1a5f4374aSIngo Molnar /* 2a5f4374aSIngo Molnar * User-space Probes (UProbes) 3a5f4374aSIngo Molnar * 4a5f4374aSIngo Molnar * This program is free software; you can redistribute it and/or modify 5a5f4374aSIngo Molnar * it under the terms of the GNU General Public License as published by 6a5f4374aSIngo Molnar * the Free Software Foundation; either version 2 of the License, or 7a5f4374aSIngo Molnar * (at your option) any later version. 8a5f4374aSIngo Molnar * 9a5f4374aSIngo Molnar * This program is distributed in the hope that it will be useful, 10a5f4374aSIngo Molnar * but WITHOUT ANY WARRANTY; without even the implied warranty of 11a5f4374aSIngo Molnar * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12a5f4374aSIngo Molnar * GNU General Public License for more details. 13a5f4374aSIngo Molnar * 14a5f4374aSIngo Molnar * You should have received a copy of the GNU General Public License 15a5f4374aSIngo Molnar * along with this program; if not, write to the Free Software 16a5f4374aSIngo Molnar * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 17a5f4374aSIngo Molnar * 1835aa621bSIngo Molnar * Copyright (C) IBM Corporation, 2008-2012 19a5f4374aSIngo Molnar * Authors: 20a5f4374aSIngo Molnar * Srikar Dronamraju 21a5f4374aSIngo Molnar * Jim Keniston 2290eec103SPeter Zijlstra * Copyright (C) 2011-2012 Red Hat, Inc., Peter Zijlstra 23a5f4374aSIngo Molnar */ 24a5f4374aSIngo Molnar 25a5f4374aSIngo Molnar #include <linux/kernel.h> 26a5f4374aSIngo Molnar #include <linux/highmem.h> 27a5f4374aSIngo Molnar #include <linux/pagemap.h> /* read_mapping_page */ 28a5f4374aSIngo Molnar #include <linux/slab.h> 29a5f4374aSIngo Molnar #include <linux/sched.h> 30e8440c14SJosh Stone #include <linux/export.h> 31a5f4374aSIngo Molnar #include <linux/rmap.h> /* anon_vma_prepare */ 32a5f4374aSIngo Molnar #include <linux/mmu_notifier.h> /* set_pte_at_notify */ 33a5f4374aSIngo Molnar #include <linux/swap.h> /* try_to_free_swap */ 340326f5a9SSrikar Dronamraju #include <linux/ptrace.h> /* user_enable_single_step */ 350326f5a9SSrikar Dronamraju #include <linux/kdebug.h> /* notifier mechanism */ 36194f8dcbSOleg Nesterov #include "../../mm/internal.h" /* munlock_vma_page */ 3732cdba1eSOleg Nesterov #include <linux/percpu-rwsem.h> 38aa59c53fSOleg Nesterov #include <linux/task_work.h> 3940814f68SOleg Nesterov #include <linux/shmem_fs.h> 40a5f4374aSIngo Molnar 41a5f4374aSIngo Molnar #include <linux/uprobes.h> 42a5f4374aSIngo Molnar 43d4b3b638SSrikar Dronamraju #define UINSNS_PER_PAGE (PAGE_SIZE/UPROBE_XOL_SLOT_BYTES) 44d4b3b638SSrikar Dronamraju #define MAX_UPROBE_XOL_SLOTS UINSNS_PER_PAGE 45d4b3b638SSrikar Dronamraju 46a5f4374aSIngo Molnar static struct rb_root uprobes_tree = RB_ROOT; 47441f1eb7SOleg Nesterov /* 48441f1eb7SOleg Nesterov * allows us to skip the uprobe_mmap if there are no uprobe events active 49441f1eb7SOleg Nesterov * at this time. Probably a fine grained per inode count is better? 50441f1eb7SOleg Nesterov */ 51441f1eb7SOleg Nesterov #define no_uprobe_events() RB_EMPTY_ROOT(&uprobes_tree) 52a5f4374aSIngo Molnar 53a5f4374aSIngo Molnar static DEFINE_SPINLOCK(uprobes_treelock); /* serialize rbtree access */ 54a5f4374aSIngo Molnar 55a5f4374aSIngo Molnar #define UPROBES_HASH_SZ 13 56a5f4374aSIngo Molnar /* serialize uprobe->pending_list */ 57a5f4374aSIngo Molnar static struct mutex uprobes_mmap_mutex[UPROBES_HASH_SZ]; 58a5f4374aSIngo Molnar #define uprobes_mmap_hash(v) (&uprobes_mmap_mutex[((unsigned long)(v)) % UPROBES_HASH_SZ]) 59a5f4374aSIngo Molnar 6032cdba1eSOleg Nesterov static struct percpu_rw_semaphore dup_mmap_sem; 6132cdba1eSOleg Nesterov 62cb9a19feSOleg Nesterov /* Have a copy of original instruction */ 6371434f2fSOleg Nesterov #define UPROBE_COPY_INSN 0 64cb9a19feSOleg Nesterov 653ff54efdSSrikar Dronamraju struct uprobe { 663ff54efdSSrikar Dronamraju struct rb_node rb_node; /* node in the rb tree */ 673ff54efdSSrikar Dronamraju atomic_t ref; 68e591c8d7SOleg Nesterov struct rw_semaphore register_rwsem; 693ff54efdSSrikar Dronamraju struct rw_semaphore consumer_rwsem; 703ff54efdSSrikar Dronamraju struct list_head pending_list; 713ff54efdSSrikar Dronamraju struct uprobe_consumer *consumers; 723ff54efdSSrikar Dronamraju struct inode *inode; /* Also hold a ref to inode */ 733ff54efdSSrikar Dronamraju loff_t offset; 7471434f2fSOleg Nesterov unsigned long flags; 75ad439356SOleg Nesterov 76ad439356SOleg Nesterov /* 77ad439356SOleg Nesterov * The generic code assumes that it has two members of unknown type 78ad439356SOleg Nesterov * owned by the arch-specific code: 79ad439356SOleg Nesterov * 80ad439356SOleg Nesterov * insn - copy_insn() saves the original instruction here for 81ad439356SOleg Nesterov * arch_uprobe_analyze_insn(). 82ad439356SOleg Nesterov * 83ad439356SOleg Nesterov * ixol - potentially modified instruction to execute out of 84ad439356SOleg Nesterov * line, copied to xol_area by xol_get_insn_slot(). 85ad439356SOleg Nesterov */ 863ff54efdSSrikar Dronamraju struct arch_uprobe arch; 873ff54efdSSrikar Dronamraju }; 883ff54efdSSrikar Dronamraju 89a5f4374aSIngo Molnar /* 90ad439356SOleg Nesterov * Execute out of line area: anonymous executable mapping installed 91ad439356SOleg Nesterov * by the probed task to execute the copy of the original instruction 92ad439356SOleg Nesterov * mangled by set_swbp(). 93ad439356SOleg Nesterov * 94c912dae6SOleg Nesterov * On a breakpoint hit, thread contests for a slot. It frees the 95c912dae6SOleg Nesterov * slot after singlestep. Currently a fixed number of slots are 96c912dae6SOleg Nesterov * allocated. 97c912dae6SOleg Nesterov */ 98c912dae6SOleg Nesterov struct xol_area { 99c912dae6SOleg Nesterov wait_queue_head_t wq; /* if all slots are busy */ 100c912dae6SOleg Nesterov atomic_t slot_count; /* number of in-use slots */ 101c912dae6SOleg Nesterov unsigned long *bitmap; /* 0 = free slot */ 102c912dae6SOleg Nesterov 103704bde3cSOleg Nesterov struct vm_special_mapping xol_mapping; 104704bde3cSOleg Nesterov struct page *pages[2]; 105c912dae6SOleg Nesterov /* 106c912dae6SOleg Nesterov * We keep the vma's vm_start rather than a pointer to the vma 107c912dae6SOleg Nesterov * itself. The probed process or a naughty kernel module could make 108c912dae6SOleg Nesterov * the vma go away, and we must handle that reasonably gracefully. 109c912dae6SOleg Nesterov */ 110c912dae6SOleg Nesterov unsigned long vaddr; /* Page(s) of instruction slots */ 111c912dae6SOleg Nesterov }; 112c912dae6SOleg Nesterov 113c912dae6SOleg Nesterov /* 114a5f4374aSIngo Molnar * valid_vma: Verify if the specified vma is an executable vma 115a5f4374aSIngo Molnar * Relax restrictions while unregistering: vm_flags might have 116a5f4374aSIngo Molnar * changed after breakpoint was inserted. 117a5f4374aSIngo Molnar * - is_register: indicates if we are in register context. 118a5f4374aSIngo Molnar * - Return 1 if the specified virtual address is in an 119a5f4374aSIngo Molnar * executable vma. 120a5f4374aSIngo Molnar */ 121a5f4374aSIngo Molnar static bool valid_vma(struct vm_area_struct *vma, bool is_register) 122a5f4374aSIngo Molnar { 12313f59c5eSOleg Nesterov vm_flags_t flags = VM_HUGETLB | VM_MAYEXEC | VM_MAYSHARE; 124a5f4374aSIngo Molnar 125e40cfce6SOleg Nesterov if (is_register) 126e40cfce6SOleg Nesterov flags |= VM_WRITE; 127a5f4374aSIngo Molnar 128e40cfce6SOleg Nesterov return vma->vm_file && (vma->vm_flags & flags) == VM_MAYEXEC; 129a5f4374aSIngo Molnar } 130a5f4374aSIngo Molnar 13157683f72SOleg Nesterov static unsigned long offset_to_vaddr(struct vm_area_struct *vma, loff_t offset) 132a5f4374aSIngo Molnar { 13357683f72SOleg Nesterov return vma->vm_start + offset - ((loff_t)vma->vm_pgoff << PAGE_SHIFT); 134a5f4374aSIngo Molnar } 135a5f4374aSIngo Molnar 136cb113b47SOleg Nesterov static loff_t vaddr_to_offset(struct vm_area_struct *vma, unsigned long vaddr) 137cb113b47SOleg Nesterov { 138cb113b47SOleg Nesterov return ((loff_t)vma->vm_pgoff << PAGE_SHIFT) + (vaddr - vma->vm_start); 139cb113b47SOleg Nesterov } 140cb113b47SOleg Nesterov 141a5f4374aSIngo Molnar /** 142a5f4374aSIngo Molnar * __replace_page - replace page in vma by new page. 143a5f4374aSIngo Molnar * based on replace_page in mm/ksm.c 144a5f4374aSIngo Molnar * 145a5f4374aSIngo Molnar * @vma: vma that holds the pte pointing to page 146c517ee74SOleg Nesterov * @addr: address the old @page is mapped at 147a5f4374aSIngo Molnar * @page: the cowed page we are replacing by kpage 148a5f4374aSIngo Molnar * @kpage: the modified page we replace page by 149a5f4374aSIngo Molnar * 150a5f4374aSIngo Molnar * Returns 0 on success, -EFAULT on failure. 151a5f4374aSIngo Molnar */ 152c517ee74SOleg Nesterov static int __replace_page(struct vm_area_struct *vma, unsigned long addr, 153bdfaa2eeSOleg Nesterov struct page *old_page, struct page *new_page) 154a5f4374aSIngo Molnar { 155a5f4374aSIngo Molnar struct mm_struct *mm = vma->vm_mm; 15614fa2daaSKirill A. Shutemov struct page_vma_mapped_walk pvmw = { 15714fa2daaSKirill A. Shutemov .page = old_page, 15814fa2daaSKirill A. Shutemov .vma = vma, 15914fa2daaSKirill A. Shutemov .address = addr, 16014fa2daaSKirill A. Shutemov }; 1619f92448cSOleg Nesterov int err; 1626bdb913fSHaggai Eran /* For mmu_notifiers */ 1636bdb913fSHaggai Eran const unsigned long mmun_start = addr; 1646bdb913fSHaggai Eran const unsigned long mmun_end = addr + PAGE_SIZE; 16500501b53SJohannes Weiner struct mem_cgroup *memcg; 16600501b53SJohannes Weiner 16714fa2daaSKirill A. Shutemov VM_BUG_ON_PAGE(PageTransHuge(old_page), old_page); 16814fa2daaSKirill A. Shutemov 169bdfaa2eeSOleg Nesterov err = mem_cgroup_try_charge(new_page, vma->vm_mm, GFP_KERNEL, &memcg, 170f627c2f5SKirill A. Shutemov false); 17100501b53SJohannes Weiner if (err) 17200501b53SJohannes Weiner return err; 173a5f4374aSIngo Molnar 174194f8dcbSOleg Nesterov /* For try_to_free_swap() and munlock_vma_page() below */ 175bdfaa2eeSOleg Nesterov lock_page(old_page); 1769f92448cSOleg Nesterov 1776bdb913fSHaggai Eran mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end); 1789f92448cSOleg Nesterov err = -EAGAIN; 17914fa2daaSKirill A. Shutemov if (!page_vma_mapped_walk(&pvmw)) { 180bdfaa2eeSOleg Nesterov mem_cgroup_cancel_charge(new_page, memcg, false); 1819f92448cSOleg Nesterov goto unlock; 1826c4687ccSOleg Nesterov } 18314fa2daaSKirill A. Shutemov VM_BUG_ON_PAGE(addr != pvmw.address, old_page); 184a5f4374aSIngo Molnar 185bdfaa2eeSOleg Nesterov get_page(new_page); 186bdfaa2eeSOleg Nesterov page_add_new_anon_rmap(new_page, vma, addr, false); 187bdfaa2eeSOleg Nesterov mem_cgroup_commit_charge(new_page, memcg, false, false); 188bdfaa2eeSOleg Nesterov lru_cache_add_active_or_unevictable(new_page, vma); 189a5f4374aSIngo Molnar 190bdfaa2eeSOleg Nesterov if (!PageAnon(old_page)) { 191bdfaa2eeSOleg Nesterov dec_mm_counter(mm, mm_counter_file(old_page)); 1927396fa81SSrikar Dronamraju inc_mm_counter(mm, MM_ANONPAGES); 1937396fa81SSrikar Dronamraju } 1947396fa81SSrikar Dronamraju 19514fa2daaSKirill A. Shutemov flush_cache_page(vma, addr, pte_pfn(*pvmw.pte)); 19614fa2daaSKirill A. Shutemov ptep_clear_flush_notify(vma, addr, pvmw.pte); 19714fa2daaSKirill A. Shutemov set_pte_at_notify(mm, addr, pvmw.pte, 19814fa2daaSKirill A. Shutemov mk_pte(new_page, vma->vm_page_prot)); 199a5f4374aSIngo Molnar 200bdfaa2eeSOleg Nesterov page_remove_rmap(old_page, false); 201bdfaa2eeSOleg Nesterov if (!page_mapped(old_page)) 202bdfaa2eeSOleg Nesterov try_to_free_swap(old_page); 20314fa2daaSKirill A. Shutemov page_vma_mapped_walk_done(&pvmw); 204a5f4374aSIngo Molnar 205194f8dcbSOleg Nesterov if (vma->vm_flags & VM_LOCKED) 206bdfaa2eeSOleg Nesterov munlock_vma_page(old_page); 207bdfaa2eeSOleg Nesterov put_page(old_page); 208194f8dcbSOleg Nesterov 2099f92448cSOleg Nesterov err = 0; 2109f92448cSOleg Nesterov unlock: 2116bdb913fSHaggai Eran mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end); 212bdfaa2eeSOleg Nesterov unlock_page(old_page); 2139f92448cSOleg Nesterov return err; 214a5f4374aSIngo Molnar } 215a5f4374aSIngo Molnar 216a5f4374aSIngo Molnar /** 2175cb4ac3aSSrikar Dronamraju * is_swbp_insn - check if instruction is breakpoint instruction. 218a5f4374aSIngo Molnar * @insn: instruction to be checked. 2195cb4ac3aSSrikar Dronamraju * Default implementation of is_swbp_insn 220a5f4374aSIngo Molnar * Returns true if @insn is a breakpoint instruction. 221a5f4374aSIngo Molnar */ 2225cb4ac3aSSrikar Dronamraju bool __weak is_swbp_insn(uprobe_opcode_t *insn) 223a5f4374aSIngo Molnar { 2245cb4ac3aSSrikar Dronamraju return *insn == UPROBE_SWBP_INSN; 225a5f4374aSIngo Molnar } 226a5f4374aSIngo Molnar 2270908ad6eSAnanth N Mavinakayanahalli /** 2280908ad6eSAnanth N Mavinakayanahalli * is_trap_insn - check if instruction is breakpoint instruction. 2290908ad6eSAnanth N Mavinakayanahalli * @insn: instruction to be checked. 2300908ad6eSAnanth N Mavinakayanahalli * Default implementation of is_trap_insn 2310908ad6eSAnanth N Mavinakayanahalli * Returns true if @insn is a breakpoint instruction. 2320908ad6eSAnanth N Mavinakayanahalli * 2330908ad6eSAnanth N Mavinakayanahalli * This function is needed for the case where an architecture has multiple 2340908ad6eSAnanth N Mavinakayanahalli * trap instructions (like powerpc). 2350908ad6eSAnanth N Mavinakayanahalli */ 2360908ad6eSAnanth N Mavinakayanahalli bool __weak is_trap_insn(uprobe_opcode_t *insn) 2370908ad6eSAnanth N Mavinakayanahalli { 2380908ad6eSAnanth N Mavinakayanahalli return is_swbp_insn(insn); 2390908ad6eSAnanth N Mavinakayanahalli } 2400908ad6eSAnanth N Mavinakayanahalli 241ab0d805cSOleg Nesterov static void copy_from_page(struct page *page, unsigned long vaddr, void *dst, int len) 242cceb55aaSOleg Nesterov { 243cceb55aaSOleg Nesterov void *kaddr = kmap_atomic(page); 244ab0d805cSOleg Nesterov memcpy(dst, kaddr + (vaddr & ~PAGE_MASK), len); 245cceb55aaSOleg Nesterov kunmap_atomic(kaddr); 246cceb55aaSOleg Nesterov } 247cceb55aaSOleg Nesterov 2485669cceeSOleg Nesterov static void copy_to_page(struct page *page, unsigned long vaddr, const void *src, int len) 2495669cceeSOleg Nesterov { 2505669cceeSOleg Nesterov void *kaddr = kmap_atomic(page); 2515669cceeSOleg Nesterov memcpy(kaddr + (vaddr & ~PAGE_MASK), src, len); 2525669cceeSOleg Nesterov kunmap_atomic(kaddr); 2535669cceeSOleg Nesterov } 2545669cceeSOleg Nesterov 255ed6f6a50SOleg Nesterov static int verify_opcode(struct page *page, unsigned long vaddr, uprobe_opcode_t *new_opcode) 256ed6f6a50SOleg Nesterov { 257ed6f6a50SOleg Nesterov uprobe_opcode_t old_opcode; 258ed6f6a50SOleg Nesterov bool is_swbp; 259ed6f6a50SOleg Nesterov 2600908ad6eSAnanth N Mavinakayanahalli /* 2610908ad6eSAnanth N Mavinakayanahalli * Note: We only check if the old_opcode is UPROBE_SWBP_INSN here. 2620908ad6eSAnanth N Mavinakayanahalli * We do not check if it is any other 'trap variant' which could 2630908ad6eSAnanth N Mavinakayanahalli * be conditional trap instruction such as the one powerpc supports. 2640908ad6eSAnanth N Mavinakayanahalli * 2650908ad6eSAnanth N Mavinakayanahalli * The logic is that we do not care if the underlying instruction 2660908ad6eSAnanth N Mavinakayanahalli * is a trap variant; uprobes always wins over any other (gdb) 2670908ad6eSAnanth N Mavinakayanahalli * breakpoint. 2680908ad6eSAnanth N Mavinakayanahalli */ 269ab0d805cSOleg Nesterov copy_from_page(page, vaddr, &old_opcode, UPROBE_SWBP_INSN_SIZE); 270ed6f6a50SOleg Nesterov is_swbp = is_swbp_insn(&old_opcode); 271ed6f6a50SOleg Nesterov 272ed6f6a50SOleg Nesterov if (is_swbp_insn(new_opcode)) { 273ed6f6a50SOleg Nesterov if (is_swbp) /* register: already installed? */ 274ed6f6a50SOleg Nesterov return 0; 275ed6f6a50SOleg Nesterov } else { 276ed6f6a50SOleg Nesterov if (!is_swbp) /* unregister: was it changed by us? */ 277076a365bSOleg Nesterov return 0; 278ed6f6a50SOleg Nesterov } 279ed6f6a50SOleg Nesterov 280ed6f6a50SOleg Nesterov return 1; 281ed6f6a50SOleg Nesterov } 282ed6f6a50SOleg Nesterov 283a5f4374aSIngo Molnar /* 284a5f4374aSIngo Molnar * NOTE: 285a5f4374aSIngo Molnar * Expect the breakpoint instruction to be the smallest size instruction for 286a5f4374aSIngo Molnar * the architecture. If an arch has variable length instruction and the 287a5f4374aSIngo Molnar * breakpoint instruction is not of the smallest length instruction 2880908ad6eSAnanth N Mavinakayanahalli * supported by that architecture then we need to modify is_trap_at_addr and 289f72d41faSOleg Nesterov * uprobe_write_opcode accordingly. This would never be a problem for archs 290f72d41faSOleg Nesterov * that have fixed length instructions. 29129dedee0SOleg Nesterov * 292f72d41faSOleg Nesterov * uprobe_write_opcode - write the opcode at a given virtual address. 293a5f4374aSIngo Molnar * @mm: the probed process address space. 294a5f4374aSIngo Molnar * @vaddr: the virtual address to store the opcode. 295a5f4374aSIngo Molnar * @opcode: opcode to be written at @vaddr. 296a5f4374aSIngo Molnar * 29729dedee0SOleg Nesterov * Called with mm->mmap_sem held for write. 298a5f4374aSIngo Molnar * Return 0 (success) or a negative errno. 299a5f4374aSIngo Molnar */ 300f72d41faSOleg Nesterov int uprobe_write_opcode(struct mm_struct *mm, unsigned long vaddr, 301cceb55aaSOleg Nesterov uprobe_opcode_t opcode) 302a5f4374aSIngo Molnar { 303a5f4374aSIngo Molnar struct page *old_page, *new_page; 304a5f4374aSIngo Molnar struct vm_area_struct *vma; 305a5f4374aSIngo Molnar int ret; 306f403072cSOleg Nesterov 3075323ce71SOleg Nesterov retry: 308a5f4374aSIngo Molnar /* Read the page with vaddr into memory */ 309c8394812SKirill A. Shutemov ret = get_user_pages_remote(NULL, mm, vaddr, 1, 310c8394812SKirill A. Shutemov FOLL_FORCE | FOLL_SPLIT, &old_page, &vma, NULL); 311a5f4374aSIngo Molnar if (ret <= 0) 312a5f4374aSIngo Molnar return ret; 313a5f4374aSIngo Molnar 314ed6f6a50SOleg Nesterov ret = verify_opcode(old_page, vaddr, &opcode); 315ed6f6a50SOleg Nesterov if (ret <= 0) 316ed6f6a50SOleg Nesterov goto put_old; 317ed6f6a50SOleg Nesterov 31829dedee0SOleg Nesterov ret = anon_vma_prepare(vma); 31929dedee0SOleg Nesterov if (ret) 32029dedee0SOleg Nesterov goto put_old; 32129dedee0SOleg Nesterov 322a5f4374aSIngo Molnar ret = -ENOMEM; 323a5f4374aSIngo Molnar new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, vaddr); 324a5f4374aSIngo Molnar if (!new_page) 3259f92448cSOleg Nesterov goto put_old; 326a5f4374aSIngo Molnar 32729dedee0SOleg Nesterov __SetPageUptodate(new_page); 3283f47107cSOleg Nesterov copy_highpage(new_page, old_page); 3293f47107cSOleg Nesterov copy_to_page(new_page, vaddr, &opcode, UPROBE_SWBP_INSN_SIZE); 330a5f4374aSIngo Molnar 331c517ee74SOleg Nesterov ret = __replace_page(vma, vaddr, old_page, new_page); 33209cbfeafSKirill A. Shutemov put_page(new_page); 3339f92448cSOleg Nesterov put_old: 334a5f4374aSIngo Molnar put_page(old_page); 335a5f4374aSIngo Molnar 3365323ce71SOleg Nesterov if (unlikely(ret == -EAGAIN)) 3375323ce71SOleg Nesterov goto retry; 338a5f4374aSIngo Molnar return ret; 339a5f4374aSIngo Molnar } 340a5f4374aSIngo Molnar 341a5f4374aSIngo Molnar /** 3425cb4ac3aSSrikar Dronamraju * set_swbp - store breakpoint at a given address. 343e3343e6aSSrikar Dronamraju * @auprobe: arch specific probepoint information. 344a5f4374aSIngo Molnar * @mm: the probed process address space. 345a5f4374aSIngo Molnar * @vaddr: the virtual address to insert the opcode. 346a5f4374aSIngo Molnar * 347a5f4374aSIngo Molnar * For mm @mm, store the breakpoint instruction at @vaddr. 348a5f4374aSIngo Molnar * Return 0 (success) or a negative errno. 349a5f4374aSIngo Molnar */ 3505cb4ac3aSSrikar Dronamraju int __weak set_swbp(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned long vaddr) 351a5f4374aSIngo Molnar { 352f72d41faSOleg Nesterov return uprobe_write_opcode(mm, vaddr, UPROBE_SWBP_INSN); 353a5f4374aSIngo Molnar } 354a5f4374aSIngo Molnar 355a5f4374aSIngo Molnar /** 356a5f4374aSIngo Molnar * set_orig_insn - Restore the original instruction. 357a5f4374aSIngo Molnar * @mm: the probed process address space. 358e3343e6aSSrikar Dronamraju * @auprobe: arch specific probepoint information. 359a5f4374aSIngo Molnar * @vaddr: the virtual address to insert the opcode. 360a5f4374aSIngo Molnar * 361a5f4374aSIngo Molnar * For mm @mm, restore the original opcode (opcode) at @vaddr. 362a5f4374aSIngo Molnar * Return 0 (success) or a negative errno. 363a5f4374aSIngo Molnar */ 364a5f4374aSIngo Molnar int __weak 365ded86e7cSOleg Nesterov set_orig_insn(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned long vaddr) 366a5f4374aSIngo Molnar { 367803200e2SOleg Nesterov return uprobe_write_opcode(mm, vaddr, *(uprobe_opcode_t *)&auprobe->insn); 368a5f4374aSIngo Molnar } 369a5f4374aSIngo Molnar 370f231722aSOleg Nesterov static struct uprobe *get_uprobe(struct uprobe *uprobe) 371f231722aSOleg Nesterov { 372f231722aSOleg Nesterov atomic_inc(&uprobe->ref); 373f231722aSOleg Nesterov return uprobe; 374f231722aSOleg Nesterov } 375f231722aSOleg Nesterov 376f231722aSOleg Nesterov static void put_uprobe(struct uprobe *uprobe) 377f231722aSOleg Nesterov { 378f231722aSOleg Nesterov if (atomic_dec_and_test(&uprobe->ref)) 379f231722aSOleg Nesterov kfree(uprobe); 380f231722aSOleg Nesterov } 381f231722aSOleg Nesterov 382a5f4374aSIngo Molnar static int match_uprobe(struct uprobe *l, struct uprobe *r) 383a5f4374aSIngo Molnar { 384a5f4374aSIngo Molnar if (l->inode < r->inode) 385a5f4374aSIngo Molnar return -1; 386a5f4374aSIngo Molnar 387a5f4374aSIngo Molnar if (l->inode > r->inode) 388a5f4374aSIngo Molnar return 1; 389a5f4374aSIngo Molnar 390a5f4374aSIngo Molnar if (l->offset < r->offset) 391a5f4374aSIngo Molnar return -1; 392a5f4374aSIngo Molnar 393a5f4374aSIngo Molnar if (l->offset > r->offset) 394a5f4374aSIngo Molnar return 1; 395a5f4374aSIngo Molnar 396a5f4374aSIngo Molnar return 0; 397a5f4374aSIngo Molnar } 398a5f4374aSIngo Molnar 399a5f4374aSIngo Molnar static struct uprobe *__find_uprobe(struct inode *inode, loff_t offset) 400a5f4374aSIngo Molnar { 401a5f4374aSIngo Molnar struct uprobe u = { .inode = inode, .offset = offset }; 402a5f4374aSIngo Molnar struct rb_node *n = uprobes_tree.rb_node; 403a5f4374aSIngo Molnar struct uprobe *uprobe; 404a5f4374aSIngo Molnar int match; 405a5f4374aSIngo Molnar 406a5f4374aSIngo Molnar while (n) { 407a5f4374aSIngo Molnar uprobe = rb_entry(n, struct uprobe, rb_node); 408a5f4374aSIngo Molnar match = match_uprobe(&u, uprobe); 409f231722aSOleg Nesterov if (!match) 410f231722aSOleg Nesterov return get_uprobe(uprobe); 411a5f4374aSIngo Molnar 412a5f4374aSIngo Molnar if (match < 0) 413a5f4374aSIngo Molnar n = n->rb_left; 414a5f4374aSIngo Molnar else 415a5f4374aSIngo Molnar n = n->rb_right; 416a5f4374aSIngo Molnar } 417a5f4374aSIngo Molnar return NULL; 418a5f4374aSIngo Molnar } 419a5f4374aSIngo Molnar 420a5f4374aSIngo Molnar /* 421a5f4374aSIngo Molnar * Find a uprobe corresponding to a given inode:offset 422a5f4374aSIngo Molnar * Acquires uprobes_treelock 423a5f4374aSIngo Molnar */ 424a5f4374aSIngo Molnar static struct uprobe *find_uprobe(struct inode *inode, loff_t offset) 425a5f4374aSIngo Molnar { 426a5f4374aSIngo Molnar struct uprobe *uprobe; 427a5f4374aSIngo Molnar 4286f47caa0SOleg Nesterov spin_lock(&uprobes_treelock); 429a5f4374aSIngo Molnar uprobe = __find_uprobe(inode, offset); 4306f47caa0SOleg Nesterov spin_unlock(&uprobes_treelock); 431a5f4374aSIngo Molnar 432a5f4374aSIngo Molnar return uprobe; 433a5f4374aSIngo Molnar } 434a5f4374aSIngo Molnar 435a5f4374aSIngo Molnar static struct uprobe *__insert_uprobe(struct uprobe *uprobe) 436a5f4374aSIngo Molnar { 437a5f4374aSIngo Molnar struct rb_node **p = &uprobes_tree.rb_node; 438a5f4374aSIngo Molnar struct rb_node *parent = NULL; 439a5f4374aSIngo Molnar struct uprobe *u; 440a5f4374aSIngo Molnar int match; 441a5f4374aSIngo Molnar 442a5f4374aSIngo Molnar while (*p) { 443a5f4374aSIngo Molnar parent = *p; 444a5f4374aSIngo Molnar u = rb_entry(parent, struct uprobe, rb_node); 445a5f4374aSIngo Molnar match = match_uprobe(uprobe, u); 446f231722aSOleg Nesterov if (!match) 447f231722aSOleg Nesterov return get_uprobe(u); 448a5f4374aSIngo Molnar 449a5f4374aSIngo Molnar if (match < 0) 450a5f4374aSIngo Molnar p = &parent->rb_left; 451a5f4374aSIngo Molnar else 452a5f4374aSIngo Molnar p = &parent->rb_right; 453a5f4374aSIngo Molnar 454a5f4374aSIngo Molnar } 455a5f4374aSIngo Molnar 456a5f4374aSIngo Molnar u = NULL; 457a5f4374aSIngo Molnar rb_link_node(&uprobe->rb_node, parent, p); 458a5f4374aSIngo Molnar rb_insert_color(&uprobe->rb_node, &uprobes_tree); 459a5f4374aSIngo Molnar /* get access + creation ref */ 460a5f4374aSIngo Molnar atomic_set(&uprobe->ref, 2); 461a5f4374aSIngo Molnar 462a5f4374aSIngo Molnar return u; 463a5f4374aSIngo Molnar } 464a5f4374aSIngo Molnar 465a5f4374aSIngo Molnar /* 466a5f4374aSIngo Molnar * Acquire uprobes_treelock. 467a5f4374aSIngo Molnar * Matching uprobe already exists in rbtree; 468a5f4374aSIngo Molnar * increment (access refcount) and return the matching uprobe. 469a5f4374aSIngo Molnar * 470a5f4374aSIngo Molnar * No matching uprobe; insert the uprobe in rb_tree; 471a5f4374aSIngo Molnar * get a double refcount (access + creation) and return NULL. 472a5f4374aSIngo Molnar */ 473a5f4374aSIngo Molnar static struct uprobe *insert_uprobe(struct uprobe *uprobe) 474a5f4374aSIngo Molnar { 475a5f4374aSIngo Molnar struct uprobe *u; 476a5f4374aSIngo Molnar 4776f47caa0SOleg Nesterov spin_lock(&uprobes_treelock); 478a5f4374aSIngo Molnar u = __insert_uprobe(uprobe); 4796f47caa0SOleg Nesterov spin_unlock(&uprobes_treelock); 480a5f4374aSIngo Molnar 481a5f4374aSIngo Molnar return u; 482a5f4374aSIngo Molnar } 483a5f4374aSIngo Molnar 484a5f4374aSIngo Molnar static struct uprobe *alloc_uprobe(struct inode *inode, loff_t offset) 485a5f4374aSIngo Molnar { 486a5f4374aSIngo Molnar struct uprobe *uprobe, *cur_uprobe; 487a5f4374aSIngo Molnar 488a5f4374aSIngo Molnar uprobe = kzalloc(sizeof(struct uprobe), GFP_KERNEL); 489a5f4374aSIngo Molnar if (!uprobe) 490a5f4374aSIngo Molnar return NULL; 491a5f4374aSIngo Molnar 492a5f4374aSIngo Molnar uprobe->inode = igrab(inode); 493a5f4374aSIngo Molnar uprobe->offset = offset; 494e591c8d7SOleg Nesterov init_rwsem(&uprobe->register_rwsem); 495a5f4374aSIngo Molnar init_rwsem(&uprobe->consumer_rwsem); 496a5f4374aSIngo Molnar 497a5f4374aSIngo Molnar /* add to uprobes_tree, sorted on inode:offset */ 498a5f4374aSIngo Molnar cur_uprobe = insert_uprobe(uprobe); 499a5f4374aSIngo Molnar /* a uprobe exists for this inode:offset combination */ 500a5f4374aSIngo Molnar if (cur_uprobe) { 501a5f4374aSIngo Molnar kfree(uprobe); 502a5f4374aSIngo Molnar uprobe = cur_uprobe; 503a5f4374aSIngo Molnar iput(inode); 504a5f4374aSIngo Molnar } 505a5f4374aSIngo Molnar 506a5f4374aSIngo Molnar return uprobe; 507a5f4374aSIngo Molnar } 508a5f4374aSIngo Molnar 5099a98e03cSOleg Nesterov static void consumer_add(struct uprobe *uprobe, struct uprobe_consumer *uc) 510a5f4374aSIngo Molnar { 511a5f4374aSIngo Molnar down_write(&uprobe->consumer_rwsem); 512e3343e6aSSrikar Dronamraju uc->next = uprobe->consumers; 513e3343e6aSSrikar Dronamraju uprobe->consumers = uc; 514a5f4374aSIngo Molnar up_write(&uprobe->consumer_rwsem); 515a5f4374aSIngo Molnar } 516a5f4374aSIngo Molnar 517a5f4374aSIngo Molnar /* 518e3343e6aSSrikar Dronamraju * For uprobe @uprobe, delete the consumer @uc. 519e3343e6aSSrikar Dronamraju * Return true if the @uc is deleted successfully 520a5f4374aSIngo Molnar * or return false. 521a5f4374aSIngo Molnar */ 522e3343e6aSSrikar Dronamraju static bool consumer_del(struct uprobe *uprobe, struct uprobe_consumer *uc) 523a5f4374aSIngo Molnar { 524a5f4374aSIngo Molnar struct uprobe_consumer **con; 525a5f4374aSIngo Molnar bool ret = false; 526a5f4374aSIngo Molnar 527a5f4374aSIngo Molnar down_write(&uprobe->consumer_rwsem); 528a5f4374aSIngo Molnar for (con = &uprobe->consumers; *con; con = &(*con)->next) { 529e3343e6aSSrikar Dronamraju if (*con == uc) { 530e3343e6aSSrikar Dronamraju *con = uc->next; 531a5f4374aSIngo Molnar ret = true; 532a5f4374aSIngo Molnar break; 533a5f4374aSIngo Molnar } 534a5f4374aSIngo Molnar } 535a5f4374aSIngo Molnar up_write(&uprobe->consumer_rwsem); 536a5f4374aSIngo Molnar 537a5f4374aSIngo Molnar return ret; 538a5f4374aSIngo Molnar } 539a5f4374aSIngo Molnar 5402ded0980SOleg Nesterov static int __copy_insn(struct address_space *mapping, struct file *filp, 5412ded0980SOleg Nesterov void *insn, int nbytes, loff_t offset) 542a5f4374aSIngo Molnar { 543a5f4374aSIngo Molnar struct page *page; 544a5f4374aSIngo Molnar /* 54540814f68SOleg Nesterov * Ensure that the page that has the original instruction is populated 54640814f68SOleg Nesterov * and in page-cache. If ->readpage == NULL it must be shmem_mapping(), 54740814f68SOleg Nesterov * see uprobe_register(). 548a5f4374aSIngo Molnar */ 54940814f68SOleg Nesterov if (mapping->a_ops->readpage) 55009cbfeafSKirill A. Shutemov page = read_mapping_page(mapping, offset >> PAGE_SHIFT, filp); 55140814f68SOleg Nesterov else 55209cbfeafSKirill A. Shutemov page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT); 553a5f4374aSIngo Molnar if (IS_ERR(page)) 554a5f4374aSIngo Molnar return PTR_ERR(page); 555a5f4374aSIngo Molnar 5562edb7b55SOleg Nesterov copy_from_page(page, offset, insn, nbytes); 55709cbfeafSKirill A. Shutemov put_page(page); 558a5f4374aSIngo Molnar 559a5f4374aSIngo Molnar return 0; 560a5f4374aSIngo Molnar } 561a5f4374aSIngo Molnar 562d436615eSOleg Nesterov static int copy_insn(struct uprobe *uprobe, struct file *filp) 563a5f4374aSIngo Molnar { 5642ded0980SOleg Nesterov struct address_space *mapping = uprobe->inode->i_mapping; 5652ded0980SOleg Nesterov loff_t offs = uprobe->offset; 566803200e2SOleg Nesterov void *insn = &uprobe->arch.insn; 567803200e2SOleg Nesterov int size = sizeof(uprobe->arch.insn); 5682ded0980SOleg Nesterov int len, err = -EIO; 569a5f4374aSIngo Molnar 5702ded0980SOleg Nesterov /* Copy only available bytes, -EIO if nothing was read */ 5712ded0980SOleg Nesterov do { 5722ded0980SOleg Nesterov if (offs >= i_size_read(uprobe->inode)) 5732ded0980SOleg Nesterov break; 574a5f4374aSIngo Molnar 5752ded0980SOleg Nesterov len = min_t(int, size, PAGE_SIZE - (offs & ~PAGE_MASK)); 5762ded0980SOleg Nesterov err = __copy_insn(mapping, filp, insn, len, offs); 577fc36f595SOleg Nesterov if (err) 5782ded0980SOleg Nesterov break; 5792ded0980SOleg Nesterov 5802ded0980SOleg Nesterov insn += len; 5812ded0980SOleg Nesterov offs += len; 5822ded0980SOleg Nesterov size -= len; 5832ded0980SOleg Nesterov } while (size); 5842ded0980SOleg Nesterov 585fc36f595SOleg Nesterov return err; 586a5f4374aSIngo Molnar } 587a5f4374aSIngo Molnar 588cb9a19feSOleg Nesterov static int prepare_uprobe(struct uprobe *uprobe, struct file *file, 589cb9a19feSOleg Nesterov struct mm_struct *mm, unsigned long vaddr) 590cb9a19feSOleg Nesterov { 591cb9a19feSOleg Nesterov int ret = 0; 592cb9a19feSOleg Nesterov 59371434f2fSOleg Nesterov if (test_bit(UPROBE_COPY_INSN, &uprobe->flags)) 594cb9a19feSOleg Nesterov return ret; 595cb9a19feSOleg Nesterov 596d4d3ccc6SOleg Nesterov /* TODO: move this into _register, until then we abuse this sem. */ 597d4d3ccc6SOleg Nesterov down_write(&uprobe->consumer_rwsem); 59871434f2fSOleg Nesterov if (test_bit(UPROBE_COPY_INSN, &uprobe->flags)) 5994710f05fSOleg Nesterov goto out; 6004710f05fSOleg Nesterov 601cb9a19feSOleg Nesterov ret = copy_insn(uprobe, file); 602cb9a19feSOleg Nesterov if (ret) 603cb9a19feSOleg Nesterov goto out; 604cb9a19feSOleg Nesterov 605cb9a19feSOleg Nesterov ret = -ENOTSUPP; 606803200e2SOleg Nesterov if (is_trap_insn((uprobe_opcode_t *)&uprobe->arch.insn)) 607cb9a19feSOleg Nesterov goto out; 608cb9a19feSOleg Nesterov 609cb9a19feSOleg Nesterov ret = arch_uprobe_analyze_insn(&uprobe->arch, mm, vaddr); 610cb9a19feSOleg Nesterov if (ret) 611cb9a19feSOleg Nesterov goto out; 612cb9a19feSOleg Nesterov 613f72d41faSOleg Nesterov /* uprobe_write_opcode() assumes we don't cross page boundary */ 614cb9a19feSOleg Nesterov BUG_ON((uprobe->offset & ~PAGE_MASK) + 615cb9a19feSOleg Nesterov UPROBE_SWBP_INSN_SIZE > PAGE_SIZE); 616cb9a19feSOleg Nesterov 617cb9a19feSOleg Nesterov smp_wmb(); /* pairs with rmb() in find_active_uprobe() */ 61871434f2fSOleg Nesterov set_bit(UPROBE_COPY_INSN, &uprobe->flags); 619cb9a19feSOleg Nesterov 620cb9a19feSOleg Nesterov out: 621d4d3ccc6SOleg Nesterov up_write(&uprobe->consumer_rwsem); 6224710f05fSOleg Nesterov 623cb9a19feSOleg Nesterov return ret; 624cb9a19feSOleg Nesterov } 625cb9a19feSOleg Nesterov 6268a7f2fa0SOleg Nesterov static inline bool consumer_filter(struct uprobe_consumer *uc, 6278a7f2fa0SOleg Nesterov enum uprobe_filter_ctx ctx, struct mm_struct *mm) 628806a98bdSOleg Nesterov { 6298a7f2fa0SOleg Nesterov return !uc->filter || uc->filter(uc, ctx, mm); 630806a98bdSOleg Nesterov } 631806a98bdSOleg Nesterov 6328a7f2fa0SOleg Nesterov static bool filter_chain(struct uprobe *uprobe, 6338a7f2fa0SOleg Nesterov enum uprobe_filter_ctx ctx, struct mm_struct *mm) 63463633cbfSOleg Nesterov { 6351ff6fee5SOleg Nesterov struct uprobe_consumer *uc; 6361ff6fee5SOleg Nesterov bool ret = false; 6371ff6fee5SOleg Nesterov 6381ff6fee5SOleg Nesterov down_read(&uprobe->consumer_rwsem); 6391ff6fee5SOleg Nesterov for (uc = uprobe->consumers; uc; uc = uc->next) { 6408a7f2fa0SOleg Nesterov ret = consumer_filter(uc, ctx, mm); 6411ff6fee5SOleg Nesterov if (ret) 6421ff6fee5SOleg Nesterov break; 6431ff6fee5SOleg Nesterov } 6441ff6fee5SOleg Nesterov up_read(&uprobe->consumer_rwsem); 6451ff6fee5SOleg Nesterov 6461ff6fee5SOleg Nesterov return ret; 64763633cbfSOleg Nesterov } 64863633cbfSOleg Nesterov 649e3343e6aSSrikar Dronamraju static int 650e3343e6aSSrikar Dronamraju install_breakpoint(struct uprobe *uprobe, struct mm_struct *mm, 651816c03fbSOleg Nesterov struct vm_area_struct *vma, unsigned long vaddr) 652a5f4374aSIngo Molnar { 653f8ac4ec9SOleg Nesterov bool first_uprobe; 654a5f4374aSIngo Molnar int ret; 655a5f4374aSIngo Molnar 656cb9a19feSOleg Nesterov ret = prepare_uprobe(uprobe, vma->vm_file, mm, vaddr); 657a5f4374aSIngo Molnar if (ret) 658a5f4374aSIngo Molnar return ret; 659a5f4374aSIngo Molnar 660f8ac4ec9SOleg Nesterov /* 661f8ac4ec9SOleg Nesterov * set MMF_HAS_UPROBES in advance for uprobe_pre_sstep_notifier(), 662f8ac4ec9SOleg Nesterov * the task can hit this breakpoint right after __replace_page(). 663f8ac4ec9SOleg Nesterov */ 664f8ac4ec9SOleg Nesterov first_uprobe = !test_bit(MMF_HAS_UPROBES, &mm->flags); 665f8ac4ec9SOleg Nesterov if (first_uprobe) 666f8ac4ec9SOleg Nesterov set_bit(MMF_HAS_UPROBES, &mm->flags); 667f8ac4ec9SOleg Nesterov 668816c03fbSOleg Nesterov ret = set_swbp(&uprobe->arch, mm, vaddr); 6699f68f672SOleg Nesterov if (!ret) 6709f68f672SOleg Nesterov clear_bit(MMF_RECALC_UPROBES, &mm->flags); 6719f68f672SOleg Nesterov else if (first_uprobe) 672f8ac4ec9SOleg Nesterov clear_bit(MMF_HAS_UPROBES, &mm->flags); 673a5f4374aSIngo Molnar 674a5f4374aSIngo Molnar return ret; 675a5f4374aSIngo Molnar } 676a5f4374aSIngo Molnar 677076a365bSOleg Nesterov static int 678816c03fbSOleg Nesterov remove_breakpoint(struct uprobe *uprobe, struct mm_struct *mm, unsigned long vaddr) 679a5f4374aSIngo Molnar { 6809f68f672SOleg Nesterov set_bit(MMF_RECALC_UPROBES, &mm->flags); 681076a365bSOleg Nesterov return set_orig_insn(&uprobe->arch, mm, vaddr); 682a5f4374aSIngo Molnar } 683a5f4374aSIngo Molnar 68406b7bcd8SOleg Nesterov static inline bool uprobe_is_active(struct uprobe *uprobe) 68506b7bcd8SOleg Nesterov { 68606b7bcd8SOleg Nesterov return !RB_EMPTY_NODE(&uprobe->rb_node); 68706b7bcd8SOleg Nesterov } 6880326f5a9SSrikar Dronamraju /* 689778b032dSOleg Nesterov * There could be threads that have already hit the breakpoint. They 690778b032dSOleg Nesterov * will recheck the current insn and restart if find_uprobe() fails. 691778b032dSOleg Nesterov * See find_active_uprobe(). 6920326f5a9SSrikar Dronamraju */ 693a5f4374aSIngo Molnar static void delete_uprobe(struct uprobe *uprobe) 694a5f4374aSIngo Molnar { 69506b7bcd8SOleg Nesterov if (WARN_ON(!uprobe_is_active(uprobe))) 69606b7bcd8SOleg Nesterov return; 69706b7bcd8SOleg Nesterov 6986f47caa0SOleg Nesterov spin_lock(&uprobes_treelock); 699a5f4374aSIngo Molnar rb_erase(&uprobe->rb_node, &uprobes_tree); 7006f47caa0SOleg Nesterov spin_unlock(&uprobes_treelock); 70106b7bcd8SOleg Nesterov RB_CLEAR_NODE(&uprobe->rb_node); /* for uprobe_is_active() */ 702a5f4374aSIngo Molnar iput(uprobe->inode); 703a5f4374aSIngo Molnar put_uprobe(uprobe); 704a5f4374aSIngo Molnar } 705a5f4374aSIngo Molnar 70626872090SOleg Nesterov struct map_info { 70726872090SOleg Nesterov struct map_info *next; 70826872090SOleg Nesterov struct mm_struct *mm; 709816c03fbSOleg Nesterov unsigned long vaddr; 71026872090SOleg Nesterov }; 71126872090SOleg Nesterov 71226872090SOleg Nesterov static inline struct map_info *free_map_info(struct map_info *info) 713a5f4374aSIngo Molnar { 71426872090SOleg Nesterov struct map_info *next = info->next; 71526872090SOleg Nesterov kfree(info); 71626872090SOleg Nesterov return next; 71726872090SOleg Nesterov } 71826872090SOleg Nesterov 71926872090SOleg Nesterov static struct map_info * 72026872090SOleg Nesterov build_map_info(struct address_space *mapping, loff_t offset, bool is_register) 72126872090SOleg Nesterov { 72226872090SOleg Nesterov unsigned long pgoff = offset >> PAGE_SHIFT; 723a5f4374aSIngo Molnar struct vm_area_struct *vma; 72426872090SOleg Nesterov struct map_info *curr = NULL; 72526872090SOleg Nesterov struct map_info *prev = NULL; 72626872090SOleg Nesterov struct map_info *info; 72726872090SOleg Nesterov int more = 0; 728a5f4374aSIngo Molnar 72926872090SOleg Nesterov again: 7304a23717aSDavidlohr Bueso i_mmap_lock_read(mapping); 7316b2dbba8SMichel Lespinasse vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) { 732a5f4374aSIngo Molnar if (!valid_vma(vma, is_register)) 733a5f4374aSIngo Molnar continue; 734a5f4374aSIngo Molnar 7357a5bfb66SOleg Nesterov if (!prev && !more) { 7367a5bfb66SOleg Nesterov /* 737c8c06efaSDavidlohr Bueso * Needs GFP_NOWAIT to avoid i_mmap_rwsem recursion through 7387a5bfb66SOleg Nesterov * reclaim. This is optimistic, no harm done if it fails. 7397a5bfb66SOleg Nesterov */ 7407a5bfb66SOleg Nesterov prev = kmalloc(sizeof(struct map_info), 7417a5bfb66SOleg Nesterov GFP_NOWAIT | __GFP_NOMEMALLOC | __GFP_NOWARN); 7427a5bfb66SOleg Nesterov if (prev) 7437a5bfb66SOleg Nesterov prev->next = NULL; 7447a5bfb66SOleg Nesterov } 74526872090SOleg Nesterov if (!prev) { 74626872090SOleg Nesterov more++; 74726872090SOleg Nesterov continue; 748a5f4374aSIngo Molnar } 749a5f4374aSIngo Molnar 75026872090SOleg Nesterov if (!atomic_inc_not_zero(&vma->vm_mm->mm_users)) 75126872090SOleg Nesterov continue; 752a5f4374aSIngo Molnar 75326872090SOleg Nesterov info = prev; 75426872090SOleg Nesterov prev = prev->next; 75526872090SOleg Nesterov info->next = curr; 75626872090SOleg Nesterov curr = info; 75726872090SOleg Nesterov 75826872090SOleg Nesterov info->mm = vma->vm_mm; 75957683f72SOleg Nesterov info->vaddr = offset_to_vaddr(vma, offset); 760a5f4374aSIngo Molnar } 7614a23717aSDavidlohr Bueso i_mmap_unlock_read(mapping); 762a5f4374aSIngo Molnar 76326872090SOleg Nesterov if (!more) 76426872090SOleg Nesterov goto out; 765a5f4374aSIngo Molnar 76626872090SOleg Nesterov prev = curr; 76726872090SOleg Nesterov while (curr) { 76826872090SOleg Nesterov mmput(curr->mm); 76926872090SOleg Nesterov curr = curr->next; 77026872090SOleg Nesterov } 77126872090SOleg Nesterov 77226872090SOleg Nesterov do { 77326872090SOleg Nesterov info = kmalloc(sizeof(struct map_info), GFP_KERNEL); 77426872090SOleg Nesterov if (!info) { 77526872090SOleg Nesterov curr = ERR_PTR(-ENOMEM); 77626872090SOleg Nesterov goto out; 77726872090SOleg Nesterov } 77826872090SOleg Nesterov info->next = prev; 77926872090SOleg Nesterov prev = info; 78026872090SOleg Nesterov } while (--more); 78126872090SOleg Nesterov 78226872090SOleg Nesterov goto again; 78326872090SOleg Nesterov out: 78426872090SOleg Nesterov while (prev) 78526872090SOleg Nesterov prev = free_map_info(prev); 78626872090SOleg Nesterov return curr; 787a5f4374aSIngo Molnar } 788a5f4374aSIngo Molnar 789bdf8647cSOleg Nesterov static int 790bdf8647cSOleg Nesterov register_for_each_vma(struct uprobe *uprobe, struct uprobe_consumer *new) 791a5f4374aSIngo Molnar { 792bdf8647cSOleg Nesterov bool is_register = !!new; 79326872090SOleg Nesterov struct map_info *info; 79426872090SOleg Nesterov int err = 0; 79526872090SOleg Nesterov 79632cdba1eSOleg Nesterov percpu_down_write(&dup_mmap_sem); 79726872090SOleg Nesterov info = build_map_info(uprobe->inode->i_mapping, 79826872090SOleg Nesterov uprobe->offset, is_register); 79932cdba1eSOleg Nesterov if (IS_ERR(info)) { 80032cdba1eSOleg Nesterov err = PTR_ERR(info); 80132cdba1eSOleg Nesterov goto out; 80232cdba1eSOleg Nesterov } 80326872090SOleg Nesterov 80426872090SOleg Nesterov while (info) { 80526872090SOleg Nesterov struct mm_struct *mm = info->mm; 806a5f4374aSIngo Molnar struct vm_area_struct *vma; 807a5f4374aSIngo Molnar 808076a365bSOleg Nesterov if (err && is_register) 80926872090SOleg Nesterov goto free; 810a5f4374aSIngo Molnar 81177fc4af1SOleg Nesterov down_write(&mm->mmap_sem); 812f4d6dfe5SOleg Nesterov vma = find_vma(mm, info->vaddr); 813f4d6dfe5SOleg Nesterov if (!vma || !valid_vma(vma, is_register) || 814f281769eSOleg Nesterov file_inode(vma->vm_file) != uprobe->inode) 81526872090SOleg Nesterov goto unlock; 81626872090SOleg Nesterov 817f4d6dfe5SOleg Nesterov if (vma->vm_start > info->vaddr || 818f4d6dfe5SOleg Nesterov vaddr_to_offset(vma, info->vaddr) != uprobe->offset) 81926872090SOleg Nesterov goto unlock; 820a5f4374aSIngo Molnar 821806a98bdSOleg Nesterov if (is_register) { 822806a98bdSOleg Nesterov /* consult only the "caller", new consumer. */ 823bdf8647cSOleg Nesterov if (consumer_filter(new, 8248a7f2fa0SOleg Nesterov UPROBE_FILTER_REGISTER, mm)) 82526872090SOleg Nesterov err = install_breakpoint(uprobe, mm, vma, info->vaddr); 826806a98bdSOleg Nesterov } else if (test_bit(MMF_HAS_UPROBES, &mm->flags)) { 8278a7f2fa0SOleg Nesterov if (!filter_chain(uprobe, 8288a7f2fa0SOleg Nesterov UPROBE_FILTER_UNREGISTER, mm)) 829076a365bSOleg Nesterov err |= remove_breakpoint(uprobe, mm, info->vaddr); 830806a98bdSOleg Nesterov } 83178f74116SOleg Nesterov 83226872090SOleg Nesterov unlock: 83326872090SOleg Nesterov up_write(&mm->mmap_sem); 83426872090SOleg Nesterov free: 83526872090SOleg Nesterov mmput(mm); 83626872090SOleg Nesterov info = free_map_info(info); 837a5f4374aSIngo Molnar } 83832cdba1eSOleg Nesterov out: 83932cdba1eSOleg Nesterov percpu_up_write(&dup_mmap_sem); 84026872090SOleg Nesterov return err; 841a5f4374aSIngo Molnar } 842a5f4374aSIngo Molnar 8439a98e03cSOleg Nesterov static int __uprobe_register(struct uprobe *uprobe, struct uprobe_consumer *uc) 844a5f4374aSIngo Molnar { 8459a98e03cSOleg Nesterov consumer_add(uprobe, uc); 846bdf8647cSOleg Nesterov return register_for_each_vma(uprobe, uc); 847a5f4374aSIngo Molnar } 848a5f4374aSIngo Molnar 84904aab9b2SOleg Nesterov static void __uprobe_unregister(struct uprobe *uprobe, struct uprobe_consumer *uc) 850a5f4374aSIngo Molnar { 85104aab9b2SOleg Nesterov int err; 852a5f4374aSIngo Molnar 85306d07139SOleg Nesterov if (WARN_ON(!consumer_del(uprobe, uc))) 85404aab9b2SOleg Nesterov return; 85504aab9b2SOleg Nesterov 856bdf8647cSOleg Nesterov err = register_for_each_vma(uprobe, NULL); 857a5f4374aSIngo Molnar /* TODO : cant unregister? schedule a worker thread */ 858bb929284SOleg Nesterov if (!uprobe->consumers && !err) 85904aab9b2SOleg Nesterov delete_uprobe(uprobe); 86004aab9b2SOleg Nesterov } 861a5f4374aSIngo Molnar 862a5f4374aSIngo Molnar /* 863a5f4374aSIngo Molnar * uprobe_register - register a probe 864a5f4374aSIngo Molnar * @inode: the file in which the probe has to be placed. 865a5f4374aSIngo Molnar * @offset: offset from the start of the file. 866e3343e6aSSrikar Dronamraju * @uc: information on howto handle the probe.. 867a5f4374aSIngo Molnar * 868a5f4374aSIngo Molnar * Apart from the access refcount, uprobe_register() takes a creation 869a5f4374aSIngo Molnar * refcount (thro alloc_uprobe) if and only if this @uprobe is getting 870a5f4374aSIngo Molnar * inserted into the rbtree (i.e first consumer for a @inode:@offset 871a5f4374aSIngo Molnar * tuple). Creation refcount stops uprobe_unregister from freeing the 872a5f4374aSIngo Molnar * @uprobe even before the register operation is complete. Creation 873e3343e6aSSrikar Dronamraju * refcount is released when the last @uc for the @uprobe 874a5f4374aSIngo Molnar * unregisters. 875a5f4374aSIngo Molnar * 876a5f4374aSIngo Molnar * Return errno if it cannot successully install probes 877a5f4374aSIngo Molnar * else return 0 (success) 878a5f4374aSIngo Molnar */ 879e3343e6aSSrikar Dronamraju int uprobe_register(struct inode *inode, loff_t offset, struct uprobe_consumer *uc) 880a5f4374aSIngo Molnar { 881a5f4374aSIngo Molnar struct uprobe *uprobe; 882a5f4374aSIngo Molnar int ret; 883a5f4374aSIngo Molnar 884ea024870SAnton Arapov /* Uprobe must have at least one set consumer */ 885ea024870SAnton Arapov if (!uc->handler && !uc->ret_handler) 886ea024870SAnton Arapov return -EINVAL; 887ea024870SAnton Arapov 88840814f68SOleg Nesterov /* copy_insn() uses read_mapping_page() or shmem_read_mapping_page() */ 88940814f68SOleg Nesterov if (!inode->i_mapping->a_ops->readpage && !shmem_mapping(inode->i_mapping)) 89041ccba02SOleg Nesterov return -EIO; 891f0744af7SOleg Nesterov /* Racy, just to catch the obvious mistakes */ 892a5f4374aSIngo Molnar if (offset > i_size_read(inode)) 893a5f4374aSIngo Molnar return -EINVAL; 894a5f4374aSIngo Molnar 89566d06dffSOleg Nesterov retry: 896a5f4374aSIngo Molnar uprobe = alloc_uprobe(inode, offset); 89766d06dffSOleg Nesterov if (!uprobe) 89866d06dffSOleg Nesterov return -ENOMEM; 89966d06dffSOleg Nesterov /* 90066d06dffSOleg Nesterov * We can race with uprobe_unregister()->delete_uprobe(). 90166d06dffSOleg Nesterov * Check uprobe_is_active() and retry if it is false. 90266d06dffSOleg Nesterov */ 903e591c8d7SOleg Nesterov down_write(&uprobe->register_rwsem); 90466d06dffSOleg Nesterov ret = -EAGAIN; 90566d06dffSOleg Nesterov if (likely(uprobe_is_active(uprobe))) { 9069a98e03cSOleg Nesterov ret = __uprobe_register(uprobe, uc); 9079a98e03cSOleg Nesterov if (ret) 90804aab9b2SOleg Nesterov __uprobe_unregister(uprobe, uc); 909a5f4374aSIngo Molnar } 91066d06dffSOleg Nesterov up_write(&uprobe->register_rwsem); 911a5f4374aSIngo Molnar put_uprobe(uprobe); 912a5f4374aSIngo Molnar 91366d06dffSOleg Nesterov if (unlikely(ret == -EAGAIN)) 91466d06dffSOleg Nesterov goto retry; 915a5f4374aSIngo Molnar return ret; 916a5f4374aSIngo Molnar } 917e8440c14SJosh Stone EXPORT_SYMBOL_GPL(uprobe_register); 918a5f4374aSIngo Molnar 919a5f4374aSIngo Molnar /* 920bdf8647cSOleg Nesterov * uprobe_apply - unregister a already registered probe. 921bdf8647cSOleg Nesterov * @inode: the file in which the probe has to be removed. 922bdf8647cSOleg Nesterov * @offset: offset from the start of the file. 923bdf8647cSOleg Nesterov * @uc: consumer which wants to add more or remove some breakpoints 924bdf8647cSOleg Nesterov * @add: add or remove the breakpoints 925bdf8647cSOleg Nesterov */ 926bdf8647cSOleg Nesterov int uprobe_apply(struct inode *inode, loff_t offset, 927bdf8647cSOleg Nesterov struct uprobe_consumer *uc, bool add) 928bdf8647cSOleg Nesterov { 929bdf8647cSOleg Nesterov struct uprobe *uprobe; 930bdf8647cSOleg Nesterov struct uprobe_consumer *con; 931bdf8647cSOleg Nesterov int ret = -ENOENT; 932bdf8647cSOleg Nesterov 933bdf8647cSOleg Nesterov uprobe = find_uprobe(inode, offset); 93406d07139SOleg Nesterov if (WARN_ON(!uprobe)) 935bdf8647cSOleg Nesterov return ret; 936bdf8647cSOleg Nesterov 937bdf8647cSOleg Nesterov down_write(&uprobe->register_rwsem); 938bdf8647cSOleg Nesterov for (con = uprobe->consumers; con && con != uc ; con = con->next) 939bdf8647cSOleg Nesterov ; 940bdf8647cSOleg Nesterov if (con) 941bdf8647cSOleg Nesterov ret = register_for_each_vma(uprobe, add ? uc : NULL); 942bdf8647cSOleg Nesterov up_write(&uprobe->register_rwsem); 943bdf8647cSOleg Nesterov put_uprobe(uprobe); 944bdf8647cSOleg Nesterov 945bdf8647cSOleg Nesterov return ret; 946bdf8647cSOleg Nesterov } 947bdf8647cSOleg Nesterov 948bdf8647cSOleg Nesterov /* 949a5f4374aSIngo Molnar * uprobe_unregister - unregister a already registered probe. 950a5f4374aSIngo Molnar * @inode: the file in which the probe has to be removed. 951a5f4374aSIngo Molnar * @offset: offset from the start of the file. 952e3343e6aSSrikar Dronamraju * @uc: identify which probe if multiple probes are colocated. 953a5f4374aSIngo Molnar */ 954e3343e6aSSrikar Dronamraju void uprobe_unregister(struct inode *inode, loff_t offset, struct uprobe_consumer *uc) 955a5f4374aSIngo Molnar { 956a5f4374aSIngo Molnar struct uprobe *uprobe; 957a5f4374aSIngo Molnar 958a5f4374aSIngo Molnar uprobe = find_uprobe(inode, offset); 95906d07139SOleg Nesterov if (WARN_ON(!uprobe)) 960a5f4374aSIngo Molnar return; 961a5f4374aSIngo Molnar 962e591c8d7SOleg Nesterov down_write(&uprobe->register_rwsem); 96304aab9b2SOleg Nesterov __uprobe_unregister(uprobe, uc); 964e591c8d7SOleg Nesterov up_write(&uprobe->register_rwsem); 965a5f4374aSIngo Molnar put_uprobe(uprobe); 966a5f4374aSIngo Molnar } 967e8440c14SJosh Stone EXPORT_SYMBOL_GPL(uprobe_unregister); 968a5f4374aSIngo Molnar 969da1816b1SOleg Nesterov static int unapply_uprobe(struct uprobe *uprobe, struct mm_struct *mm) 970da1816b1SOleg Nesterov { 971da1816b1SOleg Nesterov struct vm_area_struct *vma; 972da1816b1SOleg Nesterov int err = 0; 973da1816b1SOleg Nesterov 974da1816b1SOleg Nesterov down_read(&mm->mmap_sem); 975da1816b1SOleg Nesterov for (vma = mm->mmap; vma; vma = vma->vm_next) { 976da1816b1SOleg Nesterov unsigned long vaddr; 977da1816b1SOleg Nesterov loff_t offset; 978da1816b1SOleg Nesterov 979da1816b1SOleg Nesterov if (!valid_vma(vma, false) || 980f281769eSOleg Nesterov file_inode(vma->vm_file) != uprobe->inode) 981da1816b1SOleg Nesterov continue; 982da1816b1SOleg Nesterov 983da1816b1SOleg Nesterov offset = (loff_t)vma->vm_pgoff << PAGE_SHIFT; 984da1816b1SOleg Nesterov if (uprobe->offset < offset || 985da1816b1SOleg Nesterov uprobe->offset >= offset + vma->vm_end - vma->vm_start) 986da1816b1SOleg Nesterov continue; 987da1816b1SOleg Nesterov 988da1816b1SOleg Nesterov vaddr = offset_to_vaddr(vma, uprobe->offset); 989da1816b1SOleg Nesterov err |= remove_breakpoint(uprobe, mm, vaddr); 990da1816b1SOleg Nesterov } 991da1816b1SOleg Nesterov up_read(&mm->mmap_sem); 992da1816b1SOleg Nesterov 993da1816b1SOleg Nesterov return err; 994da1816b1SOleg Nesterov } 995da1816b1SOleg Nesterov 996891c3970SOleg Nesterov static struct rb_node * 997891c3970SOleg Nesterov find_node_in_range(struct inode *inode, loff_t min, loff_t max) 998a5f4374aSIngo Molnar { 999a5f4374aSIngo Molnar struct rb_node *n = uprobes_tree.rb_node; 1000a5f4374aSIngo Molnar 1001a5f4374aSIngo Molnar while (n) { 1002891c3970SOleg Nesterov struct uprobe *u = rb_entry(n, struct uprobe, rb_node); 1003a5f4374aSIngo Molnar 1004891c3970SOleg Nesterov if (inode < u->inode) { 1005a5f4374aSIngo Molnar n = n->rb_left; 1006891c3970SOleg Nesterov } else if (inode > u->inode) { 1007a5f4374aSIngo Molnar n = n->rb_right; 1008891c3970SOleg Nesterov } else { 1009891c3970SOleg Nesterov if (max < u->offset) 1010891c3970SOleg Nesterov n = n->rb_left; 1011891c3970SOleg Nesterov else if (min > u->offset) 1012891c3970SOleg Nesterov n = n->rb_right; 1013891c3970SOleg Nesterov else 1014891c3970SOleg Nesterov break; 1015891c3970SOleg Nesterov } 1016a5f4374aSIngo Molnar } 1017a5f4374aSIngo Molnar 1018891c3970SOleg Nesterov return n; 1019a5f4374aSIngo Molnar } 1020a5f4374aSIngo Molnar 1021a5f4374aSIngo Molnar /* 1022891c3970SOleg Nesterov * For a given range in vma, build a list of probes that need to be inserted. 1023a5f4374aSIngo Molnar */ 1024891c3970SOleg Nesterov static void build_probe_list(struct inode *inode, 1025891c3970SOleg Nesterov struct vm_area_struct *vma, 1026891c3970SOleg Nesterov unsigned long start, unsigned long end, 1027891c3970SOleg Nesterov struct list_head *head) 1028a5f4374aSIngo Molnar { 1029891c3970SOleg Nesterov loff_t min, max; 1030891c3970SOleg Nesterov struct rb_node *n, *t; 1031891c3970SOleg Nesterov struct uprobe *u; 1032891c3970SOleg Nesterov 1033891c3970SOleg Nesterov INIT_LIST_HEAD(head); 1034cb113b47SOleg Nesterov min = vaddr_to_offset(vma, start); 1035891c3970SOleg Nesterov max = min + (end - start) - 1; 1036a5f4374aSIngo Molnar 10376f47caa0SOleg Nesterov spin_lock(&uprobes_treelock); 1038891c3970SOleg Nesterov n = find_node_in_range(inode, min, max); 1039891c3970SOleg Nesterov if (n) { 1040891c3970SOleg Nesterov for (t = n; t; t = rb_prev(t)) { 1041891c3970SOleg Nesterov u = rb_entry(t, struct uprobe, rb_node); 1042891c3970SOleg Nesterov if (u->inode != inode || u->offset < min) 1043a5f4374aSIngo Molnar break; 1044891c3970SOleg Nesterov list_add(&u->pending_list, head); 1045f231722aSOleg Nesterov get_uprobe(u); 1046a5f4374aSIngo Molnar } 1047891c3970SOleg Nesterov for (t = n; (t = rb_next(t)); ) { 1048891c3970SOleg Nesterov u = rb_entry(t, struct uprobe, rb_node); 1049891c3970SOleg Nesterov if (u->inode != inode || u->offset > max) 1050891c3970SOleg Nesterov break; 1051891c3970SOleg Nesterov list_add(&u->pending_list, head); 1052f231722aSOleg Nesterov get_uprobe(u); 1053891c3970SOleg Nesterov } 1054891c3970SOleg Nesterov } 10556f47caa0SOleg Nesterov spin_unlock(&uprobes_treelock); 1056a5f4374aSIngo Molnar } 1057a5f4374aSIngo Molnar 1058a5f4374aSIngo Molnar /* 10595e5be71aSOleg Nesterov * Called from mmap_region/vma_adjust with mm->mmap_sem acquired. 1060a5f4374aSIngo Molnar * 10615e5be71aSOleg Nesterov * Currently we ignore all errors and always return 0, the callers 10625e5be71aSOleg Nesterov * can't handle the failure anyway. 1063a5f4374aSIngo Molnar */ 1064a5f4374aSIngo Molnar int uprobe_mmap(struct vm_area_struct *vma) 1065a5f4374aSIngo Molnar { 1066a5f4374aSIngo Molnar struct list_head tmp_list; 1067665605a2SOleg Nesterov struct uprobe *uprobe, *u; 1068a5f4374aSIngo Molnar struct inode *inode; 1069a5f4374aSIngo Molnar 1070441f1eb7SOleg Nesterov if (no_uprobe_events() || !valid_vma(vma, true)) 1071a5f4374aSIngo Molnar return 0; 1072a5f4374aSIngo Molnar 1073f281769eSOleg Nesterov inode = file_inode(vma->vm_file); 1074a5f4374aSIngo Molnar if (!inode) 1075a5f4374aSIngo Molnar return 0; 1076a5f4374aSIngo Molnar 1077a5f4374aSIngo Molnar mutex_lock(uprobes_mmap_hash(inode)); 1078891c3970SOleg Nesterov build_probe_list(inode, vma, vma->vm_start, vma->vm_end, &tmp_list); 1079806a98bdSOleg Nesterov /* 1080806a98bdSOleg Nesterov * We can race with uprobe_unregister(), this uprobe can be already 1081806a98bdSOleg Nesterov * removed. But in this case filter_chain() must return false, all 1082806a98bdSOleg Nesterov * consumers have gone away. 1083806a98bdSOleg Nesterov */ 1084665605a2SOleg Nesterov list_for_each_entry_safe(uprobe, u, &tmp_list, pending_list) { 1085806a98bdSOleg Nesterov if (!fatal_signal_pending(current) && 10868a7f2fa0SOleg Nesterov filter_chain(uprobe, UPROBE_FILTER_MMAP, vma->vm_mm)) { 108757683f72SOleg Nesterov unsigned long vaddr = offset_to_vaddr(vma, uprobe->offset); 10885e5be71aSOleg Nesterov install_breakpoint(uprobe, vma->vm_mm, vma, vaddr); 1089a5f4374aSIngo Molnar } 1090a5f4374aSIngo Molnar put_uprobe(uprobe); 1091a5f4374aSIngo Molnar } 1092a5f4374aSIngo Molnar mutex_unlock(uprobes_mmap_hash(inode)); 1093a5f4374aSIngo Molnar 10945e5be71aSOleg Nesterov return 0; 1095a5f4374aSIngo Molnar } 1096a5f4374aSIngo Molnar 10979f68f672SOleg Nesterov static bool 10989f68f672SOleg Nesterov vma_has_uprobes(struct vm_area_struct *vma, unsigned long start, unsigned long end) 10999f68f672SOleg Nesterov { 11009f68f672SOleg Nesterov loff_t min, max; 11019f68f672SOleg Nesterov struct inode *inode; 11029f68f672SOleg Nesterov struct rb_node *n; 11039f68f672SOleg Nesterov 1104f281769eSOleg Nesterov inode = file_inode(vma->vm_file); 11059f68f672SOleg Nesterov 11069f68f672SOleg Nesterov min = vaddr_to_offset(vma, start); 11079f68f672SOleg Nesterov max = min + (end - start) - 1; 11089f68f672SOleg Nesterov 11099f68f672SOleg Nesterov spin_lock(&uprobes_treelock); 11109f68f672SOleg Nesterov n = find_node_in_range(inode, min, max); 11119f68f672SOleg Nesterov spin_unlock(&uprobes_treelock); 11129f68f672SOleg Nesterov 11139f68f672SOleg Nesterov return !!n; 11149f68f672SOleg Nesterov } 11159f68f672SOleg Nesterov 1116682968e0SSrikar Dronamraju /* 1117682968e0SSrikar Dronamraju * Called in context of a munmap of a vma. 1118682968e0SSrikar Dronamraju */ 1119cbc91f71SSrikar Dronamraju void uprobe_munmap(struct vm_area_struct *vma, unsigned long start, unsigned long end) 1120682968e0SSrikar Dronamraju { 1121441f1eb7SOleg Nesterov if (no_uprobe_events() || !valid_vma(vma, false)) 1122682968e0SSrikar Dronamraju return; 1123682968e0SSrikar Dronamraju 11242fd611a9SOleg Nesterov if (!atomic_read(&vma->vm_mm->mm_users)) /* called by mmput() ? */ 11252fd611a9SOleg Nesterov return; 11262fd611a9SOleg Nesterov 11279f68f672SOleg Nesterov if (!test_bit(MMF_HAS_UPROBES, &vma->vm_mm->flags) || 11289f68f672SOleg Nesterov test_bit(MMF_RECALC_UPROBES, &vma->vm_mm->flags)) 1129f8ac4ec9SOleg Nesterov return; 1130f8ac4ec9SOleg Nesterov 11319f68f672SOleg Nesterov if (vma_has_uprobes(vma, start, end)) 11329f68f672SOleg Nesterov set_bit(MMF_RECALC_UPROBES, &vma->vm_mm->flags); 1133682968e0SSrikar Dronamraju } 1134682968e0SSrikar Dronamraju 1135d4b3b638SSrikar Dronamraju /* Slot allocation for XOL */ 11366441ec8bSOleg Nesterov static int xol_add_vma(struct mm_struct *mm, struct xol_area *area) 1137d4b3b638SSrikar Dronamraju { 1138704bde3cSOleg Nesterov struct vm_area_struct *vma; 1139704bde3cSOleg Nesterov int ret; 1140d4b3b638SSrikar Dronamraju 1141598fdc1dSMichal Hocko if (down_write_killable(&mm->mmap_sem)) 1142598fdc1dSMichal Hocko return -EINTR; 1143598fdc1dSMichal Hocko 1144704bde3cSOleg Nesterov if (mm->uprobes_state.xol_area) { 1145704bde3cSOleg Nesterov ret = -EALREADY; 1146d4b3b638SSrikar Dronamraju goto fail; 1147704bde3cSOleg Nesterov } 1148d4b3b638SSrikar Dronamraju 1149af0d95afSOleg Nesterov if (!area->vaddr) { 1150d4b3b638SSrikar Dronamraju /* Try to map as high as possible, this is only a hint. */ 1151af0d95afSOleg Nesterov area->vaddr = get_unmapped_area(NULL, TASK_SIZE - PAGE_SIZE, 1152af0d95afSOleg Nesterov PAGE_SIZE, 0, 0); 1153d4b3b638SSrikar Dronamraju if (area->vaddr & ~PAGE_MASK) { 1154d4b3b638SSrikar Dronamraju ret = area->vaddr; 1155d4b3b638SSrikar Dronamraju goto fail; 1156d4b3b638SSrikar Dronamraju } 1157af0d95afSOleg Nesterov } 1158d4b3b638SSrikar Dronamraju 1159704bde3cSOleg Nesterov vma = _install_special_mapping(mm, area->vaddr, PAGE_SIZE, 1160704bde3cSOleg Nesterov VM_EXEC|VM_MAYEXEC|VM_DONTCOPY|VM_IO, 1161704bde3cSOleg Nesterov &area->xol_mapping); 1162704bde3cSOleg Nesterov if (IS_ERR(vma)) { 1163704bde3cSOleg Nesterov ret = PTR_ERR(vma); 1164d4b3b638SSrikar Dronamraju goto fail; 1165704bde3cSOleg Nesterov } 1166d4b3b638SSrikar Dronamraju 1167704bde3cSOleg Nesterov ret = 0; 1168d4b3b638SSrikar Dronamraju smp_wmb(); /* pairs with get_xol_area() */ 1169d4b3b638SSrikar Dronamraju mm->uprobes_state.xol_area = area; 1170d4b3b638SSrikar Dronamraju fail: 1171d4b3b638SSrikar Dronamraju up_write(&mm->mmap_sem); 1172d4b3b638SSrikar Dronamraju 1173d4b3b638SSrikar Dronamraju return ret; 1174d4b3b638SSrikar Dronamraju } 1175d4b3b638SSrikar Dronamraju 1176af0d95afSOleg Nesterov static struct xol_area *__create_xol_area(unsigned long vaddr) 1177d4b3b638SSrikar Dronamraju { 11789b545df8SOleg Nesterov struct mm_struct *mm = current->mm; 1179e78aebfdSAnton Arapov uprobe_opcode_t insn = UPROBE_SWBP_INSN; 11806441ec8bSOleg Nesterov struct xol_area *area; 11819b545df8SOleg Nesterov 1182af0d95afSOleg Nesterov area = kmalloc(sizeof(*area), GFP_KERNEL); 1183d4b3b638SSrikar Dronamraju if (unlikely(!area)) 1184c8a82538SOleg Nesterov goto out; 1185d4b3b638SSrikar Dronamraju 1186d4b3b638SSrikar Dronamraju area->bitmap = kzalloc(BITS_TO_LONGS(UINSNS_PER_PAGE) * sizeof(long), GFP_KERNEL); 1187d4b3b638SSrikar Dronamraju if (!area->bitmap) 1188c8a82538SOleg Nesterov goto free_area; 1189c8a82538SOleg Nesterov 1190704bde3cSOleg Nesterov area->xol_mapping.name = "[uprobes]"; 1191869ae761SOleg Nesterov area->xol_mapping.fault = NULL; 1192704bde3cSOleg Nesterov area->xol_mapping.pages = area->pages; 1193f58bea2fSOleg Nesterov area->pages[0] = alloc_page(GFP_HIGHUSER); 1194f58bea2fSOleg Nesterov if (!area->pages[0]) 1195c8a82538SOleg Nesterov goto free_bitmap; 1196f58bea2fSOleg Nesterov area->pages[1] = NULL; 1197d4b3b638SSrikar Dronamraju 1198af0d95afSOleg Nesterov area->vaddr = vaddr; 1199d4b3b638SSrikar Dronamraju init_waitqueue_head(&area->wq); 12006441ec8bSOleg Nesterov /* Reserve the 1st slot for get_trampoline_vaddr() */ 12016441ec8bSOleg Nesterov set_bit(0, area->bitmap); 12026441ec8bSOleg Nesterov atomic_set(&area->slot_count, 1); 1203297e765eSMarcin Nowakowski arch_uprobe_copy_ixol(area->pages[0], 0, &insn, UPROBE_SWBP_INSN_SIZE); 1204e78aebfdSAnton Arapov 12056441ec8bSOleg Nesterov if (!xol_add_vma(mm, area)) 1206d4b3b638SSrikar Dronamraju return area; 1207d4b3b638SSrikar Dronamraju 1208f58bea2fSOleg Nesterov __free_page(area->pages[0]); 1209c8a82538SOleg Nesterov free_bitmap: 1210d4b3b638SSrikar Dronamraju kfree(area->bitmap); 1211c8a82538SOleg Nesterov free_area: 1212d4b3b638SSrikar Dronamraju kfree(area); 1213c8a82538SOleg Nesterov out: 12146441ec8bSOleg Nesterov return NULL; 12156441ec8bSOleg Nesterov } 12166441ec8bSOleg Nesterov 12176441ec8bSOleg Nesterov /* 12186441ec8bSOleg Nesterov * get_xol_area - Allocate process's xol_area if necessary. 12196441ec8bSOleg Nesterov * This area will be used for storing instructions for execution out of line. 12206441ec8bSOleg Nesterov * 12216441ec8bSOleg Nesterov * Returns the allocated area or NULL. 12226441ec8bSOleg Nesterov */ 12236441ec8bSOleg Nesterov static struct xol_area *get_xol_area(void) 12246441ec8bSOleg Nesterov { 12256441ec8bSOleg Nesterov struct mm_struct *mm = current->mm; 12266441ec8bSOleg Nesterov struct xol_area *area; 12276441ec8bSOleg Nesterov 12286441ec8bSOleg Nesterov if (!mm->uprobes_state.xol_area) 1229af0d95afSOleg Nesterov __create_xol_area(0); 12306441ec8bSOleg Nesterov 12319b545df8SOleg Nesterov area = mm->uprobes_state.xol_area; 12329b545df8SOleg Nesterov smp_read_barrier_depends(); /* pairs with wmb in xol_add_vma() */ 12339b545df8SOleg Nesterov return area; 1234d4b3b638SSrikar Dronamraju } 1235d4b3b638SSrikar Dronamraju 1236d4b3b638SSrikar Dronamraju /* 1237d4b3b638SSrikar Dronamraju * uprobe_clear_state - Free the area allocated for slots. 1238d4b3b638SSrikar Dronamraju */ 1239d4b3b638SSrikar Dronamraju void uprobe_clear_state(struct mm_struct *mm) 1240d4b3b638SSrikar Dronamraju { 1241d4b3b638SSrikar Dronamraju struct xol_area *area = mm->uprobes_state.xol_area; 1242d4b3b638SSrikar Dronamraju 1243d4b3b638SSrikar Dronamraju if (!area) 1244d4b3b638SSrikar Dronamraju return; 1245d4b3b638SSrikar Dronamraju 1246f58bea2fSOleg Nesterov put_page(area->pages[0]); 1247d4b3b638SSrikar Dronamraju kfree(area->bitmap); 1248d4b3b638SSrikar Dronamraju kfree(area); 1249d4b3b638SSrikar Dronamraju } 1250d4b3b638SSrikar Dronamraju 125132cdba1eSOleg Nesterov void uprobe_start_dup_mmap(void) 125232cdba1eSOleg Nesterov { 125332cdba1eSOleg Nesterov percpu_down_read(&dup_mmap_sem); 125432cdba1eSOleg Nesterov } 125532cdba1eSOleg Nesterov 125632cdba1eSOleg Nesterov void uprobe_end_dup_mmap(void) 125732cdba1eSOleg Nesterov { 125832cdba1eSOleg Nesterov percpu_up_read(&dup_mmap_sem); 125932cdba1eSOleg Nesterov } 126032cdba1eSOleg Nesterov 1261f8ac4ec9SOleg Nesterov void uprobe_dup_mmap(struct mm_struct *oldmm, struct mm_struct *newmm) 1262f8ac4ec9SOleg Nesterov { 126361559a81SOleg Nesterov newmm->uprobes_state.xol_area = NULL; 126461559a81SOleg Nesterov 12659f68f672SOleg Nesterov if (test_bit(MMF_HAS_UPROBES, &oldmm->flags)) { 1266f8ac4ec9SOleg Nesterov set_bit(MMF_HAS_UPROBES, &newmm->flags); 12679f68f672SOleg Nesterov /* unconditionally, dup_mmap() skips VM_DONTCOPY vmas */ 12689f68f672SOleg Nesterov set_bit(MMF_RECALC_UPROBES, &newmm->flags); 12699f68f672SOleg Nesterov } 1270f8ac4ec9SOleg Nesterov } 1271f8ac4ec9SOleg Nesterov 1272d4b3b638SSrikar Dronamraju /* 1273d4b3b638SSrikar Dronamraju * - search for a free slot. 1274d4b3b638SSrikar Dronamraju */ 1275d4b3b638SSrikar Dronamraju static unsigned long xol_take_insn_slot(struct xol_area *area) 1276d4b3b638SSrikar Dronamraju { 1277d4b3b638SSrikar Dronamraju unsigned long slot_addr; 1278d4b3b638SSrikar Dronamraju int slot_nr; 1279d4b3b638SSrikar Dronamraju 1280d4b3b638SSrikar Dronamraju do { 1281d4b3b638SSrikar Dronamraju slot_nr = find_first_zero_bit(area->bitmap, UINSNS_PER_PAGE); 1282d4b3b638SSrikar Dronamraju if (slot_nr < UINSNS_PER_PAGE) { 1283d4b3b638SSrikar Dronamraju if (!test_and_set_bit(slot_nr, area->bitmap)) 1284d4b3b638SSrikar Dronamraju break; 1285d4b3b638SSrikar Dronamraju 1286d4b3b638SSrikar Dronamraju slot_nr = UINSNS_PER_PAGE; 1287d4b3b638SSrikar Dronamraju continue; 1288d4b3b638SSrikar Dronamraju } 1289d4b3b638SSrikar Dronamraju wait_event(area->wq, (atomic_read(&area->slot_count) < UINSNS_PER_PAGE)); 1290d4b3b638SSrikar Dronamraju } while (slot_nr >= UINSNS_PER_PAGE); 1291d4b3b638SSrikar Dronamraju 1292d4b3b638SSrikar Dronamraju slot_addr = area->vaddr + (slot_nr * UPROBE_XOL_SLOT_BYTES); 1293d4b3b638SSrikar Dronamraju atomic_inc(&area->slot_count); 1294d4b3b638SSrikar Dronamraju 1295d4b3b638SSrikar Dronamraju return slot_addr; 1296d4b3b638SSrikar Dronamraju } 1297d4b3b638SSrikar Dronamraju 1298d4b3b638SSrikar Dronamraju /* 1299a6cb3f6dSOleg Nesterov * xol_get_insn_slot - allocate a slot for xol. 1300d4b3b638SSrikar Dronamraju * Returns the allocated slot address or 0. 1301d4b3b638SSrikar Dronamraju */ 1302a6cb3f6dSOleg Nesterov static unsigned long xol_get_insn_slot(struct uprobe *uprobe) 1303d4b3b638SSrikar Dronamraju { 1304d4b3b638SSrikar Dronamraju struct xol_area *area; 1305a6cb3f6dSOleg Nesterov unsigned long xol_vaddr; 1306d4b3b638SSrikar Dronamraju 13079b545df8SOleg Nesterov area = get_xol_area(); 1308d4b3b638SSrikar Dronamraju if (!area) 1309d4b3b638SSrikar Dronamraju return 0; 1310d4b3b638SSrikar Dronamraju 1311a6cb3f6dSOleg Nesterov xol_vaddr = xol_take_insn_slot(area); 1312a6cb3f6dSOleg Nesterov if (unlikely(!xol_vaddr)) 1313d4b3b638SSrikar Dronamraju return 0; 1314d4b3b638SSrikar Dronamraju 1315f58bea2fSOleg Nesterov arch_uprobe_copy_ixol(area->pages[0], xol_vaddr, 1316803200e2SOleg Nesterov &uprobe->arch.ixol, sizeof(uprobe->arch.ixol)); 1317d4b3b638SSrikar Dronamraju 1318a6cb3f6dSOleg Nesterov return xol_vaddr; 1319d4b3b638SSrikar Dronamraju } 1320d4b3b638SSrikar Dronamraju 1321d4b3b638SSrikar Dronamraju /* 1322d4b3b638SSrikar Dronamraju * xol_free_insn_slot - If slot was earlier allocated by 1323d4b3b638SSrikar Dronamraju * @xol_get_insn_slot(), make the slot available for 1324d4b3b638SSrikar Dronamraju * subsequent requests. 1325d4b3b638SSrikar Dronamraju */ 1326d4b3b638SSrikar Dronamraju static void xol_free_insn_slot(struct task_struct *tsk) 1327d4b3b638SSrikar Dronamraju { 1328d4b3b638SSrikar Dronamraju struct xol_area *area; 1329d4b3b638SSrikar Dronamraju unsigned long vma_end; 1330d4b3b638SSrikar Dronamraju unsigned long slot_addr; 1331d4b3b638SSrikar Dronamraju 1332d4b3b638SSrikar Dronamraju if (!tsk->mm || !tsk->mm->uprobes_state.xol_area || !tsk->utask) 1333d4b3b638SSrikar Dronamraju return; 1334d4b3b638SSrikar Dronamraju 1335d4b3b638SSrikar Dronamraju slot_addr = tsk->utask->xol_vaddr; 1336af4355e9SOleg Nesterov if (unlikely(!slot_addr)) 1337d4b3b638SSrikar Dronamraju return; 1338d4b3b638SSrikar Dronamraju 1339d4b3b638SSrikar Dronamraju area = tsk->mm->uprobes_state.xol_area; 1340d4b3b638SSrikar Dronamraju vma_end = area->vaddr + PAGE_SIZE; 1341d4b3b638SSrikar Dronamraju if (area->vaddr <= slot_addr && slot_addr < vma_end) { 1342d4b3b638SSrikar Dronamraju unsigned long offset; 1343d4b3b638SSrikar Dronamraju int slot_nr; 1344d4b3b638SSrikar Dronamraju 1345d4b3b638SSrikar Dronamraju offset = slot_addr - area->vaddr; 1346d4b3b638SSrikar Dronamraju slot_nr = offset / UPROBE_XOL_SLOT_BYTES; 1347d4b3b638SSrikar Dronamraju if (slot_nr >= UINSNS_PER_PAGE) 1348d4b3b638SSrikar Dronamraju return; 1349d4b3b638SSrikar Dronamraju 1350d4b3b638SSrikar Dronamraju clear_bit(slot_nr, area->bitmap); 1351d4b3b638SSrikar Dronamraju atomic_dec(&area->slot_count); 13522a742cedSOleg Nesterov smp_mb__after_atomic(); /* pairs with prepare_to_wait() */ 1353d4b3b638SSrikar Dronamraju if (waitqueue_active(&area->wq)) 1354d4b3b638SSrikar Dronamraju wake_up(&area->wq); 1355d4b3b638SSrikar Dronamraju 1356d4b3b638SSrikar Dronamraju tsk->utask->xol_vaddr = 0; 1357d4b3b638SSrikar Dronamraju } 1358d4b3b638SSrikar Dronamraju } 1359d4b3b638SSrikar Dronamraju 136072e6ae28SVictor Kamensky void __weak arch_uprobe_copy_ixol(struct page *page, unsigned long vaddr, 136172e6ae28SVictor Kamensky void *src, unsigned long len) 136272e6ae28SVictor Kamensky { 136372e6ae28SVictor Kamensky /* Initialize the slot */ 136472e6ae28SVictor Kamensky copy_to_page(page, vaddr, src, len); 136572e6ae28SVictor Kamensky 136672e6ae28SVictor Kamensky /* 136772e6ae28SVictor Kamensky * We probably need flush_icache_user_range() but it needs vma. 136872e6ae28SVictor Kamensky * This should work on most of architectures by default. If 136972e6ae28SVictor Kamensky * architecture needs to do something different it can define 137072e6ae28SVictor Kamensky * its own version of the function. 137172e6ae28SVictor Kamensky */ 137272e6ae28SVictor Kamensky flush_dcache_page(page); 137372e6ae28SVictor Kamensky } 137472e6ae28SVictor Kamensky 13750326f5a9SSrikar Dronamraju /** 13760326f5a9SSrikar Dronamraju * uprobe_get_swbp_addr - compute address of swbp given post-swbp regs 13770326f5a9SSrikar Dronamraju * @regs: Reflects the saved state of the task after it has hit a breakpoint 13780326f5a9SSrikar Dronamraju * instruction. 13790326f5a9SSrikar Dronamraju * Return the address of the breakpoint instruction. 13800326f5a9SSrikar Dronamraju */ 13810326f5a9SSrikar Dronamraju unsigned long __weak uprobe_get_swbp_addr(struct pt_regs *regs) 13820326f5a9SSrikar Dronamraju { 13830326f5a9SSrikar Dronamraju return instruction_pointer(regs) - UPROBE_SWBP_INSN_SIZE; 13840326f5a9SSrikar Dronamraju } 13850326f5a9SSrikar Dronamraju 1386b02ef20aSOleg Nesterov unsigned long uprobe_get_trap_addr(struct pt_regs *regs) 1387b02ef20aSOleg Nesterov { 1388b02ef20aSOleg Nesterov struct uprobe_task *utask = current->utask; 1389b02ef20aSOleg Nesterov 1390b02ef20aSOleg Nesterov if (unlikely(utask && utask->active_uprobe)) 1391b02ef20aSOleg Nesterov return utask->vaddr; 1392b02ef20aSOleg Nesterov 1393b02ef20aSOleg Nesterov return instruction_pointer(regs); 1394b02ef20aSOleg Nesterov } 1395b02ef20aSOleg Nesterov 13962bb5e840SOleg Nesterov static struct return_instance *free_ret_instance(struct return_instance *ri) 13972bb5e840SOleg Nesterov { 13982bb5e840SOleg Nesterov struct return_instance *next = ri->next; 13992bb5e840SOleg Nesterov put_uprobe(ri->uprobe); 14002bb5e840SOleg Nesterov kfree(ri); 14012bb5e840SOleg Nesterov return next; 14022bb5e840SOleg Nesterov } 14032bb5e840SOleg Nesterov 14040326f5a9SSrikar Dronamraju /* 14050326f5a9SSrikar Dronamraju * Called with no locks held. 14060326f5a9SSrikar Dronamraju * Called in context of a exiting or a exec-ing thread. 14070326f5a9SSrikar Dronamraju */ 14080326f5a9SSrikar Dronamraju void uprobe_free_utask(struct task_struct *t) 14090326f5a9SSrikar Dronamraju { 14100326f5a9SSrikar Dronamraju struct uprobe_task *utask = t->utask; 14112bb5e840SOleg Nesterov struct return_instance *ri; 14120326f5a9SSrikar Dronamraju 14130326f5a9SSrikar Dronamraju if (!utask) 14140326f5a9SSrikar Dronamraju return; 14150326f5a9SSrikar Dronamraju 14160326f5a9SSrikar Dronamraju if (utask->active_uprobe) 14170326f5a9SSrikar Dronamraju put_uprobe(utask->active_uprobe); 14180326f5a9SSrikar Dronamraju 14190dfd0eb8SAnton Arapov ri = utask->return_instances; 14202bb5e840SOleg Nesterov while (ri) 14212bb5e840SOleg Nesterov ri = free_ret_instance(ri); 14220dfd0eb8SAnton Arapov 1423d4b3b638SSrikar Dronamraju xol_free_insn_slot(t); 14240326f5a9SSrikar Dronamraju kfree(utask); 14250326f5a9SSrikar Dronamraju t->utask = NULL; 14260326f5a9SSrikar Dronamraju } 14270326f5a9SSrikar Dronamraju 14280326f5a9SSrikar Dronamraju /* 14295a2df662SOleg Nesterov * Allocate a uprobe_task object for the task if if necessary. 14305a2df662SOleg Nesterov * Called when the thread hits a breakpoint. 14310326f5a9SSrikar Dronamraju * 14320326f5a9SSrikar Dronamraju * Returns: 14330326f5a9SSrikar Dronamraju * - pointer to new uprobe_task on success 14340326f5a9SSrikar Dronamraju * - NULL otherwise 14350326f5a9SSrikar Dronamraju */ 14365a2df662SOleg Nesterov static struct uprobe_task *get_utask(void) 14370326f5a9SSrikar Dronamraju { 14385a2df662SOleg Nesterov if (!current->utask) 14395a2df662SOleg Nesterov current->utask = kzalloc(sizeof(struct uprobe_task), GFP_KERNEL); 14405a2df662SOleg Nesterov return current->utask; 14410326f5a9SSrikar Dronamraju } 14420326f5a9SSrikar Dronamraju 1443248d3a7bSOleg Nesterov static int dup_utask(struct task_struct *t, struct uprobe_task *o_utask) 1444248d3a7bSOleg Nesterov { 1445248d3a7bSOleg Nesterov struct uprobe_task *n_utask; 1446248d3a7bSOleg Nesterov struct return_instance **p, *o, *n; 1447248d3a7bSOleg Nesterov 1448248d3a7bSOleg Nesterov n_utask = kzalloc(sizeof(struct uprobe_task), GFP_KERNEL); 1449248d3a7bSOleg Nesterov if (!n_utask) 1450248d3a7bSOleg Nesterov return -ENOMEM; 1451248d3a7bSOleg Nesterov t->utask = n_utask; 1452248d3a7bSOleg Nesterov 1453248d3a7bSOleg Nesterov p = &n_utask->return_instances; 1454248d3a7bSOleg Nesterov for (o = o_utask->return_instances; o; o = o->next) { 1455248d3a7bSOleg Nesterov n = kmalloc(sizeof(struct return_instance), GFP_KERNEL); 1456248d3a7bSOleg Nesterov if (!n) 1457248d3a7bSOleg Nesterov return -ENOMEM; 1458248d3a7bSOleg Nesterov 1459248d3a7bSOleg Nesterov *n = *o; 1460f231722aSOleg Nesterov get_uprobe(n->uprobe); 1461248d3a7bSOleg Nesterov n->next = NULL; 1462248d3a7bSOleg Nesterov 1463248d3a7bSOleg Nesterov *p = n; 1464248d3a7bSOleg Nesterov p = &n->next; 1465248d3a7bSOleg Nesterov n_utask->depth++; 1466248d3a7bSOleg Nesterov } 1467248d3a7bSOleg Nesterov 1468248d3a7bSOleg Nesterov return 0; 1469248d3a7bSOleg Nesterov } 1470248d3a7bSOleg Nesterov 1471248d3a7bSOleg Nesterov static void uprobe_warn(struct task_struct *t, const char *msg) 1472248d3a7bSOleg Nesterov { 1473248d3a7bSOleg Nesterov pr_warn("uprobe: %s:%d failed to %s\n", 1474248d3a7bSOleg Nesterov current->comm, current->pid, msg); 1475248d3a7bSOleg Nesterov } 1476248d3a7bSOleg Nesterov 1477aa59c53fSOleg Nesterov static void dup_xol_work(struct callback_head *work) 1478aa59c53fSOleg Nesterov { 1479aa59c53fSOleg Nesterov if (current->flags & PF_EXITING) 1480aa59c53fSOleg Nesterov return; 1481aa59c53fSOleg Nesterov 1482598fdc1dSMichal Hocko if (!__create_xol_area(current->utask->dup_xol_addr) && 1483598fdc1dSMichal Hocko !fatal_signal_pending(current)) 1484aa59c53fSOleg Nesterov uprobe_warn(current, "dup xol area"); 1485aa59c53fSOleg Nesterov } 1486aa59c53fSOleg Nesterov 1487e78aebfdSAnton Arapov /* 1488b68e0749SOleg Nesterov * Called in context of a new clone/fork from copy_process. 1489b68e0749SOleg Nesterov */ 14903ab67966SOleg Nesterov void uprobe_copy_process(struct task_struct *t, unsigned long flags) 1491b68e0749SOleg Nesterov { 1492248d3a7bSOleg Nesterov struct uprobe_task *utask = current->utask; 1493248d3a7bSOleg Nesterov struct mm_struct *mm = current->mm; 1494aa59c53fSOleg Nesterov struct xol_area *area; 1495248d3a7bSOleg Nesterov 1496b68e0749SOleg Nesterov t->utask = NULL; 1497248d3a7bSOleg Nesterov 14983ab67966SOleg Nesterov if (!utask || !utask->return_instances) 14993ab67966SOleg Nesterov return; 15003ab67966SOleg Nesterov 15013ab67966SOleg Nesterov if (mm == t->mm && !(flags & CLONE_VFORK)) 1502248d3a7bSOleg Nesterov return; 1503248d3a7bSOleg Nesterov 1504248d3a7bSOleg Nesterov if (dup_utask(t, utask)) 1505248d3a7bSOleg Nesterov return uprobe_warn(t, "dup ret instances"); 1506aa59c53fSOleg Nesterov 1507aa59c53fSOleg Nesterov /* The task can fork() after dup_xol_work() fails */ 1508aa59c53fSOleg Nesterov area = mm->uprobes_state.xol_area; 1509aa59c53fSOleg Nesterov if (!area) 1510aa59c53fSOleg Nesterov return uprobe_warn(t, "dup xol area"); 1511aa59c53fSOleg Nesterov 15123ab67966SOleg Nesterov if (mm == t->mm) 15133ab67966SOleg Nesterov return; 15143ab67966SOleg Nesterov 151532473431SOleg Nesterov t->utask->dup_xol_addr = area->vaddr; 151632473431SOleg Nesterov init_task_work(&t->utask->dup_xol_work, dup_xol_work); 151732473431SOleg Nesterov task_work_add(t, &t->utask->dup_xol_work, true); 1518b68e0749SOleg Nesterov } 1519b68e0749SOleg Nesterov 1520b68e0749SOleg Nesterov /* 1521e78aebfdSAnton Arapov * Current area->vaddr notion assume the trampoline address is always 1522e78aebfdSAnton Arapov * equal area->vaddr. 1523e78aebfdSAnton Arapov * 1524e78aebfdSAnton Arapov * Returns -1 in case the xol_area is not allocated. 1525e78aebfdSAnton Arapov */ 1526e78aebfdSAnton Arapov static unsigned long get_trampoline_vaddr(void) 1527e78aebfdSAnton Arapov { 1528e78aebfdSAnton Arapov struct xol_area *area; 1529e78aebfdSAnton Arapov unsigned long trampoline_vaddr = -1; 1530e78aebfdSAnton Arapov 1531e78aebfdSAnton Arapov area = current->mm->uprobes_state.xol_area; 1532e78aebfdSAnton Arapov smp_read_barrier_depends(); 1533e78aebfdSAnton Arapov if (area) 1534e78aebfdSAnton Arapov trampoline_vaddr = area->vaddr; 1535e78aebfdSAnton Arapov 1536e78aebfdSAnton Arapov return trampoline_vaddr; 1537e78aebfdSAnton Arapov } 1538e78aebfdSAnton Arapov 1539db087ef6SOleg Nesterov static void cleanup_return_instances(struct uprobe_task *utask, bool chained, 1540db087ef6SOleg Nesterov struct pt_regs *regs) 1541a5b7e1a8SOleg Nesterov { 1542a5b7e1a8SOleg Nesterov struct return_instance *ri = utask->return_instances; 1543db087ef6SOleg Nesterov enum rp_check ctx = chained ? RP_CHECK_CHAIN_CALL : RP_CHECK_CALL; 154486dcb702SOleg Nesterov 154586dcb702SOleg Nesterov while (ri && !arch_uretprobe_is_alive(ri, ctx, regs)) { 1546a5b7e1a8SOleg Nesterov ri = free_ret_instance(ri); 1547a5b7e1a8SOleg Nesterov utask->depth--; 1548a5b7e1a8SOleg Nesterov } 1549a5b7e1a8SOleg Nesterov utask->return_instances = ri; 1550a5b7e1a8SOleg Nesterov } 1551a5b7e1a8SOleg Nesterov 15520dfd0eb8SAnton Arapov static void prepare_uretprobe(struct uprobe *uprobe, struct pt_regs *regs) 15530dfd0eb8SAnton Arapov { 15540dfd0eb8SAnton Arapov struct return_instance *ri; 15550dfd0eb8SAnton Arapov struct uprobe_task *utask; 15560dfd0eb8SAnton Arapov unsigned long orig_ret_vaddr, trampoline_vaddr; 1557db087ef6SOleg Nesterov bool chained; 15580dfd0eb8SAnton Arapov 15590dfd0eb8SAnton Arapov if (!get_xol_area()) 15600dfd0eb8SAnton Arapov return; 15610dfd0eb8SAnton Arapov 15620dfd0eb8SAnton Arapov utask = get_utask(); 15630dfd0eb8SAnton Arapov if (!utask) 15640dfd0eb8SAnton Arapov return; 15650dfd0eb8SAnton Arapov 1566ded49c55SAnton Arapov if (utask->depth >= MAX_URETPROBE_DEPTH) { 1567ded49c55SAnton Arapov printk_ratelimited(KERN_INFO "uprobe: omit uretprobe due to" 1568ded49c55SAnton Arapov " nestedness limit pid/tgid=%d/%d\n", 1569ded49c55SAnton Arapov current->pid, current->tgid); 1570ded49c55SAnton Arapov return; 1571ded49c55SAnton Arapov } 1572ded49c55SAnton Arapov 15736c58d0e4SOleg Nesterov ri = kmalloc(sizeof(struct return_instance), GFP_KERNEL); 15740dfd0eb8SAnton Arapov if (!ri) 15756c58d0e4SOleg Nesterov return; 15760dfd0eb8SAnton Arapov 15770dfd0eb8SAnton Arapov trampoline_vaddr = get_trampoline_vaddr(); 15780dfd0eb8SAnton Arapov orig_ret_vaddr = arch_uretprobe_hijack_return_addr(trampoline_vaddr, regs); 15790dfd0eb8SAnton Arapov if (orig_ret_vaddr == -1) 15800dfd0eb8SAnton Arapov goto fail; 15810dfd0eb8SAnton Arapov 1582a5b7e1a8SOleg Nesterov /* drop the entries invalidated by longjmp() */ 1583db087ef6SOleg Nesterov chained = (orig_ret_vaddr == trampoline_vaddr); 1584db087ef6SOleg Nesterov cleanup_return_instances(utask, chained, regs); 1585a5b7e1a8SOleg Nesterov 15860dfd0eb8SAnton Arapov /* 15870dfd0eb8SAnton Arapov * We don't want to keep trampoline address in stack, rather keep the 15880dfd0eb8SAnton Arapov * original return address of first caller thru all the consequent 15890dfd0eb8SAnton Arapov * instances. This also makes breakpoint unwrapping easier. 15900dfd0eb8SAnton Arapov */ 1591db087ef6SOleg Nesterov if (chained) { 15920dfd0eb8SAnton Arapov if (!utask->return_instances) { 15930dfd0eb8SAnton Arapov /* 15940dfd0eb8SAnton Arapov * This situation is not possible. Likely we have an 15950dfd0eb8SAnton Arapov * attack from user-space. 15960dfd0eb8SAnton Arapov */ 15976c58d0e4SOleg Nesterov uprobe_warn(current, "handle tail call"); 15980dfd0eb8SAnton Arapov goto fail; 15990dfd0eb8SAnton Arapov } 16000dfd0eb8SAnton Arapov orig_ret_vaddr = utask->return_instances->orig_ret_vaddr; 16010dfd0eb8SAnton Arapov } 16020dfd0eb8SAnton Arapov 1603f231722aSOleg Nesterov ri->uprobe = get_uprobe(uprobe); 16040dfd0eb8SAnton Arapov ri->func = instruction_pointer(regs); 16057b868e48SOleg Nesterov ri->stack = user_stack_pointer(regs); 16060dfd0eb8SAnton Arapov ri->orig_ret_vaddr = orig_ret_vaddr; 16070dfd0eb8SAnton Arapov ri->chained = chained; 16080dfd0eb8SAnton Arapov 1609ded49c55SAnton Arapov utask->depth++; 16100dfd0eb8SAnton Arapov ri->next = utask->return_instances; 16110dfd0eb8SAnton Arapov utask->return_instances = ri; 16120dfd0eb8SAnton Arapov 16130dfd0eb8SAnton Arapov return; 16140dfd0eb8SAnton Arapov fail: 16150dfd0eb8SAnton Arapov kfree(ri); 16160dfd0eb8SAnton Arapov } 16170dfd0eb8SAnton Arapov 16180326f5a9SSrikar Dronamraju /* Prepare to single-step probed instruction out of line. */ 16190326f5a9SSrikar Dronamraju static int 1620a6cb3f6dSOleg Nesterov pre_ssout(struct uprobe *uprobe, struct pt_regs *regs, unsigned long bp_vaddr) 16210326f5a9SSrikar Dronamraju { 1622a6cb3f6dSOleg Nesterov struct uprobe_task *utask; 1623a6cb3f6dSOleg Nesterov unsigned long xol_vaddr; 1624aba51024SOleg Nesterov int err; 1625d4b3b638SSrikar Dronamraju 1626608e7427SOleg Nesterov utask = get_utask(); 1627608e7427SOleg Nesterov if (!utask) 1628608e7427SOleg Nesterov return -ENOMEM; 1629a6cb3f6dSOleg Nesterov 1630a6cb3f6dSOleg Nesterov xol_vaddr = xol_get_insn_slot(uprobe); 1631a6cb3f6dSOleg Nesterov if (!xol_vaddr) 1632a6cb3f6dSOleg Nesterov return -ENOMEM; 1633a6cb3f6dSOleg Nesterov 1634a6cb3f6dSOleg Nesterov utask->xol_vaddr = xol_vaddr; 1635a6cb3f6dSOleg Nesterov utask->vaddr = bp_vaddr; 1636a6cb3f6dSOleg Nesterov 1637aba51024SOleg Nesterov err = arch_uprobe_pre_xol(&uprobe->arch, regs); 1638aba51024SOleg Nesterov if (unlikely(err)) { 1639aba51024SOleg Nesterov xol_free_insn_slot(current); 1640aba51024SOleg Nesterov return err; 1641aba51024SOleg Nesterov } 1642aba51024SOleg Nesterov 1643608e7427SOleg Nesterov utask->active_uprobe = uprobe; 1644608e7427SOleg Nesterov utask->state = UTASK_SSTEP; 1645aba51024SOleg Nesterov return 0; 16460326f5a9SSrikar Dronamraju } 16470326f5a9SSrikar Dronamraju 16480326f5a9SSrikar Dronamraju /* 16490326f5a9SSrikar Dronamraju * If we are singlestepping, then ensure this thread is not connected to 16500326f5a9SSrikar Dronamraju * non-fatal signals until completion of singlestep. When xol insn itself 16510326f5a9SSrikar Dronamraju * triggers the signal, restart the original insn even if the task is 16520326f5a9SSrikar Dronamraju * already SIGKILL'ed (since coredump should report the correct ip). This 16530326f5a9SSrikar Dronamraju * is even more important if the task has a handler for SIGSEGV/etc, The 16540326f5a9SSrikar Dronamraju * _same_ instruction should be repeated again after return from the signal 16550326f5a9SSrikar Dronamraju * handler, and SSTEP can never finish in this case. 16560326f5a9SSrikar Dronamraju */ 16570326f5a9SSrikar Dronamraju bool uprobe_deny_signal(void) 16580326f5a9SSrikar Dronamraju { 16590326f5a9SSrikar Dronamraju struct task_struct *t = current; 16600326f5a9SSrikar Dronamraju struct uprobe_task *utask = t->utask; 16610326f5a9SSrikar Dronamraju 16620326f5a9SSrikar Dronamraju if (likely(!utask || !utask->active_uprobe)) 16630326f5a9SSrikar Dronamraju return false; 16640326f5a9SSrikar Dronamraju 16650326f5a9SSrikar Dronamraju WARN_ON_ONCE(utask->state != UTASK_SSTEP); 16660326f5a9SSrikar Dronamraju 16670326f5a9SSrikar Dronamraju if (signal_pending(t)) { 16680326f5a9SSrikar Dronamraju spin_lock_irq(&t->sighand->siglock); 16690326f5a9SSrikar Dronamraju clear_tsk_thread_flag(t, TIF_SIGPENDING); 16700326f5a9SSrikar Dronamraju spin_unlock_irq(&t->sighand->siglock); 16710326f5a9SSrikar Dronamraju 16720326f5a9SSrikar Dronamraju if (__fatal_signal_pending(t) || arch_uprobe_xol_was_trapped(t)) { 16730326f5a9SSrikar Dronamraju utask->state = UTASK_SSTEP_TRAPPED; 16740326f5a9SSrikar Dronamraju set_tsk_thread_flag(t, TIF_UPROBE); 16750326f5a9SSrikar Dronamraju } 16760326f5a9SSrikar Dronamraju } 16770326f5a9SSrikar Dronamraju 16780326f5a9SSrikar Dronamraju return true; 16790326f5a9SSrikar Dronamraju } 16800326f5a9SSrikar Dronamraju 1681499a4f3eSOleg Nesterov static void mmf_recalc_uprobes(struct mm_struct *mm) 1682499a4f3eSOleg Nesterov { 1683499a4f3eSOleg Nesterov struct vm_area_struct *vma; 1684499a4f3eSOleg Nesterov 1685499a4f3eSOleg Nesterov for (vma = mm->mmap; vma; vma = vma->vm_next) { 1686499a4f3eSOleg Nesterov if (!valid_vma(vma, false)) 1687499a4f3eSOleg Nesterov continue; 1688499a4f3eSOleg Nesterov /* 1689499a4f3eSOleg Nesterov * This is not strictly accurate, we can race with 1690499a4f3eSOleg Nesterov * uprobe_unregister() and see the already removed 1691499a4f3eSOleg Nesterov * uprobe if delete_uprobe() was not yet called. 169263633cbfSOleg Nesterov * Or this uprobe can be filtered out. 1693499a4f3eSOleg Nesterov */ 1694499a4f3eSOleg Nesterov if (vma_has_uprobes(vma, vma->vm_start, vma->vm_end)) 1695499a4f3eSOleg Nesterov return; 1696499a4f3eSOleg Nesterov } 1697499a4f3eSOleg Nesterov 1698499a4f3eSOleg Nesterov clear_bit(MMF_HAS_UPROBES, &mm->flags); 1699499a4f3eSOleg Nesterov } 1700499a4f3eSOleg Nesterov 17010908ad6eSAnanth N Mavinakayanahalli static int is_trap_at_addr(struct mm_struct *mm, unsigned long vaddr) 1702ec75fba9SOleg Nesterov { 1703ec75fba9SOleg Nesterov struct page *page; 1704ec75fba9SOleg Nesterov uprobe_opcode_t opcode; 1705ec75fba9SOleg Nesterov int result; 1706ec75fba9SOleg Nesterov 1707ec75fba9SOleg Nesterov pagefault_disable(); 1708bd28b145SLinus Torvalds result = __get_user(opcode, (uprobe_opcode_t __user *)vaddr); 1709ec75fba9SOleg Nesterov pagefault_enable(); 1710ec75fba9SOleg Nesterov 1711ec75fba9SOleg Nesterov if (likely(result == 0)) 1712ec75fba9SOleg Nesterov goto out; 1713ec75fba9SOleg Nesterov 17141e987790SDave Hansen /* 17151e987790SDave Hansen * The NULL 'tsk' here ensures that any faults that occur here 17161e987790SDave Hansen * will not be accounted to the task. 'mm' *is* current->mm, 17171e987790SDave Hansen * but we treat this as a 'remote' access since it is 17181e987790SDave Hansen * essentially a kernel access to the memory. 17191e987790SDave Hansen */ 17209beae1eaSLorenzo Stoakes result = get_user_pages_remote(NULL, mm, vaddr, 1, FOLL_FORCE, &page, 17215b56d49fSLorenzo Stoakes NULL, NULL); 1722ec75fba9SOleg Nesterov if (result < 0) 1723ec75fba9SOleg Nesterov return result; 1724ec75fba9SOleg Nesterov 1725ab0d805cSOleg Nesterov copy_from_page(page, vaddr, &opcode, UPROBE_SWBP_INSN_SIZE); 1726ec75fba9SOleg Nesterov put_page(page); 1727ec75fba9SOleg Nesterov out: 17280908ad6eSAnanth N Mavinakayanahalli /* This needs to return true for any variant of the trap insn */ 17290908ad6eSAnanth N Mavinakayanahalli return is_trap_insn(&opcode); 1730ec75fba9SOleg Nesterov } 1731ec75fba9SOleg Nesterov 1732d790d346SOleg Nesterov static struct uprobe *find_active_uprobe(unsigned long bp_vaddr, int *is_swbp) 17330326f5a9SSrikar Dronamraju { 17343a9ea052SOleg Nesterov struct mm_struct *mm = current->mm; 17353a9ea052SOleg Nesterov struct uprobe *uprobe = NULL; 17360326f5a9SSrikar Dronamraju struct vm_area_struct *vma; 17370326f5a9SSrikar Dronamraju 17380326f5a9SSrikar Dronamraju down_read(&mm->mmap_sem); 17390326f5a9SSrikar Dronamraju vma = find_vma(mm, bp_vaddr); 17403a9ea052SOleg Nesterov if (vma && vma->vm_start <= bp_vaddr) { 17413a9ea052SOleg Nesterov if (valid_vma(vma, false)) { 1742f281769eSOleg Nesterov struct inode *inode = file_inode(vma->vm_file); 1743cb113b47SOleg Nesterov loff_t offset = vaddr_to_offset(vma, bp_vaddr); 17440326f5a9SSrikar Dronamraju 17450326f5a9SSrikar Dronamraju uprobe = find_uprobe(inode, offset); 17460326f5a9SSrikar Dronamraju } 1747d790d346SOleg Nesterov 1748d790d346SOleg Nesterov if (!uprobe) 17490908ad6eSAnanth N Mavinakayanahalli *is_swbp = is_trap_at_addr(mm, bp_vaddr); 1750d790d346SOleg Nesterov } else { 1751d790d346SOleg Nesterov *is_swbp = -EFAULT; 17523a9ea052SOleg Nesterov } 1753499a4f3eSOleg Nesterov 1754499a4f3eSOleg Nesterov if (!uprobe && test_and_clear_bit(MMF_RECALC_UPROBES, &mm->flags)) 1755499a4f3eSOleg Nesterov mmf_recalc_uprobes(mm); 17560326f5a9SSrikar Dronamraju up_read(&mm->mmap_sem); 17570326f5a9SSrikar Dronamraju 17583a9ea052SOleg Nesterov return uprobe; 17593a9ea052SOleg Nesterov } 17603a9ea052SOleg Nesterov 1761da1816b1SOleg Nesterov static void handler_chain(struct uprobe *uprobe, struct pt_regs *regs) 1762da1816b1SOleg Nesterov { 1763da1816b1SOleg Nesterov struct uprobe_consumer *uc; 1764da1816b1SOleg Nesterov int remove = UPROBE_HANDLER_REMOVE; 17650dfd0eb8SAnton Arapov bool need_prep = false; /* prepare return uprobe, when needed */ 1766da1816b1SOleg Nesterov 1767da1816b1SOleg Nesterov down_read(&uprobe->register_rwsem); 1768da1816b1SOleg Nesterov for (uc = uprobe->consumers; uc; uc = uc->next) { 1769ea024870SAnton Arapov int rc = 0; 1770da1816b1SOleg Nesterov 1771ea024870SAnton Arapov if (uc->handler) { 1772ea024870SAnton Arapov rc = uc->handler(uc, regs); 1773da1816b1SOleg Nesterov WARN(rc & ~UPROBE_HANDLER_MASK, 1774da1816b1SOleg Nesterov "bad rc=0x%x from %pf()\n", rc, uc->handler); 1775ea024870SAnton Arapov } 17760dfd0eb8SAnton Arapov 17770dfd0eb8SAnton Arapov if (uc->ret_handler) 17780dfd0eb8SAnton Arapov need_prep = true; 17790dfd0eb8SAnton Arapov 1780da1816b1SOleg Nesterov remove &= rc; 1781da1816b1SOleg Nesterov } 1782da1816b1SOleg Nesterov 17830dfd0eb8SAnton Arapov if (need_prep && !remove) 17840dfd0eb8SAnton Arapov prepare_uretprobe(uprobe, regs); /* put bp at return */ 17850dfd0eb8SAnton Arapov 1786da1816b1SOleg Nesterov if (remove && uprobe->consumers) { 1787da1816b1SOleg Nesterov WARN_ON(!uprobe_is_active(uprobe)); 1788da1816b1SOleg Nesterov unapply_uprobe(uprobe, current->mm); 1789da1816b1SOleg Nesterov } 1790da1816b1SOleg Nesterov up_read(&uprobe->register_rwsem); 1791da1816b1SOleg Nesterov } 1792da1816b1SOleg Nesterov 1793fec8898dSAnton Arapov static void 1794fec8898dSAnton Arapov handle_uretprobe_chain(struct return_instance *ri, struct pt_regs *regs) 1795fec8898dSAnton Arapov { 1796fec8898dSAnton Arapov struct uprobe *uprobe = ri->uprobe; 1797fec8898dSAnton Arapov struct uprobe_consumer *uc; 1798fec8898dSAnton Arapov 1799fec8898dSAnton Arapov down_read(&uprobe->register_rwsem); 1800fec8898dSAnton Arapov for (uc = uprobe->consumers; uc; uc = uc->next) { 1801fec8898dSAnton Arapov if (uc->ret_handler) 1802fec8898dSAnton Arapov uc->ret_handler(uc, ri->func, regs); 1803fec8898dSAnton Arapov } 1804fec8898dSAnton Arapov up_read(&uprobe->register_rwsem); 1805fec8898dSAnton Arapov } 1806fec8898dSAnton Arapov 1807a83cfeb9SOleg Nesterov static struct return_instance *find_next_ret_chain(struct return_instance *ri) 1808a83cfeb9SOleg Nesterov { 1809a83cfeb9SOleg Nesterov bool chained; 1810a83cfeb9SOleg Nesterov 1811a83cfeb9SOleg Nesterov do { 1812a83cfeb9SOleg Nesterov chained = ri->chained; 1813a83cfeb9SOleg Nesterov ri = ri->next; /* can't be NULL if chained */ 1814a83cfeb9SOleg Nesterov } while (chained); 1815a83cfeb9SOleg Nesterov 1816a83cfeb9SOleg Nesterov return ri; 1817a83cfeb9SOleg Nesterov } 1818a83cfeb9SOleg Nesterov 18190b5256c7SOleg Nesterov static void handle_trampoline(struct pt_regs *regs) 1820fec8898dSAnton Arapov { 1821fec8898dSAnton Arapov struct uprobe_task *utask; 1822a83cfeb9SOleg Nesterov struct return_instance *ri, *next; 18235eeb50deSOleg Nesterov bool valid; 1824fec8898dSAnton Arapov 1825fec8898dSAnton Arapov utask = current->utask; 1826fec8898dSAnton Arapov if (!utask) 18270b5256c7SOleg Nesterov goto sigill; 1828fec8898dSAnton Arapov 1829fec8898dSAnton Arapov ri = utask->return_instances; 1830fec8898dSAnton Arapov if (!ri) 18310b5256c7SOleg Nesterov goto sigill; 1832fec8898dSAnton Arapov 18335eeb50deSOleg Nesterov do { 1834fec8898dSAnton Arapov /* 18355eeb50deSOleg Nesterov * We should throw out the frames invalidated by longjmp(). 18365eeb50deSOleg Nesterov * If this chain is valid, then the next one should be alive 18375eeb50deSOleg Nesterov * or NULL; the latter case means that nobody but ri->func 18385eeb50deSOleg Nesterov * could hit this trampoline on return. TODO: sigaltstack(). 1839fec8898dSAnton Arapov */ 18405eeb50deSOleg Nesterov next = find_next_ret_chain(ri); 184186dcb702SOleg Nesterov valid = !next || arch_uretprobe_is_alive(next, RP_CHECK_RET, regs); 18425eeb50deSOleg Nesterov 1843fec8898dSAnton Arapov instruction_pointer_set(regs, ri->orig_ret_vaddr); 1844a83cfeb9SOleg Nesterov do { 18455eeb50deSOleg Nesterov if (valid) 1846fec8898dSAnton Arapov handle_uretprobe_chain(ri, regs); 18472bb5e840SOleg Nesterov ri = free_ret_instance(ri); 1848878b5a6eSOleg Nesterov utask->depth--; 1849a83cfeb9SOleg Nesterov } while (ri != next); 18505eeb50deSOleg Nesterov } while (!valid); 1851fec8898dSAnton Arapov 1852fec8898dSAnton Arapov utask->return_instances = ri; 18530b5256c7SOleg Nesterov return; 1854fec8898dSAnton Arapov 18550b5256c7SOleg Nesterov sigill: 18560b5256c7SOleg Nesterov uprobe_warn(current, "handle uretprobe, sending SIGILL."); 18570b5256c7SOleg Nesterov force_sig_info(SIGILL, SEND_SIG_FORCED, current); 18580b5256c7SOleg Nesterov 1859fec8898dSAnton Arapov } 1860fec8898dSAnton Arapov 18616fe50a28SDavid A. Long bool __weak arch_uprobe_ignore(struct arch_uprobe *aup, struct pt_regs *regs) 18626fe50a28SDavid A. Long { 18636fe50a28SDavid A. Long return false; 18646fe50a28SDavid A. Long } 18656fe50a28SDavid A. Long 186686dcb702SOleg Nesterov bool __weak arch_uretprobe_is_alive(struct return_instance *ret, enum rp_check ctx, 186786dcb702SOleg Nesterov struct pt_regs *regs) 186897da8976SOleg Nesterov { 186997da8976SOleg Nesterov return true; 187097da8976SOleg Nesterov } 187197da8976SOleg Nesterov 18723a9ea052SOleg Nesterov /* 18733a9ea052SOleg Nesterov * Run handler and ask thread to singlestep. 18743a9ea052SOleg Nesterov * Ensure all non-fatal signals cannot interrupt thread while it singlesteps. 18753a9ea052SOleg Nesterov */ 18763a9ea052SOleg Nesterov static void handle_swbp(struct pt_regs *regs) 18773a9ea052SOleg Nesterov { 18783a9ea052SOleg Nesterov struct uprobe *uprobe; 18793a9ea052SOleg Nesterov unsigned long bp_vaddr; 188056bb4cf6SOleg Nesterov int uninitialized_var(is_swbp); 18813a9ea052SOleg Nesterov 18823a9ea052SOleg Nesterov bp_vaddr = uprobe_get_swbp_addr(regs); 18830b5256c7SOleg Nesterov if (bp_vaddr == get_trampoline_vaddr()) 18840b5256c7SOleg Nesterov return handle_trampoline(regs); 1885fec8898dSAnton Arapov 1886fec8898dSAnton Arapov uprobe = find_active_uprobe(bp_vaddr, &is_swbp); 18870326f5a9SSrikar Dronamraju if (!uprobe) { 188856bb4cf6SOleg Nesterov if (is_swbp > 0) { 18890326f5a9SSrikar Dronamraju /* No matching uprobe; signal SIGTRAP. */ 18900326f5a9SSrikar Dronamraju send_sig(SIGTRAP, current, 0); 189156bb4cf6SOleg Nesterov } else { 189256bb4cf6SOleg Nesterov /* 189356bb4cf6SOleg Nesterov * Either we raced with uprobe_unregister() or we can't 189456bb4cf6SOleg Nesterov * access this memory. The latter is only possible if 189556bb4cf6SOleg Nesterov * another thread plays with our ->mm. In both cases 189656bb4cf6SOleg Nesterov * we can simply restart. If this vma was unmapped we 189756bb4cf6SOleg Nesterov * can pretend this insn was not executed yet and get 189856bb4cf6SOleg Nesterov * the (correct) SIGSEGV after restart. 189956bb4cf6SOleg Nesterov */ 190056bb4cf6SOleg Nesterov instruction_pointer_set(regs, bp_vaddr); 190156bb4cf6SOleg Nesterov } 19020326f5a9SSrikar Dronamraju return; 19030326f5a9SSrikar Dronamraju } 190474e59dfcSOleg Nesterov 190574e59dfcSOleg Nesterov /* change it in advance for ->handler() and restart */ 190674e59dfcSOleg Nesterov instruction_pointer_set(regs, bp_vaddr); 190774e59dfcSOleg Nesterov 1908142b18ddSOleg Nesterov /* 1909142b18ddSOleg Nesterov * TODO: move copy_insn/etc into _register and remove this hack. 1910142b18ddSOleg Nesterov * After we hit the bp, _unregister + _register can install the 1911142b18ddSOleg Nesterov * new and not-yet-analyzed uprobe at the same address, restart. 1912142b18ddSOleg Nesterov */ 1913142b18ddSOleg Nesterov smp_rmb(); /* pairs with wmb() in install_breakpoint() */ 191471434f2fSOleg Nesterov if (unlikely(!test_bit(UPROBE_COPY_INSN, &uprobe->flags))) 191574e59dfcSOleg Nesterov goto out; 19160326f5a9SSrikar Dronamraju 191772fd293aSOleg Nesterov /* Tracing handlers use ->utask to communicate with fetch methods */ 191872fd293aSOleg Nesterov if (!get_utask()) 191972fd293aSOleg Nesterov goto out; 192072fd293aSOleg Nesterov 19216fe50a28SDavid A. Long if (arch_uprobe_ignore(&uprobe->arch, regs)) 19226fe50a28SDavid A. Long goto out; 19236fe50a28SDavid A. Long 19240326f5a9SSrikar Dronamraju handler_chain(uprobe, regs); 19256fe50a28SDavid A. Long 19268a6b1732SOleg Nesterov if (arch_uprobe_skip_sstep(&uprobe->arch, regs)) 19270578a970SOleg Nesterov goto out; 19280326f5a9SSrikar Dronamraju 1929608e7427SOleg Nesterov if (!pre_ssout(uprobe, regs, bp_vaddr)) 19300326f5a9SSrikar Dronamraju return; 19310326f5a9SSrikar Dronamraju 19328a6b1732SOleg Nesterov /* arch_uprobe_skip_sstep() succeeded, or restart if can't singlestep */ 19330578a970SOleg Nesterov out: 19340326f5a9SSrikar Dronamraju put_uprobe(uprobe); 19350326f5a9SSrikar Dronamraju } 19360326f5a9SSrikar Dronamraju 19370326f5a9SSrikar Dronamraju /* 19380326f5a9SSrikar Dronamraju * Perform required fix-ups and disable singlestep. 19390326f5a9SSrikar Dronamraju * Allow pending signals to take effect. 19400326f5a9SSrikar Dronamraju */ 19410326f5a9SSrikar Dronamraju static void handle_singlestep(struct uprobe_task *utask, struct pt_regs *regs) 19420326f5a9SSrikar Dronamraju { 19430326f5a9SSrikar Dronamraju struct uprobe *uprobe; 1944014940baSOleg Nesterov int err = 0; 19450326f5a9SSrikar Dronamraju 19460326f5a9SSrikar Dronamraju uprobe = utask->active_uprobe; 19470326f5a9SSrikar Dronamraju if (utask->state == UTASK_SSTEP_ACK) 1948014940baSOleg Nesterov err = arch_uprobe_post_xol(&uprobe->arch, regs); 19490326f5a9SSrikar Dronamraju else if (utask->state == UTASK_SSTEP_TRAPPED) 19500326f5a9SSrikar Dronamraju arch_uprobe_abort_xol(&uprobe->arch, regs); 19510326f5a9SSrikar Dronamraju else 19520326f5a9SSrikar Dronamraju WARN_ON_ONCE(1); 19530326f5a9SSrikar Dronamraju 19540326f5a9SSrikar Dronamraju put_uprobe(uprobe); 19550326f5a9SSrikar Dronamraju utask->active_uprobe = NULL; 19560326f5a9SSrikar Dronamraju utask->state = UTASK_RUNNING; 1957d4b3b638SSrikar Dronamraju xol_free_insn_slot(current); 19580326f5a9SSrikar Dronamraju 19590326f5a9SSrikar Dronamraju spin_lock_irq(¤t->sighand->siglock); 19600326f5a9SSrikar Dronamraju recalc_sigpending(); /* see uprobe_deny_signal() */ 19610326f5a9SSrikar Dronamraju spin_unlock_irq(¤t->sighand->siglock); 1962014940baSOleg Nesterov 1963014940baSOleg Nesterov if (unlikely(err)) { 1964014940baSOleg Nesterov uprobe_warn(current, "execute the probed insn, sending SIGILL."); 1965014940baSOleg Nesterov force_sig_info(SIGILL, SEND_SIG_FORCED, current); 1966014940baSOleg Nesterov } 19670326f5a9SSrikar Dronamraju } 19680326f5a9SSrikar Dronamraju 19690326f5a9SSrikar Dronamraju /* 19701b08e907SOleg Nesterov * On breakpoint hit, breakpoint notifier sets the TIF_UPROBE flag and 19711b08e907SOleg Nesterov * allows the thread to return from interrupt. After that handle_swbp() 19721b08e907SOleg Nesterov * sets utask->active_uprobe. 19730326f5a9SSrikar Dronamraju * 19741b08e907SOleg Nesterov * On singlestep exception, singlestep notifier sets the TIF_UPROBE flag 19751b08e907SOleg Nesterov * and allows the thread to return from interrupt. 19760326f5a9SSrikar Dronamraju * 19770326f5a9SSrikar Dronamraju * While returning to userspace, thread notices the TIF_UPROBE flag and calls 19780326f5a9SSrikar Dronamraju * uprobe_notify_resume(). 19790326f5a9SSrikar Dronamraju */ 19800326f5a9SSrikar Dronamraju void uprobe_notify_resume(struct pt_regs *regs) 19810326f5a9SSrikar Dronamraju { 19820326f5a9SSrikar Dronamraju struct uprobe_task *utask; 19830326f5a9SSrikar Dronamraju 1984db023ea5SOleg Nesterov clear_thread_flag(TIF_UPROBE); 1985db023ea5SOleg Nesterov 19860326f5a9SSrikar Dronamraju utask = current->utask; 19871b08e907SOleg Nesterov if (utask && utask->active_uprobe) 19880326f5a9SSrikar Dronamraju handle_singlestep(utask, regs); 19891b08e907SOleg Nesterov else 19901b08e907SOleg Nesterov handle_swbp(regs); 19910326f5a9SSrikar Dronamraju } 19920326f5a9SSrikar Dronamraju 19930326f5a9SSrikar Dronamraju /* 19940326f5a9SSrikar Dronamraju * uprobe_pre_sstep_notifier gets called from interrupt context as part of 19950326f5a9SSrikar Dronamraju * notifier mechanism. Set TIF_UPROBE flag and indicate breakpoint hit. 19960326f5a9SSrikar Dronamraju */ 19970326f5a9SSrikar Dronamraju int uprobe_pre_sstep_notifier(struct pt_regs *regs) 19980326f5a9SSrikar Dronamraju { 19990dfd0eb8SAnton Arapov if (!current->mm) 20000dfd0eb8SAnton Arapov return 0; 20010dfd0eb8SAnton Arapov 20020dfd0eb8SAnton Arapov if (!test_bit(MMF_HAS_UPROBES, ¤t->mm->flags) && 20030dfd0eb8SAnton Arapov (!current->utask || !current->utask->return_instances)) 20040326f5a9SSrikar Dronamraju return 0; 20050326f5a9SSrikar Dronamraju 20060326f5a9SSrikar Dronamraju set_thread_flag(TIF_UPROBE); 20070326f5a9SSrikar Dronamraju return 1; 20080326f5a9SSrikar Dronamraju } 20090326f5a9SSrikar Dronamraju 20100326f5a9SSrikar Dronamraju /* 20110326f5a9SSrikar Dronamraju * uprobe_post_sstep_notifier gets called in interrupt context as part of notifier 20120326f5a9SSrikar Dronamraju * mechanism. Set TIF_UPROBE flag and indicate completion of singlestep. 20130326f5a9SSrikar Dronamraju */ 20140326f5a9SSrikar Dronamraju int uprobe_post_sstep_notifier(struct pt_regs *regs) 20150326f5a9SSrikar Dronamraju { 20160326f5a9SSrikar Dronamraju struct uprobe_task *utask = current->utask; 20170326f5a9SSrikar Dronamraju 20180326f5a9SSrikar Dronamraju if (!current->mm || !utask || !utask->active_uprobe) 20190326f5a9SSrikar Dronamraju /* task is currently not uprobed */ 20200326f5a9SSrikar Dronamraju return 0; 20210326f5a9SSrikar Dronamraju 20220326f5a9SSrikar Dronamraju utask->state = UTASK_SSTEP_ACK; 20230326f5a9SSrikar Dronamraju set_thread_flag(TIF_UPROBE); 20240326f5a9SSrikar Dronamraju return 1; 20250326f5a9SSrikar Dronamraju } 20260326f5a9SSrikar Dronamraju 20270326f5a9SSrikar Dronamraju static struct notifier_block uprobe_exception_nb = { 20280326f5a9SSrikar Dronamraju .notifier_call = arch_uprobe_exception_notify, 20290326f5a9SSrikar Dronamraju .priority = INT_MAX-1, /* notified after kprobes, kgdb */ 20300326f5a9SSrikar Dronamraju }; 20310326f5a9SSrikar Dronamraju 2032a5f4374aSIngo Molnar static int __init init_uprobes(void) 2033a5f4374aSIngo Molnar { 2034a5f4374aSIngo Molnar int i; 2035a5f4374aSIngo Molnar 203666d06dffSOleg Nesterov for (i = 0; i < UPROBES_HASH_SZ; i++) 2037a5f4374aSIngo Molnar mutex_init(&uprobes_mmap_mutex[i]); 20380326f5a9SSrikar Dronamraju 203932cdba1eSOleg Nesterov if (percpu_init_rwsem(&dup_mmap_sem)) 204032cdba1eSOleg Nesterov return -ENOMEM; 204132cdba1eSOleg Nesterov 20420326f5a9SSrikar Dronamraju return register_die_notifier(&uprobe_exception_nb); 2043a5f4374aSIngo Molnar } 2044736e89d9SOleg Nesterov __initcall(init_uprobes); 2045