1a5f4374aSIngo Molnar /* 2a5f4374aSIngo Molnar * User-space Probes (UProbes) 3a5f4374aSIngo Molnar * 4a5f4374aSIngo Molnar * This program is free software; you can redistribute it and/or modify 5a5f4374aSIngo Molnar * it under the terms of the GNU General Public License as published by 6a5f4374aSIngo Molnar * the Free Software Foundation; either version 2 of the License, or 7a5f4374aSIngo Molnar * (at your option) any later version. 8a5f4374aSIngo Molnar * 9a5f4374aSIngo Molnar * This program is distributed in the hope that it will be useful, 10a5f4374aSIngo Molnar * but WITHOUT ANY WARRANTY; without even the implied warranty of 11a5f4374aSIngo Molnar * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12a5f4374aSIngo Molnar * GNU General Public License for more details. 13a5f4374aSIngo Molnar * 14a5f4374aSIngo Molnar * You should have received a copy of the GNU General Public License 15a5f4374aSIngo Molnar * along with this program; if not, write to the Free Software 16a5f4374aSIngo Molnar * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 17a5f4374aSIngo Molnar * 1835aa621bSIngo Molnar * Copyright (C) IBM Corporation, 2008-2012 19a5f4374aSIngo Molnar * Authors: 20a5f4374aSIngo Molnar * Srikar Dronamraju 21a5f4374aSIngo Molnar * Jim Keniston 2235aa621bSIngo Molnar * Copyright (C) 2011-2012 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com> 23a5f4374aSIngo Molnar */ 24a5f4374aSIngo Molnar 25a5f4374aSIngo Molnar #include <linux/kernel.h> 26a5f4374aSIngo Molnar #include <linux/highmem.h> 27a5f4374aSIngo Molnar #include <linux/pagemap.h> /* read_mapping_page */ 28a5f4374aSIngo Molnar #include <linux/slab.h> 29a5f4374aSIngo Molnar #include <linux/sched.h> 30e8440c14SJosh Stone #include <linux/export.h> 31a5f4374aSIngo Molnar #include <linux/rmap.h> /* anon_vma_prepare */ 32a5f4374aSIngo Molnar #include <linux/mmu_notifier.h> /* set_pte_at_notify */ 33a5f4374aSIngo Molnar #include <linux/swap.h> /* try_to_free_swap */ 340326f5a9SSrikar Dronamraju #include <linux/ptrace.h> /* user_enable_single_step */ 350326f5a9SSrikar Dronamraju #include <linux/kdebug.h> /* notifier mechanism */ 36194f8dcbSOleg Nesterov #include "../../mm/internal.h" /* munlock_vma_page */ 3732cdba1eSOleg Nesterov #include <linux/percpu-rwsem.h> 38a5f4374aSIngo Molnar 39a5f4374aSIngo Molnar #include <linux/uprobes.h> 40a5f4374aSIngo Molnar 41d4b3b638SSrikar Dronamraju #define UINSNS_PER_PAGE (PAGE_SIZE/UPROBE_XOL_SLOT_BYTES) 42d4b3b638SSrikar Dronamraju #define MAX_UPROBE_XOL_SLOTS UINSNS_PER_PAGE 43d4b3b638SSrikar Dronamraju 44a5f4374aSIngo Molnar static struct rb_root uprobes_tree = RB_ROOT; 45441f1eb7SOleg Nesterov /* 46441f1eb7SOleg Nesterov * allows us to skip the uprobe_mmap if there are no uprobe events active 47441f1eb7SOleg Nesterov * at this time. Probably a fine grained per inode count is better? 48441f1eb7SOleg Nesterov */ 49441f1eb7SOleg Nesterov #define no_uprobe_events() RB_EMPTY_ROOT(&uprobes_tree) 50a5f4374aSIngo Molnar 51a5f4374aSIngo Molnar static DEFINE_SPINLOCK(uprobes_treelock); /* serialize rbtree access */ 52a5f4374aSIngo Molnar 53a5f4374aSIngo Molnar #define UPROBES_HASH_SZ 13 54a5f4374aSIngo Molnar /* serialize uprobe->pending_list */ 55a5f4374aSIngo Molnar static struct mutex uprobes_mmap_mutex[UPROBES_HASH_SZ]; 56a5f4374aSIngo Molnar #define uprobes_mmap_hash(v) (&uprobes_mmap_mutex[((unsigned long)(v)) % UPROBES_HASH_SZ]) 57a5f4374aSIngo Molnar 5832cdba1eSOleg Nesterov static struct percpu_rw_semaphore dup_mmap_sem; 5932cdba1eSOleg Nesterov 60cb9a19feSOleg Nesterov /* Have a copy of original instruction */ 6171434f2fSOleg Nesterov #define UPROBE_COPY_INSN 0 62cb9a19feSOleg Nesterov /* Can skip singlestep */ 63bb929284SOleg Nesterov #define UPROBE_SKIP_SSTEP 1 64cb9a19feSOleg Nesterov 653ff54efdSSrikar Dronamraju struct uprobe { 663ff54efdSSrikar Dronamraju struct rb_node rb_node; /* node in the rb tree */ 673ff54efdSSrikar Dronamraju atomic_t ref; 68e591c8d7SOleg Nesterov struct rw_semaphore register_rwsem; 693ff54efdSSrikar Dronamraju struct rw_semaphore consumer_rwsem; 703ff54efdSSrikar Dronamraju struct list_head pending_list; 713ff54efdSSrikar Dronamraju struct uprobe_consumer *consumers; 723ff54efdSSrikar Dronamraju struct inode *inode; /* Also hold a ref to inode */ 733ff54efdSSrikar Dronamraju loff_t offset; 7471434f2fSOleg Nesterov unsigned long flags; 753ff54efdSSrikar Dronamraju struct arch_uprobe arch; 763ff54efdSSrikar Dronamraju }; 773ff54efdSSrikar Dronamraju 78a5f4374aSIngo Molnar /* 79a5f4374aSIngo Molnar * valid_vma: Verify if the specified vma is an executable vma 80a5f4374aSIngo Molnar * Relax restrictions while unregistering: vm_flags might have 81a5f4374aSIngo Molnar * changed after breakpoint was inserted. 82a5f4374aSIngo Molnar * - is_register: indicates if we are in register context. 83a5f4374aSIngo Molnar * - Return 1 if the specified virtual address is in an 84a5f4374aSIngo Molnar * executable vma. 85a5f4374aSIngo Molnar */ 86a5f4374aSIngo Molnar static bool valid_vma(struct vm_area_struct *vma, bool is_register) 87a5f4374aSIngo Molnar { 88e40cfce6SOleg Nesterov vm_flags_t flags = VM_HUGETLB | VM_MAYEXEC | VM_SHARED; 89a5f4374aSIngo Molnar 90e40cfce6SOleg Nesterov if (is_register) 91e40cfce6SOleg Nesterov flags |= VM_WRITE; 92a5f4374aSIngo Molnar 93e40cfce6SOleg Nesterov return vma->vm_file && (vma->vm_flags & flags) == VM_MAYEXEC; 94a5f4374aSIngo Molnar } 95a5f4374aSIngo Molnar 9657683f72SOleg Nesterov static unsigned long offset_to_vaddr(struct vm_area_struct *vma, loff_t offset) 97a5f4374aSIngo Molnar { 9857683f72SOleg Nesterov return vma->vm_start + offset - ((loff_t)vma->vm_pgoff << PAGE_SHIFT); 99a5f4374aSIngo Molnar } 100a5f4374aSIngo Molnar 101cb113b47SOleg Nesterov static loff_t vaddr_to_offset(struct vm_area_struct *vma, unsigned long vaddr) 102cb113b47SOleg Nesterov { 103cb113b47SOleg Nesterov return ((loff_t)vma->vm_pgoff << PAGE_SHIFT) + (vaddr - vma->vm_start); 104cb113b47SOleg Nesterov } 105cb113b47SOleg Nesterov 106a5f4374aSIngo Molnar /** 107a5f4374aSIngo Molnar * __replace_page - replace page in vma by new page. 108a5f4374aSIngo Molnar * based on replace_page in mm/ksm.c 109a5f4374aSIngo Molnar * 110a5f4374aSIngo Molnar * @vma: vma that holds the pte pointing to page 111c517ee74SOleg Nesterov * @addr: address the old @page is mapped at 112a5f4374aSIngo Molnar * @page: the cowed page we are replacing by kpage 113a5f4374aSIngo Molnar * @kpage: the modified page we replace page by 114a5f4374aSIngo Molnar * 115a5f4374aSIngo Molnar * Returns 0 on success, -EFAULT on failure. 116a5f4374aSIngo Molnar */ 117c517ee74SOleg Nesterov static int __replace_page(struct vm_area_struct *vma, unsigned long addr, 118c517ee74SOleg Nesterov struct page *page, struct page *kpage) 119a5f4374aSIngo Molnar { 120a5f4374aSIngo Molnar struct mm_struct *mm = vma->vm_mm; 1215323ce71SOleg Nesterov spinlock_t *ptl; 1225323ce71SOleg Nesterov pte_t *ptep; 1239f92448cSOleg Nesterov int err; 1246bdb913fSHaggai Eran /* For mmu_notifiers */ 1256bdb913fSHaggai Eran const unsigned long mmun_start = addr; 1266bdb913fSHaggai Eran const unsigned long mmun_end = addr + PAGE_SIZE; 127a5f4374aSIngo Molnar 128194f8dcbSOleg Nesterov /* For try_to_free_swap() and munlock_vma_page() below */ 1299f92448cSOleg Nesterov lock_page(page); 1309f92448cSOleg Nesterov 1316bdb913fSHaggai Eran mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end); 1329f92448cSOleg Nesterov err = -EAGAIN; 1335323ce71SOleg Nesterov ptep = page_check_address(page, mm, addr, &ptl, 0); 134a5f4374aSIngo Molnar if (!ptep) 1359f92448cSOleg Nesterov goto unlock; 136a5f4374aSIngo Molnar 137a5f4374aSIngo Molnar get_page(kpage); 138a5f4374aSIngo Molnar page_add_new_anon_rmap(kpage, vma, addr); 139a5f4374aSIngo Molnar 1407396fa81SSrikar Dronamraju if (!PageAnon(page)) { 1417396fa81SSrikar Dronamraju dec_mm_counter(mm, MM_FILEPAGES); 1427396fa81SSrikar Dronamraju inc_mm_counter(mm, MM_ANONPAGES); 1437396fa81SSrikar Dronamraju } 1447396fa81SSrikar Dronamraju 145a5f4374aSIngo Molnar flush_cache_page(vma, addr, pte_pfn(*ptep)); 146a5f4374aSIngo Molnar ptep_clear_flush(vma, addr, ptep); 147a5f4374aSIngo Molnar set_pte_at_notify(mm, addr, ptep, mk_pte(kpage, vma->vm_page_prot)); 148a5f4374aSIngo Molnar 149a5f4374aSIngo Molnar page_remove_rmap(page); 150a5f4374aSIngo Molnar if (!page_mapped(page)) 151a5f4374aSIngo Molnar try_to_free_swap(page); 152a5f4374aSIngo Molnar pte_unmap_unlock(ptep, ptl); 153a5f4374aSIngo Molnar 154194f8dcbSOleg Nesterov if (vma->vm_flags & VM_LOCKED) 155194f8dcbSOleg Nesterov munlock_vma_page(page); 156194f8dcbSOleg Nesterov put_page(page); 157194f8dcbSOleg Nesterov 1589f92448cSOleg Nesterov err = 0; 1599f92448cSOleg Nesterov unlock: 1606bdb913fSHaggai Eran mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end); 1619f92448cSOleg Nesterov unlock_page(page); 1629f92448cSOleg Nesterov return err; 163a5f4374aSIngo Molnar } 164a5f4374aSIngo Molnar 165a5f4374aSIngo Molnar /** 1665cb4ac3aSSrikar Dronamraju * is_swbp_insn - check if instruction is breakpoint instruction. 167a5f4374aSIngo Molnar * @insn: instruction to be checked. 1685cb4ac3aSSrikar Dronamraju * Default implementation of is_swbp_insn 169a5f4374aSIngo Molnar * Returns true if @insn is a breakpoint instruction. 170a5f4374aSIngo Molnar */ 1715cb4ac3aSSrikar Dronamraju bool __weak is_swbp_insn(uprobe_opcode_t *insn) 172a5f4374aSIngo Molnar { 1735cb4ac3aSSrikar Dronamraju return *insn == UPROBE_SWBP_INSN; 174a5f4374aSIngo Molnar } 175a5f4374aSIngo Molnar 176cceb55aaSOleg Nesterov static void copy_opcode(struct page *page, unsigned long vaddr, uprobe_opcode_t *opcode) 177cceb55aaSOleg Nesterov { 178cceb55aaSOleg Nesterov void *kaddr = kmap_atomic(page); 179cceb55aaSOleg Nesterov memcpy(opcode, kaddr + (vaddr & ~PAGE_MASK), UPROBE_SWBP_INSN_SIZE); 180cceb55aaSOleg Nesterov kunmap_atomic(kaddr); 181cceb55aaSOleg Nesterov } 182cceb55aaSOleg Nesterov 183ed6f6a50SOleg Nesterov static int verify_opcode(struct page *page, unsigned long vaddr, uprobe_opcode_t *new_opcode) 184ed6f6a50SOleg Nesterov { 185ed6f6a50SOleg Nesterov uprobe_opcode_t old_opcode; 186ed6f6a50SOleg Nesterov bool is_swbp; 187ed6f6a50SOleg Nesterov 188ed6f6a50SOleg Nesterov copy_opcode(page, vaddr, &old_opcode); 189ed6f6a50SOleg Nesterov is_swbp = is_swbp_insn(&old_opcode); 190ed6f6a50SOleg Nesterov 191ed6f6a50SOleg Nesterov if (is_swbp_insn(new_opcode)) { 192ed6f6a50SOleg Nesterov if (is_swbp) /* register: already installed? */ 193ed6f6a50SOleg Nesterov return 0; 194ed6f6a50SOleg Nesterov } else { 195ed6f6a50SOleg Nesterov if (!is_swbp) /* unregister: was it changed by us? */ 196076a365bSOleg Nesterov return 0; 197ed6f6a50SOleg Nesterov } 198ed6f6a50SOleg Nesterov 199ed6f6a50SOleg Nesterov return 1; 200ed6f6a50SOleg Nesterov } 201ed6f6a50SOleg Nesterov 202a5f4374aSIngo Molnar /* 203a5f4374aSIngo Molnar * NOTE: 204a5f4374aSIngo Molnar * Expect the breakpoint instruction to be the smallest size instruction for 205a5f4374aSIngo Molnar * the architecture. If an arch has variable length instruction and the 206a5f4374aSIngo Molnar * breakpoint instruction is not of the smallest length instruction 207cceb55aaSOleg Nesterov * supported by that architecture then we need to modify is_swbp_at_addr and 208a5f4374aSIngo Molnar * write_opcode accordingly. This would never be a problem for archs that 209a5f4374aSIngo Molnar * have fixed length instructions. 210a5f4374aSIngo Molnar */ 211a5f4374aSIngo Molnar 212a5f4374aSIngo Molnar /* 213a5f4374aSIngo Molnar * write_opcode - write the opcode at a given virtual address. 214a5f4374aSIngo Molnar * @mm: the probed process address space. 215a5f4374aSIngo Molnar * @vaddr: the virtual address to store the opcode. 216a5f4374aSIngo Molnar * @opcode: opcode to be written at @vaddr. 217a5f4374aSIngo Molnar * 218a5f4374aSIngo Molnar * Called with mm->mmap_sem held (for read and with a reference to 219a5f4374aSIngo Molnar * mm). 220a5f4374aSIngo Molnar * 221a5f4374aSIngo Molnar * For mm @mm, write the opcode at @vaddr. 222a5f4374aSIngo Molnar * Return 0 (success) or a negative errno. 223a5f4374aSIngo Molnar */ 224cceb55aaSOleg Nesterov static int write_opcode(struct mm_struct *mm, unsigned long vaddr, 225cceb55aaSOleg Nesterov uprobe_opcode_t opcode) 226a5f4374aSIngo Molnar { 227a5f4374aSIngo Molnar struct page *old_page, *new_page; 228a5f4374aSIngo Molnar void *vaddr_old, *vaddr_new; 229a5f4374aSIngo Molnar struct vm_area_struct *vma; 230a5f4374aSIngo Molnar int ret; 231f403072cSOleg Nesterov 2325323ce71SOleg Nesterov retry: 233a5f4374aSIngo Molnar /* Read the page with vaddr into memory */ 23475ed82eaSOleg Nesterov ret = get_user_pages(NULL, mm, vaddr, 1, 0, 1, &old_page, &vma); 235a5f4374aSIngo Molnar if (ret <= 0) 236a5f4374aSIngo Molnar return ret; 237a5f4374aSIngo Molnar 238ed6f6a50SOleg Nesterov ret = verify_opcode(old_page, vaddr, &opcode); 239ed6f6a50SOleg Nesterov if (ret <= 0) 240ed6f6a50SOleg Nesterov goto put_old; 241ed6f6a50SOleg Nesterov 242a5f4374aSIngo Molnar ret = -ENOMEM; 243a5f4374aSIngo Molnar new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, vaddr); 244a5f4374aSIngo Molnar if (!new_page) 2459f92448cSOleg Nesterov goto put_old; 246a5f4374aSIngo Molnar 247a5f4374aSIngo Molnar __SetPageUptodate(new_page); 248a5f4374aSIngo Molnar 249a5f4374aSIngo Molnar /* copy the page now that we've got it stable */ 250a5f4374aSIngo Molnar vaddr_old = kmap_atomic(old_page); 251a5f4374aSIngo Molnar vaddr_new = kmap_atomic(new_page); 252a5f4374aSIngo Molnar 253a5f4374aSIngo Molnar memcpy(vaddr_new, vaddr_old, PAGE_SIZE); 254d9c4a30eSOleg Nesterov memcpy(vaddr_new + (vaddr & ~PAGE_MASK), &opcode, UPROBE_SWBP_INSN_SIZE); 255a5f4374aSIngo Molnar 256a5f4374aSIngo Molnar kunmap_atomic(vaddr_new); 257a5f4374aSIngo Molnar kunmap_atomic(vaddr_old); 258a5f4374aSIngo Molnar 259a5f4374aSIngo Molnar ret = anon_vma_prepare(vma); 260a5f4374aSIngo Molnar if (ret) 2619f92448cSOleg Nesterov goto put_new; 262a5f4374aSIngo Molnar 263c517ee74SOleg Nesterov ret = __replace_page(vma, vaddr, old_page, new_page); 264a5f4374aSIngo Molnar 2659f92448cSOleg Nesterov put_new: 266a5f4374aSIngo Molnar page_cache_release(new_page); 2679f92448cSOleg Nesterov put_old: 268a5f4374aSIngo Molnar put_page(old_page); 269a5f4374aSIngo Molnar 2705323ce71SOleg Nesterov if (unlikely(ret == -EAGAIN)) 2715323ce71SOleg Nesterov goto retry; 272a5f4374aSIngo Molnar return ret; 273a5f4374aSIngo Molnar } 274a5f4374aSIngo Molnar 275a5f4374aSIngo Molnar /** 2765cb4ac3aSSrikar Dronamraju * set_swbp - store breakpoint at a given address. 277e3343e6aSSrikar Dronamraju * @auprobe: arch specific probepoint information. 278a5f4374aSIngo Molnar * @mm: the probed process address space. 279a5f4374aSIngo Molnar * @vaddr: the virtual address to insert the opcode. 280a5f4374aSIngo Molnar * 281a5f4374aSIngo Molnar * For mm @mm, store the breakpoint instruction at @vaddr. 282a5f4374aSIngo Molnar * Return 0 (success) or a negative errno. 283a5f4374aSIngo Molnar */ 2845cb4ac3aSSrikar Dronamraju int __weak set_swbp(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned long vaddr) 285a5f4374aSIngo Molnar { 286cceb55aaSOleg Nesterov return write_opcode(mm, vaddr, UPROBE_SWBP_INSN); 287a5f4374aSIngo Molnar } 288a5f4374aSIngo Molnar 289a5f4374aSIngo Molnar /** 290a5f4374aSIngo Molnar * set_orig_insn - Restore the original instruction. 291a5f4374aSIngo Molnar * @mm: the probed process address space. 292e3343e6aSSrikar Dronamraju * @auprobe: arch specific probepoint information. 293a5f4374aSIngo Molnar * @vaddr: the virtual address to insert the opcode. 294a5f4374aSIngo Molnar * 295a5f4374aSIngo Molnar * For mm @mm, restore the original opcode (opcode) at @vaddr. 296a5f4374aSIngo Molnar * Return 0 (success) or a negative errno. 297a5f4374aSIngo Molnar */ 298a5f4374aSIngo Molnar int __weak 299ded86e7cSOleg Nesterov set_orig_insn(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned long vaddr) 300a5f4374aSIngo Molnar { 301cceb55aaSOleg Nesterov return write_opcode(mm, vaddr, *(uprobe_opcode_t *)auprobe->insn); 302a5f4374aSIngo Molnar } 303a5f4374aSIngo Molnar 304a5f4374aSIngo Molnar static int match_uprobe(struct uprobe *l, struct uprobe *r) 305a5f4374aSIngo Molnar { 306a5f4374aSIngo Molnar if (l->inode < r->inode) 307a5f4374aSIngo Molnar return -1; 308a5f4374aSIngo Molnar 309a5f4374aSIngo Molnar if (l->inode > r->inode) 310a5f4374aSIngo Molnar return 1; 311a5f4374aSIngo Molnar 312a5f4374aSIngo Molnar if (l->offset < r->offset) 313a5f4374aSIngo Molnar return -1; 314a5f4374aSIngo Molnar 315a5f4374aSIngo Molnar if (l->offset > r->offset) 316a5f4374aSIngo Molnar return 1; 317a5f4374aSIngo Molnar 318a5f4374aSIngo Molnar return 0; 319a5f4374aSIngo Molnar } 320a5f4374aSIngo Molnar 321a5f4374aSIngo Molnar static struct uprobe *__find_uprobe(struct inode *inode, loff_t offset) 322a5f4374aSIngo Molnar { 323a5f4374aSIngo Molnar struct uprobe u = { .inode = inode, .offset = offset }; 324a5f4374aSIngo Molnar struct rb_node *n = uprobes_tree.rb_node; 325a5f4374aSIngo Molnar struct uprobe *uprobe; 326a5f4374aSIngo Molnar int match; 327a5f4374aSIngo Molnar 328a5f4374aSIngo Molnar while (n) { 329a5f4374aSIngo Molnar uprobe = rb_entry(n, struct uprobe, rb_node); 330a5f4374aSIngo Molnar match = match_uprobe(&u, uprobe); 331a5f4374aSIngo Molnar if (!match) { 332a5f4374aSIngo Molnar atomic_inc(&uprobe->ref); 333a5f4374aSIngo Molnar return uprobe; 334a5f4374aSIngo Molnar } 335a5f4374aSIngo Molnar 336a5f4374aSIngo Molnar if (match < 0) 337a5f4374aSIngo Molnar n = n->rb_left; 338a5f4374aSIngo Molnar else 339a5f4374aSIngo Molnar n = n->rb_right; 340a5f4374aSIngo Molnar } 341a5f4374aSIngo Molnar return NULL; 342a5f4374aSIngo Molnar } 343a5f4374aSIngo Molnar 344a5f4374aSIngo Molnar /* 345a5f4374aSIngo Molnar * Find a uprobe corresponding to a given inode:offset 346a5f4374aSIngo Molnar * Acquires uprobes_treelock 347a5f4374aSIngo Molnar */ 348a5f4374aSIngo Molnar static struct uprobe *find_uprobe(struct inode *inode, loff_t offset) 349a5f4374aSIngo Molnar { 350a5f4374aSIngo Molnar struct uprobe *uprobe; 351a5f4374aSIngo Molnar 3526f47caa0SOleg Nesterov spin_lock(&uprobes_treelock); 353a5f4374aSIngo Molnar uprobe = __find_uprobe(inode, offset); 3546f47caa0SOleg Nesterov spin_unlock(&uprobes_treelock); 355a5f4374aSIngo Molnar 356a5f4374aSIngo Molnar return uprobe; 357a5f4374aSIngo Molnar } 358a5f4374aSIngo Molnar 359a5f4374aSIngo Molnar static struct uprobe *__insert_uprobe(struct uprobe *uprobe) 360a5f4374aSIngo Molnar { 361a5f4374aSIngo Molnar struct rb_node **p = &uprobes_tree.rb_node; 362a5f4374aSIngo Molnar struct rb_node *parent = NULL; 363a5f4374aSIngo Molnar struct uprobe *u; 364a5f4374aSIngo Molnar int match; 365a5f4374aSIngo Molnar 366a5f4374aSIngo Molnar while (*p) { 367a5f4374aSIngo Molnar parent = *p; 368a5f4374aSIngo Molnar u = rb_entry(parent, struct uprobe, rb_node); 369a5f4374aSIngo Molnar match = match_uprobe(uprobe, u); 370a5f4374aSIngo Molnar if (!match) { 371a5f4374aSIngo Molnar atomic_inc(&u->ref); 372a5f4374aSIngo Molnar return u; 373a5f4374aSIngo Molnar } 374a5f4374aSIngo Molnar 375a5f4374aSIngo Molnar if (match < 0) 376a5f4374aSIngo Molnar p = &parent->rb_left; 377a5f4374aSIngo Molnar else 378a5f4374aSIngo Molnar p = &parent->rb_right; 379a5f4374aSIngo Molnar 380a5f4374aSIngo Molnar } 381a5f4374aSIngo Molnar 382a5f4374aSIngo Molnar u = NULL; 383a5f4374aSIngo Molnar rb_link_node(&uprobe->rb_node, parent, p); 384a5f4374aSIngo Molnar rb_insert_color(&uprobe->rb_node, &uprobes_tree); 385a5f4374aSIngo Molnar /* get access + creation ref */ 386a5f4374aSIngo Molnar atomic_set(&uprobe->ref, 2); 387a5f4374aSIngo Molnar 388a5f4374aSIngo Molnar return u; 389a5f4374aSIngo Molnar } 390a5f4374aSIngo Molnar 391a5f4374aSIngo Molnar /* 392a5f4374aSIngo Molnar * Acquire uprobes_treelock. 393a5f4374aSIngo Molnar * Matching uprobe already exists in rbtree; 394a5f4374aSIngo Molnar * increment (access refcount) and return the matching uprobe. 395a5f4374aSIngo Molnar * 396a5f4374aSIngo Molnar * No matching uprobe; insert the uprobe in rb_tree; 397a5f4374aSIngo Molnar * get a double refcount (access + creation) and return NULL. 398a5f4374aSIngo Molnar */ 399a5f4374aSIngo Molnar static struct uprobe *insert_uprobe(struct uprobe *uprobe) 400a5f4374aSIngo Molnar { 401a5f4374aSIngo Molnar struct uprobe *u; 402a5f4374aSIngo Molnar 4036f47caa0SOleg Nesterov spin_lock(&uprobes_treelock); 404a5f4374aSIngo Molnar u = __insert_uprobe(uprobe); 4056f47caa0SOleg Nesterov spin_unlock(&uprobes_treelock); 406a5f4374aSIngo Molnar 407a5f4374aSIngo Molnar return u; 408a5f4374aSIngo Molnar } 409a5f4374aSIngo Molnar 410a5f4374aSIngo Molnar static void put_uprobe(struct uprobe *uprobe) 411a5f4374aSIngo Molnar { 412a5f4374aSIngo Molnar if (atomic_dec_and_test(&uprobe->ref)) 413a5f4374aSIngo Molnar kfree(uprobe); 414a5f4374aSIngo Molnar } 415a5f4374aSIngo Molnar 416a5f4374aSIngo Molnar static struct uprobe *alloc_uprobe(struct inode *inode, loff_t offset) 417a5f4374aSIngo Molnar { 418a5f4374aSIngo Molnar struct uprobe *uprobe, *cur_uprobe; 419a5f4374aSIngo Molnar 420a5f4374aSIngo Molnar uprobe = kzalloc(sizeof(struct uprobe), GFP_KERNEL); 421a5f4374aSIngo Molnar if (!uprobe) 422a5f4374aSIngo Molnar return NULL; 423a5f4374aSIngo Molnar 424a5f4374aSIngo Molnar uprobe->inode = igrab(inode); 425a5f4374aSIngo Molnar uprobe->offset = offset; 426e591c8d7SOleg Nesterov init_rwsem(&uprobe->register_rwsem); 427a5f4374aSIngo Molnar init_rwsem(&uprobe->consumer_rwsem); 428bbc33d05SOleg Nesterov /* For now assume that the instruction need not be single-stepped */ 429bbc33d05SOleg Nesterov __set_bit(UPROBE_SKIP_SSTEP, &uprobe->flags); 430a5f4374aSIngo Molnar 431a5f4374aSIngo Molnar /* add to uprobes_tree, sorted on inode:offset */ 432a5f4374aSIngo Molnar cur_uprobe = insert_uprobe(uprobe); 433a5f4374aSIngo Molnar 434a5f4374aSIngo Molnar /* a uprobe exists for this inode:offset combination */ 435a5f4374aSIngo Molnar if (cur_uprobe) { 436a5f4374aSIngo Molnar kfree(uprobe); 437a5f4374aSIngo Molnar uprobe = cur_uprobe; 438a5f4374aSIngo Molnar iput(inode); 439a5f4374aSIngo Molnar } 440a5f4374aSIngo Molnar 441a5f4374aSIngo Molnar return uprobe; 442a5f4374aSIngo Molnar } 443a5f4374aSIngo Molnar 4449a98e03cSOleg Nesterov static void consumer_add(struct uprobe *uprobe, struct uprobe_consumer *uc) 445a5f4374aSIngo Molnar { 446a5f4374aSIngo Molnar down_write(&uprobe->consumer_rwsem); 447e3343e6aSSrikar Dronamraju uc->next = uprobe->consumers; 448e3343e6aSSrikar Dronamraju uprobe->consumers = uc; 449a5f4374aSIngo Molnar up_write(&uprobe->consumer_rwsem); 450a5f4374aSIngo Molnar } 451a5f4374aSIngo Molnar 452a5f4374aSIngo Molnar /* 453e3343e6aSSrikar Dronamraju * For uprobe @uprobe, delete the consumer @uc. 454e3343e6aSSrikar Dronamraju * Return true if the @uc is deleted successfully 455a5f4374aSIngo Molnar * or return false. 456a5f4374aSIngo Molnar */ 457e3343e6aSSrikar Dronamraju static bool consumer_del(struct uprobe *uprobe, struct uprobe_consumer *uc) 458a5f4374aSIngo Molnar { 459a5f4374aSIngo Molnar struct uprobe_consumer **con; 460a5f4374aSIngo Molnar bool ret = false; 461a5f4374aSIngo Molnar 462a5f4374aSIngo Molnar down_write(&uprobe->consumer_rwsem); 463a5f4374aSIngo Molnar for (con = &uprobe->consumers; *con; con = &(*con)->next) { 464e3343e6aSSrikar Dronamraju if (*con == uc) { 465e3343e6aSSrikar Dronamraju *con = uc->next; 466a5f4374aSIngo Molnar ret = true; 467a5f4374aSIngo Molnar break; 468a5f4374aSIngo Molnar } 469a5f4374aSIngo Molnar } 470a5f4374aSIngo Molnar up_write(&uprobe->consumer_rwsem); 471a5f4374aSIngo Molnar 472a5f4374aSIngo Molnar return ret; 473a5f4374aSIngo Molnar } 474a5f4374aSIngo Molnar 475e3343e6aSSrikar Dronamraju static int 476d436615eSOleg Nesterov __copy_insn(struct address_space *mapping, struct file *filp, char *insn, 477593609a5SOleg Nesterov unsigned long nbytes, loff_t offset) 478a5f4374aSIngo Molnar { 479a5f4374aSIngo Molnar struct page *page; 480a5f4374aSIngo Molnar void *vaddr; 481593609a5SOleg Nesterov unsigned long off; 482593609a5SOleg Nesterov pgoff_t idx; 483a5f4374aSIngo Molnar 484a5f4374aSIngo Molnar if (!filp) 485a5f4374aSIngo Molnar return -EINVAL; 486a5f4374aSIngo Molnar 487cc359d18SOleg Nesterov if (!mapping->a_ops->readpage) 488cc359d18SOleg Nesterov return -EIO; 489cc359d18SOleg Nesterov 490593609a5SOleg Nesterov idx = offset >> PAGE_CACHE_SHIFT; 491593609a5SOleg Nesterov off = offset & ~PAGE_MASK; 492a5f4374aSIngo Molnar 493a5f4374aSIngo Molnar /* 494a5f4374aSIngo Molnar * Ensure that the page that has the original instruction is 495a5f4374aSIngo Molnar * populated and in page-cache. 496a5f4374aSIngo Molnar */ 497a5f4374aSIngo Molnar page = read_mapping_page(mapping, idx, filp); 498a5f4374aSIngo Molnar if (IS_ERR(page)) 499a5f4374aSIngo Molnar return PTR_ERR(page); 500a5f4374aSIngo Molnar 501a5f4374aSIngo Molnar vaddr = kmap_atomic(page); 502593609a5SOleg Nesterov memcpy(insn, vaddr + off, nbytes); 503a5f4374aSIngo Molnar kunmap_atomic(vaddr); 504a5f4374aSIngo Molnar page_cache_release(page); 505a5f4374aSIngo Molnar 506a5f4374aSIngo Molnar return 0; 507a5f4374aSIngo Molnar } 508a5f4374aSIngo Molnar 509d436615eSOleg Nesterov static int copy_insn(struct uprobe *uprobe, struct file *filp) 510a5f4374aSIngo Molnar { 511a5f4374aSIngo Molnar struct address_space *mapping; 512a5f4374aSIngo Molnar unsigned long nbytes; 513a5f4374aSIngo Molnar int bytes; 514a5f4374aSIngo Molnar 515d436615eSOleg Nesterov nbytes = PAGE_SIZE - (uprobe->offset & ~PAGE_MASK); 516a5f4374aSIngo Molnar mapping = uprobe->inode->i_mapping; 517a5f4374aSIngo Molnar 518a5f4374aSIngo Molnar /* Instruction at end of binary; copy only available bytes */ 519a5f4374aSIngo Molnar if (uprobe->offset + MAX_UINSN_BYTES > uprobe->inode->i_size) 520a5f4374aSIngo Molnar bytes = uprobe->inode->i_size - uprobe->offset; 521a5f4374aSIngo Molnar else 522a5f4374aSIngo Molnar bytes = MAX_UINSN_BYTES; 523a5f4374aSIngo Molnar 524a5f4374aSIngo Molnar /* Instruction at the page-boundary; copy bytes in second page */ 525a5f4374aSIngo Molnar if (nbytes < bytes) { 526fc36f595SOleg Nesterov int err = __copy_insn(mapping, filp, uprobe->arch.insn + nbytes, 527fc36f595SOleg Nesterov bytes - nbytes, uprobe->offset + nbytes); 528fc36f595SOleg Nesterov if (err) 529fc36f595SOleg Nesterov return err; 530a5f4374aSIngo Molnar bytes = nbytes; 531a5f4374aSIngo Molnar } 532d436615eSOleg Nesterov return __copy_insn(mapping, filp, uprobe->arch.insn, bytes, uprobe->offset); 533a5f4374aSIngo Molnar } 534a5f4374aSIngo Molnar 535cb9a19feSOleg Nesterov static int prepare_uprobe(struct uprobe *uprobe, struct file *file, 536cb9a19feSOleg Nesterov struct mm_struct *mm, unsigned long vaddr) 537cb9a19feSOleg Nesterov { 538cb9a19feSOleg Nesterov int ret = 0; 539cb9a19feSOleg Nesterov 54071434f2fSOleg Nesterov if (test_bit(UPROBE_COPY_INSN, &uprobe->flags)) 541cb9a19feSOleg Nesterov return ret; 542cb9a19feSOleg Nesterov 543d4d3ccc6SOleg Nesterov /* TODO: move this into _register, until then we abuse this sem. */ 544d4d3ccc6SOleg Nesterov down_write(&uprobe->consumer_rwsem); 54571434f2fSOleg Nesterov if (test_bit(UPROBE_COPY_INSN, &uprobe->flags)) 5464710f05fSOleg Nesterov goto out; 5474710f05fSOleg Nesterov 548cb9a19feSOleg Nesterov ret = copy_insn(uprobe, file); 549cb9a19feSOleg Nesterov if (ret) 550cb9a19feSOleg Nesterov goto out; 551cb9a19feSOleg Nesterov 552cb9a19feSOleg Nesterov ret = -ENOTSUPP; 553cb9a19feSOleg Nesterov if (is_swbp_insn((uprobe_opcode_t *)uprobe->arch.insn)) 554cb9a19feSOleg Nesterov goto out; 555cb9a19feSOleg Nesterov 556cb9a19feSOleg Nesterov ret = arch_uprobe_analyze_insn(&uprobe->arch, mm, vaddr); 557cb9a19feSOleg Nesterov if (ret) 558cb9a19feSOleg Nesterov goto out; 559cb9a19feSOleg Nesterov 560cb9a19feSOleg Nesterov /* write_opcode() assumes we don't cross page boundary */ 561cb9a19feSOleg Nesterov BUG_ON((uprobe->offset & ~PAGE_MASK) + 562cb9a19feSOleg Nesterov UPROBE_SWBP_INSN_SIZE > PAGE_SIZE); 563cb9a19feSOleg Nesterov 564cb9a19feSOleg Nesterov smp_wmb(); /* pairs with rmb() in find_active_uprobe() */ 56571434f2fSOleg Nesterov set_bit(UPROBE_COPY_INSN, &uprobe->flags); 566cb9a19feSOleg Nesterov 567cb9a19feSOleg Nesterov out: 568d4d3ccc6SOleg Nesterov up_write(&uprobe->consumer_rwsem); 5694710f05fSOleg Nesterov 570cb9a19feSOleg Nesterov return ret; 571cb9a19feSOleg Nesterov } 572cb9a19feSOleg Nesterov 5738a7f2fa0SOleg Nesterov static inline bool consumer_filter(struct uprobe_consumer *uc, 5748a7f2fa0SOleg Nesterov enum uprobe_filter_ctx ctx, struct mm_struct *mm) 575806a98bdSOleg Nesterov { 5768a7f2fa0SOleg Nesterov return !uc->filter || uc->filter(uc, ctx, mm); 577806a98bdSOleg Nesterov } 578806a98bdSOleg Nesterov 5798a7f2fa0SOleg Nesterov static bool filter_chain(struct uprobe *uprobe, 5808a7f2fa0SOleg Nesterov enum uprobe_filter_ctx ctx, struct mm_struct *mm) 58163633cbfSOleg Nesterov { 5821ff6fee5SOleg Nesterov struct uprobe_consumer *uc; 5831ff6fee5SOleg Nesterov bool ret = false; 5841ff6fee5SOleg Nesterov 5851ff6fee5SOleg Nesterov down_read(&uprobe->consumer_rwsem); 5861ff6fee5SOleg Nesterov for (uc = uprobe->consumers; uc; uc = uc->next) { 5878a7f2fa0SOleg Nesterov ret = consumer_filter(uc, ctx, mm); 5881ff6fee5SOleg Nesterov if (ret) 5891ff6fee5SOleg Nesterov break; 5901ff6fee5SOleg Nesterov } 5911ff6fee5SOleg Nesterov up_read(&uprobe->consumer_rwsem); 5921ff6fee5SOleg Nesterov 5931ff6fee5SOleg Nesterov return ret; 59463633cbfSOleg Nesterov } 59563633cbfSOleg Nesterov 596e3343e6aSSrikar Dronamraju static int 597e3343e6aSSrikar Dronamraju install_breakpoint(struct uprobe *uprobe, struct mm_struct *mm, 598816c03fbSOleg Nesterov struct vm_area_struct *vma, unsigned long vaddr) 599a5f4374aSIngo Molnar { 600f8ac4ec9SOleg Nesterov bool first_uprobe; 601a5f4374aSIngo Molnar int ret; 602a5f4374aSIngo Molnar 603cb9a19feSOleg Nesterov ret = prepare_uprobe(uprobe, vma->vm_file, mm, vaddr); 604a5f4374aSIngo Molnar if (ret) 605a5f4374aSIngo Molnar return ret; 606a5f4374aSIngo Molnar 607f8ac4ec9SOleg Nesterov /* 608f8ac4ec9SOleg Nesterov * set MMF_HAS_UPROBES in advance for uprobe_pre_sstep_notifier(), 609f8ac4ec9SOleg Nesterov * the task can hit this breakpoint right after __replace_page(). 610f8ac4ec9SOleg Nesterov */ 611f8ac4ec9SOleg Nesterov first_uprobe = !test_bit(MMF_HAS_UPROBES, &mm->flags); 612f8ac4ec9SOleg Nesterov if (first_uprobe) 613f8ac4ec9SOleg Nesterov set_bit(MMF_HAS_UPROBES, &mm->flags); 614f8ac4ec9SOleg Nesterov 615816c03fbSOleg Nesterov ret = set_swbp(&uprobe->arch, mm, vaddr); 6169f68f672SOleg Nesterov if (!ret) 6179f68f672SOleg Nesterov clear_bit(MMF_RECALC_UPROBES, &mm->flags); 6189f68f672SOleg Nesterov else if (first_uprobe) 619f8ac4ec9SOleg Nesterov clear_bit(MMF_HAS_UPROBES, &mm->flags); 620a5f4374aSIngo Molnar 621a5f4374aSIngo Molnar return ret; 622a5f4374aSIngo Molnar } 623a5f4374aSIngo Molnar 624076a365bSOleg Nesterov static int 625816c03fbSOleg Nesterov remove_breakpoint(struct uprobe *uprobe, struct mm_struct *mm, unsigned long vaddr) 626a5f4374aSIngo Molnar { 6279f68f672SOleg Nesterov set_bit(MMF_RECALC_UPROBES, &mm->flags); 628076a365bSOleg Nesterov return set_orig_insn(&uprobe->arch, mm, vaddr); 629a5f4374aSIngo Molnar } 630a5f4374aSIngo Molnar 63106b7bcd8SOleg Nesterov static inline bool uprobe_is_active(struct uprobe *uprobe) 63206b7bcd8SOleg Nesterov { 63306b7bcd8SOleg Nesterov return !RB_EMPTY_NODE(&uprobe->rb_node); 63406b7bcd8SOleg Nesterov } 6350326f5a9SSrikar Dronamraju /* 636778b032dSOleg Nesterov * There could be threads that have already hit the breakpoint. They 637778b032dSOleg Nesterov * will recheck the current insn and restart if find_uprobe() fails. 638778b032dSOleg Nesterov * See find_active_uprobe(). 6390326f5a9SSrikar Dronamraju */ 640a5f4374aSIngo Molnar static void delete_uprobe(struct uprobe *uprobe) 641a5f4374aSIngo Molnar { 64206b7bcd8SOleg Nesterov if (WARN_ON(!uprobe_is_active(uprobe))) 64306b7bcd8SOleg Nesterov return; 64406b7bcd8SOleg Nesterov 6456f47caa0SOleg Nesterov spin_lock(&uprobes_treelock); 646a5f4374aSIngo Molnar rb_erase(&uprobe->rb_node, &uprobes_tree); 6476f47caa0SOleg Nesterov spin_unlock(&uprobes_treelock); 64806b7bcd8SOleg Nesterov RB_CLEAR_NODE(&uprobe->rb_node); /* for uprobe_is_active() */ 649a5f4374aSIngo Molnar iput(uprobe->inode); 650a5f4374aSIngo Molnar put_uprobe(uprobe); 651a5f4374aSIngo Molnar } 652a5f4374aSIngo Molnar 65326872090SOleg Nesterov struct map_info { 65426872090SOleg Nesterov struct map_info *next; 65526872090SOleg Nesterov struct mm_struct *mm; 656816c03fbSOleg Nesterov unsigned long vaddr; 65726872090SOleg Nesterov }; 65826872090SOleg Nesterov 65926872090SOleg Nesterov static inline struct map_info *free_map_info(struct map_info *info) 660a5f4374aSIngo Molnar { 66126872090SOleg Nesterov struct map_info *next = info->next; 66226872090SOleg Nesterov kfree(info); 66326872090SOleg Nesterov return next; 66426872090SOleg Nesterov } 66526872090SOleg Nesterov 66626872090SOleg Nesterov static struct map_info * 66726872090SOleg Nesterov build_map_info(struct address_space *mapping, loff_t offset, bool is_register) 66826872090SOleg Nesterov { 66926872090SOleg Nesterov unsigned long pgoff = offset >> PAGE_SHIFT; 670a5f4374aSIngo Molnar struct vm_area_struct *vma; 67126872090SOleg Nesterov struct map_info *curr = NULL; 67226872090SOleg Nesterov struct map_info *prev = NULL; 67326872090SOleg Nesterov struct map_info *info; 67426872090SOleg Nesterov int more = 0; 675a5f4374aSIngo Molnar 67626872090SOleg Nesterov again: 67726872090SOleg Nesterov mutex_lock(&mapping->i_mmap_mutex); 6786b2dbba8SMichel Lespinasse vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) { 679a5f4374aSIngo Molnar if (!valid_vma(vma, is_register)) 680a5f4374aSIngo Molnar continue; 681a5f4374aSIngo Molnar 6827a5bfb66SOleg Nesterov if (!prev && !more) { 6837a5bfb66SOleg Nesterov /* 6847a5bfb66SOleg Nesterov * Needs GFP_NOWAIT to avoid i_mmap_mutex recursion through 6857a5bfb66SOleg Nesterov * reclaim. This is optimistic, no harm done if it fails. 6867a5bfb66SOleg Nesterov */ 6877a5bfb66SOleg Nesterov prev = kmalloc(sizeof(struct map_info), 6887a5bfb66SOleg Nesterov GFP_NOWAIT | __GFP_NOMEMALLOC | __GFP_NOWARN); 6897a5bfb66SOleg Nesterov if (prev) 6907a5bfb66SOleg Nesterov prev->next = NULL; 6917a5bfb66SOleg Nesterov } 69226872090SOleg Nesterov if (!prev) { 69326872090SOleg Nesterov more++; 69426872090SOleg Nesterov continue; 695a5f4374aSIngo Molnar } 696a5f4374aSIngo Molnar 69726872090SOleg Nesterov if (!atomic_inc_not_zero(&vma->vm_mm->mm_users)) 69826872090SOleg Nesterov continue; 699a5f4374aSIngo Molnar 70026872090SOleg Nesterov info = prev; 70126872090SOleg Nesterov prev = prev->next; 70226872090SOleg Nesterov info->next = curr; 70326872090SOleg Nesterov curr = info; 70426872090SOleg Nesterov 70526872090SOleg Nesterov info->mm = vma->vm_mm; 70657683f72SOleg Nesterov info->vaddr = offset_to_vaddr(vma, offset); 707a5f4374aSIngo Molnar } 708a5f4374aSIngo Molnar mutex_unlock(&mapping->i_mmap_mutex); 709a5f4374aSIngo Molnar 71026872090SOleg Nesterov if (!more) 71126872090SOleg Nesterov goto out; 712a5f4374aSIngo Molnar 71326872090SOleg Nesterov prev = curr; 71426872090SOleg Nesterov while (curr) { 71526872090SOleg Nesterov mmput(curr->mm); 71626872090SOleg Nesterov curr = curr->next; 71726872090SOleg Nesterov } 71826872090SOleg Nesterov 71926872090SOleg Nesterov do { 72026872090SOleg Nesterov info = kmalloc(sizeof(struct map_info), GFP_KERNEL); 72126872090SOleg Nesterov if (!info) { 72226872090SOleg Nesterov curr = ERR_PTR(-ENOMEM); 72326872090SOleg Nesterov goto out; 72426872090SOleg Nesterov } 72526872090SOleg Nesterov info->next = prev; 72626872090SOleg Nesterov prev = info; 72726872090SOleg Nesterov } while (--more); 72826872090SOleg Nesterov 72926872090SOleg Nesterov goto again; 73026872090SOleg Nesterov out: 73126872090SOleg Nesterov while (prev) 73226872090SOleg Nesterov prev = free_map_info(prev); 73326872090SOleg Nesterov return curr; 734a5f4374aSIngo Molnar } 735a5f4374aSIngo Molnar 736a5f4374aSIngo Molnar static int register_for_each_vma(struct uprobe *uprobe, bool is_register) 737a5f4374aSIngo Molnar { 73826872090SOleg Nesterov struct map_info *info; 73926872090SOleg Nesterov int err = 0; 74026872090SOleg Nesterov 74132cdba1eSOleg Nesterov percpu_down_write(&dup_mmap_sem); 74226872090SOleg Nesterov info = build_map_info(uprobe->inode->i_mapping, 74326872090SOleg Nesterov uprobe->offset, is_register); 74432cdba1eSOleg Nesterov if (IS_ERR(info)) { 74532cdba1eSOleg Nesterov err = PTR_ERR(info); 74632cdba1eSOleg Nesterov goto out; 74732cdba1eSOleg Nesterov } 74826872090SOleg Nesterov 74926872090SOleg Nesterov while (info) { 75026872090SOleg Nesterov struct mm_struct *mm = info->mm; 751a5f4374aSIngo Molnar struct vm_area_struct *vma; 752a5f4374aSIngo Molnar 753076a365bSOleg Nesterov if (err && is_register) 75426872090SOleg Nesterov goto free; 755a5f4374aSIngo Molnar 75677fc4af1SOleg Nesterov down_write(&mm->mmap_sem); 757f4d6dfe5SOleg Nesterov vma = find_vma(mm, info->vaddr); 758f4d6dfe5SOleg Nesterov if (!vma || !valid_vma(vma, is_register) || 759f4d6dfe5SOleg Nesterov vma->vm_file->f_mapping->host != uprobe->inode) 76026872090SOleg Nesterov goto unlock; 76126872090SOleg Nesterov 762f4d6dfe5SOleg Nesterov if (vma->vm_start > info->vaddr || 763f4d6dfe5SOleg Nesterov vaddr_to_offset(vma, info->vaddr) != uprobe->offset) 76426872090SOleg Nesterov goto unlock; 765a5f4374aSIngo Molnar 766806a98bdSOleg Nesterov if (is_register) { 767806a98bdSOleg Nesterov /* consult only the "caller", new consumer. */ 7688a7f2fa0SOleg Nesterov if (consumer_filter(uprobe->consumers, 7698a7f2fa0SOleg Nesterov UPROBE_FILTER_REGISTER, mm)) 77026872090SOleg Nesterov err = install_breakpoint(uprobe, mm, vma, info->vaddr); 771806a98bdSOleg Nesterov } else if (test_bit(MMF_HAS_UPROBES, &mm->flags)) { 7728a7f2fa0SOleg Nesterov if (!filter_chain(uprobe, 7738a7f2fa0SOleg Nesterov UPROBE_FILTER_UNREGISTER, mm)) 774076a365bSOleg Nesterov err |= remove_breakpoint(uprobe, mm, info->vaddr); 775806a98bdSOleg Nesterov } 77678f74116SOleg Nesterov 77726872090SOleg Nesterov unlock: 77826872090SOleg Nesterov up_write(&mm->mmap_sem); 77926872090SOleg Nesterov free: 78026872090SOleg Nesterov mmput(mm); 78126872090SOleg Nesterov info = free_map_info(info); 782a5f4374aSIngo Molnar } 78332cdba1eSOleg Nesterov out: 78432cdba1eSOleg Nesterov percpu_up_write(&dup_mmap_sem); 78526872090SOleg Nesterov return err; 786a5f4374aSIngo Molnar } 787a5f4374aSIngo Molnar 7889a98e03cSOleg Nesterov static int __uprobe_register(struct uprobe *uprobe, struct uprobe_consumer *uc) 789a5f4374aSIngo Molnar { 7909a98e03cSOleg Nesterov consumer_add(uprobe, uc); 791bb929284SOleg Nesterov return register_for_each_vma(uprobe, true); 792a5f4374aSIngo Molnar } 793a5f4374aSIngo Molnar 79404aab9b2SOleg Nesterov static void __uprobe_unregister(struct uprobe *uprobe, struct uprobe_consumer *uc) 795a5f4374aSIngo Molnar { 79604aab9b2SOleg Nesterov int err; 797a5f4374aSIngo Molnar 79804aab9b2SOleg Nesterov if (!consumer_del(uprobe, uc)) /* WARN? */ 79904aab9b2SOleg Nesterov return; 80004aab9b2SOleg Nesterov 80104aab9b2SOleg Nesterov err = register_for_each_vma(uprobe, false); 802a5f4374aSIngo Molnar /* TODO : cant unregister? schedule a worker thread */ 803bb929284SOleg Nesterov if (!uprobe->consumers && !err) 80404aab9b2SOleg Nesterov delete_uprobe(uprobe); 80504aab9b2SOleg Nesterov } 806a5f4374aSIngo Molnar 807a5f4374aSIngo Molnar /* 808a5f4374aSIngo Molnar * uprobe_register - register a probe 809a5f4374aSIngo Molnar * @inode: the file in which the probe has to be placed. 810a5f4374aSIngo Molnar * @offset: offset from the start of the file. 811e3343e6aSSrikar Dronamraju * @uc: information on howto handle the probe.. 812a5f4374aSIngo Molnar * 813a5f4374aSIngo Molnar * Apart from the access refcount, uprobe_register() takes a creation 814a5f4374aSIngo Molnar * refcount (thro alloc_uprobe) if and only if this @uprobe is getting 815a5f4374aSIngo Molnar * inserted into the rbtree (i.e first consumer for a @inode:@offset 816a5f4374aSIngo Molnar * tuple). Creation refcount stops uprobe_unregister from freeing the 817a5f4374aSIngo Molnar * @uprobe even before the register operation is complete. Creation 818e3343e6aSSrikar Dronamraju * refcount is released when the last @uc for the @uprobe 819a5f4374aSIngo Molnar * unregisters. 820a5f4374aSIngo Molnar * 821a5f4374aSIngo Molnar * Return errno if it cannot successully install probes 822a5f4374aSIngo Molnar * else return 0 (success) 823a5f4374aSIngo Molnar */ 824e3343e6aSSrikar Dronamraju int uprobe_register(struct inode *inode, loff_t offset, struct uprobe_consumer *uc) 825a5f4374aSIngo Molnar { 826a5f4374aSIngo Molnar struct uprobe *uprobe; 827a5f4374aSIngo Molnar int ret; 828a5f4374aSIngo Molnar 829f0744af7SOleg Nesterov /* Racy, just to catch the obvious mistakes */ 830a5f4374aSIngo Molnar if (offset > i_size_read(inode)) 831a5f4374aSIngo Molnar return -EINVAL; 832a5f4374aSIngo Molnar 83366d06dffSOleg Nesterov retry: 834a5f4374aSIngo Molnar uprobe = alloc_uprobe(inode, offset); 83566d06dffSOleg Nesterov if (!uprobe) 83666d06dffSOleg Nesterov return -ENOMEM; 83766d06dffSOleg Nesterov /* 83866d06dffSOleg Nesterov * We can race with uprobe_unregister()->delete_uprobe(). 83966d06dffSOleg Nesterov * Check uprobe_is_active() and retry if it is false. 84066d06dffSOleg Nesterov */ 841e591c8d7SOleg Nesterov down_write(&uprobe->register_rwsem); 84266d06dffSOleg Nesterov ret = -EAGAIN; 84366d06dffSOleg Nesterov if (likely(uprobe_is_active(uprobe))) { 8449a98e03cSOleg Nesterov ret = __uprobe_register(uprobe, uc); 8459a98e03cSOleg Nesterov if (ret) 84604aab9b2SOleg Nesterov __uprobe_unregister(uprobe, uc); 847a5f4374aSIngo Molnar } 84866d06dffSOleg Nesterov up_write(&uprobe->register_rwsem); 849a5f4374aSIngo Molnar put_uprobe(uprobe); 850a5f4374aSIngo Molnar 85166d06dffSOleg Nesterov if (unlikely(ret == -EAGAIN)) 85266d06dffSOleg Nesterov goto retry; 853a5f4374aSIngo Molnar return ret; 854a5f4374aSIngo Molnar } 855e8440c14SJosh Stone EXPORT_SYMBOL_GPL(uprobe_register); 856a5f4374aSIngo Molnar 857a5f4374aSIngo Molnar /* 858a5f4374aSIngo Molnar * uprobe_unregister - unregister a already registered probe. 859a5f4374aSIngo Molnar * @inode: the file in which the probe has to be removed. 860a5f4374aSIngo Molnar * @offset: offset from the start of the file. 861e3343e6aSSrikar Dronamraju * @uc: identify which probe if multiple probes are colocated. 862a5f4374aSIngo Molnar */ 863e3343e6aSSrikar Dronamraju void uprobe_unregister(struct inode *inode, loff_t offset, struct uprobe_consumer *uc) 864a5f4374aSIngo Molnar { 865a5f4374aSIngo Molnar struct uprobe *uprobe; 866a5f4374aSIngo Molnar 867a5f4374aSIngo Molnar uprobe = find_uprobe(inode, offset); 868a5f4374aSIngo Molnar if (!uprobe) 869a5f4374aSIngo Molnar return; 870a5f4374aSIngo Molnar 871e591c8d7SOleg Nesterov down_write(&uprobe->register_rwsem); 87204aab9b2SOleg Nesterov __uprobe_unregister(uprobe, uc); 873e591c8d7SOleg Nesterov up_write(&uprobe->register_rwsem); 874a5f4374aSIngo Molnar put_uprobe(uprobe); 875a5f4374aSIngo Molnar } 876e8440c14SJosh Stone EXPORT_SYMBOL_GPL(uprobe_unregister); 877a5f4374aSIngo Molnar 878da1816b1SOleg Nesterov static int unapply_uprobe(struct uprobe *uprobe, struct mm_struct *mm) 879da1816b1SOleg Nesterov { 880da1816b1SOleg Nesterov struct vm_area_struct *vma; 881da1816b1SOleg Nesterov int err = 0; 882da1816b1SOleg Nesterov 883da1816b1SOleg Nesterov down_read(&mm->mmap_sem); 884da1816b1SOleg Nesterov for (vma = mm->mmap; vma; vma = vma->vm_next) { 885da1816b1SOleg Nesterov unsigned long vaddr; 886da1816b1SOleg Nesterov loff_t offset; 887da1816b1SOleg Nesterov 888da1816b1SOleg Nesterov if (!valid_vma(vma, false) || 889da1816b1SOleg Nesterov vma->vm_file->f_mapping->host != uprobe->inode) 890da1816b1SOleg Nesterov continue; 891da1816b1SOleg Nesterov 892da1816b1SOleg Nesterov offset = (loff_t)vma->vm_pgoff << PAGE_SHIFT; 893da1816b1SOleg Nesterov if (uprobe->offset < offset || 894da1816b1SOleg Nesterov uprobe->offset >= offset + vma->vm_end - vma->vm_start) 895da1816b1SOleg Nesterov continue; 896da1816b1SOleg Nesterov 897da1816b1SOleg Nesterov vaddr = offset_to_vaddr(vma, uprobe->offset); 898da1816b1SOleg Nesterov err |= remove_breakpoint(uprobe, mm, vaddr); 899da1816b1SOleg Nesterov } 900da1816b1SOleg Nesterov up_read(&mm->mmap_sem); 901da1816b1SOleg Nesterov 902da1816b1SOleg Nesterov return err; 903da1816b1SOleg Nesterov } 904da1816b1SOleg Nesterov 905891c3970SOleg Nesterov static struct rb_node * 906891c3970SOleg Nesterov find_node_in_range(struct inode *inode, loff_t min, loff_t max) 907a5f4374aSIngo Molnar { 908a5f4374aSIngo Molnar struct rb_node *n = uprobes_tree.rb_node; 909a5f4374aSIngo Molnar 910a5f4374aSIngo Molnar while (n) { 911891c3970SOleg Nesterov struct uprobe *u = rb_entry(n, struct uprobe, rb_node); 912a5f4374aSIngo Molnar 913891c3970SOleg Nesterov if (inode < u->inode) { 914a5f4374aSIngo Molnar n = n->rb_left; 915891c3970SOleg Nesterov } else if (inode > u->inode) { 916a5f4374aSIngo Molnar n = n->rb_right; 917891c3970SOleg Nesterov } else { 918891c3970SOleg Nesterov if (max < u->offset) 919891c3970SOleg Nesterov n = n->rb_left; 920891c3970SOleg Nesterov else if (min > u->offset) 921891c3970SOleg Nesterov n = n->rb_right; 922891c3970SOleg Nesterov else 923891c3970SOleg Nesterov break; 924891c3970SOleg Nesterov } 925a5f4374aSIngo Molnar } 926a5f4374aSIngo Molnar 927891c3970SOleg Nesterov return n; 928a5f4374aSIngo Molnar } 929a5f4374aSIngo Molnar 930a5f4374aSIngo Molnar /* 931891c3970SOleg Nesterov * For a given range in vma, build a list of probes that need to be inserted. 932a5f4374aSIngo Molnar */ 933891c3970SOleg Nesterov static void build_probe_list(struct inode *inode, 934891c3970SOleg Nesterov struct vm_area_struct *vma, 935891c3970SOleg Nesterov unsigned long start, unsigned long end, 936891c3970SOleg Nesterov struct list_head *head) 937a5f4374aSIngo Molnar { 938891c3970SOleg Nesterov loff_t min, max; 939891c3970SOleg Nesterov struct rb_node *n, *t; 940891c3970SOleg Nesterov struct uprobe *u; 941891c3970SOleg Nesterov 942891c3970SOleg Nesterov INIT_LIST_HEAD(head); 943cb113b47SOleg Nesterov min = vaddr_to_offset(vma, start); 944891c3970SOleg Nesterov max = min + (end - start) - 1; 945a5f4374aSIngo Molnar 9466f47caa0SOleg Nesterov spin_lock(&uprobes_treelock); 947891c3970SOleg Nesterov n = find_node_in_range(inode, min, max); 948891c3970SOleg Nesterov if (n) { 949891c3970SOleg Nesterov for (t = n; t; t = rb_prev(t)) { 950891c3970SOleg Nesterov u = rb_entry(t, struct uprobe, rb_node); 951891c3970SOleg Nesterov if (u->inode != inode || u->offset < min) 952a5f4374aSIngo Molnar break; 953891c3970SOleg Nesterov list_add(&u->pending_list, head); 954891c3970SOleg Nesterov atomic_inc(&u->ref); 955a5f4374aSIngo Molnar } 956891c3970SOleg Nesterov for (t = n; (t = rb_next(t)); ) { 957891c3970SOleg Nesterov u = rb_entry(t, struct uprobe, rb_node); 958891c3970SOleg Nesterov if (u->inode != inode || u->offset > max) 959891c3970SOleg Nesterov break; 960891c3970SOleg Nesterov list_add(&u->pending_list, head); 961891c3970SOleg Nesterov atomic_inc(&u->ref); 962891c3970SOleg Nesterov } 963891c3970SOleg Nesterov } 9646f47caa0SOleg Nesterov spin_unlock(&uprobes_treelock); 965a5f4374aSIngo Molnar } 966a5f4374aSIngo Molnar 967a5f4374aSIngo Molnar /* 9685e5be71aSOleg Nesterov * Called from mmap_region/vma_adjust with mm->mmap_sem acquired. 969a5f4374aSIngo Molnar * 9705e5be71aSOleg Nesterov * Currently we ignore all errors and always return 0, the callers 9715e5be71aSOleg Nesterov * can't handle the failure anyway. 972a5f4374aSIngo Molnar */ 973a5f4374aSIngo Molnar int uprobe_mmap(struct vm_area_struct *vma) 974a5f4374aSIngo Molnar { 975a5f4374aSIngo Molnar struct list_head tmp_list; 976665605a2SOleg Nesterov struct uprobe *uprobe, *u; 977a5f4374aSIngo Molnar struct inode *inode; 978a5f4374aSIngo Molnar 979441f1eb7SOleg Nesterov if (no_uprobe_events() || !valid_vma(vma, true)) 980a5f4374aSIngo Molnar return 0; 981a5f4374aSIngo Molnar 982a5f4374aSIngo Molnar inode = vma->vm_file->f_mapping->host; 983a5f4374aSIngo Molnar if (!inode) 984a5f4374aSIngo Molnar return 0; 985a5f4374aSIngo Molnar 986a5f4374aSIngo Molnar mutex_lock(uprobes_mmap_hash(inode)); 987891c3970SOleg Nesterov build_probe_list(inode, vma, vma->vm_start, vma->vm_end, &tmp_list); 988806a98bdSOleg Nesterov /* 989806a98bdSOleg Nesterov * We can race with uprobe_unregister(), this uprobe can be already 990806a98bdSOleg Nesterov * removed. But in this case filter_chain() must return false, all 991806a98bdSOleg Nesterov * consumers have gone away. 992806a98bdSOleg Nesterov */ 993665605a2SOleg Nesterov list_for_each_entry_safe(uprobe, u, &tmp_list, pending_list) { 994806a98bdSOleg Nesterov if (!fatal_signal_pending(current) && 9958a7f2fa0SOleg Nesterov filter_chain(uprobe, UPROBE_FILTER_MMAP, vma->vm_mm)) { 99657683f72SOleg Nesterov unsigned long vaddr = offset_to_vaddr(vma, uprobe->offset); 9975e5be71aSOleg Nesterov install_breakpoint(uprobe, vma->vm_mm, vma, vaddr); 998a5f4374aSIngo Molnar } 999a5f4374aSIngo Molnar put_uprobe(uprobe); 1000a5f4374aSIngo Molnar } 1001a5f4374aSIngo Molnar mutex_unlock(uprobes_mmap_hash(inode)); 1002a5f4374aSIngo Molnar 10035e5be71aSOleg Nesterov return 0; 1004a5f4374aSIngo Molnar } 1005a5f4374aSIngo Molnar 10069f68f672SOleg Nesterov static bool 10079f68f672SOleg Nesterov vma_has_uprobes(struct vm_area_struct *vma, unsigned long start, unsigned long end) 10089f68f672SOleg Nesterov { 10099f68f672SOleg Nesterov loff_t min, max; 10109f68f672SOleg Nesterov struct inode *inode; 10119f68f672SOleg Nesterov struct rb_node *n; 10129f68f672SOleg Nesterov 10139f68f672SOleg Nesterov inode = vma->vm_file->f_mapping->host; 10149f68f672SOleg Nesterov 10159f68f672SOleg Nesterov min = vaddr_to_offset(vma, start); 10169f68f672SOleg Nesterov max = min + (end - start) - 1; 10179f68f672SOleg Nesterov 10189f68f672SOleg Nesterov spin_lock(&uprobes_treelock); 10199f68f672SOleg Nesterov n = find_node_in_range(inode, min, max); 10209f68f672SOleg Nesterov spin_unlock(&uprobes_treelock); 10219f68f672SOleg Nesterov 10229f68f672SOleg Nesterov return !!n; 10239f68f672SOleg Nesterov } 10249f68f672SOleg Nesterov 1025682968e0SSrikar Dronamraju /* 1026682968e0SSrikar Dronamraju * Called in context of a munmap of a vma. 1027682968e0SSrikar Dronamraju */ 1028cbc91f71SSrikar Dronamraju void uprobe_munmap(struct vm_area_struct *vma, unsigned long start, unsigned long end) 1029682968e0SSrikar Dronamraju { 1030441f1eb7SOleg Nesterov if (no_uprobe_events() || !valid_vma(vma, false)) 1031682968e0SSrikar Dronamraju return; 1032682968e0SSrikar Dronamraju 10332fd611a9SOleg Nesterov if (!atomic_read(&vma->vm_mm->mm_users)) /* called by mmput() ? */ 10342fd611a9SOleg Nesterov return; 10352fd611a9SOleg Nesterov 10369f68f672SOleg Nesterov if (!test_bit(MMF_HAS_UPROBES, &vma->vm_mm->flags) || 10379f68f672SOleg Nesterov test_bit(MMF_RECALC_UPROBES, &vma->vm_mm->flags)) 1038f8ac4ec9SOleg Nesterov return; 1039f8ac4ec9SOleg Nesterov 10409f68f672SOleg Nesterov if (vma_has_uprobes(vma, start, end)) 10419f68f672SOleg Nesterov set_bit(MMF_RECALC_UPROBES, &vma->vm_mm->flags); 1042682968e0SSrikar Dronamraju } 1043682968e0SSrikar Dronamraju 1044d4b3b638SSrikar Dronamraju /* Slot allocation for XOL */ 1045d4b3b638SSrikar Dronamraju static int xol_add_vma(struct xol_area *area) 1046d4b3b638SSrikar Dronamraju { 1047c8a82538SOleg Nesterov struct mm_struct *mm = current->mm; 1048c8a82538SOleg Nesterov int ret = -EALREADY; 1049d4b3b638SSrikar Dronamraju 1050d4b3b638SSrikar Dronamraju down_write(&mm->mmap_sem); 1051d4b3b638SSrikar Dronamraju if (mm->uprobes_state.xol_area) 1052d4b3b638SSrikar Dronamraju goto fail; 1053d4b3b638SSrikar Dronamraju 1054d4b3b638SSrikar Dronamraju ret = -ENOMEM; 1055d4b3b638SSrikar Dronamraju /* Try to map as high as possible, this is only a hint. */ 1056d4b3b638SSrikar Dronamraju area->vaddr = get_unmapped_area(NULL, TASK_SIZE - PAGE_SIZE, PAGE_SIZE, 0, 0); 1057d4b3b638SSrikar Dronamraju if (area->vaddr & ~PAGE_MASK) { 1058d4b3b638SSrikar Dronamraju ret = area->vaddr; 1059d4b3b638SSrikar Dronamraju goto fail; 1060d4b3b638SSrikar Dronamraju } 1061d4b3b638SSrikar Dronamraju 1062d4b3b638SSrikar Dronamraju ret = install_special_mapping(mm, area->vaddr, PAGE_SIZE, 1063d4b3b638SSrikar Dronamraju VM_EXEC|VM_MAYEXEC|VM_DONTCOPY|VM_IO, &area->page); 1064d4b3b638SSrikar Dronamraju if (ret) 1065d4b3b638SSrikar Dronamraju goto fail; 1066d4b3b638SSrikar Dronamraju 1067d4b3b638SSrikar Dronamraju smp_wmb(); /* pairs with get_xol_area() */ 1068d4b3b638SSrikar Dronamraju mm->uprobes_state.xol_area = area; 1069d4b3b638SSrikar Dronamraju ret = 0; 1070d4b3b638SSrikar Dronamraju fail: 1071d4b3b638SSrikar Dronamraju up_write(&mm->mmap_sem); 1072d4b3b638SSrikar Dronamraju 1073d4b3b638SSrikar Dronamraju return ret; 1074d4b3b638SSrikar Dronamraju } 1075d4b3b638SSrikar Dronamraju 1076d4b3b638SSrikar Dronamraju /* 10779b545df8SOleg Nesterov * get_xol_area - Allocate process's xol_area if necessary. 10789b545df8SOleg Nesterov * This area will be used for storing instructions for execution out of line. 1079d4b3b638SSrikar Dronamraju * 1080d4b3b638SSrikar Dronamraju * Returns the allocated area or NULL. 1081d4b3b638SSrikar Dronamraju */ 10829b545df8SOleg Nesterov static struct xol_area *get_xol_area(void) 1083d4b3b638SSrikar Dronamraju { 10849b545df8SOleg Nesterov struct mm_struct *mm = current->mm; 1085d4b3b638SSrikar Dronamraju struct xol_area *area; 1086d4b3b638SSrikar Dronamraju 10879b545df8SOleg Nesterov area = mm->uprobes_state.xol_area; 10889b545df8SOleg Nesterov if (area) 10899b545df8SOleg Nesterov goto ret; 10909b545df8SOleg Nesterov 1091d4b3b638SSrikar Dronamraju area = kzalloc(sizeof(*area), GFP_KERNEL); 1092d4b3b638SSrikar Dronamraju if (unlikely(!area)) 1093c8a82538SOleg Nesterov goto out; 1094d4b3b638SSrikar Dronamraju 1095d4b3b638SSrikar Dronamraju area->bitmap = kzalloc(BITS_TO_LONGS(UINSNS_PER_PAGE) * sizeof(long), GFP_KERNEL); 1096d4b3b638SSrikar Dronamraju if (!area->bitmap) 1097c8a82538SOleg Nesterov goto free_area; 1098c8a82538SOleg Nesterov 1099c8a82538SOleg Nesterov area->page = alloc_page(GFP_HIGHUSER); 1100c8a82538SOleg Nesterov if (!area->page) 1101c8a82538SOleg Nesterov goto free_bitmap; 1102d4b3b638SSrikar Dronamraju 1103d4b3b638SSrikar Dronamraju init_waitqueue_head(&area->wq); 1104d4b3b638SSrikar Dronamraju if (!xol_add_vma(area)) 1105d4b3b638SSrikar Dronamraju return area; 1106d4b3b638SSrikar Dronamraju 1107c8a82538SOleg Nesterov __free_page(area->page); 1108c8a82538SOleg Nesterov free_bitmap: 1109d4b3b638SSrikar Dronamraju kfree(area->bitmap); 1110c8a82538SOleg Nesterov free_area: 1111d4b3b638SSrikar Dronamraju kfree(area); 1112c8a82538SOleg Nesterov out: 11139b545df8SOleg Nesterov area = mm->uprobes_state.xol_area; 11149b545df8SOleg Nesterov ret: 11159b545df8SOleg Nesterov smp_read_barrier_depends(); /* pairs with wmb in xol_add_vma() */ 11169b545df8SOleg Nesterov return area; 1117d4b3b638SSrikar Dronamraju } 1118d4b3b638SSrikar Dronamraju 1119d4b3b638SSrikar Dronamraju /* 1120d4b3b638SSrikar Dronamraju * uprobe_clear_state - Free the area allocated for slots. 1121d4b3b638SSrikar Dronamraju */ 1122d4b3b638SSrikar Dronamraju void uprobe_clear_state(struct mm_struct *mm) 1123d4b3b638SSrikar Dronamraju { 1124d4b3b638SSrikar Dronamraju struct xol_area *area = mm->uprobes_state.xol_area; 1125d4b3b638SSrikar Dronamraju 1126d4b3b638SSrikar Dronamraju if (!area) 1127d4b3b638SSrikar Dronamraju return; 1128d4b3b638SSrikar Dronamraju 1129d4b3b638SSrikar Dronamraju put_page(area->page); 1130d4b3b638SSrikar Dronamraju kfree(area->bitmap); 1131d4b3b638SSrikar Dronamraju kfree(area); 1132d4b3b638SSrikar Dronamraju } 1133d4b3b638SSrikar Dronamraju 113432cdba1eSOleg Nesterov void uprobe_start_dup_mmap(void) 113532cdba1eSOleg Nesterov { 113632cdba1eSOleg Nesterov percpu_down_read(&dup_mmap_sem); 113732cdba1eSOleg Nesterov } 113832cdba1eSOleg Nesterov 113932cdba1eSOleg Nesterov void uprobe_end_dup_mmap(void) 114032cdba1eSOleg Nesterov { 114132cdba1eSOleg Nesterov percpu_up_read(&dup_mmap_sem); 114232cdba1eSOleg Nesterov } 114332cdba1eSOleg Nesterov 1144f8ac4ec9SOleg Nesterov void uprobe_dup_mmap(struct mm_struct *oldmm, struct mm_struct *newmm) 1145f8ac4ec9SOleg Nesterov { 114661559a81SOleg Nesterov newmm->uprobes_state.xol_area = NULL; 114761559a81SOleg Nesterov 11489f68f672SOleg Nesterov if (test_bit(MMF_HAS_UPROBES, &oldmm->flags)) { 1149f8ac4ec9SOleg Nesterov set_bit(MMF_HAS_UPROBES, &newmm->flags); 11509f68f672SOleg Nesterov /* unconditionally, dup_mmap() skips VM_DONTCOPY vmas */ 11519f68f672SOleg Nesterov set_bit(MMF_RECALC_UPROBES, &newmm->flags); 11529f68f672SOleg Nesterov } 1153f8ac4ec9SOleg Nesterov } 1154f8ac4ec9SOleg Nesterov 1155d4b3b638SSrikar Dronamraju /* 1156d4b3b638SSrikar Dronamraju * - search for a free slot. 1157d4b3b638SSrikar Dronamraju */ 1158d4b3b638SSrikar Dronamraju static unsigned long xol_take_insn_slot(struct xol_area *area) 1159d4b3b638SSrikar Dronamraju { 1160d4b3b638SSrikar Dronamraju unsigned long slot_addr; 1161d4b3b638SSrikar Dronamraju int slot_nr; 1162d4b3b638SSrikar Dronamraju 1163d4b3b638SSrikar Dronamraju do { 1164d4b3b638SSrikar Dronamraju slot_nr = find_first_zero_bit(area->bitmap, UINSNS_PER_PAGE); 1165d4b3b638SSrikar Dronamraju if (slot_nr < UINSNS_PER_PAGE) { 1166d4b3b638SSrikar Dronamraju if (!test_and_set_bit(slot_nr, area->bitmap)) 1167d4b3b638SSrikar Dronamraju break; 1168d4b3b638SSrikar Dronamraju 1169d4b3b638SSrikar Dronamraju slot_nr = UINSNS_PER_PAGE; 1170d4b3b638SSrikar Dronamraju continue; 1171d4b3b638SSrikar Dronamraju } 1172d4b3b638SSrikar Dronamraju wait_event(area->wq, (atomic_read(&area->slot_count) < UINSNS_PER_PAGE)); 1173d4b3b638SSrikar Dronamraju } while (slot_nr >= UINSNS_PER_PAGE); 1174d4b3b638SSrikar Dronamraju 1175d4b3b638SSrikar Dronamraju slot_addr = area->vaddr + (slot_nr * UPROBE_XOL_SLOT_BYTES); 1176d4b3b638SSrikar Dronamraju atomic_inc(&area->slot_count); 1177d4b3b638SSrikar Dronamraju 1178d4b3b638SSrikar Dronamraju return slot_addr; 1179d4b3b638SSrikar Dronamraju } 1180d4b3b638SSrikar Dronamraju 1181d4b3b638SSrikar Dronamraju /* 1182a6cb3f6dSOleg Nesterov * xol_get_insn_slot - allocate a slot for xol. 1183d4b3b638SSrikar Dronamraju * Returns the allocated slot address or 0. 1184d4b3b638SSrikar Dronamraju */ 1185a6cb3f6dSOleg Nesterov static unsigned long xol_get_insn_slot(struct uprobe *uprobe) 1186d4b3b638SSrikar Dronamraju { 1187d4b3b638SSrikar Dronamraju struct xol_area *area; 1188d4b3b638SSrikar Dronamraju unsigned long offset; 1189a6cb3f6dSOleg Nesterov unsigned long xol_vaddr; 1190d4b3b638SSrikar Dronamraju void *vaddr; 1191d4b3b638SSrikar Dronamraju 11929b545df8SOleg Nesterov area = get_xol_area(); 1193d4b3b638SSrikar Dronamraju if (!area) 1194d4b3b638SSrikar Dronamraju return 0; 1195d4b3b638SSrikar Dronamraju 1196a6cb3f6dSOleg Nesterov xol_vaddr = xol_take_insn_slot(area); 1197a6cb3f6dSOleg Nesterov if (unlikely(!xol_vaddr)) 1198d4b3b638SSrikar Dronamraju return 0; 1199d4b3b638SSrikar Dronamraju 1200a6cb3f6dSOleg Nesterov /* Initialize the slot */ 1201a6cb3f6dSOleg Nesterov offset = xol_vaddr & ~PAGE_MASK; 1202d4b3b638SSrikar Dronamraju vaddr = kmap_atomic(area->page); 1203d4b3b638SSrikar Dronamraju memcpy(vaddr + offset, uprobe->arch.insn, MAX_UINSN_BYTES); 1204d4b3b638SSrikar Dronamraju kunmap_atomic(vaddr); 120565b6ecc0SRabin Vincent /* 120665b6ecc0SRabin Vincent * We probably need flush_icache_user_range() but it needs vma. 120765b6ecc0SRabin Vincent * This should work on supported architectures too. 120865b6ecc0SRabin Vincent */ 120965b6ecc0SRabin Vincent flush_dcache_page(area->page); 1210d4b3b638SSrikar Dronamraju 1211a6cb3f6dSOleg Nesterov return xol_vaddr; 1212d4b3b638SSrikar Dronamraju } 1213d4b3b638SSrikar Dronamraju 1214d4b3b638SSrikar Dronamraju /* 1215d4b3b638SSrikar Dronamraju * xol_free_insn_slot - If slot was earlier allocated by 1216d4b3b638SSrikar Dronamraju * @xol_get_insn_slot(), make the slot available for 1217d4b3b638SSrikar Dronamraju * subsequent requests. 1218d4b3b638SSrikar Dronamraju */ 1219d4b3b638SSrikar Dronamraju static void xol_free_insn_slot(struct task_struct *tsk) 1220d4b3b638SSrikar Dronamraju { 1221d4b3b638SSrikar Dronamraju struct xol_area *area; 1222d4b3b638SSrikar Dronamraju unsigned long vma_end; 1223d4b3b638SSrikar Dronamraju unsigned long slot_addr; 1224d4b3b638SSrikar Dronamraju 1225d4b3b638SSrikar Dronamraju if (!tsk->mm || !tsk->mm->uprobes_state.xol_area || !tsk->utask) 1226d4b3b638SSrikar Dronamraju return; 1227d4b3b638SSrikar Dronamraju 1228d4b3b638SSrikar Dronamraju slot_addr = tsk->utask->xol_vaddr; 1229af4355e9SOleg Nesterov if (unlikely(!slot_addr)) 1230d4b3b638SSrikar Dronamraju return; 1231d4b3b638SSrikar Dronamraju 1232d4b3b638SSrikar Dronamraju area = tsk->mm->uprobes_state.xol_area; 1233d4b3b638SSrikar Dronamraju vma_end = area->vaddr + PAGE_SIZE; 1234d4b3b638SSrikar Dronamraju if (area->vaddr <= slot_addr && slot_addr < vma_end) { 1235d4b3b638SSrikar Dronamraju unsigned long offset; 1236d4b3b638SSrikar Dronamraju int slot_nr; 1237d4b3b638SSrikar Dronamraju 1238d4b3b638SSrikar Dronamraju offset = slot_addr - area->vaddr; 1239d4b3b638SSrikar Dronamraju slot_nr = offset / UPROBE_XOL_SLOT_BYTES; 1240d4b3b638SSrikar Dronamraju if (slot_nr >= UINSNS_PER_PAGE) 1241d4b3b638SSrikar Dronamraju return; 1242d4b3b638SSrikar Dronamraju 1243d4b3b638SSrikar Dronamraju clear_bit(slot_nr, area->bitmap); 1244d4b3b638SSrikar Dronamraju atomic_dec(&area->slot_count); 1245d4b3b638SSrikar Dronamraju if (waitqueue_active(&area->wq)) 1246d4b3b638SSrikar Dronamraju wake_up(&area->wq); 1247d4b3b638SSrikar Dronamraju 1248d4b3b638SSrikar Dronamraju tsk->utask->xol_vaddr = 0; 1249d4b3b638SSrikar Dronamraju } 1250d4b3b638SSrikar Dronamraju } 1251d4b3b638SSrikar Dronamraju 12520326f5a9SSrikar Dronamraju /** 12530326f5a9SSrikar Dronamraju * uprobe_get_swbp_addr - compute address of swbp given post-swbp regs 12540326f5a9SSrikar Dronamraju * @regs: Reflects the saved state of the task after it has hit a breakpoint 12550326f5a9SSrikar Dronamraju * instruction. 12560326f5a9SSrikar Dronamraju * Return the address of the breakpoint instruction. 12570326f5a9SSrikar Dronamraju */ 12580326f5a9SSrikar Dronamraju unsigned long __weak uprobe_get_swbp_addr(struct pt_regs *regs) 12590326f5a9SSrikar Dronamraju { 12600326f5a9SSrikar Dronamraju return instruction_pointer(regs) - UPROBE_SWBP_INSN_SIZE; 12610326f5a9SSrikar Dronamraju } 12620326f5a9SSrikar Dronamraju 12630326f5a9SSrikar Dronamraju /* 12640326f5a9SSrikar Dronamraju * Called with no locks held. 12650326f5a9SSrikar Dronamraju * Called in context of a exiting or a exec-ing thread. 12660326f5a9SSrikar Dronamraju */ 12670326f5a9SSrikar Dronamraju void uprobe_free_utask(struct task_struct *t) 12680326f5a9SSrikar Dronamraju { 12690326f5a9SSrikar Dronamraju struct uprobe_task *utask = t->utask; 12700326f5a9SSrikar Dronamraju 12710326f5a9SSrikar Dronamraju if (!utask) 12720326f5a9SSrikar Dronamraju return; 12730326f5a9SSrikar Dronamraju 12740326f5a9SSrikar Dronamraju if (utask->active_uprobe) 12750326f5a9SSrikar Dronamraju put_uprobe(utask->active_uprobe); 12760326f5a9SSrikar Dronamraju 1277d4b3b638SSrikar Dronamraju xol_free_insn_slot(t); 12780326f5a9SSrikar Dronamraju kfree(utask); 12790326f5a9SSrikar Dronamraju t->utask = NULL; 12800326f5a9SSrikar Dronamraju } 12810326f5a9SSrikar Dronamraju 12820326f5a9SSrikar Dronamraju /* 12830326f5a9SSrikar Dronamraju * Called in context of a new clone/fork from copy_process. 12840326f5a9SSrikar Dronamraju */ 12850326f5a9SSrikar Dronamraju void uprobe_copy_process(struct task_struct *t) 12860326f5a9SSrikar Dronamraju { 12870326f5a9SSrikar Dronamraju t->utask = NULL; 12880326f5a9SSrikar Dronamraju } 12890326f5a9SSrikar Dronamraju 12900326f5a9SSrikar Dronamraju /* 12915a2df662SOleg Nesterov * Allocate a uprobe_task object for the task if if necessary. 12925a2df662SOleg Nesterov * Called when the thread hits a breakpoint. 12930326f5a9SSrikar Dronamraju * 12940326f5a9SSrikar Dronamraju * Returns: 12950326f5a9SSrikar Dronamraju * - pointer to new uprobe_task on success 12960326f5a9SSrikar Dronamraju * - NULL otherwise 12970326f5a9SSrikar Dronamraju */ 12985a2df662SOleg Nesterov static struct uprobe_task *get_utask(void) 12990326f5a9SSrikar Dronamraju { 13005a2df662SOleg Nesterov if (!current->utask) 13015a2df662SOleg Nesterov current->utask = kzalloc(sizeof(struct uprobe_task), GFP_KERNEL); 13025a2df662SOleg Nesterov return current->utask; 13030326f5a9SSrikar Dronamraju } 13040326f5a9SSrikar Dronamraju 13050326f5a9SSrikar Dronamraju /* Prepare to single-step probed instruction out of line. */ 13060326f5a9SSrikar Dronamraju static int 1307a6cb3f6dSOleg Nesterov pre_ssout(struct uprobe *uprobe, struct pt_regs *regs, unsigned long bp_vaddr) 13080326f5a9SSrikar Dronamraju { 1309a6cb3f6dSOleg Nesterov struct uprobe_task *utask; 1310a6cb3f6dSOleg Nesterov unsigned long xol_vaddr; 1311aba51024SOleg Nesterov int err; 1312d4b3b638SSrikar Dronamraju 1313608e7427SOleg Nesterov utask = get_utask(); 1314608e7427SOleg Nesterov if (!utask) 1315608e7427SOleg Nesterov return -ENOMEM; 1316a6cb3f6dSOleg Nesterov 1317a6cb3f6dSOleg Nesterov xol_vaddr = xol_get_insn_slot(uprobe); 1318a6cb3f6dSOleg Nesterov if (!xol_vaddr) 1319a6cb3f6dSOleg Nesterov return -ENOMEM; 1320a6cb3f6dSOleg Nesterov 1321a6cb3f6dSOleg Nesterov utask->xol_vaddr = xol_vaddr; 1322a6cb3f6dSOleg Nesterov utask->vaddr = bp_vaddr; 1323a6cb3f6dSOleg Nesterov 1324aba51024SOleg Nesterov err = arch_uprobe_pre_xol(&uprobe->arch, regs); 1325aba51024SOleg Nesterov if (unlikely(err)) { 1326aba51024SOleg Nesterov xol_free_insn_slot(current); 1327aba51024SOleg Nesterov return err; 1328aba51024SOleg Nesterov } 1329aba51024SOleg Nesterov 1330608e7427SOleg Nesterov utask->active_uprobe = uprobe; 1331608e7427SOleg Nesterov utask->state = UTASK_SSTEP; 1332aba51024SOleg Nesterov return 0; 13330326f5a9SSrikar Dronamraju } 13340326f5a9SSrikar Dronamraju 13350326f5a9SSrikar Dronamraju /* 13360326f5a9SSrikar Dronamraju * If we are singlestepping, then ensure this thread is not connected to 13370326f5a9SSrikar Dronamraju * non-fatal signals until completion of singlestep. When xol insn itself 13380326f5a9SSrikar Dronamraju * triggers the signal, restart the original insn even if the task is 13390326f5a9SSrikar Dronamraju * already SIGKILL'ed (since coredump should report the correct ip). This 13400326f5a9SSrikar Dronamraju * is even more important if the task has a handler for SIGSEGV/etc, The 13410326f5a9SSrikar Dronamraju * _same_ instruction should be repeated again after return from the signal 13420326f5a9SSrikar Dronamraju * handler, and SSTEP can never finish in this case. 13430326f5a9SSrikar Dronamraju */ 13440326f5a9SSrikar Dronamraju bool uprobe_deny_signal(void) 13450326f5a9SSrikar Dronamraju { 13460326f5a9SSrikar Dronamraju struct task_struct *t = current; 13470326f5a9SSrikar Dronamraju struct uprobe_task *utask = t->utask; 13480326f5a9SSrikar Dronamraju 13490326f5a9SSrikar Dronamraju if (likely(!utask || !utask->active_uprobe)) 13500326f5a9SSrikar Dronamraju return false; 13510326f5a9SSrikar Dronamraju 13520326f5a9SSrikar Dronamraju WARN_ON_ONCE(utask->state != UTASK_SSTEP); 13530326f5a9SSrikar Dronamraju 13540326f5a9SSrikar Dronamraju if (signal_pending(t)) { 13550326f5a9SSrikar Dronamraju spin_lock_irq(&t->sighand->siglock); 13560326f5a9SSrikar Dronamraju clear_tsk_thread_flag(t, TIF_SIGPENDING); 13570326f5a9SSrikar Dronamraju spin_unlock_irq(&t->sighand->siglock); 13580326f5a9SSrikar Dronamraju 13590326f5a9SSrikar Dronamraju if (__fatal_signal_pending(t) || arch_uprobe_xol_was_trapped(t)) { 13600326f5a9SSrikar Dronamraju utask->state = UTASK_SSTEP_TRAPPED; 13610326f5a9SSrikar Dronamraju set_tsk_thread_flag(t, TIF_UPROBE); 13620326f5a9SSrikar Dronamraju set_tsk_thread_flag(t, TIF_NOTIFY_RESUME); 13630326f5a9SSrikar Dronamraju } 13640326f5a9SSrikar Dronamraju } 13650326f5a9SSrikar Dronamraju 13660326f5a9SSrikar Dronamraju return true; 13670326f5a9SSrikar Dronamraju } 13680326f5a9SSrikar Dronamraju 13690326f5a9SSrikar Dronamraju /* 13700326f5a9SSrikar Dronamraju * Avoid singlestepping the original instruction if the original instruction 13710326f5a9SSrikar Dronamraju * is a NOP or can be emulated. 13720326f5a9SSrikar Dronamraju */ 13730326f5a9SSrikar Dronamraju static bool can_skip_sstep(struct uprobe *uprobe, struct pt_regs *regs) 13740326f5a9SSrikar Dronamraju { 137571434f2fSOleg Nesterov if (test_bit(UPROBE_SKIP_SSTEP, &uprobe->flags)) { 13760326f5a9SSrikar Dronamraju if (arch_uprobe_skip_sstep(&uprobe->arch, regs)) 13770326f5a9SSrikar Dronamraju return true; 137871434f2fSOleg Nesterov clear_bit(UPROBE_SKIP_SSTEP, &uprobe->flags); 13790578a970SOleg Nesterov } 13800326f5a9SSrikar Dronamraju return false; 13810326f5a9SSrikar Dronamraju } 13820326f5a9SSrikar Dronamraju 1383499a4f3eSOleg Nesterov static void mmf_recalc_uprobes(struct mm_struct *mm) 1384499a4f3eSOleg Nesterov { 1385499a4f3eSOleg Nesterov struct vm_area_struct *vma; 1386499a4f3eSOleg Nesterov 1387499a4f3eSOleg Nesterov for (vma = mm->mmap; vma; vma = vma->vm_next) { 1388499a4f3eSOleg Nesterov if (!valid_vma(vma, false)) 1389499a4f3eSOleg Nesterov continue; 1390499a4f3eSOleg Nesterov /* 1391499a4f3eSOleg Nesterov * This is not strictly accurate, we can race with 1392499a4f3eSOleg Nesterov * uprobe_unregister() and see the already removed 1393499a4f3eSOleg Nesterov * uprobe if delete_uprobe() was not yet called. 139463633cbfSOleg Nesterov * Or this uprobe can be filtered out. 1395499a4f3eSOleg Nesterov */ 1396499a4f3eSOleg Nesterov if (vma_has_uprobes(vma, vma->vm_start, vma->vm_end)) 1397499a4f3eSOleg Nesterov return; 1398499a4f3eSOleg Nesterov } 1399499a4f3eSOleg Nesterov 1400499a4f3eSOleg Nesterov clear_bit(MMF_HAS_UPROBES, &mm->flags); 1401499a4f3eSOleg Nesterov } 1402499a4f3eSOleg Nesterov 1403ec75fba9SOleg Nesterov static int is_swbp_at_addr(struct mm_struct *mm, unsigned long vaddr) 1404ec75fba9SOleg Nesterov { 1405ec75fba9SOleg Nesterov struct page *page; 1406ec75fba9SOleg Nesterov uprobe_opcode_t opcode; 1407ec75fba9SOleg Nesterov int result; 1408ec75fba9SOleg Nesterov 1409ec75fba9SOleg Nesterov pagefault_disable(); 1410ec75fba9SOleg Nesterov result = __copy_from_user_inatomic(&opcode, (void __user*)vaddr, 1411ec75fba9SOleg Nesterov sizeof(opcode)); 1412ec75fba9SOleg Nesterov pagefault_enable(); 1413ec75fba9SOleg Nesterov 1414ec75fba9SOleg Nesterov if (likely(result == 0)) 1415ec75fba9SOleg Nesterov goto out; 1416ec75fba9SOleg Nesterov 1417ec75fba9SOleg Nesterov result = get_user_pages(NULL, mm, vaddr, 1, 0, 1, &page, NULL); 1418ec75fba9SOleg Nesterov if (result < 0) 1419ec75fba9SOleg Nesterov return result; 1420ec75fba9SOleg Nesterov 1421ec75fba9SOleg Nesterov copy_opcode(page, vaddr, &opcode); 1422ec75fba9SOleg Nesterov put_page(page); 1423ec75fba9SOleg Nesterov out: 1424ec75fba9SOleg Nesterov return is_swbp_insn(&opcode); 1425ec75fba9SOleg Nesterov } 1426ec75fba9SOleg Nesterov 1427d790d346SOleg Nesterov static struct uprobe *find_active_uprobe(unsigned long bp_vaddr, int *is_swbp) 14280326f5a9SSrikar Dronamraju { 14293a9ea052SOleg Nesterov struct mm_struct *mm = current->mm; 14303a9ea052SOleg Nesterov struct uprobe *uprobe = NULL; 14310326f5a9SSrikar Dronamraju struct vm_area_struct *vma; 14320326f5a9SSrikar Dronamraju 14330326f5a9SSrikar Dronamraju down_read(&mm->mmap_sem); 14340326f5a9SSrikar Dronamraju vma = find_vma(mm, bp_vaddr); 14353a9ea052SOleg Nesterov if (vma && vma->vm_start <= bp_vaddr) { 14363a9ea052SOleg Nesterov if (valid_vma(vma, false)) { 1437cb113b47SOleg Nesterov struct inode *inode = vma->vm_file->f_mapping->host; 1438cb113b47SOleg Nesterov loff_t offset = vaddr_to_offset(vma, bp_vaddr); 14390326f5a9SSrikar Dronamraju 14400326f5a9SSrikar Dronamraju uprobe = find_uprobe(inode, offset); 14410326f5a9SSrikar Dronamraju } 1442d790d346SOleg Nesterov 1443d790d346SOleg Nesterov if (!uprobe) 1444d790d346SOleg Nesterov *is_swbp = is_swbp_at_addr(mm, bp_vaddr); 1445d790d346SOleg Nesterov } else { 1446d790d346SOleg Nesterov *is_swbp = -EFAULT; 14473a9ea052SOleg Nesterov } 1448499a4f3eSOleg Nesterov 1449499a4f3eSOleg Nesterov if (!uprobe && test_and_clear_bit(MMF_RECALC_UPROBES, &mm->flags)) 1450499a4f3eSOleg Nesterov mmf_recalc_uprobes(mm); 14510326f5a9SSrikar Dronamraju up_read(&mm->mmap_sem); 14520326f5a9SSrikar Dronamraju 14533a9ea052SOleg Nesterov return uprobe; 14543a9ea052SOleg Nesterov } 14553a9ea052SOleg Nesterov 1456da1816b1SOleg Nesterov static void handler_chain(struct uprobe *uprobe, struct pt_regs *regs) 1457da1816b1SOleg Nesterov { 1458da1816b1SOleg Nesterov struct uprobe_consumer *uc; 1459da1816b1SOleg Nesterov int remove = UPROBE_HANDLER_REMOVE; 1460da1816b1SOleg Nesterov 1461da1816b1SOleg Nesterov down_read(&uprobe->register_rwsem); 1462da1816b1SOleg Nesterov for (uc = uprobe->consumers; uc; uc = uc->next) { 1463da1816b1SOleg Nesterov int rc = uc->handler(uc, regs); 1464da1816b1SOleg Nesterov 1465da1816b1SOleg Nesterov WARN(rc & ~UPROBE_HANDLER_MASK, 1466da1816b1SOleg Nesterov "bad rc=0x%x from %pf()\n", rc, uc->handler); 1467da1816b1SOleg Nesterov remove &= rc; 1468da1816b1SOleg Nesterov } 1469da1816b1SOleg Nesterov 1470da1816b1SOleg Nesterov if (remove && uprobe->consumers) { 1471da1816b1SOleg Nesterov WARN_ON(!uprobe_is_active(uprobe)); 1472da1816b1SOleg Nesterov unapply_uprobe(uprobe, current->mm); 1473da1816b1SOleg Nesterov } 1474da1816b1SOleg Nesterov up_read(&uprobe->register_rwsem); 1475da1816b1SOleg Nesterov } 1476da1816b1SOleg Nesterov 14773a9ea052SOleg Nesterov /* 14783a9ea052SOleg Nesterov * Run handler and ask thread to singlestep. 14793a9ea052SOleg Nesterov * Ensure all non-fatal signals cannot interrupt thread while it singlesteps. 14803a9ea052SOleg Nesterov */ 14813a9ea052SOleg Nesterov static void handle_swbp(struct pt_regs *regs) 14823a9ea052SOleg Nesterov { 14833a9ea052SOleg Nesterov struct uprobe *uprobe; 14843a9ea052SOleg Nesterov unsigned long bp_vaddr; 148556bb4cf6SOleg Nesterov int uninitialized_var(is_swbp); 14863a9ea052SOleg Nesterov 14873a9ea052SOleg Nesterov bp_vaddr = uprobe_get_swbp_addr(regs); 1488d790d346SOleg Nesterov uprobe = find_active_uprobe(bp_vaddr, &is_swbp); 14893a9ea052SOleg Nesterov 14900326f5a9SSrikar Dronamraju if (!uprobe) { 149156bb4cf6SOleg Nesterov if (is_swbp > 0) { 14920326f5a9SSrikar Dronamraju /* No matching uprobe; signal SIGTRAP. */ 14930326f5a9SSrikar Dronamraju send_sig(SIGTRAP, current, 0); 149456bb4cf6SOleg Nesterov } else { 149556bb4cf6SOleg Nesterov /* 149656bb4cf6SOleg Nesterov * Either we raced with uprobe_unregister() or we can't 149756bb4cf6SOleg Nesterov * access this memory. The latter is only possible if 149856bb4cf6SOleg Nesterov * another thread plays with our ->mm. In both cases 149956bb4cf6SOleg Nesterov * we can simply restart. If this vma was unmapped we 150056bb4cf6SOleg Nesterov * can pretend this insn was not executed yet and get 150156bb4cf6SOleg Nesterov * the (correct) SIGSEGV after restart. 150256bb4cf6SOleg Nesterov */ 150356bb4cf6SOleg Nesterov instruction_pointer_set(regs, bp_vaddr); 150456bb4cf6SOleg Nesterov } 15050326f5a9SSrikar Dronamraju return; 15060326f5a9SSrikar Dronamraju } 150774e59dfcSOleg Nesterov 150874e59dfcSOleg Nesterov /* change it in advance for ->handler() and restart */ 150974e59dfcSOleg Nesterov instruction_pointer_set(regs, bp_vaddr); 151074e59dfcSOleg Nesterov 1511142b18ddSOleg Nesterov /* 1512142b18ddSOleg Nesterov * TODO: move copy_insn/etc into _register and remove this hack. 1513142b18ddSOleg Nesterov * After we hit the bp, _unregister + _register can install the 1514142b18ddSOleg Nesterov * new and not-yet-analyzed uprobe at the same address, restart. 1515142b18ddSOleg Nesterov */ 1516142b18ddSOleg Nesterov smp_rmb(); /* pairs with wmb() in install_breakpoint() */ 151771434f2fSOleg Nesterov if (unlikely(!test_bit(UPROBE_COPY_INSN, &uprobe->flags))) 151874e59dfcSOleg Nesterov goto out; 15190326f5a9SSrikar Dronamraju 15200326f5a9SSrikar Dronamraju handler_chain(uprobe, regs); 15210578a970SOleg Nesterov if (can_skip_sstep(uprobe, regs)) 15220578a970SOleg Nesterov goto out; 15230326f5a9SSrikar Dronamraju 1524608e7427SOleg Nesterov if (!pre_ssout(uprobe, regs, bp_vaddr)) 15250326f5a9SSrikar Dronamraju return; 15260326f5a9SSrikar Dronamraju 152774e59dfcSOleg Nesterov /* can_skip_sstep() succeeded, or restart if can't singlestep */ 15280578a970SOleg Nesterov out: 15290326f5a9SSrikar Dronamraju put_uprobe(uprobe); 15300326f5a9SSrikar Dronamraju } 15310326f5a9SSrikar Dronamraju 15320326f5a9SSrikar Dronamraju /* 15330326f5a9SSrikar Dronamraju * Perform required fix-ups and disable singlestep. 15340326f5a9SSrikar Dronamraju * Allow pending signals to take effect. 15350326f5a9SSrikar Dronamraju */ 15360326f5a9SSrikar Dronamraju static void handle_singlestep(struct uprobe_task *utask, struct pt_regs *regs) 15370326f5a9SSrikar Dronamraju { 15380326f5a9SSrikar Dronamraju struct uprobe *uprobe; 15390326f5a9SSrikar Dronamraju 15400326f5a9SSrikar Dronamraju uprobe = utask->active_uprobe; 15410326f5a9SSrikar Dronamraju if (utask->state == UTASK_SSTEP_ACK) 15420326f5a9SSrikar Dronamraju arch_uprobe_post_xol(&uprobe->arch, regs); 15430326f5a9SSrikar Dronamraju else if (utask->state == UTASK_SSTEP_TRAPPED) 15440326f5a9SSrikar Dronamraju arch_uprobe_abort_xol(&uprobe->arch, regs); 15450326f5a9SSrikar Dronamraju else 15460326f5a9SSrikar Dronamraju WARN_ON_ONCE(1); 15470326f5a9SSrikar Dronamraju 15480326f5a9SSrikar Dronamraju put_uprobe(uprobe); 15490326f5a9SSrikar Dronamraju utask->active_uprobe = NULL; 15500326f5a9SSrikar Dronamraju utask->state = UTASK_RUNNING; 1551d4b3b638SSrikar Dronamraju xol_free_insn_slot(current); 15520326f5a9SSrikar Dronamraju 15530326f5a9SSrikar Dronamraju spin_lock_irq(¤t->sighand->siglock); 15540326f5a9SSrikar Dronamraju recalc_sigpending(); /* see uprobe_deny_signal() */ 15550326f5a9SSrikar Dronamraju spin_unlock_irq(¤t->sighand->siglock); 15560326f5a9SSrikar Dronamraju } 15570326f5a9SSrikar Dronamraju 15580326f5a9SSrikar Dronamraju /* 15591b08e907SOleg Nesterov * On breakpoint hit, breakpoint notifier sets the TIF_UPROBE flag and 15601b08e907SOleg Nesterov * allows the thread to return from interrupt. After that handle_swbp() 15611b08e907SOleg Nesterov * sets utask->active_uprobe. 15620326f5a9SSrikar Dronamraju * 15631b08e907SOleg Nesterov * On singlestep exception, singlestep notifier sets the TIF_UPROBE flag 15641b08e907SOleg Nesterov * and allows the thread to return from interrupt. 15650326f5a9SSrikar Dronamraju * 15660326f5a9SSrikar Dronamraju * While returning to userspace, thread notices the TIF_UPROBE flag and calls 15670326f5a9SSrikar Dronamraju * uprobe_notify_resume(). 15680326f5a9SSrikar Dronamraju */ 15690326f5a9SSrikar Dronamraju void uprobe_notify_resume(struct pt_regs *regs) 15700326f5a9SSrikar Dronamraju { 15710326f5a9SSrikar Dronamraju struct uprobe_task *utask; 15720326f5a9SSrikar Dronamraju 1573db023ea5SOleg Nesterov clear_thread_flag(TIF_UPROBE); 1574db023ea5SOleg Nesterov 15750326f5a9SSrikar Dronamraju utask = current->utask; 15761b08e907SOleg Nesterov if (utask && utask->active_uprobe) 15770326f5a9SSrikar Dronamraju handle_singlestep(utask, regs); 15781b08e907SOleg Nesterov else 15791b08e907SOleg Nesterov handle_swbp(regs); 15800326f5a9SSrikar Dronamraju } 15810326f5a9SSrikar Dronamraju 15820326f5a9SSrikar Dronamraju /* 15830326f5a9SSrikar Dronamraju * uprobe_pre_sstep_notifier gets called from interrupt context as part of 15840326f5a9SSrikar Dronamraju * notifier mechanism. Set TIF_UPROBE flag and indicate breakpoint hit. 15850326f5a9SSrikar Dronamraju */ 15860326f5a9SSrikar Dronamraju int uprobe_pre_sstep_notifier(struct pt_regs *regs) 15870326f5a9SSrikar Dronamraju { 1588f8ac4ec9SOleg Nesterov if (!current->mm || !test_bit(MMF_HAS_UPROBES, ¤t->mm->flags)) 15890326f5a9SSrikar Dronamraju return 0; 15900326f5a9SSrikar Dronamraju 15910326f5a9SSrikar Dronamraju set_thread_flag(TIF_UPROBE); 15920326f5a9SSrikar Dronamraju return 1; 15930326f5a9SSrikar Dronamraju } 15940326f5a9SSrikar Dronamraju 15950326f5a9SSrikar Dronamraju /* 15960326f5a9SSrikar Dronamraju * uprobe_post_sstep_notifier gets called in interrupt context as part of notifier 15970326f5a9SSrikar Dronamraju * mechanism. Set TIF_UPROBE flag and indicate completion of singlestep. 15980326f5a9SSrikar Dronamraju */ 15990326f5a9SSrikar Dronamraju int uprobe_post_sstep_notifier(struct pt_regs *regs) 16000326f5a9SSrikar Dronamraju { 16010326f5a9SSrikar Dronamraju struct uprobe_task *utask = current->utask; 16020326f5a9SSrikar Dronamraju 16030326f5a9SSrikar Dronamraju if (!current->mm || !utask || !utask->active_uprobe) 16040326f5a9SSrikar Dronamraju /* task is currently not uprobed */ 16050326f5a9SSrikar Dronamraju return 0; 16060326f5a9SSrikar Dronamraju 16070326f5a9SSrikar Dronamraju utask->state = UTASK_SSTEP_ACK; 16080326f5a9SSrikar Dronamraju set_thread_flag(TIF_UPROBE); 16090326f5a9SSrikar Dronamraju return 1; 16100326f5a9SSrikar Dronamraju } 16110326f5a9SSrikar Dronamraju 16120326f5a9SSrikar Dronamraju static struct notifier_block uprobe_exception_nb = { 16130326f5a9SSrikar Dronamraju .notifier_call = arch_uprobe_exception_notify, 16140326f5a9SSrikar Dronamraju .priority = INT_MAX-1, /* notified after kprobes, kgdb */ 16150326f5a9SSrikar Dronamraju }; 16160326f5a9SSrikar Dronamraju 1617a5f4374aSIngo Molnar static int __init init_uprobes(void) 1618a5f4374aSIngo Molnar { 1619a5f4374aSIngo Molnar int i; 1620a5f4374aSIngo Molnar 162166d06dffSOleg Nesterov for (i = 0; i < UPROBES_HASH_SZ; i++) 1622a5f4374aSIngo Molnar mutex_init(&uprobes_mmap_mutex[i]); 16230326f5a9SSrikar Dronamraju 162432cdba1eSOleg Nesterov if (percpu_init_rwsem(&dup_mmap_sem)) 162532cdba1eSOleg Nesterov return -ENOMEM; 162632cdba1eSOleg Nesterov 16270326f5a9SSrikar Dronamraju return register_die_notifier(&uprobe_exception_nb); 1628a5f4374aSIngo Molnar } 16290326f5a9SSrikar Dronamraju module_init(init_uprobes); 1630a5f4374aSIngo Molnar 1631a5f4374aSIngo Molnar static void __exit exit_uprobes(void) 1632a5f4374aSIngo Molnar { 1633a5f4374aSIngo Molnar } 1634a5f4374aSIngo Molnar module_exit(exit_uprobes); 1635