1a5f4374aSIngo Molnar /* 2a5f4374aSIngo Molnar * User-space Probes (UProbes) 3a5f4374aSIngo Molnar * 4a5f4374aSIngo Molnar * This program is free software; you can redistribute it and/or modify 5a5f4374aSIngo Molnar * it under the terms of the GNU General Public License as published by 6a5f4374aSIngo Molnar * the Free Software Foundation; either version 2 of the License, or 7a5f4374aSIngo Molnar * (at your option) any later version. 8a5f4374aSIngo Molnar * 9a5f4374aSIngo Molnar * This program is distributed in the hope that it will be useful, 10a5f4374aSIngo Molnar * but WITHOUT ANY WARRANTY; without even the implied warranty of 11a5f4374aSIngo Molnar * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12a5f4374aSIngo Molnar * GNU General Public License for more details. 13a5f4374aSIngo Molnar * 14a5f4374aSIngo Molnar * You should have received a copy of the GNU General Public License 15a5f4374aSIngo Molnar * along with this program; if not, write to the Free Software 16a5f4374aSIngo Molnar * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 17a5f4374aSIngo Molnar * 1835aa621bSIngo Molnar * Copyright (C) IBM Corporation, 2008-2012 19a5f4374aSIngo Molnar * Authors: 20a5f4374aSIngo Molnar * Srikar Dronamraju 21a5f4374aSIngo Molnar * Jim Keniston 2235aa621bSIngo Molnar * Copyright (C) 2011-2012 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com> 23a5f4374aSIngo Molnar */ 24a5f4374aSIngo Molnar 25a5f4374aSIngo Molnar #include <linux/kernel.h> 26a5f4374aSIngo Molnar #include <linux/highmem.h> 27a5f4374aSIngo Molnar #include <linux/pagemap.h> /* read_mapping_page */ 28a5f4374aSIngo Molnar #include <linux/slab.h> 29a5f4374aSIngo Molnar #include <linux/sched.h> 30a5f4374aSIngo Molnar #include <linux/rmap.h> /* anon_vma_prepare */ 31a5f4374aSIngo Molnar #include <linux/mmu_notifier.h> /* set_pte_at_notify */ 32a5f4374aSIngo Molnar #include <linux/swap.h> /* try_to_free_swap */ 330326f5a9SSrikar Dronamraju #include <linux/ptrace.h> /* user_enable_single_step */ 340326f5a9SSrikar Dronamraju #include <linux/kdebug.h> /* notifier mechanism */ 35a5f4374aSIngo Molnar 36a5f4374aSIngo Molnar #include <linux/uprobes.h> 37a5f4374aSIngo Molnar 38d4b3b638SSrikar Dronamraju #define UINSNS_PER_PAGE (PAGE_SIZE/UPROBE_XOL_SLOT_BYTES) 39d4b3b638SSrikar Dronamraju #define MAX_UPROBE_XOL_SLOTS UINSNS_PER_PAGE 40d4b3b638SSrikar Dronamraju 41a5f4374aSIngo Molnar static struct rb_root uprobes_tree = RB_ROOT; 42a5f4374aSIngo Molnar 43a5f4374aSIngo Molnar static DEFINE_SPINLOCK(uprobes_treelock); /* serialize rbtree access */ 44a5f4374aSIngo Molnar 45a5f4374aSIngo Molnar #define UPROBES_HASH_SZ 13 46a5f4374aSIngo Molnar 47a5f4374aSIngo Molnar /* serialize (un)register */ 48a5f4374aSIngo Molnar static struct mutex uprobes_mutex[UPROBES_HASH_SZ]; 49a5f4374aSIngo Molnar 50a5f4374aSIngo Molnar #define uprobes_hash(v) (&uprobes_mutex[((unsigned long)(v)) % UPROBES_HASH_SZ]) 51a5f4374aSIngo Molnar 52a5f4374aSIngo Molnar /* serialize uprobe->pending_list */ 53a5f4374aSIngo Molnar static struct mutex uprobes_mmap_mutex[UPROBES_HASH_SZ]; 54a5f4374aSIngo Molnar #define uprobes_mmap_hash(v) (&uprobes_mmap_mutex[((unsigned long)(v)) % UPROBES_HASH_SZ]) 55a5f4374aSIngo Molnar 56a5f4374aSIngo Molnar /* 57a5f4374aSIngo Molnar * uprobe_events allows us to skip the uprobe_mmap if there are no uprobe 58a5f4374aSIngo Molnar * events active at this time. Probably a fine grained per inode count is 59a5f4374aSIngo Molnar * better? 60a5f4374aSIngo Molnar */ 61a5f4374aSIngo Molnar static atomic_t uprobe_events = ATOMIC_INIT(0); 62a5f4374aSIngo Molnar 63a5f4374aSIngo Molnar /* 64a5f4374aSIngo Molnar * Maintain a temporary per vma info that can be used to search if a vma 65a5f4374aSIngo Molnar * has already been handled. This structure is introduced since extending 66a5f4374aSIngo Molnar * vm_area_struct wasnt recommended. 67a5f4374aSIngo Molnar */ 68a5f4374aSIngo Molnar struct vma_info { 69a5f4374aSIngo Molnar struct list_head probe_list; 70a5f4374aSIngo Molnar struct mm_struct *mm; 71a5f4374aSIngo Molnar loff_t vaddr; 72a5f4374aSIngo Molnar }; 73a5f4374aSIngo Molnar 743ff54efdSSrikar Dronamraju struct uprobe { 753ff54efdSSrikar Dronamraju struct rb_node rb_node; /* node in the rb tree */ 763ff54efdSSrikar Dronamraju atomic_t ref; 773ff54efdSSrikar Dronamraju struct rw_semaphore consumer_rwsem; 783ff54efdSSrikar Dronamraju struct list_head pending_list; 793ff54efdSSrikar Dronamraju struct uprobe_consumer *consumers; 803ff54efdSSrikar Dronamraju struct inode *inode; /* Also hold a ref to inode */ 813ff54efdSSrikar Dronamraju loff_t offset; 823ff54efdSSrikar Dronamraju int flags; 833ff54efdSSrikar Dronamraju struct arch_uprobe arch; 843ff54efdSSrikar Dronamraju }; 853ff54efdSSrikar Dronamraju 86a5f4374aSIngo Molnar /* 87a5f4374aSIngo Molnar * valid_vma: Verify if the specified vma is an executable vma 88a5f4374aSIngo Molnar * Relax restrictions while unregistering: vm_flags might have 89a5f4374aSIngo Molnar * changed after breakpoint was inserted. 90a5f4374aSIngo Molnar * - is_register: indicates if we are in register context. 91a5f4374aSIngo Molnar * - Return 1 if the specified virtual address is in an 92a5f4374aSIngo Molnar * executable vma. 93a5f4374aSIngo Molnar */ 94a5f4374aSIngo Molnar static bool valid_vma(struct vm_area_struct *vma, bool is_register) 95a5f4374aSIngo Molnar { 96a5f4374aSIngo Molnar if (!vma->vm_file) 97a5f4374aSIngo Molnar return false; 98a5f4374aSIngo Molnar 99a5f4374aSIngo Molnar if (!is_register) 100a5f4374aSIngo Molnar return true; 101a5f4374aSIngo Molnar 102ea131377SOleg Nesterov if ((vma->vm_flags & (VM_HUGETLB|VM_READ|VM_WRITE|VM_EXEC|VM_SHARED)) 103ea131377SOleg Nesterov == (VM_READ|VM_EXEC)) 104a5f4374aSIngo Molnar return true; 105a5f4374aSIngo Molnar 106a5f4374aSIngo Molnar return false; 107a5f4374aSIngo Molnar } 108a5f4374aSIngo Molnar 109a5f4374aSIngo Molnar static loff_t vma_address(struct vm_area_struct *vma, loff_t offset) 110a5f4374aSIngo Molnar { 111a5f4374aSIngo Molnar loff_t vaddr; 112a5f4374aSIngo Molnar 113a5f4374aSIngo Molnar vaddr = vma->vm_start + offset; 114a5f4374aSIngo Molnar vaddr -= vma->vm_pgoff << PAGE_SHIFT; 115a5f4374aSIngo Molnar 116a5f4374aSIngo Molnar return vaddr; 117a5f4374aSIngo Molnar } 118a5f4374aSIngo Molnar 119a5f4374aSIngo Molnar /** 120a5f4374aSIngo Molnar * __replace_page - replace page in vma by new page. 121a5f4374aSIngo Molnar * based on replace_page in mm/ksm.c 122a5f4374aSIngo Molnar * 123a5f4374aSIngo Molnar * @vma: vma that holds the pte pointing to page 124a5f4374aSIngo Molnar * @page: the cowed page we are replacing by kpage 125a5f4374aSIngo Molnar * @kpage: the modified page we replace page by 126a5f4374aSIngo Molnar * 127a5f4374aSIngo Molnar * Returns 0 on success, -EFAULT on failure. 128a5f4374aSIngo Molnar */ 129a5f4374aSIngo Molnar static int __replace_page(struct vm_area_struct *vma, struct page *page, struct page *kpage) 130a5f4374aSIngo Molnar { 131a5f4374aSIngo Molnar struct mm_struct *mm = vma->vm_mm; 132a5f4374aSIngo Molnar pgd_t *pgd; 133a5f4374aSIngo Molnar pud_t *pud; 134a5f4374aSIngo Molnar pmd_t *pmd; 135a5f4374aSIngo Molnar pte_t *ptep; 136a5f4374aSIngo Molnar spinlock_t *ptl; 137a5f4374aSIngo Molnar unsigned long addr; 138a5f4374aSIngo Molnar int err = -EFAULT; 139a5f4374aSIngo Molnar 140a5f4374aSIngo Molnar addr = page_address_in_vma(page, vma); 141a5f4374aSIngo Molnar if (addr == -EFAULT) 142a5f4374aSIngo Molnar goto out; 143a5f4374aSIngo Molnar 144a5f4374aSIngo Molnar pgd = pgd_offset(mm, addr); 145a5f4374aSIngo Molnar if (!pgd_present(*pgd)) 146a5f4374aSIngo Molnar goto out; 147a5f4374aSIngo Molnar 148a5f4374aSIngo Molnar pud = pud_offset(pgd, addr); 149a5f4374aSIngo Molnar if (!pud_present(*pud)) 150a5f4374aSIngo Molnar goto out; 151a5f4374aSIngo Molnar 152a5f4374aSIngo Molnar pmd = pmd_offset(pud, addr); 153a5f4374aSIngo Molnar if (!pmd_present(*pmd)) 154a5f4374aSIngo Molnar goto out; 155a5f4374aSIngo Molnar 156a5f4374aSIngo Molnar ptep = pte_offset_map_lock(mm, pmd, addr, &ptl); 157a5f4374aSIngo Molnar if (!ptep) 158a5f4374aSIngo Molnar goto out; 159a5f4374aSIngo Molnar 160a5f4374aSIngo Molnar get_page(kpage); 161a5f4374aSIngo Molnar page_add_new_anon_rmap(kpage, vma, addr); 162a5f4374aSIngo Molnar 1637396fa81SSrikar Dronamraju if (!PageAnon(page)) { 1647396fa81SSrikar Dronamraju dec_mm_counter(mm, MM_FILEPAGES); 1657396fa81SSrikar Dronamraju inc_mm_counter(mm, MM_ANONPAGES); 1667396fa81SSrikar Dronamraju } 1677396fa81SSrikar Dronamraju 168a5f4374aSIngo Molnar flush_cache_page(vma, addr, pte_pfn(*ptep)); 169a5f4374aSIngo Molnar ptep_clear_flush(vma, addr, ptep); 170a5f4374aSIngo Molnar set_pte_at_notify(mm, addr, ptep, mk_pte(kpage, vma->vm_page_prot)); 171a5f4374aSIngo Molnar 172a5f4374aSIngo Molnar page_remove_rmap(page); 173a5f4374aSIngo Molnar if (!page_mapped(page)) 174a5f4374aSIngo Molnar try_to_free_swap(page); 175a5f4374aSIngo Molnar put_page(page); 176a5f4374aSIngo Molnar pte_unmap_unlock(ptep, ptl); 177a5f4374aSIngo Molnar err = 0; 178a5f4374aSIngo Molnar 179a5f4374aSIngo Molnar out: 180a5f4374aSIngo Molnar return err; 181a5f4374aSIngo Molnar } 182a5f4374aSIngo Molnar 183a5f4374aSIngo Molnar /** 1845cb4ac3aSSrikar Dronamraju * is_swbp_insn - check if instruction is breakpoint instruction. 185a5f4374aSIngo Molnar * @insn: instruction to be checked. 1865cb4ac3aSSrikar Dronamraju * Default implementation of is_swbp_insn 187a5f4374aSIngo Molnar * Returns true if @insn is a breakpoint instruction. 188a5f4374aSIngo Molnar */ 1895cb4ac3aSSrikar Dronamraju bool __weak is_swbp_insn(uprobe_opcode_t *insn) 190a5f4374aSIngo Molnar { 1915cb4ac3aSSrikar Dronamraju return *insn == UPROBE_SWBP_INSN; 192a5f4374aSIngo Molnar } 193a5f4374aSIngo Molnar 194a5f4374aSIngo Molnar /* 195a5f4374aSIngo Molnar * NOTE: 196a5f4374aSIngo Molnar * Expect the breakpoint instruction to be the smallest size instruction for 197a5f4374aSIngo Molnar * the architecture. If an arch has variable length instruction and the 198a5f4374aSIngo Molnar * breakpoint instruction is not of the smallest length instruction 199a5f4374aSIngo Molnar * supported by that architecture then we need to modify read_opcode / 200a5f4374aSIngo Molnar * write_opcode accordingly. This would never be a problem for archs that 201a5f4374aSIngo Molnar * have fixed length instructions. 202a5f4374aSIngo Molnar */ 203a5f4374aSIngo Molnar 204a5f4374aSIngo Molnar /* 205a5f4374aSIngo Molnar * write_opcode - write the opcode at a given virtual address. 206e3343e6aSSrikar Dronamraju * @auprobe: arch breakpointing information. 207a5f4374aSIngo Molnar * @mm: the probed process address space. 208a5f4374aSIngo Molnar * @vaddr: the virtual address to store the opcode. 209a5f4374aSIngo Molnar * @opcode: opcode to be written at @vaddr. 210a5f4374aSIngo Molnar * 211a5f4374aSIngo Molnar * Called with mm->mmap_sem held (for read and with a reference to 212a5f4374aSIngo Molnar * mm). 213a5f4374aSIngo Molnar * 214a5f4374aSIngo Molnar * For mm @mm, write the opcode at @vaddr. 215a5f4374aSIngo Molnar * Return 0 (success) or a negative errno. 216a5f4374aSIngo Molnar */ 217e3343e6aSSrikar Dronamraju static int write_opcode(struct arch_uprobe *auprobe, struct mm_struct *mm, 218a5f4374aSIngo Molnar unsigned long vaddr, uprobe_opcode_t opcode) 219a5f4374aSIngo Molnar { 220a5f4374aSIngo Molnar struct page *old_page, *new_page; 221a5f4374aSIngo Molnar struct address_space *mapping; 222a5f4374aSIngo Molnar void *vaddr_old, *vaddr_new; 223a5f4374aSIngo Molnar struct vm_area_struct *vma; 2243ff54efdSSrikar Dronamraju struct uprobe *uprobe; 225a5f4374aSIngo Molnar loff_t addr; 226a5f4374aSIngo Molnar int ret; 227a5f4374aSIngo Molnar 228a5f4374aSIngo Molnar /* Read the page with vaddr into memory */ 229a5f4374aSIngo Molnar ret = get_user_pages(NULL, mm, vaddr, 1, 0, 0, &old_page, &vma); 230a5f4374aSIngo Molnar if (ret <= 0) 231a5f4374aSIngo Molnar return ret; 232a5f4374aSIngo Molnar 233a5f4374aSIngo Molnar ret = -EINVAL; 234a5f4374aSIngo Molnar 235a5f4374aSIngo Molnar /* 236a5f4374aSIngo Molnar * We are interested in text pages only. Our pages of interest 237a5f4374aSIngo Molnar * should be mapped for read and execute only. We desist from 238a5f4374aSIngo Molnar * adding probes in write mapped pages since the breakpoints 239a5f4374aSIngo Molnar * might end up in the file copy. 240a5f4374aSIngo Molnar */ 2415cb4ac3aSSrikar Dronamraju if (!valid_vma(vma, is_swbp_insn(&opcode))) 242a5f4374aSIngo Molnar goto put_out; 243a5f4374aSIngo Molnar 2443ff54efdSSrikar Dronamraju uprobe = container_of(auprobe, struct uprobe, arch); 245a5f4374aSIngo Molnar mapping = uprobe->inode->i_mapping; 246a5f4374aSIngo Molnar if (mapping != vma->vm_file->f_mapping) 247a5f4374aSIngo Molnar goto put_out; 248a5f4374aSIngo Molnar 249a5f4374aSIngo Molnar addr = vma_address(vma, uprobe->offset); 250a5f4374aSIngo Molnar if (vaddr != (unsigned long)addr) 251a5f4374aSIngo Molnar goto put_out; 252a5f4374aSIngo Molnar 253a5f4374aSIngo Molnar ret = -ENOMEM; 254a5f4374aSIngo Molnar new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, vaddr); 255a5f4374aSIngo Molnar if (!new_page) 256a5f4374aSIngo Molnar goto put_out; 257a5f4374aSIngo Molnar 258a5f4374aSIngo Molnar __SetPageUptodate(new_page); 259a5f4374aSIngo Molnar 260a5f4374aSIngo Molnar /* 261a5f4374aSIngo Molnar * lock page will serialize against do_wp_page()'s 262a5f4374aSIngo Molnar * PageAnon() handling 263a5f4374aSIngo Molnar */ 264a5f4374aSIngo Molnar lock_page(old_page); 265a5f4374aSIngo Molnar /* copy the page now that we've got it stable */ 266a5f4374aSIngo Molnar vaddr_old = kmap_atomic(old_page); 267a5f4374aSIngo Molnar vaddr_new = kmap_atomic(new_page); 268a5f4374aSIngo Molnar 269a5f4374aSIngo Molnar memcpy(vaddr_new, vaddr_old, PAGE_SIZE); 270a5f4374aSIngo Molnar 271a5f4374aSIngo Molnar /* poke the new insn in, ASSUMES we don't cross page boundary */ 272a5f4374aSIngo Molnar vaddr &= ~PAGE_MASK; 2735cb4ac3aSSrikar Dronamraju BUG_ON(vaddr + UPROBE_SWBP_INSN_SIZE > PAGE_SIZE); 2745cb4ac3aSSrikar Dronamraju memcpy(vaddr_new + vaddr, &opcode, UPROBE_SWBP_INSN_SIZE); 275a5f4374aSIngo Molnar 276a5f4374aSIngo Molnar kunmap_atomic(vaddr_new); 277a5f4374aSIngo Molnar kunmap_atomic(vaddr_old); 278a5f4374aSIngo Molnar 279a5f4374aSIngo Molnar ret = anon_vma_prepare(vma); 280a5f4374aSIngo Molnar if (ret) 281a5f4374aSIngo Molnar goto unlock_out; 282a5f4374aSIngo Molnar 283a5f4374aSIngo Molnar lock_page(new_page); 284a5f4374aSIngo Molnar ret = __replace_page(vma, old_page, new_page); 285a5f4374aSIngo Molnar unlock_page(new_page); 286a5f4374aSIngo Molnar 287a5f4374aSIngo Molnar unlock_out: 288a5f4374aSIngo Molnar unlock_page(old_page); 289a5f4374aSIngo Molnar page_cache_release(new_page); 290a5f4374aSIngo Molnar 291a5f4374aSIngo Molnar put_out: 292a5f4374aSIngo Molnar put_page(old_page); 293a5f4374aSIngo Molnar 294a5f4374aSIngo Molnar return ret; 295a5f4374aSIngo Molnar } 296a5f4374aSIngo Molnar 297a5f4374aSIngo Molnar /** 298a5f4374aSIngo Molnar * read_opcode - read the opcode at a given virtual address. 299a5f4374aSIngo Molnar * @mm: the probed process address space. 300a5f4374aSIngo Molnar * @vaddr: the virtual address to read the opcode. 301a5f4374aSIngo Molnar * @opcode: location to store the read opcode. 302a5f4374aSIngo Molnar * 303a5f4374aSIngo Molnar * Called with mm->mmap_sem held (for read and with a reference to 304a5f4374aSIngo Molnar * mm. 305a5f4374aSIngo Molnar * 306a5f4374aSIngo Molnar * For mm @mm, read the opcode at @vaddr and store it in @opcode. 307a5f4374aSIngo Molnar * Return 0 (success) or a negative errno. 308a5f4374aSIngo Molnar */ 309a5f4374aSIngo Molnar static int read_opcode(struct mm_struct *mm, unsigned long vaddr, uprobe_opcode_t *opcode) 310a5f4374aSIngo Molnar { 311a5f4374aSIngo Molnar struct page *page; 312a5f4374aSIngo Molnar void *vaddr_new; 313a5f4374aSIngo Molnar int ret; 314a5f4374aSIngo Molnar 315a3d7bb47SOleg Nesterov ret = get_user_pages(NULL, mm, vaddr, 1, 0, 1, &page, NULL); 316a5f4374aSIngo Molnar if (ret <= 0) 317a5f4374aSIngo Molnar return ret; 318a5f4374aSIngo Molnar 319a5f4374aSIngo Molnar lock_page(page); 320a5f4374aSIngo Molnar vaddr_new = kmap_atomic(page); 321a5f4374aSIngo Molnar vaddr &= ~PAGE_MASK; 3225cb4ac3aSSrikar Dronamraju memcpy(opcode, vaddr_new + vaddr, UPROBE_SWBP_INSN_SIZE); 323a5f4374aSIngo Molnar kunmap_atomic(vaddr_new); 324a5f4374aSIngo Molnar unlock_page(page); 325a5f4374aSIngo Molnar 326a5f4374aSIngo Molnar put_page(page); 327a5f4374aSIngo Molnar 328a5f4374aSIngo Molnar return 0; 329a5f4374aSIngo Molnar } 330a5f4374aSIngo Molnar 3315cb4ac3aSSrikar Dronamraju static int is_swbp_at_addr(struct mm_struct *mm, unsigned long vaddr) 332a5f4374aSIngo Molnar { 333a5f4374aSIngo Molnar uprobe_opcode_t opcode; 334a5f4374aSIngo Molnar int result; 335a5f4374aSIngo Molnar 336c00b2750SOleg Nesterov if (current->mm == mm) { 337c00b2750SOleg Nesterov pagefault_disable(); 338c00b2750SOleg Nesterov result = __copy_from_user_inatomic(&opcode, (void __user*)vaddr, 339c00b2750SOleg Nesterov sizeof(opcode)); 340c00b2750SOleg Nesterov pagefault_enable(); 341c00b2750SOleg Nesterov 342c00b2750SOleg Nesterov if (likely(result == 0)) 343c00b2750SOleg Nesterov goto out; 344c00b2750SOleg Nesterov } 345c00b2750SOleg Nesterov 346a5f4374aSIngo Molnar result = read_opcode(mm, vaddr, &opcode); 347a5f4374aSIngo Molnar if (result) 348a5f4374aSIngo Molnar return result; 349c00b2750SOleg Nesterov out: 3505cb4ac3aSSrikar Dronamraju if (is_swbp_insn(&opcode)) 351a5f4374aSIngo Molnar return 1; 352a5f4374aSIngo Molnar 353a5f4374aSIngo Molnar return 0; 354a5f4374aSIngo Molnar } 355a5f4374aSIngo Molnar 356a5f4374aSIngo Molnar /** 3575cb4ac3aSSrikar Dronamraju * set_swbp - store breakpoint at a given address. 358e3343e6aSSrikar Dronamraju * @auprobe: arch specific probepoint information. 359a5f4374aSIngo Molnar * @mm: the probed process address space. 360a5f4374aSIngo Molnar * @vaddr: the virtual address to insert the opcode. 361a5f4374aSIngo Molnar * 362a5f4374aSIngo Molnar * For mm @mm, store the breakpoint instruction at @vaddr. 363a5f4374aSIngo Molnar * Return 0 (success) or a negative errno. 364a5f4374aSIngo Molnar */ 3655cb4ac3aSSrikar Dronamraju int __weak set_swbp(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned long vaddr) 366a5f4374aSIngo Molnar { 367a5f4374aSIngo Molnar int result; 368a5f4374aSIngo Molnar 3695cb4ac3aSSrikar Dronamraju result = is_swbp_at_addr(mm, vaddr); 370a5f4374aSIngo Molnar if (result == 1) 371a5f4374aSIngo Molnar return -EEXIST; 372a5f4374aSIngo Molnar 373a5f4374aSIngo Molnar if (result) 374a5f4374aSIngo Molnar return result; 375a5f4374aSIngo Molnar 3765cb4ac3aSSrikar Dronamraju return write_opcode(auprobe, mm, vaddr, UPROBE_SWBP_INSN); 377a5f4374aSIngo Molnar } 378a5f4374aSIngo Molnar 379a5f4374aSIngo Molnar /** 380a5f4374aSIngo Molnar * set_orig_insn - Restore the original instruction. 381a5f4374aSIngo Molnar * @mm: the probed process address space. 382e3343e6aSSrikar Dronamraju * @auprobe: arch specific probepoint information. 383a5f4374aSIngo Molnar * @vaddr: the virtual address to insert the opcode. 384a5f4374aSIngo Molnar * @verify: if true, verify existance of breakpoint instruction. 385a5f4374aSIngo Molnar * 386a5f4374aSIngo Molnar * For mm @mm, restore the original opcode (opcode) at @vaddr. 387a5f4374aSIngo Molnar * Return 0 (success) or a negative errno. 388a5f4374aSIngo Molnar */ 389a5f4374aSIngo Molnar int __weak 390e3343e6aSSrikar Dronamraju set_orig_insn(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned long vaddr, bool verify) 391a5f4374aSIngo Molnar { 392a5f4374aSIngo Molnar if (verify) { 393a5f4374aSIngo Molnar int result; 394a5f4374aSIngo Molnar 3955cb4ac3aSSrikar Dronamraju result = is_swbp_at_addr(mm, vaddr); 396a5f4374aSIngo Molnar if (!result) 397a5f4374aSIngo Molnar return -EINVAL; 398a5f4374aSIngo Molnar 399a5f4374aSIngo Molnar if (result != 1) 400a5f4374aSIngo Molnar return result; 401a5f4374aSIngo Molnar } 402e3343e6aSSrikar Dronamraju return write_opcode(auprobe, mm, vaddr, *(uprobe_opcode_t *)auprobe->insn); 403a5f4374aSIngo Molnar } 404a5f4374aSIngo Molnar 405a5f4374aSIngo Molnar static int match_uprobe(struct uprobe *l, struct uprobe *r) 406a5f4374aSIngo Molnar { 407a5f4374aSIngo Molnar if (l->inode < r->inode) 408a5f4374aSIngo Molnar return -1; 409a5f4374aSIngo Molnar 410a5f4374aSIngo Molnar if (l->inode > r->inode) 411a5f4374aSIngo Molnar return 1; 412a5f4374aSIngo Molnar 413a5f4374aSIngo Molnar if (l->offset < r->offset) 414a5f4374aSIngo Molnar return -1; 415a5f4374aSIngo Molnar 416a5f4374aSIngo Molnar if (l->offset > r->offset) 417a5f4374aSIngo Molnar return 1; 418a5f4374aSIngo Molnar 419a5f4374aSIngo Molnar return 0; 420a5f4374aSIngo Molnar } 421a5f4374aSIngo Molnar 422a5f4374aSIngo Molnar static struct uprobe *__find_uprobe(struct inode *inode, loff_t offset) 423a5f4374aSIngo Molnar { 424a5f4374aSIngo Molnar struct uprobe u = { .inode = inode, .offset = offset }; 425a5f4374aSIngo Molnar struct rb_node *n = uprobes_tree.rb_node; 426a5f4374aSIngo Molnar struct uprobe *uprobe; 427a5f4374aSIngo Molnar int match; 428a5f4374aSIngo Molnar 429a5f4374aSIngo Molnar while (n) { 430a5f4374aSIngo Molnar uprobe = rb_entry(n, struct uprobe, rb_node); 431a5f4374aSIngo Molnar match = match_uprobe(&u, uprobe); 432a5f4374aSIngo Molnar if (!match) { 433a5f4374aSIngo Molnar atomic_inc(&uprobe->ref); 434a5f4374aSIngo Molnar return uprobe; 435a5f4374aSIngo Molnar } 436a5f4374aSIngo Molnar 437a5f4374aSIngo Molnar if (match < 0) 438a5f4374aSIngo Molnar n = n->rb_left; 439a5f4374aSIngo Molnar else 440a5f4374aSIngo Molnar n = n->rb_right; 441a5f4374aSIngo Molnar } 442a5f4374aSIngo Molnar return NULL; 443a5f4374aSIngo Molnar } 444a5f4374aSIngo Molnar 445a5f4374aSIngo Molnar /* 446a5f4374aSIngo Molnar * Find a uprobe corresponding to a given inode:offset 447a5f4374aSIngo Molnar * Acquires uprobes_treelock 448a5f4374aSIngo Molnar */ 449a5f4374aSIngo Molnar static struct uprobe *find_uprobe(struct inode *inode, loff_t offset) 450a5f4374aSIngo Molnar { 451a5f4374aSIngo Molnar struct uprobe *uprobe; 452a5f4374aSIngo Molnar unsigned long flags; 453a5f4374aSIngo Molnar 454a5f4374aSIngo Molnar spin_lock_irqsave(&uprobes_treelock, flags); 455a5f4374aSIngo Molnar uprobe = __find_uprobe(inode, offset); 456a5f4374aSIngo Molnar spin_unlock_irqrestore(&uprobes_treelock, flags); 457a5f4374aSIngo Molnar 458a5f4374aSIngo Molnar return uprobe; 459a5f4374aSIngo Molnar } 460a5f4374aSIngo Molnar 461a5f4374aSIngo Molnar static struct uprobe *__insert_uprobe(struct uprobe *uprobe) 462a5f4374aSIngo Molnar { 463a5f4374aSIngo Molnar struct rb_node **p = &uprobes_tree.rb_node; 464a5f4374aSIngo Molnar struct rb_node *parent = NULL; 465a5f4374aSIngo Molnar struct uprobe *u; 466a5f4374aSIngo Molnar int match; 467a5f4374aSIngo Molnar 468a5f4374aSIngo Molnar while (*p) { 469a5f4374aSIngo Molnar parent = *p; 470a5f4374aSIngo Molnar u = rb_entry(parent, struct uprobe, rb_node); 471a5f4374aSIngo Molnar match = match_uprobe(uprobe, u); 472a5f4374aSIngo Molnar if (!match) { 473a5f4374aSIngo Molnar atomic_inc(&u->ref); 474a5f4374aSIngo Molnar return u; 475a5f4374aSIngo Molnar } 476a5f4374aSIngo Molnar 477a5f4374aSIngo Molnar if (match < 0) 478a5f4374aSIngo Molnar p = &parent->rb_left; 479a5f4374aSIngo Molnar else 480a5f4374aSIngo Molnar p = &parent->rb_right; 481a5f4374aSIngo Molnar 482a5f4374aSIngo Molnar } 483a5f4374aSIngo Molnar 484a5f4374aSIngo Molnar u = NULL; 485a5f4374aSIngo Molnar rb_link_node(&uprobe->rb_node, parent, p); 486a5f4374aSIngo Molnar rb_insert_color(&uprobe->rb_node, &uprobes_tree); 487a5f4374aSIngo Molnar /* get access + creation ref */ 488a5f4374aSIngo Molnar atomic_set(&uprobe->ref, 2); 489a5f4374aSIngo Molnar 490a5f4374aSIngo Molnar return u; 491a5f4374aSIngo Molnar } 492a5f4374aSIngo Molnar 493a5f4374aSIngo Molnar /* 494a5f4374aSIngo Molnar * Acquire uprobes_treelock. 495a5f4374aSIngo Molnar * Matching uprobe already exists in rbtree; 496a5f4374aSIngo Molnar * increment (access refcount) and return the matching uprobe. 497a5f4374aSIngo Molnar * 498a5f4374aSIngo Molnar * No matching uprobe; insert the uprobe in rb_tree; 499a5f4374aSIngo Molnar * get a double refcount (access + creation) and return NULL. 500a5f4374aSIngo Molnar */ 501a5f4374aSIngo Molnar static struct uprobe *insert_uprobe(struct uprobe *uprobe) 502a5f4374aSIngo Molnar { 503a5f4374aSIngo Molnar unsigned long flags; 504a5f4374aSIngo Molnar struct uprobe *u; 505a5f4374aSIngo Molnar 506a5f4374aSIngo Molnar spin_lock_irqsave(&uprobes_treelock, flags); 507a5f4374aSIngo Molnar u = __insert_uprobe(uprobe); 508a5f4374aSIngo Molnar spin_unlock_irqrestore(&uprobes_treelock, flags); 509a5f4374aSIngo Molnar 5100326f5a9SSrikar Dronamraju /* For now assume that the instruction need not be single-stepped */ 5110326f5a9SSrikar Dronamraju uprobe->flags |= UPROBE_SKIP_SSTEP; 5120326f5a9SSrikar Dronamraju 513a5f4374aSIngo Molnar return u; 514a5f4374aSIngo Molnar } 515a5f4374aSIngo Molnar 516a5f4374aSIngo Molnar static void put_uprobe(struct uprobe *uprobe) 517a5f4374aSIngo Molnar { 518a5f4374aSIngo Molnar if (atomic_dec_and_test(&uprobe->ref)) 519a5f4374aSIngo Molnar kfree(uprobe); 520a5f4374aSIngo Molnar } 521a5f4374aSIngo Molnar 522a5f4374aSIngo Molnar static struct uprobe *alloc_uprobe(struct inode *inode, loff_t offset) 523a5f4374aSIngo Molnar { 524a5f4374aSIngo Molnar struct uprobe *uprobe, *cur_uprobe; 525a5f4374aSIngo Molnar 526a5f4374aSIngo Molnar uprobe = kzalloc(sizeof(struct uprobe), GFP_KERNEL); 527a5f4374aSIngo Molnar if (!uprobe) 528a5f4374aSIngo Molnar return NULL; 529a5f4374aSIngo Molnar 530a5f4374aSIngo Molnar uprobe->inode = igrab(inode); 531a5f4374aSIngo Molnar uprobe->offset = offset; 532a5f4374aSIngo Molnar init_rwsem(&uprobe->consumer_rwsem); 533a5f4374aSIngo Molnar INIT_LIST_HEAD(&uprobe->pending_list); 534a5f4374aSIngo Molnar 535a5f4374aSIngo Molnar /* add to uprobes_tree, sorted on inode:offset */ 536a5f4374aSIngo Molnar cur_uprobe = insert_uprobe(uprobe); 537a5f4374aSIngo Molnar 538a5f4374aSIngo Molnar /* a uprobe exists for this inode:offset combination */ 539a5f4374aSIngo Molnar if (cur_uprobe) { 540a5f4374aSIngo Molnar kfree(uprobe); 541a5f4374aSIngo Molnar uprobe = cur_uprobe; 542a5f4374aSIngo Molnar iput(inode); 543a5f4374aSIngo Molnar } else { 544a5f4374aSIngo Molnar atomic_inc(&uprobe_events); 545a5f4374aSIngo Molnar } 546a5f4374aSIngo Molnar 547a5f4374aSIngo Molnar return uprobe; 548a5f4374aSIngo Molnar } 549a5f4374aSIngo Molnar 5500326f5a9SSrikar Dronamraju static void handler_chain(struct uprobe *uprobe, struct pt_regs *regs) 5510326f5a9SSrikar Dronamraju { 5520326f5a9SSrikar Dronamraju struct uprobe_consumer *uc; 5530326f5a9SSrikar Dronamraju 5540326f5a9SSrikar Dronamraju if (!(uprobe->flags & UPROBE_RUN_HANDLER)) 5550326f5a9SSrikar Dronamraju return; 5560326f5a9SSrikar Dronamraju 5570326f5a9SSrikar Dronamraju down_read(&uprobe->consumer_rwsem); 5580326f5a9SSrikar Dronamraju for (uc = uprobe->consumers; uc; uc = uc->next) { 5590326f5a9SSrikar Dronamraju if (!uc->filter || uc->filter(uc, current)) 5600326f5a9SSrikar Dronamraju uc->handler(uc, regs); 5610326f5a9SSrikar Dronamraju } 5620326f5a9SSrikar Dronamraju up_read(&uprobe->consumer_rwsem); 5630326f5a9SSrikar Dronamraju } 5640326f5a9SSrikar Dronamraju 565a5f4374aSIngo Molnar /* Returns the previous consumer */ 566a5f4374aSIngo Molnar static struct uprobe_consumer * 567e3343e6aSSrikar Dronamraju consumer_add(struct uprobe *uprobe, struct uprobe_consumer *uc) 568a5f4374aSIngo Molnar { 569a5f4374aSIngo Molnar down_write(&uprobe->consumer_rwsem); 570e3343e6aSSrikar Dronamraju uc->next = uprobe->consumers; 571e3343e6aSSrikar Dronamraju uprobe->consumers = uc; 572a5f4374aSIngo Molnar up_write(&uprobe->consumer_rwsem); 573a5f4374aSIngo Molnar 574e3343e6aSSrikar Dronamraju return uc->next; 575a5f4374aSIngo Molnar } 576a5f4374aSIngo Molnar 577a5f4374aSIngo Molnar /* 578e3343e6aSSrikar Dronamraju * For uprobe @uprobe, delete the consumer @uc. 579e3343e6aSSrikar Dronamraju * Return true if the @uc is deleted successfully 580a5f4374aSIngo Molnar * or return false. 581a5f4374aSIngo Molnar */ 582e3343e6aSSrikar Dronamraju static bool consumer_del(struct uprobe *uprobe, struct uprobe_consumer *uc) 583a5f4374aSIngo Molnar { 584a5f4374aSIngo Molnar struct uprobe_consumer **con; 585a5f4374aSIngo Molnar bool ret = false; 586a5f4374aSIngo Molnar 587a5f4374aSIngo Molnar down_write(&uprobe->consumer_rwsem); 588a5f4374aSIngo Molnar for (con = &uprobe->consumers; *con; con = &(*con)->next) { 589e3343e6aSSrikar Dronamraju if (*con == uc) { 590e3343e6aSSrikar Dronamraju *con = uc->next; 591a5f4374aSIngo Molnar ret = true; 592a5f4374aSIngo Molnar break; 593a5f4374aSIngo Molnar } 594a5f4374aSIngo Molnar } 595a5f4374aSIngo Molnar up_write(&uprobe->consumer_rwsem); 596a5f4374aSIngo Molnar 597a5f4374aSIngo Molnar return ret; 598a5f4374aSIngo Molnar } 599a5f4374aSIngo Molnar 600e3343e6aSSrikar Dronamraju static int 601e3343e6aSSrikar Dronamraju __copy_insn(struct address_space *mapping, struct vm_area_struct *vma, char *insn, 602a5f4374aSIngo Molnar unsigned long nbytes, unsigned long offset) 603a5f4374aSIngo Molnar { 604a5f4374aSIngo Molnar struct file *filp = vma->vm_file; 605a5f4374aSIngo Molnar struct page *page; 606a5f4374aSIngo Molnar void *vaddr; 607a5f4374aSIngo Molnar unsigned long off1; 608a5f4374aSIngo Molnar unsigned long idx; 609a5f4374aSIngo Molnar 610a5f4374aSIngo Molnar if (!filp) 611a5f4374aSIngo Molnar return -EINVAL; 612a5f4374aSIngo Molnar 613cc359d18SOleg Nesterov if (!mapping->a_ops->readpage) 614cc359d18SOleg Nesterov return -EIO; 615cc359d18SOleg Nesterov 616a5f4374aSIngo Molnar idx = (unsigned long)(offset >> PAGE_CACHE_SHIFT); 617a5f4374aSIngo Molnar off1 = offset &= ~PAGE_MASK; 618a5f4374aSIngo Molnar 619a5f4374aSIngo Molnar /* 620a5f4374aSIngo Molnar * Ensure that the page that has the original instruction is 621a5f4374aSIngo Molnar * populated and in page-cache. 622a5f4374aSIngo Molnar */ 623a5f4374aSIngo Molnar page = read_mapping_page(mapping, idx, filp); 624a5f4374aSIngo Molnar if (IS_ERR(page)) 625a5f4374aSIngo Molnar return PTR_ERR(page); 626a5f4374aSIngo Molnar 627a5f4374aSIngo Molnar vaddr = kmap_atomic(page); 628a5f4374aSIngo Molnar memcpy(insn, vaddr + off1, nbytes); 629a5f4374aSIngo Molnar kunmap_atomic(vaddr); 630a5f4374aSIngo Molnar page_cache_release(page); 631a5f4374aSIngo Molnar 632a5f4374aSIngo Molnar return 0; 633a5f4374aSIngo Molnar } 634a5f4374aSIngo Molnar 635e3343e6aSSrikar Dronamraju static int 636e3343e6aSSrikar Dronamraju copy_insn(struct uprobe *uprobe, struct vm_area_struct *vma, unsigned long addr) 637a5f4374aSIngo Molnar { 638a5f4374aSIngo Molnar struct address_space *mapping; 639a5f4374aSIngo Molnar unsigned long nbytes; 640a5f4374aSIngo Molnar int bytes; 641a5f4374aSIngo Molnar 642a5f4374aSIngo Molnar addr &= ~PAGE_MASK; 643a5f4374aSIngo Molnar nbytes = PAGE_SIZE - addr; 644a5f4374aSIngo Molnar mapping = uprobe->inode->i_mapping; 645a5f4374aSIngo Molnar 646a5f4374aSIngo Molnar /* Instruction at end of binary; copy only available bytes */ 647a5f4374aSIngo Molnar if (uprobe->offset + MAX_UINSN_BYTES > uprobe->inode->i_size) 648a5f4374aSIngo Molnar bytes = uprobe->inode->i_size - uprobe->offset; 649a5f4374aSIngo Molnar else 650a5f4374aSIngo Molnar bytes = MAX_UINSN_BYTES; 651a5f4374aSIngo Molnar 652a5f4374aSIngo Molnar /* Instruction at the page-boundary; copy bytes in second page */ 653a5f4374aSIngo Molnar if (nbytes < bytes) { 6543ff54efdSSrikar Dronamraju if (__copy_insn(mapping, vma, uprobe->arch.insn + nbytes, 655a5f4374aSIngo Molnar bytes - nbytes, uprobe->offset + nbytes)) 656a5f4374aSIngo Molnar return -ENOMEM; 657a5f4374aSIngo Molnar 658a5f4374aSIngo Molnar bytes = nbytes; 659a5f4374aSIngo Molnar } 6603ff54efdSSrikar Dronamraju return __copy_insn(mapping, vma, uprobe->arch.insn, bytes, uprobe->offset); 661a5f4374aSIngo Molnar } 662a5f4374aSIngo Molnar 663682968e0SSrikar Dronamraju /* 664682968e0SSrikar Dronamraju * How mm->uprobes_state.count gets updated 665682968e0SSrikar Dronamraju * uprobe_mmap() increments the count if 666682968e0SSrikar Dronamraju * - it successfully adds a breakpoint. 667682968e0SSrikar Dronamraju * - it cannot add a breakpoint, but sees that there is a underlying 668682968e0SSrikar Dronamraju * breakpoint (via a is_swbp_at_addr()). 669682968e0SSrikar Dronamraju * 670682968e0SSrikar Dronamraju * uprobe_munmap() decrements the count if 671682968e0SSrikar Dronamraju * - it sees a underlying breakpoint, (via is_swbp_at_addr) 672682968e0SSrikar Dronamraju * (Subsequent uprobe_unregister wouldnt find the breakpoint 673682968e0SSrikar Dronamraju * unless a uprobe_mmap kicks in, since the old vma would be 674682968e0SSrikar Dronamraju * dropped just after uprobe_munmap.) 675682968e0SSrikar Dronamraju * 676682968e0SSrikar Dronamraju * uprobe_register increments the count if: 677682968e0SSrikar Dronamraju * - it successfully adds a breakpoint. 678682968e0SSrikar Dronamraju * 679682968e0SSrikar Dronamraju * uprobe_unregister decrements the count if: 680682968e0SSrikar Dronamraju * - it sees a underlying breakpoint and removes successfully. 681682968e0SSrikar Dronamraju * (via is_swbp_at_addr) 682682968e0SSrikar Dronamraju * (Subsequent uprobe_munmap wouldnt find the breakpoint 683682968e0SSrikar Dronamraju * since there is no underlying breakpoint after the 684682968e0SSrikar Dronamraju * breakpoint removal.) 685682968e0SSrikar Dronamraju */ 686e3343e6aSSrikar Dronamraju static int 687e3343e6aSSrikar Dronamraju install_breakpoint(struct uprobe *uprobe, struct mm_struct *mm, 688a5f4374aSIngo Molnar struct vm_area_struct *vma, loff_t vaddr) 689a5f4374aSIngo Molnar { 690a5f4374aSIngo Molnar unsigned long addr; 691a5f4374aSIngo Molnar int ret; 692a5f4374aSIngo Molnar 693a5f4374aSIngo Molnar /* 694a5f4374aSIngo Molnar * If probe is being deleted, unregister thread could be done with 695a5f4374aSIngo Molnar * the vma-rmap-walk through. Adding a probe now can be fatal since 696a5f4374aSIngo Molnar * nobody will be able to cleanup. Also we could be from fork or 697a5f4374aSIngo Molnar * mremap path, where the probe might have already been inserted. 698a5f4374aSIngo Molnar * Hence behave as if probe already existed. 699a5f4374aSIngo Molnar */ 700a5f4374aSIngo Molnar if (!uprobe->consumers) 701a5f4374aSIngo Molnar return -EEXIST; 702a5f4374aSIngo Molnar 703a5f4374aSIngo Molnar addr = (unsigned long)vaddr; 704a5f4374aSIngo Molnar 705900771a4SSrikar Dronamraju if (!(uprobe->flags & UPROBE_COPY_INSN)) { 706a5f4374aSIngo Molnar ret = copy_insn(uprobe, vma, addr); 707a5f4374aSIngo Molnar if (ret) 708a5f4374aSIngo Molnar return ret; 709a5f4374aSIngo Molnar 7105cb4ac3aSSrikar Dronamraju if (is_swbp_insn((uprobe_opcode_t *)uprobe->arch.insn)) 711a5f4374aSIngo Molnar return -EEXIST; 712a5f4374aSIngo Molnar 7137eb9ba5eSAnanth N Mavinakayanahalli ret = arch_uprobe_analyze_insn(&uprobe->arch, mm, addr); 714a5f4374aSIngo Molnar if (ret) 715a5f4374aSIngo Molnar return ret; 716a5f4374aSIngo Molnar 717900771a4SSrikar Dronamraju uprobe->flags |= UPROBE_COPY_INSN; 718a5f4374aSIngo Molnar } 719682968e0SSrikar Dronamraju 720682968e0SSrikar Dronamraju /* 721682968e0SSrikar Dronamraju * Ideally, should be updating the probe count after the breakpoint 722682968e0SSrikar Dronamraju * has been successfully inserted. However a thread could hit the 723682968e0SSrikar Dronamraju * breakpoint we just inserted even before the probe count is 724682968e0SSrikar Dronamraju * incremented. If this is the first breakpoint placed, breakpoint 725682968e0SSrikar Dronamraju * notifier might ignore uprobes and pass the trap to the thread. 726682968e0SSrikar Dronamraju * Hence increment before and decrement on failure. 727682968e0SSrikar Dronamraju */ 728682968e0SSrikar Dronamraju atomic_inc(&mm->uprobes_state.count); 7295cb4ac3aSSrikar Dronamraju ret = set_swbp(&uprobe->arch, mm, addr); 730682968e0SSrikar Dronamraju if (ret) 731682968e0SSrikar Dronamraju atomic_dec(&mm->uprobes_state.count); 732a5f4374aSIngo Molnar 733a5f4374aSIngo Molnar return ret; 734a5f4374aSIngo Molnar } 735a5f4374aSIngo Molnar 736e3343e6aSSrikar Dronamraju static void 737e3343e6aSSrikar Dronamraju remove_breakpoint(struct uprobe *uprobe, struct mm_struct *mm, loff_t vaddr) 738a5f4374aSIngo Molnar { 739682968e0SSrikar Dronamraju if (!set_orig_insn(&uprobe->arch, mm, (unsigned long)vaddr, true)) 740682968e0SSrikar Dronamraju atomic_dec(&mm->uprobes_state.count); 741a5f4374aSIngo Molnar } 742a5f4374aSIngo Molnar 7430326f5a9SSrikar Dronamraju /* 744778b032dSOleg Nesterov * There could be threads that have already hit the breakpoint. They 745778b032dSOleg Nesterov * will recheck the current insn and restart if find_uprobe() fails. 746778b032dSOleg Nesterov * See find_active_uprobe(). 7470326f5a9SSrikar Dronamraju */ 748a5f4374aSIngo Molnar static void delete_uprobe(struct uprobe *uprobe) 749a5f4374aSIngo Molnar { 750a5f4374aSIngo Molnar unsigned long flags; 751a5f4374aSIngo Molnar 752a5f4374aSIngo Molnar spin_lock_irqsave(&uprobes_treelock, flags); 753a5f4374aSIngo Molnar rb_erase(&uprobe->rb_node, &uprobes_tree); 754a5f4374aSIngo Molnar spin_unlock_irqrestore(&uprobes_treelock, flags); 755a5f4374aSIngo Molnar iput(uprobe->inode); 756a5f4374aSIngo Molnar put_uprobe(uprobe); 757a5f4374aSIngo Molnar atomic_dec(&uprobe_events); 758a5f4374aSIngo Molnar } 759a5f4374aSIngo Molnar 760e3343e6aSSrikar Dronamraju static struct vma_info * 761e3343e6aSSrikar Dronamraju __find_next_vma_info(struct address_space *mapping, struct list_head *head, 762e3343e6aSSrikar Dronamraju struct vma_info *vi, loff_t offset, bool is_register) 763a5f4374aSIngo Molnar { 764a5f4374aSIngo Molnar struct prio_tree_iter iter; 765a5f4374aSIngo Molnar struct vm_area_struct *vma; 766a5f4374aSIngo Molnar struct vma_info *tmpvi; 767a5f4374aSIngo Molnar unsigned long pgoff; 768a5f4374aSIngo Molnar int existing_vma; 769a5f4374aSIngo Molnar loff_t vaddr; 770a5f4374aSIngo Molnar 771a5f4374aSIngo Molnar pgoff = offset >> PAGE_SHIFT; 772a5f4374aSIngo Molnar 773a5f4374aSIngo Molnar vma_prio_tree_foreach(vma, &iter, &mapping->i_mmap, pgoff, pgoff) { 774a5f4374aSIngo Molnar if (!valid_vma(vma, is_register)) 775a5f4374aSIngo Molnar continue; 776a5f4374aSIngo Molnar 777a5f4374aSIngo Molnar existing_vma = 0; 778a5f4374aSIngo Molnar vaddr = vma_address(vma, offset); 779a5f4374aSIngo Molnar 780a5f4374aSIngo Molnar list_for_each_entry(tmpvi, head, probe_list) { 781a5f4374aSIngo Molnar if (tmpvi->mm == vma->vm_mm && tmpvi->vaddr == vaddr) { 782a5f4374aSIngo Molnar existing_vma = 1; 783a5f4374aSIngo Molnar break; 784a5f4374aSIngo Molnar } 785a5f4374aSIngo Molnar } 786a5f4374aSIngo Molnar 787a5f4374aSIngo Molnar /* 788a5f4374aSIngo Molnar * Another vma needs a probe to be installed. However skip 789a5f4374aSIngo Molnar * installing the probe if the vma is about to be unlinked. 790a5f4374aSIngo Molnar */ 791a5f4374aSIngo Molnar if (!existing_vma && atomic_inc_not_zero(&vma->vm_mm->mm_users)) { 792a5f4374aSIngo Molnar vi->mm = vma->vm_mm; 793a5f4374aSIngo Molnar vi->vaddr = vaddr; 794a5f4374aSIngo Molnar list_add(&vi->probe_list, head); 795a5f4374aSIngo Molnar 796a5f4374aSIngo Molnar return vi; 797a5f4374aSIngo Molnar } 798a5f4374aSIngo Molnar } 799a5f4374aSIngo Molnar 800a5f4374aSIngo Molnar return NULL; 801a5f4374aSIngo Molnar } 802a5f4374aSIngo Molnar 803a5f4374aSIngo Molnar /* 804a5f4374aSIngo Molnar * Iterate in the rmap prio tree and find a vma where a probe has not 805a5f4374aSIngo Molnar * yet been inserted. 806a5f4374aSIngo Molnar */ 807a5f4374aSIngo Molnar static struct vma_info * 808e3343e6aSSrikar Dronamraju find_next_vma_info(struct address_space *mapping, struct list_head *head, 809e3343e6aSSrikar Dronamraju loff_t offset, bool is_register) 810a5f4374aSIngo Molnar { 811a5f4374aSIngo Molnar struct vma_info *vi, *retvi; 812a5f4374aSIngo Molnar 813a5f4374aSIngo Molnar vi = kzalloc(sizeof(struct vma_info), GFP_KERNEL); 814a5f4374aSIngo Molnar if (!vi) 815a5f4374aSIngo Molnar return ERR_PTR(-ENOMEM); 816a5f4374aSIngo Molnar 817a5f4374aSIngo Molnar mutex_lock(&mapping->i_mmap_mutex); 818e3343e6aSSrikar Dronamraju retvi = __find_next_vma_info(mapping, head, vi, offset, is_register); 819a5f4374aSIngo Molnar mutex_unlock(&mapping->i_mmap_mutex); 820a5f4374aSIngo Molnar 821a5f4374aSIngo Molnar if (!retvi) 822a5f4374aSIngo Molnar kfree(vi); 823a5f4374aSIngo Molnar 824a5f4374aSIngo Molnar return retvi; 825a5f4374aSIngo Molnar } 826a5f4374aSIngo Molnar 827a5f4374aSIngo Molnar static int register_for_each_vma(struct uprobe *uprobe, bool is_register) 828a5f4374aSIngo Molnar { 829a5f4374aSIngo Molnar struct list_head try_list; 830a5f4374aSIngo Molnar struct vm_area_struct *vma; 831a5f4374aSIngo Molnar struct address_space *mapping; 832a5f4374aSIngo Molnar struct vma_info *vi, *tmpvi; 833a5f4374aSIngo Molnar struct mm_struct *mm; 834a5f4374aSIngo Molnar loff_t vaddr; 835a5f4374aSIngo Molnar int ret; 836a5f4374aSIngo Molnar 837a5f4374aSIngo Molnar mapping = uprobe->inode->i_mapping; 838a5f4374aSIngo Molnar INIT_LIST_HEAD(&try_list); 839a5f4374aSIngo Molnar 840a5f4374aSIngo Molnar ret = 0; 841a5f4374aSIngo Molnar 842a5f4374aSIngo Molnar for (;;) { 843e3343e6aSSrikar Dronamraju vi = find_next_vma_info(mapping, &try_list, uprobe->offset, is_register); 844a5f4374aSIngo Molnar if (!vi) 845a5f4374aSIngo Molnar break; 846a5f4374aSIngo Molnar 847a5f4374aSIngo Molnar if (IS_ERR(vi)) { 848a5f4374aSIngo Molnar ret = PTR_ERR(vi); 849a5f4374aSIngo Molnar break; 850a5f4374aSIngo Molnar } 851a5f4374aSIngo Molnar 852a5f4374aSIngo Molnar mm = vi->mm; 85377fc4af1SOleg Nesterov down_write(&mm->mmap_sem); 854a5f4374aSIngo Molnar vma = find_vma(mm, (unsigned long)vi->vaddr); 855a5f4374aSIngo Molnar if (!vma || !valid_vma(vma, is_register)) { 856a5f4374aSIngo Molnar list_del(&vi->probe_list); 857a5f4374aSIngo Molnar kfree(vi); 85877fc4af1SOleg Nesterov up_write(&mm->mmap_sem); 859a5f4374aSIngo Molnar mmput(mm); 860a5f4374aSIngo Molnar continue; 861a5f4374aSIngo Molnar } 862a5f4374aSIngo Molnar vaddr = vma_address(vma, uprobe->offset); 863a5f4374aSIngo Molnar if (vma->vm_file->f_mapping->host != uprobe->inode || 864a5f4374aSIngo Molnar vaddr != vi->vaddr) { 865a5f4374aSIngo Molnar list_del(&vi->probe_list); 866a5f4374aSIngo Molnar kfree(vi); 86777fc4af1SOleg Nesterov up_write(&mm->mmap_sem); 868a5f4374aSIngo Molnar mmput(mm); 869a5f4374aSIngo Molnar continue; 870a5f4374aSIngo Molnar } 871a5f4374aSIngo Molnar 872a5f4374aSIngo Molnar if (is_register) 873e3343e6aSSrikar Dronamraju ret = install_breakpoint(uprobe, mm, vma, vi->vaddr); 874a5f4374aSIngo Molnar else 875e3343e6aSSrikar Dronamraju remove_breakpoint(uprobe, mm, vi->vaddr); 876a5f4374aSIngo Molnar 87777fc4af1SOleg Nesterov up_write(&mm->mmap_sem); 878a5f4374aSIngo Molnar mmput(mm); 879a5f4374aSIngo Molnar if (is_register) { 880a5f4374aSIngo Molnar if (ret && ret == -EEXIST) 881a5f4374aSIngo Molnar ret = 0; 882a5f4374aSIngo Molnar if (ret) 883a5f4374aSIngo Molnar break; 884a5f4374aSIngo Molnar } 885a5f4374aSIngo Molnar } 886a5f4374aSIngo Molnar 887a5f4374aSIngo Molnar list_for_each_entry_safe(vi, tmpvi, &try_list, probe_list) { 888a5f4374aSIngo Molnar list_del(&vi->probe_list); 889a5f4374aSIngo Molnar kfree(vi); 890a5f4374aSIngo Molnar } 891a5f4374aSIngo Molnar 892a5f4374aSIngo Molnar return ret; 893a5f4374aSIngo Molnar } 894a5f4374aSIngo Molnar 895a5f4374aSIngo Molnar static int __uprobe_register(struct uprobe *uprobe) 896a5f4374aSIngo Molnar { 897a5f4374aSIngo Molnar return register_for_each_vma(uprobe, true); 898a5f4374aSIngo Molnar } 899a5f4374aSIngo Molnar 900a5f4374aSIngo Molnar static void __uprobe_unregister(struct uprobe *uprobe) 901a5f4374aSIngo Molnar { 902a5f4374aSIngo Molnar if (!register_for_each_vma(uprobe, false)) 903a5f4374aSIngo Molnar delete_uprobe(uprobe); 904a5f4374aSIngo Molnar 905a5f4374aSIngo Molnar /* TODO : cant unregister? schedule a worker thread */ 906a5f4374aSIngo Molnar } 907a5f4374aSIngo Molnar 908a5f4374aSIngo Molnar /* 909a5f4374aSIngo Molnar * uprobe_register - register a probe 910a5f4374aSIngo Molnar * @inode: the file in which the probe has to be placed. 911a5f4374aSIngo Molnar * @offset: offset from the start of the file. 912e3343e6aSSrikar Dronamraju * @uc: information on howto handle the probe.. 913a5f4374aSIngo Molnar * 914a5f4374aSIngo Molnar * Apart from the access refcount, uprobe_register() takes a creation 915a5f4374aSIngo Molnar * refcount (thro alloc_uprobe) if and only if this @uprobe is getting 916a5f4374aSIngo Molnar * inserted into the rbtree (i.e first consumer for a @inode:@offset 917a5f4374aSIngo Molnar * tuple). Creation refcount stops uprobe_unregister from freeing the 918a5f4374aSIngo Molnar * @uprobe even before the register operation is complete. Creation 919e3343e6aSSrikar Dronamraju * refcount is released when the last @uc for the @uprobe 920a5f4374aSIngo Molnar * unregisters. 921a5f4374aSIngo Molnar * 922a5f4374aSIngo Molnar * Return errno if it cannot successully install probes 923a5f4374aSIngo Molnar * else return 0 (success) 924a5f4374aSIngo Molnar */ 925e3343e6aSSrikar Dronamraju int uprobe_register(struct inode *inode, loff_t offset, struct uprobe_consumer *uc) 926a5f4374aSIngo Molnar { 927a5f4374aSIngo Molnar struct uprobe *uprobe; 928a5f4374aSIngo Molnar int ret; 929a5f4374aSIngo Molnar 930e3343e6aSSrikar Dronamraju if (!inode || !uc || uc->next) 931a5f4374aSIngo Molnar return -EINVAL; 932a5f4374aSIngo Molnar 933a5f4374aSIngo Molnar if (offset > i_size_read(inode)) 934a5f4374aSIngo Molnar return -EINVAL; 935a5f4374aSIngo Molnar 936a5f4374aSIngo Molnar ret = 0; 937a5f4374aSIngo Molnar mutex_lock(uprobes_hash(inode)); 938a5f4374aSIngo Molnar uprobe = alloc_uprobe(inode, offset); 939a5f4374aSIngo Molnar 940e3343e6aSSrikar Dronamraju if (uprobe && !consumer_add(uprobe, uc)) { 941a5f4374aSIngo Molnar ret = __uprobe_register(uprobe); 942a5f4374aSIngo Molnar if (ret) { 943a5f4374aSIngo Molnar uprobe->consumers = NULL; 944a5f4374aSIngo Molnar __uprobe_unregister(uprobe); 945a5f4374aSIngo Molnar } else { 946900771a4SSrikar Dronamraju uprobe->flags |= UPROBE_RUN_HANDLER; 947a5f4374aSIngo Molnar } 948a5f4374aSIngo Molnar } 949a5f4374aSIngo Molnar 950a5f4374aSIngo Molnar mutex_unlock(uprobes_hash(inode)); 951a5f4374aSIngo Molnar put_uprobe(uprobe); 952a5f4374aSIngo Molnar 953a5f4374aSIngo Molnar return ret; 954a5f4374aSIngo Molnar } 955a5f4374aSIngo Molnar 956a5f4374aSIngo Molnar /* 957a5f4374aSIngo Molnar * uprobe_unregister - unregister a already registered probe. 958a5f4374aSIngo Molnar * @inode: the file in which the probe has to be removed. 959a5f4374aSIngo Molnar * @offset: offset from the start of the file. 960e3343e6aSSrikar Dronamraju * @uc: identify which probe if multiple probes are colocated. 961a5f4374aSIngo Molnar */ 962e3343e6aSSrikar Dronamraju void uprobe_unregister(struct inode *inode, loff_t offset, struct uprobe_consumer *uc) 963a5f4374aSIngo Molnar { 964a5f4374aSIngo Molnar struct uprobe *uprobe; 965a5f4374aSIngo Molnar 966e3343e6aSSrikar Dronamraju if (!inode || !uc) 967a5f4374aSIngo Molnar return; 968a5f4374aSIngo Molnar 969a5f4374aSIngo Molnar uprobe = find_uprobe(inode, offset); 970a5f4374aSIngo Molnar if (!uprobe) 971a5f4374aSIngo Molnar return; 972a5f4374aSIngo Molnar 973a5f4374aSIngo Molnar mutex_lock(uprobes_hash(inode)); 974a5f4374aSIngo Molnar 975e3343e6aSSrikar Dronamraju if (consumer_del(uprobe, uc)) { 976a5f4374aSIngo Molnar if (!uprobe->consumers) { 977a5f4374aSIngo Molnar __uprobe_unregister(uprobe); 978900771a4SSrikar Dronamraju uprobe->flags &= ~UPROBE_RUN_HANDLER; 979a5f4374aSIngo Molnar } 980a5f4374aSIngo Molnar } 981a5f4374aSIngo Molnar 982a5f4374aSIngo Molnar mutex_unlock(uprobes_hash(inode)); 983a5f4374aSIngo Molnar if (uprobe) 984a5f4374aSIngo Molnar put_uprobe(uprobe); 985a5f4374aSIngo Molnar } 986a5f4374aSIngo Molnar 987a5f4374aSIngo Molnar /* 988a5f4374aSIngo Molnar * Of all the nodes that correspond to the given inode, return the node 989a5f4374aSIngo Molnar * with the least offset. 990a5f4374aSIngo Molnar */ 991a5f4374aSIngo Molnar static struct rb_node *find_least_offset_node(struct inode *inode) 992a5f4374aSIngo Molnar { 993a5f4374aSIngo Molnar struct uprobe u = { .inode = inode, .offset = 0}; 994a5f4374aSIngo Molnar struct rb_node *n = uprobes_tree.rb_node; 995a5f4374aSIngo Molnar struct rb_node *close_node = NULL; 996a5f4374aSIngo Molnar struct uprobe *uprobe; 997a5f4374aSIngo Molnar int match; 998a5f4374aSIngo Molnar 999a5f4374aSIngo Molnar while (n) { 1000a5f4374aSIngo Molnar uprobe = rb_entry(n, struct uprobe, rb_node); 1001a5f4374aSIngo Molnar match = match_uprobe(&u, uprobe); 1002a5f4374aSIngo Molnar 1003a5f4374aSIngo Molnar if (uprobe->inode == inode) 1004a5f4374aSIngo Molnar close_node = n; 1005a5f4374aSIngo Molnar 1006a5f4374aSIngo Molnar if (!match) 1007a5f4374aSIngo Molnar return close_node; 1008a5f4374aSIngo Molnar 1009a5f4374aSIngo Molnar if (match < 0) 1010a5f4374aSIngo Molnar n = n->rb_left; 1011a5f4374aSIngo Molnar else 1012a5f4374aSIngo Molnar n = n->rb_right; 1013a5f4374aSIngo Molnar } 1014a5f4374aSIngo Molnar 1015a5f4374aSIngo Molnar return close_node; 1016a5f4374aSIngo Molnar } 1017a5f4374aSIngo Molnar 1018a5f4374aSIngo Molnar /* 1019a5f4374aSIngo Molnar * For a given inode, build a list of probes that need to be inserted. 1020a5f4374aSIngo Molnar */ 1021a5f4374aSIngo Molnar static void build_probe_list(struct inode *inode, struct list_head *head) 1022a5f4374aSIngo Molnar { 1023a5f4374aSIngo Molnar struct uprobe *uprobe; 1024a5f4374aSIngo Molnar unsigned long flags; 1025a5f4374aSIngo Molnar struct rb_node *n; 1026a5f4374aSIngo Molnar 1027a5f4374aSIngo Molnar spin_lock_irqsave(&uprobes_treelock, flags); 1028a5f4374aSIngo Molnar 1029a5f4374aSIngo Molnar n = find_least_offset_node(inode); 1030a5f4374aSIngo Molnar 1031a5f4374aSIngo Molnar for (; n; n = rb_next(n)) { 1032a5f4374aSIngo Molnar uprobe = rb_entry(n, struct uprobe, rb_node); 1033a5f4374aSIngo Molnar if (uprobe->inode != inode) 1034a5f4374aSIngo Molnar break; 1035a5f4374aSIngo Molnar 1036a5f4374aSIngo Molnar list_add(&uprobe->pending_list, head); 1037a5f4374aSIngo Molnar atomic_inc(&uprobe->ref); 1038a5f4374aSIngo Molnar } 1039a5f4374aSIngo Molnar 1040a5f4374aSIngo Molnar spin_unlock_irqrestore(&uprobes_treelock, flags); 1041a5f4374aSIngo Molnar } 1042a5f4374aSIngo Molnar 1043a5f4374aSIngo Molnar /* 1044a5f4374aSIngo Molnar * Called from mmap_region. 1045a5f4374aSIngo Molnar * called with mm->mmap_sem acquired. 1046a5f4374aSIngo Molnar * 1047a5f4374aSIngo Molnar * Return -ve no if we fail to insert probes and we cannot 1048a5f4374aSIngo Molnar * bail-out. 1049a5f4374aSIngo Molnar * Return 0 otherwise. i.e: 1050a5f4374aSIngo Molnar * 1051a5f4374aSIngo Molnar * - successful insertion of probes 1052a5f4374aSIngo Molnar * - (or) no possible probes to be inserted. 1053a5f4374aSIngo Molnar * - (or) insertion of probes failed but we can bail-out. 1054a5f4374aSIngo Molnar */ 1055a5f4374aSIngo Molnar int uprobe_mmap(struct vm_area_struct *vma) 1056a5f4374aSIngo Molnar { 1057a5f4374aSIngo Molnar struct list_head tmp_list; 1058a5f4374aSIngo Molnar struct uprobe *uprobe, *u; 1059a5f4374aSIngo Molnar struct inode *inode; 1060682968e0SSrikar Dronamraju int ret, count; 1061a5f4374aSIngo Molnar 1062a5f4374aSIngo Molnar if (!atomic_read(&uprobe_events) || !valid_vma(vma, true)) 1063a5f4374aSIngo Molnar return 0; 1064a5f4374aSIngo Molnar 1065a5f4374aSIngo Molnar inode = vma->vm_file->f_mapping->host; 1066a5f4374aSIngo Molnar if (!inode) 1067a5f4374aSIngo Molnar return 0; 1068a5f4374aSIngo Molnar 1069a5f4374aSIngo Molnar INIT_LIST_HEAD(&tmp_list); 1070a5f4374aSIngo Molnar mutex_lock(uprobes_mmap_hash(inode)); 1071a5f4374aSIngo Molnar build_probe_list(inode, &tmp_list); 1072a5f4374aSIngo Molnar 1073a5f4374aSIngo Molnar ret = 0; 1074682968e0SSrikar Dronamraju count = 0; 1075a5f4374aSIngo Molnar 1076a5f4374aSIngo Molnar list_for_each_entry_safe(uprobe, u, &tmp_list, pending_list) { 1077a5f4374aSIngo Molnar loff_t vaddr; 1078a5f4374aSIngo Molnar 1079a5f4374aSIngo Molnar list_del(&uprobe->pending_list); 1080a5f4374aSIngo Molnar if (!ret) { 1081a5f4374aSIngo Molnar vaddr = vma_address(vma, uprobe->offset); 1082682968e0SSrikar Dronamraju 1083682968e0SSrikar Dronamraju if (vaddr < vma->vm_start || vaddr >= vma->vm_end) { 1084682968e0SSrikar Dronamraju put_uprobe(uprobe); 1085682968e0SSrikar Dronamraju continue; 1086a5f4374aSIngo Molnar } 1087682968e0SSrikar Dronamraju 1088682968e0SSrikar Dronamraju ret = install_breakpoint(uprobe, vma->vm_mm, vma, vaddr); 1089682968e0SSrikar Dronamraju 1090682968e0SSrikar Dronamraju /* Ignore double add: */ 1091682968e0SSrikar Dronamraju if (ret == -EEXIST) { 1092682968e0SSrikar Dronamraju ret = 0; 1093682968e0SSrikar Dronamraju 1094682968e0SSrikar Dronamraju if (!is_swbp_at_addr(vma->vm_mm, vaddr)) 1095682968e0SSrikar Dronamraju continue; 1096682968e0SSrikar Dronamraju 1097682968e0SSrikar Dronamraju /* 1098682968e0SSrikar Dronamraju * Unable to insert a breakpoint, but 1099682968e0SSrikar Dronamraju * breakpoint lies underneath. Increment the 1100682968e0SSrikar Dronamraju * probe count. 1101682968e0SSrikar Dronamraju */ 1102682968e0SSrikar Dronamraju atomic_inc(&vma->vm_mm->uprobes_state.count); 1103682968e0SSrikar Dronamraju } 1104682968e0SSrikar Dronamraju 1105682968e0SSrikar Dronamraju if (!ret) 1106682968e0SSrikar Dronamraju count++; 1107a5f4374aSIngo Molnar } 1108a5f4374aSIngo Molnar put_uprobe(uprobe); 1109a5f4374aSIngo Molnar } 1110a5f4374aSIngo Molnar 1111a5f4374aSIngo Molnar mutex_unlock(uprobes_mmap_hash(inode)); 1112a5f4374aSIngo Molnar 1113682968e0SSrikar Dronamraju if (ret) 1114682968e0SSrikar Dronamraju atomic_sub(count, &vma->vm_mm->uprobes_state.count); 1115682968e0SSrikar Dronamraju 1116a5f4374aSIngo Molnar return ret; 1117a5f4374aSIngo Molnar } 1118a5f4374aSIngo Molnar 1119682968e0SSrikar Dronamraju /* 1120682968e0SSrikar Dronamraju * Called in context of a munmap of a vma. 1121682968e0SSrikar Dronamraju */ 1122cbc91f71SSrikar Dronamraju void uprobe_munmap(struct vm_area_struct *vma, unsigned long start, unsigned long end) 1123682968e0SSrikar Dronamraju { 1124682968e0SSrikar Dronamraju struct list_head tmp_list; 1125682968e0SSrikar Dronamraju struct uprobe *uprobe, *u; 1126682968e0SSrikar Dronamraju struct inode *inode; 1127682968e0SSrikar Dronamraju 1128682968e0SSrikar Dronamraju if (!atomic_read(&uprobe_events) || !valid_vma(vma, false)) 1129682968e0SSrikar Dronamraju return; 1130682968e0SSrikar Dronamraju 1131682968e0SSrikar Dronamraju if (!atomic_read(&vma->vm_mm->uprobes_state.count)) 1132682968e0SSrikar Dronamraju return; 1133682968e0SSrikar Dronamraju 1134682968e0SSrikar Dronamraju inode = vma->vm_file->f_mapping->host; 1135682968e0SSrikar Dronamraju if (!inode) 1136682968e0SSrikar Dronamraju return; 1137682968e0SSrikar Dronamraju 1138682968e0SSrikar Dronamraju INIT_LIST_HEAD(&tmp_list); 1139682968e0SSrikar Dronamraju mutex_lock(uprobes_mmap_hash(inode)); 1140682968e0SSrikar Dronamraju build_probe_list(inode, &tmp_list); 1141682968e0SSrikar Dronamraju 1142682968e0SSrikar Dronamraju list_for_each_entry_safe(uprobe, u, &tmp_list, pending_list) { 1143682968e0SSrikar Dronamraju loff_t vaddr; 1144682968e0SSrikar Dronamraju 1145682968e0SSrikar Dronamraju list_del(&uprobe->pending_list); 1146682968e0SSrikar Dronamraju vaddr = vma_address(vma, uprobe->offset); 1147682968e0SSrikar Dronamraju 1148cbc91f71SSrikar Dronamraju if (vaddr >= start && vaddr < end) { 1149682968e0SSrikar Dronamraju /* 1150682968e0SSrikar Dronamraju * An unregister could have removed the probe before 1151682968e0SSrikar Dronamraju * unmap. So check before we decrement the count. 1152682968e0SSrikar Dronamraju */ 1153682968e0SSrikar Dronamraju if (is_swbp_at_addr(vma->vm_mm, vaddr) == 1) 1154682968e0SSrikar Dronamraju atomic_dec(&vma->vm_mm->uprobes_state.count); 1155682968e0SSrikar Dronamraju } 1156682968e0SSrikar Dronamraju put_uprobe(uprobe); 1157682968e0SSrikar Dronamraju } 1158682968e0SSrikar Dronamraju mutex_unlock(uprobes_mmap_hash(inode)); 1159682968e0SSrikar Dronamraju } 1160682968e0SSrikar Dronamraju 1161d4b3b638SSrikar Dronamraju /* Slot allocation for XOL */ 1162d4b3b638SSrikar Dronamraju static int xol_add_vma(struct xol_area *area) 1163d4b3b638SSrikar Dronamraju { 1164d4b3b638SSrikar Dronamraju struct mm_struct *mm; 1165d4b3b638SSrikar Dronamraju int ret; 1166d4b3b638SSrikar Dronamraju 1167d4b3b638SSrikar Dronamraju area->page = alloc_page(GFP_HIGHUSER); 1168d4b3b638SSrikar Dronamraju if (!area->page) 1169d4b3b638SSrikar Dronamraju return -ENOMEM; 1170d4b3b638SSrikar Dronamraju 1171d4b3b638SSrikar Dronamraju ret = -EALREADY; 1172d4b3b638SSrikar Dronamraju mm = current->mm; 1173d4b3b638SSrikar Dronamraju 1174d4b3b638SSrikar Dronamraju down_write(&mm->mmap_sem); 1175d4b3b638SSrikar Dronamraju if (mm->uprobes_state.xol_area) 1176d4b3b638SSrikar Dronamraju goto fail; 1177d4b3b638SSrikar Dronamraju 1178d4b3b638SSrikar Dronamraju ret = -ENOMEM; 1179d4b3b638SSrikar Dronamraju 1180d4b3b638SSrikar Dronamraju /* Try to map as high as possible, this is only a hint. */ 1181d4b3b638SSrikar Dronamraju area->vaddr = get_unmapped_area(NULL, TASK_SIZE - PAGE_SIZE, PAGE_SIZE, 0, 0); 1182d4b3b638SSrikar Dronamraju if (area->vaddr & ~PAGE_MASK) { 1183d4b3b638SSrikar Dronamraju ret = area->vaddr; 1184d4b3b638SSrikar Dronamraju goto fail; 1185d4b3b638SSrikar Dronamraju } 1186d4b3b638SSrikar Dronamraju 1187d4b3b638SSrikar Dronamraju ret = install_special_mapping(mm, area->vaddr, PAGE_SIZE, 1188d4b3b638SSrikar Dronamraju VM_EXEC|VM_MAYEXEC|VM_DONTCOPY|VM_IO, &area->page); 1189d4b3b638SSrikar Dronamraju if (ret) 1190d4b3b638SSrikar Dronamraju goto fail; 1191d4b3b638SSrikar Dronamraju 1192d4b3b638SSrikar Dronamraju smp_wmb(); /* pairs with get_xol_area() */ 1193d4b3b638SSrikar Dronamraju mm->uprobes_state.xol_area = area; 1194d4b3b638SSrikar Dronamraju ret = 0; 1195d4b3b638SSrikar Dronamraju 1196d4b3b638SSrikar Dronamraju fail: 1197d4b3b638SSrikar Dronamraju up_write(&mm->mmap_sem); 1198d4b3b638SSrikar Dronamraju if (ret) 1199d4b3b638SSrikar Dronamraju __free_page(area->page); 1200d4b3b638SSrikar Dronamraju 1201d4b3b638SSrikar Dronamraju return ret; 1202d4b3b638SSrikar Dronamraju } 1203d4b3b638SSrikar Dronamraju 1204d4b3b638SSrikar Dronamraju static struct xol_area *get_xol_area(struct mm_struct *mm) 1205d4b3b638SSrikar Dronamraju { 1206d4b3b638SSrikar Dronamraju struct xol_area *area; 1207d4b3b638SSrikar Dronamraju 1208d4b3b638SSrikar Dronamraju area = mm->uprobes_state.xol_area; 1209d4b3b638SSrikar Dronamraju smp_read_barrier_depends(); /* pairs with wmb in xol_add_vma() */ 1210d4b3b638SSrikar Dronamraju 1211d4b3b638SSrikar Dronamraju return area; 1212d4b3b638SSrikar Dronamraju } 1213d4b3b638SSrikar Dronamraju 1214d4b3b638SSrikar Dronamraju /* 1215d4b3b638SSrikar Dronamraju * xol_alloc_area - Allocate process's xol_area. 1216d4b3b638SSrikar Dronamraju * This area will be used for storing instructions for execution out of 1217d4b3b638SSrikar Dronamraju * line. 1218d4b3b638SSrikar Dronamraju * 1219d4b3b638SSrikar Dronamraju * Returns the allocated area or NULL. 1220d4b3b638SSrikar Dronamraju */ 1221d4b3b638SSrikar Dronamraju static struct xol_area *xol_alloc_area(void) 1222d4b3b638SSrikar Dronamraju { 1223d4b3b638SSrikar Dronamraju struct xol_area *area; 1224d4b3b638SSrikar Dronamraju 1225d4b3b638SSrikar Dronamraju area = kzalloc(sizeof(*area), GFP_KERNEL); 1226d4b3b638SSrikar Dronamraju if (unlikely(!area)) 1227d4b3b638SSrikar Dronamraju return NULL; 1228d4b3b638SSrikar Dronamraju 1229d4b3b638SSrikar Dronamraju area->bitmap = kzalloc(BITS_TO_LONGS(UINSNS_PER_PAGE) * sizeof(long), GFP_KERNEL); 1230d4b3b638SSrikar Dronamraju 1231d4b3b638SSrikar Dronamraju if (!area->bitmap) 1232d4b3b638SSrikar Dronamraju goto fail; 1233d4b3b638SSrikar Dronamraju 1234d4b3b638SSrikar Dronamraju init_waitqueue_head(&area->wq); 1235d4b3b638SSrikar Dronamraju if (!xol_add_vma(area)) 1236d4b3b638SSrikar Dronamraju return area; 1237d4b3b638SSrikar Dronamraju 1238d4b3b638SSrikar Dronamraju fail: 1239d4b3b638SSrikar Dronamraju kfree(area->bitmap); 1240d4b3b638SSrikar Dronamraju kfree(area); 1241d4b3b638SSrikar Dronamraju 1242d4b3b638SSrikar Dronamraju return get_xol_area(current->mm); 1243d4b3b638SSrikar Dronamraju } 1244d4b3b638SSrikar Dronamraju 1245d4b3b638SSrikar Dronamraju /* 1246d4b3b638SSrikar Dronamraju * uprobe_clear_state - Free the area allocated for slots. 1247d4b3b638SSrikar Dronamraju */ 1248d4b3b638SSrikar Dronamraju void uprobe_clear_state(struct mm_struct *mm) 1249d4b3b638SSrikar Dronamraju { 1250d4b3b638SSrikar Dronamraju struct xol_area *area = mm->uprobes_state.xol_area; 1251d4b3b638SSrikar Dronamraju 1252d4b3b638SSrikar Dronamraju if (!area) 1253d4b3b638SSrikar Dronamraju return; 1254d4b3b638SSrikar Dronamraju 1255d4b3b638SSrikar Dronamraju put_page(area->page); 1256d4b3b638SSrikar Dronamraju kfree(area->bitmap); 1257d4b3b638SSrikar Dronamraju kfree(area); 1258d4b3b638SSrikar Dronamraju } 1259d4b3b638SSrikar Dronamraju 1260d4b3b638SSrikar Dronamraju /* 1261d4b3b638SSrikar Dronamraju * uprobe_reset_state - Free the area allocated for slots. 1262d4b3b638SSrikar Dronamraju */ 1263d4b3b638SSrikar Dronamraju void uprobe_reset_state(struct mm_struct *mm) 1264d4b3b638SSrikar Dronamraju { 1265d4b3b638SSrikar Dronamraju mm->uprobes_state.xol_area = NULL; 1266682968e0SSrikar Dronamraju atomic_set(&mm->uprobes_state.count, 0); 1267d4b3b638SSrikar Dronamraju } 1268d4b3b638SSrikar Dronamraju 1269d4b3b638SSrikar Dronamraju /* 1270d4b3b638SSrikar Dronamraju * - search for a free slot. 1271d4b3b638SSrikar Dronamraju */ 1272d4b3b638SSrikar Dronamraju static unsigned long xol_take_insn_slot(struct xol_area *area) 1273d4b3b638SSrikar Dronamraju { 1274d4b3b638SSrikar Dronamraju unsigned long slot_addr; 1275d4b3b638SSrikar Dronamraju int slot_nr; 1276d4b3b638SSrikar Dronamraju 1277d4b3b638SSrikar Dronamraju do { 1278d4b3b638SSrikar Dronamraju slot_nr = find_first_zero_bit(area->bitmap, UINSNS_PER_PAGE); 1279d4b3b638SSrikar Dronamraju if (slot_nr < UINSNS_PER_PAGE) { 1280d4b3b638SSrikar Dronamraju if (!test_and_set_bit(slot_nr, area->bitmap)) 1281d4b3b638SSrikar Dronamraju break; 1282d4b3b638SSrikar Dronamraju 1283d4b3b638SSrikar Dronamraju slot_nr = UINSNS_PER_PAGE; 1284d4b3b638SSrikar Dronamraju continue; 1285d4b3b638SSrikar Dronamraju } 1286d4b3b638SSrikar Dronamraju wait_event(area->wq, (atomic_read(&area->slot_count) < UINSNS_PER_PAGE)); 1287d4b3b638SSrikar Dronamraju } while (slot_nr >= UINSNS_PER_PAGE); 1288d4b3b638SSrikar Dronamraju 1289d4b3b638SSrikar Dronamraju slot_addr = area->vaddr + (slot_nr * UPROBE_XOL_SLOT_BYTES); 1290d4b3b638SSrikar Dronamraju atomic_inc(&area->slot_count); 1291d4b3b638SSrikar Dronamraju 1292d4b3b638SSrikar Dronamraju return slot_addr; 1293d4b3b638SSrikar Dronamraju } 1294d4b3b638SSrikar Dronamraju 1295d4b3b638SSrikar Dronamraju /* 1296d4b3b638SSrikar Dronamraju * xol_get_insn_slot - If was not allocated a slot, then 1297d4b3b638SSrikar Dronamraju * allocate a slot. 1298d4b3b638SSrikar Dronamraju * Returns the allocated slot address or 0. 1299d4b3b638SSrikar Dronamraju */ 1300d4b3b638SSrikar Dronamraju static unsigned long xol_get_insn_slot(struct uprobe *uprobe, unsigned long slot_addr) 1301d4b3b638SSrikar Dronamraju { 1302d4b3b638SSrikar Dronamraju struct xol_area *area; 1303d4b3b638SSrikar Dronamraju unsigned long offset; 1304d4b3b638SSrikar Dronamraju void *vaddr; 1305d4b3b638SSrikar Dronamraju 1306d4b3b638SSrikar Dronamraju area = get_xol_area(current->mm); 1307d4b3b638SSrikar Dronamraju if (!area) { 1308d4b3b638SSrikar Dronamraju area = xol_alloc_area(); 1309d4b3b638SSrikar Dronamraju if (!area) 1310d4b3b638SSrikar Dronamraju return 0; 1311d4b3b638SSrikar Dronamraju } 1312d4b3b638SSrikar Dronamraju current->utask->xol_vaddr = xol_take_insn_slot(area); 1313d4b3b638SSrikar Dronamraju 1314d4b3b638SSrikar Dronamraju /* 1315d4b3b638SSrikar Dronamraju * Initialize the slot if xol_vaddr points to valid 1316d4b3b638SSrikar Dronamraju * instruction slot. 1317d4b3b638SSrikar Dronamraju */ 1318d4b3b638SSrikar Dronamraju if (unlikely(!current->utask->xol_vaddr)) 1319d4b3b638SSrikar Dronamraju return 0; 1320d4b3b638SSrikar Dronamraju 1321d4b3b638SSrikar Dronamraju current->utask->vaddr = slot_addr; 1322d4b3b638SSrikar Dronamraju offset = current->utask->xol_vaddr & ~PAGE_MASK; 1323d4b3b638SSrikar Dronamraju vaddr = kmap_atomic(area->page); 1324d4b3b638SSrikar Dronamraju memcpy(vaddr + offset, uprobe->arch.insn, MAX_UINSN_BYTES); 1325d4b3b638SSrikar Dronamraju kunmap_atomic(vaddr); 1326d4b3b638SSrikar Dronamraju 1327d4b3b638SSrikar Dronamraju return current->utask->xol_vaddr; 1328d4b3b638SSrikar Dronamraju } 1329d4b3b638SSrikar Dronamraju 1330d4b3b638SSrikar Dronamraju /* 1331d4b3b638SSrikar Dronamraju * xol_free_insn_slot - If slot was earlier allocated by 1332d4b3b638SSrikar Dronamraju * @xol_get_insn_slot(), make the slot available for 1333d4b3b638SSrikar Dronamraju * subsequent requests. 1334d4b3b638SSrikar Dronamraju */ 1335d4b3b638SSrikar Dronamraju static void xol_free_insn_slot(struct task_struct *tsk) 1336d4b3b638SSrikar Dronamraju { 1337d4b3b638SSrikar Dronamraju struct xol_area *area; 1338d4b3b638SSrikar Dronamraju unsigned long vma_end; 1339d4b3b638SSrikar Dronamraju unsigned long slot_addr; 1340d4b3b638SSrikar Dronamraju 1341d4b3b638SSrikar Dronamraju if (!tsk->mm || !tsk->mm->uprobes_state.xol_area || !tsk->utask) 1342d4b3b638SSrikar Dronamraju return; 1343d4b3b638SSrikar Dronamraju 1344d4b3b638SSrikar Dronamraju slot_addr = tsk->utask->xol_vaddr; 1345d4b3b638SSrikar Dronamraju 1346d4b3b638SSrikar Dronamraju if (unlikely(!slot_addr || IS_ERR_VALUE(slot_addr))) 1347d4b3b638SSrikar Dronamraju return; 1348d4b3b638SSrikar Dronamraju 1349d4b3b638SSrikar Dronamraju area = tsk->mm->uprobes_state.xol_area; 1350d4b3b638SSrikar Dronamraju vma_end = area->vaddr + PAGE_SIZE; 1351d4b3b638SSrikar Dronamraju if (area->vaddr <= slot_addr && slot_addr < vma_end) { 1352d4b3b638SSrikar Dronamraju unsigned long offset; 1353d4b3b638SSrikar Dronamraju int slot_nr; 1354d4b3b638SSrikar Dronamraju 1355d4b3b638SSrikar Dronamraju offset = slot_addr - area->vaddr; 1356d4b3b638SSrikar Dronamraju slot_nr = offset / UPROBE_XOL_SLOT_BYTES; 1357d4b3b638SSrikar Dronamraju if (slot_nr >= UINSNS_PER_PAGE) 1358d4b3b638SSrikar Dronamraju return; 1359d4b3b638SSrikar Dronamraju 1360d4b3b638SSrikar Dronamraju clear_bit(slot_nr, area->bitmap); 1361d4b3b638SSrikar Dronamraju atomic_dec(&area->slot_count); 1362d4b3b638SSrikar Dronamraju if (waitqueue_active(&area->wq)) 1363d4b3b638SSrikar Dronamraju wake_up(&area->wq); 1364d4b3b638SSrikar Dronamraju 1365d4b3b638SSrikar Dronamraju tsk->utask->xol_vaddr = 0; 1366d4b3b638SSrikar Dronamraju } 1367d4b3b638SSrikar Dronamraju } 1368d4b3b638SSrikar Dronamraju 13690326f5a9SSrikar Dronamraju /** 13700326f5a9SSrikar Dronamraju * uprobe_get_swbp_addr - compute address of swbp given post-swbp regs 13710326f5a9SSrikar Dronamraju * @regs: Reflects the saved state of the task after it has hit a breakpoint 13720326f5a9SSrikar Dronamraju * instruction. 13730326f5a9SSrikar Dronamraju * Return the address of the breakpoint instruction. 13740326f5a9SSrikar Dronamraju */ 13750326f5a9SSrikar Dronamraju unsigned long __weak uprobe_get_swbp_addr(struct pt_regs *regs) 13760326f5a9SSrikar Dronamraju { 13770326f5a9SSrikar Dronamraju return instruction_pointer(regs) - UPROBE_SWBP_INSN_SIZE; 13780326f5a9SSrikar Dronamraju } 13790326f5a9SSrikar Dronamraju 13800326f5a9SSrikar Dronamraju /* 13810326f5a9SSrikar Dronamraju * Called with no locks held. 13820326f5a9SSrikar Dronamraju * Called in context of a exiting or a exec-ing thread. 13830326f5a9SSrikar Dronamraju */ 13840326f5a9SSrikar Dronamraju void uprobe_free_utask(struct task_struct *t) 13850326f5a9SSrikar Dronamraju { 13860326f5a9SSrikar Dronamraju struct uprobe_task *utask = t->utask; 13870326f5a9SSrikar Dronamraju 13880326f5a9SSrikar Dronamraju if (!utask) 13890326f5a9SSrikar Dronamraju return; 13900326f5a9SSrikar Dronamraju 13910326f5a9SSrikar Dronamraju if (utask->active_uprobe) 13920326f5a9SSrikar Dronamraju put_uprobe(utask->active_uprobe); 13930326f5a9SSrikar Dronamraju 1394d4b3b638SSrikar Dronamraju xol_free_insn_slot(t); 13950326f5a9SSrikar Dronamraju kfree(utask); 13960326f5a9SSrikar Dronamraju t->utask = NULL; 13970326f5a9SSrikar Dronamraju } 13980326f5a9SSrikar Dronamraju 13990326f5a9SSrikar Dronamraju /* 14000326f5a9SSrikar Dronamraju * Called in context of a new clone/fork from copy_process. 14010326f5a9SSrikar Dronamraju */ 14020326f5a9SSrikar Dronamraju void uprobe_copy_process(struct task_struct *t) 14030326f5a9SSrikar Dronamraju { 14040326f5a9SSrikar Dronamraju t->utask = NULL; 14050326f5a9SSrikar Dronamraju } 14060326f5a9SSrikar Dronamraju 14070326f5a9SSrikar Dronamraju /* 14080326f5a9SSrikar Dronamraju * Allocate a uprobe_task object for the task. 14090326f5a9SSrikar Dronamraju * Called when the thread hits a breakpoint for the first time. 14100326f5a9SSrikar Dronamraju * 14110326f5a9SSrikar Dronamraju * Returns: 14120326f5a9SSrikar Dronamraju * - pointer to new uprobe_task on success 14130326f5a9SSrikar Dronamraju * - NULL otherwise 14140326f5a9SSrikar Dronamraju */ 14150326f5a9SSrikar Dronamraju static struct uprobe_task *add_utask(void) 14160326f5a9SSrikar Dronamraju { 14170326f5a9SSrikar Dronamraju struct uprobe_task *utask; 14180326f5a9SSrikar Dronamraju 14190326f5a9SSrikar Dronamraju utask = kzalloc(sizeof *utask, GFP_KERNEL); 14200326f5a9SSrikar Dronamraju if (unlikely(!utask)) 14210326f5a9SSrikar Dronamraju return NULL; 14220326f5a9SSrikar Dronamraju 14230326f5a9SSrikar Dronamraju utask->active_uprobe = NULL; 14240326f5a9SSrikar Dronamraju current->utask = utask; 14250326f5a9SSrikar Dronamraju return utask; 14260326f5a9SSrikar Dronamraju } 14270326f5a9SSrikar Dronamraju 14280326f5a9SSrikar Dronamraju /* Prepare to single-step probed instruction out of line. */ 14290326f5a9SSrikar Dronamraju static int 14300326f5a9SSrikar Dronamraju pre_ssout(struct uprobe *uprobe, struct pt_regs *regs, unsigned long vaddr) 14310326f5a9SSrikar Dronamraju { 1432d4b3b638SSrikar Dronamraju if (xol_get_insn_slot(uprobe, vaddr) && !arch_uprobe_pre_xol(&uprobe->arch, regs)) 1433d4b3b638SSrikar Dronamraju return 0; 1434d4b3b638SSrikar Dronamraju 14350326f5a9SSrikar Dronamraju return -EFAULT; 14360326f5a9SSrikar Dronamraju } 14370326f5a9SSrikar Dronamraju 14380326f5a9SSrikar Dronamraju /* 14390326f5a9SSrikar Dronamraju * If we are singlestepping, then ensure this thread is not connected to 14400326f5a9SSrikar Dronamraju * non-fatal signals until completion of singlestep. When xol insn itself 14410326f5a9SSrikar Dronamraju * triggers the signal, restart the original insn even if the task is 14420326f5a9SSrikar Dronamraju * already SIGKILL'ed (since coredump should report the correct ip). This 14430326f5a9SSrikar Dronamraju * is even more important if the task has a handler for SIGSEGV/etc, The 14440326f5a9SSrikar Dronamraju * _same_ instruction should be repeated again after return from the signal 14450326f5a9SSrikar Dronamraju * handler, and SSTEP can never finish in this case. 14460326f5a9SSrikar Dronamraju */ 14470326f5a9SSrikar Dronamraju bool uprobe_deny_signal(void) 14480326f5a9SSrikar Dronamraju { 14490326f5a9SSrikar Dronamraju struct task_struct *t = current; 14500326f5a9SSrikar Dronamraju struct uprobe_task *utask = t->utask; 14510326f5a9SSrikar Dronamraju 14520326f5a9SSrikar Dronamraju if (likely(!utask || !utask->active_uprobe)) 14530326f5a9SSrikar Dronamraju return false; 14540326f5a9SSrikar Dronamraju 14550326f5a9SSrikar Dronamraju WARN_ON_ONCE(utask->state != UTASK_SSTEP); 14560326f5a9SSrikar Dronamraju 14570326f5a9SSrikar Dronamraju if (signal_pending(t)) { 14580326f5a9SSrikar Dronamraju spin_lock_irq(&t->sighand->siglock); 14590326f5a9SSrikar Dronamraju clear_tsk_thread_flag(t, TIF_SIGPENDING); 14600326f5a9SSrikar Dronamraju spin_unlock_irq(&t->sighand->siglock); 14610326f5a9SSrikar Dronamraju 14620326f5a9SSrikar Dronamraju if (__fatal_signal_pending(t) || arch_uprobe_xol_was_trapped(t)) { 14630326f5a9SSrikar Dronamraju utask->state = UTASK_SSTEP_TRAPPED; 14640326f5a9SSrikar Dronamraju set_tsk_thread_flag(t, TIF_UPROBE); 14650326f5a9SSrikar Dronamraju set_tsk_thread_flag(t, TIF_NOTIFY_RESUME); 14660326f5a9SSrikar Dronamraju } 14670326f5a9SSrikar Dronamraju } 14680326f5a9SSrikar Dronamraju 14690326f5a9SSrikar Dronamraju return true; 14700326f5a9SSrikar Dronamraju } 14710326f5a9SSrikar Dronamraju 14720326f5a9SSrikar Dronamraju /* 14730326f5a9SSrikar Dronamraju * Avoid singlestepping the original instruction if the original instruction 14740326f5a9SSrikar Dronamraju * is a NOP or can be emulated. 14750326f5a9SSrikar Dronamraju */ 14760326f5a9SSrikar Dronamraju static bool can_skip_sstep(struct uprobe *uprobe, struct pt_regs *regs) 14770326f5a9SSrikar Dronamraju { 14780326f5a9SSrikar Dronamraju if (arch_uprobe_skip_sstep(&uprobe->arch, regs)) 14790326f5a9SSrikar Dronamraju return true; 14800326f5a9SSrikar Dronamraju 14810326f5a9SSrikar Dronamraju uprobe->flags &= ~UPROBE_SKIP_SSTEP; 14820326f5a9SSrikar Dronamraju return false; 14830326f5a9SSrikar Dronamraju } 14840326f5a9SSrikar Dronamraju 1485d790d346SOleg Nesterov static struct uprobe *find_active_uprobe(unsigned long bp_vaddr, int *is_swbp) 14860326f5a9SSrikar Dronamraju { 14873a9ea052SOleg Nesterov struct mm_struct *mm = current->mm; 14883a9ea052SOleg Nesterov struct uprobe *uprobe = NULL; 14890326f5a9SSrikar Dronamraju struct vm_area_struct *vma; 14900326f5a9SSrikar Dronamraju 14910326f5a9SSrikar Dronamraju down_read(&mm->mmap_sem); 14920326f5a9SSrikar Dronamraju vma = find_vma(mm, bp_vaddr); 14933a9ea052SOleg Nesterov if (vma && vma->vm_start <= bp_vaddr) { 14943a9ea052SOleg Nesterov if (valid_vma(vma, false)) { 14950326f5a9SSrikar Dronamraju struct inode *inode; 14960326f5a9SSrikar Dronamraju loff_t offset; 14970326f5a9SSrikar Dronamraju 14980326f5a9SSrikar Dronamraju inode = vma->vm_file->f_mapping->host; 14990326f5a9SSrikar Dronamraju offset = bp_vaddr - vma->vm_start; 15000326f5a9SSrikar Dronamraju offset += (vma->vm_pgoff << PAGE_SHIFT); 15010326f5a9SSrikar Dronamraju uprobe = find_uprobe(inode, offset); 15020326f5a9SSrikar Dronamraju } 1503d790d346SOleg Nesterov 1504d790d346SOleg Nesterov if (!uprobe) 1505d790d346SOleg Nesterov *is_swbp = is_swbp_at_addr(mm, bp_vaddr); 1506d790d346SOleg Nesterov } else { 1507d790d346SOleg Nesterov *is_swbp = -EFAULT; 15083a9ea052SOleg Nesterov } 15090326f5a9SSrikar Dronamraju up_read(&mm->mmap_sem); 15100326f5a9SSrikar Dronamraju 15113a9ea052SOleg Nesterov return uprobe; 15123a9ea052SOleg Nesterov } 15133a9ea052SOleg Nesterov 15143a9ea052SOleg Nesterov /* 15153a9ea052SOleg Nesterov * Run handler and ask thread to singlestep. 15163a9ea052SOleg Nesterov * Ensure all non-fatal signals cannot interrupt thread while it singlesteps. 15173a9ea052SOleg Nesterov */ 15183a9ea052SOleg Nesterov static void handle_swbp(struct pt_regs *regs) 15193a9ea052SOleg Nesterov { 15203a9ea052SOleg Nesterov struct uprobe_task *utask; 15213a9ea052SOleg Nesterov struct uprobe *uprobe; 15223a9ea052SOleg Nesterov unsigned long bp_vaddr; 152356bb4cf6SOleg Nesterov int uninitialized_var(is_swbp); 15243a9ea052SOleg Nesterov 15253a9ea052SOleg Nesterov bp_vaddr = uprobe_get_swbp_addr(regs); 1526d790d346SOleg Nesterov uprobe = find_active_uprobe(bp_vaddr, &is_swbp); 15273a9ea052SOleg Nesterov 15280326f5a9SSrikar Dronamraju if (!uprobe) { 152956bb4cf6SOleg Nesterov if (is_swbp > 0) { 15300326f5a9SSrikar Dronamraju /* No matching uprobe; signal SIGTRAP. */ 15310326f5a9SSrikar Dronamraju send_sig(SIGTRAP, current, 0); 153256bb4cf6SOleg Nesterov } else { 153356bb4cf6SOleg Nesterov /* 153456bb4cf6SOleg Nesterov * Either we raced with uprobe_unregister() or we can't 153556bb4cf6SOleg Nesterov * access this memory. The latter is only possible if 153656bb4cf6SOleg Nesterov * another thread plays with our ->mm. In both cases 153756bb4cf6SOleg Nesterov * we can simply restart. If this vma was unmapped we 153856bb4cf6SOleg Nesterov * can pretend this insn was not executed yet and get 153956bb4cf6SOleg Nesterov * the (correct) SIGSEGV after restart. 154056bb4cf6SOleg Nesterov */ 154156bb4cf6SOleg Nesterov instruction_pointer_set(regs, bp_vaddr); 154256bb4cf6SOleg Nesterov } 15430326f5a9SSrikar Dronamraju return; 15440326f5a9SSrikar Dronamraju } 15450326f5a9SSrikar Dronamraju 15460326f5a9SSrikar Dronamraju utask = current->utask; 15470326f5a9SSrikar Dronamraju if (!utask) { 15480326f5a9SSrikar Dronamraju utask = add_utask(); 15490326f5a9SSrikar Dronamraju /* Cannot allocate; re-execute the instruction. */ 15500326f5a9SSrikar Dronamraju if (!utask) 15510326f5a9SSrikar Dronamraju goto cleanup_ret; 15520326f5a9SSrikar Dronamraju } 15530326f5a9SSrikar Dronamraju utask->active_uprobe = uprobe; 15540326f5a9SSrikar Dronamraju handler_chain(uprobe, regs); 15550326f5a9SSrikar Dronamraju if (uprobe->flags & UPROBE_SKIP_SSTEP && can_skip_sstep(uprobe, regs)) 15560326f5a9SSrikar Dronamraju goto cleanup_ret; 15570326f5a9SSrikar Dronamraju 15580326f5a9SSrikar Dronamraju utask->state = UTASK_SSTEP; 15590326f5a9SSrikar Dronamraju if (!pre_ssout(uprobe, regs, bp_vaddr)) { 15600326f5a9SSrikar Dronamraju user_enable_single_step(current); 15610326f5a9SSrikar Dronamraju return; 15620326f5a9SSrikar Dronamraju } 15630326f5a9SSrikar Dronamraju 15640326f5a9SSrikar Dronamraju cleanup_ret: 15650326f5a9SSrikar Dronamraju if (utask) { 15660326f5a9SSrikar Dronamraju utask->active_uprobe = NULL; 15670326f5a9SSrikar Dronamraju utask->state = UTASK_RUNNING; 15680326f5a9SSrikar Dronamraju } 15690326f5a9SSrikar Dronamraju if (uprobe) { 15700326f5a9SSrikar Dronamraju if (!(uprobe->flags & UPROBE_SKIP_SSTEP)) 15710326f5a9SSrikar Dronamraju 15720326f5a9SSrikar Dronamraju /* 15730326f5a9SSrikar Dronamraju * cannot singlestep; cannot skip instruction; 15740326f5a9SSrikar Dronamraju * re-execute the instruction. 15750326f5a9SSrikar Dronamraju */ 15760326f5a9SSrikar Dronamraju instruction_pointer_set(regs, bp_vaddr); 15770326f5a9SSrikar Dronamraju 15780326f5a9SSrikar Dronamraju put_uprobe(uprobe); 15790326f5a9SSrikar Dronamraju } 15800326f5a9SSrikar Dronamraju } 15810326f5a9SSrikar Dronamraju 15820326f5a9SSrikar Dronamraju /* 15830326f5a9SSrikar Dronamraju * Perform required fix-ups and disable singlestep. 15840326f5a9SSrikar Dronamraju * Allow pending signals to take effect. 15850326f5a9SSrikar Dronamraju */ 15860326f5a9SSrikar Dronamraju static void handle_singlestep(struct uprobe_task *utask, struct pt_regs *regs) 15870326f5a9SSrikar Dronamraju { 15880326f5a9SSrikar Dronamraju struct uprobe *uprobe; 15890326f5a9SSrikar Dronamraju 15900326f5a9SSrikar Dronamraju uprobe = utask->active_uprobe; 15910326f5a9SSrikar Dronamraju if (utask->state == UTASK_SSTEP_ACK) 15920326f5a9SSrikar Dronamraju arch_uprobe_post_xol(&uprobe->arch, regs); 15930326f5a9SSrikar Dronamraju else if (utask->state == UTASK_SSTEP_TRAPPED) 15940326f5a9SSrikar Dronamraju arch_uprobe_abort_xol(&uprobe->arch, regs); 15950326f5a9SSrikar Dronamraju else 15960326f5a9SSrikar Dronamraju WARN_ON_ONCE(1); 15970326f5a9SSrikar Dronamraju 15980326f5a9SSrikar Dronamraju put_uprobe(uprobe); 15990326f5a9SSrikar Dronamraju utask->active_uprobe = NULL; 16000326f5a9SSrikar Dronamraju utask->state = UTASK_RUNNING; 16010326f5a9SSrikar Dronamraju user_disable_single_step(current); 1602d4b3b638SSrikar Dronamraju xol_free_insn_slot(current); 16030326f5a9SSrikar Dronamraju 16040326f5a9SSrikar Dronamraju spin_lock_irq(¤t->sighand->siglock); 16050326f5a9SSrikar Dronamraju recalc_sigpending(); /* see uprobe_deny_signal() */ 16060326f5a9SSrikar Dronamraju spin_unlock_irq(¤t->sighand->siglock); 16070326f5a9SSrikar Dronamraju } 16080326f5a9SSrikar Dronamraju 16090326f5a9SSrikar Dronamraju /* 16100326f5a9SSrikar Dronamraju * On breakpoint hit, breakpoint notifier sets the TIF_UPROBE flag. (and on 16110326f5a9SSrikar Dronamraju * subsequent probe hits on the thread sets the state to UTASK_BP_HIT) and 16120326f5a9SSrikar Dronamraju * allows the thread to return from interrupt. 16130326f5a9SSrikar Dronamraju * 16140326f5a9SSrikar Dronamraju * On singlestep exception, singlestep notifier sets the TIF_UPROBE flag and 16150326f5a9SSrikar Dronamraju * also sets the state to UTASK_SSTEP_ACK and allows the thread to return from 16160326f5a9SSrikar Dronamraju * interrupt. 16170326f5a9SSrikar Dronamraju * 16180326f5a9SSrikar Dronamraju * While returning to userspace, thread notices the TIF_UPROBE flag and calls 16190326f5a9SSrikar Dronamraju * uprobe_notify_resume(). 16200326f5a9SSrikar Dronamraju */ 16210326f5a9SSrikar Dronamraju void uprobe_notify_resume(struct pt_regs *regs) 16220326f5a9SSrikar Dronamraju { 16230326f5a9SSrikar Dronamraju struct uprobe_task *utask; 16240326f5a9SSrikar Dronamraju 16250326f5a9SSrikar Dronamraju utask = current->utask; 16260326f5a9SSrikar Dronamraju if (!utask || utask->state == UTASK_BP_HIT) 16270326f5a9SSrikar Dronamraju handle_swbp(regs); 16280326f5a9SSrikar Dronamraju else 16290326f5a9SSrikar Dronamraju handle_singlestep(utask, regs); 16300326f5a9SSrikar Dronamraju } 16310326f5a9SSrikar Dronamraju 16320326f5a9SSrikar Dronamraju /* 16330326f5a9SSrikar Dronamraju * uprobe_pre_sstep_notifier gets called from interrupt context as part of 16340326f5a9SSrikar Dronamraju * notifier mechanism. Set TIF_UPROBE flag and indicate breakpoint hit. 16350326f5a9SSrikar Dronamraju */ 16360326f5a9SSrikar Dronamraju int uprobe_pre_sstep_notifier(struct pt_regs *regs) 16370326f5a9SSrikar Dronamraju { 16380326f5a9SSrikar Dronamraju struct uprobe_task *utask; 16390326f5a9SSrikar Dronamraju 1640682968e0SSrikar Dronamraju if (!current->mm || !atomic_read(¤t->mm->uprobes_state.count)) 1641682968e0SSrikar Dronamraju /* task is currently not uprobed */ 16420326f5a9SSrikar Dronamraju return 0; 16430326f5a9SSrikar Dronamraju 16440326f5a9SSrikar Dronamraju utask = current->utask; 16450326f5a9SSrikar Dronamraju if (utask) 16460326f5a9SSrikar Dronamraju utask->state = UTASK_BP_HIT; 16470326f5a9SSrikar Dronamraju 16480326f5a9SSrikar Dronamraju set_thread_flag(TIF_UPROBE); 16490326f5a9SSrikar Dronamraju 16500326f5a9SSrikar Dronamraju return 1; 16510326f5a9SSrikar Dronamraju } 16520326f5a9SSrikar Dronamraju 16530326f5a9SSrikar Dronamraju /* 16540326f5a9SSrikar Dronamraju * uprobe_post_sstep_notifier gets called in interrupt context as part of notifier 16550326f5a9SSrikar Dronamraju * mechanism. Set TIF_UPROBE flag and indicate completion of singlestep. 16560326f5a9SSrikar Dronamraju */ 16570326f5a9SSrikar Dronamraju int uprobe_post_sstep_notifier(struct pt_regs *regs) 16580326f5a9SSrikar Dronamraju { 16590326f5a9SSrikar Dronamraju struct uprobe_task *utask = current->utask; 16600326f5a9SSrikar Dronamraju 16610326f5a9SSrikar Dronamraju if (!current->mm || !utask || !utask->active_uprobe) 16620326f5a9SSrikar Dronamraju /* task is currently not uprobed */ 16630326f5a9SSrikar Dronamraju return 0; 16640326f5a9SSrikar Dronamraju 16650326f5a9SSrikar Dronamraju utask->state = UTASK_SSTEP_ACK; 16660326f5a9SSrikar Dronamraju set_thread_flag(TIF_UPROBE); 16670326f5a9SSrikar Dronamraju return 1; 16680326f5a9SSrikar Dronamraju } 16690326f5a9SSrikar Dronamraju 16700326f5a9SSrikar Dronamraju static struct notifier_block uprobe_exception_nb = { 16710326f5a9SSrikar Dronamraju .notifier_call = arch_uprobe_exception_notify, 16720326f5a9SSrikar Dronamraju .priority = INT_MAX-1, /* notified after kprobes, kgdb */ 16730326f5a9SSrikar Dronamraju }; 16740326f5a9SSrikar Dronamraju 1675a5f4374aSIngo Molnar static int __init init_uprobes(void) 1676a5f4374aSIngo Molnar { 1677a5f4374aSIngo Molnar int i; 1678a5f4374aSIngo Molnar 1679a5f4374aSIngo Molnar for (i = 0; i < UPROBES_HASH_SZ; i++) { 1680a5f4374aSIngo Molnar mutex_init(&uprobes_mutex[i]); 1681a5f4374aSIngo Molnar mutex_init(&uprobes_mmap_mutex[i]); 1682a5f4374aSIngo Molnar } 16830326f5a9SSrikar Dronamraju 16840326f5a9SSrikar Dronamraju return register_die_notifier(&uprobe_exception_nb); 1685a5f4374aSIngo Molnar } 16860326f5a9SSrikar Dronamraju module_init(init_uprobes); 1687a5f4374aSIngo Molnar 1688a5f4374aSIngo Molnar static void __exit exit_uprobes(void) 1689a5f4374aSIngo Molnar { 1690a5f4374aSIngo Molnar } 1691a5f4374aSIngo Molnar module_exit(exit_uprobes); 1692