1a5f4374aSIngo Molnar /* 2a5f4374aSIngo Molnar * User-space Probes (UProbes) 3a5f4374aSIngo Molnar * 4a5f4374aSIngo Molnar * This program is free software; you can redistribute it and/or modify 5a5f4374aSIngo Molnar * it under the terms of the GNU General Public License as published by 6a5f4374aSIngo Molnar * the Free Software Foundation; either version 2 of the License, or 7a5f4374aSIngo Molnar * (at your option) any later version. 8a5f4374aSIngo Molnar * 9a5f4374aSIngo Molnar * This program is distributed in the hope that it will be useful, 10a5f4374aSIngo Molnar * but WITHOUT ANY WARRANTY; without even the implied warranty of 11a5f4374aSIngo Molnar * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12a5f4374aSIngo Molnar * GNU General Public License for more details. 13a5f4374aSIngo Molnar * 14a5f4374aSIngo Molnar * You should have received a copy of the GNU General Public License 15a5f4374aSIngo Molnar * along with this program; if not, write to the Free Software 16a5f4374aSIngo Molnar * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 17a5f4374aSIngo Molnar * 1835aa621bSIngo Molnar * Copyright (C) IBM Corporation, 2008-2012 19a5f4374aSIngo Molnar * Authors: 20a5f4374aSIngo Molnar * Srikar Dronamraju 21a5f4374aSIngo Molnar * Jim Keniston 2235aa621bSIngo Molnar * Copyright (C) 2011-2012 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com> 23a5f4374aSIngo Molnar */ 24a5f4374aSIngo Molnar 25a5f4374aSIngo Molnar #include <linux/kernel.h> 26a5f4374aSIngo Molnar #include <linux/highmem.h> 27a5f4374aSIngo Molnar #include <linux/pagemap.h> /* read_mapping_page */ 28a5f4374aSIngo Molnar #include <linux/slab.h> 29a5f4374aSIngo Molnar #include <linux/sched.h> 30e8440c14SJosh Stone #include <linux/export.h> 31a5f4374aSIngo Molnar #include <linux/rmap.h> /* anon_vma_prepare */ 32a5f4374aSIngo Molnar #include <linux/mmu_notifier.h> /* set_pte_at_notify */ 33a5f4374aSIngo Molnar #include <linux/swap.h> /* try_to_free_swap */ 340326f5a9SSrikar Dronamraju #include <linux/ptrace.h> /* user_enable_single_step */ 350326f5a9SSrikar Dronamraju #include <linux/kdebug.h> /* notifier mechanism */ 36194f8dcbSOleg Nesterov #include "../../mm/internal.h" /* munlock_vma_page */ 3732cdba1eSOleg Nesterov #include <linux/percpu-rwsem.h> 38a5f4374aSIngo Molnar 39a5f4374aSIngo Molnar #include <linux/uprobes.h> 40a5f4374aSIngo Molnar 41d4b3b638SSrikar Dronamraju #define UINSNS_PER_PAGE (PAGE_SIZE/UPROBE_XOL_SLOT_BYTES) 42d4b3b638SSrikar Dronamraju #define MAX_UPROBE_XOL_SLOTS UINSNS_PER_PAGE 43d4b3b638SSrikar Dronamraju 44a5f4374aSIngo Molnar static struct rb_root uprobes_tree = RB_ROOT; 45441f1eb7SOleg Nesterov /* 46441f1eb7SOleg Nesterov * allows us to skip the uprobe_mmap if there are no uprobe events active 47441f1eb7SOleg Nesterov * at this time. Probably a fine grained per inode count is better? 48441f1eb7SOleg Nesterov */ 49441f1eb7SOleg Nesterov #define no_uprobe_events() RB_EMPTY_ROOT(&uprobes_tree) 50a5f4374aSIngo Molnar 51a5f4374aSIngo Molnar static DEFINE_SPINLOCK(uprobes_treelock); /* serialize rbtree access */ 52a5f4374aSIngo Molnar 53a5f4374aSIngo Molnar #define UPROBES_HASH_SZ 13 54a5f4374aSIngo Molnar /* serialize uprobe->pending_list */ 55a5f4374aSIngo Molnar static struct mutex uprobes_mmap_mutex[UPROBES_HASH_SZ]; 56a5f4374aSIngo Molnar #define uprobes_mmap_hash(v) (&uprobes_mmap_mutex[((unsigned long)(v)) % UPROBES_HASH_SZ]) 57a5f4374aSIngo Molnar 5832cdba1eSOleg Nesterov static struct percpu_rw_semaphore dup_mmap_sem; 5932cdba1eSOleg Nesterov 60cb9a19feSOleg Nesterov /* Have a copy of original instruction */ 6171434f2fSOleg Nesterov #define UPROBE_COPY_INSN 0 62cb9a19feSOleg Nesterov /* Can skip singlestep */ 63bb929284SOleg Nesterov #define UPROBE_SKIP_SSTEP 1 64cb9a19feSOleg Nesterov 653ff54efdSSrikar Dronamraju struct uprobe { 663ff54efdSSrikar Dronamraju struct rb_node rb_node; /* node in the rb tree */ 673ff54efdSSrikar Dronamraju atomic_t ref; 68e591c8d7SOleg Nesterov struct rw_semaphore register_rwsem; 693ff54efdSSrikar Dronamraju struct rw_semaphore consumer_rwsem; 703ff54efdSSrikar Dronamraju struct list_head pending_list; 713ff54efdSSrikar Dronamraju struct uprobe_consumer *consumers; 723ff54efdSSrikar Dronamraju struct inode *inode; /* Also hold a ref to inode */ 733ff54efdSSrikar Dronamraju loff_t offset; 7471434f2fSOleg Nesterov unsigned long flags; 753ff54efdSSrikar Dronamraju struct arch_uprobe arch; 763ff54efdSSrikar Dronamraju }; 773ff54efdSSrikar Dronamraju 780dfd0eb8SAnton Arapov struct return_instance { 790dfd0eb8SAnton Arapov struct uprobe *uprobe; 800dfd0eb8SAnton Arapov unsigned long func; 810dfd0eb8SAnton Arapov unsigned long orig_ret_vaddr; /* original return address */ 820dfd0eb8SAnton Arapov bool chained; /* true, if instance is nested */ 830dfd0eb8SAnton Arapov 840dfd0eb8SAnton Arapov struct return_instance *next; /* keep as stack */ 850dfd0eb8SAnton Arapov }; 860dfd0eb8SAnton Arapov 87a5f4374aSIngo Molnar /* 88a5f4374aSIngo Molnar * valid_vma: Verify if the specified vma is an executable vma 89a5f4374aSIngo Molnar * Relax restrictions while unregistering: vm_flags might have 90a5f4374aSIngo Molnar * changed after breakpoint was inserted. 91a5f4374aSIngo Molnar * - is_register: indicates if we are in register context. 92a5f4374aSIngo Molnar * - Return 1 if the specified virtual address is in an 93a5f4374aSIngo Molnar * executable vma. 94a5f4374aSIngo Molnar */ 95a5f4374aSIngo Molnar static bool valid_vma(struct vm_area_struct *vma, bool is_register) 96a5f4374aSIngo Molnar { 97e40cfce6SOleg Nesterov vm_flags_t flags = VM_HUGETLB | VM_MAYEXEC | VM_SHARED; 98a5f4374aSIngo Molnar 99e40cfce6SOleg Nesterov if (is_register) 100e40cfce6SOleg Nesterov flags |= VM_WRITE; 101a5f4374aSIngo Molnar 102e40cfce6SOleg Nesterov return vma->vm_file && (vma->vm_flags & flags) == VM_MAYEXEC; 103a5f4374aSIngo Molnar } 104a5f4374aSIngo Molnar 10557683f72SOleg Nesterov static unsigned long offset_to_vaddr(struct vm_area_struct *vma, loff_t offset) 106a5f4374aSIngo Molnar { 10757683f72SOleg Nesterov return vma->vm_start + offset - ((loff_t)vma->vm_pgoff << PAGE_SHIFT); 108a5f4374aSIngo Molnar } 109a5f4374aSIngo Molnar 110cb113b47SOleg Nesterov static loff_t vaddr_to_offset(struct vm_area_struct *vma, unsigned long vaddr) 111cb113b47SOleg Nesterov { 112cb113b47SOleg Nesterov return ((loff_t)vma->vm_pgoff << PAGE_SHIFT) + (vaddr - vma->vm_start); 113cb113b47SOleg Nesterov } 114cb113b47SOleg Nesterov 115a5f4374aSIngo Molnar /** 116a5f4374aSIngo Molnar * __replace_page - replace page in vma by new page. 117a5f4374aSIngo Molnar * based on replace_page in mm/ksm.c 118a5f4374aSIngo Molnar * 119a5f4374aSIngo Molnar * @vma: vma that holds the pte pointing to page 120c517ee74SOleg Nesterov * @addr: address the old @page is mapped at 121a5f4374aSIngo Molnar * @page: the cowed page we are replacing by kpage 122a5f4374aSIngo Molnar * @kpage: the modified page we replace page by 123a5f4374aSIngo Molnar * 124a5f4374aSIngo Molnar * Returns 0 on success, -EFAULT on failure. 125a5f4374aSIngo Molnar */ 126c517ee74SOleg Nesterov static int __replace_page(struct vm_area_struct *vma, unsigned long addr, 127c517ee74SOleg Nesterov struct page *page, struct page *kpage) 128a5f4374aSIngo Molnar { 129a5f4374aSIngo Molnar struct mm_struct *mm = vma->vm_mm; 1305323ce71SOleg Nesterov spinlock_t *ptl; 1315323ce71SOleg Nesterov pte_t *ptep; 1329f92448cSOleg Nesterov int err; 1336bdb913fSHaggai Eran /* For mmu_notifiers */ 1346bdb913fSHaggai Eran const unsigned long mmun_start = addr; 1356bdb913fSHaggai Eran const unsigned long mmun_end = addr + PAGE_SIZE; 136a5f4374aSIngo Molnar 137194f8dcbSOleg Nesterov /* For try_to_free_swap() and munlock_vma_page() below */ 1389f92448cSOleg Nesterov lock_page(page); 1399f92448cSOleg Nesterov 1406bdb913fSHaggai Eran mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end); 1419f92448cSOleg Nesterov err = -EAGAIN; 1425323ce71SOleg Nesterov ptep = page_check_address(page, mm, addr, &ptl, 0); 143a5f4374aSIngo Molnar if (!ptep) 1449f92448cSOleg Nesterov goto unlock; 145a5f4374aSIngo Molnar 146a5f4374aSIngo Molnar get_page(kpage); 147a5f4374aSIngo Molnar page_add_new_anon_rmap(kpage, vma, addr); 148a5f4374aSIngo Molnar 1497396fa81SSrikar Dronamraju if (!PageAnon(page)) { 1507396fa81SSrikar Dronamraju dec_mm_counter(mm, MM_FILEPAGES); 1517396fa81SSrikar Dronamraju inc_mm_counter(mm, MM_ANONPAGES); 1527396fa81SSrikar Dronamraju } 1537396fa81SSrikar Dronamraju 154a5f4374aSIngo Molnar flush_cache_page(vma, addr, pte_pfn(*ptep)); 155a5f4374aSIngo Molnar ptep_clear_flush(vma, addr, ptep); 156a5f4374aSIngo Molnar set_pte_at_notify(mm, addr, ptep, mk_pte(kpage, vma->vm_page_prot)); 157a5f4374aSIngo Molnar 158a5f4374aSIngo Molnar page_remove_rmap(page); 159a5f4374aSIngo Molnar if (!page_mapped(page)) 160a5f4374aSIngo Molnar try_to_free_swap(page); 161a5f4374aSIngo Molnar pte_unmap_unlock(ptep, ptl); 162a5f4374aSIngo Molnar 163194f8dcbSOleg Nesterov if (vma->vm_flags & VM_LOCKED) 164194f8dcbSOleg Nesterov munlock_vma_page(page); 165194f8dcbSOleg Nesterov put_page(page); 166194f8dcbSOleg Nesterov 1679f92448cSOleg Nesterov err = 0; 1689f92448cSOleg Nesterov unlock: 1696bdb913fSHaggai Eran mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end); 1709f92448cSOleg Nesterov unlock_page(page); 1719f92448cSOleg Nesterov return err; 172a5f4374aSIngo Molnar } 173a5f4374aSIngo Molnar 174a5f4374aSIngo Molnar /** 1755cb4ac3aSSrikar Dronamraju * is_swbp_insn - check if instruction is breakpoint instruction. 176a5f4374aSIngo Molnar * @insn: instruction to be checked. 1775cb4ac3aSSrikar Dronamraju * Default implementation of is_swbp_insn 178a5f4374aSIngo Molnar * Returns true if @insn is a breakpoint instruction. 179a5f4374aSIngo Molnar */ 1805cb4ac3aSSrikar Dronamraju bool __weak is_swbp_insn(uprobe_opcode_t *insn) 181a5f4374aSIngo Molnar { 1825cb4ac3aSSrikar Dronamraju return *insn == UPROBE_SWBP_INSN; 183a5f4374aSIngo Molnar } 184a5f4374aSIngo Molnar 1850908ad6eSAnanth N Mavinakayanahalli /** 1860908ad6eSAnanth N Mavinakayanahalli * is_trap_insn - check if instruction is breakpoint instruction. 1870908ad6eSAnanth N Mavinakayanahalli * @insn: instruction to be checked. 1880908ad6eSAnanth N Mavinakayanahalli * Default implementation of is_trap_insn 1890908ad6eSAnanth N Mavinakayanahalli * Returns true if @insn is a breakpoint instruction. 1900908ad6eSAnanth N Mavinakayanahalli * 1910908ad6eSAnanth N Mavinakayanahalli * This function is needed for the case where an architecture has multiple 1920908ad6eSAnanth N Mavinakayanahalli * trap instructions (like powerpc). 1930908ad6eSAnanth N Mavinakayanahalli */ 1940908ad6eSAnanth N Mavinakayanahalli bool __weak is_trap_insn(uprobe_opcode_t *insn) 1950908ad6eSAnanth N Mavinakayanahalli { 1960908ad6eSAnanth N Mavinakayanahalli return is_swbp_insn(insn); 1970908ad6eSAnanth N Mavinakayanahalli } 1980908ad6eSAnanth N Mavinakayanahalli 199ab0d805cSOleg Nesterov static void copy_from_page(struct page *page, unsigned long vaddr, void *dst, int len) 200cceb55aaSOleg Nesterov { 201cceb55aaSOleg Nesterov void *kaddr = kmap_atomic(page); 202ab0d805cSOleg Nesterov memcpy(dst, kaddr + (vaddr & ~PAGE_MASK), len); 203cceb55aaSOleg Nesterov kunmap_atomic(kaddr); 204cceb55aaSOleg Nesterov } 205cceb55aaSOleg Nesterov 2065669cceeSOleg Nesterov static void copy_to_page(struct page *page, unsigned long vaddr, const void *src, int len) 2075669cceeSOleg Nesterov { 2085669cceeSOleg Nesterov void *kaddr = kmap_atomic(page); 2095669cceeSOleg Nesterov memcpy(kaddr + (vaddr & ~PAGE_MASK), src, len); 2105669cceeSOleg Nesterov kunmap_atomic(kaddr); 2115669cceeSOleg Nesterov } 2125669cceeSOleg Nesterov 213ed6f6a50SOleg Nesterov static int verify_opcode(struct page *page, unsigned long vaddr, uprobe_opcode_t *new_opcode) 214ed6f6a50SOleg Nesterov { 215ed6f6a50SOleg Nesterov uprobe_opcode_t old_opcode; 216ed6f6a50SOleg Nesterov bool is_swbp; 217ed6f6a50SOleg Nesterov 2180908ad6eSAnanth N Mavinakayanahalli /* 2190908ad6eSAnanth N Mavinakayanahalli * Note: We only check if the old_opcode is UPROBE_SWBP_INSN here. 2200908ad6eSAnanth N Mavinakayanahalli * We do not check if it is any other 'trap variant' which could 2210908ad6eSAnanth N Mavinakayanahalli * be conditional trap instruction such as the one powerpc supports. 2220908ad6eSAnanth N Mavinakayanahalli * 2230908ad6eSAnanth N Mavinakayanahalli * The logic is that we do not care if the underlying instruction 2240908ad6eSAnanth N Mavinakayanahalli * is a trap variant; uprobes always wins over any other (gdb) 2250908ad6eSAnanth N Mavinakayanahalli * breakpoint. 2260908ad6eSAnanth N Mavinakayanahalli */ 227ab0d805cSOleg Nesterov copy_from_page(page, vaddr, &old_opcode, UPROBE_SWBP_INSN_SIZE); 228ed6f6a50SOleg Nesterov is_swbp = is_swbp_insn(&old_opcode); 229ed6f6a50SOleg Nesterov 230ed6f6a50SOleg Nesterov if (is_swbp_insn(new_opcode)) { 231ed6f6a50SOleg Nesterov if (is_swbp) /* register: already installed? */ 232ed6f6a50SOleg Nesterov return 0; 233ed6f6a50SOleg Nesterov } else { 234ed6f6a50SOleg Nesterov if (!is_swbp) /* unregister: was it changed by us? */ 235076a365bSOleg Nesterov return 0; 236ed6f6a50SOleg Nesterov } 237ed6f6a50SOleg Nesterov 238ed6f6a50SOleg Nesterov return 1; 239ed6f6a50SOleg Nesterov } 240ed6f6a50SOleg Nesterov 241a5f4374aSIngo Molnar /* 242a5f4374aSIngo Molnar * NOTE: 243a5f4374aSIngo Molnar * Expect the breakpoint instruction to be the smallest size instruction for 244a5f4374aSIngo Molnar * the architecture. If an arch has variable length instruction and the 245a5f4374aSIngo Molnar * breakpoint instruction is not of the smallest length instruction 2460908ad6eSAnanth N Mavinakayanahalli * supported by that architecture then we need to modify is_trap_at_addr and 247a5f4374aSIngo Molnar * write_opcode accordingly. This would never be a problem for archs that 248a5f4374aSIngo Molnar * have fixed length instructions. 249a5f4374aSIngo Molnar */ 250a5f4374aSIngo Molnar 251a5f4374aSIngo Molnar /* 252a5f4374aSIngo Molnar * write_opcode - write the opcode at a given virtual address. 253a5f4374aSIngo Molnar * @mm: the probed process address space. 254a5f4374aSIngo Molnar * @vaddr: the virtual address to store the opcode. 255a5f4374aSIngo Molnar * @opcode: opcode to be written at @vaddr. 256a5f4374aSIngo Molnar * 257a5f4374aSIngo Molnar * Called with mm->mmap_sem held (for read and with a reference to 258a5f4374aSIngo Molnar * mm). 259a5f4374aSIngo Molnar * 260a5f4374aSIngo Molnar * For mm @mm, write the opcode at @vaddr. 261a5f4374aSIngo Molnar * Return 0 (success) or a negative errno. 262a5f4374aSIngo Molnar */ 263cceb55aaSOleg Nesterov static int write_opcode(struct mm_struct *mm, unsigned long vaddr, 264cceb55aaSOleg Nesterov uprobe_opcode_t opcode) 265a5f4374aSIngo Molnar { 266a5f4374aSIngo Molnar struct page *old_page, *new_page; 267a5f4374aSIngo Molnar struct vm_area_struct *vma; 268a5f4374aSIngo Molnar int ret; 269f403072cSOleg Nesterov 2705323ce71SOleg Nesterov retry: 271a5f4374aSIngo Molnar /* Read the page with vaddr into memory */ 27275ed82eaSOleg Nesterov ret = get_user_pages(NULL, mm, vaddr, 1, 0, 1, &old_page, &vma); 273a5f4374aSIngo Molnar if (ret <= 0) 274a5f4374aSIngo Molnar return ret; 275a5f4374aSIngo Molnar 276ed6f6a50SOleg Nesterov ret = verify_opcode(old_page, vaddr, &opcode); 277ed6f6a50SOleg Nesterov if (ret <= 0) 278ed6f6a50SOleg Nesterov goto put_old; 279ed6f6a50SOleg Nesterov 280a5f4374aSIngo Molnar ret = -ENOMEM; 281a5f4374aSIngo Molnar new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, vaddr); 282a5f4374aSIngo Molnar if (!new_page) 2839f92448cSOleg Nesterov goto put_old; 284a5f4374aSIngo Molnar 285a5f4374aSIngo Molnar __SetPageUptodate(new_page); 286a5f4374aSIngo Molnar 2873f47107cSOleg Nesterov copy_highpage(new_page, old_page); 2883f47107cSOleg Nesterov copy_to_page(new_page, vaddr, &opcode, UPROBE_SWBP_INSN_SIZE); 289a5f4374aSIngo Molnar 290a5f4374aSIngo Molnar ret = anon_vma_prepare(vma); 291a5f4374aSIngo Molnar if (ret) 2929f92448cSOleg Nesterov goto put_new; 293a5f4374aSIngo Molnar 294c517ee74SOleg Nesterov ret = __replace_page(vma, vaddr, old_page, new_page); 295a5f4374aSIngo Molnar 2969f92448cSOleg Nesterov put_new: 297a5f4374aSIngo Molnar page_cache_release(new_page); 2989f92448cSOleg Nesterov put_old: 299a5f4374aSIngo Molnar put_page(old_page); 300a5f4374aSIngo Molnar 3015323ce71SOleg Nesterov if (unlikely(ret == -EAGAIN)) 3025323ce71SOleg Nesterov goto retry; 303a5f4374aSIngo Molnar return ret; 304a5f4374aSIngo Molnar } 305a5f4374aSIngo Molnar 306a5f4374aSIngo Molnar /** 3075cb4ac3aSSrikar Dronamraju * set_swbp - store breakpoint at a given address. 308e3343e6aSSrikar Dronamraju * @auprobe: arch specific probepoint information. 309a5f4374aSIngo Molnar * @mm: the probed process address space. 310a5f4374aSIngo Molnar * @vaddr: the virtual address to insert the opcode. 311a5f4374aSIngo Molnar * 312a5f4374aSIngo Molnar * For mm @mm, store the breakpoint instruction at @vaddr. 313a5f4374aSIngo Molnar * Return 0 (success) or a negative errno. 314a5f4374aSIngo Molnar */ 3155cb4ac3aSSrikar Dronamraju int __weak set_swbp(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned long vaddr) 316a5f4374aSIngo Molnar { 317cceb55aaSOleg Nesterov return write_opcode(mm, vaddr, UPROBE_SWBP_INSN); 318a5f4374aSIngo Molnar } 319a5f4374aSIngo Molnar 320a5f4374aSIngo Molnar /** 321a5f4374aSIngo Molnar * set_orig_insn - Restore the original instruction. 322a5f4374aSIngo Molnar * @mm: the probed process address space. 323e3343e6aSSrikar Dronamraju * @auprobe: arch specific probepoint information. 324a5f4374aSIngo Molnar * @vaddr: the virtual address to insert the opcode. 325a5f4374aSIngo Molnar * 326a5f4374aSIngo Molnar * For mm @mm, restore the original opcode (opcode) at @vaddr. 327a5f4374aSIngo Molnar * Return 0 (success) or a negative errno. 328a5f4374aSIngo Molnar */ 329a5f4374aSIngo Molnar int __weak 330ded86e7cSOleg Nesterov set_orig_insn(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned long vaddr) 331a5f4374aSIngo Molnar { 332cceb55aaSOleg Nesterov return write_opcode(mm, vaddr, *(uprobe_opcode_t *)auprobe->insn); 333a5f4374aSIngo Molnar } 334a5f4374aSIngo Molnar 335a5f4374aSIngo Molnar static int match_uprobe(struct uprobe *l, struct uprobe *r) 336a5f4374aSIngo Molnar { 337a5f4374aSIngo Molnar if (l->inode < r->inode) 338a5f4374aSIngo Molnar return -1; 339a5f4374aSIngo Molnar 340a5f4374aSIngo Molnar if (l->inode > r->inode) 341a5f4374aSIngo Molnar return 1; 342a5f4374aSIngo Molnar 343a5f4374aSIngo Molnar if (l->offset < r->offset) 344a5f4374aSIngo Molnar return -1; 345a5f4374aSIngo Molnar 346a5f4374aSIngo Molnar if (l->offset > r->offset) 347a5f4374aSIngo Molnar return 1; 348a5f4374aSIngo Molnar 349a5f4374aSIngo Molnar return 0; 350a5f4374aSIngo Molnar } 351a5f4374aSIngo Molnar 352a5f4374aSIngo Molnar static struct uprobe *__find_uprobe(struct inode *inode, loff_t offset) 353a5f4374aSIngo Molnar { 354a5f4374aSIngo Molnar struct uprobe u = { .inode = inode, .offset = offset }; 355a5f4374aSIngo Molnar struct rb_node *n = uprobes_tree.rb_node; 356a5f4374aSIngo Molnar struct uprobe *uprobe; 357a5f4374aSIngo Molnar int match; 358a5f4374aSIngo Molnar 359a5f4374aSIngo Molnar while (n) { 360a5f4374aSIngo Molnar uprobe = rb_entry(n, struct uprobe, rb_node); 361a5f4374aSIngo Molnar match = match_uprobe(&u, uprobe); 362a5f4374aSIngo Molnar if (!match) { 363a5f4374aSIngo Molnar atomic_inc(&uprobe->ref); 364a5f4374aSIngo Molnar return uprobe; 365a5f4374aSIngo Molnar } 366a5f4374aSIngo Molnar 367a5f4374aSIngo Molnar if (match < 0) 368a5f4374aSIngo Molnar n = n->rb_left; 369a5f4374aSIngo Molnar else 370a5f4374aSIngo Molnar n = n->rb_right; 371a5f4374aSIngo Molnar } 372a5f4374aSIngo Molnar return NULL; 373a5f4374aSIngo Molnar } 374a5f4374aSIngo Molnar 375a5f4374aSIngo Molnar /* 376a5f4374aSIngo Molnar * Find a uprobe corresponding to a given inode:offset 377a5f4374aSIngo Molnar * Acquires uprobes_treelock 378a5f4374aSIngo Molnar */ 379a5f4374aSIngo Molnar static struct uprobe *find_uprobe(struct inode *inode, loff_t offset) 380a5f4374aSIngo Molnar { 381a5f4374aSIngo Molnar struct uprobe *uprobe; 382a5f4374aSIngo Molnar 3836f47caa0SOleg Nesterov spin_lock(&uprobes_treelock); 384a5f4374aSIngo Molnar uprobe = __find_uprobe(inode, offset); 3856f47caa0SOleg Nesterov spin_unlock(&uprobes_treelock); 386a5f4374aSIngo Molnar 387a5f4374aSIngo Molnar return uprobe; 388a5f4374aSIngo Molnar } 389a5f4374aSIngo Molnar 390a5f4374aSIngo Molnar static struct uprobe *__insert_uprobe(struct uprobe *uprobe) 391a5f4374aSIngo Molnar { 392a5f4374aSIngo Molnar struct rb_node **p = &uprobes_tree.rb_node; 393a5f4374aSIngo Molnar struct rb_node *parent = NULL; 394a5f4374aSIngo Molnar struct uprobe *u; 395a5f4374aSIngo Molnar int match; 396a5f4374aSIngo Molnar 397a5f4374aSIngo Molnar while (*p) { 398a5f4374aSIngo Molnar parent = *p; 399a5f4374aSIngo Molnar u = rb_entry(parent, struct uprobe, rb_node); 400a5f4374aSIngo Molnar match = match_uprobe(uprobe, u); 401a5f4374aSIngo Molnar if (!match) { 402a5f4374aSIngo Molnar atomic_inc(&u->ref); 403a5f4374aSIngo Molnar return u; 404a5f4374aSIngo Molnar } 405a5f4374aSIngo Molnar 406a5f4374aSIngo Molnar if (match < 0) 407a5f4374aSIngo Molnar p = &parent->rb_left; 408a5f4374aSIngo Molnar else 409a5f4374aSIngo Molnar p = &parent->rb_right; 410a5f4374aSIngo Molnar 411a5f4374aSIngo Molnar } 412a5f4374aSIngo Molnar 413a5f4374aSIngo Molnar u = NULL; 414a5f4374aSIngo Molnar rb_link_node(&uprobe->rb_node, parent, p); 415a5f4374aSIngo Molnar rb_insert_color(&uprobe->rb_node, &uprobes_tree); 416a5f4374aSIngo Molnar /* get access + creation ref */ 417a5f4374aSIngo Molnar atomic_set(&uprobe->ref, 2); 418a5f4374aSIngo Molnar 419a5f4374aSIngo Molnar return u; 420a5f4374aSIngo Molnar } 421a5f4374aSIngo Molnar 422a5f4374aSIngo Molnar /* 423a5f4374aSIngo Molnar * Acquire uprobes_treelock. 424a5f4374aSIngo Molnar * Matching uprobe already exists in rbtree; 425a5f4374aSIngo Molnar * increment (access refcount) and return the matching uprobe. 426a5f4374aSIngo Molnar * 427a5f4374aSIngo Molnar * No matching uprobe; insert the uprobe in rb_tree; 428a5f4374aSIngo Molnar * get a double refcount (access + creation) and return NULL. 429a5f4374aSIngo Molnar */ 430a5f4374aSIngo Molnar static struct uprobe *insert_uprobe(struct uprobe *uprobe) 431a5f4374aSIngo Molnar { 432a5f4374aSIngo Molnar struct uprobe *u; 433a5f4374aSIngo Molnar 4346f47caa0SOleg Nesterov spin_lock(&uprobes_treelock); 435a5f4374aSIngo Molnar u = __insert_uprobe(uprobe); 4366f47caa0SOleg Nesterov spin_unlock(&uprobes_treelock); 437a5f4374aSIngo Molnar 438a5f4374aSIngo Molnar return u; 439a5f4374aSIngo Molnar } 440a5f4374aSIngo Molnar 441a5f4374aSIngo Molnar static void put_uprobe(struct uprobe *uprobe) 442a5f4374aSIngo Molnar { 443a5f4374aSIngo Molnar if (atomic_dec_and_test(&uprobe->ref)) 444a5f4374aSIngo Molnar kfree(uprobe); 445a5f4374aSIngo Molnar } 446a5f4374aSIngo Molnar 447a5f4374aSIngo Molnar static struct uprobe *alloc_uprobe(struct inode *inode, loff_t offset) 448a5f4374aSIngo Molnar { 449a5f4374aSIngo Molnar struct uprobe *uprobe, *cur_uprobe; 450a5f4374aSIngo Molnar 451a5f4374aSIngo Molnar uprobe = kzalloc(sizeof(struct uprobe), GFP_KERNEL); 452a5f4374aSIngo Molnar if (!uprobe) 453a5f4374aSIngo Molnar return NULL; 454a5f4374aSIngo Molnar 455a5f4374aSIngo Molnar uprobe->inode = igrab(inode); 456a5f4374aSIngo Molnar uprobe->offset = offset; 457e591c8d7SOleg Nesterov init_rwsem(&uprobe->register_rwsem); 458a5f4374aSIngo Molnar init_rwsem(&uprobe->consumer_rwsem); 459bbc33d05SOleg Nesterov /* For now assume that the instruction need not be single-stepped */ 460bbc33d05SOleg Nesterov __set_bit(UPROBE_SKIP_SSTEP, &uprobe->flags); 461a5f4374aSIngo Molnar 462a5f4374aSIngo Molnar /* add to uprobes_tree, sorted on inode:offset */ 463a5f4374aSIngo Molnar cur_uprobe = insert_uprobe(uprobe); 464a5f4374aSIngo Molnar 465a5f4374aSIngo Molnar /* a uprobe exists for this inode:offset combination */ 466a5f4374aSIngo Molnar if (cur_uprobe) { 467a5f4374aSIngo Molnar kfree(uprobe); 468a5f4374aSIngo Molnar uprobe = cur_uprobe; 469a5f4374aSIngo Molnar iput(inode); 470a5f4374aSIngo Molnar } 471a5f4374aSIngo Molnar 472a5f4374aSIngo Molnar return uprobe; 473a5f4374aSIngo Molnar } 474a5f4374aSIngo Molnar 4759a98e03cSOleg Nesterov static void consumer_add(struct uprobe *uprobe, struct uprobe_consumer *uc) 476a5f4374aSIngo Molnar { 477a5f4374aSIngo Molnar down_write(&uprobe->consumer_rwsem); 478e3343e6aSSrikar Dronamraju uc->next = uprobe->consumers; 479e3343e6aSSrikar Dronamraju uprobe->consumers = uc; 480a5f4374aSIngo Molnar up_write(&uprobe->consumer_rwsem); 481a5f4374aSIngo Molnar } 482a5f4374aSIngo Molnar 483a5f4374aSIngo Molnar /* 484e3343e6aSSrikar Dronamraju * For uprobe @uprobe, delete the consumer @uc. 485e3343e6aSSrikar Dronamraju * Return true if the @uc is deleted successfully 486a5f4374aSIngo Molnar * or return false. 487a5f4374aSIngo Molnar */ 488e3343e6aSSrikar Dronamraju static bool consumer_del(struct uprobe *uprobe, struct uprobe_consumer *uc) 489a5f4374aSIngo Molnar { 490a5f4374aSIngo Molnar struct uprobe_consumer **con; 491a5f4374aSIngo Molnar bool ret = false; 492a5f4374aSIngo Molnar 493a5f4374aSIngo Molnar down_write(&uprobe->consumer_rwsem); 494a5f4374aSIngo Molnar for (con = &uprobe->consumers; *con; con = &(*con)->next) { 495e3343e6aSSrikar Dronamraju if (*con == uc) { 496e3343e6aSSrikar Dronamraju *con = uc->next; 497a5f4374aSIngo Molnar ret = true; 498a5f4374aSIngo Molnar break; 499a5f4374aSIngo Molnar } 500a5f4374aSIngo Molnar } 501a5f4374aSIngo Molnar up_write(&uprobe->consumer_rwsem); 502a5f4374aSIngo Molnar 503a5f4374aSIngo Molnar return ret; 504a5f4374aSIngo Molnar } 505a5f4374aSIngo Molnar 506e3343e6aSSrikar Dronamraju static int 507d436615eSOleg Nesterov __copy_insn(struct address_space *mapping, struct file *filp, char *insn, 508593609a5SOleg Nesterov unsigned long nbytes, loff_t offset) 509a5f4374aSIngo Molnar { 510a5f4374aSIngo Molnar struct page *page; 511a5f4374aSIngo Molnar 512cc359d18SOleg Nesterov if (!mapping->a_ops->readpage) 513cc359d18SOleg Nesterov return -EIO; 514a5f4374aSIngo Molnar /* 515a5f4374aSIngo Molnar * Ensure that the page that has the original instruction is 516a5f4374aSIngo Molnar * populated and in page-cache. 517a5f4374aSIngo Molnar */ 5182edb7b55SOleg Nesterov page = read_mapping_page(mapping, offset >> PAGE_CACHE_SHIFT, filp); 519a5f4374aSIngo Molnar if (IS_ERR(page)) 520a5f4374aSIngo Molnar return PTR_ERR(page); 521a5f4374aSIngo Molnar 5222edb7b55SOleg Nesterov copy_from_page(page, offset, insn, nbytes); 523a5f4374aSIngo Molnar page_cache_release(page); 524a5f4374aSIngo Molnar 525a5f4374aSIngo Molnar return 0; 526a5f4374aSIngo Molnar } 527a5f4374aSIngo Molnar 528d436615eSOleg Nesterov static int copy_insn(struct uprobe *uprobe, struct file *filp) 529a5f4374aSIngo Molnar { 530a5f4374aSIngo Molnar struct address_space *mapping; 531a5f4374aSIngo Molnar unsigned long nbytes; 532a5f4374aSIngo Molnar int bytes; 533a5f4374aSIngo Molnar 534d436615eSOleg Nesterov nbytes = PAGE_SIZE - (uprobe->offset & ~PAGE_MASK); 535a5f4374aSIngo Molnar mapping = uprobe->inode->i_mapping; 536a5f4374aSIngo Molnar 537a5f4374aSIngo Molnar /* Instruction at end of binary; copy only available bytes */ 538a5f4374aSIngo Molnar if (uprobe->offset + MAX_UINSN_BYTES > uprobe->inode->i_size) 539a5f4374aSIngo Molnar bytes = uprobe->inode->i_size - uprobe->offset; 540a5f4374aSIngo Molnar else 541a5f4374aSIngo Molnar bytes = MAX_UINSN_BYTES; 542a5f4374aSIngo Molnar 543a5f4374aSIngo Molnar /* Instruction at the page-boundary; copy bytes in second page */ 544a5f4374aSIngo Molnar if (nbytes < bytes) { 545fc36f595SOleg Nesterov int err = __copy_insn(mapping, filp, uprobe->arch.insn + nbytes, 546fc36f595SOleg Nesterov bytes - nbytes, uprobe->offset + nbytes); 547fc36f595SOleg Nesterov if (err) 548fc36f595SOleg Nesterov return err; 549a5f4374aSIngo Molnar bytes = nbytes; 550a5f4374aSIngo Molnar } 551d436615eSOleg Nesterov return __copy_insn(mapping, filp, uprobe->arch.insn, bytes, uprobe->offset); 552a5f4374aSIngo Molnar } 553a5f4374aSIngo Molnar 554cb9a19feSOleg Nesterov static int prepare_uprobe(struct uprobe *uprobe, struct file *file, 555cb9a19feSOleg Nesterov struct mm_struct *mm, unsigned long vaddr) 556cb9a19feSOleg Nesterov { 557cb9a19feSOleg Nesterov int ret = 0; 558cb9a19feSOleg Nesterov 55971434f2fSOleg Nesterov if (test_bit(UPROBE_COPY_INSN, &uprobe->flags)) 560cb9a19feSOleg Nesterov return ret; 561cb9a19feSOleg Nesterov 562d4d3ccc6SOleg Nesterov /* TODO: move this into _register, until then we abuse this sem. */ 563d4d3ccc6SOleg Nesterov down_write(&uprobe->consumer_rwsem); 56471434f2fSOleg Nesterov if (test_bit(UPROBE_COPY_INSN, &uprobe->flags)) 5654710f05fSOleg Nesterov goto out; 5664710f05fSOleg Nesterov 567cb9a19feSOleg Nesterov ret = copy_insn(uprobe, file); 568cb9a19feSOleg Nesterov if (ret) 569cb9a19feSOleg Nesterov goto out; 570cb9a19feSOleg Nesterov 571cb9a19feSOleg Nesterov ret = -ENOTSUPP; 5720908ad6eSAnanth N Mavinakayanahalli if (is_trap_insn((uprobe_opcode_t *)uprobe->arch.insn)) 573cb9a19feSOleg Nesterov goto out; 574cb9a19feSOleg Nesterov 575cb9a19feSOleg Nesterov ret = arch_uprobe_analyze_insn(&uprobe->arch, mm, vaddr); 576cb9a19feSOleg Nesterov if (ret) 577cb9a19feSOleg Nesterov goto out; 578cb9a19feSOleg Nesterov 579cb9a19feSOleg Nesterov /* write_opcode() assumes we don't cross page boundary */ 580cb9a19feSOleg Nesterov BUG_ON((uprobe->offset & ~PAGE_MASK) + 581cb9a19feSOleg Nesterov UPROBE_SWBP_INSN_SIZE > PAGE_SIZE); 582cb9a19feSOleg Nesterov 583cb9a19feSOleg Nesterov smp_wmb(); /* pairs with rmb() in find_active_uprobe() */ 58471434f2fSOleg Nesterov set_bit(UPROBE_COPY_INSN, &uprobe->flags); 585cb9a19feSOleg Nesterov 586cb9a19feSOleg Nesterov out: 587d4d3ccc6SOleg Nesterov up_write(&uprobe->consumer_rwsem); 5884710f05fSOleg Nesterov 589cb9a19feSOleg Nesterov return ret; 590cb9a19feSOleg Nesterov } 591cb9a19feSOleg Nesterov 5928a7f2fa0SOleg Nesterov static inline bool consumer_filter(struct uprobe_consumer *uc, 5938a7f2fa0SOleg Nesterov enum uprobe_filter_ctx ctx, struct mm_struct *mm) 594806a98bdSOleg Nesterov { 5958a7f2fa0SOleg Nesterov return !uc->filter || uc->filter(uc, ctx, mm); 596806a98bdSOleg Nesterov } 597806a98bdSOleg Nesterov 5988a7f2fa0SOleg Nesterov static bool filter_chain(struct uprobe *uprobe, 5998a7f2fa0SOleg Nesterov enum uprobe_filter_ctx ctx, struct mm_struct *mm) 60063633cbfSOleg Nesterov { 6011ff6fee5SOleg Nesterov struct uprobe_consumer *uc; 6021ff6fee5SOleg Nesterov bool ret = false; 6031ff6fee5SOleg Nesterov 6041ff6fee5SOleg Nesterov down_read(&uprobe->consumer_rwsem); 6051ff6fee5SOleg Nesterov for (uc = uprobe->consumers; uc; uc = uc->next) { 6068a7f2fa0SOleg Nesterov ret = consumer_filter(uc, ctx, mm); 6071ff6fee5SOleg Nesterov if (ret) 6081ff6fee5SOleg Nesterov break; 6091ff6fee5SOleg Nesterov } 6101ff6fee5SOleg Nesterov up_read(&uprobe->consumer_rwsem); 6111ff6fee5SOleg Nesterov 6121ff6fee5SOleg Nesterov return ret; 61363633cbfSOleg Nesterov } 61463633cbfSOleg Nesterov 615e3343e6aSSrikar Dronamraju static int 616e3343e6aSSrikar Dronamraju install_breakpoint(struct uprobe *uprobe, struct mm_struct *mm, 617816c03fbSOleg Nesterov struct vm_area_struct *vma, unsigned long vaddr) 618a5f4374aSIngo Molnar { 619f8ac4ec9SOleg Nesterov bool first_uprobe; 620a5f4374aSIngo Molnar int ret; 621a5f4374aSIngo Molnar 622cb9a19feSOleg Nesterov ret = prepare_uprobe(uprobe, vma->vm_file, mm, vaddr); 623a5f4374aSIngo Molnar if (ret) 624a5f4374aSIngo Molnar return ret; 625a5f4374aSIngo Molnar 626f8ac4ec9SOleg Nesterov /* 627f8ac4ec9SOleg Nesterov * set MMF_HAS_UPROBES in advance for uprobe_pre_sstep_notifier(), 628f8ac4ec9SOleg Nesterov * the task can hit this breakpoint right after __replace_page(). 629f8ac4ec9SOleg Nesterov */ 630f8ac4ec9SOleg Nesterov first_uprobe = !test_bit(MMF_HAS_UPROBES, &mm->flags); 631f8ac4ec9SOleg Nesterov if (first_uprobe) 632f8ac4ec9SOleg Nesterov set_bit(MMF_HAS_UPROBES, &mm->flags); 633f8ac4ec9SOleg Nesterov 634816c03fbSOleg Nesterov ret = set_swbp(&uprobe->arch, mm, vaddr); 6359f68f672SOleg Nesterov if (!ret) 6369f68f672SOleg Nesterov clear_bit(MMF_RECALC_UPROBES, &mm->flags); 6379f68f672SOleg Nesterov else if (first_uprobe) 638f8ac4ec9SOleg Nesterov clear_bit(MMF_HAS_UPROBES, &mm->flags); 639a5f4374aSIngo Molnar 640a5f4374aSIngo Molnar return ret; 641a5f4374aSIngo Molnar } 642a5f4374aSIngo Molnar 643076a365bSOleg Nesterov static int 644816c03fbSOleg Nesterov remove_breakpoint(struct uprobe *uprobe, struct mm_struct *mm, unsigned long vaddr) 645a5f4374aSIngo Molnar { 6469f68f672SOleg Nesterov set_bit(MMF_RECALC_UPROBES, &mm->flags); 647076a365bSOleg Nesterov return set_orig_insn(&uprobe->arch, mm, vaddr); 648a5f4374aSIngo Molnar } 649a5f4374aSIngo Molnar 65006b7bcd8SOleg Nesterov static inline bool uprobe_is_active(struct uprobe *uprobe) 65106b7bcd8SOleg Nesterov { 65206b7bcd8SOleg Nesterov return !RB_EMPTY_NODE(&uprobe->rb_node); 65306b7bcd8SOleg Nesterov } 6540326f5a9SSrikar Dronamraju /* 655778b032dSOleg Nesterov * There could be threads that have already hit the breakpoint. They 656778b032dSOleg Nesterov * will recheck the current insn and restart if find_uprobe() fails. 657778b032dSOleg Nesterov * See find_active_uprobe(). 6580326f5a9SSrikar Dronamraju */ 659a5f4374aSIngo Molnar static void delete_uprobe(struct uprobe *uprobe) 660a5f4374aSIngo Molnar { 66106b7bcd8SOleg Nesterov if (WARN_ON(!uprobe_is_active(uprobe))) 66206b7bcd8SOleg Nesterov return; 66306b7bcd8SOleg Nesterov 6646f47caa0SOleg Nesterov spin_lock(&uprobes_treelock); 665a5f4374aSIngo Molnar rb_erase(&uprobe->rb_node, &uprobes_tree); 6666f47caa0SOleg Nesterov spin_unlock(&uprobes_treelock); 66706b7bcd8SOleg Nesterov RB_CLEAR_NODE(&uprobe->rb_node); /* for uprobe_is_active() */ 668a5f4374aSIngo Molnar iput(uprobe->inode); 669a5f4374aSIngo Molnar put_uprobe(uprobe); 670a5f4374aSIngo Molnar } 671a5f4374aSIngo Molnar 67226872090SOleg Nesterov struct map_info { 67326872090SOleg Nesterov struct map_info *next; 67426872090SOleg Nesterov struct mm_struct *mm; 675816c03fbSOleg Nesterov unsigned long vaddr; 67626872090SOleg Nesterov }; 67726872090SOleg Nesterov 67826872090SOleg Nesterov static inline struct map_info *free_map_info(struct map_info *info) 679a5f4374aSIngo Molnar { 68026872090SOleg Nesterov struct map_info *next = info->next; 68126872090SOleg Nesterov kfree(info); 68226872090SOleg Nesterov return next; 68326872090SOleg Nesterov } 68426872090SOleg Nesterov 68526872090SOleg Nesterov static struct map_info * 68626872090SOleg Nesterov build_map_info(struct address_space *mapping, loff_t offset, bool is_register) 68726872090SOleg Nesterov { 68826872090SOleg Nesterov unsigned long pgoff = offset >> PAGE_SHIFT; 689a5f4374aSIngo Molnar struct vm_area_struct *vma; 69026872090SOleg Nesterov struct map_info *curr = NULL; 69126872090SOleg Nesterov struct map_info *prev = NULL; 69226872090SOleg Nesterov struct map_info *info; 69326872090SOleg Nesterov int more = 0; 694a5f4374aSIngo Molnar 69526872090SOleg Nesterov again: 69626872090SOleg Nesterov mutex_lock(&mapping->i_mmap_mutex); 6976b2dbba8SMichel Lespinasse vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) { 698a5f4374aSIngo Molnar if (!valid_vma(vma, is_register)) 699a5f4374aSIngo Molnar continue; 700a5f4374aSIngo Molnar 7017a5bfb66SOleg Nesterov if (!prev && !more) { 7027a5bfb66SOleg Nesterov /* 7037a5bfb66SOleg Nesterov * Needs GFP_NOWAIT to avoid i_mmap_mutex recursion through 7047a5bfb66SOleg Nesterov * reclaim. This is optimistic, no harm done if it fails. 7057a5bfb66SOleg Nesterov */ 7067a5bfb66SOleg Nesterov prev = kmalloc(sizeof(struct map_info), 7077a5bfb66SOleg Nesterov GFP_NOWAIT | __GFP_NOMEMALLOC | __GFP_NOWARN); 7087a5bfb66SOleg Nesterov if (prev) 7097a5bfb66SOleg Nesterov prev->next = NULL; 7107a5bfb66SOleg Nesterov } 71126872090SOleg Nesterov if (!prev) { 71226872090SOleg Nesterov more++; 71326872090SOleg Nesterov continue; 714a5f4374aSIngo Molnar } 715a5f4374aSIngo Molnar 71626872090SOleg Nesterov if (!atomic_inc_not_zero(&vma->vm_mm->mm_users)) 71726872090SOleg Nesterov continue; 718a5f4374aSIngo Molnar 71926872090SOleg Nesterov info = prev; 72026872090SOleg Nesterov prev = prev->next; 72126872090SOleg Nesterov info->next = curr; 72226872090SOleg Nesterov curr = info; 72326872090SOleg Nesterov 72426872090SOleg Nesterov info->mm = vma->vm_mm; 72557683f72SOleg Nesterov info->vaddr = offset_to_vaddr(vma, offset); 726a5f4374aSIngo Molnar } 727a5f4374aSIngo Molnar mutex_unlock(&mapping->i_mmap_mutex); 728a5f4374aSIngo Molnar 72926872090SOleg Nesterov if (!more) 73026872090SOleg Nesterov goto out; 731a5f4374aSIngo Molnar 73226872090SOleg Nesterov prev = curr; 73326872090SOleg Nesterov while (curr) { 73426872090SOleg Nesterov mmput(curr->mm); 73526872090SOleg Nesterov curr = curr->next; 73626872090SOleg Nesterov } 73726872090SOleg Nesterov 73826872090SOleg Nesterov do { 73926872090SOleg Nesterov info = kmalloc(sizeof(struct map_info), GFP_KERNEL); 74026872090SOleg Nesterov if (!info) { 74126872090SOleg Nesterov curr = ERR_PTR(-ENOMEM); 74226872090SOleg Nesterov goto out; 74326872090SOleg Nesterov } 74426872090SOleg Nesterov info->next = prev; 74526872090SOleg Nesterov prev = info; 74626872090SOleg Nesterov } while (--more); 74726872090SOleg Nesterov 74826872090SOleg Nesterov goto again; 74926872090SOleg Nesterov out: 75026872090SOleg Nesterov while (prev) 75126872090SOleg Nesterov prev = free_map_info(prev); 75226872090SOleg Nesterov return curr; 753a5f4374aSIngo Molnar } 754a5f4374aSIngo Molnar 755bdf8647cSOleg Nesterov static int 756bdf8647cSOleg Nesterov register_for_each_vma(struct uprobe *uprobe, struct uprobe_consumer *new) 757a5f4374aSIngo Molnar { 758bdf8647cSOleg Nesterov bool is_register = !!new; 75926872090SOleg Nesterov struct map_info *info; 76026872090SOleg Nesterov int err = 0; 76126872090SOleg Nesterov 76232cdba1eSOleg Nesterov percpu_down_write(&dup_mmap_sem); 76326872090SOleg Nesterov info = build_map_info(uprobe->inode->i_mapping, 76426872090SOleg Nesterov uprobe->offset, is_register); 76532cdba1eSOleg Nesterov if (IS_ERR(info)) { 76632cdba1eSOleg Nesterov err = PTR_ERR(info); 76732cdba1eSOleg Nesterov goto out; 76832cdba1eSOleg Nesterov } 76926872090SOleg Nesterov 77026872090SOleg Nesterov while (info) { 77126872090SOleg Nesterov struct mm_struct *mm = info->mm; 772a5f4374aSIngo Molnar struct vm_area_struct *vma; 773a5f4374aSIngo Molnar 774076a365bSOleg Nesterov if (err && is_register) 77526872090SOleg Nesterov goto free; 776a5f4374aSIngo Molnar 77777fc4af1SOleg Nesterov down_write(&mm->mmap_sem); 778f4d6dfe5SOleg Nesterov vma = find_vma(mm, info->vaddr); 779f4d6dfe5SOleg Nesterov if (!vma || !valid_vma(vma, is_register) || 780f281769eSOleg Nesterov file_inode(vma->vm_file) != uprobe->inode) 78126872090SOleg Nesterov goto unlock; 78226872090SOleg Nesterov 783f4d6dfe5SOleg Nesterov if (vma->vm_start > info->vaddr || 784f4d6dfe5SOleg Nesterov vaddr_to_offset(vma, info->vaddr) != uprobe->offset) 78526872090SOleg Nesterov goto unlock; 786a5f4374aSIngo Molnar 787806a98bdSOleg Nesterov if (is_register) { 788806a98bdSOleg Nesterov /* consult only the "caller", new consumer. */ 789bdf8647cSOleg Nesterov if (consumer_filter(new, 7908a7f2fa0SOleg Nesterov UPROBE_FILTER_REGISTER, mm)) 79126872090SOleg Nesterov err = install_breakpoint(uprobe, mm, vma, info->vaddr); 792806a98bdSOleg Nesterov } else if (test_bit(MMF_HAS_UPROBES, &mm->flags)) { 7938a7f2fa0SOleg Nesterov if (!filter_chain(uprobe, 7948a7f2fa0SOleg Nesterov UPROBE_FILTER_UNREGISTER, mm)) 795076a365bSOleg Nesterov err |= remove_breakpoint(uprobe, mm, info->vaddr); 796806a98bdSOleg Nesterov } 79778f74116SOleg Nesterov 79826872090SOleg Nesterov unlock: 79926872090SOleg Nesterov up_write(&mm->mmap_sem); 80026872090SOleg Nesterov free: 80126872090SOleg Nesterov mmput(mm); 80226872090SOleg Nesterov info = free_map_info(info); 803a5f4374aSIngo Molnar } 80432cdba1eSOleg Nesterov out: 80532cdba1eSOleg Nesterov percpu_up_write(&dup_mmap_sem); 80626872090SOleg Nesterov return err; 807a5f4374aSIngo Molnar } 808a5f4374aSIngo Molnar 8099a98e03cSOleg Nesterov static int __uprobe_register(struct uprobe *uprobe, struct uprobe_consumer *uc) 810a5f4374aSIngo Molnar { 8119a98e03cSOleg Nesterov consumer_add(uprobe, uc); 812bdf8647cSOleg Nesterov return register_for_each_vma(uprobe, uc); 813a5f4374aSIngo Molnar } 814a5f4374aSIngo Molnar 81504aab9b2SOleg Nesterov static void __uprobe_unregister(struct uprobe *uprobe, struct uprobe_consumer *uc) 816a5f4374aSIngo Molnar { 81704aab9b2SOleg Nesterov int err; 818a5f4374aSIngo Molnar 81904aab9b2SOleg Nesterov if (!consumer_del(uprobe, uc)) /* WARN? */ 82004aab9b2SOleg Nesterov return; 82104aab9b2SOleg Nesterov 822bdf8647cSOleg Nesterov err = register_for_each_vma(uprobe, NULL); 823a5f4374aSIngo Molnar /* TODO : cant unregister? schedule a worker thread */ 824bb929284SOleg Nesterov if (!uprobe->consumers && !err) 82504aab9b2SOleg Nesterov delete_uprobe(uprobe); 82604aab9b2SOleg Nesterov } 827a5f4374aSIngo Molnar 828a5f4374aSIngo Molnar /* 829a5f4374aSIngo Molnar * uprobe_register - register a probe 830a5f4374aSIngo Molnar * @inode: the file in which the probe has to be placed. 831a5f4374aSIngo Molnar * @offset: offset from the start of the file. 832e3343e6aSSrikar Dronamraju * @uc: information on howto handle the probe.. 833a5f4374aSIngo Molnar * 834a5f4374aSIngo Molnar * Apart from the access refcount, uprobe_register() takes a creation 835a5f4374aSIngo Molnar * refcount (thro alloc_uprobe) if and only if this @uprobe is getting 836a5f4374aSIngo Molnar * inserted into the rbtree (i.e first consumer for a @inode:@offset 837a5f4374aSIngo Molnar * tuple). Creation refcount stops uprobe_unregister from freeing the 838a5f4374aSIngo Molnar * @uprobe even before the register operation is complete. Creation 839e3343e6aSSrikar Dronamraju * refcount is released when the last @uc for the @uprobe 840a5f4374aSIngo Molnar * unregisters. 841a5f4374aSIngo Molnar * 842a5f4374aSIngo Molnar * Return errno if it cannot successully install probes 843a5f4374aSIngo Molnar * else return 0 (success) 844a5f4374aSIngo Molnar */ 845e3343e6aSSrikar Dronamraju int uprobe_register(struct inode *inode, loff_t offset, struct uprobe_consumer *uc) 846a5f4374aSIngo Molnar { 847a5f4374aSIngo Molnar struct uprobe *uprobe; 848a5f4374aSIngo Molnar int ret; 849a5f4374aSIngo Molnar 850ea024870SAnton Arapov /* Uprobe must have at least one set consumer */ 851ea024870SAnton Arapov if (!uc->handler && !uc->ret_handler) 852ea024870SAnton Arapov return -EINVAL; 853ea024870SAnton Arapov 854ea024870SAnton Arapov /* TODO: Implement return probes */ 855ea024870SAnton Arapov if (uc->ret_handler) 856ea024870SAnton Arapov return -ENOSYS; 857ea024870SAnton Arapov 858f0744af7SOleg Nesterov /* Racy, just to catch the obvious mistakes */ 859a5f4374aSIngo Molnar if (offset > i_size_read(inode)) 860a5f4374aSIngo Molnar return -EINVAL; 861a5f4374aSIngo Molnar 86266d06dffSOleg Nesterov retry: 863a5f4374aSIngo Molnar uprobe = alloc_uprobe(inode, offset); 86466d06dffSOleg Nesterov if (!uprobe) 86566d06dffSOleg Nesterov return -ENOMEM; 86666d06dffSOleg Nesterov /* 86766d06dffSOleg Nesterov * We can race with uprobe_unregister()->delete_uprobe(). 86866d06dffSOleg Nesterov * Check uprobe_is_active() and retry if it is false. 86966d06dffSOleg Nesterov */ 870e591c8d7SOleg Nesterov down_write(&uprobe->register_rwsem); 87166d06dffSOleg Nesterov ret = -EAGAIN; 87266d06dffSOleg Nesterov if (likely(uprobe_is_active(uprobe))) { 8739a98e03cSOleg Nesterov ret = __uprobe_register(uprobe, uc); 8749a98e03cSOleg Nesterov if (ret) 87504aab9b2SOleg Nesterov __uprobe_unregister(uprobe, uc); 876a5f4374aSIngo Molnar } 87766d06dffSOleg Nesterov up_write(&uprobe->register_rwsem); 878a5f4374aSIngo Molnar put_uprobe(uprobe); 879a5f4374aSIngo Molnar 88066d06dffSOleg Nesterov if (unlikely(ret == -EAGAIN)) 88166d06dffSOleg Nesterov goto retry; 882a5f4374aSIngo Molnar return ret; 883a5f4374aSIngo Molnar } 884e8440c14SJosh Stone EXPORT_SYMBOL_GPL(uprobe_register); 885a5f4374aSIngo Molnar 886a5f4374aSIngo Molnar /* 887bdf8647cSOleg Nesterov * uprobe_apply - unregister a already registered probe. 888bdf8647cSOleg Nesterov * @inode: the file in which the probe has to be removed. 889bdf8647cSOleg Nesterov * @offset: offset from the start of the file. 890bdf8647cSOleg Nesterov * @uc: consumer which wants to add more or remove some breakpoints 891bdf8647cSOleg Nesterov * @add: add or remove the breakpoints 892bdf8647cSOleg Nesterov */ 893bdf8647cSOleg Nesterov int uprobe_apply(struct inode *inode, loff_t offset, 894bdf8647cSOleg Nesterov struct uprobe_consumer *uc, bool add) 895bdf8647cSOleg Nesterov { 896bdf8647cSOleg Nesterov struct uprobe *uprobe; 897bdf8647cSOleg Nesterov struct uprobe_consumer *con; 898bdf8647cSOleg Nesterov int ret = -ENOENT; 899bdf8647cSOleg Nesterov 900bdf8647cSOleg Nesterov uprobe = find_uprobe(inode, offset); 901bdf8647cSOleg Nesterov if (!uprobe) 902bdf8647cSOleg Nesterov return ret; 903bdf8647cSOleg Nesterov 904bdf8647cSOleg Nesterov down_write(&uprobe->register_rwsem); 905bdf8647cSOleg Nesterov for (con = uprobe->consumers; con && con != uc ; con = con->next) 906bdf8647cSOleg Nesterov ; 907bdf8647cSOleg Nesterov if (con) 908bdf8647cSOleg Nesterov ret = register_for_each_vma(uprobe, add ? uc : NULL); 909bdf8647cSOleg Nesterov up_write(&uprobe->register_rwsem); 910bdf8647cSOleg Nesterov put_uprobe(uprobe); 911bdf8647cSOleg Nesterov 912bdf8647cSOleg Nesterov return ret; 913bdf8647cSOleg Nesterov } 914bdf8647cSOleg Nesterov 915bdf8647cSOleg Nesterov /* 916a5f4374aSIngo Molnar * uprobe_unregister - unregister a already registered probe. 917a5f4374aSIngo Molnar * @inode: the file in which the probe has to be removed. 918a5f4374aSIngo Molnar * @offset: offset from the start of the file. 919e3343e6aSSrikar Dronamraju * @uc: identify which probe if multiple probes are colocated. 920a5f4374aSIngo Molnar */ 921e3343e6aSSrikar Dronamraju void uprobe_unregister(struct inode *inode, loff_t offset, struct uprobe_consumer *uc) 922a5f4374aSIngo Molnar { 923a5f4374aSIngo Molnar struct uprobe *uprobe; 924a5f4374aSIngo Molnar 925a5f4374aSIngo Molnar uprobe = find_uprobe(inode, offset); 926a5f4374aSIngo Molnar if (!uprobe) 927a5f4374aSIngo Molnar return; 928a5f4374aSIngo Molnar 929e591c8d7SOleg Nesterov down_write(&uprobe->register_rwsem); 93004aab9b2SOleg Nesterov __uprobe_unregister(uprobe, uc); 931e591c8d7SOleg Nesterov up_write(&uprobe->register_rwsem); 932a5f4374aSIngo Molnar put_uprobe(uprobe); 933a5f4374aSIngo Molnar } 934e8440c14SJosh Stone EXPORT_SYMBOL_GPL(uprobe_unregister); 935a5f4374aSIngo Molnar 936da1816b1SOleg Nesterov static int unapply_uprobe(struct uprobe *uprobe, struct mm_struct *mm) 937da1816b1SOleg Nesterov { 938da1816b1SOleg Nesterov struct vm_area_struct *vma; 939da1816b1SOleg Nesterov int err = 0; 940da1816b1SOleg Nesterov 941da1816b1SOleg Nesterov down_read(&mm->mmap_sem); 942da1816b1SOleg Nesterov for (vma = mm->mmap; vma; vma = vma->vm_next) { 943da1816b1SOleg Nesterov unsigned long vaddr; 944da1816b1SOleg Nesterov loff_t offset; 945da1816b1SOleg Nesterov 946da1816b1SOleg Nesterov if (!valid_vma(vma, false) || 947f281769eSOleg Nesterov file_inode(vma->vm_file) != uprobe->inode) 948da1816b1SOleg Nesterov continue; 949da1816b1SOleg Nesterov 950da1816b1SOleg Nesterov offset = (loff_t)vma->vm_pgoff << PAGE_SHIFT; 951da1816b1SOleg Nesterov if (uprobe->offset < offset || 952da1816b1SOleg Nesterov uprobe->offset >= offset + vma->vm_end - vma->vm_start) 953da1816b1SOleg Nesterov continue; 954da1816b1SOleg Nesterov 955da1816b1SOleg Nesterov vaddr = offset_to_vaddr(vma, uprobe->offset); 956da1816b1SOleg Nesterov err |= remove_breakpoint(uprobe, mm, vaddr); 957da1816b1SOleg Nesterov } 958da1816b1SOleg Nesterov up_read(&mm->mmap_sem); 959da1816b1SOleg Nesterov 960da1816b1SOleg Nesterov return err; 961da1816b1SOleg Nesterov } 962da1816b1SOleg Nesterov 963891c3970SOleg Nesterov static struct rb_node * 964891c3970SOleg Nesterov find_node_in_range(struct inode *inode, loff_t min, loff_t max) 965a5f4374aSIngo Molnar { 966a5f4374aSIngo Molnar struct rb_node *n = uprobes_tree.rb_node; 967a5f4374aSIngo Molnar 968a5f4374aSIngo Molnar while (n) { 969891c3970SOleg Nesterov struct uprobe *u = rb_entry(n, struct uprobe, rb_node); 970a5f4374aSIngo Molnar 971891c3970SOleg Nesterov if (inode < u->inode) { 972a5f4374aSIngo Molnar n = n->rb_left; 973891c3970SOleg Nesterov } else if (inode > u->inode) { 974a5f4374aSIngo Molnar n = n->rb_right; 975891c3970SOleg Nesterov } else { 976891c3970SOleg Nesterov if (max < u->offset) 977891c3970SOleg Nesterov n = n->rb_left; 978891c3970SOleg Nesterov else if (min > u->offset) 979891c3970SOleg Nesterov n = n->rb_right; 980891c3970SOleg Nesterov else 981891c3970SOleg Nesterov break; 982891c3970SOleg Nesterov } 983a5f4374aSIngo Molnar } 984a5f4374aSIngo Molnar 985891c3970SOleg Nesterov return n; 986a5f4374aSIngo Molnar } 987a5f4374aSIngo Molnar 988a5f4374aSIngo Molnar /* 989891c3970SOleg Nesterov * For a given range in vma, build a list of probes that need to be inserted. 990a5f4374aSIngo Molnar */ 991891c3970SOleg Nesterov static void build_probe_list(struct inode *inode, 992891c3970SOleg Nesterov struct vm_area_struct *vma, 993891c3970SOleg Nesterov unsigned long start, unsigned long end, 994891c3970SOleg Nesterov struct list_head *head) 995a5f4374aSIngo Molnar { 996891c3970SOleg Nesterov loff_t min, max; 997891c3970SOleg Nesterov struct rb_node *n, *t; 998891c3970SOleg Nesterov struct uprobe *u; 999891c3970SOleg Nesterov 1000891c3970SOleg Nesterov INIT_LIST_HEAD(head); 1001cb113b47SOleg Nesterov min = vaddr_to_offset(vma, start); 1002891c3970SOleg Nesterov max = min + (end - start) - 1; 1003a5f4374aSIngo Molnar 10046f47caa0SOleg Nesterov spin_lock(&uprobes_treelock); 1005891c3970SOleg Nesterov n = find_node_in_range(inode, min, max); 1006891c3970SOleg Nesterov if (n) { 1007891c3970SOleg Nesterov for (t = n; t; t = rb_prev(t)) { 1008891c3970SOleg Nesterov u = rb_entry(t, struct uprobe, rb_node); 1009891c3970SOleg Nesterov if (u->inode != inode || u->offset < min) 1010a5f4374aSIngo Molnar break; 1011891c3970SOleg Nesterov list_add(&u->pending_list, head); 1012891c3970SOleg Nesterov atomic_inc(&u->ref); 1013a5f4374aSIngo Molnar } 1014891c3970SOleg Nesterov for (t = n; (t = rb_next(t)); ) { 1015891c3970SOleg Nesterov u = rb_entry(t, struct uprobe, rb_node); 1016891c3970SOleg Nesterov if (u->inode != inode || u->offset > max) 1017891c3970SOleg Nesterov break; 1018891c3970SOleg Nesterov list_add(&u->pending_list, head); 1019891c3970SOleg Nesterov atomic_inc(&u->ref); 1020891c3970SOleg Nesterov } 1021891c3970SOleg Nesterov } 10226f47caa0SOleg Nesterov spin_unlock(&uprobes_treelock); 1023a5f4374aSIngo Molnar } 1024a5f4374aSIngo Molnar 1025a5f4374aSIngo Molnar /* 10265e5be71aSOleg Nesterov * Called from mmap_region/vma_adjust with mm->mmap_sem acquired. 1027a5f4374aSIngo Molnar * 10285e5be71aSOleg Nesterov * Currently we ignore all errors and always return 0, the callers 10295e5be71aSOleg Nesterov * can't handle the failure anyway. 1030a5f4374aSIngo Molnar */ 1031a5f4374aSIngo Molnar int uprobe_mmap(struct vm_area_struct *vma) 1032a5f4374aSIngo Molnar { 1033a5f4374aSIngo Molnar struct list_head tmp_list; 1034665605a2SOleg Nesterov struct uprobe *uprobe, *u; 1035a5f4374aSIngo Molnar struct inode *inode; 1036a5f4374aSIngo Molnar 1037441f1eb7SOleg Nesterov if (no_uprobe_events() || !valid_vma(vma, true)) 1038a5f4374aSIngo Molnar return 0; 1039a5f4374aSIngo Molnar 1040f281769eSOleg Nesterov inode = file_inode(vma->vm_file); 1041a5f4374aSIngo Molnar if (!inode) 1042a5f4374aSIngo Molnar return 0; 1043a5f4374aSIngo Molnar 1044a5f4374aSIngo Molnar mutex_lock(uprobes_mmap_hash(inode)); 1045891c3970SOleg Nesterov build_probe_list(inode, vma, vma->vm_start, vma->vm_end, &tmp_list); 1046806a98bdSOleg Nesterov /* 1047806a98bdSOleg Nesterov * We can race with uprobe_unregister(), this uprobe can be already 1048806a98bdSOleg Nesterov * removed. But in this case filter_chain() must return false, all 1049806a98bdSOleg Nesterov * consumers have gone away. 1050806a98bdSOleg Nesterov */ 1051665605a2SOleg Nesterov list_for_each_entry_safe(uprobe, u, &tmp_list, pending_list) { 1052806a98bdSOleg Nesterov if (!fatal_signal_pending(current) && 10538a7f2fa0SOleg Nesterov filter_chain(uprobe, UPROBE_FILTER_MMAP, vma->vm_mm)) { 105457683f72SOleg Nesterov unsigned long vaddr = offset_to_vaddr(vma, uprobe->offset); 10555e5be71aSOleg Nesterov install_breakpoint(uprobe, vma->vm_mm, vma, vaddr); 1056a5f4374aSIngo Molnar } 1057a5f4374aSIngo Molnar put_uprobe(uprobe); 1058a5f4374aSIngo Molnar } 1059a5f4374aSIngo Molnar mutex_unlock(uprobes_mmap_hash(inode)); 1060a5f4374aSIngo Molnar 10615e5be71aSOleg Nesterov return 0; 1062a5f4374aSIngo Molnar } 1063a5f4374aSIngo Molnar 10649f68f672SOleg Nesterov static bool 10659f68f672SOleg Nesterov vma_has_uprobes(struct vm_area_struct *vma, unsigned long start, unsigned long end) 10669f68f672SOleg Nesterov { 10679f68f672SOleg Nesterov loff_t min, max; 10689f68f672SOleg Nesterov struct inode *inode; 10699f68f672SOleg Nesterov struct rb_node *n; 10709f68f672SOleg Nesterov 1071f281769eSOleg Nesterov inode = file_inode(vma->vm_file); 10729f68f672SOleg Nesterov 10739f68f672SOleg Nesterov min = vaddr_to_offset(vma, start); 10749f68f672SOleg Nesterov max = min + (end - start) - 1; 10759f68f672SOleg Nesterov 10769f68f672SOleg Nesterov spin_lock(&uprobes_treelock); 10779f68f672SOleg Nesterov n = find_node_in_range(inode, min, max); 10789f68f672SOleg Nesterov spin_unlock(&uprobes_treelock); 10799f68f672SOleg Nesterov 10809f68f672SOleg Nesterov return !!n; 10819f68f672SOleg Nesterov } 10829f68f672SOleg Nesterov 1083682968e0SSrikar Dronamraju /* 1084682968e0SSrikar Dronamraju * Called in context of a munmap of a vma. 1085682968e0SSrikar Dronamraju */ 1086cbc91f71SSrikar Dronamraju void uprobe_munmap(struct vm_area_struct *vma, unsigned long start, unsigned long end) 1087682968e0SSrikar Dronamraju { 1088441f1eb7SOleg Nesterov if (no_uprobe_events() || !valid_vma(vma, false)) 1089682968e0SSrikar Dronamraju return; 1090682968e0SSrikar Dronamraju 10912fd611a9SOleg Nesterov if (!atomic_read(&vma->vm_mm->mm_users)) /* called by mmput() ? */ 10922fd611a9SOleg Nesterov return; 10932fd611a9SOleg Nesterov 10949f68f672SOleg Nesterov if (!test_bit(MMF_HAS_UPROBES, &vma->vm_mm->flags) || 10959f68f672SOleg Nesterov test_bit(MMF_RECALC_UPROBES, &vma->vm_mm->flags)) 1096f8ac4ec9SOleg Nesterov return; 1097f8ac4ec9SOleg Nesterov 10989f68f672SOleg Nesterov if (vma_has_uprobes(vma, start, end)) 10999f68f672SOleg Nesterov set_bit(MMF_RECALC_UPROBES, &vma->vm_mm->flags); 1100682968e0SSrikar Dronamraju } 1101682968e0SSrikar Dronamraju 1102d4b3b638SSrikar Dronamraju /* Slot allocation for XOL */ 1103d4b3b638SSrikar Dronamraju static int xol_add_vma(struct xol_area *area) 1104d4b3b638SSrikar Dronamraju { 1105c8a82538SOleg Nesterov struct mm_struct *mm = current->mm; 1106c8a82538SOleg Nesterov int ret = -EALREADY; 1107d4b3b638SSrikar Dronamraju 1108d4b3b638SSrikar Dronamraju down_write(&mm->mmap_sem); 1109d4b3b638SSrikar Dronamraju if (mm->uprobes_state.xol_area) 1110d4b3b638SSrikar Dronamraju goto fail; 1111d4b3b638SSrikar Dronamraju 1112d4b3b638SSrikar Dronamraju ret = -ENOMEM; 1113d4b3b638SSrikar Dronamraju /* Try to map as high as possible, this is only a hint. */ 1114d4b3b638SSrikar Dronamraju area->vaddr = get_unmapped_area(NULL, TASK_SIZE - PAGE_SIZE, PAGE_SIZE, 0, 0); 1115d4b3b638SSrikar Dronamraju if (area->vaddr & ~PAGE_MASK) { 1116d4b3b638SSrikar Dronamraju ret = area->vaddr; 1117d4b3b638SSrikar Dronamraju goto fail; 1118d4b3b638SSrikar Dronamraju } 1119d4b3b638SSrikar Dronamraju 1120d4b3b638SSrikar Dronamraju ret = install_special_mapping(mm, area->vaddr, PAGE_SIZE, 1121d4b3b638SSrikar Dronamraju VM_EXEC|VM_MAYEXEC|VM_DONTCOPY|VM_IO, &area->page); 1122d4b3b638SSrikar Dronamraju if (ret) 1123d4b3b638SSrikar Dronamraju goto fail; 1124d4b3b638SSrikar Dronamraju 1125d4b3b638SSrikar Dronamraju smp_wmb(); /* pairs with get_xol_area() */ 1126d4b3b638SSrikar Dronamraju mm->uprobes_state.xol_area = area; 1127d4b3b638SSrikar Dronamraju ret = 0; 1128d4b3b638SSrikar Dronamraju fail: 1129d4b3b638SSrikar Dronamraju up_write(&mm->mmap_sem); 1130d4b3b638SSrikar Dronamraju 1131d4b3b638SSrikar Dronamraju return ret; 1132d4b3b638SSrikar Dronamraju } 1133d4b3b638SSrikar Dronamraju 1134d4b3b638SSrikar Dronamraju /* 11359b545df8SOleg Nesterov * get_xol_area - Allocate process's xol_area if necessary. 11369b545df8SOleg Nesterov * This area will be used for storing instructions for execution out of line. 1137d4b3b638SSrikar Dronamraju * 1138d4b3b638SSrikar Dronamraju * Returns the allocated area or NULL. 1139d4b3b638SSrikar Dronamraju */ 11409b545df8SOleg Nesterov static struct xol_area *get_xol_area(void) 1141d4b3b638SSrikar Dronamraju { 11429b545df8SOleg Nesterov struct mm_struct *mm = current->mm; 1143d4b3b638SSrikar Dronamraju struct xol_area *area; 1144e78aebfdSAnton Arapov uprobe_opcode_t insn = UPROBE_SWBP_INSN; 1145d4b3b638SSrikar Dronamraju 11469b545df8SOleg Nesterov area = mm->uprobes_state.xol_area; 11479b545df8SOleg Nesterov if (area) 11489b545df8SOleg Nesterov goto ret; 11499b545df8SOleg Nesterov 1150d4b3b638SSrikar Dronamraju area = kzalloc(sizeof(*area), GFP_KERNEL); 1151d4b3b638SSrikar Dronamraju if (unlikely(!area)) 1152c8a82538SOleg Nesterov goto out; 1153d4b3b638SSrikar Dronamraju 1154d4b3b638SSrikar Dronamraju area->bitmap = kzalloc(BITS_TO_LONGS(UINSNS_PER_PAGE) * sizeof(long), GFP_KERNEL); 1155d4b3b638SSrikar Dronamraju if (!area->bitmap) 1156c8a82538SOleg Nesterov goto free_area; 1157c8a82538SOleg Nesterov 1158c8a82538SOleg Nesterov area->page = alloc_page(GFP_HIGHUSER); 1159c8a82538SOleg Nesterov if (!area->page) 1160c8a82538SOleg Nesterov goto free_bitmap; 1161d4b3b638SSrikar Dronamraju 1162e78aebfdSAnton Arapov /* allocate first slot of task's xol_area for the return probes */ 1163e78aebfdSAnton Arapov set_bit(0, area->bitmap); 1164e78aebfdSAnton Arapov copy_to_page(area->page, 0, &insn, UPROBE_SWBP_INSN_SIZE); 1165e78aebfdSAnton Arapov atomic_set(&area->slot_count, 1); 1166d4b3b638SSrikar Dronamraju init_waitqueue_head(&area->wq); 1167e78aebfdSAnton Arapov 1168d4b3b638SSrikar Dronamraju if (!xol_add_vma(area)) 1169d4b3b638SSrikar Dronamraju return area; 1170d4b3b638SSrikar Dronamraju 1171c8a82538SOleg Nesterov __free_page(area->page); 1172c8a82538SOleg Nesterov free_bitmap: 1173d4b3b638SSrikar Dronamraju kfree(area->bitmap); 1174c8a82538SOleg Nesterov free_area: 1175d4b3b638SSrikar Dronamraju kfree(area); 1176c8a82538SOleg Nesterov out: 11779b545df8SOleg Nesterov area = mm->uprobes_state.xol_area; 11789b545df8SOleg Nesterov ret: 11799b545df8SOleg Nesterov smp_read_barrier_depends(); /* pairs with wmb in xol_add_vma() */ 11809b545df8SOleg Nesterov return area; 1181d4b3b638SSrikar Dronamraju } 1182d4b3b638SSrikar Dronamraju 1183d4b3b638SSrikar Dronamraju /* 1184d4b3b638SSrikar Dronamraju * uprobe_clear_state - Free the area allocated for slots. 1185d4b3b638SSrikar Dronamraju */ 1186d4b3b638SSrikar Dronamraju void uprobe_clear_state(struct mm_struct *mm) 1187d4b3b638SSrikar Dronamraju { 1188d4b3b638SSrikar Dronamraju struct xol_area *area = mm->uprobes_state.xol_area; 1189d4b3b638SSrikar Dronamraju 1190d4b3b638SSrikar Dronamraju if (!area) 1191d4b3b638SSrikar Dronamraju return; 1192d4b3b638SSrikar Dronamraju 1193d4b3b638SSrikar Dronamraju put_page(area->page); 1194d4b3b638SSrikar Dronamraju kfree(area->bitmap); 1195d4b3b638SSrikar Dronamraju kfree(area); 1196d4b3b638SSrikar Dronamraju } 1197d4b3b638SSrikar Dronamraju 119832cdba1eSOleg Nesterov void uprobe_start_dup_mmap(void) 119932cdba1eSOleg Nesterov { 120032cdba1eSOleg Nesterov percpu_down_read(&dup_mmap_sem); 120132cdba1eSOleg Nesterov } 120232cdba1eSOleg Nesterov 120332cdba1eSOleg Nesterov void uprobe_end_dup_mmap(void) 120432cdba1eSOleg Nesterov { 120532cdba1eSOleg Nesterov percpu_up_read(&dup_mmap_sem); 120632cdba1eSOleg Nesterov } 120732cdba1eSOleg Nesterov 1208f8ac4ec9SOleg Nesterov void uprobe_dup_mmap(struct mm_struct *oldmm, struct mm_struct *newmm) 1209f8ac4ec9SOleg Nesterov { 121061559a81SOleg Nesterov newmm->uprobes_state.xol_area = NULL; 121161559a81SOleg Nesterov 12129f68f672SOleg Nesterov if (test_bit(MMF_HAS_UPROBES, &oldmm->flags)) { 1213f8ac4ec9SOleg Nesterov set_bit(MMF_HAS_UPROBES, &newmm->flags); 12149f68f672SOleg Nesterov /* unconditionally, dup_mmap() skips VM_DONTCOPY vmas */ 12159f68f672SOleg Nesterov set_bit(MMF_RECALC_UPROBES, &newmm->flags); 12169f68f672SOleg Nesterov } 1217f8ac4ec9SOleg Nesterov } 1218f8ac4ec9SOleg Nesterov 1219d4b3b638SSrikar Dronamraju /* 1220d4b3b638SSrikar Dronamraju * - search for a free slot. 1221d4b3b638SSrikar Dronamraju */ 1222d4b3b638SSrikar Dronamraju static unsigned long xol_take_insn_slot(struct xol_area *area) 1223d4b3b638SSrikar Dronamraju { 1224d4b3b638SSrikar Dronamraju unsigned long slot_addr; 1225d4b3b638SSrikar Dronamraju int slot_nr; 1226d4b3b638SSrikar Dronamraju 1227d4b3b638SSrikar Dronamraju do { 1228d4b3b638SSrikar Dronamraju slot_nr = find_first_zero_bit(area->bitmap, UINSNS_PER_PAGE); 1229d4b3b638SSrikar Dronamraju if (slot_nr < UINSNS_PER_PAGE) { 1230d4b3b638SSrikar Dronamraju if (!test_and_set_bit(slot_nr, area->bitmap)) 1231d4b3b638SSrikar Dronamraju break; 1232d4b3b638SSrikar Dronamraju 1233d4b3b638SSrikar Dronamraju slot_nr = UINSNS_PER_PAGE; 1234d4b3b638SSrikar Dronamraju continue; 1235d4b3b638SSrikar Dronamraju } 1236d4b3b638SSrikar Dronamraju wait_event(area->wq, (atomic_read(&area->slot_count) < UINSNS_PER_PAGE)); 1237d4b3b638SSrikar Dronamraju } while (slot_nr >= UINSNS_PER_PAGE); 1238d4b3b638SSrikar Dronamraju 1239d4b3b638SSrikar Dronamraju slot_addr = area->vaddr + (slot_nr * UPROBE_XOL_SLOT_BYTES); 1240d4b3b638SSrikar Dronamraju atomic_inc(&area->slot_count); 1241d4b3b638SSrikar Dronamraju 1242d4b3b638SSrikar Dronamraju return slot_addr; 1243d4b3b638SSrikar Dronamraju } 1244d4b3b638SSrikar Dronamraju 1245d4b3b638SSrikar Dronamraju /* 1246a6cb3f6dSOleg Nesterov * xol_get_insn_slot - allocate a slot for xol. 1247d4b3b638SSrikar Dronamraju * Returns the allocated slot address or 0. 1248d4b3b638SSrikar Dronamraju */ 1249a6cb3f6dSOleg Nesterov static unsigned long xol_get_insn_slot(struct uprobe *uprobe) 1250d4b3b638SSrikar Dronamraju { 1251d4b3b638SSrikar Dronamraju struct xol_area *area; 1252a6cb3f6dSOleg Nesterov unsigned long xol_vaddr; 1253d4b3b638SSrikar Dronamraju 12549b545df8SOleg Nesterov area = get_xol_area(); 1255d4b3b638SSrikar Dronamraju if (!area) 1256d4b3b638SSrikar Dronamraju return 0; 1257d4b3b638SSrikar Dronamraju 1258a6cb3f6dSOleg Nesterov xol_vaddr = xol_take_insn_slot(area); 1259a6cb3f6dSOleg Nesterov if (unlikely(!xol_vaddr)) 1260d4b3b638SSrikar Dronamraju return 0; 1261d4b3b638SSrikar Dronamraju 1262a6cb3f6dSOleg Nesterov /* Initialize the slot */ 12635669cceeSOleg Nesterov copy_to_page(area->page, xol_vaddr, uprobe->arch.insn, MAX_UINSN_BYTES); 126465b6ecc0SRabin Vincent /* 126565b6ecc0SRabin Vincent * We probably need flush_icache_user_range() but it needs vma. 126665b6ecc0SRabin Vincent * This should work on supported architectures too. 126765b6ecc0SRabin Vincent */ 126865b6ecc0SRabin Vincent flush_dcache_page(area->page); 1269d4b3b638SSrikar Dronamraju 1270a6cb3f6dSOleg Nesterov return xol_vaddr; 1271d4b3b638SSrikar Dronamraju } 1272d4b3b638SSrikar Dronamraju 1273d4b3b638SSrikar Dronamraju /* 1274d4b3b638SSrikar Dronamraju * xol_free_insn_slot - If slot was earlier allocated by 1275d4b3b638SSrikar Dronamraju * @xol_get_insn_slot(), make the slot available for 1276d4b3b638SSrikar Dronamraju * subsequent requests. 1277d4b3b638SSrikar Dronamraju */ 1278d4b3b638SSrikar Dronamraju static void xol_free_insn_slot(struct task_struct *tsk) 1279d4b3b638SSrikar Dronamraju { 1280d4b3b638SSrikar Dronamraju struct xol_area *area; 1281d4b3b638SSrikar Dronamraju unsigned long vma_end; 1282d4b3b638SSrikar Dronamraju unsigned long slot_addr; 1283d4b3b638SSrikar Dronamraju 1284d4b3b638SSrikar Dronamraju if (!tsk->mm || !tsk->mm->uprobes_state.xol_area || !tsk->utask) 1285d4b3b638SSrikar Dronamraju return; 1286d4b3b638SSrikar Dronamraju 1287d4b3b638SSrikar Dronamraju slot_addr = tsk->utask->xol_vaddr; 1288af4355e9SOleg Nesterov if (unlikely(!slot_addr)) 1289d4b3b638SSrikar Dronamraju return; 1290d4b3b638SSrikar Dronamraju 1291d4b3b638SSrikar Dronamraju area = tsk->mm->uprobes_state.xol_area; 1292d4b3b638SSrikar Dronamraju vma_end = area->vaddr + PAGE_SIZE; 1293d4b3b638SSrikar Dronamraju if (area->vaddr <= slot_addr && slot_addr < vma_end) { 1294d4b3b638SSrikar Dronamraju unsigned long offset; 1295d4b3b638SSrikar Dronamraju int slot_nr; 1296d4b3b638SSrikar Dronamraju 1297d4b3b638SSrikar Dronamraju offset = slot_addr - area->vaddr; 1298d4b3b638SSrikar Dronamraju slot_nr = offset / UPROBE_XOL_SLOT_BYTES; 1299d4b3b638SSrikar Dronamraju if (slot_nr >= UINSNS_PER_PAGE) 1300d4b3b638SSrikar Dronamraju return; 1301d4b3b638SSrikar Dronamraju 1302d4b3b638SSrikar Dronamraju clear_bit(slot_nr, area->bitmap); 1303d4b3b638SSrikar Dronamraju atomic_dec(&area->slot_count); 1304d4b3b638SSrikar Dronamraju if (waitqueue_active(&area->wq)) 1305d4b3b638SSrikar Dronamraju wake_up(&area->wq); 1306d4b3b638SSrikar Dronamraju 1307d4b3b638SSrikar Dronamraju tsk->utask->xol_vaddr = 0; 1308d4b3b638SSrikar Dronamraju } 1309d4b3b638SSrikar Dronamraju } 1310d4b3b638SSrikar Dronamraju 13110326f5a9SSrikar Dronamraju /** 13120326f5a9SSrikar Dronamraju * uprobe_get_swbp_addr - compute address of swbp given post-swbp regs 13130326f5a9SSrikar Dronamraju * @regs: Reflects the saved state of the task after it has hit a breakpoint 13140326f5a9SSrikar Dronamraju * instruction. 13150326f5a9SSrikar Dronamraju * Return the address of the breakpoint instruction. 13160326f5a9SSrikar Dronamraju */ 13170326f5a9SSrikar Dronamraju unsigned long __weak uprobe_get_swbp_addr(struct pt_regs *regs) 13180326f5a9SSrikar Dronamraju { 13190326f5a9SSrikar Dronamraju return instruction_pointer(regs) - UPROBE_SWBP_INSN_SIZE; 13200326f5a9SSrikar Dronamraju } 13210326f5a9SSrikar Dronamraju 13220326f5a9SSrikar Dronamraju /* 13230326f5a9SSrikar Dronamraju * Called with no locks held. 13240326f5a9SSrikar Dronamraju * Called in context of a exiting or a exec-ing thread. 13250326f5a9SSrikar Dronamraju */ 13260326f5a9SSrikar Dronamraju void uprobe_free_utask(struct task_struct *t) 13270326f5a9SSrikar Dronamraju { 13280326f5a9SSrikar Dronamraju struct uprobe_task *utask = t->utask; 13290dfd0eb8SAnton Arapov struct return_instance *ri, *tmp; 13300326f5a9SSrikar Dronamraju 13310326f5a9SSrikar Dronamraju if (!utask) 13320326f5a9SSrikar Dronamraju return; 13330326f5a9SSrikar Dronamraju 13340326f5a9SSrikar Dronamraju if (utask->active_uprobe) 13350326f5a9SSrikar Dronamraju put_uprobe(utask->active_uprobe); 13360326f5a9SSrikar Dronamraju 13370dfd0eb8SAnton Arapov ri = utask->return_instances; 13380dfd0eb8SAnton Arapov while (ri) { 13390dfd0eb8SAnton Arapov tmp = ri; 13400dfd0eb8SAnton Arapov ri = ri->next; 13410dfd0eb8SAnton Arapov 13420dfd0eb8SAnton Arapov put_uprobe(tmp->uprobe); 13430dfd0eb8SAnton Arapov kfree(tmp); 13440dfd0eb8SAnton Arapov } 13450dfd0eb8SAnton Arapov 1346d4b3b638SSrikar Dronamraju xol_free_insn_slot(t); 13470326f5a9SSrikar Dronamraju kfree(utask); 13480326f5a9SSrikar Dronamraju t->utask = NULL; 13490326f5a9SSrikar Dronamraju } 13500326f5a9SSrikar Dronamraju 13510326f5a9SSrikar Dronamraju /* 13520326f5a9SSrikar Dronamraju * Called in context of a new clone/fork from copy_process. 13530326f5a9SSrikar Dronamraju */ 13540326f5a9SSrikar Dronamraju void uprobe_copy_process(struct task_struct *t) 13550326f5a9SSrikar Dronamraju { 13560326f5a9SSrikar Dronamraju t->utask = NULL; 13570326f5a9SSrikar Dronamraju } 13580326f5a9SSrikar Dronamraju 13590326f5a9SSrikar Dronamraju /* 13605a2df662SOleg Nesterov * Allocate a uprobe_task object for the task if if necessary. 13615a2df662SOleg Nesterov * Called when the thread hits a breakpoint. 13620326f5a9SSrikar Dronamraju * 13630326f5a9SSrikar Dronamraju * Returns: 13640326f5a9SSrikar Dronamraju * - pointer to new uprobe_task on success 13650326f5a9SSrikar Dronamraju * - NULL otherwise 13660326f5a9SSrikar Dronamraju */ 13675a2df662SOleg Nesterov static struct uprobe_task *get_utask(void) 13680326f5a9SSrikar Dronamraju { 13695a2df662SOleg Nesterov if (!current->utask) 13705a2df662SOleg Nesterov current->utask = kzalloc(sizeof(struct uprobe_task), GFP_KERNEL); 13715a2df662SOleg Nesterov return current->utask; 13720326f5a9SSrikar Dronamraju } 13730326f5a9SSrikar Dronamraju 1374e78aebfdSAnton Arapov /* 1375e78aebfdSAnton Arapov * Current area->vaddr notion assume the trampoline address is always 1376e78aebfdSAnton Arapov * equal area->vaddr. 1377e78aebfdSAnton Arapov * 1378e78aebfdSAnton Arapov * Returns -1 in case the xol_area is not allocated. 1379e78aebfdSAnton Arapov */ 1380e78aebfdSAnton Arapov static unsigned long get_trampoline_vaddr(void) 1381e78aebfdSAnton Arapov { 1382e78aebfdSAnton Arapov struct xol_area *area; 1383e78aebfdSAnton Arapov unsigned long trampoline_vaddr = -1; 1384e78aebfdSAnton Arapov 1385e78aebfdSAnton Arapov area = current->mm->uprobes_state.xol_area; 1386e78aebfdSAnton Arapov smp_read_barrier_depends(); 1387e78aebfdSAnton Arapov if (area) 1388e78aebfdSAnton Arapov trampoline_vaddr = area->vaddr; 1389e78aebfdSAnton Arapov 1390e78aebfdSAnton Arapov return trampoline_vaddr; 1391e78aebfdSAnton Arapov } 1392e78aebfdSAnton Arapov 13930dfd0eb8SAnton Arapov static void prepare_uretprobe(struct uprobe *uprobe, struct pt_regs *regs) 13940dfd0eb8SAnton Arapov { 13950dfd0eb8SAnton Arapov struct return_instance *ri; 13960dfd0eb8SAnton Arapov struct uprobe_task *utask; 13970dfd0eb8SAnton Arapov unsigned long orig_ret_vaddr, trampoline_vaddr; 13980dfd0eb8SAnton Arapov bool chained = false; 13990dfd0eb8SAnton Arapov 14000dfd0eb8SAnton Arapov if (!get_xol_area()) 14010dfd0eb8SAnton Arapov return; 14020dfd0eb8SAnton Arapov 14030dfd0eb8SAnton Arapov utask = get_utask(); 14040dfd0eb8SAnton Arapov if (!utask) 14050dfd0eb8SAnton Arapov return; 14060dfd0eb8SAnton Arapov 14070dfd0eb8SAnton Arapov ri = kzalloc(sizeof(struct return_instance), GFP_KERNEL); 14080dfd0eb8SAnton Arapov if (!ri) 14090dfd0eb8SAnton Arapov goto fail; 14100dfd0eb8SAnton Arapov 14110dfd0eb8SAnton Arapov trampoline_vaddr = get_trampoline_vaddr(); 14120dfd0eb8SAnton Arapov orig_ret_vaddr = arch_uretprobe_hijack_return_addr(trampoline_vaddr, regs); 14130dfd0eb8SAnton Arapov if (orig_ret_vaddr == -1) 14140dfd0eb8SAnton Arapov goto fail; 14150dfd0eb8SAnton Arapov 14160dfd0eb8SAnton Arapov /* 14170dfd0eb8SAnton Arapov * We don't want to keep trampoline address in stack, rather keep the 14180dfd0eb8SAnton Arapov * original return address of first caller thru all the consequent 14190dfd0eb8SAnton Arapov * instances. This also makes breakpoint unwrapping easier. 14200dfd0eb8SAnton Arapov */ 14210dfd0eb8SAnton Arapov if (orig_ret_vaddr == trampoline_vaddr) { 14220dfd0eb8SAnton Arapov if (!utask->return_instances) { 14230dfd0eb8SAnton Arapov /* 14240dfd0eb8SAnton Arapov * This situation is not possible. Likely we have an 14250dfd0eb8SAnton Arapov * attack from user-space. 14260dfd0eb8SAnton Arapov */ 14270dfd0eb8SAnton Arapov pr_warn("uprobe: unable to set uretprobe pid/tgid=%d/%d\n", 14280dfd0eb8SAnton Arapov current->pid, current->tgid); 14290dfd0eb8SAnton Arapov goto fail; 14300dfd0eb8SAnton Arapov } 14310dfd0eb8SAnton Arapov 14320dfd0eb8SAnton Arapov chained = true; 14330dfd0eb8SAnton Arapov orig_ret_vaddr = utask->return_instances->orig_ret_vaddr; 14340dfd0eb8SAnton Arapov } 14350dfd0eb8SAnton Arapov 14360dfd0eb8SAnton Arapov atomic_inc(&uprobe->ref); 14370dfd0eb8SAnton Arapov ri->uprobe = uprobe; 14380dfd0eb8SAnton Arapov ri->func = instruction_pointer(regs); 14390dfd0eb8SAnton Arapov ri->orig_ret_vaddr = orig_ret_vaddr; 14400dfd0eb8SAnton Arapov ri->chained = chained; 14410dfd0eb8SAnton Arapov 14420dfd0eb8SAnton Arapov /* add instance to the stack */ 14430dfd0eb8SAnton Arapov ri->next = utask->return_instances; 14440dfd0eb8SAnton Arapov utask->return_instances = ri; 14450dfd0eb8SAnton Arapov 14460dfd0eb8SAnton Arapov return; 14470dfd0eb8SAnton Arapov 14480dfd0eb8SAnton Arapov fail: 14490dfd0eb8SAnton Arapov kfree(ri); 14500dfd0eb8SAnton Arapov } 14510dfd0eb8SAnton Arapov 14520326f5a9SSrikar Dronamraju /* Prepare to single-step probed instruction out of line. */ 14530326f5a9SSrikar Dronamraju static int 1454a6cb3f6dSOleg Nesterov pre_ssout(struct uprobe *uprobe, struct pt_regs *regs, unsigned long bp_vaddr) 14550326f5a9SSrikar Dronamraju { 1456a6cb3f6dSOleg Nesterov struct uprobe_task *utask; 1457a6cb3f6dSOleg Nesterov unsigned long xol_vaddr; 1458aba51024SOleg Nesterov int err; 1459d4b3b638SSrikar Dronamraju 1460608e7427SOleg Nesterov utask = get_utask(); 1461608e7427SOleg Nesterov if (!utask) 1462608e7427SOleg Nesterov return -ENOMEM; 1463a6cb3f6dSOleg Nesterov 1464a6cb3f6dSOleg Nesterov xol_vaddr = xol_get_insn_slot(uprobe); 1465a6cb3f6dSOleg Nesterov if (!xol_vaddr) 1466a6cb3f6dSOleg Nesterov return -ENOMEM; 1467a6cb3f6dSOleg Nesterov 1468a6cb3f6dSOleg Nesterov utask->xol_vaddr = xol_vaddr; 1469a6cb3f6dSOleg Nesterov utask->vaddr = bp_vaddr; 1470a6cb3f6dSOleg Nesterov 1471aba51024SOleg Nesterov err = arch_uprobe_pre_xol(&uprobe->arch, regs); 1472aba51024SOleg Nesterov if (unlikely(err)) { 1473aba51024SOleg Nesterov xol_free_insn_slot(current); 1474aba51024SOleg Nesterov return err; 1475aba51024SOleg Nesterov } 1476aba51024SOleg Nesterov 1477608e7427SOleg Nesterov utask->active_uprobe = uprobe; 1478608e7427SOleg Nesterov utask->state = UTASK_SSTEP; 1479aba51024SOleg Nesterov return 0; 14800326f5a9SSrikar Dronamraju } 14810326f5a9SSrikar Dronamraju 14820326f5a9SSrikar Dronamraju /* 14830326f5a9SSrikar Dronamraju * If we are singlestepping, then ensure this thread is not connected to 14840326f5a9SSrikar Dronamraju * non-fatal signals until completion of singlestep. When xol insn itself 14850326f5a9SSrikar Dronamraju * triggers the signal, restart the original insn even if the task is 14860326f5a9SSrikar Dronamraju * already SIGKILL'ed (since coredump should report the correct ip). This 14870326f5a9SSrikar Dronamraju * is even more important if the task has a handler for SIGSEGV/etc, The 14880326f5a9SSrikar Dronamraju * _same_ instruction should be repeated again after return from the signal 14890326f5a9SSrikar Dronamraju * handler, and SSTEP can never finish in this case. 14900326f5a9SSrikar Dronamraju */ 14910326f5a9SSrikar Dronamraju bool uprobe_deny_signal(void) 14920326f5a9SSrikar Dronamraju { 14930326f5a9SSrikar Dronamraju struct task_struct *t = current; 14940326f5a9SSrikar Dronamraju struct uprobe_task *utask = t->utask; 14950326f5a9SSrikar Dronamraju 14960326f5a9SSrikar Dronamraju if (likely(!utask || !utask->active_uprobe)) 14970326f5a9SSrikar Dronamraju return false; 14980326f5a9SSrikar Dronamraju 14990326f5a9SSrikar Dronamraju WARN_ON_ONCE(utask->state != UTASK_SSTEP); 15000326f5a9SSrikar Dronamraju 15010326f5a9SSrikar Dronamraju if (signal_pending(t)) { 15020326f5a9SSrikar Dronamraju spin_lock_irq(&t->sighand->siglock); 15030326f5a9SSrikar Dronamraju clear_tsk_thread_flag(t, TIF_SIGPENDING); 15040326f5a9SSrikar Dronamraju spin_unlock_irq(&t->sighand->siglock); 15050326f5a9SSrikar Dronamraju 15060326f5a9SSrikar Dronamraju if (__fatal_signal_pending(t) || arch_uprobe_xol_was_trapped(t)) { 15070326f5a9SSrikar Dronamraju utask->state = UTASK_SSTEP_TRAPPED; 15080326f5a9SSrikar Dronamraju set_tsk_thread_flag(t, TIF_UPROBE); 15090326f5a9SSrikar Dronamraju set_tsk_thread_flag(t, TIF_NOTIFY_RESUME); 15100326f5a9SSrikar Dronamraju } 15110326f5a9SSrikar Dronamraju } 15120326f5a9SSrikar Dronamraju 15130326f5a9SSrikar Dronamraju return true; 15140326f5a9SSrikar Dronamraju } 15150326f5a9SSrikar Dronamraju 15160326f5a9SSrikar Dronamraju /* 15170326f5a9SSrikar Dronamraju * Avoid singlestepping the original instruction if the original instruction 15180326f5a9SSrikar Dronamraju * is a NOP or can be emulated. 15190326f5a9SSrikar Dronamraju */ 15200326f5a9SSrikar Dronamraju static bool can_skip_sstep(struct uprobe *uprobe, struct pt_regs *regs) 15210326f5a9SSrikar Dronamraju { 152271434f2fSOleg Nesterov if (test_bit(UPROBE_SKIP_SSTEP, &uprobe->flags)) { 15230326f5a9SSrikar Dronamraju if (arch_uprobe_skip_sstep(&uprobe->arch, regs)) 15240326f5a9SSrikar Dronamraju return true; 152571434f2fSOleg Nesterov clear_bit(UPROBE_SKIP_SSTEP, &uprobe->flags); 15260578a970SOleg Nesterov } 15270326f5a9SSrikar Dronamraju return false; 15280326f5a9SSrikar Dronamraju } 15290326f5a9SSrikar Dronamraju 1530499a4f3eSOleg Nesterov static void mmf_recalc_uprobes(struct mm_struct *mm) 1531499a4f3eSOleg Nesterov { 1532499a4f3eSOleg Nesterov struct vm_area_struct *vma; 1533499a4f3eSOleg Nesterov 1534499a4f3eSOleg Nesterov for (vma = mm->mmap; vma; vma = vma->vm_next) { 1535499a4f3eSOleg Nesterov if (!valid_vma(vma, false)) 1536499a4f3eSOleg Nesterov continue; 1537499a4f3eSOleg Nesterov /* 1538499a4f3eSOleg Nesterov * This is not strictly accurate, we can race with 1539499a4f3eSOleg Nesterov * uprobe_unregister() and see the already removed 1540499a4f3eSOleg Nesterov * uprobe if delete_uprobe() was not yet called. 154163633cbfSOleg Nesterov * Or this uprobe can be filtered out. 1542499a4f3eSOleg Nesterov */ 1543499a4f3eSOleg Nesterov if (vma_has_uprobes(vma, vma->vm_start, vma->vm_end)) 1544499a4f3eSOleg Nesterov return; 1545499a4f3eSOleg Nesterov } 1546499a4f3eSOleg Nesterov 1547499a4f3eSOleg Nesterov clear_bit(MMF_HAS_UPROBES, &mm->flags); 1548499a4f3eSOleg Nesterov } 1549499a4f3eSOleg Nesterov 15500908ad6eSAnanth N Mavinakayanahalli static int is_trap_at_addr(struct mm_struct *mm, unsigned long vaddr) 1551ec75fba9SOleg Nesterov { 1552ec75fba9SOleg Nesterov struct page *page; 1553ec75fba9SOleg Nesterov uprobe_opcode_t opcode; 1554ec75fba9SOleg Nesterov int result; 1555ec75fba9SOleg Nesterov 1556ec75fba9SOleg Nesterov pagefault_disable(); 1557ec75fba9SOleg Nesterov result = __copy_from_user_inatomic(&opcode, (void __user*)vaddr, 1558ec75fba9SOleg Nesterov sizeof(opcode)); 1559ec75fba9SOleg Nesterov pagefault_enable(); 1560ec75fba9SOleg Nesterov 1561ec75fba9SOleg Nesterov if (likely(result == 0)) 1562ec75fba9SOleg Nesterov goto out; 1563ec75fba9SOleg Nesterov 1564ec75fba9SOleg Nesterov result = get_user_pages(NULL, mm, vaddr, 1, 0, 1, &page, NULL); 1565ec75fba9SOleg Nesterov if (result < 0) 1566ec75fba9SOleg Nesterov return result; 1567ec75fba9SOleg Nesterov 1568ab0d805cSOleg Nesterov copy_from_page(page, vaddr, &opcode, UPROBE_SWBP_INSN_SIZE); 1569ec75fba9SOleg Nesterov put_page(page); 1570ec75fba9SOleg Nesterov out: 15710908ad6eSAnanth N Mavinakayanahalli /* This needs to return true for any variant of the trap insn */ 15720908ad6eSAnanth N Mavinakayanahalli return is_trap_insn(&opcode); 1573ec75fba9SOleg Nesterov } 1574ec75fba9SOleg Nesterov 1575d790d346SOleg Nesterov static struct uprobe *find_active_uprobe(unsigned long bp_vaddr, int *is_swbp) 15760326f5a9SSrikar Dronamraju { 15773a9ea052SOleg Nesterov struct mm_struct *mm = current->mm; 15783a9ea052SOleg Nesterov struct uprobe *uprobe = NULL; 15790326f5a9SSrikar Dronamraju struct vm_area_struct *vma; 15800326f5a9SSrikar Dronamraju 15810326f5a9SSrikar Dronamraju down_read(&mm->mmap_sem); 15820326f5a9SSrikar Dronamraju vma = find_vma(mm, bp_vaddr); 15833a9ea052SOleg Nesterov if (vma && vma->vm_start <= bp_vaddr) { 15843a9ea052SOleg Nesterov if (valid_vma(vma, false)) { 1585f281769eSOleg Nesterov struct inode *inode = file_inode(vma->vm_file); 1586cb113b47SOleg Nesterov loff_t offset = vaddr_to_offset(vma, bp_vaddr); 15870326f5a9SSrikar Dronamraju 15880326f5a9SSrikar Dronamraju uprobe = find_uprobe(inode, offset); 15890326f5a9SSrikar Dronamraju } 1590d790d346SOleg Nesterov 1591d790d346SOleg Nesterov if (!uprobe) 15920908ad6eSAnanth N Mavinakayanahalli *is_swbp = is_trap_at_addr(mm, bp_vaddr); 1593d790d346SOleg Nesterov } else { 1594d790d346SOleg Nesterov *is_swbp = -EFAULT; 15953a9ea052SOleg Nesterov } 1596499a4f3eSOleg Nesterov 1597499a4f3eSOleg Nesterov if (!uprobe && test_and_clear_bit(MMF_RECALC_UPROBES, &mm->flags)) 1598499a4f3eSOleg Nesterov mmf_recalc_uprobes(mm); 15990326f5a9SSrikar Dronamraju up_read(&mm->mmap_sem); 16000326f5a9SSrikar Dronamraju 16013a9ea052SOleg Nesterov return uprobe; 16023a9ea052SOleg Nesterov } 16033a9ea052SOleg Nesterov 1604da1816b1SOleg Nesterov static void handler_chain(struct uprobe *uprobe, struct pt_regs *regs) 1605da1816b1SOleg Nesterov { 1606da1816b1SOleg Nesterov struct uprobe_consumer *uc; 1607da1816b1SOleg Nesterov int remove = UPROBE_HANDLER_REMOVE; 16080dfd0eb8SAnton Arapov bool need_prep = false; /* prepare return uprobe, when needed */ 1609da1816b1SOleg Nesterov 1610da1816b1SOleg Nesterov down_read(&uprobe->register_rwsem); 1611da1816b1SOleg Nesterov for (uc = uprobe->consumers; uc; uc = uc->next) { 1612ea024870SAnton Arapov int rc = 0; 1613da1816b1SOleg Nesterov 1614ea024870SAnton Arapov if (uc->handler) { 1615ea024870SAnton Arapov rc = uc->handler(uc, regs); 1616da1816b1SOleg Nesterov WARN(rc & ~UPROBE_HANDLER_MASK, 1617da1816b1SOleg Nesterov "bad rc=0x%x from %pf()\n", rc, uc->handler); 1618ea024870SAnton Arapov } 16190dfd0eb8SAnton Arapov 16200dfd0eb8SAnton Arapov if (uc->ret_handler) 16210dfd0eb8SAnton Arapov need_prep = true; 16220dfd0eb8SAnton Arapov 1623da1816b1SOleg Nesterov remove &= rc; 1624da1816b1SOleg Nesterov } 1625da1816b1SOleg Nesterov 16260dfd0eb8SAnton Arapov if (need_prep && !remove) 16270dfd0eb8SAnton Arapov prepare_uretprobe(uprobe, regs); /* put bp at return */ 16280dfd0eb8SAnton Arapov 1629da1816b1SOleg Nesterov if (remove && uprobe->consumers) { 1630da1816b1SOleg Nesterov WARN_ON(!uprobe_is_active(uprobe)); 1631da1816b1SOleg Nesterov unapply_uprobe(uprobe, current->mm); 1632da1816b1SOleg Nesterov } 1633da1816b1SOleg Nesterov up_read(&uprobe->register_rwsem); 1634da1816b1SOleg Nesterov } 1635da1816b1SOleg Nesterov 16363a9ea052SOleg Nesterov /* 16373a9ea052SOleg Nesterov * Run handler and ask thread to singlestep. 16383a9ea052SOleg Nesterov * Ensure all non-fatal signals cannot interrupt thread while it singlesteps. 16393a9ea052SOleg Nesterov */ 16403a9ea052SOleg Nesterov static void handle_swbp(struct pt_regs *regs) 16413a9ea052SOleg Nesterov { 16423a9ea052SOleg Nesterov struct uprobe *uprobe; 16433a9ea052SOleg Nesterov unsigned long bp_vaddr; 164456bb4cf6SOleg Nesterov int uninitialized_var(is_swbp); 16453a9ea052SOleg Nesterov 16463a9ea052SOleg Nesterov bp_vaddr = uprobe_get_swbp_addr(regs); 1647d790d346SOleg Nesterov uprobe = find_active_uprobe(bp_vaddr, &is_swbp); 16483a9ea052SOleg Nesterov 16490326f5a9SSrikar Dronamraju if (!uprobe) { 165056bb4cf6SOleg Nesterov if (is_swbp > 0) { 16510326f5a9SSrikar Dronamraju /* No matching uprobe; signal SIGTRAP. */ 16520326f5a9SSrikar Dronamraju send_sig(SIGTRAP, current, 0); 165356bb4cf6SOleg Nesterov } else { 165456bb4cf6SOleg Nesterov /* 165556bb4cf6SOleg Nesterov * Either we raced with uprobe_unregister() or we can't 165656bb4cf6SOleg Nesterov * access this memory. The latter is only possible if 165756bb4cf6SOleg Nesterov * another thread plays with our ->mm. In both cases 165856bb4cf6SOleg Nesterov * we can simply restart. If this vma was unmapped we 165956bb4cf6SOleg Nesterov * can pretend this insn was not executed yet and get 166056bb4cf6SOleg Nesterov * the (correct) SIGSEGV after restart. 166156bb4cf6SOleg Nesterov */ 166256bb4cf6SOleg Nesterov instruction_pointer_set(regs, bp_vaddr); 166356bb4cf6SOleg Nesterov } 16640326f5a9SSrikar Dronamraju return; 16650326f5a9SSrikar Dronamraju } 166674e59dfcSOleg Nesterov 166774e59dfcSOleg Nesterov /* change it in advance for ->handler() and restart */ 166874e59dfcSOleg Nesterov instruction_pointer_set(regs, bp_vaddr); 166974e59dfcSOleg Nesterov 1670142b18ddSOleg Nesterov /* 1671142b18ddSOleg Nesterov * TODO: move copy_insn/etc into _register and remove this hack. 1672142b18ddSOleg Nesterov * After we hit the bp, _unregister + _register can install the 1673142b18ddSOleg Nesterov * new and not-yet-analyzed uprobe at the same address, restart. 1674142b18ddSOleg Nesterov */ 1675142b18ddSOleg Nesterov smp_rmb(); /* pairs with wmb() in install_breakpoint() */ 167671434f2fSOleg Nesterov if (unlikely(!test_bit(UPROBE_COPY_INSN, &uprobe->flags))) 167774e59dfcSOleg Nesterov goto out; 16780326f5a9SSrikar Dronamraju 16790326f5a9SSrikar Dronamraju handler_chain(uprobe, regs); 16800578a970SOleg Nesterov if (can_skip_sstep(uprobe, regs)) 16810578a970SOleg Nesterov goto out; 16820326f5a9SSrikar Dronamraju 1683608e7427SOleg Nesterov if (!pre_ssout(uprobe, regs, bp_vaddr)) 16840326f5a9SSrikar Dronamraju return; 16850326f5a9SSrikar Dronamraju 168674e59dfcSOleg Nesterov /* can_skip_sstep() succeeded, or restart if can't singlestep */ 16870578a970SOleg Nesterov out: 16880326f5a9SSrikar Dronamraju put_uprobe(uprobe); 16890326f5a9SSrikar Dronamraju } 16900326f5a9SSrikar Dronamraju 16910326f5a9SSrikar Dronamraju /* 16920326f5a9SSrikar Dronamraju * Perform required fix-ups and disable singlestep. 16930326f5a9SSrikar Dronamraju * Allow pending signals to take effect. 16940326f5a9SSrikar Dronamraju */ 16950326f5a9SSrikar Dronamraju static void handle_singlestep(struct uprobe_task *utask, struct pt_regs *regs) 16960326f5a9SSrikar Dronamraju { 16970326f5a9SSrikar Dronamraju struct uprobe *uprobe; 16980326f5a9SSrikar Dronamraju 16990326f5a9SSrikar Dronamraju uprobe = utask->active_uprobe; 17000326f5a9SSrikar Dronamraju if (utask->state == UTASK_SSTEP_ACK) 17010326f5a9SSrikar Dronamraju arch_uprobe_post_xol(&uprobe->arch, regs); 17020326f5a9SSrikar Dronamraju else if (utask->state == UTASK_SSTEP_TRAPPED) 17030326f5a9SSrikar Dronamraju arch_uprobe_abort_xol(&uprobe->arch, regs); 17040326f5a9SSrikar Dronamraju else 17050326f5a9SSrikar Dronamraju WARN_ON_ONCE(1); 17060326f5a9SSrikar Dronamraju 17070326f5a9SSrikar Dronamraju put_uprobe(uprobe); 17080326f5a9SSrikar Dronamraju utask->active_uprobe = NULL; 17090326f5a9SSrikar Dronamraju utask->state = UTASK_RUNNING; 1710d4b3b638SSrikar Dronamraju xol_free_insn_slot(current); 17110326f5a9SSrikar Dronamraju 17120326f5a9SSrikar Dronamraju spin_lock_irq(¤t->sighand->siglock); 17130326f5a9SSrikar Dronamraju recalc_sigpending(); /* see uprobe_deny_signal() */ 17140326f5a9SSrikar Dronamraju spin_unlock_irq(¤t->sighand->siglock); 17150326f5a9SSrikar Dronamraju } 17160326f5a9SSrikar Dronamraju 17170326f5a9SSrikar Dronamraju /* 17181b08e907SOleg Nesterov * On breakpoint hit, breakpoint notifier sets the TIF_UPROBE flag and 17191b08e907SOleg Nesterov * allows the thread to return from interrupt. After that handle_swbp() 17201b08e907SOleg Nesterov * sets utask->active_uprobe. 17210326f5a9SSrikar Dronamraju * 17221b08e907SOleg Nesterov * On singlestep exception, singlestep notifier sets the TIF_UPROBE flag 17231b08e907SOleg Nesterov * and allows the thread to return from interrupt. 17240326f5a9SSrikar Dronamraju * 17250326f5a9SSrikar Dronamraju * While returning to userspace, thread notices the TIF_UPROBE flag and calls 17260326f5a9SSrikar Dronamraju * uprobe_notify_resume(). 17270326f5a9SSrikar Dronamraju */ 17280326f5a9SSrikar Dronamraju void uprobe_notify_resume(struct pt_regs *regs) 17290326f5a9SSrikar Dronamraju { 17300326f5a9SSrikar Dronamraju struct uprobe_task *utask; 17310326f5a9SSrikar Dronamraju 1732db023ea5SOleg Nesterov clear_thread_flag(TIF_UPROBE); 1733db023ea5SOleg Nesterov 17340326f5a9SSrikar Dronamraju utask = current->utask; 17351b08e907SOleg Nesterov if (utask && utask->active_uprobe) 17360326f5a9SSrikar Dronamraju handle_singlestep(utask, regs); 17371b08e907SOleg Nesterov else 17381b08e907SOleg Nesterov handle_swbp(regs); 17390326f5a9SSrikar Dronamraju } 17400326f5a9SSrikar Dronamraju 17410326f5a9SSrikar Dronamraju /* 17420326f5a9SSrikar Dronamraju * uprobe_pre_sstep_notifier gets called from interrupt context as part of 17430326f5a9SSrikar Dronamraju * notifier mechanism. Set TIF_UPROBE flag and indicate breakpoint hit. 17440326f5a9SSrikar Dronamraju */ 17450326f5a9SSrikar Dronamraju int uprobe_pre_sstep_notifier(struct pt_regs *regs) 17460326f5a9SSrikar Dronamraju { 17470dfd0eb8SAnton Arapov if (!current->mm) 17480dfd0eb8SAnton Arapov return 0; 17490dfd0eb8SAnton Arapov 17500dfd0eb8SAnton Arapov if (!test_bit(MMF_HAS_UPROBES, ¤t->mm->flags) && 17510dfd0eb8SAnton Arapov (!current->utask || !current->utask->return_instances)) 17520326f5a9SSrikar Dronamraju return 0; 17530326f5a9SSrikar Dronamraju 17540326f5a9SSrikar Dronamraju set_thread_flag(TIF_UPROBE); 17550326f5a9SSrikar Dronamraju return 1; 17560326f5a9SSrikar Dronamraju } 17570326f5a9SSrikar Dronamraju 17580326f5a9SSrikar Dronamraju /* 17590326f5a9SSrikar Dronamraju * uprobe_post_sstep_notifier gets called in interrupt context as part of notifier 17600326f5a9SSrikar Dronamraju * mechanism. Set TIF_UPROBE flag and indicate completion of singlestep. 17610326f5a9SSrikar Dronamraju */ 17620326f5a9SSrikar Dronamraju int uprobe_post_sstep_notifier(struct pt_regs *regs) 17630326f5a9SSrikar Dronamraju { 17640326f5a9SSrikar Dronamraju struct uprobe_task *utask = current->utask; 17650326f5a9SSrikar Dronamraju 17660326f5a9SSrikar Dronamraju if (!current->mm || !utask || !utask->active_uprobe) 17670326f5a9SSrikar Dronamraju /* task is currently not uprobed */ 17680326f5a9SSrikar Dronamraju return 0; 17690326f5a9SSrikar Dronamraju 17700326f5a9SSrikar Dronamraju utask->state = UTASK_SSTEP_ACK; 17710326f5a9SSrikar Dronamraju set_thread_flag(TIF_UPROBE); 17720326f5a9SSrikar Dronamraju return 1; 17730326f5a9SSrikar Dronamraju } 17740326f5a9SSrikar Dronamraju 17750326f5a9SSrikar Dronamraju static struct notifier_block uprobe_exception_nb = { 17760326f5a9SSrikar Dronamraju .notifier_call = arch_uprobe_exception_notify, 17770326f5a9SSrikar Dronamraju .priority = INT_MAX-1, /* notified after kprobes, kgdb */ 17780326f5a9SSrikar Dronamraju }; 17790326f5a9SSrikar Dronamraju 1780a5f4374aSIngo Molnar static int __init init_uprobes(void) 1781a5f4374aSIngo Molnar { 1782a5f4374aSIngo Molnar int i; 1783a5f4374aSIngo Molnar 178466d06dffSOleg Nesterov for (i = 0; i < UPROBES_HASH_SZ; i++) 1785a5f4374aSIngo Molnar mutex_init(&uprobes_mmap_mutex[i]); 17860326f5a9SSrikar Dronamraju 178732cdba1eSOleg Nesterov if (percpu_init_rwsem(&dup_mmap_sem)) 178832cdba1eSOleg Nesterov return -ENOMEM; 178932cdba1eSOleg Nesterov 17900326f5a9SSrikar Dronamraju return register_die_notifier(&uprobe_exception_nb); 1791a5f4374aSIngo Molnar } 17920326f5a9SSrikar Dronamraju module_init(init_uprobes); 1793a5f4374aSIngo Molnar 1794a5f4374aSIngo Molnar static void __exit exit_uprobes(void) 1795a5f4374aSIngo Molnar { 1796a5f4374aSIngo Molnar } 1797a5f4374aSIngo Molnar module_exit(exit_uprobes); 1798