1a5f4374aSIngo Molnar /* 2a5f4374aSIngo Molnar * User-space Probes (UProbes) 3a5f4374aSIngo Molnar * 4a5f4374aSIngo Molnar * This program is free software; you can redistribute it and/or modify 5a5f4374aSIngo Molnar * it under the terms of the GNU General Public License as published by 6a5f4374aSIngo Molnar * the Free Software Foundation; either version 2 of the License, or 7a5f4374aSIngo Molnar * (at your option) any later version. 8a5f4374aSIngo Molnar * 9a5f4374aSIngo Molnar * This program is distributed in the hope that it will be useful, 10a5f4374aSIngo Molnar * but WITHOUT ANY WARRANTY; without even the implied warranty of 11a5f4374aSIngo Molnar * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12a5f4374aSIngo Molnar * GNU General Public License for more details. 13a5f4374aSIngo Molnar * 14a5f4374aSIngo Molnar * You should have received a copy of the GNU General Public License 15a5f4374aSIngo Molnar * along with this program; if not, write to the Free Software 16a5f4374aSIngo Molnar * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 17a5f4374aSIngo Molnar * 1835aa621bSIngo Molnar * Copyright (C) IBM Corporation, 2008-2012 19a5f4374aSIngo Molnar * Authors: 20a5f4374aSIngo Molnar * Srikar Dronamraju 21a5f4374aSIngo Molnar * Jim Keniston 2235aa621bSIngo Molnar * Copyright (C) 2011-2012 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com> 23a5f4374aSIngo Molnar */ 24a5f4374aSIngo Molnar 25a5f4374aSIngo Molnar #include <linux/kernel.h> 26a5f4374aSIngo Molnar #include <linux/highmem.h> 27a5f4374aSIngo Molnar #include <linux/pagemap.h> /* read_mapping_page */ 28a5f4374aSIngo Molnar #include <linux/slab.h> 29a5f4374aSIngo Molnar #include <linux/sched.h> 30e8440c14SJosh Stone #include <linux/export.h> 31a5f4374aSIngo Molnar #include <linux/rmap.h> /* anon_vma_prepare */ 32a5f4374aSIngo Molnar #include <linux/mmu_notifier.h> /* set_pte_at_notify */ 33a5f4374aSIngo Molnar #include <linux/swap.h> /* try_to_free_swap */ 340326f5a9SSrikar Dronamraju #include <linux/ptrace.h> /* user_enable_single_step */ 350326f5a9SSrikar Dronamraju #include <linux/kdebug.h> /* notifier mechanism */ 36194f8dcbSOleg Nesterov #include "../../mm/internal.h" /* munlock_vma_page */ 3732cdba1eSOleg Nesterov #include <linux/percpu-rwsem.h> 38aa59c53fSOleg Nesterov #include <linux/task_work.h> 39a5f4374aSIngo Molnar 40a5f4374aSIngo Molnar #include <linux/uprobes.h> 41a5f4374aSIngo Molnar 42d4b3b638SSrikar Dronamraju #define UINSNS_PER_PAGE (PAGE_SIZE/UPROBE_XOL_SLOT_BYTES) 43d4b3b638SSrikar Dronamraju #define MAX_UPROBE_XOL_SLOTS UINSNS_PER_PAGE 44d4b3b638SSrikar Dronamraju 45a5f4374aSIngo Molnar static struct rb_root uprobes_tree = RB_ROOT; 46441f1eb7SOleg Nesterov /* 47441f1eb7SOleg Nesterov * allows us to skip the uprobe_mmap if there are no uprobe events active 48441f1eb7SOleg Nesterov * at this time. Probably a fine grained per inode count is better? 49441f1eb7SOleg Nesterov */ 50441f1eb7SOleg Nesterov #define no_uprobe_events() RB_EMPTY_ROOT(&uprobes_tree) 51a5f4374aSIngo Molnar 52a5f4374aSIngo Molnar static DEFINE_SPINLOCK(uprobes_treelock); /* serialize rbtree access */ 53a5f4374aSIngo Molnar 54a5f4374aSIngo Molnar #define UPROBES_HASH_SZ 13 55a5f4374aSIngo Molnar /* serialize uprobe->pending_list */ 56a5f4374aSIngo Molnar static struct mutex uprobes_mmap_mutex[UPROBES_HASH_SZ]; 57a5f4374aSIngo Molnar #define uprobes_mmap_hash(v) (&uprobes_mmap_mutex[((unsigned long)(v)) % UPROBES_HASH_SZ]) 58a5f4374aSIngo Molnar 5932cdba1eSOleg Nesterov static struct percpu_rw_semaphore dup_mmap_sem; 6032cdba1eSOleg Nesterov 61cb9a19feSOleg Nesterov /* Have a copy of original instruction */ 6271434f2fSOleg Nesterov #define UPROBE_COPY_INSN 0 63cb9a19feSOleg Nesterov 643ff54efdSSrikar Dronamraju struct uprobe { 653ff54efdSSrikar Dronamraju struct rb_node rb_node; /* node in the rb tree */ 663ff54efdSSrikar Dronamraju atomic_t ref; 67e591c8d7SOleg Nesterov struct rw_semaphore register_rwsem; 683ff54efdSSrikar Dronamraju struct rw_semaphore consumer_rwsem; 693ff54efdSSrikar Dronamraju struct list_head pending_list; 703ff54efdSSrikar Dronamraju struct uprobe_consumer *consumers; 713ff54efdSSrikar Dronamraju struct inode *inode; /* Also hold a ref to inode */ 723ff54efdSSrikar Dronamraju loff_t offset; 7371434f2fSOleg Nesterov unsigned long flags; 74ad439356SOleg Nesterov 75ad439356SOleg Nesterov /* 76ad439356SOleg Nesterov * The generic code assumes that it has two members of unknown type 77ad439356SOleg Nesterov * owned by the arch-specific code: 78ad439356SOleg Nesterov * 79ad439356SOleg Nesterov * insn - copy_insn() saves the original instruction here for 80ad439356SOleg Nesterov * arch_uprobe_analyze_insn(). 81ad439356SOleg Nesterov * 82ad439356SOleg Nesterov * ixol - potentially modified instruction to execute out of 83ad439356SOleg Nesterov * line, copied to xol_area by xol_get_insn_slot(). 84ad439356SOleg Nesterov */ 853ff54efdSSrikar Dronamraju struct arch_uprobe arch; 863ff54efdSSrikar Dronamraju }; 873ff54efdSSrikar Dronamraju 880dfd0eb8SAnton Arapov struct return_instance { 890dfd0eb8SAnton Arapov struct uprobe *uprobe; 900dfd0eb8SAnton Arapov unsigned long func; 910dfd0eb8SAnton Arapov unsigned long orig_ret_vaddr; /* original return address */ 920dfd0eb8SAnton Arapov bool chained; /* true, if instance is nested */ 930dfd0eb8SAnton Arapov 940dfd0eb8SAnton Arapov struct return_instance *next; /* keep as stack */ 950dfd0eb8SAnton Arapov }; 960dfd0eb8SAnton Arapov 97a5f4374aSIngo Molnar /* 98ad439356SOleg Nesterov * Execute out of line area: anonymous executable mapping installed 99ad439356SOleg Nesterov * by the probed task to execute the copy of the original instruction 100ad439356SOleg Nesterov * mangled by set_swbp(). 101ad439356SOleg Nesterov * 102c912dae6SOleg Nesterov * On a breakpoint hit, thread contests for a slot. It frees the 103c912dae6SOleg Nesterov * slot after singlestep. Currently a fixed number of slots are 104c912dae6SOleg Nesterov * allocated. 105c912dae6SOleg Nesterov */ 106c912dae6SOleg Nesterov struct xol_area { 107c912dae6SOleg Nesterov wait_queue_head_t wq; /* if all slots are busy */ 108c912dae6SOleg Nesterov atomic_t slot_count; /* number of in-use slots */ 109c912dae6SOleg Nesterov unsigned long *bitmap; /* 0 = free slot */ 110c912dae6SOleg Nesterov struct page *page; 111c912dae6SOleg Nesterov 112c912dae6SOleg Nesterov /* 113c912dae6SOleg Nesterov * We keep the vma's vm_start rather than a pointer to the vma 114c912dae6SOleg Nesterov * itself. The probed process or a naughty kernel module could make 115c912dae6SOleg Nesterov * the vma go away, and we must handle that reasonably gracefully. 116c912dae6SOleg Nesterov */ 117c912dae6SOleg Nesterov unsigned long vaddr; /* Page(s) of instruction slots */ 118c912dae6SOleg Nesterov }; 119c912dae6SOleg Nesterov 120c912dae6SOleg Nesterov /* 121a5f4374aSIngo Molnar * valid_vma: Verify if the specified vma is an executable vma 122a5f4374aSIngo Molnar * Relax restrictions while unregistering: vm_flags might have 123a5f4374aSIngo Molnar * changed after breakpoint was inserted. 124a5f4374aSIngo Molnar * - is_register: indicates if we are in register context. 125a5f4374aSIngo Molnar * - Return 1 if the specified virtual address is in an 126a5f4374aSIngo Molnar * executable vma. 127a5f4374aSIngo Molnar */ 128a5f4374aSIngo Molnar static bool valid_vma(struct vm_area_struct *vma, bool is_register) 129a5f4374aSIngo Molnar { 13013f59c5eSOleg Nesterov vm_flags_t flags = VM_HUGETLB | VM_MAYEXEC | VM_MAYSHARE; 131a5f4374aSIngo Molnar 132e40cfce6SOleg Nesterov if (is_register) 133e40cfce6SOleg Nesterov flags |= VM_WRITE; 134a5f4374aSIngo Molnar 135e40cfce6SOleg Nesterov return vma->vm_file && (vma->vm_flags & flags) == VM_MAYEXEC; 136a5f4374aSIngo Molnar } 137a5f4374aSIngo Molnar 13857683f72SOleg Nesterov static unsigned long offset_to_vaddr(struct vm_area_struct *vma, loff_t offset) 139a5f4374aSIngo Molnar { 14057683f72SOleg Nesterov return vma->vm_start + offset - ((loff_t)vma->vm_pgoff << PAGE_SHIFT); 141a5f4374aSIngo Molnar } 142a5f4374aSIngo Molnar 143cb113b47SOleg Nesterov static loff_t vaddr_to_offset(struct vm_area_struct *vma, unsigned long vaddr) 144cb113b47SOleg Nesterov { 145cb113b47SOleg Nesterov return ((loff_t)vma->vm_pgoff << PAGE_SHIFT) + (vaddr - vma->vm_start); 146cb113b47SOleg Nesterov } 147cb113b47SOleg Nesterov 148a5f4374aSIngo Molnar /** 149a5f4374aSIngo Molnar * __replace_page - replace page in vma by new page. 150a5f4374aSIngo Molnar * based on replace_page in mm/ksm.c 151a5f4374aSIngo Molnar * 152a5f4374aSIngo Molnar * @vma: vma that holds the pte pointing to page 153c517ee74SOleg Nesterov * @addr: address the old @page is mapped at 154a5f4374aSIngo Molnar * @page: the cowed page we are replacing by kpage 155a5f4374aSIngo Molnar * @kpage: the modified page we replace page by 156a5f4374aSIngo Molnar * 157a5f4374aSIngo Molnar * Returns 0 on success, -EFAULT on failure. 158a5f4374aSIngo Molnar */ 159c517ee74SOleg Nesterov static int __replace_page(struct vm_area_struct *vma, unsigned long addr, 160c517ee74SOleg Nesterov struct page *page, struct page *kpage) 161a5f4374aSIngo Molnar { 162a5f4374aSIngo Molnar struct mm_struct *mm = vma->vm_mm; 1635323ce71SOleg Nesterov spinlock_t *ptl; 1645323ce71SOleg Nesterov pte_t *ptep; 1659f92448cSOleg Nesterov int err; 1666bdb913fSHaggai Eran /* For mmu_notifiers */ 1676bdb913fSHaggai Eran const unsigned long mmun_start = addr; 1686bdb913fSHaggai Eran const unsigned long mmun_end = addr + PAGE_SIZE; 169a5f4374aSIngo Molnar 170194f8dcbSOleg Nesterov /* For try_to_free_swap() and munlock_vma_page() below */ 1719f92448cSOleg Nesterov lock_page(page); 1729f92448cSOleg Nesterov 1736bdb913fSHaggai Eran mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end); 1749f92448cSOleg Nesterov err = -EAGAIN; 1755323ce71SOleg Nesterov ptep = page_check_address(page, mm, addr, &ptl, 0); 176a5f4374aSIngo Molnar if (!ptep) 1779f92448cSOleg Nesterov goto unlock; 178a5f4374aSIngo Molnar 179a5f4374aSIngo Molnar get_page(kpage); 180a5f4374aSIngo Molnar page_add_new_anon_rmap(kpage, vma, addr); 181a5f4374aSIngo Molnar 1827396fa81SSrikar Dronamraju if (!PageAnon(page)) { 1837396fa81SSrikar Dronamraju dec_mm_counter(mm, MM_FILEPAGES); 1847396fa81SSrikar Dronamraju inc_mm_counter(mm, MM_ANONPAGES); 1857396fa81SSrikar Dronamraju } 1867396fa81SSrikar Dronamraju 187a5f4374aSIngo Molnar flush_cache_page(vma, addr, pte_pfn(*ptep)); 188a5f4374aSIngo Molnar ptep_clear_flush(vma, addr, ptep); 189a5f4374aSIngo Molnar set_pte_at_notify(mm, addr, ptep, mk_pte(kpage, vma->vm_page_prot)); 190a5f4374aSIngo Molnar 191a5f4374aSIngo Molnar page_remove_rmap(page); 192a5f4374aSIngo Molnar if (!page_mapped(page)) 193a5f4374aSIngo Molnar try_to_free_swap(page); 194a5f4374aSIngo Molnar pte_unmap_unlock(ptep, ptl); 195a5f4374aSIngo Molnar 196194f8dcbSOleg Nesterov if (vma->vm_flags & VM_LOCKED) 197194f8dcbSOleg Nesterov munlock_vma_page(page); 198194f8dcbSOleg Nesterov put_page(page); 199194f8dcbSOleg Nesterov 2009f92448cSOleg Nesterov err = 0; 2019f92448cSOleg Nesterov unlock: 2026bdb913fSHaggai Eran mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end); 2039f92448cSOleg Nesterov unlock_page(page); 2049f92448cSOleg Nesterov return err; 205a5f4374aSIngo Molnar } 206a5f4374aSIngo Molnar 207a5f4374aSIngo Molnar /** 2085cb4ac3aSSrikar Dronamraju * is_swbp_insn - check if instruction is breakpoint instruction. 209a5f4374aSIngo Molnar * @insn: instruction to be checked. 2105cb4ac3aSSrikar Dronamraju * Default implementation of is_swbp_insn 211a5f4374aSIngo Molnar * Returns true if @insn is a breakpoint instruction. 212a5f4374aSIngo Molnar */ 2135cb4ac3aSSrikar Dronamraju bool __weak is_swbp_insn(uprobe_opcode_t *insn) 214a5f4374aSIngo Molnar { 2155cb4ac3aSSrikar Dronamraju return *insn == UPROBE_SWBP_INSN; 216a5f4374aSIngo Molnar } 217a5f4374aSIngo Molnar 2180908ad6eSAnanth N Mavinakayanahalli /** 2190908ad6eSAnanth N Mavinakayanahalli * is_trap_insn - check if instruction is breakpoint instruction. 2200908ad6eSAnanth N Mavinakayanahalli * @insn: instruction to be checked. 2210908ad6eSAnanth N Mavinakayanahalli * Default implementation of is_trap_insn 2220908ad6eSAnanth N Mavinakayanahalli * Returns true if @insn is a breakpoint instruction. 2230908ad6eSAnanth N Mavinakayanahalli * 2240908ad6eSAnanth N Mavinakayanahalli * This function is needed for the case where an architecture has multiple 2250908ad6eSAnanth N Mavinakayanahalli * trap instructions (like powerpc). 2260908ad6eSAnanth N Mavinakayanahalli */ 2270908ad6eSAnanth N Mavinakayanahalli bool __weak is_trap_insn(uprobe_opcode_t *insn) 2280908ad6eSAnanth N Mavinakayanahalli { 2290908ad6eSAnanth N Mavinakayanahalli return is_swbp_insn(insn); 2300908ad6eSAnanth N Mavinakayanahalli } 2310908ad6eSAnanth N Mavinakayanahalli 232ab0d805cSOleg Nesterov static void copy_from_page(struct page *page, unsigned long vaddr, void *dst, int len) 233cceb55aaSOleg Nesterov { 234cceb55aaSOleg Nesterov void *kaddr = kmap_atomic(page); 235ab0d805cSOleg Nesterov memcpy(dst, kaddr + (vaddr & ~PAGE_MASK), len); 236cceb55aaSOleg Nesterov kunmap_atomic(kaddr); 237cceb55aaSOleg Nesterov } 238cceb55aaSOleg Nesterov 2395669cceeSOleg Nesterov static void copy_to_page(struct page *page, unsigned long vaddr, const void *src, int len) 2405669cceeSOleg Nesterov { 2415669cceeSOleg Nesterov void *kaddr = kmap_atomic(page); 2425669cceeSOleg Nesterov memcpy(kaddr + (vaddr & ~PAGE_MASK), src, len); 2435669cceeSOleg Nesterov kunmap_atomic(kaddr); 2445669cceeSOleg Nesterov } 2455669cceeSOleg Nesterov 246ed6f6a50SOleg Nesterov static int verify_opcode(struct page *page, unsigned long vaddr, uprobe_opcode_t *new_opcode) 247ed6f6a50SOleg Nesterov { 248ed6f6a50SOleg Nesterov uprobe_opcode_t old_opcode; 249ed6f6a50SOleg Nesterov bool is_swbp; 250ed6f6a50SOleg Nesterov 2510908ad6eSAnanth N Mavinakayanahalli /* 2520908ad6eSAnanth N Mavinakayanahalli * Note: We only check if the old_opcode is UPROBE_SWBP_INSN here. 2530908ad6eSAnanth N Mavinakayanahalli * We do not check if it is any other 'trap variant' which could 2540908ad6eSAnanth N Mavinakayanahalli * be conditional trap instruction such as the one powerpc supports. 2550908ad6eSAnanth N Mavinakayanahalli * 2560908ad6eSAnanth N Mavinakayanahalli * The logic is that we do not care if the underlying instruction 2570908ad6eSAnanth N Mavinakayanahalli * is a trap variant; uprobes always wins over any other (gdb) 2580908ad6eSAnanth N Mavinakayanahalli * breakpoint. 2590908ad6eSAnanth N Mavinakayanahalli */ 260ab0d805cSOleg Nesterov copy_from_page(page, vaddr, &old_opcode, UPROBE_SWBP_INSN_SIZE); 261ed6f6a50SOleg Nesterov is_swbp = is_swbp_insn(&old_opcode); 262ed6f6a50SOleg Nesterov 263ed6f6a50SOleg Nesterov if (is_swbp_insn(new_opcode)) { 264ed6f6a50SOleg Nesterov if (is_swbp) /* register: already installed? */ 265ed6f6a50SOleg Nesterov return 0; 266ed6f6a50SOleg Nesterov } else { 267ed6f6a50SOleg Nesterov if (!is_swbp) /* unregister: was it changed by us? */ 268076a365bSOleg Nesterov return 0; 269ed6f6a50SOleg Nesterov } 270ed6f6a50SOleg Nesterov 271ed6f6a50SOleg Nesterov return 1; 272ed6f6a50SOleg Nesterov } 273ed6f6a50SOleg Nesterov 274a5f4374aSIngo Molnar /* 275a5f4374aSIngo Molnar * NOTE: 276a5f4374aSIngo Molnar * Expect the breakpoint instruction to be the smallest size instruction for 277a5f4374aSIngo Molnar * the architecture. If an arch has variable length instruction and the 278a5f4374aSIngo Molnar * breakpoint instruction is not of the smallest length instruction 2790908ad6eSAnanth N Mavinakayanahalli * supported by that architecture then we need to modify is_trap_at_addr and 280f72d41faSOleg Nesterov * uprobe_write_opcode accordingly. This would never be a problem for archs 281f72d41faSOleg Nesterov * that have fixed length instructions. 28229dedee0SOleg Nesterov * 283f72d41faSOleg Nesterov * uprobe_write_opcode - write the opcode at a given virtual address. 284a5f4374aSIngo Molnar * @mm: the probed process address space. 285a5f4374aSIngo Molnar * @vaddr: the virtual address to store the opcode. 286a5f4374aSIngo Molnar * @opcode: opcode to be written at @vaddr. 287a5f4374aSIngo Molnar * 28829dedee0SOleg Nesterov * Called with mm->mmap_sem held for write. 289a5f4374aSIngo Molnar * Return 0 (success) or a negative errno. 290a5f4374aSIngo Molnar */ 291f72d41faSOleg Nesterov int uprobe_write_opcode(struct mm_struct *mm, unsigned long vaddr, 292cceb55aaSOleg Nesterov uprobe_opcode_t opcode) 293a5f4374aSIngo Molnar { 294a5f4374aSIngo Molnar struct page *old_page, *new_page; 295a5f4374aSIngo Molnar struct vm_area_struct *vma; 296a5f4374aSIngo Molnar int ret; 297f403072cSOleg Nesterov 2985323ce71SOleg Nesterov retry: 299a5f4374aSIngo Molnar /* Read the page with vaddr into memory */ 30075ed82eaSOleg Nesterov ret = get_user_pages(NULL, mm, vaddr, 1, 0, 1, &old_page, &vma); 301a5f4374aSIngo Molnar if (ret <= 0) 302a5f4374aSIngo Molnar return ret; 303a5f4374aSIngo Molnar 304ed6f6a50SOleg Nesterov ret = verify_opcode(old_page, vaddr, &opcode); 305ed6f6a50SOleg Nesterov if (ret <= 0) 306ed6f6a50SOleg Nesterov goto put_old; 307ed6f6a50SOleg Nesterov 30829dedee0SOleg Nesterov ret = anon_vma_prepare(vma); 30929dedee0SOleg Nesterov if (ret) 31029dedee0SOleg Nesterov goto put_old; 31129dedee0SOleg Nesterov 312a5f4374aSIngo Molnar ret = -ENOMEM; 313a5f4374aSIngo Molnar new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, vaddr); 314a5f4374aSIngo Molnar if (!new_page) 3159f92448cSOleg Nesterov goto put_old; 316a5f4374aSIngo Molnar 31729dedee0SOleg Nesterov if (mem_cgroup_charge_anon(new_page, mm, GFP_KERNEL)) 31829dedee0SOleg Nesterov goto put_new; 319a5f4374aSIngo Molnar 32029dedee0SOleg Nesterov __SetPageUptodate(new_page); 3213f47107cSOleg Nesterov copy_highpage(new_page, old_page); 3223f47107cSOleg Nesterov copy_to_page(new_page, vaddr, &opcode, UPROBE_SWBP_INSN_SIZE); 323a5f4374aSIngo Molnar 324c517ee74SOleg Nesterov ret = __replace_page(vma, vaddr, old_page, new_page); 32529dedee0SOleg Nesterov if (ret) 32629dedee0SOleg Nesterov mem_cgroup_uncharge_page(new_page); 327a5f4374aSIngo Molnar 3289f92448cSOleg Nesterov put_new: 329a5f4374aSIngo Molnar page_cache_release(new_page); 3309f92448cSOleg Nesterov put_old: 331a5f4374aSIngo Molnar put_page(old_page); 332a5f4374aSIngo Molnar 3335323ce71SOleg Nesterov if (unlikely(ret == -EAGAIN)) 3345323ce71SOleg Nesterov goto retry; 335a5f4374aSIngo Molnar return ret; 336a5f4374aSIngo Molnar } 337a5f4374aSIngo Molnar 338a5f4374aSIngo Molnar /** 3395cb4ac3aSSrikar Dronamraju * set_swbp - store breakpoint at a given address. 340e3343e6aSSrikar Dronamraju * @auprobe: arch specific probepoint information. 341a5f4374aSIngo Molnar * @mm: the probed process address space. 342a5f4374aSIngo Molnar * @vaddr: the virtual address to insert the opcode. 343a5f4374aSIngo Molnar * 344a5f4374aSIngo Molnar * For mm @mm, store the breakpoint instruction at @vaddr. 345a5f4374aSIngo Molnar * Return 0 (success) or a negative errno. 346a5f4374aSIngo Molnar */ 3475cb4ac3aSSrikar Dronamraju int __weak set_swbp(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned long vaddr) 348a5f4374aSIngo Molnar { 349f72d41faSOleg Nesterov return uprobe_write_opcode(mm, vaddr, UPROBE_SWBP_INSN); 350a5f4374aSIngo Molnar } 351a5f4374aSIngo Molnar 352a5f4374aSIngo Molnar /** 353a5f4374aSIngo Molnar * set_orig_insn - Restore the original instruction. 354a5f4374aSIngo Molnar * @mm: the probed process address space. 355e3343e6aSSrikar Dronamraju * @auprobe: arch specific probepoint information. 356a5f4374aSIngo Molnar * @vaddr: the virtual address to insert the opcode. 357a5f4374aSIngo Molnar * 358a5f4374aSIngo Molnar * For mm @mm, restore the original opcode (opcode) at @vaddr. 359a5f4374aSIngo Molnar * Return 0 (success) or a negative errno. 360a5f4374aSIngo Molnar */ 361a5f4374aSIngo Molnar int __weak 362ded86e7cSOleg Nesterov set_orig_insn(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned long vaddr) 363a5f4374aSIngo Molnar { 364803200e2SOleg Nesterov return uprobe_write_opcode(mm, vaddr, *(uprobe_opcode_t *)&auprobe->insn); 365a5f4374aSIngo Molnar } 366a5f4374aSIngo Molnar 367a5f4374aSIngo Molnar static int match_uprobe(struct uprobe *l, struct uprobe *r) 368a5f4374aSIngo Molnar { 369a5f4374aSIngo Molnar if (l->inode < r->inode) 370a5f4374aSIngo Molnar return -1; 371a5f4374aSIngo Molnar 372a5f4374aSIngo Molnar if (l->inode > r->inode) 373a5f4374aSIngo Molnar return 1; 374a5f4374aSIngo Molnar 375a5f4374aSIngo Molnar if (l->offset < r->offset) 376a5f4374aSIngo Molnar return -1; 377a5f4374aSIngo Molnar 378a5f4374aSIngo Molnar if (l->offset > r->offset) 379a5f4374aSIngo Molnar return 1; 380a5f4374aSIngo Molnar 381a5f4374aSIngo Molnar return 0; 382a5f4374aSIngo Molnar } 383a5f4374aSIngo Molnar 384a5f4374aSIngo Molnar static struct uprobe *__find_uprobe(struct inode *inode, loff_t offset) 385a5f4374aSIngo Molnar { 386a5f4374aSIngo Molnar struct uprobe u = { .inode = inode, .offset = offset }; 387a5f4374aSIngo Molnar struct rb_node *n = uprobes_tree.rb_node; 388a5f4374aSIngo Molnar struct uprobe *uprobe; 389a5f4374aSIngo Molnar int match; 390a5f4374aSIngo Molnar 391a5f4374aSIngo Molnar while (n) { 392a5f4374aSIngo Molnar uprobe = rb_entry(n, struct uprobe, rb_node); 393a5f4374aSIngo Molnar match = match_uprobe(&u, uprobe); 394a5f4374aSIngo Molnar if (!match) { 395a5f4374aSIngo Molnar atomic_inc(&uprobe->ref); 396a5f4374aSIngo Molnar return uprobe; 397a5f4374aSIngo Molnar } 398a5f4374aSIngo Molnar 399a5f4374aSIngo Molnar if (match < 0) 400a5f4374aSIngo Molnar n = n->rb_left; 401a5f4374aSIngo Molnar else 402a5f4374aSIngo Molnar n = n->rb_right; 403a5f4374aSIngo Molnar } 404a5f4374aSIngo Molnar return NULL; 405a5f4374aSIngo Molnar } 406a5f4374aSIngo Molnar 407a5f4374aSIngo Molnar /* 408a5f4374aSIngo Molnar * Find a uprobe corresponding to a given inode:offset 409a5f4374aSIngo Molnar * Acquires uprobes_treelock 410a5f4374aSIngo Molnar */ 411a5f4374aSIngo Molnar static struct uprobe *find_uprobe(struct inode *inode, loff_t offset) 412a5f4374aSIngo Molnar { 413a5f4374aSIngo Molnar struct uprobe *uprobe; 414a5f4374aSIngo Molnar 4156f47caa0SOleg Nesterov spin_lock(&uprobes_treelock); 416a5f4374aSIngo Molnar uprobe = __find_uprobe(inode, offset); 4176f47caa0SOleg Nesterov spin_unlock(&uprobes_treelock); 418a5f4374aSIngo Molnar 419a5f4374aSIngo Molnar return uprobe; 420a5f4374aSIngo Molnar } 421a5f4374aSIngo Molnar 422a5f4374aSIngo Molnar static struct uprobe *__insert_uprobe(struct uprobe *uprobe) 423a5f4374aSIngo Molnar { 424a5f4374aSIngo Molnar struct rb_node **p = &uprobes_tree.rb_node; 425a5f4374aSIngo Molnar struct rb_node *parent = NULL; 426a5f4374aSIngo Molnar struct uprobe *u; 427a5f4374aSIngo Molnar int match; 428a5f4374aSIngo Molnar 429a5f4374aSIngo Molnar while (*p) { 430a5f4374aSIngo Molnar parent = *p; 431a5f4374aSIngo Molnar u = rb_entry(parent, struct uprobe, rb_node); 432a5f4374aSIngo Molnar match = match_uprobe(uprobe, u); 433a5f4374aSIngo Molnar if (!match) { 434a5f4374aSIngo Molnar atomic_inc(&u->ref); 435a5f4374aSIngo Molnar return u; 436a5f4374aSIngo Molnar } 437a5f4374aSIngo Molnar 438a5f4374aSIngo Molnar if (match < 0) 439a5f4374aSIngo Molnar p = &parent->rb_left; 440a5f4374aSIngo Molnar else 441a5f4374aSIngo Molnar p = &parent->rb_right; 442a5f4374aSIngo Molnar 443a5f4374aSIngo Molnar } 444a5f4374aSIngo Molnar 445a5f4374aSIngo Molnar u = NULL; 446a5f4374aSIngo Molnar rb_link_node(&uprobe->rb_node, parent, p); 447a5f4374aSIngo Molnar rb_insert_color(&uprobe->rb_node, &uprobes_tree); 448a5f4374aSIngo Molnar /* get access + creation ref */ 449a5f4374aSIngo Molnar atomic_set(&uprobe->ref, 2); 450a5f4374aSIngo Molnar 451a5f4374aSIngo Molnar return u; 452a5f4374aSIngo Molnar } 453a5f4374aSIngo Molnar 454a5f4374aSIngo Molnar /* 455a5f4374aSIngo Molnar * Acquire uprobes_treelock. 456a5f4374aSIngo Molnar * Matching uprobe already exists in rbtree; 457a5f4374aSIngo Molnar * increment (access refcount) and return the matching uprobe. 458a5f4374aSIngo Molnar * 459a5f4374aSIngo Molnar * No matching uprobe; insert the uprobe in rb_tree; 460a5f4374aSIngo Molnar * get a double refcount (access + creation) and return NULL. 461a5f4374aSIngo Molnar */ 462a5f4374aSIngo Molnar static struct uprobe *insert_uprobe(struct uprobe *uprobe) 463a5f4374aSIngo Molnar { 464a5f4374aSIngo Molnar struct uprobe *u; 465a5f4374aSIngo Molnar 4666f47caa0SOleg Nesterov spin_lock(&uprobes_treelock); 467a5f4374aSIngo Molnar u = __insert_uprobe(uprobe); 4686f47caa0SOleg Nesterov spin_unlock(&uprobes_treelock); 469a5f4374aSIngo Molnar 470a5f4374aSIngo Molnar return u; 471a5f4374aSIngo Molnar } 472a5f4374aSIngo Molnar 473a5f4374aSIngo Molnar static void put_uprobe(struct uprobe *uprobe) 474a5f4374aSIngo Molnar { 475a5f4374aSIngo Molnar if (atomic_dec_and_test(&uprobe->ref)) 476a5f4374aSIngo Molnar kfree(uprobe); 477a5f4374aSIngo Molnar } 478a5f4374aSIngo Molnar 479a5f4374aSIngo Molnar static struct uprobe *alloc_uprobe(struct inode *inode, loff_t offset) 480a5f4374aSIngo Molnar { 481a5f4374aSIngo Molnar struct uprobe *uprobe, *cur_uprobe; 482a5f4374aSIngo Molnar 483a5f4374aSIngo Molnar uprobe = kzalloc(sizeof(struct uprobe), GFP_KERNEL); 484a5f4374aSIngo Molnar if (!uprobe) 485a5f4374aSIngo Molnar return NULL; 486a5f4374aSIngo Molnar 487a5f4374aSIngo Molnar uprobe->inode = igrab(inode); 488a5f4374aSIngo Molnar uprobe->offset = offset; 489e591c8d7SOleg Nesterov init_rwsem(&uprobe->register_rwsem); 490a5f4374aSIngo Molnar init_rwsem(&uprobe->consumer_rwsem); 491a5f4374aSIngo Molnar 492a5f4374aSIngo Molnar /* add to uprobes_tree, sorted on inode:offset */ 493a5f4374aSIngo Molnar cur_uprobe = insert_uprobe(uprobe); 494a5f4374aSIngo Molnar /* a uprobe exists for this inode:offset combination */ 495a5f4374aSIngo Molnar if (cur_uprobe) { 496a5f4374aSIngo Molnar kfree(uprobe); 497a5f4374aSIngo Molnar uprobe = cur_uprobe; 498a5f4374aSIngo Molnar iput(inode); 499a5f4374aSIngo Molnar } 500a5f4374aSIngo Molnar 501a5f4374aSIngo Molnar return uprobe; 502a5f4374aSIngo Molnar } 503a5f4374aSIngo Molnar 5049a98e03cSOleg Nesterov static void consumer_add(struct uprobe *uprobe, struct uprobe_consumer *uc) 505a5f4374aSIngo Molnar { 506a5f4374aSIngo Molnar down_write(&uprobe->consumer_rwsem); 507e3343e6aSSrikar Dronamraju uc->next = uprobe->consumers; 508e3343e6aSSrikar Dronamraju uprobe->consumers = uc; 509a5f4374aSIngo Molnar up_write(&uprobe->consumer_rwsem); 510a5f4374aSIngo Molnar } 511a5f4374aSIngo Molnar 512a5f4374aSIngo Molnar /* 513e3343e6aSSrikar Dronamraju * For uprobe @uprobe, delete the consumer @uc. 514e3343e6aSSrikar Dronamraju * Return true if the @uc is deleted successfully 515a5f4374aSIngo Molnar * or return false. 516a5f4374aSIngo Molnar */ 517e3343e6aSSrikar Dronamraju static bool consumer_del(struct uprobe *uprobe, struct uprobe_consumer *uc) 518a5f4374aSIngo Molnar { 519a5f4374aSIngo Molnar struct uprobe_consumer **con; 520a5f4374aSIngo Molnar bool ret = false; 521a5f4374aSIngo Molnar 522a5f4374aSIngo Molnar down_write(&uprobe->consumer_rwsem); 523a5f4374aSIngo Molnar for (con = &uprobe->consumers; *con; con = &(*con)->next) { 524e3343e6aSSrikar Dronamraju if (*con == uc) { 525e3343e6aSSrikar Dronamraju *con = uc->next; 526a5f4374aSIngo Molnar ret = true; 527a5f4374aSIngo Molnar break; 528a5f4374aSIngo Molnar } 529a5f4374aSIngo Molnar } 530a5f4374aSIngo Molnar up_write(&uprobe->consumer_rwsem); 531a5f4374aSIngo Molnar 532a5f4374aSIngo Molnar return ret; 533a5f4374aSIngo Molnar } 534a5f4374aSIngo Molnar 5352ded0980SOleg Nesterov static int __copy_insn(struct address_space *mapping, struct file *filp, 5362ded0980SOleg Nesterov void *insn, int nbytes, loff_t offset) 537a5f4374aSIngo Molnar { 538a5f4374aSIngo Molnar struct page *page; 539a5f4374aSIngo Molnar /* 540a5f4374aSIngo Molnar * Ensure that the page that has the original instruction is 541a5f4374aSIngo Molnar * populated and in page-cache. 542a5f4374aSIngo Molnar */ 5432edb7b55SOleg Nesterov page = read_mapping_page(mapping, offset >> PAGE_CACHE_SHIFT, filp); 544a5f4374aSIngo Molnar if (IS_ERR(page)) 545a5f4374aSIngo Molnar return PTR_ERR(page); 546a5f4374aSIngo Molnar 5472edb7b55SOleg Nesterov copy_from_page(page, offset, insn, nbytes); 548a5f4374aSIngo Molnar page_cache_release(page); 549a5f4374aSIngo Molnar 550a5f4374aSIngo Molnar return 0; 551a5f4374aSIngo Molnar } 552a5f4374aSIngo Molnar 553d436615eSOleg Nesterov static int copy_insn(struct uprobe *uprobe, struct file *filp) 554a5f4374aSIngo Molnar { 5552ded0980SOleg Nesterov struct address_space *mapping = uprobe->inode->i_mapping; 5562ded0980SOleg Nesterov loff_t offs = uprobe->offset; 557803200e2SOleg Nesterov void *insn = &uprobe->arch.insn; 558803200e2SOleg Nesterov int size = sizeof(uprobe->arch.insn); 5592ded0980SOleg Nesterov int len, err = -EIO; 560a5f4374aSIngo Molnar 5612ded0980SOleg Nesterov /* Copy only available bytes, -EIO if nothing was read */ 5622ded0980SOleg Nesterov do { 5632ded0980SOleg Nesterov if (offs >= i_size_read(uprobe->inode)) 5642ded0980SOleg Nesterov break; 565a5f4374aSIngo Molnar 5662ded0980SOleg Nesterov len = min_t(int, size, PAGE_SIZE - (offs & ~PAGE_MASK)); 5672ded0980SOleg Nesterov err = __copy_insn(mapping, filp, insn, len, offs); 568fc36f595SOleg Nesterov if (err) 5692ded0980SOleg Nesterov break; 5702ded0980SOleg Nesterov 5712ded0980SOleg Nesterov insn += len; 5722ded0980SOleg Nesterov offs += len; 5732ded0980SOleg Nesterov size -= len; 5742ded0980SOleg Nesterov } while (size); 5752ded0980SOleg Nesterov 576fc36f595SOleg Nesterov return err; 577a5f4374aSIngo Molnar } 578a5f4374aSIngo Molnar 579cb9a19feSOleg Nesterov static int prepare_uprobe(struct uprobe *uprobe, struct file *file, 580cb9a19feSOleg Nesterov struct mm_struct *mm, unsigned long vaddr) 581cb9a19feSOleg Nesterov { 582cb9a19feSOleg Nesterov int ret = 0; 583cb9a19feSOleg Nesterov 58471434f2fSOleg Nesterov if (test_bit(UPROBE_COPY_INSN, &uprobe->flags)) 585cb9a19feSOleg Nesterov return ret; 586cb9a19feSOleg Nesterov 587d4d3ccc6SOleg Nesterov /* TODO: move this into _register, until then we abuse this sem. */ 588d4d3ccc6SOleg Nesterov down_write(&uprobe->consumer_rwsem); 58971434f2fSOleg Nesterov if (test_bit(UPROBE_COPY_INSN, &uprobe->flags)) 5904710f05fSOleg Nesterov goto out; 5914710f05fSOleg Nesterov 592cb9a19feSOleg Nesterov ret = copy_insn(uprobe, file); 593cb9a19feSOleg Nesterov if (ret) 594cb9a19feSOleg Nesterov goto out; 595cb9a19feSOleg Nesterov 596cb9a19feSOleg Nesterov ret = -ENOTSUPP; 597803200e2SOleg Nesterov if (is_trap_insn((uprobe_opcode_t *)&uprobe->arch.insn)) 598cb9a19feSOleg Nesterov goto out; 599cb9a19feSOleg Nesterov 600cb9a19feSOleg Nesterov ret = arch_uprobe_analyze_insn(&uprobe->arch, mm, vaddr); 601cb9a19feSOleg Nesterov if (ret) 602cb9a19feSOleg Nesterov goto out; 603cb9a19feSOleg Nesterov 604f72d41faSOleg Nesterov /* uprobe_write_opcode() assumes we don't cross page boundary */ 605cb9a19feSOleg Nesterov BUG_ON((uprobe->offset & ~PAGE_MASK) + 606cb9a19feSOleg Nesterov UPROBE_SWBP_INSN_SIZE > PAGE_SIZE); 607cb9a19feSOleg Nesterov 608cb9a19feSOleg Nesterov smp_wmb(); /* pairs with rmb() in find_active_uprobe() */ 60971434f2fSOleg Nesterov set_bit(UPROBE_COPY_INSN, &uprobe->flags); 610cb9a19feSOleg Nesterov 611cb9a19feSOleg Nesterov out: 612d4d3ccc6SOleg Nesterov up_write(&uprobe->consumer_rwsem); 6134710f05fSOleg Nesterov 614cb9a19feSOleg Nesterov return ret; 615cb9a19feSOleg Nesterov } 616cb9a19feSOleg Nesterov 6178a7f2fa0SOleg Nesterov static inline bool consumer_filter(struct uprobe_consumer *uc, 6188a7f2fa0SOleg Nesterov enum uprobe_filter_ctx ctx, struct mm_struct *mm) 619806a98bdSOleg Nesterov { 6208a7f2fa0SOleg Nesterov return !uc->filter || uc->filter(uc, ctx, mm); 621806a98bdSOleg Nesterov } 622806a98bdSOleg Nesterov 6238a7f2fa0SOleg Nesterov static bool filter_chain(struct uprobe *uprobe, 6248a7f2fa0SOleg Nesterov enum uprobe_filter_ctx ctx, struct mm_struct *mm) 62563633cbfSOleg Nesterov { 6261ff6fee5SOleg Nesterov struct uprobe_consumer *uc; 6271ff6fee5SOleg Nesterov bool ret = false; 6281ff6fee5SOleg Nesterov 6291ff6fee5SOleg Nesterov down_read(&uprobe->consumer_rwsem); 6301ff6fee5SOleg Nesterov for (uc = uprobe->consumers; uc; uc = uc->next) { 6318a7f2fa0SOleg Nesterov ret = consumer_filter(uc, ctx, mm); 6321ff6fee5SOleg Nesterov if (ret) 6331ff6fee5SOleg Nesterov break; 6341ff6fee5SOleg Nesterov } 6351ff6fee5SOleg Nesterov up_read(&uprobe->consumer_rwsem); 6361ff6fee5SOleg Nesterov 6371ff6fee5SOleg Nesterov return ret; 63863633cbfSOleg Nesterov } 63963633cbfSOleg Nesterov 640e3343e6aSSrikar Dronamraju static int 641e3343e6aSSrikar Dronamraju install_breakpoint(struct uprobe *uprobe, struct mm_struct *mm, 642816c03fbSOleg Nesterov struct vm_area_struct *vma, unsigned long vaddr) 643a5f4374aSIngo Molnar { 644f8ac4ec9SOleg Nesterov bool first_uprobe; 645a5f4374aSIngo Molnar int ret; 646a5f4374aSIngo Molnar 647cb9a19feSOleg Nesterov ret = prepare_uprobe(uprobe, vma->vm_file, mm, vaddr); 648a5f4374aSIngo Molnar if (ret) 649a5f4374aSIngo Molnar return ret; 650a5f4374aSIngo Molnar 651f8ac4ec9SOleg Nesterov /* 652f8ac4ec9SOleg Nesterov * set MMF_HAS_UPROBES in advance for uprobe_pre_sstep_notifier(), 653f8ac4ec9SOleg Nesterov * the task can hit this breakpoint right after __replace_page(). 654f8ac4ec9SOleg Nesterov */ 655f8ac4ec9SOleg Nesterov first_uprobe = !test_bit(MMF_HAS_UPROBES, &mm->flags); 656f8ac4ec9SOleg Nesterov if (first_uprobe) 657f8ac4ec9SOleg Nesterov set_bit(MMF_HAS_UPROBES, &mm->flags); 658f8ac4ec9SOleg Nesterov 659816c03fbSOleg Nesterov ret = set_swbp(&uprobe->arch, mm, vaddr); 6609f68f672SOleg Nesterov if (!ret) 6619f68f672SOleg Nesterov clear_bit(MMF_RECALC_UPROBES, &mm->flags); 6629f68f672SOleg Nesterov else if (first_uprobe) 663f8ac4ec9SOleg Nesterov clear_bit(MMF_HAS_UPROBES, &mm->flags); 664a5f4374aSIngo Molnar 665a5f4374aSIngo Molnar return ret; 666a5f4374aSIngo Molnar } 667a5f4374aSIngo Molnar 668076a365bSOleg Nesterov static int 669816c03fbSOleg Nesterov remove_breakpoint(struct uprobe *uprobe, struct mm_struct *mm, unsigned long vaddr) 670a5f4374aSIngo Molnar { 6719f68f672SOleg Nesterov set_bit(MMF_RECALC_UPROBES, &mm->flags); 672076a365bSOleg Nesterov return set_orig_insn(&uprobe->arch, mm, vaddr); 673a5f4374aSIngo Molnar } 674a5f4374aSIngo Molnar 67506b7bcd8SOleg Nesterov static inline bool uprobe_is_active(struct uprobe *uprobe) 67606b7bcd8SOleg Nesterov { 67706b7bcd8SOleg Nesterov return !RB_EMPTY_NODE(&uprobe->rb_node); 67806b7bcd8SOleg Nesterov } 6790326f5a9SSrikar Dronamraju /* 680778b032dSOleg Nesterov * There could be threads that have already hit the breakpoint. They 681778b032dSOleg Nesterov * will recheck the current insn and restart if find_uprobe() fails. 682778b032dSOleg Nesterov * See find_active_uprobe(). 6830326f5a9SSrikar Dronamraju */ 684a5f4374aSIngo Molnar static void delete_uprobe(struct uprobe *uprobe) 685a5f4374aSIngo Molnar { 68606b7bcd8SOleg Nesterov if (WARN_ON(!uprobe_is_active(uprobe))) 68706b7bcd8SOleg Nesterov return; 68806b7bcd8SOleg Nesterov 6896f47caa0SOleg Nesterov spin_lock(&uprobes_treelock); 690a5f4374aSIngo Molnar rb_erase(&uprobe->rb_node, &uprobes_tree); 6916f47caa0SOleg Nesterov spin_unlock(&uprobes_treelock); 69206b7bcd8SOleg Nesterov RB_CLEAR_NODE(&uprobe->rb_node); /* for uprobe_is_active() */ 693a5f4374aSIngo Molnar iput(uprobe->inode); 694a5f4374aSIngo Molnar put_uprobe(uprobe); 695a5f4374aSIngo Molnar } 696a5f4374aSIngo Molnar 69726872090SOleg Nesterov struct map_info { 69826872090SOleg Nesterov struct map_info *next; 69926872090SOleg Nesterov struct mm_struct *mm; 700816c03fbSOleg Nesterov unsigned long vaddr; 70126872090SOleg Nesterov }; 70226872090SOleg Nesterov 70326872090SOleg Nesterov static inline struct map_info *free_map_info(struct map_info *info) 704a5f4374aSIngo Molnar { 70526872090SOleg Nesterov struct map_info *next = info->next; 70626872090SOleg Nesterov kfree(info); 70726872090SOleg Nesterov return next; 70826872090SOleg Nesterov } 70926872090SOleg Nesterov 71026872090SOleg Nesterov static struct map_info * 71126872090SOleg Nesterov build_map_info(struct address_space *mapping, loff_t offset, bool is_register) 71226872090SOleg Nesterov { 71326872090SOleg Nesterov unsigned long pgoff = offset >> PAGE_SHIFT; 714a5f4374aSIngo Molnar struct vm_area_struct *vma; 71526872090SOleg Nesterov struct map_info *curr = NULL; 71626872090SOleg Nesterov struct map_info *prev = NULL; 71726872090SOleg Nesterov struct map_info *info; 71826872090SOleg Nesterov int more = 0; 719a5f4374aSIngo Molnar 72026872090SOleg Nesterov again: 72126872090SOleg Nesterov mutex_lock(&mapping->i_mmap_mutex); 7226b2dbba8SMichel Lespinasse vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) { 723a5f4374aSIngo Molnar if (!valid_vma(vma, is_register)) 724a5f4374aSIngo Molnar continue; 725a5f4374aSIngo Molnar 7267a5bfb66SOleg Nesterov if (!prev && !more) { 7277a5bfb66SOleg Nesterov /* 7287a5bfb66SOleg Nesterov * Needs GFP_NOWAIT to avoid i_mmap_mutex recursion through 7297a5bfb66SOleg Nesterov * reclaim. This is optimistic, no harm done if it fails. 7307a5bfb66SOleg Nesterov */ 7317a5bfb66SOleg Nesterov prev = kmalloc(sizeof(struct map_info), 7327a5bfb66SOleg Nesterov GFP_NOWAIT | __GFP_NOMEMALLOC | __GFP_NOWARN); 7337a5bfb66SOleg Nesterov if (prev) 7347a5bfb66SOleg Nesterov prev->next = NULL; 7357a5bfb66SOleg Nesterov } 73626872090SOleg Nesterov if (!prev) { 73726872090SOleg Nesterov more++; 73826872090SOleg Nesterov continue; 739a5f4374aSIngo Molnar } 740a5f4374aSIngo Molnar 74126872090SOleg Nesterov if (!atomic_inc_not_zero(&vma->vm_mm->mm_users)) 74226872090SOleg Nesterov continue; 743a5f4374aSIngo Molnar 74426872090SOleg Nesterov info = prev; 74526872090SOleg Nesterov prev = prev->next; 74626872090SOleg Nesterov info->next = curr; 74726872090SOleg Nesterov curr = info; 74826872090SOleg Nesterov 74926872090SOleg Nesterov info->mm = vma->vm_mm; 75057683f72SOleg Nesterov info->vaddr = offset_to_vaddr(vma, offset); 751a5f4374aSIngo Molnar } 752a5f4374aSIngo Molnar mutex_unlock(&mapping->i_mmap_mutex); 753a5f4374aSIngo Molnar 75426872090SOleg Nesterov if (!more) 75526872090SOleg Nesterov goto out; 756a5f4374aSIngo Molnar 75726872090SOleg Nesterov prev = curr; 75826872090SOleg Nesterov while (curr) { 75926872090SOleg Nesterov mmput(curr->mm); 76026872090SOleg Nesterov curr = curr->next; 76126872090SOleg Nesterov } 76226872090SOleg Nesterov 76326872090SOleg Nesterov do { 76426872090SOleg Nesterov info = kmalloc(sizeof(struct map_info), GFP_KERNEL); 76526872090SOleg Nesterov if (!info) { 76626872090SOleg Nesterov curr = ERR_PTR(-ENOMEM); 76726872090SOleg Nesterov goto out; 76826872090SOleg Nesterov } 76926872090SOleg Nesterov info->next = prev; 77026872090SOleg Nesterov prev = info; 77126872090SOleg Nesterov } while (--more); 77226872090SOleg Nesterov 77326872090SOleg Nesterov goto again; 77426872090SOleg Nesterov out: 77526872090SOleg Nesterov while (prev) 77626872090SOleg Nesterov prev = free_map_info(prev); 77726872090SOleg Nesterov return curr; 778a5f4374aSIngo Molnar } 779a5f4374aSIngo Molnar 780bdf8647cSOleg Nesterov static int 781bdf8647cSOleg Nesterov register_for_each_vma(struct uprobe *uprobe, struct uprobe_consumer *new) 782a5f4374aSIngo Molnar { 783bdf8647cSOleg Nesterov bool is_register = !!new; 78426872090SOleg Nesterov struct map_info *info; 78526872090SOleg Nesterov int err = 0; 78626872090SOleg Nesterov 78732cdba1eSOleg Nesterov percpu_down_write(&dup_mmap_sem); 78826872090SOleg Nesterov info = build_map_info(uprobe->inode->i_mapping, 78926872090SOleg Nesterov uprobe->offset, is_register); 79032cdba1eSOleg Nesterov if (IS_ERR(info)) { 79132cdba1eSOleg Nesterov err = PTR_ERR(info); 79232cdba1eSOleg Nesterov goto out; 79332cdba1eSOleg Nesterov } 79426872090SOleg Nesterov 79526872090SOleg Nesterov while (info) { 79626872090SOleg Nesterov struct mm_struct *mm = info->mm; 797a5f4374aSIngo Molnar struct vm_area_struct *vma; 798a5f4374aSIngo Molnar 799076a365bSOleg Nesterov if (err && is_register) 80026872090SOleg Nesterov goto free; 801a5f4374aSIngo Molnar 80277fc4af1SOleg Nesterov down_write(&mm->mmap_sem); 803f4d6dfe5SOleg Nesterov vma = find_vma(mm, info->vaddr); 804f4d6dfe5SOleg Nesterov if (!vma || !valid_vma(vma, is_register) || 805f281769eSOleg Nesterov file_inode(vma->vm_file) != uprobe->inode) 80626872090SOleg Nesterov goto unlock; 80726872090SOleg Nesterov 808f4d6dfe5SOleg Nesterov if (vma->vm_start > info->vaddr || 809f4d6dfe5SOleg Nesterov vaddr_to_offset(vma, info->vaddr) != uprobe->offset) 81026872090SOleg Nesterov goto unlock; 811a5f4374aSIngo Molnar 812806a98bdSOleg Nesterov if (is_register) { 813806a98bdSOleg Nesterov /* consult only the "caller", new consumer. */ 814bdf8647cSOleg Nesterov if (consumer_filter(new, 8158a7f2fa0SOleg Nesterov UPROBE_FILTER_REGISTER, mm)) 81626872090SOleg Nesterov err = install_breakpoint(uprobe, mm, vma, info->vaddr); 817806a98bdSOleg Nesterov } else if (test_bit(MMF_HAS_UPROBES, &mm->flags)) { 8188a7f2fa0SOleg Nesterov if (!filter_chain(uprobe, 8198a7f2fa0SOleg Nesterov UPROBE_FILTER_UNREGISTER, mm)) 820076a365bSOleg Nesterov err |= remove_breakpoint(uprobe, mm, info->vaddr); 821806a98bdSOleg Nesterov } 82278f74116SOleg Nesterov 82326872090SOleg Nesterov unlock: 82426872090SOleg Nesterov up_write(&mm->mmap_sem); 82526872090SOleg Nesterov free: 82626872090SOleg Nesterov mmput(mm); 82726872090SOleg Nesterov info = free_map_info(info); 828a5f4374aSIngo Molnar } 82932cdba1eSOleg Nesterov out: 83032cdba1eSOleg Nesterov percpu_up_write(&dup_mmap_sem); 83126872090SOleg Nesterov return err; 832a5f4374aSIngo Molnar } 833a5f4374aSIngo Molnar 8349a98e03cSOleg Nesterov static int __uprobe_register(struct uprobe *uprobe, struct uprobe_consumer *uc) 835a5f4374aSIngo Molnar { 8369a98e03cSOleg Nesterov consumer_add(uprobe, uc); 837bdf8647cSOleg Nesterov return register_for_each_vma(uprobe, uc); 838a5f4374aSIngo Molnar } 839a5f4374aSIngo Molnar 84004aab9b2SOleg Nesterov static void __uprobe_unregister(struct uprobe *uprobe, struct uprobe_consumer *uc) 841a5f4374aSIngo Molnar { 84204aab9b2SOleg Nesterov int err; 843a5f4374aSIngo Molnar 84404aab9b2SOleg Nesterov if (!consumer_del(uprobe, uc)) /* WARN? */ 84504aab9b2SOleg Nesterov return; 84604aab9b2SOleg Nesterov 847bdf8647cSOleg Nesterov err = register_for_each_vma(uprobe, NULL); 848a5f4374aSIngo Molnar /* TODO : cant unregister? schedule a worker thread */ 849bb929284SOleg Nesterov if (!uprobe->consumers && !err) 85004aab9b2SOleg Nesterov delete_uprobe(uprobe); 85104aab9b2SOleg Nesterov } 852a5f4374aSIngo Molnar 853a5f4374aSIngo Molnar /* 854a5f4374aSIngo Molnar * uprobe_register - register a probe 855a5f4374aSIngo Molnar * @inode: the file in which the probe has to be placed. 856a5f4374aSIngo Molnar * @offset: offset from the start of the file. 857e3343e6aSSrikar Dronamraju * @uc: information on howto handle the probe.. 858a5f4374aSIngo Molnar * 859a5f4374aSIngo Molnar * Apart from the access refcount, uprobe_register() takes a creation 860a5f4374aSIngo Molnar * refcount (thro alloc_uprobe) if and only if this @uprobe is getting 861a5f4374aSIngo Molnar * inserted into the rbtree (i.e first consumer for a @inode:@offset 862a5f4374aSIngo Molnar * tuple). Creation refcount stops uprobe_unregister from freeing the 863a5f4374aSIngo Molnar * @uprobe even before the register operation is complete. Creation 864e3343e6aSSrikar Dronamraju * refcount is released when the last @uc for the @uprobe 865a5f4374aSIngo Molnar * unregisters. 866a5f4374aSIngo Molnar * 867a5f4374aSIngo Molnar * Return errno if it cannot successully install probes 868a5f4374aSIngo Molnar * else return 0 (success) 869a5f4374aSIngo Molnar */ 870e3343e6aSSrikar Dronamraju int uprobe_register(struct inode *inode, loff_t offset, struct uprobe_consumer *uc) 871a5f4374aSIngo Molnar { 872a5f4374aSIngo Molnar struct uprobe *uprobe; 873a5f4374aSIngo Molnar int ret; 874a5f4374aSIngo Molnar 875ea024870SAnton Arapov /* Uprobe must have at least one set consumer */ 876ea024870SAnton Arapov if (!uc->handler && !uc->ret_handler) 877ea024870SAnton Arapov return -EINVAL; 878ea024870SAnton Arapov 87941ccba02SOleg Nesterov /* copy_insn()->read_mapping_page() needs ->readpage() */ 88041ccba02SOleg Nesterov if (!inode->i_mapping->a_ops->readpage) 88141ccba02SOleg Nesterov return -EIO; 882f0744af7SOleg Nesterov /* Racy, just to catch the obvious mistakes */ 883a5f4374aSIngo Molnar if (offset > i_size_read(inode)) 884a5f4374aSIngo Molnar return -EINVAL; 885a5f4374aSIngo Molnar 88666d06dffSOleg Nesterov retry: 887a5f4374aSIngo Molnar uprobe = alloc_uprobe(inode, offset); 88866d06dffSOleg Nesterov if (!uprobe) 88966d06dffSOleg Nesterov return -ENOMEM; 89066d06dffSOleg Nesterov /* 89166d06dffSOleg Nesterov * We can race with uprobe_unregister()->delete_uprobe(). 89266d06dffSOleg Nesterov * Check uprobe_is_active() and retry if it is false. 89366d06dffSOleg Nesterov */ 894e591c8d7SOleg Nesterov down_write(&uprobe->register_rwsem); 89566d06dffSOleg Nesterov ret = -EAGAIN; 89666d06dffSOleg Nesterov if (likely(uprobe_is_active(uprobe))) { 8979a98e03cSOleg Nesterov ret = __uprobe_register(uprobe, uc); 8989a98e03cSOleg Nesterov if (ret) 89904aab9b2SOleg Nesterov __uprobe_unregister(uprobe, uc); 900a5f4374aSIngo Molnar } 90166d06dffSOleg Nesterov up_write(&uprobe->register_rwsem); 902a5f4374aSIngo Molnar put_uprobe(uprobe); 903a5f4374aSIngo Molnar 90466d06dffSOleg Nesterov if (unlikely(ret == -EAGAIN)) 90566d06dffSOleg Nesterov goto retry; 906a5f4374aSIngo Molnar return ret; 907a5f4374aSIngo Molnar } 908e8440c14SJosh Stone EXPORT_SYMBOL_GPL(uprobe_register); 909a5f4374aSIngo Molnar 910a5f4374aSIngo Molnar /* 911bdf8647cSOleg Nesterov * uprobe_apply - unregister a already registered probe. 912bdf8647cSOleg Nesterov * @inode: the file in which the probe has to be removed. 913bdf8647cSOleg Nesterov * @offset: offset from the start of the file. 914bdf8647cSOleg Nesterov * @uc: consumer which wants to add more or remove some breakpoints 915bdf8647cSOleg Nesterov * @add: add or remove the breakpoints 916bdf8647cSOleg Nesterov */ 917bdf8647cSOleg Nesterov int uprobe_apply(struct inode *inode, loff_t offset, 918bdf8647cSOleg Nesterov struct uprobe_consumer *uc, bool add) 919bdf8647cSOleg Nesterov { 920bdf8647cSOleg Nesterov struct uprobe *uprobe; 921bdf8647cSOleg Nesterov struct uprobe_consumer *con; 922bdf8647cSOleg Nesterov int ret = -ENOENT; 923bdf8647cSOleg Nesterov 924bdf8647cSOleg Nesterov uprobe = find_uprobe(inode, offset); 925bdf8647cSOleg Nesterov if (!uprobe) 926bdf8647cSOleg Nesterov return ret; 927bdf8647cSOleg Nesterov 928bdf8647cSOleg Nesterov down_write(&uprobe->register_rwsem); 929bdf8647cSOleg Nesterov for (con = uprobe->consumers; con && con != uc ; con = con->next) 930bdf8647cSOleg Nesterov ; 931bdf8647cSOleg Nesterov if (con) 932bdf8647cSOleg Nesterov ret = register_for_each_vma(uprobe, add ? uc : NULL); 933bdf8647cSOleg Nesterov up_write(&uprobe->register_rwsem); 934bdf8647cSOleg Nesterov put_uprobe(uprobe); 935bdf8647cSOleg Nesterov 936bdf8647cSOleg Nesterov return ret; 937bdf8647cSOleg Nesterov } 938bdf8647cSOleg Nesterov 939bdf8647cSOleg Nesterov /* 940a5f4374aSIngo Molnar * uprobe_unregister - unregister a already registered probe. 941a5f4374aSIngo Molnar * @inode: the file in which the probe has to be removed. 942a5f4374aSIngo Molnar * @offset: offset from the start of the file. 943e3343e6aSSrikar Dronamraju * @uc: identify which probe if multiple probes are colocated. 944a5f4374aSIngo Molnar */ 945e3343e6aSSrikar Dronamraju void uprobe_unregister(struct inode *inode, loff_t offset, struct uprobe_consumer *uc) 946a5f4374aSIngo Molnar { 947a5f4374aSIngo Molnar struct uprobe *uprobe; 948a5f4374aSIngo Molnar 949a5f4374aSIngo Molnar uprobe = find_uprobe(inode, offset); 950a5f4374aSIngo Molnar if (!uprobe) 951a5f4374aSIngo Molnar return; 952a5f4374aSIngo Molnar 953e591c8d7SOleg Nesterov down_write(&uprobe->register_rwsem); 95404aab9b2SOleg Nesterov __uprobe_unregister(uprobe, uc); 955e591c8d7SOleg Nesterov up_write(&uprobe->register_rwsem); 956a5f4374aSIngo Molnar put_uprobe(uprobe); 957a5f4374aSIngo Molnar } 958e8440c14SJosh Stone EXPORT_SYMBOL_GPL(uprobe_unregister); 959a5f4374aSIngo Molnar 960da1816b1SOleg Nesterov static int unapply_uprobe(struct uprobe *uprobe, struct mm_struct *mm) 961da1816b1SOleg Nesterov { 962da1816b1SOleg Nesterov struct vm_area_struct *vma; 963da1816b1SOleg Nesterov int err = 0; 964da1816b1SOleg Nesterov 965da1816b1SOleg Nesterov down_read(&mm->mmap_sem); 966da1816b1SOleg Nesterov for (vma = mm->mmap; vma; vma = vma->vm_next) { 967da1816b1SOleg Nesterov unsigned long vaddr; 968da1816b1SOleg Nesterov loff_t offset; 969da1816b1SOleg Nesterov 970da1816b1SOleg Nesterov if (!valid_vma(vma, false) || 971f281769eSOleg Nesterov file_inode(vma->vm_file) != uprobe->inode) 972da1816b1SOleg Nesterov continue; 973da1816b1SOleg Nesterov 974da1816b1SOleg Nesterov offset = (loff_t)vma->vm_pgoff << PAGE_SHIFT; 975da1816b1SOleg Nesterov if (uprobe->offset < offset || 976da1816b1SOleg Nesterov uprobe->offset >= offset + vma->vm_end - vma->vm_start) 977da1816b1SOleg Nesterov continue; 978da1816b1SOleg Nesterov 979da1816b1SOleg Nesterov vaddr = offset_to_vaddr(vma, uprobe->offset); 980da1816b1SOleg Nesterov err |= remove_breakpoint(uprobe, mm, vaddr); 981da1816b1SOleg Nesterov } 982da1816b1SOleg Nesterov up_read(&mm->mmap_sem); 983da1816b1SOleg Nesterov 984da1816b1SOleg Nesterov return err; 985da1816b1SOleg Nesterov } 986da1816b1SOleg Nesterov 987891c3970SOleg Nesterov static struct rb_node * 988891c3970SOleg Nesterov find_node_in_range(struct inode *inode, loff_t min, loff_t max) 989a5f4374aSIngo Molnar { 990a5f4374aSIngo Molnar struct rb_node *n = uprobes_tree.rb_node; 991a5f4374aSIngo Molnar 992a5f4374aSIngo Molnar while (n) { 993891c3970SOleg Nesterov struct uprobe *u = rb_entry(n, struct uprobe, rb_node); 994a5f4374aSIngo Molnar 995891c3970SOleg Nesterov if (inode < u->inode) { 996a5f4374aSIngo Molnar n = n->rb_left; 997891c3970SOleg Nesterov } else if (inode > u->inode) { 998a5f4374aSIngo Molnar n = n->rb_right; 999891c3970SOleg Nesterov } else { 1000891c3970SOleg Nesterov if (max < u->offset) 1001891c3970SOleg Nesterov n = n->rb_left; 1002891c3970SOleg Nesterov else if (min > u->offset) 1003891c3970SOleg Nesterov n = n->rb_right; 1004891c3970SOleg Nesterov else 1005891c3970SOleg Nesterov break; 1006891c3970SOleg Nesterov } 1007a5f4374aSIngo Molnar } 1008a5f4374aSIngo Molnar 1009891c3970SOleg Nesterov return n; 1010a5f4374aSIngo Molnar } 1011a5f4374aSIngo Molnar 1012a5f4374aSIngo Molnar /* 1013891c3970SOleg Nesterov * For a given range in vma, build a list of probes that need to be inserted. 1014a5f4374aSIngo Molnar */ 1015891c3970SOleg Nesterov static void build_probe_list(struct inode *inode, 1016891c3970SOleg Nesterov struct vm_area_struct *vma, 1017891c3970SOleg Nesterov unsigned long start, unsigned long end, 1018891c3970SOleg Nesterov struct list_head *head) 1019a5f4374aSIngo Molnar { 1020891c3970SOleg Nesterov loff_t min, max; 1021891c3970SOleg Nesterov struct rb_node *n, *t; 1022891c3970SOleg Nesterov struct uprobe *u; 1023891c3970SOleg Nesterov 1024891c3970SOleg Nesterov INIT_LIST_HEAD(head); 1025cb113b47SOleg Nesterov min = vaddr_to_offset(vma, start); 1026891c3970SOleg Nesterov max = min + (end - start) - 1; 1027a5f4374aSIngo Molnar 10286f47caa0SOleg Nesterov spin_lock(&uprobes_treelock); 1029891c3970SOleg Nesterov n = find_node_in_range(inode, min, max); 1030891c3970SOleg Nesterov if (n) { 1031891c3970SOleg Nesterov for (t = n; t; t = rb_prev(t)) { 1032891c3970SOleg Nesterov u = rb_entry(t, struct uprobe, rb_node); 1033891c3970SOleg Nesterov if (u->inode != inode || u->offset < min) 1034a5f4374aSIngo Molnar break; 1035891c3970SOleg Nesterov list_add(&u->pending_list, head); 1036891c3970SOleg Nesterov atomic_inc(&u->ref); 1037a5f4374aSIngo Molnar } 1038891c3970SOleg Nesterov for (t = n; (t = rb_next(t)); ) { 1039891c3970SOleg Nesterov u = rb_entry(t, struct uprobe, rb_node); 1040891c3970SOleg Nesterov if (u->inode != inode || u->offset > max) 1041891c3970SOleg Nesterov break; 1042891c3970SOleg Nesterov list_add(&u->pending_list, head); 1043891c3970SOleg Nesterov atomic_inc(&u->ref); 1044891c3970SOleg Nesterov } 1045891c3970SOleg Nesterov } 10466f47caa0SOleg Nesterov spin_unlock(&uprobes_treelock); 1047a5f4374aSIngo Molnar } 1048a5f4374aSIngo Molnar 1049a5f4374aSIngo Molnar /* 10505e5be71aSOleg Nesterov * Called from mmap_region/vma_adjust with mm->mmap_sem acquired. 1051a5f4374aSIngo Molnar * 10525e5be71aSOleg Nesterov * Currently we ignore all errors and always return 0, the callers 10535e5be71aSOleg Nesterov * can't handle the failure anyway. 1054a5f4374aSIngo Molnar */ 1055a5f4374aSIngo Molnar int uprobe_mmap(struct vm_area_struct *vma) 1056a5f4374aSIngo Molnar { 1057a5f4374aSIngo Molnar struct list_head tmp_list; 1058665605a2SOleg Nesterov struct uprobe *uprobe, *u; 1059a5f4374aSIngo Molnar struct inode *inode; 1060a5f4374aSIngo Molnar 1061441f1eb7SOleg Nesterov if (no_uprobe_events() || !valid_vma(vma, true)) 1062a5f4374aSIngo Molnar return 0; 1063a5f4374aSIngo Molnar 1064f281769eSOleg Nesterov inode = file_inode(vma->vm_file); 1065a5f4374aSIngo Molnar if (!inode) 1066a5f4374aSIngo Molnar return 0; 1067a5f4374aSIngo Molnar 1068a5f4374aSIngo Molnar mutex_lock(uprobes_mmap_hash(inode)); 1069891c3970SOleg Nesterov build_probe_list(inode, vma, vma->vm_start, vma->vm_end, &tmp_list); 1070806a98bdSOleg Nesterov /* 1071806a98bdSOleg Nesterov * We can race with uprobe_unregister(), this uprobe can be already 1072806a98bdSOleg Nesterov * removed. But in this case filter_chain() must return false, all 1073806a98bdSOleg Nesterov * consumers have gone away. 1074806a98bdSOleg Nesterov */ 1075665605a2SOleg Nesterov list_for_each_entry_safe(uprobe, u, &tmp_list, pending_list) { 1076806a98bdSOleg Nesterov if (!fatal_signal_pending(current) && 10778a7f2fa0SOleg Nesterov filter_chain(uprobe, UPROBE_FILTER_MMAP, vma->vm_mm)) { 107857683f72SOleg Nesterov unsigned long vaddr = offset_to_vaddr(vma, uprobe->offset); 10795e5be71aSOleg Nesterov install_breakpoint(uprobe, vma->vm_mm, vma, vaddr); 1080a5f4374aSIngo Molnar } 1081a5f4374aSIngo Molnar put_uprobe(uprobe); 1082a5f4374aSIngo Molnar } 1083a5f4374aSIngo Molnar mutex_unlock(uprobes_mmap_hash(inode)); 1084a5f4374aSIngo Molnar 10855e5be71aSOleg Nesterov return 0; 1086a5f4374aSIngo Molnar } 1087a5f4374aSIngo Molnar 10889f68f672SOleg Nesterov static bool 10899f68f672SOleg Nesterov vma_has_uprobes(struct vm_area_struct *vma, unsigned long start, unsigned long end) 10909f68f672SOleg Nesterov { 10919f68f672SOleg Nesterov loff_t min, max; 10929f68f672SOleg Nesterov struct inode *inode; 10939f68f672SOleg Nesterov struct rb_node *n; 10949f68f672SOleg Nesterov 1095f281769eSOleg Nesterov inode = file_inode(vma->vm_file); 10969f68f672SOleg Nesterov 10979f68f672SOleg Nesterov min = vaddr_to_offset(vma, start); 10989f68f672SOleg Nesterov max = min + (end - start) - 1; 10999f68f672SOleg Nesterov 11009f68f672SOleg Nesterov spin_lock(&uprobes_treelock); 11019f68f672SOleg Nesterov n = find_node_in_range(inode, min, max); 11029f68f672SOleg Nesterov spin_unlock(&uprobes_treelock); 11039f68f672SOleg Nesterov 11049f68f672SOleg Nesterov return !!n; 11059f68f672SOleg Nesterov } 11069f68f672SOleg Nesterov 1107682968e0SSrikar Dronamraju /* 1108682968e0SSrikar Dronamraju * Called in context of a munmap of a vma. 1109682968e0SSrikar Dronamraju */ 1110cbc91f71SSrikar Dronamraju void uprobe_munmap(struct vm_area_struct *vma, unsigned long start, unsigned long end) 1111682968e0SSrikar Dronamraju { 1112441f1eb7SOleg Nesterov if (no_uprobe_events() || !valid_vma(vma, false)) 1113682968e0SSrikar Dronamraju return; 1114682968e0SSrikar Dronamraju 11152fd611a9SOleg Nesterov if (!atomic_read(&vma->vm_mm->mm_users)) /* called by mmput() ? */ 11162fd611a9SOleg Nesterov return; 11172fd611a9SOleg Nesterov 11189f68f672SOleg Nesterov if (!test_bit(MMF_HAS_UPROBES, &vma->vm_mm->flags) || 11199f68f672SOleg Nesterov test_bit(MMF_RECALC_UPROBES, &vma->vm_mm->flags)) 1120f8ac4ec9SOleg Nesterov return; 1121f8ac4ec9SOleg Nesterov 11229f68f672SOleg Nesterov if (vma_has_uprobes(vma, start, end)) 11239f68f672SOleg Nesterov set_bit(MMF_RECALC_UPROBES, &vma->vm_mm->flags); 1124682968e0SSrikar Dronamraju } 1125682968e0SSrikar Dronamraju 1126d4b3b638SSrikar Dronamraju /* Slot allocation for XOL */ 11276441ec8bSOleg Nesterov static int xol_add_vma(struct mm_struct *mm, struct xol_area *area) 1128d4b3b638SSrikar Dronamraju { 1129c8a82538SOleg Nesterov int ret = -EALREADY; 1130d4b3b638SSrikar Dronamraju 1131d4b3b638SSrikar Dronamraju down_write(&mm->mmap_sem); 1132d4b3b638SSrikar Dronamraju if (mm->uprobes_state.xol_area) 1133d4b3b638SSrikar Dronamraju goto fail; 1134d4b3b638SSrikar Dronamraju 1135af0d95afSOleg Nesterov if (!area->vaddr) { 1136d4b3b638SSrikar Dronamraju /* Try to map as high as possible, this is only a hint. */ 1137af0d95afSOleg Nesterov area->vaddr = get_unmapped_area(NULL, TASK_SIZE - PAGE_SIZE, 1138af0d95afSOleg Nesterov PAGE_SIZE, 0, 0); 1139d4b3b638SSrikar Dronamraju if (area->vaddr & ~PAGE_MASK) { 1140d4b3b638SSrikar Dronamraju ret = area->vaddr; 1141d4b3b638SSrikar Dronamraju goto fail; 1142d4b3b638SSrikar Dronamraju } 1143af0d95afSOleg Nesterov } 1144d4b3b638SSrikar Dronamraju 1145d4b3b638SSrikar Dronamraju ret = install_special_mapping(mm, area->vaddr, PAGE_SIZE, 1146d4b3b638SSrikar Dronamraju VM_EXEC|VM_MAYEXEC|VM_DONTCOPY|VM_IO, &area->page); 1147d4b3b638SSrikar Dronamraju if (ret) 1148d4b3b638SSrikar Dronamraju goto fail; 1149d4b3b638SSrikar Dronamraju 1150d4b3b638SSrikar Dronamraju smp_wmb(); /* pairs with get_xol_area() */ 1151d4b3b638SSrikar Dronamraju mm->uprobes_state.xol_area = area; 1152d4b3b638SSrikar Dronamraju fail: 1153d4b3b638SSrikar Dronamraju up_write(&mm->mmap_sem); 1154d4b3b638SSrikar Dronamraju 1155d4b3b638SSrikar Dronamraju return ret; 1156d4b3b638SSrikar Dronamraju } 1157d4b3b638SSrikar Dronamraju 1158af0d95afSOleg Nesterov static struct xol_area *__create_xol_area(unsigned long vaddr) 1159d4b3b638SSrikar Dronamraju { 11609b545df8SOleg Nesterov struct mm_struct *mm = current->mm; 1161e78aebfdSAnton Arapov uprobe_opcode_t insn = UPROBE_SWBP_INSN; 11626441ec8bSOleg Nesterov struct xol_area *area; 11639b545df8SOleg Nesterov 1164af0d95afSOleg Nesterov area = kmalloc(sizeof(*area), GFP_KERNEL); 1165d4b3b638SSrikar Dronamraju if (unlikely(!area)) 1166c8a82538SOleg Nesterov goto out; 1167d4b3b638SSrikar Dronamraju 1168d4b3b638SSrikar Dronamraju area->bitmap = kzalloc(BITS_TO_LONGS(UINSNS_PER_PAGE) * sizeof(long), GFP_KERNEL); 1169d4b3b638SSrikar Dronamraju if (!area->bitmap) 1170c8a82538SOleg Nesterov goto free_area; 1171c8a82538SOleg Nesterov 1172c8a82538SOleg Nesterov area->page = alloc_page(GFP_HIGHUSER); 1173c8a82538SOleg Nesterov if (!area->page) 1174c8a82538SOleg Nesterov goto free_bitmap; 1175d4b3b638SSrikar Dronamraju 1176af0d95afSOleg Nesterov area->vaddr = vaddr; 1177d4b3b638SSrikar Dronamraju init_waitqueue_head(&area->wq); 11786441ec8bSOleg Nesterov /* Reserve the 1st slot for get_trampoline_vaddr() */ 11796441ec8bSOleg Nesterov set_bit(0, area->bitmap); 11806441ec8bSOleg Nesterov atomic_set(&area->slot_count, 1); 11816441ec8bSOleg Nesterov copy_to_page(area->page, 0, &insn, UPROBE_SWBP_INSN_SIZE); 1182e78aebfdSAnton Arapov 11836441ec8bSOleg Nesterov if (!xol_add_vma(mm, area)) 1184d4b3b638SSrikar Dronamraju return area; 1185d4b3b638SSrikar Dronamraju 1186c8a82538SOleg Nesterov __free_page(area->page); 1187c8a82538SOleg Nesterov free_bitmap: 1188d4b3b638SSrikar Dronamraju kfree(area->bitmap); 1189c8a82538SOleg Nesterov free_area: 1190d4b3b638SSrikar Dronamraju kfree(area); 1191c8a82538SOleg Nesterov out: 11926441ec8bSOleg Nesterov return NULL; 11936441ec8bSOleg Nesterov } 11946441ec8bSOleg Nesterov 11956441ec8bSOleg Nesterov /* 11966441ec8bSOleg Nesterov * get_xol_area - Allocate process's xol_area if necessary. 11976441ec8bSOleg Nesterov * This area will be used for storing instructions for execution out of line. 11986441ec8bSOleg Nesterov * 11996441ec8bSOleg Nesterov * Returns the allocated area or NULL. 12006441ec8bSOleg Nesterov */ 12016441ec8bSOleg Nesterov static struct xol_area *get_xol_area(void) 12026441ec8bSOleg Nesterov { 12036441ec8bSOleg Nesterov struct mm_struct *mm = current->mm; 12046441ec8bSOleg Nesterov struct xol_area *area; 12056441ec8bSOleg Nesterov 12066441ec8bSOleg Nesterov if (!mm->uprobes_state.xol_area) 1207af0d95afSOleg Nesterov __create_xol_area(0); 12086441ec8bSOleg Nesterov 12099b545df8SOleg Nesterov area = mm->uprobes_state.xol_area; 12109b545df8SOleg Nesterov smp_read_barrier_depends(); /* pairs with wmb in xol_add_vma() */ 12119b545df8SOleg Nesterov return area; 1212d4b3b638SSrikar Dronamraju } 1213d4b3b638SSrikar Dronamraju 1214d4b3b638SSrikar Dronamraju /* 1215d4b3b638SSrikar Dronamraju * uprobe_clear_state - Free the area allocated for slots. 1216d4b3b638SSrikar Dronamraju */ 1217d4b3b638SSrikar Dronamraju void uprobe_clear_state(struct mm_struct *mm) 1218d4b3b638SSrikar Dronamraju { 1219d4b3b638SSrikar Dronamraju struct xol_area *area = mm->uprobes_state.xol_area; 1220d4b3b638SSrikar Dronamraju 1221d4b3b638SSrikar Dronamraju if (!area) 1222d4b3b638SSrikar Dronamraju return; 1223d4b3b638SSrikar Dronamraju 1224d4b3b638SSrikar Dronamraju put_page(area->page); 1225d4b3b638SSrikar Dronamraju kfree(area->bitmap); 1226d4b3b638SSrikar Dronamraju kfree(area); 1227d4b3b638SSrikar Dronamraju } 1228d4b3b638SSrikar Dronamraju 122932cdba1eSOleg Nesterov void uprobe_start_dup_mmap(void) 123032cdba1eSOleg Nesterov { 123132cdba1eSOleg Nesterov percpu_down_read(&dup_mmap_sem); 123232cdba1eSOleg Nesterov } 123332cdba1eSOleg Nesterov 123432cdba1eSOleg Nesterov void uprobe_end_dup_mmap(void) 123532cdba1eSOleg Nesterov { 123632cdba1eSOleg Nesterov percpu_up_read(&dup_mmap_sem); 123732cdba1eSOleg Nesterov } 123832cdba1eSOleg Nesterov 1239f8ac4ec9SOleg Nesterov void uprobe_dup_mmap(struct mm_struct *oldmm, struct mm_struct *newmm) 1240f8ac4ec9SOleg Nesterov { 124161559a81SOleg Nesterov newmm->uprobes_state.xol_area = NULL; 124261559a81SOleg Nesterov 12439f68f672SOleg Nesterov if (test_bit(MMF_HAS_UPROBES, &oldmm->flags)) { 1244f8ac4ec9SOleg Nesterov set_bit(MMF_HAS_UPROBES, &newmm->flags); 12459f68f672SOleg Nesterov /* unconditionally, dup_mmap() skips VM_DONTCOPY vmas */ 12469f68f672SOleg Nesterov set_bit(MMF_RECALC_UPROBES, &newmm->flags); 12479f68f672SOleg Nesterov } 1248f8ac4ec9SOleg Nesterov } 1249f8ac4ec9SOleg Nesterov 1250d4b3b638SSrikar Dronamraju /* 1251d4b3b638SSrikar Dronamraju * - search for a free slot. 1252d4b3b638SSrikar Dronamraju */ 1253d4b3b638SSrikar Dronamraju static unsigned long xol_take_insn_slot(struct xol_area *area) 1254d4b3b638SSrikar Dronamraju { 1255d4b3b638SSrikar Dronamraju unsigned long slot_addr; 1256d4b3b638SSrikar Dronamraju int slot_nr; 1257d4b3b638SSrikar Dronamraju 1258d4b3b638SSrikar Dronamraju do { 1259d4b3b638SSrikar Dronamraju slot_nr = find_first_zero_bit(area->bitmap, UINSNS_PER_PAGE); 1260d4b3b638SSrikar Dronamraju if (slot_nr < UINSNS_PER_PAGE) { 1261d4b3b638SSrikar Dronamraju if (!test_and_set_bit(slot_nr, area->bitmap)) 1262d4b3b638SSrikar Dronamraju break; 1263d4b3b638SSrikar Dronamraju 1264d4b3b638SSrikar Dronamraju slot_nr = UINSNS_PER_PAGE; 1265d4b3b638SSrikar Dronamraju continue; 1266d4b3b638SSrikar Dronamraju } 1267d4b3b638SSrikar Dronamraju wait_event(area->wq, (atomic_read(&area->slot_count) < UINSNS_PER_PAGE)); 1268d4b3b638SSrikar Dronamraju } while (slot_nr >= UINSNS_PER_PAGE); 1269d4b3b638SSrikar Dronamraju 1270d4b3b638SSrikar Dronamraju slot_addr = area->vaddr + (slot_nr * UPROBE_XOL_SLOT_BYTES); 1271d4b3b638SSrikar Dronamraju atomic_inc(&area->slot_count); 1272d4b3b638SSrikar Dronamraju 1273d4b3b638SSrikar Dronamraju return slot_addr; 1274d4b3b638SSrikar Dronamraju } 1275d4b3b638SSrikar Dronamraju 1276d4b3b638SSrikar Dronamraju /* 1277a6cb3f6dSOleg Nesterov * xol_get_insn_slot - allocate a slot for xol. 1278d4b3b638SSrikar Dronamraju * Returns the allocated slot address or 0. 1279d4b3b638SSrikar Dronamraju */ 1280a6cb3f6dSOleg Nesterov static unsigned long xol_get_insn_slot(struct uprobe *uprobe) 1281d4b3b638SSrikar Dronamraju { 1282d4b3b638SSrikar Dronamraju struct xol_area *area; 1283a6cb3f6dSOleg Nesterov unsigned long xol_vaddr; 1284d4b3b638SSrikar Dronamraju 12859b545df8SOleg Nesterov area = get_xol_area(); 1286d4b3b638SSrikar Dronamraju if (!area) 1287d4b3b638SSrikar Dronamraju return 0; 1288d4b3b638SSrikar Dronamraju 1289a6cb3f6dSOleg Nesterov xol_vaddr = xol_take_insn_slot(area); 1290a6cb3f6dSOleg Nesterov if (unlikely(!xol_vaddr)) 1291d4b3b638SSrikar Dronamraju return 0; 1292d4b3b638SSrikar Dronamraju 1293a6cb3f6dSOleg Nesterov /* Initialize the slot */ 12948a8de66cSOleg Nesterov copy_to_page(area->page, xol_vaddr, 1295803200e2SOleg Nesterov &uprobe->arch.ixol, sizeof(uprobe->arch.ixol)); 129665b6ecc0SRabin Vincent /* 129765b6ecc0SRabin Vincent * We probably need flush_icache_user_range() but it needs vma. 129865b6ecc0SRabin Vincent * This should work on supported architectures too. 129965b6ecc0SRabin Vincent */ 130065b6ecc0SRabin Vincent flush_dcache_page(area->page); 1301d4b3b638SSrikar Dronamraju 1302a6cb3f6dSOleg Nesterov return xol_vaddr; 1303d4b3b638SSrikar Dronamraju } 1304d4b3b638SSrikar Dronamraju 1305d4b3b638SSrikar Dronamraju /* 1306d4b3b638SSrikar Dronamraju * xol_free_insn_slot - If slot was earlier allocated by 1307d4b3b638SSrikar Dronamraju * @xol_get_insn_slot(), make the slot available for 1308d4b3b638SSrikar Dronamraju * subsequent requests. 1309d4b3b638SSrikar Dronamraju */ 1310d4b3b638SSrikar Dronamraju static void xol_free_insn_slot(struct task_struct *tsk) 1311d4b3b638SSrikar Dronamraju { 1312d4b3b638SSrikar Dronamraju struct xol_area *area; 1313d4b3b638SSrikar Dronamraju unsigned long vma_end; 1314d4b3b638SSrikar Dronamraju unsigned long slot_addr; 1315d4b3b638SSrikar Dronamraju 1316d4b3b638SSrikar Dronamraju if (!tsk->mm || !tsk->mm->uprobes_state.xol_area || !tsk->utask) 1317d4b3b638SSrikar Dronamraju return; 1318d4b3b638SSrikar Dronamraju 1319d4b3b638SSrikar Dronamraju slot_addr = tsk->utask->xol_vaddr; 1320af4355e9SOleg Nesterov if (unlikely(!slot_addr)) 1321d4b3b638SSrikar Dronamraju return; 1322d4b3b638SSrikar Dronamraju 1323d4b3b638SSrikar Dronamraju area = tsk->mm->uprobes_state.xol_area; 1324d4b3b638SSrikar Dronamraju vma_end = area->vaddr + PAGE_SIZE; 1325d4b3b638SSrikar Dronamraju if (area->vaddr <= slot_addr && slot_addr < vma_end) { 1326d4b3b638SSrikar Dronamraju unsigned long offset; 1327d4b3b638SSrikar Dronamraju int slot_nr; 1328d4b3b638SSrikar Dronamraju 1329d4b3b638SSrikar Dronamraju offset = slot_addr - area->vaddr; 1330d4b3b638SSrikar Dronamraju slot_nr = offset / UPROBE_XOL_SLOT_BYTES; 1331d4b3b638SSrikar Dronamraju if (slot_nr >= UINSNS_PER_PAGE) 1332d4b3b638SSrikar Dronamraju return; 1333d4b3b638SSrikar Dronamraju 1334d4b3b638SSrikar Dronamraju clear_bit(slot_nr, area->bitmap); 1335d4b3b638SSrikar Dronamraju atomic_dec(&area->slot_count); 1336d4b3b638SSrikar Dronamraju if (waitqueue_active(&area->wq)) 1337d4b3b638SSrikar Dronamraju wake_up(&area->wq); 1338d4b3b638SSrikar Dronamraju 1339d4b3b638SSrikar Dronamraju tsk->utask->xol_vaddr = 0; 1340d4b3b638SSrikar Dronamraju } 1341d4b3b638SSrikar Dronamraju } 1342d4b3b638SSrikar Dronamraju 13430326f5a9SSrikar Dronamraju /** 13440326f5a9SSrikar Dronamraju * uprobe_get_swbp_addr - compute address of swbp given post-swbp regs 13450326f5a9SSrikar Dronamraju * @regs: Reflects the saved state of the task after it has hit a breakpoint 13460326f5a9SSrikar Dronamraju * instruction. 13470326f5a9SSrikar Dronamraju * Return the address of the breakpoint instruction. 13480326f5a9SSrikar Dronamraju */ 13490326f5a9SSrikar Dronamraju unsigned long __weak uprobe_get_swbp_addr(struct pt_regs *regs) 13500326f5a9SSrikar Dronamraju { 13510326f5a9SSrikar Dronamraju return instruction_pointer(regs) - UPROBE_SWBP_INSN_SIZE; 13520326f5a9SSrikar Dronamraju } 13530326f5a9SSrikar Dronamraju 1354b02ef20aSOleg Nesterov unsigned long uprobe_get_trap_addr(struct pt_regs *regs) 1355b02ef20aSOleg Nesterov { 1356b02ef20aSOleg Nesterov struct uprobe_task *utask = current->utask; 1357b02ef20aSOleg Nesterov 1358b02ef20aSOleg Nesterov if (unlikely(utask && utask->active_uprobe)) 1359b02ef20aSOleg Nesterov return utask->vaddr; 1360b02ef20aSOleg Nesterov 1361b02ef20aSOleg Nesterov return instruction_pointer(regs); 1362b02ef20aSOleg Nesterov } 1363b02ef20aSOleg Nesterov 13640326f5a9SSrikar Dronamraju /* 13650326f5a9SSrikar Dronamraju * Called with no locks held. 13660326f5a9SSrikar Dronamraju * Called in context of a exiting or a exec-ing thread. 13670326f5a9SSrikar Dronamraju */ 13680326f5a9SSrikar Dronamraju void uprobe_free_utask(struct task_struct *t) 13690326f5a9SSrikar Dronamraju { 13700326f5a9SSrikar Dronamraju struct uprobe_task *utask = t->utask; 13710dfd0eb8SAnton Arapov struct return_instance *ri, *tmp; 13720326f5a9SSrikar Dronamraju 13730326f5a9SSrikar Dronamraju if (!utask) 13740326f5a9SSrikar Dronamraju return; 13750326f5a9SSrikar Dronamraju 13760326f5a9SSrikar Dronamraju if (utask->active_uprobe) 13770326f5a9SSrikar Dronamraju put_uprobe(utask->active_uprobe); 13780326f5a9SSrikar Dronamraju 13790dfd0eb8SAnton Arapov ri = utask->return_instances; 13800dfd0eb8SAnton Arapov while (ri) { 13810dfd0eb8SAnton Arapov tmp = ri; 13820dfd0eb8SAnton Arapov ri = ri->next; 13830dfd0eb8SAnton Arapov 13840dfd0eb8SAnton Arapov put_uprobe(tmp->uprobe); 13850dfd0eb8SAnton Arapov kfree(tmp); 13860dfd0eb8SAnton Arapov } 13870dfd0eb8SAnton Arapov 1388d4b3b638SSrikar Dronamraju xol_free_insn_slot(t); 13890326f5a9SSrikar Dronamraju kfree(utask); 13900326f5a9SSrikar Dronamraju t->utask = NULL; 13910326f5a9SSrikar Dronamraju } 13920326f5a9SSrikar Dronamraju 13930326f5a9SSrikar Dronamraju /* 13945a2df662SOleg Nesterov * Allocate a uprobe_task object for the task if if necessary. 13955a2df662SOleg Nesterov * Called when the thread hits a breakpoint. 13960326f5a9SSrikar Dronamraju * 13970326f5a9SSrikar Dronamraju * Returns: 13980326f5a9SSrikar Dronamraju * - pointer to new uprobe_task on success 13990326f5a9SSrikar Dronamraju * - NULL otherwise 14000326f5a9SSrikar Dronamraju */ 14015a2df662SOleg Nesterov static struct uprobe_task *get_utask(void) 14020326f5a9SSrikar Dronamraju { 14035a2df662SOleg Nesterov if (!current->utask) 14045a2df662SOleg Nesterov current->utask = kzalloc(sizeof(struct uprobe_task), GFP_KERNEL); 14055a2df662SOleg Nesterov return current->utask; 14060326f5a9SSrikar Dronamraju } 14070326f5a9SSrikar Dronamraju 1408248d3a7bSOleg Nesterov static int dup_utask(struct task_struct *t, struct uprobe_task *o_utask) 1409248d3a7bSOleg Nesterov { 1410248d3a7bSOleg Nesterov struct uprobe_task *n_utask; 1411248d3a7bSOleg Nesterov struct return_instance **p, *o, *n; 1412248d3a7bSOleg Nesterov 1413248d3a7bSOleg Nesterov n_utask = kzalloc(sizeof(struct uprobe_task), GFP_KERNEL); 1414248d3a7bSOleg Nesterov if (!n_utask) 1415248d3a7bSOleg Nesterov return -ENOMEM; 1416248d3a7bSOleg Nesterov t->utask = n_utask; 1417248d3a7bSOleg Nesterov 1418248d3a7bSOleg Nesterov p = &n_utask->return_instances; 1419248d3a7bSOleg Nesterov for (o = o_utask->return_instances; o; o = o->next) { 1420248d3a7bSOleg Nesterov n = kmalloc(sizeof(struct return_instance), GFP_KERNEL); 1421248d3a7bSOleg Nesterov if (!n) 1422248d3a7bSOleg Nesterov return -ENOMEM; 1423248d3a7bSOleg Nesterov 1424248d3a7bSOleg Nesterov *n = *o; 1425248d3a7bSOleg Nesterov atomic_inc(&n->uprobe->ref); 1426248d3a7bSOleg Nesterov n->next = NULL; 1427248d3a7bSOleg Nesterov 1428248d3a7bSOleg Nesterov *p = n; 1429248d3a7bSOleg Nesterov p = &n->next; 1430248d3a7bSOleg Nesterov n_utask->depth++; 1431248d3a7bSOleg Nesterov } 1432248d3a7bSOleg Nesterov 1433248d3a7bSOleg Nesterov return 0; 1434248d3a7bSOleg Nesterov } 1435248d3a7bSOleg Nesterov 1436248d3a7bSOleg Nesterov static void uprobe_warn(struct task_struct *t, const char *msg) 1437248d3a7bSOleg Nesterov { 1438248d3a7bSOleg Nesterov pr_warn("uprobe: %s:%d failed to %s\n", 1439248d3a7bSOleg Nesterov current->comm, current->pid, msg); 1440248d3a7bSOleg Nesterov } 1441248d3a7bSOleg Nesterov 1442aa59c53fSOleg Nesterov static void dup_xol_work(struct callback_head *work) 1443aa59c53fSOleg Nesterov { 1444aa59c53fSOleg Nesterov if (current->flags & PF_EXITING) 1445aa59c53fSOleg Nesterov return; 1446aa59c53fSOleg Nesterov 144732473431SOleg Nesterov if (!__create_xol_area(current->utask->dup_xol_addr)) 1448aa59c53fSOleg Nesterov uprobe_warn(current, "dup xol area"); 1449aa59c53fSOleg Nesterov } 1450aa59c53fSOleg Nesterov 1451e78aebfdSAnton Arapov /* 1452b68e0749SOleg Nesterov * Called in context of a new clone/fork from copy_process. 1453b68e0749SOleg Nesterov */ 14543ab67966SOleg Nesterov void uprobe_copy_process(struct task_struct *t, unsigned long flags) 1455b68e0749SOleg Nesterov { 1456248d3a7bSOleg Nesterov struct uprobe_task *utask = current->utask; 1457248d3a7bSOleg Nesterov struct mm_struct *mm = current->mm; 1458aa59c53fSOleg Nesterov struct xol_area *area; 1459248d3a7bSOleg Nesterov 1460b68e0749SOleg Nesterov t->utask = NULL; 1461248d3a7bSOleg Nesterov 14623ab67966SOleg Nesterov if (!utask || !utask->return_instances) 14633ab67966SOleg Nesterov return; 14643ab67966SOleg Nesterov 14653ab67966SOleg Nesterov if (mm == t->mm && !(flags & CLONE_VFORK)) 1466248d3a7bSOleg Nesterov return; 1467248d3a7bSOleg Nesterov 1468248d3a7bSOleg Nesterov if (dup_utask(t, utask)) 1469248d3a7bSOleg Nesterov return uprobe_warn(t, "dup ret instances"); 1470aa59c53fSOleg Nesterov 1471aa59c53fSOleg Nesterov /* The task can fork() after dup_xol_work() fails */ 1472aa59c53fSOleg Nesterov area = mm->uprobes_state.xol_area; 1473aa59c53fSOleg Nesterov if (!area) 1474aa59c53fSOleg Nesterov return uprobe_warn(t, "dup xol area"); 1475aa59c53fSOleg Nesterov 14763ab67966SOleg Nesterov if (mm == t->mm) 14773ab67966SOleg Nesterov return; 14783ab67966SOleg Nesterov 147932473431SOleg Nesterov t->utask->dup_xol_addr = area->vaddr; 148032473431SOleg Nesterov init_task_work(&t->utask->dup_xol_work, dup_xol_work); 148132473431SOleg Nesterov task_work_add(t, &t->utask->dup_xol_work, true); 1482b68e0749SOleg Nesterov } 1483b68e0749SOleg Nesterov 1484b68e0749SOleg Nesterov /* 1485e78aebfdSAnton Arapov * Current area->vaddr notion assume the trampoline address is always 1486e78aebfdSAnton Arapov * equal area->vaddr. 1487e78aebfdSAnton Arapov * 1488e78aebfdSAnton Arapov * Returns -1 in case the xol_area is not allocated. 1489e78aebfdSAnton Arapov */ 1490e78aebfdSAnton Arapov static unsigned long get_trampoline_vaddr(void) 1491e78aebfdSAnton Arapov { 1492e78aebfdSAnton Arapov struct xol_area *area; 1493e78aebfdSAnton Arapov unsigned long trampoline_vaddr = -1; 1494e78aebfdSAnton Arapov 1495e78aebfdSAnton Arapov area = current->mm->uprobes_state.xol_area; 1496e78aebfdSAnton Arapov smp_read_barrier_depends(); 1497e78aebfdSAnton Arapov if (area) 1498e78aebfdSAnton Arapov trampoline_vaddr = area->vaddr; 1499e78aebfdSAnton Arapov 1500e78aebfdSAnton Arapov return trampoline_vaddr; 1501e78aebfdSAnton Arapov } 1502e78aebfdSAnton Arapov 15030dfd0eb8SAnton Arapov static void prepare_uretprobe(struct uprobe *uprobe, struct pt_regs *regs) 15040dfd0eb8SAnton Arapov { 15050dfd0eb8SAnton Arapov struct return_instance *ri; 15060dfd0eb8SAnton Arapov struct uprobe_task *utask; 15070dfd0eb8SAnton Arapov unsigned long orig_ret_vaddr, trampoline_vaddr; 15080dfd0eb8SAnton Arapov bool chained = false; 15090dfd0eb8SAnton Arapov 15100dfd0eb8SAnton Arapov if (!get_xol_area()) 15110dfd0eb8SAnton Arapov return; 15120dfd0eb8SAnton Arapov 15130dfd0eb8SAnton Arapov utask = get_utask(); 15140dfd0eb8SAnton Arapov if (!utask) 15150dfd0eb8SAnton Arapov return; 15160dfd0eb8SAnton Arapov 1517ded49c55SAnton Arapov if (utask->depth >= MAX_URETPROBE_DEPTH) { 1518ded49c55SAnton Arapov printk_ratelimited(KERN_INFO "uprobe: omit uretprobe due to" 1519ded49c55SAnton Arapov " nestedness limit pid/tgid=%d/%d\n", 1520ded49c55SAnton Arapov current->pid, current->tgid); 1521ded49c55SAnton Arapov return; 1522ded49c55SAnton Arapov } 1523ded49c55SAnton Arapov 15240dfd0eb8SAnton Arapov ri = kzalloc(sizeof(struct return_instance), GFP_KERNEL); 15250dfd0eb8SAnton Arapov if (!ri) 15260dfd0eb8SAnton Arapov goto fail; 15270dfd0eb8SAnton Arapov 15280dfd0eb8SAnton Arapov trampoline_vaddr = get_trampoline_vaddr(); 15290dfd0eb8SAnton Arapov orig_ret_vaddr = arch_uretprobe_hijack_return_addr(trampoline_vaddr, regs); 15300dfd0eb8SAnton Arapov if (orig_ret_vaddr == -1) 15310dfd0eb8SAnton Arapov goto fail; 15320dfd0eb8SAnton Arapov 15330dfd0eb8SAnton Arapov /* 15340dfd0eb8SAnton Arapov * We don't want to keep trampoline address in stack, rather keep the 15350dfd0eb8SAnton Arapov * original return address of first caller thru all the consequent 15360dfd0eb8SAnton Arapov * instances. This also makes breakpoint unwrapping easier. 15370dfd0eb8SAnton Arapov */ 15380dfd0eb8SAnton Arapov if (orig_ret_vaddr == trampoline_vaddr) { 15390dfd0eb8SAnton Arapov if (!utask->return_instances) { 15400dfd0eb8SAnton Arapov /* 15410dfd0eb8SAnton Arapov * This situation is not possible. Likely we have an 15420dfd0eb8SAnton Arapov * attack from user-space. 15430dfd0eb8SAnton Arapov */ 15440dfd0eb8SAnton Arapov pr_warn("uprobe: unable to set uretprobe pid/tgid=%d/%d\n", 15450dfd0eb8SAnton Arapov current->pid, current->tgid); 15460dfd0eb8SAnton Arapov goto fail; 15470dfd0eb8SAnton Arapov } 15480dfd0eb8SAnton Arapov 15490dfd0eb8SAnton Arapov chained = true; 15500dfd0eb8SAnton Arapov orig_ret_vaddr = utask->return_instances->orig_ret_vaddr; 15510dfd0eb8SAnton Arapov } 15520dfd0eb8SAnton Arapov 15530dfd0eb8SAnton Arapov atomic_inc(&uprobe->ref); 15540dfd0eb8SAnton Arapov ri->uprobe = uprobe; 15550dfd0eb8SAnton Arapov ri->func = instruction_pointer(regs); 15560dfd0eb8SAnton Arapov ri->orig_ret_vaddr = orig_ret_vaddr; 15570dfd0eb8SAnton Arapov ri->chained = chained; 15580dfd0eb8SAnton Arapov 1559ded49c55SAnton Arapov utask->depth++; 1560ded49c55SAnton Arapov 15610dfd0eb8SAnton Arapov /* add instance to the stack */ 15620dfd0eb8SAnton Arapov ri->next = utask->return_instances; 15630dfd0eb8SAnton Arapov utask->return_instances = ri; 15640dfd0eb8SAnton Arapov 15650dfd0eb8SAnton Arapov return; 15660dfd0eb8SAnton Arapov 15670dfd0eb8SAnton Arapov fail: 15680dfd0eb8SAnton Arapov kfree(ri); 15690dfd0eb8SAnton Arapov } 15700dfd0eb8SAnton Arapov 15710326f5a9SSrikar Dronamraju /* Prepare to single-step probed instruction out of line. */ 15720326f5a9SSrikar Dronamraju static int 1573a6cb3f6dSOleg Nesterov pre_ssout(struct uprobe *uprobe, struct pt_regs *regs, unsigned long bp_vaddr) 15740326f5a9SSrikar Dronamraju { 1575a6cb3f6dSOleg Nesterov struct uprobe_task *utask; 1576a6cb3f6dSOleg Nesterov unsigned long xol_vaddr; 1577aba51024SOleg Nesterov int err; 1578d4b3b638SSrikar Dronamraju 1579608e7427SOleg Nesterov utask = get_utask(); 1580608e7427SOleg Nesterov if (!utask) 1581608e7427SOleg Nesterov return -ENOMEM; 1582a6cb3f6dSOleg Nesterov 1583a6cb3f6dSOleg Nesterov xol_vaddr = xol_get_insn_slot(uprobe); 1584a6cb3f6dSOleg Nesterov if (!xol_vaddr) 1585a6cb3f6dSOleg Nesterov return -ENOMEM; 1586a6cb3f6dSOleg Nesterov 1587a6cb3f6dSOleg Nesterov utask->xol_vaddr = xol_vaddr; 1588a6cb3f6dSOleg Nesterov utask->vaddr = bp_vaddr; 1589a6cb3f6dSOleg Nesterov 1590aba51024SOleg Nesterov err = arch_uprobe_pre_xol(&uprobe->arch, regs); 1591aba51024SOleg Nesterov if (unlikely(err)) { 1592aba51024SOleg Nesterov xol_free_insn_slot(current); 1593aba51024SOleg Nesterov return err; 1594aba51024SOleg Nesterov } 1595aba51024SOleg Nesterov 1596608e7427SOleg Nesterov utask->active_uprobe = uprobe; 1597608e7427SOleg Nesterov utask->state = UTASK_SSTEP; 1598aba51024SOleg Nesterov return 0; 15990326f5a9SSrikar Dronamraju } 16000326f5a9SSrikar Dronamraju 16010326f5a9SSrikar Dronamraju /* 16020326f5a9SSrikar Dronamraju * If we are singlestepping, then ensure this thread is not connected to 16030326f5a9SSrikar Dronamraju * non-fatal signals until completion of singlestep. When xol insn itself 16040326f5a9SSrikar Dronamraju * triggers the signal, restart the original insn even if the task is 16050326f5a9SSrikar Dronamraju * already SIGKILL'ed (since coredump should report the correct ip). This 16060326f5a9SSrikar Dronamraju * is even more important if the task has a handler for SIGSEGV/etc, The 16070326f5a9SSrikar Dronamraju * _same_ instruction should be repeated again after return from the signal 16080326f5a9SSrikar Dronamraju * handler, and SSTEP can never finish in this case. 16090326f5a9SSrikar Dronamraju */ 16100326f5a9SSrikar Dronamraju bool uprobe_deny_signal(void) 16110326f5a9SSrikar Dronamraju { 16120326f5a9SSrikar Dronamraju struct task_struct *t = current; 16130326f5a9SSrikar Dronamraju struct uprobe_task *utask = t->utask; 16140326f5a9SSrikar Dronamraju 16150326f5a9SSrikar Dronamraju if (likely(!utask || !utask->active_uprobe)) 16160326f5a9SSrikar Dronamraju return false; 16170326f5a9SSrikar Dronamraju 16180326f5a9SSrikar Dronamraju WARN_ON_ONCE(utask->state != UTASK_SSTEP); 16190326f5a9SSrikar Dronamraju 16200326f5a9SSrikar Dronamraju if (signal_pending(t)) { 16210326f5a9SSrikar Dronamraju spin_lock_irq(&t->sighand->siglock); 16220326f5a9SSrikar Dronamraju clear_tsk_thread_flag(t, TIF_SIGPENDING); 16230326f5a9SSrikar Dronamraju spin_unlock_irq(&t->sighand->siglock); 16240326f5a9SSrikar Dronamraju 16250326f5a9SSrikar Dronamraju if (__fatal_signal_pending(t) || arch_uprobe_xol_was_trapped(t)) { 16260326f5a9SSrikar Dronamraju utask->state = UTASK_SSTEP_TRAPPED; 16270326f5a9SSrikar Dronamraju set_tsk_thread_flag(t, TIF_UPROBE); 16280326f5a9SSrikar Dronamraju set_tsk_thread_flag(t, TIF_NOTIFY_RESUME); 16290326f5a9SSrikar Dronamraju } 16300326f5a9SSrikar Dronamraju } 16310326f5a9SSrikar Dronamraju 16320326f5a9SSrikar Dronamraju return true; 16330326f5a9SSrikar Dronamraju } 16340326f5a9SSrikar Dronamraju 1635499a4f3eSOleg Nesterov static void mmf_recalc_uprobes(struct mm_struct *mm) 1636499a4f3eSOleg Nesterov { 1637499a4f3eSOleg Nesterov struct vm_area_struct *vma; 1638499a4f3eSOleg Nesterov 1639499a4f3eSOleg Nesterov for (vma = mm->mmap; vma; vma = vma->vm_next) { 1640499a4f3eSOleg Nesterov if (!valid_vma(vma, false)) 1641499a4f3eSOleg Nesterov continue; 1642499a4f3eSOleg Nesterov /* 1643499a4f3eSOleg Nesterov * This is not strictly accurate, we can race with 1644499a4f3eSOleg Nesterov * uprobe_unregister() and see the already removed 1645499a4f3eSOleg Nesterov * uprobe if delete_uprobe() was not yet called. 164663633cbfSOleg Nesterov * Or this uprobe can be filtered out. 1647499a4f3eSOleg Nesterov */ 1648499a4f3eSOleg Nesterov if (vma_has_uprobes(vma, vma->vm_start, vma->vm_end)) 1649499a4f3eSOleg Nesterov return; 1650499a4f3eSOleg Nesterov } 1651499a4f3eSOleg Nesterov 1652499a4f3eSOleg Nesterov clear_bit(MMF_HAS_UPROBES, &mm->flags); 1653499a4f3eSOleg Nesterov } 1654499a4f3eSOleg Nesterov 16550908ad6eSAnanth N Mavinakayanahalli static int is_trap_at_addr(struct mm_struct *mm, unsigned long vaddr) 1656ec75fba9SOleg Nesterov { 1657ec75fba9SOleg Nesterov struct page *page; 1658ec75fba9SOleg Nesterov uprobe_opcode_t opcode; 1659ec75fba9SOleg Nesterov int result; 1660ec75fba9SOleg Nesterov 1661ec75fba9SOleg Nesterov pagefault_disable(); 1662ec75fba9SOleg Nesterov result = __copy_from_user_inatomic(&opcode, (void __user*)vaddr, 1663ec75fba9SOleg Nesterov sizeof(opcode)); 1664ec75fba9SOleg Nesterov pagefault_enable(); 1665ec75fba9SOleg Nesterov 1666ec75fba9SOleg Nesterov if (likely(result == 0)) 1667ec75fba9SOleg Nesterov goto out; 1668ec75fba9SOleg Nesterov 1669ec75fba9SOleg Nesterov result = get_user_pages(NULL, mm, vaddr, 1, 0, 1, &page, NULL); 1670ec75fba9SOleg Nesterov if (result < 0) 1671ec75fba9SOleg Nesterov return result; 1672ec75fba9SOleg Nesterov 1673ab0d805cSOleg Nesterov copy_from_page(page, vaddr, &opcode, UPROBE_SWBP_INSN_SIZE); 1674ec75fba9SOleg Nesterov put_page(page); 1675ec75fba9SOleg Nesterov out: 16760908ad6eSAnanth N Mavinakayanahalli /* This needs to return true for any variant of the trap insn */ 16770908ad6eSAnanth N Mavinakayanahalli return is_trap_insn(&opcode); 1678ec75fba9SOleg Nesterov } 1679ec75fba9SOleg Nesterov 1680d790d346SOleg Nesterov static struct uprobe *find_active_uprobe(unsigned long bp_vaddr, int *is_swbp) 16810326f5a9SSrikar Dronamraju { 16823a9ea052SOleg Nesterov struct mm_struct *mm = current->mm; 16833a9ea052SOleg Nesterov struct uprobe *uprobe = NULL; 16840326f5a9SSrikar Dronamraju struct vm_area_struct *vma; 16850326f5a9SSrikar Dronamraju 16860326f5a9SSrikar Dronamraju down_read(&mm->mmap_sem); 16870326f5a9SSrikar Dronamraju vma = find_vma(mm, bp_vaddr); 16883a9ea052SOleg Nesterov if (vma && vma->vm_start <= bp_vaddr) { 16893a9ea052SOleg Nesterov if (valid_vma(vma, false)) { 1690f281769eSOleg Nesterov struct inode *inode = file_inode(vma->vm_file); 1691cb113b47SOleg Nesterov loff_t offset = vaddr_to_offset(vma, bp_vaddr); 16920326f5a9SSrikar Dronamraju 16930326f5a9SSrikar Dronamraju uprobe = find_uprobe(inode, offset); 16940326f5a9SSrikar Dronamraju } 1695d790d346SOleg Nesterov 1696d790d346SOleg Nesterov if (!uprobe) 16970908ad6eSAnanth N Mavinakayanahalli *is_swbp = is_trap_at_addr(mm, bp_vaddr); 1698d790d346SOleg Nesterov } else { 1699d790d346SOleg Nesterov *is_swbp = -EFAULT; 17003a9ea052SOleg Nesterov } 1701499a4f3eSOleg Nesterov 1702499a4f3eSOleg Nesterov if (!uprobe && test_and_clear_bit(MMF_RECALC_UPROBES, &mm->flags)) 1703499a4f3eSOleg Nesterov mmf_recalc_uprobes(mm); 17040326f5a9SSrikar Dronamraju up_read(&mm->mmap_sem); 17050326f5a9SSrikar Dronamraju 17063a9ea052SOleg Nesterov return uprobe; 17073a9ea052SOleg Nesterov } 17083a9ea052SOleg Nesterov 1709da1816b1SOleg Nesterov static void handler_chain(struct uprobe *uprobe, struct pt_regs *regs) 1710da1816b1SOleg Nesterov { 1711da1816b1SOleg Nesterov struct uprobe_consumer *uc; 1712da1816b1SOleg Nesterov int remove = UPROBE_HANDLER_REMOVE; 17130dfd0eb8SAnton Arapov bool need_prep = false; /* prepare return uprobe, when needed */ 1714da1816b1SOleg Nesterov 1715da1816b1SOleg Nesterov down_read(&uprobe->register_rwsem); 1716da1816b1SOleg Nesterov for (uc = uprobe->consumers; uc; uc = uc->next) { 1717ea024870SAnton Arapov int rc = 0; 1718da1816b1SOleg Nesterov 1719ea024870SAnton Arapov if (uc->handler) { 1720ea024870SAnton Arapov rc = uc->handler(uc, regs); 1721da1816b1SOleg Nesterov WARN(rc & ~UPROBE_HANDLER_MASK, 1722da1816b1SOleg Nesterov "bad rc=0x%x from %pf()\n", rc, uc->handler); 1723ea024870SAnton Arapov } 17240dfd0eb8SAnton Arapov 17250dfd0eb8SAnton Arapov if (uc->ret_handler) 17260dfd0eb8SAnton Arapov need_prep = true; 17270dfd0eb8SAnton Arapov 1728da1816b1SOleg Nesterov remove &= rc; 1729da1816b1SOleg Nesterov } 1730da1816b1SOleg Nesterov 17310dfd0eb8SAnton Arapov if (need_prep && !remove) 17320dfd0eb8SAnton Arapov prepare_uretprobe(uprobe, regs); /* put bp at return */ 17330dfd0eb8SAnton Arapov 1734da1816b1SOleg Nesterov if (remove && uprobe->consumers) { 1735da1816b1SOleg Nesterov WARN_ON(!uprobe_is_active(uprobe)); 1736da1816b1SOleg Nesterov unapply_uprobe(uprobe, current->mm); 1737da1816b1SOleg Nesterov } 1738da1816b1SOleg Nesterov up_read(&uprobe->register_rwsem); 1739da1816b1SOleg Nesterov } 1740da1816b1SOleg Nesterov 1741fec8898dSAnton Arapov static void 1742fec8898dSAnton Arapov handle_uretprobe_chain(struct return_instance *ri, struct pt_regs *regs) 1743fec8898dSAnton Arapov { 1744fec8898dSAnton Arapov struct uprobe *uprobe = ri->uprobe; 1745fec8898dSAnton Arapov struct uprobe_consumer *uc; 1746fec8898dSAnton Arapov 1747fec8898dSAnton Arapov down_read(&uprobe->register_rwsem); 1748fec8898dSAnton Arapov for (uc = uprobe->consumers; uc; uc = uc->next) { 1749fec8898dSAnton Arapov if (uc->ret_handler) 1750fec8898dSAnton Arapov uc->ret_handler(uc, ri->func, regs); 1751fec8898dSAnton Arapov } 1752fec8898dSAnton Arapov up_read(&uprobe->register_rwsem); 1753fec8898dSAnton Arapov } 1754fec8898dSAnton Arapov 1755fec8898dSAnton Arapov static bool handle_trampoline(struct pt_regs *regs) 1756fec8898dSAnton Arapov { 1757fec8898dSAnton Arapov struct uprobe_task *utask; 1758fec8898dSAnton Arapov struct return_instance *ri, *tmp; 1759fec8898dSAnton Arapov bool chained; 1760fec8898dSAnton Arapov 1761fec8898dSAnton Arapov utask = current->utask; 1762fec8898dSAnton Arapov if (!utask) 1763fec8898dSAnton Arapov return false; 1764fec8898dSAnton Arapov 1765fec8898dSAnton Arapov ri = utask->return_instances; 1766fec8898dSAnton Arapov if (!ri) 1767fec8898dSAnton Arapov return false; 1768fec8898dSAnton Arapov 1769fec8898dSAnton Arapov /* 1770fec8898dSAnton Arapov * TODO: we should throw out return_instance's invalidated by 1771fec8898dSAnton Arapov * longjmp(), currently we assume that the probed function always 1772fec8898dSAnton Arapov * returns. 1773fec8898dSAnton Arapov */ 1774fec8898dSAnton Arapov instruction_pointer_set(regs, ri->orig_ret_vaddr); 1775fec8898dSAnton Arapov 1776fec8898dSAnton Arapov for (;;) { 1777fec8898dSAnton Arapov handle_uretprobe_chain(ri, regs); 1778fec8898dSAnton Arapov 1779fec8898dSAnton Arapov chained = ri->chained; 1780fec8898dSAnton Arapov put_uprobe(ri->uprobe); 1781fec8898dSAnton Arapov 1782fec8898dSAnton Arapov tmp = ri; 1783fec8898dSAnton Arapov ri = ri->next; 1784fec8898dSAnton Arapov kfree(tmp); 1785878b5a6eSOleg Nesterov utask->depth--; 1786fec8898dSAnton Arapov 1787fec8898dSAnton Arapov if (!chained) 1788fec8898dSAnton Arapov break; 1789fec8898dSAnton Arapov BUG_ON(!ri); 1790fec8898dSAnton Arapov } 1791fec8898dSAnton Arapov 1792fec8898dSAnton Arapov utask->return_instances = ri; 1793fec8898dSAnton Arapov 1794fec8898dSAnton Arapov return true; 1795fec8898dSAnton Arapov } 1796fec8898dSAnton Arapov 17976fe50a28SDavid A. Long bool __weak arch_uprobe_ignore(struct arch_uprobe *aup, struct pt_regs *regs) 17986fe50a28SDavid A. Long { 17996fe50a28SDavid A. Long return false; 18006fe50a28SDavid A. Long } 18016fe50a28SDavid A. Long 18023a9ea052SOleg Nesterov /* 18033a9ea052SOleg Nesterov * Run handler and ask thread to singlestep. 18043a9ea052SOleg Nesterov * Ensure all non-fatal signals cannot interrupt thread while it singlesteps. 18053a9ea052SOleg Nesterov */ 18063a9ea052SOleg Nesterov static void handle_swbp(struct pt_regs *regs) 18073a9ea052SOleg Nesterov { 18083a9ea052SOleg Nesterov struct uprobe *uprobe; 18093a9ea052SOleg Nesterov unsigned long bp_vaddr; 181056bb4cf6SOleg Nesterov int uninitialized_var(is_swbp); 18113a9ea052SOleg Nesterov 18123a9ea052SOleg Nesterov bp_vaddr = uprobe_get_swbp_addr(regs); 1813fec8898dSAnton Arapov if (bp_vaddr == get_trampoline_vaddr()) { 1814fec8898dSAnton Arapov if (handle_trampoline(regs)) 1815fec8898dSAnton Arapov return; 18163a9ea052SOleg Nesterov 1817fec8898dSAnton Arapov pr_warn("uprobe: unable to handle uretprobe pid/tgid=%d/%d\n", 1818fec8898dSAnton Arapov current->pid, current->tgid); 1819fec8898dSAnton Arapov } 1820fec8898dSAnton Arapov 1821fec8898dSAnton Arapov uprobe = find_active_uprobe(bp_vaddr, &is_swbp); 18220326f5a9SSrikar Dronamraju if (!uprobe) { 182356bb4cf6SOleg Nesterov if (is_swbp > 0) { 18240326f5a9SSrikar Dronamraju /* No matching uprobe; signal SIGTRAP. */ 18250326f5a9SSrikar Dronamraju send_sig(SIGTRAP, current, 0); 182656bb4cf6SOleg Nesterov } else { 182756bb4cf6SOleg Nesterov /* 182856bb4cf6SOleg Nesterov * Either we raced with uprobe_unregister() or we can't 182956bb4cf6SOleg Nesterov * access this memory. The latter is only possible if 183056bb4cf6SOleg Nesterov * another thread plays with our ->mm. In both cases 183156bb4cf6SOleg Nesterov * we can simply restart. If this vma was unmapped we 183256bb4cf6SOleg Nesterov * can pretend this insn was not executed yet and get 183356bb4cf6SOleg Nesterov * the (correct) SIGSEGV after restart. 183456bb4cf6SOleg Nesterov */ 183556bb4cf6SOleg Nesterov instruction_pointer_set(regs, bp_vaddr); 183656bb4cf6SOleg Nesterov } 18370326f5a9SSrikar Dronamraju return; 18380326f5a9SSrikar Dronamraju } 183974e59dfcSOleg Nesterov 184074e59dfcSOleg Nesterov /* change it in advance for ->handler() and restart */ 184174e59dfcSOleg Nesterov instruction_pointer_set(regs, bp_vaddr); 184274e59dfcSOleg Nesterov 1843142b18ddSOleg Nesterov /* 1844142b18ddSOleg Nesterov * TODO: move copy_insn/etc into _register and remove this hack. 1845142b18ddSOleg Nesterov * After we hit the bp, _unregister + _register can install the 1846142b18ddSOleg Nesterov * new and not-yet-analyzed uprobe at the same address, restart. 1847142b18ddSOleg Nesterov */ 1848142b18ddSOleg Nesterov smp_rmb(); /* pairs with wmb() in install_breakpoint() */ 184971434f2fSOleg Nesterov if (unlikely(!test_bit(UPROBE_COPY_INSN, &uprobe->flags))) 185074e59dfcSOleg Nesterov goto out; 18510326f5a9SSrikar Dronamraju 185272fd293aSOleg Nesterov /* Tracing handlers use ->utask to communicate with fetch methods */ 185372fd293aSOleg Nesterov if (!get_utask()) 185472fd293aSOleg Nesterov goto out; 185572fd293aSOleg Nesterov 18566fe50a28SDavid A. Long if (arch_uprobe_ignore(&uprobe->arch, regs)) 18576fe50a28SDavid A. Long goto out; 18586fe50a28SDavid A. Long 18590326f5a9SSrikar Dronamraju handler_chain(uprobe, regs); 18606fe50a28SDavid A. Long 18618a6b1732SOleg Nesterov if (arch_uprobe_skip_sstep(&uprobe->arch, regs)) 18620578a970SOleg Nesterov goto out; 18630326f5a9SSrikar Dronamraju 1864608e7427SOleg Nesterov if (!pre_ssout(uprobe, regs, bp_vaddr)) 18650326f5a9SSrikar Dronamraju return; 18660326f5a9SSrikar Dronamraju 18678a6b1732SOleg Nesterov /* arch_uprobe_skip_sstep() succeeded, or restart if can't singlestep */ 18680578a970SOleg Nesterov out: 18690326f5a9SSrikar Dronamraju put_uprobe(uprobe); 18700326f5a9SSrikar Dronamraju } 18710326f5a9SSrikar Dronamraju 18720326f5a9SSrikar Dronamraju /* 18730326f5a9SSrikar Dronamraju * Perform required fix-ups and disable singlestep. 18740326f5a9SSrikar Dronamraju * Allow pending signals to take effect. 18750326f5a9SSrikar Dronamraju */ 18760326f5a9SSrikar Dronamraju static void handle_singlestep(struct uprobe_task *utask, struct pt_regs *regs) 18770326f5a9SSrikar Dronamraju { 18780326f5a9SSrikar Dronamraju struct uprobe *uprobe; 1879014940baSOleg Nesterov int err = 0; 18800326f5a9SSrikar Dronamraju 18810326f5a9SSrikar Dronamraju uprobe = utask->active_uprobe; 18820326f5a9SSrikar Dronamraju if (utask->state == UTASK_SSTEP_ACK) 1883014940baSOleg Nesterov err = arch_uprobe_post_xol(&uprobe->arch, regs); 18840326f5a9SSrikar Dronamraju else if (utask->state == UTASK_SSTEP_TRAPPED) 18850326f5a9SSrikar Dronamraju arch_uprobe_abort_xol(&uprobe->arch, regs); 18860326f5a9SSrikar Dronamraju else 18870326f5a9SSrikar Dronamraju WARN_ON_ONCE(1); 18880326f5a9SSrikar Dronamraju 18890326f5a9SSrikar Dronamraju put_uprobe(uprobe); 18900326f5a9SSrikar Dronamraju utask->active_uprobe = NULL; 18910326f5a9SSrikar Dronamraju utask->state = UTASK_RUNNING; 1892d4b3b638SSrikar Dronamraju xol_free_insn_slot(current); 18930326f5a9SSrikar Dronamraju 18940326f5a9SSrikar Dronamraju spin_lock_irq(¤t->sighand->siglock); 18950326f5a9SSrikar Dronamraju recalc_sigpending(); /* see uprobe_deny_signal() */ 18960326f5a9SSrikar Dronamraju spin_unlock_irq(¤t->sighand->siglock); 1897014940baSOleg Nesterov 1898014940baSOleg Nesterov if (unlikely(err)) { 1899014940baSOleg Nesterov uprobe_warn(current, "execute the probed insn, sending SIGILL."); 1900014940baSOleg Nesterov force_sig_info(SIGILL, SEND_SIG_FORCED, current); 1901014940baSOleg Nesterov } 19020326f5a9SSrikar Dronamraju } 19030326f5a9SSrikar Dronamraju 19040326f5a9SSrikar Dronamraju /* 19051b08e907SOleg Nesterov * On breakpoint hit, breakpoint notifier sets the TIF_UPROBE flag and 19061b08e907SOleg Nesterov * allows the thread to return from interrupt. After that handle_swbp() 19071b08e907SOleg Nesterov * sets utask->active_uprobe. 19080326f5a9SSrikar Dronamraju * 19091b08e907SOleg Nesterov * On singlestep exception, singlestep notifier sets the TIF_UPROBE flag 19101b08e907SOleg Nesterov * and allows the thread to return from interrupt. 19110326f5a9SSrikar Dronamraju * 19120326f5a9SSrikar Dronamraju * While returning to userspace, thread notices the TIF_UPROBE flag and calls 19130326f5a9SSrikar Dronamraju * uprobe_notify_resume(). 19140326f5a9SSrikar Dronamraju */ 19150326f5a9SSrikar Dronamraju void uprobe_notify_resume(struct pt_regs *regs) 19160326f5a9SSrikar Dronamraju { 19170326f5a9SSrikar Dronamraju struct uprobe_task *utask; 19180326f5a9SSrikar Dronamraju 1919db023ea5SOleg Nesterov clear_thread_flag(TIF_UPROBE); 1920db023ea5SOleg Nesterov 19210326f5a9SSrikar Dronamraju utask = current->utask; 19221b08e907SOleg Nesterov if (utask && utask->active_uprobe) 19230326f5a9SSrikar Dronamraju handle_singlestep(utask, regs); 19241b08e907SOleg Nesterov else 19251b08e907SOleg Nesterov handle_swbp(regs); 19260326f5a9SSrikar Dronamraju } 19270326f5a9SSrikar Dronamraju 19280326f5a9SSrikar Dronamraju /* 19290326f5a9SSrikar Dronamraju * uprobe_pre_sstep_notifier gets called from interrupt context as part of 19300326f5a9SSrikar Dronamraju * notifier mechanism. Set TIF_UPROBE flag and indicate breakpoint hit. 19310326f5a9SSrikar Dronamraju */ 19320326f5a9SSrikar Dronamraju int uprobe_pre_sstep_notifier(struct pt_regs *regs) 19330326f5a9SSrikar Dronamraju { 19340dfd0eb8SAnton Arapov if (!current->mm) 19350dfd0eb8SAnton Arapov return 0; 19360dfd0eb8SAnton Arapov 19370dfd0eb8SAnton Arapov if (!test_bit(MMF_HAS_UPROBES, ¤t->mm->flags) && 19380dfd0eb8SAnton Arapov (!current->utask || !current->utask->return_instances)) 19390326f5a9SSrikar Dronamraju return 0; 19400326f5a9SSrikar Dronamraju 19410326f5a9SSrikar Dronamraju set_thread_flag(TIF_UPROBE); 19420326f5a9SSrikar Dronamraju return 1; 19430326f5a9SSrikar Dronamraju } 19440326f5a9SSrikar Dronamraju 19450326f5a9SSrikar Dronamraju /* 19460326f5a9SSrikar Dronamraju * uprobe_post_sstep_notifier gets called in interrupt context as part of notifier 19470326f5a9SSrikar Dronamraju * mechanism. Set TIF_UPROBE flag and indicate completion of singlestep. 19480326f5a9SSrikar Dronamraju */ 19490326f5a9SSrikar Dronamraju int uprobe_post_sstep_notifier(struct pt_regs *regs) 19500326f5a9SSrikar Dronamraju { 19510326f5a9SSrikar Dronamraju struct uprobe_task *utask = current->utask; 19520326f5a9SSrikar Dronamraju 19530326f5a9SSrikar Dronamraju if (!current->mm || !utask || !utask->active_uprobe) 19540326f5a9SSrikar Dronamraju /* task is currently not uprobed */ 19550326f5a9SSrikar Dronamraju return 0; 19560326f5a9SSrikar Dronamraju 19570326f5a9SSrikar Dronamraju utask->state = UTASK_SSTEP_ACK; 19580326f5a9SSrikar Dronamraju set_thread_flag(TIF_UPROBE); 19590326f5a9SSrikar Dronamraju return 1; 19600326f5a9SSrikar Dronamraju } 19610326f5a9SSrikar Dronamraju 19620326f5a9SSrikar Dronamraju static struct notifier_block uprobe_exception_nb = { 19630326f5a9SSrikar Dronamraju .notifier_call = arch_uprobe_exception_notify, 19640326f5a9SSrikar Dronamraju .priority = INT_MAX-1, /* notified after kprobes, kgdb */ 19650326f5a9SSrikar Dronamraju }; 19660326f5a9SSrikar Dronamraju 1967a5f4374aSIngo Molnar static int __init init_uprobes(void) 1968a5f4374aSIngo Molnar { 1969a5f4374aSIngo Molnar int i; 1970a5f4374aSIngo Molnar 197166d06dffSOleg Nesterov for (i = 0; i < UPROBES_HASH_SZ; i++) 1972a5f4374aSIngo Molnar mutex_init(&uprobes_mmap_mutex[i]); 19730326f5a9SSrikar Dronamraju 197432cdba1eSOleg Nesterov if (percpu_init_rwsem(&dup_mmap_sem)) 197532cdba1eSOleg Nesterov return -ENOMEM; 197632cdba1eSOleg Nesterov 19770326f5a9SSrikar Dronamraju return register_die_notifier(&uprobe_exception_nb); 1978a5f4374aSIngo Molnar } 1979736e89d9SOleg Nesterov __initcall(init_uprobes); 1980