1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * User-space Probes (UProbes) 4 * 5 * Copyright (C) IBM Corporation, 2008-2012 6 * Authors: 7 * Srikar Dronamraju 8 * Jim Keniston 9 * Copyright (C) 2011-2012 Red Hat, Inc., Peter Zijlstra 10 */ 11 12 #include <linux/kernel.h> 13 #include <linux/highmem.h> 14 #include <linux/pagemap.h> /* read_mapping_page */ 15 #include <linux/slab.h> 16 #include <linux/sched.h> 17 #include <linux/sched/mm.h> 18 #include <linux/sched/coredump.h> 19 #include <linux/export.h> 20 #include <linux/rmap.h> /* anon_vma_prepare */ 21 #include <linux/mmu_notifier.h> /* set_pte_at_notify */ 22 #include <linux/swap.h> /* try_to_free_swap */ 23 #include <linux/ptrace.h> /* user_enable_single_step */ 24 #include <linux/kdebug.h> /* notifier mechanism */ 25 #include "../../mm/internal.h" /* munlock_vma_page */ 26 #include <linux/percpu-rwsem.h> 27 #include <linux/task_work.h> 28 #include <linux/shmem_fs.h> 29 #include <linux/khugepaged.h> 30 31 #include <linux/uprobes.h> 32 33 #define UINSNS_PER_PAGE (PAGE_SIZE/UPROBE_XOL_SLOT_BYTES) 34 #define MAX_UPROBE_XOL_SLOTS UINSNS_PER_PAGE 35 36 static struct rb_root uprobes_tree = RB_ROOT; 37 /* 38 * allows us to skip the uprobe_mmap if there are no uprobe events active 39 * at this time. Probably a fine grained per inode count is better? 40 */ 41 #define no_uprobe_events() RB_EMPTY_ROOT(&uprobes_tree) 42 43 static DEFINE_SPINLOCK(uprobes_treelock); /* serialize rbtree access */ 44 45 #define UPROBES_HASH_SZ 13 46 /* serialize uprobe->pending_list */ 47 static struct mutex uprobes_mmap_mutex[UPROBES_HASH_SZ]; 48 #define uprobes_mmap_hash(v) (&uprobes_mmap_mutex[((unsigned long)(v)) % UPROBES_HASH_SZ]) 49 50 DEFINE_STATIC_PERCPU_RWSEM(dup_mmap_sem); 51 52 /* Have a copy of original instruction */ 53 #define UPROBE_COPY_INSN 0 54 55 struct uprobe { 56 struct rb_node rb_node; /* node in the rb tree */ 57 refcount_t ref; 58 struct rw_semaphore register_rwsem; 59 struct rw_semaphore consumer_rwsem; 60 struct list_head pending_list; 61 struct uprobe_consumer *consumers; 62 struct inode *inode; /* Also hold a ref to inode */ 63 loff_t offset; 64 loff_t ref_ctr_offset; 65 unsigned long flags; 66 67 /* 68 * The generic code assumes that it has two members of unknown type 69 * owned by the arch-specific code: 70 * 71 * insn - copy_insn() saves the original instruction here for 72 * arch_uprobe_analyze_insn(). 73 * 74 * ixol - potentially modified instruction to execute out of 75 * line, copied to xol_area by xol_get_insn_slot(). 76 */ 77 struct arch_uprobe arch; 78 }; 79 80 struct delayed_uprobe { 81 struct list_head list; 82 struct uprobe *uprobe; 83 struct mm_struct *mm; 84 }; 85 86 static DEFINE_MUTEX(delayed_uprobe_lock); 87 static LIST_HEAD(delayed_uprobe_list); 88 89 /* 90 * Execute out of line area: anonymous executable mapping installed 91 * by the probed task to execute the copy of the original instruction 92 * mangled by set_swbp(). 93 * 94 * On a breakpoint hit, thread contests for a slot. It frees the 95 * slot after singlestep. Currently a fixed number of slots are 96 * allocated. 97 */ 98 struct xol_area { 99 wait_queue_head_t wq; /* if all slots are busy */ 100 atomic_t slot_count; /* number of in-use slots */ 101 unsigned long *bitmap; /* 0 = free slot */ 102 103 struct vm_special_mapping xol_mapping; 104 struct page *pages[2]; 105 /* 106 * We keep the vma's vm_start rather than a pointer to the vma 107 * itself. The probed process or a naughty kernel module could make 108 * the vma go away, and we must handle that reasonably gracefully. 109 */ 110 unsigned long vaddr; /* Page(s) of instruction slots */ 111 }; 112 113 /* 114 * valid_vma: Verify if the specified vma is an executable vma 115 * Relax restrictions while unregistering: vm_flags might have 116 * changed after breakpoint was inserted. 117 * - is_register: indicates if we are in register context. 118 * - Return 1 if the specified virtual address is in an 119 * executable vma. 120 */ 121 static bool valid_vma(struct vm_area_struct *vma, bool is_register) 122 { 123 vm_flags_t flags = VM_HUGETLB | VM_MAYEXEC | VM_MAYSHARE; 124 125 if (is_register) 126 flags |= VM_WRITE; 127 128 return vma->vm_file && (vma->vm_flags & flags) == VM_MAYEXEC; 129 } 130 131 static unsigned long offset_to_vaddr(struct vm_area_struct *vma, loff_t offset) 132 { 133 return vma->vm_start + offset - ((loff_t)vma->vm_pgoff << PAGE_SHIFT); 134 } 135 136 static loff_t vaddr_to_offset(struct vm_area_struct *vma, unsigned long vaddr) 137 { 138 return ((loff_t)vma->vm_pgoff << PAGE_SHIFT) + (vaddr - vma->vm_start); 139 } 140 141 /** 142 * __replace_page - replace page in vma by new page. 143 * based on replace_page in mm/ksm.c 144 * 145 * @vma: vma that holds the pte pointing to page 146 * @addr: address the old @page is mapped at 147 * @old_page: the page we are replacing by new_page 148 * @new_page: the modified page we replace page by 149 * 150 * If @new_page is NULL, only unmap @old_page. 151 * 152 * Returns 0 on success, negative error code otherwise. 153 */ 154 static int __replace_page(struct vm_area_struct *vma, unsigned long addr, 155 struct page *old_page, struct page *new_page) 156 { 157 struct mm_struct *mm = vma->vm_mm; 158 struct page_vma_mapped_walk pvmw = { 159 .page = compound_head(old_page), 160 .vma = vma, 161 .address = addr, 162 }; 163 int err; 164 struct mmu_notifier_range range; 165 166 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, mm, addr, 167 addr + PAGE_SIZE); 168 169 if (new_page) { 170 err = mem_cgroup_charge(page_folio(new_page), vma->vm_mm, 171 GFP_KERNEL); 172 if (err) 173 return err; 174 } 175 176 /* For try_to_free_swap() below */ 177 lock_page(old_page); 178 179 mmu_notifier_invalidate_range_start(&range); 180 err = -EAGAIN; 181 if (!page_vma_mapped_walk(&pvmw)) 182 goto unlock; 183 VM_BUG_ON_PAGE(addr != pvmw.address, old_page); 184 185 if (new_page) { 186 get_page(new_page); 187 page_add_new_anon_rmap(new_page, vma, addr, false); 188 lru_cache_add_inactive_or_unevictable(new_page, vma); 189 } else 190 /* no new page, just dec_mm_counter for old_page */ 191 dec_mm_counter(mm, MM_ANONPAGES); 192 193 if (!PageAnon(old_page)) { 194 dec_mm_counter(mm, mm_counter_file(old_page)); 195 inc_mm_counter(mm, MM_ANONPAGES); 196 } 197 198 flush_cache_page(vma, addr, pte_pfn(*pvmw.pte)); 199 ptep_clear_flush_notify(vma, addr, pvmw.pte); 200 if (new_page) 201 set_pte_at_notify(mm, addr, pvmw.pte, 202 mk_pte(new_page, vma->vm_page_prot)); 203 204 page_remove_rmap(old_page, vma, false); 205 if (!page_mapped(old_page)) 206 try_to_free_swap(old_page); 207 page_vma_mapped_walk_done(&pvmw); 208 put_page(old_page); 209 210 err = 0; 211 unlock: 212 mmu_notifier_invalidate_range_end(&range); 213 unlock_page(old_page); 214 return err; 215 } 216 217 /** 218 * is_swbp_insn - check if instruction is breakpoint instruction. 219 * @insn: instruction to be checked. 220 * Default implementation of is_swbp_insn 221 * Returns true if @insn is a breakpoint instruction. 222 */ 223 bool __weak is_swbp_insn(uprobe_opcode_t *insn) 224 { 225 return *insn == UPROBE_SWBP_INSN; 226 } 227 228 /** 229 * is_trap_insn - check if instruction is breakpoint instruction. 230 * @insn: instruction to be checked. 231 * Default implementation of is_trap_insn 232 * Returns true if @insn is a breakpoint instruction. 233 * 234 * This function is needed for the case where an architecture has multiple 235 * trap instructions (like powerpc). 236 */ 237 bool __weak is_trap_insn(uprobe_opcode_t *insn) 238 { 239 return is_swbp_insn(insn); 240 } 241 242 static void copy_from_page(struct page *page, unsigned long vaddr, void *dst, int len) 243 { 244 void *kaddr = kmap_atomic(page); 245 memcpy(dst, kaddr + (vaddr & ~PAGE_MASK), len); 246 kunmap_atomic(kaddr); 247 } 248 249 static void copy_to_page(struct page *page, unsigned long vaddr, const void *src, int len) 250 { 251 void *kaddr = kmap_atomic(page); 252 memcpy(kaddr + (vaddr & ~PAGE_MASK), src, len); 253 kunmap_atomic(kaddr); 254 } 255 256 static int verify_opcode(struct page *page, unsigned long vaddr, uprobe_opcode_t *new_opcode) 257 { 258 uprobe_opcode_t old_opcode; 259 bool is_swbp; 260 261 /* 262 * Note: We only check if the old_opcode is UPROBE_SWBP_INSN here. 263 * We do not check if it is any other 'trap variant' which could 264 * be conditional trap instruction such as the one powerpc supports. 265 * 266 * The logic is that we do not care if the underlying instruction 267 * is a trap variant; uprobes always wins over any other (gdb) 268 * breakpoint. 269 */ 270 copy_from_page(page, vaddr, &old_opcode, UPROBE_SWBP_INSN_SIZE); 271 is_swbp = is_swbp_insn(&old_opcode); 272 273 if (is_swbp_insn(new_opcode)) { 274 if (is_swbp) /* register: already installed? */ 275 return 0; 276 } else { 277 if (!is_swbp) /* unregister: was it changed by us? */ 278 return 0; 279 } 280 281 return 1; 282 } 283 284 static struct delayed_uprobe * 285 delayed_uprobe_check(struct uprobe *uprobe, struct mm_struct *mm) 286 { 287 struct delayed_uprobe *du; 288 289 list_for_each_entry(du, &delayed_uprobe_list, list) 290 if (du->uprobe == uprobe && du->mm == mm) 291 return du; 292 return NULL; 293 } 294 295 static int delayed_uprobe_add(struct uprobe *uprobe, struct mm_struct *mm) 296 { 297 struct delayed_uprobe *du; 298 299 if (delayed_uprobe_check(uprobe, mm)) 300 return 0; 301 302 du = kzalloc(sizeof(*du), GFP_KERNEL); 303 if (!du) 304 return -ENOMEM; 305 306 du->uprobe = uprobe; 307 du->mm = mm; 308 list_add(&du->list, &delayed_uprobe_list); 309 return 0; 310 } 311 312 static void delayed_uprobe_delete(struct delayed_uprobe *du) 313 { 314 if (WARN_ON(!du)) 315 return; 316 list_del(&du->list); 317 kfree(du); 318 } 319 320 static void delayed_uprobe_remove(struct uprobe *uprobe, struct mm_struct *mm) 321 { 322 struct list_head *pos, *q; 323 struct delayed_uprobe *du; 324 325 if (!uprobe && !mm) 326 return; 327 328 list_for_each_safe(pos, q, &delayed_uprobe_list) { 329 du = list_entry(pos, struct delayed_uprobe, list); 330 331 if (uprobe && du->uprobe != uprobe) 332 continue; 333 if (mm && du->mm != mm) 334 continue; 335 336 delayed_uprobe_delete(du); 337 } 338 } 339 340 static bool valid_ref_ctr_vma(struct uprobe *uprobe, 341 struct vm_area_struct *vma) 342 { 343 unsigned long vaddr = offset_to_vaddr(vma, uprobe->ref_ctr_offset); 344 345 return uprobe->ref_ctr_offset && 346 vma->vm_file && 347 file_inode(vma->vm_file) == uprobe->inode && 348 (vma->vm_flags & (VM_WRITE|VM_SHARED)) == VM_WRITE && 349 vma->vm_start <= vaddr && 350 vma->vm_end > vaddr; 351 } 352 353 static struct vm_area_struct * 354 find_ref_ctr_vma(struct uprobe *uprobe, struct mm_struct *mm) 355 { 356 struct vm_area_struct *tmp; 357 358 for (tmp = mm->mmap; tmp; tmp = tmp->vm_next) 359 if (valid_ref_ctr_vma(uprobe, tmp)) 360 return tmp; 361 362 return NULL; 363 } 364 365 static int 366 __update_ref_ctr(struct mm_struct *mm, unsigned long vaddr, short d) 367 { 368 void *kaddr; 369 struct page *page; 370 struct vm_area_struct *vma; 371 int ret; 372 short *ptr; 373 374 if (!vaddr || !d) 375 return -EINVAL; 376 377 ret = get_user_pages_remote(mm, vaddr, 1, 378 FOLL_WRITE, &page, &vma, NULL); 379 if (unlikely(ret <= 0)) { 380 /* 381 * We are asking for 1 page. If get_user_pages_remote() fails, 382 * it may return 0, in that case we have to return error. 383 */ 384 return ret == 0 ? -EBUSY : ret; 385 } 386 387 kaddr = kmap_atomic(page); 388 ptr = kaddr + (vaddr & ~PAGE_MASK); 389 390 if (unlikely(*ptr + d < 0)) { 391 pr_warn("ref_ctr going negative. vaddr: 0x%lx, " 392 "curr val: %d, delta: %d\n", vaddr, *ptr, d); 393 ret = -EINVAL; 394 goto out; 395 } 396 397 *ptr += d; 398 ret = 0; 399 out: 400 kunmap_atomic(kaddr); 401 put_page(page); 402 return ret; 403 } 404 405 static void update_ref_ctr_warn(struct uprobe *uprobe, 406 struct mm_struct *mm, short d) 407 { 408 pr_warn("ref_ctr %s failed for inode: 0x%lx offset: " 409 "0x%llx ref_ctr_offset: 0x%llx of mm: 0x%pK\n", 410 d > 0 ? "increment" : "decrement", uprobe->inode->i_ino, 411 (unsigned long long) uprobe->offset, 412 (unsigned long long) uprobe->ref_ctr_offset, mm); 413 } 414 415 static int update_ref_ctr(struct uprobe *uprobe, struct mm_struct *mm, 416 short d) 417 { 418 struct vm_area_struct *rc_vma; 419 unsigned long rc_vaddr; 420 int ret = 0; 421 422 rc_vma = find_ref_ctr_vma(uprobe, mm); 423 424 if (rc_vma) { 425 rc_vaddr = offset_to_vaddr(rc_vma, uprobe->ref_ctr_offset); 426 ret = __update_ref_ctr(mm, rc_vaddr, d); 427 if (ret) 428 update_ref_ctr_warn(uprobe, mm, d); 429 430 if (d > 0) 431 return ret; 432 } 433 434 mutex_lock(&delayed_uprobe_lock); 435 if (d > 0) 436 ret = delayed_uprobe_add(uprobe, mm); 437 else 438 delayed_uprobe_remove(uprobe, mm); 439 mutex_unlock(&delayed_uprobe_lock); 440 441 return ret; 442 } 443 444 /* 445 * NOTE: 446 * Expect the breakpoint instruction to be the smallest size instruction for 447 * the architecture. If an arch has variable length instruction and the 448 * breakpoint instruction is not of the smallest length instruction 449 * supported by that architecture then we need to modify is_trap_at_addr and 450 * uprobe_write_opcode accordingly. This would never be a problem for archs 451 * that have fixed length instructions. 452 * 453 * uprobe_write_opcode - write the opcode at a given virtual address. 454 * @auprobe: arch specific probepoint information. 455 * @mm: the probed process address space. 456 * @vaddr: the virtual address to store the opcode. 457 * @opcode: opcode to be written at @vaddr. 458 * 459 * Called with mm->mmap_lock held for write. 460 * Return 0 (success) or a negative errno. 461 */ 462 int uprobe_write_opcode(struct arch_uprobe *auprobe, struct mm_struct *mm, 463 unsigned long vaddr, uprobe_opcode_t opcode) 464 { 465 struct uprobe *uprobe; 466 struct page *old_page, *new_page; 467 struct vm_area_struct *vma; 468 int ret, is_register, ref_ctr_updated = 0; 469 bool orig_page_huge = false; 470 unsigned int gup_flags = FOLL_FORCE; 471 472 is_register = is_swbp_insn(&opcode); 473 uprobe = container_of(auprobe, struct uprobe, arch); 474 475 retry: 476 if (is_register) 477 gup_flags |= FOLL_SPLIT_PMD; 478 /* Read the page with vaddr into memory */ 479 ret = get_user_pages_remote(mm, vaddr, 1, gup_flags, 480 &old_page, &vma, NULL); 481 if (ret <= 0) 482 return ret; 483 484 ret = verify_opcode(old_page, vaddr, &opcode); 485 if (ret <= 0) 486 goto put_old; 487 488 if (WARN(!is_register && PageCompound(old_page), 489 "uprobe unregister should never work on compound page\n")) { 490 ret = -EINVAL; 491 goto put_old; 492 } 493 494 /* We are going to replace instruction, update ref_ctr. */ 495 if (!ref_ctr_updated && uprobe->ref_ctr_offset) { 496 ret = update_ref_ctr(uprobe, mm, is_register ? 1 : -1); 497 if (ret) 498 goto put_old; 499 500 ref_ctr_updated = 1; 501 } 502 503 ret = 0; 504 if (!is_register && !PageAnon(old_page)) 505 goto put_old; 506 507 ret = anon_vma_prepare(vma); 508 if (ret) 509 goto put_old; 510 511 ret = -ENOMEM; 512 new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, vaddr); 513 if (!new_page) 514 goto put_old; 515 516 __SetPageUptodate(new_page); 517 copy_highpage(new_page, old_page); 518 copy_to_page(new_page, vaddr, &opcode, UPROBE_SWBP_INSN_SIZE); 519 520 if (!is_register) { 521 struct page *orig_page; 522 pgoff_t index; 523 524 VM_BUG_ON_PAGE(!PageAnon(old_page), old_page); 525 526 index = vaddr_to_offset(vma, vaddr & PAGE_MASK) >> PAGE_SHIFT; 527 orig_page = find_get_page(vma->vm_file->f_inode->i_mapping, 528 index); 529 530 if (orig_page) { 531 if (PageUptodate(orig_page) && 532 pages_identical(new_page, orig_page)) { 533 /* let go new_page */ 534 put_page(new_page); 535 new_page = NULL; 536 537 if (PageCompound(orig_page)) 538 orig_page_huge = true; 539 } 540 put_page(orig_page); 541 } 542 } 543 544 ret = __replace_page(vma, vaddr, old_page, new_page); 545 if (new_page) 546 put_page(new_page); 547 put_old: 548 put_page(old_page); 549 550 if (unlikely(ret == -EAGAIN)) 551 goto retry; 552 553 /* Revert back reference counter if instruction update failed. */ 554 if (ret && is_register && ref_ctr_updated) 555 update_ref_ctr(uprobe, mm, -1); 556 557 /* try collapse pmd for compound page */ 558 if (!ret && orig_page_huge) 559 collapse_pte_mapped_thp(mm, vaddr); 560 561 return ret; 562 } 563 564 /** 565 * set_swbp - store breakpoint at a given address. 566 * @auprobe: arch specific probepoint information. 567 * @mm: the probed process address space. 568 * @vaddr: the virtual address to insert the opcode. 569 * 570 * For mm @mm, store the breakpoint instruction at @vaddr. 571 * Return 0 (success) or a negative errno. 572 */ 573 int __weak set_swbp(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned long vaddr) 574 { 575 return uprobe_write_opcode(auprobe, mm, vaddr, UPROBE_SWBP_INSN); 576 } 577 578 /** 579 * set_orig_insn - Restore the original instruction. 580 * @mm: the probed process address space. 581 * @auprobe: arch specific probepoint information. 582 * @vaddr: the virtual address to insert the opcode. 583 * 584 * For mm @mm, restore the original opcode (opcode) at @vaddr. 585 * Return 0 (success) or a negative errno. 586 */ 587 int __weak 588 set_orig_insn(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned long vaddr) 589 { 590 return uprobe_write_opcode(auprobe, mm, vaddr, 591 *(uprobe_opcode_t *)&auprobe->insn); 592 } 593 594 static struct uprobe *get_uprobe(struct uprobe *uprobe) 595 { 596 refcount_inc(&uprobe->ref); 597 return uprobe; 598 } 599 600 static void put_uprobe(struct uprobe *uprobe) 601 { 602 if (refcount_dec_and_test(&uprobe->ref)) { 603 /* 604 * If application munmap(exec_vma) before uprobe_unregister() 605 * gets called, we don't get a chance to remove uprobe from 606 * delayed_uprobe_list from remove_breakpoint(). Do it here. 607 */ 608 mutex_lock(&delayed_uprobe_lock); 609 delayed_uprobe_remove(uprobe, NULL); 610 mutex_unlock(&delayed_uprobe_lock); 611 kfree(uprobe); 612 } 613 } 614 615 static __always_inline 616 int uprobe_cmp(const struct inode *l_inode, const loff_t l_offset, 617 const struct uprobe *r) 618 { 619 if (l_inode < r->inode) 620 return -1; 621 622 if (l_inode > r->inode) 623 return 1; 624 625 if (l_offset < r->offset) 626 return -1; 627 628 if (l_offset > r->offset) 629 return 1; 630 631 return 0; 632 } 633 634 #define __node_2_uprobe(node) \ 635 rb_entry((node), struct uprobe, rb_node) 636 637 struct __uprobe_key { 638 struct inode *inode; 639 loff_t offset; 640 }; 641 642 static inline int __uprobe_cmp_key(const void *key, const struct rb_node *b) 643 { 644 const struct __uprobe_key *a = key; 645 return uprobe_cmp(a->inode, a->offset, __node_2_uprobe(b)); 646 } 647 648 static inline int __uprobe_cmp(struct rb_node *a, const struct rb_node *b) 649 { 650 struct uprobe *u = __node_2_uprobe(a); 651 return uprobe_cmp(u->inode, u->offset, __node_2_uprobe(b)); 652 } 653 654 static struct uprobe *__find_uprobe(struct inode *inode, loff_t offset) 655 { 656 struct __uprobe_key key = { 657 .inode = inode, 658 .offset = offset, 659 }; 660 struct rb_node *node = rb_find(&key, &uprobes_tree, __uprobe_cmp_key); 661 662 if (node) 663 return get_uprobe(__node_2_uprobe(node)); 664 665 return NULL; 666 } 667 668 /* 669 * Find a uprobe corresponding to a given inode:offset 670 * Acquires uprobes_treelock 671 */ 672 static struct uprobe *find_uprobe(struct inode *inode, loff_t offset) 673 { 674 struct uprobe *uprobe; 675 676 spin_lock(&uprobes_treelock); 677 uprobe = __find_uprobe(inode, offset); 678 spin_unlock(&uprobes_treelock); 679 680 return uprobe; 681 } 682 683 static struct uprobe *__insert_uprobe(struct uprobe *uprobe) 684 { 685 struct rb_node *node; 686 687 node = rb_find_add(&uprobe->rb_node, &uprobes_tree, __uprobe_cmp); 688 if (node) 689 return get_uprobe(__node_2_uprobe(node)); 690 691 /* get access + creation ref */ 692 refcount_set(&uprobe->ref, 2); 693 return NULL; 694 } 695 696 /* 697 * Acquire uprobes_treelock. 698 * Matching uprobe already exists in rbtree; 699 * increment (access refcount) and return the matching uprobe. 700 * 701 * No matching uprobe; insert the uprobe in rb_tree; 702 * get a double refcount (access + creation) and return NULL. 703 */ 704 static struct uprobe *insert_uprobe(struct uprobe *uprobe) 705 { 706 struct uprobe *u; 707 708 spin_lock(&uprobes_treelock); 709 u = __insert_uprobe(uprobe); 710 spin_unlock(&uprobes_treelock); 711 712 return u; 713 } 714 715 static void 716 ref_ctr_mismatch_warn(struct uprobe *cur_uprobe, struct uprobe *uprobe) 717 { 718 pr_warn("ref_ctr_offset mismatch. inode: 0x%lx offset: 0x%llx " 719 "ref_ctr_offset(old): 0x%llx ref_ctr_offset(new): 0x%llx\n", 720 uprobe->inode->i_ino, (unsigned long long) uprobe->offset, 721 (unsigned long long) cur_uprobe->ref_ctr_offset, 722 (unsigned long long) uprobe->ref_ctr_offset); 723 } 724 725 static struct uprobe *alloc_uprobe(struct inode *inode, loff_t offset, 726 loff_t ref_ctr_offset) 727 { 728 struct uprobe *uprobe, *cur_uprobe; 729 730 uprobe = kzalloc(sizeof(struct uprobe), GFP_KERNEL); 731 if (!uprobe) 732 return NULL; 733 734 uprobe->inode = inode; 735 uprobe->offset = offset; 736 uprobe->ref_ctr_offset = ref_ctr_offset; 737 init_rwsem(&uprobe->register_rwsem); 738 init_rwsem(&uprobe->consumer_rwsem); 739 740 /* add to uprobes_tree, sorted on inode:offset */ 741 cur_uprobe = insert_uprobe(uprobe); 742 /* a uprobe exists for this inode:offset combination */ 743 if (cur_uprobe) { 744 if (cur_uprobe->ref_ctr_offset != uprobe->ref_ctr_offset) { 745 ref_ctr_mismatch_warn(cur_uprobe, uprobe); 746 put_uprobe(cur_uprobe); 747 kfree(uprobe); 748 return ERR_PTR(-EINVAL); 749 } 750 kfree(uprobe); 751 uprobe = cur_uprobe; 752 } 753 754 return uprobe; 755 } 756 757 static void consumer_add(struct uprobe *uprobe, struct uprobe_consumer *uc) 758 { 759 down_write(&uprobe->consumer_rwsem); 760 uc->next = uprobe->consumers; 761 uprobe->consumers = uc; 762 up_write(&uprobe->consumer_rwsem); 763 } 764 765 /* 766 * For uprobe @uprobe, delete the consumer @uc. 767 * Return true if the @uc is deleted successfully 768 * or return false. 769 */ 770 static bool consumer_del(struct uprobe *uprobe, struct uprobe_consumer *uc) 771 { 772 struct uprobe_consumer **con; 773 bool ret = false; 774 775 down_write(&uprobe->consumer_rwsem); 776 for (con = &uprobe->consumers; *con; con = &(*con)->next) { 777 if (*con == uc) { 778 *con = uc->next; 779 ret = true; 780 break; 781 } 782 } 783 up_write(&uprobe->consumer_rwsem); 784 785 return ret; 786 } 787 788 static int __copy_insn(struct address_space *mapping, struct file *filp, 789 void *insn, int nbytes, loff_t offset) 790 { 791 struct page *page; 792 /* 793 * Ensure that the page that has the original instruction is populated 794 * and in page-cache. If ->readpage == NULL it must be shmem_mapping(), 795 * see uprobe_register(). 796 */ 797 if (mapping->a_ops->readpage) 798 page = read_mapping_page(mapping, offset >> PAGE_SHIFT, filp); 799 else 800 page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT); 801 if (IS_ERR(page)) 802 return PTR_ERR(page); 803 804 copy_from_page(page, offset, insn, nbytes); 805 put_page(page); 806 807 return 0; 808 } 809 810 static int copy_insn(struct uprobe *uprobe, struct file *filp) 811 { 812 struct address_space *mapping = uprobe->inode->i_mapping; 813 loff_t offs = uprobe->offset; 814 void *insn = &uprobe->arch.insn; 815 int size = sizeof(uprobe->arch.insn); 816 int len, err = -EIO; 817 818 /* Copy only available bytes, -EIO if nothing was read */ 819 do { 820 if (offs >= i_size_read(uprobe->inode)) 821 break; 822 823 len = min_t(int, size, PAGE_SIZE - (offs & ~PAGE_MASK)); 824 err = __copy_insn(mapping, filp, insn, len, offs); 825 if (err) 826 break; 827 828 insn += len; 829 offs += len; 830 size -= len; 831 } while (size); 832 833 return err; 834 } 835 836 static int prepare_uprobe(struct uprobe *uprobe, struct file *file, 837 struct mm_struct *mm, unsigned long vaddr) 838 { 839 int ret = 0; 840 841 if (test_bit(UPROBE_COPY_INSN, &uprobe->flags)) 842 return ret; 843 844 /* TODO: move this into _register, until then we abuse this sem. */ 845 down_write(&uprobe->consumer_rwsem); 846 if (test_bit(UPROBE_COPY_INSN, &uprobe->flags)) 847 goto out; 848 849 ret = copy_insn(uprobe, file); 850 if (ret) 851 goto out; 852 853 ret = -ENOTSUPP; 854 if (is_trap_insn((uprobe_opcode_t *)&uprobe->arch.insn)) 855 goto out; 856 857 ret = arch_uprobe_analyze_insn(&uprobe->arch, mm, vaddr); 858 if (ret) 859 goto out; 860 861 smp_wmb(); /* pairs with the smp_rmb() in handle_swbp() */ 862 set_bit(UPROBE_COPY_INSN, &uprobe->flags); 863 864 out: 865 up_write(&uprobe->consumer_rwsem); 866 867 return ret; 868 } 869 870 static inline bool consumer_filter(struct uprobe_consumer *uc, 871 enum uprobe_filter_ctx ctx, struct mm_struct *mm) 872 { 873 return !uc->filter || uc->filter(uc, ctx, mm); 874 } 875 876 static bool filter_chain(struct uprobe *uprobe, 877 enum uprobe_filter_ctx ctx, struct mm_struct *mm) 878 { 879 struct uprobe_consumer *uc; 880 bool ret = false; 881 882 down_read(&uprobe->consumer_rwsem); 883 for (uc = uprobe->consumers; uc; uc = uc->next) { 884 ret = consumer_filter(uc, ctx, mm); 885 if (ret) 886 break; 887 } 888 up_read(&uprobe->consumer_rwsem); 889 890 return ret; 891 } 892 893 static int 894 install_breakpoint(struct uprobe *uprobe, struct mm_struct *mm, 895 struct vm_area_struct *vma, unsigned long vaddr) 896 { 897 bool first_uprobe; 898 int ret; 899 900 ret = prepare_uprobe(uprobe, vma->vm_file, mm, vaddr); 901 if (ret) 902 return ret; 903 904 /* 905 * set MMF_HAS_UPROBES in advance for uprobe_pre_sstep_notifier(), 906 * the task can hit this breakpoint right after __replace_page(). 907 */ 908 first_uprobe = !test_bit(MMF_HAS_UPROBES, &mm->flags); 909 if (first_uprobe) 910 set_bit(MMF_HAS_UPROBES, &mm->flags); 911 912 ret = set_swbp(&uprobe->arch, mm, vaddr); 913 if (!ret) 914 clear_bit(MMF_RECALC_UPROBES, &mm->flags); 915 else if (first_uprobe) 916 clear_bit(MMF_HAS_UPROBES, &mm->flags); 917 918 return ret; 919 } 920 921 static int 922 remove_breakpoint(struct uprobe *uprobe, struct mm_struct *mm, unsigned long vaddr) 923 { 924 set_bit(MMF_RECALC_UPROBES, &mm->flags); 925 return set_orig_insn(&uprobe->arch, mm, vaddr); 926 } 927 928 static inline bool uprobe_is_active(struct uprobe *uprobe) 929 { 930 return !RB_EMPTY_NODE(&uprobe->rb_node); 931 } 932 /* 933 * There could be threads that have already hit the breakpoint. They 934 * will recheck the current insn and restart if find_uprobe() fails. 935 * See find_active_uprobe(). 936 */ 937 static void delete_uprobe(struct uprobe *uprobe) 938 { 939 if (WARN_ON(!uprobe_is_active(uprobe))) 940 return; 941 942 spin_lock(&uprobes_treelock); 943 rb_erase(&uprobe->rb_node, &uprobes_tree); 944 spin_unlock(&uprobes_treelock); 945 RB_CLEAR_NODE(&uprobe->rb_node); /* for uprobe_is_active() */ 946 put_uprobe(uprobe); 947 } 948 949 struct map_info { 950 struct map_info *next; 951 struct mm_struct *mm; 952 unsigned long vaddr; 953 }; 954 955 static inline struct map_info *free_map_info(struct map_info *info) 956 { 957 struct map_info *next = info->next; 958 kfree(info); 959 return next; 960 } 961 962 static struct map_info * 963 build_map_info(struct address_space *mapping, loff_t offset, bool is_register) 964 { 965 unsigned long pgoff = offset >> PAGE_SHIFT; 966 struct vm_area_struct *vma; 967 struct map_info *curr = NULL; 968 struct map_info *prev = NULL; 969 struct map_info *info; 970 int more = 0; 971 972 again: 973 i_mmap_lock_read(mapping); 974 vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) { 975 if (!valid_vma(vma, is_register)) 976 continue; 977 978 if (!prev && !more) { 979 /* 980 * Needs GFP_NOWAIT to avoid i_mmap_rwsem recursion through 981 * reclaim. This is optimistic, no harm done if it fails. 982 */ 983 prev = kmalloc(sizeof(struct map_info), 984 GFP_NOWAIT | __GFP_NOMEMALLOC | __GFP_NOWARN); 985 if (prev) 986 prev->next = NULL; 987 } 988 if (!prev) { 989 more++; 990 continue; 991 } 992 993 if (!mmget_not_zero(vma->vm_mm)) 994 continue; 995 996 info = prev; 997 prev = prev->next; 998 info->next = curr; 999 curr = info; 1000 1001 info->mm = vma->vm_mm; 1002 info->vaddr = offset_to_vaddr(vma, offset); 1003 } 1004 i_mmap_unlock_read(mapping); 1005 1006 if (!more) 1007 goto out; 1008 1009 prev = curr; 1010 while (curr) { 1011 mmput(curr->mm); 1012 curr = curr->next; 1013 } 1014 1015 do { 1016 info = kmalloc(sizeof(struct map_info), GFP_KERNEL); 1017 if (!info) { 1018 curr = ERR_PTR(-ENOMEM); 1019 goto out; 1020 } 1021 info->next = prev; 1022 prev = info; 1023 } while (--more); 1024 1025 goto again; 1026 out: 1027 while (prev) 1028 prev = free_map_info(prev); 1029 return curr; 1030 } 1031 1032 static int 1033 register_for_each_vma(struct uprobe *uprobe, struct uprobe_consumer *new) 1034 { 1035 bool is_register = !!new; 1036 struct map_info *info; 1037 int err = 0; 1038 1039 percpu_down_write(&dup_mmap_sem); 1040 info = build_map_info(uprobe->inode->i_mapping, 1041 uprobe->offset, is_register); 1042 if (IS_ERR(info)) { 1043 err = PTR_ERR(info); 1044 goto out; 1045 } 1046 1047 while (info) { 1048 struct mm_struct *mm = info->mm; 1049 struct vm_area_struct *vma; 1050 1051 if (err && is_register) 1052 goto free; 1053 1054 mmap_write_lock(mm); 1055 vma = find_vma(mm, info->vaddr); 1056 if (!vma || !valid_vma(vma, is_register) || 1057 file_inode(vma->vm_file) != uprobe->inode) 1058 goto unlock; 1059 1060 if (vma->vm_start > info->vaddr || 1061 vaddr_to_offset(vma, info->vaddr) != uprobe->offset) 1062 goto unlock; 1063 1064 if (is_register) { 1065 /* consult only the "caller", new consumer. */ 1066 if (consumer_filter(new, 1067 UPROBE_FILTER_REGISTER, mm)) 1068 err = install_breakpoint(uprobe, mm, vma, info->vaddr); 1069 } else if (test_bit(MMF_HAS_UPROBES, &mm->flags)) { 1070 if (!filter_chain(uprobe, 1071 UPROBE_FILTER_UNREGISTER, mm)) 1072 err |= remove_breakpoint(uprobe, mm, info->vaddr); 1073 } 1074 1075 unlock: 1076 mmap_write_unlock(mm); 1077 free: 1078 mmput(mm); 1079 info = free_map_info(info); 1080 } 1081 out: 1082 percpu_up_write(&dup_mmap_sem); 1083 return err; 1084 } 1085 1086 static void 1087 __uprobe_unregister(struct uprobe *uprobe, struct uprobe_consumer *uc) 1088 { 1089 int err; 1090 1091 if (WARN_ON(!consumer_del(uprobe, uc))) 1092 return; 1093 1094 err = register_for_each_vma(uprobe, NULL); 1095 /* TODO : cant unregister? schedule a worker thread */ 1096 if (!uprobe->consumers && !err) 1097 delete_uprobe(uprobe); 1098 } 1099 1100 /* 1101 * uprobe_unregister - unregister an already registered probe. 1102 * @inode: the file in which the probe has to be removed. 1103 * @offset: offset from the start of the file. 1104 * @uc: identify which probe if multiple probes are colocated. 1105 */ 1106 void uprobe_unregister(struct inode *inode, loff_t offset, struct uprobe_consumer *uc) 1107 { 1108 struct uprobe *uprobe; 1109 1110 uprobe = find_uprobe(inode, offset); 1111 if (WARN_ON(!uprobe)) 1112 return; 1113 1114 down_write(&uprobe->register_rwsem); 1115 __uprobe_unregister(uprobe, uc); 1116 up_write(&uprobe->register_rwsem); 1117 put_uprobe(uprobe); 1118 } 1119 EXPORT_SYMBOL_GPL(uprobe_unregister); 1120 1121 /* 1122 * __uprobe_register - register a probe 1123 * @inode: the file in which the probe has to be placed. 1124 * @offset: offset from the start of the file. 1125 * @uc: information on howto handle the probe.. 1126 * 1127 * Apart from the access refcount, __uprobe_register() takes a creation 1128 * refcount (thro alloc_uprobe) if and only if this @uprobe is getting 1129 * inserted into the rbtree (i.e first consumer for a @inode:@offset 1130 * tuple). Creation refcount stops uprobe_unregister from freeing the 1131 * @uprobe even before the register operation is complete. Creation 1132 * refcount is released when the last @uc for the @uprobe 1133 * unregisters. Caller of __uprobe_register() is required to keep @inode 1134 * (and the containing mount) referenced. 1135 * 1136 * Return errno if it cannot successully install probes 1137 * else return 0 (success) 1138 */ 1139 static int __uprobe_register(struct inode *inode, loff_t offset, 1140 loff_t ref_ctr_offset, struct uprobe_consumer *uc) 1141 { 1142 struct uprobe *uprobe; 1143 int ret; 1144 1145 /* Uprobe must have at least one set consumer */ 1146 if (!uc->handler && !uc->ret_handler) 1147 return -EINVAL; 1148 1149 /* copy_insn() uses read_mapping_page() or shmem_read_mapping_page() */ 1150 if (!inode->i_mapping->a_ops->readpage && !shmem_mapping(inode->i_mapping)) 1151 return -EIO; 1152 /* Racy, just to catch the obvious mistakes */ 1153 if (offset > i_size_read(inode)) 1154 return -EINVAL; 1155 1156 /* 1157 * This ensures that copy_from_page(), copy_to_page() and 1158 * __update_ref_ctr() can't cross page boundary. 1159 */ 1160 if (!IS_ALIGNED(offset, UPROBE_SWBP_INSN_SIZE)) 1161 return -EINVAL; 1162 if (!IS_ALIGNED(ref_ctr_offset, sizeof(short))) 1163 return -EINVAL; 1164 1165 retry: 1166 uprobe = alloc_uprobe(inode, offset, ref_ctr_offset); 1167 if (!uprobe) 1168 return -ENOMEM; 1169 if (IS_ERR(uprobe)) 1170 return PTR_ERR(uprobe); 1171 1172 /* 1173 * We can race with uprobe_unregister()->delete_uprobe(). 1174 * Check uprobe_is_active() and retry if it is false. 1175 */ 1176 down_write(&uprobe->register_rwsem); 1177 ret = -EAGAIN; 1178 if (likely(uprobe_is_active(uprobe))) { 1179 consumer_add(uprobe, uc); 1180 ret = register_for_each_vma(uprobe, uc); 1181 if (ret) 1182 __uprobe_unregister(uprobe, uc); 1183 } 1184 up_write(&uprobe->register_rwsem); 1185 put_uprobe(uprobe); 1186 1187 if (unlikely(ret == -EAGAIN)) 1188 goto retry; 1189 return ret; 1190 } 1191 1192 int uprobe_register(struct inode *inode, loff_t offset, 1193 struct uprobe_consumer *uc) 1194 { 1195 return __uprobe_register(inode, offset, 0, uc); 1196 } 1197 EXPORT_SYMBOL_GPL(uprobe_register); 1198 1199 int uprobe_register_refctr(struct inode *inode, loff_t offset, 1200 loff_t ref_ctr_offset, struct uprobe_consumer *uc) 1201 { 1202 return __uprobe_register(inode, offset, ref_ctr_offset, uc); 1203 } 1204 EXPORT_SYMBOL_GPL(uprobe_register_refctr); 1205 1206 /* 1207 * uprobe_apply - unregister an already registered probe. 1208 * @inode: the file in which the probe has to be removed. 1209 * @offset: offset from the start of the file. 1210 * @uc: consumer which wants to add more or remove some breakpoints 1211 * @add: add or remove the breakpoints 1212 */ 1213 int uprobe_apply(struct inode *inode, loff_t offset, 1214 struct uprobe_consumer *uc, bool add) 1215 { 1216 struct uprobe *uprobe; 1217 struct uprobe_consumer *con; 1218 int ret = -ENOENT; 1219 1220 uprobe = find_uprobe(inode, offset); 1221 if (WARN_ON(!uprobe)) 1222 return ret; 1223 1224 down_write(&uprobe->register_rwsem); 1225 for (con = uprobe->consumers; con && con != uc ; con = con->next) 1226 ; 1227 if (con) 1228 ret = register_for_each_vma(uprobe, add ? uc : NULL); 1229 up_write(&uprobe->register_rwsem); 1230 put_uprobe(uprobe); 1231 1232 return ret; 1233 } 1234 1235 static int unapply_uprobe(struct uprobe *uprobe, struct mm_struct *mm) 1236 { 1237 struct vm_area_struct *vma; 1238 int err = 0; 1239 1240 mmap_read_lock(mm); 1241 for (vma = mm->mmap; vma; vma = vma->vm_next) { 1242 unsigned long vaddr; 1243 loff_t offset; 1244 1245 if (!valid_vma(vma, false) || 1246 file_inode(vma->vm_file) != uprobe->inode) 1247 continue; 1248 1249 offset = (loff_t)vma->vm_pgoff << PAGE_SHIFT; 1250 if (uprobe->offset < offset || 1251 uprobe->offset >= offset + vma->vm_end - vma->vm_start) 1252 continue; 1253 1254 vaddr = offset_to_vaddr(vma, uprobe->offset); 1255 err |= remove_breakpoint(uprobe, mm, vaddr); 1256 } 1257 mmap_read_unlock(mm); 1258 1259 return err; 1260 } 1261 1262 static struct rb_node * 1263 find_node_in_range(struct inode *inode, loff_t min, loff_t max) 1264 { 1265 struct rb_node *n = uprobes_tree.rb_node; 1266 1267 while (n) { 1268 struct uprobe *u = rb_entry(n, struct uprobe, rb_node); 1269 1270 if (inode < u->inode) { 1271 n = n->rb_left; 1272 } else if (inode > u->inode) { 1273 n = n->rb_right; 1274 } else { 1275 if (max < u->offset) 1276 n = n->rb_left; 1277 else if (min > u->offset) 1278 n = n->rb_right; 1279 else 1280 break; 1281 } 1282 } 1283 1284 return n; 1285 } 1286 1287 /* 1288 * For a given range in vma, build a list of probes that need to be inserted. 1289 */ 1290 static void build_probe_list(struct inode *inode, 1291 struct vm_area_struct *vma, 1292 unsigned long start, unsigned long end, 1293 struct list_head *head) 1294 { 1295 loff_t min, max; 1296 struct rb_node *n, *t; 1297 struct uprobe *u; 1298 1299 INIT_LIST_HEAD(head); 1300 min = vaddr_to_offset(vma, start); 1301 max = min + (end - start) - 1; 1302 1303 spin_lock(&uprobes_treelock); 1304 n = find_node_in_range(inode, min, max); 1305 if (n) { 1306 for (t = n; t; t = rb_prev(t)) { 1307 u = rb_entry(t, struct uprobe, rb_node); 1308 if (u->inode != inode || u->offset < min) 1309 break; 1310 list_add(&u->pending_list, head); 1311 get_uprobe(u); 1312 } 1313 for (t = n; (t = rb_next(t)); ) { 1314 u = rb_entry(t, struct uprobe, rb_node); 1315 if (u->inode != inode || u->offset > max) 1316 break; 1317 list_add(&u->pending_list, head); 1318 get_uprobe(u); 1319 } 1320 } 1321 spin_unlock(&uprobes_treelock); 1322 } 1323 1324 /* @vma contains reference counter, not the probed instruction. */ 1325 static int delayed_ref_ctr_inc(struct vm_area_struct *vma) 1326 { 1327 struct list_head *pos, *q; 1328 struct delayed_uprobe *du; 1329 unsigned long vaddr; 1330 int ret = 0, err = 0; 1331 1332 mutex_lock(&delayed_uprobe_lock); 1333 list_for_each_safe(pos, q, &delayed_uprobe_list) { 1334 du = list_entry(pos, struct delayed_uprobe, list); 1335 1336 if (du->mm != vma->vm_mm || 1337 !valid_ref_ctr_vma(du->uprobe, vma)) 1338 continue; 1339 1340 vaddr = offset_to_vaddr(vma, du->uprobe->ref_ctr_offset); 1341 ret = __update_ref_ctr(vma->vm_mm, vaddr, 1); 1342 if (ret) { 1343 update_ref_ctr_warn(du->uprobe, vma->vm_mm, 1); 1344 if (!err) 1345 err = ret; 1346 } 1347 delayed_uprobe_delete(du); 1348 } 1349 mutex_unlock(&delayed_uprobe_lock); 1350 return err; 1351 } 1352 1353 /* 1354 * Called from mmap_region/vma_adjust with mm->mmap_lock acquired. 1355 * 1356 * Currently we ignore all errors and always return 0, the callers 1357 * can't handle the failure anyway. 1358 */ 1359 int uprobe_mmap(struct vm_area_struct *vma) 1360 { 1361 struct list_head tmp_list; 1362 struct uprobe *uprobe, *u; 1363 struct inode *inode; 1364 1365 if (no_uprobe_events()) 1366 return 0; 1367 1368 if (vma->vm_file && 1369 (vma->vm_flags & (VM_WRITE|VM_SHARED)) == VM_WRITE && 1370 test_bit(MMF_HAS_UPROBES, &vma->vm_mm->flags)) 1371 delayed_ref_ctr_inc(vma); 1372 1373 if (!valid_vma(vma, true)) 1374 return 0; 1375 1376 inode = file_inode(vma->vm_file); 1377 if (!inode) 1378 return 0; 1379 1380 mutex_lock(uprobes_mmap_hash(inode)); 1381 build_probe_list(inode, vma, vma->vm_start, vma->vm_end, &tmp_list); 1382 /* 1383 * We can race with uprobe_unregister(), this uprobe can be already 1384 * removed. But in this case filter_chain() must return false, all 1385 * consumers have gone away. 1386 */ 1387 list_for_each_entry_safe(uprobe, u, &tmp_list, pending_list) { 1388 if (!fatal_signal_pending(current) && 1389 filter_chain(uprobe, UPROBE_FILTER_MMAP, vma->vm_mm)) { 1390 unsigned long vaddr = offset_to_vaddr(vma, uprobe->offset); 1391 install_breakpoint(uprobe, vma->vm_mm, vma, vaddr); 1392 } 1393 put_uprobe(uprobe); 1394 } 1395 mutex_unlock(uprobes_mmap_hash(inode)); 1396 1397 return 0; 1398 } 1399 1400 static bool 1401 vma_has_uprobes(struct vm_area_struct *vma, unsigned long start, unsigned long end) 1402 { 1403 loff_t min, max; 1404 struct inode *inode; 1405 struct rb_node *n; 1406 1407 inode = file_inode(vma->vm_file); 1408 1409 min = vaddr_to_offset(vma, start); 1410 max = min + (end - start) - 1; 1411 1412 spin_lock(&uprobes_treelock); 1413 n = find_node_in_range(inode, min, max); 1414 spin_unlock(&uprobes_treelock); 1415 1416 return !!n; 1417 } 1418 1419 /* 1420 * Called in context of a munmap of a vma. 1421 */ 1422 void uprobe_munmap(struct vm_area_struct *vma, unsigned long start, unsigned long end) 1423 { 1424 if (no_uprobe_events() || !valid_vma(vma, false)) 1425 return; 1426 1427 if (!atomic_read(&vma->vm_mm->mm_users)) /* called by mmput() ? */ 1428 return; 1429 1430 if (!test_bit(MMF_HAS_UPROBES, &vma->vm_mm->flags) || 1431 test_bit(MMF_RECALC_UPROBES, &vma->vm_mm->flags)) 1432 return; 1433 1434 if (vma_has_uprobes(vma, start, end)) 1435 set_bit(MMF_RECALC_UPROBES, &vma->vm_mm->flags); 1436 } 1437 1438 /* Slot allocation for XOL */ 1439 static int xol_add_vma(struct mm_struct *mm, struct xol_area *area) 1440 { 1441 struct vm_area_struct *vma; 1442 int ret; 1443 1444 if (mmap_write_lock_killable(mm)) 1445 return -EINTR; 1446 1447 if (mm->uprobes_state.xol_area) { 1448 ret = -EALREADY; 1449 goto fail; 1450 } 1451 1452 if (!area->vaddr) { 1453 /* Try to map as high as possible, this is only a hint. */ 1454 area->vaddr = get_unmapped_area(NULL, TASK_SIZE - PAGE_SIZE, 1455 PAGE_SIZE, 0, 0); 1456 if (IS_ERR_VALUE(area->vaddr)) { 1457 ret = area->vaddr; 1458 goto fail; 1459 } 1460 } 1461 1462 vma = _install_special_mapping(mm, area->vaddr, PAGE_SIZE, 1463 VM_EXEC|VM_MAYEXEC|VM_DONTCOPY|VM_IO, 1464 &area->xol_mapping); 1465 if (IS_ERR(vma)) { 1466 ret = PTR_ERR(vma); 1467 goto fail; 1468 } 1469 1470 ret = 0; 1471 /* pairs with get_xol_area() */ 1472 smp_store_release(&mm->uprobes_state.xol_area, area); /* ^^^ */ 1473 fail: 1474 mmap_write_unlock(mm); 1475 1476 return ret; 1477 } 1478 1479 static struct xol_area *__create_xol_area(unsigned long vaddr) 1480 { 1481 struct mm_struct *mm = current->mm; 1482 uprobe_opcode_t insn = UPROBE_SWBP_INSN; 1483 struct xol_area *area; 1484 1485 area = kmalloc(sizeof(*area), GFP_KERNEL); 1486 if (unlikely(!area)) 1487 goto out; 1488 1489 area->bitmap = kcalloc(BITS_TO_LONGS(UINSNS_PER_PAGE), sizeof(long), 1490 GFP_KERNEL); 1491 if (!area->bitmap) 1492 goto free_area; 1493 1494 area->xol_mapping.name = "[uprobes]"; 1495 area->xol_mapping.fault = NULL; 1496 area->xol_mapping.pages = area->pages; 1497 area->pages[0] = alloc_page(GFP_HIGHUSER); 1498 if (!area->pages[0]) 1499 goto free_bitmap; 1500 area->pages[1] = NULL; 1501 1502 area->vaddr = vaddr; 1503 init_waitqueue_head(&area->wq); 1504 /* Reserve the 1st slot for get_trampoline_vaddr() */ 1505 set_bit(0, area->bitmap); 1506 atomic_set(&area->slot_count, 1); 1507 arch_uprobe_copy_ixol(area->pages[0], 0, &insn, UPROBE_SWBP_INSN_SIZE); 1508 1509 if (!xol_add_vma(mm, area)) 1510 return area; 1511 1512 __free_page(area->pages[0]); 1513 free_bitmap: 1514 kfree(area->bitmap); 1515 free_area: 1516 kfree(area); 1517 out: 1518 return NULL; 1519 } 1520 1521 /* 1522 * get_xol_area - Allocate process's xol_area if necessary. 1523 * This area will be used for storing instructions for execution out of line. 1524 * 1525 * Returns the allocated area or NULL. 1526 */ 1527 static struct xol_area *get_xol_area(void) 1528 { 1529 struct mm_struct *mm = current->mm; 1530 struct xol_area *area; 1531 1532 if (!mm->uprobes_state.xol_area) 1533 __create_xol_area(0); 1534 1535 /* Pairs with xol_add_vma() smp_store_release() */ 1536 area = READ_ONCE(mm->uprobes_state.xol_area); /* ^^^ */ 1537 return area; 1538 } 1539 1540 /* 1541 * uprobe_clear_state - Free the area allocated for slots. 1542 */ 1543 void uprobe_clear_state(struct mm_struct *mm) 1544 { 1545 struct xol_area *area = mm->uprobes_state.xol_area; 1546 1547 mutex_lock(&delayed_uprobe_lock); 1548 delayed_uprobe_remove(NULL, mm); 1549 mutex_unlock(&delayed_uprobe_lock); 1550 1551 if (!area) 1552 return; 1553 1554 put_page(area->pages[0]); 1555 kfree(area->bitmap); 1556 kfree(area); 1557 } 1558 1559 void uprobe_start_dup_mmap(void) 1560 { 1561 percpu_down_read(&dup_mmap_sem); 1562 } 1563 1564 void uprobe_end_dup_mmap(void) 1565 { 1566 percpu_up_read(&dup_mmap_sem); 1567 } 1568 1569 void uprobe_dup_mmap(struct mm_struct *oldmm, struct mm_struct *newmm) 1570 { 1571 if (test_bit(MMF_HAS_UPROBES, &oldmm->flags)) { 1572 set_bit(MMF_HAS_UPROBES, &newmm->flags); 1573 /* unconditionally, dup_mmap() skips VM_DONTCOPY vmas */ 1574 set_bit(MMF_RECALC_UPROBES, &newmm->flags); 1575 } 1576 } 1577 1578 /* 1579 * - search for a free slot. 1580 */ 1581 static unsigned long xol_take_insn_slot(struct xol_area *area) 1582 { 1583 unsigned long slot_addr; 1584 int slot_nr; 1585 1586 do { 1587 slot_nr = find_first_zero_bit(area->bitmap, UINSNS_PER_PAGE); 1588 if (slot_nr < UINSNS_PER_PAGE) { 1589 if (!test_and_set_bit(slot_nr, area->bitmap)) 1590 break; 1591 1592 slot_nr = UINSNS_PER_PAGE; 1593 continue; 1594 } 1595 wait_event(area->wq, (atomic_read(&area->slot_count) < UINSNS_PER_PAGE)); 1596 } while (slot_nr >= UINSNS_PER_PAGE); 1597 1598 slot_addr = area->vaddr + (slot_nr * UPROBE_XOL_SLOT_BYTES); 1599 atomic_inc(&area->slot_count); 1600 1601 return slot_addr; 1602 } 1603 1604 /* 1605 * xol_get_insn_slot - allocate a slot for xol. 1606 * Returns the allocated slot address or 0. 1607 */ 1608 static unsigned long xol_get_insn_slot(struct uprobe *uprobe) 1609 { 1610 struct xol_area *area; 1611 unsigned long xol_vaddr; 1612 1613 area = get_xol_area(); 1614 if (!area) 1615 return 0; 1616 1617 xol_vaddr = xol_take_insn_slot(area); 1618 if (unlikely(!xol_vaddr)) 1619 return 0; 1620 1621 arch_uprobe_copy_ixol(area->pages[0], xol_vaddr, 1622 &uprobe->arch.ixol, sizeof(uprobe->arch.ixol)); 1623 1624 return xol_vaddr; 1625 } 1626 1627 /* 1628 * xol_free_insn_slot - If slot was earlier allocated by 1629 * @xol_get_insn_slot(), make the slot available for 1630 * subsequent requests. 1631 */ 1632 static void xol_free_insn_slot(struct task_struct *tsk) 1633 { 1634 struct xol_area *area; 1635 unsigned long vma_end; 1636 unsigned long slot_addr; 1637 1638 if (!tsk->mm || !tsk->mm->uprobes_state.xol_area || !tsk->utask) 1639 return; 1640 1641 slot_addr = tsk->utask->xol_vaddr; 1642 if (unlikely(!slot_addr)) 1643 return; 1644 1645 area = tsk->mm->uprobes_state.xol_area; 1646 vma_end = area->vaddr + PAGE_SIZE; 1647 if (area->vaddr <= slot_addr && slot_addr < vma_end) { 1648 unsigned long offset; 1649 int slot_nr; 1650 1651 offset = slot_addr - area->vaddr; 1652 slot_nr = offset / UPROBE_XOL_SLOT_BYTES; 1653 if (slot_nr >= UINSNS_PER_PAGE) 1654 return; 1655 1656 clear_bit(slot_nr, area->bitmap); 1657 atomic_dec(&area->slot_count); 1658 smp_mb__after_atomic(); /* pairs with prepare_to_wait() */ 1659 if (waitqueue_active(&area->wq)) 1660 wake_up(&area->wq); 1661 1662 tsk->utask->xol_vaddr = 0; 1663 } 1664 } 1665 1666 void __weak arch_uprobe_copy_ixol(struct page *page, unsigned long vaddr, 1667 void *src, unsigned long len) 1668 { 1669 /* Initialize the slot */ 1670 copy_to_page(page, vaddr, src, len); 1671 1672 /* 1673 * We probably need flush_icache_user_page() but it needs vma. 1674 * This should work on most of architectures by default. If 1675 * architecture needs to do something different it can define 1676 * its own version of the function. 1677 */ 1678 flush_dcache_page(page); 1679 } 1680 1681 /** 1682 * uprobe_get_swbp_addr - compute address of swbp given post-swbp regs 1683 * @regs: Reflects the saved state of the task after it has hit a breakpoint 1684 * instruction. 1685 * Return the address of the breakpoint instruction. 1686 */ 1687 unsigned long __weak uprobe_get_swbp_addr(struct pt_regs *regs) 1688 { 1689 return instruction_pointer(regs) - UPROBE_SWBP_INSN_SIZE; 1690 } 1691 1692 unsigned long uprobe_get_trap_addr(struct pt_regs *regs) 1693 { 1694 struct uprobe_task *utask = current->utask; 1695 1696 if (unlikely(utask && utask->active_uprobe)) 1697 return utask->vaddr; 1698 1699 return instruction_pointer(regs); 1700 } 1701 1702 static struct return_instance *free_ret_instance(struct return_instance *ri) 1703 { 1704 struct return_instance *next = ri->next; 1705 put_uprobe(ri->uprobe); 1706 kfree(ri); 1707 return next; 1708 } 1709 1710 /* 1711 * Called with no locks held. 1712 * Called in context of an exiting or an exec-ing thread. 1713 */ 1714 void uprobe_free_utask(struct task_struct *t) 1715 { 1716 struct uprobe_task *utask = t->utask; 1717 struct return_instance *ri; 1718 1719 if (!utask) 1720 return; 1721 1722 if (utask->active_uprobe) 1723 put_uprobe(utask->active_uprobe); 1724 1725 ri = utask->return_instances; 1726 while (ri) 1727 ri = free_ret_instance(ri); 1728 1729 xol_free_insn_slot(t); 1730 kfree(utask); 1731 t->utask = NULL; 1732 } 1733 1734 /* 1735 * Allocate a uprobe_task object for the task if necessary. 1736 * Called when the thread hits a breakpoint. 1737 * 1738 * Returns: 1739 * - pointer to new uprobe_task on success 1740 * - NULL otherwise 1741 */ 1742 static struct uprobe_task *get_utask(void) 1743 { 1744 if (!current->utask) 1745 current->utask = kzalloc(sizeof(struct uprobe_task), GFP_KERNEL); 1746 return current->utask; 1747 } 1748 1749 static int dup_utask(struct task_struct *t, struct uprobe_task *o_utask) 1750 { 1751 struct uprobe_task *n_utask; 1752 struct return_instance **p, *o, *n; 1753 1754 n_utask = kzalloc(sizeof(struct uprobe_task), GFP_KERNEL); 1755 if (!n_utask) 1756 return -ENOMEM; 1757 t->utask = n_utask; 1758 1759 p = &n_utask->return_instances; 1760 for (o = o_utask->return_instances; o; o = o->next) { 1761 n = kmalloc(sizeof(struct return_instance), GFP_KERNEL); 1762 if (!n) 1763 return -ENOMEM; 1764 1765 *n = *o; 1766 get_uprobe(n->uprobe); 1767 n->next = NULL; 1768 1769 *p = n; 1770 p = &n->next; 1771 n_utask->depth++; 1772 } 1773 1774 return 0; 1775 } 1776 1777 static void uprobe_warn(struct task_struct *t, const char *msg) 1778 { 1779 pr_warn("uprobe: %s:%d failed to %s\n", 1780 current->comm, current->pid, msg); 1781 } 1782 1783 static void dup_xol_work(struct callback_head *work) 1784 { 1785 if (current->flags & PF_EXITING) 1786 return; 1787 1788 if (!__create_xol_area(current->utask->dup_xol_addr) && 1789 !fatal_signal_pending(current)) 1790 uprobe_warn(current, "dup xol area"); 1791 } 1792 1793 /* 1794 * Called in context of a new clone/fork from copy_process. 1795 */ 1796 void uprobe_copy_process(struct task_struct *t, unsigned long flags) 1797 { 1798 struct uprobe_task *utask = current->utask; 1799 struct mm_struct *mm = current->mm; 1800 struct xol_area *area; 1801 1802 t->utask = NULL; 1803 1804 if (!utask || !utask->return_instances) 1805 return; 1806 1807 if (mm == t->mm && !(flags & CLONE_VFORK)) 1808 return; 1809 1810 if (dup_utask(t, utask)) 1811 return uprobe_warn(t, "dup ret instances"); 1812 1813 /* The task can fork() after dup_xol_work() fails */ 1814 area = mm->uprobes_state.xol_area; 1815 if (!area) 1816 return uprobe_warn(t, "dup xol area"); 1817 1818 if (mm == t->mm) 1819 return; 1820 1821 t->utask->dup_xol_addr = area->vaddr; 1822 init_task_work(&t->utask->dup_xol_work, dup_xol_work); 1823 task_work_add(t, &t->utask->dup_xol_work, TWA_RESUME); 1824 } 1825 1826 /* 1827 * Current area->vaddr notion assume the trampoline address is always 1828 * equal area->vaddr. 1829 * 1830 * Returns -1 in case the xol_area is not allocated. 1831 */ 1832 static unsigned long get_trampoline_vaddr(void) 1833 { 1834 struct xol_area *area; 1835 unsigned long trampoline_vaddr = -1; 1836 1837 /* Pairs with xol_add_vma() smp_store_release() */ 1838 area = READ_ONCE(current->mm->uprobes_state.xol_area); /* ^^^ */ 1839 if (area) 1840 trampoline_vaddr = area->vaddr; 1841 1842 return trampoline_vaddr; 1843 } 1844 1845 static void cleanup_return_instances(struct uprobe_task *utask, bool chained, 1846 struct pt_regs *regs) 1847 { 1848 struct return_instance *ri = utask->return_instances; 1849 enum rp_check ctx = chained ? RP_CHECK_CHAIN_CALL : RP_CHECK_CALL; 1850 1851 while (ri && !arch_uretprobe_is_alive(ri, ctx, regs)) { 1852 ri = free_ret_instance(ri); 1853 utask->depth--; 1854 } 1855 utask->return_instances = ri; 1856 } 1857 1858 static void prepare_uretprobe(struct uprobe *uprobe, struct pt_regs *regs) 1859 { 1860 struct return_instance *ri; 1861 struct uprobe_task *utask; 1862 unsigned long orig_ret_vaddr, trampoline_vaddr; 1863 bool chained; 1864 1865 if (!get_xol_area()) 1866 return; 1867 1868 utask = get_utask(); 1869 if (!utask) 1870 return; 1871 1872 if (utask->depth >= MAX_URETPROBE_DEPTH) { 1873 printk_ratelimited(KERN_INFO "uprobe: omit uretprobe due to" 1874 " nestedness limit pid/tgid=%d/%d\n", 1875 current->pid, current->tgid); 1876 return; 1877 } 1878 1879 ri = kmalloc(sizeof(struct return_instance), GFP_KERNEL); 1880 if (!ri) 1881 return; 1882 1883 trampoline_vaddr = get_trampoline_vaddr(); 1884 orig_ret_vaddr = arch_uretprobe_hijack_return_addr(trampoline_vaddr, regs); 1885 if (orig_ret_vaddr == -1) 1886 goto fail; 1887 1888 /* drop the entries invalidated by longjmp() */ 1889 chained = (orig_ret_vaddr == trampoline_vaddr); 1890 cleanup_return_instances(utask, chained, regs); 1891 1892 /* 1893 * We don't want to keep trampoline address in stack, rather keep the 1894 * original return address of first caller thru all the consequent 1895 * instances. This also makes breakpoint unwrapping easier. 1896 */ 1897 if (chained) { 1898 if (!utask->return_instances) { 1899 /* 1900 * This situation is not possible. Likely we have an 1901 * attack from user-space. 1902 */ 1903 uprobe_warn(current, "handle tail call"); 1904 goto fail; 1905 } 1906 orig_ret_vaddr = utask->return_instances->orig_ret_vaddr; 1907 } 1908 1909 ri->uprobe = get_uprobe(uprobe); 1910 ri->func = instruction_pointer(regs); 1911 ri->stack = user_stack_pointer(regs); 1912 ri->orig_ret_vaddr = orig_ret_vaddr; 1913 ri->chained = chained; 1914 1915 utask->depth++; 1916 ri->next = utask->return_instances; 1917 utask->return_instances = ri; 1918 1919 return; 1920 fail: 1921 kfree(ri); 1922 } 1923 1924 /* Prepare to single-step probed instruction out of line. */ 1925 static int 1926 pre_ssout(struct uprobe *uprobe, struct pt_regs *regs, unsigned long bp_vaddr) 1927 { 1928 struct uprobe_task *utask; 1929 unsigned long xol_vaddr; 1930 int err; 1931 1932 utask = get_utask(); 1933 if (!utask) 1934 return -ENOMEM; 1935 1936 xol_vaddr = xol_get_insn_slot(uprobe); 1937 if (!xol_vaddr) 1938 return -ENOMEM; 1939 1940 utask->xol_vaddr = xol_vaddr; 1941 utask->vaddr = bp_vaddr; 1942 1943 err = arch_uprobe_pre_xol(&uprobe->arch, regs); 1944 if (unlikely(err)) { 1945 xol_free_insn_slot(current); 1946 return err; 1947 } 1948 1949 utask->active_uprobe = uprobe; 1950 utask->state = UTASK_SSTEP; 1951 return 0; 1952 } 1953 1954 /* 1955 * If we are singlestepping, then ensure this thread is not connected to 1956 * non-fatal signals until completion of singlestep. When xol insn itself 1957 * triggers the signal, restart the original insn even if the task is 1958 * already SIGKILL'ed (since coredump should report the correct ip). This 1959 * is even more important if the task has a handler for SIGSEGV/etc, The 1960 * _same_ instruction should be repeated again after return from the signal 1961 * handler, and SSTEP can never finish in this case. 1962 */ 1963 bool uprobe_deny_signal(void) 1964 { 1965 struct task_struct *t = current; 1966 struct uprobe_task *utask = t->utask; 1967 1968 if (likely(!utask || !utask->active_uprobe)) 1969 return false; 1970 1971 WARN_ON_ONCE(utask->state != UTASK_SSTEP); 1972 1973 if (task_sigpending(t)) { 1974 spin_lock_irq(&t->sighand->siglock); 1975 clear_tsk_thread_flag(t, TIF_SIGPENDING); 1976 spin_unlock_irq(&t->sighand->siglock); 1977 1978 if (__fatal_signal_pending(t) || arch_uprobe_xol_was_trapped(t)) { 1979 utask->state = UTASK_SSTEP_TRAPPED; 1980 set_tsk_thread_flag(t, TIF_UPROBE); 1981 } 1982 } 1983 1984 return true; 1985 } 1986 1987 static void mmf_recalc_uprobes(struct mm_struct *mm) 1988 { 1989 struct vm_area_struct *vma; 1990 1991 for (vma = mm->mmap; vma; vma = vma->vm_next) { 1992 if (!valid_vma(vma, false)) 1993 continue; 1994 /* 1995 * This is not strictly accurate, we can race with 1996 * uprobe_unregister() and see the already removed 1997 * uprobe if delete_uprobe() was not yet called. 1998 * Or this uprobe can be filtered out. 1999 */ 2000 if (vma_has_uprobes(vma, vma->vm_start, vma->vm_end)) 2001 return; 2002 } 2003 2004 clear_bit(MMF_HAS_UPROBES, &mm->flags); 2005 } 2006 2007 static int is_trap_at_addr(struct mm_struct *mm, unsigned long vaddr) 2008 { 2009 struct page *page; 2010 uprobe_opcode_t opcode; 2011 int result; 2012 2013 if (WARN_ON_ONCE(!IS_ALIGNED(vaddr, UPROBE_SWBP_INSN_SIZE))) 2014 return -EINVAL; 2015 2016 pagefault_disable(); 2017 result = __get_user(opcode, (uprobe_opcode_t __user *)vaddr); 2018 pagefault_enable(); 2019 2020 if (likely(result == 0)) 2021 goto out; 2022 2023 /* 2024 * The NULL 'tsk' here ensures that any faults that occur here 2025 * will not be accounted to the task. 'mm' *is* current->mm, 2026 * but we treat this as a 'remote' access since it is 2027 * essentially a kernel access to the memory. 2028 */ 2029 result = get_user_pages_remote(mm, vaddr, 1, FOLL_FORCE, &page, 2030 NULL, NULL); 2031 if (result < 0) 2032 return result; 2033 2034 copy_from_page(page, vaddr, &opcode, UPROBE_SWBP_INSN_SIZE); 2035 put_page(page); 2036 out: 2037 /* This needs to return true for any variant of the trap insn */ 2038 return is_trap_insn(&opcode); 2039 } 2040 2041 static struct uprobe *find_active_uprobe(unsigned long bp_vaddr, int *is_swbp) 2042 { 2043 struct mm_struct *mm = current->mm; 2044 struct uprobe *uprobe = NULL; 2045 struct vm_area_struct *vma; 2046 2047 mmap_read_lock(mm); 2048 vma = vma_lookup(mm, bp_vaddr); 2049 if (vma) { 2050 if (valid_vma(vma, false)) { 2051 struct inode *inode = file_inode(vma->vm_file); 2052 loff_t offset = vaddr_to_offset(vma, bp_vaddr); 2053 2054 uprobe = find_uprobe(inode, offset); 2055 } 2056 2057 if (!uprobe) 2058 *is_swbp = is_trap_at_addr(mm, bp_vaddr); 2059 } else { 2060 *is_swbp = -EFAULT; 2061 } 2062 2063 if (!uprobe && test_and_clear_bit(MMF_RECALC_UPROBES, &mm->flags)) 2064 mmf_recalc_uprobes(mm); 2065 mmap_read_unlock(mm); 2066 2067 return uprobe; 2068 } 2069 2070 static void handler_chain(struct uprobe *uprobe, struct pt_regs *regs) 2071 { 2072 struct uprobe_consumer *uc; 2073 int remove = UPROBE_HANDLER_REMOVE; 2074 bool need_prep = false; /* prepare return uprobe, when needed */ 2075 2076 down_read(&uprobe->register_rwsem); 2077 for (uc = uprobe->consumers; uc; uc = uc->next) { 2078 int rc = 0; 2079 2080 if (uc->handler) { 2081 rc = uc->handler(uc, regs); 2082 WARN(rc & ~UPROBE_HANDLER_MASK, 2083 "bad rc=0x%x from %ps()\n", rc, uc->handler); 2084 } 2085 2086 if (uc->ret_handler) 2087 need_prep = true; 2088 2089 remove &= rc; 2090 } 2091 2092 if (need_prep && !remove) 2093 prepare_uretprobe(uprobe, regs); /* put bp at return */ 2094 2095 if (remove && uprobe->consumers) { 2096 WARN_ON(!uprobe_is_active(uprobe)); 2097 unapply_uprobe(uprobe, current->mm); 2098 } 2099 up_read(&uprobe->register_rwsem); 2100 } 2101 2102 static void 2103 handle_uretprobe_chain(struct return_instance *ri, struct pt_regs *regs) 2104 { 2105 struct uprobe *uprobe = ri->uprobe; 2106 struct uprobe_consumer *uc; 2107 2108 down_read(&uprobe->register_rwsem); 2109 for (uc = uprobe->consumers; uc; uc = uc->next) { 2110 if (uc->ret_handler) 2111 uc->ret_handler(uc, ri->func, regs); 2112 } 2113 up_read(&uprobe->register_rwsem); 2114 } 2115 2116 static struct return_instance *find_next_ret_chain(struct return_instance *ri) 2117 { 2118 bool chained; 2119 2120 do { 2121 chained = ri->chained; 2122 ri = ri->next; /* can't be NULL if chained */ 2123 } while (chained); 2124 2125 return ri; 2126 } 2127 2128 static void handle_trampoline(struct pt_regs *regs) 2129 { 2130 struct uprobe_task *utask; 2131 struct return_instance *ri, *next; 2132 bool valid; 2133 2134 utask = current->utask; 2135 if (!utask) 2136 goto sigill; 2137 2138 ri = utask->return_instances; 2139 if (!ri) 2140 goto sigill; 2141 2142 do { 2143 /* 2144 * We should throw out the frames invalidated by longjmp(). 2145 * If this chain is valid, then the next one should be alive 2146 * or NULL; the latter case means that nobody but ri->func 2147 * could hit this trampoline on return. TODO: sigaltstack(). 2148 */ 2149 next = find_next_ret_chain(ri); 2150 valid = !next || arch_uretprobe_is_alive(next, RP_CHECK_RET, regs); 2151 2152 instruction_pointer_set(regs, ri->orig_ret_vaddr); 2153 do { 2154 if (valid) 2155 handle_uretprobe_chain(ri, regs); 2156 ri = free_ret_instance(ri); 2157 utask->depth--; 2158 } while (ri != next); 2159 } while (!valid); 2160 2161 utask->return_instances = ri; 2162 return; 2163 2164 sigill: 2165 uprobe_warn(current, "handle uretprobe, sending SIGILL."); 2166 force_sig(SIGILL); 2167 2168 } 2169 2170 bool __weak arch_uprobe_ignore(struct arch_uprobe *aup, struct pt_regs *regs) 2171 { 2172 return false; 2173 } 2174 2175 bool __weak arch_uretprobe_is_alive(struct return_instance *ret, enum rp_check ctx, 2176 struct pt_regs *regs) 2177 { 2178 return true; 2179 } 2180 2181 /* 2182 * Run handler and ask thread to singlestep. 2183 * Ensure all non-fatal signals cannot interrupt thread while it singlesteps. 2184 */ 2185 static void handle_swbp(struct pt_regs *regs) 2186 { 2187 struct uprobe *uprobe; 2188 unsigned long bp_vaddr; 2189 int is_swbp; 2190 2191 bp_vaddr = uprobe_get_swbp_addr(regs); 2192 if (bp_vaddr == get_trampoline_vaddr()) 2193 return handle_trampoline(regs); 2194 2195 uprobe = find_active_uprobe(bp_vaddr, &is_swbp); 2196 if (!uprobe) { 2197 if (is_swbp > 0) { 2198 /* No matching uprobe; signal SIGTRAP. */ 2199 force_sig(SIGTRAP); 2200 } else { 2201 /* 2202 * Either we raced with uprobe_unregister() or we can't 2203 * access this memory. The latter is only possible if 2204 * another thread plays with our ->mm. In both cases 2205 * we can simply restart. If this vma was unmapped we 2206 * can pretend this insn was not executed yet and get 2207 * the (correct) SIGSEGV after restart. 2208 */ 2209 instruction_pointer_set(regs, bp_vaddr); 2210 } 2211 return; 2212 } 2213 2214 /* change it in advance for ->handler() and restart */ 2215 instruction_pointer_set(regs, bp_vaddr); 2216 2217 /* 2218 * TODO: move copy_insn/etc into _register and remove this hack. 2219 * After we hit the bp, _unregister + _register can install the 2220 * new and not-yet-analyzed uprobe at the same address, restart. 2221 */ 2222 if (unlikely(!test_bit(UPROBE_COPY_INSN, &uprobe->flags))) 2223 goto out; 2224 2225 /* 2226 * Pairs with the smp_wmb() in prepare_uprobe(). 2227 * 2228 * Guarantees that if we see the UPROBE_COPY_INSN bit set, then 2229 * we must also see the stores to &uprobe->arch performed by the 2230 * prepare_uprobe() call. 2231 */ 2232 smp_rmb(); 2233 2234 /* Tracing handlers use ->utask to communicate with fetch methods */ 2235 if (!get_utask()) 2236 goto out; 2237 2238 if (arch_uprobe_ignore(&uprobe->arch, regs)) 2239 goto out; 2240 2241 handler_chain(uprobe, regs); 2242 2243 if (arch_uprobe_skip_sstep(&uprobe->arch, regs)) 2244 goto out; 2245 2246 if (!pre_ssout(uprobe, regs, bp_vaddr)) 2247 return; 2248 2249 /* arch_uprobe_skip_sstep() succeeded, or restart if can't singlestep */ 2250 out: 2251 put_uprobe(uprobe); 2252 } 2253 2254 /* 2255 * Perform required fix-ups and disable singlestep. 2256 * Allow pending signals to take effect. 2257 */ 2258 static void handle_singlestep(struct uprobe_task *utask, struct pt_regs *regs) 2259 { 2260 struct uprobe *uprobe; 2261 int err = 0; 2262 2263 uprobe = utask->active_uprobe; 2264 if (utask->state == UTASK_SSTEP_ACK) 2265 err = arch_uprobe_post_xol(&uprobe->arch, regs); 2266 else if (utask->state == UTASK_SSTEP_TRAPPED) 2267 arch_uprobe_abort_xol(&uprobe->arch, regs); 2268 else 2269 WARN_ON_ONCE(1); 2270 2271 put_uprobe(uprobe); 2272 utask->active_uprobe = NULL; 2273 utask->state = UTASK_RUNNING; 2274 xol_free_insn_slot(current); 2275 2276 spin_lock_irq(¤t->sighand->siglock); 2277 recalc_sigpending(); /* see uprobe_deny_signal() */ 2278 spin_unlock_irq(¤t->sighand->siglock); 2279 2280 if (unlikely(err)) { 2281 uprobe_warn(current, "execute the probed insn, sending SIGILL."); 2282 force_sig(SIGILL); 2283 } 2284 } 2285 2286 /* 2287 * On breakpoint hit, breakpoint notifier sets the TIF_UPROBE flag and 2288 * allows the thread to return from interrupt. After that handle_swbp() 2289 * sets utask->active_uprobe. 2290 * 2291 * On singlestep exception, singlestep notifier sets the TIF_UPROBE flag 2292 * and allows the thread to return from interrupt. 2293 * 2294 * While returning to userspace, thread notices the TIF_UPROBE flag and calls 2295 * uprobe_notify_resume(). 2296 */ 2297 void uprobe_notify_resume(struct pt_regs *regs) 2298 { 2299 struct uprobe_task *utask; 2300 2301 clear_thread_flag(TIF_UPROBE); 2302 2303 utask = current->utask; 2304 if (utask && utask->active_uprobe) 2305 handle_singlestep(utask, regs); 2306 else 2307 handle_swbp(regs); 2308 } 2309 2310 /* 2311 * uprobe_pre_sstep_notifier gets called from interrupt context as part of 2312 * notifier mechanism. Set TIF_UPROBE flag and indicate breakpoint hit. 2313 */ 2314 int uprobe_pre_sstep_notifier(struct pt_regs *regs) 2315 { 2316 if (!current->mm) 2317 return 0; 2318 2319 if (!test_bit(MMF_HAS_UPROBES, ¤t->mm->flags) && 2320 (!current->utask || !current->utask->return_instances)) 2321 return 0; 2322 2323 set_thread_flag(TIF_UPROBE); 2324 return 1; 2325 } 2326 2327 /* 2328 * uprobe_post_sstep_notifier gets called in interrupt context as part of notifier 2329 * mechanism. Set TIF_UPROBE flag and indicate completion of singlestep. 2330 */ 2331 int uprobe_post_sstep_notifier(struct pt_regs *regs) 2332 { 2333 struct uprobe_task *utask = current->utask; 2334 2335 if (!current->mm || !utask || !utask->active_uprobe) 2336 /* task is currently not uprobed */ 2337 return 0; 2338 2339 utask->state = UTASK_SSTEP_ACK; 2340 set_thread_flag(TIF_UPROBE); 2341 return 1; 2342 } 2343 2344 static struct notifier_block uprobe_exception_nb = { 2345 .notifier_call = arch_uprobe_exception_notify, 2346 .priority = INT_MAX-1, /* notified after kprobes, kgdb */ 2347 }; 2348 2349 void __init uprobes_init(void) 2350 { 2351 int i; 2352 2353 for (i = 0; i < UPROBES_HASH_SZ; i++) 2354 mutex_init(&uprobes_mmap_mutex[i]); 2355 2356 BUG_ON(register_die_notifier(&uprobe_exception_nb)); 2357 } 2358