1 /* 2 * Kernel Probes (KProbes) 3 * kernel/kprobes.c 4 * 5 * This program is free software; you can redistribute it and/or modify 6 * it under the terms of the GNU General Public License as published by 7 * the Free Software Foundation; either version 2 of the License, or 8 * (at your option) any later version. 9 * 10 * This program is distributed in the hope that it will be useful, 11 * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 * GNU General Public License for more details. 14 * 15 * You should have received a copy of the GNU General Public License 16 * along with this program; if not, write to the Free Software 17 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 18 * 19 * Copyright (C) IBM Corporation, 2002, 2004 20 * 21 * 2002-Oct Created by Vamsi Krishna S <vamsi_krishna@in.ibm.com> Kernel 22 * Probes initial implementation (includes suggestions from 23 * Rusty Russell). 24 * 2004-Aug Updated by Prasanna S Panchamukhi <prasanna@in.ibm.com> with 25 * hlists and exceptions notifier as suggested by Andi Kleen. 26 * 2004-July Suparna Bhattacharya <suparna@in.ibm.com> added jumper probes 27 * interface to access function arguments. 28 * 2004-Sep Prasanna S Panchamukhi <prasanna@in.ibm.com> Changed Kprobes 29 * exceptions notifier to be first on the priority list. 30 * 2005-May Hien Nguyen <hien@us.ibm.com>, Jim Keniston 31 * <jkenisto@us.ibm.com> and Prasanna S Panchamukhi 32 * <prasanna@in.ibm.com> added function-return probes. 33 */ 34 #include <linux/kprobes.h> 35 #include <linux/hash.h> 36 #include <linux/init.h> 37 #include <linux/slab.h> 38 #include <linux/stddef.h> 39 #include <linux/module.h> 40 #include <linux/moduleloader.h> 41 #include <linux/kallsyms.h> 42 #include <linux/freezer.h> 43 #include <linux/seq_file.h> 44 #include <linux/debugfs.h> 45 #include <linux/kdebug.h> 46 47 #include <asm-generic/sections.h> 48 #include <asm/cacheflush.h> 49 #include <asm/errno.h> 50 #include <asm/uaccess.h> 51 52 #define KPROBE_HASH_BITS 6 53 #define KPROBE_TABLE_SIZE (1 << KPROBE_HASH_BITS) 54 55 56 /* 57 * Some oddball architectures like 64bit powerpc have function descriptors 58 * so this must be overridable. 59 */ 60 #ifndef kprobe_lookup_name 61 #define kprobe_lookup_name(name, addr) \ 62 addr = ((kprobe_opcode_t *)(kallsyms_lookup_name(name))) 63 #endif 64 65 static struct hlist_head kprobe_table[KPROBE_TABLE_SIZE]; 66 static struct hlist_head kretprobe_inst_table[KPROBE_TABLE_SIZE]; 67 68 /* NOTE: change this value only with kprobe_mutex held */ 69 static bool kprobe_enabled; 70 71 DEFINE_MUTEX(kprobe_mutex); /* Protects kprobe_table */ 72 DEFINE_SPINLOCK(kretprobe_lock); /* Protects kretprobe_inst_table */ 73 static DEFINE_PER_CPU(struct kprobe *, kprobe_instance) = NULL; 74 75 #ifdef __ARCH_WANT_KPROBES_INSN_SLOT 76 /* 77 * kprobe->ainsn.insn points to the copy of the instruction to be 78 * single-stepped. x86_64, POWER4 and above have no-exec support and 79 * stepping on the instruction on a vmalloced/kmalloced/data page 80 * is a recipe for disaster 81 */ 82 #define INSNS_PER_PAGE (PAGE_SIZE/(MAX_INSN_SIZE * sizeof(kprobe_opcode_t))) 83 84 struct kprobe_insn_page { 85 struct hlist_node hlist; 86 kprobe_opcode_t *insns; /* Page of instruction slots */ 87 char slot_used[INSNS_PER_PAGE]; 88 int nused; 89 int ngarbage; 90 }; 91 92 enum kprobe_slot_state { 93 SLOT_CLEAN = 0, 94 SLOT_DIRTY = 1, 95 SLOT_USED = 2, 96 }; 97 98 static struct hlist_head kprobe_insn_pages; 99 static int kprobe_garbage_slots; 100 static int collect_garbage_slots(void); 101 102 static int __kprobes check_safety(void) 103 { 104 int ret = 0; 105 #if defined(CONFIG_PREEMPT) && defined(CONFIG_PM) 106 ret = freeze_processes(); 107 if (ret == 0) { 108 struct task_struct *p, *q; 109 do_each_thread(p, q) { 110 if (p != current && p->state == TASK_RUNNING && 111 p->pid != 0) { 112 printk("Check failed: %s is running\n",p->comm); 113 ret = -1; 114 goto loop_end; 115 } 116 } while_each_thread(p, q); 117 } 118 loop_end: 119 thaw_processes(); 120 #else 121 synchronize_sched(); 122 #endif 123 return ret; 124 } 125 126 /** 127 * get_insn_slot() - Find a slot on an executable page for an instruction. 128 * We allocate an executable page if there's no room on existing ones. 129 */ 130 kprobe_opcode_t __kprobes *get_insn_slot(void) 131 { 132 struct kprobe_insn_page *kip; 133 struct hlist_node *pos; 134 135 retry: 136 hlist_for_each_entry(kip, pos, &kprobe_insn_pages, hlist) { 137 if (kip->nused < INSNS_PER_PAGE) { 138 int i; 139 for (i = 0; i < INSNS_PER_PAGE; i++) { 140 if (kip->slot_used[i] == SLOT_CLEAN) { 141 kip->slot_used[i] = SLOT_USED; 142 kip->nused++; 143 return kip->insns + (i * MAX_INSN_SIZE); 144 } 145 } 146 /* Surprise! No unused slots. Fix kip->nused. */ 147 kip->nused = INSNS_PER_PAGE; 148 } 149 } 150 151 /* If there are any garbage slots, collect it and try again. */ 152 if (kprobe_garbage_slots && collect_garbage_slots() == 0) { 153 goto retry; 154 } 155 /* All out of space. Need to allocate a new page. Use slot 0. */ 156 kip = kmalloc(sizeof(struct kprobe_insn_page), GFP_KERNEL); 157 if (!kip) 158 return NULL; 159 160 /* 161 * Use module_alloc so this page is within +/- 2GB of where the 162 * kernel image and loaded module images reside. This is required 163 * so x86_64 can correctly handle the %rip-relative fixups. 164 */ 165 kip->insns = module_alloc(PAGE_SIZE); 166 if (!kip->insns) { 167 kfree(kip); 168 return NULL; 169 } 170 INIT_HLIST_NODE(&kip->hlist); 171 hlist_add_head(&kip->hlist, &kprobe_insn_pages); 172 memset(kip->slot_used, SLOT_CLEAN, INSNS_PER_PAGE); 173 kip->slot_used[0] = SLOT_USED; 174 kip->nused = 1; 175 kip->ngarbage = 0; 176 return kip->insns; 177 } 178 179 /* Return 1 if all garbages are collected, otherwise 0. */ 180 static int __kprobes collect_one_slot(struct kprobe_insn_page *kip, int idx) 181 { 182 kip->slot_used[idx] = SLOT_CLEAN; 183 kip->nused--; 184 if (kip->nused == 0) { 185 /* 186 * Page is no longer in use. Free it unless 187 * it's the last one. We keep the last one 188 * so as not to have to set it up again the 189 * next time somebody inserts a probe. 190 */ 191 hlist_del(&kip->hlist); 192 if (hlist_empty(&kprobe_insn_pages)) { 193 INIT_HLIST_NODE(&kip->hlist); 194 hlist_add_head(&kip->hlist, 195 &kprobe_insn_pages); 196 } else { 197 module_free(NULL, kip->insns); 198 kfree(kip); 199 } 200 return 1; 201 } 202 return 0; 203 } 204 205 static int __kprobes collect_garbage_slots(void) 206 { 207 struct kprobe_insn_page *kip; 208 struct hlist_node *pos, *next; 209 210 /* Ensure no-one is preepmted on the garbages */ 211 if (check_safety() != 0) 212 return -EAGAIN; 213 214 hlist_for_each_entry_safe(kip, pos, next, &kprobe_insn_pages, hlist) { 215 int i; 216 if (kip->ngarbage == 0) 217 continue; 218 kip->ngarbage = 0; /* we will collect all garbages */ 219 for (i = 0; i < INSNS_PER_PAGE; i++) { 220 if (kip->slot_used[i] == SLOT_DIRTY && 221 collect_one_slot(kip, i)) 222 break; 223 } 224 } 225 kprobe_garbage_slots = 0; 226 return 0; 227 } 228 229 void __kprobes free_insn_slot(kprobe_opcode_t * slot, int dirty) 230 { 231 struct kprobe_insn_page *kip; 232 struct hlist_node *pos; 233 234 hlist_for_each_entry(kip, pos, &kprobe_insn_pages, hlist) { 235 if (kip->insns <= slot && 236 slot < kip->insns + (INSNS_PER_PAGE * MAX_INSN_SIZE)) { 237 int i = (slot - kip->insns) / MAX_INSN_SIZE; 238 if (dirty) { 239 kip->slot_used[i] = SLOT_DIRTY; 240 kip->ngarbage++; 241 } else { 242 collect_one_slot(kip, i); 243 } 244 break; 245 } 246 } 247 248 if (dirty && ++kprobe_garbage_slots > INSNS_PER_PAGE) 249 collect_garbage_slots(); 250 } 251 #endif 252 253 /* We have preemption disabled.. so it is safe to use __ versions */ 254 static inline void set_kprobe_instance(struct kprobe *kp) 255 { 256 __get_cpu_var(kprobe_instance) = kp; 257 } 258 259 static inline void reset_kprobe_instance(void) 260 { 261 __get_cpu_var(kprobe_instance) = NULL; 262 } 263 264 /* 265 * This routine is called either: 266 * - under the kprobe_mutex - during kprobe_[un]register() 267 * OR 268 * - with preemption disabled - from arch/xxx/kernel/kprobes.c 269 */ 270 struct kprobe __kprobes *get_kprobe(void *addr) 271 { 272 struct hlist_head *head; 273 struct hlist_node *node; 274 struct kprobe *p; 275 276 head = &kprobe_table[hash_ptr(addr, KPROBE_HASH_BITS)]; 277 hlist_for_each_entry_rcu(p, node, head, hlist) { 278 if (p->addr == addr) 279 return p; 280 } 281 return NULL; 282 } 283 284 /* 285 * Aggregate handlers for multiple kprobes support - these handlers 286 * take care of invoking the individual kprobe handlers on p->list 287 */ 288 static int __kprobes aggr_pre_handler(struct kprobe *p, struct pt_regs *regs) 289 { 290 struct kprobe *kp; 291 292 list_for_each_entry_rcu(kp, &p->list, list) { 293 if (kp->pre_handler) { 294 set_kprobe_instance(kp); 295 if (kp->pre_handler(kp, regs)) 296 return 1; 297 } 298 reset_kprobe_instance(); 299 } 300 return 0; 301 } 302 303 static void __kprobes aggr_post_handler(struct kprobe *p, struct pt_regs *regs, 304 unsigned long flags) 305 { 306 struct kprobe *kp; 307 308 list_for_each_entry_rcu(kp, &p->list, list) { 309 if (kp->post_handler) { 310 set_kprobe_instance(kp); 311 kp->post_handler(kp, regs, flags); 312 reset_kprobe_instance(); 313 } 314 } 315 } 316 317 static int __kprobes aggr_fault_handler(struct kprobe *p, struct pt_regs *regs, 318 int trapnr) 319 { 320 struct kprobe *cur = __get_cpu_var(kprobe_instance); 321 322 /* 323 * if we faulted "during" the execution of a user specified 324 * probe handler, invoke just that probe's fault handler 325 */ 326 if (cur && cur->fault_handler) { 327 if (cur->fault_handler(cur, regs, trapnr)) 328 return 1; 329 } 330 return 0; 331 } 332 333 static int __kprobes aggr_break_handler(struct kprobe *p, struct pt_regs *regs) 334 { 335 struct kprobe *cur = __get_cpu_var(kprobe_instance); 336 int ret = 0; 337 338 if (cur && cur->break_handler) { 339 if (cur->break_handler(cur, regs)) 340 ret = 1; 341 } 342 reset_kprobe_instance(); 343 return ret; 344 } 345 346 /* Walks the list and increments nmissed count for multiprobe case */ 347 void __kprobes kprobes_inc_nmissed_count(struct kprobe *p) 348 { 349 struct kprobe *kp; 350 if (p->pre_handler != aggr_pre_handler) { 351 p->nmissed++; 352 } else { 353 list_for_each_entry_rcu(kp, &p->list, list) 354 kp->nmissed++; 355 } 356 return; 357 } 358 359 /* Called with kretprobe_lock held */ 360 void __kprobes recycle_rp_inst(struct kretprobe_instance *ri, 361 struct hlist_head *head) 362 { 363 /* remove rp inst off the rprobe_inst_table */ 364 hlist_del(&ri->hlist); 365 if (ri->rp) { 366 /* remove rp inst off the used list */ 367 hlist_del(&ri->uflist); 368 /* put rp inst back onto the free list */ 369 INIT_HLIST_NODE(&ri->uflist); 370 hlist_add_head(&ri->uflist, &ri->rp->free_instances); 371 } else 372 /* Unregistering */ 373 hlist_add_head(&ri->hlist, head); 374 } 375 376 struct hlist_head __kprobes *kretprobe_inst_table_head(struct task_struct *tsk) 377 { 378 return &kretprobe_inst_table[hash_ptr(tsk, KPROBE_HASH_BITS)]; 379 } 380 381 /* 382 * This function is called from finish_task_switch when task tk becomes dead, 383 * so that we can recycle any function-return probe instances associated 384 * with this task. These left over instances represent probed functions 385 * that have been called but will never return. 386 */ 387 void __kprobes kprobe_flush_task(struct task_struct *tk) 388 { 389 struct kretprobe_instance *ri; 390 struct hlist_head *head, empty_rp; 391 struct hlist_node *node, *tmp; 392 unsigned long flags = 0; 393 394 INIT_HLIST_HEAD(&empty_rp); 395 spin_lock_irqsave(&kretprobe_lock, flags); 396 head = kretprobe_inst_table_head(tk); 397 hlist_for_each_entry_safe(ri, node, tmp, head, hlist) { 398 if (ri->task == tk) 399 recycle_rp_inst(ri, &empty_rp); 400 } 401 spin_unlock_irqrestore(&kretprobe_lock, flags); 402 403 hlist_for_each_entry_safe(ri, node, tmp, &empty_rp, hlist) { 404 hlist_del(&ri->hlist); 405 kfree(ri); 406 } 407 } 408 409 static inline void free_rp_inst(struct kretprobe *rp) 410 { 411 struct kretprobe_instance *ri; 412 struct hlist_node *pos, *next; 413 414 hlist_for_each_entry_safe(ri, pos, next, &rp->free_instances, uflist) { 415 hlist_del(&ri->uflist); 416 kfree(ri); 417 } 418 } 419 420 /* 421 * Keep all fields in the kprobe consistent 422 */ 423 static inline void copy_kprobe(struct kprobe *old_p, struct kprobe *p) 424 { 425 memcpy(&p->opcode, &old_p->opcode, sizeof(kprobe_opcode_t)); 426 memcpy(&p->ainsn, &old_p->ainsn, sizeof(struct arch_specific_insn)); 427 } 428 429 /* 430 * Add the new probe to old_p->list. Fail if this is the 431 * second jprobe at the address - two jprobes can't coexist 432 */ 433 static int __kprobes add_new_kprobe(struct kprobe *old_p, struct kprobe *p) 434 { 435 if (p->break_handler) { 436 if (old_p->break_handler) 437 return -EEXIST; 438 list_add_tail_rcu(&p->list, &old_p->list); 439 old_p->break_handler = aggr_break_handler; 440 } else 441 list_add_rcu(&p->list, &old_p->list); 442 if (p->post_handler && !old_p->post_handler) 443 old_p->post_handler = aggr_post_handler; 444 return 0; 445 } 446 447 /* 448 * Fill in the required fields of the "manager kprobe". Replace the 449 * earlier kprobe in the hlist with the manager kprobe 450 */ 451 static inline void add_aggr_kprobe(struct kprobe *ap, struct kprobe *p) 452 { 453 copy_kprobe(p, ap); 454 flush_insn_slot(ap); 455 ap->addr = p->addr; 456 ap->pre_handler = aggr_pre_handler; 457 ap->fault_handler = aggr_fault_handler; 458 if (p->post_handler) 459 ap->post_handler = aggr_post_handler; 460 if (p->break_handler) 461 ap->break_handler = aggr_break_handler; 462 463 INIT_LIST_HEAD(&ap->list); 464 list_add_rcu(&p->list, &ap->list); 465 466 hlist_replace_rcu(&p->hlist, &ap->hlist); 467 } 468 469 /* 470 * This is the second or subsequent kprobe at the address - handle 471 * the intricacies 472 */ 473 static int __kprobes register_aggr_kprobe(struct kprobe *old_p, 474 struct kprobe *p) 475 { 476 int ret = 0; 477 struct kprobe *ap; 478 479 if (old_p->pre_handler == aggr_pre_handler) { 480 copy_kprobe(old_p, p); 481 ret = add_new_kprobe(old_p, p); 482 } else { 483 ap = kzalloc(sizeof(struct kprobe), GFP_KERNEL); 484 if (!ap) 485 return -ENOMEM; 486 add_aggr_kprobe(ap, old_p); 487 copy_kprobe(ap, p); 488 ret = add_new_kprobe(ap, p); 489 } 490 return ret; 491 } 492 493 static int __kprobes in_kprobes_functions(unsigned long addr) 494 { 495 if (addr >= (unsigned long)__kprobes_text_start && 496 addr < (unsigned long)__kprobes_text_end) 497 return -EINVAL; 498 return 0; 499 } 500 501 /* 502 * If we have a symbol_name argument, look it up and add the offset field 503 * to it. This way, we can specify a relative address to a symbol. 504 */ 505 static kprobe_opcode_t __kprobes *kprobe_addr(struct kprobe *p) 506 { 507 kprobe_opcode_t *addr = p->addr; 508 if (p->symbol_name) { 509 if (addr) 510 return NULL; 511 kprobe_lookup_name(p->symbol_name, addr); 512 } 513 514 if (!addr) 515 return NULL; 516 return (kprobe_opcode_t *)(((char *)addr) + p->offset); 517 } 518 519 static int __kprobes __register_kprobe(struct kprobe *p, 520 unsigned long called_from) 521 { 522 int ret = 0; 523 struct kprobe *old_p; 524 struct module *probed_mod; 525 kprobe_opcode_t *addr; 526 527 addr = kprobe_addr(p); 528 if (!addr) 529 return -EINVAL; 530 p->addr = addr; 531 532 if (!kernel_text_address((unsigned long) p->addr) || 533 in_kprobes_functions((unsigned long) p->addr)) 534 return -EINVAL; 535 536 p->mod_refcounted = 0; 537 538 /* 539 * Check if are we probing a module. 540 */ 541 probed_mod = module_text_address((unsigned long) p->addr); 542 if (probed_mod) { 543 struct module *calling_mod = module_text_address(called_from); 544 /* 545 * We must allow modules to probe themself and in this case 546 * avoid incrementing the module refcount, so as to allow 547 * unloading of self probing modules. 548 */ 549 if (calling_mod && calling_mod != probed_mod) { 550 if (unlikely(!try_module_get(probed_mod))) 551 return -EINVAL; 552 p->mod_refcounted = 1; 553 } else 554 probed_mod = NULL; 555 } 556 557 p->nmissed = 0; 558 mutex_lock(&kprobe_mutex); 559 old_p = get_kprobe(p->addr); 560 if (old_p) { 561 ret = register_aggr_kprobe(old_p, p); 562 goto out; 563 } 564 565 ret = arch_prepare_kprobe(p); 566 if (ret) 567 goto out; 568 569 INIT_HLIST_NODE(&p->hlist); 570 hlist_add_head_rcu(&p->hlist, 571 &kprobe_table[hash_ptr(p->addr, KPROBE_HASH_BITS)]); 572 573 if (kprobe_enabled) 574 arch_arm_kprobe(p); 575 576 out: 577 mutex_unlock(&kprobe_mutex); 578 579 if (ret && probed_mod) 580 module_put(probed_mod); 581 return ret; 582 } 583 584 int __kprobes register_kprobe(struct kprobe *p) 585 { 586 return __register_kprobe(p, (unsigned long)__builtin_return_address(0)); 587 } 588 589 void __kprobes unregister_kprobe(struct kprobe *p) 590 { 591 struct module *mod; 592 struct kprobe *old_p, *list_p; 593 int cleanup_p; 594 595 mutex_lock(&kprobe_mutex); 596 old_p = get_kprobe(p->addr); 597 if (unlikely(!old_p)) { 598 mutex_unlock(&kprobe_mutex); 599 return; 600 } 601 if (p != old_p) { 602 list_for_each_entry_rcu(list_p, &old_p->list, list) 603 if (list_p == p) 604 /* kprobe p is a valid probe */ 605 goto valid_p; 606 mutex_unlock(&kprobe_mutex); 607 return; 608 } 609 valid_p: 610 if (old_p == p || 611 (old_p->pre_handler == aggr_pre_handler && 612 p->list.next == &old_p->list && p->list.prev == &old_p->list)) { 613 /* 614 * Only probe on the hash list. Disarm only if kprobes are 615 * enabled - otherwise, the breakpoint would already have 616 * been removed. We save on flushing icache. 617 */ 618 if (kprobe_enabled) 619 arch_disarm_kprobe(p); 620 hlist_del_rcu(&old_p->hlist); 621 cleanup_p = 1; 622 } else { 623 list_del_rcu(&p->list); 624 cleanup_p = 0; 625 } 626 627 mutex_unlock(&kprobe_mutex); 628 629 synchronize_sched(); 630 if (p->mod_refcounted) { 631 mod = module_text_address((unsigned long)p->addr); 632 if (mod) 633 module_put(mod); 634 } 635 636 if (cleanup_p) { 637 if (p != old_p) { 638 list_del_rcu(&p->list); 639 kfree(old_p); 640 } 641 arch_remove_kprobe(p); 642 } else { 643 mutex_lock(&kprobe_mutex); 644 if (p->break_handler) 645 old_p->break_handler = NULL; 646 if (p->post_handler){ 647 list_for_each_entry_rcu(list_p, &old_p->list, list){ 648 if (list_p->post_handler){ 649 cleanup_p = 2; 650 break; 651 } 652 } 653 if (cleanup_p == 0) 654 old_p->post_handler = NULL; 655 } 656 mutex_unlock(&kprobe_mutex); 657 } 658 } 659 660 static struct notifier_block kprobe_exceptions_nb = { 661 .notifier_call = kprobe_exceptions_notify, 662 .priority = 0x7fffffff /* we need to be notified first */ 663 }; 664 665 unsigned long __weak arch_deref_entry_point(void *entry) 666 { 667 return (unsigned long)entry; 668 } 669 670 int __kprobes register_jprobe(struct jprobe *jp) 671 { 672 unsigned long addr = arch_deref_entry_point(jp->entry); 673 674 if (!kernel_text_address(addr)) 675 return -EINVAL; 676 677 /* Todo: Verify probepoint is a function entry point */ 678 jp->kp.pre_handler = setjmp_pre_handler; 679 jp->kp.break_handler = longjmp_break_handler; 680 681 return __register_kprobe(&jp->kp, 682 (unsigned long)__builtin_return_address(0)); 683 } 684 685 void __kprobes unregister_jprobe(struct jprobe *jp) 686 { 687 unregister_kprobe(&jp->kp); 688 } 689 690 #ifdef CONFIG_KRETPROBES 691 /* 692 * This kprobe pre_handler is registered with every kretprobe. When probe 693 * hits it will set up the return probe. 694 */ 695 static int __kprobes pre_handler_kretprobe(struct kprobe *p, 696 struct pt_regs *regs) 697 { 698 struct kretprobe *rp = container_of(p, struct kretprobe, kp); 699 unsigned long flags = 0; 700 701 /*TODO: consider to only swap the RA after the last pre_handler fired */ 702 spin_lock_irqsave(&kretprobe_lock, flags); 703 if (!hlist_empty(&rp->free_instances)) { 704 struct kretprobe_instance *ri; 705 706 ri = hlist_entry(rp->free_instances.first, 707 struct kretprobe_instance, uflist); 708 ri->rp = rp; 709 ri->task = current; 710 711 if (rp->entry_handler && rp->entry_handler(ri, regs)) { 712 spin_unlock_irqrestore(&kretprobe_lock, flags); 713 return 0; 714 } 715 716 arch_prepare_kretprobe(ri, regs); 717 718 /* XXX(hch): why is there no hlist_move_head? */ 719 hlist_del(&ri->uflist); 720 hlist_add_head(&ri->uflist, &ri->rp->used_instances); 721 hlist_add_head(&ri->hlist, kretprobe_inst_table_head(ri->task)); 722 } else 723 rp->nmissed++; 724 spin_unlock_irqrestore(&kretprobe_lock, flags); 725 return 0; 726 } 727 728 int __kprobes register_kretprobe(struct kretprobe *rp) 729 { 730 int ret = 0; 731 struct kretprobe_instance *inst; 732 int i; 733 void *addr; 734 735 if (kretprobe_blacklist_size) { 736 addr = kprobe_addr(&rp->kp); 737 if (!addr) 738 return -EINVAL; 739 740 for (i = 0; kretprobe_blacklist[i].name != NULL; i++) { 741 if (kretprobe_blacklist[i].addr == addr) 742 return -EINVAL; 743 } 744 } 745 746 rp->kp.pre_handler = pre_handler_kretprobe; 747 rp->kp.post_handler = NULL; 748 rp->kp.fault_handler = NULL; 749 rp->kp.break_handler = NULL; 750 751 /* Pre-allocate memory for max kretprobe instances */ 752 if (rp->maxactive <= 0) { 753 #ifdef CONFIG_PREEMPT 754 rp->maxactive = max(10, 2 * NR_CPUS); 755 #else 756 rp->maxactive = NR_CPUS; 757 #endif 758 } 759 INIT_HLIST_HEAD(&rp->used_instances); 760 INIT_HLIST_HEAD(&rp->free_instances); 761 for (i = 0; i < rp->maxactive; i++) { 762 inst = kmalloc(sizeof(struct kretprobe_instance) + 763 rp->data_size, GFP_KERNEL); 764 if (inst == NULL) { 765 free_rp_inst(rp); 766 return -ENOMEM; 767 } 768 INIT_HLIST_NODE(&inst->uflist); 769 hlist_add_head(&inst->uflist, &rp->free_instances); 770 } 771 772 rp->nmissed = 0; 773 /* Establish function entry probe point */ 774 if ((ret = __register_kprobe(&rp->kp, 775 (unsigned long)__builtin_return_address(0))) != 0) 776 free_rp_inst(rp); 777 return ret; 778 } 779 780 #else /* CONFIG_KRETPROBES */ 781 int __kprobes register_kretprobe(struct kretprobe *rp) 782 { 783 return -ENOSYS; 784 } 785 786 static int __kprobes pre_handler_kretprobe(struct kprobe *p, 787 struct pt_regs *regs) 788 { 789 return 0; 790 } 791 #endif /* CONFIG_KRETPROBES */ 792 793 void __kprobes unregister_kretprobe(struct kretprobe *rp) 794 { 795 unsigned long flags; 796 struct kretprobe_instance *ri; 797 struct hlist_node *pos, *next; 798 799 unregister_kprobe(&rp->kp); 800 801 /* No race here */ 802 spin_lock_irqsave(&kretprobe_lock, flags); 803 hlist_for_each_entry_safe(ri, pos, next, &rp->used_instances, uflist) { 804 ri->rp = NULL; 805 hlist_del(&ri->uflist); 806 } 807 spin_unlock_irqrestore(&kretprobe_lock, flags); 808 free_rp_inst(rp); 809 } 810 811 static int __init init_kprobes(void) 812 { 813 int i, err = 0; 814 815 /* FIXME allocate the probe table, currently defined statically */ 816 /* initialize all list heads */ 817 for (i = 0; i < KPROBE_TABLE_SIZE; i++) { 818 INIT_HLIST_HEAD(&kprobe_table[i]); 819 INIT_HLIST_HEAD(&kretprobe_inst_table[i]); 820 } 821 822 if (kretprobe_blacklist_size) { 823 /* lookup the function address from its name */ 824 for (i = 0; kretprobe_blacklist[i].name != NULL; i++) { 825 kprobe_lookup_name(kretprobe_blacklist[i].name, 826 kretprobe_blacklist[i].addr); 827 if (!kretprobe_blacklist[i].addr) 828 printk("kretprobe: lookup failed: %s\n", 829 kretprobe_blacklist[i].name); 830 } 831 } 832 833 /* By default, kprobes are enabled */ 834 kprobe_enabled = true; 835 836 err = arch_init_kprobes(); 837 if (!err) 838 err = register_die_notifier(&kprobe_exceptions_nb); 839 840 if (!err) 841 init_test_probes(); 842 return err; 843 } 844 845 #ifdef CONFIG_DEBUG_FS 846 static void __kprobes report_probe(struct seq_file *pi, struct kprobe *p, 847 const char *sym, int offset,char *modname) 848 { 849 char *kprobe_type; 850 851 if (p->pre_handler == pre_handler_kretprobe) 852 kprobe_type = "r"; 853 else if (p->pre_handler == setjmp_pre_handler) 854 kprobe_type = "j"; 855 else 856 kprobe_type = "k"; 857 if (sym) 858 seq_printf(pi, "%p %s %s+0x%x %s\n", p->addr, kprobe_type, 859 sym, offset, (modname ? modname : " ")); 860 else 861 seq_printf(pi, "%p %s %p\n", p->addr, kprobe_type, p->addr); 862 } 863 864 static void __kprobes *kprobe_seq_start(struct seq_file *f, loff_t *pos) 865 { 866 return (*pos < KPROBE_TABLE_SIZE) ? pos : NULL; 867 } 868 869 static void __kprobes *kprobe_seq_next(struct seq_file *f, void *v, loff_t *pos) 870 { 871 (*pos)++; 872 if (*pos >= KPROBE_TABLE_SIZE) 873 return NULL; 874 return pos; 875 } 876 877 static void __kprobes kprobe_seq_stop(struct seq_file *f, void *v) 878 { 879 /* Nothing to do */ 880 } 881 882 static int __kprobes show_kprobe_addr(struct seq_file *pi, void *v) 883 { 884 struct hlist_head *head; 885 struct hlist_node *node; 886 struct kprobe *p, *kp; 887 const char *sym = NULL; 888 unsigned int i = *(loff_t *) v; 889 unsigned long offset = 0; 890 char *modname, namebuf[128]; 891 892 head = &kprobe_table[i]; 893 preempt_disable(); 894 hlist_for_each_entry_rcu(p, node, head, hlist) { 895 sym = kallsyms_lookup((unsigned long)p->addr, NULL, 896 &offset, &modname, namebuf); 897 if (p->pre_handler == aggr_pre_handler) { 898 list_for_each_entry_rcu(kp, &p->list, list) 899 report_probe(pi, kp, sym, offset, modname); 900 } else 901 report_probe(pi, p, sym, offset, modname); 902 } 903 preempt_enable(); 904 return 0; 905 } 906 907 static struct seq_operations kprobes_seq_ops = { 908 .start = kprobe_seq_start, 909 .next = kprobe_seq_next, 910 .stop = kprobe_seq_stop, 911 .show = show_kprobe_addr 912 }; 913 914 static int __kprobes kprobes_open(struct inode *inode, struct file *filp) 915 { 916 return seq_open(filp, &kprobes_seq_ops); 917 } 918 919 static struct file_operations debugfs_kprobes_operations = { 920 .open = kprobes_open, 921 .read = seq_read, 922 .llseek = seq_lseek, 923 .release = seq_release, 924 }; 925 926 static void __kprobes enable_all_kprobes(void) 927 { 928 struct hlist_head *head; 929 struct hlist_node *node; 930 struct kprobe *p; 931 unsigned int i; 932 933 mutex_lock(&kprobe_mutex); 934 935 /* If kprobes are already enabled, just return */ 936 if (kprobe_enabled) 937 goto already_enabled; 938 939 for (i = 0; i < KPROBE_TABLE_SIZE; i++) { 940 head = &kprobe_table[i]; 941 hlist_for_each_entry_rcu(p, node, head, hlist) 942 arch_arm_kprobe(p); 943 } 944 945 kprobe_enabled = true; 946 printk(KERN_INFO "Kprobes globally enabled\n"); 947 948 already_enabled: 949 mutex_unlock(&kprobe_mutex); 950 return; 951 } 952 953 static void __kprobes disable_all_kprobes(void) 954 { 955 struct hlist_head *head; 956 struct hlist_node *node; 957 struct kprobe *p; 958 unsigned int i; 959 960 mutex_lock(&kprobe_mutex); 961 962 /* If kprobes are already disabled, just return */ 963 if (!kprobe_enabled) 964 goto already_disabled; 965 966 kprobe_enabled = false; 967 printk(KERN_INFO "Kprobes globally disabled\n"); 968 for (i = 0; i < KPROBE_TABLE_SIZE; i++) { 969 head = &kprobe_table[i]; 970 hlist_for_each_entry_rcu(p, node, head, hlist) { 971 if (!arch_trampoline_kprobe(p)) 972 arch_disarm_kprobe(p); 973 } 974 } 975 976 mutex_unlock(&kprobe_mutex); 977 /* Allow all currently running kprobes to complete */ 978 synchronize_sched(); 979 return; 980 981 already_disabled: 982 mutex_unlock(&kprobe_mutex); 983 return; 984 } 985 986 /* 987 * XXX: The debugfs bool file interface doesn't allow for callbacks 988 * when the bool state is switched. We can reuse that facility when 989 * available 990 */ 991 static ssize_t read_enabled_file_bool(struct file *file, 992 char __user *user_buf, size_t count, loff_t *ppos) 993 { 994 char buf[3]; 995 996 if (kprobe_enabled) 997 buf[0] = '1'; 998 else 999 buf[0] = '0'; 1000 buf[1] = '\n'; 1001 buf[2] = 0x00; 1002 return simple_read_from_buffer(user_buf, count, ppos, buf, 2); 1003 } 1004 1005 static ssize_t write_enabled_file_bool(struct file *file, 1006 const char __user *user_buf, size_t count, loff_t *ppos) 1007 { 1008 char buf[32]; 1009 int buf_size; 1010 1011 buf_size = min(count, (sizeof(buf)-1)); 1012 if (copy_from_user(buf, user_buf, buf_size)) 1013 return -EFAULT; 1014 1015 switch (buf[0]) { 1016 case 'y': 1017 case 'Y': 1018 case '1': 1019 enable_all_kprobes(); 1020 break; 1021 case 'n': 1022 case 'N': 1023 case '0': 1024 disable_all_kprobes(); 1025 break; 1026 } 1027 1028 return count; 1029 } 1030 1031 static struct file_operations fops_kp = { 1032 .read = read_enabled_file_bool, 1033 .write = write_enabled_file_bool, 1034 }; 1035 1036 static int __kprobes debugfs_kprobe_init(void) 1037 { 1038 struct dentry *dir, *file; 1039 unsigned int value = 1; 1040 1041 dir = debugfs_create_dir("kprobes", NULL); 1042 if (!dir) 1043 return -ENOMEM; 1044 1045 file = debugfs_create_file("list", 0444, dir, NULL, 1046 &debugfs_kprobes_operations); 1047 if (!file) { 1048 debugfs_remove(dir); 1049 return -ENOMEM; 1050 } 1051 1052 file = debugfs_create_file("enabled", 0600, dir, 1053 &value, &fops_kp); 1054 if (!file) { 1055 debugfs_remove(dir); 1056 return -ENOMEM; 1057 } 1058 1059 return 0; 1060 } 1061 1062 late_initcall(debugfs_kprobe_init); 1063 #endif /* CONFIG_DEBUG_FS */ 1064 1065 module_init(init_kprobes); 1066 1067 EXPORT_SYMBOL_GPL(register_kprobe); 1068 EXPORT_SYMBOL_GPL(unregister_kprobe); 1069 EXPORT_SYMBOL_GPL(register_jprobe); 1070 EXPORT_SYMBOL_GPL(unregister_jprobe); 1071 #ifdef CONFIG_KPROBES 1072 EXPORT_SYMBOL_GPL(jprobe_return); 1073 #endif 1074 1075 #ifdef CONFIG_KPROBES 1076 EXPORT_SYMBOL_GPL(register_kretprobe); 1077 EXPORT_SYMBOL_GPL(unregister_kretprobe); 1078 #endif 1079