1 /* 2 * Kprobes-based tracing events 3 * 4 * Created by Masami Hiramatsu <mhiramat@redhat.com> 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 * 10 * This program is distributed in the hope that it will be useful, 11 * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 * GNU General Public License for more details. 14 * 15 * You should have received a copy of the GNU General Public License 16 * along with this program; if not, write to the Free Software 17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 18 */ 19 #define pr_fmt(fmt) "trace_kprobe: " fmt 20 21 #include <linux/module.h> 22 #include <linux/uaccess.h> 23 #include <linux/rculist.h> 24 #include <linux/error-injection.h> 25 26 #include "trace_probe.h" 27 28 #define KPROBE_EVENT_SYSTEM "kprobes" 29 #define KRETPROBE_MAXACTIVE_MAX 4096 30 31 /** 32 * Kprobe event core functions 33 */ 34 struct trace_kprobe { 35 struct list_head list; 36 struct kretprobe rp; /* Use rp.kp for kprobe use */ 37 unsigned long __percpu *nhit; 38 const char *symbol; /* symbol name */ 39 struct trace_probe tp; 40 }; 41 42 #define SIZEOF_TRACE_KPROBE(n) \ 43 (offsetof(struct trace_kprobe, tp.args) + \ 44 (sizeof(struct probe_arg) * (n))) 45 46 static nokprobe_inline bool trace_kprobe_is_return(struct trace_kprobe *tk) 47 { 48 return tk->rp.handler != NULL; 49 } 50 51 static nokprobe_inline const char *trace_kprobe_symbol(struct trace_kprobe *tk) 52 { 53 return tk->symbol ? tk->symbol : "unknown"; 54 } 55 56 static nokprobe_inline unsigned long trace_kprobe_offset(struct trace_kprobe *tk) 57 { 58 return tk->rp.kp.offset; 59 } 60 61 static nokprobe_inline bool trace_kprobe_has_gone(struct trace_kprobe *tk) 62 { 63 return !!(kprobe_gone(&tk->rp.kp)); 64 } 65 66 static nokprobe_inline bool trace_kprobe_within_module(struct trace_kprobe *tk, 67 struct module *mod) 68 { 69 int len = strlen(mod->name); 70 const char *name = trace_kprobe_symbol(tk); 71 return strncmp(mod->name, name, len) == 0 && name[len] == ':'; 72 } 73 74 static nokprobe_inline bool trace_kprobe_is_on_module(struct trace_kprobe *tk) 75 { 76 return !!strchr(trace_kprobe_symbol(tk), ':'); 77 } 78 79 static nokprobe_inline unsigned long trace_kprobe_nhit(struct trace_kprobe *tk) 80 { 81 unsigned long nhit = 0; 82 int cpu; 83 84 for_each_possible_cpu(cpu) 85 nhit += *per_cpu_ptr(tk->nhit, cpu); 86 87 return nhit; 88 } 89 90 bool trace_kprobe_on_func_entry(struct trace_event_call *call) 91 { 92 struct trace_kprobe *tk = (struct trace_kprobe *)call->data; 93 94 return kprobe_on_func_entry(tk->rp.kp.addr, 95 tk->rp.kp.addr ? NULL : tk->rp.kp.symbol_name, 96 tk->rp.kp.addr ? 0 : tk->rp.kp.offset); 97 } 98 99 bool trace_kprobe_error_injectable(struct trace_event_call *call) 100 { 101 struct trace_kprobe *tk = (struct trace_kprobe *)call->data; 102 unsigned long addr; 103 104 if (tk->symbol) { 105 addr = (unsigned long) 106 kallsyms_lookup_name(trace_kprobe_symbol(tk)); 107 addr += tk->rp.kp.offset; 108 } else { 109 addr = (unsigned long)tk->rp.kp.addr; 110 } 111 return within_error_injection_list(addr); 112 } 113 114 static int register_kprobe_event(struct trace_kprobe *tk); 115 static int unregister_kprobe_event(struct trace_kprobe *tk); 116 117 static DEFINE_MUTEX(probe_lock); 118 static LIST_HEAD(probe_list); 119 120 static int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs); 121 static int kretprobe_dispatcher(struct kretprobe_instance *ri, 122 struct pt_regs *regs); 123 124 /* Memory fetching by symbol */ 125 struct symbol_cache { 126 char *symbol; 127 long offset; 128 unsigned long addr; 129 }; 130 131 unsigned long update_symbol_cache(struct symbol_cache *sc) 132 { 133 sc->addr = (unsigned long)kallsyms_lookup_name(sc->symbol); 134 135 if (sc->addr) 136 sc->addr += sc->offset; 137 138 return sc->addr; 139 } 140 141 void free_symbol_cache(struct symbol_cache *sc) 142 { 143 kfree(sc->symbol); 144 kfree(sc); 145 } 146 147 struct symbol_cache *alloc_symbol_cache(const char *sym, long offset) 148 { 149 struct symbol_cache *sc; 150 151 if (!sym || strlen(sym) == 0) 152 return NULL; 153 154 sc = kzalloc(sizeof(struct symbol_cache), GFP_KERNEL); 155 if (!sc) 156 return NULL; 157 158 sc->symbol = kstrdup(sym, GFP_KERNEL); 159 if (!sc->symbol) { 160 kfree(sc); 161 return NULL; 162 } 163 sc->offset = offset; 164 update_symbol_cache(sc); 165 166 return sc; 167 } 168 169 /* 170 * Kprobes-specific fetch functions 171 */ 172 #define DEFINE_FETCH_stack(type) \ 173 static void FETCH_FUNC_NAME(stack, type)(struct pt_regs *regs, \ 174 void *offset, void *dest) \ 175 { \ 176 *(type *)dest = (type)regs_get_kernel_stack_nth(regs, \ 177 (unsigned int)((unsigned long)offset)); \ 178 } \ 179 NOKPROBE_SYMBOL(FETCH_FUNC_NAME(stack, type)); 180 181 DEFINE_BASIC_FETCH_FUNCS(stack) 182 /* No string on the stack entry */ 183 #define fetch_stack_string NULL 184 #define fetch_stack_string_size NULL 185 186 #define DEFINE_FETCH_memory(type) \ 187 static void FETCH_FUNC_NAME(memory, type)(struct pt_regs *regs, \ 188 void *addr, void *dest) \ 189 { \ 190 type retval; \ 191 if (probe_kernel_address(addr, retval)) \ 192 *(type *)dest = 0; \ 193 else \ 194 *(type *)dest = retval; \ 195 } \ 196 NOKPROBE_SYMBOL(FETCH_FUNC_NAME(memory, type)); 197 198 DEFINE_BASIC_FETCH_FUNCS(memory) 199 /* 200 * Fetch a null-terminated string. Caller MUST set *(u32 *)dest with max 201 * length and relative data location. 202 */ 203 static void FETCH_FUNC_NAME(memory, string)(struct pt_regs *regs, 204 void *addr, void *dest) 205 { 206 int maxlen = get_rloc_len(*(u32 *)dest); 207 u8 *dst = get_rloc_data(dest); 208 long ret; 209 210 if (!maxlen) 211 return; 212 213 /* 214 * Try to get string again, since the string can be changed while 215 * probing. 216 */ 217 ret = strncpy_from_unsafe(dst, addr, maxlen); 218 219 if (ret < 0) { /* Failed to fetch string */ 220 dst[0] = '\0'; 221 *(u32 *)dest = make_data_rloc(0, get_rloc_offs(*(u32 *)dest)); 222 } else { 223 *(u32 *)dest = make_data_rloc(ret, get_rloc_offs(*(u32 *)dest)); 224 } 225 } 226 NOKPROBE_SYMBOL(FETCH_FUNC_NAME(memory, string)); 227 228 /* Return the length of string -- including null terminal byte */ 229 static void FETCH_FUNC_NAME(memory, string_size)(struct pt_regs *regs, 230 void *addr, void *dest) 231 { 232 mm_segment_t old_fs; 233 int ret, len = 0; 234 u8 c; 235 236 old_fs = get_fs(); 237 set_fs(KERNEL_DS); 238 pagefault_disable(); 239 240 do { 241 ret = __copy_from_user_inatomic(&c, (u8 *)addr + len, 1); 242 len++; 243 } while (c && ret == 0 && len < MAX_STRING_SIZE); 244 245 pagefault_enable(); 246 set_fs(old_fs); 247 248 if (ret < 0) /* Failed to check the length */ 249 *(u32 *)dest = 0; 250 else 251 *(u32 *)dest = len; 252 } 253 NOKPROBE_SYMBOL(FETCH_FUNC_NAME(memory, string_size)); 254 255 #define DEFINE_FETCH_symbol(type) \ 256 void FETCH_FUNC_NAME(symbol, type)(struct pt_regs *regs, void *data, void *dest)\ 257 { \ 258 struct symbol_cache *sc = data; \ 259 if (sc->addr) \ 260 fetch_memory_##type(regs, (void *)sc->addr, dest); \ 261 else \ 262 *(type *)dest = 0; \ 263 } \ 264 NOKPROBE_SYMBOL(FETCH_FUNC_NAME(symbol, type)); 265 266 DEFINE_BASIC_FETCH_FUNCS(symbol) 267 DEFINE_FETCH_symbol(string) 268 DEFINE_FETCH_symbol(string_size) 269 270 /* kprobes don't support file_offset fetch methods */ 271 #define fetch_file_offset_u8 NULL 272 #define fetch_file_offset_u16 NULL 273 #define fetch_file_offset_u32 NULL 274 #define fetch_file_offset_u64 NULL 275 #define fetch_file_offset_string NULL 276 #define fetch_file_offset_string_size NULL 277 278 /* Fetch type information table */ 279 static const struct fetch_type kprobes_fetch_type_table[] = { 280 /* Special types */ 281 [FETCH_TYPE_STRING] = __ASSIGN_FETCH_TYPE("string", string, string, 282 sizeof(u32), 1, "__data_loc char[]"), 283 [FETCH_TYPE_STRSIZE] = __ASSIGN_FETCH_TYPE("string_size", u32, 284 string_size, sizeof(u32), 0, "u32"), 285 /* Basic types */ 286 ASSIGN_FETCH_TYPE(u8, u8, 0), 287 ASSIGN_FETCH_TYPE(u16, u16, 0), 288 ASSIGN_FETCH_TYPE(u32, u32, 0), 289 ASSIGN_FETCH_TYPE(u64, u64, 0), 290 ASSIGN_FETCH_TYPE(s8, u8, 1), 291 ASSIGN_FETCH_TYPE(s16, u16, 1), 292 ASSIGN_FETCH_TYPE(s32, u32, 1), 293 ASSIGN_FETCH_TYPE(s64, u64, 1), 294 ASSIGN_FETCH_TYPE_ALIAS(x8, u8, u8, 0), 295 ASSIGN_FETCH_TYPE_ALIAS(x16, u16, u16, 0), 296 ASSIGN_FETCH_TYPE_ALIAS(x32, u32, u32, 0), 297 ASSIGN_FETCH_TYPE_ALIAS(x64, u64, u64, 0), 298 299 ASSIGN_FETCH_TYPE_END 300 }; 301 302 /* 303 * Allocate new trace_probe and initialize it (including kprobes). 304 */ 305 static struct trace_kprobe *alloc_trace_kprobe(const char *group, 306 const char *event, 307 void *addr, 308 const char *symbol, 309 unsigned long offs, 310 int maxactive, 311 int nargs, bool is_return) 312 { 313 struct trace_kprobe *tk; 314 int ret = -ENOMEM; 315 316 tk = kzalloc(SIZEOF_TRACE_KPROBE(nargs), GFP_KERNEL); 317 if (!tk) 318 return ERR_PTR(ret); 319 320 tk->nhit = alloc_percpu(unsigned long); 321 if (!tk->nhit) 322 goto error; 323 324 if (symbol) { 325 tk->symbol = kstrdup(symbol, GFP_KERNEL); 326 if (!tk->symbol) 327 goto error; 328 tk->rp.kp.symbol_name = tk->symbol; 329 tk->rp.kp.offset = offs; 330 } else 331 tk->rp.kp.addr = addr; 332 333 if (is_return) 334 tk->rp.handler = kretprobe_dispatcher; 335 else 336 tk->rp.kp.pre_handler = kprobe_dispatcher; 337 338 tk->rp.maxactive = maxactive; 339 340 if (!event || !is_good_name(event)) { 341 ret = -EINVAL; 342 goto error; 343 } 344 345 tk->tp.call.class = &tk->tp.class; 346 tk->tp.call.name = kstrdup(event, GFP_KERNEL); 347 if (!tk->tp.call.name) 348 goto error; 349 350 if (!group || !is_good_name(group)) { 351 ret = -EINVAL; 352 goto error; 353 } 354 355 tk->tp.class.system = kstrdup(group, GFP_KERNEL); 356 if (!tk->tp.class.system) 357 goto error; 358 359 INIT_LIST_HEAD(&tk->list); 360 INIT_LIST_HEAD(&tk->tp.files); 361 return tk; 362 error: 363 kfree(tk->tp.call.name); 364 kfree(tk->symbol); 365 free_percpu(tk->nhit); 366 kfree(tk); 367 return ERR_PTR(ret); 368 } 369 370 static void free_trace_kprobe(struct trace_kprobe *tk) 371 { 372 int i; 373 374 for (i = 0; i < tk->tp.nr_args; i++) 375 traceprobe_free_probe_arg(&tk->tp.args[i]); 376 377 kfree(tk->tp.call.class->system); 378 kfree(tk->tp.call.name); 379 kfree(tk->symbol); 380 free_percpu(tk->nhit); 381 kfree(tk); 382 } 383 384 static struct trace_kprobe *find_trace_kprobe(const char *event, 385 const char *group) 386 { 387 struct trace_kprobe *tk; 388 389 list_for_each_entry(tk, &probe_list, list) 390 if (strcmp(trace_event_name(&tk->tp.call), event) == 0 && 391 strcmp(tk->tp.call.class->system, group) == 0) 392 return tk; 393 return NULL; 394 } 395 396 /* 397 * Enable trace_probe 398 * if the file is NULL, enable "perf" handler, or enable "trace" handler. 399 */ 400 static int 401 enable_trace_kprobe(struct trace_kprobe *tk, struct trace_event_file *file) 402 { 403 int ret = 0; 404 405 if (file) { 406 struct event_file_link *link; 407 408 link = kmalloc(sizeof(*link), GFP_KERNEL); 409 if (!link) { 410 ret = -ENOMEM; 411 goto out; 412 } 413 414 link->file = file; 415 list_add_tail_rcu(&link->list, &tk->tp.files); 416 417 tk->tp.flags |= TP_FLAG_TRACE; 418 } else 419 tk->tp.flags |= TP_FLAG_PROFILE; 420 421 if (trace_probe_is_registered(&tk->tp) && !trace_kprobe_has_gone(tk)) { 422 if (trace_kprobe_is_return(tk)) 423 ret = enable_kretprobe(&tk->rp); 424 else 425 ret = enable_kprobe(&tk->rp.kp); 426 } 427 out: 428 return ret; 429 } 430 431 /* 432 * Disable trace_probe 433 * if the file is NULL, disable "perf" handler, or disable "trace" handler. 434 */ 435 static int 436 disable_trace_kprobe(struct trace_kprobe *tk, struct trace_event_file *file) 437 { 438 struct event_file_link *link = NULL; 439 int wait = 0; 440 int ret = 0; 441 442 if (file) { 443 link = find_event_file_link(&tk->tp, file); 444 if (!link) { 445 ret = -EINVAL; 446 goto out; 447 } 448 449 list_del_rcu(&link->list); 450 wait = 1; 451 if (!list_empty(&tk->tp.files)) 452 goto out; 453 454 tk->tp.flags &= ~TP_FLAG_TRACE; 455 } else 456 tk->tp.flags &= ~TP_FLAG_PROFILE; 457 458 if (!trace_probe_is_enabled(&tk->tp) && trace_probe_is_registered(&tk->tp)) { 459 if (trace_kprobe_is_return(tk)) 460 disable_kretprobe(&tk->rp); 461 else 462 disable_kprobe(&tk->rp.kp); 463 wait = 1; 464 } 465 466 /* 467 * if tk is not added to any list, it must be a local trace_kprobe 468 * created with perf_event_open. We don't need to wait for these 469 * trace_kprobes 470 */ 471 if (list_empty(&tk->list)) 472 wait = 0; 473 out: 474 if (wait) { 475 /* 476 * Synchronize with kprobe_trace_func/kretprobe_trace_func 477 * to ensure disabled (all running handlers are finished). 478 * This is not only for kfree(), but also the caller, 479 * trace_remove_event_call() supposes it for releasing 480 * event_call related objects, which will be accessed in 481 * the kprobe_trace_func/kretprobe_trace_func. 482 */ 483 synchronize_sched(); 484 kfree(link); /* Ignored if link == NULL */ 485 } 486 487 return ret; 488 } 489 490 /* Internal register function - just handle k*probes and flags */ 491 static int __register_trace_kprobe(struct trace_kprobe *tk) 492 { 493 int i, ret; 494 495 if (trace_probe_is_registered(&tk->tp)) 496 return -EINVAL; 497 498 for (i = 0; i < tk->tp.nr_args; i++) 499 traceprobe_update_arg(&tk->tp.args[i]); 500 501 /* Set/clear disabled flag according to tp->flag */ 502 if (trace_probe_is_enabled(&tk->tp)) 503 tk->rp.kp.flags &= ~KPROBE_FLAG_DISABLED; 504 else 505 tk->rp.kp.flags |= KPROBE_FLAG_DISABLED; 506 507 if (trace_kprobe_is_return(tk)) 508 ret = register_kretprobe(&tk->rp); 509 else 510 ret = register_kprobe(&tk->rp.kp); 511 512 if (ret == 0) 513 tk->tp.flags |= TP_FLAG_REGISTERED; 514 else { 515 if (ret == -ENOENT && trace_kprobe_is_on_module(tk)) { 516 pr_warn("This probe might be able to register after target module is loaded. Continue.\n"); 517 ret = 0; 518 } else if (ret == -EILSEQ) { 519 pr_warn("Probing address(0x%p) is not an instruction boundary.\n", 520 tk->rp.kp.addr); 521 ret = -EINVAL; 522 } 523 } 524 525 return ret; 526 } 527 528 /* Internal unregister function - just handle k*probes and flags */ 529 static void __unregister_trace_kprobe(struct trace_kprobe *tk) 530 { 531 if (trace_probe_is_registered(&tk->tp)) { 532 if (trace_kprobe_is_return(tk)) 533 unregister_kretprobe(&tk->rp); 534 else 535 unregister_kprobe(&tk->rp.kp); 536 tk->tp.flags &= ~TP_FLAG_REGISTERED; 537 /* Cleanup kprobe for reuse */ 538 if (tk->rp.kp.symbol_name) 539 tk->rp.kp.addr = NULL; 540 } 541 } 542 543 /* Unregister a trace_probe and probe_event: call with locking probe_lock */ 544 static int unregister_trace_kprobe(struct trace_kprobe *tk) 545 { 546 /* Enabled event can not be unregistered */ 547 if (trace_probe_is_enabled(&tk->tp)) 548 return -EBUSY; 549 550 /* Will fail if probe is being used by ftrace or perf */ 551 if (unregister_kprobe_event(tk)) 552 return -EBUSY; 553 554 __unregister_trace_kprobe(tk); 555 list_del(&tk->list); 556 557 return 0; 558 } 559 560 /* Register a trace_probe and probe_event */ 561 static int register_trace_kprobe(struct trace_kprobe *tk) 562 { 563 struct trace_kprobe *old_tk; 564 int ret; 565 566 mutex_lock(&probe_lock); 567 568 /* Delete old (same name) event if exist */ 569 old_tk = find_trace_kprobe(trace_event_name(&tk->tp.call), 570 tk->tp.call.class->system); 571 if (old_tk) { 572 ret = unregister_trace_kprobe(old_tk); 573 if (ret < 0) 574 goto end; 575 free_trace_kprobe(old_tk); 576 } 577 578 /* Register new event */ 579 ret = register_kprobe_event(tk); 580 if (ret) { 581 pr_warn("Failed to register probe event(%d)\n", ret); 582 goto end; 583 } 584 585 /* Register k*probe */ 586 ret = __register_trace_kprobe(tk); 587 if (ret < 0) 588 unregister_kprobe_event(tk); 589 else 590 list_add_tail(&tk->list, &probe_list); 591 592 end: 593 mutex_unlock(&probe_lock); 594 return ret; 595 } 596 597 /* Module notifier call back, checking event on the module */ 598 static int trace_kprobe_module_callback(struct notifier_block *nb, 599 unsigned long val, void *data) 600 { 601 struct module *mod = data; 602 struct trace_kprobe *tk; 603 int ret; 604 605 if (val != MODULE_STATE_COMING) 606 return NOTIFY_DONE; 607 608 /* Update probes on coming module */ 609 mutex_lock(&probe_lock); 610 list_for_each_entry(tk, &probe_list, list) { 611 if (trace_kprobe_within_module(tk, mod)) { 612 /* Don't need to check busy - this should have gone. */ 613 __unregister_trace_kprobe(tk); 614 ret = __register_trace_kprobe(tk); 615 if (ret) 616 pr_warn("Failed to re-register probe %s on %s: %d\n", 617 trace_event_name(&tk->tp.call), 618 mod->name, ret); 619 } 620 } 621 mutex_unlock(&probe_lock); 622 623 return NOTIFY_DONE; 624 } 625 626 static struct notifier_block trace_kprobe_module_nb = { 627 .notifier_call = trace_kprobe_module_callback, 628 .priority = 1 /* Invoked after kprobe module callback */ 629 }; 630 631 /* Convert certain expected symbols into '_' when generating event names */ 632 static inline void sanitize_event_name(char *name) 633 { 634 while (*name++ != '\0') 635 if (*name == ':' || *name == '.') 636 *name = '_'; 637 } 638 639 static int create_trace_kprobe(int argc, char **argv) 640 { 641 /* 642 * Argument syntax: 643 * - Add kprobe: 644 * p[:[GRP/]EVENT] [MOD:]KSYM[+OFFS]|KADDR [FETCHARGS] 645 * - Add kretprobe: 646 * r[MAXACTIVE][:[GRP/]EVENT] [MOD:]KSYM[+0] [FETCHARGS] 647 * Fetch args: 648 * $retval : fetch return value 649 * $stack : fetch stack address 650 * $stackN : fetch Nth of stack (N:0-) 651 * $comm : fetch current task comm 652 * @ADDR : fetch memory at ADDR (ADDR should be in kernel) 653 * @SYM[+|-offs] : fetch memory at SYM +|- offs (SYM is a data symbol) 654 * %REG : fetch register REG 655 * Dereferencing memory fetch: 656 * +|-offs(ARG) : fetch memory at ARG +|- offs address. 657 * Alias name of args: 658 * NAME=FETCHARG : set NAME as alias of FETCHARG. 659 * Type of args: 660 * FETCHARG:TYPE : use TYPE instead of unsigned long. 661 */ 662 struct trace_kprobe *tk; 663 int i, ret = 0; 664 bool is_return = false, is_delete = false; 665 char *symbol = NULL, *event = NULL, *group = NULL; 666 int maxactive = 0; 667 char *arg; 668 long offset = 0; 669 void *addr = NULL; 670 char buf[MAX_EVENT_NAME_LEN]; 671 672 /* argc must be >= 1 */ 673 if (argv[0][0] == 'p') 674 is_return = false; 675 else if (argv[0][0] == 'r') 676 is_return = true; 677 else if (argv[0][0] == '-') 678 is_delete = true; 679 else { 680 pr_info("Probe definition must be started with 'p', 'r' or" 681 " '-'.\n"); 682 return -EINVAL; 683 } 684 685 event = strchr(&argv[0][1], ':'); 686 if (event) { 687 event[0] = '\0'; 688 event++; 689 } 690 if (is_return && isdigit(argv[0][1])) { 691 ret = kstrtouint(&argv[0][1], 0, &maxactive); 692 if (ret) { 693 pr_info("Failed to parse maxactive.\n"); 694 return ret; 695 } 696 /* kretprobes instances are iterated over via a list. The 697 * maximum should stay reasonable. 698 */ 699 if (maxactive > KRETPROBE_MAXACTIVE_MAX) { 700 pr_info("Maxactive is too big (%d > %d).\n", 701 maxactive, KRETPROBE_MAXACTIVE_MAX); 702 return -E2BIG; 703 } 704 } 705 706 if (event) { 707 if (strchr(event, '/')) { 708 group = event; 709 event = strchr(group, '/') + 1; 710 event[-1] = '\0'; 711 if (strlen(group) == 0) { 712 pr_info("Group name is not specified\n"); 713 return -EINVAL; 714 } 715 } 716 if (strlen(event) == 0) { 717 pr_info("Event name is not specified\n"); 718 return -EINVAL; 719 } 720 } 721 if (!group) 722 group = KPROBE_EVENT_SYSTEM; 723 724 if (is_delete) { 725 if (!event) { 726 pr_info("Delete command needs an event name.\n"); 727 return -EINVAL; 728 } 729 mutex_lock(&probe_lock); 730 tk = find_trace_kprobe(event, group); 731 if (!tk) { 732 mutex_unlock(&probe_lock); 733 pr_info("Event %s/%s doesn't exist.\n", group, event); 734 return -ENOENT; 735 } 736 /* delete an event */ 737 ret = unregister_trace_kprobe(tk); 738 if (ret == 0) 739 free_trace_kprobe(tk); 740 mutex_unlock(&probe_lock); 741 return ret; 742 } 743 744 if (argc < 2) { 745 pr_info("Probe point is not specified.\n"); 746 return -EINVAL; 747 } 748 749 /* try to parse an address. if that fails, try to read the 750 * input as a symbol. */ 751 if (kstrtoul(argv[1], 0, (unsigned long *)&addr)) { 752 /* a symbol specified */ 753 symbol = argv[1]; 754 /* TODO: support .init module functions */ 755 ret = traceprobe_split_symbol_offset(symbol, &offset); 756 if (ret || offset < 0 || offset > UINT_MAX) { 757 pr_info("Failed to parse either an address or a symbol.\n"); 758 return ret; 759 } 760 if (offset && is_return && 761 !kprobe_on_func_entry(NULL, symbol, offset)) { 762 pr_info("Given offset is not valid for return probe.\n"); 763 return -EINVAL; 764 } 765 } 766 argc -= 2; argv += 2; 767 768 /* setup a probe */ 769 if (!event) { 770 /* Make a new event name */ 771 if (symbol) 772 snprintf(buf, MAX_EVENT_NAME_LEN, "%c_%s_%ld", 773 is_return ? 'r' : 'p', symbol, offset); 774 else 775 snprintf(buf, MAX_EVENT_NAME_LEN, "%c_0x%p", 776 is_return ? 'r' : 'p', addr); 777 sanitize_event_name(buf); 778 event = buf; 779 } 780 tk = alloc_trace_kprobe(group, event, addr, symbol, offset, maxactive, 781 argc, is_return); 782 if (IS_ERR(tk)) { 783 pr_info("Failed to allocate trace_probe.(%d)\n", 784 (int)PTR_ERR(tk)); 785 return PTR_ERR(tk); 786 } 787 788 /* parse arguments */ 789 ret = 0; 790 for (i = 0; i < argc && i < MAX_TRACE_ARGS; i++) { 791 struct probe_arg *parg = &tk->tp.args[i]; 792 793 /* Increment count for freeing args in error case */ 794 tk->tp.nr_args++; 795 796 /* Parse argument name */ 797 arg = strchr(argv[i], '='); 798 if (arg) { 799 *arg++ = '\0'; 800 parg->name = kstrdup(argv[i], GFP_KERNEL); 801 } else { 802 arg = argv[i]; 803 /* If argument name is omitted, set "argN" */ 804 snprintf(buf, MAX_EVENT_NAME_LEN, "arg%d", i + 1); 805 parg->name = kstrdup(buf, GFP_KERNEL); 806 } 807 808 if (!parg->name) { 809 pr_info("Failed to allocate argument[%d] name.\n", i); 810 ret = -ENOMEM; 811 goto error; 812 } 813 814 if (!is_good_name(parg->name)) { 815 pr_info("Invalid argument[%d] name: %s\n", 816 i, parg->name); 817 ret = -EINVAL; 818 goto error; 819 } 820 821 if (traceprobe_conflict_field_name(parg->name, 822 tk->tp.args, i)) { 823 pr_info("Argument[%d] name '%s' conflicts with " 824 "another field.\n", i, argv[i]); 825 ret = -EINVAL; 826 goto error; 827 } 828 829 /* Parse fetch argument */ 830 ret = traceprobe_parse_probe_arg(arg, &tk->tp.size, parg, 831 is_return, true, 832 kprobes_fetch_type_table); 833 if (ret) { 834 pr_info("Parse error at argument[%d]. (%d)\n", i, ret); 835 goto error; 836 } 837 } 838 839 ret = register_trace_kprobe(tk); 840 if (ret) 841 goto error; 842 return 0; 843 844 error: 845 free_trace_kprobe(tk); 846 return ret; 847 } 848 849 static int release_all_trace_kprobes(void) 850 { 851 struct trace_kprobe *tk; 852 int ret = 0; 853 854 mutex_lock(&probe_lock); 855 /* Ensure no probe is in use. */ 856 list_for_each_entry(tk, &probe_list, list) 857 if (trace_probe_is_enabled(&tk->tp)) { 858 ret = -EBUSY; 859 goto end; 860 } 861 /* TODO: Use batch unregistration */ 862 while (!list_empty(&probe_list)) { 863 tk = list_entry(probe_list.next, struct trace_kprobe, list); 864 ret = unregister_trace_kprobe(tk); 865 if (ret) 866 goto end; 867 free_trace_kprobe(tk); 868 } 869 870 end: 871 mutex_unlock(&probe_lock); 872 873 return ret; 874 } 875 876 /* Probes listing interfaces */ 877 static void *probes_seq_start(struct seq_file *m, loff_t *pos) 878 { 879 mutex_lock(&probe_lock); 880 return seq_list_start(&probe_list, *pos); 881 } 882 883 static void *probes_seq_next(struct seq_file *m, void *v, loff_t *pos) 884 { 885 return seq_list_next(v, &probe_list, pos); 886 } 887 888 static void probes_seq_stop(struct seq_file *m, void *v) 889 { 890 mutex_unlock(&probe_lock); 891 } 892 893 static int probes_seq_show(struct seq_file *m, void *v) 894 { 895 struct trace_kprobe *tk = v; 896 int i; 897 898 seq_putc(m, trace_kprobe_is_return(tk) ? 'r' : 'p'); 899 seq_printf(m, ":%s/%s", tk->tp.call.class->system, 900 trace_event_name(&tk->tp.call)); 901 902 if (!tk->symbol) 903 seq_printf(m, " 0x%p", tk->rp.kp.addr); 904 else if (tk->rp.kp.offset) 905 seq_printf(m, " %s+%u", trace_kprobe_symbol(tk), 906 tk->rp.kp.offset); 907 else 908 seq_printf(m, " %s", trace_kprobe_symbol(tk)); 909 910 for (i = 0; i < tk->tp.nr_args; i++) 911 seq_printf(m, " %s=%s", tk->tp.args[i].name, tk->tp.args[i].comm); 912 seq_putc(m, '\n'); 913 914 return 0; 915 } 916 917 static const struct seq_operations probes_seq_op = { 918 .start = probes_seq_start, 919 .next = probes_seq_next, 920 .stop = probes_seq_stop, 921 .show = probes_seq_show 922 }; 923 924 static int probes_open(struct inode *inode, struct file *file) 925 { 926 int ret; 927 928 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) { 929 ret = release_all_trace_kprobes(); 930 if (ret < 0) 931 return ret; 932 } 933 934 return seq_open(file, &probes_seq_op); 935 } 936 937 static ssize_t probes_write(struct file *file, const char __user *buffer, 938 size_t count, loff_t *ppos) 939 { 940 return trace_parse_run_command(file, buffer, count, ppos, 941 create_trace_kprobe); 942 } 943 944 static const struct file_operations kprobe_events_ops = { 945 .owner = THIS_MODULE, 946 .open = probes_open, 947 .read = seq_read, 948 .llseek = seq_lseek, 949 .release = seq_release, 950 .write = probes_write, 951 }; 952 953 /* Probes profiling interfaces */ 954 static int probes_profile_seq_show(struct seq_file *m, void *v) 955 { 956 struct trace_kprobe *tk = v; 957 958 seq_printf(m, " %-44s %15lu %15lu\n", 959 trace_event_name(&tk->tp.call), 960 trace_kprobe_nhit(tk), 961 tk->rp.kp.nmissed); 962 963 return 0; 964 } 965 966 static const struct seq_operations profile_seq_op = { 967 .start = probes_seq_start, 968 .next = probes_seq_next, 969 .stop = probes_seq_stop, 970 .show = probes_profile_seq_show 971 }; 972 973 static int profile_open(struct inode *inode, struct file *file) 974 { 975 return seq_open(file, &profile_seq_op); 976 } 977 978 static const struct file_operations kprobe_profile_ops = { 979 .owner = THIS_MODULE, 980 .open = profile_open, 981 .read = seq_read, 982 .llseek = seq_lseek, 983 .release = seq_release, 984 }; 985 986 /* Kprobe handler */ 987 static nokprobe_inline void 988 __kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs, 989 struct trace_event_file *trace_file) 990 { 991 struct kprobe_trace_entry_head *entry; 992 struct ring_buffer_event *event; 993 struct ring_buffer *buffer; 994 int size, dsize, pc; 995 unsigned long irq_flags; 996 struct trace_event_call *call = &tk->tp.call; 997 998 WARN_ON(call != trace_file->event_call); 999 1000 if (trace_trigger_soft_disabled(trace_file)) 1001 return; 1002 1003 local_save_flags(irq_flags); 1004 pc = preempt_count(); 1005 1006 dsize = __get_data_size(&tk->tp, regs); 1007 size = sizeof(*entry) + tk->tp.size + dsize; 1008 1009 event = trace_event_buffer_lock_reserve(&buffer, trace_file, 1010 call->event.type, 1011 size, irq_flags, pc); 1012 if (!event) 1013 return; 1014 1015 entry = ring_buffer_event_data(event); 1016 entry->ip = (unsigned long)tk->rp.kp.addr; 1017 store_trace_args(sizeof(*entry), &tk->tp, regs, (u8 *)&entry[1], dsize); 1018 1019 event_trigger_unlock_commit_regs(trace_file, buffer, event, 1020 entry, irq_flags, pc, regs); 1021 } 1022 1023 static void 1024 kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs) 1025 { 1026 struct event_file_link *link; 1027 1028 list_for_each_entry_rcu(link, &tk->tp.files, list) 1029 __kprobe_trace_func(tk, regs, link->file); 1030 } 1031 NOKPROBE_SYMBOL(kprobe_trace_func); 1032 1033 /* Kretprobe handler */ 1034 static nokprobe_inline void 1035 __kretprobe_trace_func(struct trace_kprobe *tk, struct kretprobe_instance *ri, 1036 struct pt_regs *regs, 1037 struct trace_event_file *trace_file) 1038 { 1039 struct kretprobe_trace_entry_head *entry; 1040 struct ring_buffer_event *event; 1041 struct ring_buffer *buffer; 1042 int size, pc, dsize; 1043 unsigned long irq_flags; 1044 struct trace_event_call *call = &tk->tp.call; 1045 1046 WARN_ON(call != trace_file->event_call); 1047 1048 if (trace_trigger_soft_disabled(trace_file)) 1049 return; 1050 1051 local_save_flags(irq_flags); 1052 pc = preempt_count(); 1053 1054 dsize = __get_data_size(&tk->tp, regs); 1055 size = sizeof(*entry) + tk->tp.size + dsize; 1056 1057 event = trace_event_buffer_lock_reserve(&buffer, trace_file, 1058 call->event.type, 1059 size, irq_flags, pc); 1060 if (!event) 1061 return; 1062 1063 entry = ring_buffer_event_data(event); 1064 entry->func = (unsigned long)tk->rp.kp.addr; 1065 entry->ret_ip = (unsigned long)ri->ret_addr; 1066 store_trace_args(sizeof(*entry), &tk->tp, regs, (u8 *)&entry[1], dsize); 1067 1068 event_trigger_unlock_commit_regs(trace_file, buffer, event, 1069 entry, irq_flags, pc, regs); 1070 } 1071 1072 static void 1073 kretprobe_trace_func(struct trace_kprobe *tk, struct kretprobe_instance *ri, 1074 struct pt_regs *regs) 1075 { 1076 struct event_file_link *link; 1077 1078 list_for_each_entry_rcu(link, &tk->tp.files, list) 1079 __kretprobe_trace_func(tk, ri, regs, link->file); 1080 } 1081 NOKPROBE_SYMBOL(kretprobe_trace_func); 1082 1083 /* Event entry printers */ 1084 static enum print_line_t 1085 print_kprobe_event(struct trace_iterator *iter, int flags, 1086 struct trace_event *event) 1087 { 1088 struct kprobe_trace_entry_head *field; 1089 struct trace_seq *s = &iter->seq; 1090 struct trace_probe *tp; 1091 u8 *data; 1092 int i; 1093 1094 field = (struct kprobe_trace_entry_head *)iter->ent; 1095 tp = container_of(event, struct trace_probe, call.event); 1096 1097 trace_seq_printf(s, "%s: (", trace_event_name(&tp->call)); 1098 1099 if (!seq_print_ip_sym(s, field->ip, flags | TRACE_ITER_SYM_OFFSET)) 1100 goto out; 1101 1102 trace_seq_putc(s, ')'); 1103 1104 data = (u8 *)&field[1]; 1105 for (i = 0; i < tp->nr_args; i++) 1106 if (!tp->args[i].type->print(s, tp->args[i].name, 1107 data + tp->args[i].offset, field)) 1108 goto out; 1109 1110 trace_seq_putc(s, '\n'); 1111 out: 1112 return trace_handle_return(s); 1113 } 1114 1115 static enum print_line_t 1116 print_kretprobe_event(struct trace_iterator *iter, int flags, 1117 struct trace_event *event) 1118 { 1119 struct kretprobe_trace_entry_head *field; 1120 struct trace_seq *s = &iter->seq; 1121 struct trace_probe *tp; 1122 u8 *data; 1123 int i; 1124 1125 field = (struct kretprobe_trace_entry_head *)iter->ent; 1126 tp = container_of(event, struct trace_probe, call.event); 1127 1128 trace_seq_printf(s, "%s: (", trace_event_name(&tp->call)); 1129 1130 if (!seq_print_ip_sym(s, field->ret_ip, flags | TRACE_ITER_SYM_OFFSET)) 1131 goto out; 1132 1133 trace_seq_puts(s, " <- "); 1134 1135 if (!seq_print_ip_sym(s, field->func, flags & ~TRACE_ITER_SYM_OFFSET)) 1136 goto out; 1137 1138 trace_seq_putc(s, ')'); 1139 1140 data = (u8 *)&field[1]; 1141 for (i = 0; i < tp->nr_args; i++) 1142 if (!tp->args[i].type->print(s, tp->args[i].name, 1143 data + tp->args[i].offset, field)) 1144 goto out; 1145 1146 trace_seq_putc(s, '\n'); 1147 1148 out: 1149 return trace_handle_return(s); 1150 } 1151 1152 1153 static int kprobe_event_define_fields(struct trace_event_call *event_call) 1154 { 1155 int ret, i; 1156 struct kprobe_trace_entry_head field; 1157 struct trace_kprobe *tk = (struct trace_kprobe *)event_call->data; 1158 1159 DEFINE_FIELD(unsigned long, ip, FIELD_STRING_IP, 0); 1160 /* Set argument names as fields */ 1161 for (i = 0; i < tk->tp.nr_args; i++) { 1162 struct probe_arg *parg = &tk->tp.args[i]; 1163 1164 ret = trace_define_field(event_call, parg->type->fmttype, 1165 parg->name, 1166 sizeof(field) + parg->offset, 1167 parg->type->size, 1168 parg->type->is_signed, 1169 FILTER_OTHER); 1170 if (ret) 1171 return ret; 1172 } 1173 return 0; 1174 } 1175 1176 static int kretprobe_event_define_fields(struct trace_event_call *event_call) 1177 { 1178 int ret, i; 1179 struct kretprobe_trace_entry_head field; 1180 struct trace_kprobe *tk = (struct trace_kprobe *)event_call->data; 1181 1182 DEFINE_FIELD(unsigned long, func, FIELD_STRING_FUNC, 0); 1183 DEFINE_FIELD(unsigned long, ret_ip, FIELD_STRING_RETIP, 0); 1184 /* Set argument names as fields */ 1185 for (i = 0; i < tk->tp.nr_args; i++) { 1186 struct probe_arg *parg = &tk->tp.args[i]; 1187 1188 ret = trace_define_field(event_call, parg->type->fmttype, 1189 parg->name, 1190 sizeof(field) + parg->offset, 1191 parg->type->size, 1192 parg->type->is_signed, 1193 FILTER_OTHER); 1194 if (ret) 1195 return ret; 1196 } 1197 return 0; 1198 } 1199 1200 #ifdef CONFIG_PERF_EVENTS 1201 1202 /* Kprobe profile handler */ 1203 static int 1204 kprobe_perf_func(struct trace_kprobe *tk, struct pt_regs *regs) 1205 { 1206 struct trace_event_call *call = &tk->tp.call; 1207 struct kprobe_trace_entry_head *entry; 1208 struct hlist_head *head; 1209 int size, __size, dsize; 1210 int rctx; 1211 1212 if (bpf_prog_array_valid(call)) { 1213 unsigned long orig_ip = instruction_pointer(regs); 1214 int ret; 1215 1216 ret = trace_call_bpf(call, regs); 1217 1218 /* 1219 * We need to check and see if we modified the pc of the 1220 * pt_regs, and if so clear the kprobe and return 1 so that we 1221 * don't do the single stepping. 1222 * The ftrace kprobe handler leaves it up to us to re-enable 1223 * preemption here before returning if we've modified the ip. 1224 */ 1225 if (orig_ip != instruction_pointer(regs)) { 1226 reset_current_kprobe(); 1227 preempt_enable_no_resched(); 1228 return 1; 1229 } 1230 if (!ret) 1231 return 0; 1232 } 1233 1234 head = this_cpu_ptr(call->perf_events); 1235 if (hlist_empty(head)) 1236 return 0; 1237 1238 dsize = __get_data_size(&tk->tp, regs); 1239 __size = sizeof(*entry) + tk->tp.size + dsize; 1240 size = ALIGN(__size + sizeof(u32), sizeof(u64)); 1241 size -= sizeof(u32); 1242 1243 entry = perf_trace_buf_alloc(size, NULL, &rctx); 1244 if (!entry) 1245 return 0; 1246 1247 entry->ip = (unsigned long)tk->rp.kp.addr; 1248 memset(&entry[1], 0, dsize); 1249 store_trace_args(sizeof(*entry), &tk->tp, regs, (u8 *)&entry[1], dsize); 1250 perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs, 1251 head, NULL); 1252 return 0; 1253 } 1254 NOKPROBE_SYMBOL(kprobe_perf_func); 1255 1256 /* Kretprobe profile handler */ 1257 static void 1258 kretprobe_perf_func(struct trace_kprobe *tk, struct kretprobe_instance *ri, 1259 struct pt_regs *regs) 1260 { 1261 struct trace_event_call *call = &tk->tp.call; 1262 struct kretprobe_trace_entry_head *entry; 1263 struct hlist_head *head; 1264 int size, __size, dsize; 1265 int rctx; 1266 1267 if (bpf_prog_array_valid(call) && !trace_call_bpf(call, regs)) 1268 return; 1269 1270 head = this_cpu_ptr(call->perf_events); 1271 if (hlist_empty(head)) 1272 return; 1273 1274 dsize = __get_data_size(&tk->tp, regs); 1275 __size = sizeof(*entry) + tk->tp.size + dsize; 1276 size = ALIGN(__size + sizeof(u32), sizeof(u64)); 1277 size -= sizeof(u32); 1278 1279 entry = perf_trace_buf_alloc(size, NULL, &rctx); 1280 if (!entry) 1281 return; 1282 1283 entry->func = (unsigned long)tk->rp.kp.addr; 1284 entry->ret_ip = (unsigned long)ri->ret_addr; 1285 store_trace_args(sizeof(*entry), &tk->tp, regs, (u8 *)&entry[1], dsize); 1286 perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs, 1287 head, NULL); 1288 } 1289 NOKPROBE_SYMBOL(kretprobe_perf_func); 1290 1291 int bpf_get_kprobe_info(const struct perf_event *event, u32 *fd_type, 1292 const char **symbol, u64 *probe_offset, 1293 u64 *probe_addr, bool perf_type_tracepoint) 1294 { 1295 const char *pevent = trace_event_name(event->tp_event); 1296 const char *group = event->tp_event->class->system; 1297 struct trace_kprobe *tk; 1298 1299 if (perf_type_tracepoint) 1300 tk = find_trace_kprobe(pevent, group); 1301 else 1302 tk = event->tp_event->data; 1303 if (!tk) 1304 return -EINVAL; 1305 1306 *fd_type = trace_kprobe_is_return(tk) ? BPF_FD_TYPE_KRETPROBE 1307 : BPF_FD_TYPE_KPROBE; 1308 if (tk->symbol) { 1309 *symbol = tk->symbol; 1310 *probe_offset = tk->rp.kp.offset; 1311 *probe_addr = 0; 1312 } else { 1313 *symbol = NULL; 1314 *probe_offset = 0; 1315 *probe_addr = (unsigned long)tk->rp.kp.addr; 1316 } 1317 return 0; 1318 } 1319 #endif /* CONFIG_PERF_EVENTS */ 1320 1321 /* 1322 * called by perf_trace_init() or __ftrace_set_clr_event() under event_mutex. 1323 * 1324 * kprobe_trace_self_tests_init() does enable_trace_probe/disable_trace_probe 1325 * lockless, but we can't race with this __init function. 1326 */ 1327 static int kprobe_register(struct trace_event_call *event, 1328 enum trace_reg type, void *data) 1329 { 1330 struct trace_kprobe *tk = (struct trace_kprobe *)event->data; 1331 struct trace_event_file *file = data; 1332 1333 switch (type) { 1334 case TRACE_REG_REGISTER: 1335 return enable_trace_kprobe(tk, file); 1336 case TRACE_REG_UNREGISTER: 1337 return disable_trace_kprobe(tk, file); 1338 1339 #ifdef CONFIG_PERF_EVENTS 1340 case TRACE_REG_PERF_REGISTER: 1341 return enable_trace_kprobe(tk, NULL); 1342 case TRACE_REG_PERF_UNREGISTER: 1343 return disable_trace_kprobe(tk, NULL); 1344 case TRACE_REG_PERF_OPEN: 1345 case TRACE_REG_PERF_CLOSE: 1346 case TRACE_REG_PERF_ADD: 1347 case TRACE_REG_PERF_DEL: 1348 return 0; 1349 #endif 1350 } 1351 return 0; 1352 } 1353 1354 static int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs) 1355 { 1356 struct trace_kprobe *tk = container_of(kp, struct trace_kprobe, rp.kp); 1357 int ret = 0; 1358 1359 raw_cpu_inc(*tk->nhit); 1360 1361 if (tk->tp.flags & TP_FLAG_TRACE) 1362 kprobe_trace_func(tk, regs); 1363 #ifdef CONFIG_PERF_EVENTS 1364 if (tk->tp.flags & TP_FLAG_PROFILE) 1365 ret = kprobe_perf_func(tk, regs); 1366 #endif 1367 return ret; 1368 } 1369 NOKPROBE_SYMBOL(kprobe_dispatcher); 1370 1371 static int 1372 kretprobe_dispatcher(struct kretprobe_instance *ri, struct pt_regs *regs) 1373 { 1374 struct trace_kprobe *tk = container_of(ri->rp, struct trace_kprobe, rp); 1375 1376 raw_cpu_inc(*tk->nhit); 1377 1378 if (tk->tp.flags & TP_FLAG_TRACE) 1379 kretprobe_trace_func(tk, ri, regs); 1380 #ifdef CONFIG_PERF_EVENTS 1381 if (tk->tp.flags & TP_FLAG_PROFILE) 1382 kretprobe_perf_func(tk, ri, regs); 1383 #endif 1384 return 0; /* We don't tweek kernel, so just return 0 */ 1385 } 1386 NOKPROBE_SYMBOL(kretprobe_dispatcher); 1387 1388 static struct trace_event_functions kretprobe_funcs = { 1389 .trace = print_kretprobe_event 1390 }; 1391 1392 static struct trace_event_functions kprobe_funcs = { 1393 .trace = print_kprobe_event 1394 }; 1395 1396 static inline void init_trace_event_call(struct trace_kprobe *tk, 1397 struct trace_event_call *call) 1398 { 1399 INIT_LIST_HEAD(&call->class->fields); 1400 if (trace_kprobe_is_return(tk)) { 1401 call->event.funcs = &kretprobe_funcs; 1402 call->class->define_fields = kretprobe_event_define_fields; 1403 } else { 1404 call->event.funcs = &kprobe_funcs; 1405 call->class->define_fields = kprobe_event_define_fields; 1406 } 1407 1408 call->flags = TRACE_EVENT_FL_KPROBE; 1409 call->class->reg = kprobe_register; 1410 call->data = tk; 1411 } 1412 1413 static int register_kprobe_event(struct trace_kprobe *tk) 1414 { 1415 struct trace_event_call *call = &tk->tp.call; 1416 int ret = 0; 1417 1418 init_trace_event_call(tk, call); 1419 1420 if (set_print_fmt(&tk->tp, trace_kprobe_is_return(tk)) < 0) 1421 return -ENOMEM; 1422 ret = register_trace_event(&call->event); 1423 if (!ret) { 1424 kfree(call->print_fmt); 1425 return -ENODEV; 1426 } 1427 ret = trace_add_event_call(call); 1428 if (ret) { 1429 pr_info("Failed to register kprobe event: %s\n", 1430 trace_event_name(call)); 1431 kfree(call->print_fmt); 1432 unregister_trace_event(&call->event); 1433 } 1434 return ret; 1435 } 1436 1437 static int unregister_kprobe_event(struct trace_kprobe *tk) 1438 { 1439 int ret; 1440 1441 /* tp->event is unregistered in trace_remove_event_call() */ 1442 ret = trace_remove_event_call(&tk->tp.call); 1443 if (!ret) 1444 kfree(tk->tp.call.print_fmt); 1445 return ret; 1446 } 1447 1448 #ifdef CONFIG_PERF_EVENTS 1449 /* create a trace_kprobe, but don't add it to global lists */ 1450 struct trace_event_call * 1451 create_local_trace_kprobe(char *func, void *addr, unsigned long offs, 1452 bool is_return) 1453 { 1454 struct trace_kprobe *tk; 1455 int ret; 1456 char *event; 1457 1458 /* 1459 * local trace_kprobes are not added to probe_list, so they are never 1460 * searched in find_trace_kprobe(). Therefore, there is no concern of 1461 * duplicated name here. 1462 */ 1463 event = func ? func : "DUMMY_EVENT"; 1464 1465 tk = alloc_trace_kprobe(KPROBE_EVENT_SYSTEM, event, (void *)addr, func, 1466 offs, 0 /* maxactive */, 0 /* nargs */, 1467 is_return); 1468 1469 if (IS_ERR(tk)) { 1470 pr_info("Failed to allocate trace_probe.(%d)\n", 1471 (int)PTR_ERR(tk)); 1472 return ERR_CAST(tk); 1473 } 1474 1475 init_trace_event_call(tk, &tk->tp.call); 1476 1477 if (set_print_fmt(&tk->tp, trace_kprobe_is_return(tk)) < 0) { 1478 ret = -ENOMEM; 1479 goto error; 1480 } 1481 1482 ret = __register_trace_kprobe(tk); 1483 if (ret < 0) 1484 goto error; 1485 1486 return &tk->tp.call; 1487 error: 1488 free_trace_kprobe(tk); 1489 return ERR_PTR(ret); 1490 } 1491 1492 void destroy_local_trace_kprobe(struct trace_event_call *event_call) 1493 { 1494 struct trace_kprobe *tk; 1495 1496 tk = container_of(event_call, struct trace_kprobe, tp.call); 1497 1498 if (trace_probe_is_enabled(&tk->tp)) { 1499 WARN_ON(1); 1500 return; 1501 } 1502 1503 __unregister_trace_kprobe(tk); 1504 free_trace_kprobe(tk); 1505 } 1506 #endif /* CONFIG_PERF_EVENTS */ 1507 1508 /* Make a tracefs interface for controlling probe points */ 1509 static __init int init_kprobe_trace(void) 1510 { 1511 struct dentry *d_tracer; 1512 struct dentry *entry; 1513 1514 if (register_module_notifier(&trace_kprobe_module_nb)) 1515 return -EINVAL; 1516 1517 d_tracer = tracing_init_dentry(); 1518 if (IS_ERR(d_tracer)) 1519 return 0; 1520 1521 entry = tracefs_create_file("kprobe_events", 0644, d_tracer, 1522 NULL, &kprobe_events_ops); 1523 1524 /* Event list interface */ 1525 if (!entry) 1526 pr_warn("Could not create tracefs 'kprobe_events' entry\n"); 1527 1528 /* Profile interface */ 1529 entry = tracefs_create_file("kprobe_profile", 0444, d_tracer, 1530 NULL, &kprobe_profile_ops); 1531 1532 if (!entry) 1533 pr_warn("Could not create tracefs 'kprobe_profile' entry\n"); 1534 return 0; 1535 } 1536 fs_initcall(init_kprobe_trace); 1537 1538 1539 #ifdef CONFIG_FTRACE_STARTUP_TEST 1540 /* 1541 * The "__used" keeps gcc from removing the function symbol 1542 * from the kallsyms table. 'noinline' makes sure that there 1543 * isn't an inlined version used by the test method below 1544 */ 1545 static __used __init noinline int 1546 kprobe_trace_selftest_target(int a1, int a2, int a3, int a4, int a5, int a6) 1547 { 1548 return a1 + a2 + a3 + a4 + a5 + a6; 1549 } 1550 1551 static __init struct trace_event_file * 1552 find_trace_probe_file(struct trace_kprobe *tk, struct trace_array *tr) 1553 { 1554 struct trace_event_file *file; 1555 1556 list_for_each_entry(file, &tr->events, list) 1557 if (file->event_call == &tk->tp.call) 1558 return file; 1559 1560 return NULL; 1561 } 1562 1563 /* 1564 * Nobody but us can call enable_trace_kprobe/disable_trace_kprobe at this 1565 * stage, we can do this lockless. 1566 */ 1567 static __init int kprobe_trace_self_tests_init(void) 1568 { 1569 int ret, warn = 0; 1570 int (*target)(int, int, int, int, int, int); 1571 struct trace_kprobe *tk; 1572 struct trace_event_file *file; 1573 1574 if (tracing_is_disabled()) 1575 return -ENODEV; 1576 1577 target = kprobe_trace_selftest_target; 1578 1579 pr_info("Testing kprobe tracing: "); 1580 1581 ret = trace_run_command("p:testprobe kprobe_trace_selftest_target " 1582 "$stack $stack0 +0($stack)", 1583 create_trace_kprobe); 1584 if (WARN_ON_ONCE(ret)) { 1585 pr_warn("error on probing function entry.\n"); 1586 warn++; 1587 } else { 1588 /* Enable trace point */ 1589 tk = find_trace_kprobe("testprobe", KPROBE_EVENT_SYSTEM); 1590 if (WARN_ON_ONCE(tk == NULL)) { 1591 pr_warn("error on getting new probe.\n"); 1592 warn++; 1593 } else { 1594 file = find_trace_probe_file(tk, top_trace_array()); 1595 if (WARN_ON_ONCE(file == NULL)) { 1596 pr_warn("error on getting probe file.\n"); 1597 warn++; 1598 } else 1599 enable_trace_kprobe(tk, file); 1600 } 1601 } 1602 1603 ret = trace_run_command("r:testprobe2 kprobe_trace_selftest_target " 1604 "$retval", create_trace_kprobe); 1605 if (WARN_ON_ONCE(ret)) { 1606 pr_warn("error on probing function return.\n"); 1607 warn++; 1608 } else { 1609 /* Enable trace point */ 1610 tk = find_trace_kprobe("testprobe2", KPROBE_EVENT_SYSTEM); 1611 if (WARN_ON_ONCE(tk == NULL)) { 1612 pr_warn("error on getting 2nd new probe.\n"); 1613 warn++; 1614 } else { 1615 file = find_trace_probe_file(tk, top_trace_array()); 1616 if (WARN_ON_ONCE(file == NULL)) { 1617 pr_warn("error on getting probe file.\n"); 1618 warn++; 1619 } else 1620 enable_trace_kprobe(tk, file); 1621 } 1622 } 1623 1624 if (warn) 1625 goto end; 1626 1627 ret = target(1, 2, 3, 4, 5, 6); 1628 1629 /* 1630 * Not expecting an error here, the check is only to prevent the 1631 * optimizer from removing the call to target() as otherwise there 1632 * are no side-effects and the call is never performed. 1633 */ 1634 if (ret != 21) 1635 warn++; 1636 1637 /* Disable trace points before removing it */ 1638 tk = find_trace_kprobe("testprobe", KPROBE_EVENT_SYSTEM); 1639 if (WARN_ON_ONCE(tk == NULL)) { 1640 pr_warn("error on getting test probe.\n"); 1641 warn++; 1642 } else { 1643 if (trace_kprobe_nhit(tk) != 1) { 1644 pr_warn("incorrect number of testprobe hits\n"); 1645 warn++; 1646 } 1647 1648 file = find_trace_probe_file(tk, top_trace_array()); 1649 if (WARN_ON_ONCE(file == NULL)) { 1650 pr_warn("error on getting probe file.\n"); 1651 warn++; 1652 } else 1653 disable_trace_kprobe(tk, file); 1654 } 1655 1656 tk = find_trace_kprobe("testprobe2", KPROBE_EVENT_SYSTEM); 1657 if (WARN_ON_ONCE(tk == NULL)) { 1658 pr_warn("error on getting 2nd test probe.\n"); 1659 warn++; 1660 } else { 1661 if (trace_kprobe_nhit(tk) != 1) { 1662 pr_warn("incorrect number of testprobe2 hits\n"); 1663 warn++; 1664 } 1665 1666 file = find_trace_probe_file(tk, top_trace_array()); 1667 if (WARN_ON_ONCE(file == NULL)) { 1668 pr_warn("error on getting probe file.\n"); 1669 warn++; 1670 } else 1671 disable_trace_kprobe(tk, file); 1672 } 1673 1674 ret = trace_run_command("-:testprobe", create_trace_kprobe); 1675 if (WARN_ON_ONCE(ret)) { 1676 pr_warn("error on deleting a probe.\n"); 1677 warn++; 1678 } 1679 1680 ret = trace_run_command("-:testprobe2", create_trace_kprobe); 1681 if (WARN_ON_ONCE(ret)) { 1682 pr_warn("error on deleting a probe.\n"); 1683 warn++; 1684 } 1685 1686 end: 1687 release_all_trace_kprobes(); 1688 /* 1689 * Wait for the optimizer work to finish. Otherwise it might fiddle 1690 * with probes in already freed __init text. 1691 */ 1692 wait_for_kprobe_optimizer(); 1693 if (warn) 1694 pr_cont("NG: Some tests are failed. Please check them.\n"); 1695 else 1696 pr_cont("OK\n"); 1697 return 0; 1698 } 1699 1700 late_initcall(kprobe_trace_self_tests_init); 1701 1702 #endif 1703