1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Kprobes-based tracing events 4 * 5 * Created by Masami Hiramatsu <mhiramat@redhat.com> 6 * 7 */ 8 #define pr_fmt(fmt) "trace_kprobe: " fmt 9 10 #include <linux/module.h> 11 #include <linux/uaccess.h> 12 #include <linux/rculist.h> 13 #include <linux/error-injection.h> 14 15 #include <asm/setup.h> /* for COMMAND_LINE_SIZE */ 16 17 #include "trace_dynevent.h" 18 #include "trace_kprobe_selftest.h" 19 #include "trace_probe.h" 20 #include "trace_probe_tmpl.h" 21 22 #define KPROBE_EVENT_SYSTEM "kprobes" 23 #define KRETPROBE_MAXACTIVE_MAX 4096 24 #define MAX_KPROBE_CMDLINE_SIZE 1024 25 26 /* Kprobe early definition from command line */ 27 static char kprobe_boot_events_buf[COMMAND_LINE_SIZE] __initdata; 28 static bool kprobe_boot_events_enabled __initdata; 29 30 static int __init set_kprobe_boot_events(char *str) 31 { 32 strlcpy(kprobe_boot_events_buf, str, COMMAND_LINE_SIZE); 33 return 0; 34 } 35 __setup("kprobe_event=", set_kprobe_boot_events); 36 37 static int trace_kprobe_create(int argc, const char **argv); 38 static int trace_kprobe_show(struct seq_file *m, struct dyn_event *ev); 39 static int trace_kprobe_release(struct dyn_event *ev); 40 static bool trace_kprobe_is_busy(struct dyn_event *ev); 41 static bool trace_kprobe_match(const char *system, const char *event, 42 struct dyn_event *ev); 43 44 static struct dyn_event_operations trace_kprobe_ops = { 45 .create = trace_kprobe_create, 46 .show = trace_kprobe_show, 47 .is_busy = trace_kprobe_is_busy, 48 .free = trace_kprobe_release, 49 .match = trace_kprobe_match, 50 }; 51 52 /* 53 * Kprobe event core functions 54 */ 55 struct trace_kprobe { 56 struct dyn_event devent; 57 struct kretprobe rp; /* Use rp.kp for kprobe use */ 58 unsigned long __percpu *nhit; 59 const char *symbol; /* symbol name */ 60 struct trace_probe tp; 61 }; 62 63 static bool is_trace_kprobe(struct dyn_event *ev) 64 { 65 return ev->ops == &trace_kprobe_ops; 66 } 67 68 static struct trace_kprobe *to_trace_kprobe(struct dyn_event *ev) 69 { 70 return container_of(ev, struct trace_kprobe, devent); 71 } 72 73 /** 74 * for_each_trace_kprobe - iterate over the trace_kprobe list 75 * @pos: the struct trace_kprobe * for each entry 76 * @dpos: the struct dyn_event * to use as a loop cursor 77 */ 78 #define for_each_trace_kprobe(pos, dpos) \ 79 for_each_dyn_event(dpos) \ 80 if (is_trace_kprobe(dpos) && (pos = to_trace_kprobe(dpos))) 81 82 #define SIZEOF_TRACE_KPROBE(n) \ 83 (offsetof(struct trace_kprobe, tp.args) + \ 84 (sizeof(struct probe_arg) * (n))) 85 86 static nokprobe_inline bool trace_kprobe_is_return(struct trace_kprobe *tk) 87 { 88 return tk->rp.handler != NULL; 89 } 90 91 static nokprobe_inline const char *trace_kprobe_symbol(struct trace_kprobe *tk) 92 { 93 return tk->symbol ? tk->symbol : "unknown"; 94 } 95 96 static nokprobe_inline unsigned long trace_kprobe_offset(struct trace_kprobe *tk) 97 { 98 return tk->rp.kp.offset; 99 } 100 101 static nokprobe_inline bool trace_kprobe_has_gone(struct trace_kprobe *tk) 102 { 103 return !!(kprobe_gone(&tk->rp.kp)); 104 } 105 106 static nokprobe_inline bool trace_kprobe_within_module(struct trace_kprobe *tk, 107 struct module *mod) 108 { 109 int len = strlen(mod->name); 110 const char *name = trace_kprobe_symbol(tk); 111 return strncmp(mod->name, name, len) == 0 && name[len] == ':'; 112 } 113 114 static nokprobe_inline bool trace_kprobe_module_exist(struct trace_kprobe *tk) 115 { 116 char *p; 117 bool ret; 118 119 if (!tk->symbol) 120 return false; 121 p = strchr(tk->symbol, ':'); 122 if (!p) 123 return true; 124 *p = '\0'; 125 mutex_lock(&module_mutex); 126 ret = !!find_module(tk->symbol); 127 mutex_unlock(&module_mutex); 128 *p = ':'; 129 130 return ret; 131 } 132 133 static bool trace_kprobe_is_busy(struct dyn_event *ev) 134 { 135 struct trace_kprobe *tk = to_trace_kprobe(ev); 136 137 return trace_probe_is_enabled(&tk->tp); 138 } 139 140 static bool trace_kprobe_match(const char *system, const char *event, 141 struct dyn_event *ev) 142 { 143 struct trace_kprobe *tk = to_trace_kprobe(ev); 144 145 return strcmp(trace_probe_name(&tk->tp), event) == 0 && 146 (!system || strcmp(trace_probe_group_name(&tk->tp), system) == 0); 147 } 148 149 static nokprobe_inline unsigned long trace_kprobe_nhit(struct trace_kprobe *tk) 150 { 151 unsigned long nhit = 0; 152 int cpu; 153 154 for_each_possible_cpu(cpu) 155 nhit += *per_cpu_ptr(tk->nhit, cpu); 156 157 return nhit; 158 } 159 160 static nokprobe_inline bool trace_kprobe_is_registered(struct trace_kprobe *tk) 161 { 162 return !(list_empty(&tk->rp.kp.list) && 163 hlist_unhashed(&tk->rp.kp.hlist)); 164 } 165 166 /* Return 0 if it fails to find the symbol address */ 167 static nokprobe_inline 168 unsigned long trace_kprobe_address(struct trace_kprobe *tk) 169 { 170 unsigned long addr; 171 172 if (tk->symbol) { 173 addr = (unsigned long) 174 kallsyms_lookup_name(trace_kprobe_symbol(tk)); 175 if (addr) 176 addr += tk->rp.kp.offset; 177 } else { 178 addr = (unsigned long)tk->rp.kp.addr; 179 } 180 return addr; 181 } 182 183 bool trace_kprobe_on_func_entry(struct trace_event_call *call) 184 { 185 struct trace_kprobe *tk = (struct trace_kprobe *)call->data; 186 187 return kprobe_on_func_entry(tk->rp.kp.addr, 188 tk->rp.kp.addr ? NULL : tk->rp.kp.symbol_name, 189 tk->rp.kp.addr ? 0 : tk->rp.kp.offset); 190 } 191 192 bool trace_kprobe_error_injectable(struct trace_event_call *call) 193 { 194 struct trace_kprobe *tk = (struct trace_kprobe *)call->data; 195 196 return within_error_injection_list(trace_kprobe_address(tk)); 197 } 198 199 static int register_kprobe_event(struct trace_kprobe *tk); 200 static int unregister_kprobe_event(struct trace_kprobe *tk); 201 202 static int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs); 203 static int kretprobe_dispatcher(struct kretprobe_instance *ri, 204 struct pt_regs *regs); 205 206 static void free_trace_kprobe(struct trace_kprobe *tk) 207 { 208 if (tk) { 209 trace_probe_cleanup(&tk->tp); 210 kfree(tk->symbol); 211 free_percpu(tk->nhit); 212 kfree(tk); 213 } 214 } 215 216 /* 217 * Allocate new trace_probe and initialize it (including kprobes). 218 */ 219 static struct trace_kprobe *alloc_trace_kprobe(const char *group, 220 const char *event, 221 void *addr, 222 const char *symbol, 223 unsigned long offs, 224 int maxactive, 225 int nargs, bool is_return) 226 { 227 struct trace_kprobe *tk; 228 int ret = -ENOMEM; 229 230 tk = kzalloc(SIZEOF_TRACE_KPROBE(nargs), GFP_KERNEL); 231 if (!tk) 232 return ERR_PTR(ret); 233 234 tk->nhit = alloc_percpu(unsigned long); 235 if (!tk->nhit) 236 goto error; 237 238 if (symbol) { 239 tk->symbol = kstrdup(symbol, GFP_KERNEL); 240 if (!tk->symbol) 241 goto error; 242 tk->rp.kp.symbol_name = tk->symbol; 243 tk->rp.kp.offset = offs; 244 } else 245 tk->rp.kp.addr = addr; 246 247 if (is_return) 248 tk->rp.handler = kretprobe_dispatcher; 249 else 250 tk->rp.kp.pre_handler = kprobe_dispatcher; 251 252 tk->rp.maxactive = maxactive; 253 INIT_HLIST_NODE(&tk->rp.kp.hlist); 254 INIT_LIST_HEAD(&tk->rp.kp.list); 255 256 ret = trace_probe_init(&tk->tp, event, group); 257 if (ret < 0) 258 goto error; 259 260 dyn_event_init(&tk->devent, &trace_kprobe_ops); 261 return tk; 262 error: 263 free_trace_kprobe(tk); 264 return ERR_PTR(ret); 265 } 266 267 static struct trace_kprobe *find_trace_kprobe(const char *event, 268 const char *group) 269 { 270 struct dyn_event *pos; 271 struct trace_kprobe *tk; 272 273 for_each_trace_kprobe(tk, pos) 274 if (strcmp(trace_probe_name(&tk->tp), event) == 0 && 275 strcmp(trace_probe_group_name(&tk->tp), group) == 0) 276 return tk; 277 return NULL; 278 } 279 280 static inline int __enable_trace_kprobe(struct trace_kprobe *tk) 281 { 282 int ret = 0; 283 284 if (trace_kprobe_is_registered(tk) && !trace_kprobe_has_gone(tk)) { 285 if (trace_kprobe_is_return(tk)) 286 ret = enable_kretprobe(&tk->rp); 287 else 288 ret = enable_kprobe(&tk->rp.kp); 289 } 290 291 return ret; 292 } 293 294 /* 295 * Enable trace_probe 296 * if the file is NULL, enable "perf" handler, or enable "trace" handler. 297 */ 298 static int 299 enable_trace_kprobe(struct trace_kprobe *tk, struct trace_event_file *file) 300 { 301 bool enabled = trace_probe_is_enabled(&tk->tp); 302 int ret = 0; 303 304 if (file) { 305 ret = trace_probe_add_file(&tk->tp, file); 306 if (ret) 307 return ret; 308 } else 309 trace_probe_set_flag(&tk->tp, TP_FLAG_PROFILE); 310 311 if (enabled) 312 return 0; 313 314 ret = __enable_trace_kprobe(tk); 315 if (ret) { 316 if (file) 317 trace_probe_remove_file(&tk->tp, file); 318 else 319 trace_probe_clear_flag(&tk->tp, TP_FLAG_PROFILE); 320 } 321 322 return ret; 323 } 324 325 /* 326 * Disable trace_probe 327 * if the file is NULL, disable "perf" handler, or disable "trace" handler. 328 */ 329 static int 330 disable_trace_kprobe(struct trace_kprobe *tk, struct trace_event_file *file) 331 { 332 struct trace_probe *tp = &tk->tp; 333 int ret = 0; 334 335 if (file) { 336 if (!trace_probe_get_file_link(tp, file)) 337 return -ENOENT; 338 if (!trace_probe_has_single_file(tp)) 339 goto out; 340 trace_probe_clear_flag(tp, TP_FLAG_TRACE); 341 } else 342 trace_probe_clear_flag(tp, TP_FLAG_PROFILE); 343 344 if (!trace_probe_is_enabled(tp) && trace_kprobe_is_registered(tk)) { 345 if (trace_kprobe_is_return(tk)) 346 disable_kretprobe(&tk->rp); 347 else 348 disable_kprobe(&tk->rp.kp); 349 } 350 351 out: 352 if (file) 353 /* 354 * Synchronization is done in below function. For perf event, 355 * file == NULL and perf_trace_event_unreg() calls 356 * tracepoint_synchronize_unregister() to ensure synchronize 357 * event. We don't need to care about it. 358 */ 359 trace_probe_remove_file(tp, file); 360 361 return ret; 362 } 363 364 #if defined(CONFIG_KPROBES_ON_FTRACE) && \ 365 !defined(CONFIG_KPROBE_EVENTS_ON_NOTRACE) 366 static bool within_notrace_func(struct trace_kprobe *tk) 367 { 368 unsigned long offset, size, addr; 369 370 addr = trace_kprobe_address(tk); 371 if (!addr || !kallsyms_lookup_size_offset(addr, &size, &offset)) 372 return false; 373 374 /* Get the entry address of the target function */ 375 addr -= offset; 376 377 /* 378 * Since ftrace_location_range() does inclusive range check, we need 379 * to subtract 1 byte from the end address. 380 */ 381 return !ftrace_location_range(addr, addr + size - 1); 382 } 383 #else 384 #define within_notrace_func(tk) (false) 385 #endif 386 387 /* Internal register function - just handle k*probes and flags */ 388 static int __register_trace_kprobe(struct trace_kprobe *tk) 389 { 390 int i, ret; 391 392 if (trace_kprobe_is_registered(tk)) 393 return -EINVAL; 394 395 if (within_notrace_func(tk)) { 396 pr_warn("Could not probe notrace function %s\n", 397 trace_kprobe_symbol(tk)); 398 return -EINVAL; 399 } 400 401 for (i = 0; i < tk->tp.nr_args; i++) { 402 ret = traceprobe_update_arg(&tk->tp.args[i]); 403 if (ret) 404 return ret; 405 } 406 407 /* Set/clear disabled flag according to tp->flag */ 408 if (trace_probe_is_enabled(&tk->tp)) 409 tk->rp.kp.flags &= ~KPROBE_FLAG_DISABLED; 410 else 411 tk->rp.kp.flags |= KPROBE_FLAG_DISABLED; 412 413 if (trace_kprobe_is_return(tk)) 414 ret = register_kretprobe(&tk->rp); 415 else 416 ret = register_kprobe(&tk->rp.kp); 417 418 return ret; 419 } 420 421 /* Internal unregister function - just handle k*probes and flags */ 422 static void __unregister_trace_kprobe(struct trace_kprobe *tk) 423 { 424 if (trace_kprobe_is_registered(tk)) { 425 if (trace_kprobe_is_return(tk)) 426 unregister_kretprobe(&tk->rp); 427 else 428 unregister_kprobe(&tk->rp.kp); 429 /* Cleanup kprobe for reuse and mark it unregistered */ 430 INIT_HLIST_NODE(&tk->rp.kp.hlist); 431 INIT_LIST_HEAD(&tk->rp.kp.list); 432 if (tk->rp.kp.symbol_name) 433 tk->rp.kp.addr = NULL; 434 } 435 } 436 437 /* Unregister a trace_probe and probe_event */ 438 static int unregister_trace_kprobe(struct trace_kprobe *tk) 439 { 440 /* Enabled event can not be unregistered */ 441 if (trace_probe_is_enabled(&tk->tp)) 442 return -EBUSY; 443 444 /* Will fail if probe is being used by ftrace or perf */ 445 if (unregister_kprobe_event(tk)) 446 return -EBUSY; 447 448 __unregister_trace_kprobe(tk); 449 dyn_event_remove(&tk->devent); 450 451 return 0; 452 } 453 454 /* Register a trace_probe and probe_event */ 455 static int register_trace_kprobe(struct trace_kprobe *tk) 456 { 457 struct trace_kprobe *old_tk; 458 int ret; 459 460 mutex_lock(&event_mutex); 461 462 /* Delete old (same name) event if exist */ 463 old_tk = find_trace_kprobe(trace_probe_name(&tk->tp), 464 trace_probe_group_name(&tk->tp)); 465 if (old_tk) { 466 ret = unregister_trace_kprobe(old_tk); 467 if (ret < 0) 468 goto end; 469 free_trace_kprobe(old_tk); 470 } 471 472 /* Register new event */ 473 ret = register_kprobe_event(tk); 474 if (ret) { 475 pr_warn("Failed to register probe event(%d)\n", ret); 476 goto end; 477 } 478 479 /* Register k*probe */ 480 ret = __register_trace_kprobe(tk); 481 if (ret == -ENOENT && !trace_kprobe_module_exist(tk)) { 482 pr_warn("This probe might be able to register after target module is loaded. Continue.\n"); 483 ret = 0; 484 } 485 486 if (ret < 0) 487 unregister_kprobe_event(tk); 488 else 489 dyn_event_add(&tk->devent); 490 491 end: 492 mutex_unlock(&event_mutex); 493 return ret; 494 } 495 496 /* Module notifier call back, checking event on the module */ 497 static int trace_kprobe_module_callback(struct notifier_block *nb, 498 unsigned long val, void *data) 499 { 500 struct module *mod = data; 501 struct dyn_event *pos; 502 struct trace_kprobe *tk; 503 int ret; 504 505 if (val != MODULE_STATE_COMING) 506 return NOTIFY_DONE; 507 508 /* Update probes on coming module */ 509 mutex_lock(&event_mutex); 510 for_each_trace_kprobe(tk, pos) { 511 if (trace_kprobe_within_module(tk, mod)) { 512 /* Don't need to check busy - this should have gone. */ 513 __unregister_trace_kprobe(tk); 514 ret = __register_trace_kprobe(tk); 515 if (ret) 516 pr_warn("Failed to re-register probe %s on %s: %d\n", 517 trace_probe_name(&tk->tp), 518 mod->name, ret); 519 } 520 } 521 mutex_unlock(&event_mutex); 522 523 return NOTIFY_DONE; 524 } 525 526 static struct notifier_block trace_kprobe_module_nb = { 527 .notifier_call = trace_kprobe_module_callback, 528 .priority = 1 /* Invoked after kprobe module callback */ 529 }; 530 531 /* Convert certain expected symbols into '_' when generating event names */ 532 static inline void sanitize_event_name(char *name) 533 { 534 while (*name++ != '\0') 535 if (*name == ':' || *name == '.') 536 *name = '_'; 537 } 538 539 static int trace_kprobe_create(int argc, const char *argv[]) 540 { 541 /* 542 * Argument syntax: 543 * - Add kprobe: 544 * p[:[GRP/]EVENT] [MOD:]KSYM[+OFFS]|KADDR [FETCHARGS] 545 * - Add kretprobe: 546 * r[MAXACTIVE][:[GRP/]EVENT] [MOD:]KSYM[+0] [FETCHARGS] 547 * Fetch args: 548 * $retval : fetch return value 549 * $stack : fetch stack address 550 * $stackN : fetch Nth of stack (N:0-) 551 * $comm : fetch current task comm 552 * @ADDR : fetch memory at ADDR (ADDR should be in kernel) 553 * @SYM[+|-offs] : fetch memory at SYM +|- offs (SYM is a data symbol) 554 * %REG : fetch register REG 555 * Dereferencing memory fetch: 556 * +|-offs(ARG) : fetch memory at ARG +|- offs address. 557 * Alias name of args: 558 * NAME=FETCHARG : set NAME as alias of FETCHARG. 559 * Type of args: 560 * FETCHARG:TYPE : use TYPE instead of unsigned long. 561 */ 562 struct trace_kprobe *tk = NULL; 563 int i, len, ret = 0; 564 bool is_return = false; 565 char *symbol = NULL, *tmp = NULL; 566 const char *event = NULL, *group = KPROBE_EVENT_SYSTEM; 567 int maxactive = 0; 568 long offset = 0; 569 void *addr = NULL; 570 char buf[MAX_EVENT_NAME_LEN]; 571 unsigned int flags = TPARG_FL_KERNEL; 572 573 switch (argv[0][0]) { 574 case 'r': 575 is_return = true; 576 flags |= TPARG_FL_RETURN; 577 break; 578 case 'p': 579 break; 580 default: 581 return -ECANCELED; 582 } 583 if (argc < 2) 584 return -ECANCELED; 585 586 trace_probe_log_init("trace_kprobe", argc, argv); 587 588 event = strchr(&argv[0][1], ':'); 589 if (event) 590 event++; 591 592 if (isdigit(argv[0][1])) { 593 if (!is_return) { 594 trace_probe_log_err(1, MAXACT_NO_KPROBE); 595 goto parse_error; 596 } 597 if (event) 598 len = event - &argv[0][1] - 1; 599 else 600 len = strlen(&argv[0][1]); 601 if (len > MAX_EVENT_NAME_LEN - 1) { 602 trace_probe_log_err(1, BAD_MAXACT); 603 goto parse_error; 604 } 605 memcpy(buf, &argv[0][1], len); 606 buf[len] = '\0'; 607 ret = kstrtouint(buf, 0, &maxactive); 608 if (ret || !maxactive) { 609 trace_probe_log_err(1, BAD_MAXACT); 610 goto parse_error; 611 } 612 /* kretprobes instances are iterated over via a list. The 613 * maximum should stay reasonable. 614 */ 615 if (maxactive > KRETPROBE_MAXACTIVE_MAX) { 616 trace_probe_log_err(1, MAXACT_TOO_BIG); 617 goto parse_error; 618 } 619 } 620 621 /* try to parse an address. if that fails, try to read the 622 * input as a symbol. */ 623 if (kstrtoul(argv[1], 0, (unsigned long *)&addr)) { 624 trace_probe_log_set_index(1); 625 /* Check whether uprobe event specified */ 626 if (strchr(argv[1], '/') && strchr(argv[1], ':')) { 627 ret = -ECANCELED; 628 goto error; 629 } 630 /* a symbol specified */ 631 symbol = kstrdup(argv[1], GFP_KERNEL); 632 if (!symbol) 633 return -ENOMEM; 634 /* TODO: support .init module functions */ 635 ret = traceprobe_split_symbol_offset(symbol, &offset); 636 if (ret || offset < 0 || offset > UINT_MAX) { 637 trace_probe_log_err(0, BAD_PROBE_ADDR); 638 goto parse_error; 639 } 640 if (kprobe_on_func_entry(NULL, symbol, offset)) 641 flags |= TPARG_FL_FENTRY; 642 if (offset && is_return && !(flags & TPARG_FL_FENTRY)) { 643 trace_probe_log_err(0, BAD_RETPROBE); 644 goto parse_error; 645 } 646 } 647 648 trace_probe_log_set_index(0); 649 if (event) { 650 ret = traceprobe_parse_event_name(&event, &group, buf, 651 event - argv[0]); 652 if (ret) 653 goto parse_error; 654 } else { 655 /* Make a new event name */ 656 if (symbol) 657 snprintf(buf, MAX_EVENT_NAME_LEN, "%c_%s_%ld", 658 is_return ? 'r' : 'p', symbol, offset); 659 else 660 snprintf(buf, MAX_EVENT_NAME_LEN, "%c_0x%p", 661 is_return ? 'r' : 'p', addr); 662 sanitize_event_name(buf); 663 event = buf; 664 } 665 666 /* setup a probe */ 667 tk = alloc_trace_kprobe(group, event, addr, symbol, offset, maxactive, 668 argc - 2, is_return); 669 if (IS_ERR(tk)) { 670 ret = PTR_ERR(tk); 671 /* This must return -ENOMEM, else there is a bug */ 672 WARN_ON_ONCE(ret != -ENOMEM); 673 goto out; /* We know tk is not allocated */ 674 } 675 argc -= 2; argv += 2; 676 677 /* parse arguments */ 678 for (i = 0; i < argc && i < MAX_TRACE_ARGS; i++) { 679 tmp = kstrdup(argv[i], GFP_KERNEL); 680 if (!tmp) { 681 ret = -ENOMEM; 682 goto error; 683 } 684 685 trace_probe_log_set_index(i + 2); 686 ret = traceprobe_parse_probe_arg(&tk->tp, i, tmp, flags); 687 kfree(tmp); 688 if (ret) 689 goto error; /* This can be -ENOMEM */ 690 } 691 692 ret = traceprobe_set_print_fmt(&tk->tp, is_return); 693 if (ret < 0) 694 goto error; 695 696 ret = register_trace_kprobe(tk); 697 if (ret) { 698 trace_probe_log_set_index(1); 699 if (ret == -EILSEQ) 700 trace_probe_log_err(0, BAD_INSN_BNDRY); 701 else if (ret == -ENOENT) 702 trace_probe_log_err(0, BAD_PROBE_ADDR); 703 else if (ret != -ENOMEM) 704 trace_probe_log_err(0, FAIL_REG_PROBE); 705 goto error; 706 } 707 708 out: 709 trace_probe_log_clear(); 710 kfree(symbol); 711 return ret; 712 713 parse_error: 714 ret = -EINVAL; 715 error: 716 free_trace_kprobe(tk); 717 goto out; 718 } 719 720 static int create_or_delete_trace_kprobe(int argc, char **argv) 721 { 722 int ret; 723 724 if (argv[0][0] == '-') 725 return dyn_event_release(argc, argv, &trace_kprobe_ops); 726 727 ret = trace_kprobe_create(argc, (const char **)argv); 728 return ret == -ECANCELED ? -EINVAL : ret; 729 } 730 731 static int trace_kprobe_release(struct dyn_event *ev) 732 { 733 struct trace_kprobe *tk = to_trace_kprobe(ev); 734 int ret = unregister_trace_kprobe(tk); 735 736 if (!ret) 737 free_trace_kprobe(tk); 738 return ret; 739 } 740 741 static int trace_kprobe_show(struct seq_file *m, struct dyn_event *ev) 742 { 743 struct trace_kprobe *tk = to_trace_kprobe(ev); 744 int i; 745 746 seq_putc(m, trace_kprobe_is_return(tk) ? 'r' : 'p'); 747 seq_printf(m, ":%s/%s", trace_probe_group_name(&tk->tp), 748 trace_probe_name(&tk->tp)); 749 750 if (!tk->symbol) 751 seq_printf(m, " 0x%p", tk->rp.kp.addr); 752 else if (tk->rp.kp.offset) 753 seq_printf(m, " %s+%u", trace_kprobe_symbol(tk), 754 tk->rp.kp.offset); 755 else 756 seq_printf(m, " %s", trace_kprobe_symbol(tk)); 757 758 for (i = 0; i < tk->tp.nr_args; i++) 759 seq_printf(m, " %s=%s", tk->tp.args[i].name, tk->tp.args[i].comm); 760 seq_putc(m, '\n'); 761 762 return 0; 763 } 764 765 static int probes_seq_show(struct seq_file *m, void *v) 766 { 767 struct dyn_event *ev = v; 768 769 if (!is_trace_kprobe(ev)) 770 return 0; 771 772 return trace_kprobe_show(m, ev); 773 } 774 775 static const struct seq_operations probes_seq_op = { 776 .start = dyn_event_seq_start, 777 .next = dyn_event_seq_next, 778 .stop = dyn_event_seq_stop, 779 .show = probes_seq_show 780 }; 781 782 static int probes_open(struct inode *inode, struct file *file) 783 { 784 int ret; 785 786 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) { 787 ret = dyn_events_release_all(&trace_kprobe_ops); 788 if (ret < 0) 789 return ret; 790 } 791 792 return seq_open(file, &probes_seq_op); 793 } 794 795 static ssize_t probes_write(struct file *file, const char __user *buffer, 796 size_t count, loff_t *ppos) 797 { 798 return trace_parse_run_command(file, buffer, count, ppos, 799 create_or_delete_trace_kprobe); 800 } 801 802 static const struct file_operations kprobe_events_ops = { 803 .owner = THIS_MODULE, 804 .open = probes_open, 805 .read = seq_read, 806 .llseek = seq_lseek, 807 .release = seq_release, 808 .write = probes_write, 809 }; 810 811 /* Probes profiling interfaces */ 812 static int probes_profile_seq_show(struct seq_file *m, void *v) 813 { 814 struct dyn_event *ev = v; 815 struct trace_kprobe *tk; 816 817 if (!is_trace_kprobe(ev)) 818 return 0; 819 820 tk = to_trace_kprobe(ev); 821 seq_printf(m, " %-44s %15lu %15lu\n", 822 trace_probe_name(&tk->tp), 823 trace_kprobe_nhit(tk), 824 tk->rp.kp.nmissed); 825 826 return 0; 827 } 828 829 static const struct seq_operations profile_seq_op = { 830 .start = dyn_event_seq_start, 831 .next = dyn_event_seq_next, 832 .stop = dyn_event_seq_stop, 833 .show = probes_profile_seq_show 834 }; 835 836 static int profile_open(struct inode *inode, struct file *file) 837 { 838 return seq_open(file, &profile_seq_op); 839 } 840 841 static const struct file_operations kprobe_profile_ops = { 842 .owner = THIS_MODULE, 843 .open = profile_open, 844 .read = seq_read, 845 .llseek = seq_lseek, 846 .release = seq_release, 847 }; 848 849 /* Kprobe specific fetch functions */ 850 851 /* Return the length of string -- including null terminal byte */ 852 static nokprobe_inline int 853 fetch_store_strlen(unsigned long addr) 854 { 855 int ret, len = 0; 856 u8 c; 857 858 do { 859 ret = probe_kernel_read(&c, (u8 *)addr + len, 1); 860 len++; 861 } while (c && ret == 0 && len < MAX_STRING_SIZE); 862 863 return (ret < 0) ? ret : len; 864 } 865 866 /* Return the length of string -- including null terminal byte */ 867 static nokprobe_inline int 868 fetch_store_strlen_user(unsigned long addr) 869 { 870 const void __user *uaddr = (__force const void __user *)addr; 871 872 return strnlen_unsafe_user(uaddr, MAX_STRING_SIZE); 873 } 874 875 /* 876 * Fetch a null-terminated string. Caller MUST set *(u32 *)buf with max 877 * length and relative data location. 878 */ 879 static nokprobe_inline int 880 fetch_store_string(unsigned long addr, void *dest, void *base) 881 { 882 int maxlen = get_loc_len(*(u32 *)dest); 883 void *__dest; 884 long ret; 885 886 if (unlikely(!maxlen)) 887 return -ENOMEM; 888 889 __dest = get_loc_data(dest, base); 890 891 /* 892 * Try to get string again, since the string can be changed while 893 * probing. 894 */ 895 ret = strncpy_from_unsafe(__dest, (void *)addr, maxlen); 896 if (ret >= 0) 897 *(u32 *)dest = make_data_loc(ret, __dest - base); 898 899 return ret; 900 } 901 902 /* 903 * Fetch a null-terminated string from user. Caller MUST set *(u32 *)buf 904 * with max length and relative data location. 905 */ 906 static nokprobe_inline int 907 fetch_store_string_user(unsigned long addr, void *dest, void *base) 908 { 909 const void __user *uaddr = (__force const void __user *)addr; 910 int maxlen = get_loc_len(*(u32 *)dest); 911 void *__dest; 912 long ret; 913 914 if (unlikely(!maxlen)) 915 return -ENOMEM; 916 917 __dest = get_loc_data(dest, base); 918 919 ret = strncpy_from_unsafe_user(__dest, uaddr, maxlen); 920 if (ret >= 0) 921 *(u32 *)dest = make_data_loc(ret, __dest - base); 922 923 return ret; 924 } 925 926 static nokprobe_inline int 927 probe_mem_read(void *dest, void *src, size_t size) 928 { 929 return probe_kernel_read(dest, src, size); 930 } 931 932 static nokprobe_inline int 933 probe_mem_read_user(void *dest, void *src, size_t size) 934 { 935 const void __user *uaddr = (__force const void __user *)src; 936 937 return probe_user_read(dest, uaddr, size); 938 } 939 940 /* Note that we don't verify it, since the code does not come from user space */ 941 static int 942 process_fetch_insn(struct fetch_insn *code, struct pt_regs *regs, void *dest, 943 void *base) 944 { 945 unsigned long val; 946 947 retry: 948 /* 1st stage: get value from context */ 949 switch (code->op) { 950 case FETCH_OP_REG: 951 val = regs_get_register(regs, code->param); 952 break; 953 case FETCH_OP_STACK: 954 val = regs_get_kernel_stack_nth(regs, code->param); 955 break; 956 case FETCH_OP_STACKP: 957 val = kernel_stack_pointer(regs); 958 break; 959 case FETCH_OP_RETVAL: 960 val = regs_return_value(regs); 961 break; 962 case FETCH_OP_IMM: 963 val = code->immediate; 964 break; 965 case FETCH_OP_COMM: 966 val = (unsigned long)current->comm; 967 break; 968 #ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API 969 case FETCH_OP_ARG: 970 val = regs_get_kernel_argument(regs, code->param); 971 break; 972 #endif 973 case FETCH_NOP_SYMBOL: /* Ignore a place holder */ 974 code++; 975 goto retry; 976 default: 977 return -EILSEQ; 978 } 979 code++; 980 981 return process_fetch_insn_bottom(code, val, dest, base); 982 } 983 NOKPROBE_SYMBOL(process_fetch_insn) 984 985 /* Kprobe handler */ 986 static nokprobe_inline void 987 __kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs, 988 struct trace_event_file *trace_file) 989 { 990 struct kprobe_trace_entry_head *entry; 991 struct ring_buffer_event *event; 992 struct ring_buffer *buffer; 993 int size, dsize, pc; 994 unsigned long irq_flags; 995 struct trace_event_call *call = trace_probe_event_call(&tk->tp); 996 997 WARN_ON(call != trace_file->event_call); 998 999 if (trace_trigger_soft_disabled(trace_file)) 1000 return; 1001 1002 local_save_flags(irq_flags); 1003 pc = preempt_count(); 1004 1005 dsize = __get_data_size(&tk->tp, regs); 1006 size = sizeof(*entry) + tk->tp.size + dsize; 1007 1008 event = trace_event_buffer_lock_reserve(&buffer, trace_file, 1009 call->event.type, 1010 size, irq_flags, pc); 1011 if (!event) 1012 return; 1013 1014 entry = ring_buffer_event_data(event); 1015 entry->ip = (unsigned long)tk->rp.kp.addr; 1016 store_trace_args(&entry[1], &tk->tp, regs, sizeof(*entry), dsize); 1017 1018 event_trigger_unlock_commit_regs(trace_file, buffer, event, 1019 entry, irq_flags, pc, regs); 1020 } 1021 1022 static void 1023 kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs) 1024 { 1025 struct event_file_link *link; 1026 1027 trace_probe_for_each_link_rcu(link, &tk->tp) 1028 __kprobe_trace_func(tk, regs, link->file); 1029 } 1030 NOKPROBE_SYMBOL(kprobe_trace_func); 1031 1032 /* Kretprobe handler */ 1033 static nokprobe_inline void 1034 __kretprobe_trace_func(struct trace_kprobe *tk, struct kretprobe_instance *ri, 1035 struct pt_regs *regs, 1036 struct trace_event_file *trace_file) 1037 { 1038 struct kretprobe_trace_entry_head *entry; 1039 struct ring_buffer_event *event; 1040 struct ring_buffer *buffer; 1041 int size, pc, dsize; 1042 unsigned long irq_flags; 1043 struct trace_event_call *call = trace_probe_event_call(&tk->tp); 1044 1045 WARN_ON(call != trace_file->event_call); 1046 1047 if (trace_trigger_soft_disabled(trace_file)) 1048 return; 1049 1050 local_save_flags(irq_flags); 1051 pc = preempt_count(); 1052 1053 dsize = __get_data_size(&tk->tp, regs); 1054 size = sizeof(*entry) + tk->tp.size + dsize; 1055 1056 event = trace_event_buffer_lock_reserve(&buffer, trace_file, 1057 call->event.type, 1058 size, irq_flags, pc); 1059 if (!event) 1060 return; 1061 1062 entry = ring_buffer_event_data(event); 1063 entry->func = (unsigned long)tk->rp.kp.addr; 1064 entry->ret_ip = (unsigned long)ri->ret_addr; 1065 store_trace_args(&entry[1], &tk->tp, regs, sizeof(*entry), dsize); 1066 1067 event_trigger_unlock_commit_regs(trace_file, buffer, event, 1068 entry, irq_flags, pc, regs); 1069 } 1070 1071 static void 1072 kretprobe_trace_func(struct trace_kprobe *tk, struct kretprobe_instance *ri, 1073 struct pt_regs *regs) 1074 { 1075 struct event_file_link *link; 1076 1077 trace_probe_for_each_link_rcu(link, &tk->tp) 1078 __kretprobe_trace_func(tk, ri, regs, link->file); 1079 } 1080 NOKPROBE_SYMBOL(kretprobe_trace_func); 1081 1082 /* Event entry printers */ 1083 static enum print_line_t 1084 print_kprobe_event(struct trace_iterator *iter, int flags, 1085 struct trace_event *event) 1086 { 1087 struct kprobe_trace_entry_head *field; 1088 struct trace_seq *s = &iter->seq; 1089 struct trace_probe *tp; 1090 1091 field = (struct kprobe_trace_entry_head *)iter->ent; 1092 tp = container_of(event, struct trace_probe, call.event); 1093 1094 trace_seq_printf(s, "%s: (", trace_probe_name(tp)); 1095 1096 if (!seq_print_ip_sym(s, field->ip, flags | TRACE_ITER_SYM_OFFSET)) 1097 goto out; 1098 1099 trace_seq_putc(s, ')'); 1100 1101 if (print_probe_args(s, tp->args, tp->nr_args, 1102 (u8 *)&field[1], field) < 0) 1103 goto out; 1104 1105 trace_seq_putc(s, '\n'); 1106 out: 1107 return trace_handle_return(s); 1108 } 1109 1110 static enum print_line_t 1111 print_kretprobe_event(struct trace_iterator *iter, int flags, 1112 struct trace_event *event) 1113 { 1114 struct kretprobe_trace_entry_head *field; 1115 struct trace_seq *s = &iter->seq; 1116 struct trace_probe *tp; 1117 1118 field = (struct kretprobe_trace_entry_head *)iter->ent; 1119 tp = container_of(event, struct trace_probe, call.event); 1120 1121 trace_seq_printf(s, "%s: (", trace_probe_name(tp)); 1122 1123 if (!seq_print_ip_sym(s, field->ret_ip, flags | TRACE_ITER_SYM_OFFSET)) 1124 goto out; 1125 1126 trace_seq_puts(s, " <- "); 1127 1128 if (!seq_print_ip_sym(s, field->func, flags & ~TRACE_ITER_SYM_OFFSET)) 1129 goto out; 1130 1131 trace_seq_putc(s, ')'); 1132 1133 if (print_probe_args(s, tp->args, tp->nr_args, 1134 (u8 *)&field[1], field) < 0) 1135 goto out; 1136 1137 trace_seq_putc(s, '\n'); 1138 1139 out: 1140 return trace_handle_return(s); 1141 } 1142 1143 1144 static int kprobe_event_define_fields(struct trace_event_call *event_call) 1145 { 1146 int ret; 1147 struct kprobe_trace_entry_head field; 1148 struct trace_kprobe *tk = (struct trace_kprobe *)event_call->data; 1149 1150 DEFINE_FIELD(unsigned long, ip, FIELD_STRING_IP, 0); 1151 1152 return traceprobe_define_arg_fields(event_call, sizeof(field), &tk->tp); 1153 } 1154 1155 static int kretprobe_event_define_fields(struct trace_event_call *event_call) 1156 { 1157 int ret; 1158 struct kretprobe_trace_entry_head field; 1159 struct trace_kprobe *tk = (struct trace_kprobe *)event_call->data; 1160 1161 DEFINE_FIELD(unsigned long, func, FIELD_STRING_FUNC, 0); 1162 DEFINE_FIELD(unsigned long, ret_ip, FIELD_STRING_RETIP, 0); 1163 1164 return traceprobe_define_arg_fields(event_call, sizeof(field), &tk->tp); 1165 } 1166 1167 #ifdef CONFIG_PERF_EVENTS 1168 1169 /* Kprobe profile handler */ 1170 static int 1171 kprobe_perf_func(struct trace_kprobe *tk, struct pt_regs *regs) 1172 { 1173 struct trace_event_call *call = trace_probe_event_call(&tk->tp); 1174 struct kprobe_trace_entry_head *entry; 1175 struct hlist_head *head; 1176 int size, __size, dsize; 1177 int rctx; 1178 1179 if (bpf_prog_array_valid(call)) { 1180 unsigned long orig_ip = instruction_pointer(regs); 1181 int ret; 1182 1183 ret = trace_call_bpf(call, regs); 1184 1185 /* 1186 * We need to check and see if we modified the pc of the 1187 * pt_regs, and if so return 1 so that we don't do the 1188 * single stepping. 1189 */ 1190 if (orig_ip != instruction_pointer(regs)) 1191 return 1; 1192 if (!ret) 1193 return 0; 1194 } 1195 1196 head = this_cpu_ptr(call->perf_events); 1197 if (hlist_empty(head)) 1198 return 0; 1199 1200 dsize = __get_data_size(&tk->tp, regs); 1201 __size = sizeof(*entry) + tk->tp.size + dsize; 1202 size = ALIGN(__size + sizeof(u32), sizeof(u64)); 1203 size -= sizeof(u32); 1204 1205 entry = perf_trace_buf_alloc(size, NULL, &rctx); 1206 if (!entry) 1207 return 0; 1208 1209 entry->ip = (unsigned long)tk->rp.kp.addr; 1210 memset(&entry[1], 0, dsize); 1211 store_trace_args(&entry[1], &tk->tp, regs, sizeof(*entry), dsize); 1212 perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs, 1213 head, NULL); 1214 return 0; 1215 } 1216 NOKPROBE_SYMBOL(kprobe_perf_func); 1217 1218 /* Kretprobe profile handler */ 1219 static void 1220 kretprobe_perf_func(struct trace_kprobe *tk, struct kretprobe_instance *ri, 1221 struct pt_regs *regs) 1222 { 1223 struct trace_event_call *call = trace_probe_event_call(&tk->tp); 1224 struct kretprobe_trace_entry_head *entry; 1225 struct hlist_head *head; 1226 int size, __size, dsize; 1227 int rctx; 1228 1229 if (bpf_prog_array_valid(call) && !trace_call_bpf(call, regs)) 1230 return; 1231 1232 head = this_cpu_ptr(call->perf_events); 1233 if (hlist_empty(head)) 1234 return; 1235 1236 dsize = __get_data_size(&tk->tp, regs); 1237 __size = sizeof(*entry) + tk->tp.size + dsize; 1238 size = ALIGN(__size + sizeof(u32), sizeof(u64)); 1239 size -= sizeof(u32); 1240 1241 entry = perf_trace_buf_alloc(size, NULL, &rctx); 1242 if (!entry) 1243 return; 1244 1245 entry->func = (unsigned long)tk->rp.kp.addr; 1246 entry->ret_ip = (unsigned long)ri->ret_addr; 1247 store_trace_args(&entry[1], &tk->tp, regs, sizeof(*entry), dsize); 1248 perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs, 1249 head, NULL); 1250 } 1251 NOKPROBE_SYMBOL(kretprobe_perf_func); 1252 1253 int bpf_get_kprobe_info(const struct perf_event *event, u32 *fd_type, 1254 const char **symbol, u64 *probe_offset, 1255 u64 *probe_addr, bool perf_type_tracepoint) 1256 { 1257 const char *pevent = trace_event_name(event->tp_event); 1258 const char *group = event->tp_event->class->system; 1259 struct trace_kprobe *tk; 1260 1261 if (perf_type_tracepoint) 1262 tk = find_trace_kprobe(pevent, group); 1263 else 1264 tk = event->tp_event->data; 1265 if (!tk) 1266 return -EINVAL; 1267 1268 *fd_type = trace_kprobe_is_return(tk) ? BPF_FD_TYPE_KRETPROBE 1269 : BPF_FD_TYPE_KPROBE; 1270 if (tk->symbol) { 1271 *symbol = tk->symbol; 1272 *probe_offset = tk->rp.kp.offset; 1273 *probe_addr = 0; 1274 } else { 1275 *symbol = NULL; 1276 *probe_offset = 0; 1277 *probe_addr = (unsigned long)tk->rp.kp.addr; 1278 } 1279 return 0; 1280 } 1281 #endif /* CONFIG_PERF_EVENTS */ 1282 1283 /* 1284 * called by perf_trace_init() or __ftrace_set_clr_event() under event_mutex. 1285 * 1286 * kprobe_trace_self_tests_init() does enable_trace_probe/disable_trace_probe 1287 * lockless, but we can't race with this __init function. 1288 */ 1289 static int kprobe_register(struct trace_event_call *event, 1290 enum trace_reg type, void *data) 1291 { 1292 struct trace_kprobe *tk = (struct trace_kprobe *)event->data; 1293 struct trace_event_file *file = data; 1294 1295 switch (type) { 1296 case TRACE_REG_REGISTER: 1297 return enable_trace_kprobe(tk, file); 1298 case TRACE_REG_UNREGISTER: 1299 return disable_trace_kprobe(tk, file); 1300 1301 #ifdef CONFIG_PERF_EVENTS 1302 case TRACE_REG_PERF_REGISTER: 1303 return enable_trace_kprobe(tk, NULL); 1304 case TRACE_REG_PERF_UNREGISTER: 1305 return disable_trace_kprobe(tk, NULL); 1306 case TRACE_REG_PERF_OPEN: 1307 case TRACE_REG_PERF_CLOSE: 1308 case TRACE_REG_PERF_ADD: 1309 case TRACE_REG_PERF_DEL: 1310 return 0; 1311 #endif 1312 } 1313 return 0; 1314 } 1315 1316 static int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs) 1317 { 1318 struct trace_kprobe *tk = container_of(kp, struct trace_kprobe, rp.kp); 1319 int ret = 0; 1320 1321 raw_cpu_inc(*tk->nhit); 1322 1323 if (trace_probe_test_flag(&tk->tp, TP_FLAG_TRACE)) 1324 kprobe_trace_func(tk, regs); 1325 #ifdef CONFIG_PERF_EVENTS 1326 if (trace_probe_test_flag(&tk->tp, TP_FLAG_PROFILE)) 1327 ret = kprobe_perf_func(tk, regs); 1328 #endif 1329 return ret; 1330 } 1331 NOKPROBE_SYMBOL(kprobe_dispatcher); 1332 1333 static int 1334 kretprobe_dispatcher(struct kretprobe_instance *ri, struct pt_regs *regs) 1335 { 1336 struct trace_kprobe *tk = container_of(ri->rp, struct trace_kprobe, rp); 1337 1338 raw_cpu_inc(*tk->nhit); 1339 1340 if (trace_probe_test_flag(&tk->tp, TP_FLAG_TRACE)) 1341 kretprobe_trace_func(tk, ri, regs); 1342 #ifdef CONFIG_PERF_EVENTS 1343 if (trace_probe_test_flag(&tk->tp, TP_FLAG_PROFILE)) 1344 kretprobe_perf_func(tk, ri, regs); 1345 #endif 1346 return 0; /* We don't tweek kernel, so just return 0 */ 1347 } 1348 NOKPROBE_SYMBOL(kretprobe_dispatcher); 1349 1350 static struct trace_event_functions kretprobe_funcs = { 1351 .trace = print_kretprobe_event 1352 }; 1353 1354 static struct trace_event_functions kprobe_funcs = { 1355 .trace = print_kprobe_event 1356 }; 1357 1358 static inline void init_trace_event_call(struct trace_kprobe *tk) 1359 { 1360 struct trace_event_call *call = trace_probe_event_call(&tk->tp); 1361 1362 if (trace_kprobe_is_return(tk)) { 1363 call->event.funcs = &kretprobe_funcs; 1364 call->class->define_fields = kretprobe_event_define_fields; 1365 } else { 1366 call->event.funcs = &kprobe_funcs; 1367 call->class->define_fields = kprobe_event_define_fields; 1368 } 1369 1370 call->flags = TRACE_EVENT_FL_KPROBE; 1371 call->class->reg = kprobe_register; 1372 call->data = tk; 1373 } 1374 1375 static int register_kprobe_event(struct trace_kprobe *tk) 1376 { 1377 init_trace_event_call(tk); 1378 1379 return trace_probe_register_event_call(&tk->tp); 1380 } 1381 1382 static int unregister_kprobe_event(struct trace_kprobe *tk) 1383 { 1384 return trace_probe_unregister_event_call(&tk->tp); 1385 } 1386 1387 #ifdef CONFIG_PERF_EVENTS 1388 /* create a trace_kprobe, but don't add it to global lists */ 1389 struct trace_event_call * 1390 create_local_trace_kprobe(char *func, void *addr, unsigned long offs, 1391 bool is_return) 1392 { 1393 struct trace_kprobe *tk; 1394 int ret; 1395 char *event; 1396 1397 /* 1398 * local trace_kprobes are not added to dyn_event, so they are never 1399 * searched in find_trace_kprobe(). Therefore, there is no concern of 1400 * duplicated name here. 1401 */ 1402 event = func ? func : "DUMMY_EVENT"; 1403 1404 tk = alloc_trace_kprobe(KPROBE_EVENT_SYSTEM, event, (void *)addr, func, 1405 offs, 0 /* maxactive */, 0 /* nargs */, 1406 is_return); 1407 1408 if (IS_ERR(tk)) { 1409 pr_info("Failed to allocate trace_probe.(%d)\n", 1410 (int)PTR_ERR(tk)); 1411 return ERR_CAST(tk); 1412 } 1413 1414 init_trace_event_call(tk); 1415 1416 if (traceprobe_set_print_fmt(&tk->tp, trace_kprobe_is_return(tk)) < 0) { 1417 ret = -ENOMEM; 1418 goto error; 1419 } 1420 1421 ret = __register_trace_kprobe(tk); 1422 if (ret < 0) 1423 goto error; 1424 1425 return trace_probe_event_call(&tk->tp); 1426 error: 1427 free_trace_kprobe(tk); 1428 return ERR_PTR(ret); 1429 } 1430 1431 void destroy_local_trace_kprobe(struct trace_event_call *event_call) 1432 { 1433 struct trace_kprobe *tk; 1434 1435 tk = container_of(event_call, struct trace_kprobe, tp.call); 1436 1437 if (trace_probe_is_enabled(&tk->tp)) { 1438 WARN_ON(1); 1439 return; 1440 } 1441 1442 __unregister_trace_kprobe(tk); 1443 1444 free_trace_kprobe(tk); 1445 } 1446 #endif /* CONFIG_PERF_EVENTS */ 1447 1448 static __init void enable_boot_kprobe_events(void) 1449 { 1450 struct trace_array *tr = top_trace_array(); 1451 struct trace_event_file *file; 1452 struct trace_kprobe *tk; 1453 struct dyn_event *pos; 1454 1455 mutex_lock(&event_mutex); 1456 for_each_trace_kprobe(tk, pos) { 1457 list_for_each_entry(file, &tr->events, list) 1458 if (file->event_call == trace_probe_event_call(&tk->tp)) 1459 trace_event_enable_disable(file, 1, 0); 1460 } 1461 mutex_unlock(&event_mutex); 1462 } 1463 1464 static __init void setup_boot_kprobe_events(void) 1465 { 1466 char *p, *cmd = kprobe_boot_events_buf; 1467 int ret; 1468 1469 strreplace(kprobe_boot_events_buf, ',', ' '); 1470 1471 while (cmd && *cmd != '\0') { 1472 p = strchr(cmd, ';'); 1473 if (p) 1474 *p++ = '\0'; 1475 1476 ret = trace_run_command(cmd, create_or_delete_trace_kprobe); 1477 if (ret) 1478 pr_warn("Failed to add event(%d): %s\n", ret, cmd); 1479 else 1480 kprobe_boot_events_enabled = true; 1481 1482 cmd = p; 1483 } 1484 1485 enable_boot_kprobe_events(); 1486 } 1487 1488 /* Make a tracefs interface for controlling probe points */ 1489 static __init int init_kprobe_trace(void) 1490 { 1491 struct dentry *d_tracer; 1492 struct dentry *entry; 1493 int ret; 1494 1495 ret = dyn_event_register(&trace_kprobe_ops); 1496 if (ret) 1497 return ret; 1498 1499 if (register_module_notifier(&trace_kprobe_module_nb)) 1500 return -EINVAL; 1501 1502 d_tracer = tracing_init_dentry(); 1503 if (IS_ERR(d_tracer)) 1504 return 0; 1505 1506 entry = tracefs_create_file("kprobe_events", 0644, d_tracer, 1507 NULL, &kprobe_events_ops); 1508 1509 /* Event list interface */ 1510 if (!entry) 1511 pr_warn("Could not create tracefs 'kprobe_events' entry\n"); 1512 1513 /* Profile interface */ 1514 entry = tracefs_create_file("kprobe_profile", 0444, d_tracer, 1515 NULL, &kprobe_profile_ops); 1516 1517 if (!entry) 1518 pr_warn("Could not create tracefs 'kprobe_profile' entry\n"); 1519 1520 setup_boot_kprobe_events(); 1521 1522 return 0; 1523 } 1524 fs_initcall(init_kprobe_trace); 1525 1526 1527 #ifdef CONFIG_FTRACE_STARTUP_TEST 1528 static __init struct trace_event_file * 1529 find_trace_probe_file(struct trace_kprobe *tk, struct trace_array *tr) 1530 { 1531 struct trace_event_file *file; 1532 1533 list_for_each_entry(file, &tr->events, list) 1534 if (file->event_call == trace_probe_event_call(&tk->tp)) 1535 return file; 1536 1537 return NULL; 1538 } 1539 1540 /* 1541 * Nobody but us can call enable_trace_kprobe/disable_trace_kprobe at this 1542 * stage, we can do this lockless. 1543 */ 1544 static __init int kprobe_trace_self_tests_init(void) 1545 { 1546 int ret, warn = 0; 1547 int (*target)(int, int, int, int, int, int); 1548 struct trace_kprobe *tk; 1549 struct trace_event_file *file; 1550 1551 if (tracing_is_disabled()) 1552 return -ENODEV; 1553 1554 if (kprobe_boot_events_enabled) { 1555 pr_info("Skipping kprobe tests due to kprobe_event on cmdline\n"); 1556 return 0; 1557 } 1558 1559 target = kprobe_trace_selftest_target; 1560 1561 pr_info("Testing kprobe tracing: "); 1562 1563 ret = trace_run_command("p:testprobe kprobe_trace_selftest_target $stack $stack0 +0($stack)", 1564 create_or_delete_trace_kprobe); 1565 if (WARN_ON_ONCE(ret)) { 1566 pr_warn("error on probing function entry.\n"); 1567 warn++; 1568 } else { 1569 /* Enable trace point */ 1570 tk = find_trace_kprobe("testprobe", KPROBE_EVENT_SYSTEM); 1571 if (WARN_ON_ONCE(tk == NULL)) { 1572 pr_warn("error on getting new probe.\n"); 1573 warn++; 1574 } else { 1575 file = find_trace_probe_file(tk, top_trace_array()); 1576 if (WARN_ON_ONCE(file == NULL)) { 1577 pr_warn("error on getting probe file.\n"); 1578 warn++; 1579 } else 1580 enable_trace_kprobe(tk, file); 1581 } 1582 } 1583 1584 ret = trace_run_command("r:testprobe2 kprobe_trace_selftest_target $retval", 1585 create_or_delete_trace_kprobe); 1586 if (WARN_ON_ONCE(ret)) { 1587 pr_warn("error on probing function return.\n"); 1588 warn++; 1589 } else { 1590 /* Enable trace point */ 1591 tk = find_trace_kprobe("testprobe2", KPROBE_EVENT_SYSTEM); 1592 if (WARN_ON_ONCE(tk == NULL)) { 1593 pr_warn("error on getting 2nd new probe.\n"); 1594 warn++; 1595 } else { 1596 file = find_trace_probe_file(tk, top_trace_array()); 1597 if (WARN_ON_ONCE(file == NULL)) { 1598 pr_warn("error on getting probe file.\n"); 1599 warn++; 1600 } else 1601 enable_trace_kprobe(tk, file); 1602 } 1603 } 1604 1605 if (warn) 1606 goto end; 1607 1608 ret = target(1, 2, 3, 4, 5, 6); 1609 1610 /* 1611 * Not expecting an error here, the check is only to prevent the 1612 * optimizer from removing the call to target() as otherwise there 1613 * are no side-effects and the call is never performed. 1614 */ 1615 if (ret != 21) 1616 warn++; 1617 1618 /* Disable trace points before removing it */ 1619 tk = find_trace_kprobe("testprobe", KPROBE_EVENT_SYSTEM); 1620 if (WARN_ON_ONCE(tk == NULL)) { 1621 pr_warn("error on getting test probe.\n"); 1622 warn++; 1623 } else { 1624 if (trace_kprobe_nhit(tk) != 1) { 1625 pr_warn("incorrect number of testprobe hits\n"); 1626 warn++; 1627 } 1628 1629 file = find_trace_probe_file(tk, top_trace_array()); 1630 if (WARN_ON_ONCE(file == NULL)) { 1631 pr_warn("error on getting probe file.\n"); 1632 warn++; 1633 } else 1634 disable_trace_kprobe(tk, file); 1635 } 1636 1637 tk = find_trace_kprobe("testprobe2", KPROBE_EVENT_SYSTEM); 1638 if (WARN_ON_ONCE(tk == NULL)) { 1639 pr_warn("error on getting 2nd test probe.\n"); 1640 warn++; 1641 } else { 1642 if (trace_kprobe_nhit(tk) != 1) { 1643 pr_warn("incorrect number of testprobe2 hits\n"); 1644 warn++; 1645 } 1646 1647 file = find_trace_probe_file(tk, top_trace_array()); 1648 if (WARN_ON_ONCE(file == NULL)) { 1649 pr_warn("error on getting probe file.\n"); 1650 warn++; 1651 } else 1652 disable_trace_kprobe(tk, file); 1653 } 1654 1655 ret = trace_run_command("-:testprobe", create_or_delete_trace_kprobe); 1656 if (WARN_ON_ONCE(ret)) { 1657 pr_warn("error on deleting a probe.\n"); 1658 warn++; 1659 } 1660 1661 ret = trace_run_command("-:testprobe2", create_or_delete_trace_kprobe); 1662 if (WARN_ON_ONCE(ret)) { 1663 pr_warn("error on deleting a probe.\n"); 1664 warn++; 1665 } 1666 1667 end: 1668 ret = dyn_events_release_all(&trace_kprobe_ops); 1669 if (WARN_ON_ONCE(ret)) { 1670 pr_warn("error on cleaning up probes.\n"); 1671 warn++; 1672 } 1673 /* 1674 * Wait for the optimizer work to finish. Otherwise it might fiddle 1675 * with probes in already freed __init text. 1676 */ 1677 wait_for_kprobe_optimizer(); 1678 if (warn) 1679 pr_cont("NG: Some tests are failed. Please check them.\n"); 1680 else 1681 pr_cont("OK\n"); 1682 return 0; 1683 } 1684 1685 late_initcall(kprobe_trace_self_tests_init); 1686 1687 #endif 1688