1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Fprobe-based tracing events 4 * Copyright (C) 2022 Google LLC. 5 */ 6 #define pr_fmt(fmt) "trace_fprobe: " fmt 7 #include <asm/ptrace.h> 8 9 #include <linux/fprobe.h> 10 #include <linux/module.h> 11 #include <linux/rculist.h> 12 #include <linux/security.h> 13 #include <linux/tracepoint.h> 14 #include <linux/uaccess.h> 15 16 #include "trace_dynevent.h" 17 #include "trace_probe.h" 18 #include "trace_probe_kernel.h" 19 #include "trace_probe_tmpl.h" 20 21 #define FPROBE_EVENT_SYSTEM "fprobes" 22 #define TRACEPOINT_EVENT_SYSTEM "tracepoints" 23 #define RETHOOK_MAXACTIVE_MAX 4096 24 25 static int trace_fprobe_create(const char *raw_command); 26 static int trace_fprobe_show(struct seq_file *m, struct dyn_event *ev); 27 static int trace_fprobe_release(struct dyn_event *ev); 28 static bool trace_fprobe_is_busy(struct dyn_event *ev); 29 static bool trace_fprobe_match(const char *system, const char *event, 30 int argc, const char **argv, struct dyn_event *ev); 31 32 static struct dyn_event_operations trace_fprobe_ops = { 33 .create = trace_fprobe_create, 34 .show = trace_fprobe_show, 35 .is_busy = trace_fprobe_is_busy, 36 .free = trace_fprobe_release, 37 .match = trace_fprobe_match, 38 }; 39 40 /* 41 * Fprobe event core functions 42 */ 43 struct trace_fprobe { 44 struct dyn_event devent; 45 struct fprobe fp; 46 const char *symbol; 47 struct tracepoint *tpoint; 48 struct module *mod; 49 struct trace_probe tp; 50 }; 51 52 static bool is_trace_fprobe(struct dyn_event *ev) 53 { 54 return ev->ops == &trace_fprobe_ops; 55 } 56 57 static struct trace_fprobe *to_trace_fprobe(struct dyn_event *ev) 58 { 59 return container_of(ev, struct trace_fprobe, devent); 60 } 61 62 /** 63 * for_each_trace_fprobe - iterate over the trace_fprobe list 64 * @pos: the struct trace_fprobe * for each entry 65 * @dpos: the struct dyn_event * to use as a loop cursor 66 */ 67 #define for_each_trace_fprobe(pos, dpos) \ 68 for_each_dyn_event(dpos) \ 69 if (is_trace_fprobe(dpos) && (pos = to_trace_fprobe(dpos))) 70 71 static bool trace_fprobe_is_return(struct trace_fprobe *tf) 72 { 73 return tf->fp.exit_handler != NULL; 74 } 75 76 static bool trace_fprobe_is_tracepoint(struct trace_fprobe *tf) 77 { 78 return tf->tpoint != NULL; 79 } 80 81 static const char *trace_fprobe_symbol(struct trace_fprobe *tf) 82 { 83 return tf->symbol ? tf->symbol : "unknown"; 84 } 85 86 static bool trace_fprobe_is_busy(struct dyn_event *ev) 87 { 88 struct trace_fprobe *tf = to_trace_fprobe(ev); 89 90 return trace_probe_is_enabled(&tf->tp); 91 } 92 93 static bool trace_fprobe_match_command_head(struct trace_fprobe *tf, 94 int argc, const char **argv) 95 { 96 char buf[MAX_ARGSTR_LEN + 1]; 97 98 if (!argc) 99 return true; 100 101 snprintf(buf, sizeof(buf), "%s", trace_fprobe_symbol(tf)); 102 if (strcmp(buf, argv[0])) 103 return false; 104 argc--; argv++; 105 106 return trace_probe_match_command_args(&tf->tp, argc, argv); 107 } 108 109 static bool trace_fprobe_match(const char *system, const char *event, 110 int argc, const char **argv, struct dyn_event *ev) 111 { 112 struct trace_fprobe *tf = to_trace_fprobe(ev); 113 114 if (event[0] != '\0' && strcmp(trace_probe_name(&tf->tp), event)) 115 return false; 116 117 if (system && strcmp(trace_probe_group_name(&tf->tp), system)) 118 return false; 119 120 return trace_fprobe_match_command_head(tf, argc, argv); 121 } 122 123 static bool trace_fprobe_is_registered(struct trace_fprobe *tf) 124 { 125 return fprobe_is_registered(&tf->fp); 126 } 127 128 /* 129 * Note that we don't verify the fetch_insn code, since it does not come 130 * from user space. 131 */ 132 static int 133 process_fetch_insn(struct fetch_insn *code, void *rec, void *edata, 134 void *dest, void *base) 135 { 136 struct pt_regs *regs = rec; 137 unsigned long val; 138 int ret; 139 140 retry: 141 /* 1st stage: get value from context */ 142 switch (code->op) { 143 case FETCH_OP_STACK: 144 val = regs_get_kernel_stack_nth(regs, code->param); 145 break; 146 case FETCH_OP_STACKP: 147 val = kernel_stack_pointer(regs); 148 break; 149 case FETCH_OP_RETVAL: 150 val = regs_return_value(regs); 151 break; 152 #ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API 153 case FETCH_OP_ARG: 154 val = regs_get_kernel_argument(regs, code->param); 155 break; 156 case FETCH_OP_EDATA: 157 val = *(unsigned long *)((unsigned long)edata + code->offset); 158 break; 159 #endif 160 case FETCH_NOP_SYMBOL: /* Ignore a place holder */ 161 code++; 162 goto retry; 163 default: 164 ret = process_common_fetch_insn(code, &val); 165 if (ret < 0) 166 return ret; 167 } 168 code++; 169 170 return process_fetch_insn_bottom(code, val, dest, base); 171 } 172 NOKPROBE_SYMBOL(process_fetch_insn) 173 174 /* function entry handler */ 175 static nokprobe_inline void 176 __fentry_trace_func(struct trace_fprobe *tf, unsigned long entry_ip, 177 struct pt_regs *regs, 178 struct trace_event_file *trace_file) 179 { 180 struct fentry_trace_entry_head *entry; 181 struct trace_event_call *call = trace_probe_event_call(&tf->tp); 182 struct trace_event_buffer fbuffer; 183 int dsize; 184 185 if (WARN_ON_ONCE(call != trace_file->event_call)) 186 return; 187 188 if (trace_trigger_soft_disabled(trace_file)) 189 return; 190 191 dsize = __get_data_size(&tf->tp, regs, NULL); 192 193 entry = trace_event_buffer_reserve(&fbuffer, trace_file, 194 sizeof(*entry) + tf->tp.size + dsize); 195 if (!entry) 196 return; 197 198 fbuffer.regs = regs; 199 entry = fbuffer.entry = ring_buffer_event_data(fbuffer.event); 200 entry->ip = entry_ip; 201 store_trace_args(&entry[1], &tf->tp, regs, NULL, sizeof(*entry), dsize); 202 203 trace_event_buffer_commit(&fbuffer); 204 } 205 206 static void 207 fentry_trace_func(struct trace_fprobe *tf, unsigned long entry_ip, 208 struct pt_regs *regs) 209 { 210 struct event_file_link *link; 211 212 trace_probe_for_each_link_rcu(link, &tf->tp) 213 __fentry_trace_func(tf, entry_ip, regs, link->file); 214 } 215 NOKPROBE_SYMBOL(fentry_trace_func); 216 217 /* function exit handler */ 218 static int trace_fprobe_entry_handler(struct fprobe *fp, unsigned long entry_ip, 219 unsigned long ret_ip, struct pt_regs *regs, 220 void *entry_data) 221 { 222 struct trace_fprobe *tf = container_of(fp, struct trace_fprobe, fp); 223 224 if (tf->tp.entry_arg) 225 store_trace_entry_data(entry_data, &tf->tp, regs); 226 227 return 0; 228 } 229 NOKPROBE_SYMBOL(trace_fprobe_entry_handler) 230 231 static nokprobe_inline void 232 __fexit_trace_func(struct trace_fprobe *tf, unsigned long entry_ip, 233 unsigned long ret_ip, struct pt_regs *regs, 234 void *entry_data, struct trace_event_file *trace_file) 235 { 236 struct fexit_trace_entry_head *entry; 237 struct trace_event_buffer fbuffer; 238 struct trace_event_call *call = trace_probe_event_call(&tf->tp); 239 int dsize; 240 241 if (WARN_ON_ONCE(call != trace_file->event_call)) 242 return; 243 244 if (trace_trigger_soft_disabled(trace_file)) 245 return; 246 247 dsize = __get_data_size(&tf->tp, regs, entry_data); 248 249 entry = trace_event_buffer_reserve(&fbuffer, trace_file, 250 sizeof(*entry) + tf->tp.size + dsize); 251 if (!entry) 252 return; 253 254 fbuffer.regs = regs; 255 entry = fbuffer.entry = ring_buffer_event_data(fbuffer.event); 256 entry->func = entry_ip; 257 entry->ret_ip = ret_ip; 258 store_trace_args(&entry[1], &tf->tp, regs, entry_data, sizeof(*entry), dsize); 259 260 trace_event_buffer_commit(&fbuffer); 261 } 262 263 static void 264 fexit_trace_func(struct trace_fprobe *tf, unsigned long entry_ip, 265 unsigned long ret_ip, struct pt_regs *regs, void *entry_data) 266 { 267 struct event_file_link *link; 268 269 trace_probe_for_each_link_rcu(link, &tf->tp) 270 __fexit_trace_func(tf, entry_ip, ret_ip, regs, entry_data, link->file); 271 } 272 NOKPROBE_SYMBOL(fexit_trace_func); 273 274 #ifdef CONFIG_PERF_EVENTS 275 276 static int fentry_perf_func(struct trace_fprobe *tf, unsigned long entry_ip, 277 struct pt_regs *regs) 278 { 279 struct trace_event_call *call = trace_probe_event_call(&tf->tp); 280 struct fentry_trace_entry_head *entry; 281 struct hlist_head *head; 282 int size, __size, dsize; 283 int rctx; 284 285 head = this_cpu_ptr(call->perf_events); 286 if (hlist_empty(head)) 287 return 0; 288 289 dsize = __get_data_size(&tf->tp, regs, NULL); 290 __size = sizeof(*entry) + tf->tp.size + dsize; 291 size = ALIGN(__size + sizeof(u32), sizeof(u64)); 292 size -= sizeof(u32); 293 294 entry = perf_trace_buf_alloc(size, NULL, &rctx); 295 if (!entry) 296 return 0; 297 298 entry->ip = entry_ip; 299 memset(&entry[1], 0, dsize); 300 store_trace_args(&entry[1], &tf->tp, regs, NULL, sizeof(*entry), dsize); 301 perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs, 302 head, NULL); 303 return 0; 304 } 305 NOKPROBE_SYMBOL(fentry_perf_func); 306 307 static void 308 fexit_perf_func(struct trace_fprobe *tf, unsigned long entry_ip, 309 unsigned long ret_ip, struct pt_regs *regs, 310 void *entry_data) 311 { 312 struct trace_event_call *call = trace_probe_event_call(&tf->tp); 313 struct fexit_trace_entry_head *entry; 314 struct hlist_head *head; 315 int size, __size, dsize; 316 int rctx; 317 318 head = this_cpu_ptr(call->perf_events); 319 if (hlist_empty(head)) 320 return; 321 322 dsize = __get_data_size(&tf->tp, regs, entry_data); 323 __size = sizeof(*entry) + tf->tp.size + dsize; 324 size = ALIGN(__size + sizeof(u32), sizeof(u64)); 325 size -= sizeof(u32); 326 327 entry = perf_trace_buf_alloc(size, NULL, &rctx); 328 if (!entry) 329 return; 330 331 entry->func = entry_ip; 332 entry->ret_ip = ret_ip; 333 store_trace_args(&entry[1], &tf->tp, regs, entry_data, sizeof(*entry), dsize); 334 perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs, 335 head, NULL); 336 } 337 NOKPROBE_SYMBOL(fexit_perf_func); 338 #endif /* CONFIG_PERF_EVENTS */ 339 340 static int fentry_dispatcher(struct fprobe *fp, unsigned long entry_ip, 341 unsigned long ret_ip, struct pt_regs *regs, 342 void *entry_data) 343 { 344 struct trace_fprobe *tf = container_of(fp, struct trace_fprobe, fp); 345 int ret = 0; 346 347 if (trace_probe_test_flag(&tf->tp, TP_FLAG_TRACE)) 348 fentry_trace_func(tf, entry_ip, regs); 349 #ifdef CONFIG_PERF_EVENTS 350 if (trace_probe_test_flag(&tf->tp, TP_FLAG_PROFILE)) 351 ret = fentry_perf_func(tf, entry_ip, regs); 352 #endif 353 return ret; 354 } 355 NOKPROBE_SYMBOL(fentry_dispatcher); 356 357 static void fexit_dispatcher(struct fprobe *fp, unsigned long entry_ip, 358 unsigned long ret_ip, struct pt_regs *regs, 359 void *entry_data) 360 { 361 struct trace_fprobe *tf = container_of(fp, struct trace_fprobe, fp); 362 363 if (trace_probe_test_flag(&tf->tp, TP_FLAG_TRACE)) 364 fexit_trace_func(tf, entry_ip, ret_ip, regs, entry_data); 365 #ifdef CONFIG_PERF_EVENTS 366 if (trace_probe_test_flag(&tf->tp, TP_FLAG_PROFILE)) 367 fexit_perf_func(tf, entry_ip, ret_ip, regs, entry_data); 368 #endif 369 } 370 NOKPROBE_SYMBOL(fexit_dispatcher); 371 372 static void free_trace_fprobe(struct trace_fprobe *tf) 373 { 374 if (tf) { 375 trace_probe_cleanup(&tf->tp); 376 kfree(tf->symbol); 377 kfree(tf); 378 } 379 } 380 381 /* 382 * Allocate new trace_probe and initialize it (including fprobe). 383 */ 384 static struct trace_fprobe *alloc_trace_fprobe(const char *group, 385 const char *event, 386 const char *symbol, 387 struct tracepoint *tpoint, 388 int maxactive, 389 int nargs, bool is_return) 390 { 391 struct trace_fprobe *tf; 392 int ret = -ENOMEM; 393 394 tf = kzalloc(struct_size(tf, tp.args, nargs), GFP_KERNEL); 395 if (!tf) 396 return ERR_PTR(ret); 397 398 tf->symbol = kstrdup(symbol, GFP_KERNEL); 399 if (!tf->symbol) 400 goto error; 401 402 if (is_return) 403 tf->fp.exit_handler = fexit_dispatcher; 404 else 405 tf->fp.entry_handler = fentry_dispatcher; 406 407 tf->tpoint = tpoint; 408 tf->fp.nr_maxactive = maxactive; 409 410 ret = trace_probe_init(&tf->tp, event, group, false, nargs); 411 if (ret < 0) 412 goto error; 413 414 dyn_event_init(&tf->devent, &trace_fprobe_ops); 415 return tf; 416 error: 417 free_trace_fprobe(tf); 418 return ERR_PTR(ret); 419 } 420 421 static struct trace_fprobe *find_trace_fprobe(const char *event, 422 const char *group) 423 { 424 struct dyn_event *pos; 425 struct trace_fprobe *tf; 426 427 for_each_trace_fprobe(tf, pos) 428 if (strcmp(trace_probe_name(&tf->tp), event) == 0 && 429 strcmp(trace_probe_group_name(&tf->tp), group) == 0) 430 return tf; 431 return NULL; 432 } 433 434 static inline int __enable_trace_fprobe(struct trace_fprobe *tf) 435 { 436 if (trace_fprobe_is_registered(tf)) 437 enable_fprobe(&tf->fp); 438 439 return 0; 440 } 441 442 static void __disable_trace_fprobe(struct trace_probe *tp) 443 { 444 struct trace_fprobe *tf; 445 446 list_for_each_entry(tf, trace_probe_probe_list(tp), tp.list) { 447 if (!trace_fprobe_is_registered(tf)) 448 continue; 449 disable_fprobe(&tf->fp); 450 } 451 } 452 453 /* 454 * Enable trace_probe 455 * if the file is NULL, enable "perf" handler, or enable "trace" handler. 456 */ 457 static int enable_trace_fprobe(struct trace_event_call *call, 458 struct trace_event_file *file) 459 { 460 struct trace_probe *tp; 461 struct trace_fprobe *tf; 462 bool enabled; 463 int ret = 0; 464 465 tp = trace_probe_primary_from_call(call); 466 if (WARN_ON_ONCE(!tp)) 467 return -ENODEV; 468 enabled = trace_probe_is_enabled(tp); 469 470 /* This also changes "enabled" state */ 471 if (file) { 472 ret = trace_probe_add_file(tp, file); 473 if (ret) 474 return ret; 475 } else 476 trace_probe_set_flag(tp, TP_FLAG_PROFILE); 477 478 if (!enabled) { 479 list_for_each_entry(tf, trace_probe_probe_list(tp), tp.list) { 480 /* TODO: check the fprobe is gone */ 481 __enable_trace_fprobe(tf); 482 } 483 } 484 485 return 0; 486 } 487 488 /* 489 * Disable trace_probe 490 * if the file is NULL, disable "perf" handler, or disable "trace" handler. 491 */ 492 static int disable_trace_fprobe(struct trace_event_call *call, 493 struct trace_event_file *file) 494 { 495 struct trace_probe *tp; 496 497 tp = trace_probe_primary_from_call(call); 498 if (WARN_ON_ONCE(!tp)) 499 return -ENODEV; 500 501 if (file) { 502 if (!trace_probe_get_file_link(tp, file)) 503 return -ENOENT; 504 if (!trace_probe_has_single_file(tp)) 505 goto out; 506 trace_probe_clear_flag(tp, TP_FLAG_TRACE); 507 } else 508 trace_probe_clear_flag(tp, TP_FLAG_PROFILE); 509 510 if (!trace_probe_is_enabled(tp)) 511 __disable_trace_fprobe(tp); 512 513 out: 514 if (file) 515 /* 516 * Synchronization is done in below function. For perf event, 517 * file == NULL and perf_trace_event_unreg() calls 518 * tracepoint_synchronize_unregister() to ensure synchronize 519 * event. We don't need to care about it. 520 */ 521 trace_probe_remove_file(tp, file); 522 523 return 0; 524 } 525 526 /* Event entry printers */ 527 static enum print_line_t 528 print_fentry_event(struct trace_iterator *iter, int flags, 529 struct trace_event *event) 530 { 531 struct fentry_trace_entry_head *field; 532 struct trace_seq *s = &iter->seq; 533 struct trace_probe *tp; 534 535 field = (struct fentry_trace_entry_head *)iter->ent; 536 tp = trace_probe_primary_from_call( 537 container_of(event, struct trace_event_call, event)); 538 if (WARN_ON_ONCE(!tp)) 539 goto out; 540 541 trace_seq_printf(s, "%s: (", trace_probe_name(tp)); 542 543 if (!seq_print_ip_sym(s, field->ip, flags | TRACE_ITER_SYM_OFFSET)) 544 goto out; 545 546 trace_seq_putc(s, ')'); 547 548 if (trace_probe_print_args(s, tp->args, tp->nr_args, 549 (u8 *)&field[1], field) < 0) 550 goto out; 551 552 trace_seq_putc(s, '\n'); 553 out: 554 return trace_handle_return(s); 555 } 556 557 static enum print_line_t 558 print_fexit_event(struct trace_iterator *iter, int flags, 559 struct trace_event *event) 560 { 561 struct fexit_trace_entry_head *field; 562 struct trace_seq *s = &iter->seq; 563 struct trace_probe *tp; 564 565 field = (struct fexit_trace_entry_head *)iter->ent; 566 tp = trace_probe_primary_from_call( 567 container_of(event, struct trace_event_call, event)); 568 if (WARN_ON_ONCE(!tp)) 569 goto out; 570 571 trace_seq_printf(s, "%s: (", trace_probe_name(tp)); 572 573 if (!seq_print_ip_sym(s, field->ret_ip, flags | TRACE_ITER_SYM_OFFSET)) 574 goto out; 575 576 trace_seq_puts(s, " <- "); 577 578 if (!seq_print_ip_sym(s, field->func, flags & ~TRACE_ITER_SYM_OFFSET)) 579 goto out; 580 581 trace_seq_putc(s, ')'); 582 583 if (trace_probe_print_args(s, tp->args, tp->nr_args, 584 (u8 *)&field[1], field) < 0) 585 goto out; 586 587 trace_seq_putc(s, '\n'); 588 589 out: 590 return trace_handle_return(s); 591 } 592 593 static int fentry_event_define_fields(struct trace_event_call *event_call) 594 { 595 int ret; 596 struct fentry_trace_entry_head field; 597 struct trace_probe *tp; 598 599 tp = trace_probe_primary_from_call(event_call); 600 if (WARN_ON_ONCE(!tp)) 601 return -ENOENT; 602 603 DEFINE_FIELD(unsigned long, ip, FIELD_STRING_IP, 0); 604 605 return traceprobe_define_arg_fields(event_call, sizeof(field), tp); 606 } 607 608 static int fexit_event_define_fields(struct trace_event_call *event_call) 609 { 610 int ret; 611 struct fexit_trace_entry_head field; 612 struct trace_probe *tp; 613 614 tp = trace_probe_primary_from_call(event_call); 615 if (WARN_ON_ONCE(!tp)) 616 return -ENOENT; 617 618 DEFINE_FIELD(unsigned long, func, FIELD_STRING_FUNC, 0); 619 DEFINE_FIELD(unsigned long, ret_ip, FIELD_STRING_RETIP, 0); 620 621 return traceprobe_define_arg_fields(event_call, sizeof(field), tp); 622 } 623 624 static struct trace_event_functions fentry_funcs = { 625 .trace = print_fentry_event 626 }; 627 628 static struct trace_event_functions fexit_funcs = { 629 .trace = print_fexit_event 630 }; 631 632 static struct trace_event_fields fentry_fields_array[] = { 633 { .type = TRACE_FUNCTION_TYPE, 634 .define_fields = fentry_event_define_fields }, 635 {} 636 }; 637 638 static struct trace_event_fields fexit_fields_array[] = { 639 { .type = TRACE_FUNCTION_TYPE, 640 .define_fields = fexit_event_define_fields }, 641 {} 642 }; 643 644 static int fprobe_register(struct trace_event_call *event, 645 enum trace_reg type, void *data); 646 647 static inline void init_trace_event_call(struct trace_fprobe *tf) 648 { 649 struct trace_event_call *call = trace_probe_event_call(&tf->tp); 650 651 if (trace_fprobe_is_return(tf)) { 652 call->event.funcs = &fexit_funcs; 653 call->class->fields_array = fexit_fields_array; 654 } else { 655 call->event.funcs = &fentry_funcs; 656 call->class->fields_array = fentry_fields_array; 657 } 658 659 call->flags = TRACE_EVENT_FL_FPROBE; 660 call->class->reg = fprobe_register; 661 } 662 663 static int register_fprobe_event(struct trace_fprobe *tf) 664 { 665 init_trace_event_call(tf); 666 667 return trace_probe_register_event_call(&tf->tp); 668 } 669 670 static int unregister_fprobe_event(struct trace_fprobe *tf) 671 { 672 return trace_probe_unregister_event_call(&tf->tp); 673 } 674 675 /* Internal register function - just handle fprobe and flags */ 676 static int __register_trace_fprobe(struct trace_fprobe *tf) 677 { 678 int i, ret; 679 680 /* Should we need new LOCKDOWN flag for fprobe? */ 681 ret = security_locked_down(LOCKDOWN_KPROBES); 682 if (ret) 683 return ret; 684 685 if (trace_fprobe_is_registered(tf)) 686 return -EINVAL; 687 688 for (i = 0; i < tf->tp.nr_args; i++) { 689 ret = traceprobe_update_arg(&tf->tp.args[i]); 690 if (ret) 691 return ret; 692 } 693 694 /* Set/clear disabled flag according to tp->flag */ 695 if (trace_probe_is_enabled(&tf->tp)) 696 tf->fp.flags &= ~FPROBE_FL_DISABLED; 697 else 698 tf->fp.flags |= FPROBE_FL_DISABLED; 699 700 if (trace_fprobe_is_tracepoint(tf)) { 701 struct tracepoint *tpoint = tf->tpoint; 702 unsigned long ip = (unsigned long)tpoint->probestub; 703 /* 704 * Here, we do 2 steps to enable fprobe on a tracepoint. 705 * At first, put __probestub_##TP function on the tracepoint 706 * and put a fprobe on the stub function. 707 */ 708 ret = tracepoint_probe_register_prio_may_exist(tpoint, 709 tpoint->probestub, NULL, 0); 710 if (ret < 0) 711 return ret; 712 return register_fprobe_ips(&tf->fp, &ip, 1); 713 } 714 715 /* TODO: handle filter, nofilter or symbol list */ 716 return register_fprobe(&tf->fp, tf->symbol, NULL); 717 } 718 719 /* Internal unregister function - just handle fprobe and flags */ 720 static void __unregister_trace_fprobe(struct trace_fprobe *tf) 721 { 722 if (trace_fprobe_is_registered(tf)) { 723 unregister_fprobe(&tf->fp); 724 memset(&tf->fp, 0, sizeof(tf->fp)); 725 if (trace_fprobe_is_tracepoint(tf)) { 726 tracepoint_probe_unregister(tf->tpoint, 727 tf->tpoint->probestub, NULL); 728 tf->tpoint = NULL; 729 tf->mod = NULL; 730 } 731 } 732 } 733 734 /* TODO: make this trace_*probe common function */ 735 /* Unregister a trace_probe and probe_event */ 736 static int unregister_trace_fprobe(struct trace_fprobe *tf) 737 { 738 /* If other probes are on the event, just unregister fprobe */ 739 if (trace_probe_has_sibling(&tf->tp)) 740 goto unreg; 741 742 /* Enabled event can not be unregistered */ 743 if (trace_probe_is_enabled(&tf->tp)) 744 return -EBUSY; 745 746 /* If there's a reference to the dynamic event */ 747 if (trace_event_dyn_busy(trace_probe_event_call(&tf->tp))) 748 return -EBUSY; 749 750 /* Will fail if probe is being used by ftrace or perf */ 751 if (unregister_fprobe_event(tf)) 752 return -EBUSY; 753 754 unreg: 755 __unregister_trace_fprobe(tf); 756 dyn_event_remove(&tf->devent); 757 trace_probe_unlink(&tf->tp); 758 759 return 0; 760 } 761 762 static bool trace_fprobe_has_same_fprobe(struct trace_fprobe *orig, 763 struct trace_fprobe *comp) 764 { 765 struct trace_probe_event *tpe = orig->tp.event; 766 int i; 767 768 list_for_each_entry(orig, &tpe->probes, tp.list) { 769 if (strcmp(trace_fprobe_symbol(orig), 770 trace_fprobe_symbol(comp))) 771 continue; 772 773 /* 774 * trace_probe_compare_arg_type() ensured that nr_args and 775 * each argument name and type are same. Let's compare comm. 776 */ 777 for (i = 0; i < orig->tp.nr_args; i++) { 778 if (strcmp(orig->tp.args[i].comm, 779 comp->tp.args[i].comm)) 780 break; 781 } 782 783 if (i == orig->tp.nr_args) 784 return true; 785 } 786 787 return false; 788 } 789 790 static int append_trace_fprobe(struct trace_fprobe *tf, struct trace_fprobe *to) 791 { 792 int ret; 793 794 if (trace_fprobe_is_return(tf) != trace_fprobe_is_return(to) || 795 trace_fprobe_is_tracepoint(tf) != trace_fprobe_is_tracepoint(to)) { 796 trace_probe_log_set_index(0); 797 trace_probe_log_err(0, DIFF_PROBE_TYPE); 798 return -EEXIST; 799 } 800 ret = trace_probe_compare_arg_type(&tf->tp, &to->tp); 801 if (ret) { 802 /* Note that argument starts index = 2 */ 803 trace_probe_log_set_index(ret + 1); 804 trace_probe_log_err(0, DIFF_ARG_TYPE); 805 return -EEXIST; 806 } 807 if (trace_fprobe_has_same_fprobe(to, tf)) { 808 trace_probe_log_set_index(0); 809 trace_probe_log_err(0, SAME_PROBE); 810 return -EEXIST; 811 } 812 813 /* Append to existing event */ 814 ret = trace_probe_append(&tf->tp, &to->tp); 815 if (ret) 816 return ret; 817 818 ret = __register_trace_fprobe(tf); 819 if (ret) 820 trace_probe_unlink(&tf->tp); 821 else 822 dyn_event_add(&tf->devent, trace_probe_event_call(&tf->tp)); 823 824 return ret; 825 } 826 827 /* Register a trace_probe and probe_event */ 828 static int register_trace_fprobe(struct trace_fprobe *tf) 829 { 830 struct trace_fprobe *old_tf; 831 int ret; 832 833 mutex_lock(&event_mutex); 834 835 old_tf = find_trace_fprobe(trace_probe_name(&tf->tp), 836 trace_probe_group_name(&tf->tp)); 837 if (old_tf) { 838 ret = append_trace_fprobe(tf, old_tf); 839 goto end; 840 } 841 842 /* Register new event */ 843 ret = register_fprobe_event(tf); 844 if (ret) { 845 if (ret == -EEXIST) { 846 trace_probe_log_set_index(0); 847 trace_probe_log_err(0, EVENT_EXIST); 848 } else 849 pr_warn("Failed to register probe event(%d)\n", ret); 850 goto end; 851 } 852 853 /* Register fprobe */ 854 ret = __register_trace_fprobe(tf); 855 if (ret < 0) 856 unregister_fprobe_event(tf); 857 else 858 dyn_event_add(&tf->devent, trace_probe_event_call(&tf->tp)); 859 860 end: 861 mutex_unlock(&event_mutex); 862 return ret; 863 } 864 865 #ifdef CONFIG_MODULES 866 static int __tracepoint_probe_module_cb(struct notifier_block *self, 867 unsigned long val, void *data) 868 { 869 struct tp_module *tp_mod = data; 870 struct trace_fprobe *tf; 871 struct dyn_event *pos; 872 873 if (val != MODULE_STATE_GOING) 874 return NOTIFY_DONE; 875 876 mutex_lock(&event_mutex); 877 for_each_trace_fprobe(tf, pos) { 878 if (tp_mod->mod == tf->mod) { 879 tracepoint_probe_unregister(tf->tpoint, 880 tf->tpoint->probestub, NULL); 881 tf->tpoint = NULL; 882 tf->mod = NULL; 883 } 884 } 885 mutex_unlock(&event_mutex); 886 887 return NOTIFY_DONE; 888 } 889 890 static struct notifier_block tracepoint_module_nb = { 891 .notifier_call = __tracepoint_probe_module_cb, 892 }; 893 #endif /* CONFIG_MODULES */ 894 895 struct __find_tracepoint_cb_data { 896 const char *tp_name; 897 struct tracepoint *tpoint; 898 }; 899 900 static void __find_tracepoint_cb(struct tracepoint *tp, void *priv) 901 { 902 struct __find_tracepoint_cb_data *data = priv; 903 904 if (!data->tpoint && !strcmp(data->tp_name, tp->name)) 905 data->tpoint = tp; 906 } 907 908 static struct tracepoint *find_tracepoint(const char *tp_name) 909 { 910 struct __find_tracepoint_cb_data data = { 911 .tp_name = tp_name, 912 }; 913 914 for_each_kernel_tracepoint(__find_tracepoint_cb, &data); 915 916 return data.tpoint; 917 } 918 919 static int parse_symbol_and_return(int argc, const char *argv[], 920 char **symbol, bool *is_return, 921 bool is_tracepoint) 922 { 923 char *tmp = strchr(argv[1], '%'); 924 int i; 925 926 if (tmp) { 927 int len = tmp - argv[1]; 928 929 if (!is_tracepoint && !strcmp(tmp, "%return")) { 930 *is_return = true; 931 } else { 932 trace_probe_log_err(len, BAD_ADDR_SUFFIX); 933 return -EINVAL; 934 } 935 *symbol = kmemdup_nul(argv[1], len, GFP_KERNEL); 936 } else 937 *symbol = kstrdup(argv[1], GFP_KERNEL); 938 if (!*symbol) 939 return -ENOMEM; 940 941 if (*is_return) 942 return 0; 943 944 /* If there is $retval, this should be a return fprobe. */ 945 for (i = 2; i < argc; i++) { 946 tmp = strstr(argv[i], "$retval"); 947 if (tmp && !isalnum(tmp[7]) && tmp[7] != '_') { 948 if (is_tracepoint) { 949 trace_probe_log_set_index(i); 950 trace_probe_log_err(tmp - argv[i], RETVAL_ON_PROBE); 951 return -EINVAL; 952 } 953 *is_return = true; 954 break; 955 } 956 } 957 return 0; 958 } 959 960 static int __trace_fprobe_create(int argc, const char *argv[]) 961 { 962 /* 963 * Argument syntax: 964 * - Add fentry probe: 965 * f[:[GRP/][EVENT]] [MOD:]KSYM [FETCHARGS] 966 * - Add fexit probe: 967 * f[N][:[GRP/][EVENT]] [MOD:]KSYM%return [FETCHARGS] 968 * - Add tracepoint probe: 969 * t[:[GRP/][EVENT]] TRACEPOINT [FETCHARGS] 970 * 971 * Fetch args: 972 * $retval : fetch return value 973 * $stack : fetch stack address 974 * $stackN : fetch Nth entry of stack (N:0-) 975 * $argN : fetch Nth argument (N:1-) 976 * $comm : fetch current task comm 977 * @ADDR : fetch memory at ADDR (ADDR should be in kernel) 978 * @SYM[+|-offs] : fetch memory at SYM +|- offs (SYM is a data symbol) 979 * Dereferencing memory fetch: 980 * +|-offs(ARG) : fetch memory at ARG +|- offs address. 981 * Alias name of args: 982 * NAME=FETCHARG : set NAME as alias of FETCHARG. 983 * Type of args: 984 * FETCHARG:TYPE : use TYPE instead of unsigned long. 985 */ 986 struct trace_fprobe *tf = NULL; 987 int i, len, new_argc = 0, ret = 0; 988 bool is_return = false; 989 char *symbol = NULL; 990 const char *event = NULL, *group = FPROBE_EVENT_SYSTEM; 991 const char **new_argv = NULL; 992 int maxactive = 0; 993 char buf[MAX_EVENT_NAME_LEN]; 994 char gbuf[MAX_EVENT_NAME_LEN]; 995 char sbuf[KSYM_NAME_LEN]; 996 char abuf[MAX_BTF_ARGS_LEN]; 997 bool is_tracepoint = false; 998 struct tracepoint *tpoint = NULL; 999 struct traceprobe_parse_context ctx = { 1000 .flags = TPARG_FL_KERNEL | TPARG_FL_FPROBE, 1001 }; 1002 1003 if ((argv[0][0] != 'f' && argv[0][0] != 't') || argc < 2) 1004 return -ECANCELED; 1005 1006 if (argv[0][0] == 't') { 1007 is_tracepoint = true; 1008 group = TRACEPOINT_EVENT_SYSTEM; 1009 } 1010 1011 trace_probe_log_init("trace_fprobe", argc, argv); 1012 1013 event = strchr(&argv[0][1], ':'); 1014 if (event) 1015 event++; 1016 1017 if (isdigit(argv[0][1])) { 1018 if (event) 1019 len = event - &argv[0][1] - 1; 1020 else 1021 len = strlen(&argv[0][1]); 1022 if (len > MAX_EVENT_NAME_LEN - 1) { 1023 trace_probe_log_err(1, BAD_MAXACT); 1024 goto parse_error; 1025 } 1026 memcpy(buf, &argv[0][1], len); 1027 buf[len] = '\0'; 1028 ret = kstrtouint(buf, 0, &maxactive); 1029 if (ret || !maxactive) { 1030 trace_probe_log_err(1, BAD_MAXACT); 1031 goto parse_error; 1032 } 1033 /* fprobe rethook instances are iterated over via a list. The 1034 * maximum should stay reasonable. 1035 */ 1036 if (maxactive > RETHOOK_MAXACTIVE_MAX) { 1037 trace_probe_log_err(1, MAXACT_TOO_BIG); 1038 goto parse_error; 1039 } 1040 } 1041 1042 trace_probe_log_set_index(1); 1043 1044 /* a symbol(or tracepoint) must be specified */ 1045 ret = parse_symbol_and_return(argc, argv, &symbol, &is_return, is_tracepoint); 1046 if (ret < 0) 1047 goto parse_error; 1048 1049 if (!is_return && maxactive) { 1050 trace_probe_log_set_index(0); 1051 trace_probe_log_err(1, BAD_MAXACT_TYPE); 1052 goto parse_error; 1053 } 1054 1055 trace_probe_log_set_index(0); 1056 if (event) { 1057 ret = traceprobe_parse_event_name(&event, &group, gbuf, 1058 event - argv[0]); 1059 if (ret) 1060 goto parse_error; 1061 } 1062 1063 if (!event) { 1064 /* Make a new event name */ 1065 if (is_tracepoint) 1066 snprintf(buf, MAX_EVENT_NAME_LEN, "%s%s", 1067 isdigit(*symbol) ? "_" : "", symbol); 1068 else 1069 snprintf(buf, MAX_EVENT_NAME_LEN, "%s__%s", symbol, 1070 is_return ? "exit" : "entry"); 1071 sanitize_event_name(buf); 1072 event = buf; 1073 } 1074 1075 if (is_return) 1076 ctx.flags |= TPARG_FL_RETURN; 1077 else 1078 ctx.flags |= TPARG_FL_FENTRY; 1079 1080 if (is_tracepoint) { 1081 ctx.flags |= TPARG_FL_TPOINT; 1082 tpoint = find_tracepoint(symbol); 1083 if (!tpoint) { 1084 trace_probe_log_set_index(1); 1085 trace_probe_log_err(0, NO_TRACEPOINT); 1086 goto parse_error; 1087 } 1088 ctx.funcname = kallsyms_lookup( 1089 (unsigned long)tpoint->probestub, 1090 NULL, NULL, NULL, sbuf); 1091 } else 1092 ctx.funcname = symbol; 1093 1094 argc -= 2; argv += 2; 1095 new_argv = traceprobe_expand_meta_args(argc, argv, &new_argc, 1096 abuf, MAX_BTF_ARGS_LEN, &ctx); 1097 if (IS_ERR(new_argv)) { 1098 ret = PTR_ERR(new_argv); 1099 new_argv = NULL; 1100 goto out; 1101 } 1102 if (new_argv) { 1103 argc = new_argc; 1104 argv = new_argv; 1105 } 1106 if (argc > MAX_TRACE_ARGS) { 1107 ret = -E2BIG; 1108 goto out; 1109 } 1110 1111 /* setup a probe */ 1112 tf = alloc_trace_fprobe(group, event, symbol, tpoint, maxactive, 1113 argc, is_return); 1114 if (IS_ERR(tf)) { 1115 ret = PTR_ERR(tf); 1116 /* This must return -ENOMEM, else there is a bug */ 1117 WARN_ON_ONCE(ret != -ENOMEM); 1118 goto out; /* We know tf is not allocated */ 1119 } 1120 1121 if (is_tracepoint) 1122 tf->mod = __module_text_address( 1123 (unsigned long)tf->tpoint->probestub); 1124 1125 /* parse arguments */ 1126 for (i = 0; i < argc; i++) { 1127 trace_probe_log_set_index(i + 2); 1128 ctx.offset = 0; 1129 ret = traceprobe_parse_probe_arg(&tf->tp, i, argv[i], &ctx); 1130 if (ret) 1131 goto error; /* This can be -ENOMEM */ 1132 } 1133 1134 if (is_return && tf->tp.entry_arg) { 1135 tf->fp.entry_handler = trace_fprobe_entry_handler; 1136 tf->fp.entry_data_size = traceprobe_get_entry_data_size(&tf->tp); 1137 } 1138 1139 ret = traceprobe_set_print_fmt(&tf->tp, 1140 is_return ? PROBE_PRINT_RETURN : PROBE_PRINT_NORMAL); 1141 if (ret < 0) 1142 goto error; 1143 1144 ret = register_trace_fprobe(tf); 1145 if (ret) { 1146 trace_probe_log_set_index(1); 1147 if (ret == -EILSEQ) 1148 trace_probe_log_err(0, BAD_INSN_BNDRY); 1149 else if (ret == -ENOENT) 1150 trace_probe_log_err(0, BAD_PROBE_ADDR); 1151 else if (ret != -ENOMEM && ret != -EEXIST) 1152 trace_probe_log_err(0, FAIL_REG_PROBE); 1153 goto error; 1154 } 1155 1156 out: 1157 traceprobe_finish_parse(&ctx); 1158 trace_probe_log_clear(); 1159 kfree(new_argv); 1160 kfree(symbol); 1161 return ret; 1162 1163 parse_error: 1164 ret = -EINVAL; 1165 error: 1166 free_trace_fprobe(tf); 1167 goto out; 1168 } 1169 1170 static int trace_fprobe_create(const char *raw_command) 1171 { 1172 return trace_probe_create(raw_command, __trace_fprobe_create); 1173 } 1174 1175 static int trace_fprobe_release(struct dyn_event *ev) 1176 { 1177 struct trace_fprobe *tf = to_trace_fprobe(ev); 1178 int ret = unregister_trace_fprobe(tf); 1179 1180 if (!ret) 1181 free_trace_fprobe(tf); 1182 return ret; 1183 } 1184 1185 static int trace_fprobe_show(struct seq_file *m, struct dyn_event *ev) 1186 { 1187 struct trace_fprobe *tf = to_trace_fprobe(ev); 1188 int i; 1189 1190 if (trace_fprobe_is_tracepoint(tf)) 1191 seq_putc(m, 't'); 1192 else 1193 seq_putc(m, 'f'); 1194 if (trace_fprobe_is_return(tf) && tf->fp.nr_maxactive) 1195 seq_printf(m, "%d", tf->fp.nr_maxactive); 1196 seq_printf(m, ":%s/%s", trace_probe_group_name(&tf->tp), 1197 trace_probe_name(&tf->tp)); 1198 1199 seq_printf(m, " %s%s", trace_fprobe_symbol(tf), 1200 trace_fprobe_is_return(tf) ? "%return" : ""); 1201 1202 for (i = 0; i < tf->tp.nr_args; i++) 1203 seq_printf(m, " %s=%s", tf->tp.args[i].name, tf->tp.args[i].comm); 1204 seq_putc(m, '\n'); 1205 1206 return 0; 1207 } 1208 1209 /* 1210 * called by perf_trace_init() or __ftrace_set_clr_event() under event_mutex. 1211 */ 1212 static int fprobe_register(struct trace_event_call *event, 1213 enum trace_reg type, void *data) 1214 { 1215 struct trace_event_file *file = data; 1216 1217 switch (type) { 1218 case TRACE_REG_REGISTER: 1219 return enable_trace_fprobe(event, file); 1220 case TRACE_REG_UNREGISTER: 1221 return disable_trace_fprobe(event, file); 1222 1223 #ifdef CONFIG_PERF_EVENTS 1224 case TRACE_REG_PERF_REGISTER: 1225 return enable_trace_fprobe(event, NULL); 1226 case TRACE_REG_PERF_UNREGISTER: 1227 return disable_trace_fprobe(event, NULL); 1228 case TRACE_REG_PERF_OPEN: 1229 case TRACE_REG_PERF_CLOSE: 1230 case TRACE_REG_PERF_ADD: 1231 case TRACE_REG_PERF_DEL: 1232 return 0; 1233 #endif 1234 } 1235 return 0; 1236 } 1237 1238 /* 1239 * Register dynevent at core_initcall. This allows kernel to setup fprobe 1240 * events in postcore_initcall without tracefs. 1241 */ 1242 static __init int init_fprobe_trace_early(void) 1243 { 1244 int ret; 1245 1246 ret = dyn_event_register(&trace_fprobe_ops); 1247 if (ret) 1248 return ret; 1249 1250 #ifdef CONFIG_MODULES 1251 ret = register_tracepoint_module_notifier(&tracepoint_module_nb); 1252 if (ret) 1253 return ret; 1254 #endif 1255 1256 return 0; 1257 } 1258 core_initcall(init_fprobe_trace_early); 1259