1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2021, Microsoft Corporation. 4 * 5 * Authors: 6 * Beau Belgrave <beaub@linux.microsoft.com> 7 */ 8 9 #include <linux/bitmap.h> 10 #include <linux/cdev.h> 11 #include <linux/hashtable.h> 12 #include <linux/list.h> 13 #include <linux/io.h> 14 #include <linux/uio.h> 15 #include <linux/ioctl.h> 16 #include <linux/jhash.h> 17 #include <linux/trace_events.h> 18 #include <linux/tracefs.h> 19 #include <linux/types.h> 20 #include <linux/uaccess.h> 21 #include <uapi/linux/user_events.h> 22 #include "trace.h" 23 #include "trace_dynevent.h" 24 25 #define USER_EVENTS_PREFIX_LEN (sizeof(USER_EVENTS_PREFIX)-1) 26 27 #define FIELD_DEPTH_TYPE 0 28 #define FIELD_DEPTH_NAME 1 29 #define FIELD_DEPTH_SIZE 2 30 31 /* 32 * Limits how many trace_event calls user processes can create: 33 * Must be multiple of PAGE_SIZE. 34 */ 35 #define MAX_PAGES 1 36 #define MAX_EVENTS (MAX_PAGES * PAGE_SIZE) 37 38 /* Limit how long of an event name plus args within the subsystem. */ 39 #define MAX_EVENT_DESC 512 40 #define EVENT_NAME(user_event) ((user_event)->tracepoint.name) 41 #define MAX_FIELD_ARRAY_SIZE 1024 42 #define MAX_FIELD_ARG_NAME 256 43 44 static char *register_page_data; 45 46 static DEFINE_MUTEX(reg_mutex); 47 static DEFINE_HASHTABLE(register_table, 4); 48 static DECLARE_BITMAP(page_bitmap, MAX_EVENTS); 49 50 /* 51 * Stores per-event properties, as users register events 52 * within a file a user_event might be created if it does not 53 * already exist. These are globally used and their lifetime 54 * is tied to the refcnt member. These cannot go away until the 55 * refcnt reaches zero. 56 */ 57 struct user_event { 58 struct tracepoint tracepoint; 59 struct trace_event_call call; 60 struct trace_event_class class; 61 struct dyn_event devent; 62 struct hlist_node node; 63 struct list_head fields; 64 atomic_t refcnt; 65 int index; 66 int flags; 67 }; 68 69 /* 70 * Stores per-file events references, as users register events 71 * within a file this structure is modified and freed via RCU. 72 * The lifetime of this struct is tied to the lifetime of the file. 73 * These are not shared and only accessible by the file that created it. 74 */ 75 struct user_event_refs { 76 struct rcu_head rcu; 77 int count; 78 struct user_event *events[]; 79 }; 80 81 typedef void (*user_event_func_t) (struct user_event *user, 82 void *data, u32 datalen, 83 void *tpdata); 84 85 static int user_event_parse(char *name, char *args, char *flags, 86 struct user_event **newuser); 87 88 static u32 user_event_key(char *name) 89 { 90 return jhash(name, strlen(name), 0); 91 } 92 93 static struct list_head *user_event_get_fields(struct trace_event_call *call) 94 { 95 struct user_event *user = (struct user_event *)call->data; 96 97 return &user->fields; 98 } 99 100 /* 101 * Parses a register command for user_events 102 * Format: event_name[:FLAG1[,FLAG2...]] [field1[;field2...]] 103 * 104 * Example event named 'test' with a 20 char 'msg' field with an unsigned int 105 * 'id' field after: 106 * test char[20] msg;unsigned int id 107 * 108 * NOTE: Offsets are from the user data perspective, they are not from the 109 * trace_entry/buffer perspective. We automatically add the common properties 110 * sizes to the offset for the user. 111 */ 112 static int user_event_parse_cmd(char *raw_command, struct user_event **newuser) 113 { 114 char *name = raw_command; 115 char *args = strpbrk(name, " "); 116 char *flags; 117 118 if (args) 119 *args++ = '\0'; 120 121 flags = strpbrk(name, ":"); 122 123 if (flags) 124 *flags++ = '\0'; 125 126 return user_event_parse(name, args, flags, newuser); 127 } 128 129 static int user_field_array_size(const char *type) 130 { 131 const char *start = strchr(type, '['); 132 char val[8]; 133 char *bracket; 134 int size = 0; 135 136 if (start == NULL) 137 return -EINVAL; 138 139 if (strscpy(val, start + 1, sizeof(val)) <= 0) 140 return -EINVAL; 141 142 bracket = strchr(val, ']'); 143 144 if (!bracket) 145 return -EINVAL; 146 147 *bracket = '\0'; 148 149 if (kstrtouint(val, 0, &size)) 150 return -EINVAL; 151 152 if (size > MAX_FIELD_ARRAY_SIZE) 153 return -EINVAL; 154 155 return size; 156 } 157 158 static int user_field_size(const char *type) 159 { 160 /* long is not allowed from a user, since it's ambigious in size */ 161 if (strcmp(type, "s64") == 0) 162 return sizeof(s64); 163 if (strcmp(type, "u64") == 0) 164 return sizeof(u64); 165 if (strcmp(type, "s32") == 0) 166 return sizeof(s32); 167 if (strcmp(type, "u32") == 0) 168 return sizeof(u32); 169 if (strcmp(type, "int") == 0) 170 return sizeof(int); 171 if (strcmp(type, "unsigned int") == 0) 172 return sizeof(unsigned int); 173 if (strcmp(type, "s16") == 0) 174 return sizeof(s16); 175 if (strcmp(type, "u16") == 0) 176 return sizeof(u16); 177 if (strcmp(type, "short") == 0) 178 return sizeof(short); 179 if (strcmp(type, "unsigned short") == 0) 180 return sizeof(unsigned short); 181 if (strcmp(type, "s8") == 0) 182 return sizeof(s8); 183 if (strcmp(type, "u8") == 0) 184 return sizeof(u8); 185 if (strcmp(type, "char") == 0) 186 return sizeof(char); 187 if (strcmp(type, "unsigned char") == 0) 188 return sizeof(unsigned char); 189 if (str_has_prefix(type, "char[")) 190 return user_field_array_size(type); 191 if (str_has_prefix(type, "unsigned char[")) 192 return user_field_array_size(type); 193 if (str_has_prefix(type, "__data_loc ")) 194 return sizeof(u32); 195 if (str_has_prefix(type, "__rel_loc ")) 196 return sizeof(u32); 197 198 /* Uknown basic type, error */ 199 return -EINVAL; 200 } 201 202 static void user_event_destroy_fields(struct user_event *user) 203 { 204 struct ftrace_event_field *field, *next; 205 struct list_head *head = &user->fields; 206 207 list_for_each_entry_safe(field, next, head, link) { 208 list_del(&field->link); 209 kfree(field); 210 } 211 } 212 213 static int user_event_add_field(struct user_event *user, const char *type, 214 const char *name, int offset, int size, 215 int is_signed, int filter_type) 216 { 217 struct ftrace_event_field *field; 218 219 field = kmalloc(sizeof(*field), GFP_KERNEL); 220 221 if (!field) 222 return -ENOMEM; 223 224 field->type = type; 225 field->name = name; 226 field->offset = offset; 227 field->size = size; 228 field->is_signed = is_signed; 229 field->filter_type = filter_type; 230 231 list_add(&field->link, &user->fields); 232 233 return 0; 234 } 235 236 /* 237 * Parses the values of a field within the description 238 * Format: type name [size] 239 */ 240 static int user_event_parse_field(char *field, struct user_event *user, 241 u32 *offset) 242 { 243 char *part, *type, *name; 244 u32 depth = 0, saved_offset = *offset; 245 int len, size = -EINVAL; 246 bool is_struct = false; 247 248 field = skip_spaces(field); 249 250 if (*field == '\0') 251 return 0; 252 253 /* Handle types that have a space within */ 254 len = str_has_prefix(field, "unsigned "); 255 if (len) 256 goto skip_next; 257 258 len = str_has_prefix(field, "struct "); 259 if (len) { 260 is_struct = true; 261 goto skip_next; 262 } 263 264 len = str_has_prefix(field, "__data_loc unsigned "); 265 if (len) 266 goto skip_next; 267 268 len = str_has_prefix(field, "__data_loc "); 269 if (len) 270 goto skip_next; 271 272 len = str_has_prefix(field, "__rel_loc unsigned "); 273 if (len) 274 goto skip_next; 275 276 len = str_has_prefix(field, "__rel_loc "); 277 if (len) 278 goto skip_next; 279 280 goto parse; 281 skip_next: 282 type = field; 283 field = strpbrk(field + len, " "); 284 285 if (field == NULL) 286 return -EINVAL; 287 288 *field++ = '\0'; 289 depth++; 290 parse: 291 while ((part = strsep(&field, " ")) != NULL) { 292 switch (depth++) { 293 case FIELD_DEPTH_TYPE: 294 type = part; 295 break; 296 case FIELD_DEPTH_NAME: 297 name = part; 298 break; 299 case FIELD_DEPTH_SIZE: 300 if (!is_struct) 301 return -EINVAL; 302 303 if (kstrtou32(part, 10, &size)) 304 return -EINVAL; 305 break; 306 default: 307 return -EINVAL; 308 } 309 } 310 311 if (depth < FIELD_DEPTH_SIZE) 312 return -EINVAL; 313 314 if (depth == FIELD_DEPTH_SIZE) 315 size = user_field_size(type); 316 317 if (size == 0) 318 return -EINVAL; 319 320 if (size < 0) 321 return size; 322 323 *offset = saved_offset + size; 324 325 return user_event_add_field(user, type, name, saved_offset, size, 326 type[0] != 'u', FILTER_OTHER); 327 } 328 329 static void user_event_parse_flags(struct user_event *user, char *flags) 330 { 331 char *flag; 332 333 if (flags == NULL) 334 return; 335 336 while ((flag = strsep(&flags, ",")) != NULL) { 337 if (strcmp(flag, "BPF_ITER") == 0) 338 user->flags |= FLAG_BPF_ITER; 339 } 340 } 341 342 static int user_event_parse_fields(struct user_event *user, char *args) 343 { 344 char *field; 345 u32 offset = sizeof(struct trace_entry); 346 int ret = -EINVAL; 347 348 if (args == NULL) 349 return 0; 350 351 while ((field = strsep(&args, ";")) != NULL) { 352 ret = user_event_parse_field(field, user, &offset); 353 354 if (ret) 355 break; 356 } 357 358 return ret; 359 } 360 361 static struct trace_event_fields user_event_fields_array[1]; 362 363 static const char *user_field_format(const char *type) 364 { 365 if (strcmp(type, "s64") == 0) 366 return "%lld"; 367 if (strcmp(type, "u64") == 0) 368 return "%llu"; 369 if (strcmp(type, "s32") == 0) 370 return "%d"; 371 if (strcmp(type, "u32") == 0) 372 return "%u"; 373 if (strcmp(type, "int") == 0) 374 return "%d"; 375 if (strcmp(type, "unsigned int") == 0) 376 return "%u"; 377 if (strcmp(type, "s16") == 0) 378 return "%d"; 379 if (strcmp(type, "u16") == 0) 380 return "%u"; 381 if (strcmp(type, "short") == 0) 382 return "%d"; 383 if (strcmp(type, "unsigned short") == 0) 384 return "%u"; 385 if (strcmp(type, "s8") == 0) 386 return "%d"; 387 if (strcmp(type, "u8") == 0) 388 return "%u"; 389 if (strcmp(type, "char") == 0) 390 return "%d"; 391 if (strcmp(type, "unsigned char") == 0) 392 return "%u"; 393 if (strstr(type, "char[") != 0) 394 return "%s"; 395 396 /* Unknown, likely struct, allowed treat as 64-bit */ 397 return "%llu"; 398 } 399 400 static bool user_field_is_dyn_string(const char *type, const char **str_func) 401 { 402 if (str_has_prefix(type, "__data_loc ")) { 403 *str_func = "__get_str"; 404 goto check; 405 } 406 407 if (str_has_prefix(type, "__rel_loc ")) { 408 *str_func = "__get_rel_str"; 409 goto check; 410 } 411 412 return false; 413 check: 414 return strstr(type, "char") != 0; 415 } 416 417 #define LEN_OR_ZERO (len ? len - pos : 0) 418 static int user_event_set_print_fmt(struct user_event *user, char *buf, int len) 419 { 420 struct ftrace_event_field *field, *next; 421 struct list_head *head = &user->fields; 422 int pos = 0, depth = 0; 423 const char *str_func; 424 425 pos += snprintf(buf + pos, LEN_OR_ZERO, "\""); 426 427 list_for_each_entry_safe_reverse(field, next, head, link) { 428 if (depth != 0) 429 pos += snprintf(buf + pos, LEN_OR_ZERO, " "); 430 431 pos += snprintf(buf + pos, LEN_OR_ZERO, "%s=%s", 432 field->name, user_field_format(field->type)); 433 434 depth++; 435 } 436 437 pos += snprintf(buf + pos, LEN_OR_ZERO, "\""); 438 439 list_for_each_entry_safe_reverse(field, next, head, link) { 440 if (user_field_is_dyn_string(field->type, &str_func)) 441 pos += snprintf(buf + pos, LEN_OR_ZERO, 442 ", %s(%s)", str_func, field->name); 443 else 444 pos += snprintf(buf + pos, LEN_OR_ZERO, 445 ", REC->%s", field->name); 446 } 447 448 return pos + 1; 449 } 450 #undef LEN_OR_ZERO 451 452 static int user_event_create_print_fmt(struct user_event *user) 453 { 454 char *print_fmt; 455 int len; 456 457 len = user_event_set_print_fmt(user, NULL, 0); 458 459 print_fmt = kmalloc(len, GFP_KERNEL); 460 461 if (!print_fmt) 462 return -ENOMEM; 463 464 user_event_set_print_fmt(user, print_fmt, len); 465 466 user->call.print_fmt = print_fmt; 467 468 return 0; 469 } 470 471 static enum print_line_t user_event_print_trace(struct trace_iterator *iter, 472 int flags, 473 struct trace_event *event) 474 { 475 /* Unsafe to try to decode user provided print_fmt, use hex */ 476 trace_print_hex_dump_seq(&iter->seq, "", DUMP_PREFIX_OFFSET, 16, 477 1, iter->ent, iter->ent_size, true); 478 479 return trace_handle_return(&iter->seq); 480 } 481 482 static struct trace_event_functions user_event_funcs = { 483 .trace = user_event_print_trace, 484 }; 485 486 static int destroy_user_event(struct user_event *user) 487 { 488 int ret = 0; 489 490 /* Must destroy fields before call removal */ 491 user_event_destroy_fields(user); 492 493 ret = trace_remove_event_call(&user->call); 494 495 if (ret) 496 return ret; 497 498 dyn_event_remove(&user->devent); 499 500 register_page_data[user->index] = 0; 501 clear_bit(user->index, page_bitmap); 502 hash_del(&user->node); 503 504 kfree(user->call.print_fmt); 505 kfree(EVENT_NAME(user)); 506 kfree(user); 507 508 return ret; 509 } 510 511 static struct user_event *find_user_event(char *name, u32 *outkey) 512 { 513 struct user_event *user; 514 u32 key = user_event_key(name); 515 516 *outkey = key; 517 518 hash_for_each_possible(register_table, user, node, key) 519 if (!strcmp(EVENT_NAME(user), name)) 520 return user; 521 522 return NULL; 523 } 524 525 /* 526 * Writes the user supplied payload out to a trace file. 527 */ 528 static void user_event_ftrace(struct user_event *user, void *data, u32 datalen, 529 void *tpdata) 530 { 531 struct trace_event_file *file; 532 struct trace_entry *entry; 533 struct trace_event_buffer event_buffer; 534 535 file = (struct trace_event_file *)tpdata; 536 537 if (!file || 538 !(file->flags & EVENT_FILE_FL_ENABLED) || 539 trace_trigger_soft_disabled(file)) 540 return; 541 542 /* Allocates and fills trace_entry, + 1 of this is data payload */ 543 entry = trace_event_buffer_reserve(&event_buffer, file, 544 sizeof(*entry) + datalen); 545 546 if (unlikely(!entry)) 547 return; 548 549 memcpy(entry + 1, data, datalen); 550 551 trace_event_buffer_commit(&event_buffer); 552 } 553 554 #ifdef CONFIG_PERF_EVENTS 555 /* 556 * Writes the user supplied payload out to perf ring buffer or eBPF program. 557 */ 558 static void user_event_perf(struct user_event *user, void *data, u32 datalen, 559 void *tpdata) 560 { 561 struct hlist_head *perf_head; 562 563 if (bpf_prog_array_valid(&user->call)) { 564 struct user_bpf_context context = {0}; 565 566 context.data_len = datalen; 567 context.data_type = USER_BPF_DATA_KERNEL; 568 context.kdata = data; 569 570 trace_call_bpf(&user->call, &context); 571 } 572 573 perf_head = this_cpu_ptr(user->call.perf_events); 574 575 if (perf_head && !hlist_empty(perf_head)) { 576 struct trace_entry *perf_entry; 577 struct pt_regs *regs; 578 size_t size = sizeof(*perf_entry) + datalen; 579 int context; 580 581 perf_entry = perf_trace_buf_alloc(ALIGN(size, 8), 582 ®s, &context); 583 584 if (unlikely(!perf_entry)) 585 return; 586 587 perf_fetch_caller_regs(regs); 588 589 memcpy(perf_entry + 1, data, datalen); 590 591 perf_trace_buf_submit(perf_entry, size, context, 592 user->call.event.type, 1, regs, 593 perf_head, NULL); 594 } 595 } 596 #endif 597 598 /* 599 * Update the register page that is shared between user processes. 600 */ 601 static void update_reg_page_for(struct user_event *user) 602 { 603 struct tracepoint *tp = &user->tracepoint; 604 char status = 0; 605 606 if (atomic_read(&tp->key.enabled) > 0) { 607 struct tracepoint_func *probe_func_ptr; 608 user_event_func_t probe_func; 609 610 rcu_read_lock_sched(); 611 612 probe_func_ptr = rcu_dereference_sched(tp->funcs); 613 614 if (probe_func_ptr) { 615 do { 616 probe_func = probe_func_ptr->func; 617 618 if (probe_func == user_event_ftrace) 619 status |= EVENT_STATUS_FTRACE; 620 #ifdef CONFIG_PERF_EVENTS 621 else if (probe_func == user_event_perf) 622 status |= EVENT_STATUS_PERF; 623 #endif 624 else 625 status |= EVENT_STATUS_OTHER; 626 } while ((++probe_func_ptr)->func); 627 } 628 629 rcu_read_unlock_sched(); 630 } 631 632 register_page_data[user->index] = status; 633 } 634 635 /* 636 * Register callback for our events from tracing sub-systems. 637 */ 638 static int user_event_reg(struct trace_event_call *call, 639 enum trace_reg type, 640 void *data) 641 { 642 struct user_event *user = (struct user_event *)call->data; 643 int ret = 0; 644 645 if (!user) 646 return -ENOENT; 647 648 switch (type) { 649 case TRACE_REG_REGISTER: 650 ret = tracepoint_probe_register(call->tp, 651 call->class->probe, 652 data); 653 if (!ret) 654 goto inc; 655 break; 656 657 case TRACE_REG_UNREGISTER: 658 tracepoint_probe_unregister(call->tp, 659 call->class->probe, 660 data); 661 goto dec; 662 663 #ifdef CONFIG_PERF_EVENTS 664 case TRACE_REG_PERF_REGISTER: 665 ret = tracepoint_probe_register(call->tp, 666 call->class->perf_probe, 667 data); 668 if (!ret) 669 goto inc; 670 break; 671 672 case TRACE_REG_PERF_UNREGISTER: 673 tracepoint_probe_unregister(call->tp, 674 call->class->perf_probe, 675 data); 676 goto dec; 677 678 case TRACE_REG_PERF_OPEN: 679 case TRACE_REG_PERF_CLOSE: 680 case TRACE_REG_PERF_ADD: 681 case TRACE_REG_PERF_DEL: 682 break; 683 #endif 684 } 685 686 return ret; 687 inc: 688 atomic_inc(&user->refcnt); 689 update_reg_page_for(user); 690 return 0; 691 dec: 692 update_reg_page_for(user); 693 atomic_dec(&user->refcnt); 694 return 0; 695 } 696 697 static int user_event_create(const char *raw_command) 698 { 699 struct user_event *user; 700 char *name; 701 int ret; 702 703 if (!str_has_prefix(raw_command, USER_EVENTS_PREFIX)) 704 return -ECANCELED; 705 706 raw_command += USER_EVENTS_PREFIX_LEN; 707 raw_command = skip_spaces(raw_command); 708 709 name = kstrdup(raw_command, GFP_KERNEL); 710 711 if (!name) 712 return -ENOMEM; 713 714 mutex_lock(®_mutex); 715 ret = user_event_parse_cmd(name, &user); 716 mutex_unlock(®_mutex); 717 718 if (ret) 719 kfree(name); 720 721 return ret; 722 } 723 724 static int user_event_show(struct seq_file *m, struct dyn_event *ev) 725 { 726 struct user_event *user = container_of(ev, struct user_event, devent); 727 struct ftrace_event_field *field, *next; 728 struct list_head *head; 729 int depth = 0; 730 731 seq_printf(m, "%s%s", USER_EVENTS_PREFIX, EVENT_NAME(user)); 732 733 head = trace_get_fields(&user->call); 734 735 list_for_each_entry_safe_reverse(field, next, head, link) { 736 if (depth == 0) 737 seq_puts(m, " "); 738 else 739 seq_puts(m, "; "); 740 741 seq_printf(m, "%s %s", field->type, field->name); 742 743 if (str_has_prefix(field->type, "struct ")) 744 seq_printf(m, " %d", field->size); 745 746 depth++; 747 } 748 749 seq_puts(m, "\n"); 750 751 return 0; 752 } 753 754 static bool user_event_is_busy(struct dyn_event *ev) 755 { 756 struct user_event *user = container_of(ev, struct user_event, devent); 757 758 return atomic_read(&user->refcnt) != 0; 759 } 760 761 static int user_event_free(struct dyn_event *ev) 762 { 763 struct user_event *user = container_of(ev, struct user_event, devent); 764 765 if (atomic_read(&user->refcnt) != 0) 766 return -EBUSY; 767 768 return destroy_user_event(user); 769 } 770 771 static bool user_field_match(struct ftrace_event_field *field, int argc, 772 const char **argv, int *iout) 773 { 774 char *field_name, *arg_name; 775 int len, pos, i = *iout; 776 bool colon = false, match = false; 777 778 if (i >= argc) 779 return false; 780 781 len = MAX_FIELD_ARG_NAME; 782 field_name = kmalloc(len, GFP_KERNEL); 783 arg_name = kmalloc(len, GFP_KERNEL); 784 785 if (!arg_name || !field_name) 786 goto out; 787 788 pos = 0; 789 790 for (; i < argc; ++i) { 791 if (i != *iout) 792 pos += snprintf(arg_name + pos, len - pos, " "); 793 794 pos += snprintf(arg_name + pos, len - pos, argv[i]); 795 796 if (strchr(argv[i], ';')) { 797 ++i; 798 colon = true; 799 break; 800 } 801 } 802 803 pos = 0; 804 805 pos += snprintf(field_name + pos, len - pos, field->type); 806 pos += snprintf(field_name + pos, len - pos, " "); 807 pos += snprintf(field_name + pos, len - pos, field->name); 808 809 if (colon) 810 pos += snprintf(field_name + pos, len - pos, ";"); 811 812 *iout = i; 813 814 match = strcmp(arg_name, field_name) == 0; 815 out: 816 kfree(arg_name); 817 kfree(field_name); 818 819 return match; 820 } 821 822 static bool user_fields_match(struct user_event *user, int argc, 823 const char **argv) 824 { 825 struct ftrace_event_field *field, *next; 826 struct list_head *head = &user->fields; 827 int i = 0; 828 829 list_for_each_entry_safe_reverse(field, next, head, link) 830 if (!user_field_match(field, argc, argv, &i)) 831 return false; 832 833 if (i != argc) 834 return false; 835 836 return true; 837 } 838 839 static bool user_event_match(const char *system, const char *event, 840 int argc, const char **argv, struct dyn_event *ev) 841 { 842 struct user_event *user = container_of(ev, struct user_event, devent); 843 bool match; 844 845 match = strcmp(EVENT_NAME(user), event) == 0 && 846 (!system || strcmp(system, USER_EVENTS_SYSTEM) == 0); 847 848 if (match && argc > 0) 849 match = user_fields_match(user, argc, argv); 850 851 return match; 852 } 853 854 static struct dyn_event_operations user_event_dops = { 855 .create = user_event_create, 856 .show = user_event_show, 857 .is_busy = user_event_is_busy, 858 .free = user_event_free, 859 .match = user_event_match, 860 }; 861 862 static int user_event_trace_register(struct user_event *user) 863 { 864 int ret; 865 866 ret = register_trace_event(&user->call.event); 867 868 if (!ret) 869 return -ENODEV; 870 871 ret = trace_add_event_call(&user->call); 872 873 if (ret) 874 unregister_trace_event(&user->call.event); 875 876 return ret; 877 } 878 879 /* 880 * Parses the event name, arguments and flags then registers if successful. 881 * The name buffer lifetime is owned by this method for success cases only. 882 */ 883 static int user_event_parse(char *name, char *args, char *flags, 884 struct user_event **newuser) 885 { 886 int ret; 887 int index; 888 u32 key; 889 struct user_event *user = find_user_event(name, &key); 890 891 if (user) { 892 *newuser = user; 893 /* 894 * Name is allocated by caller, free it since it already exists. 895 * Caller only worries about failure cases for freeing. 896 */ 897 kfree(name); 898 return 0; 899 } 900 901 index = find_first_zero_bit(page_bitmap, MAX_EVENTS); 902 903 if (index == MAX_EVENTS) 904 return -EMFILE; 905 906 user = kzalloc(sizeof(*user), GFP_KERNEL); 907 908 if (!user) 909 return -ENOMEM; 910 911 INIT_LIST_HEAD(&user->class.fields); 912 INIT_LIST_HEAD(&user->fields); 913 914 user->tracepoint.name = name; 915 916 user_event_parse_flags(user, flags); 917 918 ret = user_event_parse_fields(user, args); 919 920 if (ret) 921 goto put_user; 922 923 ret = user_event_create_print_fmt(user); 924 925 if (ret) 926 goto put_user; 927 928 user->call.data = user; 929 user->call.class = &user->class; 930 user->call.name = name; 931 user->call.flags = TRACE_EVENT_FL_TRACEPOINT; 932 user->call.tp = &user->tracepoint; 933 user->call.event.funcs = &user_event_funcs; 934 935 user->class.system = USER_EVENTS_SYSTEM; 936 user->class.fields_array = user_event_fields_array; 937 user->class.get_fields = user_event_get_fields; 938 user->class.reg = user_event_reg; 939 user->class.probe = user_event_ftrace; 940 #ifdef CONFIG_PERF_EVENTS 941 user->class.perf_probe = user_event_perf; 942 #endif 943 944 mutex_lock(&event_mutex); 945 ret = user_event_trace_register(user); 946 mutex_unlock(&event_mutex); 947 948 if (ret) 949 goto put_user; 950 951 user->index = index; 952 dyn_event_init(&user->devent, &user_event_dops); 953 dyn_event_add(&user->devent, &user->call); 954 set_bit(user->index, page_bitmap); 955 hash_add(register_table, &user->node, key); 956 957 *newuser = user; 958 return 0; 959 put_user: 960 user_event_destroy_fields(user); 961 kfree(user); 962 return ret; 963 } 964 965 /* 966 * Deletes a previously created event if it is no longer being used. 967 */ 968 static int delete_user_event(char *name) 969 { 970 u32 key; 971 int ret; 972 struct user_event *user = find_user_event(name, &key); 973 974 if (!user) 975 return -ENOENT; 976 977 if (atomic_read(&user->refcnt) != 0) 978 return -EBUSY; 979 980 mutex_lock(&event_mutex); 981 ret = destroy_user_event(user); 982 mutex_unlock(&event_mutex); 983 984 return ret; 985 } 986 987 /* 988 * Validates the user payload and writes via iterator. 989 */ 990 static ssize_t user_events_write_core(struct file *file, struct iov_iter *i) 991 { 992 struct user_event_refs *refs; 993 struct user_event *user = NULL; 994 struct tracepoint *tp; 995 ssize_t ret = i->count; 996 int idx; 997 998 if (unlikely(copy_from_iter(&idx, sizeof(idx), i) != sizeof(idx))) 999 return -EFAULT; 1000 1001 rcu_read_lock_sched(); 1002 1003 refs = rcu_dereference_sched(file->private_data); 1004 1005 /* 1006 * The refs->events array is protected by RCU, and new items may be 1007 * added. But the user retrieved from indexing into the events array 1008 * shall be immutable while the file is opened. 1009 */ 1010 if (likely(refs && idx < refs->count)) 1011 user = refs->events[idx]; 1012 1013 rcu_read_unlock_sched(); 1014 1015 if (unlikely(user == NULL)) 1016 return -ENOENT; 1017 1018 tp = &user->tracepoint; 1019 1020 /* 1021 * It's possible key.enabled disables after this check, however 1022 * we don't mind if a few events are included in this condition. 1023 */ 1024 if (likely(atomic_read(&tp->key.enabled) > 0)) { 1025 struct tracepoint_func *probe_func_ptr; 1026 user_event_func_t probe_func; 1027 void *tpdata; 1028 void *kdata; 1029 u32 datalen; 1030 1031 kdata = kmalloc(i->count, GFP_KERNEL); 1032 1033 if (unlikely(!kdata)) 1034 return -ENOMEM; 1035 1036 datalen = copy_from_iter(kdata, i->count, i); 1037 1038 rcu_read_lock_sched(); 1039 1040 probe_func_ptr = rcu_dereference_sched(tp->funcs); 1041 1042 if (probe_func_ptr) { 1043 do { 1044 probe_func = probe_func_ptr->func; 1045 tpdata = probe_func_ptr->data; 1046 probe_func(user, kdata, datalen, tpdata); 1047 } while ((++probe_func_ptr)->func); 1048 } 1049 1050 rcu_read_unlock_sched(); 1051 1052 kfree(kdata); 1053 } 1054 1055 return ret; 1056 } 1057 1058 static ssize_t user_events_write(struct file *file, const char __user *ubuf, 1059 size_t count, loff_t *ppos) 1060 { 1061 struct iovec iov; 1062 struct iov_iter i; 1063 1064 if (unlikely(*ppos != 0)) 1065 return -EFAULT; 1066 1067 if (unlikely(import_single_range(READ, (char *)ubuf, count, &iov, &i))) 1068 return -EFAULT; 1069 1070 return user_events_write_core(file, &i); 1071 } 1072 1073 static ssize_t user_events_write_iter(struct kiocb *kp, struct iov_iter *i) 1074 { 1075 return user_events_write_core(kp->ki_filp, i); 1076 } 1077 1078 static int user_events_ref_add(struct file *file, struct user_event *user) 1079 { 1080 struct user_event_refs *refs, *new_refs; 1081 int i, size, count = 0; 1082 1083 refs = rcu_dereference_protected(file->private_data, 1084 lockdep_is_held(®_mutex)); 1085 1086 if (refs) { 1087 count = refs->count; 1088 1089 for (i = 0; i < count; ++i) 1090 if (refs->events[i] == user) 1091 return i; 1092 } 1093 1094 size = struct_size(refs, events, count + 1); 1095 1096 new_refs = kzalloc(size, GFP_KERNEL); 1097 1098 if (!new_refs) 1099 return -ENOMEM; 1100 1101 new_refs->count = count + 1; 1102 1103 for (i = 0; i < count; ++i) 1104 new_refs->events[i] = refs->events[i]; 1105 1106 new_refs->events[i] = user; 1107 1108 atomic_inc(&user->refcnt); 1109 1110 rcu_assign_pointer(file->private_data, new_refs); 1111 1112 if (refs) 1113 kfree_rcu(refs, rcu); 1114 1115 return i; 1116 } 1117 1118 static long user_reg_get(struct user_reg __user *ureg, struct user_reg *kreg) 1119 { 1120 u32 size; 1121 long ret; 1122 1123 ret = get_user(size, &ureg->size); 1124 1125 if (ret) 1126 return ret; 1127 1128 if (size > PAGE_SIZE) 1129 return -E2BIG; 1130 1131 return copy_struct_from_user(kreg, sizeof(*kreg), ureg, size); 1132 } 1133 1134 /* 1135 * Registers a user_event on behalf of a user process. 1136 */ 1137 static long user_events_ioctl_reg(struct file *file, unsigned long uarg) 1138 { 1139 struct user_reg __user *ureg = (struct user_reg __user *)uarg; 1140 struct user_reg reg; 1141 struct user_event *user; 1142 char *name; 1143 long ret; 1144 1145 ret = user_reg_get(ureg, ®); 1146 1147 if (ret) 1148 return ret; 1149 1150 name = strndup_user((const char __user *)(uintptr_t)reg.name_args, 1151 MAX_EVENT_DESC); 1152 1153 if (IS_ERR(name)) { 1154 ret = PTR_ERR(name); 1155 return ret; 1156 } 1157 1158 ret = user_event_parse_cmd(name, &user); 1159 1160 if (ret) { 1161 kfree(name); 1162 return ret; 1163 } 1164 1165 ret = user_events_ref_add(file, user); 1166 1167 /* Positive number is index and valid */ 1168 if (ret < 0) 1169 return ret; 1170 1171 put_user((u32)ret, &ureg->write_index); 1172 put_user(user->index, &ureg->status_index); 1173 1174 return 0; 1175 } 1176 1177 /* 1178 * Deletes a user_event on behalf of a user process. 1179 */ 1180 static long user_events_ioctl_del(struct file *file, unsigned long uarg) 1181 { 1182 void __user *ubuf = (void __user *)uarg; 1183 char *name; 1184 long ret; 1185 1186 name = strndup_user(ubuf, MAX_EVENT_DESC); 1187 1188 if (IS_ERR(name)) 1189 return PTR_ERR(name); 1190 1191 ret = delete_user_event(name); 1192 1193 kfree(name); 1194 1195 return ret; 1196 } 1197 1198 /* 1199 * Handles the ioctl from user mode to register or alter operations. 1200 */ 1201 static long user_events_ioctl(struct file *file, unsigned int cmd, 1202 unsigned long uarg) 1203 { 1204 long ret = -ENOTTY; 1205 1206 switch (cmd) { 1207 case DIAG_IOCSREG: 1208 mutex_lock(®_mutex); 1209 ret = user_events_ioctl_reg(file, uarg); 1210 mutex_unlock(®_mutex); 1211 break; 1212 1213 case DIAG_IOCSDEL: 1214 mutex_lock(®_mutex); 1215 ret = user_events_ioctl_del(file, uarg); 1216 mutex_unlock(®_mutex); 1217 break; 1218 } 1219 1220 return ret; 1221 } 1222 1223 /* 1224 * Handles the final close of the file from user mode. 1225 */ 1226 static int user_events_release(struct inode *node, struct file *file) 1227 { 1228 struct user_event_refs *refs; 1229 struct user_event *user; 1230 int i; 1231 1232 /* 1233 * Ensure refs cannot change under any situation by taking the 1234 * register mutex during the final freeing of the references. 1235 */ 1236 mutex_lock(®_mutex); 1237 1238 refs = file->private_data; 1239 1240 if (!refs) 1241 goto out; 1242 1243 /* 1244 * The lifetime of refs has reached an end, it's tied to this file. 1245 * The underlying user_events are ref counted, and cannot be freed. 1246 * After this decrement, the user_events may be freed elsewhere. 1247 */ 1248 for (i = 0; i < refs->count; ++i) { 1249 user = refs->events[i]; 1250 1251 if (user) 1252 atomic_dec(&user->refcnt); 1253 } 1254 out: 1255 file->private_data = NULL; 1256 1257 mutex_unlock(®_mutex); 1258 1259 kfree(refs); 1260 1261 return 0; 1262 } 1263 1264 static const struct file_operations user_data_fops = { 1265 .write = user_events_write, 1266 .write_iter = user_events_write_iter, 1267 .unlocked_ioctl = user_events_ioctl, 1268 .release = user_events_release, 1269 }; 1270 1271 /* 1272 * Maps the shared page into the user process for checking if event is enabled. 1273 */ 1274 static int user_status_mmap(struct file *file, struct vm_area_struct *vma) 1275 { 1276 unsigned long size = vma->vm_end - vma->vm_start; 1277 1278 if (size != MAX_EVENTS) 1279 return -EINVAL; 1280 1281 return remap_pfn_range(vma, vma->vm_start, 1282 virt_to_phys(register_page_data) >> PAGE_SHIFT, 1283 size, vm_get_page_prot(VM_READ)); 1284 } 1285 1286 static void *user_seq_start(struct seq_file *m, loff_t *pos) 1287 { 1288 if (*pos) 1289 return NULL; 1290 1291 return (void *)1; 1292 } 1293 1294 static void *user_seq_next(struct seq_file *m, void *p, loff_t *pos) 1295 { 1296 ++*pos; 1297 return NULL; 1298 } 1299 1300 static void user_seq_stop(struct seq_file *m, void *p) 1301 { 1302 } 1303 1304 static int user_seq_show(struct seq_file *m, void *p) 1305 { 1306 struct user_event *user; 1307 char status; 1308 int i, active = 0, busy = 0, flags; 1309 1310 mutex_lock(®_mutex); 1311 1312 hash_for_each(register_table, i, user, node) { 1313 status = register_page_data[user->index]; 1314 flags = user->flags; 1315 1316 seq_printf(m, "%d:%s", user->index, EVENT_NAME(user)); 1317 1318 if (flags != 0 || status != 0) 1319 seq_puts(m, " #"); 1320 1321 if (status != 0) { 1322 seq_puts(m, " Used by"); 1323 if (status & EVENT_STATUS_FTRACE) 1324 seq_puts(m, " ftrace"); 1325 if (status & EVENT_STATUS_PERF) 1326 seq_puts(m, " perf"); 1327 if (status & EVENT_STATUS_OTHER) 1328 seq_puts(m, " other"); 1329 busy++; 1330 } 1331 1332 if (flags & FLAG_BPF_ITER) 1333 seq_puts(m, " FLAG:BPF_ITER"); 1334 1335 seq_puts(m, "\n"); 1336 active++; 1337 } 1338 1339 mutex_unlock(®_mutex); 1340 1341 seq_puts(m, "\n"); 1342 seq_printf(m, "Active: %d\n", active); 1343 seq_printf(m, "Busy: %d\n", busy); 1344 seq_printf(m, "Max: %ld\n", MAX_EVENTS); 1345 1346 return 0; 1347 } 1348 1349 static const struct seq_operations user_seq_ops = { 1350 .start = user_seq_start, 1351 .next = user_seq_next, 1352 .stop = user_seq_stop, 1353 .show = user_seq_show, 1354 }; 1355 1356 static int user_status_open(struct inode *node, struct file *file) 1357 { 1358 return seq_open(file, &user_seq_ops); 1359 } 1360 1361 static const struct file_operations user_status_fops = { 1362 .open = user_status_open, 1363 .mmap = user_status_mmap, 1364 .read = seq_read, 1365 .llseek = seq_lseek, 1366 .release = seq_release, 1367 }; 1368 1369 /* 1370 * Creates a set of tracefs files to allow user mode interactions. 1371 */ 1372 static int create_user_tracefs(void) 1373 { 1374 struct dentry *edata, *emmap; 1375 1376 edata = tracefs_create_file("user_events_data", TRACE_MODE_WRITE, 1377 NULL, NULL, &user_data_fops); 1378 1379 if (!edata) { 1380 pr_warn("Could not create tracefs 'user_events_data' entry\n"); 1381 goto err; 1382 } 1383 1384 /* mmap with MAP_SHARED requires writable fd */ 1385 emmap = tracefs_create_file("user_events_status", TRACE_MODE_WRITE, 1386 NULL, NULL, &user_status_fops); 1387 1388 if (!emmap) { 1389 tracefs_remove(edata); 1390 pr_warn("Could not create tracefs 'user_events_mmap' entry\n"); 1391 goto err; 1392 } 1393 1394 return 0; 1395 err: 1396 return -ENODEV; 1397 } 1398 1399 static void set_page_reservations(bool set) 1400 { 1401 int page; 1402 1403 for (page = 0; page < MAX_PAGES; ++page) { 1404 void *addr = register_page_data + (PAGE_SIZE * page); 1405 1406 if (set) 1407 SetPageReserved(virt_to_page(addr)); 1408 else 1409 ClearPageReserved(virt_to_page(addr)); 1410 } 1411 } 1412 1413 static int __init trace_events_user_init(void) 1414 { 1415 int ret; 1416 1417 /* Zero all bits beside 0 (which is reserved for failures) */ 1418 bitmap_zero(page_bitmap, MAX_EVENTS); 1419 set_bit(0, page_bitmap); 1420 1421 register_page_data = kzalloc(MAX_EVENTS, GFP_KERNEL); 1422 1423 if (!register_page_data) 1424 return -ENOMEM; 1425 1426 set_page_reservations(true); 1427 1428 ret = create_user_tracefs(); 1429 1430 if (ret) { 1431 pr_warn("user_events could not register with tracefs\n"); 1432 set_page_reservations(false); 1433 kfree(register_page_data); 1434 return ret; 1435 } 1436 1437 if (dyn_event_register(&user_event_dops)) 1438 pr_warn("user_events could not register with dyn_events\n"); 1439 1440 return 0; 1441 } 1442 1443 fs_initcall(trace_events_user_init); 1444