1 /* SPDX-License-Identifier: GPL-2.0 */ 2 3 #ifndef _LINUX_TRACE_EVENT_H 4 #define _LINUX_TRACE_EVENT_H 5 6 #include <linux/ring_buffer.h> 7 #include <linux/trace_seq.h> 8 #include <linux/percpu.h> 9 #include <linux/hardirq.h> 10 #include <linux/perf_event.h> 11 #include <linux/tracepoint.h> 12 13 struct trace_array; 14 struct array_buffer; 15 struct tracer; 16 struct dentry; 17 struct bpf_prog; 18 union bpf_attr; 19 20 const char *trace_print_flags_seq(struct trace_seq *p, const char *delim, 21 unsigned long flags, 22 const struct trace_print_flags *flag_array); 23 24 const char *trace_print_symbols_seq(struct trace_seq *p, unsigned long val, 25 const struct trace_print_flags *symbol_array); 26 27 #if BITS_PER_LONG == 32 28 const char *trace_print_flags_seq_u64(struct trace_seq *p, const char *delim, 29 unsigned long long flags, 30 const struct trace_print_flags_u64 *flag_array); 31 32 const char *trace_print_symbols_seq_u64(struct trace_seq *p, 33 unsigned long long val, 34 const struct trace_print_flags_u64 35 *symbol_array); 36 #endif 37 38 const char *trace_print_bitmask_seq(struct trace_seq *p, void *bitmask_ptr, 39 unsigned int bitmask_size); 40 41 const char *trace_print_hex_seq(struct trace_seq *p, 42 const unsigned char *buf, int len, 43 bool concatenate); 44 45 const char *trace_print_array_seq(struct trace_seq *p, 46 const void *buf, int count, 47 size_t el_size); 48 49 const char * 50 trace_print_hex_dump_seq(struct trace_seq *p, const char *prefix_str, 51 int prefix_type, int rowsize, int groupsize, 52 const void *buf, size_t len, bool ascii); 53 54 struct trace_iterator; 55 struct trace_event; 56 57 int trace_raw_output_prep(struct trace_iterator *iter, 58 struct trace_event *event); 59 extern __printf(2, 3) 60 void trace_event_printf(struct trace_iterator *iter, const char *fmt, ...); 61 62 /* Used to find the offset and length of dynamic fields in trace events */ 63 struct trace_dynamic_info { 64 #ifdef CONFIG_CPU_BIG_ENDIAN 65 u16 len; 66 u16 offset; 67 #else 68 u16 offset; 69 u16 len; 70 #endif 71 } __packed; 72 73 /* 74 * The trace entry - the most basic unit of tracing. This is what 75 * is printed in the end as a single line in the trace output, such as: 76 * 77 * bash-15816 [01] 235.197585: idle_cpu <- irq_enter 78 */ 79 struct trace_entry { 80 unsigned short type; 81 unsigned char flags; 82 unsigned char preempt_count; 83 int pid; 84 }; 85 86 #define TRACE_EVENT_TYPE_MAX \ 87 ((1 << (sizeof(((struct trace_entry *)0)->type) * 8)) - 1) 88 89 /* 90 * Trace iterator - used by printout routines who present trace 91 * results to users and which routines might sleep, etc: 92 */ 93 struct trace_iterator { 94 struct trace_array *tr; 95 struct tracer *trace; 96 struct array_buffer *array_buffer; 97 void *private; 98 int cpu_file; 99 struct mutex mutex; 100 struct ring_buffer_iter **buffer_iter; 101 unsigned long iter_flags; 102 void *temp; /* temp holder */ 103 unsigned int temp_size; 104 char *fmt; /* modified format holder */ 105 unsigned int fmt_size; 106 long wait_index; 107 108 /* trace_seq for __print_flags() and __print_symbolic() etc. */ 109 struct trace_seq tmp_seq; 110 111 cpumask_var_t started; 112 113 /* it's true when current open file is snapshot */ 114 bool snapshot; 115 116 /* The below is zeroed out in pipe_read */ 117 struct trace_seq seq; 118 struct trace_entry *ent; 119 unsigned long lost_events; 120 int leftover; 121 int ent_size; 122 int cpu; 123 u64 ts; 124 125 loff_t pos; 126 long idx; 127 128 /* All new field here will be zeroed out in pipe_read */ 129 }; 130 131 enum trace_iter_flags { 132 TRACE_FILE_LAT_FMT = 1, 133 TRACE_FILE_ANNOTATE = 2, 134 TRACE_FILE_TIME_IN_NS = 4, 135 }; 136 137 138 typedef enum print_line_t (*trace_print_func)(struct trace_iterator *iter, 139 int flags, struct trace_event *event); 140 141 struct trace_event_functions { 142 trace_print_func trace; 143 trace_print_func raw; 144 trace_print_func hex; 145 trace_print_func binary; 146 }; 147 148 struct trace_event { 149 struct hlist_node node; 150 int type; 151 struct trace_event_functions *funcs; 152 }; 153 154 extern int register_trace_event(struct trace_event *event); 155 extern int unregister_trace_event(struct trace_event *event); 156 157 /* Return values for print_line callback */ 158 enum print_line_t { 159 TRACE_TYPE_PARTIAL_LINE = 0, /* Retry after flushing the seq */ 160 TRACE_TYPE_HANDLED = 1, 161 TRACE_TYPE_UNHANDLED = 2, /* Relay to other output functions */ 162 TRACE_TYPE_NO_CONSUME = 3 /* Handled but ask to not consume */ 163 }; 164 165 enum print_line_t trace_handle_return(struct trace_seq *s); 166 167 static inline void tracing_generic_entry_update(struct trace_entry *entry, 168 unsigned short type, 169 unsigned int trace_ctx) 170 { 171 entry->preempt_count = trace_ctx & 0xff; 172 entry->pid = current->pid; 173 entry->type = type; 174 entry->flags = trace_ctx >> 16; 175 } 176 177 unsigned int tracing_gen_ctx_irq_test(unsigned int irqs_status); 178 179 enum trace_flag_type { 180 TRACE_FLAG_IRQS_OFF = 0x01, 181 TRACE_FLAG_IRQS_NOSUPPORT = 0x02, 182 TRACE_FLAG_NEED_RESCHED = 0x04, 183 TRACE_FLAG_HARDIRQ = 0x08, 184 TRACE_FLAG_SOFTIRQ = 0x10, 185 TRACE_FLAG_PREEMPT_RESCHED = 0x20, 186 TRACE_FLAG_NMI = 0x40, 187 TRACE_FLAG_BH_OFF = 0x80, 188 }; 189 190 #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT 191 static inline unsigned int tracing_gen_ctx_flags(unsigned long irqflags) 192 { 193 unsigned int irq_status = irqs_disabled_flags(irqflags) ? 194 TRACE_FLAG_IRQS_OFF : 0; 195 return tracing_gen_ctx_irq_test(irq_status); 196 } 197 static inline unsigned int tracing_gen_ctx(void) 198 { 199 unsigned long irqflags; 200 201 local_save_flags(irqflags); 202 return tracing_gen_ctx_flags(irqflags); 203 } 204 #else 205 206 static inline unsigned int tracing_gen_ctx_flags(unsigned long irqflags) 207 { 208 return tracing_gen_ctx_irq_test(TRACE_FLAG_IRQS_NOSUPPORT); 209 } 210 static inline unsigned int tracing_gen_ctx(void) 211 { 212 return tracing_gen_ctx_irq_test(TRACE_FLAG_IRQS_NOSUPPORT); 213 } 214 #endif 215 216 static inline unsigned int tracing_gen_ctx_dec(void) 217 { 218 unsigned int trace_ctx; 219 220 trace_ctx = tracing_gen_ctx(); 221 /* 222 * Subtract one from the preemption counter if preemption is enabled, 223 * see trace_event_buffer_reserve()for details. 224 */ 225 if (IS_ENABLED(CONFIG_PREEMPTION)) 226 trace_ctx--; 227 return trace_ctx; 228 } 229 230 struct trace_event_file; 231 232 struct ring_buffer_event * 233 trace_event_buffer_lock_reserve(struct trace_buffer **current_buffer, 234 struct trace_event_file *trace_file, 235 int type, unsigned long len, 236 unsigned int trace_ctx); 237 238 #define TRACE_RECORD_CMDLINE BIT(0) 239 #define TRACE_RECORD_TGID BIT(1) 240 241 void tracing_record_taskinfo(struct task_struct *task, int flags); 242 void tracing_record_taskinfo_sched_switch(struct task_struct *prev, 243 struct task_struct *next, int flags); 244 245 void tracing_record_cmdline(struct task_struct *task); 246 void tracing_record_tgid(struct task_struct *task); 247 248 int trace_output_call(struct trace_iterator *iter, char *name, char *fmt, ...) 249 __printf(3, 4); 250 251 struct event_filter; 252 253 enum trace_reg { 254 TRACE_REG_REGISTER, 255 TRACE_REG_UNREGISTER, 256 #ifdef CONFIG_PERF_EVENTS 257 TRACE_REG_PERF_REGISTER, 258 TRACE_REG_PERF_UNREGISTER, 259 TRACE_REG_PERF_OPEN, 260 TRACE_REG_PERF_CLOSE, 261 /* 262 * These (ADD/DEL) use a 'boolean' return value, where 1 (true) means a 263 * custom action was taken and the default action is not to be 264 * performed. 265 */ 266 TRACE_REG_PERF_ADD, 267 TRACE_REG_PERF_DEL, 268 #endif 269 }; 270 271 struct trace_event_call; 272 273 #define TRACE_FUNCTION_TYPE ((const char *)~0UL) 274 275 struct trace_event_fields { 276 const char *type; 277 union { 278 struct { 279 const char *name; 280 const int size; 281 const int align; 282 const int is_signed; 283 const int filter_type; 284 const int len; 285 }; 286 int (*define_fields)(struct trace_event_call *); 287 }; 288 }; 289 290 struct trace_event_class { 291 const char *system; 292 void *probe; 293 #ifdef CONFIG_PERF_EVENTS 294 void *perf_probe; 295 #endif 296 int (*reg)(struct trace_event_call *event, 297 enum trace_reg type, void *data); 298 struct trace_event_fields *fields_array; 299 struct list_head *(*get_fields)(struct trace_event_call *); 300 struct list_head fields; 301 int (*raw_init)(struct trace_event_call *); 302 }; 303 304 extern int trace_event_reg(struct trace_event_call *event, 305 enum trace_reg type, void *data); 306 307 struct trace_event_buffer { 308 struct trace_buffer *buffer; 309 struct ring_buffer_event *event; 310 struct trace_event_file *trace_file; 311 void *entry; 312 unsigned int trace_ctx; 313 struct pt_regs *regs; 314 }; 315 316 void *trace_event_buffer_reserve(struct trace_event_buffer *fbuffer, 317 struct trace_event_file *trace_file, 318 unsigned long len); 319 320 void trace_event_buffer_commit(struct trace_event_buffer *fbuffer); 321 322 enum { 323 TRACE_EVENT_FL_FILTERED_BIT, 324 TRACE_EVENT_FL_CAP_ANY_BIT, 325 TRACE_EVENT_FL_NO_SET_FILTER_BIT, 326 TRACE_EVENT_FL_IGNORE_ENABLE_BIT, 327 TRACE_EVENT_FL_TRACEPOINT_BIT, 328 TRACE_EVENT_FL_DYNAMIC_BIT, 329 TRACE_EVENT_FL_KPROBE_BIT, 330 TRACE_EVENT_FL_UPROBE_BIT, 331 TRACE_EVENT_FL_EPROBE_BIT, 332 TRACE_EVENT_FL_FPROBE_BIT, 333 TRACE_EVENT_FL_CUSTOM_BIT, 334 }; 335 336 /* 337 * Event flags: 338 * FILTERED - The event has a filter attached 339 * CAP_ANY - Any user can enable for perf 340 * NO_SET_FILTER - Set when filter has error and is to be ignored 341 * IGNORE_ENABLE - For trace internal events, do not enable with debugfs file 342 * TRACEPOINT - Event is a tracepoint 343 * DYNAMIC - Event is a dynamic event (created at run time) 344 * KPROBE - Event is a kprobe 345 * UPROBE - Event is a uprobe 346 * EPROBE - Event is an event probe 347 * FPROBE - Event is an function probe 348 * CUSTOM - Event is a custom event (to be attached to an exsiting tracepoint) 349 * This is set when the custom event has not been attached 350 * to a tracepoint yet, then it is cleared when it is. 351 */ 352 enum { 353 TRACE_EVENT_FL_FILTERED = (1 << TRACE_EVENT_FL_FILTERED_BIT), 354 TRACE_EVENT_FL_CAP_ANY = (1 << TRACE_EVENT_FL_CAP_ANY_BIT), 355 TRACE_EVENT_FL_NO_SET_FILTER = (1 << TRACE_EVENT_FL_NO_SET_FILTER_BIT), 356 TRACE_EVENT_FL_IGNORE_ENABLE = (1 << TRACE_EVENT_FL_IGNORE_ENABLE_BIT), 357 TRACE_EVENT_FL_TRACEPOINT = (1 << TRACE_EVENT_FL_TRACEPOINT_BIT), 358 TRACE_EVENT_FL_DYNAMIC = (1 << TRACE_EVENT_FL_DYNAMIC_BIT), 359 TRACE_EVENT_FL_KPROBE = (1 << TRACE_EVENT_FL_KPROBE_BIT), 360 TRACE_EVENT_FL_UPROBE = (1 << TRACE_EVENT_FL_UPROBE_BIT), 361 TRACE_EVENT_FL_EPROBE = (1 << TRACE_EVENT_FL_EPROBE_BIT), 362 TRACE_EVENT_FL_FPROBE = (1 << TRACE_EVENT_FL_FPROBE_BIT), 363 TRACE_EVENT_FL_CUSTOM = (1 << TRACE_EVENT_FL_CUSTOM_BIT), 364 }; 365 366 #define TRACE_EVENT_FL_UKPROBE (TRACE_EVENT_FL_KPROBE | TRACE_EVENT_FL_UPROBE) 367 368 struct trace_event_call { 369 struct list_head list; 370 struct trace_event_class *class; 371 union { 372 char *name; 373 /* Set TRACE_EVENT_FL_TRACEPOINT flag when using "tp" */ 374 struct tracepoint *tp; 375 }; 376 struct trace_event event; 377 char *print_fmt; 378 struct event_filter *filter; 379 /* 380 * Static events can disappear with modules, 381 * where as dynamic ones need their own ref count. 382 */ 383 union { 384 void *module; 385 atomic_t refcnt; 386 }; 387 void *data; 388 389 /* See the TRACE_EVENT_FL_* flags above */ 390 int flags; /* static flags of different events */ 391 392 #ifdef CONFIG_PERF_EVENTS 393 int perf_refcount; 394 struct hlist_head __percpu *perf_events; 395 struct bpf_prog_array __rcu *prog_array; 396 397 int (*perf_perm)(struct trace_event_call *, 398 struct perf_event *); 399 #endif 400 }; 401 402 #ifdef CONFIG_DYNAMIC_EVENTS 403 bool trace_event_dyn_try_get_ref(struct trace_event_call *call); 404 void trace_event_dyn_put_ref(struct trace_event_call *call); 405 bool trace_event_dyn_busy(struct trace_event_call *call); 406 #else 407 static inline bool trace_event_dyn_try_get_ref(struct trace_event_call *call) 408 { 409 /* Without DYNAMIC_EVENTS configured, nothing should be calling this */ 410 return false; 411 } 412 static inline void trace_event_dyn_put_ref(struct trace_event_call *call) 413 { 414 } 415 static inline bool trace_event_dyn_busy(struct trace_event_call *call) 416 { 417 /* Nothing should call this without DYNAIMIC_EVENTS configured. */ 418 return true; 419 } 420 #endif 421 422 static inline bool trace_event_try_get_ref(struct trace_event_call *call) 423 { 424 if (call->flags & TRACE_EVENT_FL_DYNAMIC) 425 return trace_event_dyn_try_get_ref(call); 426 else 427 return try_module_get(call->module); 428 } 429 430 static inline void trace_event_put_ref(struct trace_event_call *call) 431 { 432 if (call->flags & TRACE_EVENT_FL_DYNAMIC) 433 trace_event_dyn_put_ref(call); 434 else 435 module_put(call->module); 436 } 437 438 #ifdef CONFIG_PERF_EVENTS 439 static inline bool bpf_prog_array_valid(struct trace_event_call *call) 440 { 441 /* 442 * This inline function checks whether call->prog_array 443 * is valid or not. The function is called in various places, 444 * outside rcu_read_lock/unlock, as a heuristic to speed up execution. 445 * 446 * If this function returns true, and later call->prog_array 447 * becomes false inside rcu_read_lock/unlock region, 448 * we bail out then. If this function return false, 449 * there is a risk that we might miss a few events if the checking 450 * were delayed until inside rcu_read_lock/unlock region and 451 * call->prog_array happened to become non-NULL then. 452 * 453 * Here, READ_ONCE() is used instead of rcu_access_pointer(). 454 * rcu_access_pointer() requires the actual definition of 455 * "struct bpf_prog_array" while READ_ONCE() only needs 456 * a declaration of the same type. 457 */ 458 return !!READ_ONCE(call->prog_array); 459 } 460 #endif 461 462 static inline const char * 463 trace_event_name(struct trace_event_call *call) 464 { 465 if (call->flags & TRACE_EVENT_FL_CUSTOM) 466 return call->name; 467 else if (call->flags & TRACE_EVENT_FL_TRACEPOINT) 468 return call->tp ? call->tp->name : NULL; 469 else 470 return call->name; 471 } 472 473 static inline struct list_head * 474 trace_get_fields(struct trace_event_call *event_call) 475 { 476 if (!event_call->class->get_fields) 477 return &event_call->class->fields; 478 return event_call->class->get_fields(event_call); 479 } 480 481 struct trace_subsystem_dir; 482 483 enum { 484 EVENT_FILE_FL_ENABLED_BIT, 485 EVENT_FILE_FL_RECORDED_CMD_BIT, 486 EVENT_FILE_FL_RECORDED_TGID_BIT, 487 EVENT_FILE_FL_FILTERED_BIT, 488 EVENT_FILE_FL_NO_SET_FILTER_BIT, 489 EVENT_FILE_FL_SOFT_MODE_BIT, 490 EVENT_FILE_FL_SOFT_DISABLED_BIT, 491 EVENT_FILE_FL_TRIGGER_MODE_BIT, 492 EVENT_FILE_FL_TRIGGER_COND_BIT, 493 EVENT_FILE_FL_PID_FILTER_BIT, 494 EVENT_FILE_FL_WAS_ENABLED_BIT, 495 EVENT_FILE_FL_FREED_BIT, 496 }; 497 498 extern struct trace_event_file *trace_get_event_file(const char *instance, 499 const char *system, 500 const char *event); 501 extern void trace_put_event_file(struct trace_event_file *file); 502 503 #define MAX_DYNEVENT_CMD_LEN (2048) 504 505 enum dynevent_type { 506 DYNEVENT_TYPE_SYNTH = 1, 507 DYNEVENT_TYPE_KPROBE, 508 DYNEVENT_TYPE_NONE, 509 }; 510 511 struct dynevent_cmd; 512 513 typedef int (*dynevent_create_fn_t)(struct dynevent_cmd *cmd); 514 515 struct dynevent_cmd { 516 struct seq_buf seq; 517 const char *event_name; 518 unsigned int n_fields; 519 enum dynevent_type type; 520 dynevent_create_fn_t run_command; 521 void *private_data; 522 }; 523 524 extern int dynevent_create(struct dynevent_cmd *cmd); 525 526 extern int synth_event_delete(const char *name); 527 528 extern void synth_event_cmd_init(struct dynevent_cmd *cmd, 529 char *buf, int maxlen); 530 531 extern int __synth_event_gen_cmd_start(struct dynevent_cmd *cmd, 532 const char *name, 533 struct module *mod, ...); 534 535 #define synth_event_gen_cmd_start(cmd, name, mod, ...) \ 536 __synth_event_gen_cmd_start(cmd, name, mod, ## __VA_ARGS__, NULL) 537 538 struct synth_field_desc { 539 const char *type; 540 const char *name; 541 }; 542 543 extern int synth_event_gen_cmd_array_start(struct dynevent_cmd *cmd, 544 const char *name, 545 struct module *mod, 546 struct synth_field_desc *fields, 547 unsigned int n_fields); 548 extern int synth_event_create(const char *name, 549 struct synth_field_desc *fields, 550 unsigned int n_fields, struct module *mod); 551 552 extern int synth_event_add_field(struct dynevent_cmd *cmd, 553 const char *type, 554 const char *name); 555 extern int synth_event_add_field_str(struct dynevent_cmd *cmd, 556 const char *type_name); 557 extern int synth_event_add_fields(struct dynevent_cmd *cmd, 558 struct synth_field_desc *fields, 559 unsigned int n_fields); 560 561 #define synth_event_gen_cmd_end(cmd) \ 562 dynevent_create(cmd) 563 564 struct synth_event; 565 566 struct synth_event_trace_state { 567 struct trace_event_buffer fbuffer; 568 struct synth_trace_event *entry; 569 struct trace_buffer *buffer; 570 struct synth_event *event; 571 unsigned int cur_field; 572 unsigned int n_u64; 573 bool disabled; 574 bool add_next; 575 bool add_name; 576 }; 577 578 extern int synth_event_trace(struct trace_event_file *file, 579 unsigned int n_vals, ...); 580 extern int synth_event_trace_array(struct trace_event_file *file, u64 *vals, 581 unsigned int n_vals); 582 extern int synth_event_trace_start(struct trace_event_file *file, 583 struct synth_event_trace_state *trace_state); 584 extern int synth_event_add_next_val(u64 val, 585 struct synth_event_trace_state *trace_state); 586 extern int synth_event_add_val(const char *field_name, u64 val, 587 struct synth_event_trace_state *trace_state); 588 extern int synth_event_trace_end(struct synth_event_trace_state *trace_state); 589 590 extern int kprobe_event_delete(const char *name); 591 592 extern void kprobe_event_cmd_init(struct dynevent_cmd *cmd, 593 char *buf, int maxlen); 594 595 #define kprobe_event_gen_cmd_start(cmd, name, loc, ...) \ 596 __kprobe_event_gen_cmd_start(cmd, false, name, loc, ## __VA_ARGS__, NULL) 597 598 #define kretprobe_event_gen_cmd_start(cmd, name, loc, ...) \ 599 __kprobe_event_gen_cmd_start(cmd, true, name, loc, ## __VA_ARGS__, NULL) 600 601 extern int __kprobe_event_gen_cmd_start(struct dynevent_cmd *cmd, 602 bool kretprobe, 603 const char *name, 604 const char *loc, ...); 605 606 #define kprobe_event_add_fields(cmd, ...) \ 607 __kprobe_event_add_fields(cmd, ## __VA_ARGS__, NULL) 608 609 #define kprobe_event_add_field(cmd, field) \ 610 __kprobe_event_add_fields(cmd, field, NULL) 611 612 extern int __kprobe_event_add_fields(struct dynevent_cmd *cmd, ...); 613 614 #define kprobe_event_gen_cmd_end(cmd) \ 615 dynevent_create(cmd) 616 617 #define kretprobe_event_gen_cmd_end(cmd) \ 618 dynevent_create(cmd) 619 620 /* 621 * Event file flags: 622 * ENABLED - The event is enabled 623 * RECORDED_CMD - The comms should be recorded at sched_switch 624 * RECORDED_TGID - The tgids should be recorded at sched_switch 625 * FILTERED - The event has a filter attached 626 * NO_SET_FILTER - Set when filter has error and is to be ignored 627 * SOFT_MODE - The event is enabled/disabled by SOFT_DISABLED 628 * SOFT_DISABLED - When set, do not trace the event (even though its 629 * tracepoint may be enabled) 630 * TRIGGER_MODE - When set, invoke the triggers associated with the event 631 * TRIGGER_COND - When set, one or more triggers has an associated filter 632 * PID_FILTER - When set, the event is filtered based on pid 633 * WAS_ENABLED - Set when enabled to know to clear trace on module removal 634 * FREED - File descriptor is freed, all fields should be considered invalid 635 */ 636 enum { 637 EVENT_FILE_FL_ENABLED = (1 << EVENT_FILE_FL_ENABLED_BIT), 638 EVENT_FILE_FL_RECORDED_CMD = (1 << EVENT_FILE_FL_RECORDED_CMD_BIT), 639 EVENT_FILE_FL_RECORDED_TGID = (1 << EVENT_FILE_FL_RECORDED_TGID_BIT), 640 EVENT_FILE_FL_FILTERED = (1 << EVENT_FILE_FL_FILTERED_BIT), 641 EVENT_FILE_FL_NO_SET_FILTER = (1 << EVENT_FILE_FL_NO_SET_FILTER_BIT), 642 EVENT_FILE_FL_SOFT_MODE = (1 << EVENT_FILE_FL_SOFT_MODE_BIT), 643 EVENT_FILE_FL_SOFT_DISABLED = (1 << EVENT_FILE_FL_SOFT_DISABLED_BIT), 644 EVENT_FILE_FL_TRIGGER_MODE = (1 << EVENT_FILE_FL_TRIGGER_MODE_BIT), 645 EVENT_FILE_FL_TRIGGER_COND = (1 << EVENT_FILE_FL_TRIGGER_COND_BIT), 646 EVENT_FILE_FL_PID_FILTER = (1 << EVENT_FILE_FL_PID_FILTER_BIT), 647 EVENT_FILE_FL_WAS_ENABLED = (1 << EVENT_FILE_FL_WAS_ENABLED_BIT), 648 EVENT_FILE_FL_FREED = (1 << EVENT_FILE_FL_FREED_BIT), 649 }; 650 651 struct trace_event_file { 652 struct list_head list; 653 struct trace_event_call *event_call; 654 struct event_filter __rcu *filter; 655 struct eventfs_file *ef; 656 struct trace_array *tr; 657 struct trace_subsystem_dir *system; 658 struct list_head triggers; 659 660 /* 661 * 32 bit flags: 662 * bit 0: enabled 663 * bit 1: enabled cmd record 664 * bit 2: enable/disable with the soft disable bit 665 * bit 3: soft disabled 666 * bit 4: trigger enabled 667 * 668 * Note: The bits must be set atomically to prevent races 669 * from other writers. Reads of flags do not need to be in 670 * sync as they occur in critical sections. But the way flags 671 * is currently used, these changes do not affect the code 672 * except that when a change is made, it may have a slight 673 * delay in propagating the changes to other CPUs due to 674 * caching and such. Which is mostly OK ;-) 675 */ 676 unsigned long flags; 677 atomic_t ref; /* ref count for opened files */ 678 atomic_t sm_ref; /* soft-mode reference counter */ 679 atomic_t tm_ref; /* trigger-mode reference counter */ 680 }; 681 682 #define __TRACE_EVENT_FLAGS(name, value) \ 683 static int __init trace_init_flags_##name(void) \ 684 { \ 685 event_##name.flags |= value; \ 686 return 0; \ 687 } \ 688 early_initcall(trace_init_flags_##name); 689 690 #define __TRACE_EVENT_PERF_PERM(name, expr...) \ 691 static int perf_perm_##name(struct trace_event_call *tp_event, \ 692 struct perf_event *p_event) \ 693 { \ 694 return ({ expr; }); \ 695 } \ 696 static int __init trace_init_perf_perm_##name(void) \ 697 { \ 698 event_##name.perf_perm = &perf_perm_##name; \ 699 return 0; \ 700 } \ 701 early_initcall(trace_init_perf_perm_##name); 702 703 #define PERF_MAX_TRACE_SIZE 8192 704 705 #define MAX_FILTER_STR_VAL 256U /* Should handle KSYM_SYMBOL_LEN */ 706 707 enum event_trigger_type { 708 ETT_NONE = (0), 709 ETT_TRACE_ONOFF = (1 << 0), 710 ETT_SNAPSHOT = (1 << 1), 711 ETT_STACKTRACE = (1 << 2), 712 ETT_EVENT_ENABLE = (1 << 3), 713 ETT_EVENT_HIST = (1 << 4), 714 ETT_HIST_ENABLE = (1 << 5), 715 ETT_EVENT_EPROBE = (1 << 6), 716 }; 717 718 extern int filter_match_preds(struct event_filter *filter, void *rec); 719 720 extern enum event_trigger_type 721 event_triggers_call(struct trace_event_file *file, 722 struct trace_buffer *buffer, void *rec, 723 struct ring_buffer_event *event); 724 extern void 725 event_triggers_post_call(struct trace_event_file *file, 726 enum event_trigger_type tt); 727 728 bool trace_event_ignore_this_pid(struct trace_event_file *trace_file); 729 730 bool __trace_trigger_soft_disabled(struct trace_event_file *file); 731 732 /** 733 * trace_trigger_soft_disabled - do triggers and test if soft disabled 734 * @file: The file pointer of the event to test 735 * 736 * If any triggers without filters are attached to this event, they 737 * will be called here. If the event is soft disabled and has no 738 * triggers that require testing the fields, it will return true, 739 * otherwise false. 740 */ 741 static __always_inline bool 742 trace_trigger_soft_disabled(struct trace_event_file *file) 743 { 744 unsigned long eflags = file->flags; 745 746 if (likely(!(eflags & (EVENT_FILE_FL_TRIGGER_MODE | 747 EVENT_FILE_FL_SOFT_DISABLED | 748 EVENT_FILE_FL_PID_FILTER)))) 749 return false; 750 751 if (likely(eflags & EVENT_FILE_FL_TRIGGER_COND)) 752 return false; 753 754 return __trace_trigger_soft_disabled(file); 755 } 756 757 #ifdef CONFIG_BPF_EVENTS 758 unsigned int trace_call_bpf(struct trace_event_call *call, void *ctx); 759 int perf_event_attach_bpf_prog(struct perf_event *event, struct bpf_prog *prog, u64 bpf_cookie); 760 void perf_event_detach_bpf_prog(struct perf_event *event); 761 int perf_event_query_prog_array(struct perf_event *event, void __user *info); 762 int bpf_probe_register(struct bpf_raw_event_map *btp, struct bpf_prog *prog); 763 int bpf_probe_unregister(struct bpf_raw_event_map *btp, struct bpf_prog *prog); 764 struct bpf_raw_event_map *bpf_get_raw_tracepoint(const char *name); 765 void bpf_put_raw_tracepoint(struct bpf_raw_event_map *btp); 766 int bpf_get_perf_event_info(const struct perf_event *event, u32 *prog_id, 767 u32 *fd_type, const char **buf, 768 u64 *probe_offset, u64 *probe_addr); 769 int bpf_kprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *prog); 770 int bpf_uprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *prog); 771 #else 772 static inline unsigned int trace_call_bpf(struct trace_event_call *call, void *ctx) 773 { 774 return 1; 775 } 776 777 static inline int 778 perf_event_attach_bpf_prog(struct perf_event *event, struct bpf_prog *prog, u64 bpf_cookie) 779 { 780 return -EOPNOTSUPP; 781 } 782 783 static inline void perf_event_detach_bpf_prog(struct perf_event *event) { } 784 785 static inline int 786 perf_event_query_prog_array(struct perf_event *event, void __user *info) 787 { 788 return -EOPNOTSUPP; 789 } 790 static inline int bpf_probe_register(struct bpf_raw_event_map *btp, struct bpf_prog *p) 791 { 792 return -EOPNOTSUPP; 793 } 794 static inline int bpf_probe_unregister(struct bpf_raw_event_map *btp, struct bpf_prog *p) 795 { 796 return -EOPNOTSUPP; 797 } 798 static inline struct bpf_raw_event_map *bpf_get_raw_tracepoint(const char *name) 799 { 800 return NULL; 801 } 802 static inline void bpf_put_raw_tracepoint(struct bpf_raw_event_map *btp) 803 { 804 } 805 static inline int bpf_get_perf_event_info(const struct perf_event *event, 806 u32 *prog_id, u32 *fd_type, 807 const char **buf, u64 *probe_offset, 808 u64 *probe_addr) 809 { 810 return -EOPNOTSUPP; 811 } 812 static inline int 813 bpf_kprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *prog) 814 { 815 return -EOPNOTSUPP; 816 } 817 static inline int 818 bpf_uprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *prog) 819 { 820 return -EOPNOTSUPP; 821 } 822 #endif 823 824 enum { 825 FILTER_OTHER = 0, 826 FILTER_STATIC_STRING, 827 FILTER_DYN_STRING, 828 FILTER_RDYN_STRING, 829 FILTER_PTR_STRING, 830 FILTER_TRACE_FN, 831 FILTER_CPUMASK, 832 FILTER_COMM, 833 FILTER_CPU, 834 FILTER_STACKTRACE, 835 }; 836 837 extern int trace_event_raw_init(struct trace_event_call *call); 838 extern int trace_define_field(struct trace_event_call *call, const char *type, 839 const char *name, int offset, int size, 840 int is_signed, int filter_type); 841 extern int trace_add_event_call(struct trace_event_call *call); 842 extern int trace_remove_event_call(struct trace_event_call *call); 843 extern int trace_event_get_offsets(struct trace_event_call *call); 844 845 int ftrace_set_clr_event(struct trace_array *tr, char *buf, int set); 846 int trace_set_clr_event(const char *system, const char *event, int set); 847 int trace_array_set_clr_event(struct trace_array *tr, const char *system, 848 const char *event, bool enable); 849 /* 850 * The double __builtin_constant_p is because gcc will give us an error 851 * if we try to allocate the static variable to fmt if it is not a 852 * constant. Even with the outer if statement optimizing out. 853 */ 854 #define event_trace_printk(ip, fmt, args...) \ 855 do { \ 856 __trace_printk_check_format(fmt, ##args); \ 857 tracing_record_cmdline(current); \ 858 if (__builtin_constant_p(fmt)) { \ 859 static const char *trace_printk_fmt \ 860 __section("__trace_printk_fmt") = \ 861 __builtin_constant_p(fmt) ? fmt : NULL; \ 862 \ 863 __trace_bprintk(ip, trace_printk_fmt, ##args); \ 864 } else \ 865 __trace_printk(ip, fmt, ##args); \ 866 } while (0) 867 868 #ifdef CONFIG_PERF_EVENTS 869 struct perf_event; 870 871 DECLARE_PER_CPU(struct pt_regs, perf_trace_regs); 872 DECLARE_PER_CPU(int, bpf_kprobe_override); 873 874 extern int perf_trace_init(struct perf_event *event); 875 extern void perf_trace_destroy(struct perf_event *event); 876 extern int perf_trace_add(struct perf_event *event, int flags); 877 extern void perf_trace_del(struct perf_event *event, int flags); 878 #ifdef CONFIG_KPROBE_EVENTS 879 extern int perf_kprobe_init(struct perf_event *event, bool is_retprobe); 880 extern void perf_kprobe_destroy(struct perf_event *event); 881 extern int bpf_get_kprobe_info(const struct perf_event *event, 882 u32 *fd_type, const char **symbol, 883 u64 *probe_offset, u64 *probe_addr, 884 bool perf_type_tracepoint); 885 #endif 886 #ifdef CONFIG_UPROBE_EVENTS 887 extern int perf_uprobe_init(struct perf_event *event, 888 unsigned long ref_ctr_offset, bool is_retprobe); 889 extern void perf_uprobe_destroy(struct perf_event *event); 890 extern int bpf_get_uprobe_info(const struct perf_event *event, 891 u32 *fd_type, const char **filename, 892 u64 *probe_offset, u64 *probe_addr, 893 bool perf_type_tracepoint); 894 #endif 895 extern int ftrace_profile_set_filter(struct perf_event *event, int event_id, 896 char *filter_str); 897 extern void ftrace_profile_free_filter(struct perf_event *event); 898 void perf_trace_buf_update(void *record, u16 type); 899 void *perf_trace_buf_alloc(int size, struct pt_regs **regs, int *rctxp); 900 901 int perf_event_set_bpf_prog(struct perf_event *event, struct bpf_prog *prog, u64 bpf_cookie); 902 void perf_event_free_bpf_prog(struct perf_event *event); 903 904 void bpf_trace_run1(struct bpf_prog *prog, u64 arg1); 905 void bpf_trace_run2(struct bpf_prog *prog, u64 arg1, u64 arg2); 906 void bpf_trace_run3(struct bpf_prog *prog, u64 arg1, u64 arg2, 907 u64 arg3); 908 void bpf_trace_run4(struct bpf_prog *prog, u64 arg1, u64 arg2, 909 u64 arg3, u64 arg4); 910 void bpf_trace_run5(struct bpf_prog *prog, u64 arg1, u64 arg2, 911 u64 arg3, u64 arg4, u64 arg5); 912 void bpf_trace_run6(struct bpf_prog *prog, u64 arg1, u64 arg2, 913 u64 arg3, u64 arg4, u64 arg5, u64 arg6); 914 void bpf_trace_run7(struct bpf_prog *prog, u64 arg1, u64 arg2, 915 u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7); 916 void bpf_trace_run8(struct bpf_prog *prog, u64 arg1, u64 arg2, 917 u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7, 918 u64 arg8); 919 void bpf_trace_run9(struct bpf_prog *prog, u64 arg1, u64 arg2, 920 u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7, 921 u64 arg8, u64 arg9); 922 void bpf_trace_run10(struct bpf_prog *prog, u64 arg1, u64 arg2, 923 u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7, 924 u64 arg8, u64 arg9, u64 arg10); 925 void bpf_trace_run11(struct bpf_prog *prog, u64 arg1, u64 arg2, 926 u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7, 927 u64 arg8, u64 arg9, u64 arg10, u64 arg11); 928 void bpf_trace_run12(struct bpf_prog *prog, u64 arg1, u64 arg2, 929 u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7, 930 u64 arg8, u64 arg9, u64 arg10, u64 arg11, u64 arg12); 931 void perf_trace_run_bpf_submit(void *raw_data, int size, int rctx, 932 struct trace_event_call *call, u64 count, 933 struct pt_regs *regs, struct hlist_head *head, 934 struct task_struct *task); 935 936 static inline void 937 perf_trace_buf_submit(void *raw_data, int size, int rctx, u16 type, 938 u64 count, struct pt_regs *regs, void *head, 939 struct task_struct *task) 940 { 941 perf_tp_event(type, count, raw_data, size, regs, head, rctx, task); 942 } 943 944 #endif 945 946 #define TRACE_EVENT_STR_MAX 512 947 948 /* 949 * gcc warns that you can not use a va_list in an inlined 950 * function. But lets me make it into a macro :-/ 951 */ 952 #define __trace_event_vstr_len(fmt, va) \ 953 ({ \ 954 va_list __ap; \ 955 int __ret; \ 956 \ 957 va_copy(__ap, *(va)); \ 958 __ret = vsnprintf(NULL, 0, fmt, __ap) + 1; \ 959 va_end(__ap); \ 960 \ 961 min(__ret, TRACE_EVENT_STR_MAX); \ 962 }) 963 964 #endif /* _LINUX_TRACE_EVENT_H */ 965 966 /* 967 * Note: we keep the TRACE_CUSTOM_EVENT outside the include file ifdef protection. 968 * This is due to the way trace custom events work. If a file includes two 969 * trace event headers under one "CREATE_CUSTOM_TRACE_EVENTS" the first include 970 * will override the TRACE_CUSTOM_EVENT and break the second include. 971 */ 972 973 #ifndef TRACE_CUSTOM_EVENT 974 975 #define DECLARE_CUSTOM_EVENT_CLASS(name, proto, args, tstruct, assign, print) 976 #define DEFINE_CUSTOM_EVENT(template, name, proto, args) 977 #define TRACE_CUSTOM_EVENT(name, proto, args, struct, assign, print) 978 979 #endif /* ifdef TRACE_CUSTOM_EVENT (see note above) */ 980