1 /* 2 * Performance events: 3 * 4 * Copyright (C) 2008-2009, Thomas Gleixner <tglx@linutronix.de> 5 * Copyright (C) 2008-2011, Red Hat, Inc., Ingo Molnar 6 * Copyright (C) 2008-2011, Red Hat, Inc., Peter Zijlstra 7 * 8 * Data type definitions, declarations, prototypes. 9 * 10 * Started by: Thomas Gleixner and Ingo Molnar 11 * 12 * For licencing details see kernel-base/COPYING 13 */ 14 #ifndef _LINUX_PERF_EVENT_H 15 #define _LINUX_PERF_EVENT_H 16 17 #include <uapi/linux/perf_event.h> 18 19 /* 20 * Kernel-internal data types and definitions: 21 */ 22 23 #ifdef CONFIG_PERF_EVENTS 24 # include <asm/perf_event.h> 25 # include <asm/local64.h> 26 #endif 27 28 struct perf_guest_info_callbacks { 29 int (*is_in_guest)(void); 30 int (*is_user_mode)(void); 31 unsigned long (*get_guest_ip)(void); 32 }; 33 34 #ifdef CONFIG_HAVE_HW_BREAKPOINT 35 #include <asm/hw_breakpoint.h> 36 #endif 37 38 #include <linux/list.h> 39 #include <linux/mutex.h> 40 #include <linux/rculist.h> 41 #include <linux/rcupdate.h> 42 #include <linux/spinlock.h> 43 #include <linux/hrtimer.h> 44 #include <linux/fs.h> 45 #include <linux/pid_namespace.h> 46 #include <linux/workqueue.h> 47 #include <linux/ftrace.h> 48 #include <linux/cpu.h> 49 #include <linux/irq_work.h> 50 #include <linux/static_key.h> 51 #include <linux/jump_label_ratelimit.h> 52 #include <linux/atomic.h> 53 #include <linux/sysfs.h> 54 #include <linux/perf_regs.h> 55 #include <linux/workqueue.h> 56 #include <linux/cgroup.h> 57 #include <asm/local.h> 58 59 struct perf_callchain_entry { 60 __u64 nr; 61 __u64 ip[PERF_MAX_STACK_DEPTH]; 62 }; 63 64 struct perf_raw_record { 65 u32 size; 66 void *data; 67 }; 68 69 /* 70 * branch stack layout: 71 * nr: number of taken branches stored in entries[] 72 * 73 * Note that nr can vary from sample to sample 74 * branches (to, from) are stored from most recent 75 * to least recent, i.e., entries[0] contains the most 76 * recent branch. 77 */ 78 struct perf_branch_stack { 79 __u64 nr; 80 struct perf_branch_entry entries[0]; 81 }; 82 83 struct task_struct; 84 85 /* 86 * extra PMU register associated with an event 87 */ 88 struct hw_perf_event_extra { 89 u64 config; /* register value */ 90 unsigned int reg; /* register address or index */ 91 int alloc; /* extra register already allocated */ 92 int idx; /* index in shared_regs->regs[] */ 93 }; 94 95 /** 96 * struct hw_perf_event - performance event hardware details: 97 */ 98 struct hw_perf_event { 99 #ifdef CONFIG_PERF_EVENTS 100 union { 101 struct { /* hardware */ 102 u64 config; 103 u64 last_tag; 104 unsigned long config_base; 105 unsigned long event_base; 106 int event_base_rdpmc; 107 int idx; 108 int last_cpu; 109 int flags; 110 111 struct hw_perf_event_extra extra_reg; 112 struct hw_perf_event_extra branch_reg; 113 }; 114 struct { /* software */ 115 struct hrtimer hrtimer; 116 }; 117 struct { /* tracepoint */ 118 /* for tp_event->class */ 119 struct list_head tp_list; 120 }; 121 struct { /* intel_cqm */ 122 int cqm_state; 123 u32 cqm_rmid; 124 struct list_head cqm_events_entry; 125 struct list_head cqm_groups_entry; 126 struct list_head cqm_group_entry; 127 }; 128 struct { /* itrace */ 129 int itrace_started; 130 }; 131 #ifdef CONFIG_HAVE_HW_BREAKPOINT 132 struct { /* breakpoint */ 133 /* 134 * Crufty hack to avoid the chicken and egg 135 * problem hw_breakpoint has with context 136 * creation and event initalization. 137 */ 138 struct arch_hw_breakpoint info; 139 struct list_head bp_list; 140 }; 141 #endif 142 }; 143 /* 144 * If the event is a per task event, this will point to the task in 145 * question. See the comment in perf_event_alloc(). 146 */ 147 struct task_struct *target; 148 149 /* 150 * hw_perf_event::state flags; used to track the PERF_EF_* state. 151 */ 152 #define PERF_HES_STOPPED 0x01 /* the counter is stopped */ 153 #define PERF_HES_UPTODATE 0x02 /* event->count up-to-date */ 154 #define PERF_HES_ARCH 0x04 155 156 int state; 157 158 /* 159 * The last observed hardware counter value, updated with a 160 * local64_cmpxchg() such that pmu::read() can be called nested. 161 */ 162 local64_t prev_count; 163 164 /* 165 * The period to start the next sample with. 166 */ 167 u64 sample_period; 168 169 /* 170 * The period we started this sample with. 171 */ 172 u64 last_period; 173 174 /* 175 * However much is left of the current period; note that this is 176 * a full 64bit value and allows for generation of periods longer 177 * than hardware might allow. 178 */ 179 local64_t period_left; 180 181 /* 182 * State for throttling the event, see __perf_event_overflow() and 183 * perf_adjust_freq_unthr_context(). 184 */ 185 u64 interrupts_seq; 186 u64 interrupts; 187 188 /* 189 * State for freq target events, see __perf_event_overflow() and 190 * perf_adjust_freq_unthr_context(). 191 */ 192 u64 freq_time_stamp; 193 u64 freq_count_stamp; 194 #endif 195 }; 196 197 struct perf_event; 198 199 /* 200 * Common implementation detail of pmu::{start,commit,cancel}_txn 201 */ 202 #define PERF_PMU_TXN_ADD 0x1 /* txn to add/schedule event on PMU */ 203 #define PERF_PMU_TXN_READ 0x2 /* txn to read event group from PMU */ 204 205 /** 206 * pmu::capabilities flags 207 */ 208 #define PERF_PMU_CAP_NO_INTERRUPT 0x01 209 #define PERF_PMU_CAP_NO_NMI 0x02 210 #define PERF_PMU_CAP_AUX_NO_SG 0x04 211 #define PERF_PMU_CAP_AUX_SW_DOUBLEBUF 0x08 212 #define PERF_PMU_CAP_EXCLUSIVE 0x10 213 #define PERF_PMU_CAP_ITRACE 0x20 214 215 /** 216 * struct pmu - generic performance monitoring unit 217 */ 218 struct pmu { 219 struct list_head entry; 220 221 struct module *module; 222 struct device *dev; 223 const struct attribute_group **attr_groups; 224 const char *name; 225 int type; 226 227 /* 228 * various common per-pmu feature flags 229 */ 230 int capabilities; 231 232 int * __percpu pmu_disable_count; 233 struct perf_cpu_context * __percpu pmu_cpu_context; 234 atomic_t exclusive_cnt; /* < 0: cpu; > 0: tsk */ 235 int task_ctx_nr; 236 int hrtimer_interval_ms; 237 238 /* 239 * Fully disable/enable this PMU, can be used to protect from the PMI 240 * as well as for lazy/batch writing of the MSRs. 241 */ 242 void (*pmu_enable) (struct pmu *pmu); /* optional */ 243 void (*pmu_disable) (struct pmu *pmu); /* optional */ 244 245 /* 246 * Try and initialize the event for this PMU. 247 * 248 * Returns: 249 * -ENOENT -- @event is not for this PMU 250 * 251 * -ENODEV -- @event is for this PMU but PMU not present 252 * -EBUSY -- @event is for this PMU but PMU temporarily unavailable 253 * -EINVAL -- @event is for this PMU but @event is not valid 254 * -EOPNOTSUPP -- @event is for this PMU, @event is valid, but not supported 255 * -EACCESS -- @event is for this PMU, @event is valid, but no privilidges 256 * 257 * 0 -- @event is for this PMU and valid 258 * 259 * Other error return values are allowed. 260 */ 261 int (*event_init) (struct perf_event *event); 262 263 /* 264 * Notification that the event was mapped or unmapped. Called 265 * in the context of the mapping task. 266 */ 267 void (*event_mapped) (struct perf_event *event); /*optional*/ 268 void (*event_unmapped) (struct perf_event *event); /*optional*/ 269 270 /* 271 * Flags for ->add()/->del()/ ->start()/->stop(). There are 272 * matching hw_perf_event::state flags. 273 */ 274 #define PERF_EF_START 0x01 /* start the counter when adding */ 275 #define PERF_EF_RELOAD 0x02 /* reload the counter when starting */ 276 #define PERF_EF_UPDATE 0x04 /* update the counter when stopping */ 277 278 /* 279 * Adds/Removes a counter to/from the PMU, can be done inside a 280 * transaction, see the ->*_txn() methods. 281 * 282 * The add/del callbacks will reserve all hardware resources required 283 * to service the event, this includes any counter constraint 284 * scheduling etc. 285 * 286 * Called with IRQs disabled and the PMU disabled on the CPU the event 287 * is on. 288 * 289 * ->add() called without PERF_EF_START should result in the same state 290 * as ->add() followed by ->stop(). 291 * 292 * ->del() must always PERF_EF_UPDATE stop an event. If it calls 293 * ->stop() that must deal with already being stopped without 294 * PERF_EF_UPDATE. 295 */ 296 int (*add) (struct perf_event *event, int flags); 297 void (*del) (struct perf_event *event, int flags); 298 299 /* 300 * Starts/Stops a counter present on the PMU. 301 * 302 * The PMI handler should stop the counter when perf_event_overflow() 303 * returns !0. ->start() will be used to continue. 304 * 305 * Also used to change the sample period. 306 * 307 * Called with IRQs disabled and the PMU disabled on the CPU the event 308 * is on -- will be called from NMI context with the PMU generates 309 * NMIs. 310 * 311 * ->stop() with PERF_EF_UPDATE will read the counter and update 312 * period/count values like ->read() would. 313 * 314 * ->start() with PERF_EF_RELOAD will reprogram the the counter 315 * value, must be preceded by a ->stop() with PERF_EF_UPDATE. 316 */ 317 void (*start) (struct perf_event *event, int flags); 318 void (*stop) (struct perf_event *event, int flags); 319 320 /* 321 * Updates the counter value of the event. 322 * 323 * For sampling capable PMUs this will also update the software period 324 * hw_perf_event::period_left field. 325 */ 326 void (*read) (struct perf_event *event); 327 328 /* 329 * Group events scheduling is treated as a transaction, add 330 * group events as a whole and perform one schedulability test. 331 * If the test fails, roll back the whole group 332 * 333 * Start the transaction, after this ->add() doesn't need to 334 * do schedulability tests. 335 * 336 * Optional. 337 */ 338 void (*start_txn) (struct pmu *pmu, unsigned int txn_flags); 339 /* 340 * If ->start_txn() disabled the ->add() schedulability test 341 * then ->commit_txn() is required to perform one. On success 342 * the transaction is closed. On error the transaction is kept 343 * open until ->cancel_txn() is called. 344 * 345 * Optional. 346 */ 347 int (*commit_txn) (struct pmu *pmu); 348 /* 349 * Will cancel the transaction, assumes ->del() is called 350 * for each successful ->add() during the transaction. 351 * 352 * Optional. 353 */ 354 void (*cancel_txn) (struct pmu *pmu); 355 356 /* 357 * Will return the value for perf_event_mmap_page::index for this event, 358 * if no implementation is provided it will default to: event->hw.idx + 1. 359 */ 360 int (*event_idx) (struct perf_event *event); /*optional */ 361 362 /* 363 * context-switches callback 364 */ 365 void (*sched_task) (struct perf_event_context *ctx, 366 bool sched_in); 367 /* 368 * PMU specific data size 369 */ 370 size_t task_ctx_size; 371 372 373 /* 374 * Return the count value for a counter. 375 */ 376 u64 (*count) (struct perf_event *event); /*optional*/ 377 378 /* 379 * Set up pmu-private data structures for an AUX area 380 */ 381 void *(*setup_aux) (int cpu, void **pages, 382 int nr_pages, bool overwrite); 383 /* optional */ 384 385 /* 386 * Free pmu-private AUX data structures 387 */ 388 void (*free_aux) (void *aux); /* optional */ 389 390 /* 391 * Filter events for PMU-specific reasons. 392 */ 393 int (*filter_match) (struct perf_event *event); /* optional */ 394 }; 395 396 /** 397 * enum perf_event_active_state - the states of a event 398 */ 399 enum perf_event_active_state { 400 PERF_EVENT_STATE_EXIT = -3, 401 PERF_EVENT_STATE_ERROR = -2, 402 PERF_EVENT_STATE_OFF = -1, 403 PERF_EVENT_STATE_INACTIVE = 0, 404 PERF_EVENT_STATE_ACTIVE = 1, 405 }; 406 407 struct file; 408 struct perf_sample_data; 409 410 typedef void (*perf_overflow_handler_t)(struct perf_event *, 411 struct perf_sample_data *, 412 struct pt_regs *regs); 413 414 enum perf_group_flag { 415 PERF_GROUP_SOFTWARE = 0x1, 416 }; 417 418 #define SWEVENT_HLIST_BITS 8 419 #define SWEVENT_HLIST_SIZE (1 << SWEVENT_HLIST_BITS) 420 421 struct swevent_hlist { 422 struct hlist_head heads[SWEVENT_HLIST_SIZE]; 423 struct rcu_head rcu_head; 424 }; 425 426 #define PERF_ATTACH_CONTEXT 0x01 427 #define PERF_ATTACH_GROUP 0x02 428 #define PERF_ATTACH_TASK 0x04 429 #define PERF_ATTACH_TASK_DATA 0x08 430 431 struct perf_cgroup; 432 struct ring_buffer; 433 434 /** 435 * struct perf_event - performance event kernel representation: 436 */ 437 struct perf_event { 438 #ifdef CONFIG_PERF_EVENTS 439 /* 440 * entry onto perf_event_context::event_list; 441 * modifications require ctx->lock 442 * RCU safe iterations. 443 */ 444 struct list_head event_entry; 445 446 /* 447 * XXX: group_entry and sibling_list should be mutually exclusive; 448 * either you're a sibling on a group, or you're the group leader. 449 * Rework the code to always use the same list element. 450 * 451 * Locked for modification by both ctx->mutex and ctx->lock; holding 452 * either sufficies for read. 453 */ 454 struct list_head group_entry; 455 struct list_head sibling_list; 456 457 /* 458 * We need storage to track the entries in perf_pmu_migrate_context; we 459 * cannot use the event_entry because of RCU and we want to keep the 460 * group in tact which avoids us using the other two entries. 461 */ 462 struct list_head migrate_entry; 463 464 struct hlist_node hlist_entry; 465 struct list_head active_entry; 466 int nr_siblings; 467 int group_flags; 468 struct perf_event *group_leader; 469 struct pmu *pmu; 470 471 enum perf_event_active_state state; 472 unsigned int attach_state; 473 local64_t count; 474 atomic64_t child_count; 475 476 /* 477 * These are the total time in nanoseconds that the event 478 * has been enabled (i.e. eligible to run, and the task has 479 * been scheduled in, if this is a per-task event) 480 * and running (scheduled onto the CPU), respectively. 481 * 482 * They are computed from tstamp_enabled, tstamp_running and 483 * tstamp_stopped when the event is in INACTIVE or ACTIVE state. 484 */ 485 u64 total_time_enabled; 486 u64 total_time_running; 487 488 /* 489 * These are timestamps used for computing total_time_enabled 490 * and total_time_running when the event is in INACTIVE or 491 * ACTIVE state, measured in nanoseconds from an arbitrary point 492 * in time. 493 * tstamp_enabled: the notional time when the event was enabled 494 * tstamp_running: the notional time when the event was scheduled on 495 * tstamp_stopped: in INACTIVE state, the notional time when the 496 * event was scheduled off. 497 */ 498 u64 tstamp_enabled; 499 u64 tstamp_running; 500 u64 tstamp_stopped; 501 502 /* 503 * timestamp shadows the actual context timing but it can 504 * be safely used in NMI interrupt context. It reflects the 505 * context time as it was when the event was last scheduled in. 506 * 507 * ctx_time already accounts for ctx->timestamp. Therefore to 508 * compute ctx_time for a sample, simply add perf_clock(). 509 */ 510 u64 shadow_ctx_time; 511 512 struct perf_event_attr attr; 513 u16 header_size; 514 u16 id_header_size; 515 u16 read_size; 516 struct hw_perf_event hw; 517 518 struct perf_event_context *ctx; 519 atomic_long_t refcount; 520 521 /* 522 * These accumulate total time (in nanoseconds) that children 523 * events have been enabled and running, respectively. 524 */ 525 atomic64_t child_total_time_enabled; 526 atomic64_t child_total_time_running; 527 528 /* 529 * Protect attach/detach and child_list: 530 */ 531 struct mutex child_mutex; 532 struct list_head child_list; 533 struct perf_event *parent; 534 535 int oncpu; 536 int cpu; 537 538 struct list_head owner_entry; 539 struct task_struct *owner; 540 541 /* mmap bits */ 542 struct mutex mmap_mutex; 543 atomic_t mmap_count; 544 545 struct ring_buffer *rb; 546 struct list_head rb_entry; 547 unsigned long rcu_batches; 548 int rcu_pending; 549 550 /* poll related */ 551 wait_queue_head_t waitq; 552 struct fasync_struct *fasync; 553 554 /* delayed work for NMIs and such */ 555 int pending_wakeup; 556 int pending_kill; 557 int pending_disable; 558 struct irq_work pending; 559 560 atomic_t event_limit; 561 562 void (*destroy)(struct perf_event *); 563 struct rcu_head rcu_head; 564 565 struct pid_namespace *ns; 566 u64 id; 567 568 u64 (*clock)(void); 569 perf_overflow_handler_t overflow_handler; 570 void *overflow_handler_context; 571 572 #ifdef CONFIG_EVENT_TRACING 573 struct trace_event_call *tp_event; 574 struct event_filter *filter; 575 #ifdef CONFIG_FUNCTION_TRACER 576 struct ftrace_ops ftrace_ops; 577 #endif 578 #endif 579 580 #ifdef CONFIG_CGROUP_PERF 581 struct perf_cgroup *cgrp; /* cgroup event is attach to */ 582 int cgrp_defer_enabled; 583 #endif 584 585 #endif /* CONFIG_PERF_EVENTS */ 586 }; 587 588 /** 589 * struct perf_event_context - event context structure 590 * 591 * Used as a container for task events and CPU events as well: 592 */ 593 struct perf_event_context { 594 struct pmu *pmu; 595 /* 596 * Protect the states of the events in the list, 597 * nr_active, and the list: 598 */ 599 raw_spinlock_t lock; 600 /* 601 * Protect the list of events. Locking either mutex or lock 602 * is sufficient to ensure the list doesn't change; to change 603 * the list you need to lock both the mutex and the spinlock. 604 */ 605 struct mutex mutex; 606 607 struct list_head active_ctx_list; 608 struct list_head pinned_groups; 609 struct list_head flexible_groups; 610 struct list_head event_list; 611 int nr_events; 612 int nr_active; 613 int is_active; 614 int nr_stat; 615 int nr_freq; 616 int rotate_disable; 617 atomic_t refcount; 618 struct task_struct *task; 619 620 /* 621 * Context clock, runs when context enabled. 622 */ 623 u64 time; 624 u64 timestamp; 625 626 /* 627 * These fields let us detect when two contexts have both 628 * been cloned (inherited) from a common ancestor. 629 */ 630 struct perf_event_context *parent_ctx; 631 u64 parent_gen; 632 u64 generation; 633 int pin_count; 634 int nr_cgroups; /* cgroup evts */ 635 void *task_ctx_data; /* pmu specific data */ 636 struct rcu_head rcu_head; 637 638 struct delayed_work orphans_remove; 639 bool orphans_remove_sched; 640 }; 641 642 /* 643 * Number of contexts where an event can trigger: 644 * task, softirq, hardirq, nmi. 645 */ 646 #define PERF_NR_CONTEXTS 4 647 648 /** 649 * struct perf_event_cpu_context - per cpu event context structure 650 */ 651 struct perf_cpu_context { 652 struct perf_event_context ctx; 653 struct perf_event_context *task_ctx; 654 int active_oncpu; 655 int exclusive; 656 657 raw_spinlock_t hrtimer_lock; 658 struct hrtimer hrtimer; 659 ktime_t hrtimer_interval; 660 unsigned int hrtimer_active; 661 662 struct pmu *unique_pmu; 663 struct perf_cgroup *cgrp; 664 }; 665 666 struct perf_output_handle { 667 struct perf_event *event; 668 struct ring_buffer *rb; 669 unsigned long wakeup; 670 unsigned long size; 671 union { 672 void *addr; 673 unsigned long head; 674 }; 675 int page; 676 }; 677 678 #ifdef CONFIG_CGROUP_PERF 679 680 /* 681 * perf_cgroup_info keeps track of time_enabled for a cgroup. 682 * This is a per-cpu dynamically allocated data structure. 683 */ 684 struct perf_cgroup_info { 685 u64 time; 686 u64 timestamp; 687 }; 688 689 struct perf_cgroup { 690 struct cgroup_subsys_state css; 691 struct perf_cgroup_info __percpu *info; 692 }; 693 694 /* 695 * Must ensure cgroup is pinned (css_get) before calling 696 * this function. In other words, we cannot call this function 697 * if there is no cgroup event for the current CPU context. 698 */ 699 static inline struct perf_cgroup * 700 perf_cgroup_from_task(struct task_struct *task) 701 { 702 return container_of(task_css(task, perf_event_cgrp_id), 703 struct perf_cgroup, css); 704 } 705 #endif /* CONFIG_CGROUP_PERF */ 706 707 #ifdef CONFIG_PERF_EVENTS 708 709 extern void *perf_aux_output_begin(struct perf_output_handle *handle, 710 struct perf_event *event); 711 extern void perf_aux_output_end(struct perf_output_handle *handle, 712 unsigned long size, bool truncated); 713 extern int perf_aux_output_skip(struct perf_output_handle *handle, 714 unsigned long size); 715 extern void *perf_get_aux(struct perf_output_handle *handle); 716 717 extern int perf_pmu_register(struct pmu *pmu, const char *name, int type); 718 extern void perf_pmu_unregister(struct pmu *pmu); 719 720 extern int perf_num_counters(void); 721 extern const char *perf_pmu_name(void); 722 extern void __perf_event_task_sched_in(struct task_struct *prev, 723 struct task_struct *task); 724 extern void __perf_event_task_sched_out(struct task_struct *prev, 725 struct task_struct *next); 726 extern int perf_event_init_task(struct task_struct *child); 727 extern void perf_event_exit_task(struct task_struct *child); 728 extern void perf_event_free_task(struct task_struct *task); 729 extern void perf_event_delayed_put(struct task_struct *task); 730 extern struct perf_event *perf_event_get(unsigned int fd); 731 extern const struct perf_event_attr *perf_event_attrs(struct perf_event *event); 732 extern void perf_event_print_debug(void); 733 extern void perf_pmu_disable(struct pmu *pmu); 734 extern void perf_pmu_enable(struct pmu *pmu); 735 extern void perf_sched_cb_dec(struct pmu *pmu); 736 extern void perf_sched_cb_inc(struct pmu *pmu); 737 extern int perf_event_task_disable(void); 738 extern int perf_event_task_enable(void); 739 extern int perf_event_refresh(struct perf_event *event, int refresh); 740 extern void perf_event_update_userpage(struct perf_event *event); 741 extern int perf_event_release_kernel(struct perf_event *event); 742 extern struct perf_event * 743 perf_event_create_kernel_counter(struct perf_event_attr *attr, 744 int cpu, 745 struct task_struct *task, 746 perf_overflow_handler_t callback, 747 void *context); 748 extern void perf_pmu_migrate_context(struct pmu *pmu, 749 int src_cpu, int dst_cpu); 750 extern u64 perf_event_read_local(struct perf_event *event); 751 extern u64 perf_event_read_value(struct perf_event *event, 752 u64 *enabled, u64 *running); 753 754 755 struct perf_sample_data { 756 /* 757 * Fields set by perf_sample_data_init(), group so as to 758 * minimize the cachelines touched. 759 */ 760 u64 addr; 761 struct perf_raw_record *raw; 762 struct perf_branch_stack *br_stack; 763 u64 period; 764 u64 weight; 765 u64 txn; 766 union perf_mem_data_src data_src; 767 768 /* 769 * The other fields, optionally {set,used} by 770 * perf_{prepare,output}_sample(). 771 */ 772 u64 type; 773 u64 ip; 774 struct { 775 u32 pid; 776 u32 tid; 777 } tid_entry; 778 u64 time; 779 u64 id; 780 u64 stream_id; 781 struct { 782 u32 cpu; 783 u32 reserved; 784 } cpu_entry; 785 struct perf_callchain_entry *callchain; 786 787 /* 788 * regs_user may point to task_pt_regs or to regs_user_copy, depending 789 * on arch details. 790 */ 791 struct perf_regs regs_user; 792 struct pt_regs regs_user_copy; 793 794 struct perf_regs regs_intr; 795 u64 stack_user_size; 796 } ____cacheline_aligned; 797 798 /* default value for data source */ 799 #define PERF_MEM_NA (PERF_MEM_S(OP, NA) |\ 800 PERF_MEM_S(LVL, NA) |\ 801 PERF_MEM_S(SNOOP, NA) |\ 802 PERF_MEM_S(LOCK, NA) |\ 803 PERF_MEM_S(TLB, NA)) 804 805 static inline void perf_sample_data_init(struct perf_sample_data *data, 806 u64 addr, u64 period) 807 { 808 /* remaining struct members initialized in perf_prepare_sample() */ 809 data->addr = addr; 810 data->raw = NULL; 811 data->br_stack = NULL; 812 data->period = period; 813 data->weight = 0; 814 data->data_src.val = PERF_MEM_NA; 815 data->txn = 0; 816 } 817 818 extern void perf_output_sample(struct perf_output_handle *handle, 819 struct perf_event_header *header, 820 struct perf_sample_data *data, 821 struct perf_event *event); 822 extern void perf_prepare_sample(struct perf_event_header *header, 823 struct perf_sample_data *data, 824 struct perf_event *event, 825 struct pt_regs *regs); 826 827 extern int perf_event_overflow(struct perf_event *event, 828 struct perf_sample_data *data, 829 struct pt_regs *regs); 830 831 extern void perf_event_output(struct perf_event *event, 832 struct perf_sample_data *data, 833 struct pt_regs *regs); 834 835 extern void 836 perf_event_header__init_id(struct perf_event_header *header, 837 struct perf_sample_data *data, 838 struct perf_event *event); 839 extern void 840 perf_event__output_id_sample(struct perf_event *event, 841 struct perf_output_handle *handle, 842 struct perf_sample_data *sample); 843 844 extern void 845 perf_log_lost_samples(struct perf_event *event, u64 lost); 846 847 static inline bool is_sampling_event(struct perf_event *event) 848 { 849 return event->attr.sample_period != 0; 850 } 851 852 /* 853 * Return 1 for a software event, 0 for a hardware event 854 */ 855 static inline int is_software_event(struct perf_event *event) 856 { 857 return event->pmu->task_ctx_nr == perf_sw_context; 858 } 859 860 extern struct static_key perf_swevent_enabled[PERF_COUNT_SW_MAX]; 861 862 extern void ___perf_sw_event(u32, u64, struct pt_regs *, u64); 863 extern void __perf_sw_event(u32, u64, struct pt_regs *, u64); 864 865 #ifndef perf_arch_fetch_caller_regs 866 static inline void perf_arch_fetch_caller_regs(struct pt_regs *regs, unsigned long ip) { } 867 #endif 868 869 /* 870 * Take a snapshot of the regs. Skip ip and frame pointer to 871 * the nth caller. We only need a few of the regs: 872 * - ip for PERF_SAMPLE_IP 873 * - cs for user_mode() tests 874 * - bp for callchains 875 * - eflags, for future purposes, just in case 876 */ 877 static inline void perf_fetch_caller_regs(struct pt_regs *regs) 878 { 879 memset(regs, 0, sizeof(*regs)); 880 881 perf_arch_fetch_caller_regs(regs, CALLER_ADDR0); 882 } 883 884 static __always_inline void 885 perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr) 886 { 887 if (static_key_false(&perf_swevent_enabled[event_id])) 888 __perf_sw_event(event_id, nr, regs, addr); 889 } 890 891 DECLARE_PER_CPU(struct pt_regs, __perf_regs[4]); 892 893 /* 894 * 'Special' version for the scheduler, it hard assumes no recursion, 895 * which is guaranteed by us not actually scheduling inside other swevents 896 * because those disable preemption. 897 */ 898 static __always_inline void 899 perf_sw_event_sched(u32 event_id, u64 nr, u64 addr) 900 { 901 if (static_key_false(&perf_swevent_enabled[event_id])) { 902 struct pt_regs *regs = this_cpu_ptr(&__perf_regs[0]); 903 904 perf_fetch_caller_regs(regs); 905 ___perf_sw_event(event_id, nr, regs, addr); 906 } 907 } 908 909 extern struct static_key_deferred perf_sched_events; 910 911 static __always_inline bool 912 perf_sw_migrate_enabled(void) 913 { 914 if (static_key_false(&perf_swevent_enabled[PERF_COUNT_SW_CPU_MIGRATIONS])) 915 return true; 916 return false; 917 } 918 919 static inline void perf_event_task_migrate(struct task_struct *task) 920 { 921 if (perf_sw_migrate_enabled()) 922 task->sched_migrated = 1; 923 } 924 925 static inline void perf_event_task_sched_in(struct task_struct *prev, 926 struct task_struct *task) 927 { 928 if (static_key_false(&perf_sched_events.key)) 929 __perf_event_task_sched_in(prev, task); 930 931 if (perf_sw_migrate_enabled() && task->sched_migrated) { 932 struct pt_regs *regs = this_cpu_ptr(&__perf_regs[0]); 933 934 perf_fetch_caller_regs(regs); 935 ___perf_sw_event(PERF_COUNT_SW_CPU_MIGRATIONS, 1, regs, 0); 936 task->sched_migrated = 0; 937 } 938 } 939 940 static inline void perf_event_task_sched_out(struct task_struct *prev, 941 struct task_struct *next) 942 { 943 perf_sw_event_sched(PERF_COUNT_SW_CONTEXT_SWITCHES, 1, 0); 944 945 if (static_key_false(&perf_sched_events.key)) 946 __perf_event_task_sched_out(prev, next); 947 } 948 949 static inline u64 __perf_event_count(struct perf_event *event) 950 { 951 return local64_read(&event->count) + atomic64_read(&event->child_count); 952 } 953 954 extern void perf_event_mmap(struct vm_area_struct *vma); 955 extern struct perf_guest_info_callbacks *perf_guest_cbs; 956 extern int perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *callbacks); 957 extern int perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *callbacks); 958 959 extern void perf_event_exec(void); 960 extern void perf_event_comm(struct task_struct *tsk, bool exec); 961 extern void perf_event_fork(struct task_struct *tsk); 962 963 /* Callchains */ 964 DECLARE_PER_CPU(struct perf_callchain_entry, perf_callchain_entry); 965 966 extern void perf_callchain_user(struct perf_callchain_entry *entry, struct pt_regs *regs); 967 extern void perf_callchain_kernel(struct perf_callchain_entry *entry, struct pt_regs *regs); 968 969 static inline void perf_callchain_store(struct perf_callchain_entry *entry, u64 ip) 970 { 971 if (entry->nr < PERF_MAX_STACK_DEPTH) 972 entry->ip[entry->nr++] = ip; 973 } 974 975 extern int sysctl_perf_event_paranoid; 976 extern int sysctl_perf_event_mlock; 977 extern int sysctl_perf_event_sample_rate; 978 extern int sysctl_perf_cpu_time_max_percent; 979 980 extern void perf_sample_event_took(u64 sample_len_ns); 981 982 extern int perf_proc_update_handler(struct ctl_table *table, int write, 983 void __user *buffer, size_t *lenp, 984 loff_t *ppos); 985 extern int perf_cpu_time_max_percent_handler(struct ctl_table *table, int write, 986 void __user *buffer, size_t *lenp, 987 loff_t *ppos); 988 989 990 static inline bool perf_paranoid_tracepoint_raw(void) 991 { 992 return sysctl_perf_event_paranoid > -1; 993 } 994 995 static inline bool perf_paranoid_cpu(void) 996 { 997 return sysctl_perf_event_paranoid > 0; 998 } 999 1000 static inline bool perf_paranoid_kernel(void) 1001 { 1002 return sysctl_perf_event_paranoid > 1; 1003 } 1004 1005 extern void perf_event_init(void); 1006 extern void perf_tp_event(u64 addr, u64 count, void *record, 1007 int entry_size, struct pt_regs *regs, 1008 struct hlist_head *head, int rctx, 1009 struct task_struct *task); 1010 extern void perf_bp_event(struct perf_event *event, void *data); 1011 1012 #ifndef perf_misc_flags 1013 # define perf_misc_flags(regs) \ 1014 (user_mode(regs) ? PERF_RECORD_MISC_USER : PERF_RECORD_MISC_KERNEL) 1015 # define perf_instruction_pointer(regs) instruction_pointer(regs) 1016 #endif 1017 1018 static inline bool has_branch_stack(struct perf_event *event) 1019 { 1020 return event->attr.sample_type & PERF_SAMPLE_BRANCH_STACK; 1021 } 1022 1023 static inline bool needs_branch_stack(struct perf_event *event) 1024 { 1025 return event->attr.branch_sample_type != 0; 1026 } 1027 1028 static inline bool has_aux(struct perf_event *event) 1029 { 1030 return event->pmu->setup_aux; 1031 } 1032 1033 extern int perf_output_begin(struct perf_output_handle *handle, 1034 struct perf_event *event, unsigned int size); 1035 extern void perf_output_end(struct perf_output_handle *handle); 1036 extern unsigned int perf_output_copy(struct perf_output_handle *handle, 1037 const void *buf, unsigned int len); 1038 extern unsigned int perf_output_skip(struct perf_output_handle *handle, 1039 unsigned int len); 1040 extern int perf_swevent_get_recursion_context(void); 1041 extern void perf_swevent_put_recursion_context(int rctx); 1042 extern u64 perf_swevent_set_period(struct perf_event *event); 1043 extern void perf_event_enable(struct perf_event *event); 1044 extern void perf_event_disable(struct perf_event *event); 1045 extern int __perf_event_disable(void *info); 1046 extern void perf_event_task_tick(void); 1047 #else /* !CONFIG_PERF_EVENTS: */ 1048 static inline void * 1049 perf_aux_output_begin(struct perf_output_handle *handle, 1050 struct perf_event *event) { return NULL; } 1051 static inline void 1052 perf_aux_output_end(struct perf_output_handle *handle, unsigned long size, 1053 bool truncated) { } 1054 static inline int 1055 perf_aux_output_skip(struct perf_output_handle *handle, 1056 unsigned long size) { return -EINVAL; } 1057 static inline void * 1058 perf_get_aux(struct perf_output_handle *handle) { return NULL; } 1059 static inline void 1060 perf_event_task_migrate(struct task_struct *task) { } 1061 static inline void 1062 perf_event_task_sched_in(struct task_struct *prev, 1063 struct task_struct *task) { } 1064 static inline void 1065 perf_event_task_sched_out(struct task_struct *prev, 1066 struct task_struct *next) { } 1067 static inline int perf_event_init_task(struct task_struct *child) { return 0; } 1068 static inline void perf_event_exit_task(struct task_struct *child) { } 1069 static inline void perf_event_free_task(struct task_struct *task) { } 1070 static inline void perf_event_delayed_put(struct task_struct *task) { } 1071 static inline struct perf_event *perf_event_get(unsigned int fd) { return ERR_PTR(-EINVAL); } 1072 static inline const struct perf_event_attr *perf_event_attrs(struct perf_event *event) 1073 { 1074 return ERR_PTR(-EINVAL); 1075 } 1076 static inline u64 perf_event_read_local(struct perf_event *event) { return -EINVAL; } 1077 static inline void perf_event_print_debug(void) { } 1078 static inline int perf_event_task_disable(void) { return -EINVAL; } 1079 static inline int perf_event_task_enable(void) { return -EINVAL; } 1080 static inline int perf_event_refresh(struct perf_event *event, int refresh) 1081 { 1082 return -EINVAL; 1083 } 1084 1085 static inline void 1086 perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr) { } 1087 static inline void 1088 perf_sw_event_sched(u32 event_id, u64 nr, u64 addr) { } 1089 static inline void 1090 perf_bp_event(struct perf_event *event, void *data) { } 1091 1092 static inline int perf_register_guest_info_callbacks 1093 (struct perf_guest_info_callbacks *callbacks) { return 0; } 1094 static inline int perf_unregister_guest_info_callbacks 1095 (struct perf_guest_info_callbacks *callbacks) { return 0; } 1096 1097 static inline void perf_event_mmap(struct vm_area_struct *vma) { } 1098 static inline void perf_event_exec(void) { } 1099 static inline void perf_event_comm(struct task_struct *tsk, bool exec) { } 1100 static inline void perf_event_fork(struct task_struct *tsk) { } 1101 static inline void perf_event_init(void) { } 1102 static inline int perf_swevent_get_recursion_context(void) { return -1; } 1103 static inline void perf_swevent_put_recursion_context(int rctx) { } 1104 static inline u64 perf_swevent_set_period(struct perf_event *event) { return 0; } 1105 static inline void perf_event_enable(struct perf_event *event) { } 1106 static inline void perf_event_disable(struct perf_event *event) { } 1107 static inline int __perf_event_disable(void *info) { return -1; } 1108 static inline void perf_event_task_tick(void) { } 1109 static inline int perf_event_release_kernel(struct perf_event *event) { return 0; } 1110 #endif 1111 1112 #if defined(CONFIG_PERF_EVENTS) && defined(CONFIG_NO_HZ_FULL) 1113 extern bool perf_event_can_stop_tick(void); 1114 #else 1115 static inline bool perf_event_can_stop_tick(void) { return true; } 1116 #endif 1117 1118 #if defined(CONFIG_PERF_EVENTS) && defined(CONFIG_CPU_SUP_INTEL) 1119 extern void perf_restore_debug_store(void); 1120 #else 1121 static inline void perf_restore_debug_store(void) { } 1122 #endif 1123 1124 #define perf_output_put(handle, x) perf_output_copy((handle), &(x), sizeof(x)) 1125 1126 /* 1127 * This has to have a higher priority than migration_notifier in sched/core.c. 1128 */ 1129 #define perf_cpu_notifier(fn) \ 1130 do { \ 1131 static struct notifier_block fn##_nb = \ 1132 { .notifier_call = fn, .priority = CPU_PRI_PERF }; \ 1133 unsigned long cpu = smp_processor_id(); \ 1134 unsigned long flags; \ 1135 \ 1136 cpu_notifier_register_begin(); \ 1137 fn(&fn##_nb, (unsigned long)CPU_UP_PREPARE, \ 1138 (void *)(unsigned long)cpu); \ 1139 local_irq_save(flags); \ 1140 fn(&fn##_nb, (unsigned long)CPU_STARTING, \ 1141 (void *)(unsigned long)cpu); \ 1142 local_irq_restore(flags); \ 1143 fn(&fn##_nb, (unsigned long)CPU_ONLINE, \ 1144 (void *)(unsigned long)cpu); \ 1145 __register_cpu_notifier(&fn##_nb); \ 1146 cpu_notifier_register_done(); \ 1147 } while (0) 1148 1149 /* 1150 * Bare-bones version of perf_cpu_notifier(), which doesn't invoke the 1151 * callback for already online CPUs. 1152 */ 1153 #define __perf_cpu_notifier(fn) \ 1154 do { \ 1155 static struct notifier_block fn##_nb = \ 1156 { .notifier_call = fn, .priority = CPU_PRI_PERF }; \ 1157 \ 1158 __register_cpu_notifier(&fn##_nb); \ 1159 } while (0) 1160 1161 struct perf_pmu_events_attr { 1162 struct device_attribute attr; 1163 u64 id; 1164 const char *event_str; 1165 }; 1166 1167 ssize_t perf_event_sysfs_show(struct device *dev, struct device_attribute *attr, 1168 char *page); 1169 1170 #define PMU_EVENT_ATTR(_name, _var, _id, _show) \ 1171 static struct perf_pmu_events_attr _var = { \ 1172 .attr = __ATTR(_name, 0444, _show, NULL), \ 1173 .id = _id, \ 1174 }; 1175 1176 #define PMU_EVENT_ATTR_STRING(_name, _var, _str) \ 1177 static struct perf_pmu_events_attr _var = { \ 1178 .attr = __ATTR(_name, 0444, perf_event_sysfs_show, NULL), \ 1179 .id = 0, \ 1180 .event_str = _str, \ 1181 }; 1182 1183 #define PMU_FORMAT_ATTR(_name, _format) \ 1184 static ssize_t \ 1185 _name##_show(struct device *dev, \ 1186 struct device_attribute *attr, \ 1187 char *page) \ 1188 { \ 1189 BUILD_BUG_ON(sizeof(_format) >= PAGE_SIZE); \ 1190 return sprintf(page, _format "\n"); \ 1191 } \ 1192 \ 1193 static struct device_attribute format_attr_##_name = __ATTR_RO(_name) 1194 1195 #endif /* _LINUX_PERF_EVENT_H */ 1196