1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Performance events core code: 4 * 5 * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de> 6 * Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar 7 * Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra 8 * Copyright © 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com> 9 */ 10 11 #include <linux/fs.h> 12 #include <linux/mm.h> 13 #include <linux/cpu.h> 14 #include <linux/smp.h> 15 #include <linux/idr.h> 16 #include <linux/file.h> 17 #include <linux/poll.h> 18 #include <linux/slab.h> 19 #include <linux/hash.h> 20 #include <linux/tick.h> 21 #include <linux/sysfs.h> 22 #include <linux/dcache.h> 23 #include <linux/percpu.h> 24 #include <linux/ptrace.h> 25 #include <linux/reboot.h> 26 #include <linux/vmstat.h> 27 #include <linux/device.h> 28 #include <linux/export.h> 29 #include <linux/vmalloc.h> 30 #include <linux/hardirq.h> 31 #include <linux/rculist.h> 32 #include <linux/uaccess.h> 33 #include <linux/syscalls.h> 34 #include <linux/anon_inodes.h> 35 #include <linux/kernel_stat.h> 36 #include <linux/cgroup.h> 37 #include <linux/perf_event.h> 38 #include <linux/trace_events.h> 39 #include <linux/hw_breakpoint.h> 40 #include <linux/mm_types.h> 41 #include <linux/module.h> 42 #include <linux/mman.h> 43 #include <linux/compat.h> 44 #include <linux/bpf.h> 45 #include <linux/filter.h> 46 #include <linux/namei.h> 47 #include <linux/parser.h> 48 #include <linux/sched/clock.h> 49 #include <linux/sched/mm.h> 50 #include <linux/proc_ns.h> 51 #include <linux/mount.h> 52 53 #include "internal.h" 54 55 #include <asm/irq_regs.h> 56 57 typedef int (*remote_function_f)(void *); 58 59 struct remote_function_call { 60 struct task_struct *p; 61 remote_function_f func; 62 void *info; 63 int ret; 64 }; 65 66 static void remote_function(void *data) 67 { 68 struct remote_function_call *tfc = data; 69 struct task_struct *p = tfc->p; 70 71 if (p) { 72 /* -EAGAIN */ 73 if (task_cpu(p) != smp_processor_id()) 74 return; 75 76 /* 77 * Now that we're on right CPU with IRQs disabled, we can test 78 * if we hit the right task without races. 79 */ 80 81 tfc->ret = -ESRCH; /* No such (running) process */ 82 if (p != current) 83 return; 84 } 85 86 tfc->ret = tfc->func(tfc->info); 87 } 88 89 /** 90 * task_function_call - call a function on the cpu on which a task runs 91 * @p: the task to evaluate 92 * @func: the function to be called 93 * @info: the function call argument 94 * 95 * Calls the function @func when the task is currently running. This might 96 * be on the current CPU, which just calls the function directly 97 * 98 * returns: @func return value, or 99 * -ESRCH - when the process isn't running 100 * -EAGAIN - when the process moved away 101 */ 102 static int 103 task_function_call(struct task_struct *p, remote_function_f func, void *info) 104 { 105 struct remote_function_call data = { 106 .p = p, 107 .func = func, 108 .info = info, 109 .ret = -EAGAIN, 110 }; 111 int ret; 112 113 do { 114 ret = smp_call_function_single(task_cpu(p), remote_function, &data, 1); 115 if (!ret) 116 ret = data.ret; 117 } while (ret == -EAGAIN); 118 119 return ret; 120 } 121 122 /** 123 * cpu_function_call - call a function on the cpu 124 * @func: the function to be called 125 * @info: the function call argument 126 * 127 * Calls the function @func on the remote cpu. 128 * 129 * returns: @func return value or -ENXIO when the cpu is offline 130 */ 131 static int cpu_function_call(int cpu, remote_function_f func, void *info) 132 { 133 struct remote_function_call data = { 134 .p = NULL, 135 .func = func, 136 .info = info, 137 .ret = -ENXIO, /* No such CPU */ 138 }; 139 140 smp_call_function_single(cpu, remote_function, &data, 1); 141 142 return data.ret; 143 } 144 145 static inline struct perf_cpu_context * 146 __get_cpu_context(struct perf_event_context *ctx) 147 { 148 return this_cpu_ptr(ctx->pmu->pmu_cpu_context); 149 } 150 151 static void perf_ctx_lock(struct perf_cpu_context *cpuctx, 152 struct perf_event_context *ctx) 153 { 154 raw_spin_lock(&cpuctx->ctx.lock); 155 if (ctx) 156 raw_spin_lock(&ctx->lock); 157 } 158 159 static void perf_ctx_unlock(struct perf_cpu_context *cpuctx, 160 struct perf_event_context *ctx) 161 { 162 if (ctx) 163 raw_spin_unlock(&ctx->lock); 164 raw_spin_unlock(&cpuctx->ctx.lock); 165 } 166 167 #define TASK_TOMBSTONE ((void *)-1L) 168 169 static bool is_kernel_event(struct perf_event *event) 170 { 171 return READ_ONCE(event->owner) == TASK_TOMBSTONE; 172 } 173 174 /* 175 * On task ctx scheduling... 176 * 177 * When !ctx->nr_events a task context will not be scheduled. This means 178 * we can disable the scheduler hooks (for performance) without leaving 179 * pending task ctx state. 180 * 181 * This however results in two special cases: 182 * 183 * - removing the last event from a task ctx; this is relatively straight 184 * forward and is done in __perf_remove_from_context. 185 * 186 * - adding the first event to a task ctx; this is tricky because we cannot 187 * rely on ctx->is_active and therefore cannot use event_function_call(). 188 * See perf_install_in_context(). 189 * 190 * If ctx->nr_events, then ctx->is_active and cpuctx->task_ctx are set. 191 */ 192 193 typedef void (*event_f)(struct perf_event *, struct perf_cpu_context *, 194 struct perf_event_context *, void *); 195 196 struct event_function_struct { 197 struct perf_event *event; 198 event_f func; 199 void *data; 200 }; 201 202 static int event_function(void *info) 203 { 204 struct event_function_struct *efs = info; 205 struct perf_event *event = efs->event; 206 struct perf_event_context *ctx = event->ctx; 207 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx); 208 struct perf_event_context *task_ctx = cpuctx->task_ctx; 209 int ret = 0; 210 211 lockdep_assert_irqs_disabled(); 212 213 perf_ctx_lock(cpuctx, task_ctx); 214 /* 215 * Since we do the IPI call without holding ctx->lock things can have 216 * changed, double check we hit the task we set out to hit. 217 */ 218 if (ctx->task) { 219 if (ctx->task != current) { 220 ret = -ESRCH; 221 goto unlock; 222 } 223 224 /* 225 * We only use event_function_call() on established contexts, 226 * and event_function() is only ever called when active (or 227 * rather, we'll have bailed in task_function_call() or the 228 * above ctx->task != current test), therefore we must have 229 * ctx->is_active here. 230 */ 231 WARN_ON_ONCE(!ctx->is_active); 232 /* 233 * And since we have ctx->is_active, cpuctx->task_ctx must 234 * match. 235 */ 236 WARN_ON_ONCE(task_ctx != ctx); 237 } else { 238 WARN_ON_ONCE(&cpuctx->ctx != ctx); 239 } 240 241 efs->func(event, cpuctx, ctx, efs->data); 242 unlock: 243 perf_ctx_unlock(cpuctx, task_ctx); 244 245 return ret; 246 } 247 248 static void event_function_call(struct perf_event *event, event_f func, void *data) 249 { 250 struct perf_event_context *ctx = event->ctx; 251 struct task_struct *task = READ_ONCE(ctx->task); /* verified in event_function */ 252 struct event_function_struct efs = { 253 .event = event, 254 .func = func, 255 .data = data, 256 }; 257 258 if (!event->parent) { 259 /* 260 * If this is a !child event, we must hold ctx::mutex to 261 * stabilize the the event->ctx relation. See 262 * perf_event_ctx_lock(). 263 */ 264 lockdep_assert_held(&ctx->mutex); 265 } 266 267 if (!task) { 268 cpu_function_call(event->cpu, event_function, &efs); 269 return; 270 } 271 272 if (task == TASK_TOMBSTONE) 273 return; 274 275 again: 276 if (!task_function_call(task, event_function, &efs)) 277 return; 278 279 raw_spin_lock_irq(&ctx->lock); 280 /* 281 * Reload the task pointer, it might have been changed by 282 * a concurrent perf_event_context_sched_out(). 283 */ 284 task = ctx->task; 285 if (task == TASK_TOMBSTONE) { 286 raw_spin_unlock_irq(&ctx->lock); 287 return; 288 } 289 if (ctx->is_active) { 290 raw_spin_unlock_irq(&ctx->lock); 291 goto again; 292 } 293 func(event, NULL, ctx, data); 294 raw_spin_unlock_irq(&ctx->lock); 295 } 296 297 /* 298 * Similar to event_function_call() + event_function(), but hard assumes IRQs 299 * are already disabled and we're on the right CPU. 300 */ 301 static void event_function_local(struct perf_event *event, event_f func, void *data) 302 { 303 struct perf_event_context *ctx = event->ctx; 304 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx); 305 struct task_struct *task = READ_ONCE(ctx->task); 306 struct perf_event_context *task_ctx = NULL; 307 308 lockdep_assert_irqs_disabled(); 309 310 if (task) { 311 if (task == TASK_TOMBSTONE) 312 return; 313 314 task_ctx = ctx; 315 } 316 317 perf_ctx_lock(cpuctx, task_ctx); 318 319 task = ctx->task; 320 if (task == TASK_TOMBSTONE) 321 goto unlock; 322 323 if (task) { 324 /* 325 * We must be either inactive or active and the right task, 326 * otherwise we're screwed, since we cannot IPI to somewhere 327 * else. 328 */ 329 if (ctx->is_active) { 330 if (WARN_ON_ONCE(task != current)) 331 goto unlock; 332 333 if (WARN_ON_ONCE(cpuctx->task_ctx != ctx)) 334 goto unlock; 335 } 336 } else { 337 WARN_ON_ONCE(&cpuctx->ctx != ctx); 338 } 339 340 func(event, cpuctx, ctx, data); 341 unlock: 342 perf_ctx_unlock(cpuctx, task_ctx); 343 } 344 345 #define PERF_FLAG_ALL (PERF_FLAG_FD_NO_GROUP |\ 346 PERF_FLAG_FD_OUTPUT |\ 347 PERF_FLAG_PID_CGROUP |\ 348 PERF_FLAG_FD_CLOEXEC) 349 350 /* 351 * branch priv levels that need permission checks 352 */ 353 #define PERF_SAMPLE_BRANCH_PERM_PLM \ 354 (PERF_SAMPLE_BRANCH_KERNEL |\ 355 PERF_SAMPLE_BRANCH_HV) 356 357 enum event_type_t { 358 EVENT_FLEXIBLE = 0x1, 359 EVENT_PINNED = 0x2, 360 EVENT_TIME = 0x4, 361 /* see ctx_resched() for details */ 362 EVENT_CPU = 0x8, 363 EVENT_ALL = EVENT_FLEXIBLE | EVENT_PINNED, 364 }; 365 366 /* 367 * perf_sched_events : >0 events exist 368 * perf_cgroup_events: >0 per-cpu cgroup events exist on this cpu 369 */ 370 371 static void perf_sched_delayed(struct work_struct *work); 372 DEFINE_STATIC_KEY_FALSE(perf_sched_events); 373 static DECLARE_DELAYED_WORK(perf_sched_work, perf_sched_delayed); 374 static DEFINE_MUTEX(perf_sched_mutex); 375 static atomic_t perf_sched_count; 376 377 static DEFINE_PER_CPU(atomic_t, perf_cgroup_events); 378 static DEFINE_PER_CPU(int, perf_sched_cb_usages); 379 static DEFINE_PER_CPU(struct pmu_event_list, pmu_sb_events); 380 381 static atomic_t nr_mmap_events __read_mostly; 382 static atomic_t nr_comm_events __read_mostly; 383 static atomic_t nr_namespaces_events __read_mostly; 384 static atomic_t nr_task_events __read_mostly; 385 static atomic_t nr_freq_events __read_mostly; 386 static atomic_t nr_switch_events __read_mostly; 387 static atomic_t nr_ksymbol_events __read_mostly; 388 static atomic_t nr_bpf_events __read_mostly; 389 390 static LIST_HEAD(pmus); 391 static DEFINE_MUTEX(pmus_lock); 392 static struct srcu_struct pmus_srcu; 393 static cpumask_var_t perf_online_mask; 394 395 /* 396 * perf event paranoia level: 397 * -1 - not paranoid at all 398 * 0 - disallow raw tracepoint access for unpriv 399 * 1 - disallow cpu events for unpriv 400 * 2 - disallow kernel profiling for unpriv 401 */ 402 int sysctl_perf_event_paranoid __read_mostly = 2; 403 404 /* Minimum for 512 kiB + 1 user control page */ 405 int sysctl_perf_event_mlock __read_mostly = 512 + (PAGE_SIZE / 1024); /* 'free' kiB per user */ 406 407 /* 408 * max perf event sample rate 409 */ 410 #define DEFAULT_MAX_SAMPLE_RATE 100000 411 #define DEFAULT_SAMPLE_PERIOD_NS (NSEC_PER_SEC / DEFAULT_MAX_SAMPLE_RATE) 412 #define DEFAULT_CPU_TIME_MAX_PERCENT 25 413 414 int sysctl_perf_event_sample_rate __read_mostly = DEFAULT_MAX_SAMPLE_RATE; 415 416 static int max_samples_per_tick __read_mostly = DIV_ROUND_UP(DEFAULT_MAX_SAMPLE_RATE, HZ); 417 static int perf_sample_period_ns __read_mostly = DEFAULT_SAMPLE_PERIOD_NS; 418 419 static int perf_sample_allowed_ns __read_mostly = 420 DEFAULT_SAMPLE_PERIOD_NS * DEFAULT_CPU_TIME_MAX_PERCENT / 100; 421 422 static void update_perf_cpu_limits(void) 423 { 424 u64 tmp = perf_sample_period_ns; 425 426 tmp *= sysctl_perf_cpu_time_max_percent; 427 tmp = div_u64(tmp, 100); 428 if (!tmp) 429 tmp = 1; 430 431 WRITE_ONCE(perf_sample_allowed_ns, tmp); 432 } 433 434 static bool perf_rotate_context(struct perf_cpu_context *cpuctx); 435 436 int perf_proc_update_handler(struct ctl_table *table, int write, 437 void __user *buffer, size_t *lenp, 438 loff_t *ppos) 439 { 440 int ret; 441 int perf_cpu = sysctl_perf_cpu_time_max_percent; 442 /* 443 * If throttling is disabled don't allow the write: 444 */ 445 if (write && (perf_cpu == 100 || perf_cpu == 0)) 446 return -EINVAL; 447 448 ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); 449 if (ret || !write) 450 return ret; 451 452 max_samples_per_tick = DIV_ROUND_UP(sysctl_perf_event_sample_rate, HZ); 453 perf_sample_period_ns = NSEC_PER_SEC / sysctl_perf_event_sample_rate; 454 update_perf_cpu_limits(); 455 456 return 0; 457 } 458 459 int sysctl_perf_cpu_time_max_percent __read_mostly = DEFAULT_CPU_TIME_MAX_PERCENT; 460 461 int perf_cpu_time_max_percent_handler(struct ctl_table *table, int write, 462 void __user *buffer, size_t *lenp, 463 loff_t *ppos) 464 { 465 int ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); 466 467 if (ret || !write) 468 return ret; 469 470 if (sysctl_perf_cpu_time_max_percent == 100 || 471 sysctl_perf_cpu_time_max_percent == 0) { 472 printk(KERN_WARNING 473 "perf: Dynamic interrupt throttling disabled, can hang your system!\n"); 474 WRITE_ONCE(perf_sample_allowed_ns, 0); 475 } else { 476 update_perf_cpu_limits(); 477 } 478 479 return 0; 480 } 481 482 /* 483 * perf samples are done in some very critical code paths (NMIs). 484 * If they take too much CPU time, the system can lock up and not 485 * get any real work done. This will drop the sample rate when 486 * we detect that events are taking too long. 487 */ 488 #define NR_ACCUMULATED_SAMPLES 128 489 static DEFINE_PER_CPU(u64, running_sample_length); 490 491 static u64 __report_avg; 492 static u64 __report_allowed; 493 494 static void perf_duration_warn(struct irq_work *w) 495 { 496 printk_ratelimited(KERN_INFO 497 "perf: interrupt took too long (%lld > %lld), lowering " 498 "kernel.perf_event_max_sample_rate to %d\n", 499 __report_avg, __report_allowed, 500 sysctl_perf_event_sample_rate); 501 } 502 503 static DEFINE_IRQ_WORK(perf_duration_work, perf_duration_warn); 504 505 void perf_sample_event_took(u64 sample_len_ns) 506 { 507 u64 max_len = READ_ONCE(perf_sample_allowed_ns); 508 u64 running_len; 509 u64 avg_len; 510 u32 max; 511 512 if (max_len == 0) 513 return; 514 515 /* Decay the counter by 1 average sample. */ 516 running_len = __this_cpu_read(running_sample_length); 517 running_len -= running_len/NR_ACCUMULATED_SAMPLES; 518 running_len += sample_len_ns; 519 __this_cpu_write(running_sample_length, running_len); 520 521 /* 522 * Note: this will be biased artifically low until we have 523 * seen NR_ACCUMULATED_SAMPLES. Doing it this way keeps us 524 * from having to maintain a count. 525 */ 526 avg_len = running_len/NR_ACCUMULATED_SAMPLES; 527 if (avg_len <= max_len) 528 return; 529 530 __report_avg = avg_len; 531 __report_allowed = max_len; 532 533 /* 534 * Compute a throttle threshold 25% below the current duration. 535 */ 536 avg_len += avg_len / 4; 537 max = (TICK_NSEC / 100) * sysctl_perf_cpu_time_max_percent; 538 if (avg_len < max) 539 max /= (u32)avg_len; 540 else 541 max = 1; 542 543 WRITE_ONCE(perf_sample_allowed_ns, avg_len); 544 WRITE_ONCE(max_samples_per_tick, max); 545 546 sysctl_perf_event_sample_rate = max * HZ; 547 perf_sample_period_ns = NSEC_PER_SEC / sysctl_perf_event_sample_rate; 548 549 if (!irq_work_queue(&perf_duration_work)) { 550 early_printk("perf: interrupt took too long (%lld > %lld), lowering " 551 "kernel.perf_event_max_sample_rate to %d\n", 552 __report_avg, __report_allowed, 553 sysctl_perf_event_sample_rate); 554 } 555 } 556 557 static atomic64_t perf_event_id; 558 559 static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx, 560 enum event_type_t event_type); 561 562 static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx, 563 enum event_type_t event_type, 564 struct task_struct *task); 565 566 static void update_context_time(struct perf_event_context *ctx); 567 static u64 perf_event_time(struct perf_event *event); 568 569 void __weak perf_event_print_debug(void) { } 570 571 extern __weak const char *perf_pmu_name(void) 572 { 573 return "pmu"; 574 } 575 576 static inline u64 perf_clock(void) 577 { 578 return local_clock(); 579 } 580 581 static inline u64 perf_event_clock(struct perf_event *event) 582 { 583 return event->clock(); 584 } 585 586 /* 587 * State based event timekeeping... 588 * 589 * The basic idea is to use event->state to determine which (if any) time 590 * fields to increment with the current delta. This means we only need to 591 * update timestamps when we change state or when they are explicitly requested 592 * (read). 593 * 594 * Event groups make things a little more complicated, but not terribly so. The 595 * rules for a group are that if the group leader is OFF the entire group is 596 * OFF, irrespecive of what the group member states are. This results in 597 * __perf_effective_state(). 598 * 599 * A futher ramification is that when a group leader flips between OFF and 600 * !OFF, we need to update all group member times. 601 * 602 * 603 * NOTE: perf_event_time() is based on the (cgroup) context time, and thus we 604 * need to make sure the relevant context time is updated before we try and 605 * update our timestamps. 606 */ 607 608 static __always_inline enum perf_event_state 609 __perf_effective_state(struct perf_event *event) 610 { 611 struct perf_event *leader = event->group_leader; 612 613 if (leader->state <= PERF_EVENT_STATE_OFF) 614 return leader->state; 615 616 return event->state; 617 } 618 619 static __always_inline void 620 __perf_update_times(struct perf_event *event, u64 now, u64 *enabled, u64 *running) 621 { 622 enum perf_event_state state = __perf_effective_state(event); 623 u64 delta = now - event->tstamp; 624 625 *enabled = event->total_time_enabled; 626 if (state >= PERF_EVENT_STATE_INACTIVE) 627 *enabled += delta; 628 629 *running = event->total_time_running; 630 if (state >= PERF_EVENT_STATE_ACTIVE) 631 *running += delta; 632 } 633 634 static void perf_event_update_time(struct perf_event *event) 635 { 636 u64 now = perf_event_time(event); 637 638 __perf_update_times(event, now, &event->total_time_enabled, 639 &event->total_time_running); 640 event->tstamp = now; 641 } 642 643 static void perf_event_update_sibling_time(struct perf_event *leader) 644 { 645 struct perf_event *sibling; 646 647 for_each_sibling_event(sibling, leader) 648 perf_event_update_time(sibling); 649 } 650 651 static void 652 perf_event_set_state(struct perf_event *event, enum perf_event_state state) 653 { 654 if (event->state == state) 655 return; 656 657 perf_event_update_time(event); 658 /* 659 * If a group leader gets enabled/disabled all its siblings 660 * are affected too. 661 */ 662 if ((event->state < 0) ^ (state < 0)) 663 perf_event_update_sibling_time(event); 664 665 WRITE_ONCE(event->state, state); 666 } 667 668 #ifdef CONFIG_CGROUP_PERF 669 670 static inline bool 671 perf_cgroup_match(struct perf_event *event) 672 { 673 struct perf_event_context *ctx = event->ctx; 674 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx); 675 676 /* @event doesn't care about cgroup */ 677 if (!event->cgrp) 678 return true; 679 680 /* wants specific cgroup scope but @cpuctx isn't associated with any */ 681 if (!cpuctx->cgrp) 682 return false; 683 684 /* 685 * Cgroup scoping is recursive. An event enabled for a cgroup is 686 * also enabled for all its descendant cgroups. If @cpuctx's 687 * cgroup is a descendant of @event's (the test covers identity 688 * case), it's a match. 689 */ 690 return cgroup_is_descendant(cpuctx->cgrp->css.cgroup, 691 event->cgrp->css.cgroup); 692 } 693 694 static inline void perf_detach_cgroup(struct perf_event *event) 695 { 696 css_put(&event->cgrp->css); 697 event->cgrp = NULL; 698 } 699 700 static inline int is_cgroup_event(struct perf_event *event) 701 { 702 return event->cgrp != NULL; 703 } 704 705 static inline u64 perf_cgroup_event_time(struct perf_event *event) 706 { 707 struct perf_cgroup_info *t; 708 709 t = per_cpu_ptr(event->cgrp->info, event->cpu); 710 return t->time; 711 } 712 713 static inline void __update_cgrp_time(struct perf_cgroup *cgrp) 714 { 715 struct perf_cgroup_info *info; 716 u64 now; 717 718 now = perf_clock(); 719 720 info = this_cpu_ptr(cgrp->info); 721 722 info->time += now - info->timestamp; 723 info->timestamp = now; 724 } 725 726 static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx) 727 { 728 struct perf_cgroup *cgrp = cpuctx->cgrp; 729 struct cgroup_subsys_state *css; 730 731 if (cgrp) { 732 for (css = &cgrp->css; css; css = css->parent) { 733 cgrp = container_of(css, struct perf_cgroup, css); 734 __update_cgrp_time(cgrp); 735 } 736 } 737 } 738 739 static inline void update_cgrp_time_from_event(struct perf_event *event) 740 { 741 struct perf_cgroup *cgrp; 742 743 /* 744 * ensure we access cgroup data only when needed and 745 * when we know the cgroup is pinned (css_get) 746 */ 747 if (!is_cgroup_event(event)) 748 return; 749 750 cgrp = perf_cgroup_from_task(current, event->ctx); 751 /* 752 * Do not update time when cgroup is not active 753 */ 754 if (cgroup_is_descendant(cgrp->css.cgroup, event->cgrp->css.cgroup)) 755 __update_cgrp_time(event->cgrp); 756 } 757 758 static inline void 759 perf_cgroup_set_timestamp(struct task_struct *task, 760 struct perf_event_context *ctx) 761 { 762 struct perf_cgroup *cgrp; 763 struct perf_cgroup_info *info; 764 struct cgroup_subsys_state *css; 765 766 /* 767 * ctx->lock held by caller 768 * ensure we do not access cgroup data 769 * unless we have the cgroup pinned (css_get) 770 */ 771 if (!task || !ctx->nr_cgroups) 772 return; 773 774 cgrp = perf_cgroup_from_task(task, ctx); 775 776 for (css = &cgrp->css; css; css = css->parent) { 777 cgrp = container_of(css, struct perf_cgroup, css); 778 info = this_cpu_ptr(cgrp->info); 779 info->timestamp = ctx->timestamp; 780 } 781 } 782 783 static DEFINE_PER_CPU(struct list_head, cgrp_cpuctx_list); 784 785 #define PERF_CGROUP_SWOUT 0x1 /* cgroup switch out every event */ 786 #define PERF_CGROUP_SWIN 0x2 /* cgroup switch in events based on task */ 787 788 /* 789 * reschedule events based on the cgroup constraint of task. 790 * 791 * mode SWOUT : schedule out everything 792 * mode SWIN : schedule in based on cgroup for next 793 */ 794 static void perf_cgroup_switch(struct task_struct *task, int mode) 795 { 796 struct perf_cpu_context *cpuctx; 797 struct list_head *list; 798 unsigned long flags; 799 800 /* 801 * Disable interrupts and preemption to avoid this CPU's 802 * cgrp_cpuctx_entry to change under us. 803 */ 804 local_irq_save(flags); 805 806 list = this_cpu_ptr(&cgrp_cpuctx_list); 807 list_for_each_entry(cpuctx, list, cgrp_cpuctx_entry) { 808 WARN_ON_ONCE(cpuctx->ctx.nr_cgroups == 0); 809 810 perf_ctx_lock(cpuctx, cpuctx->task_ctx); 811 perf_pmu_disable(cpuctx->ctx.pmu); 812 813 if (mode & PERF_CGROUP_SWOUT) { 814 cpu_ctx_sched_out(cpuctx, EVENT_ALL); 815 /* 816 * must not be done before ctxswout due 817 * to event_filter_match() in event_sched_out() 818 */ 819 cpuctx->cgrp = NULL; 820 } 821 822 if (mode & PERF_CGROUP_SWIN) { 823 WARN_ON_ONCE(cpuctx->cgrp); 824 /* 825 * set cgrp before ctxsw in to allow 826 * event_filter_match() to not have to pass 827 * task around 828 * we pass the cpuctx->ctx to perf_cgroup_from_task() 829 * because cgorup events are only per-cpu 830 */ 831 cpuctx->cgrp = perf_cgroup_from_task(task, 832 &cpuctx->ctx); 833 cpu_ctx_sched_in(cpuctx, EVENT_ALL, task); 834 } 835 perf_pmu_enable(cpuctx->ctx.pmu); 836 perf_ctx_unlock(cpuctx, cpuctx->task_ctx); 837 } 838 839 local_irq_restore(flags); 840 } 841 842 static inline void perf_cgroup_sched_out(struct task_struct *task, 843 struct task_struct *next) 844 { 845 struct perf_cgroup *cgrp1; 846 struct perf_cgroup *cgrp2 = NULL; 847 848 rcu_read_lock(); 849 /* 850 * we come here when we know perf_cgroup_events > 0 851 * we do not need to pass the ctx here because we know 852 * we are holding the rcu lock 853 */ 854 cgrp1 = perf_cgroup_from_task(task, NULL); 855 cgrp2 = perf_cgroup_from_task(next, NULL); 856 857 /* 858 * only schedule out current cgroup events if we know 859 * that we are switching to a different cgroup. Otherwise, 860 * do no touch the cgroup events. 861 */ 862 if (cgrp1 != cgrp2) 863 perf_cgroup_switch(task, PERF_CGROUP_SWOUT); 864 865 rcu_read_unlock(); 866 } 867 868 static inline void perf_cgroup_sched_in(struct task_struct *prev, 869 struct task_struct *task) 870 { 871 struct perf_cgroup *cgrp1; 872 struct perf_cgroup *cgrp2 = NULL; 873 874 rcu_read_lock(); 875 /* 876 * we come here when we know perf_cgroup_events > 0 877 * we do not need to pass the ctx here because we know 878 * we are holding the rcu lock 879 */ 880 cgrp1 = perf_cgroup_from_task(task, NULL); 881 cgrp2 = perf_cgroup_from_task(prev, NULL); 882 883 /* 884 * only need to schedule in cgroup events if we are changing 885 * cgroup during ctxsw. Cgroup events were not scheduled 886 * out of ctxsw out if that was not the case. 887 */ 888 if (cgrp1 != cgrp2) 889 perf_cgroup_switch(task, PERF_CGROUP_SWIN); 890 891 rcu_read_unlock(); 892 } 893 894 static inline int perf_cgroup_connect(int fd, struct perf_event *event, 895 struct perf_event_attr *attr, 896 struct perf_event *group_leader) 897 { 898 struct perf_cgroup *cgrp; 899 struct cgroup_subsys_state *css; 900 struct fd f = fdget(fd); 901 int ret = 0; 902 903 if (!f.file) 904 return -EBADF; 905 906 css = css_tryget_online_from_dir(f.file->f_path.dentry, 907 &perf_event_cgrp_subsys); 908 if (IS_ERR(css)) { 909 ret = PTR_ERR(css); 910 goto out; 911 } 912 913 cgrp = container_of(css, struct perf_cgroup, css); 914 event->cgrp = cgrp; 915 916 /* 917 * all events in a group must monitor 918 * the same cgroup because a task belongs 919 * to only one perf cgroup at a time 920 */ 921 if (group_leader && group_leader->cgrp != cgrp) { 922 perf_detach_cgroup(event); 923 ret = -EINVAL; 924 } 925 out: 926 fdput(f); 927 return ret; 928 } 929 930 static inline void 931 perf_cgroup_set_shadow_time(struct perf_event *event, u64 now) 932 { 933 struct perf_cgroup_info *t; 934 t = per_cpu_ptr(event->cgrp->info, event->cpu); 935 event->shadow_ctx_time = now - t->timestamp; 936 } 937 938 /* 939 * Update cpuctx->cgrp so that it is set when first cgroup event is added and 940 * cleared when last cgroup event is removed. 941 */ 942 static inline void 943 list_update_cgroup_event(struct perf_event *event, 944 struct perf_event_context *ctx, bool add) 945 { 946 struct perf_cpu_context *cpuctx; 947 struct list_head *cpuctx_entry; 948 949 if (!is_cgroup_event(event)) 950 return; 951 952 /* 953 * Because cgroup events are always per-cpu events, 954 * this will always be called from the right CPU. 955 */ 956 cpuctx = __get_cpu_context(ctx); 957 958 /* 959 * Since setting cpuctx->cgrp is conditional on the current @cgrp 960 * matching the event's cgroup, we must do this for every new event, 961 * because if the first would mismatch, the second would not try again 962 * and we would leave cpuctx->cgrp unset. 963 */ 964 if (add && !cpuctx->cgrp) { 965 struct perf_cgroup *cgrp = perf_cgroup_from_task(current, ctx); 966 967 if (cgroup_is_descendant(cgrp->css.cgroup, event->cgrp->css.cgroup)) 968 cpuctx->cgrp = cgrp; 969 } 970 971 if (add && ctx->nr_cgroups++) 972 return; 973 else if (!add && --ctx->nr_cgroups) 974 return; 975 976 /* no cgroup running */ 977 if (!add) 978 cpuctx->cgrp = NULL; 979 980 cpuctx_entry = &cpuctx->cgrp_cpuctx_entry; 981 if (add) 982 list_add(cpuctx_entry, this_cpu_ptr(&cgrp_cpuctx_list)); 983 else 984 list_del(cpuctx_entry); 985 } 986 987 #else /* !CONFIG_CGROUP_PERF */ 988 989 static inline bool 990 perf_cgroup_match(struct perf_event *event) 991 { 992 return true; 993 } 994 995 static inline void perf_detach_cgroup(struct perf_event *event) 996 {} 997 998 static inline int is_cgroup_event(struct perf_event *event) 999 { 1000 return 0; 1001 } 1002 1003 static inline void update_cgrp_time_from_event(struct perf_event *event) 1004 { 1005 } 1006 1007 static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx) 1008 { 1009 } 1010 1011 static inline void perf_cgroup_sched_out(struct task_struct *task, 1012 struct task_struct *next) 1013 { 1014 } 1015 1016 static inline void perf_cgroup_sched_in(struct task_struct *prev, 1017 struct task_struct *task) 1018 { 1019 } 1020 1021 static inline int perf_cgroup_connect(pid_t pid, struct perf_event *event, 1022 struct perf_event_attr *attr, 1023 struct perf_event *group_leader) 1024 { 1025 return -EINVAL; 1026 } 1027 1028 static inline void 1029 perf_cgroup_set_timestamp(struct task_struct *task, 1030 struct perf_event_context *ctx) 1031 { 1032 } 1033 1034 void 1035 perf_cgroup_switch(struct task_struct *task, struct task_struct *next) 1036 { 1037 } 1038 1039 static inline void 1040 perf_cgroup_set_shadow_time(struct perf_event *event, u64 now) 1041 { 1042 } 1043 1044 static inline u64 perf_cgroup_event_time(struct perf_event *event) 1045 { 1046 return 0; 1047 } 1048 1049 static inline void 1050 list_update_cgroup_event(struct perf_event *event, 1051 struct perf_event_context *ctx, bool add) 1052 { 1053 } 1054 1055 #endif 1056 1057 /* 1058 * set default to be dependent on timer tick just 1059 * like original code 1060 */ 1061 #define PERF_CPU_HRTIMER (1000 / HZ) 1062 /* 1063 * function must be called with interrupts disabled 1064 */ 1065 static enum hrtimer_restart perf_mux_hrtimer_handler(struct hrtimer *hr) 1066 { 1067 struct perf_cpu_context *cpuctx; 1068 bool rotations; 1069 1070 lockdep_assert_irqs_disabled(); 1071 1072 cpuctx = container_of(hr, struct perf_cpu_context, hrtimer); 1073 rotations = perf_rotate_context(cpuctx); 1074 1075 raw_spin_lock(&cpuctx->hrtimer_lock); 1076 if (rotations) 1077 hrtimer_forward_now(hr, cpuctx->hrtimer_interval); 1078 else 1079 cpuctx->hrtimer_active = 0; 1080 raw_spin_unlock(&cpuctx->hrtimer_lock); 1081 1082 return rotations ? HRTIMER_RESTART : HRTIMER_NORESTART; 1083 } 1084 1085 static void __perf_mux_hrtimer_init(struct perf_cpu_context *cpuctx, int cpu) 1086 { 1087 struct hrtimer *timer = &cpuctx->hrtimer; 1088 struct pmu *pmu = cpuctx->ctx.pmu; 1089 u64 interval; 1090 1091 /* no multiplexing needed for SW PMU */ 1092 if (pmu->task_ctx_nr == perf_sw_context) 1093 return; 1094 1095 /* 1096 * check default is sane, if not set then force to 1097 * default interval (1/tick) 1098 */ 1099 interval = pmu->hrtimer_interval_ms; 1100 if (interval < 1) 1101 interval = pmu->hrtimer_interval_ms = PERF_CPU_HRTIMER; 1102 1103 cpuctx->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * interval); 1104 1105 raw_spin_lock_init(&cpuctx->hrtimer_lock); 1106 hrtimer_init(timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED); 1107 timer->function = perf_mux_hrtimer_handler; 1108 } 1109 1110 static int perf_mux_hrtimer_restart(struct perf_cpu_context *cpuctx) 1111 { 1112 struct hrtimer *timer = &cpuctx->hrtimer; 1113 struct pmu *pmu = cpuctx->ctx.pmu; 1114 unsigned long flags; 1115 1116 /* not for SW PMU */ 1117 if (pmu->task_ctx_nr == perf_sw_context) 1118 return 0; 1119 1120 raw_spin_lock_irqsave(&cpuctx->hrtimer_lock, flags); 1121 if (!cpuctx->hrtimer_active) { 1122 cpuctx->hrtimer_active = 1; 1123 hrtimer_forward_now(timer, cpuctx->hrtimer_interval); 1124 hrtimer_start_expires(timer, HRTIMER_MODE_ABS_PINNED); 1125 } 1126 raw_spin_unlock_irqrestore(&cpuctx->hrtimer_lock, flags); 1127 1128 return 0; 1129 } 1130 1131 void perf_pmu_disable(struct pmu *pmu) 1132 { 1133 int *count = this_cpu_ptr(pmu->pmu_disable_count); 1134 if (!(*count)++) 1135 pmu->pmu_disable(pmu); 1136 } 1137 1138 void perf_pmu_enable(struct pmu *pmu) 1139 { 1140 int *count = this_cpu_ptr(pmu->pmu_disable_count); 1141 if (!--(*count)) 1142 pmu->pmu_enable(pmu); 1143 } 1144 1145 static DEFINE_PER_CPU(struct list_head, active_ctx_list); 1146 1147 /* 1148 * perf_event_ctx_activate(), perf_event_ctx_deactivate(), and 1149 * perf_event_task_tick() are fully serialized because they're strictly cpu 1150 * affine and perf_event_ctx{activate,deactivate} are called with IRQs 1151 * disabled, while perf_event_task_tick is called from IRQ context. 1152 */ 1153 static void perf_event_ctx_activate(struct perf_event_context *ctx) 1154 { 1155 struct list_head *head = this_cpu_ptr(&active_ctx_list); 1156 1157 lockdep_assert_irqs_disabled(); 1158 1159 WARN_ON(!list_empty(&ctx->active_ctx_list)); 1160 1161 list_add(&ctx->active_ctx_list, head); 1162 } 1163 1164 static void perf_event_ctx_deactivate(struct perf_event_context *ctx) 1165 { 1166 lockdep_assert_irqs_disabled(); 1167 1168 WARN_ON(list_empty(&ctx->active_ctx_list)); 1169 1170 list_del_init(&ctx->active_ctx_list); 1171 } 1172 1173 static void get_ctx(struct perf_event_context *ctx) 1174 { 1175 refcount_inc(&ctx->refcount); 1176 } 1177 1178 static void free_ctx(struct rcu_head *head) 1179 { 1180 struct perf_event_context *ctx; 1181 1182 ctx = container_of(head, struct perf_event_context, rcu_head); 1183 kfree(ctx->task_ctx_data); 1184 kfree(ctx); 1185 } 1186 1187 static void put_ctx(struct perf_event_context *ctx) 1188 { 1189 if (refcount_dec_and_test(&ctx->refcount)) { 1190 if (ctx->parent_ctx) 1191 put_ctx(ctx->parent_ctx); 1192 if (ctx->task && ctx->task != TASK_TOMBSTONE) 1193 put_task_struct(ctx->task); 1194 call_rcu(&ctx->rcu_head, free_ctx); 1195 } 1196 } 1197 1198 /* 1199 * Because of perf_event::ctx migration in sys_perf_event_open::move_group and 1200 * perf_pmu_migrate_context() we need some magic. 1201 * 1202 * Those places that change perf_event::ctx will hold both 1203 * perf_event_ctx::mutex of the 'old' and 'new' ctx value. 1204 * 1205 * Lock ordering is by mutex address. There are two other sites where 1206 * perf_event_context::mutex nests and those are: 1207 * 1208 * - perf_event_exit_task_context() [ child , 0 ] 1209 * perf_event_exit_event() 1210 * put_event() [ parent, 1 ] 1211 * 1212 * - perf_event_init_context() [ parent, 0 ] 1213 * inherit_task_group() 1214 * inherit_group() 1215 * inherit_event() 1216 * perf_event_alloc() 1217 * perf_init_event() 1218 * perf_try_init_event() [ child , 1 ] 1219 * 1220 * While it appears there is an obvious deadlock here -- the parent and child 1221 * nesting levels are inverted between the two. This is in fact safe because 1222 * life-time rules separate them. That is an exiting task cannot fork, and a 1223 * spawning task cannot (yet) exit. 1224 * 1225 * But remember that that these are parent<->child context relations, and 1226 * migration does not affect children, therefore these two orderings should not 1227 * interact. 1228 * 1229 * The change in perf_event::ctx does not affect children (as claimed above) 1230 * because the sys_perf_event_open() case will install a new event and break 1231 * the ctx parent<->child relation, and perf_pmu_migrate_context() is only 1232 * concerned with cpuctx and that doesn't have children. 1233 * 1234 * The places that change perf_event::ctx will issue: 1235 * 1236 * perf_remove_from_context(); 1237 * synchronize_rcu(); 1238 * perf_install_in_context(); 1239 * 1240 * to affect the change. The remove_from_context() + synchronize_rcu() should 1241 * quiesce the event, after which we can install it in the new location. This 1242 * means that only external vectors (perf_fops, prctl) can perturb the event 1243 * while in transit. Therefore all such accessors should also acquire 1244 * perf_event_context::mutex to serialize against this. 1245 * 1246 * However; because event->ctx can change while we're waiting to acquire 1247 * ctx->mutex we must be careful and use the below perf_event_ctx_lock() 1248 * function. 1249 * 1250 * Lock order: 1251 * cred_guard_mutex 1252 * task_struct::perf_event_mutex 1253 * perf_event_context::mutex 1254 * perf_event::child_mutex; 1255 * perf_event_context::lock 1256 * perf_event::mmap_mutex 1257 * mmap_sem 1258 * perf_addr_filters_head::lock 1259 * 1260 * cpu_hotplug_lock 1261 * pmus_lock 1262 * cpuctx->mutex / perf_event_context::mutex 1263 */ 1264 static struct perf_event_context * 1265 perf_event_ctx_lock_nested(struct perf_event *event, int nesting) 1266 { 1267 struct perf_event_context *ctx; 1268 1269 again: 1270 rcu_read_lock(); 1271 ctx = READ_ONCE(event->ctx); 1272 if (!refcount_inc_not_zero(&ctx->refcount)) { 1273 rcu_read_unlock(); 1274 goto again; 1275 } 1276 rcu_read_unlock(); 1277 1278 mutex_lock_nested(&ctx->mutex, nesting); 1279 if (event->ctx != ctx) { 1280 mutex_unlock(&ctx->mutex); 1281 put_ctx(ctx); 1282 goto again; 1283 } 1284 1285 return ctx; 1286 } 1287 1288 static inline struct perf_event_context * 1289 perf_event_ctx_lock(struct perf_event *event) 1290 { 1291 return perf_event_ctx_lock_nested(event, 0); 1292 } 1293 1294 static void perf_event_ctx_unlock(struct perf_event *event, 1295 struct perf_event_context *ctx) 1296 { 1297 mutex_unlock(&ctx->mutex); 1298 put_ctx(ctx); 1299 } 1300 1301 /* 1302 * This must be done under the ctx->lock, such as to serialize against 1303 * context_equiv(), therefore we cannot call put_ctx() since that might end up 1304 * calling scheduler related locks and ctx->lock nests inside those. 1305 */ 1306 static __must_check struct perf_event_context * 1307 unclone_ctx(struct perf_event_context *ctx) 1308 { 1309 struct perf_event_context *parent_ctx = ctx->parent_ctx; 1310 1311 lockdep_assert_held(&ctx->lock); 1312 1313 if (parent_ctx) 1314 ctx->parent_ctx = NULL; 1315 ctx->generation++; 1316 1317 return parent_ctx; 1318 } 1319 1320 static u32 perf_event_pid_type(struct perf_event *event, struct task_struct *p, 1321 enum pid_type type) 1322 { 1323 u32 nr; 1324 /* 1325 * only top level events have the pid namespace they were created in 1326 */ 1327 if (event->parent) 1328 event = event->parent; 1329 1330 nr = __task_pid_nr_ns(p, type, event->ns); 1331 /* avoid -1 if it is idle thread or runs in another ns */ 1332 if (!nr && !pid_alive(p)) 1333 nr = -1; 1334 return nr; 1335 } 1336 1337 static u32 perf_event_pid(struct perf_event *event, struct task_struct *p) 1338 { 1339 return perf_event_pid_type(event, p, PIDTYPE_TGID); 1340 } 1341 1342 static u32 perf_event_tid(struct perf_event *event, struct task_struct *p) 1343 { 1344 return perf_event_pid_type(event, p, PIDTYPE_PID); 1345 } 1346 1347 /* 1348 * If we inherit events we want to return the parent event id 1349 * to userspace. 1350 */ 1351 static u64 primary_event_id(struct perf_event *event) 1352 { 1353 u64 id = event->id; 1354 1355 if (event->parent) 1356 id = event->parent->id; 1357 1358 return id; 1359 } 1360 1361 /* 1362 * Get the perf_event_context for a task and lock it. 1363 * 1364 * This has to cope with with the fact that until it is locked, 1365 * the context could get moved to another task. 1366 */ 1367 static struct perf_event_context * 1368 perf_lock_task_context(struct task_struct *task, int ctxn, unsigned long *flags) 1369 { 1370 struct perf_event_context *ctx; 1371 1372 retry: 1373 /* 1374 * One of the few rules of preemptible RCU is that one cannot do 1375 * rcu_read_unlock() while holding a scheduler (or nested) lock when 1376 * part of the read side critical section was irqs-enabled -- see 1377 * rcu_read_unlock_special(). 1378 * 1379 * Since ctx->lock nests under rq->lock we must ensure the entire read 1380 * side critical section has interrupts disabled. 1381 */ 1382 local_irq_save(*flags); 1383 rcu_read_lock(); 1384 ctx = rcu_dereference(task->perf_event_ctxp[ctxn]); 1385 if (ctx) { 1386 /* 1387 * If this context is a clone of another, it might 1388 * get swapped for another underneath us by 1389 * perf_event_task_sched_out, though the 1390 * rcu_read_lock() protects us from any context 1391 * getting freed. Lock the context and check if it 1392 * got swapped before we could get the lock, and retry 1393 * if so. If we locked the right context, then it 1394 * can't get swapped on us any more. 1395 */ 1396 raw_spin_lock(&ctx->lock); 1397 if (ctx != rcu_dereference(task->perf_event_ctxp[ctxn])) { 1398 raw_spin_unlock(&ctx->lock); 1399 rcu_read_unlock(); 1400 local_irq_restore(*flags); 1401 goto retry; 1402 } 1403 1404 if (ctx->task == TASK_TOMBSTONE || 1405 !refcount_inc_not_zero(&ctx->refcount)) { 1406 raw_spin_unlock(&ctx->lock); 1407 ctx = NULL; 1408 } else { 1409 WARN_ON_ONCE(ctx->task != task); 1410 } 1411 } 1412 rcu_read_unlock(); 1413 if (!ctx) 1414 local_irq_restore(*flags); 1415 return ctx; 1416 } 1417 1418 /* 1419 * Get the context for a task and increment its pin_count so it 1420 * can't get swapped to another task. This also increments its 1421 * reference count so that the context can't get freed. 1422 */ 1423 static struct perf_event_context * 1424 perf_pin_task_context(struct task_struct *task, int ctxn) 1425 { 1426 struct perf_event_context *ctx; 1427 unsigned long flags; 1428 1429 ctx = perf_lock_task_context(task, ctxn, &flags); 1430 if (ctx) { 1431 ++ctx->pin_count; 1432 raw_spin_unlock_irqrestore(&ctx->lock, flags); 1433 } 1434 return ctx; 1435 } 1436 1437 static void perf_unpin_context(struct perf_event_context *ctx) 1438 { 1439 unsigned long flags; 1440 1441 raw_spin_lock_irqsave(&ctx->lock, flags); 1442 --ctx->pin_count; 1443 raw_spin_unlock_irqrestore(&ctx->lock, flags); 1444 } 1445 1446 /* 1447 * Update the record of the current time in a context. 1448 */ 1449 static void update_context_time(struct perf_event_context *ctx) 1450 { 1451 u64 now = perf_clock(); 1452 1453 ctx->time += now - ctx->timestamp; 1454 ctx->timestamp = now; 1455 } 1456 1457 static u64 perf_event_time(struct perf_event *event) 1458 { 1459 struct perf_event_context *ctx = event->ctx; 1460 1461 if (is_cgroup_event(event)) 1462 return perf_cgroup_event_time(event); 1463 1464 return ctx ? ctx->time : 0; 1465 } 1466 1467 static enum event_type_t get_event_type(struct perf_event *event) 1468 { 1469 struct perf_event_context *ctx = event->ctx; 1470 enum event_type_t event_type; 1471 1472 lockdep_assert_held(&ctx->lock); 1473 1474 /* 1475 * It's 'group type', really, because if our group leader is 1476 * pinned, so are we. 1477 */ 1478 if (event->group_leader != event) 1479 event = event->group_leader; 1480 1481 event_type = event->attr.pinned ? EVENT_PINNED : EVENT_FLEXIBLE; 1482 if (!ctx->task) 1483 event_type |= EVENT_CPU; 1484 1485 return event_type; 1486 } 1487 1488 /* 1489 * Helper function to initialize event group nodes. 1490 */ 1491 static void init_event_group(struct perf_event *event) 1492 { 1493 RB_CLEAR_NODE(&event->group_node); 1494 event->group_index = 0; 1495 } 1496 1497 /* 1498 * Extract pinned or flexible groups from the context 1499 * based on event attrs bits. 1500 */ 1501 static struct perf_event_groups * 1502 get_event_groups(struct perf_event *event, struct perf_event_context *ctx) 1503 { 1504 if (event->attr.pinned) 1505 return &ctx->pinned_groups; 1506 else 1507 return &ctx->flexible_groups; 1508 } 1509 1510 /* 1511 * Helper function to initializes perf_event_group trees. 1512 */ 1513 static void perf_event_groups_init(struct perf_event_groups *groups) 1514 { 1515 groups->tree = RB_ROOT; 1516 groups->index = 0; 1517 } 1518 1519 /* 1520 * Compare function for event groups; 1521 * 1522 * Implements complex key that first sorts by CPU and then by virtual index 1523 * which provides ordering when rotating groups for the same CPU. 1524 */ 1525 static bool 1526 perf_event_groups_less(struct perf_event *left, struct perf_event *right) 1527 { 1528 if (left->cpu < right->cpu) 1529 return true; 1530 if (left->cpu > right->cpu) 1531 return false; 1532 1533 if (left->group_index < right->group_index) 1534 return true; 1535 if (left->group_index > right->group_index) 1536 return false; 1537 1538 return false; 1539 } 1540 1541 /* 1542 * Insert @event into @groups' tree; using {@event->cpu, ++@groups->index} for 1543 * key (see perf_event_groups_less). This places it last inside the CPU 1544 * subtree. 1545 */ 1546 static void 1547 perf_event_groups_insert(struct perf_event_groups *groups, 1548 struct perf_event *event) 1549 { 1550 struct perf_event *node_event; 1551 struct rb_node *parent; 1552 struct rb_node **node; 1553 1554 event->group_index = ++groups->index; 1555 1556 node = &groups->tree.rb_node; 1557 parent = *node; 1558 1559 while (*node) { 1560 parent = *node; 1561 node_event = container_of(*node, struct perf_event, group_node); 1562 1563 if (perf_event_groups_less(event, node_event)) 1564 node = &parent->rb_left; 1565 else 1566 node = &parent->rb_right; 1567 } 1568 1569 rb_link_node(&event->group_node, parent, node); 1570 rb_insert_color(&event->group_node, &groups->tree); 1571 } 1572 1573 /* 1574 * Helper function to insert event into the pinned or flexible groups. 1575 */ 1576 static void 1577 add_event_to_groups(struct perf_event *event, struct perf_event_context *ctx) 1578 { 1579 struct perf_event_groups *groups; 1580 1581 groups = get_event_groups(event, ctx); 1582 perf_event_groups_insert(groups, event); 1583 } 1584 1585 /* 1586 * Delete a group from a tree. 1587 */ 1588 static void 1589 perf_event_groups_delete(struct perf_event_groups *groups, 1590 struct perf_event *event) 1591 { 1592 WARN_ON_ONCE(RB_EMPTY_NODE(&event->group_node) || 1593 RB_EMPTY_ROOT(&groups->tree)); 1594 1595 rb_erase(&event->group_node, &groups->tree); 1596 init_event_group(event); 1597 } 1598 1599 /* 1600 * Helper function to delete event from its groups. 1601 */ 1602 static void 1603 del_event_from_groups(struct perf_event *event, struct perf_event_context *ctx) 1604 { 1605 struct perf_event_groups *groups; 1606 1607 groups = get_event_groups(event, ctx); 1608 perf_event_groups_delete(groups, event); 1609 } 1610 1611 /* 1612 * Get the leftmost event in the @cpu subtree. 1613 */ 1614 static struct perf_event * 1615 perf_event_groups_first(struct perf_event_groups *groups, int cpu) 1616 { 1617 struct perf_event *node_event = NULL, *match = NULL; 1618 struct rb_node *node = groups->tree.rb_node; 1619 1620 while (node) { 1621 node_event = container_of(node, struct perf_event, group_node); 1622 1623 if (cpu < node_event->cpu) { 1624 node = node->rb_left; 1625 } else if (cpu > node_event->cpu) { 1626 node = node->rb_right; 1627 } else { 1628 match = node_event; 1629 node = node->rb_left; 1630 } 1631 } 1632 1633 return match; 1634 } 1635 1636 /* 1637 * Like rb_entry_next_safe() for the @cpu subtree. 1638 */ 1639 static struct perf_event * 1640 perf_event_groups_next(struct perf_event *event) 1641 { 1642 struct perf_event *next; 1643 1644 next = rb_entry_safe(rb_next(&event->group_node), typeof(*event), group_node); 1645 if (next && next->cpu == event->cpu) 1646 return next; 1647 1648 return NULL; 1649 } 1650 1651 /* 1652 * Iterate through the whole groups tree. 1653 */ 1654 #define perf_event_groups_for_each(event, groups) \ 1655 for (event = rb_entry_safe(rb_first(&((groups)->tree)), \ 1656 typeof(*event), group_node); event; \ 1657 event = rb_entry_safe(rb_next(&event->group_node), \ 1658 typeof(*event), group_node)) 1659 1660 /* 1661 * Add an event from the lists for its context. 1662 * Must be called with ctx->mutex and ctx->lock held. 1663 */ 1664 static void 1665 list_add_event(struct perf_event *event, struct perf_event_context *ctx) 1666 { 1667 lockdep_assert_held(&ctx->lock); 1668 1669 WARN_ON_ONCE(event->attach_state & PERF_ATTACH_CONTEXT); 1670 event->attach_state |= PERF_ATTACH_CONTEXT; 1671 1672 event->tstamp = perf_event_time(event); 1673 1674 /* 1675 * If we're a stand alone event or group leader, we go to the context 1676 * list, group events are kept attached to the group so that 1677 * perf_group_detach can, at all times, locate all siblings. 1678 */ 1679 if (event->group_leader == event) { 1680 event->group_caps = event->event_caps; 1681 add_event_to_groups(event, ctx); 1682 } 1683 1684 list_update_cgroup_event(event, ctx, true); 1685 1686 list_add_rcu(&event->event_entry, &ctx->event_list); 1687 ctx->nr_events++; 1688 if (event->attr.inherit_stat) 1689 ctx->nr_stat++; 1690 1691 ctx->generation++; 1692 } 1693 1694 /* 1695 * Initialize event state based on the perf_event_attr::disabled. 1696 */ 1697 static inline void perf_event__state_init(struct perf_event *event) 1698 { 1699 event->state = event->attr.disabled ? PERF_EVENT_STATE_OFF : 1700 PERF_EVENT_STATE_INACTIVE; 1701 } 1702 1703 static void __perf_event_read_size(struct perf_event *event, int nr_siblings) 1704 { 1705 int entry = sizeof(u64); /* value */ 1706 int size = 0; 1707 int nr = 1; 1708 1709 if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) 1710 size += sizeof(u64); 1711 1712 if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) 1713 size += sizeof(u64); 1714 1715 if (event->attr.read_format & PERF_FORMAT_ID) 1716 entry += sizeof(u64); 1717 1718 if (event->attr.read_format & PERF_FORMAT_GROUP) { 1719 nr += nr_siblings; 1720 size += sizeof(u64); 1721 } 1722 1723 size += entry * nr; 1724 event->read_size = size; 1725 } 1726 1727 static void __perf_event_header_size(struct perf_event *event, u64 sample_type) 1728 { 1729 struct perf_sample_data *data; 1730 u16 size = 0; 1731 1732 if (sample_type & PERF_SAMPLE_IP) 1733 size += sizeof(data->ip); 1734 1735 if (sample_type & PERF_SAMPLE_ADDR) 1736 size += sizeof(data->addr); 1737 1738 if (sample_type & PERF_SAMPLE_PERIOD) 1739 size += sizeof(data->period); 1740 1741 if (sample_type & PERF_SAMPLE_WEIGHT) 1742 size += sizeof(data->weight); 1743 1744 if (sample_type & PERF_SAMPLE_READ) 1745 size += event->read_size; 1746 1747 if (sample_type & PERF_SAMPLE_DATA_SRC) 1748 size += sizeof(data->data_src.val); 1749 1750 if (sample_type & PERF_SAMPLE_TRANSACTION) 1751 size += sizeof(data->txn); 1752 1753 if (sample_type & PERF_SAMPLE_PHYS_ADDR) 1754 size += sizeof(data->phys_addr); 1755 1756 event->header_size = size; 1757 } 1758 1759 /* 1760 * Called at perf_event creation and when events are attached/detached from a 1761 * group. 1762 */ 1763 static void perf_event__header_size(struct perf_event *event) 1764 { 1765 __perf_event_read_size(event, 1766 event->group_leader->nr_siblings); 1767 __perf_event_header_size(event, event->attr.sample_type); 1768 } 1769 1770 static void perf_event__id_header_size(struct perf_event *event) 1771 { 1772 struct perf_sample_data *data; 1773 u64 sample_type = event->attr.sample_type; 1774 u16 size = 0; 1775 1776 if (sample_type & PERF_SAMPLE_TID) 1777 size += sizeof(data->tid_entry); 1778 1779 if (sample_type & PERF_SAMPLE_TIME) 1780 size += sizeof(data->time); 1781 1782 if (sample_type & PERF_SAMPLE_IDENTIFIER) 1783 size += sizeof(data->id); 1784 1785 if (sample_type & PERF_SAMPLE_ID) 1786 size += sizeof(data->id); 1787 1788 if (sample_type & PERF_SAMPLE_STREAM_ID) 1789 size += sizeof(data->stream_id); 1790 1791 if (sample_type & PERF_SAMPLE_CPU) 1792 size += sizeof(data->cpu_entry); 1793 1794 event->id_header_size = size; 1795 } 1796 1797 static bool perf_event_validate_size(struct perf_event *event) 1798 { 1799 /* 1800 * The values computed here will be over-written when we actually 1801 * attach the event. 1802 */ 1803 __perf_event_read_size(event, event->group_leader->nr_siblings + 1); 1804 __perf_event_header_size(event, event->attr.sample_type & ~PERF_SAMPLE_READ); 1805 perf_event__id_header_size(event); 1806 1807 /* 1808 * Sum the lot; should not exceed the 64k limit we have on records. 1809 * Conservative limit to allow for callchains and other variable fields. 1810 */ 1811 if (event->read_size + event->header_size + 1812 event->id_header_size + sizeof(struct perf_event_header) >= 16*1024) 1813 return false; 1814 1815 return true; 1816 } 1817 1818 static void perf_group_attach(struct perf_event *event) 1819 { 1820 struct perf_event *group_leader = event->group_leader, *pos; 1821 1822 lockdep_assert_held(&event->ctx->lock); 1823 1824 /* 1825 * We can have double attach due to group movement in perf_event_open. 1826 */ 1827 if (event->attach_state & PERF_ATTACH_GROUP) 1828 return; 1829 1830 event->attach_state |= PERF_ATTACH_GROUP; 1831 1832 if (group_leader == event) 1833 return; 1834 1835 WARN_ON_ONCE(group_leader->ctx != event->ctx); 1836 1837 group_leader->group_caps &= event->event_caps; 1838 1839 list_add_tail(&event->sibling_list, &group_leader->sibling_list); 1840 group_leader->nr_siblings++; 1841 1842 perf_event__header_size(group_leader); 1843 1844 for_each_sibling_event(pos, group_leader) 1845 perf_event__header_size(pos); 1846 } 1847 1848 /* 1849 * Remove an event from the lists for its context. 1850 * Must be called with ctx->mutex and ctx->lock held. 1851 */ 1852 static void 1853 list_del_event(struct perf_event *event, struct perf_event_context *ctx) 1854 { 1855 WARN_ON_ONCE(event->ctx != ctx); 1856 lockdep_assert_held(&ctx->lock); 1857 1858 /* 1859 * We can have double detach due to exit/hot-unplug + close. 1860 */ 1861 if (!(event->attach_state & PERF_ATTACH_CONTEXT)) 1862 return; 1863 1864 event->attach_state &= ~PERF_ATTACH_CONTEXT; 1865 1866 list_update_cgroup_event(event, ctx, false); 1867 1868 ctx->nr_events--; 1869 if (event->attr.inherit_stat) 1870 ctx->nr_stat--; 1871 1872 list_del_rcu(&event->event_entry); 1873 1874 if (event->group_leader == event) 1875 del_event_from_groups(event, ctx); 1876 1877 /* 1878 * If event was in error state, then keep it 1879 * that way, otherwise bogus counts will be 1880 * returned on read(). The only way to get out 1881 * of error state is by explicit re-enabling 1882 * of the event 1883 */ 1884 if (event->state > PERF_EVENT_STATE_OFF) 1885 perf_event_set_state(event, PERF_EVENT_STATE_OFF); 1886 1887 ctx->generation++; 1888 } 1889 1890 static void perf_group_detach(struct perf_event *event) 1891 { 1892 struct perf_event *sibling, *tmp; 1893 struct perf_event_context *ctx = event->ctx; 1894 1895 lockdep_assert_held(&ctx->lock); 1896 1897 /* 1898 * We can have double detach due to exit/hot-unplug + close. 1899 */ 1900 if (!(event->attach_state & PERF_ATTACH_GROUP)) 1901 return; 1902 1903 event->attach_state &= ~PERF_ATTACH_GROUP; 1904 1905 /* 1906 * If this is a sibling, remove it from its group. 1907 */ 1908 if (event->group_leader != event) { 1909 list_del_init(&event->sibling_list); 1910 event->group_leader->nr_siblings--; 1911 goto out; 1912 } 1913 1914 /* 1915 * If this was a group event with sibling events then 1916 * upgrade the siblings to singleton events by adding them 1917 * to whatever list we are on. 1918 */ 1919 list_for_each_entry_safe(sibling, tmp, &event->sibling_list, sibling_list) { 1920 1921 sibling->group_leader = sibling; 1922 list_del_init(&sibling->sibling_list); 1923 1924 /* Inherit group flags from the previous leader */ 1925 sibling->group_caps = event->group_caps; 1926 1927 if (!RB_EMPTY_NODE(&event->group_node)) { 1928 add_event_to_groups(sibling, event->ctx); 1929 1930 if (sibling->state == PERF_EVENT_STATE_ACTIVE) { 1931 struct list_head *list = sibling->attr.pinned ? 1932 &ctx->pinned_active : &ctx->flexible_active; 1933 1934 list_add_tail(&sibling->active_list, list); 1935 } 1936 } 1937 1938 WARN_ON_ONCE(sibling->ctx != event->ctx); 1939 } 1940 1941 out: 1942 perf_event__header_size(event->group_leader); 1943 1944 for_each_sibling_event(tmp, event->group_leader) 1945 perf_event__header_size(tmp); 1946 } 1947 1948 static bool is_orphaned_event(struct perf_event *event) 1949 { 1950 return event->state == PERF_EVENT_STATE_DEAD; 1951 } 1952 1953 static inline int __pmu_filter_match(struct perf_event *event) 1954 { 1955 struct pmu *pmu = event->pmu; 1956 return pmu->filter_match ? pmu->filter_match(event) : 1; 1957 } 1958 1959 /* 1960 * Check whether we should attempt to schedule an event group based on 1961 * PMU-specific filtering. An event group can consist of HW and SW events, 1962 * potentially with a SW leader, so we must check all the filters, to 1963 * determine whether a group is schedulable: 1964 */ 1965 static inline int pmu_filter_match(struct perf_event *event) 1966 { 1967 struct perf_event *sibling; 1968 1969 if (!__pmu_filter_match(event)) 1970 return 0; 1971 1972 for_each_sibling_event(sibling, event) { 1973 if (!__pmu_filter_match(sibling)) 1974 return 0; 1975 } 1976 1977 return 1; 1978 } 1979 1980 static inline int 1981 event_filter_match(struct perf_event *event) 1982 { 1983 return (event->cpu == -1 || event->cpu == smp_processor_id()) && 1984 perf_cgroup_match(event) && pmu_filter_match(event); 1985 } 1986 1987 static void 1988 event_sched_out(struct perf_event *event, 1989 struct perf_cpu_context *cpuctx, 1990 struct perf_event_context *ctx) 1991 { 1992 enum perf_event_state state = PERF_EVENT_STATE_INACTIVE; 1993 1994 WARN_ON_ONCE(event->ctx != ctx); 1995 lockdep_assert_held(&ctx->lock); 1996 1997 if (event->state != PERF_EVENT_STATE_ACTIVE) 1998 return; 1999 2000 /* 2001 * Asymmetry; we only schedule events _IN_ through ctx_sched_in(), but 2002 * we can schedule events _OUT_ individually through things like 2003 * __perf_remove_from_context(). 2004 */ 2005 list_del_init(&event->active_list); 2006 2007 perf_pmu_disable(event->pmu); 2008 2009 event->pmu->del(event, 0); 2010 event->oncpu = -1; 2011 2012 if (event->pending_disable) { 2013 event->pending_disable = 0; 2014 state = PERF_EVENT_STATE_OFF; 2015 } 2016 perf_event_set_state(event, state); 2017 2018 if (!is_software_event(event)) 2019 cpuctx->active_oncpu--; 2020 if (!--ctx->nr_active) 2021 perf_event_ctx_deactivate(ctx); 2022 if (event->attr.freq && event->attr.sample_freq) 2023 ctx->nr_freq--; 2024 if (event->attr.exclusive || !cpuctx->active_oncpu) 2025 cpuctx->exclusive = 0; 2026 2027 perf_pmu_enable(event->pmu); 2028 } 2029 2030 static void 2031 group_sched_out(struct perf_event *group_event, 2032 struct perf_cpu_context *cpuctx, 2033 struct perf_event_context *ctx) 2034 { 2035 struct perf_event *event; 2036 2037 if (group_event->state != PERF_EVENT_STATE_ACTIVE) 2038 return; 2039 2040 perf_pmu_disable(ctx->pmu); 2041 2042 event_sched_out(group_event, cpuctx, ctx); 2043 2044 /* 2045 * Schedule out siblings (if any): 2046 */ 2047 for_each_sibling_event(event, group_event) 2048 event_sched_out(event, cpuctx, ctx); 2049 2050 perf_pmu_enable(ctx->pmu); 2051 2052 if (group_event->attr.exclusive) 2053 cpuctx->exclusive = 0; 2054 } 2055 2056 #define DETACH_GROUP 0x01UL 2057 2058 /* 2059 * Cross CPU call to remove a performance event 2060 * 2061 * We disable the event on the hardware level first. After that we 2062 * remove it from the context list. 2063 */ 2064 static void 2065 __perf_remove_from_context(struct perf_event *event, 2066 struct perf_cpu_context *cpuctx, 2067 struct perf_event_context *ctx, 2068 void *info) 2069 { 2070 unsigned long flags = (unsigned long)info; 2071 2072 if (ctx->is_active & EVENT_TIME) { 2073 update_context_time(ctx); 2074 update_cgrp_time_from_cpuctx(cpuctx); 2075 } 2076 2077 event_sched_out(event, cpuctx, ctx); 2078 if (flags & DETACH_GROUP) 2079 perf_group_detach(event); 2080 list_del_event(event, ctx); 2081 2082 if (!ctx->nr_events && ctx->is_active) { 2083 ctx->is_active = 0; 2084 if (ctx->task) { 2085 WARN_ON_ONCE(cpuctx->task_ctx != ctx); 2086 cpuctx->task_ctx = NULL; 2087 } 2088 } 2089 } 2090 2091 /* 2092 * Remove the event from a task's (or a CPU's) list of events. 2093 * 2094 * If event->ctx is a cloned context, callers must make sure that 2095 * every task struct that event->ctx->task could possibly point to 2096 * remains valid. This is OK when called from perf_release since 2097 * that only calls us on the top-level context, which can't be a clone. 2098 * When called from perf_event_exit_task, it's OK because the 2099 * context has been detached from its task. 2100 */ 2101 static void perf_remove_from_context(struct perf_event *event, unsigned long flags) 2102 { 2103 struct perf_event_context *ctx = event->ctx; 2104 2105 lockdep_assert_held(&ctx->mutex); 2106 2107 event_function_call(event, __perf_remove_from_context, (void *)flags); 2108 2109 /* 2110 * The above event_function_call() can NO-OP when it hits 2111 * TASK_TOMBSTONE. In that case we must already have been detached 2112 * from the context (by perf_event_exit_event()) but the grouping 2113 * might still be in-tact. 2114 */ 2115 WARN_ON_ONCE(event->attach_state & PERF_ATTACH_CONTEXT); 2116 if ((flags & DETACH_GROUP) && 2117 (event->attach_state & PERF_ATTACH_GROUP)) { 2118 /* 2119 * Since in that case we cannot possibly be scheduled, simply 2120 * detach now. 2121 */ 2122 raw_spin_lock_irq(&ctx->lock); 2123 perf_group_detach(event); 2124 raw_spin_unlock_irq(&ctx->lock); 2125 } 2126 } 2127 2128 /* 2129 * Cross CPU call to disable a performance event 2130 */ 2131 static void __perf_event_disable(struct perf_event *event, 2132 struct perf_cpu_context *cpuctx, 2133 struct perf_event_context *ctx, 2134 void *info) 2135 { 2136 if (event->state < PERF_EVENT_STATE_INACTIVE) 2137 return; 2138 2139 if (ctx->is_active & EVENT_TIME) { 2140 update_context_time(ctx); 2141 update_cgrp_time_from_event(event); 2142 } 2143 2144 if (event == event->group_leader) 2145 group_sched_out(event, cpuctx, ctx); 2146 else 2147 event_sched_out(event, cpuctx, ctx); 2148 2149 perf_event_set_state(event, PERF_EVENT_STATE_OFF); 2150 } 2151 2152 /* 2153 * Disable an event. 2154 * 2155 * If event->ctx is a cloned context, callers must make sure that 2156 * every task struct that event->ctx->task could possibly point to 2157 * remains valid. This condition is satisifed when called through 2158 * perf_event_for_each_child or perf_event_for_each because they 2159 * hold the top-level event's child_mutex, so any descendant that 2160 * goes to exit will block in perf_event_exit_event(). 2161 * 2162 * When called from perf_pending_event it's OK because event->ctx 2163 * is the current context on this CPU and preemption is disabled, 2164 * hence we can't get into perf_event_task_sched_out for this context. 2165 */ 2166 static void _perf_event_disable(struct perf_event *event) 2167 { 2168 struct perf_event_context *ctx = event->ctx; 2169 2170 raw_spin_lock_irq(&ctx->lock); 2171 if (event->state <= PERF_EVENT_STATE_OFF) { 2172 raw_spin_unlock_irq(&ctx->lock); 2173 return; 2174 } 2175 raw_spin_unlock_irq(&ctx->lock); 2176 2177 event_function_call(event, __perf_event_disable, NULL); 2178 } 2179 2180 void perf_event_disable_local(struct perf_event *event) 2181 { 2182 event_function_local(event, __perf_event_disable, NULL); 2183 } 2184 2185 /* 2186 * Strictly speaking kernel users cannot create groups and therefore this 2187 * interface does not need the perf_event_ctx_lock() magic. 2188 */ 2189 void perf_event_disable(struct perf_event *event) 2190 { 2191 struct perf_event_context *ctx; 2192 2193 ctx = perf_event_ctx_lock(event); 2194 _perf_event_disable(event); 2195 perf_event_ctx_unlock(event, ctx); 2196 } 2197 EXPORT_SYMBOL_GPL(perf_event_disable); 2198 2199 void perf_event_disable_inatomic(struct perf_event *event) 2200 { 2201 event->pending_disable = 1; 2202 irq_work_queue(&event->pending); 2203 } 2204 2205 static void perf_set_shadow_time(struct perf_event *event, 2206 struct perf_event_context *ctx) 2207 { 2208 /* 2209 * use the correct time source for the time snapshot 2210 * 2211 * We could get by without this by leveraging the 2212 * fact that to get to this function, the caller 2213 * has most likely already called update_context_time() 2214 * and update_cgrp_time_xx() and thus both timestamp 2215 * are identical (or very close). Given that tstamp is, 2216 * already adjusted for cgroup, we could say that: 2217 * tstamp - ctx->timestamp 2218 * is equivalent to 2219 * tstamp - cgrp->timestamp. 2220 * 2221 * Then, in perf_output_read(), the calculation would 2222 * work with no changes because: 2223 * - event is guaranteed scheduled in 2224 * - no scheduled out in between 2225 * - thus the timestamp would be the same 2226 * 2227 * But this is a bit hairy. 2228 * 2229 * So instead, we have an explicit cgroup call to remain 2230 * within the time time source all along. We believe it 2231 * is cleaner and simpler to understand. 2232 */ 2233 if (is_cgroup_event(event)) 2234 perf_cgroup_set_shadow_time(event, event->tstamp); 2235 else 2236 event->shadow_ctx_time = event->tstamp - ctx->timestamp; 2237 } 2238 2239 #define MAX_INTERRUPTS (~0ULL) 2240 2241 static void perf_log_throttle(struct perf_event *event, int enable); 2242 static void perf_log_itrace_start(struct perf_event *event); 2243 2244 static int 2245 event_sched_in(struct perf_event *event, 2246 struct perf_cpu_context *cpuctx, 2247 struct perf_event_context *ctx) 2248 { 2249 int ret = 0; 2250 2251 lockdep_assert_held(&ctx->lock); 2252 2253 if (event->state <= PERF_EVENT_STATE_OFF) 2254 return 0; 2255 2256 WRITE_ONCE(event->oncpu, smp_processor_id()); 2257 /* 2258 * Order event::oncpu write to happen before the ACTIVE state is 2259 * visible. This allows perf_event_{stop,read}() to observe the correct 2260 * ->oncpu if it sees ACTIVE. 2261 */ 2262 smp_wmb(); 2263 perf_event_set_state(event, PERF_EVENT_STATE_ACTIVE); 2264 2265 /* 2266 * Unthrottle events, since we scheduled we might have missed several 2267 * ticks already, also for a heavily scheduling task there is little 2268 * guarantee it'll get a tick in a timely manner. 2269 */ 2270 if (unlikely(event->hw.interrupts == MAX_INTERRUPTS)) { 2271 perf_log_throttle(event, 1); 2272 event->hw.interrupts = 0; 2273 } 2274 2275 perf_pmu_disable(event->pmu); 2276 2277 perf_set_shadow_time(event, ctx); 2278 2279 perf_log_itrace_start(event); 2280 2281 if (event->pmu->add(event, PERF_EF_START)) { 2282 perf_event_set_state(event, PERF_EVENT_STATE_INACTIVE); 2283 event->oncpu = -1; 2284 ret = -EAGAIN; 2285 goto out; 2286 } 2287 2288 if (!is_software_event(event)) 2289 cpuctx->active_oncpu++; 2290 if (!ctx->nr_active++) 2291 perf_event_ctx_activate(ctx); 2292 if (event->attr.freq && event->attr.sample_freq) 2293 ctx->nr_freq++; 2294 2295 if (event->attr.exclusive) 2296 cpuctx->exclusive = 1; 2297 2298 out: 2299 perf_pmu_enable(event->pmu); 2300 2301 return ret; 2302 } 2303 2304 static int 2305 group_sched_in(struct perf_event *group_event, 2306 struct perf_cpu_context *cpuctx, 2307 struct perf_event_context *ctx) 2308 { 2309 struct perf_event *event, *partial_group = NULL; 2310 struct pmu *pmu = ctx->pmu; 2311 2312 if (group_event->state == PERF_EVENT_STATE_OFF) 2313 return 0; 2314 2315 pmu->start_txn(pmu, PERF_PMU_TXN_ADD); 2316 2317 if (event_sched_in(group_event, cpuctx, ctx)) { 2318 pmu->cancel_txn(pmu); 2319 perf_mux_hrtimer_restart(cpuctx); 2320 return -EAGAIN; 2321 } 2322 2323 /* 2324 * Schedule in siblings as one group (if any): 2325 */ 2326 for_each_sibling_event(event, group_event) { 2327 if (event_sched_in(event, cpuctx, ctx)) { 2328 partial_group = event; 2329 goto group_error; 2330 } 2331 } 2332 2333 if (!pmu->commit_txn(pmu)) 2334 return 0; 2335 2336 group_error: 2337 /* 2338 * Groups can be scheduled in as one unit only, so undo any 2339 * partial group before returning: 2340 * The events up to the failed event are scheduled out normally. 2341 */ 2342 for_each_sibling_event(event, group_event) { 2343 if (event == partial_group) 2344 break; 2345 2346 event_sched_out(event, cpuctx, ctx); 2347 } 2348 event_sched_out(group_event, cpuctx, ctx); 2349 2350 pmu->cancel_txn(pmu); 2351 2352 perf_mux_hrtimer_restart(cpuctx); 2353 2354 return -EAGAIN; 2355 } 2356 2357 /* 2358 * Work out whether we can put this event group on the CPU now. 2359 */ 2360 static int group_can_go_on(struct perf_event *event, 2361 struct perf_cpu_context *cpuctx, 2362 int can_add_hw) 2363 { 2364 /* 2365 * Groups consisting entirely of software events can always go on. 2366 */ 2367 if (event->group_caps & PERF_EV_CAP_SOFTWARE) 2368 return 1; 2369 /* 2370 * If an exclusive group is already on, no other hardware 2371 * events can go on. 2372 */ 2373 if (cpuctx->exclusive) 2374 return 0; 2375 /* 2376 * If this group is exclusive and there are already 2377 * events on the CPU, it can't go on. 2378 */ 2379 if (event->attr.exclusive && cpuctx->active_oncpu) 2380 return 0; 2381 /* 2382 * Otherwise, try to add it if all previous groups were able 2383 * to go on. 2384 */ 2385 return can_add_hw; 2386 } 2387 2388 static void add_event_to_ctx(struct perf_event *event, 2389 struct perf_event_context *ctx) 2390 { 2391 list_add_event(event, ctx); 2392 perf_group_attach(event); 2393 } 2394 2395 static void ctx_sched_out(struct perf_event_context *ctx, 2396 struct perf_cpu_context *cpuctx, 2397 enum event_type_t event_type); 2398 static void 2399 ctx_sched_in(struct perf_event_context *ctx, 2400 struct perf_cpu_context *cpuctx, 2401 enum event_type_t event_type, 2402 struct task_struct *task); 2403 2404 static void task_ctx_sched_out(struct perf_cpu_context *cpuctx, 2405 struct perf_event_context *ctx, 2406 enum event_type_t event_type) 2407 { 2408 if (!cpuctx->task_ctx) 2409 return; 2410 2411 if (WARN_ON_ONCE(ctx != cpuctx->task_ctx)) 2412 return; 2413 2414 ctx_sched_out(ctx, cpuctx, event_type); 2415 } 2416 2417 static void perf_event_sched_in(struct perf_cpu_context *cpuctx, 2418 struct perf_event_context *ctx, 2419 struct task_struct *task) 2420 { 2421 cpu_ctx_sched_in(cpuctx, EVENT_PINNED, task); 2422 if (ctx) 2423 ctx_sched_in(ctx, cpuctx, EVENT_PINNED, task); 2424 cpu_ctx_sched_in(cpuctx, EVENT_FLEXIBLE, task); 2425 if (ctx) 2426 ctx_sched_in(ctx, cpuctx, EVENT_FLEXIBLE, task); 2427 } 2428 2429 /* 2430 * We want to maintain the following priority of scheduling: 2431 * - CPU pinned (EVENT_CPU | EVENT_PINNED) 2432 * - task pinned (EVENT_PINNED) 2433 * - CPU flexible (EVENT_CPU | EVENT_FLEXIBLE) 2434 * - task flexible (EVENT_FLEXIBLE). 2435 * 2436 * In order to avoid unscheduling and scheduling back in everything every 2437 * time an event is added, only do it for the groups of equal priority and 2438 * below. 2439 * 2440 * This can be called after a batch operation on task events, in which case 2441 * event_type is a bit mask of the types of events involved. For CPU events, 2442 * event_type is only either EVENT_PINNED or EVENT_FLEXIBLE. 2443 */ 2444 static void ctx_resched(struct perf_cpu_context *cpuctx, 2445 struct perf_event_context *task_ctx, 2446 enum event_type_t event_type) 2447 { 2448 enum event_type_t ctx_event_type; 2449 bool cpu_event = !!(event_type & EVENT_CPU); 2450 2451 /* 2452 * If pinned groups are involved, flexible groups also need to be 2453 * scheduled out. 2454 */ 2455 if (event_type & EVENT_PINNED) 2456 event_type |= EVENT_FLEXIBLE; 2457 2458 ctx_event_type = event_type & EVENT_ALL; 2459 2460 perf_pmu_disable(cpuctx->ctx.pmu); 2461 if (task_ctx) 2462 task_ctx_sched_out(cpuctx, task_ctx, event_type); 2463 2464 /* 2465 * Decide which cpu ctx groups to schedule out based on the types 2466 * of events that caused rescheduling: 2467 * - EVENT_CPU: schedule out corresponding groups; 2468 * - EVENT_PINNED task events: schedule out EVENT_FLEXIBLE groups; 2469 * - otherwise, do nothing more. 2470 */ 2471 if (cpu_event) 2472 cpu_ctx_sched_out(cpuctx, ctx_event_type); 2473 else if (ctx_event_type & EVENT_PINNED) 2474 cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE); 2475 2476 perf_event_sched_in(cpuctx, task_ctx, current); 2477 perf_pmu_enable(cpuctx->ctx.pmu); 2478 } 2479 2480 /* 2481 * Cross CPU call to install and enable a performance event 2482 * 2483 * Very similar to remote_function() + event_function() but cannot assume that 2484 * things like ctx->is_active and cpuctx->task_ctx are set. 2485 */ 2486 static int __perf_install_in_context(void *info) 2487 { 2488 struct perf_event *event = info; 2489 struct perf_event_context *ctx = event->ctx; 2490 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx); 2491 struct perf_event_context *task_ctx = cpuctx->task_ctx; 2492 bool reprogram = true; 2493 int ret = 0; 2494 2495 raw_spin_lock(&cpuctx->ctx.lock); 2496 if (ctx->task) { 2497 raw_spin_lock(&ctx->lock); 2498 task_ctx = ctx; 2499 2500 reprogram = (ctx->task == current); 2501 2502 /* 2503 * If the task is running, it must be running on this CPU, 2504 * otherwise we cannot reprogram things. 2505 * 2506 * If its not running, we don't care, ctx->lock will 2507 * serialize against it becoming runnable. 2508 */ 2509 if (task_curr(ctx->task) && !reprogram) { 2510 ret = -ESRCH; 2511 goto unlock; 2512 } 2513 2514 WARN_ON_ONCE(reprogram && cpuctx->task_ctx && cpuctx->task_ctx != ctx); 2515 } else if (task_ctx) { 2516 raw_spin_lock(&task_ctx->lock); 2517 } 2518 2519 #ifdef CONFIG_CGROUP_PERF 2520 if (is_cgroup_event(event)) { 2521 /* 2522 * If the current cgroup doesn't match the event's 2523 * cgroup, we should not try to schedule it. 2524 */ 2525 struct perf_cgroup *cgrp = perf_cgroup_from_task(current, ctx); 2526 reprogram = cgroup_is_descendant(cgrp->css.cgroup, 2527 event->cgrp->css.cgroup); 2528 } 2529 #endif 2530 2531 if (reprogram) { 2532 ctx_sched_out(ctx, cpuctx, EVENT_TIME); 2533 add_event_to_ctx(event, ctx); 2534 ctx_resched(cpuctx, task_ctx, get_event_type(event)); 2535 } else { 2536 add_event_to_ctx(event, ctx); 2537 } 2538 2539 unlock: 2540 perf_ctx_unlock(cpuctx, task_ctx); 2541 2542 return ret; 2543 } 2544 2545 /* 2546 * Attach a performance event to a context. 2547 * 2548 * Very similar to event_function_call, see comment there. 2549 */ 2550 static void 2551 perf_install_in_context(struct perf_event_context *ctx, 2552 struct perf_event *event, 2553 int cpu) 2554 { 2555 struct task_struct *task = READ_ONCE(ctx->task); 2556 2557 lockdep_assert_held(&ctx->mutex); 2558 2559 if (event->cpu != -1) 2560 event->cpu = cpu; 2561 2562 /* 2563 * Ensures that if we can observe event->ctx, both the event and ctx 2564 * will be 'complete'. See perf_iterate_sb_cpu(). 2565 */ 2566 smp_store_release(&event->ctx, ctx); 2567 2568 if (!task) { 2569 cpu_function_call(cpu, __perf_install_in_context, event); 2570 return; 2571 } 2572 2573 /* 2574 * Should not happen, we validate the ctx is still alive before calling. 2575 */ 2576 if (WARN_ON_ONCE(task == TASK_TOMBSTONE)) 2577 return; 2578 2579 /* 2580 * Installing events is tricky because we cannot rely on ctx->is_active 2581 * to be set in case this is the nr_events 0 -> 1 transition. 2582 * 2583 * Instead we use task_curr(), which tells us if the task is running. 2584 * However, since we use task_curr() outside of rq::lock, we can race 2585 * against the actual state. This means the result can be wrong. 2586 * 2587 * If we get a false positive, we retry, this is harmless. 2588 * 2589 * If we get a false negative, things are complicated. If we are after 2590 * perf_event_context_sched_in() ctx::lock will serialize us, and the 2591 * value must be correct. If we're before, it doesn't matter since 2592 * perf_event_context_sched_in() will program the counter. 2593 * 2594 * However, this hinges on the remote context switch having observed 2595 * our task->perf_event_ctxp[] store, such that it will in fact take 2596 * ctx::lock in perf_event_context_sched_in(). 2597 * 2598 * We do this by task_function_call(), if the IPI fails to hit the task 2599 * we know any future context switch of task must see the 2600 * perf_event_ctpx[] store. 2601 */ 2602 2603 /* 2604 * This smp_mb() orders the task->perf_event_ctxp[] store with the 2605 * task_cpu() load, such that if the IPI then does not find the task 2606 * running, a future context switch of that task must observe the 2607 * store. 2608 */ 2609 smp_mb(); 2610 again: 2611 if (!task_function_call(task, __perf_install_in_context, event)) 2612 return; 2613 2614 raw_spin_lock_irq(&ctx->lock); 2615 task = ctx->task; 2616 if (WARN_ON_ONCE(task == TASK_TOMBSTONE)) { 2617 /* 2618 * Cannot happen because we already checked above (which also 2619 * cannot happen), and we hold ctx->mutex, which serializes us 2620 * against perf_event_exit_task_context(). 2621 */ 2622 raw_spin_unlock_irq(&ctx->lock); 2623 return; 2624 } 2625 /* 2626 * If the task is not running, ctx->lock will avoid it becoming so, 2627 * thus we can safely install the event. 2628 */ 2629 if (task_curr(task)) { 2630 raw_spin_unlock_irq(&ctx->lock); 2631 goto again; 2632 } 2633 add_event_to_ctx(event, ctx); 2634 raw_spin_unlock_irq(&ctx->lock); 2635 } 2636 2637 /* 2638 * Cross CPU call to enable a performance event 2639 */ 2640 static void __perf_event_enable(struct perf_event *event, 2641 struct perf_cpu_context *cpuctx, 2642 struct perf_event_context *ctx, 2643 void *info) 2644 { 2645 struct perf_event *leader = event->group_leader; 2646 struct perf_event_context *task_ctx; 2647 2648 if (event->state >= PERF_EVENT_STATE_INACTIVE || 2649 event->state <= PERF_EVENT_STATE_ERROR) 2650 return; 2651 2652 if (ctx->is_active) 2653 ctx_sched_out(ctx, cpuctx, EVENT_TIME); 2654 2655 perf_event_set_state(event, PERF_EVENT_STATE_INACTIVE); 2656 2657 if (!ctx->is_active) 2658 return; 2659 2660 if (!event_filter_match(event)) { 2661 ctx_sched_in(ctx, cpuctx, EVENT_TIME, current); 2662 return; 2663 } 2664 2665 /* 2666 * If the event is in a group and isn't the group leader, 2667 * then don't put it on unless the group is on. 2668 */ 2669 if (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE) { 2670 ctx_sched_in(ctx, cpuctx, EVENT_TIME, current); 2671 return; 2672 } 2673 2674 task_ctx = cpuctx->task_ctx; 2675 if (ctx->task) 2676 WARN_ON_ONCE(task_ctx != ctx); 2677 2678 ctx_resched(cpuctx, task_ctx, get_event_type(event)); 2679 } 2680 2681 /* 2682 * Enable an event. 2683 * 2684 * If event->ctx is a cloned context, callers must make sure that 2685 * every task struct that event->ctx->task could possibly point to 2686 * remains valid. This condition is satisfied when called through 2687 * perf_event_for_each_child or perf_event_for_each as described 2688 * for perf_event_disable. 2689 */ 2690 static void _perf_event_enable(struct perf_event *event) 2691 { 2692 struct perf_event_context *ctx = event->ctx; 2693 2694 raw_spin_lock_irq(&ctx->lock); 2695 if (event->state >= PERF_EVENT_STATE_INACTIVE || 2696 event->state < PERF_EVENT_STATE_ERROR) { 2697 raw_spin_unlock_irq(&ctx->lock); 2698 return; 2699 } 2700 2701 /* 2702 * If the event is in error state, clear that first. 2703 * 2704 * That way, if we see the event in error state below, we know that it 2705 * has gone back into error state, as distinct from the task having 2706 * been scheduled away before the cross-call arrived. 2707 */ 2708 if (event->state == PERF_EVENT_STATE_ERROR) 2709 event->state = PERF_EVENT_STATE_OFF; 2710 raw_spin_unlock_irq(&ctx->lock); 2711 2712 event_function_call(event, __perf_event_enable, NULL); 2713 } 2714 2715 /* 2716 * See perf_event_disable(); 2717 */ 2718 void perf_event_enable(struct perf_event *event) 2719 { 2720 struct perf_event_context *ctx; 2721 2722 ctx = perf_event_ctx_lock(event); 2723 _perf_event_enable(event); 2724 perf_event_ctx_unlock(event, ctx); 2725 } 2726 EXPORT_SYMBOL_GPL(perf_event_enable); 2727 2728 struct stop_event_data { 2729 struct perf_event *event; 2730 unsigned int restart; 2731 }; 2732 2733 static int __perf_event_stop(void *info) 2734 { 2735 struct stop_event_data *sd = info; 2736 struct perf_event *event = sd->event; 2737 2738 /* if it's already INACTIVE, do nothing */ 2739 if (READ_ONCE(event->state) != PERF_EVENT_STATE_ACTIVE) 2740 return 0; 2741 2742 /* matches smp_wmb() in event_sched_in() */ 2743 smp_rmb(); 2744 2745 /* 2746 * There is a window with interrupts enabled before we get here, 2747 * so we need to check again lest we try to stop another CPU's event. 2748 */ 2749 if (READ_ONCE(event->oncpu) != smp_processor_id()) 2750 return -EAGAIN; 2751 2752 event->pmu->stop(event, PERF_EF_UPDATE); 2753 2754 /* 2755 * May race with the actual stop (through perf_pmu_output_stop()), 2756 * but it is only used for events with AUX ring buffer, and such 2757 * events will refuse to restart because of rb::aux_mmap_count==0, 2758 * see comments in perf_aux_output_begin(). 2759 * 2760 * Since this is happening on an event-local CPU, no trace is lost 2761 * while restarting. 2762 */ 2763 if (sd->restart) 2764 event->pmu->start(event, 0); 2765 2766 return 0; 2767 } 2768 2769 static int perf_event_stop(struct perf_event *event, int restart) 2770 { 2771 struct stop_event_data sd = { 2772 .event = event, 2773 .restart = restart, 2774 }; 2775 int ret = 0; 2776 2777 do { 2778 if (READ_ONCE(event->state) != PERF_EVENT_STATE_ACTIVE) 2779 return 0; 2780 2781 /* matches smp_wmb() in event_sched_in() */ 2782 smp_rmb(); 2783 2784 /* 2785 * We only want to restart ACTIVE events, so if the event goes 2786 * inactive here (event->oncpu==-1), there's nothing more to do; 2787 * fall through with ret==-ENXIO. 2788 */ 2789 ret = cpu_function_call(READ_ONCE(event->oncpu), 2790 __perf_event_stop, &sd); 2791 } while (ret == -EAGAIN); 2792 2793 return ret; 2794 } 2795 2796 /* 2797 * In order to contain the amount of racy and tricky in the address filter 2798 * configuration management, it is a two part process: 2799 * 2800 * (p1) when userspace mappings change as a result of (1) or (2) or (3) below, 2801 * we update the addresses of corresponding vmas in 2802 * event::addr_filter_ranges array and bump the event::addr_filters_gen; 2803 * (p2) when an event is scheduled in (pmu::add), it calls 2804 * perf_event_addr_filters_sync() which calls pmu::addr_filters_sync() 2805 * if the generation has changed since the previous call. 2806 * 2807 * If (p1) happens while the event is active, we restart it to force (p2). 2808 * 2809 * (1) perf_addr_filters_apply(): adjusting filters' offsets based on 2810 * pre-existing mappings, called once when new filters arrive via SET_FILTER 2811 * ioctl; 2812 * (2) perf_addr_filters_adjust(): adjusting filters' offsets based on newly 2813 * registered mapping, called for every new mmap(), with mm::mmap_sem down 2814 * for reading; 2815 * (3) perf_event_addr_filters_exec(): clearing filters' offsets in the process 2816 * of exec. 2817 */ 2818 void perf_event_addr_filters_sync(struct perf_event *event) 2819 { 2820 struct perf_addr_filters_head *ifh = perf_event_addr_filters(event); 2821 2822 if (!has_addr_filter(event)) 2823 return; 2824 2825 raw_spin_lock(&ifh->lock); 2826 if (event->addr_filters_gen != event->hw.addr_filters_gen) { 2827 event->pmu->addr_filters_sync(event); 2828 event->hw.addr_filters_gen = event->addr_filters_gen; 2829 } 2830 raw_spin_unlock(&ifh->lock); 2831 } 2832 EXPORT_SYMBOL_GPL(perf_event_addr_filters_sync); 2833 2834 static int _perf_event_refresh(struct perf_event *event, int refresh) 2835 { 2836 /* 2837 * not supported on inherited events 2838 */ 2839 if (event->attr.inherit || !is_sampling_event(event)) 2840 return -EINVAL; 2841 2842 atomic_add(refresh, &event->event_limit); 2843 _perf_event_enable(event); 2844 2845 return 0; 2846 } 2847 2848 /* 2849 * See perf_event_disable() 2850 */ 2851 int perf_event_refresh(struct perf_event *event, int refresh) 2852 { 2853 struct perf_event_context *ctx; 2854 int ret; 2855 2856 ctx = perf_event_ctx_lock(event); 2857 ret = _perf_event_refresh(event, refresh); 2858 perf_event_ctx_unlock(event, ctx); 2859 2860 return ret; 2861 } 2862 EXPORT_SYMBOL_GPL(perf_event_refresh); 2863 2864 static int perf_event_modify_breakpoint(struct perf_event *bp, 2865 struct perf_event_attr *attr) 2866 { 2867 int err; 2868 2869 _perf_event_disable(bp); 2870 2871 err = modify_user_hw_breakpoint_check(bp, attr, true); 2872 2873 if (!bp->attr.disabled) 2874 _perf_event_enable(bp); 2875 2876 return err; 2877 } 2878 2879 static int perf_event_modify_attr(struct perf_event *event, 2880 struct perf_event_attr *attr) 2881 { 2882 if (event->attr.type != attr->type) 2883 return -EINVAL; 2884 2885 switch (event->attr.type) { 2886 case PERF_TYPE_BREAKPOINT: 2887 return perf_event_modify_breakpoint(event, attr); 2888 default: 2889 /* Place holder for future additions. */ 2890 return -EOPNOTSUPP; 2891 } 2892 } 2893 2894 static void ctx_sched_out(struct perf_event_context *ctx, 2895 struct perf_cpu_context *cpuctx, 2896 enum event_type_t event_type) 2897 { 2898 struct perf_event *event, *tmp; 2899 int is_active = ctx->is_active; 2900 2901 lockdep_assert_held(&ctx->lock); 2902 2903 if (likely(!ctx->nr_events)) { 2904 /* 2905 * See __perf_remove_from_context(). 2906 */ 2907 WARN_ON_ONCE(ctx->is_active); 2908 if (ctx->task) 2909 WARN_ON_ONCE(cpuctx->task_ctx); 2910 return; 2911 } 2912 2913 ctx->is_active &= ~event_type; 2914 if (!(ctx->is_active & EVENT_ALL)) 2915 ctx->is_active = 0; 2916 2917 if (ctx->task) { 2918 WARN_ON_ONCE(cpuctx->task_ctx != ctx); 2919 if (!ctx->is_active) 2920 cpuctx->task_ctx = NULL; 2921 } 2922 2923 /* 2924 * Always update time if it was set; not only when it changes. 2925 * Otherwise we can 'forget' to update time for any but the last 2926 * context we sched out. For example: 2927 * 2928 * ctx_sched_out(.event_type = EVENT_FLEXIBLE) 2929 * ctx_sched_out(.event_type = EVENT_PINNED) 2930 * 2931 * would only update time for the pinned events. 2932 */ 2933 if (is_active & EVENT_TIME) { 2934 /* update (and stop) ctx time */ 2935 update_context_time(ctx); 2936 update_cgrp_time_from_cpuctx(cpuctx); 2937 } 2938 2939 is_active ^= ctx->is_active; /* changed bits */ 2940 2941 if (!ctx->nr_active || !(is_active & EVENT_ALL)) 2942 return; 2943 2944 perf_pmu_disable(ctx->pmu); 2945 if (is_active & EVENT_PINNED) { 2946 list_for_each_entry_safe(event, tmp, &ctx->pinned_active, active_list) 2947 group_sched_out(event, cpuctx, ctx); 2948 } 2949 2950 if (is_active & EVENT_FLEXIBLE) { 2951 list_for_each_entry_safe(event, tmp, &ctx->flexible_active, active_list) 2952 group_sched_out(event, cpuctx, ctx); 2953 } 2954 perf_pmu_enable(ctx->pmu); 2955 } 2956 2957 /* 2958 * Test whether two contexts are equivalent, i.e. whether they have both been 2959 * cloned from the same version of the same context. 2960 * 2961 * Equivalence is measured using a generation number in the context that is 2962 * incremented on each modification to it; see unclone_ctx(), list_add_event() 2963 * and list_del_event(). 2964 */ 2965 static int context_equiv(struct perf_event_context *ctx1, 2966 struct perf_event_context *ctx2) 2967 { 2968 lockdep_assert_held(&ctx1->lock); 2969 lockdep_assert_held(&ctx2->lock); 2970 2971 /* Pinning disables the swap optimization */ 2972 if (ctx1->pin_count || ctx2->pin_count) 2973 return 0; 2974 2975 /* If ctx1 is the parent of ctx2 */ 2976 if (ctx1 == ctx2->parent_ctx && ctx1->generation == ctx2->parent_gen) 2977 return 1; 2978 2979 /* If ctx2 is the parent of ctx1 */ 2980 if (ctx1->parent_ctx == ctx2 && ctx1->parent_gen == ctx2->generation) 2981 return 1; 2982 2983 /* 2984 * If ctx1 and ctx2 have the same parent; we flatten the parent 2985 * hierarchy, see perf_event_init_context(). 2986 */ 2987 if (ctx1->parent_ctx && ctx1->parent_ctx == ctx2->parent_ctx && 2988 ctx1->parent_gen == ctx2->parent_gen) 2989 return 1; 2990 2991 /* Unmatched */ 2992 return 0; 2993 } 2994 2995 static void __perf_event_sync_stat(struct perf_event *event, 2996 struct perf_event *next_event) 2997 { 2998 u64 value; 2999 3000 if (!event->attr.inherit_stat) 3001 return; 3002 3003 /* 3004 * Update the event value, we cannot use perf_event_read() 3005 * because we're in the middle of a context switch and have IRQs 3006 * disabled, which upsets smp_call_function_single(), however 3007 * we know the event must be on the current CPU, therefore we 3008 * don't need to use it. 3009 */ 3010 if (event->state == PERF_EVENT_STATE_ACTIVE) 3011 event->pmu->read(event); 3012 3013 perf_event_update_time(event); 3014 3015 /* 3016 * In order to keep per-task stats reliable we need to flip the event 3017 * values when we flip the contexts. 3018 */ 3019 value = local64_read(&next_event->count); 3020 value = local64_xchg(&event->count, value); 3021 local64_set(&next_event->count, value); 3022 3023 swap(event->total_time_enabled, next_event->total_time_enabled); 3024 swap(event->total_time_running, next_event->total_time_running); 3025 3026 /* 3027 * Since we swizzled the values, update the user visible data too. 3028 */ 3029 perf_event_update_userpage(event); 3030 perf_event_update_userpage(next_event); 3031 } 3032 3033 static void perf_event_sync_stat(struct perf_event_context *ctx, 3034 struct perf_event_context *next_ctx) 3035 { 3036 struct perf_event *event, *next_event; 3037 3038 if (!ctx->nr_stat) 3039 return; 3040 3041 update_context_time(ctx); 3042 3043 event = list_first_entry(&ctx->event_list, 3044 struct perf_event, event_entry); 3045 3046 next_event = list_first_entry(&next_ctx->event_list, 3047 struct perf_event, event_entry); 3048 3049 while (&event->event_entry != &ctx->event_list && 3050 &next_event->event_entry != &next_ctx->event_list) { 3051 3052 __perf_event_sync_stat(event, next_event); 3053 3054 event = list_next_entry(event, event_entry); 3055 next_event = list_next_entry(next_event, event_entry); 3056 } 3057 } 3058 3059 static void perf_event_context_sched_out(struct task_struct *task, int ctxn, 3060 struct task_struct *next) 3061 { 3062 struct perf_event_context *ctx = task->perf_event_ctxp[ctxn]; 3063 struct perf_event_context *next_ctx; 3064 struct perf_event_context *parent, *next_parent; 3065 struct perf_cpu_context *cpuctx; 3066 int do_switch = 1; 3067 3068 if (likely(!ctx)) 3069 return; 3070 3071 cpuctx = __get_cpu_context(ctx); 3072 if (!cpuctx->task_ctx) 3073 return; 3074 3075 rcu_read_lock(); 3076 next_ctx = next->perf_event_ctxp[ctxn]; 3077 if (!next_ctx) 3078 goto unlock; 3079 3080 parent = rcu_dereference(ctx->parent_ctx); 3081 next_parent = rcu_dereference(next_ctx->parent_ctx); 3082 3083 /* If neither context have a parent context; they cannot be clones. */ 3084 if (!parent && !next_parent) 3085 goto unlock; 3086 3087 if (next_parent == ctx || next_ctx == parent || next_parent == parent) { 3088 /* 3089 * Looks like the two contexts are clones, so we might be 3090 * able to optimize the context switch. We lock both 3091 * contexts and check that they are clones under the 3092 * lock (including re-checking that neither has been 3093 * uncloned in the meantime). It doesn't matter which 3094 * order we take the locks because no other cpu could 3095 * be trying to lock both of these tasks. 3096 */ 3097 raw_spin_lock(&ctx->lock); 3098 raw_spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING); 3099 if (context_equiv(ctx, next_ctx)) { 3100 WRITE_ONCE(ctx->task, next); 3101 WRITE_ONCE(next_ctx->task, task); 3102 3103 swap(ctx->task_ctx_data, next_ctx->task_ctx_data); 3104 3105 /* 3106 * RCU_INIT_POINTER here is safe because we've not 3107 * modified the ctx and the above modification of 3108 * ctx->task and ctx->task_ctx_data are immaterial 3109 * since those values are always verified under 3110 * ctx->lock which we're now holding. 3111 */ 3112 RCU_INIT_POINTER(task->perf_event_ctxp[ctxn], next_ctx); 3113 RCU_INIT_POINTER(next->perf_event_ctxp[ctxn], ctx); 3114 3115 do_switch = 0; 3116 3117 perf_event_sync_stat(ctx, next_ctx); 3118 } 3119 raw_spin_unlock(&next_ctx->lock); 3120 raw_spin_unlock(&ctx->lock); 3121 } 3122 unlock: 3123 rcu_read_unlock(); 3124 3125 if (do_switch) { 3126 raw_spin_lock(&ctx->lock); 3127 task_ctx_sched_out(cpuctx, ctx, EVENT_ALL); 3128 raw_spin_unlock(&ctx->lock); 3129 } 3130 } 3131 3132 static DEFINE_PER_CPU(struct list_head, sched_cb_list); 3133 3134 void perf_sched_cb_dec(struct pmu *pmu) 3135 { 3136 struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context); 3137 3138 this_cpu_dec(perf_sched_cb_usages); 3139 3140 if (!--cpuctx->sched_cb_usage) 3141 list_del(&cpuctx->sched_cb_entry); 3142 } 3143 3144 3145 void perf_sched_cb_inc(struct pmu *pmu) 3146 { 3147 struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context); 3148 3149 if (!cpuctx->sched_cb_usage++) 3150 list_add(&cpuctx->sched_cb_entry, this_cpu_ptr(&sched_cb_list)); 3151 3152 this_cpu_inc(perf_sched_cb_usages); 3153 } 3154 3155 /* 3156 * This function provides the context switch callback to the lower code 3157 * layer. It is invoked ONLY when the context switch callback is enabled. 3158 * 3159 * This callback is relevant even to per-cpu events; for example multi event 3160 * PEBS requires this to provide PID/TID information. This requires we flush 3161 * all queued PEBS records before we context switch to a new task. 3162 */ 3163 static void perf_pmu_sched_task(struct task_struct *prev, 3164 struct task_struct *next, 3165 bool sched_in) 3166 { 3167 struct perf_cpu_context *cpuctx; 3168 struct pmu *pmu; 3169 3170 if (prev == next) 3171 return; 3172 3173 list_for_each_entry(cpuctx, this_cpu_ptr(&sched_cb_list), sched_cb_entry) { 3174 pmu = cpuctx->ctx.pmu; /* software PMUs will not have sched_task */ 3175 3176 if (WARN_ON_ONCE(!pmu->sched_task)) 3177 continue; 3178 3179 perf_ctx_lock(cpuctx, cpuctx->task_ctx); 3180 perf_pmu_disable(pmu); 3181 3182 pmu->sched_task(cpuctx->task_ctx, sched_in); 3183 3184 perf_pmu_enable(pmu); 3185 perf_ctx_unlock(cpuctx, cpuctx->task_ctx); 3186 } 3187 } 3188 3189 static void perf_event_switch(struct task_struct *task, 3190 struct task_struct *next_prev, bool sched_in); 3191 3192 #define for_each_task_context_nr(ctxn) \ 3193 for ((ctxn) = 0; (ctxn) < perf_nr_task_contexts; (ctxn)++) 3194 3195 /* 3196 * Called from scheduler to remove the events of the current task, 3197 * with interrupts disabled. 3198 * 3199 * We stop each event and update the event value in event->count. 3200 * 3201 * This does not protect us against NMI, but disable() 3202 * sets the disabled bit in the control field of event _before_ 3203 * accessing the event control register. If a NMI hits, then it will 3204 * not restart the event. 3205 */ 3206 void __perf_event_task_sched_out(struct task_struct *task, 3207 struct task_struct *next) 3208 { 3209 int ctxn; 3210 3211 if (__this_cpu_read(perf_sched_cb_usages)) 3212 perf_pmu_sched_task(task, next, false); 3213 3214 if (atomic_read(&nr_switch_events)) 3215 perf_event_switch(task, next, false); 3216 3217 for_each_task_context_nr(ctxn) 3218 perf_event_context_sched_out(task, ctxn, next); 3219 3220 /* 3221 * if cgroup events exist on this CPU, then we need 3222 * to check if we have to switch out PMU state. 3223 * cgroup event are system-wide mode only 3224 */ 3225 if (atomic_read(this_cpu_ptr(&perf_cgroup_events))) 3226 perf_cgroup_sched_out(task, next); 3227 } 3228 3229 /* 3230 * Called with IRQs disabled 3231 */ 3232 static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx, 3233 enum event_type_t event_type) 3234 { 3235 ctx_sched_out(&cpuctx->ctx, cpuctx, event_type); 3236 } 3237 3238 static int visit_groups_merge(struct perf_event_groups *groups, int cpu, 3239 int (*func)(struct perf_event *, void *), void *data) 3240 { 3241 struct perf_event **evt, *evt1, *evt2; 3242 int ret; 3243 3244 evt1 = perf_event_groups_first(groups, -1); 3245 evt2 = perf_event_groups_first(groups, cpu); 3246 3247 while (evt1 || evt2) { 3248 if (evt1 && evt2) { 3249 if (evt1->group_index < evt2->group_index) 3250 evt = &evt1; 3251 else 3252 evt = &evt2; 3253 } else if (evt1) { 3254 evt = &evt1; 3255 } else { 3256 evt = &evt2; 3257 } 3258 3259 ret = func(*evt, data); 3260 if (ret) 3261 return ret; 3262 3263 *evt = perf_event_groups_next(*evt); 3264 } 3265 3266 return 0; 3267 } 3268 3269 struct sched_in_data { 3270 struct perf_event_context *ctx; 3271 struct perf_cpu_context *cpuctx; 3272 int can_add_hw; 3273 }; 3274 3275 static int pinned_sched_in(struct perf_event *event, void *data) 3276 { 3277 struct sched_in_data *sid = data; 3278 3279 if (event->state <= PERF_EVENT_STATE_OFF) 3280 return 0; 3281 3282 if (!event_filter_match(event)) 3283 return 0; 3284 3285 if (group_can_go_on(event, sid->cpuctx, sid->can_add_hw)) { 3286 if (!group_sched_in(event, sid->cpuctx, sid->ctx)) 3287 list_add_tail(&event->active_list, &sid->ctx->pinned_active); 3288 } 3289 3290 /* 3291 * If this pinned group hasn't been scheduled, 3292 * put it in error state. 3293 */ 3294 if (event->state == PERF_EVENT_STATE_INACTIVE) 3295 perf_event_set_state(event, PERF_EVENT_STATE_ERROR); 3296 3297 return 0; 3298 } 3299 3300 static int flexible_sched_in(struct perf_event *event, void *data) 3301 { 3302 struct sched_in_data *sid = data; 3303 3304 if (event->state <= PERF_EVENT_STATE_OFF) 3305 return 0; 3306 3307 if (!event_filter_match(event)) 3308 return 0; 3309 3310 if (group_can_go_on(event, sid->cpuctx, sid->can_add_hw)) { 3311 if (!group_sched_in(event, sid->cpuctx, sid->ctx)) 3312 list_add_tail(&event->active_list, &sid->ctx->flexible_active); 3313 else 3314 sid->can_add_hw = 0; 3315 } 3316 3317 return 0; 3318 } 3319 3320 static void 3321 ctx_pinned_sched_in(struct perf_event_context *ctx, 3322 struct perf_cpu_context *cpuctx) 3323 { 3324 struct sched_in_data sid = { 3325 .ctx = ctx, 3326 .cpuctx = cpuctx, 3327 .can_add_hw = 1, 3328 }; 3329 3330 visit_groups_merge(&ctx->pinned_groups, 3331 smp_processor_id(), 3332 pinned_sched_in, &sid); 3333 } 3334 3335 static void 3336 ctx_flexible_sched_in(struct perf_event_context *ctx, 3337 struct perf_cpu_context *cpuctx) 3338 { 3339 struct sched_in_data sid = { 3340 .ctx = ctx, 3341 .cpuctx = cpuctx, 3342 .can_add_hw = 1, 3343 }; 3344 3345 visit_groups_merge(&ctx->flexible_groups, 3346 smp_processor_id(), 3347 flexible_sched_in, &sid); 3348 } 3349 3350 static void 3351 ctx_sched_in(struct perf_event_context *ctx, 3352 struct perf_cpu_context *cpuctx, 3353 enum event_type_t event_type, 3354 struct task_struct *task) 3355 { 3356 int is_active = ctx->is_active; 3357 u64 now; 3358 3359 lockdep_assert_held(&ctx->lock); 3360 3361 if (likely(!ctx->nr_events)) 3362 return; 3363 3364 ctx->is_active |= (event_type | EVENT_TIME); 3365 if (ctx->task) { 3366 if (!is_active) 3367 cpuctx->task_ctx = ctx; 3368 else 3369 WARN_ON_ONCE(cpuctx->task_ctx != ctx); 3370 } 3371 3372 is_active ^= ctx->is_active; /* changed bits */ 3373 3374 if (is_active & EVENT_TIME) { 3375 /* start ctx time */ 3376 now = perf_clock(); 3377 ctx->timestamp = now; 3378 perf_cgroup_set_timestamp(task, ctx); 3379 } 3380 3381 /* 3382 * First go through the list and put on any pinned groups 3383 * in order to give them the best chance of going on. 3384 */ 3385 if (is_active & EVENT_PINNED) 3386 ctx_pinned_sched_in(ctx, cpuctx); 3387 3388 /* Then walk through the lower prio flexible groups */ 3389 if (is_active & EVENT_FLEXIBLE) 3390 ctx_flexible_sched_in(ctx, cpuctx); 3391 } 3392 3393 static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx, 3394 enum event_type_t event_type, 3395 struct task_struct *task) 3396 { 3397 struct perf_event_context *ctx = &cpuctx->ctx; 3398 3399 ctx_sched_in(ctx, cpuctx, event_type, task); 3400 } 3401 3402 static void perf_event_context_sched_in(struct perf_event_context *ctx, 3403 struct task_struct *task) 3404 { 3405 struct perf_cpu_context *cpuctx; 3406 3407 cpuctx = __get_cpu_context(ctx); 3408 if (cpuctx->task_ctx == ctx) 3409 return; 3410 3411 perf_ctx_lock(cpuctx, ctx); 3412 /* 3413 * We must check ctx->nr_events while holding ctx->lock, such 3414 * that we serialize against perf_install_in_context(). 3415 */ 3416 if (!ctx->nr_events) 3417 goto unlock; 3418 3419 perf_pmu_disable(ctx->pmu); 3420 /* 3421 * We want to keep the following priority order: 3422 * cpu pinned (that don't need to move), task pinned, 3423 * cpu flexible, task flexible. 3424 * 3425 * However, if task's ctx is not carrying any pinned 3426 * events, no need to flip the cpuctx's events around. 3427 */ 3428 if (!RB_EMPTY_ROOT(&ctx->pinned_groups.tree)) 3429 cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE); 3430 perf_event_sched_in(cpuctx, ctx, task); 3431 perf_pmu_enable(ctx->pmu); 3432 3433 unlock: 3434 perf_ctx_unlock(cpuctx, ctx); 3435 } 3436 3437 /* 3438 * Called from scheduler to add the events of the current task 3439 * with interrupts disabled. 3440 * 3441 * We restore the event value and then enable it. 3442 * 3443 * This does not protect us against NMI, but enable() 3444 * sets the enabled bit in the control field of event _before_ 3445 * accessing the event control register. If a NMI hits, then it will 3446 * keep the event running. 3447 */ 3448 void __perf_event_task_sched_in(struct task_struct *prev, 3449 struct task_struct *task) 3450 { 3451 struct perf_event_context *ctx; 3452 int ctxn; 3453 3454 /* 3455 * If cgroup events exist on this CPU, then we need to check if we have 3456 * to switch in PMU state; cgroup event are system-wide mode only. 3457 * 3458 * Since cgroup events are CPU events, we must schedule these in before 3459 * we schedule in the task events. 3460 */ 3461 if (atomic_read(this_cpu_ptr(&perf_cgroup_events))) 3462 perf_cgroup_sched_in(prev, task); 3463 3464 for_each_task_context_nr(ctxn) { 3465 ctx = task->perf_event_ctxp[ctxn]; 3466 if (likely(!ctx)) 3467 continue; 3468 3469 perf_event_context_sched_in(ctx, task); 3470 } 3471 3472 if (atomic_read(&nr_switch_events)) 3473 perf_event_switch(task, prev, true); 3474 3475 if (__this_cpu_read(perf_sched_cb_usages)) 3476 perf_pmu_sched_task(prev, task, true); 3477 } 3478 3479 static u64 perf_calculate_period(struct perf_event *event, u64 nsec, u64 count) 3480 { 3481 u64 frequency = event->attr.sample_freq; 3482 u64 sec = NSEC_PER_SEC; 3483 u64 divisor, dividend; 3484 3485 int count_fls, nsec_fls, frequency_fls, sec_fls; 3486 3487 count_fls = fls64(count); 3488 nsec_fls = fls64(nsec); 3489 frequency_fls = fls64(frequency); 3490 sec_fls = 30; 3491 3492 /* 3493 * We got @count in @nsec, with a target of sample_freq HZ 3494 * the target period becomes: 3495 * 3496 * @count * 10^9 3497 * period = ------------------- 3498 * @nsec * sample_freq 3499 * 3500 */ 3501 3502 /* 3503 * Reduce accuracy by one bit such that @a and @b converge 3504 * to a similar magnitude. 3505 */ 3506 #define REDUCE_FLS(a, b) \ 3507 do { \ 3508 if (a##_fls > b##_fls) { \ 3509 a >>= 1; \ 3510 a##_fls--; \ 3511 } else { \ 3512 b >>= 1; \ 3513 b##_fls--; \ 3514 } \ 3515 } while (0) 3516 3517 /* 3518 * Reduce accuracy until either term fits in a u64, then proceed with 3519 * the other, so that finally we can do a u64/u64 division. 3520 */ 3521 while (count_fls + sec_fls > 64 && nsec_fls + frequency_fls > 64) { 3522 REDUCE_FLS(nsec, frequency); 3523 REDUCE_FLS(sec, count); 3524 } 3525 3526 if (count_fls + sec_fls > 64) { 3527 divisor = nsec * frequency; 3528 3529 while (count_fls + sec_fls > 64) { 3530 REDUCE_FLS(count, sec); 3531 divisor >>= 1; 3532 } 3533 3534 dividend = count * sec; 3535 } else { 3536 dividend = count * sec; 3537 3538 while (nsec_fls + frequency_fls > 64) { 3539 REDUCE_FLS(nsec, frequency); 3540 dividend >>= 1; 3541 } 3542 3543 divisor = nsec * frequency; 3544 } 3545 3546 if (!divisor) 3547 return dividend; 3548 3549 return div64_u64(dividend, divisor); 3550 } 3551 3552 static DEFINE_PER_CPU(int, perf_throttled_count); 3553 static DEFINE_PER_CPU(u64, perf_throttled_seq); 3554 3555 static void perf_adjust_period(struct perf_event *event, u64 nsec, u64 count, bool disable) 3556 { 3557 struct hw_perf_event *hwc = &event->hw; 3558 s64 period, sample_period; 3559 s64 delta; 3560 3561 period = perf_calculate_period(event, nsec, count); 3562 3563 delta = (s64)(period - hwc->sample_period); 3564 delta = (delta + 7) / 8; /* low pass filter */ 3565 3566 sample_period = hwc->sample_period + delta; 3567 3568 if (!sample_period) 3569 sample_period = 1; 3570 3571 hwc->sample_period = sample_period; 3572 3573 if (local64_read(&hwc->period_left) > 8*sample_period) { 3574 if (disable) 3575 event->pmu->stop(event, PERF_EF_UPDATE); 3576 3577 local64_set(&hwc->period_left, 0); 3578 3579 if (disable) 3580 event->pmu->start(event, PERF_EF_RELOAD); 3581 } 3582 } 3583 3584 /* 3585 * combine freq adjustment with unthrottling to avoid two passes over the 3586 * events. At the same time, make sure, having freq events does not change 3587 * the rate of unthrottling as that would introduce bias. 3588 */ 3589 static void perf_adjust_freq_unthr_context(struct perf_event_context *ctx, 3590 int needs_unthr) 3591 { 3592 struct perf_event *event; 3593 struct hw_perf_event *hwc; 3594 u64 now, period = TICK_NSEC; 3595 s64 delta; 3596 3597 /* 3598 * only need to iterate over all events iff: 3599 * - context have events in frequency mode (needs freq adjust) 3600 * - there are events to unthrottle on this cpu 3601 */ 3602 if (!(ctx->nr_freq || needs_unthr)) 3603 return; 3604 3605 raw_spin_lock(&ctx->lock); 3606 perf_pmu_disable(ctx->pmu); 3607 3608 list_for_each_entry_rcu(event, &ctx->event_list, event_entry) { 3609 if (event->state != PERF_EVENT_STATE_ACTIVE) 3610 continue; 3611 3612 if (!event_filter_match(event)) 3613 continue; 3614 3615 perf_pmu_disable(event->pmu); 3616 3617 hwc = &event->hw; 3618 3619 if (hwc->interrupts == MAX_INTERRUPTS) { 3620 hwc->interrupts = 0; 3621 perf_log_throttle(event, 1); 3622 event->pmu->start(event, 0); 3623 } 3624 3625 if (!event->attr.freq || !event->attr.sample_freq) 3626 goto next; 3627 3628 /* 3629 * stop the event and update event->count 3630 */ 3631 event->pmu->stop(event, PERF_EF_UPDATE); 3632 3633 now = local64_read(&event->count); 3634 delta = now - hwc->freq_count_stamp; 3635 hwc->freq_count_stamp = now; 3636 3637 /* 3638 * restart the event 3639 * reload only if value has changed 3640 * we have stopped the event so tell that 3641 * to perf_adjust_period() to avoid stopping it 3642 * twice. 3643 */ 3644 if (delta > 0) 3645 perf_adjust_period(event, period, delta, false); 3646 3647 event->pmu->start(event, delta > 0 ? PERF_EF_RELOAD : 0); 3648 next: 3649 perf_pmu_enable(event->pmu); 3650 } 3651 3652 perf_pmu_enable(ctx->pmu); 3653 raw_spin_unlock(&ctx->lock); 3654 } 3655 3656 /* 3657 * Move @event to the tail of the @ctx's elegible events. 3658 */ 3659 static void rotate_ctx(struct perf_event_context *ctx, struct perf_event *event) 3660 { 3661 /* 3662 * Rotate the first entry last of non-pinned groups. Rotation might be 3663 * disabled by the inheritance code. 3664 */ 3665 if (ctx->rotate_disable) 3666 return; 3667 3668 perf_event_groups_delete(&ctx->flexible_groups, event); 3669 perf_event_groups_insert(&ctx->flexible_groups, event); 3670 } 3671 3672 static inline struct perf_event * 3673 ctx_first_active(struct perf_event_context *ctx) 3674 { 3675 return list_first_entry_or_null(&ctx->flexible_active, 3676 struct perf_event, active_list); 3677 } 3678 3679 static bool perf_rotate_context(struct perf_cpu_context *cpuctx) 3680 { 3681 struct perf_event *cpu_event = NULL, *task_event = NULL; 3682 bool cpu_rotate = false, task_rotate = false; 3683 struct perf_event_context *ctx = NULL; 3684 3685 /* 3686 * Since we run this from IRQ context, nobody can install new 3687 * events, thus the event count values are stable. 3688 */ 3689 3690 if (cpuctx->ctx.nr_events) { 3691 if (cpuctx->ctx.nr_events != cpuctx->ctx.nr_active) 3692 cpu_rotate = true; 3693 } 3694 3695 ctx = cpuctx->task_ctx; 3696 if (ctx && ctx->nr_events) { 3697 if (ctx->nr_events != ctx->nr_active) 3698 task_rotate = true; 3699 } 3700 3701 if (!(cpu_rotate || task_rotate)) 3702 return false; 3703 3704 perf_ctx_lock(cpuctx, cpuctx->task_ctx); 3705 perf_pmu_disable(cpuctx->ctx.pmu); 3706 3707 if (task_rotate) 3708 task_event = ctx_first_active(ctx); 3709 if (cpu_rotate) 3710 cpu_event = ctx_first_active(&cpuctx->ctx); 3711 3712 /* 3713 * As per the order given at ctx_resched() first 'pop' task flexible 3714 * and then, if needed CPU flexible. 3715 */ 3716 if (task_event || (ctx && cpu_event)) 3717 ctx_sched_out(ctx, cpuctx, EVENT_FLEXIBLE); 3718 if (cpu_event) 3719 cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE); 3720 3721 if (task_event) 3722 rotate_ctx(ctx, task_event); 3723 if (cpu_event) 3724 rotate_ctx(&cpuctx->ctx, cpu_event); 3725 3726 perf_event_sched_in(cpuctx, ctx, current); 3727 3728 perf_pmu_enable(cpuctx->ctx.pmu); 3729 perf_ctx_unlock(cpuctx, cpuctx->task_ctx); 3730 3731 return true; 3732 } 3733 3734 void perf_event_task_tick(void) 3735 { 3736 struct list_head *head = this_cpu_ptr(&active_ctx_list); 3737 struct perf_event_context *ctx, *tmp; 3738 int throttled; 3739 3740 lockdep_assert_irqs_disabled(); 3741 3742 __this_cpu_inc(perf_throttled_seq); 3743 throttled = __this_cpu_xchg(perf_throttled_count, 0); 3744 tick_dep_clear_cpu(smp_processor_id(), TICK_DEP_BIT_PERF_EVENTS); 3745 3746 list_for_each_entry_safe(ctx, tmp, head, active_ctx_list) 3747 perf_adjust_freq_unthr_context(ctx, throttled); 3748 } 3749 3750 static int event_enable_on_exec(struct perf_event *event, 3751 struct perf_event_context *ctx) 3752 { 3753 if (!event->attr.enable_on_exec) 3754 return 0; 3755 3756 event->attr.enable_on_exec = 0; 3757 if (event->state >= PERF_EVENT_STATE_INACTIVE) 3758 return 0; 3759 3760 perf_event_set_state(event, PERF_EVENT_STATE_INACTIVE); 3761 3762 return 1; 3763 } 3764 3765 /* 3766 * Enable all of a task's events that have been marked enable-on-exec. 3767 * This expects task == current. 3768 */ 3769 static void perf_event_enable_on_exec(int ctxn) 3770 { 3771 struct perf_event_context *ctx, *clone_ctx = NULL; 3772 enum event_type_t event_type = 0; 3773 struct perf_cpu_context *cpuctx; 3774 struct perf_event *event; 3775 unsigned long flags; 3776 int enabled = 0; 3777 3778 local_irq_save(flags); 3779 ctx = current->perf_event_ctxp[ctxn]; 3780 if (!ctx || !ctx->nr_events) 3781 goto out; 3782 3783 cpuctx = __get_cpu_context(ctx); 3784 perf_ctx_lock(cpuctx, ctx); 3785 ctx_sched_out(ctx, cpuctx, EVENT_TIME); 3786 list_for_each_entry(event, &ctx->event_list, event_entry) { 3787 enabled |= event_enable_on_exec(event, ctx); 3788 event_type |= get_event_type(event); 3789 } 3790 3791 /* 3792 * Unclone and reschedule this context if we enabled any event. 3793 */ 3794 if (enabled) { 3795 clone_ctx = unclone_ctx(ctx); 3796 ctx_resched(cpuctx, ctx, event_type); 3797 } else { 3798 ctx_sched_in(ctx, cpuctx, EVENT_TIME, current); 3799 } 3800 perf_ctx_unlock(cpuctx, ctx); 3801 3802 out: 3803 local_irq_restore(flags); 3804 3805 if (clone_ctx) 3806 put_ctx(clone_ctx); 3807 } 3808 3809 struct perf_read_data { 3810 struct perf_event *event; 3811 bool group; 3812 int ret; 3813 }; 3814 3815 static int __perf_event_read_cpu(struct perf_event *event, int event_cpu) 3816 { 3817 u16 local_pkg, event_pkg; 3818 3819 if (event->group_caps & PERF_EV_CAP_READ_ACTIVE_PKG) { 3820 int local_cpu = smp_processor_id(); 3821 3822 event_pkg = topology_physical_package_id(event_cpu); 3823 local_pkg = topology_physical_package_id(local_cpu); 3824 3825 if (event_pkg == local_pkg) 3826 return local_cpu; 3827 } 3828 3829 return event_cpu; 3830 } 3831 3832 /* 3833 * Cross CPU call to read the hardware event 3834 */ 3835 static void __perf_event_read(void *info) 3836 { 3837 struct perf_read_data *data = info; 3838 struct perf_event *sub, *event = data->event; 3839 struct perf_event_context *ctx = event->ctx; 3840 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx); 3841 struct pmu *pmu = event->pmu; 3842 3843 /* 3844 * If this is a task context, we need to check whether it is 3845 * the current task context of this cpu. If not it has been 3846 * scheduled out before the smp call arrived. In that case 3847 * event->count would have been updated to a recent sample 3848 * when the event was scheduled out. 3849 */ 3850 if (ctx->task && cpuctx->task_ctx != ctx) 3851 return; 3852 3853 raw_spin_lock(&ctx->lock); 3854 if (ctx->is_active & EVENT_TIME) { 3855 update_context_time(ctx); 3856 update_cgrp_time_from_event(event); 3857 } 3858 3859 perf_event_update_time(event); 3860 if (data->group) 3861 perf_event_update_sibling_time(event); 3862 3863 if (event->state != PERF_EVENT_STATE_ACTIVE) 3864 goto unlock; 3865 3866 if (!data->group) { 3867 pmu->read(event); 3868 data->ret = 0; 3869 goto unlock; 3870 } 3871 3872 pmu->start_txn(pmu, PERF_PMU_TXN_READ); 3873 3874 pmu->read(event); 3875 3876 for_each_sibling_event(sub, event) { 3877 if (sub->state == PERF_EVENT_STATE_ACTIVE) { 3878 /* 3879 * Use sibling's PMU rather than @event's since 3880 * sibling could be on different (eg: software) PMU. 3881 */ 3882 sub->pmu->read(sub); 3883 } 3884 } 3885 3886 data->ret = pmu->commit_txn(pmu); 3887 3888 unlock: 3889 raw_spin_unlock(&ctx->lock); 3890 } 3891 3892 static inline u64 perf_event_count(struct perf_event *event) 3893 { 3894 return local64_read(&event->count) + atomic64_read(&event->child_count); 3895 } 3896 3897 /* 3898 * NMI-safe method to read a local event, that is an event that 3899 * is: 3900 * - either for the current task, or for this CPU 3901 * - does not have inherit set, for inherited task events 3902 * will not be local and we cannot read them atomically 3903 * - must not have a pmu::count method 3904 */ 3905 int perf_event_read_local(struct perf_event *event, u64 *value, 3906 u64 *enabled, u64 *running) 3907 { 3908 unsigned long flags; 3909 int ret = 0; 3910 3911 /* 3912 * Disabling interrupts avoids all counter scheduling (context 3913 * switches, timer based rotation and IPIs). 3914 */ 3915 local_irq_save(flags); 3916 3917 /* 3918 * It must not be an event with inherit set, we cannot read 3919 * all child counters from atomic context. 3920 */ 3921 if (event->attr.inherit) { 3922 ret = -EOPNOTSUPP; 3923 goto out; 3924 } 3925 3926 /* If this is a per-task event, it must be for current */ 3927 if ((event->attach_state & PERF_ATTACH_TASK) && 3928 event->hw.target != current) { 3929 ret = -EINVAL; 3930 goto out; 3931 } 3932 3933 /* If this is a per-CPU event, it must be for this CPU */ 3934 if (!(event->attach_state & PERF_ATTACH_TASK) && 3935 event->cpu != smp_processor_id()) { 3936 ret = -EINVAL; 3937 goto out; 3938 } 3939 3940 /* If this is a pinned event it must be running on this CPU */ 3941 if (event->attr.pinned && event->oncpu != smp_processor_id()) { 3942 ret = -EBUSY; 3943 goto out; 3944 } 3945 3946 /* 3947 * If the event is currently on this CPU, its either a per-task event, 3948 * or local to this CPU. Furthermore it means its ACTIVE (otherwise 3949 * oncpu == -1). 3950 */ 3951 if (event->oncpu == smp_processor_id()) 3952 event->pmu->read(event); 3953 3954 *value = local64_read(&event->count); 3955 if (enabled || running) { 3956 u64 now = event->shadow_ctx_time + perf_clock(); 3957 u64 __enabled, __running; 3958 3959 __perf_update_times(event, now, &__enabled, &__running); 3960 if (enabled) 3961 *enabled = __enabled; 3962 if (running) 3963 *running = __running; 3964 } 3965 out: 3966 local_irq_restore(flags); 3967 3968 return ret; 3969 } 3970 3971 static int perf_event_read(struct perf_event *event, bool group) 3972 { 3973 enum perf_event_state state = READ_ONCE(event->state); 3974 int event_cpu, ret = 0; 3975 3976 /* 3977 * If event is enabled and currently active on a CPU, update the 3978 * value in the event structure: 3979 */ 3980 again: 3981 if (state == PERF_EVENT_STATE_ACTIVE) { 3982 struct perf_read_data data; 3983 3984 /* 3985 * Orders the ->state and ->oncpu loads such that if we see 3986 * ACTIVE we must also see the right ->oncpu. 3987 * 3988 * Matches the smp_wmb() from event_sched_in(). 3989 */ 3990 smp_rmb(); 3991 3992 event_cpu = READ_ONCE(event->oncpu); 3993 if ((unsigned)event_cpu >= nr_cpu_ids) 3994 return 0; 3995 3996 data = (struct perf_read_data){ 3997 .event = event, 3998 .group = group, 3999 .ret = 0, 4000 }; 4001 4002 preempt_disable(); 4003 event_cpu = __perf_event_read_cpu(event, event_cpu); 4004 4005 /* 4006 * Purposely ignore the smp_call_function_single() return 4007 * value. 4008 * 4009 * If event_cpu isn't a valid CPU it means the event got 4010 * scheduled out and that will have updated the event count. 4011 * 4012 * Therefore, either way, we'll have an up-to-date event count 4013 * after this. 4014 */ 4015 (void)smp_call_function_single(event_cpu, __perf_event_read, &data, 1); 4016 preempt_enable(); 4017 ret = data.ret; 4018 4019 } else if (state == PERF_EVENT_STATE_INACTIVE) { 4020 struct perf_event_context *ctx = event->ctx; 4021 unsigned long flags; 4022 4023 raw_spin_lock_irqsave(&ctx->lock, flags); 4024 state = event->state; 4025 if (state != PERF_EVENT_STATE_INACTIVE) { 4026 raw_spin_unlock_irqrestore(&ctx->lock, flags); 4027 goto again; 4028 } 4029 4030 /* 4031 * May read while context is not active (e.g., thread is 4032 * blocked), in that case we cannot update context time 4033 */ 4034 if (ctx->is_active & EVENT_TIME) { 4035 update_context_time(ctx); 4036 update_cgrp_time_from_event(event); 4037 } 4038 4039 perf_event_update_time(event); 4040 if (group) 4041 perf_event_update_sibling_time(event); 4042 raw_spin_unlock_irqrestore(&ctx->lock, flags); 4043 } 4044 4045 return ret; 4046 } 4047 4048 /* 4049 * Initialize the perf_event context in a task_struct: 4050 */ 4051 static void __perf_event_init_context(struct perf_event_context *ctx) 4052 { 4053 raw_spin_lock_init(&ctx->lock); 4054 mutex_init(&ctx->mutex); 4055 INIT_LIST_HEAD(&ctx->active_ctx_list); 4056 perf_event_groups_init(&ctx->pinned_groups); 4057 perf_event_groups_init(&ctx->flexible_groups); 4058 INIT_LIST_HEAD(&ctx->event_list); 4059 INIT_LIST_HEAD(&ctx->pinned_active); 4060 INIT_LIST_HEAD(&ctx->flexible_active); 4061 refcount_set(&ctx->refcount, 1); 4062 } 4063 4064 static struct perf_event_context * 4065 alloc_perf_context(struct pmu *pmu, struct task_struct *task) 4066 { 4067 struct perf_event_context *ctx; 4068 4069 ctx = kzalloc(sizeof(struct perf_event_context), GFP_KERNEL); 4070 if (!ctx) 4071 return NULL; 4072 4073 __perf_event_init_context(ctx); 4074 if (task) { 4075 ctx->task = task; 4076 get_task_struct(task); 4077 } 4078 ctx->pmu = pmu; 4079 4080 return ctx; 4081 } 4082 4083 static struct task_struct * 4084 find_lively_task_by_vpid(pid_t vpid) 4085 { 4086 struct task_struct *task; 4087 4088 rcu_read_lock(); 4089 if (!vpid) 4090 task = current; 4091 else 4092 task = find_task_by_vpid(vpid); 4093 if (task) 4094 get_task_struct(task); 4095 rcu_read_unlock(); 4096 4097 if (!task) 4098 return ERR_PTR(-ESRCH); 4099 4100 return task; 4101 } 4102 4103 /* 4104 * Returns a matching context with refcount and pincount. 4105 */ 4106 static struct perf_event_context * 4107 find_get_context(struct pmu *pmu, struct task_struct *task, 4108 struct perf_event *event) 4109 { 4110 struct perf_event_context *ctx, *clone_ctx = NULL; 4111 struct perf_cpu_context *cpuctx; 4112 void *task_ctx_data = NULL; 4113 unsigned long flags; 4114 int ctxn, err; 4115 int cpu = event->cpu; 4116 4117 if (!task) { 4118 /* Must be root to operate on a CPU event: */ 4119 if (perf_paranoid_cpu() && !capable(CAP_SYS_ADMIN)) 4120 return ERR_PTR(-EACCES); 4121 4122 cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu); 4123 ctx = &cpuctx->ctx; 4124 get_ctx(ctx); 4125 ++ctx->pin_count; 4126 4127 return ctx; 4128 } 4129 4130 err = -EINVAL; 4131 ctxn = pmu->task_ctx_nr; 4132 if (ctxn < 0) 4133 goto errout; 4134 4135 if (event->attach_state & PERF_ATTACH_TASK_DATA) { 4136 task_ctx_data = kzalloc(pmu->task_ctx_size, GFP_KERNEL); 4137 if (!task_ctx_data) { 4138 err = -ENOMEM; 4139 goto errout; 4140 } 4141 } 4142 4143 retry: 4144 ctx = perf_lock_task_context(task, ctxn, &flags); 4145 if (ctx) { 4146 clone_ctx = unclone_ctx(ctx); 4147 ++ctx->pin_count; 4148 4149 if (task_ctx_data && !ctx->task_ctx_data) { 4150 ctx->task_ctx_data = task_ctx_data; 4151 task_ctx_data = NULL; 4152 } 4153 raw_spin_unlock_irqrestore(&ctx->lock, flags); 4154 4155 if (clone_ctx) 4156 put_ctx(clone_ctx); 4157 } else { 4158 ctx = alloc_perf_context(pmu, task); 4159 err = -ENOMEM; 4160 if (!ctx) 4161 goto errout; 4162 4163 if (task_ctx_data) { 4164 ctx->task_ctx_data = task_ctx_data; 4165 task_ctx_data = NULL; 4166 } 4167 4168 err = 0; 4169 mutex_lock(&task->perf_event_mutex); 4170 /* 4171 * If it has already passed perf_event_exit_task(). 4172 * we must see PF_EXITING, it takes this mutex too. 4173 */ 4174 if (task->flags & PF_EXITING) 4175 err = -ESRCH; 4176 else if (task->perf_event_ctxp[ctxn]) 4177 err = -EAGAIN; 4178 else { 4179 get_ctx(ctx); 4180 ++ctx->pin_count; 4181 rcu_assign_pointer(task->perf_event_ctxp[ctxn], ctx); 4182 } 4183 mutex_unlock(&task->perf_event_mutex); 4184 4185 if (unlikely(err)) { 4186 put_ctx(ctx); 4187 4188 if (err == -EAGAIN) 4189 goto retry; 4190 goto errout; 4191 } 4192 } 4193 4194 kfree(task_ctx_data); 4195 return ctx; 4196 4197 errout: 4198 kfree(task_ctx_data); 4199 return ERR_PTR(err); 4200 } 4201 4202 static void perf_event_free_filter(struct perf_event *event); 4203 static void perf_event_free_bpf_prog(struct perf_event *event); 4204 4205 static void free_event_rcu(struct rcu_head *head) 4206 { 4207 struct perf_event *event; 4208 4209 event = container_of(head, struct perf_event, rcu_head); 4210 if (event->ns) 4211 put_pid_ns(event->ns); 4212 perf_event_free_filter(event); 4213 kfree(event); 4214 } 4215 4216 static void ring_buffer_attach(struct perf_event *event, 4217 struct ring_buffer *rb); 4218 4219 static void detach_sb_event(struct perf_event *event) 4220 { 4221 struct pmu_event_list *pel = per_cpu_ptr(&pmu_sb_events, event->cpu); 4222 4223 raw_spin_lock(&pel->lock); 4224 list_del_rcu(&event->sb_list); 4225 raw_spin_unlock(&pel->lock); 4226 } 4227 4228 static bool is_sb_event(struct perf_event *event) 4229 { 4230 struct perf_event_attr *attr = &event->attr; 4231 4232 if (event->parent) 4233 return false; 4234 4235 if (event->attach_state & PERF_ATTACH_TASK) 4236 return false; 4237 4238 if (attr->mmap || attr->mmap_data || attr->mmap2 || 4239 attr->comm || attr->comm_exec || 4240 attr->task || attr->ksymbol || 4241 attr->context_switch) 4242 return true; 4243 return false; 4244 } 4245 4246 static void unaccount_pmu_sb_event(struct perf_event *event) 4247 { 4248 if (is_sb_event(event)) 4249 detach_sb_event(event); 4250 } 4251 4252 static void unaccount_event_cpu(struct perf_event *event, int cpu) 4253 { 4254 if (event->parent) 4255 return; 4256 4257 if (is_cgroup_event(event)) 4258 atomic_dec(&per_cpu(perf_cgroup_events, cpu)); 4259 } 4260 4261 #ifdef CONFIG_NO_HZ_FULL 4262 static DEFINE_SPINLOCK(nr_freq_lock); 4263 #endif 4264 4265 static void unaccount_freq_event_nohz(void) 4266 { 4267 #ifdef CONFIG_NO_HZ_FULL 4268 spin_lock(&nr_freq_lock); 4269 if (atomic_dec_and_test(&nr_freq_events)) 4270 tick_nohz_dep_clear(TICK_DEP_BIT_PERF_EVENTS); 4271 spin_unlock(&nr_freq_lock); 4272 #endif 4273 } 4274 4275 static void unaccount_freq_event(void) 4276 { 4277 if (tick_nohz_full_enabled()) 4278 unaccount_freq_event_nohz(); 4279 else 4280 atomic_dec(&nr_freq_events); 4281 } 4282 4283 static void unaccount_event(struct perf_event *event) 4284 { 4285 bool dec = false; 4286 4287 if (event->parent) 4288 return; 4289 4290 if (event->attach_state & PERF_ATTACH_TASK) 4291 dec = true; 4292 if (event->attr.mmap || event->attr.mmap_data) 4293 atomic_dec(&nr_mmap_events); 4294 if (event->attr.comm) 4295 atomic_dec(&nr_comm_events); 4296 if (event->attr.namespaces) 4297 atomic_dec(&nr_namespaces_events); 4298 if (event->attr.task) 4299 atomic_dec(&nr_task_events); 4300 if (event->attr.freq) 4301 unaccount_freq_event(); 4302 if (event->attr.context_switch) { 4303 dec = true; 4304 atomic_dec(&nr_switch_events); 4305 } 4306 if (is_cgroup_event(event)) 4307 dec = true; 4308 if (has_branch_stack(event)) 4309 dec = true; 4310 if (event->attr.ksymbol) 4311 atomic_dec(&nr_ksymbol_events); 4312 if (event->attr.bpf_event) 4313 atomic_dec(&nr_bpf_events); 4314 4315 if (dec) { 4316 if (!atomic_add_unless(&perf_sched_count, -1, 1)) 4317 schedule_delayed_work(&perf_sched_work, HZ); 4318 } 4319 4320 unaccount_event_cpu(event, event->cpu); 4321 4322 unaccount_pmu_sb_event(event); 4323 } 4324 4325 static void perf_sched_delayed(struct work_struct *work) 4326 { 4327 mutex_lock(&perf_sched_mutex); 4328 if (atomic_dec_and_test(&perf_sched_count)) 4329 static_branch_disable(&perf_sched_events); 4330 mutex_unlock(&perf_sched_mutex); 4331 } 4332 4333 /* 4334 * The following implement mutual exclusion of events on "exclusive" pmus 4335 * (PERF_PMU_CAP_EXCLUSIVE). Such pmus can only have one event scheduled 4336 * at a time, so we disallow creating events that might conflict, namely: 4337 * 4338 * 1) cpu-wide events in the presence of per-task events, 4339 * 2) per-task events in the presence of cpu-wide events, 4340 * 3) two matching events on the same context. 4341 * 4342 * The former two cases are handled in the allocation path (perf_event_alloc(), 4343 * _free_event()), the latter -- before the first perf_install_in_context(). 4344 */ 4345 static int exclusive_event_init(struct perf_event *event) 4346 { 4347 struct pmu *pmu = event->pmu; 4348 4349 if (!(pmu->capabilities & PERF_PMU_CAP_EXCLUSIVE)) 4350 return 0; 4351 4352 /* 4353 * Prevent co-existence of per-task and cpu-wide events on the 4354 * same exclusive pmu. 4355 * 4356 * Negative pmu::exclusive_cnt means there are cpu-wide 4357 * events on this "exclusive" pmu, positive means there are 4358 * per-task events. 4359 * 4360 * Since this is called in perf_event_alloc() path, event::ctx 4361 * doesn't exist yet; it is, however, safe to use PERF_ATTACH_TASK 4362 * to mean "per-task event", because unlike other attach states it 4363 * never gets cleared. 4364 */ 4365 if (event->attach_state & PERF_ATTACH_TASK) { 4366 if (!atomic_inc_unless_negative(&pmu->exclusive_cnt)) 4367 return -EBUSY; 4368 } else { 4369 if (!atomic_dec_unless_positive(&pmu->exclusive_cnt)) 4370 return -EBUSY; 4371 } 4372 4373 return 0; 4374 } 4375 4376 static void exclusive_event_destroy(struct perf_event *event) 4377 { 4378 struct pmu *pmu = event->pmu; 4379 4380 if (!(pmu->capabilities & PERF_PMU_CAP_EXCLUSIVE)) 4381 return; 4382 4383 /* see comment in exclusive_event_init() */ 4384 if (event->attach_state & PERF_ATTACH_TASK) 4385 atomic_dec(&pmu->exclusive_cnt); 4386 else 4387 atomic_inc(&pmu->exclusive_cnt); 4388 } 4389 4390 static bool exclusive_event_match(struct perf_event *e1, struct perf_event *e2) 4391 { 4392 if ((e1->pmu == e2->pmu) && 4393 (e1->cpu == e2->cpu || 4394 e1->cpu == -1 || 4395 e2->cpu == -1)) 4396 return true; 4397 return false; 4398 } 4399 4400 /* Called under the same ctx::mutex as perf_install_in_context() */ 4401 static bool exclusive_event_installable(struct perf_event *event, 4402 struct perf_event_context *ctx) 4403 { 4404 struct perf_event *iter_event; 4405 struct pmu *pmu = event->pmu; 4406 4407 if (!(pmu->capabilities & PERF_PMU_CAP_EXCLUSIVE)) 4408 return true; 4409 4410 list_for_each_entry(iter_event, &ctx->event_list, event_entry) { 4411 if (exclusive_event_match(iter_event, event)) 4412 return false; 4413 } 4414 4415 return true; 4416 } 4417 4418 static void perf_addr_filters_splice(struct perf_event *event, 4419 struct list_head *head); 4420 4421 static void _free_event(struct perf_event *event) 4422 { 4423 irq_work_sync(&event->pending); 4424 4425 unaccount_event(event); 4426 4427 if (event->rb) { 4428 /* 4429 * Can happen when we close an event with re-directed output. 4430 * 4431 * Since we have a 0 refcount, perf_mmap_close() will skip 4432 * over us; possibly making our ring_buffer_put() the last. 4433 */ 4434 mutex_lock(&event->mmap_mutex); 4435 ring_buffer_attach(event, NULL); 4436 mutex_unlock(&event->mmap_mutex); 4437 } 4438 4439 if (is_cgroup_event(event)) 4440 perf_detach_cgroup(event); 4441 4442 if (!event->parent) { 4443 if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) 4444 put_callchain_buffers(); 4445 } 4446 4447 perf_event_free_bpf_prog(event); 4448 perf_addr_filters_splice(event, NULL); 4449 kfree(event->addr_filter_ranges); 4450 4451 if (event->destroy) 4452 event->destroy(event); 4453 4454 if (event->ctx) 4455 put_ctx(event->ctx); 4456 4457 if (event->hw.target) 4458 put_task_struct(event->hw.target); 4459 4460 exclusive_event_destroy(event); 4461 module_put(event->pmu->module); 4462 4463 call_rcu(&event->rcu_head, free_event_rcu); 4464 } 4465 4466 /* 4467 * Used to free events which have a known refcount of 1, such as in error paths 4468 * where the event isn't exposed yet and inherited events. 4469 */ 4470 static void free_event(struct perf_event *event) 4471 { 4472 if (WARN(atomic_long_cmpxchg(&event->refcount, 1, 0) != 1, 4473 "unexpected event refcount: %ld; ptr=%p\n", 4474 atomic_long_read(&event->refcount), event)) { 4475 /* leak to avoid use-after-free */ 4476 return; 4477 } 4478 4479 _free_event(event); 4480 } 4481 4482 /* 4483 * Remove user event from the owner task. 4484 */ 4485 static void perf_remove_from_owner(struct perf_event *event) 4486 { 4487 struct task_struct *owner; 4488 4489 rcu_read_lock(); 4490 /* 4491 * Matches the smp_store_release() in perf_event_exit_task(). If we 4492 * observe !owner it means the list deletion is complete and we can 4493 * indeed free this event, otherwise we need to serialize on 4494 * owner->perf_event_mutex. 4495 */ 4496 owner = READ_ONCE(event->owner); 4497 if (owner) { 4498 /* 4499 * Since delayed_put_task_struct() also drops the last 4500 * task reference we can safely take a new reference 4501 * while holding the rcu_read_lock(). 4502 */ 4503 get_task_struct(owner); 4504 } 4505 rcu_read_unlock(); 4506 4507 if (owner) { 4508 /* 4509 * If we're here through perf_event_exit_task() we're already 4510 * holding ctx->mutex which would be an inversion wrt. the 4511 * normal lock order. 4512 * 4513 * However we can safely take this lock because its the child 4514 * ctx->mutex. 4515 */ 4516 mutex_lock_nested(&owner->perf_event_mutex, SINGLE_DEPTH_NESTING); 4517 4518 /* 4519 * We have to re-check the event->owner field, if it is cleared 4520 * we raced with perf_event_exit_task(), acquiring the mutex 4521 * ensured they're done, and we can proceed with freeing the 4522 * event. 4523 */ 4524 if (event->owner) { 4525 list_del_init(&event->owner_entry); 4526 smp_store_release(&event->owner, NULL); 4527 } 4528 mutex_unlock(&owner->perf_event_mutex); 4529 put_task_struct(owner); 4530 } 4531 } 4532 4533 static void put_event(struct perf_event *event) 4534 { 4535 if (!atomic_long_dec_and_test(&event->refcount)) 4536 return; 4537 4538 _free_event(event); 4539 } 4540 4541 /* 4542 * Kill an event dead; while event:refcount will preserve the event 4543 * object, it will not preserve its functionality. Once the last 'user' 4544 * gives up the object, we'll destroy the thing. 4545 */ 4546 int perf_event_release_kernel(struct perf_event *event) 4547 { 4548 struct perf_event_context *ctx = event->ctx; 4549 struct perf_event *child, *tmp; 4550 LIST_HEAD(free_list); 4551 4552 /* 4553 * If we got here through err_file: fput(event_file); we will not have 4554 * attached to a context yet. 4555 */ 4556 if (!ctx) { 4557 WARN_ON_ONCE(event->attach_state & 4558 (PERF_ATTACH_CONTEXT|PERF_ATTACH_GROUP)); 4559 goto no_ctx; 4560 } 4561 4562 if (!is_kernel_event(event)) 4563 perf_remove_from_owner(event); 4564 4565 ctx = perf_event_ctx_lock(event); 4566 WARN_ON_ONCE(ctx->parent_ctx); 4567 perf_remove_from_context(event, DETACH_GROUP); 4568 4569 raw_spin_lock_irq(&ctx->lock); 4570 /* 4571 * Mark this event as STATE_DEAD, there is no external reference to it 4572 * anymore. 4573 * 4574 * Anybody acquiring event->child_mutex after the below loop _must_ 4575 * also see this, most importantly inherit_event() which will avoid 4576 * placing more children on the list. 4577 * 4578 * Thus this guarantees that we will in fact observe and kill _ALL_ 4579 * child events. 4580 */ 4581 event->state = PERF_EVENT_STATE_DEAD; 4582 raw_spin_unlock_irq(&ctx->lock); 4583 4584 perf_event_ctx_unlock(event, ctx); 4585 4586 again: 4587 mutex_lock(&event->child_mutex); 4588 list_for_each_entry(child, &event->child_list, child_list) { 4589 4590 /* 4591 * Cannot change, child events are not migrated, see the 4592 * comment with perf_event_ctx_lock_nested(). 4593 */ 4594 ctx = READ_ONCE(child->ctx); 4595 /* 4596 * Since child_mutex nests inside ctx::mutex, we must jump 4597 * through hoops. We start by grabbing a reference on the ctx. 4598 * 4599 * Since the event cannot get freed while we hold the 4600 * child_mutex, the context must also exist and have a !0 4601 * reference count. 4602 */ 4603 get_ctx(ctx); 4604 4605 /* 4606 * Now that we have a ctx ref, we can drop child_mutex, and 4607 * acquire ctx::mutex without fear of it going away. Then we 4608 * can re-acquire child_mutex. 4609 */ 4610 mutex_unlock(&event->child_mutex); 4611 mutex_lock(&ctx->mutex); 4612 mutex_lock(&event->child_mutex); 4613 4614 /* 4615 * Now that we hold ctx::mutex and child_mutex, revalidate our 4616 * state, if child is still the first entry, it didn't get freed 4617 * and we can continue doing so. 4618 */ 4619 tmp = list_first_entry_or_null(&event->child_list, 4620 struct perf_event, child_list); 4621 if (tmp == child) { 4622 perf_remove_from_context(child, DETACH_GROUP); 4623 list_move(&child->child_list, &free_list); 4624 /* 4625 * This matches the refcount bump in inherit_event(); 4626 * this can't be the last reference. 4627 */ 4628 put_event(event); 4629 } 4630 4631 mutex_unlock(&event->child_mutex); 4632 mutex_unlock(&ctx->mutex); 4633 put_ctx(ctx); 4634 goto again; 4635 } 4636 mutex_unlock(&event->child_mutex); 4637 4638 list_for_each_entry_safe(child, tmp, &free_list, child_list) { 4639 list_del(&child->child_list); 4640 free_event(child); 4641 } 4642 4643 no_ctx: 4644 put_event(event); /* Must be the 'last' reference */ 4645 return 0; 4646 } 4647 EXPORT_SYMBOL_GPL(perf_event_release_kernel); 4648 4649 /* 4650 * Called when the last reference to the file is gone. 4651 */ 4652 static int perf_release(struct inode *inode, struct file *file) 4653 { 4654 perf_event_release_kernel(file->private_data); 4655 return 0; 4656 } 4657 4658 static u64 __perf_event_read_value(struct perf_event *event, u64 *enabled, u64 *running) 4659 { 4660 struct perf_event *child; 4661 u64 total = 0; 4662 4663 *enabled = 0; 4664 *running = 0; 4665 4666 mutex_lock(&event->child_mutex); 4667 4668 (void)perf_event_read(event, false); 4669 total += perf_event_count(event); 4670 4671 *enabled += event->total_time_enabled + 4672 atomic64_read(&event->child_total_time_enabled); 4673 *running += event->total_time_running + 4674 atomic64_read(&event->child_total_time_running); 4675 4676 list_for_each_entry(child, &event->child_list, child_list) { 4677 (void)perf_event_read(child, false); 4678 total += perf_event_count(child); 4679 *enabled += child->total_time_enabled; 4680 *running += child->total_time_running; 4681 } 4682 mutex_unlock(&event->child_mutex); 4683 4684 return total; 4685 } 4686 4687 u64 perf_event_read_value(struct perf_event *event, u64 *enabled, u64 *running) 4688 { 4689 struct perf_event_context *ctx; 4690 u64 count; 4691 4692 ctx = perf_event_ctx_lock(event); 4693 count = __perf_event_read_value(event, enabled, running); 4694 perf_event_ctx_unlock(event, ctx); 4695 4696 return count; 4697 } 4698 EXPORT_SYMBOL_GPL(perf_event_read_value); 4699 4700 static int __perf_read_group_add(struct perf_event *leader, 4701 u64 read_format, u64 *values) 4702 { 4703 struct perf_event_context *ctx = leader->ctx; 4704 struct perf_event *sub; 4705 unsigned long flags; 4706 int n = 1; /* skip @nr */ 4707 int ret; 4708 4709 ret = perf_event_read(leader, true); 4710 if (ret) 4711 return ret; 4712 4713 raw_spin_lock_irqsave(&ctx->lock, flags); 4714 4715 /* 4716 * Since we co-schedule groups, {enabled,running} times of siblings 4717 * will be identical to those of the leader, so we only publish one 4718 * set. 4719 */ 4720 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) { 4721 values[n++] += leader->total_time_enabled + 4722 atomic64_read(&leader->child_total_time_enabled); 4723 } 4724 4725 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) { 4726 values[n++] += leader->total_time_running + 4727 atomic64_read(&leader->child_total_time_running); 4728 } 4729 4730 /* 4731 * Write {count,id} tuples for every sibling. 4732 */ 4733 values[n++] += perf_event_count(leader); 4734 if (read_format & PERF_FORMAT_ID) 4735 values[n++] = primary_event_id(leader); 4736 4737 for_each_sibling_event(sub, leader) { 4738 values[n++] += perf_event_count(sub); 4739 if (read_format & PERF_FORMAT_ID) 4740 values[n++] = primary_event_id(sub); 4741 } 4742 4743 raw_spin_unlock_irqrestore(&ctx->lock, flags); 4744 return 0; 4745 } 4746 4747 static int perf_read_group(struct perf_event *event, 4748 u64 read_format, char __user *buf) 4749 { 4750 struct perf_event *leader = event->group_leader, *child; 4751 struct perf_event_context *ctx = leader->ctx; 4752 int ret; 4753 u64 *values; 4754 4755 lockdep_assert_held(&ctx->mutex); 4756 4757 values = kzalloc(event->read_size, GFP_KERNEL); 4758 if (!values) 4759 return -ENOMEM; 4760 4761 values[0] = 1 + leader->nr_siblings; 4762 4763 /* 4764 * By locking the child_mutex of the leader we effectively 4765 * lock the child list of all siblings.. XXX explain how. 4766 */ 4767 mutex_lock(&leader->child_mutex); 4768 4769 ret = __perf_read_group_add(leader, read_format, values); 4770 if (ret) 4771 goto unlock; 4772 4773 list_for_each_entry(child, &leader->child_list, child_list) { 4774 ret = __perf_read_group_add(child, read_format, values); 4775 if (ret) 4776 goto unlock; 4777 } 4778 4779 mutex_unlock(&leader->child_mutex); 4780 4781 ret = event->read_size; 4782 if (copy_to_user(buf, values, event->read_size)) 4783 ret = -EFAULT; 4784 goto out; 4785 4786 unlock: 4787 mutex_unlock(&leader->child_mutex); 4788 out: 4789 kfree(values); 4790 return ret; 4791 } 4792 4793 static int perf_read_one(struct perf_event *event, 4794 u64 read_format, char __user *buf) 4795 { 4796 u64 enabled, running; 4797 u64 values[4]; 4798 int n = 0; 4799 4800 values[n++] = __perf_event_read_value(event, &enabled, &running); 4801 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) 4802 values[n++] = enabled; 4803 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) 4804 values[n++] = running; 4805 if (read_format & PERF_FORMAT_ID) 4806 values[n++] = primary_event_id(event); 4807 4808 if (copy_to_user(buf, values, n * sizeof(u64))) 4809 return -EFAULT; 4810 4811 return n * sizeof(u64); 4812 } 4813 4814 static bool is_event_hup(struct perf_event *event) 4815 { 4816 bool no_children; 4817 4818 if (event->state > PERF_EVENT_STATE_EXIT) 4819 return false; 4820 4821 mutex_lock(&event->child_mutex); 4822 no_children = list_empty(&event->child_list); 4823 mutex_unlock(&event->child_mutex); 4824 return no_children; 4825 } 4826 4827 /* 4828 * Read the performance event - simple non blocking version for now 4829 */ 4830 static ssize_t 4831 __perf_read(struct perf_event *event, char __user *buf, size_t count) 4832 { 4833 u64 read_format = event->attr.read_format; 4834 int ret; 4835 4836 /* 4837 * Return end-of-file for a read on an event that is in 4838 * error state (i.e. because it was pinned but it couldn't be 4839 * scheduled on to the CPU at some point). 4840 */ 4841 if (event->state == PERF_EVENT_STATE_ERROR) 4842 return 0; 4843 4844 if (count < event->read_size) 4845 return -ENOSPC; 4846 4847 WARN_ON_ONCE(event->ctx->parent_ctx); 4848 if (read_format & PERF_FORMAT_GROUP) 4849 ret = perf_read_group(event, read_format, buf); 4850 else 4851 ret = perf_read_one(event, read_format, buf); 4852 4853 return ret; 4854 } 4855 4856 static ssize_t 4857 perf_read(struct file *file, char __user *buf, size_t count, loff_t *ppos) 4858 { 4859 struct perf_event *event = file->private_data; 4860 struct perf_event_context *ctx; 4861 int ret; 4862 4863 ctx = perf_event_ctx_lock(event); 4864 ret = __perf_read(event, buf, count); 4865 perf_event_ctx_unlock(event, ctx); 4866 4867 return ret; 4868 } 4869 4870 static __poll_t perf_poll(struct file *file, poll_table *wait) 4871 { 4872 struct perf_event *event = file->private_data; 4873 struct ring_buffer *rb; 4874 __poll_t events = EPOLLHUP; 4875 4876 poll_wait(file, &event->waitq, wait); 4877 4878 if (is_event_hup(event)) 4879 return events; 4880 4881 /* 4882 * Pin the event->rb by taking event->mmap_mutex; otherwise 4883 * perf_event_set_output() can swizzle our rb and make us miss wakeups. 4884 */ 4885 mutex_lock(&event->mmap_mutex); 4886 rb = event->rb; 4887 if (rb) 4888 events = atomic_xchg(&rb->poll, 0); 4889 mutex_unlock(&event->mmap_mutex); 4890 return events; 4891 } 4892 4893 static void _perf_event_reset(struct perf_event *event) 4894 { 4895 (void)perf_event_read(event, false); 4896 local64_set(&event->count, 0); 4897 perf_event_update_userpage(event); 4898 } 4899 4900 /* 4901 * Holding the top-level event's child_mutex means that any 4902 * descendant process that has inherited this event will block 4903 * in perf_event_exit_event() if it goes to exit, thus satisfying the 4904 * task existence requirements of perf_event_enable/disable. 4905 */ 4906 static void perf_event_for_each_child(struct perf_event *event, 4907 void (*func)(struct perf_event *)) 4908 { 4909 struct perf_event *child; 4910 4911 WARN_ON_ONCE(event->ctx->parent_ctx); 4912 4913 mutex_lock(&event->child_mutex); 4914 func(event); 4915 list_for_each_entry(child, &event->child_list, child_list) 4916 func(child); 4917 mutex_unlock(&event->child_mutex); 4918 } 4919 4920 static void perf_event_for_each(struct perf_event *event, 4921 void (*func)(struct perf_event *)) 4922 { 4923 struct perf_event_context *ctx = event->ctx; 4924 struct perf_event *sibling; 4925 4926 lockdep_assert_held(&ctx->mutex); 4927 4928 event = event->group_leader; 4929 4930 perf_event_for_each_child(event, func); 4931 for_each_sibling_event(sibling, event) 4932 perf_event_for_each_child(sibling, func); 4933 } 4934 4935 static void __perf_event_period(struct perf_event *event, 4936 struct perf_cpu_context *cpuctx, 4937 struct perf_event_context *ctx, 4938 void *info) 4939 { 4940 u64 value = *((u64 *)info); 4941 bool active; 4942 4943 if (event->attr.freq) { 4944 event->attr.sample_freq = value; 4945 } else { 4946 event->attr.sample_period = value; 4947 event->hw.sample_period = value; 4948 } 4949 4950 active = (event->state == PERF_EVENT_STATE_ACTIVE); 4951 if (active) { 4952 perf_pmu_disable(ctx->pmu); 4953 /* 4954 * We could be throttled; unthrottle now to avoid the tick 4955 * trying to unthrottle while we already re-started the event. 4956 */ 4957 if (event->hw.interrupts == MAX_INTERRUPTS) { 4958 event->hw.interrupts = 0; 4959 perf_log_throttle(event, 1); 4960 } 4961 event->pmu->stop(event, PERF_EF_UPDATE); 4962 } 4963 4964 local64_set(&event->hw.period_left, 0); 4965 4966 if (active) { 4967 event->pmu->start(event, PERF_EF_RELOAD); 4968 perf_pmu_enable(ctx->pmu); 4969 } 4970 } 4971 4972 static int perf_event_check_period(struct perf_event *event, u64 value) 4973 { 4974 return event->pmu->check_period(event, value); 4975 } 4976 4977 static int perf_event_period(struct perf_event *event, u64 __user *arg) 4978 { 4979 u64 value; 4980 4981 if (!is_sampling_event(event)) 4982 return -EINVAL; 4983 4984 if (copy_from_user(&value, arg, sizeof(value))) 4985 return -EFAULT; 4986 4987 if (!value) 4988 return -EINVAL; 4989 4990 if (event->attr.freq && value > sysctl_perf_event_sample_rate) 4991 return -EINVAL; 4992 4993 if (perf_event_check_period(event, value)) 4994 return -EINVAL; 4995 4996 event_function_call(event, __perf_event_period, &value); 4997 4998 return 0; 4999 } 5000 5001 static const struct file_operations perf_fops; 5002 5003 static inline int perf_fget_light(int fd, struct fd *p) 5004 { 5005 struct fd f = fdget(fd); 5006 if (!f.file) 5007 return -EBADF; 5008 5009 if (f.file->f_op != &perf_fops) { 5010 fdput(f); 5011 return -EBADF; 5012 } 5013 *p = f; 5014 return 0; 5015 } 5016 5017 static int perf_event_set_output(struct perf_event *event, 5018 struct perf_event *output_event); 5019 static int perf_event_set_filter(struct perf_event *event, void __user *arg); 5020 static int perf_event_set_bpf_prog(struct perf_event *event, u32 prog_fd); 5021 static int perf_copy_attr(struct perf_event_attr __user *uattr, 5022 struct perf_event_attr *attr); 5023 5024 static long _perf_ioctl(struct perf_event *event, unsigned int cmd, unsigned long arg) 5025 { 5026 void (*func)(struct perf_event *); 5027 u32 flags = arg; 5028 5029 switch (cmd) { 5030 case PERF_EVENT_IOC_ENABLE: 5031 func = _perf_event_enable; 5032 break; 5033 case PERF_EVENT_IOC_DISABLE: 5034 func = _perf_event_disable; 5035 break; 5036 case PERF_EVENT_IOC_RESET: 5037 func = _perf_event_reset; 5038 break; 5039 5040 case PERF_EVENT_IOC_REFRESH: 5041 return _perf_event_refresh(event, arg); 5042 5043 case PERF_EVENT_IOC_PERIOD: 5044 return perf_event_period(event, (u64 __user *)arg); 5045 5046 case PERF_EVENT_IOC_ID: 5047 { 5048 u64 id = primary_event_id(event); 5049 5050 if (copy_to_user((void __user *)arg, &id, sizeof(id))) 5051 return -EFAULT; 5052 return 0; 5053 } 5054 5055 case PERF_EVENT_IOC_SET_OUTPUT: 5056 { 5057 int ret; 5058 if (arg != -1) { 5059 struct perf_event *output_event; 5060 struct fd output; 5061 ret = perf_fget_light(arg, &output); 5062 if (ret) 5063 return ret; 5064 output_event = output.file->private_data; 5065 ret = perf_event_set_output(event, output_event); 5066 fdput(output); 5067 } else { 5068 ret = perf_event_set_output(event, NULL); 5069 } 5070 return ret; 5071 } 5072 5073 case PERF_EVENT_IOC_SET_FILTER: 5074 return perf_event_set_filter(event, (void __user *)arg); 5075 5076 case PERF_EVENT_IOC_SET_BPF: 5077 return perf_event_set_bpf_prog(event, arg); 5078 5079 case PERF_EVENT_IOC_PAUSE_OUTPUT: { 5080 struct ring_buffer *rb; 5081 5082 rcu_read_lock(); 5083 rb = rcu_dereference(event->rb); 5084 if (!rb || !rb->nr_pages) { 5085 rcu_read_unlock(); 5086 return -EINVAL; 5087 } 5088 rb_toggle_paused(rb, !!arg); 5089 rcu_read_unlock(); 5090 return 0; 5091 } 5092 5093 case PERF_EVENT_IOC_QUERY_BPF: 5094 return perf_event_query_prog_array(event, (void __user *)arg); 5095 5096 case PERF_EVENT_IOC_MODIFY_ATTRIBUTES: { 5097 struct perf_event_attr new_attr; 5098 int err = perf_copy_attr((struct perf_event_attr __user *)arg, 5099 &new_attr); 5100 5101 if (err) 5102 return err; 5103 5104 return perf_event_modify_attr(event, &new_attr); 5105 } 5106 default: 5107 return -ENOTTY; 5108 } 5109 5110 if (flags & PERF_IOC_FLAG_GROUP) 5111 perf_event_for_each(event, func); 5112 else 5113 perf_event_for_each_child(event, func); 5114 5115 return 0; 5116 } 5117 5118 static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg) 5119 { 5120 struct perf_event *event = file->private_data; 5121 struct perf_event_context *ctx; 5122 long ret; 5123 5124 ctx = perf_event_ctx_lock(event); 5125 ret = _perf_ioctl(event, cmd, arg); 5126 perf_event_ctx_unlock(event, ctx); 5127 5128 return ret; 5129 } 5130 5131 #ifdef CONFIG_COMPAT 5132 static long perf_compat_ioctl(struct file *file, unsigned int cmd, 5133 unsigned long arg) 5134 { 5135 switch (_IOC_NR(cmd)) { 5136 case _IOC_NR(PERF_EVENT_IOC_SET_FILTER): 5137 case _IOC_NR(PERF_EVENT_IOC_ID): 5138 case _IOC_NR(PERF_EVENT_IOC_QUERY_BPF): 5139 case _IOC_NR(PERF_EVENT_IOC_MODIFY_ATTRIBUTES): 5140 /* Fix up pointer size (usually 4 -> 8 in 32-on-64-bit case */ 5141 if (_IOC_SIZE(cmd) == sizeof(compat_uptr_t)) { 5142 cmd &= ~IOCSIZE_MASK; 5143 cmd |= sizeof(void *) << IOCSIZE_SHIFT; 5144 } 5145 break; 5146 } 5147 return perf_ioctl(file, cmd, arg); 5148 } 5149 #else 5150 # define perf_compat_ioctl NULL 5151 #endif 5152 5153 int perf_event_task_enable(void) 5154 { 5155 struct perf_event_context *ctx; 5156 struct perf_event *event; 5157 5158 mutex_lock(¤t->perf_event_mutex); 5159 list_for_each_entry(event, ¤t->perf_event_list, owner_entry) { 5160 ctx = perf_event_ctx_lock(event); 5161 perf_event_for_each_child(event, _perf_event_enable); 5162 perf_event_ctx_unlock(event, ctx); 5163 } 5164 mutex_unlock(¤t->perf_event_mutex); 5165 5166 return 0; 5167 } 5168 5169 int perf_event_task_disable(void) 5170 { 5171 struct perf_event_context *ctx; 5172 struct perf_event *event; 5173 5174 mutex_lock(¤t->perf_event_mutex); 5175 list_for_each_entry(event, ¤t->perf_event_list, owner_entry) { 5176 ctx = perf_event_ctx_lock(event); 5177 perf_event_for_each_child(event, _perf_event_disable); 5178 perf_event_ctx_unlock(event, ctx); 5179 } 5180 mutex_unlock(¤t->perf_event_mutex); 5181 5182 return 0; 5183 } 5184 5185 static int perf_event_index(struct perf_event *event) 5186 { 5187 if (event->hw.state & PERF_HES_STOPPED) 5188 return 0; 5189 5190 if (event->state != PERF_EVENT_STATE_ACTIVE) 5191 return 0; 5192 5193 return event->pmu->event_idx(event); 5194 } 5195 5196 static void calc_timer_values(struct perf_event *event, 5197 u64 *now, 5198 u64 *enabled, 5199 u64 *running) 5200 { 5201 u64 ctx_time; 5202 5203 *now = perf_clock(); 5204 ctx_time = event->shadow_ctx_time + *now; 5205 __perf_update_times(event, ctx_time, enabled, running); 5206 } 5207 5208 static void perf_event_init_userpage(struct perf_event *event) 5209 { 5210 struct perf_event_mmap_page *userpg; 5211 struct ring_buffer *rb; 5212 5213 rcu_read_lock(); 5214 rb = rcu_dereference(event->rb); 5215 if (!rb) 5216 goto unlock; 5217 5218 userpg = rb->user_page; 5219 5220 /* Allow new userspace to detect that bit 0 is deprecated */ 5221 userpg->cap_bit0_is_deprecated = 1; 5222 userpg->size = offsetof(struct perf_event_mmap_page, __reserved); 5223 userpg->data_offset = PAGE_SIZE; 5224 userpg->data_size = perf_data_size(rb); 5225 5226 unlock: 5227 rcu_read_unlock(); 5228 } 5229 5230 void __weak arch_perf_update_userpage( 5231 struct perf_event *event, struct perf_event_mmap_page *userpg, u64 now) 5232 { 5233 } 5234 5235 /* 5236 * Callers need to ensure there can be no nesting of this function, otherwise 5237 * the seqlock logic goes bad. We can not serialize this because the arch 5238 * code calls this from NMI context. 5239 */ 5240 void perf_event_update_userpage(struct perf_event *event) 5241 { 5242 struct perf_event_mmap_page *userpg; 5243 struct ring_buffer *rb; 5244 u64 enabled, running, now; 5245 5246 rcu_read_lock(); 5247 rb = rcu_dereference(event->rb); 5248 if (!rb) 5249 goto unlock; 5250 5251 /* 5252 * compute total_time_enabled, total_time_running 5253 * based on snapshot values taken when the event 5254 * was last scheduled in. 5255 * 5256 * we cannot simply called update_context_time() 5257 * because of locking issue as we can be called in 5258 * NMI context 5259 */ 5260 calc_timer_values(event, &now, &enabled, &running); 5261 5262 userpg = rb->user_page; 5263 /* 5264 * Disable preemption to guarantee consistent time stamps are stored to 5265 * the user page. 5266 */ 5267 preempt_disable(); 5268 ++userpg->lock; 5269 barrier(); 5270 userpg->index = perf_event_index(event); 5271 userpg->offset = perf_event_count(event); 5272 if (userpg->index) 5273 userpg->offset -= local64_read(&event->hw.prev_count); 5274 5275 userpg->time_enabled = enabled + 5276 atomic64_read(&event->child_total_time_enabled); 5277 5278 userpg->time_running = running + 5279 atomic64_read(&event->child_total_time_running); 5280 5281 arch_perf_update_userpage(event, userpg, now); 5282 5283 barrier(); 5284 ++userpg->lock; 5285 preempt_enable(); 5286 unlock: 5287 rcu_read_unlock(); 5288 } 5289 EXPORT_SYMBOL_GPL(perf_event_update_userpage); 5290 5291 static vm_fault_t perf_mmap_fault(struct vm_fault *vmf) 5292 { 5293 struct perf_event *event = vmf->vma->vm_file->private_data; 5294 struct ring_buffer *rb; 5295 vm_fault_t ret = VM_FAULT_SIGBUS; 5296 5297 if (vmf->flags & FAULT_FLAG_MKWRITE) { 5298 if (vmf->pgoff == 0) 5299 ret = 0; 5300 return ret; 5301 } 5302 5303 rcu_read_lock(); 5304 rb = rcu_dereference(event->rb); 5305 if (!rb) 5306 goto unlock; 5307 5308 if (vmf->pgoff && (vmf->flags & FAULT_FLAG_WRITE)) 5309 goto unlock; 5310 5311 vmf->page = perf_mmap_to_page(rb, vmf->pgoff); 5312 if (!vmf->page) 5313 goto unlock; 5314 5315 get_page(vmf->page); 5316 vmf->page->mapping = vmf->vma->vm_file->f_mapping; 5317 vmf->page->index = vmf->pgoff; 5318 5319 ret = 0; 5320 unlock: 5321 rcu_read_unlock(); 5322 5323 return ret; 5324 } 5325 5326 static void ring_buffer_attach(struct perf_event *event, 5327 struct ring_buffer *rb) 5328 { 5329 struct ring_buffer *old_rb = NULL; 5330 unsigned long flags; 5331 5332 if (event->rb) { 5333 /* 5334 * Should be impossible, we set this when removing 5335 * event->rb_entry and wait/clear when adding event->rb_entry. 5336 */ 5337 WARN_ON_ONCE(event->rcu_pending); 5338 5339 old_rb = event->rb; 5340 spin_lock_irqsave(&old_rb->event_lock, flags); 5341 list_del_rcu(&event->rb_entry); 5342 spin_unlock_irqrestore(&old_rb->event_lock, flags); 5343 5344 event->rcu_batches = get_state_synchronize_rcu(); 5345 event->rcu_pending = 1; 5346 } 5347 5348 if (rb) { 5349 if (event->rcu_pending) { 5350 cond_synchronize_rcu(event->rcu_batches); 5351 event->rcu_pending = 0; 5352 } 5353 5354 spin_lock_irqsave(&rb->event_lock, flags); 5355 list_add_rcu(&event->rb_entry, &rb->event_list); 5356 spin_unlock_irqrestore(&rb->event_lock, flags); 5357 } 5358 5359 /* 5360 * Avoid racing with perf_mmap_close(AUX): stop the event 5361 * before swizzling the event::rb pointer; if it's getting 5362 * unmapped, its aux_mmap_count will be 0 and it won't 5363 * restart. See the comment in __perf_pmu_output_stop(). 5364 * 5365 * Data will inevitably be lost when set_output is done in 5366 * mid-air, but then again, whoever does it like this is 5367 * not in for the data anyway. 5368 */ 5369 if (has_aux(event)) 5370 perf_event_stop(event, 0); 5371 5372 rcu_assign_pointer(event->rb, rb); 5373 5374 if (old_rb) { 5375 ring_buffer_put(old_rb); 5376 /* 5377 * Since we detached before setting the new rb, so that we 5378 * could attach the new rb, we could have missed a wakeup. 5379 * Provide it now. 5380 */ 5381 wake_up_all(&event->waitq); 5382 } 5383 } 5384 5385 static void ring_buffer_wakeup(struct perf_event *event) 5386 { 5387 struct ring_buffer *rb; 5388 5389 rcu_read_lock(); 5390 rb = rcu_dereference(event->rb); 5391 if (rb) { 5392 list_for_each_entry_rcu(event, &rb->event_list, rb_entry) 5393 wake_up_all(&event->waitq); 5394 } 5395 rcu_read_unlock(); 5396 } 5397 5398 struct ring_buffer *ring_buffer_get(struct perf_event *event) 5399 { 5400 struct ring_buffer *rb; 5401 5402 rcu_read_lock(); 5403 rb = rcu_dereference(event->rb); 5404 if (rb) { 5405 if (!refcount_inc_not_zero(&rb->refcount)) 5406 rb = NULL; 5407 } 5408 rcu_read_unlock(); 5409 5410 return rb; 5411 } 5412 5413 void ring_buffer_put(struct ring_buffer *rb) 5414 { 5415 if (!refcount_dec_and_test(&rb->refcount)) 5416 return; 5417 5418 WARN_ON_ONCE(!list_empty(&rb->event_list)); 5419 5420 call_rcu(&rb->rcu_head, rb_free_rcu); 5421 } 5422 5423 static void perf_mmap_open(struct vm_area_struct *vma) 5424 { 5425 struct perf_event *event = vma->vm_file->private_data; 5426 5427 atomic_inc(&event->mmap_count); 5428 atomic_inc(&event->rb->mmap_count); 5429 5430 if (vma->vm_pgoff) 5431 atomic_inc(&event->rb->aux_mmap_count); 5432 5433 if (event->pmu->event_mapped) 5434 event->pmu->event_mapped(event, vma->vm_mm); 5435 } 5436 5437 static void perf_pmu_output_stop(struct perf_event *event); 5438 5439 /* 5440 * A buffer can be mmap()ed multiple times; either directly through the same 5441 * event, or through other events by use of perf_event_set_output(). 5442 * 5443 * In order to undo the VM accounting done by perf_mmap() we need to destroy 5444 * the buffer here, where we still have a VM context. This means we need 5445 * to detach all events redirecting to us. 5446 */ 5447 static void perf_mmap_close(struct vm_area_struct *vma) 5448 { 5449 struct perf_event *event = vma->vm_file->private_data; 5450 5451 struct ring_buffer *rb = ring_buffer_get(event); 5452 struct user_struct *mmap_user = rb->mmap_user; 5453 int mmap_locked = rb->mmap_locked; 5454 unsigned long size = perf_data_size(rb); 5455 5456 if (event->pmu->event_unmapped) 5457 event->pmu->event_unmapped(event, vma->vm_mm); 5458 5459 /* 5460 * rb->aux_mmap_count will always drop before rb->mmap_count and 5461 * event->mmap_count, so it is ok to use event->mmap_mutex to 5462 * serialize with perf_mmap here. 5463 */ 5464 if (rb_has_aux(rb) && vma->vm_pgoff == rb->aux_pgoff && 5465 atomic_dec_and_mutex_lock(&rb->aux_mmap_count, &event->mmap_mutex)) { 5466 /* 5467 * Stop all AUX events that are writing to this buffer, 5468 * so that we can free its AUX pages and corresponding PMU 5469 * data. Note that after rb::aux_mmap_count dropped to zero, 5470 * they won't start any more (see perf_aux_output_begin()). 5471 */ 5472 perf_pmu_output_stop(event); 5473 5474 /* now it's safe to free the pages */ 5475 atomic_long_sub(rb->aux_nr_pages, &mmap_user->locked_vm); 5476 vma->vm_mm->pinned_vm -= rb->aux_mmap_locked; 5477 5478 /* this has to be the last one */ 5479 rb_free_aux(rb); 5480 WARN_ON_ONCE(refcount_read(&rb->aux_refcount)); 5481 5482 mutex_unlock(&event->mmap_mutex); 5483 } 5484 5485 atomic_dec(&rb->mmap_count); 5486 5487 if (!atomic_dec_and_mutex_lock(&event->mmap_count, &event->mmap_mutex)) 5488 goto out_put; 5489 5490 ring_buffer_attach(event, NULL); 5491 mutex_unlock(&event->mmap_mutex); 5492 5493 /* If there's still other mmap()s of this buffer, we're done. */ 5494 if (atomic_read(&rb->mmap_count)) 5495 goto out_put; 5496 5497 /* 5498 * No other mmap()s, detach from all other events that might redirect 5499 * into the now unreachable buffer. Somewhat complicated by the 5500 * fact that rb::event_lock otherwise nests inside mmap_mutex. 5501 */ 5502 again: 5503 rcu_read_lock(); 5504 list_for_each_entry_rcu(event, &rb->event_list, rb_entry) { 5505 if (!atomic_long_inc_not_zero(&event->refcount)) { 5506 /* 5507 * This event is en-route to free_event() which will 5508 * detach it and remove it from the list. 5509 */ 5510 continue; 5511 } 5512 rcu_read_unlock(); 5513 5514 mutex_lock(&event->mmap_mutex); 5515 /* 5516 * Check we didn't race with perf_event_set_output() which can 5517 * swizzle the rb from under us while we were waiting to 5518 * acquire mmap_mutex. 5519 * 5520 * If we find a different rb; ignore this event, a next 5521 * iteration will no longer find it on the list. We have to 5522 * still restart the iteration to make sure we're not now 5523 * iterating the wrong list. 5524 */ 5525 if (event->rb == rb) 5526 ring_buffer_attach(event, NULL); 5527 5528 mutex_unlock(&event->mmap_mutex); 5529 put_event(event); 5530 5531 /* 5532 * Restart the iteration; either we're on the wrong list or 5533 * destroyed its integrity by doing a deletion. 5534 */ 5535 goto again; 5536 } 5537 rcu_read_unlock(); 5538 5539 /* 5540 * It could be there's still a few 0-ref events on the list; they'll 5541 * get cleaned up by free_event() -- they'll also still have their 5542 * ref on the rb and will free it whenever they are done with it. 5543 * 5544 * Aside from that, this buffer is 'fully' detached and unmapped, 5545 * undo the VM accounting. 5546 */ 5547 5548 atomic_long_sub((size >> PAGE_SHIFT) + 1, &mmap_user->locked_vm); 5549 vma->vm_mm->pinned_vm -= mmap_locked; 5550 free_uid(mmap_user); 5551 5552 out_put: 5553 ring_buffer_put(rb); /* could be last */ 5554 } 5555 5556 static const struct vm_operations_struct perf_mmap_vmops = { 5557 .open = perf_mmap_open, 5558 .close = perf_mmap_close, /* non mergeable */ 5559 .fault = perf_mmap_fault, 5560 .page_mkwrite = perf_mmap_fault, 5561 }; 5562 5563 static int perf_mmap(struct file *file, struct vm_area_struct *vma) 5564 { 5565 struct perf_event *event = file->private_data; 5566 unsigned long user_locked, user_lock_limit; 5567 struct user_struct *user = current_user(); 5568 unsigned long locked, lock_limit; 5569 struct ring_buffer *rb = NULL; 5570 unsigned long vma_size; 5571 unsigned long nr_pages; 5572 long user_extra = 0, extra = 0; 5573 int ret = 0, flags = 0; 5574 5575 /* 5576 * Don't allow mmap() of inherited per-task counters. This would 5577 * create a performance issue due to all children writing to the 5578 * same rb. 5579 */ 5580 if (event->cpu == -1 && event->attr.inherit) 5581 return -EINVAL; 5582 5583 if (!(vma->vm_flags & VM_SHARED)) 5584 return -EINVAL; 5585 5586 vma_size = vma->vm_end - vma->vm_start; 5587 5588 if (vma->vm_pgoff == 0) { 5589 nr_pages = (vma_size / PAGE_SIZE) - 1; 5590 } else { 5591 /* 5592 * AUX area mapping: if rb->aux_nr_pages != 0, it's already 5593 * mapped, all subsequent mappings should have the same size 5594 * and offset. Must be above the normal perf buffer. 5595 */ 5596 u64 aux_offset, aux_size; 5597 5598 if (!event->rb) 5599 return -EINVAL; 5600 5601 nr_pages = vma_size / PAGE_SIZE; 5602 5603 mutex_lock(&event->mmap_mutex); 5604 ret = -EINVAL; 5605 5606 rb = event->rb; 5607 if (!rb) 5608 goto aux_unlock; 5609 5610 aux_offset = READ_ONCE(rb->user_page->aux_offset); 5611 aux_size = READ_ONCE(rb->user_page->aux_size); 5612 5613 if (aux_offset < perf_data_size(rb) + PAGE_SIZE) 5614 goto aux_unlock; 5615 5616 if (aux_offset != vma->vm_pgoff << PAGE_SHIFT) 5617 goto aux_unlock; 5618 5619 /* already mapped with a different offset */ 5620 if (rb_has_aux(rb) && rb->aux_pgoff != vma->vm_pgoff) 5621 goto aux_unlock; 5622 5623 if (aux_size != vma_size || aux_size != nr_pages * PAGE_SIZE) 5624 goto aux_unlock; 5625 5626 /* already mapped with a different size */ 5627 if (rb_has_aux(rb) && rb->aux_nr_pages != nr_pages) 5628 goto aux_unlock; 5629 5630 if (!is_power_of_2(nr_pages)) 5631 goto aux_unlock; 5632 5633 if (!atomic_inc_not_zero(&rb->mmap_count)) 5634 goto aux_unlock; 5635 5636 if (rb_has_aux(rb)) { 5637 atomic_inc(&rb->aux_mmap_count); 5638 ret = 0; 5639 goto unlock; 5640 } 5641 5642 atomic_set(&rb->aux_mmap_count, 1); 5643 user_extra = nr_pages; 5644 5645 goto accounting; 5646 } 5647 5648 /* 5649 * If we have rb pages ensure they're a power-of-two number, so we 5650 * can do bitmasks instead of modulo. 5651 */ 5652 if (nr_pages != 0 && !is_power_of_2(nr_pages)) 5653 return -EINVAL; 5654 5655 if (vma_size != PAGE_SIZE * (1 + nr_pages)) 5656 return -EINVAL; 5657 5658 WARN_ON_ONCE(event->ctx->parent_ctx); 5659 again: 5660 mutex_lock(&event->mmap_mutex); 5661 if (event->rb) { 5662 if (event->rb->nr_pages != nr_pages) { 5663 ret = -EINVAL; 5664 goto unlock; 5665 } 5666 5667 if (!atomic_inc_not_zero(&event->rb->mmap_count)) { 5668 /* 5669 * Raced against perf_mmap_close() through 5670 * perf_event_set_output(). Try again, hope for better 5671 * luck. 5672 */ 5673 mutex_unlock(&event->mmap_mutex); 5674 goto again; 5675 } 5676 5677 goto unlock; 5678 } 5679 5680 user_extra = nr_pages + 1; 5681 5682 accounting: 5683 user_lock_limit = sysctl_perf_event_mlock >> (PAGE_SHIFT - 10); 5684 5685 /* 5686 * Increase the limit linearly with more CPUs: 5687 */ 5688 user_lock_limit *= num_online_cpus(); 5689 5690 user_locked = atomic_long_read(&user->locked_vm) + user_extra; 5691 5692 if (user_locked > user_lock_limit) 5693 extra = user_locked - user_lock_limit; 5694 5695 lock_limit = rlimit(RLIMIT_MEMLOCK); 5696 lock_limit >>= PAGE_SHIFT; 5697 locked = vma->vm_mm->pinned_vm + extra; 5698 5699 if ((locked > lock_limit) && perf_paranoid_tracepoint_raw() && 5700 !capable(CAP_IPC_LOCK)) { 5701 ret = -EPERM; 5702 goto unlock; 5703 } 5704 5705 WARN_ON(!rb && event->rb); 5706 5707 if (vma->vm_flags & VM_WRITE) 5708 flags |= RING_BUFFER_WRITABLE; 5709 5710 if (!rb) { 5711 rb = rb_alloc(nr_pages, 5712 event->attr.watermark ? event->attr.wakeup_watermark : 0, 5713 event->cpu, flags); 5714 5715 if (!rb) { 5716 ret = -ENOMEM; 5717 goto unlock; 5718 } 5719 5720 atomic_set(&rb->mmap_count, 1); 5721 rb->mmap_user = get_current_user(); 5722 rb->mmap_locked = extra; 5723 5724 ring_buffer_attach(event, rb); 5725 5726 perf_event_init_userpage(event); 5727 perf_event_update_userpage(event); 5728 } else { 5729 ret = rb_alloc_aux(rb, event, vma->vm_pgoff, nr_pages, 5730 event->attr.aux_watermark, flags); 5731 if (!ret) 5732 rb->aux_mmap_locked = extra; 5733 } 5734 5735 unlock: 5736 if (!ret) { 5737 atomic_long_add(user_extra, &user->locked_vm); 5738 vma->vm_mm->pinned_vm += extra; 5739 5740 atomic_inc(&event->mmap_count); 5741 } else if (rb) { 5742 atomic_dec(&rb->mmap_count); 5743 } 5744 aux_unlock: 5745 mutex_unlock(&event->mmap_mutex); 5746 5747 /* 5748 * Since pinned accounting is per vm we cannot allow fork() to copy our 5749 * vma. 5750 */ 5751 vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_DONTDUMP; 5752 vma->vm_ops = &perf_mmap_vmops; 5753 5754 if (event->pmu->event_mapped) 5755 event->pmu->event_mapped(event, vma->vm_mm); 5756 5757 return ret; 5758 } 5759 5760 static int perf_fasync(int fd, struct file *filp, int on) 5761 { 5762 struct inode *inode = file_inode(filp); 5763 struct perf_event *event = filp->private_data; 5764 int retval; 5765 5766 inode_lock(inode); 5767 retval = fasync_helper(fd, filp, on, &event->fasync); 5768 inode_unlock(inode); 5769 5770 if (retval < 0) 5771 return retval; 5772 5773 return 0; 5774 } 5775 5776 static const struct file_operations perf_fops = { 5777 .llseek = no_llseek, 5778 .release = perf_release, 5779 .read = perf_read, 5780 .poll = perf_poll, 5781 .unlocked_ioctl = perf_ioctl, 5782 .compat_ioctl = perf_compat_ioctl, 5783 .mmap = perf_mmap, 5784 .fasync = perf_fasync, 5785 }; 5786 5787 /* 5788 * Perf event wakeup 5789 * 5790 * If there's data, ensure we set the poll() state and publish everything 5791 * to user-space before waking everybody up. 5792 */ 5793 5794 static inline struct fasync_struct **perf_event_fasync(struct perf_event *event) 5795 { 5796 /* only the parent has fasync state */ 5797 if (event->parent) 5798 event = event->parent; 5799 return &event->fasync; 5800 } 5801 5802 void perf_event_wakeup(struct perf_event *event) 5803 { 5804 ring_buffer_wakeup(event); 5805 5806 if (event->pending_kill) { 5807 kill_fasync(perf_event_fasync(event), SIGIO, event->pending_kill); 5808 event->pending_kill = 0; 5809 } 5810 } 5811 5812 static void perf_pending_event(struct irq_work *entry) 5813 { 5814 struct perf_event *event = container_of(entry, 5815 struct perf_event, pending); 5816 int rctx; 5817 5818 rctx = perf_swevent_get_recursion_context(); 5819 /* 5820 * If we 'fail' here, that's OK, it means recursion is already disabled 5821 * and we won't recurse 'further'. 5822 */ 5823 5824 if (event->pending_disable) { 5825 event->pending_disable = 0; 5826 perf_event_disable_local(event); 5827 } 5828 5829 if (event->pending_wakeup) { 5830 event->pending_wakeup = 0; 5831 perf_event_wakeup(event); 5832 } 5833 5834 if (rctx >= 0) 5835 perf_swevent_put_recursion_context(rctx); 5836 } 5837 5838 /* 5839 * We assume there is only KVM supporting the callbacks. 5840 * Later on, we might change it to a list if there is 5841 * another virtualization implementation supporting the callbacks. 5842 */ 5843 struct perf_guest_info_callbacks *perf_guest_cbs; 5844 5845 int perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *cbs) 5846 { 5847 perf_guest_cbs = cbs; 5848 return 0; 5849 } 5850 EXPORT_SYMBOL_GPL(perf_register_guest_info_callbacks); 5851 5852 int perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *cbs) 5853 { 5854 perf_guest_cbs = NULL; 5855 return 0; 5856 } 5857 EXPORT_SYMBOL_GPL(perf_unregister_guest_info_callbacks); 5858 5859 static void 5860 perf_output_sample_regs(struct perf_output_handle *handle, 5861 struct pt_regs *regs, u64 mask) 5862 { 5863 int bit; 5864 DECLARE_BITMAP(_mask, 64); 5865 5866 bitmap_from_u64(_mask, mask); 5867 for_each_set_bit(bit, _mask, sizeof(mask) * BITS_PER_BYTE) { 5868 u64 val; 5869 5870 val = perf_reg_value(regs, bit); 5871 perf_output_put(handle, val); 5872 } 5873 } 5874 5875 static void perf_sample_regs_user(struct perf_regs *regs_user, 5876 struct pt_regs *regs, 5877 struct pt_regs *regs_user_copy) 5878 { 5879 if (user_mode(regs)) { 5880 regs_user->abi = perf_reg_abi(current); 5881 regs_user->regs = regs; 5882 } else if (current->mm) { 5883 perf_get_regs_user(regs_user, regs, regs_user_copy); 5884 } else { 5885 regs_user->abi = PERF_SAMPLE_REGS_ABI_NONE; 5886 regs_user->regs = NULL; 5887 } 5888 } 5889 5890 static void perf_sample_regs_intr(struct perf_regs *regs_intr, 5891 struct pt_regs *regs) 5892 { 5893 regs_intr->regs = regs; 5894 regs_intr->abi = perf_reg_abi(current); 5895 } 5896 5897 5898 /* 5899 * Get remaining task size from user stack pointer. 5900 * 5901 * It'd be better to take stack vma map and limit this more 5902 * precisly, but there's no way to get it safely under interrupt, 5903 * so using TASK_SIZE as limit. 5904 */ 5905 static u64 perf_ustack_task_size(struct pt_regs *regs) 5906 { 5907 unsigned long addr = perf_user_stack_pointer(regs); 5908 5909 if (!addr || addr >= TASK_SIZE) 5910 return 0; 5911 5912 return TASK_SIZE - addr; 5913 } 5914 5915 static u16 5916 perf_sample_ustack_size(u16 stack_size, u16 header_size, 5917 struct pt_regs *regs) 5918 { 5919 u64 task_size; 5920 5921 /* No regs, no stack pointer, no dump. */ 5922 if (!regs) 5923 return 0; 5924 5925 /* 5926 * Check if we fit in with the requested stack size into the: 5927 * - TASK_SIZE 5928 * If we don't, we limit the size to the TASK_SIZE. 5929 * 5930 * - remaining sample size 5931 * If we don't, we customize the stack size to 5932 * fit in to the remaining sample size. 5933 */ 5934 5935 task_size = min((u64) USHRT_MAX, perf_ustack_task_size(regs)); 5936 stack_size = min(stack_size, (u16) task_size); 5937 5938 /* Current header size plus static size and dynamic size. */ 5939 header_size += 2 * sizeof(u64); 5940 5941 /* Do we fit in with the current stack dump size? */ 5942 if ((u16) (header_size + stack_size) < header_size) { 5943 /* 5944 * If we overflow the maximum size for the sample, 5945 * we customize the stack dump size to fit in. 5946 */ 5947 stack_size = USHRT_MAX - header_size - sizeof(u64); 5948 stack_size = round_up(stack_size, sizeof(u64)); 5949 } 5950 5951 return stack_size; 5952 } 5953 5954 static void 5955 perf_output_sample_ustack(struct perf_output_handle *handle, u64 dump_size, 5956 struct pt_regs *regs) 5957 { 5958 /* Case of a kernel thread, nothing to dump */ 5959 if (!regs) { 5960 u64 size = 0; 5961 perf_output_put(handle, size); 5962 } else { 5963 unsigned long sp; 5964 unsigned int rem; 5965 u64 dyn_size; 5966 mm_segment_t fs; 5967 5968 /* 5969 * We dump: 5970 * static size 5971 * - the size requested by user or the best one we can fit 5972 * in to the sample max size 5973 * data 5974 * - user stack dump data 5975 * dynamic size 5976 * - the actual dumped size 5977 */ 5978 5979 /* Static size. */ 5980 perf_output_put(handle, dump_size); 5981 5982 /* Data. */ 5983 sp = perf_user_stack_pointer(regs); 5984 fs = get_fs(); 5985 set_fs(USER_DS); 5986 rem = __output_copy_user(handle, (void *) sp, dump_size); 5987 set_fs(fs); 5988 dyn_size = dump_size - rem; 5989 5990 perf_output_skip(handle, rem); 5991 5992 /* Dynamic size. */ 5993 perf_output_put(handle, dyn_size); 5994 } 5995 } 5996 5997 static void __perf_event_header__init_id(struct perf_event_header *header, 5998 struct perf_sample_data *data, 5999 struct perf_event *event) 6000 { 6001 u64 sample_type = event->attr.sample_type; 6002 6003 data->type = sample_type; 6004 header->size += event->id_header_size; 6005 6006 if (sample_type & PERF_SAMPLE_TID) { 6007 /* namespace issues */ 6008 data->tid_entry.pid = perf_event_pid(event, current); 6009 data->tid_entry.tid = perf_event_tid(event, current); 6010 } 6011 6012 if (sample_type & PERF_SAMPLE_TIME) 6013 data->time = perf_event_clock(event); 6014 6015 if (sample_type & (PERF_SAMPLE_ID | PERF_SAMPLE_IDENTIFIER)) 6016 data->id = primary_event_id(event); 6017 6018 if (sample_type & PERF_SAMPLE_STREAM_ID) 6019 data->stream_id = event->id; 6020 6021 if (sample_type & PERF_SAMPLE_CPU) { 6022 data->cpu_entry.cpu = raw_smp_processor_id(); 6023 data->cpu_entry.reserved = 0; 6024 } 6025 } 6026 6027 void perf_event_header__init_id(struct perf_event_header *header, 6028 struct perf_sample_data *data, 6029 struct perf_event *event) 6030 { 6031 if (event->attr.sample_id_all) 6032 __perf_event_header__init_id(header, data, event); 6033 } 6034 6035 static void __perf_event__output_id_sample(struct perf_output_handle *handle, 6036 struct perf_sample_data *data) 6037 { 6038 u64 sample_type = data->type; 6039 6040 if (sample_type & PERF_SAMPLE_TID) 6041 perf_output_put(handle, data->tid_entry); 6042 6043 if (sample_type & PERF_SAMPLE_TIME) 6044 perf_output_put(handle, data->time); 6045 6046 if (sample_type & PERF_SAMPLE_ID) 6047 perf_output_put(handle, data->id); 6048 6049 if (sample_type & PERF_SAMPLE_STREAM_ID) 6050 perf_output_put(handle, data->stream_id); 6051 6052 if (sample_type & PERF_SAMPLE_CPU) 6053 perf_output_put(handle, data->cpu_entry); 6054 6055 if (sample_type & PERF_SAMPLE_IDENTIFIER) 6056 perf_output_put(handle, data->id); 6057 } 6058 6059 void perf_event__output_id_sample(struct perf_event *event, 6060 struct perf_output_handle *handle, 6061 struct perf_sample_data *sample) 6062 { 6063 if (event->attr.sample_id_all) 6064 __perf_event__output_id_sample(handle, sample); 6065 } 6066 6067 static void perf_output_read_one(struct perf_output_handle *handle, 6068 struct perf_event *event, 6069 u64 enabled, u64 running) 6070 { 6071 u64 read_format = event->attr.read_format; 6072 u64 values[4]; 6073 int n = 0; 6074 6075 values[n++] = perf_event_count(event); 6076 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) { 6077 values[n++] = enabled + 6078 atomic64_read(&event->child_total_time_enabled); 6079 } 6080 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) { 6081 values[n++] = running + 6082 atomic64_read(&event->child_total_time_running); 6083 } 6084 if (read_format & PERF_FORMAT_ID) 6085 values[n++] = primary_event_id(event); 6086 6087 __output_copy(handle, values, n * sizeof(u64)); 6088 } 6089 6090 static void perf_output_read_group(struct perf_output_handle *handle, 6091 struct perf_event *event, 6092 u64 enabled, u64 running) 6093 { 6094 struct perf_event *leader = event->group_leader, *sub; 6095 u64 read_format = event->attr.read_format; 6096 u64 values[5]; 6097 int n = 0; 6098 6099 values[n++] = 1 + leader->nr_siblings; 6100 6101 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) 6102 values[n++] = enabled; 6103 6104 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) 6105 values[n++] = running; 6106 6107 if ((leader != event) && 6108 (leader->state == PERF_EVENT_STATE_ACTIVE)) 6109 leader->pmu->read(leader); 6110 6111 values[n++] = perf_event_count(leader); 6112 if (read_format & PERF_FORMAT_ID) 6113 values[n++] = primary_event_id(leader); 6114 6115 __output_copy(handle, values, n * sizeof(u64)); 6116 6117 for_each_sibling_event(sub, leader) { 6118 n = 0; 6119 6120 if ((sub != event) && 6121 (sub->state == PERF_EVENT_STATE_ACTIVE)) 6122 sub->pmu->read(sub); 6123 6124 values[n++] = perf_event_count(sub); 6125 if (read_format & PERF_FORMAT_ID) 6126 values[n++] = primary_event_id(sub); 6127 6128 __output_copy(handle, values, n * sizeof(u64)); 6129 } 6130 } 6131 6132 #define PERF_FORMAT_TOTAL_TIMES (PERF_FORMAT_TOTAL_TIME_ENABLED|\ 6133 PERF_FORMAT_TOTAL_TIME_RUNNING) 6134 6135 /* 6136 * XXX PERF_SAMPLE_READ vs inherited events seems difficult. 6137 * 6138 * The problem is that its both hard and excessively expensive to iterate the 6139 * child list, not to mention that its impossible to IPI the children running 6140 * on another CPU, from interrupt/NMI context. 6141 */ 6142 static void perf_output_read(struct perf_output_handle *handle, 6143 struct perf_event *event) 6144 { 6145 u64 enabled = 0, running = 0, now; 6146 u64 read_format = event->attr.read_format; 6147 6148 /* 6149 * compute total_time_enabled, total_time_running 6150 * based on snapshot values taken when the event 6151 * was last scheduled in. 6152 * 6153 * we cannot simply called update_context_time() 6154 * because of locking issue as we are called in 6155 * NMI context 6156 */ 6157 if (read_format & PERF_FORMAT_TOTAL_TIMES) 6158 calc_timer_values(event, &now, &enabled, &running); 6159 6160 if (event->attr.read_format & PERF_FORMAT_GROUP) 6161 perf_output_read_group(handle, event, enabled, running); 6162 else 6163 perf_output_read_one(handle, event, enabled, running); 6164 } 6165 6166 void perf_output_sample(struct perf_output_handle *handle, 6167 struct perf_event_header *header, 6168 struct perf_sample_data *data, 6169 struct perf_event *event) 6170 { 6171 u64 sample_type = data->type; 6172 6173 perf_output_put(handle, *header); 6174 6175 if (sample_type & PERF_SAMPLE_IDENTIFIER) 6176 perf_output_put(handle, data->id); 6177 6178 if (sample_type & PERF_SAMPLE_IP) 6179 perf_output_put(handle, data->ip); 6180 6181 if (sample_type & PERF_SAMPLE_TID) 6182 perf_output_put(handle, data->tid_entry); 6183 6184 if (sample_type & PERF_SAMPLE_TIME) 6185 perf_output_put(handle, data->time); 6186 6187 if (sample_type & PERF_SAMPLE_ADDR) 6188 perf_output_put(handle, data->addr); 6189 6190 if (sample_type & PERF_SAMPLE_ID) 6191 perf_output_put(handle, data->id); 6192 6193 if (sample_type & PERF_SAMPLE_STREAM_ID) 6194 perf_output_put(handle, data->stream_id); 6195 6196 if (sample_type & PERF_SAMPLE_CPU) 6197 perf_output_put(handle, data->cpu_entry); 6198 6199 if (sample_type & PERF_SAMPLE_PERIOD) 6200 perf_output_put(handle, data->period); 6201 6202 if (sample_type & PERF_SAMPLE_READ) 6203 perf_output_read(handle, event); 6204 6205 if (sample_type & PERF_SAMPLE_CALLCHAIN) { 6206 int size = 1; 6207 6208 size += data->callchain->nr; 6209 size *= sizeof(u64); 6210 __output_copy(handle, data->callchain, size); 6211 } 6212 6213 if (sample_type & PERF_SAMPLE_RAW) { 6214 struct perf_raw_record *raw = data->raw; 6215 6216 if (raw) { 6217 struct perf_raw_frag *frag = &raw->frag; 6218 6219 perf_output_put(handle, raw->size); 6220 do { 6221 if (frag->copy) { 6222 __output_custom(handle, frag->copy, 6223 frag->data, frag->size); 6224 } else { 6225 __output_copy(handle, frag->data, 6226 frag->size); 6227 } 6228 if (perf_raw_frag_last(frag)) 6229 break; 6230 frag = frag->next; 6231 } while (1); 6232 if (frag->pad) 6233 __output_skip(handle, NULL, frag->pad); 6234 } else { 6235 struct { 6236 u32 size; 6237 u32 data; 6238 } raw = { 6239 .size = sizeof(u32), 6240 .data = 0, 6241 }; 6242 perf_output_put(handle, raw); 6243 } 6244 } 6245 6246 if (sample_type & PERF_SAMPLE_BRANCH_STACK) { 6247 if (data->br_stack) { 6248 size_t size; 6249 6250 size = data->br_stack->nr 6251 * sizeof(struct perf_branch_entry); 6252 6253 perf_output_put(handle, data->br_stack->nr); 6254 perf_output_copy(handle, data->br_stack->entries, size); 6255 } else { 6256 /* 6257 * we always store at least the value of nr 6258 */ 6259 u64 nr = 0; 6260 perf_output_put(handle, nr); 6261 } 6262 } 6263 6264 if (sample_type & PERF_SAMPLE_REGS_USER) { 6265 u64 abi = data->regs_user.abi; 6266 6267 /* 6268 * If there are no regs to dump, notice it through 6269 * first u64 being zero (PERF_SAMPLE_REGS_ABI_NONE). 6270 */ 6271 perf_output_put(handle, abi); 6272 6273 if (abi) { 6274 u64 mask = event->attr.sample_regs_user; 6275 perf_output_sample_regs(handle, 6276 data->regs_user.regs, 6277 mask); 6278 } 6279 } 6280 6281 if (sample_type & PERF_SAMPLE_STACK_USER) { 6282 perf_output_sample_ustack(handle, 6283 data->stack_user_size, 6284 data->regs_user.regs); 6285 } 6286 6287 if (sample_type & PERF_SAMPLE_WEIGHT) 6288 perf_output_put(handle, data->weight); 6289 6290 if (sample_type & PERF_SAMPLE_DATA_SRC) 6291 perf_output_put(handle, data->data_src.val); 6292 6293 if (sample_type & PERF_SAMPLE_TRANSACTION) 6294 perf_output_put(handle, data->txn); 6295 6296 if (sample_type & PERF_SAMPLE_REGS_INTR) { 6297 u64 abi = data->regs_intr.abi; 6298 /* 6299 * If there are no regs to dump, notice it through 6300 * first u64 being zero (PERF_SAMPLE_REGS_ABI_NONE). 6301 */ 6302 perf_output_put(handle, abi); 6303 6304 if (abi) { 6305 u64 mask = event->attr.sample_regs_intr; 6306 6307 perf_output_sample_regs(handle, 6308 data->regs_intr.regs, 6309 mask); 6310 } 6311 } 6312 6313 if (sample_type & PERF_SAMPLE_PHYS_ADDR) 6314 perf_output_put(handle, data->phys_addr); 6315 6316 if (!event->attr.watermark) { 6317 int wakeup_events = event->attr.wakeup_events; 6318 6319 if (wakeup_events) { 6320 struct ring_buffer *rb = handle->rb; 6321 int events = local_inc_return(&rb->events); 6322 6323 if (events >= wakeup_events) { 6324 local_sub(wakeup_events, &rb->events); 6325 local_inc(&rb->wakeup); 6326 } 6327 } 6328 } 6329 } 6330 6331 static u64 perf_virt_to_phys(u64 virt) 6332 { 6333 u64 phys_addr = 0; 6334 struct page *p = NULL; 6335 6336 if (!virt) 6337 return 0; 6338 6339 if (virt >= TASK_SIZE) { 6340 /* If it's vmalloc()d memory, leave phys_addr as 0 */ 6341 if (virt_addr_valid((void *)(uintptr_t)virt) && 6342 !(virt >= VMALLOC_START && virt < VMALLOC_END)) 6343 phys_addr = (u64)virt_to_phys((void *)(uintptr_t)virt); 6344 } else { 6345 /* 6346 * Walking the pages tables for user address. 6347 * Interrupts are disabled, so it prevents any tear down 6348 * of the page tables. 6349 * Try IRQ-safe __get_user_pages_fast first. 6350 * If failed, leave phys_addr as 0. 6351 */ 6352 if ((current->mm != NULL) && 6353 (__get_user_pages_fast(virt, 1, 0, &p) == 1)) 6354 phys_addr = page_to_phys(p) + virt % PAGE_SIZE; 6355 6356 if (p) 6357 put_page(p); 6358 } 6359 6360 return phys_addr; 6361 } 6362 6363 static struct perf_callchain_entry __empty_callchain = { .nr = 0, }; 6364 6365 struct perf_callchain_entry * 6366 perf_callchain(struct perf_event *event, struct pt_regs *regs) 6367 { 6368 bool kernel = !event->attr.exclude_callchain_kernel; 6369 bool user = !event->attr.exclude_callchain_user; 6370 /* Disallow cross-task user callchains. */ 6371 bool crosstask = event->ctx->task && event->ctx->task != current; 6372 const u32 max_stack = event->attr.sample_max_stack; 6373 struct perf_callchain_entry *callchain; 6374 6375 if (!kernel && !user) 6376 return &__empty_callchain; 6377 6378 callchain = get_perf_callchain(regs, 0, kernel, user, 6379 max_stack, crosstask, true); 6380 return callchain ?: &__empty_callchain; 6381 } 6382 6383 void perf_prepare_sample(struct perf_event_header *header, 6384 struct perf_sample_data *data, 6385 struct perf_event *event, 6386 struct pt_regs *regs) 6387 { 6388 u64 sample_type = event->attr.sample_type; 6389 6390 header->type = PERF_RECORD_SAMPLE; 6391 header->size = sizeof(*header) + event->header_size; 6392 6393 header->misc = 0; 6394 header->misc |= perf_misc_flags(regs); 6395 6396 __perf_event_header__init_id(header, data, event); 6397 6398 if (sample_type & PERF_SAMPLE_IP) 6399 data->ip = perf_instruction_pointer(regs); 6400 6401 if (sample_type & PERF_SAMPLE_CALLCHAIN) { 6402 int size = 1; 6403 6404 if (!(sample_type & __PERF_SAMPLE_CALLCHAIN_EARLY)) 6405 data->callchain = perf_callchain(event, regs); 6406 6407 size += data->callchain->nr; 6408 6409 header->size += size * sizeof(u64); 6410 } 6411 6412 if (sample_type & PERF_SAMPLE_RAW) { 6413 struct perf_raw_record *raw = data->raw; 6414 int size; 6415 6416 if (raw) { 6417 struct perf_raw_frag *frag = &raw->frag; 6418 u32 sum = 0; 6419 6420 do { 6421 sum += frag->size; 6422 if (perf_raw_frag_last(frag)) 6423 break; 6424 frag = frag->next; 6425 } while (1); 6426 6427 size = round_up(sum + sizeof(u32), sizeof(u64)); 6428 raw->size = size - sizeof(u32); 6429 frag->pad = raw->size - sum; 6430 } else { 6431 size = sizeof(u64); 6432 } 6433 6434 header->size += size; 6435 } 6436 6437 if (sample_type & PERF_SAMPLE_BRANCH_STACK) { 6438 int size = sizeof(u64); /* nr */ 6439 if (data->br_stack) { 6440 size += data->br_stack->nr 6441 * sizeof(struct perf_branch_entry); 6442 } 6443 header->size += size; 6444 } 6445 6446 if (sample_type & (PERF_SAMPLE_REGS_USER | PERF_SAMPLE_STACK_USER)) 6447 perf_sample_regs_user(&data->regs_user, regs, 6448 &data->regs_user_copy); 6449 6450 if (sample_type & PERF_SAMPLE_REGS_USER) { 6451 /* regs dump ABI info */ 6452 int size = sizeof(u64); 6453 6454 if (data->regs_user.regs) { 6455 u64 mask = event->attr.sample_regs_user; 6456 size += hweight64(mask) * sizeof(u64); 6457 } 6458 6459 header->size += size; 6460 } 6461 6462 if (sample_type & PERF_SAMPLE_STACK_USER) { 6463 /* 6464 * Either we need PERF_SAMPLE_STACK_USER bit to be allways 6465 * processed as the last one or have additional check added 6466 * in case new sample type is added, because we could eat 6467 * up the rest of the sample size. 6468 */ 6469 u16 stack_size = event->attr.sample_stack_user; 6470 u16 size = sizeof(u64); 6471 6472 stack_size = perf_sample_ustack_size(stack_size, header->size, 6473 data->regs_user.regs); 6474 6475 /* 6476 * If there is something to dump, add space for the dump 6477 * itself and for the field that tells the dynamic size, 6478 * which is how many have been actually dumped. 6479 */ 6480 if (stack_size) 6481 size += sizeof(u64) + stack_size; 6482 6483 data->stack_user_size = stack_size; 6484 header->size += size; 6485 } 6486 6487 if (sample_type & PERF_SAMPLE_REGS_INTR) { 6488 /* regs dump ABI info */ 6489 int size = sizeof(u64); 6490 6491 perf_sample_regs_intr(&data->regs_intr, regs); 6492 6493 if (data->regs_intr.regs) { 6494 u64 mask = event->attr.sample_regs_intr; 6495 6496 size += hweight64(mask) * sizeof(u64); 6497 } 6498 6499 header->size += size; 6500 } 6501 6502 if (sample_type & PERF_SAMPLE_PHYS_ADDR) 6503 data->phys_addr = perf_virt_to_phys(data->addr); 6504 } 6505 6506 static __always_inline int 6507 __perf_event_output(struct perf_event *event, 6508 struct perf_sample_data *data, 6509 struct pt_regs *regs, 6510 int (*output_begin)(struct perf_output_handle *, 6511 struct perf_event *, 6512 unsigned int)) 6513 { 6514 struct perf_output_handle handle; 6515 struct perf_event_header header; 6516 int err; 6517 6518 /* protect the callchain buffers */ 6519 rcu_read_lock(); 6520 6521 perf_prepare_sample(&header, data, event, regs); 6522 6523 err = output_begin(&handle, event, header.size); 6524 if (err) 6525 goto exit; 6526 6527 perf_output_sample(&handle, &header, data, event); 6528 6529 perf_output_end(&handle); 6530 6531 exit: 6532 rcu_read_unlock(); 6533 return err; 6534 } 6535 6536 void 6537 perf_event_output_forward(struct perf_event *event, 6538 struct perf_sample_data *data, 6539 struct pt_regs *regs) 6540 { 6541 __perf_event_output(event, data, regs, perf_output_begin_forward); 6542 } 6543 6544 void 6545 perf_event_output_backward(struct perf_event *event, 6546 struct perf_sample_data *data, 6547 struct pt_regs *regs) 6548 { 6549 __perf_event_output(event, data, regs, perf_output_begin_backward); 6550 } 6551 6552 int 6553 perf_event_output(struct perf_event *event, 6554 struct perf_sample_data *data, 6555 struct pt_regs *regs) 6556 { 6557 return __perf_event_output(event, data, regs, perf_output_begin); 6558 } 6559 6560 /* 6561 * read event_id 6562 */ 6563 6564 struct perf_read_event { 6565 struct perf_event_header header; 6566 6567 u32 pid; 6568 u32 tid; 6569 }; 6570 6571 static void 6572 perf_event_read_event(struct perf_event *event, 6573 struct task_struct *task) 6574 { 6575 struct perf_output_handle handle; 6576 struct perf_sample_data sample; 6577 struct perf_read_event read_event = { 6578 .header = { 6579 .type = PERF_RECORD_READ, 6580 .misc = 0, 6581 .size = sizeof(read_event) + event->read_size, 6582 }, 6583 .pid = perf_event_pid(event, task), 6584 .tid = perf_event_tid(event, task), 6585 }; 6586 int ret; 6587 6588 perf_event_header__init_id(&read_event.header, &sample, event); 6589 ret = perf_output_begin(&handle, event, read_event.header.size); 6590 if (ret) 6591 return; 6592 6593 perf_output_put(&handle, read_event); 6594 perf_output_read(&handle, event); 6595 perf_event__output_id_sample(event, &handle, &sample); 6596 6597 perf_output_end(&handle); 6598 } 6599 6600 typedef void (perf_iterate_f)(struct perf_event *event, void *data); 6601 6602 static void 6603 perf_iterate_ctx(struct perf_event_context *ctx, 6604 perf_iterate_f output, 6605 void *data, bool all) 6606 { 6607 struct perf_event *event; 6608 6609 list_for_each_entry_rcu(event, &ctx->event_list, event_entry) { 6610 if (!all) { 6611 if (event->state < PERF_EVENT_STATE_INACTIVE) 6612 continue; 6613 if (!event_filter_match(event)) 6614 continue; 6615 } 6616 6617 output(event, data); 6618 } 6619 } 6620 6621 static void perf_iterate_sb_cpu(perf_iterate_f output, void *data) 6622 { 6623 struct pmu_event_list *pel = this_cpu_ptr(&pmu_sb_events); 6624 struct perf_event *event; 6625 6626 list_for_each_entry_rcu(event, &pel->list, sb_list) { 6627 /* 6628 * Skip events that are not fully formed yet; ensure that 6629 * if we observe event->ctx, both event and ctx will be 6630 * complete enough. See perf_install_in_context(). 6631 */ 6632 if (!smp_load_acquire(&event->ctx)) 6633 continue; 6634 6635 if (event->state < PERF_EVENT_STATE_INACTIVE) 6636 continue; 6637 if (!event_filter_match(event)) 6638 continue; 6639 output(event, data); 6640 } 6641 } 6642 6643 /* 6644 * Iterate all events that need to receive side-band events. 6645 * 6646 * For new callers; ensure that account_pmu_sb_event() includes 6647 * your event, otherwise it might not get delivered. 6648 */ 6649 static void 6650 perf_iterate_sb(perf_iterate_f output, void *data, 6651 struct perf_event_context *task_ctx) 6652 { 6653 struct perf_event_context *ctx; 6654 int ctxn; 6655 6656 rcu_read_lock(); 6657 preempt_disable(); 6658 6659 /* 6660 * If we have task_ctx != NULL we only notify the task context itself. 6661 * The task_ctx is set only for EXIT events before releasing task 6662 * context. 6663 */ 6664 if (task_ctx) { 6665 perf_iterate_ctx(task_ctx, output, data, false); 6666 goto done; 6667 } 6668 6669 perf_iterate_sb_cpu(output, data); 6670 6671 for_each_task_context_nr(ctxn) { 6672 ctx = rcu_dereference(current->perf_event_ctxp[ctxn]); 6673 if (ctx) 6674 perf_iterate_ctx(ctx, output, data, false); 6675 } 6676 done: 6677 preempt_enable(); 6678 rcu_read_unlock(); 6679 } 6680 6681 /* 6682 * Clear all file-based filters at exec, they'll have to be 6683 * re-instated when/if these objects are mmapped again. 6684 */ 6685 static void perf_event_addr_filters_exec(struct perf_event *event, void *data) 6686 { 6687 struct perf_addr_filters_head *ifh = perf_event_addr_filters(event); 6688 struct perf_addr_filter *filter; 6689 unsigned int restart = 0, count = 0; 6690 unsigned long flags; 6691 6692 if (!has_addr_filter(event)) 6693 return; 6694 6695 raw_spin_lock_irqsave(&ifh->lock, flags); 6696 list_for_each_entry(filter, &ifh->list, entry) { 6697 if (filter->path.dentry) { 6698 event->addr_filter_ranges[count].start = 0; 6699 event->addr_filter_ranges[count].size = 0; 6700 restart++; 6701 } 6702 6703 count++; 6704 } 6705 6706 if (restart) 6707 event->addr_filters_gen++; 6708 raw_spin_unlock_irqrestore(&ifh->lock, flags); 6709 6710 if (restart) 6711 perf_event_stop(event, 1); 6712 } 6713 6714 void perf_event_exec(void) 6715 { 6716 struct perf_event_context *ctx; 6717 int ctxn; 6718 6719 rcu_read_lock(); 6720 for_each_task_context_nr(ctxn) { 6721 ctx = current->perf_event_ctxp[ctxn]; 6722 if (!ctx) 6723 continue; 6724 6725 perf_event_enable_on_exec(ctxn); 6726 6727 perf_iterate_ctx(ctx, perf_event_addr_filters_exec, NULL, 6728 true); 6729 } 6730 rcu_read_unlock(); 6731 } 6732 6733 struct remote_output { 6734 struct ring_buffer *rb; 6735 int err; 6736 }; 6737 6738 static void __perf_event_output_stop(struct perf_event *event, void *data) 6739 { 6740 struct perf_event *parent = event->parent; 6741 struct remote_output *ro = data; 6742 struct ring_buffer *rb = ro->rb; 6743 struct stop_event_data sd = { 6744 .event = event, 6745 }; 6746 6747 if (!has_aux(event)) 6748 return; 6749 6750 if (!parent) 6751 parent = event; 6752 6753 /* 6754 * In case of inheritance, it will be the parent that links to the 6755 * ring-buffer, but it will be the child that's actually using it. 6756 * 6757 * We are using event::rb to determine if the event should be stopped, 6758 * however this may race with ring_buffer_attach() (through set_output), 6759 * which will make us skip the event that actually needs to be stopped. 6760 * So ring_buffer_attach() has to stop an aux event before re-assigning 6761 * its rb pointer. 6762 */ 6763 if (rcu_dereference(parent->rb) == rb) 6764 ro->err = __perf_event_stop(&sd); 6765 } 6766 6767 static int __perf_pmu_output_stop(void *info) 6768 { 6769 struct perf_event *event = info; 6770 struct pmu *pmu = event->pmu; 6771 struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context); 6772 struct remote_output ro = { 6773 .rb = event->rb, 6774 }; 6775 6776 rcu_read_lock(); 6777 perf_iterate_ctx(&cpuctx->ctx, __perf_event_output_stop, &ro, false); 6778 if (cpuctx->task_ctx) 6779 perf_iterate_ctx(cpuctx->task_ctx, __perf_event_output_stop, 6780 &ro, false); 6781 rcu_read_unlock(); 6782 6783 return ro.err; 6784 } 6785 6786 static void perf_pmu_output_stop(struct perf_event *event) 6787 { 6788 struct perf_event *iter; 6789 int err, cpu; 6790 6791 restart: 6792 rcu_read_lock(); 6793 list_for_each_entry_rcu(iter, &event->rb->event_list, rb_entry) { 6794 /* 6795 * For per-CPU events, we need to make sure that neither they 6796 * nor their children are running; for cpu==-1 events it's 6797 * sufficient to stop the event itself if it's active, since 6798 * it can't have children. 6799 */ 6800 cpu = iter->cpu; 6801 if (cpu == -1) 6802 cpu = READ_ONCE(iter->oncpu); 6803 6804 if (cpu == -1) 6805 continue; 6806 6807 err = cpu_function_call(cpu, __perf_pmu_output_stop, event); 6808 if (err == -EAGAIN) { 6809 rcu_read_unlock(); 6810 goto restart; 6811 } 6812 } 6813 rcu_read_unlock(); 6814 } 6815 6816 /* 6817 * task tracking -- fork/exit 6818 * 6819 * enabled by: attr.comm | attr.mmap | attr.mmap2 | attr.mmap_data | attr.task 6820 */ 6821 6822 struct perf_task_event { 6823 struct task_struct *task; 6824 struct perf_event_context *task_ctx; 6825 6826 struct { 6827 struct perf_event_header header; 6828 6829 u32 pid; 6830 u32 ppid; 6831 u32 tid; 6832 u32 ptid; 6833 u64 time; 6834 } event_id; 6835 }; 6836 6837 static int perf_event_task_match(struct perf_event *event) 6838 { 6839 return event->attr.comm || event->attr.mmap || 6840 event->attr.mmap2 || event->attr.mmap_data || 6841 event->attr.task; 6842 } 6843 6844 static void perf_event_task_output(struct perf_event *event, 6845 void *data) 6846 { 6847 struct perf_task_event *task_event = data; 6848 struct perf_output_handle handle; 6849 struct perf_sample_data sample; 6850 struct task_struct *task = task_event->task; 6851 int ret, size = task_event->event_id.header.size; 6852 6853 if (!perf_event_task_match(event)) 6854 return; 6855 6856 perf_event_header__init_id(&task_event->event_id.header, &sample, event); 6857 6858 ret = perf_output_begin(&handle, event, 6859 task_event->event_id.header.size); 6860 if (ret) 6861 goto out; 6862 6863 task_event->event_id.pid = perf_event_pid(event, task); 6864 task_event->event_id.ppid = perf_event_pid(event, current); 6865 6866 task_event->event_id.tid = perf_event_tid(event, task); 6867 task_event->event_id.ptid = perf_event_tid(event, current); 6868 6869 task_event->event_id.time = perf_event_clock(event); 6870 6871 perf_output_put(&handle, task_event->event_id); 6872 6873 perf_event__output_id_sample(event, &handle, &sample); 6874 6875 perf_output_end(&handle); 6876 out: 6877 task_event->event_id.header.size = size; 6878 } 6879 6880 static void perf_event_task(struct task_struct *task, 6881 struct perf_event_context *task_ctx, 6882 int new) 6883 { 6884 struct perf_task_event task_event; 6885 6886 if (!atomic_read(&nr_comm_events) && 6887 !atomic_read(&nr_mmap_events) && 6888 !atomic_read(&nr_task_events)) 6889 return; 6890 6891 task_event = (struct perf_task_event){ 6892 .task = task, 6893 .task_ctx = task_ctx, 6894 .event_id = { 6895 .header = { 6896 .type = new ? PERF_RECORD_FORK : PERF_RECORD_EXIT, 6897 .misc = 0, 6898 .size = sizeof(task_event.event_id), 6899 }, 6900 /* .pid */ 6901 /* .ppid */ 6902 /* .tid */ 6903 /* .ptid */ 6904 /* .time */ 6905 }, 6906 }; 6907 6908 perf_iterate_sb(perf_event_task_output, 6909 &task_event, 6910 task_ctx); 6911 } 6912 6913 void perf_event_fork(struct task_struct *task) 6914 { 6915 perf_event_task(task, NULL, 1); 6916 perf_event_namespaces(task); 6917 } 6918 6919 /* 6920 * comm tracking 6921 */ 6922 6923 struct perf_comm_event { 6924 struct task_struct *task; 6925 char *comm; 6926 int comm_size; 6927 6928 struct { 6929 struct perf_event_header header; 6930 6931 u32 pid; 6932 u32 tid; 6933 } event_id; 6934 }; 6935 6936 static int perf_event_comm_match(struct perf_event *event) 6937 { 6938 return event->attr.comm; 6939 } 6940 6941 static void perf_event_comm_output(struct perf_event *event, 6942 void *data) 6943 { 6944 struct perf_comm_event *comm_event = data; 6945 struct perf_output_handle handle; 6946 struct perf_sample_data sample; 6947 int size = comm_event->event_id.header.size; 6948 int ret; 6949 6950 if (!perf_event_comm_match(event)) 6951 return; 6952 6953 perf_event_header__init_id(&comm_event->event_id.header, &sample, event); 6954 ret = perf_output_begin(&handle, event, 6955 comm_event->event_id.header.size); 6956 6957 if (ret) 6958 goto out; 6959 6960 comm_event->event_id.pid = perf_event_pid(event, comm_event->task); 6961 comm_event->event_id.tid = perf_event_tid(event, comm_event->task); 6962 6963 perf_output_put(&handle, comm_event->event_id); 6964 __output_copy(&handle, comm_event->comm, 6965 comm_event->comm_size); 6966 6967 perf_event__output_id_sample(event, &handle, &sample); 6968 6969 perf_output_end(&handle); 6970 out: 6971 comm_event->event_id.header.size = size; 6972 } 6973 6974 static void perf_event_comm_event(struct perf_comm_event *comm_event) 6975 { 6976 char comm[TASK_COMM_LEN]; 6977 unsigned int size; 6978 6979 memset(comm, 0, sizeof(comm)); 6980 strlcpy(comm, comm_event->task->comm, sizeof(comm)); 6981 size = ALIGN(strlen(comm)+1, sizeof(u64)); 6982 6983 comm_event->comm = comm; 6984 comm_event->comm_size = size; 6985 6986 comm_event->event_id.header.size = sizeof(comm_event->event_id) + size; 6987 6988 perf_iterate_sb(perf_event_comm_output, 6989 comm_event, 6990 NULL); 6991 } 6992 6993 void perf_event_comm(struct task_struct *task, bool exec) 6994 { 6995 struct perf_comm_event comm_event; 6996 6997 if (!atomic_read(&nr_comm_events)) 6998 return; 6999 7000 comm_event = (struct perf_comm_event){ 7001 .task = task, 7002 /* .comm */ 7003 /* .comm_size */ 7004 .event_id = { 7005 .header = { 7006 .type = PERF_RECORD_COMM, 7007 .misc = exec ? PERF_RECORD_MISC_COMM_EXEC : 0, 7008 /* .size */ 7009 }, 7010 /* .pid */ 7011 /* .tid */ 7012 }, 7013 }; 7014 7015 perf_event_comm_event(&comm_event); 7016 } 7017 7018 /* 7019 * namespaces tracking 7020 */ 7021 7022 struct perf_namespaces_event { 7023 struct task_struct *task; 7024 7025 struct { 7026 struct perf_event_header header; 7027 7028 u32 pid; 7029 u32 tid; 7030 u64 nr_namespaces; 7031 struct perf_ns_link_info link_info[NR_NAMESPACES]; 7032 } event_id; 7033 }; 7034 7035 static int perf_event_namespaces_match(struct perf_event *event) 7036 { 7037 return event->attr.namespaces; 7038 } 7039 7040 static void perf_event_namespaces_output(struct perf_event *event, 7041 void *data) 7042 { 7043 struct perf_namespaces_event *namespaces_event = data; 7044 struct perf_output_handle handle; 7045 struct perf_sample_data sample; 7046 u16 header_size = namespaces_event->event_id.header.size; 7047 int ret; 7048 7049 if (!perf_event_namespaces_match(event)) 7050 return; 7051 7052 perf_event_header__init_id(&namespaces_event->event_id.header, 7053 &sample, event); 7054 ret = perf_output_begin(&handle, event, 7055 namespaces_event->event_id.header.size); 7056 if (ret) 7057 goto out; 7058 7059 namespaces_event->event_id.pid = perf_event_pid(event, 7060 namespaces_event->task); 7061 namespaces_event->event_id.tid = perf_event_tid(event, 7062 namespaces_event->task); 7063 7064 perf_output_put(&handle, namespaces_event->event_id); 7065 7066 perf_event__output_id_sample(event, &handle, &sample); 7067 7068 perf_output_end(&handle); 7069 out: 7070 namespaces_event->event_id.header.size = header_size; 7071 } 7072 7073 static void perf_fill_ns_link_info(struct perf_ns_link_info *ns_link_info, 7074 struct task_struct *task, 7075 const struct proc_ns_operations *ns_ops) 7076 { 7077 struct path ns_path; 7078 struct inode *ns_inode; 7079 void *error; 7080 7081 error = ns_get_path(&ns_path, task, ns_ops); 7082 if (!error) { 7083 ns_inode = ns_path.dentry->d_inode; 7084 ns_link_info->dev = new_encode_dev(ns_inode->i_sb->s_dev); 7085 ns_link_info->ino = ns_inode->i_ino; 7086 path_put(&ns_path); 7087 } 7088 } 7089 7090 void perf_event_namespaces(struct task_struct *task) 7091 { 7092 struct perf_namespaces_event namespaces_event; 7093 struct perf_ns_link_info *ns_link_info; 7094 7095 if (!atomic_read(&nr_namespaces_events)) 7096 return; 7097 7098 namespaces_event = (struct perf_namespaces_event){ 7099 .task = task, 7100 .event_id = { 7101 .header = { 7102 .type = PERF_RECORD_NAMESPACES, 7103 .misc = 0, 7104 .size = sizeof(namespaces_event.event_id), 7105 }, 7106 /* .pid */ 7107 /* .tid */ 7108 .nr_namespaces = NR_NAMESPACES, 7109 /* .link_info[NR_NAMESPACES] */ 7110 }, 7111 }; 7112 7113 ns_link_info = namespaces_event.event_id.link_info; 7114 7115 perf_fill_ns_link_info(&ns_link_info[MNT_NS_INDEX], 7116 task, &mntns_operations); 7117 7118 #ifdef CONFIG_USER_NS 7119 perf_fill_ns_link_info(&ns_link_info[USER_NS_INDEX], 7120 task, &userns_operations); 7121 #endif 7122 #ifdef CONFIG_NET_NS 7123 perf_fill_ns_link_info(&ns_link_info[NET_NS_INDEX], 7124 task, &netns_operations); 7125 #endif 7126 #ifdef CONFIG_UTS_NS 7127 perf_fill_ns_link_info(&ns_link_info[UTS_NS_INDEX], 7128 task, &utsns_operations); 7129 #endif 7130 #ifdef CONFIG_IPC_NS 7131 perf_fill_ns_link_info(&ns_link_info[IPC_NS_INDEX], 7132 task, &ipcns_operations); 7133 #endif 7134 #ifdef CONFIG_PID_NS 7135 perf_fill_ns_link_info(&ns_link_info[PID_NS_INDEX], 7136 task, &pidns_operations); 7137 #endif 7138 #ifdef CONFIG_CGROUPS 7139 perf_fill_ns_link_info(&ns_link_info[CGROUP_NS_INDEX], 7140 task, &cgroupns_operations); 7141 #endif 7142 7143 perf_iterate_sb(perf_event_namespaces_output, 7144 &namespaces_event, 7145 NULL); 7146 } 7147 7148 /* 7149 * mmap tracking 7150 */ 7151 7152 struct perf_mmap_event { 7153 struct vm_area_struct *vma; 7154 7155 const char *file_name; 7156 int file_size; 7157 int maj, min; 7158 u64 ino; 7159 u64 ino_generation; 7160 u32 prot, flags; 7161 7162 struct { 7163 struct perf_event_header header; 7164 7165 u32 pid; 7166 u32 tid; 7167 u64 start; 7168 u64 len; 7169 u64 pgoff; 7170 } event_id; 7171 }; 7172 7173 static int perf_event_mmap_match(struct perf_event *event, 7174 void *data) 7175 { 7176 struct perf_mmap_event *mmap_event = data; 7177 struct vm_area_struct *vma = mmap_event->vma; 7178 int executable = vma->vm_flags & VM_EXEC; 7179 7180 return (!executable && event->attr.mmap_data) || 7181 (executable && (event->attr.mmap || event->attr.mmap2)); 7182 } 7183 7184 static void perf_event_mmap_output(struct perf_event *event, 7185 void *data) 7186 { 7187 struct perf_mmap_event *mmap_event = data; 7188 struct perf_output_handle handle; 7189 struct perf_sample_data sample; 7190 int size = mmap_event->event_id.header.size; 7191 int ret; 7192 7193 if (!perf_event_mmap_match(event, data)) 7194 return; 7195 7196 if (event->attr.mmap2) { 7197 mmap_event->event_id.header.type = PERF_RECORD_MMAP2; 7198 mmap_event->event_id.header.size += sizeof(mmap_event->maj); 7199 mmap_event->event_id.header.size += sizeof(mmap_event->min); 7200 mmap_event->event_id.header.size += sizeof(mmap_event->ino); 7201 mmap_event->event_id.header.size += sizeof(mmap_event->ino_generation); 7202 mmap_event->event_id.header.size += sizeof(mmap_event->prot); 7203 mmap_event->event_id.header.size += sizeof(mmap_event->flags); 7204 } 7205 7206 perf_event_header__init_id(&mmap_event->event_id.header, &sample, event); 7207 ret = perf_output_begin(&handle, event, 7208 mmap_event->event_id.header.size); 7209 if (ret) 7210 goto out; 7211 7212 mmap_event->event_id.pid = perf_event_pid(event, current); 7213 mmap_event->event_id.tid = perf_event_tid(event, current); 7214 7215 perf_output_put(&handle, mmap_event->event_id); 7216 7217 if (event->attr.mmap2) { 7218 perf_output_put(&handle, mmap_event->maj); 7219 perf_output_put(&handle, mmap_event->min); 7220 perf_output_put(&handle, mmap_event->ino); 7221 perf_output_put(&handle, mmap_event->ino_generation); 7222 perf_output_put(&handle, mmap_event->prot); 7223 perf_output_put(&handle, mmap_event->flags); 7224 } 7225 7226 __output_copy(&handle, mmap_event->file_name, 7227 mmap_event->file_size); 7228 7229 perf_event__output_id_sample(event, &handle, &sample); 7230 7231 perf_output_end(&handle); 7232 out: 7233 mmap_event->event_id.header.size = size; 7234 } 7235 7236 static void perf_event_mmap_event(struct perf_mmap_event *mmap_event) 7237 { 7238 struct vm_area_struct *vma = mmap_event->vma; 7239 struct file *file = vma->vm_file; 7240 int maj = 0, min = 0; 7241 u64 ino = 0, gen = 0; 7242 u32 prot = 0, flags = 0; 7243 unsigned int size; 7244 char tmp[16]; 7245 char *buf = NULL; 7246 char *name; 7247 7248 if (vma->vm_flags & VM_READ) 7249 prot |= PROT_READ; 7250 if (vma->vm_flags & VM_WRITE) 7251 prot |= PROT_WRITE; 7252 if (vma->vm_flags & VM_EXEC) 7253 prot |= PROT_EXEC; 7254 7255 if (vma->vm_flags & VM_MAYSHARE) 7256 flags = MAP_SHARED; 7257 else 7258 flags = MAP_PRIVATE; 7259 7260 if (vma->vm_flags & VM_DENYWRITE) 7261 flags |= MAP_DENYWRITE; 7262 if (vma->vm_flags & VM_MAYEXEC) 7263 flags |= MAP_EXECUTABLE; 7264 if (vma->vm_flags & VM_LOCKED) 7265 flags |= MAP_LOCKED; 7266 if (vma->vm_flags & VM_HUGETLB) 7267 flags |= MAP_HUGETLB; 7268 7269 if (file) { 7270 struct inode *inode; 7271 dev_t dev; 7272 7273 buf = kmalloc(PATH_MAX, GFP_KERNEL); 7274 if (!buf) { 7275 name = "//enomem"; 7276 goto cpy_name; 7277 } 7278 /* 7279 * d_path() works from the end of the rb backwards, so we 7280 * need to add enough zero bytes after the string to handle 7281 * the 64bit alignment we do later. 7282 */ 7283 name = file_path(file, buf, PATH_MAX - sizeof(u64)); 7284 if (IS_ERR(name)) { 7285 name = "//toolong"; 7286 goto cpy_name; 7287 } 7288 inode = file_inode(vma->vm_file); 7289 dev = inode->i_sb->s_dev; 7290 ino = inode->i_ino; 7291 gen = inode->i_generation; 7292 maj = MAJOR(dev); 7293 min = MINOR(dev); 7294 7295 goto got_name; 7296 } else { 7297 if (vma->vm_ops && vma->vm_ops->name) { 7298 name = (char *) vma->vm_ops->name(vma); 7299 if (name) 7300 goto cpy_name; 7301 } 7302 7303 name = (char *)arch_vma_name(vma); 7304 if (name) 7305 goto cpy_name; 7306 7307 if (vma->vm_start <= vma->vm_mm->start_brk && 7308 vma->vm_end >= vma->vm_mm->brk) { 7309 name = "[heap]"; 7310 goto cpy_name; 7311 } 7312 if (vma->vm_start <= vma->vm_mm->start_stack && 7313 vma->vm_end >= vma->vm_mm->start_stack) { 7314 name = "[stack]"; 7315 goto cpy_name; 7316 } 7317 7318 name = "//anon"; 7319 goto cpy_name; 7320 } 7321 7322 cpy_name: 7323 strlcpy(tmp, name, sizeof(tmp)); 7324 name = tmp; 7325 got_name: 7326 /* 7327 * Since our buffer works in 8 byte units we need to align our string 7328 * size to a multiple of 8. However, we must guarantee the tail end is 7329 * zero'd out to avoid leaking random bits to userspace. 7330 */ 7331 size = strlen(name)+1; 7332 while (!IS_ALIGNED(size, sizeof(u64))) 7333 name[size++] = '\0'; 7334 7335 mmap_event->file_name = name; 7336 mmap_event->file_size = size; 7337 mmap_event->maj = maj; 7338 mmap_event->min = min; 7339 mmap_event->ino = ino; 7340 mmap_event->ino_generation = gen; 7341 mmap_event->prot = prot; 7342 mmap_event->flags = flags; 7343 7344 if (!(vma->vm_flags & VM_EXEC)) 7345 mmap_event->event_id.header.misc |= PERF_RECORD_MISC_MMAP_DATA; 7346 7347 mmap_event->event_id.header.size = sizeof(mmap_event->event_id) + size; 7348 7349 perf_iterate_sb(perf_event_mmap_output, 7350 mmap_event, 7351 NULL); 7352 7353 kfree(buf); 7354 } 7355 7356 /* 7357 * Check whether inode and address range match filter criteria. 7358 */ 7359 static bool perf_addr_filter_match(struct perf_addr_filter *filter, 7360 struct file *file, unsigned long offset, 7361 unsigned long size) 7362 { 7363 /* d_inode(NULL) won't be equal to any mapped user-space file */ 7364 if (!filter->path.dentry) 7365 return false; 7366 7367 if (d_inode(filter->path.dentry) != file_inode(file)) 7368 return false; 7369 7370 if (filter->offset > offset + size) 7371 return false; 7372 7373 if (filter->offset + filter->size < offset) 7374 return false; 7375 7376 return true; 7377 } 7378 7379 static bool perf_addr_filter_vma_adjust(struct perf_addr_filter *filter, 7380 struct vm_area_struct *vma, 7381 struct perf_addr_filter_range *fr) 7382 { 7383 unsigned long vma_size = vma->vm_end - vma->vm_start; 7384 unsigned long off = vma->vm_pgoff << PAGE_SHIFT; 7385 struct file *file = vma->vm_file; 7386 7387 if (!perf_addr_filter_match(filter, file, off, vma_size)) 7388 return false; 7389 7390 if (filter->offset < off) { 7391 fr->start = vma->vm_start; 7392 fr->size = min(vma_size, filter->size - (off - filter->offset)); 7393 } else { 7394 fr->start = vma->vm_start + filter->offset - off; 7395 fr->size = min(vma->vm_end - fr->start, filter->size); 7396 } 7397 7398 return true; 7399 } 7400 7401 static void __perf_addr_filters_adjust(struct perf_event *event, void *data) 7402 { 7403 struct perf_addr_filters_head *ifh = perf_event_addr_filters(event); 7404 struct vm_area_struct *vma = data; 7405 struct perf_addr_filter *filter; 7406 unsigned int restart = 0, count = 0; 7407 unsigned long flags; 7408 7409 if (!has_addr_filter(event)) 7410 return; 7411 7412 if (!vma->vm_file) 7413 return; 7414 7415 raw_spin_lock_irqsave(&ifh->lock, flags); 7416 list_for_each_entry(filter, &ifh->list, entry) { 7417 if (perf_addr_filter_vma_adjust(filter, vma, 7418 &event->addr_filter_ranges[count])) 7419 restart++; 7420 7421 count++; 7422 } 7423 7424 if (restart) 7425 event->addr_filters_gen++; 7426 raw_spin_unlock_irqrestore(&ifh->lock, flags); 7427 7428 if (restart) 7429 perf_event_stop(event, 1); 7430 } 7431 7432 /* 7433 * Adjust all task's events' filters to the new vma 7434 */ 7435 static void perf_addr_filters_adjust(struct vm_area_struct *vma) 7436 { 7437 struct perf_event_context *ctx; 7438 int ctxn; 7439 7440 /* 7441 * Data tracing isn't supported yet and as such there is no need 7442 * to keep track of anything that isn't related to executable code: 7443 */ 7444 if (!(vma->vm_flags & VM_EXEC)) 7445 return; 7446 7447 rcu_read_lock(); 7448 for_each_task_context_nr(ctxn) { 7449 ctx = rcu_dereference(current->perf_event_ctxp[ctxn]); 7450 if (!ctx) 7451 continue; 7452 7453 perf_iterate_ctx(ctx, __perf_addr_filters_adjust, vma, true); 7454 } 7455 rcu_read_unlock(); 7456 } 7457 7458 void perf_event_mmap(struct vm_area_struct *vma) 7459 { 7460 struct perf_mmap_event mmap_event; 7461 7462 if (!atomic_read(&nr_mmap_events)) 7463 return; 7464 7465 mmap_event = (struct perf_mmap_event){ 7466 .vma = vma, 7467 /* .file_name */ 7468 /* .file_size */ 7469 .event_id = { 7470 .header = { 7471 .type = PERF_RECORD_MMAP, 7472 .misc = PERF_RECORD_MISC_USER, 7473 /* .size */ 7474 }, 7475 /* .pid */ 7476 /* .tid */ 7477 .start = vma->vm_start, 7478 .len = vma->vm_end - vma->vm_start, 7479 .pgoff = (u64)vma->vm_pgoff << PAGE_SHIFT, 7480 }, 7481 /* .maj (attr_mmap2 only) */ 7482 /* .min (attr_mmap2 only) */ 7483 /* .ino (attr_mmap2 only) */ 7484 /* .ino_generation (attr_mmap2 only) */ 7485 /* .prot (attr_mmap2 only) */ 7486 /* .flags (attr_mmap2 only) */ 7487 }; 7488 7489 perf_addr_filters_adjust(vma); 7490 perf_event_mmap_event(&mmap_event); 7491 } 7492 7493 void perf_event_aux_event(struct perf_event *event, unsigned long head, 7494 unsigned long size, u64 flags) 7495 { 7496 struct perf_output_handle handle; 7497 struct perf_sample_data sample; 7498 struct perf_aux_event { 7499 struct perf_event_header header; 7500 u64 offset; 7501 u64 size; 7502 u64 flags; 7503 } rec = { 7504 .header = { 7505 .type = PERF_RECORD_AUX, 7506 .misc = 0, 7507 .size = sizeof(rec), 7508 }, 7509 .offset = head, 7510 .size = size, 7511 .flags = flags, 7512 }; 7513 int ret; 7514 7515 perf_event_header__init_id(&rec.header, &sample, event); 7516 ret = perf_output_begin(&handle, event, rec.header.size); 7517 7518 if (ret) 7519 return; 7520 7521 perf_output_put(&handle, rec); 7522 perf_event__output_id_sample(event, &handle, &sample); 7523 7524 perf_output_end(&handle); 7525 } 7526 7527 /* 7528 * Lost/dropped samples logging 7529 */ 7530 void perf_log_lost_samples(struct perf_event *event, u64 lost) 7531 { 7532 struct perf_output_handle handle; 7533 struct perf_sample_data sample; 7534 int ret; 7535 7536 struct { 7537 struct perf_event_header header; 7538 u64 lost; 7539 } lost_samples_event = { 7540 .header = { 7541 .type = PERF_RECORD_LOST_SAMPLES, 7542 .misc = 0, 7543 .size = sizeof(lost_samples_event), 7544 }, 7545 .lost = lost, 7546 }; 7547 7548 perf_event_header__init_id(&lost_samples_event.header, &sample, event); 7549 7550 ret = perf_output_begin(&handle, event, 7551 lost_samples_event.header.size); 7552 if (ret) 7553 return; 7554 7555 perf_output_put(&handle, lost_samples_event); 7556 perf_event__output_id_sample(event, &handle, &sample); 7557 perf_output_end(&handle); 7558 } 7559 7560 /* 7561 * context_switch tracking 7562 */ 7563 7564 struct perf_switch_event { 7565 struct task_struct *task; 7566 struct task_struct *next_prev; 7567 7568 struct { 7569 struct perf_event_header header; 7570 u32 next_prev_pid; 7571 u32 next_prev_tid; 7572 } event_id; 7573 }; 7574 7575 static int perf_event_switch_match(struct perf_event *event) 7576 { 7577 return event->attr.context_switch; 7578 } 7579 7580 static void perf_event_switch_output(struct perf_event *event, void *data) 7581 { 7582 struct perf_switch_event *se = data; 7583 struct perf_output_handle handle; 7584 struct perf_sample_data sample; 7585 int ret; 7586 7587 if (!perf_event_switch_match(event)) 7588 return; 7589 7590 /* Only CPU-wide events are allowed to see next/prev pid/tid */ 7591 if (event->ctx->task) { 7592 se->event_id.header.type = PERF_RECORD_SWITCH; 7593 se->event_id.header.size = sizeof(se->event_id.header); 7594 } else { 7595 se->event_id.header.type = PERF_RECORD_SWITCH_CPU_WIDE; 7596 se->event_id.header.size = sizeof(se->event_id); 7597 se->event_id.next_prev_pid = 7598 perf_event_pid(event, se->next_prev); 7599 se->event_id.next_prev_tid = 7600 perf_event_tid(event, se->next_prev); 7601 } 7602 7603 perf_event_header__init_id(&se->event_id.header, &sample, event); 7604 7605 ret = perf_output_begin(&handle, event, se->event_id.header.size); 7606 if (ret) 7607 return; 7608 7609 if (event->ctx->task) 7610 perf_output_put(&handle, se->event_id.header); 7611 else 7612 perf_output_put(&handle, se->event_id); 7613 7614 perf_event__output_id_sample(event, &handle, &sample); 7615 7616 perf_output_end(&handle); 7617 } 7618 7619 static void perf_event_switch(struct task_struct *task, 7620 struct task_struct *next_prev, bool sched_in) 7621 { 7622 struct perf_switch_event switch_event; 7623 7624 /* N.B. caller checks nr_switch_events != 0 */ 7625 7626 switch_event = (struct perf_switch_event){ 7627 .task = task, 7628 .next_prev = next_prev, 7629 .event_id = { 7630 .header = { 7631 /* .type */ 7632 .misc = sched_in ? 0 : PERF_RECORD_MISC_SWITCH_OUT, 7633 /* .size */ 7634 }, 7635 /* .next_prev_pid */ 7636 /* .next_prev_tid */ 7637 }, 7638 }; 7639 7640 if (!sched_in && task->state == TASK_RUNNING) 7641 switch_event.event_id.header.misc |= 7642 PERF_RECORD_MISC_SWITCH_OUT_PREEMPT; 7643 7644 perf_iterate_sb(perf_event_switch_output, 7645 &switch_event, 7646 NULL); 7647 } 7648 7649 /* 7650 * IRQ throttle logging 7651 */ 7652 7653 static void perf_log_throttle(struct perf_event *event, int enable) 7654 { 7655 struct perf_output_handle handle; 7656 struct perf_sample_data sample; 7657 int ret; 7658 7659 struct { 7660 struct perf_event_header header; 7661 u64 time; 7662 u64 id; 7663 u64 stream_id; 7664 } throttle_event = { 7665 .header = { 7666 .type = PERF_RECORD_THROTTLE, 7667 .misc = 0, 7668 .size = sizeof(throttle_event), 7669 }, 7670 .time = perf_event_clock(event), 7671 .id = primary_event_id(event), 7672 .stream_id = event->id, 7673 }; 7674 7675 if (enable) 7676 throttle_event.header.type = PERF_RECORD_UNTHROTTLE; 7677 7678 perf_event_header__init_id(&throttle_event.header, &sample, event); 7679 7680 ret = perf_output_begin(&handle, event, 7681 throttle_event.header.size); 7682 if (ret) 7683 return; 7684 7685 perf_output_put(&handle, throttle_event); 7686 perf_event__output_id_sample(event, &handle, &sample); 7687 perf_output_end(&handle); 7688 } 7689 7690 /* 7691 * ksymbol register/unregister tracking 7692 */ 7693 7694 struct perf_ksymbol_event { 7695 const char *name; 7696 int name_len; 7697 struct { 7698 struct perf_event_header header; 7699 u64 addr; 7700 u32 len; 7701 u16 ksym_type; 7702 u16 flags; 7703 } event_id; 7704 }; 7705 7706 static int perf_event_ksymbol_match(struct perf_event *event) 7707 { 7708 return event->attr.ksymbol; 7709 } 7710 7711 static void perf_event_ksymbol_output(struct perf_event *event, void *data) 7712 { 7713 struct perf_ksymbol_event *ksymbol_event = data; 7714 struct perf_output_handle handle; 7715 struct perf_sample_data sample; 7716 int ret; 7717 7718 if (!perf_event_ksymbol_match(event)) 7719 return; 7720 7721 perf_event_header__init_id(&ksymbol_event->event_id.header, 7722 &sample, event); 7723 ret = perf_output_begin(&handle, event, 7724 ksymbol_event->event_id.header.size); 7725 if (ret) 7726 return; 7727 7728 perf_output_put(&handle, ksymbol_event->event_id); 7729 __output_copy(&handle, ksymbol_event->name, ksymbol_event->name_len); 7730 perf_event__output_id_sample(event, &handle, &sample); 7731 7732 perf_output_end(&handle); 7733 } 7734 7735 void perf_event_ksymbol(u16 ksym_type, u64 addr, u32 len, bool unregister, 7736 const char *sym) 7737 { 7738 struct perf_ksymbol_event ksymbol_event; 7739 char name[KSYM_NAME_LEN]; 7740 u16 flags = 0; 7741 int name_len; 7742 7743 if (!atomic_read(&nr_ksymbol_events)) 7744 return; 7745 7746 if (ksym_type >= PERF_RECORD_KSYMBOL_TYPE_MAX || 7747 ksym_type == PERF_RECORD_KSYMBOL_TYPE_UNKNOWN) 7748 goto err; 7749 7750 strlcpy(name, sym, KSYM_NAME_LEN); 7751 name_len = strlen(name) + 1; 7752 while (!IS_ALIGNED(name_len, sizeof(u64))) 7753 name[name_len++] = '\0'; 7754 BUILD_BUG_ON(KSYM_NAME_LEN % sizeof(u64)); 7755 7756 if (unregister) 7757 flags |= PERF_RECORD_KSYMBOL_FLAGS_UNREGISTER; 7758 7759 ksymbol_event = (struct perf_ksymbol_event){ 7760 .name = name, 7761 .name_len = name_len, 7762 .event_id = { 7763 .header = { 7764 .type = PERF_RECORD_KSYMBOL, 7765 .size = sizeof(ksymbol_event.event_id) + 7766 name_len, 7767 }, 7768 .addr = addr, 7769 .len = len, 7770 .ksym_type = ksym_type, 7771 .flags = flags, 7772 }, 7773 }; 7774 7775 perf_iterate_sb(perf_event_ksymbol_output, &ksymbol_event, NULL); 7776 return; 7777 err: 7778 WARN_ONCE(1, "%s: Invalid KSYMBOL type 0x%x\n", __func__, ksym_type); 7779 } 7780 7781 /* 7782 * bpf program load/unload tracking 7783 */ 7784 7785 struct perf_bpf_event { 7786 struct bpf_prog *prog; 7787 struct { 7788 struct perf_event_header header; 7789 u16 type; 7790 u16 flags; 7791 u32 id; 7792 u8 tag[BPF_TAG_SIZE]; 7793 } event_id; 7794 }; 7795 7796 static int perf_event_bpf_match(struct perf_event *event) 7797 { 7798 return event->attr.bpf_event; 7799 } 7800 7801 static void perf_event_bpf_output(struct perf_event *event, void *data) 7802 { 7803 struct perf_bpf_event *bpf_event = data; 7804 struct perf_output_handle handle; 7805 struct perf_sample_data sample; 7806 int ret; 7807 7808 if (!perf_event_bpf_match(event)) 7809 return; 7810 7811 perf_event_header__init_id(&bpf_event->event_id.header, 7812 &sample, event); 7813 ret = perf_output_begin(&handle, event, 7814 bpf_event->event_id.header.size); 7815 if (ret) 7816 return; 7817 7818 perf_output_put(&handle, bpf_event->event_id); 7819 perf_event__output_id_sample(event, &handle, &sample); 7820 7821 perf_output_end(&handle); 7822 } 7823 7824 static void perf_event_bpf_emit_ksymbols(struct bpf_prog *prog, 7825 enum perf_bpf_event_type type) 7826 { 7827 bool unregister = type == PERF_BPF_EVENT_PROG_UNLOAD; 7828 char sym[KSYM_NAME_LEN]; 7829 int i; 7830 7831 if (prog->aux->func_cnt == 0) { 7832 bpf_get_prog_name(prog, sym); 7833 perf_event_ksymbol(PERF_RECORD_KSYMBOL_TYPE_BPF, 7834 (u64)(unsigned long)prog->bpf_func, 7835 prog->jited_len, unregister, sym); 7836 } else { 7837 for (i = 0; i < prog->aux->func_cnt; i++) { 7838 struct bpf_prog *subprog = prog->aux->func[i]; 7839 7840 bpf_get_prog_name(subprog, sym); 7841 perf_event_ksymbol( 7842 PERF_RECORD_KSYMBOL_TYPE_BPF, 7843 (u64)(unsigned long)subprog->bpf_func, 7844 subprog->jited_len, unregister, sym); 7845 } 7846 } 7847 } 7848 7849 void perf_event_bpf_event(struct bpf_prog *prog, 7850 enum perf_bpf_event_type type, 7851 u16 flags) 7852 { 7853 struct perf_bpf_event bpf_event; 7854 7855 if (type <= PERF_BPF_EVENT_UNKNOWN || 7856 type >= PERF_BPF_EVENT_MAX) 7857 return; 7858 7859 switch (type) { 7860 case PERF_BPF_EVENT_PROG_LOAD: 7861 case PERF_BPF_EVENT_PROG_UNLOAD: 7862 if (atomic_read(&nr_ksymbol_events)) 7863 perf_event_bpf_emit_ksymbols(prog, type); 7864 break; 7865 default: 7866 break; 7867 } 7868 7869 if (!atomic_read(&nr_bpf_events)) 7870 return; 7871 7872 bpf_event = (struct perf_bpf_event){ 7873 .prog = prog, 7874 .event_id = { 7875 .header = { 7876 .type = PERF_RECORD_BPF_EVENT, 7877 .size = sizeof(bpf_event.event_id), 7878 }, 7879 .type = type, 7880 .flags = flags, 7881 .id = prog->aux->id, 7882 }, 7883 }; 7884 7885 BUILD_BUG_ON(BPF_TAG_SIZE % sizeof(u64)); 7886 7887 memcpy(bpf_event.event_id.tag, prog->tag, BPF_TAG_SIZE); 7888 perf_iterate_sb(perf_event_bpf_output, &bpf_event, NULL); 7889 } 7890 7891 void perf_event_itrace_started(struct perf_event *event) 7892 { 7893 event->attach_state |= PERF_ATTACH_ITRACE; 7894 } 7895 7896 static void perf_log_itrace_start(struct perf_event *event) 7897 { 7898 struct perf_output_handle handle; 7899 struct perf_sample_data sample; 7900 struct perf_aux_event { 7901 struct perf_event_header header; 7902 u32 pid; 7903 u32 tid; 7904 } rec; 7905 int ret; 7906 7907 if (event->parent) 7908 event = event->parent; 7909 7910 if (!(event->pmu->capabilities & PERF_PMU_CAP_ITRACE) || 7911 event->attach_state & PERF_ATTACH_ITRACE) 7912 return; 7913 7914 rec.header.type = PERF_RECORD_ITRACE_START; 7915 rec.header.misc = 0; 7916 rec.header.size = sizeof(rec); 7917 rec.pid = perf_event_pid(event, current); 7918 rec.tid = perf_event_tid(event, current); 7919 7920 perf_event_header__init_id(&rec.header, &sample, event); 7921 ret = perf_output_begin(&handle, event, rec.header.size); 7922 7923 if (ret) 7924 return; 7925 7926 perf_output_put(&handle, rec); 7927 perf_event__output_id_sample(event, &handle, &sample); 7928 7929 perf_output_end(&handle); 7930 } 7931 7932 static int 7933 __perf_event_account_interrupt(struct perf_event *event, int throttle) 7934 { 7935 struct hw_perf_event *hwc = &event->hw; 7936 int ret = 0; 7937 u64 seq; 7938 7939 seq = __this_cpu_read(perf_throttled_seq); 7940 if (seq != hwc->interrupts_seq) { 7941 hwc->interrupts_seq = seq; 7942 hwc->interrupts = 1; 7943 } else { 7944 hwc->interrupts++; 7945 if (unlikely(throttle 7946 && hwc->interrupts >= max_samples_per_tick)) { 7947 __this_cpu_inc(perf_throttled_count); 7948 tick_dep_set_cpu(smp_processor_id(), TICK_DEP_BIT_PERF_EVENTS); 7949 hwc->interrupts = MAX_INTERRUPTS; 7950 perf_log_throttle(event, 0); 7951 ret = 1; 7952 } 7953 } 7954 7955 if (event->attr.freq) { 7956 u64 now = perf_clock(); 7957 s64 delta = now - hwc->freq_time_stamp; 7958 7959 hwc->freq_time_stamp = now; 7960 7961 if (delta > 0 && delta < 2*TICK_NSEC) 7962 perf_adjust_period(event, delta, hwc->last_period, true); 7963 } 7964 7965 return ret; 7966 } 7967 7968 int perf_event_account_interrupt(struct perf_event *event) 7969 { 7970 return __perf_event_account_interrupt(event, 1); 7971 } 7972 7973 /* 7974 * Generic event overflow handling, sampling. 7975 */ 7976 7977 static int __perf_event_overflow(struct perf_event *event, 7978 int throttle, struct perf_sample_data *data, 7979 struct pt_regs *regs) 7980 { 7981 int events = atomic_read(&event->event_limit); 7982 int ret = 0; 7983 7984 /* 7985 * Non-sampling counters might still use the PMI to fold short 7986 * hardware counters, ignore those. 7987 */ 7988 if (unlikely(!is_sampling_event(event))) 7989 return 0; 7990 7991 ret = __perf_event_account_interrupt(event, throttle); 7992 7993 /* 7994 * XXX event_limit might not quite work as expected on inherited 7995 * events 7996 */ 7997 7998 event->pending_kill = POLL_IN; 7999 if (events && atomic_dec_and_test(&event->event_limit)) { 8000 ret = 1; 8001 event->pending_kill = POLL_HUP; 8002 8003 perf_event_disable_inatomic(event); 8004 } 8005 8006 READ_ONCE(event->overflow_handler)(event, data, regs); 8007 8008 if (*perf_event_fasync(event) && event->pending_kill) { 8009 event->pending_wakeup = 1; 8010 irq_work_queue(&event->pending); 8011 } 8012 8013 return ret; 8014 } 8015 8016 int perf_event_overflow(struct perf_event *event, 8017 struct perf_sample_data *data, 8018 struct pt_regs *regs) 8019 { 8020 return __perf_event_overflow(event, 1, data, regs); 8021 } 8022 8023 /* 8024 * Generic software event infrastructure 8025 */ 8026 8027 struct swevent_htable { 8028 struct swevent_hlist *swevent_hlist; 8029 struct mutex hlist_mutex; 8030 int hlist_refcount; 8031 8032 /* Recursion avoidance in each contexts */ 8033 int recursion[PERF_NR_CONTEXTS]; 8034 }; 8035 8036 static DEFINE_PER_CPU(struct swevent_htable, swevent_htable); 8037 8038 /* 8039 * We directly increment event->count and keep a second value in 8040 * event->hw.period_left to count intervals. This period event 8041 * is kept in the range [-sample_period, 0] so that we can use the 8042 * sign as trigger. 8043 */ 8044 8045 u64 perf_swevent_set_period(struct perf_event *event) 8046 { 8047 struct hw_perf_event *hwc = &event->hw; 8048 u64 period = hwc->last_period; 8049 u64 nr, offset; 8050 s64 old, val; 8051 8052 hwc->last_period = hwc->sample_period; 8053 8054 again: 8055 old = val = local64_read(&hwc->period_left); 8056 if (val < 0) 8057 return 0; 8058 8059 nr = div64_u64(period + val, period); 8060 offset = nr * period; 8061 val -= offset; 8062 if (local64_cmpxchg(&hwc->period_left, old, val) != old) 8063 goto again; 8064 8065 return nr; 8066 } 8067 8068 static void perf_swevent_overflow(struct perf_event *event, u64 overflow, 8069 struct perf_sample_data *data, 8070 struct pt_regs *regs) 8071 { 8072 struct hw_perf_event *hwc = &event->hw; 8073 int throttle = 0; 8074 8075 if (!overflow) 8076 overflow = perf_swevent_set_period(event); 8077 8078 if (hwc->interrupts == MAX_INTERRUPTS) 8079 return; 8080 8081 for (; overflow; overflow--) { 8082 if (__perf_event_overflow(event, throttle, 8083 data, regs)) { 8084 /* 8085 * We inhibit the overflow from happening when 8086 * hwc->interrupts == MAX_INTERRUPTS. 8087 */ 8088 break; 8089 } 8090 throttle = 1; 8091 } 8092 } 8093 8094 static void perf_swevent_event(struct perf_event *event, u64 nr, 8095 struct perf_sample_data *data, 8096 struct pt_regs *regs) 8097 { 8098 struct hw_perf_event *hwc = &event->hw; 8099 8100 local64_add(nr, &event->count); 8101 8102 if (!regs) 8103 return; 8104 8105 if (!is_sampling_event(event)) 8106 return; 8107 8108 if ((event->attr.sample_type & PERF_SAMPLE_PERIOD) && !event->attr.freq) { 8109 data->period = nr; 8110 return perf_swevent_overflow(event, 1, data, regs); 8111 } else 8112 data->period = event->hw.last_period; 8113 8114 if (nr == 1 && hwc->sample_period == 1 && !event->attr.freq) 8115 return perf_swevent_overflow(event, 1, data, regs); 8116 8117 if (local64_add_negative(nr, &hwc->period_left)) 8118 return; 8119 8120 perf_swevent_overflow(event, 0, data, regs); 8121 } 8122 8123 static int perf_exclude_event(struct perf_event *event, 8124 struct pt_regs *regs) 8125 { 8126 if (event->hw.state & PERF_HES_STOPPED) 8127 return 1; 8128 8129 if (regs) { 8130 if (event->attr.exclude_user && user_mode(regs)) 8131 return 1; 8132 8133 if (event->attr.exclude_kernel && !user_mode(regs)) 8134 return 1; 8135 } 8136 8137 return 0; 8138 } 8139 8140 static int perf_swevent_match(struct perf_event *event, 8141 enum perf_type_id type, 8142 u32 event_id, 8143 struct perf_sample_data *data, 8144 struct pt_regs *regs) 8145 { 8146 if (event->attr.type != type) 8147 return 0; 8148 8149 if (event->attr.config != event_id) 8150 return 0; 8151 8152 if (perf_exclude_event(event, regs)) 8153 return 0; 8154 8155 return 1; 8156 } 8157 8158 static inline u64 swevent_hash(u64 type, u32 event_id) 8159 { 8160 u64 val = event_id | (type << 32); 8161 8162 return hash_64(val, SWEVENT_HLIST_BITS); 8163 } 8164 8165 static inline struct hlist_head * 8166 __find_swevent_head(struct swevent_hlist *hlist, u64 type, u32 event_id) 8167 { 8168 u64 hash = swevent_hash(type, event_id); 8169 8170 return &hlist->heads[hash]; 8171 } 8172 8173 /* For the read side: events when they trigger */ 8174 static inline struct hlist_head * 8175 find_swevent_head_rcu(struct swevent_htable *swhash, u64 type, u32 event_id) 8176 { 8177 struct swevent_hlist *hlist; 8178 8179 hlist = rcu_dereference(swhash->swevent_hlist); 8180 if (!hlist) 8181 return NULL; 8182 8183 return __find_swevent_head(hlist, type, event_id); 8184 } 8185 8186 /* For the event head insertion and removal in the hlist */ 8187 static inline struct hlist_head * 8188 find_swevent_head(struct swevent_htable *swhash, struct perf_event *event) 8189 { 8190 struct swevent_hlist *hlist; 8191 u32 event_id = event->attr.config; 8192 u64 type = event->attr.type; 8193 8194 /* 8195 * Event scheduling is always serialized against hlist allocation 8196 * and release. Which makes the protected version suitable here. 8197 * The context lock guarantees that. 8198 */ 8199 hlist = rcu_dereference_protected(swhash->swevent_hlist, 8200 lockdep_is_held(&event->ctx->lock)); 8201 if (!hlist) 8202 return NULL; 8203 8204 return __find_swevent_head(hlist, type, event_id); 8205 } 8206 8207 static void do_perf_sw_event(enum perf_type_id type, u32 event_id, 8208 u64 nr, 8209 struct perf_sample_data *data, 8210 struct pt_regs *regs) 8211 { 8212 struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable); 8213 struct perf_event *event; 8214 struct hlist_head *head; 8215 8216 rcu_read_lock(); 8217 head = find_swevent_head_rcu(swhash, type, event_id); 8218 if (!head) 8219 goto end; 8220 8221 hlist_for_each_entry_rcu(event, head, hlist_entry) { 8222 if (perf_swevent_match(event, type, event_id, data, regs)) 8223 perf_swevent_event(event, nr, data, regs); 8224 } 8225 end: 8226 rcu_read_unlock(); 8227 } 8228 8229 DEFINE_PER_CPU(struct pt_regs, __perf_regs[4]); 8230 8231 int perf_swevent_get_recursion_context(void) 8232 { 8233 struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable); 8234 8235 return get_recursion_context(swhash->recursion); 8236 } 8237 EXPORT_SYMBOL_GPL(perf_swevent_get_recursion_context); 8238 8239 void perf_swevent_put_recursion_context(int rctx) 8240 { 8241 struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable); 8242 8243 put_recursion_context(swhash->recursion, rctx); 8244 } 8245 8246 void ___perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr) 8247 { 8248 struct perf_sample_data data; 8249 8250 if (WARN_ON_ONCE(!regs)) 8251 return; 8252 8253 perf_sample_data_init(&data, addr, 0); 8254 do_perf_sw_event(PERF_TYPE_SOFTWARE, event_id, nr, &data, regs); 8255 } 8256 8257 void __perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr) 8258 { 8259 int rctx; 8260 8261 preempt_disable_notrace(); 8262 rctx = perf_swevent_get_recursion_context(); 8263 if (unlikely(rctx < 0)) 8264 goto fail; 8265 8266 ___perf_sw_event(event_id, nr, regs, addr); 8267 8268 perf_swevent_put_recursion_context(rctx); 8269 fail: 8270 preempt_enable_notrace(); 8271 } 8272 8273 static void perf_swevent_read(struct perf_event *event) 8274 { 8275 } 8276 8277 static int perf_swevent_add(struct perf_event *event, int flags) 8278 { 8279 struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable); 8280 struct hw_perf_event *hwc = &event->hw; 8281 struct hlist_head *head; 8282 8283 if (is_sampling_event(event)) { 8284 hwc->last_period = hwc->sample_period; 8285 perf_swevent_set_period(event); 8286 } 8287 8288 hwc->state = !(flags & PERF_EF_START); 8289 8290 head = find_swevent_head(swhash, event); 8291 if (WARN_ON_ONCE(!head)) 8292 return -EINVAL; 8293 8294 hlist_add_head_rcu(&event->hlist_entry, head); 8295 perf_event_update_userpage(event); 8296 8297 return 0; 8298 } 8299 8300 static void perf_swevent_del(struct perf_event *event, int flags) 8301 { 8302 hlist_del_rcu(&event->hlist_entry); 8303 } 8304 8305 static void perf_swevent_start(struct perf_event *event, int flags) 8306 { 8307 event->hw.state = 0; 8308 } 8309 8310 static void perf_swevent_stop(struct perf_event *event, int flags) 8311 { 8312 event->hw.state = PERF_HES_STOPPED; 8313 } 8314 8315 /* Deref the hlist from the update side */ 8316 static inline struct swevent_hlist * 8317 swevent_hlist_deref(struct swevent_htable *swhash) 8318 { 8319 return rcu_dereference_protected(swhash->swevent_hlist, 8320 lockdep_is_held(&swhash->hlist_mutex)); 8321 } 8322 8323 static void swevent_hlist_release(struct swevent_htable *swhash) 8324 { 8325 struct swevent_hlist *hlist = swevent_hlist_deref(swhash); 8326 8327 if (!hlist) 8328 return; 8329 8330 RCU_INIT_POINTER(swhash->swevent_hlist, NULL); 8331 kfree_rcu(hlist, rcu_head); 8332 } 8333 8334 static void swevent_hlist_put_cpu(int cpu) 8335 { 8336 struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu); 8337 8338 mutex_lock(&swhash->hlist_mutex); 8339 8340 if (!--swhash->hlist_refcount) 8341 swevent_hlist_release(swhash); 8342 8343 mutex_unlock(&swhash->hlist_mutex); 8344 } 8345 8346 static void swevent_hlist_put(void) 8347 { 8348 int cpu; 8349 8350 for_each_possible_cpu(cpu) 8351 swevent_hlist_put_cpu(cpu); 8352 } 8353 8354 static int swevent_hlist_get_cpu(int cpu) 8355 { 8356 struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu); 8357 int err = 0; 8358 8359 mutex_lock(&swhash->hlist_mutex); 8360 if (!swevent_hlist_deref(swhash) && 8361 cpumask_test_cpu(cpu, perf_online_mask)) { 8362 struct swevent_hlist *hlist; 8363 8364 hlist = kzalloc(sizeof(*hlist), GFP_KERNEL); 8365 if (!hlist) { 8366 err = -ENOMEM; 8367 goto exit; 8368 } 8369 rcu_assign_pointer(swhash->swevent_hlist, hlist); 8370 } 8371 swhash->hlist_refcount++; 8372 exit: 8373 mutex_unlock(&swhash->hlist_mutex); 8374 8375 return err; 8376 } 8377 8378 static int swevent_hlist_get(void) 8379 { 8380 int err, cpu, failed_cpu; 8381 8382 mutex_lock(&pmus_lock); 8383 for_each_possible_cpu(cpu) { 8384 err = swevent_hlist_get_cpu(cpu); 8385 if (err) { 8386 failed_cpu = cpu; 8387 goto fail; 8388 } 8389 } 8390 mutex_unlock(&pmus_lock); 8391 return 0; 8392 fail: 8393 for_each_possible_cpu(cpu) { 8394 if (cpu == failed_cpu) 8395 break; 8396 swevent_hlist_put_cpu(cpu); 8397 } 8398 mutex_unlock(&pmus_lock); 8399 return err; 8400 } 8401 8402 struct static_key perf_swevent_enabled[PERF_COUNT_SW_MAX]; 8403 8404 static void sw_perf_event_destroy(struct perf_event *event) 8405 { 8406 u64 event_id = event->attr.config; 8407 8408 WARN_ON(event->parent); 8409 8410 static_key_slow_dec(&perf_swevent_enabled[event_id]); 8411 swevent_hlist_put(); 8412 } 8413 8414 static int perf_swevent_init(struct perf_event *event) 8415 { 8416 u64 event_id = event->attr.config; 8417 8418 if (event->attr.type != PERF_TYPE_SOFTWARE) 8419 return -ENOENT; 8420 8421 /* 8422 * no branch sampling for software events 8423 */ 8424 if (has_branch_stack(event)) 8425 return -EOPNOTSUPP; 8426 8427 switch (event_id) { 8428 case PERF_COUNT_SW_CPU_CLOCK: 8429 case PERF_COUNT_SW_TASK_CLOCK: 8430 return -ENOENT; 8431 8432 default: 8433 break; 8434 } 8435 8436 if (event_id >= PERF_COUNT_SW_MAX) 8437 return -ENOENT; 8438 8439 if (!event->parent) { 8440 int err; 8441 8442 err = swevent_hlist_get(); 8443 if (err) 8444 return err; 8445 8446 static_key_slow_inc(&perf_swevent_enabled[event_id]); 8447 event->destroy = sw_perf_event_destroy; 8448 } 8449 8450 return 0; 8451 } 8452 8453 static struct pmu perf_swevent = { 8454 .task_ctx_nr = perf_sw_context, 8455 8456 .capabilities = PERF_PMU_CAP_NO_NMI, 8457 8458 .event_init = perf_swevent_init, 8459 .add = perf_swevent_add, 8460 .del = perf_swevent_del, 8461 .start = perf_swevent_start, 8462 .stop = perf_swevent_stop, 8463 .read = perf_swevent_read, 8464 }; 8465 8466 #ifdef CONFIG_EVENT_TRACING 8467 8468 static int perf_tp_filter_match(struct perf_event *event, 8469 struct perf_sample_data *data) 8470 { 8471 void *record = data->raw->frag.data; 8472 8473 /* only top level events have filters set */ 8474 if (event->parent) 8475 event = event->parent; 8476 8477 if (likely(!event->filter) || filter_match_preds(event->filter, record)) 8478 return 1; 8479 return 0; 8480 } 8481 8482 static int perf_tp_event_match(struct perf_event *event, 8483 struct perf_sample_data *data, 8484 struct pt_regs *regs) 8485 { 8486 if (event->hw.state & PERF_HES_STOPPED) 8487 return 0; 8488 /* 8489 * All tracepoints are from kernel-space. 8490 */ 8491 if (event->attr.exclude_kernel) 8492 return 0; 8493 8494 if (!perf_tp_filter_match(event, data)) 8495 return 0; 8496 8497 return 1; 8498 } 8499 8500 void perf_trace_run_bpf_submit(void *raw_data, int size, int rctx, 8501 struct trace_event_call *call, u64 count, 8502 struct pt_regs *regs, struct hlist_head *head, 8503 struct task_struct *task) 8504 { 8505 if (bpf_prog_array_valid(call)) { 8506 *(struct pt_regs **)raw_data = regs; 8507 if (!trace_call_bpf(call, raw_data) || hlist_empty(head)) { 8508 perf_swevent_put_recursion_context(rctx); 8509 return; 8510 } 8511 } 8512 perf_tp_event(call->event.type, count, raw_data, size, regs, head, 8513 rctx, task); 8514 } 8515 EXPORT_SYMBOL_GPL(perf_trace_run_bpf_submit); 8516 8517 void perf_tp_event(u16 event_type, u64 count, void *record, int entry_size, 8518 struct pt_regs *regs, struct hlist_head *head, int rctx, 8519 struct task_struct *task) 8520 { 8521 struct perf_sample_data data; 8522 struct perf_event *event; 8523 8524 struct perf_raw_record raw = { 8525 .frag = { 8526 .size = entry_size, 8527 .data = record, 8528 }, 8529 }; 8530 8531 perf_sample_data_init(&data, 0, 0); 8532 data.raw = &raw; 8533 8534 perf_trace_buf_update(record, event_type); 8535 8536 hlist_for_each_entry_rcu(event, head, hlist_entry) { 8537 if (perf_tp_event_match(event, &data, regs)) 8538 perf_swevent_event(event, count, &data, regs); 8539 } 8540 8541 /* 8542 * If we got specified a target task, also iterate its context and 8543 * deliver this event there too. 8544 */ 8545 if (task && task != current) { 8546 struct perf_event_context *ctx; 8547 struct trace_entry *entry = record; 8548 8549 rcu_read_lock(); 8550 ctx = rcu_dereference(task->perf_event_ctxp[perf_sw_context]); 8551 if (!ctx) 8552 goto unlock; 8553 8554 list_for_each_entry_rcu(event, &ctx->event_list, event_entry) { 8555 if (event->cpu != smp_processor_id()) 8556 continue; 8557 if (event->attr.type != PERF_TYPE_TRACEPOINT) 8558 continue; 8559 if (event->attr.config != entry->type) 8560 continue; 8561 if (perf_tp_event_match(event, &data, regs)) 8562 perf_swevent_event(event, count, &data, regs); 8563 } 8564 unlock: 8565 rcu_read_unlock(); 8566 } 8567 8568 perf_swevent_put_recursion_context(rctx); 8569 } 8570 EXPORT_SYMBOL_GPL(perf_tp_event); 8571 8572 static void tp_perf_event_destroy(struct perf_event *event) 8573 { 8574 perf_trace_destroy(event); 8575 } 8576 8577 static int perf_tp_event_init(struct perf_event *event) 8578 { 8579 int err; 8580 8581 if (event->attr.type != PERF_TYPE_TRACEPOINT) 8582 return -ENOENT; 8583 8584 /* 8585 * no branch sampling for tracepoint events 8586 */ 8587 if (has_branch_stack(event)) 8588 return -EOPNOTSUPP; 8589 8590 err = perf_trace_init(event); 8591 if (err) 8592 return err; 8593 8594 event->destroy = tp_perf_event_destroy; 8595 8596 return 0; 8597 } 8598 8599 static struct pmu perf_tracepoint = { 8600 .task_ctx_nr = perf_sw_context, 8601 8602 .event_init = perf_tp_event_init, 8603 .add = perf_trace_add, 8604 .del = perf_trace_del, 8605 .start = perf_swevent_start, 8606 .stop = perf_swevent_stop, 8607 .read = perf_swevent_read, 8608 }; 8609 8610 #if defined(CONFIG_KPROBE_EVENTS) || defined(CONFIG_UPROBE_EVENTS) 8611 /* 8612 * Flags in config, used by dynamic PMU kprobe and uprobe 8613 * The flags should match following PMU_FORMAT_ATTR(). 8614 * 8615 * PERF_PROBE_CONFIG_IS_RETPROBE if set, create kretprobe/uretprobe 8616 * if not set, create kprobe/uprobe 8617 * 8618 * The following values specify a reference counter (or semaphore in the 8619 * terminology of tools like dtrace, systemtap, etc.) Userspace Statically 8620 * Defined Tracepoints (USDT). Currently, we use 40 bit for the offset. 8621 * 8622 * PERF_UPROBE_REF_CTR_OFFSET_BITS # of bits in config as th offset 8623 * PERF_UPROBE_REF_CTR_OFFSET_SHIFT # of bits to shift left 8624 */ 8625 enum perf_probe_config { 8626 PERF_PROBE_CONFIG_IS_RETPROBE = 1U << 0, /* [k,u]retprobe */ 8627 PERF_UPROBE_REF_CTR_OFFSET_BITS = 32, 8628 PERF_UPROBE_REF_CTR_OFFSET_SHIFT = 64 - PERF_UPROBE_REF_CTR_OFFSET_BITS, 8629 }; 8630 8631 PMU_FORMAT_ATTR(retprobe, "config:0"); 8632 #endif 8633 8634 #ifdef CONFIG_KPROBE_EVENTS 8635 static struct attribute *kprobe_attrs[] = { 8636 &format_attr_retprobe.attr, 8637 NULL, 8638 }; 8639 8640 static struct attribute_group kprobe_format_group = { 8641 .name = "format", 8642 .attrs = kprobe_attrs, 8643 }; 8644 8645 static const struct attribute_group *kprobe_attr_groups[] = { 8646 &kprobe_format_group, 8647 NULL, 8648 }; 8649 8650 static int perf_kprobe_event_init(struct perf_event *event); 8651 static struct pmu perf_kprobe = { 8652 .task_ctx_nr = perf_sw_context, 8653 .event_init = perf_kprobe_event_init, 8654 .add = perf_trace_add, 8655 .del = perf_trace_del, 8656 .start = perf_swevent_start, 8657 .stop = perf_swevent_stop, 8658 .read = perf_swevent_read, 8659 .attr_groups = kprobe_attr_groups, 8660 }; 8661 8662 static int perf_kprobe_event_init(struct perf_event *event) 8663 { 8664 int err; 8665 bool is_retprobe; 8666 8667 if (event->attr.type != perf_kprobe.type) 8668 return -ENOENT; 8669 8670 if (!capable(CAP_SYS_ADMIN)) 8671 return -EACCES; 8672 8673 /* 8674 * no branch sampling for probe events 8675 */ 8676 if (has_branch_stack(event)) 8677 return -EOPNOTSUPP; 8678 8679 is_retprobe = event->attr.config & PERF_PROBE_CONFIG_IS_RETPROBE; 8680 err = perf_kprobe_init(event, is_retprobe); 8681 if (err) 8682 return err; 8683 8684 event->destroy = perf_kprobe_destroy; 8685 8686 return 0; 8687 } 8688 #endif /* CONFIG_KPROBE_EVENTS */ 8689 8690 #ifdef CONFIG_UPROBE_EVENTS 8691 PMU_FORMAT_ATTR(ref_ctr_offset, "config:32-63"); 8692 8693 static struct attribute *uprobe_attrs[] = { 8694 &format_attr_retprobe.attr, 8695 &format_attr_ref_ctr_offset.attr, 8696 NULL, 8697 }; 8698 8699 static struct attribute_group uprobe_format_group = { 8700 .name = "format", 8701 .attrs = uprobe_attrs, 8702 }; 8703 8704 static const struct attribute_group *uprobe_attr_groups[] = { 8705 &uprobe_format_group, 8706 NULL, 8707 }; 8708 8709 static int perf_uprobe_event_init(struct perf_event *event); 8710 static struct pmu perf_uprobe = { 8711 .task_ctx_nr = perf_sw_context, 8712 .event_init = perf_uprobe_event_init, 8713 .add = perf_trace_add, 8714 .del = perf_trace_del, 8715 .start = perf_swevent_start, 8716 .stop = perf_swevent_stop, 8717 .read = perf_swevent_read, 8718 .attr_groups = uprobe_attr_groups, 8719 }; 8720 8721 static int perf_uprobe_event_init(struct perf_event *event) 8722 { 8723 int err; 8724 unsigned long ref_ctr_offset; 8725 bool is_retprobe; 8726 8727 if (event->attr.type != perf_uprobe.type) 8728 return -ENOENT; 8729 8730 if (!capable(CAP_SYS_ADMIN)) 8731 return -EACCES; 8732 8733 /* 8734 * no branch sampling for probe events 8735 */ 8736 if (has_branch_stack(event)) 8737 return -EOPNOTSUPP; 8738 8739 is_retprobe = event->attr.config & PERF_PROBE_CONFIG_IS_RETPROBE; 8740 ref_ctr_offset = event->attr.config >> PERF_UPROBE_REF_CTR_OFFSET_SHIFT; 8741 err = perf_uprobe_init(event, ref_ctr_offset, is_retprobe); 8742 if (err) 8743 return err; 8744 8745 event->destroy = perf_uprobe_destroy; 8746 8747 return 0; 8748 } 8749 #endif /* CONFIG_UPROBE_EVENTS */ 8750 8751 static inline void perf_tp_register(void) 8752 { 8753 perf_pmu_register(&perf_tracepoint, "tracepoint", PERF_TYPE_TRACEPOINT); 8754 #ifdef CONFIG_KPROBE_EVENTS 8755 perf_pmu_register(&perf_kprobe, "kprobe", -1); 8756 #endif 8757 #ifdef CONFIG_UPROBE_EVENTS 8758 perf_pmu_register(&perf_uprobe, "uprobe", -1); 8759 #endif 8760 } 8761 8762 static void perf_event_free_filter(struct perf_event *event) 8763 { 8764 ftrace_profile_free_filter(event); 8765 } 8766 8767 #ifdef CONFIG_BPF_SYSCALL 8768 static void bpf_overflow_handler(struct perf_event *event, 8769 struct perf_sample_data *data, 8770 struct pt_regs *regs) 8771 { 8772 struct bpf_perf_event_data_kern ctx = { 8773 .data = data, 8774 .event = event, 8775 }; 8776 int ret = 0; 8777 8778 ctx.regs = perf_arch_bpf_user_pt_regs(regs); 8779 preempt_disable(); 8780 if (unlikely(__this_cpu_inc_return(bpf_prog_active) != 1)) 8781 goto out; 8782 rcu_read_lock(); 8783 ret = BPF_PROG_RUN(event->prog, &ctx); 8784 rcu_read_unlock(); 8785 out: 8786 __this_cpu_dec(bpf_prog_active); 8787 preempt_enable(); 8788 if (!ret) 8789 return; 8790 8791 event->orig_overflow_handler(event, data, regs); 8792 } 8793 8794 static int perf_event_set_bpf_handler(struct perf_event *event, u32 prog_fd) 8795 { 8796 struct bpf_prog *prog; 8797 8798 if (event->overflow_handler_context) 8799 /* hw breakpoint or kernel counter */ 8800 return -EINVAL; 8801 8802 if (event->prog) 8803 return -EEXIST; 8804 8805 prog = bpf_prog_get_type(prog_fd, BPF_PROG_TYPE_PERF_EVENT); 8806 if (IS_ERR(prog)) 8807 return PTR_ERR(prog); 8808 8809 event->prog = prog; 8810 event->orig_overflow_handler = READ_ONCE(event->overflow_handler); 8811 WRITE_ONCE(event->overflow_handler, bpf_overflow_handler); 8812 return 0; 8813 } 8814 8815 static void perf_event_free_bpf_handler(struct perf_event *event) 8816 { 8817 struct bpf_prog *prog = event->prog; 8818 8819 if (!prog) 8820 return; 8821 8822 WRITE_ONCE(event->overflow_handler, event->orig_overflow_handler); 8823 event->prog = NULL; 8824 bpf_prog_put(prog); 8825 } 8826 #else 8827 static int perf_event_set_bpf_handler(struct perf_event *event, u32 prog_fd) 8828 { 8829 return -EOPNOTSUPP; 8830 } 8831 static void perf_event_free_bpf_handler(struct perf_event *event) 8832 { 8833 } 8834 #endif 8835 8836 /* 8837 * returns true if the event is a tracepoint, or a kprobe/upprobe created 8838 * with perf_event_open() 8839 */ 8840 static inline bool perf_event_is_tracing(struct perf_event *event) 8841 { 8842 if (event->pmu == &perf_tracepoint) 8843 return true; 8844 #ifdef CONFIG_KPROBE_EVENTS 8845 if (event->pmu == &perf_kprobe) 8846 return true; 8847 #endif 8848 #ifdef CONFIG_UPROBE_EVENTS 8849 if (event->pmu == &perf_uprobe) 8850 return true; 8851 #endif 8852 return false; 8853 } 8854 8855 static int perf_event_set_bpf_prog(struct perf_event *event, u32 prog_fd) 8856 { 8857 bool is_kprobe, is_tracepoint, is_syscall_tp; 8858 struct bpf_prog *prog; 8859 int ret; 8860 8861 if (!perf_event_is_tracing(event)) 8862 return perf_event_set_bpf_handler(event, prog_fd); 8863 8864 is_kprobe = event->tp_event->flags & TRACE_EVENT_FL_UKPROBE; 8865 is_tracepoint = event->tp_event->flags & TRACE_EVENT_FL_TRACEPOINT; 8866 is_syscall_tp = is_syscall_trace_event(event->tp_event); 8867 if (!is_kprobe && !is_tracepoint && !is_syscall_tp) 8868 /* bpf programs can only be attached to u/kprobe or tracepoint */ 8869 return -EINVAL; 8870 8871 prog = bpf_prog_get(prog_fd); 8872 if (IS_ERR(prog)) 8873 return PTR_ERR(prog); 8874 8875 if ((is_kprobe && prog->type != BPF_PROG_TYPE_KPROBE) || 8876 (is_tracepoint && prog->type != BPF_PROG_TYPE_TRACEPOINT) || 8877 (is_syscall_tp && prog->type != BPF_PROG_TYPE_TRACEPOINT)) { 8878 /* valid fd, but invalid bpf program type */ 8879 bpf_prog_put(prog); 8880 return -EINVAL; 8881 } 8882 8883 /* Kprobe override only works for kprobes, not uprobes. */ 8884 if (prog->kprobe_override && 8885 !(event->tp_event->flags & TRACE_EVENT_FL_KPROBE)) { 8886 bpf_prog_put(prog); 8887 return -EINVAL; 8888 } 8889 8890 if (is_tracepoint || is_syscall_tp) { 8891 int off = trace_event_get_offsets(event->tp_event); 8892 8893 if (prog->aux->max_ctx_offset > off) { 8894 bpf_prog_put(prog); 8895 return -EACCES; 8896 } 8897 } 8898 8899 ret = perf_event_attach_bpf_prog(event, prog); 8900 if (ret) 8901 bpf_prog_put(prog); 8902 return ret; 8903 } 8904 8905 static void perf_event_free_bpf_prog(struct perf_event *event) 8906 { 8907 if (!perf_event_is_tracing(event)) { 8908 perf_event_free_bpf_handler(event); 8909 return; 8910 } 8911 perf_event_detach_bpf_prog(event); 8912 } 8913 8914 #else 8915 8916 static inline void perf_tp_register(void) 8917 { 8918 } 8919 8920 static void perf_event_free_filter(struct perf_event *event) 8921 { 8922 } 8923 8924 static int perf_event_set_bpf_prog(struct perf_event *event, u32 prog_fd) 8925 { 8926 return -ENOENT; 8927 } 8928 8929 static void perf_event_free_bpf_prog(struct perf_event *event) 8930 { 8931 } 8932 #endif /* CONFIG_EVENT_TRACING */ 8933 8934 #ifdef CONFIG_HAVE_HW_BREAKPOINT 8935 void perf_bp_event(struct perf_event *bp, void *data) 8936 { 8937 struct perf_sample_data sample; 8938 struct pt_regs *regs = data; 8939 8940 perf_sample_data_init(&sample, bp->attr.bp_addr, 0); 8941 8942 if (!bp->hw.state && !perf_exclude_event(bp, regs)) 8943 perf_swevent_event(bp, 1, &sample, regs); 8944 } 8945 #endif 8946 8947 /* 8948 * Allocate a new address filter 8949 */ 8950 static struct perf_addr_filter * 8951 perf_addr_filter_new(struct perf_event *event, struct list_head *filters) 8952 { 8953 int node = cpu_to_node(event->cpu == -1 ? 0 : event->cpu); 8954 struct perf_addr_filter *filter; 8955 8956 filter = kzalloc_node(sizeof(*filter), GFP_KERNEL, node); 8957 if (!filter) 8958 return NULL; 8959 8960 INIT_LIST_HEAD(&filter->entry); 8961 list_add_tail(&filter->entry, filters); 8962 8963 return filter; 8964 } 8965 8966 static void free_filters_list(struct list_head *filters) 8967 { 8968 struct perf_addr_filter *filter, *iter; 8969 8970 list_for_each_entry_safe(filter, iter, filters, entry) { 8971 path_put(&filter->path); 8972 list_del(&filter->entry); 8973 kfree(filter); 8974 } 8975 } 8976 8977 /* 8978 * Free existing address filters and optionally install new ones 8979 */ 8980 static void perf_addr_filters_splice(struct perf_event *event, 8981 struct list_head *head) 8982 { 8983 unsigned long flags; 8984 LIST_HEAD(list); 8985 8986 if (!has_addr_filter(event)) 8987 return; 8988 8989 /* don't bother with children, they don't have their own filters */ 8990 if (event->parent) 8991 return; 8992 8993 raw_spin_lock_irqsave(&event->addr_filters.lock, flags); 8994 8995 list_splice_init(&event->addr_filters.list, &list); 8996 if (head) 8997 list_splice(head, &event->addr_filters.list); 8998 8999 raw_spin_unlock_irqrestore(&event->addr_filters.lock, flags); 9000 9001 free_filters_list(&list); 9002 } 9003 9004 /* 9005 * Scan through mm's vmas and see if one of them matches the 9006 * @filter; if so, adjust filter's address range. 9007 * Called with mm::mmap_sem down for reading. 9008 */ 9009 static void perf_addr_filter_apply(struct perf_addr_filter *filter, 9010 struct mm_struct *mm, 9011 struct perf_addr_filter_range *fr) 9012 { 9013 struct vm_area_struct *vma; 9014 9015 for (vma = mm->mmap; vma; vma = vma->vm_next) { 9016 if (!vma->vm_file) 9017 continue; 9018 9019 if (perf_addr_filter_vma_adjust(filter, vma, fr)) 9020 return; 9021 } 9022 } 9023 9024 /* 9025 * Update event's address range filters based on the 9026 * task's existing mappings, if any. 9027 */ 9028 static void perf_event_addr_filters_apply(struct perf_event *event) 9029 { 9030 struct perf_addr_filters_head *ifh = perf_event_addr_filters(event); 9031 struct task_struct *task = READ_ONCE(event->ctx->task); 9032 struct perf_addr_filter *filter; 9033 struct mm_struct *mm = NULL; 9034 unsigned int count = 0; 9035 unsigned long flags; 9036 9037 /* 9038 * We may observe TASK_TOMBSTONE, which means that the event tear-down 9039 * will stop on the parent's child_mutex that our caller is also holding 9040 */ 9041 if (task == TASK_TOMBSTONE) 9042 return; 9043 9044 if (!ifh->nr_file_filters) 9045 return; 9046 9047 mm = get_task_mm(event->ctx->task); 9048 if (!mm) 9049 goto restart; 9050 9051 down_read(&mm->mmap_sem); 9052 9053 raw_spin_lock_irqsave(&ifh->lock, flags); 9054 list_for_each_entry(filter, &ifh->list, entry) { 9055 event->addr_filter_ranges[count].start = 0; 9056 event->addr_filter_ranges[count].size = 0; 9057 9058 /* 9059 * Adjust base offset if the filter is associated to a binary 9060 * that needs to be mapped: 9061 */ 9062 if (filter->path.dentry) 9063 perf_addr_filter_apply(filter, mm, &event->addr_filter_ranges[count]); 9064 9065 count++; 9066 } 9067 9068 event->addr_filters_gen++; 9069 raw_spin_unlock_irqrestore(&ifh->lock, flags); 9070 9071 up_read(&mm->mmap_sem); 9072 9073 mmput(mm); 9074 9075 restart: 9076 perf_event_stop(event, 1); 9077 } 9078 9079 /* 9080 * Address range filtering: limiting the data to certain 9081 * instruction address ranges. Filters are ioctl()ed to us from 9082 * userspace as ascii strings. 9083 * 9084 * Filter string format: 9085 * 9086 * ACTION RANGE_SPEC 9087 * where ACTION is one of the 9088 * * "filter": limit the trace to this region 9089 * * "start": start tracing from this address 9090 * * "stop": stop tracing at this address/region; 9091 * RANGE_SPEC is 9092 * * for kernel addresses: <start address>[/<size>] 9093 * * for object files: <start address>[/<size>]@</path/to/object/file> 9094 * 9095 * if <size> is not specified or is zero, the range is treated as a single 9096 * address; not valid for ACTION=="filter". 9097 */ 9098 enum { 9099 IF_ACT_NONE = -1, 9100 IF_ACT_FILTER, 9101 IF_ACT_START, 9102 IF_ACT_STOP, 9103 IF_SRC_FILE, 9104 IF_SRC_KERNEL, 9105 IF_SRC_FILEADDR, 9106 IF_SRC_KERNELADDR, 9107 }; 9108 9109 enum { 9110 IF_STATE_ACTION = 0, 9111 IF_STATE_SOURCE, 9112 IF_STATE_END, 9113 }; 9114 9115 static const match_table_t if_tokens = { 9116 { IF_ACT_FILTER, "filter" }, 9117 { IF_ACT_START, "start" }, 9118 { IF_ACT_STOP, "stop" }, 9119 { IF_SRC_FILE, "%u/%u@%s" }, 9120 { IF_SRC_KERNEL, "%u/%u" }, 9121 { IF_SRC_FILEADDR, "%u@%s" }, 9122 { IF_SRC_KERNELADDR, "%u" }, 9123 { IF_ACT_NONE, NULL }, 9124 }; 9125 9126 /* 9127 * Address filter string parser 9128 */ 9129 static int 9130 perf_event_parse_addr_filter(struct perf_event *event, char *fstr, 9131 struct list_head *filters) 9132 { 9133 struct perf_addr_filter *filter = NULL; 9134 char *start, *orig, *filename = NULL; 9135 substring_t args[MAX_OPT_ARGS]; 9136 int state = IF_STATE_ACTION, token; 9137 unsigned int kernel = 0; 9138 int ret = -EINVAL; 9139 9140 orig = fstr = kstrdup(fstr, GFP_KERNEL); 9141 if (!fstr) 9142 return -ENOMEM; 9143 9144 while ((start = strsep(&fstr, " ,\n")) != NULL) { 9145 static const enum perf_addr_filter_action_t actions[] = { 9146 [IF_ACT_FILTER] = PERF_ADDR_FILTER_ACTION_FILTER, 9147 [IF_ACT_START] = PERF_ADDR_FILTER_ACTION_START, 9148 [IF_ACT_STOP] = PERF_ADDR_FILTER_ACTION_STOP, 9149 }; 9150 ret = -EINVAL; 9151 9152 if (!*start) 9153 continue; 9154 9155 /* filter definition begins */ 9156 if (state == IF_STATE_ACTION) { 9157 filter = perf_addr_filter_new(event, filters); 9158 if (!filter) 9159 goto fail; 9160 } 9161 9162 token = match_token(start, if_tokens, args); 9163 switch (token) { 9164 case IF_ACT_FILTER: 9165 case IF_ACT_START: 9166 case IF_ACT_STOP: 9167 if (state != IF_STATE_ACTION) 9168 goto fail; 9169 9170 filter->action = actions[token]; 9171 state = IF_STATE_SOURCE; 9172 break; 9173 9174 case IF_SRC_KERNELADDR: 9175 case IF_SRC_KERNEL: 9176 kernel = 1; 9177 9178 case IF_SRC_FILEADDR: 9179 case IF_SRC_FILE: 9180 if (state != IF_STATE_SOURCE) 9181 goto fail; 9182 9183 *args[0].to = 0; 9184 ret = kstrtoul(args[0].from, 0, &filter->offset); 9185 if (ret) 9186 goto fail; 9187 9188 if (token == IF_SRC_KERNEL || token == IF_SRC_FILE) { 9189 *args[1].to = 0; 9190 ret = kstrtoul(args[1].from, 0, &filter->size); 9191 if (ret) 9192 goto fail; 9193 } 9194 9195 if (token == IF_SRC_FILE || token == IF_SRC_FILEADDR) { 9196 int fpos = token == IF_SRC_FILE ? 2 : 1; 9197 9198 filename = match_strdup(&args[fpos]); 9199 if (!filename) { 9200 ret = -ENOMEM; 9201 goto fail; 9202 } 9203 } 9204 9205 state = IF_STATE_END; 9206 break; 9207 9208 default: 9209 goto fail; 9210 } 9211 9212 /* 9213 * Filter definition is fully parsed, validate and install it. 9214 * Make sure that it doesn't contradict itself or the event's 9215 * attribute. 9216 */ 9217 if (state == IF_STATE_END) { 9218 ret = -EINVAL; 9219 if (kernel && event->attr.exclude_kernel) 9220 goto fail; 9221 9222 /* 9223 * ACTION "filter" must have a non-zero length region 9224 * specified. 9225 */ 9226 if (filter->action == PERF_ADDR_FILTER_ACTION_FILTER && 9227 !filter->size) 9228 goto fail; 9229 9230 if (!kernel) { 9231 if (!filename) 9232 goto fail; 9233 9234 /* 9235 * For now, we only support file-based filters 9236 * in per-task events; doing so for CPU-wide 9237 * events requires additional context switching 9238 * trickery, since same object code will be 9239 * mapped at different virtual addresses in 9240 * different processes. 9241 */ 9242 ret = -EOPNOTSUPP; 9243 if (!event->ctx->task) 9244 goto fail_free_name; 9245 9246 /* look up the path and grab its inode */ 9247 ret = kern_path(filename, LOOKUP_FOLLOW, 9248 &filter->path); 9249 if (ret) 9250 goto fail_free_name; 9251 9252 kfree(filename); 9253 filename = NULL; 9254 9255 ret = -EINVAL; 9256 if (!filter->path.dentry || 9257 !S_ISREG(d_inode(filter->path.dentry) 9258 ->i_mode)) 9259 goto fail; 9260 9261 event->addr_filters.nr_file_filters++; 9262 } 9263 9264 /* ready to consume more filters */ 9265 state = IF_STATE_ACTION; 9266 filter = NULL; 9267 } 9268 } 9269 9270 if (state != IF_STATE_ACTION) 9271 goto fail; 9272 9273 kfree(orig); 9274 9275 return 0; 9276 9277 fail_free_name: 9278 kfree(filename); 9279 fail: 9280 free_filters_list(filters); 9281 kfree(orig); 9282 9283 return ret; 9284 } 9285 9286 static int 9287 perf_event_set_addr_filter(struct perf_event *event, char *filter_str) 9288 { 9289 LIST_HEAD(filters); 9290 int ret; 9291 9292 /* 9293 * Since this is called in perf_ioctl() path, we're already holding 9294 * ctx::mutex. 9295 */ 9296 lockdep_assert_held(&event->ctx->mutex); 9297 9298 if (WARN_ON_ONCE(event->parent)) 9299 return -EINVAL; 9300 9301 ret = perf_event_parse_addr_filter(event, filter_str, &filters); 9302 if (ret) 9303 goto fail_clear_files; 9304 9305 ret = event->pmu->addr_filters_validate(&filters); 9306 if (ret) 9307 goto fail_free_filters; 9308 9309 /* remove existing filters, if any */ 9310 perf_addr_filters_splice(event, &filters); 9311 9312 /* install new filters */ 9313 perf_event_for_each_child(event, perf_event_addr_filters_apply); 9314 9315 return ret; 9316 9317 fail_free_filters: 9318 free_filters_list(&filters); 9319 9320 fail_clear_files: 9321 event->addr_filters.nr_file_filters = 0; 9322 9323 return ret; 9324 } 9325 9326 static int perf_event_set_filter(struct perf_event *event, void __user *arg) 9327 { 9328 int ret = -EINVAL; 9329 char *filter_str; 9330 9331 filter_str = strndup_user(arg, PAGE_SIZE); 9332 if (IS_ERR(filter_str)) 9333 return PTR_ERR(filter_str); 9334 9335 #ifdef CONFIG_EVENT_TRACING 9336 if (perf_event_is_tracing(event)) { 9337 struct perf_event_context *ctx = event->ctx; 9338 9339 /* 9340 * Beware, here be dragons!! 9341 * 9342 * the tracepoint muck will deadlock against ctx->mutex, but 9343 * the tracepoint stuff does not actually need it. So 9344 * temporarily drop ctx->mutex. As per perf_event_ctx_lock() we 9345 * already have a reference on ctx. 9346 * 9347 * This can result in event getting moved to a different ctx, 9348 * but that does not affect the tracepoint state. 9349 */ 9350 mutex_unlock(&ctx->mutex); 9351 ret = ftrace_profile_set_filter(event, event->attr.config, filter_str); 9352 mutex_lock(&ctx->mutex); 9353 } else 9354 #endif 9355 if (has_addr_filter(event)) 9356 ret = perf_event_set_addr_filter(event, filter_str); 9357 9358 kfree(filter_str); 9359 return ret; 9360 } 9361 9362 /* 9363 * hrtimer based swevent callback 9364 */ 9365 9366 static enum hrtimer_restart perf_swevent_hrtimer(struct hrtimer *hrtimer) 9367 { 9368 enum hrtimer_restart ret = HRTIMER_RESTART; 9369 struct perf_sample_data data; 9370 struct pt_regs *regs; 9371 struct perf_event *event; 9372 u64 period; 9373 9374 event = container_of(hrtimer, struct perf_event, hw.hrtimer); 9375 9376 if (event->state != PERF_EVENT_STATE_ACTIVE) 9377 return HRTIMER_NORESTART; 9378 9379 event->pmu->read(event); 9380 9381 perf_sample_data_init(&data, 0, event->hw.last_period); 9382 regs = get_irq_regs(); 9383 9384 if (regs && !perf_exclude_event(event, regs)) { 9385 if (!(event->attr.exclude_idle && is_idle_task(current))) 9386 if (__perf_event_overflow(event, 1, &data, regs)) 9387 ret = HRTIMER_NORESTART; 9388 } 9389 9390 period = max_t(u64, 10000, event->hw.sample_period); 9391 hrtimer_forward_now(hrtimer, ns_to_ktime(period)); 9392 9393 return ret; 9394 } 9395 9396 static void perf_swevent_start_hrtimer(struct perf_event *event) 9397 { 9398 struct hw_perf_event *hwc = &event->hw; 9399 s64 period; 9400 9401 if (!is_sampling_event(event)) 9402 return; 9403 9404 period = local64_read(&hwc->period_left); 9405 if (period) { 9406 if (period < 0) 9407 period = 10000; 9408 9409 local64_set(&hwc->period_left, 0); 9410 } else { 9411 period = max_t(u64, 10000, hwc->sample_period); 9412 } 9413 hrtimer_start(&hwc->hrtimer, ns_to_ktime(period), 9414 HRTIMER_MODE_REL_PINNED); 9415 } 9416 9417 static void perf_swevent_cancel_hrtimer(struct perf_event *event) 9418 { 9419 struct hw_perf_event *hwc = &event->hw; 9420 9421 if (is_sampling_event(event)) { 9422 ktime_t remaining = hrtimer_get_remaining(&hwc->hrtimer); 9423 local64_set(&hwc->period_left, ktime_to_ns(remaining)); 9424 9425 hrtimer_cancel(&hwc->hrtimer); 9426 } 9427 } 9428 9429 static void perf_swevent_init_hrtimer(struct perf_event *event) 9430 { 9431 struct hw_perf_event *hwc = &event->hw; 9432 9433 if (!is_sampling_event(event)) 9434 return; 9435 9436 hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 9437 hwc->hrtimer.function = perf_swevent_hrtimer; 9438 9439 /* 9440 * Since hrtimers have a fixed rate, we can do a static freq->period 9441 * mapping and avoid the whole period adjust feedback stuff. 9442 */ 9443 if (event->attr.freq) { 9444 long freq = event->attr.sample_freq; 9445 9446 event->attr.sample_period = NSEC_PER_SEC / freq; 9447 hwc->sample_period = event->attr.sample_period; 9448 local64_set(&hwc->period_left, hwc->sample_period); 9449 hwc->last_period = hwc->sample_period; 9450 event->attr.freq = 0; 9451 } 9452 } 9453 9454 /* 9455 * Software event: cpu wall time clock 9456 */ 9457 9458 static void cpu_clock_event_update(struct perf_event *event) 9459 { 9460 s64 prev; 9461 u64 now; 9462 9463 now = local_clock(); 9464 prev = local64_xchg(&event->hw.prev_count, now); 9465 local64_add(now - prev, &event->count); 9466 } 9467 9468 static void cpu_clock_event_start(struct perf_event *event, int flags) 9469 { 9470 local64_set(&event->hw.prev_count, local_clock()); 9471 perf_swevent_start_hrtimer(event); 9472 } 9473 9474 static void cpu_clock_event_stop(struct perf_event *event, int flags) 9475 { 9476 perf_swevent_cancel_hrtimer(event); 9477 cpu_clock_event_update(event); 9478 } 9479 9480 static int cpu_clock_event_add(struct perf_event *event, int flags) 9481 { 9482 if (flags & PERF_EF_START) 9483 cpu_clock_event_start(event, flags); 9484 perf_event_update_userpage(event); 9485 9486 return 0; 9487 } 9488 9489 static void cpu_clock_event_del(struct perf_event *event, int flags) 9490 { 9491 cpu_clock_event_stop(event, flags); 9492 } 9493 9494 static void cpu_clock_event_read(struct perf_event *event) 9495 { 9496 cpu_clock_event_update(event); 9497 } 9498 9499 static int cpu_clock_event_init(struct perf_event *event) 9500 { 9501 if (event->attr.type != PERF_TYPE_SOFTWARE) 9502 return -ENOENT; 9503 9504 if (event->attr.config != PERF_COUNT_SW_CPU_CLOCK) 9505 return -ENOENT; 9506 9507 /* 9508 * no branch sampling for software events 9509 */ 9510 if (has_branch_stack(event)) 9511 return -EOPNOTSUPP; 9512 9513 perf_swevent_init_hrtimer(event); 9514 9515 return 0; 9516 } 9517 9518 static struct pmu perf_cpu_clock = { 9519 .task_ctx_nr = perf_sw_context, 9520 9521 .capabilities = PERF_PMU_CAP_NO_NMI, 9522 9523 .event_init = cpu_clock_event_init, 9524 .add = cpu_clock_event_add, 9525 .del = cpu_clock_event_del, 9526 .start = cpu_clock_event_start, 9527 .stop = cpu_clock_event_stop, 9528 .read = cpu_clock_event_read, 9529 }; 9530 9531 /* 9532 * Software event: task time clock 9533 */ 9534 9535 static void task_clock_event_update(struct perf_event *event, u64 now) 9536 { 9537 u64 prev; 9538 s64 delta; 9539 9540 prev = local64_xchg(&event->hw.prev_count, now); 9541 delta = now - prev; 9542 local64_add(delta, &event->count); 9543 } 9544 9545 static void task_clock_event_start(struct perf_event *event, int flags) 9546 { 9547 local64_set(&event->hw.prev_count, event->ctx->time); 9548 perf_swevent_start_hrtimer(event); 9549 } 9550 9551 static void task_clock_event_stop(struct perf_event *event, int flags) 9552 { 9553 perf_swevent_cancel_hrtimer(event); 9554 task_clock_event_update(event, event->ctx->time); 9555 } 9556 9557 static int task_clock_event_add(struct perf_event *event, int flags) 9558 { 9559 if (flags & PERF_EF_START) 9560 task_clock_event_start(event, flags); 9561 perf_event_update_userpage(event); 9562 9563 return 0; 9564 } 9565 9566 static void task_clock_event_del(struct perf_event *event, int flags) 9567 { 9568 task_clock_event_stop(event, PERF_EF_UPDATE); 9569 } 9570 9571 static void task_clock_event_read(struct perf_event *event) 9572 { 9573 u64 now = perf_clock(); 9574 u64 delta = now - event->ctx->timestamp; 9575 u64 time = event->ctx->time + delta; 9576 9577 task_clock_event_update(event, time); 9578 } 9579 9580 static int task_clock_event_init(struct perf_event *event) 9581 { 9582 if (event->attr.type != PERF_TYPE_SOFTWARE) 9583 return -ENOENT; 9584 9585 if (event->attr.config != PERF_COUNT_SW_TASK_CLOCK) 9586 return -ENOENT; 9587 9588 /* 9589 * no branch sampling for software events 9590 */ 9591 if (has_branch_stack(event)) 9592 return -EOPNOTSUPP; 9593 9594 perf_swevent_init_hrtimer(event); 9595 9596 return 0; 9597 } 9598 9599 static struct pmu perf_task_clock = { 9600 .task_ctx_nr = perf_sw_context, 9601 9602 .capabilities = PERF_PMU_CAP_NO_NMI, 9603 9604 .event_init = task_clock_event_init, 9605 .add = task_clock_event_add, 9606 .del = task_clock_event_del, 9607 .start = task_clock_event_start, 9608 .stop = task_clock_event_stop, 9609 .read = task_clock_event_read, 9610 }; 9611 9612 static void perf_pmu_nop_void(struct pmu *pmu) 9613 { 9614 } 9615 9616 static void perf_pmu_nop_txn(struct pmu *pmu, unsigned int flags) 9617 { 9618 } 9619 9620 static int perf_pmu_nop_int(struct pmu *pmu) 9621 { 9622 return 0; 9623 } 9624 9625 static int perf_event_nop_int(struct perf_event *event, u64 value) 9626 { 9627 return 0; 9628 } 9629 9630 static DEFINE_PER_CPU(unsigned int, nop_txn_flags); 9631 9632 static void perf_pmu_start_txn(struct pmu *pmu, unsigned int flags) 9633 { 9634 __this_cpu_write(nop_txn_flags, flags); 9635 9636 if (flags & ~PERF_PMU_TXN_ADD) 9637 return; 9638 9639 perf_pmu_disable(pmu); 9640 } 9641 9642 static int perf_pmu_commit_txn(struct pmu *pmu) 9643 { 9644 unsigned int flags = __this_cpu_read(nop_txn_flags); 9645 9646 __this_cpu_write(nop_txn_flags, 0); 9647 9648 if (flags & ~PERF_PMU_TXN_ADD) 9649 return 0; 9650 9651 perf_pmu_enable(pmu); 9652 return 0; 9653 } 9654 9655 static void perf_pmu_cancel_txn(struct pmu *pmu) 9656 { 9657 unsigned int flags = __this_cpu_read(nop_txn_flags); 9658 9659 __this_cpu_write(nop_txn_flags, 0); 9660 9661 if (flags & ~PERF_PMU_TXN_ADD) 9662 return; 9663 9664 perf_pmu_enable(pmu); 9665 } 9666 9667 static int perf_event_idx_default(struct perf_event *event) 9668 { 9669 return 0; 9670 } 9671 9672 /* 9673 * Ensures all contexts with the same task_ctx_nr have the same 9674 * pmu_cpu_context too. 9675 */ 9676 static struct perf_cpu_context __percpu *find_pmu_context(int ctxn) 9677 { 9678 struct pmu *pmu; 9679 9680 if (ctxn < 0) 9681 return NULL; 9682 9683 list_for_each_entry(pmu, &pmus, entry) { 9684 if (pmu->task_ctx_nr == ctxn) 9685 return pmu->pmu_cpu_context; 9686 } 9687 9688 return NULL; 9689 } 9690 9691 static void free_pmu_context(struct pmu *pmu) 9692 { 9693 /* 9694 * Static contexts such as perf_sw_context have a global lifetime 9695 * and may be shared between different PMUs. Avoid freeing them 9696 * when a single PMU is going away. 9697 */ 9698 if (pmu->task_ctx_nr > perf_invalid_context) 9699 return; 9700 9701 free_percpu(pmu->pmu_cpu_context); 9702 } 9703 9704 /* 9705 * Let userspace know that this PMU supports address range filtering: 9706 */ 9707 static ssize_t nr_addr_filters_show(struct device *dev, 9708 struct device_attribute *attr, 9709 char *page) 9710 { 9711 struct pmu *pmu = dev_get_drvdata(dev); 9712 9713 return snprintf(page, PAGE_SIZE - 1, "%d\n", pmu->nr_addr_filters); 9714 } 9715 DEVICE_ATTR_RO(nr_addr_filters); 9716 9717 static struct idr pmu_idr; 9718 9719 static ssize_t 9720 type_show(struct device *dev, struct device_attribute *attr, char *page) 9721 { 9722 struct pmu *pmu = dev_get_drvdata(dev); 9723 9724 return snprintf(page, PAGE_SIZE-1, "%d\n", pmu->type); 9725 } 9726 static DEVICE_ATTR_RO(type); 9727 9728 static ssize_t 9729 perf_event_mux_interval_ms_show(struct device *dev, 9730 struct device_attribute *attr, 9731 char *page) 9732 { 9733 struct pmu *pmu = dev_get_drvdata(dev); 9734 9735 return snprintf(page, PAGE_SIZE-1, "%d\n", pmu->hrtimer_interval_ms); 9736 } 9737 9738 static DEFINE_MUTEX(mux_interval_mutex); 9739 9740 static ssize_t 9741 perf_event_mux_interval_ms_store(struct device *dev, 9742 struct device_attribute *attr, 9743 const char *buf, size_t count) 9744 { 9745 struct pmu *pmu = dev_get_drvdata(dev); 9746 int timer, cpu, ret; 9747 9748 ret = kstrtoint(buf, 0, &timer); 9749 if (ret) 9750 return ret; 9751 9752 if (timer < 1) 9753 return -EINVAL; 9754 9755 /* same value, noting to do */ 9756 if (timer == pmu->hrtimer_interval_ms) 9757 return count; 9758 9759 mutex_lock(&mux_interval_mutex); 9760 pmu->hrtimer_interval_ms = timer; 9761 9762 /* update all cpuctx for this PMU */ 9763 cpus_read_lock(); 9764 for_each_online_cpu(cpu) { 9765 struct perf_cpu_context *cpuctx; 9766 cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu); 9767 cpuctx->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * timer); 9768 9769 cpu_function_call(cpu, 9770 (remote_function_f)perf_mux_hrtimer_restart, cpuctx); 9771 } 9772 cpus_read_unlock(); 9773 mutex_unlock(&mux_interval_mutex); 9774 9775 return count; 9776 } 9777 static DEVICE_ATTR_RW(perf_event_mux_interval_ms); 9778 9779 static struct attribute *pmu_dev_attrs[] = { 9780 &dev_attr_type.attr, 9781 &dev_attr_perf_event_mux_interval_ms.attr, 9782 NULL, 9783 }; 9784 ATTRIBUTE_GROUPS(pmu_dev); 9785 9786 static int pmu_bus_running; 9787 static struct bus_type pmu_bus = { 9788 .name = "event_source", 9789 .dev_groups = pmu_dev_groups, 9790 }; 9791 9792 static void pmu_dev_release(struct device *dev) 9793 { 9794 kfree(dev); 9795 } 9796 9797 static int pmu_dev_alloc(struct pmu *pmu) 9798 { 9799 int ret = -ENOMEM; 9800 9801 pmu->dev = kzalloc(sizeof(struct device), GFP_KERNEL); 9802 if (!pmu->dev) 9803 goto out; 9804 9805 pmu->dev->groups = pmu->attr_groups; 9806 device_initialize(pmu->dev); 9807 ret = dev_set_name(pmu->dev, "%s", pmu->name); 9808 if (ret) 9809 goto free_dev; 9810 9811 dev_set_drvdata(pmu->dev, pmu); 9812 pmu->dev->bus = &pmu_bus; 9813 pmu->dev->release = pmu_dev_release; 9814 ret = device_add(pmu->dev); 9815 if (ret) 9816 goto free_dev; 9817 9818 /* For PMUs with address filters, throw in an extra attribute: */ 9819 if (pmu->nr_addr_filters) 9820 ret = device_create_file(pmu->dev, &dev_attr_nr_addr_filters); 9821 9822 if (ret) 9823 goto del_dev; 9824 9825 out: 9826 return ret; 9827 9828 del_dev: 9829 device_del(pmu->dev); 9830 9831 free_dev: 9832 put_device(pmu->dev); 9833 goto out; 9834 } 9835 9836 static struct lock_class_key cpuctx_mutex; 9837 static struct lock_class_key cpuctx_lock; 9838 9839 int perf_pmu_register(struct pmu *pmu, const char *name, int type) 9840 { 9841 int cpu, ret; 9842 9843 mutex_lock(&pmus_lock); 9844 ret = -ENOMEM; 9845 pmu->pmu_disable_count = alloc_percpu(int); 9846 if (!pmu->pmu_disable_count) 9847 goto unlock; 9848 9849 pmu->type = -1; 9850 if (!name) 9851 goto skip_type; 9852 pmu->name = name; 9853 9854 if (type < 0) { 9855 type = idr_alloc(&pmu_idr, pmu, PERF_TYPE_MAX, 0, GFP_KERNEL); 9856 if (type < 0) { 9857 ret = type; 9858 goto free_pdc; 9859 } 9860 } 9861 pmu->type = type; 9862 9863 if (pmu_bus_running) { 9864 ret = pmu_dev_alloc(pmu); 9865 if (ret) 9866 goto free_idr; 9867 } 9868 9869 skip_type: 9870 if (pmu->task_ctx_nr == perf_hw_context) { 9871 static int hw_context_taken = 0; 9872 9873 /* 9874 * Other than systems with heterogeneous CPUs, it never makes 9875 * sense for two PMUs to share perf_hw_context. PMUs which are 9876 * uncore must use perf_invalid_context. 9877 */ 9878 if (WARN_ON_ONCE(hw_context_taken && 9879 !(pmu->capabilities & PERF_PMU_CAP_HETEROGENEOUS_CPUS))) 9880 pmu->task_ctx_nr = perf_invalid_context; 9881 9882 hw_context_taken = 1; 9883 } 9884 9885 pmu->pmu_cpu_context = find_pmu_context(pmu->task_ctx_nr); 9886 if (pmu->pmu_cpu_context) 9887 goto got_cpu_context; 9888 9889 ret = -ENOMEM; 9890 pmu->pmu_cpu_context = alloc_percpu(struct perf_cpu_context); 9891 if (!pmu->pmu_cpu_context) 9892 goto free_dev; 9893 9894 for_each_possible_cpu(cpu) { 9895 struct perf_cpu_context *cpuctx; 9896 9897 cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu); 9898 __perf_event_init_context(&cpuctx->ctx); 9899 lockdep_set_class(&cpuctx->ctx.mutex, &cpuctx_mutex); 9900 lockdep_set_class(&cpuctx->ctx.lock, &cpuctx_lock); 9901 cpuctx->ctx.pmu = pmu; 9902 cpuctx->online = cpumask_test_cpu(cpu, perf_online_mask); 9903 9904 __perf_mux_hrtimer_init(cpuctx, cpu); 9905 } 9906 9907 got_cpu_context: 9908 if (!pmu->start_txn) { 9909 if (pmu->pmu_enable) { 9910 /* 9911 * If we have pmu_enable/pmu_disable calls, install 9912 * transaction stubs that use that to try and batch 9913 * hardware accesses. 9914 */ 9915 pmu->start_txn = perf_pmu_start_txn; 9916 pmu->commit_txn = perf_pmu_commit_txn; 9917 pmu->cancel_txn = perf_pmu_cancel_txn; 9918 } else { 9919 pmu->start_txn = perf_pmu_nop_txn; 9920 pmu->commit_txn = perf_pmu_nop_int; 9921 pmu->cancel_txn = perf_pmu_nop_void; 9922 } 9923 } 9924 9925 if (!pmu->pmu_enable) { 9926 pmu->pmu_enable = perf_pmu_nop_void; 9927 pmu->pmu_disable = perf_pmu_nop_void; 9928 } 9929 9930 if (!pmu->check_period) 9931 pmu->check_period = perf_event_nop_int; 9932 9933 if (!pmu->event_idx) 9934 pmu->event_idx = perf_event_idx_default; 9935 9936 list_add_rcu(&pmu->entry, &pmus); 9937 atomic_set(&pmu->exclusive_cnt, 0); 9938 ret = 0; 9939 unlock: 9940 mutex_unlock(&pmus_lock); 9941 9942 return ret; 9943 9944 free_dev: 9945 device_del(pmu->dev); 9946 put_device(pmu->dev); 9947 9948 free_idr: 9949 if (pmu->type >= PERF_TYPE_MAX) 9950 idr_remove(&pmu_idr, pmu->type); 9951 9952 free_pdc: 9953 free_percpu(pmu->pmu_disable_count); 9954 goto unlock; 9955 } 9956 EXPORT_SYMBOL_GPL(perf_pmu_register); 9957 9958 void perf_pmu_unregister(struct pmu *pmu) 9959 { 9960 mutex_lock(&pmus_lock); 9961 list_del_rcu(&pmu->entry); 9962 9963 /* 9964 * We dereference the pmu list under both SRCU and regular RCU, so 9965 * synchronize against both of those. 9966 */ 9967 synchronize_srcu(&pmus_srcu); 9968 synchronize_rcu(); 9969 9970 free_percpu(pmu->pmu_disable_count); 9971 if (pmu->type >= PERF_TYPE_MAX) 9972 idr_remove(&pmu_idr, pmu->type); 9973 if (pmu_bus_running) { 9974 if (pmu->nr_addr_filters) 9975 device_remove_file(pmu->dev, &dev_attr_nr_addr_filters); 9976 device_del(pmu->dev); 9977 put_device(pmu->dev); 9978 } 9979 free_pmu_context(pmu); 9980 mutex_unlock(&pmus_lock); 9981 } 9982 EXPORT_SYMBOL_GPL(perf_pmu_unregister); 9983 9984 static int perf_try_init_event(struct pmu *pmu, struct perf_event *event) 9985 { 9986 struct perf_event_context *ctx = NULL; 9987 int ret; 9988 9989 if (!try_module_get(pmu->module)) 9990 return -ENODEV; 9991 9992 /* 9993 * A number of pmu->event_init() methods iterate the sibling_list to, 9994 * for example, validate if the group fits on the PMU. Therefore, 9995 * if this is a sibling event, acquire the ctx->mutex to protect 9996 * the sibling_list. 9997 */ 9998 if (event->group_leader != event && pmu->task_ctx_nr != perf_sw_context) { 9999 /* 10000 * This ctx->mutex can nest when we're called through 10001 * inheritance. See the perf_event_ctx_lock_nested() comment. 10002 */ 10003 ctx = perf_event_ctx_lock_nested(event->group_leader, 10004 SINGLE_DEPTH_NESTING); 10005 BUG_ON(!ctx); 10006 } 10007 10008 event->pmu = pmu; 10009 ret = pmu->event_init(event); 10010 10011 if (ctx) 10012 perf_event_ctx_unlock(event->group_leader, ctx); 10013 10014 if (!ret) { 10015 if (pmu->capabilities & PERF_PMU_CAP_NO_EXCLUDE && 10016 event_has_any_exclude_flag(event)) { 10017 if (event->destroy) 10018 event->destroy(event); 10019 ret = -EINVAL; 10020 } 10021 } 10022 10023 if (ret) 10024 module_put(pmu->module); 10025 10026 return ret; 10027 } 10028 10029 static struct pmu *perf_init_event(struct perf_event *event) 10030 { 10031 struct pmu *pmu; 10032 int idx; 10033 int ret; 10034 10035 idx = srcu_read_lock(&pmus_srcu); 10036 10037 /* Try parent's PMU first: */ 10038 if (event->parent && event->parent->pmu) { 10039 pmu = event->parent->pmu; 10040 ret = perf_try_init_event(pmu, event); 10041 if (!ret) 10042 goto unlock; 10043 } 10044 10045 rcu_read_lock(); 10046 pmu = idr_find(&pmu_idr, event->attr.type); 10047 rcu_read_unlock(); 10048 if (pmu) { 10049 ret = perf_try_init_event(pmu, event); 10050 if (ret) 10051 pmu = ERR_PTR(ret); 10052 goto unlock; 10053 } 10054 10055 list_for_each_entry_rcu(pmu, &pmus, entry) { 10056 ret = perf_try_init_event(pmu, event); 10057 if (!ret) 10058 goto unlock; 10059 10060 if (ret != -ENOENT) { 10061 pmu = ERR_PTR(ret); 10062 goto unlock; 10063 } 10064 } 10065 pmu = ERR_PTR(-ENOENT); 10066 unlock: 10067 srcu_read_unlock(&pmus_srcu, idx); 10068 10069 return pmu; 10070 } 10071 10072 static void attach_sb_event(struct perf_event *event) 10073 { 10074 struct pmu_event_list *pel = per_cpu_ptr(&pmu_sb_events, event->cpu); 10075 10076 raw_spin_lock(&pel->lock); 10077 list_add_rcu(&event->sb_list, &pel->list); 10078 raw_spin_unlock(&pel->lock); 10079 } 10080 10081 /* 10082 * We keep a list of all !task (and therefore per-cpu) events 10083 * that need to receive side-band records. 10084 * 10085 * This avoids having to scan all the various PMU per-cpu contexts 10086 * looking for them. 10087 */ 10088 static void account_pmu_sb_event(struct perf_event *event) 10089 { 10090 if (is_sb_event(event)) 10091 attach_sb_event(event); 10092 } 10093 10094 static void account_event_cpu(struct perf_event *event, int cpu) 10095 { 10096 if (event->parent) 10097 return; 10098 10099 if (is_cgroup_event(event)) 10100 atomic_inc(&per_cpu(perf_cgroup_events, cpu)); 10101 } 10102 10103 /* Freq events need the tick to stay alive (see perf_event_task_tick). */ 10104 static void account_freq_event_nohz(void) 10105 { 10106 #ifdef CONFIG_NO_HZ_FULL 10107 /* Lock so we don't race with concurrent unaccount */ 10108 spin_lock(&nr_freq_lock); 10109 if (atomic_inc_return(&nr_freq_events) == 1) 10110 tick_nohz_dep_set(TICK_DEP_BIT_PERF_EVENTS); 10111 spin_unlock(&nr_freq_lock); 10112 #endif 10113 } 10114 10115 static void account_freq_event(void) 10116 { 10117 if (tick_nohz_full_enabled()) 10118 account_freq_event_nohz(); 10119 else 10120 atomic_inc(&nr_freq_events); 10121 } 10122 10123 10124 static void account_event(struct perf_event *event) 10125 { 10126 bool inc = false; 10127 10128 if (event->parent) 10129 return; 10130 10131 if (event->attach_state & PERF_ATTACH_TASK) 10132 inc = true; 10133 if (event->attr.mmap || event->attr.mmap_data) 10134 atomic_inc(&nr_mmap_events); 10135 if (event->attr.comm) 10136 atomic_inc(&nr_comm_events); 10137 if (event->attr.namespaces) 10138 atomic_inc(&nr_namespaces_events); 10139 if (event->attr.task) 10140 atomic_inc(&nr_task_events); 10141 if (event->attr.freq) 10142 account_freq_event(); 10143 if (event->attr.context_switch) { 10144 atomic_inc(&nr_switch_events); 10145 inc = true; 10146 } 10147 if (has_branch_stack(event)) 10148 inc = true; 10149 if (is_cgroup_event(event)) 10150 inc = true; 10151 if (event->attr.ksymbol) 10152 atomic_inc(&nr_ksymbol_events); 10153 if (event->attr.bpf_event) 10154 atomic_inc(&nr_bpf_events); 10155 10156 if (inc) { 10157 /* 10158 * We need the mutex here because static_branch_enable() 10159 * must complete *before* the perf_sched_count increment 10160 * becomes visible. 10161 */ 10162 if (atomic_inc_not_zero(&perf_sched_count)) 10163 goto enabled; 10164 10165 mutex_lock(&perf_sched_mutex); 10166 if (!atomic_read(&perf_sched_count)) { 10167 static_branch_enable(&perf_sched_events); 10168 /* 10169 * Guarantee that all CPUs observe they key change and 10170 * call the perf scheduling hooks before proceeding to 10171 * install events that need them. 10172 */ 10173 synchronize_rcu(); 10174 } 10175 /* 10176 * Now that we have waited for the sync_sched(), allow further 10177 * increments to by-pass the mutex. 10178 */ 10179 atomic_inc(&perf_sched_count); 10180 mutex_unlock(&perf_sched_mutex); 10181 } 10182 enabled: 10183 10184 account_event_cpu(event, event->cpu); 10185 10186 account_pmu_sb_event(event); 10187 } 10188 10189 /* 10190 * Allocate and initialize an event structure 10191 */ 10192 static struct perf_event * 10193 perf_event_alloc(struct perf_event_attr *attr, int cpu, 10194 struct task_struct *task, 10195 struct perf_event *group_leader, 10196 struct perf_event *parent_event, 10197 perf_overflow_handler_t overflow_handler, 10198 void *context, int cgroup_fd) 10199 { 10200 struct pmu *pmu; 10201 struct perf_event *event; 10202 struct hw_perf_event *hwc; 10203 long err = -EINVAL; 10204 10205 if ((unsigned)cpu >= nr_cpu_ids) { 10206 if (!task || cpu != -1) 10207 return ERR_PTR(-EINVAL); 10208 } 10209 10210 event = kzalloc(sizeof(*event), GFP_KERNEL); 10211 if (!event) 10212 return ERR_PTR(-ENOMEM); 10213 10214 /* 10215 * Single events are their own group leaders, with an 10216 * empty sibling list: 10217 */ 10218 if (!group_leader) 10219 group_leader = event; 10220 10221 mutex_init(&event->child_mutex); 10222 INIT_LIST_HEAD(&event->child_list); 10223 10224 INIT_LIST_HEAD(&event->event_entry); 10225 INIT_LIST_HEAD(&event->sibling_list); 10226 INIT_LIST_HEAD(&event->active_list); 10227 init_event_group(event); 10228 INIT_LIST_HEAD(&event->rb_entry); 10229 INIT_LIST_HEAD(&event->active_entry); 10230 INIT_LIST_HEAD(&event->addr_filters.list); 10231 INIT_HLIST_NODE(&event->hlist_entry); 10232 10233 10234 init_waitqueue_head(&event->waitq); 10235 init_irq_work(&event->pending, perf_pending_event); 10236 10237 mutex_init(&event->mmap_mutex); 10238 raw_spin_lock_init(&event->addr_filters.lock); 10239 10240 atomic_long_set(&event->refcount, 1); 10241 event->cpu = cpu; 10242 event->attr = *attr; 10243 event->group_leader = group_leader; 10244 event->pmu = NULL; 10245 event->oncpu = -1; 10246 10247 event->parent = parent_event; 10248 10249 event->ns = get_pid_ns(task_active_pid_ns(current)); 10250 event->id = atomic64_inc_return(&perf_event_id); 10251 10252 event->state = PERF_EVENT_STATE_INACTIVE; 10253 10254 if (task) { 10255 event->attach_state = PERF_ATTACH_TASK; 10256 /* 10257 * XXX pmu::event_init needs to know what task to account to 10258 * and we cannot use the ctx information because we need the 10259 * pmu before we get a ctx. 10260 */ 10261 get_task_struct(task); 10262 event->hw.target = task; 10263 } 10264 10265 event->clock = &local_clock; 10266 if (parent_event) 10267 event->clock = parent_event->clock; 10268 10269 if (!overflow_handler && parent_event) { 10270 overflow_handler = parent_event->overflow_handler; 10271 context = parent_event->overflow_handler_context; 10272 #if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_EVENT_TRACING) 10273 if (overflow_handler == bpf_overflow_handler) { 10274 struct bpf_prog *prog = bpf_prog_inc(parent_event->prog); 10275 10276 if (IS_ERR(prog)) { 10277 err = PTR_ERR(prog); 10278 goto err_ns; 10279 } 10280 event->prog = prog; 10281 event->orig_overflow_handler = 10282 parent_event->orig_overflow_handler; 10283 } 10284 #endif 10285 } 10286 10287 if (overflow_handler) { 10288 event->overflow_handler = overflow_handler; 10289 event->overflow_handler_context = context; 10290 } else if (is_write_backward(event)){ 10291 event->overflow_handler = perf_event_output_backward; 10292 event->overflow_handler_context = NULL; 10293 } else { 10294 event->overflow_handler = perf_event_output_forward; 10295 event->overflow_handler_context = NULL; 10296 } 10297 10298 perf_event__state_init(event); 10299 10300 pmu = NULL; 10301 10302 hwc = &event->hw; 10303 hwc->sample_period = attr->sample_period; 10304 if (attr->freq && attr->sample_freq) 10305 hwc->sample_period = 1; 10306 hwc->last_period = hwc->sample_period; 10307 10308 local64_set(&hwc->period_left, hwc->sample_period); 10309 10310 /* 10311 * We currently do not support PERF_SAMPLE_READ on inherited events. 10312 * See perf_output_read(). 10313 */ 10314 if (attr->inherit && (attr->sample_type & PERF_SAMPLE_READ)) 10315 goto err_ns; 10316 10317 if (!has_branch_stack(event)) 10318 event->attr.branch_sample_type = 0; 10319 10320 if (cgroup_fd != -1) { 10321 err = perf_cgroup_connect(cgroup_fd, event, attr, group_leader); 10322 if (err) 10323 goto err_ns; 10324 } 10325 10326 pmu = perf_init_event(event); 10327 if (IS_ERR(pmu)) { 10328 err = PTR_ERR(pmu); 10329 goto err_ns; 10330 } 10331 10332 err = exclusive_event_init(event); 10333 if (err) 10334 goto err_pmu; 10335 10336 if (has_addr_filter(event)) { 10337 event->addr_filter_ranges = kcalloc(pmu->nr_addr_filters, 10338 sizeof(struct perf_addr_filter_range), 10339 GFP_KERNEL); 10340 if (!event->addr_filter_ranges) { 10341 err = -ENOMEM; 10342 goto err_per_task; 10343 } 10344 10345 /* 10346 * Clone the parent's vma offsets: they are valid until exec() 10347 * even if the mm is not shared with the parent. 10348 */ 10349 if (event->parent) { 10350 struct perf_addr_filters_head *ifh = perf_event_addr_filters(event); 10351 10352 raw_spin_lock_irq(&ifh->lock); 10353 memcpy(event->addr_filter_ranges, 10354 event->parent->addr_filter_ranges, 10355 pmu->nr_addr_filters * sizeof(struct perf_addr_filter_range)); 10356 raw_spin_unlock_irq(&ifh->lock); 10357 } 10358 10359 /* force hw sync on the address filters */ 10360 event->addr_filters_gen = 1; 10361 } 10362 10363 if (!event->parent) { 10364 if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) { 10365 err = get_callchain_buffers(attr->sample_max_stack); 10366 if (err) 10367 goto err_addr_filters; 10368 } 10369 } 10370 10371 /* symmetric to unaccount_event() in _free_event() */ 10372 account_event(event); 10373 10374 return event; 10375 10376 err_addr_filters: 10377 kfree(event->addr_filter_ranges); 10378 10379 err_per_task: 10380 exclusive_event_destroy(event); 10381 10382 err_pmu: 10383 if (event->destroy) 10384 event->destroy(event); 10385 module_put(pmu->module); 10386 err_ns: 10387 if (is_cgroup_event(event)) 10388 perf_detach_cgroup(event); 10389 if (event->ns) 10390 put_pid_ns(event->ns); 10391 if (event->hw.target) 10392 put_task_struct(event->hw.target); 10393 kfree(event); 10394 10395 return ERR_PTR(err); 10396 } 10397 10398 static int perf_copy_attr(struct perf_event_attr __user *uattr, 10399 struct perf_event_attr *attr) 10400 { 10401 u32 size; 10402 int ret; 10403 10404 if (!access_ok(uattr, PERF_ATTR_SIZE_VER0)) 10405 return -EFAULT; 10406 10407 /* 10408 * zero the full structure, so that a short copy will be nice. 10409 */ 10410 memset(attr, 0, sizeof(*attr)); 10411 10412 ret = get_user(size, &uattr->size); 10413 if (ret) 10414 return ret; 10415 10416 if (size > PAGE_SIZE) /* silly large */ 10417 goto err_size; 10418 10419 if (!size) /* abi compat */ 10420 size = PERF_ATTR_SIZE_VER0; 10421 10422 if (size < PERF_ATTR_SIZE_VER0) 10423 goto err_size; 10424 10425 /* 10426 * If we're handed a bigger struct than we know of, 10427 * ensure all the unknown bits are 0 - i.e. new 10428 * user-space does not rely on any kernel feature 10429 * extensions we dont know about yet. 10430 */ 10431 if (size > sizeof(*attr)) { 10432 unsigned char __user *addr; 10433 unsigned char __user *end; 10434 unsigned char val; 10435 10436 addr = (void __user *)uattr + sizeof(*attr); 10437 end = (void __user *)uattr + size; 10438 10439 for (; addr < end; addr++) { 10440 ret = get_user(val, addr); 10441 if (ret) 10442 return ret; 10443 if (val) 10444 goto err_size; 10445 } 10446 size = sizeof(*attr); 10447 } 10448 10449 ret = copy_from_user(attr, uattr, size); 10450 if (ret) 10451 return -EFAULT; 10452 10453 attr->size = size; 10454 10455 if (attr->__reserved_1) 10456 return -EINVAL; 10457 10458 if (attr->sample_type & ~(PERF_SAMPLE_MAX-1)) 10459 return -EINVAL; 10460 10461 if (attr->read_format & ~(PERF_FORMAT_MAX-1)) 10462 return -EINVAL; 10463 10464 if (attr->sample_type & PERF_SAMPLE_BRANCH_STACK) { 10465 u64 mask = attr->branch_sample_type; 10466 10467 /* only using defined bits */ 10468 if (mask & ~(PERF_SAMPLE_BRANCH_MAX-1)) 10469 return -EINVAL; 10470 10471 /* at least one branch bit must be set */ 10472 if (!(mask & ~PERF_SAMPLE_BRANCH_PLM_ALL)) 10473 return -EINVAL; 10474 10475 /* propagate priv level, when not set for branch */ 10476 if (!(mask & PERF_SAMPLE_BRANCH_PLM_ALL)) { 10477 10478 /* exclude_kernel checked on syscall entry */ 10479 if (!attr->exclude_kernel) 10480 mask |= PERF_SAMPLE_BRANCH_KERNEL; 10481 10482 if (!attr->exclude_user) 10483 mask |= PERF_SAMPLE_BRANCH_USER; 10484 10485 if (!attr->exclude_hv) 10486 mask |= PERF_SAMPLE_BRANCH_HV; 10487 /* 10488 * adjust user setting (for HW filter setup) 10489 */ 10490 attr->branch_sample_type = mask; 10491 } 10492 /* privileged levels capture (kernel, hv): check permissions */ 10493 if ((mask & PERF_SAMPLE_BRANCH_PERM_PLM) 10494 && perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN)) 10495 return -EACCES; 10496 } 10497 10498 if (attr->sample_type & PERF_SAMPLE_REGS_USER) { 10499 ret = perf_reg_validate(attr->sample_regs_user); 10500 if (ret) 10501 return ret; 10502 } 10503 10504 if (attr->sample_type & PERF_SAMPLE_STACK_USER) { 10505 if (!arch_perf_have_user_stack_dump()) 10506 return -ENOSYS; 10507 10508 /* 10509 * We have __u32 type for the size, but so far 10510 * we can only use __u16 as maximum due to the 10511 * __u16 sample size limit. 10512 */ 10513 if (attr->sample_stack_user >= USHRT_MAX) 10514 return -EINVAL; 10515 else if (!IS_ALIGNED(attr->sample_stack_user, sizeof(u64))) 10516 return -EINVAL; 10517 } 10518 10519 if (!attr->sample_max_stack) 10520 attr->sample_max_stack = sysctl_perf_event_max_stack; 10521 10522 if (attr->sample_type & PERF_SAMPLE_REGS_INTR) 10523 ret = perf_reg_validate(attr->sample_regs_intr); 10524 out: 10525 return ret; 10526 10527 err_size: 10528 put_user(sizeof(*attr), &uattr->size); 10529 ret = -E2BIG; 10530 goto out; 10531 } 10532 10533 static int 10534 perf_event_set_output(struct perf_event *event, struct perf_event *output_event) 10535 { 10536 struct ring_buffer *rb = NULL; 10537 int ret = -EINVAL; 10538 10539 if (!output_event) 10540 goto set; 10541 10542 /* don't allow circular references */ 10543 if (event == output_event) 10544 goto out; 10545 10546 /* 10547 * Don't allow cross-cpu buffers 10548 */ 10549 if (output_event->cpu != event->cpu) 10550 goto out; 10551 10552 /* 10553 * If its not a per-cpu rb, it must be the same task. 10554 */ 10555 if (output_event->cpu == -1 && output_event->ctx != event->ctx) 10556 goto out; 10557 10558 /* 10559 * Mixing clocks in the same buffer is trouble you don't need. 10560 */ 10561 if (output_event->clock != event->clock) 10562 goto out; 10563 10564 /* 10565 * Either writing ring buffer from beginning or from end. 10566 * Mixing is not allowed. 10567 */ 10568 if (is_write_backward(output_event) != is_write_backward(event)) 10569 goto out; 10570 10571 /* 10572 * If both events generate aux data, they must be on the same PMU 10573 */ 10574 if (has_aux(event) && has_aux(output_event) && 10575 event->pmu != output_event->pmu) 10576 goto out; 10577 10578 set: 10579 mutex_lock(&event->mmap_mutex); 10580 /* Can't redirect output if we've got an active mmap() */ 10581 if (atomic_read(&event->mmap_count)) 10582 goto unlock; 10583 10584 if (output_event) { 10585 /* get the rb we want to redirect to */ 10586 rb = ring_buffer_get(output_event); 10587 if (!rb) 10588 goto unlock; 10589 } 10590 10591 ring_buffer_attach(event, rb); 10592 10593 ret = 0; 10594 unlock: 10595 mutex_unlock(&event->mmap_mutex); 10596 10597 out: 10598 return ret; 10599 } 10600 10601 static void mutex_lock_double(struct mutex *a, struct mutex *b) 10602 { 10603 if (b < a) 10604 swap(a, b); 10605 10606 mutex_lock(a); 10607 mutex_lock_nested(b, SINGLE_DEPTH_NESTING); 10608 } 10609 10610 static int perf_event_set_clock(struct perf_event *event, clockid_t clk_id) 10611 { 10612 bool nmi_safe = false; 10613 10614 switch (clk_id) { 10615 case CLOCK_MONOTONIC: 10616 event->clock = &ktime_get_mono_fast_ns; 10617 nmi_safe = true; 10618 break; 10619 10620 case CLOCK_MONOTONIC_RAW: 10621 event->clock = &ktime_get_raw_fast_ns; 10622 nmi_safe = true; 10623 break; 10624 10625 case CLOCK_REALTIME: 10626 event->clock = &ktime_get_real_ns; 10627 break; 10628 10629 case CLOCK_BOOTTIME: 10630 event->clock = &ktime_get_boot_ns; 10631 break; 10632 10633 case CLOCK_TAI: 10634 event->clock = &ktime_get_tai_ns; 10635 break; 10636 10637 default: 10638 return -EINVAL; 10639 } 10640 10641 if (!nmi_safe && !(event->pmu->capabilities & PERF_PMU_CAP_NO_NMI)) 10642 return -EINVAL; 10643 10644 return 0; 10645 } 10646 10647 /* 10648 * Variation on perf_event_ctx_lock_nested(), except we take two context 10649 * mutexes. 10650 */ 10651 static struct perf_event_context * 10652 __perf_event_ctx_lock_double(struct perf_event *group_leader, 10653 struct perf_event_context *ctx) 10654 { 10655 struct perf_event_context *gctx; 10656 10657 again: 10658 rcu_read_lock(); 10659 gctx = READ_ONCE(group_leader->ctx); 10660 if (!refcount_inc_not_zero(&gctx->refcount)) { 10661 rcu_read_unlock(); 10662 goto again; 10663 } 10664 rcu_read_unlock(); 10665 10666 mutex_lock_double(&gctx->mutex, &ctx->mutex); 10667 10668 if (group_leader->ctx != gctx) { 10669 mutex_unlock(&ctx->mutex); 10670 mutex_unlock(&gctx->mutex); 10671 put_ctx(gctx); 10672 goto again; 10673 } 10674 10675 return gctx; 10676 } 10677 10678 /** 10679 * sys_perf_event_open - open a performance event, associate it to a task/cpu 10680 * 10681 * @attr_uptr: event_id type attributes for monitoring/sampling 10682 * @pid: target pid 10683 * @cpu: target cpu 10684 * @group_fd: group leader event fd 10685 */ 10686 SYSCALL_DEFINE5(perf_event_open, 10687 struct perf_event_attr __user *, attr_uptr, 10688 pid_t, pid, int, cpu, int, group_fd, unsigned long, flags) 10689 { 10690 struct perf_event *group_leader = NULL, *output_event = NULL; 10691 struct perf_event *event, *sibling; 10692 struct perf_event_attr attr; 10693 struct perf_event_context *ctx, *uninitialized_var(gctx); 10694 struct file *event_file = NULL; 10695 struct fd group = {NULL, 0}; 10696 struct task_struct *task = NULL; 10697 struct pmu *pmu; 10698 int event_fd; 10699 int move_group = 0; 10700 int err; 10701 int f_flags = O_RDWR; 10702 int cgroup_fd = -1; 10703 10704 /* for future expandability... */ 10705 if (flags & ~PERF_FLAG_ALL) 10706 return -EINVAL; 10707 10708 err = perf_copy_attr(attr_uptr, &attr); 10709 if (err) 10710 return err; 10711 10712 if (!attr.exclude_kernel) { 10713 if (perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN)) 10714 return -EACCES; 10715 } 10716 10717 if (attr.namespaces) { 10718 if (!capable(CAP_SYS_ADMIN)) 10719 return -EACCES; 10720 } 10721 10722 if (attr.freq) { 10723 if (attr.sample_freq > sysctl_perf_event_sample_rate) 10724 return -EINVAL; 10725 } else { 10726 if (attr.sample_period & (1ULL << 63)) 10727 return -EINVAL; 10728 } 10729 10730 /* Only privileged users can get physical addresses */ 10731 if ((attr.sample_type & PERF_SAMPLE_PHYS_ADDR) && 10732 perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN)) 10733 return -EACCES; 10734 10735 /* 10736 * In cgroup mode, the pid argument is used to pass the fd 10737 * opened to the cgroup directory in cgroupfs. The cpu argument 10738 * designates the cpu on which to monitor threads from that 10739 * cgroup. 10740 */ 10741 if ((flags & PERF_FLAG_PID_CGROUP) && (pid == -1 || cpu == -1)) 10742 return -EINVAL; 10743 10744 if (flags & PERF_FLAG_FD_CLOEXEC) 10745 f_flags |= O_CLOEXEC; 10746 10747 event_fd = get_unused_fd_flags(f_flags); 10748 if (event_fd < 0) 10749 return event_fd; 10750 10751 if (group_fd != -1) { 10752 err = perf_fget_light(group_fd, &group); 10753 if (err) 10754 goto err_fd; 10755 group_leader = group.file->private_data; 10756 if (flags & PERF_FLAG_FD_OUTPUT) 10757 output_event = group_leader; 10758 if (flags & PERF_FLAG_FD_NO_GROUP) 10759 group_leader = NULL; 10760 } 10761 10762 if (pid != -1 && !(flags & PERF_FLAG_PID_CGROUP)) { 10763 task = find_lively_task_by_vpid(pid); 10764 if (IS_ERR(task)) { 10765 err = PTR_ERR(task); 10766 goto err_group_fd; 10767 } 10768 } 10769 10770 if (task && group_leader && 10771 group_leader->attr.inherit != attr.inherit) { 10772 err = -EINVAL; 10773 goto err_task; 10774 } 10775 10776 if (task) { 10777 err = mutex_lock_interruptible(&task->signal->cred_guard_mutex); 10778 if (err) 10779 goto err_task; 10780 10781 /* 10782 * Reuse ptrace permission checks for now. 10783 * 10784 * We must hold cred_guard_mutex across this and any potential 10785 * perf_install_in_context() call for this new event to 10786 * serialize against exec() altering our credentials (and the 10787 * perf_event_exit_task() that could imply). 10788 */ 10789 err = -EACCES; 10790 if (!ptrace_may_access(task, PTRACE_MODE_READ_REALCREDS)) 10791 goto err_cred; 10792 } 10793 10794 if (flags & PERF_FLAG_PID_CGROUP) 10795 cgroup_fd = pid; 10796 10797 event = perf_event_alloc(&attr, cpu, task, group_leader, NULL, 10798 NULL, NULL, cgroup_fd); 10799 if (IS_ERR(event)) { 10800 err = PTR_ERR(event); 10801 goto err_cred; 10802 } 10803 10804 if (is_sampling_event(event)) { 10805 if (event->pmu->capabilities & PERF_PMU_CAP_NO_INTERRUPT) { 10806 err = -EOPNOTSUPP; 10807 goto err_alloc; 10808 } 10809 } 10810 10811 /* 10812 * Special case software events and allow them to be part of 10813 * any hardware group. 10814 */ 10815 pmu = event->pmu; 10816 10817 if (attr.use_clockid) { 10818 err = perf_event_set_clock(event, attr.clockid); 10819 if (err) 10820 goto err_alloc; 10821 } 10822 10823 if (pmu->task_ctx_nr == perf_sw_context) 10824 event->event_caps |= PERF_EV_CAP_SOFTWARE; 10825 10826 if (group_leader) { 10827 if (is_software_event(event) && 10828 !in_software_context(group_leader)) { 10829 /* 10830 * If the event is a sw event, but the group_leader 10831 * is on hw context. 10832 * 10833 * Allow the addition of software events to hw 10834 * groups, this is safe because software events 10835 * never fail to schedule. 10836 */ 10837 pmu = group_leader->ctx->pmu; 10838 } else if (!is_software_event(event) && 10839 is_software_event(group_leader) && 10840 (group_leader->group_caps & PERF_EV_CAP_SOFTWARE)) { 10841 /* 10842 * In case the group is a pure software group, and we 10843 * try to add a hardware event, move the whole group to 10844 * the hardware context. 10845 */ 10846 move_group = 1; 10847 } 10848 } 10849 10850 /* 10851 * Get the target context (task or percpu): 10852 */ 10853 ctx = find_get_context(pmu, task, event); 10854 if (IS_ERR(ctx)) { 10855 err = PTR_ERR(ctx); 10856 goto err_alloc; 10857 } 10858 10859 if ((pmu->capabilities & PERF_PMU_CAP_EXCLUSIVE) && group_leader) { 10860 err = -EBUSY; 10861 goto err_context; 10862 } 10863 10864 /* 10865 * Look up the group leader (we will attach this event to it): 10866 */ 10867 if (group_leader) { 10868 err = -EINVAL; 10869 10870 /* 10871 * Do not allow a recursive hierarchy (this new sibling 10872 * becoming part of another group-sibling): 10873 */ 10874 if (group_leader->group_leader != group_leader) 10875 goto err_context; 10876 10877 /* All events in a group should have the same clock */ 10878 if (group_leader->clock != event->clock) 10879 goto err_context; 10880 10881 /* 10882 * Make sure we're both events for the same CPU; 10883 * grouping events for different CPUs is broken; since 10884 * you can never concurrently schedule them anyhow. 10885 */ 10886 if (group_leader->cpu != event->cpu) 10887 goto err_context; 10888 10889 /* 10890 * Make sure we're both on the same task, or both 10891 * per-CPU events. 10892 */ 10893 if (group_leader->ctx->task != ctx->task) 10894 goto err_context; 10895 10896 /* 10897 * Do not allow to attach to a group in a different task 10898 * or CPU context. If we're moving SW events, we'll fix 10899 * this up later, so allow that. 10900 */ 10901 if (!move_group && group_leader->ctx != ctx) 10902 goto err_context; 10903 10904 /* 10905 * Only a group leader can be exclusive or pinned 10906 */ 10907 if (attr.exclusive || attr.pinned) 10908 goto err_context; 10909 } 10910 10911 if (output_event) { 10912 err = perf_event_set_output(event, output_event); 10913 if (err) 10914 goto err_context; 10915 } 10916 10917 event_file = anon_inode_getfile("[perf_event]", &perf_fops, event, 10918 f_flags); 10919 if (IS_ERR(event_file)) { 10920 err = PTR_ERR(event_file); 10921 event_file = NULL; 10922 goto err_context; 10923 } 10924 10925 if (move_group) { 10926 gctx = __perf_event_ctx_lock_double(group_leader, ctx); 10927 10928 if (gctx->task == TASK_TOMBSTONE) { 10929 err = -ESRCH; 10930 goto err_locked; 10931 } 10932 10933 /* 10934 * Check if we raced against another sys_perf_event_open() call 10935 * moving the software group underneath us. 10936 */ 10937 if (!(group_leader->group_caps & PERF_EV_CAP_SOFTWARE)) { 10938 /* 10939 * If someone moved the group out from under us, check 10940 * if this new event wound up on the same ctx, if so 10941 * its the regular !move_group case, otherwise fail. 10942 */ 10943 if (gctx != ctx) { 10944 err = -EINVAL; 10945 goto err_locked; 10946 } else { 10947 perf_event_ctx_unlock(group_leader, gctx); 10948 move_group = 0; 10949 } 10950 } 10951 } else { 10952 mutex_lock(&ctx->mutex); 10953 } 10954 10955 if (ctx->task == TASK_TOMBSTONE) { 10956 err = -ESRCH; 10957 goto err_locked; 10958 } 10959 10960 if (!perf_event_validate_size(event)) { 10961 err = -E2BIG; 10962 goto err_locked; 10963 } 10964 10965 if (!task) { 10966 /* 10967 * Check if the @cpu we're creating an event for is online. 10968 * 10969 * We use the perf_cpu_context::ctx::mutex to serialize against 10970 * the hotplug notifiers. See perf_event_{init,exit}_cpu(). 10971 */ 10972 struct perf_cpu_context *cpuctx = 10973 container_of(ctx, struct perf_cpu_context, ctx); 10974 10975 if (!cpuctx->online) { 10976 err = -ENODEV; 10977 goto err_locked; 10978 } 10979 } 10980 10981 10982 /* 10983 * Must be under the same ctx::mutex as perf_install_in_context(), 10984 * because we need to serialize with concurrent event creation. 10985 */ 10986 if (!exclusive_event_installable(event, ctx)) { 10987 /* exclusive and group stuff are assumed mutually exclusive */ 10988 WARN_ON_ONCE(move_group); 10989 10990 err = -EBUSY; 10991 goto err_locked; 10992 } 10993 10994 WARN_ON_ONCE(ctx->parent_ctx); 10995 10996 /* 10997 * This is the point on no return; we cannot fail hereafter. This is 10998 * where we start modifying current state. 10999 */ 11000 11001 if (move_group) { 11002 /* 11003 * See perf_event_ctx_lock() for comments on the details 11004 * of swizzling perf_event::ctx. 11005 */ 11006 perf_remove_from_context(group_leader, 0); 11007 put_ctx(gctx); 11008 11009 for_each_sibling_event(sibling, group_leader) { 11010 perf_remove_from_context(sibling, 0); 11011 put_ctx(gctx); 11012 } 11013 11014 /* 11015 * Wait for everybody to stop referencing the events through 11016 * the old lists, before installing it on new lists. 11017 */ 11018 synchronize_rcu(); 11019 11020 /* 11021 * Install the group siblings before the group leader. 11022 * 11023 * Because a group leader will try and install the entire group 11024 * (through the sibling list, which is still in-tact), we can 11025 * end up with siblings installed in the wrong context. 11026 * 11027 * By installing siblings first we NO-OP because they're not 11028 * reachable through the group lists. 11029 */ 11030 for_each_sibling_event(sibling, group_leader) { 11031 perf_event__state_init(sibling); 11032 perf_install_in_context(ctx, sibling, sibling->cpu); 11033 get_ctx(ctx); 11034 } 11035 11036 /* 11037 * Removing from the context ends up with disabled 11038 * event. What we want here is event in the initial 11039 * startup state, ready to be add into new context. 11040 */ 11041 perf_event__state_init(group_leader); 11042 perf_install_in_context(ctx, group_leader, group_leader->cpu); 11043 get_ctx(ctx); 11044 } 11045 11046 /* 11047 * Precalculate sample_data sizes; do while holding ctx::mutex such 11048 * that we're serialized against further additions and before 11049 * perf_install_in_context() which is the point the event is active and 11050 * can use these values. 11051 */ 11052 perf_event__header_size(event); 11053 perf_event__id_header_size(event); 11054 11055 event->owner = current; 11056 11057 perf_install_in_context(ctx, event, event->cpu); 11058 perf_unpin_context(ctx); 11059 11060 if (move_group) 11061 perf_event_ctx_unlock(group_leader, gctx); 11062 mutex_unlock(&ctx->mutex); 11063 11064 if (task) { 11065 mutex_unlock(&task->signal->cred_guard_mutex); 11066 put_task_struct(task); 11067 } 11068 11069 mutex_lock(¤t->perf_event_mutex); 11070 list_add_tail(&event->owner_entry, ¤t->perf_event_list); 11071 mutex_unlock(¤t->perf_event_mutex); 11072 11073 /* 11074 * Drop the reference on the group_event after placing the 11075 * new event on the sibling_list. This ensures destruction 11076 * of the group leader will find the pointer to itself in 11077 * perf_group_detach(). 11078 */ 11079 fdput(group); 11080 fd_install(event_fd, event_file); 11081 return event_fd; 11082 11083 err_locked: 11084 if (move_group) 11085 perf_event_ctx_unlock(group_leader, gctx); 11086 mutex_unlock(&ctx->mutex); 11087 /* err_file: */ 11088 fput(event_file); 11089 err_context: 11090 perf_unpin_context(ctx); 11091 put_ctx(ctx); 11092 err_alloc: 11093 /* 11094 * If event_file is set, the fput() above will have called ->release() 11095 * and that will take care of freeing the event. 11096 */ 11097 if (!event_file) 11098 free_event(event); 11099 err_cred: 11100 if (task) 11101 mutex_unlock(&task->signal->cred_guard_mutex); 11102 err_task: 11103 if (task) 11104 put_task_struct(task); 11105 err_group_fd: 11106 fdput(group); 11107 err_fd: 11108 put_unused_fd(event_fd); 11109 return err; 11110 } 11111 11112 /** 11113 * perf_event_create_kernel_counter 11114 * 11115 * @attr: attributes of the counter to create 11116 * @cpu: cpu in which the counter is bound 11117 * @task: task to profile (NULL for percpu) 11118 */ 11119 struct perf_event * 11120 perf_event_create_kernel_counter(struct perf_event_attr *attr, int cpu, 11121 struct task_struct *task, 11122 perf_overflow_handler_t overflow_handler, 11123 void *context) 11124 { 11125 struct perf_event_context *ctx; 11126 struct perf_event *event; 11127 int err; 11128 11129 /* 11130 * Get the target context (task or percpu): 11131 */ 11132 11133 event = perf_event_alloc(attr, cpu, task, NULL, NULL, 11134 overflow_handler, context, -1); 11135 if (IS_ERR(event)) { 11136 err = PTR_ERR(event); 11137 goto err; 11138 } 11139 11140 /* Mark owner so we could distinguish it from user events. */ 11141 event->owner = TASK_TOMBSTONE; 11142 11143 ctx = find_get_context(event->pmu, task, event); 11144 if (IS_ERR(ctx)) { 11145 err = PTR_ERR(ctx); 11146 goto err_free; 11147 } 11148 11149 WARN_ON_ONCE(ctx->parent_ctx); 11150 mutex_lock(&ctx->mutex); 11151 if (ctx->task == TASK_TOMBSTONE) { 11152 err = -ESRCH; 11153 goto err_unlock; 11154 } 11155 11156 if (!task) { 11157 /* 11158 * Check if the @cpu we're creating an event for is online. 11159 * 11160 * We use the perf_cpu_context::ctx::mutex to serialize against 11161 * the hotplug notifiers. See perf_event_{init,exit}_cpu(). 11162 */ 11163 struct perf_cpu_context *cpuctx = 11164 container_of(ctx, struct perf_cpu_context, ctx); 11165 if (!cpuctx->online) { 11166 err = -ENODEV; 11167 goto err_unlock; 11168 } 11169 } 11170 11171 if (!exclusive_event_installable(event, ctx)) { 11172 err = -EBUSY; 11173 goto err_unlock; 11174 } 11175 11176 perf_install_in_context(ctx, event, cpu); 11177 perf_unpin_context(ctx); 11178 mutex_unlock(&ctx->mutex); 11179 11180 return event; 11181 11182 err_unlock: 11183 mutex_unlock(&ctx->mutex); 11184 perf_unpin_context(ctx); 11185 put_ctx(ctx); 11186 err_free: 11187 free_event(event); 11188 err: 11189 return ERR_PTR(err); 11190 } 11191 EXPORT_SYMBOL_GPL(perf_event_create_kernel_counter); 11192 11193 void perf_pmu_migrate_context(struct pmu *pmu, int src_cpu, int dst_cpu) 11194 { 11195 struct perf_event_context *src_ctx; 11196 struct perf_event_context *dst_ctx; 11197 struct perf_event *event, *tmp; 11198 LIST_HEAD(events); 11199 11200 src_ctx = &per_cpu_ptr(pmu->pmu_cpu_context, src_cpu)->ctx; 11201 dst_ctx = &per_cpu_ptr(pmu->pmu_cpu_context, dst_cpu)->ctx; 11202 11203 /* 11204 * See perf_event_ctx_lock() for comments on the details 11205 * of swizzling perf_event::ctx. 11206 */ 11207 mutex_lock_double(&src_ctx->mutex, &dst_ctx->mutex); 11208 list_for_each_entry_safe(event, tmp, &src_ctx->event_list, 11209 event_entry) { 11210 perf_remove_from_context(event, 0); 11211 unaccount_event_cpu(event, src_cpu); 11212 put_ctx(src_ctx); 11213 list_add(&event->migrate_entry, &events); 11214 } 11215 11216 /* 11217 * Wait for the events to quiesce before re-instating them. 11218 */ 11219 synchronize_rcu(); 11220 11221 /* 11222 * Re-instate events in 2 passes. 11223 * 11224 * Skip over group leaders and only install siblings on this first 11225 * pass, siblings will not get enabled without a leader, however a 11226 * leader will enable its siblings, even if those are still on the old 11227 * context. 11228 */ 11229 list_for_each_entry_safe(event, tmp, &events, migrate_entry) { 11230 if (event->group_leader == event) 11231 continue; 11232 11233 list_del(&event->migrate_entry); 11234 if (event->state >= PERF_EVENT_STATE_OFF) 11235 event->state = PERF_EVENT_STATE_INACTIVE; 11236 account_event_cpu(event, dst_cpu); 11237 perf_install_in_context(dst_ctx, event, dst_cpu); 11238 get_ctx(dst_ctx); 11239 } 11240 11241 /* 11242 * Once all the siblings are setup properly, install the group leaders 11243 * to make it go. 11244 */ 11245 list_for_each_entry_safe(event, tmp, &events, migrate_entry) { 11246 list_del(&event->migrate_entry); 11247 if (event->state >= PERF_EVENT_STATE_OFF) 11248 event->state = PERF_EVENT_STATE_INACTIVE; 11249 account_event_cpu(event, dst_cpu); 11250 perf_install_in_context(dst_ctx, event, dst_cpu); 11251 get_ctx(dst_ctx); 11252 } 11253 mutex_unlock(&dst_ctx->mutex); 11254 mutex_unlock(&src_ctx->mutex); 11255 } 11256 EXPORT_SYMBOL_GPL(perf_pmu_migrate_context); 11257 11258 static void sync_child_event(struct perf_event *child_event, 11259 struct task_struct *child) 11260 { 11261 struct perf_event *parent_event = child_event->parent; 11262 u64 child_val; 11263 11264 if (child_event->attr.inherit_stat) 11265 perf_event_read_event(child_event, child); 11266 11267 child_val = perf_event_count(child_event); 11268 11269 /* 11270 * Add back the child's count to the parent's count: 11271 */ 11272 atomic64_add(child_val, &parent_event->child_count); 11273 atomic64_add(child_event->total_time_enabled, 11274 &parent_event->child_total_time_enabled); 11275 atomic64_add(child_event->total_time_running, 11276 &parent_event->child_total_time_running); 11277 } 11278 11279 static void 11280 perf_event_exit_event(struct perf_event *child_event, 11281 struct perf_event_context *child_ctx, 11282 struct task_struct *child) 11283 { 11284 struct perf_event *parent_event = child_event->parent; 11285 11286 /* 11287 * Do not destroy the 'original' grouping; because of the context 11288 * switch optimization the original events could've ended up in a 11289 * random child task. 11290 * 11291 * If we were to destroy the original group, all group related 11292 * operations would cease to function properly after this random 11293 * child dies. 11294 * 11295 * Do destroy all inherited groups, we don't care about those 11296 * and being thorough is better. 11297 */ 11298 raw_spin_lock_irq(&child_ctx->lock); 11299 WARN_ON_ONCE(child_ctx->is_active); 11300 11301 if (parent_event) 11302 perf_group_detach(child_event); 11303 list_del_event(child_event, child_ctx); 11304 perf_event_set_state(child_event, PERF_EVENT_STATE_EXIT); /* is_event_hup() */ 11305 raw_spin_unlock_irq(&child_ctx->lock); 11306 11307 /* 11308 * Parent events are governed by their filedesc, retain them. 11309 */ 11310 if (!parent_event) { 11311 perf_event_wakeup(child_event); 11312 return; 11313 } 11314 /* 11315 * Child events can be cleaned up. 11316 */ 11317 11318 sync_child_event(child_event, child); 11319 11320 /* 11321 * Remove this event from the parent's list 11322 */ 11323 WARN_ON_ONCE(parent_event->ctx->parent_ctx); 11324 mutex_lock(&parent_event->child_mutex); 11325 list_del_init(&child_event->child_list); 11326 mutex_unlock(&parent_event->child_mutex); 11327 11328 /* 11329 * Kick perf_poll() for is_event_hup(). 11330 */ 11331 perf_event_wakeup(parent_event); 11332 free_event(child_event); 11333 put_event(parent_event); 11334 } 11335 11336 static void perf_event_exit_task_context(struct task_struct *child, int ctxn) 11337 { 11338 struct perf_event_context *child_ctx, *clone_ctx = NULL; 11339 struct perf_event *child_event, *next; 11340 11341 WARN_ON_ONCE(child != current); 11342 11343 child_ctx = perf_pin_task_context(child, ctxn); 11344 if (!child_ctx) 11345 return; 11346 11347 /* 11348 * In order to reduce the amount of tricky in ctx tear-down, we hold 11349 * ctx::mutex over the entire thing. This serializes against almost 11350 * everything that wants to access the ctx. 11351 * 11352 * The exception is sys_perf_event_open() / 11353 * perf_event_create_kernel_count() which does find_get_context() 11354 * without ctx::mutex (it cannot because of the move_group double mutex 11355 * lock thing). See the comments in perf_install_in_context(). 11356 */ 11357 mutex_lock(&child_ctx->mutex); 11358 11359 /* 11360 * In a single ctx::lock section, de-schedule the events and detach the 11361 * context from the task such that we cannot ever get it scheduled back 11362 * in. 11363 */ 11364 raw_spin_lock_irq(&child_ctx->lock); 11365 task_ctx_sched_out(__get_cpu_context(child_ctx), child_ctx, EVENT_ALL); 11366 11367 /* 11368 * Now that the context is inactive, destroy the task <-> ctx relation 11369 * and mark the context dead. 11370 */ 11371 RCU_INIT_POINTER(child->perf_event_ctxp[ctxn], NULL); 11372 put_ctx(child_ctx); /* cannot be last */ 11373 WRITE_ONCE(child_ctx->task, TASK_TOMBSTONE); 11374 put_task_struct(current); /* cannot be last */ 11375 11376 clone_ctx = unclone_ctx(child_ctx); 11377 raw_spin_unlock_irq(&child_ctx->lock); 11378 11379 if (clone_ctx) 11380 put_ctx(clone_ctx); 11381 11382 /* 11383 * Report the task dead after unscheduling the events so that we 11384 * won't get any samples after PERF_RECORD_EXIT. We can however still 11385 * get a few PERF_RECORD_READ events. 11386 */ 11387 perf_event_task(child, child_ctx, 0); 11388 11389 list_for_each_entry_safe(child_event, next, &child_ctx->event_list, event_entry) 11390 perf_event_exit_event(child_event, child_ctx, child); 11391 11392 mutex_unlock(&child_ctx->mutex); 11393 11394 put_ctx(child_ctx); 11395 } 11396 11397 /* 11398 * When a child task exits, feed back event values to parent events. 11399 * 11400 * Can be called with cred_guard_mutex held when called from 11401 * install_exec_creds(). 11402 */ 11403 void perf_event_exit_task(struct task_struct *child) 11404 { 11405 struct perf_event *event, *tmp; 11406 int ctxn; 11407 11408 mutex_lock(&child->perf_event_mutex); 11409 list_for_each_entry_safe(event, tmp, &child->perf_event_list, 11410 owner_entry) { 11411 list_del_init(&event->owner_entry); 11412 11413 /* 11414 * Ensure the list deletion is visible before we clear 11415 * the owner, closes a race against perf_release() where 11416 * we need to serialize on the owner->perf_event_mutex. 11417 */ 11418 smp_store_release(&event->owner, NULL); 11419 } 11420 mutex_unlock(&child->perf_event_mutex); 11421 11422 for_each_task_context_nr(ctxn) 11423 perf_event_exit_task_context(child, ctxn); 11424 11425 /* 11426 * The perf_event_exit_task_context calls perf_event_task 11427 * with child's task_ctx, which generates EXIT events for 11428 * child contexts and sets child->perf_event_ctxp[] to NULL. 11429 * At this point we need to send EXIT events to cpu contexts. 11430 */ 11431 perf_event_task(child, NULL, 0); 11432 } 11433 11434 static void perf_free_event(struct perf_event *event, 11435 struct perf_event_context *ctx) 11436 { 11437 struct perf_event *parent = event->parent; 11438 11439 if (WARN_ON_ONCE(!parent)) 11440 return; 11441 11442 mutex_lock(&parent->child_mutex); 11443 list_del_init(&event->child_list); 11444 mutex_unlock(&parent->child_mutex); 11445 11446 put_event(parent); 11447 11448 raw_spin_lock_irq(&ctx->lock); 11449 perf_group_detach(event); 11450 list_del_event(event, ctx); 11451 raw_spin_unlock_irq(&ctx->lock); 11452 free_event(event); 11453 } 11454 11455 /* 11456 * Free an unexposed, unused context as created by inheritance by 11457 * perf_event_init_task below, used by fork() in case of fail. 11458 * 11459 * Not all locks are strictly required, but take them anyway to be nice and 11460 * help out with the lockdep assertions. 11461 */ 11462 void perf_event_free_task(struct task_struct *task) 11463 { 11464 struct perf_event_context *ctx; 11465 struct perf_event *event, *tmp; 11466 int ctxn; 11467 11468 for_each_task_context_nr(ctxn) { 11469 ctx = task->perf_event_ctxp[ctxn]; 11470 if (!ctx) 11471 continue; 11472 11473 mutex_lock(&ctx->mutex); 11474 raw_spin_lock_irq(&ctx->lock); 11475 /* 11476 * Destroy the task <-> ctx relation and mark the context dead. 11477 * 11478 * This is important because even though the task hasn't been 11479 * exposed yet the context has been (through child_list). 11480 */ 11481 RCU_INIT_POINTER(task->perf_event_ctxp[ctxn], NULL); 11482 WRITE_ONCE(ctx->task, TASK_TOMBSTONE); 11483 put_task_struct(task); /* cannot be last */ 11484 raw_spin_unlock_irq(&ctx->lock); 11485 11486 list_for_each_entry_safe(event, tmp, &ctx->event_list, event_entry) 11487 perf_free_event(event, ctx); 11488 11489 mutex_unlock(&ctx->mutex); 11490 put_ctx(ctx); 11491 } 11492 } 11493 11494 void perf_event_delayed_put(struct task_struct *task) 11495 { 11496 int ctxn; 11497 11498 for_each_task_context_nr(ctxn) 11499 WARN_ON_ONCE(task->perf_event_ctxp[ctxn]); 11500 } 11501 11502 struct file *perf_event_get(unsigned int fd) 11503 { 11504 struct file *file; 11505 11506 file = fget_raw(fd); 11507 if (!file) 11508 return ERR_PTR(-EBADF); 11509 11510 if (file->f_op != &perf_fops) { 11511 fput(file); 11512 return ERR_PTR(-EBADF); 11513 } 11514 11515 return file; 11516 } 11517 11518 const struct perf_event *perf_get_event(struct file *file) 11519 { 11520 if (file->f_op != &perf_fops) 11521 return ERR_PTR(-EINVAL); 11522 11523 return file->private_data; 11524 } 11525 11526 const struct perf_event_attr *perf_event_attrs(struct perf_event *event) 11527 { 11528 if (!event) 11529 return ERR_PTR(-EINVAL); 11530 11531 return &event->attr; 11532 } 11533 11534 /* 11535 * Inherit an event from parent task to child task. 11536 * 11537 * Returns: 11538 * - valid pointer on success 11539 * - NULL for orphaned events 11540 * - IS_ERR() on error 11541 */ 11542 static struct perf_event * 11543 inherit_event(struct perf_event *parent_event, 11544 struct task_struct *parent, 11545 struct perf_event_context *parent_ctx, 11546 struct task_struct *child, 11547 struct perf_event *group_leader, 11548 struct perf_event_context *child_ctx) 11549 { 11550 enum perf_event_state parent_state = parent_event->state; 11551 struct perf_event *child_event; 11552 unsigned long flags; 11553 11554 /* 11555 * Instead of creating recursive hierarchies of events, 11556 * we link inherited events back to the original parent, 11557 * which has a filp for sure, which we use as the reference 11558 * count: 11559 */ 11560 if (parent_event->parent) 11561 parent_event = parent_event->parent; 11562 11563 child_event = perf_event_alloc(&parent_event->attr, 11564 parent_event->cpu, 11565 child, 11566 group_leader, parent_event, 11567 NULL, NULL, -1); 11568 if (IS_ERR(child_event)) 11569 return child_event; 11570 11571 11572 if ((child_event->attach_state & PERF_ATTACH_TASK_DATA) && 11573 !child_ctx->task_ctx_data) { 11574 struct pmu *pmu = child_event->pmu; 11575 11576 child_ctx->task_ctx_data = kzalloc(pmu->task_ctx_size, 11577 GFP_KERNEL); 11578 if (!child_ctx->task_ctx_data) { 11579 free_event(child_event); 11580 return NULL; 11581 } 11582 } 11583 11584 /* 11585 * is_orphaned_event() and list_add_tail(&parent_event->child_list) 11586 * must be under the same lock in order to serialize against 11587 * perf_event_release_kernel(), such that either we must observe 11588 * is_orphaned_event() or they will observe us on the child_list. 11589 */ 11590 mutex_lock(&parent_event->child_mutex); 11591 if (is_orphaned_event(parent_event) || 11592 !atomic_long_inc_not_zero(&parent_event->refcount)) { 11593 mutex_unlock(&parent_event->child_mutex); 11594 /* task_ctx_data is freed with child_ctx */ 11595 free_event(child_event); 11596 return NULL; 11597 } 11598 11599 get_ctx(child_ctx); 11600 11601 /* 11602 * Make the child state follow the state of the parent event, 11603 * not its attr.disabled bit. We hold the parent's mutex, 11604 * so we won't race with perf_event_{en, dis}able_family. 11605 */ 11606 if (parent_state >= PERF_EVENT_STATE_INACTIVE) 11607 child_event->state = PERF_EVENT_STATE_INACTIVE; 11608 else 11609 child_event->state = PERF_EVENT_STATE_OFF; 11610 11611 if (parent_event->attr.freq) { 11612 u64 sample_period = parent_event->hw.sample_period; 11613 struct hw_perf_event *hwc = &child_event->hw; 11614 11615 hwc->sample_period = sample_period; 11616 hwc->last_period = sample_period; 11617 11618 local64_set(&hwc->period_left, sample_period); 11619 } 11620 11621 child_event->ctx = child_ctx; 11622 child_event->overflow_handler = parent_event->overflow_handler; 11623 child_event->overflow_handler_context 11624 = parent_event->overflow_handler_context; 11625 11626 /* 11627 * Precalculate sample_data sizes 11628 */ 11629 perf_event__header_size(child_event); 11630 perf_event__id_header_size(child_event); 11631 11632 /* 11633 * Link it up in the child's context: 11634 */ 11635 raw_spin_lock_irqsave(&child_ctx->lock, flags); 11636 add_event_to_ctx(child_event, child_ctx); 11637 raw_spin_unlock_irqrestore(&child_ctx->lock, flags); 11638 11639 /* 11640 * Link this into the parent event's child list 11641 */ 11642 list_add_tail(&child_event->child_list, &parent_event->child_list); 11643 mutex_unlock(&parent_event->child_mutex); 11644 11645 return child_event; 11646 } 11647 11648 /* 11649 * Inherits an event group. 11650 * 11651 * This will quietly suppress orphaned events; !inherit_event() is not an error. 11652 * This matches with perf_event_release_kernel() removing all child events. 11653 * 11654 * Returns: 11655 * - 0 on success 11656 * - <0 on error 11657 */ 11658 static int inherit_group(struct perf_event *parent_event, 11659 struct task_struct *parent, 11660 struct perf_event_context *parent_ctx, 11661 struct task_struct *child, 11662 struct perf_event_context *child_ctx) 11663 { 11664 struct perf_event *leader; 11665 struct perf_event *sub; 11666 struct perf_event *child_ctr; 11667 11668 leader = inherit_event(parent_event, parent, parent_ctx, 11669 child, NULL, child_ctx); 11670 if (IS_ERR(leader)) 11671 return PTR_ERR(leader); 11672 /* 11673 * @leader can be NULL here because of is_orphaned_event(). In this 11674 * case inherit_event() will create individual events, similar to what 11675 * perf_group_detach() would do anyway. 11676 */ 11677 for_each_sibling_event(sub, parent_event) { 11678 child_ctr = inherit_event(sub, parent, parent_ctx, 11679 child, leader, child_ctx); 11680 if (IS_ERR(child_ctr)) 11681 return PTR_ERR(child_ctr); 11682 } 11683 return 0; 11684 } 11685 11686 /* 11687 * Creates the child task context and tries to inherit the event-group. 11688 * 11689 * Clears @inherited_all on !attr.inherited or error. Note that we'll leave 11690 * inherited_all set when we 'fail' to inherit an orphaned event; this is 11691 * consistent with perf_event_release_kernel() removing all child events. 11692 * 11693 * Returns: 11694 * - 0 on success 11695 * - <0 on error 11696 */ 11697 static int 11698 inherit_task_group(struct perf_event *event, struct task_struct *parent, 11699 struct perf_event_context *parent_ctx, 11700 struct task_struct *child, int ctxn, 11701 int *inherited_all) 11702 { 11703 int ret; 11704 struct perf_event_context *child_ctx; 11705 11706 if (!event->attr.inherit) { 11707 *inherited_all = 0; 11708 return 0; 11709 } 11710 11711 child_ctx = child->perf_event_ctxp[ctxn]; 11712 if (!child_ctx) { 11713 /* 11714 * This is executed from the parent task context, so 11715 * inherit events that have been marked for cloning. 11716 * First allocate and initialize a context for the 11717 * child. 11718 */ 11719 child_ctx = alloc_perf_context(parent_ctx->pmu, child); 11720 if (!child_ctx) 11721 return -ENOMEM; 11722 11723 child->perf_event_ctxp[ctxn] = child_ctx; 11724 } 11725 11726 ret = inherit_group(event, parent, parent_ctx, 11727 child, child_ctx); 11728 11729 if (ret) 11730 *inherited_all = 0; 11731 11732 return ret; 11733 } 11734 11735 /* 11736 * Initialize the perf_event context in task_struct 11737 */ 11738 static int perf_event_init_context(struct task_struct *child, int ctxn) 11739 { 11740 struct perf_event_context *child_ctx, *parent_ctx; 11741 struct perf_event_context *cloned_ctx; 11742 struct perf_event *event; 11743 struct task_struct *parent = current; 11744 int inherited_all = 1; 11745 unsigned long flags; 11746 int ret = 0; 11747 11748 if (likely(!parent->perf_event_ctxp[ctxn])) 11749 return 0; 11750 11751 /* 11752 * If the parent's context is a clone, pin it so it won't get 11753 * swapped under us. 11754 */ 11755 parent_ctx = perf_pin_task_context(parent, ctxn); 11756 if (!parent_ctx) 11757 return 0; 11758 11759 /* 11760 * No need to check if parent_ctx != NULL here; since we saw 11761 * it non-NULL earlier, the only reason for it to become NULL 11762 * is if we exit, and since we're currently in the middle of 11763 * a fork we can't be exiting at the same time. 11764 */ 11765 11766 /* 11767 * Lock the parent list. No need to lock the child - not PID 11768 * hashed yet and not running, so nobody can access it. 11769 */ 11770 mutex_lock(&parent_ctx->mutex); 11771 11772 /* 11773 * We dont have to disable NMIs - we are only looking at 11774 * the list, not manipulating it: 11775 */ 11776 perf_event_groups_for_each(event, &parent_ctx->pinned_groups) { 11777 ret = inherit_task_group(event, parent, parent_ctx, 11778 child, ctxn, &inherited_all); 11779 if (ret) 11780 goto out_unlock; 11781 } 11782 11783 /* 11784 * We can't hold ctx->lock when iterating the ->flexible_group list due 11785 * to allocations, but we need to prevent rotation because 11786 * rotate_ctx() will change the list from interrupt context. 11787 */ 11788 raw_spin_lock_irqsave(&parent_ctx->lock, flags); 11789 parent_ctx->rotate_disable = 1; 11790 raw_spin_unlock_irqrestore(&parent_ctx->lock, flags); 11791 11792 perf_event_groups_for_each(event, &parent_ctx->flexible_groups) { 11793 ret = inherit_task_group(event, parent, parent_ctx, 11794 child, ctxn, &inherited_all); 11795 if (ret) 11796 goto out_unlock; 11797 } 11798 11799 raw_spin_lock_irqsave(&parent_ctx->lock, flags); 11800 parent_ctx->rotate_disable = 0; 11801 11802 child_ctx = child->perf_event_ctxp[ctxn]; 11803 11804 if (child_ctx && inherited_all) { 11805 /* 11806 * Mark the child context as a clone of the parent 11807 * context, or of whatever the parent is a clone of. 11808 * 11809 * Note that if the parent is a clone, the holding of 11810 * parent_ctx->lock avoids it from being uncloned. 11811 */ 11812 cloned_ctx = parent_ctx->parent_ctx; 11813 if (cloned_ctx) { 11814 child_ctx->parent_ctx = cloned_ctx; 11815 child_ctx->parent_gen = parent_ctx->parent_gen; 11816 } else { 11817 child_ctx->parent_ctx = parent_ctx; 11818 child_ctx->parent_gen = parent_ctx->generation; 11819 } 11820 get_ctx(child_ctx->parent_ctx); 11821 } 11822 11823 raw_spin_unlock_irqrestore(&parent_ctx->lock, flags); 11824 out_unlock: 11825 mutex_unlock(&parent_ctx->mutex); 11826 11827 perf_unpin_context(parent_ctx); 11828 put_ctx(parent_ctx); 11829 11830 return ret; 11831 } 11832 11833 /* 11834 * Initialize the perf_event context in task_struct 11835 */ 11836 int perf_event_init_task(struct task_struct *child) 11837 { 11838 int ctxn, ret; 11839 11840 memset(child->perf_event_ctxp, 0, sizeof(child->perf_event_ctxp)); 11841 mutex_init(&child->perf_event_mutex); 11842 INIT_LIST_HEAD(&child->perf_event_list); 11843 11844 for_each_task_context_nr(ctxn) { 11845 ret = perf_event_init_context(child, ctxn); 11846 if (ret) { 11847 perf_event_free_task(child); 11848 return ret; 11849 } 11850 } 11851 11852 return 0; 11853 } 11854 11855 static void __init perf_event_init_all_cpus(void) 11856 { 11857 struct swevent_htable *swhash; 11858 int cpu; 11859 11860 zalloc_cpumask_var(&perf_online_mask, GFP_KERNEL); 11861 11862 for_each_possible_cpu(cpu) { 11863 swhash = &per_cpu(swevent_htable, cpu); 11864 mutex_init(&swhash->hlist_mutex); 11865 INIT_LIST_HEAD(&per_cpu(active_ctx_list, cpu)); 11866 11867 INIT_LIST_HEAD(&per_cpu(pmu_sb_events.list, cpu)); 11868 raw_spin_lock_init(&per_cpu(pmu_sb_events.lock, cpu)); 11869 11870 #ifdef CONFIG_CGROUP_PERF 11871 INIT_LIST_HEAD(&per_cpu(cgrp_cpuctx_list, cpu)); 11872 #endif 11873 INIT_LIST_HEAD(&per_cpu(sched_cb_list, cpu)); 11874 } 11875 } 11876 11877 void perf_swevent_init_cpu(unsigned int cpu) 11878 { 11879 struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu); 11880 11881 mutex_lock(&swhash->hlist_mutex); 11882 if (swhash->hlist_refcount > 0 && !swevent_hlist_deref(swhash)) { 11883 struct swevent_hlist *hlist; 11884 11885 hlist = kzalloc_node(sizeof(*hlist), GFP_KERNEL, cpu_to_node(cpu)); 11886 WARN_ON(!hlist); 11887 rcu_assign_pointer(swhash->swevent_hlist, hlist); 11888 } 11889 mutex_unlock(&swhash->hlist_mutex); 11890 } 11891 11892 #if defined CONFIG_HOTPLUG_CPU || defined CONFIG_KEXEC_CORE 11893 static void __perf_event_exit_context(void *__info) 11894 { 11895 struct perf_event_context *ctx = __info; 11896 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx); 11897 struct perf_event *event; 11898 11899 raw_spin_lock(&ctx->lock); 11900 ctx_sched_out(ctx, cpuctx, EVENT_TIME); 11901 list_for_each_entry(event, &ctx->event_list, event_entry) 11902 __perf_remove_from_context(event, cpuctx, ctx, (void *)DETACH_GROUP); 11903 raw_spin_unlock(&ctx->lock); 11904 } 11905 11906 static void perf_event_exit_cpu_context(int cpu) 11907 { 11908 struct perf_cpu_context *cpuctx; 11909 struct perf_event_context *ctx; 11910 struct pmu *pmu; 11911 11912 mutex_lock(&pmus_lock); 11913 list_for_each_entry(pmu, &pmus, entry) { 11914 cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu); 11915 ctx = &cpuctx->ctx; 11916 11917 mutex_lock(&ctx->mutex); 11918 smp_call_function_single(cpu, __perf_event_exit_context, ctx, 1); 11919 cpuctx->online = 0; 11920 mutex_unlock(&ctx->mutex); 11921 } 11922 cpumask_clear_cpu(cpu, perf_online_mask); 11923 mutex_unlock(&pmus_lock); 11924 } 11925 #else 11926 11927 static void perf_event_exit_cpu_context(int cpu) { } 11928 11929 #endif 11930 11931 int perf_event_init_cpu(unsigned int cpu) 11932 { 11933 struct perf_cpu_context *cpuctx; 11934 struct perf_event_context *ctx; 11935 struct pmu *pmu; 11936 11937 perf_swevent_init_cpu(cpu); 11938 11939 mutex_lock(&pmus_lock); 11940 cpumask_set_cpu(cpu, perf_online_mask); 11941 list_for_each_entry(pmu, &pmus, entry) { 11942 cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu); 11943 ctx = &cpuctx->ctx; 11944 11945 mutex_lock(&ctx->mutex); 11946 cpuctx->online = 1; 11947 mutex_unlock(&ctx->mutex); 11948 } 11949 mutex_unlock(&pmus_lock); 11950 11951 return 0; 11952 } 11953 11954 int perf_event_exit_cpu(unsigned int cpu) 11955 { 11956 perf_event_exit_cpu_context(cpu); 11957 return 0; 11958 } 11959 11960 static int 11961 perf_reboot(struct notifier_block *notifier, unsigned long val, void *v) 11962 { 11963 int cpu; 11964 11965 for_each_online_cpu(cpu) 11966 perf_event_exit_cpu(cpu); 11967 11968 return NOTIFY_OK; 11969 } 11970 11971 /* 11972 * Run the perf reboot notifier at the very last possible moment so that 11973 * the generic watchdog code runs as long as possible. 11974 */ 11975 static struct notifier_block perf_reboot_notifier = { 11976 .notifier_call = perf_reboot, 11977 .priority = INT_MIN, 11978 }; 11979 11980 void __init perf_event_init(void) 11981 { 11982 int ret; 11983 11984 idr_init(&pmu_idr); 11985 11986 perf_event_init_all_cpus(); 11987 init_srcu_struct(&pmus_srcu); 11988 perf_pmu_register(&perf_swevent, "software", PERF_TYPE_SOFTWARE); 11989 perf_pmu_register(&perf_cpu_clock, NULL, -1); 11990 perf_pmu_register(&perf_task_clock, NULL, -1); 11991 perf_tp_register(); 11992 perf_event_init_cpu(smp_processor_id()); 11993 register_reboot_notifier(&perf_reboot_notifier); 11994 11995 ret = init_hw_breakpoint(); 11996 WARN(ret, "hw_breakpoint initialization failed with: %d", ret); 11997 11998 /* 11999 * Build time assertion that we keep the data_head at the intended 12000 * location. IOW, validation we got the __reserved[] size right. 12001 */ 12002 BUILD_BUG_ON((offsetof(struct perf_event_mmap_page, data_head)) 12003 != 1024); 12004 } 12005 12006 ssize_t perf_event_sysfs_show(struct device *dev, struct device_attribute *attr, 12007 char *page) 12008 { 12009 struct perf_pmu_events_attr *pmu_attr = 12010 container_of(attr, struct perf_pmu_events_attr, attr); 12011 12012 if (pmu_attr->event_str) 12013 return sprintf(page, "%s\n", pmu_attr->event_str); 12014 12015 return 0; 12016 } 12017 EXPORT_SYMBOL_GPL(perf_event_sysfs_show); 12018 12019 static int __init perf_event_sysfs_init(void) 12020 { 12021 struct pmu *pmu; 12022 int ret; 12023 12024 mutex_lock(&pmus_lock); 12025 12026 ret = bus_register(&pmu_bus); 12027 if (ret) 12028 goto unlock; 12029 12030 list_for_each_entry(pmu, &pmus, entry) { 12031 if (!pmu->name || pmu->type < 0) 12032 continue; 12033 12034 ret = pmu_dev_alloc(pmu); 12035 WARN(ret, "Failed to register pmu: %s, reason %d\n", pmu->name, ret); 12036 } 12037 pmu_bus_running = 1; 12038 ret = 0; 12039 12040 unlock: 12041 mutex_unlock(&pmus_lock); 12042 12043 return ret; 12044 } 12045 device_initcall(perf_event_sysfs_init); 12046 12047 #ifdef CONFIG_CGROUP_PERF 12048 static struct cgroup_subsys_state * 12049 perf_cgroup_css_alloc(struct cgroup_subsys_state *parent_css) 12050 { 12051 struct perf_cgroup *jc; 12052 12053 jc = kzalloc(sizeof(*jc), GFP_KERNEL); 12054 if (!jc) 12055 return ERR_PTR(-ENOMEM); 12056 12057 jc->info = alloc_percpu(struct perf_cgroup_info); 12058 if (!jc->info) { 12059 kfree(jc); 12060 return ERR_PTR(-ENOMEM); 12061 } 12062 12063 return &jc->css; 12064 } 12065 12066 static void perf_cgroup_css_free(struct cgroup_subsys_state *css) 12067 { 12068 struct perf_cgroup *jc = container_of(css, struct perf_cgroup, css); 12069 12070 free_percpu(jc->info); 12071 kfree(jc); 12072 } 12073 12074 static int __perf_cgroup_move(void *info) 12075 { 12076 struct task_struct *task = info; 12077 rcu_read_lock(); 12078 perf_cgroup_switch(task, PERF_CGROUP_SWOUT | PERF_CGROUP_SWIN); 12079 rcu_read_unlock(); 12080 return 0; 12081 } 12082 12083 static void perf_cgroup_attach(struct cgroup_taskset *tset) 12084 { 12085 struct task_struct *task; 12086 struct cgroup_subsys_state *css; 12087 12088 cgroup_taskset_for_each(task, css, tset) 12089 task_function_call(task, __perf_cgroup_move, task); 12090 } 12091 12092 struct cgroup_subsys perf_event_cgrp_subsys = { 12093 .css_alloc = perf_cgroup_css_alloc, 12094 .css_free = perf_cgroup_css_free, 12095 .attach = perf_cgroup_attach, 12096 /* 12097 * Implicitly enable on dfl hierarchy so that perf events can 12098 * always be filtered by cgroup2 path as long as perf_event 12099 * controller is not mounted on a legacy hierarchy. 12100 */ 12101 .implicit_on_dfl = true, 12102 .threaded = true, 12103 }; 12104 #endif /* CONFIG_CGROUP_PERF */ 12105