1 /* 2 * Performance events core code: 3 * 4 * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de> 5 * Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar 6 * Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra 7 * Copyright © 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com> 8 * 9 * For licensing details see kernel-base/COPYING 10 */ 11 12 #include <linux/fs.h> 13 #include <linux/mm.h> 14 #include <linux/cpu.h> 15 #include <linux/smp.h> 16 #include <linux/idr.h> 17 #include <linux/file.h> 18 #include <linux/poll.h> 19 #include <linux/slab.h> 20 #include <linux/hash.h> 21 #include <linux/tick.h> 22 #include <linux/sysfs.h> 23 #include <linux/dcache.h> 24 #include <linux/percpu.h> 25 #include <linux/ptrace.h> 26 #include <linux/reboot.h> 27 #include <linux/vmstat.h> 28 #include <linux/device.h> 29 #include <linux/export.h> 30 #include <linux/vmalloc.h> 31 #include <linux/hardirq.h> 32 #include <linux/rculist.h> 33 #include <linux/uaccess.h> 34 #include <linux/syscalls.h> 35 #include <linux/anon_inodes.h> 36 #include <linux/kernel_stat.h> 37 #include <linux/cgroup.h> 38 #include <linux/perf_event.h> 39 #include <linux/trace_events.h> 40 #include <linux/hw_breakpoint.h> 41 #include <linux/mm_types.h> 42 #include <linux/module.h> 43 #include <linux/mman.h> 44 #include <linux/compat.h> 45 #include <linux/bpf.h> 46 #include <linux/filter.h> 47 #include <linux/namei.h> 48 #include <linux/parser.h> 49 50 #include "internal.h" 51 52 #include <asm/irq_regs.h> 53 54 typedef int (*remote_function_f)(void *); 55 56 struct remote_function_call { 57 struct task_struct *p; 58 remote_function_f func; 59 void *info; 60 int ret; 61 }; 62 63 static void remote_function(void *data) 64 { 65 struct remote_function_call *tfc = data; 66 struct task_struct *p = tfc->p; 67 68 if (p) { 69 /* -EAGAIN */ 70 if (task_cpu(p) != smp_processor_id()) 71 return; 72 73 /* 74 * Now that we're on right CPU with IRQs disabled, we can test 75 * if we hit the right task without races. 76 */ 77 78 tfc->ret = -ESRCH; /* No such (running) process */ 79 if (p != current) 80 return; 81 } 82 83 tfc->ret = tfc->func(tfc->info); 84 } 85 86 /** 87 * task_function_call - call a function on the cpu on which a task runs 88 * @p: the task to evaluate 89 * @func: the function to be called 90 * @info: the function call argument 91 * 92 * Calls the function @func when the task is currently running. This might 93 * be on the current CPU, which just calls the function directly 94 * 95 * returns: @func return value, or 96 * -ESRCH - when the process isn't running 97 * -EAGAIN - when the process moved away 98 */ 99 static int 100 task_function_call(struct task_struct *p, remote_function_f func, void *info) 101 { 102 struct remote_function_call data = { 103 .p = p, 104 .func = func, 105 .info = info, 106 .ret = -EAGAIN, 107 }; 108 int ret; 109 110 do { 111 ret = smp_call_function_single(task_cpu(p), remote_function, &data, 1); 112 if (!ret) 113 ret = data.ret; 114 } while (ret == -EAGAIN); 115 116 return ret; 117 } 118 119 /** 120 * cpu_function_call - call a function on the cpu 121 * @func: the function to be called 122 * @info: the function call argument 123 * 124 * Calls the function @func on the remote cpu. 125 * 126 * returns: @func return value or -ENXIO when the cpu is offline 127 */ 128 static int cpu_function_call(int cpu, remote_function_f func, void *info) 129 { 130 struct remote_function_call data = { 131 .p = NULL, 132 .func = func, 133 .info = info, 134 .ret = -ENXIO, /* No such CPU */ 135 }; 136 137 smp_call_function_single(cpu, remote_function, &data, 1); 138 139 return data.ret; 140 } 141 142 static inline struct perf_cpu_context * 143 __get_cpu_context(struct perf_event_context *ctx) 144 { 145 return this_cpu_ptr(ctx->pmu->pmu_cpu_context); 146 } 147 148 static void perf_ctx_lock(struct perf_cpu_context *cpuctx, 149 struct perf_event_context *ctx) 150 { 151 raw_spin_lock(&cpuctx->ctx.lock); 152 if (ctx) 153 raw_spin_lock(&ctx->lock); 154 } 155 156 static void perf_ctx_unlock(struct perf_cpu_context *cpuctx, 157 struct perf_event_context *ctx) 158 { 159 if (ctx) 160 raw_spin_unlock(&ctx->lock); 161 raw_spin_unlock(&cpuctx->ctx.lock); 162 } 163 164 #define TASK_TOMBSTONE ((void *)-1L) 165 166 static bool is_kernel_event(struct perf_event *event) 167 { 168 return READ_ONCE(event->owner) == TASK_TOMBSTONE; 169 } 170 171 /* 172 * On task ctx scheduling... 173 * 174 * When !ctx->nr_events a task context will not be scheduled. This means 175 * we can disable the scheduler hooks (for performance) without leaving 176 * pending task ctx state. 177 * 178 * This however results in two special cases: 179 * 180 * - removing the last event from a task ctx; this is relatively straight 181 * forward and is done in __perf_remove_from_context. 182 * 183 * - adding the first event to a task ctx; this is tricky because we cannot 184 * rely on ctx->is_active and therefore cannot use event_function_call(). 185 * See perf_install_in_context(). 186 * 187 * If ctx->nr_events, then ctx->is_active and cpuctx->task_ctx are set. 188 */ 189 190 typedef void (*event_f)(struct perf_event *, struct perf_cpu_context *, 191 struct perf_event_context *, void *); 192 193 struct event_function_struct { 194 struct perf_event *event; 195 event_f func; 196 void *data; 197 }; 198 199 static int event_function(void *info) 200 { 201 struct event_function_struct *efs = info; 202 struct perf_event *event = efs->event; 203 struct perf_event_context *ctx = event->ctx; 204 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx); 205 struct perf_event_context *task_ctx = cpuctx->task_ctx; 206 int ret = 0; 207 208 WARN_ON_ONCE(!irqs_disabled()); 209 210 perf_ctx_lock(cpuctx, task_ctx); 211 /* 212 * Since we do the IPI call without holding ctx->lock things can have 213 * changed, double check we hit the task we set out to hit. 214 */ 215 if (ctx->task) { 216 if (ctx->task != current) { 217 ret = -ESRCH; 218 goto unlock; 219 } 220 221 /* 222 * We only use event_function_call() on established contexts, 223 * and event_function() is only ever called when active (or 224 * rather, we'll have bailed in task_function_call() or the 225 * above ctx->task != current test), therefore we must have 226 * ctx->is_active here. 227 */ 228 WARN_ON_ONCE(!ctx->is_active); 229 /* 230 * And since we have ctx->is_active, cpuctx->task_ctx must 231 * match. 232 */ 233 WARN_ON_ONCE(task_ctx != ctx); 234 } else { 235 WARN_ON_ONCE(&cpuctx->ctx != ctx); 236 } 237 238 efs->func(event, cpuctx, ctx, efs->data); 239 unlock: 240 perf_ctx_unlock(cpuctx, task_ctx); 241 242 return ret; 243 } 244 245 static void event_function_call(struct perf_event *event, event_f func, void *data) 246 { 247 struct perf_event_context *ctx = event->ctx; 248 struct task_struct *task = READ_ONCE(ctx->task); /* verified in event_function */ 249 struct event_function_struct efs = { 250 .event = event, 251 .func = func, 252 .data = data, 253 }; 254 255 if (!event->parent) { 256 /* 257 * If this is a !child event, we must hold ctx::mutex to 258 * stabilize the the event->ctx relation. See 259 * perf_event_ctx_lock(). 260 */ 261 lockdep_assert_held(&ctx->mutex); 262 } 263 264 if (!task) { 265 cpu_function_call(event->cpu, event_function, &efs); 266 return; 267 } 268 269 if (task == TASK_TOMBSTONE) 270 return; 271 272 again: 273 if (!task_function_call(task, event_function, &efs)) 274 return; 275 276 raw_spin_lock_irq(&ctx->lock); 277 /* 278 * Reload the task pointer, it might have been changed by 279 * a concurrent perf_event_context_sched_out(). 280 */ 281 task = ctx->task; 282 if (task == TASK_TOMBSTONE) { 283 raw_spin_unlock_irq(&ctx->lock); 284 return; 285 } 286 if (ctx->is_active) { 287 raw_spin_unlock_irq(&ctx->lock); 288 goto again; 289 } 290 func(event, NULL, ctx, data); 291 raw_spin_unlock_irq(&ctx->lock); 292 } 293 294 /* 295 * Similar to event_function_call() + event_function(), but hard assumes IRQs 296 * are already disabled and we're on the right CPU. 297 */ 298 static void event_function_local(struct perf_event *event, event_f func, void *data) 299 { 300 struct perf_event_context *ctx = event->ctx; 301 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx); 302 struct task_struct *task = READ_ONCE(ctx->task); 303 struct perf_event_context *task_ctx = NULL; 304 305 WARN_ON_ONCE(!irqs_disabled()); 306 307 if (task) { 308 if (task == TASK_TOMBSTONE) 309 return; 310 311 task_ctx = ctx; 312 } 313 314 perf_ctx_lock(cpuctx, task_ctx); 315 316 task = ctx->task; 317 if (task == TASK_TOMBSTONE) 318 goto unlock; 319 320 if (task) { 321 /* 322 * We must be either inactive or active and the right task, 323 * otherwise we're screwed, since we cannot IPI to somewhere 324 * else. 325 */ 326 if (ctx->is_active) { 327 if (WARN_ON_ONCE(task != current)) 328 goto unlock; 329 330 if (WARN_ON_ONCE(cpuctx->task_ctx != ctx)) 331 goto unlock; 332 } 333 } else { 334 WARN_ON_ONCE(&cpuctx->ctx != ctx); 335 } 336 337 func(event, cpuctx, ctx, data); 338 unlock: 339 perf_ctx_unlock(cpuctx, task_ctx); 340 } 341 342 #define PERF_FLAG_ALL (PERF_FLAG_FD_NO_GROUP |\ 343 PERF_FLAG_FD_OUTPUT |\ 344 PERF_FLAG_PID_CGROUP |\ 345 PERF_FLAG_FD_CLOEXEC) 346 347 /* 348 * branch priv levels that need permission checks 349 */ 350 #define PERF_SAMPLE_BRANCH_PERM_PLM \ 351 (PERF_SAMPLE_BRANCH_KERNEL |\ 352 PERF_SAMPLE_BRANCH_HV) 353 354 enum event_type_t { 355 EVENT_FLEXIBLE = 0x1, 356 EVENT_PINNED = 0x2, 357 EVENT_TIME = 0x4, 358 EVENT_ALL = EVENT_FLEXIBLE | EVENT_PINNED, 359 }; 360 361 /* 362 * perf_sched_events : >0 events exist 363 * perf_cgroup_events: >0 per-cpu cgroup events exist on this cpu 364 */ 365 366 static void perf_sched_delayed(struct work_struct *work); 367 DEFINE_STATIC_KEY_FALSE(perf_sched_events); 368 static DECLARE_DELAYED_WORK(perf_sched_work, perf_sched_delayed); 369 static DEFINE_MUTEX(perf_sched_mutex); 370 static atomic_t perf_sched_count; 371 372 static DEFINE_PER_CPU(atomic_t, perf_cgroup_events); 373 static DEFINE_PER_CPU(int, perf_sched_cb_usages); 374 static DEFINE_PER_CPU(struct pmu_event_list, pmu_sb_events); 375 376 static atomic_t nr_mmap_events __read_mostly; 377 static atomic_t nr_comm_events __read_mostly; 378 static atomic_t nr_task_events __read_mostly; 379 static atomic_t nr_freq_events __read_mostly; 380 static atomic_t nr_switch_events __read_mostly; 381 382 static LIST_HEAD(pmus); 383 static DEFINE_MUTEX(pmus_lock); 384 static struct srcu_struct pmus_srcu; 385 386 /* 387 * perf event paranoia level: 388 * -1 - not paranoid at all 389 * 0 - disallow raw tracepoint access for unpriv 390 * 1 - disallow cpu events for unpriv 391 * 2 - disallow kernel profiling for unpriv 392 */ 393 int sysctl_perf_event_paranoid __read_mostly = 2; 394 395 /* Minimum for 512 kiB + 1 user control page */ 396 int sysctl_perf_event_mlock __read_mostly = 512 + (PAGE_SIZE / 1024); /* 'free' kiB per user */ 397 398 /* 399 * max perf event sample rate 400 */ 401 #define DEFAULT_MAX_SAMPLE_RATE 100000 402 #define DEFAULT_SAMPLE_PERIOD_NS (NSEC_PER_SEC / DEFAULT_MAX_SAMPLE_RATE) 403 #define DEFAULT_CPU_TIME_MAX_PERCENT 25 404 405 int sysctl_perf_event_sample_rate __read_mostly = DEFAULT_MAX_SAMPLE_RATE; 406 407 static int max_samples_per_tick __read_mostly = DIV_ROUND_UP(DEFAULT_MAX_SAMPLE_RATE, HZ); 408 static int perf_sample_period_ns __read_mostly = DEFAULT_SAMPLE_PERIOD_NS; 409 410 static int perf_sample_allowed_ns __read_mostly = 411 DEFAULT_SAMPLE_PERIOD_NS * DEFAULT_CPU_TIME_MAX_PERCENT / 100; 412 413 static void update_perf_cpu_limits(void) 414 { 415 u64 tmp = perf_sample_period_ns; 416 417 tmp *= sysctl_perf_cpu_time_max_percent; 418 tmp = div_u64(tmp, 100); 419 if (!tmp) 420 tmp = 1; 421 422 WRITE_ONCE(perf_sample_allowed_ns, tmp); 423 } 424 425 static int perf_rotate_context(struct perf_cpu_context *cpuctx); 426 427 int perf_proc_update_handler(struct ctl_table *table, int write, 428 void __user *buffer, size_t *lenp, 429 loff_t *ppos) 430 { 431 int ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); 432 433 if (ret || !write) 434 return ret; 435 436 /* 437 * If throttling is disabled don't allow the write: 438 */ 439 if (sysctl_perf_cpu_time_max_percent == 100 || 440 sysctl_perf_cpu_time_max_percent == 0) 441 return -EINVAL; 442 443 max_samples_per_tick = DIV_ROUND_UP(sysctl_perf_event_sample_rate, HZ); 444 perf_sample_period_ns = NSEC_PER_SEC / sysctl_perf_event_sample_rate; 445 update_perf_cpu_limits(); 446 447 return 0; 448 } 449 450 int sysctl_perf_cpu_time_max_percent __read_mostly = DEFAULT_CPU_TIME_MAX_PERCENT; 451 452 int perf_cpu_time_max_percent_handler(struct ctl_table *table, int write, 453 void __user *buffer, size_t *lenp, 454 loff_t *ppos) 455 { 456 int ret = proc_dointvec(table, write, buffer, lenp, ppos); 457 458 if (ret || !write) 459 return ret; 460 461 if (sysctl_perf_cpu_time_max_percent == 100 || 462 sysctl_perf_cpu_time_max_percent == 0) { 463 printk(KERN_WARNING 464 "perf: Dynamic interrupt throttling disabled, can hang your system!\n"); 465 WRITE_ONCE(perf_sample_allowed_ns, 0); 466 } else { 467 update_perf_cpu_limits(); 468 } 469 470 return 0; 471 } 472 473 /* 474 * perf samples are done in some very critical code paths (NMIs). 475 * If they take too much CPU time, the system can lock up and not 476 * get any real work done. This will drop the sample rate when 477 * we detect that events are taking too long. 478 */ 479 #define NR_ACCUMULATED_SAMPLES 128 480 static DEFINE_PER_CPU(u64, running_sample_length); 481 482 static u64 __report_avg; 483 static u64 __report_allowed; 484 485 static void perf_duration_warn(struct irq_work *w) 486 { 487 printk_ratelimited(KERN_INFO 488 "perf: interrupt took too long (%lld > %lld), lowering " 489 "kernel.perf_event_max_sample_rate to %d\n", 490 __report_avg, __report_allowed, 491 sysctl_perf_event_sample_rate); 492 } 493 494 static DEFINE_IRQ_WORK(perf_duration_work, perf_duration_warn); 495 496 void perf_sample_event_took(u64 sample_len_ns) 497 { 498 u64 max_len = READ_ONCE(perf_sample_allowed_ns); 499 u64 running_len; 500 u64 avg_len; 501 u32 max; 502 503 if (max_len == 0) 504 return; 505 506 /* Decay the counter by 1 average sample. */ 507 running_len = __this_cpu_read(running_sample_length); 508 running_len -= running_len/NR_ACCUMULATED_SAMPLES; 509 running_len += sample_len_ns; 510 __this_cpu_write(running_sample_length, running_len); 511 512 /* 513 * Note: this will be biased artifically low until we have 514 * seen NR_ACCUMULATED_SAMPLES. Doing it this way keeps us 515 * from having to maintain a count. 516 */ 517 avg_len = running_len/NR_ACCUMULATED_SAMPLES; 518 if (avg_len <= max_len) 519 return; 520 521 __report_avg = avg_len; 522 __report_allowed = max_len; 523 524 /* 525 * Compute a throttle threshold 25% below the current duration. 526 */ 527 avg_len += avg_len / 4; 528 max = (TICK_NSEC / 100) * sysctl_perf_cpu_time_max_percent; 529 if (avg_len < max) 530 max /= (u32)avg_len; 531 else 532 max = 1; 533 534 WRITE_ONCE(perf_sample_allowed_ns, avg_len); 535 WRITE_ONCE(max_samples_per_tick, max); 536 537 sysctl_perf_event_sample_rate = max * HZ; 538 perf_sample_period_ns = NSEC_PER_SEC / sysctl_perf_event_sample_rate; 539 540 if (!irq_work_queue(&perf_duration_work)) { 541 early_printk("perf: interrupt took too long (%lld > %lld), lowering " 542 "kernel.perf_event_max_sample_rate to %d\n", 543 __report_avg, __report_allowed, 544 sysctl_perf_event_sample_rate); 545 } 546 } 547 548 static atomic64_t perf_event_id; 549 550 static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx, 551 enum event_type_t event_type); 552 553 static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx, 554 enum event_type_t event_type, 555 struct task_struct *task); 556 557 static void update_context_time(struct perf_event_context *ctx); 558 static u64 perf_event_time(struct perf_event *event); 559 560 void __weak perf_event_print_debug(void) { } 561 562 extern __weak const char *perf_pmu_name(void) 563 { 564 return "pmu"; 565 } 566 567 static inline u64 perf_clock(void) 568 { 569 return local_clock(); 570 } 571 572 static inline u64 perf_event_clock(struct perf_event *event) 573 { 574 return event->clock(); 575 } 576 577 #ifdef CONFIG_CGROUP_PERF 578 579 static inline bool 580 perf_cgroup_match(struct perf_event *event) 581 { 582 struct perf_event_context *ctx = event->ctx; 583 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx); 584 585 /* @event doesn't care about cgroup */ 586 if (!event->cgrp) 587 return true; 588 589 /* wants specific cgroup scope but @cpuctx isn't associated with any */ 590 if (!cpuctx->cgrp) 591 return false; 592 593 /* 594 * Cgroup scoping is recursive. An event enabled for a cgroup is 595 * also enabled for all its descendant cgroups. If @cpuctx's 596 * cgroup is a descendant of @event's (the test covers identity 597 * case), it's a match. 598 */ 599 return cgroup_is_descendant(cpuctx->cgrp->css.cgroup, 600 event->cgrp->css.cgroup); 601 } 602 603 static inline void perf_detach_cgroup(struct perf_event *event) 604 { 605 css_put(&event->cgrp->css); 606 event->cgrp = NULL; 607 } 608 609 static inline int is_cgroup_event(struct perf_event *event) 610 { 611 return event->cgrp != NULL; 612 } 613 614 static inline u64 perf_cgroup_event_time(struct perf_event *event) 615 { 616 struct perf_cgroup_info *t; 617 618 t = per_cpu_ptr(event->cgrp->info, event->cpu); 619 return t->time; 620 } 621 622 static inline void __update_cgrp_time(struct perf_cgroup *cgrp) 623 { 624 struct perf_cgroup_info *info; 625 u64 now; 626 627 now = perf_clock(); 628 629 info = this_cpu_ptr(cgrp->info); 630 631 info->time += now - info->timestamp; 632 info->timestamp = now; 633 } 634 635 static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx) 636 { 637 struct perf_cgroup *cgrp_out = cpuctx->cgrp; 638 if (cgrp_out) 639 __update_cgrp_time(cgrp_out); 640 } 641 642 static inline void update_cgrp_time_from_event(struct perf_event *event) 643 { 644 struct perf_cgroup *cgrp; 645 646 /* 647 * ensure we access cgroup data only when needed and 648 * when we know the cgroup is pinned (css_get) 649 */ 650 if (!is_cgroup_event(event)) 651 return; 652 653 cgrp = perf_cgroup_from_task(current, event->ctx); 654 /* 655 * Do not update time when cgroup is not active 656 */ 657 if (cgrp == event->cgrp) 658 __update_cgrp_time(event->cgrp); 659 } 660 661 static inline void 662 perf_cgroup_set_timestamp(struct task_struct *task, 663 struct perf_event_context *ctx) 664 { 665 struct perf_cgroup *cgrp; 666 struct perf_cgroup_info *info; 667 668 /* 669 * ctx->lock held by caller 670 * ensure we do not access cgroup data 671 * unless we have the cgroup pinned (css_get) 672 */ 673 if (!task || !ctx->nr_cgroups) 674 return; 675 676 cgrp = perf_cgroup_from_task(task, ctx); 677 info = this_cpu_ptr(cgrp->info); 678 info->timestamp = ctx->timestamp; 679 } 680 681 #define PERF_CGROUP_SWOUT 0x1 /* cgroup switch out every event */ 682 #define PERF_CGROUP_SWIN 0x2 /* cgroup switch in events based on task */ 683 684 /* 685 * reschedule events based on the cgroup constraint of task. 686 * 687 * mode SWOUT : schedule out everything 688 * mode SWIN : schedule in based on cgroup for next 689 */ 690 static void perf_cgroup_switch(struct task_struct *task, int mode) 691 { 692 struct perf_cpu_context *cpuctx; 693 struct pmu *pmu; 694 unsigned long flags; 695 696 /* 697 * disable interrupts to avoid geting nr_cgroup 698 * changes via __perf_event_disable(). Also 699 * avoids preemption. 700 */ 701 local_irq_save(flags); 702 703 /* 704 * we reschedule only in the presence of cgroup 705 * constrained events. 706 */ 707 708 list_for_each_entry_rcu(pmu, &pmus, entry) { 709 cpuctx = this_cpu_ptr(pmu->pmu_cpu_context); 710 if (cpuctx->unique_pmu != pmu) 711 continue; /* ensure we process each cpuctx once */ 712 713 /* 714 * perf_cgroup_events says at least one 715 * context on this CPU has cgroup events. 716 * 717 * ctx->nr_cgroups reports the number of cgroup 718 * events for a context. 719 */ 720 if (cpuctx->ctx.nr_cgroups > 0) { 721 perf_ctx_lock(cpuctx, cpuctx->task_ctx); 722 perf_pmu_disable(cpuctx->ctx.pmu); 723 724 if (mode & PERF_CGROUP_SWOUT) { 725 cpu_ctx_sched_out(cpuctx, EVENT_ALL); 726 /* 727 * must not be done before ctxswout due 728 * to event_filter_match() in event_sched_out() 729 */ 730 cpuctx->cgrp = NULL; 731 } 732 733 if (mode & PERF_CGROUP_SWIN) { 734 WARN_ON_ONCE(cpuctx->cgrp); 735 /* 736 * set cgrp before ctxsw in to allow 737 * event_filter_match() to not have to pass 738 * task around 739 * we pass the cpuctx->ctx to perf_cgroup_from_task() 740 * because cgorup events are only per-cpu 741 */ 742 cpuctx->cgrp = perf_cgroup_from_task(task, &cpuctx->ctx); 743 cpu_ctx_sched_in(cpuctx, EVENT_ALL, task); 744 } 745 perf_pmu_enable(cpuctx->ctx.pmu); 746 perf_ctx_unlock(cpuctx, cpuctx->task_ctx); 747 } 748 } 749 750 local_irq_restore(flags); 751 } 752 753 static inline void perf_cgroup_sched_out(struct task_struct *task, 754 struct task_struct *next) 755 { 756 struct perf_cgroup *cgrp1; 757 struct perf_cgroup *cgrp2 = NULL; 758 759 rcu_read_lock(); 760 /* 761 * we come here when we know perf_cgroup_events > 0 762 * we do not need to pass the ctx here because we know 763 * we are holding the rcu lock 764 */ 765 cgrp1 = perf_cgroup_from_task(task, NULL); 766 cgrp2 = perf_cgroup_from_task(next, NULL); 767 768 /* 769 * only schedule out current cgroup events if we know 770 * that we are switching to a different cgroup. Otherwise, 771 * do no touch the cgroup events. 772 */ 773 if (cgrp1 != cgrp2) 774 perf_cgroup_switch(task, PERF_CGROUP_SWOUT); 775 776 rcu_read_unlock(); 777 } 778 779 static inline void perf_cgroup_sched_in(struct task_struct *prev, 780 struct task_struct *task) 781 { 782 struct perf_cgroup *cgrp1; 783 struct perf_cgroup *cgrp2 = NULL; 784 785 rcu_read_lock(); 786 /* 787 * we come here when we know perf_cgroup_events > 0 788 * we do not need to pass the ctx here because we know 789 * we are holding the rcu lock 790 */ 791 cgrp1 = perf_cgroup_from_task(task, NULL); 792 cgrp2 = perf_cgroup_from_task(prev, NULL); 793 794 /* 795 * only need to schedule in cgroup events if we are changing 796 * cgroup during ctxsw. Cgroup events were not scheduled 797 * out of ctxsw out if that was not the case. 798 */ 799 if (cgrp1 != cgrp2) 800 perf_cgroup_switch(task, PERF_CGROUP_SWIN); 801 802 rcu_read_unlock(); 803 } 804 805 static inline int perf_cgroup_connect(int fd, struct perf_event *event, 806 struct perf_event_attr *attr, 807 struct perf_event *group_leader) 808 { 809 struct perf_cgroup *cgrp; 810 struct cgroup_subsys_state *css; 811 struct fd f = fdget(fd); 812 int ret = 0; 813 814 if (!f.file) 815 return -EBADF; 816 817 css = css_tryget_online_from_dir(f.file->f_path.dentry, 818 &perf_event_cgrp_subsys); 819 if (IS_ERR(css)) { 820 ret = PTR_ERR(css); 821 goto out; 822 } 823 824 cgrp = container_of(css, struct perf_cgroup, css); 825 event->cgrp = cgrp; 826 827 /* 828 * all events in a group must monitor 829 * the same cgroup because a task belongs 830 * to only one perf cgroup at a time 831 */ 832 if (group_leader && group_leader->cgrp != cgrp) { 833 perf_detach_cgroup(event); 834 ret = -EINVAL; 835 } 836 out: 837 fdput(f); 838 return ret; 839 } 840 841 static inline void 842 perf_cgroup_set_shadow_time(struct perf_event *event, u64 now) 843 { 844 struct perf_cgroup_info *t; 845 t = per_cpu_ptr(event->cgrp->info, event->cpu); 846 event->shadow_ctx_time = now - t->timestamp; 847 } 848 849 static inline void 850 perf_cgroup_defer_enabled(struct perf_event *event) 851 { 852 /* 853 * when the current task's perf cgroup does not match 854 * the event's, we need to remember to call the 855 * perf_mark_enable() function the first time a task with 856 * a matching perf cgroup is scheduled in. 857 */ 858 if (is_cgroup_event(event) && !perf_cgroup_match(event)) 859 event->cgrp_defer_enabled = 1; 860 } 861 862 static inline void 863 perf_cgroup_mark_enabled(struct perf_event *event, 864 struct perf_event_context *ctx) 865 { 866 struct perf_event *sub; 867 u64 tstamp = perf_event_time(event); 868 869 if (!event->cgrp_defer_enabled) 870 return; 871 872 event->cgrp_defer_enabled = 0; 873 874 event->tstamp_enabled = tstamp - event->total_time_enabled; 875 list_for_each_entry(sub, &event->sibling_list, group_entry) { 876 if (sub->state >= PERF_EVENT_STATE_INACTIVE) { 877 sub->tstamp_enabled = tstamp - sub->total_time_enabled; 878 sub->cgrp_defer_enabled = 0; 879 } 880 } 881 } 882 883 /* 884 * Update cpuctx->cgrp so that it is set when first cgroup event is added and 885 * cleared when last cgroup event is removed. 886 */ 887 static inline void 888 list_update_cgroup_event(struct perf_event *event, 889 struct perf_event_context *ctx, bool add) 890 { 891 struct perf_cpu_context *cpuctx; 892 893 if (!is_cgroup_event(event)) 894 return; 895 896 if (add && ctx->nr_cgroups++) 897 return; 898 else if (!add && --ctx->nr_cgroups) 899 return; 900 /* 901 * Because cgroup events are always per-cpu events, 902 * this will always be called from the right CPU. 903 */ 904 cpuctx = __get_cpu_context(ctx); 905 cpuctx->cgrp = add ? event->cgrp : NULL; 906 } 907 908 #else /* !CONFIG_CGROUP_PERF */ 909 910 static inline bool 911 perf_cgroup_match(struct perf_event *event) 912 { 913 return true; 914 } 915 916 static inline void perf_detach_cgroup(struct perf_event *event) 917 {} 918 919 static inline int is_cgroup_event(struct perf_event *event) 920 { 921 return 0; 922 } 923 924 static inline u64 perf_cgroup_event_cgrp_time(struct perf_event *event) 925 { 926 return 0; 927 } 928 929 static inline void update_cgrp_time_from_event(struct perf_event *event) 930 { 931 } 932 933 static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx) 934 { 935 } 936 937 static inline void perf_cgroup_sched_out(struct task_struct *task, 938 struct task_struct *next) 939 { 940 } 941 942 static inline void perf_cgroup_sched_in(struct task_struct *prev, 943 struct task_struct *task) 944 { 945 } 946 947 static inline int perf_cgroup_connect(pid_t pid, struct perf_event *event, 948 struct perf_event_attr *attr, 949 struct perf_event *group_leader) 950 { 951 return -EINVAL; 952 } 953 954 static inline void 955 perf_cgroup_set_timestamp(struct task_struct *task, 956 struct perf_event_context *ctx) 957 { 958 } 959 960 void 961 perf_cgroup_switch(struct task_struct *task, struct task_struct *next) 962 { 963 } 964 965 static inline void 966 perf_cgroup_set_shadow_time(struct perf_event *event, u64 now) 967 { 968 } 969 970 static inline u64 perf_cgroup_event_time(struct perf_event *event) 971 { 972 return 0; 973 } 974 975 static inline void 976 perf_cgroup_defer_enabled(struct perf_event *event) 977 { 978 } 979 980 static inline void 981 perf_cgroup_mark_enabled(struct perf_event *event, 982 struct perf_event_context *ctx) 983 { 984 } 985 986 static inline void 987 list_update_cgroup_event(struct perf_event *event, 988 struct perf_event_context *ctx, bool add) 989 { 990 } 991 992 #endif 993 994 /* 995 * set default to be dependent on timer tick just 996 * like original code 997 */ 998 #define PERF_CPU_HRTIMER (1000 / HZ) 999 /* 1000 * function must be called with interrupts disbled 1001 */ 1002 static enum hrtimer_restart perf_mux_hrtimer_handler(struct hrtimer *hr) 1003 { 1004 struct perf_cpu_context *cpuctx; 1005 int rotations = 0; 1006 1007 WARN_ON(!irqs_disabled()); 1008 1009 cpuctx = container_of(hr, struct perf_cpu_context, hrtimer); 1010 rotations = perf_rotate_context(cpuctx); 1011 1012 raw_spin_lock(&cpuctx->hrtimer_lock); 1013 if (rotations) 1014 hrtimer_forward_now(hr, cpuctx->hrtimer_interval); 1015 else 1016 cpuctx->hrtimer_active = 0; 1017 raw_spin_unlock(&cpuctx->hrtimer_lock); 1018 1019 return rotations ? HRTIMER_RESTART : HRTIMER_NORESTART; 1020 } 1021 1022 static void __perf_mux_hrtimer_init(struct perf_cpu_context *cpuctx, int cpu) 1023 { 1024 struct hrtimer *timer = &cpuctx->hrtimer; 1025 struct pmu *pmu = cpuctx->ctx.pmu; 1026 u64 interval; 1027 1028 /* no multiplexing needed for SW PMU */ 1029 if (pmu->task_ctx_nr == perf_sw_context) 1030 return; 1031 1032 /* 1033 * check default is sane, if not set then force to 1034 * default interval (1/tick) 1035 */ 1036 interval = pmu->hrtimer_interval_ms; 1037 if (interval < 1) 1038 interval = pmu->hrtimer_interval_ms = PERF_CPU_HRTIMER; 1039 1040 cpuctx->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * interval); 1041 1042 raw_spin_lock_init(&cpuctx->hrtimer_lock); 1043 hrtimer_init(timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED); 1044 timer->function = perf_mux_hrtimer_handler; 1045 } 1046 1047 static int perf_mux_hrtimer_restart(struct perf_cpu_context *cpuctx) 1048 { 1049 struct hrtimer *timer = &cpuctx->hrtimer; 1050 struct pmu *pmu = cpuctx->ctx.pmu; 1051 unsigned long flags; 1052 1053 /* not for SW PMU */ 1054 if (pmu->task_ctx_nr == perf_sw_context) 1055 return 0; 1056 1057 raw_spin_lock_irqsave(&cpuctx->hrtimer_lock, flags); 1058 if (!cpuctx->hrtimer_active) { 1059 cpuctx->hrtimer_active = 1; 1060 hrtimer_forward_now(timer, cpuctx->hrtimer_interval); 1061 hrtimer_start_expires(timer, HRTIMER_MODE_ABS_PINNED); 1062 } 1063 raw_spin_unlock_irqrestore(&cpuctx->hrtimer_lock, flags); 1064 1065 return 0; 1066 } 1067 1068 void perf_pmu_disable(struct pmu *pmu) 1069 { 1070 int *count = this_cpu_ptr(pmu->pmu_disable_count); 1071 if (!(*count)++) 1072 pmu->pmu_disable(pmu); 1073 } 1074 1075 void perf_pmu_enable(struct pmu *pmu) 1076 { 1077 int *count = this_cpu_ptr(pmu->pmu_disable_count); 1078 if (!--(*count)) 1079 pmu->pmu_enable(pmu); 1080 } 1081 1082 static DEFINE_PER_CPU(struct list_head, active_ctx_list); 1083 1084 /* 1085 * perf_event_ctx_activate(), perf_event_ctx_deactivate(), and 1086 * perf_event_task_tick() are fully serialized because they're strictly cpu 1087 * affine and perf_event_ctx{activate,deactivate} are called with IRQs 1088 * disabled, while perf_event_task_tick is called from IRQ context. 1089 */ 1090 static void perf_event_ctx_activate(struct perf_event_context *ctx) 1091 { 1092 struct list_head *head = this_cpu_ptr(&active_ctx_list); 1093 1094 WARN_ON(!irqs_disabled()); 1095 1096 WARN_ON(!list_empty(&ctx->active_ctx_list)); 1097 1098 list_add(&ctx->active_ctx_list, head); 1099 } 1100 1101 static void perf_event_ctx_deactivate(struct perf_event_context *ctx) 1102 { 1103 WARN_ON(!irqs_disabled()); 1104 1105 WARN_ON(list_empty(&ctx->active_ctx_list)); 1106 1107 list_del_init(&ctx->active_ctx_list); 1108 } 1109 1110 static void get_ctx(struct perf_event_context *ctx) 1111 { 1112 WARN_ON(!atomic_inc_not_zero(&ctx->refcount)); 1113 } 1114 1115 static void free_ctx(struct rcu_head *head) 1116 { 1117 struct perf_event_context *ctx; 1118 1119 ctx = container_of(head, struct perf_event_context, rcu_head); 1120 kfree(ctx->task_ctx_data); 1121 kfree(ctx); 1122 } 1123 1124 static void put_ctx(struct perf_event_context *ctx) 1125 { 1126 if (atomic_dec_and_test(&ctx->refcount)) { 1127 if (ctx->parent_ctx) 1128 put_ctx(ctx->parent_ctx); 1129 if (ctx->task && ctx->task != TASK_TOMBSTONE) 1130 put_task_struct(ctx->task); 1131 call_rcu(&ctx->rcu_head, free_ctx); 1132 } 1133 } 1134 1135 /* 1136 * Because of perf_event::ctx migration in sys_perf_event_open::move_group and 1137 * perf_pmu_migrate_context() we need some magic. 1138 * 1139 * Those places that change perf_event::ctx will hold both 1140 * perf_event_ctx::mutex of the 'old' and 'new' ctx value. 1141 * 1142 * Lock ordering is by mutex address. There are two other sites where 1143 * perf_event_context::mutex nests and those are: 1144 * 1145 * - perf_event_exit_task_context() [ child , 0 ] 1146 * perf_event_exit_event() 1147 * put_event() [ parent, 1 ] 1148 * 1149 * - perf_event_init_context() [ parent, 0 ] 1150 * inherit_task_group() 1151 * inherit_group() 1152 * inherit_event() 1153 * perf_event_alloc() 1154 * perf_init_event() 1155 * perf_try_init_event() [ child , 1 ] 1156 * 1157 * While it appears there is an obvious deadlock here -- the parent and child 1158 * nesting levels are inverted between the two. This is in fact safe because 1159 * life-time rules separate them. That is an exiting task cannot fork, and a 1160 * spawning task cannot (yet) exit. 1161 * 1162 * But remember that that these are parent<->child context relations, and 1163 * migration does not affect children, therefore these two orderings should not 1164 * interact. 1165 * 1166 * The change in perf_event::ctx does not affect children (as claimed above) 1167 * because the sys_perf_event_open() case will install a new event and break 1168 * the ctx parent<->child relation, and perf_pmu_migrate_context() is only 1169 * concerned with cpuctx and that doesn't have children. 1170 * 1171 * The places that change perf_event::ctx will issue: 1172 * 1173 * perf_remove_from_context(); 1174 * synchronize_rcu(); 1175 * perf_install_in_context(); 1176 * 1177 * to affect the change. The remove_from_context() + synchronize_rcu() should 1178 * quiesce the event, after which we can install it in the new location. This 1179 * means that only external vectors (perf_fops, prctl) can perturb the event 1180 * while in transit. Therefore all such accessors should also acquire 1181 * perf_event_context::mutex to serialize against this. 1182 * 1183 * However; because event->ctx can change while we're waiting to acquire 1184 * ctx->mutex we must be careful and use the below perf_event_ctx_lock() 1185 * function. 1186 * 1187 * Lock order: 1188 * cred_guard_mutex 1189 * task_struct::perf_event_mutex 1190 * perf_event_context::mutex 1191 * perf_event::child_mutex; 1192 * perf_event_context::lock 1193 * perf_event::mmap_mutex 1194 * mmap_sem 1195 */ 1196 static struct perf_event_context * 1197 perf_event_ctx_lock_nested(struct perf_event *event, int nesting) 1198 { 1199 struct perf_event_context *ctx; 1200 1201 again: 1202 rcu_read_lock(); 1203 ctx = ACCESS_ONCE(event->ctx); 1204 if (!atomic_inc_not_zero(&ctx->refcount)) { 1205 rcu_read_unlock(); 1206 goto again; 1207 } 1208 rcu_read_unlock(); 1209 1210 mutex_lock_nested(&ctx->mutex, nesting); 1211 if (event->ctx != ctx) { 1212 mutex_unlock(&ctx->mutex); 1213 put_ctx(ctx); 1214 goto again; 1215 } 1216 1217 return ctx; 1218 } 1219 1220 static inline struct perf_event_context * 1221 perf_event_ctx_lock(struct perf_event *event) 1222 { 1223 return perf_event_ctx_lock_nested(event, 0); 1224 } 1225 1226 static void perf_event_ctx_unlock(struct perf_event *event, 1227 struct perf_event_context *ctx) 1228 { 1229 mutex_unlock(&ctx->mutex); 1230 put_ctx(ctx); 1231 } 1232 1233 /* 1234 * This must be done under the ctx->lock, such as to serialize against 1235 * context_equiv(), therefore we cannot call put_ctx() since that might end up 1236 * calling scheduler related locks and ctx->lock nests inside those. 1237 */ 1238 static __must_check struct perf_event_context * 1239 unclone_ctx(struct perf_event_context *ctx) 1240 { 1241 struct perf_event_context *parent_ctx = ctx->parent_ctx; 1242 1243 lockdep_assert_held(&ctx->lock); 1244 1245 if (parent_ctx) 1246 ctx->parent_ctx = NULL; 1247 ctx->generation++; 1248 1249 return parent_ctx; 1250 } 1251 1252 static u32 perf_event_pid(struct perf_event *event, struct task_struct *p) 1253 { 1254 /* 1255 * only top level events have the pid namespace they were created in 1256 */ 1257 if (event->parent) 1258 event = event->parent; 1259 1260 return task_tgid_nr_ns(p, event->ns); 1261 } 1262 1263 static u32 perf_event_tid(struct perf_event *event, struct task_struct *p) 1264 { 1265 /* 1266 * only top level events have the pid namespace they were created in 1267 */ 1268 if (event->parent) 1269 event = event->parent; 1270 1271 return task_pid_nr_ns(p, event->ns); 1272 } 1273 1274 /* 1275 * If we inherit events we want to return the parent event id 1276 * to userspace. 1277 */ 1278 static u64 primary_event_id(struct perf_event *event) 1279 { 1280 u64 id = event->id; 1281 1282 if (event->parent) 1283 id = event->parent->id; 1284 1285 return id; 1286 } 1287 1288 /* 1289 * Get the perf_event_context for a task and lock it. 1290 * 1291 * This has to cope with with the fact that until it is locked, 1292 * the context could get moved to another task. 1293 */ 1294 static struct perf_event_context * 1295 perf_lock_task_context(struct task_struct *task, int ctxn, unsigned long *flags) 1296 { 1297 struct perf_event_context *ctx; 1298 1299 retry: 1300 /* 1301 * One of the few rules of preemptible RCU is that one cannot do 1302 * rcu_read_unlock() while holding a scheduler (or nested) lock when 1303 * part of the read side critical section was irqs-enabled -- see 1304 * rcu_read_unlock_special(). 1305 * 1306 * Since ctx->lock nests under rq->lock we must ensure the entire read 1307 * side critical section has interrupts disabled. 1308 */ 1309 local_irq_save(*flags); 1310 rcu_read_lock(); 1311 ctx = rcu_dereference(task->perf_event_ctxp[ctxn]); 1312 if (ctx) { 1313 /* 1314 * If this context is a clone of another, it might 1315 * get swapped for another underneath us by 1316 * perf_event_task_sched_out, though the 1317 * rcu_read_lock() protects us from any context 1318 * getting freed. Lock the context and check if it 1319 * got swapped before we could get the lock, and retry 1320 * if so. If we locked the right context, then it 1321 * can't get swapped on us any more. 1322 */ 1323 raw_spin_lock(&ctx->lock); 1324 if (ctx != rcu_dereference(task->perf_event_ctxp[ctxn])) { 1325 raw_spin_unlock(&ctx->lock); 1326 rcu_read_unlock(); 1327 local_irq_restore(*flags); 1328 goto retry; 1329 } 1330 1331 if (ctx->task == TASK_TOMBSTONE || 1332 !atomic_inc_not_zero(&ctx->refcount)) { 1333 raw_spin_unlock(&ctx->lock); 1334 ctx = NULL; 1335 } else { 1336 WARN_ON_ONCE(ctx->task != task); 1337 } 1338 } 1339 rcu_read_unlock(); 1340 if (!ctx) 1341 local_irq_restore(*flags); 1342 return ctx; 1343 } 1344 1345 /* 1346 * Get the context for a task and increment its pin_count so it 1347 * can't get swapped to another task. This also increments its 1348 * reference count so that the context can't get freed. 1349 */ 1350 static struct perf_event_context * 1351 perf_pin_task_context(struct task_struct *task, int ctxn) 1352 { 1353 struct perf_event_context *ctx; 1354 unsigned long flags; 1355 1356 ctx = perf_lock_task_context(task, ctxn, &flags); 1357 if (ctx) { 1358 ++ctx->pin_count; 1359 raw_spin_unlock_irqrestore(&ctx->lock, flags); 1360 } 1361 return ctx; 1362 } 1363 1364 static void perf_unpin_context(struct perf_event_context *ctx) 1365 { 1366 unsigned long flags; 1367 1368 raw_spin_lock_irqsave(&ctx->lock, flags); 1369 --ctx->pin_count; 1370 raw_spin_unlock_irqrestore(&ctx->lock, flags); 1371 } 1372 1373 /* 1374 * Update the record of the current time in a context. 1375 */ 1376 static void update_context_time(struct perf_event_context *ctx) 1377 { 1378 u64 now = perf_clock(); 1379 1380 ctx->time += now - ctx->timestamp; 1381 ctx->timestamp = now; 1382 } 1383 1384 static u64 perf_event_time(struct perf_event *event) 1385 { 1386 struct perf_event_context *ctx = event->ctx; 1387 1388 if (is_cgroup_event(event)) 1389 return perf_cgroup_event_time(event); 1390 1391 return ctx ? ctx->time : 0; 1392 } 1393 1394 /* 1395 * Update the total_time_enabled and total_time_running fields for a event. 1396 */ 1397 static void update_event_times(struct perf_event *event) 1398 { 1399 struct perf_event_context *ctx = event->ctx; 1400 u64 run_end; 1401 1402 lockdep_assert_held(&ctx->lock); 1403 1404 if (event->state < PERF_EVENT_STATE_INACTIVE || 1405 event->group_leader->state < PERF_EVENT_STATE_INACTIVE) 1406 return; 1407 1408 /* 1409 * in cgroup mode, time_enabled represents 1410 * the time the event was enabled AND active 1411 * tasks were in the monitored cgroup. This is 1412 * independent of the activity of the context as 1413 * there may be a mix of cgroup and non-cgroup events. 1414 * 1415 * That is why we treat cgroup events differently 1416 * here. 1417 */ 1418 if (is_cgroup_event(event)) 1419 run_end = perf_cgroup_event_time(event); 1420 else if (ctx->is_active) 1421 run_end = ctx->time; 1422 else 1423 run_end = event->tstamp_stopped; 1424 1425 event->total_time_enabled = run_end - event->tstamp_enabled; 1426 1427 if (event->state == PERF_EVENT_STATE_INACTIVE) 1428 run_end = event->tstamp_stopped; 1429 else 1430 run_end = perf_event_time(event); 1431 1432 event->total_time_running = run_end - event->tstamp_running; 1433 1434 } 1435 1436 /* 1437 * Update total_time_enabled and total_time_running for all events in a group. 1438 */ 1439 static void update_group_times(struct perf_event *leader) 1440 { 1441 struct perf_event *event; 1442 1443 update_event_times(leader); 1444 list_for_each_entry(event, &leader->sibling_list, group_entry) 1445 update_event_times(event); 1446 } 1447 1448 static struct list_head * 1449 ctx_group_list(struct perf_event *event, struct perf_event_context *ctx) 1450 { 1451 if (event->attr.pinned) 1452 return &ctx->pinned_groups; 1453 else 1454 return &ctx->flexible_groups; 1455 } 1456 1457 /* 1458 * Add a event from the lists for its context. 1459 * Must be called with ctx->mutex and ctx->lock held. 1460 */ 1461 static void 1462 list_add_event(struct perf_event *event, struct perf_event_context *ctx) 1463 { 1464 1465 lockdep_assert_held(&ctx->lock); 1466 1467 WARN_ON_ONCE(event->attach_state & PERF_ATTACH_CONTEXT); 1468 event->attach_state |= PERF_ATTACH_CONTEXT; 1469 1470 /* 1471 * If we're a stand alone event or group leader, we go to the context 1472 * list, group events are kept attached to the group so that 1473 * perf_group_detach can, at all times, locate all siblings. 1474 */ 1475 if (event->group_leader == event) { 1476 struct list_head *list; 1477 1478 event->group_caps = event->event_caps; 1479 1480 list = ctx_group_list(event, ctx); 1481 list_add_tail(&event->group_entry, list); 1482 } 1483 1484 list_update_cgroup_event(event, ctx, true); 1485 1486 list_add_rcu(&event->event_entry, &ctx->event_list); 1487 ctx->nr_events++; 1488 if (event->attr.inherit_stat) 1489 ctx->nr_stat++; 1490 1491 ctx->generation++; 1492 } 1493 1494 /* 1495 * Initialize event state based on the perf_event_attr::disabled. 1496 */ 1497 static inline void perf_event__state_init(struct perf_event *event) 1498 { 1499 event->state = event->attr.disabled ? PERF_EVENT_STATE_OFF : 1500 PERF_EVENT_STATE_INACTIVE; 1501 } 1502 1503 static void __perf_event_read_size(struct perf_event *event, int nr_siblings) 1504 { 1505 int entry = sizeof(u64); /* value */ 1506 int size = 0; 1507 int nr = 1; 1508 1509 if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) 1510 size += sizeof(u64); 1511 1512 if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) 1513 size += sizeof(u64); 1514 1515 if (event->attr.read_format & PERF_FORMAT_ID) 1516 entry += sizeof(u64); 1517 1518 if (event->attr.read_format & PERF_FORMAT_GROUP) { 1519 nr += nr_siblings; 1520 size += sizeof(u64); 1521 } 1522 1523 size += entry * nr; 1524 event->read_size = size; 1525 } 1526 1527 static void __perf_event_header_size(struct perf_event *event, u64 sample_type) 1528 { 1529 struct perf_sample_data *data; 1530 u16 size = 0; 1531 1532 if (sample_type & PERF_SAMPLE_IP) 1533 size += sizeof(data->ip); 1534 1535 if (sample_type & PERF_SAMPLE_ADDR) 1536 size += sizeof(data->addr); 1537 1538 if (sample_type & PERF_SAMPLE_PERIOD) 1539 size += sizeof(data->period); 1540 1541 if (sample_type & PERF_SAMPLE_WEIGHT) 1542 size += sizeof(data->weight); 1543 1544 if (sample_type & PERF_SAMPLE_READ) 1545 size += event->read_size; 1546 1547 if (sample_type & PERF_SAMPLE_DATA_SRC) 1548 size += sizeof(data->data_src.val); 1549 1550 if (sample_type & PERF_SAMPLE_TRANSACTION) 1551 size += sizeof(data->txn); 1552 1553 event->header_size = size; 1554 } 1555 1556 /* 1557 * Called at perf_event creation and when events are attached/detached from a 1558 * group. 1559 */ 1560 static void perf_event__header_size(struct perf_event *event) 1561 { 1562 __perf_event_read_size(event, 1563 event->group_leader->nr_siblings); 1564 __perf_event_header_size(event, event->attr.sample_type); 1565 } 1566 1567 static void perf_event__id_header_size(struct perf_event *event) 1568 { 1569 struct perf_sample_data *data; 1570 u64 sample_type = event->attr.sample_type; 1571 u16 size = 0; 1572 1573 if (sample_type & PERF_SAMPLE_TID) 1574 size += sizeof(data->tid_entry); 1575 1576 if (sample_type & PERF_SAMPLE_TIME) 1577 size += sizeof(data->time); 1578 1579 if (sample_type & PERF_SAMPLE_IDENTIFIER) 1580 size += sizeof(data->id); 1581 1582 if (sample_type & PERF_SAMPLE_ID) 1583 size += sizeof(data->id); 1584 1585 if (sample_type & PERF_SAMPLE_STREAM_ID) 1586 size += sizeof(data->stream_id); 1587 1588 if (sample_type & PERF_SAMPLE_CPU) 1589 size += sizeof(data->cpu_entry); 1590 1591 event->id_header_size = size; 1592 } 1593 1594 static bool perf_event_validate_size(struct perf_event *event) 1595 { 1596 /* 1597 * The values computed here will be over-written when we actually 1598 * attach the event. 1599 */ 1600 __perf_event_read_size(event, event->group_leader->nr_siblings + 1); 1601 __perf_event_header_size(event, event->attr.sample_type & ~PERF_SAMPLE_READ); 1602 perf_event__id_header_size(event); 1603 1604 /* 1605 * Sum the lot; should not exceed the 64k limit we have on records. 1606 * Conservative limit to allow for callchains and other variable fields. 1607 */ 1608 if (event->read_size + event->header_size + 1609 event->id_header_size + sizeof(struct perf_event_header) >= 16*1024) 1610 return false; 1611 1612 return true; 1613 } 1614 1615 static void perf_group_attach(struct perf_event *event) 1616 { 1617 struct perf_event *group_leader = event->group_leader, *pos; 1618 1619 /* 1620 * We can have double attach due to group movement in perf_event_open. 1621 */ 1622 if (event->attach_state & PERF_ATTACH_GROUP) 1623 return; 1624 1625 event->attach_state |= PERF_ATTACH_GROUP; 1626 1627 if (group_leader == event) 1628 return; 1629 1630 WARN_ON_ONCE(group_leader->ctx != event->ctx); 1631 1632 group_leader->group_caps &= event->event_caps; 1633 1634 list_add_tail(&event->group_entry, &group_leader->sibling_list); 1635 group_leader->nr_siblings++; 1636 1637 perf_event__header_size(group_leader); 1638 1639 list_for_each_entry(pos, &group_leader->sibling_list, group_entry) 1640 perf_event__header_size(pos); 1641 } 1642 1643 /* 1644 * Remove a event from the lists for its context. 1645 * Must be called with ctx->mutex and ctx->lock held. 1646 */ 1647 static void 1648 list_del_event(struct perf_event *event, struct perf_event_context *ctx) 1649 { 1650 WARN_ON_ONCE(event->ctx != ctx); 1651 lockdep_assert_held(&ctx->lock); 1652 1653 /* 1654 * We can have double detach due to exit/hot-unplug + close. 1655 */ 1656 if (!(event->attach_state & PERF_ATTACH_CONTEXT)) 1657 return; 1658 1659 event->attach_state &= ~PERF_ATTACH_CONTEXT; 1660 1661 list_update_cgroup_event(event, ctx, false); 1662 1663 ctx->nr_events--; 1664 if (event->attr.inherit_stat) 1665 ctx->nr_stat--; 1666 1667 list_del_rcu(&event->event_entry); 1668 1669 if (event->group_leader == event) 1670 list_del_init(&event->group_entry); 1671 1672 update_group_times(event); 1673 1674 /* 1675 * If event was in error state, then keep it 1676 * that way, otherwise bogus counts will be 1677 * returned on read(). The only way to get out 1678 * of error state is by explicit re-enabling 1679 * of the event 1680 */ 1681 if (event->state > PERF_EVENT_STATE_OFF) 1682 event->state = PERF_EVENT_STATE_OFF; 1683 1684 ctx->generation++; 1685 } 1686 1687 static void perf_group_detach(struct perf_event *event) 1688 { 1689 struct perf_event *sibling, *tmp; 1690 struct list_head *list = NULL; 1691 1692 /* 1693 * We can have double detach due to exit/hot-unplug + close. 1694 */ 1695 if (!(event->attach_state & PERF_ATTACH_GROUP)) 1696 return; 1697 1698 event->attach_state &= ~PERF_ATTACH_GROUP; 1699 1700 /* 1701 * If this is a sibling, remove it from its group. 1702 */ 1703 if (event->group_leader != event) { 1704 list_del_init(&event->group_entry); 1705 event->group_leader->nr_siblings--; 1706 goto out; 1707 } 1708 1709 if (!list_empty(&event->group_entry)) 1710 list = &event->group_entry; 1711 1712 /* 1713 * If this was a group event with sibling events then 1714 * upgrade the siblings to singleton events by adding them 1715 * to whatever list we are on. 1716 */ 1717 list_for_each_entry_safe(sibling, tmp, &event->sibling_list, group_entry) { 1718 if (list) 1719 list_move_tail(&sibling->group_entry, list); 1720 sibling->group_leader = sibling; 1721 1722 /* Inherit group flags from the previous leader */ 1723 sibling->group_caps = event->group_caps; 1724 1725 WARN_ON_ONCE(sibling->ctx != event->ctx); 1726 } 1727 1728 out: 1729 perf_event__header_size(event->group_leader); 1730 1731 list_for_each_entry(tmp, &event->group_leader->sibling_list, group_entry) 1732 perf_event__header_size(tmp); 1733 } 1734 1735 static bool is_orphaned_event(struct perf_event *event) 1736 { 1737 return event->state == PERF_EVENT_STATE_DEAD; 1738 } 1739 1740 static inline int __pmu_filter_match(struct perf_event *event) 1741 { 1742 struct pmu *pmu = event->pmu; 1743 return pmu->filter_match ? pmu->filter_match(event) : 1; 1744 } 1745 1746 /* 1747 * Check whether we should attempt to schedule an event group based on 1748 * PMU-specific filtering. An event group can consist of HW and SW events, 1749 * potentially with a SW leader, so we must check all the filters, to 1750 * determine whether a group is schedulable: 1751 */ 1752 static inline int pmu_filter_match(struct perf_event *event) 1753 { 1754 struct perf_event *child; 1755 1756 if (!__pmu_filter_match(event)) 1757 return 0; 1758 1759 list_for_each_entry(child, &event->sibling_list, group_entry) { 1760 if (!__pmu_filter_match(child)) 1761 return 0; 1762 } 1763 1764 return 1; 1765 } 1766 1767 static inline int 1768 event_filter_match(struct perf_event *event) 1769 { 1770 return (event->cpu == -1 || event->cpu == smp_processor_id()) && 1771 perf_cgroup_match(event) && pmu_filter_match(event); 1772 } 1773 1774 static void 1775 event_sched_out(struct perf_event *event, 1776 struct perf_cpu_context *cpuctx, 1777 struct perf_event_context *ctx) 1778 { 1779 u64 tstamp = perf_event_time(event); 1780 u64 delta; 1781 1782 WARN_ON_ONCE(event->ctx != ctx); 1783 lockdep_assert_held(&ctx->lock); 1784 1785 /* 1786 * An event which could not be activated because of 1787 * filter mismatch still needs to have its timings 1788 * maintained, otherwise bogus information is return 1789 * via read() for time_enabled, time_running: 1790 */ 1791 if (event->state == PERF_EVENT_STATE_INACTIVE && 1792 !event_filter_match(event)) { 1793 delta = tstamp - event->tstamp_stopped; 1794 event->tstamp_running += delta; 1795 event->tstamp_stopped = tstamp; 1796 } 1797 1798 if (event->state != PERF_EVENT_STATE_ACTIVE) 1799 return; 1800 1801 perf_pmu_disable(event->pmu); 1802 1803 event->tstamp_stopped = tstamp; 1804 event->pmu->del(event, 0); 1805 event->oncpu = -1; 1806 event->state = PERF_EVENT_STATE_INACTIVE; 1807 if (event->pending_disable) { 1808 event->pending_disable = 0; 1809 event->state = PERF_EVENT_STATE_OFF; 1810 } 1811 1812 if (!is_software_event(event)) 1813 cpuctx->active_oncpu--; 1814 if (!--ctx->nr_active) 1815 perf_event_ctx_deactivate(ctx); 1816 if (event->attr.freq && event->attr.sample_freq) 1817 ctx->nr_freq--; 1818 if (event->attr.exclusive || !cpuctx->active_oncpu) 1819 cpuctx->exclusive = 0; 1820 1821 perf_pmu_enable(event->pmu); 1822 } 1823 1824 static void 1825 group_sched_out(struct perf_event *group_event, 1826 struct perf_cpu_context *cpuctx, 1827 struct perf_event_context *ctx) 1828 { 1829 struct perf_event *event; 1830 int state = group_event->state; 1831 1832 perf_pmu_disable(ctx->pmu); 1833 1834 event_sched_out(group_event, cpuctx, ctx); 1835 1836 /* 1837 * Schedule out siblings (if any): 1838 */ 1839 list_for_each_entry(event, &group_event->sibling_list, group_entry) 1840 event_sched_out(event, cpuctx, ctx); 1841 1842 perf_pmu_enable(ctx->pmu); 1843 1844 if (state == PERF_EVENT_STATE_ACTIVE && group_event->attr.exclusive) 1845 cpuctx->exclusive = 0; 1846 } 1847 1848 #define DETACH_GROUP 0x01UL 1849 1850 /* 1851 * Cross CPU call to remove a performance event 1852 * 1853 * We disable the event on the hardware level first. After that we 1854 * remove it from the context list. 1855 */ 1856 static void 1857 __perf_remove_from_context(struct perf_event *event, 1858 struct perf_cpu_context *cpuctx, 1859 struct perf_event_context *ctx, 1860 void *info) 1861 { 1862 unsigned long flags = (unsigned long)info; 1863 1864 event_sched_out(event, cpuctx, ctx); 1865 if (flags & DETACH_GROUP) 1866 perf_group_detach(event); 1867 list_del_event(event, ctx); 1868 1869 if (!ctx->nr_events && ctx->is_active) { 1870 ctx->is_active = 0; 1871 if (ctx->task) { 1872 WARN_ON_ONCE(cpuctx->task_ctx != ctx); 1873 cpuctx->task_ctx = NULL; 1874 } 1875 } 1876 } 1877 1878 /* 1879 * Remove the event from a task's (or a CPU's) list of events. 1880 * 1881 * If event->ctx is a cloned context, callers must make sure that 1882 * every task struct that event->ctx->task could possibly point to 1883 * remains valid. This is OK when called from perf_release since 1884 * that only calls us on the top-level context, which can't be a clone. 1885 * When called from perf_event_exit_task, it's OK because the 1886 * context has been detached from its task. 1887 */ 1888 static void perf_remove_from_context(struct perf_event *event, unsigned long flags) 1889 { 1890 lockdep_assert_held(&event->ctx->mutex); 1891 1892 event_function_call(event, __perf_remove_from_context, (void *)flags); 1893 } 1894 1895 /* 1896 * Cross CPU call to disable a performance event 1897 */ 1898 static void __perf_event_disable(struct perf_event *event, 1899 struct perf_cpu_context *cpuctx, 1900 struct perf_event_context *ctx, 1901 void *info) 1902 { 1903 if (event->state < PERF_EVENT_STATE_INACTIVE) 1904 return; 1905 1906 update_context_time(ctx); 1907 update_cgrp_time_from_event(event); 1908 update_group_times(event); 1909 if (event == event->group_leader) 1910 group_sched_out(event, cpuctx, ctx); 1911 else 1912 event_sched_out(event, cpuctx, ctx); 1913 event->state = PERF_EVENT_STATE_OFF; 1914 } 1915 1916 /* 1917 * Disable a event. 1918 * 1919 * If event->ctx is a cloned context, callers must make sure that 1920 * every task struct that event->ctx->task could possibly point to 1921 * remains valid. This condition is satisifed when called through 1922 * perf_event_for_each_child or perf_event_for_each because they 1923 * hold the top-level event's child_mutex, so any descendant that 1924 * goes to exit will block in perf_event_exit_event(). 1925 * 1926 * When called from perf_pending_event it's OK because event->ctx 1927 * is the current context on this CPU and preemption is disabled, 1928 * hence we can't get into perf_event_task_sched_out for this context. 1929 */ 1930 static void _perf_event_disable(struct perf_event *event) 1931 { 1932 struct perf_event_context *ctx = event->ctx; 1933 1934 raw_spin_lock_irq(&ctx->lock); 1935 if (event->state <= PERF_EVENT_STATE_OFF) { 1936 raw_spin_unlock_irq(&ctx->lock); 1937 return; 1938 } 1939 raw_spin_unlock_irq(&ctx->lock); 1940 1941 event_function_call(event, __perf_event_disable, NULL); 1942 } 1943 1944 void perf_event_disable_local(struct perf_event *event) 1945 { 1946 event_function_local(event, __perf_event_disable, NULL); 1947 } 1948 1949 /* 1950 * Strictly speaking kernel users cannot create groups and therefore this 1951 * interface does not need the perf_event_ctx_lock() magic. 1952 */ 1953 void perf_event_disable(struct perf_event *event) 1954 { 1955 struct perf_event_context *ctx; 1956 1957 ctx = perf_event_ctx_lock(event); 1958 _perf_event_disable(event); 1959 perf_event_ctx_unlock(event, ctx); 1960 } 1961 EXPORT_SYMBOL_GPL(perf_event_disable); 1962 1963 static void perf_set_shadow_time(struct perf_event *event, 1964 struct perf_event_context *ctx, 1965 u64 tstamp) 1966 { 1967 /* 1968 * use the correct time source for the time snapshot 1969 * 1970 * We could get by without this by leveraging the 1971 * fact that to get to this function, the caller 1972 * has most likely already called update_context_time() 1973 * and update_cgrp_time_xx() and thus both timestamp 1974 * are identical (or very close). Given that tstamp is, 1975 * already adjusted for cgroup, we could say that: 1976 * tstamp - ctx->timestamp 1977 * is equivalent to 1978 * tstamp - cgrp->timestamp. 1979 * 1980 * Then, in perf_output_read(), the calculation would 1981 * work with no changes because: 1982 * - event is guaranteed scheduled in 1983 * - no scheduled out in between 1984 * - thus the timestamp would be the same 1985 * 1986 * But this is a bit hairy. 1987 * 1988 * So instead, we have an explicit cgroup call to remain 1989 * within the time time source all along. We believe it 1990 * is cleaner and simpler to understand. 1991 */ 1992 if (is_cgroup_event(event)) 1993 perf_cgroup_set_shadow_time(event, tstamp); 1994 else 1995 event->shadow_ctx_time = tstamp - ctx->timestamp; 1996 } 1997 1998 #define MAX_INTERRUPTS (~0ULL) 1999 2000 static void perf_log_throttle(struct perf_event *event, int enable); 2001 static void perf_log_itrace_start(struct perf_event *event); 2002 2003 static int 2004 event_sched_in(struct perf_event *event, 2005 struct perf_cpu_context *cpuctx, 2006 struct perf_event_context *ctx) 2007 { 2008 u64 tstamp = perf_event_time(event); 2009 int ret = 0; 2010 2011 lockdep_assert_held(&ctx->lock); 2012 2013 if (event->state <= PERF_EVENT_STATE_OFF) 2014 return 0; 2015 2016 WRITE_ONCE(event->oncpu, smp_processor_id()); 2017 /* 2018 * Order event::oncpu write to happen before the ACTIVE state 2019 * is visible. 2020 */ 2021 smp_wmb(); 2022 WRITE_ONCE(event->state, PERF_EVENT_STATE_ACTIVE); 2023 2024 /* 2025 * Unthrottle events, since we scheduled we might have missed several 2026 * ticks already, also for a heavily scheduling task there is little 2027 * guarantee it'll get a tick in a timely manner. 2028 */ 2029 if (unlikely(event->hw.interrupts == MAX_INTERRUPTS)) { 2030 perf_log_throttle(event, 1); 2031 event->hw.interrupts = 0; 2032 } 2033 2034 /* 2035 * The new state must be visible before we turn it on in the hardware: 2036 */ 2037 smp_wmb(); 2038 2039 perf_pmu_disable(event->pmu); 2040 2041 perf_set_shadow_time(event, ctx, tstamp); 2042 2043 perf_log_itrace_start(event); 2044 2045 if (event->pmu->add(event, PERF_EF_START)) { 2046 event->state = PERF_EVENT_STATE_INACTIVE; 2047 event->oncpu = -1; 2048 ret = -EAGAIN; 2049 goto out; 2050 } 2051 2052 event->tstamp_running += tstamp - event->tstamp_stopped; 2053 2054 if (!is_software_event(event)) 2055 cpuctx->active_oncpu++; 2056 if (!ctx->nr_active++) 2057 perf_event_ctx_activate(ctx); 2058 if (event->attr.freq && event->attr.sample_freq) 2059 ctx->nr_freq++; 2060 2061 if (event->attr.exclusive) 2062 cpuctx->exclusive = 1; 2063 2064 out: 2065 perf_pmu_enable(event->pmu); 2066 2067 return ret; 2068 } 2069 2070 static int 2071 group_sched_in(struct perf_event *group_event, 2072 struct perf_cpu_context *cpuctx, 2073 struct perf_event_context *ctx) 2074 { 2075 struct perf_event *event, *partial_group = NULL; 2076 struct pmu *pmu = ctx->pmu; 2077 u64 now = ctx->time; 2078 bool simulate = false; 2079 2080 if (group_event->state == PERF_EVENT_STATE_OFF) 2081 return 0; 2082 2083 pmu->start_txn(pmu, PERF_PMU_TXN_ADD); 2084 2085 if (event_sched_in(group_event, cpuctx, ctx)) { 2086 pmu->cancel_txn(pmu); 2087 perf_mux_hrtimer_restart(cpuctx); 2088 return -EAGAIN; 2089 } 2090 2091 /* 2092 * Schedule in siblings as one group (if any): 2093 */ 2094 list_for_each_entry(event, &group_event->sibling_list, group_entry) { 2095 if (event_sched_in(event, cpuctx, ctx)) { 2096 partial_group = event; 2097 goto group_error; 2098 } 2099 } 2100 2101 if (!pmu->commit_txn(pmu)) 2102 return 0; 2103 2104 group_error: 2105 /* 2106 * Groups can be scheduled in as one unit only, so undo any 2107 * partial group before returning: 2108 * The events up to the failed event are scheduled out normally, 2109 * tstamp_stopped will be updated. 2110 * 2111 * The failed events and the remaining siblings need to have 2112 * their timings updated as if they had gone thru event_sched_in() 2113 * and event_sched_out(). This is required to get consistent timings 2114 * across the group. This also takes care of the case where the group 2115 * could never be scheduled by ensuring tstamp_stopped is set to mark 2116 * the time the event was actually stopped, such that time delta 2117 * calculation in update_event_times() is correct. 2118 */ 2119 list_for_each_entry(event, &group_event->sibling_list, group_entry) { 2120 if (event == partial_group) 2121 simulate = true; 2122 2123 if (simulate) { 2124 event->tstamp_running += now - event->tstamp_stopped; 2125 event->tstamp_stopped = now; 2126 } else { 2127 event_sched_out(event, cpuctx, ctx); 2128 } 2129 } 2130 event_sched_out(group_event, cpuctx, ctx); 2131 2132 pmu->cancel_txn(pmu); 2133 2134 perf_mux_hrtimer_restart(cpuctx); 2135 2136 return -EAGAIN; 2137 } 2138 2139 /* 2140 * Work out whether we can put this event group on the CPU now. 2141 */ 2142 static int group_can_go_on(struct perf_event *event, 2143 struct perf_cpu_context *cpuctx, 2144 int can_add_hw) 2145 { 2146 /* 2147 * Groups consisting entirely of software events can always go on. 2148 */ 2149 if (event->group_caps & PERF_EV_CAP_SOFTWARE) 2150 return 1; 2151 /* 2152 * If an exclusive group is already on, no other hardware 2153 * events can go on. 2154 */ 2155 if (cpuctx->exclusive) 2156 return 0; 2157 /* 2158 * If this group is exclusive and there are already 2159 * events on the CPU, it can't go on. 2160 */ 2161 if (event->attr.exclusive && cpuctx->active_oncpu) 2162 return 0; 2163 /* 2164 * Otherwise, try to add it if all previous groups were able 2165 * to go on. 2166 */ 2167 return can_add_hw; 2168 } 2169 2170 static void add_event_to_ctx(struct perf_event *event, 2171 struct perf_event_context *ctx) 2172 { 2173 u64 tstamp = perf_event_time(event); 2174 2175 list_add_event(event, ctx); 2176 perf_group_attach(event); 2177 event->tstamp_enabled = tstamp; 2178 event->tstamp_running = tstamp; 2179 event->tstamp_stopped = tstamp; 2180 } 2181 2182 static void ctx_sched_out(struct perf_event_context *ctx, 2183 struct perf_cpu_context *cpuctx, 2184 enum event_type_t event_type); 2185 static void 2186 ctx_sched_in(struct perf_event_context *ctx, 2187 struct perf_cpu_context *cpuctx, 2188 enum event_type_t event_type, 2189 struct task_struct *task); 2190 2191 static void task_ctx_sched_out(struct perf_cpu_context *cpuctx, 2192 struct perf_event_context *ctx) 2193 { 2194 if (!cpuctx->task_ctx) 2195 return; 2196 2197 if (WARN_ON_ONCE(ctx != cpuctx->task_ctx)) 2198 return; 2199 2200 ctx_sched_out(ctx, cpuctx, EVENT_ALL); 2201 } 2202 2203 static void perf_event_sched_in(struct perf_cpu_context *cpuctx, 2204 struct perf_event_context *ctx, 2205 struct task_struct *task) 2206 { 2207 cpu_ctx_sched_in(cpuctx, EVENT_PINNED, task); 2208 if (ctx) 2209 ctx_sched_in(ctx, cpuctx, EVENT_PINNED, task); 2210 cpu_ctx_sched_in(cpuctx, EVENT_FLEXIBLE, task); 2211 if (ctx) 2212 ctx_sched_in(ctx, cpuctx, EVENT_FLEXIBLE, task); 2213 } 2214 2215 static void ctx_resched(struct perf_cpu_context *cpuctx, 2216 struct perf_event_context *task_ctx) 2217 { 2218 perf_pmu_disable(cpuctx->ctx.pmu); 2219 if (task_ctx) 2220 task_ctx_sched_out(cpuctx, task_ctx); 2221 cpu_ctx_sched_out(cpuctx, EVENT_ALL); 2222 perf_event_sched_in(cpuctx, task_ctx, current); 2223 perf_pmu_enable(cpuctx->ctx.pmu); 2224 } 2225 2226 /* 2227 * Cross CPU call to install and enable a performance event 2228 * 2229 * Very similar to remote_function() + event_function() but cannot assume that 2230 * things like ctx->is_active and cpuctx->task_ctx are set. 2231 */ 2232 static int __perf_install_in_context(void *info) 2233 { 2234 struct perf_event *event = info; 2235 struct perf_event_context *ctx = event->ctx; 2236 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx); 2237 struct perf_event_context *task_ctx = cpuctx->task_ctx; 2238 bool activate = true; 2239 int ret = 0; 2240 2241 raw_spin_lock(&cpuctx->ctx.lock); 2242 if (ctx->task) { 2243 raw_spin_lock(&ctx->lock); 2244 task_ctx = ctx; 2245 2246 /* If we're on the wrong CPU, try again */ 2247 if (task_cpu(ctx->task) != smp_processor_id()) { 2248 ret = -ESRCH; 2249 goto unlock; 2250 } 2251 2252 /* 2253 * If we're on the right CPU, see if the task we target is 2254 * current, if not we don't have to activate the ctx, a future 2255 * context switch will do that for us. 2256 */ 2257 if (ctx->task != current) 2258 activate = false; 2259 else 2260 WARN_ON_ONCE(cpuctx->task_ctx && cpuctx->task_ctx != ctx); 2261 2262 } else if (task_ctx) { 2263 raw_spin_lock(&task_ctx->lock); 2264 } 2265 2266 if (activate) { 2267 ctx_sched_out(ctx, cpuctx, EVENT_TIME); 2268 add_event_to_ctx(event, ctx); 2269 ctx_resched(cpuctx, task_ctx); 2270 } else { 2271 add_event_to_ctx(event, ctx); 2272 } 2273 2274 unlock: 2275 perf_ctx_unlock(cpuctx, task_ctx); 2276 2277 return ret; 2278 } 2279 2280 /* 2281 * Attach a performance event to a context. 2282 * 2283 * Very similar to event_function_call, see comment there. 2284 */ 2285 static void 2286 perf_install_in_context(struct perf_event_context *ctx, 2287 struct perf_event *event, 2288 int cpu) 2289 { 2290 struct task_struct *task = READ_ONCE(ctx->task); 2291 2292 lockdep_assert_held(&ctx->mutex); 2293 2294 if (event->cpu != -1) 2295 event->cpu = cpu; 2296 2297 /* 2298 * Ensures that if we can observe event->ctx, both the event and ctx 2299 * will be 'complete'. See perf_iterate_sb_cpu(). 2300 */ 2301 smp_store_release(&event->ctx, ctx); 2302 2303 if (!task) { 2304 cpu_function_call(cpu, __perf_install_in_context, event); 2305 return; 2306 } 2307 2308 /* 2309 * Should not happen, we validate the ctx is still alive before calling. 2310 */ 2311 if (WARN_ON_ONCE(task == TASK_TOMBSTONE)) 2312 return; 2313 2314 /* 2315 * Installing events is tricky because we cannot rely on ctx->is_active 2316 * to be set in case this is the nr_events 0 -> 1 transition. 2317 */ 2318 again: 2319 /* 2320 * Cannot use task_function_call() because we need to run on the task's 2321 * CPU regardless of whether its current or not. 2322 */ 2323 if (!cpu_function_call(task_cpu(task), __perf_install_in_context, event)) 2324 return; 2325 2326 raw_spin_lock_irq(&ctx->lock); 2327 task = ctx->task; 2328 if (WARN_ON_ONCE(task == TASK_TOMBSTONE)) { 2329 /* 2330 * Cannot happen because we already checked above (which also 2331 * cannot happen), and we hold ctx->mutex, which serializes us 2332 * against perf_event_exit_task_context(). 2333 */ 2334 raw_spin_unlock_irq(&ctx->lock); 2335 return; 2336 } 2337 raw_spin_unlock_irq(&ctx->lock); 2338 /* 2339 * Since !ctx->is_active doesn't mean anything, we must IPI 2340 * unconditionally. 2341 */ 2342 goto again; 2343 } 2344 2345 /* 2346 * Put a event into inactive state and update time fields. 2347 * Enabling the leader of a group effectively enables all 2348 * the group members that aren't explicitly disabled, so we 2349 * have to update their ->tstamp_enabled also. 2350 * Note: this works for group members as well as group leaders 2351 * since the non-leader members' sibling_lists will be empty. 2352 */ 2353 static void __perf_event_mark_enabled(struct perf_event *event) 2354 { 2355 struct perf_event *sub; 2356 u64 tstamp = perf_event_time(event); 2357 2358 event->state = PERF_EVENT_STATE_INACTIVE; 2359 event->tstamp_enabled = tstamp - event->total_time_enabled; 2360 list_for_each_entry(sub, &event->sibling_list, group_entry) { 2361 if (sub->state >= PERF_EVENT_STATE_INACTIVE) 2362 sub->tstamp_enabled = tstamp - sub->total_time_enabled; 2363 } 2364 } 2365 2366 /* 2367 * Cross CPU call to enable a performance event 2368 */ 2369 static void __perf_event_enable(struct perf_event *event, 2370 struct perf_cpu_context *cpuctx, 2371 struct perf_event_context *ctx, 2372 void *info) 2373 { 2374 struct perf_event *leader = event->group_leader; 2375 struct perf_event_context *task_ctx; 2376 2377 if (event->state >= PERF_EVENT_STATE_INACTIVE || 2378 event->state <= PERF_EVENT_STATE_ERROR) 2379 return; 2380 2381 if (ctx->is_active) 2382 ctx_sched_out(ctx, cpuctx, EVENT_TIME); 2383 2384 __perf_event_mark_enabled(event); 2385 2386 if (!ctx->is_active) 2387 return; 2388 2389 if (!event_filter_match(event)) { 2390 if (is_cgroup_event(event)) 2391 perf_cgroup_defer_enabled(event); 2392 ctx_sched_in(ctx, cpuctx, EVENT_TIME, current); 2393 return; 2394 } 2395 2396 /* 2397 * If the event is in a group and isn't the group leader, 2398 * then don't put it on unless the group is on. 2399 */ 2400 if (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE) { 2401 ctx_sched_in(ctx, cpuctx, EVENT_TIME, current); 2402 return; 2403 } 2404 2405 task_ctx = cpuctx->task_ctx; 2406 if (ctx->task) 2407 WARN_ON_ONCE(task_ctx != ctx); 2408 2409 ctx_resched(cpuctx, task_ctx); 2410 } 2411 2412 /* 2413 * Enable a event. 2414 * 2415 * If event->ctx is a cloned context, callers must make sure that 2416 * every task struct that event->ctx->task could possibly point to 2417 * remains valid. This condition is satisfied when called through 2418 * perf_event_for_each_child or perf_event_for_each as described 2419 * for perf_event_disable. 2420 */ 2421 static void _perf_event_enable(struct perf_event *event) 2422 { 2423 struct perf_event_context *ctx = event->ctx; 2424 2425 raw_spin_lock_irq(&ctx->lock); 2426 if (event->state >= PERF_EVENT_STATE_INACTIVE || 2427 event->state < PERF_EVENT_STATE_ERROR) { 2428 raw_spin_unlock_irq(&ctx->lock); 2429 return; 2430 } 2431 2432 /* 2433 * If the event is in error state, clear that first. 2434 * 2435 * That way, if we see the event in error state below, we know that it 2436 * has gone back into error state, as distinct from the task having 2437 * been scheduled away before the cross-call arrived. 2438 */ 2439 if (event->state == PERF_EVENT_STATE_ERROR) 2440 event->state = PERF_EVENT_STATE_OFF; 2441 raw_spin_unlock_irq(&ctx->lock); 2442 2443 event_function_call(event, __perf_event_enable, NULL); 2444 } 2445 2446 /* 2447 * See perf_event_disable(); 2448 */ 2449 void perf_event_enable(struct perf_event *event) 2450 { 2451 struct perf_event_context *ctx; 2452 2453 ctx = perf_event_ctx_lock(event); 2454 _perf_event_enable(event); 2455 perf_event_ctx_unlock(event, ctx); 2456 } 2457 EXPORT_SYMBOL_GPL(perf_event_enable); 2458 2459 struct stop_event_data { 2460 struct perf_event *event; 2461 unsigned int restart; 2462 }; 2463 2464 static int __perf_event_stop(void *info) 2465 { 2466 struct stop_event_data *sd = info; 2467 struct perf_event *event = sd->event; 2468 2469 /* if it's already INACTIVE, do nothing */ 2470 if (READ_ONCE(event->state) != PERF_EVENT_STATE_ACTIVE) 2471 return 0; 2472 2473 /* matches smp_wmb() in event_sched_in() */ 2474 smp_rmb(); 2475 2476 /* 2477 * There is a window with interrupts enabled before we get here, 2478 * so we need to check again lest we try to stop another CPU's event. 2479 */ 2480 if (READ_ONCE(event->oncpu) != smp_processor_id()) 2481 return -EAGAIN; 2482 2483 event->pmu->stop(event, PERF_EF_UPDATE); 2484 2485 /* 2486 * May race with the actual stop (through perf_pmu_output_stop()), 2487 * but it is only used for events with AUX ring buffer, and such 2488 * events will refuse to restart because of rb::aux_mmap_count==0, 2489 * see comments in perf_aux_output_begin(). 2490 * 2491 * Since this is happening on a event-local CPU, no trace is lost 2492 * while restarting. 2493 */ 2494 if (sd->restart) 2495 event->pmu->start(event, 0); 2496 2497 return 0; 2498 } 2499 2500 static int perf_event_stop(struct perf_event *event, int restart) 2501 { 2502 struct stop_event_data sd = { 2503 .event = event, 2504 .restart = restart, 2505 }; 2506 int ret = 0; 2507 2508 do { 2509 if (READ_ONCE(event->state) != PERF_EVENT_STATE_ACTIVE) 2510 return 0; 2511 2512 /* matches smp_wmb() in event_sched_in() */ 2513 smp_rmb(); 2514 2515 /* 2516 * We only want to restart ACTIVE events, so if the event goes 2517 * inactive here (event->oncpu==-1), there's nothing more to do; 2518 * fall through with ret==-ENXIO. 2519 */ 2520 ret = cpu_function_call(READ_ONCE(event->oncpu), 2521 __perf_event_stop, &sd); 2522 } while (ret == -EAGAIN); 2523 2524 return ret; 2525 } 2526 2527 /* 2528 * In order to contain the amount of racy and tricky in the address filter 2529 * configuration management, it is a two part process: 2530 * 2531 * (p1) when userspace mappings change as a result of (1) or (2) or (3) below, 2532 * we update the addresses of corresponding vmas in 2533 * event::addr_filters_offs array and bump the event::addr_filters_gen; 2534 * (p2) when an event is scheduled in (pmu::add), it calls 2535 * perf_event_addr_filters_sync() which calls pmu::addr_filters_sync() 2536 * if the generation has changed since the previous call. 2537 * 2538 * If (p1) happens while the event is active, we restart it to force (p2). 2539 * 2540 * (1) perf_addr_filters_apply(): adjusting filters' offsets based on 2541 * pre-existing mappings, called once when new filters arrive via SET_FILTER 2542 * ioctl; 2543 * (2) perf_addr_filters_adjust(): adjusting filters' offsets based on newly 2544 * registered mapping, called for every new mmap(), with mm::mmap_sem down 2545 * for reading; 2546 * (3) perf_event_addr_filters_exec(): clearing filters' offsets in the process 2547 * of exec. 2548 */ 2549 void perf_event_addr_filters_sync(struct perf_event *event) 2550 { 2551 struct perf_addr_filters_head *ifh = perf_event_addr_filters(event); 2552 2553 if (!has_addr_filter(event)) 2554 return; 2555 2556 raw_spin_lock(&ifh->lock); 2557 if (event->addr_filters_gen != event->hw.addr_filters_gen) { 2558 event->pmu->addr_filters_sync(event); 2559 event->hw.addr_filters_gen = event->addr_filters_gen; 2560 } 2561 raw_spin_unlock(&ifh->lock); 2562 } 2563 EXPORT_SYMBOL_GPL(perf_event_addr_filters_sync); 2564 2565 static int _perf_event_refresh(struct perf_event *event, int refresh) 2566 { 2567 /* 2568 * not supported on inherited events 2569 */ 2570 if (event->attr.inherit || !is_sampling_event(event)) 2571 return -EINVAL; 2572 2573 atomic_add(refresh, &event->event_limit); 2574 _perf_event_enable(event); 2575 2576 return 0; 2577 } 2578 2579 /* 2580 * See perf_event_disable() 2581 */ 2582 int perf_event_refresh(struct perf_event *event, int refresh) 2583 { 2584 struct perf_event_context *ctx; 2585 int ret; 2586 2587 ctx = perf_event_ctx_lock(event); 2588 ret = _perf_event_refresh(event, refresh); 2589 perf_event_ctx_unlock(event, ctx); 2590 2591 return ret; 2592 } 2593 EXPORT_SYMBOL_GPL(perf_event_refresh); 2594 2595 static void ctx_sched_out(struct perf_event_context *ctx, 2596 struct perf_cpu_context *cpuctx, 2597 enum event_type_t event_type) 2598 { 2599 int is_active = ctx->is_active; 2600 struct perf_event *event; 2601 2602 lockdep_assert_held(&ctx->lock); 2603 2604 if (likely(!ctx->nr_events)) { 2605 /* 2606 * See __perf_remove_from_context(). 2607 */ 2608 WARN_ON_ONCE(ctx->is_active); 2609 if (ctx->task) 2610 WARN_ON_ONCE(cpuctx->task_ctx); 2611 return; 2612 } 2613 2614 ctx->is_active &= ~event_type; 2615 if (!(ctx->is_active & EVENT_ALL)) 2616 ctx->is_active = 0; 2617 2618 if (ctx->task) { 2619 WARN_ON_ONCE(cpuctx->task_ctx != ctx); 2620 if (!ctx->is_active) 2621 cpuctx->task_ctx = NULL; 2622 } 2623 2624 /* 2625 * Always update time if it was set; not only when it changes. 2626 * Otherwise we can 'forget' to update time for any but the last 2627 * context we sched out. For example: 2628 * 2629 * ctx_sched_out(.event_type = EVENT_FLEXIBLE) 2630 * ctx_sched_out(.event_type = EVENT_PINNED) 2631 * 2632 * would only update time for the pinned events. 2633 */ 2634 if (is_active & EVENT_TIME) { 2635 /* update (and stop) ctx time */ 2636 update_context_time(ctx); 2637 update_cgrp_time_from_cpuctx(cpuctx); 2638 } 2639 2640 is_active ^= ctx->is_active; /* changed bits */ 2641 2642 if (!ctx->nr_active || !(is_active & EVENT_ALL)) 2643 return; 2644 2645 perf_pmu_disable(ctx->pmu); 2646 if (is_active & EVENT_PINNED) { 2647 list_for_each_entry(event, &ctx->pinned_groups, group_entry) 2648 group_sched_out(event, cpuctx, ctx); 2649 } 2650 2651 if (is_active & EVENT_FLEXIBLE) { 2652 list_for_each_entry(event, &ctx->flexible_groups, group_entry) 2653 group_sched_out(event, cpuctx, ctx); 2654 } 2655 perf_pmu_enable(ctx->pmu); 2656 } 2657 2658 /* 2659 * Test whether two contexts are equivalent, i.e. whether they have both been 2660 * cloned from the same version of the same context. 2661 * 2662 * Equivalence is measured using a generation number in the context that is 2663 * incremented on each modification to it; see unclone_ctx(), list_add_event() 2664 * and list_del_event(). 2665 */ 2666 static int context_equiv(struct perf_event_context *ctx1, 2667 struct perf_event_context *ctx2) 2668 { 2669 lockdep_assert_held(&ctx1->lock); 2670 lockdep_assert_held(&ctx2->lock); 2671 2672 /* Pinning disables the swap optimization */ 2673 if (ctx1->pin_count || ctx2->pin_count) 2674 return 0; 2675 2676 /* If ctx1 is the parent of ctx2 */ 2677 if (ctx1 == ctx2->parent_ctx && ctx1->generation == ctx2->parent_gen) 2678 return 1; 2679 2680 /* If ctx2 is the parent of ctx1 */ 2681 if (ctx1->parent_ctx == ctx2 && ctx1->parent_gen == ctx2->generation) 2682 return 1; 2683 2684 /* 2685 * If ctx1 and ctx2 have the same parent; we flatten the parent 2686 * hierarchy, see perf_event_init_context(). 2687 */ 2688 if (ctx1->parent_ctx && ctx1->parent_ctx == ctx2->parent_ctx && 2689 ctx1->parent_gen == ctx2->parent_gen) 2690 return 1; 2691 2692 /* Unmatched */ 2693 return 0; 2694 } 2695 2696 static void __perf_event_sync_stat(struct perf_event *event, 2697 struct perf_event *next_event) 2698 { 2699 u64 value; 2700 2701 if (!event->attr.inherit_stat) 2702 return; 2703 2704 /* 2705 * Update the event value, we cannot use perf_event_read() 2706 * because we're in the middle of a context switch and have IRQs 2707 * disabled, which upsets smp_call_function_single(), however 2708 * we know the event must be on the current CPU, therefore we 2709 * don't need to use it. 2710 */ 2711 switch (event->state) { 2712 case PERF_EVENT_STATE_ACTIVE: 2713 event->pmu->read(event); 2714 /* fall-through */ 2715 2716 case PERF_EVENT_STATE_INACTIVE: 2717 update_event_times(event); 2718 break; 2719 2720 default: 2721 break; 2722 } 2723 2724 /* 2725 * In order to keep per-task stats reliable we need to flip the event 2726 * values when we flip the contexts. 2727 */ 2728 value = local64_read(&next_event->count); 2729 value = local64_xchg(&event->count, value); 2730 local64_set(&next_event->count, value); 2731 2732 swap(event->total_time_enabled, next_event->total_time_enabled); 2733 swap(event->total_time_running, next_event->total_time_running); 2734 2735 /* 2736 * Since we swizzled the values, update the user visible data too. 2737 */ 2738 perf_event_update_userpage(event); 2739 perf_event_update_userpage(next_event); 2740 } 2741 2742 static void perf_event_sync_stat(struct perf_event_context *ctx, 2743 struct perf_event_context *next_ctx) 2744 { 2745 struct perf_event *event, *next_event; 2746 2747 if (!ctx->nr_stat) 2748 return; 2749 2750 update_context_time(ctx); 2751 2752 event = list_first_entry(&ctx->event_list, 2753 struct perf_event, event_entry); 2754 2755 next_event = list_first_entry(&next_ctx->event_list, 2756 struct perf_event, event_entry); 2757 2758 while (&event->event_entry != &ctx->event_list && 2759 &next_event->event_entry != &next_ctx->event_list) { 2760 2761 __perf_event_sync_stat(event, next_event); 2762 2763 event = list_next_entry(event, event_entry); 2764 next_event = list_next_entry(next_event, event_entry); 2765 } 2766 } 2767 2768 static void perf_event_context_sched_out(struct task_struct *task, int ctxn, 2769 struct task_struct *next) 2770 { 2771 struct perf_event_context *ctx = task->perf_event_ctxp[ctxn]; 2772 struct perf_event_context *next_ctx; 2773 struct perf_event_context *parent, *next_parent; 2774 struct perf_cpu_context *cpuctx; 2775 int do_switch = 1; 2776 2777 if (likely(!ctx)) 2778 return; 2779 2780 cpuctx = __get_cpu_context(ctx); 2781 if (!cpuctx->task_ctx) 2782 return; 2783 2784 rcu_read_lock(); 2785 next_ctx = next->perf_event_ctxp[ctxn]; 2786 if (!next_ctx) 2787 goto unlock; 2788 2789 parent = rcu_dereference(ctx->parent_ctx); 2790 next_parent = rcu_dereference(next_ctx->parent_ctx); 2791 2792 /* If neither context have a parent context; they cannot be clones. */ 2793 if (!parent && !next_parent) 2794 goto unlock; 2795 2796 if (next_parent == ctx || next_ctx == parent || next_parent == parent) { 2797 /* 2798 * Looks like the two contexts are clones, so we might be 2799 * able to optimize the context switch. We lock both 2800 * contexts and check that they are clones under the 2801 * lock (including re-checking that neither has been 2802 * uncloned in the meantime). It doesn't matter which 2803 * order we take the locks because no other cpu could 2804 * be trying to lock both of these tasks. 2805 */ 2806 raw_spin_lock(&ctx->lock); 2807 raw_spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING); 2808 if (context_equiv(ctx, next_ctx)) { 2809 WRITE_ONCE(ctx->task, next); 2810 WRITE_ONCE(next_ctx->task, task); 2811 2812 swap(ctx->task_ctx_data, next_ctx->task_ctx_data); 2813 2814 /* 2815 * RCU_INIT_POINTER here is safe because we've not 2816 * modified the ctx and the above modification of 2817 * ctx->task and ctx->task_ctx_data are immaterial 2818 * since those values are always verified under 2819 * ctx->lock which we're now holding. 2820 */ 2821 RCU_INIT_POINTER(task->perf_event_ctxp[ctxn], next_ctx); 2822 RCU_INIT_POINTER(next->perf_event_ctxp[ctxn], ctx); 2823 2824 do_switch = 0; 2825 2826 perf_event_sync_stat(ctx, next_ctx); 2827 } 2828 raw_spin_unlock(&next_ctx->lock); 2829 raw_spin_unlock(&ctx->lock); 2830 } 2831 unlock: 2832 rcu_read_unlock(); 2833 2834 if (do_switch) { 2835 raw_spin_lock(&ctx->lock); 2836 task_ctx_sched_out(cpuctx, ctx); 2837 raw_spin_unlock(&ctx->lock); 2838 } 2839 } 2840 2841 static DEFINE_PER_CPU(struct list_head, sched_cb_list); 2842 2843 void perf_sched_cb_dec(struct pmu *pmu) 2844 { 2845 struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context); 2846 2847 this_cpu_dec(perf_sched_cb_usages); 2848 2849 if (!--cpuctx->sched_cb_usage) 2850 list_del(&cpuctx->sched_cb_entry); 2851 } 2852 2853 2854 void perf_sched_cb_inc(struct pmu *pmu) 2855 { 2856 struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context); 2857 2858 if (!cpuctx->sched_cb_usage++) 2859 list_add(&cpuctx->sched_cb_entry, this_cpu_ptr(&sched_cb_list)); 2860 2861 this_cpu_inc(perf_sched_cb_usages); 2862 } 2863 2864 /* 2865 * This function provides the context switch callback to the lower code 2866 * layer. It is invoked ONLY when the context switch callback is enabled. 2867 * 2868 * This callback is relevant even to per-cpu events; for example multi event 2869 * PEBS requires this to provide PID/TID information. This requires we flush 2870 * all queued PEBS records before we context switch to a new task. 2871 */ 2872 static void perf_pmu_sched_task(struct task_struct *prev, 2873 struct task_struct *next, 2874 bool sched_in) 2875 { 2876 struct perf_cpu_context *cpuctx; 2877 struct pmu *pmu; 2878 2879 if (prev == next) 2880 return; 2881 2882 list_for_each_entry(cpuctx, this_cpu_ptr(&sched_cb_list), sched_cb_entry) { 2883 pmu = cpuctx->unique_pmu; /* software PMUs will not have sched_task */ 2884 2885 if (WARN_ON_ONCE(!pmu->sched_task)) 2886 continue; 2887 2888 perf_ctx_lock(cpuctx, cpuctx->task_ctx); 2889 perf_pmu_disable(pmu); 2890 2891 pmu->sched_task(cpuctx->task_ctx, sched_in); 2892 2893 perf_pmu_enable(pmu); 2894 perf_ctx_unlock(cpuctx, cpuctx->task_ctx); 2895 } 2896 } 2897 2898 static void perf_event_switch(struct task_struct *task, 2899 struct task_struct *next_prev, bool sched_in); 2900 2901 #define for_each_task_context_nr(ctxn) \ 2902 for ((ctxn) = 0; (ctxn) < perf_nr_task_contexts; (ctxn)++) 2903 2904 /* 2905 * Called from scheduler to remove the events of the current task, 2906 * with interrupts disabled. 2907 * 2908 * We stop each event and update the event value in event->count. 2909 * 2910 * This does not protect us against NMI, but disable() 2911 * sets the disabled bit in the control field of event _before_ 2912 * accessing the event control register. If a NMI hits, then it will 2913 * not restart the event. 2914 */ 2915 void __perf_event_task_sched_out(struct task_struct *task, 2916 struct task_struct *next) 2917 { 2918 int ctxn; 2919 2920 if (__this_cpu_read(perf_sched_cb_usages)) 2921 perf_pmu_sched_task(task, next, false); 2922 2923 if (atomic_read(&nr_switch_events)) 2924 perf_event_switch(task, next, false); 2925 2926 for_each_task_context_nr(ctxn) 2927 perf_event_context_sched_out(task, ctxn, next); 2928 2929 /* 2930 * if cgroup events exist on this CPU, then we need 2931 * to check if we have to switch out PMU state. 2932 * cgroup event are system-wide mode only 2933 */ 2934 if (atomic_read(this_cpu_ptr(&perf_cgroup_events))) 2935 perf_cgroup_sched_out(task, next); 2936 } 2937 2938 /* 2939 * Called with IRQs disabled 2940 */ 2941 static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx, 2942 enum event_type_t event_type) 2943 { 2944 ctx_sched_out(&cpuctx->ctx, cpuctx, event_type); 2945 } 2946 2947 static void 2948 ctx_pinned_sched_in(struct perf_event_context *ctx, 2949 struct perf_cpu_context *cpuctx) 2950 { 2951 struct perf_event *event; 2952 2953 list_for_each_entry(event, &ctx->pinned_groups, group_entry) { 2954 if (event->state <= PERF_EVENT_STATE_OFF) 2955 continue; 2956 if (!event_filter_match(event)) 2957 continue; 2958 2959 /* may need to reset tstamp_enabled */ 2960 if (is_cgroup_event(event)) 2961 perf_cgroup_mark_enabled(event, ctx); 2962 2963 if (group_can_go_on(event, cpuctx, 1)) 2964 group_sched_in(event, cpuctx, ctx); 2965 2966 /* 2967 * If this pinned group hasn't been scheduled, 2968 * put it in error state. 2969 */ 2970 if (event->state == PERF_EVENT_STATE_INACTIVE) { 2971 update_group_times(event); 2972 event->state = PERF_EVENT_STATE_ERROR; 2973 } 2974 } 2975 } 2976 2977 static void 2978 ctx_flexible_sched_in(struct perf_event_context *ctx, 2979 struct perf_cpu_context *cpuctx) 2980 { 2981 struct perf_event *event; 2982 int can_add_hw = 1; 2983 2984 list_for_each_entry(event, &ctx->flexible_groups, group_entry) { 2985 /* Ignore events in OFF or ERROR state */ 2986 if (event->state <= PERF_EVENT_STATE_OFF) 2987 continue; 2988 /* 2989 * Listen to the 'cpu' scheduling filter constraint 2990 * of events: 2991 */ 2992 if (!event_filter_match(event)) 2993 continue; 2994 2995 /* may need to reset tstamp_enabled */ 2996 if (is_cgroup_event(event)) 2997 perf_cgroup_mark_enabled(event, ctx); 2998 2999 if (group_can_go_on(event, cpuctx, can_add_hw)) { 3000 if (group_sched_in(event, cpuctx, ctx)) 3001 can_add_hw = 0; 3002 } 3003 } 3004 } 3005 3006 static void 3007 ctx_sched_in(struct perf_event_context *ctx, 3008 struct perf_cpu_context *cpuctx, 3009 enum event_type_t event_type, 3010 struct task_struct *task) 3011 { 3012 int is_active = ctx->is_active; 3013 u64 now; 3014 3015 lockdep_assert_held(&ctx->lock); 3016 3017 if (likely(!ctx->nr_events)) 3018 return; 3019 3020 ctx->is_active |= (event_type | EVENT_TIME); 3021 if (ctx->task) { 3022 if (!is_active) 3023 cpuctx->task_ctx = ctx; 3024 else 3025 WARN_ON_ONCE(cpuctx->task_ctx != ctx); 3026 } 3027 3028 is_active ^= ctx->is_active; /* changed bits */ 3029 3030 if (is_active & EVENT_TIME) { 3031 /* start ctx time */ 3032 now = perf_clock(); 3033 ctx->timestamp = now; 3034 perf_cgroup_set_timestamp(task, ctx); 3035 } 3036 3037 /* 3038 * First go through the list and put on any pinned groups 3039 * in order to give them the best chance of going on. 3040 */ 3041 if (is_active & EVENT_PINNED) 3042 ctx_pinned_sched_in(ctx, cpuctx); 3043 3044 /* Then walk through the lower prio flexible groups */ 3045 if (is_active & EVENT_FLEXIBLE) 3046 ctx_flexible_sched_in(ctx, cpuctx); 3047 } 3048 3049 static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx, 3050 enum event_type_t event_type, 3051 struct task_struct *task) 3052 { 3053 struct perf_event_context *ctx = &cpuctx->ctx; 3054 3055 ctx_sched_in(ctx, cpuctx, event_type, task); 3056 } 3057 3058 static void perf_event_context_sched_in(struct perf_event_context *ctx, 3059 struct task_struct *task) 3060 { 3061 struct perf_cpu_context *cpuctx; 3062 3063 cpuctx = __get_cpu_context(ctx); 3064 if (cpuctx->task_ctx == ctx) 3065 return; 3066 3067 perf_ctx_lock(cpuctx, ctx); 3068 perf_pmu_disable(ctx->pmu); 3069 /* 3070 * We want to keep the following priority order: 3071 * cpu pinned (that don't need to move), task pinned, 3072 * cpu flexible, task flexible. 3073 */ 3074 cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE); 3075 perf_event_sched_in(cpuctx, ctx, task); 3076 perf_pmu_enable(ctx->pmu); 3077 perf_ctx_unlock(cpuctx, ctx); 3078 } 3079 3080 /* 3081 * Called from scheduler to add the events of the current task 3082 * with interrupts disabled. 3083 * 3084 * We restore the event value and then enable it. 3085 * 3086 * This does not protect us against NMI, but enable() 3087 * sets the enabled bit in the control field of event _before_ 3088 * accessing the event control register. If a NMI hits, then it will 3089 * keep the event running. 3090 */ 3091 void __perf_event_task_sched_in(struct task_struct *prev, 3092 struct task_struct *task) 3093 { 3094 struct perf_event_context *ctx; 3095 int ctxn; 3096 3097 /* 3098 * If cgroup events exist on this CPU, then we need to check if we have 3099 * to switch in PMU state; cgroup event are system-wide mode only. 3100 * 3101 * Since cgroup events are CPU events, we must schedule these in before 3102 * we schedule in the task events. 3103 */ 3104 if (atomic_read(this_cpu_ptr(&perf_cgroup_events))) 3105 perf_cgroup_sched_in(prev, task); 3106 3107 for_each_task_context_nr(ctxn) { 3108 ctx = task->perf_event_ctxp[ctxn]; 3109 if (likely(!ctx)) 3110 continue; 3111 3112 perf_event_context_sched_in(ctx, task); 3113 } 3114 3115 if (atomic_read(&nr_switch_events)) 3116 perf_event_switch(task, prev, true); 3117 3118 if (__this_cpu_read(perf_sched_cb_usages)) 3119 perf_pmu_sched_task(prev, task, true); 3120 } 3121 3122 static u64 perf_calculate_period(struct perf_event *event, u64 nsec, u64 count) 3123 { 3124 u64 frequency = event->attr.sample_freq; 3125 u64 sec = NSEC_PER_SEC; 3126 u64 divisor, dividend; 3127 3128 int count_fls, nsec_fls, frequency_fls, sec_fls; 3129 3130 count_fls = fls64(count); 3131 nsec_fls = fls64(nsec); 3132 frequency_fls = fls64(frequency); 3133 sec_fls = 30; 3134 3135 /* 3136 * We got @count in @nsec, with a target of sample_freq HZ 3137 * the target period becomes: 3138 * 3139 * @count * 10^9 3140 * period = ------------------- 3141 * @nsec * sample_freq 3142 * 3143 */ 3144 3145 /* 3146 * Reduce accuracy by one bit such that @a and @b converge 3147 * to a similar magnitude. 3148 */ 3149 #define REDUCE_FLS(a, b) \ 3150 do { \ 3151 if (a##_fls > b##_fls) { \ 3152 a >>= 1; \ 3153 a##_fls--; \ 3154 } else { \ 3155 b >>= 1; \ 3156 b##_fls--; \ 3157 } \ 3158 } while (0) 3159 3160 /* 3161 * Reduce accuracy until either term fits in a u64, then proceed with 3162 * the other, so that finally we can do a u64/u64 division. 3163 */ 3164 while (count_fls + sec_fls > 64 && nsec_fls + frequency_fls > 64) { 3165 REDUCE_FLS(nsec, frequency); 3166 REDUCE_FLS(sec, count); 3167 } 3168 3169 if (count_fls + sec_fls > 64) { 3170 divisor = nsec * frequency; 3171 3172 while (count_fls + sec_fls > 64) { 3173 REDUCE_FLS(count, sec); 3174 divisor >>= 1; 3175 } 3176 3177 dividend = count * sec; 3178 } else { 3179 dividend = count * sec; 3180 3181 while (nsec_fls + frequency_fls > 64) { 3182 REDUCE_FLS(nsec, frequency); 3183 dividend >>= 1; 3184 } 3185 3186 divisor = nsec * frequency; 3187 } 3188 3189 if (!divisor) 3190 return dividend; 3191 3192 return div64_u64(dividend, divisor); 3193 } 3194 3195 static DEFINE_PER_CPU(int, perf_throttled_count); 3196 static DEFINE_PER_CPU(u64, perf_throttled_seq); 3197 3198 static void perf_adjust_period(struct perf_event *event, u64 nsec, u64 count, bool disable) 3199 { 3200 struct hw_perf_event *hwc = &event->hw; 3201 s64 period, sample_period; 3202 s64 delta; 3203 3204 period = perf_calculate_period(event, nsec, count); 3205 3206 delta = (s64)(period - hwc->sample_period); 3207 delta = (delta + 7) / 8; /* low pass filter */ 3208 3209 sample_period = hwc->sample_period + delta; 3210 3211 if (!sample_period) 3212 sample_period = 1; 3213 3214 hwc->sample_period = sample_period; 3215 3216 if (local64_read(&hwc->period_left) > 8*sample_period) { 3217 if (disable) 3218 event->pmu->stop(event, PERF_EF_UPDATE); 3219 3220 local64_set(&hwc->period_left, 0); 3221 3222 if (disable) 3223 event->pmu->start(event, PERF_EF_RELOAD); 3224 } 3225 } 3226 3227 /* 3228 * combine freq adjustment with unthrottling to avoid two passes over the 3229 * events. At the same time, make sure, having freq events does not change 3230 * the rate of unthrottling as that would introduce bias. 3231 */ 3232 static void perf_adjust_freq_unthr_context(struct perf_event_context *ctx, 3233 int needs_unthr) 3234 { 3235 struct perf_event *event; 3236 struct hw_perf_event *hwc; 3237 u64 now, period = TICK_NSEC; 3238 s64 delta; 3239 3240 /* 3241 * only need to iterate over all events iff: 3242 * - context have events in frequency mode (needs freq adjust) 3243 * - there are events to unthrottle on this cpu 3244 */ 3245 if (!(ctx->nr_freq || needs_unthr)) 3246 return; 3247 3248 raw_spin_lock(&ctx->lock); 3249 perf_pmu_disable(ctx->pmu); 3250 3251 list_for_each_entry_rcu(event, &ctx->event_list, event_entry) { 3252 if (event->state != PERF_EVENT_STATE_ACTIVE) 3253 continue; 3254 3255 if (!event_filter_match(event)) 3256 continue; 3257 3258 perf_pmu_disable(event->pmu); 3259 3260 hwc = &event->hw; 3261 3262 if (hwc->interrupts == MAX_INTERRUPTS) { 3263 hwc->interrupts = 0; 3264 perf_log_throttle(event, 1); 3265 event->pmu->start(event, 0); 3266 } 3267 3268 if (!event->attr.freq || !event->attr.sample_freq) 3269 goto next; 3270 3271 /* 3272 * stop the event and update event->count 3273 */ 3274 event->pmu->stop(event, PERF_EF_UPDATE); 3275 3276 now = local64_read(&event->count); 3277 delta = now - hwc->freq_count_stamp; 3278 hwc->freq_count_stamp = now; 3279 3280 /* 3281 * restart the event 3282 * reload only if value has changed 3283 * we have stopped the event so tell that 3284 * to perf_adjust_period() to avoid stopping it 3285 * twice. 3286 */ 3287 if (delta > 0) 3288 perf_adjust_period(event, period, delta, false); 3289 3290 event->pmu->start(event, delta > 0 ? PERF_EF_RELOAD : 0); 3291 next: 3292 perf_pmu_enable(event->pmu); 3293 } 3294 3295 perf_pmu_enable(ctx->pmu); 3296 raw_spin_unlock(&ctx->lock); 3297 } 3298 3299 /* 3300 * Round-robin a context's events: 3301 */ 3302 static void rotate_ctx(struct perf_event_context *ctx) 3303 { 3304 /* 3305 * Rotate the first entry last of non-pinned groups. Rotation might be 3306 * disabled by the inheritance code. 3307 */ 3308 if (!ctx->rotate_disable) 3309 list_rotate_left(&ctx->flexible_groups); 3310 } 3311 3312 static int perf_rotate_context(struct perf_cpu_context *cpuctx) 3313 { 3314 struct perf_event_context *ctx = NULL; 3315 int rotate = 0; 3316 3317 if (cpuctx->ctx.nr_events) { 3318 if (cpuctx->ctx.nr_events != cpuctx->ctx.nr_active) 3319 rotate = 1; 3320 } 3321 3322 ctx = cpuctx->task_ctx; 3323 if (ctx && ctx->nr_events) { 3324 if (ctx->nr_events != ctx->nr_active) 3325 rotate = 1; 3326 } 3327 3328 if (!rotate) 3329 goto done; 3330 3331 perf_ctx_lock(cpuctx, cpuctx->task_ctx); 3332 perf_pmu_disable(cpuctx->ctx.pmu); 3333 3334 cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE); 3335 if (ctx) 3336 ctx_sched_out(ctx, cpuctx, EVENT_FLEXIBLE); 3337 3338 rotate_ctx(&cpuctx->ctx); 3339 if (ctx) 3340 rotate_ctx(ctx); 3341 3342 perf_event_sched_in(cpuctx, ctx, current); 3343 3344 perf_pmu_enable(cpuctx->ctx.pmu); 3345 perf_ctx_unlock(cpuctx, cpuctx->task_ctx); 3346 done: 3347 3348 return rotate; 3349 } 3350 3351 void perf_event_task_tick(void) 3352 { 3353 struct list_head *head = this_cpu_ptr(&active_ctx_list); 3354 struct perf_event_context *ctx, *tmp; 3355 int throttled; 3356 3357 WARN_ON(!irqs_disabled()); 3358 3359 __this_cpu_inc(perf_throttled_seq); 3360 throttled = __this_cpu_xchg(perf_throttled_count, 0); 3361 tick_dep_clear_cpu(smp_processor_id(), TICK_DEP_BIT_PERF_EVENTS); 3362 3363 list_for_each_entry_safe(ctx, tmp, head, active_ctx_list) 3364 perf_adjust_freq_unthr_context(ctx, throttled); 3365 } 3366 3367 static int event_enable_on_exec(struct perf_event *event, 3368 struct perf_event_context *ctx) 3369 { 3370 if (!event->attr.enable_on_exec) 3371 return 0; 3372 3373 event->attr.enable_on_exec = 0; 3374 if (event->state >= PERF_EVENT_STATE_INACTIVE) 3375 return 0; 3376 3377 __perf_event_mark_enabled(event); 3378 3379 return 1; 3380 } 3381 3382 /* 3383 * Enable all of a task's events that have been marked enable-on-exec. 3384 * This expects task == current. 3385 */ 3386 static void perf_event_enable_on_exec(int ctxn) 3387 { 3388 struct perf_event_context *ctx, *clone_ctx = NULL; 3389 struct perf_cpu_context *cpuctx; 3390 struct perf_event *event; 3391 unsigned long flags; 3392 int enabled = 0; 3393 3394 local_irq_save(flags); 3395 ctx = current->perf_event_ctxp[ctxn]; 3396 if (!ctx || !ctx->nr_events) 3397 goto out; 3398 3399 cpuctx = __get_cpu_context(ctx); 3400 perf_ctx_lock(cpuctx, ctx); 3401 ctx_sched_out(ctx, cpuctx, EVENT_TIME); 3402 list_for_each_entry(event, &ctx->event_list, event_entry) 3403 enabled |= event_enable_on_exec(event, ctx); 3404 3405 /* 3406 * Unclone and reschedule this context if we enabled any event. 3407 */ 3408 if (enabled) { 3409 clone_ctx = unclone_ctx(ctx); 3410 ctx_resched(cpuctx, ctx); 3411 } 3412 perf_ctx_unlock(cpuctx, ctx); 3413 3414 out: 3415 local_irq_restore(flags); 3416 3417 if (clone_ctx) 3418 put_ctx(clone_ctx); 3419 } 3420 3421 struct perf_read_data { 3422 struct perf_event *event; 3423 bool group; 3424 int ret; 3425 }; 3426 3427 static int find_cpu_to_read(struct perf_event *event, int local_cpu) 3428 { 3429 int event_cpu = event->oncpu; 3430 u16 local_pkg, event_pkg; 3431 3432 if (event->group_caps & PERF_EV_CAP_READ_ACTIVE_PKG) { 3433 event_pkg = topology_physical_package_id(event_cpu); 3434 local_pkg = topology_physical_package_id(local_cpu); 3435 3436 if (event_pkg == local_pkg) 3437 return local_cpu; 3438 } 3439 3440 return event_cpu; 3441 } 3442 3443 /* 3444 * Cross CPU call to read the hardware event 3445 */ 3446 static void __perf_event_read(void *info) 3447 { 3448 struct perf_read_data *data = info; 3449 struct perf_event *sub, *event = data->event; 3450 struct perf_event_context *ctx = event->ctx; 3451 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx); 3452 struct pmu *pmu = event->pmu; 3453 3454 /* 3455 * If this is a task context, we need to check whether it is 3456 * the current task context of this cpu. If not it has been 3457 * scheduled out before the smp call arrived. In that case 3458 * event->count would have been updated to a recent sample 3459 * when the event was scheduled out. 3460 */ 3461 if (ctx->task && cpuctx->task_ctx != ctx) 3462 return; 3463 3464 raw_spin_lock(&ctx->lock); 3465 if (ctx->is_active) { 3466 update_context_time(ctx); 3467 update_cgrp_time_from_event(event); 3468 } 3469 3470 update_event_times(event); 3471 if (event->state != PERF_EVENT_STATE_ACTIVE) 3472 goto unlock; 3473 3474 if (!data->group) { 3475 pmu->read(event); 3476 data->ret = 0; 3477 goto unlock; 3478 } 3479 3480 pmu->start_txn(pmu, PERF_PMU_TXN_READ); 3481 3482 pmu->read(event); 3483 3484 list_for_each_entry(sub, &event->sibling_list, group_entry) { 3485 update_event_times(sub); 3486 if (sub->state == PERF_EVENT_STATE_ACTIVE) { 3487 /* 3488 * Use sibling's PMU rather than @event's since 3489 * sibling could be on different (eg: software) PMU. 3490 */ 3491 sub->pmu->read(sub); 3492 } 3493 } 3494 3495 data->ret = pmu->commit_txn(pmu); 3496 3497 unlock: 3498 raw_spin_unlock(&ctx->lock); 3499 } 3500 3501 static inline u64 perf_event_count(struct perf_event *event) 3502 { 3503 if (event->pmu->count) 3504 return event->pmu->count(event); 3505 3506 return __perf_event_count(event); 3507 } 3508 3509 /* 3510 * NMI-safe method to read a local event, that is an event that 3511 * is: 3512 * - either for the current task, or for this CPU 3513 * - does not have inherit set, for inherited task events 3514 * will not be local and we cannot read them atomically 3515 * - must not have a pmu::count method 3516 */ 3517 u64 perf_event_read_local(struct perf_event *event) 3518 { 3519 unsigned long flags; 3520 u64 val; 3521 3522 /* 3523 * Disabling interrupts avoids all counter scheduling (context 3524 * switches, timer based rotation and IPIs). 3525 */ 3526 local_irq_save(flags); 3527 3528 /* If this is a per-task event, it must be for current */ 3529 WARN_ON_ONCE((event->attach_state & PERF_ATTACH_TASK) && 3530 event->hw.target != current); 3531 3532 /* If this is a per-CPU event, it must be for this CPU */ 3533 WARN_ON_ONCE(!(event->attach_state & PERF_ATTACH_TASK) && 3534 event->cpu != smp_processor_id()); 3535 3536 /* 3537 * It must not be an event with inherit set, we cannot read 3538 * all child counters from atomic context. 3539 */ 3540 WARN_ON_ONCE(event->attr.inherit); 3541 3542 /* 3543 * It must not have a pmu::count method, those are not 3544 * NMI safe. 3545 */ 3546 WARN_ON_ONCE(event->pmu->count); 3547 3548 /* 3549 * If the event is currently on this CPU, its either a per-task event, 3550 * or local to this CPU. Furthermore it means its ACTIVE (otherwise 3551 * oncpu == -1). 3552 */ 3553 if (event->oncpu == smp_processor_id()) 3554 event->pmu->read(event); 3555 3556 val = local64_read(&event->count); 3557 local_irq_restore(flags); 3558 3559 return val; 3560 } 3561 3562 static int perf_event_read(struct perf_event *event, bool group) 3563 { 3564 int ret = 0, cpu_to_read, local_cpu; 3565 3566 /* 3567 * If event is enabled and currently active on a CPU, update the 3568 * value in the event structure: 3569 */ 3570 if (event->state == PERF_EVENT_STATE_ACTIVE) { 3571 struct perf_read_data data = { 3572 .event = event, 3573 .group = group, 3574 .ret = 0, 3575 }; 3576 3577 local_cpu = get_cpu(); 3578 cpu_to_read = find_cpu_to_read(event, local_cpu); 3579 put_cpu(); 3580 3581 /* 3582 * Purposely ignore the smp_call_function_single() return 3583 * value. 3584 * 3585 * If event->oncpu isn't a valid CPU it means the event got 3586 * scheduled out and that will have updated the event count. 3587 * 3588 * Therefore, either way, we'll have an up-to-date event count 3589 * after this. 3590 */ 3591 (void)smp_call_function_single(cpu_to_read, __perf_event_read, &data, 1); 3592 ret = data.ret; 3593 } else if (event->state == PERF_EVENT_STATE_INACTIVE) { 3594 struct perf_event_context *ctx = event->ctx; 3595 unsigned long flags; 3596 3597 raw_spin_lock_irqsave(&ctx->lock, flags); 3598 /* 3599 * may read while context is not active 3600 * (e.g., thread is blocked), in that case 3601 * we cannot update context time 3602 */ 3603 if (ctx->is_active) { 3604 update_context_time(ctx); 3605 update_cgrp_time_from_event(event); 3606 } 3607 if (group) 3608 update_group_times(event); 3609 else 3610 update_event_times(event); 3611 raw_spin_unlock_irqrestore(&ctx->lock, flags); 3612 } 3613 3614 return ret; 3615 } 3616 3617 /* 3618 * Initialize the perf_event context in a task_struct: 3619 */ 3620 static void __perf_event_init_context(struct perf_event_context *ctx) 3621 { 3622 raw_spin_lock_init(&ctx->lock); 3623 mutex_init(&ctx->mutex); 3624 INIT_LIST_HEAD(&ctx->active_ctx_list); 3625 INIT_LIST_HEAD(&ctx->pinned_groups); 3626 INIT_LIST_HEAD(&ctx->flexible_groups); 3627 INIT_LIST_HEAD(&ctx->event_list); 3628 atomic_set(&ctx->refcount, 1); 3629 } 3630 3631 static struct perf_event_context * 3632 alloc_perf_context(struct pmu *pmu, struct task_struct *task) 3633 { 3634 struct perf_event_context *ctx; 3635 3636 ctx = kzalloc(sizeof(struct perf_event_context), GFP_KERNEL); 3637 if (!ctx) 3638 return NULL; 3639 3640 __perf_event_init_context(ctx); 3641 if (task) { 3642 ctx->task = task; 3643 get_task_struct(task); 3644 } 3645 ctx->pmu = pmu; 3646 3647 return ctx; 3648 } 3649 3650 static struct task_struct * 3651 find_lively_task_by_vpid(pid_t vpid) 3652 { 3653 struct task_struct *task; 3654 3655 rcu_read_lock(); 3656 if (!vpid) 3657 task = current; 3658 else 3659 task = find_task_by_vpid(vpid); 3660 if (task) 3661 get_task_struct(task); 3662 rcu_read_unlock(); 3663 3664 if (!task) 3665 return ERR_PTR(-ESRCH); 3666 3667 return task; 3668 } 3669 3670 /* 3671 * Returns a matching context with refcount and pincount. 3672 */ 3673 static struct perf_event_context * 3674 find_get_context(struct pmu *pmu, struct task_struct *task, 3675 struct perf_event *event) 3676 { 3677 struct perf_event_context *ctx, *clone_ctx = NULL; 3678 struct perf_cpu_context *cpuctx; 3679 void *task_ctx_data = NULL; 3680 unsigned long flags; 3681 int ctxn, err; 3682 int cpu = event->cpu; 3683 3684 if (!task) { 3685 /* Must be root to operate on a CPU event: */ 3686 if (perf_paranoid_cpu() && !capable(CAP_SYS_ADMIN)) 3687 return ERR_PTR(-EACCES); 3688 3689 /* 3690 * We could be clever and allow to attach a event to an 3691 * offline CPU and activate it when the CPU comes up, but 3692 * that's for later. 3693 */ 3694 if (!cpu_online(cpu)) 3695 return ERR_PTR(-ENODEV); 3696 3697 cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu); 3698 ctx = &cpuctx->ctx; 3699 get_ctx(ctx); 3700 ++ctx->pin_count; 3701 3702 return ctx; 3703 } 3704 3705 err = -EINVAL; 3706 ctxn = pmu->task_ctx_nr; 3707 if (ctxn < 0) 3708 goto errout; 3709 3710 if (event->attach_state & PERF_ATTACH_TASK_DATA) { 3711 task_ctx_data = kzalloc(pmu->task_ctx_size, GFP_KERNEL); 3712 if (!task_ctx_data) { 3713 err = -ENOMEM; 3714 goto errout; 3715 } 3716 } 3717 3718 retry: 3719 ctx = perf_lock_task_context(task, ctxn, &flags); 3720 if (ctx) { 3721 clone_ctx = unclone_ctx(ctx); 3722 ++ctx->pin_count; 3723 3724 if (task_ctx_data && !ctx->task_ctx_data) { 3725 ctx->task_ctx_data = task_ctx_data; 3726 task_ctx_data = NULL; 3727 } 3728 raw_spin_unlock_irqrestore(&ctx->lock, flags); 3729 3730 if (clone_ctx) 3731 put_ctx(clone_ctx); 3732 } else { 3733 ctx = alloc_perf_context(pmu, task); 3734 err = -ENOMEM; 3735 if (!ctx) 3736 goto errout; 3737 3738 if (task_ctx_data) { 3739 ctx->task_ctx_data = task_ctx_data; 3740 task_ctx_data = NULL; 3741 } 3742 3743 err = 0; 3744 mutex_lock(&task->perf_event_mutex); 3745 /* 3746 * If it has already passed perf_event_exit_task(). 3747 * we must see PF_EXITING, it takes this mutex too. 3748 */ 3749 if (task->flags & PF_EXITING) 3750 err = -ESRCH; 3751 else if (task->perf_event_ctxp[ctxn]) 3752 err = -EAGAIN; 3753 else { 3754 get_ctx(ctx); 3755 ++ctx->pin_count; 3756 rcu_assign_pointer(task->perf_event_ctxp[ctxn], ctx); 3757 } 3758 mutex_unlock(&task->perf_event_mutex); 3759 3760 if (unlikely(err)) { 3761 put_ctx(ctx); 3762 3763 if (err == -EAGAIN) 3764 goto retry; 3765 goto errout; 3766 } 3767 } 3768 3769 kfree(task_ctx_data); 3770 return ctx; 3771 3772 errout: 3773 kfree(task_ctx_data); 3774 return ERR_PTR(err); 3775 } 3776 3777 static void perf_event_free_filter(struct perf_event *event); 3778 static void perf_event_free_bpf_prog(struct perf_event *event); 3779 3780 static void free_event_rcu(struct rcu_head *head) 3781 { 3782 struct perf_event *event; 3783 3784 event = container_of(head, struct perf_event, rcu_head); 3785 if (event->ns) 3786 put_pid_ns(event->ns); 3787 perf_event_free_filter(event); 3788 kfree(event); 3789 } 3790 3791 static void ring_buffer_attach(struct perf_event *event, 3792 struct ring_buffer *rb); 3793 3794 static void detach_sb_event(struct perf_event *event) 3795 { 3796 struct pmu_event_list *pel = per_cpu_ptr(&pmu_sb_events, event->cpu); 3797 3798 raw_spin_lock(&pel->lock); 3799 list_del_rcu(&event->sb_list); 3800 raw_spin_unlock(&pel->lock); 3801 } 3802 3803 static bool is_sb_event(struct perf_event *event) 3804 { 3805 struct perf_event_attr *attr = &event->attr; 3806 3807 if (event->parent) 3808 return false; 3809 3810 if (event->attach_state & PERF_ATTACH_TASK) 3811 return false; 3812 3813 if (attr->mmap || attr->mmap_data || attr->mmap2 || 3814 attr->comm || attr->comm_exec || 3815 attr->task || 3816 attr->context_switch) 3817 return true; 3818 return false; 3819 } 3820 3821 static void unaccount_pmu_sb_event(struct perf_event *event) 3822 { 3823 if (is_sb_event(event)) 3824 detach_sb_event(event); 3825 } 3826 3827 static void unaccount_event_cpu(struct perf_event *event, int cpu) 3828 { 3829 if (event->parent) 3830 return; 3831 3832 if (is_cgroup_event(event)) 3833 atomic_dec(&per_cpu(perf_cgroup_events, cpu)); 3834 } 3835 3836 #ifdef CONFIG_NO_HZ_FULL 3837 static DEFINE_SPINLOCK(nr_freq_lock); 3838 #endif 3839 3840 static void unaccount_freq_event_nohz(void) 3841 { 3842 #ifdef CONFIG_NO_HZ_FULL 3843 spin_lock(&nr_freq_lock); 3844 if (atomic_dec_and_test(&nr_freq_events)) 3845 tick_nohz_dep_clear(TICK_DEP_BIT_PERF_EVENTS); 3846 spin_unlock(&nr_freq_lock); 3847 #endif 3848 } 3849 3850 static void unaccount_freq_event(void) 3851 { 3852 if (tick_nohz_full_enabled()) 3853 unaccount_freq_event_nohz(); 3854 else 3855 atomic_dec(&nr_freq_events); 3856 } 3857 3858 static void unaccount_event(struct perf_event *event) 3859 { 3860 bool dec = false; 3861 3862 if (event->parent) 3863 return; 3864 3865 if (event->attach_state & PERF_ATTACH_TASK) 3866 dec = true; 3867 if (event->attr.mmap || event->attr.mmap_data) 3868 atomic_dec(&nr_mmap_events); 3869 if (event->attr.comm) 3870 atomic_dec(&nr_comm_events); 3871 if (event->attr.task) 3872 atomic_dec(&nr_task_events); 3873 if (event->attr.freq) 3874 unaccount_freq_event(); 3875 if (event->attr.context_switch) { 3876 dec = true; 3877 atomic_dec(&nr_switch_events); 3878 } 3879 if (is_cgroup_event(event)) 3880 dec = true; 3881 if (has_branch_stack(event)) 3882 dec = true; 3883 3884 if (dec) { 3885 if (!atomic_add_unless(&perf_sched_count, -1, 1)) 3886 schedule_delayed_work(&perf_sched_work, HZ); 3887 } 3888 3889 unaccount_event_cpu(event, event->cpu); 3890 3891 unaccount_pmu_sb_event(event); 3892 } 3893 3894 static void perf_sched_delayed(struct work_struct *work) 3895 { 3896 mutex_lock(&perf_sched_mutex); 3897 if (atomic_dec_and_test(&perf_sched_count)) 3898 static_branch_disable(&perf_sched_events); 3899 mutex_unlock(&perf_sched_mutex); 3900 } 3901 3902 /* 3903 * The following implement mutual exclusion of events on "exclusive" pmus 3904 * (PERF_PMU_CAP_EXCLUSIVE). Such pmus can only have one event scheduled 3905 * at a time, so we disallow creating events that might conflict, namely: 3906 * 3907 * 1) cpu-wide events in the presence of per-task events, 3908 * 2) per-task events in the presence of cpu-wide events, 3909 * 3) two matching events on the same context. 3910 * 3911 * The former two cases are handled in the allocation path (perf_event_alloc(), 3912 * _free_event()), the latter -- before the first perf_install_in_context(). 3913 */ 3914 static int exclusive_event_init(struct perf_event *event) 3915 { 3916 struct pmu *pmu = event->pmu; 3917 3918 if (!(pmu->capabilities & PERF_PMU_CAP_EXCLUSIVE)) 3919 return 0; 3920 3921 /* 3922 * Prevent co-existence of per-task and cpu-wide events on the 3923 * same exclusive pmu. 3924 * 3925 * Negative pmu::exclusive_cnt means there are cpu-wide 3926 * events on this "exclusive" pmu, positive means there are 3927 * per-task events. 3928 * 3929 * Since this is called in perf_event_alloc() path, event::ctx 3930 * doesn't exist yet; it is, however, safe to use PERF_ATTACH_TASK 3931 * to mean "per-task event", because unlike other attach states it 3932 * never gets cleared. 3933 */ 3934 if (event->attach_state & PERF_ATTACH_TASK) { 3935 if (!atomic_inc_unless_negative(&pmu->exclusive_cnt)) 3936 return -EBUSY; 3937 } else { 3938 if (!atomic_dec_unless_positive(&pmu->exclusive_cnt)) 3939 return -EBUSY; 3940 } 3941 3942 return 0; 3943 } 3944 3945 static void exclusive_event_destroy(struct perf_event *event) 3946 { 3947 struct pmu *pmu = event->pmu; 3948 3949 if (!(pmu->capabilities & PERF_PMU_CAP_EXCLUSIVE)) 3950 return; 3951 3952 /* see comment in exclusive_event_init() */ 3953 if (event->attach_state & PERF_ATTACH_TASK) 3954 atomic_dec(&pmu->exclusive_cnt); 3955 else 3956 atomic_inc(&pmu->exclusive_cnt); 3957 } 3958 3959 static bool exclusive_event_match(struct perf_event *e1, struct perf_event *e2) 3960 { 3961 if ((e1->pmu == e2->pmu) && 3962 (e1->cpu == e2->cpu || 3963 e1->cpu == -1 || 3964 e2->cpu == -1)) 3965 return true; 3966 return false; 3967 } 3968 3969 /* Called under the same ctx::mutex as perf_install_in_context() */ 3970 static bool exclusive_event_installable(struct perf_event *event, 3971 struct perf_event_context *ctx) 3972 { 3973 struct perf_event *iter_event; 3974 struct pmu *pmu = event->pmu; 3975 3976 if (!(pmu->capabilities & PERF_PMU_CAP_EXCLUSIVE)) 3977 return true; 3978 3979 list_for_each_entry(iter_event, &ctx->event_list, event_entry) { 3980 if (exclusive_event_match(iter_event, event)) 3981 return false; 3982 } 3983 3984 return true; 3985 } 3986 3987 static void perf_addr_filters_splice(struct perf_event *event, 3988 struct list_head *head); 3989 3990 static void _free_event(struct perf_event *event) 3991 { 3992 irq_work_sync(&event->pending); 3993 3994 unaccount_event(event); 3995 3996 if (event->rb) { 3997 /* 3998 * Can happen when we close an event with re-directed output. 3999 * 4000 * Since we have a 0 refcount, perf_mmap_close() will skip 4001 * over us; possibly making our ring_buffer_put() the last. 4002 */ 4003 mutex_lock(&event->mmap_mutex); 4004 ring_buffer_attach(event, NULL); 4005 mutex_unlock(&event->mmap_mutex); 4006 } 4007 4008 if (is_cgroup_event(event)) 4009 perf_detach_cgroup(event); 4010 4011 if (!event->parent) { 4012 if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) 4013 put_callchain_buffers(); 4014 } 4015 4016 perf_event_free_bpf_prog(event); 4017 perf_addr_filters_splice(event, NULL); 4018 kfree(event->addr_filters_offs); 4019 4020 if (event->destroy) 4021 event->destroy(event); 4022 4023 if (event->ctx) 4024 put_ctx(event->ctx); 4025 4026 exclusive_event_destroy(event); 4027 module_put(event->pmu->module); 4028 4029 call_rcu(&event->rcu_head, free_event_rcu); 4030 } 4031 4032 /* 4033 * Used to free events which have a known refcount of 1, such as in error paths 4034 * where the event isn't exposed yet and inherited events. 4035 */ 4036 static void free_event(struct perf_event *event) 4037 { 4038 if (WARN(atomic_long_cmpxchg(&event->refcount, 1, 0) != 1, 4039 "unexpected event refcount: %ld; ptr=%p\n", 4040 atomic_long_read(&event->refcount), event)) { 4041 /* leak to avoid use-after-free */ 4042 return; 4043 } 4044 4045 _free_event(event); 4046 } 4047 4048 /* 4049 * Remove user event from the owner task. 4050 */ 4051 static void perf_remove_from_owner(struct perf_event *event) 4052 { 4053 struct task_struct *owner; 4054 4055 rcu_read_lock(); 4056 /* 4057 * Matches the smp_store_release() in perf_event_exit_task(). If we 4058 * observe !owner it means the list deletion is complete and we can 4059 * indeed free this event, otherwise we need to serialize on 4060 * owner->perf_event_mutex. 4061 */ 4062 owner = lockless_dereference(event->owner); 4063 if (owner) { 4064 /* 4065 * Since delayed_put_task_struct() also drops the last 4066 * task reference we can safely take a new reference 4067 * while holding the rcu_read_lock(). 4068 */ 4069 get_task_struct(owner); 4070 } 4071 rcu_read_unlock(); 4072 4073 if (owner) { 4074 /* 4075 * If we're here through perf_event_exit_task() we're already 4076 * holding ctx->mutex which would be an inversion wrt. the 4077 * normal lock order. 4078 * 4079 * However we can safely take this lock because its the child 4080 * ctx->mutex. 4081 */ 4082 mutex_lock_nested(&owner->perf_event_mutex, SINGLE_DEPTH_NESTING); 4083 4084 /* 4085 * We have to re-check the event->owner field, if it is cleared 4086 * we raced with perf_event_exit_task(), acquiring the mutex 4087 * ensured they're done, and we can proceed with freeing the 4088 * event. 4089 */ 4090 if (event->owner) { 4091 list_del_init(&event->owner_entry); 4092 smp_store_release(&event->owner, NULL); 4093 } 4094 mutex_unlock(&owner->perf_event_mutex); 4095 put_task_struct(owner); 4096 } 4097 } 4098 4099 static void put_event(struct perf_event *event) 4100 { 4101 if (!atomic_long_dec_and_test(&event->refcount)) 4102 return; 4103 4104 _free_event(event); 4105 } 4106 4107 /* 4108 * Kill an event dead; while event:refcount will preserve the event 4109 * object, it will not preserve its functionality. Once the last 'user' 4110 * gives up the object, we'll destroy the thing. 4111 */ 4112 int perf_event_release_kernel(struct perf_event *event) 4113 { 4114 struct perf_event_context *ctx = event->ctx; 4115 struct perf_event *child, *tmp; 4116 4117 /* 4118 * If we got here through err_file: fput(event_file); we will not have 4119 * attached to a context yet. 4120 */ 4121 if (!ctx) { 4122 WARN_ON_ONCE(event->attach_state & 4123 (PERF_ATTACH_CONTEXT|PERF_ATTACH_GROUP)); 4124 goto no_ctx; 4125 } 4126 4127 if (!is_kernel_event(event)) 4128 perf_remove_from_owner(event); 4129 4130 ctx = perf_event_ctx_lock(event); 4131 WARN_ON_ONCE(ctx->parent_ctx); 4132 perf_remove_from_context(event, DETACH_GROUP); 4133 4134 raw_spin_lock_irq(&ctx->lock); 4135 /* 4136 * Mark this even as STATE_DEAD, there is no external reference to it 4137 * anymore. 4138 * 4139 * Anybody acquiring event->child_mutex after the below loop _must_ 4140 * also see this, most importantly inherit_event() which will avoid 4141 * placing more children on the list. 4142 * 4143 * Thus this guarantees that we will in fact observe and kill _ALL_ 4144 * child events. 4145 */ 4146 event->state = PERF_EVENT_STATE_DEAD; 4147 raw_spin_unlock_irq(&ctx->lock); 4148 4149 perf_event_ctx_unlock(event, ctx); 4150 4151 again: 4152 mutex_lock(&event->child_mutex); 4153 list_for_each_entry(child, &event->child_list, child_list) { 4154 4155 /* 4156 * Cannot change, child events are not migrated, see the 4157 * comment with perf_event_ctx_lock_nested(). 4158 */ 4159 ctx = lockless_dereference(child->ctx); 4160 /* 4161 * Since child_mutex nests inside ctx::mutex, we must jump 4162 * through hoops. We start by grabbing a reference on the ctx. 4163 * 4164 * Since the event cannot get freed while we hold the 4165 * child_mutex, the context must also exist and have a !0 4166 * reference count. 4167 */ 4168 get_ctx(ctx); 4169 4170 /* 4171 * Now that we have a ctx ref, we can drop child_mutex, and 4172 * acquire ctx::mutex without fear of it going away. Then we 4173 * can re-acquire child_mutex. 4174 */ 4175 mutex_unlock(&event->child_mutex); 4176 mutex_lock(&ctx->mutex); 4177 mutex_lock(&event->child_mutex); 4178 4179 /* 4180 * Now that we hold ctx::mutex and child_mutex, revalidate our 4181 * state, if child is still the first entry, it didn't get freed 4182 * and we can continue doing so. 4183 */ 4184 tmp = list_first_entry_or_null(&event->child_list, 4185 struct perf_event, child_list); 4186 if (tmp == child) { 4187 perf_remove_from_context(child, DETACH_GROUP); 4188 list_del(&child->child_list); 4189 free_event(child); 4190 /* 4191 * This matches the refcount bump in inherit_event(); 4192 * this can't be the last reference. 4193 */ 4194 put_event(event); 4195 } 4196 4197 mutex_unlock(&event->child_mutex); 4198 mutex_unlock(&ctx->mutex); 4199 put_ctx(ctx); 4200 goto again; 4201 } 4202 mutex_unlock(&event->child_mutex); 4203 4204 no_ctx: 4205 put_event(event); /* Must be the 'last' reference */ 4206 return 0; 4207 } 4208 EXPORT_SYMBOL_GPL(perf_event_release_kernel); 4209 4210 /* 4211 * Called when the last reference to the file is gone. 4212 */ 4213 static int perf_release(struct inode *inode, struct file *file) 4214 { 4215 perf_event_release_kernel(file->private_data); 4216 return 0; 4217 } 4218 4219 u64 perf_event_read_value(struct perf_event *event, u64 *enabled, u64 *running) 4220 { 4221 struct perf_event *child; 4222 u64 total = 0; 4223 4224 *enabled = 0; 4225 *running = 0; 4226 4227 mutex_lock(&event->child_mutex); 4228 4229 (void)perf_event_read(event, false); 4230 total += perf_event_count(event); 4231 4232 *enabled += event->total_time_enabled + 4233 atomic64_read(&event->child_total_time_enabled); 4234 *running += event->total_time_running + 4235 atomic64_read(&event->child_total_time_running); 4236 4237 list_for_each_entry(child, &event->child_list, child_list) { 4238 (void)perf_event_read(child, false); 4239 total += perf_event_count(child); 4240 *enabled += child->total_time_enabled; 4241 *running += child->total_time_running; 4242 } 4243 mutex_unlock(&event->child_mutex); 4244 4245 return total; 4246 } 4247 EXPORT_SYMBOL_GPL(perf_event_read_value); 4248 4249 static int __perf_read_group_add(struct perf_event *leader, 4250 u64 read_format, u64 *values) 4251 { 4252 struct perf_event *sub; 4253 int n = 1; /* skip @nr */ 4254 int ret; 4255 4256 ret = perf_event_read(leader, true); 4257 if (ret) 4258 return ret; 4259 4260 /* 4261 * Since we co-schedule groups, {enabled,running} times of siblings 4262 * will be identical to those of the leader, so we only publish one 4263 * set. 4264 */ 4265 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) { 4266 values[n++] += leader->total_time_enabled + 4267 atomic64_read(&leader->child_total_time_enabled); 4268 } 4269 4270 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) { 4271 values[n++] += leader->total_time_running + 4272 atomic64_read(&leader->child_total_time_running); 4273 } 4274 4275 /* 4276 * Write {count,id} tuples for every sibling. 4277 */ 4278 values[n++] += perf_event_count(leader); 4279 if (read_format & PERF_FORMAT_ID) 4280 values[n++] = primary_event_id(leader); 4281 4282 list_for_each_entry(sub, &leader->sibling_list, group_entry) { 4283 values[n++] += perf_event_count(sub); 4284 if (read_format & PERF_FORMAT_ID) 4285 values[n++] = primary_event_id(sub); 4286 } 4287 4288 return 0; 4289 } 4290 4291 static int perf_read_group(struct perf_event *event, 4292 u64 read_format, char __user *buf) 4293 { 4294 struct perf_event *leader = event->group_leader, *child; 4295 struct perf_event_context *ctx = leader->ctx; 4296 int ret; 4297 u64 *values; 4298 4299 lockdep_assert_held(&ctx->mutex); 4300 4301 values = kzalloc(event->read_size, GFP_KERNEL); 4302 if (!values) 4303 return -ENOMEM; 4304 4305 values[0] = 1 + leader->nr_siblings; 4306 4307 /* 4308 * By locking the child_mutex of the leader we effectively 4309 * lock the child list of all siblings.. XXX explain how. 4310 */ 4311 mutex_lock(&leader->child_mutex); 4312 4313 ret = __perf_read_group_add(leader, read_format, values); 4314 if (ret) 4315 goto unlock; 4316 4317 list_for_each_entry(child, &leader->child_list, child_list) { 4318 ret = __perf_read_group_add(child, read_format, values); 4319 if (ret) 4320 goto unlock; 4321 } 4322 4323 mutex_unlock(&leader->child_mutex); 4324 4325 ret = event->read_size; 4326 if (copy_to_user(buf, values, event->read_size)) 4327 ret = -EFAULT; 4328 goto out; 4329 4330 unlock: 4331 mutex_unlock(&leader->child_mutex); 4332 out: 4333 kfree(values); 4334 return ret; 4335 } 4336 4337 static int perf_read_one(struct perf_event *event, 4338 u64 read_format, char __user *buf) 4339 { 4340 u64 enabled, running; 4341 u64 values[4]; 4342 int n = 0; 4343 4344 values[n++] = perf_event_read_value(event, &enabled, &running); 4345 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) 4346 values[n++] = enabled; 4347 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) 4348 values[n++] = running; 4349 if (read_format & PERF_FORMAT_ID) 4350 values[n++] = primary_event_id(event); 4351 4352 if (copy_to_user(buf, values, n * sizeof(u64))) 4353 return -EFAULT; 4354 4355 return n * sizeof(u64); 4356 } 4357 4358 static bool is_event_hup(struct perf_event *event) 4359 { 4360 bool no_children; 4361 4362 if (event->state > PERF_EVENT_STATE_EXIT) 4363 return false; 4364 4365 mutex_lock(&event->child_mutex); 4366 no_children = list_empty(&event->child_list); 4367 mutex_unlock(&event->child_mutex); 4368 return no_children; 4369 } 4370 4371 /* 4372 * Read the performance event - simple non blocking version for now 4373 */ 4374 static ssize_t 4375 __perf_read(struct perf_event *event, char __user *buf, size_t count) 4376 { 4377 u64 read_format = event->attr.read_format; 4378 int ret; 4379 4380 /* 4381 * Return end-of-file for a read on a event that is in 4382 * error state (i.e. because it was pinned but it couldn't be 4383 * scheduled on to the CPU at some point). 4384 */ 4385 if (event->state == PERF_EVENT_STATE_ERROR) 4386 return 0; 4387 4388 if (count < event->read_size) 4389 return -ENOSPC; 4390 4391 WARN_ON_ONCE(event->ctx->parent_ctx); 4392 if (read_format & PERF_FORMAT_GROUP) 4393 ret = perf_read_group(event, read_format, buf); 4394 else 4395 ret = perf_read_one(event, read_format, buf); 4396 4397 return ret; 4398 } 4399 4400 static ssize_t 4401 perf_read(struct file *file, char __user *buf, size_t count, loff_t *ppos) 4402 { 4403 struct perf_event *event = file->private_data; 4404 struct perf_event_context *ctx; 4405 int ret; 4406 4407 ctx = perf_event_ctx_lock(event); 4408 ret = __perf_read(event, buf, count); 4409 perf_event_ctx_unlock(event, ctx); 4410 4411 return ret; 4412 } 4413 4414 static unsigned int perf_poll(struct file *file, poll_table *wait) 4415 { 4416 struct perf_event *event = file->private_data; 4417 struct ring_buffer *rb; 4418 unsigned int events = POLLHUP; 4419 4420 poll_wait(file, &event->waitq, wait); 4421 4422 if (is_event_hup(event)) 4423 return events; 4424 4425 /* 4426 * Pin the event->rb by taking event->mmap_mutex; otherwise 4427 * perf_event_set_output() can swizzle our rb and make us miss wakeups. 4428 */ 4429 mutex_lock(&event->mmap_mutex); 4430 rb = event->rb; 4431 if (rb) 4432 events = atomic_xchg(&rb->poll, 0); 4433 mutex_unlock(&event->mmap_mutex); 4434 return events; 4435 } 4436 4437 static void _perf_event_reset(struct perf_event *event) 4438 { 4439 (void)perf_event_read(event, false); 4440 local64_set(&event->count, 0); 4441 perf_event_update_userpage(event); 4442 } 4443 4444 /* 4445 * Holding the top-level event's child_mutex means that any 4446 * descendant process that has inherited this event will block 4447 * in perf_event_exit_event() if it goes to exit, thus satisfying the 4448 * task existence requirements of perf_event_enable/disable. 4449 */ 4450 static void perf_event_for_each_child(struct perf_event *event, 4451 void (*func)(struct perf_event *)) 4452 { 4453 struct perf_event *child; 4454 4455 WARN_ON_ONCE(event->ctx->parent_ctx); 4456 4457 mutex_lock(&event->child_mutex); 4458 func(event); 4459 list_for_each_entry(child, &event->child_list, child_list) 4460 func(child); 4461 mutex_unlock(&event->child_mutex); 4462 } 4463 4464 static void perf_event_for_each(struct perf_event *event, 4465 void (*func)(struct perf_event *)) 4466 { 4467 struct perf_event_context *ctx = event->ctx; 4468 struct perf_event *sibling; 4469 4470 lockdep_assert_held(&ctx->mutex); 4471 4472 event = event->group_leader; 4473 4474 perf_event_for_each_child(event, func); 4475 list_for_each_entry(sibling, &event->sibling_list, group_entry) 4476 perf_event_for_each_child(sibling, func); 4477 } 4478 4479 static void __perf_event_period(struct perf_event *event, 4480 struct perf_cpu_context *cpuctx, 4481 struct perf_event_context *ctx, 4482 void *info) 4483 { 4484 u64 value = *((u64 *)info); 4485 bool active; 4486 4487 if (event->attr.freq) { 4488 event->attr.sample_freq = value; 4489 } else { 4490 event->attr.sample_period = value; 4491 event->hw.sample_period = value; 4492 } 4493 4494 active = (event->state == PERF_EVENT_STATE_ACTIVE); 4495 if (active) { 4496 perf_pmu_disable(ctx->pmu); 4497 /* 4498 * We could be throttled; unthrottle now to avoid the tick 4499 * trying to unthrottle while we already re-started the event. 4500 */ 4501 if (event->hw.interrupts == MAX_INTERRUPTS) { 4502 event->hw.interrupts = 0; 4503 perf_log_throttle(event, 1); 4504 } 4505 event->pmu->stop(event, PERF_EF_UPDATE); 4506 } 4507 4508 local64_set(&event->hw.period_left, 0); 4509 4510 if (active) { 4511 event->pmu->start(event, PERF_EF_RELOAD); 4512 perf_pmu_enable(ctx->pmu); 4513 } 4514 } 4515 4516 static int perf_event_period(struct perf_event *event, u64 __user *arg) 4517 { 4518 u64 value; 4519 4520 if (!is_sampling_event(event)) 4521 return -EINVAL; 4522 4523 if (copy_from_user(&value, arg, sizeof(value))) 4524 return -EFAULT; 4525 4526 if (!value) 4527 return -EINVAL; 4528 4529 if (event->attr.freq && value > sysctl_perf_event_sample_rate) 4530 return -EINVAL; 4531 4532 event_function_call(event, __perf_event_period, &value); 4533 4534 return 0; 4535 } 4536 4537 static const struct file_operations perf_fops; 4538 4539 static inline int perf_fget_light(int fd, struct fd *p) 4540 { 4541 struct fd f = fdget(fd); 4542 if (!f.file) 4543 return -EBADF; 4544 4545 if (f.file->f_op != &perf_fops) { 4546 fdput(f); 4547 return -EBADF; 4548 } 4549 *p = f; 4550 return 0; 4551 } 4552 4553 static int perf_event_set_output(struct perf_event *event, 4554 struct perf_event *output_event); 4555 static int perf_event_set_filter(struct perf_event *event, void __user *arg); 4556 static int perf_event_set_bpf_prog(struct perf_event *event, u32 prog_fd); 4557 4558 static long _perf_ioctl(struct perf_event *event, unsigned int cmd, unsigned long arg) 4559 { 4560 void (*func)(struct perf_event *); 4561 u32 flags = arg; 4562 4563 switch (cmd) { 4564 case PERF_EVENT_IOC_ENABLE: 4565 func = _perf_event_enable; 4566 break; 4567 case PERF_EVENT_IOC_DISABLE: 4568 func = _perf_event_disable; 4569 break; 4570 case PERF_EVENT_IOC_RESET: 4571 func = _perf_event_reset; 4572 break; 4573 4574 case PERF_EVENT_IOC_REFRESH: 4575 return _perf_event_refresh(event, arg); 4576 4577 case PERF_EVENT_IOC_PERIOD: 4578 return perf_event_period(event, (u64 __user *)arg); 4579 4580 case PERF_EVENT_IOC_ID: 4581 { 4582 u64 id = primary_event_id(event); 4583 4584 if (copy_to_user((void __user *)arg, &id, sizeof(id))) 4585 return -EFAULT; 4586 return 0; 4587 } 4588 4589 case PERF_EVENT_IOC_SET_OUTPUT: 4590 { 4591 int ret; 4592 if (arg != -1) { 4593 struct perf_event *output_event; 4594 struct fd output; 4595 ret = perf_fget_light(arg, &output); 4596 if (ret) 4597 return ret; 4598 output_event = output.file->private_data; 4599 ret = perf_event_set_output(event, output_event); 4600 fdput(output); 4601 } else { 4602 ret = perf_event_set_output(event, NULL); 4603 } 4604 return ret; 4605 } 4606 4607 case PERF_EVENT_IOC_SET_FILTER: 4608 return perf_event_set_filter(event, (void __user *)arg); 4609 4610 case PERF_EVENT_IOC_SET_BPF: 4611 return perf_event_set_bpf_prog(event, arg); 4612 4613 case PERF_EVENT_IOC_PAUSE_OUTPUT: { 4614 struct ring_buffer *rb; 4615 4616 rcu_read_lock(); 4617 rb = rcu_dereference(event->rb); 4618 if (!rb || !rb->nr_pages) { 4619 rcu_read_unlock(); 4620 return -EINVAL; 4621 } 4622 rb_toggle_paused(rb, !!arg); 4623 rcu_read_unlock(); 4624 return 0; 4625 } 4626 default: 4627 return -ENOTTY; 4628 } 4629 4630 if (flags & PERF_IOC_FLAG_GROUP) 4631 perf_event_for_each(event, func); 4632 else 4633 perf_event_for_each_child(event, func); 4634 4635 return 0; 4636 } 4637 4638 static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg) 4639 { 4640 struct perf_event *event = file->private_data; 4641 struct perf_event_context *ctx; 4642 long ret; 4643 4644 ctx = perf_event_ctx_lock(event); 4645 ret = _perf_ioctl(event, cmd, arg); 4646 perf_event_ctx_unlock(event, ctx); 4647 4648 return ret; 4649 } 4650 4651 #ifdef CONFIG_COMPAT 4652 static long perf_compat_ioctl(struct file *file, unsigned int cmd, 4653 unsigned long arg) 4654 { 4655 switch (_IOC_NR(cmd)) { 4656 case _IOC_NR(PERF_EVENT_IOC_SET_FILTER): 4657 case _IOC_NR(PERF_EVENT_IOC_ID): 4658 /* Fix up pointer size (usually 4 -> 8 in 32-on-64-bit case */ 4659 if (_IOC_SIZE(cmd) == sizeof(compat_uptr_t)) { 4660 cmd &= ~IOCSIZE_MASK; 4661 cmd |= sizeof(void *) << IOCSIZE_SHIFT; 4662 } 4663 break; 4664 } 4665 return perf_ioctl(file, cmd, arg); 4666 } 4667 #else 4668 # define perf_compat_ioctl NULL 4669 #endif 4670 4671 int perf_event_task_enable(void) 4672 { 4673 struct perf_event_context *ctx; 4674 struct perf_event *event; 4675 4676 mutex_lock(¤t->perf_event_mutex); 4677 list_for_each_entry(event, ¤t->perf_event_list, owner_entry) { 4678 ctx = perf_event_ctx_lock(event); 4679 perf_event_for_each_child(event, _perf_event_enable); 4680 perf_event_ctx_unlock(event, ctx); 4681 } 4682 mutex_unlock(¤t->perf_event_mutex); 4683 4684 return 0; 4685 } 4686 4687 int perf_event_task_disable(void) 4688 { 4689 struct perf_event_context *ctx; 4690 struct perf_event *event; 4691 4692 mutex_lock(¤t->perf_event_mutex); 4693 list_for_each_entry(event, ¤t->perf_event_list, owner_entry) { 4694 ctx = perf_event_ctx_lock(event); 4695 perf_event_for_each_child(event, _perf_event_disable); 4696 perf_event_ctx_unlock(event, ctx); 4697 } 4698 mutex_unlock(¤t->perf_event_mutex); 4699 4700 return 0; 4701 } 4702 4703 static int perf_event_index(struct perf_event *event) 4704 { 4705 if (event->hw.state & PERF_HES_STOPPED) 4706 return 0; 4707 4708 if (event->state != PERF_EVENT_STATE_ACTIVE) 4709 return 0; 4710 4711 return event->pmu->event_idx(event); 4712 } 4713 4714 static void calc_timer_values(struct perf_event *event, 4715 u64 *now, 4716 u64 *enabled, 4717 u64 *running) 4718 { 4719 u64 ctx_time; 4720 4721 *now = perf_clock(); 4722 ctx_time = event->shadow_ctx_time + *now; 4723 *enabled = ctx_time - event->tstamp_enabled; 4724 *running = ctx_time - event->tstamp_running; 4725 } 4726 4727 static void perf_event_init_userpage(struct perf_event *event) 4728 { 4729 struct perf_event_mmap_page *userpg; 4730 struct ring_buffer *rb; 4731 4732 rcu_read_lock(); 4733 rb = rcu_dereference(event->rb); 4734 if (!rb) 4735 goto unlock; 4736 4737 userpg = rb->user_page; 4738 4739 /* Allow new userspace to detect that bit 0 is deprecated */ 4740 userpg->cap_bit0_is_deprecated = 1; 4741 userpg->size = offsetof(struct perf_event_mmap_page, __reserved); 4742 userpg->data_offset = PAGE_SIZE; 4743 userpg->data_size = perf_data_size(rb); 4744 4745 unlock: 4746 rcu_read_unlock(); 4747 } 4748 4749 void __weak arch_perf_update_userpage( 4750 struct perf_event *event, struct perf_event_mmap_page *userpg, u64 now) 4751 { 4752 } 4753 4754 /* 4755 * Callers need to ensure there can be no nesting of this function, otherwise 4756 * the seqlock logic goes bad. We can not serialize this because the arch 4757 * code calls this from NMI context. 4758 */ 4759 void perf_event_update_userpage(struct perf_event *event) 4760 { 4761 struct perf_event_mmap_page *userpg; 4762 struct ring_buffer *rb; 4763 u64 enabled, running, now; 4764 4765 rcu_read_lock(); 4766 rb = rcu_dereference(event->rb); 4767 if (!rb) 4768 goto unlock; 4769 4770 /* 4771 * compute total_time_enabled, total_time_running 4772 * based on snapshot values taken when the event 4773 * was last scheduled in. 4774 * 4775 * we cannot simply called update_context_time() 4776 * because of locking issue as we can be called in 4777 * NMI context 4778 */ 4779 calc_timer_values(event, &now, &enabled, &running); 4780 4781 userpg = rb->user_page; 4782 /* 4783 * Disable preemption so as to not let the corresponding user-space 4784 * spin too long if we get preempted. 4785 */ 4786 preempt_disable(); 4787 ++userpg->lock; 4788 barrier(); 4789 userpg->index = perf_event_index(event); 4790 userpg->offset = perf_event_count(event); 4791 if (userpg->index) 4792 userpg->offset -= local64_read(&event->hw.prev_count); 4793 4794 userpg->time_enabled = enabled + 4795 atomic64_read(&event->child_total_time_enabled); 4796 4797 userpg->time_running = running + 4798 atomic64_read(&event->child_total_time_running); 4799 4800 arch_perf_update_userpage(event, userpg, now); 4801 4802 barrier(); 4803 ++userpg->lock; 4804 preempt_enable(); 4805 unlock: 4806 rcu_read_unlock(); 4807 } 4808 4809 static int perf_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf) 4810 { 4811 struct perf_event *event = vma->vm_file->private_data; 4812 struct ring_buffer *rb; 4813 int ret = VM_FAULT_SIGBUS; 4814 4815 if (vmf->flags & FAULT_FLAG_MKWRITE) { 4816 if (vmf->pgoff == 0) 4817 ret = 0; 4818 return ret; 4819 } 4820 4821 rcu_read_lock(); 4822 rb = rcu_dereference(event->rb); 4823 if (!rb) 4824 goto unlock; 4825 4826 if (vmf->pgoff && (vmf->flags & FAULT_FLAG_WRITE)) 4827 goto unlock; 4828 4829 vmf->page = perf_mmap_to_page(rb, vmf->pgoff); 4830 if (!vmf->page) 4831 goto unlock; 4832 4833 get_page(vmf->page); 4834 vmf->page->mapping = vma->vm_file->f_mapping; 4835 vmf->page->index = vmf->pgoff; 4836 4837 ret = 0; 4838 unlock: 4839 rcu_read_unlock(); 4840 4841 return ret; 4842 } 4843 4844 static void ring_buffer_attach(struct perf_event *event, 4845 struct ring_buffer *rb) 4846 { 4847 struct ring_buffer *old_rb = NULL; 4848 unsigned long flags; 4849 4850 if (event->rb) { 4851 /* 4852 * Should be impossible, we set this when removing 4853 * event->rb_entry and wait/clear when adding event->rb_entry. 4854 */ 4855 WARN_ON_ONCE(event->rcu_pending); 4856 4857 old_rb = event->rb; 4858 spin_lock_irqsave(&old_rb->event_lock, flags); 4859 list_del_rcu(&event->rb_entry); 4860 spin_unlock_irqrestore(&old_rb->event_lock, flags); 4861 4862 event->rcu_batches = get_state_synchronize_rcu(); 4863 event->rcu_pending = 1; 4864 } 4865 4866 if (rb) { 4867 if (event->rcu_pending) { 4868 cond_synchronize_rcu(event->rcu_batches); 4869 event->rcu_pending = 0; 4870 } 4871 4872 spin_lock_irqsave(&rb->event_lock, flags); 4873 list_add_rcu(&event->rb_entry, &rb->event_list); 4874 spin_unlock_irqrestore(&rb->event_lock, flags); 4875 } 4876 4877 /* 4878 * Avoid racing with perf_mmap_close(AUX): stop the event 4879 * before swizzling the event::rb pointer; if it's getting 4880 * unmapped, its aux_mmap_count will be 0 and it won't 4881 * restart. See the comment in __perf_pmu_output_stop(). 4882 * 4883 * Data will inevitably be lost when set_output is done in 4884 * mid-air, but then again, whoever does it like this is 4885 * not in for the data anyway. 4886 */ 4887 if (has_aux(event)) 4888 perf_event_stop(event, 0); 4889 4890 rcu_assign_pointer(event->rb, rb); 4891 4892 if (old_rb) { 4893 ring_buffer_put(old_rb); 4894 /* 4895 * Since we detached before setting the new rb, so that we 4896 * could attach the new rb, we could have missed a wakeup. 4897 * Provide it now. 4898 */ 4899 wake_up_all(&event->waitq); 4900 } 4901 } 4902 4903 static void ring_buffer_wakeup(struct perf_event *event) 4904 { 4905 struct ring_buffer *rb; 4906 4907 rcu_read_lock(); 4908 rb = rcu_dereference(event->rb); 4909 if (rb) { 4910 list_for_each_entry_rcu(event, &rb->event_list, rb_entry) 4911 wake_up_all(&event->waitq); 4912 } 4913 rcu_read_unlock(); 4914 } 4915 4916 struct ring_buffer *ring_buffer_get(struct perf_event *event) 4917 { 4918 struct ring_buffer *rb; 4919 4920 rcu_read_lock(); 4921 rb = rcu_dereference(event->rb); 4922 if (rb) { 4923 if (!atomic_inc_not_zero(&rb->refcount)) 4924 rb = NULL; 4925 } 4926 rcu_read_unlock(); 4927 4928 return rb; 4929 } 4930 4931 void ring_buffer_put(struct ring_buffer *rb) 4932 { 4933 if (!atomic_dec_and_test(&rb->refcount)) 4934 return; 4935 4936 WARN_ON_ONCE(!list_empty(&rb->event_list)); 4937 4938 call_rcu(&rb->rcu_head, rb_free_rcu); 4939 } 4940 4941 static void perf_mmap_open(struct vm_area_struct *vma) 4942 { 4943 struct perf_event *event = vma->vm_file->private_data; 4944 4945 atomic_inc(&event->mmap_count); 4946 atomic_inc(&event->rb->mmap_count); 4947 4948 if (vma->vm_pgoff) 4949 atomic_inc(&event->rb->aux_mmap_count); 4950 4951 if (event->pmu->event_mapped) 4952 event->pmu->event_mapped(event); 4953 } 4954 4955 static void perf_pmu_output_stop(struct perf_event *event); 4956 4957 /* 4958 * A buffer can be mmap()ed multiple times; either directly through the same 4959 * event, or through other events by use of perf_event_set_output(). 4960 * 4961 * In order to undo the VM accounting done by perf_mmap() we need to destroy 4962 * the buffer here, where we still have a VM context. This means we need 4963 * to detach all events redirecting to us. 4964 */ 4965 static void perf_mmap_close(struct vm_area_struct *vma) 4966 { 4967 struct perf_event *event = vma->vm_file->private_data; 4968 4969 struct ring_buffer *rb = ring_buffer_get(event); 4970 struct user_struct *mmap_user = rb->mmap_user; 4971 int mmap_locked = rb->mmap_locked; 4972 unsigned long size = perf_data_size(rb); 4973 4974 if (event->pmu->event_unmapped) 4975 event->pmu->event_unmapped(event); 4976 4977 /* 4978 * rb->aux_mmap_count will always drop before rb->mmap_count and 4979 * event->mmap_count, so it is ok to use event->mmap_mutex to 4980 * serialize with perf_mmap here. 4981 */ 4982 if (rb_has_aux(rb) && vma->vm_pgoff == rb->aux_pgoff && 4983 atomic_dec_and_mutex_lock(&rb->aux_mmap_count, &event->mmap_mutex)) { 4984 /* 4985 * Stop all AUX events that are writing to this buffer, 4986 * so that we can free its AUX pages and corresponding PMU 4987 * data. Note that after rb::aux_mmap_count dropped to zero, 4988 * they won't start any more (see perf_aux_output_begin()). 4989 */ 4990 perf_pmu_output_stop(event); 4991 4992 /* now it's safe to free the pages */ 4993 atomic_long_sub(rb->aux_nr_pages, &mmap_user->locked_vm); 4994 vma->vm_mm->pinned_vm -= rb->aux_mmap_locked; 4995 4996 /* this has to be the last one */ 4997 rb_free_aux(rb); 4998 WARN_ON_ONCE(atomic_read(&rb->aux_refcount)); 4999 5000 mutex_unlock(&event->mmap_mutex); 5001 } 5002 5003 atomic_dec(&rb->mmap_count); 5004 5005 if (!atomic_dec_and_mutex_lock(&event->mmap_count, &event->mmap_mutex)) 5006 goto out_put; 5007 5008 ring_buffer_attach(event, NULL); 5009 mutex_unlock(&event->mmap_mutex); 5010 5011 /* If there's still other mmap()s of this buffer, we're done. */ 5012 if (atomic_read(&rb->mmap_count)) 5013 goto out_put; 5014 5015 /* 5016 * No other mmap()s, detach from all other events that might redirect 5017 * into the now unreachable buffer. Somewhat complicated by the 5018 * fact that rb::event_lock otherwise nests inside mmap_mutex. 5019 */ 5020 again: 5021 rcu_read_lock(); 5022 list_for_each_entry_rcu(event, &rb->event_list, rb_entry) { 5023 if (!atomic_long_inc_not_zero(&event->refcount)) { 5024 /* 5025 * This event is en-route to free_event() which will 5026 * detach it and remove it from the list. 5027 */ 5028 continue; 5029 } 5030 rcu_read_unlock(); 5031 5032 mutex_lock(&event->mmap_mutex); 5033 /* 5034 * Check we didn't race with perf_event_set_output() which can 5035 * swizzle the rb from under us while we were waiting to 5036 * acquire mmap_mutex. 5037 * 5038 * If we find a different rb; ignore this event, a next 5039 * iteration will no longer find it on the list. We have to 5040 * still restart the iteration to make sure we're not now 5041 * iterating the wrong list. 5042 */ 5043 if (event->rb == rb) 5044 ring_buffer_attach(event, NULL); 5045 5046 mutex_unlock(&event->mmap_mutex); 5047 put_event(event); 5048 5049 /* 5050 * Restart the iteration; either we're on the wrong list or 5051 * destroyed its integrity by doing a deletion. 5052 */ 5053 goto again; 5054 } 5055 rcu_read_unlock(); 5056 5057 /* 5058 * It could be there's still a few 0-ref events on the list; they'll 5059 * get cleaned up by free_event() -- they'll also still have their 5060 * ref on the rb and will free it whenever they are done with it. 5061 * 5062 * Aside from that, this buffer is 'fully' detached and unmapped, 5063 * undo the VM accounting. 5064 */ 5065 5066 atomic_long_sub((size >> PAGE_SHIFT) + 1, &mmap_user->locked_vm); 5067 vma->vm_mm->pinned_vm -= mmap_locked; 5068 free_uid(mmap_user); 5069 5070 out_put: 5071 ring_buffer_put(rb); /* could be last */ 5072 } 5073 5074 static const struct vm_operations_struct perf_mmap_vmops = { 5075 .open = perf_mmap_open, 5076 .close = perf_mmap_close, /* non mergable */ 5077 .fault = perf_mmap_fault, 5078 .page_mkwrite = perf_mmap_fault, 5079 }; 5080 5081 static int perf_mmap(struct file *file, struct vm_area_struct *vma) 5082 { 5083 struct perf_event *event = file->private_data; 5084 unsigned long user_locked, user_lock_limit; 5085 struct user_struct *user = current_user(); 5086 unsigned long locked, lock_limit; 5087 struct ring_buffer *rb = NULL; 5088 unsigned long vma_size; 5089 unsigned long nr_pages; 5090 long user_extra = 0, extra = 0; 5091 int ret = 0, flags = 0; 5092 5093 /* 5094 * Don't allow mmap() of inherited per-task counters. This would 5095 * create a performance issue due to all children writing to the 5096 * same rb. 5097 */ 5098 if (event->cpu == -1 && event->attr.inherit) 5099 return -EINVAL; 5100 5101 if (!(vma->vm_flags & VM_SHARED)) 5102 return -EINVAL; 5103 5104 vma_size = vma->vm_end - vma->vm_start; 5105 5106 if (vma->vm_pgoff == 0) { 5107 nr_pages = (vma_size / PAGE_SIZE) - 1; 5108 } else { 5109 /* 5110 * AUX area mapping: if rb->aux_nr_pages != 0, it's already 5111 * mapped, all subsequent mappings should have the same size 5112 * and offset. Must be above the normal perf buffer. 5113 */ 5114 u64 aux_offset, aux_size; 5115 5116 if (!event->rb) 5117 return -EINVAL; 5118 5119 nr_pages = vma_size / PAGE_SIZE; 5120 5121 mutex_lock(&event->mmap_mutex); 5122 ret = -EINVAL; 5123 5124 rb = event->rb; 5125 if (!rb) 5126 goto aux_unlock; 5127 5128 aux_offset = ACCESS_ONCE(rb->user_page->aux_offset); 5129 aux_size = ACCESS_ONCE(rb->user_page->aux_size); 5130 5131 if (aux_offset < perf_data_size(rb) + PAGE_SIZE) 5132 goto aux_unlock; 5133 5134 if (aux_offset != vma->vm_pgoff << PAGE_SHIFT) 5135 goto aux_unlock; 5136 5137 /* already mapped with a different offset */ 5138 if (rb_has_aux(rb) && rb->aux_pgoff != vma->vm_pgoff) 5139 goto aux_unlock; 5140 5141 if (aux_size != vma_size || aux_size != nr_pages * PAGE_SIZE) 5142 goto aux_unlock; 5143 5144 /* already mapped with a different size */ 5145 if (rb_has_aux(rb) && rb->aux_nr_pages != nr_pages) 5146 goto aux_unlock; 5147 5148 if (!is_power_of_2(nr_pages)) 5149 goto aux_unlock; 5150 5151 if (!atomic_inc_not_zero(&rb->mmap_count)) 5152 goto aux_unlock; 5153 5154 if (rb_has_aux(rb)) { 5155 atomic_inc(&rb->aux_mmap_count); 5156 ret = 0; 5157 goto unlock; 5158 } 5159 5160 atomic_set(&rb->aux_mmap_count, 1); 5161 user_extra = nr_pages; 5162 5163 goto accounting; 5164 } 5165 5166 /* 5167 * If we have rb pages ensure they're a power-of-two number, so we 5168 * can do bitmasks instead of modulo. 5169 */ 5170 if (nr_pages != 0 && !is_power_of_2(nr_pages)) 5171 return -EINVAL; 5172 5173 if (vma_size != PAGE_SIZE * (1 + nr_pages)) 5174 return -EINVAL; 5175 5176 WARN_ON_ONCE(event->ctx->parent_ctx); 5177 again: 5178 mutex_lock(&event->mmap_mutex); 5179 if (event->rb) { 5180 if (event->rb->nr_pages != nr_pages) { 5181 ret = -EINVAL; 5182 goto unlock; 5183 } 5184 5185 if (!atomic_inc_not_zero(&event->rb->mmap_count)) { 5186 /* 5187 * Raced against perf_mmap_close() through 5188 * perf_event_set_output(). Try again, hope for better 5189 * luck. 5190 */ 5191 mutex_unlock(&event->mmap_mutex); 5192 goto again; 5193 } 5194 5195 goto unlock; 5196 } 5197 5198 user_extra = nr_pages + 1; 5199 5200 accounting: 5201 user_lock_limit = sysctl_perf_event_mlock >> (PAGE_SHIFT - 10); 5202 5203 /* 5204 * Increase the limit linearly with more CPUs: 5205 */ 5206 user_lock_limit *= num_online_cpus(); 5207 5208 user_locked = atomic_long_read(&user->locked_vm) + user_extra; 5209 5210 if (user_locked > user_lock_limit) 5211 extra = user_locked - user_lock_limit; 5212 5213 lock_limit = rlimit(RLIMIT_MEMLOCK); 5214 lock_limit >>= PAGE_SHIFT; 5215 locked = vma->vm_mm->pinned_vm + extra; 5216 5217 if ((locked > lock_limit) && perf_paranoid_tracepoint_raw() && 5218 !capable(CAP_IPC_LOCK)) { 5219 ret = -EPERM; 5220 goto unlock; 5221 } 5222 5223 WARN_ON(!rb && event->rb); 5224 5225 if (vma->vm_flags & VM_WRITE) 5226 flags |= RING_BUFFER_WRITABLE; 5227 5228 if (!rb) { 5229 rb = rb_alloc(nr_pages, 5230 event->attr.watermark ? event->attr.wakeup_watermark : 0, 5231 event->cpu, flags); 5232 5233 if (!rb) { 5234 ret = -ENOMEM; 5235 goto unlock; 5236 } 5237 5238 atomic_set(&rb->mmap_count, 1); 5239 rb->mmap_user = get_current_user(); 5240 rb->mmap_locked = extra; 5241 5242 ring_buffer_attach(event, rb); 5243 5244 perf_event_init_userpage(event); 5245 perf_event_update_userpage(event); 5246 } else { 5247 ret = rb_alloc_aux(rb, event, vma->vm_pgoff, nr_pages, 5248 event->attr.aux_watermark, flags); 5249 if (!ret) 5250 rb->aux_mmap_locked = extra; 5251 } 5252 5253 unlock: 5254 if (!ret) { 5255 atomic_long_add(user_extra, &user->locked_vm); 5256 vma->vm_mm->pinned_vm += extra; 5257 5258 atomic_inc(&event->mmap_count); 5259 } else if (rb) { 5260 atomic_dec(&rb->mmap_count); 5261 } 5262 aux_unlock: 5263 mutex_unlock(&event->mmap_mutex); 5264 5265 /* 5266 * Since pinned accounting is per vm we cannot allow fork() to copy our 5267 * vma. 5268 */ 5269 vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_DONTDUMP; 5270 vma->vm_ops = &perf_mmap_vmops; 5271 5272 if (event->pmu->event_mapped) 5273 event->pmu->event_mapped(event); 5274 5275 return ret; 5276 } 5277 5278 static int perf_fasync(int fd, struct file *filp, int on) 5279 { 5280 struct inode *inode = file_inode(filp); 5281 struct perf_event *event = filp->private_data; 5282 int retval; 5283 5284 inode_lock(inode); 5285 retval = fasync_helper(fd, filp, on, &event->fasync); 5286 inode_unlock(inode); 5287 5288 if (retval < 0) 5289 return retval; 5290 5291 return 0; 5292 } 5293 5294 static const struct file_operations perf_fops = { 5295 .llseek = no_llseek, 5296 .release = perf_release, 5297 .read = perf_read, 5298 .poll = perf_poll, 5299 .unlocked_ioctl = perf_ioctl, 5300 .compat_ioctl = perf_compat_ioctl, 5301 .mmap = perf_mmap, 5302 .fasync = perf_fasync, 5303 }; 5304 5305 /* 5306 * Perf event wakeup 5307 * 5308 * If there's data, ensure we set the poll() state and publish everything 5309 * to user-space before waking everybody up. 5310 */ 5311 5312 static inline struct fasync_struct **perf_event_fasync(struct perf_event *event) 5313 { 5314 /* only the parent has fasync state */ 5315 if (event->parent) 5316 event = event->parent; 5317 return &event->fasync; 5318 } 5319 5320 void perf_event_wakeup(struct perf_event *event) 5321 { 5322 ring_buffer_wakeup(event); 5323 5324 if (event->pending_kill) { 5325 kill_fasync(perf_event_fasync(event), SIGIO, event->pending_kill); 5326 event->pending_kill = 0; 5327 } 5328 } 5329 5330 static void perf_pending_event(struct irq_work *entry) 5331 { 5332 struct perf_event *event = container_of(entry, 5333 struct perf_event, pending); 5334 int rctx; 5335 5336 rctx = perf_swevent_get_recursion_context(); 5337 /* 5338 * If we 'fail' here, that's OK, it means recursion is already disabled 5339 * and we won't recurse 'further'. 5340 */ 5341 5342 if (event->pending_disable) { 5343 event->pending_disable = 0; 5344 perf_event_disable_local(event); 5345 } 5346 5347 if (event->pending_wakeup) { 5348 event->pending_wakeup = 0; 5349 perf_event_wakeup(event); 5350 } 5351 5352 if (rctx >= 0) 5353 perf_swevent_put_recursion_context(rctx); 5354 } 5355 5356 /* 5357 * We assume there is only KVM supporting the callbacks. 5358 * Later on, we might change it to a list if there is 5359 * another virtualization implementation supporting the callbacks. 5360 */ 5361 struct perf_guest_info_callbacks *perf_guest_cbs; 5362 5363 int perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *cbs) 5364 { 5365 perf_guest_cbs = cbs; 5366 return 0; 5367 } 5368 EXPORT_SYMBOL_GPL(perf_register_guest_info_callbacks); 5369 5370 int perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *cbs) 5371 { 5372 perf_guest_cbs = NULL; 5373 return 0; 5374 } 5375 EXPORT_SYMBOL_GPL(perf_unregister_guest_info_callbacks); 5376 5377 static void 5378 perf_output_sample_regs(struct perf_output_handle *handle, 5379 struct pt_regs *regs, u64 mask) 5380 { 5381 int bit; 5382 DECLARE_BITMAP(_mask, 64); 5383 5384 bitmap_from_u64(_mask, mask); 5385 for_each_set_bit(bit, _mask, sizeof(mask) * BITS_PER_BYTE) { 5386 u64 val; 5387 5388 val = perf_reg_value(regs, bit); 5389 perf_output_put(handle, val); 5390 } 5391 } 5392 5393 static void perf_sample_regs_user(struct perf_regs *regs_user, 5394 struct pt_regs *regs, 5395 struct pt_regs *regs_user_copy) 5396 { 5397 if (user_mode(regs)) { 5398 regs_user->abi = perf_reg_abi(current); 5399 regs_user->regs = regs; 5400 } else if (current->mm) { 5401 perf_get_regs_user(regs_user, regs, regs_user_copy); 5402 } else { 5403 regs_user->abi = PERF_SAMPLE_REGS_ABI_NONE; 5404 regs_user->regs = NULL; 5405 } 5406 } 5407 5408 static void perf_sample_regs_intr(struct perf_regs *regs_intr, 5409 struct pt_regs *regs) 5410 { 5411 regs_intr->regs = regs; 5412 regs_intr->abi = perf_reg_abi(current); 5413 } 5414 5415 5416 /* 5417 * Get remaining task size from user stack pointer. 5418 * 5419 * It'd be better to take stack vma map and limit this more 5420 * precisly, but there's no way to get it safely under interrupt, 5421 * so using TASK_SIZE as limit. 5422 */ 5423 static u64 perf_ustack_task_size(struct pt_regs *regs) 5424 { 5425 unsigned long addr = perf_user_stack_pointer(regs); 5426 5427 if (!addr || addr >= TASK_SIZE) 5428 return 0; 5429 5430 return TASK_SIZE - addr; 5431 } 5432 5433 static u16 5434 perf_sample_ustack_size(u16 stack_size, u16 header_size, 5435 struct pt_regs *regs) 5436 { 5437 u64 task_size; 5438 5439 /* No regs, no stack pointer, no dump. */ 5440 if (!regs) 5441 return 0; 5442 5443 /* 5444 * Check if we fit in with the requested stack size into the: 5445 * - TASK_SIZE 5446 * If we don't, we limit the size to the TASK_SIZE. 5447 * 5448 * - remaining sample size 5449 * If we don't, we customize the stack size to 5450 * fit in to the remaining sample size. 5451 */ 5452 5453 task_size = min((u64) USHRT_MAX, perf_ustack_task_size(regs)); 5454 stack_size = min(stack_size, (u16) task_size); 5455 5456 /* Current header size plus static size and dynamic size. */ 5457 header_size += 2 * sizeof(u64); 5458 5459 /* Do we fit in with the current stack dump size? */ 5460 if ((u16) (header_size + stack_size) < header_size) { 5461 /* 5462 * If we overflow the maximum size for the sample, 5463 * we customize the stack dump size to fit in. 5464 */ 5465 stack_size = USHRT_MAX - header_size - sizeof(u64); 5466 stack_size = round_up(stack_size, sizeof(u64)); 5467 } 5468 5469 return stack_size; 5470 } 5471 5472 static void 5473 perf_output_sample_ustack(struct perf_output_handle *handle, u64 dump_size, 5474 struct pt_regs *regs) 5475 { 5476 /* Case of a kernel thread, nothing to dump */ 5477 if (!regs) { 5478 u64 size = 0; 5479 perf_output_put(handle, size); 5480 } else { 5481 unsigned long sp; 5482 unsigned int rem; 5483 u64 dyn_size; 5484 5485 /* 5486 * We dump: 5487 * static size 5488 * - the size requested by user or the best one we can fit 5489 * in to the sample max size 5490 * data 5491 * - user stack dump data 5492 * dynamic size 5493 * - the actual dumped size 5494 */ 5495 5496 /* Static size. */ 5497 perf_output_put(handle, dump_size); 5498 5499 /* Data. */ 5500 sp = perf_user_stack_pointer(regs); 5501 rem = __output_copy_user(handle, (void *) sp, dump_size); 5502 dyn_size = dump_size - rem; 5503 5504 perf_output_skip(handle, rem); 5505 5506 /* Dynamic size. */ 5507 perf_output_put(handle, dyn_size); 5508 } 5509 } 5510 5511 static void __perf_event_header__init_id(struct perf_event_header *header, 5512 struct perf_sample_data *data, 5513 struct perf_event *event) 5514 { 5515 u64 sample_type = event->attr.sample_type; 5516 5517 data->type = sample_type; 5518 header->size += event->id_header_size; 5519 5520 if (sample_type & PERF_SAMPLE_TID) { 5521 /* namespace issues */ 5522 data->tid_entry.pid = perf_event_pid(event, current); 5523 data->tid_entry.tid = perf_event_tid(event, current); 5524 } 5525 5526 if (sample_type & PERF_SAMPLE_TIME) 5527 data->time = perf_event_clock(event); 5528 5529 if (sample_type & (PERF_SAMPLE_ID | PERF_SAMPLE_IDENTIFIER)) 5530 data->id = primary_event_id(event); 5531 5532 if (sample_type & PERF_SAMPLE_STREAM_ID) 5533 data->stream_id = event->id; 5534 5535 if (sample_type & PERF_SAMPLE_CPU) { 5536 data->cpu_entry.cpu = raw_smp_processor_id(); 5537 data->cpu_entry.reserved = 0; 5538 } 5539 } 5540 5541 void perf_event_header__init_id(struct perf_event_header *header, 5542 struct perf_sample_data *data, 5543 struct perf_event *event) 5544 { 5545 if (event->attr.sample_id_all) 5546 __perf_event_header__init_id(header, data, event); 5547 } 5548 5549 static void __perf_event__output_id_sample(struct perf_output_handle *handle, 5550 struct perf_sample_data *data) 5551 { 5552 u64 sample_type = data->type; 5553 5554 if (sample_type & PERF_SAMPLE_TID) 5555 perf_output_put(handle, data->tid_entry); 5556 5557 if (sample_type & PERF_SAMPLE_TIME) 5558 perf_output_put(handle, data->time); 5559 5560 if (sample_type & PERF_SAMPLE_ID) 5561 perf_output_put(handle, data->id); 5562 5563 if (sample_type & PERF_SAMPLE_STREAM_ID) 5564 perf_output_put(handle, data->stream_id); 5565 5566 if (sample_type & PERF_SAMPLE_CPU) 5567 perf_output_put(handle, data->cpu_entry); 5568 5569 if (sample_type & PERF_SAMPLE_IDENTIFIER) 5570 perf_output_put(handle, data->id); 5571 } 5572 5573 void perf_event__output_id_sample(struct perf_event *event, 5574 struct perf_output_handle *handle, 5575 struct perf_sample_data *sample) 5576 { 5577 if (event->attr.sample_id_all) 5578 __perf_event__output_id_sample(handle, sample); 5579 } 5580 5581 static void perf_output_read_one(struct perf_output_handle *handle, 5582 struct perf_event *event, 5583 u64 enabled, u64 running) 5584 { 5585 u64 read_format = event->attr.read_format; 5586 u64 values[4]; 5587 int n = 0; 5588 5589 values[n++] = perf_event_count(event); 5590 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) { 5591 values[n++] = enabled + 5592 atomic64_read(&event->child_total_time_enabled); 5593 } 5594 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) { 5595 values[n++] = running + 5596 atomic64_read(&event->child_total_time_running); 5597 } 5598 if (read_format & PERF_FORMAT_ID) 5599 values[n++] = primary_event_id(event); 5600 5601 __output_copy(handle, values, n * sizeof(u64)); 5602 } 5603 5604 /* 5605 * XXX PERF_FORMAT_GROUP vs inherited events seems difficult. 5606 */ 5607 static void perf_output_read_group(struct perf_output_handle *handle, 5608 struct perf_event *event, 5609 u64 enabled, u64 running) 5610 { 5611 struct perf_event *leader = event->group_leader, *sub; 5612 u64 read_format = event->attr.read_format; 5613 u64 values[5]; 5614 int n = 0; 5615 5616 values[n++] = 1 + leader->nr_siblings; 5617 5618 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) 5619 values[n++] = enabled; 5620 5621 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) 5622 values[n++] = running; 5623 5624 if (leader != event) 5625 leader->pmu->read(leader); 5626 5627 values[n++] = perf_event_count(leader); 5628 if (read_format & PERF_FORMAT_ID) 5629 values[n++] = primary_event_id(leader); 5630 5631 __output_copy(handle, values, n * sizeof(u64)); 5632 5633 list_for_each_entry(sub, &leader->sibling_list, group_entry) { 5634 n = 0; 5635 5636 if ((sub != event) && 5637 (sub->state == PERF_EVENT_STATE_ACTIVE)) 5638 sub->pmu->read(sub); 5639 5640 values[n++] = perf_event_count(sub); 5641 if (read_format & PERF_FORMAT_ID) 5642 values[n++] = primary_event_id(sub); 5643 5644 __output_copy(handle, values, n * sizeof(u64)); 5645 } 5646 } 5647 5648 #define PERF_FORMAT_TOTAL_TIMES (PERF_FORMAT_TOTAL_TIME_ENABLED|\ 5649 PERF_FORMAT_TOTAL_TIME_RUNNING) 5650 5651 static void perf_output_read(struct perf_output_handle *handle, 5652 struct perf_event *event) 5653 { 5654 u64 enabled = 0, running = 0, now; 5655 u64 read_format = event->attr.read_format; 5656 5657 /* 5658 * compute total_time_enabled, total_time_running 5659 * based on snapshot values taken when the event 5660 * was last scheduled in. 5661 * 5662 * we cannot simply called update_context_time() 5663 * because of locking issue as we are called in 5664 * NMI context 5665 */ 5666 if (read_format & PERF_FORMAT_TOTAL_TIMES) 5667 calc_timer_values(event, &now, &enabled, &running); 5668 5669 if (event->attr.read_format & PERF_FORMAT_GROUP) 5670 perf_output_read_group(handle, event, enabled, running); 5671 else 5672 perf_output_read_one(handle, event, enabled, running); 5673 } 5674 5675 void perf_output_sample(struct perf_output_handle *handle, 5676 struct perf_event_header *header, 5677 struct perf_sample_data *data, 5678 struct perf_event *event) 5679 { 5680 u64 sample_type = data->type; 5681 5682 perf_output_put(handle, *header); 5683 5684 if (sample_type & PERF_SAMPLE_IDENTIFIER) 5685 perf_output_put(handle, data->id); 5686 5687 if (sample_type & PERF_SAMPLE_IP) 5688 perf_output_put(handle, data->ip); 5689 5690 if (sample_type & PERF_SAMPLE_TID) 5691 perf_output_put(handle, data->tid_entry); 5692 5693 if (sample_type & PERF_SAMPLE_TIME) 5694 perf_output_put(handle, data->time); 5695 5696 if (sample_type & PERF_SAMPLE_ADDR) 5697 perf_output_put(handle, data->addr); 5698 5699 if (sample_type & PERF_SAMPLE_ID) 5700 perf_output_put(handle, data->id); 5701 5702 if (sample_type & PERF_SAMPLE_STREAM_ID) 5703 perf_output_put(handle, data->stream_id); 5704 5705 if (sample_type & PERF_SAMPLE_CPU) 5706 perf_output_put(handle, data->cpu_entry); 5707 5708 if (sample_type & PERF_SAMPLE_PERIOD) 5709 perf_output_put(handle, data->period); 5710 5711 if (sample_type & PERF_SAMPLE_READ) 5712 perf_output_read(handle, event); 5713 5714 if (sample_type & PERF_SAMPLE_CALLCHAIN) { 5715 if (data->callchain) { 5716 int size = 1; 5717 5718 if (data->callchain) 5719 size += data->callchain->nr; 5720 5721 size *= sizeof(u64); 5722 5723 __output_copy(handle, data->callchain, size); 5724 } else { 5725 u64 nr = 0; 5726 perf_output_put(handle, nr); 5727 } 5728 } 5729 5730 if (sample_type & PERF_SAMPLE_RAW) { 5731 struct perf_raw_record *raw = data->raw; 5732 5733 if (raw) { 5734 struct perf_raw_frag *frag = &raw->frag; 5735 5736 perf_output_put(handle, raw->size); 5737 do { 5738 if (frag->copy) { 5739 __output_custom(handle, frag->copy, 5740 frag->data, frag->size); 5741 } else { 5742 __output_copy(handle, frag->data, 5743 frag->size); 5744 } 5745 if (perf_raw_frag_last(frag)) 5746 break; 5747 frag = frag->next; 5748 } while (1); 5749 if (frag->pad) 5750 __output_skip(handle, NULL, frag->pad); 5751 } else { 5752 struct { 5753 u32 size; 5754 u32 data; 5755 } raw = { 5756 .size = sizeof(u32), 5757 .data = 0, 5758 }; 5759 perf_output_put(handle, raw); 5760 } 5761 } 5762 5763 if (sample_type & PERF_SAMPLE_BRANCH_STACK) { 5764 if (data->br_stack) { 5765 size_t size; 5766 5767 size = data->br_stack->nr 5768 * sizeof(struct perf_branch_entry); 5769 5770 perf_output_put(handle, data->br_stack->nr); 5771 perf_output_copy(handle, data->br_stack->entries, size); 5772 } else { 5773 /* 5774 * we always store at least the value of nr 5775 */ 5776 u64 nr = 0; 5777 perf_output_put(handle, nr); 5778 } 5779 } 5780 5781 if (sample_type & PERF_SAMPLE_REGS_USER) { 5782 u64 abi = data->regs_user.abi; 5783 5784 /* 5785 * If there are no regs to dump, notice it through 5786 * first u64 being zero (PERF_SAMPLE_REGS_ABI_NONE). 5787 */ 5788 perf_output_put(handle, abi); 5789 5790 if (abi) { 5791 u64 mask = event->attr.sample_regs_user; 5792 perf_output_sample_regs(handle, 5793 data->regs_user.regs, 5794 mask); 5795 } 5796 } 5797 5798 if (sample_type & PERF_SAMPLE_STACK_USER) { 5799 perf_output_sample_ustack(handle, 5800 data->stack_user_size, 5801 data->regs_user.regs); 5802 } 5803 5804 if (sample_type & PERF_SAMPLE_WEIGHT) 5805 perf_output_put(handle, data->weight); 5806 5807 if (sample_type & PERF_SAMPLE_DATA_SRC) 5808 perf_output_put(handle, data->data_src.val); 5809 5810 if (sample_type & PERF_SAMPLE_TRANSACTION) 5811 perf_output_put(handle, data->txn); 5812 5813 if (sample_type & PERF_SAMPLE_REGS_INTR) { 5814 u64 abi = data->regs_intr.abi; 5815 /* 5816 * If there are no regs to dump, notice it through 5817 * first u64 being zero (PERF_SAMPLE_REGS_ABI_NONE). 5818 */ 5819 perf_output_put(handle, abi); 5820 5821 if (abi) { 5822 u64 mask = event->attr.sample_regs_intr; 5823 5824 perf_output_sample_regs(handle, 5825 data->regs_intr.regs, 5826 mask); 5827 } 5828 } 5829 5830 if (!event->attr.watermark) { 5831 int wakeup_events = event->attr.wakeup_events; 5832 5833 if (wakeup_events) { 5834 struct ring_buffer *rb = handle->rb; 5835 int events = local_inc_return(&rb->events); 5836 5837 if (events >= wakeup_events) { 5838 local_sub(wakeup_events, &rb->events); 5839 local_inc(&rb->wakeup); 5840 } 5841 } 5842 } 5843 } 5844 5845 void perf_prepare_sample(struct perf_event_header *header, 5846 struct perf_sample_data *data, 5847 struct perf_event *event, 5848 struct pt_regs *regs) 5849 { 5850 u64 sample_type = event->attr.sample_type; 5851 5852 header->type = PERF_RECORD_SAMPLE; 5853 header->size = sizeof(*header) + event->header_size; 5854 5855 header->misc = 0; 5856 header->misc |= perf_misc_flags(regs); 5857 5858 __perf_event_header__init_id(header, data, event); 5859 5860 if (sample_type & PERF_SAMPLE_IP) 5861 data->ip = perf_instruction_pointer(regs); 5862 5863 if (sample_type & PERF_SAMPLE_CALLCHAIN) { 5864 int size = 1; 5865 5866 data->callchain = perf_callchain(event, regs); 5867 5868 if (data->callchain) 5869 size += data->callchain->nr; 5870 5871 header->size += size * sizeof(u64); 5872 } 5873 5874 if (sample_type & PERF_SAMPLE_RAW) { 5875 struct perf_raw_record *raw = data->raw; 5876 int size; 5877 5878 if (raw) { 5879 struct perf_raw_frag *frag = &raw->frag; 5880 u32 sum = 0; 5881 5882 do { 5883 sum += frag->size; 5884 if (perf_raw_frag_last(frag)) 5885 break; 5886 frag = frag->next; 5887 } while (1); 5888 5889 size = round_up(sum + sizeof(u32), sizeof(u64)); 5890 raw->size = size - sizeof(u32); 5891 frag->pad = raw->size - sum; 5892 } else { 5893 size = sizeof(u64); 5894 } 5895 5896 header->size += size; 5897 } 5898 5899 if (sample_type & PERF_SAMPLE_BRANCH_STACK) { 5900 int size = sizeof(u64); /* nr */ 5901 if (data->br_stack) { 5902 size += data->br_stack->nr 5903 * sizeof(struct perf_branch_entry); 5904 } 5905 header->size += size; 5906 } 5907 5908 if (sample_type & (PERF_SAMPLE_REGS_USER | PERF_SAMPLE_STACK_USER)) 5909 perf_sample_regs_user(&data->regs_user, regs, 5910 &data->regs_user_copy); 5911 5912 if (sample_type & PERF_SAMPLE_REGS_USER) { 5913 /* regs dump ABI info */ 5914 int size = sizeof(u64); 5915 5916 if (data->regs_user.regs) { 5917 u64 mask = event->attr.sample_regs_user; 5918 size += hweight64(mask) * sizeof(u64); 5919 } 5920 5921 header->size += size; 5922 } 5923 5924 if (sample_type & PERF_SAMPLE_STACK_USER) { 5925 /* 5926 * Either we need PERF_SAMPLE_STACK_USER bit to be allways 5927 * processed as the last one or have additional check added 5928 * in case new sample type is added, because we could eat 5929 * up the rest of the sample size. 5930 */ 5931 u16 stack_size = event->attr.sample_stack_user; 5932 u16 size = sizeof(u64); 5933 5934 stack_size = perf_sample_ustack_size(stack_size, header->size, 5935 data->regs_user.regs); 5936 5937 /* 5938 * If there is something to dump, add space for the dump 5939 * itself and for the field that tells the dynamic size, 5940 * which is how many have been actually dumped. 5941 */ 5942 if (stack_size) 5943 size += sizeof(u64) + stack_size; 5944 5945 data->stack_user_size = stack_size; 5946 header->size += size; 5947 } 5948 5949 if (sample_type & PERF_SAMPLE_REGS_INTR) { 5950 /* regs dump ABI info */ 5951 int size = sizeof(u64); 5952 5953 perf_sample_regs_intr(&data->regs_intr, regs); 5954 5955 if (data->regs_intr.regs) { 5956 u64 mask = event->attr.sample_regs_intr; 5957 5958 size += hweight64(mask) * sizeof(u64); 5959 } 5960 5961 header->size += size; 5962 } 5963 } 5964 5965 static void __always_inline 5966 __perf_event_output(struct perf_event *event, 5967 struct perf_sample_data *data, 5968 struct pt_regs *regs, 5969 int (*output_begin)(struct perf_output_handle *, 5970 struct perf_event *, 5971 unsigned int)) 5972 { 5973 struct perf_output_handle handle; 5974 struct perf_event_header header; 5975 5976 /* protect the callchain buffers */ 5977 rcu_read_lock(); 5978 5979 perf_prepare_sample(&header, data, event, regs); 5980 5981 if (output_begin(&handle, event, header.size)) 5982 goto exit; 5983 5984 perf_output_sample(&handle, &header, data, event); 5985 5986 perf_output_end(&handle); 5987 5988 exit: 5989 rcu_read_unlock(); 5990 } 5991 5992 void 5993 perf_event_output_forward(struct perf_event *event, 5994 struct perf_sample_data *data, 5995 struct pt_regs *regs) 5996 { 5997 __perf_event_output(event, data, regs, perf_output_begin_forward); 5998 } 5999 6000 void 6001 perf_event_output_backward(struct perf_event *event, 6002 struct perf_sample_data *data, 6003 struct pt_regs *regs) 6004 { 6005 __perf_event_output(event, data, regs, perf_output_begin_backward); 6006 } 6007 6008 void 6009 perf_event_output(struct perf_event *event, 6010 struct perf_sample_data *data, 6011 struct pt_regs *regs) 6012 { 6013 __perf_event_output(event, data, regs, perf_output_begin); 6014 } 6015 6016 /* 6017 * read event_id 6018 */ 6019 6020 struct perf_read_event { 6021 struct perf_event_header header; 6022 6023 u32 pid; 6024 u32 tid; 6025 }; 6026 6027 static void 6028 perf_event_read_event(struct perf_event *event, 6029 struct task_struct *task) 6030 { 6031 struct perf_output_handle handle; 6032 struct perf_sample_data sample; 6033 struct perf_read_event read_event = { 6034 .header = { 6035 .type = PERF_RECORD_READ, 6036 .misc = 0, 6037 .size = sizeof(read_event) + event->read_size, 6038 }, 6039 .pid = perf_event_pid(event, task), 6040 .tid = perf_event_tid(event, task), 6041 }; 6042 int ret; 6043 6044 perf_event_header__init_id(&read_event.header, &sample, event); 6045 ret = perf_output_begin(&handle, event, read_event.header.size); 6046 if (ret) 6047 return; 6048 6049 perf_output_put(&handle, read_event); 6050 perf_output_read(&handle, event); 6051 perf_event__output_id_sample(event, &handle, &sample); 6052 6053 perf_output_end(&handle); 6054 } 6055 6056 typedef void (perf_iterate_f)(struct perf_event *event, void *data); 6057 6058 static void 6059 perf_iterate_ctx(struct perf_event_context *ctx, 6060 perf_iterate_f output, 6061 void *data, bool all) 6062 { 6063 struct perf_event *event; 6064 6065 list_for_each_entry_rcu(event, &ctx->event_list, event_entry) { 6066 if (!all) { 6067 if (event->state < PERF_EVENT_STATE_INACTIVE) 6068 continue; 6069 if (!event_filter_match(event)) 6070 continue; 6071 } 6072 6073 output(event, data); 6074 } 6075 } 6076 6077 static void perf_iterate_sb_cpu(perf_iterate_f output, void *data) 6078 { 6079 struct pmu_event_list *pel = this_cpu_ptr(&pmu_sb_events); 6080 struct perf_event *event; 6081 6082 list_for_each_entry_rcu(event, &pel->list, sb_list) { 6083 /* 6084 * Skip events that are not fully formed yet; ensure that 6085 * if we observe event->ctx, both event and ctx will be 6086 * complete enough. See perf_install_in_context(). 6087 */ 6088 if (!smp_load_acquire(&event->ctx)) 6089 continue; 6090 6091 if (event->state < PERF_EVENT_STATE_INACTIVE) 6092 continue; 6093 if (!event_filter_match(event)) 6094 continue; 6095 output(event, data); 6096 } 6097 } 6098 6099 /* 6100 * Iterate all events that need to receive side-band events. 6101 * 6102 * For new callers; ensure that account_pmu_sb_event() includes 6103 * your event, otherwise it might not get delivered. 6104 */ 6105 static void 6106 perf_iterate_sb(perf_iterate_f output, void *data, 6107 struct perf_event_context *task_ctx) 6108 { 6109 struct perf_event_context *ctx; 6110 int ctxn; 6111 6112 rcu_read_lock(); 6113 preempt_disable(); 6114 6115 /* 6116 * If we have task_ctx != NULL we only notify the task context itself. 6117 * The task_ctx is set only for EXIT events before releasing task 6118 * context. 6119 */ 6120 if (task_ctx) { 6121 perf_iterate_ctx(task_ctx, output, data, false); 6122 goto done; 6123 } 6124 6125 perf_iterate_sb_cpu(output, data); 6126 6127 for_each_task_context_nr(ctxn) { 6128 ctx = rcu_dereference(current->perf_event_ctxp[ctxn]); 6129 if (ctx) 6130 perf_iterate_ctx(ctx, output, data, false); 6131 } 6132 done: 6133 preempt_enable(); 6134 rcu_read_unlock(); 6135 } 6136 6137 /* 6138 * Clear all file-based filters at exec, they'll have to be 6139 * re-instated when/if these objects are mmapped again. 6140 */ 6141 static void perf_event_addr_filters_exec(struct perf_event *event, void *data) 6142 { 6143 struct perf_addr_filters_head *ifh = perf_event_addr_filters(event); 6144 struct perf_addr_filter *filter; 6145 unsigned int restart = 0, count = 0; 6146 unsigned long flags; 6147 6148 if (!has_addr_filter(event)) 6149 return; 6150 6151 raw_spin_lock_irqsave(&ifh->lock, flags); 6152 list_for_each_entry(filter, &ifh->list, entry) { 6153 if (filter->inode) { 6154 event->addr_filters_offs[count] = 0; 6155 restart++; 6156 } 6157 6158 count++; 6159 } 6160 6161 if (restart) 6162 event->addr_filters_gen++; 6163 raw_spin_unlock_irqrestore(&ifh->lock, flags); 6164 6165 if (restart) 6166 perf_event_stop(event, 1); 6167 } 6168 6169 void perf_event_exec(void) 6170 { 6171 struct perf_event_context *ctx; 6172 int ctxn; 6173 6174 rcu_read_lock(); 6175 for_each_task_context_nr(ctxn) { 6176 ctx = current->perf_event_ctxp[ctxn]; 6177 if (!ctx) 6178 continue; 6179 6180 perf_event_enable_on_exec(ctxn); 6181 6182 perf_iterate_ctx(ctx, perf_event_addr_filters_exec, NULL, 6183 true); 6184 } 6185 rcu_read_unlock(); 6186 } 6187 6188 struct remote_output { 6189 struct ring_buffer *rb; 6190 int err; 6191 }; 6192 6193 static void __perf_event_output_stop(struct perf_event *event, void *data) 6194 { 6195 struct perf_event *parent = event->parent; 6196 struct remote_output *ro = data; 6197 struct ring_buffer *rb = ro->rb; 6198 struct stop_event_data sd = { 6199 .event = event, 6200 }; 6201 6202 if (!has_aux(event)) 6203 return; 6204 6205 if (!parent) 6206 parent = event; 6207 6208 /* 6209 * In case of inheritance, it will be the parent that links to the 6210 * ring-buffer, but it will be the child that's actually using it. 6211 * 6212 * We are using event::rb to determine if the event should be stopped, 6213 * however this may race with ring_buffer_attach() (through set_output), 6214 * which will make us skip the event that actually needs to be stopped. 6215 * So ring_buffer_attach() has to stop an aux event before re-assigning 6216 * its rb pointer. 6217 */ 6218 if (rcu_dereference(parent->rb) == rb) 6219 ro->err = __perf_event_stop(&sd); 6220 } 6221 6222 static int __perf_pmu_output_stop(void *info) 6223 { 6224 struct perf_event *event = info; 6225 struct pmu *pmu = event->pmu; 6226 struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context); 6227 struct remote_output ro = { 6228 .rb = event->rb, 6229 }; 6230 6231 rcu_read_lock(); 6232 perf_iterate_ctx(&cpuctx->ctx, __perf_event_output_stop, &ro, false); 6233 if (cpuctx->task_ctx) 6234 perf_iterate_ctx(cpuctx->task_ctx, __perf_event_output_stop, 6235 &ro, false); 6236 rcu_read_unlock(); 6237 6238 return ro.err; 6239 } 6240 6241 static void perf_pmu_output_stop(struct perf_event *event) 6242 { 6243 struct perf_event *iter; 6244 int err, cpu; 6245 6246 restart: 6247 rcu_read_lock(); 6248 list_for_each_entry_rcu(iter, &event->rb->event_list, rb_entry) { 6249 /* 6250 * For per-CPU events, we need to make sure that neither they 6251 * nor their children are running; for cpu==-1 events it's 6252 * sufficient to stop the event itself if it's active, since 6253 * it can't have children. 6254 */ 6255 cpu = iter->cpu; 6256 if (cpu == -1) 6257 cpu = READ_ONCE(iter->oncpu); 6258 6259 if (cpu == -1) 6260 continue; 6261 6262 err = cpu_function_call(cpu, __perf_pmu_output_stop, event); 6263 if (err == -EAGAIN) { 6264 rcu_read_unlock(); 6265 goto restart; 6266 } 6267 } 6268 rcu_read_unlock(); 6269 } 6270 6271 /* 6272 * task tracking -- fork/exit 6273 * 6274 * enabled by: attr.comm | attr.mmap | attr.mmap2 | attr.mmap_data | attr.task 6275 */ 6276 6277 struct perf_task_event { 6278 struct task_struct *task; 6279 struct perf_event_context *task_ctx; 6280 6281 struct { 6282 struct perf_event_header header; 6283 6284 u32 pid; 6285 u32 ppid; 6286 u32 tid; 6287 u32 ptid; 6288 u64 time; 6289 } event_id; 6290 }; 6291 6292 static int perf_event_task_match(struct perf_event *event) 6293 { 6294 return event->attr.comm || event->attr.mmap || 6295 event->attr.mmap2 || event->attr.mmap_data || 6296 event->attr.task; 6297 } 6298 6299 static void perf_event_task_output(struct perf_event *event, 6300 void *data) 6301 { 6302 struct perf_task_event *task_event = data; 6303 struct perf_output_handle handle; 6304 struct perf_sample_data sample; 6305 struct task_struct *task = task_event->task; 6306 int ret, size = task_event->event_id.header.size; 6307 6308 if (!perf_event_task_match(event)) 6309 return; 6310 6311 perf_event_header__init_id(&task_event->event_id.header, &sample, event); 6312 6313 ret = perf_output_begin(&handle, event, 6314 task_event->event_id.header.size); 6315 if (ret) 6316 goto out; 6317 6318 task_event->event_id.pid = perf_event_pid(event, task); 6319 task_event->event_id.ppid = perf_event_pid(event, current); 6320 6321 task_event->event_id.tid = perf_event_tid(event, task); 6322 task_event->event_id.ptid = perf_event_tid(event, current); 6323 6324 task_event->event_id.time = perf_event_clock(event); 6325 6326 perf_output_put(&handle, task_event->event_id); 6327 6328 perf_event__output_id_sample(event, &handle, &sample); 6329 6330 perf_output_end(&handle); 6331 out: 6332 task_event->event_id.header.size = size; 6333 } 6334 6335 static void perf_event_task(struct task_struct *task, 6336 struct perf_event_context *task_ctx, 6337 int new) 6338 { 6339 struct perf_task_event task_event; 6340 6341 if (!atomic_read(&nr_comm_events) && 6342 !atomic_read(&nr_mmap_events) && 6343 !atomic_read(&nr_task_events)) 6344 return; 6345 6346 task_event = (struct perf_task_event){ 6347 .task = task, 6348 .task_ctx = task_ctx, 6349 .event_id = { 6350 .header = { 6351 .type = new ? PERF_RECORD_FORK : PERF_RECORD_EXIT, 6352 .misc = 0, 6353 .size = sizeof(task_event.event_id), 6354 }, 6355 /* .pid */ 6356 /* .ppid */ 6357 /* .tid */ 6358 /* .ptid */ 6359 /* .time */ 6360 }, 6361 }; 6362 6363 perf_iterate_sb(perf_event_task_output, 6364 &task_event, 6365 task_ctx); 6366 } 6367 6368 void perf_event_fork(struct task_struct *task) 6369 { 6370 perf_event_task(task, NULL, 1); 6371 } 6372 6373 /* 6374 * comm tracking 6375 */ 6376 6377 struct perf_comm_event { 6378 struct task_struct *task; 6379 char *comm; 6380 int comm_size; 6381 6382 struct { 6383 struct perf_event_header header; 6384 6385 u32 pid; 6386 u32 tid; 6387 } event_id; 6388 }; 6389 6390 static int perf_event_comm_match(struct perf_event *event) 6391 { 6392 return event->attr.comm; 6393 } 6394 6395 static void perf_event_comm_output(struct perf_event *event, 6396 void *data) 6397 { 6398 struct perf_comm_event *comm_event = data; 6399 struct perf_output_handle handle; 6400 struct perf_sample_data sample; 6401 int size = comm_event->event_id.header.size; 6402 int ret; 6403 6404 if (!perf_event_comm_match(event)) 6405 return; 6406 6407 perf_event_header__init_id(&comm_event->event_id.header, &sample, event); 6408 ret = perf_output_begin(&handle, event, 6409 comm_event->event_id.header.size); 6410 6411 if (ret) 6412 goto out; 6413 6414 comm_event->event_id.pid = perf_event_pid(event, comm_event->task); 6415 comm_event->event_id.tid = perf_event_tid(event, comm_event->task); 6416 6417 perf_output_put(&handle, comm_event->event_id); 6418 __output_copy(&handle, comm_event->comm, 6419 comm_event->comm_size); 6420 6421 perf_event__output_id_sample(event, &handle, &sample); 6422 6423 perf_output_end(&handle); 6424 out: 6425 comm_event->event_id.header.size = size; 6426 } 6427 6428 static void perf_event_comm_event(struct perf_comm_event *comm_event) 6429 { 6430 char comm[TASK_COMM_LEN]; 6431 unsigned int size; 6432 6433 memset(comm, 0, sizeof(comm)); 6434 strlcpy(comm, comm_event->task->comm, sizeof(comm)); 6435 size = ALIGN(strlen(comm)+1, sizeof(u64)); 6436 6437 comm_event->comm = comm; 6438 comm_event->comm_size = size; 6439 6440 comm_event->event_id.header.size = sizeof(comm_event->event_id) + size; 6441 6442 perf_iterate_sb(perf_event_comm_output, 6443 comm_event, 6444 NULL); 6445 } 6446 6447 void perf_event_comm(struct task_struct *task, bool exec) 6448 { 6449 struct perf_comm_event comm_event; 6450 6451 if (!atomic_read(&nr_comm_events)) 6452 return; 6453 6454 comm_event = (struct perf_comm_event){ 6455 .task = task, 6456 /* .comm */ 6457 /* .comm_size */ 6458 .event_id = { 6459 .header = { 6460 .type = PERF_RECORD_COMM, 6461 .misc = exec ? PERF_RECORD_MISC_COMM_EXEC : 0, 6462 /* .size */ 6463 }, 6464 /* .pid */ 6465 /* .tid */ 6466 }, 6467 }; 6468 6469 perf_event_comm_event(&comm_event); 6470 } 6471 6472 /* 6473 * mmap tracking 6474 */ 6475 6476 struct perf_mmap_event { 6477 struct vm_area_struct *vma; 6478 6479 const char *file_name; 6480 int file_size; 6481 int maj, min; 6482 u64 ino; 6483 u64 ino_generation; 6484 u32 prot, flags; 6485 6486 struct { 6487 struct perf_event_header header; 6488 6489 u32 pid; 6490 u32 tid; 6491 u64 start; 6492 u64 len; 6493 u64 pgoff; 6494 } event_id; 6495 }; 6496 6497 static int perf_event_mmap_match(struct perf_event *event, 6498 void *data) 6499 { 6500 struct perf_mmap_event *mmap_event = data; 6501 struct vm_area_struct *vma = mmap_event->vma; 6502 int executable = vma->vm_flags & VM_EXEC; 6503 6504 return (!executable && event->attr.mmap_data) || 6505 (executable && (event->attr.mmap || event->attr.mmap2)); 6506 } 6507 6508 static void perf_event_mmap_output(struct perf_event *event, 6509 void *data) 6510 { 6511 struct perf_mmap_event *mmap_event = data; 6512 struct perf_output_handle handle; 6513 struct perf_sample_data sample; 6514 int size = mmap_event->event_id.header.size; 6515 int ret; 6516 6517 if (!perf_event_mmap_match(event, data)) 6518 return; 6519 6520 if (event->attr.mmap2) { 6521 mmap_event->event_id.header.type = PERF_RECORD_MMAP2; 6522 mmap_event->event_id.header.size += sizeof(mmap_event->maj); 6523 mmap_event->event_id.header.size += sizeof(mmap_event->min); 6524 mmap_event->event_id.header.size += sizeof(mmap_event->ino); 6525 mmap_event->event_id.header.size += sizeof(mmap_event->ino_generation); 6526 mmap_event->event_id.header.size += sizeof(mmap_event->prot); 6527 mmap_event->event_id.header.size += sizeof(mmap_event->flags); 6528 } 6529 6530 perf_event_header__init_id(&mmap_event->event_id.header, &sample, event); 6531 ret = perf_output_begin(&handle, event, 6532 mmap_event->event_id.header.size); 6533 if (ret) 6534 goto out; 6535 6536 mmap_event->event_id.pid = perf_event_pid(event, current); 6537 mmap_event->event_id.tid = perf_event_tid(event, current); 6538 6539 perf_output_put(&handle, mmap_event->event_id); 6540 6541 if (event->attr.mmap2) { 6542 perf_output_put(&handle, mmap_event->maj); 6543 perf_output_put(&handle, mmap_event->min); 6544 perf_output_put(&handle, mmap_event->ino); 6545 perf_output_put(&handle, mmap_event->ino_generation); 6546 perf_output_put(&handle, mmap_event->prot); 6547 perf_output_put(&handle, mmap_event->flags); 6548 } 6549 6550 __output_copy(&handle, mmap_event->file_name, 6551 mmap_event->file_size); 6552 6553 perf_event__output_id_sample(event, &handle, &sample); 6554 6555 perf_output_end(&handle); 6556 out: 6557 mmap_event->event_id.header.size = size; 6558 } 6559 6560 static void perf_event_mmap_event(struct perf_mmap_event *mmap_event) 6561 { 6562 struct vm_area_struct *vma = mmap_event->vma; 6563 struct file *file = vma->vm_file; 6564 int maj = 0, min = 0; 6565 u64 ino = 0, gen = 0; 6566 u32 prot = 0, flags = 0; 6567 unsigned int size; 6568 char tmp[16]; 6569 char *buf = NULL; 6570 char *name; 6571 6572 if (file) { 6573 struct inode *inode; 6574 dev_t dev; 6575 6576 buf = kmalloc(PATH_MAX, GFP_KERNEL); 6577 if (!buf) { 6578 name = "//enomem"; 6579 goto cpy_name; 6580 } 6581 /* 6582 * d_path() works from the end of the rb backwards, so we 6583 * need to add enough zero bytes after the string to handle 6584 * the 64bit alignment we do later. 6585 */ 6586 name = file_path(file, buf, PATH_MAX - sizeof(u64)); 6587 if (IS_ERR(name)) { 6588 name = "//toolong"; 6589 goto cpy_name; 6590 } 6591 inode = file_inode(vma->vm_file); 6592 dev = inode->i_sb->s_dev; 6593 ino = inode->i_ino; 6594 gen = inode->i_generation; 6595 maj = MAJOR(dev); 6596 min = MINOR(dev); 6597 6598 if (vma->vm_flags & VM_READ) 6599 prot |= PROT_READ; 6600 if (vma->vm_flags & VM_WRITE) 6601 prot |= PROT_WRITE; 6602 if (vma->vm_flags & VM_EXEC) 6603 prot |= PROT_EXEC; 6604 6605 if (vma->vm_flags & VM_MAYSHARE) 6606 flags = MAP_SHARED; 6607 else 6608 flags = MAP_PRIVATE; 6609 6610 if (vma->vm_flags & VM_DENYWRITE) 6611 flags |= MAP_DENYWRITE; 6612 if (vma->vm_flags & VM_MAYEXEC) 6613 flags |= MAP_EXECUTABLE; 6614 if (vma->vm_flags & VM_LOCKED) 6615 flags |= MAP_LOCKED; 6616 if (vma->vm_flags & VM_HUGETLB) 6617 flags |= MAP_HUGETLB; 6618 6619 goto got_name; 6620 } else { 6621 if (vma->vm_ops && vma->vm_ops->name) { 6622 name = (char *) vma->vm_ops->name(vma); 6623 if (name) 6624 goto cpy_name; 6625 } 6626 6627 name = (char *)arch_vma_name(vma); 6628 if (name) 6629 goto cpy_name; 6630 6631 if (vma->vm_start <= vma->vm_mm->start_brk && 6632 vma->vm_end >= vma->vm_mm->brk) { 6633 name = "[heap]"; 6634 goto cpy_name; 6635 } 6636 if (vma->vm_start <= vma->vm_mm->start_stack && 6637 vma->vm_end >= vma->vm_mm->start_stack) { 6638 name = "[stack]"; 6639 goto cpy_name; 6640 } 6641 6642 name = "//anon"; 6643 goto cpy_name; 6644 } 6645 6646 cpy_name: 6647 strlcpy(tmp, name, sizeof(tmp)); 6648 name = tmp; 6649 got_name: 6650 /* 6651 * Since our buffer works in 8 byte units we need to align our string 6652 * size to a multiple of 8. However, we must guarantee the tail end is 6653 * zero'd out to avoid leaking random bits to userspace. 6654 */ 6655 size = strlen(name)+1; 6656 while (!IS_ALIGNED(size, sizeof(u64))) 6657 name[size++] = '\0'; 6658 6659 mmap_event->file_name = name; 6660 mmap_event->file_size = size; 6661 mmap_event->maj = maj; 6662 mmap_event->min = min; 6663 mmap_event->ino = ino; 6664 mmap_event->ino_generation = gen; 6665 mmap_event->prot = prot; 6666 mmap_event->flags = flags; 6667 6668 if (!(vma->vm_flags & VM_EXEC)) 6669 mmap_event->event_id.header.misc |= PERF_RECORD_MISC_MMAP_DATA; 6670 6671 mmap_event->event_id.header.size = sizeof(mmap_event->event_id) + size; 6672 6673 perf_iterate_sb(perf_event_mmap_output, 6674 mmap_event, 6675 NULL); 6676 6677 kfree(buf); 6678 } 6679 6680 /* 6681 * Check whether inode and address range match filter criteria. 6682 */ 6683 static bool perf_addr_filter_match(struct perf_addr_filter *filter, 6684 struct file *file, unsigned long offset, 6685 unsigned long size) 6686 { 6687 if (filter->inode != file->f_inode) 6688 return false; 6689 6690 if (filter->offset > offset + size) 6691 return false; 6692 6693 if (filter->offset + filter->size < offset) 6694 return false; 6695 6696 return true; 6697 } 6698 6699 static void __perf_addr_filters_adjust(struct perf_event *event, void *data) 6700 { 6701 struct perf_addr_filters_head *ifh = perf_event_addr_filters(event); 6702 struct vm_area_struct *vma = data; 6703 unsigned long off = vma->vm_pgoff << PAGE_SHIFT, flags; 6704 struct file *file = vma->vm_file; 6705 struct perf_addr_filter *filter; 6706 unsigned int restart = 0, count = 0; 6707 6708 if (!has_addr_filter(event)) 6709 return; 6710 6711 if (!file) 6712 return; 6713 6714 raw_spin_lock_irqsave(&ifh->lock, flags); 6715 list_for_each_entry(filter, &ifh->list, entry) { 6716 if (perf_addr_filter_match(filter, file, off, 6717 vma->vm_end - vma->vm_start)) { 6718 event->addr_filters_offs[count] = vma->vm_start; 6719 restart++; 6720 } 6721 6722 count++; 6723 } 6724 6725 if (restart) 6726 event->addr_filters_gen++; 6727 raw_spin_unlock_irqrestore(&ifh->lock, flags); 6728 6729 if (restart) 6730 perf_event_stop(event, 1); 6731 } 6732 6733 /* 6734 * Adjust all task's events' filters to the new vma 6735 */ 6736 static void perf_addr_filters_adjust(struct vm_area_struct *vma) 6737 { 6738 struct perf_event_context *ctx; 6739 int ctxn; 6740 6741 /* 6742 * Data tracing isn't supported yet and as such there is no need 6743 * to keep track of anything that isn't related to executable code: 6744 */ 6745 if (!(vma->vm_flags & VM_EXEC)) 6746 return; 6747 6748 rcu_read_lock(); 6749 for_each_task_context_nr(ctxn) { 6750 ctx = rcu_dereference(current->perf_event_ctxp[ctxn]); 6751 if (!ctx) 6752 continue; 6753 6754 perf_iterate_ctx(ctx, __perf_addr_filters_adjust, vma, true); 6755 } 6756 rcu_read_unlock(); 6757 } 6758 6759 void perf_event_mmap(struct vm_area_struct *vma) 6760 { 6761 struct perf_mmap_event mmap_event; 6762 6763 if (!atomic_read(&nr_mmap_events)) 6764 return; 6765 6766 mmap_event = (struct perf_mmap_event){ 6767 .vma = vma, 6768 /* .file_name */ 6769 /* .file_size */ 6770 .event_id = { 6771 .header = { 6772 .type = PERF_RECORD_MMAP, 6773 .misc = PERF_RECORD_MISC_USER, 6774 /* .size */ 6775 }, 6776 /* .pid */ 6777 /* .tid */ 6778 .start = vma->vm_start, 6779 .len = vma->vm_end - vma->vm_start, 6780 .pgoff = (u64)vma->vm_pgoff << PAGE_SHIFT, 6781 }, 6782 /* .maj (attr_mmap2 only) */ 6783 /* .min (attr_mmap2 only) */ 6784 /* .ino (attr_mmap2 only) */ 6785 /* .ino_generation (attr_mmap2 only) */ 6786 /* .prot (attr_mmap2 only) */ 6787 /* .flags (attr_mmap2 only) */ 6788 }; 6789 6790 perf_addr_filters_adjust(vma); 6791 perf_event_mmap_event(&mmap_event); 6792 } 6793 6794 void perf_event_aux_event(struct perf_event *event, unsigned long head, 6795 unsigned long size, u64 flags) 6796 { 6797 struct perf_output_handle handle; 6798 struct perf_sample_data sample; 6799 struct perf_aux_event { 6800 struct perf_event_header header; 6801 u64 offset; 6802 u64 size; 6803 u64 flags; 6804 } rec = { 6805 .header = { 6806 .type = PERF_RECORD_AUX, 6807 .misc = 0, 6808 .size = sizeof(rec), 6809 }, 6810 .offset = head, 6811 .size = size, 6812 .flags = flags, 6813 }; 6814 int ret; 6815 6816 perf_event_header__init_id(&rec.header, &sample, event); 6817 ret = perf_output_begin(&handle, event, rec.header.size); 6818 6819 if (ret) 6820 return; 6821 6822 perf_output_put(&handle, rec); 6823 perf_event__output_id_sample(event, &handle, &sample); 6824 6825 perf_output_end(&handle); 6826 } 6827 6828 /* 6829 * Lost/dropped samples logging 6830 */ 6831 void perf_log_lost_samples(struct perf_event *event, u64 lost) 6832 { 6833 struct perf_output_handle handle; 6834 struct perf_sample_data sample; 6835 int ret; 6836 6837 struct { 6838 struct perf_event_header header; 6839 u64 lost; 6840 } lost_samples_event = { 6841 .header = { 6842 .type = PERF_RECORD_LOST_SAMPLES, 6843 .misc = 0, 6844 .size = sizeof(lost_samples_event), 6845 }, 6846 .lost = lost, 6847 }; 6848 6849 perf_event_header__init_id(&lost_samples_event.header, &sample, event); 6850 6851 ret = perf_output_begin(&handle, event, 6852 lost_samples_event.header.size); 6853 if (ret) 6854 return; 6855 6856 perf_output_put(&handle, lost_samples_event); 6857 perf_event__output_id_sample(event, &handle, &sample); 6858 perf_output_end(&handle); 6859 } 6860 6861 /* 6862 * context_switch tracking 6863 */ 6864 6865 struct perf_switch_event { 6866 struct task_struct *task; 6867 struct task_struct *next_prev; 6868 6869 struct { 6870 struct perf_event_header header; 6871 u32 next_prev_pid; 6872 u32 next_prev_tid; 6873 } event_id; 6874 }; 6875 6876 static int perf_event_switch_match(struct perf_event *event) 6877 { 6878 return event->attr.context_switch; 6879 } 6880 6881 static void perf_event_switch_output(struct perf_event *event, void *data) 6882 { 6883 struct perf_switch_event *se = data; 6884 struct perf_output_handle handle; 6885 struct perf_sample_data sample; 6886 int ret; 6887 6888 if (!perf_event_switch_match(event)) 6889 return; 6890 6891 /* Only CPU-wide events are allowed to see next/prev pid/tid */ 6892 if (event->ctx->task) { 6893 se->event_id.header.type = PERF_RECORD_SWITCH; 6894 se->event_id.header.size = sizeof(se->event_id.header); 6895 } else { 6896 se->event_id.header.type = PERF_RECORD_SWITCH_CPU_WIDE; 6897 se->event_id.header.size = sizeof(se->event_id); 6898 se->event_id.next_prev_pid = 6899 perf_event_pid(event, se->next_prev); 6900 se->event_id.next_prev_tid = 6901 perf_event_tid(event, se->next_prev); 6902 } 6903 6904 perf_event_header__init_id(&se->event_id.header, &sample, event); 6905 6906 ret = perf_output_begin(&handle, event, se->event_id.header.size); 6907 if (ret) 6908 return; 6909 6910 if (event->ctx->task) 6911 perf_output_put(&handle, se->event_id.header); 6912 else 6913 perf_output_put(&handle, se->event_id); 6914 6915 perf_event__output_id_sample(event, &handle, &sample); 6916 6917 perf_output_end(&handle); 6918 } 6919 6920 static void perf_event_switch(struct task_struct *task, 6921 struct task_struct *next_prev, bool sched_in) 6922 { 6923 struct perf_switch_event switch_event; 6924 6925 /* N.B. caller checks nr_switch_events != 0 */ 6926 6927 switch_event = (struct perf_switch_event){ 6928 .task = task, 6929 .next_prev = next_prev, 6930 .event_id = { 6931 .header = { 6932 /* .type */ 6933 .misc = sched_in ? 0 : PERF_RECORD_MISC_SWITCH_OUT, 6934 /* .size */ 6935 }, 6936 /* .next_prev_pid */ 6937 /* .next_prev_tid */ 6938 }, 6939 }; 6940 6941 perf_iterate_sb(perf_event_switch_output, 6942 &switch_event, 6943 NULL); 6944 } 6945 6946 /* 6947 * IRQ throttle logging 6948 */ 6949 6950 static void perf_log_throttle(struct perf_event *event, int enable) 6951 { 6952 struct perf_output_handle handle; 6953 struct perf_sample_data sample; 6954 int ret; 6955 6956 struct { 6957 struct perf_event_header header; 6958 u64 time; 6959 u64 id; 6960 u64 stream_id; 6961 } throttle_event = { 6962 .header = { 6963 .type = PERF_RECORD_THROTTLE, 6964 .misc = 0, 6965 .size = sizeof(throttle_event), 6966 }, 6967 .time = perf_event_clock(event), 6968 .id = primary_event_id(event), 6969 .stream_id = event->id, 6970 }; 6971 6972 if (enable) 6973 throttle_event.header.type = PERF_RECORD_UNTHROTTLE; 6974 6975 perf_event_header__init_id(&throttle_event.header, &sample, event); 6976 6977 ret = perf_output_begin(&handle, event, 6978 throttle_event.header.size); 6979 if (ret) 6980 return; 6981 6982 perf_output_put(&handle, throttle_event); 6983 perf_event__output_id_sample(event, &handle, &sample); 6984 perf_output_end(&handle); 6985 } 6986 6987 static void perf_log_itrace_start(struct perf_event *event) 6988 { 6989 struct perf_output_handle handle; 6990 struct perf_sample_data sample; 6991 struct perf_aux_event { 6992 struct perf_event_header header; 6993 u32 pid; 6994 u32 tid; 6995 } rec; 6996 int ret; 6997 6998 if (event->parent) 6999 event = event->parent; 7000 7001 if (!(event->pmu->capabilities & PERF_PMU_CAP_ITRACE) || 7002 event->hw.itrace_started) 7003 return; 7004 7005 rec.header.type = PERF_RECORD_ITRACE_START; 7006 rec.header.misc = 0; 7007 rec.header.size = sizeof(rec); 7008 rec.pid = perf_event_pid(event, current); 7009 rec.tid = perf_event_tid(event, current); 7010 7011 perf_event_header__init_id(&rec.header, &sample, event); 7012 ret = perf_output_begin(&handle, event, rec.header.size); 7013 7014 if (ret) 7015 return; 7016 7017 perf_output_put(&handle, rec); 7018 perf_event__output_id_sample(event, &handle, &sample); 7019 7020 perf_output_end(&handle); 7021 } 7022 7023 /* 7024 * Generic event overflow handling, sampling. 7025 */ 7026 7027 static int __perf_event_overflow(struct perf_event *event, 7028 int throttle, struct perf_sample_data *data, 7029 struct pt_regs *regs) 7030 { 7031 int events = atomic_read(&event->event_limit); 7032 struct hw_perf_event *hwc = &event->hw; 7033 u64 seq; 7034 int ret = 0; 7035 7036 /* 7037 * Non-sampling counters might still use the PMI to fold short 7038 * hardware counters, ignore those. 7039 */ 7040 if (unlikely(!is_sampling_event(event))) 7041 return 0; 7042 7043 seq = __this_cpu_read(perf_throttled_seq); 7044 if (seq != hwc->interrupts_seq) { 7045 hwc->interrupts_seq = seq; 7046 hwc->interrupts = 1; 7047 } else { 7048 hwc->interrupts++; 7049 if (unlikely(throttle 7050 && hwc->interrupts >= max_samples_per_tick)) { 7051 __this_cpu_inc(perf_throttled_count); 7052 tick_dep_set_cpu(smp_processor_id(), TICK_DEP_BIT_PERF_EVENTS); 7053 hwc->interrupts = MAX_INTERRUPTS; 7054 perf_log_throttle(event, 0); 7055 ret = 1; 7056 } 7057 } 7058 7059 if (event->attr.freq) { 7060 u64 now = perf_clock(); 7061 s64 delta = now - hwc->freq_time_stamp; 7062 7063 hwc->freq_time_stamp = now; 7064 7065 if (delta > 0 && delta < 2*TICK_NSEC) 7066 perf_adjust_period(event, delta, hwc->last_period, true); 7067 } 7068 7069 /* 7070 * XXX event_limit might not quite work as expected on inherited 7071 * events 7072 */ 7073 7074 event->pending_kill = POLL_IN; 7075 if (events && atomic_dec_and_test(&event->event_limit)) { 7076 ret = 1; 7077 event->pending_kill = POLL_HUP; 7078 event->pending_disable = 1; 7079 irq_work_queue(&event->pending); 7080 } 7081 7082 READ_ONCE(event->overflow_handler)(event, data, regs); 7083 7084 if (*perf_event_fasync(event) && event->pending_kill) { 7085 event->pending_wakeup = 1; 7086 irq_work_queue(&event->pending); 7087 } 7088 7089 return ret; 7090 } 7091 7092 int perf_event_overflow(struct perf_event *event, 7093 struct perf_sample_data *data, 7094 struct pt_regs *regs) 7095 { 7096 return __perf_event_overflow(event, 1, data, regs); 7097 } 7098 7099 /* 7100 * Generic software event infrastructure 7101 */ 7102 7103 struct swevent_htable { 7104 struct swevent_hlist *swevent_hlist; 7105 struct mutex hlist_mutex; 7106 int hlist_refcount; 7107 7108 /* Recursion avoidance in each contexts */ 7109 int recursion[PERF_NR_CONTEXTS]; 7110 }; 7111 7112 static DEFINE_PER_CPU(struct swevent_htable, swevent_htable); 7113 7114 /* 7115 * We directly increment event->count and keep a second value in 7116 * event->hw.period_left to count intervals. This period event 7117 * is kept in the range [-sample_period, 0] so that we can use the 7118 * sign as trigger. 7119 */ 7120 7121 u64 perf_swevent_set_period(struct perf_event *event) 7122 { 7123 struct hw_perf_event *hwc = &event->hw; 7124 u64 period = hwc->last_period; 7125 u64 nr, offset; 7126 s64 old, val; 7127 7128 hwc->last_period = hwc->sample_period; 7129 7130 again: 7131 old = val = local64_read(&hwc->period_left); 7132 if (val < 0) 7133 return 0; 7134 7135 nr = div64_u64(period + val, period); 7136 offset = nr * period; 7137 val -= offset; 7138 if (local64_cmpxchg(&hwc->period_left, old, val) != old) 7139 goto again; 7140 7141 return nr; 7142 } 7143 7144 static void perf_swevent_overflow(struct perf_event *event, u64 overflow, 7145 struct perf_sample_data *data, 7146 struct pt_regs *regs) 7147 { 7148 struct hw_perf_event *hwc = &event->hw; 7149 int throttle = 0; 7150 7151 if (!overflow) 7152 overflow = perf_swevent_set_period(event); 7153 7154 if (hwc->interrupts == MAX_INTERRUPTS) 7155 return; 7156 7157 for (; overflow; overflow--) { 7158 if (__perf_event_overflow(event, throttle, 7159 data, regs)) { 7160 /* 7161 * We inhibit the overflow from happening when 7162 * hwc->interrupts == MAX_INTERRUPTS. 7163 */ 7164 break; 7165 } 7166 throttle = 1; 7167 } 7168 } 7169 7170 static void perf_swevent_event(struct perf_event *event, u64 nr, 7171 struct perf_sample_data *data, 7172 struct pt_regs *regs) 7173 { 7174 struct hw_perf_event *hwc = &event->hw; 7175 7176 local64_add(nr, &event->count); 7177 7178 if (!regs) 7179 return; 7180 7181 if (!is_sampling_event(event)) 7182 return; 7183 7184 if ((event->attr.sample_type & PERF_SAMPLE_PERIOD) && !event->attr.freq) { 7185 data->period = nr; 7186 return perf_swevent_overflow(event, 1, data, regs); 7187 } else 7188 data->period = event->hw.last_period; 7189 7190 if (nr == 1 && hwc->sample_period == 1 && !event->attr.freq) 7191 return perf_swevent_overflow(event, 1, data, regs); 7192 7193 if (local64_add_negative(nr, &hwc->period_left)) 7194 return; 7195 7196 perf_swevent_overflow(event, 0, data, regs); 7197 } 7198 7199 static int perf_exclude_event(struct perf_event *event, 7200 struct pt_regs *regs) 7201 { 7202 if (event->hw.state & PERF_HES_STOPPED) 7203 return 1; 7204 7205 if (regs) { 7206 if (event->attr.exclude_user && user_mode(regs)) 7207 return 1; 7208 7209 if (event->attr.exclude_kernel && !user_mode(regs)) 7210 return 1; 7211 } 7212 7213 return 0; 7214 } 7215 7216 static int perf_swevent_match(struct perf_event *event, 7217 enum perf_type_id type, 7218 u32 event_id, 7219 struct perf_sample_data *data, 7220 struct pt_regs *regs) 7221 { 7222 if (event->attr.type != type) 7223 return 0; 7224 7225 if (event->attr.config != event_id) 7226 return 0; 7227 7228 if (perf_exclude_event(event, regs)) 7229 return 0; 7230 7231 return 1; 7232 } 7233 7234 static inline u64 swevent_hash(u64 type, u32 event_id) 7235 { 7236 u64 val = event_id | (type << 32); 7237 7238 return hash_64(val, SWEVENT_HLIST_BITS); 7239 } 7240 7241 static inline struct hlist_head * 7242 __find_swevent_head(struct swevent_hlist *hlist, u64 type, u32 event_id) 7243 { 7244 u64 hash = swevent_hash(type, event_id); 7245 7246 return &hlist->heads[hash]; 7247 } 7248 7249 /* For the read side: events when they trigger */ 7250 static inline struct hlist_head * 7251 find_swevent_head_rcu(struct swevent_htable *swhash, u64 type, u32 event_id) 7252 { 7253 struct swevent_hlist *hlist; 7254 7255 hlist = rcu_dereference(swhash->swevent_hlist); 7256 if (!hlist) 7257 return NULL; 7258 7259 return __find_swevent_head(hlist, type, event_id); 7260 } 7261 7262 /* For the event head insertion and removal in the hlist */ 7263 static inline struct hlist_head * 7264 find_swevent_head(struct swevent_htable *swhash, struct perf_event *event) 7265 { 7266 struct swevent_hlist *hlist; 7267 u32 event_id = event->attr.config; 7268 u64 type = event->attr.type; 7269 7270 /* 7271 * Event scheduling is always serialized against hlist allocation 7272 * and release. Which makes the protected version suitable here. 7273 * The context lock guarantees that. 7274 */ 7275 hlist = rcu_dereference_protected(swhash->swevent_hlist, 7276 lockdep_is_held(&event->ctx->lock)); 7277 if (!hlist) 7278 return NULL; 7279 7280 return __find_swevent_head(hlist, type, event_id); 7281 } 7282 7283 static void do_perf_sw_event(enum perf_type_id type, u32 event_id, 7284 u64 nr, 7285 struct perf_sample_data *data, 7286 struct pt_regs *regs) 7287 { 7288 struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable); 7289 struct perf_event *event; 7290 struct hlist_head *head; 7291 7292 rcu_read_lock(); 7293 head = find_swevent_head_rcu(swhash, type, event_id); 7294 if (!head) 7295 goto end; 7296 7297 hlist_for_each_entry_rcu(event, head, hlist_entry) { 7298 if (perf_swevent_match(event, type, event_id, data, regs)) 7299 perf_swevent_event(event, nr, data, regs); 7300 } 7301 end: 7302 rcu_read_unlock(); 7303 } 7304 7305 DEFINE_PER_CPU(struct pt_regs, __perf_regs[4]); 7306 7307 int perf_swevent_get_recursion_context(void) 7308 { 7309 struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable); 7310 7311 return get_recursion_context(swhash->recursion); 7312 } 7313 EXPORT_SYMBOL_GPL(perf_swevent_get_recursion_context); 7314 7315 void perf_swevent_put_recursion_context(int rctx) 7316 { 7317 struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable); 7318 7319 put_recursion_context(swhash->recursion, rctx); 7320 } 7321 7322 void ___perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr) 7323 { 7324 struct perf_sample_data data; 7325 7326 if (WARN_ON_ONCE(!regs)) 7327 return; 7328 7329 perf_sample_data_init(&data, addr, 0); 7330 do_perf_sw_event(PERF_TYPE_SOFTWARE, event_id, nr, &data, regs); 7331 } 7332 7333 void __perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr) 7334 { 7335 int rctx; 7336 7337 preempt_disable_notrace(); 7338 rctx = perf_swevent_get_recursion_context(); 7339 if (unlikely(rctx < 0)) 7340 goto fail; 7341 7342 ___perf_sw_event(event_id, nr, regs, addr); 7343 7344 perf_swevent_put_recursion_context(rctx); 7345 fail: 7346 preempt_enable_notrace(); 7347 } 7348 7349 static void perf_swevent_read(struct perf_event *event) 7350 { 7351 } 7352 7353 static int perf_swevent_add(struct perf_event *event, int flags) 7354 { 7355 struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable); 7356 struct hw_perf_event *hwc = &event->hw; 7357 struct hlist_head *head; 7358 7359 if (is_sampling_event(event)) { 7360 hwc->last_period = hwc->sample_period; 7361 perf_swevent_set_period(event); 7362 } 7363 7364 hwc->state = !(flags & PERF_EF_START); 7365 7366 head = find_swevent_head(swhash, event); 7367 if (WARN_ON_ONCE(!head)) 7368 return -EINVAL; 7369 7370 hlist_add_head_rcu(&event->hlist_entry, head); 7371 perf_event_update_userpage(event); 7372 7373 return 0; 7374 } 7375 7376 static void perf_swevent_del(struct perf_event *event, int flags) 7377 { 7378 hlist_del_rcu(&event->hlist_entry); 7379 } 7380 7381 static void perf_swevent_start(struct perf_event *event, int flags) 7382 { 7383 event->hw.state = 0; 7384 } 7385 7386 static void perf_swevent_stop(struct perf_event *event, int flags) 7387 { 7388 event->hw.state = PERF_HES_STOPPED; 7389 } 7390 7391 /* Deref the hlist from the update side */ 7392 static inline struct swevent_hlist * 7393 swevent_hlist_deref(struct swevent_htable *swhash) 7394 { 7395 return rcu_dereference_protected(swhash->swevent_hlist, 7396 lockdep_is_held(&swhash->hlist_mutex)); 7397 } 7398 7399 static void swevent_hlist_release(struct swevent_htable *swhash) 7400 { 7401 struct swevent_hlist *hlist = swevent_hlist_deref(swhash); 7402 7403 if (!hlist) 7404 return; 7405 7406 RCU_INIT_POINTER(swhash->swevent_hlist, NULL); 7407 kfree_rcu(hlist, rcu_head); 7408 } 7409 7410 static void swevent_hlist_put_cpu(int cpu) 7411 { 7412 struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu); 7413 7414 mutex_lock(&swhash->hlist_mutex); 7415 7416 if (!--swhash->hlist_refcount) 7417 swevent_hlist_release(swhash); 7418 7419 mutex_unlock(&swhash->hlist_mutex); 7420 } 7421 7422 static void swevent_hlist_put(void) 7423 { 7424 int cpu; 7425 7426 for_each_possible_cpu(cpu) 7427 swevent_hlist_put_cpu(cpu); 7428 } 7429 7430 static int swevent_hlist_get_cpu(int cpu) 7431 { 7432 struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu); 7433 int err = 0; 7434 7435 mutex_lock(&swhash->hlist_mutex); 7436 if (!swevent_hlist_deref(swhash) && cpu_online(cpu)) { 7437 struct swevent_hlist *hlist; 7438 7439 hlist = kzalloc(sizeof(*hlist), GFP_KERNEL); 7440 if (!hlist) { 7441 err = -ENOMEM; 7442 goto exit; 7443 } 7444 rcu_assign_pointer(swhash->swevent_hlist, hlist); 7445 } 7446 swhash->hlist_refcount++; 7447 exit: 7448 mutex_unlock(&swhash->hlist_mutex); 7449 7450 return err; 7451 } 7452 7453 static int swevent_hlist_get(void) 7454 { 7455 int err, cpu, failed_cpu; 7456 7457 get_online_cpus(); 7458 for_each_possible_cpu(cpu) { 7459 err = swevent_hlist_get_cpu(cpu); 7460 if (err) { 7461 failed_cpu = cpu; 7462 goto fail; 7463 } 7464 } 7465 put_online_cpus(); 7466 7467 return 0; 7468 fail: 7469 for_each_possible_cpu(cpu) { 7470 if (cpu == failed_cpu) 7471 break; 7472 swevent_hlist_put_cpu(cpu); 7473 } 7474 7475 put_online_cpus(); 7476 return err; 7477 } 7478 7479 struct static_key perf_swevent_enabled[PERF_COUNT_SW_MAX]; 7480 7481 static void sw_perf_event_destroy(struct perf_event *event) 7482 { 7483 u64 event_id = event->attr.config; 7484 7485 WARN_ON(event->parent); 7486 7487 static_key_slow_dec(&perf_swevent_enabled[event_id]); 7488 swevent_hlist_put(); 7489 } 7490 7491 static int perf_swevent_init(struct perf_event *event) 7492 { 7493 u64 event_id = event->attr.config; 7494 7495 if (event->attr.type != PERF_TYPE_SOFTWARE) 7496 return -ENOENT; 7497 7498 /* 7499 * no branch sampling for software events 7500 */ 7501 if (has_branch_stack(event)) 7502 return -EOPNOTSUPP; 7503 7504 switch (event_id) { 7505 case PERF_COUNT_SW_CPU_CLOCK: 7506 case PERF_COUNT_SW_TASK_CLOCK: 7507 return -ENOENT; 7508 7509 default: 7510 break; 7511 } 7512 7513 if (event_id >= PERF_COUNT_SW_MAX) 7514 return -ENOENT; 7515 7516 if (!event->parent) { 7517 int err; 7518 7519 err = swevent_hlist_get(); 7520 if (err) 7521 return err; 7522 7523 static_key_slow_inc(&perf_swevent_enabled[event_id]); 7524 event->destroy = sw_perf_event_destroy; 7525 } 7526 7527 return 0; 7528 } 7529 7530 static struct pmu perf_swevent = { 7531 .task_ctx_nr = perf_sw_context, 7532 7533 .capabilities = PERF_PMU_CAP_NO_NMI, 7534 7535 .event_init = perf_swevent_init, 7536 .add = perf_swevent_add, 7537 .del = perf_swevent_del, 7538 .start = perf_swevent_start, 7539 .stop = perf_swevent_stop, 7540 .read = perf_swevent_read, 7541 }; 7542 7543 #ifdef CONFIG_EVENT_TRACING 7544 7545 static int perf_tp_filter_match(struct perf_event *event, 7546 struct perf_sample_data *data) 7547 { 7548 void *record = data->raw->frag.data; 7549 7550 /* only top level events have filters set */ 7551 if (event->parent) 7552 event = event->parent; 7553 7554 if (likely(!event->filter) || filter_match_preds(event->filter, record)) 7555 return 1; 7556 return 0; 7557 } 7558 7559 static int perf_tp_event_match(struct perf_event *event, 7560 struct perf_sample_data *data, 7561 struct pt_regs *regs) 7562 { 7563 if (event->hw.state & PERF_HES_STOPPED) 7564 return 0; 7565 /* 7566 * All tracepoints are from kernel-space. 7567 */ 7568 if (event->attr.exclude_kernel) 7569 return 0; 7570 7571 if (!perf_tp_filter_match(event, data)) 7572 return 0; 7573 7574 return 1; 7575 } 7576 7577 void perf_trace_run_bpf_submit(void *raw_data, int size, int rctx, 7578 struct trace_event_call *call, u64 count, 7579 struct pt_regs *regs, struct hlist_head *head, 7580 struct task_struct *task) 7581 { 7582 struct bpf_prog *prog = call->prog; 7583 7584 if (prog) { 7585 *(struct pt_regs **)raw_data = regs; 7586 if (!trace_call_bpf(prog, raw_data) || hlist_empty(head)) { 7587 perf_swevent_put_recursion_context(rctx); 7588 return; 7589 } 7590 } 7591 perf_tp_event(call->event.type, count, raw_data, size, regs, head, 7592 rctx, task); 7593 } 7594 EXPORT_SYMBOL_GPL(perf_trace_run_bpf_submit); 7595 7596 void perf_tp_event(u16 event_type, u64 count, void *record, int entry_size, 7597 struct pt_regs *regs, struct hlist_head *head, int rctx, 7598 struct task_struct *task) 7599 { 7600 struct perf_sample_data data; 7601 struct perf_event *event; 7602 7603 struct perf_raw_record raw = { 7604 .frag = { 7605 .size = entry_size, 7606 .data = record, 7607 }, 7608 }; 7609 7610 perf_sample_data_init(&data, 0, 0); 7611 data.raw = &raw; 7612 7613 perf_trace_buf_update(record, event_type); 7614 7615 hlist_for_each_entry_rcu(event, head, hlist_entry) { 7616 if (perf_tp_event_match(event, &data, regs)) 7617 perf_swevent_event(event, count, &data, regs); 7618 } 7619 7620 /* 7621 * If we got specified a target task, also iterate its context and 7622 * deliver this event there too. 7623 */ 7624 if (task && task != current) { 7625 struct perf_event_context *ctx; 7626 struct trace_entry *entry = record; 7627 7628 rcu_read_lock(); 7629 ctx = rcu_dereference(task->perf_event_ctxp[perf_sw_context]); 7630 if (!ctx) 7631 goto unlock; 7632 7633 list_for_each_entry_rcu(event, &ctx->event_list, event_entry) { 7634 if (event->attr.type != PERF_TYPE_TRACEPOINT) 7635 continue; 7636 if (event->attr.config != entry->type) 7637 continue; 7638 if (perf_tp_event_match(event, &data, regs)) 7639 perf_swevent_event(event, count, &data, regs); 7640 } 7641 unlock: 7642 rcu_read_unlock(); 7643 } 7644 7645 perf_swevent_put_recursion_context(rctx); 7646 } 7647 EXPORT_SYMBOL_GPL(perf_tp_event); 7648 7649 static void tp_perf_event_destroy(struct perf_event *event) 7650 { 7651 perf_trace_destroy(event); 7652 } 7653 7654 static int perf_tp_event_init(struct perf_event *event) 7655 { 7656 int err; 7657 7658 if (event->attr.type != PERF_TYPE_TRACEPOINT) 7659 return -ENOENT; 7660 7661 /* 7662 * no branch sampling for tracepoint events 7663 */ 7664 if (has_branch_stack(event)) 7665 return -EOPNOTSUPP; 7666 7667 err = perf_trace_init(event); 7668 if (err) 7669 return err; 7670 7671 event->destroy = tp_perf_event_destroy; 7672 7673 return 0; 7674 } 7675 7676 static struct pmu perf_tracepoint = { 7677 .task_ctx_nr = perf_sw_context, 7678 7679 .event_init = perf_tp_event_init, 7680 .add = perf_trace_add, 7681 .del = perf_trace_del, 7682 .start = perf_swevent_start, 7683 .stop = perf_swevent_stop, 7684 .read = perf_swevent_read, 7685 }; 7686 7687 static inline void perf_tp_register(void) 7688 { 7689 perf_pmu_register(&perf_tracepoint, "tracepoint", PERF_TYPE_TRACEPOINT); 7690 } 7691 7692 static void perf_event_free_filter(struct perf_event *event) 7693 { 7694 ftrace_profile_free_filter(event); 7695 } 7696 7697 #ifdef CONFIG_BPF_SYSCALL 7698 static void bpf_overflow_handler(struct perf_event *event, 7699 struct perf_sample_data *data, 7700 struct pt_regs *regs) 7701 { 7702 struct bpf_perf_event_data_kern ctx = { 7703 .data = data, 7704 .regs = regs, 7705 }; 7706 int ret = 0; 7707 7708 preempt_disable(); 7709 if (unlikely(__this_cpu_inc_return(bpf_prog_active) != 1)) 7710 goto out; 7711 rcu_read_lock(); 7712 ret = BPF_PROG_RUN(event->prog, (void *)&ctx); 7713 rcu_read_unlock(); 7714 out: 7715 __this_cpu_dec(bpf_prog_active); 7716 preempt_enable(); 7717 if (!ret) 7718 return; 7719 7720 event->orig_overflow_handler(event, data, regs); 7721 } 7722 7723 static int perf_event_set_bpf_handler(struct perf_event *event, u32 prog_fd) 7724 { 7725 struct bpf_prog *prog; 7726 7727 if (event->overflow_handler_context) 7728 /* hw breakpoint or kernel counter */ 7729 return -EINVAL; 7730 7731 if (event->prog) 7732 return -EEXIST; 7733 7734 prog = bpf_prog_get_type(prog_fd, BPF_PROG_TYPE_PERF_EVENT); 7735 if (IS_ERR(prog)) 7736 return PTR_ERR(prog); 7737 7738 event->prog = prog; 7739 event->orig_overflow_handler = READ_ONCE(event->overflow_handler); 7740 WRITE_ONCE(event->overflow_handler, bpf_overflow_handler); 7741 return 0; 7742 } 7743 7744 static void perf_event_free_bpf_handler(struct perf_event *event) 7745 { 7746 struct bpf_prog *prog = event->prog; 7747 7748 if (!prog) 7749 return; 7750 7751 WRITE_ONCE(event->overflow_handler, event->orig_overflow_handler); 7752 event->prog = NULL; 7753 bpf_prog_put(prog); 7754 } 7755 #else 7756 static int perf_event_set_bpf_handler(struct perf_event *event, u32 prog_fd) 7757 { 7758 return -EOPNOTSUPP; 7759 } 7760 static void perf_event_free_bpf_handler(struct perf_event *event) 7761 { 7762 } 7763 #endif 7764 7765 static int perf_event_set_bpf_prog(struct perf_event *event, u32 prog_fd) 7766 { 7767 bool is_kprobe, is_tracepoint; 7768 struct bpf_prog *prog; 7769 7770 if (event->attr.type == PERF_TYPE_HARDWARE || 7771 event->attr.type == PERF_TYPE_SOFTWARE) 7772 return perf_event_set_bpf_handler(event, prog_fd); 7773 7774 if (event->attr.type != PERF_TYPE_TRACEPOINT) 7775 return -EINVAL; 7776 7777 if (event->tp_event->prog) 7778 return -EEXIST; 7779 7780 is_kprobe = event->tp_event->flags & TRACE_EVENT_FL_UKPROBE; 7781 is_tracepoint = event->tp_event->flags & TRACE_EVENT_FL_TRACEPOINT; 7782 if (!is_kprobe && !is_tracepoint) 7783 /* bpf programs can only be attached to u/kprobe or tracepoint */ 7784 return -EINVAL; 7785 7786 prog = bpf_prog_get(prog_fd); 7787 if (IS_ERR(prog)) 7788 return PTR_ERR(prog); 7789 7790 if ((is_kprobe && prog->type != BPF_PROG_TYPE_KPROBE) || 7791 (is_tracepoint && prog->type != BPF_PROG_TYPE_TRACEPOINT)) { 7792 /* valid fd, but invalid bpf program type */ 7793 bpf_prog_put(prog); 7794 return -EINVAL; 7795 } 7796 7797 if (is_tracepoint) { 7798 int off = trace_event_get_offsets(event->tp_event); 7799 7800 if (prog->aux->max_ctx_offset > off) { 7801 bpf_prog_put(prog); 7802 return -EACCES; 7803 } 7804 } 7805 event->tp_event->prog = prog; 7806 7807 return 0; 7808 } 7809 7810 static void perf_event_free_bpf_prog(struct perf_event *event) 7811 { 7812 struct bpf_prog *prog; 7813 7814 perf_event_free_bpf_handler(event); 7815 7816 if (!event->tp_event) 7817 return; 7818 7819 prog = event->tp_event->prog; 7820 if (prog) { 7821 event->tp_event->prog = NULL; 7822 bpf_prog_put(prog); 7823 } 7824 } 7825 7826 #else 7827 7828 static inline void perf_tp_register(void) 7829 { 7830 } 7831 7832 static void perf_event_free_filter(struct perf_event *event) 7833 { 7834 } 7835 7836 static int perf_event_set_bpf_prog(struct perf_event *event, u32 prog_fd) 7837 { 7838 return -ENOENT; 7839 } 7840 7841 static void perf_event_free_bpf_prog(struct perf_event *event) 7842 { 7843 } 7844 #endif /* CONFIG_EVENT_TRACING */ 7845 7846 #ifdef CONFIG_HAVE_HW_BREAKPOINT 7847 void perf_bp_event(struct perf_event *bp, void *data) 7848 { 7849 struct perf_sample_data sample; 7850 struct pt_regs *regs = data; 7851 7852 perf_sample_data_init(&sample, bp->attr.bp_addr, 0); 7853 7854 if (!bp->hw.state && !perf_exclude_event(bp, regs)) 7855 perf_swevent_event(bp, 1, &sample, regs); 7856 } 7857 #endif 7858 7859 /* 7860 * Allocate a new address filter 7861 */ 7862 static struct perf_addr_filter * 7863 perf_addr_filter_new(struct perf_event *event, struct list_head *filters) 7864 { 7865 int node = cpu_to_node(event->cpu == -1 ? 0 : event->cpu); 7866 struct perf_addr_filter *filter; 7867 7868 filter = kzalloc_node(sizeof(*filter), GFP_KERNEL, node); 7869 if (!filter) 7870 return NULL; 7871 7872 INIT_LIST_HEAD(&filter->entry); 7873 list_add_tail(&filter->entry, filters); 7874 7875 return filter; 7876 } 7877 7878 static void free_filters_list(struct list_head *filters) 7879 { 7880 struct perf_addr_filter *filter, *iter; 7881 7882 list_for_each_entry_safe(filter, iter, filters, entry) { 7883 if (filter->inode) 7884 iput(filter->inode); 7885 list_del(&filter->entry); 7886 kfree(filter); 7887 } 7888 } 7889 7890 /* 7891 * Free existing address filters and optionally install new ones 7892 */ 7893 static void perf_addr_filters_splice(struct perf_event *event, 7894 struct list_head *head) 7895 { 7896 unsigned long flags; 7897 LIST_HEAD(list); 7898 7899 if (!has_addr_filter(event)) 7900 return; 7901 7902 /* don't bother with children, they don't have their own filters */ 7903 if (event->parent) 7904 return; 7905 7906 raw_spin_lock_irqsave(&event->addr_filters.lock, flags); 7907 7908 list_splice_init(&event->addr_filters.list, &list); 7909 if (head) 7910 list_splice(head, &event->addr_filters.list); 7911 7912 raw_spin_unlock_irqrestore(&event->addr_filters.lock, flags); 7913 7914 free_filters_list(&list); 7915 } 7916 7917 /* 7918 * Scan through mm's vmas and see if one of them matches the 7919 * @filter; if so, adjust filter's address range. 7920 * Called with mm::mmap_sem down for reading. 7921 */ 7922 static unsigned long perf_addr_filter_apply(struct perf_addr_filter *filter, 7923 struct mm_struct *mm) 7924 { 7925 struct vm_area_struct *vma; 7926 7927 for (vma = mm->mmap; vma; vma = vma->vm_next) { 7928 struct file *file = vma->vm_file; 7929 unsigned long off = vma->vm_pgoff << PAGE_SHIFT; 7930 unsigned long vma_size = vma->vm_end - vma->vm_start; 7931 7932 if (!file) 7933 continue; 7934 7935 if (!perf_addr_filter_match(filter, file, off, vma_size)) 7936 continue; 7937 7938 return vma->vm_start; 7939 } 7940 7941 return 0; 7942 } 7943 7944 /* 7945 * Update event's address range filters based on the 7946 * task's existing mappings, if any. 7947 */ 7948 static void perf_event_addr_filters_apply(struct perf_event *event) 7949 { 7950 struct perf_addr_filters_head *ifh = perf_event_addr_filters(event); 7951 struct task_struct *task = READ_ONCE(event->ctx->task); 7952 struct perf_addr_filter *filter; 7953 struct mm_struct *mm = NULL; 7954 unsigned int count = 0; 7955 unsigned long flags; 7956 7957 /* 7958 * We may observe TASK_TOMBSTONE, which means that the event tear-down 7959 * will stop on the parent's child_mutex that our caller is also holding 7960 */ 7961 if (task == TASK_TOMBSTONE) 7962 return; 7963 7964 mm = get_task_mm(event->ctx->task); 7965 if (!mm) 7966 goto restart; 7967 7968 down_read(&mm->mmap_sem); 7969 7970 raw_spin_lock_irqsave(&ifh->lock, flags); 7971 list_for_each_entry(filter, &ifh->list, entry) { 7972 event->addr_filters_offs[count] = 0; 7973 7974 /* 7975 * Adjust base offset if the filter is associated to a binary 7976 * that needs to be mapped: 7977 */ 7978 if (filter->inode) 7979 event->addr_filters_offs[count] = 7980 perf_addr_filter_apply(filter, mm); 7981 7982 count++; 7983 } 7984 7985 event->addr_filters_gen++; 7986 raw_spin_unlock_irqrestore(&ifh->lock, flags); 7987 7988 up_read(&mm->mmap_sem); 7989 7990 mmput(mm); 7991 7992 restart: 7993 perf_event_stop(event, 1); 7994 } 7995 7996 /* 7997 * Address range filtering: limiting the data to certain 7998 * instruction address ranges. Filters are ioctl()ed to us from 7999 * userspace as ascii strings. 8000 * 8001 * Filter string format: 8002 * 8003 * ACTION RANGE_SPEC 8004 * where ACTION is one of the 8005 * * "filter": limit the trace to this region 8006 * * "start": start tracing from this address 8007 * * "stop": stop tracing at this address/region; 8008 * RANGE_SPEC is 8009 * * for kernel addresses: <start address>[/<size>] 8010 * * for object files: <start address>[/<size>]@</path/to/object/file> 8011 * 8012 * if <size> is not specified, the range is treated as a single address. 8013 */ 8014 enum { 8015 IF_ACT_FILTER, 8016 IF_ACT_START, 8017 IF_ACT_STOP, 8018 IF_SRC_FILE, 8019 IF_SRC_KERNEL, 8020 IF_SRC_FILEADDR, 8021 IF_SRC_KERNELADDR, 8022 }; 8023 8024 enum { 8025 IF_STATE_ACTION = 0, 8026 IF_STATE_SOURCE, 8027 IF_STATE_END, 8028 }; 8029 8030 static const match_table_t if_tokens = { 8031 { IF_ACT_FILTER, "filter" }, 8032 { IF_ACT_START, "start" }, 8033 { IF_ACT_STOP, "stop" }, 8034 { IF_SRC_FILE, "%u/%u@%s" }, 8035 { IF_SRC_KERNEL, "%u/%u" }, 8036 { IF_SRC_FILEADDR, "%u@%s" }, 8037 { IF_SRC_KERNELADDR, "%u" }, 8038 }; 8039 8040 /* 8041 * Address filter string parser 8042 */ 8043 static int 8044 perf_event_parse_addr_filter(struct perf_event *event, char *fstr, 8045 struct list_head *filters) 8046 { 8047 struct perf_addr_filter *filter = NULL; 8048 char *start, *orig, *filename = NULL; 8049 struct path path; 8050 substring_t args[MAX_OPT_ARGS]; 8051 int state = IF_STATE_ACTION, token; 8052 unsigned int kernel = 0; 8053 int ret = -EINVAL; 8054 8055 orig = fstr = kstrdup(fstr, GFP_KERNEL); 8056 if (!fstr) 8057 return -ENOMEM; 8058 8059 while ((start = strsep(&fstr, " ,\n")) != NULL) { 8060 ret = -EINVAL; 8061 8062 if (!*start) 8063 continue; 8064 8065 /* filter definition begins */ 8066 if (state == IF_STATE_ACTION) { 8067 filter = perf_addr_filter_new(event, filters); 8068 if (!filter) 8069 goto fail; 8070 } 8071 8072 token = match_token(start, if_tokens, args); 8073 switch (token) { 8074 case IF_ACT_FILTER: 8075 case IF_ACT_START: 8076 filter->filter = 1; 8077 8078 case IF_ACT_STOP: 8079 if (state != IF_STATE_ACTION) 8080 goto fail; 8081 8082 state = IF_STATE_SOURCE; 8083 break; 8084 8085 case IF_SRC_KERNELADDR: 8086 case IF_SRC_KERNEL: 8087 kernel = 1; 8088 8089 case IF_SRC_FILEADDR: 8090 case IF_SRC_FILE: 8091 if (state != IF_STATE_SOURCE) 8092 goto fail; 8093 8094 if (token == IF_SRC_FILE || token == IF_SRC_KERNEL) 8095 filter->range = 1; 8096 8097 *args[0].to = 0; 8098 ret = kstrtoul(args[0].from, 0, &filter->offset); 8099 if (ret) 8100 goto fail; 8101 8102 if (filter->range) { 8103 *args[1].to = 0; 8104 ret = kstrtoul(args[1].from, 0, &filter->size); 8105 if (ret) 8106 goto fail; 8107 } 8108 8109 if (token == IF_SRC_FILE || token == IF_SRC_FILEADDR) { 8110 int fpos = filter->range ? 2 : 1; 8111 8112 filename = match_strdup(&args[fpos]); 8113 if (!filename) { 8114 ret = -ENOMEM; 8115 goto fail; 8116 } 8117 } 8118 8119 state = IF_STATE_END; 8120 break; 8121 8122 default: 8123 goto fail; 8124 } 8125 8126 /* 8127 * Filter definition is fully parsed, validate and install it. 8128 * Make sure that it doesn't contradict itself or the event's 8129 * attribute. 8130 */ 8131 if (state == IF_STATE_END) { 8132 if (kernel && event->attr.exclude_kernel) 8133 goto fail; 8134 8135 if (!kernel) { 8136 if (!filename) 8137 goto fail; 8138 8139 /* look up the path and grab its inode */ 8140 ret = kern_path(filename, LOOKUP_FOLLOW, &path); 8141 if (ret) 8142 goto fail_free_name; 8143 8144 filter->inode = igrab(d_inode(path.dentry)); 8145 path_put(&path); 8146 kfree(filename); 8147 filename = NULL; 8148 8149 ret = -EINVAL; 8150 if (!filter->inode || 8151 !S_ISREG(filter->inode->i_mode)) 8152 /* free_filters_list() will iput() */ 8153 goto fail; 8154 } 8155 8156 /* ready to consume more filters */ 8157 state = IF_STATE_ACTION; 8158 filter = NULL; 8159 } 8160 } 8161 8162 if (state != IF_STATE_ACTION) 8163 goto fail; 8164 8165 kfree(orig); 8166 8167 return 0; 8168 8169 fail_free_name: 8170 kfree(filename); 8171 fail: 8172 free_filters_list(filters); 8173 kfree(orig); 8174 8175 return ret; 8176 } 8177 8178 static int 8179 perf_event_set_addr_filter(struct perf_event *event, char *filter_str) 8180 { 8181 LIST_HEAD(filters); 8182 int ret; 8183 8184 /* 8185 * Since this is called in perf_ioctl() path, we're already holding 8186 * ctx::mutex. 8187 */ 8188 lockdep_assert_held(&event->ctx->mutex); 8189 8190 if (WARN_ON_ONCE(event->parent)) 8191 return -EINVAL; 8192 8193 /* 8194 * For now, we only support filtering in per-task events; doing so 8195 * for CPU-wide events requires additional context switching trickery, 8196 * since same object code will be mapped at different virtual 8197 * addresses in different processes. 8198 */ 8199 if (!event->ctx->task) 8200 return -EOPNOTSUPP; 8201 8202 ret = perf_event_parse_addr_filter(event, filter_str, &filters); 8203 if (ret) 8204 return ret; 8205 8206 ret = event->pmu->addr_filters_validate(&filters); 8207 if (ret) { 8208 free_filters_list(&filters); 8209 return ret; 8210 } 8211 8212 /* remove existing filters, if any */ 8213 perf_addr_filters_splice(event, &filters); 8214 8215 /* install new filters */ 8216 perf_event_for_each_child(event, perf_event_addr_filters_apply); 8217 8218 return ret; 8219 } 8220 8221 static int perf_event_set_filter(struct perf_event *event, void __user *arg) 8222 { 8223 char *filter_str; 8224 int ret = -EINVAL; 8225 8226 if ((event->attr.type != PERF_TYPE_TRACEPOINT || 8227 !IS_ENABLED(CONFIG_EVENT_TRACING)) && 8228 !has_addr_filter(event)) 8229 return -EINVAL; 8230 8231 filter_str = strndup_user(arg, PAGE_SIZE); 8232 if (IS_ERR(filter_str)) 8233 return PTR_ERR(filter_str); 8234 8235 if (IS_ENABLED(CONFIG_EVENT_TRACING) && 8236 event->attr.type == PERF_TYPE_TRACEPOINT) 8237 ret = ftrace_profile_set_filter(event, event->attr.config, 8238 filter_str); 8239 else if (has_addr_filter(event)) 8240 ret = perf_event_set_addr_filter(event, filter_str); 8241 8242 kfree(filter_str); 8243 return ret; 8244 } 8245 8246 /* 8247 * hrtimer based swevent callback 8248 */ 8249 8250 static enum hrtimer_restart perf_swevent_hrtimer(struct hrtimer *hrtimer) 8251 { 8252 enum hrtimer_restart ret = HRTIMER_RESTART; 8253 struct perf_sample_data data; 8254 struct pt_regs *regs; 8255 struct perf_event *event; 8256 u64 period; 8257 8258 event = container_of(hrtimer, struct perf_event, hw.hrtimer); 8259 8260 if (event->state != PERF_EVENT_STATE_ACTIVE) 8261 return HRTIMER_NORESTART; 8262 8263 event->pmu->read(event); 8264 8265 perf_sample_data_init(&data, 0, event->hw.last_period); 8266 regs = get_irq_regs(); 8267 8268 if (regs && !perf_exclude_event(event, regs)) { 8269 if (!(event->attr.exclude_idle && is_idle_task(current))) 8270 if (__perf_event_overflow(event, 1, &data, regs)) 8271 ret = HRTIMER_NORESTART; 8272 } 8273 8274 period = max_t(u64, 10000, event->hw.sample_period); 8275 hrtimer_forward_now(hrtimer, ns_to_ktime(period)); 8276 8277 return ret; 8278 } 8279 8280 static void perf_swevent_start_hrtimer(struct perf_event *event) 8281 { 8282 struct hw_perf_event *hwc = &event->hw; 8283 s64 period; 8284 8285 if (!is_sampling_event(event)) 8286 return; 8287 8288 period = local64_read(&hwc->period_left); 8289 if (period) { 8290 if (period < 0) 8291 period = 10000; 8292 8293 local64_set(&hwc->period_left, 0); 8294 } else { 8295 period = max_t(u64, 10000, hwc->sample_period); 8296 } 8297 hrtimer_start(&hwc->hrtimer, ns_to_ktime(period), 8298 HRTIMER_MODE_REL_PINNED); 8299 } 8300 8301 static void perf_swevent_cancel_hrtimer(struct perf_event *event) 8302 { 8303 struct hw_perf_event *hwc = &event->hw; 8304 8305 if (is_sampling_event(event)) { 8306 ktime_t remaining = hrtimer_get_remaining(&hwc->hrtimer); 8307 local64_set(&hwc->period_left, ktime_to_ns(remaining)); 8308 8309 hrtimer_cancel(&hwc->hrtimer); 8310 } 8311 } 8312 8313 static void perf_swevent_init_hrtimer(struct perf_event *event) 8314 { 8315 struct hw_perf_event *hwc = &event->hw; 8316 8317 if (!is_sampling_event(event)) 8318 return; 8319 8320 hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 8321 hwc->hrtimer.function = perf_swevent_hrtimer; 8322 8323 /* 8324 * Since hrtimers have a fixed rate, we can do a static freq->period 8325 * mapping and avoid the whole period adjust feedback stuff. 8326 */ 8327 if (event->attr.freq) { 8328 long freq = event->attr.sample_freq; 8329 8330 event->attr.sample_period = NSEC_PER_SEC / freq; 8331 hwc->sample_period = event->attr.sample_period; 8332 local64_set(&hwc->period_left, hwc->sample_period); 8333 hwc->last_period = hwc->sample_period; 8334 event->attr.freq = 0; 8335 } 8336 } 8337 8338 /* 8339 * Software event: cpu wall time clock 8340 */ 8341 8342 static void cpu_clock_event_update(struct perf_event *event) 8343 { 8344 s64 prev; 8345 u64 now; 8346 8347 now = local_clock(); 8348 prev = local64_xchg(&event->hw.prev_count, now); 8349 local64_add(now - prev, &event->count); 8350 } 8351 8352 static void cpu_clock_event_start(struct perf_event *event, int flags) 8353 { 8354 local64_set(&event->hw.prev_count, local_clock()); 8355 perf_swevent_start_hrtimer(event); 8356 } 8357 8358 static void cpu_clock_event_stop(struct perf_event *event, int flags) 8359 { 8360 perf_swevent_cancel_hrtimer(event); 8361 cpu_clock_event_update(event); 8362 } 8363 8364 static int cpu_clock_event_add(struct perf_event *event, int flags) 8365 { 8366 if (flags & PERF_EF_START) 8367 cpu_clock_event_start(event, flags); 8368 perf_event_update_userpage(event); 8369 8370 return 0; 8371 } 8372 8373 static void cpu_clock_event_del(struct perf_event *event, int flags) 8374 { 8375 cpu_clock_event_stop(event, flags); 8376 } 8377 8378 static void cpu_clock_event_read(struct perf_event *event) 8379 { 8380 cpu_clock_event_update(event); 8381 } 8382 8383 static int cpu_clock_event_init(struct perf_event *event) 8384 { 8385 if (event->attr.type != PERF_TYPE_SOFTWARE) 8386 return -ENOENT; 8387 8388 if (event->attr.config != PERF_COUNT_SW_CPU_CLOCK) 8389 return -ENOENT; 8390 8391 /* 8392 * no branch sampling for software events 8393 */ 8394 if (has_branch_stack(event)) 8395 return -EOPNOTSUPP; 8396 8397 perf_swevent_init_hrtimer(event); 8398 8399 return 0; 8400 } 8401 8402 static struct pmu perf_cpu_clock = { 8403 .task_ctx_nr = perf_sw_context, 8404 8405 .capabilities = PERF_PMU_CAP_NO_NMI, 8406 8407 .event_init = cpu_clock_event_init, 8408 .add = cpu_clock_event_add, 8409 .del = cpu_clock_event_del, 8410 .start = cpu_clock_event_start, 8411 .stop = cpu_clock_event_stop, 8412 .read = cpu_clock_event_read, 8413 }; 8414 8415 /* 8416 * Software event: task time clock 8417 */ 8418 8419 static void task_clock_event_update(struct perf_event *event, u64 now) 8420 { 8421 u64 prev; 8422 s64 delta; 8423 8424 prev = local64_xchg(&event->hw.prev_count, now); 8425 delta = now - prev; 8426 local64_add(delta, &event->count); 8427 } 8428 8429 static void task_clock_event_start(struct perf_event *event, int flags) 8430 { 8431 local64_set(&event->hw.prev_count, event->ctx->time); 8432 perf_swevent_start_hrtimer(event); 8433 } 8434 8435 static void task_clock_event_stop(struct perf_event *event, int flags) 8436 { 8437 perf_swevent_cancel_hrtimer(event); 8438 task_clock_event_update(event, event->ctx->time); 8439 } 8440 8441 static int task_clock_event_add(struct perf_event *event, int flags) 8442 { 8443 if (flags & PERF_EF_START) 8444 task_clock_event_start(event, flags); 8445 perf_event_update_userpage(event); 8446 8447 return 0; 8448 } 8449 8450 static void task_clock_event_del(struct perf_event *event, int flags) 8451 { 8452 task_clock_event_stop(event, PERF_EF_UPDATE); 8453 } 8454 8455 static void task_clock_event_read(struct perf_event *event) 8456 { 8457 u64 now = perf_clock(); 8458 u64 delta = now - event->ctx->timestamp; 8459 u64 time = event->ctx->time + delta; 8460 8461 task_clock_event_update(event, time); 8462 } 8463 8464 static int task_clock_event_init(struct perf_event *event) 8465 { 8466 if (event->attr.type != PERF_TYPE_SOFTWARE) 8467 return -ENOENT; 8468 8469 if (event->attr.config != PERF_COUNT_SW_TASK_CLOCK) 8470 return -ENOENT; 8471 8472 /* 8473 * no branch sampling for software events 8474 */ 8475 if (has_branch_stack(event)) 8476 return -EOPNOTSUPP; 8477 8478 perf_swevent_init_hrtimer(event); 8479 8480 return 0; 8481 } 8482 8483 static struct pmu perf_task_clock = { 8484 .task_ctx_nr = perf_sw_context, 8485 8486 .capabilities = PERF_PMU_CAP_NO_NMI, 8487 8488 .event_init = task_clock_event_init, 8489 .add = task_clock_event_add, 8490 .del = task_clock_event_del, 8491 .start = task_clock_event_start, 8492 .stop = task_clock_event_stop, 8493 .read = task_clock_event_read, 8494 }; 8495 8496 static void perf_pmu_nop_void(struct pmu *pmu) 8497 { 8498 } 8499 8500 static void perf_pmu_nop_txn(struct pmu *pmu, unsigned int flags) 8501 { 8502 } 8503 8504 static int perf_pmu_nop_int(struct pmu *pmu) 8505 { 8506 return 0; 8507 } 8508 8509 static DEFINE_PER_CPU(unsigned int, nop_txn_flags); 8510 8511 static void perf_pmu_start_txn(struct pmu *pmu, unsigned int flags) 8512 { 8513 __this_cpu_write(nop_txn_flags, flags); 8514 8515 if (flags & ~PERF_PMU_TXN_ADD) 8516 return; 8517 8518 perf_pmu_disable(pmu); 8519 } 8520 8521 static int perf_pmu_commit_txn(struct pmu *pmu) 8522 { 8523 unsigned int flags = __this_cpu_read(nop_txn_flags); 8524 8525 __this_cpu_write(nop_txn_flags, 0); 8526 8527 if (flags & ~PERF_PMU_TXN_ADD) 8528 return 0; 8529 8530 perf_pmu_enable(pmu); 8531 return 0; 8532 } 8533 8534 static void perf_pmu_cancel_txn(struct pmu *pmu) 8535 { 8536 unsigned int flags = __this_cpu_read(nop_txn_flags); 8537 8538 __this_cpu_write(nop_txn_flags, 0); 8539 8540 if (flags & ~PERF_PMU_TXN_ADD) 8541 return; 8542 8543 perf_pmu_enable(pmu); 8544 } 8545 8546 static int perf_event_idx_default(struct perf_event *event) 8547 { 8548 return 0; 8549 } 8550 8551 /* 8552 * Ensures all contexts with the same task_ctx_nr have the same 8553 * pmu_cpu_context too. 8554 */ 8555 static struct perf_cpu_context __percpu *find_pmu_context(int ctxn) 8556 { 8557 struct pmu *pmu; 8558 8559 if (ctxn < 0) 8560 return NULL; 8561 8562 list_for_each_entry(pmu, &pmus, entry) { 8563 if (pmu->task_ctx_nr == ctxn) 8564 return pmu->pmu_cpu_context; 8565 } 8566 8567 return NULL; 8568 } 8569 8570 static void update_pmu_context(struct pmu *pmu, struct pmu *old_pmu) 8571 { 8572 int cpu; 8573 8574 for_each_possible_cpu(cpu) { 8575 struct perf_cpu_context *cpuctx; 8576 8577 cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu); 8578 8579 if (cpuctx->unique_pmu == old_pmu) 8580 cpuctx->unique_pmu = pmu; 8581 } 8582 } 8583 8584 static void free_pmu_context(struct pmu *pmu) 8585 { 8586 struct pmu *i; 8587 8588 mutex_lock(&pmus_lock); 8589 /* 8590 * Like a real lame refcount. 8591 */ 8592 list_for_each_entry(i, &pmus, entry) { 8593 if (i->pmu_cpu_context == pmu->pmu_cpu_context) { 8594 update_pmu_context(i, pmu); 8595 goto out; 8596 } 8597 } 8598 8599 free_percpu(pmu->pmu_cpu_context); 8600 out: 8601 mutex_unlock(&pmus_lock); 8602 } 8603 8604 /* 8605 * Let userspace know that this PMU supports address range filtering: 8606 */ 8607 static ssize_t nr_addr_filters_show(struct device *dev, 8608 struct device_attribute *attr, 8609 char *page) 8610 { 8611 struct pmu *pmu = dev_get_drvdata(dev); 8612 8613 return snprintf(page, PAGE_SIZE - 1, "%d\n", pmu->nr_addr_filters); 8614 } 8615 DEVICE_ATTR_RO(nr_addr_filters); 8616 8617 static struct idr pmu_idr; 8618 8619 static ssize_t 8620 type_show(struct device *dev, struct device_attribute *attr, char *page) 8621 { 8622 struct pmu *pmu = dev_get_drvdata(dev); 8623 8624 return snprintf(page, PAGE_SIZE-1, "%d\n", pmu->type); 8625 } 8626 static DEVICE_ATTR_RO(type); 8627 8628 static ssize_t 8629 perf_event_mux_interval_ms_show(struct device *dev, 8630 struct device_attribute *attr, 8631 char *page) 8632 { 8633 struct pmu *pmu = dev_get_drvdata(dev); 8634 8635 return snprintf(page, PAGE_SIZE-1, "%d\n", pmu->hrtimer_interval_ms); 8636 } 8637 8638 static DEFINE_MUTEX(mux_interval_mutex); 8639 8640 static ssize_t 8641 perf_event_mux_interval_ms_store(struct device *dev, 8642 struct device_attribute *attr, 8643 const char *buf, size_t count) 8644 { 8645 struct pmu *pmu = dev_get_drvdata(dev); 8646 int timer, cpu, ret; 8647 8648 ret = kstrtoint(buf, 0, &timer); 8649 if (ret) 8650 return ret; 8651 8652 if (timer < 1) 8653 return -EINVAL; 8654 8655 /* same value, noting to do */ 8656 if (timer == pmu->hrtimer_interval_ms) 8657 return count; 8658 8659 mutex_lock(&mux_interval_mutex); 8660 pmu->hrtimer_interval_ms = timer; 8661 8662 /* update all cpuctx for this PMU */ 8663 get_online_cpus(); 8664 for_each_online_cpu(cpu) { 8665 struct perf_cpu_context *cpuctx; 8666 cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu); 8667 cpuctx->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * timer); 8668 8669 cpu_function_call(cpu, 8670 (remote_function_f)perf_mux_hrtimer_restart, cpuctx); 8671 } 8672 put_online_cpus(); 8673 mutex_unlock(&mux_interval_mutex); 8674 8675 return count; 8676 } 8677 static DEVICE_ATTR_RW(perf_event_mux_interval_ms); 8678 8679 static struct attribute *pmu_dev_attrs[] = { 8680 &dev_attr_type.attr, 8681 &dev_attr_perf_event_mux_interval_ms.attr, 8682 NULL, 8683 }; 8684 ATTRIBUTE_GROUPS(pmu_dev); 8685 8686 static int pmu_bus_running; 8687 static struct bus_type pmu_bus = { 8688 .name = "event_source", 8689 .dev_groups = pmu_dev_groups, 8690 }; 8691 8692 static void pmu_dev_release(struct device *dev) 8693 { 8694 kfree(dev); 8695 } 8696 8697 static int pmu_dev_alloc(struct pmu *pmu) 8698 { 8699 int ret = -ENOMEM; 8700 8701 pmu->dev = kzalloc(sizeof(struct device), GFP_KERNEL); 8702 if (!pmu->dev) 8703 goto out; 8704 8705 pmu->dev->groups = pmu->attr_groups; 8706 device_initialize(pmu->dev); 8707 ret = dev_set_name(pmu->dev, "%s", pmu->name); 8708 if (ret) 8709 goto free_dev; 8710 8711 dev_set_drvdata(pmu->dev, pmu); 8712 pmu->dev->bus = &pmu_bus; 8713 pmu->dev->release = pmu_dev_release; 8714 ret = device_add(pmu->dev); 8715 if (ret) 8716 goto free_dev; 8717 8718 /* For PMUs with address filters, throw in an extra attribute: */ 8719 if (pmu->nr_addr_filters) 8720 ret = device_create_file(pmu->dev, &dev_attr_nr_addr_filters); 8721 8722 if (ret) 8723 goto del_dev; 8724 8725 out: 8726 return ret; 8727 8728 del_dev: 8729 device_del(pmu->dev); 8730 8731 free_dev: 8732 put_device(pmu->dev); 8733 goto out; 8734 } 8735 8736 static struct lock_class_key cpuctx_mutex; 8737 static struct lock_class_key cpuctx_lock; 8738 8739 int perf_pmu_register(struct pmu *pmu, const char *name, int type) 8740 { 8741 int cpu, ret; 8742 8743 mutex_lock(&pmus_lock); 8744 ret = -ENOMEM; 8745 pmu->pmu_disable_count = alloc_percpu(int); 8746 if (!pmu->pmu_disable_count) 8747 goto unlock; 8748 8749 pmu->type = -1; 8750 if (!name) 8751 goto skip_type; 8752 pmu->name = name; 8753 8754 if (type < 0) { 8755 type = idr_alloc(&pmu_idr, pmu, PERF_TYPE_MAX, 0, GFP_KERNEL); 8756 if (type < 0) { 8757 ret = type; 8758 goto free_pdc; 8759 } 8760 } 8761 pmu->type = type; 8762 8763 if (pmu_bus_running) { 8764 ret = pmu_dev_alloc(pmu); 8765 if (ret) 8766 goto free_idr; 8767 } 8768 8769 skip_type: 8770 if (pmu->task_ctx_nr == perf_hw_context) { 8771 static int hw_context_taken = 0; 8772 8773 /* 8774 * Other than systems with heterogeneous CPUs, it never makes 8775 * sense for two PMUs to share perf_hw_context. PMUs which are 8776 * uncore must use perf_invalid_context. 8777 */ 8778 if (WARN_ON_ONCE(hw_context_taken && 8779 !(pmu->capabilities & PERF_PMU_CAP_HETEROGENEOUS_CPUS))) 8780 pmu->task_ctx_nr = perf_invalid_context; 8781 8782 hw_context_taken = 1; 8783 } 8784 8785 pmu->pmu_cpu_context = find_pmu_context(pmu->task_ctx_nr); 8786 if (pmu->pmu_cpu_context) 8787 goto got_cpu_context; 8788 8789 ret = -ENOMEM; 8790 pmu->pmu_cpu_context = alloc_percpu(struct perf_cpu_context); 8791 if (!pmu->pmu_cpu_context) 8792 goto free_dev; 8793 8794 for_each_possible_cpu(cpu) { 8795 struct perf_cpu_context *cpuctx; 8796 8797 cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu); 8798 __perf_event_init_context(&cpuctx->ctx); 8799 lockdep_set_class(&cpuctx->ctx.mutex, &cpuctx_mutex); 8800 lockdep_set_class(&cpuctx->ctx.lock, &cpuctx_lock); 8801 cpuctx->ctx.pmu = pmu; 8802 8803 __perf_mux_hrtimer_init(cpuctx, cpu); 8804 8805 cpuctx->unique_pmu = pmu; 8806 } 8807 8808 got_cpu_context: 8809 if (!pmu->start_txn) { 8810 if (pmu->pmu_enable) { 8811 /* 8812 * If we have pmu_enable/pmu_disable calls, install 8813 * transaction stubs that use that to try and batch 8814 * hardware accesses. 8815 */ 8816 pmu->start_txn = perf_pmu_start_txn; 8817 pmu->commit_txn = perf_pmu_commit_txn; 8818 pmu->cancel_txn = perf_pmu_cancel_txn; 8819 } else { 8820 pmu->start_txn = perf_pmu_nop_txn; 8821 pmu->commit_txn = perf_pmu_nop_int; 8822 pmu->cancel_txn = perf_pmu_nop_void; 8823 } 8824 } 8825 8826 if (!pmu->pmu_enable) { 8827 pmu->pmu_enable = perf_pmu_nop_void; 8828 pmu->pmu_disable = perf_pmu_nop_void; 8829 } 8830 8831 if (!pmu->event_idx) 8832 pmu->event_idx = perf_event_idx_default; 8833 8834 list_add_rcu(&pmu->entry, &pmus); 8835 atomic_set(&pmu->exclusive_cnt, 0); 8836 ret = 0; 8837 unlock: 8838 mutex_unlock(&pmus_lock); 8839 8840 return ret; 8841 8842 free_dev: 8843 device_del(pmu->dev); 8844 put_device(pmu->dev); 8845 8846 free_idr: 8847 if (pmu->type >= PERF_TYPE_MAX) 8848 idr_remove(&pmu_idr, pmu->type); 8849 8850 free_pdc: 8851 free_percpu(pmu->pmu_disable_count); 8852 goto unlock; 8853 } 8854 EXPORT_SYMBOL_GPL(perf_pmu_register); 8855 8856 void perf_pmu_unregister(struct pmu *pmu) 8857 { 8858 mutex_lock(&pmus_lock); 8859 list_del_rcu(&pmu->entry); 8860 mutex_unlock(&pmus_lock); 8861 8862 /* 8863 * We dereference the pmu list under both SRCU and regular RCU, so 8864 * synchronize against both of those. 8865 */ 8866 synchronize_srcu(&pmus_srcu); 8867 synchronize_rcu(); 8868 8869 free_percpu(pmu->pmu_disable_count); 8870 if (pmu->type >= PERF_TYPE_MAX) 8871 idr_remove(&pmu_idr, pmu->type); 8872 if (pmu->nr_addr_filters) 8873 device_remove_file(pmu->dev, &dev_attr_nr_addr_filters); 8874 device_del(pmu->dev); 8875 put_device(pmu->dev); 8876 free_pmu_context(pmu); 8877 } 8878 EXPORT_SYMBOL_GPL(perf_pmu_unregister); 8879 8880 static int perf_try_init_event(struct pmu *pmu, struct perf_event *event) 8881 { 8882 struct perf_event_context *ctx = NULL; 8883 int ret; 8884 8885 if (!try_module_get(pmu->module)) 8886 return -ENODEV; 8887 8888 if (event->group_leader != event) { 8889 /* 8890 * This ctx->mutex can nest when we're called through 8891 * inheritance. See the perf_event_ctx_lock_nested() comment. 8892 */ 8893 ctx = perf_event_ctx_lock_nested(event->group_leader, 8894 SINGLE_DEPTH_NESTING); 8895 BUG_ON(!ctx); 8896 } 8897 8898 event->pmu = pmu; 8899 ret = pmu->event_init(event); 8900 8901 if (ctx) 8902 perf_event_ctx_unlock(event->group_leader, ctx); 8903 8904 if (ret) 8905 module_put(pmu->module); 8906 8907 return ret; 8908 } 8909 8910 static struct pmu *perf_init_event(struct perf_event *event) 8911 { 8912 struct pmu *pmu = NULL; 8913 int idx; 8914 int ret; 8915 8916 idx = srcu_read_lock(&pmus_srcu); 8917 8918 rcu_read_lock(); 8919 pmu = idr_find(&pmu_idr, event->attr.type); 8920 rcu_read_unlock(); 8921 if (pmu) { 8922 ret = perf_try_init_event(pmu, event); 8923 if (ret) 8924 pmu = ERR_PTR(ret); 8925 goto unlock; 8926 } 8927 8928 list_for_each_entry_rcu(pmu, &pmus, entry) { 8929 ret = perf_try_init_event(pmu, event); 8930 if (!ret) 8931 goto unlock; 8932 8933 if (ret != -ENOENT) { 8934 pmu = ERR_PTR(ret); 8935 goto unlock; 8936 } 8937 } 8938 pmu = ERR_PTR(-ENOENT); 8939 unlock: 8940 srcu_read_unlock(&pmus_srcu, idx); 8941 8942 return pmu; 8943 } 8944 8945 static void attach_sb_event(struct perf_event *event) 8946 { 8947 struct pmu_event_list *pel = per_cpu_ptr(&pmu_sb_events, event->cpu); 8948 8949 raw_spin_lock(&pel->lock); 8950 list_add_rcu(&event->sb_list, &pel->list); 8951 raw_spin_unlock(&pel->lock); 8952 } 8953 8954 /* 8955 * We keep a list of all !task (and therefore per-cpu) events 8956 * that need to receive side-band records. 8957 * 8958 * This avoids having to scan all the various PMU per-cpu contexts 8959 * looking for them. 8960 */ 8961 static void account_pmu_sb_event(struct perf_event *event) 8962 { 8963 if (is_sb_event(event)) 8964 attach_sb_event(event); 8965 } 8966 8967 static void account_event_cpu(struct perf_event *event, int cpu) 8968 { 8969 if (event->parent) 8970 return; 8971 8972 if (is_cgroup_event(event)) 8973 atomic_inc(&per_cpu(perf_cgroup_events, cpu)); 8974 } 8975 8976 /* Freq events need the tick to stay alive (see perf_event_task_tick). */ 8977 static void account_freq_event_nohz(void) 8978 { 8979 #ifdef CONFIG_NO_HZ_FULL 8980 /* Lock so we don't race with concurrent unaccount */ 8981 spin_lock(&nr_freq_lock); 8982 if (atomic_inc_return(&nr_freq_events) == 1) 8983 tick_nohz_dep_set(TICK_DEP_BIT_PERF_EVENTS); 8984 spin_unlock(&nr_freq_lock); 8985 #endif 8986 } 8987 8988 static void account_freq_event(void) 8989 { 8990 if (tick_nohz_full_enabled()) 8991 account_freq_event_nohz(); 8992 else 8993 atomic_inc(&nr_freq_events); 8994 } 8995 8996 8997 static void account_event(struct perf_event *event) 8998 { 8999 bool inc = false; 9000 9001 if (event->parent) 9002 return; 9003 9004 if (event->attach_state & PERF_ATTACH_TASK) 9005 inc = true; 9006 if (event->attr.mmap || event->attr.mmap_data) 9007 atomic_inc(&nr_mmap_events); 9008 if (event->attr.comm) 9009 atomic_inc(&nr_comm_events); 9010 if (event->attr.task) 9011 atomic_inc(&nr_task_events); 9012 if (event->attr.freq) 9013 account_freq_event(); 9014 if (event->attr.context_switch) { 9015 atomic_inc(&nr_switch_events); 9016 inc = true; 9017 } 9018 if (has_branch_stack(event)) 9019 inc = true; 9020 if (is_cgroup_event(event)) 9021 inc = true; 9022 9023 if (inc) { 9024 if (atomic_inc_not_zero(&perf_sched_count)) 9025 goto enabled; 9026 9027 mutex_lock(&perf_sched_mutex); 9028 if (!atomic_read(&perf_sched_count)) { 9029 static_branch_enable(&perf_sched_events); 9030 /* 9031 * Guarantee that all CPUs observe they key change and 9032 * call the perf scheduling hooks before proceeding to 9033 * install events that need them. 9034 */ 9035 synchronize_sched(); 9036 } 9037 /* 9038 * Now that we have waited for the sync_sched(), allow further 9039 * increments to by-pass the mutex. 9040 */ 9041 atomic_inc(&perf_sched_count); 9042 mutex_unlock(&perf_sched_mutex); 9043 } 9044 enabled: 9045 9046 account_event_cpu(event, event->cpu); 9047 9048 account_pmu_sb_event(event); 9049 } 9050 9051 /* 9052 * Allocate and initialize a event structure 9053 */ 9054 static struct perf_event * 9055 perf_event_alloc(struct perf_event_attr *attr, int cpu, 9056 struct task_struct *task, 9057 struct perf_event *group_leader, 9058 struct perf_event *parent_event, 9059 perf_overflow_handler_t overflow_handler, 9060 void *context, int cgroup_fd) 9061 { 9062 struct pmu *pmu; 9063 struct perf_event *event; 9064 struct hw_perf_event *hwc; 9065 long err = -EINVAL; 9066 9067 if ((unsigned)cpu >= nr_cpu_ids) { 9068 if (!task || cpu != -1) 9069 return ERR_PTR(-EINVAL); 9070 } 9071 9072 event = kzalloc(sizeof(*event), GFP_KERNEL); 9073 if (!event) 9074 return ERR_PTR(-ENOMEM); 9075 9076 /* 9077 * Single events are their own group leaders, with an 9078 * empty sibling list: 9079 */ 9080 if (!group_leader) 9081 group_leader = event; 9082 9083 mutex_init(&event->child_mutex); 9084 INIT_LIST_HEAD(&event->child_list); 9085 9086 INIT_LIST_HEAD(&event->group_entry); 9087 INIT_LIST_HEAD(&event->event_entry); 9088 INIT_LIST_HEAD(&event->sibling_list); 9089 INIT_LIST_HEAD(&event->rb_entry); 9090 INIT_LIST_HEAD(&event->active_entry); 9091 INIT_LIST_HEAD(&event->addr_filters.list); 9092 INIT_HLIST_NODE(&event->hlist_entry); 9093 9094 9095 init_waitqueue_head(&event->waitq); 9096 init_irq_work(&event->pending, perf_pending_event); 9097 9098 mutex_init(&event->mmap_mutex); 9099 raw_spin_lock_init(&event->addr_filters.lock); 9100 9101 atomic_long_set(&event->refcount, 1); 9102 event->cpu = cpu; 9103 event->attr = *attr; 9104 event->group_leader = group_leader; 9105 event->pmu = NULL; 9106 event->oncpu = -1; 9107 9108 event->parent = parent_event; 9109 9110 event->ns = get_pid_ns(task_active_pid_ns(current)); 9111 event->id = atomic64_inc_return(&perf_event_id); 9112 9113 event->state = PERF_EVENT_STATE_INACTIVE; 9114 9115 if (task) { 9116 event->attach_state = PERF_ATTACH_TASK; 9117 /* 9118 * XXX pmu::event_init needs to know what task to account to 9119 * and we cannot use the ctx information because we need the 9120 * pmu before we get a ctx. 9121 */ 9122 event->hw.target = task; 9123 } 9124 9125 event->clock = &local_clock; 9126 if (parent_event) 9127 event->clock = parent_event->clock; 9128 9129 if (!overflow_handler && parent_event) { 9130 overflow_handler = parent_event->overflow_handler; 9131 context = parent_event->overflow_handler_context; 9132 #if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_EVENT_TRACING) 9133 if (overflow_handler == bpf_overflow_handler) { 9134 struct bpf_prog *prog = bpf_prog_inc(parent_event->prog); 9135 9136 if (IS_ERR(prog)) { 9137 err = PTR_ERR(prog); 9138 goto err_ns; 9139 } 9140 event->prog = prog; 9141 event->orig_overflow_handler = 9142 parent_event->orig_overflow_handler; 9143 } 9144 #endif 9145 } 9146 9147 if (overflow_handler) { 9148 event->overflow_handler = overflow_handler; 9149 event->overflow_handler_context = context; 9150 } else if (is_write_backward(event)){ 9151 event->overflow_handler = perf_event_output_backward; 9152 event->overflow_handler_context = NULL; 9153 } else { 9154 event->overflow_handler = perf_event_output_forward; 9155 event->overflow_handler_context = NULL; 9156 } 9157 9158 perf_event__state_init(event); 9159 9160 pmu = NULL; 9161 9162 hwc = &event->hw; 9163 hwc->sample_period = attr->sample_period; 9164 if (attr->freq && attr->sample_freq) 9165 hwc->sample_period = 1; 9166 hwc->last_period = hwc->sample_period; 9167 9168 local64_set(&hwc->period_left, hwc->sample_period); 9169 9170 /* 9171 * we currently do not support PERF_FORMAT_GROUP on inherited events 9172 */ 9173 if (attr->inherit && (attr->read_format & PERF_FORMAT_GROUP)) 9174 goto err_ns; 9175 9176 if (!has_branch_stack(event)) 9177 event->attr.branch_sample_type = 0; 9178 9179 if (cgroup_fd != -1) { 9180 err = perf_cgroup_connect(cgroup_fd, event, attr, group_leader); 9181 if (err) 9182 goto err_ns; 9183 } 9184 9185 pmu = perf_init_event(event); 9186 if (!pmu) 9187 goto err_ns; 9188 else if (IS_ERR(pmu)) { 9189 err = PTR_ERR(pmu); 9190 goto err_ns; 9191 } 9192 9193 err = exclusive_event_init(event); 9194 if (err) 9195 goto err_pmu; 9196 9197 if (has_addr_filter(event)) { 9198 event->addr_filters_offs = kcalloc(pmu->nr_addr_filters, 9199 sizeof(unsigned long), 9200 GFP_KERNEL); 9201 if (!event->addr_filters_offs) 9202 goto err_per_task; 9203 9204 /* force hw sync on the address filters */ 9205 event->addr_filters_gen = 1; 9206 } 9207 9208 if (!event->parent) { 9209 if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) { 9210 err = get_callchain_buffers(attr->sample_max_stack); 9211 if (err) 9212 goto err_addr_filters; 9213 } 9214 } 9215 9216 /* symmetric to unaccount_event() in _free_event() */ 9217 account_event(event); 9218 9219 return event; 9220 9221 err_addr_filters: 9222 kfree(event->addr_filters_offs); 9223 9224 err_per_task: 9225 exclusive_event_destroy(event); 9226 9227 err_pmu: 9228 if (event->destroy) 9229 event->destroy(event); 9230 module_put(pmu->module); 9231 err_ns: 9232 if (is_cgroup_event(event)) 9233 perf_detach_cgroup(event); 9234 if (event->ns) 9235 put_pid_ns(event->ns); 9236 kfree(event); 9237 9238 return ERR_PTR(err); 9239 } 9240 9241 static int perf_copy_attr(struct perf_event_attr __user *uattr, 9242 struct perf_event_attr *attr) 9243 { 9244 u32 size; 9245 int ret; 9246 9247 if (!access_ok(VERIFY_WRITE, uattr, PERF_ATTR_SIZE_VER0)) 9248 return -EFAULT; 9249 9250 /* 9251 * zero the full structure, so that a short copy will be nice. 9252 */ 9253 memset(attr, 0, sizeof(*attr)); 9254 9255 ret = get_user(size, &uattr->size); 9256 if (ret) 9257 return ret; 9258 9259 if (size > PAGE_SIZE) /* silly large */ 9260 goto err_size; 9261 9262 if (!size) /* abi compat */ 9263 size = PERF_ATTR_SIZE_VER0; 9264 9265 if (size < PERF_ATTR_SIZE_VER0) 9266 goto err_size; 9267 9268 /* 9269 * If we're handed a bigger struct than we know of, 9270 * ensure all the unknown bits are 0 - i.e. new 9271 * user-space does not rely on any kernel feature 9272 * extensions we dont know about yet. 9273 */ 9274 if (size > sizeof(*attr)) { 9275 unsigned char __user *addr; 9276 unsigned char __user *end; 9277 unsigned char val; 9278 9279 addr = (void __user *)uattr + sizeof(*attr); 9280 end = (void __user *)uattr + size; 9281 9282 for (; addr < end; addr++) { 9283 ret = get_user(val, addr); 9284 if (ret) 9285 return ret; 9286 if (val) 9287 goto err_size; 9288 } 9289 size = sizeof(*attr); 9290 } 9291 9292 ret = copy_from_user(attr, uattr, size); 9293 if (ret) 9294 return -EFAULT; 9295 9296 if (attr->__reserved_1) 9297 return -EINVAL; 9298 9299 if (attr->sample_type & ~(PERF_SAMPLE_MAX-1)) 9300 return -EINVAL; 9301 9302 if (attr->read_format & ~(PERF_FORMAT_MAX-1)) 9303 return -EINVAL; 9304 9305 if (attr->sample_type & PERF_SAMPLE_BRANCH_STACK) { 9306 u64 mask = attr->branch_sample_type; 9307 9308 /* only using defined bits */ 9309 if (mask & ~(PERF_SAMPLE_BRANCH_MAX-1)) 9310 return -EINVAL; 9311 9312 /* at least one branch bit must be set */ 9313 if (!(mask & ~PERF_SAMPLE_BRANCH_PLM_ALL)) 9314 return -EINVAL; 9315 9316 /* propagate priv level, when not set for branch */ 9317 if (!(mask & PERF_SAMPLE_BRANCH_PLM_ALL)) { 9318 9319 /* exclude_kernel checked on syscall entry */ 9320 if (!attr->exclude_kernel) 9321 mask |= PERF_SAMPLE_BRANCH_KERNEL; 9322 9323 if (!attr->exclude_user) 9324 mask |= PERF_SAMPLE_BRANCH_USER; 9325 9326 if (!attr->exclude_hv) 9327 mask |= PERF_SAMPLE_BRANCH_HV; 9328 /* 9329 * adjust user setting (for HW filter setup) 9330 */ 9331 attr->branch_sample_type = mask; 9332 } 9333 /* privileged levels capture (kernel, hv): check permissions */ 9334 if ((mask & PERF_SAMPLE_BRANCH_PERM_PLM) 9335 && perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN)) 9336 return -EACCES; 9337 } 9338 9339 if (attr->sample_type & PERF_SAMPLE_REGS_USER) { 9340 ret = perf_reg_validate(attr->sample_regs_user); 9341 if (ret) 9342 return ret; 9343 } 9344 9345 if (attr->sample_type & PERF_SAMPLE_STACK_USER) { 9346 if (!arch_perf_have_user_stack_dump()) 9347 return -ENOSYS; 9348 9349 /* 9350 * We have __u32 type for the size, but so far 9351 * we can only use __u16 as maximum due to the 9352 * __u16 sample size limit. 9353 */ 9354 if (attr->sample_stack_user >= USHRT_MAX) 9355 ret = -EINVAL; 9356 else if (!IS_ALIGNED(attr->sample_stack_user, sizeof(u64))) 9357 ret = -EINVAL; 9358 } 9359 9360 if (attr->sample_type & PERF_SAMPLE_REGS_INTR) 9361 ret = perf_reg_validate(attr->sample_regs_intr); 9362 out: 9363 return ret; 9364 9365 err_size: 9366 put_user(sizeof(*attr), &uattr->size); 9367 ret = -E2BIG; 9368 goto out; 9369 } 9370 9371 static int 9372 perf_event_set_output(struct perf_event *event, struct perf_event *output_event) 9373 { 9374 struct ring_buffer *rb = NULL; 9375 int ret = -EINVAL; 9376 9377 if (!output_event) 9378 goto set; 9379 9380 /* don't allow circular references */ 9381 if (event == output_event) 9382 goto out; 9383 9384 /* 9385 * Don't allow cross-cpu buffers 9386 */ 9387 if (output_event->cpu != event->cpu) 9388 goto out; 9389 9390 /* 9391 * If its not a per-cpu rb, it must be the same task. 9392 */ 9393 if (output_event->cpu == -1 && output_event->ctx != event->ctx) 9394 goto out; 9395 9396 /* 9397 * Mixing clocks in the same buffer is trouble you don't need. 9398 */ 9399 if (output_event->clock != event->clock) 9400 goto out; 9401 9402 /* 9403 * Either writing ring buffer from beginning or from end. 9404 * Mixing is not allowed. 9405 */ 9406 if (is_write_backward(output_event) != is_write_backward(event)) 9407 goto out; 9408 9409 /* 9410 * If both events generate aux data, they must be on the same PMU 9411 */ 9412 if (has_aux(event) && has_aux(output_event) && 9413 event->pmu != output_event->pmu) 9414 goto out; 9415 9416 set: 9417 mutex_lock(&event->mmap_mutex); 9418 /* Can't redirect output if we've got an active mmap() */ 9419 if (atomic_read(&event->mmap_count)) 9420 goto unlock; 9421 9422 if (output_event) { 9423 /* get the rb we want to redirect to */ 9424 rb = ring_buffer_get(output_event); 9425 if (!rb) 9426 goto unlock; 9427 } 9428 9429 ring_buffer_attach(event, rb); 9430 9431 ret = 0; 9432 unlock: 9433 mutex_unlock(&event->mmap_mutex); 9434 9435 out: 9436 return ret; 9437 } 9438 9439 static void mutex_lock_double(struct mutex *a, struct mutex *b) 9440 { 9441 if (b < a) 9442 swap(a, b); 9443 9444 mutex_lock(a); 9445 mutex_lock_nested(b, SINGLE_DEPTH_NESTING); 9446 } 9447 9448 static int perf_event_set_clock(struct perf_event *event, clockid_t clk_id) 9449 { 9450 bool nmi_safe = false; 9451 9452 switch (clk_id) { 9453 case CLOCK_MONOTONIC: 9454 event->clock = &ktime_get_mono_fast_ns; 9455 nmi_safe = true; 9456 break; 9457 9458 case CLOCK_MONOTONIC_RAW: 9459 event->clock = &ktime_get_raw_fast_ns; 9460 nmi_safe = true; 9461 break; 9462 9463 case CLOCK_REALTIME: 9464 event->clock = &ktime_get_real_ns; 9465 break; 9466 9467 case CLOCK_BOOTTIME: 9468 event->clock = &ktime_get_boot_ns; 9469 break; 9470 9471 case CLOCK_TAI: 9472 event->clock = &ktime_get_tai_ns; 9473 break; 9474 9475 default: 9476 return -EINVAL; 9477 } 9478 9479 if (!nmi_safe && !(event->pmu->capabilities & PERF_PMU_CAP_NO_NMI)) 9480 return -EINVAL; 9481 9482 return 0; 9483 } 9484 9485 /** 9486 * sys_perf_event_open - open a performance event, associate it to a task/cpu 9487 * 9488 * @attr_uptr: event_id type attributes for monitoring/sampling 9489 * @pid: target pid 9490 * @cpu: target cpu 9491 * @group_fd: group leader event fd 9492 */ 9493 SYSCALL_DEFINE5(perf_event_open, 9494 struct perf_event_attr __user *, attr_uptr, 9495 pid_t, pid, int, cpu, int, group_fd, unsigned long, flags) 9496 { 9497 struct perf_event *group_leader = NULL, *output_event = NULL; 9498 struct perf_event *event, *sibling; 9499 struct perf_event_attr attr; 9500 struct perf_event_context *ctx, *uninitialized_var(gctx); 9501 struct file *event_file = NULL; 9502 struct fd group = {NULL, 0}; 9503 struct task_struct *task = NULL; 9504 struct pmu *pmu; 9505 int event_fd; 9506 int move_group = 0; 9507 int err; 9508 int f_flags = O_RDWR; 9509 int cgroup_fd = -1; 9510 9511 /* for future expandability... */ 9512 if (flags & ~PERF_FLAG_ALL) 9513 return -EINVAL; 9514 9515 err = perf_copy_attr(attr_uptr, &attr); 9516 if (err) 9517 return err; 9518 9519 if (!attr.exclude_kernel) { 9520 if (perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN)) 9521 return -EACCES; 9522 } 9523 9524 if (attr.freq) { 9525 if (attr.sample_freq > sysctl_perf_event_sample_rate) 9526 return -EINVAL; 9527 } else { 9528 if (attr.sample_period & (1ULL << 63)) 9529 return -EINVAL; 9530 } 9531 9532 if (!attr.sample_max_stack) 9533 attr.sample_max_stack = sysctl_perf_event_max_stack; 9534 9535 /* 9536 * In cgroup mode, the pid argument is used to pass the fd 9537 * opened to the cgroup directory in cgroupfs. The cpu argument 9538 * designates the cpu on which to monitor threads from that 9539 * cgroup. 9540 */ 9541 if ((flags & PERF_FLAG_PID_CGROUP) && (pid == -1 || cpu == -1)) 9542 return -EINVAL; 9543 9544 if (flags & PERF_FLAG_FD_CLOEXEC) 9545 f_flags |= O_CLOEXEC; 9546 9547 event_fd = get_unused_fd_flags(f_flags); 9548 if (event_fd < 0) 9549 return event_fd; 9550 9551 if (group_fd != -1) { 9552 err = perf_fget_light(group_fd, &group); 9553 if (err) 9554 goto err_fd; 9555 group_leader = group.file->private_data; 9556 if (flags & PERF_FLAG_FD_OUTPUT) 9557 output_event = group_leader; 9558 if (flags & PERF_FLAG_FD_NO_GROUP) 9559 group_leader = NULL; 9560 } 9561 9562 if (pid != -1 && !(flags & PERF_FLAG_PID_CGROUP)) { 9563 task = find_lively_task_by_vpid(pid); 9564 if (IS_ERR(task)) { 9565 err = PTR_ERR(task); 9566 goto err_group_fd; 9567 } 9568 } 9569 9570 if (task && group_leader && 9571 group_leader->attr.inherit != attr.inherit) { 9572 err = -EINVAL; 9573 goto err_task; 9574 } 9575 9576 get_online_cpus(); 9577 9578 if (task) { 9579 err = mutex_lock_interruptible(&task->signal->cred_guard_mutex); 9580 if (err) 9581 goto err_cpus; 9582 9583 /* 9584 * Reuse ptrace permission checks for now. 9585 * 9586 * We must hold cred_guard_mutex across this and any potential 9587 * perf_install_in_context() call for this new event to 9588 * serialize against exec() altering our credentials (and the 9589 * perf_event_exit_task() that could imply). 9590 */ 9591 err = -EACCES; 9592 if (!ptrace_may_access(task, PTRACE_MODE_READ_REALCREDS)) 9593 goto err_cred; 9594 } 9595 9596 if (flags & PERF_FLAG_PID_CGROUP) 9597 cgroup_fd = pid; 9598 9599 event = perf_event_alloc(&attr, cpu, task, group_leader, NULL, 9600 NULL, NULL, cgroup_fd); 9601 if (IS_ERR(event)) { 9602 err = PTR_ERR(event); 9603 goto err_cred; 9604 } 9605 9606 if (is_sampling_event(event)) { 9607 if (event->pmu->capabilities & PERF_PMU_CAP_NO_INTERRUPT) { 9608 err = -EOPNOTSUPP; 9609 goto err_alloc; 9610 } 9611 } 9612 9613 /* 9614 * Special case software events and allow them to be part of 9615 * any hardware group. 9616 */ 9617 pmu = event->pmu; 9618 9619 if (attr.use_clockid) { 9620 err = perf_event_set_clock(event, attr.clockid); 9621 if (err) 9622 goto err_alloc; 9623 } 9624 9625 if (pmu->task_ctx_nr == perf_sw_context) 9626 event->event_caps |= PERF_EV_CAP_SOFTWARE; 9627 9628 if (group_leader && 9629 (is_software_event(event) != is_software_event(group_leader))) { 9630 if (is_software_event(event)) { 9631 /* 9632 * If event and group_leader are not both a software 9633 * event, and event is, then group leader is not. 9634 * 9635 * Allow the addition of software events to !software 9636 * groups, this is safe because software events never 9637 * fail to schedule. 9638 */ 9639 pmu = group_leader->pmu; 9640 } else if (is_software_event(group_leader) && 9641 (group_leader->group_caps & PERF_EV_CAP_SOFTWARE)) { 9642 /* 9643 * In case the group is a pure software group, and we 9644 * try to add a hardware event, move the whole group to 9645 * the hardware context. 9646 */ 9647 move_group = 1; 9648 } 9649 } 9650 9651 /* 9652 * Get the target context (task or percpu): 9653 */ 9654 ctx = find_get_context(pmu, task, event); 9655 if (IS_ERR(ctx)) { 9656 err = PTR_ERR(ctx); 9657 goto err_alloc; 9658 } 9659 9660 if ((pmu->capabilities & PERF_PMU_CAP_EXCLUSIVE) && group_leader) { 9661 err = -EBUSY; 9662 goto err_context; 9663 } 9664 9665 /* 9666 * Look up the group leader (we will attach this event to it): 9667 */ 9668 if (group_leader) { 9669 err = -EINVAL; 9670 9671 /* 9672 * Do not allow a recursive hierarchy (this new sibling 9673 * becoming part of another group-sibling): 9674 */ 9675 if (group_leader->group_leader != group_leader) 9676 goto err_context; 9677 9678 /* All events in a group should have the same clock */ 9679 if (group_leader->clock != event->clock) 9680 goto err_context; 9681 9682 /* 9683 * Do not allow to attach to a group in a different 9684 * task or CPU context: 9685 */ 9686 if (move_group) { 9687 /* 9688 * Make sure we're both on the same task, or both 9689 * per-cpu events. 9690 */ 9691 if (group_leader->ctx->task != ctx->task) 9692 goto err_context; 9693 9694 /* 9695 * Make sure we're both events for the same CPU; 9696 * grouping events for different CPUs is broken; since 9697 * you can never concurrently schedule them anyhow. 9698 */ 9699 if (group_leader->cpu != event->cpu) 9700 goto err_context; 9701 } else { 9702 if (group_leader->ctx != ctx) 9703 goto err_context; 9704 } 9705 9706 /* 9707 * Only a group leader can be exclusive or pinned 9708 */ 9709 if (attr.exclusive || attr.pinned) 9710 goto err_context; 9711 } 9712 9713 if (output_event) { 9714 err = perf_event_set_output(event, output_event); 9715 if (err) 9716 goto err_context; 9717 } 9718 9719 event_file = anon_inode_getfile("[perf_event]", &perf_fops, event, 9720 f_flags); 9721 if (IS_ERR(event_file)) { 9722 err = PTR_ERR(event_file); 9723 event_file = NULL; 9724 goto err_context; 9725 } 9726 9727 if (move_group) { 9728 gctx = group_leader->ctx; 9729 mutex_lock_double(&gctx->mutex, &ctx->mutex); 9730 if (gctx->task == TASK_TOMBSTONE) { 9731 err = -ESRCH; 9732 goto err_locked; 9733 } 9734 } else { 9735 mutex_lock(&ctx->mutex); 9736 } 9737 9738 if (ctx->task == TASK_TOMBSTONE) { 9739 err = -ESRCH; 9740 goto err_locked; 9741 } 9742 9743 if (!perf_event_validate_size(event)) { 9744 err = -E2BIG; 9745 goto err_locked; 9746 } 9747 9748 /* 9749 * Must be under the same ctx::mutex as perf_install_in_context(), 9750 * because we need to serialize with concurrent event creation. 9751 */ 9752 if (!exclusive_event_installable(event, ctx)) { 9753 /* exclusive and group stuff are assumed mutually exclusive */ 9754 WARN_ON_ONCE(move_group); 9755 9756 err = -EBUSY; 9757 goto err_locked; 9758 } 9759 9760 WARN_ON_ONCE(ctx->parent_ctx); 9761 9762 /* 9763 * This is the point on no return; we cannot fail hereafter. This is 9764 * where we start modifying current state. 9765 */ 9766 9767 if (move_group) { 9768 /* 9769 * See perf_event_ctx_lock() for comments on the details 9770 * of swizzling perf_event::ctx. 9771 */ 9772 perf_remove_from_context(group_leader, 0); 9773 9774 list_for_each_entry(sibling, &group_leader->sibling_list, 9775 group_entry) { 9776 perf_remove_from_context(sibling, 0); 9777 put_ctx(gctx); 9778 } 9779 9780 /* 9781 * Wait for everybody to stop referencing the events through 9782 * the old lists, before installing it on new lists. 9783 */ 9784 synchronize_rcu(); 9785 9786 /* 9787 * Install the group siblings before the group leader. 9788 * 9789 * Because a group leader will try and install the entire group 9790 * (through the sibling list, which is still in-tact), we can 9791 * end up with siblings installed in the wrong context. 9792 * 9793 * By installing siblings first we NO-OP because they're not 9794 * reachable through the group lists. 9795 */ 9796 list_for_each_entry(sibling, &group_leader->sibling_list, 9797 group_entry) { 9798 perf_event__state_init(sibling); 9799 perf_install_in_context(ctx, sibling, sibling->cpu); 9800 get_ctx(ctx); 9801 } 9802 9803 /* 9804 * Removing from the context ends up with disabled 9805 * event. What we want here is event in the initial 9806 * startup state, ready to be add into new context. 9807 */ 9808 perf_event__state_init(group_leader); 9809 perf_install_in_context(ctx, group_leader, group_leader->cpu); 9810 get_ctx(ctx); 9811 9812 /* 9813 * Now that all events are installed in @ctx, nothing 9814 * references @gctx anymore, so drop the last reference we have 9815 * on it. 9816 */ 9817 put_ctx(gctx); 9818 } 9819 9820 /* 9821 * Precalculate sample_data sizes; do while holding ctx::mutex such 9822 * that we're serialized against further additions and before 9823 * perf_install_in_context() which is the point the event is active and 9824 * can use these values. 9825 */ 9826 perf_event__header_size(event); 9827 perf_event__id_header_size(event); 9828 9829 event->owner = current; 9830 9831 perf_install_in_context(ctx, event, event->cpu); 9832 perf_unpin_context(ctx); 9833 9834 if (move_group) 9835 mutex_unlock(&gctx->mutex); 9836 mutex_unlock(&ctx->mutex); 9837 9838 if (task) { 9839 mutex_unlock(&task->signal->cred_guard_mutex); 9840 put_task_struct(task); 9841 } 9842 9843 put_online_cpus(); 9844 9845 mutex_lock(¤t->perf_event_mutex); 9846 list_add_tail(&event->owner_entry, ¤t->perf_event_list); 9847 mutex_unlock(¤t->perf_event_mutex); 9848 9849 /* 9850 * Drop the reference on the group_event after placing the 9851 * new event on the sibling_list. This ensures destruction 9852 * of the group leader will find the pointer to itself in 9853 * perf_group_detach(). 9854 */ 9855 fdput(group); 9856 fd_install(event_fd, event_file); 9857 return event_fd; 9858 9859 err_locked: 9860 if (move_group) 9861 mutex_unlock(&gctx->mutex); 9862 mutex_unlock(&ctx->mutex); 9863 /* err_file: */ 9864 fput(event_file); 9865 err_context: 9866 perf_unpin_context(ctx); 9867 put_ctx(ctx); 9868 err_alloc: 9869 /* 9870 * If event_file is set, the fput() above will have called ->release() 9871 * and that will take care of freeing the event. 9872 */ 9873 if (!event_file) 9874 free_event(event); 9875 err_cred: 9876 if (task) 9877 mutex_unlock(&task->signal->cred_guard_mutex); 9878 err_cpus: 9879 put_online_cpus(); 9880 err_task: 9881 if (task) 9882 put_task_struct(task); 9883 err_group_fd: 9884 fdput(group); 9885 err_fd: 9886 put_unused_fd(event_fd); 9887 return err; 9888 } 9889 9890 /** 9891 * perf_event_create_kernel_counter 9892 * 9893 * @attr: attributes of the counter to create 9894 * @cpu: cpu in which the counter is bound 9895 * @task: task to profile (NULL for percpu) 9896 */ 9897 struct perf_event * 9898 perf_event_create_kernel_counter(struct perf_event_attr *attr, int cpu, 9899 struct task_struct *task, 9900 perf_overflow_handler_t overflow_handler, 9901 void *context) 9902 { 9903 struct perf_event_context *ctx; 9904 struct perf_event *event; 9905 int err; 9906 9907 /* 9908 * Get the target context (task or percpu): 9909 */ 9910 9911 event = perf_event_alloc(attr, cpu, task, NULL, NULL, 9912 overflow_handler, context, -1); 9913 if (IS_ERR(event)) { 9914 err = PTR_ERR(event); 9915 goto err; 9916 } 9917 9918 /* Mark owner so we could distinguish it from user events. */ 9919 event->owner = TASK_TOMBSTONE; 9920 9921 ctx = find_get_context(event->pmu, task, event); 9922 if (IS_ERR(ctx)) { 9923 err = PTR_ERR(ctx); 9924 goto err_free; 9925 } 9926 9927 WARN_ON_ONCE(ctx->parent_ctx); 9928 mutex_lock(&ctx->mutex); 9929 if (ctx->task == TASK_TOMBSTONE) { 9930 err = -ESRCH; 9931 goto err_unlock; 9932 } 9933 9934 if (!exclusive_event_installable(event, ctx)) { 9935 err = -EBUSY; 9936 goto err_unlock; 9937 } 9938 9939 perf_install_in_context(ctx, event, cpu); 9940 perf_unpin_context(ctx); 9941 mutex_unlock(&ctx->mutex); 9942 9943 return event; 9944 9945 err_unlock: 9946 mutex_unlock(&ctx->mutex); 9947 perf_unpin_context(ctx); 9948 put_ctx(ctx); 9949 err_free: 9950 free_event(event); 9951 err: 9952 return ERR_PTR(err); 9953 } 9954 EXPORT_SYMBOL_GPL(perf_event_create_kernel_counter); 9955 9956 void perf_pmu_migrate_context(struct pmu *pmu, int src_cpu, int dst_cpu) 9957 { 9958 struct perf_event_context *src_ctx; 9959 struct perf_event_context *dst_ctx; 9960 struct perf_event *event, *tmp; 9961 LIST_HEAD(events); 9962 9963 src_ctx = &per_cpu_ptr(pmu->pmu_cpu_context, src_cpu)->ctx; 9964 dst_ctx = &per_cpu_ptr(pmu->pmu_cpu_context, dst_cpu)->ctx; 9965 9966 /* 9967 * See perf_event_ctx_lock() for comments on the details 9968 * of swizzling perf_event::ctx. 9969 */ 9970 mutex_lock_double(&src_ctx->mutex, &dst_ctx->mutex); 9971 list_for_each_entry_safe(event, tmp, &src_ctx->event_list, 9972 event_entry) { 9973 perf_remove_from_context(event, 0); 9974 unaccount_event_cpu(event, src_cpu); 9975 put_ctx(src_ctx); 9976 list_add(&event->migrate_entry, &events); 9977 } 9978 9979 /* 9980 * Wait for the events to quiesce before re-instating them. 9981 */ 9982 synchronize_rcu(); 9983 9984 /* 9985 * Re-instate events in 2 passes. 9986 * 9987 * Skip over group leaders and only install siblings on this first 9988 * pass, siblings will not get enabled without a leader, however a 9989 * leader will enable its siblings, even if those are still on the old 9990 * context. 9991 */ 9992 list_for_each_entry_safe(event, tmp, &events, migrate_entry) { 9993 if (event->group_leader == event) 9994 continue; 9995 9996 list_del(&event->migrate_entry); 9997 if (event->state >= PERF_EVENT_STATE_OFF) 9998 event->state = PERF_EVENT_STATE_INACTIVE; 9999 account_event_cpu(event, dst_cpu); 10000 perf_install_in_context(dst_ctx, event, dst_cpu); 10001 get_ctx(dst_ctx); 10002 } 10003 10004 /* 10005 * Once all the siblings are setup properly, install the group leaders 10006 * to make it go. 10007 */ 10008 list_for_each_entry_safe(event, tmp, &events, migrate_entry) { 10009 list_del(&event->migrate_entry); 10010 if (event->state >= PERF_EVENT_STATE_OFF) 10011 event->state = PERF_EVENT_STATE_INACTIVE; 10012 account_event_cpu(event, dst_cpu); 10013 perf_install_in_context(dst_ctx, event, dst_cpu); 10014 get_ctx(dst_ctx); 10015 } 10016 mutex_unlock(&dst_ctx->mutex); 10017 mutex_unlock(&src_ctx->mutex); 10018 } 10019 EXPORT_SYMBOL_GPL(perf_pmu_migrate_context); 10020 10021 static void sync_child_event(struct perf_event *child_event, 10022 struct task_struct *child) 10023 { 10024 struct perf_event *parent_event = child_event->parent; 10025 u64 child_val; 10026 10027 if (child_event->attr.inherit_stat) 10028 perf_event_read_event(child_event, child); 10029 10030 child_val = perf_event_count(child_event); 10031 10032 /* 10033 * Add back the child's count to the parent's count: 10034 */ 10035 atomic64_add(child_val, &parent_event->child_count); 10036 atomic64_add(child_event->total_time_enabled, 10037 &parent_event->child_total_time_enabled); 10038 atomic64_add(child_event->total_time_running, 10039 &parent_event->child_total_time_running); 10040 } 10041 10042 static void 10043 perf_event_exit_event(struct perf_event *child_event, 10044 struct perf_event_context *child_ctx, 10045 struct task_struct *child) 10046 { 10047 struct perf_event *parent_event = child_event->parent; 10048 10049 /* 10050 * Do not destroy the 'original' grouping; because of the context 10051 * switch optimization the original events could've ended up in a 10052 * random child task. 10053 * 10054 * If we were to destroy the original group, all group related 10055 * operations would cease to function properly after this random 10056 * child dies. 10057 * 10058 * Do destroy all inherited groups, we don't care about those 10059 * and being thorough is better. 10060 */ 10061 raw_spin_lock_irq(&child_ctx->lock); 10062 WARN_ON_ONCE(child_ctx->is_active); 10063 10064 if (parent_event) 10065 perf_group_detach(child_event); 10066 list_del_event(child_event, child_ctx); 10067 child_event->state = PERF_EVENT_STATE_EXIT; /* is_event_hup() */ 10068 raw_spin_unlock_irq(&child_ctx->lock); 10069 10070 /* 10071 * Parent events are governed by their filedesc, retain them. 10072 */ 10073 if (!parent_event) { 10074 perf_event_wakeup(child_event); 10075 return; 10076 } 10077 /* 10078 * Child events can be cleaned up. 10079 */ 10080 10081 sync_child_event(child_event, child); 10082 10083 /* 10084 * Remove this event from the parent's list 10085 */ 10086 WARN_ON_ONCE(parent_event->ctx->parent_ctx); 10087 mutex_lock(&parent_event->child_mutex); 10088 list_del_init(&child_event->child_list); 10089 mutex_unlock(&parent_event->child_mutex); 10090 10091 /* 10092 * Kick perf_poll() for is_event_hup(). 10093 */ 10094 perf_event_wakeup(parent_event); 10095 free_event(child_event); 10096 put_event(parent_event); 10097 } 10098 10099 static void perf_event_exit_task_context(struct task_struct *child, int ctxn) 10100 { 10101 struct perf_event_context *child_ctx, *clone_ctx = NULL; 10102 struct perf_event *child_event, *next; 10103 10104 WARN_ON_ONCE(child != current); 10105 10106 child_ctx = perf_pin_task_context(child, ctxn); 10107 if (!child_ctx) 10108 return; 10109 10110 /* 10111 * In order to reduce the amount of tricky in ctx tear-down, we hold 10112 * ctx::mutex over the entire thing. This serializes against almost 10113 * everything that wants to access the ctx. 10114 * 10115 * The exception is sys_perf_event_open() / 10116 * perf_event_create_kernel_count() which does find_get_context() 10117 * without ctx::mutex (it cannot because of the move_group double mutex 10118 * lock thing). See the comments in perf_install_in_context(). 10119 */ 10120 mutex_lock(&child_ctx->mutex); 10121 10122 /* 10123 * In a single ctx::lock section, de-schedule the events and detach the 10124 * context from the task such that we cannot ever get it scheduled back 10125 * in. 10126 */ 10127 raw_spin_lock_irq(&child_ctx->lock); 10128 task_ctx_sched_out(__get_cpu_context(child_ctx), child_ctx); 10129 10130 /* 10131 * Now that the context is inactive, destroy the task <-> ctx relation 10132 * and mark the context dead. 10133 */ 10134 RCU_INIT_POINTER(child->perf_event_ctxp[ctxn], NULL); 10135 put_ctx(child_ctx); /* cannot be last */ 10136 WRITE_ONCE(child_ctx->task, TASK_TOMBSTONE); 10137 put_task_struct(current); /* cannot be last */ 10138 10139 clone_ctx = unclone_ctx(child_ctx); 10140 raw_spin_unlock_irq(&child_ctx->lock); 10141 10142 if (clone_ctx) 10143 put_ctx(clone_ctx); 10144 10145 /* 10146 * Report the task dead after unscheduling the events so that we 10147 * won't get any samples after PERF_RECORD_EXIT. We can however still 10148 * get a few PERF_RECORD_READ events. 10149 */ 10150 perf_event_task(child, child_ctx, 0); 10151 10152 list_for_each_entry_safe(child_event, next, &child_ctx->event_list, event_entry) 10153 perf_event_exit_event(child_event, child_ctx, child); 10154 10155 mutex_unlock(&child_ctx->mutex); 10156 10157 put_ctx(child_ctx); 10158 } 10159 10160 /* 10161 * When a child task exits, feed back event values to parent events. 10162 * 10163 * Can be called with cred_guard_mutex held when called from 10164 * install_exec_creds(). 10165 */ 10166 void perf_event_exit_task(struct task_struct *child) 10167 { 10168 struct perf_event *event, *tmp; 10169 int ctxn; 10170 10171 mutex_lock(&child->perf_event_mutex); 10172 list_for_each_entry_safe(event, tmp, &child->perf_event_list, 10173 owner_entry) { 10174 list_del_init(&event->owner_entry); 10175 10176 /* 10177 * Ensure the list deletion is visible before we clear 10178 * the owner, closes a race against perf_release() where 10179 * we need to serialize on the owner->perf_event_mutex. 10180 */ 10181 smp_store_release(&event->owner, NULL); 10182 } 10183 mutex_unlock(&child->perf_event_mutex); 10184 10185 for_each_task_context_nr(ctxn) 10186 perf_event_exit_task_context(child, ctxn); 10187 10188 /* 10189 * The perf_event_exit_task_context calls perf_event_task 10190 * with child's task_ctx, which generates EXIT events for 10191 * child contexts and sets child->perf_event_ctxp[] to NULL. 10192 * At this point we need to send EXIT events to cpu contexts. 10193 */ 10194 perf_event_task(child, NULL, 0); 10195 } 10196 10197 static void perf_free_event(struct perf_event *event, 10198 struct perf_event_context *ctx) 10199 { 10200 struct perf_event *parent = event->parent; 10201 10202 if (WARN_ON_ONCE(!parent)) 10203 return; 10204 10205 mutex_lock(&parent->child_mutex); 10206 list_del_init(&event->child_list); 10207 mutex_unlock(&parent->child_mutex); 10208 10209 put_event(parent); 10210 10211 raw_spin_lock_irq(&ctx->lock); 10212 perf_group_detach(event); 10213 list_del_event(event, ctx); 10214 raw_spin_unlock_irq(&ctx->lock); 10215 free_event(event); 10216 } 10217 10218 /* 10219 * Free an unexposed, unused context as created by inheritance by 10220 * perf_event_init_task below, used by fork() in case of fail. 10221 * 10222 * Not all locks are strictly required, but take them anyway to be nice and 10223 * help out with the lockdep assertions. 10224 */ 10225 void perf_event_free_task(struct task_struct *task) 10226 { 10227 struct perf_event_context *ctx; 10228 struct perf_event *event, *tmp; 10229 int ctxn; 10230 10231 for_each_task_context_nr(ctxn) { 10232 ctx = task->perf_event_ctxp[ctxn]; 10233 if (!ctx) 10234 continue; 10235 10236 mutex_lock(&ctx->mutex); 10237 again: 10238 list_for_each_entry_safe(event, tmp, &ctx->pinned_groups, 10239 group_entry) 10240 perf_free_event(event, ctx); 10241 10242 list_for_each_entry_safe(event, tmp, &ctx->flexible_groups, 10243 group_entry) 10244 perf_free_event(event, ctx); 10245 10246 if (!list_empty(&ctx->pinned_groups) || 10247 !list_empty(&ctx->flexible_groups)) 10248 goto again; 10249 10250 mutex_unlock(&ctx->mutex); 10251 10252 put_ctx(ctx); 10253 } 10254 } 10255 10256 void perf_event_delayed_put(struct task_struct *task) 10257 { 10258 int ctxn; 10259 10260 for_each_task_context_nr(ctxn) 10261 WARN_ON_ONCE(task->perf_event_ctxp[ctxn]); 10262 } 10263 10264 struct file *perf_event_get(unsigned int fd) 10265 { 10266 struct file *file; 10267 10268 file = fget_raw(fd); 10269 if (!file) 10270 return ERR_PTR(-EBADF); 10271 10272 if (file->f_op != &perf_fops) { 10273 fput(file); 10274 return ERR_PTR(-EBADF); 10275 } 10276 10277 return file; 10278 } 10279 10280 const struct perf_event_attr *perf_event_attrs(struct perf_event *event) 10281 { 10282 if (!event) 10283 return ERR_PTR(-EINVAL); 10284 10285 return &event->attr; 10286 } 10287 10288 /* 10289 * inherit a event from parent task to child task: 10290 */ 10291 static struct perf_event * 10292 inherit_event(struct perf_event *parent_event, 10293 struct task_struct *parent, 10294 struct perf_event_context *parent_ctx, 10295 struct task_struct *child, 10296 struct perf_event *group_leader, 10297 struct perf_event_context *child_ctx) 10298 { 10299 enum perf_event_active_state parent_state = parent_event->state; 10300 struct perf_event *child_event; 10301 unsigned long flags; 10302 10303 /* 10304 * Instead of creating recursive hierarchies of events, 10305 * we link inherited events back to the original parent, 10306 * which has a filp for sure, which we use as the reference 10307 * count: 10308 */ 10309 if (parent_event->parent) 10310 parent_event = parent_event->parent; 10311 10312 child_event = perf_event_alloc(&parent_event->attr, 10313 parent_event->cpu, 10314 child, 10315 group_leader, parent_event, 10316 NULL, NULL, -1); 10317 if (IS_ERR(child_event)) 10318 return child_event; 10319 10320 /* 10321 * is_orphaned_event() and list_add_tail(&parent_event->child_list) 10322 * must be under the same lock in order to serialize against 10323 * perf_event_release_kernel(), such that either we must observe 10324 * is_orphaned_event() or they will observe us on the child_list. 10325 */ 10326 mutex_lock(&parent_event->child_mutex); 10327 if (is_orphaned_event(parent_event) || 10328 !atomic_long_inc_not_zero(&parent_event->refcount)) { 10329 mutex_unlock(&parent_event->child_mutex); 10330 free_event(child_event); 10331 return NULL; 10332 } 10333 10334 get_ctx(child_ctx); 10335 10336 /* 10337 * Make the child state follow the state of the parent event, 10338 * not its attr.disabled bit. We hold the parent's mutex, 10339 * so we won't race with perf_event_{en, dis}able_family. 10340 */ 10341 if (parent_state >= PERF_EVENT_STATE_INACTIVE) 10342 child_event->state = PERF_EVENT_STATE_INACTIVE; 10343 else 10344 child_event->state = PERF_EVENT_STATE_OFF; 10345 10346 if (parent_event->attr.freq) { 10347 u64 sample_period = parent_event->hw.sample_period; 10348 struct hw_perf_event *hwc = &child_event->hw; 10349 10350 hwc->sample_period = sample_period; 10351 hwc->last_period = sample_period; 10352 10353 local64_set(&hwc->period_left, sample_period); 10354 } 10355 10356 child_event->ctx = child_ctx; 10357 child_event->overflow_handler = parent_event->overflow_handler; 10358 child_event->overflow_handler_context 10359 = parent_event->overflow_handler_context; 10360 10361 /* 10362 * Precalculate sample_data sizes 10363 */ 10364 perf_event__header_size(child_event); 10365 perf_event__id_header_size(child_event); 10366 10367 /* 10368 * Link it up in the child's context: 10369 */ 10370 raw_spin_lock_irqsave(&child_ctx->lock, flags); 10371 add_event_to_ctx(child_event, child_ctx); 10372 raw_spin_unlock_irqrestore(&child_ctx->lock, flags); 10373 10374 /* 10375 * Link this into the parent event's child list 10376 */ 10377 list_add_tail(&child_event->child_list, &parent_event->child_list); 10378 mutex_unlock(&parent_event->child_mutex); 10379 10380 return child_event; 10381 } 10382 10383 static int inherit_group(struct perf_event *parent_event, 10384 struct task_struct *parent, 10385 struct perf_event_context *parent_ctx, 10386 struct task_struct *child, 10387 struct perf_event_context *child_ctx) 10388 { 10389 struct perf_event *leader; 10390 struct perf_event *sub; 10391 struct perf_event *child_ctr; 10392 10393 leader = inherit_event(parent_event, parent, parent_ctx, 10394 child, NULL, child_ctx); 10395 if (IS_ERR(leader)) 10396 return PTR_ERR(leader); 10397 list_for_each_entry(sub, &parent_event->sibling_list, group_entry) { 10398 child_ctr = inherit_event(sub, parent, parent_ctx, 10399 child, leader, child_ctx); 10400 if (IS_ERR(child_ctr)) 10401 return PTR_ERR(child_ctr); 10402 } 10403 return 0; 10404 } 10405 10406 static int 10407 inherit_task_group(struct perf_event *event, struct task_struct *parent, 10408 struct perf_event_context *parent_ctx, 10409 struct task_struct *child, int ctxn, 10410 int *inherited_all) 10411 { 10412 int ret; 10413 struct perf_event_context *child_ctx; 10414 10415 if (!event->attr.inherit) { 10416 *inherited_all = 0; 10417 return 0; 10418 } 10419 10420 child_ctx = child->perf_event_ctxp[ctxn]; 10421 if (!child_ctx) { 10422 /* 10423 * This is executed from the parent task context, so 10424 * inherit events that have been marked for cloning. 10425 * First allocate and initialize a context for the 10426 * child. 10427 */ 10428 10429 child_ctx = alloc_perf_context(parent_ctx->pmu, child); 10430 if (!child_ctx) 10431 return -ENOMEM; 10432 10433 child->perf_event_ctxp[ctxn] = child_ctx; 10434 } 10435 10436 ret = inherit_group(event, parent, parent_ctx, 10437 child, child_ctx); 10438 10439 if (ret) 10440 *inherited_all = 0; 10441 10442 return ret; 10443 } 10444 10445 /* 10446 * Initialize the perf_event context in task_struct 10447 */ 10448 static int perf_event_init_context(struct task_struct *child, int ctxn) 10449 { 10450 struct perf_event_context *child_ctx, *parent_ctx; 10451 struct perf_event_context *cloned_ctx; 10452 struct perf_event *event; 10453 struct task_struct *parent = current; 10454 int inherited_all = 1; 10455 unsigned long flags; 10456 int ret = 0; 10457 10458 if (likely(!parent->perf_event_ctxp[ctxn])) 10459 return 0; 10460 10461 /* 10462 * If the parent's context is a clone, pin it so it won't get 10463 * swapped under us. 10464 */ 10465 parent_ctx = perf_pin_task_context(parent, ctxn); 10466 if (!parent_ctx) 10467 return 0; 10468 10469 /* 10470 * No need to check if parent_ctx != NULL here; since we saw 10471 * it non-NULL earlier, the only reason for it to become NULL 10472 * is if we exit, and since we're currently in the middle of 10473 * a fork we can't be exiting at the same time. 10474 */ 10475 10476 /* 10477 * Lock the parent list. No need to lock the child - not PID 10478 * hashed yet and not running, so nobody can access it. 10479 */ 10480 mutex_lock(&parent_ctx->mutex); 10481 10482 /* 10483 * We dont have to disable NMIs - we are only looking at 10484 * the list, not manipulating it: 10485 */ 10486 list_for_each_entry(event, &parent_ctx->pinned_groups, group_entry) { 10487 ret = inherit_task_group(event, parent, parent_ctx, 10488 child, ctxn, &inherited_all); 10489 if (ret) 10490 break; 10491 } 10492 10493 /* 10494 * We can't hold ctx->lock when iterating the ->flexible_group list due 10495 * to allocations, but we need to prevent rotation because 10496 * rotate_ctx() will change the list from interrupt context. 10497 */ 10498 raw_spin_lock_irqsave(&parent_ctx->lock, flags); 10499 parent_ctx->rotate_disable = 1; 10500 raw_spin_unlock_irqrestore(&parent_ctx->lock, flags); 10501 10502 list_for_each_entry(event, &parent_ctx->flexible_groups, group_entry) { 10503 ret = inherit_task_group(event, parent, parent_ctx, 10504 child, ctxn, &inherited_all); 10505 if (ret) 10506 break; 10507 } 10508 10509 raw_spin_lock_irqsave(&parent_ctx->lock, flags); 10510 parent_ctx->rotate_disable = 0; 10511 10512 child_ctx = child->perf_event_ctxp[ctxn]; 10513 10514 if (child_ctx && inherited_all) { 10515 /* 10516 * Mark the child context as a clone of the parent 10517 * context, or of whatever the parent is a clone of. 10518 * 10519 * Note that if the parent is a clone, the holding of 10520 * parent_ctx->lock avoids it from being uncloned. 10521 */ 10522 cloned_ctx = parent_ctx->parent_ctx; 10523 if (cloned_ctx) { 10524 child_ctx->parent_ctx = cloned_ctx; 10525 child_ctx->parent_gen = parent_ctx->parent_gen; 10526 } else { 10527 child_ctx->parent_ctx = parent_ctx; 10528 child_ctx->parent_gen = parent_ctx->generation; 10529 } 10530 get_ctx(child_ctx->parent_ctx); 10531 } 10532 10533 raw_spin_unlock_irqrestore(&parent_ctx->lock, flags); 10534 mutex_unlock(&parent_ctx->mutex); 10535 10536 perf_unpin_context(parent_ctx); 10537 put_ctx(parent_ctx); 10538 10539 return ret; 10540 } 10541 10542 /* 10543 * Initialize the perf_event context in task_struct 10544 */ 10545 int perf_event_init_task(struct task_struct *child) 10546 { 10547 int ctxn, ret; 10548 10549 memset(child->perf_event_ctxp, 0, sizeof(child->perf_event_ctxp)); 10550 mutex_init(&child->perf_event_mutex); 10551 INIT_LIST_HEAD(&child->perf_event_list); 10552 10553 for_each_task_context_nr(ctxn) { 10554 ret = perf_event_init_context(child, ctxn); 10555 if (ret) { 10556 perf_event_free_task(child); 10557 return ret; 10558 } 10559 } 10560 10561 return 0; 10562 } 10563 10564 static void __init perf_event_init_all_cpus(void) 10565 { 10566 struct swevent_htable *swhash; 10567 int cpu; 10568 10569 for_each_possible_cpu(cpu) { 10570 swhash = &per_cpu(swevent_htable, cpu); 10571 mutex_init(&swhash->hlist_mutex); 10572 INIT_LIST_HEAD(&per_cpu(active_ctx_list, cpu)); 10573 10574 INIT_LIST_HEAD(&per_cpu(pmu_sb_events.list, cpu)); 10575 raw_spin_lock_init(&per_cpu(pmu_sb_events.lock, cpu)); 10576 10577 INIT_LIST_HEAD(&per_cpu(sched_cb_list, cpu)); 10578 } 10579 } 10580 10581 int perf_event_init_cpu(unsigned int cpu) 10582 { 10583 struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu); 10584 10585 mutex_lock(&swhash->hlist_mutex); 10586 if (swhash->hlist_refcount > 0 && !swevent_hlist_deref(swhash)) { 10587 struct swevent_hlist *hlist; 10588 10589 hlist = kzalloc_node(sizeof(*hlist), GFP_KERNEL, cpu_to_node(cpu)); 10590 WARN_ON(!hlist); 10591 rcu_assign_pointer(swhash->swevent_hlist, hlist); 10592 } 10593 mutex_unlock(&swhash->hlist_mutex); 10594 return 0; 10595 } 10596 10597 #if defined CONFIG_HOTPLUG_CPU || defined CONFIG_KEXEC_CORE 10598 static void __perf_event_exit_context(void *__info) 10599 { 10600 struct perf_event_context *ctx = __info; 10601 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx); 10602 struct perf_event *event; 10603 10604 raw_spin_lock(&ctx->lock); 10605 list_for_each_entry(event, &ctx->event_list, event_entry) 10606 __perf_remove_from_context(event, cpuctx, ctx, (void *)DETACH_GROUP); 10607 raw_spin_unlock(&ctx->lock); 10608 } 10609 10610 static void perf_event_exit_cpu_context(int cpu) 10611 { 10612 struct perf_event_context *ctx; 10613 struct pmu *pmu; 10614 int idx; 10615 10616 idx = srcu_read_lock(&pmus_srcu); 10617 list_for_each_entry_rcu(pmu, &pmus, entry) { 10618 ctx = &per_cpu_ptr(pmu->pmu_cpu_context, cpu)->ctx; 10619 10620 mutex_lock(&ctx->mutex); 10621 smp_call_function_single(cpu, __perf_event_exit_context, ctx, 1); 10622 mutex_unlock(&ctx->mutex); 10623 } 10624 srcu_read_unlock(&pmus_srcu, idx); 10625 } 10626 #else 10627 10628 static void perf_event_exit_cpu_context(int cpu) { } 10629 10630 #endif 10631 10632 int perf_event_exit_cpu(unsigned int cpu) 10633 { 10634 perf_event_exit_cpu_context(cpu); 10635 return 0; 10636 } 10637 10638 static int 10639 perf_reboot(struct notifier_block *notifier, unsigned long val, void *v) 10640 { 10641 int cpu; 10642 10643 for_each_online_cpu(cpu) 10644 perf_event_exit_cpu(cpu); 10645 10646 return NOTIFY_OK; 10647 } 10648 10649 /* 10650 * Run the perf reboot notifier at the very last possible moment so that 10651 * the generic watchdog code runs as long as possible. 10652 */ 10653 static struct notifier_block perf_reboot_notifier = { 10654 .notifier_call = perf_reboot, 10655 .priority = INT_MIN, 10656 }; 10657 10658 void __init perf_event_init(void) 10659 { 10660 int ret; 10661 10662 idr_init(&pmu_idr); 10663 10664 perf_event_init_all_cpus(); 10665 init_srcu_struct(&pmus_srcu); 10666 perf_pmu_register(&perf_swevent, "software", PERF_TYPE_SOFTWARE); 10667 perf_pmu_register(&perf_cpu_clock, NULL, -1); 10668 perf_pmu_register(&perf_task_clock, NULL, -1); 10669 perf_tp_register(); 10670 perf_event_init_cpu(smp_processor_id()); 10671 register_reboot_notifier(&perf_reboot_notifier); 10672 10673 ret = init_hw_breakpoint(); 10674 WARN(ret, "hw_breakpoint initialization failed with: %d", ret); 10675 10676 /* 10677 * Build time assertion that we keep the data_head at the intended 10678 * location. IOW, validation we got the __reserved[] size right. 10679 */ 10680 BUILD_BUG_ON((offsetof(struct perf_event_mmap_page, data_head)) 10681 != 1024); 10682 } 10683 10684 ssize_t perf_event_sysfs_show(struct device *dev, struct device_attribute *attr, 10685 char *page) 10686 { 10687 struct perf_pmu_events_attr *pmu_attr = 10688 container_of(attr, struct perf_pmu_events_attr, attr); 10689 10690 if (pmu_attr->event_str) 10691 return sprintf(page, "%s\n", pmu_attr->event_str); 10692 10693 return 0; 10694 } 10695 EXPORT_SYMBOL_GPL(perf_event_sysfs_show); 10696 10697 static int __init perf_event_sysfs_init(void) 10698 { 10699 struct pmu *pmu; 10700 int ret; 10701 10702 mutex_lock(&pmus_lock); 10703 10704 ret = bus_register(&pmu_bus); 10705 if (ret) 10706 goto unlock; 10707 10708 list_for_each_entry(pmu, &pmus, entry) { 10709 if (!pmu->name || pmu->type < 0) 10710 continue; 10711 10712 ret = pmu_dev_alloc(pmu); 10713 WARN(ret, "Failed to register pmu: %s, reason %d\n", pmu->name, ret); 10714 } 10715 pmu_bus_running = 1; 10716 ret = 0; 10717 10718 unlock: 10719 mutex_unlock(&pmus_lock); 10720 10721 return ret; 10722 } 10723 device_initcall(perf_event_sysfs_init); 10724 10725 #ifdef CONFIG_CGROUP_PERF 10726 static struct cgroup_subsys_state * 10727 perf_cgroup_css_alloc(struct cgroup_subsys_state *parent_css) 10728 { 10729 struct perf_cgroup *jc; 10730 10731 jc = kzalloc(sizeof(*jc), GFP_KERNEL); 10732 if (!jc) 10733 return ERR_PTR(-ENOMEM); 10734 10735 jc->info = alloc_percpu(struct perf_cgroup_info); 10736 if (!jc->info) { 10737 kfree(jc); 10738 return ERR_PTR(-ENOMEM); 10739 } 10740 10741 return &jc->css; 10742 } 10743 10744 static void perf_cgroup_css_free(struct cgroup_subsys_state *css) 10745 { 10746 struct perf_cgroup *jc = container_of(css, struct perf_cgroup, css); 10747 10748 free_percpu(jc->info); 10749 kfree(jc); 10750 } 10751 10752 static int __perf_cgroup_move(void *info) 10753 { 10754 struct task_struct *task = info; 10755 rcu_read_lock(); 10756 perf_cgroup_switch(task, PERF_CGROUP_SWOUT | PERF_CGROUP_SWIN); 10757 rcu_read_unlock(); 10758 return 0; 10759 } 10760 10761 static void perf_cgroup_attach(struct cgroup_taskset *tset) 10762 { 10763 struct task_struct *task; 10764 struct cgroup_subsys_state *css; 10765 10766 cgroup_taskset_for_each(task, css, tset) 10767 task_function_call(task, __perf_cgroup_move, task); 10768 } 10769 10770 struct cgroup_subsys perf_event_cgrp_subsys = { 10771 .css_alloc = perf_cgroup_css_alloc, 10772 .css_free = perf_cgroup_css_free, 10773 .attach = perf_cgroup_attach, 10774 }; 10775 #endif /* CONFIG_CGROUP_PERF */ 10776