1 /* 2 * Performance events core code: 3 * 4 * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de> 5 * Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar 6 * Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra 7 * Copyright © 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com> 8 * 9 * For licensing details see kernel-base/COPYING 10 */ 11 12 #include <linux/fs.h> 13 #include <linux/mm.h> 14 #include <linux/cpu.h> 15 #include <linux/smp.h> 16 #include <linux/idr.h> 17 #include <linux/file.h> 18 #include <linux/poll.h> 19 #include <linux/slab.h> 20 #include <linux/hash.h> 21 #include <linux/tick.h> 22 #include <linux/sysfs.h> 23 #include <linux/dcache.h> 24 #include <linux/percpu.h> 25 #include <linux/ptrace.h> 26 #include <linux/reboot.h> 27 #include <linux/vmstat.h> 28 #include <linux/device.h> 29 #include <linux/export.h> 30 #include <linux/vmalloc.h> 31 #include <linux/hardirq.h> 32 #include <linux/rculist.h> 33 #include <linux/uaccess.h> 34 #include <linux/syscalls.h> 35 #include <linux/anon_inodes.h> 36 #include <linux/kernel_stat.h> 37 #include <linux/cgroup.h> 38 #include <linux/perf_event.h> 39 #include <linux/trace_events.h> 40 #include <linux/hw_breakpoint.h> 41 #include <linux/mm_types.h> 42 #include <linux/module.h> 43 #include <linux/mman.h> 44 #include <linux/compat.h> 45 #include <linux/bpf.h> 46 #include <linux/filter.h> 47 48 #include "internal.h" 49 50 #include <asm/irq_regs.h> 51 52 typedef int (*remote_function_f)(void *); 53 54 struct remote_function_call { 55 struct task_struct *p; 56 remote_function_f func; 57 void *info; 58 int ret; 59 }; 60 61 static void remote_function(void *data) 62 { 63 struct remote_function_call *tfc = data; 64 struct task_struct *p = tfc->p; 65 66 if (p) { 67 tfc->ret = -EAGAIN; 68 if (task_cpu(p) != smp_processor_id() || !task_curr(p)) 69 return; 70 } 71 72 tfc->ret = tfc->func(tfc->info); 73 } 74 75 /** 76 * task_function_call - call a function on the cpu on which a task runs 77 * @p: the task to evaluate 78 * @func: the function to be called 79 * @info: the function call argument 80 * 81 * Calls the function @func when the task is currently running. This might 82 * be on the current CPU, which just calls the function directly 83 * 84 * returns: @func return value, or 85 * -ESRCH - when the process isn't running 86 * -EAGAIN - when the process moved away 87 */ 88 static int 89 task_function_call(struct task_struct *p, remote_function_f func, void *info) 90 { 91 struct remote_function_call data = { 92 .p = p, 93 .func = func, 94 .info = info, 95 .ret = -ESRCH, /* No such (running) process */ 96 }; 97 98 if (task_curr(p)) 99 smp_call_function_single(task_cpu(p), remote_function, &data, 1); 100 101 return data.ret; 102 } 103 104 /** 105 * cpu_function_call - call a function on the cpu 106 * @func: the function to be called 107 * @info: the function call argument 108 * 109 * Calls the function @func on the remote cpu. 110 * 111 * returns: @func return value or -ENXIO when the cpu is offline 112 */ 113 static int cpu_function_call(int cpu, remote_function_f func, void *info) 114 { 115 struct remote_function_call data = { 116 .p = NULL, 117 .func = func, 118 .info = info, 119 .ret = -ENXIO, /* No such CPU */ 120 }; 121 122 smp_call_function_single(cpu, remote_function, &data, 1); 123 124 return data.ret; 125 } 126 127 static inline struct perf_cpu_context * 128 __get_cpu_context(struct perf_event_context *ctx) 129 { 130 return this_cpu_ptr(ctx->pmu->pmu_cpu_context); 131 } 132 133 static void perf_ctx_lock(struct perf_cpu_context *cpuctx, 134 struct perf_event_context *ctx) 135 { 136 raw_spin_lock(&cpuctx->ctx.lock); 137 if (ctx) 138 raw_spin_lock(&ctx->lock); 139 } 140 141 static void perf_ctx_unlock(struct perf_cpu_context *cpuctx, 142 struct perf_event_context *ctx) 143 { 144 if (ctx) 145 raw_spin_unlock(&ctx->lock); 146 raw_spin_unlock(&cpuctx->ctx.lock); 147 } 148 149 #define TASK_TOMBSTONE ((void *)-1L) 150 151 static bool is_kernel_event(struct perf_event *event) 152 { 153 return READ_ONCE(event->owner) == TASK_TOMBSTONE; 154 } 155 156 /* 157 * On task ctx scheduling... 158 * 159 * When !ctx->nr_events a task context will not be scheduled. This means 160 * we can disable the scheduler hooks (for performance) without leaving 161 * pending task ctx state. 162 * 163 * This however results in two special cases: 164 * 165 * - removing the last event from a task ctx; this is relatively straight 166 * forward and is done in __perf_remove_from_context. 167 * 168 * - adding the first event to a task ctx; this is tricky because we cannot 169 * rely on ctx->is_active and therefore cannot use event_function_call(). 170 * See perf_install_in_context(). 171 * 172 * This is because we need a ctx->lock serialized variable (ctx->is_active) 173 * to reliably determine if a particular task/context is scheduled in. The 174 * task_curr() use in task_function_call() is racy in that a remote context 175 * switch is not a single atomic operation. 176 * 177 * As is, the situation is 'safe' because we set rq->curr before we do the 178 * actual context switch. This means that task_curr() will fail early, but 179 * we'll continue spinning on ctx->is_active until we've passed 180 * perf_event_task_sched_out(). 181 * 182 * Without this ctx->lock serialized variable we could have race where we find 183 * the task (and hence the context) would not be active while in fact they are. 184 * 185 * If ctx->nr_events, then ctx->is_active and cpuctx->task_ctx are set. 186 */ 187 188 typedef void (*event_f)(struct perf_event *, struct perf_cpu_context *, 189 struct perf_event_context *, void *); 190 191 struct event_function_struct { 192 struct perf_event *event; 193 event_f func; 194 void *data; 195 }; 196 197 static int event_function(void *info) 198 { 199 struct event_function_struct *efs = info; 200 struct perf_event *event = efs->event; 201 struct perf_event_context *ctx = event->ctx; 202 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx); 203 struct perf_event_context *task_ctx = cpuctx->task_ctx; 204 int ret = 0; 205 206 WARN_ON_ONCE(!irqs_disabled()); 207 208 perf_ctx_lock(cpuctx, task_ctx); 209 /* 210 * Since we do the IPI call without holding ctx->lock things can have 211 * changed, double check we hit the task we set out to hit. 212 */ 213 if (ctx->task) { 214 if (ctx->task != current) { 215 ret = -EAGAIN; 216 goto unlock; 217 } 218 219 /* 220 * We only use event_function_call() on established contexts, 221 * and event_function() is only ever called when active (or 222 * rather, we'll have bailed in task_function_call() or the 223 * above ctx->task != current test), therefore we must have 224 * ctx->is_active here. 225 */ 226 WARN_ON_ONCE(!ctx->is_active); 227 /* 228 * And since we have ctx->is_active, cpuctx->task_ctx must 229 * match. 230 */ 231 WARN_ON_ONCE(task_ctx != ctx); 232 } else { 233 WARN_ON_ONCE(&cpuctx->ctx != ctx); 234 } 235 236 efs->func(event, cpuctx, ctx, efs->data); 237 unlock: 238 perf_ctx_unlock(cpuctx, task_ctx); 239 240 return ret; 241 } 242 243 static void event_function_local(struct perf_event *event, event_f func, void *data) 244 { 245 struct event_function_struct efs = { 246 .event = event, 247 .func = func, 248 .data = data, 249 }; 250 251 int ret = event_function(&efs); 252 WARN_ON_ONCE(ret); 253 } 254 255 static void event_function_call(struct perf_event *event, event_f func, void *data) 256 { 257 struct perf_event_context *ctx = event->ctx; 258 struct task_struct *task = READ_ONCE(ctx->task); /* verified in event_function */ 259 struct event_function_struct efs = { 260 .event = event, 261 .func = func, 262 .data = data, 263 }; 264 265 if (!event->parent) { 266 /* 267 * If this is a !child event, we must hold ctx::mutex to 268 * stabilize the the event->ctx relation. See 269 * perf_event_ctx_lock(). 270 */ 271 lockdep_assert_held(&ctx->mutex); 272 } 273 274 if (!task) { 275 cpu_function_call(event->cpu, event_function, &efs); 276 return; 277 } 278 279 again: 280 if (task == TASK_TOMBSTONE) 281 return; 282 283 if (!task_function_call(task, event_function, &efs)) 284 return; 285 286 raw_spin_lock_irq(&ctx->lock); 287 /* 288 * Reload the task pointer, it might have been changed by 289 * a concurrent perf_event_context_sched_out(). 290 */ 291 task = ctx->task; 292 if (task != TASK_TOMBSTONE) { 293 if (ctx->is_active) { 294 raw_spin_unlock_irq(&ctx->lock); 295 goto again; 296 } 297 func(event, NULL, ctx, data); 298 } 299 raw_spin_unlock_irq(&ctx->lock); 300 } 301 302 #define PERF_FLAG_ALL (PERF_FLAG_FD_NO_GROUP |\ 303 PERF_FLAG_FD_OUTPUT |\ 304 PERF_FLAG_PID_CGROUP |\ 305 PERF_FLAG_FD_CLOEXEC) 306 307 /* 308 * branch priv levels that need permission checks 309 */ 310 #define PERF_SAMPLE_BRANCH_PERM_PLM \ 311 (PERF_SAMPLE_BRANCH_KERNEL |\ 312 PERF_SAMPLE_BRANCH_HV) 313 314 enum event_type_t { 315 EVENT_FLEXIBLE = 0x1, 316 EVENT_PINNED = 0x2, 317 EVENT_ALL = EVENT_FLEXIBLE | EVENT_PINNED, 318 }; 319 320 /* 321 * perf_sched_events : >0 events exist 322 * perf_cgroup_events: >0 per-cpu cgroup events exist on this cpu 323 */ 324 struct static_key_deferred perf_sched_events __read_mostly; 325 static DEFINE_PER_CPU(atomic_t, perf_cgroup_events); 326 static DEFINE_PER_CPU(int, perf_sched_cb_usages); 327 328 static atomic_t nr_mmap_events __read_mostly; 329 static atomic_t nr_comm_events __read_mostly; 330 static atomic_t nr_task_events __read_mostly; 331 static atomic_t nr_freq_events __read_mostly; 332 static atomic_t nr_switch_events __read_mostly; 333 334 static LIST_HEAD(pmus); 335 static DEFINE_MUTEX(pmus_lock); 336 static struct srcu_struct pmus_srcu; 337 338 /* 339 * perf event paranoia level: 340 * -1 - not paranoid at all 341 * 0 - disallow raw tracepoint access for unpriv 342 * 1 - disallow cpu events for unpriv 343 * 2 - disallow kernel profiling for unpriv 344 */ 345 int sysctl_perf_event_paranoid __read_mostly = 1; 346 347 /* Minimum for 512 kiB + 1 user control page */ 348 int sysctl_perf_event_mlock __read_mostly = 512 + (PAGE_SIZE / 1024); /* 'free' kiB per user */ 349 350 /* 351 * max perf event sample rate 352 */ 353 #define DEFAULT_MAX_SAMPLE_RATE 100000 354 #define DEFAULT_SAMPLE_PERIOD_NS (NSEC_PER_SEC / DEFAULT_MAX_SAMPLE_RATE) 355 #define DEFAULT_CPU_TIME_MAX_PERCENT 25 356 357 int sysctl_perf_event_sample_rate __read_mostly = DEFAULT_MAX_SAMPLE_RATE; 358 359 static int max_samples_per_tick __read_mostly = DIV_ROUND_UP(DEFAULT_MAX_SAMPLE_RATE, HZ); 360 static int perf_sample_period_ns __read_mostly = DEFAULT_SAMPLE_PERIOD_NS; 361 362 static int perf_sample_allowed_ns __read_mostly = 363 DEFAULT_SAMPLE_PERIOD_NS * DEFAULT_CPU_TIME_MAX_PERCENT / 100; 364 365 static void update_perf_cpu_limits(void) 366 { 367 u64 tmp = perf_sample_period_ns; 368 369 tmp *= sysctl_perf_cpu_time_max_percent; 370 do_div(tmp, 100); 371 ACCESS_ONCE(perf_sample_allowed_ns) = tmp; 372 } 373 374 static int perf_rotate_context(struct perf_cpu_context *cpuctx); 375 376 int perf_proc_update_handler(struct ctl_table *table, int write, 377 void __user *buffer, size_t *lenp, 378 loff_t *ppos) 379 { 380 int ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); 381 382 if (ret || !write) 383 return ret; 384 385 max_samples_per_tick = DIV_ROUND_UP(sysctl_perf_event_sample_rate, HZ); 386 perf_sample_period_ns = NSEC_PER_SEC / sysctl_perf_event_sample_rate; 387 update_perf_cpu_limits(); 388 389 return 0; 390 } 391 392 int sysctl_perf_cpu_time_max_percent __read_mostly = DEFAULT_CPU_TIME_MAX_PERCENT; 393 394 int perf_cpu_time_max_percent_handler(struct ctl_table *table, int write, 395 void __user *buffer, size_t *lenp, 396 loff_t *ppos) 397 { 398 int ret = proc_dointvec(table, write, buffer, lenp, ppos); 399 400 if (ret || !write) 401 return ret; 402 403 update_perf_cpu_limits(); 404 405 return 0; 406 } 407 408 /* 409 * perf samples are done in some very critical code paths (NMIs). 410 * If they take too much CPU time, the system can lock up and not 411 * get any real work done. This will drop the sample rate when 412 * we detect that events are taking too long. 413 */ 414 #define NR_ACCUMULATED_SAMPLES 128 415 static DEFINE_PER_CPU(u64, running_sample_length); 416 417 static void perf_duration_warn(struct irq_work *w) 418 { 419 u64 allowed_ns = ACCESS_ONCE(perf_sample_allowed_ns); 420 u64 avg_local_sample_len; 421 u64 local_samples_len; 422 423 local_samples_len = __this_cpu_read(running_sample_length); 424 avg_local_sample_len = local_samples_len/NR_ACCUMULATED_SAMPLES; 425 426 printk_ratelimited(KERN_WARNING 427 "perf interrupt took too long (%lld > %lld), lowering " 428 "kernel.perf_event_max_sample_rate to %d\n", 429 avg_local_sample_len, allowed_ns >> 1, 430 sysctl_perf_event_sample_rate); 431 } 432 433 static DEFINE_IRQ_WORK(perf_duration_work, perf_duration_warn); 434 435 void perf_sample_event_took(u64 sample_len_ns) 436 { 437 u64 allowed_ns = ACCESS_ONCE(perf_sample_allowed_ns); 438 u64 avg_local_sample_len; 439 u64 local_samples_len; 440 441 if (allowed_ns == 0) 442 return; 443 444 /* decay the counter by 1 average sample */ 445 local_samples_len = __this_cpu_read(running_sample_length); 446 local_samples_len -= local_samples_len/NR_ACCUMULATED_SAMPLES; 447 local_samples_len += sample_len_ns; 448 __this_cpu_write(running_sample_length, local_samples_len); 449 450 /* 451 * note: this will be biased artifically low until we have 452 * seen NR_ACCUMULATED_SAMPLES. Doing it this way keeps us 453 * from having to maintain a count. 454 */ 455 avg_local_sample_len = local_samples_len/NR_ACCUMULATED_SAMPLES; 456 457 if (avg_local_sample_len <= allowed_ns) 458 return; 459 460 if (max_samples_per_tick <= 1) 461 return; 462 463 max_samples_per_tick = DIV_ROUND_UP(max_samples_per_tick, 2); 464 sysctl_perf_event_sample_rate = max_samples_per_tick * HZ; 465 perf_sample_period_ns = NSEC_PER_SEC / sysctl_perf_event_sample_rate; 466 467 update_perf_cpu_limits(); 468 469 if (!irq_work_queue(&perf_duration_work)) { 470 early_printk("perf interrupt took too long (%lld > %lld), lowering " 471 "kernel.perf_event_max_sample_rate to %d\n", 472 avg_local_sample_len, allowed_ns >> 1, 473 sysctl_perf_event_sample_rate); 474 } 475 } 476 477 static atomic64_t perf_event_id; 478 479 static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx, 480 enum event_type_t event_type); 481 482 static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx, 483 enum event_type_t event_type, 484 struct task_struct *task); 485 486 static void update_context_time(struct perf_event_context *ctx); 487 static u64 perf_event_time(struct perf_event *event); 488 489 void __weak perf_event_print_debug(void) { } 490 491 extern __weak const char *perf_pmu_name(void) 492 { 493 return "pmu"; 494 } 495 496 static inline u64 perf_clock(void) 497 { 498 return local_clock(); 499 } 500 501 static inline u64 perf_event_clock(struct perf_event *event) 502 { 503 return event->clock(); 504 } 505 506 #ifdef CONFIG_CGROUP_PERF 507 508 static inline bool 509 perf_cgroup_match(struct perf_event *event) 510 { 511 struct perf_event_context *ctx = event->ctx; 512 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx); 513 514 /* @event doesn't care about cgroup */ 515 if (!event->cgrp) 516 return true; 517 518 /* wants specific cgroup scope but @cpuctx isn't associated with any */ 519 if (!cpuctx->cgrp) 520 return false; 521 522 /* 523 * Cgroup scoping is recursive. An event enabled for a cgroup is 524 * also enabled for all its descendant cgroups. If @cpuctx's 525 * cgroup is a descendant of @event's (the test covers identity 526 * case), it's a match. 527 */ 528 return cgroup_is_descendant(cpuctx->cgrp->css.cgroup, 529 event->cgrp->css.cgroup); 530 } 531 532 static inline void perf_detach_cgroup(struct perf_event *event) 533 { 534 css_put(&event->cgrp->css); 535 event->cgrp = NULL; 536 } 537 538 static inline int is_cgroup_event(struct perf_event *event) 539 { 540 return event->cgrp != NULL; 541 } 542 543 static inline u64 perf_cgroup_event_time(struct perf_event *event) 544 { 545 struct perf_cgroup_info *t; 546 547 t = per_cpu_ptr(event->cgrp->info, event->cpu); 548 return t->time; 549 } 550 551 static inline void __update_cgrp_time(struct perf_cgroup *cgrp) 552 { 553 struct perf_cgroup_info *info; 554 u64 now; 555 556 now = perf_clock(); 557 558 info = this_cpu_ptr(cgrp->info); 559 560 info->time += now - info->timestamp; 561 info->timestamp = now; 562 } 563 564 static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx) 565 { 566 struct perf_cgroup *cgrp_out = cpuctx->cgrp; 567 if (cgrp_out) 568 __update_cgrp_time(cgrp_out); 569 } 570 571 static inline void update_cgrp_time_from_event(struct perf_event *event) 572 { 573 struct perf_cgroup *cgrp; 574 575 /* 576 * ensure we access cgroup data only when needed and 577 * when we know the cgroup is pinned (css_get) 578 */ 579 if (!is_cgroup_event(event)) 580 return; 581 582 cgrp = perf_cgroup_from_task(current, event->ctx); 583 /* 584 * Do not update time when cgroup is not active 585 */ 586 if (cgrp == event->cgrp) 587 __update_cgrp_time(event->cgrp); 588 } 589 590 static inline void 591 perf_cgroup_set_timestamp(struct task_struct *task, 592 struct perf_event_context *ctx) 593 { 594 struct perf_cgroup *cgrp; 595 struct perf_cgroup_info *info; 596 597 /* 598 * ctx->lock held by caller 599 * ensure we do not access cgroup data 600 * unless we have the cgroup pinned (css_get) 601 */ 602 if (!task || !ctx->nr_cgroups) 603 return; 604 605 cgrp = perf_cgroup_from_task(task, ctx); 606 info = this_cpu_ptr(cgrp->info); 607 info->timestamp = ctx->timestamp; 608 } 609 610 #define PERF_CGROUP_SWOUT 0x1 /* cgroup switch out every event */ 611 #define PERF_CGROUP_SWIN 0x2 /* cgroup switch in events based on task */ 612 613 /* 614 * reschedule events based on the cgroup constraint of task. 615 * 616 * mode SWOUT : schedule out everything 617 * mode SWIN : schedule in based on cgroup for next 618 */ 619 static void perf_cgroup_switch(struct task_struct *task, int mode) 620 { 621 struct perf_cpu_context *cpuctx; 622 struct pmu *pmu; 623 unsigned long flags; 624 625 /* 626 * disable interrupts to avoid geting nr_cgroup 627 * changes via __perf_event_disable(). Also 628 * avoids preemption. 629 */ 630 local_irq_save(flags); 631 632 /* 633 * we reschedule only in the presence of cgroup 634 * constrained events. 635 */ 636 637 list_for_each_entry_rcu(pmu, &pmus, entry) { 638 cpuctx = this_cpu_ptr(pmu->pmu_cpu_context); 639 if (cpuctx->unique_pmu != pmu) 640 continue; /* ensure we process each cpuctx once */ 641 642 /* 643 * perf_cgroup_events says at least one 644 * context on this CPU has cgroup events. 645 * 646 * ctx->nr_cgroups reports the number of cgroup 647 * events for a context. 648 */ 649 if (cpuctx->ctx.nr_cgroups > 0) { 650 perf_ctx_lock(cpuctx, cpuctx->task_ctx); 651 perf_pmu_disable(cpuctx->ctx.pmu); 652 653 if (mode & PERF_CGROUP_SWOUT) { 654 cpu_ctx_sched_out(cpuctx, EVENT_ALL); 655 /* 656 * must not be done before ctxswout due 657 * to event_filter_match() in event_sched_out() 658 */ 659 cpuctx->cgrp = NULL; 660 } 661 662 if (mode & PERF_CGROUP_SWIN) { 663 WARN_ON_ONCE(cpuctx->cgrp); 664 /* 665 * set cgrp before ctxsw in to allow 666 * event_filter_match() to not have to pass 667 * task around 668 * we pass the cpuctx->ctx to perf_cgroup_from_task() 669 * because cgorup events are only per-cpu 670 */ 671 cpuctx->cgrp = perf_cgroup_from_task(task, &cpuctx->ctx); 672 cpu_ctx_sched_in(cpuctx, EVENT_ALL, task); 673 } 674 perf_pmu_enable(cpuctx->ctx.pmu); 675 perf_ctx_unlock(cpuctx, cpuctx->task_ctx); 676 } 677 } 678 679 local_irq_restore(flags); 680 } 681 682 static inline void perf_cgroup_sched_out(struct task_struct *task, 683 struct task_struct *next) 684 { 685 struct perf_cgroup *cgrp1; 686 struct perf_cgroup *cgrp2 = NULL; 687 688 rcu_read_lock(); 689 /* 690 * we come here when we know perf_cgroup_events > 0 691 * we do not need to pass the ctx here because we know 692 * we are holding the rcu lock 693 */ 694 cgrp1 = perf_cgroup_from_task(task, NULL); 695 cgrp2 = perf_cgroup_from_task(next, NULL); 696 697 /* 698 * only schedule out current cgroup events if we know 699 * that we are switching to a different cgroup. Otherwise, 700 * do no touch the cgroup events. 701 */ 702 if (cgrp1 != cgrp2) 703 perf_cgroup_switch(task, PERF_CGROUP_SWOUT); 704 705 rcu_read_unlock(); 706 } 707 708 static inline void perf_cgroup_sched_in(struct task_struct *prev, 709 struct task_struct *task) 710 { 711 struct perf_cgroup *cgrp1; 712 struct perf_cgroup *cgrp2 = NULL; 713 714 rcu_read_lock(); 715 /* 716 * we come here when we know perf_cgroup_events > 0 717 * we do not need to pass the ctx here because we know 718 * we are holding the rcu lock 719 */ 720 cgrp1 = perf_cgroup_from_task(task, NULL); 721 cgrp2 = perf_cgroup_from_task(prev, NULL); 722 723 /* 724 * only need to schedule in cgroup events if we are changing 725 * cgroup during ctxsw. Cgroup events were not scheduled 726 * out of ctxsw out if that was not the case. 727 */ 728 if (cgrp1 != cgrp2) 729 perf_cgroup_switch(task, PERF_CGROUP_SWIN); 730 731 rcu_read_unlock(); 732 } 733 734 static inline int perf_cgroup_connect(int fd, struct perf_event *event, 735 struct perf_event_attr *attr, 736 struct perf_event *group_leader) 737 { 738 struct perf_cgroup *cgrp; 739 struct cgroup_subsys_state *css; 740 struct fd f = fdget(fd); 741 int ret = 0; 742 743 if (!f.file) 744 return -EBADF; 745 746 css = css_tryget_online_from_dir(f.file->f_path.dentry, 747 &perf_event_cgrp_subsys); 748 if (IS_ERR(css)) { 749 ret = PTR_ERR(css); 750 goto out; 751 } 752 753 cgrp = container_of(css, struct perf_cgroup, css); 754 event->cgrp = cgrp; 755 756 /* 757 * all events in a group must monitor 758 * the same cgroup because a task belongs 759 * to only one perf cgroup at a time 760 */ 761 if (group_leader && group_leader->cgrp != cgrp) { 762 perf_detach_cgroup(event); 763 ret = -EINVAL; 764 } 765 out: 766 fdput(f); 767 return ret; 768 } 769 770 static inline void 771 perf_cgroup_set_shadow_time(struct perf_event *event, u64 now) 772 { 773 struct perf_cgroup_info *t; 774 t = per_cpu_ptr(event->cgrp->info, event->cpu); 775 event->shadow_ctx_time = now - t->timestamp; 776 } 777 778 static inline void 779 perf_cgroup_defer_enabled(struct perf_event *event) 780 { 781 /* 782 * when the current task's perf cgroup does not match 783 * the event's, we need to remember to call the 784 * perf_mark_enable() function the first time a task with 785 * a matching perf cgroup is scheduled in. 786 */ 787 if (is_cgroup_event(event) && !perf_cgroup_match(event)) 788 event->cgrp_defer_enabled = 1; 789 } 790 791 static inline void 792 perf_cgroup_mark_enabled(struct perf_event *event, 793 struct perf_event_context *ctx) 794 { 795 struct perf_event *sub; 796 u64 tstamp = perf_event_time(event); 797 798 if (!event->cgrp_defer_enabled) 799 return; 800 801 event->cgrp_defer_enabled = 0; 802 803 event->tstamp_enabled = tstamp - event->total_time_enabled; 804 list_for_each_entry(sub, &event->sibling_list, group_entry) { 805 if (sub->state >= PERF_EVENT_STATE_INACTIVE) { 806 sub->tstamp_enabled = tstamp - sub->total_time_enabled; 807 sub->cgrp_defer_enabled = 0; 808 } 809 } 810 } 811 #else /* !CONFIG_CGROUP_PERF */ 812 813 static inline bool 814 perf_cgroup_match(struct perf_event *event) 815 { 816 return true; 817 } 818 819 static inline void perf_detach_cgroup(struct perf_event *event) 820 {} 821 822 static inline int is_cgroup_event(struct perf_event *event) 823 { 824 return 0; 825 } 826 827 static inline u64 perf_cgroup_event_cgrp_time(struct perf_event *event) 828 { 829 return 0; 830 } 831 832 static inline void update_cgrp_time_from_event(struct perf_event *event) 833 { 834 } 835 836 static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx) 837 { 838 } 839 840 static inline void perf_cgroup_sched_out(struct task_struct *task, 841 struct task_struct *next) 842 { 843 } 844 845 static inline void perf_cgroup_sched_in(struct task_struct *prev, 846 struct task_struct *task) 847 { 848 } 849 850 static inline int perf_cgroup_connect(pid_t pid, struct perf_event *event, 851 struct perf_event_attr *attr, 852 struct perf_event *group_leader) 853 { 854 return -EINVAL; 855 } 856 857 static inline void 858 perf_cgroup_set_timestamp(struct task_struct *task, 859 struct perf_event_context *ctx) 860 { 861 } 862 863 void 864 perf_cgroup_switch(struct task_struct *task, struct task_struct *next) 865 { 866 } 867 868 static inline void 869 perf_cgroup_set_shadow_time(struct perf_event *event, u64 now) 870 { 871 } 872 873 static inline u64 perf_cgroup_event_time(struct perf_event *event) 874 { 875 return 0; 876 } 877 878 static inline void 879 perf_cgroup_defer_enabled(struct perf_event *event) 880 { 881 } 882 883 static inline void 884 perf_cgroup_mark_enabled(struct perf_event *event, 885 struct perf_event_context *ctx) 886 { 887 } 888 #endif 889 890 /* 891 * set default to be dependent on timer tick just 892 * like original code 893 */ 894 #define PERF_CPU_HRTIMER (1000 / HZ) 895 /* 896 * function must be called with interrupts disbled 897 */ 898 static enum hrtimer_restart perf_mux_hrtimer_handler(struct hrtimer *hr) 899 { 900 struct perf_cpu_context *cpuctx; 901 int rotations = 0; 902 903 WARN_ON(!irqs_disabled()); 904 905 cpuctx = container_of(hr, struct perf_cpu_context, hrtimer); 906 rotations = perf_rotate_context(cpuctx); 907 908 raw_spin_lock(&cpuctx->hrtimer_lock); 909 if (rotations) 910 hrtimer_forward_now(hr, cpuctx->hrtimer_interval); 911 else 912 cpuctx->hrtimer_active = 0; 913 raw_spin_unlock(&cpuctx->hrtimer_lock); 914 915 return rotations ? HRTIMER_RESTART : HRTIMER_NORESTART; 916 } 917 918 static void __perf_mux_hrtimer_init(struct perf_cpu_context *cpuctx, int cpu) 919 { 920 struct hrtimer *timer = &cpuctx->hrtimer; 921 struct pmu *pmu = cpuctx->ctx.pmu; 922 u64 interval; 923 924 /* no multiplexing needed for SW PMU */ 925 if (pmu->task_ctx_nr == perf_sw_context) 926 return; 927 928 /* 929 * check default is sane, if not set then force to 930 * default interval (1/tick) 931 */ 932 interval = pmu->hrtimer_interval_ms; 933 if (interval < 1) 934 interval = pmu->hrtimer_interval_ms = PERF_CPU_HRTIMER; 935 936 cpuctx->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * interval); 937 938 raw_spin_lock_init(&cpuctx->hrtimer_lock); 939 hrtimer_init(timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED); 940 timer->function = perf_mux_hrtimer_handler; 941 } 942 943 static int perf_mux_hrtimer_restart(struct perf_cpu_context *cpuctx) 944 { 945 struct hrtimer *timer = &cpuctx->hrtimer; 946 struct pmu *pmu = cpuctx->ctx.pmu; 947 unsigned long flags; 948 949 /* not for SW PMU */ 950 if (pmu->task_ctx_nr == perf_sw_context) 951 return 0; 952 953 raw_spin_lock_irqsave(&cpuctx->hrtimer_lock, flags); 954 if (!cpuctx->hrtimer_active) { 955 cpuctx->hrtimer_active = 1; 956 hrtimer_forward_now(timer, cpuctx->hrtimer_interval); 957 hrtimer_start_expires(timer, HRTIMER_MODE_ABS_PINNED); 958 } 959 raw_spin_unlock_irqrestore(&cpuctx->hrtimer_lock, flags); 960 961 return 0; 962 } 963 964 void perf_pmu_disable(struct pmu *pmu) 965 { 966 int *count = this_cpu_ptr(pmu->pmu_disable_count); 967 if (!(*count)++) 968 pmu->pmu_disable(pmu); 969 } 970 971 void perf_pmu_enable(struct pmu *pmu) 972 { 973 int *count = this_cpu_ptr(pmu->pmu_disable_count); 974 if (!--(*count)) 975 pmu->pmu_enable(pmu); 976 } 977 978 static DEFINE_PER_CPU(struct list_head, active_ctx_list); 979 980 /* 981 * perf_event_ctx_activate(), perf_event_ctx_deactivate(), and 982 * perf_event_task_tick() are fully serialized because they're strictly cpu 983 * affine and perf_event_ctx{activate,deactivate} are called with IRQs 984 * disabled, while perf_event_task_tick is called from IRQ context. 985 */ 986 static void perf_event_ctx_activate(struct perf_event_context *ctx) 987 { 988 struct list_head *head = this_cpu_ptr(&active_ctx_list); 989 990 WARN_ON(!irqs_disabled()); 991 992 WARN_ON(!list_empty(&ctx->active_ctx_list)); 993 994 list_add(&ctx->active_ctx_list, head); 995 } 996 997 static void perf_event_ctx_deactivate(struct perf_event_context *ctx) 998 { 999 WARN_ON(!irqs_disabled()); 1000 1001 WARN_ON(list_empty(&ctx->active_ctx_list)); 1002 1003 list_del_init(&ctx->active_ctx_list); 1004 } 1005 1006 static void get_ctx(struct perf_event_context *ctx) 1007 { 1008 WARN_ON(!atomic_inc_not_zero(&ctx->refcount)); 1009 } 1010 1011 static void free_ctx(struct rcu_head *head) 1012 { 1013 struct perf_event_context *ctx; 1014 1015 ctx = container_of(head, struct perf_event_context, rcu_head); 1016 kfree(ctx->task_ctx_data); 1017 kfree(ctx); 1018 } 1019 1020 static void put_ctx(struct perf_event_context *ctx) 1021 { 1022 if (atomic_dec_and_test(&ctx->refcount)) { 1023 if (ctx->parent_ctx) 1024 put_ctx(ctx->parent_ctx); 1025 if (ctx->task && ctx->task != TASK_TOMBSTONE) 1026 put_task_struct(ctx->task); 1027 call_rcu(&ctx->rcu_head, free_ctx); 1028 } 1029 } 1030 1031 /* 1032 * Because of perf_event::ctx migration in sys_perf_event_open::move_group and 1033 * perf_pmu_migrate_context() we need some magic. 1034 * 1035 * Those places that change perf_event::ctx will hold both 1036 * perf_event_ctx::mutex of the 'old' and 'new' ctx value. 1037 * 1038 * Lock ordering is by mutex address. There are two other sites where 1039 * perf_event_context::mutex nests and those are: 1040 * 1041 * - perf_event_exit_task_context() [ child , 0 ] 1042 * perf_event_exit_event() 1043 * put_event() [ parent, 1 ] 1044 * 1045 * - perf_event_init_context() [ parent, 0 ] 1046 * inherit_task_group() 1047 * inherit_group() 1048 * inherit_event() 1049 * perf_event_alloc() 1050 * perf_init_event() 1051 * perf_try_init_event() [ child , 1 ] 1052 * 1053 * While it appears there is an obvious deadlock here -- the parent and child 1054 * nesting levels are inverted between the two. This is in fact safe because 1055 * life-time rules separate them. That is an exiting task cannot fork, and a 1056 * spawning task cannot (yet) exit. 1057 * 1058 * But remember that that these are parent<->child context relations, and 1059 * migration does not affect children, therefore these two orderings should not 1060 * interact. 1061 * 1062 * The change in perf_event::ctx does not affect children (as claimed above) 1063 * because the sys_perf_event_open() case will install a new event and break 1064 * the ctx parent<->child relation, and perf_pmu_migrate_context() is only 1065 * concerned with cpuctx and that doesn't have children. 1066 * 1067 * The places that change perf_event::ctx will issue: 1068 * 1069 * perf_remove_from_context(); 1070 * synchronize_rcu(); 1071 * perf_install_in_context(); 1072 * 1073 * to affect the change. The remove_from_context() + synchronize_rcu() should 1074 * quiesce the event, after which we can install it in the new location. This 1075 * means that only external vectors (perf_fops, prctl) can perturb the event 1076 * while in transit. Therefore all such accessors should also acquire 1077 * perf_event_context::mutex to serialize against this. 1078 * 1079 * However; because event->ctx can change while we're waiting to acquire 1080 * ctx->mutex we must be careful and use the below perf_event_ctx_lock() 1081 * function. 1082 * 1083 * Lock order: 1084 * task_struct::perf_event_mutex 1085 * perf_event_context::mutex 1086 * perf_event::child_mutex; 1087 * perf_event_context::lock 1088 * perf_event::mmap_mutex 1089 * mmap_sem 1090 */ 1091 static struct perf_event_context * 1092 perf_event_ctx_lock_nested(struct perf_event *event, int nesting) 1093 { 1094 struct perf_event_context *ctx; 1095 1096 again: 1097 rcu_read_lock(); 1098 ctx = ACCESS_ONCE(event->ctx); 1099 if (!atomic_inc_not_zero(&ctx->refcount)) { 1100 rcu_read_unlock(); 1101 goto again; 1102 } 1103 rcu_read_unlock(); 1104 1105 mutex_lock_nested(&ctx->mutex, nesting); 1106 if (event->ctx != ctx) { 1107 mutex_unlock(&ctx->mutex); 1108 put_ctx(ctx); 1109 goto again; 1110 } 1111 1112 return ctx; 1113 } 1114 1115 static inline struct perf_event_context * 1116 perf_event_ctx_lock(struct perf_event *event) 1117 { 1118 return perf_event_ctx_lock_nested(event, 0); 1119 } 1120 1121 static void perf_event_ctx_unlock(struct perf_event *event, 1122 struct perf_event_context *ctx) 1123 { 1124 mutex_unlock(&ctx->mutex); 1125 put_ctx(ctx); 1126 } 1127 1128 /* 1129 * This must be done under the ctx->lock, such as to serialize against 1130 * context_equiv(), therefore we cannot call put_ctx() since that might end up 1131 * calling scheduler related locks and ctx->lock nests inside those. 1132 */ 1133 static __must_check struct perf_event_context * 1134 unclone_ctx(struct perf_event_context *ctx) 1135 { 1136 struct perf_event_context *parent_ctx = ctx->parent_ctx; 1137 1138 lockdep_assert_held(&ctx->lock); 1139 1140 if (parent_ctx) 1141 ctx->parent_ctx = NULL; 1142 ctx->generation++; 1143 1144 return parent_ctx; 1145 } 1146 1147 static u32 perf_event_pid(struct perf_event *event, struct task_struct *p) 1148 { 1149 /* 1150 * only top level events have the pid namespace they were created in 1151 */ 1152 if (event->parent) 1153 event = event->parent; 1154 1155 return task_tgid_nr_ns(p, event->ns); 1156 } 1157 1158 static u32 perf_event_tid(struct perf_event *event, struct task_struct *p) 1159 { 1160 /* 1161 * only top level events have the pid namespace they were created in 1162 */ 1163 if (event->parent) 1164 event = event->parent; 1165 1166 return task_pid_nr_ns(p, event->ns); 1167 } 1168 1169 /* 1170 * If we inherit events we want to return the parent event id 1171 * to userspace. 1172 */ 1173 static u64 primary_event_id(struct perf_event *event) 1174 { 1175 u64 id = event->id; 1176 1177 if (event->parent) 1178 id = event->parent->id; 1179 1180 return id; 1181 } 1182 1183 /* 1184 * Get the perf_event_context for a task and lock it. 1185 * 1186 * This has to cope with with the fact that until it is locked, 1187 * the context could get moved to another task. 1188 */ 1189 static struct perf_event_context * 1190 perf_lock_task_context(struct task_struct *task, int ctxn, unsigned long *flags) 1191 { 1192 struct perf_event_context *ctx; 1193 1194 retry: 1195 /* 1196 * One of the few rules of preemptible RCU is that one cannot do 1197 * rcu_read_unlock() while holding a scheduler (or nested) lock when 1198 * part of the read side critical section was irqs-enabled -- see 1199 * rcu_read_unlock_special(). 1200 * 1201 * Since ctx->lock nests under rq->lock we must ensure the entire read 1202 * side critical section has interrupts disabled. 1203 */ 1204 local_irq_save(*flags); 1205 rcu_read_lock(); 1206 ctx = rcu_dereference(task->perf_event_ctxp[ctxn]); 1207 if (ctx) { 1208 /* 1209 * If this context is a clone of another, it might 1210 * get swapped for another underneath us by 1211 * perf_event_task_sched_out, though the 1212 * rcu_read_lock() protects us from any context 1213 * getting freed. Lock the context and check if it 1214 * got swapped before we could get the lock, and retry 1215 * if so. If we locked the right context, then it 1216 * can't get swapped on us any more. 1217 */ 1218 raw_spin_lock(&ctx->lock); 1219 if (ctx != rcu_dereference(task->perf_event_ctxp[ctxn])) { 1220 raw_spin_unlock(&ctx->lock); 1221 rcu_read_unlock(); 1222 local_irq_restore(*flags); 1223 goto retry; 1224 } 1225 1226 if (ctx->task == TASK_TOMBSTONE || 1227 !atomic_inc_not_zero(&ctx->refcount)) { 1228 raw_spin_unlock(&ctx->lock); 1229 ctx = NULL; 1230 } else { 1231 WARN_ON_ONCE(ctx->task != task); 1232 } 1233 } 1234 rcu_read_unlock(); 1235 if (!ctx) 1236 local_irq_restore(*flags); 1237 return ctx; 1238 } 1239 1240 /* 1241 * Get the context for a task and increment its pin_count so it 1242 * can't get swapped to another task. This also increments its 1243 * reference count so that the context can't get freed. 1244 */ 1245 static struct perf_event_context * 1246 perf_pin_task_context(struct task_struct *task, int ctxn) 1247 { 1248 struct perf_event_context *ctx; 1249 unsigned long flags; 1250 1251 ctx = perf_lock_task_context(task, ctxn, &flags); 1252 if (ctx) { 1253 ++ctx->pin_count; 1254 raw_spin_unlock_irqrestore(&ctx->lock, flags); 1255 } 1256 return ctx; 1257 } 1258 1259 static void perf_unpin_context(struct perf_event_context *ctx) 1260 { 1261 unsigned long flags; 1262 1263 raw_spin_lock_irqsave(&ctx->lock, flags); 1264 --ctx->pin_count; 1265 raw_spin_unlock_irqrestore(&ctx->lock, flags); 1266 } 1267 1268 /* 1269 * Update the record of the current time in a context. 1270 */ 1271 static void update_context_time(struct perf_event_context *ctx) 1272 { 1273 u64 now = perf_clock(); 1274 1275 ctx->time += now - ctx->timestamp; 1276 ctx->timestamp = now; 1277 } 1278 1279 static u64 perf_event_time(struct perf_event *event) 1280 { 1281 struct perf_event_context *ctx = event->ctx; 1282 1283 if (is_cgroup_event(event)) 1284 return perf_cgroup_event_time(event); 1285 1286 return ctx ? ctx->time : 0; 1287 } 1288 1289 /* 1290 * Update the total_time_enabled and total_time_running fields for a event. 1291 * The caller of this function needs to hold the ctx->lock. 1292 */ 1293 static void update_event_times(struct perf_event *event) 1294 { 1295 struct perf_event_context *ctx = event->ctx; 1296 u64 run_end; 1297 1298 if (event->state < PERF_EVENT_STATE_INACTIVE || 1299 event->group_leader->state < PERF_EVENT_STATE_INACTIVE) 1300 return; 1301 /* 1302 * in cgroup mode, time_enabled represents 1303 * the time the event was enabled AND active 1304 * tasks were in the monitored cgroup. This is 1305 * independent of the activity of the context as 1306 * there may be a mix of cgroup and non-cgroup events. 1307 * 1308 * That is why we treat cgroup events differently 1309 * here. 1310 */ 1311 if (is_cgroup_event(event)) 1312 run_end = perf_cgroup_event_time(event); 1313 else if (ctx->is_active) 1314 run_end = ctx->time; 1315 else 1316 run_end = event->tstamp_stopped; 1317 1318 event->total_time_enabled = run_end - event->tstamp_enabled; 1319 1320 if (event->state == PERF_EVENT_STATE_INACTIVE) 1321 run_end = event->tstamp_stopped; 1322 else 1323 run_end = perf_event_time(event); 1324 1325 event->total_time_running = run_end - event->tstamp_running; 1326 1327 } 1328 1329 /* 1330 * Update total_time_enabled and total_time_running for all events in a group. 1331 */ 1332 static void update_group_times(struct perf_event *leader) 1333 { 1334 struct perf_event *event; 1335 1336 update_event_times(leader); 1337 list_for_each_entry(event, &leader->sibling_list, group_entry) 1338 update_event_times(event); 1339 } 1340 1341 static struct list_head * 1342 ctx_group_list(struct perf_event *event, struct perf_event_context *ctx) 1343 { 1344 if (event->attr.pinned) 1345 return &ctx->pinned_groups; 1346 else 1347 return &ctx->flexible_groups; 1348 } 1349 1350 /* 1351 * Add a event from the lists for its context. 1352 * Must be called with ctx->mutex and ctx->lock held. 1353 */ 1354 static void 1355 list_add_event(struct perf_event *event, struct perf_event_context *ctx) 1356 { 1357 lockdep_assert_held(&ctx->lock); 1358 1359 WARN_ON_ONCE(event->attach_state & PERF_ATTACH_CONTEXT); 1360 event->attach_state |= PERF_ATTACH_CONTEXT; 1361 1362 /* 1363 * If we're a stand alone event or group leader, we go to the context 1364 * list, group events are kept attached to the group so that 1365 * perf_group_detach can, at all times, locate all siblings. 1366 */ 1367 if (event->group_leader == event) { 1368 struct list_head *list; 1369 1370 if (is_software_event(event)) 1371 event->group_flags |= PERF_GROUP_SOFTWARE; 1372 1373 list = ctx_group_list(event, ctx); 1374 list_add_tail(&event->group_entry, list); 1375 } 1376 1377 if (is_cgroup_event(event)) 1378 ctx->nr_cgroups++; 1379 1380 list_add_rcu(&event->event_entry, &ctx->event_list); 1381 ctx->nr_events++; 1382 if (event->attr.inherit_stat) 1383 ctx->nr_stat++; 1384 1385 ctx->generation++; 1386 } 1387 1388 /* 1389 * Initialize event state based on the perf_event_attr::disabled. 1390 */ 1391 static inline void perf_event__state_init(struct perf_event *event) 1392 { 1393 event->state = event->attr.disabled ? PERF_EVENT_STATE_OFF : 1394 PERF_EVENT_STATE_INACTIVE; 1395 } 1396 1397 static void __perf_event_read_size(struct perf_event *event, int nr_siblings) 1398 { 1399 int entry = sizeof(u64); /* value */ 1400 int size = 0; 1401 int nr = 1; 1402 1403 if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) 1404 size += sizeof(u64); 1405 1406 if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) 1407 size += sizeof(u64); 1408 1409 if (event->attr.read_format & PERF_FORMAT_ID) 1410 entry += sizeof(u64); 1411 1412 if (event->attr.read_format & PERF_FORMAT_GROUP) { 1413 nr += nr_siblings; 1414 size += sizeof(u64); 1415 } 1416 1417 size += entry * nr; 1418 event->read_size = size; 1419 } 1420 1421 static void __perf_event_header_size(struct perf_event *event, u64 sample_type) 1422 { 1423 struct perf_sample_data *data; 1424 u16 size = 0; 1425 1426 if (sample_type & PERF_SAMPLE_IP) 1427 size += sizeof(data->ip); 1428 1429 if (sample_type & PERF_SAMPLE_ADDR) 1430 size += sizeof(data->addr); 1431 1432 if (sample_type & PERF_SAMPLE_PERIOD) 1433 size += sizeof(data->period); 1434 1435 if (sample_type & PERF_SAMPLE_WEIGHT) 1436 size += sizeof(data->weight); 1437 1438 if (sample_type & PERF_SAMPLE_READ) 1439 size += event->read_size; 1440 1441 if (sample_type & PERF_SAMPLE_DATA_SRC) 1442 size += sizeof(data->data_src.val); 1443 1444 if (sample_type & PERF_SAMPLE_TRANSACTION) 1445 size += sizeof(data->txn); 1446 1447 event->header_size = size; 1448 } 1449 1450 /* 1451 * Called at perf_event creation and when events are attached/detached from a 1452 * group. 1453 */ 1454 static void perf_event__header_size(struct perf_event *event) 1455 { 1456 __perf_event_read_size(event, 1457 event->group_leader->nr_siblings); 1458 __perf_event_header_size(event, event->attr.sample_type); 1459 } 1460 1461 static void perf_event__id_header_size(struct perf_event *event) 1462 { 1463 struct perf_sample_data *data; 1464 u64 sample_type = event->attr.sample_type; 1465 u16 size = 0; 1466 1467 if (sample_type & PERF_SAMPLE_TID) 1468 size += sizeof(data->tid_entry); 1469 1470 if (sample_type & PERF_SAMPLE_TIME) 1471 size += sizeof(data->time); 1472 1473 if (sample_type & PERF_SAMPLE_IDENTIFIER) 1474 size += sizeof(data->id); 1475 1476 if (sample_type & PERF_SAMPLE_ID) 1477 size += sizeof(data->id); 1478 1479 if (sample_type & PERF_SAMPLE_STREAM_ID) 1480 size += sizeof(data->stream_id); 1481 1482 if (sample_type & PERF_SAMPLE_CPU) 1483 size += sizeof(data->cpu_entry); 1484 1485 event->id_header_size = size; 1486 } 1487 1488 static bool perf_event_validate_size(struct perf_event *event) 1489 { 1490 /* 1491 * The values computed here will be over-written when we actually 1492 * attach the event. 1493 */ 1494 __perf_event_read_size(event, event->group_leader->nr_siblings + 1); 1495 __perf_event_header_size(event, event->attr.sample_type & ~PERF_SAMPLE_READ); 1496 perf_event__id_header_size(event); 1497 1498 /* 1499 * Sum the lot; should not exceed the 64k limit we have on records. 1500 * Conservative limit to allow for callchains and other variable fields. 1501 */ 1502 if (event->read_size + event->header_size + 1503 event->id_header_size + sizeof(struct perf_event_header) >= 16*1024) 1504 return false; 1505 1506 return true; 1507 } 1508 1509 static void perf_group_attach(struct perf_event *event) 1510 { 1511 struct perf_event *group_leader = event->group_leader, *pos; 1512 1513 /* 1514 * We can have double attach due to group movement in perf_event_open. 1515 */ 1516 if (event->attach_state & PERF_ATTACH_GROUP) 1517 return; 1518 1519 event->attach_state |= PERF_ATTACH_GROUP; 1520 1521 if (group_leader == event) 1522 return; 1523 1524 WARN_ON_ONCE(group_leader->ctx != event->ctx); 1525 1526 if (group_leader->group_flags & PERF_GROUP_SOFTWARE && 1527 !is_software_event(event)) 1528 group_leader->group_flags &= ~PERF_GROUP_SOFTWARE; 1529 1530 list_add_tail(&event->group_entry, &group_leader->sibling_list); 1531 group_leader->nr_siblings++; 1532 1533 perf_event__header_size(group_leader); 1534 1535 list_for_each_entry(pos, &group_leader->sibling_list, group_entry) 1536 perf_event__header_size(pos); 1537 } 1538 1539 /* 1540 * Remove a event from the lists for its context. 1541 * Must be called with ctx->mutex and ctx->lock held. 1542 */ 1543 static void 1544 list_del_event(struct perf_event *event, struct perf_event_context *ctx) 1545 { 1546 struct perf_cpu_context *cpuctx; 1547 1548 WARN_ON_ONCE(event->ctx != ctx); 1549 lockdep_assert_held(&ctx->lock); 1550 1551 /* 1552 * We can have double detach due to exit/hot-unplug + close. 1553 */ 1554 if (!(event->attach_state & PERF_ATTACH_CONTEXT)) 1555 return; 1556 1557 event->attach_state &= ~PERF_ATTACH_CONTEXT; 1558 1559 if (is_cgroup_event(event)) { 1560 ctx->nr_cgroups--; 1561 /* 1562 * Because cgroup events are always per-cpu events, this will 1563 * always be called from the right CPU. 1564 */ 1565 cpuctx = __get_cpu_context(ctx); 1566 /* 1567 * If there are no more cgroup events then clear cgrp to avoid 1568 * stale pointer in update_cgrp_time_from_cpuctx(). 1569 */ 1570 if (!ctx->nr_cgroups) 1571 cpuctx->cgrp = NULL; 1572 } 1573 1574 ctx->nr_events--; 1575 if (event->attr.inherit_stat) 1576 ctx->nr_stat--; 1577 1578 list_del_rcu(&event->event_entry); 1579 1580 if (event->group_leader == event) 1581 list_del_init(&event->group_entry); 1582 1583 update_group_times(event); 1584 1585 /* 1586 * If event was in error state, then keep it 1587 * that way, otherwise bogus counts will be 1588 * returned on read(). The only way to get out 1589 * of error state is by explicit re-enabling 1590 * of the event 1591 */ 1592 if (event->state > PERF_EVENT_STATE_OFF) 1593 event->state = PERF_EVENT_STATE_OFF; 1594 1595 ctx->generation++; 1596 } 1597 1598 static void perf_group_detach(struct perf_event *event) 1599 { 1600 struct perf_event *sibling, *tmp; 1601 struct list_head *list = NULL; 1602 1603 /* 1604 * We can have double detach due to exit/hot-unplug + close. 1605 */ 1606 if (!(event->attach_state & PERF_ATTACH_GROUP)) 1607 return; 1608 1609 event->attach_state &= ~PERF_ATTACH_GROUP; 1610 1611 /* 1612 * If this is a sibling, remove it from its group. 1613 */ 1614 if (event->group_leader != event) { 1615 list_del_init(&event->group_entry); 1616 event->group_leader->nr_siblings--; 1617 goto out; 1618 } 1619 1620 if (!list_empty(&event->group_entry)) 1621 list = &event->group_entry; 1622 1623 /* 1624 * If this was a group event with sibling events then 1625 * upgrade the siblings to singleton events by adding them 1626 * to whatever list we are on. 1627 */ 1628 list_for_each_entry_safe(sibling, tmp, &event->sibling_list, group_entry) { 1629 if (list) 1630 list_move_tail(&sibling->group_entry, list); 1631 sibling->group_leader = sibling; 1632 1633 /* Inherit group flags from the previous leader */ 1634 sibling->group_flags = event->group_flags; 1635 1636 WARN_ON_ONCE(sibling->ctx != event->ctx); 1637 } 1638 1639 out: 1640 perf_event__header_size(event->group_leader); 1641 1642 list_for_each_entry(tmp, &event->group_leader->sibling_list, group_entry) 1643 perf_event__header_size(tmp); 1644 } 1645 1646 static bool is_orphaned_event(struct perf_event *event) 1647 { 1648 return event->state == PERF_EVENT_STATE_EXIT; 1649 } 1650 1651 static inline int pmu_filter_match(struct perf_event *event) 1652 { 1653 struct pmu *pmu = event->pmu; 1654 return pmu->filter_match ? pmu->filter_match(event) : 1; 1655 } 1656 1657 static inline int 1658 event_filter_match(struct perf_event *event) 1659 { 1660 return (event->cpu == -1 || event->cpu == smp_processor_id()) 1661 && perf_cgroup_match(event) && pmu_filter_match(event); 1662 } 1663 1664 static void 1665 event_sched_out(struct perf_event *event, 1666 struct perf_cpu_context *cpuctx, 1667 struct perf_event_context *ctx) 1668 { 1669 u64 tstamp = perf_event_time(event); 1670 u64 delta; 1671 1672 WARN_ON_ONCE(event->ctx != ctx); 1673 lockdep_assert_held(&ctx->lock); 1674 1675 /* 1676 * An event which could not be activated because of 1677 * filter mismatch still needs to have its timings 1678 * maintained, otherwise bogus information is return 1679 * via read() for time_enabled, time_running: 1680 */ 1681 if (event->state == PERF_EVENT_STATE_INACTIVE 1682 && !event_filter_match(event)) { 1683 delta = tstamp - event->tstamp_stopped; 1684 event->tstamp_running += delta; 1685 event->tstamp_stopped = tstamp; 1686 } 1687 1688 if (event->state != PERF_EVENT_STATE_ACTIVE) 1689 return; 1690 1691 perf_pmu_disable(event->pmu); 1692 1693 event->state = PERF_EVENT_STATE_INACTIVE; 1694 if (event->pending_disable) { 1695 event->pending_disable = 0; 1696 event->state = PERF_EVENT_STATE_OFF; 1697 } 1698 event->tstamp_stopped = tstamp; 1699 event->pmu->del(event, 0); 1700 event->oncpu = -1; 1701 1702 if (!is_software_event(event)) 1703 cpuctx->active_oncpu--; 1704 if (!--ctx->nr_active) 1705 perf_event_ctx_deactivate(ctx); 1706 if (event->attr.freq && event->attr.sample_freq) 1707 ctx->nr_freq--; 1708 if (event->attr.exclusive || !cpuctx->active_oncpu) 1709 cpuctx->exclusive = 0; 1710 1711 perf_pmu_enable(event->pmu); 1712 } 1713 1714 static void 1715 group_sched_out(struct perf_event *group_event, 1716 struct perf_cpu_context *cpuctx, 1717 struct perf_event_context *ctx) 1718 { 1719 struct perf_event *event; 1720 int state = group_event->state; 1721 1722 event_sched_out(group_event, cpuctx, ctx); 1723 1724 /* 1725 * Schedule out siblings (if any): 1726 */ 1727 list_for_each_entry(event, &group_event->sibling_list, group_entry) 1728 event_sched_out(event, cpuctx, ctx); 1729 1730 if (state == PERF_EVENT_STATE_ACTIVE && group_event->attr.exclusive) 1731 cpuctx->exclusive = 0; 1732 } 1733 1734 #define DETACH_GROUP 0x01UL 1735 #define DETACH_STATE 0x02UL 1736 1737 /* 1738 * Cross CPU call to remove a performance event 1739 * 1740 * We disable the event on the hardware level first. After that we 1741 * remove it from the context list. 1742 */ 1743 static void 1744 __perf_remove_from_context(struct perf_event *event, 1745 struct perf_cpu_context *cpuctx, 1746 struct perf_event_context *ctx, 1747 void *info) 1748 { 1749 unsigned long flags = (unsigned long)info; 1750 1751 event_sched_out(event, cpuctx, ctx); 1752 if (flags & DETACH_GROUP) 1753 perf_group_detach(event); 1754 list_del_event(event, ctx); 1755 if (flags & DETACH_STATE) 1756 event->state = PERF_EVENT_STATE_EXIT; 1757 1758 if (!ctx->nr_events && ctx->is_active) { 1759 ctx->is_active = 0; 1760 if (ctx->task) { 1761 WARN_ON_ONCE(cpuctx->task_ctx != ctx); 1762 cpuctx->task_ctx = NULL; 1763 } 1764 } 1765 } 1766 1767 /* 1768 * Remove the event from a task's (or a CPU's) list of events. 1769 * 1770 * If event->ctx is a cloned context, callers must make sure that 1771 * every task struct that event->ctx->task could possibly point to 1772 * remains valid. This is OK when called from perf_release since 1773 * that only calls us on the top-level context, which can't be a clone. 1774 * When called from perf_event_exit_task, it's OK because the 1775 * context has been detached from its task. 1776 */ 1777 static void perf_remove_from_context(struct perf_event *event, unsigned long flags) 1778 { 1779 lockdep_assert_held(&event->ctx->mutex); 1780 1781 event_function_call(event, __perf_remove_from_context, (void *)flags); 1782 } 1783 1784 /* 1785 * Cross CPU call to disable a performance event 1786 */ 1787 static void __perf_event_disable(struct perf_event *event, 1788 struct perf_cpu_context *cpuctx, 1789 struct perf_event_context *ctx, 1790 void *info) 1791 { 1792 if (event->state < PERF_EVENT_STATE_INACTIVE) 1793 return; 1794 1795 update_context_time(ctx); 1796 update_cgrp_time_from_event(event); 1797 update_group_times(event); 1798 if (event == event->group_leader) 1799 group_sched_out(event, cpuctx, ctx); 1800 else 1801 event_sched_out(event, cpuctx, ctx); 1802 event->state = PERF_EVENT_STATE_OFF; 1803 } 1804 1805 /* 1806 * Disable a event. 1807 * 1808 * If event->ctx is a cloned context, callers must make sure that 1809 * every task struct that event->ctx->task could possibly point to 1810 * remains valid. This condition is satisifed when called through 1811 * perf_event_for_each_child or perf_event_for_each because they 1812 * hold the top-level event's child_mutex, so any descendant that 1813 * goes to exit will block in perf_event_exit_event(). 1814 * 1815 * When called from perf_pending_event it's OK because event->ctx 1816 * is the current context on this CPU and preemption is disabled, 1817 * hence we can't get into perf_event_task_sched_out for this context. 1818 */ 1819 static void _perf_event_disable(struct perf_event *event) 1820 { 1821 struct perf_event_context *ctx = event->ctx; 1822 1823 raw_spin_lock_irq(&ctx->lock); 1824 if (event->state <= PERF_EVENT_STATE_OFF) { 1825 raw_spin_unlock_irq(&ctx->lock); 1826 return; 1827 } 1828 raw_spin_unlock_irq(&ctx->lock); 1829 1830 event_function_call(event, __perf_event_disable, NULL); 1831 } 1832 1833 void perf_event_disable_local(struct perf_event *event) 1834 { 1835 event_function_local(event, __perf_event_disable, NULL); 1836 } 1837 1838 /* 1839 * Strictly speaking kernel users cannot create groups and therefore this 1840 * interface does not need the perf_event_ctx_lock() magic. 1841 */ 1842 void perf_event_disable(struct perf_event *event) 1843 { 1844 struct perf_event_context *ctx; 1845 1846 ctx = perf_event_ctx_lock(event); 1847 _perf_event_disable(event); 1848 perf_event_ctx_unlock(event, ctx); 1849 } 1850 EXPORT_SYMBOL_GPL(perf_event_disable); 1851 1852 static void perf_set_shadow_time(struct perf_event *event, 1853 struct perf_event_context *ctx, 1854 u64 tstamp) 1855 { 1856 /* 1857 * use the correct time source for the time snapshot 1858 * 1859 * We could get by without this by leveraging the 1860 * fact that to get to this function, the caller 1861 * has most likely already called update_context_time() 1862 * and update_cgrp_time_xx() and thus both timestamp 1863 * are identical (or very close). Given that tstamp is, 1864 * already adjusted for cgroup, we could say that: 1865 * tstamp - ctx->timestamp 1866 * is equivalent to 1867 * tstamp - cgrp->timestamp. 1868 * 1869 * Then, in perf_output_read(), the calculation would 1870 * work with no changes because: 1871 * - event is guaranteed scheduled in 1872 * - no scheduled out in between 1873 * - thus the timestamp would be the same 1874 * 1875 * But this is a bit hairy. 1876 * 1877 * So instead, we have an explicit cgroup call to remain 1878 * within the time time source all along. We believe it 1879 * is cleaner and simpler to understand. 1880 */ 1881 if (is_cgroup_event(event)) 1882 perf_cgroup_set_shadow_time(event, tstamp); 1883 else 1884 event->shadow_ctx_time = tstamp - ctx->timestamp; 1885 } 1886 1887 #define MAX_INTERRUPTS (~0ULL) 1888 1889 static void perf_log_throttle(struct perf_event *event, int enable); 1890 static void perf_log_itrace_start(struct perf_event *event); 1891 1892 static int 1893 event_sched_in(struct perf_event *event, 1894 struct perf_cpu_context *cpuctx, 1895 struct perf_event_context *ctx) 1896 { 1897 u64 tstamp = perf_event_time(event); 1898 int ret = 0; 1899 1900 lockdep_assert_held(&ctx->lock); 1901 1902 if (event->state <= PERF_EVENT_STATE_OFF) 1903 return 0; 1904 1905 event->state = PERF_EVENT_STATE_ACTIVE; 1906 event->oncpu = smp_processor_id(); 1907 1908 /* 1909 * Unthrottle events, since we scheduled we might have missed several 1910 * ticks already, also for a heavily scheduling task there is little 1911 * guarantee it'll get a tick in a timely manner. 1912 */ 1913 if (unlikely(event->hw.interrupts == MAX_INTERRUPTS)) { 1914 perf_log_throttle(event, 1); 1915 event->hw.interrupts = 0; 1916 } 1917 1918 /* 1919 * The new state must be visible before we turn it on in the hardware: 1920 */ 1921 smp_wmb(); 1922 1923 perf_pmu_disable(event->pmu); 1924 1925 perf_set_shadow_time(event, ctx, tstamp); 1926 1927 perf_log_itrace_start(event); 1928 1929 if (event->pmu->add(event, PERF_EF_START)) { 1930 event->state = PERF_EVENT_STATE_INACTIVE; 1931 event->oncpu = -1; 1932 ret = -EAGAIN; 1933 goto out; 1934 } 1935 1936 event->tstamp_running += tstamp - event->tstamp_stopped; 1937 1938 if (!is_software_event(event)) 1939 cpuctx->active_oncpu++; 1940 if (!ctx->nr_active++) 1941 perf_event_ctx_activate(ctx); 1942 if (event->attr.freq && event->attr.sample_freq) 1943 ctx->nr_freq++; 1944 1945 if (event->attr.exclusive) 1946 cpuctx->exclusive = 1; 1947 1948 out: 1949 perf_pmu_enable(event->pmu); 1950 1951 return ret; 1952 } 1953 1954 static int 1955 group_sched_in(struct perf_event *group_event, 1956 struct perf_cpu_context *cpuctx, 1957 struct perf_event_context *ctx) 1958 { 1959 struct perf_event *event, *partial_group = NULL; 1960 struct pmu *pmu = ctx->pmu; 1961 u64 now = ctx->time; 1962 bool simulate = false; 1963 1964 if (group_event->state == PERF_EVENT_STATE_OFF) 1965 return 0; 1966 1967 pmu->start_txn(pmu, PERF_PMU_TXN_ADD); 1968 1969 if (event_sched_in(group_event, cpuctx, ctx)) { 1970 pmu->cancel_txn(pmu); 1971 perf_mux_hrtimer_restart(cpuctx); 1972 return -EAGAIN; 1973 } 1974 1975 /* 1976 * Schedule in siblings as one group (if any): 1977 */ 1978 list_for_each_entry(event, &group_event->sibling_list, group_entry) { 1979 if (event_sched_in(event, cpuctx, ctx)) { 1980 partial_group = event; 1981 goto group_error; 1982 } 1983 } 1984 1985 if (!pmu->commit_txn(pmu)) 1986 return 0; 1987 1988 group_error: 1989 /* 1990 * Groups can be scheduled in as one unit only, so undo any 1991 * partial group before returning: 1992 * The events up to the failed event are scheduled out normally, 1993 * tstamp_stopped will be updated. 1994 * 1995 * The failed events and the remaining siblings need to have 1996 * their timings updated as if they had gone thru event_sched_in() 1997 * and event_sched_out(). This is required to get consistent timings 1998 * across the group. This also takes care of the case where the group 1999 * could never be scheduled by ensuring tstamp_stopped is set to mark 2000 * the time the event was actually stopped, such that time delta 2001 * calculation in update_event_times() is correct. 2002 */ 2003 list_for_each_entry(event, &group_event->sibling_list, group_entry) { 2004 if (event == partial_group) 2005 simulate = true; 2006 2007 if (simulate) { 2008 event->tstamp_running += now - event->tstamp_stopped; 2009 event->tstamp_stopped = now; 2010 } else { 2011 event_sched_out(event, cpuctx, ctx); 2012 } 2013 } 2014 event_sched_out(group_event, cpuctx, ctx); 2015 2016 pmu->cancel_txn(pmu); 2017 2018 perf_mux_hrtimer_restart(cpuctx); 2019 2020 return -EAGAIN; 2021 } 2022 2023 /* 2024 * Work out whether we can put this event group on the CPU now. 2025 */ 2026 static int group_can_go_on(struct perf_event *event, 2027 struct perf_cpu_context *cpuctx, 2028 int can_add_hw) 2029 { 2030 /* 2031 * Groups consisting entirely of software events can always go on. 2032 */ 2033 if (event->group_flags & PERF_GROUP_SOFTWARE) 2034 return 1; 2035 /* 2036 * If an exclusive group is already on, no other hardware 2037 * events can go on. 2038 */ 2039 if (cpuctx->exclusive) 2040 return 0; 2041 /* 2042 * If this group is exclusive and there are already 2043 * events on the CPU, it can't go on. 2044 */ 2045 if (event->attr.exclusive && cpuctx->active_oncpu) 2046 return 0; 2047 /* 2048 * Otherwise, try to add it if all previous groups were able 2049 * to go on. 2050 */ 2051 return can_add_hw; 2052 } 2053 2054 static void add_event_to_ctx(struct perf_event *event, 2055 struct perf_event_context *ctx) 2056 { 2057 u64 tstamp = perf_event_time(event); 2058 2059 list_add_event(event, ctx); 2060 perf_group_attach(event); 2061 event->tstamp_enabled = tstamp; 2062 event->tstamp_running = tstamp; 2063 event->tstamp_stopped = tstamp; 2064 } 2065 2066 static void task_ctx_sched_out(struct perf_cpu_context *cpuctx, 2067 struct perf_event_context *ctx); 2068 static void 2069 ctx_sched_in(struct perf_event_context *ctx, 2070 struct perf_cpu_context *cpuctx, 2071 enum event_type_t event_type, 2072 struct task_struct *task); 2073 2074 static void perf_event_sched_in(struct perf_cpu_context *cpuctx, 2075 struct perf_event_context *ctx, 2076 struct task_struct *task) 2077 { 2078 cpu_ctx_sched_in(cpuctx, EVENT_PINNED, task); 2079 if (ctx) 2080 ctx_sched_in(ctx, cpuctx, EVENT_PINNED, task); 2081 cpu_ctx_sched_in(cpuctx, EVENT_FLEXIBLE, task); 2082 if (ctx) 2083 ctx_sched_in(ctx, cpuctx, EVENT_FLEXIBLE, task); 2084 } 2085 2086 static void ctx_resched(struct perf_cpu_context *cpuctx, 2087 struct perf_event_context *task_ctx) 2088 { 2089 perf_pmu_disable(cpuctx->ctx.pmu); 2090 if (task_ctx) 2091 task_ctx_sched_out(cpuctx, task_ctx); 2092 cpu_ctx_sched_out(cpuctx, EVENT_ALL); 2093 perf_event_sched_in(cpuctx, task_ctx, current); 2094 perf_pmu_enable(cpuctx->ctx.pmu); 2095 } 2096 2097 /* 2098 * Cross CPU call to install and enable a performance event 2099 * 2100 * Must be called with ctx->mutex held 2101 */ 2102 static int __perf_install_in_context(void *info) 2103 { 2104 struct perf_event_context *ctx = info; 2105 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx); 2106 struct perf_event_context *task_ctx = cpuctx->task_ctx; 2107 2108 raw_spin_lock(&cpuctx->ctx.lock); 2109 if (ctx->task) { 2110 raw_spin_lock(&ctx->lock); 2111 /* 2112 * If we hit the 'wrong' task, we've since scheduled and 2113 * everything should be sorted, nothing to do! 2114 */ 2115 task_ctx = ctx; 2116 if (ctx->task != current) 2117 goto unlock; 2118 2119 /* 2120 * If task_ctx is set, it had better be to us. 2121 */ 2122 WARN_ON_ONCE(cpuctx->task_ctx != ctx && cpuctx->task_ctx); 2123 } else if (task_ctx) { 2124 raw_spin_lock(&task_ctx->lock); 2125 } 2126 2127 ctx_resched(cpuctx, task_ctx); 2128 unlock: 2129 perf_ctx_unlock(cpuctx, task_ctx); 2130 2131 return 0; 2132 } 2133 2134 /* 2135 * Attach a performance event to a context 2136 */ 2137 static void 2138 perf_install_in_context(struct perf_event_context *ctx, 2139 struct perf_event *event, 2140 int cpu) 2141 { 2142 struct task_struct *task = NULL; 2143 2144 lockdep_assert_held(&ctx->mutex); 2145 2146 event->ctx = ctx; 2147 if (event->cpu != -1) 2148 event->cpu = cpu; 2149 2150 /* 2151 * Installing events is tricky because we cannot rely on ctx->is_active 2152 * to be set in case this is the nr_events 0 -> 1 transition. 2153 * 2154 * So what we do is we add the event to the list here, which will allow 2155 * a future context switch to DTRT and then send a racy IPI. If the IPI 2156 * fails to hit the right task, this means a context switch must have 2157 * happened and that will have taken care of business. 2158 */ 2159 raw_spin_lock_irq(&ctx->lock); 2160 task = ctx->task; 2161 /* 2162 * Worse, we cannot even rely on the ctx actually existing anymore. If 2163 * between find_get_context() and perf_install_in_context() the task 2164 * went through perf_event_exit_task() its dead and we should not be 2165 * adding new events. 2166 */ 2167 if (task == TASK_TOMBSTONE) { 2168 raw_spin_unlock_irq(&ctx->lock); 2169 return; 2170 } 2171 update_context_time(ctx); 2172 /* 2173 * Update cgrp time only if current cgrp matches event->cgrp. 2174 * Must be done before calling add_event_to_ctx(). 2175 */ 2176 update_cgrp_time_from_event(event); 2177 add_event_to_ctx(event, ctx); 2178 raw_spin_unlock_irq(&ctx->lock); 2179 2180 if (task) 2181 task_function_call(task, __perf_install_in_context, ctx); 2182 else 2183 cpu_function_call(cpu, __perf_install_in_context, ctx); 2184 } 2185 2186 /* 2187 * Put a event into inactive state and update time fields. 2188 * Enabling the leader of a group effectively enables all 2189 * the group members that aren't explicitly disabled, so we 2190 * have to update their ->tstamp_enabled also. 2191 * Note: this works for group members as well as group leaders 2192 * since the non-leader members' sibling_lists will be empty. 2193 */ 2194 static void __perf_event_mark_enabled(struct perf_event *event) 2195 { 2196 struct perf_event *sub; 2197 u64 tstamp = perf_event_time(event); 2198 2199 event->state = PERF_EVENT_STATE_INACTIVE; 2200 event->tstamp_enabled = tstamp - event->total_time_enabled; 2201 list_for_each_entry(sub, &event->sibling_list, group_entry) { 2202 if (sub->state >= PERF_EVENT_STATE_INACTIVE) 2203 sub->tstamp_enabled = tstamp - sub->total_time_enabled; 2204 } 2205 } 2206 2207 /* 2208 * Cross CPU call to enable a performance event 2209 */ 2210 static void __perf_event_enable(struct perf_event *event, 2211 struct perf_cpu_context *cpuctx, 2212 struct perf_event_context *ctx, 2213 void *info) 2214 { 2215 struct perf_event *leader = event->group_leader; 2216 struct perf_event_context *task_ctx; 2217 2218 if (event->state >= PERF_EVENT_STATE_INACTIVE || 2219 event->state <= PERF_EVENT_STATE_ERROR) 2220 return; 2221 2222 update_context_time(ctx); 2223 __perf_event_mark_enabled(event); 2224 2225 if (!ctx->is_active) 2226 return; 2227 2228 if (!event_filter_match(event)) { 2229 if (is_cgroup_event(event)) { 2230 perf_cgroup_set_timestamp(current, ctx); // XXX ? 2231 perf_cgroup_defer_enabled(event); 2232 } 2233 return; 2234 } 2235 2236 /* 2237 * If the event is in a group and isn't the group leader, 2238 * then don't put it on unless the group is on. 2239 */ 2240 if (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE) 2241 return; 2242 2243 task_ctx = cpuctx->task_ctx; 2244 if (ctx->task) 2245 WARN_ON_ONCE(task_ctx != ctx); 2246 2247 ctx_resched(cpuctx, task_ctx); 2248 } 2249 2250 /* 2251 * Enable a event. 2252 * 2253 * If event->ctx is a cloned context, callers must make sure that 2254 * every task struct that event->ctx->task could possibly point to 2255 * remains valid. This condition is satisfied when called through 2256 * perf_event_for_each_child or perf_event_for_each as described 2257 * for perf_event_disable. 2258 */ 2259 static void _perf_event_enable(struct perf_event *event) 2260 { 2261 struct perf_event_context *ctx = event->ctx; 2262 2263 raw_spin_lock_irq(&ctx->lock); 2264 if (event->state >= PERF_EVENT_STATE_INACTIVE || 2265 event->state < PERF_EVENT_STATE_ERROR) { 2266 raw_spin_unlock_irq(&ctx->lock); 2267 return; 2268 } 2269 2270 /* 2271 * If the event is in error state, clear that first. 2272 * 2273 * That way, if we see the event in error state below, we know that it 2274 * has gone back into error state, as distinct from the task having 2275 * been scheduled away before the cross-call arrived. 2276 */ 2277 if (event->state == PERF_EVENT_STATE_ERROR) 2278 event->state = PERF_EVENT_STATE_OFF; 2279 raw_spin_unlock_irq(&ctx->lock); 2280 2281 event_function_call(event, __perf_event_enable, NULL); 2282 } 2283 2284 /* 2285 * See perf_event_disable(); 2286 */ 2287 void perf_event_enable(struct perf_event *event) 2288 { 2289 struct perf_event_context *ctx; 2290 2291 ctx = perf_event_ctx_lock(event); 2292 _perf_event_enable(event); 2293 perf_event_ctx_unlock(event, ctx); 2294 } 2295 EXPORT_SYMBOL_GPL(perf_event_enable); 2296 2297 static int _perf_event_refresh(struct perf_event *event, int refresh) 2298 { 2299 /* 2300 * not supported on inherited events 2301 */ 2302 if (event->attr.inherit || !is_sampling_event(event)) 2303 return -EINVAL; 2304 2305 atomic_add(refresh, &event->event_limit); 2306 _perf_event_enable(event); 2307 2308 return 0; 2309 } 2310 2311 /* 2312 * See perf_event_disable() 2313 */ 2314 int perf_event_refresh(struct perf_event *event, int refresh) 2315 { 2316 struct perf_event_context *ctx; 2317 int ret; 2318 2319 ctx = perf_event_ctx_lock(event); 2320 ret = _perf_event_refresh(event, refresh); 2321 perf_event_ctx_unlock(event, ctx); 2322 2323 return ret; 2324 } 2325 EXPORT_SYMBOL_GPL(perf_event_refresh); 2326 2327 static void ctx_sched_out(struct perf_event_context *ctx, 2328 struct perf_cpu_context *cpuctx, 2329 enum event_type_t event_type) 2330 { 2331 int is_active = ctx->is_active; 2332 struct perf_event *event; 2333 2334 lockdep_assert_held(&ctx->lock); 2335 2336 if (likely(!ctx->nr_events)) { 2337 /* 2338 * See __perf_remove_from_context(). 2339 */ 2340 WARN_ON_ONCE(ctx->is_active); 2341 if (ctx->task) 2342 WARN_ON_ONCE(cpuctx->task_ctx); 2343 return; 2344 } 2345 2346 ctx->is_active &= ~event_type; 2347 if (ctx->task) { 2348 WARN_ON_ONCE(cpuctx->task_ctx != ctx); 2349 if (!ctx->is_active) 2350 cpuctx->task_ctx = NULL; 2351 } 2352 2353 update_context_time(ctx); 2354 update_cgrp_time_from_cpuctx(cpuctx); 2355 if (!ctx->nr_active) 2356 return; 2357 2358 perf_pmu_disable(ctx->pmu); 2359 if ((is_active & EVENT_PINNED) && (event_type & EVENT_PINNED)) { 2360 list_for_each_entry(event, &ctx->pinned_groups, group_entry) 2361 group_sched_out(event, cpuctx, ctx); 2362 } 2363 2364 if ((is_active & EVENT_FLEXIBLE) && (event_type & EVENT_FLEXIBLE)) { 2365 list_for_each_entry(event, &ctx->flexible_groups, group_entry) 2366 group_sched_out(event, cpuctx, ctx); 2367 } 2368 perf_pmu_enable(ctx->pmu); 2369 } 2370 2371 /* 2372 * Test whether two contexts are equivalent, i.e. whether they have both been 2373 * cloned from the same version of the same context. 2374 * 2375 * Equivalence is measured using a generation number in the context that is 2376 * incremented on each modification to it; see unclone_ctx(), list_add_event() 2377 * and list_del_event(). 2378 */ 2379 static int context_equiv(struct perf_event_context *ctx1, 2380 struct perf_event_context *ctx2) 2381 { 2382 lockdep_assert_held(&ctx1->lock); 2383 lockdep_assert_held(&ctx2->lock); 2384 2385 /* Pinning disables the swap optimization */ 2386 if (ctx1->pin_count || ctx2->pin_count) 2387 return 0; 2388 2389 /* If ctx1 is the parent of ctx2 */ 2390 if (ctx1 == ctx2->parent_ctx && ctx1->generation == ctx2->parent_gen) 2391 return 1; 2392 2393 /* If ctx2 is the parent of ctx1 */ 2394 if (ctx1->parent_ctx == ctx2 && ctx1->parent_gen == ctx2->generation) 2395 return 1; 2396 2397 /* 2398 * If ctx1 and ctx2 have the same parent; we flatten the parent 2399 * hierarchy, see perf_event_init_context(). 2400 */ 2401 if (ctx1->parent_ctx && ctx1->parent_ctx == ctx2->parent_ctx && 2402 ctx1->parent_gen == ctx2->parent_gen) 2403 return 1; 2404 2405 /* Unmatched */ 2406 return 0; 2407 } 2408 2409 static void __perf_event_sync_stat(struct perf_event *event, 2410 struct perf_event *next_event) 2411 { 2412 u64 value; 2413 2414 if (!event->attr.inherit_stat) 2415 return; 2416 2417 /* 2418 * Update the event value, we cannot use perf_event_read() 2419 * because we're in the middle of a context switch and have IRQs 2420 * disabled, which upsets smp_call_function_single(), however 2421 * we know the event must be on the current CPU, therefore we 2422 * don't need to use it. 2423 */ 2424 switch (event->state) { 2425 case PERF_EVENT_STATE_ACTIVE: 2426 event->pmu->read(event); 2427 /* fall-through */ 2428 2429 case PERF_EVENT_STATE_INACTIVE: 2430 update_event_times(event); 2431 break; 2432 2433 default: 2434 break; 2435 } 2436 2437 /* 2438 * In order to keep per-task stats reliable we need to flip the event 2439 * values when we flip the contexts. 2440 */ 2441 value = local64_read(&next_event->count); 2442 value = local64_xchg(&event->count, value); 2443 local64_set(&next_event->count, value); 2444 2445 swap(event->total_time_enabled, next_event->total_time_enabled); 2446 swap(event->total_time_running, next_event->total_time_running); 2447 2448 /* 2449 * Since we swizzled the values, update the user visible data too. 2450 */ 2451 perf_event_update_userpage(event); 2452 perf_event_update_userpage(next_event); 2453 } 2454 2455 static void perf_event_sync_stat(struct perf_event_context *ctx, 2456 struct perf_event_context *next_ctx) 2457 { 2458 struct perf_event *event, *next_event; 2459 2460 if (!ctx->nr_stat) 2461 return; 2462 2463 update_context_time(ctx); 2464 2465 event = list_first_entry(&ctx->event_list, 2466 struct perf_event, event_entry); 2467 2468 next_event = list_first_entry(&next_ctx->event_list, 2469 struct perf_event, event_entry); 2470 2471 while (&event->event_entry != &ctx->event_list && 2472 &next_event->event_entry != &next_ctx->event_list) { 2473 2474 __perf_event_sync_stat(event, next_event); 2475 2476 event = list_next_entry(event, event_entry); 2477 next_event = list_next_entry(next_event, event_entry); 2478 } 2479 } 2480 2481 static void perf_event_context_sched_out(struct task_struct *task, int ctxn, 2482 struct task_struct *next) 2483 { 2484 struct perf_event_context *ctx = task->perf_event_ctxp[ctxn]; 2485 struct perf_event_context *next_ctx; 2486 struct perf_event_context *parent, *next_parent; 2487 struct perf_cpu_context *cpuctx; 2488 int do_switch = 1; 2489 2490 if (likely(!ctx)) 2491 return; 2492 2493 cpuctx = __get_cpu_context(ctx); 2494 if (!cpuctx->task_ctx) 2495 return; 2496 2497 rcu_read_lock(); 2498 next_ctx = next->perf_event_ctxp[ctxn]; 2499 if (!next_ctx) 2500 goto unlock; 2501 2502 parent = rcu_dereference(ctx->parent_ctx); 2503 next_parent = rcu_dereference(next_ctx->parent_ctx); 2504 2505 /* If neither context have a parent context; they cannot be clones. */ 2506 if (!parent && !next_parent) 2507 goto unlock; 2508 2509 if (next_parent == ctx || next_ctx == parent || next_parent == parent) { 2510 /* 2511 * Looks like the two contexts are clones, so we might be 2512 * able to optimize the context switch. We lock both 2513 * contexts and check that they are clones under the 2514 * lock (including re-checking that neither has been 2515 * uncloned in the meantime). It doesn't matter which 2516 * order we take the locks because no other cpu could 2517 * be trying to lock both of these tasks. 2518 */ 2519 raw_spin_lock(&ctx->lock); 2520 raw_spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING); 2521 if (context_equiv(ctx, next_ctx)) { 2522 WRITE_ONCE(ctx->task, next); 2523 WRITE_ONCE(next_ctx->task, task); 2524 2525 swap(ctx->task_ctx_data, next_ctx->task_ctx_data); 2526 2527 /* 2528 * RCU_INIT_POINTER here is safe because we've not 2529 * modified the ctx and the above modification of 2530 * ctx->task and ctx->task_ctx_data are immaterial 2531 * since those values are always verified under 2532 * ctx->lock which we're now holding. 2533 */ 2534 RCU_INIT_POINTER(task->perf_event_ctxp[ctxn], next_ctx); 2535 RCU_INIT_POINTER(next->perf_event_ctxp[ctxn], ctx); 2536 2537 do_switch = 0; 2538 2539 perf_event_sync_stat(ctx, next_ctx); 2540 } 2541 raw_spin_unlock(&next_ctx->lock); 2542 raw_spin_unlock(&ctx->lock); 2543 } 2544 unlock: 2545 rcu_read_unlock(); 2546 2547 if (do_switch) { 2548 raw_spin_lock(&ctx->lock); 2549 task_ctx_sched_out(cpuctx, ctx); 2550 raw_spin_unlock(&ctx->lock); 2551 } 2552 } 2553 2554 void perf_sched_cb_dec(struct pmu *pmu) 2555 { 2556 this_cpu_dec(perf_sched_cb_usages); 2557 } 2558 2559 void perf_sched_cb_inc(struct pmu *pmu) 2560 { 2561 this_cpu_inc(perf_sched_cb_usages); 2562 } 2563 2564 /* 2565 * This function provides the context switch callback to the lower code 2566 * layer. It is invoked ONLY when the context switch callback is enabled. 2567 */ 2568 static void perf_pmu_sched_task(struct task_struct *prev, 2569 struct task_struct *next, 2570 bool sched_in) 2571 { 2572 struct perf_cpu_context *cpuctx; 2573 struct pmu *pmu; 2574 unsigned long flags; 2575 2576 if (prev == next) 2577 return; 2578 2579 local_irq_save(flags); 2580 2581 rcu_read_lock(); 2582 2583 list_for_each_entry_rcu(pmu, &pmus, entry) { 2584 if (pmu->sched_task) { 2585 cpuctx = this_cpu_ptr(pmu->pmu_cpu_context); 2586 2587 perf_ctx_lock(cpuctx, cpuctx->task_ctx); 2588 2589 perf_pmu_disable(pmu); 2590 2591 pmu->sched_task(cpuctx->task_ctx, sched_in); 2592 2593 perf_pmu_enable(pmu); 2594 2595 perf_ctx_unlock(cpuctx, cpuctx->task_ctx); 2596 } 2597 } 2598 2599 rcu_read_unlock(); 2600 2601 local_irq_restore(flags); 2602 } 2603 2604 static void perf_event_switch(struct task_struct *task, 2605 struct task_struct *next_prev, bool sched_in); 2606 2607 #define for_each_task_context_nr(ctxn) \ 2608 for ((ctxn) = 0; (ctxn) < perf_nr_task_contexts; (ctxn)++) 2609 2610 /* 2611 * Called from scheduler to remove the events of the current task, 2612 * with interrupts disabled. 2613 * 2614 * We stop each event and update the event value in event->count. 2615 * 2616 * This does not protect us against NMI, but disable() 2617 * sets the disabled bit in the control field of event _before_ 2618 * accessing the event control register. If a NMI hits, then it will 2619 * not restart the event. 2620 */ 2621 void __perf_event_task_sched_out(struct task_struct *task, 2622 struct task_struct *next) 2623 { 2624 int ctxn; 2625 2626 if (__this_cpu_read(perf_sched_cb_usages)) 2627 perf_pmu_sched_task(task, next, false); 2628 2629 if (atomic_read(&nr_switch_events)) 2630 perf_event_switch(task, next, false); 2631 2632 for_each_task_context_nr(ctxn) 2633 perf_event_context_sched_out(task, ctxn, next); 2634 2635 /* 2636 * if cgroup events exist on this CPU, then we need 2637 * to check if we have to switch out PMU state. 2638 * cgroup event are system-wide mode only 2639 */ 2640 if (atomic_read(this_cpu_ptr(&perf_cgroup_events))) 2641 perf_cgroup_sched_out(task, next); 2642 } 2643 2644 static void task_ctx_sched_out(struct perf_cpu_context *cpuctx, 2645 struct perf_event_context *ctx) 2646 { 2647 if (!cpuctx->task_ctx) 2648 return; 2649 2650 if (WARN_ON_ONCE(ctx != cpuctx->task_ctx)) 2651 return; 2652 2653 ctx_sched_out(ctx, cpuctx, EVENT_ALL); 2654 } 2655 2656 /* 2657 * Called with IRQs disabled 2658 */ 2659 static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx, 2660 enum event_type_t event_type) 2661 { 2662 ctx_sched_out(&cpuctx->ctx, cpuctx, event_type); 2663 } 2664 2665 static void 2666 ctx_pinned_sched_in(struct perf_event_context *ctx, 2667 struct perf_cpu_context *cpuctx) 2668 { 2669 struct perf_event *event; 2670 2671 list_for_each_entry(event, &ctx->pinned_groups, group_entry) { 2672 if (event->state <= PERF_EVENT_STATE_OFF) 2673 continue; 2674 if (!event_filter_match(event)) 2675 continue; 2676 2677 /* may need to reset tstamp_enabled */ 2678 if (is_cgroup_event(event)) 2679 perf_cgroup_mark_enabled(event, ctx); 2680 2681 if (group_can_go_on(event, cpuctx, 1)) 2682 group_sched_in(event, cpuctx, ctx); 2683 2684 /* 2685 * If this pinned group hasn't been scheduled, 2686 * put it in error state. 2687 */ 2688 if (event->state == PERF_EVENT_STATE_INACTIVE) { 2689 update_group_times(event); 2690 event->state = PERF_EVENT_STATE_ERROR; 2691 } 2692 } 2693 } 2694 2695 static void 2696 ctx_flexible_sched_in(struct perf_event_context *ctx, 2697 struct perf_cpu_context *cpuctx) 2698 { 2699 struct perf_event *event; 2700 int can_add_hw = 1; 2701 2702 list_for_each_entry(event, &ctx->flexible_groups, group_entry) { 2703 /* Ignore events in OFF or ERROR state */ 2704 if (event->state <= PERF_EVENT_STATE_OFF) 2705 continue; 2706 /* 2707 * Listen to the 'cpu' scheduling filter constraint 2708 * of events: 2709 */ 2710 if (!event_filter_match(event)) 2711 continue; 2712 2713 /* may need to reset tstamp_enabled */ 2714 if (is_cgroup_event(event)) 2715 perf_cgroup_mark_enabled(event, ctx); 2716 2717 if (group_can_go_on(event, cpuctx, can_add_hw)) { 2718 if (group_sched_in(event, cpuctx, ctx)) 2719 can_add_hw = 0; 2720 } 2721 } 2722 } 2723 2724 static void 2725 ctx_sched_in(struct perf_event_context *ctx, 2726 struct perf_cpu_context *cpuctx, 2727 enum event_type_t event_type, 2728 struct task_struct *task) 2729 { 2730 int is_active = ctx->is_active; 2731 u64 now; 2732 2733 lockdep_assert_held(&ctx->lock); 2734 2735 if (likely(!ctx->nr_events)) 2736 return; 2737 2738 ctx->is_active |= event_type; 2739 if (ctx->task) { 2740 if (!is_active) 2741 cpuctx->task_ctx = ctx; 2742 else 2743 WARN_ON_ONCE(cpuctx->task_ctx != ctx); 2744 } 2745 2746 now = perf_clock(); 2747 ctx->timestamp = now; 2748 perf_cgroup_set_timestamp(task, ctx); 2749 /* 2750 * First go through the list and put on any pinned groups 2751 * in order to give them the best chance of going on. 2752 */ 2753 if (!(is_active & EVENT_PINNED) && (event_type & EVENT_PINNED)) 2754 ctx_pinned_sched_in(ctx, cpuctx); 2755 2756 /* Then walk through the lower prio flexible groups */ 2757 if (!(is_active & EVENT_FLEXIBLE) && (event_type & EVENT_FLEXIBLE)) 2758 ctx_flexible_sched_in(ctx, cpuctx); 2759 } 2760 2761 static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx, 2762 enum event_type_t event_type, 2763 struct task_struct *task) 2764 { 2765 struct perf_event_context *ctx = &cpuctx->ctx; 2766 2767 ctx_sched_in(ctx, cpuctx, event_type, task); 2768 } 2769 2770 static void perf_event_context_sched_in(struct perf_event_context *ctx, 2771 struct task_struct *task) 2772 { 2773 struct perf_cpu_context *cpuctx; 2774 2775 cpuctx = __get_cpu_context(ctx); 2776 if (cpuctx->task_ctx == ctx) 2777 return; 2778 2779 perf_ctx_lock(cpuctx, ctx); 2780 perf_pmu_disable(ctx->pmu); 2781 /* 2782 * We want to keep the following priority order: 2783 * cpu pinned (that don't need to move), task pinned, 2784 * cpu flexible, task flexible. 2785 */ 2786 cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE); 2787 perf_event_sched_in(cpuctx, ctx, task); 2788 perf_pmu_enable(ctx->pmu); 2789 perf_ctx_unlock(cpuctx, ctx); 2790 } 2791 2792 /* 2793 * Called from scheduler to add the events of the current task 2794 * with interrupts disabled. 2795 * 2796 * We restore the event value and then enable it. 2797 * 2798 * This does not protect us against NMI, but enable() 2799 * sets the enabled bit in the control field of event _before_ 2800 * accessing the event control register. If a NMI hits, then it will 2801 * keep the event running. 2802 */ 2803 void __perf_event_task_sched_in(struct task_struct *prev, 2804 struct task_struct *task) 2805 { 2806 struct perf_event_context *ctx; 2807 int ctxn; 2808 2809 /* 2810 * If cgroup events exist on this CPU, then we need to check if we have 2811 * to switch in PMU state; cgroup event are system-wide mode only. 2812 * 2813 * Since cgroup events are CPU events, we must schedule these in before 2814 * we schedule in the task events. 2815 */ 2816 if (atomic_read(this_cpu_ptr(&perf_cgroup_events))) 2817 perf_cgroup_sched_in(prev, task); 2818 2819 for_each_task_context_nr(ctxn) { 2820 ctx = task->perf_event_ctxp[ctxn]; 2821 if (likely(!ctx)) 2822 continue; 2823 2824 perf_event_context_sched_in(ctx, task); 2825 } 2826 2827 if (atomic_read(&nr_switch_events)) 2828 perf_event_switch(task, prev, true); 2829 2830 if (__this_cpu_read(perf_sched_cb_usages)) 2831 perf_pmu_sched_task(prev, task, true); 2832 } 2833 2834 static u64 perf_calculate_period(struct perf_event *event, u64 nsec, u64 count) 2835 { 2836 u64 frequency = event->attr.sample_freq; 2837 u64 sec = NSEC_PER_SEC; 2838 u64 divisor, dividend; 2839 2840 int count_fls, nsec_fls, frequency_fls, sec_fls; 2841 2842 count_fls = fls64(count); 2843 nsec_fls = fls64(nsec); 2844 frequency_fls = fls64(frequency); 2845 sec_fls = 30; 2846 2847 /* 2848 * We got @count in @nsec, with a target of sample_freq HZ 2849 * the target period becomes: 2850 * 2851 * @count * 10^9 2852 * period = ------------------- 2853 * @nsec * sample_freq 2854 * 2855 */ 2856 2857 /* 2858 * Reduce accuracy by one bit such that @a and @b converge 2859 * to a similar magnitude. 2860 */ 2861 #define REDUCE_FLS(a, b) \ 2862 do { \ 2863 if (a##_fls > b##_fls) { \ 2864 a >>= 1; \ 2865 a##_fls--; \ 2866 } else { \ 2867 b >>= 1; \ 2868 b##_fls--; \ 2869 } \ 2870 } while (0) 2871 2872 /* 2873 * Reduce accuracy until either term fits in a u64, then proceed with 2874 * the other, so that finally we can do a u64/u64 division. 2875 */ 2876 while (count_fls + sec_fls > 64 && nsec_fls + frequency_fls > 64) { 2877 REDUCE_FLS(nsec, frequency); 2878 REDUCE_FLS(sec, count); 2879 } 2880 2881 if (count_fls + sec_fls > 64) { 2882 divisor = nsec * frequency; 2883 2884 while (count_fls + sec_fls > 64) { 2885 REDUCE_FLS(count, sec); 2886 divisor >>= 1; 2887 } 2888 2889 dividend = count * sec; 2890 } else { 2891 dividend = count * sec; 2892 2893 while (nsec_fls + frequency_fls > 64) { 2894 REDUCE_FLS(nsec, frequency); 2895 dividend >>= 1; 2896 } 2897 2898 divisor = nsec * frequency; 2899 } 2900 2901 if (!divisor) 2902 return dividend; 2903 2904 return div64_u64(dividend, divisor); 2905 } 2906 2907 static DEFINE_PER_CPU(int, perf_throttled_count); 2908 static DEFINE_PER_CPU(u64, perf_throttled_seq); 2909 2910 static void perf_adjust_period(struct perf_event *event, u64 nsec, u64 count, bool disable) 2911 { 2912 struct hw_perf_event *hwc = &event->hw; 2913 s64 period, sample_period; 2914 s64 delta; 2915 2916 period = perf_calculate_period(event, nsec, count); 2917 2918 delta = (s64)(period - hwc->sample_period); 2919 delta = (delta + 7) / 8; /* low pass filter */ 2920 2921 sample_period = hwc->sample_period + delta; 2922 2923 if (!sample_period) 2924 sample_period = 1; 2925 2926 hwc->sample_period = sample_period; 2927 2928 if (local64_read(&hwc->period_left) > 8*sample_period) { 2929 if (disable) 2930 event->pmu->stop(event, PERF_EF_UPDATE); 2931 2932 local64_set(&hwc->period_left, 0); 2933 2934 if (disable) 2935 event->pmu->start(event, PERF_EF_RELOAD); 2936 } 2937 } 2938 2939 /* 2940 * combine freq adjustment with unthrottling to avoid two passes over the 2941 * events. At the same time, make sure, having freq events does not change 2942 * the rate of unthrottling as that would introduce bias. 2943 */ 2944 static void perf_adjust_freq_unthr_context(struct perf_event_context *ctx, 2945 int needs_unthr) 2946 { 2947 struct perf_event *event; 2948 struct hw_perf_event *hwc; 2949 u64 now, period = TICK_NSEC; 2950 s64 delta; 2951 2952 /* 2953 * only need to iterate over all events iff: 2954 * - context have events in frequency mode (needs freq adjust) 2955 * - there are events to unthrottle on this cpu 2956 */ 2957 if (!(ctx->nr_freq || needs_unthr)) 2958 return; 2959 2960 raw_spin_lock(&ctx->lock); 2961 perf_pmu_disable(ctx->pmu); 2962 2963 list_for_each_entry_rcu(event, &ctx->event_list, event_entry) { 2964 if (event->state != PERF_EVENT_STATE_ACTIVE) 2965 continue; 2966 2967 if (!event_filter_match(event)) 2968 continue; 2969 2970 perf_pmu_disable(event->pmu); 2971 2972 hwc = &event->hw; 2973 2974 if (hwc->interrupts == MAX_INTERRUPTS) { 2975 hwc->interrupts = 0; 2976 perf_log_throttle(event, 1); 2977 event->pmu->start(event, 0); 2978 } 2979 2980 if (!event->attr.freq || !event->attr.sample_freq) 2981 goto next; 2982 2983 /* 2984 * stop the event and update event->count 2985 */ 2986 event->pmu->stop(event, PERF_EF_UPDATE); 2987 2988 now = local64_read(&event->count); 2989 delta = now - hwc->freq_count_stamp; 2990 hwc->freq_count_stamp = now; 2991 2992 /* 2993 * restart the event 2994 * reload only if value has changed 2995 * we have stopped the event so tell that 2996 * to perf_adjust_period() to avoid stopping it 2997 * twice. 2998 */ 2999 if (delta > 0) 3000 perf_adjust_period(event, period, delta, false); 3001 3002 event->pmu->start(event, delta > 0 ? PERF_EF_RELOAD : 0); 3003 next: 3004 perf_pmu_enable(event->pmu); 3005 } 3006 3007 perf_pmu_enable(ctx->pmu); 3008 raw_spin_unlock(&ctx->lock); 3009 } 3010 3011 /* 3012 * Round-robin a context's events: 3013 */ 3014 static void rotate_ctx(struct perf_event_context *ctx) 3015 { 3016 /* 3017 * Rotate the first entry last of non-pinned groups. Rotation might be 3018 * disabled by the inheritance code. 3019 */ 3020 if (!ctx->rotate_disable) 3021 list_rotate_left(&ctx->flexible_groups); 3022 } 3023 3024 static int perf_rotate_context(struct perf_cpu_context *cpuctx) 3025 { 3026 struct perf_event_context *ctx = NULL; 3027 int rotate = 0; 3028 3029 if (cpuctx->ctx.nr_events) { 3030 if (cpuctx->ctx.nr_events != cpuctx->ctx.nr_active) 3031 rotate = 1; 3032 } 3033 3034 ctx = cpuctx->task_ctx; 3035 if (ctx && ctx->nr_events) { 3036 if (ctx->nr_events != ctx->nr_active) 3037 rotate = 1; 3038 } 3039 3040 if (!rotate) 3041 goto done; 3042 3043 perf_ctx_lock(cpuctx, cpuctx->task_ctx); 3044 perf_pmu_disable(cpuctx->ctx.pmu); 3045 3046 cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE); 3047 if (ctx) 3048 ctx_sched_out(ctx, cpuctx, EVENT_FLEXIBLE); 3049 3050 rotate_ctx(&cpuctx->ctx); 3051 if (ctx) 3052 rotate_ctx(ctx); 3053 3054 perf_event_sched_in(cpuctx, ctx, current); 3055 3056 perf_pmu_enable(cpuctx->ctx.pmu); 3057 perf_ctx_unlock(cpuctx, cpuctx->task_ctx); 3058 done: 3059 3060 return rotate; 3061 } 3062 3063 #ifdef CONFIG_NO_HZ_FULL 3064 bool perf_event_can_stop_tick(void) 3065 { 3066 if (atomic_read(&nr_freq_events) || 3067 __this_cpu_read(perf_throttled_count)) 3068 return false; 3069 else 3070 return true; 3071 } 3072 #endif 3073 3074 void perf_event_task_tick(void) 3075 { 3076 struct list_head *head = this_cpu_ptr(&active_ctx_list); 3077 struct perf_event_context *ctx, *tmp; 3078 int throttled; 3079 3080 WARN_ON(!irqs_disabled()); 3081 3082 __this_cpu_inc(perf_throttled_seq); 3083 throttled = __this_cpu_xchg(perf_throttled_count, 0); 3084 3085 list_for_each_entry_safe(ctx, tmp, head, active_ctx_list) 3086 perf_adjust_freq_unthr_context(ctx, throttled); 3087 } 3088 3089 static int event_enable_on_exec(struct perf_event *event, 3090 struct perf_event_context *ctx) 3091 { 3092 if (!event->attr.enable_on_exec) 3093 return 0; 3094 3095 event->attr.enable_on_exec = 0; 3096 if (event->state >= PERF_EVENT_STATE_INACTIVE) 3097 return 0; 3098 3099 __perf_event_mark_enabled(event); 3100 3101 return 1; 3102 } 3103 3104 /* 3105 * Enable all of a task's events that have been marked enable-on-exec. 3106 * This expects task == current. 3107 */ 3108 static void perf_event_enable_on_exec(int ctxn) 3109 { 3110 struct perf_event_context *ctx, *clone_ctx = NULL; 3111 struct perf_cpu_context *cpuctx; 3112 struct perf_event *event; 3113 unsigned long flags; 3114 int enabled = 0; 3115 3116 local_irq_save(flags); 3117 ctx = current->perf_event_ctxp[ctxn]; 3118 if (!ctx || !ctx->nr_events) 3119 goto out; 3120 3121 cpuctx = __get_cpu_context(ctx); 3122 perf_ctx_lock(cpuctx, ctx); 3123 list_for_each_entry(event, &ctx->event_list, event_entry) 3124 enabled |= event_enable_on_exec(event, ctx); 3125 3126 /* 3127 * Unclone and reschedule this context if we enabled any event. 3128 */ 3129 if (enabled) { 3130 clone_ctx = unclone_ctx(ctx); 3131 ctx_resched(cpuctx, ctx); 3132 } 3133 perf_ctx_unlock(cpuctx, ctx); 3134 3135 out: 3136 local_irq_restore(flags); 3137 3138 if (clone_ctx) 3139 put_ctx(clone_ctx); 3140 } 3141 3142 void perf_event_exec(void) 3143 { 3144 int ctxn; 3145 3146 rcu_read_lock(); 3147 for_each_task_context_nr(ctxn) 3148 perf_event_enable_on_exec(ctxn); 3149 rcu_read_unlock(); 3150 } 3151 3152 struct perf_read_data { 3153 struct perf_event *event; 3154 bool group; 3155 int ret; 3156 }; 3157 3158 /* 3159 * Cross CPU call to read the hardware event 3160 */ 3161 static void __perf_event_read(void *info) 3162 { 3163 struct perf_read_data *data = info; 3164 struct perf_event *sub, *event = data->event; 3165 struct perf_event_context *ctx = event->ctx; 3166 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx); 3167 struct pmu *pmu = event->pmu; 3168 3169 /* 3170 * If this is a task context, we need to check whether it is 3171 * the current task context of this cpu. If not it has been 3172 * scheduled out before the smp call arrived. In that case 3173 * event->count would have been updated to a recent sample 3174 * when the event was scheduled out. 3175 */ 3176 if (ctx->task && cpuctx->task_ctx != ctx) 3177 return; 3178 3179 raw_spin_lock(&ctx->lock); 3180 if (ctx->is_active) { 3181 update_context_time(ctx); 3182 update_cgrp_time_from_event(event); 3183 } 3184 3185 update_event_times(event); 3186 if (event->state != PERF_EVENT_STATE_ACTIVE) 3187 goto unlock; 3188 3189 if (!data->group) { 3190 pmu->read(event); 3191 data->ret = 0; 3192 goto unlock; 3193 } 3194 3195 pmu->start_txn(pmu, PERF_PMU_TXN_READ); 3196 3197 pmu->read(event); 3198 3199 list_for_each_entry(sub, &event->sibling_list, group_entry) { 3200 update_event_times(sub); 3201 if (sub->state == PERF_EVENT_STATE_ACTIVE) { 3202 /* 3203 * Use sibling's PMU rather than @event's since 3204 * sibling could be on different (eg: software) PMU. 3205 */ 3206 sub->pmu->read(sub); 3207 } 3208 } 3209 3210 data->ret = pmu->commit_txn(pmu); 3211 3212 unlock: 3213 raw_spin_unlock(&ctx->lock); 3214 } 3215 3216 static inline u64 perf_event_count(struct perf_event *event) 3217 { 3218 if (event->pmu->count) 3219 return event->pmu->count(event); 3220 3221 return __perf_event_count(event); 3222 } 3223 3224 /* 3225 * NMI-safe method to read a local event, that is an event that 3226 * is: 3227 * - either for the current task, or for this CPU 3228 * - does not have inherit set, for inherited task events 3229 * will not be local and we cannot read them atomically 3230 * - must not have a pmu::count method 3231 */ 3232 u64 perf_event_read_local(struct perf_event *event) 3233 { 3234 unsigned long flags; 3235 u64 val; 3236 3237 /* 3238 * Disabling interrupts avoids all counter scheduling (context 3239 * switches, timer based rotation and IPIs). 3240 */ 3241 local_irq_save(flags); 3242 3243 /* If this is a per-task event, it must be for current */ 3244 WARN_ON_ONCE((event->attach_state & PERF_ATTACH_TASK) && 3245 event->hw.target != current); 3246 3247 /* If this is a per-CPU event, it must be for this CPU */ 3248 WARN_ON_ONCE(!(event->attach_state & PERF_ATTACH_TASK) && 3249 event->cpu != smp_processor_id()); 3250 3251 /* 3252 * It must not be an event with inherit set, we cannot read 3253 * all child counters from atomic context. 3254 */ 3255 WARN_ON_ONCE(event->attr.inherit); 3256 3257 /* 3258 * It must not have a pmu::count method, those are not 3259 * NMI safe. 3260 */ 3261 WARN_ON_ONCE(event->pmu->count); 3262 3263 /* 3264 * If the event is currently on this CPU, its either a per-task event, 3265 * or local to this CPU. Furthermore it means its ACTIVE (otherwise 3266 * oncpu == -1). 3267 */ 3268 if (event->oncpu == smp_processor_id()) 3269 event->pmu->read(event); 3270 3271 val = local64_read(&event->count); 3272 local_irq_restore(flags); 3273 3274 return val; 3275 } 3276 3277 static int perf_event_read(struct perf_event *event, bool group) 3278 { 3279 int ret = 0; 3280 3281 /* 3282 * If event is enabled and currently active on a CPU, update the 3283 * value in the event structure: 3284 */ 3285 if (event->state == PERF_EVENT_STATE_ACTIVE) { 3286 struct perf_read_data data = { 3287 .event = event, 3288 .group = group, 3289 .ret = 0, 3290 }; 3291 smp_call_function_single(event->oncpu, 3292 __perf_event_read, &data, 1); 3293 ret = data.ret; 3294 } else if (event->state == PERF_EVENT_STATE_INACTIVE) { 3295 struct perf_event_context *ctx = event->ctx; 3296 unsigned long flags; 3297 3298 raw_spin_lock_irqsave(&ctx->lock, flags); 3299 /* 3300 * may read while context is not active 3301 * (e.g., thread is blocked), in that case 3302 * we cannot update context time 3303 */ 3304 if (ctx->is_active) { 3305 update_context_time(ctx); 3306 update_cgrp_time_from_event(event); 3307 } 3308 if (group) 3309 update_group_times(event); 3310 else 3311 update_event_times(event); 3312 raw_spin_unlock_irqrestore(&ctx->lock, flags); 3313 } 3314 3315 return ret; 3316 } 3317 3318 /* 3319 * Initialize the perf_event context in a task_struct: 3320 */ 3321 static void __perf_event_init_context(struct perf_event_context *ctx) 3322 { 3323 raw_spin_lock_init(&ctx->lock); 3324 mutex_init(&ctx->mutex); 3325 INIT_LIST_HEAD(&ctx->active_ctx_list); 3326 INIT_LIST_HEAD(&ctx->pinned_groups); 3327 INIT_LIST_HEAD(&ctx->flexible_groups); 3328 INIT_LIST_HEAD(&ctx->event_list); 3329 atomic_set(&ctx->refcount, 1); 3330 } 3331 3332 static struct perf_event_context * 3333 alloc_perf_context(struct pmu *pmu, struct task_struct *task) 3334 { 3335 struct perf_event_context *ctx; 3336 3337 ctx = kzalloc(sizeof(struct perf_event_context), GFP_KERNEL); 3338 if (!ctx) 3339 return NULL; 3340 3341 __perf_event_init_context(ctx); 3342 if (task) { 3343 ctx->task = task; 3344 get_task_struct(task); 3345 } 3346 ctx->pmu = pmu; 3347 3348 return ctx; 3349 } 3350 3351 static struct task_struct * 3352 find_lively_task_by_vpid(pid_t vpid) 3353 { 3354 struct task_struct *task; 3355 int err; 3356 3357 rcu_read_lock(); 3358 if (!vpid) 3359 task = current; 3360 else 3361 task = find_task_by_vpid(vpid); 3362 if (task) 3363 get_task_struct(task); 3364 rcu_read_unlock(); 3365 3366 if (!task) 3367 return ERR_PTR(-ESRCH); 3368 3369 /* Reuse ptrace permission checks for now. */ 3370 err = -EACCES; 3371 if (!ptrace_may_access(task, PTRACE_MODE_READ_REALCREDS)) 3372 goto errout; 3373 3374 return task; 3375 errout: 3376 put_task_struct(task); 3377 return ERR_PTR(err); 3378 3379 } 3380 3381 /* 3382 * Returns a matching context with refcount and pincount. 3383 */ 3384 static struct perf_event_context * 3385 find_get_context(struct pmu *pmu, struct task_struct *task, 3386 struct perf_event *event) 3387 { 3388 struct perf_event_context *ctx, *clone_ctx = NULL; 3389 struct perf_cpu_context *cpuctx; 3390 void *task_ctx_data = NULL; 3391 unsigned long flags; 3392 int ctxn, err; 3393 int cpu = event->cpu; 3394 3395 if (!task) { 3396 /* Must be root to operate on a CPU event: */ 3397 if (perf_paranoid_cpu() && !capable(CAP_SYS_ADMIN)) 3398 return ERR_PTR(-EACCES); 3399 3400 /* 3401 * We could be clever and allow to attach a event to an 3402 * offline CPU and activate it when the CPU comes up, but 3403 * that's for later. 3404 */ 3405 if (!cpu_online(cpu)) 3406 return ERR_PTR(-ENODEV); 3407 3408 cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu); 3409 ctx = &cpuctx->ctx; 3410 get_ctx(ctx); 3411 ++ctx->pin_count; 3412 3413 return ctx; 3414 } 3415 3416 err = -EINVAL; 3417 ctxn = pmu->task_ctx_nr; 3418 if (ctxn < 0) 3419 goto errout; 3420 3421 if (event->attach_state & PERF_ATTACH_TASK_DATA) { 3422 task_ctx_data = kzalloc(pmu->task_ctx_size, GFP_KERNEL); 3423 if (!task_ctx_data) { 3424 err = -ENOMEM; 3425 goto errout; 3426 } 3427 } 3428 3429 retry: 3430 ctx = perf_lock_task_context(task, ctxn, &flags); 3431 if (ctx) { 3432 clone_ctx = unclone_ctx(ctx); 3433 ++ctx->pin_count; 3434 3435 if (task_ctx_data && !ctx->task_ctx_data) { 3436 ctx->task_ctx_data = task_ctx_data; 3437 task_ctx_data = NULL; 3438 } 3439 raw_spin_unlock_irqrestore(&ctx->lock, flags); 3440 3441 if (clone_ctx) 3442 put_ctx(clone_ctx); 3443 } else { 3444 ctx = alloc_perf_context(pmu, task); 3445 err = -ENOMEM; 3446 if (!ctx) 3447 goto errout; 3448 3449 if (task_ctx_data) { 3450 ctx->task_ctx_data = task_ctx_data; 3451 task_ctx_data = NULL; 3452 } 3453 3454 err = 0; 3455 mutex_lock(&task->perf_event_mutex); 3456 /* 3457 * If it has already passed perf_event_exit_task(). 3458 * we must see PF_EXITING, it takes this mutex too. 3459 */ 3460 if (task->flags & PF_EXITING) 3461 err = -ESRCH; 3462 else if (task->perf_event_ctxp[ctxn]) 3463 err = -EAGAIN; 3464 else { 3465 get_ctx(ctx); 3466 ++ctx->pin_count; 3467 rcu_assign_pointer(task->perf_event_ctxp[ctxn], ctx); 3468 } 3469 mutex_unlock(&task->perf_event_mutex); 3470 3471 if (unlikely(err)) { 3472 put_ctx(ctx); 3473 3474 if (err == -EAGAIN) 3475 goto retry; 3476 goto errout; 3477 } 3478 } 3479 3480 kfree(task_ctx_data); 3481 return ctx; 3482 3483 errout: 3484 kfree(task_ctx_data); 3485 return ERR_PTR(err); 3486 } 3487 3488 static void perf_event_free_filter(struct perf_event *event); 3489 static void perf_event_free_bpf_prog(struct perf_event *event); 3490 3491 static void free_event_rcu(struct rcu_head *head) 3492 { 3493 struct perf_event *event; 3494 3495 event = container_of(head, struct perf_event, rcu_head); 3496 if (event->ns) 3497 put_pid_ns(event->ns); 3498 perf_event_free_filter(event); 3499 kfree(event); 3500 } 3501 3502 static void ring_buffer_attach(struct perf_event *event, 3503 struct ring_buffer *rb); 3504 3505 static void unaccount_event_cpu(struct perf_event *event, int cpu) 3506 { 3507 if (event->parent) 3508 return; 3509 3510 if (is_cgroup_event(event)) 3511 atomic_dec(&per_cpu(perf_cgroup_events, cpu)); 3512 } 3513 3514 static void unaccount_event(struct perf_event *event) 3515 { 3516 bool dec = false; 3517 3518 if (event->parent) 3519 return; 3520 3521 if (event->attach_state & PERF_ATTACH_TASK) 3522 dec = true; 3523 if (event->attr.mmap || event->attr.mmap_data) 3524 atomic_dec(&nr_mmap_events); 3525 if (event->attr.comm) 3526 atomic_dec(&nr_comm_events); 3527 if (event->attr.task) 3528 atomic_dec(&nr_task_events); 3529 if (event->attr.freq) 3530 atomic_dec(&nr_freq_events); 3531 if (event->attr.context_switch) { 3532 dec = true; 3533 atomic_dec(&nr_switch_events); 3534 } 3535 if (is_cgroup_event(event)) 3536 dec = true; 3537 if (has_branch_stack(event)) 3538 dec = true; 3539 3540 if (dec) 3541 static_key_slow_dec_deferred(&perf_sched_events); 3542 3543 unaccount_event_cpu(event, event->cpu); 3544 } 3545 3546 /* 3547 * The following implement mutual exclusion of events on "exclusive" pmus 3548 * (PERF_PMU_CAP_EXCLUSIVE). Such pmus can only have one event scheduled 3549 * at a time, so we disallow creating events that might conflict, namely: 3550 * 3551 * 1) cpu-wide events in the presence of per-task events, 3552 * 2) per-task events in the presence of cpu-wide events, 3553 * 3) two matching events on the same context. 3554 * 3555 * The former two cases are handled in the allocation path (perf_event_alloc(), 3556 * _free_event()), the latter -- before the first perf_install_in_context(). 3557 */ 3558 static int exclusive_event_init(struct perf_event *event) 3559 { 3560 struct pmu *pmu = event->pmu; 3561 3562 if (!(pmu->capabilities & PERF_PMU_CAP_EXCLUSIVE)) 3563 return 0; 3564 3565 /* 3566 * Prevent co-existence of per-task and cpu-wide events on the 3567 * same exclusive pmu. 3568 * 3569 * Negative pmu::exclusive_cnt means there are cpu-wide 3570 * events on this "exclusive" pmu, positive means there are 3571 * per-task events. 3572 * 3573 * Since this is called in perf_event_alloc() path, event::ctx 3574 * doesn't exist yet; it is, however, safe to use PERF_ATTACH_TASK 3575 * to mean "per-task event", because unlike other attach states it 3576 * never gets cleared. 3577 */ 3578 if (event->attach_state & PERF_ATTACH_TASK) { 3579 if (!atomic_inc_unless_negative(&pmu->exclusive_cnt)) 3580 return -EBUSY; 3581 } else { 3582 if (!atomic_dec_unless_positive(&pmu->exclusive_cnt)) 3583 return -EBUSY; 3584 } 3585 3586 return 0; 3587 } 3588 3589 static void exclusive_event_destroy(struct perf_event *event) 3590 { 3591 struct pmu *pmu = event->pmu; 3592 3593 if (!(pmu->capabilities & PERF_PMU_CAP_EXCLUSIVE)) 3594 return; 3595 3596 /* see comment in exclusive_event_init() */ 3597 if (event->attach_state & PERF_ATTACH_TASK) 3598 atomic_dec(&pmu->exclusive_cnt); 3599 else 3600 atomic_inc(&pmu->exclusive_cnt); 3601 } 3602 3603 static bool exclusive_event_match(struct perf_event *e1, struct perf_event *e2) 3604 { 3605 if ((e1->pmu->capabilities & PERF_PMU_CAP_EXCLUSIVE) && 3606 (e1->cpu == e2->cpu || 3607 e1->cpu == -1 || 3608 e2->cpu == -1)) 3609 return true; 3610 return false; 3611 } 3612 3613 /* Called under the same ctx::mutex as perf_install_in_context() */ 3614 static bool exclusive_event_installable(struct perf_event *event, 3615 struct perf_event_context *ctx) 3616 { 3617 struct perf_event *iter_event; 3618 struct pmu *pmu = event->pmu; 3619 3620 if (!(pmu->capabilities & PERF_PMU_CAP_EXCLUSIVE)) 3621 return true; 3622 3623 list_for_each_entry(iter_event, &ctx->event_list, event_entry) { 3624 if (exclusive_event_match(iter_event, event)) 3625 return false; 3626 } 3627 3628 return true; 3629 } 3630 3631 static void _free_event(struct perf_event *event) 3632 { 3633 irq_work_sync(&event->pending); 3634 3635 unaccount_event(event); 3636 3637 if (event->rb) { 3638 /* 3639 * Can happen when we close an event with re-directed output. 3640 * 3641 * Since we have a 0 refcount, perf_mmap_close() will skip 3642 * over us; possibly making our ring_buffer_put() the last. 3643 */ 3644 mutex_lock(&event->mmap_mutex); 3645 ring_buffer_attach(event, NULL); 3646 mutex_unlock(&event->mmap_mutex); 3647 } 3648 3649 if (is_cgroup_event(event)) 3650 perf_detach_cgroup(event); 3651 3652 if (!event->parent) { 3653 if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) 3654 put_callchain_buffers(); 3655 } 3656 3657 perf_event_free_bpf_prog(event); 3658 3659 if (event->destroy) 3660 event->destroy(event); 3661 3662 if (event->ctx) 3663 put_ctx(event->ctx); 3664 3665 if (event->pmu) { 3666 exclusive_event_destroy(event); 3667 module_put(event->pmu->module); 3668 } 3669 3670 call_rcu(&event->rcu_head, free_event_rcu); 3671 } 3672 3673 /* 3674 * Used to free events which have a known refcount of 1, such as in error paths 3675 * where the event isn't exposed yet and inherited events. 3676 */ 3677 static void free_event(struct perf_event *event) 3678 { 3679 if (WARN(atomic_long_cmpxchg(&event->refcount, 1, 0) != 1, 3680 "unexpected event refcount: %ld; ptr=%p\n", 3681 atomic_long_read(&event->refcount), event)) { 3682 /* leak to avoid use-after-free */ 3683 return; 3684 } 3685 3686 _free_event(event); 3687 } 3688 3689 /* 3690 * Remove user event from the owner task. 3691 */ 3692 static void perf_remove_from_owner(struct perf_event *event) 3693 { 3694 struct task_struct *owner; 3695 3696 rcu_read_lock(); 3697 /* 3698 * Matches the smp_store_release() in perf_event_exit_task(). If we 3699 * observe !owner it means the list deletion is complete and we can 3700 * indeed free this event, otherwise we need to serialize on 3701 * owner->perf_event_mutex. 3702 */ 3703 owner = lockless_dereference(event->owner); 3704 if (owner) { 3705 /* 3706 * Since delayed_put_task_struct() also drops the last 3707 * task reference we can safely take a new reference 3708 * while holding the rcu_read_lock(). 3709 */ 3710 get_task_struct(owner); 3711 } 3712 rcu_read_unlock(); 3713 3714 if (owner) { 3715 /* 3716 * If we're here through perf_event_exit_task() we're already 3717 * holding ctx->mutex which would be an inversion wrt. the 3718 * normal lock order. 3719 * 3720 * However we can safely take this lock because its the child 3721 * ctx->mutex. 3722 */ 3723 mutex_lock_nested(&owner->perf_event_mutex, SINGLE_DEPTH_NESTING); 3724 3725 /* 3726 * We have to re-check the event->owner field, if it is cleared 3727 * we raced with perf_event_exit_task(), acquiring the mutex 3728 * ensured they're done, and we can proceed with freeing the 3729 * event. 3730 */ 3731 if (event->owner) { 3732 list_del_init(&event->owner_entry); 3733 smp_store_release(&event->owner, NULL); 3734 } 3735 mutex_unlock(&owner->perf_event_mutex); 3736 put_task_struct(owner); 3737 } 3738 } 3739 3740 static void put_event(struct perf_event *event) 3741 { 3742 if (!atomic_long_dec_and_test(&event->refcount)) 3743 return; 3744 3745 _free_event(event); 3746 } 3747 3748 /* 3749 * Kill an event dead; while event:refcount will preserve the event 3750 * object, it will not preserve its functionality. Once the last 'user' 3751 * gives up the object, we'll destroy the thing. 3752 */ 3753 int perf_event_release_kernel(struct perf_event *event) 3754 { 3755 struct perf_event_context *ctx; 3756 struct perf_event *child, *tmp; 3757 3758 if (!is_kernel_event(event)) 3759 perf_remove_from_owner(event); 3760 3761 ctx = perf_event_ctx_lock(event); 3762 WARN_ON_ONCE(ctx->parent_ctx); 3763 perf_remove_from_context(event, DETACH_GROUP | DETACH_STATE); 3764 perf_event_ctx_unlock(event, ctx); 3765 3766 /* 3767 * At this point we must have event->state == PERF_EVENT_STATE_EXIT, 3768 * either from the above perf_remove_from_context() or through 3769 * perf_event_exit_event(). 3770 * 3771 * Therefore, anybody acquiring event->child_mutex after the below 3772 * loop _must_ also see this, most importantly inherit_event() which 3773 * will avoid placing more children on the list. 3774 * 3775 * Thus this guarantees that we will in fact observe and kill _ALL_ 3776 * child events. 3777 */ 3778 WARN_ON_ONCE(event->state != PERF_EVENT_STATE_EXIT); 3779 3780 again: 3781 mutex_lock(&event->child_mutex); 3782 list_for_each_entry(child, &event->child_list, child_list) { 3783 3784 /* 3785 * Cannot change, child events are not migrated, see the 3786 * comment with perf_event_ctx_lock_nested(). 3787 */ 3788 ctx = lockless_dereference(child->ctx); 3789 /* 3790 * Since child_mutex nests inside ctx::mutex, we must jump 3791 * through hoops. We start by grabbing a reference on the ctx. 3792 * 3793 * Since the event cannot get freed while we hold the 3794 * child_mutex, the context must also exist and have a !0 3795 * reference count. 3796 */ 3797 get_ctx(ctx); 3798 3799 /* 3800 * Now that we have a ctx ref, we can drop child_mutex, and 3801 * acquire ctx::mutex without fear of it going away. Then we 3802 * can re-acquire child_mutex. 3803 */ 3804 mutex_unlock(&event->child_mutex); 3805 mutex_lock(&ctx->mutex); 3806 mutex_lock(&event->child_mutex); 3807 3808 /* 3809 * Now that we hold ctx::mutex and child_mutex, revalidate our 3810 * state, if child is still the first entry, it didn't get freed 3811 * and we can continue doing so. 3812 */ 3813 tmp = list_first_entry_or_null(&event->child_list, 3814 struct perf_event, child_list); 3815 if (tmp == child) { 3816 perf_remove_from_context(child, DETACH_GROUP); 3817 list_del(&child->child_list); 3818 free_event(child); 3819 /* 3820 * This matches the refcount bump in inherit_event(); 3821 * this can't be the last reference. 3822 */ 3823 put_event(event); 3824 } 3825 3826 mutex_unlock(&event->child_mutex); 3827 mutex_unlock(&ctx->mutex); 3828 put_ctx(ctx); 3829 goto again; 3830 } 3831 mutex_unlock(&event->child_mutex); 3832 3833 /* Must be the last reference */ 3834 put_event(event); 3835 return 0; 3836 } 3837 EXPORT_SYMBOL_GPL(perf_event_release_kernel); 3838 3839 /* 3840 * Called when the last reference to the file is gone. 3841 */ 3842 static int perf_release(struct inode *inode, struct file *file) 3843 { 3844 perf_event_release_kernel(file->private_data); 3845 return 0; 3846 } 3847 3848 u64 perf_event_read_value(struct perf_event *event, u64 *enabled, u64 *running) 3849 { 3850 struct perf_event *child; 3851 u64 total = 0; 3852 3853 *enabled = 0; 3854 *running = 0; 3855 3856 mutex_lock(&event->child_mutex); 3857 3858 (void)perf_event_read(event, false); 3859 total += perf_event_count(event); 3860 3861 *enabled += event->total_time_enabled + 3862 atomic64_read(&event->child_total_time_enabled); 3863 *running += event->total_time_running + 3864 atomic64_read(&event->child_total_time_running); 3865 3866 list_for_each_entry(child, &event->child_list, child_list) { 3867 (void)perf_event_read(child, false); 3868 total += perf_event_count(child); 3869 *enabled += child->total_time_enabled; 3870 *running += child->total_time_running; 3871 } 3872 mutex_unlock(&event->child_mutex); 3873 3874 return total; 3875 } 3876 EXPORT_SYMBOL_GPL(perf_event_read_value); 3877 3878 static int __perf_read_group_add(struct perf_event *leader, 3879 u64 read_format, u64 *values) 3880 { 3881 struct perf_event *sub; 3882 int n = 1; /* skip @nr */ 3883 int ret; 3884 3885 ret = perf_event_read(leader, true); 3886 if (ret) 3887 return ret; 3888 3889 /* 3890 * Since we co-schedule groups, {enabled,running} times of siblings 3891 * will be identical to those of the leader, so we only publish one 3892 * set. 3893 */ 3894 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) { 3895 values[n++] += leader->total_time_enabled + 3896 atomic64_read(&leader->child_total_time_enabled); 3897 } 3898 3899 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) { 3900 values[n++] += leader->total_time_running + 3901 atomic64_read(&leader->child_total_time_running); 3902 } 3903 3904 /* 3905 * Write {count,id} tuples for every sibling. 3906 */ 3907 values[n++] += perf_event_count(leader); 3908 if (read_format & PERF_FORMAT_ID) 3909 values[n++] = primary_event_id(leader); 3910 3911 list_for_each_entry(sub, &leader->sibling_list, group_entry) { 3912 values[n++] += perf_event_count(sub); 3913 if (read_format & PERF_FORMAT_ID) 3914 values[n++] = primary_event_id(sub); 3915 } 3916 3917 return 0; 3918 } 3919 3920 static int perf_read_group(struct perf_event *event, 3921 u64 read_format, char __user *buf) 3922 { 3923 struct perf_event *leader = event->group_leader, *child; 3924 struct perf_event_context *ctx = leader->ctx; 3925 int ret; 3926 u64 *values; 3927 3928 lockdep_assert_held(&ctx->mutex); 3929 3930 values = kzalloc(event->read_size, GFP_KERNEL); 3931 if (!values) 3932 return -ENOMEM; 3933 3934 values[0] = 1 + leader->nr_siblings; 3935 3936 /* 3937 * By locking the child_mutex of the leader we effectively 3938 * lock the child list of all siblings.. XXX explain how. 3939 */ 3940 mutex_lock(&leader->child_mutex); 3941 3942 ret = __perf_read_group_add(leader, read_format, values); 3943 if (ret) 3944 goto unlock; 3945 3946 list_for_each_entry(child, &leader->child_list, child_list) { 3947 ret = __perf_read_group_add(child, read_format, values); 3948 if (ret) 3949 goto unlock; 3950 } 3951 3952 mutex_unlock(&leader->child_mutex); 3953 3954 ret = event->read_size; 3955 if (copy_to_user(buf, values, event->read_size)) 3956 ret = -EFAULT; 3957 goto out; 3958 3959 unlock: 3960 mutex_unlock(&leader->child_mutex); 3961 out: 3962 kfree(values); 3963 return ret; 3964 } 3965 3966 static int perf_read_one(struct perf_event *event, 3967 u64 read_format, char __user *buf) 3968 { 3969 u64 enabled, running; 3970 u64 values[4]; 3971 int n = 0; 3972 3973 values[n++] = perf_event_read_value(event, &enabled, &running); 3974 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) 3975 values[n++] = enabled; 3976 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) 3977 values[n++] = running; 3978 if (read_format & PERF_FORMAT_ID) 3979 values[n++] = primary_event_id(event); 3980 3981 if (copy_to_user(buf, values, n * sizeof(u64))) 3982 return -EFAULT; 3983 3984 return n * sizeof(u64); 3985 } 3986 3987 static bool is_event_hup(struct perf_event *event) 3988 { 3989 bool no_children; 3990 3991 if (event->state != PERF_EVENT_STATE_EXIT) 3992 return false; 3993 3994 mutex_lock(&event->child_mutex); 3995 no_children = list_empty(&event->child_list); 3996 mutex_unlock(&event->child_mutex); 3997 return no_children; 3998 } 3999 4000 /* 4001 * Read the performance event - simple non blocking version for now 4002 */ 4003 static ssize_t 4004 __perf_read(struct perf_event *event, char __user *buf, size_t count) 4005 { 4006 u64 read_format = event->attr.read_format; 4007 int ret; 4008 4009 /* 4010 * Return end-of-file for a read on a event that is in 4011 * error state (i.e. because it was pinned but it couldn't be 4012 * scheduled on to the CPU at some point). 4013 */ 4014 if (event->state == PERF_EVENT_STATE_ERROR) 4015 return 0; 4016 4017 if (count < event->read_size) 4018 return -ENOSPC; 4019 4020 WARN_ON_ONCE(event->ctx->parent_ctx); 4021 if (read_format & PERF_FORMAT_GROUP) 4022 ret = perf_read_group(event, read_format, buf); 4023 else 4024 ret = perf_read_one(event, read_format, buf); 4025 4026 return ret; 4027 } 4028 4029 static ssize_t 4030 perf_read(struct file *file, char __user *buf, size_t count, loff_t *ppos) 4031 { 4032 struct perf_event *event = file->private_data; 4033 struct perf_event_context *ctx; 4034 int ret; 4035 4036 ctx = perf_event_ctx_lock(event); 4037 ret = __perf_read(event, buf, count); 4038 perf_event_ctx_unlock(event, ctx); 4039 4040 return ret; 4041 } 4042 4043 static unsigned int perf_poll(struct file *file, poll_table *wait) 4044 { 4045 struct perf_event *event = file->private_data; 4046 struct ring_buffer *rb; 4047 unsigned int events = POLLHUP; 4048 4049 poll_wait(file, &event->waitq, wait); 4050 4051 if (is_event_hup(event)) 4052 return events; 4053 4054 /* 4055 * Pin the event->rb by taking event->mmap_mutex; otherwise 4056 * perf_event_set_output() can swizzle our rb and make us miss wakeups. 4057 */ 4058 mutex_lock(&event->mmap_mutex); 4059 rb = event->rb; 4060 if (rb) 4061 events = atomic_xchg(&rb->poll, 0); 4062 mutex_unlock(&event->mmap_mutex); 4063 return events; 4064 } 4065 4066 static void _perf_event_reset(struct perf_event *event) 4067 { 4068 (void)perf_event_read(event, false); 4069 local64_set(&event->count, 0); 4070 perf_event_update_userpage(event); 4071 } 4072 4073 /* 4074 * Holding the top-level event's child_mutex means that any 4075 * descendant process that has inherited this event will block 4076 * in perf_event_exit_event() if it goes to exit, thus satisfying the 4077 * task existence requirements of perf_event_enable/disable. 4078 */ 4079 static void perf_event_for_each_child(struct perf_event *event, 4080 void (*func)(struct perf_event *)) 4081 { 4082 struct perf_event *child; 4083 4084 WARN_ON_ONCE(event->ctx->parent_ctx); 4085 4086 mutex_lock(&event->child_mutex); 4087 func(event); 4088 list_for_each_entry(child, &event->child_list, child_list) 4089 func(child); 4090 mutex_unlock(&event->child_mutex); 4091 } 4092 4093 static void perf_event_for_each(struct perf_event *event, 4094 void (*func)(struct perf_event *)) 4095 { 4096 struct perf_event_context *ctx = event->ctx; 4097 struct perf_event *sibling; 4098 4099 lockdep_assert_held(&ctx->mutex); 4100 4101 event = event->group_leader; 4102 4103 perf_event_for_each_child(event, func); 4104 list_for_each_entry(sibling, &event->sibling_list, group_entry) 4105 perf_event_for_each_child(sibling, func); 4106 } 4107 4108 static void __perf_event_period(struct perf_event *event, 4109 struct perf_cpu_context *cpuctx, 4110 struct perf_event_context *ctx, 4111 void *info) 4112 { 4113 u64 value = *((u64 *)info); 4114 bool active; 4115 4116 if (event->attr.freq) { 4117 event->attr.sample_freq = value; 4118 } else { 4119 event->attr.sample_period = value; 4120 event->hw.sample_period = value; 4121 } 4122 4123 active = (event->state == PERF_EVENT_STATE_ACTIVE); 4124 if (active) { 4125 perf_pmu_disable(ctx->pmu); 4126 event->pmu->stop(event, PERF_EF_UPDATE); 4127 } 4128 4129 local64_set(&event->hw.period_left, 0); 4130 4131 if (active) { 4132 event->pmu->start(event, PERF_EF_RELOAD); 4133 perf_pmu_enable(ctx->pmu); 4134 } 4135 } 4136 4137 static int perf_event_period(struct perf_event *event, u64 __user *arg) 4138 { 4139 u64 value; 4140 4141 if (!is_sampling_event(event)) 4142 return -EINVAL; 4143 4144 if (copy_from_user(&value, arg, sizeof(value))) 4145 return -EFAULT; 4146 4147 if (!value) 4148 return -EINVAL; 4149 4150 if (event->attr.freq && value > sysctl_perf_event_sample_rate) 4151 return -EINVAL; 4152 4153 event_function_call(event, __perf_event_period, &value); 4154 4155 return 0; 4156 } 4157 4158 static const struct file_operations perf_fops; 4159 4160 static inline int perf_fget_light(int fd, struct fd *p) 4161 { 4162 struct fd f = fdget(fd); 4163 if (!f.file) 4164 return -EBADF; 4165 4166 if (f.file->f_op != &perf_fops) { 4167 fdput(f); 4168 return -EBADF; 4169 } 4170 *p = f; 4171 return 0; 4172 } 4173 4174 static int perf_event_set_output(struct perf_event *event, 4175 struct perf_event *output_event); 4176 static int perf_event_set_filter(struct perf_event *event, void __user *arg); 4177 static int perf_event_set_bpf_prog(struct perf_event *event, u32 prog_fd); 4178 4179 static long _perf_ioctl(struct perf_event *event, unsigned int cmd, unsigned long arg) 4180 { 4181 void (*func)(struct perf_event *); 4182 u32 flags = arg; 4183 4184 switch (cmd) { 4185 case PERF_EVENT_IOC_ENABLE: 4186 func = _perf_event_enable; 4187 break; 4188 case PERF_EVENT_IOC_DISABLE: 4189 func = _perf_event_disable; 4190 break; 4191 case PERF_EVENT_IOC_RESET: 4192 func = _perf_event_reset; 4193 break; 4194 4195 case PERF_EVENT_IOC_REFRESH: 4196 return _perf_event_refresh(event, arg); 4197 4198 case PERF_EVENT_IOC_PERIOD: 4199 return perf_event_period(event, (u64 __user *)arg); 4200 4201 case PERF_EVENT_IOC_ID: 4202 { 4203 u64 id = primary_event_id(event); 4204 4205 if (copy_to_user((void __user *)arg, &id, sizeof(id))) 4206 return -EFAULT; 4207 return 0; 4208 } 4209 4210 case PERF_EVENT_IOC_SET_OUTPUT: 4211 { 4212 int ret; 4213 if (arg != -1) { 4214 struct perf_event *output_event; 4215 struct fd output; 4216 ret = perf_fget_light(arg, &output); 4217 if (ret) 4218 return ret; 4219 output_event = output.file->private_data; 4220 ret = perf_event_set_output(event, output_event); 4221 fdput(output); 4222 } else { 4223 ret = perf_event_set_output(event, NULL); 4224 } 4225 return ret; 4226 } 4227 4228 case PERF_EVENT_IOC_SET_FILTER: 4229 return perf_event_set_filter(event, (void __user *)arg); 4230 4231 case PERF_EVENT_IOC_SET_BPF: 4232 return perf_event_set_bpf_prog(event, arg); 4233 4234 default: 4235 return -ENOTTY; 4236 } 4237 4238 if (flags & PERF_IOC_FLAG_GROUP) 4239 perf_event_for_each(event, func); 4240 else 4241 perf_event_for_each_child(event, func); 4242 4243 return 0; 4244 } 4245 4246 static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg) 4247 { 4248 struct perf_event *event = file->private_data; 4249 struct perf_event_context *ctx; 4250 long ret; 4251 4252 ctx = perf_event_ctx_lock(event); 4253 ret = _perf_ioctl(event, cmd, arg); 4254 perf_event_ctx_unlock(event, ctx); 4255 4256 return ret; 4257 } 4258 4259 #ifdef CONFIG_COMPAT 4260 static long perf_compat_ioctl(struct file *file, unsigned int cmd, 4261 unsigned long arg) 4262 { 4263 switch (_IOC_NR(cmd)) { 4264 case _IOC_NR(PERF_EVENT_IOC_SET_FILTER): 4265 case _IOC_NR(PERF_EVENT_IOC_ID): 4266 /* Fix up pointer size (usually 4 -> 8 in 32-on-64-bit case */ 4267 if (_IOC_SIZE(cmd) == sizeof(compat_uptr_t)) { 4268 cmd &= ~IOCSIZE_MASK; 4269 cmd |= sizeof(void *) << IOCSIZE_SHIFT; 4270 } 4271 break; 4272 } 4273 return perf_ioctl(file, cmd, arg); 4274 } 4275 #else 4276 # define perf_compat_ioctl NULL 4277 #endif 4278 4279 int perf_event_task_enable(void) 4280 { 4281 struct perf_event_context *ctx; 4282 struct perf_event *event; 4283 4284 mutex_lock(¤t->perf_event_mutex); 4285 list_for_each_entry(event, ¤t->perf_event_list, owner_entry) { 4286 ctx = perf_event_ctx_lock(event); 4287 perf_event_for_each_child(event, _perf_event_enable); 4288 perf_event_ctx_unlock(event, ctx); 4289 } 4290 mutex_unlock(¤t->perf_event_mutex); 4291 4292 return 0; 4293 } 4294 4295 int perf_event_task_disable(void) 4296 { 4297 struct perf_event_context *ctx; 4298 struct perf_event *event; 4299 4300 mutex_lock(¤t->perf_event_mutex); 4301 list_for_each_entry(event, ¤t->perf_event_list, owner_entry) { 4302 ctx = perf_event_ctx_lock(event); 4303 perf_event_for_each_child(event, _perf_event_disable); 4304 perf_event_ctx_unlock(event, ctx); 4305 } 4306 mutex_unlock(¤t->perf_event_mutex); 4307 4308 return 0; 4309 } 4310 4311 static int perf_event_index(struct perf_event *event) 4312 { 4313 if (event->hw.state & PERF_HES_STOPPED) 4314 return 0; 4315 4316 if (event->state != PERF_EVENT_STATE_ACTIVE) 4317 return 0; 4318 4319 return event->pmu->event_idx(event); 4320 } 4321 4322 static void calc_timer_values(struct perf_event *event, 4323 u64 *now, 4324 u64 *enabled, 4325 u64 *running) 4326 { 4327 u64 ctx_time; 4328 4329 *now = perf_clock(); 4330 ctx_time = event->shadow_ctx_time + *now; 4331 *enabled = ctx_time - event->tstamp_enabled; 4332 *running = ctx_time - event->tstamp_running; 4333 } 4334 4335 static void perf_event_init_userpage(struct perf_event *event) 4336 { 4337 struct perf_event_mmap_page *userpg; 4338 struct ring_buffer *rb; 4339 4340 rcu_read_lock(); 4341 rb = rcu_dereference(event->rb); 4342 if (!rb) 4343 goto unlock; 4344 4345 userpg = rb->user_page; 4346 4347 /* Allow new userspace to detect that bit 0 is deprecated */ 4348 userpg->cap_bit0_is_deprecated = 1; 4349 userpg->size = offsetof(struct perf_event_mmap_page, __reserved); 4350 userpg->data_offset = PAGE_SIZE; 4351 userpg->data_size = perf_data_size(rb); 4352 4353 unlock: 4354 rcu_read_unlock(); 4355 } 4356 4357 void __weak arch_perf_update_userpage( 4358 struct perf_event *event, struct perf_event_mmap_page *userpg, u64 now) 4359 { 4360 } 4361 4362 /* 4363 * Callers need to ensure there can be no nesting of this function, otherwise 4364 * the seqlock logic goes bad. We can not serialize this because the arch 4365 * code calls this from NMI context. 4366 */ 4367 void perf_event_update_userpage(struct perf_event *event) 4368 { 4369 struct perf_event_mmap_page *userpg; 4370 struct ring_buffer *rb; 4371 u64 enabled, running, now; 4372 4373 rcu_read_lock(); 4374 rb = rcu_dereference(event->rb); 4375 if (!rb) 4376 goto unlock; 4377 4378 /* 4379 * compute total_time_enabled, total_time_running 4380 * based on snapshot values taken when the event 4381 * was last scheduled in. 4382 * 4383 * we cannot simply called update_context_time() 4384 * because of locking issue as we can be called in 4385 * NMI context 4386 */ 4387 calc_timer_values(event, &now, &enabled, &running); 4388 4389 userpg = rb->user_page; 4390 /* 4391 * Disable preemption so as to not let the corresponding user-space 4392 * spin too long if we get preempted. 4393 */ 4394 preempt_disable(); 4395 ++userpg->lock; 4396 barrier(); 4397 userpg->index = perf_event_index(event); 4398 userpg->offset = perf_event_count(event); 4399 if (userpg->index) 4400 userpg->offset -= local64_read(&event->hw.prev_count); 4401 4402 userpg->time_enabled = enabled + 4403 atomic64_read(&event->child_total_time_enabled); 4404 4405 userpg->time_running = running + 4406 atomic64_read(&event->child_total_time_running); 4407 4408 arch_perf_update_userpage(event, userpg, now); 4409 4410 barrier(); 4411 ++userpg->lock; 4412 preempt_enable(); 4413 unlock: 4414 rcu_read_unlock(); 4415 } 4416 4417 static int perf_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf) 4418 { 4419 struct perf_event *event = vma->vm_file->private_data; 4420 struct ring_buffer *rb; 4421 int ret = VM_FAULT_SIGBUS; 4422 4423 if (vmf->flags & FAULT_FLAG_MKWRITE) { 4424 if (vmf->pgoff == 0) 4425 ret = 0; 4426 return ret; 4427 } 4428 4429 rcu_read_lock(); 4430 rb = rcu_dereference(event->rb); 4431 if (!rb) 4432 goto unlock; 4433 4434 if (vmf->pgoff && (vmf->flags & FAULT_FLAG_WRITE)) 4435 goto unlock; 4436 4437 vmf->page = perf_mmap_to_page(rb, vmf->pgoff); 4438 if (!vmf->page) 4439 goto unlock; 4440 4441 get_page(vmf->page); 4442 vmf->page->mapping = vma->vm_file->f_mapping; 4443 vmf->page->index = vmf->pgoff; 4444 4445 ret = 0; 4446 unlock: 4447 rcu_read_unlock(); 4448 4449 return ret; 4450 } 4451 4452 static void ring_buffer_attach(struct perf_event *event, 4453 struct ring_buffer *rb) 4454 { 4455 struct ring_buffer *old_rb = NULL; 4456 unsigned long flags; 4457 4458 if (event->rb) { 4459 /* 4460 * Should be impossible, we set this when removing 4461 * event->rb_entry and wait/clear when adding event->rb_entry. 4462 */ 4463 WARN_ON_ONCE(event->rcu_pending); 4464 4465 old_rb = event->rb; 4466 spin_lock_irqsave(&old_rb->event_lock, flags); 4467 list_del_rcu(&event->rb_entry); 4468 spin_unlock_irqrestore(&old_rb->event_lock, flags); 4469 4470 event->rcu_batches = get_state_synchronize_rcu(); 4471 event->rcu_pending = 1; 4472 } 4473 4474 if (rb) { 4475 if (event->rcu_pending) { 4476 cond_synchronize_rcu(event->rcu_batches); 4477 event->rcu_pending = 0; 4478 } 4479 4480 spin_lock_irqsave(&rb->event_lock, flags); 4481 list_add_rcu(&event->rb_entry, &rb->event_list); 4482 spin_unlock_irqrestore(&rb->event_lock, flags); 4483 } 4484 4485 rcu_assign_pointer(event->rb, rb); 4486 4487 if (old_rb) { 4488 ring_buffer_put(old_rb); 4489 /* 4490 * Since we detached before setting the new rb, so that we 4491 * could attach the new rb, we could have missed a wakeup. 4492 * Provide it now. 4493 */ 4494 wake_up_all(&event->waitq); 4495 } 4496 } 4497 4498 static void ring_buffer_wakeup(struct perf_event *event) 4499 { 4500 struct ring_buffer *rb; 4501 4502 rcu_read_lock(); 4503 rb = rcu_dereference(event->rb); 4504 if (rb) { 4505 list_for_each_entry_rcu(event, &rb->event_list, rb_entry) 4506 wake_up_all(&event->waitq); 4507 } 4508 rcu_read_unlock(); 4509 } 4510 4511 struct ring_buffer *ring_buffer_get(struct perf_event *event) 4512 { 4513 struct ring_buffer *rb; 4514 4515 rcu_read_lock(); 4516 rb = rcu_dereference(event->rb); 4517 if (rb) { 4518 if (!atomic_inc_not_zero(&rb->refcount)) 4519 rb = NULL; 4520 } 4521 rcu_read_unlock(); 4522 4523 return rb; 4524 } 4525 4526 void ring_buffer_put(struct ring_buffer *rb) 4527 { 4528 if (!atomic_dec_and_test(&rb->refcount)) 4529 return; 4530 4531 WARN_ON_ONCE(!list_empty(&rb->event_list)); 4532 4533 call_rcu(&rb->rcu_head, rb_free_rcu); 4534 } 4535 4536 static void perf_mmap_open(struct vm_area_struct *vma) 4537 { 4538 struct perf_event *event = vma->vm_file->private_data; 4539 4540 atomic_inc(&event->mmap_count); 4541 atomic_inc(&event->rb->mmap_count); 4542 4543 if (vma->vm_pgoff) 4544 atomic_inc(&event->rb->aux_mmap_count); 4545 4546 if (event->pmu->event_mapped) 4547 event->pmu->event_mapped(event); 4548 } 4549 4550 /* 4551 * A buffer can be mmap()ed multiple times; either directly through the same 4552 * event, or through other events by use of perf_event_set_output(). 4553 * 4554 * In order to undo the VM accounting done by perf_mmap() we need to destroy 4555 * the buffer here, where we still have a VM context. This means we need 4556 * to detach all events redirecting to us. 4557 */ 4558 static void perf_mmap_close(struct vm_area_struct *vma) 4559 { 4560 struct perf_event *event = vma->vm_file->private_data; 4561 4562 struct ring_buffer *rb = ring_buffer_get(event); 4563 struct user_struct *mmap_user = rb->mmap_user; 4564 int mmap_locked = rb->mmap_locked; 4565 unsigned long size = perf_data_size(rb); 4566 4567 if (event->pmu->event_unmapped) 4568 event->pmu->event_unmapped(event); 4569 4570 /* 4571 * rb->aux_mmap_count will always drop before rb->mmap_count and 4572 * event->mmap_count, so it is ok to use event->mmap_mutex to 4573 * serialize with perf_mmap here. 4574 */ 4575 if (rb_has_aux(rb) && vma->vm_pgoff == rb->aux_pgoff && 4576 atomic_dec_and_mutex_lock(&rb->aux_mmap_count, &event->mmap_mutex)) { 4577 atomic_long_sub(rb->aux_nr_pages, &mmap_user->locked_vm); 4578 vma->vm_mm->pinned_vm -= rb->aux_mmap_locked; 4579 4580 rb_free_aux(rb); 4581 mutex_unlock(&event->mmap_mutex); 4582 } 4583 4584 atomic_dec(&rb->mmap_count); 4585 4586 if (!atomic_dec_and_mutex_lock(&event->mmap_count, &event->mmap_mutex)) 4587 goto out_put; 4588 4589 ring_buffer_attach(event, NULL); 4590 mutex_unlock(&event->mmap_mutex); 4591 4592 /* If there's still other mmap()s of this buffer, we're done. */ 4593 if (atomic_read(&rb->mmap_count)) 4594 goto out_put; 4595 4596 /* 4597 * No other mmap()s, detach from all other events that might redirect 4598 * into the now unreachable buffer. Somewhat complicated by the 4599 * fact that rb::event_lock otherwise nests inside mmap_mutex. 4600 */ 4601 again: 4602 rcu_read_lock(); 4603 list_for_each_entry_rcu(event, &rb->event_list, rb_entry) { 4604 if (!atomic_long_inc_not_zero(&event->refcount)) { 4605 /* 4606 * This event is en-route to free_event() which will 4607 * detach it and remove it from the list. 4608 */ 4609 continue; 4610 } 4611 rcu_read_unlock(); 4612 4613 mutex_lock(&event->mmap_mutex); 4614 /* 4615 * Check we didn't race with perf_event_set_output() which can 4616 * swizzle the rb from under us while we were waiting to 4617 * acquire mmap_mutex. 4618 * 4619 * If we find a different rb; ignore this event, a next 4620 * iteration will no longer find it on the list. We have to 4621 * still restart the iteration to make sure we're not now 4622 * iterating the wrong list. 4623 */ 4624 if (event->rb == rb) 4625 ring_buffer_attach(event, NULL); 4626 4627 mutex_unlock(&event->mmap_mutex); 4628 put_event(event); 4629 4630 /* 4631 * Restart the iteration; either we're on the wrong list or 4632 * destroyed its integrity by doing a deletion. 4633 */ 4634 goto again; 4635 } 4636 rcu_read_unlock(); 4637 4638 /* 4639 * It could be there's still a few 0-ref events on the list; they'll 4640 * get cleaned up by free_event() -- they'll also still have their 4641 * ref on the rb and will free it whenever they are done with it. 4642 * 4643 * Aside from that, this buffer is 'fully' detached and unmapped, 4644 * undo the VM accounting. 4645 */ 4646 4647 atomic_long_sub((size >> PAGE_SHIFT) + 1, &mmap_user->locked_vm); 4648 vma->vm_mm->pinned_vm -= mmap_locked; 4649 free_uid(mmap_user); 4650 4651 out_put: 4652 ring_buffer_put(rb); /* could be last */ 4653 } 4654 4655 static const struct vm_operations_struct perf_mmap_vmops = { 4656 .open = perf_mmap_open, 4657 .close = perf_mmap_close, /* non mergable */ 4658 .fault = perf_mmap_fault, 4659 .page_mkwrite = perf_mmap_fault, 4660 }; 4661 4662 static int perf_mmap(struct file *file, struct vm_area_struct *vma) 4663 { 4664 struct perf_event *event = file->private_data; 4665 unsigned long user_locked, user_lock_limit; 4666 struct user_struct *user = current_user(); 4667 unsigned long locked, lock_limit; 4668 struct ring_buffer *rb = NULL; 4669 unsigned long vma_size; 4670 unsigned long nr_pages; 4671 long user_extra = 0, extra = 0; 4672 int ret = 0, flags = 0; 4673 4674 /* 4675 * Don't allow mmap() of inherited per-task counters. This would 4676 * create a performance issue due to all children writing to the 4677 * same rb. 4678 */ 4679 if (event->cpu == -1 && event->attr.inherit) 4680 return -EINVAL; 4681 4682 if (!(vma->vm_flags & VM_SHARED)) 4683 return -EINVAL; 4684 4685 vma_size = vma->vm_end - vma->vm_start; 4686 4687 if (vma->vm_pgoff == 0) { 4688 nr_pages = (vma_size / PAGE_SIZE) - 1; 4689 } else { 4690 /* 4691 * AUX area mapping: if rb->aux_nr_pages != 0, it's already 4692 * mapped, all subsequent mappings should have the same size 4693 * and offset. Must be above the normal perf buffer. 4694 */ 4695 u64 aux_offset, aux_size; 4696 4697 if (!event->rb) 4698 return -EINVAL; 4699 4700 nr_pages = vma_size / PAGE_SIZE; 4701 4702 mutex_lock(&event->mmap_mutex); 4703 ret = -EINVAL; 4704 4705 rb = event->rb; 4706 if (!rb) 4707 goto aux_unlock; 4708 4709 aux_offset = ACCESS_ONCE(rb->user_page->aux_offset); 4710 aux_size = ACCESS_ONCE(rb->user_page->aux_size); 4711 4712 if (aux_offset < perf_data_size(rb) + PAGE_SIZE) 4713 goto aux_unlock; 4714 4715 if (aux_offset != vma->vm_pgoff << PAGE_SHIFT) 4716 goto aux_unlock; 4717 4718 /* already mapped with a different offset */ 4719 if (rb_has_aux(rb) && rb->aux_pgoff != vma->vm_pgoff) 4720 goto aux_unlock; 4721 4722 if (aux_size != vma_size || aux_size != nr_pages * PAGE_SIZE) 4723 goto aux_unlock; 4724 4725 /* already mapped with a different size */ 4726 if (rb_has_aux(rb) && rb->aux_nr_pages != nr_pages) 4727 goto aux_unlock; 4728 4729 if (!is_power_of_2(nr_pages)) 4730 goto aux_unlock; 4731 4732 if (!atomic_inc_not_zero(&rb->mmap_count)) 4733 goto aux_unlock; 4734 4735 if (rb_has_aux(rb)) { 4736 atomic_inc(&rb->aux_mmap_count); 4737 ret = 0; 4738 goto unlock; 4739 } 4740 4741 atomic_set(&rb->aux_mmap_count, 1); 4742 user_extra = nr_pages; 4743 4744 goto accounting; 4745 } 4746 4747 /* 4748 * If we have rb pages ensure they're a power-of-two number, so we 4749 * can do bitmasks instead of modulo. 4750 */ 4751 if (nr_pages != 0 && !is_power_of_2(nr_pages)) 4752 return -EINVAL; 4753 4754 if (vma_size != PAGE_SIZE * (1 + nr_pages)) 4755 return -EINVAL; 4756 4757 WARN_ON_ONCE(event->ctx->parent_ctx); 4758 again: 4759 mutex_lock(&event->mmap_mutex); 4760 if (event->rb) { 4761 if (event->rb->nr_pages != nr_pages) { 4762 ret = -EINVAL; 4763 goto unlock; 4764 } 4765 4766 if (!atomic_inc_not_zero(&event->rb->mmap_count)) { 4767 /* 4768 * Raced against perf_mmap_close() through 4769 * perf_event_set_output(). Try again, hope for better 4770 * luck. 4771 */ 4772 mutex_unlock(&event->mmap_mutex); 4773 goto again; 4774 } 4775 4776 goto unlock; 4777 } 4778 4779 user_extra = nr_pages + 1; 4780 4781 accounting: 4782 user_lock_limit = sysctl_perf_event_mlock >> (PAGE_SHIFT - 10); 4783 4784 /* 4785 * Increase the limit linearly with more CPUs: 4786 */ 4787 user_lock_limit *= num_online_cpus(); 4788 4789 user_locked = atomic_long_read(&user->locked_vm) + user_extra; 4790 4791 if (user_locked > user_lock_limit) 4792 extra = user_locked - user_lock_limit; 4793 4794 lock_limit = rlimit(RLIMIT_MEMLOCK); 4795 lock_limit >>= PAGE_SHIFT; 4796 locked = vma->vm_mm->pinned_vm + extra; 4797 4798 if ((locked > lock_limit) && perf_paranoid_tracepoint_raw() && 4799 !capable(CAP_IPC_LOCK)) { 4800 ret = -EPERM; 4801 goto unlock; 4802 } 4803 4804 WARN_ON(!rb && event->rb); 4805 4806 if (vma->vm_flags & VM_WRITE) 4807 flags |= RING_BUFFER_WRITABLE; 4808 4809 if (!rb) { 4810 rb = rb_alloc(nr_pages, 4811 event->attr.watermark ? event->attr.wakeup_watermark : 0, 4812 event->cpu, flags); 4813 4814 if (!rb) { 4815 ret = -ENOMEM; 4816 goto unlock; 4817 } 4818 4819 atomic_set(&rb->mmap_count, 1); 4820 rb->mmap_user = get_current_user(); 4821 rb->mmap_locked = extra; 4822 4823 ring_buffer_attach(event, rb); 4824 4825 perf_event_init_userpage(event); 4826 perf_event_update_userpage(event); 4827 } else { 4828 ret = rb_alloc_aux(rb, event, vma->vm_pgoff, nr_pages, 4829 event->attr.aux_watermark, flags); 4830 if (!ret) 4831 rb->aux_mmap_locked = extra; 4832 } 4833 4834 unlock: 4835 if (!ret) { 4836 atomic_long_add(user_extra, &user->locked_vm); 4837 vma->vm_mm->pinned_vm += extra; 4838 4839 atomic_inc(&event->mmap_count); 4840 } else if (rb) { 4841 atomic_dec(&rb->mmap_count); 4842 } 4843 aux_unlock: 4844 mutex_unlock(&event->mmap_mutex); 4845 4846 /* 4847 * Since pinned accounting is per vm we cannot allow fork() to copy our 4848 * vma. 4849 */ 4850 vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_DONTDUMP; 4851 vma->vm_ops = &perf_mmap_vmops; 4852 4853 if (event->pmu->event_mapped) 4854 event->pmu->event_mapped(event); 4855 4856 return ret; 4857 } 4858 4859 static int perf_fasync(int fd, struct file *filp, int on) 4860 { 4861 struct inode *inode = file_inode(filp); 4862 struct perf_event *event = filp->private_data; 4863 int retval; 4864 4865 inode_lock(inode); 4866 retval = fasync_helper(fd, filp, on, &event->fasync); 4867 inode_unlock(inode); 4868 4869 if (retval < 0) 4870 return retval; 4871 4872 return 0; 4873 } 4874 4875 static const struct file_operations perf_fops = { 4876 .llseek = no_llseek, 4877 .release = perf_release, 4878 .read = perf_read, 4879 .poll = perf_poll, 4880 .unlocked_ioctl = perf_ioctl, 4881 .compat_ioctl = perf_compat_ioctl, 4882 .mmap = perf_mmap, 4883 .fasync = perf_fasync, 4884 }; 4885 4886 /* 4887 * Perf event wakeup 4888 * 4889 * If there's data, ensure we set the poll() state and publish everything 4890 * to user-space before waking everybody up. 4891 */ 4892 4893 static inline struct fasync_struct **perf_event_fasync(struct perf_event *event) 4894 { 4895 /* only the parent has fasync state */ 4896 if (event->parent) 4897 event = event->parent; 4898 return &event->fasync; 4899 } 4900 4901 void perf_event_wakeup(struct perf_event *event) 4902 { 4903 ring_buffer_wakeup(event); 4904 4905 if (event->pending_kill) { 4906 kill_fasync(perf_event_fasync(event), SIGIO, event->pending_kill); 4907 event->pending_kill = 0; 4908 } 4909 } 4910 4911 static void perf_pending_event(struct irq_work *entry) 4912 { 4913 struct perf_event *event = container_of(entry, 4914 struct perf_event, pending); 4915 int rctx; 4916 4917 rctx = perf_swevent_get_recursion_context(); 4918 /* 4919 * If we 'fail' here, that's OK, it means recursion is already disabled 4920 * and we won't recurse 'further'. 4921 */ 4922 4923 if (event->pending_disable) { 4924 event->pending_disable = 0; 4925 perf_event_disable_local(event); 4926 } 4927 4928 if (event->pending_wakeup) { 4929 event->pending_wakeup = 0; 4930 perf_event_wakeup(event); 4931 } 4932 4933 if (rctx >= 0) 4934 perf_swevent_put_recursion_context(rctx); 4935 } 4936 4937 /* 4938 * We assume there is only KVM supporting the callbacks. 4939 * Later on, we might change it to a list if there is 4940 * another virtualization implementation supporting the callbacks. 4941 */ 4942 struct perf_guest_info_callbacks *perf_guest_cbs; 4943 4944 int perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *cbs) 4945 { 4946 perf_guest_cbs = cbs; 4947 return 0; 4948 } 4949 EXPORT_SYMBOL_GPL(perf_register_guest_info_callbacks); 4950 4951 int perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *cbs) 4952 { 4953 perf_guest_cbs = NULL; 4954 return 0; 4955 } 4956 EXPORT_SYMBOL_GPL(perf_unregister_guest_info_callbacks); 4957 4958 static void 4959 perf_output_sample_regs(struct perf_output_handle *handle, 4960 struct pt_regs *regs, u64 mask) 4961 { 4962 int bit; 4963 4964 for_each_set_bit(bit, (const unsigned long *) &mask, 4965 sizeof(mask) * BITS_PER_BYTE) { 4966 u64 val; 4967 4968 val = perf_reg_value(regs, bit); 4969 perf_output_put(handle, val); 4970 } 4971 } 4972 4973 static void perf_sample_regs_user(struct perf_regs *regs_user, 4974 struct pt_regs *regs, 4975 struct pt_regs *regs_user_copy) 4976 { 4977 if (user_mode(regs)) { 4978 regs_user->abi = perf_reg_abi(current); 4979 regs_user->regs = regs; 4980 } else if (current->mm) { 4981 perf_get_regs_user(regs_user, regs, regs_user_copy); 4982 } else { 4983 regs_user->abi = PERF_SAMPLE_REGS_ABI_NONE; 4984 regs_user->regs = NULL; 4985 } 4986 } 4987 4988 static void perf_sample_regs_intr(struct perf_regs *regs_intr, 4989 struct pt_regs *regs) 4990 { 4991 regs_intr->regs = regs; 4992 regs_intr->abi = perf_reg_abi(current); 4993 } 4994 4995 4996 /* 4997 * Get remaining task size from user stack pointer. 4998 * 4999 * It'd be better to take stack vma map and limit this more 5000 * precisly, but there's no way to get it safely under interrupt, 5001 * so using TASK_SIZE as limit. 5002 */ 5003 static u64 perf_ustack_task_size(struct pt_regs *regs) 5004 { 5005 unsigned long addr = perf_user_stack_pointer(regs); 5006 5007 if (!addr || addr >= TASK_SIZE) 5008 return 0; 5009 5010 return TASK_SIZE - addr; 5011 } 5012 5013 static u16 5014 perf_sample_ustack_size(u16 stack_size, u16 header_size, 5015 struct pt_regs *regs) 5016 { 5017 u64 task_size; 5018 5019 /* No regs, no stack pointer, no dump. */ 5020 if (!regs) 5021 return 0; 5022 5023 /* 5024 * Check if we fit in with the requested stack size into the: 5025 * - TASK_SIZE 5026 * If we don't, we limit the size to the TASK_SIZE. 5027 * 5028 * - remaining sample size 5029 * If we don't, we customize the stack size to 5030 * fit in to the remaining sample size. 5031 */ 5032 5033 task_size = min((u64) USHRT_MAX, perf_ustack_task_size(regs)); 5034 stack_size = min(stack_size, (u16) task_size); 5035 5036 /* Current header size plus static size and dynamic size. */ 5037 header_size += 2 * sizeof(u64); 5038 5039 /* Do we fit in with the current stack dump size? */ 5040 if ((u16) (header_size + stack_size) < header_size) { 5041 /* 5042 * If we overflow the maximum size for the sample, 5043 * we customize the stack dump size to fit in. 5044 */ 5045 stack_size = USHRT_MAX - header_size - sizeof(u64); 5046 stack_size = round_up(stack_size, sizeof(u64)); 5047 } 5048 5049 return stack_size; 5050 } 5051 5052 static void 5053 perf_output_sample_ustack(struct perf_output_handle *handle, u64 dump_size, 5054 struct pt_regs *regs) 5055 { 5056 /* Case of a kernel thread, nothing to dump */ 5057 if (!regs) { 5058 u64 size = 0; 5059 perf_output_put(handle, size); 5060 } else { 5061 unsigned long sp; 5062 unsigned int rem; 5063 u64 dyn_size; 5064 5065 /* 5066 * We dump: 5067 * static size 5068 * - the size requested by user or the best one we can fit 5069 * in to the sample max size 5070 * data 5071 * - user stack dump data 5072 * dynamic size 5073 * - the actual dumped size 5074 */ 5075 5076 /* Static size. */ 5077 perf_output_put(handle, dump_size); 5078 5079 /* Data. */ 5080 sp = perf_user_stack_pointer(regs); 5081 rem = __output_copy_user(handle, (void *) sp, dump_size); 5082 dyn_size = dump_size - rem; 5083 5084 perf_output_skip(handle, rem); 5085 5086 /* Dynamic size. */ 5087 perf_output_put(handle, dyn_size); 5088 } 5089 } 5090 5091 static void __perf_event_header__init_id(struct perf_event_header *header, 5092 struct perf_sample_data *data, 5093 struct perf_event *event) 5094 { 5095 u64 sample_type = event->attr.sample_type; 5096 5097 data->type = sample_type; 5098 header->size += event->id_header_size; 5099 5100 if (sample_type & PERF_SAMPLE_TID) { 5101 /* namespace issues */ 5102 data->tid_entry.pid = perf_event_pid(event, current); 5103 data->tid_entry.tid = perf_event_tid(event, current); 5104 } 5105 5106 if (sample_type & PERF_SAMPLE_TIME) 5107 data->time = perf_event_clock(event); 5108 5109 if (sample_type & (PERF_SAMPLE_ID | PERF_SAMPLE_IDENTIFIER)) 5110 data->id = primary_event_id(event); 5111 5112 if (sample_type & PERF_SAMPLE_STREAM_ID) 5113 data->stream_id = event->id; 5114 5115 if (sample_type & PERF_SAMPLE_CPU) { 5116 data->cpu_entry.cpu = raw_smp_processor_id(); 5117 data->cpu_entry.reserved = 0; 5118 } 5119 } 5120 5121 void perf_event_header__init_id(struct perf_event_header *header, 5122 struct perf_sample_data *data, 5123 struct perf_event *event) 5124 { 5125 if (event->attr.sample_id_all) 5126 __perf_event_header__init_id(header, data, event); 5127 } 5128 5129 static void __perf_event__output_id_sample(struct perf_output_handle *handle, 5130 struct perf_sample_data *data) 5131 { 5132 u64 sample_type = data->type; 5133 5134 if (sample_type & PERF_SAMPLE_TID) 5135 perf_output_put(handle, data->tid_entry); 5136 5137 if (sample_type & PERF_SAMPLE_TIME) 5138 perf_output_put(handle, data->time); 5139 5140 if (sample_type & PERF_SAMPLE_ID) 5141 perf_output_put(handle, data->id); 5142 5143 if (sample_type & PERF_SAMPLE_STREAM_ID) 5144 perf_output_put(handle, data->stream_id); 5145 5146 if (sample_type & PERF_SAMPLE_CPU) 5147 perf_output_put(handle, data->cpu_entry); 5148 5149 if (sample_type & PERF_SAMPLE_IDENTIFIER) 5150 perf_output_put(handle, data->id); 5151 } 5152 5153 void perf_event__output_id_sample(struct perf_event *event, 5154 struct perf_output_handle *handle, 5155 struct perf_sample_data *sample) 5156 { 5157 if (event->attr.sample_id_all) 5158 __perf_event__output_id_sample(handle, sample); 5159 } 5160 5161 static void perf_output_read_one(struct perf_output_handle *handle, 5162 struct perf_event *event, 5163 u64 enabled, u64 running) 5164 { 5165 u64 read_format = event->attr.read_format; 5166 u64 values[4]; 5167 int n = 0; 5168 5169 values[n++] = perf_event_count(event); 5170 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) { 5171 values[n++] = enabled + 5172 atomic64_read(&event->child_total_time_enabled); 5173 } 5174 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) { 5175 values[n++] = running + 5176 atomic64_read(&event->child_total_time_running); 5177 } 5178 if (read_format & PERF_FORMAT_ID) 5179 values[n++] = primary_event_id(event); 5180 5181 __output_copy(handle, values, n * sizeof(u64)); 5182 } 5183 5184 /* 5185 * XXX PERF_FORMAT_GROUP vs inherited events seems difficult. 5186 */ 5187 static void perf_output_read_group(struct perf_output_handle *handle, 5188 struct perf_event *event, 5189 u64 enabled, u64 running) 5190 { 5191 struct perf_event *leader = event->group_leader, *sub; 5192 u64 read_format = event->attr.read_format; 5193 u64 values[5]; 5194 int n = 0; 5195 5196 values[n++] = 1 + leader->nr_siblings; 5197 5198 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) 5199 values[n++] = enabled; 5200 5201 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) 5202 values[n++] = running; 5203 5204 if (leader != event) 5205 leader->pmu->read(leader); 5206 5207 values[n++] = perf_event_count(leader); 5208 if (read_format & PERF_FORMAT_ID) 5209 values[n++] = primary_event_id(leader); 5210 5211 __output_copy(handle, values, n * sizeof(u64)); 5212 5213 list_for_each_entry(sub, &leader->sibling_list, group_entry) { 5214 n = 0; 5215 5216 if ((sub != event) && 5217 (sub->state == PERF_EVENT_STATE_ACTIVE)) 5218 sub->pmu->read(sub); 5219 5220 values[n++] = perf_event_count(sub); 5221 if (read_format & PERF_FORMAT_ID) 5222 values[n++] = primary_event_id(sub); 5223 5224 __output_copy(handle, values, n * sizeof(u64)); 5225 } 5226 } 5227 5228 #define PERF_FORMAT_TOTAL_TIMES (PERF_FORMAT_TOTAL_TIME_ENABLED|\ 5229 PERF_FORMAT_TOTAL_TIME_RUNNING) 5230 5231 static void perf_output_read(struct perf_output_handle *handle, 5232 struct perf_event *event) 5233 { 5234 u64 enabled = 0, running = 0, now; 5235 u64 read_format = event->attr.read_format; 5236 5237 /* 5238 * compute total_time_enabled, total_time_running 5239 * based on snapshot values taken when the event 5240 * was last scheduled in. 5241 * 5242 * we cannot simply called update_context_time() 5243 * because of locking issue as we are called in 5244 * NMI context 5245 */ 5246 if (read_format & PERF_FORMAT_TOTAL_TIMES) 5247 calc_timer_values(event, &now, &enabled, &running); 5248 5249 if (event->attr.read_format & PERF_FORMAT_GROUP) 5250 perf_output_read_group(handle, event, enabled, running); 5251 else 5252 perf_output_read_one(handle, event, enabled, running); 5253 } 5254 5255 void perf_output_sample(struct perf_output_handle *handle, 5256 struct perf_event_header *header, 5257 struct perf_sample_data *data, 5258 struct perf_event *event) 5259 { 5260 u64 sample_type = data->type; 5261 5262 perf_output_put(handle, *header); 5263 5264 if (sample_type & PERF_SAMPLE_IDENTIFIER) 5265 perf_output_put(handle, data->id); 5266 5267 if (sample_type & PERF_SAMPLE_IP) 5268 perf_output_put(handle, data->ip); 5269 5270 if (sample_type & PERF_SAMPLE_TID) 5271 perf_output_put(handle, data->tid_entry); 5272 5273 if (sample_type & PERF_SAMPLE_TIME) 5274 perf_output_put(handle, data->time); 5275 5276 if (sample_type & PERF_SAMPLE_ADDR) 5277 perf_output_put(handle, data->addr); 5278 5279 if (sample_type & PERF_SAMPLE_ID) 5280 perf_output_put(handle, data->id); 5281 5282 if (sample_type & PERF_SAMPLE_STREAM_ID) 5283 perf_output_put(handle, data->stream_id); 5284 5285 if (sample_type & PERF_SAMPLE_CPU) 5286 perf_output_put(handle, data->cpu_entry); 5287 5288 if (sample_type & PERF_SAMPLE_PERIOD) 5289 perf_output_put(handle, data->period); 5290 5291 if (sample_type & PERF_SAMPLE_READ) 5292 perf_output_read(handle, event); 5293 5294 if (sample_type & PERF_SAMPLE_CALLCHAIN) { 5295 if (data->callchain) { 5296 int size = 1; 5297 5298 if (data->callchain) 5299 size += data->callchain->nr; 5300 5301 size *= sizeof(u64); 5302 5303 __output_copy(handle, data->callchain, size); 5304 } else { 5305 u64 nr = 0; 5306 perf_output_put(handle, nr); 5307 } 5308 } 5309 5310 if (sample_type & PERF_SAMPLE_RAW) { 5311 if (data->raw) { 5312 u32 raw_size = data->raw->size; 5313 u32 real_size = round_up(raw_size + sizeof(u32), 5314 sizeof(u64)) - sizeof(u32); 5315 u64 zero = 0; 5316 5317 perf_output_put(handle, real_size); 5318 __output_copy(handle, data->raw->data, raw_size); 5319 if (real_size - raw_size) 5320 __output_copy(handle, &zero, real_size - raw_size); 5321 } else { 5322 struct { 5323 u32 size; 5324 u32 data; 5325 } raw = { 5326 .size = sizeof(u32), 5327 .data = 0, 5328 }; 5329 perf_output_put(handle, raw); 5330 } 5331 } 5332 5333 if (sample_type & PERF_SAMPLE_BRANCH_STACK) { 5334 if (data->br_stack) { 5335 size_t size; 5336 5337 size = data->br_stack->nr 5338 * sizeof(struct perf_branch_entry); 5339 5340 perf_output_put(handle, data->br_stack->nr); 5341 perf_output_copy(handle, data->br_stack->entries, size); 5342 } else { 5343 /* 5344 * we always store at least the value of nr 5345 */ 5346 u64 nr = 0; 5347 perf_output_put(handle, nr); 5348 } 5349 } 5350 5351 if (sample_type & PERF_SAMPLE_REGS_USER) { 5352 u64 abi = data->regs_user.abi; 5353 5354 /* 5355 * If there are no regs to dump, notice it through 5356 * first u64 being zero (PERF_SAMPLE_REGS_ABI_NONE). 5357 */ 5358 perf_output_put(handle, abi); 5359 5360 if (abi) { 5361 u64 mask = event->attr.sample_regs_user; 5362 perf_output_sample_regs(handle, 5363 data->regs_user.regs, 5364 mask); 5365 } 5366 } 5367 5368 if (sample_type & PERF_SAMPLE_STACK_USER) { 5369 perf_output_sample_ustack(handle, 5370 data->stack_user_size, 5371 data->regs_user.regs); 5372 } 5373 5374 if (sample_type & PERF_SAMPLE_WEIGHT) 5375 perf_output_put(handle, data->weight); 5376 5377 if (sample_type & PERF_SAMPLE_DATA_SRC) 5378 perf_output_put(handle, data->data_src.val); 5379 5380 if (sample_type & PERF_SAMPLE_TRANSACTION) 5381 perf_output_put(handle, data->txn); 5382 5383 if (sample_type & PERF_SAMPLE_REGS_INTR) { 5384 u64 abi = data->regs_intr.abi; 5385 /* 5386 * If there are no regs to dump, notice it through 5387 * first u64 being zero (PERF_SAMPLE_REGS_ABI_NONE). 5388 */ 5389 perf_output_put(handle, abi); 5390 5391 if (abi) { 5392 u64 mask = event->attr.sample_regs_intr; 5393 5394 perf_output_sample_regs(handle, 5395 data->regs_intr.regs, 5396 mask); 5397 } 5398 } 5399 5400 if (!event->attr.watermark) { 5401 int wakeup_events = event->attr.wakeup_events; 5402 5403 if (wakeup_events) { 5404 struct ring_buffer *rb = handle->rb; 5405 int events = local_inc_return(&rb->events); 5406 5407 if (events >= wakeup_events) { 5408 local_sub(wakeup_events, &rb->events); 5409 local_inc(&rb->wakeup); 5410 } 5411 } 5412 } 5413 } 5414 5415 void perf_prepare_sample(struct perf_event_header *header, 5416 struct perf_sample_data *data, 5417 struct perf_event *event, 5418 struct pt_regs *regs) 5419 { 5420 u64 sample_type = event->attr.sample_type; 5421 5422 header->type = PERF_RECORD_SAMPLE; 5423 header->size = sizeof(*header) + event->header_size; 5424 5425 header->misc = 0; 5426 header->misc |= perf_misc_flags(regs); 5427 5428 __perf_event_header__init_id(header, data, event); 5429 5430 if (sample_type & PERF_SAMPLE_IP) 5431 data->ip = perf_instruction_pointer(regs); 5432 5433 if (sample_type & PERF_SAMPLE_CALLCHAIN) { 5434 int size = 1; 5435 5436 data->callchain = perf_callchain(event, regs); 5437 5438 if (data->callchain) 5439 size += data->callchain->nr; 5440 5441 header->size += size * sizeof(u64); 5442 } 5443 5444 if (sample_type & PERF_SAMPLE_RAW) { 5445 int size = sizeof(u32); 5446 5447 if (data->raw) 5448 size += data->raw->size; 5449 else 5450 size += sizeof(u32); 5451 5452 header->size += round_up(size, sizeof(u64)); 5453 } 5454 5455 if (sample_type & PERF_SAMPLE_BRANCH_STACK) { 5456 int size = sizeof(u64); /* nr */ 5457 if (data->br_stack) { 5458 size += data->br_stack->nr 5459 * sizeof(struct perf_branch_entry); 5460 } 5461 header->size += size; 5462 } 5463 5464 if (sample_type & (PERF_SAMPLE_REGS_USER | PERF_SAMPLE_STACK_USER)) 5465 perf_sample_regs_user(&data->regs_user, regs, 5466 &data->regs_user_copy); 5467 5468 if (sample_type & PERF_SAMPLE_REGS_USER) { 5469 /* regs dump ABI info */ 5470 int size = sizeof(u64); 5471 5472 if (data->regs_user.regs) { 5473 u64 mask = event->attr.sample_regs_user; 5474 size += hweight64(mask) * sizeof(u64); 5475 } 5476 5477 header->size += size; 5478 } 5479 5480 if (sample_type & PERF_SAMPLE_STACK_USER) { 5481 /* 5482 * Either we need PERF_SAMPLE_STACK_USER bit to be allways 5483 * processed as the last one or have additional check added 5484 * in case new sample type is added, because we could eat 5485 * up the rest of the sample size. 5486 */ 5487 u16 stack_size = event->attr.sample_stack_user; 5488 u16 size = sizeof(u64); 5489 5490 stack_size = perf_sample_ustack_size(stack_size, header->size, 5491 data->regs_user.regs); 5492 5493 /* 5494 * If there is something to dump, add space for the dump 5495 * itself and for the field that tells the dynamic size, 5496 * which is how many have been actually dumped. 5497 */ 5498 if (stack_size) 5499 size += sizeof(u64) + stack_size; 5500 5501 data->stack_user_size = stack_size; 5502 header->size += size; 5503 } 5504 5505 if (sample_type & PERF_SAMPLE_REGS_INTR) { 5506 /* regs dump ABI info */ 5507 int size = sizeof(u64); 5508 5509 perf_sample_regs_intr(&data->regs_intr, regs); 5510 5511 if (data->regs_intr.regs) { 5512 u64 mask = event->attr.sample_regs_intr; 5513 5514 size += hweight64(mask) * sizeof(u64); 5515 } 5516 5517 header->size += size; 5518 } 5519 } 5520 5521 void perf_event_output(struct perf_event *event, 5522 struct perf_sample_data *data, 5523 struct pt_regs *regs) 5524 { 5525 struct perf_output_handle handle; 5526 struct perf_event_header header; 5527 5528 /* protect the callchain buffers */ 5529 rcu_read_lock(); 5530 5531 perf_prepare_sample(&header, data, event, regs); 5532 5533 if (perf_output_begin(&handle, event, header.size)) 5534 goto exit; 5535 5536 perf_output_sample(&handle, &header, data, event); 5537 5538 perf_output_end(&handle); 5539 5540 exit: 5541 rcu_read_unlock(); 5542 } 5543 5544 /* 5545 * read event_id 5546 */ 5547 5548 struct perf_read_event { 5549 struct perf_event_header header; 5550 5551 u32 pid; 5552 u32 tid; 5553 }; 5554 5555 static void 5556 perf_event_read_event(struct perf_event *event, 5557 struct task_struct *task) 5558 { 5559 struct perf_output_handle handle; 5560 struct perf_sample_data sample; 5561 struct perf_read_event read_event = { 5562 .header = { 5563 .type = PERF_RECORD_READ, 5564 .misc = 0, 5565 .size = sizeof(read_event) + event->read_size, 5566 }, 5567 .pid = perf_event_pid(event, task), 5568 .tid = perf_event_tid(event, task), 5569 }; 5570 int ret; 5571 5572 perf_event_header__init_id(&read_event.header, &sample, event); 5573 ret = perf_output_begin(&handle, event, read_event.header.size); 5574 if (ret) 5575 return; 5576 5577 perf_output_put(&handle, read_event); 5578 perf_output_read(&handle, event); 5579 perf_event__output_id_sample(event, &handle, &sample); 5580 5581 perf_output_end(&handle); 5582 } 5583 5584 typedef void (perf_event_aux_output_cb)(struct perf_event *event, void *data); 5585 5586 static void 5587 perf_event_aux_ctx(struct perf_event_context *ctx, 5588 perf_event_aux_output_cb output, 5589 void *data) 5590 { 5591 struct perf_event *event; 5592 5593 list_for_each_entry_rcu(event, &ctx->event_list, event_entry) { 5594 if (event->state < PERF_EVENT_STATE_INACTIVE) 5595 continue; 5596 if (!event_filter_match(event)) 5597 continue; 5598 output(event, data); 5599 } 5600 } 5601 5602 static void 5603 perf_event_aux_task_ctx(perf_event_aux_output_cb output, void *data, 5604 struct perf_event_context *task_ctx) 5605 { 5606 rcu_read_lock(); 5607 preempt_disable(); 5608 perf_event_aux_ctx(task_ctx, output, data); 5609 preempt_enable(); 5610 rcu_read_unlock(); 5611 } 5612 5613 static void 5614 perf_event_aux(perf_event_aux_output_cb output, void *data, 5615 struct perf_event_context *task_ctx) 5616 { 5617 struct perf_cpu_context *cpuctx; 5618 struct perf_event_context *ctx; 5619 struct pmu *pmu; 5620 int ctxn; 5621 5622 /* 5623 * If we have task_ctx != NULL we only notify 5624 * the task context itself. The task_ctx is set 5625 * only for EXIT events before releasing task 5626 * context. 5627 */ 5628 if (task_ctx) { 5629 perf_event_aux_task_ctx(output, data, task_ctx); 5630 return; 5631 } 5632 5633 rcu_read_lock(); 5634 list_for_each_entry_rcu(pmu, &pmus, entry) { 5635 cpuctx = get_cpu_ptr(pmu->pmu_cpu_context); 5636 if (cpuctx->unique_pmu != pmu) 5637 goto next; 5638 perf_event_aux_ctx(&cpuctx->ctx, output, data); 5639 ctxn = pmu->task_ctx_nr; 5640 if (ctxn < 0) 5641 goto next; 5642 ctx = rcu_dereference(current->perf_event_ctxp[ctxn]); 5643 if (ctx) 5644 perf_event_aux_ctx(ctx, output, data); 5645 next: 5646 put_cpu_ptr(pmu->pmu_cpu_context); 5647 } 5648 rcu_read_unlock(); 5649 } 5650 5651 /* 5652 * task tracking -- fork/exit 5653 * 5654 * enabled by: attr.comm | attr.mmap | attr.mmap2 | attr.mmap_data | attr.task 5655 */ 5656 5657 struct perf_task_event { 5658 struct task_struct *task; 5659 struct perf_event_context *task_ctx; 5660 5661 struct { 5662 struct perf_event_header header; 5663 5664 u32 pid; 5665 u32 ppid; 5666 u32 tid; 5667 u32 ptid; 5668 u64 time; 5669 } event_id; 5670 }; 5671 5672 static int perf_event_task_match(struct perf_event *event) 5673 { 5674 return event->attr.comm || event->attr.mmap || 5675 event->attr.mmap2 || event->attr.mmap_data || 5676 event->attr.task; 5677 } 5678 5679 static void perf_event_task_output(struct perf_event *event, 5680 void *data) 5681 { 5682 struct perf_task_event *task_event = data; 5683 struct perf_output_handle handle; 5684 struct perf_sample_data sample; 5685 struct task_struct *task = task_event->task; 5686 int ret, size = task_event->event_id.header.size; 5687 5688 if (!perf_event_task_match(event)) 5689 return; 5690 5691 perf_event_header__init_id(&task_event->event_id.header, &sample, event); 5692 5693 ret = perf_output_begin(&handle, event, 5694 task_event->event_id.header.size); 5695 if (ret) 5696 goto out; 5697 5698 task_event->event_id.pid = perf_event_pid(event, task); 5699 task_event->event_id.ppid = perf_event_pid(event, current); 5700 5701 task_event->event_id.tid = perf_event_tid(event, task); 5702 task_event->event_id.ptid = perf_event_tid(event, current); 5703 5704 task_event->event_id.time = perf_event_clock(event); 5705 5706 perf_output_put(&handle, task_event->event_id); 5707 5708 perf_event__output_id_sample(event, &handle, &sample); 5709 5710 perf_output_end(&handle); 5711 out: 5712 task_event->event_id.header.size = size; 5713 } 5714 5715 static void perf_event_task(struct task_struct *task, 5716 struct perf_event_context *task_ctx, 5717 int new) 5718 { 5719 struct perf_task_event task_event; 5720 5721 if (!atomic_read(&nr_comm_events) && 5722 !atomic_read(&nr_mmap_events) && 5723 !atomic_read(&nr_task_events)) 5724 return; 5725 5726 task_event = (struct perf_task_event){ 5727 .task = task, 5728 .task_ctx = task_ctx, 5729 .event_id = { 5730 .header = { 5731 .type = new ? PERF_RECORD_FORK : PERF_RECORD_EXIT, 5732 .misc = 0, 5733 .size = sizeof(task_event.event_id), 5734 }, 5735 /* .pid */ 5736 /* .ppid */ 5737 /* .tid */ 5738 /* .ptid */ 5739 /* .time */ 5740 }, 5741 }; 5742 5743 perf_event_aux(perf_event_task_output, 5744 &task_event, 5745 task_ctx); 5746 } 5747 5748 void perf_event_fork(struct task_struct *task) 5749 { 5750 perf_event_task(task, NULL, 1); 5751 } 5752 5753 /* 5754 * comm tracking 5755 */ 5756 5757 struct perf_comm_event { 5758 struct task_struct *task; 5759 char *comm; 5760 int comm_size; 5761 5762 struct { 5763 struct perf_event_header header; 5764 5765 u32 pid; 5766 u32 tid; 5767 } event_id; 5768 }; 5769 5770 static int perf_event_comm_match(struct perf_event *event) 5771 { 5772 return event->attr.comm; 5773 } 5774 5775 static void perf_event_comm_output(struct perf_event *event, 5776 void *data) 5777 { 5778 struct perf_comm_event *comm_event = data; 5779 struct perf_output_handle handle; 5780 struct perf_sample_data sample; 5781 int size = comm_event->event_id.header.size; 5782 int ret; 5783 5784 if (!perf_event_comm_match(event)) 5785 return; 5786 5787 perf_event_header__init_id(&comm_event->event_id.header, &sample, event); 5788 ret = perf_output_begin(&handle, event, 5789 comm_event->event_id.header.size); 5790 5791 if (ret) 5792 goto out; 5793 5794 comm_event->event_id.pid = perf_event_pid(event, comm_event->task); 5795 comm_event->event_id.tid = perf_event_tid(event, comm_event->task); 5796 5797 perf_output_put(&handle, comm_event->event_id); 5798 __output_copy(&handle, comm_event->comm, 5799 comm_event->comm_size); 5800 5801 perf_event__output_id_sample(event, &handle, &sample); 5802 5803 perf_output_end(&handle); 5804 out: 5805 comm_event->event_id.header.size = size; 5806 } 5807 5808 static void perf_event_comm_event(struct perf_comm_event *comm_event) 5809 { 5810 char comm[TASK_COMM_LEN]; 5811 unsigned int size; 5812 5813 memset(comm, 0, sizeof(comm)); 5814 strlcpy(comm, comm_event->task->comm, sizeof(comm)); 5815 size = ALIGN(strlen(comm)+1, sizeof(u64)); 5816 5817 comm_event->comm = comm; 5818 comm_event->comm_size = size; 5819 5820 comm_event->event_id.header.size = sizeof(comm_event->event_id) + size; 5821 5822 perf_event_aux(perf_event_comm_output, 5823 comm_event, 5824 NULL); 5825 } 5826 5827 void perf_event_comm(struct task_struct *task, bool exec) 5828 { 5829 struct perf_comm_event comm_event; 5830 5831 if (!atomic_read(&nr_comm_events)) 5832 return; 5833 5834 comm_event = (struct perf_comm_event){ 5835 .task = task, 5836 /* .comm */ 5837 /* .comm_size */ 5838 .event_id = { 5839 .header = { 5840 .type = PERF_RECORD_COMM, 5841 .misc = exec ? PERF_RECORD_MISC_COMM_EXEC : 0, 5842 /* .size */ 5843 }, 5844 /* .pid */ 5845 /* .tid */ 5846 }, 5847 }; 5848 5849 perf_event_comm_event(&comm_event); 5850 } 5851 5852 /* 5853 * mmap tracking 5854 */ 5855 5856 struct perf_mmap_event { 5857 struct vm_area_struct *vma; 5858 5859 const char *file_name; 5860 int file_size; 5861 int maj, min; 5862 u64 ino; 5863 u64 ino_generation; 5864 u32 prot, flags; 5865 5866 struct { 5867 struct perf_event_header header; 5868 5869 u32 pid; 5870 u32 tid; 5871 u64 start; 5872 u64 len; 5873 u64 pgoff; 5874 } event_id; 5875 }; 5876 5877 static int perf_event_mmap_match(struct perf_event *event, 5878 void *data) 5879 { 5880 struct perf_mmap_event *mmap_event = data; 5881 struct vm_area_struct *vma = mmap_event->vma; 5882 int executable = vma->vm_flags & VM_EXEC; 5883 5884 return (!executable && event->attr.mmap_data) || 5885 (executable && (event->attr.mmap || event->attr.mmap2)); 5886 } 5887 5888 static void perf_event_mmap_output(struct perf_event *event, 5889 void *data) 5890 { 5891 struct perf_mmap_event *mmap_event = data; 5892 struct perf_output_handle handle; 5893 struct perf_sample_data sample; 5894 int size = mmap_event->event_id.header.size; 5895 int ret; 5896 5897 if (!perf_event_mmap_match(event, data)) 5898 return; 5899 5900 if (event->attr.mmap2) { 5901 mmap_event->event_id.header.type = PERF_RECORD_MMAP2; 5902 mmap_event->event_id.header.size += sizeof(mmap_event->maj); 5903 mmap_event->event_id.header.size += sizeof(mmap_event->min); 5904 mmap_event->event_id.header.size += sizeof(mmap_event->ino); 5905 mmap_event->event_id.header.size += sizeof(mmap_event->ino_generation); 5906 mmap_event->event_id.header.size += sizeof(mmap_event->prot); 5907 mmap_event->event_id.header.size += sizeof(mmap_event->flags); 5908 } 5909 5910 perf_event_header__init_id(&mmap_event->event_id.header, &sample, event); 5911 ret = perf_output_begin(&handle, event, 5912 mmap_event->event_id.header.size); 5913 if (ret) 5914 goto out; 5915 5916 mmap_event->event_id.pid = perf_event_pid(event, current); 5917 mmap_event->event_id.tid = perf_event_tid(event, current); 5918 5919 perf_output_put(&handle, mmap_event->event_id); 5920 5921 if (event->attr.mmap2) { 5922 perf_output_put(&handle, mmap_event->maj); 5923 perf_output_put(&handle, mmap_event->min); 5924 perf_output_put(&handle, mmap_event->ino); 5925 perf_output_put(&handle, mmap_event->ino_generation); 5926 perf_output_put(&handle, mmap_event->prot); 5927 perf_output_put(&handle, mmap_event->flags); 5928 } 5929 5930 __output_copy(&handle, mmap_event->file_name, 5931 mmap_event->file_size); 5932 5933 perf_event__output_id_sample(event, &handle, &sample); 5934 5935 perf_output_end(&handle); 5936 out: 5937 mmap_event->event_id.header.size = size; 5938 } 5939 5940 static void perf_event_mmap_event(struct perf_mmap_event *mmap_event) 5941 { 5942 struct vm_area_struct *vma = mmap_event->vma; 5943 struct file *file = vma->vm_file; 5944 int maj = 0, min = 0; 5945 u64 ino = 0, gen = 0; 5946 u32 prot = 0, flags = 0; 5947 unsigned int size; 5948 char tmp[16]; 5949 char *buf = NULL; 5950 char *name; 5951 5952 if (file) { 5953 struct inode *inode; 5954 dev_t dev; 5955 5956 buf = kmalloc(PATH_MAX, GFP_KERNEL); 5957 if (!buf) { 5958 name = "//enomem"; 5959 goto cpy_name; 5960 } 5961 /* 5962 * d_path() works from the end of the rb backwards, so we 5963 * need to add enough zero bytes after the string to handle 5964 * the 64bit alignment we do later. 5965 */ 5966 name = file_path(file, buf, PATH_MAX - sizeof(u64)); 5967 if (IS_ERR(name)) { 5968 name = "//toolong"; 5969 goto cpy_name; 5970 } 5971 inode = file_inode(vma->vm_file); 5972 dev = inode->i_sb->s_dev; 5973 ino = inode->i_ino; 5974 gen = inode->i_generation; 5975 maj = MAJOR(dev); 5976 min = MINOR(dev); 5977 5978 if (vma->vm_flags & VM_READ) 5979 prot |= PROT_READ; 5980 if (vma->vm_flags & VM_WRITE) 5981 prot |= PROT_WRITE; 5982 if (vma->vm_flags & VM_EXEC) 5983 prot |= PROT_EXEC; 5984 5985 if (vma->vm_flags & VM_MAYSHARE) 5986 flags = MAP_SHARED; 5987 else 5988 flags = MAP_PRIVATE; 5989 5990 if (vma->vm_flags & VM_DENYWRITE) 5991 flags |= MAP_DENYWRITE; 5992 if (vma->vm_flags & VM_MAYEXEC) 5993 flags |= MAP_EXECUTABLE; 5994 if (vma->vm_flags & VM_LOCKED) 5995 flags |= MAP_LOCKED; 5996 if (vma->vm_flags & VM_HUGETLB) 5997 flags |= MAP_HUGETLB; 5998 5999 goto got_name; 6000 } else { 6001 if (vma->vm_ops && vma->vm_ops->name) { 6002 name = (char *) vma->vm_ops->name(vma); 6003 if (name) 6004 goto cpy_name; 6005 } 6006 6007 name = (char *)arch_vma_name(vma); 6008 if (name) 6009 goto cpy_name; 6010 6011 if (vma->vm_start <= vma->vm_mm->start_brk && 6012 vma->vm_end >= vma->vm_mm->brk) { 6013 name = "[heap]"; 6014 goto cpy_name; 6015 } 6016 if (vma->vm_start <= vma->vm_mm->start_stack && 6017 vma->vm_end >= vma->vm_mm->start_stack) { 6018 name = "[stack]"; 6019 goto cpy_name; 6020 } 6021 6022 name = "//anon"; 6023 goto cpy_name; 6024 } 6025 6026 cpy_name: 6027 strlcpy(tmp, name, sizeof(tmp)); 6028 name = tmp; 6029 got_name: 6030 /* 6031 * Since our buffer works in 8 byte units we need to align our string 6032 * size to a multiple of 8. However, we must guarantee the tail end is 6033 * zero'd out to avoid leaking random bits to userspace. 6034 */ 6035 size = strlen(name)+1; 6036 while (!IS_ALIGNED(size, sizeof(u64))) 6037 name[size++] = '\0'; 6038 6039 mmap_event->file_name = name; 6040 mmap_event->file_size = size; 6041 mmap_event->maj = maj; 6042 mmap_event->min = min; 6043 mmap_event->ino = ino; 6044 mmap_event->ino_generation = gen; 6045 mmap_event->prot = prot; 6046 mmap_event->flags = flags; 6047 6048 if (!(vma->vm_flags & VM_EXEC)) 6049 mmap_event->event_id.header.misc |= PERF_RECORD_MISC_MMAP_DATA; 6050 6051 mmap_event->event_id.header.size = sizeof(mmap_event->event_id) + size; 6052 6053 perf_event_aux(perf_event_mmap_output, 6054 mmap_event, 6055 NULL); 6056 6057 kfree(buf); 6058 } 6059 6060 void perf_event_mmap(struct vm_area_struct *vma) 6061 { 6062 struct perf_mmap_event mmap_event; 6063 6064 if (!atomic_read(&nr_mmap_events)) 6065 return; 6066 6067 mmap_event = (struct perf_mmap_event){ 6068 .vma = vma, 6069 /* .file_name */ 6070 /* .file_size */ 6071 .event_id = { 6072 .header = { 6073 .type = PERF_RECORD_MMAP, 6074 .misc = PERF_RECORD_MISC_USER, 6075 /* .size */ 6076 }, 6077 /* .pid */ 6078 /* .tid */ 6079 .start = vma->vm_start, 6080 .len = vma->vm_end - vma->vm_start, 6081 .pgoff = (u64)vma->vm_pgoff << PAGE_SHIFT, 6082 }, 6083 /* .maj (attr_mmap2 only) */ 6084 /* .min (attr_mmap2 only) */ 6085 /* .ino (attr_mmap2 only) */ 6086 /* .ino_generation (attr_mmap2 only) */ 6087 /* .prot (attr_mmap2 only) */ 6088 /* .flags (attr_mmap2 only) */ 6089 }; 6090 6091 perf_event_mmap_event(&mmap_event); 6092 } 6093 6094 void perf_event_aux_event(struct perf_event *event, unsigned long head, 6095 unsigned long size, u64 flags) 6096 { 6097 struct perf_output_handle handle; 6098 struct perf_sample_data sample; 6099 struct perf_aux_event { 6100 struct perf_event_header header; 6101 u64 offset; 6102 u64 size; 6103 u64 flags; 6104 } rec = { 6105 .header = { 6106 .type = PERF_RECORD_AUX, 6107 .misc = 0, 6108 .size = sizeof(rec), 6109 }, 6110 .offset = head, 6111 .size = size, 6112 .flags = flags, 6113 }; 6114 int ret; 6115 6116 perf_event_header__init_id(&rec.header, &sample, event); 6117 ret = perf_output_begin(&handle, event, rec.header.size); 6118 6119 if (ret) 6120 return; 6121 6122 perf_output_put(&handle, rec); 6123 perf_event__output_id_sample(event, &handle, &sample); 6124 6125 perf_output_end(&handle); 6126 } 6127 6128 /* 6129 * Lost/dropped samples logging 6130 */ 6131 void perf_log_lost_samples(struct perf_event *event, u64 lost) 6132 { 6133 struct perf_output_handle handle; 6134 struct perf_sample_data sample; 6135 int ret; 6136 6137 struct { 6138 struct perf_event_header header; 6139 u64 lost; 6140 } lost_samples_event = { 6141 .header = { 6142 .type = PERF_RECORD_LOST_SAMPLES, 6143 .misc = 0, 6144 .size = sizeof(lost_samples_event), 6145 }, 6146 .lost = lost, 6147 }; 6148 6149 perf_event_header__init_id(&lost_samples_event.header, &sample, event); 6150 6151 ret = perf_output_begin(&handle, event, 6152 lost_samples_event.header.size); 6153 if (ret) 6154 return; 6155 6156 perf_output_put(&handle, lost_samples_event); 6157 perf_event__output_id_sample(event, &handle, &sample); 6158 perf_output_end(&handle); 6159 } 6160 6161 /* 6162 * context_switch tracking 6163 */ 6164 6165 struct perf_switch_event { 6166 struct task_struct *task; 6167 struct task_struct *next_prev; 6168 6169 struct { 6170 struct perf_event_header header; 6171 u32 next_prev_pid; 6172 u32 next_prev_tid; 6173 } event_id; 6174 }; 6175 6176 static int perf_event_switch_match(struct perf_event *event) 6177 { 6178 return event->attr.context_switch; 6179 } 6180 6181 static void perf_event_switch_output(struct perf_event *event, void *data) 6182 { 6183 struct perf_switch_event *se = data; 6184 struct perf_output_handle handle; 6185 struct perf_sample_data sample; 6186 int ret; 6187 6188 if (!perf_event_switch_match(event)) 6189 return; 6190 6191 /* Only CPU-wide events are allowed to see next/prev pid/tid */ 6192 if (event->ctx->task) { 6193 se->event_id.header.type = PERF_RECORD_SWITCH; 6194 se->event_id.header.size = sizeof(se->event_id.header); 6195 } else { 6196 se->event_id.header.type = PERF_RECORD_SWITCH_CPU_WIDE; 6197 se->event_id.header.size = sizeof(se->event_id); 6198 se->event_id.next_prev_pid = 6199 perf_event_pid(event, se->next_prev); 6200 se->event_id.next_prev_tid = 6201 perf_event_tid(event, se->next_prev); 6202 } 6203 6204 perf_event_header__init_id(&se->event_id.header, &sample, event); 6205 6206 ret = perf_output_begin(&handle, event, se->event_id.header.size); 6207 if (ret) 6208 return; 6209 6210 if (event->ctx->task) 6211 perf_output_put(&handle, se->event_id.header); 6212 else 6213 perf_output_put(&handle, se->event_id); 6214 6215 perf_event__output_id_sample(event, &handle, &sample); 6216 6217 perf_output_end(&handle); 6218 } 6219 6220 static void perf_event_switch(struct task_struct *task, 6221 struct task_struct *next_prev, bool sched_in) 6222 { 6223 struct perf_switch_event switch_event; 6224 6225 /* N.B. caller checks nr_switch_events != 0 */ 6226 6227 switch_event = (struct perf_switch_event){ 6228 .task = task, 6229 .next_prev = next_prev, 6230 .event_id = { 6231 .header = { 6232 /* .type */ 6233 .misc = sched_in ? 0 : PERF_RECORD_MISC_SWITCH_OUT, 6234 /* .size */ 6235 }, 6236 /* .next_prev_pid */ 6237 /* .next_prev_tid */ 6238 }, 6239 }; 6240 6241 perf_event_aux(perf_event_switch_output, 6242 &switch_event, 6243 NULL); 6244 } 6245 6246 /* 6247 * IRQ throttle logging 6248 */ 6249 6250 static void perf_log_throttle(struct perf_event *event, int enable) 6251 { 6252 struct perf_output_handle handle; 6253 struct perf_sample_data sample; 6254 int ret; 6255 6256 struct { 6257 struct perf_event_header header; 6258 u64 time; 6259 u64 id; 6260 u64 stream_id; 6261 } throttle_event = { 6262 .header = { 6263 .type = PERF_RECORD_THROTTLE, 6264 .misc = 0, 6265 .size = sizeof(throttle_event), 6266 }, 6267 .time = perf_event_clock(event), 6268 .id = primary_event_id(event), 6269 .stream_id = event->id, 6270 }; 6271 6272 if (enable) 6273 throttle_event.header.type = PERF_RECORD_UNTHROTTLE; 6274 6275 perf_event_header__init_id(&throttle_event.header, &sample, event); 6276 6277 ret = perf_output_begin(&handle, event, 6278 throttle_event.header.size); 6279 if (ret) 6280 return; 6281 6282 perf_output_put(&handle, throttle_event); 6283 perf_event__output_id_sample(event, &handle, &sample); 6284 perf_output_end(&handle); 6285 } 6286 6287 static void perf_log_itrace_start(struct perf_event *event) 6288 { 6289 struct perf_output_handle handle; 6290 struct perf_sample_data sample; 6291 struct perf_aux_event { 6292 struct perf_event_header header; 6293 u32 pid; 6294 u32 tid; 6295 } rec; 6296 int ret; 6297 6298 if (event->parent) 6299 event = event->parent; 6300 6301 if (!(event->pmu->capabilities & PERF_PMU_CAP_ITRACE) || 6302 event->hw.itrace_started) 6303 return; 6304 6305 rec.header.type = PERF_RECORD_ITRACE_START; 6306 rec.header.misc = 0; 6307 rec.header.size = sizeof(rec); 6308 rec.pid = perf_event_pid(event, current); 6309 rec.tid = perf_event_tid(event, current); 6310 6311 perf_event_header__init_id(&rec.header, &sample, event); 6312 ret = perf_output_begin(&handle, event, rec.header.size); 6313 6314 if (ret) 6315 return; 6316 6317 perf_output_put(&handle, rec); 6318 perf_event__output_id_sample(event, &handle, &sample); 6319 6320 perf_output_end(&handle); 6321 } 6322 6323 /* 6324 * Generic event overflow handling, sampling. 6325 */ 6326 6327 static int __perf_event_overflow(struct perf_event *event, 6328 int throttle, struct perf_sample_data *data, 6329 struct pt_regs *regs) 6330 { 6331 int events = atomic_read(&event->event_limit); 6332 struct hw_perf_event *hwc = &event->hw; 6333 u64 seq; 6334 int ret = 0; 6335 6336 /* 6337 * Non-sampling counters might still use the PMI to fold short 6338 * hardware counters, ignore those. 6339 */ 6340 if (unlikely(!is_sampling_event(event))) 6341 return 0; 6342 6343 seq = __this_cpu_read(perf_throttled_seq); 6344 if (seq != hwc->interrupts_seq) { 6345 hwc->interrupts_seq = seq; 6346 hwc->interrupts = 1; 6347 } else { 6348 hwc->interrupts++; 6349 if (unlikely(throttle 6350 && hwc->interrupts >= max_samples_per_tick)) { 6351 __this_cpu_inc(perf_throttled_count); 6352 hwc->interrupts = MAX_INTERRUPTS; 6353 perf_log_throttle(event, 0); 6354 tick_nohz_full_kick(); 6355 ret = 1; 6356 } 6357 } 6358 6359 if (event->attr.freq) { 6360 u64 now = perf_clock(); 6361 s64 delta = now - hwc->freq_time_stamp; 6362 6363 hwc->freq_time_stamp = now; 6364 6365 if (delta > 0 && delta < 2*TICK_NSEC) 6366 perf_adjust_period(event, delta, hwc->last_period, true); 6367 } 6368 6369 /* 6370 * XXX event_limit might not quite work as expected on inherited 6371 * events 6372 */ 6373 6374 event->pending_kill = POLL_IN; 6375 if (events && atomic_dec_and_test(&event->event_limit)) { 6376 ret = 1; 6377 event->pending_kill = POLL_HUP; 6378 event->pending_disable = 1; 6379 irq_work_queue(&event->pending); 6380 } 6381 6382 if (event->overflow_handler) 6383 event->overflow_handler(event, data, regs); 6384 else 6385 perf_event_output(event, data, regs); 6386 6387 if (*perf_event_fasync(event) && event->pending_kill) { 6388 event->pending_wakeup = 1; 6389 irq_work_queue(&event->pending); 6390 } 6391 6392 return ret; 6393 } 6394 6395 int perf_event_overflow(struct perf_event *event, 6396 struct perf_sample_data *data, 6397 struct pt_regs *regs) 6398 { 6399 return __perf_event_overflow(event, 1, data, regs); 6400 } 6401 6402 /* 6403 * Generic software event infrastructure 6404 */ 6405 6406 struct swevent_htable { 6407 struct swevent_hlist *swevent_hlist; 6408 struct mutex hlist_mutex; 6409 int hlist_refcount; 6410 6411 /* Recursion avoidance in each contexts */ 6412 int recursion[PERF_NR_CONTEXTS]; 6413 }; 6414 6415 static DEFINE_PER_CPU(struct swevent_htable, swevent_htable); 6416 6417 /* 6418 * We directly increment event->count and keep a second value in 6419 * event->hw.period_left to count intervals. This period event 6420 * is kept in the range [-sample_period, 0] so that we can use the 6421 * sign as trigger. 6422 */ 6423 6424 u64 perf_swevent_set_period(struct perf_event *event) 6425 { 6426 struct hw_perf_event *hwc = &event->hw; 6427 u64 period = hwc->last_period; 6428 u64 nr, offset; 6429 s64 old, val; 6430 6431 hwc->last_period = hwc->sample_period; 6432 6433 again: 6434 old = val = local64_read(&hwc->period_left); 6435 if (val < 0) 6436 return 0; 6437 6438 nr = div64_u64(period + val, period); 6439 offset = nr * period; 6440 val -= offset; 6441 if (local64_cmpxchg(&hwc->period_left, old, val) != old) 6442 goto again; 6443 6444 return nr; 6445 } 6446 6447 static void perf_swevent_overflow(struct perf_event *event, u64 overflow, 6448 struct perf_sample_data *data, 6449 struct pt_regs *regs) 6450 { 6451 struct hw_perf_event *hwc = &event->hw; 6452 int throttle = 0; 6453 6454 if (!overflow) 6455 overflow = perf_swevent_set_period(event); 6456 6457 if (hwc->interrupts == MAX_INTERRUPTS) 6458 return; 6459 6460 for (; overflow; overflow--) { 6461 if (__perf_event_overflow(event, throttle, 6462 data, regs)) { 6463 /* 6464 * We inhibit the overflow from happening when 6465 * hwc->interrupts == MAX_INTERRUPTS. 6466 */ 6467 break; 6468 } 6469 throttle = 1; 6470 } 6471 } 6472 6473 static void perf_swevent_event(struct perf_event *event, u64 nr, 6474 struct perf_sample_data *data, 6475 struct pt_regs *regs) 6476 { 6477 struct hw_perf_event *hwc = &event->hw; 6478 6479 local64_add(nr, &event->count); 6480 6481 if (!regs) 6482 return; 6483 6484 if (!is_sampling_event(event)) 6485 return; 6486 6487 if ((event->attr.sample_type & PERF_SAMPLE_PERIOD) && !event->attr.freq) { 6488 data->period = nr; 6489 return perf_swevent_overflow(event, 1, data, regs); 6490 } else 6491 data->period = event->hw.last_period; 6492 6493 if (nr == 1 && hwc->sample_period == 1 && !event->attr.freq) 6494 return perf_swevent_overflow(event, 1, data, regs); 6495 6496 if (local64_add_negative(nr, &hwc->period_left)) 6497 return; 6498 6499 perf_swevent_overflow(event, 0, data, regs); 6500 } 6501 6502 static int perf_exclude_event(struct perf_event *event, 6503 struct pt_regs *regs) 6504 { 6505 if (event->hw.state & PERF_HES_STOPPED) 6506 return 1; 6507 6508 if (regs) { 6509 if (event->attr.exclude_user && user_mode(regs)) 6510 return 1; 6511 6512 if (event->attr.exclude_kernel && !user_mode(regs)) 6513 return 1; 6514 } 6515 6516 return 0; 6517 } 6518 6519 static int perf_swevent_match(struct perf_event *event, 6520 enum perf_type_id type, 6521 u32 event_id, 6522 struct perf_sample_data *data, 6523 struct pt_regs *regs) 6524 { 6525 if (event->attr.type != type) 6526 return 0; 6527 6528 if (event->attr.config != event_id) 6529 return 0; 6530 6531 if (perf_exclude_event(event, regs)) 6532 return 0; 6533 6534 return 1; 6535 } 6536 6537 static inline u64 swevent_hash(u64 type, u32 event_id) 6538 { 6539 u64 val = event_id | (type << 32); 6540 6541 return hash_64(val, SWEVENT_HLIST_BITS); 6542 } 6543 6544 static inline struct hlist_head * 6545 __find_swevent_head(struct swevent_hlist *hlist, u64 type, u32 event_id) 6546 { 6547 u64 hash = swevent_hash(type, event_id); 6548 6549 return &hlist->heads[hash]; 6550 } 6551 6552 /* For the read side: events when they trigger */ 6553 static inline struct hlist_head * 6554 find_swevent_head_rcu(struct swevent_htable *swhash, u64 type, u32 event_id) 6555 { 6556 struct swevent_hlist *hlist; 6557 6558 hlist = rcu_dereference(swhash->swevent_hlist); 6559 if (!hlist) 6560 return NULL; 6561 6562 return __find_swevent_head(hlist, type, event_id); 6563 } 6564 6565 /* For the event head insertion and removal in the hlist */ 6566 static inline struct hlist_head * 6567 find_swevent_head(struct swevent_htable *swhash, struct perf_event *event) 6568 { 6569 struct swevent_hlist *hlist; 6570 u32 event_id = event->attr.config; 6571 u64 type = event->attr.type; 6572 6573 /* 6574 * Event scheduling is always serialized against hlist allocation 6575 * and release. Which makes the protected version suitable here. 6576 * The context lock guarantees that. 6577 */ 6578 hlist = rcu_dereference_protected(swhash->swevent_hlist, 6579 lockdep_is_held(&event->ctx->lock)); 6580 if (!hlist) 6581 return NULL; 6582 6583 return __find_swevent_head(hlist, type, event_id); 6584 } 6585 6586 static void do_perf_sw_event(enum perf_type_id type, u32 event_id, 6587 u64 nr, 6588 struct perf_sample_data *data, 6589 struct pt_regs *regs) 6590 { 6591 struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable); 6592 struct perf_event *event; 6593 struct hlist_head *head; 6594 6595 rcu_read_lock(); 6596 head = find_swevent_head_rcu(swhash, type, event_id); 6597 if (!head) 6598 goto end; 6599 6600 hlist_for_each_entry_rcu(event, head, hlist_entry) { 6601 if (perf_swevent_match(event, type, event_id, data, regs)) 6602 perf_swevent_event(event, nr, data, regs); 6603 } 6604 end: 6605 rcu_read_unlock(); 6606 } 6607 6608 DEFINE_PER_CPU(struct pt_regs, __perf_regs[4]); 6609 6610 int perf_swevent_get_recursion_context(void) 6611 { 6612 struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable); 6613 6614 return get_recursion_context(swhash->recursion); 6615 } 6616 EXPORT_SYMBOL_GPL(perf_swevent_get_recursion_context); 6617 6618 inline void perf_swevent_put_recursion_context(int rctx) 6619 { 6620 struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable); 6621 6622 put_recursion_context(swhash->recursion, rctx); 6623 } 6624 6625 void ___perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr) 6626 { 6627 struct perf_sample_data data; 6628 6629 if (WARN_ON_ONCE(!regs)) 6630 return; 6631 6632 perf_sample_data_init(&data, addr, 0); 6633 do_perf_sw_event(PERF_TYPE_SOFTWARE, event_id, nr, &data, regs); 6634 } 6635 6636 void __perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr) 6637 { 6638 int rctx; 6639 6640 preempt_disable_notrace(); 6641 rctx = perf_swevent_get_recursion_context(); 6642 if (unlikely(rctx < 0)) 6643 goto fail; 6644 6645 ___perf_sw_event(event_id, nr, regs, addr); 6646 6647 perf_swevent_put_recursion_context(rctx); 6648 fail: 6649 preempt_enable_notrace(); 6650 } 6651 6652 static void perf_swevent_read(struct perf_event *event) 6653 { 6654 } 6655 6656 static int perf_swevent_add(struct perf_event *event, int flags) 6657 { 6658 struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable); 6659 struct hw_perf_event *hwc = &event->hw; 6660 struct hlist_head *head; 6661 6662 if (is_sampling_event(event)) { 6663 hwc->last_period = hwc->sample_period; 6664 perf_swevent_set_period(event); 6665 } 6666 6667 hwc->state = !(flags & PERF_EF_START); 6668 6669 head = find_swevent_head(swhash, event); 6670 if (WARN_ON_ONCE(!head)) 6671 return -EINVAL; 6672 6673 hlist_add_head_rcu(&event->hlist_entry, head); 6674 perf_event_update_userpage(event); 6675 6676 return 0; 6677 } 6678 6679 static void perf_swevent_del(struct perf_event *event, int flags) 6680 { 6681 hlist_del_rcu(&event->hlist_entry); 6682 } 6683 6684 static void perf_swevent_start(struct perf_event *event, int flags) 6685 { 6686 event->hw.state = 0; 6687 } 6688 6689 static void perf_swevent_stop(struct perf_event *event, int flags) 6690 { 6691 event->hw.state = PERF_HES_STOPPED; 6692 } 6693 6694 /* Deref the hlist from the update side */ 6695 static inline struct swevent_hlist * 6696 swevent_hlist_deref(struct swevent_htable *swhash) 6697 { 6698 return rcu_dereference_protected(swhash->swevent_hlist, 6699 lockdep_is_held(&swhash->hlist_mutex)); 6700 } 6701 6702 static void swevent_hlist_release(struct swevent_htable *swhash) 6703 { 6704 struct swevent_hlist *hlist = swevent_hlist_deref(swhash); 6705 6706 if (!hlist) 6707 return; 6708 6709 RCU_INIT_POINTER(swhash->swevent_hlist, NULL); 6710 kfree_rcu(hlist, rcu_head); 6711 } 6712 6713 static void swevent_hlist_put_cpu(struct perf_event *event, int cpu) 6714 { 6715 struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu); 6716 6717 mutex_lock(&swhash->hlist_mutex); 6718 6719 if (!--swhash->hlist_refcount) 6720 swevent_hlist_release(swhash); 6721 6722 mutex_unlock(&swhash->hlist_mutex); 6723 } 6724 6725 static void swevent_hlist_put(struct perf_event *event) 6726 { 6727 int cpu; 6728 6729 for_each_possible_cpu(cpu) 6730 swevent_hlist_put_cpu(event, cpu); 6731 } 6732 6733 static int swevent_hlist_get_cpu(struct perf_event *event, int cpu) 6734 { 6735 struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu); 6736 int err = 0; 6737 6738 mutex_lock(&swhash->hlist_mutex); 6739 if (!swevent_hlist_deref(swhash) && cpu_online(cpu)) { 6740 struct swevent_hlist *hlist; 6741 6742 hlist = kzalloc(sizeof(*hlist), GFP_KERNEL); 6743 if (!hlist) { 6744 err = -ENOMEM; 6745 goto exit; 6746 } 6747 rcu_assign_pointer(swhash->swevent_hlist, hlist); 6748 } 6749 swhash->hlist_refcount++; 6750 exit: 6751 mutex_unlock(&swhash->hlist_mutex); 6752 6753 return err; 6754 } 6755 6756 static int swevent_hlist_get(struct perf_event *event) 6757 { 6758 int err; 6759 int cpu, failed_cpu; 6760 6761 get_online_cpus(); 6762 for_each_possible_cpu(cpu) { 6763 err = swevent_hlist_get_cpu(event, cpu); 6764 if (err) { 6765 failed_cpu = cpu; 6766 goto fail; 6767 } 6768 } 6769 put_online_cpus(); 6770 6771 return 0; 6772 fail: 6773 for_each_possible_cpu(cpu) { 6774 if (cpu == failed_cpu) 6775 break; 6776 swevent_hlist_put_cpu(event, cpu); 6777 } 6778 6779 put_online_cpus(); 6780 return err; 6781 } 6782 6783 struct static_key perf_swevent_enabled[PERF_COUNT_SW_MAX]; 6784 6785 static void sw_perf_event_destroy(struct perf_event *event) 6786 { 6787 u64 event_id = event->attr.config; 6788 6789 WARN_ON(event->parent); 6790 6791 static_key_slow_dec(&perf_swevent_enabled[event_id]); 6792 swevent_hlist_put(event); 6793 } 6794 6795 static int perf_swevent_init(struct perf_event *event) 6796 { 6797 u64 event_id = event->attr.config; 6798 6799 if (event->attr.type != PERF_TYPE_SOFTWARE) 6800 return -ENOENT; 6801 6802 /* 6803 * no branch sampling for software events 6804 */ 6805 if (has_branch_stack(event)) 6806 return -EOPNOTSUPP; 6807 6808 switch (event_id) { 6809 case PERF_COUNT_SW_CPU_CLOCK: 6810 case PERF_COUNT_SW_TASK_CLOCK: 6811 return -ENOENT; 6812 6813 default: 6814 break; 6815 } 6816 6817 if (event_id >= PERF_COUNT_SW_MAX) 6818 return -ENOENT; 6819 6820 if (!event->parent) { 6821 int err; 6822 6823 err = swevent_hlist_get(event); 6824 if (err) 6825 return err; 6826 6827 static_key_slow_inc(&perf_swevent_enabled[event_id]); 6828 event->destroy = sw_perf_event_destroy; 6829 } 6830 6831 return 0; 6832 } 6833 6834 static struct pmu perf_swevent = { 6835 .task_ctx_nr = perf_sw_context, 6836 6837 .capabilities = PERF_PMU_CAP_NO_NMI, 6838 6839 .event_init = perf_swevent_init, 6840 .add = perf_swevent_add, 6841 .del = perf_swevent_del, 6842 .start = perf_swevent_start, 6843 .stop = perf_swevent_stop, 6844 .read = perf_swevent_read, 6845 }; 6846 6847 #ifdef CONFIG_EVENT_TRACING 6848 6849 static int perf_tp_filter_match(struct perf_event *event, 6850 struct perf_sample_data *data) 6851 { 6852 void *record = data->raw->data; 6853 6854 /* only top level events have filters set */ 6855 if (event->parent) 6856 event = event->parent; 6857 6858 if (likely(!event->filter) || filter_match_preds(event->filter, record)) 6859 return 1; 6860 return 0; 6861 } 6862 6863 static int perf_tp_event_match(struct perf_event *event, 6864 struct perf_sample_data *data, 6865 struct pt_regs *regs) 6866 { 6867 if (event->hw.state & PERF_HES_STOPPED) 6868 return 0; 6869 /* 6870 * All tracepoints are from kernel-space. 6871 */ 6872 if (event->attr.exclude_kernel) 6873 return 0; 6874 6875 if (!perf_tp_filter_match(event, data)) 6876 return 0; 6877 6878 return 1; 6879 } 6880 6881 void perf_tp_event(u64 addr, u64 count, void *record, int entry_size, 6882 struct pt_regs *regs, struct hlist_head *head, int rctx, 6883 struct task_struct *task) 6884 { 6885 struct perf_sample_data data; 6886 struct perf_event *event; 6887 6888 struct perf_raw_record raw = { 6889 .size = entry_size, 6890 .data = record, 6891 }; 6892 6893 perf_sample_data_init(&data, addr, 0); 6894 data.raw = &raw; 6895 6896 hlist_for_each_entry_rcu(event, head, hlist_entry) { 6897 if (perf_tp_event_match(event, &data, regs)) 6898 perf_swevent_event(event, count, &data, regs); 6899 } 6900 6901 /* 6902 * If we got specified a target task, also iterate its context and 6903 * deliver this event there too. 6904 */ 6905 if (task && task != current) { 6906 struct perf_event_context *ctx; 6907 struct trace_entry *entry = record; 6908 6909 rcu_read_lock(); 6910 ctx = rcu_dereference(task->perf_event_ctxp[perf_sw_context]); 6911 if (!ctx) 6912 goto unlock; 6913 6914 list_for_each_entry_rcu(event, &ctx->event_list, event_entry) { 6915 if (event->attr.type != PERF_TYPE_TRACEPOINT) 6916 continue; 6917 if (event->attr.config != entry->type) 6918 continue; 6919 if (perf_tp_event_match(event, &data, regs)) 6920 perf_swevent_event(event, count, &data, regs); 6921 } 6922 unlock: 6923 rcu_read_unlock(); 6924 } 6925 6926 perf_swevent_put_recursion_context(rctx); 6927 } 6928 EXPORT_SYMBOL_GPL(perf_tp_event); 6929 6930 static void tp_perf_event_destroy(struct perf_event *event) 6931 { 6932 perf_trace_destroy(event); 6933 } 6934 6935 static int perf_tp_event_init(struct perf_event *event) 6936 { 6937 int err; 6938 6939 if (event->attr.type != PERF_TYPE_TRACEPOINT) 6940 return -ENOENT; 6941 6942 /* 6943 * no branch sampling for tracepoint events 6944 */ 6945 if (has_branch_stack(event)) 6946 return -EOPNOTSUPP; 6947 6948 err = perf_trace_init(event); 6949 if (err) 6950 return err; 6951 6952 event->destroy = tp_perf_event_destroy; 6953 6954 return 0; 6955 } 6956 6957 static struct pmu perf_tracepoint = { 6958 .task_ctx_nr = perf_sw_context, 6959 6960 .event_init = perf_tp_event_init, 6961 .add = perf_trace_add, 6962 .del = perf_trace_del, 6963 .start = perf_swevent_start, 6964 .stop = perf_swevent_stop, 6965 .read = perf_swevent_read, 6966 }; 6967 6968 static inline void perf_tp_register(void) 6969 { 6970 perf_pmu_register(&perf_tracepoint, "tracepoint", PERF_TYPE_TRACEPOINT); 6971 } 6972 6973 static int perf_event_set_filter(struct perf_event *event, void __user *arg) 6974 { 6975 char *filter_str; 6976 int ret; 6977 6978 if (event->attr.type != PERF_TYPE_TRACEPOINT) 6979 return -EINVAL; 6980 6981 filter_str = strndup_user(arg, PAGE_SIZE); 6982 if (IS_ERR(filter_str)) 6983 return PTR_ERR(filter_str); 6984 6985 ret = ftrace_profile_set_filter(event, event->attr.config, filter_str); 6986 6987 kfree(filter_str); 6988 return ret; 6989 } 6990 6991 static void perf_event_free_filter(struct perf_event *event) 6992 { 6993 ftrace_profile_free_filter(event); 6994 } 6995 6996 static int perf_event_set_bpf_prog(struct perf_event *event, u32 prog_fd) 6997 { 6998 struct bpf_prog *prog; 6999 7000 if (event->attr.type != PERF_TYPE_TRACEPOINT) 7001 return -EINVAL; 7002 7003 if (event->tp_event->prog) 7004 return -EEXIST; 7005 7006 if (!(event->tp_event->flags & TRACE_EVENT_FL_UKPROBE)) 7007 /* bpf programs can only be attached to u/kprobes */ 7008 return -EINVAL; 7009 7010 prog = bpf_prog_get(prog_fd); 7011 if (IS_ERR(prog)) 7012 return PTR_ERR(prog); 7013 7014 if (prog->type != BPF_PROG_TYPE_KPROBE) { 7015 /* valid fd, but invalid bpf program type */ 7016 bpf_prog_put(prog); 7017 return -EINVAL; 7018 } 7019 7020 event->tp_event->prog = prog; 7021 7022 return 0; 7023 } 7024 7025 static void perf_event_free_bpf_prog(struct perf_event *event) 7026 { 7027 struct bpf_prog *prog; 7028 7029 if (!event->tp_event) 7030 return; 7031 7032 prog = event->tp_event->prog; 7033 if (prog) { 7034 event->tp_event->prog = NULL; 7035 bpf_prog_put(prog); 7036 } 7037 } 7038 7039 #else 7040 7041 static inline void perf_tp_register(void) 7042 { 7043 } 7044 7045 static int perf_event_set_filter(struct perf_event *event, void __user *arg) 7046 { 7047 return -ENOENT; 7048 } 7049 7050 static void perf_event_free_filter(struct perf_event *event) 7051 { 7052 } 7053 7054 static int perf_event_set_bpf_prog(struct perf_event *event, u32 prog_fd) 7055 { 7056 return -ENOENT; 7057 } 7058 7059 static void perf_event_free_bpf_prog(struct perf_event *event) 7060 { 7061 } 7062 #endif /* CONFIG_EVENT_TRACING */ 7063 7064 #ifdef CONFIG_HAVE_HW_BREAKPOINT 7065 void perf_bp_event(struct perf_event *bp, void *data) 7066 { 7067 struct perf_sample_data sample; 7068 struct pt_regs *regs = data; 7069 7070 perf_sample_data_init(&sample, bp->attr.bp_addr, 0); 7071 7072 if (!bp->hw.state && !perf_exclude_event(bp, regs)) 7073 perf_swevent_event(bp, 1, &sample, regs); 7074 } 7075 #endif 7076 7077 /* 7078 * hrtimer based swevent callback 7079 */ 7080 7081 static enum hrtimer_restart perf_swevent_hrtimer(struct hrtimer *hrtimer) 7082 { 7083 enum hrtimer_restart ret = HRTIMER_RESTART; 7084 struct perf_sample_data data; 7085 struct pt_regs *regs; 7086 struct perf_event *event; 7087 u64 period; 7088 7089 event = container_of(hrtimer, struct perf_event, hw.hrtimer); 7090 7091 if (event->state != PERF_EVENT_STATE_ACTIVE) 7092 return HRTIMER_NORESTART; 7093 7094 event->pmu->read(event); 7095 7096 perf_sample_data_init(&data, 0, event->hw.last_period); 7097 regs = get_irq_regs(); 7098 7099 if (regs && !perf_exclude_event(event, regs)) { 7100 if (!(event->attr.exclude_idle && is_idle_task(current))) 7101 if (__perf_event_overflow(event, 1, &data, regs)) 7102 ret = HRTIMER_NORESTART; 7103 } 7104 7105 period = max_t(u64, 10000, event->hw.sample_period); 7106 hrtimer_forward_now(hrtimer, ns_to_ktime(period)); 7107 7108 return ret; 7109 } 7110 7111 static void perf_swevent_start_hrtimer(struct perf_event *event) 7112 { 7113 struct hw_perf_event *hwc = &event->hw; 7114 s64 period; 7115 7116 if (!is_sampling_event(event)) 7117 return; 7118 7119 period = local64_read(&hwc->period_left); 7120 if (period) { 7121 if (period < 0) 7122 period = 10000; 7123 7124 local64_set(&hwc->period_left, 0); 7125 } else { 7126 period = max_t(u64, 10000, hwc->sample_period); 7127 } 7128 hrtimer_start(&hwc->hrtimer, ns_to_ktime(period), 7129 HRTIMER_MODE_REL_PINNED); 7130 } 7131 7132 static void perf_swevent_cancel_hrtimer(struct perf_event *event) 7133 { 7134 struct hw_perf_event *hwc = &event->hw; 7135 7136 if (is_sampling_event(event)) { 7137 ktime_t remaining = hrtimer_get_remaining(&hwc->hrtimer); 7138 local64_set(&hwc->period_left, ktime_to_ns(remaining)); 7139 7140 hrtimer_cancel(&hwc->hrtimer); 7141 } 7142 } 7143 7144 static void perf_swevent_init_hrtimer(struct perf_event *event) 7145 { 7146 struct hw_perf_event *hwc = &event->hw; 7147 7148 if (!is_sampling_event(event)) 7149 return; 7150 7151 hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 7152 hwc->hrtimer.function = perf_swevent_hrtimer; 7153 7154 /* 7155 * Since hrtimers have a fixed rate, we can do a static freq->period 7156 * mapping and avoid the whole period adjust feedback stuff. 7157 */ 7158 if (event->attr.freq) { 7159 long freq = event->attr.sample_freq; 7160 7161 event->attr.sample_period = NSEC_PER_SEC / freq; 7162 hwc->sample_period = event->attr.sample_period; 7163 local64_set(&hwc->period_left, hwc->sample_period); 7164 hwc->last_period = hwc->sample_period; 7165 event->attr.freq = 0; 7166 } 7167 } 7168 7169 /* 7170 * Software event: cpu wall time clock 7171 */ 7172 7173 static void cpu_clock_event_update(struct perf_event *event) 7174 { 7175 s64 prev; 7176 u64 now; 7177 7178 now = local_clock(); 7179 prev = local64_xchg(&event->hw.prev_count, now); 7180 local64_add(now - prev, &event->count); 7181 } 7182 7183 static void cpu_clock_event_start(struct perf_event *event, int flags) 7184 { 7185 local64_set(&event->hw.prev_count, local_clock()); 7186 perf_swevent_start_hrtimer(event); 7187 } 7188 7189 static void cpu_clock_event_stop(struct perf_event *event, int flags) 7190 { 7191 perf_swevent_cancel_hrtimer(event); 7192 cpu_clock_event_update(event); 7193 } 7194 7195 static int cpu_clock_event_add(struct perf_event *event, int flags) 7196 { 7197 if (flags & PERF_EF_START) 7198 cpu_clock_event_start(event, flags); 7199 perf_event_update_userpage(event); 7200 7201 return 0; 7202 } 7203 7204 static void cpu_clock_event_del(struct perf_event *event, int flags) 7205 { 7206 cpu_clock_event_stop(event, flags); 7207 } 7208 7209 static void cpu_clock_event_read(struct perf_event *event) 7210 { 7211 cpu_clock_event_update(event); 7212 } 7213 7214 static int cpu_clock_event_init(struct perf_event *event) 7215 { 7216 if (event->attr.type != PERF_TYPE_SOFTWARE) 7217 return -ENOENT; 7218 7219 if (event->attr.config != PERF_COUNT_SW_CPU_CLOCK) 7220 return -ENOENT; 7221 7222 /* 7223 * no branch sampling for software events 7224 */ 7225 if (has_branch_stack(event)) 7226 return -EOPNOTSUPP; 7227 7228 perf_swevent_init_hrtimer(event); 7229 7230 return 0; 7231 } 7232 7233 static struct pmu perf_cpu_clock = { 7234 .task_ctx_nr = perf_sw_context, 7235 7236 .capabilities = PERF_PMU_CAP_NO_NMI, 7237 7238 .event_init = cpu_clock_event_init, 7239 .add = cpu_clock_event_add, 7240 .del = cpu_clock_event_del, 7241 .start = cpu_clock_event_start, 7242 .stop = cpu_clock_event_stop, 7243 .read = cpu_clock_event_read, 7244 }; 7245 7246 /* 7247 * Software event: task time clock 7248 */ 7249 7250 static void task_clock_event_update(struct perf_event *event, u64 now) 7251 { 7252 u64 prev; 7253 s64 delta; 7254 7255 prev = local64_xchg(&event->hw.prev_count, now); 7256 delta = now - prev; 7257 local64_add(delta, &event->count); 7258 } 7259 7260 static void task_clock_event_start(struct perf_event *event, int flags) 7261 { 7262 local64_set(&event->hw.prev_count, event->ctx->time); 7263 perf_swevent_start_hrtimer(event); 7264 } 7265 7266 static void task_clock_event_stop(struct perf_event *event, int flags) 7267 { 7268 perf_swevent_cancel_hrtimer(event); 7269 task_clock_event_update(event, event->ctx->time); 7270 } 7271 7272 static int task_clock_event_add(struct perf_event *event, int flags) 7273 { 7274 if (flags & PERF_EF_START) 7275 task_clock_event_start(event, flags); 7276 perf_event_update_userpage(event); 7277 7278 return 0; 7279 } 7280 7281 static void task_clock_event_del(struct perf_event *event, int flags) 7282 { 7283 task_clock_event_stop(event, PERF_EF_UPDATE); 7284 } 7285 7286 static void task_clock_event_read(struct perf_event *event) 7287 { 7288 u64 now = perf_clock(); 7289 u64 delta = now - event->ctx->timestamp; 7290 u64 time = event->ctx->time + delta; 7291 7292 task_clock_event_update(event, time); 7293 } 7294 7295 static int task_clock_event_init(struct perf_event *event) 7296 { 7297 if (event->attr.type != PERF_TYPE_SOFTWARE) 7298 return -ENOENT; 7299 7300 if (event->attr.config != PERF_COUNT_SW_TASK_CLOCK) 7301 return -ENOENT; 7302 7303 /* 7304 * no branch sampling for software events 7305 */ 7306 if (has_branch_stack(event)) 7307 return -EOPNOTSUPP; 7308 7309 perf_swevent_init_hrtimer(event); 7310 7311 return 0; 7312 } 7313 7314 static struct pmu perf_task_clock = { 7315 .task_ctx_nr = perf_sw_context, 7316 7317 .capabilities = PERF_PMU_CAP_NO_NMI, 7318 7319 .event_init = task_clock_event_init, 7320 .add = task_clock_event_add, 7321 .del = task_clock_event_del, 7322 .start = task_clock_event_start, 7323 .stop = task_clock_event_stop, 7324 .read = task_clock_event_read, 7325 }; 7326 7327 static void perf_pmu_nop_void(struct pmu *pmu) 7328 { 7329 } 7330 7331 static void perf_pmu_nop_txn(struct pmu *pmu, unsigned int flags) 7332 { 7333 } 7334 7335 static int perf_pmu_nop_int(struct pmu *pmu) 7336 { 7337 return 0; 7338 } 7339 7340 static DEFINE_PER_CPU(unsigned int, nop_txn_flags); 7341 7342 static void perf_pmu_start_txn(struct pmu *pmu, unsigned int flags) 7343 { 7344 __this_cpu_write(nop_txn_flags, flags); 7345 7346 if (flags & ~PERF_PMU_TXN_ADD) 7347 return; 7348 7349 perf_pmu_disable(pmu); 7350 } 7351 7352 static int perf_pmu_commit_txn(struct pmu *pmu) 7353 { 7354 unsigned int flags = __this_cpu_read(nop_txn_flags); 7355 7356 __this_cpu_write(nop_txn_flags, 0); 7357 7358 if (flags & ~PERF_PMU_TXN_ADD) 7359 return 0; 7360 7361 perf_pmu_enable(pmu); 7362 return 0; 7363 } 7364 7365 static void perf_pmu_cancel_txn(struct pmu *pmu) 7366 { 7367 unsigned int flags = __this_cpu_read(nop_txn_flags); 7368 7369 __this_cpu_write(nop_txn_flags, 0); 7370 7371 if (flags & ~PERF_PMU_TXN_ADD) 7372 return; 7373 7374 perf_pmu_enable(pmu); 7375 } 7376 7377 static int perf_event_idx_default(struct perf_event *event) 7378 { 7379 return 0; 7380 } 7381 7382 /* 7383 * Ensures all contexts with the same task_ctx_nr have the same 7384 * pmu_cpu_context too. 7385 */ 7386 static struct perf_cpu_context __percpu *find_pmu_context(int ctxn) 7387 { 7388 struct pmu *pmu; 7389 7390 if (ctxn < 0) 7391 return NULL; 7392 7393 list_for_each_entry(pmu, &pmus, entry) { 7394 if (pmu->task_ctx_nr == ctxn) 7395 return pmu->pmu_cpu_context; 7396 } 7397 7398 return NULL; 7399 } 7400 7401 static void update_pmu_context(struct pmu *pmu, struct pmu *old_pmu) 7402 { 7403 int cpu; 7404 7405 for_each_possible_cpu(cpu) { 7406 struct perf_cpu_context *cpuctx; 7407 7408 cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu); 7409 7410 if (cpuctx->unique_pmu == old_pmu) 7411 cpuctx->unique_pmu = pmu; 7412 } 7413 } 7414 7415 static void free_pmu_context(struct pmu *pmu) 7416 { 7417 struct pmu *i; 7418 7419 mutex_lock(&pmus_lock); 7420 /* 7421 * Like a real lame refcount. 7422 */ 7423 list_for_each_entry(i, &pmus, entry) { 7424 if (i->pmu_cpu_context == pmu->pmu_cpu_context) { 7425 update_pmu_context(i, pmu); 7426 goto out; 7427 } 7428 } 7429 7430 free_percpu(pmu->pmu_cpu_context); 7431 out: 7432 mutex_unlock(&pmus_lock); 7433 } 7434 static struct idr pmu_idr; 7435 7436 static ssize_t 7437 type_show(struct device *dev, struct device_attribute *attr, char *page) 7438 { 7439 struct pmu *pmu = dev_get_drvdata(dev); 7440 7441 return snprintf(page, PAGE_SIZE-1, "%d\n", pmu->type); 7442 } 7443 static DEVICE_ATTR_RO(type); 7444 7445 static ssize_t 7446 perf_event_mux_interval_ms_show(struct device *dev, 7447 struct device_attribute *attr, 7448 char *page) 7449 { 7450 struct pmu *pmu = dev_get_drvdata(dev); 7451 7452 return snprintf(page, PAGE_SIZE-1, "%d\n", pmu->hrtimer_interval_ms); 7453 } 7454 7455 static DEFINE_MUTEX(mux_interval_mutex); 7456 7457 static ssize_t 7458 perf_event_mux_interval_ms_store(struct device *dev, 7459 struct device_attribute *attr, 7460 const char *buf, size_t count) 7461 { 7462 struct pmu *pmu = dev_get_drvdata(dev); 7463 int timer, cpu, ret; 7464 7465 ret = kstrtoint(buf, 0, &timer); 7466 if (ret) 7467 return ret; 7468 7469 if (timer < 1) 7470 return -EINVAL; 7471 7472 /* same value, noting to do */ 7473 if (timer == pmu->hrtimer_interval_ms) 7474 return count; 7475 7476 mutex_lock(&mux_interval_mutex); 7477 pmu->hrtimer_interval_ms = timer; 7478 7479 /* update all cpuctx for this PMU */ 7480 get_online_cpus(); 7481 for_each_online_cpu(cpu) { 7482 struct perf_cpu_context *cpuctx; 7483 cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu); 7484 cpuctx->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * timer); 7485 7486 cpu_function_call(cpu, 7487 (remote_function_f)perf_mux_hrtimer_restart, cpuctx); 7488 } 7489 put_online_cpus(); 7490 mutex_unlock(&mux_interval_mutex); 7491 7492 return count; 7493 } 7494 static DEVICE_ATTR_RW(perf_event_mux_interval_ms); 7495 7496 static struct attribute *pmu_dev_attrs[] = { 7497 &dev_attr_type.attr, 7498 &dev_attr_perf_event_mux_interval_ms.attr, 7499 NULL, 7500 }; 7501 ATTRIBUTE_GROUPS(pmu_dev); 7502 7503 static int pmu_bus_running; 7504 static struct bus_type pmu_bus = { 7505 .name = "event_source", 7506 .dev_groups = pmu_dev_groups, 7507 }; 7508 7509 static void pmu_dev_release(struct device *dev) 7510 { 7511 kfree(dev); 7512 } 7513 7514 static int pmu_dev_alloc(struct pmu *pmu) 7515 { 7516 int ret = -ENOMEM; 7517 7518 pmu->dev = kzalloc(sizeof(struct device), GFP_KERNEL); 7519 if (!pmu->dev) 7520 goto out; 7521 7522 pmu->dev->groups = pmu->attr_groups; 7523 device_initialize(pmu->dev); 7524 ret = dev_set_name(pmu->dev, "%s", pmu->name); 7525 if (ret) 7526 goto free_dev; 7527 7528 dev_set_drvdata(pmu->dev, pmu); 7529 pmu->dev->bus = &pmu_bus; 7530 pmu->dev->release = pmu_dev_release; 7531 ret = device_add(pmu->dev); 7532 if (ret) 7533 goto free_dev; 7534 7535 out: 7536 return ret; 7537 7538 free_dev: 7539 put_device(pmu->dev); 7540 goto out; 7541 } 7542 7543 static struct lock_class_key cpuctx_mutex; 7544 static struct lock_class_key cpuctx_lock; 7545 7546 int perf_pmu_register(struct pmu *pmu, const char *name, int type) 7547 { 7548 int cpu, ret; 7549 7550 mutex_lock(&pmus_lock); 7551 ret = -ENOMEM; 7552 pmu->pmu_disable_count = alloc_percpu(int); 7553 if (!pmu->pmu_disable_count) 7554 goto unlock; 7555 7556 pmu->type = -1; 7557 if (!name) 7558 goto skip_type; 7559 pmu->name = name; 7560 7561 if (type < 0) { 7562 type = idr_alloc(&pmu_idr, pmu, PERF_TYPE_MAX, 0, GFP_KERNEL); 7563 if (type < 0) { 7564 ret = type; 7565 goto free_pdc; 7566 } 7567 } 7568 pmu->type = type; 7569 7570 if (pmu_bus_running) { 7571 ret = pmu_dev_alloc(pmu); 7572 if (ret) 7573 goto free_idr; 7574 } 7575 7576 skip_type: 7577 pmu->pmu_cpu_context = find_pmu_context(pmu->task_ctx_nr); 7578 if (pmu->pmu_cpu_context) 7579 goto got_cpu_context; 7580 7581 ret = -ENOMEM; 7582 pmu->pmu_cpu_context = alloc_percpu(struct perf_cpu_context); 7583 if (!pmu->pmu_cpu_context) 7584 goto free_dev; 7585 7586 for_each_possible_cpu(cpu) { 7587 struct perf_cpu_context *cpuctx; 7588 7589 cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu); 7590 __perf_event_init_context(&cpuctx->ctx); 7591 lockdep_set_class(&cpuctx->ctx.mutex, &cpuctx_mutex); 7592 lockdep_set_class(&cpuctx->ctx.lock, &cpuctx_lock); 7593 cpuctx->ctx.pmu = pmu; 7594 7595 __perf_mux_hrtimer_init(cpuctx, cpu); 7596 7597 cpuctx->unique_pmu = pmu; 7598 } 7599 7600 got_cpu_context: 7601 if (!pmu->start_txn) { 7602 if (pmu->pmu_enable) { 7603 /* 7604 * If we have pmu_enable/pmu_disable calls, install 7605 * transaction stubs that use that to try and batch 7606 * hardware accesses. 7607 */ 7608 pmu->start_txn = perf_pmu_start_txn; 7609 pmu->commit_txn = perf_pmu_commit_txn; 7610 pmu->cancel_txn = perf_pmu_cancel_txn; 7611 } else { 7612 pmu->start_txn = perf_pmu_nop_txn; 7613 pmu->commit_txn = perf_pmu_nop_int; 7614 pmu->cancel_txn = perf_pmu_nop_void; 7615 } 7616 } 7617 7618 if (!pmu->pmu_enable) { 7619 pmu->pmu_enable = perf_pmu_nop_void; 7620 pmu->pmu_disable = perf_pmu_nop_void; 7621 } 7622 7623 if (!pmu->event_idx) 7624 pmu->event_idx = perf_event_idx_default; 7625 7626 list_add_rcu(&pmu->entry, &pmus); 7627 atomic_set(&pmu->exclusive_cnt, 0); 7628 ret = 0; 7629 unlock: 7630 mutex_unlock(&pmus_lock); 7631 7632 return ret; 7633 7634 free_dev: 7635 device_del(pmu->dev); 7636 put_device(pmu->dev); 7637 7638 free_idr: 7639 if (pmu->type >= PERF_TYPE_MAX) 7640 idr_remove(&pmu_idr, pmu->type); 7641 7642 free_pdc: 7643 free_percpu(pmu->pmu_disable_count); 7644 goto unlock; 7645 } 7646 EXPORT_SYMBOL_GPL(perf_pmu_register); 7647 7648 void perf_pmu_unregister(struct pmu *pmu) 7649 { 7650 mutex_lock(&pmus_lock); 7651 list_del_rcu(&pmu->entry); 7652 mutex_unlock(&pmus_lock); 7653 7654 /* 7655 * We dereference the pmu list under both SRCU and regular RCU, so 7656 * synchronize against both of those. 7657 */ 7658 synchronize_srcu(&pmus_srcu); 7659 synchronize_rcu(); 7660 7661 free_percpu(pmu->pmu_disable_count); 7662 if (pmu->type >= PERF_TYPE_MAX) 7663 idr_remove(&pmu_idr, pmu->type); 7664 device_del(pmu->dev); 7665 put_device(pmu->dev); 7666 free_pmu_context(pmu); 7667 } 7668 EXPORT_SYMBOL_GPL(perf_pmu_unregister); 7669 7670 static int perf_try_init_event(struct pmu *pmu, struct perf_event *event) 7671 { 7672 struct perf_event_context *ctx = NULL; 7673 int ret; 7674 7675 if (!try_module_get(pmu->module)) 7676 return -ENODEV; 7677 7678 if (event->group_leader != event) { 7679 /* 7680 * This ctx->mutex can nest when we're called through 7681 * inheritance. See the perf_event_ctx_lock_nested() comment. 7682 */ 7683 ctx = perf_event_ctx_lock_nested(event->group_leader, 7684 SINGLE_DEPTH_NESTING); 7685 BUG_ON(!ctx); 7686 } 7687 7688 event->pmu = pmu; 7689 ret = pmu->event_init(event); 7690 7691 if (ctx) 7692 perf_event_ctx_unlock(event->group_leader, ctx); 7693 7694 if (ret) 7695 module_put(pmu->module); 7696 7697 return ret; 7698 } 7699 7700 static struct pmu *perf_init_event(struct perf_event *event) 7701 { 7702 struct pmu *pmu = NULL; 7703 int idx; 7704 int ret; 7705 7706 idx = srcu_read_lock(&pmus_srcu); 7707 7708 rcu_read_lock(); 7709 pmu = idr_find(&pmu_idr, event->attr.type); 7710 rcu_read_unlock(); 7711 if (pmu) { 7712 ret = perf_try_init_event(pmu, event); 7713 if (ret) 7714 pmu = ERR_PTR(ret); 7715 goto unlock; 7716 } 7717 7718 list_for_each_entry_rcu(pmu, &pmus, entry) { 7719 ret = perf_try_init_event(pmu, event); 7720 if (!ret) 7721 goto unlock; 7722 7723 if (ret != -ENOENT) { 7724 pmu = ERR_PTR(ret); 7725 goto unlock; 7726 } 7727 } 7728 pmu = ERR_PTR(-ENOENT); 7729 unlock: 7730 srcu_read_unlock(&pmus_srcu, idx); 7731 7732 return pmu; 7733 } 7734 7735 static void account_event_cpu(struct perf_event *event, int cpu) 7736 { 7737 if (event->parent) 7738 return; 7739 7740 if (is_cgroup_event(event)) 7741 atomic_inc(&per_cpu(perf_cgroup_events, cpu)); 7742 } 7743 7744 static void account_event(struct perf_event *event) 7745 { 7746 bool inc = false; 7747 7748 if (event->parent) 7749 return; 7750 7751 if (event->attach_state & PERF_ATTACH_TASK) 7752 inc = true; 7753 if (event->attr.mmap || event->attr.mmap_data) 7754 atomic_inc(&nr_mmap_events); 7755 if (event->attr.comm) 7756 atomic_inc(&nr_comm_events); 7757 if (event->attr.task) 7758 atomic_inc(&nr_task_events); 7759 if (event->attr.freq) { 7760 if (atomic_inc_return(&nr_freq_events) == 1) 7761 tick_nohz_full_kick_all(); 7762 } 7763 if (event->attr.context_switch) { 7764 atomic_inc(&nr_switch_events); 7765 inc = true; 7766 } 7767 if (has_branch_stack(event)) 7768 inc = true; 7769 if (is_cgroup_event(event)) 7770 inc = true; 7771 7772 if (inc) 7773 static_key_slow_inc(&perf_sched_events.key); 7774 7775 account_event_cpu(event, event->cpu); 7776 } 7777 7778 /* 7779 * Allocate and initialize a event structure 7780 */ 7781 static struct perf_event * 7782 perf_event_alloc(struct perf_event_attr *attr, int cpu, 7783 struct task_struct *task, 7784 struct perf_event *group_leader, 7785 struct perf_event *parent_event, 7786 perf_overflow_handler_t overflow_handler, 7787 void *context, int cgroup_fd) 7788 { 7789 struct pmu *pmu; 7790 struct perf_event *event; 7791 struct hw_perf_event *hwc; 7792 long err = -EINVAL; 7793 7794 if ((unsigned)cpu >= nr_cpu_ids) { 7795 if (!task || cpu != -1) 7796 return ERR_PTR(-EINVAL); 7797 } 7798 7799 event = kzalloc(sizeof(*event), GFP_KERNEL); 7800 if (!event) 7801 return ERR_PTR(-ENOMEM); 7802 7803 /* 7804 * Single events are their own group leaders, with an 7805 * empty sibling list: 7806 */ 7807 if (!group_leader) 7808 group_leader = event; 7809 7810 mutex_init(&event->child_mutex); 7811 INIT_LIST_HEAD(&event->child_list); 7812 7813 INIT_LIST_HEAD(&event->group_entry); 7814 INIT_LIST_HEAD(&event->event_entry); 7815 INIT_LIST_HEAD(&event->sibling_list); 7816 INIT_LIST_HEAD(&event->rb_entry); 7817 INIT_LIST_HEAD(&event->active_entry); 7818 INIT_HLIST_NODE(&event->hlist_entry); 7819 7820 7821 init_waitqueue_head(&event->waitq); 7822 init_irq_work(&event->pending, perf_pending_event); 7823 7824 mutex_init(&event->mmap_mutex); 7825 7826 atomic_long_set(&event->refcount, 1); 7827 event->cpu = cpu; 7828 event->attr = *attr; 7829 event->group_leader = group_leader; 7830 event->pmu = NULL; 7831 event->oncpu = -1; 7832 7833 event->parent = parent_event; 7834 7835 event->ns = get_pid_ns(task_active_pid_ns(current)); 7836 event->id = atomic64_inc_return(&perf_event_id); 7837 7838 event->state = PERF_EVENT_STATE_INACTIVE; 7839 7840 if (task) { 7841 event->attach_state = PERF_ATTACH_TASK; 7842 /* 7843 * XXX pmu::event_init needs to know what task to account to 7844 * and we cannot use the ctx information because we need the 7845 * pmu before we get a ctx. 7846 */ 7847 event->hw.target = task; 7848 } 7849 7850 event->clock = &local_clock; 7851 if (parent_event) 7852 event->clock = parent_event->clock; 7853 7854 if (!overflow_handler && parent_event) { 7855 overflow_handler = parent_event->overflow_handler; 7856 context = parent_event->overflow_handler_context; 7857 } 7858 7859 event->overflow_handler = overflow_handler; 7860 event->overflow_handler_context = context; 7861 7862 perf_event__state_init(event); 7863 7864 pmu = NULL; 7865 7866 hwc = &event->hw; 7867 hwc->sample_period = attr->sample_period; 7868 if (attr->freq && attr->sample_freq) 7869 hwc->sample_period = 1; 7870 hwc->last_period = hwc->sample_period; 7871 7872 local64_set(&hwc->period_left, hwc->sample_period); 7873 7874 /* 7875 * we currently do not support PERF_FORMAT_GROUP on inherited events 7876 */ 7877 if (attr->inherit && (attr->read_format & PERF_FORMAT_GROUP)) 7878 goto err_ns; 7879 7880 if (!has_branch_stack(event)) 7881 event->attr.branch_sample_type = 0; 7882 7883 if (cgroup_fd != -1) { 7884 err = perf_cgroup_connect(cgroup_fd, event, attr, group_leader); 7885 if (err) 7886 goto err_ns; 7887 } 7888 7889 pmu = perf_init_event(event); 7890 if (!pmu) 7891 goto err_ns; 7892 else if (IS_ERR(pmu)) { 7893 err = PTR_ERR(pmu); 7894 goto err_ns; 7895 } 7896 7897 err = exclusive_event_init(event); 7898 if (err) 7899 goto err_pmu; 7900 7901 if (!event->parent) { 7902 if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) { 7903 err = get_callchain_buffers(); 7904 if (err) 7905 goto err_per_task; 7906 } 7907 } 7908 7909 return event; 7910 7911 err_per_task: 7912 exclusive_event_destroy(event); 7913 7914 err_pmu: 7915 if (event->destroy) 7916 event->destroy(event); 7917 module_put(pmu->module); 7918 err_ns: 7919 if (is_cgroup_event(event)) 7920 perf_detach_cgroup(event); 7921 if (event->ns) 7922 put_pid_ns(event->ns); 7923 kfree(event); 7924 7925 return ERR_PTR(err); 7926 } 7927 7928 static int perf_copy_attr(struct perf_event_attr __user *uattr, 7929 struct perf_event_attr *attr) 7930 { 7931 u32 size; 7932 int ret; 7933 7934 if (!access_ok(VERIFY_WRITE, uattr, PERF_ATTR_SIZE_VER0)) 7935 return -EFAULT; 7936 7937 /* 7938 * zero the full structure, so that a short copy will be nice. 7939 */ 7940 memset(attr, 0, sizeof(*attr)); 7941 7942 ret = get_user(size, &uattr->size); 7943 if (ret) 7944 return ret; 7945 7946 if (size > PAGE_SIZE) /* silly large */ 7947 goto err_size; 7948 7949 if (!size) /* abi compat */ 7950 size = PERF_ATTR_SIZE_VER0; 7951 7952 if (size < PERF_ATTR_SIZE_VER0) 7953 goto err_size; 7954 7955 /* 7956 * If we're handed a bigger struct than we know of, 7957 * ensure all the unknown bits are 0 - i.e. new 7958 * user-space does not rely on any kernel feature 7959 * extensions we dont know about yet. 7960 */ 7961 if (size > sizeof(*attr)) { 7962 unsigned char __user *addr; 7963 unsigned char __user *end; 7964 unsigned char val; 7965 7966 addr = (void __user *)uattr + sizeof(*attr); 7967 end = (void __user *)uattr + size; 7968 7969 for (; addr < end; addr++) { 7970 ret = get_user(val, addr); 7971 if (ret) 7972 return ret; 7973 if (val) 7974 goto err_size; 7975 } 7976 size = sizeof(*attr); 7977 } 7978 7979 ret = copy_from_user(attr, uattr, size); 7980 if (ret) 7981 return -EFAULT; 7982 7983 if (attr->__reserved_1) 7984 return -EINVAL; 7985 7986 if (attr->sample_type & ~(PERF_SAMPLE_MAX-1)) 7987 return -EINVAL; 7988 7989 if (attr->read_format & ~(PERF_FORMAT_MAX-1)) 7990 return -EINVAL; 7991 7992 if (attr->sample_type & PERF_SAMPLE_BRANCH_STACK) { 7993 u64 mask = attr->branch_sample_type; 7994 7995 /* only using defined bits */ 7996 if (mask & ~(PERF_SAMPLE_BRANCH_MAX-1)) 7997 return -EINVAL; 7998 7999 /* at least one branch bit must be set */ 8000 if (!(mask & ~PERF_SAMPLE_BRANCH_PLM_ALL)) 8001 return -EINVAL; 8002 8003 /* propagate priv level, when not set for branch */ 8004 if (!(mask & PERF_SAMPLE_BRANCH_PLM_ALL)) { 8005 8006 /* exclude_kernel checked on syscall entry */ 8007 if (!attr->exclude_kernel) 8008 mask |= PERF_SAMPLE_BRANCH_KERNEL; 8009 8010 if (!attr->exclude_user) 8011 mask |= PERF_SAMPLE_BRANCH_USER; 8012 8013 if (!attr->exclude_hv) 8014 mask |= PERF_SAMPLE_BRANCH_HV; 8015 /* 8016 * adjust user setting (for HW filter setup) 8017 */ 8018 attr->branch_sample_type = mask; 8019 } 8020 /* privileged levels capture (kernel, hv): check permissions */ 8021 if ((mask & PERF_SAMPLE_BRANCH_PERM_PLM) 8022 && perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN)) 8023 return -EACCES; 8024 } 8025 8026 if (attr->sample_type & PERF_SAMPLE_REGS_USER) { 8027 ret = perf_reg_validate(attr->sample_regs_user); 8028 if (ret) 8029 return ret; 8030 } 8031 8032 if (attr->sample_type & PERF_SAMPLE_STACK_USER) { 8033 if (!arch_perf_have_user_stack_dump()) 8034 return -ENOSYS; 8035 8036 /* 8037 * We have __u32 type for the size, but so far 8038 * we can only use __u16 as maximum due to the 8039 * __u16 sample size limit. 8040 */ 8041 if (attr->sample_stack_user >= USHRT_MAX) 8042 ret = -EINVAL; 8043 else if (!IS_ALIGNED(attr->sample_stack_user, sizeof(u64))) 8044 ret = -EINVAL; 8045 } 8046 8047 if (attr->sample_type & PERF_SAMPLE_REGS_INTR) 8048 ret = perf_reg_validate(attr->sample_regs_intr); 8049 out: 8050 return ret; 8051 8052 err_size: 8053 put_user(sizeof(*attr), &uattr->size); 8054 ret = -E2BIG; 8055 goto out; 8056 } 8057 8058 static int 8059 perf_event_set_output(struct perf_event *event, struct perf_event *output_event) 8060 { 8061 struct ring_buffer *rb = NULL; 8062 int ret = -EINVAL; 8063 8064 if (!output_event) 8065 goto set; 8066 8067 /* don't allow circular references */ 8068 if (event == output_event) 8069 goto out; 8070 8071 /* 8072 * Don't allow cross-cpu buffers 8073 */ 8074 if (output_event->cpu != event->cpu) 8075 goto out; 8076 8077 /* 8078 * If its not a per-cpu rb, it must be the same task. 8079 */ 8080 if (output_event->cpu == -1 && output_event->ctx != event->ctx) 8081 goto out; 8082 8083 /* 8084 * Mixing clocks in the same buffer is trouble you don't need. 8085 */ 8086 if (output_event->clock != event->clock) 8087 goto out; 8088 8089 /* 8090 * If both events generate aux data, they must be on the same PMU 8091 */ 8092 if (has_aux(event) && has_aux(output_event) && 8093 event->pmu != output_event->pmu) 8094 goto out; 8095 8096 set: 8097 mutex_lock(&event->mmap_mutex); 8098 /* Can't redirect output if we've got an active mmap() */ 8099 if (atomic_read(&event->mmap_count)) 8100 goto unlock; 8101 8102 if (output_event) { 8103 /* get the rb we want to redirect to */ 8104 rb = ring_buffer_get(output_event); 8105 if (!rb) 8106 goto unlock; 8107 } 8108 8109 ring_buffer_attach(event, rb); 8110 8111 ret = 0; 8112 unlock: 8113 mutex_unlock(&event->mmap_mutex); 8114 8115 out: 8116 return ret; 8117 } 8118 8119 static void mutex_lock_double(struct mutex *a, struct mutex *b) 8120 { 8121 if (b < a) 8122 swap(a, b); 8123 8124 mutex_lock(a); 8125 mutex_lock_nested(b, SINGLE_DEPTH_NESTING); 8126 } 8127 8128 static int perf_event_set_clock(struct perf_event *event, clockid_t clk_id) 8129 { 8130 bool nmi_safe = false; 8131 8132 switch (clk_id) { 8133 case CLOCK_MONOTONIC: 8134 event->clock = &ktime_get_mono_fast_ns; 8135 nmi_safe = true; 8136 break; 8137 8138 case CLOCK_MONOTONIC_RAW: 8139 event->clock = &ktime_get_raw_fast_ns; 8140 nmi_safe = true; 8141 break; 8142 8143 case CLOCK_REALTIME: 8144 event->clock = &ktime_get_real_ns; 8145 break; 8146 8147 case CLOCK_BOOTTIME: 8148 event->clock = &ktime_get_boot_ns; 8149 break; 8150 8151 case CLOCK_TAI: 8152 event->clock = &ktime_get_tai_ns; 8153 break; 8154 8155 default: 8156 return -EINVAL; 8157 } 8158 8159 if (!nmi_safe && !(event->pmu->capabilities & PERF_PMU_CAP_NO_NMI)) 8160 return -EINVAL; 8161 8162 return 0; 8163 } 8164 8165 /** 8166 * sys_perf_event_open - open a performance event, associate it to a task/cpu 8167 * 8168 * @attr_uptr: event_id type attributes for monitoring/sampling 8169 * @pid: target pid 8170 * @cpu: target cpu 8171 * @group_fd: group leader event fd 8172 */ 8173 SYSCALL_DEFINE5(perf_event_open, 8174 struct perf_event_attr __user *, attr_uptr, 8175 pid_t, pid, int, cpu, int, group_fd, unsigned long, flags) 8176 { 8177 struct perf_event *group_leader = NULL, *output_event = NULL; 8178 struct perf_event *event, *sibling; 8179 struct perf_event_attr attr; 8180 struct perf_event_context *ctx, *uninitialized_var(gctx); 8181 struct file *event_file = NULL; 8182 struct fd group = {NULL, 0}; 8183 struct task_struct *task = NULL; 8184 struct pmu *pmu; 8185 int event_fd; 8186 int move_group = 0; 8187 int err; 8188 int f_flags = O_RDWR; 8189 int cgroup_fd = -1; 8190 8191 /* for future expandability... */ 8192 if (flags & ~PERF_FLAG_ALL) 8193 return -EINVAL; 8194 8195 err = perf_copy_attr(attr_uptr, &attr); 8196 if (err) 8197 return err; 8198 8199 if (!attr.exclude_kernel) { 8200 if (perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN)) 8201 return -EACCES; 8202 } 8203 8204 if (attr.freq) { 8205 if (attr.sample_freq > sysctl_perf_event_sample_rate) 8206 return -EINVAL; 8207 } else { 8208 if (attr.sample_period & (1ULL << 63)) 8209 return -EINVAL; 8210 } 8211 8212 /* 8213 * In cgroup mode, the pid argument is used to pass the fd 8214 * opened to the cgroup directory in cgroupfs. The cpu argument 8215 * designates the cpu on which to monitor threads from that 8216 * cgroup. 8217 */ 8218 if ((flags & PERF_FLAG_PID_CGROUP) && (pid == -1 || cpu == -1)) 8219 return -EINVAL; 8220 8221 if (flags & PERF_FLAG_FD_CLOEXEC) 8222 f_flags |= O_CLOEXEC; 8223 8224 event_fd = get_unused_fd_flags(f_flags); 8225 if (event_fd < 0) 8226 return event_fd; 8227 8228 if (group_fd != -1) { 8229 err = perf_fget_light(group_fd, &group); 8230 if (err) 8231 goto err_fd; 8232 group_leader = group.file->private_data; 8233 if (flags & PERF_FLAG_FD_OUTPUT) 8234 output_event = group_leader; 8235 if (flags & PERF_FLAG_FD_NO_GROUP) 8236 group_leader = NULL; 8237 } 8238 8239 if (pid != -1 && !(flags & PERF_FLAG_PID_CGROUP)) { 8240 task = find_lively_task_by_vpid(pid); 8241 if (IS_ERR(task)) { 8242 err = PTR_ERR(task); 8243 goto err_group_fd; 8244 } 8245 } 8246 8247 if (task && group_leader && 8248 group_leader->attr.inherit != attr.inherit) { 8249 err = -EINVAL; 8250 goto err_task; 8251 } 8252 8253 get_online_cpus(); 8254 8255 if (flags & PERF_FLAG_PID_CGROUP) 8256 cgroup_fd = pid; 8257 8258 event = perf_event_alloc(&attr, cpu, task, group_leader, NULL, 8259 NULL, NULL, cgroup_fd); 8260 if (IS_ERR(event)) { 8261 err = PTR_ERR(event); 8262 goto err_cpus; 8263 } 8264 8265 if (is_sampling_event(event)) { 8266 if (event->pmu->capabilities & PERF_PMU_CAP_NO_INTERRUPT) { 8267 err = -ENOTSUPP; 8268 goto err_alloc; 8269 } 8270 } 8271 8272 account_event(event); 8273 8274 /* 8275 * Special case software events and allow them to be part of 8276 * any hardware group. 8277 */ 8278 pmu = event->pmu; 8279 8280 if (attr.use_clockid) { 8281 err = perf_event_set_clock(event, attr.clockid); 8282 if (err) 8283 goto err_alloc; 8284 } 8285 8286 if (group_leader && 8287 (is_software_event(event) != is_software_event(group_leader))) { 8288 if (is_software_event(event)) { 8289 /* 8290 * If event and group_leader are not both a software 8291 * event, and event is, then group leader is not. 8292 * 8293 * Allow the addition of software events to !software 8294 * groups, this is safe because software events never 8295 * fail to schedule. 8296 */ 8297 pmu = group_leader->pmu; 8298 } else if (is_software_event(group_leader) && 8299 (group_leader->group_flags & PERF_GROUP_SOFTWARE)) { 8300 /* 8301 * In case the group is a pure software group, and we 8302 * try to add a hardware event, move the whole group to 8303 * the hardware context. 8304 */ 8305 move_group = 1; 8306 } 8307 } 8308 8309 /* 8310 * Get the target context (task or percpu): 8311 */ 8312 ctx = find_get_context(pmu, task, event); 8313 if (IS_ERR(ctx)) { 8314 err = PTR_ERR(ctx); 8315 goto err_alloc; 8316 } 8317 8318 if ((pmu->capabilities & PERF_PMU_CAP_EXCLUSIVE) && group_leader) { 8319 err = -EBUSY; 8320 goto err_context; 8321 } 8322 8323 if (task) { 8324 put_task_struct(task); 8325 task = NULL; 8326 } 8327 8328 /* 8329 * Look up the group leader (we will attach this event to it): 8330 */ 8331 if (group_leader) { 8332 err = -EINVAL; 8333 8334 /* 8335 * Do not allow a recursive hierarchy (this new sibling 8336 * becoming part of another group-sibling): 8337 */ 8338 if (group_leader->group_leader != group_leader) 8339 goto err_context; 8340 8341 /* All events in a group should have the same clock */ 8342 if (group_leader->clock != event->clock) 8343 goto err_context; 8344 8345 /* 8346 * Do not allow to attach to a group in a different 8347 * task or CPU context: 8348 */ 8349 if (move_group) { 8350 /* 8351 * Make sure we're both on the same task, or both 8352 * per-cpu events. 8353 */ 8354 if (group_leader->ctx->task != ctx->task) 8355 goto err_context; 8356 8357 /* 8358 * Make sure we're both events for the same CPU; 8359 * grouping events for different CPUs is broken; since 8360 * you can never concurrently schedule them anyhow. 8361 */ 8362 if (group_leader->cpu != event->cpu) 8363 goto err_context; 8364 } else { 8365 if (group_leader->ctx != ctx) 8366 goto err_context; 8367 } 8368 8369 /* 8370 * Only a group leader can be exclusive or pinned 8371 */ 8372 if (attr.exclusive || attr.pinned) 8373 goto err_context; 8374 } 8375 8376 if (output_event) { 8377 err = perf_event_set_output(event, output_event); 8378 if (err) 8379 goto err_context; 8380 } 8381 8382 event_file = anon_inode_getfile("[perf_event]", &perf_fops, event, 8383 f_flags); 8384 if (IS_ERR(event_file)) { 8385 err = PTR_ERR(event_file); 8386 goto err_context; 8387 } 8388 8389 if (move_group) { 8390 gctx = group_leader->ctx; 8391 mutex_lock_double(&gctx->mutex, &ctx->mutex); 8392 } else { 8393 mutex_lock(&ctx->mutex); 8394 } 8395 8396 if (!perf_event_validate_size(event)) { 8397 err = -E2BIG; 8398 goto err_locked; 8399 } 8400 8401 /* 8402 * Must be under the same ctx::mutex as perf_install_in_context(), 8403 * because we need to serialize with concurrent event creation. 8404 */ 8405 if (!exclusive_event_installable(event, ctx)) { 8406 /* exclusive and group stuff are assumed mutually exclusive */ 8407 WARN_ON_ONCE(move_group); 8408 8409 err = -EBUSY; 8410 goto err_locked; 8411 } 8412 8413 WARN_ON_ONCE(ctx->parent_ctx); 8414 8415 if (move_group) { 8416 /* 8417 * See perf_event_ctx_lock() for comments on the details 8418 * of swizzling perf_event::ctx. 8419 */ 8420 perf_remove_from_context(group_leader, 0); 8421 8422 list_for_each_entry(sibling, &group_leader->sibling_list, 8423 group_entry) { 8424 perf_remove_from_context(sibling, 0); 8425 put_ctx(gctx); 8426 } 8427 8428 /* 8429 * Wait for everybody to stop referencing the events through 8430 * the old lists, before installing it on new lists. 8431 */ 8432 synchronize_rcu(); 8433 8434 /* 8435 * Install the group siblings before the group leader. 8436 * 8437 * Because a group leader will try and install the entire group 8438 * (through the sibling list, which is still in-tact), we can 8439 * end up with siblings installed in the wrong context. 8440 * 8441 * By installing siblings first we NO-OP because they're not 8442 * reachable through the group lists. 8443 */ 8444 list_for_each_entry(sibling, &group_leader->sibling_list, 8445 group_entry) { 8446 perf_event__state_init(sibling); 8447 perf_install_in_context(ctx, sibling, sibling->cpu); 8448 get_ctx(ctx); 8449 } 8450 8451 /* 8452 * Removing from the context ends up with disabled 8453 * event. What we want here is event in the initial 8454 * startup state, ready to be add into new context. 8455 */ 8456 perf_event__state_init(group_leader); 8457 perf_install_in_context(ctx, group_leader, group_leader->cpu); 8458 get_ctx(ctx); 8459 8460 /* 8461 * Now that all events are installed in @ctx, nothing 8462 * references @gctx anymore, so drop the last reference we have 8463 * on it. 8464 */ 8465 put_ctx(gctx); 8466 } 8467 8468 /* 8469 * Precalculate sample_data sizes; do while holding ctx::mutex such 8470 * that we're serialized against further additions and before 8471 * perf_install_in_context() which is the point the event is active and 8472 * can use these values. 8473 */ 8474 perf_event__header_size(event); 8475 perf_event__id_header_size(event); 8476 8477 event->owner = current; 8478 8479 perf_install_in_context(ctx, event, event->cpu); 8480 perf_unpin_context(ctx); 8481 8482 if (move_group) 8483 mutex_unlock(&gctx->mutex); 8484 mutex_unlock(&ctx->mutex); 8485 8486 put_online_cpus(); 8487 8488 mutex_lock(¤t->perf_event_mutex); 8489 list_add_tail(&event->owner_entry, ¤t->perf_event_list); 8490 mutex_unlock(¤t->perf_event_mutex); 8491 8492 /* 8493 * Drop the reference on the group_event after placing the 8494 * new event on the sibling_list. This ensures destruction 8495 * of the group leader will find the pointer to itself in 8496 * perf_group_detach(). 8497 */ 8498 fdput(group); 8499 fd_install(event_fd, event_file); 8500 return event_fd; 8501 8502 err_locked: 8503 if (move_group) 8504 mutex_unlock(&gctx->mutex); 8505 mutex_unlock(&ctx->mutex); 8506 /* err_file: */ 8507 fput(event_file); 8508 err_context: 8509 perf_unpin_context(ctx); 8510 put_ctx(ctx); 8511 err_alloc: 8512 free_event(event); 8513 err_cpus: 8514 put_online_cpus(); 8515 err_task: 8516 if (task) 8517 put_task_struct(task); 8518 err_group_fd: 8519 fdput(group); 8520 err_fd: 8521 put_unused_fd(event_fd); 8522 return err; 8523 } 8524 8525 /** 8526 * perf_event_create_kernel_counter 8527 * 8528 * @attr: attributes of the counter to create 8529 * @cpu: cpu in which the counter is bound 8530 * @task: task to profile (NULL for percpu) 8531 */ 8532 struct perf_event * 8533 perf_event_create_kernel_counter(struct perf_event_attr *attr, int cpu, 8534 struct task_struct *task, 8535 perf_overflow_handler_t overflow_handler, 8536 void *context) 8537 { 8538 struct perf_event_context *ctx; 8539 struct perf_event *event; 8540 int err; 8541 8542 /* 8543 * Get the target context (task or percpu): 8544 */ 8545 8546 event = perf_event_alloc(attr, cpu, task, NULL, NULL, 8547 overflow_handler, context, -1); 8548 if (IS_ERR(event)) { 8549 err = PTR_ERR(event); 8550 goto err; 8551 } 8552 8553 /* Mark owner so we could distinguish it from user events. */ 8554 event->owner = TASK_TOMBSTONE; 8555 8556 account_event(event); 8557 8558 ctx = find_get_context(event->pmu, task, event); 8559 if (IS_ERR(ctx)) { 8560 err = PTR_ERR(ctx); 8561 goto err_free; 8562 } 8563 8564 WARN_ON_ONCE(ctx->parent_ctx); 8565 mutex_lock(&ctx->mutex); 8566 if (!exclusive_event_installable(event, ctx)) { 8567 mutex_unlock(&ctx->mutex); 8568 perf_unpin_context(ctx); 8569 put_ctx(ctx); 8570 err = -EBUSY; 8571 goto err_free; 8572 } 8573 8574 perf_install_in_context(ctx, event, cpu); 8575 perf_unpin_context(ctx); 8576 mutex_unlock(&ctx->mutex); 8577 8578 return event; 8579 8580 err_free: 8581 free_event(event); 8582 err: 8583 return ERR_PTR(err); 8584 } 8585 EXPORT_SYMBOL_GPL(perf_event_create_kernel_counter); 8586 8587 void perf_pmu_migrate_context(struct pmu *pmu, int src_cpu, int dst_cpu) 8588 { 8589 struct perf_event_context *src_ctx; 8590 struct perf_event_context *dst_ctx; 8591 struct perf_event *event, *tmp; 8592 LIST_HEAD(events); 8593 8594 src_ctx = &per_cpu_ptr(pmu->pmu_cpu_context, src_cpu)->ctx; 8595 dst_ctx = &per_cpu_ptr(pmu->pmu_cpu_context, dst_cpu)->ctx; 8596 8597 /* 8598 * See perf_event_ctx_lock() for comments on the details 8599 * of swizzling perf_event::ctx. 8600 */ 8601 mutex_lock_double(&src_ctx->mutex, &dst_ctx->mutex); 8602 list_for_each_entry_safe(event, tmp, &src_ctx->event_list, 8603 event_entry) { 8604 perf_remove_from_context(event, 0); 8605 unaccount_event_cpu(event, src_cpu); 8606 put_ctx(src_ctx); 8607 list_add(&event->migrate_entry, &events); 8608 } 8609 8610 /* 8611 * Wait for the events to quiesce before re-instating them. 8612 */ 8613 synchronize_rcu(); 8614 8615 /* 8616 * Re-instate events in 2 passes. 8617 * 8618 * Skip over group leaders and only install siblings on this first 8619 * pass, siblings will not get enabled without a leader, however a 8620 * leader will enable its siblings, even if those are still on the old 8621 * context. 8622 */ 8623 list_for_each_entry_safe(event, tmp, &events, migrate_entry) { 8624 if (event->group_leader == event) 8625 continue; 8626 8627 list_del(&event->migrate_entry); 8628 if (event->state >= PERF_EVENT_STATE_OFF) 8629 event->state = PERF_EVENT_STATE_INACTIVE; 8630 account_event_cpu(event, dst_cpu); 8631 perf_install_in_context(dst_ctx, event, dst_cpu); 8632 get_ctx(dst_ctx); 8633 } 8634 8635 /* 8636 * Once all the siblings are setup properly, install the group leaders 8637 * to make it go. 8638 */ 8639 list_for_each_entry_safe(event, tmp, &events, migrate_entry) { 8640 list_del(&event->migrate_entry); 8641 if (event->state >= PERF_EVENT_STATE_OFF) 8642 event->state = PERF_EVENT_STATE_INACTIVE; 8643 account_event_cpu(event, dst_cpu); 8644 perf_install_in_context(dst_ctx, event, dst_cpu); 8645 get_ctx(dst_ctx); 8646 } 8647 mutex_unlock(&dst_ctx->mutex); 8648 mutex_unlock(&src_ctx->mutex); 8649 } 8650 EXPORT_SYMBOL_GPL(perf_pmu_migrate_context); 8651 8652 static void sync_child_event(struct perf_event *child_event, 8653 struct task_struct *child) 8654 { 8655 struct perf_event *parent_event = child_event->parent; 8656 u64 child_val; 8657 8658 if (child_event->attr.inherit_stat) 8659 perf_event_read_event(child_event, child); 8660 8661 child_val = perf_event_count(child_event); 8662 8663 /* 8664 * Add back the child's count to the parent's count: 8665 */ 8666 atomic64_add(child_val, &parent_event->child_count); 8667 atomic64_add(child_event->total_time_enabled, 8668 &parent_event->child_total_time_enabled); 8669 atomic64_add(child_event->total_time_running, 8670 &parent_event->child_total_time_running); 8671 } 8672 8673 static void 8674 perf_event_exit_event(struct perf_event *child_event, 8675 struct perf_event_context *child_ctx, 8676 struct task_struct *child) 8677 { 8678 struct perf_event *parent_event = child_event->parent; 8679 8680 /* 8681 * Do not destroy the 'original' grouping; because of the context 8682 * switch optimization the original events could've ended up in a 8683 * random child task. 8684 * 8685 * If we were to destroy the original group, all group related 8686 * operations would cease to function properly after this random 8687 * child dies. 8688 * 8689 * Do destroy all inherited groups, we don't care about those 8690 * and being thorough is better. 8691 */ 8692 raw_spin_lock_irq(&child_ctx->lock); 8693 WARN_ON_ONCE(child_ctx->is_active); 8694 8695 if (parent_event) 8696 perf_group_detach(child_event); 8697 list_del_event(child_event, child_ctx); 8698 child_event->state = PERF_EVENT_STATE_EXIT; /* see perf_event_release_kernel() */ 8699 raw_spin_unlock_irq(&child_ctx->lock); 8700 8701 /* 8702 * Parent events are governed by their filedesc, retain them. 8703 */ 8704 if (!parent_event) { 8705 perf_event_wakeup(child_event); 8706 return; 8707 } 8708 /* 8709 * Child events can be cleaned up. 8710 */ 8711 8712 sync_child_event(child_event, child); 8713 8714 /* 8715 * Remove this event from the parent's list 8716 */ 8717 WARN_ON_ONCE(parent_event->ctx->parent_ctx); 8718 mutex_lock(&parent_event->child_mutex); 8719 list_del_init(&child_event->child_list); 8720 mutex_unlock(&parent_event->child_mutex); 8721 8722 /* 8723 * Kick perf_poll() for is_event_hup(). 8724 */ 8725 perf_event_wakeup(parent_event); 8726 free_event(child_event); 8727 put_event(parent_event); 8728 } 8729 8730 static void perf_event_exit_task_context(struct task_struct *child, int ctxn) 8731 { 8732 struct perf_event_context *child_ctx, *clone_ctx = NULL; 8733 struct perf_event *child_event, *next; 8734 8735 WARN_ON_ONCE(child != current); 8736 8737 child_ctx = perf_pin_task_context(child, ctxn); 8738 if (!child_ctx) 8739 return; 8740 8741 /* 8742 * In order to reduce the amount of tricky in ctx tear-down, we hold 8743 * ctx::mutex over the entire thing. This serializes against almost 8744 * everything that wants to access the ctx. 8745 * 8746 * The exception is sys_perf_event_open() / 8747 * perf_event_create_kernel_count() which does find_get_context() 8748 * without ctx::mutex (it cannot because of the move_group double mutex 8749 * lock thing). See the comments in perf_install_in_context(). 8750 */ 8751 mutex_lock(&child_ctx->mutex); 8752 8753 /* 8754 * In a single ctx::lock section, de-schedule the events and detach the 8755 * context from the task such that we cannot ever get it scheduled back 8756 * in. 8757 */ 8758 raw_spin_lock_irq(&child_ctx->lock); 8759 task_ctx_sched_out(__get_cpu_context(child_ctx), child_ctx); 8760 8761 /* 8762 * Now that the context is inactive, destroy the task <-> ctx relation 8763 * and mark the context dead. 8764 */ 8765 RCU_INIT_POINTER(child->perf_event_ctxp[ctxn], NULL); 8766 put_ctx(child_ctx); /* cannot be last */ 8767 WRITE_ONCE(child_ctx->task, TASK_TOMBSTONE); 8768 put_task_struct(current); /* cannot be last */ 8769 8770 clone_ctx = unclone_ctx(child_ctx); 8771 raw_spin_unlock_irq(&child_ctx->lock); 8772 8773 if (clone_ctx) 8774 put_ctx(clone_ctx); 8775 8776 /* 8777 * Report the task dead after unscheduling the events so that we 8778 * won't get any samples after PERF_RECORD_EXIT. We can however still 8779 * get a few PERF_RECORD_READ events. 8780 */ 8781 perf_event_task(child, child_ctx, 0); 8782 8783 list_for_each_entry_safe(child_event, next, &child_ctx->event_list, event_entry) 8784 perf_event_exit_event(child_event, child_ctx, child); 8785 8786 mutex_unlock(&child_ctx->mutex); 8787 8788 put_ctx(child_ctx); 8789 } 8790 8791 /* 8792 * When a child task exits, feed back event values to parent events. 8793 */ 8794 void perf_event_exit_task(struct task_struct *child) 8795 { 8796 struct perf_event *event, *tmp; 8797 int ctxn; 8798 8799 mutex_lock(&child->perf_event_mutex); 8800 list_for_each_entry_safe(event, tmp, &child->perf_event_list, 8801 owner_entry) { 8802 list_del_init(&event->owner_entry); 8803 8804 /* 8805 * Ensure the list deletion is visible before we clear 8806 * the owner, closes a race against perf_release() where 8807 * we need to serialize on the owner->perf_event_mutex. 8808 */ 8809 smp_store_release(&event->owner, NULL); 8810 } 8811 mutex_unlock(&child->perf_event_mutex); 8812 8813 for_each_task_context_nr(ctxn) 8814 perf_event_exit_task_context(child, ctxn); 8815 8816 /* 8817 * The perf_event_exit_task_context calls perf_event_task 8818 * with child's task_ctx, which generates EXIT events for 8819 * child contexts and sets child->perf_event_ctxp[] to NULL. 8820 * At this point we need to send EXIT events to cpu contexts. 8821 */ 8822 perf_event_task(child, NULL, 0); 8823 } 8824 8825 static void perf_free_event(struct perf_event *event, 8826 struct perf_event_context *ctx) 8827 { 8828 struct perf_event *parent = event->parent; 8829 8830 if (WARN_ON_ONCE(!parent)) 8831 return; 8832 8833 mutex_lock(&parent->child_mutex); 8834 list_del_init(&event->child_list); 8835 mutex_unlock(&parent->child_mutex); 8836 8837 put_event(parent); 8838 8839 raw_spin_lock_irq(&ctx->lock); 8840 perf_group_detach(event); 8841 list_del_event(event, ctx); 8842 raw_spin_unlock_irq(&ctx->lock); 8843 free_event(event); 8844 } 8845 8846 /* 8847 * Free an unexposed, unused context as created by inheritance by 8848 * perf_event_init_task below, used by fork() in case of fail. 8849 * 8850 * Not all locks are strictly required, but take them anyway to be nice and 8851 * help out with the lockdep assertions. 8852 */ 8853 void perf_event_free_task(struct task_struct *task) 8854 { 8855 struct perf_event_context *ctx; 8856 struct perf_event *event, *tmp; 8857 int ctxn; 8858 8859 for_each_task_context_nr(ctxn) { 8860 ctx = task->perf_event_ctxp[ctxn]; 8861 if (!ctx) 8862 continue; 8863 8864 mutex_lock(&ctx->mutex); 8865 again: 8866 list_for_each_entry_safe(event, tmp, &ctx->pinned_groups, 8867 group_entry) 8868 perf_free_event(event, ctx); 8869 8870 list_for_each_entry_safe(event, tmp, &ctx->flexible_groups, 8871 group_entry) 8872 perf_free_event(event, ctx); 8873 8874 if (!list_empty(&ctx->pinned_groups) || 8875 !list_empty(&ctx->flexible_groups)) 8876 goto again; 8877 8878 mutex_unlock(&ctx->mutex); 8879 8880 put_ctx(ctx); 8881 } 8882 } 8883 8884 void perf_event_delayed_put(struct task_struct *task) 8885 { 8886 int ctxn; 8887 8888 for_each_task_context_nr(ctxn) 8889 WARN_ON_ONCE(task->perf_event_ctxp[ctxn]); 8890 } 8891 8892 struct file *perf_event_get(unsigned int fd) 8893 { 8894 struct file *file; 8895 8896 file = fget_raw(fd); 8897 if (!file) 8898 return ERR_PTR(-EBADF); 8899 8900 if (file->f_op != &perf_fops) { 8901 fput(file); 8902 return ERR_PTR(-EBADF); 8903 } 8904 8905 return file; 8906 } 8907 8908 const struct perf_event_attr *perf_event_attrs(struct perf_event *event) 8909 { 8910 if (!event) 8911 return ERR_PTR(-EINVAL); 8912 8913 return &event->attr; 8914 } 8915 8916 /* 8917 * inherit a event from parent task to child task: 8918 */ 8919 static struct perf_event * 8920 inherit_event(struct perf_event *parent_event, 8921 struct task_struct *parent, 8922 struct perf_event_context *parent_ctx, 8923 struct task_struct *child, 8924 struct perf_event *group_leader, 8925 struct perf_event_context *child_ctx) 8926 { 8927 enum perf_event_active_state parent_state = parent_event->state; 8928 struct perf_event *child_event; 8929 unsigned long flags; 8930 8931 /* 8932 * Instead of creating recursive hierarchies of events, 8933 * we link inherited events back to the original parent, 8934 * which has a filp for sure, which we use as the reference 8935 * count: 8936 */ 8937 if (parent_event->parent) 8938 parent_event = parent_event->parent; 8939 8940 child_event = perf_event_alloc(&parent_event->attr, 8941 parent_event->cpu, 8942 child, 8943 group_leader, parent_event, 8944 NULL, NULL, -1); 8945 if (IS_ERR(child_event)) 8946 return child_event; 8947 8948 /* 8949 * is_orphaned_event() and list_add_tail(&parent_event->child_list) 8950 * must be under the same lock in order to serialize against 8951 * perf_event_release_kernel(), such that either we must observe 8952 * is_orphaned_event() or they will observe us on the child_list. 8953 */ 8954 mutex_lock(&parent_event->child_mutex); 8955 if (is_orphaned_event(parent_event) || 8956 !atomic_long_inc_not_zero(&parent_event->refcount)) { 8957 mutex_unlock(&parent_event->child_mutex); 8958 free_event(child_event); 8959 return NULL; 8960 } 8961 8962 get_ctx(child_ctx); 8963 8964 /* 8965 * Make the child state follow the state of the parent event, 8966 * not its attr.disabled bit. We hold the parent's mutex, 8967 * so we won't race with perf_event_{en, dis}able_family. 8968 */ 8969 if (parent_state >= PERF_EVENT_STATE_INACTIVE) 8970 child_event->state = PERF_EVENT_STATE_INACTIVE; 8971 else 8972 child_event->state = PERF_EVENT_STATE_OFF; 8973 8974 if (parent_event->attr.freq) { 8975 u64 sample_period = parent_event->hw.sample_period; 8976 struct hw_perf_event *hwc = &child_event->hw; 8977 8978 hwc->sample_period = sample_period; 8979 hwc->last_period = sample_period; 8980 8981 local64_set(&hwc->period_left, sample_period); 8982 } 8983 8984 child_event->ctx = child_ctx; 8985 child_event->overflow_handler = parent_event->overflow_handler; 8986 child_event->overflow_handler_context 8987 = parent_event->overflow_handler_context; 8988 8989 /* 8990 * Precalculate sample_data sizes 8991 */ 8992 perf_event__header_size(child_event); 8993 perf_event__id_header_size(child_event); 8994 8995 /* 8996 * Link it up in the child's context: 8997 */ 8998 raw_spin_lock_irqsave(&child_ctx->lock, flags); 8999 add_event_to_ctx(child_event, child_ctx); 9000 raw_spin_unlock_irqrestore(&child_ctx->lock, flags); 9001 9002 /* 9003 * Link this into the parent event's child list 9004 */ 9005 list_add_tail(&child_event->child_list, &parent_event->child_list); 9006 mutex_unlock(&parent_event->child_mutex); 9007 9008 return child_event; 9009 } 9010 9011 static int inherit_group(struct perf_event *parent_event, 9012 struct task_struct *parent, 9013 struct perf_event_context *parent_ctx, 9014 struct task_struct *child, 9015 struct perf_event_context *child_ctx) 9016 { 9017 struct perf_event *leader; 9018 struct perf_event *sub; 9019 struct perf_event *child_ctr; 9020 9021 leader = inherit_event(parent_event, parent, parent_ctx, 9022 child, NULL, child_ctx); 9023 if (IS_ERR(leader)) 9024 return PTR_ERR(leader); 9025 list_for_each_entry(sub, &parent_event->sibling_list, group_entry) { 9026 child_ctr = inherit_event(sub, parent, parent_ctx, 9027 child, leader, child_ctx); 9028 if (IS_ERR(child_ctr)) 9029 return PTR_ERR(child_ctr); 9030 } 9031 return 0; 9032 } 9033 9034 static int 9035 inherit_task_group(struct perf_event *event, struct task_struct *parent, 9036 struct perf_event_context *parent_ctx, 9037 struct task_struct *child, int ctxn, 9038 int *inherited_all) 9039 { 9040 int ret; 9041 struct perf_event_context *child_ctx; 9042 9043 if (!event->attr.inherit) { 9044 *inherited_all = 0; 9045 return 0; 9046 } 9047 9048 child_ctx = child->perf_event_ctxp[ctxn]; 9049 if (!child_ctx) { 9050 /* 9051 * This is executed from the parent task context, so 9052 * inherit events that have been marked for cloning. 9053 * First allocate and initialize a context for the 9054 * child. 9055 */ 9056 9057 child_ctx = alloc_perf_context(parent_ctx->pmu, child); 9058 if (!child_ctx) 9059 return -ENOMEM; 9060 9061 child->perf_event_ctxp[ctxn] = child_ctx; 9062 } 9063 9064 ret = inherit_group(event, parent, parent_ctx, 9065 child, child_ctx); 9066 9067 if (ret) 9068 *inherited_all = 0; 9069 9070 return ret; 9071 } 9072 9073 /* 9074 * Initialize the perf_event context in task_struct 9075 */ 9076 static int perf_event_init_context(struct task_struct *child, int ctxn) 9077 { 9078 struct perf_event_context *child_ctx, *parent_ctx; 9079 struct perf_event_context *cloned_ctx; 9080 struct perf_event *event; 9081 struct task_struct *parent = current; 9082 int inherited_all = 1; 9083 unsigned long flags; 9084 int ret = 0; 9085 9086 if (likely(!parent->perf_event_ctxp[ctxn])) 9087 return 0; 9088 9089 /* 9090 * If the parent's context is a clone, pin it so it won't get 9091 * swapped under us. 9092 */ 9093 parent_ctx = perf_pin_task_context(parent, ctxn); 9094 if (!parent_ctx) 9095 return 0; 9096 9097 /* 9098 * No need to check if parent_ctx != NULL here; since we saw 9099 * it non-NULL earlier, the only reason for it to become NULL 9100 * is if we exit, and since we're currently in the middle of 9101 * a fork we can't be exiting at the same time. 9102 */ 9103 9104 /* 9105 * Lock the parent list. No need to lock the child - not PID 9106 * hashed yet and not running, so nobody can access it. 9107 */ 9108 mutex_lock(&parent_ctx->mutex); 9109 9110 /* 9111 * We dont have to disable NMIs - we are only looking at 9112 * the list, not manipulating it: 9113 */ 9114 list_for_each_entry(event, &parent_ctx->pinned_groups, group_entry) { 9115 ret = inherit_task_group(event, parent, parent_ctx, 9116 child, ctxn, &inherited_all); 9117 if (ret) 9118 break; 9119 } 9120 9121 /* 9122 * We can't hold ctx->lock when iterating the ->flexible_group list due 9123 * to allocations, but we need to prevent rotation because 9124 * rotate_ctx() will change the list from interrupt context. 9125 */ 9126 raw_spin_lock_irqsave(&parent_ctx->lock, flags); 9127 parent_ctx->rotate_disable = 1; 9128 raw_spin_unlock_irqrestore(&parent_ctx->lock, flags); 9129 9130 list_for_each_entry(event, &parent_ctx->flexible_groups, group_entry) { 9131 ret = inherit_task_group(event, parent, parent_ctx, 9132 child, ctxn, &inherited_all); 9133 if (ret) 9134 break; 9135 } 9136 9137 raw_spin_lock_irqsave(&parent_ctx->lock, flags); 9138 parent_ctx->rotate_disable = 0; 9139 9140 child_ctx = child->perf_event_ctxp[ctxn]; 9141 9142 if (child_ctx && inherited_all) { 9143 /* 9144 * Mark the child context as a clone of the parent 9145 * context, or of whatever the parent is a clone of. 9146 * 9147 * Note that if the parent is a clone, the holding of 9148 * parent_ctx->lock avoids it from being uncloned. 9149 */ 9150 cloned_ctx = parent_ctx->parent_ctx; 9151 if (cloned_ctx) { 9152 child_ctx->parent_ctx = cloned_ctx; 9153 child_ctx->parent_gen = parent_ctx->parent_gen; 9154 } else { 9155 child_ctx->parent_ctx = parent_ctx; 9156 child_ctx->parent_gen = parent_ctx->generation; 9157 } 9158 get_ctx(child_ctx->parent_ctx); 9159 } 9160 9161 raw_spin_unlock_irqrestore(&parent_ctx->lock, flags); 9162 mutex_unlock(&parent_ctx->mutex); 9163 9164 perf_unpin_context(parent_ctx); 9165 put_ctx(parent_ctx); 9166 9167 return ret; 9168 } 9169 9170 /* 9171 * Initialize the perf_event context in task_struct 9172 */ 9173 int perf_event_init_task(struct task_struct *child) 9174 { 9175 int ctxn, ret; 9176 9177 memset(child->perf_event_ctxp, 0, sizeof(child->perf_event_ctxp)); 9178 mutex_init(&child->perf_event_mutex); 9179 INIT_LIST_HEAD(&child->perf_event_list); 9180 9181 for_each_task_context_nr(ctxn) { 9182 ret = perf_event_init_context(child, ctxn); 9183 if (ret) { 9184 perf_event_free_task(child); 9185 return ret; 9186 } 9187 } 9188 9189 return 0; 9190 } 9191 9192 static void __init perf_event_init_all_cpus(void) 9193 { 9194 struct swevent_htable *swhash; 9195 int cpu; 9196 9197 for_each_possible_cpu(cpu) { 9198 swhash = &per_cpu(swevent_htable, cpu); 9199 mutex_init(&swhash->hlist_mutex); 9200 INIT_LIST_HEAD(&per_cpu(active_ctx_list, cpu)); 9201 } 9202 } 9203 9204 static void perf_event_init_cpu(int cpu) 9205 { 9206 struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu); 9207 9208 mutex_lock(&swhash->hlist_mutex); 9209 if (swhash->hlist_refcount > 0) { 9210 struct swevent_hlist *hlist; 9211 9212 hlist = kzalloc_node(sizeof(*hlist), GFP_KERNEL, cpu_to_node(cpu)); 9213 WARN_ON(!hlist); 9214 rcu_assign_pointer(swhash->swevent_hlist, hlist); 9215 } 9216 mutex_unlock(&swhash->hlist_mutex); 9217 } 9218 9219 #if defined CONFIG_HOTPLUG_CPU || defined CONFIG_KEXEC_CORE 9220 static void __perf_event_exit_context(void *__info) 9221 { 9222 struct perf_event_context *ctx = __info; 9223 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx); 9224 struct perf_event *event; 9225 9226 raw_spin_lock(&ctx->lock); 9227 list_for_each_entry(event, &ctx->event_list, event_entry) 9228 __perf_remove_from_context(event, cpuctx, ctx, (void *)DETACH_GROUP); 9229 raw_spin_unlock(&ctx->lock); 9230 } 9231 9232 static void perf_event_exit_cpu_context(int cpu) 9233 { 9234 struct perf_event_context *ctx; 9235 struct pmu *pmu; 9236 int idx; 9237 9238 idx = srcu_read_lock(&pmus_srcu); 9239 list_for_each_entry_rcu(pmu, &pmus, entry) { 9240 ctx = &per_cpu_ptr(pmu->pmu_cpu_context, cpu)->ctx; 9241 9242 mutex_lock(&ctx->mutex); 9243 smp_call_function_single(cpu, __perf_event_exit_context, ctx, 1); 9244 mutex_unlock(&ctx->mutex); 9245 } 9246 srcu_read_unlock(&pmus_srcu, idx); 9247 } 9248 9249 static void perf_event_exit_cpu(int cpu) 9250 { 9251 perf_event_exit_cpu_context(cpu); 9252 } 9253 #else 9254 static inline void perf_event_exit_cpu(int cpu) { } 9255 #endif 9256 9257 static int 9258 perf_reboot(struct notifier_block *notifier, unsigned long val, void *v) 9259 { 9260 int cpu; 9261 9262 for_each_online_cpu(cpu) 9263 perf_event_exit_cpu(cpu); 9264 9265 return NOTIFY_OK; 9266 } 9267 9268 /* 9269 * Run the perf reboot notifier at the very last possible moment so that 9270 * the generic watchdog code runs as long as possible. 9271 */ 9272 static struct notifier_block perf_reboot_notifier = { 9273 .notifier_call = perf_reboot, 9274 .priority = INT_MIN, 9275 }; 9276 9277 static int 9278 perf_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu) 9279 { 9280 unsigned int cpu = (long)hcpu; 9281 9282 switch (action & ~CPU_TASKS_FROZEN) { 9283 9284 case CPU_UP_PREPARE: 9285 case CPU_DOWN_FAILED: 9286 perf_event_init_cpu(cpu); 9287 break; 9288 9289 case CPU_UP_CANCELED: 9290 case CPU_DOWN_PREPARE: 9291 perf_event_exit_cpu(cpu); 9292 break; 9293 default: 9294 break; 9295 } 9296 9297 return NOTIFY_OK; 9298 } 9299 9300 void __init perf_event_init(void) 9301 { 9302 int ret; 9303 9304 idr_init(&pmu_idr); 9305 9306 perf_event_init_all_cpus(); 9307 init_srcu_struct(&pmus_srcu); 9308 perf_pmu_register(&perf_swevent, "software", PERF_TYPE_SOFTWARE); 9309 perf_pmu_register(&perf_cpu_clock, NULL, -1); 9310 perf_pmu_register(&perf_task_clock, NULL, -1); 9311 perf_tp_register(); 9312 perf_cpu_notifier(perf_cpu_notify); 9313 register_reboot_notifier(&perf_reboot_notifier); 9314 9315 ret = init_hw_breakpoint(); 9316 WARN(ret, "hw_breakpoint initialization failed with: %d", ret); 9317 9318 /* do not patch jump label more than once per second */ 9319 jump_label_rate_limit(&perf_sched_events, HZ); 9320 9321 /* 9322 * Build time assertion that we keep the data_head at the intended 9323 * location. IOW, validation we got the __reserved[] size right. 9324 */ 9325 BUILD_BUG_ON((offsetof(struct perf_event_mmap_page, data_head)) 9326 != 1024); 9327 } 9328 9329 ssize_t perf_event_sysfs_show(struct device *dev, struct device_attribute *attr, 9330 char *page) 9331 { 9332 struct perf_pmu_events_attr *pmu_attr = 9333 container_of(attr, struct perf_pmu_events_attr, attr); 9334 9335 if (pmu_attr->event_str) 9336 return sprintf(page, "%s\n", pmu_attr->event_str); 9337 9338 return 0; 9339 } 9340 9341 static int __init perf_event_sysfs_init(void) 9342 { 9343 struct pmu *pmu; 9344 int ret; 9345 9346 mutex_lock(&pmus_lock); 9347 9348 ret = bus_register(&pmu_bus); 9349 if (ret) 9350 goto unlock; 9351 9352 list_for_each_entry(pmu, &pmus, entry) { 9353 if (!pmu->name || pmu->type < 0) 9354 continue; 9355 9356 ret = pmu_dev_alloc(pmu); 9357 WARN(ret, "Failed to register pmu: %s, reason %d\n", pmu->name, ret); 9358 } 9359 pmu_bus_running = 1; 9360 ret = 0; 9361 9362 unlock: 9363 mutex_unlock(&pmus_lock); 9364 9365 return ret; 9366 } 9367 device_initcall(perf_event_sysfs_init); 9368 9369 #ifdef CONFIG_CGROUP_PERF 9370 static struct cgroup_subsys_state * 9371 perf_cgroup_css_alloc(struct cgroup_subsys_state *parent_css) 9372 { 9373 struct perf_cgroup *jc; 9374 9375 jc = kzalloc(sizeof(*jc), GFP_KERNEL); 9376 if (!jc) 9377 return ERR_PTR(-ENOMEM); 9378 9379 jc->info = alloc_percpu(struct perf_cgroup_info); 9380 if (!jc->info) { 9381 kfree(jc); 9382 return ERR_PTR(-ENOMEM); 9383 } 9384 9385 return &jc->css; 9386 } 9387 9388 static void perf_cgroup_css_free(struct cgroup_subsys_state *css) 9389 { 9390 struct perf_cgroup *jc = container_of(css, struct perf_cgroup, css); 9391 9392 free_percpu(jc->info); 9393 kfree(jc); 9394 } 9395 9396 static int __perf_cgroup_move(void *info) 9397 { 9398 struct task_struct *task = info; 9399 rcu_read_lock(); 9400 perf_cgroup_switch(task, PERF_CGROUP_SWOUT | PERF_CGROUP_SWIN); 9401 rcu_read_unlock(); 9402 return 0; 9403 } 9404 9405 static void perf_cgroup_attach(struct cgroup_taskset *tset) 9406 { 9407 struct task_struct *task; 9408 struct cgroup_subsys_state *css; 9409 9410 cgroup_taskset_for_each(task, css, tset) 9411 task_function_call(task, __perf_cgroup_move, task); 9412 } 9413 9414 struct cgroup_subsys perf_event_cgrp_subsys = { 9415 .css_alloc = perf_cgroup_css_alloc, 9416 .css_free = perf_cgroup_css_free, 9417 .attach = perf_cgroup_attach, 9418 }; 9419 #endif /* CONFIG_CGROUP_PERF */ 9420