1 /* 2 * Performance events core code: 3 * 4 * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de> 5 * Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar 6 * Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com> 7 * Copyright © 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com> 8 * 9 * For licensing details see kernel-base/COPYING 10 */ 11 12 #include <linux/fs.h> 13 #include <linux/mm.h> 14 #include <linux/cpu.h> 15 #include <linux/smp.h> 16 #include <linux/idr.h> 17 #include <linux/file.h> 18 #include <linux/poll.h> 19 #include <linux/slab.h> 20 #include <linux/hash.h> 21 #include <linux/tick.h> 22 #include <linux/sysfs.h> 23 #include <linux/dcache.h> 24 #include <linux/percpu.h> 25 #include <linux/ptrace.h> 26 #include <linux/reboot.h> 27 #include <linux/vmstat.h> 28 #include <linux/device.h> 29 #include <linux/export.h> 30 #include <linux/vmalloc.h> 31 #include <linux/hardirq.h> 32 #include <linux/rculist.h> 33 #include <linux/uaccess.h> 34 #include <linux/syscalls.h> 35 #include <linux/anon_inodes.h> 36 #include <linux/kernel_stat.h> 37 #include <linux/perf_event.h> 38 #include <linux/ftrace_event.h> 39 #include <linux/hw_breakpoint.h> 40 #include <linux/mm_types.h> 41 #include <linux/cgroup.h> 42 43 #include "internal.h" 44 45 #include <asm/irq_regs.h> 46 47 struct remote_function_call { 48 struct task_struct *p; 49 int (*func)(void *info); 50 void *info; 51 int ret; 52 }; 53 54 static void remote_function(void *data) 55 { 56 struct remote_function_call *tfc = data; 57 struct task_struct *p = tfc->p; 58 59 if (p) { 60 tfc->ret = -EAGAIN; 61 if (task_cpu(p) != smp_processor_id() || !task_curr(p)) 62 return; 63 } 64 65 tfc->ret = tfc->func(tfc->info); 66 } 67 68 /** 69 * task_function_call - call a function on the cpu on which a task runs 70 * @p: the task to evaluate 71 * @func: the function to be called 72 * @info: the function call argument 73 * 74 * Calls the function @func when the task is currently running. This might 75 * be on the current CPU, which just calls the function directly 76 * 77 * returns: @func return value, or 78 * -ESRCH - when the process isn't running 79 * -EAGAIN - when the process moved away 80 */ 81 static int 82 task_function_call(struct task_struct *p, int (*func) (void *info), void *info) 83 { 84 struct remote_function_call data = { 85 .p = p, 86 .func = func, 87 .info = info, 88 .ret = -ESRCH, /* No such (running) process */ 89 }; 90 91 if (task_curr(p)) 92 smp_call_function_single(task_cpu(p), remote_function, &data, 1); 93 94 return data.ret; 95 } 96 97 /** 98 * cpu_function_call - call a function on the cpu 99 * @func: the function to be called 100 * @info: the function call argument 101 * 102 * Calls the function @func on the remote cpu. 103 * 104 * returns: @func return value or -ENXIO when the cpu is offline 105 */ 106 static int cpu_function_call(int cpu, int (*func) (void *info), void *info) 107 { 108 struct remote_function_call data = { 109 .p = NULL, 110 .func = func, 111 .info = info, 112 .ret = -ENXIO, /* No such CPU */ 113 }; 114 115 smp_call_function_single(cpu, remote_function, &data, 1); 116 117 return data.ret; 118 } 119 120 #define PERF_FLAG_ALL (PERF_FLAG_FD_NO_GROUP |\ 121 PERF_FLAG_FD_OUTPUT |\ 122 PERF_FLAG_PID_CGROUP |\ 123 PERF_FLAG_FD_CLOEXEC) 124 125 /* 126 * branch priv levels that need permission checks 127 */ 128 #define PERF_SAMPLE_BRANCH_PERM_PLM \ 129 (PERF_SAMPLE_BRANCH_KERNEL |\ 130 PERF_SAMPLE_BRANCH_HV) 131 132 enum event_type_t { 133 EVENT_FLEXIBLE = 0x1, 134 EVENT_PINNED = 0x2, 135 EVENT_ALL = EVENT_FLEXIBLE | EVENT_PINNED, 136 }; 137 138 /* 139 * perf_sched_events : >0 events exist 140 * perf_cgroup_events: >0 per-cpu cgroup events exist on this cpu 141 */ 142 struct static_key_deferred perf_sched_events __read_mostly; 143 static DEFINE_PER_CPU(atomic_t, perf_cgroup_events); 144 static DEFINE_PER_CPU(atomic_t, perf_branch_stack_events); 145 146 static atomic_t nr_mmap_events __read_mostly; 147 static atomic_t nr_comm_events __read_mostly; 148 static atomic_t nr_task_events __read_mostly; 149 static atomic_t nr_freq_events __read_mostly; 150 151 static LIST_HEAD(pmus); 152 static DEFINE_MUTEX(pmus_lock); 153 static struct srcu_struct pmus_srcu; 154 155 /* 156 * perf event paranoia level: 157 * -1 - not paranoid at all 158 * 0 - disallow raw tracepoint access for unpriv 159 * 1 - disallow cpu events for unpriv 160 * 2 - disallow kernel profiling for unpriv 161 */ 162 int sysctl_perf_event_paranoid __read_mostly = 1; 163 164 /* Minimum for 512 kiB + 1 user control page */ 165 int sysctl_perf_event_mlock __read_mostly = 512 + (PAGE_SIZE / 1024); /* 'free' kiB per user */ 166 167 /* 168 * max perf event sample rate 169 */ 170 #define DEFAULT_MAX_SAMPLE_RATE 100000 171 #define DEFAULT_SAMPLE_PERIOD_NS (NSEC_PER_SEC / DEFAULT_MAX_SAMPLE_RATE) 172 #define DEFAULT_CPU_TIME_MAX_PERCENT 25 173 174 int sysctl_perf_event_sample_rate __read_mostly = DEFAULT_MAX_SAMPLE_RATE; 175 176 static int max_samples_per_tick __read_mostly = DIV_ROUND_UP(DEFAULT_MAX_SAMPLE_RATE, HZ); 177 static int perf_sample_period_ns __read_mostly = DEFAULT_SAMPLE_PERIOD_NS; 178 179 static int perf_sample_allowed_ns __read_mostly = 180 DEFAULT_SAMPLE_PERIOD_NS * DEFAULT_CPU_TIME_MAX_PERCENT / 100; 181 182 void update_perf_cpu_limits(void) 183 { 184 u64 tmp = perf_sample_period_ns; 185 186 tmp *= sysctl_perf_cpu_time_max_percent; 187 do_div(tmp, 100); 188 ACCESS_ONCE(perf_sample_allowed_ns) = tmp; 189 } 190 191 static int perf_rotate_context(struct perf_cpu_context *cpuctx); 192 193 int perf_proc_update_handler(struct ctl_table *table, int write, 194 void __user *buffer, size_t *lenp, 195 loff_t *ppos) 196 { 197 int ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); 198 199 if (ret || !write) 200 return ret; 201 202 max_samples_per_tick = DIV_ROUND_UP(sysctl_perf_event_sample_rate, HZ); 203 perf_sample_period_ns = NSEC_PER_SEC / sysctl_perf_event_sample_rate; 204 update_perf_cpu_limits(); 205 206 return 0; 207 } 208 209 int sysctl_perf_cpu_time_max_percent __read_mostly = DEFAULT_CPU_TIME_MAX_PERCENT; 210 211 int perf_cpu_time_max_percent_handler(struct ctl_table *table, int write, 212 void __user *buffer, size_t *lenp, 213 loff_t *ppos) 214 { 215 int ret = proc_dointvec(table, write, buffer, lenp, ppos); 216 217 if (ret || !write) 218 return ret; 219 220 update_perf_cpu_limits(); 221 222 return 0; 223 } 224 225 /* 226 * perf samples are done in some very critical code paths (NMIs). 227 * If they take too much CPU time, the system can lock up and not 228 * get any real work done. This will drop the sample rate when 229 * we detect that events are taking too long. 230 */ 231 #define NR_ACCUMULATED_SAMPLES 128 232 static DEFINE_PER_CPU(u64, running_sample_length); 233 234 static void perf_duration_warn(struct irq_work *w) 235 { 236 u64 allowed_ns = ACCESS_ONCE(perf_sample_allowed_ns); 237 u64 avg_local_sample_len; 238 u64 local_samples_len; 239 240 local_samples_len = __get_cpu_var(running_sample_length); 241 avg_local_sample_len = local_samples_len/NR_ACCUMULATED_SAMPLES; 242 243 printk_ratelimited(KERN_WARNING 244 "perf interrupt took too long (%lld > %lld), lowering " 245 "kernel.perf_event_max_sample_rate to %d\n", 246 avg_local_sample_len, allowed_ns >> 1, 247 sysctl_perf_event_sample_rate); 248 } 249 250 static DEFINE_IRQ_WORK(perf_duration_work, perf_duration_warn); 251 252 void perf_sample_event_took(u64 sample_len_ns) 253 { 254 u64 allowed_ns = ACCESS_ONCE(perf_sample_allowed_ns); 255 u64 avg_local_sample_len; 256 u64 local_samples_len; 257 258 if (allowed_ns == 0) 259 return; 260 261 /* decay the counter by 1 average sample */ 262 local_samples_len = __get_cpu_var(running_sample_length); 263 local_samples_len -= local_samples_len/NR_ACCUMULATED_SAMPLES; 264 local_samples_len += sample_len_ns; 265 __get_cpu_var(running_sample_length) = local_samples_len; 266 267 /* 268 * note: this will be biased artifically low until we have 269 * seen NR_ACCUMULATED_SAMPLES. Doing it this way keeps us 270 * from having to maintain a count. 271 */ 272 avg_local_sample_len = local_samples_len/NR_ACCUMULATED_SAMPLES; 273 274 if (avg_local_sample_len <= allowed_ns) 275 return; 276 277 if (max_samples_per_tick <= 1) 278 return; 279 280 max_samples_per_tick = DIV_ROUND_UP(max_samples_per_tick, 2); 281 sysctl_perf_event_sample_rate = max_samples_per_tick * HZ; 282 perf_sample_period_ns = NSEC_PER_SEC / sysctl_perf_event_sample_rate; 283 284 update_perf_cpu_limits(); 285 286 if (!irq_work_queue(&perf_duration_work)) { 287 early_printk("perf interrupt took too long (%lld > %lld), lowering " 288 "kernel.perf_event_max_sample_rate to %d\n", 289 avg_local_sample_len, allowed_ns >> 1, 290 sysctl_perf_event_sample_rate); 291 } 292 } 293 294 static atomic64_t perf_event_id; 295 296 static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx, 297 enum event_type_t event_type); 298 299 static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx, 300 enum event_type_t event_type, 301 struct task_struct *task); 302 303 static void update_context_time(struct perf_event_context *ctx); 304 static u64 perf_event_time(struct perf_event *event); 305 306 void __weak perf_event_print_debug(void) { } 307 308 extern __weak const char *perf_pmu_name(void) 309 { 310 return "pmu"; 311 } 312 313 static inline u64 perf_clock(void) 314 { 315 return local_clock(); 316 } 317 318 static inline struct perf_cpu_context * 319 __get_cpu_context(struct perf_event_context *ctx) 320 { 321 return this_cpu_ptr(ctx->pmu->pmu_cpu_context); 322 } 323 324 static void perf_ctx_lock(struct perf_cpu_context *cpuctx, 325 struct perf_event_context *ctx) 326 { 327 raw_spin_lock(&cpuctx->ctx.lock); 328 if (ctx) 329 raw_spin_lock(&ctx->lock); 330 } 331 332 static void perf_ctx_unlock(struct perf_cpu_context *cpuctx, 333 struct perf_event_context *ctx) 334 { 335 if (ctx) 336 raw_spin_unlock(&ctx->lock); 337 raw_spin_unlock(&cpuctx->ctx.lock); 338 } 339 340 #ifdef CONFIG_CGROUP_PERF 341 342 /* 343 * perf_cgroup_info keeps track of time_enabled for a cgroup. 344 * This is a per-cpu dynamically allocated data structure. 345 */ 346 struct perf_cgroup_info { 347 u64 time; 348 u64 timestamp; 349 }; 350 351 struct perf_cgroup { 352 struct cgroup_subsys_state css; 353 struct perf_cgroup_info __percpu *info; 354 }; 355 356 /* 357 * Must ensure cgroup is pinned (css_get) before calling 358 * this function. In other words, we cannot call this function 359 * if there is no cgroup event for the current CPU context. 360 */ 361 static inline struct perf_cgroup * 362 perf_cgroup_from_task(struct task_struct *task) 363 { 364 return container_of(task_css(task, perf_event_cgrp_id), 365 struct perf_cgroup, css); 366 } 367 368 static inline bool 369 perf_cgroup_match(struct perf_event *event) 370 { 371 struct perf_event_context *ctx = event->ctx; 372 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx); 373 374 /* @event doesn't care about cgroup */ 375 if (!event->cgrp) 376 return true; 377 378 /* wants specific cgroup scope but @cpuctx isn't associated with any */ 379 if (!cpuctx->cgrp) 380 return false; 381 382 /* 383 * Cgroup scoping is recursive. An event enabled for a cgroup is 384 * also enabled for all its descendant cgroups. If @cpuctx's 385 * cgroup is a descendant of @event's (the test covers identity 386 * case), it's a match. 387 */ 388 return cgroup_is_descendant(cpuctx->cgrp->css.cgroup, 389 event->cgrp->css.cgroup); 390 } 391 392 static inline void perf_put_cgroup(struct perf_event *event) 393 { 394 css_put(&event->cgrp->css); 395 } 396 397 static inline void perf_detach_cgroup(struct perf_event *event) 398 { 399 perf_put_cgroup(event); 400 event->cgrp = NULL; 401 } 402 403 static inline int is_cgroup_event(struct perf_event *event) 404 { 405 return event->cgrp != NULL; 406 } 407 408 static inline u64 perf_cgroup_event_time(struct perf_event *event) 409 { 410 struct perf_cgroup_info *t; 411 412 t = per_cpu_ptr(event->cgrp->info, event->cpu); 413 return t->time; 414 } 415 416 static inline void __update_cgrp_time(struct perf_cgroup *cgrp) 417 { 418 struct perf_cgroup_info *info; 419 u64 now; 420 421 now = perf_clock(); 422 423 info = this_cpu_ptr(cgrp->info); 424 425 info->time += now - info->timestamp; 426 info->timestamp = now; 427 } 428 429 static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx) 430 { 431 struct perf_cgroup *cgrp_out = cpuctx->cgrp; 432 if (cgrp_out) 433 __update_cgrp_time(cgrp_out); 434 } 435 436 static inline void update_cgrp_time_from_event(struct perf_event *event) 437 { 438 struct perf_cgroup *cgrp; 439 440 /* 441 * ensure we access cgroup data only when needed and 442 * when we know the cgroup is pinned (css_get) 443 */ 444 if (!is_cgroup_event(event)) 445 return; 446 447 cgrp = perf_cgroup_from_task(current); 448 /* 449 * Do not update time when cgroup is not active 450 */ 451 if (cgrp == event->cgrp) 452 __update_cgrp_time(event->cgrp); 453 } 454 455 static inline void 456 perf_cgroup_set_timestamp(struct task_struct *task, 457 struct perf_event_context *ctx) 458 { 459 struct perf_cgroup *cgrp; 460 struct perf_cgroup_info *info; 461 462 /* 463 * ctx->lock held by caller 464 * ensure we do not access cgroup data 465 * unless we have the cgroup pinned (css_get) 466 */ 467 if (!task || !ctx->nr_cgroups) 468 return; 469 470 cgrp = perf_cgroup_from_task(task); 471 info = this_cpu_ptr(cgrp->info); 472 info->timestamp = ctx->timestamp; 473 } 474 475 #define PERF_CGROUP_SWOUT 0x1 /* cgroup switch out every event */ 476 #define PERF_CGROUP_SWIN 0x2 /* cgroup switch in events based on task */ 477 478 /* 479 * reschedule events based on the cgroup constraint of task. 480 * 481 * mode SWOUT : schedule out everything 482 * mode SWIN : schedule in based on cgroup for next 483 */ 484 void perf_cgroup_switch(struct task_struct *task, int mode) 485 { 486 struct perf_cpu_context *cpuctx; 487 struct pmu *pmu; 488 unsigned long flags; 489 490 /* 491 * disable interrupts to avoid geting nr_cgroup 492 * changes via __perf_event_disable(). Also 493 * avoids preemption. 494 */ 495 local_irq_save(flags); 496 497 /* 498 * we reschedule only in the presence of cgroup 499 * constrained events. 500 */ 501 rcu_read_lock(); 502 503 list_for_each_entry_rcu(pmu, &pmus, entry) { 504 cpuctx = this_cpu_ptr(pmu->pmu_cpu_context); 505 if (cpuctx->unique_pmu != pmu) 506 continue; /* ensure we process each cpuctx once */ 507 508 /* 509 * perf_cgroup_events says at least one 510 * context on this CPU has cgroup events. 511 * 512 * ctx->nr_cgroups reports the number of cgroup 513 * events for a context. 514 */ 515 if (cpuctx->ctx.nr_cgroups > 0) { 516 perf_ctx_lock(cpuctx, cpuctx->task_ctx); 517 perf_pmu_disable(cpuctx->ctx.pmu); 518 519 if (mode & PERF_CGROUP_SWOUT) { 520 cpu_ctx_sched_out(cpuctx, EVENT_ALL); 521 /* 522 * must not be done before ctxswout due 523 * to event_filter_match() in event_sched_out() 524 */ 525 cpuctx->cgrp = NULL; 526 } 527 528 if (mode & PERF_CGROUP_SWIN) { 529 WARN_ON_ONCE(cpuctx->cgrp); 530 /* 531 * set cgrp before ctxsw in to allow 532 * event_filter_match() to not have to pass 533 * task around 534 */ 535 cpuctx->cgrp = perf_cgroup_from_task(task); 536 cpu_ctx_sched_in(cpuctx, EVENT_ALL, task); 537 } 538 perf_pmu_enable(cpuctx->ctx.pmu); 539 perf_ctx_unlock(cpuctx, cpuctx->task_ctx); 540 } 541 } 542 543 rcu_read_unlock(); 544 545 local_irq_restore(flags); 546 } 547 548 static inline void perf_cgroup_sched_out(struct task_struct *task, 549 struct task_struct *next) 550 { 551 struct perf_cgroup *cgrp1; 552 struct perf_cgroup *cgrp2 = NULL; 553 554 /* 555 * we come here when we know perf_cgroup_events > 0 556 */ 557 cgrp1 = perf_cgroup_from_task(task); 558 559 /* 560 * next is NULL when called from perf_event_enable_on_exec() 561 * that will systematically cause a cgroup_switch() 562 */ 563 if (next) 564 cgrp2 = perf_cgroup_from_task(next); 565 566 /* 567 * only schedule out current cgroup events if we know 568 * that we are switching to a different cgroup. Otherwise, 569 * do no touch the cgroup events. 570 */ 571 if (cgrp1 != cgrp2) 572 perf_cgroup_switch(task, PERF_CGROUP_SWOUT); 573 } 574 575 static inline void perf_cgroup_sched_in(struct task_struct *prev, 576 struct task_struct *task) 577 { 578 struct perf_cgroup *cgrp1; 579 struct perf_cgroup *cgrp2 = NULL; 580 581 /* 582 * we come here when we know perf_cgroup_events > 0 583 */ 584 cgrp1 = perf_cgroup_from_task(task); 585 586 /* prev can never be NULL */ 587 cgrp2 = perf_cgroup_from_task(prev); 588 589 /* 590 * only need to schedule in cgroup events if we are changing 591 * cgroup during ctxsw. Cgroup events were not scheduled 592 * out of ctxsw out if that was not the case. 593 */ 594 if (cgrp1 != cgrp2) 595 perf_cgroup_switch(task, PERF_CGROUP_SWIN); 596 } 597 598 static inline int perf_cgroup_connect(int fd, struct perf_event *event, 599 struct perf_event_attr *attr, 600 struct perf_event *group_leader) 601 { 602 struct perf_cgroup *cgrp; 603 struct cgroup_subsys_state *css; 604 struct fd f = fdget(fd); 605 int ret = 0; 606 607 if (!f.file) 608 return -EBADF; 609 610 css = css_tryget_from_dir(f.file->f_dentry, &perf_event_cgrp_subsys); 611 if (IS_ERR(css)) { 612 ret = PTR_ERR(css); 613 goto out; 614 } 615 616 cgrp = container_of(css, struct perf_cgroup, css); 617 event->cgrp = cgrp; 618 619 /* 620 * all events in a group must monitor 621 * the same cgroup because a task belongs 622 * to only one perf cgroup at a time 623 */ 624 if (group_leader && group_leader->cgrp != cgrp) { 625 perf_detach_cgroup(event); 626 ret = -EINVAL; 627 } 628 out: 629 fdput(f); 630 return ret; 631 } 632 633 static inline void 634 perf_cgroup_set_shadow_time(struct perf_event *event, u64 now) 635 { 636 struct perf_cgroup_info *t; 637 t = per_cpu_ptr(event->cgrp->info, event->cpu); 638 event->shadow_ctx_time = now - t->timestamp; 639 } 640 641 static inline void 642 perf_cgroup_defer_enabled(struct perf_event *event) 643 { 644 /* 645 * when the current task's perf cgroup does not match 646 * the event's, we need to remember to call the 647 * perf_mark_enable() function the first time a task with 648 * a matching perf cgroup is scheduled in. 649 */ 650 if (is_cgroup_event(event) && !perf_cgroup_match(event)) 651 event->cgrp_defer_enabled = 1; 652 } 653 654 static inline void 655 perf_cgroup_mark_enabled(struct perf_event *event, 656 struct perf_event_context *ctx) 657 { 658 struct perf_event *sub; 659 u64 tstamp = perf_event_time(event); 660 661 if (!event->cgrp_defer_enabled) 662 return; 663 664 event->cgrp_defer_enabled = 0; 665 666 event->tstamp_enabled = tstamp - event->total_time_enabled; 667 list_for_each_entry(sub, &event->sibling_list, group_entry) { 668 if (sub->state >= PERF_EVENT_STATE_INACTIVE) { 669 sub->tstamp_enabled = tstamp - sub->total_time_enabled; 670 sub->cgrp_defer_enabled = 0; 671 } 672 } 673 } 674 #else /* !CONFIG_CGROUP_PERF */ 675 676 static inline bool 677 perf_cgroup_match(struct perf_event *event) 678 { 679 return true; 680 } 681 682 static inline void perf_detach_cgroup(struct perf_event *event) 683 {} 684 685 static inline int is_cgroup_event(struct perf_event *event) 686 { 687 return 0; 688 } 689 690 static inline u64 perf_cgroup_event_cgrp_time(struct perf_event *event) 691 { 692 return 0; 693 } 694 695 static inline void update_cgrp_time_from_event(struct perf_event *event) 696 { 697 } 698 699 static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx) 700 { 701 } 702 703 static inline void perf_cgroup_sched_out(struct task_struct *task, 704 struct task_struct *next) 705 { 706 } 707 708 static inline void perf_cgroup_sched_in(struct task_struct *prev, 709 struct task_struct *task) 710 { 711 } 712 713 static inline int perf_cgroup_connect(pid_t pid, struct perf_event *event, 714 struct perf_event_attr *attr, 715 struct perf_event *group_leader) 716 { 717 return -EINVAL; 718 } 719 720 static inline void 721 perf_cgroup_set_timestamp(struct task_struct *task, 722 struct perf_event_context *ctx) 723 { 724 } 725 726 void 727 perf_cgroup_switch(struct task_struct *task, struct task_struct *next) 728 { 729 } 730 731 static inline void 732 perf_cgroup_set_shadow_time(struct perf_event *event, u64 now) 733 { 734 } 735 736 static inline u64 perf_cgroup_event_time(struct perf_event *event) 737 { 738 return 0; 739 } 740 741 static inline void 742 perf_cgroup_defer_enabled(struct perf_event *event) 743 { 744 } 745 746 static inline void 747 perf_cgroup_mark_enabled(struct perf_event *event, 748 struct perf_event_context *ctx) 749 { 750 } 751 #endif 752 753 /* 754 * set default to be dependent on timer tick just 755 * like original code 756 */ 757 #define PERF_CPU_HRTIMER (1000 / HZ) 758 /* 759 * function must be called with interrupts disbled 760 */ 761 static enum hrtimer_restart perf_cpu_hrtimer_handler(struct hrtimer *hr) 762 { 763 struct perf_cpu_context *cpuctx; 764 enum hrtimer_restart ret = HRTIMER_NORESTART; 765 int rotations = 0; 766 767 WARN_ON(!irqs_disabled()); 768 769 cpuctx = container_of(hr, struct perf_cpu_context, hrtimer); 770 771 rotations = perf_rotate_context(cpuctx); 772 773 /* 774 * arm timer if needed 775 */ 776 if (rotations) { 777 hrtimer_forward_now(hr, cpuctx->hrtimer_interval); 778 ret = HRTIMER_RESTART; 779 } 780 781 return ret; 782 } 783 784 /* CPU is going down */ 785 void perf_cpu_hrtimer_cancel(int cpu) 786 { 787 struct perf_cpu_context *cpuctx; 788 struct pmu *pmu; 789 unsigned long flags; 790 791 if (WARN_ON(cpu != smp_processor_id())) 792 return; 793 794 local_irq_save(flags); 795 796 rcu_read_lock(); 797 798 list_for_each_entry_rcu(pmu, &pmus, entry) { 799 cpuctx = this_cpu_ptr(pmu->pmu_cpu_context); 800 801 if (pmu->task_ctx_nr == perf_sw_context) 802 continue; 803 804 hrtimer_cancel(&cpuctx->hrtimer); 805 } 806 807 rcu_read_unlock(); 808 809 local_irq_restore(flags); 810 } 811 812 static void __perf_cpu_hrtimer_init(struct perf_cpu_context *cpuctx, int cpu) 813 { 814 struct hrtimer *hr = &cpuctx->hrtimer; 815 struct pmu *pmu = cpuctx->ctx.pmu; 816 int timer; 817 818 /* no multiplexing needed for SW PMU */ 819 if (pmu->task_ctx_nr == perf_sw_context) 820 return; 821 822 /* 823 * check default is sane, if not set then force to 824 * default interval (1/tick) 825 */ 826 timer = pmu->hrtimer_interval_ms; 827 if (timer < 1) 828 timer = pmu->hrtimer_interval_ms = PERF_CPU_HRTIMER; 829 830 cpuctx->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * timer); 831 832 hrtimer_init(hr, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED); 833 hr->function = perf_cpu_hrtimer_handler; 834 } 835 836 static void perf_cpu_hrtimer_restart(struct perf_cpu_context *cpuctx) 837 { 838 struct hrtimer *hr = &cpuctx->hrtimer; 839 struct pmu *pmu = cpuctx->ctx.pmu; 840 841 /* not for SW PMU */ 842 if (pmu->task_ctx_nr == perf_sw_context) 843 return; 844 845 if (hrtimer_active(hr)) 846 return; 847 848 if (!hrtimer_callback_running(hr)) 849 __hrtimer_start_range_ns(hr, cpuctx->hrtimer_interval, 850 0, HRTIMER_MODE_REL_PINNED, 0); 851 } 852 853 void perf_pmu_disable(struct pmu *pmu) 854 { 855 int *count = this_cpu_ptr(pmu->pmu_disable_count); 856 if (!(*count)++) 857 pmu->pmu_disable(pmu); 858 } 859 860 void perf_pmu_enable(struct pmu *pmu) 861 { 862 int *count = this_cpu_ptr(pmu->pmu_disable_count); 863 if (!--(*count)) 864 pmu->pmu_enable(pmu); 865 } 866 867 static DEFINE_PER_CPU(struct list_head, rotation_list); 868 869 /* 870 * perf_pmu_rotate_start() and perf_rotate_context() are fully serialized 871 * because they're strictly cpu affine and rotate_start is called with IRQs 872 * disabled, while rotate_context is called from IRQ context. 873 */ 874 static void perf_pmu_rotate_start(struct pmu *pmu) 875 { 876 struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context); 877 struct list_head *head = &__get_cpu_var(rotation_list); 878 879 WARN_ON(!irqs_disabled()); 880 881 if (list_empty(&cpuctx->rotation_list)) 882 list_add(&cpuctx->rotation_list, head); 883 } 884 885 static void get_ctx(struct perf_event_context *ctx) 886 { 887 WARN_ON(!atomic_inc_not_zero(&ctx->refcount)); 888 } 889 890 static void put_ctx(struct perf_event_context *ctx) 891 { 892 if (atomic_dec_and_test(&ctx->refcount)) { 893 if (ctx->parent_ctx) 894 put_ctx(ctx->parent_ctx); 895 if (ctx->task) 896 put_task_struct(ctx->task); 897 kfree_rcu(ctx, rcu_head); 898 } 899 } 900 901 static void unclone_ctx(struct perf_event_context *ctx) 902 { 903 if (ctx->parent_ctx) { 904 put_ctx(ctx->parent_ctx); 905 ctx->parent_ctx = NULL; 906 } 907 ctx->generation++; 908 } 909 910 static u32 perf_event_pid(struct perf_event *event, struct task_struct *p) 911 { 912 /* 913 * only top level events have the pid namespace they were created in 914 */ 915 if (event->parent) 916 event = event->parent; 917 918 return task_tgid_nr_ns(p, event->ns); 919 } 920 921 static u32 perf_event_tid(struct perf_event *event, struct task_struct *p) 922 { 923 /* 924 * only top level events have the pid namespace they were created in 925 */ 926 if (event->parent) 927 event = event->parent; 928 929 return task_pid_nr_ns(p, event->ns); 930 } 931 932 /* 933 * If we inherit events we want to return the parent event id 934 * to userspace. 935 */ 936 static u64 primary_event_id(struct perf_event *event) 937 { 938 u64 id = event->id; 939 940 if (event->parent) 941 id = event->parent->id; 942 943 return id; 944 } 945 946 /* 947 * Get the perf_event_context for a task and lock it. 948 * This has to cope with with the fact that until it is locked, 949 * the context could get moved to another task. 950 */ 951 static struct perf_event_context * 952 perf_lock_task_context(struct task_struct *task, int ctxn, unsigned long *flags) 953 { 954 struct perf_event_context *ctx; 955 956 retry: 957 /* 958 * One of the few rules of preemptible RCU is that one cannot do 959 * rcu_read_unlock() while holding a scheduler (or nested) lock when 960 * part of the read side critical section was preemptible -- see 961 * rcu_read_unlock_special(). 962 * 963 * Since ctx->lock nests under rq->lock we must ensure the entire read 964 * side critical section is non-preemptible. 965 */ 966 preempt_disable(); 967 rcu_read_lock(); 968 ctx = rcu_dereference(task->perf_event_ctxp[ctxn]); 969 if (ctx) { 970 /* 971 * If this context is a clone of another, it might 972 * get swapped for another underneath us by 973 * perf_event_task_sched_out, though the 974 * rcu_read_lock() protects us from any context 975 * getting freed. Lock the context and check if it 976 * got swapped before we could get the lock, and retry 977 * if so. If we locked the right context, then it 978 * can't get swapped on us any more. 979 */ 980 raw_spin_lock_irqsave(&ctx->lock, *flags); 981 if (ctx != rcu_dereference(task->perf_event_ctxp[ctxn])) { 982 raw_spin_unlock_irqrestore(&ctx->lock, *flags); 983 rcu_read_unlock(); 984 preempt_enable(); 985 goto retry; 986 } 987 988 if (!atomic_inc_not_zero(&ctx->refcount)) { 989 raw_spin_unlock_irqrestore(&ctx->lock, *flags); 990 ctx = NULL; 991 } 992 } 993 rcu_read_unlock(); 994 preempt_enable(); 995 return ctx; 996 } 997 998 /* 999 * Get the context for a task and increment its pin_count so it 1000 * can't get swapped to another task. This also increments its 1001 * reference count so that the context can't get freed. 1002 */ 1003 static struct perf_event_context * 1004 perf_pin_task_context(struct task_struct *task, int ctxn) 1005 { 1006 struct perf_event_context *ctx; 1007 unsigned long flags; 1008 1009 ctx = perf_lock_task_context(task, ctxn, &flags); 1010 if (ctx) { 1011 ++ctx->pin_count; 1012 raw_spin_unlock_irqrestore(&ctx->lock, flags); 1013 } 1014 return ctx; 1015 } 1016 1017 static void perf_unpin_context(struct perf_event_context *ctx) 1018 { 1019 unsigned long flags; 1020 1021 raw_spin_lock_irqsave(&ctx->lock, flags); 1022 --ctx->pin_count; 1023 raw_spin_unlock_irqrestore(&ctx->lock, flags); 1024 } 1025 1026 /* 1027 * Update the record of the current time in a context. 1028 */ 1029 static void update_context_time(struct perf_event_context *ctx) 1030 { 1031 u64 now = perf_clock(); 1032 1033 ctx->time += now - ctx->timestamp; 1034 ctx->timestamp = now; 1035 } 1036 1037 static u64 perf_event_time(struct perf_event *event) 1038 { 1039 struct perf_event_context *ctx = event->ctx; 1040 1041 if (is_cgroup_event(event)) 1042 return perf_cgroup_event_time(event); 1043 1044 return ctx ? ctx->time : 0; 1045 } 1046 1047 /* 1048 * Update the total_time_enabled and total_time_running fields for a event. 1049 * The caller of this function needs to hold the ctx->lock. 1050 */ 1051 static void update_event_times(struct perf_event *event) 1052 { 1053 struct perf_event_context *ctx = event->ctx; 1054 u64 run_end; 1055 1056 if (event->state < PERF_EVENT_STATE_INACTIVE || 1057 event->group_leader->state < PERF_EVENT_STATE_INACTIVE) 1058 return; 1059 /* 1060 * in cgroup mode, time_enabled represents 1061 * the time the event was enabled AND active 1062 * tasks were in the monitored cgroup. This is 1063 * independent of the activity of the context as 1064 * there may be a mix of cgroup and non-cgroup events. 1065 * 1066 * That is why we treat cgroup events differently 1067 * here. 1068 */ 1069 if (is_cgroup_event(event)) 1070 run_end = perf_cgroup_event_time(event); 1071 else if (ctx->is_active) 1072 run_end = ctx->time; 1073 else 1074 run_end = event->tstamp_stopped; 1075 1076 event->total_time_enabled = run_end - event->tstamp_enabled; 1077 1078 if (event->state == PERF_EVENT_STATE_INACTIVE) 1079 run_end = event->tstamp_stopped; 1080 else 1081 run_end = perf_event_time(event); 1082 1083 event->total_time_running = run_end - event->tstamp_running; 1084 1085 } 1086 1087 /* 1088 * Update total_time_enabled and total_time_running for all events in a group. 1089 */ 1090 static void update_group_times(struct perf_event *leader) 1091 { 1092 struct perf_event *event; 1093 1094 update_event_times(leader); 1095 list_for_each_entry(event, &leader->sibling_list, group_entry) 1096 update_event_times(event); 1097 } 1098 1099 static struct list_head * 1100 ctx_group_list(struct perf_event *event, struct perf_event_context *ctx) 1101 { 1102 if (event->attr.pinned) 1103 return &ctx->pinned_groups; 1104 else 1105 return &ctx->flexible_groups; 1106 } 1107 1108 /* 1109 * Add a event from the lists for its context. 1110 * Must be called with ctx->mutex and ctx->lock held. 1111 */ 1112 static void 1113 list_add_event(struct perf_event *event, struct perf_event_context *ctx) 1114 { 1115 WARN_ON_ONCE(event->attach_state & PERF_ATTACH_CONTEXT); 1116 event->attach_state |= PERF_ATTACH_CONTEXT; 1117 1118 /* 1119 * If we're a stand alone event or group leader, we go to the context 1120 * list, group events are kept attached to the group so that 1121 * perf_group_detach can, at all times, locate all siblings. 1122 */ 1123 if (event->group_leader == event) { 1124 struct list_head *list; 1125 1126 if (is_software_event(event)) 1127 event->group_flags |= PERF_GROUP_SOFTWARE; 1128 1129 list = ctx_group_list(event, ctx); 1130 list_add_tail(&event->group_entry, list); 1131 } 1132 1133 if (is_cgroup_event(event)) 1134 ctx->nr_cgroups++; 1135 1136 if (has_branch_stack(event)) 1137 ctx->nr_branch_stack++; 1138 1139 list_add_rcu(&event->event_entry, &ctx->event_list); 1140 if (!ctx->nr_events) 1141 perf_pmu_rotate_start(ctx->pmu); 1142 ctx->nr_events++; 1143 if (event->attr.inherit_stat) 1144 ctx->nr_stat++; 1145 1146 ctx->generation++; 1147 } 1148 1149 /* 1150 * Initialize event state based on the perf_event_attr::disabled. 1151 */ 1152 static inline void perf_event__state_init(struct perf_event *event) 1153 { 1154 event->state = event->attr.disabled ? PERF_EVENT_STATE_OFF : 1155 PERF_EVENT_STATE_INACTIVE; 1156 } 1157 1158 /* 1159 * Called at perf_event creation and when events are attached/detached from a 1160 * group. 1161 */ 1162 static void perf_event__read_size(struct perf_event *event) 1163 { 1164 int entry = sizeof(u64); /* value */ 1165 int size = 0; 1166 int nr = 1; 1167 1168 if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) 1169 size += sizeof(u64); 1170 1171 if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) 1172 size += sizeof(u64); 1173 1174 if (event->attr.read_format & PERF_FORMAT_ID) 1175 entry += sizeof(u64); 1176 1177 if (event->attr.read_format & PERF_FORMAT_GROUP) { 1178 nr += event->group_leader->nr_siblings; 1179 size += sizeof(u64); 1180 } 1181 1182 size += entry * nr; 1183 event->read_size = size; 1184 } 1185 1186 static void perf_event__header_size(struct perf_event *event) 1187 { 1188 struct perf_sample_data *data; 1189 u64 sample_type = event->attr.sample_type; 1190 u16 size = 0; 1191 1192 perf_event__read_size(event); 1193 1194 if (sample_type & PERF_SAMPLE_IP) 1195 size += sizeof(data->ip); 1196 1197 if (sample_type & PERF_SAMPLE_ADDR) 1198 size += sizeof(data->addr); 1199 1200 if (sample_type & PERF_SAMPLE_PERIOD) 1201 size += sizeof(data->period); 1202 1203 if (sample_type & PERF_SAMPLE_WEIGHT) 1204 size += sizeof(data->weight); 1205 1206 if (sample_type & PERF_SAMPLE_READ) 1207 size += event->read_size; 1208 1209 if (sample_type & PERF_SAMPLE_DATA_SRC) 1210 size += sizeof(data->data_src.val); 1211 1212 if (sample_type & PERF_SAMPLE_TRANSACTION) 1213 size += sizeof(data->txn); 1214 1215 event->header_size = size; 1216 } 1217 1218 static void perf_event__id_header_size(struct perf_event *event) 1219 { 1220 struct perf_sample_data *data; 1221 u64 sample_type = event->attr.sample_type; 1222 u16 size = 0; 1223 1224 if (sample_type & PERF_SAMPLE_TID) 1225 size += sizeof(data->tid_entry); 1226 1227 if (sample_type & PERF_SAMPLE_TIME) 1228 size += sizeof(data->time); 1229 1230 if (sample_type & PERF_SAMPLE_IDENTIFIER) 1231 size += sizeof(data->id); 1232 1233 if (sample_type & PERF_SAMPLE_ID) 1234 size += sizeof(data->id); 1235 1236 if (sample_type & PERF_SAMPLE_STREAM_ID) 1237 size += sizeof(data->stream_id); 1238 1239 if (sample_type & PERF_SAMPLE_CPU) 1240 size += sizeof(data->cpu_entry); 1241 1242 event->id_header_size = size; 1243 } 1244 1245 static void perf_group_attach(struct perf_event *event) 1246 { 1247 struct perf_event *group_leader = event->group_leader, *pos; 1248 1249 /* 1250 * We can have double attach due to group movement in perf_event_open. 1251 */ 1252 if (event->attach_state & PERF_ATTACH_GROUP) 1253 return; 1254 1255 event->attach_state |= PERF_ATTACH_GROUP; 1256 1257 if (group_leader == event) 1258 return; 1259 1260 if (group_leader->group_flags & PERF_GROUP_SOFTWARE && 1261 !is_software_event(event)) 1262 group_leader->group_flags &= ~PERF_GROUP_SOFTWARE; 1263 1264 list_add_tail(&event->group_entry, &group_leader->sibling_list); 1265 group_leader->nr_siblings++; 1266 1267 perf_event__header_size(group_leader); 1268 1269 list_for_each_entry(pos, &group_leader->sibling_list, group_entry) 1270 perf_event__header_size(pos); 1271 } 1272 1273 /* 1274 * Remove a event from the lists for its context. 1275 * Must be called with ctx->mutex and ctx->lock held. 1276 */ 1277 static void 1278 list_del_event(struct perf_event *event, struct perf_event_context *ctx) 1279 { 1280 struct perf_cpu_context *cpuctx; 1281 /* 1282 * We can have double detach due to exit/hot-unplug + close. 1283 */ 1284 if (!(event->attach_state & PERF_ATTACH_CONTEXT)) 1285 return; 1286 1287 event->attach_state &= ~PERF_ATTACH_CONTEXT; 1288 1289 if (is_cgroup_event(event)) { 1290 ctx->nr_cgroups--; 1291 cpuctx = __get_cpu_context(ctx); 1292 /* 1293 * if there are no more cgroup events 1294 * then cler cgrp to avoid stale pointer 1295 * in update_cgrp_time_from_cpuctx() 1296 */ 1297 if (!ctx->nr_cgroups) 1298 cpuctx->cgrp = NULL; 1299 } 1300 1301 if (has_branch_stack(event)) 1302 ctx->nr_branch_stack--; 1303 1304 ctx->nr_events--; 1305 if (event->attr.inherit_stat) 1306 ctx->nr_stat--; 1307 1308 list_del_rcu(&event->event_entry); 1309 1310 if (event->group_leader == event) 1311 list_del_init(&event->group_entry); 1312 1313 update_group_times(event); 1314 1315 /* 1316 * If event was in error state, then keep it 1317 * that way, otherwise bogus counts will be 1318 * returned on read(). The only way to get out 1319 * of error state is by explicit re-enabling 1320 * of the event 1321 */ 1322 if (event->state > PERF_EVENT_STATE_OFF) 1323 event->state = PERF_EVENT_STATE_OFF; 1324 1325 ctx->generation++; 1326 } 1327 1328 static void perf_group_detach(struct perf_event *event) 1329 { 1330 struct perf_event *sibling, *tmp; 1331 struct list_head *list = NULL; 1332 1333 /* 1334 * We can have double detach due to exit/hot-unplug + close. 1335 */ 1336 if (!(event->attach_state & PERF_ATTACH_GROUP)) 1337 return; 1338 1339 event->attach_state &= ~PERF_ATTACH_GROUP; 1340 1341 /* 1342 * If this is a sibling, remove it from its group. 1343 */ 1344 if (event->group_leader != event) { 1345 list_del_init(&event->group_entry); 1346 event->group_leader->nr_siblings--; 1347 goto out; 1348 } 1349 1350 if (!list_empty(&event->group_entry)) 1351 list = &event->group_entry; 1352 1353 /* 1354 * If this was a group event with sibling events then 1355 * upgrade the siblings to singleton events by adding them 1356 * to whatever list we are on. 1357 */ 1358 list_for_each_entry_safe(sibling, tmp, &event->sibling_list, group_entry) { 1359 if (list) 1360 list_move_tail(&sibling->group_entry, list); 1361 sibling->group_leader = sibling; 1362 1363 /* Inherit group flags from the previous leader */ 1364 sibling->group_flags = event->group_flags; 1365 } 1366 1367 out: 1368 perf_event__header_size(event->group_leader); 1369 1370 list_for_each_entry(tmp, &event->group_leader->sibling_list, group_entry) 1371 perf_event__header_size(tmp); 1372 } 1373 1374 static inline int 1375 event_filter_match(struct perf_event *event) 1376 { 1377 return (event->cpu == -1 || event->cpu == smp_processor_id()) 1378 && perf_cgroup_match(event); 1379 } 1380 1381 static void 1382 event_sched_out(struct perf_event *event, 1383 struct perf_cpu_context *cpuctx, 1384 struct perf_event_context *ctx) 1385 { 1386 u64 tstamp = perf_event_time(event); 1387 u64 delta; 1388 /* 1389 * An event which could not be activated because of 1390 * filter mismatch still needs to have its timings 1391 * maintained, otherwise bogus information is return 1392 * via read() for time_enabled, time_running: 1393 */ 1394 if (event->state == PERF_EVENT_STATE_INACTIVE 1395 && !event_filter_match(event)) { 1396 delta = tstamp - event->tstamp_stopped; 1397 event->tstamp_running += delta; 1398 event->tstamp_stopped = tstamp; 1399 } 1400 1401 if (event->state != PERF_EVENT_STATE_ACTIVE) 1402 return; 1403 1404 perf_pmu_disable(event->pmu); 1405 1406 event->state = PERF_EVENT_STATE_INACTIVE; 1407 if (event->pending_disable) { 1408 event->pending_disable = 0; 1409 event->state = PERF_EVENT_STATE_OFF; 1410 } 1411 event->tstamp_stopped = tstamp; 1412 event->pmu->del(event, 0); 1413 event->oncpu = -1; 1414 1415 if (!is_software_event(event)) 1416 cpuctx->active_oncpu--; 1417 ctx->nr_active--; 1418 if (event->attr.freq && event->attr.sample_freq) 1419 ctx->nr_freq--; 1420 if (event->attr.exclusive || !cpuctx->active_oncpu) 1421 cpuctx->exclusive = 0; 1422 1423 perf_pmu_enable(event->pmu); 1424 } 1425 1426 static void 1427 group_sched_out(struct perf_event *group_event, 1428 struct perf_cpu_context *cpuctx, 1429 struct perf_event_context *ctx) 1430 { 1431 struct perf_event *event; 1432 int state = group_event->state; 1433 1434 event_sched_out(group_event, cpuctx, ctx); 1435 1436 /* 1437 * Schedule out siblings (if any): 1438 */ 1439 list_for_each_entry(event, &group_event->sibling_list, group_entry) 1440 event_sched_out(event, cpuctx, ctx); 1441 1442 if (state == PERF_EVENT_STATE_ACTIVE && group_event->attr.exclusive) 1443 cpuctx->exclusive = 0; 1444 } 1445 1446 /* 1447 * Cross CPU call to remove a performance event 1448 * 1449 * We disable the event on the hardware level first. After that we 1450 * remove it from the context list. 1451 */ 1452 static int __perf_remove_from_context(void *info) 1453 { 1454 struct perf_event *event = info; 1455 struct perf_event_context *ctx = event->ctx; 1456 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx); 1457 1458 raw_spin_lock(&ctx->lock); 1459 event_sched_out(event, cpuctx, ctx); 1460 list_del_event(event, ctx); 1461 if (!ctx->nr_events && cpuctx->task_ctx == ctx) { 1462 ctx->is_active = 0; 1463 cpuctx->task_ctx = NULL; 1464 } 1465 raw_spin_unlock(&ctx->lock); 1466 1467 return 0; 1468 } 1469 1470 1471 /* 1472 * Remove the event from a task's (or a CPU's) list of events. 1473 * 1474 * CPU events are removed with a smp call. For task events we only 1475 * call when the task is on a CPU. 1476 * 1477 * If event->ctx is a cloned context, callers must make sure that 1478 * every task struct that event->ctx->task could possibly point to 1479 * remains valid. This is OK when called from perf_release since 1480 * that only calls us on the top-level context, which can't be a clone. 1481 * When called from perf_event_exit_task, it's OK because the 1482 * context has been detached from its task. 1483 */ 1484 static void perf_remove_from_context(struct perf_event *event) 1485 { 1486 struct perf_event_context *ctx = event->ctx; 1487 struct task_struct *task = ctx->task; 1488 1489 lockdep_assert_held(&ctx->mutex); 1490 1491 if (!task) { 1492 /* 1493 * Per cpu events are removed via an smp call and 1494 * the removal is always successful. 1495 */ 1496 cpu_function_call(event->cpu, __perf_remove_from_context, event); 1497 return; 1498 } 1499 1500 retry: 1501 if (!task_function_call(task, __perf_remove_from_context, event)) 1502 return; 1503 1504 raw_spin_lock_irq(&ctx->lock); 1505 /* 1506 * If we failed to find a running task, but find the context active now 1507 * that we've acquired the ctx->lock, retry. 1508 */ 1509 if (ctx->is_active) { 1510 raw_spin_unlock_irq(&ctx->lock); 1511 goto retry; 1512 } 1513 1514 /* 1515 * Since the task isn't running, its safe to remove the event, us 1516 * holding the ctx->lock ensures the task won't get scheduled in. 1517 */ 1518 list_del_event(event, ctx); 1519 raw_spin_unlock_irq(&ctx->lock); 1520 } 1521 1522 /* 1523 * Cross CPU call to disable a performance event 1524 */ 1525 int __perf_event_disable(void *info) 1526 { 1527 struct perf_event *event = info; 1528 struct perf_event_context *ctx = event->ctx; 1529 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx); 1530 1531 /* 1532 * If this is a per-task event, need to check whether this 1533 * event's task is the current task on this cpu. 1534 * 1535 * Can trigger due to concurrent perf_event_context_sched_out() 1536 * flipping contexts around. 1537 */ 1538 if (ctx->task && cpuctx->task_ctx != ctx) 1539 return -EINVAL; 1540 1541 raw_spin_lock(&ctx->lock); 1542 1543 /* 1544 * If the event is on, turn it off. 1545 * If it is in error state, leave it in error state. 1546 */ 1547 if (event->state >= PERF_EVENT_STATE_INACTIVE) { 1548 update_context_time(ctx); 1549 update_cgrp_time_from_event(event); 1550 update_group_times(event); 1551 if (event == event->group_leader) 1552 group_sched_out(event, cpuctx, ctx); 1553 else 1554 event_sched_out(event, cpuctx, ctx); 1555 event->state = PERF_EVENT_STATE_OFF; 1556 } 1557 1558 raw_spin_unlock(&ctx->lock); 1559 1560 return 0; 1561 } 1562 1563 /* 1564 * Disable a event. 1565 * 1566 * If event->ctx is a cloned context, callers must make sure that 1567 * every task struct that event->ctx->task could possibly point to 1568 * remains valid. This condition is satisifed when called through 1569 * perf_event_for_each_child or perf_event_for_each because they 1570 * hold the top-level event's child_mutex, so any descendant that 1571 * goes to exit will block in sync_child_event. 1572 * When called from perf_pending_event it's OK because event->ctx 1573 * is the current context on this CPU and preemption is disabled, 1574 * hence we can't get into perf_event_task_sched_out for this context. 1575 */ 1576 void perf_event_disable(struct perf_event *event) 1577 { 1578 struct perf_event_context *ctx = event->ctx; 1579 struct task_struct *task = ctx->task; 1580 1581 if (!task) { 1582 /* 1583 * Disable the event on the cpu that it's on 1584 */ 1585 cpu_function_call(event->cpu, __perf_event_disable, event); 1586 return; 1587 } 1588 1589 retry: 1590 if (!task_function_call(task, __perf_event_disable, event)) 1591 return; 1592 1593 raw_spin_lock_irq(&ctx->lock); 1594 /* 1595 * If the event is still active, we need to retry the cross-call. 1596 */ 1597 if (event->state == PERF_EVENT_STATE_ACTIVE) { 1598 raw_spin_unlock_irq(&ctx->lock); 1599 /* 1600 * Reload the task pointer, it might have been changed by 1601 * a concurrent perf_event_context_sched_out(). 1602 */ 1603 task = ctx->task; 1604 goto retry; 1605 } 1606 1607 /* 1608 * Since we have the lock this context can't be scheduled 1609 * in, so we can change the state safely. 1610 */ 1611 if (event->state == PERF_EVENT_STATE_INACTIVE) { 1612 update_group_times(event); 1613 event->state = PERF_EVENT_STATE_OFF; 1614 } 1615 raw_spin_unlock_irq(&ctx->lock); 1616 } 1617 EXPORT_SYMBOL_GPL(perf_event_disable); 1618 1619 static void perf_set_shadow_time(struct perf_event *event, 1620 struct perf_event_context *ctx, 1621 u64 tstamp) 1622 { 1623 /* 1624 * use the correct time source for the time snapshot 1625 * 1626 * We could get by without this by leveraging the 1627 * fact that to get to this function, the caller 1628 * has most likely already called update_context_time() 1629 * and update_cgrp_time_xx() and thus both timestamp 1630 * are identical (or very close). Given that tstamp is, 1631 * already adjusted for cgroup, we could say that: 1632 * tstamp - ctx->timestamp 1633 * is equivalent to 1634 * tstamp - cgrp->timestamp. 1635 * 1636 * Then, in perf_output_read(), the calculation would 1637 * work with no changes because: 1638 * - event is guaranteed scheduled in 1639 * - no scheduled out in between 1640 * - thus the timestamp would be the same 1641 * 1642 * But this is a bit hairy. 1643 * 1644 * So instead, we have an explicit cgroup call to remain 1645 * within the time time source all along. We believe it 1646 * is cleaner and simpler to understand. 1647 */ 1648 if (is_cgroup_event(event)) 1649 perf_cgroup_set_shadow_time(event, tstamp); 1650 else 1651 event->shadow_ctx_time = tstamp - ctx->timestamp; 1652 } 1653 1654 #define MAX_INTERRUPTS (~0ULL) 1655 1656 static void perf_log_throttle(struct perf_event *event, int enable); 1657 1658 static int 1659 event_sched_in(struct perf_event *event, 1660 struct perf_cpu_context *cpuctx, 1661 struct perf_event_context *ctx) 1662 { 1663 u64 tstamp = perf_event_time(event); 1664 int ret = 0; 1665 1666 if (event->state <= PERF_EVENT_STATE_OFF) 1667 return 0; 1668 1669 event->state = PERF_EVENT_STATE_ACTIVE; 1670 event->oncpu = smp_processor_id(); 1671 1672 /* 1673 * Unthrottle events, since we scheduled we might have missed several 1674 * ticks already, also for a heavily scheduling task there is little 1675 * guarantee it'll get a tick in a timely manner. 1676 */ 1677 if (unlikely(event->hw.interrupts == MAX_INTERRUPTS)) { 1678 perf_log_throttle(event, 1); 1679 event->hw.interrupts = 0; 1680 } 1681 1682 /* 1683 * The new state must be visible before we turn it on in the hardware: 1684 */ 1685 smp_wmb(); 1686 1687 perf_pmu_disable(event->pmu); 1688 1689 if (event->pmu->add(event, PERF_EF_START)) { 1690 event->state = PERF_EVENT_STATE_INACTIVE; 1691 event->oncpu = -1; 1692 ret = -EAGAIN; 1693 goto out; 1694 } 1695 1696 event->tstamp_running += tstamp - event->tstamp_stopped; 1697 1698 perf_set_shadow_time(event, ctx, tstamp); 1699 1700 if (!is_software_event(event)) 1701 cpuctx->active_oncpu++; 1702 ctx->nr_active++; 1703 if (event->attr.freq && event->attr.sample_freq) 1704 ctx->nr_freq++; 1705 1706 if (event->attr.exclusive) 1707 cpuctx->exclusive = 1; 1708 1709 out: 1710 perf_pmu_enable(event->pmu); 1711 1712 return ret; 1713 } 1714 1715 static int 1716 group_sched_in(struct perf_event *group_event, 1717 struct perf_cpu_context *cpuctx, 1718 struct perf_event_context *ctx) 1719 { 1720 struct perf_event *event, *partial_group = NULL; 1721 struct pmu *pmu = ctx->pmu; 1722 u64 now = ctx->time; 1723 bool simulate = false; 1724 1725 if (group_event->state == PERF_EVENT_STATE_OFF) 1726 return 0; 1727 1728 pmu->start_txn(pmu); 1729 1730 if (event_sched_in(group_event, cpuctx, ctx)) { 1731 pmu->cancel_txn(pmu); 1732 perf_cpu_hrtimer_restart(cpuctx); 1733 return -EAGAIN; 1734 } 1735 1736 /* 1737 * Schedule in siblings as one group (if any): 1738 */ 1739 list_for_each_entry(event, &group_event->sibling_list, group_entry) { 1740 if (event_sched_in(event, cpuctx, ctx)) { 1741 partial_group = event; 1742 goto group_error; 1743 } 1744 } 1745 1746 if (!pmu->commit_txn(pmu)) 1747 return 0; 1748 1749 group_error: 1750 /* 1751 * Groups can be scheduled in as one unit only, so undo any 1752 * partial group before returning: 1753 * The events up to the failed event are scheduled out normally, 1754 * tstamp_stopped will be updated. 1755 * 1756 * The failed events and the remaining siblings need to have 1757 * their timings updated as if they had gone thru event_sched_in() 1758 * and event_sched_out(). This is required to get consistent timings 1759 * across the group. This also takes care of the case where the group 1760 * could never be scheduled by ensuring tstamp_stopped is set to mark 1761 * the time the event was actually stopped, such that time delta 1762 * calculation in update_event_times() is correct. 1763 */ 1764 list_for_each_entry(event, &group_event->sibling_list, group_entry) { 1765 if (event == partial_group) 1766 simulate = true; 1767 1768 if (simulate) { 1769 event->tstamp_running += now - event->tstamp_stopped; 1770 event->tstamp_stopped = now; 1771 } else { 1772 event_sched_out(event, cpuctx, ctx); 1773 } 1774 } 1775 event_sched_out(group_event, cpuctx, ctx); 1776 1777 pmu->cancel_txn(pmu); 1778 1779 perf_cpu_hrtimer_restart(cpuctx); 1780 1781 return -EAGAIN; 1782 } 1783 1784 /* 1785 * Work out whether we can put this event group on the CPU now. 1786 */ 1787 static int group_can_go_on(struct perf_event *event, 1788 struct perf_cpu_context *cpuctx, 1789 int can_add_hw) 1790 { 1791 /* 1792 * Groups consisting entirely of software events can always go on. 1793 */ 1794 if (event->group_flags & PERF_GROUP_SOFTWARE) 1795 return 1; 1796 /* 1797 * If an exclusive group is already on, no other hardware 1798 * events can go on. 1799 */ 1800 if (cpuctx->exclusive) 1801 return 0; 1802 /* 1803 * If this group is exclusive and there are already 1804 * events on the CPU, it can't go on. 1805 */ 1806 if (event->attr.exclusive && cpuctx->active_oncpu) 1807 return 0; 1808 /* 1809 * Otherwise, try to add it if all previous groups were able 1810 * to go on. 1811 */ 1812 return can_add_hw; 1813 } 1814 1815 static void add_event_to_ctx(struct perf_event *event, 1816 struct perf_event_context *ctx) 1817 { 1818 u64 tstamp = perf_event_time(event); 1819 1820 list_add_event(event, ctx); 1821 perf_group_attach(event); 1822 event->tstamp_enabled = tstamp; 1823 event->tstamp_running = tstamp; 1824 event->tstamp_stopped = tstamp; 1825 } 1826 1827 static void task_ctx_sched_out(struct perf_event_context *ctx); 1828 static void 1829 ctx_sched_in(struct perf_event_context *ctx, 1830 struct perf_cpu_context *cpuctx, 1831 enum event_type_t event_type, 1832 struct task_struct *task); 1833 1834 static void perf_event_sched_in(struct perf_cpu_context *cpuctx, 1835 struct perf_event_context *ctx, 1836 struct task_struct *task) 1837 { 1838 cpu_ctx_sched_in(cpuctx, EVENT_PINNED, task); 1839 if (ctx) 1840 ctx_sched_in(ctx, cpuctx, EVENT_PINNED, task); 1841 cpu_ctx_sched_in(cpuctx, EVENT_FLEXIBLE, task); 1842 if (ctx) 1843 ctx_sched_in(ctx, cpuctx, EVENT_FLEXIBLE, task); 1844 } 1845 1846 /* 1847 * Cross CPU call to install and enable a performance event 1848 * 1849 * Must be called with ctx->mutex held 1850 */ 1851 static int __perf_install_in_context(void *info) 1852 { 1853 struct perf_event *event = info; 1854 struct perf_event_context *ctx = event->ctx; 1855 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx); 1856 struct perf_event_context *task_ctx = cpuctx->task_ctx; 1857 struct task_struct *task = current; 1858 1859 perf_ctx_lock(cpuctx, task_ctx); 1860 perf_pmu_disable(cpuctx->ctx.pmu); 1861 1862 /* 1863 * If there was an active task_ctx schedule it out. 1864 */ 1865 if (task_ctx) 1866 task_ctx_sched_out(task_ctx); 1867 1868 /* 1869 * If the context we're installing events in is not the 1870 * active task_ctx, flip them. 1871 */ 1872 if (ctx->task && task_ctx != ctx) { 1873 if (task_ctx) 1874 raw_spin_unlock(&task_ctx->lock); 1875 raw_spin_lock(&ctx->lock); 1876 task_ctx = ctx; 1877 } 1878 1879 if (task_ctx) { 1880 cpuctx->task_ctx = task_ctx; 1881 task = task_ctx->task; 1882 } 1883 1884 cpu_ctx_sched_out(cpuctx, EVENT_ALL); 1885 1886 update_context_time(ctx); 1887 /* 1888 * update cgrp time only if current cgrp 1889 * matches event->cgrp. Must be done before 1890 * calling add_event_to_ctx() 1891 */ 1892 update_cgrp_time_from_event(event); 1893 1894 add_event_to_ctx(event, ctx); 1895 1896 /* 1897 * Schedule everything back in 1898 */ 1899 perf_event_sched_in(cpuctx, task_ctx, task); 1900 1901 perf_pmu_enable(cpuctx->ctx.pmu); 1902 perf_ctx_unlock(cpuctx, task_ctx); 1903 1904 return 0; 1905 } 1906 1907 /* 1908 * Attach a performance event to a context 1909 * 1910 * First we add the event to the list with the hardware enable bit 1911 * in event->hw_config cleared. 1912 * 1913 * If the event is attached to a task which is on a CPU we use a smp 1914 * call to enable it in the task context. The task might have been 1915 * scheduled away, but we check this in the smp call again. 1916 */ 1917 static void 1918 perf_install_in_context(struct perf_event_context *ctx, 1919 struct perf_event *event, 1920 int cpu) 1921 { 1922 struct task_struct *task = ctx->task; 1923 1924 lockdep_assert_held(&ctx->mutex); 1925 1926 event->ctx = ctx; 1927 if (event->cpu != -1) 1928 event->cpu = cpu; 1929 1930 if (!task) { 1931 /* 1932 * Per cpu events are installed via an smp call and 1933 * the install is always successful. 1934 */ 1935 cpu_function_call(cpu, __perf_install_in_context, event); 1936 return; 1937 } 1938 1939 retry: 1940 if (!task_function_call(task, __perf_install_in_context, event)) 1941 return; 1942 1943 raw_spin_lock_irq(&ctx->lock); 1944 /* 1945 * If we failed to find a running task, but find the context active now 1946 * that we've acquired the ctx->lock, retry. 1947 */ 1948 if (ctx->is_active) { 1949 raw_spin_unlock_irq(&ctx->lock); 1950 goto retry; 1951 } 1952 1953 /* 1954 * Since the task isn't running, its safe to add the event, us holding 1955 * the ctx->lock ensures the task won't get scheduled in. 1956 */ 1957 add_event_to_ctx(event, ctx); 1958 raw_spin_unlock_irq(&ctx->lock); 1959 } 1960 1961 /* 1962 * Put a event into inactive state and update time fields. 1963 * Enabling the leader of a group effectively enables all 1964 * the group members that aren't explicitly disabled, so we 1965 * have to update their ->tstamp_enabled also. 1966 * Note: this works for group members as well as group leaders 1967 * since the non-leader members' sibling_lists will be empty. 1968 */ 1969 static void __perf_event_mark_enabled(struct perf_event *event) 1970 { 1971 struct perf_event *sub; 1972 u64 tstamp = perf_event_time(event); 1973 1974 event->state = PERF_EVENT_STATE_INACTIVE; 1975 event->tstamp_enabled = tstamp - event->total_time_enabled; 1976 list_for_each_entry(sub, &event->sibling_list, group_entry) { 1977 if (sub->state >= PERF_EVENT_STATE_INACTIVE) 1978 sub->tstamp_enabled = tstamp - sub->total_time_enabled; 1979 } 1980 } 1981 1982 /* 1983 * Cross CPU call to enable a performance event 1984 */ 1985 static int __perf_event_enable(void *info) 1986 { 1987 struct perf_event *event = info; 1988 struct perf_event_context *ctx = event->ctx; 1989 struct perf_event *leader = event->group_leader; 1990 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx); 1991 int err; 1992 1993 /* 1994 * There's a time window between 'ctx->is_active' check 1995 * in perf_event_enable function and this place having: 1996 * - IRQs on 1997 * - ctx->lock unlocked 1998 * 1999 * where the task could be killed and 'ctx' deactivated 2000 * by perf_event_exit_task. 2001 */ 2002 if (!ctx->is_active) 2003 return -EINVAL; 2004 2005 raw_spin_lock(&ctx->lock); 2006 update_context_time(ctx); 2007 2008 if (event->state >= PERF_EVENT_STATE_INACTIVE) 2009 goto unlock; 2010 2011 /* 2012 * set current task's cgroup time reference point 2013 */ 2014 perf_cgroup_set_timestamp(current, ctx); 2015 2016 __perf_event_mark_enabled(event); 2017 2018 if (!event_filter_match(event)) { 2019 if (is_cgroup_event(event)) 2020 perf_cgroup_defer_enabled(event); 2021 goto unlock; 2022 } 2023 2024 /* 2025 * If the event is in a group and isn't the group leader, 2026 * then don't put it on unless the group is on. 2027 */ 2028 if (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE) 2029 goto unlock; 2030 2031 if (!group_can_go_on(event, cpuctx, 1)) { 2032 err = -EEXIST; 2033 } else { 2034 if (event == leader) 2035 err = group_sched_in(event, cpuctx, ctx); 2036 else 2037 err = event_sched_in(event, cpuctx, ctx); 2038 } 2039 2040 if (err) { 2041 /* 2042 * If this event can't go on and it's part of a 2043 * group, then the whole group has to come off. 2044 */ 2045 if (leader != event) { 2046 group_sched_out(leader, cpuctx, ctx); 2047 perf_cpu_hrtimer_restart(cpuctx); 2048 } 2049 if (leader->attr.pinned) { 2050 update_group_times(leader); 2051 leader->state = PERF_EVENT_STATE_ERROR; 2052 } 2053 } 2054 2055 unlock: 2056 raw_spin_unlock(&ctx->lock); 2057 2058 return 0; 2059 } 2060 2061 /* 2062 * Enable a event. 2063 * 2064 * If event->ctx is a cloned context, callers must make sure that 2065 * every task struct that event->ctx->task could possibly point to 2066 * remains valid. This condition is satisfied when called through 2067 * perf_event_for_each_child or perf_event_for_each as described 2068 * for perf_event_disable. 2069 */ 2070 void perf_event_enable(struct perf_event *event) 2071 { 2072 struct perf_event_context *ctx = event->ctx; 2073 struct task_struct *task = ctx->task; 2074 2075 if (!task) { 2076 /* 2077 * Enable the event on the cpu that it's on 2078 */ 2079 cpu_function_call(event->cpu, __perf_event_enable, event); 2080 return; 2081 } 2082 2083 raw_spin_lock_irq(&ctx->lock); 2084 if (event->state >= PERF_EVENT_STATE_INACTIVE) 2085 goto out; 2086 2087 /* 2088 * If the event is in error state, clear that first. 2089 * That way, if we see the event in error state below, we 2090 * know that it has gone back into error state, as distinct 2091 * from the task having been scheduled away before the 2092 * cross-call arrived. 2093 */ 2094 if (event->state == PERF_EVENT_STATE_ERROR) 2095 event->state = PERF_EVENT_STATE_OFF; 2096 2097 retry: 2098 if (!ctx->is_active) { 2099 __perf_event_mark_enabled(event); 2100 goto out; 2101 } 2102 2103 raw_spin_unlock_irq(&ctx->lock); 2104 2105 if (!task_function_call(task, __perf_event_enable, event)) 2106 return; 2107 2108 raw_spin_lock_irq(&ctx->lock); 2109 2110 /* 2111 * If the context is active and the event is still off, 2112 * we need to retry the cross-call. 2113 */ 2114 if (ctx->is_active && event->state == PERF_EVENT_STATE_OFF) { 2115 /* 2116 * task could have been flipped by a concurrent 2117 * perf_event_context_sched_out() 2118 */ 2119 task = ctx->task; 2120 goto retry; 2121 } 2122 2123 out: 2124 raw_spin_unlock_irq(&ctx->lock); 2125 } 2126 EXPORT_SYMBOL_GPL(perf_event_enable); 2127 2128 int perf_event_refresh(struct perf_event *event, int refresh) 2129 { 2130 /* 2131 * not supported on inherited events 2132 */ 2133 if (event->attr.inherit || !is_sampling_event(event)) 2134 return -EINVAL; 2135 2136 atomic_add(refresh, &event->event_limit); 2137 perf_event_enable(event); 2138 2139 return 0; 2140 } 2141 EXPORT_SYMBOL_GPL(perf_event_refresh); 2142 2143 static void ctx_sched_out(struct perf_event_context *ctx, 2144 struct perf_cpu_context *cpuctx, 2145 enum event_type_t event_type) 2146 { 2147 struct perf_event *event; 2148 int is_active = ctx->is_active; 2149 2150 ctx->is_active &= ~event_type; 2151 if (likely(!ctx->nr_events)) 2152 return; 2153 2154 update_context_time(ctx); 2155 update_cgrp_time_from_cpuctx(cpuctx); 2156 if (!ctx->nr_active) 2157 return; 2158 2159 perf_pmu_disable(ctx->pmu); 2160 if ((is_active & EVENT_PINNED) && (event_type & EVENT_PINNED)) { 2161 list_for_each_entry(event, &ctx->pinned_groups, group_entry) 2162 group_sched_out(event, cpuctx, ctx); 2163 } 2164 2165 if ((is_active & EVENT_FLEXIBLE) && (event_type & EVENT_FLEXIBLE)) { 2166 list_for_each_entry(event, &ctx->flexible_groups, group_entry) 2167 group_sched_out(event, cpuctx, ctx); 2168 } 2169 perf_pmu_enable(ctx->pmu); 2170 } 2171 2172 /* 2173 * Test whether two contexts are equivalent, i.e. whether they have both been 2174 * cloned from the same version of the same context. 2175 * 2176 * Equivalence is measured using a generation number in the context that is 2177 * incremented on each modification to it; see unclone_ctx(), list_add_event() 2178 * and list_del_event(). 2179 */ 2180 static int context_equiv(struct perf_event_context *ctx1, 2181 struct perf_event_context *ctx2) 2182 { 2183 /* Pinning disables the swap optimization */ 2184 if (ctx1->pin_count || ctx2->pin_count) 2185 return 0; 2186 2187 /* If ctx1 is the parent of ctx2 */ 2188 if (ctx1 == ctx2->parent_ctx && ctx1->generation == ctx2->parent_gen) 2189 return 1; 2190 2191 /* If ctx2 is the parent of ctx1 */ 2192 if (ctx1->parent_ctx == ctx2 && ctx1->parent_gen == ctx2->generation) 2193 return 1; 2194 2195 /* 2196 * If ctx1 and ctx2 have the same parent; we flatten the parent 2197 * hierarchy, see perf_event_init_context(). 2198 */ 2199 if (ctx1->parent_ctx && ctx1->parent_ctx == ctx2->parent_ctx && 2200 ctx1->parent_gen == ctx2->parent_gen) 2201 return 1; 2202 2203 /* Unmatched */ 2204 return 0; 2205 } 2206 2207 static void __perf_event_sync_stat(struct perf_event *event, 2208 struct perf_event *next_event) 2209 { 2210 u64 value; 2211 2212 if (!event->attr.inherit_stat) 2213 return; 2214 2215 /* 2216 * Update the event value, we cannot use perf_event_read() 2217 * because we're in the middle of a context switch and have IRQs 2218 * disabled, which upsets smp_call_function_single(), however 2219 * we know the event must be on the current CPU, therefore we 2220 * don't need to use it. 2221 */ 2222 switch (event->state) { 2223 case PERF_EVENT_STATE_ACTIVE: 2224 event->pmu->read(event); 2225 /* fall-through */ 2226 2227 case PERF_EVENT_STATE_INACTIVE: 2228 update_event_times(event); 2229 break; 2230 2231 default: 2232 break; 2233 } 2234 2235 /* 2236 * In order to keep per-task stats reliable we need to flip the event 2237 * values when we flip the contexts. 2238 */ 2239 value = local64_read(&next_event->count); 2240 value = local64_xchg(&event->count, value); 2241 local64_set(&next_event->count, value); 2242 2243 swap(event->total_time_enabled, next_event->total_time_enabled); 2244 swap(event->total_time_running, next_event->total_time_running); 2245 2246 /* 2247 * Since we swizzled the values, update the user visible data too. 2248 */ 2249 perf_event_update_userpage(event); 2250 perf_event_update_userpage(next_event); 2251 } 2252 2253 static void perf_event_sync_stat(struct perf_event_context *ctx, 2254 struct perf_event_context *next_ctx) 2255 { 2256 struct perf_event *event, *next_event; 2257 2258 if (!ctx->nr_stat) 2259 return; 2260 2261 update_context_time(ctx); 2262 2263 event = list_first_entry(&ctx->event_list, 2264 struct perf_event, event_entry); 2265 2266 next_event = list_first_entry(&next_ctx->event_list, 2267 struct perf_event, event_entry); 2268 2269 while (&event->event_entry != &ctx->event_list && 2270 &next_event->event_entry != &next_ctx->event_list) { 2271 2272 __perf_event_sync_stat(event, next_event); 2273 2274 event = list_next_entry(event, event_entry); 2275 next_event = list_next_entry(next_event, event_entry); 2276 } 2277 } 2278 2279 static void perf_event_context_sched_out(struct task_struct *task, int ctxn, 2280 struct task_struct *next) 2281 { 2282 struct perf_event_context *ctx = task->perf_event_ctxp[ctxn]; 2283 struct perf_event_context *next_ctx; 2284 struct perf_event_context *parent, *next_parent; 2285 struct perf_cpu_context *cpuctx; 2286 int do_switch = 1; 2287 2288 if (likely(!ctx)) 2289 return; 2290 2291 cpuctx = __get_cpu_context(ctx); 2292 if (!cpuctx->task_ctx) 2293 return; 2294 2295 rcu_read_lock(); 2296 next_ctx = next->perf_event_ctxp[ctxn]; 2297 if (!next_ctx) 2298 goto unlock; 2299 2300 parent = rcu_dereference(ctx->parent_ctx); 2301 next_parent = rcu_dereference(next_ctx->parent_ctx); 2302 2303 /* If neither context have a parent context; they cannot be clones. */ 2304 if (!parent && !next_parent) 2305 goto unlock; 2306 2307 if (next_parent == ctx || next_ctx == parent || next_parent == parent) { 2308 /* 2309 * Looks like the two contexts are clones, so we might be 2310 * able to optimize the context switch. We lock both 2311 * contexts and check that they are clones under the 2312 * lock (including re-checking that neither has been 2313 * uncloned in the meantime). It doesn't matter which 2314 * order we take the locks because no other cpu could 2315 * be trying to lock both of these tasks. 2316 */ 2317 raw_spin_lock(&ctx->lock); 2318 raw_spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING); 2319 if (context_equiv(ctx, next_ctx)) { 2320 /* 2321 * XXX do we need a memory barrier of sorts 2322 * wrt to rcu_dereference() of perf_event_ctxp 2323 */ 2324 task->perf_event_ctxp[ctxn] = next_ctx; 2325 next->perf_event_ctxp[ctxn] = ctx; 2326 ctx->task = next; 2327 next_ctx->task = task; 2328 do_switch = 0; 2329 2330 perf_event_sync_stat(ctx, next_ctx); 2331 } 2332 raw_spin_unlock(&next_ctx->lock); 2333 raw_spin_unlock(&ctx->lock); 2334 } 2335 unlock: 2336 rcu_read_unlock(); 2337 2338 if (do_switch) { 2339 raw_spin_lock(&ctx->lock); 2340 ctx_sched_out(ctx, cpuctx, EVENT_ALL); 2341 cpuctx->task_ctx = NULL; 2342 raw_spin_unlock(&ctx->lock); 2343 } 2344 } 2345 2346 #define for_each_task_context_nr(ctxn) \ 2347 for ((ctxn) = 0; (ctxn) < perf_nr_task_contexts; (ctxn)++) 2348 2349 /* 2350 * Called from scheduler to remove the events of the current task, 2351 * with interrupts disabled. 2352 * 2353 * We stop each event and update the event value in event->count. 2354 * 2355 * This does not protect us against NMI, but disable() 2356 * sets the disabled bit in the control field of event _before_ 2357 * accessing the event control register. If a NMI hits, then it will 2358 * not restart the event. 2359 */ 2360 void __perf_event_task_sched_out(struct task_struct *task, 2361 struct task_struct *next) 2362 { 2363 int ctxn; 2364 2365 for_each_task_context_nr(ctxn) 2366 perf_event_context_sched_out(task, ctxn, next); 2367 2368 /* 2369 * if cgroup events exist on this CPU, then we need 2370 * to check if we have to switch out PMU state. 2371 * cgroup event are system-wide mode only 2372 */ 2373 if (atomic_read(&__get_cpu_var(perf_cgroup_events))) 2374 perf_cgroup_sched_out(task, next); 2375 } 2376 2377 static void task_ctx_sched_out(struct perf_event_context *ctx) 2378 { 2379 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx); 2380 2381 if (!cpuctx->task_ctx) 2382 return; 2383 2384 if (WARN_ON_ONCE(ctx != cpuctx->task_ctx)) 2385 return; 2386 2387 ctx_sched_out(ctx, cpuctx, EVENT_ALL); 2388 cpuctx->task_ctx = NULL; 2389 } 2390 2391 /* 2392 * Called with IRQs disabled 2393 */ 2394 static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx, 2395 enum event_type_t event_type) 2396 { 2397 ctx_sched_out(&cpuctx->ctx, cpuctx, event_type); 2398 } 2399 2400 static void 2401 ctx_pinned_sched_in(struct perf_event_context *ctx, 2402 struct perf_cpu_context *cpuctx) 2403 { 2404 struct perf_event *event; 2405 2406 list_for_each_entry(event, &ctx->pinned_groups, group_entry) { 2407 if (event->state <= PERF_EVENT_STATE_OFF) 2408 continue; 2409 if (!event_filter_match(event)) 2410 continue; 2411 2412 /* may need to reset tstamp_enabled */ 2413 if (is_cgroup_event(event)) 2414 perf_cgroup_mark_enabled(event, ctx); 2415 2416 if (group_can_go_on(event, cpuctx, 1)) 2417 group_sched_in(event, cpuctx, ctx); 2418 2419 /* 2420 * If this pinned group hasn't been scheduled, 2421 * put it in error state. 2422 */ 2423 if (event->state == PERF_EVENT_STATE_INACTIVE) { 2424 update_group_times(event); 2425 event->state = PERF_EVENT_STATE_ERROR; 2426 } 2427 } 2428 } 2429 2430 static void 2431 ctx_flexible_sched_in(struct perf_event_context *ctx, 2432 struct perf_cpu_context *cpuctx) 2433 { 2434 struct perf_event *event; 2435 int can_add_hw = 1; 2436 2437 list_for_each_entry(event, &ctx->flexible_groups, group_entry) { 2438 /* Ignore events in OFF or ERROR state */ 2439 if (event->state <= PERF_EVENT_STATE_OFF) 2440 continue; 2441 /* 2442 * Listen to the 'cpu' scheduling filter constraint 2443 * of events: 2444 */ 2445 if (!event_filter_match(event)) 2446 continue; 2447 2448 /* may need to reset tstamp_enabled */ 2449 if (is_cgroup_event(event)) 2450 perf_cgroup_mark_enabled(event, ctx); 2451 2452 if (group_can_go_on(event, cpuctx, can_add_hw)) { 2453 if (group_sched_in(event, cpuctx, ctx)) 2454 can_add_hw = 0; 2455 } 2456 } 2457 } 2458 2459 static void 2460 ctx_sched_in(struct perf_event_context *ctx, 2461 struct perf_cpu_context *cpuctx, 2462 enum event_type_t event_type, 2463 struct task_struct *task) 2464 { 2465 u64 now; 2466 int is_active = ctx->is_active; 2467 2468 ctx->is_active |= event_type; 2469 if (likely(!ctx->nr_events)) 2470 return; 2471 2472 now = perf_clock(); 2473 ctx->timestamp = now; 2474 perf_cgroup_set_timestamp(task, ctx); 2475 /* 2476 * First go through the list and put on any pinned groups 2477 * in order to give them the best chance of going on. 2478 */ 2479 if (!(is_active & EVENT_PINNED) && (event_type & EVENT_PINNED)) 2480 ctx_pinned_sched_in(ctx, cpuctx); 2481 2482 /* Then walk through the lower prio flexible groups */ 2483 if (!(is_active & EVENT_FLEXIBLE) && (event_type & EVENT_FLEXIBLE)) 2484 ctx_flexible_sched_in(ctx, cpuctx); 2485 } 2486 2487 static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx, 2488 enum event_type_t event_type, 2489 struct task_struct *task) 2490 { 2491 struct perf_event_context *ctx = &cpuctx->ctx; 2492 2493 ctx_sched_in(ctx, cpuctx, event_type, task); 2494 } 2495 2496 static void perf_event_context_sched_in(struct perf_event_context *ctx, 2497 struct task_struct *task) 2498 { 2499 struct perf_cpu_context *cpuctx; 2500 2501 cpuctx = __get_cpu_context(ctx); 2502 if (cpuctx->task_ctx == ctx) 2503 return; 2504 2505 perf_ctx_lock(cpuctx, ctx); 2506 perf_pmu_disable(ctx->pmu); 2507 /* 2508 * We want to keep the following priority order: 2509 * cpu pinned (that don't need to move), task pinned, 2510 * cpu flexible, task flexible. 2511 */ 2512 cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE); 2513 2514 if (ctx->nr_events) 2515 cpuctx->task_ctx = ctx; 2516 2517 perf_event_sched_in(cpuctx, cpuctx->task_ctx, task); 2518 2519 perf_pmu_enable(ctx->pmu); 2520 perf_ctx_unlock(cpuctx, ctx); 2521 2522 /* 2523 * Since these rotations are per-cpu, we need to ensure the 2524 * cpu-context we got scheduled on is actually rotating. 2525 */ 2526 perf_pmu_rotate_start(ctx->pmu); 2527 } 2528 2529 /* 2530 * When sampling the branck stack in system-wide, it may be necessary 2531 * to flush the stack on context switch. This happens when the branch 2532 * stack does not tag its entries with the pid of the current task. 2533 * Otherwise it becomes impossible to associate a branch entry with a 2534 * task. This ambiguity is more likely to appear when the branch stack 2535 * supports priv level filtering and the user sets it to monitor only 2536 * at the user level (which could be a useful measurement in system-wide 2537 * mode). In that case, the risk is high of having a branch stack with 2538 * branch from multiple tasks. Flushing may mean dropping the existing 2539 * entries or stashing them somewhere in the PMU specific code layer. 2540 * 2541 * This function provides the context switch callback to the lower code 2542 * layer. It is invoked ONLY when there is at least one system-wide context 2543 * with at least one active event using taken branch sampling. 2544 */ 2545 static void perf_branch_stack_sched_in(struct task_struct *prev, 2546 struct task_struct *task) 2547 { 2548 struct perf_cpu_context *cpuctx; 2549 struct pmu *pmu; 2550 unsigned long flags; 2551 2552 /* no need to flush branch stack if not changing task */ 2553 if (prev == task) 2554 return; 2555 2556 local_irq_save(flags); 2557 2558 rcu_read_lock(); 2559 2560 list_for_each_entry_rcu(pmu, &pmus, entry) { 2561 cpuctx = this_cpu_ptr(pmu->pmu_cpu_context); 2562 2563 /* 2564 * check if the context has at least one 2565 * event using PERF_SAMPLE_BRANCH_STACK 2566 */ 2567 if (cpuctx->ctx.nr_branch_stack > 0 2568 && pmu->flush_branch_stack) { 2569 2570 perf_ctx_lock(cpuctx, cpuctx->task_ctx); 2571 2572 perf_pmu_disable(pmu); 2573 2574 pmu->flush_branch_stack(); 2575 2576 perf_pmu_enable(pmu); 2577 2578 perf_ctx_unlock(cpuctx, cpuctx->task_ctx); 2579 } 2580 } 2581 2582 rcu_read_unlock(); 2583 2584 local_irq_restore(flags); 2585 } 2586 2587 /* 2588 * Called from scheduler to add the events of the current task 2589 * with interrupts disabled. 2590 * 2591 * We restore the event value and then enable it. 2592 * 2593 * This does not protect us against NMI, but enable() 2594 * sets the enabled bit in the control field of event _before_ 2595 * accessing the event control register. If a NMI hits, then it will 2596 * keep the event running. 2597 */ 2598 void __perf_event_task_sched_in(struct task_struct *prev, 2599 struct task_struct *task) 2600 { 2601 struct perf_event_context *ctx; 2602 int ctxn; 2603 2604 for_each_task_context_nr(ctxn) { 2605 ctx = task->perf_event_ctxp[ctxn]; 2606 if (likely(!ctx)) 2607 continue; 2608 2609 perf_event_context_sched_in(ctx, task); 2610 } 2611 /* 2612 * if cgroup events exist on this CPU, then we need 2613 * to check if we have to switch in PMU state. 2614 * cgroup event are system-wide mode only 2615 */ 2616 if (atomic_read(&__get_cpu_var(perf_cgroup_events))) 2617 perf_cgroup_sched_in(prev, task); 2618 2619 /* check for system-wide branch_stack events */ 2620 if (atomic_read(&__get_cpu_var(perf_branch_stack_events))) 2621 perf_branch_stack_sched_in(prev, task); 2622 } 2623 2624 static u64 perf_calculate_period(struct perf_event *event, u64 nsec, u64 count) 2625 { 2626 u64 frequency = event->attr.sample_freq; 2627 u64 sec = NSEC_PER_SEC; 2628 u64 divisor, dividend; 2629 2630 int count_fls, nsec_fls, frequency_fls, sec_fls; 2631 2632 count_fls = fls64(count); 2633 nsec_fls = fls64(nsec); 2634 frequency_fls = fls64(frequency); 2635 sec_fls = 30; 2636 2637 /* 2638 * We got @count in @nsec, with a target of sample_freq HZ 2639 * the target period becomes: 2640 * 2641 * @count * 10^9 2642 * period = ------------------- 2643 * @nsec * sample_freq 2644 * 2645 */ 2646 2647 /* 2648 * Reduce accuracy by one bit such that @a and @b converge 2649 * to a similar magnitude. 2650 */ 2651 #define REDUCE_FLS(a, b) \ 2652 do { \ 2653 if (a##_fls > b##_fls) { \ 2654 a >>= 1; \ 2655 a##_fls--; \ 2656 } else { \ 2657 b >>= 1; \ 2658 b##_fls--; \ 2659 } \ 2660 } while (0) 2661 2662 /* 2663 * Reduce accuracy until either term fits in a u64, then proceed with 2664 * the other, so that finally we can do a u64/u64 division. 2665 */ 2666 while (count_fls + sec_fls > 64 && nsec_fls + frequency_fls > 64) { 2667 REDUCE_FLS(nsec, frequency); 2668 REDUCE_FLS(sec, count); 2669 } 2670 2671 if (count_fls + sec_fls > 64) { 2672 divisor = nsec * frequency; 2673 2674 while (count_fls + sec_fls > 64) { 2675 REDUCE_FLS(count, sec); 2676 divisor >>= 1; 2677 } 2678 2679 dividend = count * sec; 2680 } else { 2681 dividend = count * sec; 2682 2683 while (nsec_fls + frequency_fls > 64) { 2684 REDUCE_FLS(nsec, frequency); 2685 dividend >>= 1; 2686 } 2687 2688 divisor = nsec * frequency; 2689 } 2690 2691 if (!divisor) 2692 return dividend; 2693 2694 return div64_u64(dividend, divisor); 2695 } 2696 2697 static DEFINE_PER_CPU(int, perf_throttled_count); 2698 static DEFINE_PER_CPU(u64, perf_throttled_seq); 2699 2700 static void perf_adjust_period(struct perf_event *event, u64 nsec, u64 count, bool disable) 2701 { 2702 struct hw_perf_event *hwc = &event->hw; 2703 s64 period, sample_period; 2704 s64 delta; 2705 2706 period = perf_calculate_period(event, nsec, count); 2707 2708 delta = (s64)(period - hwc->sample_period); 2709 delta = (delta + 7) / 8; /* low pass filter */ 2710 2711 sample_period = hwc->sample_period + delta; 2712 2713 if (!sample_period) 2714 sample_period = 1; 2715 2716 hwc->sample_period = sample_period; 2717 2718 if (local64_read(&hwc->period_left) > 8*sample_period) { 2719 if (disable) 2720 event->pmu->stop(event, PERF_EF_UPDATE); 2721 2722 local64_set(&hwc->period_left, 0); 2723 2724 if (disable) 2725 event->pmu->start(event, PERF_EF_RELOAD); 2726 } 2727 } 2728 2729 /* 2730 * combine freq adjustment with unthrottling to avoid two passes over the 2731 * events. At the same time, make sure, having freq events does not change 2732 * the rate of unthrottling as that would introduce bias. 2733 */ 2734 static void perf_adjust_freq_unthr_context(struct perf_event_context *ctx, 2735 int needs_unthr) 2736 { 2737 struct perf_event *event; 2738 struct hw_perf_event *hwc; 2739 u64 now, period = TICK_NSEC; 2740 s64 delta; 2741 2742 /* 2743 * only need to iterate over all events iff: 2744 * - context have events in frequency mode (needs freq adjust) 2745 * - there are events to unthrottle on this cpu 2746 */ 2747 if (!(ctx->nr_freq || needs_unthr)) 2748 return; 2749 2750 raw_spin_lock(&ctx->lock); 2751 perf_pmu_disable(ctx->pmu); 2752 2753 list_for_each_entry_rcu(event, &ctx->event_list, event_entry) { 2754 if (event->state != PERF_EVENT_STATE_ACTIVE) 2755 continue; 2756 2757 if (!event_filter_match(event)) 2758 continue; 2759 2760 perf_pmu_disable(event->pmu); 2761 2762 hwc = &event->hw; 2763 2764 if (hwc->interrupts == MAX_INTERRUPTS) { 2765 hwc->interrupts = 0; 2766 perf_log_throttle(event, 1); 2767 event->pmu->start(event, 0); 2768 } 2769 2770 if (!event->attr.freq || !event->attr.sample_freq) 2771 goto next; 2772 2773 /* 2774 * stop the event and update event->count 2775 */ 2776 event->pmu->stop(event, PERF_EF_UPDATE); 2777 2778 now = local64_read(&event->count); 2779 delta = now - hwc->freq_count_stamp; 2780 hwc->freq_count_stamp = now; 2781 2782 /* 2783 * restart the event 2784 * reload only if value has changed 2785 * we have stopped the event so tell that 2786 * to perf_adjust_period() to avoid stopping it 2787 * twice. 2788 */ 2789 if (delta > 0) 2790 perf_adjust_period(event, period, delta, false); 2791 2792 event->pmu->start(event, delta > 0 ? PERF_EF_RELOAD : 0); 2793 next: 2794 perf_pmu_enable(event->pmu); 2795 } 2796 2797 perf_pmu_enable(ctx->pmu); 2798 raw_spin_unlock(&ctx->lock); 2799 } 2800 2801 /* 2802 * Round-robin a context's events: 2803 */ 2804 static void rotate_ctx(struct perf_event_context *ctx) 2805 { 2806 /* 2807 * Rotate the first entry last of non-pinned groups. Rotation might be 2808 * disabled by the inheritance code. 2809 */ 2810 if (!ctx->rotate_disable) 2811 list_rotate_left(&ctx->flexible_groups); 2812 } 2813 2814 /* 2815 * perf_pmu_rotate_start() and perf_rotate_context() are fully serialized 2816 * because they're strictly cpu affine and rotate_start is called with IRQs 2817 * disabled, while rotate_context is called from IRQ context. 2818 */ 2819 static int perf_rotate_context(struct perf_cpu_context *cpuctx) 2820 { 2821 struct perf_event_context *ctx = NULL; 2822 int rotate = 0, remove = 1; 2823 2824 if (cpuctx->ctx.nr_events) { 2825 remove = 0; 2826 if (cpuctx->ctx.nr_events != cpuctx->ctx.nr_active) 2827 rotate = 1; 2828 } 2829 2830 ctx = cpuctx->task_ctx; 2831 if (ctx && ctx->nr_events) { 2832 remove = 0; 2833 if (ctx->nr_events != ctx->nr_active) 2834 rotate = 1; 2835 } 2836 2837 if (!rotate) 2838 goto done; 2839 2840 perf_ctx_lock(cpuctx, cpuctx->task_ctx); 2841 perf_pmu_disable(cpuctx->ctx.pmu); 2842 2843 cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE); 2844 if (ctx) 2845 ctx_sched_out(ctx, cpuctx, EVENT_FLEXIBLE); 2846 2847 rotate_ctx(&cpuctx->ctx); 2848 if (ctx) 2849 rotate_ctx(ctx); 2850 2851 perf_event_sched_in(cpuctx, ctx, current); 2852 2853 perf_pmu_enable(cpuctx->ctx.pmu); 2854 perf_ctx_unlock(cpuctx, cpuctx->task_ctx); 2855 done: 2856 if (remove) 2857 list_del_init(&cpuctx->rotation_list); 2858 2859 return rotate; 2860 } 2861 2862 #ifdef CONFIG_NO_HZ_FULL 2863 bool perf_event_can_stop_tick(void) 2864 { 2865 if (atomic_read(&nr_freq_events) || 2866 __this_cpu_read(perf_throttled_count)) 2867 return false; 2868 else 2869 return true; 2870 } 2871 #endif 2872 2873 void perf_event_task_tick(void) 2874 { 2875 struct list_head *head = &__get_cpu_var(rotation_list); 2876 struct perf_cpu_context *cpuctx, *tmp; 2877 struct perf_event_context *ctx; 2878 int throttled; 2879 2880 WARN_ON(!irqs_disabled()); 2881 2882 __this_cpu_inc(perf_throttled_seq); 2883 throttled = __this_cpu_xchg(perf_throttled_count, 0); 2884 2885 list_for_each_entry_safe(cpuctx, tmp, head, rotation_list) { 2886 ctx = &cpuctx->ctx; 2887 perf_adjust_freq_unthr_context(ctx, throttled); 2888 2889 ctx = cpuctx->task_ctx; 2890 if (ctx) 2891 perf_adjust_freq_unthr_context(ctx, throttled); 2892 } 2893 } 2894 2895 static int event_enable_on_exec(struct perf_event *event, 2896 struct perf_event_context *ctx) 2897 { 2898 if (!event->attr.enable_on_exec) 2899 return 0; 2900 2901 event->attr.enable_on_exec = 0; 2902 if (event->state >= PERF_EVENT_STATE_INACTIVE) 2903 return 0; 2904 2905 __perf_event_mark_enabled(event); 2906 2907 return 1; 2908 } 2909 2910 /* 2911 * Enable all of a task's events that have been marked enable-on-exec. 2912 * This expects task == current. 2913 */ 2914 static void perf_event_enable_on_exec(struct perf_event_context *ctx) 2915 { 2916 struct perf_event *event; 2917 unsigned long flags; 2918 int enabled = 0; 2919 int ret; 2920 2921 local_irq_save(flags); 2922 if (!ctx || !ctx->nr_events) 2923 goto out; 2924 2925 /* 2926 * We must ctxsw out cgroup events to avoid conflict 2927 * when invoking perf_task_event_sched_in() later on 2928 * in this function. Otherwise we end up trying to 2929 * ctxswin cgroup events which are already scheduled 2930 * in. 2931 */ 2932 perf_cgroup_sched_out(current, NULL); 2933 2934 raw_spin_lock(&ctx->lock); 2935 task_ctx_sched_out(ctx); 2936 2937 list_for_each_entry(event, &ctx->event_list, event_entry) { 2938 ret = event_enable_on_exec(event, ctx); 2939 if (ret) 2940 enabled = 1; 2941 } 2942 2943 /* 2944 * Unclone this context if we enabled any event. 2945 */ 2946 if (enabled) 2947 unclone_ctx(ctx); 2948 2949 raw_spin_unlock(&ctx->lock); 2950 2951 /* 2952 * Also calls ctxswin for cgroup events, if any: 2953 */ 2954 perf_event_context_sched_in(ctx, ctx->task); 2955 out: 2956 local_irq_restore(flags); 2957 } 2958 2959 /* 2960 * Cross CPU call to read the hardware event 2961 */ 2962 static void __perf_event_read(void *info) 2963 { 2964 struct perf_event *event = info; 2965 struct perf_event_context *ctx = event->ctx; 2966 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx); 2967 2968 /* 2969 * If this is a task context, we need to check whether it is 2970 * the current task context of this cpu. If not it has been 2971 * scheduled out before the smp call arrived. In that case 2972 * event->count would have been updated to a recent sample 2973 * when the event was scheduled out. 2974 */ 2975 if (ctx->task && cpuctx->task_ctx != ctx) 2976 return; 2977 2978 raw_spin_lock(&ctx->lock); 2979 if (ctx->is_active) { 2980 update_context_time(ctx); 2981 update_cgrp_time_from_event(event); 2982 } 2983 update_event_times(event); 2984 if (event->state == PERF_EVENT_STATE_ACTIVE) 2985 event->pmu->read(event); 2986 raw_spin_unlock(&ctx->lock); 2987 } 2988 2989 static inline u64 perf_event_count(struct perf_event *event) 2990 { 2991 return local64_read(&event->count) + atomic64_read(&event->child_count); 2992 } 2993 2994 static u64 perf_event_read(struct perf_event *event) 2995 { 2996 /* 2997 * If event is enabled and currently active on a CPU, update the 2998 * value in the event structure: 2999 */ 3000 if (event->state == PERF_EVENT_STATE_ACTIVE) { 3001 smp_call_function_single(event->oncpu, 3002 __perf_event_read, event, 1); 3003 } else if (event->state == PERF_EVENT_STATE_INACTIVE) { 3004 struct perf_event_context *ctx = event->ctx; 3005 unsigned long flags; 3006 3007 raw_spin_lock_irqsave(&ctx->lock, flags); 3008 /* 3009 * may read while context is not active 3010 * (e.g., thread is blocked), in that case 3011 * we cannot update context time 3012 */ 3013 if (ctx->is_active) { 3014 update_context_time(ctx); 3015 update_cgrp_time_from_event(event); 3016 } 3017 update_event_times(event); 3018 raw_spin_unlock_irqrestore(&ctx->lock, flags); 3019 } 3020 3021 return perf_event_count(event); 3022 } 3023 3024 /* 3025 * Initialize the perf_event context in a task_struct: 3026 */ 3027 static void __perf_event_init_context(struct perf_event_context *ctx) 3028 { 3029 raw_spin_lock_init(&ctx->lock); 3030 mutex_init(&ctx->mutex); 3031 INIT_LIST_HEAD(&ctx->pinned_groups); 3032 INIT_LIST_HEAD(&ctx->flexible_groups); 3033 INIT_LIST_HEAD(&ctx->event_list); 3034 atomic_set(&ctx->refcount, 1); 3035 } 3036 3037 static struct perf_event_context * 3038 alloc_perf_context(struct pmu *pmu, struct task_struct *task) 3039 { 3040 struct perf_event_context *ctx; 3041 3042 ctx = kzalloc(sizeof(struct perf_event_context), GFP_KERNEL); 3043 if (!ctx) 3044 return NULL; 3045 3046 __perf_event_init_context(ctx); 3047 if (task) { 3048 ctx->task = task; 3049 get_task_struct(task); 3050 } 3051 ctx->pmu = pmu; 3052 3053 return ctx; 3054 } 3055 3056 static struct task_struct * 3057 find_lively_task_by_vpid(pid_t vpid) 3058 { 3059 struct task_struct *task; 3060 int err; 3061 3062 rcu_read_lock(); 3063 if (!vpid) 3064 task = current; 3065 else 3066 task = find_task_by_vpid(vpid); 3067 if (task) 3068 get_task_struct(task); 3069 rcu_read_unlock(); 3070 3071 if (!task) 3072 return ERR_PTR(-ESRCH); 3073 3074 /* Reuse ptrace permission checks for now. */ 3075 err = -EACCES; 3076 if (!ptrace_may_access(task, PTRACE_MODE_READ)) 3077 goto errout; 3078 3079 return task; 3080 errout: 3081 put_task_struct(task); 3082 return ERR_PTR(err); 3083 3084 } 3085 3086 /* 3087 * Returns a matching context with refcount and pincount. 3088 */ 3089 static struct perf_event_context * 3090 find_get_context(struct pmu *pmu, struct task_struct *task, int cpu) 3091 { 3092 struct perf_event_context *ctx; 3093 struct perf_cpu_context *cpuctx; 3094 unsigned long flags; 3095 int ctxn, err; 3096 3097 if (!task) { 3098 /* Must be root to operate on a CPU event: */ 3099 if (perf_paranoid_cpu() && !capable(CAP_SYS_ADMIN)) 3100 return ERR_PTR(-EACCES); 3101 3102 /* 3103 * We could be clever and allow to attach a event to an 3104 * offline CPU and activate it when the CPU comes up, but 3105 * that's for later. 3106 */ 3107 if (!cpu_online(cpu)) 3108 return ERR_PTR(-ENODEV); 3109 3110 cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu); 3111 ctx = &cpuctx->ctx; 3112 get_ctx(ctx); 3113 ++ctx->pin_count; 3114 3115 return ctx; 3116 } 3117 3118 err = -EINVAL; 3119 ctxn = pmu->task_ctx_nr; 3120 if (ctxn < 0) 3121 goto errout; 3122 3123 retry: 3124 ctx = perf_lock_task_context(task, ctxn, &flags); 3125 if (ctx) { 3126 unclone_ctx(ctx); 3127 ++ctx->pin_count; 3128 raw_spin_unlock_irqrestore(&ctx->lock, flags); 3129 } else { 3130 ctx = alloc_perf_context(pmu, task); 3131 err = -ENOMEM; 3132 if (!ctx) 3133 goto errout; 3134 3135 err = 0; 3136 mutex_lock(&task->perf_event_mutex); 3137 /* 3138 * If it has already passed perf_event_exit_task(). 3139 * we must see PF_EXITING, it takes this mutex too. 3140 */ 3141 if (task->flags & PF_EXITING) 3142 err = -ESRCH; 3143 else if (task->perf_event_ctxp[ctxn]) 3144 err = -EAGAIN; 3145 else { 3146 get_ctx(ctx); 3147 ++ctx->pin_count; 3148 rcu_assign_pointer(task->perf_event_ctxp[ctxn], ctx); 3149 } 3150 mutex_unlock(&task->perf_event_mutex); 3151 3152 if (unlikely(err)) { 3153 put_ctx(ctx); 3154 3155 if (err == -EAGAIN) 3156 goto retry; 3157 goto errout; 3158 } 3159 } 3160 3161 return ctx; 3162 3163 errout: 3164 return ERR_PTR(err); 3165 } 3166 3167 static void perf_event_free_filter(struct perf_event *event); 3168 3169 static void free_event_rcu(struct rcu_head *head) 3170 { 3171 struct perf_event *event; 3172 3173 event = container_of(head, struct perf_event, rcu_head); 3174 if (event->ns) 3175 put_pid_ns(event->ns); 3176 perf_event_free_filter(event); 3177 kfree(event); 3178 } 3179 3180 static void ring_buffer_put(struct ring_buffer *rb); 3181 static void ring_buffer_detach(struct perf_event *event, struct ring_buffer *rb); 3182 3183 static void unaccount_event_cpu(struct perf_event *event, int cpu) 3184 { 3185 if (event->parent) 3186 return; 3187 3188 if (has_branch_stack(event)) { 3189 if (!(event->attach_state & PERF_ATTACH_TASK)) 3190 atomic_dec(&per_cpu(perf_branch_stack_events, cpu)); 3191 } 3192 if (is_cgroup_event(event)) 3193 atomic_dec(&per_cpu(perf_cgroup_events, cpu)); 3194 } 3195 3196 static void unaccount_event(struct perf_event *event) 3197 { 3198 if (event->parent) 3199 return; 3200 3201 if (event->attach_state & PERF_ATTACH_TASK) 3202 static_key_slow_dec_deferred(&perf_sched_events); 3203 if (event->attr.mmap || event->attr.mmap_data) 3204 atomic_dec(&nr_mmap_events); 3205 if (event->attr.comm) 3206 atomic_dec(&nr_comm_events); 3207 if (event->attr.task) 3208 atomic_dec(&nr_task_events); 3209 if (event->attr.freq) 3210 atomic_dec(&nr_freq_events); 3211 if (is_cgroup_event(event)) 3212 static_key_slow_dec_deferred(&perf_sched_events); 3213 if (has_branch_stack(event)) 3214 static_key_slow_dec_deferred(&perf_sched_events); 3215 3216 unaccount_event_cpu(event, event->cpu); 3217 } 3218 3219 static void __free_event(struct perf_event *event) 3220 { 3221 if (!event->parent) { 3222 if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) 3223 put_callchain_buffers(); 3224 } 3225 3226 if (event->destroy) 3227 event->destroy(event); 3228 3229 if (event->ctx) 3230 put_ctx(event->ctx); 3231 3232 call_rcu(&event->rcu_head, free_event_rcu); 3233 } 3234 static void free_event(struct perf_event *event) 3235 { 3236 irq_work_sync(&event->pending); 3237 3238 unaccount_event(event); 3239 3240 if (event->rb) { 3241 struct ring_buffer *rb; 3242 3243 /* 3244 * Can happen when we close an event with re-directed output. 3245 * 3246 * Since we have a 0 refcount, perf_mmap_close() will skip 3247 * over us; possibly making our ring_buffer_put() the last. 3248 */ 3249 mutex_lock(&event->mmap_mutex); 3250 rb = event->rb; 3251 if (rb) { 3252 rcu_assign_pointer(event->rb, NULL); 3253 ring_buffer_detach(event, rb); 3254 ring_buffer_put(rb); /* could be last */ 3255 } 3256 mutex_unlock(&event->mmap_mutex); 3257 } 3258 3259 if (is_cgroup_event(event)) 3260 perf_detach_cgroup(event); 3261 3262 3263 __free_event(event); 3264 } 3265 3266 int perf_event_release_kernel(struct perf_event *event) 3267 { 3268 struct perf_event_context *ctx = event->ctx; 3269 3270 WARN_ON_ONCE(ctx->parent_ctx); 3271 /* 3272 * There are two ways this annotation is useful: 3273 * 3274 * 1) there is a lock recursion from perf_event_exit_task 3275 * see the comment there. 3276 * 3277 * 2) there is a lock-inversion with mmap_sem through 3278 * perf_event_read_group(), which takes faults while 3279 * holding ctx->mutex, however this is called after 3280 * the last filedesc died, so there is no possibility 3281 * to trigger the AB-BA case. 3282 */ 3283 mutex_lock_nested(&ctx->mutex, SINGLE_DEPTH_NESTING); 3284 raw_spin_lock_irq(&ctx->lock); 3285 perf_group_detach(event); 3286 raw_spin_unlock_irq(&ctx->lock); 3287 perf_remove_from_context(event); 3288 mutex_unlock(&ctx->mutex); 3289 3290 free_event(event); 3291 3292 return 0; 3293 } 3294 EXPORT_SYMBOL_GPL(perf_event_release_kernel); 3295 3296 /* 3297 * Called when the last reference to the file is gone. 3298 */ 3299 static void put_event(struct perf_event *event) 3300 { 3301 struct task_struct *owner; 3302 3303 if (!atomic_long_dec_and_test(&event->refcount)) 3304 return; 3305 3306 rcu_read_lock(); 3307 owner = ACCESS_ONCE(event->owner); 3308 /* 3309 * Matches the smp_wmb() in perf_event_exit_task(). If we observe 3310 * !owner it means the list deletion is complete and we can indeed 3311 * free this event, otherwise we need to serialize on 3312 * owner->perf_event_mutex. 3313 */ 3314 smp_read_barrier_depends(); 3315 if (owner) { 3316 /* 3317 * Since delayed_put_task_struct() also drops the last 3318 * task reference we can safely take a new reference 3319 * while holding the rcu_read_lock(). 3320 */ 3321 get_task_struct(owner); 3322 } 3323 rcu_read_unlock(); 3324 3325 if (owner) { 3326 mutex_lock(&owner->perf_event_mutex); 3327 /* 3328 * We have to re-check the event->owner field, if it is cleared 3329 * we raced with perf_event_exit_task(), acquiring the mutex 3330 * ensured they're done, and we can proceed with freeing the 3331 * event. 3332 */ 3333 if (event->owner) 3334 list_del_init(&event->owner_entry); 3335 mutex_unlock(&owner->perf_event_mutex); 3336 put_task_struct(owner); 3337 } 3338 3339 perf_event_release_kernel(event); 3340 } 3341 3342 static int perf_release(struct inode *inode, struct file *file) 3343 { 3344 put_event(file->private_data); 3345 return 0; 3346 } 3347 3348 u64 perf_event_read_value(struct perf_event *event, u64 *enabled, u64 *running) 3349 { 3350 struct perf_event *child; 3351 u64 total = 0; 3352 3353 *enabled = 0; 3354 *running = 0; 3355 3356 mutex_lock(&event->child_mutex); 3357 total += perf_event_read(event); 3358 *enabled += event->total_time_enabled + 3359 atomic64_read(&event->child_total_time_enabled); 3360 *running += event->total_time_running + 3361 atomic64_read(&event->child_total_time_running); 3362 3363 list_for_each_entry(child, &event->child_list, child_list) { 3364 total += perf_event_read(child); 3365 *enabled += child->total_time_enabled; 3366 *running += child->total_time_running; 3367 } 3368 mutex_unlock(&event->child_mutex); 3369 3370 return total; 3371 } 3372 EXPORT_SYMBOL_GPL(perf_event_read_value); 3373 3374 static int perf_event_read_group(struct perf_event *event, 3375 u64 read_format, char __user *buf) 3376 { 3377 struct perf_event *leader = event->group_leader, *sub; 3378 int n = 0, size = 0, ret = -EFAULT; 3379 struct perf_event_context *ctx = leader->ctx; 3380 u64 values[5]; 3381 u64 count, enabled, running; 3382 3383 mutex_lock(&ctx->mutex); 3384 count = perf_event_read_value(leader, &enabled, &running); 3385 3386 values[n++] = 1 + leader->nr_siblings; 3387 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) 3388 values[n++] = enabled; 3389 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) 3390 values[n++] = running; 3391 values[n++] = count; 3392 if (read_format & PERF_FORMAT_ID) 3393 values[n++] = primary_event_id(leader); 3394 3395 size = n * sizeof(u64); 3396 3397 if (copy_to_user(buf, values, size)) 3398 goto unlock; 3399 3400 ret = size; 3401 3402 list_for_each_entry(sub, &leader->sibling_list, group_entry) { 3403 n = 0; 3404 3405 values[n++] = perf_event_read_value(sub, &enabled, &running); 3406 if (read_format & PERF_FORMAT_ID) 3407 values[n++] = primary_event_id(sub); 3408 3409 size = n * sizeof(u64); 3410 3411 if (copy_to_user(buf + ret, values, size)) { 3412 ret = -EFAULT; 3413 goto unlock; 3414 } 3415 3416 ret += size; 3417 } 3418 unlock: 3419 mutex_unlock(&ctx->mutex); 3420 3421 return ret; 3422 } 3423 3424 static int perf_event_read_one(struct perf_event *event, 3425 u64 read_format, char __user *buf) 3426 { 3427 u64 enabled, running; 3428 u64 values[4]; 3429 int n = 0; 3430 3431 values[n++] = perf_event_read_value(event, &enabled, &running); 3432 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) 3433 values[n++] = enabled; 3434 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) 3435 values[n++] = running; 3436 if (read_format & PERF_FORMAT_ID) 3437 values[n++] = primary_event_id(event); 3438 3439 if (copy_to_user(buf, values, n * sizeof(u64))) 3440 return -EFAULT; 3441 3442 return n * sizeof(u64); 3443 } 3444 3445 /* 3446 * Read the performance event - simple non blocking version for now 3447 */ 3448 static ssize_t 3449 perf_read_hw(struct perf_event *event, char __user *buf, size_t count) 3450 { 3451 u64 read_format = event->attr.read_format; 3452 int ret; 3453 3454 /* 3455 * Return end-of-file for a read on a event that is in 3456 * error state (i.e. because it was pinned but it couldn't be 3457 * scheduled on to the CPU at some point). 3458 */ 3459 if (event->state == PERF_EVENT_STATE_ERROR) 3460 return 0; 3461 3462 if (count < event->read_size) 3463 return -ENOSPC; 3464 3465 WARN_ON_ONCE(event->ctx->parent_ctx); 3466 if (read_format & PERF_FORMAT_GROUP) 3467 ret = perf_event_read_group(event, read_format, buf); 3468 else 3469 ret = perf_event_read_one(event, read_format, buf); 3470 3471 return ret; 3472 } 3473 3474 static ssize_t 3475 perf_read(struct file *file, char __user *buf, size_t count, loff_t *ppos) 3476 { 3477 struct perf_event *event = file->private_data; 3478 3479 return perf_read_hw(event, buf, count); 3480 } 3481 3482 static unsigned int perf_poll(struct file *file, poll_table *wait) 3483 { 3484 struct perf_event *event = file->private_data; 3485 struct ring_buffer *rb; 3486 unsigned int events = POLL_HUP; 3487 3488 /* 3489 * Pin the event->rb by taking event->mmap_mutex; otherwise 3490 * perf_event_set_output() can swizzle our rb and make us miss wakeups. 3491 */ 3492 mutex_lock(&event->mmap_mutex); 3493 rb = event->rb; 3494 if (rb) 3495 events = atomic_xchg(&rb->poll, 0); 3496 mutex_unlock(&event->mmap_mutex); 3497 3498 poll_wait(file, &event->waitq, wait); 3499 3500 return events; 3501 } 3502 3503 static void perf_event_reset(struct perf_event *event) 3504 { 3505 (void)perf_event_read(event); 3506 local64_set(&event->count, 0); 3507 perf_event_update_userpage(event); 3508 } 3509 3510 /* 3511 * Holding the top-level event's child_mutex means that any 3512 * descendant process that has inherited this event will block 3513 * in sync_child_event if it goes to exit, thus satisfying the 3514 * task existence requirements of perf_event_enable/disable. 3515 */ 3516 static void perf_event_for_each_child(struct perf_event *event, 3517 void (*func)(struct perf_event *)) 3518 { 3519 struct perf_event *child; 3520 3521 WARN_ON_ONCE(event->ctx->parent_ctx); 3522 mutex_lock(&event->child_mutex); 3523 func(event); 3524 list_for_each_entry(child, &event->child_list, child_list) 3525 func(child); 3526 mutex_unlock(&event->child_mutex); 3527 } 3528 3529 static void perf_event_for_each(struct perf_event *event, 3530 void (*func)(struct perf_event *)) 3531 { 3532 struct perf_event_context *ctx = event->ctx; 3533 struct perf_event *sibling; 3534 3535 WARN_ON_ONCE(ctx->parent_ctx); 3536 mutex_lock(&ctx->mutex); 3537 event = event->group_leader; 3538 3539 perf_event_for_each_child(event, func); 3540 list_for_each_entry(sibling, &event->sibling_list, group_entry) 3541 perf_event_for_each_child(sibling, func); 3542 mutex_unlock(&ctx->mutex); 3543 } 3544 3545 static int perf_event_period(struct perf_event *event, u64 __user *arg) 3546 { 3547 struct perf_event_context *ctx = event->ctx; 3548 int ret = 0, active; 3549 u64 value; 3550 3551 if (!is_sampling_event(event)) 3552 return -EINVAL; 3553 3554 if (copy_from_user(&value, arg, sizeof(value))) 3555 return -EFAULT; 3556 3557 if (!value) 3558 return -EINVAL; 3559 3560 raw_spin_lock_irq(&ctx->lock); 3561 if (event->attr.freq) { 3562 if (value > sysctl_perf_event_sample_rate) { 3563 ret = -EINVAL; 3564 goto unlock; 3565 } 3566 3567 event->attr.sample_freq = value; 3568 } else { 3569 event->attr.sample_period = value; 3570 event->hw.sample_period = value; 3571 } 3572 3573 active = (event->state == PERF_EVENT_STATE_ACTIVE); 3574 if (active) { 3575 perf_pmu_disable(ctx->pmu); 3576 event->pmu->stop(event, PERF_EF_UPDATE); 3577 } 3578 3579 local64_set(&event->hw.period_left, 0); 3580 3581 if (active) { 3582 event->pmu->start(event, PERF_EF_RELOAD); 3583 perf_pmu_enable(ctx->pmu); 3584 } 3585 3586 unlock: 3587 raw_spin_unlock_irq(&ctx->lock); 3588 3589 return ret; 3590 } 3591 3592 static const struct file_operations perf_fops; 3593 3594 static inline int perf_fget_light(int fd, struct fd *p) 3595 { 3596 struct fd f = fdget(fd); 3597 if (!f.file) 3598 return -EBADF; 3599 3600 if (f.file->f_op != &perf_fops) { 3601 fdput(f); 3602 return -EBADF; 3603 } 3604 *p = f; 3605 return 0; 3606 } 3607 3608 static int perf_event_set_output(struct perf_event *event, 3609 struct perf_event *output_event); 3610 static int perf_event_set_filter(struct perf_event *event, void __user *arg); 3611 3612 static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg) 3613 { 3614 struct perf_event *event = file->private_data; 3615 void (*func)(struct perf_event *); 3616 u32 flags = arg; 3617 3618 switch (cmd) { 3619 case PERF_EVENT_IOC_ENABLE: 3620 func = perf_event_enable; 3621 break; 3622 case PERF_EVENT_IOC_DISABLE: 3623 func = perf_event_disable; 3624 break; 3625 case PERF_EVENT_IOC_RESET: 3626 func = perf_event_reset; 3627 break; 3628 3629 case PERF_EVENT_IOC_REFRESH: 3630 return perf_event_refresh(event, arg); 3631 3632 case PERF_EVENT_IOC_PERIOD: 3633 return perf_event_period(event, (u64 __user *)arg); 3634 3635 case PERF_EVENT_IOC_ID: 3636 { 3637 u64 id = primary_event_id(event); 3638 3639 if (copy_to_user((void __user *)arg, &id, sizeof(id))) 3640 return -EFAULT; 3641 return 0; 3642 } 3643 3644 case PERF_EVENT_IOC_SET_OUTPUT: 3645 { 3646 int ret; 3647 if (arg != -1) { 3648 struct perf_event *output_event; 3649 struct fd output; 3650 ret = perf_fget_light(arg, &output); 3651 if (ret) 3652 return ret; 3653 output_event = output.file->private_data; 3654 ret = perf_event_set_output(event, output_event); 3655 fdput(output); 3656 } else { 3657 ret = perf_event_set_output(event, NULL); 3658 } 3659 return ret; 3660 } 3661 3662 case PERF_EVENT_IOC_SET_FILTER: 3663 return perf_event_set_filter(event, (void __user *)arg); 3664 3665 default: 3666 return -ENOTTY; 3667 } 3668 3669 if (flags & PERF_IOC_FLAG_GROUP) 3670 perf_event_for_each(event, func); 3671 else 3672 perf_event_for_each_child(event, func); 3673 3674 return 0; 3675 } 3676 3677 int perf_event_task_enable(void) 3678 { 3679 struct perf_event *event; 3680 3681 mutex_lock(¤t->perf_event_mutex); 3682 list_for_each_entry(event, ¤t->perf_event_list, owner_entry) 3683 perf_event_for_each_child(event, perf_event_enable); 3684 mutex_unlock(¤t->perf_event_mutex); 3685 3686 return 0; 3687 } 3688 3689 int perf_event_task_disable(void) 3690 { 3691 struct perf_event *event; 3692 3693 mutex_lock(¤t->perf_event_mutex); 3694 list_for_each_entry(event, ¤t->perf_event_list, owner_entry) 3695 perf_event_for_each_child(event, perf_event_disable); 3696 mutex_unlock(¤t->perf_event_mutex); 3697 3698 return 0; 3699 } 3700 3701 static int perf_event_index(struct perf_event *event) 3702 { 3703 if (event->hw.state & PERF_HES_STOPPED) 3704 return 0; 3705 3706 if (event->state != PERF_EVENT_STATE_ACTIVE) 3707 return 0; 3708 3709 return event->pmu->event_idx(event); 3710 } 3711 3712 static void calc_timer_values(struct perf_event *event, 3713 u64 *now, 3714 u64 *enabled, 3715 u64 *running) 3716 { 3717 u64 ctx_time; 3718 3719 *now = perf_clock(); 3720 ctx_time = event->shadow_ctx_time + *now; 3721 *enabled = ctx_time - event->tstamp_enabled; 3722 *running = ctx_time - event->tstamp_running; 3723 } 3724 3725 static void perf_event_init_userpage(struct perf_event *event) 3726 { 3727 struct perf_event_mmap_page *userpg; 3728 struct ring_buffer *rb; 3729 3730 rcu_read_lock(); 3731 rb = rcu_dereference(event->rb); 3732 if (!rb) 3733 goto unlock; 3734 3735 userpg = rb->user_page; 3736 3737 /* Allow new userspace to detect that bit 0 is deprecated */ 3738 userpg->cap_bit0_is_deprecated = 1; 3739 userpg->size = offsetof(struct perf_event_mmap_page, __reserved); 3740 3741 unlock: 3742 rcu_read_unlock(); 3743 } 3744 3745 void __weak arch_perf_update_userpage(struct perf_event_mmap_page *userpg, u64 now) 3746 { 3747 } 3748 3749 /* 3750 * Callers need to ensure there can be no nesting of this function, otherwise 3751 * the seqlock logic goes bad. We can not serialize this because the arch 3752 * code calls this from NMI context. 3753 */ 3754 void perf_event_update_userpage(struct perf_event *event) 3755 { 3756 struct perf_event_mmap_page *userpg; 3757 struct ring_buffer *rb; 3758 u64 enabled, running, now; 3759 3760 rcu_read_lock(); 3761 rb = rcu_dereference(event->rb); 3762 if (!rb) 3763 goto unlock; 3764 3765 /* 3766 * compute total_time_enabled, total_time_running 3767 * based on snapshot values taken when the event 3768 * was last scheduled in. 3769 * 3770 * we cannot simply called update_context_time() 3771 * because of locking issue as we can be called in 3772 * NMI context 3773 */ 3774 calc_timer_values(event, &now, &enabled, &running); 3775 3776 userpg = rb->user_page; 3777 /* 3778 * Disable preemption so as to not let the corresponding user-space 3779 * spin too long if we get preempted. 3780 */ 3781 preempt_disable(); 3782 ++userpg->lock; 3783 barrier(); 3784 userpg->index = perf_event_index(event); 3785 userpg->offset = perf_event_count(event); 3786 if (userpg->index) 3787 userpg->offset -= local64_read(&event->hw.prev_count); 3788 3789 userpg->time_enabled = enabled + 3790 atomic64_read(&event->child_total_time_enabled); 3791 3792 userpg->time_running = running + 3793 atomic64_read(&event->child_total_time_running); 3794 3795 arch_perf_update_userpage(userpg, now); 3796 3797 barrier(); 3798 ++userpg->lock; 3799 preempt_enable(); 3800 unlock: 3801 rcu_read_unlock(); 3802 } 3803 3804 static int perf_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf) 3805 { 3806 struct perf_event *event = vma->vm_file->private_data; 3807 struct ring_buffer *rb; 3808 int ret = VM_FAULT_SIGBUS; 3809 3810 if (vmf->flags & FAULT_FLAG_MKWRITE) { 3811 if (vmf->pgoff == 0) 3812 ret = 0; 3813 return ret; 3814 } 3815 3816 rcu_read_lock(); 3817 rb = rcu_dereference(event->rb); 3818 if (!rb) 3819 goto unlock; 3820 3821 if (vmf->pgoff && (vmf->flags & FAULT_FLAG_WRITE)) 3822 goto unlock; 3823 3824 vmf->page = perf_mmap_to_page(rb, vmf->pgoff); 3825 if (!vmf->page) 3826 goto unlock; 3827 3828 get_page(vmf->page); 3829 vmf->page->mapping = vma->vm_file->f_mapping; 3830 vmf->page->index = vmf->pgoff; 3831 3832 ret = 0; 3833 unlock: 3834 rcu_read_unlock(); 3835 3836 return ret; 3837 } 3838 3839 static void ring_buffer_attach(struct perf_event *event, 3840 struct ring_buffer *rb) 3841 { 3842 unsigned long flags; 3843 3844 if (!list_empty(&event->rb_entry)) 3845 return; 3846 3847 spin_lock_irqsave(&rb->event_lock, flags); 3848 if (list_empty(&event->rb_entry)) 3849 list_add(&event->rb_entry, &rb->event_list); 3850 spin_unlock_irqrestore(&rb->event_lock, flags); 3851 } 3852 3853 static void ring_buffer_detach(struct perf_event *event, struct ring_buffer *rb) 3854 { 3855 unsigned long flags; 3856 3857 if (list_empty(&event->rb_entry)) 3858 return; 3859 3860 spin_lock_irqsave(&rb->event_lock, flags); 3861 list_del_init(&event->rb_entry); 3862 wake_up_all(&event->waitq); 3863 spin_unlock_irqrestore(&rb->event_lock, flags); 3864 } 3865 3866 static void ring_buffer_wakeup(struct perf_event *event) 3867 { 3868 struct ring_buffer *rb; 3869 3870 rcu_read_lock(); 3871 rb = rcu_dereference(event->rb); 3872 if (rb) { 3873 list_for_each_entry_rcu(event, &rb->event_list, rb_entry) 3874 wake_up_all(&event->waitq); 3875 } 3876 rcu_read_unlock(); 3877 } 3878 3879 static void rb_free_rcu(struct rcu_head *rcu_head) 3880 { 3881 struct ring_buffer *rb; 3882 3883 rb = container_of(rcu_head, struct ring_buffer, rcu_head); 3884 rb_free(rb); 3885 } 3886 3887 static struct ring_buffer *ring_buffer_get(struct perf_event *event) 3888 { 3889 struct ring_buffer *rb; 3890 3891 rcu_read_lock(); 3892 rb = rcu_dereference(event->rb); 3893 if (rb) { 3894 if (!atomic_inc_not_zero(&rb->refcount)) 3895 rb = NULL; 3896 } 3897 rcu_read_unlock(); 3898 3899 return rb; 3900 } 3901 3902 static void ring_buffer_put(struct ring_buffer *rb) 3903 { 3904 if (!atomic_dec_and_test(&rb->refcount)) 3905 return; 3906 3907 WARN_ON_ONCE(!list_empty(&rb->event_list)); 3908 3909 call_rcu(&rb->rcu_head, rb_free_rcu); 3910 } 3911 3912 static void perf_mmap_open(struct vm_area_struct *vma) 3913 { 3914 struct perf_event *event = vma->vm_file->private_data; 3915 3916 atomic_inc(&event->mmap_count); 3917 atomic_inc(&event->rb->mmap_count); 3918 } 3919 3920 /* 3921 * A buffer can be mmap()ed multiple times; either directly through the same 3922 * event, or through other events by use of perf_event_set_output(). 3923 * 3924 * In order to undo the VM accounting done by perf_mmap() we need to destroy 3925 * the buffer here, where we still have a VM context. This means we need 3926 * to detach all events redirecting to us. 3927 */ 3928 static void perf_mmap_close(struct vm_area_struct *vma) 3929 { 3930 struct perf_event *event = vma->vm_file->private_data; 3931 3932 struct ring_buffer *rb = event->rb; 3933 struct user_struct *mmap_user = rb->mmap_user; 3934 int mmap_locked = rb->mmap_locked; 3935 unsigned long size = perf_data_size(rb); 3936 3937 atomic_dec(&rb->mmap_count); 3938 3939 if (!atomic_dec_and_mutex_lock(&event->mmap_count, &event->mmap_mutex)) 3940 return; 3941 3942 /* Detach current event from the buffer. */ 3943 rcu_assign_pointer(event->rb, NULL); 3944 ring_buffer_detach(event, rb); 3945 mutex_unlock(&event->mmap_mutex); 3946 3947 /* If there's still other mmap()s of this buffer, we're done. */ 3948 if (atomic_read(&rb->mmap_count)) { 3949 ring_buffer_put(rb); /* can't be last */ 3950 return; 3951 } 3952 3953 /* 3954 * No other mmap()s, detach from all other events that might redirect 3955 * into the now unreachable buffer. Somewhat complicated by the 3956 * fact that rb::event_lock otherwise nests inside mmap_mutex. 3957 */ 3958 again: 3959 rcu_read_lock(); 3960 list_for_each_entry_rcu(event, &rb->event_list, rb_entry) { 3961 if (!atomic_long_inc_not_zero(&event->refcount)) { 3962 /* 3963 * This event is en-route to free_event() which will 3964 * detach it and remove it from the list. 3965 */ 3966 continue; 3967 } 3968 rcu_read_unlock(); 3969 3970 mutex_lock(&event->mmap_mutex); 3971 /* 3972 * Check we didn't race with perf_event_set_output() which can 3973 * swizzle the rb from under us while we were waiting to 3974 * acquire mmap_mutex. 3975 * 3976 * If we find a different rb; ignore this event, a next 3977 * iteration will no longer find it on the list. We have to 3978 * still restart the iteration to make sure we're not now 3979 * iterating the wrong list. 3980 */ 3981 if (event->rb == rb) { 3982 rcu_assign_pointer(event->rb, NULL); 3983 ring_buffer_detach(event, rb); 3984 ring_buffer_put(rb); /* can't be last, we still have one */ 3985 } 3986 mutex_unlock(&event->mmap_mutex); 3987 put_event(event); 3988 3989 /* 3990 * Restart the iteration; either we're on the wrong list or 3991 * destroyed its integrity by doing a deletion. 3992 */ 3993 goto again; 3994 } 3995 rcu_read_unlock(); 3996 3997 /* 3998 * It could be there's still a few 0-ref events on the list; they'll 3999 * get cleaned up by free_event() -- they'll also still have their 4000 * ref on the rb and will free it whenever they are done with it. 4001 * 4002 * Aside from that, this buffer is 'fully' detached and unmapped, 4003 * undo the VM accounting. 4004 */ 4005 4006 atomic_long_sub((size >> PAGE_SHIFT) + 1, &mmap_user->locked_vm); 4007 vma->vm_mm->pinned_vm -= mmap_locked; 4008 free_uid(mmap_user); 4009 4010 ring_buffer_put(rb); /* could be last */ 4011 } 4012 4013 static const struct vm_operations_struct perf_mmap_vmops = { 4014 .open = perf_mmap_open, 4015 .close = perf_mmap_close, 4016 .fault = perf_mmap_fault, 4017 .page_mkwrite = perf_mmap_fault, 4018 }; 4019 4020 static int perf_mmap(struct file *file, struct vm_area_struct *vma) 4021 { 4022 struct perf_event *event = file->private_data; 4023 unsigned long user_locked, user_lock_limit; 4024 struct user_struct *user = current_user(); 4025 unsigned long locked, lock_limit; 4026 struct ring_buffer *rb; 4027 unsigned long vma_size; 4028 unsigned long nr_pages; 4029 long user_extra, extra; 4030 int ret = 0, flags = 0; 4031 4032 /* 4033 * Don't allow mmap() of inherited per-task counters. This would 4034 * create a performance issue due to all children writing to the 4035 * same rb. 4036 */ 4037 if (event->cpu == -1 && event->attr.inherit) 4038 return -EINVAL; 4039 4040 if (!(vma->vm_flags & VM_SHARED)) 4041 return -EINVAL; 4042 4043 vma_size = vma->vm_end - vma->vm_start; 4044 nr_pages = (vma_size / PAGE_SIZE) - 1; 4045 4046 /* 4047 * If we have rb pages ensure they're a power-of-two number, so we 4048 * can do bitmasks instead of modulo. 4049 */ 4050 if (nr_pages != 0 && !is_power_of_2(nr_pages)) 4051 return -EINVAL; 4052 4053 if (vma_size != PAGE_SIZE * (1 + nr_pages)) 4054 return -EINVAL; 4055 4056 if (vma->vm_pgoff != 0) 4057 return -EINVAL; 4058 4059 WARN_ON_ONCE(event->ctx->parent_ctx); 4060 again: 4061 mutex_lock(&event->mmap_mutex); 4062 if (event->rb) { 4063 if (event->rb->nr_pages != nr_pages) { 4064 ret = -EINVAL; 4065 goto unlock; 4066 } 4067 4068 if (!atomic_inc_not_zero(&event->rb->mmap_count)) { 4069 /* 4070 * Raced against perf_mmap_close() through 4071 * perf_event_set_output(). Try again, hope for better 4072 * luck. 4073 */ 4074 mutex_unlock(&event->mmap_mutex); 4075 goto again; 4076 } 4077 4078 goto unlock; 4079 } 4080 4081 user_extra = nr_pages + 1; 4082 user_lock_limit = sysctl_perf_event_mlock >> (PAGE_SHIFT - 10); 4083 4084 /* 4085 * Increase the limit linearly with more CPUs: 4086 */ 4087 user_lock_limit *= num_online_cpus(); 4088 4089 user_locked = atomic_long_read(&user->locked_vm) + user_extra; 4090 4091 extra = 0; 4092 if (user_locked > user_lock_limit) 4093 extra = user_locked - user_lock_limit; 4094 4095 lock_limit = rlimit(RLIMIT_MEMLOCK); 4096 lock_limit >>= PAGE_SHIFT; 4097 locked = vma->vm_mm->pinned_vm + extra; 4098 4099 if ((locked > lock_limit) && perf_paranoid_tracepoint_raw() && 4100 !capable(CAP_IPC_LOCK)) { 4101 ret = -EPERM; 4102 goto unlock; 4103 } 4104 4105 WARN_ON(event->rb); 4106 4107 if (vma->vm_flags & VM_WRITE) 4108 flags |= RING_BUFFER_WRITABLE; 4109 4110 rb = rb_alloc(nr_pages, 4111 event->attr.watermark ? event->attr.wakeup_watermark : 0, 4112 event->cpu, flags); 4113 4114 if (!rb) { 4115 ret = -ENOMEM; 4116 goto unlock; 4117 } 4118 4119 atomic_set(&rb->mmap_count, 1); 4120 rb->mmap_locked = extra; 4121 rb->mmap_user = get_current_user(); 4122 4123 atomic_long_add(user_extra, &user->locked_vm); 4124 vma->vm_mm->pinned_vm += extra; 4125 4126 ring_buffer_attach(event, rb); 4127 rcu_assign_pointer(event->rb, rb); 4128 4129 perf_event_init_userpage(event); 4130 perf_event_update_userpage(event); 4131 4132 unlock: 4133 if (!ret) 4134 atomic_inc(&event->mmap_count); 4135 mutex_unlock(&event->mmap_mutex); 4136 4137 /* 4138 * Since pinned accounting is per vm we cannot allow fork() to copy our 4139 * vma. 4140 */ 4141 vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_DONTDUMP; 4142 vma->vm_ops = &perf_mmap_vmops; 4143 4144 return ret; 4145 } 4146 4147 static int perf_fasync(int fd, struct file *filp, int on) 4148 { 4149 struct inode *inode = file_inode(filp); 4150 struct perf_event *event = filp->private_data; 4151 int retval; 4152 4153 mutex_lock(&inode->i_mutex); 4154 retval = fasync_helper(fd, filp, on, &event->fasync); 4155 mutex_unlock(&inode->i_mutex); 4156 4157 if (retval < 0) 4158 return retval; 4159 4160 return 0; 4161 } 4162 4163 static const struct file_operations perf_fops = { 4164 .llseek = no_llseek, 4165 .release = perf_release, 4166 .read = perf_read, 4167 .poll = perf_poll, 4168 .unlocked_ioctl = perf_ioctl, 4169 .compat_ioctl = perf_ioctl, 4170 .mmap = perf_mmap, 4171 .fasync = perf_fasync, 4172 }; 4173 4174 /* 4175 * Perf event wakeup 4176 * 4177 * If there's data, ensure we set the poll() state and publish everything 4178 * to user-space before waking everybody up. 4179 */ 4180 4181 void perf_event_wakeup(struct perf_event *event) 4182 { 4183 ring_buffer_wakeup(event); 4184 4185 if (event->pending_kill) { 4186 kill_fasync(&event->fasync, SIGIO, event->pending_kill); 4187 event->pending_kill = 0; 4188 } 4189 } 4190 4191 static void perf_pending_event(struct irq_work *entry) 4192 { 4193 struct perf_event *event = container_of(entry, 4194 struct perf_event, pending); 4195 4196 if (event->pending_disable) { 4197 event->pending_disable = 0; 4198 __perf_event_disable(event); 4199 } 4200 4201 if (event->pending_wakeup) { 4202 event->pending_wakeup = 0; 4203 perf_event_wakeup(event); 4204 } 4205 } 4206 4207 /* 4208 * We assume there is only KVM supporting the callbacks. 4209 * Later on, we might change it to a list if there is 4210 * another virtualization implementation supporting the callbacks. 4211 */ 4212 struct perf_guest_info_callbacks *perf_guest_cbs; 4213 4214 int perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *cbs) 4215 { 4216 perf_guest_cbs = cbs; 4217 return 0; 4218 } 4219 EXPORT_SYMBOL_GPL(perf_register_guest_info_callbacks); 4220 4221 int perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *cbs) 4222 { 4223 perf_guest_cbs = NULL; 4224 return 0; 4225 } 4226 EXPORT_SYMBOL_GPL(perf_unregister_guest_info_callbacks); 4227 4228 static void 4229 perf_output_sample_regs(struct perf_output_handle *handle, 4230 struct pt_regs *regs, u64 mask) 4231 { 4232 int bit; 4233 4234 for_each_set_bit(bit, (const unsigned long *) &mask, 4235 sizeof(mask) * BITS_PER_BYTE) { 4236 u64 val; 4237 4238 val = perf_reg_value(regs, bit); 4239 perf_output_put(handle, val); 4240 } 4241 } 4242 4243 static void perf_sample_regs_user(struct perf_regs_user *regs_user, 4244 struct pt_regs *regs) 4245 { 4246 if (!user_mode(regs)) { 4247 if (current->mm) 4248 regs = task_pt_regs(current); 4249 else 4250 regs = NULL; 4251 } 4252 4253 if (regs) { 4254 regs_user->regs = regs; 4255 regs_user->abi = perf_reg_abi(current); 4256 } 4257 } 4258 4259 /* 4260 * Get remaining task size from user stack pointer. 4261 * 4262 * It'd be better to take stack vma map and limit this more 4263 * precisly, but there's no way to get it safely under interrupt, 4264 * so using TASK_SIZE as limit. 4265 */ 4266 static u64 perf_ustack_task_size(struct pt_regs *regs) 4267 { 4268 unsigned long addr = perf_user_stack_pointer(regs); 4269 4270 if (!addr || addr >= TASK_SIZE) 4271 return 0; 4272 4273 return TASK_SIZE - addr; 4274 } 4275 4276 static u16 4277 perf_sample_ustack_size(u16 stack_size, u16 header_size, 4278 struct pt_regs *regs) 4279 { 4280 u64 task_size; 4281 4282 /* No regs, no stack pointer, no dump. */ 4283 if (!regs) 4284 return 0; 4285 4286 /* 4287 * Check if we fit in with the requested stack size into the: 4288 * - TASK_SIZE 4289 * If we don't, we limit the size to the TASK_SIZE. 4290 * 4291 * - remaining sample size 4292 * If we don't, we customize the stack size to 4293 * fit in to the remaining sample size. 4294 */ 4295 4296 task_size = min((u64) USHRT_MAX, perf_ustack_task_size(regs)); 4297 stack_size = min(stack_size, (u16) task_size); 4298 4299 /* Current header size plus static size and dynamic size. */ 4300 header_size += 2 * sizeof(u64); 4301 4302 /* Do we fit in with the current stack dump size? */ 4303 if ((u16) (header_size + stack_size) < header_size) { 4304 /* 4305 * If we overflow the maximum size for the sample, 4306 * we customize the stack dump size to fit in. 4307 */ 4308 stack_size = USHRT_MAX - header_size - sizeof(u64); 4309 stack_size = round_up(stack_size, sizeof(u64)); 4310 } 4311 4312 return stack_size; 4313 } 4314 4315 static void 4316 perf_output_sample_ustack(struct perf_output_handle *handle, u64 dump_size, 4317 struct pt_regs *regs) 4318 { 4319 /* Case of a kernel thread, nothing to dump */ 4320 if (!regs) { 4321 u64 size = 0; 4322 perf_output_put(handle, size); 4323 } else { 4324 unsigned long sp; 4325 unsigned int rem; 4326 u64 dyn_size; 4327 4328 /* 4329 * We dump: 4330 * static size 4331 * - the size requested by user or the best one we can fit 4332 * in to the sample max size 4333 * data 4334 * - user stack dump data 4335 * dynamic size 4336 * - the actual dumped size 4337 */ 4338 4339 /* Static size. */ 4340 perf_output_put(handle, dump_size); 4341 4342 /* Data. */ 4343 sp = perf_user_stack_pointer(regs); 4344 rem = __output_copy_user(handle, (void *) sp, dump_size); 4345 dyn_size = dump_size - rem; 4346 4347 perf_output_skip(handle, rem); 4348 4349 /* Dynamic size. */ 4350 perf_output_put(handle, dyn_size); 4351 } 4352 } 4353 4354 static void __perf_event_header__init_id(struct perf_event_header *header, 4355 struct perf_sample_data *data, 4356 struct perf_event *event) 4357 { 4358 u64 sample_type = event->attr.sample_type; 4359 4360 data->type = sample_type; 4361 header->size += event->id_header_size; 4362 4363 if (sample_type & PERF_SAMPLE_TID) { 4364 /* namespace issues */ 4365 data->tid_entry.pid = perf_event_pid(event, current); 4366 data->tid_entry.tid = perf_event_tid(event, current); 4367 } 4368 4369 if (sample_type & PERF_SAMPLE_TIME) 4370 data->time = perf_clock(); 4371 4372 if (sample_type & (PERF_SAMPLE_ID | PERF_SAMPLE_IDENTIFIER)) 4373 data->id = primary_event_id(event); 4374 4375 if (sample_type & PERF_SAMPLE_STREAM_ID) 4376 data->stream_id = event->id; 4377 4378 if (sample_type & PERF_SAMPLE_CPU) { 4379 data->cpu_entry.cpu = raw_smp_processor_id(); 4380 data->cpu_entry.reserved = 0; 4381 } 4382 } 4383 4384 void perf_event_header__init_id(struct perf_event_header *header, 4385 struct perf_sample_data *data, 4386 struct perf_event *event) 4387 { 4388 if (event->attr.sample_id_all) 4389 __perf_event_header__init_id(header, data, event); 4390 } 4391 4392 static void __perf_event__output_id_sample(struct perf_output_handle *handle, 4393 struct perf_sample_data *data) 4394 { 4395 u64 sample_type = data->type; 4396 4397 if (sample_type & PERF_SAMPLE_TID) 4398 perf_output_put(handle, data->tid_entry); 4399 4400 if (sample_type & PERF_SAMPLE_TIME) 4401 perf_output_put(handle, data->time); 4402 4403 if (sample_type & PERF_SAMPLE_ID) 4404 perf_output_put(handle, data->id); 4405 4406 if (sample_type & PERF_SAMPLE_STREAM_ID) 4407 perf_output_put(handle, data->stream_id); 4408 4409 if (sample_type & PERF_SAMPLE_CPU) 4410 perf_output_put(handle, data->cpu_entry); 4411 4412 if (sample_type & PERF_SAMPLE_IDENTIFIER) 4413 perf_output_put(handle, data->id); 4414 } 4415 4416 void perf_event__output_id_sample(struct perf_event *event, 4417 struct perf_output_handle *handle, 4418 struct perf_sample_data *sample) 4419 { 4420 if (event->attr.sample_id_all) 4421 __perf_event__output_id_sample(handle, sample); 4422 } 4423 4424 static void perf_output_read_one(struct perf_output_handle *handle, 4425 struct perf_event *event, 4426 u64 enabled, u64 running) 4427 { 4428 u64 read_format = event->attr.read_format; 4429 u64 values[4]; 4430 int n = 0; 4431 4432 values[n++] = perf_event_count(event); 4433 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) { 4434 values[n++] = enabled + 4435 atomic64_read(&event->child_total_time_enabled); 4436 } 4437 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) { 4438 values[n++] = running + 4439 atomic64_read(&event->child_total_time_running); 4440 } 4441 if (read_format & PERF_FORMAT_ID) 4442 values[n++] = primary_event_id(event); 4443 4444 __output_copy(handle, values, n * sizeof(u64)); 4445 } 4446 4447 /* 4448 * XXX PERF_FORMAT_GROUP vs inherited events seems difficult. 4449 */ 4450 static void perf_output_read_group(struct perf_output_handle *handle, 4451 struct perf_event *event, 4452 u64 enabled, u64 running) 4453 { 4454 struct perf_event *leader = event->group_leader, *sub; 4455 u64 read_format = event->attr.read_format; 4456 u64 values[5]; 4457 int n = 0; 4458 4459 values[n++] = 1 + leader->nr_siblings; 4460 4461 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) 4462 values[n++] = enabled; 4463 4464 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) 4465 values[n++] = running; 4466 4467 if (leader != event) 4468 leader->pmu->read(leader); 4469 4470 values[n++] = perf_event_count(leader); 4471 if (read_format & PERF_FORMAT_ID) 4472 values[n++] = primary_event_id(leader); 4473 4474 __output_copy(handle, values, n * sizeof(u64)); 4475 4476 list_for_each_entry(sub, &leader->sibling_list, group_entry) { 4477 n = 0; 4478 4479 if ((sub != event) && 4480 (sub->state == PERF_EVENT_STATE_ACTIVE)) 4481 sub->pmu->read(sub); 4482 4483 values[n++] = perf_event_count(sub); 4484 if (read_format & PERF_FORMAT_ID) 4485 values[n++] = primary_event_id(sub); 4486 4487 __output_copy(handle, values, n * sizeof(u64)); 4488 } 4489 } 4490 4491 #define PERF_FORMAT_TOTAL_TIMES (PERF_FORMAT_TOTAL_TIME_ENABLED|\ 4492 PERF_FORMAT_TOTAL_TIME_RUNNING) 4493 4494 static void perf_output_read(struct perf_output_handle *handle, 4495 struct perf_event *event) 4496 { 4497 u64 enabled = 0, running = 0, now; 4498 u64 read_format = event->attr.read_format; 4499 4500 /* 4501 * compute total_time_enabled, total_time_running 4502 * based on snapshot values taken when the event 4503 * was last scheduled in. 4504 * 4505 * we cannot simply called update_context_time() 4506 * because of locking issue as we are called in 4507 * NMI context 4508 */ 4509 if (read_format & PERF_FORMAT_TOTAL_TIMES) 4510 calc_timer_values(event, &now, &enabled, &running); 4511 4512 if (event->attr.read_format & PERF_FORMAT_GROUP) 4513 perf_output_read_group(handle, event, enabled, running); 4514 else 4515 perf_output_read_one(handle, event, enabled, running); 4516 } 4517 4518 void perf_output_sample(struct perf_output_handle *handle, 4519 struct perf_event_header *header, 4520 struct perf_sample_data *data, 4521 struct perf_event *event) 4522 { 4523 u64 sample_type = data->type; 4524 4525 perf_output_put(handle, *header); 4526 4527 if (sample_type & PERF_SAMPLE_IDENTIFIER) 4528 perf_output_put(handle, data->id); 4529 4530 if (sample_type & PERF_SAMPLE_IP) 4531 perf_output_put(handle, data->ip); 4532 4533 if (sample_type & PERF_SAMPLE_TID) 4534 perf_output_put(handle, data->tid_entry); 4535 4536 if (sample_type & PERF_SAMPLE_TIME) 4537 perf_output_put(handle, data->time); 4538 4539 if (sample_type & PERF_SAMPLE_ADDR) 4540 perf_output_put(handle, data->addr); 4541 4542 if (sample_type & PERF_SAMPLE_ID) 4543 perf_output_put(handle, data->id); 4544 4545 if (sample_type & PERF_SAMPLE_STREAM_ID) 4546 perf_output_put(handle, data->stream_id); 4547 4548 if (sample_type & PERF_SAMPLE_CPU) 4549 perf_output_put(handle, data->cpu_entry); 4550 4551 if (sample_type & PERF_SAMPLE_PERIOD) 4552 perf_output_put(handle, data->period); 4553 4554 if (sample_type & PERF_SAMPLE_READ) 4555 perf_output_read(handle, event); 4556 4557 if (sample_type & PERF_SAMPLE_CALLCHAIN) { 4558 if (data->callchain) { 4559 int size = 1; 4560 4561 if (data->callchain) 4562 size += data->callchain->nr; 4563 4564 size *= sizeof(u64); 4565 4566 __output_copy(handle, data->callchain, size); 4567 } else { 4568 u64 nr = 0; 4569 perf_output_put(handle, nr); 4570 } 4571 } 4572 4573 if (sample_type & PERF_SAMPLE_RAW) { 4574 if (data->raw) { 4575 perf_output_put(handle, data->raw->size); 4576 __output_copy(handle, data->raw->data, 4577 data->raw->size); 4578 } else { 4579 struct { 4580 u32 size; 4581 u32 data; 4582 } raw = { 4583 .size = sizeof(u32), 4584 .data = 0, 4585 }; 4586 perf_output_put(handle, raw); 4587 } 4588 } 4589 4590 if (sample_type & PERF_SAMPLE_BRANCH_STACK) { 4591 if (data->br_stack) { 4592 size_t size; 4593 4594 size = data->br_stack->nr 4595 * sizeof(struct perf_branch_entry); 4596 4597 perf_output_put(handle, data->br_stack->nr); 4598 perf_output_copy(handle, data->br_stack->entries, size); 4599 } else { 4600 /* 4601 * we always store at least the value of nr 4602 */ 4603 u64 nr = 0; 4604 perf_output_put(handle, nr); 4605 } 4606 } 4607 4608 if (sample_type & PERF_SAMPLE_REGS_USER) { 4609 u64 abi = data->regs_user.abi; 4610 4611 /* 4612 * If there are no regs to dump, notice it through 4613 * first u64 being zero (PERF_SAMPLE_REGS_ABI_NONE). 4614 */ 4615 perf_output_put(handle, abi); 4616 4617 if (abi) { 4618 u64 mask = event->attr.sample_regs_user; 4619 perf_output_sample_regs(handle, 4620 data->regs_user.regs, 4621 mask); 4622 } 4623 } 4624 4625 if (sample_type & PERF_SAMPLE_STACK_USER) { 4626 perf_output_sample_ustack(handle, 4627 data->stack_user_size, 4628 data->regs_user.regs); 4629 } 4630 4631 if (sample_type & PERF_SAMPLE_WEIGHT) 4632 perf_output_put(handle, data->weight); 4633 4634 if (sample_type & PERF_SAMPLE_DATA_SRC) 4635 perf_output_put(handle, data->data_src.val); 4636 4637 if (sample_type & PERF_SAMPLE_TRANSACTION) 4638 perf_output_put(handle, data->txn); 4639 4640 if (!event->attr.watermark) { 4641 int wakeup_events = event->attr.wakeup_events; 4642 4643 if (wakeup_events) { 4644 struct ring_buffer *rb = handle->rb; 4645 int events = local_inc_return(&rb->events); 4646 4647 if (events >= wakeup_events) { 4648 local_sub(wakeup_events, &rb->events); 4649 local_inc(&rb->wakeup); 4650 } 4651 } 4652 } 4653 } 4654 4655 void perf_prepare_sample(struct perf_event_header *header, 4656 struct perf_sample_data *data, 4657 struct perf_event *event, 4658 struct pt_regs *regs) 4659 { 4660 u64 sample_type = event->attr.sample_type; 4661 4662 header->type = PERF_RECORD_SAMPLE; 4663 header->size = sizeof(*header) + event->header_size; 4664 4665 header->misc = 0; 4666 header->misc |= perf_misc_flags(regs); 4667 4668 __perf_event_header__init_id(header, data, event); 4669 4670 if (sample_type & PERF_SAMPLE_IP) 4671 data->ip = perf_instruction_pointer(regs); 4672 4673 if (sample_type & PERF_SAMPLE_CALLCHAIN) { 4674 int size = 1; 4675 4676 data->callchain = perf_callchain(event, regs); 4677 4678 if (data->callchain) 4679 size += data->callchain->nr; 4680 4681 header->size += size * sizeof(u64); 4682 } 4683 4684 if (sample_type & PERF_SAMPLE_RAW) { 4685 int size = sizeof(u32); 4686 4687 if (data->raw) 4688 size += data->raw->size; 4689 else 4690 size += sizeof(u32); 4691 4692 WARN_ON_ONCE(size & (sizeof(u64)-1)); 4693 header->size += size; 4694 } 4695 4696 if (sample_type & PERF_SAMPLE_BRANCH_STACK) { 4697 int size = sizeof(u64); /* nr */ 4698 if (data->br_stack) { 4699 size += data->br_stack->nr 4700 * sizeof(struct perf_branch_entry); 4701 } 4702 header->size += size; 4703 } 4704 4705 if (sample_type & PERF_SAMPLE_REGS_USER) { 4706 /* regs dump ABI info */ 4707 int size = sizeof(u64); 4708 4709 perf_sample_regs_user(&data->regs_user, regs); 4710 4711 if (data->regs_user.regs) { 4712 u64 mask = event->attr.sample_regs_user; 4713 size += hweight64(mask) * sizeof(u64); 4714 } 4715 4716 header->size += size; 4717 } 4718 4719 if (sample_type & PERF_SAMPLE_STACK_USER) { 4720 /* 4721 * Either we need PERF_SAMPLE_STACK_USER bit to be allways 4722 * processed as the last one or have additional check added 4723 * in case new sample type is added, because we could eat 4724 * up the rest of the sample size. 4725 */ 4726 struct perf_regs_user *uregs = &data->regs_user; 4727 u16 stack_size = event->attr.sample_stack_user; 4728 u16 size = sizeof(u64); 4729 4730 if (!uregs->abi) 4731 perf_sample_regs_user(uregs, regs); 4732 4733 stack_size = perf_sample_ustack_size(stack_size, header->size, 4734 uregs->regs); 4735 4736 /* 4737 * If there is something to dump, add space for the dump 4738 * itself and for the field that tells the dynamic size, 4739 * which is how many have been actually dumped. 4740 */ 4741 if (stack_size) 4742 size += sizeof(u64) + stack_size; 4743 4744 data->stack_user_size = stack_size; 4745 header->size += size; 4746 } 4747 } 4748 4749 static void perf_event_output(struct perf_event *event, 4750 struct perf_sample_data *data, 4751 struct pt_regs *regs) 4752 { 4753 struct perf_output_handle handle; 4754 struct perf_event_header header; 4755 4756 /* protect the callchain buffers */ 4757 rcu_read_lock(); 4758 4759 perf_prepare_sample(&header, data, event, regs); 4760 4761 if (perf_output_begin(&handle, event, header.size)) 4762 goto exit; 4763 4764 perf_output_sample(&handle, &header, data, event); 4765 4766 perf_output_end(&handle); 4767 4768 exit: 4769 rcu_read_unlock(); 4770 } 4771 4772 /* 4773 * read event_id 4774 */ 4775 4776 struct perf_read_event { 4777 struct perf_event_header header; 4778 4779 u32 pid; 4780 u32 tid; 4781 }; 4782 4783 static void 4784 perf_event_read_event(struct perf_event *event, 4785 struct task_struct *task) 4786 { 4787 struct perf_output_handle handle; 4788 struct perf_sample_data sample; 4789 struct perf_read_event read_event = { 4790 .header = { 4791 .type = PERF_RECORD_READ, 4792 .misc = 0, 4793 .size = sizeof(read_event) + event->read_size, 4794 }, 4795 .pid = perf_event_pid(event, task), 4796 .tid = perf_event_tid(event, task), 4797 }; 4798 int ret; 4799 4800 perf_event_header__init_id(&read_event.header, &sample, event); 4801 ret = perf_output_begin(&handle, event, read_event.header.size); 4802 if (ret) 4803 return; 4804 4805 perf_output_put(&handle, read_event); 4806 perf_output_read(&handle, event); 4807 perf_event__output_id_sample(event, &handle, &sample); 4808 4809 perf_output_end(&handle); 4810 } 4811 4812 typedef void (perf_event_aux_output_cb)(struct perf_event *event, void *data); 4813 4814 static void 4815 perf_event_aux_ctx(struct perf_event_context *ctx, 4816 perf_event_aux_output_cb output, 4817 void *data) 4818 { 4819 struct perf_event *event; 4820 4821 list_for_each_entry_rcu(event, &ctx->event_list, event_entry) { 4822 if (event->state < PERF_EVENT_STATE_INACTIVE) 4823 continue; 4824 if (!event_filter_match(event)) 4825 continue; 4826 output(event, data); 4827 } 4828 } 4829 4830 static void 4831 perf_event_aux(perf_event_aux_output_cb output, void *data, 4832 struct perf_event_context *task_ctx) 4833 { 4834 struct perf_cpu_context *cpuctx; 4835 struct perf_event_context *ctx; 4836 struct pmu *pmu; 4837 int ctxn; 4838 4839 rcu_read_lock(); 4840 list_for_each_entry_rcu(pmu, &pmus, entry) { 4841 cpuctx = get_cpu_ptr(pmu->pmu_cpu_context); 4842 if (cpuctx->unique_pmu != pmu) 4843 goto next; 4844 perf_event_aux_ctx(&cpuctx->ctx, output, data); 4845 if (task_ctx) 4846 goto next; 4847 ctxn = pmu->task_ctx_nr; 4848 if (ctxn < 0) 4849 goto next; 4850 ctx = rcu_dereference(current->perf_event_ctxp[ctxn]); 4851 if (ctx) 4852 perf_event_aux_ctx(ctx, output, data); 4853 next: 4854 put_cpu_ptr(pmu->pmu_cpu_context); 4855 } 4856 4857 if (task_ctx) { 4858 preempt_disable(); 4859 perf_event_aux_ctx(task_ctx, output, data); 4860 preempt_enable(); 4861 } 4862 rcu_read_unlock(); 4863 } 4864 4865 /* 4866 * task tracking -- fork/exit 4867 * 4868 * enabled by: attr.comm | attr.mmap | attr.mmap2 | attr.mmap_data | attr.task 4869 */ 4870 4871 struct perf_task_event { 4872 struct task_struct *task; 4873 struct perf_event_context *task_ctx; 4874 4875 struct { 4876 struct perf_event_header header; 4877 4878 u32 pid; 4879 u32 ppid; 4880 u32 tid; 4881 u32 ptid; 4882 u64 time; 4883 } event_id; 4884 }; 4885 4886 static int perf_event_task_match(struct perf_event *event) 4887 { 4888 return event->attr.comm || event->attr.mmap || 4889 event->attr.mmap2 || event->attr.mmap_data || 4890 event->attr.task; 4891 } 4892 4893 static void perf_event_task_output(struct perf_event *event, 4894 void *data) 4895 { 4896 struct perf_task_event *task_event = data; 4897 struct perf_output_handle handle; 4898 struct perf_sample_data sample; 4899 struct task_struct *task = task_event->task; 4900 int ret, size = task_event->event_id.header.size; 4901 4902 if (!perf_event_task_match(event)) 4903 return; 4904 4905 perf_event_header__init_id(&task_event->event_id.header, &sample, event); 4906 4907 ret = perf_output_begin(&handle, event, 4908 task_event->event_id.header.size); 4909 if (ret) 4910 goto out; 4911 4912 task_event->event_id.pid = perf_event_pid(event, task); 4913 task_event->event_id.ppid = perf_event_pid(event, current); 4914 4915 task_event->event_id.tid = perf_event_tid(event, task); 4916 task_event->event_id.ptid = perf_event_tid(event, current); 4917 4918 perf_output_put(&handle, task_event->event_id); 4919 4920 perf_event__output_id_sample(event, &handle, &sample); 4921 4922 perf_output_end(&handle); 4923 out: 4924 task_event->event_id.header.size = size; 4925 } 4926 4927 static void perf_event_task(struct task_struct *task, 4928 struct perf_event_context *task_ctx, 4929 int new) 4930 { 4931 struct perf_task_event task_event; 4932 4933 if (!atomic_read(&nr_comm_events) && 4934 !atomic_read(&nr_mmap_events) && 4935 !atomic_read(&nr_task_events)) 4936 return; 4937 4938 task_event = (struct perf_task_event){ 4939 .task = task, 4940 .task_ctx = task_ctx, 4941 .event_id = { 4942 .header = { 4943 .type = new ? PERF_RECORD_FORK : PERF_RECORD_EXIT, 4944 .misc = 0, 4945 .size = sizeof(task_event.event_id), 4946 }, 4947 /* .pid */ 4948 /* .ppid */ 4949 /* .tid */ 4950 /* .ptid */ 4951 .time = perf_clock(), 4952 }, 4953 }; 4954 4955 perf_event_aux(perf_event_task_output, 4956 &task_event, 4957 task_ctx); 4958 } 4959 4960 void perf_event_fork(struct task_struct *task) 4961 { 4962 perf_event_task(task, NULL, 1); 4963 } 4964 4965 /* 4966 * comm tracking 4967 */ 4968 4969 struct perf_comm_event { 4970 struct task_struct *task; 4971 char *comm; 4972 int comm_size; 4973 4974 struct { 4975 struct perf_event_header header; 4976 4977 u32 pid; 4978 u32 tid; 4979 } event_id; 4980 }; 4981 4982 static int perf_event_comm_match(struct perf_event *event) 4983 { 4984 return event->attr.comm; 4985 } 4986 4987 static void perf_event_comm_output(struct perf_event *event, 4988 void *data) 4989 { 4990 struct perf_comm_event *comm_event = data; 4991 struct perf_output_handle handle; 4992 struct perf_sample_data sample; 4993 int size = comm_event->event_id.header.size; 4994 int ret; 4995 4996 if (!perf_event_comm_match(event)) 4997 return; 4998 4999 perf_event_header__init_id(&comm_event->event_id.header, &sample, event); 5000 ret = perf_output_begin(&handle, event, 5001 comm_event->event_id.header.size); 5002 5003 if (ret) 5004 goto out; 5005 5006 comm_event->event_id.pid = perf_event_pid(event, comm_event->task); 5007 comm_event->event_id.tid = perf_event_tid(event, comm_event->task); 5008 5009 perf_output_put(&handle, comm_event->event_id); 5010 __output_copy(&handle, comm_event->comm, 5011 comm_event->comm_size); 5012 5013 perf_event__output_id_sample(event, &handle, &sample); 5014 5015 perf_output_end(&handle); 5016 out: 5017 comm_event->event_id.header.size = size; 5018 } 5019 5020 static void perf_event_comm_event(struct perf_comm_event *comm_event) 5021 { 5022 char comm[TASK_COMM_LEN]; 5023 unsigned int size; 5024 5025 memset(comm, 0, sizeof(comm)); 5026 strlcpy(comm, comm_event->task->comm, sizeof(comm)); 5027 size = ALIGN(strlen(comm)+1, sizeof(u64)); 5028 5029 comm_event->comm = comm; 5030 comm_event->comm_size = size; 5031 5032 comm_event->event_id.header.size = sizeof(comm_event->event_id) + size; 5033 5034 perf_event_aux(perf_event_comm_output, 5035 comm_event, 5036 NULL); 5037 } 5038 5039 void perf_event_comm(struct task_struct *task) 5040 { 5041 struct perf_comm_event comm_event; 5042 struct perf_event_context *ctx; 5043 int ctxn; 5044 5045 rcu_read_lock(); 5046 for_each_task_context_nr(ctxn) { 5047 ctx = task->perf_event_ctxp[ctxn]; 5048 if (!ctx) 5049 continue; 5050 5051 perf_event_enable_on_exec(ctx); 5052 } 5053 rcu_read_unlock(); 5054 5055 if (!atomic_read(&nr_comm_events)) 5056 return; 5057 5058 comm_event = (struct perf_comm_event){ 5059 .task = task, 5060 /* .comm */ 5061 /* .comm_size */ 5062 .event_id = { 5063 .header = { 5064 .type = PERF_RECORD_COMM, 5065 .misc = 0, 5066 /* .size */ 5067 }, 5068 /* .pid */ 5069 /* .tid */ 5070 }, 5071 }; 5072 5073 perf_event_comm_event(&comm_event); 5074 } 5075 5076 /* 5077 * mmap tracking 5078 */ 5079 5080 struct perf_mmap_event { 5081 struct vm_area_struct *vma; 5082 5083 const char *file_name; 5084 int file_size; 5085 int maj, min; 5086 u64 ino; 5087 u64 ino_generation; 5088 5089 struct { 5090 struct perf_event_header header; 5091 5092 u32 pid; 5093 u32 tid; 5094 u64 start; 5095 u64 len; 5096 u64 pgoff; 5097 } event_id; 5098 }; 5099 5100 static int perf_event_mmap_match(struct perf_event *event, 5101 void *data) 5102 { 5103 struct perf_mmap_event *mmap_event = data; 5104 struct vm_area_struct *vma = mmap_event->vma; 5105 int executable = vma->vm_flags & VM_EXEC; 5106 5107 return (!executable && event->attr.mmap_data) || 5108 (executable && (event->attr.mmap || event->attr.mmap2)); 5109 } 5110 5111 static void perf_event_mmap_output(struct perf_event *event, 5112 void *data) 5113 { 5114 struct perf_mmap_event *mmap_event = data; 5115 struct perf_output_handle handle; 5116 struct perf_sample_data sample; 5117 int size = mmap_event->event_id.header.size; 5118 int ret; 5119 5120 if (!perf_event_mmap_match(event, data)) 5121 return; 5122 5123 if (event->attr.mmap2) { 5124 mmap_event->event_id.header.type = PERF_RECORD_MMAP2; 5125 mmap_event->event_id.header.size += sizeof(mmap_event->maj); 5126 mmap_event->event_id.header.size += sizeof(mmap_event->min); 5127 mmap_event->event_id.header.size += sizeof(mmap_event->ino); 5128 mmap_event->event_id.header.size += sizeof(mmap_event->ino_generation); 5129 } 5130 5131 perf_event_header__init_id(&mmap_event->event_id.header, &sample, event); 5132 ret = perf_output_begin(&handle, event, 5133 mmap_event->event_id.header.size); 5134 if (ret) 5135 goto out; 5136 5137 mmap_event->event_id.pid = perf_event_pid(event, current); 5138 mmap_event->event_id.tid = perf_event_tid(event, current); 5139 5140 perf_output_put(&handle, mmap_event->event_id); 5141 5142 if (event->attr.mmap2) { 5143 perf_output_put(&handle, mmap_event->maj); 5144 perf_output_put(&handle, mmap_event->min); 5145 perf_output_put(&handle, mmap_event->ino); 5146 perf_output_put(&handle, mmap_event->ino_generation); 5147 } 5148 5149 __output_copy(&handle, mmap_event->file_name, 5150 mmap_event->file_size); 5151 5152 perf_event__output_id_sample(event, &handle, &sample); 5153 5154 perf_output_end(&handle); 5155 out: 5156 mmap_event->event_id.header.size = size; 5157 } 5158 5159 static void perf_event_mmap_event(struct perf_mmap_event *mmap_event) 5160 { 5161 struct vm_area_struct *vma = mmap_event->vma; 5162 struct file *file = vma->vm_file; 5163 int maj = 0, min = 0; 5164 u64 ino = 0, gen = 0; 5165 unsigned int size; 5166 char tmp[16]; 5167 char *buf = NULL; 5168 char *name; 5169 5170 if (file) { 5171 struct inode *inode; 5172 dev_t dev; 5173 5174 buf = kmalloc(PATH_MAX, GFP_KERNEL); 5175 if (!buf) { 5176 name = "//enomem"; 5177 goto cpy_name; 5178 } 5179 /* 5180 * d_path() works from the end of the rb backwards, so we 5181 * need to add enough zero bytes after the string to handle 5182 * the 64bit alignment we do later. 5183 */ 5184 name = d_path(&file->f_path, buf, PATH_MAX - sizeof(u64)); 5185 if (IS_ERR(name)) { 5186 name = "//toolong"; 5187 goto cpy_name; 5188 } 5189 inode = file_inode(vma->vm_file); 5190 dev = inode->i_sb->s_dev; 5191 ino = inode->i_ino; 5192 gen = inode->i_generation; 5193 maj = MAJOR(dev); 5194 min = MINOR(dev); 5195 goto got_name; 5196 } else { 5197 name = (char *)arch_vma_name(vma); 5198 if (name) 5199 goto cpy_name; 5200 5201 if (vma->vm_start <= vma->vm_mm->start_brk && 5202 vma->vm_end >= vma->vm_mm->brk) { 5203 name = "[heap]"; 5204 goto cpy_name; 5205 } 5206 if (vma->vm_start <= vma->vm_mm->start_stack && 5207 vma->vm_end >= vma->vm_mm->start_stack) { 5208 name = "[stack]"; 5209 goto cpy_name; 5210 } 5211 5212 name = "//anon"; 5213 goto cpy_name; 5214 } 5215 5216 cpy_name: 5217 strlcpy(tmp, name, sizeof(tmp)); 5218 name = tmp; 5219 got_name: 5220 /* 5221 * Since our buffer works in 8 byte units we need to align our string 5222 * size to a multiple of 8. However, we must guarantee the tail end is 5223 * zero'd out to avoid leaking random bits to userspace. 5224 */ 5225 size = strlen(name)+1; 5226 while (!IS_ALIGNED(size, sizeof(u64))) 5227 name[size++] = '\0'; 5228 5229 mmap_event->file_name = name; 5230 mmap_event->file_size = size; 5231 mmap_event->maj = maj; 5232 mmap_event->min = min; 5233 mmap_event->ino = ino; 5234 mmap_event->ino_generation = gen; 5235 5236 if (!(vma->vm_flags & VM_EXEC)) 5237 mmap_event->event_id.header.misc |= PERF_RECORD_MISC_MMAP_DATA; 5238 5239 mmap_event->event_id.header.size = sizeof(mmap_event->event_id) + size; 5240 5241 perf_event_aux(perf_event_mmap_output, 5242 mmap_event, 5243 NULL); 5244 5245 kfree(buf); 5246 } 5247 5248 void perf_event_mmap(struct vm_area_struct *vma) 5249 { 5250 struct perf_mmap_event mmap_event; 5251 5252 if (!atomic_read(&nr_mmap_events)) 5253 return; 5254 5255 mmap_event = (struct perf_mmap_event){ 5256 .vma = vma, 5257 /* .file_name */ 5258 /* .file_size */ 5259 .event_id = { 5260 .header = { 5261 .type = PERF_RECORD_MMAP, 5262 .misc = PERF_RECORD_MISC_USER, 5263 /* .size */ 5264 }, 5265 /* .pid */ 5266 /* .tid */ 5267 .start = vma->vm_start, 5268 .len = vma->vm_end - vma->vm_start, 5269 .pgoff = (u64)vma->vm_pgoff << PAGE_SHIFT, 5270 }, 5271 /* .maj (attr_mmap2 only) */ 5272 /* .min (attr_mmap2 only) */ 5273 /* .ino (attr_mmap2 only) */ 5274 /* .ino_generation (attr_mmap2 only) */ 5275 }; 5276 5277 perf_event_mmap_event(&mmap_event); 5278 } 5279 5280 /* 5281 * IRQ throttle logging 5282 */ 5283 5284 static void perf_log_throttle(struct perf_event *event, int enable) 5285 { 5286 struct perf_output_handle handle; 5287 struct perf_sample_data sample; 5288 int ret; 5289 5290 struct { 5291 struct perf_event_header header; 5292 u64 time; 5293 u64 id; 5294 u64 stream_id; 5295 } throttle_event = { 5296 .header = { 5297 .type = PERF_RECORD_THROTTLE, 5298 .misc = 0, 5299 .size = sizeof(throttle_event), 5300 }, 5301 .time = perf_clock(), 5302 .id = primary_event_id(event), 5303 .stream_id = event->id, 5304 }; 5305 5306 if (enable) 5307 throttle_event.header.type = PERF_RECORD_UNTHROTTLE; 5308 5309 perf_event_header__init_id(&throttle_event.header, &sample, event); 5310 5311 ret = perf_output_begin(&handle, event, 5312 throttle_event.header.size); 5313 if (ret) 5314 return; 5315 5316 perf_output_put(&handle, throttle_event); 5317 perf_event__output_id_sample(event, &handle, &sample); 5318 perf_output_end(&handle); 5319 } 5320 5321 /* 5322 * Generic event overflow handling, sampling. 5323 */ 5324 5325 static int __perf_event_overflow(struct perf_event *event, 5326 int throttle, struct perf_sample_data *data, 5327 struct pt_regs *regs) 5328 { 5329 int events = atomic_read(&event->event_limit); 5330 struct hw_perf_event *hwc = &event->hw; 5331 u64 seq; 5332 int ret = 0; 5333 5334 /* 5335 * Non-sampling counters might still use the PMI to fold short 5336 * hardware counters, ignore those. 5337 */ 5338 if (unlikely(!is_sampling_event(event))) 5339 return 0; 5340 5341 seq = __this_cpu_read(perf_throttled_seq); 5342 if (seq != hwc->interrupts_seq) { 5343 hwc->interrupts_seq = seq; 5344 hwc->interrupts = 1; 5345 } else { 5346 hwc->interrupts++; 5347 if (unlikely(throttle 5348 && hwc->interrupts >= max_samples_per_tick)) { 5349 __this_cpu_inc(perf_throttled_count); 5350 hwc->interrupts = MAX_INTERRUPTS; 5351 perf_log_throttle(event, 0); 5352 tick_nohz_full_kick(); 5353 ret = 1; 5354 } 5355 } 5356 5357 if (event->attr.freq) { 5358 u64 now = perf_clock(); 5359 s64 delta = now - hwc->freq_time_stamp; 5360 5361 hwc->freq_time_stamp = now; 5362 5363 if (delta > 0 && delta < 2*TICK_NSEC) 5364 perf_adjust_period(event, delta, hwc->last_period, true); 5365 } 5366 5367 /* 5368 * XXX event_limit might not quite work as expected on inherited 5369 * events 5370 */ 5371 5372 event->pending_kill = POLL_IN; 5373 if (events && atomic_dec_and_test(&event->event_limit)) { 5374 ret = 1; 5375 event->pending_kill = POLL_HUP; 5376 event->pending_disable = 1; 5377 irq_work_queue(&event->pending); 5378 } 5379 5380 if (event->overflow_handler) 5381 event->overflow_handler(event, data, regs); 5382 else 5383 perf_event_output(event, data, regs); 5384 5385 if (event->fasync && event->pending_kill) { 5386 event->pending_wakeup = 1; 5387 irq_work_queue(&event->pending); 5388 } 5389 5390 return ret; 5391 } 5392 5393 int perf_event_overflow(struct perf_event *event, 5394 struct perf_sample_data *data, 5395 struct pt_regs *regs) 5396 { 5397 return __perf_event_overflow(event, 1, data, regs); 5398 } 5399 5400 /* 5401 * Generic software event infrastructure 5402 */ 5403 5404 struct swevent_htable { 5405 struct swevent_hlist *swevent_hlist; 5406 struct mutex hlist_mutex; 5407 int hlist_refcount; 5408 5409 /* Recursion avoidance in each contexts */ 5410 int recursion[PERF_NR_CONTEXTS]; 5411 }; 5412 5413 static DEFINE_PER_CPU(struct swevent_htable, swevent_htable); 5414 5415 /* 5416 * We directly increment event->count and keep a second value in 5417 * event->hw.period_left to count intervals. This period event 5418 * is kept in the range [-sample_period, 0] so that we can use the 5419 * sign as trigger. 5420 */ 5421 5422 u64 perf_swevent_set_period(struct perf_event *event) 5423 { 5424 struct hw_perf_event *hwc = &event->hw; 5425 u64 period = hwc->last_period; 5426 u64 nr, offset; 5427 s64 old, val; 5428 5429 hwc->last_period = hwc->sample_period; 5430 5431 again: 5432 old = val = local64_read(&hwc->period_left); 5433 if (val < 0) 5434 return 0; 5435 5436 nr = div64_u64(period + val, period); 5437 offset = nr * period; 5438 val -= offset; 5439 if (local64_cmpxchg(&hwc->period_left, old, val) != old) 5440 goto again; 5441 5442 return nr; 5443 } 5444 5445 static void perf_swevent_overflow(struct perf_event *event, u64 overflow, 5446 struct perf_sample_data *data, 5447 struct pt_regs *regs) 5448 { 5449 struct hw_perf_event *hwc = &event->hw; 5450 int throttle = 0; 5451 5452 if (!overflow) 5453 overflow = perf_swevent_set_period(event); 5454 5455 if (hwc->interrupts == MAX_INTERRUPTS) 5456 return; 5457 5458 for (; overflow; overflow--) { 5459 if (__perf_event_overflow(event, throttle, 5460 data, regs)) { 5461 /* 5462 * We inhibit the overflow from happening when 5463 * hwc->interrupts == MAX_INTERRUPTS. 5464 */ 5465 break; 5466 } 5467 throttle = 1; 5468 } 5469 } 5470 5471 static void perf_swevent_event(struct perf_event *event, u64 nr, 5472 struct perf_sample_data *data, 5473 struct pt_regs *regs) 5474 { 5475 struct hw_perf_event *hwc = &event->hw; 5476 5477 local64_add(nr, &event->count); 5478 5479 if (!regs) 5480 return; 5481 5482 if (!is_sampling_event(event)) 5483 return; 5484 5485 if ((event->attr.sample_type & PERF_SAMPLE_PERIOD) && !event->attr.freq) { 5486 data->period = nr; 5487 return perf_swevent_overflow(event, 1, data, regs); 5488 } else 5489 data->period = event->hw.last_period; 5490 5491 if (nr == 1 && hwc->sample_period == 1 && !event->attr.freq) 5492 return perf_swevent_overflow(event, 1, data, regs); 5493 5494 if (local64_add_negative(nr, &hwc->period_left)) 5495 return; 5496 5497 perf_swevent_overflow(event, 0, data, regs); 5498 } 5499 5500 static int perf_exclude_event(struct perf_event *event, 5501 struct pt_regs *regs) 5502 { 5503 if (event->hw.state & PERF_HES_STOPPED) 5504 return 1; 5505 5506 if (regs) { 5507 if (event->attr.exclude_user && user_mode(regs)) 5508 return 1; 5509 5510 if (event->attr.exclude_kernel && !user_mode(regs)) 5511 return 1; 5512 } 5513 5514 return 0; 5515 } 5516 5517 static int perf_swevent_match(struct perf_event *event, 5518 enum perf_type_id type, 5519 u32 event_id, 5520 struct perf_sample_data *data, 5521 struct pt_regs *regs) 5522 { 5523 if (event->attr.type != type) 5524 return 0; 5525 5526 if (event->attr.config != event_id) 5527 return 0; 5528 5529 if (perf_exclude_event(event, regs)) 5530 return 0; 5531 5532 return 1; 5533 } 5534 5535 static inline u64 swevent_hash(u64 type, u32 event_id) 5536 { 5537 u64 val = event_id | (type << 32); 5538 5539 return hash_64(val, SWEVENT_HLIST_BITS); 5540 } 5541 5542 static inline struct hlist_head * 5543 __find_swevent_head(struct swevent_hlist *hlist, u64 type, u32 event_id) 5544 { 5545 u64 hash = swevent_hash(type, event_id); 5546 5547 return &hlist->heads[hash]; 5548 } 5549 5550 /* For the read side: events when they trigger */ 5551 static inline struct hlist_head * 5552 find_swevent_head_rcu(struct swevent_htable *swhash, u64 type, u32 event_id) 5553 { 5554 struct swevent_hlist *hlist; 5555 5556 hlist = rcu_dereference(swhash->swevent_hlist); 5557 if (!hlist) 5558 return NULL; 5559 5560 return __find_swevent_head(hlist, type, event_id); 5561 } 5562 5563 /* For the event head insertion and removal in the hlist */ 5564 static inline struct hlist_head * 5565 find_swevent_head(struct swevent_htable *swhash, struct perf_event *event) 5566 { 5567 struct swevent_hlist *hlist; 5568 u32 event_id = event->attr.config; 5569 u64 type = event->attr.type; 5570 5571 /* 5572 * Event scheduling is always serialized against hlist allocation 5573 * and release. Which makes the protected version suitable here. 5574 * The context lock guarantees that. 5575 */ 5576 hlist = rcu_dereference_protected(swhash->swevent_hlist, 5577 lockdep_is_held(&event->ctx->lock)); 5578 if (!hlist) 5579 return NULL; 5580 5581 return __find_swevent_head(hlist, type, event_id); 5582 } 5583 5584 static void do_perf_sw_event(enum perf_type_id type, u32 event_id, 5585 u64 nr, 5586 struct perf_sample_data *data, 5587 struct pt_regs *regs) 5588 { 5589 struct swevent_htable *swhash = &__get_cpu_var(swevent_htable); 5590 struct perf_event *event; 5591 struct hlist_head *head; 5592 5593 rcu_read_lock(); 5594 head = find_swevent_head_rcu(swhash, type, event_id); 5595 if (!head) 5596 goto end; 5597 5598 hlist_for_each_entry_rcu(event, head, hlist_entry) { 5599 if (perf_swevent_match(event, type, event_id, data, regs)) 5600 perf_swevent_event(event, nr, data, regs); 5601 } 5602 end: 5603 rcu_read_unlock(); 5604 } 5605 5606 int perf_swevent_get_recursion_context(void) 5607 { 5608 struct swevent_htable *swhash = &__get_cpu_var(swevent_htable); 5609 5610 return get_recursion_context(swhash->recursion); 5611 } 5612 EXPORT_SYMBOL_GPL(perf_swevent_get_recursion_context); 5613 5614 inline void perf_swevent_put_recursion_context(int rctx) 5615 { 5616 struct swevent_htable *swhash = &__get_cpu_var(swevent_htable); 5617 5618 put_recursion_context(swhash->recursion, rctx); 5619 } 5620 5621 void __perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr) 5622 { 5623 struct perf_sample_data data; 5624 int rctx; 5625 5626 preempt_disable_notrace(); 5627 rctx = perf_swevent_get_recursion_context(); 5628 if (rctx < 0) 5629 return; 5630 5631 perf_sample_data_init(&data, addr, 0); 5632 5633 do_perf_sw_event(PERF_TYPE_SOFTWARE, event_id, nr, &data, regs); 5634 5635 perf_swevent_put_recursion_context(rctx); 5636 preempt_enable_notrace(); 5637 } 5638 5639 static void perf_swevent_read(struct perf_event *event) 5640 { 5641 } 5642 5643 static int perf_swevent_add(struct perf_event *event, int flags) 5644 { 5645 struct swevent_htable *swhash = &__get_cpu_var(swevent_htable); 5646 struct hw_perf_event *hwc = &event->hw; 5647 struct hlist_head *head; 5648 5649 if (is_sampling_event(event)) { 5650 hwc->last_period = hwc->sample_period; 5651 perf_swevent_set_period(event); 5652 } 5653 5654 hwc->state = !(flags & PERF_EF_START); 5655 5656 head = find_swevent_head(swhash, event); 5657 if (WARN_ON_ONCE(!head)) 5658 return -EINVAL; 5659 5660 hlist_add_head_rcu(&event->hlist_entry, head); 5661 5662 return 0; 5663 } 5664 5665 static void perf_swevent_del(struct perf_event *event, int flags) 5666 { 5667 hlist_del_rcu(&event->hlist_entry); 5668 } 5669 5670 static void perf_swevent_start(struct perf_event *event, int flags) 5671 { 5672 event->hw.state = 0; 5673 } 5674 5675 static void perf_swevent_stop(struct perf_event *event, int flags) 5676 { 5677 event->hw.state = PERF_HES_STOPPED; 5678 } 5679 5680 /* Deref the hlist from the update side */ 5681 static inline struct swevent_hlist * 5682 swevent_hlist_deref(struct swevent_htable *swhash) 5683 { 5684 return rcu_dereference_protected(swhash->swevent_hlist, 5685 lockdep_is_held(&swhash->hlist_mutex)); 5686 } 5687 5688 static void swevent_hlist_release(struct swevent_htable *swhash) 5689 { 5690 struct swevent_hlist *hlist = swevent_hlist_deref(swhash); 5691 5692 if (!hlist) 5693 return; 5694 5695 rcu_assign_pointer(swhash->swevent_hlist, NULL); 5696 kfree_rcu(hlist, rcu_head); 5697 } 5698 5699 static void swevent_hlist_put_cpu(struct perf_event *event, int cpu) 5700 { 5701 struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu); 5702 5703 mutex_lock(&swhash->hlist_mutex); 5704 5705 if (!--swhash->hlist_refcount) 5706 swevent_hlist_release(swhash); 5707 5708 mutex_unlock(&swhash->hlist_mutex); 5709 } 5710 5711 static void swevent_hlist_put(struct perf_event *event) 5712 { 5713 int cpu; 5714 5715 for_each_possible_cpu(cpu) 5716 swevent_hlist_put_cpu(event, cpu); 5717 } 5718 5719 static int swevent_hlist_get_cpu(struct perf_event *event, int cpu) 5720 { 5721 struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu); 5722 int err = 0; 5723 5724 mutex_lock(&swhash->hlist_mutex); 5725 5726 if (!swevent_hlist_deref(swhash) && cpu_online(cpu)) { 5727 struct swevent_hlist *hlist; 5728 5729 hlist = kzalloc(sizeof(*hlist), GFP_KERNEL); 5730 if (!hlist) { 5731 err = -ENOMEM; 5732 goto exit; 5733 } 5734 rcu_assign_pointer(swhash->swevent_hlist, hlist); 5735 } 5736 swhash->hlist_refcount++; 5737 exit: 5738 mutex_unlock(&swhash->hlist_mutex); 5739 5740 return err; 5741 } 5742 5743 static int swevent_hlist_get(struct perf_event *event) 5744 { 5745 int err; 5746 int cpu, failed_cpu; 5747 5748 get_online_cpus(); 5749 for_each_possible_cpu(cpu) { 5750 err = swevent_hlist_get_cpu(event, cpu); 5751 if (err) { 5752 failed_cpu = cpu; 5753 goto fail; 5754 } 5755 } 5756 put_online_cpus(); 5757 5758 return 0; 5759 fail: 5760 for_each_possible_cpu(cpu) { 5761 if (cpu == failed_cpu) 5762 break; 5763 swevent_hlist_put_cpu(event, cpu); 5764 } 5765 5766 put_online_cpus(); 5767 return err; 5768 } 5769 5770 struct static_key perf_swevent_enabled[PERF_COUNT_SW_MAX]; 5771 5772 static void sw_perf_event_destroy(struct perf_event *event) 5773 { 5774 u64 event_id = event->attr.config; 5775 5776 WARN_ON(event->parent); 5777 5778 static_key_slow_dec(&perf_swevent_enabled[event_id]); 5779 swevent_hlist_put(event); 5780 } 5781 5782 static int perf_swevent_init(struct perf_event *event) 5783 { 5784 u64 event_id = event->attr.config; 5785 5786 if (event->attr.type != PERF_TYPE_SOFTWARE) 5787 return -ENOENT; 5788 5789 /* 5790 * no branch sampling for software events 5791 */ 5792 if (has_branch_stack(event)) 5793 return -EOPNOTSUPP; 5794 5795 switch (event_id) { 5796 case PERF_COUNT_SW_CPU_CLOCK: 5797 case PERF_COUNT_SW_TASK_CLOCK: 5798 return -ENOENT; 5799 5800 default: 5801 break; 5802 } 5803 5804 if (event_id >= PERF_COUNT_SW_MAX) 5805 return -ENOENT; 5806 5807 if (!event->parent) { 5808 int err; 5809 5810 err = swevent_hlist_get(event); 5811 if (err) 5812 return err; 5813 5814 static_key_slow_inc(&perf_swevent_enabled[event_id]); 5815 event->destroy = sw_perf_event_destroy; 5816 } 5817 5818 return 0; 5819 } 5820 5821 static int perf_swevent_event_idx(struct perf_event *event) 5822 { 5823 return 0; 5824 } 5825 5826 static struct pmu perf_swevent = { 5827 .task_ctx_nr = perf_sw_context, 5828 5829 .event_init = perf_swevent_init, 5830 .add = perf_swevent_add, 5831 .del = perf_swevent_del, 5832 .start = perf_swevent_start, 5833 .stop = perf_swevent_stop, 5834 .read = perf_swevent_read, 5835 5836 .event_idx = perf_swevent_event_idx, 5837 }; 5838 5839 #ifdef CONFIG_EVENT_TRACING 5840 5841 static int perf_tp_filter_match(struct perf_event *event, 5842 struct perf_sample_data *data) 5843 { 5844 void *record = data->raw->data; 5845 5846 if (likely(!event->filter) || filter_match_preds(event->filter, record)) 5847 return 1; 5848 return 0; 5849 } 5850 5851 static int perf_tp_event_match(struct perf_event *event, 5852 struct perf_sample_data *data, 5853 struct pt_regs *regs) 5854 { 5855 if (event->hw.state & PERF_HES_STOPPED) 5856 return 0; 5857 /* 5858 * All tracepoints are from kernel-space. 5859 */ 5860 if (event->attr.exclude_kernel) 5861 return 0; 5862 5863 if (!perf_tp_filter_match(event, data)) 5864 return 0; 5865 5866 return 1; 5867 } 5868 5869 void perf_tp_event(u64 addr, u64 count, void *record, int entry_size, 5870 struct pt_regs *regs, struct hlist_head *head, int rctx, 5871 struct task_struct *task) 5872 { 5873 struct perf_sample_data data; 5874 struct perf_event *event; 5875 5876 struct perf_raw_record raw = { 5877 .size = entry_size, 5878 .data = record, 5879 }; 5880 5881 perf_sample_data_init(&data, addr, 0); 5882 data.raw = &raw; 5883 5884 hlist_for_each_entry_rcu(event, head, hlist_entry) { 5885 if (perf_tp_event_match(event, &data, regs)) 5886 perf_swevent_event(event, count, &data, regs); 5887 } 5888 5889 /* 5890 * If we got specified a target task, also iterate its context and 5891 * deliver this event there too. 5892 */ 5893 if (task && task != current) { 5894 struct perf_event_context *ctx; 5895 struct trace_entry *entry = record; 5896 5897 rcu_read_lock(); 5898 ctx = rcu_dereference(task->perf_event_ctxp[perf_sw_context]); 5899 if (!ctx) 5900 goto unlock; 5901 5902 list_for_each_entry_rcu(event, &ctx->event_list, event_entry) { 5903 if (event->attr.type != PERF_TYPE_TRACEPOINT) 5904 continue; 5905 if (event->attr.config != entry->type) 5906 continue; 5907 if (perf_tp_event_match(event, &data, regs)) 5908 perf_swevent_event(event, count, &data, regs); 5909 } 5910 unlock: 5911 rcu_read_unlock(); 5912 } 5913 5914 perf_swevent_put_recursion_context(rctx); 5915 } 5916 EXPORT_SYMBOL_GPL(perf_tp_event); 5917 5918 static void tp_perf_event_destroy(struct perf_event *event) 5919 { 5920 perf_trace_destroy(event); 5921 } 5922 5923 static int perf_tp_event_init(struct perf_event *event) 5924 { 5925 int err; 5926 5927 if (event->attr.type != PERF_TYPE_TRACEPOINT) 5928 return -ENOENT; 5929 5930 /* 5931 * no branch sampling for tracepoint events 5932 */ 5933 if (has_branch_stack(event)) 5934 return -EOPNOTSUPP; 5935 5936 err = perf_trace_init(event); 5937 if (err) 5938 return err; 5939 5940 event->destroy = tp_perf_event_destroy; 5941 5942 return 0; 5943 } 5944 5945 static struct pmu perf_tracepoint = { 5946 .task_ctx_nr = perf_sw_context, 5947 5948 .event_init = perf_tp_event_init, 5949 .add = perf_trace_add, 5950 .del = perf_trace_del, 5951 .start = perf_swevent_start, 5952 .stop = perf_swevent_stop, 5953 .read = perf_swevent_read, 5954 5955 .event_idx = perf_swevent_event_idx, 5956 }; 5957 5958 static inline void perf_tp_register(void) 5959 { 5960 perf_pmu_register(&perf_tracepoint, "tracepoint", PERF_TYPE_TRACEPOINT); 5961 } 5962 5963 static int perf_event_set_filter(struct perf_event *event, void __user *arg) 5964 { 5965 char *filter_str; 5966 int ret; 5967 5968 if (event->attr.type != PERF_TYPE_TRACEPOINT) 5969 return -EINVAL; 5970 5971 filter_str = strndup_user(arg, PAGE_SIZE); 5972 if (IS_ERR(filter_str)) 5973 return PTR_ERR(filter_str); 5974 5975 ret = ftrace_profile_set_filter(event, event->attr.config, filter_str); 5976 5977 kfree(filter_str); 5978 return ret; 5979 } 5980 5981 static void perf_event_free_filter(struct perf_event *event) 5982 { 5983 ftrace_profile_free_filter(event); 5984 } 5985 5986 #else 5987 5988 static inline void perf_tp_register(void) 5989 { 5990 } 5991 5992 static int perf_event_set_filter(struct perf_event *event, void __user *arg) 5993 { 5994 return -ENOENT; 5995 } 5996 5997 static void perf_event_free_filter(struct perf_event *event) 5998 { 5999 } 6000 6001 #endif /* CONFIG_EVENT_TRACING */ 6002 6003 #ifdef CONFIG_HAVE_HW_BREAKPOINT 6004 void perf_bp_event(struct perf_event *bp, void *data) 6005 { 6006 struct perf_sample_data sample; 6007 struct pt_regs *regs = data; 6008 6009 perf_sample_data_init(&sample, bp->attr.bp_addr, 0); 6010 6011 if (!bp->hw.state && !perf_exclude_event(bp, regs)) 6012 perf_swevent_event(bp, 1, &sample, regs); 6013 } 6014 #endif 6015 6016 /* 6017 * hrtimer based swevent callback 6018 */ 6019 6020 static enum hrtimer_restart perf_swevent_hrtimer(struct hrtimer *hrtimer) 6021 { 6022 enum hrtimer_restart ret = HRTIMER_RESTART; 6023 struct perf_sample_data data; 6024 struct pt_regs *regs; 6025 struct perf_event *event; 6026 u64 period; 6027 6028 event = container_of(hrtimer, struct perf_event, hw.hrtimer); 6029 6030 if (event->state != PERF_EVENT_STATE_ACTIVE) 6031 return HRTIMER_NORESTART; 6032 6033 event->pmu->read(event); 6034 6035 perf_sample_data_init(&data, 0, event->hw.last_period); 6036 regs = get_irq_regs(); 6037 6038 if (regs && !perf_exclude_event(event, regs)) { 6039 if (!(event->attr.exclude_idle && is_idle_task(current))) 6040 if (__perf_event_overflow(event, 1, &data, regs)) 6041 ret = HRTIMER_NORESTART; 6042 } 6043 6044 period = max_t(u64, 10000, event->hw.sample_period); 6045 hrtimer_forward_now(hrtimer, ns_to_ktime(period)); 6046 6047 return ret; 6048 } 6049 6050 static void perf_swevent_start_hrtimer(struct perf_event *event) 6051 { 6052 struct hw_perf_event *hwc = &event->hw; 6053 s64 period; 6054 6055 if (!is_sampling_event(event)) 6056 return; 6057 6058 period = local64_read(&hwc->period_left); 6059 if (period) { 6060 if (period < 0) 6061 period = 10000; 6062 6063 local64_set(&hwc->period_left, 0); 6064 } else { 6065 period = max_t(u64, 10000, hwc->sample_period); 6066 } 6067 __hrtimer_start_range_ns(&hwc->hrtimer, 6068 ns_to_ktime(period), 0, 6069 HRTIMER_MODE_REL_PINNED, 0); 6070 } 6071 6072 static void perf_swevent_cancel_hrtimer(struct perf_event *event) 6073 { 6074 struct hw_perf_event *hwc = &event->hw; 6075 6076 if (is_sampling_event(event)) { 6077 ktime_t remaining = hrtimer_get_remaining(&hwc->hrtimer); 6078 local64_set(&hwc->period_left, ktime_to_ns(remaining)); 6079 6080 hrtimer_cancel(&hwc->hrtimer); 6081 } 6082 } 6083 6084 static void perf_swevent_init_hrtimer(struct perf_event *event) 6085 { 6086 struct hw_perf_event *hwc = &event->hw; 6087 6088 if (!is_sampling_event(event)) 6089 return; 6090 6091 hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 6092 hwc->hrtimer.function = perf_swevent_hrtimer; 6093 6094 /* 6095 * Since hrtimers have a fixed rate, we can do a static freq->period 6096 * mapping and avoid the whole period adjust feedback stuff. 6097 */ 6098 if (event->attr.freq) { 6099 long freq = event->attr.sample_freq; 6100 6101 event->attr.sample_period = NSEC_PER_SEC / freq; 6102 hwc->sample_period = event->attr.sample_period; 6103 local64_set(&hwc->period_left, hwc->sample_period); 6104 hwc->last_period = hwc->sample_period; 6105 event->attr.freq = 0; 6106 } 6107 } 6108 6109 /* 6110 * Software event: cpu wall time clock 6111 */ 6112 6113 static void cpu_clock_event_update(struct perf_event *event) 6114 { 6115 s64 prev; 6116 u64 now; 6117 6118 now = local_clock(); 6119 prev = local64_xchg(&event->hw.prev_count, now); 6120 local64_add(now - prev, &event->count); 6121 } 6122 6123 static void cpu_clock_event_start(struct perf_event *event, int flags) 6124 { 6125 local64_set(&event->hw.prev_count, local_clock()); 6126 perf_swevent_start_hrtimer(event); 6127 } 6128 6129 static void cpu_clock_event_stop(struct perf_event *event, int flags) 6130 { 6131 perf_swevent_cancel_hrtimer(event); 6132 cpu_clock_event_update(event); 6133 } 6134 6135 static int cpu_clock_event_add(struct perf_event *event, int flags) 6136 { 6137 if (flags & PERF_EF_START) 6138 cpu_clock_event_start(event, flags); 6139 6140 return 0; 6141 } 6142 6143 static void cpu_clock_event_del(struct perf_event *event, int flags) 6144 { 6145 cpu_clock_event_stop(event, flags); 6146 } 6147 6148 static void cpu_clock_event_read(struct perf_event *event) 6149 { 6150 cpu_clock_event_update(event); 6151 } 6152 6153 static int cpu_clock_event_init(struct perf_event *event) 6154 { 6155 if (event->attr.type != PERF_TYPE_SOFTWARE) 6156 return -ENOENT; 6157 6158 if (event->attr.config != PERF_COUNT_SW_CPU_CLOCK) 6159 return -ENOENT; 6160 6161 /* 6162 * no branch sampling for software events 6163 */ 6164 if (has_branch_stack(event)) 6165 return -EOPNOTSUPP; 6166 6167 perf_swevent_init_hrtimer(event); 6168 6169 return 0; 6170 } 6171 6172 static struct pmu perf_cpu_clock = { 6173 .task_ctx_nr = perf_sw_context, 6174 6175 .event_init = cpu_clock_event_init, 6176 .add = cpu_clock_event_add, 6177 .del = cpu_clock_event_del, 6178 .start = cpu_clock_event_start, 6179 .stop = cpu_clock_event_stop, 6180 .read = cpu_clock_event_read, 6181 6182 .event_idx = perf_swevent_event_idx, 6183 }; 6184 6185 /* 6186 * Software event: task time clock 6187 */ 6188 6189 static void task_clock_event_update(struct perf_event *event, u64 now) 6190 { 6191 u64 prev; 6192 s64 delta; 6193 6194 prev = local64_xchg(&event->hw.prev_count, now); 6195 delta = now - prev; 6196 local64_add(delta, &event->count); 6197 } 6198 6199 static void task_clock_event_start(struct perf_event *event, int flags) 6200 { 6201 local64_set(&event->hw.prev_count, event->ctx->time); 6202 perf_swevent_start_hrtimer(event); 6203 } 6204 6205 static void task_clock_event_stop(struct perf_event *event, int flags) 6206 { 6207 perf_swevent_cancel_hrtimer(event); 6208 task_clock_event_update(event, event->ctx->time); 6209 } 6210 6211 static int task_clock_event_add(struct perf_event *event, int flags) 6212 { 6213 if (flags & PERF_EF_START) 6214 task_clock_event_start(event, flags); 6215 6216 return 0; 6217 } 6218 6219 static void task_clock_event_del(struct perf_event *event, int flags) 6220 { 6221 task_clock_event_stop(event, PERF_EF_UPDATE); 6222 } 6223 6224 static void task_clock_event_read(struct perf_event *event) 6225 { 6226 u64 now = perf_clock(); 6227 u64 delta = now - event->ctx->timestamp; 6228 u64 time = event->ctx->time + delta; 6229 6230 task_clock_event_update(event, time); 6231 } 6232 6233 static int task_clock_event_init(struct perf_event *event) 6234 { 6235 if (event->attr.type != PERF_TYPE_SOFTWARE) 6236 return -ENOENT; 6237 6238 if (event->attr.config != PERF_COUNT_SW_TASK_CLOCK) 6239 return -ENOENT; 6240 6241 /* 6242 * no branch sampling for software events 6243 */ 6244 if (has_branch_stack(event)) 6245 return -EOPNOTSUPP; 6246 6247 perf_swevent_init_hrtimer(event); 6248 6249 return 0; 6250 } 6251 6252 static struct pmu perf_task_clock = { 6253 .task_ctx_nr = perf_sw_context, 6254 6255 .event_init = task_clock_event_init, 6256 .add = task_clock_event_add, 6257 .del = task_clock_event_del, 6258 .start = task_clock_event_start, 6259 .stop = task_clock_event_stop, 6260 .read = task_clock_event_read, 6261 6262 .event_idx = perf_swevent_event_idx, 6263 }; 6264 6265 static void perf_pmu_nop_void(struct pmu *pmu) 6266 { 6267 } 6268 6269 static int perf_pmu_nop_int(struct pmu *pmu) 6270 { 6271 return 0; 6272 } 6273 6274 static void perf_pmu_start_txn(struct pmu *pmu) 6275 { 6276 perf_pmu_disable(pmu); 6277 } 6278 6279 static int perf_pmu_commit_txn(struct pmu *pmu) 6280 { 6281 perf_pmu_enable(pmu); 6282 return 0; 6283 } 6284 6285 static void perf_pmu_cancel_txn(struct pmu *pmu) 6286 { 6287 perf_pmu_enable(pmu); 6288 } 6289 6290 static int perf_event_idx_default(struct perf_event *event) 6291 { 6292 return event->hw.idx + 1; 6293 } 6294 6295 /* 6296 * Ensures all contexts with the same task_ctx_nr have the same 6297 * pmu_cpu_context too. 6298 */ 6299 static struct perf_cpu_context __percpu *find_pmu_context(int ctxn) 6300 { 6301 struct pmu *pmu; 6302 6303 if (ctxn < 0) 6304 return NULL; 6305 6306 list_for_each_entry(pmu, &pmus, entry) { 6307 if (pmu->task_ctx_nr == ctxn) 6308 return pmu->pmu_cpu_context; 6309 } 6310 6311 return NULL; 6312 } 6313 6314 static void update_pmu_context(struct pmu *pmu, struct pmu *old_pmu) 6315 { 6316 int cpu; 6317 6318 for_each_possible_cpu(cpu) { 6319 struct perf_cpu_context *cpuctx; 6320 6321 cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu); 6322 6323 if (cpuctx->unique_pmu == old_pmu) 6324 cpuctx->unique_pmu = pmu; 6325 } 6326 } 6327 6328 static void free_pmu_context(struct pmu *pmu) 6329 { 6330 struct pmu *i; 6331 6332 mutex_lock(&pmus_lock); 6333 /* 6334 * Like a real lame refcount. 6335 */ 6336 list_for_each_entry(i, &pmus, entry) { 6337 if (i->pmu_cpu_context == pmu->pmu_cpu_context) { 6338 update_pmu_context(i, pmu); 6339 goto out; 6340 } 6341 } 6342 6343 free_percpu(pmu->pmu_cpu_context); 6344 out: 6345 mutex_unlock(&pmus_lock); 6346 } 6347 static struct idr pmu_idr; 6348 6349 static ssize_t 6350 type_show(struct device *dev, struct device_attribute *attr, char *page) 6351 { 6352 struct pmu *pmu = dev_get_drvdata(dev); 6353 6354 return snprintf(page, PAGE_SIZE-1, "%d\n", pmu->type); 6355 } 6356 static DEVICE_ATTR_RO(type); 6357 6358 static ssize_t 6359 perf_event_mux_interval_ms_show(struct device *dev, 6360 struct device_attribute *attr, 6361 char *page) 6362 { 6363 struct pmu *pmu = dev_get_drvdata(dev); 6364 6365 return snprintf(page, PAGE_SIZE-1, "%d\n", pmu->hrtimer_interval_ms); 6366 } 6367 6368 static ssize_t 6369 perf_event_mux_interval_ms_store(struct device *dev, 6370 struct device_attribute *attr, 6371 const char *buf, size_t count) 6372 { 6373 struct pmu *pmu = dev_get_drvdata(dev); 6374 int timer, cpu, ret; 6375 6376 ret = kstrtoint(buf, 0, &timer); 6377 if (ret) 6378 return ret; 6379 6380 if (timer < 1) 6381 return -EINVAL; 6382 6383 /* same value, noting to do */ 6384 if (timer == pmu->hrtimer_interval_ms) 6385 return count; 6386 6387 pmu->hrtimer_interval_ms = timer; 6388 6389 /* update all cpuctx for this PMU */ 6390 for_each_possible_cpu(cpu) { 6391 struct perf_cpu_context *cpuctx; 6392 cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu); 6393 cpuctx->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * timer); 6394 6395 if (hrtimer_active(&cpuctx->hrtimer)) 6396 hrtimer_forward_now(&cpuctx->hrtimer, cpuctx->hrtimer_interval); 6397 } 6398 6399 return count; 6400 } 6401 static DEVICE_ATTR_RW(perf_event_mux_interval_ms); 6402 6403 static struct attribute *pmu_dev_attrs[] = { 6404 &dev_attr_type.attr, 6405 &dev_attr_perf_event_mux_interval_ms.attr, 6406 NULL, 6407 }; 6408 ATTRIBUTE_GROUPS(pmu_dev); 6409 6410 static int pmu_bus_running; 6411 static struct bus_type pmu_bus = { 6412 .name = "event_source", 6413 .dev_groups = pmu_dev_groups, 6414 }; 6415 6416 static void pmu_dev_release(struct device *dev) 6417 { 6418 kfree(dev); 6419 } 6420 6421 static int pmu_dev_alloc(struct pmu *pmu) 6422 { 6423 int ret = -ENOMEM; 6424 6425 pmu->dev = kzalloc(sizeof(struct device), GFP_KERNEL); 6426 if (!pmu->dev) 6427 goto out; 6428 6429 pmu->dev->groups = pmu->attr_groups; 6430 device_initialize(pmu->dev); 6431 ret = dev_set_name(pmu->dev, "%s", pmu->name); 6432 if (ret) 6433 goto free_dev; 6434 6435 dev_set_drvdata(pmu->dev, pmu); 6436 pmu->dev->bus = &pmu_bus; 6437 pmu->dev->release = pmu_dev_release; 6438 ret = device_add(pmu->dev); 6439 if (ret) 6440 goto free_dev; 6441 6442 out: 6443 return ret; 6444 6445 free_dev: 6446 put_device(pmu->dev); 6447 goto out; 6448 } 6449 6450 static struct lock_class_key cpuctx_mutex; 6451 static struct lock_class_key cpuctx_lock; 6452 6453 int perf_pmu_register(struct pmu *pmu, const char *name, int type) 6454 { 6455 int cpu, ret; 6456 6457 mutex_lock(&pmus_lock); 6458 ret = -ENOMEM; 6459 pmu->pmu_disable_count = alloc_percpu(int); 6460 if (!pmu->pmu_disable_count) 6461 goto unlock; 6462 6463 pmu->type = -1; 6464 if (!name) 6465 goto skip_type; 6466 pmu->name = name; 6467 6468 if (type < 0) { 6469 type = idr_alloc(&pmu_idr, pmu, PERF_TYPE_MAX, 0, GFP_KERNEL); 6470 if (type < 0) { 6471 ret = type; 6472 goto free_pdc; 6473 } 6474 } 6475 pmu->type = type; 6476 6477 if (pmu_bus_running) { 6478 ret = pmu_dev_alloc(pmu); 6479 if (ret) 6480 goto free_idr; 6481 } 6482 6483 skip_type: 6484 pmu->pmu_cpu_context = find_pmu_context(pmu->task_ctx_nr); 6485 if (pmu->pmu_cpu_context) 6486 goto got_cpu_context; 6487 6488 ret = -ENOMEM; 6489 pmu->pmu_cpu_context = alloc_percpu(struct perf_cpu_context); 6490 if (!pmu->pmu_cpu_context) 6491 goto free_dev; 6492 6493 for_each_possible_cpu(cpu) { 6494 struct perf_cpu_context *cpuctx; 6495 6496 cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu); 6497 __perf_event_init_context(&cpuctx->ctx); 6498 lockdep_set_class(&cpuctx->ctx.mutex, &cpuctx_mutex); 6499 lockdep_set_class(&cpuctx->ctx.lock, &cpuctx_lock); 6500 cpuctx->ctx.type = cpu_context; 6501 cpuctx->ctx.pmu = pmu; 6502 6503 __perf_cpu_hrtimer_init(cpuctx, cpu); 6504 6505 INIT_LIST_HEAD(&cpuctx->rotation_list); 6506 cpuctx->unique_pmu = pmu; 6507 } 6508 6509 got_cpu_context: 6510 if (!pmu->start_txn) { 6511 if (pmu->pmu_enable) { 6512 /* 6513 * If we have pmu_enable/pmu_disable calls, install 6514 * transaction stubs that use that to try and batch 6515 * hardware accesses. 6516 */ 6517 pmu->start_txn = perf_pmu_start_txn; 6518 pmu->commit_txn = perf_pmu_commit_txn; 6519 pmu->cancel_txn = perf_pmu_cancel_txn; 6520 } else { 6521 pmu->start_txn = perf_pmu_nop_void; 6522 pmu->commit_txn = perf_pmu_nop_int; 6523 pmu->cancel_txn = perf_pmu_nop_void; 6524 } 6525 } 6526 6527 if (!pmu->pmu_enable) { 6528 pmu->pmu_enable = perf_pmu_nop_void; 6529 pmu->pmu_disable = perf_pmu_nop_void; 6530 } 6531 6532 if (!pmu->event_idx) 6533 pmu->event_idx = perf_event_idx_default; 6534 6535 list_add_rcu(&pmu->entry, &pmus); 6536 ret = 0; 6537 unlock: 6538 mutex_unlock(&pmus_lock); 6539 6540 return ret; 6541 6542 free_dev: 6543 device_del(pmu->dev); 6544 put_device(pmu->dev); 6545 6546 free_idr: 6547 if (pmu->type >= PERF_TYPE_MAX) 6548 idr_remove(&pmu_idr, pmu->type); 6549 6550 free_pdc: 6551 free_percpu(pmu->pmu_disable_count); 6552 goto unlock; 6553 } 6554 6555 void perf_pmu_unregister(struct pmu *pmu) 6556 { 6557 mutex_lock(&pmus_lock); 6558 list_del_rcu(&pmu->entry); 6559 mutex_unlock(&pmus_lock); 6560 6561 /* 6562 * We dereference the pmu list under both SRCU and regular RCU, so 6563 * synchronize against both of those. 6564 */ 6565 synchronize_srcu(&pmus_srcu); 6566 synchronize_rcu(); 6567 6568 free_percpu(pmu->pmu_disable_count); 6569 if (pmu->type >= PERF_TYPE_MAX) 6570 idr_remove(&pmu_idr, pmu->type); 6571 device_del(pmu->dev); 6572 put_device(pmu->dev); 6573 free_pmu_context(pmu); 6574 } 6575 6576 struct pmu *perf_init_event(struct perf_event *event) 6577 { 6578 struct pmu *pmu = NULL; 6579 int idx; 6580 int ret; 6581 6582 idx = srcu_read_lock(&pmus_srcu); 6583 6584 rcu_read_lock(); 6585 pmu = idr_find(&pmu_idr, event->attr.type); 6586 rcu_read_unlock(); 6587 if (pmu) { 6588 event->pmu = pmu; 6589 ret = pmu->event_init(event); 6590 if (ret) 6591 pmu = ERR_PTR(ret); 6592 goto unlock; 6593 } 6594 6595 list_for_each_entry_rcu(pmu, &pmus, entry) { 6596 event->pmu = pmu; 6597 ret = pmu->event_init(event); 6598 if (!ret) 6599 goto unlock; 6600 6601 if (ret != -ENOENT) { 6602 pmu = ERR_PTR(ret); 6603 goto unlock; 6604 } 6605 } 6606 pmu = ERR_PTR(-ENOENT); 6607 unlock: 6608 srcu_read_unlock(&pmus_srcu, idx); 6609 6610 return pmu; 6611 } 6612 6613 static void account_event_cpu(struct perf_event *event, int cpu) 6614 { 6615 if (event->parent) 6616 return; 6617 6618 if (has_branch_stack(event)) { 6619 if (!(event->attach_state & PERF_ATTACH_TASK)) 6620 atomic_inc(&per_cpu(perf_branch_stack_events, cpu)); 6621 } 6622 if (is_cgroup_event(event)) 6623 atomic_inc(&per_cpu(perf_cgroup_events, cpu)); 6624 } 6625 6626 static void account_event(struct perf_event *event) 6627 { 6628 if (event->parent) 6629 return; 6630 6631 if (event->attach_state & PERF_ATTACH_TASK) 6632 static_key_slow_inc(&perf_sched_events.key); 6633 if (event->attr.mmap || event->attr.mmap_data) 6634 atomic_inc(&nr_mmap_events); 6635 if (event->attr.comm) 6636 atomic_inc(&nr_comm_events); 6637 if (event->attr.task) 6638 atomic_inc(&nr_task_events); 6639 if (event->attr.freq) { 6640 if (atomic_inc_return(&nr_freq_events) == 1) 6641 tick_nohz_full_kick_all(); 6642 } 6643 if (has_branch_stack(event)) 6644 static_key_slow_inc(&perf_sched_events.key); 6645 if (is_cgroup_event(event)) 6646 static_key_slow_inc(&perf_sched_events.key); 6647 6648 account_event_cpu(event, event->cpu); 6649 } 6650 6651 /* 6652 * Allocate and initialize a event structure 6653 */ 6654 static struct perf_event * 6655 perf_event_alloc(struct perf_event_attr *attr, int cpu, 6656 struct task_struct *task, 6657 struct perf_event *group_leader, 6658 struct perf_event *parent_event, 6659 perf_overflow_handler_t overflow_handler, 6660 void *context) 6661 { 6662 struct pmu *pmu; 6663 struct perf_event *event; 6664 struct hw_perf_event *hwc; 6665 long err = -EINVAL; 6666 6667 if ((unsigned)cpu >= nr_cpu_ids) { 6668 if (!task || cpu != -1) 6669 return ERR_PTR(-EINVAL); 6670 } 6671 6672 event = kzalloc(sizeof(*event), GFP_KERNEL); 6673 if (!event) 6674 return ERR_PTR(-ENOMEM); 6675 6676 /* 6677 * Single events are their own group leaders, with an 6678 * empty sibling list: 6679 */ 6680 if (!group_leader) 6681 group_leader = event; 6682 6683 mutex_init(&event->child_mutex); 6684 INIT_LIST_HEAD(&event->child_list); 6685 6686 INIT_LIST_HEAD(&event->group_entry); 6687 INIT_LIST_HEAD(&event->event_entry); 6688 INIT_LIST_HEAD(&event->sibling_list); 6689 INIT_LIST_HEAD(&event->rb_entry); 6690 INIT_LIST_HEAD(&event->active_entry); 6691 INIT_HLIST_NODE(&event->hlist_entry); 6692 6693 6694 init_waitqueue_head(&event->waitq); 6695 init_irq_work(&event->pending, perf_pending_event); 6696 6697 mutex_init(&event->mmap_mutex); 6698 6699 atomic_long_set(&event->refcount, 1); 6700 event->cpu = cpu; 6701 event->attr = *attr; 6702 event->group_leader = group_leader; 6703 event->pmu = NULL; 6704 event->oncpu = -1; 6705 6706 event->parent = parent_event; 6707 6708 event->ns = get_pid_ns(task_active_pid_ns(current)); 6709 event->id = atomic64_inc_return(&perf_event_id); 6710 6711 event->state = PERF_EVENT_STATE_INACTIVE; 6712 6713 if (task) { 6714 event->attach_state = PERF_ATTACH_TASK; 6715 6716 if (attr->type == PERF_TYPE_TRACEPOINT) 6717 event->hw.tp_target = task; 6718 #ifdef CONFIG_HAVE_HW_BREAKPOINT 6719 /* 6720 * hw_breakpoint is a bit difficult here.. 6721 */ 6722 else if (attr->type == PERF_TYPE_BREAKPOINT) 6723 event->hw.bp_target = task; 6724 #endif 6725 } 6726 6727 if (!overflow_handler && parent_event) { 6728 overflow_handler = parent_event->overflow_handler; 6729 context = parent_event->overflow_handler_context; 6730 } 6731 6732 event->overflow_handler = overflow_handler; 6733 event->overflow_handler_context = context; 6734 6735 perf_event__state_init(event); 6736 6737 pmu = NULL; 6738 6739 hwc = &event->hw; 6740 hwc->sample_period = attr->sample_period; 6741 if (attr->freq && attr->sample_freq) 6742 hwc->sample_period = 1; 6743 hwc->last_period = hwc->sample_period; 6744 6745 local64_set(&hwc->period_left, hwc->sample_period); 6746 6747 /* 6748 * we currently do not support PERF_FORMAT_GROUP on inherited events 6749 */ 6750 if (attr->inherit && (attr->read_format & PERF_FORMAT_GROUP)) 6751 goto err_ns; 6752 6753 pmu = perf_init_event(event); 6754 if (!pmu) 6755 goto err_ns; 6756 else if (IS_ERR(pmu)) { 6757 err = PTR_ERR(pmu); 6758 goto err_ns; 6759 } 6760 6761 if (!event->parent) { 6762 if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) { 6763 err = get_callchain_buffers(); 6764 if (err) 6765 goto err_pmu; 6766 } 6767 } 6768 6769 return event; 6770 6771 err_pmu: 6772 if (event->destroy) 6773 event->destroy(event); 6774 err_ns: 6775 if (event->ns) 6776 put_pid_ns(event->ns); 6777 kfree(event); 6778 6779 return ERR_PTR(err); 6780 } 6781 6782 static int perf_copy_attr(struct perf_event_attr __user *uattr, 6783 struct perf_event_attr *attr) 6784 { 6785 u32 size; 6786 int ret; 6787 6788 if (!access_ok(VERIFY_WRITE, uattr, PERF_ATTR_SIZE_VER0)) 6789 return -EFAULT; 6790 6791 /* 6792 * zero the full structure, so that a short copy will be nice. 6793 */ 6794 memset(attr, 0, sizeof(*attr)); 6795 6796 ret = get_user(size, &uattr->size); 6797 if (ret) 6798 return ret; 6799 6800 if (size > PAGE_SIZE) /* silly large */ 6801 goto err_size; 6802 6803 if (!size) /* abi compat */ 6804 size = PERF_ATTR_SIZE_VER0; 6805 6806 if (size < PERF_ATTR_SIZE_VER0) 6807 goto err_size; 6808 6809 /* 6810 * If we're handed a bigger struct than we know of, 6811 * ensure all the unknown bits are 0 - i.e. new 6812 * user-space does not rely on any kernel feature 6813 * extensions we dont know about yet. 6814 */ 6815 if (size > sizeof(*attr)) { 6816 unsigned char __user *addr; 6817 unsigned char __user *end; 6818 unsigned char val; 6819 6820 addr = (void __user *)uattr + sizeof(*attr); 6821 end = (void __user *)uattr + size; 6822 6823 for (; addr < end; addr++) { 6824 ret = get_user(val, addr); 6825 if (ret) 6826 return ret; 6827 if (val) 6828 goto err_size; 6829 } 6830 size = sizeof(*attr); 6831 } 6832 6833 ret = copy_from_user(attr, uattr, size); 6834 if (ret) 6835 return -EFAULT; 6836 6837 /* disabled for now */ 6838 if (attr->mmap2) 6839 return -EINVAL; 6840 6841 if (attr->__reserved_1) 6842 return -EINVAL; 6843 6844 if (attr->sample_type & ~(PERF_SAMPLE_MAX-1)) 6845 return -EINVAL; 6846 6847 if (attr->read_format & ~(PERF_FORMAT_MAX-1)) 6848 return -EINVAL; 6849 6850 if (attr->sample_type & PERF_SAMPLE_BRANCH_STACK) { 6851 u64 mask = attr->branch_sample_type; 6852 6853 /* only using defined bits */ 6854 if (mask & ~(PERF_SAMPLE_BRANCH_MAX-1)) 6855 return -EINVAL; 6856 6857 /* at least one branch bit must be set */ 6858 if (!(mask & ~PERF_SAMPLE_BRANCH_PLM_ALL)) 6859 return -EINVAL; 6860 6861 /* propagate priv level, when not set for branch */ 6862 if (!(mask & PERF_SAMPLE_BRANCH_PLM_ALL)) { 6863 6864 /* exclude_kernel checked on syscall entry */ 6865 if (!attr->exclude_kernel) 6866 mask |= PERF_SAMPLE_BRANCH_KERNEL; 6867 6868 if (!attr->exclude_user) 6869 mask |= PERF_SAMPLE_BRANCH_USER; 6870 6871 if (!attr->exclude_hv) 6872 mask |= PERF_SAMPLE_BRANCH_HV; 6873 /* 6874 * adjust user setting (for HW filter setup) 6875 */ 6876 attr->branch_sample_type = mask; 6877 } 6878 /* privileged levels capture (kernel, hv): check permissions */ 6879 if ((mask & PERF_SAMPLE_BRANCH_PERM_PLM) 6880 && perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN)) 6881 return -EACCES; 6882 } 6883 6884 if (attr->sample_type & PERF_SAMPLE_REGS_USER) { 6885 ret = perf_reg_validate(attr->sample_regs_user); 6886 if (ret) 6887 return ret; 6888 } 6889 6890 if (attr->sample_type & PERF_SAMPLE_STACK_USER) { 6891 if (!arch_perf_have_user_stack_dump()) 6892 return -ENOSYS; 6893 6894 /* 6895 * We have __u32 type for the size, but so far 6896 * we can only use __u16 as maximum due to the 6897 * __u16 sample size limit. 6898 */ 6899 if (attr->sample_stack_user >= USHRT_MAX) 6900 ret = -EINVAL; 6901 else if (!IS_ALIGNED(attr->sample_stack_user, sizeof(u64))) 6902 ret = -EINVAL; 6903 } 6904 6905 out: 6906 return ret; 6907 6908 err_size: 6909 put_user(sizeof(*attr), &uattr->size); 6910 ret = -E2BIG; 6911 goto out; 6912 } 6913 6914 static int 6915 perf_event_set_output(struct perf_event *event, struct perf_event *output_event) 6916 { 6917 struct ring_buffer *rb = NULL, *old_rb = NULL; 6918 int ret = -EINVAL; 6919 6920 if (!output_event) 6921 goto set; 6922 6923 /* don't allow circular references */ 6924 if (event == output_event) 6925 goto out; 6926 6927 /* 6928 * Don't allow cross-cpu buffers 6929 */ 6930 if (output_event->cpu != event->cpu) 6931 goto out; 6932 6933 /* 6934 * If its not a per-cpu rb, it must be the same task. 6935 */ 6936 if (output_event->cpu == -1 && output_event->ctx != event->ctx) 6937 goto out; 6938 6939 set: 6940 mutex_lock(&event->mmap_mutex); 6941 /* Can't redirect output if we've got an active mmap() */ 6942 if (atomic_read(&event->mmap_count)) 6943 goto unlock; 6944 6945 old_rb = event->rb; 6946 6947 if (output_event) { 6948 /* get the rb we want to redirect to */ 6949 rb = ring_buffer_get(output_event); 6950 if (!rb) 6951 goto unlock; 6952 } 6953 6954 if (old_rb) 6955 ring_buffer_detach(event, old_rb); 6956 6957 if (rb) 6958 ring_buffer_attach(event, rb); 6959 6960 rcu_assign_pointer(event->rb, rb); 6961 6962 if (old_rb) { 6963 ring_buffer_put(old_rb); 6964 /* 6965 * Since we detached before setting the new rb, so that we 6966 * could attach the new rb, we could have missed a wakeup. 6967 * Provide it now. 6968 */ 6969 wake_up_all(&event->waitq); 6970 } 6971 6972 ret = 0; 6973 unlock: 6974 mutex_unlock(&event->mmap_mutex); 6975 6976 out: 6977 return ret; 6978 } 6979 6980 /** 6981 * sys_perf_event_open - open a performance event, associate it to a task/cpu 6982 * 6983 * @attr_uptr: event_id type attributes for monitoring/sampling 6984 * @pid: target pid 6985 * @cpu: target cpu 6986 * @group_fd: group leader event fd 6987 */ 6988 SYSCALL_DEFINE5(perf_event_open, 6989 struct perf_event_attr __user *, attr_uptr, 6990 pid_t, pid, int, cpu, int, group_fd, unsigned long, flags) 6991 { 6992 struct perf_event *group_leader = NULL, *output_event = NULL; 6993 struct perf_event *event, *sibling; 6994 struct perf_event_attr attr; 6995 struct perf_event_context *ctx; 6996 struct file *event_file = NULL; 6997 struct fd group = {NULL, 0}; 6998 struct task_struct *task = NULL; 6999 struct pmu *pmu; 7000 int event_fd; 7001 int move_group = 0; 7002 int err; 7003 int f_flags = O_RDWR; 7004 7005 /* for future expandability... */ 7006 if (flags & ~PERF_FLAG_ALL) 7007 return -EINVAL; 7008 7009 err = perf_copy_attr(attr_uptr, &attr); 7010 if (err) 7011 return err; 7012 7013 if (!attr.exclude_kernel) { 7014 if (perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN)) 7015 return -EACCES; 7016 } 7017 7018 if (attr.freq) { 7019 if (attr.sample_freq > sysctl_perf_event_sample_rate) 7020 return -EINVAL; 7021 } 7022 7023 /* 7024 * In cgroup mode, the pid argument is used to pass the fd 7025 * opened to the cgroup directory in cgroupfs. The cpu argument 7026 * designates the cpu on which to monitor threads from that 7027 * cgroup. 7028 */ 7029 if ((flags & PERF_FLAG_PID_CGROUP) && (pid == -1 || cpu == -1)) 7030 return -EINVAL; 7031 7032 if (flags & PERF_FLAG_FD_CLOEXEC) 7033 f_flags |= O_CLOEXEC; 7034 7035 event_fd = get_unused_fd_flags(f_flags); 7036 if (event_fd < 0) 7037 return event_fd; 7038 7039 if (group_fd != -1) { 7040 err = perf_fget_light(group_fd, &group); 7041 if (err) 7042 goto err_fd; 7043 group_leader = group.file->private_data; 7044 if (flags & PERF_FLAG_FD_OUTPUT) 7045 output_event = group_leader; 7046 if (flags & PERF_FLAG_FD_NO_GROUP) 7047 group_leader = NULL; 7048 } 7049 7050 if (pid != -1 && !(flags & PERF_FLAG_PID_CGROUP)) { 7051 task = find_lively_task_by_vpid(pid); 7052 if (IS_ERR(task)) { 7053 err = PTR_ERR(task); 7054 goto err_group_fd; 7055 } 7056 } 7057 7058 get_online_cpus(); 7059 7060 event = perf_event_alloc(&attr, cpu, task, group_leader, NULL, 7061 NULL, NULL); 7062 if (IS_ERR(event)) { 7063 err = PTR_ERR(event); 7064 goto err_task; 7065 } 7066 7067 if (flags & PERF_FLAG_PID_CGROUP) { 7068 err = perf_cgroup_connect(pid, event, &attr, group_leader); 7069 if (err) { 7070 __free_event(event); 7071 goto err_task; 7072 } 7073 } 7074 7075 account_event(event); 7076 7077 /* 7078 * Special case software events and allow them to be part of 7079 * any hardware group. 7080 */ 7081 pmu = event->pmu; 7082 7083 if (group_leader && 7084 (is_software_event(event) != is_software_event(group_leader))) { 7085 if (is_software_event(event)) { 7086 /* 7087 * If event and group_leader are not both a software 7088 * event, and event is, then group leader is not. 7089 * 7090 * Allow the addition of software events to !software 7091 * groups, this is safe because software events never 7092 * fail to schedule. 7093 */ 7094 pmu = group_leader->pmu; 7095 } else if (is_software_event(group_leader) && 7096 (group_leader->group_flags & PERF_GROUP_SOFTWARE)) { 7097 /* 7098 * In case the group is a pure software group, and we 7099 * try to add a hardware event, move the whole group to 7100 * the hardware context. 7101 */ 7102 move_group = 1; 7103 } 7104 } 7105 7106 /* 7107 * Get the target context (task or percpu): 7108 */ 7109 ctx = find_get_context(pmu, task, event->cpu); 7110 if (IS_ERR(ctx)) { 7111 err = PTR_ERR(ctx); 7112 goto err_alloc; 7113 } 7114 7115 if (task) { 7116 put_task_struct(task); 7117 task = NULL; 7118 } 7119 7120 /* 7121 * Look up the group leader (we will attach this event to it): 7122 */ 7123 if (group_leader) { 7124 err = -EINVAL; 7125 7126 /* 7127 * Do not allow a recursive hierarchy (this new sibling 7128 * becoming part of another group-sibling): 7129 */ 7130 if (group_leader->group_leader != group_leader) 7131 goto err_context; 7132 /* 7133 * Do not allow to attach to a group in a different 7134 * task or CPU context: 7135 */ 7136 if (move_group) { 7137 if (group_leader->ctx->type != ctx->type) 7138 goto err_context; 7139 } else { 7140 if (group_leader->ctx != ctx) 7141 goto err_context; 7142 } 7143 7144 /* 7145 * Only a group leader can be exclusive or pinned 7146 */ 7147 if (attr.exclusive || attr.pinned) 7148 goto err_context; 7149 } 7150 7151 if (output_event) { 7152 err = perf_event_set_output(event, output_event); 7153 if (err) 7154 goto err_context; 7155 } 7156 7157 event_file = anon_inode_getfile("[perf_event]", &perf_fops, event, 7158 f_flags); 7159 if (IS_ERR(event_file)) { 7160 err = PTR_ERR(event_file); 7161 goto err_context; 7162 } 7163 7164 if (move_group) { 7165 struct perf_event_context *gctx = group_leader->ctx; 7166 7167 mutex_lock(&gctx->mutex); 7168 perf_remove_from_context(group_leader); 7169 7170 /* 7171 * Removing from the context ends up with disabled 7172 * event. What we want here is event in the initial 7173 * startup state, ready to be add into new context. 7174 */ 7175 perf_event__state_init(group_leader); 7176 list_for_each_entry(sibling, &group_leader->sibling_list, 7177 group_entry) { 7178 perf_remove_from_context(sibling); 7179 perf_event__state_init(sibling); 7180 put_ctx(gctx); 7181 } 7182 mutex_unlock(&gctx->mutex); 7183 put_ctx(gctx); 7184 } 7185 7186 WARN_ON_ONCE(ctx->parent_ctx); 7187 mutex_lock(&ctx->mutex); 7188 7189 if (move_group) { 7190 synchronize_rcu(); 7191 perf_install_in_context(ctx, group_leader, event->cpu); 7192 get_ctx(ctx); 7193 list_for_each_entry(sibling, &group_leader->sibling_list, 7194 group_entry) { 7195 perf_install_in_context(ctx, sibling, event->cpu); 7196 get_ctx(ctx); 7197 } 7198 } 7199 7200 perf_install_in_context(ctx, event, event->cpu); 7201 perf_unpin_context(ctx); 7202 mutex_unlock(&ctx->mutex); 7203 7204 put_online_cpus(); 7205 7206 event->owner = current; 7207 7208 mutex_lock(¤t->perf_event_mutex); 7209 list_add_tail(&event->owner_entry, ¤t->perf_event_list); 7210 mutex_unlock(¤t->perf_event_mutex); 7211 7212 /* 7213 * Precalculate sample_data sizes 7214 */ 7215 perf_event__header_size(event); 7216 perf_event__id_header_size(event); 7217 7218 /* 7219 * Drop the reference on the group_event after placing the 7220 * new event on the sibling_list. This ensures destruction 7221 * of the group leader will find the pointer to itself in 7222 * perf_group_detach(). 7223 */ 7224 fdput(group); 7225 fd_install(event_fd, event_file); 7226 return event_fd; 7227 7228 err_context: 7229 perf_unpin_context(ctx); 7230 put_ctx(ctx); 7231 err_alloc: 7232 free_event(event); 7233 err_task: 7234 put_online_cpus(); 7235 if (task) 7236 put_task_struct(task); 7237 err_group_fd: 7238 fdput(group); 7239 err_fd: 7240 put_unused_fd(event_fd); 7241 return err; 7242 } 7243 7244 /** 7245 * perf_event_create_kernel_counter 7246 * 7247 * @attr: attributes of the counter to create 7248 * @cpu: cpu in which the counter is bound 7249 * @task: task to profile (NULL for percpu) 7250 */ 7251 struct perf_event * 7252 perf_event_create_kernel_counter(struct perf_event_attr *attr, int cpu, 7253 struct task_struct *task, 7254 perf_overflow_handler_t overflow_handler, 7255 void *context) 7256 { 7257 struct perf_event_context *ctx; 7258 struct perf_event *event; 7259 int err; 7260 7261 /* 7262 * Get the target context (task or percpu): 7263 */ 7264 7265 event = perf_event_alloc(attr, cpu, task, NULL, NULL, 7266 overflow_handler, context); 7267 if (IS_ERR(event)) { 7268 err = PTR_ERR(event); 7269 goto err; 7270 } 7271 7272 account_event(event); 7273 7274 ctx = find_get_context(event->pmu, task, cpu); 7275 if (IS_ERR(ctx)) { 7276 err = PTR_ERR(ctx); 7277 goto err_free; 7278 } 7279 7280 WARN_ON_ONCE(ctx->parent_ctx); 7281 mutex_lock(&ctx->mutex); 7282 perf_install_in_context(ctx, event, cpu); 7283 perf_unpin_context(ctx); 7284 mutex_unlock(&ctx->mutex); 7285 7286 return event; 7287 7288 err_free: 7289 free_event(event); 7290 err: 7291 return ERR_PTR(err); 7292 } 7293 EXPORT_SYMBOL_GPL(perf_event_create_kernel_counter); 7294 7295 void perf_pmu_migrate_context(struct pmu *pmu, int src_cpu, int dst_cpu) 7296 { 7297 struct perf_event_context *src_ctx; 7298 struct perf_event_context *dst_ctx; 7299 struct perf_event *event, *tmp; 7300 LIST_HEAD(events); 7301 7302 src_ctx = &per_cpu_ptr(pmu->pmu_cpu_context, src_cpu)->ctx; 7303 dst_ctx = &per_cpu_ptr(pmu->pmu_cpu_context, dst_cpu)->ctx; 7304 7305 mutex_lock(&src_ctx->mutex); 7306 list_for_each_entry_safe(event, tmp, &src_ctx->event_list, 7307 event_entry) { 7308 perf_remove_from_context(event); 7309 unaccount_event_cpu(event, src_cpu); 7310 put_ctx(src_ctx); 7311 list_add(&event->migrate_entry, &events); 7312 } 7313 mutex_unlock(&src_ctx->mutex); 7314 7315 synchronize_rcu(); 7316 7317 mutex_lock(&dst_ctx->mutex); 7318 list_for_each_entry_safe(event, tmp, &events, migrate_entry) { 7319 list_del(&event->migrate_entry); 7320 if (event->state >= PERF_EVENT_STATE_OFF) 7321 event->state = PERF_EVENT_STATE_INACTIVE; 7322 account_event_cpu(event, dst_cpu); 7323 perf_install_in_context(dst_ctx, event, dst_cpu); 7324 get_ctx(dst_ctx); 7325 } 7326 mutex_unlock(&dst_ctx->mutex); 7327 } 7328 EXPORT_SYMBOL_GPL(perf_pmu_migrate_context); 7329 7330 static void sync_child_event(struct perf_event *child_event, 7331 struct task_struct *child) 7332 { 7333 struct perf_event *parent_event = child_event->parent; 7334 u64 child_val; 7335 7336 if (child_event->attr.inherit_stat) 7337 perf_event_read_event(child_event, child); 7338 7339 child_val = perf_event_count(child_event); 7340 7341 /* 7342 * Add back the child's count to the parent's count: 7343 */ 7344 atomic64_add(child_val, &parent_event->child_count); 7345 atomic64_add(child_event->total_time_enabled, 7346 &parent_event->child_total_time_enabled); 7347 atomic64_add(child_event->total_time_running, 7348 &parent_event->child_total_time_running); 7349 7350 /* 7351 * Remove this event from the parent's list 7352 */ 7353 WARN_ON_ONCE(parent_event->ctx->parent_ctx); 7354 mutex_lock(&parent_event->child_mutex); 7355 list_del_init(&child_event->child_list); 7356 mutex_unlock(&parent_event->child_mutex); 7357 7358 /* 7359 * Release the parent event, if this was the last 7360 * reference to it. 7361 */ 7362 put_event(parent_event); 7363 } 7364 7365 static void 7366 __perf_event_exit_task(struct perf_event *child_event, 7367 struct perf_event_context *child_ctx, 7368 struct task_struct *child) 7369 { 7370 if (child_event->parent) { 7371 raw_spin_lock_irq(&child_ctx->lock); 7372 perf_group_detach(child_event); 7373 raw_spin_unlock_irq(&child_ctx->lock); 7374 } 7375 7376 perf_remove_from_context(child_event); 7377 7378 /* 7379 * It can happen that the parent exits first, and has events 7380 * that are still around due to the child reference. These 7381 * events need to be zapped. 7382 */ 7383 if (child_event->parent) { 7384 sync_child_event(child_event, child); 7385 free_event(child_event); 7386 } 7387 } 7388 7389 static void perf_event_exit_task_context(struct task_struct *child, int ctxn) 7390 { 7391 struct perf_event *child_event, *tmp; 7392 struct perf_event_context *child_ctx; 7393 unsigned long flags; 7394 7395 if (likely(!child->perf_event_ctxp[ctxn])) { 7396 perf_event_task(child, NULL, 0); 7397 return; 7398 } 7399 7400 local_irq_save(flags); 7401 /* 7402 * We can't reschedule here because interrupts are disabled, 7403 * and either child is current or it is a task that can't be 7404 * scheduled, so we are now safe from rescheduling changing 7405 * our context. 7406 */ 7407 child_ctx = rcu_dereference_raw(child->perf_event_ctxp[ctxn]); 7408 7409 /* 7410 * Take the context lock here so that if find_get_context is 7411 * reading child->perf_event_ctxp, we wait until it has 7412 * incremented the context's refcount before we do put_ctx below. 7413 */ 7414 raw_spin_lock(&child_ctx->lock); 7415 task_ctx_sched_out(child_ctx); 7416 child->perf_event_ctxp[ctxn] = NULL; 7417 /* 7418 * If this context is a clone; unclone it so it can't get 7419 * swapped to another process while we're removing all 7420 * the events from it. 7421 */ 7422 unclone_ctx(child_ctx); 7423 update_context_time(child_ctx); 7424 raw_spin_unlock_irqrestore(&child_ctx->lock, flags); 7425 7426 /* 7427 * Report the task dead after unscheduling the events so that we 7428 * won't get any samples after PERF_RECORD_EXIT. We can however still 7429 * get a few PERF_RECORD_READ events. 7430 */ 7431 perf_event_task(child, child_ctx, 0); 7432 7433 /* 7434 * We can recurse on the same lock type through: 7435 * 7436 * __perf_event_exit_task() 7437 * sync_child_event() 7438 * put_event() 7439 * mutex_lock(&ctx->mutex) 7440 * 7441 * But since its the parent context it won't be the same instance. 7442 */ 7443 mutex_lock(&child_ctx->mutex); 7444 7445 again: 7446 list_for_each_entry_safe(child_event, tmp, &child_ctx->pinned_groups, 7447 group_entry) 7448 __perf_event_exit_task(child_event, child_ctx, child); 7449 7450 list_for_each_entry_safe(child_event, tmp, &child_ctx->flexible_groups, 7451 group_entry) 7452 __perf_event_exit_task(child_event, child_ctx, child); 7453 7454 /* 7455 * If the last event was a group event, it will have appended all 7456 * its siblings to the list, but we obtained 'tmp' before that which 7457 * will still point to the list head terminating the iteration. 7458 */ 7459 if (!list_empty(&child_ctx->pinned_groups) || 7460 !list_empty(&child_ctx->flexible_groups)) 7461 goto again; 7462 7463 mutex_unlock(&child_ctx->mutex); 7464 7465 put_ctx(child_ctx); 7466 } 7467 7468 /* 7469 * When a child task exits, feed back event values to parent events. 7470 */ 7471 void perf_event_exit_task(struct task_struct *child) 7472 { 7473 struct perf_event *event, *tmp; 7474 int ctxn; 7475 7476 mutex_lock(&child->perf_event_mutex); 7477 list_for_each_entry_safe(event, tmp, &child->perf_event_list, 7478 owner_entry) { 7479 list_del_init(&event->owner_entry); 7480 7481 /* 7482 * Ensure the list deletion is visible before we clear 7483 * the owner, closes a race against perf_release() where 7484 * we need to serialize on the owner->perf_event_mutex. 7485 */ 7486 smp_wmb(); 7487 event->owner = NULL; 7488 } 7489 mutex_unlock(&child->perf_event_mutex); 7490 7491 for_each_task_context_nr(ctxn) 7492 perf_event_exit_task_context(child, ctxn); 7493 } 7494 7495 static void perf_free_event(struct perf_event *event, 7496 struct perf_event_context *ctx) 7497 { 7498 struct perf_event *parent = event->parent; 7499 7500 if (WARN_ON_ONCE(!parent)) 7501 return; 7502 7503 mutex_lock(&parent->child_mutex); 7504 list_del_init(&event->child_list); 7505 mutex_unlock(&parent->child_mutex); 7506 7507 put_event(parent); 7508 7509 perf_group_detach(event); 7510 list_del_event(event, ctx); 7511 free_event(event); 7512 } 7513 7514 /* 7515 * free an unexposed, unused context as created by inheritance by 7516 * perf_event_init_task below, used by fork() in case of fail. 7517 */ 7518 void perf_event_free_task(struct task_struct *task) 7519 { 7520 struct perf_event_context *ctx; 7521 struct perf_event *event, *tmp; 7522 int ctxn; 7523 7524 for_each_task_context_nr(ctxn) { 7525 ctx = task->perf_event_ctxp[ctxn]; 7526 if (!ctx) 7527 continue; 7528 7529 mutex_lock(&ctx->mutex); 7530 again: 7531 list_for_each_entry_safe(event, tmp, &ctx->pinned_groups, 7532 group_entry) 7533 perf_free_event(event, ctx); 7534 7535 list_for_each_entry_safe(event, tmp, &ctx->flexible_groups, 7536 group_entry) 7537 perf_free_event(event, ctx); 7538 7539 if (!list_empty(&ctx->pinned_groups) || 7540 !list_empty(&ctx->flexible_groups)) 7541 goto again; 7542 7543 mutex_unlock(&ctx->mutex); 7544 7545 put_ctx(ctx); 7546 } 7547 } 7548 7549 void perf_event_delayed_put(struct task_struct *task) 7550 { 7551 int ctxn; 7552 7553 for_each_task_context_nr(ctxn) 7554 WARN_ON_ONCE(task->perf_event_ctxp[ctxn]); 7555 } 7556 7557 /* 7558 * inherit a event from parent task to child task: 7559 */ 7560 static struct perf_event * 7561 inherit_event(struct perf_event *parent_event, 7562 struct task_struct *parent, 7563 struct perf_event_context *parent_ctx, 7564 struct task_struct *child, 7565 struct perf_event *group_leader, 7566 struct perf_event_context *child_ctx) 7567 { 7568 struct perf_event *child_event; 7569 unsigned long flags; 7570 7571 /* 7572 * Instead of creating recursive hierarchies of events, 7573 * we link inherited events back to the original parent, 7574 * which has a filp for sure, which we use as the reference 7575 * count: 7576 */ 7577 if (parent_event->parent) 7578 parent_event = parent_event->parent; 7579 7580 child_event = perf_event_alloc(&parent_event->attr, 7581 parent_event->cpu, 7582 child, 7583 group_leader, parent_event, 7584 NULL, NULL); 7585 if (IS_ERR(child_event)) 7586 return child_event; 7587 7588 if (!atomic_long_inc_not_zero(&parent_event->refcount)) { 7589 free_event(child_event); 7590 return NULL; 7591 } 7592 7593 get_ctx(child_ctx); 7594 7595 /* 7596 * Make the child state follow the state of the parent event, 7597 * not its attr.disabled bit. We hold the parent's mutex, 7598 * so we won't race with perf_event_{en, dis}able_family. 7599 */ 7600 if (parent_event->state >= PERF_EVENT_STATE_INACTIVE) 7601 child_event->state = PERF_EVENT_STATE_INACTIVE; 7602 else 7603 child_event->state = PERF_EVENT_STATE_OFF; 7604 7605 if (parent_event->attr.freq) { 7606 u64 sample_period = parent_event->hw.sample_period; 7607 struct hw_perf_event *hwc = &child_event->hw; 7608 7609 hwc->sample_period = sample_period; 7610 hwc->last_period = sample_period; 7611 7612 local64_set(&hwc->period_left, sample_period); 7613 } 7614 7615 child_event->ctx = child_ctx; 7616 child_event->overflow_handler = parent_event->overflow_handler; 7617 child_event->overflow_handler_context 7618 = parent_event->overflow_handler_context; 7619 7620 /* 7621 * Precalculate sample_data sizes 7622 */ 7623 perf_event__header_size(child_event); 7624 perf_event__id_header_size(child_event); 7625 7626 /* 7627 * Link it up in the child's context: 7628 */ 7629 raw_spin_lock_irqsave(&child_ctx->lock, flags); 7630 add_event_to_ctx(child_event, child_ctx); 7631 raw_spin_unlock_irqrestore(&child_ctx->lock, flags); 7632 7633 /* 7634 * Link this into the parent event's child list 7635 */ 7636 WARN_ON_ONCE(parent_event->ctx->parent_ctx); 7637 mutex_lock(&parent_event->child_mutex); 7638 list_add_tail(&child_event->child_list, &parent_event->child_list); 7639 mutex_unlock(&parent_event->child_mutex); 7640 7641 return child_event; 7642 } 7643 7644 static int inherit_group(struct perf_event *parent_event, 7645 struct task_struct *parent, 7646 struct perf_event_context *parent_ctx, 7647 struct task_struct *child, 7648 struct perf_event_context *child_ctx) 7649 { 7650 struct perf_event *leader; 7651 struct perf_event *sub; 7652 struct perf_event *child_ctr; 7653 7654 leader = inherit_event(parent_event, parent, parent_ctx, 7655 child, NULL, child_ctx); 7656 if (IS_ERR(leader)) 7657 return PTR_ERR(leader); 7658 list_for_each_entry(sub, &parent_event->sibling_list, group_entry) { 7659 child_ctr = inherit_event(sub, parent, parent_ctx, 7660 child, leader, child_ctx); 7661 if (IS_ERR(child_ctr)) 7662 return PTR_ERR(child_ctr); 7663 } 7664 return 0; 7665 } 7666 7667 static int 7668 inherit_task_group(struct perf_event *event, struct task_struct *parent, 7669 struct perf_event_context *parent_ctx, 7670 struct task_struct *child, int ctxn, 7671 int *inherited_all) 7672 { 7673 int ret; 7674 struct perf_event_context *child_ctx; 7675 7676 if (!event->attr.inherit) { 7677 *inherited_all = 0; 7678 return 0; 7679 } 7680 7681 child_ctx = child->perf_event_ctxp[ctxn]; 7682 if (!child_ctx) { 7683 /* 7684 * This is executed from the parent task context, so 7685 * inherit events that have been marked for cloning. 7686 * First allocate and initialize a context for the 7687 * child. 7688 */ 7689 7690 child_ctx = alloc_perf_context(parent_ctx->pmu, child); 7691 if (!child_ctx) 7692 return -ENOMEM; 7693 7694 child->perf_event_ctxp[ctxn] = child_ctx; 7695 } 7696 7697 ret = inherit_group(event, parent, parent_ctx, 7698 child, child_ctx); 7699 7700 if (ret) 7701 *inherited_all = 0; 7702 7703 return ret; 7704 } 7705 7706 /* 7707 * Initialize the perf_event context in task_struct 7708 */ 7709 int perf_event_init_context(struct task_struct *child, int ctxn) 7710 { 7711 struct perf_event_context *child_ctx, *parent_ctx; 7712 struct perf_event_context *cloned_ctx; 7713 struct perf_event *event; 7714 struct task_struct *parent = current; 7715 int inherited_all = 1; 7716 unsigned long flags; 7717 int ret = 0; 7718 7719 if (likely(!parent->perf_event_ctxp[ctxn])) 7720 return 0; 7721 7722 /* 7723 * If the parent's context is a clone, pin it so it won't get 7724 * swapped under us. 7725 */ 7726 parent_ctx = perf_pin_task_context(parent, ctxn); 7727 7728 /* 7729 * No need to check if parent_ctx != NULL here; since we saw 7730 * it non-NULL earlier, the only reason for it to become NULL 7731 * is if we exit, and since we're currently in the middle of 7732 * a fork we can't be exiting at the same time. 7733 */ 7734 7735 /* 7736 * Lock the parent list. No need to lock the child - not PID 7737 * hashed yet and not running, so nobody can access it. 7738 */ 7739 mutex_lock(&parent_ctx->mutex); 7740 7741 /* 7742 * We dont have to disable NMIs - we are only looking at 7743 * the list, not manipulating it: 7744 */ 7745 list_for_each_entry(event, &parent_ctx->pinned_groups, group_entry) { 7746 ret = inherit_task_group(event, parent, parent_ctx, 7747 child, ctxn, &inherited_all); 7748 if (ret) 7749 break; 7750 } 7751 7752 /* 7753 * We can't hold ctx->lock when iterating the ->flexible_group list due 7754 * to allocations, but we need to prevent rotation because 7755 * rotate_ctx() will change the list from interrupt context. 7756 */ 7757 raw_spin_lock_irqsave(&parent_ctx->lock, flags); 7758 parent_ctx->rotate_disable = 1; 7759 raw_spin_unlock_irqrestore(&parent_ctx->lock, flags); 7760 7761 list_for_each_entry(event, &parent_ctx->flexible_groups, group_entry) { 7762 ret = inherit_task_group(event, parent, parent_ctx, 7763 child, ctxn, &inherited_all); 7764 if (ret) 7765 break; 7766 } 7767 7768 raw_spin_lock_irqsave(&parent_ctx->lock, flags); 7769 parent_ctx->rotate_disable = 0; 7770 7771 child_ctx = child->perf_event_ctxp[ctxn]; 7772 7773 if (child_ctx && inherited_all) { 7774 /* 7775 * Mark the child context as a clone of the parent 7776 * context, or of whatever the parent is a clone of. 7777 * 7778 * Note that if the parent is a clone, the holding of 7779 * parent_ctx->lock avoids it from being uncloned. 7780 */ 7781 cloned_ctx = parent_ctx->parent_ctx; 7782 if (cloned_ctx) { 7783 child_ctx->parent_ctx = cloned_ctx; 7784 child_ctx->parent_gen = parent_ctx->parent_gen; 7785 } else { 7786 child_ctx->parent_ctx = parent_ctx; 7787 child_ctx->parent_gen = parent_ctx->generation; 7788 } 7789 get_ctx(child_ctx->parent_ctx); 7790 } 7791 7792 raw_spin_unlock_irqrestore(&parent_ctx->lock, flags); 7793 mutex_unlock(&parent_ctx->mutex); 7794 7795 perf_unpin_context(parent_ctx); 7796 put_ctx(parent_ctx); 7797 7798 return ret; 7799 } 7800 7801 /* 7802 * Initialize the perf_event context in task_struct 7803 */ 7804 int perf_event_init_task(struct task_struct *child) 7805 { 7806 int ctxn, ret; 7807 7808 memset(child->perf_event_ctxp, 0, sizeof(child->perf_event_ctxp)); 7809 mutex_init(&child->perf_event_mutex); 7810 INIT_LIST_HEAD(&child->perf_event_list); 7811 7812 for_each_task_context_nr(ctxn) { 7813 ret = perf_event_init_context(child, ctxn); 7814 if (ret) 7815 return ret; 7816 } 7817 7818 return 0; 7819 } 7820 7821 static void __init perf_event_init_all_cpus(void) 7822 { 7823 struct swevent_htable *swhash; 7824 int cpu; 7825 7826 for_each_possible_cpu(cpu) { 7827 swhash = &per_cpu(swevent_htable, cpu); 7828 mutex_init(&swhash->hlist_mutex); 7829 INIT_LIST_HEAD(&per_cpu(rotation_list, cpu)); 7830 } 7831 } 7832 7833 static void perf_event_init_cpu(int cpu) 7834 { 7835 struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu); 7836 7837 mutex_lock(&swhash->hlist_mutex); 7838 if (swhash->hlist_refcount > 0) { 7839 struct swevent_hlist *hlist; 7840 7841 hlist = kzalloc_node(sizeof(*hlist), GFP_KERNEL, cpu_to_node(cpu)); 7842 WARN_ON(!hlist); 7843 rcu_assign_pointer(swhash->swevent_hlist, hlist); 7844 } 7845 mutex_unlock(&swhash->hlist_mutex); 7846 } 7847 7848 #if defined CONFIG_HOTPLUG_CPU || defined CONFIG_KEXEC 7849 static void perf_pmu_rotate_stop(struct pmu *pmu) 7850 { 7851 struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context); 7852 7853 WARN_ON(!irqs_disabled()); 7854 7855 list_del_init(&cpuctx->rotation_list); 7856 } 7857 7858 static void __perf_event_exit_context(void *__info) 7859 { 7860 struct perf_event_context *ctx = __info; 7861 struct perf_event *event; 7862 7863 perf_pmu_rotate_stop(ctx->pmu); 7864 7865 rcu_read_lock(); 7866 list_for_each_entry_rcu(event, &ctx->event_list, event_entry) 7867 __perf_remove_from_context(event); 7868 rcu_read_unlock(); 7869 } 7870 7871 static void perf_event_exit_cpu_context(int cpu) 7872 { 7873 struct perf_event_context *ctx; 7874 struct pmu *pmu; 7875 int idx; 7876 7877 idx = srcu_read_lock(&pmus_srcu); 7878 list_for_each_entry_rcu(pmu, &pmus, entry) { 7879 ctx = &per_cpu_ptr(pmu->pmu_cpu_context, cpu)->ctx; 7880 7881 mutex_lock(&ctx->mutex); 7882 smp_call_function_single(cpu, __perf_event_exit_context, ctx, 1); 7883 mutex_unlock(&ctx->mutex); 7884 } 7885 srcu_read_unlock(&pmus_srcu, idx); 7886 } 7887 7888 static void perf_event_exit_cpu(int cpu) 7889 { 7890 struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu); 7891 7892 perf_event_exit_cpu_context(cpu); 7893 7894 mutex_lock(&swhash->hlist_mutex); 7895 swevent_hlist_release(swhash); 7896 mutex_unlock(&swhash->hlist_mutex); 7897 } 7898 #else 7899 static inline void perf_event_exit_cpu(int cpu) { } 7900 #endif 7901 7902 static int 7903 perf_reboot(struct notifier_block *notifier, unsigned long val, void *v) 7904 { 7905 int cpu; 7906 7907 for_each_online_cpu(cpu) 7908 perf_event_exit_cpu(cpu); 7909 7910 return NOTIFY_OK; 7911 } 7912 7913 /* 7914 * Run the perf reboot notifier at the very last possible moment so that 7915 * the generic watchdog code runs as long as possible. 7916 */ 7917 static struct notifier_block perf_reboot_notifier = { 7918 .notifier_call = perf_reboot, 7919 .priority = INT_MIN, 7920 }; 7921 7922 static int 7923 perf_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu) 7924 { 7925 unsigned int cpu = (long)hcpu; 7926 7927 switch (action & ~CPU_TASKS_FROZEN) { 7928 7929 case CPU_UP_PREPARE: 7930 case CPU_DOWN_FAILED: 7931 perf_event_init_cpu(cpu); 7932 break; 7933 7934 case CPU_UP_CANCELED: 7935 case CPU_DOWN_PREPARE: 7936 perf_event_exit_cpu(cpu); 7937 break; 7938 default: 7939 break; 7940 } 7941 7942 return NOTIFY_OK; 7943 } 7944 7945 void __init perf_event_init(void) 7946 { 7947 int ret; 7948 7949 idr_init(&pmu_idr); 7950 7951 perf_event_init_all_cpus(); 7952 init_srcu_struct(&pmus_srcu); 7953 perf_pmu_register(&perf_swevent, "software", PERF_TYPE_SOFTWARE); 7954 perf_pmu_register(&perf_cpu_clock, NULL, -1); 7955 perf_pmu_register(&perf_task_clock, NULL, -1); 7956 perf_tp_register(); 7957 perf_cpu_notifier(perf_cpu_notify); 7958 register_reboot_notifier(&perf_reboot_notifier); 7959 7960 ret = init_hw_breakpoint(); 7961 WARN(ret, "hw_breakpoint initialization failed with: %d", ret); 7962 7963 /* do not patch jump label more than once per second */ 7964 jump_label_rate_limit(&perf_sched_events, HZ); 7965 7966 /* 7967 * Build time assertion that we keep the data_head at the intended 7968 * location. IOW, validation we got the __reserved[] size right. 7969 */ 7970 BUILD_BUG_ON((offsetof(struct perf_event_mmap_page, data_head)) 7971 != 1024); 7972 } 7973 7974 static int __init perf_event_sysfs_init(void) 7975 { 7976 struct pmu *pmu; 7977 int ret; 7978 7979 mutex_lock(&pmus_lock); 7980 7981 ret = bus_register(&pmu_bus); 7982 if (ret) 7983 goto unlock; 7984 7985 list_for_each_entry(pmu, &pmus, entry) { 7986 if (!pmu->name || pmu->type < 0) 7987 continue; 7988 7989 ret = pmu_dev_alloc(pmu); 7990 WARN(ret, "Failed to register pmu: %s, reason %d\n", pmu->name, ret); 7991 } 7992 pmu_bus_running = 1; 7993 ret = 0; 7994 7995 unlock: 7996 mutex_unlock(&pmus_lock); 7997 7998 return ret; 7999 } 8000 device_initcall(perf_event_sysfs_init); 8001 8002 #ifdef CONFIG_CGROUP_PERF 8003 static struct cgroup_subsys_state * 8004 perf_cgroup_css_alloc(struct cgroup_subsys_state *parent_css) 8005 { 8006 struct perf_cgroup *jc; 8007 8008 jc = kzalloc(sizeof(*jc), GFP_KERNEL); 8009 if (!jc) 8010 return ERR_PTR(-ENOMEM); 8011 8012 jc->info = alloc_percpu(struct perf_cgroup_info); 8013 if (!jc->info) { 8014 kfree(jc); 8015 return ERR_PTR(-ENOMEM); 8016 } 8017 8018 return &jc->css; 8019 } 8020 8021 static void perf_cgroup_css_free(struct cgroup_subsys_state *css) 8022 { 8023 struct perf_cgroup *jc = container_of(css, struct perf_cgroup, css); 8024 8025 free_percpu(jc->info); 8026 kfree(jc); 8027 } 8028 8029 static int __perf_cgroup_move(void *info) 8030 { 8031 struct task_struct *task = info; 8032 perf_cgroup_switch(task, PERF_CGROUP_SWOUT | PERF_CGROUP_SWIN); 8033 return 0; 8034 } 8035 8036 static void perf_cgroup_attach(struct cgroup_subsys_state *css, 8037 struct cgroup_taskset *tset) 8038 { 8039 struct task_struct *task; 8040 8041 cgroup_taskset_for_each(task, tset) 8042 task_function_call(task, __perf_cgroup_move, task); 8043 } 8044 8045 static void perf_cgroup_exit(struct cgroup_subsys_state *css, 8046 struct cgroup_subsys_state *old_css, 8047 struct task_struct *task) 8048 { 8049 /* 8050 * cgroup_exit() is called in the copy_process() failure path. 8051 * Ignore this case since the task hasn't ran yet, this avoids 8052 * trying to poke a half freed task state from generic code. 8053 */ 8054 if (!(task->flags & PF_EXITING)) 8055 return; 8056 8057 task_function_call(task, __perf_cgroup_move, task); 8058 } 8059 8060 struct cgroup_subsys perf_event_cgrp_subsys = { 8061 .css_alloc = perf_cgroup_css_alloc, 8062 .css_free = perf_cgroup_css_free, 8063 .exit = perf_cgroup_exit, 8064 .attach = perf_cgroup_attach, 8065 }; 8066 #endif /* CONFIG_CGROUP_PERF */ 8067