1 /* 2 * Performance events core code: 3 * 4 * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de> 5 * Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar 6 * Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com> 7 * Copyright © 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com> 8 * 9 * For licensing details see kernel-base/COPYING 10 */ 11 12 #include <linux/fs.h> 13 #include <linux/mm.h> 14 #include <linux/cpu.h> 15 #include <linux/smp.h> 16 #include <linux/idr.h> 17 #include <linux/file.h> 18 #include <linux/poll.h> 19 #include <linux/slab.h> 20 #include <linux/hash.h> 21 #include <linux/tick.h> 22 #include <linux/sysfs.h> 23 #include <linux/dcache.h> 24 #include <linux/percpu.h> 25 #include <linux/ptrace.h> 26 #include <linux/reboot.h> 27 #include <linux/vmstat.h> 28 #include <linux/device.h> 29 #include <linux/export.h> 30 #include <linux/vmalloc.h> 31 #include <linux/hardirq.h> 32 #include <linux/rculist.h> 33 #include <linux/uaccess.h> 34 #include <linux/syscalls.h> 35 #include <linux/anon_inodes.h> 36 #include <linux/kernel_stat.h> 37 #include <linux/perf_event.h> 38 #include <linux/ftrace_event.h> 39 #include <linux/hw_breakpoint.h> 40 #include <linux/mm_types.h> 41 #include <linux/cgroup.h> 42 #include <linux/module.h> 43 #include <linux/mman.h> 44 #include <linux/compat.h> 45 46 #include "internal.h" 47 48 #include <asm/irq_regs.h> 49 50 static struct workqueue_struct *perf_wq; 51 52 struct remote_function_call { 53 struct task_struct *p; 54 int (*func)(void *info); 55 void *info; 56 int ret; 57 }; 58 59 static void remote_function(void *data) 60 { 61 struct remote_function_call *tfc = data; 62 struct task_struct *p = tfc->p; 63 64 if (p) { 65 tfc->ret = -EAGAIN; 66 if (task_cpu(p) != smp_processor_id() || !task_curr(p)) 67 return; 68 } 69 70 tfc->ret = tfc->func(tfc->info); 71 } 72 73 /** 74 * task_function_call - call a function on the cpu on which a task runs 75 * @p: the task to evaluate 76 * @func: the function to be called 77 * @info: the function call argument 78 * 79 * Calls the function @func when the task is currently running. This might 80 * be on the current CPU, which just calls the function directly 81 * 82 * returns: @func return value, or 83 * -ESRCH - when the process isn't running 84 * -EAGAIN - when the process moved away 85 */ 86 static int 87 task_function_call(struct task_struct *p, int (*func) (void *info), void *info) 88 { 89 struct remote_function_call data = { 90 .p = p, 91 .func = func, 92 .info = info, 93 .ret = -ESRCH, /* No such (running) process */ 94 }; 95 96 if (task_curr(p)) 97 smp_call_function_single(task_cpu(p), remote_function, &data, 1); 98 99 return data.ret; 100 } 101 102 /** 103 * cpu_function_call - call a function on the cpu 104 * @func: the function to be called 105 * @info: the function call argument 106 * 107 * Calls the function @func on the remote cpu. 108 * 109 * returns: @func return value or -ENXIO when the cpu is offline 110 */ 111 static int cpu_function_call(int cpu, int (*func) (void *info), void *info) 112 { 113 struct remote_function_call data = { 114 .p = NULL, 115 .func = func, 116 .info = info, 117 .ret = -ENXIO, /* No such CPU */ 118 }; 119 120 smp_call_function_single(cpu, remote_function, &data, 1); 121 122 return data.ret; 123 } 124 125 #define EVENT_OWNER_KERNEL ((void *) -1) 126 127 static bool is_kernel_event(struct perf_event *event) 128 { 129 return event->owner == EVENT_OWNER_KERNEL; 130 } 131 132 #define PERF_FLAG_ALL (PERF_FLAG_FD_NO_GROUP |\ 133 PERF_FLAG_FD_OUTPUT |\ 134 PERF_FLAG_PID_CGROUP |\ 135 PERF_FLAG_FD_CLOEXEC) 136 137 /* 138 * branch priv levels that need permission checks 139 */ 140 #define PERF_SAMPLE_BRANCH_PERM_PLM \ 141 (PERF_SAMPLE_BRANCH_KERNEL |\ 142 PERF_SAMPLE_BRANCH_HV) 143 144 enum event_type_t { 145 EVENT_FLEXIBLE = 0x1, 146 EVENT_PINNED = 0x2, 147 EVENT_ALL = EVENT_FLEXIBLE | EVENT_PINNED, 148 }; 149 150 /* 151 * perf_sched_events : >0 events exist 152 * perf_cgroup_events: >0 per-cpu cgroup events exist on this cpu 153 */ 154 struct static_key_deferred perf_sched_events __read_mostly; 155 static DEFINE_PER_CPU(atomic_t, perf_cgroup_events); 156 static DEFINE_PER_CPU(atomic_t, perf_branch_stack_events); 157 158 static atomic_t nr_mmap_events __read_mostly; 159 static atomic_t nr_comm_events __read_mostly; 160 static atomic_t nr_task_events __read_mostly; 161 static atomic_t nr_freq_events __read_mostly; 162 163 static LIST_HEAD(pmus); 164 static DEFINE_MUTEX(pmus_lock); 165 static struct srcu_struct pmus_srcu; 166 167 /* 168 * perf event paranoia level: 169 * -1 - not paranoid at all 170 * 0 - disallow raw tracepoint access for unpriv 171 * 1 - disallow cpu events for unpriv 172 * 2 - disallow kernel profiling for unpriv 173 */ 174 int sysctl_perf_event_paranoid __read_mostly = 1; 175 176 /* Minimum for 512 kiB + 1 user control page */ 177 int sysctl_perf_event_mlock __read_mostly = 512 + (PAGE_SIZE / 1024); /* 'free' kiB per user */ 178 179 /* 180 * max perf event sample rate 181 */ 182 #define DEFAULT_MAX_SAMPLE_RATE 100000 183 #define DEFAULT_SAMPLE_PERIOD_NS (NSEC_PER_SEC / DEFAULT_MAX_SAMPLE_RATE) 184 #define DEFAULT_CPU_TIME_MAX_PERCENT 25 185 186 int sysctl_perf_event_sample_rate __read_mostly = DEFAULT_MAX_SAMPLE_RATE; 187 188 static int max_samples_per_tick __read_mostly = DIV_ROUND_UP(DEFAULT_MAX_SAMPLE_RATE, HZ); 189 static int perf_sample_period_ns __read_mostly = DEFAULT_SAMPLE_PERIOD_NS; 190 191 static int perf_sample_allowed_ns __read_mostly = 192 DEFAULT_SAMPLE_PERIOD_NS * DEFAULT_CPU_TIME_MAX_PERCENT / 100; 193 194 void update_perf_cpu_limits(void) 195 { 196 u64 tmp = perf_sample_period_ns; 197 198 tmp *= sysctl_perf_cpu_time_max_percent; 199 do_div(tmp, 100); 200 ACCESS_ONCE(perf_sample_allowed_ns) = tmp; 201 } 202 203 static int perf_rotate_context(struct perf_cpu_context *cpuctx); 204 205 int perf_proc_update_handler(struct ctl_table *table, int write, 206 void __user *buffer, size_t *lenp, 207 loff_t *ppos) 208 { 209 int ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); 210 211 if (ret || !write) 212 return ret; 213 214 max_samples_per_tick = DIV_ROUND_UP(sysctl_perf_event_sample_rate, HZ); 215 perf_sample_period_ns = NSEC_PER_SEC / sysctl_perf_event_sample_rate; 216 update_perf_cpu_limits(); 217 218 return 0; 219 } 220 221 int sysctl_perf_cpu_time_max_percent __read_mostly = DEFAULT_CPU_TIME_MAX_PERCENT; 222 223 int perf_cpu_time_max_percent_handler(struct ctl_table *table, int write, 224 void __user *buffer, size_t *lenp, 225 loff_t *ppos) 226 { 227 int ret = proc_dointvec(table, write, buffer, lenp, ppos); 228 229 if (ret || !write) 230 return ret; 231 232 update_perf_cpu_limits(); 233 234 return 0; 235 } 236 237 /* 238 * perf samples are done in some very critical code paths (NMIs). 239 * If they take too much CPU time, the system can lock up and not 240 * get any real work done. This will drop the sample rate when 241 * we detect that events are taking too long. 242 */ 243 #define NR_ACCUMULATED_SAMPLES 128 244 static DEFINE_PER_CPU(u64, running_sample_length); 245 246 static void perf_duration_warn(struct irq_work *w) 247 { 248 u64 allowed_ns = ACCESS_ONCE(perf_sample_allowed_ns); 249 u64 avg_local_sample_len; 250 u64 local_samples_len; 251 252 local_samples_len = __this_cpu_read(running_sample_length); 253 avg_local_sample_len = local_samples_len/NR_ACCUMULATED_SAMPLES; 254 255 printk_ratelimited(KERN_WARNING 256 "perf interrupt took too long (%lld > %lld), lowering " 257 "kernel.perf_event_max_sample_rate to %d\n", 258 avg_local_sample_len, allowed_ns >> 1, 259 sysctl_perf_event_sample_rate); 260 } 261 262 static DEFINE_IRQ_WORK(perf_duration_work, perf_duration_warn); 263 264 void perf_sample_event_took(u64 sample_len_ns) 265 { 266 u64 allowed_ns = ACCESS_ONCE(perf_sample_allowed_ns); 267 u64 avg_local_sample_len; 268 u64 local_samples_len; 269 270 if (allowed_ns == 0) 271 return; 272 273 /* decay the counter by 1 average sample */ 274 local_samples_len = __this_cpu_read(running_sample_length); 275 local_samples_len -= local_samples_len/NR_ACCUMULATED_SAMPLES; 276 local_samples_len += sample_len_ns; 277 __this_cpu_write(running_sample_length, local_samples_len); 278 279 /* 280 * note: this will be biased artifically low until we have 281 * seen NR_ACCUMULATED_SAMPLES. Doing it this way keeps us 282 * from having to maintain a count. 283 */ 284 avg_local_sample_len = local_samples_len/NR_ACCUMULATED_SAMPLES; 285 286 if (avg_local_sample_len <= allowed_ns) 287 return; 288 289 if (max_samples_per_tick <= 1) 290 return; 291 292 max_samples_per_tick = DIV_ROUND_UP(max_samples_per_tick, 2); 293 sysctl_perf_event_sample_rate = max_samples_per_tick * HZ; 294 perf_sample_period_ns = NSEC_PER_SEC / sysctl_perf_event_sample_rate; 295 296 update_perf_cpu_limits(); 297 298 if (!irq_work_queue(&perf_duration_work)) { 299 early_printk("perf interrupt took too long (%lld > %lld), lowering " 300 "kernel.perf_event_max_sample_rate to %d\n", 301 avg_local_sample_len, allowed_ns >> 1, 302 sysctl_perf_event_sample_rate); 303 } 304 } 305 306 static atomic64_t perf_event_id; 307 308 static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx, 309 enum event_type_t event_type); 310 311 static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx, 312 enum event_type_t event_type, 313 struct task_struct *task); 314 315 static void update_context_time(struct perf_event_context *ctx); 316 static u64 perf_event_time(struct perf_event *event); 317 318 void __weak perf_event_print_debug(void) { } 319 320 extern __weak const char *perf_pmu_name(void) 321 { 322 return "pmu"; 323 } 324 325 static inline u64 perf_clock(void) 326 { 327 return local_clock(); 328 } 329 330 static inline struct perf_cpu_context * 331 __get_cpu_context(struct perf_event_context *ctx) 332 { 333 return this_cpu_ptr(ctx->pmu->pmu_cpu_context); 334 } 335 336 static void perf_ctx_lock(struct perf_cpu_context *cpuctx, 337 struct perf_event_context *ctx) 338 { 339 raw_spin_lock(&cpuctx->ctx.lock); 340 if (ctx) 341 raw_spin_lock(&ctx->lock); 342 } 343 344 static void perf_ctx_unlock(struct perf_cpu_context *cpuctx, 345 struct perf_event_context *ctx) 346 { 347 if (ctx) 348 raw_spin_unlock(&ctx->lock); 349 raw_spin_unlock(&cpuctx->ctx.lock); 350 } 351 352 #ifdef CONFIG_CGROUP_PERF 353 354 /* 355 * perf_cgroup_info keeps track of time_enabled for a cgroup. 356 * This is a per-cpu dynamically allocated data structure. 357 */ 358 struct perf_cgroup_info { 359 u64 time; 360 u64 timestamp; 361 }; 362 363 struct perf_cgroup { 364 struct cgroup_subsys_state css; 365 struct perf_cgroup_info __percpu *info; 366 }; 367 368 /* 369 * Must ensure cgroup is pinned (css_get) before calling 370 * this function. In other words, we cannot call this function 371 * if there is no cgroup event for the current CPU context. 372 */ 373 static inline struct perf_cgroup * 374 perf_cgroup_from_task(struct task_struct *task) 375 { 376 return container_of(task_css(task, perf_event_cgrp_id), 377 struct perf_cgroup, css); 378 } 379 380 static inline bool 381 perf_cgroup_match(struct perf_event *event) 382 { 383 struct perf_event_context *ctx = event->ctx; 384 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx); 385 386 /* @event doesn't care about cgroup */ 387 if (!event->cgrp) 388 return true; 389 390 /* wants specific cgroup scope but @cpuctx isn't associated with any */ 391 if (!cpuctx->cgrp) 392 return false; 393 394 /* 395 * Cgroup scoping is recursive. An event enabled for a cgroup is 396 * also enabled for all its descendant cgroups. If @cpuctx's 397 * cgroup is a descendant of @event's (the test covers identity 398 * case), it's a match. 399 */ 400 return cgroup_is_descendant(cpuctx->cgrp->css.cgroup, 401 event->cgrp->css.cgroup); 402 } 403 404 static inline void perf_detach_cgroup(struct perf_event *event) 405 { 406 css_put(&event->cgrp->css); 407 event->cgrp = NULL; 408 } 409 410 static inline int is_cgroup_event(struct perf_event *event) 411 { 412 return event->cgrp != NULL; 413 } 414 415 static inline u64 perf_cgroup_event_time(struct perf_event *event) 416 { 417 struct perf_cgroup_info *t; 418 419 t = per_cpu_ptr(event->cgrp->info, event->cpu); 420 return t->time; 421 } 422 423 static inline void __update_cgrp_time(struct perf_cgroup *cgrp) 424 { 425 struct perf_cgroup_info *info; 426 u64 now; 427 428 now = perf_clock(); 429 430 info = this_cpu_ptr(cgrp->info); 431 432 info->time += now - info->timestamp; 433 info->timestamp = now; 434 } 435 436 static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx) 437 { 438 struct perf_cgroup *cgrp_out = cpuctx->cgrp; 439 if (cgrp_out) 440 __update_cgrp_time(cgrp_out); 441 } 442 443 static inline void update_cgrp_time_from_event(struct perf_event *event) 444 { 445 struct perf_cgroup *cgrp; 446 447 /* 448 * ensure we access cgroup data only when needed and 449 * when we know the cgroup is pinned (css_get) 450 */ 451 if (!is_cgroup_event(event)) 452 return; 453 454 cgrp = perf_cgroup_from_task(current); 455 /* 456 * Do not update time when cgroup is not active 457 */ 458 if (cgrp == event->cgrp) 459 __update_cgrp_time(event->cgrp); 460 } 461 462 static inline void 463 perf_cgroup_set_timestamp(struct task_struct *task, 464 struct perf_event_context *ctx) 465 { 466 struct perf_cgroup *cgrp; 467 struct perf_cgroup_info *info; 468 469 /* 470 * ctx->lock held by caller 471 * ensure we do not access cgroup data 472 * unless we have the cgroup pinned (css_get) 473 */ 474 if (!task || !ctx->nr_cgroups) 475 return; 476 477 cgrp = perf_cgroup_from_task(task); 478 info = this_cpu_ptr(cgrp->info); 479 info->timestamp = ctx->timestamp; 480 } 481 482 #define PERF_CGROUP_SWOUT 0x1 /* cgroup switch out every event */ 483 #define PERF_CGROUP_SWIN 0x2 /* cgroup switch in events based on task */ 484 485 /* 486 * reschedule events based on the cgroup constraint of task. 487 * 488 * mode SWOUT : schedule out everything 489 * mode SWIN : schedule in based on cgroup for next 490 */ 491 void perf_cgroup_switch(struct task_struct *task, int mode) 492 { 493 struct perf_cpu_context *cpuctx; 494 struct pmu *pmu; 495 unsigned long flags; 496 497 /* 498 * disable interrupts to avoid geting nr_cgroup 499 * changes via __perf_event_disable(). Also 500 * avoids preemption. 501 */ 502 local_irq_save(flags); 503 504 /* 505 * we reschedule only in the presence of cgroup 506 * constrained events. 507 */ 508 rcu_read_lock(); 509 510 list_for_each_entry_rcu(pmu, &pmus, entry) { 511 cpuctx = this_cpu_ptr(pmu->pmu_cpu_context); 512 if (cpuctx->unique_pmu != pmu) 513 continue; /* ensure we process each cpuctx once */ 514 515 /* 516 * perf_cgroup_events says at least one 517 * context on this CPU has cgroup events. 518 * 519 * ctx->nr_cgroups reports the number of cgroup 520 * events for a context. 521 */ 522 if (cpuctx->ctx.nr_cgroups > 0) { 523 perf_ctx_lock(cpuctx, cpuctx->task_ctx); 524 perf_pmu_disable(cpuctx->ctx.pmu); 525 526 if (mode & PERF_CGROUP_SWOUT) { 527 cpu_ctx_sched_out(cpuctx, EVENT_ALL); 528 /* 529 * must not be done before ctxswout due 530 * to event_filter_match() in event_sched_out() 531 */ 532 cpuctx->cgrp = NULL; 533 } 534 535 if (mode & PERF_CGROUP_SWIN) { 536 WARN_ON_ONCE(cpuctx->cgrp); 537 /* 538 * set cgrp before ctxsw in to allow 539 * event_filter_match() to not have to pass 540 * task around 541 */ 542 cpuctx->cgrp = perf_cgroup_from_task(task); 543 cpu_ctx_sched_in(cpuctx, EVENT_ALL, task); 544 } 545 perf_pmu_enable(cpuctx->ctx.pmu); 546 perf_ctx_unlock(cpuctx, cpuctx->task_ctx); 547 } 548 } 549 550 rcu_read_unlock(); 551 552 local_irq_restore(flags); 553 } 554 555 static inline void perf_cgroup_sched_out(struct task_struct *task, 556 struct task_struct *next) 557 { 558 struct perf_cgroup *cgrp1; 559 struct perf_cgroup *cgrp2 = NULL; 560 561 /* 562 * we come here when we know perf_cgroup_events > 0 563 */ 564 cgrp1 = perf_cgroup_from_task(task); 565 566 /* 567 * next is NULL when called from perf_event_enable_on_exec() 568 * that will systematically cause a cgroup_switch() 569 */ 570 if (next) 571 cgrp2 = perf_cgroup_from_task(next); 572 573 /* 574 * only schedule out current cgroup events if we know 575 * that we are switching to a different cgroup. Otherwise, 576 * do no touch the cgroup events. 577 */ 578 if (cgrp1 != cgrp2) 579 perf_cgroup_switch(task, PERF_CGROUP_SWOUT); 580 } 581 582 static inline void perf_cgroup_sched_in(struct task_struct *prev, 583 struct task_struct *task) 584 { 585 struct perf_cgroup *cgrp1; 586 struct perf_cgroup *cgrp2 = NULL; 587 588 /* 589 * we come here when we know perf_cgroup_events > 0 590 */ 591 cgrp1 = perf_cgroup_from_task(task); 592 593 /* prev can never be NULL */ 594 cgrp2 = perf_cgroup_from_task(prev); 595 596 /* 597 * only need to schedule in cgroup events if we are changing 598 * cgroup during ctxsw. Cgroup events were not scheduled 599 * out of ctxsw out if that was not the case. 600 */ 601 if (cgrp1 != cgrp2) 602 perf_cgroup_switch(task, PERF_CGROUP_SWIN); 603 } 604 605 static inline int perf_cgroup_connect(int fd, struct perf_event *event, 606 struct perf_event_attr *attr, 607 struct perf_event *group_leader) 608 { 609 struct perf_cgroup *cgrp; 610 struct cgroup_subsys_state *css; 611 struct fd f = fdget(fd); 612 int ret = 0; 613 614 if (!f.file) 615 return -EBADF; 616 617 css = css_tryget_online_from_dir(f.file->f_path.dentry, 618 &perf_event_cgrp_subsys); 619 if (IS_ERR(css)) { 620 ret = PTR_ERR(css); 621 goto out; 622 } 623 624 cgrp = container_of(css, struct perf_cgroup, css); 625 event->cgrp = cgrp; 626 627 /* 628 * all events in a group must monitor 629 * the same cgroup because a task belongs 630 * to only one perf cgroup at a time 631 */ 632 if (group_leader && group_leader->cgrp != cgrp) { 633 perf_detach_cgroup(event); 634 ret = -EINVAL; 635 } 636 out: 637 fdput(f); 638 return ret; 639 } 640 641 static inline void 642 perf_cgroup_set_shadow_time(struct perf_event *event, u64 now) 643 { 644 struct perf_cgroup_info *t; 645 t = per_cpu_ptr(event->cgrp->info, event->cpu); 646 event->shadow_ctx_time = now - t->timestamp; 647 } 648 649 static inline void 650 perf_cgroup_defer_enabled(struct perf_event *event) 651 { 652 /* 653 * when the current task's perf cgroup does not match 654 * the event's, we need to remember to call the 655 * perf_mark_enable() function the first time a task with 656 * a matching perf cgroup is scheduled in. 657 */ 658 if (is_cgroup_event(event) && !perf_cgroup_match(event)) 659 event->cgrp_defer_enabled = 1; 660 } 661 662 static inline void 663 perf_cgroup_mark_enabled(struct perf_event *event, 664 struct perf_event_context *ctx) 665 { 666 struct perf_event *sub; 667 u64 tstamp = perf_event_time(event); 668 669 if (!event->cgrp_defer_enabled) 670 return; 671 672 event->cgrp_defer_enabled = 0; 673 674 event->tstamp_enabled = tstamp - event->total_time_enabled; 675 list_for_each_entry(sub, &event->sibling_list, group_entry) { 676 if (sub->state >= PERF_EVENT_STATE_INACTIVE) { 677 sub->tstamp_enabled = tstamp - sub->total_time_enabled; 678 sub->cgrp_defer_enabled = 0; 679 } 680 } 681 } 682 #else /* !CONFIG_CGROUP_PERF */ 683 684 static inline bool 685 perf_cgroup_match(struct perf_event *event) 686 { 687 return true; 688 } 689 690 static inline void perf_detach_cgroup(struct perf_event *event) 691 {} 692 693 static inline int is_cgroup_event(struct perf_event *event) 694 { 695 return 0; 696 } 697 698 static inline u64 perf_cgroup_event_cgrp_time(struct perf_event *event) 699 { 700 return 0; 701 } 702 703 static inline void update_cgrp_time_from_event(struct perf_event *event) 704 { 705 } 706 707 static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx) 708 { 709 } 710 711 static inline void perf_cgroup_sched_out(struct task_struct *task, 712 struct task_struct *next) 713 { 714 } 715 716 static inline void perf_cgroup_sched_in(struct task_struct *prev, 717 struct task_struct *task) 718 { 719 } 720 721 static inline int perf_cgroup_connect(pid_t pid, struct perf_event *event, 722 struct perf_event_attr *attr, 723 struct perf_event *group_leader) 724 { 725 return -EINVAL; 726 } 727 728 static inline void 729 perf_cgroup_set_timestamp(struct task_struct *task, 730 struct perf_event_context *ctx) 731 { 732 } 733 734 void 735 perf_cgroup_switch(struct task_struct *task, struct task_struct *next) 736 { 737 } 738 739 static inline void 740 perf_cgroup_set_shadow_time(struct perf_event *event, u64 now) 741 { 742 } 743 744 static inline u64 perf_cgroup_event_time(struct perf_event *event) 745 { 746 return 0; 747 } 748 749 static inline void 750 perf_cgroup_defer_enabled(struct perf_event *event) 751 { 752 } 753 754 static inline void 755 perf_cgroup_mark_enabled(struct perf_event *event, 756 struct perf_event_context *ctx) 757 { 758 } 759 #endif 760 761 /* 762 * set default to be dependent on timer tick just 763 * like original code 764 */ 765 #define PERF_CPU_HRTIMER (1000 / HZ) 766 /* 767 * function must be called with interrupts disbled 768 */ 769 static enum hrtimer_restart perf_cpu_hrtimer_handler(struct hrtimer *hr) 770 { 771 struct perf_cpu_context *cpuctx; 772 enum hrtimer_restart ret = HRTIMER_NORESTART; 773 int rotations = 0; 774 775 WARN_ON(!irqs_disabled()); 776 777 cpuctx = container_of(hr, struct perf_cpu_context, hrtimer); 778 779 rotations = perf_rotate_context(cpuctx); 780 781 /* 782 * arm timer if needed 783 */ 784 if (rotations) { 785 hrtimer_forward_now(hr, cpuctx->hrtimer_interval); 786 ret = HRTIMER_RESTART; 787 } 788 789 return ret; 790 } 791 792 /* CPU is going down */ 793 void perf_cpu_hrtimer_cancel(int cpu) 794 { 795 struct perf_cpu_context *cpuctx; 796 struct pmu *pmu; 797 unsigned long flags; 798 799 if (WARN_ON(cpu != smp_processor_id())) 800 return; 801 802 local_irq_save(flags); 803 804 rcu_read_lock(); 805 806 list_for_each_entry_rcu(pmu, &pmus, entry) { 807 cpuctx = this_cpu_ptr(pmu->pmu_cpu_context); 808 809 if (pmu->task_ctx_nr == perf_sw_context) 810 continue; 811 812 hrtimer_cancel(&cpuctx->hrtimer); 813 } 814 815 rcu_read_unlock(); 816 817 local_irq_restore(flags); 818 } 819 820 static void __perf_cpu_hrtimer_init(struct perf_cpu_context *cpuctx, int cpu) 821 { 822 struct hrtimer *hr = &cpuctx->hrtimer; 823 struct pmu *pmu = cpuctx->ctx.pmu; 824 int timer; 825 826 /* no multiplexing needed for SW PMU */ 827 if (pmu->task_ctx_nr == perf_sw_context) 828 return; 829 830 /* 831 * check default is sane, if not set then force to 832 * default interval (1/tick) 833 */ 834 timer = pmu->hrtimer_interval_ms; 835 if (timer < 1) 836 timer = pmu->hrtimer_interval_ms = PERF_CPU_HRTIMER; 837 838 cpuctx->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * timer); 839 840 hrtimer_init(hr, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED); 841 hr->function = perf_cpu_hrtimer_handler; 842 } 843 844 static void perf_cpu_hrtimer_restart(struct perf_cpu_context *cpuctx) 845 { 846 struct hrtimer *hr = &cpuctx->hrtimer; 847 struct pmu *pmu = cpuctx->ctx.pmu; 848 849 /* not for SW PMU */ 850 if (pmu->task_ctx_nr == perf_sw_context) 851 return; 852 853 if (hrtimer_active(hr)) 854 return; 855 856 if (!hrtimer_callback_running(hr)) 857 __hrtimer_start_range_ns(hr, cpuctx->hrtimer_interval, 858 0, HRTIMER_MODE_REL_PINNED, 0); 859 } 860 861 void perf_pmu_disable(struct pmu *pmu) 862 { 863 int *count = this_cpu_ptr(pmu->pmu_disable_count); 864 if (!(*count)++) 865 pmu->pmu_disable(pmu); 866 } 867 868 void perf_pmu_enable(struct pmu *pmu) 869 { 870 int *count = this_cpu_ptr(pmu->pmu_disable_count); 871 if (!--(*count)) 872 pmu->pmu_enable(pmu); 873 } 874 875 static DEFINE_PER_CPU(struct list_head, active_ctx_list); 876 877 /* 878 * perf_event_ctx_activate(), perf_event_ctx_deactivate(), and 879 * perf_event_task_tick() are fully serialized because they're strictly cpu 880 * affine and perf_event_ctx{activate,deactivate} are called with IRQs 881 * disabled, while perf_event_task_tick is called from IRQ context. 882 */ 883 static void perf_event_ctx_activate(struct perf_event_context *ctx) 884 { 885 struct list_head *head = this_cpu_ptr(&active_ctx_list); 886 887 WARN_ON(!irqs_disabled()); 888 889 WARN_ON(!list_empty(&ctx->active_ctx_list)); 890 891 list_add(&ctx->active_ctx_list, head); 892 } 893 894 static void perf_event_ctx_deactivate(struct perf_event_context *ctx) 895 { 896 WARN_ON(!irqs_disabled()); 897 898 WARN_ON(list_empty(&ctx->active_ctx_list)); 899 900 list_del_init(&ctx->active_ctx_list); 901 } 902 903 static void get_ctx(struct perf_event_context *ctx) 904 { 905 WARN_ON(!atomic_inc_not_zero(&ctx->refcount)); 906 } 907 908 static void put_ctx(struct perf_event_context *ctx) 909 { 910 if (atomic_dec_and_test(&ctx->refcount)) { 911 if (ctx->parent_ctx) 912 put_ctx(ctx->parent_ctx); 913 if (ctx->task) 914 put_task_struct(ctx->task); 915 kfree_rcu(ctx, rcu_head); 916 } 917 } 918 919 /* 920 * Because of perf_event::ctx migration in sys_perf_event_open::move_group and 921 * perf_pmu_migrate_context() we need some magic. 922 * 923 * Those places that change perf_event::ctx will hold both 924 * perf_event_ctx::mutex of the 'old' and 'new' ctx value. 925 * 926 * Lock ordering is by mutex address. There is one other site where 927 * perf_event_context::mutex nests and that is put_event(). But remember that 928 * that is a parent<->child context relation, and migration does not affect 929 * children, therefore these two orderings should not interact. 930 * 931 * The change in perf_event::ctx does not affect children (as claimed above) 932 * because the sys_perf_event_open() case will install a new event and break 933 * the ctx parent<->child relation, and perf_pmu_migrate_context() is only 934 * concerned with cpuctx and that doesn't have children. 935 * 936 * The places that change perf_event::ctx will issue: 937 * 938 * perf_remove_from_context(); 939 * synchronize_rcu(); 940 * perf_install_in_context(); 941 * 942 * to affect the change. The remove_from_context() + synchronize_rcu() should 943 * quiesce the event, after which we can install it in the new location. This 944 * means that only external vectors (perf_fops, prctl) can perturb the event 945 * while in transit. Therefore all such accessors should also acquire 946 * perf_event_context::mutex to serialize against this. 947 * 948 * However; because event->ctx can change while we're waiting to acquire 949 * ctx->mutex we must be careful and use the below perf_event_ctx_lock() 950 * function. 951 * 952 * Lock order: 953 * task_struct::perf_event_mutex 954 * perf_event_context::mutex 955 * perf_event_context::lock 956 * perf_event::child_mutex; 957 * perf_event::mmap_mutex 958 * mmap_sem 959 */ 960 static struct perf_event_context * 961 perf_event_ctx_lock_nested(struct perf_event *event, int nesting) 962 { 963 struct perf_event_context *ctx; 964 965 again: 966 rcu_read_lock(); 967 ctx = ACCESS_ONCE(event->ctx); 968 if (!atomic_inc_not_zero(&ctx->refcount)) { 969 rcu_read_unlock(); 970 goto again; 971 } 972 rcu_read_unlock(); 973 974 mutex_lock_nested(&ctx->mutex, nesting); 975 if (event->ctx != ctx) { 976 mutex_unlock(&ctx->mutex); 977 put_ctx(ctx); 978 goto again; 979 } 980 981 return ctx; 982 } 983 984 static inline struct perf_event_context * 985 perf_event_ctx_lock(struct perf_event *event) 986 { 987 return perf_event_ctx_lock_nested(event, 0); 988 } 989 990 static void perf_event_ctx_unlock(struct perf_event *event, 991 struct perf_event_context *ctx) 992 { 993 mutex_unlock(&ctx->mutex); 994 put_ctx(ctx); 995 } 996 997 /* 998 * This must be done under the ctx->lock, such as to serialize against 999 * context_equiv(), therefore we cannot call put_ctx() since that might end up 1000 * calling scheduler related locks and ctx->lock nests inside those. 1001 */ 1002 static __must_check struct perf_event_context * 1003 unclone_ctx(struct perf_event_context *ctx) 1004 { 1005 struct perf_event_context *parent_ctx = ctx->parent_ctx; 1006 1007 lockdep_assert_held(&ctx->lock); 1008 1009 if (parent_ctx) 1010 ctx->parent_ctx = NULL; 1011 ctx->generation++; 1012 1013 return parent_ctx; 1014 } 1015 1016 static u32 perf_event_pid(struct perf_event *event, struct task_struct *p) 1017 { 1018 /* 1019 * only top level events have the pid namespace they were created in 1020 */ 1021 if (event->parent) 1022 event = event->parent; 1023 1024 return task_tgid_nr_ns(p, event->ns); 1025 } 1026 1027 static u32 perf_event_tid(struct perf_event *event, struct task_struct *p) 1028 { 1029 /* 1030 * only top level events have the pid namespace they were created in 1031 */ 1032 if (event->parent) 1033 event = event->parent; 1034 1035 return task_pid_nr_ns(p, event->ns); 1036 } 1037 1038 /* 1039 * If we inherit events we want to return the parent event id 1040 * to userspace. 1041 */ 1042 static u64 primary_event_id(struct perf_event *event) 1043 { 1044 u64 id = event->id; 1045 1046 if (event->parent) 1047 id = event->parent->id; 1048 1049 return id; 1050 } 1051 1052 /* 1053 * Get the perf_event_context for a task and lock it. 1054 * This has to cope with with the fact that until it is locked, 1055 * the context could get moved to another task. 1056 */ 1057 static struct perf_event_context * 1058 perf_lock_task_context(struct task_struct *task, int ctxn, unsigned long *flags) 1059 { 1060 struct perf_event_context *ctx; 1061 1062 retry: 1063 /* 1064 * One of the few rules of preemptible RCU is that one cannot do 1065 * rcu_read_unlock() while holding a scheduler (or nested) lock when 1066 * part of the read side critical section was preemptible -- see 1067 * rcu_read_unlock_special(). 1068 * 1069 * Since ctx->lock nests under rq->lock we must ensure the entire read 1070 * side critical section is non-preemptible. 1071 */ 1072 preempt_disable(); 1073 rcu_read_lock(); 1074 ctx = rcu_dereference(task->perf_event_ctxp[ctxn]); 1075 if (ctx) { 1076 /* 1077 * If this context is a clone of another, it might 1078 * get swapped for another underneath us by 1079 * perf_event_task_sched_out, though the 1080 * rcu_read_lock() protects us from any context 1081 * getting freed. Lock the context and check if it 1082 * got swapped before we could get the lock, and retry 1083 * if so. If we locked the right context, then it 1084 * can't get swapped on us any more. 1085 */ 1086 raw_spin_lock_irqsave(&ctx->lock, *flags); 1087 if (ctx != rcu_dereference(task->perf_event_ctxp[ctxn])) { 1088 raw_spin_unlock_irqrestore(&ctx->lock, *flags); 1089 rcu_read_unlock(); 1090 preempt_enable(); 1091 goto retry; 1092 } 1093 1094 if (!atomic_inc_not_zero(&ctx->refcount)) { 1095 raw_spin_unlock_irqrestore(&ctx->lock, *flags); 1096 ctx = NULL; 1097 } 1098 } 1099 rcu_read_unlock(); 1100 preempt_enable(); 1101 return ctx; 1102 } 1103 1104 /* 1105 * Get the context for a task and increment its pin_count so it 1106 * can't get swapped to another task. This also increments its 1107 * reference count so that the context can't get freed. 1108 */ 1109 static struct perf_event_context * 1110 perf_pin_task_context(struct task_struct *task, int ctxn) 1111 { 1112 struct perf_event_context *ctx; 1113 unsigned long flags; 1114 1115 ctx = perf_lock_task_context(task, ctxn, &flags); 1116 if (ctx) { 1117 ++ctx->pin_count; 1118 raw_spin_unlock_irqrestore(&ctx->lock, flags); 1119 } 1120 return ctx; 1121 } 1122 1123 static void perf_unpin_context(struct perf_event_context *ctx) 1124 { 1125 unsigned long flags; 1126 1127 raw_spin_lock_irqsave(&ctx->lock, flags); 1128 --ctx->pin_count; 1129 raw_spin_unlock_irqrestore(&ctx->lock, flags); 1130 } 1131 1132 /* 1133 * Update the record of the current time in a context. 1134 */ 1135 static void update_context_time(struct perf_event_context *ctx) 1136 { 1137 u64 now = perf_clock(); 1138 1139 ctx->time += now - ctx->timestamp; 1140 ctx->timestamp = now; 1141 } 1142 1143 static u64 perf_event_time(struct perf_event *event) 1144 { 1145 struct perf_event_context *ctx = event->ctx; 1146 1147 if (is_cgroup_event(event)) 1148 return perf_cgroup_event_time(event); 1149 1150 return ctx ? ctx->time : 0; 1151 } 1152 1153 /* 1154 * Update the total_time_enabled and total_time_running fields for a event. 1155 * The caller of this function needs to hold the ctx->lock. 1156 */ 1157 static void update_event_times(struct perf_event *event) 1158 { 1159 struct perf_event_context *ctx = event->ctx; 1160 u64 run_end; 1161 1162 if (event->state < PERF_EVENT_STATE_INACTIVE || 1163 event->group_leader->state < PERF_EVENT_STATE_INACTIVE) 1164 return; 1165 /* 1166 * in cgroup mode, time_enabled represents 1167 * the time the event was enabled AND active 1168 * tasks were in the monitored cgroup. This is 1169 * independent of the activity of the context as 1170 * there may be a mix of cgroup and non-cgroup events. 1171 * 1172 * That is why we treat cgroup events differently 1173 * here. 1174 */ 1175 if (is_cgroup_event(event)) 1176 run_end = perf_cgroup_event_time(event); 1177 else if (ctx->is_active) 1178 run_end = ctx->time; 1179 else 1180 run_end = event->tstamp_stopped; 1181 1182 event->total_time_enabled = run_end - event->tstamp_enabled; 1183 1184 if (event->state == PERF_EVENT_STATE_INACTIVE) 1185 run_end = event->tstamp_stopped; 1186 else 1187 run_end = perf_event_time(event); 1188 1189 event->total_time_running = run_end - event->tstamp_running; 1190 1191 } 1192 1193 /* 1194 * Update total_time_enabled and total_time_running for all events in a group. 1195 */ 1196 static void update_group_times(struct perf_event *leader) 1197 { 1198 struct perf_event *event; 1199 1200 update_event_times(leader); 1201 list_for_each_entry(event, &leader->sibling_list, group_entry) 1202 update_event_times(event); 1203 } 1204 1205 static struct list_head * 1206 ctx_group_list(struct perf_event *event, struct perf_event_context *ctx) 1207 { 1208 if (event->attr.pinned) 1209 return &ctx->pinned_groups; 1210 else 1211 return &ctx->flexible_groups; 1212 } 1213 1214 /* 1215 * Add a event from the lists for its context. 1216 * Must be called with ctx->mutex and ctx->lock held. 1217 */ 1218 static void 1219 list_add_event(struct perf_event *event, struct perf_event_context *ctx) 1220 { 1221 WARN_ON_ONCE(event->attach_state & PERF_ATTACH_CONTEXT); 1222 event->attach_state |= PERF_ATTACH_CONTEXT; 1223 1224 /* 1225 * If we're a stand alone event or group leader, we go to the context 1226 * list, group events are kept attached to the group so that 1227 * perf_group_detach can, at all times, locate all siblings. 1228 */ 1229 if (event->group_leader == event) { 1230 struct list_head *list; 1231 1232 if (is_software_event(event)) 1233 event->group_flags |= PERF_GROUP_SOFTWARE; 1234 1235 list = ctx_group_list(event, ctx); 1236 list_add_tail(&event->group_entry, list); 1237 } 1238 1239 if (is_cgroup_event(event)) 1240 ctx->nr_cgroups++; 1241 1242 if (has_branch_stack(event)) 1243 ctx->nr_branch_stack++; 1244 1245 list_add_rcu(&event->event_entry, &ctx->event_list); 1246 ctx->nr_events++; 1247 if (event->attr.inherit_stat) 1248 ctx->nr_stat++; 1249 1250 ctx->generation++; 1251 } 1252 1253 /* 1254 * Initialize event state based on the perf_event_attr::disabled. 1255 */ 1256 static inline void perf_event__state_init(struct perf_event *event) 1257 { 1258 event->state = event->attr.disabled ? PERF_EVENT_STATE_OFF : 1259 PERF_EVENT_STATE_INACTIVE; 1260 } 1261 1262 /* 1263 * Called at perf_event creation and when events are attached/detached from a 1264 * group. 1265 */ 1266 static void perf_event__read_size(struct perf_event *event) 1267 { 1268 int entry = sizeof(u64); /* value */ 1269 int size = 0; 1270 int nr = 1; 1271 1272 if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) 1273 size += sizeof(u64); 1274 1275 if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) 1276 size += sizeof(u64); 1277 1278 if (event->attr.read_format & PERF_FORMAT_ID) 1279 entry += sizeof(u64); 1280 1281 if (event->attr.read_format & PERF_FORMAT_GROUP) { 1282 nr += event->group_leader->nr_siblings; 1283 size += sizeof(u64); 1284 } 1285 1286 size += entry * nr; 1287 event->read_size = size; 1288 } 1289 1290 static void perf_event__header_size(struct perf_event *event) 1291 { 1292 struct perf_sample_data *data; 1293 u64 sample_type = event->attr.sample_type; 1294 u16 size = 0; 1295 1296 perf_event__read_size(event); 1297 1298 if (sample_type & PERF_SAMPLE_IP) 1299 size += sizeof(data->ip); 1300 1301 if (sample_type & PERF_SAMPLE_ADDR) 1302 size += sizeof(data->addr); 1303 1304 if (sample_type & PERF_SAMPLE_PERIOD) 1305 size += sizeof(data->period); 1306 1307 if (sample_type & PERF_SAMPLE_WEIGHT) 1308 size += sizeof(data->weight); 1309 1310 if (sample_type & PERF_SAMPLE_READ) 1311 size += event->read_size; 1312 1313 if (sample_type & PERF_SAMPLE_DATA_SRC) 1314 size += sizeof(data->data_src.val); 1315 1316 if (sample_type & PERF_SAMPLE_TRANSACTION) 1317 size += sizeof(data->txn); 1318 1319 event->header_size = size; 1320 } 1321 1322 static void perf_event__id_header_size(struct perf_event *event) 1323 { 1324 struct perf_sample_data *data; 1325 u64 sample_type = event->attr.sample_type; 1326 u16 size = 0; 1327 1328 if (sample_type & PERF_SAMPLE_TID) 1329 size += sizeof(data->tid_entry); 1330 1331 if (sample_type & PERF_SAMPLE_TIME) 1332 size += sizeof(data->time); 1333 1334 if (sample_type & PERF_SAMPLE_IDENTIFIER) 1335 size += sizeof(data->id); 1336 1337 if (sample_type & PERF_SAMPLE_ID) 1338 size += sizeof(data->id); 1339 1340 if (sample_type & PERF_SAMPLE_STREAM_ID) 1341 size += sizeof(data->stream_id); 1342 1343 if (sample_type & PERF_SAMPLE_CPU) 1344 size += sizeof(data->cpu_entry); 1345 1346 event->id_header_size = size; 1347 } 1348 1349 static void perf_group_attach(struct perf_event *event) 1350 { 1351 struct perf_event *group_leader = event->group_leader, *pos; 1352 1353 /* 1354 * We can have double attach due to group movement in perf_event_open. 1355 */ 1356 if (event->attach_state & PERF_ATTACH_GROUP) 1357 return; 1358 1359 event->attach_state |= PERF_ATTACH_GROUP; 1360 1361 if (group_leader == event) 1362 return; 1363 1364 WARN_ON_ONCE(group_leader->ctx != event->ctx); 1365 1366 if (group_leader->group_flags & PERF_GROUP_SOFTWARE && 1367 !is_software_event(event)) 1368 group_leader->group_flags &= ~PERF_GROUP_SOFTWARE; 1369 1370 list_add_tail(&event->group_entry, &group_leader->sibling_list); 1371 group_leader->nr_siblings++; 1372 1373 perf_event__header_size(group_leader); 1374 1375 list_for_each_entry(pos, &group_leader->sibling_list, group_entry) 1376 perf_event__header_size(pos); 1377 } 1378 1379 /* 1380 * Remove a event from the lists for its context. 1381 * Must be called with ctx->mutex and ctx->lock held. 1382 */ 1383 static void 1384 list_del_event(struct perf_event *event, struct perf_event_context *ctx) 1385 { 1386 struct perf_cpu_context *cpuctx; 1387 1388 WARN_ON_ONCE(event->ctx != ctx); 1389 lockdep_assert_held(&ctx->lock); 1390 1391 /* 1392 * We can have double detach due to exit/hot-unplug + close. 1393 */ 1394 if (!(event->attach_state & PERF_ATTACH_CONTEXT)) 1395 return; 1396 1397 event->attach_state &= ~PERF_ATTACH_CONTEXT; 1398 1399 if (is_cgroup_event(event)) { 1400 ctx->nr_cgroups--; 1401 cpuctx = __get_cpu_context(ctx); 1402 /* 1403 * if there are no more cgroup events 1404 * then cler cgrp to avoid stale pointer 1405 * in update_cgrp_time_from_cpuctx() 1406 */ 1407 if (!ctx->nr_cgroups) 1408 cpuctx->cgrp = NULL; 1409 } 1410 1411 if (has_branch_stack(event)) 1412 ctx->nr_branch_stack--; 1413 1414 ctx->nr_events--; 1415 if (event->attr.inherit_stat) 1416 ctx->nr_stat--; 1417 1418 list_del_rcu(&event->event_entry); 1419 1420 if (event->group_leader == event) 1421 list_del_init(&event->group_entry); 1422 1423 update_group_times(event); 1424 1425 /* 1426 * If event was in error state, then keep it 1427 * that way, otherwise bogus counts will be 1428 * returned on read(). The only way to get out 1429 * of error state is by explicit re-enabling 1430 * of the event 1431 */ 1432 if (event->state > PERF_EVENT_STATE_OFF) 1433 event->state = PERF_EVENT_STATE_OFF; 1434 1435 ctx->generation++; 1436 } 1437 1438 static void perf_group_detach(struct perf_event *event) 1439 { 1440 struct perf_event *sibling, *tmp; 1441 struct list_head *list = NULL; 1442 1443 /* 1444 * We can have double detach due to exit/hot-unplug + close. 1445 */ 1446 if (!(event->attach_state & PERF_ATTACH_GROUP)) 1447 return; 1448 1449 event->attach_state &= ~PERF_ATTACH_GROUP; 1450 1451 /* 1452 * If this is a sibling, remove it from its group. 1453 */ 1454 if (event->group_leader != event) { 1455 list_del_init(&event->group_entry); 1456 event->group_leader->nr_siblings--; 1457 goto out; 1458 } 1459 1460 if (!list_empty(&event->group_entry)) 1461 list = &event->group_entry; 1462 1463 /* 1464 * If this was a group event with sibling events then 1465 * upgrade the siblings to singleton events by adding them 1466 * to whatever list we are on. 1467 */ 1468 list_for_each_entry_safe(sibling, tmp, &event->sibling_list, group_entry) { 1469 if (list) 1470 list_move_tail(&sibling->group_entry, list); 1471 sibling->group_leader = sibling; 1472 1473 /* Inherit group flags from the previous leader */ 1474 sibling->group_flags = event->group_flags; 1475 1476 WARN_ON_ONCE(sibling->ctx != event->ctx); 1477 } 1478 1479 out: 1480 perf_event__header_size(event->group_leader); 1481 1482 list_for_each_entry(tmp, &event->group_leader->sibling_list, group_entry) 1483 perf_event__header_size(tmp); 1484 } 1485 1486 /* 1487 * User event without the task. 1488 */ 1489 static bool is_orphaned_event(struct perf_event *event) 1490 { 1491 return event && !is_kernel_event(event) && !event->owner; 1492 } 1493 1494 /* 1495 * Event has a parent but parent's task finished and it's 1496 * alive only because of children holding refference. 1497 */ 1498 static bool is_orphaned_child(struct perf_event *event) 1499 { 1500 return is_orphaned_event(event->parent); 1501 } 1502 1503 static void orphans_remove_work(struct work_struct *work); 1504 1505 static void schedule_orphans_remove(struct perf_event_context *ctx) 1506 { 1507 if (!ctx->task || ctx->orphans_remove_sched || !perf_wq) 1508 return; 1509 1510 if (queue_delayed_work(perf_wq, &ctx->orphans_remove, 1)) { 1511 get_ctx(ctx); 1512 ctx->orphans_remove_sched = true; 1513 } 1514 } 1515 1516 static int __init perf_workqueue_init(void) 1517 { 1518 perf_wq = create_singlethread_workqueue("perf"); 1519 WARN(!perf_wq, "failed to create perf workqueue\n"); 1520 return perf_wq ? 0 : -1; 1521 } 1522 1523 core_initcall(perf_workqueue_init); 1524 1525 static inline int 1526 event_filter_match(struct perf_event *event) 1527 { 1528 return (event->cpu == -1 || event->cpu == smp_processor_id()) 1529 && perf_cgroup_match(event); 1530 } 1531 1532 static void 1533 event_sched_out(struct perf_event *event, 1534 struct perf_cpu_context *cpuctx, 1535 struct perf_event_context *ctx) 1536 { 1537 u64 tstamp = perf_event_time(event); 1538 u64 delta; 1539 1540 WARN_ON_ONCE(event->ctx != ctx); 1541 lockdep_assert_held(&ctx->lock); 1542 1543 /* 1544 * An event which could not be activated because of 1545 * filter mismatch still needs to have its timings 1546 * maintained, otherwise bogus information is return 1547 * via read() for time_enabled, time_running: 1548 */ 1549 if (event->state == PERF_EVENT_STATE_INACTIVE 1550 && !event_filter_match(event)) { 1551 delta = tstamp - event->tstamp_stopped; 1552 event->tstamp_running += delta; 1553 event->tstamp_stopped = tstamp; 1554 } 1555 1556 if (event->state != PERF_EVENT_STATE_ACTIVE) 1557 return; 1558 1559 perf_pmu_disable(event->pmu); 1560 1561 event->state = PERF_EVENT_STATE_INACTIVE; 1562 if (event->pending_disable) { 1563 event->pending_disable = 0; 1564 event->state = PERF_EVENT_STATE_OFF; 1565 } 1566 event->tstamp_stopped = tstamp; 1567 event->pmu->del(event, 0); 1568 event->oncpu = -1; 1569 1570 if (!is_software_event(event)) 1571 cpuctx->active_oncpu--; 1572 if (!--ctx->nr_active) 1573 perf_event_ctx_deactivate(ctx); 1574 if (event->attr.freq && event->attr.sample_freq) 1575 ctx->nr_freq--; 1576 if (event->attr.exclusive || !cpuctx->active_oncpu) 1577 cpuctx->exclusive = 0; 1578 1579 if (is_orphaned_child(event)) 1580 schedule_orphans_remove(ctx); 1581 1582 perf_pmu_enable(event->pmu); 1583 } 1584 1585 static void 1586 group_sched_out(struct perf_event *group_event, 1587 struct perf_cpu_context *cpuctx, 1588 struct perf_event_context *ctx) 1589 { 1590 struct perf_event *event; 1591 int state = group_event->state; 1592 1593 event_sched_out(group_event, cpuctx, ctx); 1594 1595 /* 1596 * Schedule out siblings (if any): 1597 */ 1598 list_for_each_entry(event, &group_event->sibling_list, group_entry) 1599 event_sched_out(event, cpuctx, ctx); 1600 1601 if (state == PERF_EVENT_STATE_ACTIVE && group_event->attr.exclusive) 1602 cpuctx->exclusive = 0; 1603 } 1604 1605 struct remove_event { 1606 struct perf_event *event; 1607 bool detach_group; 1608 }; 1609 1610 /* 1611 * Cross CPU call to remove a performance event 1612 * 1613 * We disable the event on the hardware level first. After that we 1614 * remove it from the context list. 1615 */ 1616 static int __perf_remove_from_context(void *info) 1617 { 1618 struct remove_event *re = info; 1619 struct perf_event *event = re->event; 1620 struct perf_event_context *ctx = event->ctx; 1621 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx); 1622 1623 raw_spin_lock(&ctx->lock); 1624 event_sched_out(event, cpuctx, ctx); 1625 if (re->detach_group) 1626 perf_group_detach(event); 1627 list_del_event(event, ctx); 1628 if (!ctx->nr_events && cpuctx->task_ctx == ctx) { 1629 ctx->is_active = 0; 1630 cpuctx->task_ctx = NULL; 1631 } 1632 raw_spin_unlock(&ctx->lock); 1633 1634 return 0; 1635 } 1636 1637 1638 /* 1639 * Remove the event from a task's (or a CPU's) list of events. 1640 * 1641 * CPU events are removed with a smp call. For task events we only 1642 * call when the task is on a CPU. 1643 * 1644 * If event->ctx is a cloned context, callers must make sure that 1645 * every task struct that event->ctx->task could possibly point to 1646 * remains valid. This is OK when called from perf_release since 1647 * that only calls us on the top-level context, which can't be a clone. 1648 * When called from perf_event_exit_task, it's OK because the 1649 * context has been detached from its task. 1650 */ 1651 static void perf_remove_from_context(struct perf_event *event, bool detach_group) 1652 { 1653 struct perf_event_context *ctx = event->ctx; 1654 struct task_struct *task = ctx->task; 1655 struct remove_event re = { 1656 .event = event, 1657 .detach_group = detach_group, 1658 }; 1659 1660 lockdep_assert_held(&ctx->mutex); 1661 1662 if (!task) { 1663 /* 1664 * Per cpu events are removed via an smp call. The removal can 1665 * fail if the CPU is currently offline, but in that case we 1666 * already called __perf_remove_from_context from 1667 * perf_event_exit_cpu. 1668 */ 1669 cpu_function_call(event->cpu, __perf_remove_from_context, &re); 1670 return; 1671 } 1672 1673 retry: 1674 if (!task_function_call(task, __perf_remove_from_context, &re)) 1675 return; 1676 1677 raw_spin_lock_irq(&ctx->lock); 1678 /* 1679 * If we failed to find a running task, but find the context active now 1680 * that we've acquired the ctx->lock, retry. 1681 */ 1682 if (ctx->is_active) { 1683 raw_spin_unlock_irq(&ctx->lock); 1684 /* 1685 * Reload the task pointer, it might have been changed by 1686 * a concurrent perf_event_context_sched_out(). 1687 */ 1688 task = ctx->task; 1689 goto retry; 1690 } 1691 1692 /* 1693 * Since the task isn't running, its safe to remove the event, us 1694 * holding the ctx->lock ensures the task won't get scheduled in. 1695 */ 1696 if (detach_group) 1697 perf_group_detach(event); 1698 list_del_event(event, ctx); 1699 raw_spin_unlock_irq(&ctx->lock); 1700 } 1701 1702 /* 1703 * Cross CPU call to disable a performance event 1704 */ 1705 int __perf_event_disable(void *info) 1706 { 1707 struct perf_event *event = info; 1708 struct perf_event_context *ctx = event->ctx; 1709 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx); 1710 1711 /* 1712 * If this is a per-task event, need to check whether this 1713 * event's task is the current task on this cpu. 1714 * 1715 * Can trigger due to concurrent perf_event_context_sched_out() 1716 * flipping contexts around. 1717 */ 1718 if (ctx->task && cpuctx->task_ctx != ctx) 1719 return -EINVAL; 1720 1721 raw_spin_lock(&ctx->lock); 1722 1723 /* 1724 * If the event is on, turn it off. 1725 * If it is in error state, leave it in error state. 1726 */ 1727 if (event->state >= PERF_EVENT_STATE_INACTIVE) { 1728 update_context_time(ctx); 1729 update_cgrp_time_from_event(event); 1730 update_group_times(event); 1731 if (event == event->group_leader) 1732 group_sched_out(event, cpuctx, ctx); 1733 else 1734 event_sched_out(event, cpuctx, ctx); 1735 event->state = PERF_EVENT_STATE_OFF; 1736 } 1737 1738 raw_spin_unlock(&ctx->lock); 1739 1740 return 0; 1741 } 1742 1743 /* 1744 * Disable a event. 1745 * 1746 * If event->ctx is a cloned context, callers must make sure that 1747 * every task struct that event->ctx->task could possibly point to 1748 * remains valid. This condition is satisifed when called through 1749 * perf_event_for_each_child or perf_event_for_each because they 1750 * hold the top-level event's child_mutex, so any descendant that 1751 * goes to exit will block in sync_child_event. 1752 * When called from perf_pending_event it's OK because event->ctx 1753 * is the current context on this CPU and preemption is disabled, 1754 * hence we can't get into perf_event_task_sched_out for this context. 1755 */ 1756 static void _perf_event_disable(struct perf_event *event) 1757 { 1758 struct perf_event_context *ctx = event->ctx; 1759 struct task_struct *task = ctx->task; 1760 1761 if (!task) { 1762 /* 1763 * Disable the event on the cpu that it's on 1764 */ 1765 cpu_function_call(event->cpu, __perf_event_disable, event); 1766 return; 1767 } 1768 1769 retry: 1770 if (!task_function_call(task, __perf_event_disable, event)) 1771 return; 1772 1773 raw_spin_lock_irq(&ctx->lock); 1774 /* 1775 * If the event is still active, we need to retry the cross-call. 1776 */ 1777 if (event->state == PERF_EVENT_STATE_ACTIVE) { 1778 raw_spin_unlock_irq(&ctx->lock); 1779 /* 1780 * Reload the task pointer, it might have been changed by 1781 * a concurrent perf_event_context_sched_out(). 1782 */ 1783 task = ctx->task; 1784 goto retry; 1785 } 1786 1787 /* 1788 * Since we have the lock this context can't be scheduled 1789 * in, so we can change the state safely. 1790 */ 1791 if (event->state == PERF_EVENT_STATE_INACTIVE) { 1792 update_group_times(event); 1793 event->state = PERF_EVENT_STATE_OFF; 1794 } 1795 raw_spin_unlock_irq(&ctx->lock); 1796 } 1797 1798 /* 1799 * Strictly speaking kernel users cannot create groups and therefore this 1800 * interface does not need the perf_event_ctx_lock() magic. 1801 */ 1802 void perf_event_disable(struct perf_event *event) 1803 { 1804 struct perf_event_context *ctx; 1805 1806 ctx = perf_event_ctx_lock(event); 1807 _perf_event_disable(event); 1808 perf_event_ctx_unlock(event, ctx); 1809 } 1810 EXPORT_SYMBOL_GPL(perf_event_disable); 1811 1812 static void perf_set_shadow_time(struct perf_event *event, 1813 struct perf_event_context *ctx, 1814 u64 tstamp) 1815 { 1816 /* 1817 * use the correct time source for the time snapshot 1818 * 1819 * We could get by without this by leveraging the 1820 * fact that to get to this function, the caller 1821 * has most likely already called update_context_time() 1822 * and update_cgrp_time_xx() and thus both timestamp 1823 * are identical (or very close). Given that tstamp is, 1824 * already adjusted for cgroup, we could say that: 1825 * tstamp - ctx->timestamp 1826 * is equivalent to 1827 * tstamp - cgrp->timestamp. 1828 * 1829 * Then, in perf_output_read(), the calculation would 1830 * work with no changes because: 1831 * - event is guaranteed scheduled in 1832 * - no scheduled out in between 1833 * - thus the timestamp would be the same 1834 * 1835 * But this is a bit hairy. 1836 * 1837 * So instead, we have an explicit cgroup call to remain 1838 * within the time time source all along. We believe it 1839 * is cleaner and simpler to understand. 1840 */ 1841 if (is_cgroup_event(event)) 1842 perf_cgroup_set_shadow_time(event, tstamp); 1843 else 1844 event->shadow_ctx_time = tstamp - ctx->timestamp; 1845 } 1846 1847 #define MAX_INTERRUPTS (~0ULL) 1848 1849 static void perf_log_throttle(struct perf_event *event, int enable); 1850 1851 static int 1852 event_sched_in(struct perf_event *event, 1853 struct perf_cpu_context *cpuctx, 1854 struct perf_event_context *ctx) 1855 { 1856 u64 tstamp = perf_event_time(event); 1857 int ret = 0; 1858 1859 lockdep_assert_held(&ctx->lock); 1860 1861 if (event->state <= PERF_EVENT_STATE_OFF) 1862 return 0; 1863 1864 event->state = PERF_EVENT_STATE_ACTIVE; 1865 event->oncpu = smp_processor_id(); 1866 1867 /* 1868 * Unthrottle events, since we scheduled we might have missed several 1869 * ticks already, also for a heavily scheduling task there is little 1870 * guarantee it'll get a tick in a timely manner. 1871 */ 1872 if (unlikely(event->hw.interrupts == MAX_INTERRUPTS)) { 1873 perf_log_throttle(event, 1); 1874 event->hw.interrupts = 0; 1875 } 1876 1877 /* 1878 * The new state must be visible before we turn it on in the hardware: 1879 */ 1880 smp_wmb(); 1881 1882 perf_pmu_disable(event->pmu); 1883 1884 if (event->pmu->add(event, PERF_EF_START)) { 1885 event->state = PERF_EVENT_STATE_INACTIVE; 1886 event->oncpu = -1; 1887 ret = -EAGAIN; 1888 goto out; 1889 } 1890 1891 event->tstamp_running += tstamp - event->tstamp_stopped; 1892 1893 perf_set_shadow_time(event, ctx, tstamp); 1894 1895 if (!is_software_event(event)) 1896 cpuctx->active_oncpu++; 1897 if (!ctx->nr_active++) 1898 perf_event_ctx_activate(ctx); 1899 if (event->attr.freq && event->attr.sample_freq) 1900 ctx->nr_freq++; 1901 1902 if (event->attr.exclusive) 1903 cpuctx->exclusive = 1; 1904 1905 if (is_orphaned_child(event)) 1906 schedule_orphans_remove(ctx); 1907 1908 out: 1909 perf_pmu_enable(event->pmu); 1910 1911 return ret; 1912 } 1913 1914 static int 1915 group_sched_in(struct perf_event *group_event, 1916 struct perf_cpu_context *cpuctx, 1917 struct perf_event_context *ctx) 1918 { 1919 struct perf_event *event, *partial_group = NULL; 1920 struct pmu *pmu = ctx->pmu; 1921 u64 now = ctx->time; 1922 bool simulate = false; 1923 1924 if (group_event->state == PERF_EVENT_STATE_OFF) 1925 return 0; 1926 1927 pmu->start_txn(pmu); 1928 1929 if (event_sched_in(group_event, cpuctx, ctx)) { 1930 pmu->cancel_txn(pmu); 1931 perf_cpu_hrtimer_restart(cpuctx); 1932 return -EAGAIN; 1933 } 1934 1935 /* 1936 * Schedule in siblings as one group (if any): 1937 */ 1938 list_for_each_entry(event, &group_event->sibling_list, group_entry) { 1939 if (event_sched_in(event, cpuctx, ctx)) { 1940 partial_group = event; 1941 goto group_error; 1942 } 1943 } 1944 1945 if (!pmu->commit_txn(pmu)) 1946 return 0; 1947 1948 group_error: 1949 /* 1950 * Groups can be scheduled in as one unit only, so undo any 1951 * partial group before returning: 1952 * The events up to the failed event are scheduled out normally, 1953 * tstamp_stopped will be updated. 1954 * 1955 * The failed events and the remaining siblings need to have 1956 * their timings updated as if they had gone thru event_sched_in() 1957 * and event_sched_out(). This is required to get consistent timings 1958 * across the group. This also takes care of the case where the group 1959 * could never be scheduled by ensuring tstamp_stopped is set to mark 1960 * the time the event was actually stopped, such that time delta 1961 * calculation in update_event_times() is correct. 1962 */ 1963 list_for_each_entry(event, &group_event->sibling_list, group_entry) { 1964 if (event == partial_group) 1965 simulate = true; 1966 1967 if (simulate) { 1968 event->tstamp_running += now - event->tstamp_stopped; 1969 event->tstamp_stopped = now; 1970 } else { 1971 event_sched_out(event, cpuctx, ctx); 1972 } 1973 } 1974 event_sched_out(group_event, cpuctx, ctx); 1975 1976 pmu->cancel_txn(pmu); 1977 1978 perf_cpu_hrtimer_restart(cpuctx); 1979 1980 return -EAGAIN; 1981 } 1982 1983 /* 1984 * Work out whether we can put this event group on the CPU now. 1985 */ 1986 static int group_can_go_on(struct perf_event *event, 1987 struct perf_cpu_context *cpuctx, 1988 int can_add_hw) 1989 { 1990 /* 1991 * Groups consisting entirely of software events can always go on. 1992 */ 1993 if (event->group_flags & PERF_GROUP_SOFTWARE) 1994 return 1; 1995 /* 1996 * If an exclusive group is already on, no other hardware 1997 * events can go on. 1998 */ 1999 if (cpuctx->exclusive) 2000 return 0; 2001 /* 2002 * If this group is exclusive and there are already 2003 * events on the CPU, it can't go on. 2004 */ 2005 if (event->attr.exclusive && cpuctx->active_oncpu) 2006 return 0; 2007 /* 2008 * Otherwise, try to add it if all previous groups were able 2009 * to go on. 2010 */ 2011 return can_add_hw; 2012 } 2013 2014 static void add_event_to_ctx(struct perf_event *event, 2015 struct perf_event_context *ctx) 2016 { 2017 u64 tstamp = perf_event_time(event); 2018 2019 list_add_event(event, ctx); 2020 perf_group_attach(event); 2021 event->tstamp_enabled = tstamp; 2022 event->tstamp_running = tstamp; 2023 event->tstamp_stopped = tstamp; 2024 } 2025 2026 static void task_ctx_sched_out(struct perf_event_context *ctx); 2027 static void 2028 ctx_sched_in(struct perf_event_context *ctx, 2029 struct perf_cpu_context *cpuctx, 2030 enum event_type_t event_type, 2031 struct task_struct *task); 2032 2033 static void perf_event_sched_in(struct perf_cpu_context *cpuctx, 2034 struct perf_event_context *ctx, 2035 struct task_struct *task) 2036 { 2037 cpu_ctx_sched_in(cpuctx, EVENT_PINNED, task); 2038 if (ctx) 2039 ctx_sched_in(ctx, cpuctx, EVENT_PINNED, task); 2040 cpu_ctx_sched_in(cpuctx, EVENT_FLEXIBLE, task); 2041 if (ctx) 2042 ctx_sched_in(ctx, cpuctx, EVENT_FLEXIBLE, task); 2043 } 2044 2045 /* 2046 * Cross CPU call to install and enable a performance event 2047 * 2048 * Must be called with ctx->mutex held 2049 */ 2050 static int __perf_install_in_context(void *info) 2051 { 2052 struct perf_event *event = info; 2053 struct perf_event_context *ctx = event->ctx; 2054 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx); 2055 struct perf_event_context *task_ctx = cpuctx->task_ctx; 2056 struct task_struct *task = current; 2057 2058 perf_ctx_lock(cpuctx, task_ctx); 2059 perf_pmu_disable(cpuctx->ctx.pmu); 2060 2061 /* 2062 * If there was an active task_ctx schedule it out. 2063 */ 2064 if (task_ctx) 2065 task_ctx_sched_out(task_ctx); 2066 2067 /* 2068 * If the context we're installing events in is not the 2069 * active task_ctx, flip them. 2070 */ 2071 if (ctx->task && task_ctx != ctx) { 2072 if (task_ctx) 2073 raw_spin_unlock(&task_ctx->lock); 2074 raw_spin_lock(&ctx->lock); 2075 task_ctx = ctx; 2076 } 2077 2078 if (task_ctx) { 2079 cpuctx->task_ctx = task_ctx; 2080 task = task_ctx->task; 2081 } 2082 2083 cpu_ctx_sched_out(cpuctx, EVENT_ALL); 2084 2085 update_context_time(ctx); 2086 /* 2087 * update cgrp time only if current cgrp 2088 * matches event->cgrp. Must be done before 2089 * calling add_event_to_ctx() 2090 */ 2091 update_cgrp_time_from_event(event); 2092 2093 add_event_to_ctx(event, ctx); 2094 2095 /* 2096 * Schedule everything back in 2097 */ 2098 perf_event_sched_in(cpuctx, task_ctx, task); 2099 2100 perf_pmu_enable(cpuctx->ctx.pmu); 2101 perf_ctx_unlock(cpuctx, task_ctx); 2102 2103 return 0; 2104 } 2105 2106 /* 2107 * Attach a performance event to a context 2108 * 2109 * First we add the event to the list with the hardware enable bit 2110 * in event->hw_config cleared. 2111 * 2112 * If the event is attached to a task which is on a CPU we use a smp 2113 * call to enable it in the task context. The task might have been 2114 * scheduled away, but we check this in the smp call again. 2115 */ 2116 static void 2117 perf_install_in_context(struct perf_event_context *ctx, 2118 struct perf_event *event, 2119 int cpu) 2120 { 2121 struct task_struct *task = ctx->task; 2122 2123 lockdep_assert_held(&ctx->mutex); 2124 2125 event->ctx = ctx; 2126 if (event->cpu != -1) 2127 event->cpu = cpu; 2128 2129 if (!task) { 2130 /* 2131 * Per cpu events are installed via an smp call and 2132 * the install is always successful. 2133 */ 2134 cpu_function_call(cpu, __perf_install_in_context, event); 2135 return; 2136 } 2137 2138 retry: 2139 if (!task_function_call(task, __perf_install_in_context, event)) 2140 return; 2141 2142 raw_spin_lock_irq(&ctx->lock); 2143 /* 2144 * If we failed to find a running task, but find the context active now 2145 * that we've acquired the ctx->lock, retry. 2146 */ 2147 if (ctx->is_active) { 2148 raw_spin_unlock_irq(&ctx->lock); 2149 /* 2150 * Reload the task pointer, it might have been changed by 2151 * a concurrent perf_event_context_sched_out(). 2152 */ 2153 task = ctx->task; 2154 goto retry; 2155 } 2156 2157 /* 2158 * Since the task isn't running, its safe to add the event, us holding 2159 * the ctx->lock ensures the task won't get scheduled in. 2160 */ 2161 add_event_to_ctx(event, ctx); 2162 raw_spin_unlock_irq(&ctx->lock); 2163 } 2164 2165 /* 2166 * Put a event into inactive state and update time fields. 2167 * Enabling the leader of a group effectively enables all 2168 * the group members that aren't explicitly disabled, so we 2169 * have to update their ->tstamp_enabled also. 2170 * Note: this works for group members as well as group leaders 2171 * since the non-leader members' sibling_lists will be empty. 2172 */ 2173 static void __perf_event_mark_enabled(struct perf_event *event) 2174 { 2175 struct perf_event *sub; 2176 u64 tstamp = perf_event_time(event); 2177 2178 event->state = PERF_EVENT_STATE_INACTIVE; 2179 event->tstamp_enabled = tstamp - event->total_time_enabled; 2180 list_for_each_entry(sub, &event->sibling_list, group_entry) { 2181 if (sub->state >= PERF_EVENT_STATE_INACTIVE) 2182 sub->tstamp_enabled = tstamp - sub->total_time_enabled; 2183 } 2184 } 2185 2186 /* 2187 * Cross CPU call to enable a performance event 2188 */ 2189 static int __perf_event_enable(void *info) 2190 { 2191 struct perf_event *event = info; 2192 struct perf_event_context *ctx = event->ctx; 2193 struct perf_event *leader = event->group_leader; 2194 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx); 2195 int err; 2196 2197 /* 2198 * There's a time window between 'ctx->is_active' check 2199 * in perf_event_enable function and this place having: 2200 * - IRQs on 2201 * - ctx->lock unlocked 2202 * 2203 * where the task could be killed and 'ctx' deactivated 2204 * by perf_event_exit_task. 2205 */ 2206 if (!ctx->is_active) 2207 return -EINVAL; 2208 2209 raw_spin_lock(&ctx->lock); 2210 update_context_time(ctx); 2211 2212 if (event->state >= PERF_EVENT_STATE_INACTIVE) 2213 goto unlock; 2214 2215 /* 2216 * set current task's cgroup time reference point 2217 */ 2218 perf_cgroup_set_timestamp(current, ctx); 2219 2220 __perf_event_mark_enabled(event); 2221 2222 if (!event_filter_match(event)) { 2223 if (is_cgroup_event(event)) 2224 perf_cgroup_defer_enabled(event); 2225 goto unlock; 2226 } 2227 2228 /* 2229 * If the event is in a group and isn't the group leader, 2230 * then don't put it on unless the group is on. 2231 */ 2232 if (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE) 2233 goto unlock; 2234 2235 if (!group_can_go_on(event, cpuctx, 1)) { 2236 err = -EEXIST; 2237 } else { 2238 if (event == leader) 2239 err = group_sched_in(event, cpuctx, ctx); 2240 else 2241 err = event_sched_in(event, cpuctx, ctx); 2242 } 2243 2244 if (err) { 2245 /* 2246 * If this event can't go on and it's part of a 2247 * group, then the whole group has to come off. 2248 */ 2249 if (leader != event) { 2250 group_sched_out(leader, cpuctx, ctx); 2251 perf_cpu_hrtimer_restart(cpuctx); 2252 } 2253 if (leader->attr.pinned) { 2254 update_group_times(leader); 2255 leader->state = PERF_EVENT_STATE_ERROR; 2256 } 2257 } 2258 2259 unlock: 2260 raw_spin_unlock(&ctx->lock); 2261 2262 return 0; 2263 } 2264 2265 /* 2266 * Enable a event. 2267 * 2268 * If event->ctx is a cloned context, callers must make sure that 2269 * every task struct that event->ctx->task could possibly point to 2270 * remains valid. This condition is satisfied when called through 2271 * perf_event_for_each_child or perf_event_for_each as described 2272 * for perf_event_disable. 2273 */ 2274 static void _perf_event_enable(struct perf_event *event) 2275 { 2276 struct perf_event_context *ctx = event->ctx; 2277 struct task_struct *task = ctx->task; 2278 2279 if (!task) { 2280 /* 2281 * Enable the event on the cpu that it's on 2282 */ 2283 cpu_function_call(event->cpu, __perf_event_enable, event); 2284 return; 2285 } 2286 2287 raw_spin_lock_irq(&ctx->lock); 2288 if (event->state >= PERF_EVENT_STATE_INACTIVE) 2289 goto out; 2290 2291 /* 2292 * If the event is in error state, clear that first. 2293 * That way, if we see the event in error state below, we 2294 * know that it has gone back into error state, as distinct 2295 * from the task having been scheduled away before the 2296 * cross-call arrived. 2297 */ 2298 if (event->state == PERF_EVENT_STATE_ERROR) 2299 event->state = PERF_EVENT_STATE_OFF; 2300 2301 retry: 2302 if (!ctx->is_active) { 2303 __perf_event_mark_enabled(event); 2304 goto out; 2305 } 2306 2307 raw_spin_unlock_irq(&ctx->lock); 2308 2309 if (!task_function_call(task, __perf_event_enable, event)) 2310 return; 2311 2312 raw_spin_lock_irq(&ctx->lock); 2313 2314 /* 2315 * If the context is active and the event is still off, 2316 * we need to retry the cross-call. 2317 */ 2318 if (ctx->is_active && event->state == PERF_EVENT_STATE_OFF) { 2319 /* 2320 * task could have been flipped by a concurrent 2321 * perf_event_context_sched_out() 2322 */ 2323 task = ctx->task; 2324 goto retry; 2325 } 2326 2327 out: 2328 raw_spin_unlock_irq(&ctx->lock); 2329 } 2330 2331 /* 2332 * See perf_event_disable(); 2333 */ 2334 void perf_event_enable(struct perf_event *event) 2335 { 2336 struct perf_event_context *ctx; 2337 2338 ctx = perf_event_ctx_lock(event); 2339 _perf_event_enable(event); 2340 perf_event_ctx_unlock(event, ctx); 2341 } 2342 EXPORT_SYMBOL_GPL(perf_event_enable); 2343 2344 static int _perf_event_refresh(struct perf_event *event, int refresh) 2345 { 2346 /* 2347 * not supported on inherited events 2348 */ 2349 if (event->attr.inherit || !is_sampling_event(event)) 2350 return -EINVAL; 2351 2352 atomic_add(refresh, &event->event_limit); 2353 _perf_event_enable(event); 2354 2355 return 0; 2356 } 2357 2358 /* 2359 * See perf_event_disable() 2360 */ 2361 int perf_event_refresh(struct perf_event *event, int refresh) 2362 { 2363 struct perf_event_context *ctx; 2364 int ret; 2365 2366 ctx = perf_event_ctx_lock(event); 2367 ret = _perf_event_refresh(event, refresh); 2368 perf_event_ctx_unlock(event, ctx); 2369 2370 return ret; 2371 } 2372 EXPORT_SYMBOL_GPL(perf_event_refresh); 2373 2374 static void ctx_sched_out(struct perf_event_context *ctx, 2375 struct perf_cpu_context *cpuctx, 2376 enum event_type_t event_type) 2377 { 2378 struct perf_event *event; 2379 int is_active = ctx->is_active; 2380 2381 ctx->is_active &= ~event_type; 2382 if (likely(!ctx->nr_events)) 2383 return; 2384 2385 update_context_time(ctx); 2386 update_cgrp_time_from_cpuctx(cpuctx); 2387 if (!ctx->nr_active) 2388 return; 2389 2390 perf_pmu_disable(ctx->pmu); 2391 if ((is_active & EVENT_PINNED) && (event_type & EVENT_PINNED)) { 2392 list_for_each_entry(event, &ctx->pinned_groups, group_entry) 2393 group_sched_out(event, cpuctx, ctx); 2394 } 2395 2396 if ((is_active & EVENT_FLEXIBLE) && (event_type & EVENT_FLEXIBLE)) { 2397 list_for_each_entry(event, &ctx->flexible_groups, group_entry) 2398 group_sched_out(event, cpuctx, ctx); 2399 } 2400 perf_pmu_enable(ctx->pmu); 2401 } 2402 2403 /* 2404 * Test whether two contexts are equivalent, i.e. whether they have both been 2405 * cloned from the same version of the same context. 2406 * 2407 * Equivalence is measured using a generation number in the context that is 2408 * incremented on each modification to it; see unclone_ctx(), list_add_event() 2409 * and list_del_event(). 2410 */ 2411 static int context_equiv(struct perf_event_context *ctx1, 2412 struct perf_event_context *ctx2) 2413 { 2414 lockdep_assert_held(&ctx1->lock); 2415 lockdep_assert_held(&ctx2->lock); 2416 2417 /* Pinning disables the swap optimization */ 2418 if (ctx1->pin_count || ctx2->pin_count) 2419 return 0; 2420 2421 /* If ctx1 is the parent of ctx2 */ 2422 if (ctx1 == ctx2->parent_ctx && ctx1->generation == ctx2->parent_gen) 2423 return 1; 2424 2425 /* If ctx2 is the parent of ctx1 */ 2426 if (ctx1->parent_ctx == ctx2 && ctx1->parent_gen == ctx2->generation) 2427 return 1; 2428 2429 /* 2430 * If ctx1 and ctx2 have the same parent; we flatten the parent 2431 * hierarchy, see perf_event_init_context(). 2432 */ 2433 if (ctx1->parent_ctx && ctx1->parent_ctx == ctx2->parent_ctx && 2434 ctx1->parent_gen == ctx2->parent_gen) 2435 return 1; 2436 2437 /* Unmatched */ 2438 return 0; 2439 } 2440 2441 static void __perf_event_sync_stat(struct perf_event *event, 2442 struct perf_event *next_event) 2443 { 2444 u64 value; 2445 2446 if (!event->attr.inherit_stat) 2447 return; 2448 2449 /* 2450 * Update the event value, we cannot use perf_event_read() 2451 * because we're in the middle of a context switch and have IRQs 2452 * disabled, which upsets smp_call_function_single(), however 2453 * we know the event must be on the current CPU, therefore we 2454 * don't need to use it. 2455 */ 2456 switch (event->state) { 2457 case PERF_EVENT_STATE_ACTIVE: 2458 event->pmu->read(event); 2459 /* fall-through */ 2460 2461 case PERF_EVENT_STATE_INACTIVE: 2462 update_event_times(event); 2463 break; 2464 2465 default: 2466 break; 2467 } 2468 2469 /* 2470 * In order to keep per-task stats reliable we need to flip the event 2471 * values when we flip the contexts. 2472 */ 2473 value = local64_read(&next_event->count); 2474 value = local64_xchg(&event->count, value); 2475 local64_set(&next_event->count, value); 2476 2477 swap(event->total_time_enabled, next_event->total_time_enabled); 2478 swap(event->total_time_running, next_event->total_time_running); 2479 2480 /* 2481 * Since we swizzled the values, update the user visible data too. 2482 */ 2483 perf_event_update_userpage(event); 2484 perf_event_update_userpage(next_event); 2485 } 2486 2487 static void perf_event_sync_stat(struct perf_event_context *ctx, 2488 struct perf_event_context *next_ctx) 2489 { 2490 struct perf_event *event, *next_event; 2491 2492 if (!ctx->nr_stat) 2493 return; 2494 2495 update_context_time(ctx); 2496 2497 event = list_first_entry(&ctx->event_list, 2498 struct perf_event, event_entry); 2499 2500 next_event = list_first_entry(&next_ctx->event_list, 2501 struct perf_event, event_entry); 2502 2503 while (&event->event_entry != &ctx->event_list && 2504 &next_event->event_entry != &next_ctx->event_list) { 2505 2506 __perf_event_sync_stat(event, next_event); 2507 2508 event = list_next_entry(event, event_entry); 2509 next_event = list_next_entry(next_event, event_entry); 2510 } 2511 } 2512 2513 static void perf_event_context_sched_out(struct task_struct *task, int ctxn, 2514 struct task_struct *next) 2515 { 2516 struct perf_event_context *ctx = task->perf_event_ctxp[ctxn]; 2517 struct perf_event_context *next_ctx; 2518 struct perf_event_context *parent, *next_parent; 2519 struct perf_cpu_context *cpuctx; 2520 int do_switch = 1; 2521 2522 if (likely(!ctx)) 2523 return; 2524 2525 cpuctx = __get_cpu_context(ctx); 2526 if (!cpuctx->task_ctx) 2527 return; 2528 2529 rcu_read_lock(); 2530 next_ctx = next->perf_event_ctxp[ctxn]; 2531 if (!next_ctx) 2532 goto unlock; 2533 2534 parent = rcu_dereference(ctx->parent_ctx); 2535 next_parent = rcu_dereference(next_ctx->parent_ctx); 2536 2537 /* If neither context have a parent context; they cannot be clones. */ 2538 if (!parent && !next_parent) 2539 goto unlock; 2540 2541 if (next_parent == ctx || next_ctx == parent || next_parent == parent) { 2542 /* 2543 * Looks like the two contexts are clones, so we might be 2544 * able to optimize the context switch. We lock both 2545 * contexts and check that they are clones under the 2546 * lock (including re-checking that neither has been 2547 * uncloned in the meantime). It doesn't matter which 2548 * order we take the locks because no other cpu could 2549 * be trying to lock both of these tasks. 2550 */ 2551 raw_spin_lock(&ctx->lock); 2552 raw_spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING); 2553 if (context_equiv(ctx, next_ctx)) { 2554 /* 2555 * XXX do we need a memory barrier of sorts 2556 * wrt to rcu_dereference() of perf_event_ctxp 2557 */ 2558 task->perf_event_ctxp[ctxn] = next_ctx; 2559 next->perf_event_ctxp[ctxn] = ctx; 2560 ctx->task = next; 2561 next_ctx->task = task; 2562 do_switch = 0; 2563 2564 perf_event_sync_stat(ctx, next_ctx); 2565 } 2566 raw_spin_unlock(&next_ctx->lock); 2567 raw_spin_unlock(&ctx->lock); 2568 } 2569 unlock: 2570 rcu_read_unlock(); 2571 2572 if (do_switch) { 2573 raw_spin_lock(&ctx->lock); 2574 ctx_sched_out(ctx, cpuctx, EVENT_ALL); 2575 cpuctx->task_ctx = NULL; 2576 raw_spin_unlock(&ctx->lock); 2577 } 2578 } 2579 2580 #define for_each_task_context_nr(ctxn) \ 2581 for ((ctxn) = 0; (ctxn) < perf_nr_task_contexts; (ctxn)++) 2582 2583 /* 2584 * Called from scheduler to remove the events of the current task, 2585 * with interrupts disabled. 2586 * 2587 * We stop each event and update the event value in event->count. 2588 * 2589 * This does not protect us against NMI, but disable() 2590 * sets the disabled bit in the control field of event _before_ 2591 * accessing the event control register. If a NMI hits, then it will 2592 * not restart the event. 2593 */ 2594 void __perf_event_task_sched_out(struct task_struct *task, 2595 struct task_struct *next) 2596 { 2597 int ctxn; 2598 2599 for_each_task_context_nr(ctxn) 2600 perf_event_context_sched_out(task, ctxn, next); 2601 2602 /* 2603 * if cgroup events exist on this CPU, then we need 2604 * to check if we have to switch out PMU state. 2605 * cgroup event are system-wide mode only 2606 */ 2607 if (atomic_read(this_cpu_ptr(&perf_cgroup_events))) 2608 perf_cgroup_sched_out(task, next); 2609 } 2610 2611 static void task_ctx_sched_out(struct perf_event_context *ctx) 2612 { 2613 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx); 2614 2615 if (!cpuctx->task_ctx) 2616 return; 2617 2618 if (WARN_ON_ONCE(ctx != cpuctx->task_ctx)) 2619 return; 2620 2621 ctx_sched_out(ctx, cpuctx, EVENT_ALL); 2622 cpuctx->task_ctx = NULL; 2623 } 2624 2625 /* 2626 * Called with IRQs disabled 2627 */ 2628 static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx, 2629 enum event_type_t event_type) 2630 { 2631 ctx_sched_out(&cpuctx->ctx, cpuctx, event_type); 2632 } 2633 2634 static void 2635 ctx_pinned_sched_in(struct perf_event_context *ctx, 2636 struct perf_cpu_context *cpuctx) 2637 { 2638 struct perf_event *event; 2639 2640 list_for_each_entry(event, &ctx->pinned_groups, group_entry) { 2641 if (event->state <= PERF_EVENT_STATE_OFF) 2642 continue; 2643 if (!event_filter_match(event)) 2644 continue; 2645 2646 /* may need to reset tstamp_enabled */ 2647 if (is_cgroup_event(event)) 2648 perf_cgroup_mark_enabled(event, ctx); 2649 2650 if (group_can_go_on(event, cpuctx, 1)) 2651 group_sched_in(event, cpuctx, ctx); 2652 2653 /* 2654 * If this pinned group hasn't been scheduled, 2655 * put it in error state. 2656 */ 2657 if (event->state == PERF_EVENT_STATE_INACTIVE) { 2658 update_group_times(event); 2659 event->state = PERF_EVENT_STATE_ERROR; 2660 } 2661 } 2662 } 2663 2664 static void 2665 ctx_flexible_sched_in(struct perf_event_context *ctx, 2666 struct perf_cpu_context *cpuctx) 2667 { 2668 struct perf_event *event; 2669 int can_add_hw = 1; 2670 2671 list_for_each_entry(event, &ctx->flexible_groups, group_entry) { 2672 /* Ignore events in OFF or ERROR state */ 2673 if (event->state <= PERF_EVENT_STATE_OFF) 2674 continue; 2675 /* 2676 * Listen to the 'cpu' scheduling filter constraint 2677 * of events: 2678 */ 2679 if (!event_filter_match(event)) 2680 continue; 2681 2682 /* may need to reset tstamp_enabled */ 2683 if (is_cgroup_event(event)) 2684 perf_cgroup_mark_enabled(event, ctx); 2685 2686 if (group_can_go_on(event, cpuctx, can_add_hw)) { 2687 if (group_sched_in(event, cpuctx, ctx)) 2688 can_add_hw = 0; 2689 } 2690 } 2691 } 2692 2693 static void 2694 ctx_sched_in(struct perf_event_context *ctx, 2695 struct perf_cpu_context *cpuctx, 2696 enum event_type_t event_type, 2697 struct task_struct *task) 2698 { 2699 u64 now; 2700 int is_active = ctx->is_active; 2701 2702 ctx->is_active |= event_type; 2703 if (likely(!ctx->nr_events)) 2704 return; 2705 2706 now = perf_clock(); 2707 ctx->timestamp = now; 2708 perf_cgroup_set_timestamp(task, ctx); 2709 /* 2710 * First go through the list and put on any pinned groups 2711 * in order to give them the best chance of going on. 2712 */ 2713 if (!(is_active & EVENT_PINNED) && (event_type & EVENT_PINNED)) 2714 ctx_pinned_sched_in(ctx, cpuctx); 2715 2716 /* Then walk through the lower prio flexible groups */ 2717 if (!(is_active & EVENT_FLEXIBLE) && (event_type & EVENT_FLEXIBLE)) 2718 ctx_flexible_sched_in(ctx, cpuctx); 2719 } 2720 2721 static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx, 2722 enum event_type_t event_type, 2723 struct task_struct *task) 2724 { 2725 struct perf_event_context *ctx = &cpuctx->ctx; 2726 2727 ctx_sched_in(ctx, cpuctx, event_type, task); 2728 } 2729 2730 static void perf_event_context_sched_in(struct perf_event_context *ctx, 2731 struct task_struct *task) 2732 { 2733 struct perf_cpu_context *cpuctx; 2734 2735 cpuctx = __get_cpu_context(ctx); 2736 if (cpuctx->task_ctx == ctx) 2737 return; 2738 2739 perf_ctx_lock(cpuctx, ctx); 2740 perf_pmu_disable(ctx->pmu); 2741 /* 2742 * We want to keep the following priority order: 2743 * cpu pinned (that don't need to move), task pinned, 2744 * cpu flexible, task flexible. 2745 */ 2746 cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE); 2747 2748 if (ctx->nr_events) 2749 cpuctx->task_ctx = ctx; 2750 2751 perf_event_sched_in(cpuctx, cpuctx->task_ctx, task); 2752 2753 perf_pmu_enable(ctx->pmu); 2754 perf_ctx_unlock(cpuctx, ctx); 2755 } 2756 2757 /* 2758 * When sampling the branck stack in system-wide, it may be necessary 2759 * to flush the stack on context switch. This happens when the branch 2760 * stack does not tag its entries with the pid of the current task. 2761 * Otherwise it becomes impossible to associate a branch entry with a 2762 * task. This ambiguity is more likely to appear when the branch stack 2763 * supports priv level filtering and the user sets it to monitor only 2764 * at the user level (which could be a useful measurement in system-wide 2765 * mode). In that case, the risk is high of having a branch stack with 2766 * branch from multiple tasks. Flushing may mean dropping the existing 2767 * entries or stashing them somewhere in the PMU specific code layer. 2768 * 2769 * This function provides the context switch callback to the lower code 2770 * layer. It is invoked ONLY when there is at least one system-wide context 2771 * with at least one active event using taken branch sampling. 2772 */ 2773 static void perf_branch_stack_sched_in(struct task_struct *prev, 2774 struct task_struct *task) 2775 { 2776 struct perf_cpu_context *cpuctx; 2777 struct pmu *pmu; 2778 unsigned long flags; 2779 2780 /* no need to flush branch stack if not changing task */ 2781 if (prev == task) 2782 return; 2783 2784 local_irq_save(flags); 2785 2786 rcu_read_lock(); 2787 2788 list_for_each_entry_rcu(pmu, &pmus, entry) { 2789 cpuctx = this_cpu_ptr(pmu->pmu_cpu_context); 2790 2791 /* 2792 * check if the context has at least one 2793 * event using PERF_SAMPLE_BRANCH_STACK 2794 */ 2795 if (cpuctx->ctx.nr_branch_stack > 0 2796 && pmu->flush_branch_stack) { 2797 2798 perf_ctx_lock(cpuctx, cpuctx->task_ctx); 2799 2800 perf_pmu_disable(pmu); 2801 2802 pmu->flush_branch_stack(); 2803 2804 perf_pmu_enable(pmu); 2805 2806 perf_ctx_unlock(cpuctx, cpuctx->task_ctx); 2807 } 2808 } 2809 2810 rcu_read_unlock(); 2811 2812 local_irq_restore(flags); 2813 } 2814 2815 /* 2816 * Called from scheduler to add the events of the current task 2817 * with interrupts disabled. 2818 * 2819 * We restore the event value and then enable it. 2820 * 2821 * This does not protect us against NMI, but enable() 2822 * sets the enabled bit in the control field of event _before_ 2823 * accessing the event control register. If a NMI hits, then it will 2824 * keep the event running. 2825 */ 2826 void __perf_event_task_sched_in(struct task_struct *prev, 2827 struct task_struct *task) 2828 { 2829 struct perf_event_context *ctx; 2830 int ctxn; 2831 2832 for_each_task_context_nr(ctxn) { 2833 ctx = task->perf_event_ctxp[ctxn]; 2834 if (likely(!ctx)) 2835 continue; 2836 2837 perf_event_context_sched_in(ctx, task); 2838 } 2839 /* 2840 * if cgroup events exist on this CPU, then we need 2841 * to check if we have to switch in PMU state. 2842 * cgroup event are system-wide mode only 2843 */ 2844 if (atomic_read(this_cpu_ptr(&perf_cgroup_events))) 2845 perf_cgroup_sched_in(prev, task); 2846 2847 /* check for system-wide branch_stack events */ 2848 if (atomic_read(this_cpu_ptr(&perf_branch_stack_events))) 2849 perf_branch_stack_sched_in(prev, task); 2850 } 2851 2852 static u64 perf_calculate_period(struct perf_event *event, u64 nsec, u64 count) 2853 { 2854 u64 frequency = event->attr.sample_freq; 2855 u64 sec = NSEC_PER_SEC; 2856 u64 divisor, dividend; 2857 2858 int count_fls, nsec_fls, frequency_fls, sec_fls; 2859 2860 count_fls = fls64(count); 2861 nsec_fls = fls64(nsec); 2862 frequency_fls = fls64(frequency); 2863 sec_fls = 30; 2864 2865 /* 2866 * We got @count in @nsec, with a target of sample_freq HZ 2867 * the target period becomes: 2868 * 2869 * @count * 10^9 2870 * period = ------------------- 2871 * @nsec * sample_freq 2872 * 2873 */ 2874 2875 /* 2876 * Reduce accuracy by one bit such that @a and @b converge 2877 * to a similar magnitude. 2878 */ 2879 #define REDUCE_FLS(a, b) \ 2880 do { \ 2881 if (a##_fls > b##_fls) { \ 2882 a >>= 1; \ 2883 a##_fls--; \ 2884 } else { \ 2885 b >>= 1; \ 2886 b##_fls--; \ 2887 } \ 2888 } while (0) 2889 2890 /* 2891 * Reduce accuracy until either term fits in a u64, then proceed with 2892 * the other, so that finally we can do a u64/u64 division. 2893 */ 2894 while (count_fls + sec_fls > 64 && nsec_fls + frequency_fls > 64) { 2895 REDUCE_FLS(nsec, frequency); 2896 REDUCE_FLS(sec, count); 2897 } 2898 2899 if (count_fls + sec_fls > 64) { 2900 divisor = nsec * frequency; 2901 2902 while (count_fls + sec_fls > 64) { 2903 REDUCE_FLS(count, sec); 2904 divisor >>= 1; 2905 } 2906 2907 dividend = count * sec; 2908 } else { 2909 dividend = count * sec; 2910 2911 while (nsec_fls + frequency_fls > 64) { 2912 REDUCE_FLS(nsec, frequency); 2913 dividend >>= 1; 2914 } 2915 2916 divisor = nsec * frequency; 2917 } 2918 2919 if (!divisor) 2920 return dividend; 2921 2922 return div64_u64(dividend, divisor); 2923 } 2924 2925 static DEFINE_PER_CPU(int, perf_throttled_count); 2926 static DEFINE_PER_CPU(u64, perf_throttled_seq); 2927 2928 static void perf_adjust_period(struct perf_event *event, u64 nsec, u64 count, bool disable) 2929 { 2930 struct hw_perf_event *hwc = &event->hw; 2931 s64 period, sample_period; 2932 s64 delta; 2933 2934 period = perf_calculate_period(event, nsec, count); 2935 2936 delta = (s64)(period - hwc->sample_period); 2937 delta = (delta + 7) / 8; /* low pass filter */ 2938 2939 sample_period = hwc->sample_period + delta; 2940 2941 if (!sample_period) 2942 sample_period = 1; 2943 2944 hwc->sample_period = sample_period; 2945 2946 if (local64_read(&hwc->period_left) > 8*sample_period) { 2947 if (disable) 2948 event->pmu->stop(event, PERF_EF_UPDATE); 2949 2950 local64_set(&hwc->period_left, 0); 2951 2952 if (disable) 2953 event->pmu->start(event, PERF_EF_RELOAD); 2954 } 2955 } 2956 2957 /* 2958 * combine freq adjustment with unthrottling to avoid two passes over the 2959 * events. At the same time, make sure, having freq events does not change 2960 * the rate of unthrottling as that would introduce bias. 2961 */ 2962 static void perf_adjust_freq_unthr_context(struct perf_event_context *ctx, 2963 int needs_unthr) 2964 { 2965 struct perf_event *event; 2966 struct hw_perf_event *hwc; 2967 u64 now, period = TICK_NSEC; 2968 s64 delta; 2969 2970 /* 2971 * only need to iterate over all events iff: 2972 * - context have events in frequency mode (needs freq adjust) 2973 * - there are events to unthrottle on this cpu 2974 */ 2975 if (!(ctx->nr_freq || needs_unthr)) 2976 return; 2977 2978 raw_spin_lock(&ctx->lock); 2979 perf_pmu_disable(ctx->pmu); 2980 2981 list_for_each_entry_rcu(event, &ctx->event_list, event_entry) { 2982 if (event->state != PERF_EVENT_STATE_ACTIVE) 2983 continue; 2984 2985 if (!event_filter_match(event)) 2986 continue; 2987 2988 perf_pmu_disable(event->pmu); 2989 2990 hwc = &event->hw; 2991 2992 if (hwc->interrupts == MAX_INTERRUPTS) { 2993 hwc->interrupts = 0; 2994 perf_log_throttle(event, 1); 2995 event->pmu->start(event, 0); 2996 } 2997 2998 if (!event->attr.freq || !event->attr.sample_freq) 2999 goto next; 3000 3001 /* 3002 * stop the event and update event->count 3003 */ 3004 event->pmu->stop(event, PERF_EF_UPDATE); 3005 3006 now = local64_read(&event->count); 3007 delta = now - hwc->freq_count_stamp; 3008 hwc->freq_count_stamp = now; 3009 3010 /* 3011 * restart the event 3012 * reload only if value has changed 3013 * we have stopped the event so tell that 3014 * to perf_adjust_period() to avoid stopping it 3015 * twice. 3016 */ 3017 if (delta > 0) 3018 perf_adjust_period(event, period, delta, false); 3019 3020 event->pmu->start(event, delta > 0 ? PERF_EF_RELOAD : 0); 3021 next: 3022 perf_pmu_enable(event->pmu); 3023 } 3024 3025 perf_pmu_enable(ctx->pmu); 3026 raw_spin_unlock(&ctx->lock); 3027 } 3028 3029 /* 3030 * Round-robin a context's events: 3031 */ 3032 static void rotate_ctx(struct perf_event_context *ctx) 3033 { 3034 /* 3035 * Rotate the first entry last of non-pinned groups. Rotation might be 3036 * disabled by the inheritance code. 3037 */ 3038 if (!ctx->rotate_disable) 3039 list_rotate_left(&ctx->flexible_groups); 3040 } 3041 3042 static int perf_rotate_context(struct perf_cpu_context *cpuctx) 3043 { 3044 struct perf_event_context *ctx = NULL; 3045 int rotate = 0; 3046 3047 if (cpuctx->ctx.nr_events) { 3048 if (cpuctx->ctx.nr_events != cpuctx->ctx.nr_active) 3049 rotate = 1; 3050 } 3051 3052 ctx = cpuctx->task_ctx; 3053 if (ctx && ctx->nr_events) { 3054 if (ctx->nr_events != ctx->nr_active) 3055 rotate = 1; 3056 } 3057 3058 if (!rotate) 3059 goto done; 3060 3061 perf_ctx_lock(cpuctx, cpuctx->task_ctx); 3062 perf_pmu_disable(cpuctx->ctx.pmu); 3063 3064 cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE); 3065 if (ctx) 3066 ctx_sched_out(ctx, cpuctx, EVENT_FLEXIBLE); 3067 3068 rotate_ctx(&cpuctx->ctx); 3069 if (ctx) 3070 rotate_ctx(ctx); 3071 3072 perf_event_sched_in(cpuctx, ctx, current); 3073 3074 perf_pmu_enable(cpuctx->ctx.pmu); 3075 perf_ctx_unlock(cpuctx, cpuctx->task_ctx); 3076 done: 3077 3078 return rotate; 3079 } 3080 3081 #ifdef CONFIG_NO_HZ_FULL 3082 bool perf_event_can_stop_tick(void) 3083 { 3084 if (atomic_read(&nr_freq_events) || 3085 __this_cpu_read(perf_throttled_count)) 3086 return false; 3087 else 3088 return true; 3089 } 3090 #endif 3091 3092 void perf_event_task_tick(void) 3093 { 3094 struct list_head *head = this_cpu_ptr(&active_ctx_list); 3095 struct perf_event_context *ctx, *tmp; 3096 int throttled; 3097 3098 WARN_ON(!irqs_disabled()); 3099 3100 __this_cpu_inc(perf_throttled_seq); 3101 throttled = __this_cpu_xchg(perf_throttled_count, 0); 3102 3103 list_for_each_entry_safe(ctx, tmp, head, active_ctx_list) 3104 perf_adjust_freq_unthr_context(ctx, throttled); 3105 } 3106 3107 static int event_enable_on_exec(struct perf_event *event, 3108 struct perf_event_context *ctx) 3109 { 3110 if (!event->attr.enable_on_exec) 3111 return 0; 3112 3113 event->attr.enable_on_exec = 0; 3114 if (event->state >= PERF_EVENT_STATE_INACTIVE) 3115 return 0; 3116 3117 __perf_event_mark_enabled(event); 3118 3119 return 1; 3120 } 3121 3122 /* 3123 * Enable all of a task's events that have been marked enable-on-exec. 3124 * This expects task == current. 3125 */ 3126 static void perf_event_enable_on_exec(struct perf_event_context *ctx) 3127 { 3128 struct perf_event_context *clone_ctx = NULL; 3129 struct perf_event *event; 3130 unsigned long flags; 3131 int enabled = 0; 3132 int ret; 3133 3134 local_irq_save(flags); 3135 if (!ctx || !ctx->nr_events) 3136 goto out; 3137 3138 /* 3139 * We must ctxsw out cgroup events to avoid conflict 3140 * when invoking perf_task_event_sched_in() later on 3141 * in this function. Otherwise we end up trying to 3142 * ctxswin cgroup events which are already scheduled 3143 * in. 3144 */ 3145 perf_cgroup_sched_out(current, NULL); 3146 3147 raw_spin_lock(&ctx->lock); 3148 task_ctx_sched_out(ctx); 3149 3150 list_for_each_entry(event, &ctx->event_list, event_entry) { 3151 ret = event_enable_on_exec(event, ctx); 3152 if (ret) 3153 enabled = 1; 3154 } 3155 3156 /* 3157 * Unclone this context if we enabled any event. 3158 */ 3159 if (enabled) 3160 clone_ctx = unclone_ctx(ctx); 3161 3162 raw_spin_unlock(&ctx->lock); 3163 3164 /* 3165 * Also calls ctxswin for cgroup events, if any: 3166 */ 3167 perf_event_context_sched_in(ctx, ctx->task); 3168 out: 3169 local_irq_restore(flags); 3170 3171 if (clone_ctx) 3172 put_ctx(clone_ctx); 3173 } 3174 3175 void perf_event_exec(void) 3176 { 3177 struct perf_event_context *ctx; 3178 int ctxn; 3179 3180 rcu_read_lock(); 3181 for_each_task_context_nr(ctxn) { 3182 ctx = current->perf_event_ctxp[ctxn]; 3183 if (!ctx) 3184 continue; 3185 3186 perf_event_enable_on_exec(ctx); 3187 } 3188 rcu_read_unlock(); 3189 } 3190 3191 /* 3192 * Cross CPU call to read the hardware event 3193 */ 3194 static void __perf_event_read(void *info) 3195 { 3196 struct perf_event *event = info; 3197 struct perf_event_context *ctx = event->ctx; 3198 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx); 3199 3200 /* 3201 * If this is a task context, we need to check whether it is 3202 * the current task context of this cpu. If not it has been 3203 * scheduled out before the smp call arrived. In that case 3204 * event->count would have been updated to a recent sample 3205 * when the event was scheduled out. 3206 */ 3207 if (ctx->task && cpuctx->task_ctx != ctx) 3208 return; 3209 3210 raw_spin_lock(&ctx->lock); 3211 if (ctx->is_active) { 3212 update_context_time(ctx); 3213 update_cgrp_time_from_event(event); 3214 } 3215 update_event_times(event); 3216 if (event->state == PERF_EVENT_STATE_ACTIVE) 3217 event->pmu->read(event); 3218 raw_spin_unlock(&ctx->lock); 3219 } 3220 3221 static inline u64 perf_event_count(struct perf_event *event) 3222 { 3223 return local64_read(&event->count) + atomic64_read(&event->child_count); 3224 } 3225 3226 static u64 perf_event_read(struct perf_event *event) 3227 { 3228 /* 3229 * If event is enabled and currently active on a CPU, update the 3230 * value in the event structure: 3231 */ 3232 if (event->state == PERF_EVENT_STATE_ACTIVE) { 3233 smp_call_function_single(event->oncpu, 3234 __perf_event_read, event, 1); 3235 } else if (event->state == PERF_EVENT_STATE_INACTIVE) { 3236 struct perf_event_context *ctx = event->ctx; 3237 unsigned long flags; 3238 3239 raw_spin_lock_irqsave(&ctx->lock, flags); 3240 /* 3241 * may read while context is not active 3242 * (e.g., thread is blocked), in that case 3243 * we cannot update context time 3244 */ 3245 if (ctx->is_active) { 3246 update_context_time(ctx); 3247 update_cgrp_time_from_event(event); 3248 } 3249 update_event_times(event); 3250 raw_spin_unlock_irqrestore(&ctx->lock, flags); 3251 } 3252 3253 return perf_event_count(event); 3254 } 3255 3256 /* 3257 * Initialize the perf_event context in a task_struct: 3258 */ 3259 static void __perf_event_init_context(struct perf_event_context *ctx) 3260 { 3261 raw_spin_lock_init(&ctx->lock); 3262 mutex_init(&ctx->mutex); 3263 INIT_LIST_HEAD(&ctx->active_ctx_list); 3264 INIT_LIST_HEAD(&ctx->pinned_groups); 3265 INIT_LIST_HEAD(&ctx->flexible_groups); 3266 INIT_LIST_HEAD(&ctx->event_list); 3267 atomic_set(&ctx->refcount, 1); 3268 INIT_DELAYED_WORK(&ctx->orphans_remove, orphans_remove_work); 3269 } 3270 3271 static struct perf_event_context * 3272 alloc_perf_context(struct pmu *pmu, struct task_struct *task) 3273 { 3274 struct perf_event_context *ctx; 3275 3276 ctx = kzalloc(sizeof(struct perf_event_context), GFP_KERNEL); 3277 if (!ctx) 3278 return NULL; 3279 3280 __perf_event_init_context(ctx); 3281 if (task) { 3282 ctx->task = task; 3283 get_task_struct(task); 3284 } 3285 ctx->pmu = pmu; 3286 3287 return ctx; 3288 } 3289 3290 static struct task_struct * 3291 find_lively_task_by_vpid(pid_t vpid) 3292 { 3293 struct task_struct *task; 3294 int err; 3295 3296 rcu_read_lock(); 3297 if (!vpid) 3298 task = current; 3299 else 3300 task = find_task_by_vpid(vpid); 3301 if (task) 3302 get_task_struct(task); 3303 rcu_read_unlock(); 3304 3305 if (!task) 3306 return ERR_PTR(-ESRCH); 3307 3308 /* Reuse ptrace permission checks for now. */ 3309 err = -EACCES; 3310 if (!ptrace_may_access(task, PTRACE_MODE_READ)) 3311 goto errout; 3312 3313 return task; 3314 errout: 3315 put_task_struct(task); 3316 return ERR_PTR(err); 3317 3318 } 3319 3320 /* 3321 * Returns a matching context with refcount and pincount. 3322 */ 3323 static struct perf_event_context * 3324 find_get_context(struct pmu *pmu, struct task_struct *task, int cpu) 3325 { 3326 struct perf_event_context *ctx, *clone_ctx = NULL; 3327 struct perf_cpu_context *cpuctx; 3328 unsigned long flags; 3329 int ctxn, err; 3330 3331 if (!task) { 3332 /* Must be root to operate on a CPU event: */ 3333 if (perf_paranoid_cpu() && !capable(CAP_SYS_ADMIN)) 3334 return ERR_PTR(-EACCES); 3335 3336 /* 3337 * We could be clever and allow to attach a event to an 3338 * offline CPU and activate it when the CPU comes up, but 3339 * that's for later. 3340 */ 3341 if (!cpu_online(cpu)) 3342 return ERR_PTR(-ENODEV); 3343 3344 cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu); 3345 ctx = &cpuctx->ctx; 3346 get_ctx(ctx); 3347 ++ctx->pin_count; 3348 3349 return ctx; 3350 } 3351 3352 err = -EINVAL; 3353 ctxn = pmu->task_ctx_nr; 3354 if (ctxn < 0) 3355 goto errout; 3356 3357 retry: 3358 ctx = perf_lock_task_context(task, ctxn, &flags); 3359 if (ctx) { 3360 clone_ctx = unclone_ctx(ctx); 3361 ++ctx->pin_count; 3362 raw_spin_unlock_irqrestore(&ctx->lock, flags); 3363 3364 if (clone_ctx) 3365 put_ctx(clone_ctx); 3366 } else { 3367 ctx = alloc_perf_context(pmu, task); 3368 err = -ENOMEM; 3369 if (!ctx) 3370 goto errout; 3371 3372 err = 0; 3373 mutex_lock(&task->perf_event_mutex); 3374 /* 3375 * If it has already passed perf_event_exit_task(). 3376 * we must see PF_EXITING, it takes this mutex too. 3377 */ 3378 if (task->flags & PF_EXITING) 3379 err = -ESRCH; 3380 else if (task->perf_event_ctxp[ctxn]) 3381 err = -EAGAIN; 3382 else { 3383 get_ctx(ctx); 3384 ++ctx->pin_count; 3385 rcu_assign_pointer(task->perf_event_ctxp[ctxn], ctx); 3386 } 3387 mutex_unlock(&task->perf_event_mutex); 3388 3389 if (unlikely(err)) { 3390 put_ctx(ctx); 3391 3392 if (err == -EAGAIN) 3393 goto retry; 3394 goto errout; 3395 } 3396 } 3397 3398 return ctx; 3399 3400 errout: 3401 return ERR_PTR(err); 3402 } 3403 3404 static void perf_event_free_filter(struct perf_event *event); 3405 3406 static void free_event_rcu(struct rcu_head *head) 3407 { 3408 struct perf_event *event; 3409 3410 event = container_of(head, struct perf_event, rcu_head); 3411 if (event->ns) 3412 put_pid_ns(event->ns); 3413 perf_event_free_filter(event); 3414 kfree(event); 3415 } 3416 3417 static void ring_buffer_put(struct ring_buffer *rb); 3418 static void ring_buffer_attach(struct perf_event *event, 3419 struct ring_buffer *rb); 3420 3421 static void unaccount_event_cpu(struct perf_event *event, int cpu) 3422 { 3423 if (event->parent) 3424 return; 3425 3426 if (has_branch_stack(event)) { 3427 if (!(event->attach_state & PERF_ATTACH_TASK)) 3428 atomic_dec(&per_cpu(perf_branch_stack_events, cpu)); 3429 } 3430 if (is_cgroup_event(event)) 3431 atomic_dec(&per_cpu(perf_cgroup_events, cpu)); 3432 } 3433 3434 static void unaccount_event(struct perf_event *event) 3435 { 3436 if (event->parent) 3437 return; 3438 3439 if (event->attach_state & PERF_ATTACH_TASK) 3440 static_key_slow_dec_deferred(&perf_sched_events); 3441 if (event->attr.mmap || event->attr.mmap_data) 3442 atomic_dec(&nr_mmap_events); 3443 if (event->attr.comm) 3444 atomic_dec(&nr_comm_events); 3445 if (event->attr.task) 3446 atomic_dec(&nr_task_events); 3447 if (event->attr.freq) 3448 atomic_dec(&nr_freq_events); 3449 if (is_cgroup_event(event)) 3450 static_key_slow_dec_deferred(&perf_sched_events); 3451 if (has_branch_stack(event)) 3452 static_key_slow_dec_deferred(&perf_sched_events); 3453 3454 unaccount_event_cpu(event, event->cpu); 3455 } 3456 3457 static void __free_event(struct perf_event *event) 3458 { 3459 if (!event->parent) { 3460 if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) 3461 put_callchain_buffers(); 3462 } 3463 3464 if (event->destroy) 3465 event->destroy(event); 3466 3467 if (event->ctx) 3468 put_ctx(event->ctx); 3469 3470 if (event->pmu) 3471 module_put(event->pmu->module); 3472 3473 call_rcu(&event->rcu_head, free_event_rcu); 3474 } 3475 3476 static void _free_event(struct perf_event *event) 3477 { 3478 irq_work_sync(&event->pending); 3479 3480 unaccount_event(event); 3481 3482 if (event->rb) { 3483 /* 3484 * Can happen when we close an event with re-directed output. 3485 * 3486 * Since we have a 0 refcount, perf_mmap_close() will skip 3487 * over us; possibly making our ring_buffer_put() the last. 3488 */ 3489 mutex_lock(&event->mmap_mutex); 3490 ring_buffer_attach(event, NULL); 3491 mutex_unlock(&event->mmap_mutex); 3492 } 3493 3494 if (is_cgroup_event(event)) 3495 perf_detach_cgroup(event); 3496 3497 __free_event(event); 3498 } 3499 3500 /* 3501 * Used to free events which have a known refcount of 1, such as in error paths 3502 * where the event isn't exposed yet and inherited events. 3503 */ 3504 static void free_event(struct perf_event *event) 3505 { 3506 if (WARN(atomic_long_cmpxchg(&event->refcount, 1, 0) != 1, 3507 "unexpected event refcount: %ld; ptr=%p\n", 3508 atomic_long_read(&event->refcount), event)) { 3509 /* leak to avoid use-after-free */ 3510 return; 3511 } 3512 3513 _free_event(event); 3514 } 3515 3516 /* 3517 * Remove user event from the owner task. 3518 */ 3519 static void perf_remove_from_owner(struct perf_event *event) 3520 { 3521 struct task_struct *owner; 3522 3523 rcu_read_lock(); 3524 owner = ACCESS_ONCE(event->owner); 3525 /* 3526 * Matches the smp_wmb() in perf_event_exit_task(). If we observe 3527 * !owner it means the list deletion is complete and we can indeed 3528 * free this event, otherwise we need to serialize on 3529 * owner->perf_event_mutex. 3530 */ 3531 smp_read_barrier_depends(); 3532 if (owner) { 3533 /* 3534 * Since delayed_put_task_struct() also drops the last 3535 * task reference we can safely take a new reference 3536 * while holding the rcu_read_lock(). 3537 */ 3538 get_task_struct(owner); 3539 } 3540 rcu_read_unlock(); 3541 3542 if (owner) { 3543 /* 3544 * If we're here through perf_event_exit_task() we're already 3545 * holding ctx->mutex which would be an inversion wrt. the 3546 * normal lock order. 3547 * 3548 * However we can safely take this lock because its the child 3549 * ctx->mutex. 3550 */ 3551 mutex_lock_nested(&owner->perf_event_mutex, SINGLE_DEPTH_NESTING); 3552 3553 /* 3554 * We have to re-check the event->owner field, if it is cleared 3555 * we raced with perf_event_exit_task(), acquiring the mutex 3556 * ensured they're done, and we can proceed with freeing the 3557 * event. 3558 */ 3559 if (event->owner) 3560 list_del_init(&event->owner_entry); 3561 mutex_unlock(&owner->perf_event_mutex); 3562 put_task_struct(owner); 3563 } 3564 } 3565 3566 /* 3567 * Called when the last reference to the file is gone. 3568 */ 3569 static void put_event(struct perf_event *event) 3570 { 3571 struct perf_event_context *ctx; 3572 3573 if (!atomic_long_dec_and_test(&event->refcount)) 3574 return; 3575 3576 if (!is_kernel_event(event)) 3577 perf_remove_from_owner(event); 3578 3579 /* 3580 * There are two ways this annotation is useful: 3581 * 3582 * 1) there is a lock recursion from perf_event_exit_task 3583 * see the comment there. 3584 * 3585 * 2) there is a lock-inversion with mmap_sem through 3586 * perf_event_read_group(), which takes faults while 3587 * holding ctx->mutex, however this is called after 3588 * the last filedesc died, so there is no possibility 3589 * to trigger the AB-BA case. 3590 */ 3591 ctx = perf_event_ctx_lock_nested(event, SINGLE_DEPTH_NESTING); 3592 WARN_ON_ONCE(ctx->parent_ctx); 3593 perf_remove_from_context(event, true); 3594 mutex_unlock(&ctx->mutex); 3595 3596 _free_event(event); 3597 } 3598 3599 int perf_event_release_kernel(struct perf_event *event) 3600 { 3601 put_event(event); 3602 return 0; 3603 } 3604 EXPORT_SYMBOL_GPL(perf_event_release_kernel); 3605 3606 static int perf_release(struct inode *inode, struct file *file) 3607 { 3608 put_event(file->private_data); 3609 return 0; 3610 } 3611 3612 /* 3613 * Remove all orphanes events from the context. 3614 */ 3615 static void orphans_remove_work(struct work_struct *work) 3616 { 3617 struct perf_event_context *ctx; 3618 struct perf_event *event, *tmp; 3619 3620 ctx = container_of(work, struct perf_event_context, 3621 orphans_remove.work); 3622 3623 mutex_lock(&ctx->mutex); 3624 list_for_each_entry_safe(event, tmp, &ctx->event_list, event_entry) { 3625 struct perf_event *parent_event = event->parent; 3626 3627 if (!is_orphaned_child(event)) 3628 continue; 3629 3630 perf_remove_from_context(event, true); 3631 3632 mutex_lock(&parent_event->child_mutex); 3633 list_del_init(&event->child_list); 3634 mutex_unlock(&parent_event->child_mutex); 3635 3636 free_event(event); 3637 put_event(parent_event); 3638 } 3639 3640 raw_spin_lock_irq(&ctx->lock); 3641 ctx->orphans_remove_sched = false; 3642 raw_spin_unlock_irq(&ctx->lock); 3643 mutex_unlock(&ctx->mutex); 3644 3645 put_ctx(ctx); 3646 } 3647 3648 u64 perf_event_read_value(struct perf_event *event, u64 *enabled, u64 *running) 3649 { 3650 struct perf_event *child; 3651 u64 total = 0; 3652 3653 *enabled = 0; 3654 *running = 0; 3655 3656 mutex_lock(&event->child_mutex); 3657 total += perf_event_read(event); 3658 *enabled += event->total_time_enabled + 3659 atomic64_read(&event->child_total_time_enabled); 3660 *running += event->total_time_running + 3661 atomic64_read(&event->child_total_time_running); 3662 3663 list_for_each_entry(child, &event->child_list, child_list) { 3664 total += perf_event_read(child); 3665 *enabled += child->total_time_enabled; 3666 *running += child->total_time_running; 3667 } 3668 mutex_unlock(&event->child_mutex); 3669 3670 return total; 3671 } 3672 EXPORT_SYMBOL_GPL(perf_event_read_value); 3673 3674 static int perf_event_read_group(struct perf_event *event, 3675 u64 read_format, char __user *buf) 3676 { 3677 struct perf_event *leader = event->group_leader, *sub; 3678 struct perf_event_context *ctx = leader->ctx; 3679 int n = 0, size = 0, ret; 3680 u64 count, enabled, running; 3681 u64 values[5]; 3682 3683 lockdep_assert_held(&ctx->mutex); 3684 3685 count = perf_event_read_value(leader, &enabled, &running); 3686 3687 values[n++] = 1 + leader->nr_siblings; 3688 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) 3689 values[n++] = enabled; 3690 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) 3691 values[n++] = running; 3692 values[n++] = count; 3693 if (read_format & PERF_FORMAT_ID) 3694 values[n++] = primary_event_id(leader); 3695 3696 size = n * sizeof(u64); 3697 3698 if (copy_to_user(buf, values, size)) 3699 return -EFAULT; 3700 3701 ret = size; 3702 3703 list_for_each_entry(sub, &leader->sibling_list, group_entry) { 3704 n = 0; 3705 3706 values[n++] = perf_event_read_value(sub, &enabled, &running); 3707 if (read_format & PERF_FORMAT_ID) 3708 values[n++] = primary_event_id(sub); 3709 3710 size = n * sizeof(u64); 3711 3712 if (copy_to_user(buf + ret, values, size)) { 3713 return -EFAULT; 3714 } 3715 3716 ret += size; 3717 } 3718 3719 return ret; 3720 } 3721 3722 static int perf_event_read_one(struct perf_event *event, 3723 u64 read_format, char __user *buf) 3724 { 3725 u64 enabled, running; 3726 u64 values[4]; 3727 int n = 0; 3728 3729 values[n++] = perf_event_read_value(event, &enabled, &running); 3730 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) 3731 values[n++] = enabled; 3732 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) 3733 values[n++] = running; 3734 if (read_format & PERF_FORMAT_ID) 3735 values[n++] = primary_event_id(event); 3736 3737 if (copy_to_user(buf, values, n * sizeof(u64))) 3738 return -EFAULT; 3739 3740 return n * sizeof(u64); 3741 } 3742 3743 static bool is_event_hup(struct perf_event *event) 3744 { 3745 bool no_children; 3746 3747 if (event->state != PERF_EVENT_STATE_EXIT) 3748 return false; 3749 3750 mutex_lock(&event->child_mutex); 3751 no_children = list_empty(&event->child_list); 3752 mutex_unlock(&event->child_mutex); 3753 return no_children; 3754 } 3755 3756 /* 3757 * Read the performance event - simple non blocking version for now 3758 */ 3759 static ssize_t 3760 perf_read_hw(struct perf_event *event, char __user *buf, size_t count) 3761 { 3762 u64 read_format = event->attr.read_format; 3763 int ret; 3764 3765 /* 3766 * Return end-of-file for a read on a event that is in 3767 * error state (i.e. because it was pinned but it couldn't be 3768 * scheduled on to the CPU at some point). 3769 */ 3770 if (event->state == PERF_EVENT_STATE_ERROR) 3771 return 0; 3772 3773 if (count < event->read_size) 3774 return -ENOSPC; 3775 3776 WARN_ON_ONCE(event->ctx->parent_ctx); 3777 if (read_format & PERF_FORMAT_GROUP) 3778 ret = perf_event_read_group(event, read_format, buf); 3779 else 3780 ret = perf_event_read_one(event, read_format, buf); 3781 3782 return ret; 3783 } 3784 3785 static ssize_t 3786 perf_read(struct file *file, char __user *buf, size_t count, loff_t *ppos) 3787 { 3788 struct perf_event *event = file->private_data; 3789 struct perf_event_context *ctx; 3790 int ret; 3791 3792 ctx = perf_event_ctx_lock(event); 3793 ret = perf_read_hw(event, buf, count); 3794 perf_event_ctx_unlock(event, ctx); 3795 3796 return ret; 3797 } 3798 3799 static unsigned int perf_poll(struct file *file, poll_table *wait) 3800 { 3801 struct perf_event *event = file->private_data; 3802 struct ring_buffer *rb; 3803 unsigned int events = POLLHUP; 3804 3805 poll_wait(file, &event->waitq, wait); 3806 3807 if (is_event_hup(event)) 3808 return events; 3809 3810 /* 3811 * Pin the event->rb by taking event->mmap_mutex; otherwise 3812 * perf_event_set_output() can swizzle our rb and make us miss wakeups. 3813 */ 3814 mutex_lock(&event->mmap_mutex); 3815 rb = event->rb; 3816 if (rb) 3817 events = atomic_xchg(&rb->poll, 0); 3818 mutex_unlock(&event->mmap_mutex); 3819 return events; 3820 } 3821 3822 static void _perf_event_reset(struct perf_event *event) 3823 { 3824 (void)perf_event_read(event); 3825 local64_set(&event->count, 0); 3826 perf_event_update_userpage(event); 3827 } 3828 3829 /* 3830 * Holding the top-level event's child_mutex means that any 3831 * descendant process that has inherited this event will block 3832 * in sync_child_event if it goes to exit, thus satisfying the 3833 * task existence requirements of perf_event_enable/disable. 3834 */ 3835 static void perf_event_for_each_child(struct perf_event *event, 3836 void (*func)(struct perf_event *)) 3837 { 3838 struct perf_event *child; 3839 3840 WARN_ON_ONCE(event->ctx->parent_ctx); 3841 3842 mutex_lock(&event->child_mutex); 3843 func(event); 3844 list_for_each_entry(child, &event->child_list, child_list) 3845 func(child); 3846 mutex_unlock(&event->child_mutex); 3847 } 3848 3849 static void perf_event_for_each(struct perf_event *event, 3850 void (*func)(struct perf_event *)) 3851 { 3852 struct perf_event_context *ctx = event->ctx; 3853 struct perf_event *sibling; 3854 3855 lockdep_assert_held(&ctx->mutex); 3856 3857 event = event->group_leader; 3858 3859 perf_event_for_each_child(event, func); 3860 list_for_each_entry(sibling, &event->sibling_list, group_entry) 3861 perf_event_for_each_child(sibling, func); 3862 } 3863 3864 static int perf_event_period(struct perf_event *event, u64 __user *arg) 3865 { 3866 struct perf_event_context *ctx = event->ctx; 3867 int ret = 0, active; 3868 u64 value; 3869 3870 if (!is_sampling_event(event)) 3871 return -EINVAL; 3872 3873 if (copy_from_user(&value, arg, sizeof(value))) 3874 return -EFAULT; 3875 3876 if (!value) 3877 return -EINVAL; 3878 3879 raw_spin_lock_irq(&ctx->lock); 3880 if (event->attr.freq) { 3881 if (value > sysctl_perf_event_sample_rate) { 3882 ret = -EINVAL; 3883 goto unlock; 3884 } 3885 3886 event->attr.sample_freq = value; 3887 } else { 3888 event->attr.sample_period = value; 3889 event->hw.sample_period = value; 3890 } 3891 3892 active = (event->state == PERF_EVENT_STATE_ACTIVE); 3893 if (active) { 3894 perf_pmu_disable(ctx->pmu); 3895 event->pmu->stop(event, PERF_EF_UPDATE); 3896 } 3897 3898 local64_set(&event->hw.period_left, 0); 3899 3900 if (active) { 3901 event->pmu->start(event, PERF_EF_RELOAD); 3902 perf_pmu_enable(ctx->pmu); 3903 } 3904 3905 unlock: 3906 raw_spin_unlock_irq(&ctx->lock); 3907 3908 return ret; 3909 } 3910 3911 static const struct file_operations perf_fops; 3912 3913 static inline int perf_fget_light(int fd, struct fd *p) 3914 { 3915 struct fd f = fdget(fd); 3916 if (!f.file) 3917 return -EBADF; 3918 3919 if (f.file->f_op != &perf_fops) { 3920 fdput(f); 3921 return -EBADF; 3922 } 3923 *p = f; 3924 return 0; 3925 } 3926 3927 static int perf_event_set_output(struct perf_event *event, 3928 struct perf_event *output_event); 3929 static int perf_event_set_filter(struct perf_event *event, void __user *arg); 3930 3931 static long _perf_ioctl(struct perf_event *event, unsigned int cmd, unsigned long arg) 3932 { 3933 void (*func)(struct perf_event *); 3934 u32 flags = arg; 3935 3936 switch (cmd) { 3937 case PERF_EVENT_IOC_ENABLE: 3938 func = _perf_event_enable; 3939 break; 3940 case PERF_EVENT_IOC_DISABLE: 3941 func = _perf_event_disable; 3942 break; 3943 case PERF_EVENT_IOC_RESET: 3944 func = _perf_event_reset; 3945 break; 3946 3947 case PERF_EVENT_IOC_REFRESH: 3948 return _perf_event_refresh(event, arg); 3949 3950 case PERF_EVENT_IOC_PERIOD: 3951 return perf_event_period(event, (u64 __user *)arg); 3952 3953 case PERF_EVENT_IOC_ID: 3954 { 3955 u64 id = primary_event_id(event); 3956 3957 if (copy_to_user((void __user *)arg, &id, sizeof(id))) 3958 return -EFAULT; 3959 return 0; 3960 } 3961 3962 case PERF_EVENT_IOC_SET_OUTPUT: 3963 { 3964 int ret; 3965 if (arg != -1) { 3966 struct perf_event *output_event; 3967 struct fd output; 3968 ret = perf_fget_light(arg, &output); 3969 if (ret) 3970 return ret; 3971 output_event = output.file->private_data; 3972 ret = perf_event_set_output(event, output_event); 3973 fdput(output); 3974 } else { 3975 ret = perf_event_set_output(event, NULL); 3976 } 3977 return ret; 3978 } 3979 3980 case PERF_EVENT_IOC_SET_FILTER: 3981 return perf_event_set_filter(event, (void __user *)arg); 3982 3983 default: 3984 return -ENOTTY; 3985 } 3986 3987 if (flags & PERF_IOC_FLAG_GROUP) 3988 perf_event_for_each(event, func); 3989 else 3990 perf_event_for_each_child(event, func); 3991 3992 return 0; 3993 } 3994 3995 static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg) 3996 { 3997 struct perf_event *event = file->private_data; 3998 struct perf_event_context *ctx; 3999 long ret; 4000 4001 ctx = perf_event_ctx_lock(event); 4002 ret = _perf_ioctl(event, cmd, arg); 4003 perf_event_ctx_unlock(event, ctx); 4004 4005 return ret; 4006 } 4007 4008 #ifdef CONFIG_COMPAT 4009 static long perf_compat_ioctl(struct file *file, unsigned int cmd, 4010 unsigned long arg) 4011 { 4012 switch (_IOC_NR(cmd)) { 4013 case _IOC_NR(PERF_EVENT_IOC_SET_FILTER): 4014 case _IOC_NR(PERF_EVENT_IOC_ID): 4015 /* Fix up pointer size (usually 4 -> 8 in 32-on-64-bit case */ 4016 if (_IOC_SIZE(cmd) == sizeof(compat_uptr_t)) { 4017 cmd &= ~IOCSIZE_MASK; 4018 cmd |= sizeof(void *) << IOCSIZE_SHIFT; 4019 } 4020 break; 4021 } 4022 return perf_ioctl(file, cmd, arg); 4023 } 4024 #else 4025 # define perf_compat_ioctl NULL 4026 #endif 4027 4028 int perf_event_task_enable(void) 4029 { 4030 struct perf_event_context *ctx; 4031 struct perf_event *event; 4032 4033 mutex_lock(¤t->perf_event_mutex); 4034 list_for_each_entry(event, ¤t->perf_event_list, owner_entry) { 4035 ctx = perf_event_ctx_lock(event); 4036 perf_event_for_each_child(event, _perf_event_enable); 4037 perf_event_ctx_unlock(event, ctx); 4038 } 4039 mutex_unlock(¤t->perf_event_mutex); 4040 4041 return 0; 4042 } 4043 4044 int perf_event_task_disable(void) 4045 { 4046 struct perf_event_context *ctx; 4047 struct perf_event *event; 4048 4049 mutex_lock(¤t->perf_event_mutex); 4050 list_for_each_entry(event, ¤t->perf_event_list, owner_entry) { 4051 ctx = perf_event_ctx_lock(event); 4052 perf_event_for_each_child(event, _perf_event_disable); 4053 perf_event_ctx_unlock(event, ctx); 4054 } 4055 mutex_unlock(¤t->perf_event_mutex); 4056 4057 return 0; 4058 } 4059 4060 static int perf_event_index(struct perf_event *event) 4061 { 4062 if (event->hw.state & PERF_HES_STOPPED) 4063 return 0; 4064 4065 if (event->state != PERF_EVENT_STATE_ACTIVE) 4066 return 0; 4067 4068 return event->pmu->event_idx(event); 4069 } 4070 4071 static void calc_timer_values(struct perf_event *event, 4072 u64 *now, 4073 u64 *enabled, 4074 u64 *running) 4075 { 4076 u64 ctx_time; 4077 4078 *now = perf_clock(); 4079 ctx_time = event->shadow_ctx_time + *now; 4080 *enabled = ctx_time - event->tstamp_enabled; 4081 *running = ctx_time - event->tstamp_running; 4082 } 4083 4084 static void perf_event_init_userpage(struct perf_event *event) 4085 { 4086 struct perf_event_mmap_page *userpg; 4087 struct ring_buffer *rb; 4088 4089 rcu_read_lock(); 4090 rb = rcu_dereference(event->rb); 4091 if (!rb) 4092 goto unlock; 4093 4094 userpg = rb->user_page; 4095 4096 /* Allow new userspace to detect that bit 0 is deprecated */ 4097 userpg->cap_bit0_is_deprecated = 1; 4098 userpg->size = offsetof(struct perf_event_mmap_page, __reserved); 4099 4100 unlock: 4101 rcu_read_unlock(); 4102 } 4103 4104 void __weak arch_perf_update_userpage( 4105 struct perf_event *event, struct perf_event_mmap_page *userpg, u64 now) 4106 { 4107 } 4108 4109 /* 4110 * Callers need to ensure there can be no nesting of this function, otherwise 4111 * the seqlock logic goes bad. We can not serialize this because the arch 4112 * code calls this from NMI context. 4113 */ 4114 void perf_event_update_userpage(struct perf_event *event) 4115 { 4116 struct perf_event_mmap_page *userpg; 4117 struct ring_buffer *rb; 4118 u64 enabled, running, now; 4119 4120 rcu_read_lock(); 4121 rb = rcu_dereference(event->rb); 4122 if (!rb) 4123 goto unlock; 4124 4125 /* 4126 * compute total_time_enabled, total_time_running 4127 * based on snapshot values taken when the event 4128 * was last scheduled in. 4129 * 4130 * we cannot simply called update_context_time() 4131 * because of locking issue as we can be called in 4132 * NMI context 4133 */ 4134 calc_timer_values(event, &now, &enabled, &running); 4135 4136 userpg = rb->user_page; 4137 /* 4138 * Disable preemption so as to not let the corresponding user-space 4139 * spin too long if we get preempted. 4140 */ 4141 preempt_disable(); 4142 ++userpg->lock; 4143 barrier(); 4144 userpg->index = perf_event_index(event); 4145 userpg->offset = perf_event_count(event); 4146 if (userpg->index) 4147 userpg->offset -= local64_read(&event->hw.prev_count); 4148 4149 userpg->time_enabled = enabled + 4150 atomic64_read(&event->child_total_time_enabled); 4151 4152 userpg->time_running = running + 4153 atomic64_read(&event->child_total_time_running); 4154 4155 arch_perf_update_userpage(event, userpg, now); 4156 4157 barrier(); 4158 ++userpg->lock; 4159 preempt_enable(); 4160 unlock: 4161 rcu_read_unlock(); 4162 } 4163 4164 static int perf_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf) 4165 { 4166 struct perf_event *event = vma->vm_file->private_data; 4167 struct ring_buffer *rb; 4168 int ret = VM_FAULT_SIGBUS; 4169 4170 if (vmf->flags & FAULT_FLAG_MKWRITE) { 4171 if (vmf->pgoff == 0) 4172 ret = 0; 4173 return ret; 4174 } 4175 4176 rcu_read_lock(); 4177 rb = rcu_dereference(event->rb); 4178 if (!rb) 4179 goto unlock; 4180 4181 if (vmf->pgoff && (vmf->flags & FAULT_FLAG_WRITE)) 4182 goto unlock; 4183 4184 vmf->page = perf_mmap_to_page(rb, vmf->pgoff); 4185 if (!vmf->page) 4186 goto unlock; 4187 4188 get_page(vmf->page); 4189 vmf->page->mapping = vma->vm_file->f_mapping; 4190 vmf->page->index = vmf->pgoff; 4191 4192 ret = 0; 4193 unlock: 4194 rcu_read_unlock(); 4195 4196 return ret; 4197 } 4198 4199 static void ring_buffer_attach(struct perf_event *event, 4200 struct ring_buffer *rb) 4201 { 4202 struct ring_buffer *old_rb = NULL; 4203 unsigned long flags; 4204 4205 if (event->rb) { 4206 /* 4207 * Should be impossible, we set this when removing 4208 * event->rb_entry and wait/clear when adding event->rb_entry. 4209 */ 4210 WARN_ON_ONCE(event->rcu_pending); 4211 4212 old_rb = event->rb; 4213 event->rcu_batches = get_state_synchronize_rcu(); 4214 event->rcu_pending = 1; 4215 4216 spin_lock_irqsave(&old_rb->event_lock, flags); 4217 list_del_rcu(&event->rb_entry); 4218 spin_unlock_irqrestore(&old_rb->event_lock, flags); 4219 } 4220 4221 if (event->rcu_pending && rb) { 4222 cond_synchronize_rcu(event->rcu_batches); 4223 event->rcu_pending = 0; 4224 } 4225 4226 if (rb) { 4227 spin_lock_irqsave(&rb->event_lock, flags); 4228 list_add_rcu(&event->rb_entry, &rb->event_list); 4229 spin_unlock_irqrestore(&rb->event_lock, flags); 4230 } 4231 4232 rcu_assign_pointer(event->rb, rb); 4233 4234 if (old_rb) { 4235 ring_buffer_put(old_rb); 4236 /* 4237 * Since we detached before setting the new rb, so that we 4238 * could attach the new rb, we could have missed a wakeup. 4239 * Provide it now. 4240 */ 4241 wake_up_all(&event->waitq); 4242 } 4243 } 4244 4245 static void ring_buffer_wakeup(struct perf_event *event) 4246 { 4247 struct ring_buffer *rb; 4248 4249 rcu_read_lock(); 4250 rb = rcu_dereference(event->rb); 4251 if (rb) { 4252 list_for_each_entry_rcu(event, &rb->event_list, rb_entry) 4253 wake_up_all(&event->waitq); 4254 } 4255 rcu_read_unlock(); 4256 } 4257 4258 static void rb_free_rcu(struct rcu_head *rcu_head) 4259 { 4260 struct ring_buffer *rb; 4261 4262 rb = container_of(rcu_head, struct ring_buffer, rcu_head); 4263 rb_free(rb); 4264 } 4265 4266 static struct ring_buffer *ring_buffer_get(struct perf_event *event) 4267 { 4268 struct ring_buffer *rb; 4269 4270 rcu_read_lock(); 4271 rb = rcu_dereference(event->rb); 4272 if (rb) { 4273 if (!atomic_inc_not_zero(&rb->refcount)) 4274 rb = NULL; 4275 } 4276 rcu_read_unlock(); 4277 4278 return rb; 4279 } 4280 4281 static void ring_buffer_put(struct ring_buffer *rb) 4282 { 4283 if (!atomic_dec_and_test(&rb->refcount)) 4284 return; 4285 4286 WARN_ON_ONCE(!list_empty(&rb->event_list)); 4287 4288 call_rcu(&rb->rcu_head, rb_free_rcu); 4289 } 4290 4291 static void perf_mmap_open(struct vm_area_struct *vma) 4292 { 4293 struct perf_event *event = vma->vm_file->private_data; 4294 4295 atomic_inc(&event->mmap_count); 4296 atomic_inc(&event->rb->mmap_count); 4297 4298 if (event->pmu->event_mapped) 4299 event->pmu->event_mapped(event); 4300 } 4301 4302 /* 4303 * A buffer can be mmap()ed multiple times; either directly through the same 4304 * event, or through other events by use of perf_event_set_output(). 4305 * 4306 * In order to undo the VM accounting done by perf_mmap() we need to destroy 4307 * the buffer here, where we still have a VM context. This means we need 4308 * to detach all events redirecting to us. 4309 */ 4310 static void perf_mmap_close(struct vm_area_struct *vma) 4311 { 4312 struct perf_event *event = vma->vm_file->private_data; 4313 4314 struct ring_buffer *rb = ring_buffer_get(event); 4315 struct user_struct *mmap_user = rb->mmap_user; 4316 int mmap_locked = rb->mmap_locked; 4317 unsigned long size = perf_data_size(rb); 4318 4319 if (event->pmu->event_unmapped) 4320 event->pmu->event_unmapped(event); 4321 4322 atomic_dec(&rb->mmap_count); 4323 4324 if (!atomic_dec_and_mutex_lock(&event->mmap_count, &event->mmap_mutex)) 4325 goto out_put; 4326 4327 ring_buffer_attach(event, NULL); 4328 mutex_unlock(&event->mmap_mutex); 4329 4330 /* If there's still other mmap()s of this buffer, we're done. */ 4331 if (atomic_read(&rb->mmap_count)) 4332 goto out_put; 4333 4334 /* 4335 * No other mmap()s, detach from all other events that might redirect 4336 * into the now unreachable buffer. Somewhat complicated by the 4337 * fact that rb::event_lock otherwise nests inside mmap_mutex. 4338 */ 4339 again: 4340 rcu_read_lock(); 4341 list_for_each_entry_rcu(event, &rb->event_list, rb_entry) { 4342 if (!atomic_long_inc_not_zero(&event->refcount)) { 4343 /* 4344 * This event is en-route to free_event() which will 4345 * detach it and remove it from the list. 4346 */ 4347 continue; 4348 } 4349 rcu_read_unlock(); 4350 4351 mutex_lock(&event->mmap_mutex); 4352 /* 4353 * Check we didn't race with perf_event_set_output() which can 4354 * swizzle the rb from under us while we were waiting to 4355 * acquire mmap_mutex. 4356 * 4357 * If we find a different rb; ignore this event, a next 4358 * iteration will no longer find it on the list. We have to 4359 * still restart the iteration to make sure we're not now 4360 * iterating the wrong list. 4361 */ 4362 if (event->rb == rb) 4363 ring_buffer_attach(event, NULL); 4364 4365 mutex_unlock(&event->mmap_mutex); 4366 put_event(event); 4367 4368 /* 4369 * Restart the iteration; either we're on the wrong list or 4370 * destroyed its integrity by doing a deletion. 4371 */ 4372 goto again; 4373 } 4374 rcu_read_unlock(); 4375 4376 /* 4377 * It could be there's still a few 0-ref events on the list; they'll 4378 * get cleaned up by free_event() -- they'll also still have their 4379 * ref on the rb and will free it whenever they are done with it. 4380 * 4381 * Aside from that, this buffer is 'fully' detached and unmapped, 4382 * undo the VM accounting. 4383 */ 4384 4385 atomic_long_sub((size >> PAGE_SHIFT) + 1, &mmap_user->locked_vm); 4386 vma->vm_mm->pinned_vm -= mmap_locked; 4387 free_uid(mmap_user); 4388 4389 out_put: 4390 ring_buffer_put(rb); /* could be last */ 4391 } 4392 4393 static const struct vm_operations_struct perf_mmap_vmops = { 4394 .open = perf_mmap_open, 4395 .close = perf_mmap_close, 4396 .fault = perf_mmap_fault, 4397 .page_mkwrite = perf_mmap_fault, 4398 }; 4399 4400 static int perf_mmap(struct file *file, struct vm_area_struct *vma) 4401 { 4402 struct perf_event *event = file->private_data; 4403 unsigned long user_locked, user_lock_limit; 4404 struct user_struct *user = current_user(); 4405 unsigned long locked, lock_limit; 4406 struct ring_buffer *rb; 4407 unsigned long vma_size; 4408 unsigned long nr_pages; 4409 long user_extra, extra; 4410 int ret = 0, flags = 0; 4411 4412 /* 4413 * Don't allow mmap() of inherited per-task counters. This would 4414 * create a performance issue due to all children writing to the 4415 * same rb. 4416 */ 4417 if (event->cpu == -1 && event->attr.inherit) 4418 return -EINVAL; 4419 4420 if (!(vma->vm_flags & VM_SHARED)) 4421 return -EINVAL; 4422 4423 vma_size = vma->vm_end - vma->vm_start; 4424 nr_pages = (vma_size / PAGE_SIZE) - 1; 4425 4426 /* 4427 * If we have rb pages ensure they're a power-of-two number, so we 4428 * can do bitmasks instead of modulo. 4429 */ 4430 if (nr_pages != 0 && !is_power_of_2(nr_pages)) 4431 return -EINVAL; 4432 4433 if (vma_size != PAGE_SIZE * (1 + nr_pages)) 4434 return -EINVAL; 4435 4436 if (vma->vm_pgoff != 0) 4437 return -EINVAL; 4438 4439 WARN_ON_ONCE(event->ctx->parent_ctx); 4440 again: 4441 mutex_lock(&event->mmap_mutex); 4442 if (event->rb) { 4443 if (event->rb->nr_pages != nr_pages) { 4444 ret = -EINVAL; 4445 goto unlock; 4446 } 4447 4448 if (!atomic_inc_not_zero(&event->rb->mmap_count)) { 4449 /* 4450 * Raced against perf_mmap_close() through 4451 * perf_event_set_output(). Try again, hope for better 4452 * luck. 4453 */ 4454 mutex_unlock(&event->mmap_mutex); 4455 goto again; 4456 } 4457 4458 goto unlock; 4459 } 4460 4461 user_extra = nr_pages + 1; 4462 user_lock_limit = sysctl_perf_event_mlock >> (PAGE_SHIFT - 10); 4463 4464 /* 4465 * Increase the limit linearly with more CPUs: 4466 */ 4467 user_lock_limit *= num_online_cpus(); 4468 4469 user_locked = atomic_long_read(&user->locked_vm) + user_extra; 4470 4471 extra = 0; 4472 if (user_locked > user_lock_limit) 4473 extra = user_locked - user_lock_limit; 4474 4475 lock_limit = rlimit(RLIMIT_MEMLOCK); 4476 lock_limit >>= PAGE_SHIFT; 4477 locked = vma->vm_mm->pinned_vm + extra; 4478 4479 if ((locked > lock_limit) && perf_paranoid_tracepoint_raw() && 4480 !capable(CAP_IPC_LOCK)) { 4481 ret = -EPERM; 4482 goto unlock; 4483 } 4484 4485 WARN_ON(event->rb); 4486 4487 if (vma->vm_flags & VM_WRITE) 4488 flags |= RING_BUFFER_WRITABLE; 4489 4490 rb = rb_alloc(nr_pages, 4491 event->attr.watermark ? event->attr.wakeup_watermark : 0, 4492 event->cpu, flags); 4493 4494 if (!rb) { 4495 ret = -ENOMEM; 4496 goto unlock; 4497 } 4498 4499 atomic_set(&rb->mmap_count, 1); 4500 rb->mmap_locked = extra; 4501 rb->mmap_user = get_current_user(); 4502 4503 atomic_long_add(user_extra, &user->locked_vm); 4504 vma->vm_mm->pinned_vm += extra; 4505 4506 ring_buffer_attach(event, rb); 4507 4508 perf_event_init_userpage(event); 4509 perf_event_update_userpage(event); 4510 4511 unlock: 4512 if (!ret) 4513 atomic_inc(&event->mmap_count); 4514 mutex_unlock(&event->mmap_mutex); 4515 4516 /* 4517 * Since pinned accounting is per vm we cannot allow fork() to copy our 4518 * vma. 4519 */ 4520 vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_DONTDUMP; 4521 vma->vm_ops = &perf_mmap_vmops; 4522 4523 if (event->pmu->event_mapped) 4524 event->pmu->event_mapped(event); 4525 4526 return ret; 4527 } 4528 4529 static int perf_fasync(int fd, struct file *filp, int on) 4530 { 4531 struct inode *inode = file_inode(filp); 4532 struct perf_event *event = filp->private_data; 4533 int retval; 4534 4535 mutex_lock(&inode->i_mutex); 4536 retval = fasync_helper(fd, filp, on, &event->fasync); 4537 mutex_unlock(&inode->i_mutex); 4538 4539 if (retval < 0) 4540 return retval; 4541 4542 return 0; 4543 } 4544 4545 static const struct file_operations perf_fops = { 4546 .llseek = no_llseek, 4547 .release = perf_release, 4548 .read = perf_read, 4549 .poll = perf_poll, 4550 .unlocked_ioctl = perf_ioctl, 4551 .compat_ioctl = perf_compat_ioctl, 4552 .mmap = perf_mmap, 4553 .fasync = perf_fasync, 4554 }; 4555 4556 /* 4557 * Perf event wakeup 4558 * 4559 * If there's data, ensure we set the poll() state and publish everything 4560 * to user-space before waking everybody up. 4561 */ 4562 4563 void perf_event_wakeup(struct perf_event *event) 4564 { 4565 ring_buffer_wakeup(event); 4566 4567 if (event->pending_kill) { 4568 kill_fasync(&event->fasync, SIGIO, event->pending_kill); 4569 event->pending_kill = 0; 4570 } 4571 } 4572 4573 static void perf_pending_event(struct irq_work *entry) 4574 { 4575 struct perf_event *event = container_of(entry, 4576 struct perf_event, pending); 4577 4578 if (event->pending_disable) { 4579 event->pending_disable = 0; 4580 __perf_event_disable(event); 4581 } 4582 4583 if (event->pending_wakeup) { 4584 event->pending_wakeup = 0; 4585 perf_event_wakeup(event); 4586 } 4587 } 4588 4589 /* 4590 * We assume there is only KVM supporting the callbacks. 4591 * Later on, we might change it to a list if there is 4592 * another virtualization implementation supporting the callbacks. 4593 */ 4594 struct perf_guest_info_callbacks *perf_guest_cbs; 4595 4596 int perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *cbs) 4597 { 4598 perf_guest_cbs = cbs; 4599 return 0; 4600 } 4601 EXPORT_SYMBOL_GPL(perf_register_guest_info_callbacks); 4602 4603 int perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *cbs) 4604 { 4605 perf_guest_cbs = NULL; 4606 return 0; 4607 } 4608 EXPORT_SYMBOL_GPL(perf_unregister_guest_info_callbacks); 4609 4610 static void 4611 perf_output_sample_regs(struct perf_output_handle *handle, 4612 struct pt_regs *regs, u64 mask) 4613 { 4614 int bit; 4615 4616 for_each_set_bit(bit, (const unsigned long *) &mask, 4617 sizeof(mask) * BITS_PER_BYTE) { 4618 u64 val; 4619 4620 val = perf_reg_value(regs, bit); 4621 perf_output_put(handle, val); 4622 } 4623 } 4624 4625 static void perf_sample_regs_user(struct perf_regs *regs_user, 4626 struct pt_regs *regs, 4627 struct pt_regs *regs_user_copy) 4628 { 4629 if (user_mode(regs)) { 4630 regs_user->abi = perf_reg_abi(current); 4631 regs_user->regs = regs; 4632 } else if (current->mm) { 4633 perf_get_regs_user(regs_user, regs, regs_user_copy); 4634 } else { 4635 regs_user->abi = PERF_SAMPLE_REGS_ABI_NONE; 4636 regs_user->regs = NULL; 4637 } 4638 } 4639 4640 static void perf_sample_regs_intr(struct perf_regs *regs_intr, 4641 struct pt_regs *regs) 4642 { 4643 regs_intr->regs = regs; 4644 regs_intr->abi = perf_reg_abi(current); 4645 } 4646 4647 4648 /* 4649 * Get remaining task size from user stack pointer. 4650 * 4651 * It'd be better to take stack vma map and limit this more 4652 * precisly, but there's no way to get it safely under interrupt, 4653 * so using TASK_SIZE as limit. 4654 */ 4655 static u64 perf_ustack_task_size(struct pt_regs *regs) 4656 { 4657 unsigned long addr = perf_user_stack_pointer(regs); 4658 4659 if (!addr || addr >= TASK_SIZE) 4660 return 0; 4661 4662 return TASK_SIZE - addr; 4663 } 4664 4665 static u16 4666 perf_sample_ustack_size(u16 stack_size, u16 header_size, 4667 struct pt_regs *regs) 4668 { 4669 u64 task_size; 4670 4671 /* No regs, no stack pointer, no dump. */ 4672 if (!regs) 4673 return 0; 4674 4675 /* 4676 * Check if we fit in with the requested stack size into the: 4677 * - TASK_SIZE 4678 * If we don't, we limit the size to the TASK_SIZE. 4679 * 4680 * - remaining sample size 4681 * If we don't, we customize the stack size to 4682 * fit in to the remaining sample size. 4683 */ 4684 4685 task_size = min((u64) USHRT_MAX, perf_ustack_task_size(regs)); 4686 stack_size = min(stack_size, (u16) task_size); 4687 4688 /* Current header size plus static size and dynamic size. */ 4689 header_size += 2 * sizeof(u64); 4690 4691 /* Do we fit in with the current stack dump size? */ 4692 if ((u16) (header_size + stack_size) < header_size) { 4693 /* 4694 * If we overflow the maximum size for the sample, 4695 * we customize the stack dump size to fit in. 4696 */ 4697 stack_size = USHRT_MAX - header_size - sizeof(u64); 4698 stack_size = round_up(stack_size, sizeof(u64)); 4699 } 4700 4701 return stack_size; 4702 } 4703 4704 static void 4705 perf_output_sample_ustack(struct perf_output_handle *handle, u64 dump_size, 4706 struct pt_regs *regs) 4707 { 4708 /* Case of a kernel thread, nothing to dump */ 4709 if (!regs) { 4710 u64 size = 0; 4711 perf_output_put(handle, size); 4712 } else { 4713 unsigned long sp; 4714 unsigned int rem; 4715 u64 dyn_size; 4716 4717 /* 4718 * We dump: 4719 * static size 4720 * - the size requested by user or the best one we can fit 4721 * in to the sample max size 4722 * data 4723 * - user stack dump data 4724 * dynamic size 4725 * - the actual dumped size 4726 */ 4727 4728 /* Static size. */ 4729 perf_output_put(handle, dump_size); 4730 4731 /* Data. */ 4732 sp = perf_user_stack_pointer(regs); 4733 rem = __output_copy_user(handle, (void *) sp, dump_size); 4734 dyn_size = dump_size - rem; 4735 4736 perf_output_skip(handle, rem); 4737 4738 /* Dynamic size. */ 4739 perf_output_put(handle, dyn_size); 4740 } 4741 } 4742 4743 static void __perf_event_header__init_id(struct perf_event_header *header, 4744 struct perf_sample_data *data, 4745 struct perf_event *event) 4746 { 4747 u64 sample_type = event->attr.sample_type; 4748 4749 data->type = sample_type; 4750 header->size += event->id_header_size; 4751 4752 if (sample_type & PERF_SAMPLE_TID) { 4753 /* namespace issues */ 4754 data->tid_entry.pid = perf_event_pid(event, current); 4755 data->tid_entry.tid = perf_event_tid(event, current); 4756 } 4757 4758 if (sample_type & PERF_SAMPLE_TIME) 4759 data->time = perf_clock(); 4760 4761 if (sample_type & (PERF_SAMPLE_ID | PERF_SAMPLE_IDENTIFIER)) 4762 data->id = primary_event_id(event); 4763 4764 if (sample_type & PERF_SAMPLE_STREAM_ID) 4765 data->stream_id = event->id; 4766 4767 if (sample_type & PERF_SAMPLE_CPU) { 4768 data->cpu_entry.cpu = raw_smp_processor_id(); 4769 data->cpu_entry.reserved = 0; 4770 } 4771 } 4772 4773 void perf_event_header__init_id(struct perf_event_header *header, 4774 struct perf_sample_data *data, 4775 struct perf_event *event) 4776 { 4777 if (event->attr.sample_id_all) 4778 __perf_event_header__init_id(header, data, event); 4779 } 4780 4781 static void __perf_event__output_id_sample(struct perf_output_handle *handle, 4782 struct perf_sample_data *data) 4783 { 4784 u64 sample_type = data->type; 4785 4786 if (sample_type & PERF_SAMPLE_TID) 4787 perf_output_put(handle, data->tid_entry); 4788 4789 if (sample_type & PERF_SAMPLE_TIME) 4790 perf_output_put(handle, data->time); 4791 4792 if (sample_type & PERF_SAMPLE_ID) 4793 perf_output_put(handle, data->id); 4794 4795 if (sample_type & PERF_SAMPLE_STREAM_ID) 4796 perf_output_put(handle, data->stream_id); 4797 4798 if (sample_type & PERF_SAMPLE_CPU) 4799 perf_output_put(handle, data->cpu_entry); 4800 4801 if (sample_type & PERF_SAMPLE_IDENTIFIER) 4802 perf_output_put(handle, data->id); 4803 } 4804 4805 void perf_event__output_id_sample(struct perf_event *event, 4806 struct perf_output_handle *handle, 4807 struct perf_sample_data *sample) 4808 { 4809 if (event->attr.sample_id_all) 4810 __perf_event__output_id_sample(handle, sample); 4811 } 4812 4813 static void perf_output_read_one(struct perf_output_handle *handle, 4814 struct perf_event *event, 4815 u64 enabled, u64 running) 4816 { 4817 u64 read_format = event->attr.read_format; 4818 u64 values[4]; 4819 int n = 0; 4820 4821 values[n++] = perf_event_count(event); 4822 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) { 4823 values[n++] = enabled + 4824 atomic64_read(&event->child_total_time_enabled); 4825 } 4826 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) { 4827 values[n++] = running + 4828 atomic64_read(&event->child_total_time_running); 4829 } 4830 if (read_format & PERF_FORMAT_ID) 4831 values[n++] = primary_event_id(event); 4832 4833 __output_copy(handle, values, n * sizeof(u64)); 4834 } 4835 4836 /* 4837 * XXX PERF_FORMAT_GROUP vs inherited events seems difficult. 4838 */ 4839 static void perf_output_read_group(struct perf_output_handle *handle, 4840 struct perf_event *event, 4841 u64 enabled, u64 running) 4842 { 4843 struct perf_event *leader = event->group_leader, *sub; 4844 u64 read_format = event->attr.read_format; 4845 u64 values[5]; 4846 int n = 0; 4847 4848 values[n++] = 1 + leader->nr_siblings; 4849 4850 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) 4851 values[n++] = enabled; 4852 4853 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) 4854 values[n++] = running; 4855 4856 if (leader != event) 4857 leader->pmu->read(leader); 4858 4859 values[n++] = perf_event_count(leader); 4860 if (read_format & PERF_FORMAT_ID) 4861 values[n++] = primary_event_id(leader); 4862 4863 __output_copy(handle, values, n * sizeof(u64)); 4864 4865 list_for_each_entry(sub, &leader->sibling_list, group_entry) { 4866 n = 0; 4867 4868 if ((sub != event) && 4869 (sub->state == PERF_EVENT_STATE_ACTIVE)) 4870 sub->pmu->read(sub); 4871 4872 values[n++] = perf_event_count(sub); 4873 if (read_format & PERF_FORMAT_ID) 4874 values[n++] = primary_event_id(sub); 4875 4876 __output_copy(handle, values, n * sizeof(u64)); 4877 } 4878 } 4879 4880 #define PERF_FORMAT_TOTAL_TIMES (PERF_FORMAT_TOTAL_TIME_ENABLED|\ 4881 PERF_FORMAT_TOTAL_TIME_RUNNING) 4882 4883 static void perf_output_read(struct perf_output_handle *handle, 4884 struct perf_event *event) 4885 { 4886 u64 enabled = 0, running = 0, now; 4887 u64 read_format = event->attr.read_format; 4888 4889 /* 4890 * compute total_time_enabled, total_time_running 4891 * based on snapshot values taken when the event 4892 * was last scheduled in. 4893 * 4894 * we cannot simply called update_context_time() 4895 * because of locking issue as we are called in 4896 * NMI context 4897 */ 4898 if (read_format & PERF_FORMAT_TOTAL_TIMES) 4899 calc_timer_values(event, &now, &enabled, &running); 4900 4901 if (event->attr.read_format & PERF_FORMAT_GROUP) 4902 perf_output_read_group(handle, event, enabled, running); 4903 else 4904 perf_output_read_one(handle, event, enabled, running); 4905 } 4906 4907 void perf_output_sample(struct perf_output_handle *handle, 4908 struct perf_event_header *header, 4909 struct perf_sample_data *data, 4910 struct perf_event *event) 4911 { 4912 u64 sample_type = data->type; 4913 4914 perf_output_put(handle, *header); 4915 4916 if (sample_type & PERF_SAMPLE_IDENTIFIER) 4917 perf_output_put(handle, data->id); 4918 4919 if (sample_type & PERF_SAMPLE_IP) 4920 perf_output_put(handle, data->ip); 4921 4922 if (sample_type & PERF_SAMPLE_TID) 4923 perf_output_put(handle, data->tid_entry); 4924 4925 if (sample_type & PERF_SAMPLE_TIME) 4926 perf_output_put(handle, data->time); 4927 4928 if (sample_type & PERF_SAMPLE_ADDR) 4929 perf_output_put(handle, data->addr); 4930 4931 if (sample_type & PERF_SAMPLE_ID) 4932 perf_output_put(handle, data->id); 4933 4934 if (sample_type & PERF_SAMPLE_STREAM_ID) 4935 perf_output_put(handle, data->stream_id); 4936 4937 if (sample_type & PERF_SAMPLE_CPU) 4938 perf_output_put(handle, data->cpu_entry); 4939 4940 if (sample_type & PERF_SAMPLE_PERIOD) 4941 perf_output_put(handle, data->period); 4942 4943 if (sample_type & PERF_SAMPLE_READ) 4944 perf_output_read(handle, event); 4945 4946 if (sample_type & PERF_SAMPLE_CALLCHAIN) { 4947 if (data->callchain) { 4948 int size = 1; 4949 4950 if (data->callchain) 4951 size += data->callchain->nr; 4952 4953 size *= sizeof(u64); 4954 4955 __output_copy(handle, data->callchain, size); 4956 } else { 4957 u64 nr = 0; 4958 perf_output_put(handle, nr); 4959 } 4960 } 4961 4962 if (sample_type & PERF_SAMPLE_RAW) { 4963 if (data->raw) { 4964 perf_output_put(handle, data->raw->size); 4965 __output_copy(handle, data->raw->data, 4966 data->raw->size); 4967 } else { 4968 struct { 4969 u32 size; 4970 u32 data; 4971 } raw = { 4972 .size = sizeof(u32), 4973 .data = 0, 4974 }; 4975 perf_output_put(handle, raw); 4976 } 4977 } 4978 4979 if (sample_type & PERF_SAMPLE_BRANCH_STACK) { 4980 if (data->br_stack) { 4981 size_t size; 4982 4983 size = data->br_stack->nr 4984 * sizeof(struct perf_branch_entry); 4985 4986 perf_output_put(handle, data->br_stack->nr); 4987 perf_output_copy(handle, data->br_stack->entries, size); 4988 } else { 4989 /* 4990 * we always store at least the value of nr 4991 */ 4992 u64 nr = 0; 4993 perf_output_put(handle, nr); 4994 } 4995 } 4996 4997 if (sample_type & PERF_SAMPLE_REGS_USER) { 4998 u64 abi = data->regs_user.abi; 4999 5000 /* 5001 * If there are no regs to dump, notice it through 5002 * first u64 being zero (PERF_SAMPLE_REGS_ABI_NONE). 5003 */ 5004 perf_output_put(handle, abi); 5005 5006 if (abi) { 5007 u64 mask = event->attr.sample_regs_user; 5008 perf_output_sample_regs(handle, 5009 data->regs_user.regs, 5010 mask); 5011 } 5012 } 5013 5014 if (sample_type & PERF_SAMPLE_STACK_USER) { 5015 perf_output_sample_ustack(handle, 5016 data->stack_user_size, 5017 data->regs_user.regs); 5018 } 5019 5020 if (sample_type & PERF_SAMPLE_WEIGHT) 5021 perf_output_put(handle, data->weight); 5022 5023 if (sample_type & PERF_SAMPLE_DATA_SRC) 5024 perf_output_put(handle, data->data_src.val); 5025 5026 if (sample_type & PERF_SAMPLE_TRANSACTION) 5027 perf_output_put(handle, data->txn); 5028 5029 if (sample_type & PERF_SAMPLE_REGS_INTR) { 5030 u64 abi = data->regs_intr.abi; 5031 /* 5032 * If there are no regs to dump, notice it through 5033 * first u64 being zero (PERF_SAMPLE_REGS_ABI_NONE). 5034 */ 5035 perf_output_put(handle, abi); 5036 5037 if (abi) { 5038 u64 mask = event->attr.sample_regs_intr; 5039 5040 perf_output_sample_regs(handle, 5041 data->regs_intr.regs, 5042 mask); 5043 } 5044 } 5045 5046 if (!event->attr.watermark) { 5047 int wakeup_events = event->attr.wakeup_events; 5048 5049 if (wakeup_events) { 5050 struct ring_buffer *rb = handle->rb; 5051 int events = local_inc_return(&rb->events); 5052 5053 if (events >= wakeup_events) { 5054 local_sub(wakeup_events, &rb->events); 5055 local_inc(&rb->wakeup); 5056 } 5057 } 5058 } 5059 } 5060 5061 void perf_prepare_sample(struct perf_event_header *header, 5062 struct perf_sample_data *data, 5063 struct perf_event *event, 5064 struct pt_regs *regs) 5065 { 5066 u64 sample_type = event->attr.sample_type; 5067 5068 header->type = PERF_RECORD_SAMPLE; 5069 header->size = sizeof(*header) + event->header_size; 5070 5071 header->misc = 0; 5072 header->misc |= perf_misc_flags(regs); 5073 5074 __perf_event_header__init_id(header, data, event); 5075 5076 if (sample_type & PERF_SAMPLE_IP) 5077 data->ip = perf_instruction_pointer(regs); 5078 5079 if (sample_type & PERF_SAMPLE_CALLCHAIN) { 5080 int size = 1; 5081 5082 data->callchain = perf_callchain(event, regs); 5083 5084 if (data->callchain) 5085 size += data->callchain->nr; 5086 5087 header->size += size * sizeof(u64); 5088 } 5089 5090 if (sample_type & PERF_SAMPLE_RAW) { 5091 int size = sizeof(u32); 5092 5093 if (data->raw) 5094 size += data->raw->size; 5095 else 5096 size += sizeof(u32); 5097 5098 WARN_ON_ONCE(size & (sizeof(u64)-1)); 5099 header->size += size; 5100 } 5101 5102 if (sample_type & PERF_SAMPLE_BRANCH_STACK) { 5103 int size = sizeof(u64); /* nr */ 5104 if (data->br_stack) { 5105 size += data->br_stack->nr 5106 * sizeof(struct perf_branch_entry); 5107 } 5108 header->size += size; 5109 } 5110 5111 if (sample_type & (PERF_SAMPLE_REGS_USER | PERF_SAMPLE_STACK_USER)) 5112 perf_sample_regs_user(&data->regs_user, regs, 5113 &data->regs_user_copy); 5114 5115 if (sample_type & PERF_SAMPLE_REGS_USER) { 5116 /* regs dump ABI info */ 5117 int size = sizeof(u64); 5118 5119 if (data->regs_user.regs) { 5120 u64 mask = event->attr.sample_regs_user; 5121 size += hweight64(mask) * sizeof(u64); 5122 } 5123 5124 header->size += size; 5125 } 5126 5127 if (sample_type & PERF_SAMPLE_STACK_USER) { 5128 /* 5129 * Either we need PERF_SAMPLE_STACK_USER bit to be allways 5130 * processed as the last one or have additional check added 5131 * in case new sample type is added, because we could eat 5132 * up the rest of the sample size. 5133 */ 5134 u16 stack_size = event->attr.sample_stack_user; 5135 u16 size = sizeof(u64); 5136 5137 stack_size = perf_sample_ustack_size(stack_size, header->size, 5138 data->regs_user.regs); 5139 5140 /* 5141 * If there is something to dump, add space for the dump 5142 * itself and for the field that tells the dynamic size, 5143 * which is how many have been actually dumped. 5144 */ 5145 if (stack_size) 5146 size += sizeof(u64) + stack_size; 5147 5148 data->stack_user_size = stack_size; 5149 header->size += size; 5150 } 5151 5152 if (sample_type & PERF_SAMPLE_REGS_INTR) { 5153 /* regs dump ABI info */ 5154 int size = sizeof(u64); 5155 5156 perf_sample_regs_intr(&data->regs_intr, regs); 5157 5158 if (data->regs_intr.regs) { 5159 u64 mask = event->attr.sample_regs_intr; 5160 5161 size += hweight64(mask) * sizeof(u64); 5162 } 5163 5164 header->size += size; 5165 } 5166 } 5167 5168 static void perf_event_output(struct perf_event *event, 5169 struct perf_sample_data *data, 5170 struct pt_regs *regs) 5171 { 5172 struct perf_output_handle handle; 5173 struct perf_event_header header; 5174 5175 /* protect the callchain buffers */ 5176 rcu_read_lock(); 5177 5178 perf_prepare_sample(&header, data, event, regs); 5179 5180 if (perf_output_begin(&handle, event, header.size)) 5181 goto exit; 5182 5183 perf_output_sample(&handle, &header, data, event); 5184 5185 perf_output_end(&handle); 5186 5187 exit: 5188 rcu_read_unlock(); 5189 } 5190 5191 /* 5192 * read event_id 5193 */ 5194 5195 struct perf_read_event { 5196 struct perf_event_header header; 5197 5198 u32 pid; 5199 u32 tid; 5200 }; 5201 5202 static void 5203 perf_event_read_event(struct perf_event *event, 5204 struct task_struct *task) 5205 { 5206 struct perf_output_handle handle; 5207 struct perf_sample_data sample; 5208 struct perf_read_event read_event = { 5209 .header = { 5210 .type = PERF_RECORD_READ, 5211 .misc = 0, 5212 .size = sizeof(read_event) + event->read_size, 5213 }, 5214 .pid = perf_event_pid(event, task), 5215 .tid = perf_event_tid(event, task), 5216 }; 5217 int ret; 5218 5219 perf_event_header__init_id(&read_event.header, &sample, event); 5220 ret = perf_output_begin(&handle, event, read_event.header.size); 5221 if (ret) 5222 return; 5223 5224 perf_output_put(&handle, read_event); 5225 perf_output_read(&handle, event); 5226 perf_event__output_id_sample(event, &handle, &sample); 5227 5228 perf_output_end(&handle); 5229 } 5230 5231 typedef void (perf_event_aux_output_cb)(struct perf_event *event, void *data); 5232 5233 static void 5234 perf_event_aux_ctx(struct perf_event_context *ctx, 5235 perf_event_aux_output_cb output, 5236 void *data) 5237 { 5238 struct perf_event *event; 5239 5240 list_for_each_entry_rcu(event, &ctx->event_list, event_entry) { 5241 if (event->state < PERF_EVENT_STATE_INACTIVE) 5242 continue; 5243 if (!event_filter_match(event)) 5244 continue; 5245 output(event, data); 5246 } 5247 } 5248 5249 static void 5250 perf_event_aux(perf_event_aux_output_cb output, void *data, 5251 struct perf_event_context *task_ctx) 5252 { 5253 struct perf_cpu_context *cpuctx; 5254 struct perf_event_context *ctx; 5255 struct pmu *pmu; 5256 int ctxn; 5257 5258 rcu_read_lock(); 5259 list_for_each_entry_rcu(pmu, &pmus, entry) { 5260 cpuctx = get_cpu_ptr(pmu->pmu_cpu_context); 5261 if (cpuctx->unique_pmu != pmu) 5262 goto next; 5263 perf_event_aux_ctx(&cpuctx->ctx, output, data); 5264 if (task_ctx) 5265 goto next; 5266 ctxn = pmu->task_ctx_nr; 5267 if (ctxn < 0) 5268 goto next; 5269 ctx = rcu_dereference(current->perf_event_ctxp[ctxn]); 5270 if (ctx) 5271 perf_event_aux_ctx(ctx, output, data); 5272 next: 5273 put_cpu_ptr(pmu->pmu_cpu_context); 5274 } 5275 5276 if (task_ctx) { 5277 preempt_disable(); 5278 perf_event_aux_ctx(task_ctx, output, data); 5279 preempt_enable(); 5280 } 5281 rcu_read_unlock(); 5282 } 5283 5284 /* 5285 * task tracking -- fork/exit 5286 * 5287 * enabled by: attr.comm | attr.mmap | attr.mmap2 | attr.mmap_data | attr.task 5288 */ 5289 5290 struct perf_task_event { 5291 struct task_struct *task; 5292 struct perf_event_context *task_ctx; 5293 5294 struct { 5295 struct perf_event_header header; 5296 5297 u32 pid; 5298 u32 ppid; 5299 u32 tid; 5300 u32 ptid; 5301 u64 time; 5302 } event_id; 5303 }; 5304 5305 static int perf_event_task_match(struct perf_event *event) 5306 { 5307 return event->attr.comm || event->attr.mmap || 5308 event->attr.mmap2 || event->attr.mmap_data || 5309 event->attr.task; 5310 } 5311 5312 static void perf_event_task_output(struct perf_event *event, 5313 void *data) 5314 { 5315 struct perf_task_event *task_event = data; 5316 struct perf_output_handle handle; 5317 struct perf_sample_data sample; 5318 struct task_struct *task = task_event->task; 5319 int ret, size = task_event->event_id.header.size; 5320 5321 if (!perf_event_task_match(event)) 5322 return; 5323 5324 perf_event_header__init_id(&task_event->event_id.header, &sample, event); 5325 5326 ret = perf_output_begin(&handle, event, 5327 task_event->event_id.header.size); 5328 if (ret) 5329 goto out; 5330 5331 task_event->event_id.pid = perf_event_pid(event, task); 5332 task_event->event_id.ppid = perf_event_pid(event, current); 5333 5334 task_event->event_id.tid = perf_event_tid(event, task); 5335 task_event->event_id.ptid = perf_event_tid(event, current); 5336 5337 perf_output_put(&handle, task_event->event_id); 5338 5339 perf_event__output_id_sample(event, &handle, &sample); 5340 5341 perf_output_end(&handle); 5342 out: 5343 task_event->event_id.header.size = size; 5344 } 5345 5346 static void perf_event_task(struct task_struct *task, 5347 struct perf_event_context *task_ctx, 5348 int new) 5349 { 5350 struct perf_task_event task_event; 5351 5352 if (!atomic_read(&nr_comm_events) && 5353 !atomic_read(&nr_mmap_events) && 5354 !atomic_read(&nr_task_events)) 5355 return; 5356 5357 task_event = (struct perf_task_event){ 5358 .task = task, 5359 .task_ctx = task_ctx, 5360 .event_id = { 5361 .header = { 5362 .type = new ? PERF_RECORD_FORK : PERF_RECORD_EXIT, 5363 .misc = 0, 5364 .size = sizeof(task_event.event_id), 5365 }, 5366 /* .pid */ 5367 /* .ppid */ 5368 /* .tid */ 5369 /* .ptid */ 5370 .time = perf_clock(), 5371 }, 5372 }; 5373 5374 perf_event_aux(perf_event_task_output, 5375 &task_event, 5376 task_ctx); 5377 } 5378 5379 void perf_event_fork(struct task_struct *task) 5380 { 5381 perf_event_task(task, NULL, 1); 5382 } 5383 5384 /* 5385 * comm tracking 5386 */ 5387 5388 struct perf_comm_event { 5389 struct task_struct *task; 5390 char *comm; 5391 int comm_size; 5392 5393 struct { 5394 struct perf_event_header header; 5395 5396 u32 pid; 5397 u32 tid; 5398 } event_id; 5399 }; 5400 5401 static int perf_event_comm_match(struct perf_event *event) 5402 { 5403 return event->attr.comm; 5404 } 5405 5406 static void perf_event_comm_output(struct perf_event *event, 5407 void *data) 5408 { 5409 struct perf_comm_event *comm_event = data; 5410 struct perf_output_handle handle; 5411 struct perf_sample_data sample; 5412 int size = comm_event->event_id.header.size; 5413 int ret; 5414 5415 if (!perf_event_comm_match(event)) 5416 return; 5417 5418 perf_event_header__init_id(&comm_event->event_id.header, &sample, event); 5419 ret = perf_output_begin(&handle, event, 5420 comm_event->event_id.header.size); 5421 5422 if (ret) 5423 goto out; 5424 5425 comm_event->event_id.pid = perf_event_pid(event, comm_event->task); 5426 comm_event->event_id.tid = perf_event_tid(event, comm_event->task); 5427 5428 perf_output_put(&handle, comm_event->event_id); 5429 __output_copy(&handle, comm_event->comm, 5430 comm_event->comm_size); 5431 5432 perf_event__output_id_sample(event, &handle, &sample); 5433 5434 perf_output_end(&handle); 5435 out: 5436 comm_event->event_id.header.size = size; 5437 } 5438 5439 static void perf_event_comm_event(struct perf_comm_event *comm_event) 5440 { 5441 char comm[TASK_COMM_LEN]; 5442 unsigned int size; 5443 5444 memset(comm, 0, sizeof(comm)); 5445 strlcpy(comm, comm_event->task->comm, sizeof(comm)); 5446 size = ALIGN(strlen(comm)+1, sizeof(u64)); 5447 5448 comm_event->comm = comm; 5449 comm_event->comm_size = size; 5450 5451 comm_event->event_id.header.size = sizeof(comm_event->event_id) + size; 5452 5453 perf_event_aux(perf_event_comm_output, 5454 comm_event, 5455 NULL); 5456 } 5457 5458 void perf_event_comm(struct task_struct *task, bool exec) 5459 { 5460 struct perf_comm_event comm_event; 5461 5462 if (!atomic_read(&nr_comm_events)) 5463 return; 5464 5465 comm_event = (struct perf_comm_event){ 5466 .task = task, 5467 /* .comm */ 5468 /* .comm_size */ 5469 .event_id = { 5470 .header = { 5471 .type = PERF_RECORD_COMM, 5472 .misc = exec ? PERF_RECORD_MISC_COMM_EXEC : 0, 5473 /* .size */ 5474 }, 5475 /* .pid */ 5476 /* .tid */ 5477 }, 5478 }; 5479 5480 perf_event_comm_event(&comm_event); 5481 } 5482 5483 /* 5484 * mmap tracking 5485 */ 5486 5487 struct perf_mmap_event { 5488 struct vm_area_struct *vma; 5489 5490 const char *file_name; 5491 int file_size; 5492 int maj, min; 5493 u64 ino; 5494 u64 ino_generation; 5495 u32 prot, flags; 5496 5497 struct { 5498 struct perf_event_header header; 5499 5500 u32 pid; 5501 u32 tid; 5502 u64 start; 5503 u64 len; 5504 u64 pgoff; 5505 } event_id; 5506 }; 5507 5508 static int perf_event_mmap_match(struct perf_event *event, 5509 void *data) 5510 { 5511 struct perf_mmap_event *mmap_event = data; 5512 struct vm_area_struct *vma = mmap_event->vma; 5513 int executable = vma->vm_flags & VM_EXEC; 5514 5515 return (!executable && event->attr.mmap_data) || 5516 (executable && (event->attr.mmap || event->attr.mmap2)); 5517 } 5518 5519 static void perf_event_mmap_output(struct perf_event *event, 5520 void *data) 5521 { 5522 struct perf_mmap_event *mmap_event = data; 5523 struct perf_output_handle handle; 5524 struct perf_sample_data sample; 5525 int size = mmap_event->event_id.header.size; 5526 int ret; 5527 5528 if (!perf_event_mmap_match(event, data)) 5529 return; 5530 5531 if (event->attr.mmap2) { 5532 mmap_event->event_id.header.type = PERF_RECORD_MMAP2; 5533 mmap_event->event_id.header.size += sizeof(mmap_event->maj); 5534 mmap_event->event_id.header.size += sizeof(mmap_event->min); 5535 mmap_event->event_id.header.size += sizeof(mmap_event->ino); 5536 mmap_event->event_id.header.size += sizeof(mmap_event->ino_generation); 5537 mmap_event->event_id.header.size += sizeof(mmap_event->prot); 5538 mmap_event->event_id.header.size += sizeof(mmap_event->flags); 5539 } 5540 5541 perf_event_header__init_id(&mmap_event->event_id.header, &sample, event); 5542 ret = perf_output_begin(&handle, event, 5543 mmap_event->event_id.header.size); 5544 if (ret) 5545 goto out; 5546 5547 mmap_event->event_id.pid = perf_event_pid(event, current); 5548 mmap_event->event_id.tid = perf_event_tid(event, current); 5549 5550 perf_output_put(&handle, mmap_event->event_id); 5551 5552 if (event->attr.mmap2) { 5553 perf_output_put(&handle, mmap_event->maj); 5554 perf_output_put(&handle, mmap_event->min); 5555 perf_output_put(&handle, mmap_event->ino); 5556 perf_output_put(&handle, mmap_event->ino_generation); 5557 perf_output_put(&handle, mmap_event->prot); 5558 perf_output_put(&handle, mmap_event->flags); 5559 } 5560 5561 __output_copy(&handle, mmap_event->file_name, 5562 mmap_event->file_size); 5563 5564 perf_event__output_id_sample(event, &handle, &sample); 5565 5566 perf_output_end(&handle); 5567 out: 5568 mmap_event->event_id.header.size = size; 5569 } 5570 5571 static void perf_event_mmap_event(struct perf_mmap_event *mmap_event) 5572 { 5573 struct vm_area_struct *vma = mmap_event->vma; 5574 struct file *file = vma->vm_file; 5575 int maj = 0, min = 0; 5576 u64 ino = 0, gen = 0; 5577 u32 prot = 0, flags = 0; 5578 unsigned int size; 5579 char tmp[16]; 5580 char *buf = NULL; 5581 char *name; 5582 5583 if (file) { 5584 struct inode *inode; 5585 dev_t dev; 5586 5587 buf = kmalloc(PATH_MAX, GFP_KERNEL); 5588 if (!buf) { 5589 name = "//enomem"; 5590 goto cpy_name; 5591 } 5592 /* 5593 * d_path() works from the end of the rb backwards, so we 5594 * need to add enough zero bytes after the string to handle 5595 * the 64bit alignment we do later. 5596 */ 5597 name = d_path(&file->f_path, buf, PATH_MAX - sizeof(u64)); 5598 if (IS_ERR(name)) { 5599 name = "//toolong"; 5600 goto cpy_name; 5601 } 5602 inode = file_inode(vma->vm_file); 5603 dev = inode->i_sb->s_dev; 5604 ino = inode->i_ino; 5605 gen = inode->i_generation; 5606 maj = MAJOR(dev); 5607 min = MINOR(dev); 5608 5609 if (vma->vm_flags & VM_READ) 5610 prot |= PROT_READ; 5611 if (vma->vm_flags & VM_WRITE) 5612 prot |= PROT_WRITE; 5613 if (vma->vm_flags & VM_EXEC) 5614 prot |= PROT_EXEC; 5615 5616 if (vma->vm_flags & VM_MAYSHARE) 5617 flags = MAP_SHARED; 5618 else 5619 flags = MAP_PRIVATE; 5620 5621 if (vma->vm_flags & VM_DENYWRITE) 5622 flags |= MAP_DENYWRITE; 5623 if (vma->vm_flags & VM_MAYEXEC) 5624 flags |= MAP_EXECUTABLE; 5625 if (vma->vm_flags & VM_LOCKED) 5626 flags |= MAP_LOCKED; 5627 if (vma->vm_flags & VM_HUGETLB) 5628 flags |= MAP_HUGETLB; 5629 5630 goto got_name; 5631 } else { 5632 if (vma->vm_ops && vma->vm_ops->name) { 5633 name = (char *) vma->vm_ops->name(vma); 5634 if (name) 5635 goto cpy_name; 5636 } 5637 5638 name = (char *)arch_vma_name(vma); 5639 if (name) 5640 goto cpy_name; 5641 5642 if (vma->vm_start <= vma->vm_mm->start_brk && 5643 vma->vm_end >= vma->vm_mm->brk) { 5644 name = "[heap]"; 5645 goto cpy_name; 5646 } 5647 if (vma->vm_start <= vma->vm_mm->start_stack && 5648 vma->vm_end >= vma->vm_mm->start_stack) { 5649 name = "[stack]"; 5650 goto cpy_name; 5651 } 5652 5653 name = "//anon"; 5654 goto cpy_name; 5655 } 5656 5657 cpy_name: 5658 strlcpy(tmp, name, sizeof(tmp)); 5659 name = tmp; 5660 got_name: 5661 /* 5662 * Since our buffer works in 8 byte units we need to align our string 5663 * size to a multiple of 8. However, we must guarantee the tail end is 5664 * zero'd out to avoid leaking random bits to userspace. 5665 */ 5666 size = strlen(name)+1; 5667 while (!IS_ALIGNED(size, sizeof(u64))) 5668 name[size++] = '\0'; 5669 5670 mmap_event->file_name = name; 5671 mmap_event->file_size = size; 5672 mmap_event->maj = maj; 5673 mmap_event->min = min; 5674 mmap_event->ino = ino; 5675 mmap_event->ino_generation = gen; 5676 mmap_event->prot = prot; 5677 mmap_event->flags = flags; 5678 5679 if (!(vma->vm_flags & VM_EXEC)) 5680 mmap_event->event_id.header.misc |= PERF_RECORD_MISC_MMAP_DATA; 5681 5682 mmap_event->event_id.header.size = sizeof(mmap_event->event_id) + size; 5683 5684 perf_event_aux(perf_event_mmap_output, 5685 mmap_event, 5686 NULL); 5687 5688 kfree(buf); 5689 } 5690 5691 void perf_event_mmap(struct vm_area_struct *vma) 5692 { 5693 struct perf_mmap_event mmap_event; 5694 5695 if (!atomic_read(&nr_mmap_events)) 5696 return; 5697 5698 mmap_event = (struct perf_mmap_event){ 5699 .vma = vma, 5700 /* .file_name */ 5701 /* .file_size */ 5702 .event_id = { 5703 .header = { 5704 .type = PERF_RECORD_MMAP, 5705 .misc = PERF_RECORD_MISC_USER, 5706 /* .size */ 5707 }, 5708 /* .pid */ 5709 /* .tid */ 5710 .start = vma->vm_start, 5711 .len = vma->vm_end - vma->vm_start, 5712 .pgoff = (u64)vma->vm_pgoff << PAGE_SHIFT, 5713 }, 5714 /* .maj (attr_mmap2 only) */ 5715 /* .min (attr_mmap2 only) */ 5716 /* .ino (attr_mmap2 only) */ 5717 /* .ino_generation (attr_mmap2 only) */ 5718 /* .prot (attr_mmap2 only) */ 5719 /* .flags (attr_mmap2 only) */ 5720 }; 5721 5722 perf_event_mmap_event(&mmap_event); 5723 } 5724 5725 /* 5726 * IRQ throttle logging 5727 */ 5728 5729 static void perf_log_throttle(struct perf_event *event, int enable) 5730 { 5731 struct perf_output_handle handle; 5732 struct perf_sample_data sample; 5733 int ret; 5734 5735 struct { 5736 struct perf_event_header header; 5737 u64 time; 5738 u64 id; 5739 u64 stream_id; 5740 } throttle_event = { 5741 .header = { 5742 .type = PERF_RECORD_THROTTLE, 5743 .misc = 0, 5744 .size = sizeof(throttle_event), 5745 }, 5746 .time = perf_clock(), 5747 .id = primary_event_id(event), 5748 .stream_id = event->id, 5749 }; 5750 5751 if (enable) 5752 throttle_event.header.type = PERF_RECORD_UNTHROTTLE; 5753 5754 perf_event_header__init_id(&throttle_event.header, &sample, event); 5755 5756 ret = perf_output_begin(&handle, event, 5757 throttle_event.header.size); 5758 if (ret) 5759 return; 5760 5761 perf_output_put(&handle, throttle_event); 5762 perf_event__output_id_sample(event, &handle, &sample); 5763 perf_output_end(&handle); 5764 } 5765 5766 /* 5767 * Generic event overflow handling, sampling. 5768 */ 5769 5770 static int __perf_event_overflow(struct perf_event *event, 5771 int throttle, struct perf_sample_data *data, 5772 struct pt_regs *regs) 5773 { 5774 int events = atomic_read(&event->event_limit); 5775 struct hw_perf_event *hwc = &event->hw; 5776 u64 seq; 5777 int ret = 0; 5778 5779 /* 5780 * Non-sampling counters might still use the PMI to fold short 5781 * hardware counters, ignore those. 5782 */ 5783 if (unlikely(!is_sampling_event(event))) 5784 return 0; 5785 5786 seq = __this_cpu_read(perf_throttled_seq); 5787 if (seq != hwc->interrupts_seq) { 5788 hwc->interrupts_seq = seq; 5789 hwc->interrupts = 1; 5790 } else { 5791 hwc->interrupts++; 5792 if (unlikely(throttle 5793 && hwc->interrupts >= max_samples_per_tick)) { 5794 __this_cpu_inc(perf_throttled_count); 5795 hwc->interrupts = MAX_INTERRUPTS; 5796 perf_log_throttle(event, 0); 5797 tick_nohz_full_kick(); 5798 ret = 1; 5799 } 5800 } 5801 5802 if (event->attr.freq) { 5803 u64 now = perf_clock(); 5804 s64 delta = now - hwc->freq_time_stamp; 5805 5806 hwc->freq_time_stamp = now; 5807 5808 if (delta > 0 && delta < 2*TICK_NSEC) 5809 perf_adjust_period(event, delta, hwc->last_period, true); 5810 } 5811 5812 /* 5813 * XXX event_limit might not quite work as expected on inherited 5814 * events 5815 */ 5816 5817 event->pending_kill = POLL_IN; 5818 if (events && atomic_dec_and_test(&event->event_limit)) { 5819 ret = 1; 5820 event->pending_kill = POLL_HUP; 5821 event->pending_disable = 1; 5822 irq_work_queue(&event->pending); 5823 } 5824 5825 if (event->overflow_handler) 5826 event->overflow_handler(event, data, regs); 5827 else 5828 perf_event_output(event, data, regs); 5829 5830 if (event->fasync && event->pending_kill) { 5831 event->pending_wakeup = 1; 5832 irq_work_queue(&event->pending); 5833 } 5834 5835 return ret; 5836 } 5837 5838 int perf_event_overflow(struct perf_event *event, 5839 struct perf_sample_data *data, 5840 struct pt_regs *regs) 5841 { 5842 return __perf_event_overflow(event, 1, data, regs); 5843 } 5844 5845 /* 5846 * Generic software event infrastructure 5847 */ 5848 5849 struct swevent_htable { 5850 struct swevent_hlist *swevent_hlist; 5851 struct mutex hlist_mutex; 5852 int hlist_refcount; 5853 5854 /* Recursion avoidance in each contexts */ 5855 int recursion[PERF_NR_CONTEXTS]; 5856 5857 /* Keeps track of cpu being initialized/exited */ 5858 bool online; 5859 }; 5860 5861 static DEFINE_PER_CPU(struct swevent_htable, swevent_htable); 5862 5863 /* 5864 * We directly increment event->count and keep a second value in 5865 * event->hw.period_left to count intervals. This period event 5866 * is kept in the range [-sample_period, 0] so that we can use the 5867 * sign as trigger. 5868 */ 5869 5870 u64 perf_swevent_set_period(struct perf_event *event) 5871 { 5872 struct hw_perf_event *hwc = &event->hw; 5873 u64 period = hwc->last_period; 5874 u64 nr, offset; 5875 s64 old, val; 5876 5877 hwc->last_period = hwc->sample_period; 5878 5879 again: 5880 old = val = local64_read(&hwc->period_left); 5881 if (val < 0) 5882 return 0; 5883 5884 nr = div64_u64(period + val, period); 5885 offset = nr * period; 5886 val -= offset; 5887 if (local64_cmpxchg(&hwc->period_left, old, val) != old) 5888 goto again; 5889 5890 return nr; 5891 } 5892 5893 static void perf_swevent_overflow(struct perf_event *event, u64 overflow, 5894 struct perf_sample_data *data, 5895 struct pt_regs *regs) 5896 { 5897 struct hw_perf_event *hwc = &event->hw; 5898 int throttle = 0; 5899 5900 if (!overflow) 5901 overflow = perf_swevent_set_period(event); 5902 5903 if (hwc->interrupts == MAX_INTERRUPTS) 5904 return; 5905 5906 for (; overflow; overflow--) { 5907 if (__perf_event_overflow(event, throttle, 5908 data, regs)) { 5909 /* 5910 * We inhibit the overflow from happening when 5911 * hwc->interrupts == MAX_INTERRUPTS. 5912 */ 5913 break; 5914 } 5915 throttle = 1; 5916 } 5917 } 5918 5919 static void perf_swevent_event(struct perf_event *event, u64 nr, 5920 struct perf_sample_data *data, 5921 struct pt_regs *regs) 5922 { 5923 struct hw_perf_event *hwc = &event->hw; 5924 5925 local64_add(nr, &event->count); 5926 5927 if (!regs) 5928 return; 5929 5930 if (!is_sampling_event(event)) 5931 return; 5932 5933 if ((event->attr.sample_type & PERF_SAMPLE_PERIOD) && !event->attr.freq) { 5934 data->period = nr; 5935 return perf_swevent_overflow(event, 1, data, regs); 5936 } else 5937 data->period = event->hw.last_period; 5938 5939 if (nr == 1 && hwc->sample_period == 1 && !event->attr.freq) 5940 return perf_swevent_overflow(event, 1, data, regs); 5941 5942 if (local64_add_negative(nr, &hwc->period_left)) 5943 return; 5944 5945 perf_swevent_overflow(event, 0, data, regs); 5946 } 5947 5948 static int perf_exclude_event(struct perf_event *event, 5949 struct pt_regs *regs) 5950 { 5951 if (event->hw.state & PERF_HES_STOPPED) 5952 return 1; 5953 5954 if (regs) { 5955 if (event->attr.exclude_user && user_mode(regs)) 5956 return 1; 5957 5958 if (event->attr.exclude_kernel && !user_mode(regs)) 5959 return 1; 5960 } 5961 5962 return 0; 5963 } 5964 5965 static int perf_swevent_match(struct perf_event *event, 5966 enum perf_type_id type, 5967 u32 event_id, 5968 struct perf_sample_data *data, 5969 struct pt_regs *regs) 5970 { 5971 if (event->attr.type != type) 5972 return 0; 5973 5974 if (event->attr.config != event_id) 5975 return 0; 5976 5977 if (perf_exclude_event(event, regs)) 5978 return 0; 5979 5980 return 1; 5981 } 5982 5983 static inline u64 swevent_hash(u64 type, u32 event_id) 5984 { 5985 u64 val = event_id | (type << 32); 5986 5987 return hash_64(val, SWEVENT_HLIST_BITS); 5988 } 5989 5990 static inline struct hlist_head * 5991 __find_swevent_head(struct swevent_hlist *hlist, u64 type, u32 event_id) 5992 { 5993 u64 hash = swevent_hash(type, event_id); 5994 5995 return &hlist->heads[hash]; 5996 } 5997 5998 /* For the read side: events when they trigger */ 5999 static inline struct hlist_head * 6000 find_swevent_head_rcu(struct swevent_htable *swhash, u64 type, u32 event_id) 6001 { 6002 struct swevent_hlist *hlist; 6003 6004 hlist = rcu_dereference(swhash->swevent_hlist); 6005 if (!hlist) 6006 return NULL; 6007 6008 return __find_swevent_head(hlist, type, event_id); 6009 } 6010 6011 /* For the event head insertion and removal in the hlist */ 6012 static inline struct hlist_head * 6013 find_swevent_head(struct swevent_htable *swhash, struct perf_event *event) 6014 { 6015 struct swevent_hlist *hlist; 6016 u32 event_id = event->attr.config; 6017 u64 type = event->attr.type; 6018 6019 /* 6020 * Event scheduling is always serialized against hlist allocation 6021 * and release. Which makes the protected version suitable here. 6022 * The context lock guarantees that. 6023 */ 6024 hlist = rcu_dereference_protected(swhash->swevent_hlist, 6025 lockdep_is_held(&event->ctx->lock)); 6026 if (!hlist) 6027 return NULL; 6028 6029 return __find_swevent_head(hlist, type, event_id); 6030 } 6031 6032 static void do_perf_sw_event(enum perf_type_id type, u32 event_id, 6033 u64 nr, 6034 struct perf_sample_data *data, 6035 struct pt_regs *regs) 6036 { 6037 struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable); 6038 struct perf_event *event; 6039 struct hlist_head *head; 6040 6041 rcu_read_lock(); 6042 head = find_swevent_head_rcu(swhash, type, event_id); 6043 if (!head) 6044 goto end; 6045 6046 hlist_for_each_entry_rcu(event, head, hlist_entry) { 6047 if (perf_swevent_match(event, type, event_id, data, regs)) 6048 perf_swevent_event(event, nr, data, regs); 6049 } 6050 end: 6051 rcu_read_unlock(); 6052 } 6053 6054 DEFINE_PER_CPU(struct pt_regs, __perf_regs[4]); 6055 6056 int perf_swevent_get_recursion_context(void) 6057 { 6058 struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable); 6059 6060 return get_recursion_context(swhash->recursion); 6061 } 6062 EXPORT_SYMBOL_GPL(perf_swevent_get_recursion_context); 6063 6064 inline void perf_swevent_put_recursion_context(int rctx) 6065 { 6066 struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable); 6067 6068 put_recursion_context(swhash->recursion, rctx); 6069 } 6070 6071 void ___perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr) 6072 { 6073 struct perf_sample_data data; 6074 6075 if (WARN_ON_ONCE(!regs)) 6076 return; 6077 6078 perf_sample_data_init(&data, addr, 0); 6079 do_perf_sw_event(PERF_TYPE_SOFTWARE, event_id, nr, &data, regs); 6080 } 6081 6082 void __perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr) 6083 { 6084 int rctx; 6085 6086 preempt_disable_notrace(); 6087 rctx = perf_swevent_get_recursion_context(); 6088 if (unlikely(rctx < 0)) 6089 goto fail; 6090 6091 ___perf_sw_event(event_id, nr, regs, addr); 6092 6093 perf_swevent_put_recursion_context(rctx); 6094 fail: 6095 preempt_enable_notrace(); 6096 } 6097 6098 static void perf_swevent_read(struct perf_event *event) 6099 { 6100 } 6101 6102 static int perf_swevent_add(struct perf_event *event, int flags) 6103 { 6104 struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable); 6105 struct hw_perf_event *hwc = &event->hw; 6106 struct hlist_head *head; 6107 6108 if (is_sampling_event(event)) { 6109 hwc->last_period = hwc->sample_period; 6110 perf_swevent_set_period(event); 6111 } 6112 6113 hwc->state = !(flags & PERF_EF_START); 6114 6115 head = find_swevent_head(swhash, event); 6116 if (!head) { 6117 /* 6118 * We can race with cpu hotplug code. Do not 6119 * WARN if the cpu just got unplugged. 6120 */ 6121 WARN_ON_ONCE(swhash->online); 6122 return -EINVAL; 6123 } 6124 6125 hlist_add_head_rcu(&event->hlist_entry, head); 6126 6127 return 0; 6128 } 6129 6130 static void perf_swevent_del(struct perf_event *event, int flags) 6131 { 6132 hlist_del_rcu(&event->hlist_entry); 6133 } 6134 6135 static void perf_swevent_start(struct perf_event *event, int flags) 6136 { 6137 event->hw.state = 0; 6138 } 6139 6140 static void perf_swevent_stop(struct perf_event *event, int flags) 6141 { 6142 event->hw.state = PERF_HES_STOPPED; 6143 } 6144 6145 /* Deref the hlist from the update side */ 6146 static inline struct swevent_hlist * 6147 swevent_hlist_deref(struct swevent_htable *swhash) 6148 { 6149 return rcu_dereference_protected(swhash->swevent_hlist, 6150 lockdep_is_held(&swhash->hlist_mutex)); 6151 } 6152 6153 static void swevent_hlist_release(struct swevent_htable *swhash) 6154 { 6155 struct swevent_hlist *hlist = swevent_hlist_deref(swhash); 6156 6157 if (!hlist) 6158 return; 6159 6160 RCU_INIT_POINTER(swhash->swevent_hlist, NULL); 6161 kfree_rcu(hlist, rcu_head); 6162 } 6163 6164 static void swevent_hlist_put_cpu(struct perf_event *event, int cpu) 6165 { 6166 struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu); 6167 6168 mutex_lock(&swhash->hlist_mutex); 6169 6170 if (!--swhash->hlist_refcount) 6171 swevent_hlist_release(swhash); 6172 6173 mutex_unlock(&swhash->hlist_mutex); 6174 } 6175 6176 static void swevent_hlist_put(struct perf_event *event) 6177 { 6178 int cpu; 6179 6180 for_each_possible_cpu(cpu) 6181 swevent_hlist_put_cpu(event, cpu); 6182 } 6183 6184 static int swevent_hlist_get_cpu(struct perf_event *event, int cpu) 6185 { 6186 struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu); 6187 int err = 0; 6188 6189 mutex_lock(&swhash->hlist_mutex); 6190 6191 if (!swevent_hlist_deref(swhash) && cpu_online(cpu)) { 6192 struct swevent_hlist *hlist; 6193 6194 hlist = kzalloc(sizeof(*hlist), GFP_KERNEL); 6195 if (!hlist) { 6196 err = -ENOMEM; 6197 goto exit; 6198 } 6199 rcu_assign_pointer(swhash->swevent_hlist, hlist); 6200 } 6201 swhash->hlist_refcount++; 6202 exit: 6203 mutex_unlock(&swhash->hlist_mutex); 6204 6205 return err; 6206 } 6207 6208 static int swevent_hlist_get(struct perf_event *event) 6209 { 6210 int err; 6211 int cpu, failed_cpu; 6212 6213 get_online_cpus(); 6214 for_each_possible_cpu(cpu) { 6215 err = swevent_hlist_get_cpu(event, cpu); 6216 if (err) { 6217 failed_cpu = cpu; 6218 goto fail; 6219 } 6220 } 6221 put_online_cpus(); 6222 6223 return 0; 6224 fail: 6225 for_each_possible_cpu(cpu) { 6226 if (cpu == failed_cpu) 6227 break; 6228 swevent_hlist_put_cpu(event, cpu); 6229 } 6230 6231 put_online_cpus(); 6232 return err; 6233 } 6234 6235 struct static_key perf_swevent_enabled[PERF_COUNT_SW_MAX]; 6236 6237 static void sw_perf_event_destroy(struct perf_event *event) 6238 { 6239 u64 event_id = event->attr.config; 6240 6241 WARN_ON(event->parent); 6242 6243 static_key_slow_dec(&perf_swevent_enabled[event_id]); 6244 swevent_hlist_put(event); 6245 } 6246 6247 static int perf_swevent_init(struct perf_event *event) 6248 { 6249 u64 event_id = event->attr.config; 6250 6251 if (event->attr.type != PERF_TYPE_SOFTWARE) 6252 return -ENOENT; 6253 6254 /* 6255 * no branch sampling for software events 6256 */ 6257 if (has_branch_stack(event)) 6258 return -EOPNOTSUPP; 6259 6260 switch (event_id) { 6261 case PERF_COUNT_SW_CPU_CLOCK: 6262 case PERF_COUNT_SW_TASK_CLOCK: 6263 return -ENOENT; 6264 6265 default: 6266 break; 6267 } 6268 6269 if (event_id >= PERF_COUNT_SW_MAX) 6270 return -ENOENT; 6271 6272 if (!event->parent) { 6273 int err; 6274 6275 err = swevent_hlist_get(event); 6276 if (err) 6277 return err; 6278 6279 static_key_slow_inc(&perf_swevent_enabled[event_id]); 6280 event->destroy = sw_perf_event_destroy; 6281 } 6282 6283 return 0; 6284 } 6285 6286 static struct pmu perf_swevent = { 6287 .task_ctx_nr = perf_sw_context, 6288 6289 .event_init = perf_swevent_init, 6290 .add = perf_swevent_add, 6291 .del = perf_swevent_del, 6292 .start = perf_swevent_start, 6293 .stop = perf_swevent_stop, 6294 .read = perf_swevent_read, 6295 }; 6296 6297 #ifdef CONFIG_EVENT_TRACING 6298 6299 static int perf_tp_filter_match(struct perf_event *event, 6300 struct perf_sample_data *data) 6301 { 6302 void *record = data->raw->data; 6303 6304 if (likely(!event->filter) || filter_match_preds(event->filter, record)) 6305 return 1; 6306 return 0; 6307 } 6308 6309 static int perf_tp_event_match(struct perf_event *event, 6310 struct perf_sample_data *data, 6311 struct pt_regs *regs) 6312 { 6313 if (event->hw.state & PERF_HES_STOPPED) 6314 return 0; 6315 /* 6316 * All tracepoints are from kernel-space. 6317 */ 6318 if (event->attr.exclude_kernel) 6319 return 0; 6320 6321 if (!perf_tp_filter_match(event, data)) 6322 return 0; 6323 6324 return 1; 6325 } 6326 6327 void perf_tp_event(u64 addr, u64 count, void *record, int entry_size, 6328 struct pt_regs *regs, struct hlist_head *head, int rctx, 6329 struct task_struct *task) 6330 { 6331 struct perf_sample_data data; 6332 struct perf_event *event; 6333 6334 struct perf_raw_record raw = { 6335 .size = entry_size, 6336 .data = record, 6337 }; 6338 6339 perf_sample_data_init(&data, addr, 0); 6340 data.raw = &raw; 6341 6342 hlist_for_each_entry_rcu(event, head, hlist_entry) { 6343 if (perf_tp_event_match(event, &data, regs)) 6344 perf_swevent_event(event, count, &data, regs); 6345 } 6346 6347 /* 6348 * If we got specified a target task, also iterate its context and 6349 * deliver this event there too. 6350 */ 6351 if (task && task != current) { 6352 struct perf_event_context *ctx; 6353 struct trace_entry *entry = record; 6354 6355 rcu_read_lock(); 6356 ctx = rcu_dereference(task->perf_event_ctxp[perf_sw_context]); 6357 if (!ctx) 6358 goto unlock; 6359 6360 list_for_each_entry_rcu(event, &ctx->event_list, event_entry) { 6361 if (event->attr.type != PERF_TYPE_TRACEPOINT) 6362 continue; 6363 if (event->attr.config != entry->type) 6364 continue; 6365 if (perf_tp_event_match(event, &data, regs)) 6366 perf_swevent_event(event, count, &data, regs); 6367 } 6368 unlock: 6369 rcu_read_unlock(); 6370 } 6371 6372 perf_swevent_put_recursion_context(rctx); 6373 } 6374 EXPORT_SYMBOL_GPL(perf_tp_event); 6375 6376 static void tp_perf_event_destroy(struct perf_event *event) 6377 { 6378 perf_trace_destroy(event); 6379 } 6380 6381 static int perf_tp_event_init(struct perf_event *event) 6382 { 6383 int err; 6384 6385 if (event->attr.type != PERF_TYPE_TRACEPOINT) 6386 return -ENOENT; 6387 6388 /* 6389 * no branch sampling for tracepoint events 6390 */ 6391 if (has_branch_stack(event)) 6392 return -EOPNOTSUPP; 6393 6394 err = perf_trace_init(event); 6395 if (err) 6396 return err; 6397 6398 event->destroy = tp_perf_event_destroy; 6399 6400 return 0; 6401 } 6402 6403 static struct pmu perf_tracepoint = { 6404 .task_ctx_nr = perf_sw_context, 6405 6406 .event_init = perf_tp_event_init, 6407 .add = perf_trace_add, 6408 .del = perf_trace_del, 6409 .start = perf_swevent_start, 6410 .stop = perf_swevent_stop, 6411 .read = perf_swevent_read, 6412 }; 6413 6414 static inline void perf_tp_register(void) 6415 { 6416 perf_pmu_register(&perf_tracepoint, "tracepoint", PERF_TYPE_TRACEPOINT); 6417 } 6418 6419 static int perf_event_set_filter(struct perf_event *event, void __user *arg) 6420 { 6421 char *filter_str; 6422 int ret; 6423 6424 if (event->attr.type != PERF_TYPE_TRACEPOINT) 6425 return -EINVAL; 6426 6427 filter_str = strndup_user(arg, PAGE_SIZE); 6428 if (IS_ERR(filter_str)) 6429 return PTR_ERR(filter_str); 6430 6431 ret = ftrace_profile_set_filter(event, event->attr.config, filter_str); 6432 6433 kfree(filter_str); 6434 return ret; 6435 } 6436 6437 static void perf_event_free_filter(struct perf_event *event) 6438 { 6439 ftrace_profile_free_filter(event); 6440 } 6441 6442 #else 6443 6444 static inline void perf_tp_register(void) 6445 { 6446 } 6447 6448 static int perf_event_set_filter(struct perf_event *event, void __user *arg) 6449 { 6450 return -ENOENT; 6451 } 6452 6453 static void perf_event_free_filter(struct perf_event *event) 6454 { 6455 } 6456 6457 #endif /* CONFIG_EVENT_TRACING */ 6458 6459 #ifdef CONFIG_HAVE_HW_BREAKPOINT 6460 void perf_bp_event(struct perf_event *bp, void *data) 6461 { 6462 struct perf_sample_data sample; 6463 struct pt_regs *regs = data; 6464 6465 perf_sample_data_init(&sample, bp->attr.bp_addr, 0); 6466 6467 if (!bp->hw.state && !perf_exclude_event(bp, regs)) 6468 perf_swevent_event(bp, 1, &sample, regs); 6469 } 6470 #endif 6471 6472 /* 6473 * hrtimer based swevent callback 6474 */ 6475 6476 static enum hrtimer_restart perf_swevent_hrtimer(struct hrtimer *hrtimer) 6477 { 6478 enum hrtimer_restart ret = HRTIMER_RESTART; 6479 struct perf_sample_data data; 6480 struct pt_regs *regs; 6481 struct perf_event *event; 6482 u64 period; 6483 6484 event = container_of(hrtimer, struct perf_event, hw.hrtimer); 6485 6486 if (event->state != PERF_EVENT_STATE_ACTIVE) 6487 return HRTIMER_NORESTART; 6488 6489 event->pmu->read(event); 6490 6491 perf_sample_data_init(&data, 0, event->hw.last_period); 6492 regs = get_irq_regs(); 6493 6494 if (regs && !perf_exclude_event(event, regs)) { 6495 if (!(event->attr.exclude_idle && is_idle_task(current))) 6496 if (__perf_event_overflow(event, 1, &data, regs)) 6497 ret = HRTIMER_NORESTART; 6498 } 6499 6500 period = max_t(u64, 10000, event->hw.sample_period); 6501 hrtimer_forward_now(hrtimer, ns_to_ktime(period)); 6502 6503 return ret; 6504 } 6505 6506 static void perf_swevent_start_hrtimer(struct perf_event *event) 6507 { 6508 struct hw_perf_event *hwc = &event->hw; 6509 s64 period; 6510 6511 if (!is_sampling_event(event)) 6512 return; 6513 6514 period = local64_read(&hwc->period_left); 6515 if (period) { 6516 if (period < 0) 6517 period = 10000; 6518 6519 local64_set(&hwc->period_left, 0); 6520 } else { 6521 period = max_t(u64, 10000, hwc->sample_period); 6522 } 6523 __hrtimer_start_range_ns(&hwc->hrtimer, 6524 ns_to_ktime(period), 0, 6525 HRTIMER_MODE_REL_PINNED, 0); 6526 } 6527 6528 static void perf_swevent_cancel_hrtimer(struct perf_event *event) 6529 { 6530 struct hw_perf_event *hwc = &event->hw; 6531 6532 if (is_sampling_event(event)) { 6533 ktime_t remaining = hrtimer_get_remaining(&hwc->hrtimer); 6534 local64_set(&hwc->period_left, ktime_to_ns(remaining)); 6535 6536 hrtimer_cancel(&hwc->hrtimer); 6537 } 6538 } 6539 6540 static void perf_swevent_init_hrtimer(struct perf_event *event) 6541 { 6542 struct hw_perf_event *hwc = &event->hw; 6543 6544 if (!is_sampling_event(event)) 6545 return; 6546 6547 hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 6548 hwc->hrtimer.function = perf_swevent_hrtimer; 6549 6550 /* 6551 * Since hrtimers have a fixed rate, we can do a static freq->period 6552 * mapping and avoid the whole period adjust feedback stuff. 6553 */ 6554 if (event->attr.freq) { 6555 long freq = event->attr.sample_freq; 6556 6557 event->attr.sample_period = NSEC_PER_SEC / freq; 6558 hwc->sample_period = event->attr.sample_period; 6559 local64_set(&hwc->period_left, hwc->sample_period); 6560 hwc->last_period = hwc->sample_period; 6561 event->attr.freq = 0; 6562 } 6563 } 6564 6565 /* 6566 * Software event: cpu wall time clock 6567 */ 6568 6569 static void cpu_clock_event_update(struct perf_event *event) 6570 { 6571 s64 prev; 6572 u64 now; 6573 6574 now = local_clock(); 6575 prev = local64_xchg(&event->hw.prev_count, now); 6576 local64_add(now - prev, &event->count); 6577 } 6578 6579 static void cpu_clock_event_start(struct perf_event *event, int flags) 6580 { 6581 local64_set(&event->hw.prev_count, local_clock()); 6582 perf_swevent_start_hrtimer(event); 6583 } 6584 6585 static void cpu_clock_event_stop(struct perf_event *event, int flags) 6586 { 6587 perf_swevent_cancel_hrtimer(event); 6588 cpu_clock_event_update(event); 6589 } 6590 6591 static int cpu_clock_event_add(struct perf_event *event, int flags) 6592 { 6593 if (flags & PERF_EF_START) 6594 cpu_clock_event_start(event, flags); 6595 6596 return 0; 6597 } 6598 6599 static void cpu_clock_event_del(struct perf_event *event, int flags) 6600 { 6601 cpu_clock_event_stop(event, flags); 6602 } 6603 6604 static void cpu_clock_event_read(struct perf_event *event) 6605 { 6606 cpu_clock_event_update(event); 6607 } 6608 6609 static int cpu_clock_event_init(struct perf_event *event) 6610 { 6611 if (event->attr.type != PERF_TYPE_SOFTWARE) 6612 return -ENOENT; 6613 6614 if (event->attr.config != PERF_COUNT_SW_CPU_CLOCK) 6615 return -ENOENT; 6616 6617 /* 6618 * no branch sampling for software events 6619 */ 6620 if (has_branch_stack(event)) 6621 return -EOPNOTSUPP; 6622 6623 perf_swevent_init_hrtimer(event); 6624 6625 return 0; 6626 } 6627 6628 static struct pmu perf_cpu_clock = { 6629 .task_ctx_nr = perf_sw_context, 6630 6631 .event_init = cpu_clock_event_init, 6632 .add = cpu_clock_event_add, 6633 .del = cpu_clock_event_del, 6634 .start = cpu_clock_event_start, 6635 .stop = cpu_clock_event_stop, 6636 .read = cpu_clock_event_read, 6637 }; 6638 6639 /* 6640 * Software event: task time clock 6641 */ 6642 6643 static void task_clock_event_update(struct perf_event *event, u64 now) 6644 { 6645 u64 prev; 6646 s64 delta; 6647 6648 prev = local64_xchg(&event->hw.prev_count, now); 6649 delta = now - prev; 6650 local64_add(delta, &event->count); 6651 } 6652 6653 static void task_clock_event_start(struct perf_event *event, int flags) 6654 { 6655 local64_set(&event->hw.prev_count, event->ctx->time); 6656 perf_swevent_start_hrtimer(event); 6657 } 6658 6659 static void task_clock_event_stop(struct perf_event *event, int flags) 6660 { 6661 perf_swevent_cancel_hrtimer(event); 6662 task_clock_event_update(event, event->ctx->time); 6663 } 6664 6665 static int task_clock_event_add(struct perf_event *event, int flags) 6666 { 6667 if (flags & PERF_EF_START) 6668 task_clock_event_start(event, flags); 6669 6670 return 0; 6671 } 6672 6673 static void task_clock_event_del(struct perf_event *event, int flags) 6674 { 6675 task_clock_event_stop(event, PERF_EF_UPDATE); 6676 } 6677 6678 static void task_clock_event_read(struct perf_event *event) 6679 { 6680 u64 now = perf_clock(); 6681 u64 delta = now - event->ctx->timestamp; 6682 u64 time = event->ctx->time + delta; 6683 6684 task_clock_event_update(event, time); 6685 } 6686 6687 static int task_clock_event_init(struct perf_event *event) 6688 { 6689 if (event->attr.type != PERF_TYPE_SOFTWARE) 6690 return -ENOENT; 6691 6692 if (event->attr.config != PERF_COUNT_SW_TASK_CLOCK) 6693 return -ENOENT; 6694 6695 /* 6696 * no branch sampling for software events 6697 */ 6698 if (has_branch_stack(event)) 6699 return -EOPNOTSUPP; 6700 6701 perf_swevent_init_hrtimer(event); 6702 6703 return 0; 6704 } 6705 6706 static struct pmu perf_task_clock = { 6707 .task_ctx_nr = perf_sw_context, 6708 6709 .event_init = task_clock_event_init, 6710 .add = task_clock_event_add, 6711 .del = task_clock_event_del, 6712 .start = task_clock_event_start, 6713 .stop = task_clock_event_stop, 6714 .read = task_clock_event_read, 6715 }; 6716 6717 static void perf_pmu_nop_void(struct pmu *pmu) 6718 { 6719 } 6720 6721 static int perf_pmu_nop_int(struct pmu *pmu) 6722 { 6723 return 0; 6724 } 6725 6726 static void perf_pmu_start_txn(struct pmu *pmu) 6727 { 6728 perf_pmu_disable(pmu); 6729 } 6730 6731 static int perf_pmu_commit_txn(struct pmu *pmu) 6732 { 6733 perf_pmu_enable(pmu); 6734 return 0; 6735 } 6736 6737 static void perf_pmu_cancel_txn(struct pmu *pmu) 6738 { 6739 perf_pmu_enable(pmu); 6740 } 6741 6742 static int perf_event_idx_default(struct perf_event *event) 6743 { 6744 return 0; 6745 } 6746 6747 /* 6748 * Ensures all contexts with the same task_ctx_nr have the same 6749 * pmu_cpu_context too. 6750 */ 6751 static struct perf_cpu_context __percpu *find_pmu_context(int ctxn) 6752 { 6753 struct pmu *pmu; 6754 6755 if (ctxn < 0) 6756 return NULL; 6757 6758 list_for_each_entry(pmu, &pmus, entry) { 6759 if (pmu->task_ctx_nr == ctxn) 6760 return pmu->pmu_cpu_context; 6761 } 6762 6763 return NULL; 6764 } 6765 6766 static void update_pmu_context(struct pmu *pmu, struct pmu *old_pmu) 6767 { 6768 int cpu; 6769 6770 for_each_possible_cpu(cpu) { 6771 struct perf_cpu_context *cpuctx; 6772 6773 cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu); 6774 6775 if (cpuctx->unique_pmu == old_pmu) 6776 cpuctx->unique_pmu = pmu; 6777 } 6778 } 6779 6780 static void free_pmu_context(struct pmu *pmu) 6781 { 6782 struct pmu *i; 6783 6784 mutex_lock(&pmus_lock); 6785 /* 6786 * Like a real lame refcount. 6787 */ 6788 list_for_each_entry(i, &pmus, entry) { 6789 if (i->pmu_cpu_context == pmu->pmu_cpu_context) { 6790 update_pmu_context(i, pmu); 6791 goto out; 6792 } 6793 } 6794 6795 free_percpu(pmu->pmu_cpu_context); 6796 out: 6797 mutex_unlock(&pmus_lock); 6798 } 6799 static struct idr pmu_idr; 6800 6801 static ssize_t 6802 type_show(struct device *dev, struct device_attribute *attr, char *page) 6803 { 6804 struct pmu *pmu = dev_get_drvdata(dev); 6805 6806 return snprintf(page, PAGE_SIZE-1, "%d\n", pmu->type); 6807 } 6808 static DEVICE_ATTR_RO(type); 6809 6810 static ssize_t 6811 perf_event_mux_interval_ms_show(struct device *dev, 6812 struct device_attribute *attr, 6813 char *page) 6814 { 6815 struct pmu *pmu = dev_get_drvdata(dev); 6816 6817 return snprintf(page, PAGE_SIZE-1, "%d\n", pmu->hrtimer_interval_ms); 6818 } 6819 6820 static ssize_t 6821 perf_event_mux_interval_ms_store(struct device *dev, 6822 struct device_attribute *attr, 6823 const char *buf, size_t count) 6824 { 6825 struct pmu *pmu = dev_get_drvdata(dev); 6826 int timer, cpu, ret; 6827 6828 ret = kstrtoint(buf, 0, &timer); 6829 if (ret) 6830 return ret; 6831 6832 if (timer < 1) 6833 return -EINVAL; 6834 6835 /* same value, noting to do */ 6836 if (timer == pmu->hrtimer_interval_ms) 6837 return count; 6838 6839 pmu->hrtimer_interval_ms = timer; 6840 6841 /* update all cpuctx for this PMU */ 6842 for_each_possible_cpu(cpu) { 6843 struct perf_cpu_context *cpuctx; 6844 cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu); 6845 cpuctx->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * timer); 6846 6847 if (hrtimer_active(&cpuctx->hrtimer)) 6848 hrtimer_forward_now(&cpuctx->hrtimer, cpuctx->hrtimer_interval); 6849 } 6850 6851 return count; 6852 } 6853 static DEVICE_ATTR_RW(perf_event_mux_interval_ms); 6854 6855 static struct attribute *pmu_dev_attrs[] = { 6856 &dev_attr_type.attr, 6857 &dev_attr_perf_event_mux_interval_ms.attr, 6858 NULL, 6859 }; 6860 ATTRIBUTE_GROUPS(pmu_dev); 6861 6862 static int pmu_bus_running; 6863 static struct bus_type pmu_bus = { 6864 .name = "event_source", 6865 .dev_groups = pmu_dev_groups, 6866 }; 6867 6868 static void pmu_dev_release(struct device *dev) 6869 { 6870 kfree(dev); 6871 } 6872 6873 static int pmu_dev_alloc(struct pmu *pmu) 6874 { 6875 int ret = -ENOMEM; 6876 6877 pmu->dev = kzalloc(sizeof(struct device), GFP_KERNEL); 6878 if (!pmu->dev) 6879 goto out; 6880 6881 pmu->dev->groups = pmu->attr_groups; 6882 device_initialize(pmu->dev); 6883 ret = dev_set_name(pmu->dev, "%s", pmu->name); 6884 if (ret) 6885 goto free_dev; 6886 6887 dev_set_drvdata(pmu->dev, pmu); 6888 pmu->dev->bus = &pmu_bus; 6889 pmu->dev->release = pmu_dev_release; 6890 ret = device_add(pmu->dev); 6891 if (ret) 6892 goto free_dev; 6893 6894 out: 6895 return ret; 6896 6897 free_dev: 6898 put_device(pmu->dev); 6899 goto out; 6900 } 6901 6902 static struct lock_class_key cpuctx_mutex; 6903 static struct lock_class_key cpuctx_lock; 6904 6905 int perf_pmu_register(struct pmu *pmu, const char *name, int type) 6906 { 6907 int cpu, ret; 6908 6909 mutex_lock(&pmus_lock); 6910 ret = -ENOMEM; 6911 pmu->pmu_disable_count = alloc_percpu(int); 6912 if (!pmu->pmu_disable_count) 6913 goto unlock; 6914 6915 pmu->type = -1; 6916 if (!name) 6917 goto skip_type; 6918 pmu->name = name; 6919 6920 if (type < 0) { 6921 type = idr_alloc(&pmu_idr, pmu, PERF_TYPE_MAX, 0, GFP_KERNEL); 6922 if (type < 0) { 6923 ret = type; 6924 goto free_pdc; 6925 } 6926 } 6927 pmu->type = type; 6928 6929 if (pmu_bus_running) { 6930 ret = pmu_dev_alloc(pmu); 6931 if (ret) 6932 goto free_idr; 6933 } 6934 6935 skip_type: 6936 pmu->pmu_cpu_context = find_pmu_context(pmu->task_ctx_nr); 6937 if (pmu->pmu_cpu_context) 6938 goto got_cpu_context; 6939 6940 ret = -ENOMEM; 6941 pmu->pmu_cpu_context = alloc_percpu(struct perf_cpu_context); 6942 if (!pmu->pmu_cpu_context) 6943 goto free_dev; 6944 6945 for_each_possible_cpu(cpu) { 6946 struct perf_cpu_context *cpuctx; 6947 6948 cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu); 6949 __perf_event_init_context(&cpuctx->ctx); 6950 lockdep_set_class(&cpuctx->ctx.mutex, &cpuctx_mutex); 6951 lockdep_set_class(&cpuctx->ctx.lock, &cpuctx_lock); 6952 cpuctx->ctx.pmu = pmu; 6953 6954 __perf_cpu_hrtimer_init(cpuctx, cpu); 6955 6956 cpuctx->unique_pmu = pmu; 6957 } 6958 6959 got_cpu_context: 6960 if (!pmu->start_txn) { 6961 if (pmu->pmu_enable) { 6962 /* 6963 * If we have pmu_enable/pmu_disable calls, install 6964 * transaction stubs that use that to try and batch 6965 * hardware accesses. 6966 */ 6967 pmu->start_txn = perf_pmu_start_txn; 6968 pmu->commit_txn = perf_pmu_commit_txn; 6969 pmu->cancel_txn = perf_pmu_cancel_txn; 6970 } else { 6971 pmu->start_txn = perf_pmu_nop_void; 6972 pmu->commit_txn = perf_pmu_nop_int; 6973 pmu->cancel_txn = perf_pmu_nop_void; 6974 } 6975 } 6976 6977 if (!pmu->pmu_enable) { 6978 pmu->pmu_enable = perf_pmu_nop_void; 6979 pmu->pmu_disable = perf_pmu_nop_void; 6980 } 6981 6982 if (!pmu->event_idx) 6983 pmu->event_idx = perf_event_idx_default; 6984 6985 list_add_rcu(&pmu->entry, &pmus); 6986 ret = 0; 6987 unlock: 6988 mutex_unlock(&pmus_lock); 6989 6990 return ret; 6991 6992 free_dev: 6993 device_del(pmu->dev); 6994 put_device(pmu->dev); 6995 6996 free_idr: 6997 if (pmu->type >= PERF_TYPE_MAX) 6998 idr_remove(&pmu_idr, pmu->type); 6999 7000 free_pdc: 7001 free_percpu(pmu->pmu_disable_count); 7002 goto unlock; 7003 } 7004 EXPORT_SYMBOL_GPL(perf_pmu_register); 7005 7006 void perf_pmu_unregister(struct pmu *pmu) 7007 { 7008 mutex_lock(&pmus_lock); 7009 list_del_rcu(&pmu->entry); 7010 mutex_unlock(&pmus_lock); 7011 7012 /* 7013 * We dereference the pmu list under both SRCU and regular RCU, so 7014 * synchronize against both of those. 7015 */ 7016 synchronize_srcu(&pmus_srcu); 7017 synchronize_rcu(); 7018 7019 free_percpu(pmu->pmu_disable_count); 7020 if (pmu->type >= PERF_TYPE_MAX) 7021 idr_remove(&pmu_idr, pmu->type); 7022 device_del(pmu->dev); 7023 put_device(pmu->dev); 7024 free_pmu_context(pmu); 7025 } 7026 EXPORT_SYMBOL_GPL(perf_pmu_unregister); 7027 7028 static int perf_try_init_event(struct pmu *pmu, struct perf_event *event) 7029 { 7030 int ret; 7031 7032 if (!try_module_get(pmu->module)) 7033 return -ENODEV; 7034 event->pmu = pmu; 7035 ret = pmu->event_init(event); 7036 if (ret) 7037 module_put(pmu->module); 7038 7039 return ret; 7040 } 7041 7042 struct pmu *perf_init_event(struct perf_event *event) 7043 { 7044 struct pmu *pmu = NULL; 7045 int idx; 7046 int ret; 7047 7048 idx = srcu_read_lock(&pmus_srcu); 7049 7050 rcu_read_lock(); 7051 pmu = idr_find(&pmu_idr, event->attr.type); 7052 rcu_read_unlock(); 7053 if (pmu) { 7054 ret = perf_try_init_event(pmu, event); 7055 if (ret) 7056 pmu = ERR_PTR(ret); 7057 goto unlock; 7058 } 7059 7060 list_for_each_entry_rcu(pmu, &pmus, entry) { 7061 ret = perf_try_init_event(pmu, event); 7062 if (!ret) 7063 goto unlock; 7064 7065 if (ret != -ENOENT) { 7066 pmu = ERR_PTR(ret); 7067 goto unlock; 7068 } 7069 } 7070 pmu = ERR_PTR(-ENOENT); 7071 unlock: 7072 srcu_read_unlock(&pmus_srcu, idx); 7073 7074 return pmu; 7075 } 7076 7077 static void account_event_cpu(struct perf_event *event, int cpu) 7078 { 7079 if (event->parent) 7080 return; 7081 7082 if (has_branch_stack(event)) { 7083 if (!(event->attach_state & PERF_ATTACH_TASK)) 7084 atomic_inc(&per_cpu(perf_branch_stack_events, cpu)); 7085 } 7086 if (is_cgroup_event(event)) 7087 atomic_inc(&per_cpu(perf_cgroup_events, cpu)); 7088 } 7089 7090 static void account_event(struct perf_event *event) 7091 { 7092 if (event->parent) 7093 return; 7094 7095 if (event->attach_state & PERF_ATTACH_TASK) 7096 static_key_slow_inc(&perf_sched_events.key); 7097 if (event->attr.mmap || event->attr.mmap_data) 7098 atomic_inc(&nr_mmap_events); 7099 if (event->attr.comm) 7100 atomic_inc(&nr_comm_events); 7101 if (event->attr.task) 7102 atomic_inc(&nr_task_events); 7103 if (event->attr.freq) { 7104 if (atomic_inc_return(&nr_freq_events) == 1) 7105 tick_nohz_full_kick_all(); 7106 } 7107 if (has_branch_stack(event)) 7108 static_key_slow_inc(&perf_sched_events.key); 7109 if (is_cgroup_event(event)) 7110 static_key_slow_inc(&perf_sched_events.key); 7111 7112 account_event_cpu(event, event->cpu); 7113 } 7114 7115 /* 7116 * Allocate and initialize a event structure 7117 */ 7118 static struct perf_event * 7119 perf_event_alloc(struct perf_event_attr *attr, int cpu, 7120 struct task_struct *task, 7121 struct perf_event *group_leader, 7122 struct perf_event *parent_event, 7123 perf_overflow_handler_t overflow_handler, 7124 void *context) 7125 { 7126 struct pmu *pmu; 7127 struct perf_event *event; 7128 struct hw_perf_event *hwc; 7129 long err = -EINVAL; 7130 7131 if ((unsigned)cpu >= nr_cpu_ids) { 7132 if (!task || cpu != -1) 7133 return ERR_PTR(-EINVAL); 7134 } 7135 7136 event = kzalloc(sizeof(*event), GFP_KERNEL); 7137 if (!event) 7138 return ERR_PTR(-ENOMEM); 7139 7140 /* 7141 * Single events are their own group leaders, with an 7142 * empty sibling list: 7143 */ 7144 if (!group_leader) 7145 group_leader = event; 7146 7147 mutex_init(&event->child_mutex); 7148 INIT_LIST_HEAD(&event->child_list); 7149 7150 INIT_LIST_HEAD(&event->group_entry); 7151 INIT_LIST_HEAD(&event->event_entry); 7152 INIT_LIST_HEAD(&event->sibling_list); 7153 INIT_LIST_HEAD(&event->rb_entry); 7154 INIT_LIST_HEAD(&event->active_entry); 7155 INIT_HLIST_NODE(&event->hlist_entry); 7156 7157 7158 init_waitqueue_head(&event->waitq); 7159 init_irq_work(&event->pending, perf_pending_event); 7160 7161 mutex_init(&event->mmap_mutex); 7162 7163 atomic_long_set(&event->refcount, 1); 7164 event->cpu = cpu; 7165 event->attr = *attr; 7166 event->group_leader = group_leader; 7167 event->pmu = NULL; 7168 event->oncpu = -1; 7169 7170 event->parent = parent_event; 7171 7172 event->ns = get_pid_ns(task_active_pid_ns(current)); 7173 event->id = atomic64_inc_return(&perf_event_id); 7174 7175 event->state = PERF_EVENT_STATE_INACTIVE; 7176 7177 if (task) { 7178 event->attach_state = PERF_ATTACH_TASK; 7179 7180 if (attr->type == PERF_TYPE_TRACEPOINT) 7181 event->hw.tp_target = task; 7182 #ifdef CONFIG_HAVE_HW_BREAKPOINT 7183 /* 7184 * hw_breakpoint is a bit difficult here.. 7185 */ 7186 else if (attr->type == PERF_TYPE_BREAKPOINT) 7187 event->hw.bp_target = task; 7188 #endif 7189 } 7190 7191 if (!overflow_handler && parent_event) { 7192 overflow_handler = parent_event->overflow_handler; 7193 context = parent_event->overflow_handler_context; 7194 } 7195 7196 event->overflow_handler = overflow_handler; 7197 event->overflow_handler_context = context; 7198 7199 perf_event__state_init(event); 7200 7201 pmu = NULL; 7202 7203 hwc = &event->hw; 7204 hwc->sample_period = attr->sample_period; 7205 if (attr->freq && attr->sample_freq) 7206 hwc->sample_period = 1; 7207 hwc->last_period = hwc->sample_period; 7208 7209 local64_set(&hwc->period_left, hwc->sample_period); 7210 7211 /* 7212 * we currently do not support PERF_FORMAT_GROUP on inherited events 7213 */ 7214 if (attr->inherit && (attr->read_format & PERF_FORMAT_GROUP)) 7215 goto err_ns; 7216 7217 pmu = perf_init_event(event); 7218 if (!pmu) 7219 goto err_ns; 7220 else if (IS_ERR(pmu)) { 7221 err = PTR_ERR(pmu); 7222 goto err_ns; 7223 } 7224 7225 if (!event->parent) { 7226 if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) { 7227 err = get_callchain_buffers(); 7228 if (err) 7229 goto err_pmu; 7230 } 7231 } 7232 7233 return event; 7234 7235 err_pmu: 7236 if (event->destroy) 7237 event->destroy(event); 7238 module_put(pmu->module); 7239 err_ns: 7240 if (event->ns) 7241 put_pid_ns(event->ns); 7242 kfree(event); 7243 7244 return ERR_PTR(err); 7245 } 7246 7247 static int perf_copy_attr(struct perf_event_attr __user *uattr, 7248 struct perf_event_attr *attr) 7249 { 7250 u32 size; 7251 int ret; 7252 7253 if (!access_ok(VERIFY_WRITE, uattr, PERF_ATTR_SIZE_VER0)) 7254 return -EFAULT; 7255 7256 /* 7257 * zero the full structure, so that a short copy will be nice. 7258 */ 7259 memset(attr, 0, sizeof(*attr)); 7260 7261 ret = get_user(size, &uattr->size); 7262 if (ret) 7263 return ret; 7264 7265 if (size > PAGE_SIZE) /* silly large */ 7266 goto err_size; 7267 7268 if (!size) /* abi compat */ 7269 size = PERF_ATTR_SIZE_VER0; 7270 7271 if (size < PERF_ATTR_SIZE_VER0) 7272 goto err_size; 7273 7274 /* 7275 * If we're handed a bigger struct than we know of, 7276 * ensure all the unknown bits are 0 - i.e. new 7277 * user-space does not rely on any kernel feature 7278 * extensions we dont know about yet. 7279 */ 7280 if (size > sizeof(*attr)) { 7281 unsigned char __user *addr; 7282 unsigned char __user *end; 7283 unsigned char val; 7284 7285 addr = (void __user *)uattr + sizeof(*attr); 7286 end = (void __user *)uattr + size; 7287 7288 for (; addr < end; addr++) { 7289 ret = get_user(val, addr); 7290 if (ret) 7291 return ret; 7292 if (val) 7293 goto err_size; 7294 } 7295 size = sizeof(*attr); 7296 } 7297 7298 ret = copy_from_user(attr, uattr, size); 7299 if (ret) 7300 return -EFAULT; 7301 7302 if (attr->__reserved_1) 7303 return -EINVAL; 7304 7305 if (attr->sample_type & ~(PERF_SAMPLE_MAX-1)) 7306 return -EINVAL; 7307 7308 if (attr->read_format & ~(PERF_FORMAT_MAX-1)) 7309 return -EINVAL; 7310 7311 if (attr->sample_type & PERF_SAMPLE_BRANCH_STACK) { 7312 u64 mask = attr->branch_sample_type; 7313 7314 /* only using defined bits */ 7315 if (mask & ~(PERF_SAMPLE_BRANCH_MAX-1)) 7316 return -EINVAL; 7317 7318 /* at least one branch bit must be set */ 7319 if (!(mask & ~PERF_SAMPLE_BRANCH_PLM_ALL)) 7320 return -EINVAL; 7321 7322 /* propagate priv level, when not set for branch */ 7323 if (!(mask & PERF_SAMPLE_BRANCH_PLM_ALL)) { 7324 7325 /* exclude_kernel checked on syscall entry */ 7326 if (!attr->exclude_kernel) 7327 mask |= PERF_SAMPLE_BRANCH_KERNEL; 7328 7329 if (!attr->exclude_user) 7330 mask |= PERF_SAMPLE_BRANCH_USER; 7331 7332 if (!attr->exclude_hv) 7333 mask |= PERF_SAMPLE_BRANCH_HV; 7334 /* 7335 * adjust user setting (for HW filter setup) 7336 */ 7337 attr->branch_sample_type = mask; 7338 } 7339 /* privileged levels capture (kernel, hv): check permissions */ 7340 if ((mask & PERF_SAMPLE_BRANCH_PERM_PLM) 7341 && perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN)) 7342 return -EACCES; 7343 } 7344 7345 if (attr->sample_type & PERF_SAMPLE_REGS_USER) { 7346 ret = perf_reg_validate(attr->sample_regs_user); 7347 if (ret) 7348 return ret; 7349 } 7350 7351 if (attr->sample_type & PERF_SAMPLE_STACK_USER) { 7352 if (!arch_perf_have_user_stack_dump()) 7353 return -ENOSYS; 7354 7355 /* 7356 * We have __u32 type for the size, but so far 7357 * we can only use __u16 as maximum due to the 7358 * __u16 sample size limit. 7359 */ 7360 if (attr->sample_stack_user >= USHRT_MAX) 7361 ret = -EINVAL; 7362 else if (!IS_ALIGNED(attr->sample_stack_user, sizeof(u64))) 7363 ret = -EINVAL; 7364 } 7365 7366 if (attr->sample_type & PERF_SAMPLE_REGS_INTR) 7367 ret = perf_reg_validate(attr->sample_regs_intr); 7368 out: 7369 return ret; 7370 7371 err_size: 7372 put_user(sizeof(*attr), &uattr->size); 7373 ret = -E2BIG; 7374 goto out; 7375 } 7376 7377 static int 7378 perf_event_set_output(struct perf_event *event, struct perf_event *output_event) 7379 { 7380 struct ring_buffer *rb = NULL; 7381 int ret = -EINVAL; 7382 7383 if (!output_event) 7384 goto set; 7385 7386 /* don't allow circular references */ 7387 if (event == output_event) 7388 goto out; 7389 7390 /* 7391 * Don't allow cross-cpu buffers 7392 */ 7393 if (output_event->cpu != event->cpu) 7394 goto out; 7395 7396 /* 7397 * If its not a per-cpu rb, it must be the same task. 7398 */ 7399 if (output_event->cpu == -1 && output_event->ctx != event->ctx) 7400 goto out; 7401 7402 set: 7403 mutex_lock(&event->mmap_mutex); 7404 /* Can't redirect output if we've got an active mmap() */ 7405 if (atomic_read(&event->mmap_count)) 7406 goto unlock; 7407 7408 if (output_event) { 7409 /* get the rb we want to redirect to */ 7410 rb = ring_buffer_get(output_event); 7411 if (!rb) 7412 goto unlock; 7413 } 7414 7415 ring_buffer_attach(event, rb); 7416 7417 ret = 0; 7418 unlock: 7419 mutex_unlock(&event->mmap_mutex); 7420 7421 out: 7422 return ret; 7423 } 7424 7425 static void mutex_lock_double(struct mutex *a, struct mutex *b) 7426 { 7427 if (b < a) 7428 swap(a, b); 7429 7430 mutex_lock(a); 7431 mutex_lock_nested(b, SINGLE_DEPTH_NESTING); 7432 } 7433 7434 /** 7435 * sys_perf_event_open - open a performance event, associate it to a task/cpu 7436 * 7437 * @attr_uptr: event_id type attributes for monitoring/sampling 7438 * @pid: target pid 7439 * @cpu: target cpu 7440 * @group_fd: group leader event fd 7441 */ 7442 SYSCALL_DEFINE5(perf_event_open, 7443 struct perf_event_attr __user *, attr_uptr, 7444 pid_t, pid, int, cpu, int, group_fd, unsigned long, flags) 7445 { 7446 struct perf_event *group_leader = NULL, *output_event = NULL; 7447 struct perf_event *event, *sibling; 7448 struct perf_event_attr attr; 7449 struct perf_event_context *ctx, *uninitialized_var(gctx); 7450 struct file *event_file = NULL; 7451 struct fd group = {NULL, 0}; 7452 struct task_struct *task = NULL; 7453 struct pmu *pmu; 7454 int event_fd; 7455 int move_group = 0; 7456 int err; 7457 int f_flags = O_RDWR; 7458 7459 /* for future expandability... */ 7460 if (flags & ~PERF_FLAG_ALL) 7461 return -EINVAL; 7462 7463 err = perf_copy_attr(attr_uptr, &attr); 7464 if (err) 7465 return err; 7466 7467 if (!attr.exclude_kernel) { 7468 if (perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN)) 7469 return -EACCES; 7470 } 7471 7472 if (attr.freq) { 7473 if (attr.sample_freq > sysctl_perf_event_sample_rate) 7474 return -EINVAL; 7475 } else { 7476 if (attr.sample_period & (1ULL << 63)) 7477 return -EINVAL; 7478 } 7479 7480 /* 7481 * In cgroup mode, the pid argument is used to pass the fd 7482 * opened to the cgroup directory in cgroupfs. The cpu argument 7483 * designates the cpu on which to monitor threads from that 7484 * cgroup. 7485 */ 7486 if ((flags & PERF_FLAG_PID_CGROUP) && (pid == -1 || cpu == -1)) 7487 return -EINVAL; 7488 7489 if (flags & PERF_FLAG_FD_CLOEXEC) 7490 f_flags |= O_CLOEXEC; 7491 7492 event_fd = get_unused_fd_flags(f_flags); 7493 if (event_fd < 0) 7494 return event_fd; 7495 7496 if (group_fd != -1) { 7497 err = perf_fget_light(group_fd, &group); 7498 if (err) 7499 goto err_fd; 7500 group_leader = group.file->private_data; 7501 if (flags & PERF_FLAG_FD_OUTPUT) 7502 output_event = group_leader; 7503 if (flags & PERF_FLAG_FD_NO_GROUP) 7504 group_leader = NULL; 7505 } 7506 7507 if (pid != -1 && !(flags & PERF_FLAG_PID_CGROUP)) { 7508 task = find_lively_task_by_vpid(pid); 7509 if (IS_ERR(task)) { 7510 err = PTR_ERR(task); 7511 goto err_group_fd; 7512 } 7513 } 7514 7515 if (task && group_leader && 7516 group_leader->attr.inherit != attr.inherit) { 7517 err = -EINVAL; 7518 goto err_task; 7519 } 7520 7521 get_online_cpus(); 7522 7523 event = perf_event_alloc(&attr, cpu, task, group_leader, NULL, 7524 NULL, NULL); 7525 if (IS_ERR(event)) { 7526 err = PTR_ERR(event); 7527 goto err_cpus; 7528 } 7529 7530 if (flags & PERF_FLAG_PID_CGROUP) { 7531 err = perf_cgroup_connect(pid, event, &attr, group_leader); 7532 if (err) { 7533 __free_event(event); 7534 goto err_cpus; 7535 } 7536 } 7537 7538 if (is_sampling_event(event)) { 7539 if (event->pmu->capabilities & PERF_PMU_CAP_NO_INTERRUPT) { 7540 err = -ENOTSUPP; 7541 goto err_alloc; 7542 } 7543 } 7544 7545 account_event(event); 7546 7547 /* 7548 * Special case software events and allow them to be part of 7549 * any hardware group. 7550 */ 7551 pmu = event->pmu; 7552 7553 if (group_leader && 7554 (is_software_event(event) != is_software_event(group_leader))) { 7555 if (is_software_event(event)) { 7556 /* 7557 * If event and group_leader are not both a software 7558 * event, and event is, then group leader is not. 7559 * 7560 * Allow the addition of software events to !software 7561 * groups, this is safe because software events never 7562 * fail to schedule. 7563 */ 7564 pmu = group_leader->pmu; 7565 } else if (is_software_event(group_leader) && 7566 (group_leader->group_flags & PERF_GROUP_SOFTWARE)) { 7567 /* 7568 * In case the group is a pure software group, and we 7569 * try to add a hardware event, move the whole group to 7570 * the hardware context. 7571 */ 7572 move_group = 1; 7573 } 7574 } 7575 7576 /* 7577 * Get the target context (task or percpu): 7578 */ 7579 ctx = find_get_context(pmu, task, event->cpu); 7580 if (IS_ERR(ctx)) { 7581 err = PTR_ERR(ctx); 7582 goto err_alloc; 7583 } 7584 7585 if (task) { 7586 put_task_struct(task); 7587 task = NULL; 7588 } 7589 7590 /* 7591 * Look up the group leader (we will attach this event to it): 7592 */ 7593 if (group_leader) { 7594 err = -EINVAL; 7595 7596 /* 7597 * Do not allow a recursive hierarchy (this new sibling 7598 * becoming part of another group-sibling): 7599 */ 7600 if (group_leader->group_leader != group_leader) 7601 goto err_context; 7602 /* 7603 * Do not allow to attach to a group in a different 7604 * task or CPU context: 7605 */ 7606 if (move_group) { 7607 /* 7608 * Make sure we're both on the same task, or both 7609 * per-cpu events. 7610 */ 7611 if (group_leader->ctx->task != ctx->task) 7612 goto err_context; 7613 7614 /* 7615 * Make sure we're both events for the same CPU; 7616 * grouping events for different CPUs is broken; since 7617 * you can never concurrently schedule them anyhow. 7618 */ 7619 if (group_leader->cpu != event->cpu) 7620 goto err_context; 7621 } else { 7622 if (group_leader->ctx != ctx) 7623 goto err_context; 7624 } 7625 7626 /* 7627 * Only a group leader can be exclusive or pinned 7628 */ 7629 if (attr.exclusive || attr.pinned) 7630 goto err_context; 7631 } 7632 7633 if (output_event) { 7634 err = perf_event_set_output(event, output_event); 7635 if (err) 7636 goto err_context; 7637 } 7638 7639 event_file = anon_inode_getfile("[perf_event]", &perf_fops, event, 7640 f_flags); 7641 if (IS_ERR(event_file)) { 7642 err = PTR_ERR(event_file); 7643 goto err_context; 7644 } 7645 7646 if (move_group) { 7647 gctx = group_leader->ctx; 7648 7649 /* 7650 * See perf_event_ctx_lock() for comments on the details 7651 * of swizzling perf_event::ctx. 7652 */ 7653 mutex_lock_double(&gctx->mutex, &ctx->mutex); 7654 7655 perf_remove_from_context(group_leader, false); 7656 7657 list_for_each_entry(sibling, &group_leader->sibling_list, 7658 group_entry) { 7659 perf_remove_from_context(sibling, false); 7660 put_ctx(gctx); 7661 } 7662 } else { 7663 mutex_lock(&ctx->mutex); 7664 } 7665 7666 WARN_ON_ONCE(ctx->parent_ctx); 7667 7668 if (move_group) { 7669 /* 7670 * Wait for everybody to stop referencing the events through 7671 * the old lists, before installing it on new lists. 7672 */ 7673 synchronize_rcu(); 7674 7675 /* 7676 * Install the group siblings before the group leader. 7677 * 7678 * Because a group leader will try and install the entire group 7679 * (through the sibling list, which is still in-tact), we can 7680 * end up with siblings installed in the wrong context. 7681 * 7682 * By installing siblings first we NO-OP because they're not 7683 * reachable through the group lists. 7684 */ 7685 list_for_each_entry(sibling, &group_leader->sibling_list, 7686 group_entry) { 7687 perf_event__state_init(sibling); 7688 perf_install_in_context(ctx, sibling, sibling->cpu); 7689 get_ctx(ctx); 7690 } 7691 7692 /* 7693 * Removing from the context ends up with disabled 7694 * event. What we want here is event in the initial 7695 * startup state, ready to be add into new context. 7696 */ 7697 perf_event__state_init(group_leader); 7698 perf_install_in_context(ctx, group_leader, group_leader->cpu); 7699 get_ctx(ctx); 7700 } 7701 7702 perf_install_in_context(ctx, event, event->cpu); 7703 perf_unpin_context(ctx); 7704 7705 if (move_group) { 7706 mutex_unlock(&gctx->mutex); 7707 put_ctx(gctx); 7708 } 7709 mutex_unlock(&ctx->mutex); 7710 7711 put_online_cpus(); 7712 7713 event->owner = current; 7714 7715 mutex_lock(¤t->perf_event_mutex); 7716 list_add_tail(&event->owner_entry, ¤t->perf_event_list); 7717 mutex_unlock(¤t->perf_event_mutex); 7718 7719 /* 7720 * Precalculate sample_data sizes 7721 */ 7722 perf_event__header_size(event); 7723 perf_event__id_header_size(event); 7724 7725 /* 7726 * Drop the reference on the group_event after placing the 7727 * new event on the sibling_list. This ensures destruction 7728 * of the group leader will find the pointer to itself in 7729 * perf_group_detach(). 7730 */ 7731 fdput(group); 7732 fd_install(event_fd, event_file); 7733 return event_fd; 7734 7735 err_context: 7736 perf_unpin_context(ctx); 7737 put_ctx(ctx); 7738 err_alloc: 7739 free_event(event); 7740 err_cpus: 7741 put_online_cpus(); 7742 err_task: 7743 if (task) 7744 put_task_struct(task); 7745 err_group_fd: 7746 fdput(group); 7747 err_fd: 7748 put_unused_fd(event_fd); 7749 return err; 7750 } 7751 7752 /** 7753 * perf_event_create_kernel_counter 7754 * 7755 * @attr: attributes of the counter to create 7756 * @cpu: cpu in which the counter is bound 7757 * @task: task to profile (NULL for percpu) 7758 */ 7759 struct perf_event * 7760 perf_event_create_kernel_counter(struct perf_event_attr *attr, int cpu, 7761 struct task_struct *task, 7762 perf_overflow_handler_t overflow_handler, 7763 void *context) 7764 { 7765 struct perf_event_context *ctx; 7766 struct perf_event *event; 7767 int err; 7768 7769 /* 7770 * Get the target context (task or percpu): 7771 */ 7772 7773 event = perf_event_alloc(attr, cpu, task, NULL, NULL, 7774 overflow_handler, context); 7775 if (IS_ERR(event)) { 7776 err = PTR_ERR(event); 7777 goto err; 7778 } 7779 7780 /* Mark owner so we could distinguish it from user events. */ 7781 event->owner = EVENT_OWNER_KERNEL; 7782 7783 account_event(event); 7784 7785 ctx = find_get_context(event->pmu, task, cpu); 7786 if (IS_ERR(ctx)) { 7787 err = PTR_ERR(ctx); 7788 goto err_free; 7789 } 7790 7791 WARN_ON_ONCE(ctx->parent_ctx); 7792 mutex_lock(&ctx->mutex); 7793 perf_install_in_context(ctx, event, cpu); 7794 perf_unpin_context(ctx); 7795 mutex_unlock(&ctx->mutex); 7796 7797 return event; 7798 7799 err_free: 7800 free_event(event); 7801 err: 7802 return ERR_PTR(err); 7803 } 7804 EXPORT_SYMBOL_GPL(perf_event_create_kernel_counter); 7805 7806 void perf_pmu_migrate_context(struct pmu *pmu, int src_cpu, int dst_cpu) 7807 { 7808 struct perf_event_context *src_ctx; 7809 struct perf_event_context *dst_ctx; 7810 struct perf_event *event, *tmp; 7811 LIST_HEAD(events); 7812 7813 src_ctx = &per_cpu_ptr(pmu->pmu_cpu_context, src_cpu)->ctx; 7814 dst_ctx = &per_cpu_ptr(pmu->pmu_cpu_context, dst_cpu)->ctx; 7815 7816 /* 7817 * See perf_event_ctx_lock() for comments on the details 7818 * of swizzling perf_event::ctx. 7819 */ 7820 mutex_lock_double(&src_ctx->mutex, &dst_ctx->mutex); 7821 list_for_each_entry_safe(event, tmp, &src_ctx->event_list, 7822 event_entry) { 7823 perf_remove_from_context(event, false); 7824 unaccount_event_cpu(event, src_cpu); 7825 put_ctx(src_ctx); 7826 list_add(&event->migrate_entry, &events); 7827 } 7828 7829 /* 7830 * Wait for the events to quiesce before re-instating them. 7831 */ 7832 synchronize_rcu(); 7833 7834 /* 7835 * Re-instate events in 2 passes. 7836 * 7837 * Skip over group leaders and only install siblings on this first 7838 * pass, siblings will not get enabled without a leader, however a 7839 * leader will enable its siblings, even if those are still on the old 7840 * context. 7841 */ 7842 list_for_each_entry_safe(event, tmp, &events, migrate_entry) { 7843 if (event->group_leader == event) 7844 continue; 7845 7846 list_del(&event->migrate_entry); 7847 if (event->state >= PERF_EVENT_STATE_OFF) 7848 event->state = PERF_EVENT_STATE_INACTIVE; 7849 account_event_cpu(event, dst_cpu); 7850 perf_install_in_context(dst_ctx, event, dst_cpu); 7851 get_ctx(dst_ctx); 7852 } 7853 7854 /* 7855 * Once all the siblings are setup properly, install the group leaders 7856 * to make it go. 7857 */ 7858 list_for_each_entry_safe(event, tmp, &events, migrate_entry) { 7859 list_del(&event->migrate_entry); 7860 if (event->state >= PERF_EVENT_STATE_OFF) 7861 event->state = PERF_EVENT_STATE_INACTIVE; 7862 account_event_cpu(event, dst_cpu); 7863 perf_install_in_context(dst_ctx, event, dst_cpu); 7864 get_ctx(dst_ctx); 7865 } 7866 mutex_unlock(&dst_ctx->mutex); 7867 mutex_unlock(&src_ctx->mutex); 7868 } 7869 EXPORT_SYMBOL_GPL(perf_pmu_migrate_context); 7870 7871 static void sync_child_event(struct perf_event *child_event, 7872 struct task_struct *child) 7873 { 7874 struct perf_event *parent_event = child_event->parent; 7875 u64 child_val; 7876 7877 if (child_event->attr.inherit_stat) 7878 perf_event_read_event(child_event, child); 7879 7880 child_val = perf_event_count(child_event); 7881 7882 /* 7883 * Add back the child's count to the parent's count: 7884 */ 7885 atomic64_add(child_val, &parent_event->child_count); 7886 atomic64_add(child_event->total_time_enabled, 7887 &parent_event->child_total_time_enabled); 7888 atomic64_add(child_event->total_time_running, 7889 &parent_event->child_total_time_running); 7890 7891 /* 7892 * Remove this event from the parent's list 7893 */ 7894 WARN_ON_ONCE(parent_event->ctx->parent_ctx); 7895 mutex_lock(&parent_event->child_mutex); 7896 list_del_init(&child_event->child_list); 7897 mutex_unlock(&parent_event->child_mutex); 7898 7899 /* 7900 * Make sure user/parent get notified, that we just 7901 * lost one event. 7902 */ 7903 perf_event_wakeup(parent_event); 7904 7905 /* 7906 * Release the parent event, if this was the last 7907 * reference to it. 7908 */ 7909 put_event(parent_event); 7910 } 7911 7912 static void 7913 __perf_event_exit_task(struct perf_event *child_event, 7914 struct perf_event_context *child_ctx, 7915 struct task_struct *child) 7916 { 7917 /* 7918 * Do not destroy the 'original' grouping; because of the context 7919 * switch optimization the original events could've ended up in a 7920 * random child task. 7921 * 7922 * If we were to destroy the original group, all group related 7923 * operations would cease to function properly after this random 7924 * child dies. 7925 * 7926 * Do destroy all inherited groups, we don't care about those 7927 * and being thorough is better. 7928 */ 7929 perf_remove_from_context(child_event, !!child_event->parent); 7930 7931 /* 7932 * It can happen that the parent exits first, and has events 7933 * that are still around due to the child reference. These 7934 * events need to be zapped. 7935 */ 7936 if (child_event->parent) { 7937 sync_child_event(child_event, child); 7938 free_event(child_event); 7939 } else { 7940 child_event->state = PERF_EVENT_STATE_EXIT; 7941 perf_event_wakeup(child_event); 7942 } 7943 } 7944 7945 static void perf_event_exit_task_context(struct task_struct *child, int ctxn) 7946 { 7947 struct perf_event *child_event, *next; 7948 struct perf_event_context *child_ctx, *clone_ctx = NULL; 7949 unsigned long flags; 7950 7951 if (likely(!child->perf_event_ctxp[ctxn])) { 7952 perf_event_task(child, NULL, 0); 7953 return; 7954 } 7955 7956 local_irq_save(flags); 7957 /* 7958 * We can't reschedule here because interrupts are disabled, 7959 * and either child is current or it is a task that can't be 7960 * scheduled, so we are now safe from rescheduling changing 7961 * our context. 7962 */ 7963 child_ctx = rcu_dereference_raw(child->perf_event_ctxp[ctxn]); 7964 7965 /* 7966 * Take the context lock here so that if find_get_context is 7967 * reading child->perf_event_ctxp, we wait until it has 7968 * incremented the context's refcount before we do put_ctx below. 7969 */ 7970 raw_spin_lock(&child_ctx->lock); 7971 task_ctx_sched_out(child_ctx); 7972 child->perf_event_ctxp[ctxn] = NULL; 7973 7974 /* 7975 * If this context is a clone; unclone it so it can't get 7976 * swapped to another process while we're removing all 7977 * the events from it. 7978 */ 7979 clone_ctx = unclone_ctx(child_ctx); 7980 update_context_time(child_ctx); 7981 raw_spin_unlock_irqrestore(&child_ctx->lock, flags); 7982 7983 if (clone_ctx) 7984 put_ctx(clone_ctx); 7985 7986 /* 7987 * Report the task dead after unscheduling the events so that we 7988 * won't get any samples after PERF_RECORD_EXIT. We can however still 7989 * get a few PERF_RECORD_READ events. 7990 */ 7991 perf_event_task(child, child_ctx, 0); 7992 7993 /* 7994 * We can recurse on the same lock type through: 7995 * 7996 * __perf_event_exit_task() 7997 * sync_child_event() 7998 * put_event() 7999 * mutex_lock(&ctx->mutex) 8000 * 8001 * But since its the parent context it won't be the same instance. 8002 */ 8003 mutex_lock(&child_ctx->mutex); 8004 8005 list_for_each_entry_safe(child_event, next, &child_ctx->event_list, event_entry) 8006 __perf_event_exit_task(child_event, child_ctx, child); 8007 8008 mutex_unlock(&child_ctx->mutex); 8009 8010 put_ctx(child_ctx); 8011 } 8012 8013 /* 8014 * When a child task exits, feed back event values to parent events. 8015 */ 8016 void perf_event_exit_task(struct task_struct *child) 8017 { 8018 struct perf_event *event, *tmp; 8019 int ctxn; 8020 8021 mutex_lock(&child->perf_event_mutex); 8022 list_for_each_entry_safe(event, tmp, &child->perf_event_list, 8023 owner_entry) { 8024 list_del_init(&event->owner_entry); 8025 8026 /* 8027 * Ensure the list deletion is visible before we clear 8028 * the owner, closes a race against perf_release() where 8029 * we need to serialize on the owner->perf_event_mutex. 8030 */ 8031 smp_wmb(); 8032 event->owner = NULL; 8033 } 8034 mutex_unlock(&child->perf_event_mutex); 8035 8036 for_each_task_context_nr(ctxn) 8037 perf_event_exit_task_context(child, ctxn); 8038 } 8039 8040 static void perf_free_event(struct perf_event *event, 8041 struct perf_event_context *ctx) 8042 { 8043 struct perf_event *parent = event->parent; 8044 8045 if (WARN_ON_ONCE(!parent)) 8046 return; 8047 8048 mutex_lock(&parent->child_mutex); 8049 list_del_init(&event->child_list); 8050 mutex_unlock(&parent->child_mutex); 8051 8052 put_event(parent); 8053 8054 raw_spin_lock_irq(&ctx->lock); 8055 perf_group_detach(event); 8056 list_del_event(event, ctx); 8057 raw_spin_unlock_irq(&ctx->lock); 8058 free_event(event); 8059 } 8060 8061 /* 8062 * Free an unexposed, unused context as created by inheritance by 8063 * perf_event_init_task below, used by fork() in case of fail. 8064 * 8065 * Not all locks are strictly required, but take them anyway to be nice and 8066 * help out with the lockdep assertions. 8067 */ 8068 void perf_event_free_task(struct task_struct *task) 8069 { 8070 struct perf_event_context *ctx; 8071 struct perf_event *event, *tmp; 8072 int ctxn; 8073 8074 for_each_task_context_nr(ctxn) { 8075 ctx = task->perf_event_ctxp[ctxn]; 8076 if (!ctx) 8077 continue; 8078 8079 mutex_lock(&ctx->mutex); 8080 again: 8081 list_for_each_entry_safe(event, tmp, &ctx->pinned_groups, 8082 group_entry) 8083 perf_free_event(event, ctx); 8084 8085 list_for_each_entry_safe(event, tmp, &ctx->flexible_groups, 8086 group_entry) 8087 perf_free_event(event, ctx); 8088 8089 if (!list_empty(&ctx->pinned_groups) || 8090 !list_empty(&ctx->flexible_groups)) 8091 goto again; 8092 8093 mutex_unlock(&ctx->mutex); 8094 8095 put_ctx(ctx); 8096 } 8097 } 8098 8099 void perf_event_delayed_put(struct task_struct *task) 8100 { 8101 int ctxn; 8102 8103 for_each_task_context_nr(ctxn) 8104 WARN_ON_ONCE(task->perf_event_ctxp[ctxn]); 8105 } 8106 8107 /* 8108 * inherit a event from parent task to child task: 8109 */ 8110 static struct perf_event * 8111 inherit_event(struct perf_event *parent_event, 8112 struct task_struct *parent, 8113 struct perf_event_context *parent_ctx, 8114 struct task_struct *child, 8115 struct perf_event *group_leader, 8116 struct perf_event_context *child_ctx) 8117 { 8118 enum perf_event_active_state parent_state = parent_event->state; 8119 struct perf_event *child_event; 8120 unsigned long flags; 8121 8122 /* 8123 * Instead of creating recursive hierarchies of events, 8124 * we link inherited events back to the original parent, 8125 * which has a filp for sure, which we use as the reference 8126 * count: 8127 */ 8128 if (parent_event->parent) 8129 parent_event = parent_event->parent; 8130 8131 child_event = perf_event_alloc(&parent_event->attr, 8132 parent_event->cpu, 8133 child, 8134 group_leader, parent_event, 8135 NULL, NULL); 8136 if (IS_ERR(child_event)) 8137 return child_event; 8138 8139 if (is_orphaned_event(parent_event) || 8140 !atomic_long_inc_not_zero(&parent_event->refcount)) { 8141 free_event(child_event); 8142 return NULL; 8143 } 8144 8145 get_ctx(child_ctx); 8146 8147 /* 8148 * Make the child state follow the state of the parent event, 8149 * not its attr.disabled bit. We hold the parent's mutex, 8150 * so we won't race with perf_event_{en, dis}able_family. 8151 */ 8152 if (parent_state >= PERF_EVENT_STATE_INACTIVE) 8153 child_event->state = PERF_EVENT_STATE_INACTIVE; 8154 else 8155 child_event->state = PERF_EVENT_STATE_OFF; 8156 8157 if (parent_event->attr.freq) { 8158 u64 sample_period = parent_event->hw.sample_period; 8159 struct hw_perf_event *hwc = &child_event->hw; 8160 8161 hwc->sample_period = sample_period; 8162 hwc->last_period = sample_period; 8163 8164 local64_set(&hwc->period_left, sample_period); 8165 } 8166 8167 child_event->ctx = child_ctx; 8168 child_event->overflow_handler = parent_event->overflow_handler; 8169 child_event->overflow_handler_context 8170 = parent_event->overflow_handler_context; 8171 8172 /* 8173 * Precalculate sample_data sizes 8174 */ 8175 perf_event__header_size(child_event); 8176 perf_event__id_header_size(child_event); 8177 8178 /* 8179 * Link it up in the child's context: 8180 */ 8181 raw_spin_lock_irqsave(&child_ctx->lock, flags); 8182 add_event_to_ctx(child_event, child_ctx); 8183 raw_spin_unlock_irqrestore(&child_ctx->lock, flags); 8184 8185 /* 8186 * Link this into the parent event's child list 8187 */ 8188 WARN_ON_ONCE(parent_event->ctx->parent_ctx); 8189 mutex_lock(&parent_event->child_mutex); 8190 list_add_tail(&child_event->child_list, &parent_event->child_list); 8191 mutex_unlock(&parent_event->child_mutex); 8192 8193 return child_event; 8194 } 8195 8196 static int inherit_group(struct perf_event *parent_event, 8197 struct task_struct *parent, 8198 struct perf_event_context *parent_ctx, 8199 struct task_struct *child, 8200 struct perf_event_context *child_ctx) 8201 { 8202 struct perf_event *leader; 8203 struct perf_event *sub; 8204 struct perf_event *child_ctr; 8205 8206 leader = inherit_event(parent_event, parent, parent_ctx, 8207 child, NULL, child_ctx); 8208 if (IS_ERR(leader)) 8209 return PTR_ERR(leader); 8210 list_for_each_entry(sub, &parent_event->sibling_list, group_entry) { 8211 child_ctr = inherit_event(sub, parent, parent_ctx, 8212 child, leader, child_ctx); 8213 if (IS_ERR(child_ctr)) 8214 return PTR_ERR(child_ctr); 8215 } 8216 return 0; 8217 } 8218 8219 static int 8220 inherit_task_group(struct perf_event *event, struct task_struct *parent, 8221 struct perf_event_context *parent_ctx, 8222 struct task_struct *child, int ctxn, 8223 int *inherited_all) 8224 { 8225 int ret; 8226 struct perf_event_context *child_ctx; 8227 8228 if (!event->attr.inherit) { 8229 *inherited_all = 0; 8230 return 0; 8231 } 8232 8233 child_ctx = child->perf_event_ctxp[ctxn]; 8234 if (!child_ctx) { 8235 /* 8236 * This is executed from the parent task context, so 8237 * inherit events that have been marked for cloning. 8238 * First allocate and initialize a context for the 8239 * child. 8240 */ 8241 8242 child_ctx = alloc_perf_context(parent_ctx->pmu, child); 8243 if (!child_ctx) 8244 return -ENOMEM; 8245 8246 child->perf_event_ctxp[ctxn] = child_ctx; 8247 } 8248 8249 ret = inherit_group(event, parent, parent_ctx, 8250 child, child_ctx); 8251 8252 if (ret) 8253 *inherited_all = 0; 8254 8255 return ret; 8256 } 8257 8258 /* 8259 * Initialize the perf_event context in task_struct 8260 */ 8261 static int perf_event_init_context(struct task_struct *child, int ctxn) 8262 { 8263 struct perf_event_context *child_ctx, *parent_ctx; 8264 struct perf_event_context *cloned_ctx; 8265 struct perf_event *event; 8266 struct task_struct *parent = current; 8267 int inherited_all = 1; 8268 unsigned long flags; 8269 int ret = 0; 8270 8271 if (likely(!parent->perf_event_ctxp[ctxn])) 8272 return 0; 8273 8274 /* 8275 * If the parent's context is a clone, pin it so it won't get 8276 * swapped under us. 8277 */ 8278 parent_ctx = perf_pin_task_context(parent, ctxn); 8279 if (!parent_ctx) 8280 return 0; 8281 8282 /* 8283 * No need to check if parent_ctx != NULL here; since we saw 8284 * it non-NULL earlier, the only reason for it to become NULL 8285 * is if we exit, and since we're currently in the middle of 8286 * a fork we can't be exiting at the same time. 8287 */ 8288 8289 /* 8290 * Lock the parent list. No need to lock the child - not PID 8291 * hashed yet and not running, so nobody can access it. 8292 */ 8293 mutex_lock(&parent_ctx->mutex); 8294 8295 /* 8296 * We dont have to disable NMIs - we are only looking at 8297 * the list, not manipulating it: 8298 */ 8299 list_for_each_entry(event, &parent_ctx->pinned_groups, group_entry) { 8300 ret = inherit_task_group(event, parent, parent_ctx, 8301 child, ctxn, &inherited_all); 8302 if (ret) 8303 break; 8304 } 8305 8306 /* 8307 * We can't hold ctx->lock when iterating the ->flexible_group list due 8308 * to allocations, but we need to prevent rotation because 8309 * rotate_ctx() will change the list from interrupt context. 8310 */ 8311 raw_spin_lock_irqsave(&parent_ctx->lock, flags); 8312 parent_ctx->rotate_disable = 1; 8313 raw_spin_unlock_irqrestore(&parent_ctx->lock, flags); 8314 8315 list_for_each_entry(event, &parent_ctx->flexible_groups, group_entry) { 8316 ret = inherit_task_group(event, parent, parent_ctx, 8317 child, ctxn, &inherited_all); 8318 if (ret) 8319 break; 8320 } 8321 8322 raw_spin_lock_irqsave(&parent_ctx->lock, flags); 8323 parent_ctx->rotate_disable = 0; 8324 8325 child_ctx = child->perf_event_ctxp[ctxn]; 8326 8327 if (child_ctx && inherited_all) { 8328 /* 8329 * Mark the child context as a clone of the parent 8330 * context, or of whatever the parent is a clone of. 8331 * 8332 * Note that if the parent is a clone, the holding of 8333 * parent_ctx->lock avoids it from being uncloned. 8334 */ 8335 cloned_ctx = parent_ctx->parent_ctx; 8336 if (cloned_ctx) { 8337 child_ctx->parent_ctx = cloned_ctx; 8338 child_ctx->parent_gen = parent_ctx->parent_gen; 8339 } else { 8340 child_ctx->parent_ctx = parent_ctx; 8341 child_ctx->parent_gen = parent_ctx->generation; 8342 } 8343 get_ctx(child_ctx->parent_ctx); 8344 } 8345 8346 raw_spin_unlock_irqrestore(&parent_ctx->lock, flags); 8347 mutex_unlock(&parent_ctx->mutex); 8348 8349 perf_unpin_context(parent_ctx); 8350 put_ctx(parent_ctx); 8351 8352 return ret; 8353 } 8354 8355 /* 8356 * Initialize the perf_event context in task_struct 8357 */ 8358 int perf_event_init_task(struct task_struct *child) 8359 { 8360 int ctxn, ret; 8361 8362 memset(child->perf_event_ctxp, 0, sizeof(child->perf_event_ctxp)); 8363 mutex_init(&child->perf_event_mutex); 8364 INIT_LIST_HEAD(&child->perf_event_list); 8365 8366 for_each_task_context_nr(ctxn) { 8367 ret = perf_event_init_context(child, ctxn); 8368 if (ret) { 8369 perf_event_free_task(child); 8370 return ret; 8371 } 8372 } 8373 8374 return 0; 8375 } 8376 8377 static void __init perf_event_init_all_cpus(void) 8378 { 8379 struct swevent_htable *swhash; 8380 int cpu; 8381 8382 for_each_possible_cpu(cpu) { 8383 swhash = &per_cpu(swevent_htable, cpu); 8384 mutex_init(&swhash->hlist_mutex); 8385 INIT_LIST_HEAD(&per_cpu(active_ctx_list, cpu)); 8386 } 8387 } 8388 8389 static void perf_event_init_cpu(int cpu) 8390 { 8391 struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu); 8392 8393 mutex_lock(&swhash->hlist_mutex); 8394 swhash->online = true; 8395 if (swhash->hlist_refcount > 0) { 8396 struct swevent_hlist *hlist; 8397 8398 hlist = kzalloc_node(sizeof(*hlist), GFP_KERNEL, cpu_to_node(cpu)); 8399 WARN_ON(!hlist); 8400 rcu_assign_pointer(swhash->swevent_hlist, hlist); 8401 } 8402 mutex_unlock(&swhash->hlist_mutex); 8403 } 8404 8405 #if defined CONFIG_HOTPLUG_CPU || defined CONFIG_KEXEC 8406 static void __perf_event_exit_context(void *__info) 8407 { 8408 struct remove_event re = { .detach_group = true }; 8409 struct perf_event_context *ctx = __info; 8410 8411 rcu_read_lock(); 8412 list_for_each_entry_rcu(re.event, &ctx->event_list, event_entry) 8413 __perf_remove_from_context(&re); 8414 rcu_read_unlock(); 8415 } 8416 8417 static void perf_event_exit_cpu_context(int cpu) 8418 { 8419 struct perf_event_context *ctx; 8420 struct pmu *pmu; 8421 int idx; 8422 8423 idx = srcu_read_lock(&pmus_srcu); 8424 list_for_each_entry_rcu(pmu, &pmus, entry) { 8425 ctx = &per_cpu_ptr(pmu->pmu_cpu_context, cpu)->ctx; 8426 8427 mutex_lock(&ctx->mutex); 8428 smp_call_function_single(cpu, __perf_event_exit_context, ctx, 1); 8429 mutex_unlock(&ctx->mutex); 8430 } 8431 srcu_read_unlock(&pmus_srcu, idx); 8432 } 8433 8434 static void perf_event_exit_cpu(int cpu) 8435 { 8436 struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu); 8437 8438 perf_event_exit_cpu_context(cpu); 8439 8440 mutex_lock(&swhash->hlist_mutex); 8441 swhash->online = false; 8442 swevent_hlist_release(swhash); 8443 mutex_unlock(&swhash->hlist_mutex); 8444 } 8445 #else 8446 static inline void perf_event_exit_cpu(int cpu) { } 8447 #endif 8448 8449 static int 8450 perf_reboot(struct notifier_block *notifier, unsigned long val, void *v) 8451 { 8452 int cpu; 8453 8454 for_each_online_cpu(cpu) 8455 perf_event_exit_cpu(cpu); 8456 8457 return NOTIFY_OK; 8458 } 8459 8460 /* 8461 * Run the perf reboot notifier at the very last possible moment so that 8462 * the generic watchdog code runs as long as possible. 8463 */ 8464 static struct notifier_block perf_reboot_notifier = { 8465 .notifier_call = perf_reboot, 8466 .priority = INT_MIN, 8467 }; 8468 8469 static int 8470 perf_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu) 8471 { 8472 unsigned int cpu = (long)hcpu; 8473 8474 switch (action & ~CPU_TASKS_FROZEN) { 8475 8476 case CPU_UP_PREPARE: 8477 case CPU_DOWN_FAILED: 8478 perf_event_init_cpu(cpu); 8479 break; 8480 8481 case CPU_UP_CANCELED: 8482 case CPU_DOWN_PREPARE: 8483 perf_event_exit_cpu(cpu); 8484 break; 8485 default: 8486 break; 8487 } 8488 8489 return NOTIFY_OK; 8490 } 8491 8492 void __init perf_event_init(void) 8493 { 8494 int ret; 8495 8496 idr_init(&pmu_idr); 8497 8498 perf_event_init_all_cpus(); 8499 init_srcu_struct(&pmus_srcu); 8500 perf_pmu_register(&perf_swevent, "software", PERF_TYPE_SOFTWARE); 8501 perf_pmu_register(&perf_cpu_clock, NULL, -1); 8502 perf_pmu_register(&perf_task_clock, NULL, -1); 8503 perf_tp_register(); 8504 perf_cpu_notifier(perf_cpu_notify); 8505 register_reboot_notifier(&perf_reboot_notifier); 8506 8507 ret = init_hw_breakpoint(); 8508 WARN(ret, "hw_breakpoint initialization failed with: %d", ret); 8509 8510 /* do not patch jump label more than once per second */ 8511 jump_label_rate_limit(&perf_sched_events, HZ); 8512 8513 /* 8514 * Build time assertion that we keep the data_head at the intended 8515 * location. IOW, validation we got the __reserved[] size right. 8516 */ 8517 BUILD_BUG_ON((offsetof(struct perf_event_mmap_page, data_head)) 8518 != 1024); 8519 } 8520 8521 ssize_t perf_event_sysfs_show(struct device *dev, struct device_attribute *attr, 8522 char *page) 8523 { 8524 struct perf_pmu_events_attr *pmu_attr = 8525 container_of(attr, struct perf_pmu_events_attr, attr); 8526 8527 if (pmu_attr->event_str) 8528 return sprintf(page, "%s\n", pmu_attr->event_str); 8529 8530 return 0; 8531 } 8532 8533 static int __init perf_event_sysfs_init(void) 8534 { 8535 struct pmu *pmu; 8536 int ret; 8537 8538 mutex_lock(&pmus_lock); 8539 8540 ret = bus_register(&pmu_bus); 8541 if (ret) 8542 goto unlock; 8543 8544 list_for_each_entry(pmu, &pmus, entry) { 8545 if (!pmu->name || pmu->type < 0) 8546 continue; 8547 8548 ret = pmu_dev_alloc(pmu); 8549 WARN(ret, "Failed to register pmu: %s, reason %d\n", pmu->name, ret); 8550 } 8551 pmu_bus_running = 1; 8552 ret = 0; 8553 8554 unlock: 8555 mutex_unlock(&pmus_lock); 8556 8557 return ret; 8558 } 8559 device_initcall(perf_event_sysfs_init); 8560 8561 #ifdef CONFIG_CGROUP_PERF 8562 static struct cgroup_subsys_state * 8563 perf_cgroup_css_alloc(struct cgroup_subsys_state *parent_css) 8564 { 8565 struct perf_cgroup *jc; 8566 8567 jc = kzalloc(sizeof(*jc), GFP_KERNEL); 8568 if (!jc) 8569 return ERR_PTR(-ENOMEM); 8570 8571 jc->info = alloc_percpu(struct perf_cgroup_info); 8572 if (!jc->info) { 8573 kfree(jc); 8574 return ERR_PTR(-ENOMEM); 8575 } 8576 8577 return &jc->css; 8578 } 8579 8580 static void perf_cgroup_css_free(struct cgroup_subsys_state *css) 8581 { 8582 struct perf_cgroup *jc = container_of(css, struct perf_cgroup, css); 8583 8584 free_percpu(jc->info); 8585 kfree(jc); 8586 } 8587 8588 static int __perf_cgroup_move(void *info) 8589 { 8590 struct task_struct *task = info; 8591 perf_cgroup_switch(task, PERF_CGROUP_SWOUT | PERF_CGROUP_SWIN); 8592 return 0; 8593 } 8594 8595 static void perf_cgroup_attach(struct cgroup_subsys_state *css, 8596 struct cgroup_taskset *tset) 8597 { 8598 struct task_struct *task; 8599 8600 cgroup_taskset_for_each(task, tset) 8601 task_function_call(task, __perf_cgroup_move, task); 8602 } 8603 8604 static void perf_cgroup_exit(struct cgroup_subsys_state *css, 8605 struct cgroup_subsys_state *old_css, 8606 struct task_struct *task) 8607 { 8608 /* 8609 * cgroup_exit() is called in the copy_process() failure path. 8610 * Ignore this case since the task hasn't ran yet, this avoids 8611 * trying to poke a half freed task state from generic code. 8612 */ 8613 if (!(task->flags & PF_EXITING)) 8614 return; 8615 8616 task_function_call(task, __perf_cgroup_move, task); 8617 } 8618 8619 struct cgroup_subsys perf_event_cgrp_subsys = { 8620 .css_alloc = perf_cgroup_css_alloc, 8621 .css_free = perf_cgroup_css_free, 8622 .exit = perf_cgroup_exit, 8623 .attach = perf_cgroup_attach, 8624 }; 8625 #endif /* CONFIG_CGROUP_PERF */ 8626