xref: /openbmc/linux/kernel/events/core.c (revision 6774def6)
1 /*
2  * Performance events core code:
3  *
4  *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
5  *  Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar
6  *  Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
7  *  Copyright  ©  2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
8  *
9  * For licensing details see kernel-base/COPYING
10  */
11 
12 #include <linux/fs.h>
13 #include <linux/mm.h>
14 #include <linux/cpu.h>
15 #include <linux/smp.h>
16 #include <linux/idr.h>
17 #include <linux/file.h>
18 #include <linux/poll.h>
19 #include <linux/slab.h>
20 #include <linux/hash.h>
21 #include <linux/tick.h>
22 #include <linux/sysfs.h>
23 #include <linux/dcache.h>
24 #include <linux/percpu.h>
25 #include <linux/ptrace.h>
26 #include <linux/reboot.h>
27 #include <linux/vmstat.h>
28 #include <linux/device.h>
29 #include <linux/export.h>
30 #include <linux/vmalloc.h>
31 #include <linux/hardirq.h>
32 #include <linux/rculist.h>
33 #include <linux/uaccess.h>
34 #include <linux/syscalls.h>
35 #include <linux/anon_inodes.h>
36 #include <linux/kernel_stat.h>
37 #include <linux/perf_event.h>
38 #include <linux/ftrace_event.h>
39 #include <linux/hw_breakpoint.h>
40 #include <linux/mm_types.h>
41 #include <linux/cgroup.h>
42 #include <linux/module.h>
43 #include <linux/mman.h>
44 #include <linux/compat.h>
45 
46 #include "internal.h"
47 
48 #include <asm/irq_regs.h>
49 
50 static struct workqueue_struct *perf_wq;
51 
52 struct remote_function_call {
53 	struct task_struct	*p;
54 	int			(*func)(void *info);
55 	void			*info;
56 	int			ret;
57 };
58 
59 static void remote_function(void *data)
60 {
61 	struct remote_function_call *tfc = data;
62 	struct task_struct *p = tfc->p;
63 
64 	if (p) {
65 		tfc->ret = -EAGAIN;
66 		if (task_cpu(p) != smp_processor_id() || !task_curr(p))
67 			return;
68 	}
69 
70 	tfc->ret = tfc->func(tfc->info);
71 }
72 
73 /**
74  * task_function_call - call a function on the cpu on which a task runs
75  * @p:		the task to evaluate
76  * @func:	the function to be called
77  * @info:	the function call argument
78  *
79  * Calls the function @func when the task is currently running. This might
80  * be on the current CPU, which just calls the function directly
81  *
82  * returns: @func return value, or
83  *	    -ESRCH  - when the process isn't running
84  *	    -EAGAIN - when the process moved away
85  */
86 static int
87 task_function_call(struct task_struct *p, int (*func) (void *info), void *info)
88 {
89 	struct remote_function_call data = {
90 		.p	= p,
91 		.func	= func,
92 		.info	= info,
93 		.ret	= -ESRCH, /* No such (running) process */
94 	};
95 
96 	if (task_curr(p))
97 		smp_call_function_single(task_cpu(p), remote_function, &data, 1);
98 
99 	return data.ret;
100 }
101 
102 /**
103  * cpu_function_call - call a function on the cpu
104  * @func:	the function to be called
105  * @info:	the function call argument
106  *
107  * Calls the function @func on the remote cpu.
108  *
109  * returns: @func return value or -ENXIO when the cpu is offline
110  */
111 static int cpu_function_call(int cpu, int (*func) (void *info), void *info)
112 {
113 	struct remote_function_call data = {
114 		.p	= NULL,
115 		.func	= func,
116 		.info	= info,
117 		.ret	= -ENXIO, /* No such CPU */
118 	};
119 
120 	smp_call_function_single(cpu, remote_function, &data, 1);
121 
122 	return data.ret;
123 }
124 
125 #define EVENT_OWNER_KERNEL ((void *) -1)
126 
127 static bool is_kernel_event(struct perf_event *event)
128 {
129 	return event->owner == EVENT_OWNER_KERNEL;
130 }
131 
132 #define PERF_FLAG_ALL (PERF_FLAG_FD_NO_GROUP |\
133 		       PERF_FLAG_FD_OUTPUT  |\
134 		       PERF_FLAG_PID_CGROUP |\
135 		       PERF_FLAG_FD_CLOEXEC)
136 
137 /*
138  * branch priv levels that need permission checks
139  */
140 #define PERF_SAMPLE_BRANCH_PERM_PLM \
141 	(PERF_SAMPLE_BRANCH_KERNEL |\
142 	 PERF_SAMPLE_BRANCH_HV)
143 
144 enum event_type_t {
145 	EVENT_FLEXIBLE = 0x1,
146 	EVENT_PINNED = 0x2,
147 	EVENT_ALL = EVENT_FLEXIBLE | EVENT_PINNED,
148 };
149 
150 /*
151  * perf_sched_events : >0 events exist
152  * perf_cgroup_events: >0 per-cpu cgroup events exist on this cpu
153  */
154 struct static_key_deferred perf_sched_events __read_mostly;
155 static DEFINE_PER_CPU(atomic_t, perf_cgroup_events);
156 static DEFINE_PER_CPU(atomic_t, perf_branch_stack_events);
157 
158 static atomic_t nr_mmap_events __read_mostly;
159 static atomic_t nr_comm_events __read_mostly;
160 static atomic_t nr_task_events __read_mostly;
161 static atomic_t nr_freq_events __read_mostly;
162 
163 static LIST_HEAD(pmus);
164 static DEFINE_MUTEX(pmus_lock);
165 static struct srcu_struct pmus_srcu;
166 
167 /*
168  * perf event paranoia level:
169  *  -1 - not paranoid at all
170  *   0 - disallow raw tracepoint access for unpriv
171  *   1 - disallow cpu events for unpriv
172  *   2 - disallow kernel profiling for unpriv
173  */
174 int sysctl_perf_event_paranoid __read_mostly = 1;
175 
176 /* Minimum for 512 kiB + 1 user control page */
177 int sysctl_perf_event_mlock __read_mostly = 512 + (PAGE_SIZE / 1024); /* 'free' kiB per user */
178 
179 /*
180  * max perf event sample rate
181  */
182 #define DEFAULT_MAX_SAMPLE_RATE		100000
183 #define DEFAULT_SAMPLE_PERIOD_NS	(NSEC_PER_SEC / DEFAULT_MAX_SAMPLE_RATE)
184 #define DEFAULT_CPU_TIME_MAX_PERCENT	25
185 
186 int sysctl_perf_event_sample_rate __read_mostly	= DEFAULT_MAX_SAMPLE_RATE;
187 
188 static int max_samples_per_tick __read_mostly	= DIV_ROUND_UP(DEFAULT_MAX_SAMPLE_RATE, HZ);
189 static int perf_sample_period_ns __read_mostly	= DEFAULT_SAMPLE_PERIOD_NS;
190 
191 static int perf_sample_allowed_ns __read_mostly =
192 	DEFAULT_SAMPLE_PERIOD_NS * DEFAULT_CPU_TIME_MAX_PERCENT / 100;
193 
194 void update_perf_cpu_limits(void)
195 {
196 	u64 tmp = perf_sample_period_ns;
197 
198 	tmp *= sysctl_perf_cpu_time_max_percent;
199 	do_div(tmp, 100);
200 	ACCESS_ONCE(perf_sample_allowed_ns) = tmp;
201 }
202 
203 static int perf_rotate_context(struct perf_cpu_context *cpuctx);
204 
205 int perf_proc_update_handler(struct ctl_table *table, int write,
206 		void __user *buffer, size_t *lenp,
207 		loff_t *ppos)
208 {
209 	int ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
210 
211 	if (ret || !write)
212 		return ret;
213 
214 	max_samples_per_tick = DIV_ROUND_UP(sysctl_perf_event_sample_rate, HZ);
215 	perf_sample_period_ns = NSEC_PER_SEC / sysctl_perf_event_sample_rate;
216 	update_perf_cpu_limits();
217 
218 	return 0;
219 }
220 
221 int sysctl_perf_cpu_time_max_percent __read_mostly = DEFAULT_CPU_TIME_MAX_PERCENT;
222 
223 int perf_cpu_time_max_percent_handler(struct ctl_table *table, int write,
224 				void __user *buffer, size_t *lenp,
225 				loff_t *ppos)
226 {
227 	int ret = proc_dointvec(table, write, buffer, lenp, ppos);
228 
229 	if (ret || !write)
230 		return ret;
231 
232 	update_perf_cpu_limits();
233 
234 	return 0;
235 }
236 
237 /*
238  * perf samples are done in some very critical code paths (NMIs).
239  * If they take too much CPU time, the system can lock up and not
240  * get any real work done.  This will drop the sample rate when
241  * we detect that events are taking too long.
242  */
243 #define NR_ACCUMULATED_SAMPLES 128
244 static DEFINE_PER_CPU(u64, running_sample_length);
245 
246 static void perf_duration_warn(struct irq_work *w)
247 {
248 	u64 allowed_ns = ACCESS_ONCE(perf_sample_allowed_ns);
249 	u64 avg_local_sample_len;
250 	u64 local_samples_len;
251 
252 	local_samples_len = __this_cpu_read(running_sample_length);
253 	avg_local_sample_len = local_samples_len/NR_ACCUMULATED_SAMPLES;
254 
255 	printk_ratelimited(KERN_WARNING
256 			"perf interrupt took too long (%lld > %lld), lowering "
257 			"kernel.perf_event_max_sample_rate to %d\n",
258 			avg_local_sample_len, allowed_ns >> 1,
259 			sysctl_perf_event_sample_rate);
260 }
261 
262 static DEFINE_IRQ_WORK(perf_duration_work, perf_duration_warn);
263 
264 void perf_sample_event_took(u64 sample_len_ns)
265 {
266 	u64 allowed_ns = ACCESS_ONCE(perf_sample_allowed_ns);
267 	u64 avg_local_sample_len;
268 	u64 local_samples_len;
269 
270 	if (allowed_ns == 0)
271 		return;
272 
273 	/* decay the counter by 1 average sample */
274 	local_samples_len = __this_cpu_read(running_sample_length);
275 	local_samples_len -= local_samples_len/NR_ACCUMULATED_SAMPLES;
276 	local_samples_len += sample_len_ns;
277 	__this_cpu_write(running_sample_length, local_samples_len);
278 
279 	/*
280 	 * note: this will be biased artifically low until we have
281 	 * seen NR_ACCUMULATED_SAMPLES.  Doing it this way keeps us
282 	 * from having to maintain a count.
283 	 */
284 	avg_local_sample_len = local_samples_len/NR_ACCUMULATED_SAMPLES;
285 
286 	if (avg_local_sample_len <= allowed_ns)
287 		return;
288 
289 	if (max_samples_per_tick <= 1)
290 		return;
291 
292 	max_samples_per_tick = DIV_ROUND_UP(max_samples_per_tick, 2);
293 	sysctl_perf_event_sample_rate = max_samples_per_tick * HZ;
294 	perf_sample_period_ns = NSEC_PER_SEC / sysctl_perf_event_sample_rate;
295 
296 	update_perf_cpu_limits();
297 
298 	if (!irq_work_queue(&perf_duration_work)) {
299 		early_printk("perf interrupt took too long (%lld > %lld), lowering "
300 			     "kernel.perf_event_max_sample_rate to %d\n",
301 			     avg_local_sample_len, allowed_ns >> 1,
302 			     sysctl_perf_event_sample_rate);
303 	}
304 }
305 
306 static atomic64_t perf_event_id;
307 
308 static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,
309 			      enum event_type_t event_type);
310 
311 static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx,
312 			     enum event_type_t event_type,
313 			     struct task_struct *task);
314 
315 static void update_context_time(struct perf_event_context *ctx);
316 static u64 perf_event_time(struct perf_event *event);
317 
318 void __weak perf_event_print_debug(void)	{ }
319 
320 extern __weak const char *perf_pmu_name(void)
321 {
322 	return "pmu";
323 }
324 
325 static inline u64 perf_clock(void)
326 {
327 	return local_clock();
328 }
329 
330 static inline struct perf_cpu_context *
331 __get_cpu_context(struct perf_event_context *ctx)
332 {
333 	return this_cpu_ptr(ctx->pmu->pmu_cpu_context);
334 }
335 
336 static void perf_ctx_lock(struct perf_cpu_context *cpuctx,
337 			  struct perf_event_context *ctx)
338 {
339 	raw_spin_lock(&cpuctx->ctx.lock);
340 	if (ctx)
341 		raw_spin_lock(&ctx->lock);
342 }
343 
344 static void perf_ctx_unlock(struct perf_cpu_context *cpuctx,
345 			    struct perf_event_context *ctx)
346 {
347 	if (ctx)
348 		raw_spin_unlock(&ctx->lock);
349 	raw_spin_unlock(&cpuctx->ctx.lock);
350 }
351 
352 #ifdef CONFIG_CGROUP_PERF
353 
354 /*
355  * perf_cgroup_info keeps track of time_enabled for a cgroup.
356  * This is a per-cpu dynamically allocated data structure.
357  */
358 struct perf_cgroup_info {
359 	u64				time;
360 	u64				timestamp;
361 };
362 
363 struct perf_cgroup {
364 	struct cgroup_subsys_state	css;
365 	struct perf_cgroup_info	__percpu *info;
366 };
367 
368 /*
369  * Must ensure cgroup is pinned (css_get) before calling
370  * this function. In other words, we cannot call this function
371  * if there is no cgroup event for the current CPU context.
372  */
373 static inline struct perf_cgroup *
374 perf_cgroup_from_task(struct task_struct *task)
375 {
376 	return container_of(task_css(task, perf_event_cgrp_id),
377 			    struct perf_cgroup, css);
378 }
379 
380 static inline bool
381 perf_cgroup_match(struct perf_event *event)
382 {
383 	struct perf_event_context *ctx = event->ctx;
384 	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
385 
386 	/* @event doesn't care about cgroup */
387 	if (!event->cgrp)
388 		return true;
389 
390 	/* wants specific cgroup scope but @cpuctx isn't associated with any */
391 	if (!cpuctx->cgrp)
392 		return false;
393 
394 	/*
395 	 * Cgroup scoping is recursive.  An event enabled for a cgroup is
396 	 * also enabled for all its descendant cgroups.  If @cpuctx's
397 	 * cgroup is a descendant of @event's (the test covers identity
398 	 * case), it's a match.
399 	 */
400 	return cgroup_is_descendant(cpuctx->cgrp->css.cgroup,
401 				    event->cgrp->css.cgroup);
402 }
403 
404 static inline void perf_detach_cgroup(struct perf_event *event)
405 {
406 	css_put(&event->cgrp->css);
407 	event->cgrp = NULL;
408 }
409 
410 static inline int is_cgroup_event(struct perf_event *event)
411 {
412 	return event->cgrp != NULL;
413 }
414 
415 static inline u64 perf_cgroup_event_time(struct perf_event *event)
416 {
417 	struct perf_cgroup_info *t;
418 
419 	t = per_cpu_ptr(event->cgrp->info, event->cpu);
420 	return t->time;
421 }
422 
423 static inline void __update_cgrp_time(struct perf_cgroup *cgrp)
424 {
425 	struct perf_cgroup_info *info;
426 	u64 now;
427 
428 	now = perf_clock();
429 
430 	info = this_cpu_ptr(cgrp->info);
431 
432 	info->time += now - info->timestamp;
433 	info->timestamp = now;
434 }
435 
436 static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx)
437 {
438 	struct perf_cgroup *cgrp_out = cpuctx->cgrp;
439 	if (cgrp_out)
440 		__update_cgrp_time(cgrp_out);
441 }
442 
443 static inline void update_cgrp_time_from_event(struct perf_event *event)
444 {
445 	struct perf_cgroup *cgrp;
446 
447 	/*
448 	 * ensure we access cgroup data only when needed and
449 	 * when we know the cgroup is pinned (css_get)
450 	 */
451 	if (!is_cgroup_event(event))
452 		return;
453 
454 	cgrp = perf_cgroup_from_task(current);
455 	/*
456 	 * Do not update time when cgroup is not active
457 	 */
458 	if (cgrp == event->cgrp)
459 		__update_cgrp_time(event->cgrp);
460 }
461 
462 static inline void
463 perf_cgroup_set_timestamp(struct task_struct *task,
464 			  struct perf_event_context *ctx)
465 {
466 	struct perf_cgroup *cgrp;
467 	struct perf_cgroup_info *info;
468 
469 	/*
470 	 * ctx->lock held by caller
471 	 * ensure we do not access cgroup data
472 	 * unless we have the cgroup pinned (css_get)
473 	 */
474 	if (!task || !ctx->nr_cgroups)
475 		return;
476 
477 	cgrp = perf_cgroup_from_task(task);
478 	info = this_cpu_ptr(cgrp->info);
479 	info->timestamp = ctx->timestamp;
480 }
481 
482 #define PERF_CGROUP_SWOUT	0x1 /* cgroup switch out every event */
483 #define PERF_CGROUP_SWIN	0x2 /* cgroup switch in events based on task */
484 
485 /*
486  * reschedule events based on the cgroup constraint of task.
487  *
488  * mode SWOUT : schedule out everything
489  * mode SWIN : schedule in based on cgroup for next
490  */
491 void perf_cgroup_switch(struct task_struct *task, int mode)
492 {
493 	struct perf_cpu_context *cpuctx;
494 	struct pmu *pmu;
495 	unsigned long flags;
496 
497 	/*
498 	 * disable interrupts to avoid geting nr_cgroup
499 	 * changes via __perf_event_disable(). Also
500 	 * avoids preemption.
501 	 */
502 	local_irq_save(flags);
503 
504 	/*
505 	 * we reschedule only in the presence of cgroup
506 	 * constrained events.
507 	 */
508 	rcu_read_lock();
509 
510 	list_for_each_entry_rcu(pmu, &pmus, entry) {
511 		cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
512 		if (cpuctx->unique_pmu != pmu)
513 			continue; /* ensure we process each cpuctx once */
514 
515 		/*
516 		 * perf_cgroup_events says at least one
517 		 * context on this CPU has cgroup events.
518 		 *
519 		 * ctx->nr_cgroups reports the number of cgroup
520 		 * events for a context.
521 		 */
522 		if (cpuctx->ctx.nr_cgroups > 0) {
523 			perf_ctx_lock(cpuctx, cpuctx->task_ctx);
524 			perf_pmu_disable(cpuctx->ctx.pmu);
525 
526 			if (mode & PERF_CGROUP_SWOUT) {
527 				cpu_ctx_sched_out(cpuctx, EVENT_ALL);
528 				/*
529 				 * must not be done before ctxswout due
530 				 * to event_filter_match() in event_sched_out()
531 				 */
532 				cpuctx->cgrp = NULL;
533 			}
534 
535 			if (mode & PERF_CGROUP_SWIN) {
536 				WARN_ON_ONCE(cpuctx->cgrp);
537 				/*
538 				 * set cgrp before ctxsw in to allow
539 				 * event_filter_match() to not have to pass
540 				 * task around
541 				 */
542 				cpuctx->cgrp = perf_cgroup_from_task(task);
543 				cpu_ctx_sched_in(cpuctx, EVENT_ALL, task);
544 			}
545 			perf_pmu_enable(cpuctx->ctx.pmu);
546 			perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
547 		}
548 	}
549 
550 	rcu_read_unlock();
551 
552 	local_irq_restore(flags);
553 }
554 
555 static inline void perf_cgroup_sched_out(struct task_struct *task,
556 					 struct task_struct *next)
557 {
558 	struct perf_cgroup *cgrp1;
559 	struct perf_cgroup *cgrp2 = NULL;
560 
561 	/*
562 	 * we come here when we know perf_cgroup_events > 0
563 	 */
564 	cgrp1 = perf_cgroup_from_task(task);
565 
566 	/*
567 	 * next is NULL when called from perf_event_enable_on_exec()
568 	 * that will systematically cause a cgroup_switch()
569 	 */
570 	if (next)
571 		cgrp2 = perf_cgroup_from_task(next);
572 
573 	/*
574 	 * only schedule out current cgroup events if we know
575 	 * that we are switching to a different cgroup. Otherwise,
576 	 * do no touch the cgroup events.
577 	 */
578 	if (cgrp1 != cgrp2)
579 		perf_cgroup_switch(task, PERF_CGROUP_SWOUT);
580 }
581 
582 static inline void perf_cgroup_sched_in(struct task_struct *prev,
583 					struct task_struct *task)
584 {
585 	struct perf_cgroup *cgrp1;
586 	struct perf_cgroup *cgrp2 = NULL;
587 
588 	/*
589 	 * we come here when we know perf_cgroup_events > 0
590 	 */
591 	cgrp1 = perf_cgroup_from_task(task);
592 
593 	/* prev can never be NULL */
594 	cgrp2 = perf_cgroup_from_task(prev);
595 
596 	/*
597 	 * only need to schedule in cgroup events if we are changing
598 	 * cgroup during ctxsw. Cgroup events were not scheduled
599 	 * out of ctxsw out if that was not the case.
600 	 */
601 	if (cgrp1 != cgrp2)
602 		perf_cgroup_switch(task, PERF_CGROUP_SWIN);
603 }
604 
605 static inline int perf_cgroup_connect(int fd, struct perf_event *event,
606 				      struct perf_event_attr *attr,
607 				      struct perf_event *group_leader)
608 {
609 	struct perf_cgroup *cgrp;
610 	struct cgroup_subsys_state *css;
611 	struct fd f = fdget(fd);
612 	int ret = 0;
613 
614 	if (!f.file)
615 		return -EBADF;
616 
617 	css = css_tryget_online_from_dir(f.file->f_dentry,
618 					 &perf_event_cgrp_subsys);
619 	if (IS_ERR(css)) {
620 		ret = PTR_ERR(css);
621 		goto out;
622 	}
623 
624 	cgrp = container_of(css, struct perf_cgroup, css);
625 	event->cgrp = cgrp;
626 
627 	/*
628 	 * all events in a group must monitor
629 	 * the same cgroup because a task belongs
630 	 * to only one perf cgroup at a time
631 	 */
632 	if (group_leader && group_leader->cgrp != cgrp) {
633 		perf_detach_cgroup(event);
634 		ret = -EINVAL;
635 	}
636 out:
637 	fdput(f);
638 	return ret;
639 }
640 
641 static inline void
642 perf_cgroup_set_shadow_time(struct perf_event *event, u64 now)
643 {
644 	struct perf_cgroup_info *t;
645 	t = per_cpu_ptr(event->cgrp->info, event->cpu);
646 	event->shadow_ctx_time = now - t->timestamp;
647 }
648 
649 static inline void
650 perf_cgroup_defer_enabled(struct perf_event *event)
651 {
652 	/*
653 	 * when the current task's perf cgroup does not match
654 	 * the event's, we need to remember to call the
655 	 * perf_mark_enable() function the first time a task with
656 	 * a matching perf cgroup is scheduled in.
657 	 */
658 	if (is_cgroup_event(event) && !perf_cgroup_match(event))
659 		event->cgrp_defer_enabled = 1;
660 }
661 
662 static inline void
663 perf_cgroup_mark_enabled(struct perf_event *event,
664 			 struct perf_event_context *ctx)
665 {
666 	struct perf_event *sub;
667 	u64 tstamp = perf_event_time(event);
668 
669 	if (!event->cgrp_defer_enabled)
670 		return;
671 
672 	event->cgrp_defer_enabled = 0;
673 
674 	event->tstamp_enabled = tstamp - event->total_time_enabled;
675 	list_for_each_entry(sub, &event->sibling_list, group_entry) {
676 		if (sub->state >= PERF_EVENT_STATE_INACTIVE) {
677 			sub->tstamp_enabled = tstamp - sub->total_time_enabled;
678 			sub->cgrp_defer_enabled = 0;
679 		}
680 	}
681 }
682 #else /* !CONFIG_CGROUP_PERF */
683 
684 static inline bool
685 perf_cgroup_match(struct perf_event *event)
686 {
687 	return true;
688 }
689 
690 static inline void perf_detach_cgroup(struct perf_event *event)
691 {}
692 
693 static inline int is_cgroup_event(struct perf_event *event)
694 {
695 	return 0;
696 }
697 
698 static inline u64 perf_cgroup_event_cgrp_time(struct perf_event *event)
699 {
700 	return 0;
701 }
702 
703 static inline void update_cgrp_time_from_event(struct perf_event *event)
704 {
705 }
706 
707 static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx)
708 {
709 }
710 
711 static inline void perf_cgroup_sched_out(struct task_struct *task,
712 					 struct task_struct *next)
713 {
714 }
715 
716 static inline void perf_cgroup_sched_in(struct task_struct *prev,
717 					struct task_struct *task)
718 {
719 }
720 
721 static inline int perf_cgroup_connect(pid_t pid, struct perf_event *event,
722 				      struct perf_event_attr *attr,
723 				      struct perf_event *group_leader)
724 {
725 	return -EINVAL;
726 }
727 
728 static inline void
729 perf_cgroup_set_timestamp(struct task_struct *task,
730 			  struct perf_event_context *ctx)
731 {
732 }
733 
734 void
735 perf_cgroup_switch(struct task_struct *task, struct task_struct *next)
736 {
737 }
738 
739 static inline void
740 perf_cgroup_set_shadow_time(struct perf_event *event, u64 now)
741 {
742 }
743 
744 static inline u64 perf_cgroup_event_time(struct perf_event *event)
745 {
746 	return 0;
747 }
748 
749 static inline void
750 perf_cgroup_defer_enabled(struct perf_event *event)
751 {
752 }
753 
754 static inline void
755 perf_cgroup_mark_enabled(struct perf_event *event,
756 			 struct perf_event_context *ctx)
757 {
758 }
759 #endif
760 
761 /*
762  * set default to be dependent on timer tick just
763  * like original code
764  */
765 #define PERF_CPU_HRTIMER (1000 / HZ)
766 /*
767  * function must be called with interrupts disbled
768  */
769 static enum hrtimer_restart perf_cpu_hrtimer_handler(struct hrtimer *hr)
770 {
771 	struct perf_cpu_context *cpuctx;
772 	enum hrtimer_restart ret = HRTIMER_NORESTART;
773 	int rotations = 0;
774 
775 	WARN_ON(!irqs_disabled());
776 
777 	cpuctx = container_of(hr, struct perf_cpu_context, hrtimer);
778 
779 	rotations = perf_rotate_context(cpuctx);
780 
781 	/*
782 	 * arm timer if needed
783 	 */
784 	if (rotations) {
785 		hrtimer_forward_now(hr, cpuctx->hrtimer_interval);
786 		ret = HRTIMER_RESTART;
787 	}
788 
789 	return ret;
790 }
791 
792 /* CPU is going down */
793 void perf_cpu_hrtimer_cancel(int cpu)
794 {
795 	struct perf_cpu_context *cpuctx;
796 	struct pmu *pmu;
797 	unsigned long flags;
798 
799 	if (WARN_ON(cpu != smp_processor_id()))
800 		return;
801 
802 	local_irq_save(flags);
803 
804 	rcu_read_lock();
805 
806 	list_for_each_entry_rcu(pmu, &pmus, entry) {
807 		cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
808 
809 		if (pmu->task_ctx_nr == perf_sw_context)
810 			continue;
811 
812 		hrtimer_cancel(&cpuctx->hrtimer);
813 	}
814 
815 	rcu_read_unlock();
816 
817 	local_irq_restore(flags);
818 }
819 
820 static void __perf_cpu_hrtimer_init(struct perf_cpu_context *cpuctx, int cpu)
821 {
822 	struct hrtimer *hr = &cpuctx->hrtimer;
823 	struct pmu *pmu = cpuctx->ctx.pmu;
824 	int timer;
825 
826 	/* no multiplexing needed for SW PMU */
827 	if (pmu->task_ctx_nr == perf_sw_context)
828 		return;
829 
830 	/*
831 	 * check default is sane, if not set then force to
832 	 * default interval (1/tick)
833 	 */
834 	timer = pmu->hrtimer_interval_ms;
835 	if (timer < 1)
836 		timer = pmu->hrtimer_interval_ms = PERF_CPU_HRTIMER;
837 
838 	cpuctx->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * timer);
839 
840 	hrtimer_init(hr, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);
841 	hr->function = perf_cpu_hrtimer_handler;
842 }
843 
844 static void perf_cpu_hrtimer_restart(struct perf_cpu_context *cpuctx)
845 {
846 	struct hrtimer *hr = &cpuctx->hrtimer;
847 	struct pmu *pmu = cpuctx->ctx.pmu;
848 
849 	/* not for SW PMU */
850 	if (pmu->task_ctx_nr == perf_sw_context)
851 		return;
852 
853 	if (hrtimer_active(hr))
854 		return;
855 
856 	if (!hrtimer_callback_running(hr))
857 		__hrtimer_start_range_ns(hr, cpuctx->hrtimer_interval,
858 					 0, HRTIMER_MODE_REL_PINNED, 0);
859 }
860 
861 void perf_pmu_disable(struct pmu *pmu)
862 {
863 	int *count = this_cpu_ptr(pmu->pmu_disable_count);
864 	if (!(*count)++)
865 		pmu->pmu_disable(pmu);
866 }
867 
868 void perf_pmu_enable(struct pmu *pmu)
869 {
870 	int *count = this_cpu_ptr(pmu->pmu_disable_count);
871 	if (!--(*count))
872 		pmu->pmu_enable(pmu);
873 }
874 
875 static DEFINE_PER_CPU(struct list_head, rotation_list);
876 
877 /*
878  * perf_pmu_rotate_start() and perf_rotate_context() are fully serialized
879  * because they're strictly cpu affine and rotate_start is called with IRQs
880  * disabled, while rotate_context is called from IRQ context.
881  */
882 static void perf_pmu_rotate_start(struct pmu *pmu)
883 {
884 	struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
885 	struct list_head *head = this_cpu_ptr(&rotation_list);
886 
887 	WARN_ON(!irqs_disabled());
888 
889 	if (list_empty(&cpuctx->rotation_list))
890 		list_add(&cpuctx->rotation_list, head);
891 }
892 
893 static void get_ctx(struct perf_event_context *ctx)
894 {
895 	WARN_ON(!atomic_inc_not_zero(&ctx->refcount));
896 }
897 
898 static void put_ctx(struct perf_event_context *ctx)
899 {
900 	if (atomic_dec_and_test(&ctx->refcount)) {
901 		if (ctx->parent_ctx)
902 			put_ctx(ctx->parent_ctx);
903 		if (ctx->task)
904 			put_task_struct(ctx->task);
905 		kfree_rcu(ctx, rcu_head);
906 	}
907 }
908 
909 /*
910  * This must be done under the ctx->lock, such as to serialize against
911  * context_equiv(), therefore we cannot call put_ctx() since that might end up
912  * calling scheduler related locks and ctx->lock nests inside those.
913  */
914 static __must_check struct perf_event_context *
915 unclone_ctx(struct perf_event_context *ctx)
916 {
917 	struct perf_event_context *parent_ctx = ctx->parent_ctx;
918 
919 	lockdep_assert_held(&ctx->lock);
920 
921 	if (parent_ctx)
922 		ctx->parent_ctx = NULL;
923 	ctx->generation++;
924 
925 	return parent_ctx;
926 }
927 
928 static u32 perf_event_pid(struct perf_event *event, struct task_struct *p)
929 {
930 	/*
931 	 * only top level events have the pid namespace they were created in
932 	 */
933 	if (event->parent)
934 		event = event->parent;
935 
936 	return task_tgid_nr_ns(p, event->ns);
937 }
938 
939 static u32 perf_event_tid(struct perf_event *event, struct task_struct *p)
940 {
941 	/*
942 	 * only top level events have the pid namespace they were created in
943 	 */
944 	if (event->parent)
945 		event = event->parent;
946 
947 	return task_pid_nr_ns(p, event->ns);
948 }
949 
950 /*
951  * If we inherit events we want to return the parent event id
952  * to userspace.
953  */
954 static u64 primary_event_id(struct perf_event *event)
955 {
956 	u64 id = event->id;
957 
958 	if (event->parent)
959 		id = event->parent->id;
960 
961 	return id;
962 }
963 
964 /*
965  * Get the perf_event_context for a task and lock it.
966  * This has to cope with with the fact that until it is locked,
967  * the context could get moved to another task.
968  */
969 static struct perf_event_context *
970 perf_lock_task_context(struct task_struct *task, int ctxn, unsigned long *flags)
971 {
972 	struct perf_event_context *ctx;
973 
974 retry:
975 	/*
976 	 * One of the few rules of preemptible RCU is that one cannot do
977 	 * rcu_read_unlock() while holding a scheduler (or nested) lock when
978 	 * part of the read side critical section was preemptible -- see
979 	 * rcu_read_unlock_special().
980 	 *
981 	 * Since ctx->lock nests under rq->lock we must ensure the entire read
982 	 * side critical section is non-preemptible.
983 	 */
984 	preempt_disable();
985 	rcu_read_lock();
986 	ctx = rcu_dereference(task->perf_event_ctxp[ctxn]);
987 	if (ctx) {
988 		/*
989 		 * If this context is a clone of another, it might
990 		 * get swapped for another underneath us by
991 		 * perf_event_task_sched_out, though the
992 		 * rcu_read_lock() protects us from any context
993 		 * getting freed.  Lock the context and check if it
994 		 * got swapped before we could get the lock, and retry
995 		 * if so.  If we locked the right context, then it
996 		 * can't get swapped on us any more.
997 		 */
998 		raw_spin_lock_irqsave(&ctx->lock, *flags);
999 		if (ctx != rcu_dereference(task->perf_event_ctxp[ctxn])) {
1000 			raw_spin_unlock_irqrestore(&ctx->lock, *flags);
1001 			rcu_read_unlock();
1002 			preempt_enable();
1003 			goto retry;
1004 		}
1005 
1006 		if (!atomic_inc_not_zero(&ctx->refcount)) {
1007 			raw_spin_unlock_irqrestore(&ctx->lock, *flags);
1008 			ctx = NULL;
1009 		}
1010 	}
1011 	rcu_read_unlock();
1012 	preempt_enable();
1013 	return ctx;
1014 }
1015 
1016 /*
1017  * Get the context for a task and increment its pin_count so it
1018  * can't get swapped to another task.  This also increments its
1019  * reference count so that the context can't get freed.
1020  */
1021 static struct perf_event_context *
1022 perf_pin_task_context(struct task_struct *task, int ctxn)
1023 {
1024 	struct perf_event_context *ctx;
1025 	unsigned long flags;
1026 
1027 	ctx = perf_lock_task_context(task, ctxn, &flags);
1028 	if (ctx) {
1029 		++ctx->pin_count;
1030 		raw_spin_unlock_irqrestore(&ctx->lock, flags);
1031 	}
1032 	return ctx;
1033 }
1034 
1035 static void perf_unpin_context(struct perf_event_context *ctx)
1036 {
1037 	unsigned long flags;
1038 
1039 	raw_spin_lock_irqsave(&ctx->lock, flags);
1040 	--ctx->pin_count;
1041 	raw_spin_unlock_irqrestore(&ctx->lock, flags);
1042 }
1043 
1044 /*
1045  * Update the record of the current time in a context.
1046  */
1047 static void update_context_time(struct perf_event_context *ctx)
1048 {
1049 	u64 now = perf_clock();
1050 
1051 	ctx->time += now - ctx->timestamp;
1052 	ctx->timestamp = now;
1053 }
1054 
1055 static u64 perf_event_time(struct perf_event *event)
1056 {
1057 	struct perf_event_context *ctx = event->ctx;
1058 
1059 	if (is_cgroup_event(event))
1060 		return perf_cgroup_event_time(event);
1061 
1062 	return ctx ? ctx->time : 0;
1063 }
1064 
1065 /*
1066  * Update the total_time_enabled and total_time_running fields for a event.
1067  * The caller of this function needs to hold the ctx->lock.
1068  */
1069 static void update_event_times(struct perf_event *event)
1070 {
1071 	struct perf_event_context *ctx = event->ctx;
1072 	u64 run_end;
1073 
1074 	if (event->state < PERF_EVENT_STATE_INACTIVE ||
1075 	    event->group_leader->state < PERF_EVENT_STATE_INACTIVE)
1076 		return;
1077 	/*
1078 	 * in cgroup mode, time_enabled represents
1079 	 * the time the event was enabled AND active
1080 	 * tasks were in the monitored cgroup. This is
1081 	 * independent of the activity of the context as
1082 	 * there may be a mix of cgroup and non-cgroup events.
1083 	 *
1084 	 * That is why we treat cgroup events differently
1085 	 * here.
1086 	 */
1087 	if (is_cgroup_event(event))
1088 		run_end = perf_cgroup_event_time(event);
1089 	else if (ctx->is_active)
1090 		run_end = ctx->time;
1091 	else
1092 		run_end = event->tstamp_stopped;
1093 
1094 	event->total_time_enabled = run_end - event->tstamp_enabled;
1095 
1096 	if (event->state == PERF_EVENT_STATE_INACTIVE)
1097 		run_end = event->tstamp_stopped;
1098 	else
1099 		run_end = perf_event_time(event);
1100 
1101 	event->total_time_running = run_end - event->tstamp_running;
1102 
1103 }
1104 
1105 /*
1106  * Update total_time_enabled and total_time_running for all events in a group.
1107  */
1108 static void update_group_times(struct perf_event *leader)
1109 {
1110 	struct perf_event *event;
1111 
1112 	update_event_times(leader);
1113 	list_for_each_entry(event, &leader->sibling_list, group_entry)
1114 		update_event_times(event);
1115 }
1116 
1117 static struct list_head *
1118 ctx_group_list(struct perf_event *event, struct perf_event_context *ctx)
1119 {
1120 	if (event->attr.pinned)
1121 		return &ctx->pinned_groups;
1122 	else
1123 		return &ctx->flexible_groups;
1124 }
1125 
1126 /*
1127  * Add a event from the lists for its context.
1128  * Must be called with ctx->mutex and ctx->lock held.
1129  */
1130 static void
1131 list_add_event(struct perf_event *event, struct perf_event_context *ctx)
1132 {
1133 	WARN_ON_ONCE(event->attach_state & PERF_ATTACH_CONTEXT);
1134 	event->attach_state |= PERF_ATTACH_CONTEXT;
1135 
1136 	/*
1137 	 * If we're a stand alone event or group leader, we go to the context
1138 	 * list, group events are kept attached to the group so that
1139 	 * perf_group_detach can, at all times, locate all siblings.
1140 	 */
1141 	if (event->group_leader == event) {
1142 		struct list_head *list;
1143 
1144 		if (is_software_event(event))
1145 			event->group_flags |= PERF_GROUP_SOFTWARE;
1146 
1147 		list = ctx_group_list(event, ctx);
1148 		list_add_tail(&event->group_entry, list);
1149 	}
1150 
1151 	if (is_cgroup_event(event))
1152 		ctx->nr_cgroups++;
1153 
1154 	if (has_branch_stack(event))
1155 		ctx->nr_branch_stack++;
1156 
1157 	list_add_rcu(&event->event_entry, &ctx->event_list);
1158 	if (!ctx->nr_events)
1159 		perf_pmu_rotate_start(ctx->pmu);
1160 	ctx->nr_events++;
1161 	if (event->attr.inherit_stat)
1162 		ctx->nr_stat++;
1163 
1164 	ctx->generation++;
1165 }
1166 
1167 /*
1168  * Initialize event state based on the perf_event_attr::disabled.
1169  */
1170 static inline void perf_event__state_init(struct perf_event *event)
1171 {
1172 	event->state = event->attr.disabled ? PERF_EVENT_STATE_OFF :
1173 					      PERF_EVENT_STATE_INACTIVE;
1174 }
1175 
1176 /*
1177  * Called at perf_event creation and when events are attached/detached from a
1178  * group.
1179  */
1180 static void perf_event__read_size(struct perf_event *event)
1181 {
1182 	int entry = sizeof(u64); /* value */
1183 	int size = 0;
1184 	int nr = 1;
1185 
1186 	if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
1187 		size += sizeof(u64);
1188 
1189 	if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
1190 		size += sizeof(u64);
1191 
1192 	if (event->attr.read_format & PERF_FORMAT_ID)
1193 		entry += sizeof(u64);
1194 
1195 	if (event->attr.read_format & PERF_FORMAT_GROUP) {
1196 		nr += event->group_leader->nr_siblings;
1197 		size += sizeof(u64);
1198 	}
1199 
1200 	size += entry * nr;
1201 	event->read_size = size;
1202 }
1203 
1204 static void perf_event__header_size(struct perf_event *event)
1205 {
1206 	struct perf_sample_data *data;
1207 	u64 sample_type = event->attr.sample_type;
1208 	u16 size = 0;
1209 
1210 	perf_event__read_size(event);
1211 
1212 	if (sample_type & PERF_SAMPLE_IP)
1213 		size += sizeof(data->ip);
1214 
1215 	if (sample_type & PERF_SAMPLE_ADDR)
1216 		size += sizeof(data->addr);
1217 
1218 	if (sample_type & PERF_SAMPLE_PERIOD)
1219 		size += sizeof(data->period);
1220 
1221 	if (sample_type & PERF_SAMPLE_WEIGHT)
1222 		size += sizeof(data->weight);
1223 
1224 	if (sample_type & PERF_SAMPLE_READ)
1225 		size += event->read_size;
1226 
1227 	if (sample_type & PERF_SAMPLE_DATA_SRC)
1228 		size += sizeof(data->data_src.val);
1229 
1230 	if (sample_type & PERF_SAMPLE_TRANSACTION)
1231 		size += sizeof(data->txn);
1232 
1233 	event->header_size = size;
1234 }
1235 
1236 static void perf_event__id_header_size(struct perf_event *event)
1237 {
1238 	struct perf_sample_data *data;
1239 	u64 sample_type = event->attr.sample_type;
1240 	u16 size = 0;
1241 
1242 	if (sample_type & PERF_SAMPLE_TID)
1243 		size += sizeof(data->tid_entry);
1244 
1245 	if (sample_type & PERF_SAMPLE_TIME)
1246 		size += sizeof(data->time);
1247 
1248 	if (sample_type & PERF_SAMPLE_IDENTIFIER)
1249 		size += sizeof(data->id);
1250 
1251 	if (sample_type & PERF_SAMPLE_ID)
1252 		size += sizeof(data->id);
1253 
1254 	if (sample_type & PERF_SAMPLE_STREAM_ID)
1255 		size += sizeof(data->stream_id);
1256 
1257 	if (sample_type & PERF_SAMPLE_CPU)
1258 		size += sizeof(data->cpu_entry);
1259 
1260 	event->id_header_size = size;
1261 }
1262 
1263 static void perf_group_attach(struct perf_event *event)
1264 {
1265 	struct perf_event *group_leader = event->group_leader, *pos;
1266 
1267 	/*
1268 	 * We can have double attach due to group movement in perf_event_open.
1269 	 */
1270 	if (event->attach_state & PERF_ATTACH_GROUP)
1271 		return;
1272 
1273 	event->attach_state |= PERF_ATTACH_GROUP;
1274 
1275 	if (group_leader == event)
1276 		return;
1277 
1278 	if (group_leader->group_flags & PERF_GROUP_SOFTWARE &&
1279 			!is_software_event(event))
1280 		group_leader->group_flags &= ~PERF_GROUP_SOFTWARE;
1281 
1282 	list_add_tail(&event->group_entry, &group_leader->sibling_list);
1283 	group_leader->nr_siblings++;
1284 
1285 	perf_event__header_size(group_leader);
1286 
1287 	list_for_each_entry(pos, &group_leader->sibling_list, group_entry)
1288 		perf_event__header_size(pos);
1289 }
1290 
1291 /*
1292  * Remove a event from the lists for its context.
1293  * Must be called with ctx->mutex and ctx->lock held.
1294  */
1295 static void
1296 list_del_event(struct perf_event *event, struct perf_event_context *ctx)
1297 {
1298 	struct perf_cpu_context *cpuctx;
1299 	/*
1300 	 * We can have double detach due to exit/hot-unplug + close.
1301 	 */
1302 	if (!(event->attach_state & PERF_ATTACH_CONTEXT))
1303 		return;
1304 
1305 	event->attach_state &= ~PERF_ATTACH_CONTEXT;
1306 
1307 	if (is_cgroup_event(event)) {
1308 		ctx->nr_cgroups--;
1309 		cpuctx = __get_cpu_context(ctx);
1310 		/*
1311 		 * if there are no more cgroup events
1312 		 * then cler cgrp to avoid stale pointer
1313 		 * in update_cgrp_time_from_cpuctx()
1314 		 */
1315 		if (!ctx->nr_cgroups)
1316 			cpuctx->cgrp = NULL;
1317 	}
1318 
1319 	if (has_branch_stack(event))
1320 		ctx->nr_branch_stack--;
1321 
1322 	ctx->nr_events--;
1323 	if (event->attr.inherit_stat)
1324 		ctx->nr_stat--;
1325 
1326 	list_del_rcu(&event->event_entry);
1327 
1328 	if (event->group_leader == event)
1329 		list_del_init(&event->group_entry);
1330 
1331 	update_group_times(event);
1332 
1333 	/*
1334 	 * If event was in error state, then keep it
1335 	 * that way, otherwise bogus counts will be
1336 	 * returned on read(). The only way to get out
1337 	 * of error state is by explicit re-enabling
1338 	 * of the event
1339 	 */
1340 	if (event->state > PERF_EVENT_STATE_OFF)
1341 		event->state = PERF_EVENT_STATE_OFF;
1342 
1343 	ctx->generation++;
1344 }
1345 
1346 static void perf_group_detach(struct perf_event *event)
1347 {
1348 	struct perf_event *sibling, *tmp;
1349 	struct list_head *list = NULL;
1350 
1351 	/*
1352 	 * We can have double detach due to exit/hot-unplug + close.
1353 	 */
1354 	if (!(event->attach_state & PERF_ATTACH_GROUP))
1355 		return;
1356 
1357 	event->attach_state &= ~PERF_ATTACH_GROUP;
1358 
1359 	/*
1360 	 * If this is a sibling, remove it from its group.
1361 	 */
1362 	if (event->group_leader != event) {
1363 		list_del_init(&event->group_entry);
1364 		event->group_leader->nr_siblings--;
1365 		goto out;
1366 	}
1367 
1368 	if (!list_empty(&event->group_entry))
1369 		list = &event->group_entry;
1370 
1371 	/*
1372 	 * If this was a group event with sibling events then
1373 	 * upgrade the siblings to singleton events by adding them
1374 	 * to whatever list we are on.
1375 	 */
1376 	list_for_each_entry_safe(sibling, tmp, &event->sibling_list, group_entry) {
1377 		if (list)
1378 			list_move_tail(&sibling->group_entry, list);
1379 		sibling->group_leader = sibling;
1380 
1381 		/* Inherit group flags from the previous leader */
1382 		sibling->group_flags = event->group_flags;
1383 	}
1384 
1385 out:
1386 	perf_event__header_size(event->group_leader);
1387 
1388 	list_for_each_entry(tmp, &event->group_leader->sibling_list, group_entry)
1389 		perf_event__header_size(tmp);
1390 }
1391 
1392 /*
1393  * User event without the task.
1394  */
1395 static bool is_orphaned_event(struct perf_event *event)
1396 {
1397 	return event && !is_kernel_event(event) && !event->owner;
1398 }
1399 
1400 /*
1401  * Event has a parent but parent's task finished and it's
1402  * alive only because of children holding refference.
1403  */
1404 static bool is_orphaned_child(struct perf_event *event)
1405 {
1406 	return is_orphaned_event(event->parent);
1407 }
1408 
1409 static void orphans_remove_work(struct work_struct *work);
1410 
1411 static void schedule_orphans_remove(struct perf_event_context *ctx)
1412 {
1413 	if (!ctx->task || ctx->orphans_remove_sched || !perf_wq)
1414 		return;
1415 
1416 	if (queue_delayed_work(perf_wq, &ctx->orphans_remove, 1)) {
1417 		get_ctx(ctx);
1418 		ctx->orphans_remove_sched = true;
1419 	}
1420 }
1421 
1422 static int __init perf_workqueue_init(void)
1423 {
1424 	perf_wq = create_singlethread_workqueue("perf");
1425 	WARN(!perf_wq, "failed to create perf workqueue\n");
1426 	return perf_wq ? 0 : -1;
1427 }
1428 
1429 core_initcall(perf_workqueue_init);
1430 
1431 static inline int
1432 event_filter_match(struct perf_event *event)
1433 {
1434 	return (event->cpu == -1 || event->cpu == smp_processor_id())
1435 	    && perf_cgroup_match(event);
1436 }
1437 
1438 static void
1439 event_sched_out(struct perf_event *event,
1440 		  struct perf_cpu_context *cpuctx,
1441 		  struct perf_event_context *ctx)
1442 {
1443 	u64 tstamp = perf_event_time(event);
1444 	u64 delta;
1445 	/*
1446 	 * An event which could not be activated because of
1447 	 * filter mismatch still needs to have its timings
1448 	 * maintained, otherwise bogus information is return
1449 	 * via read() for time_enabled, time_running:
1450 	 */
1451 	if (event->state == PERF_EVENT_STATE_INACTIVE
1452 	    && !event_filter_match(event)) {
1453 		delta = tstamp - event->tstamp_stopped;
1454 		event->tstamp_running += delta;
1455 		event->tstamp_stopped = tstamp;
1456 	}
1457 
1458 	if (event->state != PERF_EVENT_STATE_ACTIVE)
1459 		return;
1460 
1461 	perf_pmu_disable(event->pmu);
1462 
1463 	event->state = PERF_EVENT_STATE_INACTIVE;
1464 	if (event->pending_disable) {
1465 		event->pending_disable = 0;
1466 		event->state = PERF_EVENT_STATE_OFF;
1467 	}
1468 	event->tstamp_stopped = tstamp;
1469 	event->pmu->del(event, 0);
1470 	event->oncpu = -1;
1471 
1472 	if (!is_software_event(event))
1473 		cpuctx->active_oncpu--;
1474 	ctx->nr_active--;
1475 	if (event->attr.freq && event->attr.sample_freq)
1476 		ctx->nr_freq--;
1477 	if (event->attr.exclusive || !cpuctx->active_oncpu)
1478 		cpuctx->exclusive = 0;
1479 
1480 	if (is_orphaned_child(event))
1481 		schedule_orphans_remove(ctx);
1482 
1483 	perf_pmu_enable(event->pmu);
1484 }
1485 
1486 static void
1487 group_sched_out(struct perf_event *group_event,
1488 		struct perf_cpu_context *cpuctx,
1489 		struct perf_event_context *ctx)
1490 {
1491 	struct perf_event *event;
1492 	int state = group_event->state;
1493 
1494 	event_sched_out(group_event, cpuctx, ctx);
1495 
1496 	/*
1497 	 * Schedule out siblings (if any):
1498 	 */
1499 	list_for_each_entry(event, &group_event->sibling_list, group_entry)
1500 		event_sched_out(event, cpuctx, ctx);
1501 
1502 	if (state == PERF_EVENT_STATE_ACTIVE && group_event->attr.exclusive)
1503 		cpuctx->exclusive = 0;
1504 }
1505 
1506 struct remove_event {
1507 	struct perf_event *event;
1508 	bool detach_group;
1509 };
1510 
1511 /*
1512  * Cross CPU call to remove a performance event
1513  *
1514  * We disable the event on the hardware level first. After that we
1515  * remove it from the context list.
1516  */
1517 static int __perf_remove_from_context(void *info)
1518 {
1519 	struct remove_event *re = info;
1520 	struct perf_event *event = re->event;
1521 	struct perf_event_context *ctx = event->ctx;
1522 	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
1523 
1524 	raw_spin_lock(&ctx->lock);
1525 	event_sched_out(event, cpuctx, ctx);
1526 	if (re->detach_group)
1527 		perf_group_detach(event);
1528 	list_del_event(event, ctx);
1529 	if (!ctx->nr_events && cpuctx->task_ctx == ctx) {
1530 		ctx->is_active = 0;
1531 		cpuctx->task_ctx = NULL;
1532 	}
1533 	raw_spin_unlock(&ctx->lock);
1534 
1535 	return 0;
1536 }
1537 
1538 
1539 /*
1540  * Remove the event from a task's (or a CPU's) list of events.
1541  *
1542  * CPU events are removed with a smp call. For task events we only
1543  * call when the task is on a CPU.
1544  *
1545  * If event->ctx is a cloned context, callers must make sure that
1546  * every task struct that event->ctx->task could possibly point to
1547  * remains valid.  This is OK when called from perf_release since
1548  * that only calls us on the top-level context, which can't be a clone.
1549  * When called from perf_event_exit_task, it's OK because the
1550  * context has been detached from its task.
1551  */
1552 static void perf_remove_from_context(struct perf_event *event, bool detach_group)
1553 {
1554 	struct perf_event_context *ctx = event->ctx;
1555 	struct task_struct *task = ctx->task;
1556 	struct remove_event re = {
1557 		.event = event,
1558 		.detach_group = detach_group,
1559 	};
1560 
1561 	lockdep_assert_held(&ctx->mutex);
1562 
1563 	if (!task) {
1564 		/*
1565 		 * Per cpu events are removed via an smp call and
1566 		 * the removal is always successful.
1567 		 */
1568 		cpu_function_call(event->cpu, __perf_remove_from_context, &re);
1569 		return;
1570 	}
1571 
1572 retry:
1573 	if (!task_function_call(task, __perf_remove_from_context, &re))
1574 		return;
1575 
1576 	raw_spin_lock_irq(&ctx->lock);
1577 	/*
1578 	 * If we failed to find a running task, but find the context active now
1579 	 * that we've acquired the ctx->lock, retry.
1580 	 */
1581 	if (ctx->is_active) {
1582 		raw_spin_unlock_irq(&ctx->lock);
1583 		/*
1584 		 * Reload the task pointer, it might have been changed by
1585 		 * a concurrent perf_event_context_sched_out().
1586 		 */
1587 		task = ctx->task;
1588 		goto retry;
1589 	}
1590 
1591 	/*
1592 	 * Since the task isn't running, its safe to remove the event, us
1593 	 * holding the ctx->lock ensures the task won't get scheduled in.
1594 	 */
1595 	if (detach_group)
1596 		perf_group_detach(event);
1597 	list_del_event(event, ctx);
1598 	raw_spin_unlock_irq(&ctx->lock);
1599 }
1600 
1601 /*
1602  * Cross CPU call to disable a performance event
1603  */
1604 int __perf_event_disable(void *info)
1605 {
1606 	struct perf_event *event = info;
1607 	struct perf_event_context *ctx = event->ctx;
1608 	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
1609 
1610 	/*
1611 	 * If this is a per-task event, need to check whether this
1612 	 * event's task is the current task on this cpu.
1613 	 *
1614 	 * Can trigger due to concurrent perf_event_context_sched_out()
1615 	 * flipping contexts around.
1616 	 */
1617 	if (ctx->task && cpuctx->task_ctx != ctx)
1618 		return -EINVAL;
1619 
1620 	raw_spin_lock(&ctx->lock);
1621 
1622 	/*
1623 	 * If the event is on, turn it off.
1624 	 * If it is in error state, leave it in error state.
1625 	 */
1626 	if (event->state >= PERF_EVENT_STATE_INACTIVE) {
1627 		update_context_time(ctx);
1628 		update_cgrp_time_from_event(event);
1629 		update_group_times(event);
1630 		if (event == event->group_leader)
1631 			group_sched_out(event, cpuctx, ctx);
1632 		else
1633 			event_sched_out(event, cpuctx, ctx);
1634 		event->state = PERF_EVENT_STATE_OFF;
1635 	}
1636 
1637 	raw_spin_unlock(&ctx->lock);
1638 
1639 	return 0;
1640 }
1641 
1642 /*
1643  * Disable a event.
1644  *
1645  * If event->ctx is a cloned context, callers must make sure that
1646  * every task struct that event->ctx->task could possibly point to
1647  * remains valid.  This condition is satisifed when called through
1648  * perf_event_for_each_child or perf_event_for_each because they
1649  * hold the top-level event's child_mutex, so any descendant that
1650  * goes to exit will block in sync_child_event.
1651  * When called from perf_pending_event it's OK because event->ctx
1652  * is the current context on this CPU and preemption is disabled,
1653  * hence we can't get into perf_event_task_sched_out for this context.
1654  */
1655 void perf_event_disable(struct perf_event *event)
1656 {
1657 	struct perf_event_context *ctx = event->ctx;
1658 	struct task_struct *task = ctx->task;
1659 
1660 	if (!task) {
1661 		/*
1662 		 * Disable the event on the cpu that it's on
1663 		 */
1664 		cpu_function_call(event->cpu, __perf_event_disable, event);
1665 		return;
1666 	}
1667 
1668 retry:
1669 	if (!task_function_call(task, __perf_event_disable, event))
1670 		return;
1671 
1672 	raw_spin_lock_irq(&ctx->lock);
1673 	/*
1674 	 * If the event is still active, we need to retry the cross-call.
1675 	 */
1676 	if (event->state == PERF_EVENT_STATE_ACTIVE) {
1677 		raw_spin_unlock_irq(&ctx->lock);
1678 		/*
1679 		 * Reload the task pointer, it might have been changed by
1680 		 * a concurrent perf_event_context_sched_out().
1681 		 */
1682 		task = ctx->task;
1683 		goto retry;
1684 	}
1685 
1686 	/*
1687 	 * Since we have the lock this context can't be scheduled
1688 	 * in, so we can change the state safely.
1689 	 */
1690 	if (event->state == PERF_EVENT_STATE_INACTIVE) {
1691 		update_group_times(event);
1692 		event->state = PERF_EVENT_STATE_OFF;
1693 	}
1694 	raw_spin_unlock_irq(&ctx->lock);
1695 }
1696 EXPORT_SYMBOL_GPL(perf_event_disable);
1697 
1698 static void perf_set_shadow_time(struct perf_event *event,
1699 				 struct perf_event_context *ctx,
1700 				 u64 tstamp)
1701 {
1702 	/*
1703 	 * use the correct time source for the time snapshot
1704 	 *
1705 	 * We could get by without this by leveraging the
1706 	 * fact that to get to this function, the caller
1707 	 * has most likely already called update_context_time()
1708 	 * and update_cgrp_time_xx() and thus both timestamp
1709 	 * are identical (or very close). Given that tstamp is,
1710 	 * already adjusted for cgroup, we could say that:
1711 	 *    tstamp - ctx->timestamp
1712 	 * is equivalent to
1713 	 *    tstamp - cgrp->timestamp.
1714 	 *
1715 	 * Then, in perf_output_read(), the calculation would
1716 	 * work with no changes because:
1717 	 * - event is guaranteed scheduled in
1718 	 * - no scheduled out in between
1719 	 * - thus the timestamp would be the same
1720 	 *
1721 	 * But this is a bit hairy.
1722 	 *
1723 	 * So instead, we have an explicit cgroup call to remain
1724 	 * within the time time source all along. We believe it
1725 	 * is cleaner and simpler to understand.
1726 	 */
1727 	if (is_cgroup_event(event))
1728 		perf_cgroup_set_shadow_time(event, tstamp);
1729 	else
1730 		event->shadow_ctx_time = tstamp - ctx->timestamp;
1731 }
1732 
1733 #define MAX_INTERRUPTS (~0ULL)
1734 
1735 static void perf_log_throttle(struct perf_event *event, int enable);
1736 
1737 static int
1738 event_sched_in(struct perf_event *event,
1739 		 struct perf_cpu_context *cpuctx,
1740 		 struct perf_event_context *ctx)
1741 {
1742 	u64 tstamp = perf_event_time(event);
1743 	int ret = 0;
1744 
1745 	lockdep_assert_held(&ctx->lock);
1746 
1747 	if (event->state <= PERF_EVENT_STATE_OFF)
1748 		return 0;
1749 
1750 	event->state = PERF_EVENT_STATE_ACTIVE;
1751 	event->oncpu = smp_processor_id();
1752 
1753 	/*
1754 	 * Unthrottle events, since we scheduled we might have missed several
1755 	 * ticks already, also for a heavily scheduling task there is little
1756 	 * guarantee it'll get a tick in a timely manner.
1757 	 */
1758 	if (unlikely(event->hw.interrupts == MAX_INTERRUPTS)) {
1759 		perf_log_throttle(event, 1);
1760 		event->hw.interrupts = 0;
1761 	}
1762 
1763 	/*
1764 	 * The new state must be visible before we turn it on in the hardware:
1765 	 */
1766 	smp_wmb();
1767 
1768 	perf_pmu_disable(event->pmu);
1769 
1770 	if (event->pmu->add(event, PERF_EF_START)) {
1771 		event->state = PERF_EVENT_STATE_INACTIVE;
1772 		event->oncpu = -1;
1773 		ret = -EAGAIN;
1774 		goto out;
1775 	}
1776 
1777 	event->tstamp_running += tstamp - event->tstamp_stopped;
1778 
1779 	perf_set_shadow_time(event, ctx, tstamp);
1780 
1781 	if (!is_software_event(event))
1782 		cpuctx->active_oncpu++;
1783 	ctx->nr_active++;
1784 	if (event->attr.freq && event->attr.sample_freq)
1785 		ctx->nr_freq++;
1786 
1787 	if (event->attr.exclusive)
1788 		cpuctx->exclusive = 1;
1789 
1790 	if (is_orphaned_child(event))
1791 		schedule_orphans_remove(ctx);
1792 
1793 out:
1794 	perf_pmu_enable(event->pmu);
1795 
1796 	return ret;
1797 }
1798 
1799 static int
1800 group_sched_in(struct perf_event *group_event,
1801 	       struct perf_cpu_context *cpuctx,
1802 	       struct perf_event_context *ctx)
1803 {
1804 	struct perf_event *event, *partial_group = NULL;
1805 	struct pmu *pmu = ctx->pmu;
1806 	u64 now = ctx->time;
1807 	bool simulate = false;
1808 
1809 	if (group_event->state == PERF_EVENT_STATE_OFF)
1810 		return 0;
1811 
1812 	pmu->start_txn(pmu);
1813 
1814 	if (event_sched_in(group_event, cpuctx, ctx)) {
1815 		pmu->cancel_txn(pmu);
1816 		perf_cpu_hrtimer_restart(cpuctx);
1817 		return -EAGAIN;
1818 	}
1819 
1820 	/*
1821 	 * Schedule in siblings as one group (if any):
1822 	 */
1823 	list_for_each_entry(event, &group_event->sibling_list, group_entry) {
1824 		if (event_sched_in(event, cpuctx, ctx)) {
1825 			partial_group = event;
1826 			goto group_error;
1827 		}
1828 	}
1829 
1830 	if (!pmu->commit_txn(pmu))
1831 		return 0;
1832 
1833 group_error:
1834 	/*
1835 	 * Groups can be scheduled in as one unit only, so undo any
1836 	 * partial group before returning:
1837 	 * The events up to the failed event are scheduled out normally,
1838 	 * tstamp_stopped will be updated.
1839 	 *
1840 	 * The failed events and the remaining siblings need to have
1841 	 * their timings updated as if they had gone thru event_sched_in()
1842 	 * and event_sched_out(). This is required to get consistent timings
1843 	 * across the group. This also takes care of the case where the group
1844 	 * could never be scheduled by ensuring tstamp_stopped is set to mark
1845 	 * the time the event was actually stopped, such that time delta
1846 	 * calculation in update_event_times() is correct.
1847 	 */
1848 	list_for_each_entry(event, &group_event->sibling_list, group_entry) {
1849 		if (event == partial_group)
1850 			simulate = true;
1851 
1852 		if (simulate) {
1853 			event->tstamp_running += now - event->tstamp_stopped;
1854 			event->tstamp_stopped = now;
1855 		} else {
1856 			event_sched_out(event, cpuctx, ctx);
1857 		}
1858 	}
1859 	event_sched_out(group_event, cpuctx, ctx);
1860 
1861 	pmu->cancel_txn(pmu);
1862 
1863 	perf_cpu_hrtimer_restart(cpuctx);
1864 
1865 	return -EAGAIN;
1866 }
1867 
1868 /*
1869  * Work out whether we can put this event group on the CPU now.
1870  */
1871 static int group_can_go_on(struct perf_event *event,
1872 			   struct perf_cpu_context *cpuctx,
1873 			   int can_add_hw)
1874 {
1875 	/*
1876 	 * Groups consisting entirely of software events can always go on.
1877 	 */
1878 	if (event->group_flags & PERF_GROUP_SOFTWARE)
1879 		return 1;
1880 	/*
1881 	 * If an exclusive group is already on, no other hardware
1882 	 * events can go on.
1883 	 */
1884 	if (cpuctx->exclusive)
1885 		return 0;
1886 	/*
1887 	 * If this group is exclusive and there are already
1888 	 * events on the CPU, it can't go on.
1889 	 */
1890 	if (event->attr.exclusive && cpuctx->active_oncpu)
1891 		return 0;
1892 	/*
1893 	 * Otherwise, try to add it if all previous groups were able
1894 	 * to go on.
1895 	 */
1896 	return can_add_hw;
1897 }
1898 
1899 static void add_event_to_ctx(struct perf_event *event,
1900 			       struct perf_event_context *ctx)
1901 {
1902 	u64 tstamp = perf_event_time(event);
1903 
1904 	list_add_event(event, ctx);
1905 	perf_group_attach(event);
1906 	event->tstamp_enabled = tstamp;
1907 	event->tstamp_running = tstamp;
1908 	event->tstamp_stopped = tstamp;
1909 }
1910 
1911 static void task_ctx_sched_out(struct perf_event_context *ctx);
1912 static void
1913 ctx_sched_in(struct perf_event_context *ctx,
1914 	     struct perf_cpu_context *cpuctx,
1915 	     enum event_type_t event_type,
1916 	     struct task_struct *task);
1917 
1918 static void perf_event_sched_in(struct perf_cpu_context *cpuctx,
1919 				struct perf_event_context *ctx,
1920 				struct task_struct *task)
1921 {
1922 	cpu_ctx_sched_in(cpuctx, EVENT_PINNED, task);
1923 	if (ctx)
1924 		ctx_sched_in(ctx, cpuctx, EVENT_PINNED, task);
1925 	cpu_ctx_sched_in(cpuctx, EVENT_FLEXIBLE, task);
1926 	if (ctx)
1927 		ctx_sched_in(ctx, cpuctx, EVENT_FLEXIBLE, task);
1928 }
1929 
1930 /*
1931  * Cross CPU call to install and enable a performance event
1932  *
1933  * Must be called with ctx->mutex held
1934  */
1935 static int  __perf_install_in_context(void *info)
1936 {
1937 	struct perf_event *event = info;
1938 	struct perf_event_context *ctx = event->ctx;
1939 	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
1940 	struct perf_event_context *task_ctx = cpuctx->task_ctx;
1941 	struct task_struct *task = current;
1942 
1943 	perf_ctx_lock(cpuctx, task_ctx);
1944 	perf_pmu_disable(cpuctx->ctx.pmu);
1945 
1946 	/*
1947 	 * If there was an active task_ctx schedule it out.
1948 	 */
1949 	if (task_ctx)
1950 		task_ctx_sched_out(task_ctx);
1951 
1952 	/*
1953 	 * If the context we're installing events in is not the
1954 	 * active task_ctx, flip them.
1955 	 */
1956 	if (ctx->task && task_ctx != ctx) {
1957 		if (task_ctx)
1958 			raw_spin_unlock(&task_ctx->lock);
1959 		raw_spin_lock(&ctx->lock);
1960 		task_ctx = ctx;
1961 	}
1962 
1963 	if (task_ctx) {
1964 		cpuctx->task_ctx = task_ctx;
1965 		task = task_ctx->task;
1966 	}
1967 
1968 	cpu_ctx_sched_out(cpuctx, EVENT_ALL);
1969 
1970 	update_context_time(ctx);
1971 	/*
1972 	 * update cgrp time only if current cgrp
1973 	 * matches event->cgrp. Must be done before
1974 	 * calling add_event_to_ctx()
1975 	 */
1976 	update_cgrp_time_from_event(event);
1977 
1978 	add_event_to_ctx(event, ctx);
1979 
1980 	/*
1981 	 * Schedule everything back in
1982 	 */
1983 	perf_event_sched_in(cpuctx, task_ctx, task);
1984 
1985 	perf_pmu_enable(cpuctx->ctx.pmu);
1986 	perf_ctx_unlock(cpuctx, task_ctx);
1987 
1988 	return 0;
1989 }
1990 
1991 /*
1992  * Attach a performance event to a context
1993  *
1994  * First we add the event to the list with the hardware enable bit
1995  * in event->hw_config cleared.
1996  *
1997  * If the event is attached to a task which is on a CPU we use a smp
1998  * call to enable it in the task context. The task might have been
1999  * scheduled away, but we check this in the smp call again.
2000  */
2001 static void
2002 perf_install_in_context(struct perf_event_context *ctx,
2003 			struct perf_event *event,
2004 			int cpu)
2005 {
2006 	struct task_struct *task = ctx->task;
2007 
2008 	lockdep_assert_held(&ctx->mutex);
2009 
2010 	event->ctx = ctx;
2011 	if (event->cpu != -1)
2012 		event->cpu = cpu;
2013 
2014 	if (!task) {
2015 		/*
2016 		 * Per cpu events are installed via an smp call and
2017 		 * the install is always successful.
2018 		 */
2019 		cpu_function_call(cpu, __perf_install_in_context, event);
2020 		return;
2021 	}
2022 
2023 retry:
2024 	if (!task_function_call(task, __perf_install_in_context, event))
2025 		return;
2026 
2027 	raw_spin_lock_irq(&ctx->lock);
2028 	/*
2029 	 * If we failed to find a running task, but find the context active now
2030 	 * that we've acquired the ctx->lock, retry.
2031 	 */
2032 	if (ctx->is_active) {
2033 		raw_spin_unlock_irq(&ctx->lock);
2034 		/*
2035 		 * Reload the task pointer, it might have been changed by
2036 		 * a concurrent perf_event_context_sched_out().
2037 		 */
2038 		task = ctx->task;
2039 		goto retry;
2040 	}
2041 
2042 	/*
2043 	 * Since the task isn't running, its safe to add the event, us holding
2044 	 * the ctx->lock ensures the task won't get scheduled in.
2045 	 */
2046 	add_event_to_ctx(event, ctx);
2047 	raw_spin_unlock_irq(&ctx->lock);
2048 }
2049 
2050 /*
2051  * Put a event into inactive state and update time fields.
2052  * Enabling the leader of a group effectively enables all
2053  * the group members that aren't explicitly disabled, so we
2054  * have to update their ->tstamp_enabled also.
2055  * Note: this works for group members as well as group leaders
2056  * since the non-leader members' sibling_lists will be empty.
2057  */
2058 static void __perf_event_mark_enabled(struct perf_event *event)
2059 {
2060 	struct perf_event *sub;
2061 	u64 tstamp = perf_event_time(event);
2062 
2063 	event->state = PERF_EVENT_STATE_INACTIVE;
2064 	event->tstamp_enabled = tstamp - event->total_time_enabled;
2065 	list_for_each_entry(sub, &event->sibling_list, group_entry) {
2066 		if (sub->state >= PERF_EVENT_STATE_INACTIVE)
2067 			sub->tstamp_enabled = tstamp - sub->total_time_enabled;
2068 	}
2069 }
2070 
2071 /*
2072  * Cross CPU call to enable a performance event
2073  */
2074 static int __perf_event_enable(void *info)
2075 {
2076 	struct perf_event *event = info;
2077 	struct perf_event_context *ctx = event->ctx;
2078 	struct perf_event *leader = event->group_leader;
2079 	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
2080 	int err;
2081 
2082 	/*
2083 	 * There's a time window between 'ctx->is_active' check
2084 	 * in perf_event_enable function and this place having:
2085 	 *   - IRQs on
2086 	 *   - ctx->lock unlocked
2087 	 *
2088 	 * where the task could be killed and 'ctx' deactivated
2089 	 * by perf_event_exit_task.
2090 	 */
2091 	if (!ctx->is_active)
2092 		return -EINVAL;
2093 
2094 	raw_spin_lock(&ctx->lock);
2095 	update_context_time(ctx);
2096 
2097 	if (event->state >= PERF_EVENT_STATE_INACTIVE)
2098 		goto unlock;
2099 
2100 	/*
2101 	 * set current task's cgroup time reference point
2102 	 */
2103 	perf_cgroup_set_timestamp(current, ctx);
2104 
2105 	__perf_event_mark_enabled(event);
2106 
2107 	if (!event_filter_match(event)) {
2108 		if (is_cgroup_event(event))
2109 			perf_cgroup_defer_enabled(event);
2110 		goto unlock;
2111 	}
2112 
2113 	/*
2114 	 * If the event is in a group and isn't the group leader,
2115 	 * then don't put it on unless the group is on.
2116 	 */
2117 	if (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE)
2118 		goto unlock;
2119 
2120 	if (!group_can_go_on(event, cpuctx, 1)) {
2121 		err = -EEXIST;
2122 	} else {
2123 		if (event == leader)
2124 			err = group_sched_in(event, cpuctx, ctx);
2125 		else
2126 			err = event_sched_in(event, cpuctx, ctx);
2127 	}
2128 
2129 	if (err) {
2130 		/*
2131 		 * If this event can't go on and it's part of a
2132 		 * group, then the whole group has to come off.
2133 		 */
2134 		if (leader != event) {
2135 			group_sched_out(leader, cpuctx, ctx);
2136 			perf_cpu_hrtimer_restart(cpuctx);
2137 		}
2138 		if (leader->attr.pinned) {
2139 			update_group_times(leader);
2140 			leader->state = PERF_EVENT_STATE_ERROR;
2141 		}
2142 	}
2143 
2144 unlock:
2145 	raw_spin_unlock(&ctx->lock);
2146 
2147 	return 0;
2148 }
2149 
2150 /*
2151  * Enable a event.
2152  *
2153  * If event->ctx is a cloned context, callers must make sure that
2154  * every task struct that event->ctx->task could possibly point to
2155  * remains valid.  This condition is satisfied when called through
2156  * perf_event_for_each_child or perf_event_for_each as described
2157  * for perf_event_disable.
2158  */
2159 void perf_event_enable(struct perf_event *event)
2160 {
2161 	struct perf_event_context *ctx = event->ctx;
2162 	struct task_struct *task = ctx->task;
2163 
2164 	if (!task) {
2165 		/*
2166 		 * Enable the event on the cpu that it's on
2167 		 */
2168 		cpu_function_call(event->cpu, __perf_event_enable, event);
2169 		return;
2170 	}
2171 
2172 	raw_spin_lock_irq(&ctx->lock);
2173 	if (event->state >= PERF_EVENT_STATE_INACTIVE)
2174 		goto out;
2175 
2176 	/*
2177 	 * If the event is in error state, clear that first.
2178 	 * That way, if we see the event in error state below, we
2179 	 * know that it has gone back into error state, as distinct
2180 	 * from the task having been scheduled away before the
2181 	 * cross-call arrived.
2182 	 */
2183 	if (event->state == PERF_EVENT_STATE_ERROR)
2184 		event->state = PERF_EVENT_STATE_OFF;
2185 
2186 retry:
2187 	if (!ctx->is_active) {
2188 		__perf_event_mark_enabled(event);
2189 		goto out;
2190 	}
2191 
2192 	raw_spin_unlock_irq(&ctx->lock);
2193 
2194 	if (!task_function_call(task, __perf_event_enable, event))
2195 		return;
2196 
2197 	raw_spin_lock_irq(&ctx->lock);
2198 
2199 	/*
2200 	 * If the context is active and the event is still off,
2201 	 * we need to retry the cross-call.
2202 	 */
2203 	if (ctx->is_active && event->state == PERF_EVENT_STATE_OFF) {
2204 		/*
2205 		 * task could have been flipped by a concurrent
2206 		 * perf_event_context_sched_out()
2207 		 */
2208 		task = ctx->task;
2209 		goto retry;
2210 	}
2211 
2212 out:
2213 	raw_spin_unlock_irq(&ctx->lock);
2214 }
2215 EXPORT_SYMBOL_GPL(perf_event_enable);
2216 
2217 int perf_event_refresh(struct perf_event *event, int refresh)
2218 {
2219 	/*
2220 	 * not supported on inherited events
2221 	 */
2222 	if (event->attr.inherit || !is_sampling_event(event))
2223 		return -EINVAL;
2224 
2225 	atomic_add(refresh, &event->event_limit);
2226 	perf_event_enable(event);
2227 
2228 	return 0;
2229 }
2230 EXPORT_SYMBOL_GPL(perf_event_refresh);
2231 
2232 static void ctx_sched_out(struct perf_event_context *ctx,
2233 			  struct perf_cpu_context *cpuctx,
2234 			  enum event_type_t event_type)
2235 {
2236 	struct perf_event *event;
2237 	int is_active = ctx->is_active;
2238 
2239 	ctx->is_active &= ~event_type;
2240 	if (likely(!ctx->nr_events))
2241 		return;
2242 
2243 	update_context_time(ctx);
2244 	update_cgrp_time_from_cpuctx(cpuctx);
2245 	if (!ctx->nr_active)
2246 		return;
2247 
2248 	perf_pmu_disable(ctx->pmu);
2249 	if ((is_active & EVENT_PINNED) && (event_type & EVENT_PINNED)) {
2250 		list_for_each_entry(event, &ctx->pinned_groups, group_entry)
2251 			group_sched_out(event, cpuctx, ctx);
2252 	}
2253 
2254 	if ((is_active & EVENT_FLEXIBLE) && (event_type & EVENT_FLEXIBLE)) {
2255 		list_for_each_entry(event, &ctx->flexible_groups, group_entry)
2256 			group_sched_out(event, cpuctx, ctx);
2257 	}
2258 	perf_pmu_enable(ctx->pmu);
2259 }
2260 
2261 /*
2262  * Test whether two contexts are equivalent, i.e. whether they have both been
2263  * cloned from the same version of the same context.
2264  *
2265  * Equivalence is measured using a generation number in the context that is
2266  * incremented on each modification to it; see unclone_ctx(), list_add_event()
2267  * and list_del_event().
2268  */
2269 static int context_equiv(struct perf_event_context *ctx1,
2270 			 struct perf_event_context *ctx2)
2271 {
2272 	lockdep_assert_held(&ctx1->lock);
2273 	lockdep_assert_held(&ctx2->lock);
2274 
2275 	/* Pinning disables the swap optimization */
2276 	if (ctx1->pin_count || ctx2->pin_count)
2277 		return 0;
2278 
2279 	/* If ctx1 is the parent of ctx2 */
2280 	if (ctx1 == ctx2->parent_ctx && ctx1->generation == ctx2->parent_gen)
2281 		return 1;
2282 
2283 	/* If ctx2 is the parent of ctx1 */
2284 	if (ctx1->parent_ctx == ctx2 && ctx1->parent_gen == ctx2->generation)
2285 		return 1;
2286 
2287 	/*
2288 	 * If ctx1 and ctx2 have the same parent; we flatten the parent
2289 	 * hierarchy, see perf_event_init_context().
2290 	 */
2291 	if (ctx1->parent_ctx && ctx1->parent_ctx == ctx2->parent_ctx &&
2292 			ctx1->parent_gen == ctx2->parent_gen)
2293 		return 1;
2294 
2295 	/* Unmatched */
2296 	return 0;
2297 }
2298 
2299 static void __perf_event_sync_stat(struct perf_event *event,
2300 				     struct perf_event *next_event)
2301 {
2302 	u64 value;
2303 
2304 	if (!event->attr.inherit_stat)
2305 		return;
2306 
2307 	/*
2308 	 * Update the event value, we cannot use perf_event_read()
2309 	 * because we're in the middle of a context switch and have IRQs
2310 	 * disabled, which upsets smp_call_function_single(), however
2311 	 * we know the event must be on the current CPU, therefore we
2312 	 * don't need to use it.
2313 	 */
2314 	switch (event->state) {
2315 	case PERF_EVENT_STATE_ACTIVE:
2316 		event->pmu->read(event);
2317 		/* fall-through */
2318 
2319 	case PERF_EVENT_STATE_INACTIVE:
2320 		update_event_times(event);
2321 		break;
2322 
2323 	default:
2324 		break;
2325 	}
2326 
2327 	/*
2328 	 * In order to keep per-task stats reliable we need to flip the event
2329 	 * values when we flip the contexts.
2330 	 */
2331 	value = local64_read(&next_event->count);
2332 	value = local64_xchg(&event->count, value);
2333 	local64_set(&next_event->count, value);
2334 
2335 	swap(event->total_time_enabled, next_event->total_time_enabled);
2336 	swap(event->total_time_running, next_event->total_time_running);
2337 
2338 	/*
2339 	 * Since we swizzled the values, update the user visible data too.
2340 	 */
2341 	perf_event_update_userpage(event);
2342 	perf_event_update_userpage(next_event);
2343 }
2344 
2345 static void perf_event_sync_stat(struct perf_event_context *ctx,
2346 				   struct perf_event_context *next_ctx)
2347 {
2348 	struct perf_event *event, *next_event;
2349 
2350 	if (!ctx->nr_stat)
2351 		return;
2352 
2353 	update_context_time(ctx);
2354 
2355 	event = list_first_entry(&ctx->event_list,
2356 				   struct perf_event, event_entry);
2357 
2358 	next_event = list_first_entry(&next_ctx->event_list,
2359 					struct perf_event, event_entry);
2360 
2361 	while (&event->event_entry != &ctx->event_list &&
2362 	       &next_event->event_entry != &next_ctx->event_list) {
2363 
2364 		__perf_event_sync_stat(event, next_event);
2365 
2366 		event = list_next_entry(event, event_entry);
2367 		next_event = list_next_entry(next_event, event_entry);
2368 	}
2369 }
2370 
2371 static void perf_event_context_sched_out(struct task_struct *task, int ctxn,
2372 					 struct task_struct *next)
2373 {
2374 	struct perf_event_context *ctx = task->perf_event_ctxp[ctxn];
2375 	struct perf_event_context *next_ctx;
2376 	struct perf_event_context *parent, *next_parent;
2377 	struct perf_cpu_context *cpuctx;
2378 	int do_switch = 1;
2379 
2380 	if (likely(!ctx))
2381 		return;
2382 
2383 	cpuctx = __get_cpu_context(ctx);
2384 	if (!cpuctx->task_ctx)
2385 		return;
2386 
2387 	rcu_read_lock();
2388 	next_ctx = next->perf_event_ctxp[ctxn];
2389 	if (!next_ctx)
2390 		goto unlock;
2391 
2392 	parent = rcu_dereference(ctx->parent_ctx);
2393 	next_parent = rcu_dereference(next_ctx->parent_ctx);
2394 
2395 	/* If neither context have a parent context; they cannot be clones. */
2396 	if (!parent && !next_parent)
2397 		goto unlock;
2398 
2399 	if (next_parent == ctx || next_ctx == parent || next_parent == parent) {
2400 		/*
2401 		 * Looks like the two contexts are clones, so we might be
2402 		 * able to optimize the context switch.  We lock both
2403 		 * contexts and check that they are clones under the
2404 		 * lock (including re-checking that neither has been
2405 		 * uncloned in the meantime).  It doesn't matter which
2406 		 * order we take the locks because no other cpu could
2407 		 * be trying to lock both of these tasks.
2408 		 */
2409 		raw_spin_lock(&ctx->lock);
2410 		raw_spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING);
2411 		if (context_equiv(ctx, next_ctx)) {
2412 			/*
2413 			 * XXX do we need a memory barrier of sorts
2414 			 * wrt to rcu_dereference() of perf_event_ctxp
2415 			 */
2416 			task->perf_event_ctxp[ctxn] = next_ctx;
2417 			next->perf_event_ctxp[ctxn] = ctx;
2418 			ctx->task = next;
2419 			next_ctx->task = task;
2420 			do_switch = 0;
2421 
2422 			perf_event_sync_stat(ctx, next_ctx);
2423 		}
2424 		raw_spin_unlock(&next_ctx->lock);
2425 		raw_spin_unlock(&ctx->lock);
2426 	}
2427 unlock:
2428 	rcu_read_unlock();
2429 
2430 	if (do_switch) {
2431 		raw_spin_lock(&ctx->lock);
2432 		ctx_sched_out(ctx, cpuctx, EVENT_ALL);
2433 		cpuctx->task_ctx = NULL;
2434 		raw_spin_unlock(&ctx->lock);
2435 	}
2436 }
2437 
2438 #define for_each_task_context_nr(ctxn)					\
2439 	for ((ctxn) = 0; (ctxn) < perf_nr_task_contexts; (ctxn)++)
2440 
2441 /*
2442  * Called from scheduler to remove the events of the current task,
2443  * with interrupts disabled.
2444  *
2445  * We stop each event and update the event value in event->count.
2446  *
2447  * This does not protect us against NMI, but disable()
2448  * sets the disabled bit in the control field of event _before_
2449  * accessing the event control register. If a NMI hits, then it will
2450  * not restart the event.
2451  */
2452 void __perf_event_task_sched_out(struct task_struct *task,
2453 				 struct task_struct *next)
2454 {
2455 	int ctxn;
2456 
2457 	for_each_task_context_nr(ctxn)
2458 		perf_event_context_sched_out(task, ctxn, next);
2459 
2460 	/*
2461 	 * if cgroup events exist on this CPU, then we need
2462 	 * to check if we have to switch out PMU state.
2463 	 * cgroup event are system-wide mode only
2464 	 */
2465 	if (atomic_read(this_cpu_ptr(&perf_cgroup_events)))
2466 		perf_cgroup_sched_out(task, next);
2467 }
2468 
2469 static void task_ctx_sched_out(struct perf_event_context *ctx)
2470 {
2471 	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
2472 
2473 	if (!cpuctx->task_ctx)
2474 		return;
2475 
2476 	if (WARN_ON_ONCE(ctx != cpuctx->task_ctx))
2477 		return;
2478 
2479 	ctx_sched_out(ctx, cpuctx, EVENT_ALL);
2480 	cpuctx->task_ctx = NULL;
2481 }
2482 
2483 /*
2484  * Called with IRQs disabled
2485  */
2486 static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,
2487 			      enum event_type_t event_type)
2488 {
2489 	ctx_sched_out(&cpuctx->ctx, cpuctx, event_type);
2490 }
2491 
2492 static void
2493 ctx_pinned_sched_in(struct perf_event_context *ctx,
2494 		    struct perf_cpu_context *cpuctx)
2495 {
2496 	struct perf_event *event;
2497 
2498 	list_for_each_entry(event, &ctx->pinned_groups, group_entry) {
2499 		if (event->state <= PERF_EVENT_STATE_OFF)
2500 			continue;
2501 		if (!event_filter_match(event))
2502 			continue;
2503 
2504 		/* may need to reset tstamp_enabled */
2505 		if (is_cgroup_event(event))
2506 			perf_cgroup_mark_enabled(event, ctx);
2507 
2508 		if (group_can_go_on(event, cpuctx, 1))
2509 			group_sched_in(event, cpuctx, ctx);
2510 
2511 		/*
2512 		 * If this pinned group hasn't been scheduled,
2513 		 * put it in error state.
2514 		 */
2515 		if (event->state == PERF_EVENT_STATE_INACTIVE) {
2516 			update_group_times(event);
2517 			event->state = PERF_EVENT_STATE_ERROR;
2518 		}
2519 	}
2520 }
2521 
2522 static void
2523 ctx_flexible_sched_in(struct perf_event_context *ctx,
2524 		      struct perf_cpu_context *cpuctx)
2525 {
2526 	struct perf_event *event;
2527 	int can_add_hw = 1;
2528 
2529 	list_for_each_entry(event, &ctx->flexible_groups, group_entry) {
2530 		/* Ignore events in OFF or ERROR state */
2531 		if (event->state <= PERF_EVENT_STATE_OFF)
2532 			continue;
2533 		/*
2534 		 * Listen to the 'cpu' scheduling filter constraint
2535 		 * of events:
2536 		 */
2537 		if (!event_filter_match(event))
2538 			continue;
2539 
2540 		/* may need to reset tstamp_enabled */
2541 		if (is_cgroup_event(event))
2542 			perf_cgroup_mark_enabled(event, ctx);
2543 
2544 		if (group_can_go_on(event, cpuctx, can_add_hw)) {
2545 			if (group_sched_in(event, cpuctx, ctx))
2546 				can_add_hw = 0;
2547 		}
2548 	}
2549 }
2550 
2551 static void
2552 ctx_sched_in(struct perf_event_context *ctx,
2553 	     struct perf_cpu_context *cpuctx,
2554 	     enum event_type_t event_type,
2555 	     struct task_struct *task)
2556 {
2557 	u64 now;
2558 	int is_active = ctx->is_active;
2559 
2560 	ctx->is_active |= event_type;
2561 	if (likely(!ctx->nr_events))
2562 		return;
2563 
2564 	now = perf_clock();
2565 	ctx->timestamp = now;
2566 	perf_cgroup_set_timestamp(task, ctx);
2567 	/*
2568 	 * First go through the list and put on any pinned groups
2569 	 * in order to give them the best chance of going on.
2570 	 */
2571 	if (!(is_active & EVENT_PINNED) && (event_type & EVENT_PINNED))
2572 		ctx_pinned_sched_in(ctx, cpuctx);
2573 
2574 	/* Then walk through the lower prio flexible groups */
2575 	if (!(is_active & EVENT_FLEXIBLE) && (event_type & EVENT_FLEXIBLE))
2576 		ctx_flexible_sched_in(ctx, cpuctx);
2577 }
2578 
2579 static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx,
2580 			     enum event_type_t event_type,
2581 			     struct task_struct *task)
2582 {
2583 	struct perf_event_context *ctx = &cpuctx->ctx;
2584 
2585 	ctx_sched_in(ctx, cpuctx, event_type, task);
2586 }
2587 
2588 static void perf_event_context_sched_in(struct perf_event_context *ctx,
2589 					struct task_struct *task)
2590 {
2591 	struct perf_cpu_context *cpuctx;
2592 
2593 	cpuctx = __get_cpu_context(ctx);
2594 	if (cpuctx->task_ctx == ctx)
2595 		return;
2596 
2597 	perf_ctx_lock(cpuctx, ctx);
2598 	perf_pmu_disable(ctx->pmu);
2599 	/*
2600 	 * We want to keep the following priority order:
2601 	 * cpu pinned (that don't need to move), task pinned,
2602 	 * cpu flexible, task flexible.
2603 	 */
2604 	cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);
2605 
2606 	if (ctx->nr_events)
2607 		cpuctx->task_ctx = ctx;
2608 
2609 	perf_event_sched_in(cpuctx, cpuctx->task_ctx, task);
2610 
2611 	perf_pmu_enable(ctx->pmu);
2612 	perf_ctx_unlock(cpuctx, ctx);
2613 
2614 	/*
2615 	 * Since these rotations are per-cpu, we need to ensure the
2616 	 * cpu-context we got scheduled on is actually rotating.
2617 	 */
2618 	perf_pmu_rotate_start(ctx->pmu);
2619 }
2620 
2621 /*
2622  * When sampling the branck stack in system-wide, it may be necessary
2623  * to flush the stack on context switch. This happens when the branch
2624  * stack does not tag its entries with the pid of the current task.
2625  * Otherwise it becomes impossible to associate a branch entry with a
2626  * task. This ambiguity is more likely to appear when the branch stack
2627  * supports priv level filtering and the user sets it to monitor only
2628  * at the user level (which could be a useful measurement in system-wide
2629  * mode). In that case, the risk is high of having a branch stack with
2630  * branch from multiple tasks. Flushing may mean dropping the existing
2631  * entries or stashing them somewhere in the PMU specific code layer.
2632  *
2633  * This function provides the context switch callback to the lower code
2634  * layer. It is invoked ONLY when there is at least one system-wide context
2635  * with at least one active event using taken branch sampling.
2636  */
2637 static void perf_branch_stack_sched_in(struct task_struct *prev,
2638 				       struct task_struct *task)
2639 {
2640 	struct perf_cpu_context *cpuctx;
2641 	struct pmu *pmu;
2642 	unsigned long flags;
2643 
2644 	/* no need to flush branch stack if not changing task */
2645 	if (prev == task)
2646 		return;
2647 
2648 	local_irq_save(flags);
2649 
2650 	rcu_read_lock();
2651 
2652 	list_for_each_entry_rcu(pmu, &pmus, entry) {
2653 		cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
2654 
2655 		/*
2656 		 * check if the context has at least one
2657 		 * event using PERF_SAMPLE_BRANCH_STACK
2658 		 */
2659 		if (cpuctx->ctx.nr_branch_stack > 0
2660 		    && pmu->flush_branch_stack) {
2661 
2662 			perf_ctx_lock(cpuctx, cpuctx->task_ctx);
2663 
2664 			perf_pmu_disable(pmu);
2665 
2666 			pmu->flush_branch_stack();
2667 
2668 			perf_pmu_enable(pmu);
2669 
2670 			perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
2671 		}
2672 	}
2673 
2674 	rcu_read_unlock();
2675 
2676 	local_irq_restore(flags);
2677 }
2678 
2679 /*
2680  * Called from scheduler to add the events of the current task
2681  * with interrupts disabled.
2682  *
2683  * We restore the event value and then enable it.
2684  *
2685  * This does not protect us against NMI, but enable()
2686  * sets the enabled bit in the control field of event _before_
2687  * accessing the event control register. If a NMI hits, then it will
2688  * keep the event running.
2689  */
2690 void __perf_event_task_sched_in(struct task_struct *prev,
2691 				struct task_struct *task)
2692 {
2693 	struct perf_event_context *ctx;
2694 	int ctxn;
2695 
2696 	for_each_task_context_nr(ctxn) {
2697 		ctx = task->perf_event_ctxp[ctxn];
2698 		if (likely(!ctx))
2699 			continue;
2700 
2701 		perf_event_context_sched_in(ctx, task);
2702 	}
2703 	/*
2704 	 * if cgroup events exist on this CPU, then we need
2705 	 * to check if we have to switch in PMU state.
2706 	 * cgroup event are system-wide mode only
2707 	 */
2708 	if (atomic_read(this_cpu_ptr(&perf_cgroup_events)))
2709 		perf_cgroup_sched_in(prev, task);
2710 
2711 	/* check for system-wide branch_stack events */
2712 	if (atomic_read(this_cpu_ptr(&perf_branch_stack_events)))
2713 		perf_branch_stack_sched_in(prev, task);
2714 }
2715 
2716 static u64 perf_calculate_period(struct perf_event *event, u64 nsec, u64 count)
2717 {
2718 	u64 frequency = event->attr.sample_freq;
2719 	u64 sec = NSEC_PER_SEC;
2720 	u64 divisor, dividend;
2721 
2722 	int count_fls, nsec_fls, frequency_fls, sec_fls;
2723 
2724 	count_fls = fls64(count);
2725 	nsec_fls = fls64(nsec);
2726 	frequency_fls = fls64(frequency);
2727 	sec_fls = 30;
2728 
2729 	/*
2730 	 * We got @count in @nsec, with a target of sample_freq HZ
2731 	 * the target period becomes:
2732 	 *
2733 	 *             @count * 10^9
2734 	 * period = -------------------
2735 	 *          @nsec * sample_freq
2736 	 *
2737 	 */
2738 
2739 	/*
2740 	 * Reduce accuracy by one bit such that @a and @b converge
2741 	 * to a similar magnitude.
2742 	 */
2743 #define REDUCE_FLS(a, b)		\
2744 do {					\
2745 	if (a##_fls > b##_fls) {	\
2746 		a >>= 1;		\
2747 		a##_fls--;		\
2748 	} else {			\
2749 		b >>= 1;		\
2750 		b##_fls--;		\
2751 	}				\
2752 } while (0)
2753 
2754 	/*
2755 	 * Reduce accuracy until either term fits in a u64, then proceed with
2756 	 * the other, so that finally we can do a u64/u64 division.
2757 	 */
2758 	while (count_fls + sec_fls > 64 && nsec_fls + frequency_fls > 64) {
2759 		REDUCE_FLS(nsec, frequency);
2760 		REDUCE_FLS(sec, count);
2761 	}
2762 
2763 	if (count_fls + sec_fls > 64) {
2764 		divisor = nsec * frequency;
2765 
2766 		while (count_fls + sec_fls > 64) {
2767 			REDUCE_FLS(count, sec);
2768 			divisor >>= 1;
2769 		}
2770 
2771 		dividend = count * sec;
2772 	} else {
2773 		dividend = count * sec;
2774 
2775 		while (nsec_fls + frequency_fls > 64) {
2776 			REDUCE_FLS(nsec, frequency);
2777 			dividend >>= 1;
2778 		}
2779 
2780 		divisor = nsec * frequency;
2781 	}
2782 
2783 	if (!divisor)
2784 		return dividend;
2785 
2786 	return div64_u64(dividend, divisor);
2787 }
2788 
2789 static DEFINE_PER_CPU(int, perf_throttled_count);
2790 static DEFINE_PER_CPU(u64, perf_throttled_seq);
2791 
2792 static void perf_adjust_period(struct perf_event *event, u64 nsec, u64 count, bool disable)
2793 {
2794 	struct hw_perf_event *hwc = &event->hw;
2795 	s64 period, sample_period;
2796 	s64 delta;
2797 
2798 	period = perf_calculate_period(event, nsec, count);
2799 
2800 	delta = (s64)(period - hwc->sample_period);
2801 	delta = (delta + 7) / 8; /* low pass filter */
2802 
2803 	sample_period = hwc->sample_period + delta;
2804 
2805 	if (!sample_period)
2806 		sample_period = 1;
2807 
2808 	hwc->sample_period = sample_period;
2809 
2810 	if (local64_read(&hwc->period_left) > 8*sample_period) {
2811 		if (disable)
2812 			event->pmu->stop(event, PERF_EF_UPDATE);
2813 
2814 		local64_set(&hwc->period_left, 0);
2815 
2816 		if (disable)
2817 			event->pmu->start(event, PERF_EF_RELOAD);
2818 	}
2819 }
2820 
2821 /*
2822  * combine freq adjustment with unthrottling to avoid two passes over the
2823  * events. At the same time, make sure, having freq events does not change
2824  * the rate of unthrottling as that would introduce bias.
2825  */
2826 static void perf_adjust_freq_unthr_context(struct perf_event_context *ctx,
2827 					   int needs_unthr)
2828 {
2829 	struct perf_event *event;
2830 	struct hw_perf_event *hwc;
2831 	u64 now, period = TICK_NSEC;
2832 	s64 delta;
2833 
2834 	/*
2835 	 * only need to iterate over all events iff:
2836 	 * - context have events in frequency mode (needs freq adjust)
2837 	 * - there are events to unthrottle on this cpu
2838 	 */
2839 	if (!(ctx->nr_freq || needs_unthr))
2840 		return;
2841 
2842 	raw_spin_lock(&ctx->lock);
2843 	perf_pmu_disable(ctx->pmu);
2844 
2845 	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
2846 		if (event->state != PERF_EVENT_STATE_ACTIVE)
2847 			continue;
2848 
2849 		if (!event_filter_match(event))
2850 			continue;
2851 
2852 		perf_pmu_disable(event->pmu);
2853 
2854 		hwc = &event->hw;
2855 
2856 		if (hwc->interrupts == MAX_INTERRUPTS) {
2857 			hwc->interrupts = 0;
2858 			perf_log_throttle(event, 1);
2859 			event->pmu->start(event, 0);
2860 		}
2861 
2862 		if (!event->attr.freq || !event->attr.sample_freq)
2863 			goto next;
2864 
2865 		/*
2866 		 * stop the event and update event->count
2867 		 */
2868 		event->pmu->stop(event, PERF_EF_UPDATE);
2869 
2870 		now = local64_read(&event->count);
2871 		delta = now - hwc->freq_count_stamp;
2872 		hwc->freq_count_stamp = now;
2873 
2874 		/*
2875 		 * restart the event
2876 		 * reload only if value has changed
2877 		 * we have stopped the event so tell that
2878 		 * to perf_adjust_period() to avoid stopping it
2879 		 * twice.
2880 		 */
2881 		if (delta > 0)
2882 			perf_adjust_period(event, period, delta, false);
2883 
2884 		event->pmu->start(event, delta > 0 ? PERF_EF_RELOAD : 0);
2885 	next:
2886 		perf_pmu_enable(event->pmu);
2887 	}
2888 
2889 	perf_pmu_enable(ctx->pmu);
2890 	raw_spin_unlock(&ctx->lock);
2891 }
2892 
2893 /*
2894  * Round-robin a context's events:
2895  */
2896 static void rotate_ctx(struct perf_event_context *ctx)
2897 {
2898 	/*
2899 	 * Rotate the first entry last of non-pinned groups. Rotation might be
2900 	 * disabled by the inheritance code.
2901 	 */
2902 	if (!ctx->rotate_disable)
2903 		list_rotate_left(&ctx->flexible_groups);
2904 }
2905 
2906 /*
2907  * perf_pmu_rotate_start() and perf_rotate_context() are fully serialized
2908  * because they're strictly cpu affine and rotate_start is called with IRQs
2909  * disabled, while rotate_context is called from IRQ context.
2910  */
2911 static int perf_rotate_context(struct perf_cpu_context *cpuctx)
2912 {
2913 	struct perf_event_context *ctx = NULL;
2914 	int rotate = 0, remove = 1;
2915 
2916 	if (cpuctx->ctx.nr_events) {
2917 		remove = 0;
2918 		if (cpuctx->ctx.nr_events != cpuctx->ctx.nr_active)
2919 			rotate = 1;
2920 	}
2921 
2922 	ctx = cpuctx->task_ctx;
2923 	if (ctx && ctx->nr_events) {
2924 		remove = 0;
2925 		if (ctx->nr_events != ctx->nr_active)
2926 			rotate = 1;
2927 	}
2928 
2929 	if (!rotate)
2930 		goto done;
2931 
2932 	perf_ctx_lock(cpuctx, cpuctx->task_ctx);
2933 	perf_pmu_disable(cpuctx->ctx.pmu);
2934 
2935 	cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);
2936 	if (ctx)
2937 		ctx_sched_out(ctx, cpuctx, EVENT_FLEXIBLE);
2938 
2939 	rotate_ctx(&cpuctx->ctx);
2940 	if (ctx)
2941 		rotate_ctx(ctx);
2942 
2943 	perf_event_sched_in(cpuctx, ctx, current);
2944 
2945 	perf_pmu_enable(cpuctx->ctx.pmu);
2946 	perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
2947 done:
2948 	if (remove)
2949 		list_del_init(&cpuctx->rotation_list);
2950 
2951 	return rotate;
2952 }
2953 
2954 #ifdef CONFIG_NO_HZ_FULL
2955 bool perf_event_can_stop_tick(void)
2956 {
2957 	if (atomic_read(&nr_freq_events) ||
2958 	    __this_cpu_read(perf_throttled_count))
2959 		return false;
2960 	else
2961 		return true;
2962 }
2963 #endif
2964 
2965 void perf_event_task_tick(void)
2966 {
2967 	struct list_head *head = this_cpu_ptr(&rotation_list);
2968 	struct perf_cpu_context *cpuctx, *tmp;
2969 	struct perf_event_context *ctx;
2970 	int throttled;
2971 
2972 	WARN_ON(!irqs_disabled());
2973 
2974 	__this_cpu_inc(perf_throttled_seq);
2975 	throttled = __this_cpu_xchg(perf_throttled_count, 0);
2976 
2977 	list_for_each_entry_safe(cpuctx, tmp, head, rotation_list) {
2978 		ctx = &cpuctx->ctx;
2979 		perf_adjust_freq_unthr_context(ctx, throttled);
2980 
2981 		ctx = cpuctx->task_ctx;
2982 		if (ctx)
2983 			perf_adjust_freq_unthr_context(ctx, throttled);
2984 	}
2985 }
2986 
2987 static int event_enable_on_exec(struct perf_event *event,
2988 				struct perf_event_context *ctx)
2989 {
2990 	if (!event->attr.enable_on_exec)
2991 		return 0;
2992 
2993 	event->attr.enable_on_exec = 0;
2994 	if (event->state >= PERF_EVENT_STATE_INACTIVE)
2995 		return 0;
2996 
2997 	__perf_event_mark_enabled(event);
2998 
2999 	return 1;
3000 }
3001 
3002 /*
3003  * Enable all of a task's events that have been marked enable-on-exec.
3004  * This expects task == current.
3005  */
3006 static void perf_event_enable_on_exec(struct perf_event_context *ctx)
3007 {
3008 	struct perf_event_context *clone_ctx = NULL;
3009 	struct perf_event *event;
3010 	unsigned long flags;
3011 	int enabled = 0;
3012 	int ret;
3013 
3014 	local_irq_save(flags);
3015 	if (!ctx || !ctx->nr_events)
3016 		goto out;
3017 
3018 	/*
3019 	 * We must ctxsw out cgroup events to avoid conflict
3020 	 * when invoking perf_task_event_sched_in() later on
3021 	 * in this function. Otherwise we end up trying to
3022 	 * ctxswin cgroup events which are already scheduled
3023 	 * in.
3024 	 */
3025 	perf_cgroup_sched_out(current, NULL);
3026 
3027 	raw_spin_lock(&ctx->lock);
3028 	task_ctx_sched_out(ctx);
3029 
3030 	list_for_each_entry(event, &ctx->event_list, event_entry) {
3031 		ret = event_enable_on_exec(event, ctx);
3032 		if (ret)
3033 			enabled = 1;
3034 	}
3035 
3036 	/*
3037 	 * Unclone this context if we enabled any event.
3038 	 */
3039 	if (enabled)
3040 		clone_ctx = unclone_ctx(ctx);
3041 
3042 	raw_spin_unlock(&ctx->lock);
3043 
3044 	/*
3045 	 * Also calls ctxswin for cgroup events, if any:
3046 	 */
3047 	perf_event_context_sched_in(ctx, ctx->task);
3048 out:
3049 	local_irq_restore(flags);
3050 
3051 	if (clone_ctx)
3052 		put_ctx(clone_ctx);
3053 }
3054 
3055 void perf_event_exec(void)
3056 {
3057 	struct perf_event_context *ctx;
3058 	int ctxn;
3059 
3060 	rcu_read_lock();
3061 	for_each_task_context_nr(ctxn) {
3062 		ctx = current->perf_event_ctxp[ctxn];
3063 		if (!ctx)
3064 			continue;
3065 
3066 		perf_event_enable_on_exec(ctx);
3067 	}
3068 	rcu_read_unlock();
3069 }
3070 
3071 /*
3072  * Cross CPU call to read the hardware event
3073  */
3074 static void __perf_event_read(void *info)
3075 {
3076 	struct perf_event *event = info;
3077 	struct perf_event_context *ctx = event->ctx;
3078 	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
3079 
3080 	/*
3081 	 * If this is a task context, we need to check whether it is
3082 	 * the current task context of this cpu.  If not it has been
3083 	 * scheduled out before the smp call arrived.  In that case
3084 	 * event->count would have been updated to a recent sample
3085 	 * when the event was scheduled out.
3086 	 */
3087 	if (ctx->task && cpuctx->task_ctx != ctx)
3088 		return;
3089 
3090 	raw_spin_lock(&ctx->lock);
3091 	if (ctx->is_active) {
3092 		update_context_time(ctx);
3093 		update_cgrp_time_from_event(event);
3094 	}
3095 	update_event_times(event);
3096 	if (event->state == PERF_EVENT_STATE_ACTIVE)
3097 		event->pmu->read(event);
3098 	raw_spin_unlock(&ctx->lock);
3099 }
3100 
3101 static inline u64 perf_event_count(struct perf_event *event)
3102 {
3103 	return local64_read(&event->count) + atomic64_read(&event->child_count);
3104 }
3105 
3106 static u64 perf_event_read(struct perf_event *event)
3107 {
3108 	/*
3109 	 * If event is enabled and currently active on a CPU, update the
3110 	 * value in the event structure:
3111 	 */
3112 	if (event->state == PERF_EVENT_STATE_ACTIVE) {
3113 		smp_call_function_single(event->oncpu,
3114 					 __perf_event_read, event, 1);
3115 	} else if (event->state == PERF_EVENT_STATE_INACTIVE) {
3116 		struct perf_event_context *ctx = event->ctx;
3117 		unsigned long flags;
3118 
3119 		raw_spin_lock_irqsave(&ctx->lock, flags);
3120 		/*
3121 		 * may read while context is not active
3122 		 * (e.g., thread is blocked), in that case
3123 		 * we cannot update context time
3124 		 */
3125 		if (ctx->is_active) {
3126 			update_context_time(ctx);
3127 			update_cgrp_time_from_event(event);
3128 		}
3129 		update_event_times(event);
3130 		raw_spin_unlock_irqrestore(&ctx->lock, flags);
3131 	}
3132 
3133 	return perf_event_count(event);
3134 }
3135 
3136 /*
3137  * Initialize the perf_event context in a task_struct:
3138  */
3139 static void __perf_event_init_context(struct perf_event_context *ctx)
3140 {
3141 	raw_spin_lock_init(&ctx->lock);
3142 	mutex_init(&ctx->mutex);
3143 	INIT_LIST_HEAD(&ctx->pinned_groups);
3144 	INIT_LIST_HEAD(&ctx->flexible_groups);
3145 	INIT_LIST_HEAD(&ctx->event_list);
3146 	atomic_set(&ctx->refcount, 1);
3147 	INIT_DELAYED_WORK(&ctx->orphans_remove, orphans_remove_work);
3148 }
3149 
3150 static struct perf_event_context *
3151 alloc_perf_context(struct pmu *pmu, struct task_struct *task)
3152 {
3153 	struct perf_event_context *ctx;
3154 
3155 	ctx = kzalloc(sizeof(struct perf_event_context), GFP_KERNEL);
3156 	if (!ctx)
3157 		return NULL;
3158 
3159 	__perf_event_init_context(ctx);
3160 	if (task) {
3161 		ctx->task = task;
3162 		get_task_struct(task);
3163 	}
3164 	ctx->pmu = pmu;
3165 
3166 	return ctx;
3167 }
3168 
3169 static struct task_struct *
3170 find_lively_task_by_vpid(pid_t vpid)
3171 {
3172 	struct task_struct *task;
3173 	int err;
3174 
3175 	rcu_read_lock();
3176 	if (!vpid)
3177 		task = current;
3178 	else
3179 		task = find_task_by_vpid(vpid);
3180 	if (task)
3181 		get_task_struct(task);
3182 	rcu_read_unlock();
3183 
3184 	if (!task)
3185 		return ERR_PTR(-ESRCH);
3186 
3187 	/* Reuse ptrace permission checks for now. */
3188 	err = -EACCES;
3189 	if (!ptrace_may_access(task, PTRACE_MODE_READ))
3190 		goto errout;
3191 
3192 	return task;
3193 errout:
3194 	put_task_struct(task);
3195 	return ERR_PTR(err);
3196 
3197 }
3198 
3199 /*
3200  * Returns a matching context with refcount and pincount.
3201  */
3202 static struct perf_event_context *
3203 find_get_context(struct pmu *pmu, struct task_struct *task, int cpu)
3204 {
3205 	struct perf_event_context *ctx, *clone_ctx = NULL;
3206 	struct perf_cpu_context *cpuctx;
3207 	unsigned long flags;
3208 	int ctxn, err;
3209 
3210 	if (!task) {
3211 		/* Must be root to operate on a CPU event: */
3212 		if (perf_paranoid_cpu() && !capable(CAP_SYS_ADMIN))
3213 			return ERR_PTR(-EACCES);
3214 
3215 		/*
3216 		 * We could be clever and allow to attach a event to an
3217 		 * offline CPU and activate it when the CPU comes up, but
3218 		 * that's for later.
3219 		 */
3220 		if (!cpu_online(cpu))
3221 			return ERR_PTR(-ENODEV);
3222 
3223 		cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
3224 		ctx = &cpuctx->ctx;
3225 		get_ctx(ctx);
3226 		++ctx->pin_count;
3227 
3228 		return ctx;
3229 	}
3230 
3231 	err = -EINVAL;
3232 	ctxn = pmu->task_ctx_nr;
3233 	if (ctxn < 0)
3234 		goto errout;
3235 
3236 retry:
3237 	ctx = perf_lock_task_context(task, ctxn, &flags);
3238 	if (ctx) {
3239 		clone_ctx = unclone_ctx(ctx);
3240 		++ctx->pin_count;
3241 		raw_spin_unlock_irqrestore(&ctx->lock, flags);
3242 
3243 		if (clone_ctx)
3244 			put_ctx(clone_ctx);
3245 	} else {
3246 		ctx = alloc_perf_context(pmu, task);
3247 		err = -ENOMEM;
3248 		if (!ctx)
3249 			goto errout;
3250 
3251 		err = 0;
3252 		mutex_lock(&task->perf_event_mutex);
3253 		/*
3254 		 * If it has already passed perf_event_exit_task().
3255 		 * we must see PF_EXITING, it takes this mutex too.
3256 		 */
3257 		if (task->flags & PF_EXITING)
3258 			err = -ESRCH;
3259 		else if (task->perf_event_ctxp[ctxn])
3260 			err = -EAGAIN;
3261 		else {
3262 			get_ctx(ctx);
3263 			++ctx->pin_count;
3264 			rcu_assign_pointer(task->perf_event_ctxp[ctxn], ctx);
3265 		}
3266 		mutex_unlock(&task->perf_event_mutex);
3267 
3268 		if (unlikely(err)) {
3269 			put_ctx(ctx);
3270 
3271 			if (err == -EAGAIN)
3272 				goto retry;
3273 			goto errout;
3274 		}
3275 	}
3276 
3277 	return ctx;
3278 
3279 errout:
3280 	return ERR_PTR(err);
3281 }
3282 
3283 static void perf_event_free_filter(struct perf_event *event);
3284 
3285 static void free_event_rcu(struct rcu_head *head)
3286 {
3287 	struct perf_event *event;
3288 
3289 	event = container_of(head, struct perf_event, rcu_head);
3290 	if (event->ns)
3291 		put_pid_ns(event->ns);
3292 	perf_event_free_filter(event);
3293 	kfree(event);
3294 }
3295 
3296 static void ring_buffer_put(struct ring_buffer *rb);
3297 static void ring_buffer_attach(struct perf_event *event,
3298 			       struct ring_buffer *rb);
3299 
3300 static void unaccount_event_cpu(struct perf_event *event, int cpu)
3301 {
3302 	if (event->parent)
3303 		return;
3304 
3305 	if (has_branch_stack(event)) {
3306 		if (!(event->attach_state & PERF_ATTACH_TASK))
3307 			atomic_dec(&per_cpu(perf_branch_stack_events, cpu));
3308 	}
3309 	if (is_cgroup_event(event))
3310 		atomic_dec(&per_cpu(perf_cgroup_events, cpu));
3311 }
3312 
3313 static void unaccount_event(struct perf_event *event)
3314 {
3315 	if (event->parent)
3316 		return;
3317 
3318 	if (event->attach_state & PERF_ATTACH_TASK)
3319 		static_key_slow_dec_deferred(&perf_sched_events);
3320 	if (event->attr.mmap || event->attr.mmap_data)
3321 		atomic_dec(&nr_mmap_events);
3322 	if (event->attr.comm)
3323 		atomic_dec(&nr_comm_events);
3324 	if (event->attr.task)
3325 		atomic_dec(&nr_task_events);
3326 	if (event->attr.freq)
3327 		atomic_dec(&nr_freq_events);
3328 	if (is_cgroup_event(event))
3329 		static_key_slow_dec_deferred(&perf_sched_events);
3330 	if (has_branch_stack(event))
3331 		static_key_slow_dec_deferred(&perf_sched_events);
3332 
3333 	unaccount_event_cpu(event, event->cpu);
3334 }
3335 
3336 static void __free_event(struct perf_event *event)
3337 {
3338 	if (!event->parent) {
3339 		if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN)
3340 			put_callchain_buffers();
3341 	}
3342 
3343 	if (event->destroy)
3344 		event->destroy(event);
3345 
3346 	if (event->ctx)
3347 		put_ctx(event->ctx);
3348 
3349 	if (event->pmu)
3350 		module_put(event->pmu->module);
3351 
3352 	call_rcu(&event->rcu_head, free_event_rcu);
3353 }
3354 
3355 static void _free_event(struct perf_event *event)
3356 {
3357 	irq_work_sync(&event->pending);
3358 
3359 	unaccount_event(event);
3360 
3361 	if (event->rb) {
3362 		/*
3363 		 * Can happen when we close an event with re-directed output.
3364 		 *
3365 		 * Since we have a 0 refcount, perf_mmap_close() will skip
3366 		 * over us; possibly making our ring_buffer_put() the last.
3367 		 */
3368 		mutex_lock(&event->mmap_mutex);
3369 		ring_buffer_attach(event, NULL);
3370 		mutex_unlock(&event->mmap_mutex);
3371 	}
3372 
3373 	if (is_cgroup_event(event))
3374 		perf_detach_cgroup(event);
3375 
3376 	__free_event(event);
3377 }
3378 
3379 /*
3380  * Used to free events which have a known refcount of 1, such as in error paths
3381  * where the event isn't exposed yet and inherited events.
3382  */
3383 static void free_event(struct perf_event *event)
3384 {
3385 	if (WARN(atomic_long_cmpxchg(&event->refcount, 1, 0) != 1,
3386 				"unexpected event refcount: %ld; ptr=%p\n",
3387 				atomic_long_read(&event->refcount), event)) {
3388 		/* leak to avoid use-after-free */
3389 		return;
3390 	}
3391 
3392 	_free_event(event);
3393 }
3394 
3395 /*
3396  * Remove user event from the owner task.
3397  */
3398 static void perf_remove_from_owner(struct perf_event *event)
3399 {
3400 	struct task_struct *owner;
3401 
3402 	rcu_read_lock();
3403 	owner = ACCESS_ONCE(event->owner);
3404 	/*
3405 	 * Matches the smp_wmb() in perf_event_exit_task(). If we observe
3406 	 * !owner it means the list deletion is complete and we can indeed
3407 	 * free this event, otherwise we need to serialize on
3408 	 * owner->perf_event_mutex.
3409 	 */
3410 	smp_read_barrier_depends();
3411 	if (owner) {
3412 		/*
3413 		 * Since delayed_put_task_struct() also drops the last
3414 		 * task reference we can safely take a new reference
3415 		 * while holding the rcu_read_lock().
3416 		 */
3417 		get_task_struct(owner);
3418 	}
3419 	rcu_read_unlock();
3420 
3421 	if (owner) {
3422 		mutex_lock(&owner->perf_event_mutex);
3423 		/*
3424 		 * We have to re-check the event->owner field, if it is cleared
3425 		 * we raced with perf_event_exit_task(), acquiring the mutex
3426 		 * ensured they're done, and we can proceed with freeing the
3427 		 * event.
3428 		 */
3429 		if (event->owner)
3430 			list_del_init(&event->owner_entry);
3431 		mutex_unlock(&owner->perf_event_mutex);
3432 		put_task_struct(owner);
3433 	}
3434 }
3435 
3436 /*
3437  * Called when the last reference to the file is gone.
3438  */
3439 static void put_event(struct perf_event *event)
3440 {
3441 	struct perf_event_context *ctx = event->ctx;
3442 
3443 	if (!atomic_long_dec_and_test(&event->refcount))
3444 		return;
3445 
3446 	if (!is_kernel_event(event))
3447 		perf_remove_from_owner(event);
3448 
3449 	WARN_ON_ONCE(ctx->parent_ctx);
3450 	/*
3451 	 * There are two ways this annotation is useful:
3452 	 *
3453 	 *  1) there is a lock recursion from perf_event_exit_task
3454 	 *     see the comment there.
3455 	 *
3456 	 *  2) there is a lock-inversion with mmap_sem through
3457 	 *     perf_event_read_group(), which takes faults while
3458 	 *     holding ctx->mutex, however this is called after
3459 	 *     the last filedesc died, so there is no possibility
3460 	 *     to trigger the AB-BA case.
3461 	 */
3462 	mutex_lock_nested(&ctx->mutex, SINGLE_DEPTH_NESTING);
3463 	perf_remove_from_context(event, true);
3464 	mutex_unlock(&ctx->mutex);
3465 
3466 	_free_event(event);
3467 }
3468 
3469 int perf_event_release_kernel(struct perf_event *event)
3470 {
3471 	put_event(event);
3472 	return 0;
3473 }
3474 EXPORT_SYMBOL_GPL(perf_event_release_kernel);
3475 
3476 static int perf_release(struct inode *inode, struct file *file)
3477 {
3478 	put_event(file->private_data);
3479 	return 0;
3480 }
3481 
3482 /*
3483  * Remove all orphanes events from the context.
3484  */
3485 static void orphans_remove_work(struct work_struct *work)
3486 {
3487 	struct perf_event_context *ctx;
3488 	struct perf_event *event, *tmp;
3489 
3490 	ctx = container_of(work, struct perf_event_context,
3491 			   orphans_remove.work);
3492 
3493 	mutex_lock(&ctx->mutex);
3494 	list_for_each_entry_safe(event, tmp, &ctx->event_list, event_entry) {
3495 		struct perf_event *parent_event = event->parent;
3496 
3497 		if (!is_orphaned_child(event))
3498 			continue;
3499 
3500 		perf_remove_from_context(event, true);
3501 
3502 		mutex_lock(&parent_event->child_mutex);
3503 		list_del_init(&event->child_list);
3504 		mutex_unlock(&parent_event->child_mutex);
3505 
3506 		free_event(event);
3507 		put_event(parent_event);
3508 	}
3509 
3510 	raw_spin_lock_irq(&ctx->lock);
3511 	ctx->orphans_remove_sched = false;
3512 	raw_spin_unlock_irq(&ctx->lock);
3513 	mutex_unlock(&ctx->mutex);
3514 
3515 	put_ctx(ctx);
3516 }
3517 
3518 u64 perf_event_read_value(struct perf_event *event, u64 *enabled, u64 *running)
3519 {
3520 	struct perf_event *child;
3521 	u64 total = 0;
3522 
3523 	*enabled = 0;
3524 	*running = 0;
3525 
3526 	mutex_lock(&event->child_mutex);
3527 	total += perf_event_read(event);
3528 	*enabled += event->total_time_enabled +
3529 			atomic64_read(&event->child_total_time_enabled);
3530 	*running += event->total_time_running +
3531 			atomic64_read(&event->child_total_time_running);
3532 
3533 	list_for_each_entry(child, &event->child_list, child_list) {
3534 		total += perf_event_read(child);
3535 		*enabled += child->total_time_enabled;
3536 		*running += child->total_time_running;
3537 	}
3538 	mutex_unlock(&event->child_mutex);
3539 
3540 	return total;
3541 }
3542 EXPORT_SYMBOL_GPL(perf_event_read_value);
3543 
3544 static int perf_event_read_group(struct perf_event *event,
3545 				   u64 read_format, char __user *buf)
3546 {
3547 	struct perf_event *leader = event->group_leader, *sub;
3548 	int n = 0, size = 0, ret = -EFAULT;
3549 	struct perf_event_context *ctx = leader->ctx;
3550 	u64 values[5];
3551 	u64 count, enabled, running;
3552 
3553 	mutex_lock(&ctx->mutex);
3554 	count = perf_event_read_value(leader, &enabled, &running);
3555 
3556 	values[n++] = 1 + leader->nr_siblings;
3557 	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
3558 		values[n++] = enabled;
3559 	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
3560 		values[n++] = running;
3561 	values[n++] = count;
3562 	if (read_format & PERF_FORMAT_ID)
3563 		values[n++] = primary_event_id(leader);
3564 
3565 	size = n * sizeof(u64);
3566 
3567 	if (copy_to_user(buf, values, size))
3568 		goto unlock;
3569 
3570 	ret = size;
3571 
3572 	list_for_each_entry(sub, &leader->sibling_list, group_entry) {
3573 		n = 0;
3574 
3575 		values[n++] = perf_event_read_value(sub, &enabled, &running);
3576 		if (read_format & PERF_FORMAT_ID)
3577 			values[n++] = primary_event_id(sub);
3578 
3579 		size = n * sizeof(u64);
3580 
3581 		if (copy_to_user(buf + ret, values, size)) {
3582 			ret = -EFAULT;
3583 			goto unlock;
3584 		}
3585 
3586 		ret += size;
3587 	}
3588 unlock:
3589 	mutex_unlock(&ctx->mutex);
3590 
3591 	return ret;
3592 }
3593 
3594 static int perf_event_read_one(struct perf_event *event,
3595 				 u64 read_format, char __user *buf)
3596 {
3597 	u64 enabled, running;
3598 	u64 values[4];
3599 	int n = 0;
3600 
3601 	values[n++] = perf_event_read_value(event, &enabled, &running);
3602 	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
3603 		values[n++] = enabled;
3604 	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
3605 		values[n++] = running;
3606 	if (read_format & PERF_FORMAT_ID)
3607 		values[n++] = primary_event_id(event);
3608 
3609 	if (copy_to_user(buf, values, n * sizeof(u64)))
3610 		return -EFAULT;
3611 
3612 	return n * sizeof(u64);
3613 }
3614 
3615 static bool is_event_hup(struct perf_event *event)
3616 {
3617 	bool no_children;
3618 
3619 	if (event->state != PERF_EVENT_STATE_EXIT)
3620 		return false;
3621 
3622 	mutex_lock(&event->child_mutex);
3623 	no_children = list_empty(&event->child_list);
3624 	mutex_unlock(&event->child_mutex);
3625 	return no_children;
3626 }
3627 
3628 /*
3629  * Read the performance event - simple non blocking version for now
3630  */
3631 static ssize_t
3632 perf_read_hw(struct perf_event *event, char __user *buf, size_t count)
3633 {
3634 	u64 read_format = event->attr.read_format;
3635 	int ret;
3636 
3637 	/*
3638 	 * Return end-of-file for a read on a event that is in
3639 	 * error state (i.e. because it was pinned but it couldn't be
3640 	 * scheduled on to the CPU at some point).
3641 	 */
3642 	if (event->state == PERF_EVENT_STATE_ERROR)
3643 		return 0;
3644 
3645 	if (count < event->read_size)
3646 		return -ENOSPC;
3647 
3648 	WARN_ON_ONCE(event->ctx->parent_ctx);
3649 	if (read_format & PERF_FORMAT_GROUP)
3650 		ret = perf_event_read_group(event, read_format, buf);
3651 	else
3652 		ret = perf_event_read_one(event, read_format, buf);
3653 
3654 	return ret;
3655 }
3656 
3657 static ssize_t
3658 perf_read(struct file *file, char __user *buf, size_t count, loff_t *ppos)
3659 {
3660 	struct perf_event *event = file->private_data;
3661 
3662 	return perf_read_hw(event, buf, count);
3663 }
3664 
3665 static unsigned int perf_poll(struct file *file, poll_table *wait)
3666 {
3667 	struct perf_event *event = file->private_data;
3668 	struct ring_buffer *rb;
3669 	unsigned int events = POLLHUP;
3670 
3671 	poll_wait(file, &event->waitq, wait);
3672 
3673 	if (is_event_hup(event))
3674 		return events;
3675 
3676 	/*
3677 	 * Pin the event->rb by taking event->mmap_mutex; otherwise
3678 	 * perf_event_set_output() can swizzle our rb and make us miss wakeups.
3679 	 */
3680 	mutex_lock(&event->mmap_mutex);
3681 	rb = event->rb;
3682 	if (rb)
3683 		events = atomic_xchg(&rb->poll, 0);
3684 	mutex_unlock(&event->mmap_mutex);
3685 	return events;
3686 }
3687 
3688 static void perf_event_reset(struct perf_event *event)
3689 {
3690 	(void)perf_event_read(event);
3691 	local64_set(&event->count, 0);
3692 	perf_event_update_userpage(event);
3693 }
3694 
3695 /*
3696  * Holding the top-level event's child_mutex means that any
3697  * descendant process that has inherited this event will block
3698  * in sync_child_event if it goes to exit, thus satisfying the
3699  * task existence requirements of perf_event_enable/disable.
3700  */
3701 static void perf_event_for_each_child(struct perf_event *event,
3702 					void (*func)(struct perf_event *))
3703 {
3704 	struct perf_event *child;
3705 
3706 	WARN_ON_ONCE(event->ctx->parent_ctx);
3707 	mutex_lock(&event->child_mutex);
3708 	func(event);
3709 	list_for_each_entry(child, &event->child_list, child_list)
3710 		func(child);
3711 	mutex_unlock(&event->child_mutex);
3712 }
3713 
3714 static void perf_event_for_each(struct perf_event *event,
3715 				  void (*func)(struct perf_event *))
3716 {
3717 	struct perf_event_context *ctx = event->ctx;
3718 	struct perf_event *sibling;
3719 
3720 	WARN_ON_ONCE(ctx->parent_ctx);
3721 	mutex_lock(&ctx->mutex);
3722 	event = event->group_leader;
3723 
3724 	perf_event_for_each_child(event, func);
3725 	list_for_each_entry(sibling, &event->sibling_list, group_entry)
3726 		perf_event_for_each_child(sibling, func);
3727 	mutex_unlock(&ctx->mutex);
3728 }
3729 
3730 static int perf_event_period(struct perf_event *event, u64 __user *arg)
3731 {
3732 	struct perf_event_context *ctx = event->ctx;
3733 	int ret = 0, active;
3734 	u64 value;
3735 
3736 	if (!is_sampling_event(event))
3737 		return -EINVAL;
3738 
3739 	if (copy_from_user(&value, arg, sizeof(value)))
3740 		return -EFAULT;
3741 
3742 	if (!value)
3743 		return -EINVAL;
3744 
3745 	raw_spin_lock_irq(&ctx->lock);
3746 	if (event->attr.freq) {
3747 		if (value > sysctl_perf_event_sample_rate) {
3748 			ret = -EINVAL;
3749 			goto unlock;
3750 		}
3751 
3752 		event->attr.sample_freq = value;
3753 	} else {
3754 		event->attr.sample_period = value;
3755 		event->hw.sample_period = value;
3756 	}
3757 
3758 	active = (event->state == PERF_EVENT_STATE_ACTIVE);
3759 	if (active) {
3760 		perf_pmu_disable(ctx->pmu);
3761 		event->pmu->stop(event, PERF_EF_UPDATE);
3762 	}
3763 
3764 	local64_set(&event->hw.period_left, 0);
3765 
3766 	if (active) {
3767 		event->pmu->start(event, PERF_EF_RELOAD);
3768 		perf_pmu_enable(ctx->pmu);
3769 	}
3770 
3771 unlock:
3772 	raw_spin_unlock_irq(&ctx->lock);
3773 
3774 	return ret;
3775 }
3776 
3777 static const struct file_operations perf_fops;
3778 
3779 static inline int perf_fget_light(int fd, struct fd *p)
3780 {
3781 	struct fd f = fdget(fd);
3782 	if (!f.file)
3783 		return -EBADF;
3784 
3785 	if (f.file->f_op != &perf_fops) {
3786 		fdput(f);
3787 		return -EBADF;
3788 	}
3789 	*p = f;
3790 	return 0;
3791 }
3792 
3793 static int perf_event_set_output(struct perf_event *event,
3794 				 struct perf_event *output_event);
3795 static int perf_event_set_filter(struct perf_event *event, void __user *arg);
3796 
3797 static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
3798 {
3799 	struct perf_event *event = file->private_data;
3800 	void (*func)(struct perf_event *);
3801 	u32 flags = arg;
3802 
3803 	switch (cmd) {
3804 	case PERF_EVENT_IOC_ENABLE:
3805 		func = perf_event_enable;
3806 		break;
3807 	case PERF_EVENT_IOC_DISABLE:
3808 		func = perf_event_disable;
3809 		break;
3810 	case PERF_EVENT_IOC_RESET:
3811 		func = perf_event_reset;
3812 		break;
3813 
3814 	case PERF_EVENT_IOC_REFRESH:
3815 		return perf_event_refresh(event, arg);
3816 
3817 	case PERF_EVENT_IOC_PERIOD:
3818 		return perf_event_period(event, (u64 __user *)arg);
3819 
3820 	case PERF_EVENT_IOC_ID:
3821 	{
3822 		u64 id = primary_event_id(event);
3823 
3824 		if (copy_to_user((void __user *)arg, &id, sizeof(id)))
3825 			return -EFAULT;
3826 		return 0;
3827 	}
3828 
3829 	case PERF_EVENT_IOC_SET_OUTPUT:
3830 	{
3831 		int ret;
3832 		if (arg != -1) {
3833 			struct perf_event *output_event;
3834 			struct fd output;
3835 			ret = perf_fget_light(arg, &output);
3836 			if (ret)
3837 				return ret;
3838 			output_event = output.file->private_data;
3839 			ret = perf_event_set_output(event, output_event);
3840 			fdput(output);
3841 		} else {
3842 			ret = perf_event_set_output(event, NULL);
3843 		}
3844 		return ret;
3845 	}
3846 
3847 	case PERF_EVENT_IOC_SET_FILTER:
3848 		return perf_event_set_filter(event, (void __user *)arg);
3849 
3850 	default:
3851 		return -ENOTTY;
3852 	}
3853 
3854 	if (flags & PERF_IOC_FLAG_GROUP)
3855 		perf_event_for_each(event, func);
3856 	else
3857 		perf_event_for_each_child(event, func);
3858 
3859 	return 0;
3860 }
3861 
3862 #ifdef CONFIG_COMPAT
3863 static long perf_compat_ioctl(struct file *file, unsigned int cmd,
3864 				unsigned long arg)
3865 {
3866 	switch (_IOC_NR(cmd)) {
3867 	case _IOC_NR(PERF_EVENT_IOC_SET_FILTER):
3868 	case _IOC_NR(PERF_EVENT_IOC_ID):
3869 		/* Fix up pointer size (usually 4 -> 8 in 32-on-64-bit case */
3870 		if (_IOC_SIZE(cmd) == sizeof(compat_uptr_t)) {
3871 			cmd &= ~IOCSIZE_MASK;
3872 			cmd |= sizeof(void *) << IOCSIZE_SHIFT;
3873 		}
3874 		break;
3875 	}
3876 	return perf_ioctl(file, cmd, arg);
3877 }
3878 #else
3879 # define perf_compat_ioctl NULL
3880 #endif
3881 
3882 int perf_event_task_enable(void)
3883 {
3884 	struct perf_event *event;
3885 
3886 	mutex_lock(&current->perf_event_mutex);
3887 	list_for_each_entry(event, &current->perf_event_list, owner_entry)
3888 		perf_event_for_each_child(event, perf_event_enable);
3889 	mutex_unlock(&current->perf_event_mutex);
3890 
3891 	return 0;
3892 }
3893 
3894 int perf_event_task_disable(void)
3895 {
3896 	struct perf_event *event;
3897 
3898 	mutex_lock(&current->perf_event_mutex);
3899 	list_for_each_entry(event, &current->perf_event_list, owner_entry)
3900 		perf_event_for_each_child(event, perf_event_disable);
3901 	mutex_unlock(&current->perf_event_mutex);
3902 
3903 	return 0;
3904 }
3905 
3906 static int perf_event_index(struct perf_event *event)
3907 {
3908 	if (event->hw.state & PERF_HES_STOPPED)
3909 		return 0;
3910 
3911 	if (event->state != PERF_EVENT_STATE_ACTIVE)
3912 		return 0;
3913 
3914 	return event->pmu->event_idx(event);
3915 }
3916 
3917 static void calc_timer_values(struct perf_event *event,
3918 				u64 *now,
3919 				u64 *enabled,
3920 				u64 *running)
3921 {
3922 	u64 ctx_time;
3923 
3924 	*now = perf_clock();
3925 	ctx_time = event->shadow_ctx_time + *now;
3926 	*enabled = ctx_time - event->tstamp_enabled;
3927 	*running = ctx_time - event->tstamp_running;
3928 }
3929 
3930 static void perf_event_init_userpage(struct perf_event *event)
3931 {
3932 	struct perf_event_mmap_page *userpg;
3933 	struct ring_buffer *rb;
3934 
3935 	rcu_read_lock();
3936 	rb = rcu_dereference(event->rb);
3937 	if (!rb)
3938 		goto unlock;
3939 
3940 	userpg = rb->user_page;
3941 
3942 	/* Allow new userspace to detect that bit 0 is deprecated */
3943 	userpg->cap_bit0_is_deprecated = 1;
3944 	userpg->size = offsetof(struct perf_event_mmap_page, __reserved);
3945 
3946 unlock:
3947 	rcu_read_unlock();
3948 }
3949 
3950 void __weak arch_perf_update_userpage(struct perf_event_mmap_page *userpg, u64 now)
3951 {
3952 }
3953 
3954 /*
3955  * Callers need to ensure there can be no nesting of this function, otherwise
3956  * the seqlock logic goes bad. We can not serialize this because the arch
3957  * code calls this from NMI context.
3958  */
3959 void perf_event_update_userpage(struct perf_event *event)
3960 {
3961 	struct perf_event_mmap_page *userpg;
3962 	struct ring_buffer *rb;
3963 	u64 enabled, running, now;
3964 
3965 	rcu_read_lock();
3966 	rb = rcu_dereference(event->rb);
3967 	if (!rb)
3968 		goto unlock;
3969 
3970 	/*
3971 	 * compute total_time_enabled, total_time_running
3972 	 * based on snapshot values taken when the event
3973 	 * was last scheduled in.
3974 	 *
3975 	 * we cannot simply called update_context_time()
3976 	 * because of locking issue as we can be called in
3977 	 * NMI context
3978 	 */
3979 	calc_timer_values(event, &now, &enabled, &running);
3980 
3981 	userpg = rb->user_page;
3982 	/*
3983 	 * Disable preemption so as to not let the corresponding user-space
3984 	 * spin too long if we get preempted.
3985 	 */
3986 	preempt_disable();
3987 	++userpg->lock;
3988 	barrier();
3989 	userpg->index = perf_event_index(event);
3990 	userpg->offset = perf_event_count(event);
3991 	if (userpg->index)
3992 		userpg->offset -= local64_read(&event->hw.prev_count);
3993 
3994 	userpg->time_enabled = enabled +
3995 			atomic64_read(&event->child_total_time_enabled);
3996 
3997 	userpg->time_running = running +
3998 			atomic64_read(&event->child_total_time_running);
3999 
4000 	arch_perf_update_userpage(userpg, now);
4001 
4002 	barrier();
4003 	++userpg->lock;
4004 	preempt_enable();
4005 unlock:
4006 	rcu_read_unlock();
4007 }
4008 
4009 static int perf_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
4010 {
4011 	struct perf_event *event = vma->vm_file->private_data;
4012 	struct ring_buffer *rb;
4013 	int ret = VM_FAULT_SIGBUS;
4014 
4015 	if (vmf->flags & FAULT_FLAG_MKWRITE) {
4016 		if (vmf->pgoff == 0)
4017 			ret = 0;
4018 		return ret;
4019 	}
4020 
4021 	rcu_read_lock();
4022 	rb = rcu_dereference(event->rb);
4023 	if (!rb)
4024 		goto unlock;
4025 
4026 	if (vmf->pgoff && (vmf->flags & FAULT_FLAG_WRITE))
4027 		goto unlock;
4028 
4029 	vmf->page = perf_mmap_to_page(rb, vmf->pgoff);
4030 	if (!vmf->page)
4031 		goto unlock;
4032 
4033 	get_page(vmf->page);
4034 	vmf->page->mapping = vma->vm_file->f_mapping;
4035 	vmf->page->index   = vmf->pgoff;
4036 
4037 	ret = 0;
4038 unlock:
4039 	rcu_read_unlock();
4040 
4041 	return ret;
4042 }
4043 
4044 static void ring_buffer_attach(struct perf_event *event,
4045 			       struct ring_buffer *rb)
4046 {
4047 	struct ring_buffer *old_rb = NULL;
4048 	unsigned long flags;
4049 
4050 	if (event->rb) {
4051 		/*
4052 		 * Should be impossible, we set this when removing
4053 		 * event->rb_entry and wait/clear when adding event->rb_entry.
4054 		 */
4055 		WARN_ON_ONCE(event->rcu_pending);
4056 
4057 		old_rb = event->rb;
4058 		event->rcu_batches = get_state_synchronize_rcu();
4059 		event->rcu_pending = 1;
4060 
4061 		spin_lock_irqsave(&old_rb->event_lock, flags);
4062 		list_del_rcu(&event->rb_entry);
4063 		spin_unlock_irqrestore(&old_rb->event_lock, flags);
4064 	}
4065 
4066 	if (event->rcu_pending && rb) {
4067 		cond_synchronize_rcu(event->rcu_batches);
4068 		event->rcu_pending = 0;
4069 	}
4070 
4071 	if (rb) {
4072 		spin_lock_irqsave(&rb->event_lock, flags);
4073 		list_add_rcu(&event->rb_entry, &rb->event_list);
4074 		spin_unlock_irqrestore(&rb->event_lock, flags);
4075 	}
4076 
4077 	rcu_assign_pointer(event->rb, rb);
4078 
4079 	if (old_rb) {
4080 		ring_buffer_put(old_rb);
4081 		/*
4082 		 * Since we detached before setting the new rb, so that we
4083 		 * could attach the new rb, we could have missed a wakeup.
4084 		 * Provide it now.
4085 		 */
4086 		wake_up_all(&event->waitq);
4087 	}
4088 }
4089 
4090 static void ring_buffer_wakeup(struct perf_event *event)
4091 {
4092 	struct ring_buffer *rb;
4093 
4094 	rcu_read_lock();
4095 	rb = rcu_dereference(event->rb);
4096 	if (rb) {
4097 		list_for_each_entry_rcu(event, &rb->event_list, rb_entry)
4098 			wake_up_all(&event->waitq);
4099 	}
4100 	rcu_read_unlock();
4101 }
4102 
4103 static void rb_free_rcu(struct rcu_head *rcu_head)
4104 {
4105 	struct ring_buffer *rb;
4106 
4107 	rb = container_of(rcu_head, struct ring_buffer, rcu_head);
4108 	rb_free(rb);
4109 }
4110 
4111 static struct ring_buffer *ring_buffer_get(struct perf_event *event)
4112 {
4113 	struct ring_buffer *rb;
4114 
4115 	rcu_read_lock();
4116 	rb = rcu_dereference(event->rb);
4117 	if (rb) {
4118 		if (!atomic_inc_not_zero(&rb->refcount))
4119 			rb = NULL;
4120 	}
4121 	rcu_read_unlock();
4122 
4123 	return rb;
4124 }
4125 
4126 static void ring_buffer_put(struct ring_buffer *rb)
4127 {
4128 	if (!atomic_dec_and_test(&rb->refcount))
4129 		return;
4130 
4131 	WARN_ON_ONCE(!list_empty(&rb->event_list));
4132 
4133 	call_rcu(&rb->rcu_head, rb_free_rcu);
4134 }
4135 
4136 static void perf_mmap_open(struct vm_area_struct *vma)
4137 {
4138 	struct perf_event *event = vma->vm_file->private_data;
4139 
4140 	atomic_inc(&event->mmap_count);
4141 	atomic_inc(&event->rb->mmap_count);
4142 }
4143 
4144 /*
4145  * A buffer can be mmap()ed multiple times; either directly through the same
4146  * event, or through other events by use of perf_event_set_output().
4147  *
4148  * In order to undo the VM accounting done by perf_mmap() we need to destroy
4149  * the buffer here, where we still have a VM context. This means we need
4150  * to detach all events redirecting to us.
4151  */
4152 static void perf_mmap_close(struct vm_area_struct *vma)
4153 {
4154 	struct perf_event *event = vma->vm_file->private_data;
4155 
4156 	struct ring_buffer *rb = ring_buffer_get(event);
4157 	struct user_struct *mmap_user = rb->mmap_user;
4158 	int mmap_locked = rb->mmap_locked;
4159 	unsigned long size = perf_data_size(rb);
4160 
4161 	atomic_dec(&rb->mmap_count);
4162 
4163 	if (!atomic_dec_and_mutex_lock(&event->mmap_count, &event->mmap_mutex))
4164 		goto out_put;
4165 
4166 	ring_buffer_attach(event, NULL);
4167 	mutex_unlock(&event->mmap_mutex);
4168 
4169 	/* If there's still other mmap()s of this buffer, we're done. */
4170 	if (atomic_read(&rb->mmap_count))
4171 		goto out_put;
4172 
4173 	/*
4174 	 * No other mmap()s, detach from all other events that might redirect
4175 	 * into the now unreachable buffer. Somewhat complicated by the
4176 	 * fact that rb::event_lock otherwise nests inside mmap_mutex.
4177 	 */
4178 again:
4179 	rcu_read_lock();
4180 	list_for_each_entry_rcu(event, &rb->event_list, rb_entry) {
4181 		if (!atomic_long_inc_not_zero(&event->refcount)) {
4182 			/*
4183 			 * This event is en-route to free_event() which will
4184 			 * detach it and remove it from the list.
4185 			 */
4186 			continue;
4187 		}
4188 		rcu_read_unlock();
4189 
4190 		mutex_lock(&event->mmap_mutex);
4191 		/*
4192 		 * Check we didn't race with perf_event_set_output() which can
4193 		 * swizzle the rb from under us while we were waiting to
4194 		 * acquire mmap_mutex.
4195 		 *
4196 		 * If we find a different rb; ignore this event, a next
4197 		 * iteration will no longer find it on the list. We have to
4198 		 * still restart the iteration to make sure we're not now
4199 		 * iterating the wrong list.
4200 		 */
4201 		if (event->rb == rb)
4202 			ring_buffer_attach(event, NULL);
4203 
4204 		mutex_unlock(&event->mmap_mutex);
4205 		put_event(event);
4206 
4207 		/*
4208 		 * Restart the iteration; either we're on the wrong list or
4209 		 * destroyed its integrity by doing a deletion.
4210 		 */
4211 		goto again;
4212 	}
4213 	rcu_read_unlock();
4214 
4215 	/*
4216 	 * It could be there's still a few 0-ref events on the list; they'll
4217 	 * get cleaned up by free_event() -- they'll also still have their
4218 	 * ref on the rb and will free it whenever they are done with it.
4219 	 *
4220 	 * Aside from that, this buffer is 'fully' detached and unmapped,
4221 	 * undo the VM accounting.
4222 	 */
4223 
4224 	atomic_long_sub((size >> PAGE_SHIFT) + 1, &mmap_user->locked_vm);
4225 	vma->vm_mm->pinned_vm -= mmap_locked;
4226 	free_uid(mmap_user);
4227 
4228 out_put:
4229 	ring_buffer_put(rb); /* could be last */
4230 }
4231 
4232 static const struct vm_operations_struct perf_mmap_vmops = {
4233 	.open		= perf_mmap_open,
4234 	.close		= perf_mmap_close,
4235 	.fault		= perf_mmap_fault,
4236 	.page_mkwrite	= perf_mmap_fault,
4237 };
4238 
4239 static int perf_mmap(struct file *file, struct vm_area_struct *vma)
4240 {
4241 	struct perf_event *event = file->private_data;
4242 	unsigned long user_locked, user_lock_limit;
4243 	struct user_struct *user = current_user();
4244 	unsigned long locked, lock_limit;
4245 	struct ring_buffer *rb;
4246 	unsigned long vma_size;
4247 	unsigned long nr_pages;
4248 	long user_extra, extra;
4249 	int ret = 0, flags = 0;
4250 
4251 	/*
4252 	 * Don't allow mmap() of inherited per-task counters. This would
4253 	 * create a performance issue due to all children writing to the
4254 	 * same rb.
4255 	 */
4256 	if (event->cpu == -1 && event->attr.inherit)
4257 		return -EINVAL;
4258 
4259 	if (!(vma->vm_flags & VM_SHARED))
4260 		return -EINVAL;
4261 
4262 	vma_size = vma->vm_end - vma->vm_start;
4263 	nr_pages = (vma_size / PAGE_SIZE) - 1;
4264 
4265 	/*
4266 	 * If we have rb pages ensure they're a power-of-two number, so we
4267 	 * can do bitmasks instead of modulo.
4268 	 */
4269 	if (nr_pages != 0 && !is_power_of_2(nr_pages))
4270 		return -EINVAL;
4271 
4272 	if (vma_size != PAGE_SIZE * (1 + nr_pages))
4273 		return -EINVAL;
4274 
4275 	if (vma->vm_pgoff != 0)
4276 		return -EINVAL;
4277 
4278 	WARN_ON_ONCE(event->ctx->parent_ctx);
4279 again:
4280 	mutex_lock(&event->mmap_mutex);
4281 	if (event->rb) {
4282 		if (event->rb->nr_pages != nr_pages) {
4283 			ret = -EINVAL;
4284 			goto unlock;
4285 		}
4286 
4287 		if (!atomic_inc_not_zero(&event->rb->mmap_count)) {
4288 			/*
4289 			 * Raced against perf_mmap_close() through
4290 			 * perf_event_set_output(). Try again, hope for better
4291 			 * luck.
4292 			 */
4293 			mutex_unlock(&event->mmap_mutex);
4294 			goto again;
4295 		}
4296 
4297 		goto unlock;
4298 	}
4299 
4300 	user_extra = nr_pages + 1;
4301 	user_lock_limit = sysctl_perf_event_mlock >> (PAGE_SHIFT - 10);
4302 
4303 	/*
4304 	 * Increase the limit linearly with more CPUs:
4305 	 */
4306 	user_lock_limit *= num_online_cpus();
4307 
4308 	user_locked = atomic_long_read(&user->locked_vm) + user_extra;
4309 
4310 	extra = 0;
4311 	if (user_locked > user_lock_limit)
4312 		extra = user_locked - user_lock_limit;
4313 
4314 	lock_limit = rlimit(RLIMIT_MEMLOCK);
4315 	lock_limit >>= PAGE_SHIFT;
4316 	locked = vma->vm_mm->pinned_vm + extra;
4317 
4318 	if ((locked > lock_limit) && perf_paranoid_tracepoint_raw() &&
4319 		!capable(CAP_IPC_LOCK)) {
4320 		ret = -EPERM;
4321 		goto unlock;
4322 	}
4323 
4324 	WARN_ON(event->rb);
4325 
4326 	if (vma->vm_flags & VM_WRITE)
4327 		flags |= RING_BUFFER_WRITABLE;
4328 
4329 	rb = rb_alloc(nr_pages,
4330 		event->attr.watermark ? event->attr.wakeup_watermark : 0,
4331 		event->cpu, flags);
4332 
4333 	if (!rb) {
4334 		ret = -ENOMEM;
4335 		goto unlock;
4336 	}
4337 
4338 	atomic_set(&rb->mmap_count, 1);
4339 	rb->mmap_locked = extra;
4340 	rb->mmap_user = get_current_user();
4341 
4342 	atomic_long_add(user_extra, &user->locked_vm);
4343 	vma->vm_mm->pinned_vm += extra;
4344 
4345 	ring_buffer_attach(event, rb);
4346 
4347 	perf_event_init_userpage(event);
4348 	perf_event_update_userpage(event);
4349 
4350 unlock:
4351 	if (!ret)
4352 		atomic_inc(&event->mmap_count);
4353 	mutex_unlock(&event->mmap_mutex);
4354 
4355 	/*
4356 	 * Since pinned accounting is per vm we cannot allow fork() to copy our
4357 	 * vma.
4358 	 */
4359 	vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_DONTDUMP;
4360 	vma->vm_ops = &perf_mmap_vmops;
4361 
4362 	return ret;
4363 }
4364 
4365 static int perf_fasync(int fd, struct file *filp, int on)
4366 {
4367 	struct inode *inode = file_inode(filp);
4368 	struct perf_event *event = filp->private_data;
4369 	int retval;
4370 
4371 	mutex_lock(&inode->i_mutex);
4372 	retval = fasync_helper(fd, filp, on, &event->fasync);
4373 	mutex_unlock(&inode->i_mutex);
4374 
4375 	if (retval < 0)
4376 		return retval;
4377 
4378 	return 0;
4379 }
4380 
4381 static const struct file_operations perf_fops = {
4382 	.llseek			= no_llseek,
4383 	.release		= perf_release,
4384 	.read			= perf_read,
4385 	.poll			= perf_poll,
4386 	.unlocked_ioctl		= perf_ioctl,
4387 	.compat_ioctl		= perf_compat_ioctl,
4388 	.mmap			= perf_mmap,
4389 	.fasync			= perf_fasync,
4390 };
4391 
4392 /*
4393  * Perf event wakeup
4394  *
4395  * If there's data, ensure we set the poll() state and publish everything
4396  * to user-space before waking everybody up.
4397  */
4398 
4399 void perf_event_wakeup(struct perf_event *event)
4400 {
4401 	ring_buffer_wakeup(event);
4402 
4403 	if (event->pending_kill) {
4404 		kill_fasync(&event->fasync, SIGIO, event->pending_kill);
4405 		event->pending_kill = 0;
4406 	}
4407 }
4408 
4409 static void perf_pending_event(struct irq_work *entry)
4410 {
4411 	struct perf_event *event = container_of(entry,
4412 			struct perf_event, pending);
4413 
4414 	if (event->pending_disable) {
4415 		event->pending_disable = 0;
4416 		__perf_event_disable(event);
4417 	}
4418 
4419 	if (event->pending_wakeup) {
4420 		event->pending_wakeup = 0;
4421 		perf_event_wakeup(event);
4422 	}
4423 }
4424 
4425 /*
4426  * We assume there is only KVM supporting the callbacks.
4427  * Later on, we might change it to a list if there is
4428  * another virtualization implementation supporting the callbacks.
4429  */
4430 struct perf_guest_info_callbacks *perf_guest_cbs;
4431 
4432 int perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *cbs)
4433 {
4434 	perf_guest_cbs = cbs;
4435 	return 0;
4436 }
4437 EXPORT_SYMBOL_GPL(perf_register_guest_info_callbacks);
4438 
4439 int perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *cbs)
4440 {
4441 	perf_guest_cbs = NULL;
4442 	return 0;
4443 }
4444 EXPORT_SYMBOL_GPL(perf_unregister_guest_info_callbacks);
4445 
4446 static void
4447 perf_output_sample_regs(struct perf_output_handle *handle,
4448 			struct pt_regs *regs, u64 mask)
4449 {
4450 	int bit;
4451 
4452 	for_each_set_bit(bit, (const unsigned long *) &mask,
4453 			 sizeof(mask) * BITS_PER_BYTE) {
4454 		u64 val;
4455 
4456 		val = perf_reg_value(regs, bit);
4457 		perf_output_put(handle, val);
4458 	}
4459 }
4460 
4461 static void perf_sample_regs_user(struct perf_regs_user *regs_user,
4462 				  struct pt_regs *regs)
4463 {
4464 	if (!user_mode(regs)) {
4465 		if (current->mm)
4466 			regs = task_pt_regs(current);
4467 		else
4468 			regs = NULL;
4469 	}
4470 
4471 	if (regs) {
4472 		regs_user->regs = regs;
4473 		regs_user->abi  = perf_reg_abi(current);
4474 	}
4475 }
4476 
4477 /*
4478  * Get remaining task size from user stack pointer.
4479  *
4480  * It'd be better to take stack vma map and limit this more
4481  * precisly, but there's no way to get it safely under interrupt,
4482  * so using TASK_SIZE as limit.
4483  */
4484 static u64 perf_ustack_task_size(struct pt_regs *regs)
4485 {
4486 	unsigned long addr = perf_user_stack_pointer(regs);
4487 
4488 	if (!addr || addr >= TASK_SIZE)
4489 		return 0;
4490 
4491 	return TASK_SIZE - addr;
4492 }
4493 
4494 static u16
4495 perf_sample_ustack_size(u16 stack_size, u16 header_size,
4496 			struct pt_regs *regs)
4497 {
4498 	u64 task_size;
4499 
4500 	/* No regs, no stack pointer, no dump. */
4501 	if (!regs)
4502 		return 0;
4503 
4504 	/*
4505 	 * Check if we fit in with the requested stack size into the:
4506 	 * - TASK_SIZE
4507 	 *   If we don't, we limit the size to the TASK_SIZE.
4508 	 *
4509 	 * - remaining sample size
4510 	 *   If we don't, we customize the stack size to
4511 	 *   fit in to the remaining sample size.
4512 	 */
4513 
4514 	task_size  = min((u64) USHRT_MAX, perf_ustack_task_size(regs));
4515 	stack_size = min(stack_size, (u16) task_size);
4516 
4517 	/* Current header size plus static size and dynamic size. */
4518 	header_size += 2 * sizeof(u64);
4519 
4520 	/* Do we fit in with the current stack dump size? */
4521 	if ((u16) (header_size + stack_size) < header_size) {
4522 		/*
4523 		 * If we overflow the maximum size for the sample,
4524 		 * we customize the stack dump size to fit in.
4525 		 */
4526 		stack_size = USHRT_MAX - header_size - sizeof(u64);
4527 		stack_size = round_up(stack_size, sizeof(u64));
4528 	}
4529 
4530 	return stack_size;
4531 }
4532 
4533 static void
4534 perf_output_sample_ustack(struct perf_output_handle *handle, u64 dump_size,
4535 			  struct pt_regs *regs)
4536 {
4537 	/* Case of a kernel thread, nothing to dump */
4538 	if (!regs) {
4539 		u64 size = 0;
4540 		perf_output_put(handle, size);
4541 	} else {
4542 		unsigned long sp;
4543 		unsigned int rem;
4544 		u64 dyn_size;
4545 
4546 		/*
4547 		 * We dump:
4548 		 * static size
4549 		 *   - the size requested by user or the best one we can fit
4550 		 *     in to the sample max size
4551 		 * data
4552 		 *   - user stack dump data
4553 		 * dynamic size
4554 		 *   - the actual dumped size
4555 		 */
4556 
4557 		/* Static size. */
4558 		perf_output_put(handle, dump_size);
4559 
4560 		/* Data. */
4561 		sp = perf_user_stack_pointer(regs);
4562 		rem = __output_copy_user(handle, (void *) sp, dump_size);
4563 		dyn_size = dump_size - rem;
4564 
4565 		perf_output_skip(handle, rem);
4566 
4567 		/* Dynamic size. */
4568 		perf_output_put(handle, dyn_size);
4569 	}
4570 }
4571 
4572 static void __perf_event_header__init_id(struct perf_event_header *header,
4573 					 struct perf_sample_data *data,
4574 					 struct perf_event *event)
4575 {
4576 	u64 sample_type = event->attr.sample_type;
4577 
4578 	data->type = sample_type;
4579 	header->size += event->id_header_size;
4580 
4581 	if (sample_type & PERF_SAMPLE_TID) {
4582 		/* namespace issues */
4583 		data->tid_entry.pid = perf_event_pid(event, current);
4584 		data->tid_entry.tid = perf_event_tid(event, current);
4585 	}
4586 
4587 	if (sample_type & PERF_SAMPLE_TIME)
4588 		data->time = perf_clock();
4589 
4590 	if (sample_type & (PERF_SAMPLE_ID | PERF_SAMPLE_IDENTIFIER))
4591 		data->id = primary_event_id(event);
4592 
4593 	if (sample_type & PERF_SAMPLE_STREAM_ID)
4594 		data->stream_id = event->id;
4595 
4596 	if (sample_type & PERF_SAMPLE_CPU) {
4597 		data->cpu_entry.cpu	 = raw_smp_processor_id();
4598 		data->cpu_entry.reserved = 0;
4599 	}
4600 }
4601 
4602 void perf_event_header__init_id(struct perf_event_header *header,
4603 				struct perf_sample_data *data,
4604 				struct perf_event *event)
4605 {
4606 	if (event->attr.sample_id_all)
4607 		__perf_event_header__init_id(header, data, event);
4608 }
4609 
4610 static void __perf_event__output_id_sample(struct perf_output_handle *handle,
4611 					   struct perf_sample_data *data)
4612 {
4613 	u64 sample_type = data->type;
4614 
4615 	if (sample_type & PERF_SAMPLE_TID)
4616 		perf_output_put(handle, data->tid_entry);
4617 
4618 	if (sample_type & PERF_SAMPLE_TIME)
4619 		perf_output_put(handle, data->time);
4620 
4621 	if (sample_type & PERF_SAMPLE_ID)
4622 		perf_output_put(handle, data->id);
4623 
4624 	if (sample_type & PERF_SAMPLE_STREAM_ID)
4625 		perf_output_put(handle, data->stream_id);
4626 
4627 	if (sample_type & PERF_SAMPLE_CPU)
4628 		perf_output_put(handle, data->cpu_entry);
4629 
4630 	if (sample_type & PERF_SAMPLE_IDENTIFIER)
4631 		perf_output_put(handle, data->id);
4632 }
4633 
4634 void perf_event__output_id_sample(struct perf_event *event,
4635 				  struct perf_output_handle *handle,
4636 				  struct perf_sample_data *sample)
4637 {
4638 	if (event->attr.sample_id_all)
4639 		__perf_event__output_id_sample(handle, sample);
4640 }
4641 
4642 static void perf_output_read_one(struct perf_output_handle *handle,
4643 				 struct perf_event *event,
4644 				 u64 enabled, u64 running)
4645 {
4646 	u64 read_format = event->attr.read_format;
4647 	u64 values[4];
4648 	int n = 0;
4649 
4650 	values[n++] = perf_event_count(event);
4651 	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
4652 		values[n++] = enabled +
4653 			atomic64_read(&event->child_total_time_enabled);
4654 	}
4655 	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
4656 		values[n++] = running +
4657 			atomic64_read(&event->child_total_time_running);
4658 	}
4659 	if (read_format & PERF_FORMAT_ID)
4660 		values[n++] = primary_event_id(event);
4661 
4662 	__output_copy(handle, values, n * sizeof(u64));
4663 }
4664 
4665 /*
4666  * XXX PERF_FORMAT_GROUP vs inherited events seems difficult.
4667  */
4668 static void perf_output_read_group(struct perf_output_handle *handle,
4669 			    struct perf_event *event,
4670 			    u64 enabled, u64 running)
4671 {
4672 	struct perf_event *leader = event->group_leader, *sub;
4673 	u64 read_format = event->attr.read_format;
4674 	u64 values[5];
4675 	int n = 0;
4676 
4677 	values[n++] = 1 + leader->nr_siblings;
4678 
4679 	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
4680 		values[n++] = enabled;
4681 
4682 	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
4683 		values[n++] = running;
4684 
4685 	if (leader != event)
4686 		leader->pmu->read(leader);
4687 
4688 	values[n++] = perf_event_count(leader);
4689 	if (read_format & PERF_FORMAT_ID)
4690 		values[n++] = primary_event_id(leader);
4691 
4692 	__output_copy(handle, values, n * sizeof(u64));
4693 
4694 	list_for_each_entry(sub, &leader->sibling_list, group_entry) {
4695 		n = 0;
4696 
4697 		if ((sub != event) &&
4698 		    (sub->state == PERF_EVENT_STATE_ACTIVE))
4699 			sub->pmu->read(sub);
4700 
4701 		values[n++] = perf_event_count(sub);
4702 		if (read_format & PERF_FORMAT_ID)
4703 			values[n++] = primary_event_id(sub);
4704 
4705 		__output_copy(handle, values, n * sizeof(u64));
4706 	}
4707 }
4708 
4709 #define PERF_FORMAT_TOTAL_TIMES (PERF_FORMAT_TOTAL_TIME_ENABLED|\
4710 				 PERF_FORMAT_TOTAL_TIME_RUNNING)
4711 
4712 static void perf_output_read(struct perf_output_handle *handle,
4713 			     struct perf_event *event)
4714 {
4715 	u64 enabled = 0, running = 0, now;
4716 	u64 read_format = event->attr.read_format;
4717 
4718 	/*
4719 	 * compute total_time_enabled, total_time_running
4720 	 * based on snapshot values taken when the event
4721 	 * was last scheduled in.
4722 	 *
4723 	 * we cannot simply called update_context_time()
4724 	 * because of locking issue as we are called in
4725 	 * NMI context
4726 	 */
4727 	if (read_format & PERF_FORMAT_TOTAL_TIMES)
4728 		calc_timer_values(event, &now, &enabled, &running);
4729 
4730 	if (event->attr.read_format & PERF_FORMAT_GROUP)
4731 		perf_output_read_group(handle, event, enabled, running);
4732 	else
4733 		perf_output_read_one(handle, event, enabled, running);
4734 }
4735 
4736 void perf_output_sample(struct perf_output_handle *handle,
4737 			struct perf_event_header *header,
4738 			struct perf_sample_data *data,
4739 			struct perf_event *event)
4740 {
4741 	u64 sample_type = data->type;
4742 
4743 	perf_output_put(handle, *header);
4744 
4745 	if (sample_type & PERF_SAMPLE_IDENTIFIER)
4746 		perf_output_put(handle, data->id);
4747 
4748 	if (sample_type & PERF_SAMPLE_IP)
4749 		perf_output_put(handle, data->ip);
4750 
4751 	if (sample_type & PERF_SAMPLE_TID)
4752 		perf_output_put(handle, data->tid_entry);
4753 
4754 	if (sample_type & PERF_SAMPLE_TIME)
4755 		perf_output_put(handle, data->time);
4756 
4757 	if (sample_type & PERF_SAMPLE_ADDR)
4758 		perf_output_put(handle, data->addr);
4759 
4760 	if (sample_type & PERF_SAMPLE_ID)
4761 		perf_output_put(handle, data->id);
4762 
4763 	if (sample_type & PERF_SAMPLE_STREAM_ID)
4764 		perf_output_put(handle, data->stream_id);
4765 
4766 	if (sample_type & PERF_SAMPLE_CPU)
4767 		perf_output_put(handle, data->cpu_entry);
4768 
4769 	if (sample_type & PERF_SAMPLE_PERIOD)
4770 		perf_output_put(handle, data->period);
4771 
4772 	if (sample_type & PERF_SAMPLE_READ)
4773 		perf_output_read(handle, event);
4774 
4775 	if (sample_type & PERF_SAMPLE_CALLCHAIN) {
4776 		if (data->callchain) {
4777 			int size = 1;
4778 
4779 			if (data->callchain)
4780 				size += data->callchain->nr;
4781 
4782 			size *= sizeof(u64);
4783 
4784 			__output_copy(handle, data->callchain, size);
4785 		} else {
4786 			u64 nr = 0;
4787 			perf_output_put(handle, nr);
4788 		}
4789 	}
4790 
4791 	if (sample_type & PERF_SAMPLE_RAW) {
4792 		if (data->raw) {
4793 			perf_output_put(handle, data->raw->size);
4794 			__output_copy(handle, data->raw->data,
4795 					   data->raw->size);
4796 		} else {
4797 			struct {
4798 				u32	size;
4799 				u32	data;
4800 			} raw = {
4801 				.size = sizeof(u32),
4802 				.data = 0,
4803 			};
4804 			perf_output_put(handle, raw);
4805 		}
4806 	}
4807 
4808 	if (sample_type & PERF_SAMPLE_BRANCH_STACK) {
4809 		if (data->br_stack) {
4810 			size_t size;
4811 
4812 			size = data->br_stack->nr
4813 			     * sizeof(struct perf_branch_entry);
4814 
4815 			perf_output_put(handle, data->br_stack->nr);
4816 			perf_output_copy(handle, data->br_stack->entries, size);
4817 		} else {
4818 			/*
4819 			 * we always store at least the value of nr
4820 			 */
4821 			u64 nr = 0;
4822 			perf_output_put(handle, nr);
4823 		}
4824 	}
4825 
4826 	if (sample_type & PERF_SAMPLE_REGS_USER) {
4827 		u64 abi = data->regs_user.abi;
4828 
4829 		/*
4830 		 * If there are no regs to dump, notice it through
4831 		 * first u64 being zero (PERF_SAMPLE_REGS_ABI_NONE).
4832 		 */
4833 		perf_output_put(handle, abi);
4834 
4835 		if (abi) {
4836 			u64 mask = event->attr.sample_regs_user;
4837 			perf_output_sample_regs(handle,
4838 						data->regs_user.regs,
4839 						mask);
4840 		}
4841 	}
4842 
4843 	if (sample_type & PERF_SAMPLE_STACK_USER) {
4844 		perf_output_sample_ustack(handle,
4845 					  data->stack_user_size,
4846 					  data->regs_user.regs);
4847 	}
4848 
4849 	if (sample_type & PERF_SAMPLE_WEIGHT)
4850 		perf_output_put(handle, data->weight);
4851 
4852 	if (sample_type & PERF_SAMPLE_DATA_SRC)
4853 		perf_output_put(handle, data->data_src.val);
4854 
4855 	if (sample_type & PERF_SAMPLE_TRANSACTION)
4856 		perf_output_put(handle, data->txn);
4857 
4858 	if (!event->attr.watermark) {
4859 		int wakeup_events = event->attr.wakeup_events;
4860 
4861 		if (wakeup_events) {
4862 			struct ring_buffer *rb = handle->rb;
4863 			int events = local_inc_return(&rb->events);
4864 
4865 			if (events >= wakeup_events) {
4866 				local_sub(wakeup_events, &rb->events);
4867 				local_inc(&rb->wakeup);
4868 			}
4869 		}
4870 	}
4871 }
4872 
4873 void perf_prepare_sample(struct perf_event_header *header,
4874 			 struct perf_sample_data *data,
4875 			 struct perf_event *event,
4876 			 struct pt_regs *regs)
4877 {
4878 	u64 sample_type = event->attr.sample_type;
4879 
4880 	header->type = PERF_RECORD_SAMPLE;
4881 	header->size = sizeof(*header) + event->header_size;
4882 
4883 	header->misc = 0;
4884 	header->misc |= perf_misc_flags(regs);
4885 
4886 	__perf_event_header__init_id(header, data, event);
4887 
4888 	if (sample_type & PERF_SAMPLE_IP)
4889 		data->ip = perf_instruction_pointer(regs);
4890 
4891 	if (sample_type & PERF_SAMPLE_CALLCHAIN) {
4892 		int size = 1;
4893 
4894 		data->callchain = perf_callchain(event, regs);
4895 
4896 		if (data->callchain)
4897 			size += data->callchain->nr;
4898 
4899 		header->size += size * sizeof(u64);
4900 	}
4901 
4902 	if (sample_type & PERF_SAMPLE_RAW) {
4903 		int size = sizeof(u32);
4904 
4905 		if (data->raw)
4906 			size += data->raw->size;
4907 		else
4908 			size += sizeof(u32);
4909 
4910 		WARN_ON_ONCE(size & (sizeof(u64)-1));
4911 		header->size += size;
4912 	}
4913 
4914 	if (sample_type & PERF_SAMPLE_BRANCH_STACK) {
4915 		int size = sizeof(u64); /* nr */
4916 		if (data->br_stack) {
4917 			size += data->br_stack->nr
4918 			      * sizeof(struct perf_branch_entry);
4919 		}
4920 		header->size += size;
4921 	}
4922 
4923 	if (sample_type & PERF_SAMPLE_REGS_USER) {
4924 		/* regs dump ABI info */
4925 		int size = sizeof(u64);
4926 
4927 		perf_sample_regs_user(&data->regs_user, regs);
4928 
4929 		if (data->regs_user.regs) {
4930 			u64 mask = event->attr.sample_regs_user;
4931 			size += hweight64(mask) * sizeof(u64);
4932 		}
4933 
4934 		header->size += size;
4935 	}
4936 
4937 	if (sample_type & PERF_SAMPLE_STACK_USER) {
4938 		/*
4939 		 * Either we need PERF_SAMPLE_STACK_USER bit to be allways
4940 		 * processed as the last one or have additional check added
4941 		 * in case new sample type is added, because we could eat
4942 		 * up the rest of the sample size.
4943 		 */
4944 		struct perf_regs_user *uregs = &data->regs_user;
4945 		u16 stack_size = event->attr.sample_stack_user;
4946 		u16 size = sizeof(u64);
4947 
4948 		if (!uregs->abi)
4949 			perf_sample_regs_user(uregs, regs);
4950 
4951 		stack_size = perf_sample_ustack_size(stack_size, header->size,
4952 						     uregs->regs);
4953 
4954 		/*
4955 		 * If there is something to dump, add space for the dump
4956 		 * itself and for the field that tells the dynamic size,
4957 		 * which is how many have been actually dumped.
4958 		 */
4959 		if (stack_size)
4960 			size += sizeof(u64) + stack_size;
4961 
4962 		data->stack_user_size = stack_size;
4963 		header->size += size;
4964 	}
4965 }
4966 
4967 static void perf_event_output(struct perf_event *event,
4968 				struct perf_sample_data *data,
4969 				struct pt_regs *regs)
4970 {
4971 	struct perf_output_handle handle;
4972 	struct perf_event_header header;
4973 
4974 	/* protect the callchain buffers */
4975 	rcu_read_lock();
4976 
4977 	perf_prepare_sample(&header, data, event, regs);
4978 
4979 	if (perf_output_begin(&handle, event, header.size))
4980 		goto exit;
4981 
4982 	perf_output_sample(&handle, &header, data, event);
4983 
4984 	perf_output_end(&handle);
4985 
4986 exit:
4987 	rcu_read_unlock();
4988 }
4989 
4990 /*
4991  * read event_id
4992  */
4993 
4994 struct perf_read_event {
4995 	struct perf_event_header	header;
4996 
4997 	u32				pid;
4998 	u32				tid;
4999 };
5000 
5001 static void
5002 perf_event_read_event(struct perf_event *event,
5003 			struct task_struct *task)
5004 {
5005 	struct perf_output_handle handle;
5006 	struct perf_sample_data sample;
5007 	struct perf_read_event read_event = {
5008 		.header = {
5009 			.type = PERF_RECORD_READ,
5010 			.misc = 0,
5011 			.size = sizeof(read_event) + event->read_size,
5012 		},
5013 		.pid = perf_event_pid(event, task),
5014 		.tid = perf_event_tid(event, task),
5015 	};
5016 	int ret;
5017 
5018 	perf_event_header__init_id(&read_event.header, &sample, event);
5019 	ret = perf_output_begin(&handle, event, read_event.header.size);
5020 	if (ret)
5021 		return;
5022 
5023 	perf_output_put(&handle, read_event);
5024 	perf_output_read(&handle, event);
5025 	perf_event__output_id_sample(event, &handle, &sample);
5026 
5027 	perf_output_end(&handle);
5028 }
5029 
5030 typedef void (perf_event_aux_output_cb)(struct perf_event *event, void *data);
5031 
5032 static void
5033 perf_event_aux_ctx(struct perf_event_context *ctx,
5034 		   perf_event_aux_output_cb output,
5035 		   void *data)
5036 {
5037 	struct perf_event *event;
5038 
5039 	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
5040 		if (event->state < PERF_EVENT_STATE_INACTIVE)
5041 			continue;
5042 		if (!event_filter_match(event))
5043 			continue;
5044 		output(event, data);
5045 	}
5046 }
5047 
5048 static void
5049 perf_event_aux(perf_event_aux_output_cb output, void *data,
5050 	       struct perf_event_context *task_ctx)
5051 {
5052 	struct perf_cpu_context *cpuctx;
5053 	struct perf_event_context *ctx;
5054 	struct pmu *pmu;
5055 	int ctxn;
5056 
5057 	rcu_read_lock();
5058 	list_for_each_entry_rcu(pmu, &pmus, entry) {
5059 		cpuctx = get_cpu_ptr(pmu->pmu_cpu_context);
5060 		if (cpuctx->unique_pmu != pmu)
5061 			goto next;
5062 		perf_event_aux_ctx(&cpuctx->ctx, output, data);
5063 		if (task_ctx)
5064 			goto next;
5065 		ctxn = pmu->task_ctx_nr;
5066 		if (ctxn < 0)
5067 			goto next;
5068 		ctx = rcu_dereference(current->perf_event_ctxp[ctxn]);
5069 		if (ctx)
5070 			perf_event_aux_ctx(ctx, output, data);
5071 next:
5072 		put_cpu_ptr(pmu->pmu_cpu_context);
5073 	}
5074 
5075 	if (task_ctx) {
5076 		preempt_disable();
5077 		perf_event_aux_ctx(task_ctx, output, data);
5078 		preempt_enable();
5079 	}
5080 	rcu_read_unlock();
5081 }
5082 
5083 /*
5084  * task tracking -- fork/exit
5085  *
5086  * enabled by: attr.comm | attr.mmap | attr.mmap2 | attr.mmap_data | attr.task
5087  */
5088 
5089 struct perf_task_event {
5090 	struct task_struct		*task;
5091 	struct perf_event_context	*task_ctx;
5092 
5093 	struct {
5094 		struct perf_event_header	header;
5095 
5096 		u32				pid;
5097 		u32				ppid;
5098 		u32				tid;
5099 		u32				ptid;
5100 		u64				time;
5101 	} event_id;
5102 };
5103 
5104 static int perf_event_task_match(struct perf_event *event)
5105 {
5106 	return event->attr.comm  || event->attr.mmap ||
5107 	       event->attr.mmap2 || event->attr.mmap_data ||
5108 	       event->attr.task;
5109 }
5110 
5111 static void perf_event_task_output(struct perf_event *event,
5112 				   void *data)
5113 {
5114 	struct perf_task_event *task_event = data;
5115 	struct perf_output_handle handle;
5116 	struct perf_sample_data	sample;
5117 	struct task_struct *task = task_event->task;
5118 	int ret, size = task_event->event_id.header.size;
5119 
5120 	if (!perf_event_task_match(event))
5121 		return;
5122 
5123 	perf_event_header__init_id(&task_event->event_id.header, &sample, event);
5124 
5125 	ret = perf_output_begin(&handle, event,
5126 				task_event->event_id.header.size);
5127 	if (ret)
5128 		goto out;
5129 
5130 	task_event->event_id.pid = perf_event_pid(event, task);
5131 	task_event->event_id.ppid = perf_event_pid(event, current);
5132 
5133 	task_event->event_id.tid = perf_event_tid(event, task);
5134 	task_event->event_id.ptid = perf_event_tid(event, current);
5135 
5136 	perf_output_put(&handle, task_event->event_id);
5137 
5138 	perf_event__output_id_sample(event, &handle, &sample);
5139 
5140 	perf_output_end(&handle);
5141 out:
5142 	task_event->event_id.header.size = size;
5143 }
5144 
5145 static void perf_event_task(struct task_struct *task,
5146 			      struct perf_event_context *task_ctx,
5147 			      int new)
5148 {
5149 	struct perf_task_event task_event;
5150 
5151 	if (!atomic_read(&nr_comm_events) &&
5152 	    !atomic_read(&nr_mmap_events) &&
5153 	    !atomic_read(&nr_task_events))
5154 		return;
5155 
5156 	task_event = (struct perf_task_event){
5157 		.task	  = task,
5158 		.task_ctx = task_ctx,
5159 		.event_id    = {
5160 			.header = {
5161 				.type = new ? PERF_RECORD_FORK : PERF_RECORD_EXIT,
5162 				.misc = 0,
5163 				.size = sizeof(task_event.event_id),
5164 			},
5165 			/* .pid  */
5166 			/* .ppid */
5167 			/* .tid  */
5168 			/* .ptid */
5169 			.time = perf_clock(),
5170 		},
5171 	};
5172 
5173 	perf_event_aux(perf_event_task_output,
5174 		       &task_event,
5175 		       task_ctx);
5176 }
5177 
5178 void perf_event_fork(struct task_struct *task)
5179 {
5180 	perf_event_task(task, NULL, 1);
5181 }
5182 
5183 /*
5184  * comm tracking
5185  */
5186 
5187 struct perf_comm_event {
5188 	struct task_struct	*task;
5189 	char			*comm;
5190 	int			comm_size;
5191 
5192 	struct {
5193 		struct perf_event_header	header;
5194 
5195 		u32				pid;
5196 		u32				tid;
5197 	} event_id;
5198 };
5199 
5200 static int perf_event_comm_match(struct perf_event *event)
5201 {
5202 	return event->attr.comm;
5203 }
5204 
5205 static void perf_event_comm_output(struct perf_event *event,
5206 				   void *data)
5207 {
5208 	struct perf_comm_event *comm_event = data;
5209 	struct perf_output_handle handle;
5210 	struct perf_sample_data sample;
5211 	int size = comm_event->event_id.header.size;
5212 	int ret;
5213 
5214 	if (!perf_event_comm_match(event))
5215 		return;
5216 
5217 	perf_event_header__init_id(&comm_event->event_id.header, &sample, event);
5218 	ret = perf_output_begin(&handle, event,
5219 				comm_event->event_id.header.size);
5220 
5221 	if (ret)
5222 		goto out;
5223 
5224 	comm_event->event_id.pid = perf_event_pid(event, comm_event->task);
5225 	comm_event->event_id.tid = perf_event_tid(event, comm_event->task);
5226 
5227 	perf_output_put(&handle, comm_event->event_id);
5228 	__output_copy(&handle, comm_event->comm,
5229 				   comm_event->comm_size);
5230 
5231 	perf_event__output_id_sample(event, &handle, &sample);
5232 
5233 	perf_output_end(&handle);
5234 out:
5235 	comm_event->event_id.header.size = size;
5236 }
5237 
5238 static void perf_event_comm_event(struct perf_comm_event *comm_event)
5239 {
5240 	char comm[TASK_COMM_LEN];
5241 	unsigned int size;
5242 
5243 	memset(comm, 0, sizeof(comm));
5244 	strlcpy(comm, comm_event->task->comm, sizeof(comm));
5245 	size = ALIGN(strlen(comm)+1, sizeof(u64));
5246 
5247 	comm_event->comm = comm;
5248 	comm_event->comm_size = size;
5249 
5250 	comm_event->event_id.header.size = sizeof(comm_event->event_id) + size;
5251 
5252 	perf_event_aux(perf_event_comm_output,
5253 		       comm_event,
5254 		       NULL);
5255 }
5256 
5257 void perf_event_comm(struct task_struct *task, bool exec)
5258 {
5259 	struct perf_comm_event comm_event;
5260 
5261 	if (!atomic_read(&nr_comm_events))
5262 		return;
5263 
5264 	comm_event = (struct perf_comm_event){
5265 		.task	= task,
5266 		/* .comm      */
5267 		/* .comm_size */
5268 		.event_id  = {
5269 			.header = {
5270 				.type = PERF_RECORD_COMM,
5271 				.misc = exec ? PERF_RECORD_MISC_COMM_EXEC : 0,
5272 				/* .size */
5273 			},
5274 			/* .pid */
5275 			/* .tid */
5276 		},
5277 	};
5278 
5279 	perf_event_comm_event(&comm_event);
5280 }
5281 
5282 /*
5283  * mmap tracking
5284  */
5285 
5286 struct perf_mmap_event {
5287 	struct vm_area_struct	*vma;
5288 
5289 	const char		*file_name;
5290 	int			file_size;
5291 	int			maj, min;
5292 	u64			ino;
5293 	u64			ino_generation;
5294 	u32			prot, flags;
5295 
5296 	struct {
5297 		struct perf_event_header	header;
5298 
5299 		u32				pid;
5300 		u32				tid;
5301 		u64				start;
5302 		u64				len;
5303 		u64				pgoff;
5304 	} event_id;
5305 };
5306 
5307 static int perf_event_mmap_match(struct perf_event *event,
5308 				 void *data)
5309 {
5310 	struct perf_mmap_event *mmap_event = data;
5311 	struct vm_area_struct *vma = mmap_event->vma;
5312 	int executable = vma->vm_flags & VM_EXEC;
5313 
5314 	return (!executable && event->attr.mmap_data) ||
5315 	       (executable && (event->attr.mmap || event->attr.mmap2));
5316 }
5317 
5318 static void perf_event_mmap_output(struct perf_event *event,
5319 				   void *data)
5320 {
5321 	struct perf_mmap_event *mmap_event = data;
5322 	struct perf_output_handle handle;
5323 	struct perf_sample_data sample;
5324 	int size = mmap_event->event_id.header.size;
5325 	int ret;
5326 
5327 	if (!perf_event_mmap_match(event, data))
5328 		return;
5329 
5330 	if (event->attr.mmap2) {
5331 		mmap_event->event_id.header.type = PERF_RECORD_MMAP2;
5332 		mmap_event->event_id.header.size += sizeof(mmap_event->maj);
5333 		mmap_event->event_id.header.size += sizeof(mmap_event->min);
5334 		mmap_event->event_id.header.size += sizeof(mmap_event->ino);
5335 		mmap_event->event_id.header.size += sizeof(mmap_event->ino_generation);
5336 		mmap_event->event_id.header.size += sizeof(mmap_event->prot);
5337 		mmap_event->event_id.header.size += sizeof(mmap_event->flags);
5338 	}
5339 
5340 	perf_event_header__init_id(&mmap_event->event_id.header, &sample, event);
5341 	ret = perf_output_begin(&handle, event,
5342 				mmap_event->event_id.header.size);
5343 	if (ret)
5344 		goto out;
5345 
5346 	mmap_event->event_id.pid = perf_event_pid(event, current);
5347 	mmap_event->event_id.tid = perf_event_tid(event, current);
5348 
5349 	perf_output_put(&handle, mmap_event->event_id);
5350 
5351 	if (event->attr.mmap2) {
5352 		perf_output_put(&handle, mmap_event->maj);
5353 		perf_output_put(&handle, mmap_event->min);
5354 		perf_output_put(&handle, mmap_event->ino);
5355 		perf_output_put(&handle, mmap_event->ino_generation);
5356 		perf_output_put(&handle, mmap_event->prot);
5357 		perf_output_put(&handle, mmap_event->flags);
5358 	}
5359 
5360 	__output_copy(&handle, mmap_event->file_name,
5361 				   mmap_event->file_size);
5362 
5363 	perf_event__output_id_sample(event, &handle, &sample);
5364 
5365 	perf_output_end(&handle);
5366 out:
5367 	mmap_event->event_id.header.size = size;
5368 }
5369 
5370 static void perf_event_mmap_event(struct perf_mmap_event *mmap_event)
5371 {
5372 	struct vm_area_struct *vma = mmap_event->vma;
5373 	struct file *file = vma->vm_file;
5374 	int maj = 0, min = 0;
5375 	u64 ino = 0, gen = 0;
5376 	u32 prot = 0, flags = 0;
5377 	unsigned int size;
5378 	char tmp[16];
5379 	char *buf = NULL;
5380 	char *name;
5381 
5382 	if (file) {
5383 		struct inode *inode;
5384 		dev_t dev;
5385 
5386 		buf = kmalloc(PATH_MAX, GFP_KERNEL);
5387 		if (!buf) {
5388 			name = "//enomem";
5389 			goto cpy_name;
5390 		}
5391 		/*
5392 		 * d_path() works from the end of the rb backwards, so we
5393 		 * need to add enough zero bytes after the string to handle
5394 		 * the 64bit alignment we do later.
5395 		 */
5396 		name = d_path(&file->f_path, buf, PATH_MAX - sizeof(u64));
5397 		if (IS_ERR(name)) {
5398 			name = "//toolong";
5399 			goto cpy_name;
5400 		}
5401 		inode = file_inode(vma->vm_file);
5402 		dev = inode->i_sb->s_dev;
5403 		ino = inode->i_ino;
5404 		gen = inode->i_generation;
5405 		maj = MAJOR(dev);
5406 		min = MINOR(dev);
5407 
5408 		if (vma->vm_flags & VM_READ)
5409 			prot |= PROT_READ;
5410 		if (vma->vm_flags & VM_WRITE)
5411 			prot |= PROT_WRITE;
5412 		if (vma->vm_flags & VM_EXEC)
5413 			prot |= PROT_EXEC;
5414 
5415 		if (vma->vm_flags & VM_MAYSHARE)
5416 			flags = MAP_SHARED;
5417 		else
5418 			flags = MAP_PRIVATE;
5419 
5420 		if (vma->vm_flags & VM_DENYWRITE)
5421 			flags |= MAP_DENYWRITE;
5422 		if (vma->vm_flags & VM_MAYEXEC)
5423 			flags |= MAP_EXECUTABLE;
5424 		if (vma->vm_flags & VM_LOCKED)
5425 			flags |= MAP_LOCKED;
5426 		if (vma->vm_flags & VM_HUGETLB)
5427 			flags |= MAP_HUGETLB;
5428 
5429 		goto got_name;
5430 	} else {
5431 		if (vma->vm_ops && vma->vm_ops->name) {
5432 			name = (char *) vma->vm_ops->name(vma);
5433 			if (name)
5434 				goto cpy_name;
5435 		}
5436 
5437 		name = (char *)arch_vma_name(vma);
5438 		if (name)
5439 			goto cpy_name;
5440 
5441 		if (vma->vm_start <= vma->vm_mm->start_brk &&
5442 				vma->vm_end >= vma->vm_mm->brk) {
5443 			name = "[heap]";
5444 			goto cpy_name;
5445 		}
5446 		if (vma->vm_start <= vma->vm_mm->start_stack &&
5447 				vma->vm_end >= vma->vm_mm->start_stack) {
5448 			name = "[stack]";
5449 			goto cpy_name;
5450 		}
5451 
5452 		name = "//anon";
5453 		goto cpy_name;
5454 	}
5455 
5456 cpy_name:
5457 	strlcpy(tmp, name, sizeof(tmp));
5458 	name = tmp;
5459 got_name:
5460 	/*
5461 	 * Since our buffer works in 8 byte units we need to align our string
5462 	 * size to a multiple of 8. However, we must guarantee the tail end is
5463 	 * zero'd out to avoid leaking random bits to userspace.
5464 	 */
5465 	size = strlen(name)+1;
5466 	while (!IS_ALIGNED(size, sizeof(u64)))
5467 		name[size++] = '\0';
5468 
5469 	mmap_event->file_name = name;
5470 	mmap_event->file_size = size;
5471 	mmap_event->maj = maj;
5472 	mmap_event->min = min;
5473 	mmap_event->ino = ino;
5474 	mmap_event->ino_generation = gen;
5475 	mmap_event->prot = prot;
5476 	mmap_event->flags = flags;
5477 
5478 	if (!(vma->vm_flags & VM_EXEC))
5479 		mmap_event->event_id.header.misc |= PERF_RECORD_MISC_MMAP_DATA;
5480 
5481 	mmap_event->event_id.header.size = sizeof(mmap_event->event_id) + size;
5482 
5483 	perf_event_aux(perf_event_mmap_output,
5484 		       mmap_event,
5485 		       NULL);
5486 
5487 	kfree(buf);
5488 }
5489 
5490 void perf_event_mmap(struct vm_area_struct *vma)
5491 {
5492 	struct perf_mmap_event mmap_event;
5493 
5494 	if (!atomic_read(&nr_mmap_events))
5495 		return;
5496 
5497 	mmap_event = (struct perf_mmap_event){
5498 		.vma	= vma,
5499 		/* .file_name */
5500 		/* .file_size */
5501 		.event_id  = {
5502 			.header = {
5503 				.type = PERF_RECORD_MMAP,
5504 				.misc = PERF_RECORD_MISC_USER,
5505 				/* .size */
5506 			},
5507 			/* .pid */
5508 			/* .tid */
5509 			.start  = vma->vm_start,
5510 			.len    = vma->vm_end - vma->vm_start,
5511 			.pgoff  = (u64)vma->vm_pgoff << PAGE_SHIFT,
5512 		},
5513 		/* .maj (attr_mmap2 only) */
5514 		/* .min (attr_mmap2 only) */
5515 		/* .ino (attr_mmap2 only) */
5516 		/* .ino_generation (attr_mmap2 only) */
5517 		/* .prot (attr_mmap2 only) */
5518 		/* .flags (attr_mmap2 only) */
5519 	};
5520 
5521 	perf_event_mmap_event(&mmap_event);
5522 }
5523 
5524 /*
5525  * IRQ throttle logging
5526  */
5527 
5528 static void perf_log_throttle(struct perf_event *event, int enable)
5529 {
5530 	struct perf_output_handle handle;
5531 	struct perf_sample_data sample;
5532 	int ret;
5533 
5534 	struct {
5535 		struct perf_event_header	header;
5536 		u64				time;
5537 		u64				id;
5538 		u64				stream_id;
5539 	} throttle_event = {
5540 		.header = {
5541 			.type = PERF_RECORD_THROTTLE,
5542 			.misc = 0,
5543 			.size = sizeof(throttle_event),
5544 		},
5545 		.time		= perf_clock(),
5546 		.id		= primary_event_id(event),
5547 		.stream_id	= event->id,
5548 	};
5549 
5550 	if (enable)
5551 		throttle_event.header.type = PERF_RECORD_UNTHROTTLE;
5552 
5553 	perf_event_header__init_id(&throttle_event.header, &sample, event);
5554 
5555 	ret = perf_output_begin(&handle, event,
5556 				throttle_event.header.size);
5557 	if (ret)
5558 		return;
5559 
5560 	perf_output_put(&handle, throttle_event);
5561 	perf_event__output_id_sample(event, &handle, &sample);
5562 	perf_output_end(&handle);
5563 }
5564 
5565 /*
5566  * Generic event overflow handling, sampling.
5567  */
5568 
5569 static int __perf_event_overflow(struct perf_event *event,
5570 				   int throttle, struct perf_sample_data *data,
5571 				   struct pt_regs *regs)
5572 {
5573 	int events = atomic_read(&event->event_limit);
5574 	struct hw_perf_event *hwc = &event->hw;
5575 	u64 seq;
5576 	int ret = 0;
5577 
5578 	/*
5579 	 * Non-sampling counters might still use the PMI to fold short
5580 	 * hardware counters, ignore those.
5581 	 */
5582 	if (unlikely(!is_sampling_event(event)))
5583 		return 0;
5584 
5585 	seq = __this_cpu_read(perf_throttled_seq);
5586 	if (seq != hwc->interrupts_seq) {
5587 		hwc->interrupts_seq = seq;
5588 		hwc->interrupts = 1;
5589 	} else {
5590 		hwc->interrupts++;
5591 		if (unlikely(throttle
5592 			     && hwc->interrupts >= max_samples_per_tick)) {
5593 			__this_cpu_inc(perf_throttled_count);
5594 			hwc->interrupts = MAX_INTERRUPTS;
5595 			perf_log_throttle(event, 0);
5596 			tick_nohz_full_kick();
5597 			ret = 1;
5598 		}
5599 	}
5600 
5601 	if (event->attr.freq) {
5602 		u64 now = perf_clock();
5603 		s64 delta = now - hwc->freq_time_stamp;
5604 
5605 		hwc->freq_time_stamp = now;
5606 
5607 		if (delta > 0 && delta < 2*TICK_NSEC)
5608 			perf_adjust_period(event, delta, hwc->last_period, true);
5609 	}
5610 
5611 	/*
5612 	 * XXX event_limit might not quite work as expected on inherited
5613 	 * events
5614 	 */
5615 
5616 	event->pending_kill = POLL_IN;
5617 	if (events && atomic_dec_and_test(&event->event_limit)) {
5618 		ret = 1;
5619 		event->pending_kill = POLL_HUP;
5620 		event->pending_disable = 1;
5621 		irq_work_queue(&event->pending);
5622 	}
5623 
5624 	if (event->overflow_handler)
5625 		event->overflow_handler(event, data, regs);
5626 	else
5627 		perf_event_output(event, data, regs);
5628 
5629 	if (event->fasync && event->pending_kill) {
5630 		event->pending_wakeup = 1;
5631 		irq_work_queue(&event->pending);
5632 	}
5633 
5634 	return ret;
5635 }
5636 
5637 int perf_event_overflow(struct perf_event *event,
5638 			  struct perf_sample_data *data,
5639 			  struct pt_regs *regs)
5640 {
5641 	return __perf_event_overflow(event, 1, data, regs);
5642 }
5643 
5644 /*
5645  * Generic software event infrastructure
5646  */
5647 
5648 struct swevent_htable {
5649 	struct swevent_hlist		*swevent_hlist;
5650 	struct mutex			hlist_mutex;
5651 	int				hlist_refcount;
5652 
5653 	/* Recursion avoidance in each contexts */
5654 	int				recursion[PERF_NR_CONTEXTS];
5655 
5656 	/* Keeps track of cpu being initialized/exited */
5657 	bool				online;
5658 };
5659 
5660 static DEFINE_PER_CPU(struct swevent_htable, swevent_htable);
5661 
5662 /*
5663  * We directly increment event->count and keep a second value in
5664  * event->hw.period_left to count intervals. This period event
5665  * is kept in the range [-sample_period, 0] so that we can use the
5666  * sign as trigger.
5667  */
5668 
5669 u64 perf_swevent_set_period(struct perf_event *event)
5670 {
5671 	struct hw_perf_event *hwc = &event->hw;
5672 	u64 period = hwc->last_period;
5673 	u64 nr, offset;
5674 	s64 old, val;
5675 
5676 	hwc->last_period = hwc->sample_period;
5677 
5678 again:
5679 	old = val = local64_read(&hwc->period_left);
5680 	if (val < 0)
5681 		return 0;
5682 
5683 	nr = div64_u64(period + val, period);
5684 	offset = nr * period;
5685 	val -= offset;
5686 	if (local64_cmpxchg(&hwc->period_left, old, val) != old)
5687 		goto again;
5688 
5689 	return nr;
5690 }
5691 
5692 static void perf_swevent_overflow(struct perf_event *event, u64 overflow,
5693 				    struct perf_sample_data *data,
5694 				    struct pt_regs *regs)
5695 {
5696 	struct hw_perf_event *hwc = &event->hw;
5697 	int throttle = 0;
5698 
5699 	if (!overflow)
5700 		overflow = perf_swevent_set_period(event);
5701 
5702 	if (hwc->interrupts == MAX_INTERRUPTS)
5703 		return;
5704 
5705 	for (; overflow; overflow--) {
5706 		if (__perf_event_overflow(event, throttle,
5707 					    data, regs)) {
5708 			/*
5709 			 * We inhibit the overflow from happening when
5710 			 * hwc->interrupts == MAX_INTERRUPTS.
5711 			 */
5712 			break;
5713 		}
5714 		throttle = 1;
5715 	}
5716 }
5717 
5718 static void perf_swevent_event(struct perf_event *event, u64 nr,
5719 			       struct perf_sample_data *data,
5720 			       struct pt_regs *regs)
5721 {
5722 	struct hw_perf_event *hwc = &event->hw;
5723 
5724 	local64_add(nr, &event->count);
5725 
5726 	if (!regs)
5727 		return;
5728 
5729 	if (!is_sampling_event(event))
5730 		return;
5731 
5732 	if ((event->attr.sample_type & PERF_SAMPLE_PERIOD) && !event->attr.freq) {
5733 		data->period = nr;
5734 		return perf_swevent_overflow(event, 1, data, regs);
5735 	} else
5736 		data->period = event->hw.last_period;
5737 
5738 	if (nr == 1 && hwc->sample_period == 1 && !event->attr.freq)
5739 		return perf_swevent_overflow(event, 1, data, regs);
5740 
5741 	if (local64_add_negative(nr, &hwc->period_left))
5742 		return;
5743 
5744 	perf_swevent_overflow(event, 0, data, regs);
5745 }
5746 
5747 static int perf_exclude_event(struct perf_event *event,
5748 			      struct pt_regs *regs)
5749 {
5750 	if (event->hw.state & PERF_HES_STOPPED)
5751 		return 1;
5752 
5753 	if (regs) {
5754 		if (event->attr.exclude_user && user_mode(regs))
5755 			return 1;
5756 
5757 		if (event->attr.exclude_kernel && !user_mode(regs))
5758 			return 1;
5759 	}
5760 
5761 	return 0;
5762 }
5763 
5764 static int perf_swevent_match(struct perf_event *event,
5765 				enum perf_type_id type,
5766 				u32 event_id,
5767 				struct perf_sample_data *data,
5768 				struct pt_regs *regs)
5769 {
5770 	if (event->attr.type != type)
5771 		return 0;
5772 
5773 	if (event->attr.config != event_id)
5774 		return 0;
5775 
5776 	if (perf_exclude_event(event, regs))
5777 		return 0;
5778 
5779 	return 1;
5780 }
5781 
5782 static inline u64 swevent_hash(u64 type, u32 event_id)
5783 {
5784 	u64 val = event_id | (type << 32);
5785 
5786 	return hash_64(val, SWEVENT_HLIST_BITS);
5787 }
5788 
5789 static inline struct hlist_head *
5790 __find_swevent_head(struct swevent_hlist *hlist, u64 type, u32 event_id)
5791 {
5792 	u64 hash = swevent_hash(type, event_id);
5793 
5794 	return &hlist->heads[hash];
5795 }
5796 
5797 /* For the read side: events when they trigger */
5798 static inline struct hlist_head *
5799 find_swevent_head_rcu(struct swevent_htable *swhash, u64 type, u32 event_id)
5800 {
5801 	struct swevent_hlist *hlist;
5802 
5803 	hlist = rcu_dereference(swhash->swevent_hlist);
5804 	if (!hlist)
5805 		return NULL;
5806 
5807 	return __find_swevent_head(hlist, type, event_id);
5808 }
5809 
5810 /* For the event head insertion and removal in the hlist */
5811 static inline struct hlist_head *
5812 find_swevent_head(struct swevent_htable *swhash, struct perf_event *event)
5813 {
5814 	struct swevent_hlist *hlist;
5815 	u32 event_id = event->attr.config;
5816 	u64 type = event->attr.type;
5817 
5818 	/*
5819 	 * Event scheduling is always serialized against hlist allocation
5820 	 * and release. Which makes the protected version suitable here.
5821 	 * The context lock guarantees that.
5822 	 */
5823 	hlist = rcu_dereference_protected(swhash->swevent_hlist,
5824 					  lockdep_is_held(&event->ctx->lock));
5825 	if (!hlist)
5826 		return NULL;
5827 
5828 	return __find_swevent_head(hlist, type, event_id);
5829 }
5830 
5831 static void do_perf_sw_event(enum perf_type_id type, u32 event_id,
5832 				    u64 nr,
5833 				    struct perf_sample_data *data,
5834 				    struct pt_regs *regs)
5835 {
5836 	struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable);
5837 	struct perf_event *event;
5838 	struct hlist_head *head;
5839 
5840 	rcu_read_lock();
5841 	head = find_swevent_head_rcu(swhash, type, event_id);
5842 	if (!head)
5843 		goto end;
5844 
5845 	hlist_for_each_entry_rcu(event, head, hlist_entry) {
5846 		if (perf_swevent_match(event, type, event_id, data, regs))
5847 			perf_swevent_event(event, nr, data, regs);
5848 	}
5849 end:
5850 	rcu_read_unlock();
5851 }
5852 
5853 int perf_swevent_get_recursion_context(void)
5854 {
5855 	struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable);
5856 
5857 	return get_recursion_context(swhash->recursion);
5858 }
5859 EXPORT_SYMBOL_GPL(perf_swevent_get_recursion_context);
5860 
5861 inline void perf_swevent_put_recursion_context(int rctx)
5862 {
5863 	struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable);
5864 
5865 	put_recursion_context(swhash->recursion, rctx);
5866 }
5867 
5868 void __perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr)
5869 {
5870 	struct perf_sample_data data;
5871 	int rctx;
5872 
5873 	preempt_disable_notrace();
5874 	rctx = perf_swevent_get_recursion_context();
5875 	if (rctx < 0)
5876 		return;
5877 
5878 	perf_sample_data_init(&data, addr, 0);
5879 
5880 	do_perf_sw_event(PERF_TYPE_SOFTWARE, event_id, nr, &data, regs);
5881 
5882 	perf_swevent_put_recursion_context(rctx);
5883 	preempt_enable_notrace();
5884 }
5885 
5886 static void perf_swevent_read(struct perf_event *event)
5887 {
5888 }
5889 
5890 static int perf_swevent_add(struct perf_event *event, int flags)
5891 {
5892 	struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable);
5893 	struct hw_perf_event *hwc = &event->hw;
5894 	struct hlist_head *head;
5895 
5896 	if (is_sampling_event(event)) {
5897 		hwc->last_period = hwc->sample_period;
5898 		perf_swevent_set_period(event);
5899 	}
5900 
5901 	hwc->state = !(flags & PERF_EF_START);
5902 
5903 	head = find_swevent_head(swhash, event);
5904 	if (!head) {
5905 		/*
5906 		 * We can race with cpu hotplug code. Do not
5907 		 * WARN if the cpu just got unplugged.
5908 		 */
5909 		WARN_ON_ONCE(swhash->online);
5910 		return -EINVAL;
5911 	}
5912 
5913 	hlist_add_head_rcu(&event->hlist_entry, head);
5914 
5915 	return 0;
5916 }
5917 
5918 static void perf_swevent_del(struct perf_event *event, int flags)
5919 {
5920 	hlist_del_rcu(&event->hlist_entry);
5921 }
5922 
5923 static void perf_swevent_start(struct perf_event *event, int flags)
5924 {
5925 	event->hw.state = 0;
5926 }
5927 
5928 static void perf_swevent_stop(struct perf_event *event, int flags)
5929 {
5930 	event->hw.state = PERF_HES_STOPPED;
5931 }
5932 
5933 /* Deref the hlist from the update side */
5934 static inline struct swevent_hlist *
5935 swevent_hlist_deref(struct swevent_htable *swhash)
5936 {
5937 	return rcu_dereference_protected(swhash->swevent_hlist,
5938 					 lockdep_is_held(&swhash->hlist_mutex));
5939 }
5940 
5941 static void swevent_hlist_release(struct swevent_htable *swhash)
5942 {
5943 	struct swevent_hlist *hlist = swevent_hlist_deref(swhash);
5944 
5945 	if (!hlist)
5946 		return;
5947 
5948 	RCU_INIT_POINTER(swhash->swevent_hlist, NULL);
5949 	kfree_rcu(hlist, rcu_head);
5950 }
5951 
5952 static void swevent_hlist_put_cpu(struct perf_event *event, int cpu)
5953 {
5954 	struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
5955 
5956 	mutex_lock(&swhash->hlist_mutex);
5957 
5958 	if (!--swhash->hlist_refcount)
5959 		swevent_hlist_release(swhash);
5960 
5961 	mutex_unlock(&swhash->hlist_mutex);
5962 }
5963 
5964 static void swevent_hlist_put(struct perf_event *event)
5965 {
5966 	int cpu;
5967 
5968 	for_each_possible_cpu(cpu)
5969 		swevent_hlist_put_cpu(event, cpu);
5970 }
5971 
5972 static int swevent_hlist_get_cpu(struct perf_event *event, int cpu)
5973 {
5974 	struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
5975 	int err = 0;
5976 
5977 	mutex_lock(&swhash->hlist_mutex);
5978 
5979 	if (!swevent_hlist_deref(swhash) && cpu_online(cpu)) {
5980 		struct swevent_hlist *hlist;
5981 
5982 		hlist = kzalloc(sizeof(*hlist), GFP_KERNEL);
5983 		if (!hlist) {
5984 			err = -ENOMEM;
5985 			goto exit;
5986 		}
5987 		rcu_assign_pointer(swhash->swevent_hlist, hlist);
5988 	}
5989 	swhash->hlist_refcount++;
5990 exit:
5991 	mutex_unlock(&swhash->hlist_mutex);
5992 
5993 	return err;
5994 }
5995 
5996 static int swevent_hlist_get(struct perf_event *event)
5997 {
5998 	int err;
5999 	int cpu, failed_cpu;
6000 
6001 	get_online_cpus();
6002 	for_each_possible_cpu(cpu) {
6003 		err = swevent_hlist_get_cpu(event, cpu);
6004 		if (err) {
6005 			failed_cpu = cpu;
6006 			goto fail;
6007 		}
6008 	}
6009 	put_online_cpus();
6010 
6011 	return 0;
6012 fail:
6013 	for_each_possible_cpu(cpu) {
6014 		if (cpu == failed_cpu)
6015 			break;
6016 		swevent_hlist_put_cpu(event, cpu);
6017 	}
6018 
6019 	put_online_cpus();
6020 	return err;
6021 }
6022 
6023 struct static_key perf_swevent_enabled[PERF_COUNT_SW_MAX];
6024 
6025 static void sw_perf_event_destroy(struct perf_event *event)
6026 {
6027 	u64 event_id = event->attr.config;
6028 
6029 	WARN_ON(event->parent);
6030 
6031 	static_key_slow_dec(&perf_swevent_enabled[event_id]);
6032 	swevent_hlist_put(event);
6033 }
6034 
6035 static int perf_swevent_init(struct perf_event *event)
6036 {
6037 	u64 event_id = event->attr.config;
6038 
6039 	if (event->attr.type != PERF_TYPE_SOFTWARE)
6040 		return -ENOENT;
6041 
6042 	/*
6043 	 * no branch sampling for software events
6044 	 */
6045 	if (has_branch_stack(event))
6046 		return -EOPNOTSUPP;
6047 
6048 	switch (event_id) {
6049 	case PERF_COUNT_SW_CPU_CLOCK:
6050 	case PERF_COUNT_SW_TASK_CLOCK:
6051 		return -ENOENT;
6052 
6053 	default:
6054 		break;
6055 	}
6056 
6057 	if (event_id >= PERF_COUNT_SW_MAX)
6058 		return -ENOENT;
6059 
6060 	if (!event->parent) {
6061 		int err;
6062 
6063 		err = swevent_hlist_get(event);
6064 		if (err)
6065 			return err;
6066 
6067 		static_key_slow_inc(&perf_swevent_enabled[event_id]);
6068 		event->destroy = sw_perf_event_destroy;
6069 	}
6070 
6071 	return 0;
6072 }
6073 
6074 static struct pmu perf_swevent = {
6075 	.task_ctx_nr	= perf_sw_context,
6076 
6077 	.event_init	= perf_swevent_init,
6078 	.add		= perf_swevent_add,
6079 	.del		= perf_swevent_del,
6080 	.start		= perf_swevent_start,
6081 	.stop		= perf_swevent_stop,
6082 	.read		= perf_swevent_read,
6083 };
6084 
6085 #ifdef CONFIG_EVENT_TRACING
6086 
6087 static int perf_tp_filter_match(struct perf_event *event,
6088 				struct perf_sample_data *data)
6089 {
6090 	void *record = data->raw->data;
6091 
6092 	if (likely(!event->filter) || filter_match_preds(event->filter, record))
6093 		return 1;
6094 	return 0;
6095 }
6096 
6097 static int perf_tp_event_match(struct perf_event *event,
6098 				struct perf_sample_data *data,
6099 				struct pt_regs *regs)
6100 {
6101 	if (event->hw.state & PERF_HES_STOPPED)
6102 		return 0;
6103 	/*
6104 	 * All tracepoints are from kernel-space.
6105 	 */
6106 	if (event->attr.exclude_kernel)
6107 		return 0;
6108 
6109 	if (!perf_tp_filter_match(event, data))
6110 		return 0;
6111 
6112 	return 1;
6113 }
6114 
6115 void perf_tp_event(u64 addr, u64 count, void *record, int entry_size,
6116 		   struct pt_regs *regs, struct hlist_head *head, int rctx,
6117 		   struct task_struct *task)
6118 {
6119 	struct perf_sample_data data;
6120 	struct perf_event *event;
6121 
6122 	struct perf_raw_record raw = {
6123 		.size = entry_size,
6124 		.data = record,
6125 	};
6126 
6127 	perf_sample_data_init(&data, addr, 0);
6128 	data.raw = &raw;
6129 
6130 	hlist_for_each_entry_rcu(event, head, hlist_entry) {
6131 		if (perf_tp_event_match(event, &data, regs))
6132 			perf_swevent_event(event, count, &data, regs);
6133 	}
6134 
6135 	/*
6136 	 * If we got specified a target task, also iterate its context and
6137 	 * deliver this event there too.
6138 	 */
6139 	if (task && task != current) {
6140 		struct perf_event_context *ctx;
6141 		struct trace_entry *entry = record;
6142 
6143 		rcu_read_lock();
6144 		ctx = rcu_dereference(task->perf_event_ctxp[perf_sw_context]);
6145 		if (!ctx)
6146 			goto unlock;
6147 
6148 		list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
6149 			if (event->attr.type != PERF_TYPE_TRACEPOINT)
6150 				continue;
6151 			if (event->attr.config != entry->type)
6152 				continue;
6153 			if (perf_tp_event_match(event, &data, regs))
6154 				perf_swevent_event(event, count, &data, regs);
6155 		}
6156 unlock:
6157 		rcu_read_unlock();
6158 	}
6159 
6160 	perf_swevent_put_recursion_context(rctx);
6161 }
6162 EXPORT_SYMBOL_GPL(perf_tp_event);
6163 
6164 static void tp_perf_event_destroy(struct perf_event *event)
6165 {
6166 	perf_trace_destroy(event);
6167 }
6168 
6169 static int perf_tp_event_init(struct perf_event *event)
6170 {
6171 	int err;
6172 
6173 	if (event->attr.type != PERF_TYPE_TRACEPOINT)
6174 		return -ENOENT;
6175 
6176 	/*
6177 	 * no branch sampling for tracepoint events
6178 	 */
6179 	if (has_branch_stack(event))
6180 		return -EOPNOTSUPP;
6181 
6182 	err = perf_trace_init(event);
6183 	if (err)
6184 		return err;
6185 
6186 	event->destroy = tp_perf_event_destroy;
6187 
6188 	return 0;
6189 }
6190 
6191 static struct pmu perf_tracepoint = {
6192 	.task_ctx_nr	= perf_sw_context,
6193 
6194 	.event_init	= perf_tp_event_init,
6195 	.add		= perf_trace_add,
6196 	.del		= perf_trace_del,
6197 	.start		= perf_swevent_start,
6198 	.stop		= perf_swevent_stop,
6199 	.read		= perf_swevent_read,
6200 };
6201 
6202 static inline void perf_tp_register(void)
6203 {
6204 	perf_pmu_register(&perf_tracepoint, "tracepoint", PERF_TYPE_TRACEPOINT);
6205 }
6206 
6207 static int perf_event_set_filter(struct perf_event *event, void __user *arg)
6208 {
6209 	char *filter_str;
6210 	int ret;
6211 
6212 	if (event->attr.type != PERF_TYPE_TRACEPOINT)
6213 		return -EINVAL;
6214 
6215 	filter_str = strndup_user(arg, PAGE_SIZE);
6216 	if (IS_ERR(filter_str))
6217 		return PTR_ERR(filter_str);
6218 
6219 	ret = ftrace_profile_set_filter(event, event->attr.config, filter_str);
6220 
6221 	kfree(filter_str);
6222 	return ret;
6223 }
6224 
6225 static void perf_event_free_filter(struct perf_event *event)
6226 {
6227 	ftrace_profile_free_filter(event);
6228 }
6229 
6230 #else
6231 
6232 static inline void perf_tp_register(void)
6233 {
6234 }
6235 
6236 static int perf_event_set_filter(struct perf_event *event, void __user *arg)
6237 {
6238 	return -ENOENT;
6239 }
6240 
6241 static void perf_event_free_filter(struct perf_event *event)
6242 {
6243 }
6244 
6245 #endif /* CONFIG_EVENT_TRACING */
6246 
6247 #ifdef CONFIG_HAVE_HW_BREAKPOINT
6248 void perf_bp_event(struct perf_event *bp, void *data)
6249 {
6250 	struct perf_sample_data sample;
6251 	struct pt_regs *regs = data;
6252 
6253 	perf_sample_data_init(&sample, bp->attr.bp_addr, 0);
6254 
6255 	if (!bp->hw.state && !perf_exclude_event(bp, regs))
6256 		perf_swevent_event(bp, 1, &sample, regs);
6257 }
6258 #endif
6259 
6260 /*
6261  * hrtimer based swevent callback
6262  */
6263 
6264 static enum hrtimer_restart perf_swevent_hrtimer(struct hrtimer *hrtimer)
6265 {
6266 	enum hrtimer_restart ret = HRTIMER_RESTART;
6267 	struct perf_sample_data data;
6268 	struct pt_regs *regs;
6269 	struct perf_event *event;
6270 	u64 period;
6271 
6272 	event = container_of(hrtimer, struct perf_event, hw.hrtimer);
6273 
6274 	if (event->state != PERF_EVENT_STATE_ACTIVE)
6275 		return HRTIMER_NORESTART;
6276 
6277 	event->pmu->read(event);
6278 
6279 	perf_sample_data_init(&data, 0, event->hw.last_period);
6280 	regs = get_irq_regs();
6281 
6282 	if (regs && !perf_exclude_event(event, regs)) {
6283 		if (!(event->attr.exclude_idle && is_idle_task(current)))
6284 			if (__perf_event_overflow(event, 1, &data, regs))
6285 				ret = HRTIMER_NORESTART;
6286 	}
6287 
6288 	period = max_t(u64, 10000, event->hw.sample_period);
6289 	hrtimer_forward_now(hrtimer, ns_to_ktime(period));
6290 
6291 	return ret;
6292 }
6293 
6294 static void perf_swevent_start_hrtimer(struct perf_event *event)
6295 {
6296 	struct hw_perf_event *hwc = &event->hw;
6297 	s64 period;
6298 
6299 	if (!is_sampling_event(event))
6300 		return;
6301 
6302 	period = local64_read(&hwc->period_left);
6303 	if (period) {
6304 		if (period < 0)
6305 			period = 10000;
6306 
6307 		local64_set(&hwc->period_left, 0);
6308 	} else {
6309 		period = max_t(u64, 10000, hwc->sample_period);
6310 	}
6311 	__hrtimer_start_range_ns(&hwc->hrtimer,
6312 				ns_to_ktime(period), 0,
6313 				HRTIMER_MODE_REL_PINNED, 0);
6314 }
6315 
6316 static void perf_swevent_cancel_hrtimer(struct perf_event *event)
6317 {
6318 	struct hw_perf_event *hwc = &event->hw;
6319 
6320 	if (is_sampling_event(event)) {
6321 		ktime_t remaining = hrtimer_get_remaining(&hwc->hrtimer);
6322 		local64_set(&hwc->period_left, ktime_to_ns(remaining));
6323 
6324 		hrtimer_cancel(&hwc->hrtimer);
6325 	}
6326 }
6327 
6328 static void perf_swevent_init_hrtimer(struct perf_event *event)
6329 {
6330 	struct hw_perf_event *hwc = &event->hw;
6331 
6332 	if (!is_sampling_event(event))
6333 		return;
6334 
6335 	hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
6336 	hwc->hrtimer.function = perf_swevent_hrtimer;
6337 
6338 	/*
6339 	 * Since hrtimers have a fixed rate, we can do a static freq->period
6340 	 * mapping and avoid the whole period adjust feedback stuff.
6341 	 */
6342 	if (event->attr.freq) {
6343 		long freq = event->attr.sample_freq;
6344 
6345 		event->attr.sample_period = NSEC_PER_SEC / freq;
6346 		hwc->sample_period = event->attr.sample_period;
6347 		local64_set(&hwc->period_left, hwc->sample_period);
6348 		hwc->last_period = hwc->sample_period;
6349 		event->attr.freq = 0;
6350 	}
6351 }
6352 
6353 /*
6354  * Software event: cpu wall time clock
6355  */
6356 
6357 static void cpu_clock_event_update(struct perf_event *event)
6358 {
6359 	s64 prev;
6360 	u64 now;
6361 
6362 	now = local_clock();
6363 	prev = local64_xchg(&event->hw.prev_count, now);
6364 	local64_add(now - prev, &event->count);
6365 }
6366 
6367 static void cpu_clock_event_start(struct perf_event *event, int flags)
6368 {
6369 	local64_set(&event->hw.prev_count, local_clock());
6370 	perf_swevent_start_hrtimer(event);
6371 }
6372 
6373 static void cpu_clock_event_stop(struct perf_event *event, int flags)
6374 {
6375 	perf_swevent_cancel_hrtimer(event);
6376 	cpu_clock_event_update(event);
6377 }
6378 
6379 static int cpu_clock_event_add(struct perf_event *event, int flags)
6380 {
6381 	if (flags & PERF_EF_START)
6382 		cpu_clock_event_start(event, flags);
6383 
6384 	return 0;
6385 }
6386 
6387 static void cpu_clock_event_del(struct perf_event *event, int flags)
6388 {
6389 	cpu_clock_event_stop(event, flags);
6390 }
6391 
6392 static void cpu_clock_event_read(struct perf_event *event)
6393 {
6394 	cpu_clock_event_update(event);
6395 }
6396 
6397 static int cpu_clock_event_init(struct perf_event *event)
6398 {
6399 	if (event->attr.type != PERF_TYPE_SOFTWARE)
6400 		return -ENOENT;
6401 
6402 	if (event->attr.config != PERF_COUNT_SW_CPU_CLOCK)
6403 		return -ENOENT;
6404 
6405 	/*
6406 	 * no branch sampling for software events
6407 	 */
6408 	if (has_branch_stack(event))
6409 		return -EOPNOTSUPP;
6410 
6411 	perf_swevent_init_hrtimer(event);
6412 
6413 	return 0;
6414 }
6415 
6416 static struct pmu perf_cpu_clock = {
6417 	.task_ctx_nr	= perf_sw_context,
6418 
6419 	.event_init	= cpu_clock_event_init,
6420 	.add		= cpu_clock_event_add,
6421 	.del		= cpu_clock_event_del,
6422 	.start		= cpu_clock_event_start,
6423 	.stop		= cpu_clock_event_stop,
6424 	.read		= cpu_clock_event_read,
6425 };
6426 
6427 /*
6428  * Software event: task time clock
6429  */
6430 
6431 static void task_clock_event_update(struct perf_event *event, u64 now)
6432 {
6433 	u64 prev;
6434 	s64 delta;
6435 
6436 	prev = local64_xchg(&event->hw.prev_count, now);
6437 	delta = now - prev;
6438 	local64_add(delta, &event->count);
6439 }
6440 
6441 static void task_clock_event_start(struct perf_event *event, int flags)
6442 {
6443 	local64_set(&event->hw.prev_count, event->ctx->time);
6444 	perf_swevent_start_hrtimer(event);
6445 }
6446 
6447 static void task_clock_event_stop(struct perf_event *event, int flags)
6448 {
6449 	perf_swevent_cancel_hrtimer(event);
6450 	task_clock_event_update(event, event->ctx->time);
6451 }
6452 
6453 static int task_clock_event_add(struct perf_event *event, int flags)
6454 {
6455 	if (flags & PERF_EF_START)
6456 		task_clock_event_start(event, flags);
6457 
6458 	return 0;
6459 }
6460 
6461 static void task_clock_event_del(struct perf_event *event, int flags)
6462 {
6463 	task_clock_event_stop(event, PERF_EF_UPDATE);
6464 }
6465 
6466 static void task_clock_event_read(struct perf_event *event)
6467 {
6468 	u64 now = perf_clock();
6469 	u64 delta = now - event->ctx->timestamp;
6470 	u64 time = event->ctx->time + delta;
6471 
6472 	task_clock_event_update(event, time);
6473 }
6474 
6475 static int task_clock_event_init(struct perf_event *event)
6476 {
6477 	if (event->attr.type != PERF_TYPE_SOFTWARE)
6478 		return -ENOENT;
6479 
6480 	if (event->attr.config != PERF_COUNT_SW_TASK_CLOCK)
6481 		return -ENOENT;
6482 
6483 	/*
6484 	 * no branch sampling for software events
6485 	 */
6486 	if (has_branch_stack(event))
6487 		return -EOPNOTSUPP;
6488 
6489 	perf_swevent_init_hrtimer(event);
6490 
6491 	return 0;
6492 }
6493 
6494 static struct pmu perf_task_clock = {
6495 	.task_ctx_nr	= perf_sw_context,
6496 
6497 	.event_init	= task_clock_event_init,
6498 	.add		= task_clock_event_add,
6499 	.del		= task_clock_event_del,
6500 	.start		= task_clock_event_start,
6501 	.stop		= task_clock_event_stop,
6502 	.read		= task_clock_event_read,
6503 };
6504 
6505 static void perf_pmu_nop_void(struct pmu *pmu)
6506 {
6507 }
6508 
6509 static int perf_pmu_nop_int(struct pmu *pmu)
6510 {
6511 	return 0;
6512 }
6513 
6514 static void perf_pmu_start_txn(struct pmu *pmu)
6515 {
6516 	perf_pmu_disable(pmu);
6517 }
6518 
6519 static int perf_pmu_commit_txn(struct pmu *pmu)
6520 {
6521 	perf_pmu_enable(pmu);
6522 	return 0;
6523 }
6524 
6525 static void perf_pmu_cancel_txn(struct pmu *pmu)
6526 {
6527 	perf_pmu_enable(pmu);
6528 }
6529 
6530 static int perf_event_idx_default(struct perf_event *event)
6531 {
6532 	return 0;
6533 }
6534 
6535 /*
6536  * Ensures all contexts with the same task_ctx_nr have the same
6537  * pmu_cpu_context too.
6538  */
6539 static struct perf_cpu_context __percpu *find_pmu_context(int ctxn)
6540 {
6541 	struct pmu *pmu;
6542 
6543 	if (ctxn < 0)
6544 		return NULL;
6545 
6546 	list_for_each_entry(pmu, &pmus, entry) {
6547 		if (pmu->task_ctx_nr == ctxn)
6548 			return pmu->pmu_cpu_context;
6549 	}
6550 
6551 	return NULL;
6552 }
6553 
6554 static void update_pmu_context(struct pmu *pmu, struct pmu *old_pmu)
6555 {
6556 	int cpu;
6557 
6558 	for_each_possible_cpu(cpu) {
6559 		struct perf_cpu_context *cpuctx;
6560 
6561 		cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
6562 
6563 		if (cpuctx->unique_pmu == old_pmu)
6564 			cpuctx->unique_pmu = pmu;
6565 	}
6566 }
6567 
6568 static void free_pmu_context(struct pmu *pmu)
6569 {
6570 	struct pmu *i;
6571 
6572 	mutex_lock(&pmus_lock);
6573 	/*
6574 	 * Like a real lame refcount.
6575 	 */
6576 	list_for_each_entry(i, &pmus, entry) {
6577 		if (i->pmu_cpu_context == pmu->pmu_cpu_context) {
6578 			update_pmu_context(i, pmu);
6579 			goto out;
6580 		}
6581 	}
6582 
6583 	free_percpu(pmu->pmu_cpu_context);
6584 out:
6585 	mutex_unlock(&pmus_lock);
6586 }
6587 static struct idr pmu_idr;
6588 
6589 static ssize_t
6590 type_show(struct device *dev, struct device_attribute *attr, char *page)
6591 {
6592 	struct pmu *pmu = dev_get_drvdata(dev);
6593 
6594 	return snprintf(page, PAGE_SIZE-1, "%d\n", pmu->type);
6595 }
6596 static DEVICE_ATTR_RO(type);
6597 
6598 static ssize_t
6599 perf_event_mux_interval_ms_show(struct device *dev,
6600 				struct device_attribute *attr,
6601 				char *page)
6602 {
6603 	struct pmu *pmu = dev_get_drvdata(dev);
6604 
6605 	return snprintf(page, PAGE_SIZE-1, "%d\n", pmu->hrtimer_interval_ms);
6606 }
6607 
6608 static ssize_t
6609 perf_event_mux_interval_ms_store(struct device *dev,
6610 				 struct device_attribute *attr,
6611 				 const char *buf, size_t count)
6612 {
6613 	struct pmu *pmu = dev_get_drvdata(dev);
6614 	int timer, cpu, ret;
6615 
6616 	ret = kstrtoint(buf, 0, &timer);
6617 	if (ret)
6618 		return ret;
6619 
6620 	if (timer < 1)
6621 		return -EINVAL;
6622 
6623 	/* same value, noting to do */
6624 	if (timer == pmu->hrtimer_interval_ms)
6625 		return count;
6626 
6627 	pmu->hrtimer_interval_ms = timer;
6628 
6629 	/* update all cpuctx for this PMU */
6630 	for_each_possible_cpu(cpu) {
6631 		struct perf_cpu_context *cpuctx;
6632 		cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
6633 		cpuctx->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * timer);
6634 
6635 		if (hrtimer_active(&cpuctx->hrtimer))
6636 			hrtimer_forward_now(&cpuctx->hrtimer, cpuctx->hrtimer_interval);
6637 	}
6638 
6639 	return count;
6640 }
6641 static DEVICE_ATTR_RW(perf_event_mux_interval_ms);
6642 
6643 static struct attribute *pmu_dev_attrs[] = {
6644 	&dev_attr_type.attr,
6645 	&dev_attr_perf_event_mux_interval_ms.attr,
6646 	NULL,
6647 };
6648 ATTRIBUTE_GROUPS(pmu_dev);
6649 
6650 static int pmu_bus_running;
6651 static struct bus_type pmu_bus = {
6652 	.name		= "event_source",
6653 	.dev_groups	= pmu_dev_groups,
6654 };
6655 
6656 static void pmu_dev_release(struct device *dev)
6657 {
6658 	kfree(dev);
6659 }
6660 
6661 static int pmu_dev_alloc(struct pmu *pmu)
6662 {
6663 	int ret = -ENOMEM;
6664 
6665 	pmu->dev = kzalloc(sizeof(struct device), GFP_KERNEL);
6666 	if (!pmu->dev)
6667 		goto out;
6668 
6669 	pmu->dev->groups = pmu->attr_groups;
6670 	device_initialize(pmu->dev);
6671 	ret = dev_set_name(pmu->dev, "%s", pmu->name);
6672 	if (ret)
6673 		goto free_dev;
6674 
6675 	dev_set_drvdata(pmu->dev, pmu);
6676 	pmu->dev->bus = &pmu_bus;
6677 	pmu->dev->release = pmu_dev_release;
6678 	ret = device_add(pmu->dev);
6679 	if (ret)
6680 		goto free_dev;
6681 
6682 out:
6683 	return ret;
6684 
6685 free_dev:
6686 	put_device(pmu->dev);
6687 	goto out;
6688 }
6689 
6690 static struct lock_class_key cpuctx_mutex;
6691 static struct lock_class_key cpuctx_lock;
6692 
6693 int perf_pmu_register(struct pmu *pmu, const char *name, int type)
6694 {
6695 	int cpu, ret;
6696 
6697 	mutex_lock(&pmus_lock);
6698 	ret = -ENOMEM;
6699 	pmu->pmu_disable_count = alloc_percpu(int);
6700 	if (!pmu->pmu_disable_count)
6701 		goto unlock;
6702 
6703 	pmu->type = -1;
6704 	if (!name)
6705 		goto skip_type;
6706 	pmu->name = name;
6707 
6708 	if (type < 0) {
6709 		type = idr_alloc(&pmu_idr, pmu, PERF_TYPE_MAX, 0, GFP_KERNEL);
6710 		if (type < 0) {
6711 			ret = type;
6712 			goto free_pdc;
6713 		}
6714 	}
6715 	pmu->type = type;
6716 
6717 	if (pmu_bus_running) {
6718 		ret = pmu_dev_alloc(pmu);
6719 		if (ret)
6720 			goto free_idr;
6721 	}
6722 
6723 skip_type:
6724 	pmu->pmu_cpu_context = find_pmu_context(pmu->task_ctx_nr);
6725 	if (pmu->pmu_cpu_context)
6726 		goto got_cpu_context;
6727 
6728 	ret = -ENOMEM;
6729 	pmu->pmu_cpu_context = alloc_percpu(struct perf_cpu_context);
6730 	if (!pmu->pmu_cpu_context)
6731 		goto free_dev;
6732 
6733 	for_each_possible_cpu(cpu) {
6734 		struct perf_cpu_context *cpuctx;
6735 
6736 		cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
6737 		__perf_event_init_context(&cpuctx->ctx);
6738 		lockdep_set_class(&cpuctx->ctx.mutex, &cpuctx_mutex);
6739 		lockdep_set_class(&cpuctx->ctx.lock, &cpuctx_lock);
6740 		cpuctx->ctx.type = cpu_context;
6741 		cpuctx->ctx.pmu = pmu;
6742 
6743 		__perf_cpu_hrtimer_init(cpuctx, cpu);
6744 
6745 		INIT_LIST_HEAD(&cpuctx->rotation_list);
6746 		cpuctx->unique_pmu = pmu;
6747 	}
6748 
6749 got_cpu_context:
6750 	if (!pmu->start_txn) {
6751 		if (pmu->pmu_enable) {
6752 			/*
6753 			 * If we have pmu_enable/pmu_disable calls, install
6754 			 * transaction stubs that use that to try and batch
6755 			 * hardware accesses.
6756 			 */
6757 			pmu->start_txn  = perf_pmu_start_txn;
6758 			pmu->commit_txn = perf_pmu_commit_txn;
6759 			pmu->cancel_txn = perf_pmu_cancel_txn;
6760 		} else {
6761 			pmu->start_txn  = perf_pmu_nop_void;
6762 			pmu->commit_txn = perf_pmu_nop_int;
6763 			pmu->cancel_txn = perf_pmu_nop_void;
6764 		}
6765 	}
6766 
6767 	if (!pmu->pmu_enable) {
6768 		pmu->pmu_enable  = perf_pmu_nop_void;
6769 		pmu->pmu_disable = perf_pmu_nop_void;
6770 	}
6771 
6772 	if (!pmu->event_idx)
6773 		pmu->event_idx = perf_event_idx_default;
6774 
6775 	list_add_rcu(&pmu->entry, &pmus);
6776 	ret = 0;
6777 unlock:
6778 	mutex_unlock(&pmus_lock);
6779 
6780 	return ret;
6781 
6782 free_dev:
6783 	device_del(pmu->dev);
6784 	put_device(pmu->dev);
6785 
6786 free_idr:
6787 	if (pmu->type >= PERF_TYPE_MAX)
6788 		idr_remove(&pmu_idr, pmu->type);
6789 
6790 free_pdc:
6791 	free_percpu(pmu->pmu_disable_count);
6792 	goto unlock;
6793 }
6794 EXPORT_SYMBOL_GPL(perf_pmu_register);
6795 
6796 void perf_pmu_unregister(struct pmu *pmu)
6797 {
6798 	mutex_lock(&pmus_lock);
6799 	list_del_rcu(&pmu->entry);
6800 	mutex_unlock(&pmus_lock);
6801 
6802 	/*
6803 	 * We dereference the pmu list under both SRCU and regular RCU, so
6804 	 * synchronize against both of those.
6805 	 */
6806 	synchronize_srcu(&pmus_srcu);
6807 	synchronize_rcu();
6808 
6809 	free_percpu(pmu->pmu_disable_count);
6810 	if (pmu->type >= PERF_TYPE_MAX)
6811 		idr_remove(&pmu_idr, pmu->type);
6812 	device_del(pmu->dev);
6813 	put_device(pmu->dev);
6814 	free_pmu_context(pmu);
6815 }
6816 EXPORT_SYMBOL_GPL(perf_pmu_unregister);
6817 
6818 struct pmu *perf_init_event(struct perf_event *event)
6819 {
6820 	struct pmu *pmu = NULL;
6821 	int idx;
6822 	int ret;
6823 
6824 	idx = srcu_read_lock(&pmus_srcu);
6825 
6826 	rcu_read_lock();
6827 	pmu = idr_find(&pmu_idr, event->attr.type);
6828 	rcu_read_unlock();
6829 	if (pmu) {
6830 		if (!try_module_get(pmu->module)) {
6831 			pmu = ERR_PTR(-ENODEV);
6832 			goto unlock;
6833 		}
6834 		event->pmu = pmu;
6835 		ret = pmu->event_init(event);
6836 		if (ret)
6837 			pmu = ERR_PTR(ret);
6838 		goto unlock;
6839 	}
6840 
6841 	list_for_each_entry_rcu(pmu, &pmus, entry) {
6842 		if (!try_module_get(pmu->module)) {
6843 			pmu = ERR_PTR(-ENODEV);
6844 			goto unlock;
6845 		}
6846 		event->pmu = pmu;
6847 		ret = pmu->event_init(event);
6848 		if (!ret)
6849 			goto unlock;
6850 
6851 		if (ret != -ENOENT) {
6852 			pmu = ERR_PTR(ret);
6853 			goto unlock;
6854 		}
6855 	}
6856 	pmu = ERR_PTR(-ENOENT);
6857 unlock:
6858 	srcu_read_unlock(&pmus_srcu, idx);
6859 
6860 	return pmu;
6861 }
6862 
6863 static void account_event_cpu(struct perf_event *event, int cpu)
6864 {
6865 	if (event->parent)
6866 		return;
6867 
6868 	if (has_branch_stack(event)) {
6869 		if (!(event->attach_state & PERF_ATTACH_TASK))
6870 			atomic_inc(&per_cpu(perf_branch_stack_events, cpu));
6871 	}
6872 	if (is_cgroup_event(event))
6873 		atomic_inc(&per_cpu(perf_cgroup_events, cpu));
6874 }
6875 
6876 static void account_event(struct perf_event *event)
6877 {
6878 	if (event->parent)
6879 		return;
6880 
6881 	if (event->attach_state & PERF_ATTACH_TASK)
6882 		static_key_slow_inc(&perf_sched_events.key);
6883 	if (event->attr.mmap || event->attr.mmap_data)
6884 		atomic_inc(&nr_mmap_events);
6885 	if (event->attr.comm)
6886 		atomic_inc(&nr_comm_events);
6887 	if (event->attr.task)
6888 		atomic_inc(&nr_task_events);
6889 	if (event->attr.freq) {
6890 		if (atomic_inc_return(&nr_freq_events) == 1)
6891 			tick_nohz_full_kick_all();
6892 	}
6893 	if (has_branch_stack(event))
6894 		static_key_slow_inc(&perf_sched_events.key);
6895 	if (is_cgroup_event(event))
6896 		static_key_slow_inc(&perf_sched_events.key);
6897 
6898 	account_event_cpu(event, event->cpu);
6899 }
6900 
6901 /*
6902  * Allocate and initialize a event structure
6903  */
6904 static struct perf_event *
6905 perf_event_alloc(struct perf_event_attr *attr, int cpu,
6906 		 struct task_struct *task,
6907 		 struct perf_event *group_leader,
6908 		 struct perf_event *parent_event,
6909 		 perf_overflow_handler_t overflow_handler,
6910 		 void *context)
6911 {
6912 	struct pmu *pmu;
6913 	struct perf_event *event;
6914 	struct hw_perf_event *hwc;
6915 	long err = -EINVAL;
6916 
6917 	if ((unsigned)cpu >= nr_cpu_ids) {
6918 		if (!task || cpu != -1)
6919 			return ERR_PTR(-EINVAL);
6920 	}
6921 
6922 	event = kzalloc(sizeof(*event), GFP_KERNEL);
6923 	if (!event)
6924 		return ERR_PTR(-ENOMEM);
6925 
6926 	/*
6927 	 * Single events are their own group leaders, with an
6928 	 * empty sibling list:
6929 	 */
6930 	if (!group_leader)
6931 		group_leader = event;
6932 
6933 	mutex_init(&event->child_mutex);
6934 	INIT_LIST_HEAD(&event->child_list);
6935 
6936 	INIT_LIST_HEAD(&event->group_entry);
6937 	INIT_LIST_HEAD(&event->event_entry);
6938 	INIT_LIST_HEAD(&event->sibling_list);
6939 	INIT_LIST_HEAD(&event->rb_entry);
6940 	INIT_LIST_HEAD(&event->active_entry);
6941 	INIT_HLIST_NODE(&event->hlist_entry);
6942 
6943 
6944 	init_waitqueue_head(&event->waitq);
6945 	init_irq_work(&event->pending, perf_pending_event);
6946 
6947 	mutex_init(&event->mmap_mutex);
6948 
6949 	atomic_long_set(&event->refcount, 1);
6950 	event->cpu		= cpu;
6951 	event->attr		= *attr;
6952 	event->group_leader	= group_leader;
6953 	event->pmu		= NULL;
6954 	event->oncpu		= -1;
6955 
6956 	event->parent		= parent_event;
6957 
6958 	event->ns		= get_pid_ns(task_active_pid_ns(current));
6959 	event->id		= atomic64_inc_return(&perf_event_id);
6960 
6961 	event->state		= PERF_EVENT_STATE_INACTIVE;
6962 
6963 	if (task) {
6964 		event->attach_state = PERF_ATTACH_TASK;
6965 
6966 		if (attr->type == PERF_TYPE_TRACEPOINT)
6967 			event->hw.tp_target = task;
6968 #ifdef CONFIG_HAVE_HW_BREAKPOINT
6969 		/*
6970 		 * hw_breakpoint is a bit difficult here..
6971 		 */
6972 		else if (attr->type == PERF_TYPE_BREAKPOINT)
6973 			event->hw.bp_target = task;
6974 #endif
6975 	}
6976 
6977 	if (!overflow_handler && parent_event) {
6978 		overflow_handler = parent_event->overflow_handler;
6979 		context = parent_event->overflow_handler_context;
6980 	}
6981 
6982 	event->overflow_handler	= overflow_handler;
6983 	event->overflow_handler_context = context;
6984 
6985 	perf_event__state_init(event);
6986 
6987 	pmu = NULL;
6988 
6989 	hwc = &event->hw;
6990 	hwc->sample_period = attr->sample_period;
6991 	if (attr->freq && attr->sample_freq)
6992 		hwc->sample_period = 1;
6993 	hwc->last_period = hwc->sample_period;
6994 
6995 	local64_set(&hwc->period_left, hwc->sample_period);
6996 
6997 	/*
6998 	 * we currently do not support PERF_FORMAT_GROUP on inherited events
6999 	 */
7000 	if (attr->inherit && (attr->read_format & PERF_FORMAT_GROUP))
7001 		goto err_ns;
7002 
7003 	pmu = perf_init_event(event);
7004 	if (!pmu)
7005 		goto err_ns;
7006 	else if (IS_ERR(pmu)) {
7007 		err = PTR_ERR(pmu);
7008 		goto err_ns;
7009 	}
7010 
7011 	if (!event->parent) {
7012 		if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) {
7013 			err = get_callchain_buffers();
7014 			if (err)
7015 				goto err_pmu;
7016 		}
7017 	}
7018 
7019 	return event;
7020 
7021 err_pmu:
7022 	if (event->destroy)
7023 		event->destroy(event);
7024 	module_put(pmu->module);
7025 err_ns:
7026 	if (event->ns)
7027 		put_pid_ns(event->ns);
7028 	kfree(event);
7029 
7030 	return ERR_PTR(err);
7031 }
7032 
7033 static int perf_copy_attr(struct perf_event_attr __user *uattr,
7034 			  struct perf_event_attr *attr)
7035 {
7036 	u32 size;
7037 	int ret;
7038 
7039 	if (!access_ok(VERIFY_WRITE, uattr, PERF_ATTR_SIZE_VER0))
7040 		return -EFAULT;
7041 
7042 	/*
7043 	 * zero the full structure, so that a short copy will be nice.
7044 	 */
7045 	memset(attr, 0, sizeof(*attr));
7046 
7047 	ret = get_user(size, &uattr->size);
7048 	if (ret)
7049 		return ret;
7050 
7051 	if (size > PAGE_SIZE)	/* silly large */
7052 		goto err_size;
7053 
7054 	if (!size)		/* abi compat */
7055 		size = PERF_ATTR_SIZE_VER0;
7056 
7057 	if (size < PERF_ATTR_SIZE_VER0)
7058 		goto err_size;
7059 
7060 	/*
7061 	 * If we're handed a bigger struct than we know of,
7062 	 * ensure all the unknown bits are 0 - i.e. new
7063 	 * user-space does not rely on any kernel feature
7064 	 * extensions we dont know about yet.
7065 	 */
7066 	if (size > sizeof(*attr)) {
7067 		unsigned char __user *addr;
7068 		unsigned char __user *end;
7069 		unsigned char val;
7070 
7071 		addr = (void __user *)uattr + sizeof(*attr);
7072 		end  = (void __user *)uattr + size;
7073 
7074 		for (; addr < end; addr++) {
7075 			ret = get_user(val, addr);
7076 			if (ret)
7077 				return ret;
7078 			if (val)
7079 				goto err_size;
7080 		}
7081 		size = sizeof(*attr);
7082 	}
7083 
7084 	ret = copy_from_user(attr, uattr, size);
7085 	if (ret)
7086 		return -EFAULT;
7087 
7088 	if (attr->__reserved_1)
7089 		return -EINVAL;
7090 
7091 	if (attr->sample_type & ~(PERF_SAMPLE_MAX-1))
7092 		return -EINVAL;
7093 
7094 	if (attr->read_format & ~(PERF_FORMAT_MAX-1))
7095 		return -EINVAL;
7096 
7097 	if (attr->sample_type & PERF_SAMPLE_BRANCH_STACK) {
7098 		u64 mask = attr->branch_sample_type;
7099 
7100 		/* only using defined bits */
7101 		if (mask & ~(PERF_SAMPLE_BRANCH_MAX-1))
7102 			return -EINVAL;
7103 
7104 		/* at least one branch bit must be set */
7105 		if (!(mask & ~PERF_SAMPLE_BRANCH_PLM_ALL))
7106 			return -EINVAL;
7107 
7108 		/* propagate priv level, when not set for branch */
7109 		if (!(mask & PERF_SAMPLE_BRANCH_PLM_ALL)) {
7110 
7111 			/* exclude_kernel checked on syscall entry */
7112 			if (!attr->exclude_kernel)
7113 				mask |= PERF_SAMPLE_BRANCH_KERNEL;
7114 
7115 			if (!attr->exclude_user)
7116 				mask |= PERF_SAMPLE_BRANCH_USER;
7117 
7118 			if (!attr->exclude_hv)
7119 				mask |= PERF_SAMPLE_BRANCH_HV;
7120 			/*
7121 			 * adjust user setting (for HW filter setup)
7122 			 */
7123 			attr->branch_sample_type = mask;
7124 		}
7125 		/* privileged levels capture (kernel, hv): check permissions */
7126 		if ((mask & PERF_SAMPLE_BRANCH_PERM_PLM)
7127 		    && perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN))
7128 			return -EACCES;
7129 	}
7130 
7131 	if (attr->sample_type & PERF_SAMPLE_REGS_USER) {
7132 		ret = perf_reg_validate(attr->sample_regs_user);
7133 		if (ret)
7134 			return ret;
7135 	}
7136 
7137 	if (attr->sample_type & PERF_SAMPLE_STACK_USER) {
7138 		if (!arch_perf_have_user_stack_dump())
7139 			return -ENOSYS;
7140 
7141 		/*
7142 		 * We have __u32 type for the size, but so far
7143 		 * we can only use __u16 as maximum due to the
7144 		 * __u16 sample size limit.
7145 		 */
7146 		if (attr->sample_stack_user >= USHRT_MAX)
7147 			ret = -EINVAL;
7148 		else if (!IS_ALIGNED(attr->sample_stack_user, sizeof(u64)))
7149 			ret = -EINVAL;
7150 	}
7151 
7152 out:
7153 	return ret;
7154 
7155 err_size:
7156 	put_user(sizeof(*attr), &uattr->size);
7157 	ret = -E2BIG;
7158 	goto out;
7159 }
7160 
7161 static int
7162 perf_event_set_output(struct perf_event *event, struct perf_event *output_event)
7163 {
7164 	struct ring_buffer *rb = NULL;
7165 	int ret = -EINVAL;
7166 
7167 	if (!output_event)
7168 		goto set;
7169 
7170 	/* don't allow circular references */
7171 	if (event == output_event)
7172 		goto out;
7173 
7174 	/*
7175 	 * Don't allow cross-cpu buffers
7176 	 */
7177 	if (output_event->cpu != event->cpu)
7178 		goto out;
7179 
7180 	/*
7181 	 * If its not a per-cpu rb, it must be the same task.
7182 	 */
7183 	if (output_event->cpu == -1 && output_event->ctx != event->ctx)
7184 		goto out;
7185 
7186 set:
7187 	mutex_lock(&event->mmap_mutex);
7188 	/* Can't redirect output if we've got an active mmap() */
7189 	if (atomic_read(&event->mmap_count))
7190 		goto unlock;
7191 
7192 	if (output_event) {
7193 		/* get the rb we want to redirect to */
7194 		rb = ring_buffer_get(output_event);
7195 		if (!rb)
7196 			goto unlock;
7197 	}
7198 
7199 	ring_buffer_attach(event, rb);
7200 
7201 	ret = 0;
7202 unlock:
7203 	mutex_unlock(&event->mmap_mutex);
7204 
7205 out:
7206 	return ret;
7207 }
7208 
7209 /**
7210  * sys_perf_event_open - open a performance event, associate it to a task/cpu
7211  *
7212  * @attr_uptr:	event_id type attributes for monitoring/sampling
7213  * @pid:		target pid
7214  * @cpu:		target cpu
7215  * @group_fd:		group leader event fd
7216  */
7217 SYSCALL_DEFINE5(perf_event_open,
7218 		struct perf_event_attr __user *, attr_uptr,
7219 		pid_t, pid, int, cpu, int, group_fd, unsigned long, flags)
7220 {
7221 	struct perf_event *group_leader = NULL, *output_event = NULL;
7222 	struct perf_event *event, *sibling;
7223 	struct perf_event_attr attr;
7224 	struct perf_event_context *ctx;
7225 	struct file *event_file = NULL;
7226 	struct fd group = {NULL, 0};
7227 	struct task_struct *task = NULL;
7228 	struct pmu *pmu;
7229 	int event_fd;
7230 	int move_group = 0;
7231 	int err;
7232 	int f_flags = O_RDWR;
7233 
7234 	/* for future expandability... */
7235 	if (flags & ~PERF_FLAG_ALL)
7236 		return -EINVAL;
7237 
7238 	err = perf_copy_attr(attr_uptr, &attr);
7239 	if (err)
7240 		return err;
7241 
7242 	if (!attr.exclude_kernel) {
7243 		if (perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN))
7244 			return -EACCES;
7245 	}
7246 
7247 	if (attr.freq) {
7248 		if (attr.sample_freq > sysctl_perf_event_sample_rate)
7249 			return -EINVAL;
7250 	} else {
7251 		if (attr.sample_period & (1ULL << 63))
7252 			return -EINVAL;
7253 	}
7254 
7255 	/*
7256 	 * In cgroup mode, the pid argument is used to pass the fd
7257 	 * opened to the cgroup directory in cgroupfs. The cpu argument
7258 	 * designates the cpu on which to monitor threads from that
7259 	 * cgroup.
7260 	 */
7261 	if ((flags & PERF_FLAG_PID_CGROUP) && (pid == -1 || cpu == -1))
7262 		return -EINVAL;
7263 
7264 	if (flags & PERF_FLAG_FD_CLOEXEC)
7265 		f_flags |= O_CLOEXEC;
7266 
7267 	event_fd = get_unused_fd_flags(f_flags);
7268 	if (event_fd < 0)
7269 		return event_fd;
7270 
7271 	if (group_fd != -1) {
7272 		err = perf_fget_light(group_fd, &group);
7273 		if (err)
7274 			goto err_fd;
7275 		group_leader = group.file->private_data;
7276 		if (flags & PERF_FLAG_FD_OUTPUT)
7277 			output_event = group_leader;
7278 		if (flags & PERF_FLAG_FD_NO_GROUP)
7279 			group_leader = NULL;
7280 	}
7281 
7282 	if (pid != -1 && !(flags & PERF_FLAG_PID_CGROUP)) {
7283 		task = find_lively_task_by_vpid(pid);
7284 		if (IS_ERR(task)) {
7285 			err = PTR_ERR(task);
7286 			goto err_group_fd;
7287 		}
7288 	}
7289 
7290 	if (task && group_leader &&
7291 	    group_leader->attr.inherit != attr.inherit) {
7292 		err = -EINVAL;
7293 		goto err_task;
7294 	}
7295 
7296 	get_online_cpus();
7297 
7298 	event = perf_event_alloc(&attr, cpu, task, group_leader, NULL,
7299 				 NULL, NULL);
7300 	if (IS_ERR(event)) {
7301 		err = PTR_ERR(event);
7302 		goto err_cpus;
7303 	}
7304 
7305 	if (flags & PERF_FLAG_PID_CGROUP) {
7306 		err = perf_cgroup_connect(pid, event, &attr, group_leader);
7307 		if (err) {
7308 			__free_event(event);
7309 			goto err_cpus;
7310 		}
7311 	}
7312 
7313 	if (is_sampling_event(event)) {
7314 		if (event->pmu->capabilities & PERF_PMU_CAP_NO_INTERRUPT) {
7315 			err = -ENOTSUPP;
7316 			goto err_alloc;
7317 		}
7318 	}
7319 
7320 	account_event(event);
7321 
7322 	/*
7323 	 * Special case software events and allow them to be part of
7324 	 * any hardware group.
7325 	 */
7326 	pmu = event->pmu;
7327 
7328 	if (group_leader &&
7329 	    (is_software_event(event) != is_software_event(group_leader))) {
7330 		if (is_software_event(event)) {
7331 			/*
7332 			 * If event and group_leader are not both a software
7333 			 * event, and event is, then group leader is not.
7334 			 *
7335 			 * Allow the addition of software events to !software
7336 			 * groups, this is safe because software events never
7337 			 * fail to schedule.
7338 			 */
7339 			pmu = group_leader->pmu;
7340 		} else if (is_software_event(group_leader) &&
7341 			   (group_leader->group_flags & PERF_GROUP_SOFTWARE)) {
7342 			/*
7343 			 * In case the group is a pure software group, and we
7344 			 * try to add a hardware event, move the whole group to
7345 			 * the hardware context.
7346 			 */
7347 			move_group = 1;
7348 		}
7349 	}
7350 
7351 	/*
7352 	 * Get the target context (task or percpu):
7353 	 */
7354 	ctx = find_get_context(pmu, task, event->cpu);
7355 	if (IS_ERR(ctx)) {
7356 		err = PTR_ERR(ctx);
7357 		goto err_alloc;
7358 	}
7359 
7360 	if (task) {
7361 		put_task_struct(task);
7362 		task = NULL;
7363 	}
7364 
7365 	/*
7366 	 * Look up the group leader (we will attach this event to it):
7367 	 */
7368 	if (group_leader) {
7369 		err = -EINVAL;
7370 
7371 		/*
7372 		 * Do not allow a recursive hierarchy (this new sibling
7373 		 * becoming part of another group-sibling):
7374 		 */
7375 		if (group_leader->group_leader != group_leader)
7376 			goto err_context;
7377 		/*
7378 		 * Do not allow to attach to a group in a different
7379 		 * task or CPU context:
7380 		 */
7381 		if (move_group) {
7382 			if (group_leader->ctx->type != ctx->type)
7383 				goto err_context;
7384 		} else {
7385 			if (group_leader->ctx != ctx)
7386 				goto err_context;
7387 		}
7388 
7389 		/*
7390 		 * Only a group leader can be exclusive or pinned
7391 		 */
7392 		if (attr.exclusive || attr.pinned)
7393 			goto err_context;
7394 	}
7395 
7396 	if (output_event) {
7397 		err = perf_event_set_output(event, output_event);
7398 		if (err)
7399 			goto err_context;
7400 	}
7401 
7402 	event_file = anon_inode_getfile("[perf_event]", &perf_fops, event,
7403 					f_flags);
7404 	if (IS_ERR(event_file)) {
7405 		err = PTR_ERR(event_file);
7406 		goto err_context;
7407 	}
7408 
7409 	if (move_group) {
7410 		struct perf_event_context *gctx = group_leader->ctx;
7411 
7412 		mutex_lock(&gctx->mutex);
7413 		perf_remove_from_context(group_leader, false);
7414 
7415 		/*
7416 		 * Removing from the context ends up with disabled
7417 		 * event. What we want here is event in the initial
7418 		 * startup state, ready to be add into new context.
7419 		 */
7420 		perf_event__state_init(group_leader);
7421 		list_for_each_entry(sibling, &group_leader->sibling_list,
7422 				    group_entry) {
7423 			perf_remove_from_context(sibling, false);
7424 			perf_event__state_init(sibling);
7425 			put_ctx(gctx);
7426 		}
7427 		mutex_unlock(&gctx->mutex);
7428 		put_ctx(gctx);
7429 	}
7430 
7431 	WARN_ON_ONCE(ctx->parent_ctx);
7432 	mutex_lock(&ctx->mutex);
7433 
7434 	if (move_group) {
7435 		synchronize_rcu();
7436 		perf_install_in_context(ctx, group_leader, event->cpu);
7437 		get_ctx(ctx);
7438 		list_for_each_entry(sibling, &group_leader->sibling_list,
7439 				    group_entry) {
7440 			perf_install_in_context(ctx, sibling, event->cpu);
7441 			get_ctx(ctx);
7442 		}
7443 	}
7444 
7445 	perf_install_in_context(ctx, event, event->cpu);
7446 	perf_unpin_context(ctx);
7447 	mutex_unlock(&ctx->mutex);
7448 
7449 	put_online_cpus();
7450 
7451 	event->owner = current;
7452 
7453 	mutex_lock(&current->perf_event_mutex);
7454 	list_add_tail(&event->owner_entry, &current->perf_event_list);
7455 	mutex_unlock(&current->perf_event_mutex);
7456 
7457 	/*
7458 	 * Precalculate sample_data sizes
7459 	 */
7460 	perf_event__header_size(event);
7461 	perf_event__id_header_size(event);
7462 
7463 	/*
7464 	 * Drop the reference on the group_event after placing the
7465 	 * new event on the sibling_list. This ensures destruction
7466 	 * of the group leader will find the pointer to itself in
7467 	 * perf_group_detach().
7468 	 */
7469 	fdput(group);
7470 	fd_install(event_fd, event_file);
7471 	return event_fd;
7472 
7473 err_context:
7474 	perf_unpin_context(ctx);
7475 	put_ctx(ctx);
7476 err_alloc:
7477 	free_event(event);
7478 err_cpus:
7479 	put_online_cpus();
7480 err_task:
7481 	if (task)
7482 		put_task_struct(task);
7483 err_group_fd:
7484 	fdput(group);
7485 err_fd:
7486 	put_unused_fd(event_fd);
7487 	return err;
7488 }
7489 
7490 /**
7491  * perf_event_create_kernel_counter
7492  *
7493  * @attr: attributes of the counter to create
7494  * @cpu: cpu in which the counter is bound
7495  * @task: task to profile (NULL for percpu)
7496  */
7497 struct perf_event *
7498 perf_event_create_kernel_counter(struct perf_event_attr *attr, int cpu,
7499 				 struct task_struct *task,
7500 				 perf_overflow_handler_t overflow_handler,
7501 				 void *context)
7502 {
7503 	struct perf_event_context *ctx;
7504 	struct perf_event *event;
7505 	int err;
7506 
7507 	/*
7508 	 * Get the target context (task or percpu):
7509 	 */
7510 
7511 	event = perf_event_alloc(attr, cpu, task, NULL, NULL,
7512 				 overflow_handler, context);
7513 	if (IS_ERR(event)) {
7514 		err = PTR_ERR(event);
7515 		goto err;
7516 	}
7517 
7518 	/* Mark owner so we could distinguish it from user events. */
7519 	event->owner = EVENT_OWNER_KERNEL;
7520 
7521 	account_event(event);
7522 
7523 	ctx = find_get_context(event->pmu, task, cpu);
7524 	if (IS_ERR(ctx)) {
7525 		err = PTR_ERR(ctx);
7526 		goto err_free;
7527 	}
7528 
7529 	WARN_ON_ONCE(ctx->parent_ctx);
7530 	mutex_lock(&ctx->mutex);
7531 	perf_install_in_context(ctx, event, cpu);
7532 	perf_unpin_context(ctx);
7533 	mutex_unlock(&ctx->mutex);
7534 
7535 	return event;
7536 
7537 err_free:
7538 	free_event(event);
7539 err:
7540 	return ERR_PTR(err);
7541 }
7542 EXPORT_SYMBOL_GPL(perf_event_create_kernel_counter);
7543 
7544 void perf_pmu_migrate_context(struct pmu *pmu, int src_cpu, int dst_cpu)
7545 {
7546 	struct perf_event_context *src_ctx;
7547 	struct perf_event_context *dst_ctx;
7548 	struct perf_event *event, *tmp;
7549 	LIST_HEAD(events);
7550 
7551 	src_ctx = &per_cpu_ptr(pmu->pmu_cpu_context, src_cpu)->ctx;
7552 	dst_ctx = &per_cpu_ptr(pmu->pmu_cpu_context, dst_cpu)->ctx;
7553 
7554 	mutex_lock(&src_ctx->mutex);
7555 	list_for_each_entry_safe(event, tmp, &src_ctx->event_list,
7556 				 event_entry) {
7557 		perf_remove_from_context(event, false);
7558 		unaccount_event_cpu(event, src_cpu);
7559 		put_ctx(src_ctx);
7560 		list_add(&event->migrate_entry, &events);
7561 	}
7562 	mutex_unlock(&src_ctx->mutex);
7563 
7564 	synchronize_rcu();
7565 
7566 	mutex_lock(&dst_ctx->mutex);
7567 	list_for_each_entry_safe(event, tmp, &events, migrate_entry) {
7568 		list_del(&event->migrate_entry);
7569 		if (event->state >= PERF_EVENT_STATE_OFF)
7570 			event->state = PERF_EVENT_STATE_INACTIVE;
7571 		account_event_cpu(event, dst_cpu);
7572 		perf_install_in_context(dst_ctx, event, dst_cpu);
7573 		get_ctx(dst_ctx);
7574 	}
7575 	mutex_unlock(&dst_ctx->mutex);
7576 }
7577 EXPORT_SYMBOL_GPL(perf_pmu_migrate_context);
7578 
7579 static void sync_child_event(struct perf_event *child_event,
7580 			       struct task_struct *child)
7581 {
7582 	struct perf_event *parent_event = child_event->parent;
7583 	u64 child_val;
7584 
7585 	if (child_event->attr.inherit_stat)
7586 		perf_event_read_event(child_event, child);
7587 
7588 	child_val = perf_event_count(child_event);
7589 
7590 	/*
7591 	 * Add back the child's count to the parent's count:
7592 	 */
7593 	atomic64_add(child_val, &parent_event->child_count);
7594 	atomic64_add(child_event->total_time_enabled,
7595 		     &parent_event->child_total_time_enabled);
7596 	atomic64_add(child_event->total_time_running,
7597 		     &parent_event->child_total_time_running);
7598 
7599 	/*
7600 	 * Remove this event from the parent's list
7601 	 */
7602 	WARN_ON_ONCE(parent_event->ctx->parent_ctx);
7603 	mutex_lock(&parent_event->child_mutex);
7604 	list_del_init(&child_event->child_list);
7605 	mutex_unlock(&parent_event->child_mutex);
7606 
7607 	/*
7608 	 * Make sure user/parent get notified, that we just
7609 	 * lost one event.
7610 	 */
7611 	perf_event_wakeup(parent_event);
7612 
7613 	/*
7614 	 * Release the parent event, if this was the last
7615 	 * reference to it.
7616 	 */
7617 	put_event(parent_event);
7618 }
7619 
7620 static void
7621 __perf_event_exit_task(struct perf_event *child_event,
7622 			 struct perf_event_context *child_ctx,
7623 			 struct task_struct *child)
7624 {
7625 	/*
7626 	 * Do not destroy the 'original' grouping; because of the context
7627 	 * switch optimization the original events could've ended up in a
7628 	 * random child task.
7629 	 *
7630 	 * If we were to destroy the original group, all group related
7631 	 * operations would cease to function properly after this random
7632 	 * child dies.
7633 	 *
7634 	 * Do destroy all inherited groups, we don't care about those
7635 	 * and being thorough is better.
7636 	 */
7637 	perf_remove_from_context(child_event, !!child_event->parent);
7638 
7639 	/*
7640 	 * It can happen that the parent exits first, and has events
7641 	 * that are still around due to the child reference. These
7642 	 * events need to be zapped.
7643 	 */
7644 	if (child_event->parent) {
7645 		sync_child_event(child_event, child);
7646 		free_event(child_event);
7647 	} else {
7648 		child_event->state = PERF_EVENT_STATE_EXIT;
7649 		perf_event_wakeup(child_event);
7650 	}
7651 }
7652 
7653 static void perf_event_exit_task_context(struct task_struct *child, int ctxn)
7654 {
7655 	struct perf_event *child_event, *next;
7656 	struct perf_event_context *child_ctx, *clone_ctx = NULL;
7657 	unsigned long flags;
7658 
7659 	if (likely(!child->perf_event_ctxp[ctxn])) {
7660 		perf_event_task(child, NULL, 0);
7661 		return;
7662 	}
7663 
7664 	local_irq_save(flags);
7665 	/*
7666 	 * We can't reschedule here because interrupts are disabled,
7667 	 * and either child is current or it is a task that can't be
7668 	 * scheduled, so we are now safe from rescheduling changing
7669 	 * our context.
7670 	 */
7671 	child_ctx = rcu_dereference_raw(child->perf_event_ctxp[ctxn]);
7672 
7673 	/*
7674 	 * Take the context lock here so that if find_get_context is
7675 	 * reading child->perf_event_ctxp, we wait until it has
7676 	 * incremented the context's refcount before we do put_ctx below.
7677 	 */
7678 	raw_spin_lock(&child_ctx->lock);
7679 	task_ctx_sched_out(child_ctx);
7680 	child->perf_event_ctxp[ctxn] = NULL;
7681 
7682 	/*
7683 	 * If this context is a clone; unclone it so it can't get
7684 	 * swapped to another process while we're removing all
7685 	 * the events from it.
7686 	 */
7687 	clone_ctx = unclone_ctx(child_ctx);
7688 	update_context_time(child_ctx);
7689 	raw_spin_unlock_irqrestore(&child_ctx->lock, flags);
7690 
7691 	if (clone_ctx)
7692 		put_ctx(clone_ctx);
7693 
7694 	/*
7695 	 * Report the task dead after unscheduling the events so that we
7696 	 * won't get any samples after PERF_RECORD_EXIT. We can however still
7697 	 * get a few PERF_RECORD_READ events.
7698 	 */
7699 	perf_event_task(child, child_ctx, 0);
7700 
7701 	/*
7702 	 * We can recurse on the same lock type through:
7703 	 *
7704 	 *   __perf_event_exit_task()
7705 	 *     sync_child_event()
7706 	 *       put_event()
7707 	 *         mutex_lock(&ctx->mutex)
7708 	 *
7709 	 * But since its the parent context it won't be the same instance.
7710 	 */
7711 	mutex_lock(&child_ctx->mutex);
7712 
7713 	list_for_each_entry_safe(child_event, next, &child_ctx->event_list, event_entry)
7714 		__perf_event_exit_task(child_event, child_ctx, child);
7715 
7716 	mutex_unlock(&child_ctx->mutex);
7717 
7718 	put_ctx(child_ctx);
7719 }
7720 
7721 /*
7722  * When a child task exits, feed back event values to parent events.
7723  */
7724 void perf_event_exit_task(struct task_struct *child)
7725 {
7726 	struct perf_event *event, *tmp;
7727 	int ctxn;
7728 
7729 	mutex_lock(&child->perf_event_mutex);
7730 	list_for_each_entry_safe(event, tmp, &child->perf_event_list,
7731 				 owner_entry) {
7732 		list_del_init(&event->owner_entry);
7733 
7734 		/*
7735 		 * Ensure the list deletion is visible before we clear
7736 		 * the owner, closes a race against perf_release() where
7737 		 * we need to serialize on the owner->perf_event_mutex.
7738 		 */
7739 		smp_wmb();
7740 		event->owner = NULL;
7741 	}
7742 	mutex_unlock(&child->perf_event_mutex);
7743 
7744 	for_each_task_context_nr(ctxn)
7745 		perf_event_exit_task_context(child, ctxn);
7746 }
7747 
7748 static void perf_free_event(struct perf_event *event,
7749 			    struct perf_event_context *ctx)
7750 {
7751 	struct perf_event *parent = event->parent;
7752 
7753 	if (WARN_ON_ONCE(!parent))
7754 		return;
7755 
7756 	mutex_lock(&parent->child_mutex);
7757 	list_del_init(&event->child_list);
7758 	mutex_unlock(&parent->child_mutex);
7759 
7760 	put_event(parent);
7761 
7762 	perf_group_detach(event);
7763 	list_del_event(event, ctx);
7764 	free_event(event);
7765 }
7766 
7767 /*
7768  * free an unexposed, unused context as created by inheritance by
7769  * perf_event_init_task below, used by fork() in case of fail.
7770  */
7771 void perf_event_free_task(struct task_struct *task)
7772 {
7773 	struct perf_event_context *ctx;
7774 	struct perf_event *event, *tmp;
7775 	int ctxn;
7776 
7777 	for_each_task_context_nr(ctxn) {
7778 		ctx = task->perf_event_ctxp[ctxn];
7779 		if (!ctx)
7780 			continue;
7781 
7782 		mutex_lock(&ctx->mutex);
7783 again:
7784 		list_for_each_entry_safe(event, tmp, &ctx->pinned_groups,
7785 				group_entry)
7786 			perf_free_event(event, ctx);
7787 
7788 		list_for_each_entry_safe(event, tmp, &ctx->flexible_groups,
7789 				group_entry)
7790 			perf_free_event(event, ctx);
7791 
7792 		if (!list_empty(&ctx->pinned_groups) ||
7793 				!list_empty(&ctx->flexible_groups))
7794 			goto again;
7795 
7796 		mutex_unlock(&ctx->mutex);
7797 
7798 		put_ctx(ctx);
7799 	}
7800 }
7801 
7802 void perf_event_delayed_put(struct task_struct *task)
7803 {
7804 	int ctxn;
7805 
7806 	for_each_task_context_nr(ctxn)
7807 		WARN_ON_ONCE(task->perf_event_ctxp[ctxn]);
7808 }
7809 
7810 /*
7811  * inherit a event from parent task to child task:
7812  */
7813 static struct perf_event *
7814 inherit_event(struct perf_event *parent_event,
7815 	      struct task_struct *parent,
7816 	      struct perf_event_context *parent_ctx,
7817 	      struct task_struct *child,
7818 	      struct perf_event *group_leader,
7819 	      struct perf_event_context *child_ctx)
7820 {
7821 	enum perf_event_active_state parent_state = parent_event->state;
7822 	struct perf_event *child_event;
7823 	unsigned long flags;
7824 
7825 	/*
7826 	 * Instead of creating recursive hierarchies of events,
7827 	 * we link inherited events back to the original parent,
7828 	 * which has a filp for sure, which we use as the reference
7829 	 * count:
7830 	 */
7831 	if (parent_event->parent)
7832 		parent_event = parent_event->parent;
7833 
7834 	child_event = perf_event_alloc(&parent_event->attr,
7835 					   parent_event->cpu,
7836 					   child,
7837 					   group_leader, parent_event,
7838 				           NULL, NULL);
7839 	if (IS_ERR(child_event))
7840 		return child_event;
7841 
7842 	if (is_orphaned_event(parent_event) ||
7843 	    !atomic_long_inc_not_zero(&parent_event->refcount)) {
7844 		free_event(child_event);
7845 		return NULL;
7846 	}
7847 
7848 	get_ctx(child_ctx);
7849 
7850 	/*
7851 	 * Make the child state follow the state of the parent event,
7852 	 * not its attr.disabled bit.  We hold the parent's mutex,
7853 	 * so we won't race with perf_event_{en, dis}able_family.
7854 	 */
7855 	if (parent_state >= PERF_EVENT_STATE_INACTIVE)
7856 		child_event->state = PERF_EVENT_STATE_INACTIVE;
7857 	else
7858 		child_event->state = PERF_EVENT_STATE_OFF;
7859 
7860 	if (parent_event->attr.freq) {
7861 		u64 sample_period = parent_event->hw.sample_period;
7862 		struct hw_perf_event *hwc = &child_event->hw;
7863 
7864 		hwc->sample_period = sample_period;
7865 		hwc->last_period   = sample_period;
7866 
7867 		local64_set(&hwc->period_left, sample_period);
7868 	}
7869 
7870 	child_event->ctx = child_ctx;
7871 	child_event->overflow_handler = parent_event->overflow_handler;
7872 	child_event->overflow_handler_context
7873 		= parent_event->overflow_handler_context;
7874 
7875 	/*
7876 	 * Precalculate sample_data sizes
7877 	 */
7878 	perf_event__header_size(child_event);
7879 	perf_event__id_header_size(child_event);
7880 
7881 	/*
7882 	 * Link it up in the child's context:
7883 	 */
7884 	raw_spin_lock_irqsave(&child_ctx->lock, flags);
7885 	add_event_to_ctx(child_event, child_ctx);
7886 	raw_spin_unlock_irqrestore(&child_ctx->lock, flags);
7887 
7888 	/*
7889 	 * Link this into the parent event's child list
7890 	 */
7891 	WARN_ON_ONCE(parent_event->ctx->parent_ctx);
7892 	mutex_lock(&parent_event->child_mutex);
7893 	list_add_tail(&child_event->child_list, &parent_event->child_list);
7894 	mutex_unlock(&parent_event->child_mutex);
7895 
7896 	return child_event;
7897 }
7898 
7899 static int inherit_group(struct perf_event *parent_event,
7900 	      struct task_struct *parent,
7901 	      struct perf_event_context *parent_ctx,
7902 	      struct task_struct *child,
7903 	      struct perf_event_context *child_ctx)
7904 {
7905 	struct perf_event *leader;
7906 	struct perf_event *sub;
7907 	struct perf_event *child_ctr;
7908 
7909 	leader = inherit_event(parent_event, parent, parent_ctx,
7910 				 child, NULL, child_ctx);
7911 	if (IS_ERR(leader))
7912 		return PTR_ERR(leader);
7913 	list_for_each_entry(sub, &parent_event->sibling_list, group_entry) {
7914 		child_ctr = inherit_event(sub, parent, parent_ctx,
7915 					    child, leader, child_ctx);
7916 		if (IS_ERR(child_ctr))
7917 			return PTR_ERR(child_ctr);
7918 	}
7919 	return 0;
7920 }
7921 
7922 static int
7923 inherit_task_group(struct perf_event *event, struct task_struct *parent,
7924 		   struct perf_event_context *parent_ctx,
7925 		   struct task_struct *child, int ctxn,
7926 		   int *inherited_all)
7927 {
7928 	int ret;
7929 	struct perf_event_context *child_ctx;
7930 
7931 	if (!event->attr.inherit) {
7932 		*inherited_all = 0;
7933 		return 0;
7934 	}
7935 
7936 	child_ctx = child->perf_event_ctxp[ctxn];
7937 	if (!child_ctx) {
7938 		/*
7939 		 * This is executed from the parent task context, so
7940 		 * inherit events that have been marked for cloning.
7941 		 * First allocate and initialize a context for the
7942 		 * child.
7943 		 */
7944 
7945 		child_ctx = alloc_perf_context(parent_ctx->pmu, child);
7946 		if (!child_ctx)
7947 			return -ENOMEM;
7948 
7949 		child->perf_event_ctxp[ctxn] = child_ctx;
7950 	}
7951 
7952 	ret = inherit_group(event, parent, parent_ctx,
7953 			    child, child_ctx);
7954 
7955 	if (ret)
7956 		*inherited_all = 0;
7957 
7958 	return ret;
7959 }
7960 
7961 /*
7962  * Initialize the perf_event context in task_struct
7963  */
7964 static int perf_event_init_context(struct task_struct *child, int ctxn)
7965 {
7966 	struct perf_event_context *child_ctx, *parent_ctx;
7967 	struct perf_event_context *cloned_ctx;
7968 	struct perf_event *event;
7969 	struct task_struct *parent = current;
7970 	int inherited_all = 1;
7971 	unsigned long flags;
7972 	int ret = 0;
7973 
7974 	if (likely(!parent->perf_event_ctxp[ctxn]))
7975 		return 0;
7976 
7977 	/*
7978 	 * If the parent's context is a clone, pin it so it won't get
7979 	 * swapped under us.
7980 	 */
7981 	parent_ctx = perf_pin_task_context(parent, ctxn);
7982 	if (!parent_ctx)
7983 		return 0;
7984 
7985 	/*
7986 	 * No need to check if parent_ctx != NULL here; since we saw
7987 	 * it non-NULL earlier, the only reason for it to become NULL
7988 	 * is if we exit, and since we're currently in the middle of
7989 	 * a fork we can't be exiting at the same time.
7990 	 */
7991 
7992 	/*
7993 	 * Lock the parent list. No need to lock the child - not PID
7994 	 * hashed yet and not running, so nobody can access it.
7995 	 */
7996 	mutex_lock(&parent_ctx->mutex);
7997 
7998 	/*
7999 	 * We dont have to disable NMIs - we are only looking at
8000 	 * the list, not manipulating it:
8001 	 */
8002 	list_for_each_entry(event, &parent_ctx->pinned_groups, group_entry) {
8003 		ret = inherit_task_group(event, parent, parent_ctx,
8004 					 child, ctxn, &inherited_all);
8005 		if (ret)
8006 			break;
8007 	}
8008 
8009 	/*
8010 	 * We can't hold ctx->lock when iterating the ->flexible_group list due
8011 	 * to allocations, but we need to prevent rotation because
8012 	 * rotate_ctx() will change the list from interrupt context.
8013 	 */
8014 	raw_spin_lock_irqsave(&parent_ctx->lock, flags);
8015 	parent_ctx->rotate_disable = 1;
8016 	raw_spin_unlock_irqrestore(&parent_ctx->lock, flags);
8017 
8018 	list_for_each_entry(event, &parent_ctx->flexible_groups, group_entry) {
8019 		ret = inherit_task_group(event, parent, parent_ctx,
8020 					 child, ctxn, &inherited_all);
8021 		if (ret)
8022 			break;
8023 	}
8024 
8025 	raw_spin_lock_irqsave(&parent_ctx->lock, flags);
8026 	parent_ctx->rotate_disable = 0;
8027 
8028 	child_ctx = child->perf_event_ctxp[ctxn];
8029 
8030 	if (child_ctx && inherited_all) {
8031 		/*
8032 		 * Mark the child context as a clone of the parent
8033 		 * context, or of whatever the parent is a clone of.
8034 		 *
8035 		 * Note that if the parent is a clone, the holding of
8036 		 * parent_ctx->lock avoids it from being uncloned.
8037 		 */
8038 		cloned_ctx = parent_ctx->parent_ctx;
8039 		if (cloned_ctx) {
8040 			child_ctx->parent_ctx = cloned_ctx;
8041 			child_ctx->parent_gen = parent_ctx->parent_gen;
8042 		} else {
8043 			child_ctx->parent_ctx = parent_ctx;
8044 			child_ctx->parent_gen = parent_ctx->generation;
8045 		}
8046 		get_ctx(child_ctx->parent_ctx);
8047 	}
8048 
8049 	raw_spin_unlock_irqrestore(&parent_ctx->lock, flags);
8050 	mutex_unlock(&parent_ctx->mutex);
8051 
8052 	perf_unpin_context(parent_ctx);
8053 	put_ctx(parent_ctx);
8054 
8055 	return ret;
8056 }
8057 
8058 /*
8059  * Initialize the perf_event context in task_struct
8060  */
8061 int perf_event_init_task(struct task_struct *child)
8062 {
8063 	int ctxn, ret;
8064 
8065 	memset(child->perf_event_ctxp, 0, sizeof(child->perf_event_ctxp));
8066 	mutex_init(&child->perf_event_mutex);
8067 	INIT_LIST_HEAD(&child->perf_event_list);
8068 
8069 	for_each_task_context_nr(ctxn) {
8070 		ret = perf_event_init_context(child, ctxn);
8071 		if (ret) {
8072 			perf_event_free_task(child);
8073 			return ret;
8074 		}
8075 	}
8076 
8077 	return 0;
8078 }
8079 
8080 static void __init perf_event_init_all_cpus(void)
8081 {
8082 	struct swevent_htable *swhash;
8083 	int cpu;
8084 
8085 	for_each_possible_cpu(cpu) {
8086 		swhash = &per_cpu(swevent_htable, cpu);
8087 		mutex_init(&swhash->hlist_mutex);
8088 		INIT_LIST_HEAD(&per_cpu(rotation_list, cpu));
8089 	}
8090 }
8091 
8092 static void perf_event_init_cpu(int cpu)
8093 {
8094 	struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
8095 
8096 	mutex_lock(&swhash->hlist_mutex);
8097 	swhash->online = true;
8098 	if (swhash->hlist_refcount > 0) {
8099 		struct swevent_hlist *hlist;
8100 
8101 		hlist = kzalloc_node(sizeof(*hlist), GFP_KERNEL, cpu_to_node(cpu));
8102 		WARN_ON(!hlist);
8103 		rcu_assign_pointer(swhash->swevent_hlist, hlist);
8104 	}
8105 	mutex_unlock(&swhash->hlist_mutex);
8106 }
8107 
8108 #if defined CONFIG_HOTPLUG_CPU || defined CONFIG_KEXEC
8109 static void perf_pmu_rotate_stop(struct pmu *pmu)
8110 {
8111 	struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
8112 
8113 	WARN_ON(!irqs_disabled());
8114 
8115 	list_del_init(&cpuctx->rotation_list);
8116 }
8117 
8118 static void __perf_event_exit_context(void *__info)
8119 {
8120 	struct remove_event re = { .detach_group = false };
8121 	struct perf_event_context *ctx = __info;
8122 
8123 	perf_pmu_rotate_stop(ctx->pmu);
8124 
8125 	rcu_read_lock();
8126 	list_for_each_entry_rcu(re.event, &ctx->event_list, event_entry)
8127 		__perf_remove_from_context(&re);
8128 	rcu_read_unlock();
8129 }
8130 
8131 static void perf_event_exit_cpu_context(int cpu)
8132 {
8133 	struct perf_event_context *ctx;
8134 	struct pmu *pmu;
8135 	int idx;
8136 
8137 	idx = srcu_read_lock(&pmus_srcu);
8138 	list_for_each_entry_rcu(pmu, &pmus, entry) {
8139 		ctx = &per_cpu_ptr(pmu->pmu_cpu_context, cpu)->ctx;
8140 
8141 		mutex_lock(&ctx->mutex);
8142 		smp_call_function_single(cpu, __perf_event_exit_context, ctx, 1);
8143 		mutex_unlock(&ctx->mutex);
8144 	}
8145 	srcu_read_unlock(&pmus_srcu, idx);
8146 }
8147 
8148 static void perf_event_exit_cpu(int cpu)
8149 {
8150 	struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
8151 
8152 	perf_event_exit_cpu_context(cpu);
8153 
8154 	mutex_lock(&swhash->hlist_mutex);
8155 	swhash->online = false;
8156 	swevent_hlist_release(swhash);
8157 	mutex_unlock(&swhash->hlist_mutex);
8158 }
8159 #else
8160 static inline void perf_event_exit_cpu(int cpu) { }
8161 #endif
8162 
8163 static int
8164 perf_reboot(struct notifier_block *notifier, unsigned long val, void *v)
8165 {
8166 	int cpu;
8167 
8168 	for_each_online_cpu(cpu)
8169 		perf_event_exit_cpu(cpu);
8170 
8171 	return NOTIFY_OK;
8172 }
8173 
8174 /*
8175  * Run the perf reboot notifier at the very last possible moment so that
8176  * the generic watchdog code runs as long as possible.
8177  */
8178 static struct notifier_block perf_reboot_notifier = {
8179 	.notifier_call = perf_reboot,
8180 	.priority = INT_MIN,
8181 };
8182 
8183 static int
8184 perf_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
8185 {
8186 	unsigned int cpu = (long)hcpu;
8187 
8188 	switch (action & ~CPU_TASKS_FROZEN) {
8189 
8190 	case CPU_UP_PREPARE:
8191 	case CPU_DOWN_FAILED:
8192 		perf_event_init_cpu(cpu);
8193 		break;
8194 
8195 	case CPU_UP_CANCELED:
8196 	case CPU_DOWN_PREPARE:
8197 		perf_event_exit_cpu(cpu);
8198 		break;
8199 	default:
8200 		break;
8201 	}
8202 
8203 	return NOTIFY_OK;
8204 }
8205 
8206 void __init perf_event_init(void)
8207 {
8208 	int ret;
8209 
8210 	idr_init(&pmu_idr);
8211 
8212 	perf_event_init_all_cpus();
8213 	init_srcu_struct(&pmus_srcu);
8214 	perf_pmu_register(&perf_swevent, "software", PERF_TYPE_SOFTWARE);
8215 	perf_pmu_register(&perf_cpu_clock, NULL, -1);
8216 	perf_pmu_register(&perf_task_clock, NULL, -1);
8217 	perf_tp_register();
8218 	perf_cpu_notifier(perf_cpu_notify);
8219 	register_reboot_notifier(&perf_reboot_notifier);
8220 
8221 	ret = init_hw_breakpoint();
8222 	WARN(ret, "hw_breakpoint initialization failed with: %d", ret);
8223 
8224 	/* do not patch jump label more than once per second */
8225 	jump_label_rate_limit(&perf_sched_events, HZ);
8226 
8227 	/*
8228 	 * Build time assertion that we keep the data_head at the intended
8229 	 * location.  IOW, validation we got the __reserved[] size right.
8230 	 */
8231 	BUILD_BUG_ON((offsetof(struct perf_event_mmap_page, data_head))
8232 		     != 1024);
8233 }
8234 
8235 static int __init perf_event_sysfs_init(void)
8236 {
8237 	struct pmu *pmu;
8238 	int ret;
8239 
8240 	mutex_lock(&pmus_lock);
8241 
8242 	ret = bus_register(&pmu_bus);
8243 	if (ret)
8244 		goto unlock;
8245 
8246 	list_for_each_entry(pmu, &pmus, entry) {
8247 		if (!pmu->name || pmu->type < 0)
8248 			continue;
8249 
8250 		ret = pmu_dev_alloc(pmu);
8251 		WARN(ret, "Failed to register pmu: %s, reason %d\n", pmu->name, ret);
8252 	}
8253 	pmu_bus_running = 1;
8254 	ret = 0;
8255 
8256 unlock:
8257 	mutex_unlock(&pmus_lock);
8258 
8259 	return ret;
8260 }
8261 device_initcall(perf_event_sysfs_init);
8262 
8263 #ifdef CONFIG_CGROUP_PERF
8264 static struct cgroup_subsys_state *
8265 perf_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
8266 {
8267 	struct perf_cgroup *jc;
8268 
8269 	jc = kzalloc(sizeof(*jc), GFP_KERNEL);
8270 	if (!jc)
8271 		return ERR_PTR(-ENOMEM);
8272 
8273 	jc->info = alloc_percpu(struct perf_cgroup_info);
8274 	if (!jc->info) {
8275 		kfree(jc);
8276 		return ERR_PTR(-ENOMEM);
8277 	}
8278 
8279 	return &jc->css;
8280 }
8281 
8282 static void perf_cgroup_css_free(struct cgroup_subsys_state *css)
8283 {
8284 	struct perf_cgroup *jc = container_of(css, struct perf_cgroup, css);
8285 
8286 	free_percpu(jc->info);
8287 	kfree(jc);
8288 }
8289 
8290 static int __perf_cgroup_move(void *info)
8291 {
8292 	struct task_struct *task = info;
8293 	perf_cgroup_switch(task, PERF_CGROUP_SWOUT | PERF_CGROUP_SWIN);
8294 	return 0;
8295 }
8296 
8297 static void perf_cgroup_attach(struct cgroup_subsys_state *css,
8298 			       struct cgroup_taskset *tset)
8299 {
8300 	struct task_struct *task;
8301 
8302 	cgroup_taskset_for_each(task, tset)
8303 		task_function_call(task, __perf_cgroup_move, task);
8304 }
8305 
8306 static void perf_cgroup_exit(struct cgroup_subsys_state *css,
8307 			     struct cgroup_subsys_state *old_css,
8308 			     struct task_struct *task)
8309 {
8310 	/*
8311 	 * cgroup_exit() is called in the copy_process() failure path.
8312 	 * Ignore this case since the task hasn't ran yet, this avoids
8313 	 * trying to poke a half freed task state from generic code.
8314 	 */
8315 	if (!(task->flags & PF_EXITING))
8316 		return;
8317 
8318 	task_function_call(task, __perf_cgroup_move, task);
8319 }
8320 
8321 struct cgroup_subsys perf_event_cgrp_subsys = {
8322 	.css_alloc	= perf_cgroup_css_alloc,
8323 	.css_free	= perf_cgroup_css_free,
8324 	.exit		= perf_cgroup_exit,
8325 	.attach		= perf_cgroup_attach,
8326 };
8327 #endif /* CONFIG_CGROUP_PERF */
8328