xref: /openbmc/linux/kernel/events/core.c (revision 52fb57e7)
1 /*
2  * Performance events core code:
3  *
4  *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
5  *  Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar
6  *  Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
7  *  Copyright  ©  2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
8  *
9  * For licensing details see kernel-base/COPYING
10  */
11 
12 #include <linux/fs.h>
13 #include <linux/mm.h>
14 #include <linux/cpu.h>
15 #include <linux/smp.h>
16 #include <linux/idr.h>
17 #include <linux/file.h>
18 #include <linux/poll.h>
19 #include <linux/slab.h>
20 #include <linux/hash.h>
21 #include <linux/tick.h>
22 #include <linux/sysfs.h>
23 #include <linux/dcache.h>
24 #include <linux/percpu.h>
25 #include <linux/ptrace.h>
26 #include <linux/reboot.h>
27 #include <linux/vmstat.h>
28 #include <linux/device.h>
29 #include <linux/export.h>
30 #include <linux/vmalloc.h>
31 #include <linux/hardirq.h>
32 #include <linux/rculist.h>
33 #include <linux/uaccess.h>
34 #include <linux/syscalls.h>
35 #include <linux/anon_inodes.h>
36 #include <linux/kernel_stat.h>
37 #include <linux/cgroup.h>
38 #include <linux/perf_event.h>
39 #include <linux/ftrace_event.h>
40 #include <linux/hw_breakpoint.h>
41 #include <linux/mm_types.h>
42 #include <linux/module.h>
43 #include <linux/mman.h>
44 #include <linux/compat.h>
45 #include <linux/bpf.h>
46 #include <linux/filter.h>
47 
48 #include "internal.h"
49 
50 #include <asm/irq_regs.h>
51 
52 static struct workqueue_struct *perf_wq;
53 
54 struct remote_function_call {
55 	struct task_struct	*p;
56 	int			(*func)(void *info);
57 	void			*info;
58 	int			ret;
59 };
60 
61 static void remote_function(void *data)
62 {
63 	struct remote_function_call *tfc = data;
64 	struct task_struct *p = tfc->p;
65 
66 	if (p) {
67 		tfc->ret = -EAGAIN;
68 		if (task_cpu(p) != smp_processor_id() || !task_curr(p))
69 			return;
70 	}
71 
72 	tfc->ret = tfc->func(tfc->info);
73 }
74 
75 /**
76  * task_function_call - call a function on the cpu on which a task runs
77  * @p:		the task to evaluate
78  * @func:	the function to be called
79  * @info:	the function call argument
80  *
81  * Calls the function @func when the task is currently running. This might
82  * be on the current CPU, which just calls the function directly
83  *
84  * returns: @func return value, or
85  *	    -ESRCH  - when the process isn't running
86  *	    -EAGAIN - when the process moved away
87  */
88 static int
89 task_function_call(struct task_struct *p, int (*func) (void *info), void *info)
90 {
91 	struct remote_function_call data = {
92 		.p	= p,
93 		.func	= func,
94 		.info	= info,
95 		.ret	= -ESRCH, /* No such (running) process */
96 	};
97 
98 	if (task_curr(p))
99 		smp_call_function_single(task_cpu(p), remote_function, &data, 1);
100 
101 	return data.ret;
102 }
103 
104 /**
105  * cpu_function_call - call a function on the cpu
106  * @func:	the function to be called
107  * @info:	the function call argument
108  *
109  * Calls the function @func on the remote cpu.
110  *
111  * returns: @func return value or -ENXIO when the cpu is offline
112  */
113 static int cpu_function_call(int cpu, int (*func) (void *info), void *info)
114 {
115 	struct remote_function_call data = {
116 		.p	= NULL,
117 		.func	= func,
118 		.info	= info,
119 		.ret	= -ENXIO, /* No such CPU */
120 	};
121 
122 	smp_call_function_single(cpu, remote_function, &data, 1);
123 
124 	return data.ret;
125 }
126 
127 #define EVENT_OWNER_KERNEL ((void *) -1)
128 
129 static bool is_kernel_event(struct perf_event *event)
130 {
131 	return event->owner == EVENT_OWNER_KERNEL;
132 }
133 
134 #define PERF_FLAG_ALL (PERF_FLAG_FD_NO_GROUP |\
135 		       PERF_FLAG_FD_OUTPUT  |\
136 		       PERF_FLAG_PID_CGROUP |\
137 		       PERF_FLAG_FD_CLOEXEC)
138 
139 /*
140  * branch priv levels that need permission checks
141  */
142 #define PERF_SAMPLE_BRANCH_PERM_PLM \
143 	(PERF_SAMPLE_BRANCH_KERNEL |\
144 	 PERF_SAMPLE_BRANCH_HV)
145 
146 enum event_type_t {
147 	EVENT_FLEXIBLE = 0x1,
148 	EVENT_PINNED = 0x2,
149 	EVENT_ALL = EVENT_FLEXIBLE | EVENT_PINNED,
150 };
151 
152 /*
153  * perf_sched_events : >0 events exist
154  * perf_cgroup_events: >0 per-cpu cgroup events exist on this cpu
155  */
156 struct static_key_deferred perf_sched_events __read_mostly;
157 static DEFINE_PER_CPU(atomic_t, perf_cgroup_events);
158 static DEFINE_PER_CPU(int, perf_sched_cb_usages);
159 
160 static atomic_t nr_mmap_events __read_mostly;
161 static atomic_t nr_comm_events __read_mostly;
162 static atomic_t nr_task_events __read_mostly;
163 static atomic_t nr_freq_events __read_mostly;
164 
165 static LIST_HEAD(pmus);
166 static DEFINE_MUTEX(pmus_lock);
167 static struct srcu_struct pmus_srcu;
168 
169 /*
170  * perf event paranoia level:
171  *  -1 - not paranoid at all
172  *   0 - disallow raw tracepoint access for unpriv
173  *   1 - disallow cpu events for unpriv
174  *   2 - disallow kernel profiling for unpriv
175  */
176 int sysctl_perf_event_paranoid __read_mostly = 1;
177 
178 /* Minimum for 512 kiB + 1 user control page */
179 int sysctl_perf_event_mlock __read_mostly = 512 + (PAGE_SIZE / 1024); /* 'free' kiB per user */
180 
181 /*
182  * max perf event sample rate
183  */
184 #define DEFAULT_MAX_SAMPLE_RATE		100000
185 #define DEFAULT_SAMPLE_PERIOD_NS	(NSEC_PER_SEC / DEFAULT_MAX_SAMPLE_RATE)
186 #define DEFAULT_CPU_TIME_MAX_PERCENT	25
187 
188 int sysctl_perf_event_sample_rate __read_mostly	= DEFAULT_MAX_SAMPLE_RATE;
189 
190 static int max_samples_per_tick __read_mostly	= DIV_ROUND_UP(DEFAULT_MAX_SAMPLE_RATE, HZ);
191 static int perf_sample_period_ns __read_mostly	= DEFAULT_SAMPLE_PERIOD_NS;
192 
193 static int perf_sample_allowed_ns __read_mostly =
194 	DEFAULT_SAMPLE_PERIOD_NS * DEFAULT_CPU_TIME_MAX_PERCENT / 100;
195 
196 void update_perf_cpu_limits(void)
197 {
198 	u64 tmp = perf_sample_period_ns;
199 
200 	tmp *= sysctl_perf_cpu_time_max_percent;
201 	do_div(tmp, 100);
202 	ACCESS_ONCE(perf_sample_allowed_ns) = tmp;
203 }
204 
205 static int perf_rotate_context(struct perf_cpu_context *cpuctx);
206 
207 int perf_proc_update_handler(struct ctl_table *table, int write,
208 		void __user *buffer, size_t *lenp,
209 		loff_t *ppos)
210 {
211 	int ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
212 
213 	if (ret || !write)
214 		return ret;
215 
216 	max_samples_per_tick = DIV_ROUND_UP(sysctl_perf_event_sample_rate, HZ);
217 	perf_sample_period_ns = NSEC_PER_SEC / sysctl_perf_event_sample_rate;
218 	update_perf_cpu_limits();
219 
220 	return 0;
221 }
222 
223 int sysctl_perf_cpu_time_max_percent __read_mostly = DEFAULT_CPU_TIME_MAX_PERCENT;
224 
225 int perf_cpu_time_max_percent_handler(struct ctl_table *table, int write,
226 				void __user *buffer, size_t *lenp,
227 				loff_t *ppos)
228 {
229 	int ret = proc_dointvec(table, write, buffer, lenp, ppos);
230 
231 	if (ret || !write)
232 		return ret;
233 
234 	update_perf_cpu_limits();
235 
236 	return 0;
237 }
238 
239 /*
240  * perf samples are done in some very critical code paths (NMIs).
241  * If they take too much CPU time, the system can lock up and not
242  * get any real work done.  This will drop the sample rate when
243  * we detect that events are taking too long.
244  */
245 #define NR_ACCUMULATED_SAMPLES 128
246 static DEFINE_PER_CPU(u64, running_sample_length);
247 
248 static void perf_duration_warn(struct irq_work *w)
249 {
250 	u64 allowed_ns = ACCESS_ONCE(perf_sample_allowed_ns);
251 	u64 avg_local_sample_len;
252 	u64 local_samples_len;
253 
254 	local_samples_len = __this_cpu_read(running_sample_length);
255 	avg_local_sample_len = local_samples_len/NR_ACCUMULATED_SAMPLES;
256 
257 	printk_ratelimited(KERN_WARNING
258 			"perf interrupt took too long (%lld > %lld), lowering "
259 			"kernel.perf_event_max_sample_rate to %d\n",
260 			avg_local_sample_len, allowed_ns >> 1,
261 			sysctl_perf_event_sample_rate);
262 }
263 
264 static DEFINE_IRQ_WORK(perf_duration_work, perf_duration_warn);
265 
266 void perf_sample_event_took(u64 sample_len_ns)
267 {
268 	u64 allowed_ns = ACCESS_ONCE(perf_sample_allowed_ns);
269 	u64 avg_local_sample_len;
270 	u64 local_samples_len;
271 
272 	if (allowed_ns == 0)
273 		return;
274 
275 	/* decay the counter by 1 average sample */
276 	local_samples_len = __this_cpu_read(running_sample_length);
277 	local_samples_len -= local_samples_len/NR_ACCUMULATED_SAMPLES;
278 	local_samples_len += sample_len_ns;
279 	__this_cpu_write(running_sample_length, local_samples_len);
280 
281 	/*
282 	 * note: this will be biased artifically low until we have
283 	 * seen NR_ACCUMULATED_SAMPLES.  Doing it this way keeps us
284 	 * from having to maintain a count.
285 	 */
286 	avg_local_sample_len = local_samples_len/NR_ACCUMULATED_SAMPLES;
287 
288 	if (avg_local_sample_len <= allowed_ns)
289 		return;
290 
291 	if (max_samples_per_tick <= 1)
292 		return;
293 
294 	max_samples_per_tick = DIV_ROUND_UP(max_samples_per_tick, 2);
295 	sysctl_perf_event_sample_rate = max_samples_per_tick * HZ;
296 	perf_sample_period_ns = NSEC_PER_SEC / sysctl_perf_event_sample_rate;
297 
298 	update_perf_cpu_limits();
299 
300 	if (!irq_work_queue(&perf_duration_work)) {
301 		early_printk("perf interrupt took too long (%lld > %lld), lowering "
302 			     "kernel.perf_event_max_sample_rate to %d\n",
303 			     avg_local_sample_len, allowed_ns >> 1,
304 			     sysctl_perf_event_sample_rate);
305 	}
306 }
307 
308 static atomic64_t perf_event_id;
309 
310 static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,
311 			      enum event_type_t event_type);
312 
313 static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx,
314 			     enum event_type_t event_type,
315 			     struct task_struct *task);
316 
317 static void update_context_time(struct perf_event_context *ctx);
318 static u64 perf_event_time(struct perf_event *event);
319 
320 void __weak perf_event_print_debug(void)	{ }
321 
322 extern __weak const char *perf_pmu_name(void)
323 {
324 	return "pmu";
325 }
326 
327 static inline u64 perf_clock(void)
328 {
329 	return local_clock();
330 }
331 
332 static inline u64 perf_event_clock(struct perf_event *event)
333 {
334 	return event->clock();
335 }
336 
337 static inline struct perf_cpu_context *
338 __get_cpu_context(struct perf_event_context *ctx)
339 {
340 	return this_cpu_ptr(ctx->pmu->pmu_cpu_context);
341 }
342 
343 static void perf_ctx_lock(struct perf_cpu_context *cpuctx,
344 			  struct perf_event_context *ctx)
345 {
346 	raw_spin_lock(&cpuctx->ctx.lock);
347 	if (ctx)
348 		raw_spin_lock(&ctx->lock);
349 }
350 
351 static void perf_ctx_unlock(struct perf_cpu_context *cpuctx,
352 			    struct perf_event_context *ctx)
353 {
354 	if (ctx)
355 		raw_spin_unlock(&ctx->lock);
356 	raw_spin_unlock(&cpuctx->ctx.lock);
357 }
358 
359 #ifdef CONFIG_CGROUP_PERF
360 
361 static inline bool
362 perf_cgroup_match(struct perf_event *event)
363 {
364 	struct perf_event_context *ctx = event->ctx;
365 	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
366 
367 	/* @event doesn't care about cgroup */
368 	if (!event->cgrp)
369 		return true;
370 
371 	/* wants specific cgroup scope but @cpuctx isn't associated with any */
372 	if (!cpuctx->cgrp)
373 		return false;
374 
375 	/*
376 	 * Cgroup scoping is recursive.  An event enabled for a cgroup is
377 	 * also enabled for all its descendant cgroups.  If @cpuctx's
378 	 * cgroup is a descendant of @event's (the test covers identity
379 	 * case), it's a match.
380 	 */
381 	return cgroup_is_descendant(cpuctx->cgrp->css.cgroup,
382 				    event->cgrp->css.cgroup);
383 }
384 
385 static inline void perf_detach_cgroup(struct perf_event *event)
386 {
387 	css_put(&event->cgrp->css);
388 	event->cgrp = NULL;
389 }
390 
391 static inline int is_cgroup_event(struct perf_event *event)
392 {
393 	return event->cgrp != NULL;
394 }
395 
396 static inline u64 perf_cgroup_event_time(struct perf_event *event)
397 {
398 	struct perf_cgroup_info *t;
399 
400 	t = per_cpu_ptr(event->cgrp->info, event->cpu);
401 	return t->time;
402 }
403 
404 static inline void __update_cgrp_time(struct perf_cgroup *cgrp)
405 {
406 	struct perf_cgroup_info *info;
407 	u64 now;
408 
409 	now = perf_clock();
410 
411 	info = this_cpu_ptr(cgrp->info);
412 
413 	info->time += now - info->timestamp;
414 	info->timestamp = now;
415 }
416 
417 static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx)
418 {
419 	struct perf_cgroup *cgrp_out = cpuctx->cgrp;
420 	if (cgrp_out)
421 		__update_cgrp_time(cgrp_out);
422 }
423 
424 static inline void update_cgrp_time_from_event(struct perf_event *event)
425 {
426 	struct perf_cgroup *cgrp;
427 
428 	/*
429 	 * ensure we access cgroup data only when needed and
430 	 * when we know the cgroup is pinned (css_get)
431 	 */
432 	if (!is_cgroup_event(event))
433 		return;
434 
435 	cgrp = perf_cgroup_from_task(current);
436 	/*
437 	 * Do not update time when cgroup is not active
438 	 */
439 	if (cgrp == event->cgrp)
440 		__update_cgrp_time(event->cgrp);
441 }
442 
443 static inline void
444 perf_cgroup_set_timestamp(struct task_struct *task,
445 			  struct perf_event_context *ctx)
446 {
447 	struct perf_cgroup *cgrp;
448 	struct perf_cgroup_info *info;
449 
450 	/*
451 	 * ctx->lock held by caller
452 	 * ensure we do not access cgroup data
453 	 * unless we have the cgroup pinned (css_get)
454 	 */
455 	if (!task || !ctx->nr_cgroups)
456 		return;
457 
458 	cgrp = perf_cgroup_from_task(task);
459 	info = this_cpu_ptr(cgrp->info);
460 	info->timestamp = ctx->timestamp;
461 }
462 
463 #define PERF_CGROUP_SWOUT	0x1 /* cgroup switch out every event */
464 #define PERF_CGROUP_SWIN	0x2 /* cgroup switch in events based on task */
465 
466 /*
467  * reschedule events based on the cgroup constraint of task.
468  *
469  * mode SWOUT : schedule out everything
470  * mode SWIN : schedule in based on cgroup for next
471  */
472 void perf_cgroup_switch(struct task_struct *task, int mode)
473 {
474 	struct perf_cpu_context *cpuctx;
475 	struct pmu *pmu;
476 	unsigned long flags;
477 
478 	/*
479 	 * disable interrupts to avoid geting nr_cgroup
480 	 * changes via __perf_event_disable(). Also
481 	 * avoids preemption.
482 	 */
483 	local_irq_save(flags);
484 
485 	/*
486 	 * we reschedule only in the presence of cgroup
487 	 * constrained events.
488 	 */
489 	rcu_read_lock();
490 
491 	list_for_each_entry_rcu(pmu, &pmus, entry) {
492 		cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
493 		if (cpuctx->unique_pmu != pmu)
494 			continue; /* ensure we process each cpuctx once */
495 
496 		/*
497 		 * perf_cgroup_events says at least one
498 		 * context on this CPU has cgroup events.
499 		 *
500 		 * ctx->nr_cgroups reports the number of cgroup
501 		 * events for a context.
502 		 */
503 		if (cpuctx->ctx.nr_cgroups > 0) {
504 			perf_ctx_lock(cpuctx, cpuctx->task_ctx);
505 			perf_pmu_disable(cpuctx->ctx.pmu);
506 
507 			if (mode & PERF_CGROUP_SWOUT) {
508 				cpu_ctx_sched_out(cpuctx, EVENT_ALL);
509 				/*
510 				 * must not be done before ctxswout due
511 				 * to event_filter_match() in event_sched_out()
512 				 */
513 				cpuctx->cgrp = NULL;
514 			}
515 
516 			if (mode & PERF_CGROUP_SWIN) {
517 				WARN_ON_ONCE(cpuctx->cgrp);
518 				/*
519 				 * set cgrp before ctxsw in to allow
520 				 * event_filter_match() to not have to pass
521 				 * task around
522 				 */
523 				cpuctx->cgrp = perf_cgroup_from_task(task);
524 				cpu_ctx_sched_in(cpuctx, EVENT_ALL, task);
525 			}
526 			perf_pmu_enable(cpuctx->ctx.pmu);
527 			perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
528 		}
529 	}
530 
531 	rcu_read_unlock();
532 
533 	local_irq_restore(flags);
534 }
535 
536 static inline void perf_cgroup_sched_out(struct task_struct *task,
537 					 struct task_struct *next)
538 {
539 	struct perf_cgroup *cgrp1;
540 	struct perf_cgroup *cgrp2 = NULL;
541 
542 	/*
543 	 * we come here when we know perf_cgroup_events > 0
544 	 */
545 	cgrp1 = perf_cgroup_from_task(task);
546 
547 	/*
548 	 * next is NULL when called from perf_event_enable_on_exec()
549 	 * that will systematically cause a cgroup_switch()
550 	 */
551 	if (next)
552 		cgrp2 = perf_cgroup_from_task(next);
553 
554 	/*
555 	 * only schedule out current cgroup events if we know
556 	 * that we are switching to a different cgroup. Otherwise,
557 	 * do no touch the cgroup events.
558 	 */
559 	if (cgrp1 != cgrp2)
560 		perf_cgroup_switch(task, PERF_CGROUP_SWOUT);
561 }
562 
563 static inline void perf_cgroup_sched_in(struct task_struct *prev,
564 					struct task_struct *task)
565 {
566 	struct perf_cgroup *cgrp1;
567 	struct perf_cgroup *cgrp2 = NULL;
568 
569 	/*
570 	 * we come here when we know perf_cgroup_events > 0
571 	 */
572 	cgrp1 = perf_cgroup_from_task(task);
573 
574 	/* prev can never be NULL */
575 	cgrp2 = perf_cgroup_from_task(prev);
576 
577 	/*
578 	 * only need to schedule in cgroup events if we are changing
579 	 * cgroup during ctxsw. Cgroup events were not scheduled
580 	 * out of ctxsw out if that was not the case.
581 	 */
582 	if (cgrp1 != cgrp2)
583 		perf_cgroup_switch(task, PERF_CGROUP_SWIN);
584 }
585 
586 static inline int perf_cgroup_connect(int fd, struct perf_event *event,
587 				      struct perf_event_attr *attr,
588 				      struct perf_event *group_leader)
589 {
590 	struct perf_cgroup *cgrp;
591 	struct cgroup_subsys_state *css;
592 	struct fd f = fdget(fd);
593 	int ret = 0;
594 
595 	if (!f.file)
596 		return -EBADF;
597 
598 	css = css_tryget_online_from_dir(f.file->f_path.dentry,
599 					 &perf_event_cgrp_subsys);
600 	if (IS_ERR(css)) {
601 		ret = PTR_ERR(css);
602 		goto out;
603 	}
604 
605 	cgrp = container_of(css, struct perf_cgroup, css);
606 	event->cgrp = cgrp;
607 
608 	/*
609 	 * all events in a group must monitor
610 	 * the same cgroup because a task belongs
611 	 * to only one perf cgroup at a time
612 	 */
613 	if (group_leader && group_leader->cgrp != cgrp) {
614 		perf_detach_cgroup(event);
615 		ret = -EINVAL;
616 	}
617 out:
618 	fdput(f);
619 	return ret;
620 }
621 
622 static inline void
623 perf_cgroup_set_shadow_time(struct perf_event *event, u64 now)
624 {
625 	struct perf_cgroup_info *t;
626 	t = per_cpu_ptr(event->cgrp->info, event->cpu);
627 	event->shadow_ctx_time = now - t->timestamp;
628 }
629 
630 static inline void
631 perf_cgroup_defer_enabled(struct perf_event *event)
632 {
633 	/*
634 	 * when the current task's perf cgroup does not match
635 	 * the event's, we need to remember to call the
636 	 * perf_mark_enable() function the first time a task with
637 	 * a matching perf cgroup is scheduled in.
638 	 */
639 	if (is_cgroup_event(event) && !perf_cgroup_match(event))
640 		event->cgrp_defer_enabled = 1;
641 }
642 
643 static inline void
644 perf_cgroup_mark_enabled(struct perf_event *event,
645 			 struct perf_event_context *ctx)
646 {
647 	struct perf_event *sub;
648 	u64 tstamp = perf_event_time(event);
649 
650 	if (!event->cgrp_defer_enabled)
651 		return;
652 
653 	event->cgrp_defer_enabled = 0;
654 
655 	event->tstamp_enabled = tstamp - event->total_time_enabled;
656 	list_for_each_entry(sub, &event->sibling_list, group_entry) {
657 		if (sub->state >= PERF_EVENT_STATE_INACTIVE) {
658 			sub->tstamp_enabled = tstamp - sub->total_time_enabled;
659 			sub->cgrp_defer_enabled = 0;
660 		}
661 	}
662 }
663 #else /* !CONFIG_CGROUP_PERF */
664 
665 static inline bool
666 perf_cgroup_match(struct perf_event *event)
667 {
668 	return true;
669 }
670 
671 static inline void perf_detach_cgroup(struct perf_event *event)
672 {}
673 
674 static inline int is_cgroup_event(struct perf_event *event)
675 {
676 	return 0;
677 }
678 
679 static inline u64 perf_cgroup_event_cgrp_time(struct perf_event *event)
680 {
681 	return 0;
682 }
683 
684 static inline void update_cgrp_time_from_event(struct perf_event *event)
685 {
686 }
687 
688 static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx)
689 {
690 }
691 
692 static inline void perf_cgroup_sched_out(struct task_struct *task,
693 					 struct task_struct *next)
694 {
695 }
696 
697 static inline void perf_cgroup_sched_in(struct task_struct *prev,
698 					struct task_struct *task)
699 {
700 }
701 
702 static inline int perf_cgroup_connect(pid_t pid, struct perf_event *event,
703 				      struct perf_event_attr *attr,
704 				      struct perf_event *group_leader)
705 {
706 	return -EINVAL;
707 }
708 
709 static inline void
710 perf_cgroup_set_timestamp(struct task_struct *task,
711 			  struct perf_event_context *ctx)
712 {
713 }
714 
715 void
716 perf_cgroup_switch(struct task_struct *task, struct task_struct *next)
717 {
718 }
719 
720 static inline void
721 perf_cgroup_set_shadow_time(struct perf_event *event, u64 now)
722 {
723 }
724 
725 static inline u64 perf_cgroup_event_time(struct perf_event *event)
726 {
727 	return 0;
728 }
729 
730 static inline void
731 perf_cgroup_defer_enabled(struct perf_event *event)
732 {
733 }
734 
735 static inline void
736 perf_cgroup_mark_enabled(struct perf_event *event,
737 			 struct perf_event_context *ctx)
738 {
739 }
740 #endif
741 
742 /*
743  * set default to be dependent on timer tick just
744  * like original code
745  */
746 #define PERF_CPU_HRTIMER (1000 / HZ)
747 /*
748  * function must be called with interrupts disbled
749  */
750 static enum hrtimer_restart perf_cpu_hrtimer_handler(struct hrtimer *hr)
751 {
752 	struct perf_cpu_context *cpuctx;
753 	enum hrtimer_restart ret = HRTIMER_NORESTART;
754 	int rotations = 0;
755 
756 	WARN_ON(!irqs_disabled());
757 
758 	cpuctx = container_of(hr, struct perf_cpu_context, hrtimer);
759 
760 	rotations = perf_rotate_context(cpuctx);
761 
762 	/*
763 	 * arm timer if needed
764 	 */
765 	if (rotations) {
766 		hrtimer_forward_now(hr, cpuctx->hrtimer_interval);
767 		ret = HRTIMER_RESTART;
768 	}
769 
770 	return ret;
771 }
772 
773 /* CPU is going down */
774 void perf_cpu_hrtimer_cancel(int cpu)
775 {
776 	struct perf_cpu_context *cpuctx;
777 	struct pmu *pmu;
778 	unsigned long flags;
779 
780 	if (WARN_ON(cpu != smp_processor_id()))
781 		return;
782 
783 	local_irq_save(flags);
784 
785 	rcu_read_lock();
786 
787 	list_for_each_entry_rcu(pmu, &pmus, entry) {
788 		cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
789 
790 		if (pmu->task_ctx_nr == perf_sw_context)
791 			continue;
792 
793 		hrtimer_cancel(&cpuctx->hrtimer);
794 	}
795 
796 	rcu_read_unlock();
797 
798 	local_irq_restore(flags);
799 }
800 
801 static void __perf_cpu_hrtimer_init(struct perf_cpu_context *cpuctx, int cpu)
802 {
803 	struct hrtimer *hr = &cpuctx->hrtimer;
804 	struct pmu *pmu = cpuctx->ctx.pmu;
805 	int timer;
806 
807 	/* no multiplexing needed for SW PMU */
808 	if (pmu->task_ctx_nr == perf_sw_context)
809 		return;
810 
811 	/*
812 	 * check default is sane, if not set then force to
813 	 * default interval (1/tick)
814 	 */
815 	timer = pmu->hrtimer_interval_ms;
816 	if (timer < 1)
817 		timer = pmu->hrtimer_interval_ms = PERF_CPU_HRTIMER;
818 
819 	cpuctx->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * timer);
820 
821 	hrtimer_init(hr, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);
822 	hr->function = perf_cpu_hrtimer_handler;
823 }
824 
825 static void perf_cpu_hrtimer_restart(struct perf_cpu_context *cpuctx)
826 {
827 	struct hrtimer *hr = &cpuctx->hrtimer;
828 	struct pmu *pmu = cpuctx->ctx.pmu;
829 
830 	/* not for SW PMU */
831 	if (pmu->task_ctx_nr == perf_sw_context)
832 		return;
833 
834 	if (hrtimer_active(hr))
835 		return;
836 
837 	if (!hrtimer_callback_running(hr))
838 		__hrtimer_start_range_ns(hr, cpuctx->hrtimer_interval,
839 					 0, HRTIMER_MODE_REL_PINNED, 0);
840 }
841 
842 void perf_pmu_disable(struct pmu *pmu)
843 {
844 	int *count = this_cpu_ptr(pmu->pmu_disable_count);
845 	if (!(*count)++)
846 		pmu->pmu_disable(pmu);
847 }
848 
849 void perf_pmu_enable(struct pmu *pmu)
850 {
851 	int *count = this_cpu_ptr(pmu->pmu_disable_count);
852 	if (!--(*count))
853 		pmu->pmu_enable(pmu);
854 }
855 
856 static DEFINE_PER_CPU(struct list_head, active_ctx_list);
857 
858 /*
859  * perf_event_ctx_activate(), perf_event_ctx_deactivate(), and
860  * perf_event_task_tick() are fully serialized because they're strictly cpu
861  * affine and perf_event_ctx{activate,deactivate} are called with IRQs
862  * disabled, while perf_event_task_tick is called from IRQ context.
863  */
864 static void perf_event_ctx_activate(struct perf_event_context *ctx)
865 {
866 	struct list_head *head = this_cpu_ptr(&active_ctx_list);
867 
868 	WARN_ON(!irqs_disabled());
869 
870 	WARN_ON(!list_empty(&ctx->active_ctx_list));
871 
872 	list_add(&ctx->active_ctx_list, head);
873 }
874 
875 static void perf_event_ctx_deactivate(struct perf_event_context *ctx)
876 {
877 	WARN_ON(!irqs_disabled());
878 
879 	WARN_ON(list_empty(&ctx->active_ctx_list));
880 
881 	list_del_init(&ctx->active_ctx_list);
882 }
883 
884 static void get_ctx(struct perf_event_context *ctx)
885 {
886 	WARN_ON(!atomic_inc_not_zero(&ctx->refcount));
887 }
888 
889 static void free_ctx(struct rcu_head *head)
890 {
891 	struct perf_event_context *ctx;
892 
893 	ctx = container_of(head, struct perf_event_context, rcu_head);
894 	kfree(ctx->task_ctx_data);
895 	kfree(ctx);
896 }
897 
898 static void put_ctx(struct perf_event_context *ctx)
899 {
900 	if (atomic_dec_and_test(&ctx->refcount)) {
901 		if (ctx->parent_ctx)
902 			put_ctx(ctx->parent_ctx);
903 		if (ctx->task)
904 			put_task_struct(ctx->task);
905 		call_rcu(&ctx->rcu_head, free_ctx);
906 	}
907 }
908 
909 /*
910  * Because of perf_event::ctx migration in sys_perf_event_open::move_group and
911  * perf_pmu_migrate_context() we need some magic.
912  *
913  * Those places that change perf_event::ctx will hold both
914  * perf_event_ctx::mutex of the 'old' and 'new' ctx value.
915  *
916  * Lock ordering is by mutex address. There are two other sites where
917  * perf_event_context::mutex nests and those are:
918  *
919  *  - perf_event_exit_task_context()	[ child , 0 ]
920  *      __perf_event_exit_task()
921  *        sync_child_event()
922  *          put_event()			[ parent, 1 ]
923  *
924  *  - perf_event_init_context()		[ parent, 0 ]
925  *      inherit_task_group()
926  *        inherit_group()
927  *          inherit_event()
928  *            perf_event_alloc()
929  *              perf_init_event()
930  *                perf_try_init_event()	[ child , 1 ]
931  *
932  * While it appears there is an obvious deadlock here -- the parent and child
933  * nesting levels are inverted between the two. This is in fact safe because
934  * life-time rules separate them. That is an exiting task cannot fork, and a
935  * spawning task cannot (yet) exit.
936  *
937  * But remember that that these are parent<->child context relations, and
938  * migration does not affect children, therefore these two orderings should not
939  * interact.
940  *
941  * The change in perf_event::ctx does not affect children (as claimed above)
942  * because the sys_perf_event_open() case will install a new event and break
943  * the ctx parent<->child relation, and perf_pmu_migrate_context() is only
944  * concerned with cpuctx and that doesn't have children.
945  *
946  * The places that change perf_event::ctx will issue:
947  *
948  *   perf_remove_from_context();
949  *   synchronize_rcu();
950  *   perf_install_in_context();
951  *
952  * to affect the change. The remove_from_context() + synchronize_rcu() should
953  * quiesce the event, after which we can install it in the new location. This
954  * means that only external vectors (perf_fops, prctl) can perturb the event
955  * while in transit. Therefore all such accessors should also acquire
956  * perf_event_context::mutex to serialize against this.
957  *
958  * However; because event->ctx can change while we're waiting to acquire
959  * ctx->mutex we must be careful and use the below perf_event_ctx_lock()
960  * function.
961  *
962  * Lock order:
963  *	task_struct::perf_event_mutex
964  *	  perf_event_context::mutex
965  *	    perf_event_context::lock
966  *	    perf_event::child_mutex;
967  *	    perf_event::mmap_mutex
968  *	    mmap_sem
969  */
970 static struct perf_event_context *
971 perf_event_ctx_lock_nested(struct perf_event *event, int nesting)
972 {
973 	struct perf_event_context *ctx;
974 
975 again:
976 	rcu_read_lock();
977 	ctx = ACCESS_ONCE(event->ctx);
978 	if (!atomic_inc_not_zero(&ctx->refcount)) {
979 		rcu_read_unlock();
980 		goto again;
981 	}
982 	rcu_read_unlock();
983 
984 	mutex_lock_nested(&ctx->mutex, nesting);
985 	if (event->ctx != ctx) {
986 		mutex_unlock(&ctx->mutex);
987 		put_ctx(ctx);
988 		goto again;
989 	}
990 
991 	return ctx;
992 }
993 
994 static inline struct perf_event_context *
995 perf_event_ctx_lock(struct perf_event *event)
996 {
997 	return perf_event_ctx_lock_nested(event, 0);
998 }
999 
1000 static void perf_event_ctx_unlock(struct perf_event *event,
1001 				  struct perf_event_context *ctx)
1002 {
1003 	mutex_unlock(&ctx->mutex);
1004 	put_ctx(ctx);
1005 }
1006 
1007 /*
1008  * This must be done under the ctx->lock, such as to serialize against
1009  * context_equiv(), therefore we cannot call put_ctx() since that might end up
1010  * calling scheduler related locks and ctx->lock nests inside those.
1011  */
1012 static __must_check struct perf_event_context *
1013 unclone_ctx(struct perf_event_context *ctx)
1014 {
1015 	struct perf_event_context *parent_ctx = ctx->parent_ctx;
1016 
1017 	lockdep_assert_held(&ctx->lock);
1018 
1019 	if (parent_ctx)
1020 		ctx->parent_ctx = NULL;
1021 	ctx->generation++;
1022 
1023 	return parent_ctx;
1024 }
1025 
1026 static u32 perf_event_pid(struct perf_event *event, struct task_struct *p)
1027 {
1028 	/*
1029 	 * only top level events have the pid namespace they were created in
1030 	 */
1031 	if (event->parent)
1032 		event = event->parent;
1033 
1034 	return task_tgid_nr_ns(p, event->ns);
1035 }
1036 
1037 static u32 perf_event_tid(struct perf_event *event, struct task_struct *p)
1038 {
1039 	/*
1040 	 * only top level events have the pid namespace they were created in
1041 	 */
1042 	if (event->parent)
1043 		event = event->parent;
1044 
1045 	return task_pid_nr_ns(p, event->ns);
1046 }
1047 
1048 /*
1049  * If we inherit events we want to return the parent event id
1050  * to userspace.
1051  */
1052 static u64 primary_event_id(struct perf_event *event)
1053 {
1054 	u64 id = event->id;
1055 
1056 	if (event->parent)
1057 		id = event->parent->id;
1058 
1059 	return id;
1060 }
1061 
1062 /*
1063  * Get the perf_event_context for a task and lock it.
1064  * This has to cope with with the fact that until it is locked,
1065  * the context could get moved to another task.
1066  */
1067 static struct perf_event_context *
1068 perf_lock_task_context(struct task_struct *task, int ctxn, unsigned long *flags)
1069 {
1070 	struct perf_event_context *ctx;
1071 
1072 retry:
1073 	/*
1074 	 * One of the few rules of preemptible RCU is that one cannot do
1075 	 * rcu_read_unlock() while holding a scheduler (or nested) lock when
1076 	 * part of the read side critical section was preemptible -- see
1077 	 * rcu_read_unlock_special().
1078 	 *
1079 	 * Since ctx->lock nests under rq->lock we must ensure the entire read
1080 	 * side critical section is non-preemptible.
1081 	 */
1082 	preempt_disable();
1083 	rcu_read_lock();
1084 	ctx = rcu_dereference(task->perf_event_ctxp[ctxn]);
1085 	if (ctx) {
1086 		/*
1087 		 * If this context is a clone of another, it might
1088 		 * get swapped for another underneath us by
1089 		 * perf_event_task_sched_out, though the
1090 		 * rcu_read_lock() protects us from any context
1091 		 * getting freed.  Lock the context and check if it
1092 		 * got swapped before we could get the lock, and retry
1093 		 * if so.  If we locked the right context, then it
1094 		 * can't get swapped on us any more.
1095 		 */
1096 		raw_spin_lock_irqsave(&ctx->lock, *flags);
1097 		if (ctx != rcu_dereference(task->perf_event_ctxp[ctxn])) {
1098 			raw_spin_unlock_irqrestore(&ctx->lock, *flags);
1099 			rcu_read_unlock();
1100 			preempt_enable();
1101 			goto retry;
1102 		}
1103 
1104 		if (!atomic_inc_not_zero(&ctx->refcount)) {
1105 			raw_spin_unlock_irqrestore(&ctx->lock, *flags);
1106 			ctx = NULL;
1107 		}
1108 	}
1109 	rcu_read_unlock();
1110 	preempt_enable();
1111 	return ctx;
1112 }
1113 
1114 /*
1115  * Get the context for a task and increment its pin_count so it
1116  * can't get swapped to another task.  This also increments its
1117  * reference count so that the context can't get freed.
1118  */
1119 static struct perf_event_context *
1120 perf_pin_task_context(struct task_struct *task, int ctxn)
1121 {
1122 	struct perf_event_context *ctx;
1123 	unsigned long flags;
1124 
1125 	ctx = perf_lock_task_context(task, ctxn, &flags);
1126 	if (ctx) {
1127 		++ctx->pin_count;
1128 		raw_spin_unlock_irqrestore(&ctx->lock, flags);
1129 	}
1130 	return ctx;
1131 }
1132 
1133 static void perf_unpin_context(struct perf_event_context *ctx)
1134 {
1135 	unsigned long flags;
1136 
1137 	raw_spin_lock_irqsave(&ctx->lock, flags);
1138 	--ctx->pin_count;
1139 	raw_spin_unlock_irqrestore(&ctx->lock, flags);
1140 }
1141 
1142 /*
1143  * Update the record of the current time in a context.
1144  */
1145 static void update_context_time(struct perf_event_context *ctx)
1146 {
1147 	u64 now = perf_clock();
1148 
1149 	ctx->time += now - ctx->timestamp;
1150 	ctx->timestamp = now;
1151 }
1152 
1153 static u64 perf_event_time(struct perf_event *event)
1154 {
1155 	struct perf_event_context *ctx = event->ctx;
1156 
1157 	if (is_cgroup_event(event))
1158 		return perf_cgroup_event_time(event);
1159 
1160 	return ctx ? ctx->time : 0;
1161 }
1162 
1163 /*
1164  * Update the total_time_enabled and total_time_running fields for a event.
1165  * The caller of this function needs to hold the ctx->lock.
1166  */
1167 static void update_event_times(struct perf_event *event)
1168 {
1169 	struct perf_event_context *ctx = event->ctx;
1170 	u64 run_end;
1171 
1172 	if (event->state < PERF_EVENT_STATE_INACTIVE ||
1173 	    event->group_leader->state < PERF_EVENT_STATE_INACTIVE)
1174 		return;
1175 	/*
1176 	 * in cgroup mode, time_enabled represents
1177 	 * the time the event was enabled AND active
1178 	 * tasks were in the monitored cgroup. This is
1179 	 * independent of the activity of the context as
1180 	 * there may be a mix of cgroup and non-cgroup events.
1181 	 *
1182 	 * That is why we treat cgroup events differently
1183 	 * here.
1184 	 */
1185 	if (is_cgroup_event(event))
1186 		run_end = perf_cgroup_event_time(event);
1187 	else if (ctx->is_active)
1188 		run_end = ctx->time;
1189 	else
1190 		run_end = event->tstamp_stopped;
1191 
1192 	event->total_time_enabled = run_end - event->tstamp_enabled;
1193 
1194 	if (event->state == PERF_EVENT_STATE_INACTIVE)
1195 		run_end = event->tstamp_stopped;
1196 	else
1197 		run_end = perf_event_time(event);
1198 
1199 	event->total_time_running = run_end - event->tstamp_running;
1200 
1201 }
1202 
1203 /*
1204  * Update total_time_enabled and total_time_running for all events in a group.
1205  */
1206 static void update_group_times(struct perf_event *leader)
1207 {
1208 	struct perf_event *event;
1209 
1210 	update_event_times(leader);
1211 	list_for_each_entry(event, &leader->sibling_list, group_entry)
1212 		update_event_times(event);
1213 }
1214 
1215 static struct list_head *
1216 ctx_group_list(struct perf_event *event, struct perf_event_context *ctx)
1217 {
1218 	if (event->attr.pinned)
1219 		return &ctx->pinned_groups;
1220 	else
1221 		return &ctx->flexible_groups;
1222 }
1223 
1224 /*
1225  * Add a event from the lists for its context.
1226  * Must be called with ctx->mutex and ctx->lock held.
1227  */
1228 static void
1229 list_add_event(struct perf_event *event, struct perf_event_context *ctx)
1230 {
1231 	WARN_ON_ONCE(event->attach_state & PERF_ATTACH_CONTEXT);
1232 	event->attach_state |= PERF_ATTACH_CONTEXT;
1233 
1234 	/*
1235 	 * If we're a stand alone event or group leader, we go to the context
1236 	 * list, group events are kept attached to the group so that
1237 	 * perf_group_detach can, at all times, locate all siblings.
1238 	 */
1239 	if (event->group_leader == event) {
1240 		struct list_head *list;
1241 
1242 		if (is_software_event(event))
1243 			event->group_flags |= PERF_GROUP_SOFTWARE;
1244 
1245 		list = ctx_group_list(event, ctx);
1246 		list_add_tail(&event->group_entry, list);
1247 	}
1248 
1249 	if (is_cgroup_event(event))
1250 		ctx->nr_cgroups++;
1251 
1252 	list_add_rcu(&event->event_entry, &ctx->event_list);
1253 	ctx->nr_events++;
1254 	if (event->attr.inherit_stat)
1255 		ctx->nr_stat++;
1256 
1257 	ctx->generation++;
1258 }
1259 
1260 /*
1261  * Initialize event state based on the perf_event_attr::disabled.
1262  */
1263 static inline void perf_event__state_init(struct perf_event *event)
1264 {
1265 	event->state = event->attr.disabled ? PERF_EVENT_STATE_OFF :
1266 					      PERF_EVENT_STATE_INACTIVE;
1267 }
1268 
1269 /*
1270  * Called at perf_event creation and when events are attached/detached from a
1271  * group.
1272  */
1273 static void perf_event__read_size(struct perf_event *event)
1274 {
1275 	int entry = sizeof(u64); /* value */
1276 	int size = 0;
1277 	int nr = 1;
1278 
1279 	if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
1280 		size += sizeof(u64);
1281 
1282 	if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
1283 		size += sizeof(u64);
1284 
1285 	if (event->attr.read_format & PERF_FORMAT_ID)
1286 		entry += sizeof(u64);
1287 
1288 	if (event->attr.read_format & PERF_FORMAT_GROUP) {
1289 		nr += event->group_leader->nr_siblings;
1290 		size += sizeof(u64);
1291 	}
1292 
1293 	size += entry * nr;
1294 	event->read_size = size;
1295 }
1296 
1297 static void perf_event__header_size(struct perf_event *event)
1298 {
1299 	struct perf_sample_data *data;
1300 	u64 sample_type = event->attr.sample_type;
1301 	u16 size = 0;
1302 
1303 	perf_event__read_size(event);
1304 
1305 	if (sample_type & PERF_SAMPLE_IP)
1306 		size += sizeof(data->ip);
1307 
1308 	if (sample_type & PERF_SAMPLE_ADDR)
1309 		size += sizeof(data->addr);
1310 
1311 	if (sample_type & PERF_SAMPLE_PERIOD)
1312 		size += sizeof(data->period);
1313 
1314 	if (sample_type & PERF_SAMPLE_WEIGHT)
1315 		size += sizeof(data->weight);
1316 
1317 	if (sample_type & PERF_SAMPLE_READ)
1318 		size += event->read_size;
1319 
1320 	if (sample_type & PERF_SAMPLE_DATA_SRC)
1321 		size += sizeof(data->data_src.val);
1322 
1323 	if (sample_type & PERF_SAMPLE_TRANSACTION)
1324 		size += sizeof(data->txn);
1325 
1326 	event->header_size = size;
1327 }
1328 
1329 static void perf_event__id_header_size(struct perf_event *event)
1330 {
1331 	struct perf_sample_data *data;
1332 	u64 sample_type = event->attr.sample_type;
1333 	u16 size = 0;
1334 
1335 	if (sample_type & PERF_SAMPLE_TID)
1336 		size += sizeof(data->tid_entry);
1337 
1338 	if (sample_type & PERF_SAMPLE_TIME)
1339 		size += sizeof(data->time);
1340 
1341 	if (sample_type & PERF_SAMPLE_IDENTIFIER)
1342 		size += sizeof(data->id);
1343 
1344 	if (sample_type & PERF_SAMPLE_ID)
1345 		size += sizeof(data->id);
1346 
1347 	if (sample_type & PERF_SAMPLE_STREAM_ID)
1348 		size += sizeof(data->stream_id);
1349 
1350 	if (sample_type & PERF_SAMPLE_CPU)
1351 		size += sizeof(data->cpu_entry);
1352 
1353 	event->id_header_size = size;
1354 }
1355 
1356 static void perf_group_attach(struct perf_event *event)
1357 {
1358 	struct perf_event *group_leader = event->group_leader, *pos;
1359 
1360 	/*
1361 	 * We can have double attach due to group movement in perf_event_open.
1362 	 */
1363 	if (event->attach_state & PERF_ATTACH_GROUP)
1364 		return;
1365 
1366 	event->attach_state |= PERF_ATTACH_GROUP;
1367 
1368 	if (group_leader == event)
1369 		return;
1370 
1371 	WARN_ON_ONCE(group_leader->ctx != event->ctx);
1372 
1373 	if (group_leader->group_flags & PERF_GROUP_SOFTWARE &&
1374 			!is_software_event(event))
1375 		group_leader->group_flags &= ~PERF_GROUP_SOFTWARE;
1376 
1377 	list_add_tail(&event->group_entry, &group_leader->sibling_list);
1378 	group_leader->nr_siblings++;
1379 
1380 	perf_event__header_size(group_leader);
1381 
1382 	list_for_each_entry(pos, &group_leader->sibling_list, group_entry)
1383 		perf_event__header_size(pos);
1384 }
1385 
1386 /*
1387  * Remove a event from the lists for its context.
1388  * Must be called with ctx->mutex and ctx->lock held.
1389  */
1390 static void
1391 list_del_event(struct perf_event *event, struct perf_event_context *ctx)
1392 {
1393 	struct perf_cpu_context *cpuctx;
1394 
1395 	WARN_ON_ONCE(event->ctx != ctx);
1396 	lockdep_assert_held(&ctx->lock);
1397 
1398 	/*
1399 	 * We can have double detach due to exit/hot-unplug + close.
1400 	 */
1401 	if (!(event->attach_state & PERF_ATTACH_CONTEXT))
1402 		return;
1403 
1404 	event->attach_state &= ~PERF_ATTACH_CONTEXT;
1405 
1406 	if (is_cgroup_event(event)) {
1407 		ctx->nr_cgroups--;
1408 		cpuctx = __get_cpu_context(ctx);
1409 		/*
1410 		 * if there are no more cgroup events
1411 		 * then cler cgrp to avoid stale pointer
1412 		 * in update_cgrp_time_from_cpuctx()
1413 		 */
1414 		if (!ctx->nr_cgroups)
1415 			cpuctx->cgrp = NULL;
1416 	}
1417 
1418 	ctx->nr_events--;
1419 	if (event->attr.inherit_stat)
1420 		ctx->nr_stat--;
1421 
1422 	list_del_rcu(&event->event_entry);
1423 
1424 	if (event->group_leader == event)
1425 		list_del_init(&event->group_entry);
1426 
1427 	update_group_times(event);
1428 
1429 	/*
1430 	 * If event was in error state, then keep it
1431 	 * that way, otherwise bogus counts will be
1432 	 * returned on read(). The only way to get out
1433 	 * of error state is by explicit re-enabling
1434 	 * of the event
1435 	 */
1436 	if (event->state > PERF_EVENT_STATE_OFF)
1437 		event->state = PERF_EVENT_STATE_OFF;
1438 
1439 	ctx->generation++;
1440 }
1441 
1442 static void perf_group_detach(struct perf_event *event)
1443 {
1444 	struct perf_event *sibling, *tmp;
1445 	struct list_head *list = NULL;
1446 
1447 	/*
1448 	 * We can have double detach due to exit/hot-unplug + close.
1449 	 */
1450 	if (!(event->attach_state & PERF_ATTACH_GROUP))
1451 		return;
1452 
1453 	event->attach_state &= ~PERF_ATTACH_GROUP;
1454 
1455 	/*
1456 	 * If this is a sibling, remove it from its group.
1457 	 */
1458 	if (event->group_leader != event) {
1459 		list_del_init(&event->group_entry);
1460 		event->group_leader->nr_siblings--;
1461 		goto out;
1462 	}
1463 
1464 	if (!list_empty(&event->group_entry))
1465 		list = &event->group_entry;
1466 
1467 	/*
1468 	 * If this was a group event with sibling events then
1469 	 * upgrade the siblings to singleton events by adding them
1470 	 * to whatever list we are on.
1471 	 */
1472 	list_for_each_entry_safe(sibling, tmp, &event->sibling_list, group_entry) {
1473 		if (list)
1474 			list_move_tail(&sibling->group_entry, list);
1475 		sibling->group_leader = sibling;
1476 
1477 		/* Inherit group flags from the previous leader */
1478 		sibling->group_flags = event->group_flags;
1479 
1480 		WARN_ON_ONCE(sibling->ctx != event->ctx);
1481 	}
1482 
1483 out:
1484 	perf_event__header_size(event->group_leader);
1485 
1486 	list_for_each_entry(tmp, &event->group_leader->sibling_list, group_entry)
1487 		perf_event__header_size(tmp);
1488 }
1489 
1490 /*
1491  * User event without the task.
1492  */
1493 static bool is_orphaned_event(struct perf_event *event)
1494 {
1495 	return event && !is_kernel_event(event) && !event->owner;
1496 }
1497 
1498 /*
1499  * Event has a parent but parent's task finished and it's
1500  * alive only because of children holding refference.
1501  */
1502 static bool is_orphaned_child(struct perf_event *event)
1503 {
1504 	return is_orphaned_event(event->parent);
1505 }
1506 
1507 static void orphans_remove_work(struct work_struct *work);
1508 
1509 static void schedule_orphans_remove(struct perf_event_context *ctx)
1510 {
1511 	if (!ctx->task || ctx->orphans_remove_sched || !perf_wq)
1512 		return;
1513 
1514 	if (queue_delayed_work(perf_wq, &ctx->orphans_remove, 1)) {
1515 		get_ctx(ctx);
1516 		ctx->orphans_remove_sched = true;
1517 	}
1518 }
1519 
1520 static int __init perf_workqueue_init(void)
1521 {
1522 	perf_wq = create_singlethread_workqueue("perf");
1523 	WARN(!perf_wq, "failed to create perf workqueue\n");
1524 	return perf_wq ? 0 : -1;
1525 }
1526 
1527 core_initcall(perf_workqueue_init);
1528 
1529 static inline int
1530 event_filter_match(struct perf_event *event)
1531 {
1532 	return (event->cpu == -1 || event->cpu == smp_processor_id())
1533 	    && perf_cgroup_match(event);
1534 }
1535 
1536 static void
1537 event_sched_out(struct perf_event *event,
1538 		  struct perf_cpu_context *cpuctx,
1539 		  struct perf_event_context *ctx)
1540 {
1541 	u64 tstamp = perf_event_time(event);
1542 	u64 delta;
1543 
1544 	WARN_ON_ONCE(event->ctx != ctx);
1545 	lockdep_assert_held(&ctx->lock);
1546 
1547 	/*
1548 	 * An event which could not be activated because of
1549 	 * filter mismatch still needs to have its timings
1550 	 * maintained, otherwise bogus information is return
1551 	 * via read() for time_enabled, time_running:
1552 	 */
1553 	if (event->state == PERF_EVENT_STATE_INACTIVE
1554 	    && !event_filter_match(event)) {
1555 		delta = tstamp - event->tstamp_stopped;
1556 		event->tstamp_running += delta;
1557 		event->tstamp_stopped = tstamp;
1558 	}
1559 
1560 	if (event->state != PERF_EVENT_STATE_ACTIVE)
1561 		return;
1562 
1563 	perf_pmu_disable(event->pmu);
1564 
1565 	event->state = PERF_EVENT_STATE_INACTIVE;
1566 	if (event->pending_disable) {
1567 		event->pending_disable = 0;
1568 		event->state = PERF_EVENT_STATE_OFF;
1569 	}
1570 	event->tstamp_stopped = tstamp;
1571 	event->pmu->del(event, 0);
1572 	event->oncpu = -1;
1573 
1574 	if (!is_software_event(event))
1575 		cpuctx->active_oncpu--;
1576 	if (!--ctx->nr_active)
1577 		perf_event_ctx_deactivate(ctx);
1578 	if (event->attr.freq && event->attr.sample_freq)
1579 		ctx->nr_freq--;
1580 	if (event->attr.exclusive || !cpuctx->active_oncpu)
1581 		cpuctx->exclusive = 0;
1582 
1583 	if (is_orphaned_child(event))
1584 		schedule_orphans_remove(ctx);
1585 
1586 	perf_pmu_enable(event->pmu);
1587 }
1588 
1589 static void
1590 group_sched_out(struct perf_event *group_event,
1591 		struct perf_cpu_context *cpuctx,
1592 		struct perf_event_context *ctx)
1593 {
1594 	struct perf_event *event;
1595 	int state = group_event->state;
1596 
1597 	event_sched_out(group_event, cpuctx, ctx);
1598 
1599 	/*
1600 	 * Schedule out siblings (if any):
1601 	 */
1602 	list_for_each_entry(event, &group_event->sibling_list, group_entry)
1603 		event_sched_out(event, cpuctx, ctx);
1604 
1605 	if (state == PERF_EVENT_STATE_ACTIVE && group_event->attr.exclusive)
1606 		cpuctx->exclusive = 0;
1607 }
1608 
1609 struct remove_event {
1610 	struct perf_event *event;
1611 	bool detach_group;
1612 };
1613 
1614 /*
1615  * Cross CPU call to remove a performance event
1616  *
1617  * We disable the event on the hardware level first. After that we
1618  * remove it from the context list.
1619  */
1620 static int __perf_remove_from_context(void *info)
1621 {
1622 	struct remove_event *re = info;
1623 	struct perf_event *event = re->event;
1624 	struct perf_event_context *ctx = event->ctx;
1625 	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
1626 
1627 	raw_spin_lock(&ctx->lock);
1628 	event_sched_out(event, cpuctx, ctx);
1629 	if (re->detach_group)
1630 		perf_group_detach(event);
1631 	list_del_event(event, ctx);
1632 	if (!ctx->nr_events && cpuctx->task_ctx == ctx) {
1633 		ctx->is_active = 0;
1634 		cpuctx->task_ctx = NULL;
1635 	}
1636 	raw_spin_unlock(&ctx->lock);
1637 
1638 	return 0;
1639 }
1640 
1641 
1642 /*
1643  * Remove the event from a task's (or a CPU's) list of events.
1644  *
1645  * CPU events are removed with a smp call. For task events we only
1646  * call when the task is on a CPU.
1647  *
1648  * If event->ctx is a cloned context, callers must make sure that
1649  * every task struct that event->ctx->task could possibly point to
1650  * remains valid.  This is OK when called from perf_release since
1651  * that only calls us on the top-level context, which can't be a clone.
1652  * When called from perf_event_exit_task, it's OK because the
1653  * context has been detached from its task.
1654  */
1655 static void perf_remove_from_context(struct perf_event *event, bool detach_group)
1656 {
1657 	struct perf_event_context *ctx = event->ctx;
1658 	struct task_struct *task = ctx->task;
1659 	struct remove_event re = {
1660 		.event = event,
1661 		.detach_group = detach_group,
1662 	};
1663 
1664 	lockdep_assert_held(&ctx->mutex);
1665 
1666 	if (!task) {
1667 		/*
1668 		 * Per cpu events are removed via an smp call. The removal can
1669 		 * fail if the CPU is currently offline, but in that case we
1670 		 * already called __perf_remove_from_context from
1671 		 * perf_event_exit_cpu.
1672 		 */
1673 		cpu_function_call(event->cpu, __perf_remove_from_context, &re);
1674 		return;
1675 	}
1676 
1677 retry:
1678 	if (!task_function_call(task, __perf_remove_from_context, &re))
1679 		return;
1680 
1681 	raw_spin_lock_irq(&ctx->lock);
1682 	/*
1683 	 * If we failed to find a running task, but find the context active now
1684 	 * that we've acquired the ctx->lock, retry.
1685 	 */
1686 	if (ctx->is_active) {
1687 		raw_spin_unlock_irq(&ctx->lock);
1688 		/*
1689 		 * Reload the task pointer, it might have been changed by
1690 		 * a concurrent perf_event_context_sched_out().
1691 		 */
1692 		task = ctx->task;
1693 		goto retry;
1694 	}
1695 
1696 	/*
1697 	 * Since the task isn't running, its safe to remove the event, us
1698 	 * holding the ctx->lock ensures the task won't get scheduled in.
1699 	 */
1700 	if (detach_group)
1701 		perf_group_detach(event);
1702 	list_del_event(event, ctx);
1703 	raw_spin_unlock_irq(&ctx->lock);
1704 }
1705 
1706 /*
1707  * Cross CPU call to disable a performance event
1708  */
1709 int __perf_event_disable(void *info)
1710 {
1711 	struct perf_event *event = info;
1712 	struct perf_event_context *ctx = event->ctx;
1713 	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
1714 
1715 	/*
1716 	 * If this is a per-task event, need to check whether this
1717 	 * event's task is the current task on this cpu.
1718 	 *
1719 	 * Can trigger due to concurrent perf_event_context_sched_out()
1720 	 * flipping contexts around.
1721 	 */
1722 	if (ctx->task && cpuctx->task_ctx != ctx)
1723 		return -EINVAL;
1724 
1725 	raw_spin_lock(&ctx->lock);
1726 
1727 	/*
1728 	 * If the event is on, turn it off.
1729 	 * If it is in error state, leave it in error state.
1730 	 */
1731 	if (event->state >= PERF_EVENT_STATE_INACTIVE) {
1732 		update_context_time(ctx);
1733 		update_cgrp_time_from_event(event);
1734 		update_group_times(event);
1735 		if (event == event->group_leader)
1736 			group_sched_out(event, cpuctx, ctx);
1737 		else
1738 			event_sched_out(event, cpuctx, ctx);
1739 		event->state = PERF_EVENT_STATE_OFF;
1740 	}
1741 
1742 	raw_spin_unlock(&ctx->lock);
1743 
1744 	return 0;
1745 }
1746 
1747 /*
1748  * Disable a event.
1749  *
1750  * If event->ctx is a cloned context, callers must make sure that
1751  * every task struct that event->ctx->task could possibly point to
1752  * remains valid.  This condition is satisifed when called through
1753  * perf_event_for_each_child or perf_event_for_each because they
1754  * hold the top-level event's child_mutex, so any descendant that
1755  * goes to exit will block in sync_child_event.
1756  * When called from perf_pending_event it's OK because event->ctx
1757  * is the current context on this CPU and preemption is disabled,
1758  * hence we can't get into perf_event_task_sched_out for this context.
1759  */
1760 static void _perf_event_disable(struct perf_event *event)
1761 {
1762 	struct perf_event_context *ctx = event->ctx;
1763 	struct task_struct *task = ctx->task;
1764 
1765 	if (!task) {
1766 		/*
1767 		 * Disable the event on the cpu that it's on
1768 		 */
1769 		cpu_function_call(event->cpu, __perf_event_disable, event);
1770 		return;
1771 	}
1772 
1773 retry:
1774 	if (!task_function_call(task, __perf_event_disable, event))
1775 		return;
1776 
1777 	raw_spin_lock_irq(&ctx->lock);
1778 	/*
1779 	 * If the event is still active, we need to retry the cross-call.
1780 	 */
1781 	if (event->state == PERF_EVENT_STATE_ACTIVE) {
1782 		raw_spin_unlock_irq(&ctx->lock);
1783 		/*
1784 		 * Reload the task pointer, it might have been changed by
1785 		 * a concurrent perf_event_context_sched_out().
1786 		 */
1787 		task = ctx->task;
1788 		goto retry;
1789 	}
1790 
1791 	/*
1792 	 * Since we have the lock this context can't be scheduled
1793 	 * in, so we can change the state safely.
1794 	 */
1795 	if (event->state == PERF_EVENT_STATE_INACTIVE) {
1796 		update_group_times(event);
1797 		event->state = PERF_EVENT_STATE_OFF;
1798 	}
1799 	raw_spin_unlock_irq(&ctx->lock);
1800 }
1801 
1802 /*
1803  * Strictly speaking kernel users cannot create groups and therefore this
1804  * interface does not need the perf_event_ctx_lock() magic.
1805  */
1806 void perf_event_disable(struct perf_event *event)
1807 {
1808 	struct perf_event_context *ctx;
1809 
1810 	ctx = perf_event_ctx_lock(event);
1811 	_perf_event_disable(event);
1812 	perf_event_ctx_unlock(event, ctx);
1813 }
1814 EXPORT_SYMBOL_GPL(perf_event_disable);
1815 
1816 static void perf_set_shadow_time(struct perf_event *event,
1817 				 struct perf_event_context *ctx,
1818 				 u64 tstamp)
1819 {
1820 	/*
1821 	 * use the correct time source for the time snapshot
1822 	 *
1823 	 * We could get by without this by leveraging the
1824 	 * fact that to get to this function, the caller
1825 	 * has most likely already called update_context_time()
1826 	 * and update_cgrp_time_xx() and thus both timestamp
1827 	 * are identical (or very close). Given that tstamp is,
1828 	 * already adjusted for cgroup, we could say that:
1829 	 *    tstamp - ctx->timestamp
1830 	 * is equivalent to
1831 	 *    tstamp - cgrp->timestamp.
1832 	 *
1833 	 * Then, in perf_output_read(), the calculation would
1834 	 * work with no changes because:
1835 	 * - event is guaranteed scheduled in
1836 	 * - no scheduled out in between
1837 	 * - thus the timestamp would be the same
1838 	 *
1839 	 * But this is a bit hairy.
1840 	 *
1841 	 * So instead, we have an explicit cgroup call to remain
1842 	 * within the time time source all along. We believe it
1843 	 * is cleaner and simpler to understand.
1844 	 */
1845 	if (is_cgroup_event(event))
1846 		perf_cgroup_set_shadow_time(event, tstamp);
1847 	else
1848 		event->shadow_ctx_time = tstamp - ctx->timestamp;
1849 }
1850 
1851 #define MAX_INTERRUPTS (~0ULL)
1852 
1853 static void perf_log_throttle(struct perf_event *event, int enable);
1854 static void perf_log_itrace_start(struct perf_event *event);
1855 
1856 static int
1857 event_sched_in(struct perf_event *event,
1858 		 struct perf_cpu_context *cpuctx,
1859 		 struct perf_event_context *ctx)
1860 {
1861 	u64 tstamp = perf_event_time(event);
1862 	int ret = 0;
1863 
1864 	lockdep_assert_held(&ctx->lock);
1865 
1866 	if (event->state <= PERF_EVENT_STATE_OFF)
1867 		return 0;
1868 
1869 	event->state = PERF_EVENT_STATE_ACTIVE;
1870 	event->oncpu = smp_processor_id();
1871 
1872 	/*
1873 	 * Unthrottle events, since we scheduled we might have missed several
1874 	 * ticks already, also for a heavily scheduling task there is little
1875 	 * guarantee it'll get a tick in a timely manner.
1876 	 */
1877 	if (unlikely(event->hw.interrupts == MAX_INTERRUPTS)) {
1878 		perf_log_throttle(event, 1);
1879 		event->hw.interrupts = 0;
1880 	}
1881 
1882 	/*
1883 	 * The new state must be visible before we turn it on in the hardware:
1884 	 */
1885 	smp_wmb();
1886 
1887 	perf_pmu_disable(event->pmu);
1888 
1889 	event->tstamp_running += tstamp - event->tstamp_stopped;
1890 
1891 	perf_set_shadow_time(event, ctx, tstamp);
1892 
1893 	perf_log_itrace_start(event);
1894 
1895 	if (event->pmu->add(event, PERF_EF_START)) {
1896 		event->state = PERF_EVENT_STATE_INACTIVE;
1897 		event->oncpu = -1;
1898 		ret = -EAGAIN;
1899 		goto out;
1900 	}
1901 
1902 	if (!is_software_event(event))
1903 		cpuctx->active_oncpu++;
1904 	if (!ctx->nr_active++)
1905 		perf_event_ctx_activate(ctx);
1906 	if (event->attr.freq && event->attr.sample_freq)
1907 		ctx->nr_freq++;
1908 
1909 	if (event->attr.exclusive)
1910 		cpuctx->exclusive = 1;
1911 
1912 	if (is_orphaned_child(event))
1913 		schedule_orphans_remove(ctx);
1914 
1915 out:
1916 	perf_pmu_enable(event->pmu);
1917 
1918 	return ret;
1919 }
1920 
1921 static int
1922 group_sched_in(struct perf_event *group_event,
1923 	       struct perf_cpu_context *cpuctx,
1924 	       struct perf_event_context *ctx)
1925 {
1926 	struct perf_event *event, *partial_group = NULL;
1927 	struct pmu *pmu = ctx->pmu;
1928 	u64 now = ctx->time;
1929 	bool simulate = false;
1930 
1931 	if (group_event->state == PERF_EVENT_STATE_OFF)
1932 		return 0;
1933 
1934 	pmu->start_txn(pmu);
1935 
1936 	if (event_sched_in(group_event, cpuctx, ctx)) {
1937 		pmu->cancel_txn(pmu);
1938 		perf_cpu_hrtimer_restart(cpuctx);
1939 		return -EAGAIN;
1940 	}
1941 
1942 	/*
1943 	 * Schedule in siblings as one group (if any):
1944 	 */
1945 	list_for_each_entry(event, &group_event->sibling_list, group_entry) {
1946 		if (event_sched_in(event, cpuctx, ctx)) {
1947 			partial_group = event;
1948 			goto group_error;
1949 		}
1950 	}
1951 
1952 	if (!pmu->commit_txn(pmu))
1953 		return 0;
1954 
1955 group_error:
1956 	/*
1957 	 * Groups can be scheduled in as one unit only, so undo any
1958 	 * partial group before returning:
1959 	 * The events up to the failed event are scheduled out normally,
1960 	 * tstamp_stopped will be updated.
1961 	 *
1962 	 * The failed events and the remaining siblings need to have
1963 	 * their timings updated as if they had gone thru event_sched_in()
1964 	 * and event_sched_out(). This is required to get consistent timings
1965 	 * across the group. This also takes care of the case where the group
1966 	 * could never be scheduled by ensuring tstamp_stopped is set to mark
1967 	 * the time the event was actually stopped, such that time delta
1968 	 * calculation in update_event_times() is correct.
1969 	 */
1970 	list_for_each_entry(event, &group_event->sibling_list, group_entry) {
1971 		if (event == partial_group)
1972 			simulate = true;
1973 
1974 		if (simulate) {
1975 			event->tstamp_running += now - event->tstamp_stopped;
1976 			event->tstamp_stopped = now;
1977 		} else {
1978 			event_sched_out(event, cpuctx, ctx);
1979 		}
1980 	}
1981 	event_sched_out(group_event, cpuctx, ctx);
1982 
1983 	pmu->cancel_txn(pmu);
1984 
1985 	perf_cpu_hrtimer_restart(cpuctx);
1986 
1987 	return -EAGAIN;
1988 }
1989 
1990 /*
1991  * Work out whether we can put this event group on the CPU now.
1992  */
1993 static int group_can_go_on(struct perf_event *event,
1994 			   struct perf_cpu_context *cpuctx,
1995 			   int can_add_hw)
1996 {
1997 	/*
1998 	 * Groups consisting entirely of software events can always go on.
1999 	 */
2000 	if (event->group_flags & PERF_GROUP_SOFTWARE)
2001 		return 1;
2002 	/*
2003 	 * If an exclusive group is already on, no other hardware
2004 	 * events can go on.
2005 	 */
2006 	if (cpuctx->exclusive)
2007 		return 0;
2008 	/*
2009 	 * If this group is exclusive and there are already
2010 	 * events on the CPU, it can't go on.
2011 	 */
2012 	if (event->attr.exclusive && cpuctx->active_oncpu)
2013 		return 0;
2014 	/*
2015 	 * Otherwise, try to add it if all previous groups were able
2016 	 * to go on.
2017 	 */
2018 	return can_add_hw;
2019 }
2020 
2021 static void add_event_to_ctx(struct perf_event *event,
2022 			       struct perf_event_context *ctx)
2023 {
2024 	u64 tstamp = perf_event_time(event);
2025 
2026 	list_add_event(event, ctx);
2027 	perf_group_attach(event);
2028 	event->tstamp_enabled = tstamp;
2029 	event->tstamp_running = tstamp;
2030 	event->tstamp_stopped = tstamp;
2031 }
2032 
2033 static void task_ctx_sched_out(struct perf_event_context *ctx);
2034 static void
2035 ctx_sched_in(struct perf_event_context *ctx,
2036 	     struct perf_cpu_context *cpuctx,
2037 	     enum event_type_t event_type,
2038 	     struct task_struct *task);
2039 
2040 static void perf_event_sched_in(struct perf_cpu_context *cpuctx,
2041 				struct perf_event_context *ctx,
2042 				struct task_struct *task)
2043 {
2044 	cpu_ctx_sched_in(cpuctx, EVENT_PINNED, task);
2045 	if (ctx)
2046 		ctx_sched_in(ctx, cpuctx, EVENT_PINNED, task);
2047 	cpu_ctx_sched_in(cpuctx, EVENT_FLEXIBLE, task);
2048 	if (ctx)
2049 		ctx_sched_in(ctx, cpuctx, EVENT_FLEXIBLE, task);
2050 }
2051 
2052 /*
2053  * Cross CPU call to install and enable a performance event
2054  *
2055  * Must be called with ctx->mutex held
2056  */
2057 static int  __perf_install_in_context(void *info)
2058 {
2059 	struct perf_event *event = info;
2060 	struct perf_event_context *ctx = event->ctx;
2061 	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
2062 	struct perf_event_context *task_ctx = cpuctx->task_ctx;
2063 	struct task_struct *task = current;
2064 
2065 	perf_ctx_lock(cpuctx, task_ctx);
2066 	perf_pmu_disable(cpuctx->ctx.pmu);
2067 
2068 	/*
2069 	 * If there was an active task_ctx schedule it out.
2070 	 */
2071 	if (task_ctx)
2072 		task_ctx_sched_out(task_ctx);
2073 
2074 	/*
2075 	 * If the context we're installing events in is not the
2076 	 * active task_ctx, flip them.
2077 	 */
2078 	if (ctx->task && task_ctx != ctx) {
2079 		if (task_ctx)
2080 			raw_spin_unlock(&task_ctx->lock);
2081 		raw_spin_lock(&ctx->lock);
2082 		task_ctx = ctx;
2083 	}
2084 
2085 	if (task_ctx) {
2086 		cpuctx->task_ctx = task_ctx;
2087 		task = task_ctx->task;
2088 	}
2089 
2090 	cpu_ctx_sched_out(cpuctx, EVENT_ALL);
2091 
2092 	update_context_time(ctx);
2093 	/*
2094 	 * update cgrp time only if current cgrp
2095 	 * matches event->cgrp. Must be done before
2096 	 * calling add_event_to_ctx()
2097 	 */
2098 	update_cgrp_time_from_event(event);
2099 
2100 	add_event_to_ctx(event, ctx);
2101 
2102 	/*
2103 	 * Schedule everything back in
2104 	 */
2105 	perf_event_sched_in(cpuctx, task_ctx, task);
2106 
2107 	perf_pmu_enable(cpuctx->ctx.pmu);
2108 	perf_ctx_unlock(cpuctx, task_ctx);
2109 
2110 	return 0;
2111 }
2112 
2113 /*
2114  * Attach a performance event to a context
2115  *
2116  * First we add the event to the list with the hardware enable bit
2117  * in event->hw_config cleared.
2118  *
2119  * If the event is attached to a task which is on a CPU we use a smp
2120  * call to enable it in the task context. The task might have been
2121  * scheduled away, but we check this in the smp call again.
2122  */
2123 static void
2124 perf_install_in_context(struct perf_event_context *ctx,
2125 			struct perf_event *event,
2126 			int cpu)
2127 {
2128 	struct task_struct *task = ctx->task;
2129 
2130 	lockdep_assert_held(&ctx->mutex);
2131 
2132 	event->ctx = ctx;
2133 	if (event->cpu != -1)
2134 		event->cpu = cpu;
2135 
2136 	if (!task) {
2137 		/*
2138 		 * Per cpu events are installed via an smp call and
2139 		 * the install is always successful.
2140 		 */
2141 		cpu_function_call(cpu, __perf_install_in_context, event);
2142 		return;
2143 	}
2144 
2145 retry:
2146 	if (!task_function_call(task, __perf_install_in_context, event))
2147 		return;
2148 
2149 	raw_spin_lock_irq(&ctx->lock);
2150 	/*
2151 	 * If we failed to find a running task, but find the context active now
2152 	 * that we've acquired the ctx->lock, retry.
2153 	 */
2154 	if (ctx->is_active) {
2155 		raw_spin_unlock_irq(&ctx->lock);
2156 		/*
2157 		 * Reload the task pointer, it might have been changed by
2158 		 * a concurrent perf_event_context_sched_out().
2159 		 */
2160 		task = ctx->task;
2161 		goto retry;
2162 	}
2163 
2164 	/*
2165 	 * Since the task isn't running, its safe to add the event, us holding
2166 	 * the ctx->lock ensures the task won't get scheduled in.
2167 	 */
2168 	add_event_to_ctx(event, ctx);
2169 	raw_spin_unlock_irq(&ctx->lock);
2170 }
2171 
2172 /*
2173  * Put a event into inactive state and update time fields.
2174  * Enabling the leader of a group effectively enables all
2175  * the group members that aren't explicitly disabled, so we
2176  * have to update their ->tstamp_enabled also.
2177  * Note: this works for group members as well as group leaders
2178  * since the non-leader members' sibling_lists will be empty.
2179  */
2180 static void __perf_event_mark_enabled(struct perf_event *event)
2181 {
2182 	struct perf_event *sub;
2183 	u64 tstamp = perf_event_time(event);
2184 
2185 	event->state = PERF_EVENT_STATE_INACTIVE;
2186 	event->tstamp_enabled = tstamp - event->total_time_enabled;
2187 	list_for_each_entry(sub, &event->sibling_list, group_entry) {
2188 		if (sub->state >= PERF_EVENT_STATE_INACTIVE)
2189 			sub->tstamp_enabled = tstamp - sub->total_time_enabled;
2190 	}
2191 }
2192 
2193 /*
2194  * Cross CPU call to enable a performance event
2195  */
2196 static int __perf_event_enable(void *info)
2197 {
2198 	struct perf_event *event = info;
2199 	struct perf_event_context *ctx = event->ctx;
2200 	struct perf_event *leader = event->group_leader;
2201 	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
2202 	int err;
2203 
2204 	/*
2205 	 * There's a time window between 'ctx->is_active' check
2206 	 * in perf_event_enable function and this place having:
2207 	 *   - IRQs on
2208 	 *   - ctx->lock unlocked
2209 	 *
2210 	 * where the task could be killed and 'ctx' deactivated
2211 	 * by perf_event_exit_task.
2212 	 */
2213 	if (!ctx->is_active)
2214 		return -EINVAL;
2215 
2216 	raw_spin_lock(&ctx->lock);
2217 	update_context_time(ctx);
2218 
2219 	if (event->state >= PERF_EVENT_STATE_INACTIVE)
2220 		goto unlock;
2221 
2222 	/*
2223 	 * set current task's cgroup time reference point
2224 	 */
2225 	perf_cgroup_set_timestamp(current, ctx);
2226 
2227 	__perf_event_mark_enabled(event);
2228 
2229 	if (!event_filter_match(event)) {
2230 		if (is_cgroup_event(event))
2231 			perf_cgroup_defer_enabled(event);
2232 		goto unlock;
2233 	}
2234 
2235 	/*
2236 	 * If the event is in a group and isn't the group leader,
2237 	 * then don't put it on unless the group is on.
2238 	 */
2239 	if (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE)
2240 		goto unlock;
2241 
2242 	if (!group_can_go_on(event, cpuctx, 1)) {
2243 		err = -EEXIST;
2244 	} else {
2245 		if (event == leader)
2246 			err = group_sched_in(event, cpuctx, ctx);
2247 		else
2248 			err = event_sched_in(event, cpuctx, ctx);
2249 	}
2250 
2251 	if (err) {
2252 		/*
2253 		 * If this event can't go on and it's part of a
2254 		 * group, then the whole group has to come off.
2255 		 */
2256 		if (leader != event) {
2257 			group_sched_out(leader, cpuctx, ctx);
2258 			perf_cpu_hrtimer_restart(cpuctx);
2259 		}
2260 		if (leader->attr.pinned) {
2261 			update_group_times(leader);
2262 			leader->state = PERF_EVENT_STATE_ERROR;
2263 		}
2264 	}
2265 
2266 unlock:
2267 	raw_spin_unlock(&ctx->lock);
2268 
2269 	return 0;
2270 }
2271 
2272 /*
2273  * Enable a event.
2274  *
2275  * If event->ctx is a cloned context, callers must make sure that
2276  * every task struct that event->ctx->task could possibly point to
2277  * remains valid.  This condition is satisfied when called through
2278  * perf_event_for_each_child or perf_event_for_each as described
2279  * for perf_event_disable.
2280  */
2281 static void _perf_event_enable(struct perf_event *event)
2282 {
2283 	struct perf_event_context *ctx = event->ctx;
2284 	struct task_struct *task = ctx->task;
2285 
2286 	if (!task) {
2287 		/*
2288 		 * Enable the event on the cpu that it's on
2289 		 */
2290 		cpu_function_call(event->cpu, __perf_event_enable, event);
2291 		return;
2292 	}
2293 
2294 	raw_spin_lock_irq(&ctx->lock);
2295 	if (event->state >= PERF_EVENT_STATE_INACTIVE)
2296 		goto out;
2297 
2298 	/*
2299 	 * If the event is in error state, clear that first.
2300 	 * That way, if we see the event in error state below, we
2301 	 * know that it has gone back into error state, as distinct
2302 	 * from the task having been scheduled away before the
2303 	 * cross-call arrived.
2304 	 */
2305 	if (event->state == PERF_EVENT_STATE_ERROR)
2306 		event->state = PERF_EVENT_STATE_OFF;
2307 
2308 retry:
2309 	if (!ctx->is_active) {
2310 		__perf_event_mark_enabled(event);
2311 		goto out;
2312 	}
2313 
2314 	raw_spin_unlock_irq(&ctx->lock);
2315 
2316 	if (!task_function_call(task, __perf_event_enable, event))
2317 		return;
2318 
2319 	raw_spin_lock_irq(&ctx->lock);
2320 
2321 	/*
2322 	 * If the context is active and the event is still off,
2323 	 * we need to retry the cross-call.
2324 	 */
2325 	if (ctx->is_active && event->state == PERF_EVENT_STATE_OFF) {
2326 		/*
2327 		 * task could have been flipped by a concurrent
2328 		 * perf_event_context_sched_out()
2329 		 */
2330 		task = ctx->task;
2331 		goto retry;
2332 	}
2333 
2334 out:
2335 	raw_spin_unlock_irq(&ctx->lock);
2336 }
2337 
2338 /*
2339  * See perf_event_disable();
2340  */
2341 void perf_event_enable(struct perf_event *event)
2342 {
2343 	struct perf_event_context *ctx;
2344 
2345 	ctx = perf_event_ctx_lock(event);
2346 	_perf_event_enable(event);
2347 	perf_event_ctx_unlock(event, ctx);
2348 }
2349 EXPORT_SYMBOL_GPL(perf_event_enable);
2350 
2351 static int _perf_event_refresh(struct perf_event *event, int refresh)
2352 {
2353 	/*
2354 	 * not supported on inherited events
2355 	 */
2356 	if (event->attr.inherit || !is_sampling_event(event))
2357 		return -EINVAL;
2358 
2359 	atomic_add(refresh, &event->event_limit);
2360 	_perf_event_enable(event);
2361 
2362 	return 0;
2363 }
2364 
2365 /*
2366  * See perf_event_disable()
2367  */
2368 int perf_event_refresh(struct perf_event *event, int refresh)
2369 {
2370 	struct perf_event_context *ctx;
2371 	int ret;
2372 
2373 	ctx = perf_event_ctx_lock(event);
2374 	ret = _perf_event_refresh(event, refresh);
2375 	perf_event_ctx_unlock(event, ctx);
2376 
2377 	return ret;
2378 }
2379 EXPORT_SYMBOL_GPL(perf_event_refresh);
2380 
2381 static void ctx_sched_out(struct perf_event_context *ctx,
2382 			  struct perf_cpu_context *cpuctx,
2383 			  enum event_type_t event_type)
2384 {
2385 	struct perf_event *event;
2386 	int is_active = ctx->is_active;
2387 
2388 	ctx->is_active &= ~event_type;
2389 	if (likely(!ctx->nr_events))
2390 		return;
2391 
2392 	update_context_time(ctx);
2393 	update_cgrp_time_from_cpuctx(cpuctx);
2394 	if (!ctx->nr_active)
2395 		return;
2396 
2397 	perf_pmu_disable(ctx->pmu);
2398 	if ((is_active & EVENT_PINNED) && (event_type & EVENT_PINNED)) {
2399 		list_for_each_entry(event, &ctx->pinned_groups, group_entry)
2400 			group_sched_out(event, cpuctx, ctx);
2401 	}
2402 
2403 	if ((is_active & EVENT_FLEXIBLE) && (event_type & EVENT_FLEXIBLE)) {
2404 		list_for_each_entry(event, &ctx->flexible_groups, group_entry)
2405 			group_sched_out(event, cpuctx, ctx);
2406 	}
2407 	perf_pmu_enable(ctx->pmu);
2408 }
2409 
2410 /*
2411  * Test whether two contexts are equivalent, i.e. whether they have both been
2412  * cloned from the same version of the same context.
2413  *
2414  * Equivalence is measured using a generation number in the context that is
2415  * incremented on each modification to it; see unclone_ctx(), list_add_event()
2416  * and list_del_event().
2417  */
2418 static int context_equiv(struct perf_event_context *ctx1,
2419 			 struct perf_event_context *ctx2)
2420 {
2421 	lockdep_assert_held(&ctx1->lock);
2422 	lockdep_assert_held(&ctx2->lock);
2423 
2424 	/* Pinning disables the swap optimization */
2425 	if (ctx1->pin_count || ctx2->pin_count)
2426 		return 0;
2427 
2428 	/* If ctx1 is the parent of ctx2 */
2429 	if (ctx1 == ctx2->parent_ctx && ctx1->generation == ctx2->parent_gen)
2430 		return 1;
2431 
2432 	/* If ctx2 is the parent of ctx1 */
2433 	if (ctx1->parent_ctx == ctx2 && ctx1->parent_gen == ctx2->generation)
2434 		return 1;
2435 
2436 	/*
2437 	 * If ctx1 and ctx2 have the same parent; we flatten the parent
2438 	 * hierarchy, see perf_event_init_context().
2439 	 */
2440 	if (ctx1->parent_ctx && ctx1->parent_ctx == ctx2->parent_ctx &&
2441 			ctx1->parent_gen == ctx2->parent_gen)
2442 		return 1;
2443 
2444 	/* Unmatched */
2445 	return 0;
2446 }
2447 
2448 static void __perf_event_sync_stat(struct perf_event *event,
2449 				     struct perf_event *next_event)
2450 {
2451 	u64 value;
2452 
2453 	if (!event->attr.inherit_stat)
2454 		return;
2455 
2456 	/*
2457 	 * Update the event value, we cannot use perf_event_read()
2458 	 * because we're in the middle of a context switch and have IRQs
2459 	 * disabled, which upsets smp_call_function_single(), however
2460 	 * we know the event must be on the current CPU, therefore we
2461 	 * don't need to use it.
2462 	 */
2463 	switch (event->state) {
2464 	case PERF_EVENT_STATE_ACTIVE:
2465 		event->pmu->read(event);
2466 		/* fall-through */
2467 
2468 	case PERF_EVENT_STATE_INACTIVE:
2469 		update_event_times(event);
2470 		break;
2471 
2472 	default:
2473 		break;
2474 	}
2475 
2476 	/*
2477 	 * In order to keep per-task stats reliable we need to flip the event
2478 	 * values when we flip the contexts.
2479 	 */
2480 	value = local64_read(&next_event->count);
2481 	value = local64_xchg(&event->count, value);
2482 	local64_set(&next_event->count, value);
2483 
2484 	swap(event->total_time_enabled, next_event->total_time_enabled);
2485 	swap(event->total_time_running, next_event->total_time_running);
2486 
2487 	/*
2488 	 * Since we swizzled the values, update the user visible data too.
2489 	 */
2490 	perf_event_update_userpage(event);
2491 	perf_event_update_userpage(next_event);
2492 }
2493 
2494 static void perf_event_sync_stat(struct perf_event_context *ctx,
2495 				   struct perf_event_context *next_ctx)
2496 {
2497 	struct perf_event *event, *next_event;
2498 
2499 	if (!ctx->nr_stat)
2500 		return;
2501 
2502 	update_context_time(ctx);
2503 
2504 	event = list_first_entry(&ctx->event_list,
2505 				   struct perf_event, event_entry);
2506 
2507 	next_event = list_first_entry(&next_ctx->event_list,
2508 					struct perf_event, event_entry);
2509 
2510 	while (&event->event_entry != &ctx->event_list &&
2511 	       &next_event->event_entry != &next_ctx->event_list) {
2512 
2513 		__perf_event_sync_stat(event, next_event);
2514 
2515 		event = list_next_entry(event, event_entry);
2516 		next_event = list_next_entry(next_event, event_entry);
2517 	}
2518 }
2519 
2520 static void perf_event_context_sched_out(struct task_struct *task, int ctxn,
2521 					 struct task_struct *next)
2522 {
2523 	struct perf_event_context *ctx = task->perf_event_ctxp[ctxn];
2524 	struct perf_event_context *next_ctx;
2525 	struct perf_event_context *parent, *next_parent;
2526 	struct perf_cpu_context *cpuctx;
2527 	int do_switch = 1;
2528 
2529 	if (likely(!ctx))
2530 		return;
2531 
2532 	cpuctx = __get_cpu_context(ctx);
2533 	if (!cpuctx->task_ctx)
2534 		return;
2535 
2536 	rcu_read_lock();
2537 	next_ctx = next->perf_event_ctxp[ctxn];
2538 	if (!next_ctx)
2539 		goto unlock;
2540 
2541 	parent = rcu_dereference(ctx->parent_ctx);
2542 	next_parent = rcu_dereference(next_ctx->parent_ctx);
2543 
2544 	/* If neither context have a parent context; they cannot be clones. */
2545 	if (!parent && !next_parent)
2546 		goto unlock;
2547 
2548 	if (next_parent == ctx || next_ctx == parent || next_parent == parent) {
2549 		/*
2550 		 * Looks like the two contexts are clones, so we might be
2551 		 * able to optimize the context switch.  We lock both
2552 		 * contexts and check that they are clones under the
2553 		 * lock (including re-checking that neither has been
2554 		 * uncloned in the meantime).  It doesn't matter which
2555 		 * order we take the locks because no other cpu could
2556 		 * be trying to lock both of these tasks.
2557 		 */
2558 		raw_spin_lock(&ctx->lock);
2559 		raw_spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING);
2560 		if (context_equiv(ctx, next_ctx)) {
2561 			/*
2562 			 * XXX do we need a memory barrier of sorts
2563 			 * wrt to rcu_dereference() of perf_event_ctxp
2564 			 */
2565 			task->perf_event_ctxp[ctxn] = next_ctx;
2566 			next->perf_event_ctxp[ctxn] = ctx;
2567 			ctx->task = next;
2568 			next_ctx->task = task;
2569 
2570 			swap(ctx->task_ctx_data, next_ctx->task_ctx_data);
2571 
2572 			do_switch = 0;
2573 
2574 			perf_event_sync_stat(ctx, next_ctx);
2575 		}
2576 		raw_spin_unlock(&next_ctx->lock);
2577 		raw_spin_unlock(&ctx->lock);
2578 	}
2579 unlock:
2580 	rcu_read_unlock();
2581 
2582 	if (do_switch) {
2583 		raw_spin_lock(&ctx->lock);
2584 		ctx_sched_out(ctx, cpuctx, EVENT_ALL);
2585 		cpuctx->task_ctx = NULL;
2586 		raw_spin_unlock(&ctx->lock);
2587 	}
2588 }
2589 
2590 void perf_sched_cb_dec(struct pmu *pmu)
2591 {
2592 	this_cpu_dec(perf_sched_cb_usages);
2593 }
2594 
2595 void perf_sched_cb_inc(struct pmu *pmu)
2596 {
2597 	this_cpu_inc(perf_sched_cb_usages);
2598 }
2599 
2600 /*
2601  * This function provides the context switch callback to the lower code
2602  * layer. It is invoked ONLY when the context switch callback is enabled.
2603  */
2604 static void perf_pmu_sched_task(struct task_struct *prev,
2605 				struct task_struct *next,
2606 				bool sched_in)
2607 {
2608 	struct perf_cpu_context *cpuctx;
2609 	struct pmu *pmu;
2610 	unsigned long flags;
2611 
2612 	if (prev == next)
2613 		return;
2614 
2615 	local_irq_save(flags);
2616 
2617 	rcu_read_lock();
2618 
2619 	list_for_each_entry_rcu(pmu, &pmus, entry) {
2620 		if (pmu->sched_task) {
2621 			cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
2622 
2623 			perf_ctx_lock(cpuctx, cpuctx->task_ctx);
2624 
2625 			perf_pmu_disable(pmu);
2626 
2627 			pmu->sched_task(cpuctx->task_ctx, sched_in);
2628 
2629 			perf_pmu_enable(pmu);
2630 
2631 			perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
2632 		}
2633 	}
2634 
2635 	rcu_read_unlock();
2636 
2637 	local_irq_restore(flags);
2638 }
2639 
2640 #define for_each_task_context_nr(ctxn)					\
2641 	for ((ctxn) = 0; (ctxn) < perf_nr_task_contexts; (ctxn)++)
2642 
2643 /*
2644  * Called from scheduler to remove the events of the current task,
2645  * with interrupts disabled.
2646  *
2647  * We stop each event and update the event value in event->count.
2648  *
2649  * This does not protect us against NMI, but disable()
2650  * sets the disabled bit in the control field of event _before_
2651  * accessing the event control register. If a NMI hits, then it will
2652  * not restart the event.
2653  */
2654 void __perf_event_task_sched_out(struct task_struct *task,
2655 				 struct task_struct *next)
2656 {
2657 	int ctxn;
2658 
2659 	if (__this_cpu_read(perf_sched_cb_usages))
2660 		perf_pmu_sched_task(task, next, false);
2661 
2662 	for_each_task_context_nr(ctxn)
2663 		perf_event_context_sched_out(task, ctxn, next);
2664 
2665 	/*
2666 	 * if cgroup events exist on this CPU, then we need
2667 	 * to check if we have to switch out PMU state.
2668 	 * cgroup event are system-wide mode only
2669 	 */
2670 	if (atomic_read(this_cpu_ptr(&perf_cgroup_events)))
2671 		perf_cgroup_sched_out(task, next);
2672 }
2673 
2674 static void task_ctx_sched_out(struct perf_event_context *ctx)
2675 {
2676 	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
2677 
2678 	if (!cpuctx->task_ctx)
2679 		return;
2680 
2681 	if (WARN_ON_ONCE(ctx != cpuctx->task_ctx))
2682 		return;
2683 
2684 	ctx_sched_out(ctx, cpuctx, EVENT_ALL);
2685 	cpuctx->task_ctx = NULL;
2686 }
2687 
2688 /*
2689  * Called with IRQs disabled
2690  */
2691 static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,
2692 			      enum event_type_t event_type)
2693 {
2694 	ctx_sched_out(&cpuctx->ctx, cpuctx, event_type);
2695 }
2696 
2697 static void
2698 ctx_pinned_sched_in(struct perf_event_context *ctx,
2699 		    struct perf_cpu_context *cpuctx)
2700 {
2701 	struct perf_event *event;
2702 
2703 	list_for_each_entry(event, &ctx->pinned_groups, group_entry) {
2704 		if (event->state <= PERF_EVENT_STATE_OFF)
2705 			continue;
2706 		if (!event_filter_match(event))
2707 			continue;
2708 
2709 		/* may need to reset tstamp_enabled */
2710 		if (is_cgroup_event(event))
2711 			perf_cgroup_mark_enabled(event, ctx);
2712 
2713 		if (group_can_go_on(event, cpuctx, 1))
2714 			group_sched_in(event, cpuctx, ctx);
2715 
2716 		/*
2717 		 * If this pinned group hasn't been scheduled,
2718 		 * put it in error state.
2719 		 */
2720 		if (event->state == PERF_EVENT_STATE_INACTIVE) {
2721 			update_group_times(event);
2722 			event->state = PERF_EVENT_STATE_ERROR;
2723 		}
2724 	}
2725 }
2726 
2727 static void
2728 ctx_flexible_sched_in(struct perf_event_context *ctx,
2729 		      struct perf_cpu_context *cpuctx)
2730 {
2731 	struct perf_event *event;
2732 	int can_add_hw = 1;
2733 
2734 	list_for_each_entry(event, &ctx->flexible_groups, group_entry) {
2735 		/* Ignore events in OFF or ERROR state */
2736 		if (event->state <= PERF_EVENT_STATE_OFF)
2737 			continue;
2738 		/*
2739 		 * Listen to the 'cpu' scheduling filter constraint
2740 		 * of events:
2741 		 */
2742 		if (!event_filter_match(event))
2743 			continue;
2744 
2745 		/* may need to reset tstamp_enabled */
2746 		if (is_cgroup_event(event))
2747 			perf_cgroup_mark_enabled(event, ctx);
2748 
2749 		if (group_can_go_on(event, cpuctx, can_add_hw)) {
2750 			if (group_sched_in(event, cpuctx, ctx))
2751 				can_add_hw = 0;
2752 		}
2753 	}
2754 }
2755 
2756 static void
2757 ctx_sched_in(struct perf_event_context *ctx,
2758 	     struct perf_cpu_context *cpuctx,
2759 	     enum event_type_t event_type,
2760 	     struct task_struct *task)
2761 {
2762 	u64 now;
2763 	int is_active = ctx->is_active;
2764 
2765 	ctx->is_active |= event_type;
2766 	if (likely(!ctx->nr_events))
2767 		return;
2768 
2769 	now = perf_clock();
2770 	ctx->timestamp = now;
2771 	perf_cgroup_set_timestamp(task, ctx);
2772 	/*
2773 	 * First go through the list and put on any pinned groups
2774 	 * in order to give them the best chance of going on.
2775 	 */
2776 	if (!(is_active & EVENT_PINNED) && (event_type & EVENT_PINNED))
2777 		ctx_pinned_sched_in(ctx, cpuctx);
2778 
2779 	/* Then walk through the lower prio flexible groups */
2780 	if (!(is_active & EVENT_FLEXIBLE) && (event_type & EVENT_FLEXIBLE))
2781 		ctx_flexible_sched_in(ctx, cpuctx);
2782 }
2783 
2784 static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx,
2785 			     enum event_type_t event_type,
2786 			     struct task_struct *task)
2787 {
2788 	struct perf_event_context *ctx = &cpuctx->ctx;
2789 
2790 	ctx_sched_in(ctx, cpuctx, event_type, task);
2791 }
2792 
2793 static void perf_event_context_sched_in(struct perf_event_context *ctx,
2794 					struct task_struct *task)
2795 {
2796 	struct perf_cpu_context *cpuctx;
2797 
2798 	cpuctx = __get_cpu_context(ctx);
2799 	if (cpuctx->task_ctx == ctx)
2800 		return;
2801 
2802 	perf_ctx_lock(cpuctx, ctx);
2803 	perf_pmu_disable(ctx->pmu);
2804 	/*
2805 	 * We want to keep the following priority order:
2806 	 * cpu pinned (that don't need to move), task pinned,
2807 	 * cpu flexible, task flexible.
2808 	 */
2809 	cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);
2810 
2811 	if (ctx->nr_events)
2812 		cpuctx->task_ctx = ctx;
2813 
2814 	perf_event_sched_in(cpuctx, cpuctx->task_ctx, task);
2815 
2816 	perf_pmu_enable(ctx->pmu);
2817 	perf_ctx_unlock(cpuctx, ctx);
2818 }
2819 
2820 /*
2821  * Called from scheduler to add the events of the current task
2822  * with interrupts disabled.
2823  *
2824  * We restore the event value and then enable it.
2825  *
2826  * This does not protect us against NMI, but enable()
2827  * sets the enabled bit in the control field of event _before_
2828  * accessing the event control register. If a NMI hits, then it will
2829  * keep the event running.
2830  */
2831 void __perf_event_task_sched_in(struct task_struct *prev,
2832 				struct task_struct *task)
2833 {
2834 	struct perf_event_context *ctx;
2835 	int ctxn;
2836 
2837 	for_each_task_context_nr(ctxn) {
2838 		ctx = task->perf_event_ctxp[ctxn];
2839 		if (likely(!ctx))
2840 			continue;
2841 
2842 		perf_event_context_sched_in(ctx, task);
2843 	}
2844 	/*
2845 	 * if cgroup events exist on this CPU, then we need
2846 	 * to check if we have to switch in PMU state.
2847 	 * cgroup event are system-wide mode only
2848 	 */
2849 	if (atomic_read(this_cpu_ptr(&perf_cgroup_events)))
2850 		perf_cgroup_sched_in(prev, task);
2851 
2852 	if (__this_cpu_read(perf_sched_cb_usages))
2853 		perf_pmu_sched_task(prev, task, true);
2854 }
2855 
2856 static u64 perf_calculate_period(struct perf_event *event, u64 nsec, u64 count)
2857 {
2858 	u64 frequency = event->attr.sample_freq;
2859 	u64 sec = NSEC_PER_SEC;
2860 	u64 divisor, dividend;
2861 
2862 	int count_fls, nsec_fls, frequency_fls, sec_fls;
2863 
2864 	count_fls = fls64(count);
2865 	nsec_fls = fls64(nsec);
2866 	frequency_fls = fls64(frequency);
2867 	sec_fls = 30;
2868 
2869 	/*
2870 	 * We got @count in @nsec, with a target of sample_freq HZ
2871 	 * the target period becomes:
2872 	 *
2873 	 *             @count * 10^9
2874 	 * period = -------------------
2875 	 *          @nsec * sample_freq
2876 	 *
2877 	 */
2878 
2879 	/*
2880 	 * Reduce accuracy by one bit such that @a and @b converge
2881 	 * to a similar magnitude.
2882 	 */
2883 #define REDUCE_FLS(a, b)		\
2884 do {					\
2885 	if (a##_fls > b##_fls) {	\
2886 		a >>= 1;		\
2887 		a##_fls--;		\
2888 	} else {			\
2889 		b >>= 1;		\
2890 		b##_fls--;		\
2891 	}				\
2892 } while (0)
2893 
2894 	/*
2895 	 * Reduce accuracy until either term fits in a u64, then proceed with
2896 	 * the other, so that finally we can do a u64/u64 division.
2897 	 */
2898 	while (count_fls + sec_fls > 64 && nsec_fls + frequency_fls > 64) {
2899 		REDUCE_FLS(nsec, frequency);
2900 		REDUCE_FLS(sec, count);
2901 	}
2902 
2903 	if (count_fls + sec_fls > 64) {
2904 		divisor = nsec * frequency;
2905 
2906 		while (count_fls + sec_fls > 64) {
2907 			REDUCE_FLS(count, sec);
2908 			divisor >>= 1;
2909 		}
2910 
2911 		dividend = count * sec;
2912 	} else {
2913 		dividend = count * sec;
2914 
2915 		while (nsec_fls + frequency_fls > 64) {
2916 			REDUCE_FLS(nsec, frequency);
2917 			dividend >>= 1;
2918 		}
2919 
2920 		divisor = nsec * frequency;
2921 	}
2922 
2923 	if (!divisor)
2924 		return dividend;
2925 
2926 	return div64_u64(dividend, divisor);
2927 }
2928 
2929 static DEFINE_PER_CPU(int, perf_throttled_count);
2930 static DEFINE_PER_CPU(u64, perf_throttled_seq);
2931 
2932 static void perf_adjust_period(struct perf_event *event, u64 nsec, u64 count, bool disable)
2933 {
2934 	struct hw_perf_event *hwc = &event->hw;
2935 	s64 period, sample_period;
2936 	s64 delta;
2937 
2938 	period = perf_calculate_period(event, nsec, count);
2939 
2940 	delta = (s64)(period - hwc->sample_period);
2941 	delta = (delta + 7) / 8; /* low pass filter */
2942 
2943 	sample_period = hwc->sample_period + delta;
2944 
2945 	if (!sample_period)
2946 		sample_period = 1;
2947 
2948 	hwc->sample_period = sample_period;
2949 
2950 	if (local64_read(&hwc->period_left) > 8*sample_period) {
2951 		if (disable)
2952 			event->pmu->stop(event, PERF_EF_UPDATE);
2953 
2954 		local64_set(&hwc->period_left, 0);
2955 
2956 		if (disable)
2957 			event->pmu->start(event, PERF_EF_RELOAD);
2958 	}
2959 }
2960 
2961 /*
2962  * combine freq adjustment with unthrottling to avoid two passes over the
2963  * events. At the same time, make sure, having freq events does not change
2964  * the rate of unthrottling as that would introduce bias.
2965  */
2966 static void perf_adjust_freq_unthr_context(struct perf_event_context *ctx,
2967 					   int needs_unthr)
2968 {
2969 	struct perf_event *event;
2970 	struct hw_perf_event *hwc;
2971 	u64 now, period = TICK_NSEC;
2972 	s64 delta;
2973 
2974 	/*
2975 	 * only need to iterate over all events iff:
2976 	 * - context have events in frequency mode (needs freq adjust)
2977 	 * - there are events to unthrottle on this cpu
2978 	 */
2979 	if (!(ctx->nr_freq || needs_unthr))
2980 		return;
2981 
2982 	raw_spin_lock(&ctx->lock);
2983 	perf_pmu_disable(ctx->pmu);
2984 
2985 	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
2986 		if (event->state != PERF_EVENT_STATE_ACTIVE)
2987 			continue;
2988 
2989 		if (!event_filter_match(event))
2990 			continue;
2991 
2992 		perf_pmu_disable(event->pmu);
2993 
2994 		hwc = &event->hw;
2995 
2996 		if (hwc->interrupts == MAX_INTERRUPTS) {
2997 			hwc->interrupts = 0;
2998 			perf_log_throttle(event, 1);
2999 			event->pmu->start(event, 0);
3000 		}
3001 
3002 		if (!event->attr.freq || !event->attr.sample_freq)
3003 			goto next;
3004 
3005 		/*
3006 		 * stop the event and update event->count
3007 		 */
3008 		event->pmu->stop(event, PERF_EF_UPDATE);
3009 
3010 		now = local64_read(&event->count);
3011 		delta = now - hwc->freq_count_stamp;
3012 		hwc->freq_count_stamp = now;
3013 
3014 		/*
3015 		 * restart the event
3016 		 * reload only if value has changed
3017 		 * we have stopped the event so tell that
3018 		 * to perf_adjust_period() to avoid stopping it
3019 		 * twice.
3020 		 */
3021 		if (delta > 0)
3022 			perf_adjust_period(event, period, delta, false);
3023 
3024 		event->pmu->start(event, delta > 0 ? PERF_EF_RELOAD : 0);
3025 	next:
3026 		perf_pmu_enable(event->pmu);
3027 	}
3028 
3029 	perf_pmu_enable(ctx->pmu);
3030 	raw_spin_unlock(&ctx->lock);
3031 }
3032 
3033 /*
3034  * Round-robin a context's events:
3035  */
3036 static void rotate_ctx(struct perf_event_context *ctx)
3037 {
3038 	/*
3039 	 * Rotate the first entry last of non-pinned groups. Rotation might be
3040 	 * disabled by the inheritance code.
3041 	 */
3042 	if (!ctx->rotate_disable)
3043 		list_rotate_left(&ctx->flexible_groups);
3044 }
3045 
3046 static int perf_rotate_context(struct perf_cpu_context *cpuctx)
3047 {
3048 	struct perf_event_context *ctx = NULL;
3049 	int rotate = 0;
3050 
3051 	if (cpuctx->ctx.nr_events) {
3052 		if (cpuctx->ctx.nr_events != cpuctx->ctx.nr_active)
3053 			rotate = 1;
3054 	}
3055 
3056 	ctx = cpuctx->task_ctx;
3057 	if (ctx && ctx->nr_events) {
3058 		if (ctx->nr_events != ctx->nr_active)
3059 			rotate = 1;
3060 	}
3061 
3062 	if (!rotate)
3063 		goto done;
3064 
3065 	perf_ctx_lock(cpuctx, cpuctx->task_ctx);
3066 	perf_pmu_disable(cpuctx->ctx.pmu);
3067 
3068 	cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);
3069 	if (ctx)
3070 		ctx_sched_out(ctx, cpuctx, EVENT_FLEXIBLE);
3071 
3072 	rotate_ctx(&cpuctx->ctx);
3073 	if (ctx)
3074 		rotate_ctx(ctx);
3075 
3076 	perf_event_sched_in(cpuctx, ctx, current);
3077 
3078 	perf_pmu_enable(cpuctx->ctx.pmu);
3079 	perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
3080 done:
3081 
3082 	return rotate;
3083 }
3084 
3085 #ifdef CONFIG_NO_HZ_FULL
3086 bool perf_event_can_stop_tick(void)
3087 {
3088 	if (atomic_read(&nr_freq_events) ||
3089 	    __this_cpu_read(perf_throttled_count))
3090 		return false;
3091 	else
3092 		return true;
3093 }
3094 #endif
3095 
3096 void perf_event_task_tick(void)
3097 {
3098 	struct list_head *head = this_cpu_ptr(&active_ctx_list);
3099 	struct perf_event_context *ctx, *tmp;
3100 	int throttled;
3101 
3102 	WARN_ON(!irqs_disabled());
3103 
3104 	__this_cpu_inc(perf_throttled_seq);
3105 	throttled = __this_cpu_xchg(perf_throttled_count, 0);
3106 
3107 	list_for_each_entry_safe(ctx, tmp, head, active_ctx_list)
3108 		perf_adjust_freq_unthr_context(ctx, throttled);
3109 }
3110 
3111 static int event_enable_on_exec(struct perf_event *event,
3112 				struct perf_event_context *ctx)
3113 {
3114 	if (!event->attr.enable_on_exec)
3115 		return 0;
3116 
3117 	event->attr.enable_on_exec = 0;
3118 	if (event->state >= PERF_EVENT_STATE_INACTIVE)
3119 		return 0;
3120 
3121 	__perf_event_mark_enabled(event);
3122 
3123 	return 1;
3124 }
3125 
3126 /*
3127  * Enable all of a task's events that have been marked enable-on-exec.
3128  * This expects task == current.
3129  */
3130 static void perf_event_enable_on_exec(struct perf_event_context *ctx)
3131 {
3132 	struct perf_event_context *clone_ctx = NULL;
3133 	struct perf_event *event;
3134 	unsigned long flags;
3135 	int enabled = 0;
3136 	int ret;
3137 
3138 	local_irq_save(flags);
3139 	if (!ctx || !ctx->nr_events)
3140 		goto out;
3141 
3142 	/*
3143 	 * We must ctxsw out cgroup events to avoid conflict
3144 	 * when invoking perf_task_event_sched_in() later on
3145 	 * in this function. Otherwise we end up trying to
3146 	 * ctxswin cgroup events which are already scheduled
3147 	 * in.
3148 	 */
3149 	perf_cgroup_sched_out(current, NULL);
3150 
3151 	raw_spin_lock(&ctx->lock);
3152 	task_ctx_sched_out(ctx);
3153 
3154 	list_for_each_entry(event, &ctx->event_list, event_entry) {
3155 		ret = event_enable_on_exec(event, ctx);
3156 		if (ret)
3157 			enabled = 1;
3158 	}
3159 
3160 	/*
3161 	 * Unclone this context if we enabled any event.
3162 	 */
3163 	if (enabled)
3164 		clone_ctx = unclone_ctx(ctx);
3165 
3166 	raw_spin_unlock(&ctx->lock);
3167 
3168 	/*
3169 	 * Also calls ctxswin for cgroup events, if any:
3170 	 */
3171 	perf_event_context_sched_in(ctx, ctx->task);
3172 out:
3173 	local_irq_restore(flags);
3174 
3175 	if (clone_ctx)
3176 		put_ctx(clone_ctx);
3177 }
3178 
3179 void perf_event_exec(void)
3180 {
3181 	struct perf_event_context *ctx;
3182 	int ctxn;
3183 
3184 	rcu_read_lock();
3185 	for_each_task_context_nr(ctxn) {
3186 		ctx = current->perf_event_ctxp[ctxn];
3187 		if (!ctx)
3188 			continue;
3189 
3190 		perf_event_enable_on_exec(ctx);
3191 	}
3192 	rcu_read_unlock();
3193 }
3194 
3195 /*
3196  * Cross CPU call to read the hardware event
3197  */
3198 static void __perf_event_read(void *info)
3199 {
3200 	struct perf_event *event = info;
3201 	struct perf_event_context *ctx = event->ctx;
3202 	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
3203 
3204 	/*
3205 	 * If this is a task context, we need to check whether it is
3206 	 * the current task context of this cpu.  If not it has been
3207 	 * scheduled out before the smp call arrived.  In that case
3208 	 * event->count would have been updated to a recent sample
3209 	 * when the event was scheduled out.
3210 	 */
3211 	if (ctx->task && cpuctx->task_ctx != ctx)
3212 		return;
3213 
3214 	raw_spin_lock(&ctx->lock);
3215 	if (ctx->is_active) {
3216 		update_context_time(ctx);
3217 		update_cgrp_time_from_event(event);
3218 	}
3219 	update_event_times(event);
3220 	if (event->state == PERF_EVENT_STATE_ACTIVE)
3221 		event->pmu->read(event);
3222 	raw_spin_unlock(&ctx->lock);
3223 }
3224 
3225 static inline u64 perf_event_count(struct perf_event *event)
3226 {
3227 	if (event->pmu->count)
3228 		return event->pmu->count(event);
3229 
3230 	return __perf_event_count(event);
3231 }
3232 
3233 static u64 perf_event_read(struct perf_event *event)
3234 {
3235 	/*
3236 	 * If event is enabled and currently active on a CPU, update the
3237 	 * value in the event structure:
3238 	 */
3239 	if (event->state == PERF_EVENT_STATE_ACTIVE) {
3240 		smp_call_function_single(event->oncpu,
3241 					 __perf_event_read, event, 1);
3242 	} else if (event->state == PERF_EVENT_STATE_INACTIVE) {
3243 		struct perf_event_context *ctx = event->ctx;
3244 		unsigned long flags;
3245 
3246 		raw_spin_lock_irqsave(&ctx->lock, flags);
3247 		/*
3248 		 * may read while context is not active
3249 		 * (e.g., thread is blocked), in that case
3250 		 * we cannot update context time
3251 		 */
3252 		if (ctx->is_active) {
3253 			update_context_time(ctx);
3254 			update_cgrp_time_from_event(event);
3255 		}
3256 		update_event_times(event);
3257 		raw_spin_unlock_irqrestore(&ctx->lock, flags);
3258 	}
3259 
3260 	return perf_event_count(event);
3261 }
3262 
3263 /*
3264  * Initialize the perf_event context in a task_struct:
3265  */
3266 static void __perf_event_init_context(struct perf_event_context *ctx)
3267 {
3268 	raw_spin_lock_init(&ctx->lock);
3269 	mutex_init(&ctx->mutex);
3270 	INIT_LIST_HEAD(&ctx->active_ctx_list);
3271 	INIT_LIST_HEAD(&ctx->pinned_groups);
3272 	INIT_LIST_HEAD(&ctx->flexible_groups);
3273 	INIT_LIST_HEAD(&ctx->event_list);
3274 	atomic_set(&ctx->refcount, 1);
3275 	INIT_DELAYED_WORK(&ctx->orphans_remove, orphans_remove_work);
3276 }
3277 
3278 static struct perf_event_context *
3279 alloc_perf_context(struct pmu *pmu, struct task_struct *task)
3280 {
3281 	struct perf_event_context *ctx;
3282 
3283 	ctx = kzalloc(sizeof(struct perf_event_context), GFP_KERNEL);
3284 	if (!ctx)
3285 		return NULL;
3286 
3287 	__perf_event_init_context(ctx);
3288 	if (task) {
3289 		ctx->task = task;
3290 		get_task_struct(task);
3291 	}
3292 	ctx->pmu = pmu;
3293 
3294 	return ctx;
3295 }
3296 
3297 static struct task_struct *
3298 find_lively_task_by_vpid(pid_t vpid)
3299 {
3300 	struct task_struct *task;
3301 	int err;
3302 
3303 	rcu_read_lock();
3304 	if (!vpid)
3305 		task = current;
3306 	else
3307 		task = find_task_by_vpid(vpid);
3308 	if (task)
3309 		get_task_struct(task);
3310 	rcu_read_unlock();
3311 
3312 	if (!task)
3313 		return ERR_PTR(-ESRCH);
3314 
3315 	/* Reuse ptrace permission checks for now. */
3316 	err = -EACCES;
3317 	if (!ptrace_may_access(task, PTRACE_MODE_READ))
3318 		goto errout;
3319 
3320 	return task;
3321 errout:
3322 	put_task_struct(task);
3323 	return ERR_PTR(err);
3324 
3325 }
3326 
3327 /*
3328  * Returns a matching context with refcount and pincount.
3329  */
3330 static struct perf_event_context *
3331 find_get_context(struct pmu *pmu, struct task_struct *task,
3332 		struct perf_event *event)
3333 {
3334 	struct perf_event_context *ctx, *clone_ctx = NULL;
3335 	struct perf_cpu_context *cpuctx;
3336 	void *task_ctx_data = NULL;
3337 	unsigned long flags;
3338 	int ctxn, err;
3339 	int cpu = event->cpu;
3340 
3341 	if (!task) {
3342 		/* Must be root to operate on a CPU event: */
3343 		if (perf_paranoid_cpu() && !capable(CAP_SYS_ADMIN))
3344 			return ERR_PTR(-EACCES);
3345 
3346 		/*
3347 		 * We could be clever and allow to attach a event to an
3348 		 * offline CPU and activate it when the CPU comes up, but
3349 		 * that's for later.
3350 		 */
3351 		if (!cpu_online(cpu))
3352 			return ERR_PTR(-ENODEV);
3353 
3354 		cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
3355 		ctx = &cpuctx->ctx;
3356 		get_ctx(ctx);
3357 		++ctx->pin_count;
3358 
3359 		return ctx;
3360 	}
3361 
3362 	err = -EINVAL;
3363 	ctxn = pmu->task_ctx_nr;
3364 	if (ctxn < 0)
3365 		goto errout;
3366 
3367 	if (event->attach_state & PERF_ATTACH_TASK_DATA) {
3368 		task_ctx_data = kzalloc(pmu->task_ctx_size, GFP_KERNEL);
3369 		if (!task_ctx_data) {
3370 			err = -ENOMEM;
3371 			goto errout;
3372 		}
3373 	}
3374 
3375 retry:
3376 	ctx = perf_lock_task_context(task, ctxn, &flags);
3377 	if (ctx) {
3378 		clone_ctx = unclone_ctx(ctx);
3379 		++ctx->pin_count;
3380 
3381 		if (task_ctx_data && !ctx->task_ctx_data) {
3382 			ctx->task_ctx_data = task_ctx_data;
3383 			task_ctx_data = NULL;
3384 		}
3385 		raw_spin_unlock_irqrestore(&ctx->lock, flags);
3386 
3387 		if (clone_ctx)
3388 			put_ctx(clone_ctx);
3389 	} else {
3390 		ctx = alloc_perf_context(pmu, task);
3391 		err = -ENOMEM;
3392 		if (!ctx)
3393 			goto errout;
3394 
3395 		if (task_ctx_data) {
3396 			ctx->task_ctx_data = task_ctx_data;
3397 			task_ctx_data = NULL;
3398 		}
3399 
3400 		err = 0;
3401 		mutex_lock(&task->perf_event_mutex);
3402 		/*
3403 		 * If it has already passed perf_event_exit_task().
3404 		 * we must see PF_EXITING, it takes this mutex too.
3405 		 */
3406 		if (task->flags & PF_EXITING)
3407 			err = -ESRCH;
3408 		else if (task->perf_event_ctxp[ctxn])
3409 			err = -EAGAIN;
3410 		else {
3411 			get_ctx(ctx);
3412 			++ctx->pin_count;
3413 			rcu_assign_pointer(task->perf_event_ctxp[ctxn], ctx);
3414 		}
3415 		mutex_unlock(&task->perf_event_mutex);
3416 
3417 		if (unlikely(err)) {
3418 			put_ctx(ctx);
3419 
3420 			if (err == -EAGAIN)
3421 				goto retry;
3422 			goto errout;
3423 		}
3424 	}
3425 
3426 	kfree(task_ctx_data);
3427 	return ctx;
3428 
3429 errout:
3430 	kfree(task_ctx_data);
3431 	return ERR_PTR(err);
3432 }
3433 
3434 static void perf_event_free_filter(struct perf_event *event);
3435 static void perf_event_free_bpf_prog(struct perf_event *event);
3436 
3437 static void free_event_rcu(struct rcu_head *head)
3438 {
3439 	struct perf_event *event;
3440 
3441 	event = container_of(head, struct perf_event, rcu_head);
3442 	if (event->ns)
3443 		put_pid_ns(event->ns);
3444 	perf_event_free_filter(event);
3445 	perf_event_free_bpf_prog(event);
3446 	kfree(event);
3447 }
3448 
3449 static void ring_buffer_attach(struct perf_event *event,
3450 			       struct ring_buffer *rb);
3451 
3452 static void unaccount_event_cpu(struct perf_event *event, int cpu)
3453 {
3454 	if (event->parent)
3455 		return;
3456 
3457 	if (is_cgroup_event(event))
3458 		atomic_dec(&per_cpu(perf_cgroup_events, cpu));
3459 }
3460 
3461 static void unaccount_event(struct perf_event *event)
3462 {
3463 	if (event->parent)
3464 		return;
3465 
3466 	if (event->attach_state & PERF_ATTACH_TASK)
3467 		static_key_slow_dec_deferred(&perf_sched_events);
3468 	if (event->attr.mmap || event->attr.mmap_data)
3469 		atomic_dec(&nr_mmap_events);
3470 	if (event->attr.comm)
3471 		atomic_dec(&nr_comm_events);
3472 	if (event->attr.task)
3473 		atomic_dec(&nr_task_events);
3474 	if (event->attr.freq)
3475 		atomic_dec(&nr_freq_events);
3476 	if (is_cgroup_event(event))
3477 		static_key_slow_dec_deferred(&perf_sched_events);
3478 	if (has_branch_stack(event))
3479 		static_key_slow_dec_deferred(&perf_sched_events);
3480 
3481 	unaccount_event_cpu(event, event->cpu);
3482 }
3483 
3484 /*
3485  * The following implement mutual exclusion of events on "exclusive" pmus
3486  * (PERF_PMU_CAP_EXCLUSIVE). Such pmus can only have one event scheduled
3487  * at a time, so we disallow creating events that might conflict, namely:
3488  *
3489  *  1) cpu-wide events in the presence of per-task events,
3490  *  2) per-task events in the presence of cpu-wide events,
3491  *  3) two matching events on the same context.
3492  *
3493  * The former two cases are handled in the allocation path (perf_event_alloc(),
3494  * __free_event()), the latter -- before the first perf_install_in_context().
3495  */
3496 static int exclusive_event_init(struct perf_event *event)
3497 {
3498 	struct pmu *pmu = event->pmu;
3499 
3500 	if (!(pmu->capabilities & PERF_PMU_CAP_EXCLUSIVE))
3501 		return 0;
3502 
3503 	/*
3504 	 * Prevent co-existence of per-task and cpu-wide events on the
3505 	 * same exclusive pmu.
3506 	 *
3507 	 * Negative pmu::exclusive_cnt means there are cpu-wide
3508 	 * events on this "exclusive" pmu, positive means there are
3509 	 * per-task events.
3510 	 *
3511 	 * Since this is called in perf_event_alloc() path, event::ctx
3512 	 * doesn't exist yet; it is, however, safe to use PERF_ATTACH_TASK
3513 	 * to mean "per-task event", because unlike other attach states it
3514 	 * never gets cleared.
3515 	 */
3516 	if (event->attach_state & PERF_ATTACH_TASK) {
3517 		if (!atomic_inc_unless_negative(&pmu->exclusive_cnt))
3518 			return -EBUSY;
3519 	} else {
3520 		if (!atomic_dec_unless_positive(&pmu->exclusive_cnt))
3521 			return -EBUSY;
3522 	}
3523 
3524 	return 0;
3525 }
3526 
3527 static void exclusive_event_destroy(struct perf_event *event)
3528 {
3529 	struct pmu *pmu = event->pmu;
3530 
3531 	if (!(pmu->capabilities & PERF_PMU_CAP_EXCLUSIVE))
3532 		return;
3533 
3534 	/* see comment in exclusive_event_init() */
3535 	if (event->attach_state & PERF_ATTACH_TASK)
3536 		atomic_dec(&pmu->exclusive_cnt);
3537 	else
3538 		atomic_inc(&pmu->exclusive_cnt);
3539 }
3540 
3541 static bool exclusive_event_match(struct perf_event *e1, struct perf_event *e2)
3542 {
3543 	if ((e1->pmu->capabilities & PERF_PMU_CAP_EXCLUSIVE) &&
3544 	    (e1->cpu == e2->cpu ||
3545 	     e1->cpu == -1 ||
3546 	     e2->cpu == -1))
3547 		return true;
3548 	return false;
3549 }
3550 
3551 /* Called under the same ctx::mutex as perf_install_in_context() */
3552 static bool exclusive_event_installable(struct perf_event *event,
3553 					struct perf_event_context *ctx)
3554 {
3555 	struct perf_event *iter_event;
3556 	struct pmu *pmu = event->pmu;
3557 
3558 	if (!(pmu->capabilities & PERF_PMU_CAP_EXCLUSIVE))
3559 		return true;
3560 
3561 	list_for_each_entry(iter_event, &ctx->event_list, event_entry) {
3562 		if (exclusive_event_match(iter_event, event))
3563 			return false;
3564 	}
3565 
3566 	return true;
3567 }
3568 
3569 static void __free_event(struct perf_event *event)
3570 {
3571 	if (!event->parent) {
3572 		if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN)
3573 			put_callchain_buffers();
3574 	}
3575 
3576 	if (event->destroy)
3577 		event->destroy(event);
3578 
3579 	if (event->ctx)
3580 		put_ctx(event->ctx);
3581 
3582 	if (event->pmu) {
3583 		exclusive_event_destroy(event);
3584 		module_put(event->pmu->module);
3585 	}
3586 
3587 	call_rcu(&event->rcu_head, free_event_rcu);
3588 }
3589 
3590 static void _free_event(struct perf_event *event)
3591 {
3592 	irq_work_sync(&event->pending);
3593 
3594 	unaccount_event(event);
3595 
3596 	if (event->rb) {
3597 		/*
3598 		 * Can happen when we close an event with re-directed output.
3599 		 *
3600 		 * Since we have a 0 refcount, perf_mmap_close() will skip
3601 		 * over us; possibly making our ring_buffer_put() the last.
3602 		 */
3603 		mutex_lock(&event->mmap_mutex);
3604 		ring_buffer_attach(event, NULL);
3605 		mutex_unlock(&event->mmap_mutex);
3606 	}
3607 
3608 	if (is_cgroup_event(event))
3609 		perf_detach_cgroup(event);
3610 
3611 	__free_event(event);
3612 }
3613 
3614 /*
3615  * Used to free events which have a known refcount of 1, such as in error paths
3616  * where the event isn't exposed yet and inherited events.
3617  */
3618 static void free_event(struct perf_event *event)
3619 {
3620 	if (WARN(atomic_long_cmpxchg(&event->refcount, 1, 0) != 1,
3621 				"unexpected event refcount: %ld; ptr=%p\n",
3622 				atomic_long_read(&event->refcount), event)) {
3623 		/* leak to avoid use-after-free */
3624 		return;
3625 	}
3626 
3627 	_free_event(event);
3628 }
3629 
3630 /*
3631  * Remove user event from the owner task.
3632  */
3633 static void perf_remove_from_owner(struct perf_event *event)
3634 {
3635 	struct task_struct *owner;
3636 
3637 	rcu_read_lock();
3638 	owner = ACCESS_ONCE(event->owner);
3639 	/*
3640 	 * Matches the smp_wmb() in perf_event_exit_task(). If we observe
3641 	 * !owner it means the list deletion is complete and we can indeed
3642 	 * free this event, otherwise we need to serialize on
3643 	 * owner->perf_event_mutex.
3644 	 */
3645 	smp_read_barrier_depends();
3646 	if (owner) {
3647 		/*
3648 		 * Since delayed_put_task_struct() also drops the last
3649 		 * task reference we can safely take a new reference
3650 		 * while holding the rcu_read_lock().
3651 		 */
3652 		get_task_struct(owner);
3653 	}
3654 	rcu_read_unlock();
3655 
3656 	if (owner) {
3657 		/*
3658 		 * If we're here through perf_event_exit_task() we're already
3659 		 * holding ctx->mutex which would be an inversion wrt. the
3660 		 * normal lock order.
3661 		 *
3662 		 * However we can safely take this lock because its the child
3663 		 * ctx->mutex.
3664 		 */
3665 		mutex_lock_nested(&owner->perf_event_mutex, SINGLE_DEPTH_NESTING);
3666 
3667 		/*
3668 		 * We have to re-check the event->owner field, if it is cleared
3669 		 * we raced with perf_event_exit_task(), acquiring the mutex
3670 		 * ensured they're done, and we can proceed with freeing the
3671 		 * event.
3672 		 */
3673 		if (event->owner)
3674 			list_del_init(&event->owner_entry);
3675 		mutex_unlock(&owner->perf_event_mutex);
3676 		put_task_struct(owner);
3677 	}
3678 }
3679 
3680 static void put_event(struct perf_event *event)
3681 {
3682 	struct perf_event_context *ctx;
3683 
3684 	if (!atomic_long_dec_and_test(&event->refcount))
3685 		return;
3686 
3687 	if (!is_kernel_event(event))
3688 		perf_remove_from_owner(event);
3689 
3690 	/*
3691 	 * There are two ways this annotation is useful:
3692 	 *
3693 	 *  1) there is a lock recursion from perf_event_exit_task
3694 	 *     see the comment there.
3695 	 *
3696 	 *  2) there is a lock-inversion with mmap_sem through
3697 	 *     perf_event_read_group(), which takes faults while
3698 	 *     holding ctx->mutex, however this is called after
3699 	 *     the last filedesc died, so there is no possibility
3700 	 *     to trigger the AB-BA case.
3701 	 */
3702 	ctx = perf_event_ctx_lock_nested(event, SINGLE_DEPTH_NESTING);
3703 	WARN_ON_ONCE(ctx->parent_ctx);
3704 	perf_remove_from_context(event, true);
3705 	perf_event_ctx_unlock(event, ctx);
3706 
3707 	_free_event(event);
3708 }
3709 
3710 int perf_event_release_kernel(struct perf_event *event)
3711 {
3712 	put_event(event);
3713 	return 0;
3714 }
3715 EXPORT_SYMBOL_GPL(perf_event_release_kernel);
3716 
3717 /*
3718  * Called when the last reference to the file is gone.
3719  */
3720 static int perf_release(struct inode *inode, struct file *file)
3721 {
3722 	put_event(file->private_data);
3723 	return 0;
3724 }
3725 
3726 /*
3727  * Remove all orphanes events from the context.
3728  */
3729 static void orphans_remove_work(struct work_struct *work)
3730 {
3731 	struct perf_event_context *ctx;
3732 	struct perf_event *event, *tmp;
3733 
3734 	ctx = container_of(work, struct perf_event_context,
3735 			   orphans_remove.work);
3736 
3737 	mutex_lock(&ctx->mutex);
3738 	list_for_each_entry_safe(event, tmp, &ctx->event_list, event_entry) {
3739 		struct perf_event *parent_event = event->parent;
3740 
3741 		if (!is_orphaned_child(event))
3742 			continue;
3743 
3744 		perf_remove_from_context(event, true);
3745 
3746 		mutex_lock(&parent_event->child_mutex);
3747 		list_del_init(&event->child_list);
3748 		mutex_unlock(&parent_event->child_mutex);
3749 
3750 		free_event(event);
3751 		put_event(parent_event);
3752 	}
3753 
3754 	raw_spin_lock_irq(&ctx->lock);
3755 	ctx->orphans_remove_sched = false;
3756 	raw_spin_unlock_irq(&ctx->lock);
3757 	mutex_unlock(&ctx->mutex);
3758 
3759 	put_ctx(ctx);
3760 }
3761 
3762 u64 perf_event_read_value(struct perf_event *event, u64 *enabled, u64 *running)
3763 {
3764 	struct perf_event *child;
3765 	u64 total = 0;
3766 
3767 	*enabled = 0;
3768 	*running = 0;
3769 
3770 	mutex_lock(&event->child_mutex);
3771 	total += perf_event_read(event);
3772 	*enabled += event->total_time_enabled +
3773 			atomic64_read(&event->child_total_time_enabled);
3774 	*running += event->total_time_running +
3775 			atomic64_read(&event->child_total_time_running);
3776 
3777 	list_for_each_entry(child, &event->child_list, child_list) {
3778 		total += perf_event_read(child);
3779 		*enabled += child->total_time_enabled;
3780 		*running += child->total_time_running;
3781 	}
3782 	mutex_unlock(&event->child_mutex);
3783 
3784 	return total;
3785 }
3786 EXPORT_SYMBOL_GPL(perf_event_read_value);
3787 
3788 static int perf_event_read_group(struct perf_event *event,
3789 				   u64 read_format, char __user *buf)
3790 {
3791 	struct perf_event *leader = event->group_leader, *sub;
3792 	struct perf_event_context *ctx = leader->ctx;
3793 	int n = 0, size = 0, ret;
3794 	u64 count, enabled, running;
3795 	u64 values[5];
3796 
3797 	lockdep_assert_held(&ctx->mutex);
3798 
3799 	count = perf_event_read_value(leader, &enabled, &running);
3800 
3801 	values[n++] = 1 + leader->nr_siblings;
3802 	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
3803 		values[n++] = enabled;
3804 	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
3805 		values[n++] = running;
3806 	values[n++] = count;
3807 	if (read_format & PERF_FORMAT_ID)
3808 		values[n++] = primary_event_id(leader);
3809 
3810 	size = n * sizeof(u64);
3811 
3812 	if (copy_to_user(buf, values, size))
3813 		return -EFAULT;
3814 
3815 	ret = size;
3816 
3817 	list_for_each_entry(sub, &leader->sibling_list, group_entry) {
3818 		n = 0;
3819 
3820 		values[n++] = perf_event_read_value(sub, &enabled, &running);
3821 		if (read_format & PERF_FORMAT_ID)
3822 			values[n++] = primary_event_id(sub);
3823 
3824 		size = n * sizeof(u64);
3825 
3826 		if (copy_to_user(buf + ret, values, size)) {
3827 			return -EFAULT;
3828 		}
3829 
3830 		ret += size;
3831 	}
3832 
3833 	return ret;
3834 }
3835 
3836 static int perf_event_read_one(struct perf_event *event,
3837 				 u64 read_format, char __user *buf)
3838 {
3839 	u64 enabled, running;
3840 	u64 values[4];
3841 	int n = 0;
3842 
3843 	values[n++] = perf_event_read_value(event, &enabled, &running);
3844 	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
3845 		values[n++] = enabled;
3846 	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
3847 		values[n++] = running;
3848 	if (read_format & PERF_FORMAT_ID)
3849 		values[n++] = primary_event_id(event);
3850 
3851 	if (copy_to_user(buf, values, n * sizeof(u64)))
3852 		return -EFAULT;
3853 
3854 	return n * sizeof(u64);
3855 }
3856 
3857 static bool is_event_hup(struct perf_event *event)
3858 {
3859 	bool no_children;
3860 
3861 	if (event->state != PERF_EVENT_STATE_EXIT)
3862 		return false;
3863 
3864 	mutex_lock(&event->child_mutex);
3865 	no_children = list_empty(&event->child_list);
3866 	mutex_unlock(&event->child_mutex);
3867 	return no_children;
3868 }
3869 
3870 /*
3871  * Read the performance event - simple non blocking version for now
3872  */
3873 static ssize_t
3874 perf_read_hw(struct perf_event *event, char __user *buf, size_t count)
3875 {
3876 	u64 read_format = event->attr.read_format;
3877 	int ret;
3878 
3879 	/*
3880 	 * Return end-of-file for a read on a event that is in
3881 	 * error state (i.e. because it was pinned but it couldn't be
3882 	 * scheduled on to the CPU at some point).
3883 	 */
3884 	if (event->state == PERF_EVENT_STATE_ERROR)
3885 		return 0;
3886 
3887 	if (count < event->read_size)
3888 		return -ENOSPC;
3889 
3890 	WARN_ON_ONCE(event->ctx->parent_ctx);
3891 	if (read_format & PERF_FORMAT_GROUP)
3892 		ret = perf_event_read_group(event, read_format, buf);
3893 	else
3894 		ret = perf_event_read_one(event, read_format, buf);
3895 
3896 	return ret;
3897 }
3898 
3899 static ssize_t
3900 perf_read(struct file *file, char __user *buf, size_t count, loff_t *ppos)
3901 {
3902 	struct perf_event *event = file->private_data;
3903 	struct perf_event_context *ctx;
3904 	int ret;
3905 
3906 	ctx = perf_event_ctx_lock(event);
3907 	ret = perf_read_hw(event, buf, count);
3908 	perf_event_ctx_unlock(event, ctx);
3909 
3910 	return ret;
3911 }
3912 
3913 static unsigned int perf_poll(struct file *file, poll_table *wait)
3914 {
3915 	struct perf_event *event = file->private_data;
3916 	struct ring_buffer *rb;
3917 	unsigned int events = POLLHUP;
3918 
3919 	poll_wait(file, &event->waitq, wait);
3920 
3921 	if (is_event_hup(event))
3922 		return events;
3923 
3924 	/*
3925 	 * Pin the event->rb by taking event->mmap_mutex; otherwise
3926 	 * perf_event_set_output() can swizzle our rb and make us miss wakeups.
3927 	 */
3928 	mutex_lock(&event->mmap_mutex);
3929 	rb = event->rb;
3930 	if (rb)
3931 		events = atomic_xchg(&rb->poll, 0);
3932 	mutex_unlock(&event->mmap_mutex);
3933 	return events;
3934 }
3935 
3936 static void _perf_event_reset(struct perf_event *event)
3937 {
3938 	(void)perf_event_read(event);
3939 	local64_set(&event->count, 0);
3940 	perf_event_update_userpage(event);
3941 }
3942 
3943 /*
3944  * Holding the top-level event's child_mutex means that any
3945  * descendant process that has inherited this event will block
3946  * in sync_child_event if it goes to exit, thus satisfying the
3947  * task existence requirements of perf_event_enable/disable.
3948  */
3949 static void perf_event_for_each_child(struct perf_event *event,
3950 					void (*func)(struct perf_event *))
3951 {
3952 	struct perf_event *child;
3953 
3954 	WARN_ON_ONCE(event->ctx->parent_ctx);
3955 
3956 	mutex_lock(&event->child_mutex);
3957 	func(event);
3958 	list_for_each_entry(child, &event->child_list, child_list)
3959 		func(child);
3960 	mutex_unlock(&event->child_mutex);
3961 }
3962 
3963 static void perf_event_for_each(struct perf_event *event,
3964 				  void (*func)(struct perf_event *))
3965 {
3966 	struct perf_event_context *ctx = event->ctx;
3967 	struct perf_event *sibling;
3968 
3969 	lockdep_assert_held(&ctx->mutex);
3970 
3971 	event = event->group_leader;
3972 
3973 	perf_event_for_each_child(event, func);
3974 	list_for_each_entry(sibling, &event->sibling_list, group_entry)
3975 		perf_event_for_each_child(sibling, func);
3976 }
3977 
3978 static int perf_event_period(struct perf_event *event, u64 __user *arg)
3979 {
3980 	struct perf_event_context *ctx = event->ctx;
3981 	int ret = 0, active;
3982 	u64 value;
3983 
3984 	if (!is_sampling_event(event))
3985 		return -EINVAL;
3986 
3987 	if (copy_from_user(&value, arg, sizeof(value)))
3988 		return -EFAULT;
3989 
3990 	if (!value)
3991 		return -EINVAL;
3992 
3993 	raw_spin_lock_irq(&ctx->lock);
3994 	if (event->attr.freq) {
3995 		if (value > sysctl_perf_event_sample_rate) {
3996 			ret = -EINVAL;
3997 			goto unlock;
3998 		}
3999 
4000 		event->attr.sample_freq = value;
4001 	} else {
4002 		event->attr.sample_period = value;
4003 		event->hw.sample_period = value;
4004 	}
4005 
4006 	active = (event->state == PERF_EVENT_STATE_ACTIVE);
4007 	if (active) {
4008 		perf_pmu_disable(ctx->pmu);
4009 		event->pmu->stop(event, PERF_EF_UPDATE);
4010 	}
4011 
4012 	local64_set(&event->hw.period_left, 0);
4013 
4014 	if (active) {
4015 		event->pmu->start(event, PERF_EF_RELOAD);
4016 		perf_pmu_enable(ctx->pmu);
4017 	}
4018 
4019 unlock:
4020 	raw_spin_unlock_irq(&ctx->lock);
4021 
4022 	return ret;
4023 }
4024 
4025 static const struct file_operations perf_fops;
4026 
4027 static inline int perf_fget_light(int fd, struct fd *p)
4028 {
4029 	struct fd f = fdget(fd);
4030 	if (!f.file)
4031 		return -EBADF;
4032 
4033 	if (f.file->f_op != &perf_fops) {
4034 		fdput(f);
4035 		return -EBADF;
4036 	}
4037 	*p = f;
4038 	return 0;
4039 }
4040 
4041 static int perf_event_set_output(struct perf_event *event,
4042 				 struct perf_event *output_event);
4043 static int perf_event_set_filter(struct perf_event *event, void __user *arg);
4044 static int perf_event_set_bpf_prog(struct perf_event *event, u32 prog_fd);
4045 
4046 static long _perf_ioctl(struct perf_event *event, unsigned int cmd, unsigned long arg)
4047 {
4048 	void (*func)(struct perf_event *);
4049 	u32 flags = arg;
4050 
4051 	switch (cmd) {
4052 	case PERF_EVENT_IOC_ENABLE:
4053 		func = _perf_event_enable;
4054 		break;
4055 	case PERF_EVENT_IOC_DISABLE:
4056 		func = _perf_event_disable;
4057 		break;
4058 	case PERF_EVENT_IOC_RESET:
4059 		func = _perf_event_reset;
4060 		break;
4061 
4062 	case PERF_EVENT_IOC_REFRESH:
4063 		return _perf_event_refresh(event, arg);
4064 
4065 	case PERF_EVENT_IOC_PERIOD:
4066 		return perf_event_period(event, (u64 __user *)arg);
4067 
4068 	case PERF_EVENT_IOC_ID:
4069 	{
4070 		u64 id = primary_event_id(event);
4071 
4072 		if (copy_to_user((void __user *)arg, &id, sizeof(id)))
4073 			return -EFAULT;
4074 		return 0;
4075 	}
4076 
4077 	case PERF_EVENT_IOC_SET_OUTPUT:
4078 	{
4079 		int ret;
4080 		if (arg != -1) {
4081 			struct perf_event *output_event;
4082 			struct fd output;
4083 			ret = perf_fget_light(arg, &output);
4084 			if (ret)
4085 				return ret;
4086 			output_event = output.file->private_data;
4087 			ret = perf_event_set_output(event, output_event);
4088 			fdput(output);
4089 		} else {
4090 			ret = perf_event_set_output(event, NULL);
4091 		}
4092 		return ret;
4093 	}
4094 
4095 	case PERF_EVENT_IOC_SET_FILTER:
4096 		return perf_event_set_filter(event, (void __user *)arg);
4097 
4098 	case PERF_EVENT_IOC_SET_BPF:
4099 		return perf_event_set_bpf_prog(event, arg);
4100 
4101 	default:
4102 		return -ENOTTY;
4103 	}
4104 
4105 	if (flags & PERF_IOC_FLAG_GROUP)
4106 		perf_event_for_each(event, func);
4107 	else
4108 		perf_event_for_each_child(event, func);
4109 
4110 	return 0;
4111 }
4112 
4113 static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
4114 {
4115 	struct perf_event *event = file->private_data;
4116 	struct perf_event_context *ctx;
4117 	long ret;
4118 
4119 	ctx = perf_event_ctx_lock(event);
4120 	ret = _perf_ioctl(event, cmd, arg);
4121 	perf_event_ctx_unlock(event, ctx);
4122 
4123 	return ret;
4124 }
4125 
4126 #ifdef CONFIG_COMPAT
4127 static long perf_compat_ioctl(struct file *file, unsigned int cmd,
4128 				unsigned long arg)
4129 {
4130 	switch (_IOC_NR(cmd)) {
4131 	case _IOC_NR(PERF_EVENT_IOC_SET_FILTER):
4132 	case _IOC_NR(PERF_EVENT_IOC_ID):
4133 		/* Fix up pointer size (usually 4 -> 8 in 32-on-64-bit case */
4134 		if (_IOC_SIZE(cmd) == sizeof(compat_uptr_t)) {
4135 			cmd &= ~IOCSIZE_MASK;
4136 			cmd |= sizeof(void *) << IOCSIZE_SHIFT;
4137 		}
4138 		break;
4139 	}
4140 	return perf_ioctl(file, cmd, arg);
4141 }
4142 #else
4143 # define perf_compat_ioctl NULL
4144 #endif
4145 
4146 int perf_event_task_enable(void)
4147 {
4148 	struct perf_event_context *ctx;
4149 	struct perf_event *event;
4150 
4151 	mutex_lock(&current->perf_event_mutex);
4152 	list_for_each_entry(event, &current->perf_event_list, owner_entry) {
4153 		ctx = perf_event_ctx_lock(event);
4154 		perf_event_for_each_child(event, _perf_event_enable);
4155 		perf_event_ctx_unlock(event, ctx);
4156 	}
4157 	mutex_unlock(&current->perf_event_mutex);
4158 
4159 	return 0;
4160 }
4161 
4162 int perf_event_task_disable(void)
4163 {
4164 	struct perf_event_context *ctx;
4165 	struct perf_event *event;
4166 
4167 	mutex_lock(&current->perf_event_mutex);
4168 	list_for_each_entry(event, &current->perf_event_list, owner_entry) {
4169 		ctx = perf_event_ctx_lock(event);
4170 		perf_event_for_each_child(event, _perf_event_disable);
4171 		perf_event_ctx_unlock(event, ctx);
4172 	}
4173 	mutex_unlock(&current->perf_event_mutex);
4174 
4175 	return 0;
4176 }
4177 
4178 static int perf_event_index(struct perf_event *event)
4179 {
4180 	if (event->hw.state & PERF_HES_STOPPED)
4181 		return 0;
4182 
4183 	if (event->state != PERF_EVENT_STATE_ACTIVE)
4184 		return 0;
4185 
4186 	return event->pmu->event_idx(event);
4187 }
4188 
4189 static void calc_timer_values(struct perf_event *event,
4190 				u64 *now,
4191 				u64 *enabled,
4192 				u64 *running)
4193 {
4194 	u64 ctx_time;
4195 
4196 	*now = perf_clock();
4197 	ctx_time = event->shadow_ctx_time + *now;
4198 	*enabled = ctx_time - event->tstamp_enabled;
4199 	*running = ctx_time - event->tstamp_running;
4200 }
4201 
4202 static void perf_event_init_userpage(struct perf_event *event)
4203 {
4204 	struct perf_event_mmap_page *userpg;
4205 	struct ring_buffer *rb;
4206 
4207 	rcu_read_lock();
4208 	rb = rcu_dereference(event->rb);
4209 	if (!rb)
4210 		goto unlock;
4211 
4212 	userpg = rb->user_page;
4213 
4214 	/* Allow new userspace to detect that bit 0 is deprecated */
4215 	userpg->cap_bit0_is_deprecated = 1;
4216 	userpg->size = offsetof(struct perf_event_mmap_page, __reserved);
4217 	userpg->data_offset = PAGE_SIZE;
4218 	userpg->data_size = perf_data_size(rb);
4219 
4220 unlock:
4221 	rcu_read_unlock();
4222 }
4223 
4224 void __weak arch_perf_update_userpage(
4225 	struct perf_event *event, struct perf_event_mmap_page *userpg, u64 now)
4226 {
4227 }
4228 
4229 /*
4230  * Callers need to ensure there can be no nesting of this function, otherwise
4231  * the seqlock logic goes bad. We can not serialize this because the arch
4232  * code calls this from NMI context.
4233  */
4234 void perf_event_update_userpage(struct perf_event *event)
4235 {
4236 	struct perf_event_mmap_page *userpg;
4237 	struct ring_buffer *rb;
4238 	u64 enabled, running, now;
4239 
4240 	rcu_read_lock();
4241 	rb = rcu_dereference(event->rb);
4242 	if (!rb)
4243 		goto unlock;
4244 
4245 	/*
4246 	 * compute total_time_enabled, total_time_running
4247 	 * based on snapshot values taken when the event
4248 	 * was last scheduled in.
4249 	 *
4250 	 * we cannot simply called update_context_time()
4251 	 * because of locking issue as we can be called in
4252 	 * NMI context
4253 	 */
4254 	calc_timer_values(event, &now, &enabled, &running);
4255 
4256 	userpg = rb->user_page;
4257 	/*
4258 	 * Disable preemption so as to not let the corresponding user-space
4259 	 * spin too long if we get preempted.
4260 	 */
4261 	preempt_disable();
4262 	++userpg->lock;
4263 	barrier();
4264 	userpg->index = perf_event_index(event);
4265 	userpg->offset = perf_event_count(event);
4266 	if (userpg->index)
4267 		userpg->offset -= local64_read(&event->hw.prev_count);
4268 
4269 	userpg->time_enabled = enabled +
4270 			atomic64_read(&event->child_total_time_enabled);
4271 
4272 	userpg->time_running = running +
4273 			atomic64_read(&event->child_total_time_running);
4274 
4275 	arch_perf_update_userpage(event, userpg, now);
4276 
4277 	barrier();
4278 	++userpg->lock;
4279 	preempt_enable();
4280 unlock:
4281 	rcu_read_unlock();
4282 }
4283 
4284 static int perf_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
4285 {
4286 	struct perf_event *event = vma->vm_file->private_data;
4287 	struct ring_buffer *rb;
4288 	int ret = VM_FAULT_SIGBUS;
4289 
4290 	if (vmf->flags & FAULT_FLAG_MKWRITE) {
4291 		if (vmf->pgoff == 0)
4292 			ret = 0;
4293 		return ret;
4294 	}
4295 
4296 	rcu_read_lock();
4297 	rb = rcu_dereference(event->rb);
4298 	if (!rb)
4299 		goto unlock;
4300 
4301 	if (vmf->pgoff && (vmf->flags & FAULT_FLAG_WRITE))
4302 		goto unlock;
4303 
4304 	vmf->page = perf_mmap_to_page(rb, vmf->pgoff);
4305 	if (!vmf->page)
4306 		goto unlock;
4307 
4308 	get_page(vmf->page);
4309 	vmf->page->mapping = vma->vm_file->f_mapping;
4310 	vmf->page->index   = vmf->pgoff;
4311 
4312 	ret = 0;
4313 unlock:
4314 	rcu_read_unlock();
4315 
4316 	return ret;
4317 }
4318 
4319 static void ring_buffer_attach(struct perf_event *event,
4320 			       struct ring_buffer *rb)
4321 {
4322 	struct ring_buffer *old_rb = NULL;
4323 	unsigned long flags;
4324 
4325 	if (event->rb) {
4326 		/*
4327 		 * Should be impossible, we set this when removing
4328 		 * event->rb_entry and wait/clear when adding event->rb_entry.
4329 		 */
4330 		WARN_ON_ONCE(event->rcu_pending);
4331 
4332 		old_rb = event->rb;
4333 		event->rcu_batches = get_state_synchronize_rcu();
4334 		event->rcu_pending = 1;
4335 
4336 		spin_lock_irqsave(&old_rb->event_lock, flags);
4337 		list_del_rcu(&event->rb_entry);
4338 		spin_unlock_irqrestore(&old_rb->event_lock, flags);
4339 	}
4340 
4341 	if (event->rcu_pending && rb) {
4342 		cond_synchronize_rcu(event->rcu_batches);
4343 		event->rcu_pending = 0;
4344 	}
4345 
4346 	if (rb) {
4347 		spin_lock_irqsave(&rb->event_lock, flags);
4348 		list_add_rcu(&event->rb_entry, &rb->event_list);
4349 		spin_unlock_irqrestore(&rb->event_lock, flags);
4350 	}
4351 
4352 	rcu_assign_pointer(event->rb, rb);
4353 
4354 	if (old_rb) {
4355 		ring_buffer_put(old_rb);
4356 		/*
4357 		 * Since we detached before setting the new rb, so that we
4358 		 * could attach the new rb, we could have missed a wakeup.
4359 		 * Provide it now.
4360 		 */
4361 		wake_up_all(&event->waitq);
4362 	}
4363 }
4364 
4365 static void ring_buffer_wakeup(struct perf_event *event)
4366 {
4367 	struct ring_buffer *rb;
4368 
4369 	rcu_read_lock();
4370 	rb = rcu_dereference(event->rb);
4371 	if (rb) {
4372 		list_for_each_entry_rcu(event, &rb->event_list, rb_entry)
4373 			wake_up_all(&event->waitq);
4374 	}
4375 	rcu_read_unlock();
4376 }
4377 
4378 static void rb_free_rcu(struct rcu_head *rcu_head)
4379 {
4380 	struct ring_buffer *rb;
4381 
4382 	rb = container_of(rcu_head, struct ring_buffer, rcu_head);
4383 	rb_free(rb);
4384 }
4385 
4386 struct ring_buffer *ring_buffer_get(struct perf_event *event)
4387 {
4388 	struct ring_buffer *rb;
4389 
4390 	rcu_read_lock();
4391 	rb = rcu_dereference(event->rb);
4392 	if (rb) {
4393 		if (!atomic_inc_not_zero(&rb->refcount))
4394 			rb = NULL;
4395 	}
4396 	rcu_read_unlock();
4397 
4398 	return rb;
4399 }
4400 
4401 void ring_buffer_put(struct ring_buffer *rb)
4402 {
4403 	if (!atomic_dec_and_test(&rb->refcount))
4404 		return;
4405 
4406 	WARN_ON_ONCE(!list_empty(&rb->event_list));
4407 
4408 	call_rcu(&rb->rcu_head, rb_free_rcu);
4409 }
4410 
4411 static void perf_mmap_open(struct vm_area_struct *vma)
4412 {
4413 	struct perf_event *event = vma->vm_file->private_data;
4414 
4415 	atomic_inc(&event->mmap_count);
4416 	atomic_inc(&event->rb->mmap_count);
4417 
4418 	if (vma->vm_pgoff)
4419 		atomic_inc(&event->rb->aux_mmap_count);
4420 
4421 	if (event->pmu->event_mapped)
4422 		event->pmu->event_mapped(event);
4423 }
4424 
4425 /*
4426  * A buffer can be mmap()ed multiple times; either directly through the same
4427  * event, or through other events by use of perf_event_set_output().
4428  *
4429  * In order to undo the VM accounting done by perf_mmap() we need to destroy
4430  * the buffer here, where we still have a VM context. This means we need
4431  * to detach all events redirecting to us.
4432  */
4433 static void perf_mmap_close(struct vm_area_struct *vma)
4434 {
4435 	struct perf_event *event = vma->vm_file->private_data;
4436 
4437 	struct ring_buffer *rb = ring_buffer_get(event);
4438 	struct user_struct *mmap_user = rb->mmap_user;
4439 	int mmap_locked = rb->mmap_locked;
4440 	unsigned long size = perf_data_size(rb);
4441 
4442 	if (event->pmu->event_unmapped)
4443 		event->pmu->event_unmapped(event);
4444 
4445 	/*
4446 	 * rb->aux_mmap_count will always drop before rb->mmap_count and
4447 	 * event->mmap_count, so it is ok to use event->mmap_mutex to
4448 	 * serialize with perf_mmap here.
4449 	 */
4450 	if (rb_has_aux(rb) && vma->vm_pgoff == rb->aux_pgoff &&
4451 	    atomic_dec_and_mutex_lock(&rb->aux_mmap_count, &event->mmap_mutex)) {
4452 		atomic_long_sub(rb->aux_nr_pages, &mmap_user->locked_vm);
4453 		vma->vm_mm->pinned_vm -= rb->aux_mmap_locked;
4454 
4455 		rb_free_aux(rb);
4456 		mutex_unlock(&event->mmap_mutex);
4457 	}
4458 
4459 	atomic_dec(&rb->mmap_count);
4460 
4461 	if (!atomic_dec_and_mutex_lock(&event->mmap_count, &event->mmap_mutex))
4462 		goto out_put;
4463 
4464 	ring_buffer_attach(event, NULL);
4465 	mutex_unlock(&event->mmap_mutex);
4466 
4467 	/* If there's still other mmap()s of this buffer, we're done. */
4468 	if (atomic_read(&rb->mmap_count))
4469 		goto out_put;
4470 
4471 	/*
4472 	 * No other mmap()s, detach from all other events that might redirect
4473 	 * into the now unreachable buffer. Somewhat complicated by the
4474 	 * fact that rb::event_lock otherwise nests inside mmap_mutex.
4475 	 */
4476 again:
4477 	rcu_read_lock();
4478 	list_for_each_entry_rcu(event, &rb->event_list, rb_entry) {
4479 		if (!atomic_long_inc_not_zero(&event->refcount)) {
4480 			/*
4481 			 * This event is en-route to free_event() which will
4482 			 * detach it and remove it from the list.
4483 			 */
4484 			continue;
4485 		}
4486 		rcu_read_unlock();
4487 
4488 		mutex_lock(&event->mmap_mutex);
4489 		/*
4490 		 * Check we didn't race with perf_event_set_output() which can
4491 		 * swizzle the rb from under us while we were waiting to
4492 		 * acquire mmap_mutex.
4493 		 *
4494 		 * If we find a different rb; ignore this event, a next
4495 		 * iteration will no longer find it on the list. We have to
4496 		 * still restart the iteration to make sure we're not now
4497 		 * iterating the wrong list.
4498 		 */
4499 		if (event->rb == rb)
4500 			ring_buffer_attach(event, NULL);
4501 
4502 		mutex_unlock(&event->mmap_mutex);
4503 		put_event(event);
4504 
4505 		/*
4506 		 * Restart the iteration; either we're on the wrong list or
4507 		 * destroyed its integrity by doing a deletion.
4508 		 */
4509 		goto again;
4510 	}
4511 	rcu_read_unlock();
4512 
4513 	/*
4514 	 * It could be there's still a few 0-ref events on the list; they'll
4515 	 * get cleaned up by free_event() -- they'll also still have their
4516 	 * ref on the rb and will free it whenever they are done with it.
4517 	 *
4518 	 * Aside from that, this buffer is 'fully' detached and unmapped,
4519 	 * undo the VM accounting.
4520 	 */
4521 
4522 	atomic_long_sub((size >> PAGE_SHIFT) + 1, &mmap_user->locked_vm);
4523 	vma->vm_mm->pinned_vm -= mmap_locked;
4524 	free_uid(mmap_user);
4525 
4526 out_put:
4527 	ring_buffer_put(rb); /* could be last */
4528 }
4529 
4530 static const struct vm_operations_struct perf_mmap_vmops = {
4531 	.open		= perf_mmap_open,
4532 	.close		= perf_mmap_close, /* non mergable */
4533 	.fault		= perf_mmap_fault,
4534 	.page_mkwrite	= perf_mmap_fault,
4535 };
4536 
4537 static int perf_mmap(struct file *file, struct vm_area_struct *vma)
4538 {
4539 	struct perf_event *event = file->private_data;
4540 	unsigned long user_locked, user_lock_limit;
4541 	struct user_struct *user = current_user();
4542 	unsigned long locked, lock_limit;
4543 	struct ring_buffer *rb = NULL;
4544 	unsigned long vma_size;
4545 	unsigned long nr_pages;
4546 	long user_extra = 0, extra = 0;
4547 	int ret = 0, flags = 0;
4548 
4549 	/*
4550 	 * Don't allow mmap() of inherited per-task counters. This would
4551 	 * create a performance issue due to all children writing to the
4552 	 * same rb.
4553 	 */
4554 	if (event->cpu == -1 && event->attr.inherit)
4555 		return -EINVAL;
4556 
4557 	if (!(vma->vm_flags & VM_SHARED))
4558 		return -EINVAL;
4559 
4560 	vma_size = vma->vm_end - vma->vm_start;
4561 
4562 	if (vma->vm_pgoff == 0) {
4563 		nr_pages = (vma_size / PAGE_SIZE) - 1;
4564 	} else {
4565 		/*
4566 		 * AUX area mapping: if rb->aux_nr_pages != 0, it's already
4567 		 * mapped, all subsequent mappings should have the same size
4568 		 * and offset. Must be above the normal perf buffer.
4569 		 */
4570 		u64 aux_offset, aux_size;
4571 
4572 		if (!event->rb)
4573 			return -EINVAL;
4574 
4575 		nr_pages = vma_size / PAGE_SIZE;
4576 
4577 		mutex_lock(&event->mmap_mutex);
4578 		ret = -EINVAL;
4579 
4580 		rb = event->rb;
4581 		if (!rb)
4582 			goto aux_unlock;
4583 
4584 		aux_offset = ACCESS_ONCE(rb->user_page->aux_offset);
4585 		aux_size = ACCESS_ONCE(rb->user_page->aux_size);
4586 
4587 		if (aux_offset < perf_data_size(rb) + PAGE_SIZE)
4588 			goto aux_unlock;
4589 
4590 		if (aux_offset != vma->vm_pgoff << PAGE_SHIFT)
4591 			goto aux_unlock;
4592 
4593 		/* already mapped with a different offset */
4594 		if (rb_has_aux(rb) && rb->aux_pgoff != vma->vm_pgoff)
4595 			goto aux_unlock;
4596 
4597 		if (aux_size != vma_size || aux_size != nr_pages * PAGE_SIZE)
4598 			goto aux_unlock;
4599 
4600 		/* already mapped with a different size */
4601 		if (rb_has_aux(rb) && rb->aux_nr_pages != nr_pages)
4602 			goto aux_unlock;
4603 
4604 		if (!is_power_of_2(nr_pages))
4605 			goto aux_unlock;
4606 
4607 		if (!atomic_inc_not_zero(&rb->mmap_count))
4608 			goto aux_unlock;
4609 
4610 		if (rb_has_aux(rb)) {
4611 			atomic_inc(&rb->aux_mmap_count);
4612 			ret = 0;
4613 			goto unlock;
4614 		}
4615 
4616 		atomic_set(&rb->aux_mmap_count, 1);
4617 		user_extra = nr_pages;
4618 
4619 		goto accounting;
4620 	}
4621 
4622 	/*
4623 	 * If we have rb pages ensure they're a power-of-two number, so we
4624 	 * can do bitmasks instead of modulo.
4625 	 */
4626 	if (nr_pages != 0 && !is_power_of_2(nr_pages))
4627 		return -EINVAL;
4628 
4629 	if (vma_size != PAGE_SIZE * (1 + nr_pages))
4630 		return -EINVAL;
4631 
4632 	WARN_ON_ONCE(event->ctx->parent_ctx);
4633 again:
4634 	mutex_lock(&event->mmap_mutex);
4635 	if (event->rb) {
4636 		if (event->rb->nr_pages != nr_pages) {
4637 			ret = -EINVAL;
4638 			goto unlock;
4639 		}
4640 
4641 		if (!atomic_inc_not_zero(&event->rb->mmap_count)) {
4642 			/*
4643 			 * Raced against perf_mmap_close() through
4644 			 * perf_event_set_output(). Try again, hope for better
4645 			 * luck.
4646 			 */
4647 			mutex_unlock(&event->mmap_mutex);
4648 			goto again;
4649 		}
4650 
4651 		goto unlock;
4652 	}
4653 
4654 	user_extra = nr_pages + 1;
4655 
4656 accounting:
4657 	user_lock_limit = sysctl_perf_event_mlock >> (PAGE_SHIFT - 10);
4658 
4659 	/*
4660 	 * Increase the limit linearly with more CPUs:
4661 	 */
4662 	user_lock_limit *= num_online_cpus();
4663 
4664 	user_locked = atomic_long_read(&user->locked_vm) + user_extra;
4665 
4666 	if (user_locked > user_lock_limit)
4667 		extra = user_locked - user_lock_limit;
4668 
4669 	lock_limit = rlimit(RLIMIT_MEMLOCK);
4670 	lock_limit >>= PAGE_SHIFT;
4671 	locked = vma->vm_mm->pinned_vm + extra;
4672 
4673 	if ((locked > lock_limit) && perf_paranoid_tracepoint_raw() &&
4674 		!capable(CAP_IPC_LOCK)) {
4675 		ret = -EPERM;
4676 		goto unlock;
4677 	}
4678 
4679 	WARN_ON(!rb && event->rb);
4680 
4681 	if (vma->vm_flags & VM_WRITE)
4682 		flags |= RING_BUFFER_WRITABLE;
4683 
4684 	if (!rb) {
4685 		rb = rb_alloc(nr_pages,
4686 			      event->attr.watermark ? event->attr.wakeup_watermark : 0,
4687 			      event->cpu, flags);
4688 
4689 		if (!rb) {
4690 			ret = -ENOMEM;
4691 			goto unlock;
4692 		}
4693 
4694 		atomic_set(&rb->mmap_count, 1);
4695 		rb->mmap_user = get_current_user();
4696 		rb->mmap_locked = extra;
4697 
4698 		ring_buffer_attach(event, rb);
4699 
4700 		perf_event_init_userpage(event);
4701 		perf_event_update_userpage(event);
4702 	} else {
4703 		ret = rb_alloc_aux(rb, event, vma->vm_pgoff, nr_pages,
4704 				   event->attr.aux_watermark, flags);
4705 		if (!ret)
4706 			rb->aux_mmap_locked = extra;
4707 	}
4708 
4709 unlock:
4710 	if (!ret) {
4711 		atomic_long_add(user_extra, &user->locked_vm);
4712 		vma->vm_mm->pinned_vm += extra;
4713 
4714 		atomic_inc(&event->mmap_count);
4715 	} else if (rb) {
4716 		atomic_dec(&rb->mmap_count);
4717 	}
4718 aux_unlock:
4719 	mutex_unlock(&event->mmap_mutex);
4720 
4721 	/*
4722 	 * Since pinned accounting is per vm we cannot allow fork() to copy our
4723 	 * vma.
4724 	 */
4725 	vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_DONTDUMP;
4726 	vma->vm_ops = &perf_mmap_vmops;
4727 
4728 	if (event->pmu->event_mapped)
4729 		event->pmu->event_mapped(event);
4730 
4731 	return ret;
4732 }
4733 
4734 static int perf_fasync(int fd, struct file *filp, int on)
4735 {
4736 	struct inode *inode = file_inode(filp);
4737 	struct perf_event *event = filp->private_data;
4738 	int retval;
4739 
4740 	mutex_lock(&inode->i_mutex);
4741 	retval = fasync_helper(fd, filp, on, &event->fasync);
4742 	mutex_unlock(&inode->i_mutex);
4743 
4744 	if (retval < 0)
4745 		return retval;
4746 
4747 	return 0;
4748 }
4749 
4750 static const struct file_operations perf_fops = {
4751 	.llseek			= no_llseek,
4752 	.release		= perf_release,
4753 	.read			= perf_read,
4754 	.poll			= perf_poll,
4755 	.unlocked_ioctl		= perf_ioctl,
4756 	.compat_ioctl		= perf_compat_ioctl,
4757 	.mmap			= perf_mmap,
4758 	.fasync			= perf_fasync,
4759 };
4760 
4761 /*
4762  * Perf event wakeup
4763  *
4764  * If there's data, ensure we set the poll() state and publish everything
4765  * to user-space before waking everybody up.
4766  */
4767 
4768 void perf_event_wakeup(struct perf_event *event)
4769 {
4770 	ring_buffer_wakeup(event);
4771 
4772 	if (event->pending_kill) {
4773 		kill_fasync(&event->fasync, SIGIO, event->pending_kill);
4774 		event->pending_kill = 0;
4775 	}
4776 }
4777 
4778 static void perf_pending_event(struct irq_work *entry)
4779 {
4780 	struct perf_event *event = container_of(entry,
4781 			struct perf_event, pending);
4782 	int rctx;
4783 
4784 	rctx = perf_swevent_get_recursion_context();
4785 	/*
4786 	 * If we 'fail' here, that's OK, it means recursion is already disabled
4787 	 * and we won't recurse 'further'.
4788 	 */
4789 
4790 	if (event->pending_disable) {
4791 		event->pending_disable = 0;
4792 		__perf_event_disable(event);
4793 	}
4794 
4795 	if (event->pending_wakeup) {
4796 		event->pending_wakeup = 0;
4797 		perf_event_wakeup(event);
4798 	}
4799 
4800 	if (rctx >= 0)
4801 		perf_swevent_put_recursion_context(rctx);
4802 }
4803 
4804 /*
4805  * We assume there is only KVM supporting the callbacks.
4806  * Later on, we might change it to a list if there is
4807  * another virtualization implementation supporting the callbacks.
4808  */
4809 struct perf_guest_info_callbacks *perf_guest_cbs;
4810 
4811 int perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *cbs)
4812 {
4813 	perf_guest_cbs = cbs;
4814 	return 0;
4815 }
4816 EXPORT_SYMBOL_GPL(perf_register_guest_info_callbacks);
4817 
4818 int perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *cbs)
4819 {
4820 	perf_guest_cbs = NULL;
4821 	return 0;
4822 }
4823 EXPORT_SYMBOL_GPL(perf_unregister_guest_info_callbacks);
4824 
4825 static void
4826 perf_output_sample_regs(struct perf_output_handle *handle,
4827 			struct pt_regs *regs, u64 mask)
4828 {
4829 	int bit;
4830 
4831 	for_each_set_bit(bit, (const unsigned long *) &mask,
4832 			 sizeof(mask) * BITS_PER_BYTE) {
4833 		u64 val;
4834 
4835 		val = perf_reg_value(regs, bit);
4836 		perf_output_put(handle, val);
4837 	}
4838 }
4839 
4840 static void perf_sample_regs_user(struct perf_regs *regs_user,
4841 				  struct pt_regs *regs,
4842 				  struct pt_regs *regs_user_copy)
4843 {
4844 	if (user_mode(regs)) {
4845 		regs_user->abi = perf_reg_abi(current);
4846 		regs_user->regs = regs;
4847 	} else if (current->mm) {
4848 		perf_get_regs_user(regs_user, regs, regs_user_copy);
4849 	} else {
4850 		regs_user->abi = PERF_SAMPLE_REGS_ABI_NONE;
4851 		regs_user->regs = NULL;
4852 	}
4853 }
4854 
4855 static void perf_sample_regs_intr(struct perf_regs *regs_intr,
4856 				  struct pt_regs *regs)
4857 {
4858 	regs_intr->regs = regs;
4859 	regs_intr->abi  = perf_reg_abi(current);
4860 }
4861 
4862 
4863 /*
4864  * Get remaining task size from user stack pointer.
4865  *
4866  * It'd be better to take stack vma map and limit this more
4867  * precisly, but there's no way to get it safely under interrupt,
4868  * so using TASK_SIZE as limit.
4869  */
4870 static u64 perf_ustack_task_size(struct pt_regs *regs)
4871 {
4872 	unsigned long addr = perf_user_stack_pointer(regs);
4873 
4874 	if (!addr || addr >= TASK_SIZE)
4875 		return 0;
4876 
4877 	return TASK_SIZE - addr;
4878 }
4879 
4880 static u16
4881 perf_sample_ustack_size(u16 stack_size, u16 header_size,
4882 			struct pt_regs *regs)
4883 {
4884 	u64 task_size;
4885 
4886 	/* No regs, no stack pointer, no dump. */
4887 	if (!regs)
4888 		return 0;
4889 
4890 	/*
4891 	 * Check if we fit in with the requested stack size into the:
4892 	 * - TASK_SIZE
4893 	 *   If we don't, we limit the size to the TASK_SIZE.
4894 	 *
4895 	 * - remaining sample size
4896 	 *   If we don't, we customize the stack size to
4897 	 *   fit in to the remaining sample size.
4898 	 */
4899 
4900 	task_size  = min((u64) USHRT_MAX, perf_ustack_task_size(regs));
4901 	stack_size = min(stack_size, (u16) task_size);
4902 
4903 	/* Current header size plus static size and dynamic size. */
4904 	header_size += 2 * sizeof(u64);
4905 
4906 	/* Do we fit in with the current stack dump size? */
4907 	if ((u16) (header_size + stack_size) < header_size) {
4908 		/*
4909 		 * If we overflow the maximum size for the sample,
4910 		 * we customize the stack dump size to fit in.
4911 		 */
4912 		stack_size = USHRT_MAX - header_size - sizeof(u64);
4913 		stack_size = round_up(stack_size, sizeof(u64));
4914 	}
4915 
4916 	return stack_size;
4917 }
4918 
4919 static void
4920 perf_output_sample_ustack(struct perf_output_handle *handle, u64 dump_size,
4921 			  struct pt_regs *regs)
4922 {
4923 	/* Case of a kernel thread, nothing to dump */
4924 	if (!regs) {
4925 		u64 size = 0;
4926 		perf_output_put(handle, size);
4927 	} else {
4928 		unsigned long sp;
4929 		unsigned int rem;
4930 		u64 dyn_size;
4931 
4932 		/*
4933 		 * We dump:
4934 		 * static size
4935 		 *   - the size requested by user or the best one we can fit
4936 		 *     in to the sample max size
4937 		 * data
4938 		 *   - user stack dump data
4939 		 * dynamic size
4940 		 *   - the actual dumped size
4941 		 */
4942 
4943 		/* Static size. */
4944 		perf_output_put(handle, dump_size);
4945 
4946 		/* Data. */
4947 		sp = perf_user_stack_pointer(regs);
4948 		rem = __output_copy_user(handle, (void *) sp, dump_size);
4949 		dyn_size = dump_size - rem;
4950 
4951 		perf_output_skip(handle, rem);
4952 
4953 		/* Dynamic size. */
4954 		perf_output_put(handle, dyn_size);
4955 	}
4956 }
4957 
4958 static void __perf_event_header__init_id(struct perf_event_header *header,
4959 					 struct perf_sample_data *data,
4960 					 struct perf_event *event)
4961 {
4962 	u64 sample_type = event->attr.sample_type;
4963 
4964 	data->type = sample_type;
4965 	header->size += event->id_header_size;
4966 
4967 	if (sample_type & PERF_SAMPLE_TID) {
4968 		/* namespace issues */
4969 		data->tid_entry.pid = perf_event_pid(event, current);
4970 		data->tid_entry.tid = perf_event_tid(event, current);
4971 	}
4972 
4973 	if (sample_type & PERF_SAMPLE_TIME)
4974 		data->time = perf_event_clock(event);
4975 
4976 	if (sample_type & (PERF_SAMPLE_ID | PERF_SAMPLE_IDENTIFIER))
4977 		data->id = primary_event_id(event);
4978 
4979 	if (sample_type & PERF_SAMPLE_STREAM_ID)
4980 		data->stream_id = event->id;
4981 
4982 	if (sample_type & PERF_SAMPLE_CPU) {
4983 		data->cpu_entry.cpu	 = raw_smp_processor_id();
4984 		data->cpu_entry.reserved = 0;
4985 	}
4986 }
4987 
4988 void perf_event_header__init_id(struct perf_event_header *header,
4989 				struct perf_sample_data *data,
4990 				struct perf_event *event)
4991 {
4992 	if (event->attr.sample_id_all)
4993 		__perf_event_header__init_id(header, data, event);
4994 }
4995 
4996 static void __perf_event__output_id_sample(struct perf_output_handle *handle,
4997 					   struct perf_sample_data *data)
4998 {
4999 	u64 sample_type = data->type;
5000 
5001 	if (sample_type & PERF_SAMPLE_TID)
5002 		perf_output_put(handle, data->tid_entry);
5003 
5004 	if (sample_type & PERF_SAMPLE_TIME)
5005 		perf_output_put(handle, data->time);
5006 
5007 	if (sample_type & PERF_SAMPLE_ID)
5008 		perf_output_put(handle, data->id);
5009 
5010 	if (sample_type & PERF_SAMPLE_STREAM_ID)
5011 		perf_output_put(handle, data->stream_id);
5012 
5013 	if (sample_type & PERF_SAMPLE_CPU)
5014 		perf_output_put(handle, data->cpu_entry);
5015 
5016 	if (sample_type & PERF_SAMPLE_IDENTIFIER)
5017 		perf_output_put(handle, data->id);
5018 }
5019 
5020 void perf_event__output_id_sample(struct perf_event *event,
5021 				  struct perf_output_handle *handle,
5022 				  struct perf_sample_data *sample)
5023 {
5024 	if (event->attr.sample_id_all)
5025 		__perf_event__output_id_sample(handle, sample);
5026 }
5027 
5028 static void perf_output_read_one(struct perf_output_handle *handle,
5029 				 struct perf_event *event,
5030 				 u64 enabled, u64 running)
5031 {
5032 	u64 read_format = event->attr.read_format;
5033 	u64 values[4];
5034 	int n = 0;
5035 
5036 	values[n++] = perf_event_count(event);
5037 	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
5038 		values[n++] = enabled +
5039 			atomic64_read(&event->child_total_time_enabled);
5040 	}
5041 	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
5042 		values[n++] = running +
5043 			atomic64_read(&event->child_total_time_running);
5044 	}
5045 	if (read_format & PERF_FORMAT_ID)
5046 		values[n++] = primary_event_id(event);
5047 
5048 	__output_copy(handle, values, n * sizeof(u64));
5049 }
5050 
5051 /*
5052  * XXX PERF_FORMAT_GROUP vs inherited events seems difficult.
5053  */
5054 static void perf_output_read_group(struct perf_output_handle *handle,
5055 			    struct perf_event *event,
5056 			    u64 enabled, u64 running)
5057 {
5058 	struct perf_event *leader = event->group_leader, *sub;
5059 	u64 read_format = event->attr.read_format;
5060 	u64 values[5];
5061 	int n = 0;
5062 
5063 	values[n++] = 1 + leader->nr_siblings;
5064 
5065 	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
5066 		values[n++] = enabled;
5067 
5068 	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
5069 		values[n++] = running;
5070 
5071 	if (leader != event)
5072 		leader->pmu->read(leader);
5073 
5074 	values[n++] = perf_event_count(leader);
5075 	if (read_format & PERF_FORMAT_ID)
5076 		values[n++] = primary_event_id(leader);
5077 
5078 	__output_copy(handle, values, n * sizeof(u64));
5079 
5080 	list_for_each_entry(sub, &leader->sibling_list, group_entry) {
5081 		n = 0;
5082 
5083 		if ((sub != event) &&
5084 		    (sub->state == PERF_EVENT_STATE_ACTIVE))
5085 			sub->pmu->read(sub);
5086 
5087 		values[n++] = perf_event_count(sub);
5088 		if (read_format & PERF_FORMAT_ID)
5089 			values[n++] = primary_event_id(sub);
5090 
5091 		__output_copy(handle, values, n * sizeof(u64));
5092 	}
5093 }
5094 
5095 #define PERF_FORMAT_TOTAL_TIMES (PERF_FORMAT_TOTAL_TIME_ENABLED|\
5096 				 PERF_FORMAT_TOTAL_TIME_RUNNING)
5097 
5098 static void perf_output_read(struct perf_output_handle *handle,
5099 			     struct perf_event *event)
5100 {
5101 	u64 enabled = 0, running = 0, now;
5102 	u64 read_format = event->attr.read_format;
5103 
5104 	/*
5105 	 * compute total_time_enabled, total_time_running
5106 	 * based on snapshot values taken when the event
5107 	 * was last scheduled in.
5108 	 *
5109 	 * we cannot simply called update_context_time()
5110 	 * because of locking issue as we are called in
5111 	 * NMI context
5112 	 */
5113 	if (read_format & PERF_FORMAT_TOTAL_TIMES)
5114 		calc_timer_values(event, &now, &enabled, &running);
5115 
5116 	if (event->attr.read_format & PERF_FORMAT_GROUP)
5117 		perf_output_read_group(handle, event, enabled, running);
5118 	else
5119 		perf_output_read_one(handle, event, enabled, running);
5120 }
5121 
5122 void perf_output_sample(struct perf_output_handle *handle,
5123 			struct perf_event_header *header,
5124 			struct perf_sample_data *data,
5125 			struct perf_event *event)
5126 {
5127 	u64 sample_type = data->type;
5128 
5129 	perf_output_put(handle, *header);
5130 
5131 	if (sample_type & PERF_SAMPLE_IDENTIFIER)
5132 		perf_output_put(handle, data->id);
5133 
5134 	if (sample_type & PERF_SAMPLE_IP)
5135 		perf_output_put(handle, data->ip);
5136 
5137 	if (sample_type & PERF_SAMPLE_TID)
5138 		perf_output_put(handle, data->tid_entry);
5139 
5140 	if (sample_type & PERF_SAMPLE_TIME)
5141 		perf_output_put(handle, data->time);
5142 
5143 	if (sample_type & PERF_SAMPLE_ADDR)
5144 		perf_output_put(handle, data->addr);
5145 
5146 	if (sample_type & PERF_SAMPLE_ID)
5147 		perf_output_put(handle, data->id);
5148 
5149 	if (sample_type & PERF_SAMPLE_STREAM_ID)
5150 		perf_output_put(handle, data->stream_id);
5151 
5152 	if (sample_type & PERF_SAMPLE_CPU)
5153 		perf_output_put(handle, data->cpu_entry);
5154 
5155 	if (sample_type & PERF_SAMPLE_PERIOD)
5156 		perf_output_put(handle, data->period);
5157 
5158 	if (sample_type & PERF_SAMPLE_READ)
5159 		perf_output_read(handle, event);
5160 
5161 	if (sample_type & PERF_SAMPLE_CALLCHAIN) {
5162 		if (data->callchain) {
5163 			int size = 1;
5164 
5165 			if (data->callchain)
5166 				size += data->callchain->nr;
5167 
5168 			size *= sizeof(u64);
5169 
5170 			__output_copy(handle, data->callchain, size);
5171 		} else {
5172 			u64 nr = 0;
5173 			perf_output_put(handle, nr);
5174 		}
5175 	}
5176 
5177 	if (sample_type & PERF_SAMPLE_RAW) {
5178 		if (data->raw) {
5179 			perf_output_put(handle, data->raw->size);
5180 			__output_copy(handle, data->raw->data,
5181 					   data->raw->size);
5182 		} else {
5183 			struct {
5184 				u32	size;
5185 				u32	data;
5186 			} raw = {
5187 				.size = sizeof(u32),
5188 				.data = 0,
5189 			};
5190 			perf_output_put(handle, raw);
5191 		}
5192 	}
5193 
5194 	if (sample_type & PERF_SAMPLE_BRANCH_STACK) {
5195 		if (data->br_stack) {
5196 			size_t size;
5197 
5198 			size = data->br_stack->nr
5199 			     * sizeof(struct perf_branch_entry);
5200 
5201 			perf_output_put(handle, data->br_stack->nr);
5202 			perf_output_copy(handle, data->br_stack->entries, size);
5203 		} else {
5204 			/*
5205 			 * we always store at least the value of nr
5206 			 */
5207 			u64 nr = 0;
5208 			perf_output_put(handle, nr);
5209 		}
5210 	}
5211 
5212 	if (sample_type & PERF_SAMPLE_REGS_USER) {
5213 		u64 abi = data->regs_user.abi;
5214 
5215 		/*
5216 		 * If there are no regs to dump, notice it through
5217 		 * first u64 being zero (PERF_SAMPLE_REGS_ABI_NONE).
5218 		 */
5219 		perf_output_put(handle, abi);
5220 
5221 		if (abi) {
5222 			u64 mask = event->attr.sample_regs_user;
5223 			perf_output_sample_regs(handle,
5224 						data->regs_user.regs,
5225 						mask);
5226 		}
5227 	}
5228 
5229 	if (sample_type & PERF_SAMPLE_STACK_USER) {
5230 		perf_output_sample_ustack(handle,
5231 					  data->stack_user_size,
5232 					  data->regs_user.regs);
5233 	}
5234 
5235 	if (sample_type & PERF_SAMPLE_WEIGHT)
5236 		perf_output_put(handle, data->weight);
5237 
5238 	if (sample_type & PERF_SAMPLE_DATA_SRC)
5239 		perf_output_put(handle, data->data_src.val);
5240 
5241 	if (sample_type & PERF_SAMPLE_TRANSACTION)
5242 		perf_output_put(handle, data->txn);
5243 
5244 	if (sample_type & PERF_SAMPLE_REGS_INTR) {
5245 		u64 abi = data->regs_intr.abi;
5246 		/*
5247 		 * If there are no regs to dump, notice it through
5248 		 * first u64 being zero (PERF_SAMPLE_REGS_ABI_NONE).
5249 		 */
5250 		perf_output_put(handle, abi);
5251 
5252 		if (abi) {
5253 			u64 mask = event->attr.sample_regs_intr;
5254 
5255 			perf_output_sample_regs(handle,
5256 						data->regs_intr.regs,
5257 						mask);
5258 		}
5259 	}
5260 
5261 	if (!event->attr.watermark) {
5262 		int wakeup_events = event->attr.wakeup_events;
5263 
5264 		if (wakeup_events) {
5265 			struct ring_buffer *rb = handle->rb;
5266 			int events = local_inc_return(&rb->events);
5267 
5268 			if (events >= wakeup_events) {
5269 				local_sub(wakeup_events, &rb->events);
5270 				local_inc(&rb->wakeup);
5271 			}
5272 		}
5273 	}
5274 }
5275 
5276 void perf_prepare_sample(struct perf_event_header *header,
5277 			 struct perf_sample_data *data,
5278 			 struct perf_event *event,
5279 			 struct pt_regs *regs)
5280 {
5281 	u64 sample_type = event->attr.sample_type;
5282 
5283 	header->type = PERF_RECORD_SAMPLE;
5284 	header->size = sizeof(*header) + event->header_size;
5285 
5286 	header->misc = 0;
5287 	header->misc |= perf_misc_flags(regs);
5288 
5289 	__perf_event_header__init_id(header, data, event);
5290 
5291 	if (sample_type & PERF_SAMPLE_IP)
5292 		data->ip = perf_instruction_pointer(regs);
5293 
5294 	if (sample_type & PERF_SAMPLE_CALLCHAIN) {
5295 		int size = 1;
5296 
5297 		data->callchain = perf_callchain(event, regs);
5298 
5299 		if (data->callchain)
5300 			size += data->callchain->nr;
5301 
5302 		header->size += size * sizeof(u64);
5303 	}
5304 
5305 	if (sample_type & PERF_SAMPLE_RAW) {
5306 		int size = sizeof(u32);
5307 
5308 		if (data->raw)
5309 			size += data->raw->size;
5310 		else
5311 			size += sizeof(u32);
5312 
5313 		WARN_ON_ONCE(size & (sizeof(u64)-1));
5314 		header->size += size;
5315 	}
5316 
5317 	if (sample_type & PERF_SAMPLE_BRANCH_STACK) {
5318 		int size = sizeof(u64); /* nr */
5319 		if (data->br_stack) {
5320 			size += data->br_stack->nr
5321 			      * sizeof(struct perf_branch_entry);
5322 		}
5323 		header->size += size;
5324 	}
5325 
5326 	if (sample_type & (PERF_SAMPLE_REGS_USER | PERF_SAMPLE_STACK_USER))
5327 		perf_sample_regs_user(&data->regs_user, regs,
5328 				      &data->regs_user_copy);
5329 
5330 	if (sample_type & PERF_SAMPLE_REGS_USER) {
5331 		/* regs dump ABI info */
5332 		int size = sizeof(u64);
5333 
5334 		if (data->regs_user.regs) {
5335 			u64 mask = event->attr.sample_regs_user;
5336 			size += hweight64(mask) * sizeof(u64);
5337 		}
5338 
5339 		header->size += size;
5340 	}
5341 
5342 	if (sample_type & PERF_SAMPLE_STACK_USER) {
5343 		/*
5344 		 * Either we need PERF_SAMPLE_STACK_USER bit to be allways
5345 		 * processed as the last one or have additional check added
5346 		 * in case new sample type is added, because we could eat
5347 		 * up the rest of the sample size.
5348 		 */
5349 		u16 stack_size = event->attr.sample_stack_user;
5350 		u16 size = sizeof(u64);
5351 
5352 		stack_size = perf_sample_ustack_size(stack_size, header->size,
5353 						     data->regs_user.regs);
5354 
5355 		/*
5356 		 * If there is something to dump, add space for the dump
5357 		 * itself and for the field that tells the dynamic size,
5358 		 * which is how many have been actually dumped.
5359 		 */
5360 		if (stack_size)
5361 			size += sizeof(u64) + stack_size;
5362 
5363 		data->stack_user_size = stack_size;
5364 		header->size += size;
5365 	}
5366 
5367 	if (sample_type & PERF_SAMPLE_REGS_INTR) {
5368 		/* regs dump ABI info */
5369 		int size = sizeof(u64);
5370 
5371 		perf_sample_regs_intr(&data->regs_intr, regs);
5372 
5373 		if (data->regs_intr.regs) {
5374 			u64 mask = event->attr.sample_regs_intr;
5375 
5376 			size += hweight64(mask) * sizeof(u64);
5377 		}
5378 
5379 		header->size += size;
5380 	}
5381 }
5382 
5383 static void perf_event_output(struct perf_event *event,
5384 				struct perf_sample_data *data,
5385 				struct pt_regs *regs)
5386 {
5387 	struct perf_output_handle handle;
5388 	struct perf_event_header header;
5389 
5390 	/* protect the callchain buffers */
5391 	rcu_read_lock();
5392 
5393 	perf_prepare_sample(&header, data, event, regs);
5394 
5395 	if (perf_output_begin(&handle, event, header.size))
5396 		goto exit;
5397 
5398 	perf_output_sample(&handle, &header, data, event);
5399 
5400 	perf_output_end(&handle);
5401 
5402 exit:
5403 	rcu_read_unlock();
5404 }
5405 
5406 /*
5407  * read event_id
5408  */
5409 
5410 struct perf_read_event {
5411 	struct perf_event_header	header;
5412 
5413 	u32				pid;
5414 	u32				tid;
5415 };
5416 
5417 static void
5418 perf_event_read_event(struct perf_event *event,
5419 			struct task_struct *task)
5420 {
5421 	struct perf_output_handle handle;
5422 	struct perf_sample_data sample;
5423 	struct perf_read_event read_event = {
5424 		.header = {
5425 			.type = PERF_RECORD_READ,
5426 			.misc = 0,
5427 			.size = sizeof(read_event) + event->read_size,
5428 		},
5429 		.pid = perf_event_pid(event, task),
5430 		.tid = perf_event_tid(event, task),
5431 	};
5432 	int ret;
5433 
5434 	perf_event_header__init_id(&read_event.header, &sample, event);
5435 	ret = perf_output_begin(&handle, event, read_event.header.size);
5436 	if (ret)
5437 		return;
5438 
5439 	perf_output_put(&handle, read_event);
5440 	perf_output_read(&handle, event);
5441 	perf_event__output_id_sample(event, &handle, &sample);
5442 
5443 	perf_output_end(&handle);
5444 }
5445 
5446 typedef void (perf_event_aux_output_cb)(struct perf_event *event, void *data);
5447 
5448 static void
5449 perf_event_aux_ctx(struct perf_event_context *ctx,
5450 		   perf_event_aux_output_cb output,
5451 		   void *data)
5452 {
5453 	struct perf_event *event;
5454 
5455 	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
5456 		if (event->state < PERF_EVENT_STATE_INACTIVE)
5457 			continue;
5458 		if (!event_filter_match(event))
5459 			continue;
5460 		output(event, data);
5461 	}
5462 }
5463 
5464 static void
5465 perf_event_aux(perf_event_aux_output_cb output, void *data,
5466 	       struct perf_event_context *task_ctx)
5467 {
5468 	struct perf_cpu_context *cpuctx;
5469 	struct perf_event_context *ctx;
5470 	struct pmu *pmu;
5471 	int ctxn;
5472 
5473 	rcu_read_lock();
5474 	list_for_each_entry_rcu(pmu, &pmus, entry) {
5475 		cpuctx = get_cpu_ptr(pmu->pmu_cpu_context);
5476 		if (cpuctx->unique_pmu != pmu)
5477 			goto next;
5478 		perf_event_aux_ctx(&cpuctx->ctx, output, data);
5479 		if (task_ctx)
5480 			goto next;
5481 		ctxn = pmu->task_ctx_nr;
5482 		if (ctxn < 0)
5483 			goto next;
5484 		ctx = rcu_dereference(current->perf_event_ctxp[ctxn]);
5485 		if (ctx)
5486 			perf_event_aux_ctx(ctx, output, data);
5487 next:
5488 		put_cpu_ptr(pmu->pmu_cpu_context);
5489 	}
5490 
5491 	if (task_ctx) {
5492 		preempt_disable();
5493 		perf_event_aux_ctx(task_ctx, output, data);
5494 		preempt_enable();
5495 	}
5496 	rcu_read_unlock();
5497 }
5498 
5499 /*
5500  * task tracking -- fork/exit
5501  *
5502  * enabled by: attr.comm | attr.mmap | attr.mmap2 | attr.mmap_data | attr.task
5503  */
5504 
5505 struct perf_task_event {
5506 	struct task_struct		*task;
5507 	struct perf_event_context	*task_ctx;
5508 
5509 	struct {
5510 		struct perf_event_header	header;
5511 
5512 		u32				pid;
5513 		u32				ppid;
5514 		u32				tid;
5515 		u32				ptid;
5516 		u64				time;
5517 	} event_id;
5518 };
5519 
5520 static int perf_event_task_match(struct perf_event *event)
5521 {
5522 	return event->attr.comm  || event->attr.mmap ||
5523 	       event->attr.mmap2 || event->attr.mmap_data ||
5524 	       event->attr.task;
5525 }
5526 
5527 static void perf_event_task_output(struct perf_event *event,
5528 				   void *data)
5529 {
5530 	struct perf_task_event *task_event = data;
5531 	struct perf_output_handle handle;
5532 	struct perf_sample_data	sample;
5533 	struct task_struct *task = task_event->task;
5534 	int ret, size = task_event->event_id.header.size;
5535 
5536 	if (!perf_event_task_match(event))
5537 		return;
5538 
5539 	perf_event_header__init_id(&task_event->event_id.header, &sample, event);
5540 
5541 	ret = perf_output_begin(&handle, event,
5542 				task_event->event_id.header.size);
5543 	if (ret)
5544 		goto out;
5545 
5546 	task_event->event_id.pid = perf_event_pid(event, task);
5547 	task_event->event_id.ppid = perf_event_pid(event, current);
5548 
5549 	task_event->event_id.tid = perf_event_tid(event, task);
5550 	task_event->event_id.ptid = perf_event_tid(event, current);
5551 
5552 	task_event->event_id.time = perf_event_clock(event);
5553 
5554 	perf_output_put(&handle, task_event->event_id);
5555 
5556 	perf_event__output_id_sample(event, &handle, &sample);
5557 
5558 	perf_output_end(&handle);
5559 out:
5560 	task_event->event_id.header.size = size;
5561 }
5562 
5563 static void perf_event_task(struct task_struct *task,
5564 			      struct perf_event_context *task_ctx,
5565 			      int new)
5566 {
5567 	struct perf_task_event task_event;
5568 
5569 	if (!atomic_read(&nr_comm_events) &&
5570 	    !atomic_read(&nr_mmap_events) &&
5571 	    !atomic_read(&nr_task_events))
5572 		return;
5573 
5574 	task_event = (struct perf_task_event){
5575 		.task	  = task,
5576 		.task_ctx = task_ctx,
5577 		.event_id    = {
5578 			.header = {
5579 				.type = new ? PERF_RECORD_FORK : PERF_RECORD_EXIT,
5580 				.misc = 0,
5581 				.size = sizeof(task_event.event_id),
5582 			},
5583 			/* .pid  */
5584 			/* .ppid */
5585 			/* .tid  */
5586 			/* .ptid */
5587 			/* .time */
5588 		},
5589 	};
5590 
5591 	perf_event_aux(perf_event_task_output,
5592 		       &task_event,
5593 		       task_ctx);
5594 }
5595 
5596 void perf_event_fork(struct task_struct *task)
5597 {
5598 	perf_event_task(task, NULL, 1);
5599 }
5600 
5601 /*
5602  * comm tracking
5603  */
5604 
5605 struct perf_comm_event {
5606 	struct task_struct	*task;
5607 	char			*comm;
5608 	int			comm_size;
5609 
5610 	struct {
5611 		struct perf_event_header	header;
5612 
5613 		u32				pid;
5614 		u32				tid;
5615 	} event_id;
5616 };
5617 
5618 static int perf_event_comm_match(struct perf_event *event)
5619 {
5620 	return event->attr.comm;
5621 }
5622 
5623 static void perf_event_comm_output(struct perf_event *event,
5624 				   void *data)
5625 {
5626 	struct perf_comm_event *comm_event = data;
5627 	struct perf_output_handle handle;
5628 	struct perf_sample_data sample;
5629 	int size = comm_event->event_id.header.size;
5630 	int ret;
5631 
5632 	if (!perf_event_comm_match(event))
5633 		return;
5634 
5635 	perf_event_header__init_id(&comm_event->event_id.header, &sample, event);
5636 	ret = perf_output_begin(&handle, event,
5637 				comm_event->event_id.header.size);
5638 
5639 	if (ret)
5640 		goto out;
5641 
5642 	comm_event->event_id.pid = perf_event_pid(event, comm_event->task);
5643 	comm_event->event_id.tid = perf_event_tid(event, comm_event->task);
5644 
5645 	perf_output_put(&handle, comm_event->event_id);
5646 	__output_copy(&handle, comm_event->comm,
5647 				   comm_event->comm_size);
5648 
5649 	perf_event__output_id_sample(event, &handle, &sample);
5650 
5651 	perf_output_end(&handle);
5652 out:
5653 	comm_event->event_id.header.size = size;
5654 }
5655 
5656 static void perf_event_comm_event(struct perf_comm_event *comm_event)
5657 {
5658 	char comm[TASK_COMM_LEN];
5659 	unsigned int size;
5660 
5661 	memset(comm, 0, sizeof(comm));
5662 	strlcpy(comm, comm_event->task->comm, sizeof(comm));
5663 	size = ALIGN(strlen(comm)+1, sizeof(u64));
5664 
5665 	comm_event->comm = comm;
5666 	comm_event->comm_size = size;
5667 
5668 	comm_event->event_id.header.size = sizeof(comm_event->event_id) + size;
5669 
5670 	perf_event_aux(perf_event_comm_output,
5671 		       comm_event,
5672 		       NULL);
5673 }
5674 
5675 void perf_event_comm(struct task_struct *task, bool exec)
5676 {
5677 	struct perf_comm_event comm_event;
5678 
5679 	if (!atomic_read(&nr_comm_events))
5680 		return;
5681 
5682 	comm_event = (struct perf_comm_event){
5683 		.task	= task,
5684 		/* .comm      */
5685 		/* .comm_size */
5686 		.event_id  = {
5687 			.header = {
5688 				.type = PERF_RECORD_COMM,
5689 				.misc = exec ? PERF_RECORD_MISC_COMM_EXEC : 0,
5690 				/* .size */
5691 			},
5692 			/* .pid */
5693 			/* .tid */
5694 		},
5695 	};
5696 
5697 	perf_event_comm_event(&comm_event);
5698 }
5699 
5700 /*
5701  * mmap tracking
5702  */
5703 
5704 struct perf_mmap_event {
5705 	struct vm_area_struct	*vma;
5706 
5707 	const char		*file_name;
5708 	int			file_size;
5709 	int			maj, min;
5710 	u64			ino;
5711 	u64			ino_generation;
5712 	u32			prot, flags;
5713 
5714 	struct {
5715 		struct perf_event_header	header;
5716 
5717 		u32				pid;
5718 		u32				tid;
5719 		u64				start;
5720 		u64				len;
5721 		u64				pgoff;
5722 	} event_id;
5723 };
5724 
5725 static int perf_event_mmap_match(struct perf_event *event,
5726 				 void *data)
5727 {
5728 	struct perf_mmap_event *mmap_event = data;
5729 	struct vm_area_struct *vma = mmap_event->vma;
5730 	int executable = vma->vm_flags & VM_EXEC;
5731 
5732 	return (!executable && event->attr.mmap_data) ||
5733 	       (executable && (event->attr.mmap || event->attr.mmap2));
5734 }
5735 
5736 static void perf_event_mmap_output(struct perf_event *event,
5737 				   void *data)
5738 {
5739 	struct perf_mmap_event *mmap_event = data;
5740 	struct perf_output_handle handle;
5741 	struct perf_sample_data sample;
5742 	int size = mmap_event->event_id.header.size;
5743 	int ret;
5744 
5745 	if (!perf_event_mmap_match(event, data))
5746 		return;
5747 
5748 	if (event->attr.mmap2) {
5749 		mmap_event->event_id.header.type = PERF_RECORD_MMAP2;
5750 		mmap_event->event_id.header.size += sizeof(mmap_event->maj);
5751 		mmap_event->event_id.header.size += sizeof(mmap_event->min);
5752 		mmap_event->event_id.header.size += sizeof(mmap_event->ino);
5753 		mmap_event->event_id.header.size += sizeof(mmap_event->ino_generation);
5754 		mmap_event->event_id.header.size += sizeof(mmap_event->prot);
5755 		mmap_event->event_id.header.size += sizeof(mmap_event->flags);
5756 	}
5757 
5758 	perf_event_header__init_id(&mmap_event->event_id.header, &sample, event);
5759 	ret = perf_output_begin(&handle, event,
5760 				mmap_event->event_id.header.size);
5761 	if (ret)
5762 		goto out;
5763 
5764 	mmap_event->event_id.pid = perf_event_pid(event, current);
5765 	mmap_event->event_id.tid = perf_event_tid(event, current);
5766 
5767 	perf_output_put(&handle, mmap_event->event_id);
5768 
5769 	if (event->attr.mmap2) {
5770 		perf_output_put(&handle, mmap_event->maj);
5771 		perf_output_put(&handle, mmap_event->min);
5772 		perf_output_put(&handle, mmap_event->ino);
5773 		perf_output_put(&handle, mmap_event->ino_generation);
5774 		perf_output_put(&handle, mmap_event->prot);
5775 		perf_output_put(&handle, mmap_event->flags);
5776 	}
5777 
5778 	__output_copy(&handle, mmap_event->file_name,
5779 				   mmap_event->file_size);
5780 
5781 	perf_event__output_id_sample(event, &handle, &sample);
5782 
5783 	perf_output_end(&handle);
5784 out:
5785 	mmap_event->event_id.header.size = size;
5786 }
5787 
5788 static void perf_event_mmap_event(struct perf_mmap_event *mmap_event)
5789 {
5790 	struct vm_area_struct *vma = mmap_event->vma;
5791 	struct file *file = vma->vm_file;
5792 	int maj = 0, min = 0;
5793 	u64 ino = 0, gen = 0;
5794 	u32 prot = 0, flags = 0;
5795 	unsigned int size;
5796 	char tmp[16];
5797 	char *buf = NULL;
5798 	char *name;
5799 
5800 	if (file) {
5801 		struct inode *inode;
5802 		dev_t dev;
5803 
5804 		buf = kmalloc(PATH_MAX, GFP_KERNEL);
5805 		if (!buf) {
5806 			name = "//enomem";
5807 			goto cpy_name;
5808 		}
5809 		/*
5810 		 * d_path() works from the end of the rb backwards, so we
5811 		 * need to add enough zero bytes after the string to handle
5812 		 * the 64bit alignment we do later.
5813 		 */
5814 		name = d_path(&file->f_path, buf, PATH_MAX - sizeof(u64));
5815 		if (IS_ERR(name)) {
5816 			name = "//toolong";
5817 			goto cpy_name;
5818 		}
5819 		inode = file_inode(vma->vm_file);
5820 		dev = inode->i_sb->s_dev;
5821 		ino = inode->i_ino;
5822 		gen = inode->i_generation;
5823 		maj = MAJOR(dev);
5824 		min = MINOR(dev);
5825 
5826 		if (vma->vm_flags & VM_READ)
5827 			prot |= PROT_READ;
5828 		if (vma->vm_flags & VM_WRITE)
5829 			prot |= PROT_WRITE;
5830 		if (vma->vm_flags & VM_EXEC)
5831 			prot |= PROT_EXEC;
5832 
5833 		if (vma->vm_flags & VM_MAYSHARE)
5834 			flags = MAP_SHARED;
5835 		else
5836 			flags = MAP_PRIVATE;
5837 
5838 		if (vma->vm_flags & VM_DENYWRITE)
5839 			flags |= MAP_DENYWRITE;
5840 		if (vma->vm_flags & VM_MAYEXEC)
5841 			flags |= MAP_EXECUTABLE;
5842 		if (vma->vm_flags & VM_LOCKED)
5843 			flags |= MAP_LOCKED;
5844 		if (vma->vm_flags & VM_HUGETLB)
5845 			flags |= MAP_HUGETLB;
5846 
5847 		goto got_name;
5848 	} else {
5849 		if (vma->vm_ops && vma->vm_ops->name) {
5850 			name = (char *) vma->vm_ops->name(vma);
5851 			if (name)
5852 				goto cpy_name;
5853 		}
5854 
5855 		name = (char *)arch_vma_name(vma);
5856 		if (name)
5857 			goto cpy_name;
5858 
5859 		if (vma->vm_start <= vma->vm_mm->start_brk &&
5860 				vma->vm_end >= vma->vm_mm->brk) {
5861 			name = "[heap]";
5862 			goto cpy_name;
5863 		}
5864 		if (vma->vm_start <= vma->vm_mm->start_stack &&
5865 				vma->vm_end >= vma->vm_mm->start_stack) {
5866 			name = "[stack]";
5867 			goto cpy_name;
5868 		}
5869 
5870 		name = "//anon";
5871 		goto cpy_name;
5872 	}
5873 
5874 cpy_name:
5875 	strlcpy(tmp, name, sizeof(tmp));
5876 	name = tmp;
5877 got_name:
5878 	/*
5879 	 * Since our buffer works in 8 byte units we need to align our string
5880 	 * size to a multiple of 8. However, we must guarantee the tail end is
5881 	 * zero'd out to avoid leaking random bits to userspace.
5882 	 */
5883 	size = strlen(name)+1;
5884 	while (!IS_ALIGNED(size, sizeof(u64)))
5885 		name[size++] = '\0';
5886 
5887 	mmap_event->file_name = name;
5888 	mmap_event->file_size = size;
5889 	mmap_event->maj = maj;
5890 	mmap_event->min = min;
5891 	mmap_event->ino = ino;
5892 	mmap_event->ino_generation = gen;
5893 	mmap_event->prot = prot;
5894 	mmap_event->flags = flags;
5895 
5896 	if (!(vma->vm_flags & VM_EXEC))
5897 		mmap_event->event_id.header.misc |= PERF_RECORD_MISC_MMAP_DATA;
5898 
5899 	mmap_event->event_id.header.size = sizeof(mmap_event->event_id) + size;
5900 
5901 	perf_event_aux(perf_event_mmap_output,
5902 		       mmap_event,
5903 		       NULL);
5904 
5905 	kfree(buf);
5906 }
5907 
5908 void perf_event_mmap(struct vm_area_struct *vma)
5909 {
5910 	struct perf_mmap_event mmap_event;
5911 
5912 	if (!atomic_read(&nr_mmap_events))
5913 		return;
5914 
5915 	mmap_event = (struct perf_mmap_event){
5916 		.vma	= vma,
5917 		/* .file_name */
5918 		/* .file_size */
5919 		.event_id  = {
5920 			.header = {
5921 				.type = PERF_RECORD_MMAP,
5922 				.misc = PERF_RECORD_MISC_USER,
5923 				/* .size */
5924 			},
5925 			/* .pid */
5926 			/* .tid */
5927 			.start  = vma->vm_start,
5928 			.len    = vma->vm_end - vma->vm_start,
5929 			.pgoff  = (u64)vma->vm_pgoff << PAGE_SHIFT,
5930 		},
5931 		/* .maj (attr_mmap2 only) */
5932 		/* .min (attr_mmap2 only) */
5933 		/* .ino (attr_mmap2 only) */
5934 		/* .ino_generation (attr_mmap2 only) */
5935 		/* .prot (attr_mmap2 only) */
5936 		/* .flags (attr_mmap2 only) */
5937 	};
5938 
5939 	perf_event_mmap_event(&mmap_event);
5940 }
5941 
5942 void perf_event_aux_event(struct perf_event *event, unsigned long head,
5943 			  unsigned long size, u64 flags)
5944 {
5945 	struct perf_output_handle handle;
5946 	struct perf_sample_data sample;
5947 	struct perf_aux_event {
5948 		struct perf_event_header	header;
5949 		u64				offset;
5950 		u64				size;
5951 		u64				flags;
5952 	} rec = {
5953 		.header = {
5954 			.type = PERF_RECORD_AUX,
5955 			.misc = 0,
5956 			.size = sizeof(rec),
5957 		},
5958 		.offset		= head,
5959 		.size		= size,
5960 		.flags		= flags,
5961 	};
5962 	int ret;
5963 
5964 	perf_event_header__init_id(&rec.header, &sample, event);
5965 	ret = perf_output_begin(&handle, event, rec.header.size);
5966 
5967 	if (ret)
5968 		return;
5969 
5970 	perf_output_put(&handle, rec);
5971 	perf_event__output_id_sample(event, &handle, &sample);
5972 
5973 	perf_output_end(&handle);
5974 }
5975 
5976 /*
5977  * IRQ throttle logging
5978  */
5979 
5980 static void perf_log_throttle(struct perf_event *event, int enable)
5981 {
5982 	struct perf_output_handle handle;
5983 	struct perf_sample_data sample;
5984 	int ret;
5985 
5986 	struct {
5987 		struct perf_event_header	header;
5988 		u64				time;
5989 		u64				id;
5990 		u64				stream_id;
5991 	} throttle_event = {
5992 		.header = {
5993 			.type = PERF_RECORD_THROTTLE,
5994 			.misc = 0,
5995 			.size = sizeof(throttle_event),
5996 		},
5997 		.time		= perf_event_clock(event),
5998 		.id		= primary_event_id(event),
5999 		.stream_id	= event->id,
6000 	};
6001 
6002 	if (enable)
6003 		throttle_event.header.type = PERF_RECORD_UNTHROTTLE;
6004 
6005 	perf_event_header__init_id(&throttle_event.header, &sample, event);
6006 
6007 	ret = perf_output_begin(&handle, event,
6008 				throttle_event.header.size);
6009 	if (ret)
6010 		return;
6011 
6012 	perf_output_put(&handle, throttle_event);
6013 	perf_event__output_id_sample(event, &handle, &sample);
6014 	perf_output_end(&handle);
6015 }
6016 
6017 static void perf_log_itrace_start(struct perf_event *event)
6018 {
6019 	struct perf_output_handle handle;
6020 	struct perf_sample_data sample;
6021 	struct perf_aux_event {
6022 		struct perf_event_header        header;
6023 		u32				pid;
6024 		u32				tid;
6025 	} rec;
6026 	int ret;
6027 
6028 	if (event->parent)
6029 		event = event->parent;
6030 
6031 	if (!(event->pmu->capabilities & PERF_PMU_CAP_ITRACE) ||
6032 	    event->hw.itrace_started)
6033 		return;
6034 
6035 	event->hw.itrace_started = 1;
6036 
6037 	rec.header.type	= PERF_RECORD_ITRACE_START;
6038 	rec.header.misc	= 0;
6039 	rec.header.size	= sizeof(rec);
6040 	rec.pid	= perf_event_pid(event, current);
6041 	rec.tid	= perf_event_tid(event, current);
6042 
6043 	perf_event_header__init_id(&rec.header, &sample, event);
6044 	ret = perf_output_begin(&handle, event, rec.header.size);
6045 
6046 	if (ret)
6047 		return;
6048 
6049 	perf_output_put(&handle, rec);
6050 	perf_event__output_id_sample(event, &handle, &sample);
6051 
6052 	perf_output_end(&handle);
6053 }
6054 
6055 /*
6056  * Generic event overflow handling, sampling.
6057  */
6058 
6059 static int __perf_event_overflow(struct perf_event *event,
6060 				   int throttle, struct perf_sample_data *data,
6061 				   struct pt_regs *regs)
6062 {
6063 	int events = atomic_read(&event->event_limit);
6064 	struct hw_perf_event *hwc = &event->hw;
6065 	u64 seq;
6066 	int ret = 0;
6067 
6068 	/*
6069 	 * Non-sampling counters might still use the PMI to fold short
6070 	 * hardware counters, ignore those.
6071 	 */
6072 	if (unlikely(!is_sampling_event(event)))
6073 		return 0;
6074 
6075 	seq = __this_cpu_read(perf_throttled_seq);
6076 	if (seq != hwc->interrupts_seq) {
6077 		hwc->interrupts_seq = seq;
6078 		hwc->interrupts = 1;
6079 	} else {
6080 		hwc->interrupts++;
6081 		if (unlikely(throttle
6082 			     && hwc->interrupts >= max_samples_per_tick)) {
6083 			__this_cpu_inc(perf_throttled_count);
6084 			hwc->interrupts = MAX_INTERRUPTS;
6085 			perf_log_throttle(event, 0);
6086 			tick_nohz_full_kick();
6087 			ret = 1;
6088 		}
6089 	}
6090 
6091 	if (event->attr.freq) {
6092 		u64 now = perf_clock();
6093 		s64 delta = now - hwc->freq_time_stamp;
6094 
6095 		hwc->freq_time_stamp = now;
6096 
6097 		if (delta > 0 && delta < 2*TICK_NSEC)
6098 			perf_adjust_period(event, delta, hwc->last_period, true);
6099 	}
6100 
6101 	/*
6102 	 * XXX event_limit might not quite work as expected on inherited
6103 	 * events
6104 	 */
6105 
6106 	event->pending_kill = POLL_IN;
6107 	if (events && atomic_dec_and_test(&event->event_limit)) {
6108 		ret = 1;
6109 		event->pending_kill = POLL_HUP;
6110 		event->pending_disable = 1;
6111 		irq_work_queue(&event->pending);
6112 	}
6113 
6114 	if (event->overflow_handler)
6115 		event->overflow_handler(event, data, regs);
6116 	else
6117 		perf_event_output(event, data, regs);
6118 
6119 	if (event->fasync && event->pending_kill) {
6120 		event->pending_wakeup = 1;
6121 		irq_work_queue(&event->pending);
6122 	}
6123 
6124 	return ret;
6125 }
6126 
6127 int perf_event_overflow(struct perf_event *event,
6128 			  struct perf_sample_data *data,
6129 			  struct pt_regs *regs)
6130 {
6131 	return __perf_event_overflow(event, 1, data, regs);
6132 }
6133 
6134 /*
6135  * Generic software event infrastructure
6136  */
6137 
6138 struct swevent_htable {
6139 	struct swevent_hlist		*swevent_hlist;
6140 	struct mutex			hlist_mutex;
6141 	int				hlist_refcount;
6142 
6143 	/* Recursion avoidance in each contexts */
6144 	int				recursion[PERF_NR_CONTEXTS];
6145 
6146 	/* Keeps track of cpu being initialized/exited */
6147 	bool				online;
6148 };
6149 
6150 static DEFINE_PER_CPU(struct swevent_htable, swevent_htable);
6151 
6152 /*
6153  * We directly increment event->count and keep a second value in
6154  * event->hw.period_left to count intervals. This period event
6155  * is kept in the range [-sample_period, 0] so that we can use the
6156  * sign as trigger.
6157  */
6158 
6159 u64 perf_swevent_set_period(struct perf_event *event)
6160 {
6161 	struct hw_perf_event *hwc = &event->hw;
6162 	u64 period = hwc->last_period;
6163 	u64 nr, offset;
6164 	s64 old, val;
6165 
6166 	hwc->last_period = hwc->sample_period;
6167 
6168 again:
6169 	old = val = local64_read(&hwc->period_left);
6170 	if (val < 0)
6171 		return 0;
6172 
6173 	nr = div64_u64(period + val, period);
6174 	offset = nr * period;
6175 	val -= offset;
6176 	if (local64_cmpxchg(&hwc->period_left, old, val) != old)
6177 		goto again;
6178 
6179 	return nr;
6180 }
6181 
6182 static void perf_swevent_overflow(struct perf_event *event, u64 overflow,
6183 				    struct perf_sample_data *data,
6184 				    struct pt_regs *regs)
6185 {
6186 	struct hw_perf_event *hwc = &event->hw;
6187 	int throttle = 0;
6188 
6189 	if (!overflow)
6190 		overflow = perf_swevent_set_period(event);
6191 
6192 	if (hwc->interrupts == MAX_INTERRUPTS)
6193 		return;
6194 
6195 	for (; overflow; overflow--) {
6196 		if (__perf_event_overflow(event, throttle,
6197 					    data, regs)) {
6198 			/*
6199 			 * We inhibit the overflow from happening when
6200 			 * hwc->interrupts == MAX_INTERRUPTS.
6201 			 */
6202 			break;
6203 		}
6204 		throttle = 1;
6205 	}
6206 }
6207 
6208 static void perf_swevent_event(struct perf_event *event, u64 nr,
6209 			       struct perf_sample_data *data,
6210 			       struct pt_regs *regs)
6211 {
6212 	struct hw_perf_event *hwc = &event->hw;
6213 
6214 	local64_add(nr, &event->count);
6215 
6216 	if (!regs)
6217 		return;
6218 
6219 	if (!is_sampling_event(event))
6220 		return;
6221 
6222 	if ((event->attr.sample_type & PERF_SAMPLE_PERIOD) && !event->attr.freq) {
6223 		data->period = nr;
6224 		return perf_swevent_overflow(event, 1, data, regs);
6225 	} else
6226 		data->period = event->hw.last_period;
6227 
6228 	if (nr == 1 && hwc->sample_period == 1 && !event->attr.freq)
6229 		return perf_swevent_overflow(event, 1, data, regs);
6230 
6231 	if (local64_add_negative(nr, &hwc->period_left))
6232 		return;
6233 
6234 	perf_swevent_overflow(event, 0, data, regs);
6235 }
6236 
6237 static int perf_exclude_event(struct perf_event *event,
6238 			      struct pt_regs *regs)
6239 {
6240 	if (event->hw.state & PERF_HES_STOPPED)
6241 		return 1;
6242 
6243 	if (regs) {
6244 		if (event->attr.exclude_user && user_mode(regs))
6245 			return 1;
6246 
6247 		if (event->attr.exclude_kernel && !user_mode(regs))
6248 			return 1;
6249 	}
6250 
6251 	return 0;
6252 }
6253 
6254 static int perf_swevent_match(struct perf_event *event,
6255 				enum perf_type_id type,
6256 				u32 event_id,
6257 				struct perf_sample_data *data,
6258 				struct pt_regs *regs)
6259 {
6260 	if (event->attr.type != type)
6261 		return 0;
6262 
6263 	if (event->attr.config != event_id)
6264 		return 0;
6265 
6266 	if (perf_exclude_event(event, regs))
6267 		return 0;
6268 
6269 	return 1;
6270 }
6271 
6272 static inline u64 swevent_hash(u64 type, u32 event_id)
6273 {
6274 	u64 val = event_id | (type << 32);
6275 
6276 	return hash_64(val, SWEVENT_HLIST_BITS);
6277 }
6278 
6279 static inline struct hlist_head *
6280 __find_swevent_head(struct swevent_hlist *hlist, u64 type, u32 event_id)
6281 {
6282 	u64 hash = swevent_hash(type, event_id);
6283 
6284 	return &hlist->heads[hash];
6285 }
6286 
6287 /* For the read side: events when they trigger */
6288 static inline struct hlist_head *
6289 find_swevent_head_rcu(struct swevent_htable *swhash, u64 type, u32 event_id)
6290 {
6291 	struct swevent_hlist *hlist;
6292 
6293 	hlist = rcu_dereference(swhash->swevent_hlist);
6294 	if (!hlist)
6295 		return NULL;
6296 
6297 	return __find_swevent_head(hlist, type, event_id);
6298 }
6299 
6300 /* For the event head insertion and removal in the hlist */
6301 static inline struct hlist_head *
6302 find_swevent_head(struct swevent_htable *swhash, struct perf_event *event)
6303 {
6304 	struct swevent_hlist *hlist;
6305 	u32 event_id = event->attr.config;
6306 	u64 type = event->attr.type;
6307 
6308 	/*
6309 	 * Event scheduling is always serialized against hlist allocation
6310 	 * and release. Which makes the protected version suitable here.
6311 	 * The context lock guarantees that.
6312 	 */
6313 	hlist = rcu_dereference_protected(swhash->swevent_hlist,
6314 					  lockdep_is_held(&event->ctx->lock));
6315 	if (!hlist)
6316 		return NULL;
6317 
6318 	return __find_swevent_head(hlist, type, event_id);
6319 }
6320 
6321 static void do_perf_sw_event(enum perf_type_id type, u32 event_id,
6322 				    u64 nr,
6323 				    struct perf_sample_data *data,
6324 				    struct pt_regs *regs)
6325 {
6326 	struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable);
6327 	struct perf_event *event;
6328 	struct hlist_head *head;
6329 
6330 	rcu_read_lock();
6331 	head = find_swevent_head_rcu(swhash, type, event_id);
6332 	if (!head)
6333 		goto end;
6334 
6335 	hlist_for_each_entry_rcu(event, head, hlist_entry) {
6336 		if (perf_swevent_match(event, type, event_id, data, regs))
6337 			perf_swevent_event(event, nr, data, regs);
6338 	}
6339 end:
6340 	rcu_read_unlock();
6341 }
6342 
6343 DEFINE_PER_CPU(struct pt_regs, __perf_regs[4]);
6344 
6345 int perf_swevent_get_recursion_context(void)
6346 {
6347 	struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable);
6348 
6349 	return get_recursion_context(swhash->recursion);
6350 }
6351 EXPORT_SYMBOL_GPL(perf_swevent_get_recursion_context);
6352 
6353 inline void perf_swevent_put_recursion_context(int rctx)
6354 {
6355 	struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable);
6356 
6357 	put_recursion_context(swhash->recursion, rctx);
6358 }
6359 
6360 void ___perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr)
6361 {
6362 	struct perf_sample_data data;
6363 
6364 	if (WARN_ON_ONCE(!regs))
6365 		return;
6366 
6367 	perf_sample_data_init(&data, addr, 0);
6368 	do_perf_sw_event(PERF_TYPE_SOFTWARE, event_id, nr, &data, regs);
6369 }
6370 
6371 void __perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr)
6372 {
6373 	int rctx;
6374 
6375 	preempt_disable_notrace();
6376 	rctx = perf_swevent_get_recursion_context();
6377 	if (unlikely(rctx < 0))
6378 		goto fail;
6379 
6380 	___perf_sw_event(event_id, nr, regs, addr);
6381 
6382 	perf_swevent_put_recursion_context(rctx);
6383 fail:
6384 	preempt_enable_notrace();
6385 }
6386 
6387 static void perf_swevent_read(struct perf_event *event)
6388 {
6389 }
6390 
6391 static int perf_swevent_add(struct perf_event *event, int flags)
6392 {
6393 	struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable);
6394 	struct hw_perf_event *hwc = &event->hw;
6395 	struct hlist_head *head;
6396 
6397 	if (is_sampling_event(event)) {
6398 		hwc->last_period = hwc->sample_period;
6399 		perf_swevent_set_period(event);
6400 	}
6401 
6402 	hwc->state = !(flags & PERF_EF_START);
6403 
6404 	head = find_swevent_head(swhash, event);
6405 	if (!head) {
6406 		/*
6407 		 * We can race with cpu hotplug code. Do not
6408 		 * WARN if the cpu just got unplugged.
6409 		 */
6410 		WARN_ON_ONCE(swhash->online);
6411 		return -EINVAL;
6412 	}
6413 
6414 	hlist_add_head_rcu(&event->hlist_entry, head);
6415 	perf_event_update_userpage(event);
6416 
6417 	return 0;
6418 }
6419 
6420 static void perf_swevent_del(struct perf_event *event, int flags)
6421 {
6422 	hlist_del_rcu(&event->hlist_entry);
6423 }
6424 
6425 static void perf_swevent_start(struct perf_event *event, int flags)
6426 {
6427 	event->hw.state = 0;
6428 }
6429 
6430 static void perf_swevent_stop(struct perf_event *event, int flags)
6431 {
6432 	event->hw.state = PERF_HES_STOPPED;
6433 }
6434 
6435 /* Deref the hlist from the update side */
6436 static inline struct swevent_hlist *
6437 swevent_hlist_deref(struct swevent_htable *swhash)
6438 {
6439 	return rcu_dereference_protected(swhash->swevent_hlist,
6440 					 lockdep_is_held(&swhash->hlist_mutex));
6441 }
6442 
6443 static void swevent_hlist_release(struct swevent_htable *swhash)
6444 {
6445 	struct swevent_hlist *hlist = swevent_hlist_deref(swhash);
6446 
6447 	if (!hlist)
6448 		return;
6449 
6450 	RCU_INIT_POINTER(swhash->swevent_hlist, NULL);
6451 	kfree_rcu(hlist, rcu_head);
6452 }
6453 
6454 static void swevent_hlist_put_cpu(struct perf_event *event, int cpu)
6455 {
6456 	struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
6457 
6458 	mutex_lock(&swhash->hlist_mutex);
6459 
6460 	if (!--swhash->hlist_refcount)
6461 		swevent_hlist_release(swhash);
6462 
6463 	mutex_unlock(&swhash->hlist_mutex);
6464 }
6465 
6466 static void swevent_hlist_put(struct perf_event *event)
6467 {
6468 	int cpu;
6469 
6470 	for_each_possible_cpu(cpu)
6471 		swevent_hlist_put_cpu(event, cpu);
6472 }
6473 
6474 static int swevent_hlist_get_cpu(struct perf_event *event, int cpu)
6475 {
6476 	struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
6477 	int err = 0;
6478 
6479 	mutex_lock(&swhash->hlist_mutex);
6480 
6481 	if (!swevent_hlist_deref(swhash) && cpu_online(cpu)) {
6482 		struct swevent_hlist *hlist;
6483 
6484 		hlist = kzalloc(sizeof(*hlist), GFP_KERNEL);
6485 		if (!hlist) {
6486 			err = -ENOMEM;
6487 			goto exit;
6488 		}
6489 		rcu_assign_pointer(swhash->swevent_hlist, hlist);
6490 	}
6491 	swhash->hlist_refcount++;
6492 exit:
6493 	mutex_unlock(&swhash->hlist_mutex);
6494 
6495 	return err;
6496 }
6497 
6498 static int swevent_hlist_get(struct perf_event *event)
6499 {
6500 	int err;
6501 	int cpu, failed_cpu;
6502 
6503 	get_online_cpus();
6504 	for_each_possible_cpu(cpu) {
6505 		err = swevent_hlist_get_cpu(event, cpu);
6506 		if (err) {
6507 			failed_cpu = cpu;
6508 			goto fail;
6509 		}
6510 	}
6511 	put_online_cpus();
6512 
6513 	return 0;
6514 fail:
6515 	for_each_possible_cpu(cpu) {
6516 		if (cpu == failed_cpu)
6517 			break;
6518 		swevent_hlist_put_cpu(event, cpu);
6519 	}
6520 
6521 	put_online_cpus();
6522 	return err;
6523 }
6524 
6525 struct static_key perf_swevent_enabled[PERF_COUNT_SW_MAX];
6526 
6527 static void sw_perf_event_destroy(struct perf_event *event)
6528 {
6529 	u64 event_id = event->attr.config;
6530 
6531 	WARN_ON(event->parent);
6532 
6533 	static_key_slow_dec(&perf_swevent_enabled[event_id]);
6534 	swevent_hlist_put(event);
6535 }
6536 
6537 static int perf_swevent_init(struct perf_event *event)
6538 {
6539 	u64 event_id = event->attr.config;
6540 
6541 	if (event->attr.type != PERF_TYPE_SOFTWARE)
6542 		return -ENOENT;
6543 
6544 	/*
6545 	 * no branch sampling for software events
6546 	 */
6547 	if (has_branch_stack(event))
6548 		return -EOPNOTSUPP;
6549 
6550 	switch (event_id) {
6551 	case PERF_COUNT_SW_CPU_CLOCK:
6552 	case PERF_COUNT_SW_TASK_CLOCK:
6553 		return -ENOENT;
6554 
6555 	default:
6556 		break;
6557 	}
6558 
6559 	if (event_id >= PERF_COUNT_SW_MAX)
6560 		return -ENOENT;
6561 
6562 	if (!event->parent) {
6563 		int err;
6564 
6565 		err = swevent_hlist_get(event);
6566 		if (err)
6567 			return err;
6568 
6569 		static_key_slow_inc(&perf_swevent_enabled[event_id]);
6570 		event->destroy = sw_perf_event_destroy;
6571 	}
6572 
6573 	return 0;
6574 }
6575 
6576 static struct pmu perf_swevent = {
6577 	.task_ctx_nr	= perf_sw_context,
6578 
6579 	.capabilities	= PERF_PMU_CAP_NO_NMI,
6580 
6581 	.event_init	= perf_swevent_init,
6582 	.add		= perf_swevent_add,
6583 	.del		= perf_swevent_del,
6584 	.start		= perf_swevent_start,
6585 	.stop		= perf_swevent_stop,
6586 	.read		= perf_swevent_read,
6587 };
6588 
6589 #ifdef CONFIG_EVENT_TRACING
6590 
6591 static int perf_tp_filter_match(struct perf_event *event,
6592 				struct perf_sample_data *data)
6593 {
6594 	void *record = data->raw->data;
6595 
6596 	if (likely(!event->filter) || filter_match_preds(event->filter, record))
6597 		return 1;
6598 	return 0;
6599 }
6600 
6601 static int perf_tp_event_match(struct perf_event *event,
6602 				struct perf_sample_data *data,
6603 				struct pt_regs *regs)
6604 {
6605 	if (event->hw.state & PERF_HES_STOPPED)
6606 		return 0;
6607 	/*
6608 	 * All tracepoints are from kernel-space.
6609 	 */
6610 	if (event->attr.exclude_kernel)
6611 		return 0;
6612 
6613 	if (!perf_tp_filter_match(event, data))
6614 		return 0;
6615 
6616 	return 1;
6617 }
6618 
6619 void perf_tp_event(u64 addr, u64 count, void *record, int entry_size,
6620 		   struct pt_regs *regs, struct hlist_head *head, int rctx,
6621 		   struct task_struct *task)
6622 {
6623 	struct perf_sample_data data;
6624 	struct perf_event *event;
6625 
6626 	struct perf_raw_record raw = {
6627 		.size = entry_size,
6628 		.data = record,
6629 	};
6630 
6631 	perf_sample_data_init(&data, addr, 0);
6632 	data.raw = &raw;
6633 
6634 	hlist_for_each_entry_rcu(event, head, hlist_entry) {
6635 		if (perf_tp_event_match(event, &data, regs))
6636 			perf_swevent_event(event, count, &data, regs);
6637 	}
6638 
6639 	/*
6640 	 * If we got specified a target task, also iterate its context and
6641 	 * deliver this event there too.
6642 	 */
6643 	if (task && task != current) {
6644 		struct perf_event_context *ctx;
6645 		struct trace_entry *entry = record;
6646 
6647 		rcu_read_lock();
6648 		ctx = rcu_dereference(task->perf_event_ctxp[perf_sw_context]);
6649 		if (!ctx)
6650 			goto unlock;
6651 
6652 		list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
6653 			if (event->attr.type != PERF_TYPE_TRACEPOINT)
6654 				continue;
6655 			if (event->attr.config != entry->type)
6656 				continue;
6657 			if (perf_tp_event_match(event, &data, regs))
6658 				perf_swevent_event(event, count, &data, regs);
6659 		}
6660 unlock:
6661 		rcu_read_unlock();
6662 	}
6663 
6664 	perf_swevent_put_recursion_context(rctx);
6665 }
6666 EXPORT_SYMBOL_GPL(perf_tp_event);
6667 
6668 static void tp_perf_event_destroy(struct perf_event *event)
6669 {
6670 	perf_trace_destroy(event);
6671 }
6672 
6673 static int perf_tp_event_init(struct perf_event *event)
6674 {
6675 	int err;
6676 
6677 	if (event->attr.type != PERF_TYPE_TRACEPOINT)
6678 		return -ENOENT;
6679 
6680 	/*
6681 	 * no branch sampling for tracepoint events
6682 	 */
6683 	if (has_branch_stack(event))
6684 		return -EOPNOTSUPP;
6685 
6686 	err = perf_trace_init(event);
6687 	if (err)
6688 		return err;
6689 
6690 	event->destroy = tp_perf_event_destroy;
6691 
6692 	return 0;
6693 }
6694 
6695 static struct pmu perf_tracepoint = {
6696 	.task_ctx_nr	= perf_sw_context,
6697 
6698 	.event_init	= perf_tp_event_init,
6699 	.add		= perf_trace_add,
6700 	.del		= perf_trace_del,
6701 	.start		= perf_swevent_start,
6702 	.stop		= perf_swevent_stop,
6703 	.read		= perf_swevent_read,
6704 };
6705 
6706 static inline void perf_tp_register(void)
6707 {
6708 	perf_pmu_register(&perf_tracepoint, "tracepoint", PERF_TYPE_TRACEPOINT);
6709 }
6710 
6711 static int perf_event_set_filter(struct perf_event *event, void __user *arg)
6712 {
6713 	char *filter_str;
6714 	int ret;
6715 
6716 	if (event->attr.type != PERF_TYPE_TRACEPOINT)
6717 		return -EINVAL;
6718 
6719 	filter_str = strndup_user(arg, PAGE_SIZE);
6720 	if (IS_ERR(filter_str))
6721 		return PTR_ERR(filter_str);
6722 
6723 	ret = ftrace_profile_set_filter(event, event->attr.config, filter_str);
6724 
6725 	kfree(filter_str);
6726 	return ret;
6727 }
6728 
6729 static void perf_event_free_filter(struct perf_event *event)
6730 {
6731 	ftrace_profile_free_filter(event);
6732 }
6733 
6734 static int perf_event_set_bpf_prog(struct perf_event *event, u32 prog_fd)
6735 {
6736 	struct bpf_prog *prog;
6737 
6738 	if (event->attr.type != PERF_TYPE_TRACEPOINT)
6739 		return -EINVAL;
6740 
6741 	if (event->tp_event->prog)
6742 		return -EEXIST;
6743 
6744 	if (!(event->tp_event->flags & TRACE_EVENT_FL_KPROBE))
6745 		/* bpf programs can only be attached to kprobes */
6746 		return -EINVAL;
6747 
6748 	prog = bpf_prog_get(prog_fd);
6749 	if (IS_ERR(prog))
6750 		return PTR_ERR(prog);
6751 
6752 	if (prog->type != BPF_PROG_TYPE_KPROBE) {
6753 		/* valid fd, but invalid bpf program type */
6754 		bpf_prog_put(prog);
6755 		return -EINVAL;
6756 	}
6757 
6758 	event->tp_event->prog = prog;
6759 
6760 	return 0;
6761 }
6762 
6763 static void perf_event_free_bpf_prog(struct perf_event *event)
6764 {
6765 	struct bpf_prog *prog;
6766 
6767 	if (!event->tp_event)
6768 		return;
6769 
6770 	prog = event->tp_event->prog;
6771 	if (prog) {
6772 		event->tp_event->prog = NULL;
6773 		bpf_prog_put(prog);
6774 	}
6775 }
6776 
6777 #else
6778 
6779 static inline void perf_tp_register(void)
6780 {
6781 }
6782 
6783 static int perf_event_set_filter(struct perf_event *event, void __user *arg)
6784 {
6785 	return -ENOENT;
6786 }
6787 
6788 static void perf_event_free_filter(struct perf_event *event)
6789 {
6790 }
6791 
6792 static int perf_event_set_bpf_prog(struct perf_event *event, u32 prog_fd)
6793 {
6794 	return -ENOENT;
6795 }
6796 
6797 static void perf_event_free_bpf_prog(struct perf_event *event)
6798 {
6799 }
6800 #endif /* CONFIG_EVENT_TRACING */
6801 
6802 #ifdef CONFIG_HAVE_HW_BREAKPOINT
6803 void perf_bp_event(struct perf_event *bp, void *data)
6804 {
6805 	struct perf_sample_data sample;
6806 	struct pt_regs *regs = data;
6807 
6808 	perf_sample_data_init(&sample, bp->attr.bp_addr, 0);
6809 
6810 	if (!bp->hw.state && !perf_exclude_event(bp, regs))
6811 		perf_swevent_event(bp, 1, &sample, regs);
6812 }
6813 #endif
6814 
6815 /*
6816  * hrtimer based swevent callback
6817  */
6818 
6819 static enum hrtimer_restart perf_swevent_hrtimer(struct hrtimer *hrtimer)
6820 {
6821 	enum hrtimer_restart ret = HRTIMER_RESTART;
6822 	struct perf_sample_data data;
6823 	struct pt_regs *regs;
6824 	struct perf_event *event;
6825 	u64 period;
6826 
6827 	event = container_of(hrtimer, struct perf_event, hw.hrtimer);
6828 
6829 	if (event->state != PERF_EVENT_STATE_ACTIVE)
6830 		return HRTIMER_NORESTART;
6831 
6832 	event->pmu->read(event);
6833 
6834 	perf_sample_data_init(&data, 0, event->hw.last_period);
6835 	regs = get_irq_regs();
6836 
6837 	if (regs && !perf_exclude_event(event, regs)) {
6838 		if (!(event->attr.exclude_idle && is_idle_task(current)))
6839 			if (__perf_event_overflow(event, 1, &data, regs))
6840 				ret = HRTIMER_NORESTART;
6841 	}
6842 
6843 	period = max_t(u64, 10000, event->hw.sample_period);
6844 	hrtimer_forward_now(hrtimer, ns_to_ktime(period));
6845 
6846 	return ret;
6847 }
6848 
6849 static void perf_swevent_start_hrtimer(struct perf_event *event)
6850 {
6851 	struct hw_perf_event *hwc = &event->hw;
6852 	s64 period;
6853 
6854 	if (!is_sampling_event(event))
6855 		return;
6856 
6857 	period = local64_read(&hwc->period_left);
6858 	if (period) {
6859 		if (period < 0)
6860 			period = 10000;
6861 
6862 		local64_set(&hwc->period_left, 0);
6863 	} else {
6864 		period = max_t(u64, 10000, hwc->sample_period);
6865 	}
6866 	__hrtimer_start_range_ns(&hwc->hrtimer,
6867 				ns_to_ktime(period), 0,
6868 				HRTIMER_MODE_REL_PINNED, 0);
6869 }
6870 
6871 static void perf_swevent_cancel_hrtimer(struct perf_event *event)
6872 {
6873 	struct hw_perf_event *hwc = &event->hw;
6874 
6875 	if (is_sampling_event(event)) {
6876 		ktime_t remaining = hrtimer_get_remaining(&hwc->hrtimer);
6877 		local64_set(&hwc->period_left, ktime_to_ns(remaining));
6878 
6879 		hrtimer_cancel(&hwc->hrtimer);
6880 	}
6881 }
6882 
6883 static void perf_swevent_init_hrtimer(struct perf_event *event)
6884 {
6885 	struct hw_perf_event *hwc = &event->hw;
6886 
6887 	if (!is_sampling_event(event))
6888 		return;
6889 
6890 	hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
6891 	hwc->hrtimer.function = perf_swevent_hrtimer;
6892 
6893 	/*
6894 	 * Since hrtimers have a fixed rate, we can do a static freq->period
6895 	 * mapping and avoid the whole period adjust feedback stuff.
6896 	 */
6897 	if (event->attr.freq) {
6898 		long freq = event->attr.sample_freq;
6899 
6900 		event->attr.sample_period = NSEC_PER_SEC / freq;
6901 		hwc->sample_period = event->attr.sample_period;
6902 		local64_set(&hwc->period_left, hwc->sample_period);
6903 		hwc->last_period = hwc->sample_period;
6904 		event->attr.freq = 0;
6905 	}
6906 }
6907 
6908 /*
6909  * Software event: cpu wall time clock
6910  */
6911 
6912 static void cpu_clock_event_update(struct perf_event *event)
6913 {
6914 	s64 prev;
6915 	u64 now;
6916 
6917 	now = local_clock();
6918 	prev = local64_xchg(&event->hw.prev_count, now);
6919 	local64_add(now - prev, &event->count);
6920 }
6921 
6922 static void cpu_clock_event_start(struct perf_event *event, int flags)
6923 {
6924 	local64_set(&event->hw.prev_count, local_clock());
6925 	perf_swevent_start_hrtimer(event);
6926 }
6927 
6928 static void cpu_clock_event_stop(struct perf_event *event, int flags)
6929 {
6930 	perf_swevent_cancel_hrtimer(event);
6931 	cpu_clock_event_update(event);
6932 }
6933 
6934 static int cpu_clock_event_add(struct perf_event *event, int flags)
6935 {
6936 	if (flags & PERF_EF_START)
6937 		cpu_clock_event_start(event, flags);
6938 	perf_event_update_userpage(event);
6939 
6940 	return 0;
6941 }
6942 
6943 static void cpu_clock_event_del(struct perf_event *event, int flags)
6944 {
6945 	cpu_clock_event_stop(event, flags);
6946 }
6947 
6948 static void cpu_clock_event_read(struct perf_event *event)
6949 {
6950 	cpu_clock_event_update(event);
6951 }
6952 
6953 static int cpu_clock_event_init(struct perf_event *event)
6954 {
6955 	if (event->attr.type != PERF_TYPE_SOFTWARE)
6956 		return -ENOENT;
6957 
6958 	if (event->attr.config != PERF_COUNT_SW_CPU_CLOCK)
6959 		return -ENOENT;
6960 
6961 	/*
6962 	 * no branch sampling for software events
6963 	 */
6964 	if (has_branch_stack(event))
6965 		return -EOPNOTSUPP;
6966 
6967 	perf_swevent_init_hrtimer(event);
6968 
6969 	return 0;
6970 }
6971 
6972 static struct pmu perf_cpu_clock = {
6973 	.task_ctx_nr	= perf_sw_context,
6974 
6975 	.capabilities	= PERF_PMU_CAP_NO_NMI,
6976 
6977 	.event_init	= cpu_clock_event_init,
6978 	.add		= cpu_clock_event_add,
6979 	.del		= cpu_clock_event_del,
6980 	.start		= cpu_clock_event_start,
6981 	.stop		= cpu_clock_event_stop,
6982 	.read		= cpu_clock_event_read,
6983 };
6984 
6985 /*
6986  * Software event: task time clock
6987  */
6988 
6989 static void task_clock_event_update(struct perf_event *event, u64 now)
6990 {
6991 	u64 prev;
6992 	s64 delta;
6993 
6994 	prev = local64_xchg(&event->hw.prev_count, now);
6995 	delta = now - prev;
6996 	local64_add(delta, &event->count);
6997 }
6998 
6999 static void task_clock_event_start(struct perf_event *event, int flags)
7000 {
7001 	local64_set(&event->hw.prev_count, event->ctx->time);
7002 	perf_swevent_start_hrtimer(event);
7003 }
7004 
7005 static void task_clock_event_stop(struct perf_event *event, int flags)
7006 {
7007 	perf_swevent_cancel_hrtimer(event);
7008 	task_clock_event_update(event, event->ctx->time);
7009 }
7010 
7011 static int task_clock_event_add(struct perf_event *event, int flags)
7012 {
7013 	if (flags & PERF_EF_START)
7014 		task_clock_event_start(event, flags);
7015 	perf_event_update_userpage(event);
7016 
7017 	return 0;
7018 }
7019 
7020 static void task_clock_event_del(struct perf_event *event, int flags)
7021 {
7022 	task_clock_event_stop(event, PERF_EF_UPDATE);
7023 }
7024 
7025 static void task_clock_event_read(struct perf_event *event)
7026 {
7027 	u64 now = perf_clock();
7028 	u64 delta = now - event->ctx->timestamp;
7029 	u64 time = event->ctx->time + delta;
7030 
7031 	task_clock_event_update(event, time);
7032 }
7033 
7034 static int task_clock_event_init(struct perf_event *event)
7035 {
7036 	if (event->attr.type != PERF_TYPE_SOFTWARE)
7037 		return -ENOENT;
7038 
7039 	if (event->attr.config != PERF_COUNT_SW_TASK_CLOCK)
7040 		return -ENOENT;
7041 
7042 	/*
7043 	 * no branch sampling for software events
7044 	 */
7045 	if (has_branch_stack(event))
7046 		return -EOPNOTSUPP;
7047 
7048 	perf_swevent_init_hrtimer(event);
7049 
7050 	return 0;
7051 }
7052 
7053 static struct pmu perf_task_clock = {
7054 	.task_ctx_nr	= perf_sw_context,
7055 
7056 	.capabilities	= PERF_PMU_CAP_NO_NMI,
7057 
7058 	.event_init	= task_clock_event_init,
7059 	.add		= task_clock_event_add,
7060 	.del		= task_clock_event_del,
7061 	.start		= task_clock_event_start,
7062 	.stop		= task_clock_event_stop,
7063 	.read		= task_clock_event_read,
7064 };
7065 
7066 static void perf_pmu_nop_void(struct pmu *pmu)
7067 {
7068 }
7069 
7070 static int perf_pmu_nop_int(struct pmu *pmu)
7071 {
7072 	return 0;
7073 }
7074 
7075 static void perf_pmu_start_txn(struct pmu *pmu)
7076 {
7077 	perf_pmu_disable(pmu);
7078 }
7079 
7080 static int perf_pmu_commit_txn(struct pmu *pmu)
7081 {
7082 	perf_pmu_enable(pmu);
7083 	return 0;
7084 }
7085 
7086 static void perf_pmu_cancel_txn(struct pmu *pmu)
7087 {
7088 	perf_pmu_enable(pmu);
7089 }
7090 
7091 static int perf_event_idx_default(struct perf_event *event)
7092 {
7093 	return 0;
7094 }
7095 
7096 /*
7097  * Ensures all contexts with the same task_ctx_nr have the same
7098  * pmu_cpu_context too.
7099  */
7100 static struct perf_cpu_context __percpu *find_pmu_context(int ctxn)
7101 {
7102 	struct pmu *pmu;
7103 
7104 	if (ctxn < 0)
7105 		return NULL;
7106 
7107 	list_for_each_entry(pmu, &pmus, entry) {
7108 		if (pmu->task_ctx_nr == ctxn)
7109 			return pmu->pmu_cpu_context;
7110 	}
7111 
7112 	return NULL;
7113 }
7114 
7115 static void update_pmu_context(struct pmu *pmu, struct pmu *old_pmu)
7116 {
7117 	int cpu;
7118 
7119 	for_each_possible_cpu(cpu) {
7120 		struct perf_cpu_context *cpuctx;
7121 
7122 		cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
7123 
7124 		if (cpuctx->unique_pmu == old_pmu)
7125 			cpuctx->unique_pmu = pmu;
7126 	}
7127 }
7128 
7129 static void free_pmu_context(struct pmu *pmu)
7130 {
7131 	struct pmu *i;
7132 
7133 	mutex_lock(&pmus_lock);
7134 	/*
7135 	 * Like a real lame refcount.
7136 	 */
7137 	list_for_each_entry(i, &pmus, entry) {
7138 		if (i->pmu_cpu_context == pmu->pmu_cpu_context) {
7139 			update_pmu_context(i, pmu);
7140 			goto out;
7141 		}
7142 	}
7143 
7144 	free_percpu(pmu->pmu_cpu_context);
7145 out:
7146 	mutex_unlock(&pmus_lock);
7147 }
7148 static struct idr pmu_idr;
7149 
7150 static ssize_t
7151 type_show(struct device *dev, struct device_attribute *attr, char *page)
7152 {
7153 	struct pmu *pmu = dev_get_drvdata(dev);
7154 
7155 	return snprintf(page, PAGE_SIZE-1, "%d\n", pmu->type);
7156 }
7157 static DEVICE_ATTR_RO(type);
7158 
7159 static ssize_t
7160 perf_event_mux_interval_ms_show(struct device *dev,
7161 				struct device_attribute *attr,
7162 				char *page)
7163 {
7164 	struct pmu *pmu = dev_get_drvdata(dev);
7165 
7166 	return snprintf(page, PAGE_SIZE-1, "%d\n", pmu->hrtimer_interval_ms);
7167 }
7168 
7169 static ssize_t
7170 perf_event_mux_interval_ms_store(struct device *dev,
7171 				 struct device_attribute *attr,
7172 				 const char *buf, size_t count)
7173 {
7174 	struct pmu *pmu = dev_get_drvdata(dev);
7175 	int timer, cpu, ret;
7176 
7177 	ret = kstrtoint(buf, 0, &timer);
7178 	if (ret)
7179 		return ret;
7180 
7181 	if (timer < 1)
7182 		return -EINVAL;
7183 
7184 	/* same value, noting to do */
7185 	if (timer == pmu->hrtimer_interval_ms)
7186 		return count;
7187 
7188 	pmu->hrtimer_interval_ms = timer;
7189 
7190 	/* update all cpuctx for this PMU */
7191 	for_each_possible_cpu(cpu) {
7192 		struct perf_cpu_context *cpuctx;
7193 		cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
7194 		cpuctx->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * timer);
7195 
7196 		if (hrtimer_active(&cpuctx->hrtimer))
7197 			hrtimer_forward_now(&cpuctx->hrtimer, cpuctx->hrtimer_interval);
7198 	}
7199 
7200 	return count;
7201 }
7202 static DEVICE_ATTR_RW(perf_event_mux_interval_ms);
7203 
7204 static struct attribute *pmu_dev_attrs[] = {
7205 	&dev_attr_type.attr,
7206 	&dev_attr_perf_event_mux_interval_ms.attr,
7207 	NULL,
7208 };
7209 ATTRIBUTE_GROUPS(pmu_dev);
7210 
7211 static int pmu_bus_running;
7212 static struct bus_type pmu_bus = {
7213 	.name		= "event_source",
7214 	.dev_groups	= pmu_dev_groups,
7215 };
7216 
7217 static void pmu_dev_release(struct device *dev)
7218 {
7219 	kfree(dev);
7220 }
7221 
7222 static int pmu_dev_alloc(struct pmu *pmu)
7223 {
7224 	int ret = -ENOMEM;
7225 
7226 	pmu->dev = kzalloc(sizeof(struct device), GFP_KERNEL);
7227 	if (!pmu->dev)
7228 		goto out;
7229 
7230 	pmu->dev->groups = pmu->attr_groups;
7231 	device_initialize(pmu->dev);
7232 	ret = dev_set_name(pmu->dev, "%s", pmu->name);
7233 	if (ret)
7234 		goto free_dev;
7235 
7236 	dev_set_drvdata(pmu->dev, pmu);
7237 	pmu->dev->bus = &pmu_bus;
7238 	pmu->dev->release = pmu_dev_release;
7239 	ret = device_add(pmu->dev);
7240 	if (ret)
7241 		goto free_dev;
7242 
7243 out:
7244 	return ret;
7245 
7246 free_dev:
7247 	put_device(pmu->dev);
7248 	goto out;
7249 }
7250 
7251 static struct lock_class_key cpuctx_mutex;
7252 static struct lock_class_key cpuctx_lock;
7253 
7254 int perf_pmu_register(struct pmu *pmu, const char *name, int type)
7255 {
7256 	int cpu, ret;
7257 
7258 	mutex_lock(&pmus_lock);
7259 	ret = -ENOMEM;
7260 	pmu->pmu_disable_count = alloc_percpu(int);
7261 	if (!pmu->pmu_disable_count)
7262 		goto unlock;
7263 
7264 	pmu->type = -1;
7265 	if (!name)
7266 		goto skip_type;
7267 	pmu->name = name;
7268 
7269 	if (type < 0) {
7270 		type = idr_alloc(&pmu_idr, pmu, PERF_TYPE_MAX, 0, GFP_KERNEL);
7271 		if (type < 0) {
7272 			ret = type;
7273 			goto free_pdc;
7274 		}
7275 	}
7276 	pmu->type = type;
7277 
7278 	if (pmu_bus_running) {
7279 		ret = pmu_dev_alloc(pmu);
7280 		if (ret)
7281 			goto free_idr;
7282 	}
7283 
7284 skip_type:
7285 	pmu->pmu_cpu_context = find_pmu_context(pmu->task_ctx_nr);
7286 	if (pmu->pmu_cpu_context)
7287 		goto got_cpu_context;
7288 
7289 	ret = -ENOMEM;
7290 	pmu->pmu_cpu_context = alloc_percpu(struct perf_cpu_context);
7291 	if (!pmu->pmu_cpu_context)
7292 		goto free_dev;
7293 
7294 	for_each_possible_cpu(cpu) {
7295 		struct perf_cpu_context *cpuctx;
7296 
7297 		cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
7298 		__perf_event_init_context(&cpuctx->ctx);
7299 		lockdep_set_class(&cpuctx->ctx.mutex, &cpuctx_mutex);
7300 		lockdep_set_class(&cpuctx->ctx.lock, &cpuctx_lock);
7301 		cpuctx->ctx.pmu = pmu;
7302 
7303 		__perf_cpu_hrtimer_init(cpuctx, cpu);
7304 
7305 		cpuctx->unique_pmu = pmu;
7306 	}
7307 
7308 got_cpu_context:
7309 	if (!pmu->start_txn) {
7310 		if (pmu->pmu_enable) {
7311 			/*
7312 			 * If we have pmu_enable/pmu_disable calls, install
7313 			 * transaction stubs that use that to try and batch
7314 			 * hardware accesses.
7315 			 */
7316 			pmu->start_txn  = perf_pmu_start_txn;
7317 			pmu->commit_txn = perf_pmu_commit_txn;
7318 			pmu->cancel_txn = perf_pmu_cancel_txn;
7319 		} else {
7320 			pmu->start_txn  = perf_pmu_nop_void;
7321 			pmu->commit_txn = perf_pmu_nop_int;
7322 			pmu->cancel_txn = perf_pmu_nop_void;
7323 		}
7324 	}
7325 
7326 	if (!pmu->pmu_enable) {
7327 		pmu->pmu_enable  = perf_pmu_nop_void;
7328 		pmu->pmu_disable = perf_pmu_nop_void;
7329 	}
7330 
7331 	if (!pmu->event_idx)
7332 		pmu->event_idx = perf_event_idx_default;
7333 
7334 	list_add_rcu(&pmu->entry, &pmus);
7335 	atomic_set(&pmu->exclusive_cnt, 0);
7336 	ret = 0;
7337 unlock:
7338 	mutex_unlock(&pmus_lock);
7339 
7340 	return ret;
7341 
7342 free_dev:
7343 	device_del(pmu->dev);
7344 	put_device(pmu->dev);
7345 
7346 free_idr:
7347 	if (pmu->type >= PERF_TYPE_MAX)
7348 		idr_remove(&pmu_idr, pmu->type);
7349 
7350 free_pdc:
7351 	free_percpu(pmu->pmu_disable_count);
7352 	goto unlock;
7353 }
7354 EXPORT_SYMBOL_GPL(perf_pmu_register);
7355 
7356 void perf_pmu_unregister(struct pmu *pmu)
7357 {
7358 	mutex_lock(&pmus_lock);
7359 	list_del_rcu(&pmu->entry);
7360 	mutex_unlock(&pmus_lock);
7361 
7362 	/*
7363 	 * We dereference the pmu list under both SRCU and regular RCU, so
7364 	 * synchronize against both of those.
7365 	 */
7366 	synchronize_srcu(&pmus_srcu);
7367 	synchronize_rcu();
7368 
7369 	free_percpu(pmu->pmu_disable_count);
7370 	if (pmu->type >= PERF_TYPE_MAX)
7371 		idr_remove(&pmu_idr, pmu->type);
7372 	device_del(pmu->dev);
7373 	put_device(pmu->dev);
7374 	free_pmu_context(pmu);
7375 }
7376 EXPORT_SYMBOL_GPL(perf_pmu_unregister);
7377 
7378 static int perf_try_init_event(struct pmu *pmu, struct perf_event *event)
7379 {
7380 	struct perf_event_context *ctx = NULL;
7381 	int ret;
7382 
7383 	if (!try_module_get(pmu->module))
7384 		return -ENODEV;
7385 
7386 	if (event->group_leader != event) {
7387 		/*
7388 		 * This ctx->mutex can nest when we're called through
7389 		 * inheritance. See the perf_event_ctx_lock_nested() comment.
7390 		 */
7391 		ctx = perf_event_ctx_lock_nested(event->group_leader,
7392 						 SINGLE_DEPTH_NESTING);
7393 		BUG_ON(!ctx);
7394 	}
7395 
7396 	event->pmu = pmu;
7397 	ret = pmu->event_init(event);
7398 
7399 	if (ctx)
7400 		perf_event_ctx_unlock(event->group_leader, ctx);
7401 
7402 	if (ret)
7403 		module_put(pmu->module);
7404 
7405 	return ret;
7406 }
7407 
7408 struct pmu *perf_init_event(struct perf_event *event)
7409 {
7410 	struct pmu *pmu = NULL;
7411 	int idx;
7412 	int ret;
7413 
7414 	idx = srcu_read_lock(&pmus_srcu);
7415 
7416 	rcu_read_lock();
7417 	pmu = idr_find(&pmu_idr, event->attr.type);
7418 	rcu_read_unlock();
7419 	if (pmu) {
7420 		ret = perf_try_init_event(pmu, event);
7421 		if (ret)
7422 			pmu = ERR_PTR(ret);
7423 		goto unlock;
7424 	}
7425 
7426 	list_for_each_entry_rcu(pmu, &pmus, entry) {
7427 		ret = perf_try_init_event(pmu, event);
7428 		if (!ret)
7429 			goto unlock;
7430 
7431 		if (ret != -ENOENT) {
7432 			pmu = ERR_PTR(ret);
7433 			goto unlock;
7434 		}
7435 	}
7436 	pmu = ERR_PTR(-ENOENT);
7437 unlock:
7438 	srcu_read_unlock(&pmus_srcu, idx);
7439 
7440 	return pmu;
7441 }
7442 
7443 static void account_event_cpu(struct perf_event *event, int cpu)
7444 {
7445 	if (event->parent)
7446 		return;
7447 
7448 	if (is_cgroup_event(event))
7449 		atomic_inc(&per_cpu(perf_cgroup_events, cpu));
7450 }
7451 
7452 static void account_event(struct perf_event *event)
7453 {
7454 	if (event->parent)
7455 		return;
7456 
7457 	if (event->attach_state & PERF_ATTACH_TASK)
7458 		static_key_slow_inc(&perf_sched_events.key);
7459 	if (event->attr.mmap || event->attr.mmap_data)
7460 		atomic_inc(&nr_mmap_events);
7461 	if (event->attr.comm)
7462 		atomic_inc(&nr_comm_events);
7463 	if (event->attr.task)
7464 		atomic_inc(&nr_task_events);
7465 	if (event->attr.freq) {
7466 		if (atomic_inc_return(&nr_freq_events) == 1)
7467 			tick_nohz_full_kick_all();
7468 	}
7469 	if (has_branch_stack(event))
7470 		static_key_slow_inc(&perf_sched_events.key);
7471 	if (is_cgroup_event(event))
7472 		static_key_slow_inc(&perf_sched_events.key);
7473 
7474 	account_event_cpu(event, event->cpu);
7475 }
7476 
7477 /*
7478  * Allocate and initialize a event structure
7479  */
7480 static struct perf_event *
7481 perf_event_alloc(struct perf_event_attr *attr, int cpu,
7482 		 struct task_struct *task,
7483 		 struct perf_event *group_leader,
7484 		 struct perf_event *parent_event,
7485 		 perf_overflow_handler_t overflow_handler,
7486 		 void *context, int cgroup_fd)
7487 {
7488 	struct pmu *pmu;
7489 	struct perf_event *event;
7490 	struct hw_perf_event *hwc;
7491 	long err = -EINVAL;
7492 
7493 	if ((unsigned)cpu >= nr_cpu_ids) {
7494 		if (!task || cpu != -1)
7495 			return ERR_PTR(-EINVAL);
7496 	}
7497 
7498 	event = kzalloc(sizeof(*event), GFP_KERNEL);
7499 	if (!event)
7500 		return ERR_PTR(-ENOMEM);
7501 
7502 	/*
7503 	 * Single events are their own group leaders, with an
7504 	 * empty sibling list:
7505 	 */
7506 	if (!group_leader)
7507 		group_leader = event;
7508 
7509 	mutex_init(&event->child_mutex);
7510 	INIT_LIST_HEAD(&event->child_list);
7511 
7512 	INIT_LIST_HEAD(&event->group_entry);
7513 	INIT_LIST_HEAD(&event->event_entry);
7514 	INIT_LIST_HEAD(&event->sibling_list);
7515 	INIT_LIST_HEAD(&event->rb_entry);
7516 	INIT_LIST_HEAD(&event->active_entry);
7517 	INIT_HLIST_NODE(&event->hlist_entry);
7518 
7519 
7520 	init_waitqueue_head(&event->waitq);
7521 	init_irq_work(&event->pending, perf_pending_event);
7522 
7523 	mutex_init(&event->mmap_mutex);
7524 
7525 	atomic_long_set(&event->refcount, 1);
7526 	event->cpu		= cpu;
7527 	event->attr		= *attr;
7528 	event->group_leader	= group_leader;
7529 	event->pmu		= NULL;
7530 	event->oncpu		= -1;
7531 
7532 	event->parent		= parent_event;
7533 
7534 	event->ns		= get_pid_ns(task_active_pid_ns(current));
7535 	event->id		= atomic64_inc_return(&perf_event_id);
7536 
7537 	event->state		= PERF_EVENT_STATE_INACTIVE;
7538 
7539 	if (task) {
7540 		event->attach_state = PERF_ATTACH_TASK;
7541 		/*
7542 		 * XXX pmu::event_init needs to know what task to account to
7543 		 * and we cannot use the ctx information because we need the
7544 		 * pmu before we get a ctx.
7545 		 */
7546 		event->hw.target = task;
7547 	}
7548 
7549 	event->clock = &local_clock;
7550 	if (parent_event)
7551 		event->clock = parent_event->clock;
7552 
7553 	if (!overflow_handler && parent_event) {
7554 		overflow_handler = parent_event->overflow_handler;
7555 		context = parent_event->overflow_handler_context;
7556 	}
7557 
7558 	event->overflow_handler	= overflow_handler;
7559 	event->overflow_handler_context = context;
7560 
7561 	perf_event__state_init(event);
7562 
7563 	pmu = NULL;
7564 
7565 	hwc = &event->hw;
7566 	hwc->sample_period = attr->sample_period;
7567 	if (attr->freq && attr->sample_freq)
7568 		hwc->sample_period = 1;
7569 	hwc->last_period = hwc->sample_period;
7570 
7571 	local64_set(&hwc->period_left, hwc->sample_period);
7572 
7573 	/*
7574 	 * we currently do not support PERF_FORMAT_GROUP on inherited events
7575 	 */
7576 	if (attr->inherit && (attr->read_format & PERF_FORMAT_GROUP))
7577 		goto err_ns;
7578 
7579 	if (!has_branch_stack(event))
7580 		event->attr.branch_sample_type = 0;
7581 
7582 	if (cgroup_fd != -1) {
7583 		err = perf_cgroup_connect(cgroup_fd, event, attr, group_leader);
7584 		if (err)
7585 			goto err_ns;
7586 	}
7587 
7588 	pmu = perf_init_event(event);
7589 	if (!pmu)
7590 		goto err_ns;
7591 	else if (IS_ERR(pmu)) {
7592 		err = PTR_ERR(pmu);
7593 		goto err_ns;
7594 	}
7595 
7596 	err = exclusive_event_init(event);
7597 	if (err)
7598 		goto err_pmu;
7599 
7600 	if (!event->parent) {
7601 		if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) {
7602 			err = get_callchain_buffers();
7603 			if (err)
7604 				goto err_per_task;
7605 		}
7606 	}
7607 
7608 	return event;
7609 
7610 err_per_task:
7611 	exclusive_event_destroy(event);
7612 
7613 err_pmu:
7614 	if (event->destroy)
7615 		event->destroy(event);
7616 	module_put(pmu->module);
7617 err_ns:
7618 	if (is_cgroup_event(event))
7619 		perf_detach_cgroup(event);
7620 	if (event->ns)
7621 		put_pid_ns(event->ns);
7622 	kfree(event);
7623 
7624 	return ERR_PTR(err);
7625 }
7626 
7627 static int perf_copy_attr(struct perf_event_attr __user *uattr,
7628 			  struct perf_event_attr *attr)
7629 {
7630 	u32 size;
7631 	int ret;
7632 
7633 	if (!access_ok(VERIFY_WRITE, uattr, PERF_ATTR_SIZE_VER0))
7634 		return -EFAULT;
7635 
7636 	/*
7637 	 * zero the full structure, so that a short copy will be nice.
7638 	 */
7639 	memset(attr, 0, sizeof(*attr));
7640 
7641 	ret = get_user(size, &uattr->size);
7642 	if (ret)
7643 		return ret;
7644 
7645 	if (size > PAGE_SIZE)	/* silly large */
7646 		goto err_size;
7647 
7648 	if (!size)		/* abi compat */
7649 		size = PERF_ATTR_SIZE_VER0;
7650 
7651 	if (size < PERF_ATTR_SIZE_VER0)
7652 		goto err_size;
7653 
7654 	/*
7655 	 * If we're handed a bigger struct than we know of,
7656 	 * ensure all the unknown bits are 0 - i.e. new
7657 	 * user-space does not rely on any kernel feature
7658 	 * extensions we dont know about yet.
7659 	 */
7660 	if (size > sizeof(*attr)) {
7661 		unsigned char __user *addr;
7662 		unsigned char __user *end;
7663 		unsigned char val;
7664 
7665 		addr = (void __user *)uattr + sizeof(*attr);
7666 		end  = (void __user *)uattr + size;
7667 
7668 		for (; addr < end; addr++) {
7669 			ret = get_user(val, addr);
7670 			if (ret)
7671 				return ret;
7672 			if (val)
7673 				goto err_size;
7674 		}
7675 		size = sizeof(*attr);
7676 	}
7677 
7678 	ret = copy_from_user(attr, uattr, size);
7679 	if (ret)
7680 		return -EFAULT;
7681 
7682 	if (attr->__reserved_1)
7683 		return -EINVAL;
7684 
7685 	if (attr->sample_type & ~(PERF_SAMPLE_MAX-1))
7686 		return -EINVAL;
7687 
7688 	if (attr->read_format & ~(PERF_FORMAT_MAX-1))
7689 		return -EINVAL;
7690 
7691 	if (attr->sample_type & PERF_SAMPLE_BRANCH_STACK) {
7692 		u64 mask = attr->branch_sample_type;
7693 
7694 		/* only using defined bits */
7695 		if (mask & ~(PERF_SAMPLE_BRANCH_MAX-1))
7696 			return -EINVAL;
7697 
7698 		/* at least one branch bit must be set */
7699 		if (!(mask & ~PERF_SAMPLE_BRANCH_PLM_ALL))
7700 			return -EINVAL;
7701 
7702 		/* propagate priv level, when not set for branch */
7703 		if (!(mask & PERF_SAMPLE_BRANCH_PLM_ALL)) {
7704 
7705 			/* exclude_kernel checked on syscall entry */
7706 			if (!attr->exclude_kernel)
7707 				mask |= PERF_SAMPLE_BRANCH_KERNEL;
7708 
7709 			if (!attr->exclude_user)
7710 				mask |= PERF_SAMPLE_BRANCH_USER;
7711 
7712 			if (!attr->exclude_hv)
7713 				mask |= PERF_SAMPLE_BRANCH_HV;
7714 			/*
7715 			 * adjust user setting (for HW filter setup)
7716 			 */
7717 			attr->branch_sample_type = mask;
7718 		}
7719 		/* privileged levels capture (kernel, hv): check permissions */
7720 		if ((mask & PERF_SAMPLE_BRANCH_PERM_PLM)
7721 		    && perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN))
7722 			return -EACCES;
7723 	}
7724 
7725 	if (attr->sample_type & PERF_SAMPLE_REGS_USER) {
7726 		ret = perf_reg_validate(attr->sample_regs_user);
7727 		if (ret)
7728 			return ret;
7729 	}
7730 
7731 	if (attr->sample_type & PERF_SAMPLE_STACK_USER) {
7732 		if (!arch_perf_have_user_stack_dump())
7733 			return -ENOSYS;
7734 
7735 		/*
7736 		 * We have __u32 type for the size, but so far
7737 		 * we can only use __u16 as maximum due to the
7738 		 * __u16 sample size limit.
7739 		 */
7740 		if (attr->sample_stack_user >= USHRT_MAX)
7741 			ret = -EINVAL;
7742 		else if (!IS_ALIGNED(attr->sample_stack_user, sizeof(u64)))
7743 			ret = -EINVAL;
7744 	}
7745 
7746 	if (attr->sample_type & PERF_SAMPLE_REGS_INTR)
7747 		ret = perf_reg_validate(attr->sample_regs_intr);
7748 out:
7749 	return ret;
7750 
7751 err_size:
7752 	put_user(sizeof(*attr), &uattr->size);
7753 	ret = -E2BIG;
7754 	goto out;
7755 }
7756 
7757 static int
7758 perf_event_set_output(struct perf_event *event, struct perf_event *output_event)
7759 {
7760 	struct ring_buffer *rb = NULL;
7761 	int ret = -EINVAL;
7762 
7763 	if (!output_event)
7764 		goto set;
7765 
7766 	/* don't allow circular references */
7767 	if (event == output_event)
7768 		goto out;
7769 
7770 	/*
7771 	 * Don't allow cross-cpu buffers
7772 	 */
7773 	if (output_event->cpu != event->cpu)
7774 		goto out;
7775 
7776 	/*
7777 	 * If its not a per-cpu rb, it must be the same task.
7778 	 */
7779 	if (output_event->cpu == -1 && output_event->ctx != event->ctx)
7780 		goto out;
7781 
7782 	/*
7783 	 * Mixing clocks in the same buffer is trouble you don't need.
7784 	 */
7785 	if (output_event->clock != event->clock)
7786 		goto out;
7787 
7788 	/*
7789 	 * If both events generate aux data, they must be on the same PMU
7790 	 */
7791 	if (has_aux(event) && has_aux(output_event) &&
7792 	    event->pmu != output_event->pmu)
7793 		goto out;
7794 
7795 set:
7796 	mutex_lock(&event->mmap_mutex);
7797 	/* Can't redirect output if we've got an active mmap() */
7798 	if (atomic_read(&event->mmap_count))
7799 		goto unlock;
7800 
7801 	if (output_event) {
7802 		/* get the rb we want to redirect to */
7803 		rb = ring_buffer_get(output_event);
7804 		if (!rb)
7805 			goto unlock;
7806 	}
7807 
7808 	ring_buffer_attach(event, rb);
7809 
7810 	ret = 0;
7811 unlock:
7812 	mutex_unlock(&event->mmap_mutex);
7813 
7814 out:
7815 	return ret;
7816 }
7817 
7818 static void mutex_lock_double(struct mutex *a, struct mutex *b)
7819 {
7820 	if (b < a)
7821 		swap(a, b);
7822 
7823 	mutex_lock(a);
7824 	mutex_lock_nested(b, SINGLE_DEPTH_NESTING);
7825 }
7826 
7827 static int perf_event_set_clock(struct perf_event *event, clockid_t clk_id)
7828 {
7829 	bool nmi_safe = false;
7830 
7831 	switch (clk_id) {
7832 	case CLOCK_MONOTONIC:
7833 		event->clock = &ktime_get_mono_fast_ns;
7834 		nmi_safe = true;
7835 		break;
7836 
7837 	case CLOCK_MONOTONIC_RAW:
7838 		event->clock = &ktime_get_raw_fast_ns;
7839 		nmi_safe = true;
7840 		break;
7841 
7842 	case CLOCK_REALTIME:
7843 		event->clock = &ktime_get_real_ns;
7844 		break;
7845 
7846 	case CLOCK_BOOTTIME:
7847 		event->clock = &ktime_get_boot_ns;
7848 		break;
7849 
7850 	case CLOCK_TAI:
7851 		event->clock = &ktime_get_tai_ns;
7852 		break;
7853 
7854 	default:
7855 		return -EINVAL;
7856 	}
7857 
7858 	if (!nmi_safe && !(event->pmu->capabilities & PERF_PMU_CAP_NO_NMI))
7859 		return -EINVAL;
7860 
7861 	return 0;
7862 }
7863 
7864 /**
7865  * sys_perf_event_open - open a performance event, associate it to a task/cpu
7866  *
7867  * @attr_uptr:	event_id type attributes for monitoring/sampling
7868  * @pid:		target pid
7869  * @cpu:		target cpu
7870  * @group_fd:		group leader event fd
7871  */
7872 SYSCALL_DEFINE5(perf_event_open,
7873 		struct perf_event_attr __user *, attr_uptr,
7874 		pid_t, pid, int, cpu, int, group_fd, unsigned long, flags)
7875 {
7876 	struct perf_event *group_leader = NULL, *output_event = NULL;
7877 	struct perf_event *event, *sibling;
7878 	struct perf_event_attr attr;
7879 	struct perf_event_context *ctx, *uninitialized_var(gctx);
7880 	struct file *event_file = NULL;
7881 	struct fd group = {NULL, 0};
7882 	struct task_struct *task = NULL;
7883 	struct pmu *pmu;
7884 	int event_fd;
7885 	int move_group = 0;
7886 	int err;
7887 	int f_flags = O_RDWR;
7888 	int cgroup_fd = -1;
7889 
7890 	/* for future expandability... */
7891 	if (flags & ~PERF_FLAG_ALL)
7892 		return -EINVAL;
7893 
7894 	err = perf_copy_attr(attr_uptr, &attr);
7895 	if (err)
7896 		return err;
7897 
7898 	if (!attr.exclude_kernel) {
7899 		if (perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN))
7900 			return -EACCES;
7901 	}
7902 
7903 	if (attr.freq) {
7904 		if (attr.sample_freq > sysctl_perf_event_sample_rate)
7905 			return -EINVAL;
7906 	} else {
7907 		if (attr.sample_period & (1ULL << 63))
7908 			return -EINVAL;
7909 	}
7910 
7911 	/*
7912 	 * In cgroup mode, the pid argument is used to pass the fd
7913 	 * opened to the cgroup directory in cgroupfs. The cpu argument
7914 	 * designates the cpu on which to monitor threads from that
7915 	 * cgroup.
7916 	 */
7917 	if ((flags & PERF_FLAG_PID_CGROUP) && (pid == -1 || cpu == -1))
7918 		return -EINVAL;
7919 
7920 	if (flags & PERF_FLAG_FD_CLOEXEC)
7921 		f_flags |= O_CLOEXEC;
7922 
7923 	event_fd = get_unused_fd_flags(f_flags);
7924 	if (event_fd < 0)
7925 		return event_fd;
7926 
7927 	if (group_fd != -1) {
7928 		err = perf_fget_light(group_fd, &group);
7929 		if (err)
7930 			goto err_fd;
7931 		group_leader = group.file->private_data;
7932 		if (flags & PERF_FLAG_FD_OUTPUT)
7933 			output_event = group_leader;
7934 		if (flags & PERF_FLAG_FD_NO_GROUP)
7935 			group_leader = NULL;
7936 	}
7937 
7938 	if (pid != -1 && !(flags & PERF_FLAG_PID_CGROUP)) {
7939 		task = find_lively_task_by_vpid(pid);
7940 		if (IS_ERR(task)) {
7941 			err = PTR_ERR(task);
7942 			goto err_group_fd;
7943 		}
7944 	}
7945 
7946 	if (task && group_leader &&
7947 	    group_leader->attr.inherit != attr.inherit) {
7948 		err = -EINVAL;
7949 		goto err_task;
7950 	}
7951 
7952 	get_online_cpus();
7953 
7954 	if (flags & PERF_FLAG_PID_CGROUP)
7955 		cgroup_fd = pid;
7956 
7957 	event = perf_event_alloc(&attr, cpu, task, group_leader, NULL,
7958 				 NULL, NULL, cgroup_fd);
7959 	if (IS_ERR(event)) {
7960 		err = PTR_ERR(event);
7961 		goto err_cpus;
7962 	}
7963 
7964 	if (is_sampling_event(event)) {
7965 		if (event->pmu->capabilities & PERF_PMU_CAP_NO_INTERRUPT) {
7966 			err = -ENOTSUPP;
7967 			goto err_alloc;
7968 		}
7969 	}
7970 
7971 	account_event(event);
7972 
7973 	/*
7974 	 * Special case software events and allow them to be part of
7975 	 * any hardware group.
7976 	 */
7977 	pmu = event->pmu;
7978 
7979 	if (attr.use_clockid) {
7980 		err = perf_event_set_clock(event, attr.clockid);
7981 		if (err)
7982 			goto err_alloc;
7983 	}
7984 
7985 	if (group_leader &&
7986 	    (is_software_event(event) != is_software_event(group_leader))) {
7987 		if (is_software_event(event)) {
7988 			/*
7989 			 * If event and group_leader are not both a software
7990 			 * event, and event is, then group leader is not.
7991 			 *
7992 			 * Allow the addition of software events to !software
7993 			 * groups, this is safe because software events never
7994 			 * fail to schedule.
7995 			 */
7996 			pmu = group_leader->pmu;
7997 		} else if (is_software_event(group_leader) &&
7998 			   (group_leader->group_flags & PERF_GROUP_SOFTWARE)) {
7999 			/*
8000 			 * In case the group is a pure software group, and we
8001 			 * try to add a hardware event, move the whole group to
8002 			 * the hardware context.
8003 			 */
8004 			move_group = 1;
8005 		}
8006 	}
8007 
8008 	/*
8009 	 * Get the target context (task or percpu):
8010 	 */
8011 	ctx = find_get_context(pmu, task, event);
8012 	if (IS_ERR(ctx)) {
8013 		err = PTR_ERR(ctx);
8014 		goto err_alloc;
8015 	}
8016 
8017 	if ((pmu->capabilities & PERF_PMU_CAP_EXCLUSIVE) && group_leader) {
8018 		err = -EBUSY;
8019 		goto err_context;
8020 	}
8021 
8022 	if (task) {
8023 		put_task_struct(task);
8024 		task = NULL;
8025 	}
8026 
8027 	/*
8028 	 * Look up the group leader (we will attach this event to it):
8029 	 */
8030 	if (group_leader) {
8031 		err = -EINVAL;
8032 
8033 		/*
8034 		 * Do not allow a recursive hierarchy (this new sibling
8035 		 * becoming part of another group-sibling):
8036 		 */
8037 		if (group_leader->group_leader != group_leader)
8038 			goto err_context;
8039 
8040 		/* All events in a group should have the same clock */
8041 		if (group_leader->clock != event->clock)
8042 			goto err_context;
8043 
8044 		/*
8045 		 * Do not allow to attach to a group in a different
8046 		 * task or CPU context:
8047 		 */
8048 		if (move_group) {
8049 			/*
8050 			 * Make sure we're both on the same task, or both
8051 			 * per-cpu events.
8052 			 */
8053 			if (group_leader->ctx->task != ctx->task)
8054 				goto err_context;
8055 
8056 			/*
8057 			 * Make sure we're both events for the same CPU;
8058 			 * grouping events for different CPUs is broken; since
8059 			 * you can never concurrently schedule them anyhow.
8060 			 */
8061 			if (group_leader->cpu != event->cpu)
8062 				goto err_context;
8063 		} else {
8064 			if (group_leader->ctx != ctx)
8065 				goto err_context;
8066 		}
8067 
8068 		/*
8069 		 * Only a group leader can be exclusive or pinned
8070 		 */
8071 		if (attr.exclusive || attr.pinned)
8072 			goto err_context;
8073 	}
8074 
8075 	if (output_event) {
8076 		err = perf_event_set_output(event, output_event);
8077 		if (err)
8078 			goto err_context;
8079 	}
8080 
8081 	event_file = anon_inode_getfile("[perf_event]", &perf_fops, event,
8082 					f_flags);
8083 	if (IS_ERR(event_file)) {
8084 		err = PTR_ERR(event_file);
8085 		goto err_context;
8086 	}
8087 
8088 	if (move_group) {
8089 		gctx = group_leader->ctx;
8090 
8091 		/*
8092 		 * See perf_event_ctx_lock() for comments on the details
8093 		 * of swizzling perf_event::ctx.
8094 		 */
8095 		mutex_lock_double(&gctx->mutex, &ctx->mutex);
8096 
8097 		perf_remove_from_context(group_leader, false);
8098 
8099 		list_for_each_entry(sibling, &group_leader->sibling_list,
8100 				    group_entry) {
8101 			perf_remove_from_context(sibling, false);
8102 			put_ctx(gctx);
8103 		}
8104 	} else {
8105 		mutex_lock(&ctx->mutex);
8106 	}
8107 
8108 	WARN_ON_ONCE(ctx->parent_ctx);
8109 
8110 	if (move_group) {
8111 		/*
8112 		 * Wait for everybody to stop referencing the events through
8113 		 * the old lists, before installing it on new lists.
8114 		 */
8115 		synchronize_rcu();
8116 
8117 		/*
8118 		 * Install the group siblings before the group leader.
8119 		 *
8120 		 * Because a group leader will try and install the entire group
8121 		 * (through the sibling list, which is still in-tact), we can
8122 		 * end up with siblings installed in the wrong context.
8123 		 *
8124 		 * By installing siblings first we NO-OP because they're not
8125 		 * reachable through the group lists.
8126 		 */
8127 		list_for_each_entry(sibling, &group_leader->sibling_list,
8128 				    group_entry) {
8129 			perf_event__state_init(sibling);
8130 			perf_install_in_context(ctx, sibling, sibling->cpu);
8131 			get_ctx(ctx);
8132 		}
8133 
8134 		/*
8135 		 * Removing from the context ends up with disabled
8136 		 * event. What we want here is event in the initial
8137 		 * startup state, ready to be add into new context.
8138 		 */
8139 		perf_event__state_init(group_leader);
8140 		perf_install_in_context(ctx, group_leader, group_leader->cpu);
8141 		get_ctx(ctx);
8142 	}
8143 
8144 	if (!exclusive_event_installable(event, ctx)) {
8145 		err = -EBUSY;
8146 		mutex_unlock(&ctx->mutex);
8147 		fput(event_file);
8148 		goto err_context;
8149 	}
8150 
8151 	perf_install_in_context(ctx, event, event->cpu);
8152 	perf_unpin_context(ctx);
8153 
8154 	if (move_group) {
8155 		mutex_unlock(&gctx->mutex);
8156 		put_ctx(gctx);
8157 	}
8158 	mutex_unlock(&ctx->mutex);
8159 
8160 	put_online_cpus();
8161 
8162 	event->owner = current;
8163 
8164 	mutex_lock(&current->perf_event_mutex);
8165 	list_add_tail(&event->owner_entry, &current->perf_event_list);
8166 	mutex_unlock(&current->perf_event_mutex);
8167 
8168 	/*
8169 	 * Precalculate sample_data sizes
8170 	 */
8171 	perf_event__header_size(event);
8172 	perf_event__id_header_size(event);
8173 
8174 	/*
8175 	 * Drop the reference on the group_event after placing the
8176 	 * new event on the sibling_list. This ensures destruction
8177 	 * of the group leader will find the pointer to itself in
8178 	 * perf_group_detach().
8179 	 */
8180 	fdput(group);
8181 	fd_install(event_fd, event_file);
8182 	return event_fd;
8183 
8184 err_context:
8185 	perf_unpin_context(ctx);
8186 	put_ctx(ctx);
8187 err_alloc:
8188 	free_event(event);
8189 err_cpus:
8190 	put_online_cpus();
8191 err_task:
8192 	if (task)
8193 		put_task_struct(task);
8194 err_group_fd:
8195 	fdput(group);
8196 err_fd:
8197 	put_unused_fd(event_fd);
8198 	return err;
8199 }
8200 
8201 /**
8202  * perf_event_create_kernel_counter
8203  *
8204  * @attr: attributes of the counter to create
8205  * @cpu: cpu in which the counter is bound
8206  * @task: task to profile (NULL for percpu)
8207  */
8208 struct perf_event *
8209 perf_event_create_kernel_counter(struct perf_event_attr *attr, int cpu,
8210 				 struct task_struct *task,
8211 				 perf_overflow_handler_t overflow_handler,
8212 				 void *context)
8213 {
8214 	struct perf_event_context *ctx;
8215 	struct perf_event *event;
8216 	int err;
8217 
8218 	/*
8219 	 * Get the target context (task or percpu):
8220 	 */
8221 
8222 	event = perf_event_alloc(attr, cpu, task, NULL, NULL,
8223 				 overflow_handler, context, -1);
8224 	if (IS_ERR(event)) {
8225 		err = PTR_ERR(event);
8226 		goto err;
8227 	}
8228 
8229 	/* Mark owner so we could distinguish it from user events. */
8230 	event->owner = EVENT_OWNER_KERNEL;
8231 
8232 	account_event(event);
8233 
8234 	ctx = find_get_context(event->pmu, task, event);
8235 	if (IS_ERR(ctx)) {
8236 		err = PTR_ERR(ctx);
8237 		goto err_free;
8238 	}
8239 
8240 	WARN_ON_ONCE(ctx->parent_ctx);
8241 	mutex_lock(&ctx->mutex);
8242 	if (!exclusive_event_installable(event, ctx)) {
8243 		mutex_unlock(&ctx->mutex);
8244 		perf_unpin_context(ctx);
8245 		put_ctx(ctx);
8246 		err = -EBUSY;
8247 		goto err_free;
8248 	}
8249 
8250 	perf_install_in_context(ctx, event, cpu);
8251 	perf_unpin_context(ctx);
8252 	mutex_unlock(&ctx->mutex);
8253 
8254 	return event;
8255 
8256 err_free:
8257 	free_event(event);
8258 err:
8259 	return ERR_PTR(err);
8260 }
8261 EXPORT_SYMBOL_GPL(perf_event_create_kernel_counter);
8262 
8263 void perf_pmu_migrate_context(struct pmu *pmu, int src_cpu, int dst_cpu)
8264 {
8265 	struct perf_event_context *src_ctx;
8266 	struct perf_event_context *dst_ctx;
8267 	struct perf_event *event, *tmp;
8268 	LIST_HEAD(events);
8269 
8270 	src_ctx = &per_cpu_ptr(pmu->pmu_cpu_context, src_cpu)->ctx;
8271 	dst_ctx = &per_cpu_ptr(pmu->pmu_cpu_context, dst_cpu)->ctx;
8272 
8273 	/*
8274 	 * See perf_event_ctx_lock() for comments on the details
8275 	 * of swizzling perf_event::ctx.
8276 	 */
8277 	mutex_lock_double(&src_ctx->mutex, &dst_ctx->mutex);
8278 	list_for_each_entry_safe(event, tmp, &src_ctx->event_list,
8279 				 event_entry) {
8280 		perf_remove_from_context(event, false);
8281 		unaccount_event_cpu(event, src_cpu);
8282 		put_ctx(src_ctx);
8283 		list_add(&event->migrate_entry, &events);
8284 	}
8285 
8286 	/*
8287 	 * Wait for the events to quiesce before re-instating them.
8288 	 */
8289 	synchronize_rcu();
8290 
8291 	/*
8292 	 * Re-instate events in 2 passes.
8293 	 *
8294 	 * Skip over group leaders and only install siblings on this first
8295 	 * pass, siblings will not get enabled without a leader, however a
8296 	 * leader will enable its siblings, even if those are still on the old
8297 	 * context.
8298 	 */
8299 	list_for_each_entry_safe(event, tmp, &events, migrate_entry) {
8300 		if (event->group_leader == event)
8301 			continue;
8302 
8303 		list_del(&event->migrate_entry);
8304 		if (event->state >= PERF_EVENT_STATE_OFF)
8305 			event->state = PERF_EVENT_STATE_INACTIVE;
8306 		account_event_cpu(event, dst_cpu);
8307 		perf_install_in_context(dst_ctx, event, dst_cpu);
8308 		get_ctx(dst_ctx);
8309 	}
8310 
8311 	/*
8312 	 * Once all the siblings are setup properly, install the group leaders
8313 	 * to make it go.
8314 	 */
8315 	list_for_each_entry_safe(event, tmp, &events, migrate_entry) {
8316 		list_del(&event->migrate_entry);
8317 		if (event->state >= PERF_EVENT_STATE_OFF)
8318 			event->state = PERF_EVENT_STATE_INACTIVE;
8319 		account_event_cpu(event, dst_cpu);
8320 		perf_install_in_context(dst_ctx, event, dst_cpu);
8321 		get_ctx(dst_ctx);
8322 	}
8323 	mutex_unlock(&dst_ctx->mutex);
8324 	mutex_unlock(&src_ctx->mutex);
8325 }
8326 EXPORT_SYMBOL_GPL(perf_pmu_migrate_context);
8327 
8328 static void sync_child_event(struct perf_event *child_event,
8329 			       struct task_struct *child)
8330 {
8331 	struct perf_event *parent_event = child_event->parent;
8332 	u64 child_val;
8333 
8334 	if (child_event->attr.inherit_stat)
8335 		perf_event_read_event(child_event, child);
8336 
8337 	child_val = perf_event_count(child_event);
8338 
8339 	/*
8340 	 * Add back the child's count to the parent's count:
8341 	 */
8342 	atomic64_add(child_val, &parent_event->child_count);
8343 	atomic64_add(child_event->total_time_enabled,
8344 		     &parent_event->child_total_time_enabled);
8345 	atomic64_add(child_event->total_time_running,
8346 		     &parent_event->child_total_time_running);
8347 
8348 	/*
8349 	 * Remove this event from the parent's list
8350 	 */
8351 	WARN_ON_ONCE(parent_event->ctx->parent_ctx);
8352 	mutex_lock(&parent_event->child_mutex);
8353 	list_del_init(&child_event->child_list);
8354 	mutex_unlock(&parent_event->child_mutex);
8355 
8356 	/*
8357 	 * Make sure user/parent get notified, that we just
8358 	 * lost one event.
8359 	 */
8360 	perf_event_wakeup(parent_event);
8361 
8362 	/*
8363 	 * Release the parent event, if this was the last
8364 	 * reference to it.
8365 	 */
8366 	put_event(parent_event);
8367 }
8368 
8369 static void
8370 __perf_event_exit_task(struct perf_event *child_event,
8371 			 struct perf_event_context *child_ctx,
8372 			 struct task_struct *child)
8373 {
8374 	/*
8375 	 * Do not destroy the 'original' grouping; because of the context
8376 	 * switch optimization the original events could've ended up in a
8377 	 * random child task.
8378 	 *
8379 	 * If we were to destroy the original group, all group related
8380 	 * operations would cease to function properly after this random
8381 	 * child dies.
8382 	 *
8383 	 * Do destroy all inherited groups, we don't care about those
8384 	 * and being thorough is better.
8385 	 */
8386 	perf_remove_from_context(child_event, !!child_event->parent);
8387 
8388 	/*
8389 	 * It can happen that the parent exits first, and has events
8390 	 * that are still around due to the child reference. These
8391 	 * events need to be zapped.
8392 	 */
8393 	if (child_event->parent) {
8394 		sync_child_event(child_event, child);
8395 		free_event(child_event);
8396 	} else {
8397 		child_event->state = PERF_EVENT_STATE_EXIT;
8398 		perf_event_wakeup(child_event);
8399 	}
8400 }
8401 
8402 static void perf_event_exit_task_context(struct task_struct *child, int ctxn)
8403 {
8404 	struct perf_event *child_event, *next;
8405 	struct perf_event_context *child_ctx, *clone_ctx = NULL;
8406 	unsigned long flags;
8407 
8408 	if (likely(!child->perf_event_ctxp[ctxn])) {
8409 		perf_event_task(child, NULL, 0);
8410 		return;
8411 	}
8412 
8413 	local_irq_save(flags);
8414 	/*
8415 	 * We can't reschedule here because interrupts are disabled,
8416 	 * and either child is current or it is a task that can't be
8417 	 * scheduled, so we are now safe from rescheduling changing
8418 	 * our context.
8419 	 */
8420 	child_ctx = rcu_dereference_raw(child->perf_event_ctxp[ctxn]);
8421 
8422 	/*
8423 	 * Take the context lock here so that if find_get_context is
8424 	 * reading child->perf_event_ctxp, we wait until it has
8425 	 * incremented the context's refcount before we do put_ctx below.
8426 	 */
8427 	raw_spin_lock(&child_ctx->lock);
8428 	task_ctx_sched_out(child_ctx);
8429 	child->perf_event_ctxp[ctxn] = NULL;
8430 
8431 	/*
8432 	 * If this context is a clone; unclone it so it can't get
8433 	 * swapped to another process while we're removing all
8434 	 * the events from it.
8435 	 */
8436 	clone_ctx = unclone_ctx(child_ctx);
8437 	update_context_time(child_ctx);
8438 	raw_spin_unlock_irqrestore(&child_ctx->lock, flags);
8439 
8440 	if (clone_ctx)
8441 		put_ctx(clone_ctx);
8442 
8443 	/*
8444 	 * Report the task dead after unscheduling the events so that we
8445 	 * won't get any samples after PERF_RECORD_EXIT. We can however still
8446 	 * get a few PERF_RECORD_READ events.
8447 	 */
8448 	perf_event_task(child, child_ctx, 0);
8449 
8450 	/*
8451 	 * We can recurse on the same lock type through:
8452 	 *
8453 	 *   __perf_event_exit_task()
8454 	 *     sync_child_event()
8455 	 *       put_event()
8456 	 *         mutex_lock(&ctx->mutex)
8457 	 *
8458 	 * But since its the parent context it won't be the same instance.
8459 	 */
8460 	mutex_lock(&child_ctx->mutex);
8461 
8462 	list_for_each_entry_safe(child_event, next, &child_ctx->event_list, event_entry)
8463 		__perf_event_exit_task(child_event, child_ctx, child);
8464 
8465 	mutex_unlock(&child_ctx->mutex);
8466 
8467 	put_ctx(child_ctx);
8468 }
8469 
8470 /*
8471  * When a child task exits, feed back event values to parent events.
8472  */
8473 void perf_event_exit_task(struct task_struct *child)
8474 {
8475 	struct perf_event *event, *tmp;
8476 	int ctxn;
8477 
8478 	mutex_lock(&child->perf_event_mutex);
8479 	list_for_each_entry_safe(event, tmp, &child->perf_event_list,
8480 				 owner_entry) {
8481 		list_del_init(&event->owner_entry);
8482 
8483 		/*
8484 		 * Ensure the list deletion is visible before we clear
8485 		 * the owner, closes a race against perf_release() where
8486 		 * we need to serialize on the owner->perf_event_mutex.
8487 		 */
8488 		smp_wmb();
8489 		event->owner = NULL;
8490 	}
8491 	mutex_unlock(&child->perf_event_mutex);
8492 
8493 	for_each_task_context_nr(ctxn)
8494 		perf_event_exit_task_context(child, ctxn);
8495 }
8496 
8497 static void perf_free_event(struct perf_event *event,
8498 			    struct perf_event_context *ctx)
8499 {
8500 	struct perf_event *parent = event->parent;
8501 
8502 	if (WARN_ON_ONCE(!parent))
8503 		return;
8504 
8505 	mutex_lock(&parent->child_mutex);
8506 	list_del_init(&event->child_list);
8507 	mutex_unlock(&parent->child_mutex);
8508 
8509 	put_event(parent);
8510 
8511 	raw_spin_lock_irq(&ctx->lock);
8512 	perf_group_detach(event);
8513 	list_del_event(event, ctx);
8514 	raw_spin_unlock_irq(&ctx->lock);
8515 	free_event(event);
8516 }
8517 
8518 /*
8519  * Free an unexposed, unused context as created by inheritance by
8520  * perf_event_init_task below, used by fork() in case of fail.
8521  *
8522  * Not all locks are strictly required, but take them anyway to be nice and
8523  * help out with the lockdep assertions.
8524  */
8525 void perf_event_free_task(struct task_struct *task)
8526 {
8527 	struct perf_event_context *ctx;
8528 	struct perf_event *event, *tmp;
8529 	int ctxn;
8530 
8531 	for_each_task_context_nr(ctxn) {
8532 		ctx = task->perf_event_ctxp[ctxn];
8533 		if (!ctx)
8534 			continue;
8535 
8536 		mutex_lock(&ctx->mutex);
8537 again:
8538 		list_for_each_entry_safe(event, tmp, &ctx->pinned_groups,
8539 				group_entry)
8540 			perf_free_event(event, ctx);
8541 
8542 		list_for_each_entry_safe(event, tmp, &ctx->flexible_groups,
8543 				group_entry)
8544 			perf_free_event(event, ctx);
8545 
8546 		if (!list_empty(&ctx->pinned_groups) ||
8547 				!list_empty(&ctx->flexible_groups))
8548 			goto again;
8549 
8550 		mutex_unlock(&ctx->mutex);
8551 
8552 		put_ctx(ctx);
8553 	}
8554 }
8555 
8556 void perf_event_delayed_put(struct task_struct *task)
8557 {
8558 	int ctxn;
8559 
8560 	for_each_task_context_nr(ctxn)
8561 		WARN_ON_ONCE(task->perf_event_ctxp[ctxn]);
8562 }
8563 
8564 /*
8565  * inherit a event from parent task to child task:
8566  */
8567 static struct perf_event *
8568 inherit_event(struct perf_event *parent_event,
8569 	      struct task_struct *parent,
8570 	      struct perf_event_context *parent_ctx,
8571 	      struct task_struct *child,
8572 	      struct perf_event *group_leader,
8573 	      struct perf_event_context *child_ctx)
8574 {
8575 	enum perf_event_active_state parent_state = parent_event->state;
8576 	struct perf_event *child_event;
8577 	unsigned long flags;
8578 
8579 	/*
8580 	 * Instead of creating recursive hierarchies of events,
8581 	 * we link inherited events back to the original parent,
8582 	 * which has a filp for sure, which we use as the reference
8583 	 * count:
8584 	 */
8585 	if (parent_event->parent)
8586 		parent_event = parent_event->parent;
8587 
8588 	child_event = perf_event_alloc(&parent_event->attr,
8589 					   parent_event->cpu,
8590 					   child,
8591 					   group_leader, parent_event,
8592 					   NULL, NULL, -1);
8593 	if (IS_ERR(child_event))
8594 		return child_event;
8595 
8596 	if (is_orphaned_event(parent_event) ||
8597 	    !atomic_long_inc_not_zero(&parent_event->refcount)) {
8598 		free_event(child_event);
8599 		return NULL;
8600 	}
8601 
8602 	get_ctx(child_ctx);
8603 
8604 	/*
8605 	 * Make the child state follow the state of the parent event,
8606 	 * not its attr.disabled bit.  We hold the parent's mutex,
8607 	 * so we won't race with perf_event_{en, dis}able_family.
8608 	 */
8609 	if (parent_state >= PERF_EVENT_STATE_INACTIVE)
8610 		child_event->state = PERF_EVENT_STATE_INACTIVE;
8611 	else
8612 		child_event->state = PERF_EVENT_STATE_OFF;
8613 
8614 	if (parent_event->attr.freq) {
8615 		u64 sample_period = parent_event->hw.sample_period;
8616 		struct hw_perf_event *hwc = &child_event->hw;
8617 
8618 		hwc->sample_period = sample_period;
8619 		hwc->last_period   = sample_period;
8620 
8621 		local64_set(&hwc->period_left, sample_period);
8622 	}
8623 
8624 	child_event->ctx = child_ctx;
8625 	child_event->overflow_handler = parent_event->overflow_handler;
8626 	child_event->overflow_handler_context
8627 		= parent_event->overflow_handler_context;
8628 
8629 	/*
8630 	 * Precalculate sample_data sizes
8631 	 */
8632 	perf_event__header_size(child_event);
8633 	perf_event__id_header_size(child_event);
8634 
8635 	/*
8636 	 * Link it up in the child's context:
8637 	 */
8638 	raw_spin_lock_irqsave(&child_ctx->lock, flags);
8639 	add_event_to_ctx(child_event, child_ctx);
8640 	raw_spin_unlock_irqrestore(&child_ctx->lock, flags);
8641 
8642 	/*
8643 	 * Link this into the parent event's child list
8644 	 */
8645 	WARN_ON_ONCE(parent_event->ctx->parent_ctx);
8646 	mutex_lock(&parent_event->child_mutex);
8647 	list_add_tail(&child_event->child_list, &parent_event->child_list);
8648 	mutex_unlock(&parent_event->child_mutex);
8649 
8650 	return child_event;
8651 }
8652 
8653 static int inherit_group(struct perf_event *parent_event,
8654 	      struct task_struct *parent,
8655 	      struct perf_event_context *parent_ctx,
8656 	      struct task_struct *child,
8657 	      struct perf_event_context *child_ctx)
8658 {
8659 	struct perf_event *leader;
8660 	struct perf_event *sub;
8661 	struct perf_event *child_ctr;
8662 
8663 	leader = inherit_event(parent_event, parent, parent_ctx,
8664 				 child, NULL, child_ctx);
8665 	if (IS_ERR(leader))
8666 		return PTR_ERR(leader);
8667 	list_for_each_entry(sub, &parent_event->sibling_list, group_entry) {
8668 		child_ctr = inherit_event(sub, parent, parent_ctx,
8669 					    child, leader, child_ctx);
8670 		if (IS_ERR(child_ctr))
8671 			return PTR_ERR(child_ctr);
8672 	}
8673 	return 0;
8674 }
8675 
8676 static int
8677 inherit_task_group(struct perf_event *event, struct task_struct *parent,
8678 		   struct perf_event_context *parent_ctx,
8679 		   struct task_struct *child, int ctxn,
8680 		   int *inherited_all)
8681 {
8682 	int ret;
8683 	struct perf_event_context *child_ctx;
8684 
8685 	if (!event->attr.inherit) {
8686 		*inherited_all = 0;
8687 		return 0;
8688 	}
8689 
8690 	child_ctx = child->perf_event_ctxp[ctxn];
8691 	if (!child_ctx) {
8692 		/*
8693 		 * This is executed from the parent task context, so
8694 		 * inherit events that have been marked for cloning.
8695 		 * First allocate and initialize a context for the
8696 		 * child.
8697 		 */
8698 
8699 		child_ctx = alloc_perf_context(parent_ctx->pmu, child);
8700 		if (!child_ctx)
8701 			return -ENOMEM;
8702 
8703 		child->perf_event_ctxp[ctxn] = child_ctx;
8704 	}
8705 
8706 	ret = inherit_group(event, parent, parent_ctx,
8707 			    child, child_ctx);
8708 
8709 	if (ret)
8710 		*inherited_all = 0;
8711 
8712 	return ret;
8713 }
8714 
8715 /*
8716  * Initialize the perf_event context in task_struct
8717  */
8718 static int perf_event_init_context(struct task_struct *child, int ctxn)
8719 {
8720 	struct perf_event_context *child_ctx, *parent_ctx;
8721 	struct perf_event_context *cloned_ctx;
8722 	struct perf_event *event;
8723 	struct task_struct *parent = current;
8724 	int inherited_all = 1;
8725 	unsigned long flags;
8726 	int ret = 0;
8727 
8728 	if (likely(!parent->perf_event_ctxp[ctxn]))
8729 		return 0;
8730 
8731 	/*
8732 	 * If the parent's context is a clone, pin it so it won't get
8733 	 * swapped under us.
8734 	 */
8735 	parent_ctx = perf_pin_task_context(parent, ctxn);
8736 	if (!parent_ctx)
8737 		return 0;
8738 
8739 	/*
8740 	 * No need to check if parent_ctx != NULL here; since we saw
8741 	 * it non-NULL earlier, the only reason for it to become NULL
8742 	 * is if we exit, and since we're currently in the middle of
8743 	 * a fork we can't be exiting at the same time.
8744 	 */
8745 
8746 	/*
8747 	 * Lock the parent list. No need to lock the child - not PID
8748 	 * hashed yet and not running, so nobody can access it.
8749 	 */
8750 	mutex_lock(&parent_ctx->mutex);
8751 
8752 	/*
8753 	 * We dont have to disable NMIs - we are only looking at
8754 	 * the list, not manipulating it:
8755 	 */
8756 	list_for_each_entry(event, &parent_ctx->pinned_groups, group_entry) {
8757 		ret = inherit_task_group(event, parent, parent_ctx,
8758 					 child, ctxn, &inherited_all);
8759 		if (ret)
8760 			break;
8761 	}
8762 
8763 	/*
8764 	 * We can't hold ctx->lock when iterating the ->flexible_group list due
8765 	 * to allocations, but we need to prevent rotation because
8766 	 * rotate_ctx() will change the list from interrupt context.
8767 	 */
8768 	raw_spin_lock_irqsave(&parent_ctx->lock, flags);
8769 	parent_ctx->rotate_disable = 1;
8770 	raw_spin_unlock_irqrestore(&parent_ctx->lock, flags);
8771 
8772 	list_for_each_entry(event, &parent_ctx->flexible_groups, group_entry) {
8773 		ret = inherit_task_group(event, parent, parent_ctx,
8774 					 child, ctxn, &inherited_all);
8775 		if (ret)
8776 			break;
8777 	}
8778 
8779 	raw_spin_lock_irqsave(&parent_ctx->lock, flags);
8780 	parent_ctx->rotate_disable = 0;
8781 
8782 	child_ctx = child->perf_event_ctxp[ctxn];
8783 
8784 	if (child_ctx && inherited_all) {
8785 		/*
8786 		 * Mark the child context as a clone of the parent
8787 		 * context, or of whatever the parent is a clone of.
8788 		 *
8789 		 * Note that if the parent is a clone, the holding of
8790 		 * parent_ctx->lock avoids it from being uncloned.
8791 		 */
8792 		cloned_ctx = parent_ctx->parent_ctx;
8793 		if (cloned_ctx) {
8794 			child_ctx->parent_ctx = cloned_ctx;
8795 			child_ctx->parent_gen = parent_ctx->parent_gen;
8796 		} else {
8797 			child_ctx->parent_ctx = parent_ctx;
8798 			child_ctx->parent_gen = parent_ctx->generation;
8799 		}
8800 		get_ctx(child_ctx->parent_ctx);
8801 	}
8802 
8803 	raw_spin_unlock_irqrestore(&parent_ctx->lock, flags);
8804 	mutex_unlock(&parent_ctx->mutex);
8805 
8806 	perf_unpin_context(parent_ctx);
8807 	put_ctx(parent_ctx);
8808 
8809 	return ret;
8810 }
8811 
8812 /*
8813  * Initialize the perf_event context in task_struct
8814  */
8815 int perf_event_init_task(struct task_struct *child)
8816 {
8817 	int ctxn, ret;
8818 
8819 	memset(child->perf_event_ctxp, 0, sizeof(child->perf_event_ctxp));
8820 	mutex_init(&child->perf_event_mutex);
8821 	INIT_LIST_HEAD(&child->perf_event_list);
8822 
8823 	for_each_task_context_nr(ctxn) {
8824 		ret = perf_event_init_context(child, ctxn);
8825 		if (ret) {
8826 			perf_event_free_task(child);
8827 			return ret;
8828 		}
8829 	}
8830 
8831 	return 0;
8832 }
8833 
8834 static void __init perf_event_init_all_cpus(void)
8835 {
8836 	struct swevent_htable *swhash;
8837 	int cpu;
8838 
8839 	for_each_possible_cpu(cpu) {
8840 		swhash = &per_cpu(swevent_htable, cpu);
8841 		mutex_init(&swhash->hlist_mutex);
8842 		INIT_LIST_HEAD(&per_cpu(active_ctx_list, cpu));
8843 	}
8844 }
8845 
8846 static void perf_event_init_cpu(int cpu)
8847 {
8848 	struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
8849 
8850 	mutex_lock(&swhash->hlist_mutex);
8851 	swhash->online = true;
8852 	if (swhash->hlist_refcount > 0) {
8853 		struct swevent_hlist *hlist;
8854 
8855 		hlist = kzalloc_node(sizeof(*hlist), GFP_KERNEL, cpu_to_node(cpu));
8856 		WARN_ON(!hlist);
8857 		rcu_assign_pointer(swhash->swevent_hlist, hlist);
8858 	}
8859 	mutex_unlock(&swhash->hlist_mutex);
8860 }
8861 
8862 #if defined CONFIG_HOTPLUG_CPU || defined CONFIG_KEXEC
8863 static void __perf_event_exit_context(void *__info)
8864 {
8865 	struct remove_event re = { .detach_group = true };
8866 	struct perf_event_context *ctx = __info;
8867 
8868 	rcu_read_lock();
8869 	list_for_each_entry_rcu(re.event, &ctx->event_list, event_entry)
8870 		__perf_remove_from_context(&re);
8871 	rcu_read_unlock();
8872 }
8873 
8874 static void perf_event_exit_cpu_context(int cpu)
8875 {
8876 	struct perf_event_context *ctx;
8877 	struct pmu *pmu;
8878 	int idx;
8879 
8880 	idx = srcu_read_lock(&pmus_srcu);
8881 	list_for_each_entry_rcu(pmu, &pmus, entry) {
8882 		ctx = &per_cpu_ptr(pmu->pmu_cpu_context, cpu)->ctx;
8883 
8884 		mutex_lock(&ctx->mutex);
8885 		smp_call_function_single(cpu, __perf_event_exit_context, ctx, 1);
8886 		mutex_unlock(&ctx->mutex);
8887 	}
8888 	srcu_read_unlock(&pmus_srcu, idx);
8889 }
8890 
8891 static void perf_event_exit_cpu(int cpu)
8892 {
8893 	struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
8894 
8895 	perf_event_exit_cpu_context(cpu);
8896 
8897 	mutex_lock(&swhash->hlist_mutex);
8898 	swhash->online = false;
8899 	swevent_hlist_release(swhash);
8900 	mutex_unlock(&swhash->hlist_mutex);
8901 }
8902 #else
8903 static inline void perf_event_exit_cpu(int cpu) { }
8904 #endif
8905 
8906 static int
8907 perf_reboot(struct notifier_block *notifier, unsigned long val, void *v)
8908 {
8909 	int cpu;
8910 
8911 	for_each_online_cpu(cpu)
8912 		perf_event_exit_cpu(cpu);
8913 
8914 	return NOTIFY_OK;
8915 }
8916 
8917 /*
8918  * Run the perf reboot notifier at the very last possible moment so that
8919  * the generic watchdog code runs as long as possible.
8920  */
8921 static struct notifier_block perf_reboot_notifier = {
8922 	.notifier_call = perf_reboot,
8923 	.priority = INT_MIN,
8924 };
8925 
8926 static int
8927 perf_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
8928 {
8929 	unsigned int cpu = (long)hcpu;
8930 
8931 	switch (action & ~CPU_TASKS_FROZEN) {
8932 
8933 	case CPU_UP_PREPARE:
8934 	case CPU_DOWN_FAILED:
8935 		perf_event_init_cpu(cpu);
8936 		break;
8937 
8938 	case CPU_UP_CANCELED:
8939 	case CPU_DOWN_PREPARE:
8940 		perf_event_exit_cpu(cpu);
8941 		break;
8942 	default:
8943 		break;
8944 	}
8945 
8946 	return NOTIFY_OK;
8947 }
8948 
8949 void __init perf_event_init(void)
8950 {
8951 	int ret;
8952 
8953 	idr_init(&pmu_idr);
8954 
8955 	perf_event_init_all_cpus();
8956 	init_srcu_struct(&pmus_srcu);
8957 	perf_pmu_register(&perf_swevent, "software", PERF_TYPE_SOFTWARE);
8958 	perf_pmu_register(&perf_cpu_clock, NULL, -1);
8959 	perf_pmu_register(&perf_task_clock, NULL, -1);
8960 	perf_tp_register();
8961 	perf_cpu_notifier(perf_cpu_notify);
8962 	register_reboot_notifier(&perf_reboot_notifier);
8963 
8964 	ret = init_hw_breakpoint();
8965 	WARN(ret, "hw_breakpoint initialization failed with: %d", ret);
8966 
8967 	/* do not patch jump label more than once per second */
8968 	jump_label_rate_limit(&perf_sched_events, HZ);
8969 
8970 	/*
8971 	 * Build time assertion that we keep the data_head at the intended
8972 	 * location.  IOW, validation we got the __reserved[] size right.
8973 	 */
8974 	BUILD_BUG_ON((offsetof(struct perf_event_mmap_page, data_head))
8975 		     != 1024);
8976 }
8977 
8978 ssize_t perf_event_sysfs_show(struct device *dev, struct device_attribute *attr,
8979 			      char *page)
8980 {
8981 	struct perf_pmu_events_attr *pmu_attr =
8982 		container_of(attr, struct perf_pmu_events_attr, attr);
8983 
8984 	if (pmu_attr->event_str)
8985 		return sprintf(page, "%s\n", pmu_attr->event_str);
8986 
8987 	return 0;
8988 }
8989 
8990 static int __init perf_event_sysfs_init(void)
8991 {
8992 	struct pmu *pmu;
8993 	int ret;
8994 
8995 	mutex_lock(&pmus_lock);
8996 
8997 	ret = bus_register(&pmu_bus);
8998 	if (ret)
8999 		goto unlock;
9000 
9001 	list_for_each_entry(pmu, &pmus, entry) {
9002 		if (!pmu->name || pmu->type < 0)
9003 			continue;
9004 
9005 		ret = pmu_dev_alloc(pmu);
9006 		WARN(ret, "Failed to register pmu: %s, reason %d\n", pmu->name, ret);
9007 	}
9008 	pmu_bus_running = 1;
9009 	ret = 0;
9010 
9011 unlock:
9012 	mutex_unlock(&pmus_lock);
9013 
9014 	return ret;
9015 }
9016 device_initcall(perf_event_sysfs_init);
9017 
9018 #ifdef CONFIG_CGROUP_PERF
9019 static struct cgroup_subsys_state *
9020 perf_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
9021 {
9022 	struct perf_cgroup *jc;
9023 
9024 	jc = kzalloc(sizeof(*jc), GFP_KERNEL);
9025 	if (!jc)
9026 		return ERR_PTR(-ENOMEM);
9027 
9028 	jc->info = alloc_percpu(struct perf_cgroup_info);
9029 	if (!jc->info) {
9030 		kfree(jc);
9031 		return ERR_PTR(-ENOMEM);
9032 	}
9033 
9034 	return &jc->css;
9035 }
9036 
9037 static void perf_cgroup_css_free(struct cgroup_subsys_state *css)
9038 {
9039 	struct perf_cgroup *jc = container_of(css, struct perf_cgroup, css);
9040 
9041 	free_percpu(jc->info);
9042 	kfree(jc);
9043 }
9044 
9045 static int __perf_cgroup_move(void *info)
9046 {
9047 	struct task_struct *task = info;
9048 	perf_cgroup_switch(task, PERF_CGROUP_SWOUT | PERF_CGROUP_SWIN);
9049 	return 0;
9050 }
9051 
9052 static void perf_cgroup_attach(struct cgroup_subsys_state *css,
9053 			       struct cgroup_taskset *tset)
9054 {
9055 	struct task_struct *task;
9056 
9057 	cgroup_taskset_for_each(task, tset)
9058 		task_function_call(task, __perf_cgroup_move, task);
9059 }
9060 
9061 static void perf_cgroup_exit(struct cgroup_subsys_state *css,
9062 			     struct cgroup_subsys_state *old_css,
9063 			     struct task_struct *task)
9064 {
9065 	/*
9066 	 * cgroup_exit() is called in the copy_process() failure path.
9067 	 * Ignore this case since the task hasn't ran yet, this avoids
9068 	 * trying to poke a half freed task state from generic code.
9069 	 */
9070 	if (!(task->flags & PF_EXITING))
9071 		return;
9072 
9073 	task_function_call(task, __perf_cgroup_move, task);
9074 }
9075 
9076 struct cgroup_subsys perf_event_cgrp_subsys = {
9077 	.css_alloc	= perf_cgroup_css_alloc,
9078 	.css_free	= perf_cgroup_css_free,
9079 	.exit		= perf_cgroup_exit,
9080 	.attach		= perf_cgroup_attach,
9081 };
9082 #endif /* CONFIG_CGROUP_PERF */
9083