xref: /openbmc/linux/kernel/events/core.c (revision 519b58bb)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Performance events core code:
4  *
5  *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
6  *  Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar
7  *  Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra
8  *  Copyright  ©  2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
9  */
10 
11 #include <linux/fs.h>
12 #include <linux/mm.h>
13 #include <linux/cpu.h>
14 #include <linux/smp.h>
15 #include <linux/idr.h>
16 #include <linux/file.h>
17 #include <linux/poll.h>
18 #include <linux/slab.h>
19 #include <linux/hash.h>
20 #include <linux/tick.h>
21 #include <linux/sysfs.h>
22 #include <linux/dcache.h>
23 #include <linux/percpu.h>
24 #include <linux/ptrace.h>
25 #include <linux/reboot.h>
26 #include <linux/vmstat.h>
27 #include <linux/device.h>
28 #include <linux/export.h>
29 #include <linux/vmalloc.h>
30 #include <linux/hardirq.h>
31 #include <linux/hugetlb.h>
32 #include <linux/rculist.h>
33 #include <linux/uaccess.h>
34 #include <linux/syscalls.h>
35 #include <linux/anon_inodes.h>
36 #include <linux/kernel_stat.h>
37 #include <linux/cgroup.h>
38 #include <linux/perf_event.h>
39 #include <linux/trace_events.h>
40 #include <linux/hw_breakpoint.h>
41 #include <linux/mm_types.h>
42 #include <linux/module.h>
43 #include <linux/mman.h>
44 #include <linux/compat.h>
45 #include <linux/bpf.h>
46 #include <linux/filter.h>
47 #include <linux/namei.h>
48 #include <linux/parser.h>
49 #include <linux/sched/clock.h>
50 #include <linux/sched/mm.h>
51 #include <linux/proc_ns.h>
52 #include <linux/mount.h>
53 #include <linux/min_heap.h>
54 #include <linux/highmem.h>
55 #include <linux/pgtable.h>
56 #include <linux/buildid.h>
57 #include <linux/task_work.h>
58 
59 #include "internal.h"
60 
61 #include <asm/irq_regs.h>
62 
63 typedef int (*remote_function_f)(void *);
64 
65 struct remote_function_call {
66 	struct task_struct	*p;
67 	remote_function_f	func;
68 	void			*info;
69 	int			ret;
70 };
71 
72 static void remote_function(void *data)
73 {
74 	struct remote_function_call *tfc = data;
75 	struct task_struct *p = tfc->p;
76 
77 	if (p) {
78 		/* -EAGAIN */
79 		if (task_cpu(p) != smp_processor_id())
80 			return;
81 
82 		/*
83 		 * Now that we're on right CPU with IRQs disabled, we can test
84 		 * if we hit the right task without races.
85 		 */
86 
87 		tfc->ret = -ESRCH; /* No such (running) process */
88 		if (p != current)
89 			return;
90 	}
91 
92 	tfc->ret = tfc->func(tfc->info);
93 }
94 
95 /**
96  * task_function_call - call a function on the cpu on which a task runs
97  * @p:		the task to evaluate
98  * @func:	the function to be called
99  * @info:	the function call argument
100  *
101  * Calls the function @func when the task is currently running. This might
102  * be on the current CPU, which just calls the function directly.  This will
103  * retry due to any failures in smp_call_function_single(), such as if the
104  * task_cpu() goes offline concurrently.
105  *
106  * returns @func return value or -ESRCH or -ENXIO when the process isn't running
107  */
108 static int
109 task_function_call(struct task_struct *p, remote_function_f func, void *info)
110 {
111 	struct remote_function_call data = {
112 		.p	= p,
113 		.func	= func,
114 		.info	= info,
115 		.ret	= -EAGAIN,
116 	};
117 	int ret;
118 
119 	for (;;) {
120 		ret = smp_call_function_single(task_cpu(p), remote_function,
121 					       &data, 1);
122 		if (!ret)
123 			ret = data.ret;
124 
125 		if (ret != -EAGAIN)
126 			break;
127 
128 		cond_resched();
129 	}
130 
131 	return ret;
132 }
133 
134 /**
135  * cpu_function_call - call a function on the cpu
136  * @cpu:	target cpu to queue this function
137  * @func:	the function to be called
138  * @info:	the function call argument
139  *
140  * Calls the function @func on the remote cpu.
141  *
142  * returns: @func return value or -ENXIO when the cpu is offline
143  */
144 static int cpu_function_call(int cpu, remote_function_f func, void *info)
145 {
146 	struct remote_function_call data = {
147 		.p	= NULL,
148 		.func	= func,
149 		.info	= info,
150 		.ret	= -ENXIO, /* No such CPU */
151 	};
152 
153 	smp_call_function_single(cpu, remote_function, &data, 1);
154 
155 	return data.ret;
156 }
157 
158 static inline struct perf_cpu_context *
159 __get_cpu_context(struct perf_event_context *ctx)
160 {
161 	return this_cpu_ptr(ctx->pmu->pmu_cpu_context);
162 }
163 
164 static void perf_ctx_lock(struct perf_cpu_context *cpuctx,
165 			  struct perf_event_context *ctx)
166 {
167 	raw_spin_lock(&cpuctx->ctx.lock);
168 	if (ctx)
169 		raw_spin_lock(&ctx->lock);
170 }
171 
172 static void perf_ctx_unlock(struct perf_cpu_context *cpuctx,
173 			    struct perf_event_context *ctx)
174 {
175 	if (ctx)
176 		raw_spin_unlock(&ctx->lock);
177 	raw_spin_unlock(&cpuctx->ctx.lock);
178 }
179 
180 #define TASK_TOMBSTONE ((void *)-1L)
181 
182 static bool is_kernel_event(struct perf_event *event)
183 {
184 	return READ_ONCE(event->owner) == TASK_TOMBSTONE;
185 }
186 
187 /*
188  * On task ctx scheduling...
189  *
190  * When !ctx->nr_events a task context will not be scheduled. This means
191  * we can disable the scheduler hooks (for performance) without leaving
192  * pending task ctx state.
193  *
194  * This however results in two special cases:
195  *
196  *  - removing the last event from a task ctx; this is relatively straight
197  *    forward and is done in __perf_remove_from_context.
198  *
199  *  - adding the first event to a task ctx; this is tricky because we cannot
200  *    rely on ctx->is_active and therefore cannot use event_function_call().
201  *    See perf_install_in_context().
202  *
203  * If ctx->nr_events, then ctx->is_active and cpuctx->task_ctx are set.
204  */
205 
206 typedef void (*event_f)(struct perf_event *, struct perf_cpu_context *,
207 			struct perf_event_context *, void *);
208 
209 struct event_function_struct {
210 	struct perf_event *event;
211 	event_f func;
212 	void *data;
213 };
214 
215 static int event_function(void *info)
216 {
217 	struct event_function_struct *efs = info;
218 	struct perf_event *event = efs->event;
219 	struct perf_event_context *ctx = event->ctx;
220 	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
221 	struct perf_event_context *task_ctx = cpuctx->task_ctx;
222 	int ret = 0;
223 
224 	lockdep_assert_irqs_disabled();
225 
226 	perf_ctx_lock(cpuctx, task_ctx);
227 	/*
228 	 * Since we do the IPI call without holding ctx->lock things can have
229 	 * changed, double check we hit the task we set out to hit.
230 	 */
231 	if (ctx->task) {
232 		if (ctx->task != current) {
233 			ret = -ESRCH;
234 			goto unlock;
235 		}
236 
237 		/*
238 		 * We only use event_function_call() on established contexts,
239 		 * and event_function() is only ever called when active (or
240 		 * rather, we'll have bailed in task_function_call() or the
241 		 * above ctx->task != current test), therefore we must have
242 		 * ctx->is_active here.
243 		 */
244 		WARN_ON_ONCE(!ctx->is_active);
245 		/*
246 		 * And since we have ctx->is_active, cpuctx->task_ctx must
247 		 * match.
248 		 */
249 		WARN_ON_ONCE(task_ctx != ctx);
250 	} else {
251 		WARN_ON_ONCE(&cpuctx->ctx != ctx);
252 	}
253 
254 	efs->func(event, cpuctx, ctx, efs->data);
255 unlock:
256 	perf_ctx_unlock(cpuctx, task_ctx);
257 
258 	return ret;
259 }
260 
261 static void event_function_call(struct perf_event *event, event_f func, void *data)
262 {
263 	struct perf_event_context *ctx = event->ctx;
264 	struct task_struct *task = READ_ONCE(ctx->task); /* verified in event_function */
265 	struct event_function_struct efs = {
266 		.event = event,
267 		.func = func,
268 		.data = data,
269 	};
270 
271 	if (!event->parent) {
272 		/*
273 		 * If this is a !child event, we must hold ctx::mutex to
274 		 * stabilize the event->ctx relation. See
275 		 * perf_event_ctx_lock().
276 		 */
277 		lockdep_assert_held(&ctx->mutex);
278 	}
279 
280 	if (!task) {
281 		cpu_function_call(event->cpu, event_function, &efs);
282 		return;
283 	}
284 
285 	if (task == TASK_TOMBSTONE)
286 		return;
287 
288 again:
289 	if (!task_function_call(task, event_function, &efs))
290 		return;
291 
292 	raw_spin_lock_irq(&ctx->lock);
293 	/*
294 	 * Reload the task pointer, it might have been changed by
295 	 * a concurrent perf_event_context_sched_out().
296 	 */
297 	task = ctx->task;
298 	if (task == TASK_TOMBSTONE) {
299 		raw_spin_unlock_irq(&ctx->lock);
300 		return;
301 	}
302 	if (ctx->is_active) {
303 		raw_spin_unlock_irq(&ctx->lock);
304 		goto again;
305 	}
306 	func(event, NULL, ctx, data);
307 	raw_spin_unlock_irq(&ctx->lock);
308 }
309 
310 /*
311  * Similar to event_function_call() + event_function(), but hard assumes IRQs
312  * are already disabled and we're on the right CPU.
313  */
314 static void event_function_local(struct perf_event *event, event_f func, void *data)
315 {
316 	struct perf_event_context *ctx = event->ctx;
317 	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
318 	struct task_struct *task = READ_ONCE(ctx->task);
319 	struct perf_event_context *task_ctx = NULL;
320 
321 	lockdep_assert_irqs_disabled();
322 
323 	if (task) {
324 		if (task == TASK_TOMBSTONE)
325 			return;
326 
327 		task_ctx = ctx;
328 	}
329 
330 	perf_ctx_lock(cpuctx, task_ctx);
331 
332 	task = ctx->task;
333 	if (task == TASK_TOMBSTONE)
334 		goto unlock;
335 
336 	if (task) {
337 		/*
338 		 * We must be either inactive or active and the right task,
339 		 * otherwise we're screwed, since we cannot IPI to somewhere
340 		 * else.
341 		 */
342 		if (ctx->is_active) {
343 			if (WARN_ON_ONCE(task != current))
344 				goto unlock;
345 
346 			if (WARN_ON_ONCE(cpuctx->task_ctx != ctx))
347 				goto unlock;
348 		}
349 	} else {
350 		WARN_ON_ONCE(&cpuctx->ctx != ctx);
351 	}
352 
353 	func(event, cpuctx, ctx, data);
354 unlock:
355 	perf_ctx_unlock(cpuctx, task_ctx);
356 }
357 
358 #define PERF_FLAG_ALL (PERF_FLAG_FD_NO_GROUP |\
359 		       PERF_FLAG_FD_OUTPUT  |\
360 		       PERF_FLAG_PID_CGROUP |\
361 		       PERF_FLAG_FD_CLOEXEC)
362 
363 /*
364  * branch priv levels that need permission checks
365  */
366 #define PERF_SAMPLE_BRANCH_PERM_PLM \
367 	(PERF_SAMPLE_BRANCH_KERNEL |\
368 	 PERF_SAMPLE_BRANCH_HV)
369 
370 enum event_type_t {
371 	EVENT_FLEXIBLE = 0x1,
372 	EVENT_PINNED = 0x2,
373 	EVENT_TIME = 0x4,
374 	/* see ctx_resched() for details */
375 	EVENT_CPU = 0x8,
376 	EVENT_ALL = EVENT_FLEXIBLE | EVENT_PINNED,
377 };
378 
379 /*
380  * perf_sched_events : >0 events exist
381  * perf_cgroup_events: >0 per-cpu cgroup events exist on this cpu
382  */
383 
384 static void perf_sched_delayed(struct work_struct *work);
385 DEFINE_STATIC_KEY_FALSE(perf_sched_events);
386 static DECLARE_DELAYED_WORK(perf_sched_work, perf_sched_delayed);
387 static DEFINE_MUTEX(perf_sched_mutex);
388 static atomic_t perf_sched_count;
389 
390 static DEFINE_PER_CPU(atomic_t, perf_cgroup_events);
391 static DEFINE_PER_CPU(int, perf_sched_cb_usages);
392 static DEFINE_PER_CPU(struct pmu_event_list, pmu_sb_events);
393 
394 static atomic_t nr_mmap_events __read_mostly;
395 static atomic_t nr_comm_events __read_mostly;
396 static atomic_t nr_namespaces_events __read_mostly;
397 static atomic_t nr_task_events __read_mostly;
398 static atomic_t nr_freq_events __read_mostly;
399 static atomic_t nr_switch_events __read_mostly;
400 static atomic_t nr_ksymbol_events __read_mostly;
401 static atomic_t nr_bpf_events __read_mostly;
402 static atomic_t nr_cgroup_events __read_mostly;
403 static atomic_t nr_text_poke_events __read_mostly;
404 static atomic_t nr_build_id_events __read_mostly;
405 
406 static LIST_HEAD(pmus);
407 static DEFINE_MUTEX(pmus_lock);
408 static struct srcu_struct pmus_srcu;
409 static cpumask_var_t perf_online_mask;
410 static struct kmem_cache *perf_event_cache;
411 
412 /*
413  * perf event paranoia level:
414  *  -1 - not paranoid at all
415  *   0 - disallow raw tracepoint access for unpriv
416  *   1 - disallow cpu events for unpriv
417  *   2 - disallow kernel profiling for unpriv
418  */
419 int sysctl_perf_event_paranoid __read_mostly = 2;
420 
421 /* Minimum for 512 kiB + 1 user control page */
422 int sysctl_perf_event_mlock __read_mostly = 512 + (PAGE_SIZE / 1024); /* 'free' kiB per user */
423 
424 /*
425  * max perf event sample rate
426  */
427 #define DEFAULT_MAX_SAMPLE_RATE		100000
428 #define DEFAULT_SAMPLE_PERIOD_NS	(NSEC_PER_SEC / DEFAULT_MAX_SAMPLE_RATE)
429 #define DEFAULT_CPU_TIME_MAX_PERCENT	25
430 
431 int sysctl_perf_event_sample_rate __read_mostly	= DEFAULT_MAX_SAMPLE_RATE;
432 
433 static int max_samples_per_tick __read_mostly	= DIV_ROUND_UP(DEFAULT_MAX_SAMPLE_RATE, HZ);
434 static int perf_sample_period_ns __read_mostly	= DEFAULT_SAMPLE_PERIOD_NS;
435 
436 static int perf_sample_allowed_ns __read_mostly =
437 	DEFAULT_SAMPLE_PERIOD_NS * DEFAULT_CPU_TIME_MAX_PERCENT / 100;
438 
439 static void update_perf_cpu_limits(void)
440 {
441 	u64 tmp = perf_sample_period_ns;
442 
443 	tmp *= sysctl_perf_cpu_time_max_percent;
444 	tmp = div_u64(tmp, 100);
445 	if (!tmp)
446 		tmp = 1;
447 
448 	WRITE_ONCE(perf_sample_allowed_ns, tmp);
449 }
450 
451 static bool perf_rotate_context(struct perf_cpu_context *cpuctx);
452 
453 int perf_proc_update_handler(struct ctl_table *table, int write,
454 		void *buffer, size_t *lenp, loff_t *ppos)
455 {
456 	int ret;
457 	int perf_cpu = sysctl_perf_cpu_time_max_percent;
458 	/*
459 	 * If throttling is disabled don't allow the write:
460 	 */
461 	if (write && (perf_cpu == 100 || perf_cpu == 0))
462 		return -EINVAL;
463 
464 	ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
465 	if (ret || !write)
466 		return ret;
467 
468 	max_samples_per_tick = DIV_ROUND_UP(sysctl_perf_event_sample_rate, HZ);
469 	perf_sample_period_ns = NSEC_PER_SEC / sysctl_perf_event_sample_rate;
470 	update_perf_cpu_limits();
471 
472 	return 0;
473 }
474 
475 int sysctl_perf_cpu_time_max_percent __read_mostly = DEFAULT_CPU_TIME_MAX_PERCENT;
476 
477 int perf_cpu_time_max_percent_handler(struct ctl_table *table, int write,
478 		void *buffer, size_t *lenp, loff_t *ppos)
479 {
480 	int ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
481 
482 	if (ret || !write)
483 		return ret;
484 
485 	if (sysctl_perf_cpu_time_max_percent == 100 ||
486 	    sysctl_perf_cpu_time_max_percent == 0) {
487 		printk(KERN_WARNING
488 		       "perf: Dynamic interrupt throttling disabled, can hang your system!\n");
489 		WRITE_ONCE(perf_sample_allowed_ns, 0);
490 	} else {
491 		update_perf_cpu_limits();
492 	}
493 
494 	return 0;
495 }
496 
497 /*
498  * perf samples are done in some very critical code paths (NMIs).
499  * If they take too much CPU time, the system can lock up and not
500  * get any real work done.  This will drop the sample rate when
501  * we detect that events are taking too long.
502  */
503 #define NR_ACCUMULATED_SAMPLES 128
504 static DEFINE_PER_CPU(u64, running_sample_length);
505 
506 static u64 __report_avg;
507 static u64 __report_allowed;
508 
509 static void perf_duration_warn(struct irq_work *w)
510 {
511 	printk_ratelimited(KERN_INFO
512 		"perf: interrupt took too long (%lld > %lld), lowering "
513 		"kernel.perf_event_max_sample_rate to %d\n",
514 		__report_avg, __report_allowed,
515 		sysctl_perf_event_sample_rate);
516 }
517 
518 static DEFINE_IRQ_WORK(perf_duration_work, perf_duration_warn);
519 
520 void perf_sample_event_took(u64 sample_len_ns)
521 {
522 	u64 max_len = READ_ONCE(perf_sample_allowed_ns);
523 	u64 running_len;
524 	u64 avg_len;
525 	u32 max;
526 
527 	if (max_len == 0)
528 		return;
529 
530 	/* Decay the counter by 1 average sample. */
531 	running_len = __this_cpu_read(running_sample_length);
532 	running_len -= running_len/NR_ACCUMULATED_SAMPLES;
533 	running_len += sample_len_ns;
534 	__this_cpu_write(running_sample_length, running_len);
535 
536 	/*
537 	 * Note: this will be biased artifically low until we have
538 	 * seen NR_ACCUMULATED_SAMPLES. Doing it this way keeps us
539 	 * from having to maintain a count.
540 	 */
541 	avg_len = running_len/NR_ACCUMULATED_SAMPLES;
542 	if (avg_len <= max_len)
543 		return;
544 
545 	__report_avg = avg_len;
546 	__report_allowed = max_len;
547 
548 	/*
549 	 * Compute a throttle threshold 25% below the current duration.
550 	 */
551 	avg_len += avg_len / 4;
552 	max = (TICK_NSEC / 100) * sysctl_perf_cpu_time_max_percent;
553 	if (avg_len < max)
554 		max /= (u32)avg_len;
555 	else
556 		max = 1;
557 
558 	WRITE_ONCE(perf_sample_allowed_ns, avg_len);
559 	WRITE_ONCE(max_samples_per_tick, max);
560 
561 	sysctl_perf_event_sample_rate = max * HZ;
562 	perf_sample_period_ns = NSEC_PER_SEC / sysctl_perf_event_sample_rate;
563 
564 	if (!irq_work_queue(&perf_duration_work)) {
565 		early_printk("perf: interrupt took too long (%lld > %lld), lowering "
566 			     "kernel.perf_event_max_sample_rate to %d\n",
567 			     __report_avg, __report_allowed,
568 			     sysctl_perf_event_sample_rate);
569 	}
570 }
571 
572 static atomic64_t perf_event_id;
573 
574 static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,
575 			      enum event_type_t event_type);
576 
577 static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx,
578 			     enum event_type_t event_type);
579 
580 static void update_context_time(struct perf_event_context *ctx);
581 static u64 perf_event_time(struct perf_event *event);
582 
583 void __weak perf_event_print_debug(void)	{ }
584 
585 static inline u64 perf_clock(void)
586 {
587 	return local_clock();
588 }
589 
590 static inline u64 perf_event_clock(struct perf_event *event)
591 {
592 	return event->clock();
593 }
594 
595 /*
596  * State based event timekeeping...
597  *
598  * The basic idea is to use event->state to determine which (if any) time
599  * fields to increment with the current delta. This means we only need to
600  * update timestamps when we change state or when they are explicitly requested
601  * (read).
602  *
603  * Event groups make things a little more complicated, but not terribly so. The
604  * rules for a group are that if the group leader is OFF the entire group is
605  * OFF, irrespecive of what the group member states are. This results in
606  * __perf_effective_state().
607  *
608  * A futher ramification is that when a group leader flips between OFF and
609  * !OFF, we need to update all group member times.
610  *
611  *
612  * NOTE: perf_event_time() is based on the (cgroup) context time, and thus we
613  * need to make sure the relevant context time is updated before we try and
614  * update our timestamps.
615  */
616 
617 static __always_inline enum perf_event_state
618 __perf_effective_state(struct perf_event *event)
619 {
620 	struct perf_event *leader = event->group_leader;
621 
622 	if (leader->state <= PERF_EVENT_STATE_OFF)
623 		return leader->state;
624 
625 	return event->state;
626 }
627 
628 static __always_inline void
629 __perf_update_times(struct perf_event *event, u64 now, u64 *enabled, u64 *running)
630 {
631 	enum perf_event_state state = __perf_effective_state(event);
632 	u64 delta = now - event->tstamp;
633 
634 	*enabled = event->total_time_enabled;
635 	if (state >= PERF_EVENT_STATE_INACTIVE)
636 		*enabled += delta;
637 
638 	*running = event->total_time_running;
639 	if (state >= PERF_EVENT_STATE_ACTIVE)
640 		*running += delta;
641 }
642 
643 static void perf_event_update_time(struct perf_event *event)
644 {
645 	u64 now = perf_event_time(event);
646 
647 	__perf_update_times(event, now, &event->total_time_enabled,
648 					&event->total_time_running);
649 	event->tstamp = now;
650 }
651 
652 static void perf_event_update_sibling_time(struct perf_event *leader)
653 {
654 	struct perf_event *sibling;
655 
656 	for_each_sibling_event(sibling, leader)
657 		perf_event_update_time(sibling);
658 }
659 
660 static void
661 perf_event_set_state(struct perf_event *event, enum perf_event_state state)
662 {
663 	if (event->state == state)
664 		return;
665 
666 	perf_event_update_time(event);
667 	/*
668 	 * If a group leader gets enabled/disabled all its siblings
669 	 * are affected too.
670 	 */
671 	if ((event->state < 0) ^ (state < 0))
672 		perf_event_update_sibling_time(event);
673 
674 	WRITE_ONCE(event->state, state);
675 }
676 
677 /*
678  * UP store-release, load-acquire
679  */
680 
681 #define __store_release(ptr, val)					\
682 do {									\
683 	barrier();							\
684 	WRITE_ONCE(*(ptr), (val));					\
685 } while (0)
686 
687 #define __load_acquire(ptr)						\
688 ({									\
689 	__unqual_scalar_typeof(*(ptr)) ___p = READ_ONCE(*(ptr));	\
690 	barrier();							\
691 	___p;								\
692 })
693 
694 #ifdef CONFIG_CGROUP_PERF
695 
696 static inline bool
697 perf_cgroup_match(struct perf_event *event)
698 {
699 	struct perf_event_context *ctx = event->ctx;
700 	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
701 
702 	/* @event doesn't care about cgroup */
703 	if (!event->cgrp)
704 		return true;
705 
706 	/* wants specific cgroup scope but @cpuctx isn't associated with any */
707 	if (!cpuctx->cgrp)
708 		return false;
709 
710 	/*
711 	 * Cgroup scoping is recursive.  An event enabled for a cgroup is
712 	 * also enabled for all its descendant cgroups.  If @cpuctx's
713 	 * cgroup is a descendant of @event's (the test covers identity
714 	 * case), it's a match.
715 	 */
716 	return cgroup_is_descendant(cpuctx->cgrp->css.cgroup,
717 				    event->cgrp->css.cgroup);
718 }
719 
720 static inline void perf_detach_cgroup(struct perf_event *event)
721 {
722 	css_put(&event->cgrp->css);
723 	event->cgrp = NULL;
724 }
725 
726 static inline int is_cgroup_event(struct perf_event *event)
727 {
728 	return event->cgrp != NULL;
729 }
730 
731 static inline u64 perf_cgroup_event_time(struct perf_event *event)
732 {
733 	struct perf_cgroup_info *t;
734 
735 	t = per_cpu_ptr(event->cgrp->info, event->cpu);
736 	return t->time;
737 }
738 
739 static inline u64 perf_cgroup_event_time_now(struct perf_event *event, u64 now)
740 {
741 	struct perf_cgroup_info *t;
742 
743 	t = per_cpu_ptr(event->cgrp->info, event->cpu);
744 	if (!__load_acquire(&t->active))
745 		return t->time;
746 	now += READ_ONCE(t->timeoffset);
747 	return now;
748 }
749 
750 static inline void __update_cgrp_time(struct perf_cgroup_info *info, u64 now, bool adv)
751 {
752 	if (adv)
753 		info->time += now - info->timestamp;
754 	info->timestamp = now;
755 	/*
756 	 * see update_context_time()
757 	 */
758 	WRITE_ONCE(info->timeoffset, info->time - info->timestamp);
759 }
760 
761 static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx, bool final)
762 {
763 	struct perf_cgroup *cgrp = cpuctx->cgrp;
764 	struct cgroup_subsys_state *css;
765 	struct perf_cgroup_info *info;
766 
767 	if (cgrp) {
768 		u64 now = perf_clock();
769 
770 		for (css = &cgrp->css; css; css = css->parent) {
771 			cgrp = container_of(css, struct perf_cgroup, css);
772 			info = this_cpu_ptr(cgrp->info);
773 
774 			__update_cgrp_time(info, now, true);
775 			if (final)
776 				__store_release(&info->active, 0);
777 		}
778 	}
779 }
780 
781 static inline void update_cgrp_time_from_event(struct perf_event *event)
782 {
783 	struct perf_cgroup_info *info;
784 
785 	/*
786 	 * ensure we access cgroup data only when needed and
787 	 * when we know the cgroup is pinned (css_get)
788 	 */
789 	if (!is_cgroup_event(event))
790 		return;
791 
792 	info = this_cpu_ptr(event->cgrp->info);
793 	/*
794 	 * Do not update time when cgroup is not active
795 	 */
796 	if (info->active)
797 		__update_cgrp_time(info, perf_clock(), true);
798 }
799 
800 static inline void
801 perf_cgroup_set_timestamp(struct perf_cpu_context *cpuctx)
802 {
803 	struct perf_event_context *ctx = &cpuctx->ctx;
804 	struct perf_cgroup *cgrp = cpuctx->cgrp;
805 	struct perf_cgroup_info *info;
806 	struct cgroup_subsys_state *css;
807 
808 	/*
809 	 * ctx->lock held by caller
810 	 * ensure we do not access cgroup data
811 	 * unless we have the cgroup pinned (css_get)
812 	 */
813 	if (!cgrp)
814 		return;
815 
816 	WARN_ON_ONCE(!ctx->nr_cgroups);
817 
818 	for (css = &cgrp->css; css; css = css->parent) {
819 		cgrp = container_of(css, struct perf_cgroup, css);
820 		info = this_cpu_ptr(cgrp->info);
821 		__update_cgrp_time(info, ctx->timestamp, false);
822 		__store_release(&info->active, 1);
823 	}
824 }
825 
826 static DEFINE_PER_CPU(struct list_head, cgrp_cpuctx_list);
827 
828 /*
829  * reschedule events based on the cgroup constraint of task.
830  */
831 static void perf_cgroup_switch(struct task_struct *task)
832 {
833 	struct perf_cgroup *cgrp;
834 	struct perf_cpu_context *cpuctx, *tmp;
835 	struct list_head *list;
836 	unsigned long flags;
837 
838 	/*
839 	 * Disable interrupts and preemption to avoid this CPU's
840 	 * cgrp_cpuctx_entry to change under us.
841 	 */
842 	local_irq_save(flags);
843 
844 	cgrp = perf_cgroup_from_task(task, NULL);
845 
846 	list = this_cpu_ptr(&cgrp_cpuctx_list);
847 	list_for_each_entry_safe(cpuctx, tmp, list, cgrp_cpuctx_entry) {
848 		WARN_ON_ONCE(cpuctx->ctx.nr_cgroups == 0);
849 		if (READ_ONCE(cpuctx->cgrp) == cgrp)
850 			continue;
851 
852 		perf_ctx_lock(cpuctx, cpuctx->task_ctx);
853 		perf_pmu_disable(cpuctx->ctx.pmu);
854 
855 		cpu_ctx_sched_out(cpuctx, EVENT_ALL);
856 		/*
857 		 * must not be done before ctxswout due
858 		 * to update_cgrp_time_from_cpuctx() in
859 		 * ctx_sched_out()
860 		 */
861 		cpuctx->cgrp = cgrp;
862 		/*
863 		 * set cgrp before ctxsw in to allow
864 		 * perf_cgroup_set_timestamp() in ctx_sched_in()
865 		 * to not have to pass task around
866 		 */
867 		cpu_ctx_sched_in(cpuctx, EVENT_ALL);
868 
869 		perf_pmu_enable(cpuctx->ctx.pmu);
870 		perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
871 	}
872 
873 	local_irq_restore(flags);
874 }
875 
876 static int perf_cgroup_ensure_storage(struct perf_event *event,
877 				struct cgroup_subsys_state *css)
878 {
879 	struct perf_cpu_context *cpuctx;
880 	struct perf_event **storage;
881 	int cpu, heap_size, ret = 0;
882 
883 	/*
884 	 * Allow storage to have sufficent space for an iterator for each
885 	 * possibly nested cgroup plus an iterator for events with no cgroup.
886 	 */
887 	for (heap_size = 1; css; css = css->parent)
888 		heap_size++;
889 
890 	for_each_possible_cpu(cpu) {
891 		cpuctx = per_cpu_ptr(event->pmu->pmu_cpu_context, cpu);
892 		if (heap_size <= cpuctx->heap_size)
893 			continue;
894 
895 		storage = kmalloc_node(heap_size * sizeof(struct perf_event *),
896 				       GFP_KERNEL, cpu_to_node(cpu));
897 		if (!storage) {
898 			ret = -ENOMEM;
899 			break;
900 		}
901 
902 		raw_spin_lock_irq(&cpuctx->ctx.lock);
903 		if (cpuctx->heap_size < heap_size) {
904 			swap(cpuctx->heap, storage);
905 			if (storage == cpuctx->heap_default)
906 				storage = NULL;
907 			cpuctx->heap_size = heap_size;
908 		}
909 		raw_spin_unlock_irq(&cpuctx->ctx.lock);
910 
911 		kfree(storage);
912 	}
913 
914 	return ret;
915 }
916 
917 static inline int perf_cgroup_connect(int fd, struct perf_event *event,
918 				      struct perf_event_attr *attr,
919 				      struct perf_event *group_leader)
920 {
921 	struct perf_cgroup *cgrp;
922 	struct cgroup_subsys_state *css;
923 	struct fd f = fdget(fd);
924 	int ret = 0;
925 
926 	if (!f.file)
927 		return -EBADF;
928 
929 	css = css_tryget_online_from_dir(f.file->f_path.dentry,
930 					 &perf_event_cgrp_subsys);
931 	if (IS_ERR(css)) {
932 		ret = PTR_ERR(css);
933 		goto out;
934 	}
935 
936 	ret = perf_cgroup_ensure_storage(event, css);
937 	if (ret)
938 		goto out;
939 
940 	cgrp = container_of(css, struct perf_cgroup, css);
941 	event->cgrp = cgrp;
942 
943 	/*
944 	 * all events in a group must monitor
945 	 * the same cgroup because a task belongs
946 	 * to only one perf cgroup at a time
947 	 */
948 	if (group_leader && group_leader->cgrp != cgrp) {
949 		perf_detach_cgroup(event);
950 		ret = -EINVAL;
951 	}
952 out:
953 	fdput(f);
954 	return ret;
955 }
956 
957 static inline void
958 perf_cgroup_event_enable(struct perf_event *event, struct perf_event_context *ctx)
959 {
960 	struct perf_cpu_context *cpuctx;
961 
962 	if (!is_cgroup_event(event))
963 		return;
964 
965 	/*
966 	 * Because cgroup events are always per-cpu events,
967 	 * @ctx == &cpuctx->ctx.
968 	 */
969 	cpuctx = container_of(ctx, struct perf_cpu_context, ctx);
970 
971 	if (ctx->nr_cgroups++)
972 		return;
973 
974 	cpuctx->cgrp = perf_cgroup_from_task(current, ctx);
975 	list_add(&cpuctx->cgrp_cpuctx_entry,
976 			per_cpu_ptr(&cgrp_cpuctx_list, event->cpu));
977 }
978 
979 static inline void
980 perf_cgroup_event_disable(struct perf_event *event, struct perf_event_context *ctx)
981 {
982 	struct perf_cpu_context *cpuctx;
983 
984 	if (!is_cgroup_event(event))
985 		return;
986 
987 	/*
988 	 * Because cgroup events are always per-cpu events,
989 	 * @ctx == &cpuctx->ctx.
990 	 */
991 	cpuctx = container_of(ctx, struct perf_cpu_context, ctx);
992 
993 	if (--ctx->nr_cgroups)
994 		return;
995 
996 	cpuctx->cgrp = NULL;
997 	list_del(&cpuctx->cgrp_cpuctx_entry);
998 }
999 
1000 #else /* !CONFIG_CGROUP_PERF */
1001 
1002 static inline bool
1003 perf_cgroup_match(struct perf_event *event)
1004 {
1005 	return true;
1006 }
1007 
1008 static inline void perf_detach_cgroup(struct perf_event *event)
1009 {}
1010 
1011 static inline int is_cgroup_event(struct perf_event *event)
1012 {
1013 	return 0;
1014 }
1015 
1016 static inline void update_cgrp_time_from_event(struct perf_event *event)
1017 {
1018 }
1019 
1020 static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx,
1021 						bool final)
1022 {
1023 }
1024 
1025 static inline int perf_cgroup_connect(pid_t pid, struct perf_event *event,
1026 				      struct perf_event_attr *attr,
1027 				      struct perf_event *group_leader)
1028 {
1029 	return -EINVAL;
1030 }
1031 
1032 static inline void
1033 perf_cgroup_set_timestamp(struct perf_cpu_context *cpuctx)
1034 {
1035 }
1036 
1037 static inline u64 perf_cgroup_event_time(struct perf_event *event)
1038 {
1039 	return 0;
1040 }
1041 
1042 static inline u64 perf_cgroup_event_time_now(struct perf_event *event, u64 now)
1043 {
1044 	return 0;
1045 }
1046 
1047 static inline void
1048 perf_cgroup_event_enable(struct perf_event *event, struct perf_event_context *ctx)
1049 {
1050 }
1051 
1052 static inline void
1053 perf_cgroup_event_disable(struct perf_event *event, struct perf_event_context *ctx)
1054 {
1055 }
1056 
1057 static void perf_cgroup_switch(struct task_struct *task)
1058 {
1059 }
1060 #endif
1061 
1062 /*
1063  * set default to be dependent on timer tick just
1064  * like original code
1065  */
1066 #define PERF_CPU_HRTIMER (1000 / HZ)
1067 /*
1068  * function must be called with interrupts disabled
1069  */
1070 static enum hrtimer_restart perf_mux_hrtimer_handler(struct hrtimer *hr)
1071 {
1072 	struct perf_cpu_context *cpuctx;
1073 	bool rotations;
1074 
1075 	lockdep_assert_irqs_disabled();
1076 
1077 	cpuctx = container_of(hr, struct perf_cpu_context, hrtimer);
1078 	rotations = perf_rotate_context(cpuctx);
1079 
1080 	raw_spin_lock(&cpuctx->hrtimer_lock);
1081 	if (rotations)
1082 		hrtimer_forward_now(hr, cpuctx->hrtimer_interval);
1083 	else
1084 		cpuctx->hrtimer_active = 0;
1085 	raw_spin_unlock(&cpuctx->hrtimer_lock);
1086 
1087 	return rotations ? HRTIMER_RESTART : HRTIMER_NORESTART;
1088 }
1089 
1090 static void __perf_mux_hrtimer_init(struct perf_cpu_context *cpuctx, int cpu)
1091 {
1092 	struct hrtimer *timer = &cpuctx->hrtimer;
1093 	struct pmu *pmu = cpuctx->ctx.pmu;
1094 	u64 interval;
1095 
1096 	/* no multiplexing needed for SW PMU */
1097 	if (pmu->task_ctx_nr == perf_sw_context)
1098 		return;
1099 
1100 	/*
1101 	 * check default is sane, if not set then force to
1102 	 * default interval (1/tick)
1103 	 */
1104 	interval = pmu->hrtimer_interval_ms;
1105 	if (interval < 1)
1106 		interval = pmu->hrtimer_interval_ms = PERF_CPU_HRTIMER;
1107 
1108 	cpuctx->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * interval);
1109 
1110 	raw_spin_lock_init(&cpuctx->hrtimer_lock);
1111 	hrtimer_init(timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED_HARD);
1112 	timer->function = perf_mux_hrtimer_handler;
1113 }
1114 
1115 static int perf_mux_hrtimer_restart(struct perf_cpu_context *cpuctx)
1116 {
1117 	struct hrtimer *timer = &cpuctx->hrtimer;
1118 	struct pmu *pmu = cpuctx->ctx.pmu;
1119 	unsigned long flags;
1120 
1121 	/* not for SW PMU */
1122 	if (pmu->task_ctx_nr == perf_sw_context)
1123 		return 0;
1124 
1125 	raw_spin_lock_irqsave(&cpuctx->hrtimer_lock, flags);
1126 	if (!cpuctx->hrtimer_active) {
1127 		cpuctx->hrtimer_active = 1;
1128 		hrtimer_forward_now(timer, cpuctx->hrtimer_interval);
1129 		hrtimer_start_expires(timer, HRTIMER_MODE_ABS_PINNED_HARD);
1130 	}
1131 	raw_spin_unlock_irqrestore(&cpuctx->hrtimer_lock, flags);
1132 
1133 	return 0;
1134 }
1135 
1136 void perf_pmu_disable(struct pmu *pmu)
1137 {
1138 	int *count = this_cpu_ptr(pmu->pmu_disable_count);
1139 	if (!(*count)++)
1140 		pmu->pmu_disable(pmu);
1141 }
1142 
1143 void perf_pmu_enable(struct pmu *pmu)
1144 {
1145 	int *count = this_cpu_ptr(pmu->pmu_disable_count);
1146 	if (!--(*count))
1147 		pmu->pmu_enable(pmu);
1148 }
1149 
1150 static DEFINE_PER_CPU(struct list_head, active_ctx_list);
1151 
1152 /*
1153  * perf_event_ctx_activate(), perf_event_ctx_deactivate(), and
1154  * perf_event_task_tick() are fully serialized because they're strictly cpu
1155  * affine and perf_event_ctx{activate,deactivate} are called with IRQs
1156  * disabled, while perf_event_task_tick is called from IRQ context.
1157  */
1158 static void perf_event_ctx_activate(struct perf_event_context *ctx)
1159 {
1160 	struct list_head *head = this_cpu_ptr(&active_ctx_list);
1161 
1162 	lockdep_assert_irqs_disabled();
1163 
1164 	WARN_ON(!list_empty(&ctx->active_ctx_list));
1165 
1166 	list_add(&ctx->active_ctx_list, head);
1167 }
1168 
1169 static void perf_event_ctx_deactivate(struct perf_event_context *ctx)
1170 {
1171 	lockdep_assert_irqs_disabled();
1172 
1173 	WARN_ON(list_empty(&ctx->active_ctx_list));
1174 
1175 	list_del_init(&ctx->active_ctx_list);
1176 }
1177 
1178 static void get_ctx(struct perf_event_context *ctx)
1179 {
1180 	refcount_inc(&ctx->refcount);
1181 }
1182 
1183 static void *alloc_task_ctx_data(struct pmu *pmu)
1184 {
1185 	if (pmu->task_ctx_cache)
1186 		return kmem_cache_zalloc(pmu->task_ctx_cache, GFP_KERNEL);
1187 
1188 	return NULL;
1189 }
1190 
1191 static void free_task_ctx_data(struct pmu *pmu, void *task_ctx_data)
1192 {
1193 	if (pmu->task_ctx_cache && task_ctx_data)
1194 		kmem_cache_free(pmu->task_ctx_cache, task_ctx_data);
1195 }
1196 
1197 static void free_ctx(struct rcu_head *head)
1198 {
1199 	struct perf_event_context *ctx;
1200 
1201 	ctx = container_of(head, struct perf_event_context, rcu_head);
1202 	free_task_ctx_data(ctx->pmu, ctx->task_ctx_data);
1203 	kfree(ctx);
1204 }
1205 
1206 static void put_ctx(struct perf_event_context *ctx)
1207 {
1208 	if (refcount_dec_and_test(&ctx->refcount)) {
1209 		if (ctx->parent_ctx)
1210 			put_ctx(ctx->parent_ctx);
1211 		if (ctx->task && ctx->task != TASK_TOMBSTONE)
1212 			put_task_struct(ctx->task);
1213 		call_rcu(&ctx->rcu_head, free_ctx);
1214 	}
1215 }
1216 
1217 /*
1218  * Because of perf_event::ctx migration in sys_perf_event_open::move_group and
1219  * perf_pmu_migrate_context() we need some magic.
1220  *
1221  * Those places that change perf_event::ctx will hold both
1222  * perf_event_ctx::mutex of the 'old' and 'new' ctx value.
1223  *
1224  * Lock ordering is by mutex address. There are two other sites where
1225  * perf_event_context::mutex nests and those are:
1226  *
1227  *  - perf_event_exit_task_context()	[ child , 0 ]
1228  *      perf_event_exit_event()
1229  *        put_event()			[ parent, 1 ]
1230  *
1231  *  - perf_event_init_context()		[ parent, 0 ]
1232  *      inherit_task_group()
1233  *        inherit_group()
1234  *          inherit_event()
1235  *            perf_event_alloc()
1236  *              perf_init_event()
1237  *                perf_try_init_event()	[ child , 1 ]
1238  *
1239  * While it appears there is an obvious deadlock here -- the parent and child
1240  * nesting levels are inverted between the two. This is in fact safe because
1241  * life-time rules separate them. That is an exiting task cannot fork, and a
1242  * spawning task cannot (yet) exit.
1243  *
1244  * But remember that these are parent<->child context relations, and
1245  * migration does not affect children, therefore these two orderings should not
1246  * interact.
1247  *
1248  * The change in perf_event::ctx does not affect children (as claimed above)
1249  * because the sys_perf_event_open() case will install a new event and break
1250  * the ctx parent<->child relation, and perf_pmu_migrate_context() is only
1251  * concerned with cpuctx and that doesn't have children.
1252  *
1253  * The places that change perf_event::ctx will issue:
1254  *
1255  *   perf_remove_from_context();
1256  *   synchronize_rcu();
1257  *   perf_install_in_context();
1258  *
1259  * to affect the change. The remove_from_context() + synchronize_rcu() should
1260  * quiesce the event, after which we can install it in the new location. This
1261  * means that only external vectors (perf_fops, prctl) can perturb the event
1262  * while in transit. Therefore all such accessors should also acquire
1263  * perf_event_context::mutex to serialize against this.
1264  *
1265  * However; because event->ctx can change while we're waiting to acquire
1266  * ctx->mutex we must be careful and use the below perf_event_ctx_lock()
1267  * function.
1268  *
1269  * Lock order:
1270  *    exec_update_lock
1271  *	task_struct::perf_event_mutex
1272  *	  perf_event_context::mutex
1273  *	    perf_event::child_mutex;
1274  *	      perf_event_context::lock
1275  *	    perf_event::mmap_mutex
1276  *	    mmap_lock
1277  *	      perf_addr_filters_head::lock
1278  *
1279  *    cpu_hotplug_lock
1280  *      pmus_lock
1281  *	  cpuctx->mutex / perf_event_context::mutex
1282  */
1283 static struct perf_event_context *
1284 perf_event_ctx_lock_nested(struct perf_event *event, int nesting)
1285 {
1286 	struct perf_event_context *ctx;
1287 
1288 again:
1289 	rcu_read_lock();
1290 	ctx = READ_ONCE(event->ctx);
1291 	if (!refcount_inc_not_zero(&ctx->refcount)) {
1292 		rcu_read_unlock();
1293 		goto again;
1294 	}
1295 	rcu_read_unlock();
1296 
1297 	mutex_lock_nested(&ctx->mutex, nesting);
1298 	if (event->ctx != ctx) {
1299 		mutex_unlock(&ctx->mutex);
1300 		put_ctx(ctx);
1301 		goto again;
1302 	}
1303 
1304 	return ctx;
1305 }
1306 
1307 static inline struct perf_event_context *
1308 perf_event_ctx_lock(struct perf_event *event)
1309 {
1310 	return perf_event_ctx_lock_nested(event, 0);
1311 }
1312 
1313 static void perf_event_ctx_unlock(struct perf_event *event,
1314 				  struct perf_event_context *ctx)
1315 {
1316 	mutex_unlock(&ctx->mutex);
1317 	put_ctx(ctx);
1318 }
1319 
1320 /*
1321  * This must be done under the ctx->lock, such as to serialize against
1322  * context_equiv(), therefore we cannot call put_ctx() since that might end up
1323  * calling scheduler related locks and ctx->lock nests inside those.
1324  */
1325 static __must_check struct perf_event_context *
1326 unclone_ctx(struct perf_event_context *ctx)
1327 {
1328 	struct perf_event_context *parent_ctx = ctx->parent_ctx;
1329 
1330 	lockdep_assert_held(&ctx->lock);
1331 
1332 	if (parent_ctx)
1333 		ctx->parent_ctx = NULL;
1334 	ctx->generation++;
1335 
1336 	return parent_ctx;
1337 }
1338 
1339 static u32 perf_event_pid_type(struct perf_event *event, struct task_struct *p,
1340 				enum pid_type type)
1341 {
1342 	u32 nr;
1343 	/*
1344 	 * only top level events have the pid namespace they were created in
1345 	 */
1346 	if (event->parent)
1347 		event = event->parent;
1348 
1349 	nr = __task_pid_nr_ns(p, type, event->ns);
1350 	/* avoid -1 if it is idle thread or runs in another ns */
1351 	if (!nr && !pid_alive(p))
1352 		nr = -1;
1353 	return nr;
1354 }
1355 
1356 static u32 perf_event_pid(struct perf_event *event, struct task_struct *p)
1357 {
1358 	return perf_event_pid_type(event, p, PIDTYPE_TGID);
1359 }
1360 
1361 static u32 perf_event_tid(struct perf_event *event, struct task_struct *p)
1362 {
1363 	return perf_event_pid_type(event, p, PIDTYPE_PID);
1364 }
1365 
1366 /*
1367  * If we inherit events we want to return the parent event id
1368  * to userspace.
1369  */
1370 static u64 primary_event_id(struct perf_event *event)
1371 {
1372 	u64 id = event->id;
1373 
1374 	if (event->parent)
1375 		id = event->parent->id;
1376 
1377 	return id;
1378 }
1379 
1380 /*
1381  * Get the perf_event_context for a task and lock it.
1382  *
1383  * This has to cope with the fact that until it is locked,
1384  * the context could get moved to another task.
1385  */
1386 static struct perf_event_context *
1387 perf_lock_task_context(struct task_struct *task, int ctxn, unsigned long *flags)
1388 {
1389 	struct perf_event_context *ctx;
1390 
1391 retry:
1392 	/*
1393 	 * One of the few rules of preemptible RCU is that one cannot do
1394 	 * rcu_read_unlock() while holding a scheduler (or nested) lock when
1395 	 * part of the read side critical section was irqs-enabled -- see
1396 	 * rcu_read_unlock_special().
1397 	 *
1398 	 * Since ctx->lock nests under rq->lock we must ensure the entire read
1399 	 * side critical section has interrupts disabled.
1400 	 */
1401 	local_irq_save(*flags);
1402 	rcu_read_lock();
1403 	ctx = rcu_dereference(task->perf_event_ctxp[ctxn]);
1404 	if (ctx) {
1405 		/*
1406 		 * If this context is a clone of another, it might
1407 		 * get swapped for another underneath us by
1408 		 * perf_event_task_sched_out, though the
1409 		 * rcu_read_lock() protects us from any context
1410 		 * getting freed.  Lock the context and check if it
1411 		 * got swapped before we could get the lock, and retry
1412 		 * if so.  If we locked the right context, then it
1413 		 * can't get swapped on us any more.
1414 		 */
1415 		raw_spin_lock(&ctx->lock);
1416 		if (ctx != rcu_dereference(task->perf_event_ctxp[ctxn])) {
1417 			raw_spin_unlock(&ctx->lock);
1418 			rcu_read_unlock();
1419 			local_irq_restore(*flags);
1420 			goto retry;
1421 		}
1422 
1423 		if (ctx->task == TASK_TOMBSTONE ||
1424 		    !refcount_inc_not_zero(&ctx->refcount)) {
1425 			raw_spin_unlock(&ctx->lock);
1426 			ctx = NULL;
1427 		} else {
1428 			WARN_ON_ONCE(ctx->task != task);
1429 		}
1430 	}
1431 	rcu_read_unlock();
1432 	if (!ctx)
1433 		local_irq_restore(*flags);
1434 	return ctx;
1435 }
1436 
1437 /*
1438  * Get the context for a task and increment its pin_count so it
1439  * can't get swapped to another task.  This also increments its
1440  * reference count so that the context can't get freed.
1441  */
1442 static struct perf_event_context *
1443 perf_pin_task_context(struct task_struct *task, int ctxn)
1444 {
1445 	struct perf_event_context *ctx;
1446 	unsigned long flags;
1447 
1448 	ctx = perf_lock_task_context(task, ctxn, &flags);
1449 	if (ctx) {
1450 		++ctx->pin_count;
1451 		raw_spin_unlock_irqrestore(&ctx->lock, flags);
1452 	}
1453 	return ctx;
1454 }
1455 
1456 static void perf_unpin_context(struct perf_event_context *ctx)
1457 {
1458 	unsigned long flags;
1459 
1460 	raw_spin_lock_irqsave(&ctx->lock, flags);
1461 	--ctx->pin_count;
1462 	raw_spin_unlock_irqrestore(&ctx->lock, flags);
1463 }
1464 
1465 /*
1466  * Update the record of the current time in a context.
1467  */
1468 static void __update_context_time(struct perf_event_context *ctx, bool adv)
1469 {
1470 	u64 now = perf_clock();
1471 
1472 	lockdep_assert_held(&ctx->lock);
1473 
1474 	if (adv)
1475 		ctx->time += now - ctx->timestamp;
1476 	ctx->timestamp = now;
1477 
1478 	/*
1479 	 * The above: time' = time + (now - timestamp), can be re-arranged
1480 	 * into: time` = now + (time - timestamp), which gives a single value
1481 	 * offset to compute future time without locks on.
1482 	 *
1483 	 * See perf_event_time_now(), which can be used from NMI context where
1484 	 * it's (obviously) not possible to acquire ctx->lock in order to read
1485 	 * both the above values in a consistent manner.
1486 	 */
1487 	WRITE_ONCE(ctx->timeoffset, ctx->time - ctx->timestamp);
1488 }
1489 
1490 static void update_context_time(struct perf_event_context *ctx)
1491 {
1492 	__update_context_time(ctx, true);
1493 }
1494 
1495 static u64 perf_event_time(struct perf_event *event)
1496 {
1497 	struct perf_event_context *ctx = event->ctx;
1498 
1499 	if (unlikely(!ctx))
1500 		return 0;
1501 
1502 	if (is_cgroup_event(event))
1503 		return perf_cgroup_event_time(event);
1504 
1505 	return ctx->time;
1506 }
1507 
1508 static u64 perf_event_time_now(struct perf_event *event, u64 now)
1509 {
1510 	struct perf_event_context *ctx = event->ctx;
1511 
1512 	if (unlikely(!ctx))
1513 		return 0;
1514 
1515 	if (is_cgroup_event(event))
1516 		return perf_cgroup_event_time_now(event, now);
1517 
1518 	if (!(__load_acquire(&ctx->is_active) & EVENT_TIME))
1519 		return ctx->time;
1520 
1521 	now += READ_ONCE(ctx->timeoffset);
1522 	return now;
1523 }
1524 
1525 static enum event_type_t get_event_type(struct perf_event *event)
1526 {
1527 	struct perf_event_context *ctx = event->ctx;
1528 	enum event_type_t event_type;
1529 
1530 	lockdep_assert_held(&ctx->lock);
1531 
1532 	/*
1533 	 * It's 'group type', really, because if our group leader is
1534 	 * pinned, so are we.
1535 	 */
1536 	if (event->group_leader != event)
1537 		event = event->group_leader;
1538 
1539 	event_type = event->attr.pinned ? EVENT_PINNED : EVENT_FLEXIBLE;
1540 	if (!ctx->task)
1541 		event_type |= EVENT_CPU;
1542 
1543 	return event_type;
1544 }
1545 
1546 /*
1547  * Helper function to initialize event group nodes.
1548  */
1549 static void init_event_group(struct perf_event *event)
1550 {
1551 	RB_CLEAR_NODE(&event->group_node);
1552 	event->group_index = 0;
1553 }
1554 
1555 /*
1556  * Extract pinned or flexible groups from the context
1557  * based on event attrs bits.
1558  */
1559 static struct perf_event_groups *
1560 get_event_groups(struct perf_event *event, struct perf_event_context *ctx)
1561 {
1562 	if (event->attr.pinned)
1563 		return &ctx->pinned_groups;
1564 	else
1565 		return &ctx->flexible_groups;
1566 }
1567 
1568 /*
1569  * Helper function to initializes perf_event_group trees.
1570  */
1571 static void perf_event_groups_init(struct perf_event_groups *groups)
1572 {
1573 	groups->tree = RB_ROOT;
1574 	groups->index = 0;
1575 }
1576 
1577 static inline struct cgroup *event_cgroup(const struct perf_event *event)
1578 {
1579 	struct cgroup *cgroup = NULL;
1580 
1581 #ifdef CONFIG_CGROUP_PERF
1582 	if (event->cgrp)
1583 		cgroup = event->cgrp->css.cgroup;
1584 #endif
1585 
1586 	return cgroup;
1587 }
1588 
1589 /*
1590  * Compare function for event groups;
1591  *
1592  * Implements complex key that first sorts by CPU and then by virtual index
1593  * which provides ordering when rotating groups for the same CPU.
1594  */
1595 static __always_inline int
1596 perf_event_groups_cmp(const int left_cpu, const struct cgroup *left_cgroup,
1597 		      const u64 left_group_index, const struct perf_event *right)
1598 {
1599 	if (left_cpu < right->cpu)
1600 		return -1;
1601 	if (left_cpu > right->cpu)
1602 		return 1;
1603 
1604 #ifdef CONFIG_CGROUP_PERF
1605 	{
1606 		const struct cgroup *right_cgroup = event_cgroup(right);
1607 
1608 		if (left_cgroup != right_cgroup) {
1609 			if (!left_cgroup) {
1610 				/*
1611 				 * Left has no cgroup but right does, no
1612 				 * cgroups come first.
1613 				 */
1614 				return -1;
1615 			}
1616 			if (!right_cgroup) {
1617 				/*
1618 				 * Right has no cgroup but left does, no
1619 				 * cgroups come first.
1620 				 */
1621 				return 1;
1622 			}
1623 			/* Two dissimilar cgroups, order by id. */
1624 			if (cgroup_id(left_cgroup) < cgroup_id(right_cgroup))
1625 				return -1;
1626 
1627 			return 1;
1628 		}
1629 	}
1630 #endif
1631 
1632 	if (left_group_index < right->group_index)
1633 		return -1;
1634 	if (left_group_index > right->group_index)
1635 		return 1;
1636 
1637 	return 0;
1638 }
1639 
1640 #define __node_2_pe(node) \
1641 	rb_entry((node), struct perf_event, group_node)
1642 
1643 static inline bool __group_less(struct rb_node *a, const struct rb_node *b)
1644 {
1645 	struct perf_event *e = __node_2_pe(a);
1646 	return perf_event_groups_cmp(e->cpu, event_cgroup(e), e->group_index,
1647 				     __node_2_pe(b)) < 0;
1648 }
1649 
1650 struct __group_key {
1651 	int cpu;
1652 	struct cgroup *cgroup;
1653 };
1654 
1655 static inline int __group_cmp(const void *key, const struct rb_node *node)
1656 {
1657 	const struct __group_key *a = key;
1658 	const struct perf_event *b = __node_2_pe(node);
1659 
1660 	/* partial/subtree match: @cpu, @cgroup; ignore: @group_index */
1661 	return perf_event_groups_cmp(a->cpu, a->cgroup, b->group_index, b);
1662 }
1663 
1664 /*
1665  * Insert @event into @groups' tree; using {@event->cpu, ++@groups->index} for
1666  * key (see perf_event_groups_less). This places it last inside the CPU
1667  * subtree.
1668  */
1669 static void
1670 perf_event_groups_insert(struct perf_event_groups *groups,
1671 			 struct perf_event *event)
1672 {
1673 	event->group_index = ++groups->index;
1674 
1675 	rb_add(&event->group_node, &groups->tree, __group_less);
1676 }
1677 
1678 /*
1679  * Helper function to insert event into the pinned or flexible groups.
1680  */
1681 static void
1682 add_event_to_groups(struct perf_event *event, struct perf_event_context *ctx)
1683 {
1684 	struct perf_event_groups *groups;
1685 
1686 	groups = get_event_groups(event, ctx);
1687 	perf_event_groups_insert(groups, event);
1688 }
1689 
1690 /*
1691  * Delete a group from a tree.
1692  */
1693 static void
1694 perf_event_groups_delete(struct perf_event_groups *groups,
1695 			 struct perf_event *event)
1696 {
1697 	WARN_ON_ONCE(RB_EMPTY_NODE(&event->group_node) ||
1698 		     RB_EMPTY_ROOT(&groups->tree));
1699 
1700 	rb_erase(&event->group_node, &groups->tree);
1701 	init_event_group(event);
1702 }
1703 
1704 /*
1705  * Helper function to delete event from its groups.
1706  */
1707 static void
1708 del_event_from_groups(struct perf_event *event, struct perf_event_context *ctx)
1709 {
1710 	struct perf_event_groups *groups;
1711 
1712 	groups = get_event_groups(event, ctx);
1713 	perf_event_groups_delete(groups, event);
1714 }
1715 
1716 /*
1717  * Get the leftmost event in the cpu/cgroup subtree.
1718  */
1719 static struct perf_event *
1720 perf_event_groups_first(struct perf_event_groups *groups, int cpu,
1721 			struct cgroup *cgrp)
1722 {
1723 	struct __group_key key = {
1724 		.cpu = cpu,
1725 		.cgroup = cgrp,
1726 	};
1727 	struct rb_node *node;
1728 
1729 	node = rb_find_first(&key, &groups->tree, __group_cmp);
1730 	if (node)
1731 		return __node_2_pe(node);
1732 
1733 	return NULL;
1734 }
1735 
1736 /*
1737  * Like rb_entry_next_safe() for the @cpu subtree.
1738  */
1739 static struct perf_event *
1740 perf_event_groups_next(struct perf_event *event)
1741 {
1742 	struct __group_key key = {
1743 		.cpu = event->cpu,
1744 		.cgroup = event_cgroup(event),
1745 	};
1746 	struct rb_node *next;
1747 
1748 	next = rb_next_match(&key, &event->group_node, __group_cmp);
1749 	if (next)
1750 		return __node_2_pe(next);
1751 
1752 	return NULL;
1753 }
1754 
1755 /*
1756  * Iterate through the whole groups tree.
1757  */
1758 #define perf_event_groups_for_each(event, groups)			\
1759 	for (event = rb_entry_safe(rb_first(&((groups)->tree)),		\
1760 				typeof(*event), group_node); event;	\
1761 		event = rb_entry_safe(rb_next(&event->group_node),	\
1762 				typeof(*event), group_node))
1763 
1764 /*
1765  * Add an event from the lists for its context.
1766  * Must be called with ctx->mutex and ctx->lock held.
1767  */
1768 static void
1769 list_add_event(struct perf_event *event, struct perf_event_context *ctx)
1770 {
1771 	lockdep_assert_held(&ctx->lock);
1772 
1773 	WARN_ON_ONCE(event->attach_state & PERF_ATTACH_CONTEXT);
1774 	event->attach_state |= PERF_ATTACH_CONTEXT;
1775 
1776 	event->tstamp = perf_event_time(event);
1777 
1778 	/*
1779 	 * If we're a stand alone event or group leader, we go to the context
1780 	 * list, group events are kept attached to the group so that
1781 	 * perf_group_detach can, at all times, locate all siblings.
1782 	 */
1783 	if (event->group_leader == event) {
1784 		event->group_caps = event->event_caps;
1785 		add_event_to_groups(event, ctx);
1786 	}
1787 
1788 	list_add_rcu(&event->event_entry, &ctx->event_list);
1789 	ctx->nr_events++;
1790 	if (event->hw.flags & PERF_EVENT_FLAG_USER_READ_CNT)
1791 		ctx->nr_user++;
1792 	if (event->attr.inherit_stat)
1793 		ctx->nr_stat++;
1794 
1795 	if (event->state > PERF_EVENT_STATE_OFF)
1796 		perf_cgroup_event_enable(event, ctx);
1797 
1798 	ctx->generation++;
1799 }
1800 
1801 /*
1802  * Initialize event state based on the perf_event_attr::disabled.
1803  */
1804 static inline void perf_event__state_init(struct perf_event *event)
1805 {
1806 	event->state = event->attr.disabled ? PERF_EVENT_STATE_OFF :
1807 					      PERF_EVENT_STATE_INACTIVE;
1808 }
1809 
1810 static void __perf_event_read_size(struct perf_event *event, int nr_siblings)
1811 {
1812 	int entry = sizeof(u64); /* value */
1813 	int size = 0;
1814 	int nr = 1;
1815 
1816 	if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
1817 		size += sizeof(u64);
1818 
1819 	if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
1820 		size += sizeof(u64);
1821 
1822 	if (event->attr.read_format & PERF_FORMAT_ID)
1823 		entry += sizeof(u64);
1824 
1825 	if (event->attr.read_format & PERF_FORMAT_LOST)
1826 		entry += sizeof(u64);
1827 
1828 	if (event->attr.read_format & PERF_FORMAT_GROUP) {
1829 		nr += nr_siblings;
1830 		size += sizeof(u64);
1831 	}
1832 
1833 	size += entry * nr;
1834 	event->read_size = size;
1835 }
1836 
1837 static void __perf_event_header_size(struct perf_event *event, u64 sample_type)
1838 {
1839 	struct perf_sample_data *data;
1840 	u16 size = 0;
1841 
1842 	if (sample_type & PERF_SAMPLE_IP)
1843 		size += sizeof(data->ip);
1844 
1845 	if (sample_type & PERF_SAMPLE_ADDR)
1846 		size += sizeof(data->addr);
1847 
1848 	if (sample_type & PERF_SAMPLE_PERIOD)
1849 		size += sizeof(data->period);
1850 
1851 	if (sample_type & PERF_SAMPLE_WEIGHT_TYPE)
1852 		size += sizeof(data->weight.full);
1853 
1854 	if (sample_type & PERF_SAMPLE_READ)
1855 		size += event->read_size;
1856 
1857 	if (sample_type & PERF_SAMPLE_DATA_SRC)
1858 		size += sizeof(data->data_src.val);
1859 
1860 	if (sample_type & PERF_SAMPLE_TRANSACTION)
1861 		size += sizeof(data->txn);
1862 
1863 	if (sample_type & PERF_SAMPLE_PHYS_ADDR)
1864 		size += sizeof(data->phys_addr);
1865 
1866 	if (sample_type & PERF_SAMPLE_CGROUP)
1867 		size += sizeof(data->cgroup);
1868 
1869 	if (sample_type & PERF_SAMPLE_DATA_PAGE_SIZE)
1870 		size += sizeof(data->data_page_size);
1871 
1872 	if (sample_type & PERF_SAMPLE_CODE_PAGE_SIZE)
1873 		size += sizeof(data->code_page_size);
1874 
1875 	event->header_size = size;
1876 }
1877 
1878 /*
1879  * Called at perf_event creation and when events are attached/detached from a
1880  * group.
1881  */
1882 static void perf_event__header_size(struct perf_event *event)
1883 {
1884 	__perf_event_read_size(event,
1885 			       event->group_leader->nr_siblings);
1886 	__perf_event_header_size(event, event->attr.sample_type);
1887 }
1888 
1889 static void perf_event__id_header_size(struct perf_event *event)
1890 {
1891 	struct perf_sample_data *data;
1892 	u64 sample_type = event->attr.sample_type;
1893 	u16 size = 0;
1894 
1895 	if (sample_type & PERF_SAMPLE_TID)
1896 		size += sizeof(data->tid_entry);
1897 
1898 	if (sample_type & PERF_SAMPLE_TIME)
1899 		size += sizeof(data->time);
1900 
1901 	if (sample_type & PERF_SAMPLE_IDENTIFIER)
1902 		size += sizeof(data->id);
1903 
1904 	if (sample_type & PERF_SAMPLE_ID)
1905 		size += sizeof(data->id);
1906 
1907 	if (sample_type & PERF_SAMPLE_STREAM_ID)
1908 		size += sizeof(data->stream_id);
1909 
1910 	if (sample_type & PERF_SAMPLE_CPU)
1911 		size += sizeof(data->cpu_entry);
1912 
1913 	event->id_header_size = size;
1914 }
1915 
1916 static bool perf_event_validate_size(struct perf_event *event)
1917 {
1918 	/*
1919 	 * The values computed here will be over-written when we actually
1920 	 * attach the event.
1921 	 */
1922 	__perf_event_read_size(event, event->group_leader->nr_siblings + 1);
1923 	__perf_event_header_size(event, event->attr.sample_type & ~PERF_SAMPLE_READ);
1924 	perf_event__id_header_size(event);
1925 
1926 	/*
1927 	 * Sum the lot; should not exceed the 64k limit we have on records.
1928 	 * Conservative limit to allow for callchains and other variable fields.
1929 	 */
1930 	if (event->read_size + event->header_size +
1931 	    event->id_header_size + sizeof(struct perf_event_header) >= 16*1024)
1932 		return false;
1933 
1934 	return true;
1935 }
1936 
1937 static void perf_group_attach(struct perf_event *event)
1938 {
1939 	struct perf_event *group_leader = event->group_leader, *pos;
1940 
1941 	lockdep_assert_held(&event->ctx->lock);
1942 
1943 	/*
1944 	 * We can have double attach due to group movement in perf_event_open.
1945 	 */
1946 	if (event->attach_state & PERF_ATTACH_GROUP)
1947 		return;
1948 
1949 	event->attach_state |= PERF_ATTACH_GROUP;
1950 
1951 	if (group_leader == event)
1952 		return;
1953 
1954 	WARN_ON_ONCE(group_leader->ctx != event->ctx);
1955 
1956 	group_leader->group_caps &= event->event_caps;
1957 
1958 	list_add_tail(&event->sibling_list, &group_leader->sibling_list);
1959 	group_leader->nr_siblings++;
1960 
1961 	perf_event__header_size(group_leader);
1962 
1963 	for_each_sibling_event(pos, group_leader)
1964 		perf_event__header_size(pos);
1965 }
1966 
1967 /*
1968  * Remove an event from the lists for its context.
1969  * Must be called with ctx->mutex and ctx->lock held.
1970  */
1971 static void
1972 list_del_event(struct perf_event *event, struct perf_event_context *ctx)
1973 {
1974 	WARN_ON_ONCE(event->ctx != ctx);
1975 	lockdep_assert_held(&ctx->lock);
1976 
1977 	/*
1978 	 * We can have double detach due to exit/hot-unplug + close.
1979 	 */
1980 	if (!(event->attach_state & PERF_ATTACH_CONTEXT))
1981 		return;
1982 
1983 	event->attach_state &= ~PERF_ATTACH_CONTEXT;
1984 
1985 	ctx->nr_events--;
1986 	if (event->hw.flags & PERF_EVENT_FLAG_USER_READ_CNT)
1987 		ctx->nr_user--;
1988 	if (event->attr.inherit_stat)
1989 		ctx->nr_stat--;
1990 
1991 	list_del_rcu(&event->event_entry);
1992 
1993 	if (event->group_leader == event)
1994 		del_event_from_groups(event, ctx);
1995 
1996 	/*
1997 	 * If event was in error state, then keep it
1998 	 * that way, otherwise bogus counts will be
1999 	 * returned on read(). The only way to get out
2000 	 * of error state is by explicit re-enabling
2001 	 * of the event
2002 	 */
2003 	if (event->state > PERF_EVENT_STATE_OFF) {
2004 		perf_cgroup_event_disable(event, ctx);
2005 		perf_event_set_state(event, PERF_EVENT_STATE_OFF);
2006 	}
2007 
2008 	ctx->generation++;
2009 }
2010 
2011 static int
2012 perf_aux_output_match(struct perf_event *event, struct perf_event *aux_event)
2013 {
2014 	if (!has_aux(aux_event))
2015 		return 0;
2016 
2017 	if (!event->pmu->aux_output_match)
2018 		return 0;
2019 
2020 	return event->pmu->aux_output_match(aux_event);
2021 }
2022 
2023 static void put_event(struct perf_event *event);
2024 static void event_sched_out(struct perf_event *event,
2025 			    struct perf_cpu_context *cpuctx,
2026 			    struct perf_event_context *ctx);
2027 
2028 static void perf_put_aux_event(struct perf_event *event)
2029 {
2030 	struct perf_event_context *ctx = event->ctx;
2031 	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
2032 	struct perf_event *iter;
2033 
2034 	/*
2035 	 * If event uses aux_event tear down the link
2036 	 */
2037 	if (event->aux_event) {
2038 		iter = event->aux_event;
2039 		event->aux_event = NULL;
2040 		put_event(iter);
2041 		return;
2042 	}
2043 
2044 	/*
2045 	 * If the event is an aux_event, tear down all links to
2046 	 * it from other events.
2047 	 */
2048 	for_each_sibling_event(iter, event->group_leader) {
2049 		if (iter->aux_event != event)
2050 			continue;
2051 
2052 		iter->aux_event = NULL;
2053 		put_event(event);
2054 
2055 		/*
2056 		 * If it's ACTIVE, schedule it out and put it into ERROR
2057 		 * state so that we don't try to schedule it again. Note
2058 		 * that perf_event_enable() will clear the ERROR status.
2059 		 */
2060 		event_sched_out(iter, cpuctx, ctx);
2061 		perf_event_set_state(event, PERF_EVENT_STATE_ERROR);
2062 	}
2063 }
2064 
2065 static bool perf_need_aux_event(struct perf_event *event)
2066 {
2067 	return !!event->attr.aux_output || !!event->attr.aux_sample_size;
2068 }
2069 
2070 static int perf_get_aux_event(struct perf_event *event,
2071 			      struct perf_event *group_leader)
2072 {
2073 	/*
2074 	 * Our group leader must be an aux event if we want to be
2075 	 * an aux_output. This way, the aux event will precede its
2076 	 * aux_output events in the group, and therefore will always
2077 	 * schedule first.
2078 	 */
2079 	if (!group_leader)
2080 		return 0;
2081 
2082 	/*
2083 	 * aux_output and aux_sample_size are mutually exclusive.
2084 	 */
2085 	if (event->attr.aux_output && event->attr.aux_sample_size)
2086 		return 0;
2087 
2088 	if (event->attr.aux_output &&
2089 	    !perf_aux_output_match(event, group_leader))
2090 		return 0;
2091 
2092 	if (event->attr.aux_sample_size && !group_leader->pmu->snapshot_aux)
2093 		return 0;
2094 
2095 	if (!atomic_long_inc_not_zero(&group_leader->refcount))
2096 		return 0;
2097 
2098 	/*
2099 	 * Link aux_outputs to their aux event; this is undone in
2100 	 * perf_group_detach() by perf_put_aux_event(). When the
2101 	 * group in torn down, the aux_output events loose their
2102 	 * link to the aux_event and can't schedule any more.
2103 	 */
2104 	event->aux_event = group_leader;
2105 
2106 	return 1;
2107 }
2108 
2109 static inline struct list_head *get_event_list(struct perf_event *event)
2110 {
2111 	struct perf_event_context *ctx = event->ctx;
2112 	return event->attr.pinned ? &ctx->pinned_active : &ctx->flexible_active;
2113 }
2114 
2115 /*
2116  * Events that have PERF_EV_CAP_SIBLING require being part of a group and
2117  * cannot exist on their own, schedule them out and move them into the ERROR
2118  * state. Also see _perf_event_enable(), it will not be able to recover
2119  * this ERROR state.
2120  */
2121 static inline void perf_remove_sibling_event(struct perf_event *event)
2122 {
2123 	struct perf_event_context *ctx = event->ctx;
2124 	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
2125 
2126 	event_sched_out(event, cpuctx, ctx);
2127 	perf_event_set_state(event, PERF_EVENT_STATE_ERROR);
2128 }
2129 
2130 static void perf_group_detach(struct perf_event *event)
2131 {
2132 	struct perf_event *leader = event->group_leader;
2133 	struct perf_event *sibling, *tmp;
2134 	struct perf_event_context *ctx = event->ctx;
2135 
2136 	lockdep_assert_held(&ctx->lock);
2137 
2138 	/*
2139 	 * We can have double detach due to exit/hot-unplug + close.
2140 	 */
2141 	if (!(event->attach_state & PERF_ATTACH_GROUP))
2142 		return;
2143 
2144 	event->attach_state &= ~PERF_ATTACH_GROUP;
2145 
2146 	perf_put_aux_event(event);
2147 
2148 	/*
2149 	 * If this is a sibling, remove it from its group.
2150 	 */
2151 	if (leader != event) {
2152 		list_del_init(&event->sibling_list);
2153 		event->group_leader->nr_siblings--;
2154 		goto out;
2155 	}
2156 
2157 	/*
2158 	 * If this was a group event with sibling events then
2159 	 * upgrade the siblings to singleton events by adding them
2160 	 * to whatever list we are on.
2161 	 */
2162 	list_for_each_entry_safe(sibling, tmp, &event->sibling_list, sibling_list) {
2163 
2164 		if (sibling->event_caps & PERF_EV_CAP_SIBLING)
2165 			perf_remove_sibling_event(sibling);
2166 
2167 		sibling->group_leader = sibling;
2168 		list_del_init(&sibling->sibling_list);
2169 
2170 		/* Inherit group flags from the previous leader */
2171 		sibling->group_caps = event->group_caps;
2172 
2173 		if (!RB_EMPTY_NODE(&event->group_node)) {
2174 			add_event_to_groups(sibling, event->ctx);
2175 
2176 			if (sibling->state == PERF_EVENT_STATE_ACTIVE)
2177 				list_add_tail(&sibling->active_list, get_event_list(sibling));
2178 		}
2179 
2180 		WARN_ON_ONCE(sibling->ctx != event->ctx);
2181 	}
2182 
2183 out:
2184 	for_each_sibling_event(tmp, leader)
2185 		perf_event__header_size(tmp);
2186 
2187 	perf_event__header_size(leader);
2188 }
2189 
2190 static void sync_child_event(struct perf_event *child_event);
2191 
2192 static void perf_child_detach(struct perf_event *event)
2193 {
2194 	struct perf_event *parent_event = event->parent;
2195 
2196 	if (!(event->attach_state & PERF_ATTACH_CHILD))
2197 		return;
2198 
2199 	event->attach_state &= ~PERF_ATTACH_CHILD;
2200 
2201 	if (WARN_ON_ONCE(!parent_event))
2202 		return;
2203 
2204 	lockdep_assert_held(&parent_event->child_mutex);
2205 
2206 	sync_child_event(event);
2207 	list_del_init(&event->child_list);
2208 }
2209 
2210 static bool is_orphaned_event(struct perf_event *event)
2211 {
2212 	return event->state == PERF_EVENT_STATE_DEAD;
2213 }
2214 
2215 static inline int __pmu_filter_match(struct perf_event *event)
2216 {
2217 	struct pmu *pmu = event->pmu;
2218 	return pmu->filter_match ? pmu->filter_match(event) : 1;
2219 }
2220 
2221 /*
2222  * Check whether we should attempt to schedule an event group based on
2223  * PMU-specific filtering. An event group can consist of HW and SW events,
2224  * potentially with a SW leader, so we must check all the filters, to
2225  * determine whether a group is schedulable:
2226  */
2227 static inline int pmu_filter_match(struct perf_event *event)
2228 {
2229 	struct perf_event *sibling;
2230 	unsigned long flags;
2231 	int ret = 1;
2232 
2233 	if (!__pmu_filter_match(event))
2234 		return 0;
2235 
2236 	local_irq_save(flags);
2237 	for_each_sibling_event(sibling, event) {
2238 		if (!__pmu_filter_match(sibling)) {
2239 			ret = 0;
2240 			break;
2241 		}
2242 	}
2243 	local_irq_restore(flags);
2244 
2245 	return ret;
2246 }
2247 
2248 static inline int
2249 event_filter_match(struct perf_event *event)
2250 {
2251 	return (event->cpu == -1 || event->cpu == smp_processor_id()) &&
2252 	       perf_cgroup_match(event) && pmu_filter_match(event);
2253 }
2254 
2255 static void
2256 event_sched_out(struct perf_event *event,
2257 		  struct perf_cpu_context *cpuctx,
2258 		  struct perf_event_context *ctx)
2259 {
2260 	enum perf_event_state state = PERF_EVENT_STATE_INACTIVE;
2261 
2262 	WARN_ON_ONCE(event->ctx != ctx);
2263 	lockdep_assert_held(&ctx->lock);
2264 
2265 	if (event->state != PERF_EVENT_STATE_ACTIVE)
2266 		return;
2267 
2268 	/*
2269 	 * Asymmetry; we only schedule events _IN_ through ctx_sched_in(), but
2270 	 * we can schedule events _OUT_ individually through things like
2271 	 * __perf_remove_from_context().
2272 	 */
2273 	list_del_init(&event->active_list);
2274 
2275 	perf_pmu_disable(event->pmu);
2276 
2277 	event->pmu->del(event, 0);
2278 	event->oncpu = -1;
2279 
2280 	if (event->pending_disable) {
2281 		event->pending_disable = 0;
2282 		perf_cgroup_event_disable(event, ctx);
2283 		state = PERF_EVENT_STATE_OFF;
2284 	}
2285 
2286 	if (event->pending_sigtrap) {
2287 		bool dec = true;
2288 
2289 		event->pending_sigtrap = 0;
2290 		if (state != PERF_EVENT_STATE_OFF &&
2291 		    !event->pending_work) {
2292 			event->pending_work = 1;
2293 			dec = false;
2294 			task_work_add(current, &event->pending_task, TWA_RESUME);
2295 		}
2296 		if (dec)
2297 			local_dec(&event->ctx->nr_pending);
2298 	}
2299 
2300 	perf_event_set_state(event, state);
2301 
2302 	if (!is_software_event(event))
2303 		cpuctx->active_oncpu--;
2304 	if (!--ctx->nr_active)
2305 		perf_event_ctx_deactivate(ctx);
2306 	if (event->attr.freq && event->attr.sample_freq)
2307 		ctx->nr_freq--;
2308 	if (event->attr.exclusive || !cpuctx->active_oncpu)
2309 		cpuctx->exclusive = 0;
2310 
2311 	perf_pmu_enable(event->pmu);
2312 }
2313 
2314 static void
2315 group_sched_out(struct perf_event *group_event,
2316 		struct perf_cpu_context *cpuctx,
2317 		struct perf_event_context *ctx)
2318 {
2319 	struct perf_event *event;
2320 
2321 	if (group_event->state != PERF_EVENT_STATE_ACTIVE)
2322 		return;
2323 
2324 	perf_pmu_disable(ctx->pmu);
2325 
2326 	event_sched_out(group_event, cpuctx, ctx);
2327 
2328 	/*
2329 	 * Schedule out siblings (if any):
2330 	 */
2331 	for_each_sibling_event(event, group_event)
2332 		event_sched_out(event, cpuctx, ctx);
2333 
2334 	perf_pmu_enable(ctx->pmu);
2335 }
2336 
2337 #define DETACH_GROUP	0x01UL
2338 #define DETACH_CHILD	0x02UL
2339 
2340 /*
2341  * Cross CPU call to remove a performance event
2342  *
2343  * We disable the event on the hardware level first. After that we
2344  * remove it from the context list.
2345  */
2346 static void
2347 __perf_remove_from_context(struct perf_event *event,
2348 			   struct perf_cpu_context *cpuctx,
2349 			   struct perf_event_context *ctx,
2350 			   void *info)
2351 {
2352 	unsigned long flags = (unsigned long)info;
2353 
2354 	if (ctx->is_active & EVENT_TIME) {
2355 		update_context_time(ctx);
2356 		update_cgrp_time_from_cpuctx(cpuctx, false);
2357 	}
2358 
2359 	event_sched_out(event, cpuctx, ctx);
2360 	if (flags & DETACH_GROUP)
2361 		perf_group_detach(event);
2362 	if (flags & DETACH_CHILD)
2363 		perf_child_detach(event);
2364 	list_del_event(event, ctx);
2365 
2366 	if (!ctx->nr_events && ctx->is_active) {
2367 		if (ctx == &cpuctx->ctx)
2368 			update_cgrp_time_from_cpuctx(cpuctx, true);
2369 
2370 		ctx->is_active = 0;
2371 		ctx->rotate_necessary = 0;
2372 		if (ctx->task) {
2373 			WARN_ON_ONCE(cpuctx->task_ctx != ctx);
2374 			cpuctx->task_ctx = NULL;
2375 		}
2376 	}
2377 }
2378 
2379 /*
2380  * Remove the event from a task's (or a CPU's) list of events.
2381  *
2382  * If event->ctx is a cloned context, callers must make sure that
2383  * every task struct that event->ctx->task could possibly point to
2384  * remains valid.  This is OK when called from perf_release since
2385  * that only calls us on the top-level context, which can't be a clone.
2386  * When called from perf_event_exit_task, it's OK because the
2387  * context has been detached from its task.
2388  */
2389 static void perf_remove_from_context(struct perf_event *event, unsigned long flags)
2390 {
2391 	struct perf_event_context *ctx = event->ctx;
2392 
2393 	lockdep_assert_held(&ctx->mutex);
2394 
2395 	/*
2396 	 * Because of perf_event_exit_task(), perf_remove_from_context() ought
2397 	 * to work in the face of TASK_TOMBSTONE, unlike every other
2398 	 * event_function_call() user.
2399 	 */
2400 	raw_spin_lock_irq(&ctx->lock);
2401 	/*
2402 	 * Cgroup events are per-cpu events, and must IPI because of
2403 	 * cgrp_cpuctx_list.
2404 	 */
2405 	if (!ctx->is_active && !is_cgroup_event(event)) {
2406 		__perf_remove_from_context(event, __get_cpu_context(ctx),
2407 					   ctx, (void *)flags);
2408 		raw_spin_unlock_irq(&ctx->lock);
2409 		return;
2410 	}
2411 	raw_spin_unlock_irq(&ctx->lock);
2412 
2413 	event_function_call(event, __perf_remove_from_context, (void *)flags);
2414 }
2415 
2416 /*
2417  * Cross CPU call to disable a performance event
2418  */
2419 static void __perf_event_disable(struct perf_event *event,
2420 				 struct perf_cpu_context *cpuctx,
2421 				 struct perf_event_context *ctx,
2422 				 void *info)
2423 {
2424 	if (event->state < PERF_EVENT_STATE_INACTIVE)
2425 		return;
2426 
2427 	if (ctx->is_active & EVENT_TIME) {
2428 		update_context_time(ctx);
2429 		update_cgrp_time_from_event(event);
2430 	}
2431 
2432 	if (event == event->group_leader)
2433 		group_sched_out(event, cpuctx, ctx);
2434 	else
2435 		event_sched_out(event, cpuctx, ctx);
2436 
2437 	perf_event_set_state(event, PERF_EVENT_STATE_OFF);
2438 	perf_cgroup_event_disable(event, ctx);
2439 }
2440 
2441 /*
2442  * Disable an event.
2443  *
2444  * If event->ctx is a cloned context, callers must make sure that
2445  * every task struct that event->ctx->task could possibly point to
2446  * remains valid.  This condition is satisfied when called through
2447  * perf_event_for_each_child or perf_event_for_each because they
2448  * hold the top-level event's child_mutex, so any descendant that
2449  * goes to exit will block in perf_event_exit_event().
2450  *
2451  * When called from perf_pending_irq it's OK because event->ctx
2452  * is the current context on this CPU and preemption is disabled,
2453  * hence we can't get into perf_event_task_sched_out for this context.
2454  */
2455 static void _perf_event_disable(struct perf_event *event)
2456 {
2457 	struct perf_event_context *ctx = event->ctx;
2458 
2459 	raw_spin_lock_irq(&ctx->lock);
2460 	if (event->state <= PERF_EVENT_STATE_OFF) {
2461 		raw_spin_unlock_irq(&ctx->lock);
2462 		return;
2463 	}
2464 	raw_spin_unlock_irq(&ctx->lock);
2465 
2466 	event_function_call(event, __perf_event_disable, NULL);
2467 }
2468 
2469 void perf_event_disable_local(struct perf_event *event)
2470 {
2471 	event_function_local(event, __perf_event_disable, NULL);
2472 }
2473 
2474 /*
2475  * Strictly speaking kernel users cannot create groups and therefore this
2476  * interface does not need the perf_event_ctx_lock() magic.
2477  */
2478 void perf_event_disable(struct perf_event *event)
2479 {
2480 	struct perf_event_context *ctx;
2481 
2482 	ctx = perf_event_ctx_lock(event);
2483 	_perf_event_disable(event);
2484 	perf_event_ctx_unlock(event, ctx);
2485 }
2486 EXPORT_SYMBOL_GPL(perf_event_disable);
2487 
2488 void perf_event_disable_inatomic(struct perf_event *event)
2489 {
2490 	event->pending_disable = 1;
2491 	irq_work_queue(&event->pending_irq);
2492 }
2493 
2494 #define MAX_INTERRUPTS (~0ULL)
2495 
2496 static void perf_log_throttle(struct perf_event *event, int enable);
2497 static void perf_log_itrace_start(struct perf_event *event);
2498 
2499 static int
2500 event_sched_in(struct perf_event *event,
2501 		 struct perf_cpu_context *cpuctx,
2502 		 struct perf_event_context *ctx)
2503 {
2504 	int ret = 0;
2505 
2506 	WARN_ON_ONCE(event->ctx != ctx);
2507 
2508 	lockdep_assert_held(&ctx->lock);
2509 
2510 	if (event->state <= PERF_EVENT_STATE_OFF)
2511 		return 0;
2512 
2513 	WRITE_ONCE(event->oncpu, smp_processor_id());
2514 	/*
2515 	 * Order event::oncpu write to happen before the ACTIVE state is
2516 	 * visible. This allows perf_event_{stop,read}() to observe the correct
2517 	 * ->oncpu if it sees ACTIVE.
2518 	 */
2519 	smp_wmb();
2520 	perf_event_set_state(event, PERF_EVENT_STATE_ACTIVE);
2521 
2522 	/*
2523 	 * Unthrottle events, since we scheduled we might have missed several
2524 	 * ticks already, also for a heavily scheduling task there is little
2525 	 * guarantee it'll get a tick in a timely manner.
2526 	 */
2527 	if (unlikely(event->hw.interrupts == MAX_INTERRUPTS)) {
2528 		perf_log_throttle(event, 1);
2529 		event->hw.interrupts = 0;
2530 	}
2531 
2532 	perf_pmu_disable(event->pmu);
2533 
2534 	perf_log_itrace_start(event);
2535 
2536 	if (event->pmu->add(event, PERF_EF_START)) {
2537 		perf_event_set_state(event, PERF_EVENT_STATE_INACTIVE);
2538 		event->oncpu = -1;
2539 		ret = -EAGAIN;
2540 		goto out;
2541 	}
2542 
2543 	if (!is_software_event(event))
2544 		cpuctx->active_oncpu++;
2545 	if (!ctx->nr_active++)
2546 		perf_event_ctx_activate(ctx);
2547 	if (event->attr.freq && event->attr.sample_freq)
2548 		ctx->nr_freq++;
2549 
2550 	if (event->attr.exclusive)
2551 		cpuctx->exclusive = 1;
2552 
2553 out:
2554 	perf_pmu_enable(event->pmu);
2555 
2556 	return ret;
2557 }
2558 
2559 static int
2560 group_sched_in(struct perf_event *group_event,
2561 	       struct perf_cpu_context *cpuctx,
2562 	       struct perf_event_context *ctx)
2563 {
2564 	struct perf_event *event, *partial_group = NULL;
2565 	struct pmu *pmu = ctx->pmu;
2566 
2567 	if (group_event->state == PERF_EVENT_STATE_OFF)
2568 		return 0;
2569 
2570 	pmu->start_txn(pmu, PERF_PMU_TXN_ADD);
2571 
2572 	if (event_sched_in(group_event, cpuctx, ctx))
2573 		goto error;
2574 
2575 	/*
2576 	 * Schedule in siblings as one group (if any):
2577 	 */
2578 	for_each_sibling_event(event, group_event) {
2579 		if (event_sched_in(event, cpuctx, ctx)) {
2580 			partial_group = event;
2581 			goto group_error;
2582 		}
2583 	}
2584 
2585 	if (!pmu->commit_txn(pmu))
2586 		return 0;
2587 
2588 group_error:
2589 	/*
2590 	 * Groups can be scheduled in as one unit only, so undo any
2591 	 * partial group before returning:
2592 	 * The events up to the failed event are scheduled out normally.
2593 	 */
2594 	for_each_sibling_event(event, group_event) {
2595 		if (event == partial_group)
2596 			break;
2597 
2598 		event_sched_out(event, cpuctx, ctx);
2599 	}
2600 	event_sched_out(group_event, cpuctx, ctx);
2601 
2602 error:
2603 	pmu->cancel_txn(pmu);
2604 	return -EAGAIN;
2605 }
2606 
2607 /*
2608  * Work out whether we can put this event group on the CPU now.
2609  */
2610 static int group_can_go_on(struct perf_event *event,
2611 			   struct perf_cpu_context *cpuctx,
2612 			   int can_add_hw)
2613 {
2614 	/*
2615 	 * Groups consisting entirely of software events can always go on.
2616 	 */
2617 	if (event->group_caps & PERF_EV_CAP_SOFTWARE)
2618 		return 1;
2619 	/*
2620 	 * If an exclusive group is already on, no other hardware
2621 	 * events can go on.
2622 	 */
2623 	if (cpuctx->exclusive)
2624 		return 0;
2625 	/*
2626 	 * If this group is exclusive and there are already
2627 	 * events on the CPU, it can't go on.
2628 	 */
2629 	if (event->attr.exclusive && !list_empty(get_event_list(event)))
2630 		return 0;
2631 	/*
2632 	 * Otherwise, try to add it if all previous groups were able
2633 	 * to go on.
2634 	 */
2635 	return can_add_hw;
2636 }
2637 
2638 static void add_event_to_ctx(struct perf_event *event,
2639 			       struct perf_event_context *ctx)
2640 {
2641 	list_add_event(event, ctx);
2642 	perf_group_attach(event);
2643 }
2644 
2645 static void ctx_sched_out(struct perf_event_context *ctx,
2646 			  struct perf_cpu_context *cpuctx,
2647 			  enum event_type_t event_type);
2648 static void
2649 ctx_sched_in(struct perf_event_context *ctx,
2650 	     struct perf_cpu_context *cpuctx,
2651 	     enum event_type_t event_type);
2652 
2653 static void task_ctx_sched_out(struct perf_cpu_context *cpuctx,
2654 			       struct perf_event_context *ctx,
2655 			       enum event_type_t event_type)
2656 {
2657 	if (!cpuctx->task_ctx)
2658 		return;
2659 
2660 	if (WARN_ON_ONCE(ctx != cpuctx->task_ctx))
2661 		return;
2662 
2663 	ctx_sched_out(ctx, cpuctx, event_type);
2664 }
2665 
2666 static void perf_event_sched_in(struct perf_cpu_context *cpuctx,
2667 				struct perf_event_context *ctx)
2668 {
2669 	cpu_ctx_sched_in(cpuctx, EVENT_PINNED);
2670 	if (ctx)
2671 		ctx_sched_in(ctx, cpuctx, EVENT_PINNED);
2672 	cpu_ctx_sched_in(cpuctx, EVENT_FLEXIBLE);
2673 	if (ctx)
2674 		ctx_sched_in(ctx, cpuctx, EVENT_FLEXIBLE);
2675 }
2676 
2677 /*
2678  * We want to maintain the following priority of scheduling:
2679  *  - CPU pinned (EVENT_CPU | EVENT_PINNED)
2680  *  - task pinned (EVENT_PINNED)
2681  *  - CPU flexible (EVENT_CPU | EVENT_FLEXIBLE)
2682  *  - task flexible (EVENT_FLEXIBLE).
2683  *
2684  * In order to avoid unscheduling and scheduling back in everything every
2685  * time an event is added, only do it for the groups of equal priority and
2686  * below.
2687  *
2688  * This can be called after a batch operation on task events, in which case
2689  * event_type is a bit mask of the types of events involved. For CPU events,
2690  * event_type is only either EVENT_PINNED or EVENT_FLEXIBLE.
2691  */
2692 static void ctx_resched(struct perf_cpu_context *cpuctx,
2693 			struct perf_event_context *task_ctx,
2694 			enum event_type_t event_type)
2695 {
2696 	enum event_type_t ctx_event_type;
2697 	bool cpu_event = !!(event_type & EVENT_CPU);
2698 
2699 	/*
2700 	 * If pinned groups are involved, flexible groups also need to be
2701 	 * scheduled out.
2702 	 */
2703 	if (event_type & EVENT_PINNED)
2704 		event_type |= EVENT_FLEXIBLE;
2705 
2706 	ctx_event_type = event_type & EVENT_ALL;
2707 
2708 	perf_pmu_disable(cpuctx->ctx.pmu);
2709 	if (task_ctx)
2710 		task_ctx_sched_out(cpuctx, task_ctx, event_type);
2711 
2712 	/*
2713 	 * Decide which cpu ctx groups to schedule out based on the types
2714 	 * of events that caused rescheduling:
2715 	 *  - EVENT_CPU: schedule out corresponding groups;
2716 	 *  - EVENT_PINNED task events: schedule out EVENT_FLEXIBLE groups;
2717 	 *  - otherwise, do nothing more.
2718 	 */
2719 	if (cpu_event)
2720 		cpu_ctx_sched_out(cpuctx, ctx_event_type);
2721 	else if (ctx_event_type & EVENT_PINNED)
2722 		cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);
2723 
2724 	perf_event_sched_in(cpuctx, task_ctx);
2725 	perf_pmu_enable(cpuctx->ctx.pmu);
2726 }
2727 
2728 void perf_pmu_resched(struct pmu *pmu)
2729 {
2730 	struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
2731 	struct perf_event_context *task_ctx = cpuctx->task_ctx;
2732 
2733 	perf_ctx_lock(cpuctx, task_ctx);
2734 	ctx_resched(cpuctx, task_ctx, EVENT_ALL|EVENT_CPU);
2735 	perf_ctx_unlock(cpuctx, task_ctx);
2736 }
2737 
2738 /*
2739  * Cross CPU call to install and enable a performance event
2740  *
2741  * Very similar to remote_function() + event_function() but cannot assume that
2742  * things like ctx->is_active and cpuctx->task_ctx are set.
2743  */
2744 static int  __perf_install_in_context(void *info)
2745 {
2746 	struct perf_event *event = info;
2747 	struct perf_event_context *ctx = event->ctx;
2748 	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
2749 	struct perf_event_context *task_ctx = cpuctx->task_ctx;
2750 	bool reprogram = true;
2751 	int ret = 0;
2752 
2753 	raw_spin_lock(&cpuctx->ctx.lock);
2754 	if (ctx->task) {
2755 		raw_spin_lock(&ctx->lock);
2756 		task_ctx = ctx;
2757 
2758 		reprogram = (ctx->task == current);
2759 
2760 		/*
2761 		 * If the task is running, it must be running on this CPU,
2762 		 * otherwise we cannot reprogram things.
2763 		 *
2764 		 * If its not running, we don't care, ctx->lock will
2765 		 * serialize against it becoming runnable.
2766 		 */
2767 		if (task_curr(ctx->task) && !reprogram) {
2768 			ret = -ESRCH;
2769 			goto unlock;
2770 		}
2771 
2772 		WARN_ON_ONCE(reprogram && cpuctx->task_ctx && cpuctx->task_ctx != ctx);
2773 	} else if (task_ctx) {
2774 		raw_spin_lock(&task_ctx->lock);
2775 	}
2776 
2777 #ifdef CONFIG_CGROUP_PERF
2778 	if (event->state > PERF_EVENT_STATE_OFF && is_cgroup_event(event)) {
2779 		/*
2780 		 * If the current cgroup doesn't match the event's
2781 		 * cgroup, we should not try to schedule it.
2782 		 */
2783 		struct perf_cgroup *cgrp = perf_cgroup_from_task(current, ctx);
2784 		reprogram = cgroup_is_descendant(cgrp->css.cgroup,
2785 					event->cgrp->css.cgroup);
2786 	}
2787 #endif
2788 
2789 	if (reprogram) {
2790 		ctx_sched_out(ctx, cpuctx, EVENT_TIME);
2791 		add_event_to_ctx(event, ctx);
2792 		ctx_resched(cpuctx, task_ctx, get_event_type(event));
2793 	} else {
2794 		add_event_to_ctx(event, ctx);
2795 	}
2796 
2797 unlock:
2798 	perf_ctx_unlock(cpuctx, task_ctx);
2799 
2800 	return ret;
2801 }
2802 
2803 static bool exclusive_event_installable(struct perf_event *event,
2804 					struct perf_event_context *ctx);
2805 
2806 /*
2807  * Attach a performance event to a context.
2808  *
2809  * Very similar to event_function_call, see comment there.
2810  */
2811 static void
2812 perf_install_in_context(struct perf_event_context *ctx,
2813 			struct perf_event *event,
2814 			int cpu)
2815 {
2816 	struct task_struct *task = READ_ONCE(ctx->task);
2817 
2818 	lockdep_assert_held(&ctx->mutex);
2819 
2820 	WARN_ON_ONCE(!exclusive_event_installable(event, ctx));
2821 
2822 	if (event->cpu != -1)
2823 		event->cpu = cpu;
2824 
2825 	/*
2826 	 * Ensures that if we can observe event->ctx, both the event and ctx
2827 	 * will be 'complete'. See perf_iterate_sb_cpu().
2828 	 */
2829 	smp_store_release(&event->ctx, ctx);
2830 
2831 	/*
2832 	 * perf_event_attr::disabled events will not run and can be initialized
2833 	 * without IPI. Except when this is the first event for the context, in
2834 	 * that case we need the magic of the IPI to set ctx->is_active.
2835 	 * Similarly, cgroup events for the context also needs the IPI to
2836 	 * manipulate the cgrp_cpuctx_list.
2837 	 *
2838 	 * The IOC_ENABLE that is sure to follow the creation of a disabled
2839 	 * event will issue the IPI and reprogram the hardware.
2840 	 */
2841 	if (__perf_effective_state(event) == PERF_EVENT_STATE_OFF &&
2842 	    ctx->nr_events && !is_cgroup_event(event)) {
2843 		raw_spin_lock_irq(&ctx->lock);
2844 		if (ctx->task == TASK_TOMBSTONE) {
2845 			raw_spin_unlock_irq(&ctx->lock);
2846 			return;
2847 		}
2848 		add_event_to_ctx(event, ctx);
2849 		raw_spin_unlock_irq(&ctx->lock);
2850 		return;
2851 	}
2852 
2853 	if (!task) {
2854 		cpu_function_call(cpu, __perf_install_in_context, event);
2855 		return;
2856 	}
2857 
2858 	/*
2859 	 * Should not happen, we validate the ctx is still alive before calling.
2860 	 */
2861 	if (WARN_ON_ONCE(task == TASK_TOMBSTONE))
2862 		return;
2863 
2864 	/*
2865 	 * Installing events is tricky because we cannot rely on ctx->is_active
2866 	 * to be set in case this is the nr_events 0 -> 1 transition.
2867 	 *
2868 	 * Instead we use task_curr(), which tells us if the task is running.
2869 	 * However, since we use task_curr() outside of rq::lock, we can race
2870 	 * against the actual state. This means the result can be wrong.
2871 	 *
2872 	 * If we get a false positive, we retry, this is harmless.
2873 	 *
2874 	 * If we get a false negative, things are complicated. If we are after
2875 	 * perf_event_context_sched_in() ctx::lock will serialize us, and the
2876 	 * value must be correct. If we're before, it doesn't matter since
2877 	 * perf_event_context_sched_in() will program the counter.
2878 	 *
2879 	 * However, this hinges on the remote context switch having observed
2880 	 * our task->perf_event_ctxp[] store, such that it will in fact take
2881 	 * ctx::lock in perf_event_context_sched_in().
2882 	 *
2883 	 * We do this by task_function_call(), if the IPI fails to hit the task
2884 	 * we know any future context switch of task must see the
2885 	 * perf_event_ctpx[] store.
2886 	 */
2887 
2888 	/*
2889 	 * This smp_mb() orders the task->perf_event_ctxp[] store with the
2890 	 * task_cpu() load, such that if the IPI then does not find the task
2891 	 * running, a future context switch of that task must observe the
2892 	 * store.
2893 	 */
2894 	smp_mb();
2895 again:
2896 	if (!task_function_call(task, __perf_install_in_context, event))
2897 		return;
2898 
2899 	raw_spin_lock_irq(&ctx->lock);
2900 	task = ctx->task;
2901 	if (WARN_ON_ONCE(task == TASK_TOMBSTONE)) {
2902 		/*
2903 		 * Cannot happen because we already checked above (which also
2904 		 * cannot happen), and we hold ctx->mutex, which serializes us
2905 		 * against perf_event_exit_task_context().
2906 		 */
2907 		raw_spin_unlock_irq(&ctx->lock);
2908 		return;
2909 	}
2910 	/*
2911 	 * If the task is not running, ctx->lock will avoid it becoming so,
2912 	 * thus we can safely install the event.
2913 	 */
2914 	if (task_curr(task)) {
2915 		raw_spin_unlock_irq(&ctx->lock);
2916 		goto again;
2917 	}
2918 	add_event_to_ctx(event, ctx);
2919 	raw_spin_unlock_irq(&ctx->lock);
2920 }
2921 
2922 /*
2923  * Cross CPU call to enable a performance event
2924  */
2925 static void __perf_event_enable(struct perf_event *event,
2926 				struct perf_cpu_context *cpuctx,
2927 				struct perf_event_context *ctx,
2928 				void *info)
2929 {
2930 	struct perf_event *leader = event->group_leader;
2931 	struct perf_event_context *task_ctx;
2932 
2933 	if (event->state >= PERF_EVENT_STATE_INACTIVE ||
2934 	    event->state <= PERF_EVENT_STATE_ERROR)
2935 		return;
2936 
2937 	if (ctx->is_active)
2938 		ctx_sched_out(ctx, cpuctx, EVENT_TIME);
2939 
2940 	perf_event_set_state(event, PERF_EVENT_STATE_INACTIVE);
2941 	perf_cgroup_event_enable(event, ctx);
2942 
2943 	if (!ctx->is_active)
2944 		return;
2945 
2946 	if (!event_filter_match(event)) {
2947 		ctx_sched_in(ctx, cpuctx, EVENT_TIME);
2948 		return;
2949 	}
2950 
2951 	/*
2952 	 * If the event is in a group and isn't the group leader,
2953 	 * then don't put it on unless the group is on.
2954 	 */
2955 	if (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE) {
2956 		ctx_sched_in(ctx, cpuctx, EVENT_TIME);
2957 		return;
2958 	}
2959 
2960 	task_ctx = cpuctx->task_ctx;
2961 	if (ctx->task)
2962 		WARN_ON_ONCE(task_ctx != ctx);
2963 
2964 	ctx_resched(cpuctx, task_ctx, get_event_type(event));
2965 }
2966 
2967 /*
2968  * Enable an event.
2969  *
2970  * If event->ctx is a cloned context, callers must make sure that
2971  * every task struct that event->ctx->task could possibly point to
2972  * remains valid.  This condition is satisfied when called through
2973  * perf_event_for_each_child or perf_event_for_each as described
2974  * for perf_event_disable.
2975  */
2976 static void _perf_event_enable(struct perf_event *event)
2977 {
2978 	struct perf_event_context *ctx = event->ctx;
2979 
2980 	raw_spin_lock_irq(&ctx->lock);
2981 	if (event->state >= PERF_EVENT_STATE_INACTIVE ||
2982 	    event->state <  PERF_EVENT_STATE_ERROR) {
2983 out:
2984 		raw_spin_unlock_irq(&ctx->lock);
2985 		return;
2986 	}
2987 
2988 	/*
2989 	 * If the event is in error state, clear that first.
2990 	 *
2991 	 * That way, if we see the event in error state below, we know that it
2992 	 * has gone back into error state, as distinct from the task having
2993 	 * been scheduled away before the cross-call arrived.
2994 	 */
2995 	if (event->state == PERF_EVENT_STATE_ERROR) {
2996 		/*
2997 		 * Detached SIBLING events cannot leave ERROR state.
2998 		 */
2999 		if (event->event_caps & PERF_EV_CAP_SIBLING &&
3000 		    event->group_leader == event)
3001 			goto out;
3002 
3003 		event->state = PERF_EVENT_STATE_OFF;
3004 	}
3005 	raw_spin_unlock_irq(&ctx->lock);
3006 
3007 	event_function_call(event, __perf_event_enable, NULL);
3008 }
3009 
3010 /*
3011  * See perf_event_disable();
3012  */
3013 void perf_event_enable(struct perf_event *event)
3014 {
3015 	struct perf_event_context *ctx;
3016 
3017 	ctx = perf_event_ctx_lock(event);
3018 	_perf_event_enable(event);
3019 	perf_event_ctx_unlock(event, ctx);
3020 }
3021 EXPORT_SYMBOL_GPL(perf_event_enable);
3022 
3023 struct stop_event_data {
3024 	struct perf_event	*event;
3025 	unsigned int		restart;
3026 };
3027 
3028 static int __perf_event_stop(void *info)
3029 {
3030 	struct stop_event_data *sd = info;
3031 	struct perf_event *event = sd->event;
3032 
3033 	/* if it's already INACTIVE, do nothing */
3034 	if (READ_ONCE(event->state) != PERF_EVENT_STATE_ACTIVE)
3035 		return 0;
3036 
3037 	/* matches smp_wmb() in event_sched_in() */
3038 	smp_rmb();
3039 
3040 	/*
3041 	 * There is a window with interrupts enabled before we get here,
3042 	 * so we need to check again lest we try to stop another CPU's event.
3043 	 */
3044 	if (READ_ONCE(event->oncpu) != smp_processor_id())
3045 		return -EAGAIN;
3046 
3047 	event->pmu->stop(event, PERF_EF_UPDATE);
3048 
3049 	/*
3050 	 * May race with the actual stop (through perf_pmu_output_stop()),
3051 	 * but it is only used for events with AUX ring buffer, and such
3052 	 * events will refuse to restart because of rb::aux_mmap_count==0,
3053 	 * see comments in perf_aux_output_begin().
3054 	 *
3055 	 * Since this is happening on an event-local CPU, no trace is lost
3056 	 * while restarting.
3057 	 */
3058 	if (sd->restart)
3059 		event->pmu->start(event, 0);
3060 
3061 	return 0;
3062 }
3063 
3064 static int perf_event_stop(struct perf_event *event, int restart)
3065 {
3066 	struct stop_event_data sd = {
3067 		.event		= event,
3068 		.restart	= restart,
3069 	};
3070 	int ret = 0;
3071 
3072 	do {
3073 		if (READ_ONCE(event->state) != PERF_EVENT_STATE_ACTIVE)
3074 			return 0;
3075 
3076 		/* matches smp_wmb() in event_sched_in() */
3077 		smp_rmb();
3078 
3079 		/*
3080 		 * We only want to restart ACTIVE events, so if the event goes
3081 		 * inactive here (event->oncpu==-1), there's nothing more to do;
3082 		 * fall through with ret==-ENXIO.
3083 		 */
3084 		ret = cpu_function_call(READ_ONCE(event->oncpu),
3085 					__perf_event_stop, &sd);
3086 	} while (ret == -EAGAIN);
3087 
3088 	return ret;
3089 }
3090 
3091 /*
3092  * In order to contain the amount of racy and tricky in the address filter
3093  * configuration management, it is a two part process:
3094  *
3095  * (p1) when userspace mappings change as a result of (1) or (2) or (3) below,
3096  *      we update the addresses of corresponding vmas in
3097  *	event::addr_filter_ranges array and bump the event::addr_filters_gen;
3098  * (p2) when an event is scheduled in (pmu::add), it calls
3099  *      perf_event_addr_filters_sync() which calls pmu::addr_filters_sync()
3100  *      if the generation has changed since the previous call.
3101  *
3102  * If (p1) happens while the event is active, we restart it to force (p2).
3103  *
3104  * (1) perf_addr_filters_apply(): adjusting filters' offsets based on
3105  *     pre-existing mappings, called once when new filters arrive via SET_FILTER
3106  *     ioctl;
3107  * (2) perf_addr_filters_adjust(): adjusting filters' offsets based on newly
3108  *     registered mapping, called for every new mmap(), with mm::mmap_lock down
3109  *     for reading;
3110  * (3) perf_event_addr_filters_exec(): clearing filters' offsets in the process
3111  *     of exec.
3112  */
3113 void perf_event_addr_filters_sync(struct perf_event *event)
3114 {
3115 	struct perf_addr_filters_head *ifh = perf_event_addr_filters(event);
3116 
3117 	if (!has_addr_filter(event))
3118 		return;
3119 
3120 	raw_spin_lock(&ifh->lock);
3121 	if (event->addr_filters_gen != event->hw.addr_filters_gen) {
3122 		event->pmu->addr_filters_sync(event);
3123 		event->hw.addr_filters_gen = event->addr_filters_gen;
3124 	}
3125 	raw_spin_unlock(&ifh->lock);
3126 }
3127 EXPORT_SYMBOL_GPL(perf_event_addr_filters_sync);
3128 
3129 static int _perf_event_refresh(struct perf_event *event, int refresh)
3130 {
3131 	/*
3132 	 * not supported on inherited events
3133 	 */
3134 	if (event->attr.inherit || !is_sampling_event(event))
3135 		return -EINVAL;
3136 
3137 	atomic_add(refresh, &event->event_limit);
3138 	_perf_event_enable(event);
3139 
3140 	return 0;
3141 }
3142 
3143 /*
3144  * See perf_event_disable()
3145  */
3146 int perf_event_refresh(struct perf_event *event, int refresh)
3147 {
3148 	struct perf_event_context *ctx;
3149 	int ret;
3150 
3151 	ctx = perf_event_ctx_lock(event);
3152 	ret = _perf_event_refresh(event, refresh);
3153 	perf_event_ctx_unlock(event, ctx);
3154 
3155 	return ret;
3156 }
3157 EXPORT_SYMBOL_GPL(perf_event_refresh);
3158 
3159 static int perf_event_modify_breakpoint(struct perf_event *bp,
3160 					 struct perf_event_attr *attr)
3161 {
3162 	int err;
3163 
3164 	_perf_event_disable(bp);
3165 
3166 	err = modify_user_hw_breakpoint_check(bp, attr, true);
3167 
3168 	if (!bp->attr.disabled)
3169 		_perf_event_enable(bp);
3170 
3171 	return err;
3172 }
3173 
3174 /*
3175  * Copy event-type-independent attributes that may be modified.
3176  */
3177 static void perf_event_modify_copy_attr(struct perf_event_attr *to,
3178 					const struct perf_event_attr *from)
3179 {
3180 	to->sig_data = from->sig_data;
3181 }
3182 
3183 static int perf_event_modify_attr(struct perf_event *event,
3184 				  struct perf_event_attr *attr)
3185 {
3186 	int (*func)(struct perf_event *, struct perf_event_attr *);
3187 	struct perf_event *child;
3188 	int err;
3189 
3190 	if (event->attr.type != attr->type)
3191 		return -EINVAL;
3192 
3193 	switch (event->attr.type) {
3194 	case PERF_TYPE_BREAKPOINT:
3195 		func = perf_event_modify_breakpoint;
3196 		break;
3197 	default:
3198 		/* Place holder for future additions. */
3199 		return -EOPNOTSUPP;
3200 	}
3201 
3202 	WARN_ON_ONCE(event->ctx->parent_ctx);
3203 
3204 	mutex_lock(&event->child_mutex);
3205 	/*
3206 	 * Event-type-independent attributes must be copied before event-type
3207 	 * modification, which will validate that final attributes match the
3208 	 * source attributes after all relevant attributes have been copied.
3209 	 */
3210 	perf_event_modify_copy_attr(&event->attr, attr);
3211 	err = func(event, attr);
3212 	if (err)
3213 		goto out;
3214 	list_for_each_entry(child, &event->child_list, child_list) {
3215 		perf_event_modify_copy_attr(&child->attr, attr);
3216 		err = func(child, attr);
3217 		if (err)
3218 			goto out;
3219 	}
3220 out:
3221 	mutex_unlock(&event->child_mutex);
3222 	return err;
3223 }
3224 
3225 static void ctx_sched_out(struct perf_event_context *ctx,
3226 			  struct perf_cpu_context *cpuctx,
3227 			  enum event_type_t event_type)
3228 {
3229 	struct perf_event *event, *tmp;
3230 	int is_active = ctx->is_active;
3231 
3232 	lockdep_assert_held(&ctx->lock);
3233 
3234 	if (likely(!ctx->nr_events)) {
3235 		/*
3236 		 * See __perf_remove_from_context().
3237 		 */
3238 		WARN_ON_ONCE(ctx->is_active);
3239 		if (ctx->task)
3240 			WARN_ON_ONCE(cpuctx->task_ctx);
3241 		return;
3242 	}
3243 
3244 	/*
3245 	 * Always update time if it was set; not only when it changes.
3246 	 * Otherwise we can 'forget' to update time for any but the last
3247 	 * context we sched out. For example:
3248 	 *
3249 	 *   ctx_sched_out(.event_type = EVENT_FLEXIBLE)
3250 	 *   ctx_sched_out(.event_type = EVENT_PINNED)
3251 	 *
3252 	 * would only update time for the pinned events.
3253 	 */
3254 	if (is_active & EVENT_TIME) {
3255 		/* update (and stop) ctx time */
3256 		update_context_time(ctx);
3257 		update_cgrp_time_from_cpuctx(cpuctx, ctx == &cpuctx->ctx);
3258 		/*
3259 		 * CPU-release for the below ->is_active store,
3260 		 * see __load_acquire() in perf_event_time_now()
3261 		 */
3262 		barrier();
3263 	}
3264 
3265 	ctx->is_active &= ~event_type;
3266 	if (!(ctx->is_active & EVENT_ALL))
3267 		ctx->is_active = 0;
3268 
3269 	if (ctx->task) {
3270 		WARN_ON_ONCE(cpuctx->task_ctx != ctx);
3271 		if (!ctx->is_active)
3272 			cpuctx->task_ctx = NULL;
3273 	}
3274 
3275 	is_active ^= ctx->is_active; /* changed bits */
3276 
3277 	if (!ctx->nr_active || !(is_active & EVENT_ALL))
3278 		return;
3279 
3280 	perf_pmu_disable(ctx->pmu);
3281 	if (is_active & EVENT_PINNED) {
3282 		list_for_each_entry_safe(event, tmp, &ctx->pinned_active, active_list)
3283 			group_sched_out(event, cpuctx, ctx);
3284 	}
3285 
3286 	if (is_active & EVENT_FLEXIBLE) {
3287 		list_for_each_entry_safe(event, tmp, &ctx->flexible_active, active_list)
3288 			group_sched_out(event, cpuctx, ctx);
3289 
3290 		/*
3291 		 * Since we cleared EVENT_FLEXIBLE, also clear
3292 		 * rotate_necessary, is will be reset by
3293 		 * ctx_flexible_sched_in() when needed.
3294 		 */
3295 		ctx->rotate_necessary = 0;
3296 	}
3297 	perf_pmu_enable(ctx->pmu);
3298 }
3299 
3300 /*
3301  * Test whether two contexts are equivalent, i.e. whether they have both been
3302  * cloned from the same version of the same context.
3303  *
3304  * Equivalence is measured using a generation number in the context that is
3305  * incremented on each modification to it; see unclone_ctx(), list_add_event()
3306  * and list_del_event().
3307  */
3308 static int context_equiv(struct perf_event_context *ctx1,
3309 			 struct perf_event_context *ctx2)
3310 {
3311 	lockdep_assert_held(&ctx1->lock);
3312 	lockdep_assert_held(&ctx2->lock);
3313 
3314 	/* Pinning disables the swap optimization */
3315 	if (ctx1->pin_count || ctx2->pin_count)
3316 		return 0;
3317 
3318 	/* If ctx1 is the parent of ctx2 */
3319 	if (ctx1 == ctx2->parent_ctx && ctx1->generation == ctx2->parent_gen)
3320 		return 1;
3321 
3322 	/* If ctx2 is the parent of ctx1 */
3323 	if (ctx1->parent_ctx == ctx2 && ctx1->parent_gen == ctx2->generation)
3324 		return 1;
3325 
3326 	/*
3327 	 * If ctx1 and ctx2 have the same parent; we flatten the parent
3328 	 * hierarchy, see perf_event_init_context().
3329 	 */
3330 	if (ctx1->parent_ctx && ctx1->parent_ctx == ctx2->parent_ctx &&
3331 			ctx1->parent_gen == ctx2->parent_gen)
3332 		return 1;
3333 
3334 	/* Unmatched */
3335 	return 0;
3336 }
3337 
3338 static void __perf_event_sync_stat(struct perf_event *event,
3339 				     struct perf_event *next_event)
3340 {
3341 	u64 value;
3342 
3343 	if (!event->attr.inherit_stat)
3344 		return;
3345 
3346 	/*
3347 	 * Update the event value, we cannot use perf_event_read()
3348 	 * because we're in the middle of a context switch and have IRQs
3349 	 * disabled, which upsets smp_call_function_single(), however
3350 	 * we know the event must be on the current CPU, therefore we
3351 	 * don't need to use it.
3352 	 */
3353 	if (event->state == PERF_EVENT_STATE_ACTIVE)
3354 		event->pmu->read(event);
3355 
3356 	perf_event_update_time(event);
3357 
3358 	/*
3359 	 * In order to keep per-task stats reliable we need to flip the event
3360 	 * values when we flip the contexts.
3361 	 */
3362 	value = local64_read(&next_event->count);
3363 	value = local64_xchg(&event->count, value);
3364 	local64_set(&next_event->count, value);
3365 
3366 	swap(event->total_time_enabled, next_event->total_time_enabled);
3367 	swap(event->total_time_running, next_event->total_time_running);
3368 
3369 	/*
3370 	 * Since we swizzled the values, update the user visible data too.
3371 	 */
3372 	perf_event_update_userpage(event);
3373 	perf_event_update_userpage(next_event);
3374 }
3375 
3376 static void perf_event_sync_stat(struct perf_event_context *ctx,
3377 				   struct perf_event_context *next_ctx)
3378 {
3379 	struct perf_event *event, *next_event;
3380 
3381 	if (!ctx->nr_stat)
3382 		return;
3383 
3384 	update_context_time(ctx);
3385 
3386 	event = list_first_entry(&ctx->event_list,
3387 				   struct perf_event, event_entry);
3388 
3389 	next_event = list_first_entry(&next_ctx->event_list,
3390 					struct perf_event, event_entry);
3391 
3392 	while (&event->event_entry != &ctx->event_list &&
3393 	       &next_event->event_entry != &next_ctx->event_list) {
3394 
3395 		__perf_event_sync_stat(event, next_event);
3396 
3397 		event = list_next_entry(event, event_entry);
3398 		next_event = list_next_entry(next_event, event_entry);
3399 	}
3400 }
3401 
3402 static void perf_event_context_sched_out(struct task_struct *task, int ctxn,
3403 					 struct task_struct *next)
3404 {
3405 	struct perf_event_context *ctx = task->perf_event_ctxp[ctxn];
3406 	struct perf_event_context *next_ctx;
3407 	struct perf_event_context *parent, *next_parent;
3408 	struct perf_cpu_context *cpuctx;
3409 	int do_switch = 1;
3410 	struct pmu *pmu;
3411 
3412 	if (likely(!ctx))
3413 		return;
3414 
3415 	pmu = ctx->pmu;
3416 	cpuctx = __get_cpu_context(ctx);
3417 	if (!cpuctx->task_ctx)
3418 		return;
3419 
3420 	rcu_read_lock();
3421 	next_ctx = next->perf_event_ctxp[ctxn];
3422 	if (!next_ctx)
3423 		goto unlock;
3424 
3425 	parent = rcu_dereference(ctx->parent_ctx);
3426 	next_parent = rcu_dereference(next_ctx->parent_ctx);
3427 
3428 	/* If neither context have a parent context; they cannot be clones. */
3429 	if (!parent && !next_parent)
3430 		goto unlock;
3431 
3432 	if (next_parent == ctx || next_ctx == parent || next_parent == parent) {
3433 		/*
3434 		 * Looks like the two contexts are clones, so we might be
3435 		 * able to optimize the context switch.  We lock both
3436 		 * contexts and check that they are clones under the
3437 		 * lock (including re-checking that neither has been
3438 		 * uncloned in the meantime).  It doesn't matter which
3439 		 * order we take the locks because no other cpu could
3440 		 * be trying to lock both of these tasks.
3441 		 */
3442 		raw_spin_lock(&ctx->lock);
3443 		raw_spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING);
3444 		if (context_equiv(ctx, next_ctx)) {
3445 
3446 			perf_pmu_disable(pmu);
3447 
3448 			/* PMIs are disabled; ctx->nr_pending is stable. */
3449 			if (local_read(&ctx->nr_pending) ||
3450 			    local_read(&next_ctx->nr_pending)) {
3451 				/*
3452 				 * Must not swap out ctx when there's pending
3453 				 * events that rely on the ctx->task relation.
3454 				 */
3455 				raw_spin_unlock(&next_ctx->lock);
3456 				rcu_read_unlock();
3457 				goto inside_switch;
3458 			}
3459 
3460 			WRITE_ONCE(ctx->task, next);
3461 			WRITE_ONCE(next_ctx->task, task);
3462 
3463 			if (cpuctx->sched_cb_usage && pmu->sched_task)
3464 				pmu->sched_task(ctx, false);
3465 
3466 			/*
3467 			 * PMU specific parts of task perf context can require
3468 			 * additional synchronization. As an example of such
3469 			 * synchronization see implementation details of Intel
3470 			 * LBR call stack data profiling;
3471 			 */
3472 			if (pmu->swap_task_ctx)
3473 				pmu->swap_task_ctx(ctx, next_ctx);
3474 			else
3475 				swap(ctx->task_ctx_data, next_ctx->task_ctx_data);
3476 
3477 			perf_pmu_enable(pmu);
3478 
3479 			/*
3480 			 * RCU_INIT_POINTER here is safe because we've not
3481 			 * modified the ctx and the above modification of
3482 			 * ctx->task and ctx->task_ctx_data are immaterial
3483 			 * since those values are always verified under
3484 			 * ctx->lock which we're now holding.
3485 			 */
3486 			RCU_INIT_POINTER(task->perf_event_ctxp[ctxn], next_ctx);
3487 			RCU_INIT_POINTER(next->perf_event_ctxp[ctxn], ctx);
3488 
3489 			do_switch = 0;
3490 
3491 			perf_event_sync_stat(ctx, next_ctx);
3492 		}
3493 		raw_spin_unlock(&next_ctx->lock);
3494 		raw_spin_unlock(&ctx->lock);
3495 	}
3496 unlock:
3497 	rcu_read_unlock();
3498 
3499 	if (do_switch) {
3500 		raw_spin_lock(&ctx->lock);
3501 		perf_pmu_disable(pmu);
3502 
3503 inside_switch:
3504 		if (cpuctx->sched_cb_usage && pmu->sched_task)
3505 			pmu->sched_task(ctx, false);
3506 		task_ctx_sched_out(cpuctx, ctx, EVENT_ALL);
3507 
3508 		perf_pmu_enable(pmu);
3509 		raw_spin_unlock(&ctx->lock);
3510 	}
3511 }
3512 
3513 static DEFINE_PER_CPU(struct list_head, sched_cb_list);
3514 
3515 void perf_sched_cb_dec(struct pmu *pmu)
3516 {
3517 	struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
3518 
3519 	this_cpu_dec(perf_sched_cb_usages);
3520 
3521 	if (!--cpuctx->sched_cb_usage)
3522 		list_del(&cpuctx->sched_cb_entry);
3523 }
3524 
3525 
3526 void perf_sched_cb_inc(struct pmu *pmu)
3527 {
3528 	struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
3529 
3530 	if (!cpuctx->sched_cb_usage++)
3531 		list_add(&cpuctx->sched_cb_entry, this_cpu_ptr(&sched_cb_list));
3532 
3533 	this_cpu_inc(perf_sched_cb_usages);
3534 }
3535 
3536 /*
3537  * This function provides the context switch callback to the lower code
3538  * layer. It is invoked ONLY when the context switch callback is enabled.
3539  *
3540  * This callback is relevant even to per-cpu events; for example multi event
3541  * PEBS requires this to provide PID/TID information. This requires we flush
3542  * all queued PEBS records before we context switch to a new task.
3543  */
3544 static void __perf_pmu_sched_task(struct perf_cpu_context *cpuctx, bool sched_in)
3545 {
3546 	struct pmu *pmu;
3547 
3548 	pmu = cpuctx->ctx.pmu; /* software PMUs will not have sched_task */
3549 
3550 	if (WARN_ON_ONCE(!pmu->sched_task))
3551 		return;
3552 
3553 	perf_ctx_lock(cpuctx, cpuctx->task_ctx);
3554 	perf_pmu_disable(pmu);
3555 
3556 	pmu->sched_task(cpuctx->task_ctx, sched_in);
3557 
3558 	perf_pmu_enable(pmu);
3559 	perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
3560 }
3561 
3562 static void perf_pmu_sched_task(struct task_struct *prev,
3563 				struct task_struct *next,
3564 				bool sched_in)
3565 {
3566 	struct perf_cpu_context *cpuctx;
3567 
3568 	if (prev == next)
3569 		return;
3570 
3571 	list_for_each_entry(cpuctx, this_cpu_ptr(&sched_cb_list), sched_cb_entry) {
3572 		/* will be handled in perf_event_context_sched_in/out */
3573 		if (cpuctx->task_ctx)
3574 			continue;
3575 
3576 		__perf_pmu_sched_task(cpuctx, sched_in);
3577 	}
3578 }
3579 
3580 static void perf_event_switch(struct task_struct *task,
3581 			      struct task_struct *next_prev, bool sched_in);
3582 
3583 #define for_each_task_context_nr(ctxn)					\
3584 	for ((ctxn) = 0; (ctxn) < perf_nr_task_contexts; (ctxn)++)
3585 
3586 /*
3587  * Called from scheduler to remove the events of the current task,
3588  * with interrupts disabled.
3589  *
3590  * We stop each event and update the event value in event->count.
3591  *
3592  * This does not protect us against NMI, but disable()
3593  * sets the disabled bit in the control field of event _before_
3594  * accessing the event control register. If a NMI hits, then it will
3595  * not restart the event.
3596  */
3597 void __perf_event_task_sched_out(struct task_struct *task,
3598 				 struct task_struct *next)
3599 {
3600 	int ctxn;
3601 
3602 	if (__this_cpu_read(perf_sched_cb_usages))
3603 		perf_pmu_sched_task(task, next, false);
3604 
3605 	if (atomic_read(&nr_switch_events))
3606 		perf_event_switch(task, next, false);
3607 
3608 	for_each_task_context_nr(ctxn)
3609 		perf_event_context_sched_out(task, ctxn, next);
3610 
3611 	/*
3612 	 * if cgroup events exist on this CPU, then we need
3613 	 * to check if we have to switch out PMU state.
3614 	 * cgroup event are system-wide mode only
3615 	 */
3616 	if (atomic_read(this_cpu_ptr(&perf_cgroup_events)))
3617 		perf_cgroup_switch(next);
3618 }
3619 
3620 /*
3621  * Called with IRQs disabled
3622  */
3623 static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,
3624 			      enum event_type_t event_type)
3625 {
3626 	ctx_sched_out(&cpuctx->ctx, cpuctx, event_type);
3627 }
3628 
3629 static bool perf_less_group_idx(const void *l, const void *r)
3630 {
3631 	const struct perf_event *le = *(const struct perf_event **)l;
3632 	const struct perf_event *re = *(const struct perf_event **)r;
3633 
3634 	return le->group_index < re->group_index;
3635 }
3636 
3637 static void swap_ptr(void *l, void *r)
3638 {
3639 	void **lp = l, **rp = r;
3640 
3641 	swap(*lp, *rp);
3642 }
3643 
3644 static const struct min_heap_callbacks perf_min_heap = {
3645 	.elem_size = sizeof(struct perf_event *),
3646 	.less = perf_less_group_idx,
3647 	.swp = swap_ptr,
3648 };
3649 
3650 static void __heap_add(struct min_heap *heap, struct perf_event *event)
3651 {
3652 	struct perf_event **itrs = heap->data;
3653 
3654 	if (event) {
3655 		itrs[heap->nr] = event;
3656 		heap->nr++;
3657 	}
3658 }
3659 
3660 static noinline int visit_groups_merge(struct perf_cpu_context *cpuctx,
3661 				struct perf_event_groups *groups, int cpu,
3662 				int (*func)(struct perf_event *, void *),
3663 				void *data)
3664 {
3665 #ifdef CONFIG_CGROUP_PERF
3666 	struct cgroup_subsys_state *css = NULL;
3667 #endif
3668 	/* Space for per CPU and/or any CPU event iterators. */
3669 	struct perf_event *itrs[2];
3670 	struct min_heap event_heap;
3671 	struct perf_event **evt;
3672 	int ret;
3673 
3674 	if (cpuctx) {
3675 		event_heap = (struct min_heap){
3676 			.data = cpuctx->heap,
3677 			.nr = 0,
3678 			.size = cpuctx->heap_size,
3679 		};
3680 
3681 		lockdep_assert_held(&cpuctx->ctx.lock);
3682 
3683 #ifdef CONFIG_CGROUP_PERF
3684 		if (cpuctx->cgrp)
3685 			css = &cpuctx->cgrp->css;
3686 #endif
3687 	} else {
3688 		event_heap = (struct min_heap){
3689 			.data = itrs,
3690 			.nr = 0,
3691 			.size = ARRAY_SIZE(itrs),
3692 		};
3693 		/* Events not within a CPU context may be on any CPU. */
3694 		__heap_add(&event_heap, perf_event_groups_first(groups, -1, NULL));
3695 	}
3696 	evt = event_heap.data;
3697 
3698 	__heap_add(&event_heap, perf_event_groups_first(groups, cpu, NULL));
3699 
3700 #ifdef CONFIG_CGROUP_PERF
3701 	for (; css; css = css->parent)
3702 		__heap_add(&event_heap, perf_event_groups_first(groups, cpu, css->cgroup));
3703 #endif
3704 
3705 	min_heapify_all(&event_heap, &perf_min_heap);
3706 
3707 	while (event_heap.nr) {
3708 		ret = func(*evt, data);
3709 		if (ret)
3710 			return ret;
3711 
3712 		*evt = perf_event_groups_next(*evt);
3713 		if (*evt)
3714 			min_heapify(&event_heap, 0, &perf_min_heap);
3715 		else
3716 			min_heap_pop(&event_heap, &perf_min_heap);
3717 	}
3718 
3719 	return 0;
3720 }
3721 
3722 /*
3723  * Because the userpage is strictly per-event (there is no concept of context,
3724  * so there cannot be a context indirection), every userpage must be updated
3725  * when context time starts :-(
3726  *
3727  * IOW, we must not miss EVENT_TIME edges.
3728  */
3729 static inline bool event_update_userpage(struct perf_event *event)
3730 {
3731 	if (likely(!atomic_read(&event->mmap_count)))
3732 		return false;
3733 
3734 	perf_event_update_time(event);
3735 	perf_event_update_userpage(event);
3736 
3737 	return true;
3738 }
3739 
3740 static inline void group_update_userpage(struct perf_event *group_event)
3741 {
3742 	struct perf_event *event;
3743 
3744 	if (!event_update_userpage(group_event))
3745 		return;
3746 
3747 	for_each_sibling_event(event, group_event)
3748 		event_update_userpage(event);
3749 }
3750 
3751 static int merge_sched_in(struct perf_event *event, void *data)
3752 {
3753 	struct perf_event_context *ctx = event->ctx;
3754 	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
3755 	int *can_add_hw = data;
3756 
3757 	if (event->state <= PERF_EVENT_STATE_OFF)
3758 		return 0;
3759 
3760 	if (!event_filter_match(event))
3761 		return 0;
3762 
3763 	if (group_can_go_on(event, cpuctx, *can_add_hw)) {
3764 		if (!group_sched_in(event, cpuctx, ctx))
3765 			list_add_tail(&event->active_list, get_event_list(event));
3766 	}
3767 
3768 	if (event->state == PERF_EVENT_STATE_INACTIVE) {
3769 		*can_add_hw = 0;
3770 		if (event->attr.pinned) {
3771 			perf_cgroup_event_disable(event, ctx);
3772 			perf_event_set_state(event, PERF_EVENT_STATE_ERROR);
3773 		} else {
3774 			ctx->rotate_necessary = 1;
3775 			perf_mux_hrtimer_restart(cpuctx);
3776 			group_update_userpage(event);
3777 		}
3778 	}
3779 
3780 	return 0;
3781 }
3782 
3783 static void
3784 ctx_pinned_sched_in(struct perf_event_context *ctx,
3785 		    struct perf_cpu_context *cpuctx)
3786 {
3787 	int can_add_hw = 1;
3788 
3789 	if (ctx != &cpuctx->ctx)
3790 		cpuctx = NULL;
3791 
3792 	visit_groups_merge(cpuctx, &ctx->pinned_groups,
3793 			   smp_processor_id(),
3794 			   merge_sched_in, &can_add_hw);
3795 }
3796 
3797 static void
3798 ctx_flexible_sched_in(struct perf_event_context *ctx,
3799 		      struct perf_cpu_context *cpuctx)
3800 {
3801 	int can_add_hw = 1;
3802 
3803 	if (ctx != &cpuctx->ctx)
3804 		cpuctx = NULL;
3805 
3806 	visit_groups_merge(cpuctx, &ctx->flexible_groups,
3807 			   smp_processor_id(),
3808 			   merge_sched_in, &can_add_hw);
3809 }
3810 
3811 static void
3812 ctx_sched_in(struct perf_event_context *ctx,
3813 	     struct perf_cpu_context *cpuctx,
3814 	     enum event_type_t event_type)
3815 {
3816 	int is_active = ctx->is_active;
3817 
3818 	lockdep_assert_held(&ctx->lock);
3819 
3820 	if (likely(!ctx->nr_events))
3821 		return;
3822 
3823 	if (is_active ^ EVENT_TIME) {
3824 		/* start ctx time */
3825 		__update_context_time(ctx, false);
3826 		perf_cgroup_set_timestamp(cpuctx);
3827 		/*
3828 		 * CPU-release for the below ->is_active store,
3829 		 * see __load_acquire() in perf_event_time_now()
3830 		 */
3831 		barrier();
3832 	}
3833 
3834 	ctx->is_active |= (event_type | EVENT_TIME);
3835 	if (ctx->task) {
3836 		if (!is_active)
3837 			cpuctx->task_ctx = ctx;
3838 		else
3839 			WARN_ON_ONCE(cpuctx->task_ctx != ctx);
3840 	}
3841 
3842 	is_active ^= ctx->is_active; /* changed bits */
3843 
3844 	/*
3845 	 * First go through the list and put on any pinned groups
3846 	 * in order to give them the best chance of going on.
3847 	 */
3848 	if (is_active & EVENT_PINNED)
3849 		ctx_pinned_sched_in(ctx, cpuctx);
3850 
3851 	/* Then walk through the lower prio flexible groups */
3852 	if (is_active & EVENT_FLEXIBLE)
3853 		ctx_flexible_sched_in(ctx, cpuctx);
3854 }
3855 
3856 static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx,
3857 			     enum event_type_t event_type)
3858 {
3859 	struct perf_event_context *ctx = &cpuctx->ctx;
3860 
3861 	ctx_sched_in(ctx, cpuctx, event_type);
3862 }
3863 
3864 static void perf_event_context_sched_in(struct perf_event_context *ctx,
3865 					struct task_struct *task)
3866 {
3867 	struct perf_cpu_context *cpuctx;
3868 	struct pmu *pmu;
3869 
3870 	cpuctx = __get_cpu_context(ctx);
3871 
3872 	/*
3873 	 * HACK: for HETEROGENEOUS the task context might have switched to a
3874 	 * different PMU, force (re)set the context,
3875 	 */
3876 	pmu = ctx->pmu = cpuctx->ctx.pmu;
3877 
3878 	if (cpuctx->task_ctx == ctx) {
3879 		if (cpuctx->sched_cb_usage)
3880 			__perf_pmu_sched_task(cpuctx, true);
3881 		return;
3882 	}
3883 
3884 	perf_ctx_lock(cpuctx, ctx);
3885 	/*
3886 	 * We must check ctx->nr_events while holding ctx->lock, such
3887 	 * that we serialize against perf_install_in_context().
3888 	 */
3889 	if (!ctx->nr_events)
3890 		goto unlock;
3891 
3892 	perf_pmu_disable(pmu);
3893 	/*
3894 	 * We want to keep the following priority order:
3895 	 * cpu pinned (that don't need to move), task pinned,
3896 	 * cpu flexible, task flexible.
3897 	 *
3898 	 * However, if task's ctx is not carrying any pinned
3899 	 * events, no need to flip the cpuctx's events around.
3900 	 */
3901 	if (!RB_EMPTY_ROOT(&ctx->pinned_groups.tree))
3902 		cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);
3903 	perf_event_sched_in(cpuctx, ctx);
3904 
3905 	if (cpuctx->sched_cb_usage && pmu->sched_task)
3906 		pmu->sched_task(cpuctx->task_ctx, true);
3907 
3908 	perf_pmu_enable(pmu);
3909 
3910 unlock:
3911 	perf_ctx_unlock(cpuctx, ctx);
3912 }
3913 
3914 /*
3915  * Called from scheduler to add the events of the current task
3916  * with interrupts disabled.
3917  *
3918  * We restore the event value and then enable it.
3919  *
3920  * This does not protect us against NMI, but enable()
3921  * sets the enabled bit in the control field of event _before_
3922  * accessing the event control register. If a NMI hits, then it will
3923  * keep the event running.
3924  */
3925 void __perf_event_task_sched_in(struct task_struct *prev,
3926 				struct task_struct *task)
3927 {
3928 	struct perf_event_context *ctx;
3929 	int ctxn;
3930 
3931 	for_each_task_context_nr(ctxn) {
3932 		ctx = task->perf_event_ctxp[ctxn];
3933 		if (likely(!ctx))
3934 			continue;
3935 
3936 		perf_event_context_sched_in(ctx, task);
3937 	}
3938 
3939 	if (atomic_read(&nr_switch_events))
3940 		perf_event_switch(task, prev, true);
3941 
3942 	if (__this_cpu_read(perf_sched_cb_usages))
3943 		perf_pmu_sched_task(prev, task, true);
3944 }
3945 
3946 static u64 perf_calculate_period(struct perf_event *event, u64 nsec, u64 count)
3947 {
3948 	u64 frequency = event->attr.sample_freq;
3949 	u64 sec = NSEC_PER_SEC;
3950 	u64 divisor, dividend;
3951 
3952 	int count_fls, nsec_fls, frequency_fls, sec_fls;
3953 
3954 	count_fls = fls64(count);
3955 	nsec_fls = fls64(nsec);
3956 	frequency_fls = fls64(frequency);
3957 	sec_fls = 30;
3958 
3959 	/*
3960 	 * We got @count in @nsec, with a target of sample_freq HZ
3961 	 * the target period becomes:
3962 	 *
3963 	 *             @count * 10^9
3964 	 * period = -------------------
3965 	 *          @nsec * sample_freq
3966 	 *
3967 	 */
3968 
3969 	/*
3970 	 * Reduce accuracy by one bit such that @a and @b converge
3971 	 * to a similar magnitude.
3972 	 */
3973 #define REDUCE_FLS(a, b)		\
3974 do {					\
3975 	if (a##_fls > b##_fls) {	\
3976 		a >>= 1;		\
3977 		a##_fls--;		\
3978 	} else {			\
3979 		b >>= 1;		\
3980 		b##_fls--;		\
3981 	}				\
3982 } while (0)
3983 
3984 	/*
3985 	 * Reduce accuracy until either term fits in a u64, then proceed with
3986 	 * the other, so that finally we can do a u64/u64 division.
3987 	 */
3988 	while (count_fls + sec_fls > 64 && nsec_fls + frequency_fls > 64) {
3989 		REDUCE_FLS(nsec, frequency);
3990 		REDUCE_FLS(sec, count);
3991 	}
3992 
3993 	if (count_fls + sec_fls > 64) {
3994 		divisor = nsec * frequency;
3995 
3996 		while (count_fls + sec_fls > 64) {
3997 			REDUCE_FLS(count, sec);
3998 			divisor >>= 1;
3999 		}
4000 
4001 		dividend = count * sec;
4002 	} else {
4003 		dividend = count * sec;
4004 
4005 		while (nsec_fls + frequency_fls > 64) {
4006 			REDUCE_FLS(nsec, frequency);
4007 			dividend >>= 1;
4008 		}
4009 
4010 		divisor = nsec * frequency;
4011 	}
4012 
4013 	if (!divisor)
4014 		return dividend;
4015 
4016 	return div64_u64(dividend, divisor);
4017 }
4018 
4019 static DEFINE_PER_CPU(int, perf_throttled_count);
4020 static DEFINE_PER_CPU(u64, perf_throttled_seq);
4021 
4022 static void perf_adjust_period(struct perf_event *event, u64 nsec, u64 count, bool disable)
4023 {
4024 	struct hw_perf_event *hwc = &event->hw;
4025 	s64 period, sample_period;
4026 	s64 delta;
4027 
4028 	period = perf_calculate_period(event, nsec, count);
4029 
4030 	delta = (s64)(period - hwc->sample_period);
4031 	delta = (delta + 7) / 8; /* low pass filter */
4032 
4033 	sample_period = hwc->sample_period + delta;
4034 
4035 	if (!sample_period)
4036 		sample_period = 1;
4037 
4038 	hwc->sample_period = sample_period;
4039 
4040 	if (local64_read(&hwc->period_left) > 8*sample_period) {
4041 		if (disable)
4042 			event->pmu->stop(event, PERF_EF_UPDATE);
4043 
4044 		local64_set(&hwc->period_left, 0);
4045 
4046 		if (disable)
4047 			event->pmu->start(event, PERF_EF_RELOAD);
4048 	}
4049 }
4050 
4051 /*
4052  * combine freq adjustment with unthrottling to avoid two passes over the
4053  * events. At the same time, make sure, having freq events does not change
4054  * the rate of unthrottling as that would introduce bias.
4055  */
4056 static void perf_adjust_freq_unthr_context(struct perf_event_context *ctx,
4057 					   int needs_unthr)
4058 {
4059 	struct perf_event *event;
4060 	struct hw_perf_event *hwc;
4061 	u64 now, period = TICK_NSEC;
4062 	s64 delta;
4063 
4064 	/*
4065 	 * only need to iterate over all events iff:
4066 	 * - context have events in frequency mode (needs freq adjust)
4067 	 * - there are events to unthrottle on this cpu
4068 	 */
4069 	if (!(ctx->nr_freq || needs_unthr))
4070 		return;
4071 
4072 	raw_spin_lock(&ctx->lock);
4073 	perf_pmu_disable(ctx->pmu);
4074 
4075 	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
4076 		if (event->state != PERF_EVENT_STATE_ACTIVE)
4077 			continue;
4078 
4079 		if (!event_filter_match(event))
4080 			continue;
4081 
4082 		perf_pmu_disable(event->pmu);
4083 
4084 		hwc = &event->hw;
4085 
4086 		if (hwc->interrupts == MAX_INTERRUPTS) {
4087 			hwc->interrupts = 0;
4088 			perf_log_throttle(event, 1);
4089 			event->pmu->start(event, 0);
4090 		}
4091 
4092 		if (!event->attr.freq || !event->attr.sample_freq)
4093 			goto next;
4094 
4095 		/*
4096 		 * stop the event and update event->count
4097 		 */
4098 		event->pmu->stop(event, PERF_EF_UPDATE);
4099 
4100 		now = local64_read(&event->count);
4101 		delta = now - hwc->freq_count_stamp;
4102 		hwc->freq_count_stamp = now;
4103 
4104 		/*
4105 		 * restart the event
4106 		 * reload only if value has changed
4107 		 * we have stopped the event so tell that
4108 		 * to perf_adjust_period() to avoid stopping it
4109 		 * twice.
4110 		 */
4111 		if (delta > 0)
4112 			perf_adjust_period(event, period, delta, false);
4113 
4114 		event->pmu->start(event, delta > 0 ? PERF_EF_RELOAD : 0);
4115 	next:
4116 		perf_pmu_enable(event->pmu);
4117 	}
4118 
4119 	perf_pmu_enable(ctx->pmu);
4120 	raw_spin_unlock(&ctx->lock);
4121 }
4122 
4123 /*
4124  * Move @event to the tail of the @ctx's elegible events.
4125  */
4126 static void rotate_ctx(struct perf_event_context *ctx, struct perf_event *event)
4127 {
4128 	/*
4129 	 * Rotate the first entry last of non-pinned groups. Rotation might be
4130 	 * disabled by the inheritance code.
4131 	 */
4132 	if (ctx->rotate_disable)
4133 		return;
4134 
4135 	perf_event_groups_delete(&ctx->flexible_groups, event);
4136 	perf_event_groups_insert(&ctx->flexible_groups, event);
4137 }
4138 
4139 /* pick an event from the flexible_groups to rotate */
4140 static inline struct perf_event *
4141 ctx_event_to_rotate(struct perf_event_context *ctx)
4142 {
4143 	struct perf_event *event;
4144 
4145 	/* pick the first active flexible event */
4146 	event = list_first_entry_or_null(&ctx->flexible_active,
4147 					 struct perf_event, active_list);
4148 
4149 	/* if no active flexible event, pick the first event */
4150 	if (!event) {
4151 		event = rb_entry_safe(rb_first(&ctx->flexible_groups.tree),
4152 				      typeof(*event), group_node);
4153 	}
4154 
4155 	/*
4156 	 * Unconditionally clear rotate_necessary; if ctx_flexible_sched_in()
4157 	 * finds there are unschedulable events, it will set it again.
4158 	 */
4159 	ctx->rotate_necessary = 0;
4160 
4161 	return event;
4162 }
4163 
4164 static bool perf_rotate_context(struct perf_cpu_context *cpuctx)
4165 {
4166 	struct perf_event *cpu_event = NULL, *task_event = NULL;
4167 	struct perf_event_context *task_ctx = NULL;
4168 	int cpu_rotate, task_rotate;
4169 
4170 	/*
4171 	 * Since we run this from IRQ context, nobody can install new
4172 	 * events, thus the event count values are stable.
4173 	 */
4174 
4175 	cpu_rotate = cpuctx->ctx.rotate_necessary;
4176 	task_ctx = cpuctx->task_ctx;
4177 	task_rotate = task_ctx ? task_ctx->rotate_necessary : 0;
4178 
4179 	if (!(cpu_rotate || task_rotate))
4180 		return false;
4181 
4182 	perf_ctx_lock(cpuctx, cpuctx->task_ctx);
4183 	perf_pmu_disable(cpuctx->ctx.pmu);
4184 
4185 	if (task_rotate)
4186 		task_event = ctx_event_to_rotate(task_ctx);
4187 	if (cpu_rotate)
4188 		cpu_event = ctx_event_to_rotate(&cpuctx->ctx);
4189 
4190 	/*
4191 	 * As per the order given at ctx_resched() first 'pop' task flexible
4192 	 * and then, if needed CPU flexible.
4193 	 */
4194 	if (task_event || (task_ctx && cpu_event))
4195 		ctx_sched_out(task_ctx, cpuctx, EVENT_FLEXIBLE);
4196 	if (cpu_event)
4197 		cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);
4198 
4199 	if (task_event)
4200 		rotate_ctx(task_ctx, task_event);
4201 	if (cpu_event)
4202 		rotate_ctx(&cpuctx->ctx, cpu_event);
4203 
4204 	perf_event_sched_in(cpuctx, task_ctx);
4205 
4206 	perf_pmu_enable(cpuctx->ctx.pmu);
4207 	perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
4208 
4209 	return true;
4210 }
4211 
4212 void perf_event_task_tick(void)
4213 {
4214 	struct list_head *head = this_cpu_ptr(&active_ctx_list);
4215 	struct perf_event_context *ctx, *tmp;
4216 	int throttled;
4217 
4218 	lockdep_assert_irqs_disabled();
4219 
4220 	__this_cpu_inc(perf_throttled_seq);
4221 	throttled = __this_cpu_xchg(perf_throttled_count, 0);
4222 	tick_dep_clear_cpu(smp_processor_id(), TICK_DEP_BIT_PERF_EVENTS);
4223 
4224 	list_for_each_entry_safe(ctx, tmp, head, active_ctx_list)
4225 		perf_adjust_freq_unthr_context(ctx, throttled);
4226 }
4227 
4228 static int event_enable_on_exec(struct perf_event *event,
4229 				struct perf_event_context *ctx)
4230 {
4231 	if (!event->attr.enable_on_exec)
4232 		return 0;
4233 
4234 	event->attr.enable_on_exec = 0;
4235 	if (event->state >= PERF_EVENT_STATE_INACTIVE)
4236 		return 0;
4237 
4238 	perf_event_set_state(event, PERF_EVENT_STATE_INACTIVE);
4239 
4240 	return 1;
4241 }
4242 
4243 /*
4244  * Enable all of a task's events that have been marked enable-on-exec.
4245  * This expects task == current.
4246  */
4247 static void perf_event_enable_on_exec(int ctxn)
4248 {
4249 	struct perf_event_context *ctx, *clone_ctx = NULL;
4250 	enum event_type_t event_type = 0;
4251 	struct perf_cpu_context *cpuctx;
4252 	struct perf_event *event;
4253 	unsigned long flags;
4254 	int enabled = 0;
4255 
4256 	local_irq_save(flags);
4257 	ctx = current->perf_event_ctxp[ctxn];
4258 	if (!ctx || !ctx->nr_events)
4259 		goto out;
4260 
4261 	cpuctx = __get_cpu_context(ctx);
4262 	perf_ctx_lock(cpuctx, ctx);
4263 	ctx_sched_out(ctx, cpuctx, EVENT_TIME);
4264 	list_for_each_entry(event, &ctx->event_list, event_entry) {
4265 		enabled |= event_enable_on_exec(event, ctx);
4266 		event_type |= get_event_type(event);
4267 	}
4268 
4269 	/*
4270 	 * Unclone and reschedule this context if we enabled any event.
4271 	 */
4272 	if (enabled) {
4273 		clone_ctx = unclone_ctx(ctx);
4274 		ctx_resched(cpuctx, ctx, event_type);
4275 	} else {
4276 		ctx_sched_in(ctx, cpuctx, EVENT_TIME);
4277 	}
4278 	perf_ctx_unlock(cpuctx, ctx);
4279 
4280 out:
4281 	local_irq_restore(flags);
4282 
4283 	if (clone_ctx)
4284 		put_ctx(clone_ctx);
4285 }
4286 
4287 static void perf_remove_from_owner(struct perf_event *event);
4288 static void perf_event_exit_event(struct perf_event *event,
4289 				  struct perf_event_context *ctx);
4290 
4291 /*
4292  * Removes all events from the current task that have been marked
4293  * remove-on-exec, and feeds their values back to parent events.
4294  */
4295 static void perf_event_remove_on_exec(int ctxn)
4296 {
4297 	struct perf_event_context *ctx, *clone_ctx = NULL;
4298 	struct perf_event *event, *next;
4299 	unsigned long flags;
4300 	bool modified = false;
4301 
4302 	ctx = perf_pin_task_context(current, ctxn);
4303 	if (!ctx)
4304 		return;
4305 
4306 	mutex_lock(&ctx->mutex);
4307 
4308 	if (WARN_ON_ONCE(ctx->task != current))
4309 		goto unlock;
4310 
4311 	list_for_each_entry_safe(event, next, &ctx->event_list, event_entry) {
4312 		if (!event->attr.remove_on_exec)
4313 			continue;
4314 
4315 		if (!is_kernel_event(event))
4316 			perf_remove_from_owner(event);
4317 
4318 		modified = true;
4319 
4320 		perf_event_exit_event(event, ctx);
4321 	}
4322 
4323 	raw_spin_lock_irqsave(&ctx->lock, flags);
4324 	if (modified)
4325 		clone_ctx = unclone_ctx(ctx);
4326 	--ctx->pin_count;
4327 	raw_spin_unlock_irqrestore(&ctx->lock, flags);
4328 
4329 unlock:
4330 	mutex_unlock(&ctx->mutex);
4331 
4332 	put_ctx(ctx);
4333 	if (clone_ctx)
4334 		put_ctx(clone_ctx);
4335 }
4336 
4337 struct perf_read_data {
4338 	struct perf_event *event;
4339 	bool group;
4340 	int ret;
4341 };
4342 
4343 static int __perf_event_read_cpu(struct perf_event *event, int event_cpu)
4344 {
4345 	u16 local_pkg, event_pkg;
4346 
4347 	if (event->group_caps & PERF_EV_CAP_READ_ACTIVE_PKG) {
4348 		int local_cpu = smp_processor_id();
4349 
4350 		event_pkg = topology_physical_package_id(event_cpu);
4351 		local_pkg = topology_physical_package_id(local_cpu);
4352 
4353 		if (event_pkg == local_pkg)
4354 			return local_cpu;
4355 	}
4356 
4357 	return event_cpu;
4358 }
4359 
4360 /*
4361  * Cross CPU call to read the hardware event
4362  */
4363 static void __perf_event_read(void *info)
4364 {
4365 	struct perf_read_data *data = info;
4366 	struct perf_event *sub, *event = data->event;
4367 	struct perf_event_context *ctx = event->ctx;
4368 	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
4369 	struct pmu *pmu = event->pmu;
4370 
4371 	/*
4372 	 * If this is a task context, we need to check whether it is
4373 	 * the current task context of this cpu.  If not it has been
4374 	 * scheduled out before the smp call arrived.  In that case
4375 	 * event->count would have been updated to a recent sample
4376 	 * when the event was scheduled out.
4377 	 */
4378 	if (ctx->task && cpuctx->task_ctx != ctx)
4379 		return;
4380 
4381 	raw_spin_lock(&ctx->lock);
4382 	if (ctx->is_active & EVENT_TIME) {
4383 		update_context_time(ctx);
4384 		update_cgrp_time_from_event(event);
4385 	}
4386 
4387 	perf_event_update_time(event);
4388 	if (data->group)
4389 		perf_event_update_sibling_time(event);
4390 
4391 	if (event->state != PERF_EVENT_STATE_ACTIVE)
4392 		goto unlock;
4393 
4394 	if (!data->group) {
4395 		pmu->read(event);
4396 		data->ret = 0;
4397 		goto unlock;
4398 	}
4399 
4400 	pmu->start_txn(pmu, PERF_PMU_TXN_READ);
4401 
4402 	pmu->read(event);
4403 
4404 	for_each_sibling_event(sub, event) {
4405 		if (sub->state == PERF_EVENT_STATE_ACTIVE) {
4406 			/*
4407 			 * Use sibling's PMU rather than @event's since
4408 			 * sibling could be on different (eg: software) PMU.
4409 			 */
4410 			sub->pmu->read(sub);
4411 		}
4412 	}
4413 
4414 	data->ret = pmu->commit_txn(pmu);
4415 
4416 unlock:
4417 	raw_spin_unlock(&ctx->lock);
4418 }
4419 
4420 static inline u64 perf_event_count(struct perf_event *event)
4421 {
4422 	return local64_read(&event->count) + atomic64_read(&event->child_count);
4423 }
4424 
4425 static void calc_timer_values(struct perf_event *event,
4426 				u64 *now,
4427 				u64 *enabled,
4428 				u64 *running)
4429 {
4430 	u64 ctx_time;
4431 
4432 	*now = perf_clock();
4433 	ctx_time = perf_event_time_now(event, *now);
4434 	__perf_update_times(event, ctx_time, enabled, running);
4435 }
4436 
4437 /*
4438  * NMI-safe method to read a local event, that is an event that
4439  * is:
4440  *   - either for the current task, or for this CPU
4441  *   - does not have inherit set, for inherited task events
4442  *     will not be local and we cannot read them atomically
4443  *   - must not have a pmu::count method
4444  */
4445 int perf_event_read_local(struct perf_event *event, u64 *value,
4446 			  u64 *enabled, u64 *running)
4447 {
4448 	unsigned long flags;
4449 	int ret = 0;
4450 
4451 	/*
4452 	 * Disabling interrupts avoids all counter scheduling (context
4453 	 * switches, timer based rotation and IPIs).
4454 	 */
4455 	local_irq_save(flags);
4456 
4457 	/*
4458 	 * It must not be an event with inherit set, we cannot read
4459 	 * all child counters from atomic context.
4460 	 */
4461 	if (event->attr.inherit) {
4462 		ret = -EOPNOTSUPP;
4463 		goto out;
4464 	}
4465 
4466 	/* If this is a per-task event, it must be for current */
4467 	if ((event->attach_state & PERF_ATTACH_TASK) &&
4468 	    event->hw.target != current) {
4469 		ret = -EINVAL;
4470 		goto out;
4471 	}
4472 
4473 	/* If this is a per-CPU event, it must be for this CPU */
4474 	if (!(event->attach_state & PERF_ATTACH_TASK) &&
4475 	    event->cpu != smp_processor_id()) {
4476 		ret = -EINVAL;
4477 		goto out;
4478 	}
4479 
4480 	/* If this is a pinned event it must be running on this CPU */
4481 	if (event->attr.pinned && event->oncpu != smp_processor_id()) {
4482 		ret = -EBUSY;
4483 		goto out;
4484 	}
4485 
4486 	/*
4487 	 * If the event is currently on this CPU, its either a per-task event,
4488 	 * or local to this CPU. Furthermore it means its ACTIVE (otherwise
4489 	 * oncpu == -1).
4490 	 */
4491 	if (event->oncpu == smp_processor_id())
4492 		event->pmu->read(event);
4493 
4494 	*value = local64_read(&event->count);
4495 	if (enabled || running) {
4496 		u64 __enabled, __running, __now;
4497 
4498 		calc_timer_values(event, &__now, &__enabled, &__running);
4499 		if (enabled)
4500 			*enabled = __enabled;
4501 		if (running)
4502 			*running = __running;
4503 	}
4504 out:
4505 	local_irq_restore(flags);
4506 
4507 	return ret;
4508 }
4509 
4510 static int perf_event_read(struct perf_event *event, bool group)
4511 {
4512 	enum perf_event_state state = READ_ONCE(event->state);
4513 	int event_cpu, ret = 0;
4514 
4515 	/*
4516 	 * If event is enabled and currently active on a CPU, update the
4517 	 * value in the event structure:
4518 	 */
4519 again:
4520 	if (state == PERF_EVENT_STATE_ACTIVE) {
4521 		struct perf_read_data data;
4522 
4523 		/*
4524 		 * Orders the ->state and ->oncpu loads such that if we see
4525 		 * ACTIVE we must also see the right ->oncpu.
4526 		 *
4527 		 * Matches the smp_wmb() from event_sched_in().
4528 		 */
4529 		smp_rmb();
4530 
4531 		event_cpu = READ_ONCE(event->oncpu);
4532 		if ((unsigned)event_cpu >= nr_cpu_ids)
4533 			return 0;
4534 
4535 		data = (struct perf_read_data){
4536 			.event = event,
4537 			.group = group,
4538 			.ret = 0,
4539 		};
4540 
4541 		preempt_disable();
4542 		event_cpu = __perf_event_read_cpu(event, event_cpu);
4543 
4544 		/*
4545 		 * Purposely ignore the smp_call_function_single() return
4546 		 * value.
4547 		 *
4548 		 * If event_cpu isn't a valid CPU it means the event got
4549 		 * scheduled out and that will have updated the event count.
4550 		 *
4551 		 * Therefore, either way, we'll have an up-to-date event count
4552 		 * after this.
4553 		 */
4554 		(void)smp_call_function_single(event_cpu, __perf_event_read, &data, 1);
4555 		preempt_enable();
4556 		ret = data.ret;
4557 
4558 	} else if (state == PERF_EVENT_STATE_INACTIVE) {
4559 		struct perf_event_context *ctx = event->ctx;
4560 		unsigned long flags;
4561 
4562 		raw_spin_lock_irqsave(&ctx->lock, flags);
4563 		state = event->state;
4564 		if (state != PERF_EVENT_STATE_INACTIVE) {
4565 			raw_spin_unlock_irqrestore(&ctx->lock, flags);
4566 			goto again;
4567 		}
4568 
4569 		/*
4570 		 * May read while context is not active (e.g., thread is
4571 		 * blocked), in that case we cannot update context time
4572 		 */
4573 		if (ctx->is_active & EVENT_TIME) {
4574 			update_context_time(ctx);
4575 			update_cgrp_time_from_event(event);
4576 		}
4577 
4578 		perf_event_update_time(event);
4579 		if (group)
4580 			perf_event_update_sibling_time(event);
4581 		raw_spin_unlock_irqrestore(&ctx->lock, flags);
4582 	}
4583 
4584 	return ret;
4585 }
4586 
4587 /*
4588  * Initialize the perf_event context in a task_struct:
4589  */
4590 static void __perf_event_init_context(struct perf_event_context *ctx)
4591 {
4592 	raw_spin_lock_init(&ctx->lock);
4593 	mutex_init(&ctx->mutex);
4594 	INIT_LIST_HEAD(&ctx->active_ctx_list);
4595 	perf_event_groups_init(&ctx->pinned_groups);
4596 	perf_event_groups_init(&ctx->flexible_groups);
4597 	INIT_LIST_HEAD(&ctx->event_list);
4598 	INIT_LIST_HEAD(&ctx->pinned_active);
4599 	INIT_LIST_HEAD(&ctx->flexible_active);
4600 	refcount_set(&ctx->refcount, 1);
4601 }
4602 
4603 static struct perf_event_context *
4604 alloc_perf_context(struct pmu *pmu, struct task_struct *task)
4605 {
4606 	struct perf_event_context *ctx;
4607 
4608 	ctx = kzalloc(sizeof(struct perf_event_context), GFP_KERNEL);
4609 	if (!ctx)
4610 		return NULL;
4611 
4612 	__perf_event_init_context(ctx);
4613 	if (task)
4614 		ctx->task = get_task_struct(task);
4615 	ctx->pmu = pmu;
4616 
4617 	return ctx;
4618 }
4619 
4620 static struct task_struct *
4621 find_lively_task_by_vpid(pid_t vpid)
4622 {
4623 	struct task_struct *task;
4624 
4625 	rcu_read_lock();
4626 	if (!vpid)
4627 		task = current;
4628 	else
4629 		task = find_task_by_vpid(vpid);
4630 	if (task)
4631 		get_task_struct(task);
4632 	rcu_read_unlock();
4633 
4634 	if (!task)
4635 		return ERR_PTR(-ESRCH);
4636 
4637 	return task;
4638 }
4639 
4640 /*
4641  * Returns a matching context with refcount and pincount.
4642  */
4643 static struct perf_event_context *
4644 find_get_context(struct pmu *pmu, struct task_struct *task,
4645 		struct perf_event *event)
4646 {
4647 	struct perf_event_context *ctx, *clone_ctx = NULL;
4648 	struct perf_cpu_context *cpuctx;
4649 	void *task_ctx_data = NULL;
4650 	unsigned long flags;
4651 	int ctxn, err;
4652 	int cpu = event->cpu;
4653 
4654 	if (!task) {
4655 		/* Must be root to operate on a CPU event: */
4656 		err = perf_allow_cpu(&event->attr);
4657 		if (err)
4658 			return ERR_PTR(err);
4659 
4660 		cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
4661 		ctx = &cpuctx->ctx;
4662 		get_ctx(ctx);
4663 		raw_spin_lock_irqsave(&ctx->lock, flags);
4664 		++ctx->pin_count;
4665 		raw_spin_unlock_irqrestore(&ctx->lock, flags);
4666 
4667 		return ctx;
4668 	}
4669 
4670 	err = -EINVAL;
4671 	ctxn = pmu->task_ctx_nr;
4672 	if (ctxn < 0)
4673 		goto errout;
4674 
4675 	if (event->attach_state & PERF_ATTACH_TASK_DATA) {
4676 		task_ctx_data = alloc_task_ctx_data(pmu);
4677 		if (!task_ctx_data) {
4678 			err = -ENOMEM;
4679 			goto errout;
4680 		}
4681 	}
4682 
4683 retry:
4684 	ctx = perf_lock_task_context(task, ctxn, &flags);
4685 	if (ctx) {
4686 		clone_ctx = unclone_ctx(ctx);
4687 		++ctx->pin_count;
4688 
4689 		if (task_ctx_data && !ctx->task_ctx_data) {
4690 			ctx->task_ctx_data = task_ctx_data;
4691 			task_ctx_data = NULL;
4692 		}
4693 		raw_spin_unlock_irqrestore(&ctx->lock, flags);
4694 
4695 		if (clone_ctx)
4696 			put_ctx(clone_ctx);
4697 	} else {
4698 		ctx = alloc_perf_context(pmu, task);
4699 		err = -ENOMEM;
4700 		if (!ctx)
4701 			goto errout;
4702 
4703 		if (task_ctx_data) {
4704 			ctx->task_ctx_data = task_ctx_data;
4705 			task_ctx_data = NULL;
4706 		}
4707 
4708 		err = 0;
4709 		mutex_lock(&task->perf_event_mutex);
4710 		/*
4711 		 * If it has already passed perf_event_exit_task().
4712 		 * we must see PF_EXITING, it takes this mutex too.
4713 		 */
4714 		if (task->flags & PF_EXITING)
4715 			err = -ESRCH;
4716 		else if (task->perf_event_ctxp[ctxn])
4717 			err = -EAGAIN;
4718 		else {
4719 			get_ctx(ctx);
4720 			++ctx->pin_count;
4721 			rcu_assign_pointer(task->perf_event_ctxp[ctxn], ctx);
4722 		}
4723 		mutex_unlock(&task->perf_event_mutex);
4724 
4725 		if (unlikely(err)) {
4726 			put_ctx(ctx);
4727 
4728 			if (err == -EAGAIN)
4729 				goto retry;
4730 			goto errout;
4731 		}
4732 	}
4733 
4734 	free_task_ctx_data(pmu, task_ctx_data);
4735 	return ctx;
4736 
4737 errout:
4738 	free_task_ctx_data(pmu, task_ctx_data);
4739 	return ERR_PTR(err);
4740 }
4741 
4742 static void perf_event_free_filter(struct perf_event *event);
4743 
4744 static void free_event_rcu(struct rcu_head *head)
4745 {
4746 	struct perf_event *event;
4747 
4748 	event = container_of(head, struct perf_event, rcu_head);
4749 	if (event->ns)
4750 		put_pid_ns(event->ns);
4751 	perf_event_free_filter(event);
4752 	kmem_cache_free(perf_event_cache, event);
4753 }
4754 
4755 static void ring_buffer_attach(struct perf_event *event,
4756 			       struct perf_buffer *rb);
4757 
4758 static void detach_sb_event(struct perf_event *event)
4759 {
4760 	struct pmu_event_list *pel = per_cpu_ptr(&pmu_sb_events, event->cpu);
4761 
4762 	raw_spin_lock(&pel->lock);
4763 	list_del_rcu(&event->sb_list);
4764 	raw_spin_unlock(&pel->lock);
4765 }
4766 
4767 static bool is_sb_event(struct perf_event *event)
4768 {
4769 	struct perf_event_attr *attr = &event->attr;
4770 
4771 	if (event->parent)
4772 		return false;
4773 
4774 	if (event->attach_state & PERF_ATTACH_TASK)
4775 		return false;
4776 
4777 	if (attr->mmap || attr->mmap_data || attr->mmap2 ||
4778 	    attr->comm || attr->comm_exec ||
4779 	    attr->task || attr->ksymbol ||
4780 	    attr->context_switch || attr->text_poke ||
4781 	    attr->bpf_event)
4782 		return true;
4783 	return false;
4784 }
4785 
4786 static void unaccount_pmu_sb_event(struct perf_event *event)
4787 {
4788 	if (is_sb_event(event))
4789 		detach_sb_event(event);
4790 }
4791 
4792 static void unaccount_event_cpu(struct perf_event *event, int cpu)
4793 {
4794 	if (event->parent)
4795 		return;
4796 
4797 	if (is_cgroup_event(event))
4798 		atomic_dec(&per_cpu(perf_cgroup_events, cpu));
4799 }
4800 
4801 #ifdef CONFIG_NO_HZ_FULL
4802 static DEFINE_SPINLOCK(nr_freq_lock);
4803 #endif
4804 
4805 static void unaccount_freq_event_nohz(void)
4806 {
4807 #ifdef CONFIG_NO_HZ_FULL
4808 	spin_lock(&nr_freq_lock);
4809 	if (atomic_dec_and_test(&nr_freq_events))
4810 		tick_nohz_dep_clear(TICK_DEP_BIT_PERF_EVENTS);
4811 	spin_unlock(&nr_freq_lock);
4812 #endif
4813 }
4814 
4815 static void unaccount_freq_event(void)
4816 {
4817 	if (tick_nohz_full_enabled())
4818 		unaccount_freq_event_nohz();
4819 	else
4820 		atomic_dec(&nr_freq_events);
4821 }
4822 
4823 static void unaccount_event(struct perf_event *event)
4824 {
4825 	bool dec = false;
4826 
4827 	if (event->parent)
4828 		return;
4829 
4830 	if (event->attach_state & (PERF_ATTACH_TASK | PERF_ATTACH_SCHED_CB))
4831 		dec = true;
4832 	if (event->attr.mmap || event->attr.mmap_data)
4833 		atomic_dec(&nr_mmap_events);
4834 	if (event->attr.build_id)
4835 		atomic_dec(&nr_build_id_events);
4836 	if (event->attr.comm)
4837 		atomic_dec(&nr_comm_events);
4838 	if (event->attr.namespaces)
4839 		atomic_dec(&nr_namespaces_events);
4840 	if (event->attr.cgroup)
4841 		atomic_dec(&nr_cgroup_events);
4842 	if (event->attr.task)
4843 		atomic_dec(&nr_task_events);
4844 	if (event->attr.freq)
4845 		unaccount_freq_event();
4846 	if (event->attr.context_switch) {
4847 		dec = true;
4848 		atomic_dec(&nr_switch_events);
4849 	}
4850 	if (is_cgroup_event(event))
4851 		dec = true;
4852 	if (has_branch_stack(event))
4853 		dec = true;
4854 	if (event->attr.ksymbol)
4855 		atomic_dec(&nr_ksymbol_events);
4856 	if (event->attr.bpf_event)
4857 		atomic_dec(&nr_bpf_events);
4858 	if (event->attr.text_poke)
4859 		atomic_dec(&nr_text_poke_events);
4860 
4861 	if (dec) {
4862 		if (!atomic_add_unless(&perf_sched_count, -1, 1))
4863 			schedule_delayed_work(&perf_sched_work, HZ);
4864 	}
4865 
4866 	unaccount_event_cpu(event, event->cpu);
4867 
4868 	unaccount_pmu_sb_event(event);
4869 }
4870 
4871 static void perf_sched_delayed(struct work_struct *work)
4872 {
4873 	mutex_lock(&perf_sched_mutex);
4874 	if (atomic_dec_and_test(&perf_sched_count))
4875 		static_branch_disable(&perf_sched_events);
4876 	mutex_unlock(&perf_sched_mutex);
4877 }
4878 
4879 /*
4880  * The following implement mutual exclusion of events on "exclusive" pmus
4881  * (PERF_PMU_CAP_EXCLUSIVE). Such pmus can only have one event scheduled
4882  * at a time, so we disallow creating events that might conflict, namely:
4883  *
4884  *  1) cpu-wide events in the presence of per-task events,
4885  *  2) per-task events in the presence of cpu-wide events,
4886  *  3) two matching events on the same context.
4887  *
4888  * The former two cases are handled in the allocation path (perf_event_alloc(),
4889  * _free_event()), the latter -- before the first perf_install_in_context().
4890  */
4891 static int exclusive_event_init(struct perf_event *event)
4892 {
4893 	struct pmu *pmu = event->pmu;
4894 
4895 	if (!is_exclusive_pmu(pmu))
4896 		return 0;
4897 
4898 	/*
4899 	 * Prevent co-existence of per-task and cpu-wide events on the
4900 	 * same exclusive pmu.
4901 	 *
4902 	 * Negative pmu::exclusive_cnt means there are cpu-wide
4903 	 * events on this "exclusive" pmu, positive means there are
4904 	 * per-task events.
4905 	 *
4906 	 * Since this is called in perf_event_alloc() path, event::ctx
4907 	 * doesn't exist yet; it is, however, safe to use PERF_ATTACH_TASK
4908 	 * to mean "per-task event", because unlike other attach states it
4909 	 * never gets cleared.
4910 	 */
4911 	if (event->attach_state & PERF_ATTACH_TASK) {
4912 		if (!atomic_inc_unless_negative(&pmu->exclusive_cnt))
4913 			return -EBUSY;
4914 	} else {
4915 		if (!atomic_dec_unless_positive(&pmu->exclusive_cnt))
4916 			return -EBUSY;
4917 	}
4918 
4919 	return 0;
4920 }
4921 
4922 static void exclusive_event_destroy(struct perf_event *event)
4923 {
4924 	struct pmu *pmu = event->pmu;
4925 
4926 	if (!is_exclusive_pmu(pmu))
4927 		return;
4928 
4929 	/* see comment in exclusive_event_init() */
4930 	if (event->attach_state & PERF_ATTACH_TASK)
4931 		atomic_dec(&pmu->exclusive_cnt);
4932 	else
4933 		atomic_inc(&pmu->exclusive_cnt);
4934 }
4935 
4936 static bool exclusive_event_match(struct perf_event *e1, struct perf_event *e2)
4937 {
4938 	if ((e1->pmu == e2->pmu) &&
4939 	    (e1->cpu == e2->cpu ||
4940 	     e1->cpu == -1 ||
4941 	     e2->cpu == -1))
4942 		return true;
4943 	return false;
4944 }
4945 
4946 static bool exclusive_event_installable(struct perf_event *event,
4947 					struct perf_event_context *ctx)
4948 {
4949 	struct perf_event *iter_event;
4950 	struct pmu *pmu = event->pmu;
4951 
4952 	lockdep_assert_held(&ctx->mutex);
4953 
4954 	if (!is_exclusive_pmu(pmu))
4955 		return true;
4956 
4957 	list_for_each_entry(iter_event, &ctx->event_list, event_entry) {
4958 		if (exclusive_event_match(iter_event, event))
4959 			return false;
4960 	}
4961 
4962 	return true;
4963 }
4964 
4965 static void perf_addr_filters_splice(struct perf_event *event,
4966 				       struct list_head *head);
4967 
4968 static void _free_event(struct perf_event *event)
4969 {
4970 	irq_work_sync(&event->pending_irq);
4971 
4972 	unaccount_event(event);
4973 
4974 	security_perf_event_free(event);
4975 
4976 	if (event->rb) {
4977 		/*
4978 		 * Can happen when we close an event with re-directed output.
4979 		 *
4980 		 * Since we have a 0 refcount, perf_mmap_close() will skip
4981 		 * over us; possibly making our ring_buffer_put() the last.
4982 		 */
4983 		mutex_lock(&event->mmap_mutex);
4984 		ring_buffer_attach(event, NULL);
4985 		mutex_unlock(&event->mmap_mutex);
4986 	}
4987 
4988 	if (is_cgroup_event(event))
4989 		perf_detach_cgroup(event);
4990 
4991 	if (!event->parent) {
4992 		if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN)
4993 			put_callchain_buffers();
4994 	}
4995 
4996 	perf_event_free_bpf_prog(event);
4997 	perf_addr_filters_splice(event, NULL);
4998 	kfree(event->addr_filter_ranges);
4999 
5000 	if (event->destroy)
5001 		event->destroy(event);
5002 
5003 	/*
5004 	 * Must be after ->destroy(), due to uprobe_perf_close() using
5005 	 * hw.target.
5006 	 */
5007 	if (event->hw.target)
5008 		put_task_struct(event->hw.target);
5009 
5010 	/*
5011 	 * perf_event_free_task() relies on put_ctx() being 'last', in particular
5012 	 * all task references must be cleaned up.
5013 	 */
5014 	if (event->ctx)
5015 		put_ctx(event->ctx);
5016 
5017 	exclusive_event_destroy(event);
5018 	module_put(event->pmu->module);
5019 
5020 	call_rcu(&event->rcu_head, free_event_rcu);
5021 }
5022 
5023 /*
5024  * Used to free events which have a known refcount of 1, such as in error paths
5025  * where the event isn't exposed yet and inherited events.
5026  */
5027 static void free_event(struct perf_event *event)
5028 {
5029 	if (WARN(atomic_long_cmpxchg(&event->refcount, 1, 0) != 1,
5030 				"unexpected event refcount: %ld; ptr=%p\n",
5031 				atomic_long_read(&event->refcount), event)) {
5032 		/* leak to avoid use-after-free */
5033 		return;
5034 	}
5035 
5036 	_free_event(event);
5037 }
5038 
5039 /*
5040  * Remove user event from the owner task.
5041  */
5042 static void perf_remove_from_owner(struct perf_event *event)
5043 {
5044 	struct task_struct *owner;
5045 
5046 	rcu_read_lock();
5047 	/*
5048 	 * Matches the smp_store_release() in perf_event_exit_task(). If we
5049 	 * observe !owner it means the list deletion is complete and we can
5050 	 * indeed free this event, otherwise we need to serialize on
5051 	 * owner->perf_event_mutex.
5052 	 */
5053 	owner = READ_ONCE(event->owner);
5054 	if (owner) {
5055 		/*
5056 		 * Since delayed_put_task_struct() also drops the last
5057 		 * task reference we can safely take a new reference
5058 		 * while holding the rcu_read_lock().
5059 		 */
5060 		get_task_struct(owner);
5061 	}
5062 	rcu_read_unlock();
5063 
5064 	if (owner) {
5065 		/*
5066 		 * If we're here through perf_event_exit_task() we're already
5067 		 * holding ctx->mutex which would be an inversion wrt. the
5068 		 * normal lock order.
5069 		 *
5070 		 * However we can safely take this lock because its the child
5071 		 * ctx->mutex.
5072 		 */
5073 		mutex_lock_nested(&owner->perf_event_mutex, SINGLE_DEPTH_NESTING);
5074 
5075 		/*
5076 		 * We have to re-check the event->owner field, if it is cleared
5077 		 * we raced with perf_event_exit_task(), acquiring the mutex
5078 		 * ensured they're done, and we can proceed with freeing the
5079 		 * event.
5080 		 */
5081 		if (event->owner) {
5082 			list_del_init(&event->owner_entry);
5083 			smp_store_release(&event->owner, NULL);
5084 		}
5085 		mutex_unlock(&owner->perf_event_mutex);
5086 		put_task_struct(owner);
5087 	}
5088 }
5089 
5090 static void put_event(struct perf_event *event)
5091 {
5092 	if (!atomic_long_dec_and_test(&event->refcount))
5093 		return;
5094 
5095 	_free_event(event);
5096 }
5097 
5098 /*
5099  * Kill an event dead; while event:refcount will preserve the event
5100  * object, it will not preserve its functionality. Once the last 'user'
5101  * gives up the object, we'll destroy the thing.
5102  */
5103 int perf_event_release_kernel(struct perf_event *event)
5104 {
5105 	struct perf_event_context *ctx = event->ctx;
5106 	struct perf_event *child, *tmp;
5107 	LIST_HEAD(free_list);
5108 
5109 	/*
5110 	 * If we got here through err_file: fput(event_file); we will not have
5111 	 * attached to a context yet.
5112 	 */
5113 	if (!ctx) {
5114 		WARN_ON_ONCE(event->attach_state &
5115 				(PERF_ATTACH_CONTEXT|PERF_ATTACH_GROUP));
5116 		goto no_ctx;
5117 	}
5118 
5119 	if (!is_kernel_event(event))
5120 		perf_remove_from_owner(event);
5121 
5122 	ctx = perf_event_ctx_lock(event);
5123 	WARN_ON_ONCE(ctx->parent_ctx);
5124 	perf_remove_from_context(event, DETACH_GROUP);
5125 
5126 	raw_spin_lock_irq(&ctx->lock);
5127 	/*
5128 	 * Mark this event as STATE_DEAD, there is no external reference to it
5129 	 * anymore.
5130 	 *
5131 	 * Anybody acquiring event->child_mutex after the below loop _must_
5132 	 * also see this, most importantly inherit_event() which will avoid
5133 	 * placing more children on the list.
5134 	 *
5135 	 * Thus this guarantees that we will in fact observe and kill _ALL_
5136 	 * child events.
5137 	 */
5138 	event->state = PERF_EVENT_STATE_DEAD;
5139 	raw_spin_unlock_irq(&ctx->lock);
5140 
5141 	perf_event_ctx_unlock(event, ctx);
5142 
5143 again:
5144 	mutex_lock(&event->child_mutex);
5145 	list_for_each_entry(child, &event->child_list, child_list) {
5146 
5147 		/*
5148 		 * Cannot change, child events are not migrated, see the
5149 		 * comment with perf_event_ctx_lock_nested().
5150 		 */
5151 		ctx = READ_ONCE(child->ctx);
5152 		/*
5153 		 * Since child_mutex nests inside ctx::mutex, we must jump
5154 		 * through hoops. We start by grabbing a reference on the ctx.
5155 		 *
5156 		 * Since the event cannot get freed while we hold the
5157 		 * child_mutex, the context must also exist and have a !0
5158 		 * reference count.
5159 		 */
5160 		get_ctx(ctx);
5161 
5162 		/*
5163 		 * Now that we have a ctx ref, we can drop child_mutex, and
5164 		 * acquire ctx::mutex without fear of it going away. Then we
5165 		 * can re-acquire child_mutex.
5166 		 */
5167 		mutex_unlock(&event->child_mutex);
5168 		mutex_lock(&ctx->mutex);
5169 		mutex_lock(&event->child_mutex);
5170 
5171 		/*
5172 		 * Now that we hold ctx::mutex and child_mutex, revalidate our
5173 		 * state, if child is still the first entry, it didn't get freed
5174 		 * and we can continue doing so.
5175 		 */
5176 		tmp = list_first_entry_or_null(&event->child_list,
5177 					       struct perf_event, child_list);
5178 		if (tmp == child) {
5179 			perf_remove_from_context(child, DETACH_GROUP);
5180 			list_move(&child->child_list, &free_list);
5181 			/*
5182 			 * This matches the refcount bump in inherit_event();
5183 			 * this can't be the last reference.
5184 			 */
5185 			put_event(event);
5186 		}
5187 
5188 		mutex_unlock(&event->child_mutex);
5189 		mutex_unlock(&ctx->mutex);
5190 		put_ctx(ctx);
5191 		goto again;
5192 	}
5193 	mutex_unlock(&event->child_mutex);
5194 
5195 	list_for_each_entry_safe(child, tmp, &free_list, child_list) {
5196 		void *var = &child->ctx->refcount;
5197 
5198 		list_del(&child->child_list);
5199 		free_event(child);
5200 
5201 		/*
5202 		 * Wake any perf_event_free_task() waiting for this event to be
5203 		 * freed.
5204 		 */
5205 		smp_mb(); /* pairs with wait_var_event() */
5206 		wake_up_var(var);
5207 	}
5208 
5209 no_ctx:
5210 	put_event(event); /* Must be the 'last' reference */
5211 	return 0;
5212 }
5213 EXPORT_SYMBOL_GPL(perf_event_release_kernel);
5214 
5215 /*
5216  * Called when the last reference to the file is gone.
5217  */
5218 static int perf_release(struct inode *inode, struct file *file)
5219 {
5220 	perf_event_release_kernel(file->private_data);
5221 	return 0;
5222 }
5223 
5224 static u64 __perf_event_read_value(struct perf_event *event, u64 *enabled, u64 *running)
5225 {
5226 	struct perf_event *child;
5227 	u64 total = 0;
5228 
5229 	*enabled = 0;
5230 	*running = 0;
5231 
5232 	mutex_lock(&event->child_mutex);
5233 
5234 	(void)perf_event_read(event, false);
5235 	total += perf_event_count(event);
5236 
5237 	*enabled += event->total_time_enabled +
5238 			atomic64_read(&event->child_total_time_enabled);
5239 	*running += event->total_time_running +
5240 			atomic64_read(&event->child_total_time_running);
5241 
5242 	list_for_each_entry(child, &event->child_list, child_list) {
5243 		(void)perf_event_read(child, false);
5244 		total += perf_event_count(child);
5245 		*enabled += child->total_time_enabled;
5246 		*running += child->total_time_running;
5247 	}
5248 	mutex_unlock(&event->child_mutex);
5249 
5250 	return total;
5251 }
5252 
5253 u64 perf_event_read_value(struct perf_event *event, u64 *enabled, u64 *running)
5254 {
5255 	struct perf_event_context *ctx;
5256 	u64 count;
5257 
5258 	ctx = perf_event_ctx_lock(event);
5259 	count = __perf_event_read_value(event, enabled, running);
5260 	perf_event_ctx_unlock(event, ctx);
5261 
5262 	return count;
5263 }
5264 EXPORT_SYMBOL_GPL(perf_event_read_value);
5265 
5266 static int __perf_read_group_add(struct perf_event *leader,
5267 					u64 read_format, u64 *values)
5268 {
5269 	struct perf_event_context *ctx = leader->ctx;
5270 	struct perf_event *sub;
5271 	unsigned long flags;
5272 	int n = 1; /* skip @nr */
5273 	int ret;
5274 
5275 	ret = perf_event_read(leader, true);
5276 	if (ret)
5277 		return ret;
5278 
5279 	raw_spin_lock_irqsave(&ctx->lock, flags);
5280 
5281 	/*
5282 	 * Since we co-schedule groups, {enabled,running} times of siblings
5283 	 * will be identical to those of the leader, so we only publish one
5284 	 * set.
5285 	 */
5286 	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
5287 		values[n++] += leader->total_time_enabled +
5288 			atomic64_read(&leader->child_total_time_enabled);
5289 	}
5290 
5291 	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
5292 		values[n++] += leader->total_time_running +
5293 			atomic64_read(&leader->child_total_time_running);
5294 	}
5295 
5296 	/*
5297 	 * Write {count,id} tuples for every sibling.
5298 	 */
5299 	values[n++] += perf_event_count(leader);
5300 	if (read_format & PERF_FORMAT_ID)
5301 		values[n++] = primary_event_id(leader);
5302 	if (read_format & PERF_FORMAT_LOST)
5303 		values[n++] = atomic64_read(&leader->lost_samples);
5304 
5305 	for_each_sibling_event(sub, leader) {
5306 		values[n++] += perf_event_count(sub);
5307 		if (read_format & PERF_FORMAT_ID)
5308 			values[n++] = primary_event_id(sub);
5309 		if (read_format & PERF_FORMAT_LOST)
5310 			values[n++] = atomic64_read(&sub->lost_samples);
5311 	}
5312 
5313 	raw_spin_unlock_irqrestore(&ctx->lock, flags);
5314 	return 0;
5315 }
5316 
5317 static int perf_read_group(struct perf_event *event,
5318 				   u64 read_format, char __user *buf)
5319 {
5320 	struct perf_event *leader = event->group_leader, *child;
5321 	struct perf_event_context *ctx = leader->ctx;
5322 	int ret;
5323 	u64 *values;
5324 
5325 	lockdep_assert_held(&ctx->mutex);
5326 
5327 	values = kzalloc(event->read_size, GFP_KERNEL);
5328 	if (!values)
5329 		return -ENOMEM;
5330 
5331 	values[0] = 1 + leader->nr_siblings;
5332 
5333 	/*
5334 	 * By locking the child_mutex of the leader we effectively
5335 	 * lock the child list of all siblings.. XXX explain how.
5336 	 */
5337 	mutex_lock(&leader->child_mutex);
5338 
5339 	ret = __perf_read_group_add(leader, read_format, values);
5340 	if (ret)
5341 		goto unlock;
5342 
5343 	list_for_each_entry(child, &leader->child_list, child_list) {
5344 		ret = __perf_read_group_add(child, read_format, values);
5345 		if (ret)
5346 			goto unlock;
5347 	}
5348 
5349 	mutex_unlock(&leader->child_mutex);
5350 
5351 	ret = event->read_size;
5352 	if (copy_to_user(buf, values, event->read_size))
5353 		ret = -EFAULT;
5354 	goto out;
5355 
5356 unlock:
5357 	mutex_unlock(&leader->child_mutex);
5358 out:
5359 	kfree(values);
5360 	return ret;
5361 }
5362 
5363 static int perf_read_one(struct perf_event *event,
5364 				 u64 read_format, char __user *buf)
5365 {
5366 	u64 enabled, running;
5367 	u64 values[5];
5368 	int n = 0;
5369 
5370 	values[n++] = __perf_event_read_value(event, &enabled, &running);
5371 	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
5372 		values[n++] = enabled;
5373 	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
5374 		values[n++] = running;
5375 	if (read_format & PERF_FORMAT_ID)
5376 		values[n++] = primary_event_id(event);
5377 	if (read_format & PERF_FORMAT_LOST)
5378 		values[n++] = atomic64_read(&event->lost_samples);
5379 
5380 	if (copy_to_user(buf, values, n * sizeof(u64)))
5381 		return -EFAULT;
5382 
5383 	return n * sizeof(u64);
5384 }
5385 
5386 static bool is_event_hup(struct perf_event *event)
5387 {
5388 	bool no_children;
5389 
5390 	if (event->state > PERF_EVENT_STATE_EXIT)
5391 		return false;
5392 
5393 	mutex_lock(&event->child_mutex);
5394 	no_children = list_empty(&event->child_list);
5395 	mutex_unlock(&event->child_mutex);
5396 	return no_children;
5397 }
5398 
5399 /*
5400  * Read the performance event - simple non blocking version for now
5401  */
5402 static ssize_t
5403 __perf_read(struct perf_event *event, char __user *buf, size_t count)
5404 {
5405 	u64 read_format = event->attr.read_format;
5406 	int ret;
5407 
5408 	/*
5409 	 * Return end-of-file for a read on an event that is in
5410 	 * error state (i.e. because it was pinned but it couldn't be
5411 	 * scheduled on to the CPU at some point).
5412 	 */
5413 	if (event->state == PERF_EVENT_STATE_ERROR)
5414 		return 0;
5415 
5416 	if (count < event->read_size)
5417 		return -ENOSPC;
5418 
5419 	WARN_ON_ONCE(event->ctx->parent_ctx);
5420 	if (read_format & PERF_FORMAT_GROUP)
5421 		ret = perf_read_group(event, read_format, buf);
5422 	else
5423 		ret = perf_read_one(event, read_format, buf);
5424 
5425 	return ret;
5426 }
5427 
5428 static ssize_t
5429 perf_read(struct file *file, char __user *buf, size_t count, loff_t *ppos)
5430 {
5431 	struct perf_event *event = file->private_data;
5432 	struct perf_event_context *ctx;
5433 	int ret;
5434 
5435 	ret = security_perf_event_read(event);
5436 	if (ret)
5437 		return ret;
5438 
5439 	ctx = perf_event_ctx_lock(event);
5440 	ret = __perf_read(event, buf, count);
5441 	perf_event_ctx_unlock(event, ctx);
5442 
5443 	return ret;
5444 }
5445 
5446 static __poll_t perf_poll(struct file *file, poll_table *wait)
5447 {
5448 	struct perf_event *event = file->private_data;
5449 	struct perf_buffer *rb;
5450 	__poll_t events = EPOLLHUP;
5451 
5452 	poll_wait(file, &event->waitq, wait);
5453 
5454 	if (is_event_hup(event))
5455 		return events;
5456 
5457 	/*
5458 	 * Pin the event->rb by taking event->mmap_mutex; otherwise
5459 	 * perf_event_set_output() can swizzle our rb and make us miss wakeups.
5460 	 */
5461 	mutex_lock(&event->mmap_mutex);
5462 	rb = event->rb;
5463 	if (rb)
5464 		events = atomic_xchg(&rb->poll, 0);
5465 	mutex_unlock(&event->mmap_mutex);
5466 	return events;
5467 }
5468 
5469 static void _perf_event_reset(struct perf_event *event)
5470 {
5471 	(void)perf_event_read(event, false);
5472 	local64_set(&event->count, 0);
5473 	perf_event_update_userpage(event);
5474 }
5475 
5476 /* Assume it's not an event with inherit set. */
5477 u64 perf_event_pause(struct perf_event *event, bool reset)
5478 {
5479 	struct perf_event_context *ctx;
5480 	u64 count;
5481 
5482 	ctx = perf_event_ctx_lock(event);
5483 	WARN_ON_ONCE(event->attr.inherit);
5484 	_perf_event_disable(event);
5485 	count = local64_read(&event->count);
5486 	if (reset)
5487 		local64_set(&event->count, 0);
5488 	perf_event_ctx_unlock(event, ctx);
5489 
5490 	return count;
5491 }
5492 EXPORT_SYMBOL_GPL(perf_event_pause);
5493 
5494 /*
5495  * Holding the top-level event's child_mutex means that any
5496  * descendant process that has inherited this event will block
5497  * in perf_event_exit_event() if it goes to exit, thus satisfying the
5498  * task existence requirements of perf_event_enable/disable.
5499  */
5500 static void perf_event_for_each_child(struct perf_event *event,
5501 					void (*func)(struct perf_event *))
5502 {
5503 	struct perf_event *child;
5504 
5505 	WARN_ON_ONCE(event->ctx->parent_ctx);
5506 
5507 	mutex_lock(&event->child_mutex);
5508 	func(event);
5509 	list_for_each_entry(child, &event->child_list, child_list)
5510 		func(child);
5511 	mutex_unlock(&event->child_mutex);
5512 }
5513 
5514 static void perf_event_for_each(struct perf_event *event,
5515 				  void (*func)(struct perf_event *))
5516 {
5517 	struct perf_event_context *ctx = event->ctx;
5518 	struct perf_event *sibling;
5519 
5520 	lockdep_assert_held(&ctx->mutex);
5521 
5522 	event = event->group_leader;
5523 
5524 	perf_event_for_each_child(event, func);
5525 	for_each_sibling_event(sibling, event)
5526 		perf_event_for_each_child(sibling, func);
5527 }
5528 
5529 static void __perf_event_period(struct perf_event *event,
5530 				struct perf_cpu_context *cpuctx,
5531 				struct perf_event_context *ctx,
5532 				void *info)
5533 {
5534 	u64 value = *((u64 *)info);
5535 	bool active;
5536 
5537 	if (event->attr.freq) {
5538 		event->attr.sample_freq = value;
5539 	} else {
5540 		event->attr.sample_period = value;
5541 		event->hw.sample_period = value;
5542 	}
5543 
5544 	active = (event->state == PERF_EVENT_STATE_ACTIVE);
5545 	if (active) {
5546 		perf_pmu_disable(ctx->pmu);
5547 		/*
5548 		 * We could be throttled; unthrottle now to avoid the tick
5549 		 * trying to unthrottle while we already re-started the event.
5550 		 */
5551 		if (event->hw.interrupts == MAX_INTERRUPTS) {
5552 			event->hw.interrupts = 0;
5553 			perf_log_throttle(event, 1);
5554 		}
5555 		event->pmu->stop(event, PERF_EF_UPDATE);
5556 	}
5557 
5558 	local64_set(&event->hw.period_left, 0);
5559 
5560 	if (active) {
5561 		event->pmu->start(event, PERF_EF_RELOAD);
5562 		perf_pmu_enable(ctx->pmu);
5563 	}
5564 }
5565 
5566 static int perf_event_check_period(struct perf_event *event, u64 value)
5567 {
5568 	return event->pmu->check_period(event, value);
5569 }
5570 
5571 static int _perf_event_period(struct perf_event *event, u64 value)
5572 {
5573 	if (!is_sampling_event(event))
5574 		return -EINVAL;
5575 
5576 	if (!value)
5577 		return -EINVAL;
5578 
5579 	if (event->attr.freq && value > sysctl_perf_event_sample_rate)
5580 		return -EINVAL;
5581 
5582 	if (perf_event_check_period(event, value))
5583 		return -EINVAL;
5584 
5585 	if (!event->attr.freq && (value & (1ULL << 63)))
5586 		return -EINVAL;
5587 
5588 	event_function_call(event, __perf_event_period, &value);
5589 
5590 	return 0;
5591 }
5592 
5593 int perf_event_period(struct perf_event *event, u64 value)
5594 {
5595 	struct perf_event_context *ctx;
5596 	int ret;
5597 
5598 	ctx = perf_event_ctx_lock(event);
5599 	ret = _perf_event_period(event, value);
5600 	perf_event_ctx_unlock(event, ctx);
5601 
5602 	return ret;
5603 }
5604 EXPORT_SYMBOL_GPL(perf_event_period);
5605 
5606 static const struct file_operations perf_fops;
5607 
5608 static inline int perf_fget_light(int fd, struct fd *p)
5609 {
5610 	struct fd f = fdget(fd);
5611 	if (!f.file)
5612 		return -EBADF;
5613 
5614 	if (f.file->f_op != &perf_fops) {
5615 		fdput(f);
5616 		return -EBADF;
5617 	}
5618 	*p = f;
5619 	return 0;
5620 }
5621 
5622 static int perf_event_set_output(struct perf_event *event,
5623 				 struct perf_event *output_event);
5624 static int perf_event_set_filter(struct perf_event *event, void __user *arg);
5625 static int perf_copy_attr(struct perf_event_attr __user *uattr,
5626 			  struct perf_event_attr *attr);
5627 
5628 static long _perf_ioctl(struct perf_event *event, unsigned int cmd, unsigned long arg)
5629 {
5630 	void (*func)(struct perf_event *);
5631 	u32 flags = arg;
5632 
5633 	switch (cmd) {
5634 	case PERF_EVENT_IOC_ENABLE:
5635 		func = _perf_event_enable;
5636 		break;
5637 	case PERF_EVENT_IOC_DISABLE:
5638 		func = _perf_event_disable;
5639 		break;
5640 	case PERF_EVENT_IOC_RESET:
5641 		func = _perf_event_reset;
5642 		break;
5643 
5644 	case PERF_EVENT_IOC_REFRESH:
5645 		return _perf_event_refresh(event, arg);
5646 
5647 	case PERF_EVENT_IOC_PERIOD:
5648 	{
5649 		u64 value;
5650 
5651 		if (copy_from_user(&value, (u64 __user *)arg, sizeof(value)))
5652 			return -EFAULT;
5653 
5654 		return _perf_event_period(event, value);
5655 	}
5656 	case PERF_EVENT_IOC_ID:
5657 	{
5658 		u64 id = primary_event_id(event);
5659 
5660 		if (copy_to_user((void __user *)arg, &id, sizeof(id)))
5661 			return -EFAULT;
5662 		return 0;
5663 	}
5664 
5665 	case PERF_EVENT_IOC_SET_OUTPUT:
5666 	{
5667 		int ret;
5668 		if (arg != -1) {
5669 			struct perf_event *output_event;
5670 			struct fd output;
5671 			ret = perf_fget_light(arg, &output);
5672 			if (ret)
5673 				return ret;
5674 			output_event = output.file->private_data;
5675 			ret = perf_event_set_output(event, output_event);
5676 			fdput(output);
5677 		} else {
5678 			ret = perf_event_set_output(event, NULL);
5679 		}
5680 		return ret;
5681 	}
5682 
5683 	case PERF_EVENT_IOC_SET_FILTER:
5684 		return perf_event_set_filter(event, (void __user *)arg);
5685 
5686 	case PERF_EVENT_IOC_SET_BPF:
5687 	{
5688 		struct bpf_prog *prog;
5689 		int err;
5690 
5691 		prog = bpf_prog_get(arg);
5692 		if (IS_ERR(prog))
5693 			return PTR_ERR(prog);
5694 
5695 		err = perf_event_set_bpf_prog(event, prog, 0);
5696 		if (err) {
5697 			bpf_prog_put(prog);
5698 			return err;
5699 		}
5700 
5701 		return 0;
5702 	}
5703 
5704 	case PERF_EVENT_IOC_PAUSE_OUTPUT: {
5705 		struct perf_buffer *rb;
5706 
5707 		rcu_read_lock();
5708 		rb = rcu_dereference(event->rb);
5709 		if (!rb || !rb->nr_pages) {
5710 			rcu_read_unlock();
5711 			return -EINVAL;
5712 		}
5713 		rb_toggle_paused(rb, !!arg);
5714 		rcu_read_unlock();
5715 		return 0;
5716 	}
5717 
5718 	case PERF_EVENT_IOC_QUERY_BPF:
5719 		return perf_event_query_prog_array(event, (void __user *)arg);
5720 
5721 	case PERF_EVENT_IOC_MODIFY_ATTRIBUTES: {
5722 		struct perf_event_attr new_attr;
5723 		int err = perf_copy_attr((struct perf_event_attr __user *)arg,
5724 					 &new_attr);
5725 
5726 		if (err)
5727 			return err;
5728 
5729 		return perf_event_modify_attr(event,  &new_attr);
5730 	}
5731 	default:
5732 		return -ENOTTY;
5733 	}
5734 
5735 	if (flags & PERF_IOC_FLAG_GROUP)
5736 		perf_event_for_each(event, func);
5737 	else
5738 		perf_event_for_each_child(event, func);
5739 
5740 	return 0;
5741 }
5742 
5743 static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
5744 {
5745 	struct perf_event *event = file->private_data;
5746 	struct perf_event_context *ctx;
5747 	long ret;
5748 
5749 	/* Treat ioctl like writes as it is likely a mutating operation. */
5750 	ret = security_perf_event_write(event);
5751 	if (ret)
5752 		return ret;
5753 
5754 	ctx = perf_event_ctx_lock(event);
5755 	ret = _perf_ioctl(event, cmd, arg);
5756 	perf_event_ctx_unlock(event, ctx);
5757 
5758 	return ret;
5759 }
5760 
5761 #ifdef CONFIG_COMPAT
5762 static long perf_compat_ioctl(struct file *file, unsigned int cmd,
5763 				unsigned long arg)
5764 {
5765 	switch (_IOC_NR(cmd)) {
5766 	case _IOC_NR(PERF_EVENT_IOC_SET_FILTER):
5767 	case _IOC_NR(PERF_EVENT_IOC_ID):
5768 	case _IOC_NR(PERF_EVENT_IOC_QUERY_BPF):
5769 	case _IOC_NR(PERF_EVENT_IOC_MODIFY_ATTRIBUTES):
5770 		/* Fix up pointer size (usually 4 -> 8 in 32-on-64-bit case */
5771 		if (_IOC_SIZE(cmd) == sizeof(compat_uptr_t)) {
5772 			cmd &= ~IOCSIZE_MASK;
5773 			cmd |= sizeof(void *) << IOCSIZE_SHIFT;
5774 		}
5775 		break;
5776 	}
5777 	return perf_ioctl(file, cmd, arg);
5778 }
5779 #else
5780 # define perf_compat_ioctl NULL
5781 #endif
5782 
5783 int perf_event_task_enable(void)
5784 {
5785 	struct perf_event_context *ctx;
5786 	struct perf_event *event;
5787 
5788 	mutex_lock(&current->perf_event_mutex);
5789 	list_for_each_entry(event, &current->perf_event_list, owner_entry) {
5790 		ctx = perf_event_ctx_lock(event);
5791 		perf_event_for_each_child(event, _perf_event_enable);
5792 		perf_event_ctx_unlock(event, ctx);
5793 	}
5794 	mutex_unlock(&current->perf_event_mutex);
5795 
5796 	return 0;
5797 }
5798 
5799 int perf_event_task_disable(void)
5800 {
5801 	struct perf_event_context *ctx;
5802 	struct perf_event *event;
5803 
5804 	mutex_lock(&current->perf_event_mutex);
5805 	list_for_each_entry(event, &current->perf_event_list, owner_entry) {
5806 		ctx = perf_event_ctx_lock(event);
5807 		perf_event_for_each_child(event, _perf_event_disable);
5808 		perf_event_ctx_unlock(event, ctx);
5809 	}
5810 	mutex_unlock(&current->perf_event_mutex);
5811 
5812 	return 0;
5813 }
5814 
5815 static int perf_event_index(struct perf_event *event)
5816 {
5817 	if (event->hw.state & PERF_HES_STOPPED)
5818 		return 0;
5819 
5820 	if (event->state != PERF_EVENT_STATE_ACTIVE)
5821 		return 0;
5822 
5823 	return event->pmu->event_idx(event);
5824 }
5825 
5826 static void perf_event_init_userpage(struct perf_event *event)
5827 {
5828 	struct perf_event_mmap_page *userpg;
5829 	struct perf_buffer *rb;
5830 
5831 	rcu_read_lock();
5832 	rb = rcu_dereference(event->rb);
5833 	if (!rb)
5834 		goto unlock;
5835 
5836 	userpg = rb->user_page;
5837 
5838 	/* Allow new userspace to detect that bit 0 is deprecated */
5839 	userpg->cap_bit0_is_deprecated = 1;
5840 	userpg->size = offsetof(struct perf_event_mmap_page, __reserved);
5841 	userpg->data_offset = PAGE_SIZE;
5842 	userpg->data_size = perf_data_size(rb);
5843 
5844 unlock:
5845 	rcu_read_unlock();
5846 }
5847 
5848 void __weak arch_perf_update_userpage(
5849 	struct perf_event *event, struct perf_event_mmap_page *userpg, u64 now)
5850 {
5851 }
5852 
5853 /*
5854  * Callers need to ensure there can be no nesting of this function, otherwise
5855  * the seqlock logic goes bad. We can not serialize this because the arch
5856  * code calls this from NMI context.
5857  */
5858 void perf_event_update_userpage(struct perf_event *event)
5859 {
5860 	struct perf_event_mmap_page *userpg;
5861 	struct perf_buffer *rb;
5862 	u64 enabled, running, now;
5863 
5864 	rcu_read_lock();
5865 	rb = rcu_dereference(event->rb);
5866 	if (!rb)
5867 		goto unlock;
5868 
5869 	/*
5870 	 * compute total_time_enabled, total_time_running
5871 	 * based on snapshot values taken when the event
5872 	 * was last scheduled in.
5873 	 *
5874 	 * we cannot simply called update_context_time()
5875 	 * because of locking issue as we can be called in
5876 	 * NMI context
5877 	 */
5878 	calc_timer_values(event, &now, &enabled, &running);
5879 
5880 	userpg = rb->user_page;
5881 	/*
5882 	 * Disable preemption to guarantee consistent time stamps are stored to
5883 	 * the user page.
5884 	 */
5885 	preempt_disable();
5886 	++userpg->lock;
5887 	barrier();
5888 	userpg->index = perf_event_index(event);
5889 	userpg->offset = perf_event_count(event);
5890 	if (userpg->index)
5891 		userpg->offset -= local64_read(&event->hw.prev_count);
5892 
5893 	userpg->time_enabled = enabled +
5894 			atomic64_read(&event->child_total_time_enabled);
5895 
5896 	userpg->time_running = running +
5897 			atomic64_read(&event->child_total_time_running);
5898 
5899 	arch_perf_update_userpage(event, userpg, now);
5900 
5901 	barrier();
5902 	++userpg->lock;
5903 	preempt_enable();
5904 unlock:
5905 	rcu_read_unlock();
5906 }
5907 EXPORT_SYMBOL_GPL(perf_event_update_userpage);
5908 
5909 static vm_fault_t perf_mmap_fault(struct vm_fault *vmf)
5910 {
5911 	struct perf_event *event = vmf->vma->vm_file->private_data;
5912 	struct perf_buffer *rb;
5913 	vm_fault_t ret = VM_FAULT_SIGBUS;
5914 
5915 	if (vmf->flags & FAULT_FLAG_MKWRITE) {
5916 		if (vmf->pgoff == 0)
5917 			ret = 0;
5918 		return ret;
5919 	}
5920 
5921 	rcu_read_lock();
5922 	rb = rcu_dereference(event->rb);
5923 	if (!rb)
5924 		goto unlock;
5925 
5926 	if (vmf->pgoff && (vmf->flags & FAULT_FLAG_WRITE))
5927 		goto unlock;
5928 
5929 	vmf->page = perf_mmap_to_page(rb, vmf->pgoff);
5930 	if (!vmf->page)
5931 		goto unlock;
5932 
5933 	get_page(vmf->page);
5934 	vmf->page->mapping = vmf->vma->vm_file->f_mapping;
5935 	vmf->page->index   = vmf->pgoff;
5936 
5937 	ret = 0;
5938 unlock:
5939 	rcu_read_unlock();
5940 
5941 	return ret;
5942 }
5943 
5944 static void ring_buffer_attach(struct perf_event *event,
5945 			       struct perf_buffer *rb)
5946 {
5947 	struct perf_buffer *old_rb = NULL;
5948 	unsigned long flags;
5949 
5950 	WARN_ON_ONCE(event->parent);
5951 
5952 	if (event->rb) {
5953 		/*
5954 		 * Should be impossible, we set this when removing
5955 		 * event->rb_entry and wait/clear when adding event->rb_entry.
5956 		 */
5957 		WARN_ON_ONCE(event->rcu_pending);
5958 
5959 		old_rb = event->rb;
5960 		spin_lock_irqsave(&old_rb->event_lock, flags);
5961 		list_del_rcu(&event->rb_entry);
5962 		spin_unlock_irqrestore(&old_rb->event_lock, flags);
5963 
5964 		event->rcu_batches = get_state_synchronize_rcu();
5965 		event->rcu_pending = 1;
5966 	}
5967 
5968 	if (rb) {
5969 		if (event->rcu_pending) {
5970 			cond_synchronize_rcu(event->rcu_batches);
5971 			event->rcu_pending = 0;
5972 		}
5973 
5974 		spin_lock_irqsave(&rb->event_lock, flags);
5975 		list_add_rcu(&event->rb_entry, &rb->event_list);
5976 		spin_unlock_irqrestore(&rb->event_lock, flags);
5977 	}
5978 
5979 	/*
5980 	 * Avoid racing with perf_mmap_close(AUX): stop the event
5981 	 * before swizzling the event::rb pointer; if it's getting
5982 	 * unmapped, its aux_mmap_count will be 0 and it won't
5983 	 * restart. See the comment in __perf_pmu_output_stop().
5984 	 *
5985 	 * Data will inevitably be lost when set_output is done in
5986 	 * mid-air, but then again, whoever does it like this is
5987 	 * not in for the data anyway.
5988 	 */
5989 	if (has_aux(event))
5990 		perf_event_stop(event, 0);
5991 
5992 	rcu_assign_pointer(event->rb, rb);
5993 
5994 	if (old_rb) {
5995 		ring_buffer_put(old_rb);
5996 		/*
5997 		 * Since we detached before setting the new rb, so that we
5998 		 * could attach the new rb, we could have missed a wakeup.
5999 		 * Provide it now.
6000 		 */
6001 		wake_up_all(&event->waitq);
6002 	}
6003 }
6004 
6005 static void ring_buffer_wakeup(struct perf_event *event)
6006 {
6007 	struct perf_buffer *rb;
6008 
6009 	if (event->parent)
6010 		event = event->parent;
6011 
6012 	rcu_read_lock();
6013 	rb = rcu_dereference(event->rb);
6014 	if (rb) {
6015 		list_for_each_entry_rcu(event, &rb->event_list, rb_entry)
6016 			wake_up_all(&event->waitq);
6017 	}
6018 	rcu_read_unlock();
6019 }
6020 
6021 struct perf_buffer *ring_buffer_get(struct perf_event *event)
6022 {
6023 	struct perf_buffer *rb;
6024 
6025 	if (event->parent)
6026 		event = event->parent;
6027 
6028 	rcu_read_lock();
6029 	rb = rcu_dereference(event->rb);
6030 	if (rb) {
6031 		if (!refcount_inc_not_zero(&rb->refcount))
6032 			rb = NULL;
6033 	}
6034 	rcu_read_unlock();
6035 
6036 	return rb;
6037 }
6038 
6039 void ring_buffer_put(struct perf_buffer *rb)
6040 {
6041 	if (!refcount_dec_and_test(&rb->refcount))
6042 		return;
6043 
6044 	WARN_ON_ONCE(!list_empty(&rb->event_list));
6045 
6046 	call_rcu(&rb->rcu_head, rb_free_rcu);
6047 }
6048 
6049 static void perf_mmap_open(struct vm_area_struct *vma)
6050 {
6051 	struct perf_event *event = vma->vm_file->private_data;
6052 
6053 	atomic_inc(&event->mmap_count);
6054 	atomic_inc(&event->rb->mmap_count);
6055 
6056 	if (vma->vm_pgoff)
6057 		atomic_inc(&event->rb->aux_mmap_count);
6058 
6059 	if (event->pmu->event_mapped)
6060 		event->pmu->event_mapped(event, vma->vm_mm);
6061 }
6062 
6063 static void perf_pmu_output_stop(struct perf_event *event);
6064 
6065 /*
6066  * A buffer can be mmap()ed multiple times; either directly through the same
6067  * event, or through other events by use of perf_event_set_output().
6068  *
6069  * In order to undo the VM accounting done by perf_mmap() we need to destroy
6070  * the buffer here, where we still have a VM context. This means we need
6071  * to detach all events redirecting to us.
6072  */
6073 static void perf_mmap_close(struct vm_area_struct *vma)
6074 {
6075 	struct perf_event *event = vma->vm_file->private_data;
6076 	struct perf_buffer *rb = ring_buffer_get(event);
6077 	struct user_struct *mmap_user = rb->mmap_user;
6078 	int mmap_locked = rb->mmap_locked;
6079 	unsigned long size = perf_data_size(rb);
6080 	bool detach_rest = false;
6081 
6082 	if (event->pmu->event_unmapped)
6083 		event->pmu->event_unmapped(event, vma->vm_mm);
6084 
6085 	/*
6086 	 * rb->aux_mmap_count will always drop before rb->mmap_count and
6087 	 * event->mmap_count, so it is ok to use event->mmap_mutex to
6088 	 * serialize with perf_mmap here.
6089 	 */
6090 	if (rb_has_aux(rb) && vma->vm_pgoff == rb->aux_pgoff &&
6091 	    atomic_dec_and_mutex_lock(&rb->aux_mmap_count, &event->mmap_mutex)) {
6092 		/*
6093 		 * Stop all AUX events that are writing to this buffer,
6094 		 * so that we can free its AUX pages and corresponding PMU
6095 		 * data. Note that after rb::aux_mmap_count dropped to zero,
6096 		 * they won't start any more (see perf_aux_output_begin()).
6097 		 */
6098 		perf_pmu_output_stop(event);
6099 
6100 		/* now it's safe to free the pages */
6101 		atomic_long_sub(rb->aux_nr_pages - rb->aux_mmap_locked, &mmap_user->locked_vm);
6102 		atomic64_sub(rb->aux_mmap_locked, &vma->vm_mm->pinned_vm);
6103 
6104 		/* this has to be the last one */
6105 		rb_free_aux(rb);
6106 		WARN_ON_ONCE(refcount_read(&rb->aux_refcount));
6107 
6108 		mutex_unlock(&event->mmap_mutex);
6109 	}
6110 
6111 	if (atomic_dec_and_test(&rb->mmap_count))
6112 		detach_rest = true;
6113 
6114 	if (!atomic_dec_and_mutex_lock(&event->mmap_count, &event->mmap_mutex))
6115 		goto out_put;
6116 
6117 	ring_buffer_attach(event, NULL);
6118 	mutex_unlock(&event->mmap_mutex);
6119 
6120 	/* If there's still other mmap()s of this buffer, we're done. */
6121 	if (!detach_rest)
6122 		goto out_put;
6123 
6124 	/*
6125 	 * No other mmap()s, detach from all other events that might redirect
6126 	 * into the now unreachable buffer. Somewhat complicated by the
6127 	 * fact that rb::event_lock otherwise nests inside mmap_mutex.
6128 	 */
6129 again:
6130 	rcu_read_lock();
6131 	list_for_each_entry_rcu(event, &rb->event_list, rb_entry) {
6132 		if (!atomic_long_inc_not_zero(&event->refcount)) {
6133 			/*
6134 			 * This event is en-route to free_event() which will
6135 			 * detach it and remove it from the list.
6136 			 */
6137 			continue;
6138 		}
6139 		rcu_read_unlock();
6140 
6141 		mutex_lock(&event->mmap_mutex);
6142 		/*
6143 		 * Check we didn't race with perf_event_set_output() which can
6144 		 * swizzle the rb from under us while we were waiting to
6145 		 * acquire mmap_mutex.
6146 		 *
6147 		 * If we find a different rb; ignore this event, a next
6148 		 * iteration will no longer find it on the list. We have to
6149 		 * still restart the iteration to make sure we're not now
6150 		 * iterating the wrong list.
6151 		 */
6152 		if (event->rb == rb)
6153 			ring_buffer_attach(event, NULL);
6154 
6155 		mutex_unlock(&event->mmap_mutex);
6156 		put_event(event);
6157 
6158 		/*
6159 		 * Restart the iteration; either we're on the wrong list or
6160 		 * destroyed its integrity by doing a deletion.
6161 		 */
6162 		goto again;
6163 	}
6164 	rcu_read_unlock();
6165 
6166 	/*
6167 	 * It could be there's still a few 0-ref events on the list; they'll
6168 	 * get cleaned up by free_event() -- they'll also still have their
6169 	 * ref on the rb and will free it whenever they are done with it.
6170 	 *
6171 	 * Aside from that, this buffer is 'fully' detached and unmapped,
6172 	 * undo the VM accounting.
6173 	 */
6174 
6175 	atomic_long_sub((size >> PAGE_SHIFT) + 1 - mmap_locked,
6176 			&mmap_user->locked_vm);
6177 	atomic64_sub(mmap_locked, &vma->vm_mm->pinned_vm);
6178 	free_uid(mmap_user);
6179 
6180 out_put:
6181 	ring_buffer_put(rb); /* could be last */
6182 }
6183 
6184 static const struct vm_operations_struct perf_mmap_vmops = {
6185 	.open		= perf_mmap_open,
6186 	.close		= perf_mmap_close, /* non mergeable */
6187 	.fault		= perf_mmap_fault,
6188 	.page_mkwrite	= perf_mmap_fault,
6189 };
6190 
6191 static int perf_mmap(struct file *file, struct vm_area_struct *vma)
6192 {
6193 	struct perf_event *event = file->private_data;
6194 	unsigned long user_locked, user_lock_limit;
6195 	struct user_struct *user = current_user();
6196 	struct perf_buffer *rb = NULL;
6197 	unsigned long locked, lock_limit;
6198 	unsigned long vma_size;
6199 	unsigned long nr_pages;
6200 	long user_extra = 0, extra = 0;
6201 	int ret = 0, flags = 0;
6202 
6203 	/*
6204 	 * Don't allow mmap() of inherited per-task counters. This would
6205 	 * create a performance issue due to all children writing to the
6206 	 * same rb.
6207 	 */
6208 	if (event->cpu == -1 && event->attr.inherit)
6209 		return -EINVAL;
6210 
6211 	if (!(vma->vm_flags & VM_SHARED))
6212 		return -EINVAL;
6213 
6214 	ret = security_perf_event_read(event);
6215 	if (ret)
6216 		return ret;
6217 
6218 	vma_size = vma->vm_end - vma->vm_start;
6219 
6220 	if (vma->vm_pgoff == 0) {
6221 		nr_pages = (vma_size / PAGE_SIZE) - 1;
6222 	} else {
6223 		/*
6224 		 * AUX area mapping: if rb->aux_nr_pages != 0, it's already
6225 		 * mapped, all subsequent mappings should have the same size
6226 		 * and offset. Must be above the normal perf buffer.
6227 		 */
6228 		u64 aux_offset, aux_size;
6229 
6230 		if (!event->rb)
6231 			return -EINVAL;
6232 
6233 		nr_pages = vma_size / PAGE_SIZE;
6234 
6235 		mutex_lock(&event->mmap_mutex);
6236 		ret = -EINVAL;
6237 
6238 		rb = event->rb;
6239 		if (!rb)
6240 			goto aux_unlock;
6241 
6242 		aux_offset = READ_ONCE(rb->user_page->aux_offset);
6243 		aux_size = READ_ONCE(rb->user_page->aux_size);
6244 
6245 		if (aux_offset < perf_data_size(rb) + PAGE_SIZE)
6246 			goto aux_unlock;
6247 
6248 		if (aux_offset != vma->vm_pgoff << PAGE_SHIFT)
6249 			goto aux_unlock;
6250 
6251 		/* already mapped with a different offset */
6252 		if (rb_has_aux(rb) && rb->aux_pgoff != vma->vm_pgoff)
6253 			goto aux_unlock;
6254 
6255 		if (aux_size != vma_size || aux_size != nr_pages * PAGE_SIZE)
6256 			goto aux_unlock;
6257 
6258 		/* already mapped with a different size */
6259 		if (rb_has_aux(rb) && rb->aux_nr_pages != nr_pages)
6260 			goto aux_unlock;
6261 
6262 		if (!is_power_of_2(nr_pages))
6263 			goto aux_unlock;
6264 
6265 		if (!atomic_inc_not_zero(&rb->mmap_count))
6266 			goto aux_unlock;
6267 
6268 		if (rb_has_aux(rb)) {
6269 			atomic_inc(&rb->aux_mmap_count);
6270 			ret = 0;
6271 			goto unlock;
6272 		}
6273 
6274 		atomic_set(&rb->aux_mmap_count, 1);
6275 		user_extra = nr_pages;
6276 
6277 		goto accounting;
6278 	}
6279 
6280 	/*
6281 	 * If we have rb pages ensure they're a power-of-two number, so we
6282 	 * can do bitmasks instead of modulo.
6283 	 */
6284 	if (nr_pages != 0 && !is_power_of_2(nr_pages))
6285 		return -EINVAL;
6286 
6287 	if (vma_size != PAGE_SIZE * (1 + nr_pages))
6288 		return -EINVAL;
6289 
6290 	WARN_ON_ONCE(event->ctx->parent_ctx);
6291 again:
6292 	mutex_lock(&event->mmap_mutex);
6293 	if (event->rb) {
6294 		if (data_page_nr(event->rb) != nr_pages) {
6295 			ret = -EINVAL;
6296 			goto unlock;
6297 		}
6298 
6299 		if (!atomic_inc_not_zero(&event->rb->mmap_count)) {
6300 			/*
6301 			 * Raced against perf_mmap_close(); remove the
6302 			 * event and try again.
6303 			 */
6304 			ring_buffer_attach(event, NULL);
6305 			mutex_unlock(&event->mmap_mutex);
6306 			goto again;
6307 		}
6308 
6309 		goto unlock;
6310 	}
6311 
6312 	user_extra = nr_pages + 1;
6313 
6314 accounting:
6315 	user_lock_limit = sysctl_perf_event_mlock >> (PAGE_SHIFT - 10);
6316 
6317 	/*
6318 	 * Increase the limit linearly with more CPUs:
6319 	 */
6320 	user_lock_limit *= num_online_cpus();
6321 
6322 	user_locked = atomic_long_read(&user->locked_vm);
6323 
6324 	/*
6325 	 * sysctl_perf_event_mlock may have changed, so that
6326 	 *     user->locked_vm > user_lock_limit
6327 	 */
6328 	if (user_locked > user_lock_limit)
6329 		user_locked = user_lock_limit;
6330 	user_locked += user_extra;
6331 
6332 	if (user_locked > user_lock_limit) {
6333 		/*
6334 		 * charge locked_vm until it hits user_lock_limit;
6335 		 * charge the rest from pinned_vm
6336 		 */
6337 		extra = user_locked - user_lock_limit;
6338 		user_extra -= extra;
6339 	}
6340 
6341 	lock_limit = rlimit(RLIMIT_MEMLOCK);
6342 	lock_limit >>= PAGE_SHIFT;
6343 	locked = atomic64_read(&vma->vm_mm->pinned_vm) + extra;
6344 
6345 	if ((locked > lock_limit) && perf_is_paranoid() &&
6346 		!capable(CAP_IPC_LOCK)) {
6347 		ret = -EPERM;
6348 		goto unlock;
6349 	}
6350 
6351 	WARN_ON(!rb && event->rb);
6352 
6353 	if (vma->vm_flags & VM_WRITE)
6354 		flags |= RING_BUFFER_WRITABLE;
6355 
6356 	if (!rb) {
6357 		rb = rb_alloc(nr_pages,
6358 			      event->attr.watermark ? event->attr.wakeup_watermark : 0,
6359 			      event->cpu, flags);
6360 
6361 		if (!rb) {
6362 			ret = -ENOMEM;
6363 			goto unlock;
6364 		}
6365 
6366 		atomic_set(&rb->mmap_count, 1);
6367 		rb->mmap_user = get_current_user();
6368 		rb->mmap_locked = extra;
6369 
6370 		ring_buffer_attach(event, rb);
6371 
6372 		perf_event_update_time(event);
6373 		perf_event_init_userpage(event);
6374 		perf_event_update_userpage(event);
6375 	} else {
6376 		ret = rb_alloc_aux(rb, event, vma->vm_pgoff, nr_pages,
6377 				   event->attr.aux_watermark, flags);
6378 		if (!ret)
6379 			rb->aux_mmap_locked = extra;
6380 	}
6381 
6382 unlock:
6383 	if (!ret) {
6384 		atomic_long_add(user_extra, &user->locked_vm);
6385 		atomic64_add(extra, &vma->vm_mm->pinned_vm);
6386 
6387 		atomic_inc(&event->mmap_count);
6388 	} else if (rb) {
6389 		atomic_dec(&rb->mmap_count);
6390 	}
6391 aux_unlock:
6392 	mutex_unlock(&event->mmap_mutex);
6393 
6394 	/*
6395 	 * Since pinned accounting is per vm we cannot allow fork() to copy our
6396 	 * vma.
6397 	 */
6398 	vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_DONTDUMP;
6399 	vma->vm_ops = &perf_mmap_vmops;
6400 
6401 	if (event->pmu->event_mapped)
6402 		event->pmu->event_mapped(event, vma->vm_mm);
6403 
6404 	return ret;
6405 }
6406 
6407 static int perf_fasync(int fd, struct file *filp, int on)
6408 {
6409 	struct inode *inode = file_inode(filp);
6410 	struct perf_event *event = filp->private_data;
6411 	int retval;
6412 
6413 	inode_lock(inode);
6414 	retval = fasync_helper(fd, filp, on, &event->fasync);
6415 	inode_unlock(inode);
6416 
6417 	if (retval < 0)
6418 		return retval;
6419 
6420 	return 0;
6421 }
6422 
6423 static const struct file_operations perf_fops = {
6424 	.llseek			= no_llseek,
6425 	.release		= perf_release,
6426 	.read			= perf_read,
6427 	.poll			= perf_poll,
6428 	.unlocked_ioctl		= perf_ioctl,
6429 	.compat_ioctl		= perf_compat_ioctl,
6430 	.mmap			= perf_mmap,
6431 	.fasync			= perf_fasync,
6432 };
6433 
6434 /*
6435  * Perf event wakeup
6436  *
6437  * If there's data, ensure we set the poll() state and publish everything
6438  * to user-space before waking everybody up.
6439  */
6440 
6441 static inline struct fasync_struct **perf_event_fasync(struct perf_event *event)
6442 {
6443 	/* only the parent has fasync state */
6444 	if (event->parent)
6445 		event = event->parent;
6446 	return &event->fasync;
6447 }
6448 
6449 void perf_event_wakeup(struct perf_event *event)
6450 {
6451 	ring_buffer_wakeup(event);
6452 
6453 	if (event->pending_kill) {
6454 		kill_fasync(perf_event_fasync(event), SIGIO, event->pending_kill);
6455 		event->pending_kill = 0;
6456 	}
6457 }
6458 
6459 static void perf_sigtrap(struct perf_event *event)
6460 {
6461 	/*
6462 	 * We'd expect this to only occur if the irq_work is delayed and either
6463 	 * ctx->task or current has changed in the meantime. This can be the
6464 	 * case on architectures that do not implement arch_irq_work_raise().
6465 	 */
6466 	if (WARN_ON_ONCE(event->ctx->task != current))
6467 		return;
6468 
6469 	/*
6470 	 * Both perf_pending_task() and perf_pending_irq() can race with the
6471 	 * task exiting.
6472 	 */
6473 	if (current->flags & PF_EXITING)
6474 		return;
6475 
6476 	send_sig_perf((void __user *)event->pending_addr,
6477 		      event->attr.type, event->attr.sig_data);
6478 }
6479 
6480 /*
6481  * Deliver the pending work in-event-context or follow the context.
6482  */
6483 static void __perf_pending_irq(struct perf_event *event)
6484 {
6485 	int cpu = READ_ONCE(event->oncpu);
6486 
6487 	/*
6488 	 * If the event isn't running; we done. event_sched_out() will have
6489 	 * taken care of things.
6490 	 */
6491 	if (cpu < 0)
6492 		return;
6493 
6494 	/*
6495 	 * Yay, we hit home and are in the context of the event.
6496 	 */
6497 	if (cpu == smp_processor_id()) {
6498 		if (event->pending_sigtrap) {
6499 			event->pending_sigtrap = 0;
6500 			perf_sigtrap(event);
6501 			local_dec(&event->ctx->nr_pending);
6502 		}
6503 		if (event->pending_disable) {
6504 			event->pending_disable = 0;
6505 			perf_event_disable_local(event);
6506 		}
6507 		return;
6508 	}
6509 
6510 	/*
6511 	 *  CPU-A			CPU-B
6512 	 *
6513 	 *  perf_event_disable_inatomic()
6514 	 *    @pending_disable = CPU-A;
6515 	 *    irq_work_queue();
6516 	 *
6517 	 *  sched-out
6518 	 *    @pending_disable = -1;
6519 	 *
6520 	 *				sched-in
6521 	 *				perf_event_disable_inatomic()
6522 	 *				  @pending_disable = CPU-B;
6523 	 *				  irq_work_queue(); // FAILS
6524 	 *
6525 	 *  irq_work_run()
6526 	 *    perf_pending_irq()
6527 	 *
6528 	 * But the event runs on CPU-B and wants disabling there.
6529 	 */
6530 	irq_work_queue_on(&event->pending_irq, cpu);
6531 }
6532 
6533 static void perf_pending_irq(struct irq_work *entry)
6534 {
6535 	struct perf_event *event = container_of(entry, struct perf_event, pending_irq);
6536 	int rctx;
6537 
6538 	/*
6539 	 * If we 'fail' here, that's OK, it means recursion is already disabled
6540 	 * and we won't recurse 'further'.
6541 	 */
6542 	rctx = perf_swevent_get_recursion_context();
6543 
6544 	/*
6545 	 * The wakeup isn't bound to the context of the event -- it can happen
6546 	 * irrespective of where the event is.
6547 	 */
6548 	if (event->pending_wakeup) {
6549 		event->pending_wakeup = 0;
6550 		perf_event_wakeup(event);
6551 	}
6552 
6553 	__perf_pending_irq(event);
6554 
6555 	if (rctx >= 0)
6556 		perf_swevent_put_recursion_context(rctx);
6557 }
6558 
6559 static void perf_pending_task(struct callback_head *head)
6560 {
6561 	struct perf_event *event = container_of(head, struct perf_event, pending_task);
6562 	int rctx;
6563 
6564 	/*
6565 	 * If we 'fail' here, that's OK, it means recursion is already disabled
6566 	 * and we won't recurse 'further'.
6567 	 */
6568 	preempt_disable_notrace();
6569 	rctx = perf_swevent_get_recursion_context();
6570 
6571 	if (event->pending_work) {
6572 		event->pending_work = 0;
6573 		perf_sigtrap(event);
6574 		local_dec(&event->ctx->nr_pending);
6575 	}
6576 
6577 	if (rctx >= 0)
6578 		perf_swevent_put_recursion_context(rctx);
6579 	preempt_enable_notrace();
6580 }
6581 
6582 #ifdef CONFIG_GUEST_PERF_EVENTS
6583 struct perf_guest_info_callbacks __rcu *perf_guest_cbs;
6584 
6585 DEFINE_STATIC_CALL_RET0(__perf_guest_state, *perf_guest_cbs->state);
6586 DEFINE_STATIC_CALL_RET0(__perf_guest_get_ip, *perf_guest_cbs->get_ip);
6587 DEFINE_STATIC_CALL_RET0(__perf_guest_handle_intel_pt_intr, *perf_guest_cbs->handle_intel_pt_intr);
6588 
6589 void perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *cbs)
6590 {
6591 	if (WARN_ON_ONCE(rcu_access_pointer(perf_guest_cbs)))
6592 		return;
6593 
6594 	rcu_assign_pointer(perf_guest_cbs, cbs);
6595 	static_call_update(__perf_guest_state, cbs->state);
6596 	static_call_update(__perf_guest_get_ip, cbs->get_ip);
6597 
6598 	/* Implementing ->handle_intel_pt_intr is optional. */
6599 	if (cbs->handle_intel_pt_intr)
6600 		static_call_update(__perf_guest_handle_intel_pt_intr,
6601 				   cbs->handle_intel_pt_intr);
6602 }
6603 EXPORT_SYMBOL_GPL(perf_register_guest_info_callbacks);
6604 
6605 void perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *cbs)
6606 {
6607 	if (WARN_ON_ONCE(rcu_access_pointer(perf_guest_cbs) != cbs))
6608 		return;
6609 
6610 	rcu_assign_pointer(perf_guest_cbs, NULL);
6611 	static_call_update(__perf_guest_state, (void *)&__static_call_return0);
6612 	static_call_update(__perf_guest_get_ip, (void *)&__static_call_return0);
6613 	static_call_update(__perf_guest_handle_intel_pt_intr,
6614 			   (void *)&__static_call_return0);
6615 	synchronize_rcu();
6616 }
6617 EXPORT_SYMBOL_GPL(perf_unregister_guest_info_callbacks);
6618 #endif
6619 
6620 static void
6621 perf_output_sample_regs(struct perf_output_handle *handle,
6622 			struct pt_regs *regs, u64 mask)
6623 {
6624 	int bit;
6625 	DECLARE_BITMAP(_mask, 64);
6626 
6627 	bitmap_from_u64(_mask, mask);
6628 	for_each_set_bit(bit, _mask, sizeof(mask) * BITS_PER_BYTE) {
6629 		u64 val;
6630 
6631 		val = perf_reg_value(regs, bit);
6632 		perf_output_put(handle, val);
6633 	}
6634 }
6635 
6636 static void perf_sample_regs_user(struct perf_regs *regs_user,
6637 				  struct pt_regs *regs)
6638 {
6639 	if (user_mode(regs)) {
6640 		regs_user->abi = perf_reg_abi(current);
6641 		regs_user->regs = regs;
6642 	} else if (!(current->flags & PF_KTHREAD)) {
6643 		perf_get_regs_user(regs_user, regs);
6644 	} else {
6645 		regs_user->abi = PERF_SAMPLE_REGS_ABI_NONE;
6646 		regs_user->regs = NULL;
6647 	}
6648 }
6649 
6650 static void perf_sample_regs_intr(struct perf_regs *regs_intr,
6651 				  struct pt_regs *regs)
6652 {
6653 	regs_intr->regs = regs;
6654 	regs_intr->abi  = perf_reg_abi(current);
6655 }
6656 
6657 
6658 /*
6659  * Get remaining task size from user stack pointer.
6660  *
6661  * It'd be better to take stack vma map and limit this more
6662  * precisely, but there's no way to get it safely under interrupt,
6663  * so using TASK_SIZE as limit.
6664  */
6665 static u64 perf_ustack_task_size(struct pt_regs *regs)
6666 {
6667 	unsigned long addr = perf_user_stack_pointer(regs);
6668 
6669 	if (!addr || addr >= TASK_SIZE)
6670 		return 0;
6671 
6672 	return TASK_SIZE - addr;
6673 }
6674 
6675 static u16
6676 perf_sample_ustack_size(u16 stack_size, u16 header_size,
6677 			struct pt_regs *regs)
6678 {
6679 	u64 task_size;
6680 
6681 	/* No regs, no stack pointer, no dump. */
6682 	if (!regs)
6683 		return 0;
6684 
6685 	/*
6686 	 * Check if we fit in with the requested stack size into the:
6687 	 * - TASK_SIZE
6688 	 *   If we don't, we limit the size to the TASK_SIZE.
6689 	 *
6690 	 * - remaining sample size
6691 	 *   If we don't, we customize the stack size to
6692 	 *   fit in to the remaining sample size.
6693 	 */
6694 
6695 	task_size  = min((u64) USHRT_MAX, perf_ustack_task_size(regs));
6696 	stack_size = min(stack_size, (u16) task_size);
6697 
6698 	/* Current header size plus static size and dynamic size. */
6699 	header_size += 2 * sizeof(u64);
6700 
6701 	/* Do we fit in with the current stack dump size? */
6702 	if ((u16) (header_size + stack_size) < header_size) {
6703 		/*
6704 		 * If we overflow the maximum size for the sample,
6705 		 * we customize the stack dump size to fit in.
6706 		 */
6707 		stack_size = USHRT_MAX - header_size - sizeof(u64);
6708 		stack_size = round_up(stack_size, sizeof(u64));
6709 	}
6710 
6711 	return stack_size;
6712 }
6713 
6714 static void
6715 perf_output_sample_ustack(struct perf_output_handle *handle, u64 dump_size,
6716 			  struct pt_regs *regs)
6717 {
6718 	/* Case of a kernel thread, nothing to dump */
6719 	if (!regs) {
6720 		u64 size = 0;
6721 		perf_output_put(handle, size);
6722 	} else {
6723 		unsigned long sp;
6724 		unsigned int rem;
6725 		u64 dyn_size;
6726 
6727 		/*
6728 		 * We dump:
6729 		 * static size
6730 		 *   - the size requested by user or the best one we can fit
6731 		 *     in to the sample max size
6732 		 * data
6733 		 *   - user stack dump data
6734 		 * dynamic size
6735 		 *   - the actual dumped size
6736 		 */
6737 
6738 		/* Static size. */
6739 		perf_output_put(handle, dump_size);
6740 
6741 		/* Data. */
6742 		sp = perf_user_stack_pointer(regs);
6743 		rem = __output_copy_user(handle, (void *) sp, dump_size);
6744 		dyn_size = dump_size - rem;
6745 
6746 		perf_output_skip(handle, rem);
6747 
6748 		/* Dynamic size. */
6749 		perf_output_put(handle, dyn_size);
6750 	}
6751 }
6752 
6753 static unsigned long perf_prepare_sample_aux(struct perf_event *event,
6754 					  struct perf_sample_data *data,
6755 					  size_t size)
6756 {
6757 	struct perf_event *sampler = event->aux_event;
6758 	struct perf_buffer *rb;
6759 
6760 	data->aux_size = 0;
6761 
6762 	if (!sampler)
6763 		goto out;
6764 
6765 	if (WARN_ON_ONCE(READ_ONCE(sampler->state) != PERF_EVENT_STATE_ACTIVE))
6766 		goto out;
6767 
6768 	if (WARN_ON_ONCE(READ_ONCE(sampler->oncpu) != smp_processor_id()))
6769 		goto out;
6770 
6771 	rb = ring_buffer_get(sampler);
6772 	if (!rb)
6773 		goto out;
6774 
6775 	/*
6776 	 * If this is an NMI hit inside sampling code, don't take
6777 	 * the sample. See also perf_aux_sample_output().
6778 	 */
6779 	if (READ_ONCE(rb->aux_in_sampling)) {
6780 		data->aux_size = 0;
6781 	} else {
6782 		size = min_t(size_t, size, perf_aux_size(rb));
6783 		data->aux_size = ALIGN(size, sizeof(u64));
6784 	}
6785 	ring_buffer_put(rb);
6786 
6787 out:
6788 	return data->aux_size;
6789 }
6790 
6791 static long perf_pmu_snapshot_aux(struct perf_buffer *rb,
6792                                  struct perf_event *event,
6793                                  struct perf_output_handle *handle,
6794                                  unsigned long size)
6795 {
6796 	unsigned long flags;
6797 	long ret;
6798 
6799 	/*
6800 	 * Normal ->start()/->stop() callbacks run in IRQ mode in scheduler
6801 	 * paths. If we start calling them in NMI context, they may race with
6802 	 * the IRQ ones, that is, for example, re-starting an event that's just
6803 	 * been stopped, which is why we're using a separate callback that
6804 	 * doesn't change the event state.
6805 	 *
6806 	 * IRQs need to be disabled to prevent IPIs from racing with us.
6807 	 */
6808 	local_irq_save(flags);
6809 	/*
6810 	 * Guard against NMI hits inside the critical section;
6811 	 * see also perf_prepare_sample_aux().
6812 	 */
6813 	WRITE_ONCE(rb->aux_in_sampling, 1);
6814 	barrier();
6815 
6816 	ret = event->pmu->snapshot_aux(event, handle, size);
6817 
6818 	barrier();
6819 	WRITE_ONCE(rb->aux_in_sampling, 0);
6820 	local_irq_restore(flags);
6821 
6822 	return ret;
6823 }
6824 
6825 static void perf_aux_sample_output(struct perf_event *event,
6826 				   struct perf_output_handle *handle,
6827 				   struct perf_sample_data *data)
6828 {
6829 	struct perf_event *sampler = event->aux_event;
6830 	struct perf_buffer *rb;
6831 	unsigned long pad;
6832 	long size;
6833 
6834 	if (WARN_ON_ONCE(!sampler || !data->aux_size))
6835 		return;
6836 
6837 	rb = ring_buffer_get(sampler);
6838 	if (!rb)
6839 		return;
6840 
6841 	size = perf_pmu_snapshot_aux(rb, sampler, handle, data->aux_size);
6842 
6843 	/*
6844 	 * An error here means that perf_output_copy() failed (returned a
6845 	 * non-zero surplus that it didn't copy), which in its current
6846 	 * enlightened implementation is not possible. If that changes, we'd
6847 	 * like to know.
6848 	 */
6849 	if (WARN_ON_ONCE(size < 0))
6850 		goto out_put;
6851 
6852 	/*
6853 	 * The pad comes from ALIGN()ing data->aux_size up to u64 in
6854 	 * perf_prepare_sample_aux(), so should not be more than that.
6855 	 */
6856 	pad = data->aux_size - size;
6857 	if (WARN_ON_ONCE(pad >= sizeof(u64)))
6858 		pad = 8;
6859 
6860 	if (pad) {
6861 		u64 zero = 0;
6862 		perf_output_copy(handle, &zero, pad);
6863 	}
6864 
6865 out_put:
6866 	ring_buffer_put(rb);
6867 }
6868 
6869 static void __perf_event_header__init_id(struct perf_event_header *header,
6870 					 struct perf_sample_data *data,
6871 					 struct perf_event *event,
6872 					 u64 sample_type)
6873 {
6874 	data->type = event->attr.sample_type;
6875 	header->size += event->id_header_size;
6876 
6877 	if (sample_type & PERF_SAMPLE_TID) {
6878 		/* namespace issues */
6879 		data->tid_entry.pid = perf_event_pid(event, current);
6880 		data->tid_entry.tid = perf_event_tid(event, current);
6881 	}
6882 
6883 	if (sample_type & PERF_SAMPLE_TIME)
6884 		data->time = perf_event_clock(event);
6885 
6886 	if (sample_type & (PERF_SAMPLE_ID | PERF_SAMPLE_IDENTIFIER))
6887 		data->id = primary_event_id(event);
6888 
6889 	if (sample_type & PERF_SAMPLE_STREAM_ID)
6890 		data->stream_id = event->id;
6891 
6892 	if (sample_type & PERF_SAMPLE_CPU) {
6893 		data->cpu_entry.cpu	 = raw_smp_processor_id();
6894 		data->cpu_entry.reserved = 0;
6895 	}
6896 }
6897 
6898 void perf_event_header__init_id(struct perf_event_header *header,
6899 				struct perf_sample_data *data,
6900 				struct perf_event *event)
6901 {
6902 	if (event->attr.sample_id_all)
6903 		__perf_event_header__init_id(header, data, event, event->attr.sample_type);
6904 }
6905 
6906 static void __perf_event__output_id_sample(struct perf_output_handle *handle,
6907 					   struct perf_sample_data *data)
6908 {
6909 	u64 sample_type = data->type;
6910 
6911 	if (sample_type & PERF_SAMPLE_TID)
6912 		perf_output_put(handle, data->tid_entry);
6913 
6914 	if (sample_type & PERF_SAMPLE_TIME)
6915 		perf_output_put(handle, data->time);
6916 
6917 	if (sample_type & PERF_SAMPLE_ID)
6918 		perf_output_put(handle, data->id);
6919 
6920 	if (sample_type & PERF_SAMPLE_STREAM_ID)
6921 		perf_output_put(handle, data->stream_id);
6922 
6923 	if (sample_type & PERF_SAMPLE_CPU)
6924 		perf_output_put(handle, data->cpu_entry);
6925 
6926 	if (sample_type & PERF_SAMPLE_IDENTIFIER)
6927 		perf_output_put(handle, data->id);
6928 }
6929 
6930 void perf_event__output_id_sample(struct perf_event *event,
6931 				  struct perf_output_handle *handle,
6932 				  struct perf_sample_data *sample)
6933 {
6934 	if (event->attr.sample_id_all)
6935 		__perf_event__output_id_sample(handle, sample);
6936 }
6937 
6938 static void perf_output_read_one(struct perf_output_handle *handle,
6939 				 struct perf_event *event,
6940 				 u64 enabled, u64 running)
6941 {
6942 	u64 read_format = event->attr.read_format;
6943 	u64 values[5];
6944 	int n = 0;
6945 
6946 	values[n++] = perf_event_count(event);
6947 	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
6948 		values[n++] = enabled +
6949 			atomic64_read(&event->child_total_time_enabled);
6950 	}
6951 	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
6952 		values[n++] = running +
6953 			atomic64_read(&event->child_total_time_running);
6954 	}
6955 	if (read_format & PERF_FORMAT_ID)
6956 		values[n++] = primary_event_id(event);
6957 	if (read_format & PERF_FORMAT_LOST)
6958 		values[n++] = atomic64_read(&event->lost_samples);
6959 
6960 	__output_copy(handle, values, n * sizeof(u64));
6961 }
6962 
6963 static void perf_output_read_group(struct perf_output_handle *handle,
6964 			    struct perf_event *event,
6965 			    u64 enabled, u64 running)
6966 {
6967 	struct perf_event *leader = event->group_leader, *sub;
6968 	u64 read_format = event->attr.read_format;
6969 	unsigned long flags;
6970 	u64 values[6];
6971 	int n = 0;
6972 
6973 	/*
6974 	 * Disabling interrupts avoids all counter scheduling
6975 	 * (context switches, timer based rotation and IPIs).
6976 	 */
6977 	local_irq_save(flags);
6978 
6979 	values[n++] = 1 + leader->nr_siblings;
6980 
6981 	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
6982 		values[n++] = enabled;
6983 
6984 	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
6985 		values[n++] = running;
6986 
6987 	if ((leader != event) &&
6988 	    (leader->state == PERF_EVENT_STATE_ACTIVE))
6989 		leader->pmu->read(leader);
6990 
6991 	values[n++] = perf_event_count(leader);
6992 	if (read_format & PERF_FORMAT_ID)
6993 		values[n++] = primary_event_id(leader);
6994 	if (read_format & PERF_FORMAT_LOST)
6995 		values[n++] = atomic64_read(&leader->lost_samples);
6996 
6997 	__output_copy(handle, values, n * sizeof(u64));
6998 
6999 	for_each_sibling_event(sub, leader) {
7000 		n = 0;
7001 
7002 		if ((sub != event) &&
7003 		    (sub->state == PERF_EVENT_STATE_ACTIVE))
7004 			sub->pmu->read(sub);
7005 
7006 		values[n++] = perf_event_count(sub);
7007 		if (read_format & PERF_FORMAT_ID)
7008 			values[n++] = primary_event_id(sub);
7009 		if (read_format & PERF_FORMAT_LOST)
7010 			values[n++] = atomic64_read(&sub->lost_samples);
7011 
7012 		__output_copy(handle, values, n * sizeof(u64));
7013 	}
7014 
7015 	local_irq_restore(flags);
7016 }
7017 
7018 #define PERF_FORMAT_TOTAL_TIMES (PERF_FORMAT_TOTAL_TIME_ENABLED|\
7019 				 PERF_FORMAT_TOTAL_TIME_RUNNING)
7020 
7021 /*
7022  * XXX PERF_SAMPLE_READ vs inherited events seems difficult.
7023  *
7024  * The problem is that its both hard and excessively expensive to iterate the
7025  * child list, not to mention that its impossible to IPI the children running
7026  * on another CPU, from interrupt/NMI context.
7027  */
7028 static void perf_output_read(struct perf_output_handle *handle,
7029 			     struct perf_event *event)
7030 {
7031 	u64 enabled = 0, running = 0, now;
7032 	u64 read_format = event->attr.read_format;
7033 
7034 	/*
7035 	 * compute total_time_enabled, total_time_running
7036 	 * based on snapshot values taken when the event
7037 	 * was last scheduled in.
7038 	 *
7039 	 * we cannot simply called update_context_time()
7040 	 * because of locking issue as we are called in
7041 	 * NMI context
7042 	 */
7043 	if (read_format & PERF_FORMAT_TOTAL_TIMES)
7044 		calc_timer_values(event, &now, &enabled, &running);
7045 
7046 	if (event->attr.read_format & PERF_FORMAT_GROUP)
7047 		perf_output_read_group(handle, event, enabled, running);
7048 	else
7049 		perf_output_read_one(handle, event, enabled, running);
7050 }
7051 
7052 void perf_output_sample(struct perf_output_handle *handle,
7053 			struct perf_event_header *header,
7054 			struct perf_sample_data *data,
7055 			struct perf_event *event)
7056 {
7057 	u64 sample_type = data->type;
7058 
7059 	perf_output_put(handle, *header);
7060 
7061 	if (sample_type & PERF_SAMPLE_IDENTIFIER)
7062 		perf_output_put(handle, data->id);
7063 
7064 	if (sample_type & PERF_SAMPLE_IP)
7065 		perf_output_put(handle, data->ip);
7066 
7067 	if (sample_type & PERF_SAMPLE_TID)
7068 		perf_output_put(handle, data->tid_entry);
7069 
7070 	if (sample_type & PERF_SAMPLE_TIME)
7071 		perf_output_put(handle, data->time);
7072 
7073 	if (sample_type & PERF_SAMPLE_ADDR)
7074 		perf_output_put(handle, data->addr);
7075 
7076 	if (sample_type & PERF_SAMPLE_ID)
7077 		perf_output_put(handle, data->id);
7078 
7079 	if (sample_type & PERF_SAMPLE_STREAM_ID)
7080 		perf_output_put(handle, data->stream_id);
7081 
7082 	if (sample_type & PERF_SAMPLE_CPU)
7083 		perf_output_put(handle, data->cpu_entry);
7084 
7085 	if (sample_type & PERF_SAMPLE_PERIOD)
7086 		perf_output_put(handle, data->period);
7087 
7088 	if (sample_type & PERF_SAMPLE_READ)
7089 		perf_output_read(handle, event);
7090 
7091 	if (sample_type & PERF_SAMPLE_CALLCHAIN) {
7092 		int size = 1;
7093 
7094 		size += data->callchain->nr;
7095 		size *= sizeof(u64);
7096 		__output_copy(handle, data->callchain, size);
7097 	}
7098 
7099 	if (sample_type & PERF_SAMPLE_RAW) {
7100 		struct perf_raw_record *raw = data->raw;
7101 
7102 		if (raw) {
7103 			struct perf_raw_frag *frag = &raw->frag;
7104 
7105 			perf_output_put(handle, raw->size);
7106 			do {
7107 				if (frag->copy) {
7108 					__output_custom(handle, frag->copy,
7109 							frag->data, frag->size);
7110 				} else {
7111 					__output_copy(handle, frag->data,
7112 						      frag->size);
7113 				}
7114 				if (perf_raw_frag_last(frag))
7115 					break;
7116 				frag = frag->next;
7117 			} while (1);
7118 			if (frag->pad)
7119 				__output_skip(handle, NULL, frag->pad);
7120 		} else {
7121 			struct {
7122 				u32	size;
7123 				u32	data;
7124 			} raw = {
7125 				.size = sizeof(u32),
7126 				.data = 0,
7127 			};
7128 			perf_output_put(handle, raw);
7129 		}
7130 	}
7131 
7132 	if (sample_type & PERF_SAMPLE_BRANCH_STACK) {
7133 		if (data->sample_flags & PERF_SAMPLE_BRANCH_STACK) {
7134 			size_t size;
7135 
7136 			size = data->br_stack->nr
7137 			     * sizeof(struct perf_branch_entry);
7138 
7139 			perf_output_put(handle, data->br_stack->nr);
7140 			if (branch_sample_hw_index(event))
7141 				perf_output_put(handle, data->br_stack->hw_idx);
7142 			perf_output_copy(handle, data->br_stack->entries, size);
7143 		} else {
7144 			/*
7145 			 * we always store at least the value of nr
7146 			 */
7147 			u64 nr = 0;
7148 			perf_output_put(handle, nr);
7149 		}
7150 	}
7151 
7152 	if (sample_type & PERF_SAMPLE_REGS_USER) {
7153 		u64 abi = data->regs_user.abi;
7154 
7155 		/*
7156 		 * If there are no regs to dump, notice it through
7157 		 * first u64 being zero (PERF_SAMPLE_REGS_ABI_NONE).
7158 		 */
7159 		perf_output_put(handle, abi);
7160 
7161 		if (abi) {
7162 			u64 mask = event->attr.sample_regs_user;
7163 			perf_output_sample_regs(handle,
7164 						data->regs_user.regs,
7165 						mask);
7166 		}
7167 	}
7168 
7169 	if (sample_type & PERF_SAMPLE_STACK_USER) {
7170 		perf_output_sample_ustack(handle,
7171 					  data->stack_user_size,
7172 					  data->regs_user.regs);
7173 	}
7174 
7175 	if (sample_type & PERF_SAMPLE_WEIGHT_TYPE)
7176 		perf_output_put(handle, data->weight.full);
7177 
7178 	if (sample_type & PERF_SAMPLE_DATA_SRC)
7179 		perf_output_put(handle, data->data_src.val);
7180 
7181 	if (sample_type & PERF_SAMPLE_TRANSACTION)
7182 		perf_output_put(handle, data->txn);
7183 
7184 	if (sample_type & PERF_SAMPLE_REGS_INTR) {
7185 		u64 abi = data->regs_intr.abi;
7186 		/*
7187 		 * If there are no regs to dump, notice it through
7188 		 * first u64 being zero (PERF_SAMPLE_REGS_ABI_NONE).
7189 		 */
7190 		perf_output_put(handle, abi);
7191 
7192 		if (abi) {
7193 			u64 mask = event->attr.sample_regs_intr;
7194 
7195 			perf_output_sample_regs(handle,
7196 						data->regs_intr.regs,
7197 						mask);
7198 		}
7199 	}
7200 
7201 	if (sample_type & PERF_SAMPLE_PHYS_ADDR)
7202 		perf_output_put(handle, data->phys_addr);
7203 
7204 	if (sample_type & PERF_SAMPLE_CGROUP)
7205 		perf_output_put(handle, data->cgroup);
7206 
7207 	if (sample_type & PERF_SAMPLE_DATA_PAGE_SIZE)
7208 		perf_output_put(handle, data->data_page_size);
7209 
7210 	if (sample_type & PERF_SAMPLE_CODE_PAGE_SIZE)
7211 		perf_output_put(handle, data->code_page_size);
7212 
7213 	if (sample_type & PERF_SAMPLE_AUX) {
7214 		perf_output_put(handle, data->aux_size);
7215 
7216 		if (data->aux_size)
7217 			perf_aux_sample_output(event, handle, data);
7218 	}
7219 
7220 	if (!event->attr.watermark) {
7221 		int wakeup_events = event->attr.wakeup_events;
7222 
7223 		if (wakeup_events) {
7224 			struct perf_buffer *rb = handle->rb;
7225 			int events = local_inc_return(&rb->events);
7226 
7227 			if (events >= wakeup_events) {
7228 				local_sub(wakeup_events, &rb->events);
7229 				local_inc(&rb->wakeup);
7230 			}
7231 		}
7232 	}
7233 }
7234 
7235 static u64 perf_virt_to_phys(u64 virt)
7236 {
7237 	u64 phys_addr = 0;
7238 
7239 	if (!virt)
7240 		return 0;
7241 
7242 	if (virt >= TASK_SIZE) {
7243 		/* If it's vmalloc()d memory, leave phys_addr as 0 */
7244 		if (virt_addr_valid((void *)(uintptr_t)virt) &&
7245 		    !(virt >= VMALLOC_START && virt < VMALLOC_END))
7246 			phys_addr = (u64)virt_to_phys((void *)(uintptr_t)virt);
7247 	} else {
7248 		/*
7249 		 * Walking the pages tables for user address.
7250 		 * Interrupts are disabled, so it prevents any tear down
7251 		 * of the page tables.
7252 		 * Try IRQ-safe get_user_page_fast_only first.
7253 		 * If failed, leave phys_addr as 0.
7254 		 */
7255 		if (current->mm != NULL) {
7256 			struct page *p;
7257 
7258 			pagefault_disable();
7259 			if (get_user_page_fast_only(virt, 0, &p)) {
7260 				phys_addr = page_to_phys(p) + virt % PAGE_SIZE;
7261 				put_page(p);
7262 			}
7263 			pagefault_enable();
7264 		}
7265 	}
7266 
7267 	return phys_addr;
7268 }
7269 
7270 /*
7271  * Return the pagetable size of a given virtual address.
7272  */
7273 static u64 perf_get_pgtable_size(struct mm_struct *mm, unsigned long addr)
7274 {
7275 	u64 size = 0;
7276 
7277 #ifdef CONFIG_HAVE_FAST_GUP
7278 	pgd_t *pgdp, pgd;
7279 	p4d_t *p4dp, p4d;
7280 	pud_t *pudp, pud;
7281 	pmd_t *pmdp, pmd;
7282 	pte_t *ptep, pte;
7283 
7284 	pgdp = pgd_offset(mm, addr);
7285 	pgd = READ_ONCE(*pgdp);
7286 	if (pgd_none(pgd))
7287 		return 0;
7288 
7289 	if (pgd_leaf(pgd))
7290 		return pgd_leaf_size(pgd);
7291 
7292 	p4dp = p4d_offset_lockless(pgdp, pgd, addr);
7293 	p4d = READ_ONCE(*p4dp);
7294 	if (!p4d_present(p4d))
7295 		return 0;
7296 
7297 	if (p4d_leaf(p4d))
7298 		return p4d_leaf_size(p4d);
7299 
7300 	pudp = pud_offset_lockless(p4dp, p4d, addr);
7301 	pud = READ_ONCE(*pudp);
7302 	if (!pud_present(pud))
7303 		return 0;
7304 
7305 	if (pud_leaf(pud))
7306 		return pud_leaf_size(pud);
7307 
7308 	pmdp = pmd_offset_lockless(pudp, pud, addr);
7309 	pmd = READ_ONCE(*pmdp);
7310 	if (!pmd_present(pmd))
7311 		return 0;
7312 
7313 	if (pmd_leaf(pmd))
7314 		return pmd_leaf_size(pmd);
7315 
7316 	ptep = pte_offset_map(&pmd, addr);
7317 	pte = ptep_get_lockless(ptep);
7318 	if (pte_present(pte))
7319 		size = pte_leaf_size(pte);
7320 	pte_unmap(ptep);
7321 #endif /* CONFIG_HAVE_FAST_GUP */
7322 
7323 	return size;
7324 }
7325 
7326 static u64 perf_get_page_size(unsigned long addr)
7327 {
7328 	struct mm_struct *mm;
7329 	unsigned long flags;
7330 	u64 size;
7331 
7332 	if (!addr)
7333 		return 0;
7334 
7335 	/*
7336 	 * Software page-table walkers must disable IRQs,
7337 	 * which prevents any tear down of the page tables.
7338 	 */
7339 	local_irq_save(flags);
7340 
7341 	mm = current->mm;
7342 	if (!mm) {
7343 		/*
7344 		 * For kernel threads and the like, use init_mm so that
7345 		 * we can find kernel memory.
7346 		 */
7347 		mm = &init_mm;
7348 	}
7349 
7350 	size = perf_get_pgtable_size(mm, addr);
7351 
7352 	local_irq_restore(flags);
7353 
7354 	return size;
7355 }
7356 
7357 static struct perf_callchain_entry __empty_callchain = { .nr = 0, };
7358 
7359 struct perf_callchain_entry *
7360 perf_callchain(struct perf_event *event, struct pt_regs *regs)
7361 {
7362 	bool kernel = !event->attr.exclude_callchain_kernel;
7363 	bool user   = !event->attr.exclude_callchain_user;
7364 	/* Disallow cross-task user callchains. */
7365 	bool crosstask = event->ctx->task && event->ctx->task != current;
7366 	const u32 max_stack = event->attr.sample_max_stack;
7367 	struct perf_callchain_entry *callchain;
7368 
7369 	if (!kernel && !user)
7370 		return &__empty_callchain;
7371 
7372 	callchain = get_perf_callchain(regs, 0, kernel, user,
7373 				       max_stack, crosstask, true);
7374 	return callchain ?: &__empty_callchain;
7375 }
7376 
7377 void perf_prepare_sample(struct perf_event_header *header,
7378 			 struct perf_sample_data *data,
7379 			 struct perf_event *event,
7380 			 struct pt_regs *regs)
7381 {
7382 	u64 sample_type = event->attr.sample_type;
7383 	u64 filtered_sample_type;
7384 
7385 	header->type = PERF_RECORD_SAMPLE;
7386 	header->size = sizeof(*header) + event->header_size;
7387 
7388 	header->misc = 0;
7389 	header->misc |= perf_misc_flags(regs);
7390 
7391 	/*
7392 	 * Clear the sample flags that have already been done by the
7393 	 * PMU driver.
7394 	 */
7395 	filtered_sample_type = sample_type & ~data->sample_flags;
7396 	__perf_event_header__init_id(header, data, event, filtered_sample_type);
7397 
7398 	if (sample_type & (PERF_SAMPLE_IP | PERF_SAMPLE_CODE_PAGE_SIZE))
7399 		data->ip = perf_instruction_pointer(regs);
7400 
7401 	if (sample_type & PERF_SAMPLE_CALLCHAIN) {
7402 		int size = 1;
7403 
7404 		if (filtered_sample_type & PERF_SAMPLE_CALLCHAIN)
7405 			data->callchain = perf_callchain(event, regs);
7406 
7407 		size += data->callchain->nr;
7408 
7409 		header->size += size * sizeof(u64);
7410 	}
7411 
7412 	if (sample_type & PERF_SAMPLE_RAW) {
7413 		struct perf_raw_record *raw = data->raw;
7414 		int size;
7415 
7416 		if (raw && (data->sample_flags & PERF_SAMPLE_RAW)) {
7417 			struct perf_raw_frag *frag = &raw->frag;
7418 			u32 sum = 0;
7419 
7420 			do {
7421 				sum += frag->size;
7422 				if (perf_raw_frag_last(frag))
7423 					break;
7424 				frag = frag->next;
7425 			} while (1);
7426 
7427 			size = round_up(sum + sizeof(u32), sizeof(u64));
7428 			raw->size = size - sizeof(u32);
7429 			frag->pad = raw->size - sum;
7430 		} else {
7431 			size = sizeof(u64);
7432 			data->raw = NULL;
7433 		}
7434 
7435 		header->size += size;
7436 	}
7437 
7438 	if (sample_type & PERF_SAMPLE_BRANCH_STACK) {
7439 		int size = sizeof(u64); /* nr */
7440 		if (data->sample_flags & PERF_SAMPLE_BRANCH_STACK) {
7441 			if (branch_sample_hw_index(event))
7442 				size += sizeof(u64);
7443 
7444 			size += data->br_stack->nr
7445 			      * sizeof(struct perf_branch_entry);
7446 		}
7447 		header->size += size;
7448 	}
7449 
7450 	if (sample_type & (PERF_SAMPLE_REGS_USER | PERF_SAMPLE_STACK_USER))
7451 		perf_sample_regs_user(&data->regs_user, regs);
7452 
7453 	if (sample_type & PERF_SAMPLE_REGS_USER) {
7454 		/* regs dump ABI info */
7455 		int size = sizeof(u64);
7456 
7457 		if (data->regs_user.regs) {
7458 			u64 mask = event->attr.sample_regs_user;
7459 			size += hweight64(mask) * sizeof(u64);
7460 		}
7461 
7462 		header->size += size;
7463 	}
7464 
7465 	if (sample_type & PERF_SAMPLE_STACK_USER) {
7466 		/*
7467 		 * Either we need PERF_SAMPLE_STACK_USER bit to be always
7468 		 * processed as the last one or have additional check added
7469 		 * in case new sample type is added, because we could eat
7470 		 * up the rest of the sample size.
7471 		 */
7472 		u16 stack_size = event->attr.sample_stack_user;
7473 		u16 size = sizeof(u64);
7474 
7475 		stack_size = perf_sample_ustack_size(stack_size, header->size,
7476 						     data->regs_user.regs);
7477 
7478 		/*
7479 		 * If there is something to dump, add space for the dump
7480 		 * itself and for the field that tells the dynamic size,
7481 		 * which is how many have been actually dumped.
7482 		 */
7483 		if (stack_size)
7484 			size += sizeof(u64) + stack_size;
7485 
7486 		data->stack_user_size = stack_size;
7487 		header->size += size;
7488 	}
7489 
7490 	if (filtered_sample_type & PERF_SAMPLE_WEIGHT_TYPE)
7491 		data->weight.full = 0;
7492 
7493 	if (filtered_sample_type & PERF_SAMPLE_DATA_SRC)
7494 		data->data_src.val = PERF_MEM_NA;
7495 
7496 	if (filtered_sample_type & PERF_SAMPLE_TRANSACTION)
7497 		data->txn = 0;
7498 
7499 	if (sample_type & (PERF_SAMPLE_ADDR | PERF_SAMPLE_PHYS_ADDR | PERF_SAMPLE_DATA_PAGE_SIZE)) {
7500 		if (filtered_sample_type & PERF_SAMPLE_ADDR)
7501 			data->addr = 0;
7502 	}
7503 
7504 	if (sample_type & PERF_SAMPLE_REGS_INTR) {
7505 		/* regs dump ABI info */
7506 		int size = sizeof(u64);
7507 
7508 		perf_sample_regs_intr(&data->regs_intr, regs);
7509 
7510 		if (data->regs_intr.regs) {
7511 			u64 mask = event->attr.sample_regs_intr;
7512 
7513 			size += hweight64(mask) * sizeof(u64);
7514 		}
7515 
7516 		header->size += size;
7517 	}
7518 
7519 	if (sample_type & PERF_SAMPLE_PHYS_ADDR &&
7520 	    filtered_sample_type & PERF_SAMPLE_PHYS_ADDR)
7521 		data->phys_addr = perf_virt_to_phys(data->addr);
7522 
7523 #ifdef CONFIG_CGROUP_PERF
7524 	if (sample_type & PERF_SAMPLE_CGROUP) {
7525 		struct cgroup *cgrp;
7526 
7527 		/* protected by RCU */
7528 		cgrp = task_css_check(current, perf_event_cgrp_id, 1)->cgroup;
7529 		data->cgroup = cgroup_id(cgrp);
7530 	}
7531 #endif
7532 
7533 	/*
7534 	 * PERF_DATA_PAGE_SIZE requires PERF_SAMPLE_ADDR. If the user doesn't
7535 	 * require PERF_SAMPLE_ADDR, kernel implicitly retrieve the data->addr,
7536 	 * but the value will not dump to the userspace.
7537 	 */
7538 	if (sample_type & PERF_SAMPLE_DATA_PAGE_SIZE)
7539 		data->data_page_size = perf_get_page_size(data->addr);
7540 
7541 	if (sample_type & PERF_SAMPLE_CODE_PAGE_SIZE)
7542 		data->code_page_size = perf_get_page_size(data->ip);
7543 
7544 	if (sample_type & PERF_SAMPLE_AUX) {
7545 		u64 size;
7546 
7547 		header->size += sizeof(u64); /* size */
7548 
7549 		/*
7550 		 * Given the 16bit nature of header::size, an AUX sample can
7551 		 * easily overflow it, what with all the preceding sample bits.
7552 		 * Make sure this doesn't happen by using up to U16_MAX bytes
7553 		 * per sample in total (rounded down to 8 byte boundary).
7554 		 */
7555 		size = min_t(size_t, U16_MAX - header->size,
7556 			     event->attr.aux_sample_size);
7557 		size = rounddown(size, 8);
7558 		size = perf_prepare_sample_aux(event, data, size);
7559 
7560 		WARN_ON_ONCE(size + header->size > U16_MAX);
7561 		header->size += size;
7562 	}
7563 	/*
7564 	 * If you're adding more sample types here, you likely need to do
7565 	 * something about the overflowing header::size, like repurpose the
7566 	 * lowest 3 bits of size, which should be always zero at the moment.
7567 	 * This raises a more important question, do we really need 512k sized
7568 	 * samples and why, so good argumentation is in order for whatever you
7569 	 * do here next.
7570 	 */
7571 	WARN_ON_ONCE(header->size & 7);
7572 }
7573 
7574 static __always_inline int
7575 __perf_event_output(struct perf_event *event,
7576 		    struct perf_sample_data *data,
7577 		    struct pt_regs *regs,
7578 		    int (*output_begin)(struct perf_output_handle *,
7579 					struct perf_sample_data *,
7580 					struct perf_event *,
7581 					unsigned int))
7582 {
7583 	struct perf_output_handle handle;
7584 	struct perf_event_header header;
7585 	int err;
7586 
7587 	/* protect the callchain buffers */
7588 	rcu_read_lock();
7589 
7590 	perf_prepare_sample(&header, data, event, regs);
7591 
7592 	err = output_begin(&handle, data, event, header.size);
7593 	if (err)
7594 		goto exit;
7595 
7596 	perf_output_sample(&handle, &header, data, event);
7597 
7598 	perf_output_end(&handle);
7599 
7600 exit:
7601 	rcu_read_unlock();
7602 	return err;
7603 }
7604 
7605 void
7606 perf_event_output_forward(struct perf_event *event,
7607 			 struct perf_sample_data *data,
7608 			 struct pt_regs *regs)
7609 {
7610 	__perf_event_output(event, data, regs, perf_output_begin_forward);
7611 }
7612 
7613 void
7614 perf_event_output_backward(struct perf_event *event,
7615 			   struct perf_sample_data *data,
7616 			   struct pt_regs *regs)
7617 {
7618 	__perf_event_output(event, data, regs, perf_output_begin_backward);
7619 }
7620 
7621 int
7622 perf_event_output(struct perf_event *event,
7623 		  struct perf_sample_data *data,
7624 		  struct pt_regs *regs)
7625 {
7626 	return __perf_event_output(event, data, regs, perf_output_begin);
7627 }
7628 
7629 /*
7630  * read event_id
7631  */
7632 
7633 struct perf_read_event {
7634 	struct perf_event_header	header;
7635 
7636 	u32				pid;
7637 	u32				tid;
7638 };
7639 
7640 static void
7641 perf_event_read_event(struct perf_event *event,
7642 			struct task_struct *task)
7643 {
7644 	struct perf_output_handle handle;
7645 	struct perf_sample_data sample;
7646 	struct perf_read_event read_event = {
7647 		.header = {
7648 			.type = PERF_RECORD_READ,
7649 			.misc = 0,
7650 			.size = sizeof(read_event) + event->read_size,
7651 		},
7652 		.pid = perf_event_pid(event, task),
7653 		.tid = perf_event_tid(event, task),
7654 	};
7655 	int ret;
7656 
7657 	perf_event_header__init_id(&read_event.header, &sample, event);
7658 	ret = perf_output_begin(&handle, &sample, event, read_event.header.size);
7659 	if (ret)
7660 		return;
7661 
7662 	perf_output_put(&handle, read_event);
7663 	perf_output_read(&handle, event);
7664 	perf_event__output_id_sample(event, &handle, &sample);
7665 
7666 	perf_output_end(&handle);
7667 }
7668 
7669 typedef void (perf_iterate_f)(struct perf_event *event, void *data);
7670 
7671 static void
7672 perf_iterate_ctx(struct perf_event_context *ctx,
7673 		   perf_iterate_f output,
7674 		   void *data, bool all)
7675 {
7676 	struct perf_event *event;
7677 
7678 	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
7679 		if (!all) {
7680 			if (event->state < PERF_EVENT_STATE_INACTIVE)
7681 				continue;
7682 			if (!event_filter_match(event))
7683 				continue;
7684 		}
7685 
7686 		output(event, data);
7687 	}
7688 }
7689 
7690 static void perf_iterate_sb_cpu(perf_iterate_f output, void *data)
7691 {
7692 	struct pmu_event_list *pel = this_cpu_ptr(&pmu_sb_events);
7693 	struct perf_event *event;
7694 
7695 	list_for_each_entry_rcu(event, &pel->list, sb_list) {
7696 		/*
7697 		 * Skip events that are not fully formed yet; ensure that
7698 		 * if we observe event->ctx, both event and ctx will be
7699 		 * complete enough. See perf_install_in_context().
7700 		 */
7701 		if (!smp_load_acquire(&event->ctx))
7702 			continue;
7703 
7704 		if (event->state < PERF_EVENT_STATE_INACTIVE)
7705 			continue;
7706 		if (!event_filter_match(event))
7707 			continue;
7708 		output(event, data);
7709 	}
7710 }
7711 
7712 /*
7713  * Iterate all events that need to receive side-band events.
7714  *
7715  * For new callers; ensure that account_pmu_sb_event() includes
7716  * your event, otherwise it might not get delivered.
7717  */
7718 static void
7719 perf_iterate_sb(perf_iterate_f output, void *data,
7720 	       struct perf_event_context *task_ctx)
7721 {
7722 	struct perf_event_context *ctx;
7723 	int ctxn;
7724 
7725 	rcu_read_lock();
7726 	preempt_disable();
7727 
7728 	/*
7729 	 * If we have task_ctx != NULL we only notify the task context itself.
7730 	 * The task_ctx is set only for EXIT events before releasing task
7731 	 * context.
7732 	 */
7733 	if (task_ctx) {
7734 		perf_iterate_ctx(task_ctx, output, data, false);
7735 		goto done;
7736 	}
7737 
7738 	perf_iterate_sb_cpu(output, data);
7739 
7740 	for_each_task_context_nr(ctxn) {
7741 		ctx = rcu_dereference(current->perf_event_ctxp[ctxn]);
7742 		if (ctx)
7743 			perf_iterate_ctx(ctx, output, data, false);
7744 	}
7745 done:
7746 	preempt_enable();
7747 	rcu_read_unlock();
7748 }
7749 
7750 /*
7751  * Clear all file-based filters at exec, they'll have to be
7752  * re-instated when/if these objects are mmapped again.
7753  */
7754 static void perf_event_addr_filters_exec(struct perf_event *event, void *data)
7755 {
7756 	struct perf_addr_filters_head *ifh = perf_event_addr_filters(event);
7757 	struct perf_addr_filter *filter;
7758 	unsigned int restart = 0, count = 0;
7759 	unsigned long flags;
7760 
7761 	if (!has_addr_filter(event))
7762 		return;
7763 
7764 	raw_spin_lock_irqsave(&ifh->lock, flags);
7765 	list_for_each_entry(filter, &ifh->list, entry) {
7766 		if (filter->path.dentry) {
7767 			event->addr_filter_ranges[count].start = 0;
7768 			event->addr_filter_ranges[count].size = 0;
7769 			restart++;
7770 		}
7771 
7772 		count++;
7773 	}
7774 
7775 	if (restart)
7776 		event->addr_filters_gen++;
7777 	raw_spin_unlock_irqrestore(&ifh->lock, flags);
7778 
7779 	if (restart)
7780 		perf_event_stop(event, 1);
7781 }
7782 
7783 void perf_event_exec(void)
7784 {
7785 	struct perf_event_context *ctx;
7786 	int ctxn;
7787 
7788 	for_each_task_context_nr(ctxn) {
7789 		perf_event_enable_on_exec(ctxn);
7790 		perf_event_remove_on_exec(ctxn);
7791 
7792 		rcu_read_lock();
7793 		ctx = rcu_dereference(current->perf_event_ctxp[ctxn]);
7794 		if (ctx) {
7795 			perf_iterate_ctx(ctx, perf_event_addr_filters_exec,
7796 					 NULL, true);
7797 		}
7798 		rcu_read_unlock();
7799 	}
7800 }
7801 
7802 struct remote_output {
7803 	struct perf_buffer	*rb;
7804 	int			err;
7805 };
7806 
7807 static void __perf_event_output_stop(struct perf_event *event, void *data)
7808 {
7809 	struct perf_event *parent = event->parent;
7810 	struct remote_output *ro = data;
7811 	struct perf_buffer *rb = ro->rb;
7812 	struct stop_event_data sd = {
7813 		.event	= event,
7814 	};
7815 
7816 	if (!has_aux(event))
7817 		return;
7818 
7819 	if (!parent)
7820 		parent = event;
7821 
7822 	/*
7823 	 * In case of inheritance, it will be the parent that links to the
7824 	 * ring-buffer, but it will be the child that's actually using it.
7825 	 *
7826 	 * We are using event::rb to determine if the event should be stopped,
7827 	 * however this may race with ring_buffer_attach() (through set_output),
7828 	 * which will make us skip the event that actually needs to be stopped.
7829 	 * So ring_buffer_attach() has to stop an aux event before re-assigning
7830 	 * its rb pointer.
7831 	 */
7832 	if (rcu_dereference(parent->rb) == rb)
7833 		ro->err = __perf_event_stop(&sd);
7834 }
7835 
7836 static int __perf_pmu_output_stop(void *info)
7837 {
7838 	struct perf_event *event = info;
7839 	struct pmu *pmu = event->ctx->pmu;
7840 	struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
7841 	struct remote_output ro = {
7842 		.rb	= event->rb,
7843 	};
7844 
7845 	rcu_read_lock();
7846 	perf_iterate_ctx(&cpuctx->ctx, __perf_event_output_stop, &ro, false);
7847 	if (cpuctx->task_ctx)
7848 		perf_iterate_ctx(cpuctx->task_ctx, __perf_event_output_stop,
7849 				   &ro, false);
7850 	rcu_read_unlock();
7851 
7852 	return ro.err;
7853 }
7854 
7855 static void perf_pmu_output_stop(struct perf_event *event)
7856 {
7857 	struct perf_event *iter;
7858 	int err, cpu;
7859 
7860 restart:
7861 	rcu_read_lock();
7862 	list_for_each_entry_rcu(iter, &event->rb->event_list, rb_entry) {
7863 		/*
7864 		 * For per-CPU events, we need to make sure that neither they
7865 		 * nor their children are running; for cpu==-1 events it's
7866 		 * sufficient to stop the event itself if it's active, since
7867 		 * it can't have children.
7868 		 */
7869 		cpu = iter->cpu;
7870 		if (cpu == -1)
7871 			cpu = READ_ONCE(iter->oncpu);
7872 
7873 		if (cpu == -1)
7874 			continue;
7875 
7876 		err = cpu_function_call(cpu, __perf_pmu_output_stop, event);
7877 		if (err == -EAGAIN) {
7878 			rcu_read_unlock();
7879 			goto restart;
7880 		}
7881 	}
7882 	rcu_read_unlock();
7883 }
7884 
7885 /*
7886  * task tracking -- fork/exit
7887  *
7888  * enabled by: attr.comm | attr.mmap | attr.mmap2 | attr.mmap_data | attr.task
7889  */
7890 
7891 struct perf_task_event {
7892 	struct task_struct		*task;
7893 	struct perf_event_context	*task_ctx;
7894 
7895 	struct {
7896 		struct perf_event_header	header;
7897 
7898 		u32				pid;
7899 		u32				ppid;
7900 		u32				tid;
7901 		u32				ptid;
7902 		u64				time;
7903 	} event_id;
7904 };
7905 
7906 static int perf_event_task_match(struct perf_event *event)
7907 {
7908 	return event->attr.comm  || event->attr.mmap ||
7909 	       event->attr.mmap2 || event->attr.mmap_data ||
7910 	       event->attr.task;
7911 }
7912 
7913 static void perf_event_task_output(struct perf_event *event,
7914 				   void *data)
7915 {
7916 	struct perf_task_event *task_event = data;
7917 	struct perf_output_handle handle;
7918 	struct perf_sample_data	sample;
7919 	struct task_struct *task = task_event->task;
7920 	int ret, size = task_event->event_id.header.size;
7921 
7922 	if (!perf_event_task_match(event))
7923 		return;
7924 
7925 	perf_event_header__init_id(&task_event->event_id.header, &sample, event);
7926 
7927 	ret = perf_output_begin(&handle, &sample, event,
7928 				task_event->event_id.header.size);
7929 	if (ret)
7930 		goto out;
7931 
7932 	task_event->event_id.pid = perf_event_pid(event, task);
7933 	task_event->event_id.tid = perf_event_tid(event, task);
7934 
7935 	if (task_event->event_id.header.type == PERF_RECORD_EXIT) {
7936 		task_event->event_id.ppid = perf_event_pid(event,
7937 							task->real_parent);
7938 		task_event->event_id.ptid = perf_event_pid(event,
7939 							task->real_parent);
7940 	} else {  /* PERF_RECORD_FORK */
7941 		task_event->event_id.ppid = perf_event_pid(event, current);
7942 		task_event->event_id.ptid = perf_event_tid(event, current);
7943 	}
7944 
7945 	task_event->event_id.time = perf_event_clock(event);
7946 
7947 	perf_output_put(&handle, task_event->event_id);
7948 
7949 	perf_event__output_id_sample(event, &handle, &sample);
7950 
7951 	perf_output_end(&handle);
7952 out:
7953 	task_event->event_id.header.size = size;
7954 }
7955 
7956 static void perf_event_task(struct task_struct *task,
7957 			      struct perf_event_context *task_ctx,
7958 			      int new)
7959 {
7960 	struct perf_task_event task_event;
7961 
7962 	if (!atomic_read(&nr_comm_events) &&
7963 	    !atomic_read(&nr_mmap_events) &&
7964 	    !atomic_read(&nr_task_events))
7965 		return;
7966 
7967 	task_event = (struct perf_task_event){
7968 		.task	  = task,
7969 		.task_ctx = task_ctx,
7970 		.event_id    = {
7971 			.header = {
7972 				.type = new ? PERF_RECORD_FORK : PERF_RECORD_EXIT,
7973 				.misc = 0,
7974 				.size = sizeof(task_event.event_id),
7975 			},
7976 			/* .pid  */
7977 			/* .ppid */
7978 			/* .tid  */
7979 			/* .ptid */
7980 			/* .time */
7981 		},
7982 	};
7983 
7984 	perf_iterate_sb(perf_event_task_output,
7985 		       &task_event,
7986 		       task_ctx);
7987 }
7988 
7989 void perf_event_fork(struct task_struct *task)
7990 {
7991 	perf_event_task(task, NULL, 1);
7992 	perf_event_namespaces(task);
7993 }
7994 
7995 /*
7996  * comm tracking
7997  */
7998 
7999 struct perf_comm_event {
8000 	struct task_struct	*task;
8001 	char			*comm;
8002 	int			comm_size;
8003 
8004 	struct {
8005 		struct perf_event_header	header;
8006 
8007 		u32				pid;
8008 		u32				tid;
8009 	} event_id;
8010 };
8011 
8012 static int perf_event_comm_match(struct perf_event *event)
8013 {
8014 	return event->attr.comm;
8015 }
8016 
8017 static void perf_event_comm_output(struct perf_event *event,
8018 				   void *data)
8019 {
8020 	struct perf_comm_event *comm_event = data;
8021 	struct perf_output_handle handle;
8022 	struct perf_sample_data sample;
8023 	int size = comm_event->event_id.header.size;
8024 	int ret;
8025 
8026 	if (!perf_event_comm_match(event))
8027 		return;
8028 
8029 	perf_event_header__init_id(&comm_event->event_id.header, &sample, event);
8030 	ret = perf_output_begin(&handle, &sample, event,
8031 				comm_event->event_id.header.size);
8032 
8033 	if (ret)
8034 		goto out;
8035 
8036 	comm_event->event_id.pid = perf_event_pid(event, comm_event->task);
8037 	comm_event->event_id.tid = perf_event_tid(event, comm_event->task);
8038 
8039 	perf_output_put(&handle, comm_event->event_id);
8040 	__output_copy(&handle, comm_event->comm,
8041 				   comm_event->comm_size);
8042 
8043 	perf_event__output_id_sample(event, &handle, &sample);
8044 
8045 	perf_output_end(&handle);
8046 out:
8047 	comm_event->event_id.header.size = size;
8048 }
8049 
8050 static void perf_event_comm_event(struct perf_comm_event *comm_event)
8051 {
8052 	char comm[TASK_COMM_LEN];
8053 	unsigned int size;
8054 
8055 	memset(comm, 0, sizeof(comm));
8056 	strlcpy(comm, comm_event->task->comm, sizeof(comm));
8057 	size = ALIGN(strlen(comm)+1, sizeof(u64));
8058 
8059 	comm_event->comm = comm;
8060 	comm_event->comm_size = size;
8061 
8062 	comm_event->event_id.header.size = sizeof(comm_event->event_id) + size;
8063 
8064 	perf_iterate_sb(perf_event_comm_output,
8065 		       comm_event,
8066 		       NULL);
8067 }
8068 
8069 void perf_event_comm(struct task_struct *task, bool exec)
8070 {
8071 	struct perf_comm_event comm_event;
8072 
8073 	if (!atomic_read(&nr_comm_events))
8074 		return;
8075 
8076 	comm_event = (struct perf_comm_event){
8077 		.task	= task,
8078 		/* .comm      */
8079 		/* .comm_size */
8080 		.event_id  = {
8081 			.header = {
8082 				.type = PERF_RECORD_COMM,
8083 				.misc = exec ? PERF_RECORD_MISC_COMM_EXEC : 0,
8084 				/* .size */
8085 			},
8086 			/* .pid */
8087 			/* .tid */
8088 		},
8089 	};
8090 
8091 	perf_event_comm_event(&comm_event);
8092 }
8093 
8094 /*
8095  * namespaces tracking
8096  */
8097 
8098 struct perf_namespaces_event {
8099 	struct task_struct		*task;
8100 
8101 	struct {
8102 		struct perf_event_header	header;
8103 
8104 		u32				pid;
8105 		u32				tid;
8106 		u64				nr_namespaces;
8107 		struct perf_ns_link_info	link_info[NR_NAMESPACES];
8108 	} event_id;
8109 };
8110 
8111 static int perf_event_namespaces_match(struct perf_event *event)
8112 {
8113 	return event->attr.namespaces;
8114 }
8115 
8116 static void perf_event_namespaces_output(struct perf_event *event,
8117 					 void *data)
8118 {
8119 	struct perf_namespaces_event *namespaces_event = data;
8120 	struct perf_output_handle handle;
8121 	struct perf_sample_data sample;
8122 	u16 header_size = namespaces_event->event_id.header.size;
8123 	int ret;
8124 
8125 	if (!perf_event_namespaces_match(event))
8126 		return;
8127 
8128 	perf_event_header__init_id(&namespaces_event->event_id.header,
8129 				   &sample, event);
8130 	ret = perf_output_begin(&handle, &sample, event,
8131 				namespaces_event->event_id.header.size);
8132 	if (ret)
8133 		goto out;
8134 
8135 	namespaces_event->event_id.pid = perf_event_pid(event,
8136 							namespaces_event->task);
8137 	namespaces_event->event_id.tid = perf_event_tid(event,
8138 							namespaces_event->task);
8139 
8140 	perf_output_put(&handle, namespaces_event->event_id);
8141 
8142 	perf_event__output_id_sample(event, &handle, &sample);
8143 
8144 	perf_output_end(&handle);
8145 out:
8146 	namespaces_event->event_id.header.size = header_size;
8147 }
8148 
8149 static void perf_fill_ns_link_info(struct perf_ns_link_info *ns_link_info,
8150 				   struct task_struct *task,
8151 				   const struct proc_ns_operations *ns_ops)
8152 {
8153 	struct path ns_path;
8154 	struct inode *ns_inode;
8155 	int error;
8156 
8157 	error = ns_get_path(&ns_path, task, ns_ops);
8158 	if (!error) {
8159 		ns_inode = ns_path.dentry->d_inode;
8160 		ns_link_info->dev = new_encode_dev(ns_inode->i_sb->s_dev);
8161 		ns_link_info->ino = ns_inode->i_ino;
8162 		path_put(&ns_path);
8163 	}
8164 }
8165 
8166 void perf_event_namespaces(struct task_struct *task)
8167 {
8168 	struct perf_namespaces_event namespaces_event;
8169 	struct perf_ns_link_info *ns_link_info;
8170 
8171 	if (!atomic_read(&nr_namespaces_events))
8172 		return;
8173 
8174 	namespaces_event = (struct perf_namespaces_event){
8175 		.task	= task,
8176 		.event_id  = {
8177 			.header = {
8178 				.type = PERF_RECORD_NAMESPACES,
8179 				.misc = 0,
8180 				.size = sizeof(namespaces_event.event_id),
8181 			},
8182 			/* .pid */
8183 			/* .tid */
8184 			.nr_namespaces = NR_NAMESPACES,
8185 			/* .link_info[NR_NAMESPACES] */
8186 		},
8187 	};
8188 
8189 	ns_link_info = namespaces_event.event_id.link_info;
8190 
8191 	perf_fill_ns_link_info(&ns_link_info[MNT_NS_INDEX],
8192 			       task, &mntns_operations);
8193 
8194 #ifdef CONFIG_USER_NS
8195 	perf_fill_ns_link_info(&ns_link_info[USER_NS_INDEX],
8196 			       task, &userns_operations);
8197 #endif
8198 #ifdef CONFIG_NET_NS
8199 	perf_fill_ns_link_info(&ns_link_info[NET_NS_INDEX],
8200 			       task, &netns_operations);
8201 #endif
8202 #ifdef CONFIG_UTS_NS
8203 	perf_fill_ns_link_info(&ns_link_info[UTS_NS_INDEX],
8204 			       task, &utsns_operations);
8205 #endif
8206 #ifdef CONFIG_IPC_NS
8207 	perf_fill_ns_link_info(&ns_link_info[IPC_NS_INDEX],
8208 			       task, &ipcns_operations);
8209 #endif
8210 #ifdef CONFIG_PID_NS
8211 	perf_fill_ns_link_info(&ns_link_info[PID_NS_INDEX],
8212 			       task, &pidns_operations);
8213 #endif
8214 #ifdef CONFIG_CGROUPS
8215 	perf_fill_ns_link_info(&ns_link_info[CGROUP_NS_INDEX],
8216 			       task, &cgroupns_operations);
8217 #endif
8218 
8219 	perf_iterate_sb(perf_event_namespaces_output,
8220 			&namespaces_event,
8221 			NULL);
8222 }
8223 
8224 /*
8225  * cgroup tracking
8226  */
8227 #ifdef CONFIG_CGROUP_PERF
8228 
8229 struct perf_cgroup_event {
8230 	char				*path;
8231 	int				path_size;
8232 	struct {
8233 		struct perf_event_header	header;
8234 		u64				id;
8235 		char				path[];
8236 	} event_id;
8237 };
8238 
8239 static int perf_event_cgroup_match(struct perf_event *event)
8240 {
8241 	return event->attr.cgroup;
8242 }
8243 
8244 static void perf_event_cgroup_output(struct perf_event *event, void *data)
8245 {
8246 	struct perf_cgroup_event *cgroup_event = data;
8247 	struct perf_output_handle handle;
8248 	struct perf_sample_data sample;
8249 	u16 header_size = cgroup_event->event_id.header.size;
8250 	int ret;
8251 
8252 	if (!perf_event_cgroup_match(event))
8253 		return;
8254 
8255 	perf_event_header__init_id(&cgroup_event->event_id.header,
8256 				   &sample, event);
8257 	ret = perf_output_begin(&handle, &sample, event,
8258 				cgroup_event->event_id.header.size);
8259 	if (ret)
8260 		goto out;
8261 
8262 	perf_output_put(&handle, cgroup_event->event_id);
8263 	__output_copy(&handle, cgroup_event->path, cgroup_event->path_size);
8264 
8265 	perf_event__output_id_sample(event, &handle, &sample);
8266 
8267 	perf_output_end(&handle);
8268 out:
8269 	cgroup_event->event_id.header.size = header_size;
8270 }
8271 
8272 static void perf_event_cgroup(struct cgroup *cgrp)
8273 {
8274 	struct perf_cgroup_event cgroup_event;
8275 	char path_enomem[16] = "//enomem";
8276 	char *pathname;
8277 	size_t size;
8278 
8279 	if (!atomic_read(&nr_cgroup_events))
8280 		return;
8281 
8282 	cgroup_event = (struct perf_cgroup_event){
8283 		.event_id  = {
8284 			.header = {
8285 				.type = PERF_RECORD_CGROUP,
8286 				.misc = 0,
8287 				.size = sizeof(cgroup_event.event_id),
8288 			},
8289 			.id = cgroup_id(cgrp),
8290 		},
8291 	};
8292 
8293 	pathname = kmalloc(PATH_MAX, GFP_KERNEL);
8294 	if (pathname == NULL) {
8295 		cgroup_event.path = path_enomem;
8296 	} else {
8297 		/* just to be sure to have enough space for alignment */
8298 		cgroup_path(cgrp, pathname, PATH_MAX - sizeof(u64));
8299 		cgroup_event.path = pathname;
8300 	}
8301 
8302 	/*
8303 	 * Since our buffer works in 8 byte units we need to align our string
8304 	 * size to a multiple of 8. However, we must guarantee the tail end is
8305 	 * zero'd out to avoid leaking random bits to userspace.
8306 	 */
8307 	size = strlen(cgroup_event.path) + 1;
8308 	while (!IS_ALIGNED(size, sizeof(u64)))
8309 		cgroup_event.path[size++] = '\0';
8310 
8311 	cgroup_event.event_id.header.size += size;
8312 	cgroup_event.path_size = size;
8313 
8314 	perf_iterate_sb(perf_event_cgroup_output,
8315 			&cgroup_event,
8316 			NULL);
8317 
8318 	kfree(pathname);
8319 }
8320 
8321 #endif
8322 
8323 /*
8324  * mmap tracking
8325  */
8326 
8327 struct perf_mmap_event {
8328 	struct vm_area_struct	*vma;
8329 
8330 	const char		*file_name;
8331 	int			file_size;
8332 	int			maj, min;
8333 	u64			ino;
8334 	u64			ino_generation;
8335 	u32			prot, flags;
8336 	u8			build_id[BUILD_ID_SIZE_MAX];
8337 	u32			build_id_size;
8338 
8339 	struct {
8340 		struct perf_event_header	header;
8341 
8342 		u32				pid;
8343 		u32				tid;
8344 		u64				start;
8345 		u64				len;
8346 		u64				pgoff;
8347 	} event_id;
8348 };
8349 
8350 static int perf_event_mmap_match(struct perf_event *event,
8351 				 void *data)
8352 {
8353 	struct perf_mmap_event *mmap_event = data;
8354 	struct vm_area_struct *vma = mmap_event->vma;
8355 	int executable = vma->vm_flags & VM_EXEC;
8356 
8357 	return (!executable && event->attr.mmap_data) ||
8358 	       (executable && (event->attr.mmap || event->attr.mmap2));
8359 }
8360 
8361 static void perf_event_mmap_output(struct perf_event *event,
8362 				   void *data)
8363 {
8364 	struct perf_mmap_event *mmap_event = data;
8365 	struct perf_output_handle handle;
8366 	struct perf_sample_data sample;
8367 	int size = mmap_event->event_id.header.size;
8368 	u32 type = mmap_event->event_id.header.type;
8369 	bool use_build_id;
8370 	int ret;
8371 
8372 	if (!perf_event_mmap_match(event, data))
8373 		return;
8374 
8375 	if (event->attr.mmap2) {
8376 		mmap_event->event_id.header.type = PERF_RECORD_MMAP2;
8377 		mmap_event->event_id.header.size += sizeof(mmap_event->maj);
8378 		mmap_event->event_id.header.size += sizeof(mmap_event->min);
8379 		mmap_event->event_id.header.size += sizeof(mmap_event->ino);
8380 		mmap_event->event_id.header.size += sizeof(mmap_event->ino_generation);
8381 		mmap_event->event_id.header.size += sizeof(mmap_event->prot);
8382 		mmap_event->event_id.header.size += sizeof(mmap_event->flags);
8383 	}
8384 
8385 	perf_event_header__init_id(&mmap_event->event_id.header, &sample, event);
8386 	ret = perf_output_begin(&handle, &sample, event,
8387 				mmap_event->event_id.header.size);
8388 	if (ret)
8389 		goto out;
8390 
8391 	mmap_event->event_id.pid = perf_event_pid(event, current);
8392 	mmap_event->event_id.tid = perf_event_tid(event, current);
8393 
8394 	use_build_id = event->attr.build_id && mmap_event->build_id_size;
8395 
8396 	if (event->attr.mmap2 && use_build_id)
8397 		mmap_event->event_id.header.misc |= PERF_RECORD_MISC_MMAP_BUILD_ID;
8398 
8399 	perf_output_put(&handle, mmap_event->event_id);
8400 
8401 	if (event->attr.mmap2) {
8402 		if (use_build_id) {
8403 			u8 size[4] = { (u8) mmap_event->build_id_size, 0, 0, 0 };
8404 
8405 			__output_copy(&handle, size, 4);
8406 			__output_copy(&handle, mmap_event->build_id, BUILD_ID_SIZE_MAX);
8407 		} else {
8408 			perf_output_put(&handle, mmap_event->maj);
8409 			perf_output_put(&handle, mmap_event->min);
8410 			perf_output_put(&handle, mmap_event->ino);
8411 			perf_output_put(&handle, mmap_event->ino_generation);
8412 		}
8413 		perf_output_put(&handle, mmap_event->prot);
8414 		perf_output_put(&handle, mmap_event->flags);
8415 	}
8416 
8417 	__output_copy(&handle, mmap_event->file_name,
8418 				   mmap_event->file_size);
8419 
8420 	perf_event__output_id_sample(event, &handle, &sample);
8421 
8422 	perf_output_end(&handle);
8423 out:
8424 	mmap_event->event_id.header.size = size;
8425 	mmap_event->event_id.header.type = type;
8426 }
8427 
8428 static void perf_event_mmap_event(struct perf_mmap_event *mmap_event)
8429 {
8430 	struct vm_area_struct *vma = mmap_event->vma;
8431 	struct file *file = vma->vm_file;
8432 	int maj = 0, min = 0;
8433 	u64 ino = 0, gen = 0;
8434 	u32 prot = 0, flags = 0;
8435 	unsigned int size;
8436 	char tmp[16];
8437 	char *buf = NULL;
8438 	char *name;
8439 
8440 	if (vma->vm_flags & VM_READ)
8441 		prot |= PROT_READ;
8442 	if (vma->vm_flags & VM_WRITE)
8443 		prot |= PROT_WRITE;
8444 	if (vma->vm_flags & VM_EXEC)
8445 		prot |= PROT_EXEC;
8446 
8447 	if (vma->vm_flags & VM_MAYSHARE)
8448 		flags = MAP_SHARED;
8449 	else
8450 		flags = MAP_PRIVATE;
8451 
8452 	if (vma->vm_flags & VM_LOCKED)
8453 		flags |= MAP_LOCKED;
8454 	if (is_vm_hugetlb_page(vma))
8455 		flags |= MAP_HUGETLB;
8456 
8457 	if (file) {
8458 		struct inode *inode;
8459 		dev_t dev;
8460 
8461 		buf = kmalloc(PATH_MAX, GFP_KERNEL);
8462 		if (!buf) {
8463 			name = "//enomem";
8464 			goto cpy_name;
8465 		}
8466 		/*
8467 		 * d_path() works from the end of the rb backwards, so we
8468 		 * need to add enough zero bytes after the string to handle
8469 		 * the 64bit alignment we do later.
8470 		 */
8471 		name = file_path(file, buf, PATH_MAX - sizeof(u64));
8472 		if (IS_ERR(name)) {
8473 			name = "//toolong";
8474 			goto cpy_name;
8475 		}
8476 		inode = file_inode(vma->vm_file);
8477 		dev = inode->i_sb->s_dev;
8478 		ino = inode->i_ino;
8479 		gen = inode->i_generation;
8480 		maj = MAJOR(dev);
8481 		min = MINOR(dev);
8482 
8483 		goto got_name;
8484 	} else {
8485 		if (vma->vm_ops && vma->vm_ops->name) {
8486 			name = (char *) vma->vm_ops->name(vma);
8487 			if (name)
8488 				goto cpy_name;
8489 		}
8490 
8491 		name = (char *)arch_vma_name(vma);
8492 		if (name)
8493 			goto cpy_name;
8494 
8495 		if (vma->vm_start <= vma->vm_mm->start_brk &&
8496 				vma->vm_end >= vma->vm_mm->brk) {
8497 			name = "[heap]";
8498 			goto cpy_name;
8499 		}
8500 		if (vma->vm_start <= vma->vm_mm->start_stack &&
8501 				vma->vm_end >= vma->vm_mm->start_stack) {
8502 			name = "[stack]";
8503 			goto cpy_name;
8504 		}
8505 
8506 		name = "//anon";
8507 		goto cpy_name;
8508 	}
8509 
8510 cpy_name:
8511 	strlcpy(tmp, name, sizeof(tmp));
8512 	name = tmp;
8513 got_name:
8514 	/*
8515 	 * Since our buffer works in 8 byte units we need to align our string
8516 	 * size to a multiple of 8. However, we must guarantee the tail end is
8517 	 * zero'd out to avoid leaking random bits to userspace.
8518 	 */
8519 	size = strlen(name)+1;
8520 	while (!IS_ALIGNED(size, sizeof(u64)))
8521 		name[size++] = '\0';
8522 
8523 	mmap_event->file_name = name;
8524 	mmap_event->file_size = size;
8525 	mmap_event->maj = maj;
8526 	mmap_event->min = min;
8527 	mmap_event->ino = ino;
8528 	mmap_event->ino_generation = gen;
8529 	mmap_event->prot = prot;
8530 	mmap_event->flags = flags;
8531 
8532 	if (!(vma->vm_flags & VM_EXEC))
8533 		mmap_event->event_id.header.misc |= PERF_RECORD_MISC_MMAP_DATA;
8534 
8535 	mmap_event->event_id.header.size = sizeof(mmap_event->event_id) + size;
8536 
8537 	if (atomic_read(&nr_build_id_events))
8538 		build_id_parse(vma, mmap_event->build_id, &mmap_event->build_id_size);
8539 
8540 	perf_iterate_sb(perf_event_mmap_output,
8541 		       mmap_event,
8542 		       NULL);
8543 
8544 	kfree(buf);
8545 }
8546 
8547 /*
8548  * Check whether inode and address range match filter criteria.
8549  */
8550 static bool perf_addr_filter_match(struct perf_addr_filter *filter,
8551 				     struct file *file, unsigned long offset,
8552 				     unsigned long size)
8553 {
8554 	/* d_inode(NULL) won't be equal to any mapped user-space file */
8555 	if (!filter->path.dentry)
8556 		return false;
8557 
8558 	if (d_inode(filter->path.dentry) != file_inode(file))
8559 		return false;
8560 
8561 	if (filter->offset > offset + size)
8562 		return false;
8563 
8564 	if (filter->offset + filter->size < offset)
8565 		return false;
8566 
8567 	return true;
8568 }
8569 
8570 static bool perf_addr_filter_vma_adjust(struct perf_addr_filter *filter,
8571 					struct vm_area_struct *vma,
8572 					struct perf_addr_filter_range *fr)
8573 {
8574 	unsigned long vma_size = vma->vm_end - vma->vm_start;
8575 	unsigned long off = vma->vm_pgoff << PAGE_SHIFT;
8576 	struct file *file = vma->vm_file;
8577 
8578 	if (!perf_addr_filter_match(filter, file, off, vma_size))
8579 		return false;
8580 
8581 	if (filter->offset < off) {
8582 		fr->start = vma->vm_start;
8583 		fr->size = min(vma_size, filter->size - (off - filter->offset));
8584 	} else {
8585 		fr->start = vma->vm_start + filter->offset - off;
8586 		fr->size = min(vma->vm_end - fr->start, filter->size);
8587 	}
8588 
8589 	return true;
8590 }
8591 
8592 static void __perf_addr_filters_adjust(struct perf_event *event, void *data)
8593 {
8594 	struct perf_addr_filters_head *ifh = perf_event_addr_filters(event);
8595 	struct vm_area_struct *vma = data;
8596 	struct perf_addr_filter *filter;
8597 	unsigned int restart = 0, count = 0;
8598 	unsigned long flags;
8599 
8600 	if (!has_addr_filter(event))
8601 		return;
8602 
8603 	if (!vma->vm_file)
8604 		return;
8605 
8606 	raw_spin_lock_irqsave(&ifh->lock, flags);
8607 	list_for_each_entry(filter, &ifh->list, entry) {
8608 		if (perf_addr_filter_vma_adjust(filter, vma,
8609 						&event->addr_filter_ranges[count]))
8610 			restart++;
8611 
8612 		count++;
8613 	}
8614 
8615 	if (restart)
8616 		event->addr_filters_gen++;
8617 	raw_spin_unlock_irqrestore(&ifh->lock, flags);
8618 
8619 	if (restart)
8620 		perf_event_stop(event, 1);
8621 }
8622 
8623 /*
8624  * Adjust all task's events' filters to the new vma
8625  */
8626 static void perf_addr_filters_adjust(struct vm_area_struct *vma)
8627 {
8628 	struct perf_event_context *ctx;
8629 	int ctxn;
8630 
8631 	/*
8632 	 * Data tracing isn't supported yet and as such there is no need
8633 	 * to keep track of anything that isn't related to executable code:
8634 	 */
8635 	if (!(vma->vm_flags & VM_EXEC))
8636 		return;
8637 
8638 	rcu_read_lock();
8639 	for_each_task_context_nr(ctxn) {
8640 		ctx = rcu_dereference(current->perf_event_ctxp[ctxn]);
8641 		if (!ctx)
8642 			continue;
8643 
8644 		perf_iterate_ctx(ctx, __perf_addr_filters_adjust, vma, true);
8645 	}
8646 	rcu_read_unlock();
8647 }
8648 
8649 void perf_event_mmap(struct vm_area_struct *vma)
8650 {
8651 	struct perf_mmap_event mmap_event;
8652 
8653 	if (!atomic_read(&nr_mmap_events))
8654 		return;
8655 
8656 	mmap_event = (struct perf_mmap_event){
8657 		.vma	= vma,
8658 		/* .file_name */
8659 		/* .file_size */
8660 		.event_id  = {
8661 			.header = {
8662 				.type = PERF_RECORD_MMAP,
8663 				.misc = PERF_RECORD_MISC_USER,
8664 				/* .size */
8665 			},
8666 			/* .pid */
8667 			/* .tid */
8668 			.start  = vma->vm_start,
8669 			.len    = vma->vm_end - vma->vm_start,
8670 			.pgoff  = (u64)vma->vm_pgoff << PAGE_SHIFT,
8671 		},
8672 		/* .maj (attr_mmap2 only) */
8673 		/* .min (attr_mmap2 only) */
8674 		/* .ino (attr_mmap2 only) */
8675 		/* .ino_generation (attr_mmap2 only) */
8676 		/* .prot (attr_mmap2 only) */
8677 		/* .flags (attr_mmap2 only) */
8678 	};
8679 
8680 	perf_addr_filters_adjust(vma);
8681 	perf_event_mmap_event(&mmap_event);
8682 }
8683 
8684 void perf_event_aux_event(struct perf_event *event, unsigned long head,
8685 			  unsigned long size, u64 flags)
8686 {
8687 	struct perf_output_handle handle;
8688 	struct perf_sample_data sample;
8689 	struct perf_aux_event {
8690 		struct perf_event_header	header;
8691 		u64				offset;
8692 		u64				size;
8693 		u64				flags;
8694 	} rec = {
8695 		.header = {
8696 			.type = PERF_RECORD_AUX,
8697 			.misc = 0,
8698 			.size = sizeof(rec),
8699 		},
8700 		.offset		= head,
8701 		.size		= size,
8702 		.flags		= flags,
8703 	};
8704 	int ret;
8705 
8706 	perf_event_header__init_id(&rec.header, &sample, event);
8707 	ret = perf_output_begin(&handle, &sample, event, rec.header.size);
8708 
8709 	if (ret)
8710 		return;
8711 
8712 	perf_output_put(&handle, rec);
8713 	perf_event__output_id_sample(event, &handle, &sample);
8714 
8715 	perf_output_end(&handle);
8716 }
8717 
8718 /*
8719  * Lost/dropped samples logging
8720  */
8721 void perf_log_lost_samples(struct perf_event *event, u64 lost)
8722 {
8723 	struct perf_output_handle handle;
8724 	struct perf_sample_data sample;
8725 	int ret;
8726 
8727 	struct {
8728 		struct perf_event_header	header;
8729 		u64				lost;
8730 	} lost_samples_event = {
8731 		.header = {
8732 			.type = PERF_RECORD_LOST_SAMPLES,
8733 			.misc = 0,
8734 			.size = sizeof(lost_samples_event),
8735 		},
8736 		.lost		= lost,
8737 	};
8738 
8739 	perf_event_header__init_id(&lost_samples_event.header, &sample, event);
8740 
8741 	ret = perf_output_begin(&handle, &sample, event,
8742 				lost_samples_event.header.size);
8743 	if (ret)
8744 		return;
8745 
8746 	perf_output_put(&handle, lost_samples_event);
8747 	perf_event__output_id_sample(event, &handle, &sample);
8748 	perf_output_end(&handle);
8749 }
8750 
8751 /*
8752  * context_switch tracking
8753  */
8754 
8755 struct perf_switch_event {
8756 	struct task_struct	*task;
8757 	struct task_struct	*next_prev;
8758 
8759 	struct {
8760 		struct perf_event_header	header;
8761 		u32				next_prev_pid;
8762 		u32				next_prev_tid;
8763 	} event_id;
8764 };
8765 
8766 static int perf_event_switch_match(struct perf_event *event)
8767 {
8768 	return event->attr.context_switch;
8769 }
8770 
8771 static void perf_event_switch_output(struct perf_event *event, void *data)
8772 {
8773 	struct perf_switch_event *se = data;
8774 	struct perf_output_handle handle;
8775 	struct perf_sample_data sample;
8776 	int ret;
8777 
8778 	if (!perf_event_switch_match(event))
8779 		return;
8780 
8781 	/* Only CPU-wide events are allowed to see next/prev pid/tid */
8782 	if (event->ctx->task) {
8783 		se->event_id.header.type = PERF_RECORD_SWITCH;
8784 		se->event_id.header.size = sizeof(se->event_id.header);
8785 	} else {
8786 		se->event_id.header.type = PERF_RECORD_SWITCH_CPU_WIDE;
8787 		se->event_id.header.size = sizeof(se->event_id);
8788 		se->event_id.next_prev_pid =
8789 					perf_event_pid(event, se->next_prev);
8790 		se->event_id.next_prev_tid =
8791 					perf_event_tid(event, se->next_prev);
8792 	}
8793 
8794 	perf_event_header__init_id(&se->event_id.header, &sample, event);
8795 
8796 	ret = perf_output_begin(&handle, &sample, event, se->event_id.header.size);
8797 	if (ret)
8798 		return;
8799 
8800 	if (event->ctx->task)
8801 		perf_output_put(&handle, se->event_id.header);
8802 	else
8803 		perf_output_put(&handle, se->event_id);
8804 
8805 	perf_event__output_id_sample(event, &handle, &sample);
8806 
8807 	perf_output_end(&handle);
8808 }
8809 
8810 static void perf_event_switch(struct task_struct *task,
8811 			      struct task_struct *next_prev, bool sched_in)
8812 {
8813 	struct perf_switch_event switch_event;
8814 
8815 	/* N.B. caller checks nr_switch_events != 0 */
8816 
8817 	switch_event = (struct perf_switch_event){
8818 		.task		= task,
8819 		.next_prev	= next_prev,
8820 		.event_id	= {
8821 			.header = {
8822 				/* .type */
8823 				.misc = sched_in ? 0 : PERF_RECORD_MISC_SWITCH_OUT,
8824 				/* .size */
8825 			},
8826 			/* .next_prev_pid */
8827 			/* .next_prev_tid */
8828 		},
8829 	};
8830 
8831 	if (!sched_in && task->on_rq) {
8832 		switch_event.event_id.header.misc |=
8833 				PERF_RECORD_MISC_SWITCH_OUT_PREEMPT;
8834 	}
8835 
8836 	perf_iterate_sb(perf_event_switch_output, &switch_event, NULL);
8837 }
8838 
8839 /*
8840  * IRQ throttle logging
8841  */
8842 
8843 static void perf_log_throttle(struct perf_event *event, int enable)
8844 {
8845 	struct perf_output_handle handle;
8846 	struct perf_sample_data sample;
8847 	int ret;
8848 
8849 	struct {
8850 		struct perf_event_header	header;
8851 		u64				time;
8852 		u64				id;
8853 		u64				stream_id;
8854 	} throttle_event = {
8855 		.header = {
8856 			.type = PERF_RECORD_THROTTLE,
8857 			.misc = 0,
8858 			.size = sizeof(throttle_event),
8859 		},
8860 		.time		= perf_event_clock(event),
8861 		.id		= primary_event_id(event),
8862 		.stream_id	= event->id,
8863 	};
8864 
8865 	if (enable)
8866 		throttle_event.header.type = PERF_RECORD_UNTHROTTLE;
8867 
8868 	perf_event_header__init_id(&throttle_event.header, &sample, event);
8869 
8870 	ret = perf_output_begin(&handle, &sample, event,
8871 				throttle_event.header.size);
8872 	if (ret)
8873 		return;
8874 
8875 	perf_output_put(&handle, throttle_event);
8876 	perf_event__output_id_sample(event, &handle, &sample);
8877 	perf_output_end(&handle);
8878 }
8879 
8880 /*
8881  * ksymbol register/unregister tracking
8882  */
8883 
8884 struct perf_ksymbol_event {
8885 	const char	*name;
8886 	int		name_len;
8887 	struct {
8888 		struct perf_event_header        header;
8889 		u64				addr;
8890 		u32				len;
8891 		u16				ksym_type;
8892 		u16				flags;
8893 	} event_id;
8894 };
8895 
8896 static int perf_event_ksymbol_match(struct perf_event *event)
8897 {
8898 	return event->attr.ksymbol;
8899 }
8900 
8901 static void perf_event_ksymbol_output(struct perf_event *event, void *data)
8902 {
8903 	struct perf_ksymbol_event *ksymbol_event = data;
8904 	struct perf_output_handle handle;
8905 	struct perf_sample_data sample;
8906 	int ret;
8907 
8908 	if (!perf_event_ksymbol_match(event))
8909 		return;
8910 
8911 	perf_event_header__init_id(&ksymbol_event->event_id.header,
8912 				   &sample, event);
8913 	ret = perf_output_begin(&handle, &sample, event,
8914 				ksymbol_event->event_id.header.size);
8915 	if (ret)
8916 		return;
8917 
8918 	perf_output_put(&handle, ksymbol_event->event_id);
8919 	__output_copy(&handle, ksymbol_event->name, ksymbol_event->name_len);
8920 	perf_event__output_id_sample(event, &handle, &sample);
8921 
8922 	perf_output_end(&handle);
8923 }
8924 
8925 void perf_event_ksymbol(u16 ksym_type, u64 addr, u32 len, bool unregister,
8926 			const char *sym)
8927 {
8928 	struct perf_ksymbol_event ksymbol_event;
8929 	char name[KSYM_NAME_LEN];
8930 	u16 flags = 0;
8931 	int name_len;
8932 
8933 	if (!atomic_read(&nr_ksymbol_events))
8934 		return;
8935 
8936 	if (ksym_type >= PERF_RECORD_KSYMBOL_TYPE_MAX ||
8937 	    ksym_type == PERF_RECORD_KSYMBOL_TYPE_UNKNOWN)
8938 		goto err;
8939 
8940 	strlcpy(name, sym, KSYM_NAME_LEN);
8941 	name_len = strlen(name) + 1;
8942 	while (!IS_ALIGNED(name_len, sizeof(u64)))
8943 		name[name_len++] = '\0';
8944 	BUILD_BUG_ON(KSYM_NAME_LEN % sizeof(u64));
8945 
8946 	if (unregister)
8947 		flags |= PERF_RECORD_KSYMBOL_FLAGS_UNREGISTER;
8948 
8949 	ksymbol_event = (struct perf_ksymbol_event){
8950 		.name = name,
8951 		.name_len = name_len,
8952 		.event_id = {
8953 			.header = {
8954 				.type = PERF_RECORD_KSYMBOL,
8955 				.size = sizeof(ksymbol_event.event_id) +
8956 					name_len,
8957 			},
8958 			.addr = addr,
8959 			.len = len,
8960 			.ksym_type = ksym_type,
8961 			.flags = flags,
8962 		},
8963 	};
8964 
8965 	perf_iterate_sb(perf_event_ksymbol_output, &ksymbol_event, NULL);
8966 	return;
8967 err:
8968 	WARN_ONCE(1, "%s: Invalid KSYMBOL type 0x%x\n", __func__, ksym_type);
8969 }
8970 
8971 /*
8972  * bpf program load/unload tracking
8973  */
8974 
8975 struct perf_bpf_event {
8976 	struct bpf_prog	*prog;
8977 	struct {
8978 		struct perf_event_header        header;
8979 		u16				type;
8980 		u16				flags;
8981 		u32				id;
8982 		u8				tag[BPF_TAG_SIZE];
8983 	} event_id;
8984 };
8985 
8986 static int perf_event_bpf_match(struct perf_event *event)
8987 {
8988 	return event->attr.bpf_event;
8989 }
8990 
8991 static void perf_event_bpf_output(struct perf_event *event, void *data)
8992 {
8993 	struct perf_bpf_event *bpf_event = data;
8994 	struct perf_output_handle handle;
8995 	struct perf_sample_data sample;
8996 	int ret;
8997 
8998 	if (!perf_event_bpf_match(event))
8999 		return;
9000 
9001 	perf_event_header__init_id(&bpf_event->event_id.header,
9002 				   &sample, event);
9003 	ret = perf_output_begin(&handle, data, event,
9004 				bpf_event->event_id.header.size);
9005 	if (ret)
9006 		return;
9007 
9008 	perf_output_put(&handle, bpf_event->event_id);
9009 	perf_event__output_id_sample(event, &handle, &sample);
9010 
9011 	perf_output_end(&handle);
9012 }
9013 
9014 static void perf_event_bpf_emit_ksymbols(struct bpf_prog *prog,
9015 					 enum perf_bpf_event_type type)
9016 {
9017 	bool unregister = type == PERF_BPF_EVENT_PROG_UNLOAD;
9018 	int i;
9019 
9020 	if (prog->aux->func_cnt == 0) {
9021 		perf_event_ksymbol(PERF_RECORD_KSYMBOL_TYPE_BPF,
9022 				   (u64)(unsigned long)prog->bpf_func,
9023 				   prog->jited_len, unregister,
9024 				   prog->aux->ksym.name);
9025 	} else {
9026 		for (i = 0; i < prog->aux->func_cnt; i++) {
9027 			struct bpf_prog *subprog = prog->aux->func[i];
9028 
9029 			perf_event_ksymbol(
9030 				PERF_RECORD_KSYMBOL_TYPE_BPF,
9031 				(u64)(unsigned long)subprog->bpf_func,
9032 				subprog->jited_len, unregister,
9033 				prog->aux->ksym.name);
9034 		}
9035 	}
9036 }
9037 
9038 void perf_event_bpf_event(struct bpf_prog *prog,
9039 			  enum perf_bpf_event_type type,
9040 			  u16 flags)
9041 {
9042 	struct perf_bpf_event bpf_event;
9043 
9044 	if (type <= PERF_BPF_EVENT_UNKNOWN ||
9045 	    type >= PERF_BPF_EVENT_MAX)
9046 		return;
9047 
9048 	switch (type) {
9049 	case PERF_BPF_EVENT_PROG_LOAD:
9050 	case PERF_BPF_EVENT_PROG_UNLOAD:
9051 		if (atomic_read(&nr_ksymbol_events))
9052 			perf_event_bpf_emit_ksymbols(prog, type);
9053 		break;
9054 	default:
9055 		break;
9056 	}
9057 
9058 	if (!atomic_read(&nr_bpf_events))
9059 		return;
9060 
9061 	bpf_event = (struct perf_bpf_event){
9062 		.prog = prog,
9063 		.event_id = {
9064 			.header = {
9065 				.type = PERF_RECORD_BPF_EVENT,
9066 				.size = sizeof(bpf_event.event_id),
9067 			},
9068 			.type = type,
9069 			.flags = flags,
9070 			.id = prog->aux->id,
9071 		},
9072 	};
9073 
9074 	BUILD_BUG_ON(BPF_TAG_SIZE % sizeof(u64));
9075 
9076 	memcpy(bpf_event.event_id.tag, prog->tag, BPF_TAG_SIZE);
9077 	perf_iterate_sb(perf_event_bpf_output, &bpf_event, NULL);
9078 }
9079 
9080 struct perf_text_poke_event {
9081 	const void		*old_bytes;
9082 	const void		*new_bytes;
9083 	size_t			pad;
9084 	u16			old_len;
9085 	u16			new_len;
9086 
9087 	struct {
9088 		struct perf_event_header	header;
9089 
9090 		u64				addr;
9091 	} event_id;
9092 };
9093 
9094 static int perf_event_text_poke_match(struct perf_event *event)
9095 {
9096 	return event->attr.text_poke;
9097 }
9098 
9099 static void perf_event_text_poke_output(struct perf_event *event, void *data)
9100 {
9101 	struct perf_text_poke_event *text_poke_event = data;
9102 	struct perf_output_handle handle;
9103 	struct perf_sample_data sample;
9104 	u64 padding = 0;
9105 	int ret;
9106 
9107 	if (!perf_event_text_poke_match(event))
9108 		return;
9109 
9110 	perf_event_header__init_id(&text_poke_event->event_id.header, &sample, event);
9111 
9112 	ret = perf_output_begin(&handle, &sample, event,
9113 				text_poke_event->event_id.header.size);
9114 	if (ret)
9115 		return;
9116 
9117 	perf_output_put(&handle, text_poke_event->event_id);
9118 	perf_output_put(&handle, text_poke_event->old_len);
9119 	perf_output_put(&handle, text_poke_event->new_len);
9120 
9121 	__output_copy(&handle, text_poke_event->old_bytes, text_poke_event->old_len);
9122 	__output_copy(&handle, text_poke_event->new_bytes, text_poke_event->new_len);
9123 
9124 	if (text_poke_event->pad)
9125 		__output_copy(&handle, &padding, text_poke_event->pad);
9126 
9127 	perf_event__output_id_sample(event, &handle, &sample);
9128 
9129 	perf_output_end(&handle);
9130 }
9131 
9132 void perf_event_text_poke(const void *addr, const void *old_bytes,
9133 			  size_t old_len, const void *new_bytes, size_t new_len)
9134 {
9135 	struct perf_text_poke_event text_poke_event;
9136 	size_t tot, pad;
9137 
9138 	if (!atomic_read(&nr_text_poke_events))
9139 		return;
9140 
9141 	tot  = sizeof(text_poke_event.old_len) + old_len;
9142 	tot += sizeof(text_poke_event.new_len) + new_len;
9143 	pad  = ALIGN(tot, sizeof(u64)) - tot;
9144 
9145 	text_poke_event = (struct perf_text_poke_event){
9146 		.old_bytes    = old_bytes,
9147 		.new_bytes    = new_bytes,
9148 		.pad          = pad,
9149 		.old_len      = old_len,
9150 		.new_len      = new_len,
9151 		.event_id  = {
9152 			.header = {
9153 				.type = PERF_RECORD_TEXT_POKE,
9154 				.misc = PERF_RECORD_MISC_KERNEL,
9155 				.size = sizeof(text_poke_event.event_id) + tot + pad,
9156 			},
9157 			.addr = (unsigned long)addr,
9158 		},
9159 	};
9160 
9161 	perf_iterate_sb(perf_event_text_poke_output, &text_poke_event, NULL);
9162 }
9163 
9164 void perf_event_itrace_started(struct perf_event *event)
9165 {
9166 	event->attach_state |= PERF_ATTACH_ITRACE;
9167 }
9168 
9169 static void perf_log_itrace_start(struct perf_event *event)
9170 {
9171 	struct perf_output_handle handle;
9172 	struct perf_sample_data sample;
9173 	struct perf_aux_event {
9174 		struct perf_event_header        header;
9175 		u32				pid;
9176 		u32				tid;
9177 	} rec;
9178 	int ret;
9179 
9180 	if (event->parent)
9181 		event = event->parent;
9182 
9183 	if (!(event->pmu->capabilities & PERF_PMU_CAP_ITRACE) ||
9184 	    event->attach_state & PERF_ATTACH_ITRACE)
9185 		return;
9186 
9187 	rec.header.type	= PERF_RECORD_ITRACE_START;
9188 	rec.header.misc	= 0;
9189 	rec.header.size	= sizeof(rec);
9190 	rec.pid	= perf_event_pid(event, current);
9191 	rec.tid	= perf_event_tid(event, current);
9192 
9193 	perf_event_header__init_id(&rec.header, &sample, event);
9194 	ret = perf_output_begin(&handle, &sample, event, rec.header.size);
9195 
9196 	if (ret)
9197 		return;
9198 
9199 	perf_output_put(&handle, rec);
9200 	perf_event__output_id_sample(event, &handle, &sample);
9201 
9202 	perf_output_end(&handle);
9203 }
9204 
9205 void perf_report_aux_output_id(struct perf_event *event, u64 hw_id)
9206 {
9207 	struct perf_output_handle handle;
9208 	struct perf_sample_data sample;
9209 	struct perf_aux_event {
9210 		struct perf_event_header        header;
9211 		u64				hw_id;
9212 	} rec;
9213 	int ret;
9214 
9215 	if (event->parent)
9216 		event = event->parent;
9217 
9218 	rec.header.type	= PERF_RECORD_AUX_OUTPUT_HW_ID;
9219 	rec.header.misc	= 0;
9220 	rec.header.size	= sizeof(rec);
9221 	rec.hw_id	= hw_id;
9222 
9223 	perf_event_header__init_id(&rec.header, &sample, event);
9224 	ret = perf_output_begin(&handle, &sample, event, rec.header.size);
9225 
9226 	if (ret)
9227 		return;
9228 
9229 	perf_output_put(&handle, rec);
9230 	perf_event__output_id_sample(event, &handle, &sample);
9231 
9232 	perf_output_end(&handle);
9233 }
9234 
9235 static int
9236 __perf_event_account_interrupt(struct perf_event *event, int throttle)
9237 {
9238 	struct hw_perf_event *hwc = &event->hw;
9239 	int ret = 0;
9240 	u64 seq;
9241 
9242 	seq = __this_cpu_read(perf_throttled_seq);
9243 	if (seq != hwc->interrupts_seq) {
9244 		hwc->interrupts_seq = seq;
9245 		hwc->interrupts = 1;
9246 	} else {
9247 		hwc->interrupts++;
9248 		if (unlikely(throttle
9249 			     && hwc->interrupts >= max_samples_per_tick)) {
9250 			__this_cpu_inc(perf_throttled_count);
9251 			tick_dep_set_cpu(smp_processor_id(), TICK_DEP_BIT_PERF_EVENTS);
9252 			hwc->interrupts = MAX_INTERRUPTS;
9253 			perf_log_throttle(event, 0);
9254 			ret = 1;
9255 		}
9256 	}
9257 
9258 	if (event->attr.freq) {
9259 		u64 now = perf_clock();
9260 		s64 delta = now - hwc->freq_time_stamp;
9261 
9262 		hwc->freq_time_stamp = now;
9263 
9264 		if (delta > 0 && delta < 2*TICK_NSEC)
9265 			perf_adjust_period(event, delta, hwc->last_period, true);
9266 	}
9267 
9268 	return ret;
9269 }
9270 
9271 int perf_event_account_interrupt(struct perf_event *event)
9272 {
9273 	return __perf_event_account_interrupt(event, 1);
9274 }
9275 
9276 /*
9277  * Generic event overflow handling, sampling.
9278  */
9279 
9280 static int __perf_event_overflow(struct perf_event *event,
9281 				 int throttle, struct perf_sample_data *data,
9282 				 struct pt_regs *regs)
9283 {
9284 	int events = atomic_read(&event->event_limit);
9285 	int ret = 0;
9286 
9287 	/*
9288 	 * Non-sampling counters might still use the PMI to fold short
9289 	 * hardware counters, ignore those.
9290 	 */
9291 	if (unlikely(!is_sampling_event(event)))
9292 		return 0;
9293 
9294 	ret = __perf_event_account_interrupt(event, throttle);
9295 
9296 	/*
9297 	 * XXX event_limit might not quite work as expected on inherited
9298 	 * events
9299 	 */
9300 
9301 	event->pending_kill = POLL_IN;
9302 	if (events && atomic_dec_and_test(&event->event_limit)) {
9303 		ret = 1;
9304 		event->pending_kill = POLL_HUP;
9305 		perf_event_disable_inatomic(event);
9306 	}
9307 
9308 	if (event->attr.sigtrap) {
9309 		/*
9310 		 * Should not be able to return to user space without processing
9311 		 * pending_sigtrap (kernel events can overflow multiple times).
9312 		 */
9313 		WARN_ON_ONCE(event->pending_sigtrap && event->attr.exclude_kernel);
9314 		if (!event->pending_sigtrap) {
9315 			event->pending_sigtrap = 1;
9316 			local_inc(&event->ctx->nr_pending);
9317 		}
9318 		event->pending_addr = data->addr;
9319 		irq_work_queue(&event->pending_irq);
9320 	}
9321 
9322 	READ_ONCE(event->overflow_handler)(event, data, regs);
9323 
9324 	if (*perf_event_fasync(event) && event->pending_kill) {
9325 		event->pending_wakeup = 1;
9326 		irq_work_queue(&event->pending_irq);
9327 	}
9328 
9329 	return ret;
9330 }
9331 
9332 int perf_event_overflow(struct perf_event *event,
9333 			struct perf_sample_data *data,
9334 			struct pt_regs *regs)
9335 {
9336 	return __perf_event_overflow(event, 1, data, regs);
9337 }
9338 
9339 /*
9340  * Generic software event infrastructure
9341  */
9342 
9343 struct swevent_htable {
9344 	struct swevent_hlist		*swevent_hlist;
9345 	struct mutex			hlist_mutex;
9346 	int				hlist_refcount;
9347 
9348 	/* Recursion avoidance in each contexts */
9349 	int				recursion[PERF_NR_CONTEXTS];
9350 };
9351 
9352 static DEFINE_PER_CPU(struct swevent_htable, swevent_htable);
9353 
9354 /*
9355  * We directly increment event->count and keep a second value in
9356  * event->hw.period_left to count intervals. This period event
9357  * is kept in the range [-sample_period, 0] so that we can use the
9358  * sign as trigger.
9359  */
9360 
9361 u64 perf_swevent_set_period(struct perf_event *event)
9362 {
9363 	struct hw_perf_event *hwc = &event->hw;
9364 	u64 period = hwc->last_period;
9365 	u64 nr, offset;
9366 	s64 old, val;
9367 
9368 	hwc->last_period = hwc->sample_period;
9369 
9370 again:
9371 	old = val = local64_read(&hwc->period_left);
9372 	if (val < 0)
9373 		return 0;
9374 
9375 	nr = div64_u64(period + val, period);
9376 	offset = nr * period;
9377 	val -= offset;
9378 	if (local64_cmpxchg(&hwc->period_left, old, val) != old)
9379 		goto again;
9380 
9381 	return nr;
9382 }
9383 
9384 static void perf_swevent_overflow(struct perf_event *event, u64 overflow,
9385 				    struct perf_sample_data *data,
9386 				    struct pt_regs *regs)
9387 {
9388 	struct hw_perf_event *hwc = &event->hw;
9389 	int throttle = 0;
9390 
9391 	if (!overflow)
9392 		overflow = perf_swevent_set_period(event);
9393 
9394 	if (hwc->interrupts == MAX_INTERRUPTS)
9395 		return;
9396 
9397 	for (; overflow; overflow--) {
9398 		if (__perf_event_overflow(event, throttle,
9399 					    data, regs)) {
9400 			/*
9401 			 * We inhibit the overflow from happening when
9402 			 * hwc->interrupts == MAX_INTERRUPTS.
9403 			 */
9404 			break;
9405 		}
9406 		throttle = 1;
9407 	}
9408 }
9409 
9410 static void perf_swevent_event(struct perf_event *event, u64 nr,
9411 			       struct perf_sample_data *data,
9412 			       struct pt_regs *regs)
9413 {
9414 	struct hw_perf_event *hwc = &event->hw;
9415 
9416 	local64_add(nr, &event->count);
9417 
9418 	if (!regs)
9419 		return;
9420 
9421 	if (!is_sampling_event(event))
9422 		return;
9423 
9424 	if ((event->attr.sample_type & PERF_SAMPLE_PERIOD) && !event->attr.freq) {
9425 		data->period = nr;
9426 		return perf_swevent_overflow(event, 1, data, regs);
9427 	} else
9428 		data->period = event->hw.last_period;
9429 
9430 	if (nr == 1 && hwc->sample_period == 1 && !event->attr.freq)
9431 		return perf_swevent_overflow(event, 1, data, regs);
9432 
9433 	if (local64_add_negative(nr, &hwc->period_left))
9434 		return;
9435 
9436 	perf_swevent_overflow(event, 0, data, regs);
9437 }
9438 
9439 static int perf_exclude_event(struct perf_event *event,
9440 			      struct pt_regs *regs)
9441 {
9442 	if (event->hw.state & PERF_HES_STOPPED)
9443 		return 1;
9444 
9445 	if (regs) {
9446 		if (event->attr.exclude_user && user_mode(regs))
9447 			return 1;
9448 
9449 		if (event->attr.exclude_kernel && !user_mode(regs))
9450 			return 1;
9451 	}
9452 
9453 	return 0;
9454 }
9455 
9456 static int perf_swevent_match(struct perf_event *event,
9457 				enum perf_type_id type,
9458 				u32 event_id,
9459 				struct perf_sample_data *data,
9460 				struct pt_regs *regs)
9461 {
9462 	if (event->attr.type != type)
9463 		return 0;
9464 
9465 	if (event->attr.config != event_id)
9466 		return 0;
9467 
9468 	if (perf_exclude_event(event, regs))
9469 		return 0;
9470 
9471 	return 1;
9472 }
9473 
9474 static inline u64 swevent_hash(u64 type, u32 event_id)
9475 {
9476 	u64 val = event_id | (type << 32);
9477 
9478 	return hash_64(val, SWEVENT_HLIST_BITS);
9479 }
9480 
9481 static inline struct hlist_head *
9482 __find_swevent_head(struct swevent_hlist *hlist, u64 type, u32 event_id)
9483 {
9484 	u64 hash = swevent_hash(type, event_id);
9485 
9486 	return &hlist->heads[hash];
9487 }
9488 
9489 /* For the read side: events when they trigger */
9490 static inline struct hlist_head *
9491 find_swevent_head_rcu(struct swevent_htable *swhash, u64 type, u32 event_id)
9492 {
9493 	struct swevent_hlist *hlist;
9494 
9495 	hlist = rcu_dereference(swhash->swevent_hlist);
9496 	if (!hlist)
9497 		return NULL;
9498 
9499 	return __find_swevent_head(hlist, type, event_id);
9500 }
9501 
9502 /* For the event head insertion and removal in the hlist */
9503 static inline struct hlist_head *
9504 find_swevent_head(struct swevent_htable *swhash, struct perf_event *event)
9505 {
9506 	struct swevent_hlist *hlist;
9507 	u32 event_id = event->attr.config;
9508 	u64 type = event->attr.type;
9509 
9510 	/*
9511 	 * Event scheduling is always serialized against hlist allocation
9512 	 * and release. Which makes the protected version suitable here.
9513 	 * The context lock guarantees that.
9514 	 */
9515 	hlist = rcu_dereference_protected(swhash->swevent_hlist,
9516 					  lockdep_is_held(&event->ctx->lock));
9517 	if (!hlist)
9518 		return NULL;
9519 
9520 	return __find_swevent_head(hlist, type, event_id);
9521 }
9522 
9523 static void do_perf_sw_event(enum perf_type_id type, u32 event_id,
9524 				    u64 nr,
9525 				    struct perf_sample_data *data,
9526 				    struct pt_regs *regs)
9527 {
9528 	struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable);
9529 	struct perf_event *event;
9530 	struct hlist_head *head;
9531 
9532 	rcu_read_lock();
9533 	head = find_swevent_head_rcu(swhash, type, event_id);
9534 	if (!head)
9535 		goto end;
9536 
9537 	hlist_for_each_entry_rcu(event, head, hlist_entry) {
9538 		if (perf_swevent_match(event, type, event_id, data, regs))
9539 			perf_swevent_event(event, nr, data, regs);
9540 	}
9541 end:
9542 	rcu_read_unlock();
9543 }
9544 
9545 DEFINE_PER_CPU(struct pt_regs, __perf_regs[4]);
9546 
9547 int perf_swevent_get_recursion_context(void)
9548 {
9549 	struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable);
9550 
9551 	return get_recursion_context(swhash->recursion);
9552 }
9553 EXPORT_SYMBOL_GPL(perf_swevent_get_recursion_context);
9554 
9555 void perf_swevent_put_recursion_context(int rctx)
9556 {
9557 	struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable);
9558 
9559 	put_recursion_context(swhash->recursion, rctx);
9560 }
9561 
9562 void ___perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr)
9563 {
9564 	struct perf_sample_data data;
9565 
9566 	if (WARN_ON_ONCE(!regs))
9567 		return;
9568 
9569 	perf_sample_data_init(&data, addr, 0);
9570 	do_perf_sw_event(PERF_TYPE_SOFTWARE, event_id, nr, &data, regs);
9571 }
9572 
9573 void __perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr)
9574 {
9575 	int rctx;
9576 
9577 	preempt_disable_notrace();
9578 	rctx = perf_swevent_get_recursion_context();
9579 	if (unlikely(rctx < 0))
9580 		goto fail;
9581 
9582 	___perf_sw_event(event_id, nr, regs, addr);
9583 
9584 	perf_swevent_put_recursion_context(rctx);
9585 fail:
9586 	preempt_enable_notrace();
9587 }
9588 
9589 static void perf_swevent_read(struct perf_event *event)
9590 {
9591 }
9592 
9593 static int perf_swevent_add(struct perf_event *event, int flags)
9594 {
9595 	struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable);
9596 	struct hw_perf_event *hwc = &event->hw;
9597 	struct hlist_head *head;
9598 
9599 	if (is_sampling_event(event)) {
9600 		hwc->last_period = hwc->sample_period;
9601 		perf_swevent_set_period(event);
9602 	}
9603 
9604 	hwc->state = !(flags & PERF_EF_START);
9605 
9606 	head = find_swevent_head(swhash, event);
9607 	if (WARN_ON_ONCE(!head))
9608 		return -EINVAL;
9609 
9610 	hlist_add_head_rcu(&event->hlist_entry, head);
9611 	perf_event_update_userpage(event);
9612 
9613 	return 0;
9614 }
9615 
9616 static void perf_swevent_del(struct perf_event *event, int flags)
9617 {
9618 	hlist_del_rcu(&event->hlist_entry);
9619 }
9620 
9621 static void perf_swevent_start(struct perf_event *event, int flags)
9622 {
9623 	event->hw.state = 0;
9624 }
9625 
9626 static void perf_swevent_stop(struct perf_event *event, int flags)
9627 {
9628 	event->hw.state = PERF_HES_STOPPED;
9629 }
9630 
9631 /* Deref the hlist from the update side */
9632 static inline struct swevent_hlist *
9633 swevent_hlist_deref(struct swevent_htable *swhash)
9634 {
9635 	return rcu_dereference_protected(swhash->swevent_hlist,
9636 					 lockdep_is_held(&swhash->hlist_mutex));
9637 }
9638 
9639 static void swevent_hlist_release(struct swevent_htable *swhash)
9640 {
9641 	struct swevent_hlist *hlist = swevent_hlist_deref(swhash);
9642 
9643 	if (!hlist)
9644 		return;
9645 
9646 	RCU_INIT_POINTER(swhash->swevent_hlist, NULL);
9647 	kfree_rcu(hlist, rcu_head);
9648 }
9649 
9650 static void swevent_hlist_put_cpu(int cpu)
9651 {
9652 	struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
9653 
9654 	mutex_lock(&swhash->hlist_mutex);
9655 
9656 	if (!--swhash->hlist_refcount)
9657 		swevent_hlist_release(swhash);
9658 
9659 	mutex_unlock(&swhash->hlist_mutex);
9660 }
9661 
9662 static void swevent_hlist_put(void)
9663 {
9664 	int cpu;
9665 
9666 	for_each_possible_cpu(cpu)
9667 		swevent_hlist_put_cpu(cpu);
9668 }
9669 
9670 static int swevent_hlist_get_cpu(int cpu)
9671 {
9672 	struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
9673 	int err = 0;
9674 
9675 	mutex_lock(&swhash->hlist_mutex);
9676 	if (!swevent_hlist_deref(swhash) &&
9677 	    cpumask_test_cpu(cpu, perf_online_mask)) {
9678 		struct swevent_hlist *hlist;
9679 
9680 		hlist = kzalloc(sizeof(*hlist), GFP_KERNEL);
9681 		if (!hlist) {
9682 			err = -ENOMEM;
9683 			goto exit;
9684 		}
9685 		rcu_assign_pointer(swhash->swevent_hlist, hlist);
9686 	}
9687 	swhash->hlist_refcount++;
9688 exit:
9689 	mutex_unlock(&swhash->hlist_mutex);
9690 
9691 	return err;
9692 }
9693 
9694 static int swevent_hlist_get(void)
9695 {
9696 	int err, cpu, failed_cpu;
9697 
9698 	mutex_lock(&pmus_lock);
9699 	for_each_possible_cpu(cpu) {
9700 		err = swevent_hlist_get_cpu(cpu);
9701 		if (err) {
9702 			failed_cpu = cpu;
9703 			goto fail;
9704 		}
9705 	}
9706 	mutex_unlock(&pmus_lock);
9707 	return 0;
9708 fail:
9709 	for_each_possible_cpu(cpu) {
9710 		if (cpu == failed_cpu)
9711 			break;
9712 		swevent_hlist_put_cpu(cpu);
9713 	}
9714 	mutex_unlock(&pmus_lock);
9715 	return err;
9716 }
9717 
9718 struct static_key perf_swevent_enabled[PERF_COUNT_SW_MAX];
9719 
9720 static void sw_perf_event_destroy(struct perf_event *event)
9721 {
9722 	u64 event_id = event->attr.config;
9723 
9724 	WARN_ON(event->parent);
9725 
9726 	static_key_slow_dec(&perf_swevent_enabled[event_id]);
9727 	swevent_hlist_put();
9728 }
9729 
9730 static int perf_swevent_init(struct perf_event *event)
9731 {
9732 	u64 event_id = event->attr.config;
9733 
9734 	if (event->attr.type != PERF_TYPE_SOFTWARE)
9735 		return -ENOENT;
9736 
9737 	/*
9738 	 * no branch sampling for software events
9739 	 */
9740 	if (has_branch_stack(event))
9741 		return -EOPNOTSUPP;
9742 
9743 	switch (event_id) {
9744 	case PERF_COUNT_SW_CPU_CLOCK:
9745 	case PERF_COUNT_SW_TASK_CLOCK:
9746 		return -ENOENT;
9747 
9748 	default:
9749 		break;
9750 	}
9751 
9752 	if (event_id >= PERF_COUNT_SW_MAX)
9753 		return -ENOENT;
9754 
9755 	if (!event->parent) {
9756 		int err;
9757 
9758 		err = swevent_hlist_get();
9759 		if (err)
9760 			return err;
9761 
9762 		static_key_slow_inc(&perf_swevent_enabled[event_id]);
9763 		event->destroy = sw_perf_event_destroy;
9764 	}
9765 
9766 	return 0;
9767 }
9768 
9769 static struct pmu perf_swevent = {
9770 	.task_ctx_nr	= perf_sw_context,
9771 
9772 	.capabilities	= PERF_PMU_CAP_NO_NMI,
9773 
9774 	.event_init	= perf_swevent_init,
9775 	.add		= perf_swevent_add,
9776 	.del		= perf_swevent_del,
9777 	.start		= perf_swevent_start,
9778 	.stop		= perf_swevent_stop,
9779 	.read		= perf_swevent_read,
9780 };
9781 
9782 #ifdef CONFIG_EVENT_TRACING
9783 
9784 static int perf_tp_filter_match(struct perf_event *event,
9785 				struct perf_sample_data *data)
9786 {
9787 	void *record = data->raw->frag.data;
9788 
9789 	/* only top level events have filters set */
9790 	if (event->parent)
9791 		event = event->parent;
9792 
9793 	if (likely(!event->filter) || filter_match_preds(event->filter, record))
9794 		return 1;
9795 	return 0;
9796 }
9797 
9798 static int perf_tp_event_match(struct perf_event *event,
9799 				struct perf_sample_data *data,
9800 				struct pt_regs *regs)
9801 {
9802 	if (event->hw.state & PERF_HES_STOPPED)
9803 		return 0;
9804 	/*
9805 	 * If exclude_kernel, only trace user-space tracepoints (uprobes)
9806 	 */
9807 	if (event->attr.exclude_kernel && !user_mode(regs))
9808 		return 0;
9809 
9810 	if (!perf_tp_filter_match(event, data))
9811 		return 0;
9812 
9813 	return 1;
9814 }
9815 
9816 void perf_trace_run_bpf_submit(void *raw_data, int size, int rctx,
9817 			       struct trace_event_call *call, u64 count,
9818 			       struct pt_regs *regs, struct hlist_head *head,
9819 			       struct task_struct *task)
9820 {
9821 	if (bpf_prog_array_valid(call)) {
9822 		*(struct pt_regs **)raw_data = regs;
9823 		if (!trace_call_bpf(call, raw_data) || hlist_empty(head)) {
9824 			perf_swevent_put_recursion_context(rctx);
9825 			return;
9826 		}
9827 	}
9828 	perf_tp_event(call->event.type, count, raw_data, size, regs, head,
9829 		      rctx, task);
9830 }
9831 EXPORT_SYMBOL_GPL(perf_trace_run_bpf_submit);
9832 
9833 void perf_tp_event(u16 event_type, u64 count, void *record, int entry_size,
9834 		   struct pt_regs *regs, struct hlist_head *head, int rctx,
9835 		   struct task_struct *task)
9836 {
9837 	struct perf_sample_data data;
9838 	struct perf_event *event;
9839 
9840 	struct perf_raw_record raw = {
9841 		.frag = {
9842 			.size = entry_size,
9843 			.data = record,
9844 		},
9845 	};
9846 
9847 	perf_sample_data_init(&data, 0, 0);
9848 	data.raw = &raw;
9849 	data.sample_flags |= PERF_SAMPLE_RAW;
9850 
9851 	perf_trace_buf_update(record, event_type);
9852 
9853 	hlist_for_each_entry_rcu(event, head, hlist_entry) {
9854 		if (perf_tp_event_match(event, &data, regs))
9855 			perf_swevent_event(event, count, &data, regs);
9856 	}
9857 
9858 	/*
9859 	 * If we got specified a target task, also iterate its context and
9860 	 * deliver this event there too.
9861 	 */
9862 	if (task && task != current) {
9863 		struct perf_event_context *ctx;
9864 		struct trace_entry *entry = record;
9865 
9866 		rcu_read_lock();
9867 		ctx = rcu_dereference(task->perf_event_ctxp[perf_sw_context]);
9868 		if (!ctx)
9869 			goto unlock;
9870 
9871 		list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
9872 			if (event->cpu != smp_processor_id())
9873 				continue;
9874 			if (event->attr.type != PERF_TYPE_TRACEPOINT)
9875 				continue;
9876 			if (event->attr.config != entry->type)
9877 				continue;
9878 			/* Cannot deliver synchronous signal to other task. */
9879 			if (event->attr.sigtrap)
9880 				continue;
9881 			if (perf_tp_event_match(event, &data, regs))
9882 				perf_swevent_event(event, count, &data, regs);
9883 		}
9884 unlock:
9885 		rcu_read_unlock();
9886 	}
9887 
9888 	perf_swevent_put_recursion_context(rctx);
9889 }
9890 EXPORT_SYMBOL_GPL(perf_tp_event);
9891 
9892 static void tp_perf_event_destroy(struct perf_event *event)
9893 {
9894 	perf_trace_destroy(event);
9895 }
9896 
9897 static int perf_tp_event_init(struct perf_event *event)
9898 {
9899 	int err;
9900 
9901 	if (event->attr.type != PERF_TYPE_TRACEPOINT)
9902 		return -ENOENT;
9903 
9904 	/*
9905 	 * no branch sampling for tracepoint events
9906 	 */
9907 	if (has_branch_stack(event))
9908 		return -EOPNOTSUPP;
9909 
9910 	err = perf_trace_init(event);
9911 	if (err)
9912 		return err;
9913 
9914 	event->destroy = tp_perf_event_destroy;
9915 
9916 	return 0;
9917 }
9918 
9919 static struct pmu perf_tracepoint = {
9920 	.task_ctx_nr	= perf_sw_context,
9921 
9922 	.event_init	= perf_tp_event_init,
9923 	.add		= perf_trace_add,
9924 	.del		= perf_trace_del,
9925 	.start		= perf_swevent_start,
9926 	.stop		= perf_swevent_stop,
9927 	.read		= perf_swevent_read,
9928 };
9929 
9930 #if defined(CONFIG_KPROBE_EVENTS) || defined(CONFIG_UPROBE_EVENTS)
9931 /*
9932  * Flags in config, used by dynamic PMU kprobe and uprobe
9933  * The flags should match following PMU_FORMAT_ATTR().
9934  *
9935  * PERF_PROBE_CONFIG_IS_RETPROBE if set, create kretprobe/uretprobe
9936  *                               if not set, create kprobe/uprobe
9937  *
9938  * The following values specify a reference counter (or semaphore in the
9939  * terminology of tools like dtrace, systemtap, etc.) Userspace Statically
9940  * Defined Tracepoints (USDT). Currently, we use 40 bit for the offset.
9941  *
9942  * PERF_UPROBE_REF_CTR_OFFSET_BITS	# of bits in config as th offset
9943  * PERF_UPROBE_REF_CTR_OFFSET_SHIFT	# of bits to shift left
9944  */
9945 enum perf_probe_config {
9946 	PERF_PROBE_CONFIG_IS_RETPROBE = 1U << 0,  /* [k,u]retprobe */
9947 	PERF_UPROBE_REF_CTR_OFFSET_BITS = 32,
9948 	PERF_UPROBE_REF_CTR_OFFSET_SHIFT = 64 - PERF_UPROBE_REF_CTR_OFFSET_BITS,
9949 };
9950 
9951 PMU_FORMAT_ATTR(retprobe, "config:0");
9952 #endif
9953 
9954 #ifdef CONFIG_KPROBE_EVENTS
9955 static struct attribute *kprobe_attrs[] = {
9956 	&format_attr_retprobe.attr,
9957 	NULL,
9958 };
9959 
9960 static struct attribute_group kprobe_format_group = {
9961 	.name = "format",
9962 	.attrs = kprobe_attrs,
9963 };
9964 
9965 static const struct attribute_group *kprobe_attr_groups[] = {
9966 	&kprobe_format_group,
9967 	NULL,
9968 };
9969 
9970 static int perf_kprobe_event_init(struct perf_event *event);
9971 static struct pmu perf_kprobe = {
9972 	.task_ctx_nr	= perf_sw_context,
9973 	.event_init	= perf_kprobe_event_init,
9974 	.add		= perf_trace_add,
9975 	.del		= perf_trace_del,
9976 	.start		= perf_swevent_start,
9977 	.stop		= perf_swevent_stop,
9978 	.read		= perf_swevent_read,
9979 	.attr_groups	= kprobe_attr_groups,
9980 };
9981 
9982 static int perf_kprobe_event_init(struct perf_event *event)
9983 {
9984 	int err;
9985 	bool is_retprobe;
9986 
9987 	if (event->attr.type != perf_kprobe.type)
9988 		return -ENOENT;
9989 
9990 	if (!perfmon_capable())
9991 		return -EACCES;
9992 
9993 	/*
9994 	 * no branch sampling for probe events
9995 	 */
9996 	if (has_branch_stack(event))
9997 		return -EOPNOTSUPP;
9998 
9999 	is_retprobe = event->attr.config & PERF_PROBE_CONFIG_IS_RETPROBE;
10000 	err = perf_kprobe_init(event, is_retprobe);
10001 	if (err)
10002 		return err;
10003 
10004 	event->destroy = perf_kprobe_destroy;
10005 
10006 	return 0;
10007 }
10008 #endif /* CONFIG_KPROBE_EVENTS */
10009 
10010 #ifdef CONFIG_UPROBE_EVENTS
10011 PMU_FORMAT_ATTR(ref_ctr_offset, "config:32-63");
10012 
10013 static struct attribute *uprobe_attrs[] = {
10014 	&format_attr_retprobe.attr,
10015 	&format_attr_ref_ctr_offset.attr,
10016 	NULL,
10017 };
10018 
10019 static struct attribute_group uprobe_format_group = {
10020 	.name = "format",
10021 	.attrs = uprobe_attrs,
10022 };
10023 
10024 static const struct attribute_group *uprobe_attr_groups[] = {
10025 	&uprobe_format_group,
10026 	NULL,
10027 };
10028 
10029 static int perf_uprobe_event_init(struct perf_event *event);
10030 static struct pmu perf_uprobe = {
10031 	.task_ctx_nr	= perf_sw_context,
10032 	.event_init	= perf_uprobe_event_init,
10033 	.add		= perf_trace_add,
10034 	.del		= perf_trace_del,
10035 	.start		= perf_swevent_start,
10036 	.stop		= perf_swevent_stop,
10037 	.read		= perf_swevent_read,
10038 	.attr_groups	= uprobe_attr_groups,
10039 };
10040 
10041 static int perf_uprobe_event_init(struct perf_event *event)
10042 {
10043 	int err;
10044 	unsigned long ref_ctr_offset;
10045 	bool is_retprobe;
10046 
10047 	if (event->attr.type != perf_uprobe.type)
10048 		return -ENOENT;
10049 
10050 	if (!perfmon_capable())
10051 		return -EACCES;
10052 
10053 	/*
10054 	 * no branch sampling for probe events
10055 	 */
10056 	if (has_branch_stack(event))
10057 		return -EOPNOTSUPP;
10058 
10059 	is_retprobe = event->attr.config & PERF_PROBE_CONFIG_IS_RETPROBE;
10060 	ref_ctr_offset = event->attr.config >> PERF_UPROBE_REF_CTR_OFFSET_SHIFT;
10061 	err = perf_uprobe_init(event, ref_ctr_offset, is_retprobe);
10062 	if (err)
10063 		return err;
10064 
10065 	event->destroy = perf_uprobe_destroy;
10066 
10067 	return 0;
10068 }
10069 #endif /* CONFIG_UPROBE_EVENTS */
10070 
10071 static inline void perf_tp_register(void)
10072 {
10073 	perf_pmu_register(&perf_tracepoint, "tracepoint", PERF_TYPE_TRACEPOINT);
10074 #ifdef CONFIG_KPROBE_EVENTS
10075 	perf_pmu_register(&perf_kprobe, "kprobe", -1);
10076 #endif
10077 #ifdef CONFIG_UPROBE_EVENTS
10078 	perf_pmu_register(&perf_uprobe, "uprobe", -1);
10079 #endif
10080 }
10081 
10082 static void perf_event_free_filter(struct perf_event *event)
10083 {
10084 	ftrace_profile_free_filter(event);
10085 }
10086 
10087 #ifdef CONFIG_BPF_SYSCALL
10088 static void bpf_overflow_handler(struct perf_event *event,
10089 				 struct perf_sample_data *data,
10090 				 struct pt_regs *regs)
10091 {
10092 	struct bpf_perf_event_data_kern ctx = {
10093 		.data = data,
10094 		.event = event,
10095 	};
10096 	struct bpf_prog *prog;
10097 	int ret = 0;
10098 
10099 	ctx.regs = perf_arch_bpf_user_pt_regs(regs);
10100 	if (unlikely(__this_cpu_inc_return(bpf_prog_active) != 1))
10101 		goto out;
10102 	rcu_read_lock();
10103 	prog = READ_ONCE(event->prog);
10104 	if (prog) {
10105 		if (prog->call_get_stack &&
10106 		    (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) &&
10107 		    !(data->sample_flags & PERF_SAMPLE_CALLCHAIN)) {
10108 			data->callchain = perf_callchain(event, regs);
10109 			data->sample_flags |= PERF_SAMPLE_CALLCHAIN;
10110 		}
10111 
10112 		ret = bpf_prog_run(prog, &ctx);
10113 	}
10114 	rcu_read_unlock();
10115 out:
10116 	__this_cpu_dec(bpf_prog_active);
10117 	if (!ret)
10118 		return;
10119 
10120 	event->orig_overflow_handler(event, data, regs);
10121 }
10122 
10123 static int perf_event_set_bpf_handler(struct perf_event *event,
10124 				      struct bpf_prog *prog,
10125 				      u64 bpf_cookie)
10126 {
10127 	if (event->overflow_handler_context)
10128 		/* hw breakpoint or kernel counter */
10129 		return -EINVAL;
10130 
10131 	if (event->prog)
10132 		return -EEXIST;
10133 
10134 	if (prog->type != BPF_PROG_TYPE_PERF_EVENT)
10135 		return -EINVAL;
10136 
10137 	if (event->attr.precise_ip &&
10138 	    prog->call_get_stack &&
10139 	    (!(event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) ||
10140 	     event->attr.exclude_callchain_kernel ||
10141 	     event->attr.exclude_callchain_user)) {
10142 		/*
10143 		 * On perf_event with precise_ip, calling bpf_get_stack()
10144 		 * may trigger unwinder warnings and occasional crashes.
10145 		 * bpf_get_[stack|stackid] works around this issue by using
10146 		 * callchain attached to perf_sample_data. If the
10147 		 * perf_event does not full (kernel and user) callchain
10148 		 * attached to perf_sample_data, do not allow attaching BPF
10149 		 * program that calls bpf_get_[stack|stackid].
10150 		 */
10151 		return -EPROTO;
10152 	}
10153 
10154 	event->prog = prog;
10155 	event->bpf_cookie = bpf_cookie;
10156 	event->orig_overflow_handler = READ_ONCE(event->overflow_handler);
10157 	WRITE_ONCE(event->overflow_handler, bpf_overflow_handler);
10158 	return 0;
10159 }
10160 
10161 static void perf_event_free_bpf_handler(struct perf_event *event)
10162 {
10163 	struct bpf_prog *prog = event->prog;
10164 
10165 	if (!prog)
10166 		return;
10167 
10168 	WRITE_ONCE(event->overflow_handler, event->orig_overflow_handler);
10169 	event->prog = NULL;
10170 	bpf_prog_put(prog);
10171 }
10172 #else
10173 static int perf_event_set_bpf_handler(struct perf_event *event,
10174 				      struct bpf_prog *prog,
10175 				      u64 bpf_cookie)
10176 {
10177 	return -EOPNOTSUPP;
10178 }
10179 static void perf_event_free_bpf_handler(struct perf_event *event)
10180 {
10181 }
10182 #endif
10183 
10184 /*
10185  * returns true if the event is a tracepoint, or a kprobe/upprobe created
10186  * with perf_event_open()
10187  */
10188 static inline bool perf_event_is_tracing(struct perf_event *event)
10189 {
10190 	if (event->pmu == &perf_tracepoint)
10191 		return true;
10192 #ifdef CONFIG_KPROBE_EVENTS
10193 	if (event->pmu == &perf_kprobe)
10194 		return true;
10195 #endif
10196 #ifdef CONFIG_UPROBE_EVENTS
10197 	if (event->pmu == &perf_uprobe)
10198 		return true;
10199 #endif
10200 	return false;
10201 }
10202 
10203 int perf_event_set_bpf_prog(struct perf_event *event, struct bpf_prog *prog,
10204 			    u64 bpf_cookie)
10205 {
10206 	bool is_kprobe, is_uprobe, is_tracepoint, is_syscall_tp;
10207 
10208 	if (!perf_event_is_tracing(event))
10209 		return perf_event_set_bpf_handler(event, prog, bpf_cookie);
10210 
10211 	is_kprobe = event->tp_event->flags & TRACE_EVENT_FL_KPROBE;
10212 	is_uprobe = event->tp_event->flags & TRACE_EVENT_FL_UPROBE;
10213 	is_tracepoint = event->tp_event->flags & TRACE_EVENT_FL_TRACEPOINT;
10214 	is_syscall_tp = is_syscall_trace_event(event->tp_event);
10215 	if (!is_kprobe && !is_uprobe && !is_tracepoint && !is_syscall_tp)
10216 		/* bpf programs can only be attached to u/kprobe or tracepoint */
10217 		return -EINVAL;
10218 
10219 	if (((is_kprobe || is_uprobe) && prog->type != BPF_PROG_TYPE_KPROBE) ||
10220 	    (is_tracepoint && prog->type != BPF_PROG_TYPE_TRACEPOINT) ||
10221 	    (is_syscall_tp && prog->type != BPF_PROG_TYPE_TRACEPOINT))
10222 		return -EINVAL;
10223 
10224 	if (prog->type == BPF_PROG_TYPE_KPROBE && prog->aux->sleepable && !is_uprobe)
10225 		/* only uprobe programs are allowed to be sleepable */
10226 		return -EINVAL;
10227 
10228 	/* Kprobe override only works for kprobes, not uprobes. */
10229 	if (prog->kprobe_override && !is_kprobe)
10230 		return -EINVAL;
10231 
10232 	if (is_tracepoint || is_syscall_tp) {
10233 		int off = trace_event_get_offsets(event->tp_event);
10234 
10235 		if (prog->aux->max_ctx_offset > off)
10236 			return -EACCES;
10237 	}
10238 
10239 	return perf_event_attach_bpf_prog(event, prog, bpf_cookie);
10240 }
10241 
10242 void perf_event_free_bpf_prog(struct perf_event *event)
10243 {
10244 	if (!perf_event_is_tracing(event)) {
10245 		perf_event_free_bpf_handler(event);
10246 		return;
10247 	}
10248 	perf_event_detach_bpf_prog(event);
10249 }
10250 
10251 #else
10252 
10253 static inline void perf_tp_register(void)
10254 {
10255 }
10256 
10257 static void perf_event_free_filter(struct perf_event *event)
10258 {
10259 }
10260 
10261 int perf_event_set_bpf_prog(struct perf_event *event, struct bpf_prog *prog,
10262 			    u64 bpf_cookie)
10263 {
10264 	return -ENOENT;
10265 }
10266 
10267 void perf_event_free_bpf_prog(struct perf_event *event)
10268 {
10269 }
10270 #endif /* CONFIG_EVENT_TRACING */
10271 
10272 #ifdef CONFIG_HAVE_HW_BREAKPOINT
10273 void perf_bp_event(struct perf_event *bp, void *data)
10274 {
10275 	struct perf_sample_data sample;
10276 	struct pt_regs *regs = data;
10277 
10278 	perf_sample_data_init(&sample, bp->attr.bp_addr, 0);
10279 
10280 	if (!bp->hw.state && !perf_exclude_event(bp, regs))
10281 		perf_swevent_event(bp, 1, &sample, regs);
10282 }
10283 #endif
10284 
10285 /*
10286  * Allocate a new address filter
10287  */
10288 static struct perf_addr_filter *
10289 perf_addr_filter_new(struct perf_event *event, struct list_head *filters)
10290 {
10291 	int node = cpu_to_node(event->cpu == -1 ? 0 : event->cpu);
10292 	struct perf_addr_filter *filter;
10293 
10294 	filter = kzalloc_node(sizeof(*filter), GFP_KERNEL, node);
10295 	if (!filter)
10296 		return NULL;
10297 
10298 	INIT_LIST_HEAD(&filter->entry);
10299 	list_add_tail(&filter->entry, filters);
10300 
10301 	return filter;
10302 }
10303 
10304 static void free_filters_list(struct list_head *filters)
10305 {
10306 	struct perf_addr_filter *filter, *iter;
10307 
10308 	list_for_each_entry_safe(filter, iter, filters, entry) {
10309 		path_put(&filter->path);
10310 		list_del(&filter->entry);
10311 		kfree(filter);
10312 	}
10313 }
10314 
10315 /*
10316  * Free existing address filters and optionally install new ones
10317  */
10318 static void perf_addr_filters_splice(struct perf_event *event,
10319 				     struct list_head *head)
10320 {
10321 	unsigned long flags;
10322 	LIST_HEAD(list);
10323 
10324 	if (!has_addr_filter(event))
10325 		return;
10326 
10327 	/* don't bother with children, they don't have their own filters */
10328 	if (event->parent)
10329 		return;
10330 
10331 	raw_spin_lock_irqsave(&event->addr_filters.lock, flags);
10332 
10333 	list_splice_init(&event->addr_filters.list, &list);
10334 	if (head)
10335 		list_splice(head, &event->addr_filters.list);
10336 
10337 	raw_spin_unlock_irqrestore(&event->addr_filters.lock, flags);
10338 
10339 	free_filters_list(&list);
10340 }
10341 
10342 /*
10343  * Scan through mm's vmas and see if one of them matches the
10344  * @filter; if so, adjust filter's address range.
10345  * Called with mm::mmap_lock down for reading.
10346  */
10347 static void perf_addr_filter_apply(struct perf_addr_filter *filter,
10348 				   struct mm_struct *mm,
10349 				   struct perf_addr_filter_range *fr)
10350 {
10351 	struct vm_area_struct *vma;
10352 	VMA_ITERATOR(vmi, mm, 0);
10353 
10354 	for_each_vma(vmi, vma) {
10355 		if (!vma->vm_file)
10356 			continue;
10357 
10358 		if (perf_addr_filter_vma_adjust(filter, vma, fr))
10359 			return;
10360 	}
10361 }
10362 
10363 /*
10364  * Update event's address range filters based on the
10365  * task's existing mappings, if any.
10366  */
10367 static void perf_event_addr_filters_apply(struct perf_event *event)
10368 {
10369 	struct perf_addr_filters_head *ifh = perf_event_addr_filters(event);
10370 	struct task_struct *task = READ_ONCE(event->ctx->task);
10371 	struct perf_addr_filter *filter;
10372 	struct mm_struct *mm = NULL;
10373 	unsigned int count = 0;
10374 	unsigned long flags;
10375 
10376 	/*
10377 	 * We may observe TASK_TOMBSTONE, which means that the event tear-down
10378 	 * will stop on the parent's child_mutex that our caller is also holding
10379 	 */
10380 	if (task == TASK_TOMBSTONE)
10381 		return;
10382 
10383 	if (ifh->nr_file_filters) {
10384 		mm = get_task_mm(task);
10385 		if (!mm)
10386 			goto restart;
10387 
10388 		mmap_read_lock(mm);
10389 	}
10390 
10391 	raw_spin_lock_irqsave(&ifh->lock, flags);
10392 	list_for_each_entry(filter, &ifh->list, entry) {
10393 		if (filter->path.dentry) {
10394 			/*
10395 			 * Adjust base offset if the filter is associated to a
10396 			 * binary that needs to be mapped:
10397 			 */
10398 			event->addr_filter_ranges[count].start = 0;
10399 			event->addr_filter_ranges[count].size = 0;
10400 
10401 			perf_addr_filter_apply(filter, mm, &event->addr_filter_ranges[count]);
10402 		} else {
10403 			event->addr_filter_ranges[count].start = filter->offset;
10404 			event->addr_filter_ranges[count].size  = filter->size;
10405 		}
10406 
10407 		count++;
10408 	}
10409 
10410 	event->addr_filters_gen++;
10411 	raw_spin_unlock_irqrestore(&ifh->lock, flags);
10412 
10413 	if (ifh->nr_file_filters) {
10414 		mmap_read_unlock(mm);
10415 
10416 		mmput(mm);
10417 	}
10418 
10419 restart:
10420 	perf_event_stop(event, 1);
10421 }
10422 
10423 /*
10424  * Address range filtering: limiting the data to certain
10425  * instruction address ranges. Filters are ioctl()ed to us from
10426  * userspace as ascii strings.
10427  *
10428  * Filter string format:
10429  *
10430  * ACTION RANGE_SPEC
10431  * where ACTION is one of the
10432  *  * "filter": limit the trace to this region
10433  *  * "start": start tracing from this address
10434  *  * "stop": stop tracing at this address/region;
10435  * RANGE_SPEC is
10436  *  * for kernel addresses: <start address>[/<size>]
10437  *  * for object files:     <start address>[/<size>]@</path/to/object/file>
10438  *
10439  * if <size> is not specified or is zero, the range is treated as a single
10440  * address; not valid for ACTION=="filter".
10441  */
10442 enum {
10443 	IF_ACT_NONE = -1,
10444 	IF_ACT_FILTER,
10445 	IF_ACT_START,
10446 	IF_ACT_STOP,
10447 	IF_SRC_FILE,
10448 	IF_SRC_KERNEL,
10449 	IF_SRC_FILEADDR,
10450 	IF_SRC_KERNELADDR,
10451 };
10452 
10453 enum {
10454 	IF_STATE_ACTION = 0,
10455 	IF_STATE_SOURCE,
10456 	IF_STATE_END,
10457 };
10458 
10459 static const match_table_t if_tokens = {
10460 	{ IF_ACT_FILTER,	"filter" },
10461 	{ IF_ACT_START,		"start" },
10462 	{ IF_ACT_STOP,		"stop" },
10463 	{ IF_SRC_FILE,		"%u/%u@%s" },
10464 	{ IF_SRC_KERNEL,	"%u/%u" },
10465 	{ IF_SRC_FILEADDR,	"%u@%s" },
10466 	{ IF_SRC_KERNELADDR,	"%u" },
10467 	{ IF_ACT_NONE,		NULL },
10468 };
10469 
10470 /*
10471  * Address filter string parser
10472  */
10473 static int
10474 perf_event_parse_addr_filter(struct perf_event *event, char *fstr,
10475 			     struct list_head *filters)
10476 {
10477 	struct perf_addr_filter *filter = NULL;
10478 	char *start, *orig, *filename = NULL;
10479 	substring_t args[MAX_OPT_ARGS];
10480 	int state = IF_STATE_ACTION, token;
10481 	unsigned int kernel = 0;
10482 	int ret = -EINVAL;
10483 
10484 	orig = fstr = kstrdup(fstr, GFP_KERNEL);
10485 	if (!fstr)
10486 		return -ENOMEM;
10487 
10488 	while ((start = strsep(&fstr, " ,\n")) != NULL) {
10489 		static const enum perf_addr_filter_action_t actions[] = {
10490 			[IF_ACT_FILTER]	= PERF_ADDR_FILTER_ACTION_FILTER,
10491 			[IF_ACT_START]	= PERF_ADDR_FILTER_ACTION_START,
10492 			[IF_ACT_STOP]	= PERF_ADDR_FILTER_ACTION_STOP,
10493 		};
10494 		ret = -EINVAL;
10495 
10496 		if (!*start)
10497 			continue;
10498 
10499 		/* filter definition begins */
10500 		if (state == IF_STATE_ACTION) {
10501 			filter = perf_addr_filter_new(event, filters);
10502 			if (!filter)
10503 				goto fail;
10504 		}
10505 
10506 		token = match_token(start, if_tokens, args);
10507 		switch (token) {
10508 		case IF_ACT_FILTER:
10509 		case IF_ACT_START:
10510 		case IF_ACT_STOP:
10511 			if (state != IF_STATE_ACTION)
10512 				goto fail;
10513 
10514 			filter->action = actions[token];
10515 			state = IF_STATE_SOURCE;
10516 			break;
10517 
10518 		case IF_SRC_KERNELADDR:
10519 		case IF_SRC_KERNEL:
10520 			kernel = 1;
10521 			fallthrough;
10522 
10523 		case IF_SRC_FILEADDR:
10524 		case IF_SRC_FILE:
10525 			if (state != IF_STATE_SOURCE)
10526 				goto fail;
10527 
10528 			*args[0].to = 0;
10529 			ret = kstrtoul(args[0].from, 0, &filter->offset);
10530 			if (ret)
10531 				goto fail;
10532 
10533 			if (token == IF_SRC_KERNEL || token == IF_SRC_FILE) {
10534 				*args[1].to = 0;
10535 				ret = kstrtoul(args[1].from, 0, &filter->size);
10536 				if (ret)
10537 					goto fail;
10538 			}
10539 
10540 			if (token == IF_SRC_FILE || token == IF_SRC_FILEADDR) {
10541 				int fpos = token == IF_SRC_FILE ? 2 : 1;
10542 
10543 				kfree(filename);
10544 				filename = match_strdup(&args[fpos]);
10545 				if (!filename) {
10546 					ret = -ENOMEM;
10547 					goto fail;
10548 				}
10549 			}
10550 
10551 			state = IF_STATE_END;
10552 			break;
10553 
10554 		default:
10555 			goto fail;
10556 		}
10557 
10558 		/*
10559 		 * Filter definition is fully parsed, validate and install it.
10560 		 * Make sure that it doesn't contradict itself or the event's
10561 		 * attribute.
10562 		 */
10563 		if (state == IF_STATE_END) {
10564 			ret = -EINVAL;
10565 
10566 			/*
10567 			 * ACTION "filter" must have a non-zero length region
10568 			 * specified.
10569 			 */
10570 			if (filter->action == PERF_ADDR_FILTER_ACTION_FILTER &&
10571 			    !filter->size)
10572 				goto fail;
10573 
10574 			if (!kernel) {
10575 				if (!filename)
10576 					goto fail;
10577 
10578 				/*
10579 				 * For now, we only support file-based filters
10580 				 * in per-task events; doing so for CPU-wide
10581 				 * events requires additional context switching
10582 				 * trickery, since same object code will be
10583 				 * mapped at different virtual addresses in
10584 				 * different processes.
10585 				 */
10586 				ret = -EOPNOTSUPP;
10587 				if (!event->ctx->task)
10588 					goto fail;
10589 
10590 				/* look up the path and grab its inode */
10591 				ret = kern_path(filename, LOOKUP_FOLLOW,
10592 						&filter->path);
10593 				if (ret)
10594 					goto fail;
10595 
10596 				ret = -EINVAL;
10597 				if (!filter->path.dentry ||
10598 				    !S_ISREG(d_inode(filter->path.dentry)
10599 					     ->i_mode))
10600 					goto fail;
10601 
10602 				event->addr_filters.nr_file_filters++;
10603 			}
10604 
10605 			/* ready to consume more filters */
10606 			kfree(filename);
10607 			filename = NULL;
10608 			state = IF_STATE_ACTION;
10609 			filter = NULL;
10610 			kernel = 0;
10611 		}
10612 	}
10613 
10614 	if (state != IF_STATE_ACTION)
10615 		goto fail;
10616 
10617 	kfree(filename);
10618 	kfree(orig);
10619 
10620 	return 0;
10621 
10622 fail:
10623 	kfree(filename);
10624 	free_filters_list(filters);
10625 	kfree(orig);
10626 
10627 	return ret;
10628 }
10629 
10630 static int
10631 perf_event_set_addr_filter(struct perf_event *event, char *filter_str)
10632 {
10633 	LIST_HEAD(filters);
10634 	int ret;
10635 
10636 	/*
10637 	 * Since this is called in perf_ioctl() path, we're already holding
10638 	 * ctx::mutex.
10639 	 */
10640 	lockdep_assert_held(&event->ctx->mutex);
10641 
10642 	if (WARN_ON_ONCE(event->parent))
10643 		return -EINVAL;
10644 
10645 	ret = perf_event_parse_addr_filter(event, filter_str, &filters);
10646 	if (ret)
10647 		goto fail_clear_files;
10648 
10649 	ret = event->pmu->addr_filters_validate(&filters);
10650 	if (ret)
10651 		goto fail_free_filters;
10652 
10653 	/* remove existing filters, if any */
10654 	perf_addr_filters_splice(event, &filters);
10655 
10656 	/* install new filters */
10657 	perf_event_for_each_child(event, perf_event_addr_filters_apply);
10658 
10659 	return ret;
10660 
10661 fail_free_filters:
10662 	free_filters_list(&filters);
10663 
10664 fail_clear_files:
10665 	event->addr_filters.nr_file_filters = 0;
10666 
10667 	return ret;
10668 }
10669 
10670 static int perf_event_set_filter(struct perf_event *event, void __user *arg)
10671 {
10672 	int ret = -EINVAL;
10673 	char *filter_str;
10674 
10675 	filter_str = strndup_user(arg, PAGE_SIZE);
10676 	if (IS_ERR(filter_str))
10677 		return PTR_ERR(filter_str);
10678 
10679 #ifdef CONFIG_EVENT_TRACING
10680 	if (perf_event_is_tracing(event)) {
10681 		struct perf_event_context *ctx = event->ctx;
10682 
10683 		/*
10684 		 * Beware, here be dragons!!
10685 		 *
10686 		 * the tracepoint muck will deadlock against ctx->mutex, but
10687 		 * the tracepoint stuff does not actually need it. So
10688 		 * temporarily drop ctx->mutex. As per perf_event_ctx_lock() we
10689 		 * already have a reference on ctx.
10690 		 *
10691 		 * This can result in event getting moved to a different ctx,
10692 		 * but that does not affect the tracepoint state.
10693 		 */
10694 		mutex_unlock(&ctx->mutex);
10695 		ret = ftrace_profile_set_filter(event, event->attr.config, filter_str);
10696 		mutex_lock(&ctx->mutex);
10697 	} else
10698 #endif
10699 	if (has_addr_filter(event))
10700 		ret = perf_event_set_addr_filter(event, filter_str);
10701 
10702 	kfree(filter_str);
10703 	return ret;
10704 }
10705 
10706 /*
10707  * hrtimer based swevent callback
10708  */
10709 
10710 static enum hrtimer_restart perf_swevent_hrtimer(struct hrtimer *hrtimer)
10711 {
10712 	enum hrtimer_restart ret = HRTIMER_RESTART;
10713 	struct perf_sample_data data;
10714 	struct pt_regs *regs;
10715 	struct perf_event *event;
10716 	u64 period;
10717 
10718 	event = container_of(hrtimer, struct perf_event, hw.hrtimer);
10719 
10720 	if (event->state != PERF_EVENT_STATE_ACTIVE)
10721 		return HRTIMER_NORESTART;
10722 
10723 	event->pmu->read(event);
10724 
10725 	perf_sample_data_init(&data, 0, event->hw.last_period);
10726 	regs = get_irq_regs();
10727 
10728 	if (regs && !perf_exclude_event(event, regs)) {
10729 		if (!(event->attr.exclude_idle && is_idle_task(current)))
10730 			if (__perf_event_overflow(event, 1, &data, regs))
10731 				ret = HRTIMER_NORESTART;
10732 	}
10733 
10734 	period = max_t(u64, 10000, event->hw.sample_period);
10735 	hrtimer_forward_now(hrtimer, ns_to_ktime(period));
10736 
10737 	return ret;
10738 }
10739 
10740 static void perf_swevent_start_hrtimer(struct perf_event *event)
10741 {
10742 	struct hw_perf_event *hwc = &event->hw;
10743 	s64 period;
10744 
10745 	if (!is_sampling_event(event))
10746 		return;
10747 
10748 	period = local64_read(&hwc->period_left);
10749 	if (period) {
10750 		if (period < 0)
10751 			period = 10000;
10752 
10753 		local64_set(&hwc->period_left, 0);
10754 	} else {
10755 		period = max_t(u64, 10000, hwc->sample_period);
10756 	}
10757 	hrtimer_start(&hwc->hrtimer, ns_to_ktime(period),
10758 		      HRTIMER_MODE_REL_PINNED_HARD);
10759 }
10760 
10761 static void perf_swevent_cancel_hrtimer(struct perf_event *event)
10762 {
10763 	struct hw_perf_event *hwc = &event->hw;
10764 
10765 	if (is_sampling_event(event)) {
10766 		ktime_t remaining = hrtimer_get_remaining(&hwc->hrtimer);
10767 		local64_set(&hwc->period_left, ktime_to_ns(remaining));
10768 
10769 		hrtimer_cancel(&hwc->hrtimer);
10770 	}
10771 }
10772 
10773 static void perf_swevent_init_hrtimer(struct perf_event *event)
10774 {
10775 	struct hw_perf_event *hwc = &event->hw;
10776 
10777 	if (!is_sampling_event(event))
10778 		return;
10779 
10780 	hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD);
10781 	hwc->hrtimer.function = perf_swevent_hrtimer;
10782 
10783 	/*
10784 	 * Since hrtimers have a fixed rate, we can do a static freq->period
10785 	 * mapping and avoid the whole period adjust feedback stuff.
10786 	 */
10787 	if (event->attr.freq) {
10788 		long freq = event->attr.sample_freq;
10789 
10790 		event->attr.sample_period = NSEC_PER_SEC / freq;
10791 		hwc->sample_period = event->attr.sample_period;
10792 		local64_set(&hwc->period_left, hwc->sample_period);
10793 		hwc->last_period = hwc->sample_period;
10794 		event->attr.freq = 0;
10795 	}
10796 }
10797 
10798 /*
10799  * Software event: cpu wall time clock
10800  */
10801 
10802 static void cpu_clock_event_update(struct perf_event *event)
10803 {
10804 	s64 prev;
10805 	u64 now;
10806 
10807 	now = local_clock();
10808 	prev = local64_xchg(&event->hw.prev_count, now);
10809 	local64_add(now - prev, &event->count);
10810 }
10811 
10812 static void cpu_clock_event_start(struct perf_event *event, int flags)
10813 {
10814 	local64_set(&event->hw.prev_count, local_clock());
10815 	perf_swevent_start_hrtimer(event);
10816 }
10817 
10818 static void cpu_clock_event_stop(struct perf_event *event, int flags)
10819 {
10820 	perf_swevent_cancel_hrtimer(event);
10821 	cpu_clock_event_update(event);
10822 }
10823 
10824 static int cpu_clock_event_add(struct perf_event *event, int flags)
10825 {
10826 	if (flags & PERF_EF_START)
10827 		cpu_clock_event_start(event, flags);
10828 	perf_event_update_userpage(event);
10829 
10830 	return 0;
10831 }
10832 
10833 static void cpu_clock_event_del(struct perf_event *event, int flags)
10834 {
10835 	cpu_clock_event_stop(event, flags);
10836 }
10837 
10838 static void cpu_clock_event_read(struct perf_event *event)
10839 {
10840 	cpu_clock_event_update(event);
10841 }
10842 
10843 static int cpu_clock_event_init(struct perf_event *event)
10844 {
10845 	if (event->attr.type != PERF_TYPE_SOFTWARE)
10846 		return -ENOENT;
10847 
10848 	if (event->attr.config != PERF_COUNT_SW_CPU_CLOCK)
10849 		return -ENOENT;
10850 
10851 	/*
10852 	 * no branch sampling for software events
10853 	 */
10854 	if (has_branch_stack(event))
10855 		return -EOPNOTSUPP;
10856 
10857 	perf_swevent_init_hrtimer(event);
10858 
10859 	return 0;
10860 }
10861 
10862 static struct pmu perf_cpu_clock = {
10863 	.task_ctx_nr	= perf_sw_context,
10864 
10865 	.capabilities	= PERF_PMU_CAP_NO_NMI,
10866 
10867 	.event_init	= cpu_clock_event_init,
10868 	.add		= cpu_clock_event_add,
10869 	.del		= cpu_clock_event_del,
10870 	.start		= cpu_clock_event_start,
10871 	.stop		= cpu_clock_event_stop,
10872 	.read		= cpu_clock_event_read,
10873 };
10874 
10875 /*
10876  * Software event: task time clock
10877  */
10878 
10879 static void task_clock_event_update(struct perf_event *event, u64 now)
10880 {
10881 	u64 prev;
10882 	s64 delta;
10883 
10884 	prev = local64_xchg(&event->hw.prev_count, now);
10885 	delta = now - prev;
10886 	local64_add(delta, &event->count);
10887 }
10888 
10889 static void task_clock_event_start(struct perf_event *event, int flags)
10890 {
10891 	local64_set(&event->hw.prev_count, event->ctx->time);
10892 	perf_swevent_start_hrtimer(event);
10893 }
10894 
10895 static void task_clock_event_stop(struct perf_event *event, int flags)
10896 {
10897 	perf_swevent_cancel_hrtimer(event);
10898 	task_clock_event_update(event, event->ctx->time);
10899 }
10900 
10901 static int task_clock_event_add(struct perf_event *event, int flags)
10902 {
10903 	if (flags & PERF_EF_START)
10904 		task_clock_event_start(event, flags);
10905 	perf_event_update_userpage(event);
10906 
10907 	return 0;
10908 }
10909 
10910 static void task_clock_event_del(struct perf_event *event, int flags)
10911 {
10912 	task_clock_event_stop(event, PERF_EF_UPDATE);
10913 }
10914 
10915 static void task_clock_event_read(struct perf_event *event)
10916 {
10917 	u64 now = perf_clock();
10918 	u64 delta = now - event->ctx->timestamp;
10919 	u64 time = event->ctx->time + delta;
10920 
10921 	task_clock_event_update(event, time);
10922 }
10923 
10924 static int task_clock_event_init(struct perf_event *event)
10925 {
10926 	if (event->attr.type != PERF_TYPE_SOFTWARE)
10927 		return -ENOENT;
10928 
10929 	if (event->attr.config != PERF_COUNT_SW_TASK_CLOCK)
10930 		return -ENOENT;
10931 
10932 	/*
10933 	 * no branch sampling for software events
10934 	 */
10935 	if (has_branch_stack(event))
10936 		return -EOPNOTSUPP;
10937 
10938 	perf_swevent_init_hrtimer(event);
10939 
10940 	return 0;
10941 }
10942 
10943 static struct pmu perf_task_clock = {
10944 	.task_ctx_nr	= perf_sw_context,
10945 
10946 	.capabilities	= PERF_PMU_CAP_NO_NMI,
10947 
10948 	.event_init	= task_clock_event_init,
10949 	.add		= task_clock_event_add,
10950 	.del		= task_clock_event_del,
10951 	.start		= task_clock_event_start,
10952 	.stop		= task_clock_event_stop,
10953 	.read		= task_clock_event_read,
10954 };
10955 
10956 static void perf_pmu_nop_void(struct pmu *pmu)
10957 {
10958 }
10959 
10960 static void perf_pmu_nop_txn(struct pmu *pmu, unsigned int flags)
10961 {
10962 }
10963 
10964 static int perf_pmu_nop_int(struct pmu *pmu)
10965 {
10966 	return 0;
10967 }
10968 
10969 static int perf_event_nop_int(struct perf_event *event, u64 value)
10970 {
10971 	return 0;
10972 }
10973 
10974 static DEFINE_PER_CPU(unsigned int, nop_txn_flags);
10975 
10976 static void perf_pmu_start_txn(struct pmu *pmu, unsigned int flags)
10977 {
10978 	__this_cpu_write(nop_txn_flags, flags);
10979 
10980 	if (flags & ~PERF_PMU_TXN_ADD)
10981 		return;
10982 
10983 	perf_pmu_disable(pmu);
10984 }
10985 
10986 static int perf_pmu_commit_txn(struct pmu *pmu)
10987 {
10988 	unsigned int flags = __this_cpu_read(nop_txn_flags);
10989 
10990 	__this_cpu_write(nop_txn_flags, 0);
10991 
10992 	if (flags & ~PERF_PMU_TXN_ADD)
10993 		return 0;
10994 
10995 	perf_pmu_enable(pmu);
10996 	return 0;
10997 }
10998 
10999 static void perf_pmu_cancel_txn(struct pmu *pmu)
11000 {
11001 	unsigned int flags =  __this_cpu_read(nop_txn_flags);
11002 
11003 	__this_cpu_write(nop_txn_flags, 0);
11004 
11005 	if (flags & ~PERF_PMU_TXN_ADD)
11006 		return;
11007 
11008 	perf_pmu_enable(pmu);
11009 }
11010 
11011 static int perf_event_idx_default(struct perf_event *event)
11012 {
11013 	return 0;
11014 }
11015 
11016 /*
11017  * Ensures all contexts with the same task_ctx_nr have the same
11018  * pmu_cpu_context too.
11019  */
11020 static struct perf_cpu_context __percpu *find_pmu_context(int ctxn)
11021 {
11022 	struct pmu *pmu;
11023 
11024 	if (ctxn < 0)
11025 		return NULL;
11026 
11027 	list_for_each_entry(pmu, &pmus, entry) {
11028 		if (pmu->task_ctx_nr == ctxn)
11029 			return pmu->pmu_cpu_context;
11030 	}
11031 
11032 	return NULL;
11033 }
11034 
11035 static void free_pmu_context(struct pmu *pmu)
11036 {
11037 	/*
11038 	 * Static contexts such as perf_sw_context have a global lifetime
11039 	 * and may be shared between different PMUs. Avoid freeing them
11040 	 * when a single PMU is going away.
11041 	 */
11042 	if (pmu->task_ctx_nr > perf_invalid_context)
11043 		return;
11044 
11045 	free_percpu(pmu->pmu_cpu_context);
11046 }
11047 
11048 /*
11049  * Let userspace know that this PMU supports address range filtering:
11050  */
11051 static ssize_t nr_addr_filters_show(struct device *dev,
11052 				    struct device_attribute *attr,
11053 				    char *page)
11054 {
11055 	struct pmu *pmu = dev_get_drvdata(dev);
11056 
11057 	return scnprintf(page, PAGE_SIZE - 1, "%d\n", pmu->nr_addr_filters);
11058 }
11059 DEVICE_ATTR_RO(nr_addr_filters);
11060 
11061 static struct idr pmu_idr;
11062 
11063 static ssize_t
11064 type_show(struct device *dev, struct device_attribute *attr, char *page)
11065 {
11066 	struct pmu *pmu = dev_get_drvdata(dev);
11067 
11068 	return scnprintf(page, PAGE_SIZE - 1, "%d\n", pmu->type);
11069 }
11070 static DEVICE_ATTR_RO(type);
11071 
11072 static ssize_t
11073 perf_event_mux_interval_ms_show(struct device *dev,
11074 				struct device_attribute *attr,
11075 				char *page)
11076 {
11077 	struct pmu *pmu = dev_get_drvdata(dev);
11078 
11079 	return scnprintf(page, PAGE_SIZE - 1, "%d\n", pmu->hrtimer_interval_ms);
11080 }
11081 
11082 static DEFINE_MUTEX(mux_interval_mutex);
11083 
11084 static ssize_t
11085 perf_event_mux_interval_ms_store(struct device *dev,
11086 				 struct device_attribute *attr,
11087 				 const char *buf, size_t count)
11088 {
11089 	struct pmu *pmu = dev_get_drvdata(dev);
11090 	int timer, cpu, ret;
11091 
11092 	ret = kstrtoint(buf, 0, &timer);
11093 	if (ret)
11094 		return ret;
11095 
11096 	if (timer < 1)
11097 		return -EINVAL;
11098 
11099 	/* same value, noting to do */
11100 	if (timer == pmu->hrtimer_interval_ms)
11101 		return count;
11102 
11103 	mutex_lock(&mux_interval_mutex);
11104 	pmu->hrtimer_interval_ms = timer;
11105 
11106 	/* update all cpuctx for this PMU */
11107 	cpus_read_lock();
11108 	for_each_online_cpu(cpu) {
11109 		struct perf_cpu_context *cpuctx;
11110 		cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
11111 		cpuctx->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * timer);
11112 
11113 		cpu_function_call(cpu,
11114 			(remote_function_f)perf_mux_hrtimer_restart, cpuctx);
11115 	}
11116 	cpus_read_unlock();
11117 	mutex_unlock(&mux_interval_mutex);
11118 
11119 	return count;
11120 }
11121 static DEVICE_ATTR_RW(perf_event_mux_interval_ms);
11122 
11123 static struct attribute *pmu_dev_attrs[] = {
11124 	&dev_attr_type.attr,
11125 	&dev_attr_perf_event_mux_interval_ms.attr,
11126 	NULL,
11127 };
11128 ATTRIBUTE_GROUPS(pmu_dev);
11129 
11130 static int pmu_bus_running;
11131 static struct bus_type pmu_bus = {
11132 	.name		= "event_source",
11133 	.dev_groups	= pmu_dev_groups,
11134 };
11135 
11136 static void pmu_dev_release(struct device *dev)
11137 {
11138 	kfree(dev);
11139 }
11140 
11141 static int pmu_dev_alloc(struct pmu *pmu)
11142 {
11143 	int ret = -ENOMEM;
11144 
11145 	pmu->dev = kzalloc(sizeof(struct device), GFP_KERNEL);
11146 	if (!pmu->dev)
11147 		goto out;
11148 
11149 	pmu->dev->groups = pmu->attr_groups;
11150 	device_initialize(pmu->dev);
11151 	ret = dev_set_name(pmu->dev, "%s", pmu->name);
11152 	if (ret)
11153 		goto free_dev;
11154 
11155 	dev_set_drvdata(pmu->dev, pmu);
11156 	pmu->dev->bus = &pmu_bus;
11157 	pmu->dev->release = pmu_dev_release;
11158 	ret = device_add(pmu->dev);
11159 	if (ret)
11160 		goto free_dev;
11161 
11162 	/* For PMUs with address filters, throw in an extra attribute: */
11163 	if (pmu->nr_addr_filters)
11164 		ret = device_create_file(pmu->dev, &dev_attr_nr_addr_filters);
11165 
11166 	if (ret)
11167 		goto del_dev;
11168 
11169 	if (pmu->attr_update)
11170 		ret = sysfs_update_groups(&pmu->dev->kobj, pmu->attr_update);
11171 
11172 	if (ret)
11173 		goto del_dev;
11174 
11175 out:
11176 	return ret;
11177 
11178 del_dev:
11179 	device_del(pmu->dev);
11180 
11181 free_dev:
11182 	put_device(pmu->dev);
11183 	goto out;
11184 }
11185 
11186 static struct lock_class_key cpuctx_mutex;
11187 static struct lock_class_key cpuctx_lock;
11188 
11189 int perf_pmu_register(struct pmu *pmu, const char *name, int type)
11190 {
11191 	int cpu, ret, max = PERF_TYPE_MAX;
11192 
11193 	mutex_lock(&pmus_lock);
11194 	ret = -ENOMEM;
11195 	pmu->pmu_disable_count = alloc_percpu(int);
11196 	if (!pmu->pmu_disable_count)
11197 		goto unlock;
11198 
11199 	pmu->type = -1;
11200 	if (!name)
11201 		goto skip_type;
11202 	pmu->name = name;
11203 
11204 	if (type != PERF_TYPE_SOFTWARE) {
11205 		if (type >= 0)
11206 			max = type;
11207 
11208 		ret = idr_alloc(&pmu_idr, pmu, max, 0, GFP_KERNEL);
11209 		if (ret < 0)
11210 			goto free_pdc;
11211 
11212 		WARN_ON(type >= 0 && ret != type);
11213 
11214 		type = ret;
11215 	}
11216 	pmu->type = type;
11217 
11218 	if (pmu_bus_running) {
11219 		ret = pmu_dev_alloc(pmu);
11220 		if (ret)
11221 			goto free_idr;
11222 	}
11223 
11224 skip_type:
11225 	if (pmu->task_ctx_nr == perf_hw_context) {
11226 		static int hw_context_taken = 0;
11227 
11228 		/*
11229 		 * Other than systems with heterogeneous CPUs, it never makes
11230 		 * sense for two PMUs to share perf_hw_context. PMUs which are
11231 		 * uncore must use perf_invalid_context.
11232 		 */
11233 		if (WARN_ON_ONCE(hw_context_taken &&
11234 		    !(pmu->capabilities & PERF_PMU_CAP_HETEROGENEOUS_CPUS)))
11235 			pmu->task_ctx_nr = perf_invalid_context;
11236 
11237 		hw_context_taken = 1;
11238 	}
11239 
11240 	pmu->pmu_cpu_context = find_pmu_context(pmu->task_ctx_nr);
11241 	if (pmu->pmu_cpu_context)
11242 		goto got_cpu_context;
11243 
11244 	ret = -ENOMEM;
11245 	pmu->pmu_cpu_context = alloc_percpu(struct perf_cpu_context);
11246 	if (!pmu->pmu_cpu_context)
11247 		goto free_dev;
11248 
11249 	for_each_possible_cpu(cpu) {
11250 		struct perf_cpu_context *cpuctx;
11251 
11252 		cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
11253 		__perf_event_init_context(&cpuctx->ctx);
11254 		lockdep_set_class(&cpuctx->ctx.mutex, &cpuctx_mutex);
11255 		lockdep_set_class(&cpuctx->ctx.lock, &cpuctx_lock);
11256 		cpuctx->ctx.pmu = pmu;
11257 		cpuctx->online = cpumask_test_cpu(cpu, perf_online_mask);
11258 
11259 		__perf_mux_hrtimer_init(cpuctx, cpu);
11260 
11261 		cpuctx->heap_size = ARRAY_SIZE(cpuctx->heap_default);
11262 		cpuctx->heap = cpuctx->heap_default;
11263 	}
11264 
11265 got_cpu_context:
11266 	if (!pmu->start_txn) {
11267 		if (pmu->pmu_enable) {
11268 			/*
11269 			 * If we have pmu_enable/pmu_disable calls, install
11270 			 * transaction stubs that use that to try and batch
11271 			 * hardware accesses.
11272 			 */
11273 			pmu->start_txn  = perf_pmu_start_txn;
11274 			pmu->commit_txn = perf_pmu_commit_txn;
11275 			pmu->cancel_txn = perf_pmu_cancel_txn;
11276 		} else {
11277 			pmu->start_txn  = perf_pmu_nop_txn;
11278 			pmu->commit_txn = perf_pmu_nop_int;
11279 			pmu->cancel_txn = perf_pmu_nop_void;
11280 		}
11281 	}
11282 
11283 	if (!pmu->pmu_enable) {
11284 		pmu->pmu_enable  = perf_pmu_nop_void;
11285 		pmu->pmu_disable = perf_pmu_nop_void;
11286 	}
11287 
11288 	if (!pmu->check_period)
11289 		pmu->check_period = perf_event_nop_int;
11290 
11291 	if (!pmu->event_idx)
11292 		pmu->event_idx = perf_event_idx_default;
11293 
11294 	/*
11295 	 * Ensure the TYPE_SOFTWARE PMUs are at the head of the list,
11296 	 * since these cannot be in the IDR. This way the linear search
11297 	 * is fast, provided a valid software event is provided.
11298 	 */
11299 	if (type == PERF_TYPE_SOFTWARE || !name)
11300 		list_add_rcu(&pmu->entry, &pmus);
11301 	else
11302 		list_add_tail_rcu(&pmu->entry, &pmus);
11303 
11304 	atomic_set(&pmu->exclusive_cnt, 0);
11305 	ret = 0;
11306 unlock:
11307 	mutex_unlock(&pmus_lock);
11308 
11309 	return ret;
11310 
11311 free_dev:
11312 	device_del(pmu->dev);
11313 	put_device(pmu->dev);
11314 
11315 free_idr:
11316 	if (pmu->type != PERF_TYPE_SOFTWARE)
11317 		idr_remove(&pmu_idr, pmu->type);
11318 
11319 free_pdc:
11320 	free_percpu(pmu->pmu_disable_count);
11321 	goto unlock;
11322 }
11323 EXPORT_SYMBOL_GPL(perf_pmu_register);
11324 
11325 void perf_pmu_unregister(struct pmu *pmu)
11326 {
11327 	mutex_lock(&pmus_lock);
11328 	list_del_rcu(&pmu->entry);
11329 
11330 	/*
11331 	 * We dereference the pmu list under both SRCU and regular RCU, so
11332 	 * synchronize against both of those.
11333 	 */
11334 	synchronize_srcu(&pmus_srcu);
11335 	synchronize_rcu();
11336 
11337 	free_percpu(pmu->pmu_disable_count);
11338 	if (pmu->type != PERF_TYPE_SOFTWARE)
11339 		idr_remove(&pmu_idr, pmu->type);
11340 	if (pmu_bus_running) {
11341 		if (pmu->nr_addr_filters)
11342 			device_remove_file(pmu->dev, &dev_attr_nr_addr_filters);
11343 		device_del(pmu->dev);
11344 		put_device(pmu->dev);
11345 	}
11346 	free_pmu_context(pmu);
11347 	mutex_unlock(&pmus_lock);
11348 }
11349 EXPORT_SYMBOL_GPL(perf_pmu_unregister);
11350 
11351 static inline bool has_extended_regs(struct perf_event *event)
11352 {
11353 	return (event->attr.sample_regs_user & PERF_REG_EXTENDED_MASK) ||
11354 	       (event->attr.sample_regs_intr & PERF_REG_EXTENDED_MASK);
11355 }
11356 
11357 static int perf_try_init_event(struct pmu *pmu, struct perf_event *event)
11358 {
11359 	struct perf_event_context *ctx = NULL;
11360 	int ret;
11361 
11362 	if (!try_module_get(pmu->module))
11363 		return -ENODEV;
11364 
11365 	/*
11366 	 * A number of pmu->event_init() methods iterate the sibling_list to,
11367 	 * for example, validate if the group fits on the PMU. Therefore,
11368 	 * if this is a sibling event, acquire the ctx->mutex to protect
11369 	 * the sibling_list.
11370 	 */
11371 	if (event->group_leader != event && pmu->task_ctx_nr != perf_sw_context) {
11372 		/*
11373 		 * This ctx->mutex can nest when we're called through
11374 		 * inheritance. See the perf_event_ctx_lock_nested() comment.
11375 		 */
11376 		ctx = perf_event_ctx_lock_nested(event->group_leader,
11377 						 SINGLE_DEPTH_NESTING);
11378 		BUG_ON(!ctx);
11379 	}
11380 
11381 	event->pmu = pmu;
11382 	ret = pmu->event_init(event);
11383 
11384 	if (ctx)
11385 		perf_event_ctx_unlock(event->group_leader, ctx);
11386 
11387 	if (!ret) {
11388 		if (!(pmu->capabilities & PERF_PMU_CAP_EXTENDED_REGS) &&
11389 		    has_extended_regs(event))
11390 			ret = -EOPNOTSUPP;
11391 
11392 		if (pmu->capabilities & PERF_PMU_CAP_NO_EXCLUDE &&
11393 		    event_has_any_exclude_flag(event))
11394 			ret = -EINVAL;
11395 
11396 		if (ret && event->destroy)
11397 			event->destroy(event);
11398 	}
11399 
11400 	if (ret)
11401 		module_put(pmu->module);
11402 
11403 	return ret;
11404 }
11405 
11406 static struct pmu *perf_init_event(struct perf_event *event)
11407 {
11408 	bool extended_type = false;
11409 	int idx, type, ret;
11410 	struct pmu *pmu;
11411 
11412 	idx = srcu_read_lock(&pmus_srcu);
11413 
11414 	/* Try parent's PMU first: */
11415 	if (event->parent && event->parent->pmu) {
11416 		pmu = event->parent->pmu;
11417 		ret = perf_try_init_event(pmu, event);
11418 		if (!ret)
11419 			goto unlock;
11420 	}
11421 
11422 	/*
11423 	 * PERF_TYPE_HARDWARE and PERF_TYPE_HW_CACHE
11424 	 * are often aliases for PERF_TYPE_RAW.
11425 	 */
11426 	type = event->attr.type;
11427 	if (type == PERF_TYPE_HARDWARE || type == PERF_TYPE_HW_CACHE) {
11428 		type = event->attr.config >> PERF_PMU_TYPE_SHIFT;
11429 		if (!type) {
11430 			type = PERF_TYPE_RAW;
11431 		} else {
11432 			extended_type = true;
11433 			event->attr.config &= PERF_HW_EVENT_MASK;
11434 		}
11435 	}
11436 
11437 again:
11438 	rcu_read_lock();
11439 	pmu = idr_find(&pmu_idr, type);
11440 	rcu_read_unlock();
11441 	if (pmu) {
11442 		if (event->attr.type != type && type != PERF_TYPE_RAW &&
11443 		    !(pmu->capabilities & PERF_PMU_CAP_EXTENDED_HW_TYPE))
11444 			goto fail;
11445 
11446 		ret = perf_try_init_event(pmu, event);
11447 		if (ret == -ENOENT && event->attr.type != type && !extended_type) {
11448 			type = event->attr.type;
11449 			goto again;
11450 		}
11451 
11452 		if (ret)
11453 			pmu = ERR_PTR(ret);
11454 
11455 		goto unlock;
11456 	}
11457 
11458 	list_for_each_entry_rcu(pmu, &pmus, entry, lockdep_is_held(&pmus_srcu)) {
11459 		ret = perf_try_init_event(pmu, event);
11460 		if (!ret)
11461 			goto unlock;
11462 
11463 		if (ret != -ENOENT) {
11464 			pmu = ERR_PTR(ret);
11465 			goto unlock;
11466 		}
11467 	}
11468 fail:
11469 	pmu = ERR_PTR(-ENOENT);
11470 unlock:
11471 	srcu_read_unlock(&pmus_srcu, idx);
11472 
11473 	return pmu;
11474 }
11475 
11476 static void attach_sb_event(struct perf_event *event)
11477 {
11478 	struct pmu_event_list *pel = per_cpu_ptr(&pmu_sb_events, event->cpu);
11479 
11480 	raw_spin_lock(&pel->lock);
11481 	list_add_rcu(&event->sb_list, &pel->list);
11482 	raw_spin_unlock(&pel->lock);
11483 }
11484 
11485 /*
11486  * We keep a list of all !task (and therefore per-cpu) events
11487  * that need to receive side-band records.
11488  *
11489  * This avoids having to scan all the various PMU per-cpu contexts
11490  * looking for them.
11491  */
11492 static void account_pmu_sb_event(struct perf_event *event)
11493 {
11494 	if (is_sb_event(event))
11495 		attach_sb_event(event);
11496 }
11497 
11498 static void account_event_cpu(struct perf_event *event, int cpu)
11499 {
11500 	if (event->parent)
11501 		return;
11502 
11503 	if (is_cgroup_event(event))
11504 		atomic_inc(&per_cpu(perf_cgroup_events, cpu));
11505 }
11506 
11507 /* Freq events need the tick to stay alive (see perf_event_task_tick). */
11508 static void account_freq_event_nohz(void)
11509 {
11510 #ifdef CONFIG_NO_HZ_FULL
11511 	/* Lock so we don't race with concurrent unaccount */
11512 	spin_lock(&nr_freq_lock);
11513 	if (atomic_inc_return(&nr_freq_events) == 1)
11514 		tick_nohz_dep_set(TICK_DEP_BIT_PERF_EVENTS);
11515 	spin_unlock(&nr_freq_lock);
11516 #endif
11517 }
11518 
11519 static void account_freq_event(void)
11520 {
11521 	if (tick_nohz_full_enabled())
11522 		account_freq_event_nohz();
11523 	else
11524 		atomic_inc(&nr_freq_events);
11525 }
11526 
11527 
11528 static void account_event(struct perf_event *event)
11529 {
11530 	bool inc = false;
11531 
11532 	if (event->parent)
11533 		return;
11534 
11535 	if (event->attach_state & (PERF_ATTACH_TASK | PERF_ATTACH_SCHED_CB))
11536 		inc = true;
11537 	if (event->attr.mmap || event->attr.mmap_data)
11538 		atomic_inc(&nr_mmap_events);
11539 	if (event->attr.build_id)
11540 		atomic_inc(&nr_build_id_events);
11541 	if (event->attr.comm)
11542 		atomic_inc(&nr_comm_events);
11543 	if (event->attr.namespaces)
11544 		atomic_inc(&nr_namespaces_events);
11545 	if (event->attr.cgroup)
11546 		atomic_inc(&nr_cgroup_events);
11547 	if (event->attr.task)
11548 		atomic_inc(&nr_task_events);
11549 	if (event->attr.freq)
11550 		account_freq_event();
11551 	if (event->attr.context_switch) {
11552 		atomic_inc(&nr_switch_events);
11553 		inc = true;
11554 	}
11555 	if (has_branch_stack(event))
11556 		inc = true;
11557 	if (is_cgroup_event(event))
11558 		inc = true;
11559 	if (event->attr.ksymbol)
11560 		atomic_inc(&nr_ksymbol_events);
11561 	if (event->attr.bpf_event)
11562 		atomic_inc(&nr_bpf_events);
11563 	if (event->attr.text_poke)
11564 		atomic_inc(&nr_text_poke_events);
11565 
11566 	if (inc) {
11567 		/*
11568 		 * We need the mutex here because static_branch_enable()
11569 		 * must complete *before* the perf_sched_count increment
11570 		 * becomes visible.
11571 		 */
11572 		if (atomic_inc_not_zero(&perf_sched_count))
11573 			goto enabled;
11574 
11575 		mutex_lock(&perf_sched_mutex);
11576 		if (!atomic_read(&perf_sched_count)) {
11577 			static_branch_enable(&perf_sched_events);
11578 			/*
11579 			 * Guarantee that all CPUs observe they key change and
11580 			 * call the perf scheduling hooks before proceeding to
11581 			 * install events that need them.
11582 			 */
11583 			synchronize_rcu();
11584 		}
11585 		/*
11586 		 * Now that we have waited for the sync_sched(), allow further
11587 		 * increments to by-pass the mutex.
11588 		 */
11589 		atomic_inc(&perf_sched_count);
11590 		mutex_unlock(&perf_sched_mutex);
11591 	}
11592 enabled:
11593 
11594 	account_event_cpu(event, event->cpu);
11595 
11596 	account_pmu_sb_event(event);
11597 }
11598 
11599 /*
11600  * Allocate and initialize an event structure
11601  */
11602 static struct perf_event *
11603 perf_event_alloc(struct perf_event_attr *attr, int cpu,
11604 		 struct task_struct *task,
11605 		 struct perf_event *group_leader,
11606 		 struct perf_event *parent_event,
11607 		 perf_overflow_handler_t overflow_handler,
11608 		 void *context, int cgroup_fd)
11609 {
11610 	struct pmu *pmu;
11611 	struct perf_event *event;
11612 	struct hw_perf_event *hwc;
11613 	long err = -EINVAL;
11614 	int node;
11615 
11616 	if ((unsigned)cpu >= nr_cpu_ids) {
11617 		if (!task || cpu != -1)
11618 			return ERR_PTR(-EINVAL);
11619 	}
11620 	if (attr->sigtrap && !task) {
11621 		/* Requires a task: avoid signalling random tasks. */
11622 		return ERR_PTR(-EINVAL);
11623 	}
11624 
11625 	node = (cpu >= 0) ? cpu_to_node(cpu) : -1;
11626 	event = kmem_cache_alloc_node(perf_event_cache, GFP_KERNEL | __GFP_ZERO,
11627 				      node);
11628 	if (!event)
11629 		return ERR_PTR(-ENOMEM);
11630 
11631 	/*
11632 	 * Single events are their own group leaders, with an
11633 	 * empty sibling list:
11634 	 */
11635 	if (!group_leader)
11636 		group_leader = event;
11637 
11638 	mutex_init(&event->child_mutex);
11639 	INIT_LIST_HEAD(&event->child_list);
11640 
11641 	INIT_LIST_HEAD(&event->event_entry);
11642 	INIT_LIST_HEAD(&event->sibling_list);
11643 	INIT_LIST_HEAD(&event->active_list);
11644 	init_event_group(event);
11645 	INIT_LIST_HEAD(&event->rb_entry);
11646 	INIT_LIST_HEAD(&event->active_entry);
11647 	INIT_LIST_HEAD(&event->addr_filters.list);
11648 	INIT_HLIST_NODE(&event->hlist_entry);
11649 
11650 
11651 	init_waitqueue_head(&event->waitq);
11652 	init_irq_work(&event->pending_irq, perf_pending_irq);
11653 	init_task_work(&event->pending_task, perf_pending_task);
11654 
11655 	mutex_init(&event->mmap_mutex);
11656 	raw_spin_lock_init(&event->addr_filters.lock);
11657 
11658 	atomic_long_set(&event->refcount, 1);
11659 	event->cpu		= cpu;
11660 	event->attr		= *attr;
11661 	event->group_leader	= group_leader;
11662 	event->pmu		= NULL;
11663 	event->oncpu		= -1;
11664 
11665 	event->parent		= parent_event;
11666 
11667 	event->ns		= get_pid_ns(task_active_pid_ns(current));
11668 	event->id		= atomic64_inc_return(&perf_event_id);
11669 
11670 	event->state		= PERF_EVENT_STATE_INACTIVE;
11671 
11672 	if (parent_event)
11673 		event->event_caps = parent_event->event_caps;
11674 
11675 	if (task) {
11676 		event->attach_state = PERF_ATTACH_TASK;
11677 		/*
11678 		 * XXX pmu::event_init needs to know what task to account to
11679 		 * and we cannot use the ctx information because we need the
11680 		 * pmu before we get a ctx.
11681 		 */
11682 		event->hw.target = get_task_struct(task);
11683 	}
11684 
11685 	event->clock = &local_clock;
11686 	if (parent_event)
11687 		event->clock = parent_event->clock;
11688 
11689 	if (!overflow_handler && parent_event) {
11690 		overflow_handler = parent_event->overflow_handler;
11691 		context = parent_event->overflow_handler_context;
11692 #if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_EVENT_TRACING)
11693 		if (overflow_handler == bpf_overflow_handler) {
11694 			struct bpf_prog *prog = parent_event->prog;
11695 
11696 			bpf_prog_inc(prog);
11697 			event->prog = prog;
11698 			event->orig_overflow_handler =
11699 				parent_event->orig_overflow_handler;
11700 		}
11701 #endif
11702 	}
11703 
11704 	if (overflow_handler) {
11705 		event->overflow_handler	= overflow_handler;
11706 		event->overflow_handler_context = context;
11707 	} else if (is_write_backward(event)){
11708 		event->overflow_handler = perf_event_output_backward;
11709 		event->overflow_handler_context = NULL;
11710 	} else {
11711 		event->overflow_handler = perf_event_output_forward;
11712 		event->overflow_handler_context = NULL;
11713 	}
11714 
11715 	perf_event__state_init(event);
11716 
11717 	pmu = NULL;
11718 
11719 	hwc = &event->hw;
11720 	hwc->sample_period = attr->sample_period;
11721 	if (attr->freq && attr->sample_freq)
11722 		hwc->sample_period = 1;
11723 	hwc->last_period = hwc->sample_period;
11724 
11725 	local64_set(&hwc->period_left, hwc->sample_period);
11726 
11727 	/*
11728 	 * We currently do not support PERF_SAMPLE_READ on inherited events.
11729 	 * See perf_output_read().
11730 	 */
11731 	if (attr->inherit && (attr->sample_type & PERF_SAMPLE_READ))
11732 		goto err_ns;
11733 
11734 	if (!has_branch_stack(event))
11735 		event->attr.branch_sample_type = 0;
11736 
11737 	pmu = perf_init_event(event);
11738 	if (IS_ERR(pmu)) {
11739 		err = PTR_ERR(pmu);
11740 		goto err_ns;
11741 	}
11742 
11743 	/*
11744 	 * Disallow uncore-cgroup events, they don't make sense as the cgroup will
11745 	 * be different on other CPUs in the uncore mask.
11746 	 */
11747 	if (pmu->task_ctx_nr == perf_invalid_context && cgroup_fd != -1) {
11748 		err = -EINVAL;
11749 		goto err_pmu;
11750 	}
11751 
11752 	if (event->attr.aux_output &&
11753 	    !(pmu->capabilities & PERF_PMU_CAP_AUX_OUTPUT)) {
11754 		err = -EOPNOTSUPP;
11755 		goto err_pmu;
11756 	}
11757 
11758 	if (cgroup_fd != -1) {
11759 		err = perf_cgroup_connect(cgroup_fd, event, attr, group_leader);
11760 		if (err)
11761 			goto err_pmu;
11762 	}
11763 
11764 	err = exclusive_event_init(event);
11765 	if (err)
11766 		goto err_pmu;
11767 
11768 	if (has_addr_filter(event)) {
11769 		event->addr_filter_ranges = kcalloc(pmu->nr_addr_filters,
11770 						    sizeof(struct perf_addr_filter_range),
11771 						    GFP_KERNEL);
11772 		if (!event->addr_filter_ranges) {
11773 			err = -ENOMEM;
11774 			goto err_per_task;
11775 		}
11776 
11777 		/*
11778 		 * Clone the parent's vma offsets: they are valid until exec()
11779 		 * even if the mm is not shared with the parent.
11780 		 */
11781 		if (event->parent) {
11782 			struct perf_addr_filters_head *ifh = perf_event_addr_filters(event);
11783 
11784 			raw_spin_lock_irq(&ifh->lock);
11785 			memcpy(event->addr_filter_ranges,
11786 			       event->parent->addr_filter_ranges,
11787 			       pmu->nr_addr_filters * sizeof(struct perf_addr_filter_range));
11788 			raw_spin_unlock_irq(&ifh->lock);
11789 		}
11790 
11791 		/* force hw sync on the address filters */
11792 		event->addr_filters_gen = 1;
11793 	}
11794 
11795 	if (!event->parent) {
11796 		if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) {
11797 			err = get_callchain_buffers(attr->sample_max_stack);
11798 			if (err)
11799 				goto err_addr_filters;
11800 		}
11801 	}
11802 
11803 	err = security_perf_event_alloc(event);
11804 	if (err)
11805 		goto err_callchain_buffer;
11806 
11807 	/* symmetric to unaccount_event() in _free_event() */
11808 	account_event(event);
11809 
11810 	return event;
11811 
11812 err_callchain_buffer:
11813 	if (!event->parent) {
11814 		if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN)
11815 			put_callchain_buffers();
11816 	}
11817 err_addr_filters:
11818 	kfree(event->addr_filter_ranges);
11819 
11820 err_per_task:
11821 	exclusive_event_destroy(event);
11822 
11823 err_pmu:
11824 	if (is_cgroup_event(event))
11825 		perf_detach_cgroup(event);
11826 	if (event->destroy)
11827 		event->destroy(event);
11828 	module_put(pmu->module);
11829 err_ns:
11830 	if (event->hw.target)
11831 		put_task_struct(event->hw.target);
11832 	call_rcu(&event->rcu_head, free_event_rcu);
11833 
11834 	return ERR_PTR(err);
11835 }
11836 
11837 static int perf_copy_attr(struct perf_event_attr __user *uattr,
11838 			  struct perf_event_attr *attr)
11839 {
11840 	u32 size;
11841 	int ret;
11842 
11843 	/* Zero the full structure, so that a short copy will be nice. */
11844 	memset(attr, 0, sizeof(*attr));
11845 
11846 	ret = get_user(size, &uattr->size);
11847 	if (ret)
11848 		return ret;
11849 
11850 	/* ABI compatibility quirk: */
11851 	if (!size)
11852 		size = PERF_ATTR_SIZE_VER0;
11853 	if (size < PERF_ATTR_SIZE_VER0 || size > PAGE_SIZE)
11854 		goto err_size;
11855 
11856 	ret = copy_struct_from_user(attr, sizeof(*attr), uattr, size);
11857 	if (ret) {
11858 		if (ret == -E2BIG)
11859 			goto err_size;
11860 		return ret;
11861 	}
11862 
11863 	attr->size = size;
11864 
11865 	if (attr->__reserved_1 || attr->__reserved_2 || attr->__reserved_3)
11866 		return -EINVAL;
11867 
11868 	if (attr->sample_type & ~(PERF_SAMPLE_MAX-1))
11869 		return -EINVAL;
11870 
11871 	if (attr->read_format & ~(PERF_FORMAT_MAX-1))
11872 		return -EINVAL;
11873 
11874 	if (attr->sample_type & PERF_SAMPLE_BRANCH_STACK) {
11875 		u64 mask = attr->branch_sample_type;
11876 
11877 		/* only using defined bits */
11878 		if (mask & ~(PERF_SAMPLE_BRANCH_MAX-1))
11879 			return -EINVAL;
11880 
11881 		/* at least one branch bit must be set */
11882 		if (!(mask & ~PERF_SAMPLE_BRANCH_PLM_ALL))
11883 			return -EINVAL;
11884 
11885 		/* propagate priv level, when not set for branch */
11886 		if (!(mask & PERF_SAMPLE_BRANCH_PLM_ALL)) {
11887 
11888 			/* exclude_kernel checked on syscall entry */
11889 			if (!attr->exclude_kernel)
11890 				mask |= PERF_SAMPLE_BRANCH_KERNEL;
11891 
11892 			if (!attr->exclude_user)
11893 				mask |= PERF_SAMPLE_BRANCH_USER;
11894 
11895 			if (!attr->exclude_hv)
11896 				mask |= PERF_SAMPLE_BRANCH_HV;
11897 			/*
11898 			 * adjust user setting (for HW filter setup)
11899 			 */
11900 			attr->branch_sample_type = mask;
11901 		}
11902 		/* privileged levels capture (kernel, hv): check permissions */
11903 		if (mask & PERF_SAMPLE_BRANCH_PERM_PLM) {
11904 			ret = perf_allow_kernel(attr);
11905 			if (ret)
11906 				return ret;
11907 		}
11908 	}
11909 
11910 	if (attr->sample_type & PERF_SAMPLE_REGS_USER) {
11911 		ret = perf_reg_validate(attr->sample_regs_user);
11912 		if (ret)
11913 			return ret;
11914 	}
11915 
11916 	if (attr->sample_type & PERF_SAMPLE_STACK_USER) {
11917 		if (!arch_perf_have_user_stack_dump())
11918 			return -ENOSYS;
11919 
11920 		/*
11921 		 * We have __u32 type for the size, but so far
11922 		 * we can only use __u16 as maximum due to the
11923 		 * __u16 sample size limit.
11924 		 */
11925 		if (attr->sample_stack_user >= USHRT_MAX)
11926 			return -EINVAL;
11927 		else if (!IS_ALIGNED(attr->sample_stack_user, sizeof(u64)))
11928 			return -EINVAL;
11929 	}
11930 
11931 	if (!attr->sample_max_stack)
11932 		attr->sample_max_stack = sysctl_perf_event_max_stack;
11933 
11934 	if (attr->sample_type & PERF_SAMPLE_REGS_INTR)
11935 		ret = perf_reg_validate(attr->sample_regs_intr);
11936 
11937 #ifndef CONFIG_CGROUP_PERF
11938 	if (attr->sample_type & PERF_SAMPLE_CGROUP)
11939 		return -EINVAL;
11940 #endif
11941 	if ((attr->sample_type & PERF_SAMPLE_WEIGHT) &&
11942 	    (attr->sample_type & PERF_SAMPLE_WEIGHT_STRUCT))
11943 		return -EINVAL;
11944 
11945 	if (!attr->inherit && attr->inherit_thread)
11946 		return -EINVAL;
11947 
11948 	if (attr->remove_on_exec && attr->enable_on_exec)
11949 		return -EINVAL;
11950 
11951 	if (attr->sigtrap && !attr->remove_on_exec)
11952 		return -EINVAL;
11953 
11954 out:
11955 	return ret;
11956 
11957 err_size:
11958 	put_user(sizeof(*attr), &uattr->size);
11959 	ret = -E2BIG;
11960 	goto out;
11961 }
11962 
11963 static void mutex_lock_double(struct mutex *a, struct mutex *b)
11964 {
11965 	if (b < a)
11966 		swap(a, b);
11967 
11968 	mutex_lock(a);
11969 	mutex_lock_nested(b, SINGLE_DEPTH_NESTING);
11970 }
11971 
11972 static int
11973 perf_event_set_output(struct perf_event *event, struct perf_event *output_event)
11974 {
11975 	struct perf_buffer *rb = NULL;
11976 	int ret = -EINVAL;
11977 
11978 	if (!output_event) {
11979 		mutex_lock(&event->mmap_mutex);
11980 		goto set;
11981 	}
11982 
11983 	/* don't allow circular references */
11984 	if (event == output_event)
11985 		goto out;
11986 
11987 	/*
11988 	 * Don't allow cross-cpu buffers
11989 	 */
11990 	if (output_event->cpu != event->cpu)
11991 		goto out;
11992 
11993 	/*
11994 	 * If its not a per-cpu rb, it must be the same task.
11995 	 */
11996 	if (output_event->cpu == -1 && output_event->ctx != event->ctx)
11997 		goto out;
11998 
11999 	/*
12000 	 * Mixing clocks in the same buffer is trouble you don't need.
12001 	 */
12002 	if (output_event->clock != event->clock)
12003 		goto out;
12004 
12005 	/*
12006 	 * Either writing ring buffer from beginning or from end.
12007 	 * Mixing is not allowed.
12008 	 */
12009 	if (is_write_backward(output_event) != is_write_backward(event))
12010 		goto out;
12011 
12012 	/*
12013 	 * If both events generate aux data, they must be on the same PMU
12014 	 */
12015 	if (has_aux(event) && has_aux(output_event) &&
12016 	    event->pmu != output_event->pmu)
12017 		goto out;
12018 
12019 	/*
12020 	 * Hold both mmap_mutex to serialize against perf_mmap_close().  Since
12021 	 * output_event is already on rb->event_list, and the list iteration
12022 	 * restarts after every removal, it is guaranteed this new event is
12023 	 * observed *OR* if output_event is already removed, it's guaranteed we
12024 	 * observe !rb->mmap_count.
12025 	 */
12026 	mutex_lock_double(&event->mmap_mutex, &output_event->mmap_mutex);
12027 set:
12028 	/* Can't redirect output if we've got an active mmap() */
12029 	if (atomic_read(&event->mmap_count))
12030 		goto unlock;
12031 
12032 	if (output_event) {
12033 		/* get the rb we want to redirect to */
12034 		rb = ring_buffer_get(output_event);
12035 		if (!rb)
12036 			goto unlock;
12037 
12038 		/* did we race against perf_mmap_close() */
12039 		if (!atomic_read(&rb->mmap_count)) {
12040 			ring_buffer_put(rb);
12041 			goto unlock;
12042 		}
12043 	}
12044 
12045 	ring_buffer_attach(event, rb);
12046 
12047 	ret = 0;
12048 unlock:
12049 	mutex_unlock(&event->mmap_mutex);
12050 	if (output_event)
12051 		mutex_unlock(&output_event->mmap_mutex);
12052 
12053 out:
12054 	return ret;
12055 }
12056 
12057 static int perf_event_set_clock(struct perf_event *event, clockid_t clk_id)
12058 {
12059 	bool nmi_safe = false;
12060 
12061 	switch (clk_id) {
12062 	case CLOCK_MONOTONIC:
12063 		event->clock = &ktime_get_mono_fast_ns;
12064 		nmi_safe = true;
12065 		break;
12066 
12067 	case CLOCK_MONOTONIC_RAW:
12068 		event->clock = &ktime_get_raw_fast_ns;
12069 		nmi_safe = true;
12070 		break;
12071 
12072 	case CLOCK_REALTIME:
12073 		event->clock = &ktime_get_real_ns;
12074 		break;
12075 
12076 	case CLOCK_BOOTTIME:
12077 		event->clock = &ktime_get_boottime_ns;
12078 		break;
12079 
12080 	case CLOCK_TAI:
12081 		event->clock = &ktime_get_clocktai_ns;
12082 		break;
12083 
12084 	default:
12085 		return -EINVAL;
12086 	}
12087 
12088 	if (!nmi_safe && !(event->pmu->capabilities & PERF_PMU_CAP_NO_NMI))
12089 		return -EINVAL;
12090 
12091 	return 0;
12092 }
12093 
12094 /*
12095  * Variation on perf_event_ctx_lock_nested(), except we take two context
12096  * mutexes.
12097  */
12098 static struct perf_event_context *
12099 __perf_event_ctx_lock_double(struct perf_event *group_leader,
12100 			     struct perf_event_context *ctx)
12101 {
12102 	struct perf_event_context *gctx;
12103 
12104 again:
12105 	rcu_read_lock();
12106 	gctx = READ_ONCE(group_leader->ctx);
12107 	if (!refcount_inc_not_zero(&gctx->refcount)) {
12108 		rcu_read_unlock();
12109 		goto again;
12110 	}
12111 	rcu_read_unlock();
12112 
12113 	mutex_lock_double(&gctx->mutex, &ctx->mutex);
12114 
12115 	if (group_leader->ctx != gctx) {
12116 		mutex_unlock(&ctx->mutex);
12117 		mutex_unlock(&gctx->mutex);
12118 		put_ctx(gctx);
12119 		goto again;
12120 	}
12121 
12122 	return gctx;
12123 }
12124 
12125 static bool
12126 perf_check_permission(struct perf_event_attr *attr, struct task_struct *task)
12127 {
12128 	unsigned int ptrace_mode = PTRACE_MODE_READ_REALCREDS;
12129 	bool is_capable = perfmon_capable();
12130 
12131 	if (attr->sigtrap) {
12132 		/*
12133 		 * perf_event_attr::sigtrap sends signals to the other task.
12134 		 * Require the current task to also have CAP_KILL.
12135 		 */
12136 		rcu_read_lock();
12137 		is_capable &= ns_capable(__task_cred(task)->user_ns, CAP_KILL);
12138 		rcu_read_unlock();
12139 
12140 		/*
12141 		 * If the required capabilities aren't available, checks for
12142 		 * ptrace permissions: upgrade to ATTACH, since sending signals
12143 		 * can effectively change the target task.
12144 		 */
12145 		ptrace_mode = PTRACE_MODE_ATTACH_REALCREDS;
12146 	}
12147 
12148 	/*
12149 	 * Preserve ptrace permission check for backwards compatibility. The
12150 	 * ptrace check also includes checks that the current task and other
12151 	 * task have matching uids, and is therefore not done here explicitly.
12152 	 */
12153 	return is_capable || ptrace_may_access(task, ptrace_mode);
12154 }
12155 
12156 /**
12157  * sys_perf_event_open - open a performance event, associate it to a task/cpu
12158  *
12159  * @attr_uptr:	event_id type attributes for monitoring/sampling
12160  * @pid:		target pid
12161  * @cpu:		target cpu
12162  * @group_fd:		group leader event fd
12163  * @flags:		perf event open flags
12164  */
12165 SYSCALL_DEFINE5(perf_event_open,
12166 		struct perf_event_attr __user *, attr_uptr,
12167 		pid_t, pid, int, cpu, int, group_fd, unsigned long, flags)
12168 {
12169 	struct perf_event *group_leader = NULL, *output_event = NULL;
12170 	struct perf_event *event, *sibling;
12171 	struct perf_event_attr attr;
12172 	struct perf_event_context *ctx, *gctx;
12173 	struct file *event_file = NULL;
12174 	struct fd group = {NULL, 0};
12175 	struct task_struct *task = NULL;
12176 	struct pmu *pmu;
12177 	int event_fd;
12178 	int move_group = 0;
12179 	int err;
12180 	int f_flags = O_RDWR;
12181 	int cgroup_fd = -1;
12182 
12183 	/* for future expandability... */
12184 	if (flags & ~PERF_FLAG_ALL)
12185 		return -EINVAL;
12186 
12187 	/* Do we allow access to perf_event_open(2) ? */
12188 	err = security_perf_event_open(&attr, PERF_SECURITY_OPEN);
12189 	if (err)
12190 		return err;
12191 
12192 	err = perf_copy_attr(attr_uptr, &attr);
12193 	if (err)
12194 		return err;
12195 
12196 	if (!attr.exclude_kernel) {
12197 		err = perf_allow_kernel(&attr);
12198 		if (err)
12199 			return err;
12200 	}
12201 
12202 	if (attr.namespaces) {
12203 		if (!perfmon_capable())
12204 			return -EACCES;
12205 	}
12206 
12207 	if (attr.freq) {
12208 		if (attr.sample_freq > sysctl_perf_event_sample_rate)
12209 			return -EINVAL;
12210 	} else {
12211 		if (attr.sample_period & (1ULL << 63))
12212 			return -EINVAL;
12213 	}
12214 
12215 	/* Only privileged users can get physical addresses */
12216 	if ((attr.sample_type & PERF_SAMPLE_PHYS_ADDR)) {
12217 		err = perf_allow_kernel(&attr);
12218 		if (err)
12219 			return err;
12220 	}
12221 
12222 	/* REGS_INTR can leak data, lockdown must prevent this */
12223 	if (attr.sample_type & PERF_SAMPLE_REGS_INTR) {
12224 		err = security_locked_down(LOCKDOWN_PERF);
12225 		if (err)
12226 			return err;
12227 	}
12228 
12229 	/*
12230 	 * In cgroup mode, the pid argument is used to pass the fd
12231 	 * opened to the cgroup directory in cgroupfs. The cpu argument
12232 	 * designates the cpu on which to monitor threads from that
12233 	 * cgroup.
12234 	 */
12235 	if ((flags & PERF_FLAG_PID_CGROUP) && (pid == -1 || cpu == -1))
12236 		return -EINVAL;
12237 
12238 	if (flags & PERF_FLAG_FD_CLOEXEC)
12239 		f_flags |= O_CLOEXEC;
12240 
12241 	event_fd = get_unused_fd_flags(f_flags);
12242 	if (event_fd < 0)
12243 		return event_fd;
12244 
12245 	if (group_fd != -1) {
12246 		err = perf_fget_light(group_fd, &group);
12247 		if (err)
12248 			goto err_fd;
12249 		group_leader = group.file->private_data;
12250 		if (flags & PERF_FLAG_FD_OUTPUT)
12251 			output_event = group_leader;
12252 		if (flags & PERF_FLAG_FD_NO_GROUP)
12253 			group_leader = NULL;
12254 	}
12255 
12256 	if (pid != -1 && !(flags & PERF_FLAG_PID_CGROUP)) {
12257 		task = find_lively_task_by_vpid(pid);
12258 		if (IS_ERR(task)) {
12259 			err = PTR_ERR(task);
12260 			goto err_group_fd;
12261 		}
12262 	}
12263 
12264 	if (task && group_leader &&
12265 	    group_leader->attr.inherit != attr.inherit) {
12266 		err = -EINVAL;
12267 		goto err_task;
12268 	}
12269 
12270 	if (flags & PERF_FLAG_PID_CGROUP)
12271 		cgroup_fd = pid;
12272 
12273 	event = perf_event_alloc(&attr, cpu, task, group_leader, NULL,
12274 				 NULL, NULL, cgroup_fd);
12275 	if (IS_ERR(event)) {
12276 		err = PTR_ERR(event);
12277 		goto err_task;
12278 	}
12279 
12280 	if (is_sampling_event(event)) {
12281 		if (event->pmu->capabilities & PERF_PMU_CAP_NO_INTERRUPT) {
12282 			err = -EOPNOTSUPP;
12283 			goto err_alloc;
12284 		}
12285 	}
12286 
12287 	/*
12288 	 * Special case software events and allow them to be part of
12289 	 * any hardware group.
12290 	 */
12291 	pmu = event->pmu;
12292 
12293 	if (attr.use_clockid) {
12294 		err = perf_event_set_clock(event, attr.clockid);
12295 		if (err)
12296 			goto err_alloc;
12297 	}
12298 
12299 	if (pmu->task_ctx_nr == perf_sw_context)
12300 		event->event_caps |= PERF_EV_CAP_SOFTWARE;
12301 
12302 	if (group_leader) {
12303 		if (is_software_event(event) &&
12304 		    !in_software_context(group_leader)) {
12305 			/*
12306 			 * If the event is a sw event, but the group_leader
12307 			 * is on hw context.
12308 			 *
12309 			 * Allow the addition of software events to hw
12310 			 * groups, this is safe because software events
12311 			 * never fail to schedule.
12312 			 */
12313 			pmu = group_leader->ctx->pmu;
12314 		} else if (!is_software_event(event) &&
12315 			   is_software_event(group_leader) &&
12316 			   (group_leader->group_caps & PERF_EV_CAP_SOFTWARE)) {
12317 			/*
12318 			 * In case the group is a pure software group, and we
12319 			 * try to add a hardware event, move the whole group to
12320 			 * the hardware context.
12321 			 */
12322 			move_group = 1;
12323 		}
12324 	}
12325 
12326 	/*
12327 	 * Get the target context (task or percpu):
12328 	 */
12329 	ctx = find_get_context(pmu, task, event);
12330 	if (IS_ERR(ctx)) {
12331 		err = PTR_ERR(ctx);
12332 		goto err_alloc;
12333 	}
12334 
12335 	/*
12336 	 * Look up the group leader (we will attach this event to it):
12337 	 */
12338 	if (group_leader) {
12339 		err = -EINVAL;
12340 
12341 		/*
12342 		 * Do not allow a recursive hierarchy (this new sibling
12343 		 * becoming part of another group-sibling):
12344 		 */
12345 		if (group_leader->group_leader != group_leader)
12346 			goto err_context;
12347 
12348 		/* All events in a group should have the same clock */
12349 		if (group_leader->clock != event->clock)
12350 			goto err_context;
12351 
12352 		/*
12353 		 * Make sure we're both events for the same CPU;
12354 		 * grouping events for different CPUs is broken; since
12355 		 * you can never concurrently schedule them anyhow.
12356 		 */
12357 		if (group_leader->cpu != event->cpu)
12358 			goto err_context;
12359 
12360 		/*
12361 		 * Make sure we're both on the same task, or both
12362 		 * per-CPU events.
12363 		 */
12364 		if (group_leader->ctx->task != ctx->task)
12365 			goto err_context;
12366 
12367 		/*
12368 		 * Do not allow to attach to a group in a different task
12369 		 * or CPU context. If we're moving SW events, we'll fix
12370 		 * this up later, so allow that.
12371 		 *
12372 		 * Racy, not holding group_leader->ctx->mutex, see comment with
12373 		 * perf_event_ctx_lock().
12374 		 */
12375 		if (!move_group && group_leader->ctx != ctx)
12376 			goto err_context;
12377 
12378 		/*
12379 		 * Only a group leader can be exclusive or pinned
12380 		 */
12381 		if (attr.exclusive || attr.pinned)
12382 			goto err_context;
12383 	}
12384 
12385 	if (output_event) {
12386 		err = perf_event_set_output(event, output_event);
12387 		if (err)
12388 			goto err_context;
12389 	}
12390 
12391 	event_file = anon_inode_getfile("[perf_event]", &perf_fops, event,
12392 					f_flags);
12393 	if (IS_ERR(event_file)) {
12394 		err = PTR_ERR(event_file);
12395 		event_file = NULL;
12396 		goto err_context;
12397 	}
12398 
12399 	if (task) {
12400 		err = down_read_interruptible(&task->signal->exec_update_lock);
12401 		if (err)
12402 			goto err_file;
12403 
12404 		/*
12405 		 * We must hold exec_update_lock across this and any potential
12406 		 * perf_install_in_context() call for this new event to
12407 		 * serialize against exec() altering our credentials (and the
12408 		 * perf_event_exit_task() that could imply).
12409 		 */
12410 		err = -EACCES;
12411 		if (!perf_check_permission(&attr, task))
12412 			goto err_cred;
12413 	}
12414 
12415 	if (move_group) {
12416 		gctx = __perf_event_ctx_lock_double(group_leader, ctx);
12417 
12418 		if (gctx->task == TASK_TOMBSTONE) {
12419 			err = -ESRCH;
12420 			goto err_locked;
12421 		}
12422 
12423 		/*
12424 		 * Check if we raced against another sys_perf_event_open() call
12425 		 * moving the software group underneath us.
12426 		 */
12427 		if (!(group_leader->group_caps & PERF_EV_CAP_SOFTWARE)) {
12428 			/*
12429 			 * If someone moved the group out from under us, check
12430 			 * if this new event wound up on the same ctx, if so
12431 			 * its the regular !move_group case, otherwise fail.
12432 			 */
12433 			if (gctx != ctx) {
12434 				err = -EINVAL;
12435 				goto err_locked;
12436 			} else {
12437 				perf_event_ctx_unlock(group_leader, gctx);
12438 				move_group = 0;
12439 				goto not_move_group;
12440 			}
12441 		}
12442 
12443 		/*
12444 		 * Failure to create exclusive events returns -EBUSY.
12445 		 */
12446 		err = -EBUSY;
12447 		if (!exclusive_event_installable(group_leader, ctx))
12448 			goto err_locked;
12449 
12450 		for_each_sibling_event(sibling, group_leader) {
12451 			if (!exclusive_event_installable(sibling, ctx))
12452 				goto err_locked;
12453 		}
12454 	} else {
12455 		mutex_lock(&ctx->mutex);
12456 
12457 		/*
12458 		 * Now that we hold ctx->lock, (re)validate group_leader->ctx == ctx,
12459 		 * see the group_leader && !move_group test earlier.
12460 		 */
12461 		if (group_leader && group_leader->ctx != ctx) {
12462 			err = -EINVAL;
12463 			goto err_locked;
12464 		}
12465 	}
12466 not_move_group:
12467 
12468 	if (ctx->task == TASK_TOMBSTONE) {
12469 		err = -ESRCH;
12470 		goto err_locked;
12471 	}
12472 
12473 	if (!perf_event_validate_size(event)) {
12474 		err = -E2BIG;
12475 		goto err_locked;
12476 	}
12477 
12478 	if (!task) {
12479 		/*
12480 		 * Check if the @cpu we're creating an event for is online.
12481 		 *
12482 		 * We use the perf_cpu_context::ctx::mutex to serialize against
12483 		 * the hotplug notifiers. See perf_event_{init,exit}_cpu().
12484 		 */
12485 		struct perf_cpu_context *cpuctx =
12486 			container_of(ctx, struct perf_cpu_context, ctx);
12487 
12488 		if (!cpuctx->online) {
12489 			err = -ENODEV;
12490 			goto err_locked;
12491 		}
12492 	}
12493 
12494 	if (perf_need_aux_event(event) && !perf_get_aux_event(event, group_leader)) {
12495 		err = -EINVAL;
12496 		goto err_locked;
12497 	}
12498 
12499 	/*
12500 	 * Must be under the same ctx::mutex as perf_install_in_context(),
12501 	 * because we need to serialize with concurrent event creation.
12502 	 */
12503 	if (!exclusive_event_installable(event, ctx)) {
12504 		err = -EBUSY;
12505 		goto err_locked;
12506 	}
12507 
12508 	WARN_ON_ONCE(ctx->parent_ctx);
12509 
12510 	/*
12511 	 * This is the point on no return; we cannot fail hereafter. This is
12512 	 * where we start modifying current state.
12513 	 */
12514 
12515 	if (move_group) {
12516 		/*
12517 		 * See perf_event_ctx_lock() for comments on the details
12518 		 * of swizzling perf_event::ctx.
12519 		 */
12520 		perf_remove_from_context(group_leader, 0);
12521 		put_ctx(gctx);
12522 
12523 		for_each_sibling_event(sibling, group_leader) {
12524 			perf_remove_from_context(sibling, 0);
12525 			put_ctx(gctx);
12526 		}
12527 
12528 		/*
12529 		 * Wait for everybody to stop referencing the events through
12530 		 * the old lists, before installing it on new lists.
12531 		 */
12532 		synchronize_rcu();
12533 
12534 		/*
12535 		 * Install the group siblings before the group leader.
12536 		 *
12537 		 * Because a group leader will try and install the entire group
12538 		 * (through the sibling list, which is still in-tact), we can
12539 		 * end up with siblings installed in the wrong context.
12540 		 *
12541 		 * By installing siblings first we NO-OP because they're not
12542 		 * reachable through the group lists.
12543 		 */
12544 		for_each_sibling_event(sibling, group_leader) {
12545 			perf_event__state_init(sibling);
12546 			perf_install_in_context(ctx, sibling, sibling->cpu);
12547 			get_ctx(ctx);
12548 		}
12549 
12550 		/*
12551 		 * Removing from the context ends up with disabled
12552 		 * event. What we want here is event in the initial
12553 		 * startup state, ready to be add into new context.
12554 		 */
12555 		perf_event__state_init(group_leader);
12556 		perf_install_in_context(ctx, group_leader, group_leader->cpu);
12557 		get_ctx(ctx);
12558 	}
12559 
12560 	/*
12561 	 * Precalculate sample_data sizes; do while holding ctx::mutex such
12562 	 * that we're serialized against further additions and before
12563 	 * perf_install_in_context() which is the point the event is active and
12564 	 * can use these values.
12565 	 */
12566 	perf_event__header_size(event);
12567 	perf_event__id_header_size(event);
12568 
12569 	event->owner = current;
12570 
12571 	perf_install_in_context(ctx, event, event->cpu);
12572 	perf_unpin_context(ctx);
12573 
12574 	if (move_group)
12575 		perf_event_ctx_unlock(group_leader, gctx);
12576 	mutex_unlock(&ctx->mutex);
12577 
12578 	if (task) {
12579 		up_read(&task->signal->exec_update_lock);
12580 		put_task_struct(task);
12581 	}
12582 
12583 	mutex_lock(&current->perf_event_mutex);
12584 	list_add_tail(&event->owner_entry, &current->perf_event_list);
12585 	mutex_unlock(&current->perf_event_mutex);
12586 
12587 	/*
12588 	 * Drop the reference on the group_event after placing the
12589 	 * new event on the sibling_list. This ensures destruction
12590 	 * of the group leader will find the pointer to itself in
12591 	 * perf_group_detach().
12592 	 */
12593 	fdput(group);
12594 	fd_install(event_fd, event_file);
12595 	return event_fd;
12596 
12597 err_locked:
12598 	if (move_group)
12599 		perf_event_ctx_unlock(group_leader, gctx);
12600 	mutex_unlock(&ctx->mutex);
12601 err_cred:
12602 	if (task)
12603 		up_read(&task->signal->exec_update_lock);
12604 err_file:
12605 	fput(event_file);
12606 err_context:
12607 	perf_unpin_context(ctx);
12608 	put_ctx(ctx);
12609 err_alloc:
12610 	/*
12611 	 * If event_file is set, the fput() above will have called ->release()
12612 	 * and that will take care of freeing the event.
12613 	 */
12614 	if (!event_file)
12615 		free_event(event);
12616 err_task:
12617 	if (task)
12618 		put_task_struct(task);
12619 err_group_fd:
12620 	fdput(group);
12621 err_fd:
12622 	put_unused_fd(event_fd);
12623 	return err;
12624 }
12625 
12626 /**
12627  * perf_event_create_kernel_counter
12628  *
12629  * @attr: attributes of the counter to create
12630  * @cpu: cpu in which the counter is bound
12631  * @task: task to profile (NULL for percpu)
12632  * @overflow_handler: callback to trigger when we hit the event
12633  * @context: context data could be used in overflow_handler callback
12634  */
12635 struct perf_event *
12636 perf_event_create_kernel_counter(struct perf_event_attr *attr, int cpu,
12637 				 struct task_struct *task,
12638 				 perf_overflow_handler_t overflow_handler,
12639 				 void *context)
12640 {
12641 	struct perf_event_context *ctx;
12642 	struct perf_event *event;
12643 	int err;
12644 
12645 	/*
12646 	 * Grouping is not supported for kernel events, neither is 'AUX',
12647 	 * make sure the caller's intentions are adjusted.
12648 	 */
12649 	if (attr->aux_output)
12650 		return ERR_PTR(-EINVAL);
12651 
12652 	event = perf_event_alloc(attr, cpu, task, NULL, NULL,
12653 				 overflow_handler, context, -1);
12654 	if (IS_ERR(event)) {
12655 		err = PTR_ERR(event);
12656 		goto err;
12657 	}
12658 
12659 	/* Mark owner so we could distinguish it from user events. */
12660 	event->owner = TASK_TOMBSTONE;
12661 
12662 	/*
12663 	 * Get the target context (task or percpu):
12664 	 */
12665 	ctx = find_get_context(event->pmu, task, event);
12666 	if (IS_ERR(ctx)) {
12667 		err = PTR_ERR(ctx);
12668 		goto err_free;
12669 	}
12670 
12671 	WARN_ON_ONCE(ctx->parent_ctx);
12672 	mutex_lock(&ctx->mutex);
12673 	if (ctx->task == TASK_TOMBSTONE) {
12674 		err = -ESRCH;
12675 		goto err_unlock;
12676 	}
12677 
12678 	if (!task) {
12679 		/*
12680 		 * Check if the @cpu we're creating an event for is online.
12681 		 *
12682 		 * We use the perf_cpu_context::ctx::mutex to serialize against
12683 		 * the hotplug notifiers. See perf_event_{init,exit}_cpu().
12684 		 */
12685 		struct perf_cpu_context *cpuctx =
12686 			container_of(ctx, struct perf_cpu_context, ctx);
12687 		if (!cpuctx->online) {
12688 			err = -ENODEV;
12689 			goto err_unlock;
12690 		}
12691 	}
12692 
12693 	if (!exclusive_event_installable(event, ctx)) {
12694 		err = -EBUSY;
12695 		goto err_unlock;
12696 	}
12697 
12698 	perf_install_in_context(ctx, event, event->cpu);
12699 	perf_unpin_context(ctx);
12700 	mutex_unlock(&ctx->mutex);
12701 
12702 	return event;
12703 
12704 err_unlock:
12705 	mutex_unlock(&ctx->mutex);
12706 	perf_unpin_context(ctx);
12707 	put_ctx(ctx);
12708 err_free:
12709 	free_event(event);
12710 err:
12711 	return ERR_PTR(err);
12712 }
12713 EXPORT_SYMBOL_GPL(perf_event_create_kernel_counter);
12714 
12715 void perf_pmu_migrate_context(struct pmu *pmu, int src_cpu, int dst_cpu)
12716 {
12717 	struct perf_event_context *src_ctx;
12718 	struct perf_event_context *dst_ctx;
12719 	struct perf_event *event, *tmp;
12720 	LIST_HEAD(events);
12721 
12722 	src_ctx = &per_cpu_ptr(pmu->pmu_cpu_context, src_cpu)->ctx;
12723 	dst_ctx = &per_cpu_ptr(pmu->pmu_cpu_context, dst_cpu)->ctx;
12724 
12725 	/*
12726 	 * See perf_event_ctx_lock() for comments on the details
12727 	 * of swizzling perf_event::ctx.
12728 	 */
12729 	mutex_lock_double(&src_ctx->mutex, &dst_ctx->mutex);
12730 	list_for_each_entry_safe(event, tmp, &src_ctx->event_list,
12731 				 event_entry) {
12732 		perf_remove_from_context(event, 0);
12733 		unaccount_event_cpu(event, src_cpu);
12734 		put_ctx(src_ctx);
12735 		list_add(&event->migrate_entry, &events);
12736 	}
12737 
12738 	/*
12739 	 * Wait for the events to quiesce before re-instating them.
12740 	 */
12741 	synchronize_rcu();
12742 
12743 	/*
12744 	 * Re-instate events in 2 passes.
12745 	 *
12746 	 * Skip over group leaders and only install siblings on this first
12747 	 * pass, siblings will not get enabled without a leader, however a
12748 	 * leader will enable its siblings, even if those are still on the old
12749 	 * context.
12750 	 */
12751 	list_for_each_entry_safe(event, tmp, &events, migrate_entry) {
12752 		if (event->group_leader == event)
12753 			continue;
12754 
12755 		list_del(&event->migrate_entry);
12756 		if (event->state >= PERF_EVENT_STATE_OFF)
12757 			event->state = PERF_EVENT_STATE_INACTIVE;
12758 		account_event_cpu(event, dst_cpu);
12759 		perf_install_in_context(dst_ctx, event, dst_cpu);
12760 		get_ctx(dst_ctx);
12761 	}
12762 
12763 	/*
12764 	 * Once all the siblings are setup properly, install the group leaders
12765 	 * to make it go.
12766 	 */
12767 	list_for_each_entry_safe(event, tmp, &events, migrate_entry) {
12768 		list_del(&event->migrate_entry);
12769 		if (event->state >= PERF_EVENT_STATE_OFF)
12770 			event->state = PERF_EVENT_STATE_INACTIVE;
12771 		account_event_cpu(event, dst_cpu);
12772 		perf_install_in_context(dst_ctx, event, dst_cpu);
12773 		get_ctx(dst_ctx);
12774 	}
12775 	mutex_unlock(&dst_ctx->mutex);
12776 	mutex_unlock(&src_ctx->mutex);
12777 }
12778 EXPORT_SYMBOL_GPL(perf_pmu_migrate_context);
12779 
12780 static void sync_child_event(struct perf_event *child_event)
12781 {
12782 	struct perf_event *parent_event = child_event->parent;
12783 	u64 child_val;
12784 
12785 	if (child_event->attr.inherit_stat) {
12786 		struct task_struct *task = child_event->ctx->task;
12787 
12788 		if (task && task != TASK_TOMBSTONE)
12789 			perf_event_read_event(child_event, task);
12790 	}
12791 
12792 	child_val = perf_event_count(child_event);
12793 
12794 	/*
12795 	 * Add back the child's count to the parent's count:
12796 	 */
12797 	atomic64_add(child_val, &parent_event->child_count);
12798 	atomic64_add(child_event->total_time_enabled,
12799 		     &parent_event->child_total_time_enabled);
12800 	atomic64_add(child_event->total_time_running,
12801 		     &parent_event->child_total_time_running);
12802 }
12803 
12804 static void
12805 perf_event_exit_event(struct perf_event *event, struct perf_event_context *ctx)
12806 {
12807 	struct perf_event *parent_event = event->parent;
12808 	unsigned long detach_flags = 0;
12809 
12810 	if (parent_event) {
12811 		/*
12812 		 * Do not destroy the 'original' grouping; because of the
12813 		 * context switch optimization the original events could've
12814 		 * ended up in a random child task.
12815 		 *
12816 		 * If we were to destroy the original group, all group related
12817 		 * operations would cease to function properly after this
12818 		 * random child dies.
12819 		 *
12820 		 * Do destroy all inherited groups, we don't care about those
12821 		 * and being thorough is better.
12822 		 */
12823 		detach_flags = DETACH_GROUP | DETACH_CHILD;
12824 		mutex_lock(&parent_event->child_mutex);
12825 	}
12826 
12827 	perf_remove_from_context(event, detach_flags);
12828 
12829 	raw_spin_lock_irq(&ctx->lock);
12830 	if (event->state > PERF_EVENT_STATE_EXIT)
12831 		perf_event_set_state(event, PERF_EVENT_STATE_EXIT);
12832 	raw_spin_unlock_irq(&ctx->lock);
12833 
12834 	/*
12835 	 * Child events can be freed.
12836 	 */
12837 	if (parent_event) {
12838 		mutex_unlock(&parent_event->child_mutex);
12839 		/*
12840 		 * Kick perf_poll() for is_event_hup();
12841 		 */
12842 		perf_event_wakeup(parent_event);
12843 		free_event(event);
12844 		put_event(parent_event);
12845 		return;
12846 	}
12847 
12848 	/*
12849 	 * Parent events are governed by their filedesc, retain them.
12850 	 */
12851 	perf_event_wakeup(event);
12852 }
12853 
12854 static void perf_event_exit_task_context(struct task_struct *child, int ctxn)
12855 {
12856 	struct perf_event_context *child_ctx, *clone_ctx = NULL;
12857 	struct perf_event *child_event, *next;
12858 
12859 	WARN_ON_ONCE(child != current);
12860 
12861 	child_ctx = perf_pin_task_context(child, ctxn);
12862 	if (!child_ctx)
12863 		return;
12864 
12865 	/*
12866 	 * In order to reduce the amount of tricky in ctx tear-down, we hold
12867 	 * ctx::mutex over the entire thing. This serializes against almost
12868 	 * everything that wants to access the ctx.
12869 	 *
12870 	 * The exception is sys_perf_event_open() /
12871 	 * perf_event_create_kernel_count() which does find_get_context()
12872 	 * without ctx::mutex (it cannot because of the move_group double mutex
12873 	 * lock thing). See the comments in perf_install_in_context().
12874 	 */
12875 	mutex_lock(&child_ctx->mutex);
12876 
12877 	/*
12878 	 * In a single ctx::lock section, de-schedule the events and detach the
12879 	 * context from the task such that we cannot ever get it scheduled back
12880 	 * in.
12881 	 */
12882 	raw_spin_lock_irq(&child_ctx->lock);
12883 	task_ctx_sched_out(__get_cpu_context(child_ctx), child_ctx, EVENT_ALL);
12884 
12885 	/*
12886 	 * Now that the context is inactive, destroy the task <-> ctx relation
12887 	 * and mark the context dead.
12888 	 */
12889 	RCU_INIT_POINTER(child->perf_event_ctxp[ctxn], NULL);
12890 	put_ctx(child_ctx); /* cannot be last */
12891 	WRITE_ONCE(child_ctx->task, TASK_TOMBSTONE);
12892 	put_task_struct(current); /* cannot be last */
12893 
12894 	clone_ctx = unclone_ctx(child_ctx);
12895 	raw_spin_unlock_irq(&child_ctx->lock);
12896 
12897 	if (clone_ctx)
12898 		put_ctx(clone_ctx);
12899 
12900 	/*
12901 	 * Report the task dead after unscheduling the events so that we
12902 	 * won't get any samples after PERF_RECORD_EXIT. We can however still
12903 	 * get a few PERF_RECORD_READ events.
12904 	 */
12905 	perf_event_task(child, child_ctx, 0);
12906 
12907 	list_for_each_entry_safe(child_event, next, &child_ctx->event_list, event_entry)
12908 		perf_event_exit_event(child_event, child_ctx);
12909 
12910 	mutex_unlock(&child_ctx->mutex);
12911 
12912 	put_ctx(child_ctx);
12913 }
12914 
12915 /*
12916  * When a child task exits, feed back event values to parent events.
12917  *
12918  * Can be called with exec_update_lock held when called from
12919  * setup_new_exec().
12920  */
12921 void perf_event_exit_task(struct task_struct *child)
12922 {
12923 	struct perf_event *event, *tmp;
12924 	int ctxn;
12925 
12926 	mutex_lock(&child->perf_event_mutex);
12927 	list_for_each_entry_safe(event, tmp, &child->perf_event_list,
12928 				 owner_entry) {
12929 		list_del_init(&event->owner_entry);
12930 
12931 		/*
12932 		 * Ensure the list deletion is visible before we clear
12933 		 * the owner, closes a race against perf_release() where
12934 		 * we need to serialize on the owner->perf_event_mutex.
12935 		 */
12936 		smp_store_release(&event->owner, NULL);
12937 	}
12938 	mutex_unlock(&child->perf_event_mutex);
12939 
12940 	for_each_task_context_nr(ctxn)
12941 		perf_event_exit_task_context(child, ctxn);
12942 
12943 	/*
12944 	 * The perf_event_exit_task_context calls perf_event_task
12945 	 * with child's task_ctx, which generates EXIT events for
12946 	 * child contexts and sets child->perf_event_ctxp[] to NULL.
12947 	 * At this point we need to send EXIT events to cpu contexts.
12948 	 */
12949 	perf_event_task(child, NULL, 0);
12950 }
12951 
12952 static void perf_free_event(struct perf_event *event,
12953 			    struct perf_event_context *ctx)
12954 {
12955 	struct perf_event *parent = event->parent;
12956 
12957 	if (WARN_ON_ONCE(!parent))
12958 		return;
12959 
12960 	mutex_lock(&parent->child_mutex);
12961 	list_del_init(&event->child_list);
12962 	mutex_unlock(&parent->child_mutex);
12963 
12964 	put_event(parent);
12965 
12966 	raw_spin_lock_irq(&ctx->lock);
12967 	perf_group_detach(event);
12968 	list_del_event(event, ctx);
12969 	raw_spin_unlock_irq(&ctx->lock);
12970 	free_event(event);
12971 }
12972 
12973 /*
12974  * Free a context as created by inheritance by perf_event_init_task() below,
12975  * used by fork() in case of fail.
12976  *
12977  * Even though the task has never lived, the context and events have been
12978  * exposed through the child_list, so we must take care tearing it all down.
12979  */
12980 void perf_event_free_task(struct task_struct *task)
12981 {
12982 	struct perf_event_context *ctx;
12983 	struct perf_event *event, *tmp;
12984 	int ctxn;
12985 
12986 	for_each_task_context_nr(ctxn) {
12987 		ctx = task->perf_event_ctxp[ctxn];
12988 		if (!ctx)
12989 			continue;
12990 
12991 		mutex_lock(&ctx->mutex);
12992 		raw_spin_lock_irq(&ctx->lock);
12993 		/*
12994 		 * Destroy the task <-> ctx relation and mark the context dead.
12995 		 *
12996 		 * This is important because even though the task hasn't been
12997 		 * exposed yet the context has been (through child_list).
12998 		 */
12999 		RCU_INIT_POINTER(task->perf_event_ctxp[ctxn], NULL);
13000 		WRITE_ONCE(ctx->task, TASK_TOMBSTONE);
13001 		put_task_struct(task); /* cannot be last */
13002 		raw_spin_unlock_irq(&ctx->lock);
13003 
13004 		list_for_each_entry_safe(event, tmp, &ctx->event_list, event_entry)
13005 			perf_free_event(event, ctx);
13006 
13007 		mutex_unlock(&ctx->mutex);
13008 
13009 		/*
13010 		 * perf_event_release_kernel() could've stolen some of our
13011 		 * child events and still have them on its free_list. In that
13012 		 * case we must wait for these events to have been freed (in
13013 		 * particular all their references to this task must've been
13014 		 * dropped).
13015 		 *
13016 		 * Without this copy_process() will unconditionally free this
13017 		 * task (irrespective of its reference count) and
13018 		 * _free_event()'s put_task_struct(event->hw.target) will be a
13019 		 * use-after-free.
13020 		 *
13021 		 * Wait for all events to drop their context reference.
13022 		 */
13023 		wait_var_event(&ctx->refcount, refcount_read(&ctx->refcount) == 1);
13024 		put_ctx(ctx); /* must be last */
13025 	}
13026 }
13027 
13028 void perf_event_delayed_put(struct task_struct *task)
13029 {
13030 	int ctxn;
13031 
13032 	for_each_task_context_nr(ctxn)
13033 		WARN_ON_ONCE(task->perf_event_ctxp[ctxn]);
13034 }
13035 
13036 struct file *perf_event_get(unsigned int fd)
13037 {
13038 	struct file *file = fget(fd);
13039 	if (!file)
13040 		return ERR_PTR(-EBADF);
13041 
13042 	if (file->f_op != &perf_fops) {
13043 		fput(file);
13044 		return ERR_PTR(-EBADF);
13045 	}
13046 
13047 	return file;
13048 }
13049 
13050 const struct perf_event *perf_get_event(struct file *file)
13051 {
13052 	if (file->f_op != &perf_fops)
13053 		return ERR_PTR(-EINVAL);
13054 
13055 	return file->private_data;
13056 }
13057 
13058 const struct perf_event_attr *perf_event_attrs(struct perf_event *event)
13059 {
13060 	if (!event)
13061 		return ERR_PTR(-EINVAL);
13062 
13063 	return &event->attr;
13064 }
13065 
13066 /*
13067  * Inherit an event from parent task to child task.
13068  *
13069  * Returns:
13070  *  - valid pointer on success
13071  *  - NULL for orphaned events
13072  *  - IS_ERR() on error
13073  */
13074 static struct perf_event *
13075 inherit_event(struct perf_event *parent_event,
13076 	      struct task_struct *parent,
13077 	      struct perf_event_context *parent_ctx,
13078 	      struct task_struct *child,
13079 	      struct perf_event *group_leader,
13080 	      struct perf_event_context *child_ctx)
13081 {
13082 	enum perf_event_state parent_state = parent_event->state;
13083 	struct perf_event *child_event;
13084 	unsigned long flags;
13085 
13086 	/*
13087 	 * Instead of creating recursive hierarchies of events,
13088 	 * we link inherited events back to the original parent,
13089 	 * which has a filp for sure, which we use as the reference
13090 	 * count:
13091 	 */
13092 	if (parent_event->parent)
13093 		parent_event = parent_event->parent;
13094 
13095 	child_event = perf_event_alloc(&parent_event->attr,
13096 					   parent_event->cpu,
13097 					   child,
13098 					   group_leader, parent_event,
13099 					   NULL, NULL, -1);
13100 	if (IS_ERR(child_event))
13101 		return child_event;
13102 
13103 
13104 	if ((child_event->attach_state & PERF_ATTACH_TASK_DATA) &&
13105 	    !child_ctx->task_ctx_data) {
13106 		struct pmu *pmu = child_event->pmu;
13107 
13108 		child_ctx->task_ctx_data = alloc_task_ctx_data(pmu);
13109 		if (!child_ctx->task_ctx_data) {
13110 			free_event(child_event);
13111 			return ERR_PTR(-ENOMEM);
13112 		}
13113 	}
13114 
13115 	/*
13116 	 * is_orphaned_event() and list_add_tail(&parent_event->child_list)
13117 	 * must be under the same lock in order to serialize against
13118 	 * perf_event_release_kernel(), such that either we must observe
13119 	 * is_orphaned_event() or they will observe us on the child_list.
13120 	 */
13121 	mutex_lock(&parent_event->child_mutex);
13122 	if (is_orphaned_event(parent_event) ||
13123 	    !atomic_long_inc_not_zero(&parent_event->refcount)) {
13124 		mutex_unlock(&parent_event->child_mutex);
13125 		/* task_ctx_data is freed with child_ctx */
13126 		free_event(child_event);
13127 		return NULL;
13128 	}
13129 
13130 	get_ctx(child_ctx);
13131 
13132 	/*
13133 	 * Make the child state follow the state of the parent event,
13134 	 * not its attr.disabled bit.  We hold the parent's mutex,
13135 	 * so we won't race with perf_event_{en, dis}able_family.
13136 	 */
13137 	if (parent_state >= PERF_EVENT_STATE_INACTIVE)
13138 		child_event->state = PERF_EVENT_STATE_INACTIVE;
13139 	else
13140 		child_event->state = PERF_EVENT_STATE_OFF;
13141 
13142 	if (parent_event->attr.freq) {
13143 		u64 sample_period = parent_event->hw.sample_period;
13144 		struct hw_perf_event *hwc = &child_event->hw;
13145 
13146 		hwc->sample_period = sample_period;
13147 		hwc->last_period   = sample_period;
13148 
13149 		local64_set(&hwc->period_left, sample_period);
13150 	}
13151 
13152 	child_event->ctx = child_ctx;
13153 	child_event->overflow_handler = parent_event->overflow_handler;
13154 	child_event->overflow_handler_context
13155 		= parent_event->overflow_handler_context;
13156 
13157 	/*
13158 	 * Precalculate sample_data sizes
13159 	 */
13160 	perf_event__header_size(child_event);
13161 	perf_event__id_header_size(child_event);
13162 
13163 	/*
13164 	 * Link it up in the child's context:
13165 	 */
13166 	raw_spin_lock_irqsave(&child_ctx->lock, flags);
13167 	add_event_to_ctx(child_event, child_ctx);
13168 	child_event->attach_state |= PERF_ATTACH_CHILD;
13169 	raw_spin_unlock_irqrestore(&child_ctx->lock, flags);
13170 
13171 	/*
13172 	 * Link this into the parent event's child list
13173 	 */
13174 	list_add_tail(&child_event->child_list, &parent_event->child_list);
13175 	mutex_unlock(&parent_event->child_mutex);
13176 
13177 	return child_event;
13178 }
13179 
13180 /*
13181  * Inherits an event group.
13182  *
13183  * This will quietly suppress orphaned events; !inherit_event() is not an error.
13184  * This matches with perf_event_release_kernel() removing all child events.
13185  *
13186  * Returns:
13187  *  - 0 on success
13188  *  - <0 on error
13189  */
13190 static int inherit_group(struct perf_event *parent_event,
13191 	      struct task_struct *parent,
13192 	      struct perf_event_context *parent_ctx,
13193 	      struct task_struct *child,
13194 	      struct perf_event_context *child_ctx)
13195 {
13196 	struct perf_event *leader;
13197 	struct perf_event *sub;
13198 	struct perf_event *child_ctr;
13199 
13200 	leader = inherit_event(parent_event, parent, parent_ctx,
13201 				 child, NULL, child_ctx);
13202 	if (IS_ERR(leader))
13203 		return PTR_ERR(leader);
13204 	/*
13205 	 * @leader can be NULL here because of is_orphaned_event(). In this
13206 	 * case inherit_event() will create individual events, similar to what
13207 	 * perf_group_detach() would do anyway.
13208 	 */
13209 	for_each_sibling_event(sub, parent_event) {
13210 		child_ctr = inherit_event(sub, parent, parent_ctx,
13211 					    child, leader, child_ctx);
13212 		if (IS_ERR(child_ctr))
13213 			return PTR_ERR(child_ctr);
13214 
13215 		if (sub->aux_event == parent_event && child_ctr &&
13216 		    !perf_get_aux_event(child_ctr, leader))
13217 			return -EINVAL;
13218 	}
13219 	return 0;
13220 }
13221 
13222 /*
13223  * Creates the child task context and tries to inherit the event-group.
13224  *
13225  * Clears @inherited_all on !attr.inherited or error. Note that we'll leave
13226  * inherited_all set when we 'fail' to inherit an orphaned event; this is
13227  * consistent with perf_event_release_kernel() removing all child events.
13228  *
13229  * Returns:
13230  *  - 0 on success
13231  *  - <0 on error
13232  */
13233 static int
13234 inherit_task_group(struct perf_event *event, struct task_struct *parent,
13235 		   struct perf_event_context *parent_ctx,
13236 		   struct task_struct *child, int ctxn,
13237 		   u64 clone_flags, int *inherited_all)
13238 {
13239 	int ret;
13240 	struct perf_event_context *child_ctx;
13241 
13242 	if (!event->attr.inherit ||
13243 	    (event->attr.inherit_thread && !(clone_flags & CLONE_THREAD)) ||
13244 	    /* Do not inherit if sigtrap and signal handlers were cleared. */
13245 	    (event->attr.sigtrap && (clone_flags & CLONE_CLEAR_SIGHAND))) {
13246 		*inherited_all = 0;
13247 		return 0;
13248 	}
13249 
13250 	child_ctx = child->perf_event_ctxp[ctxn];
13251 	if (!child_ctx) {
13252 		/*
13253 		 * This is executed from the parent task context, so
13254 		 * inherit events that have been marked for cloning.
13255 		 * First allocate and initialize a context for the
13256 		 * child.
13257 		 */
13258 		child_ctx = alloc_perf_context(parent_ctx->pmu, child);
13259 		if (!child_ctx)
13260 			return -ENOMEM;
13261 
13262 		child->perf_event_ctxp[ctxn] = child_ctx;
13263 	}
13264 
13265 	ret = inherit_group(event, parent, parent_ctx,
13266 			    child, child_ctx);
13267 
13268 	if (ret)
13269 		*inherited_all = 0;
13270 
13271 	return ret;
13272 }
13273 
13274 /*
13275  * Initialize the perf_event context in task_struct
13276  */
13277 static int perf_event_init_context(struct task_struct *child, int ctxn,
13278 				   u64 clone_flags)
13279 {
13280 	struct perf_event_context *child_ctx, *parent_ctx;
13281 	struct perf_event_context *cloned_ctx;
13282 	struct perf_event *event;
13283 	struct task_struct *parent = current;
13284 	int inherited_all = 1;
13285 	unsigned long flags;
13286 	int ret = 0;
13287 
13288 	if (likely(!parent->perf_event_ctxp[ctxn]))
13289 		return 0;
13290 
13291 	/*
13292 	 * If the parent's context is a clone, pin it so it won't get
13293 	 * swapped under us.
13294 	 */
13295 	parent_ctx = perf_pin_task_context(parent, ctxn);
13296 	if (!parent_ctx)
13297 		return 0;
13298 
13299 	/*
13300 	 * No need to check if parent_ctx != NULL here; since we saw
13301 	 * it non-NULL earlier, the only reason for it to become NULL
13302 	 * is if we exit, and since we're currently in the middle of
13303 	 * a fork we can't be exiting at the same time.
13304 	 */
13305 
13306 	/*
13307 	 * Lock the parent list. No need to lock the child - not PID
13308 	 * hashed yet and not running, so nobody can access it.
13309 	 */
13310 	mutex_lock(&parent_ctx->mutex);
13311 
13312 	/*
13313 	 * We dont have to disable NMIs - we are only looking at
13314 	 * the list, not manipulating it:
13315 	 */
13316 	perf_event_groups_for_each(event, &parent_ctx->pinned_groups) {
13317 		ret = inherit_task_group(event, parent, parent_ctx,
13318 					 child, ctxn, clone_flags,
13319 					 &inherited_all);
13320 		if (ret)
13321 			goto out_unlock;
13322 	}
13323 
13324 	/*
13325 	 * We can't hold ctx->lock when iterating the ->flexible_group list due
13326 	 * to allocations, but we need to prevent rotation because
13327 	 * rotate_ctx() will change the list from interrupt context.
13328 	 */
13329 	raw_spin_lock_irqsave(&parent_ctx->lock, flags);
13330 	parent_ctx->rotate_disable = 1;
13331 	raw_spin_unlock_irqrestore(&parent_ctx->lock, flags);
13332 
13333 	perf_event_groups_for_each(event, &parent_ctx->flexible_groups) {
13334 		ret = inherit_task_group(event, parent, parent_ctx,
13335 					 child, ctxn, clone_flags,
13336 					 &inherited_all);
13337 		if (ret)
13338 			goto out_unlock;
13339 	}
13340 
13341 	raw_spin_lock_irqsave(&parent_ctx->lock, flags);
13342 	parent_ctx->rotate_disable = 0;
13343 
13344 	child_ctx = child->perf_event_ctxp[ctxn];
13345 
13346 	if (child_ctx && inherited_all) {
13347 		/*
13348 		 * Mark the child context as a clone of the parent
13349 		 * context, or of whatever the parent is a clone of.
13350 		 *
13351 		 * Note that if the parent is a clone, the holding of
13352 		 * parent_ctx->lock avoids it from being uncloned.
13353 		 */
13354 		cloned_ctx = parent_ctx->parent_ctx;
13355 		if (cloned_ctx) {
13356 			child_ctx->parent_ctx = cloned_ctx;
13357 			child_ctx->parent_gen = parent_ctx->parent_gen;
13358 		} else {
13359 			child_ctx->parent_ctx = parent_ctx;
13360 			child_ctx->parent_gen = parent_ctx->generation;
13361 		}
13362 		get_ctx(child_ctx->parent_ctx);
13363 	}
13364 
13365 	raw_spin_unlock_irqrestore(&parent_ctx->lock, flags);
13366 out_unlock:
13367 	mutex_unlock(&parent_ctx->mutex);
13368 
13369 	perf_unpin_context(parent_ctx);
13370 	put_ctx(parent_ctx);
13371 
13372 	return ret;
13373 }
13374 
13375 /*
13376  * Initialize the perf_event context in task_struct
13377  */
13378 int perf_event_init_task(struct task_struct *child, u64 clone_flags)
13379 {
13380 	int ctxn, ret;
13381 
13382 	memset(child->perf_event_ctxp, 0, sizeof(child->perf_event_ctxp));
13383 	mutex_init(&child->perf_event_mutex);
13384 	INIT_LIST_HEAD(&child->perf_event_list);
13385 
13386 	for_each_task_context_nr(ctxn) {
13387 		ret = perf_event_init_context(child, ctxn, clone_flags);
13388 		if (ret) {
13389 			perf_event_free_task(child);
13390 			return ret;
13391 		}
13392 	}
13393 
13394 	return 0;
13395 }
13396 
13397 static void __init perf_event_init_all_cpus(void)
13398 {
13399 	struct swevent_htable *swhash;
13400 	int cpu;
13401 
13402 	zalloc_cpumask_var(&perf_online_mask, GFP_KERNEL);
13403 
13404 	for_each_possible_cpu(cpu) {
13405 		swhash = &per_cpu(swevent_htable, cpu);
13406 		mutex_init(&swhash->hlist_mutex);
13407 		INIT_LIST_HEAD(&per_cpu(active_ctx_list, cpu));
13408 
13409 		INIT_LIST_HEAD(&per_cpu(pmu_sb_events.list, cpu));
13410 		raw_spin_lock_init(&per_cpu(pmu_sb_events.lock, cpu));
13411 
13412 #ifdef CONFIG_CGROUP_PERF
13413 		INIT_LIST_HEAD(&per_cpu(cgrp_cpuctx_list, cpu));
13414 #endif
13415 		INIT_LIST_HEAD(&per_cpu(sched_cb_list, cpu));
13416 	}
13417 }
13418 
13419 static void perf_swevent_init_cpu(unsigned int cpu)
13420 {
13421 	struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
13422 
13423 	mutex_lock(&swhash->hlist_mutex);
13424 	if (swhash->hlist_refcount > 0 && !swevent_hlist_deref(swhash)) {
13425 		struct swevent_hlist *hlist;
13426 
13427 		hlist = kzalloc_node(sizeof(*hlist), GFP_KERNEL, cpu_to_node(cpu));
13428 		WARN_ON(!hlist);
13429 		rcu_assign_pointer(swhash->swevent_hlist, hlist);
13430 	}
13431 	mutex_unlock(&swhash->hlist_mutex);
13432 }
13433 
13434 #if defined CONFIG_HOTPLUG_CPU || defined CONFIG_KEXEC_CORE
13435 static void __perf_event_exit_context(void *__info)
13436 {
13437 	struct perf_event_context *ctx = __info;
13438 	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
13439 	struct perf_event *event;
13440 
13441 	raw_spin_lock(&ctx->lock);
13442 	ctx_sched_out(ctx, cpuctx, EVENT_TIME);
13443 	list_for_each_entry(event, &ctx->event_list, event_entry)
13444 		__perf_remove_from_context(event, cpuctx, ctx, (void *)DETACH_GROUP);
13445 	raw_spin_unlock(&ctx->lock);
13446 }
13447 
13448 static void perf_event_exit_cpu_context(int cpu)
13449 {
13450 	struct perf_cpu_context *cpuctx;
13451 	struct perf_event_context *ctx;
13452 	struct pmu *pmu;
13453 
13454 	mutex_lock(&pmus_lock);
13455 	list_for_each_entry(pmu, &pmus, entry) {
13456 		cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
13457 		ctx = &cpuctx->ctx;
13458 
13459 		mutex_lock(&ctx->mutex);
13460 		smp_call_function_single(cpu, __perf_event_exit_context, ctx, 1);
13461 		cpuctx->online = 0;
13462 		mutex_unlock(&ctx->mutex);
13463 	}
13464 	cpumask_clear_cpu(cpu, perf_online_mask);
13465 	mutex_unlock(&pmus_lock);
13466 }
13467 #else
13468 
13469 static void perf_event_exit_cpu_context(int cpu) { }
13470 
13471 #endif
13472 
13473 int perf_event_init_cpu(unsigned int cpu)
13474 {
13475 	struct perf_cpu_context *cpuctx;
13476 	struct perf_event_context *ctx;
13477 	struct pmu *pmu;
13478 
13479 	perf_swevent_init_cpu(cpu);
13480 
13481 	mutex_lock(&pmus_lock);
13482 	cpumask_set_cpu(cpu, perf_online_mask);
13483 	list_for_each_entry(pmu, &pmus, entry) {
13484 		cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
13485 		ctx = &cpuctx->ctx;
13486 
13487 		mutex_lock(&ctx->mutex);
13488 		cpuctx->online = 1;
13489 		mutex_unlock(&ctx->mutex);
13490 	}
13491 	mutex_unlock(&pmus_lock);
13492 
13493 	return 0;
13494 }
13495 
13496 int perf_event_exit_cpu(unsigned int cpu)
13497 {
13498 	perf_event_exit_cpu_context(cpu);
13499 	return 0;
13500 }
13501 
13502 static int
13503 perf_reboot(struct notifier_block *notifier, unsigned long val, void *v)
13504 {
13505 	int cpu;
13506 
13507 	for_each_online_cpu(cpu)
13508 		perf_event_exit_cpu(cpu);
13509 
13510 	return NOTIFY_OK;
13511 }
13512 
13513 /*
13514  * Run the perf reboot notifier at the very last possible moment so that
13515  * the generic watchdog code runs as long as possible.
13516  */
13517 static struct notifier_block perf_reboot_notifier = {
13518 	.notifier_call = perf_reboot,
13519 	.priority = INT_MIN,
13520 };
13521 
13522 void __init perf_event_init(void)
13523 {
13524 	int ret;
13525 
13526 	idr_init(&pmu_idr);
13527 
13528 	perf_event_init_all_cpus();
13529 	init_srcu_struct(&pmus_srcu);
13530 	perf_pmu_register(&perf_swevent, "software", PERF_TYPE_SOFTWARE);
13531 	perf_pmu_register(&perf_cpu_clock, NULL, -1);
13532 	perf_pmu_register(&perf_task_clock, NULL, -1);
13533 	perf_tp_register();
13534 	perf_event_init_cpu(smp_processor_id());
13535 	register_reboot_notifier(&perf_reboot_notifier);
13536 
13537 	ret = init_hw_breakpoint();
13538 	WARN(ret, "hw_breakpoint initialization failed with: %d", ret);
13539 
13540 	perf_event_cache = KMEM_CACHE(perf_event, SLAB_PANIC);
13541 
13542 	/*
13543 	 * Build time assertion that we keep the data_head at the intended
13544 	 * location.  IOW, validation we got the __reserved[] size right.
13545 	 */
13546 	BUILD_BUG_ON((offsetof(struct perf_event_mmap_page, data_head))
13547 		     != 1024);
13548 }
13549 
13550 ssize_t perf_event_sysfs_show(struct device *dev, struct device_attribute *attr,
13551 			      char *page)
13552 {
13553 	struct perf_pmu_events_attr *pmu_attr =
13554 		container_of(attr, struct perf_pmu_events_attr, attr);
13555 
13556 	if (pmu_attr->event_str)
13557 		return sprintf(page, "%s\n", pmu_attr->event_str);
13558 
13559 	return 0;
13560 }
13561 EXPORT_SYMBOL_GPL(perf_event_sysfs_show);
13562 
13563 static int __init perf_event_sysfs_init(void)
13564 {
13565 	struct pmu *pmu;
13566 	int ret;
13567 
13568 	mutex_lock(&pmus_lock);
13569 
13570 	ret = bus_register(&pmu_bus);
13571 	if (ret)
13572 		goto unlock;
13573 
13574 	list_for_each_entry(pmu, &pmus, entry) {
13575 		if (!pmu->name || pmu->type < 0)
13576 			continue;
13577 
13578 		ret = pmu_dev_alloc(pmu);
13579 		WARN(ret, "Failed to register pmu: %s, reason %d\n", pmu->name, ret);
13580 	}
13581 	pmu_bus_running = 1;
13582 	ret = 0;
13583 
13584 unlock:
13585 	mutex_unlock(&pmus_lock);
13586 
13587 	return ret;
13588 }
13589 device_initcall(perf_event_sysfs_init);
13590 
13591 #ifdef CONFIG_CGROUP_PERF
13592 static struct cgroup_subsys_state *
13593 perf_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
13594 {
13595 	struct perf_cgroup *jc;
13596 
13597 	jc = kzalloc(sizeof(*jc), GFP_KERNEL);
13598 	if (!jc)
13599 		return ERR_PTR(-ENOMEM);
13600 
13601 	jc->info = alloc_percpu(struct perf_cgroup_info);
13602 	if (!jc->info) {
13603 		kfree(jc);
13604 		return ERR_PTR(-ENOMEM);
13605 	}
13606 
13607 	return &jc->css;
13608 }
13609 
13610 static void perf_cgroup_css_free(struct cgroup_subsys_state *css)
13611 {
13612 	struct perf_cgroup *jc = container_of(css, struct perf_cgroup, css);
13613 
13614 	free_percpu(jc->info);
13615 	kfree(jc);
13616 }
13617 
13618 static int perf_cgroup_css_online(struct cgroup_subsys_state *css)
13619 {
13620 	perf_event_cgroup(css->cgroup);
13621 	return 0;
13622 }
13623 
13624 static int __perf_cgroup_move(void *info)
13625 {
13626 	struct task_struct *task = info;
13627 	rcu_read_lock();
13628 	perf_cgroup_switch(task);
13629 	rcu_read_unlock();
13630 	return 0;
13631 }
13632 
13633 static void perf_cgroup_attach(struct cgroup_taskset *tset)
13634 {
13635 	struct task_struct *task;
13636 	struct cgroup_subsys_state *css;
13637 
13638 	cgroup_taskset_for_each(task, css, tset)
13639 		task_function_call(task, __perf_cgroup_move, task);
13640 }
13641 
13642 struct cgroup_subsys perf_event_cgrp_subsys = {
13643 	.css_alloc	= perf_cgroup_css_alloc,
13644 	.css_free	= perf_cgroup_css_free,
13645 	.css_online	= perf_cgroup_css_online,
13646 	.attach		= perf_cgroup_attach,
13647 	/*
13648 	 * Implicitly enable on dfl hierarchy so that perf events can
13649 	 * always be filtered by cgroup2 path as long as perf_event
13650 	 * controller is not mounted on a legacy hierarchy.
13651 	 */
13652 	.implicit_on_dfl = true,
13653 	.threaded	= true,
13654 };
13655 #endif /* CONFIG_CGROUP_PERF */
13656 
13657 DEFINE_STATIC_CALL_RET0(perf_snapshot_branch_stack, perf_snapshot_branch_stack_t);
13658