12874c5fdSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-or-later
2f2699491SMichael Ellerman /*
3f2699491SMichael Ellerman * Performance event support - Freescale Embedded Performance Monitor
4f2699491SMichael Ellerman *
5f2699491SMichael Ellerman * Copyright 2008-2009 Paul Mackerras, IBM Corporation.
6f2699491SMichael Ellerman * Copyright 2010 Freescale Semiconductor, Inc.
7f2699491SMichael Ellerman */
8f2699491SMichael Ellerman #include <linux/kernel.h>
9f2699491SMichael Ellerman #include <linux/sched.h>
10f2699491SMichael Ellerman #include <linux/perf_event.h>
11f2699491SMichael Ellerman #include <linux/percpu.h>
12f2699491SMichael Ellerman #include <linux/hardirq.h>
13f2699491SMichael Ellerman #include <asm/reg_fsl_emb.h>
14f2699491SMichael Ellerman #include <asm/pmc.h>
15f2699491SMichael Ellerman #include <asm/machdep.h>
16f2699491SMichael Ellerman #include <asm/firmware.h>
17f2699491SMichael Ellerman #include <asm/ptrace.h>
18f2699491SMichael Ellerman
19f2699491SMichael Ellerman struct cpu_hw_events {
20f2699491SMichael Ellerman int n_events;
21f2699491SMichael Ellerman int disabled;
22f2699491SMichael Ellerman u8 pmcs_enabled;
23f2699491SMichael Ellerman struct perf_event *event[MAX_HWEVENTS];
24f2699491SMichael Ellerman };
25f2699491SMichael Ellerman static DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events);
26f2699491SMichael Ellerman
27f2699491SMichael Ellerman static struct fsl_emb_pmu *ppmu;
28f2699491SMichael Ellerman
29f2699491SMichael Ellerman /* Number of perf_events counting hardware events */
30f2699491SMichael Ellerman static atomic_t num_events;
31f2699491SMichael Ellerman /* Used to avoid races in calling reserve/release_pmc_hardware */
32f2699491SMichael Ellerman static DEFINE_MUTEX(pmc_reserve_mutex);
33f2699491SMichael Ellerman
34f2699491SMichael Ellerman static void perf_event_interrupt(struct pt_regs *regs);
35f2699491SMichael Ellerman
36f2699491SMichael Ellerman /*
37f2699491SMichael Ellerman * Read one performance monitor counter (PMC).
38f2699491SMichael Ellerman */
read_pmc(int idx)39f2699491SMichael Ellerman static unsigned long read_pmc(int idx)
40f2699491SMichael Ellerman {
41f2699491SMichael Ellerman unsigned long val;
42f2699491SMichael Ellerman
43f2699491SMichael Ellerman switch (idx) {
44f2699491SMichael Ellerman case 0:
45f2699491SMichael Ellerman val = mfpmr(PMRN_PMC0);
46f2699491SMichael Ellerman break;
47f2699491SMichael Ellerman case 1:
48f2699491SMichael Ellerman val = mfpmr(PMRN_PMC1);
49f2699491SMichael Ellerman break;
50f2699491SMichael Ellerman case 2:
51f2699491SMichael Ellerman val = mfpmr(PMRN_PMC2);
52f2699491SMichael Ellerman break;
53f2699491SMichael Ellerman case 3:
54f2699491SMichael Ellerman val = mfpmr(PMRN_PMC3);
55f2699491SMichael Ellerman break;
565815c434SLijun Pan case 4:
575815c434SLijun Pan val = mfpmr(PMRN_PMC4);
585815c434SLijun Pan break;
595815c434SLijun Pan case 5:
605815c434SLijun Pan val = mfpmr(PMRN_PMC5);
615815c434SLijun Pan break;
62f2699491SMichael Ellerman default:
63f2699491SMichael Ellerman printk(KERN_ERR "oops trying to read PMC%d\n", idx);
64f2699491SMichael Ellerman val = 0;
65f2699491SMichael Ellerman }
66f2699491SMichael Ellerman return val;
67f2699491SMichael Ellerman }
68f2699491SMichael Ellerman
69f2699491SMichael Ellerman /*
70f2699491SMichael Ellerman * Write one PMC.
71f2699491SMichael Ellerman */
write_pmc(int idx,unsigned long val)72f2699491SMichael Ellerman static void write_pmc(int idx, unsigned long val)
73f2699491SMichael Ellerman {
74f2699491SMichael Ellerman switch (idx) {
75f2699491SMichael Ellerman case 0:
76f2699491SMichael Ellerman mtpmr(PMRN_PMC0, val);
77f2699491SMichael Ellerman break;
78f2699491SMichael Ellerman case 1:
79f2699491SMichael Ellerman mtpmr(PMRN_PMC1, val);
80f2699491SMichael Ellerman break;
81f2699491SMichael Ellerman case 2:
82f2699491SMichael Ellerman mtpmr(PMRN_PMC2, val);
83f2699491SMichael Ellerman break;
84f2699491SMichael Ellerman case 3:
85f2699491SMichael Ellerman mtpmr(PMRN_PMC3, val);
86f2699491SMichael Ellerman break;
875815c434SLijun Pan case 4:
885815c434SLijun Pan mtpmr(PMRN_PMC4, val);
895815c434SLijun Pan break;
905815c434SLijun Pan case 5:
915815c434SLijun Pan mtpmr(PMRN_PMC5, val);
925815c434SLijun Pan break;
93f2699491SMichael Ellerman default:
94f2699491SMichael Ellerman printk(KERN_ERR "oops trying to write PMC%d\n", idx);
95f2699491SMichael Ellerman }
96f2699491SMichael Ellerman
97f2699491SMichael Ellerman isync();
98f2699491SMichael Ellerman }
99f2699491SMichael Ellerman
100f2699491SMichael Ellerman /*
101f2699491SMichael Ellerman * Write one local control A register
102f2699491SMichael Ellerman */
write_pmlca(int idx,unsigned long val)103f2699491SMichael Ellerman static void write_pmlca(int idx, unsigned long val)
104f2699491SMichael Ellerman {
105f2699491SMichael Ellerman switch (idx) {
106f2699491SMichael Ellerman case 0:
107f2699491SMichael Ellerman mtpmr(PMRN_PMLCA0, val);
108f2699491SMichael Ellerman break;
109f2699491SMichael Ellerman case 1:
110f2699491SMichael Ellerman mtpmr(PMRN_PMLCA1, val);
111f2699491SMichael Ellerman break;
112f2699491SMichael Ellerman case 2:
113f2699491SMichael Ellerman mtpmr(PMRN_PMLCA2, val);
114f2699491SMichael Ellerman break;
115f2699491SMichael Ellerman case 3:
116f2699491SMichael Ellerman mtpmr(PMRN_PMLCA3, val);
117f2699491SMichael Ellerman break;
1185815c434SLijun Pan case 4:
1195815c434SLijun Pan mtpmr(PMRN_PMLCA4, val);
1205815c434SLijun Pan break;
1215815c434SLijun Pan case 5:
1225815c434SLijun Pan mtpmr(PMRN_PMLCA5, val);
1235815c434SLijun Pan break;
124f2699491SMichael Ellerman default:
125f2699491SMichael Ellerman printk(KERN_ERR "oops trying to write PMLCA%d\n", idx);
126f2699491SMichael Ellerman }
127f2699491SMichael Ellerman
128f2699491SMichael Ellerman isync();
129f2699491SMichael Ellerman }
130f2699491SMichael Ellerman
131f2699491SMichael Ellerman /*
132f2699491SMichael Ellerman * Write one local control B register
133f2699491SMichael Ellerman */
write_pmlcb(int idx,unsigned long val)134f2699491SMichael Ellerman static void write_pmlcb(int idx, unsigned long val)
135f2699491SMichael Ellerman {
136f2699491SMichael Ellerman switch (idx) {
137f2699491SMichael Ellerman case 0:
138f2699491SMichael Ellerman mtpmr(PMRN_PMLCB0, val);
139f2699491SMichael Ellerman break;
140f2699491SMichael Ellerman case 1:
141f2699491SMichael Ellerman mtpmr(PMRN_PMLCB1, val);
142f2699491SMichael Ellerman break;
143f2699491SMichael Ellerman case 2:
144f2699491SMichael Ellerman mtpmr(PMRN_PMLCB2, val);
145f2699491SMichael Ellerman break;
146f2699491SMichael Ellerman case 3:
147f2699491SMichael Ellerman mtpmr(PMRN_PMLCB3, val);
148f2699491SMichael Ellerman break;
1495815c434SLijun Pan case 4:
1505815c434SLijun Pan mtpmr(PMRN_PMLCB4, val);
1515815c434SLijun Pan break;
1525815c434SLijun Pan case 5:
1535815c434SLijun Pan mtpmr(PMRN_PMLCB5, val);
1545815c434SLijun Pan break;
155f2699491SMichael Ellerman default:
156f2699491SMichael Ellerman printk(KERN_ERR "oops trying to write PMLCB%d\n", idx);
157f2699491SMichael Ellerman }
158f2699491SMichael Ellerman
159f2699491SMichael Ellerman isync();
160f2699491SMichael Ellerman }
161f2699491SMichael Ellerman
fsl_emb_pmu_read(struct perf_event * event)162f2699491SMichael Ellerman static void fsl_emb_pmu_read(struct perf_event *event)
163f2699491SMichael Ellerman {
164f2699491SMichael Ellerman s64 val, delta, prev;
165f2699491SMichael Ellerman
166f2699491SMichael Ellerman if (event->hw.state & PERF_HES_STOPPED)
167f2699491SMichael Ellerman return;
168f2699491SMichael Ellerman
169f2699491SMichael Ellerman /*
170f2699491SMichael Ellerman * Performance monitor interrupts come even when interrupts
171f2699491SMichael Ellerman * are soft-disabled, as long as interrupts are hard-enabled.
172f2699491SMichael Ellerman * Therefore we treat them like NMIs.
173f2699491SMichael Ellerman */
174f2699491SMichael Ellerman do {
175f2699491SMichael Ellerman prev = local64_read(&event->hw.prev_count);
176f2699491SMichael Ellerman barrier();
177f2699491SMichael Ellerman val = read_pmc(event->hw.idx);
178f2699491SMichael Ellerman } while (local64_cmpxchg(&event->hw.prev_count, prev, val) != prev);
179f2699491SMichael Ellerman
180f2699491SMichael Ellerman /* The counters are only 32 bits wide */
181f2699491SMichael Ellerman delta = (val - prev) & 0xfffffffful;
182f2699491SMichael Ellerman local64_add(delta, &event->count);
183f2699491SMichael Ellerman local64_sub(delta, &event->hw.period_left);
184f2699491SMichael Ellerman }
185f2699491SMichael Ellerman
186f2699491SMichael Ellerman /*
187f2699491SMichael Ellerman * Disable all events to prevent PMU interrupts and to allow
188f2699491SMichael Ellerman * events to be added or removed.
189f2699491SMichael Ellerman */
fsl_emb_pmu_disable(struct pmu * pmu)190f2699491SMichael Ellerman static void fsl_emb_pmu_disable(struct pmu *pmu)
191f2699491SMichael Ellerman {
192f2699491SMichael Ellerman struct cpu_hw_events *cpuhw;
193f2699491SMichael Ellerman unsigned long flags;
194f2699491SMichael Ellerman
195f2699491SMichael Ellerman local_irq_save(flags);
19669111bacSChristoph Lameter cpuhw = this_cpu_ptr(&cpu_hw_events);
197f2699491SMichael Ellerman
198f2699491SMichael Ellerman if (!cpuhw->disabled) {
199f2699491SMichael Ellerman cpuhw->disabled = 1;
200f2699491SMichael Ellerman
201f2699491SMichael Ellerman /*
202f2699491SMichael Ellerman * Check if we ever enabled the PMU on this cpu.
203f2699491SMichael Ellerman */
204f2699491SMichael Ellerman if (!cpuhw->pmcs_enabled) {
205f2699491SMichael Ellerman ppc_enable_pmcs();
206f2699491SMichael Ellerman cpuhw->pmcs_enabled = 1;
207f2699491SMichael Ellerman }
208f2699491SMichael Ellerman
209f2699491SMichael Ellerman if (atomic_read(&num_events)) {
210f2699491SMichael Ellerman /*
211f2699491SMichael Ellerman * Set the 'freeze all counters' bit, and disable
212f2699491SMichael Ellerman * interrupts. The barrier is to make sure the
213f2699491SMichael Ellerman * mtpmr has been executed and the PMU has frozen
214f2699491SMichael Ellerman * the events before we return.
215f2699491SMichael Ellerman */
216f2699491SMichael Ellerman
217f2699491SMichael Ellerman mtpmr(PMRN_PMGC0, PMGC0_FAC);
218f2699491SMichael Ellerman isync();
219f2699491SMichael Ellerman }
220f2699491SMichael Ellerman }
221f2699491SMichael Ellerman local_irq_restore(flags);
222f2699491SMichael Ellerman }
223f2699491SMichael Ellerman
224f2699491SMichael Ellerman /*
225f2699491SMichael Ellerman * Re-enable all events if disable == 0.
226f2699491SMichael Ellerman * If we were previously disabled and events were added, then
227f2699491SMichael Ellerman * put the new config on the PMU.
228f2699491SMichael Ellerman */
fsl_emb_pmu_enable(struct pmu * pmu)229f2699491SMichael Ellerman static void fsl_emb_pmu_enable(struct pmu *pmu)
230f2699491SMichael Ellerman {
231f2699491SMichael Ellerman struct cpu_hw_events *cpuhw;
232f2699491SMichael Ellerman unsigned long flags;
233f2699491SMichael Ellerman
234f2699491SMichael Ellerman local_irq_save(flags);
23569111bacSChristoph Lameter cpuhw = this_cpu_ptr(&cpu_hw_events);
236f2699491SMichael Ellerman if (!cpuhw->disabled)
237f2699491SMichael Ellerman goto out;
238f2699491SMichael Ellerman
239f2699491SMichael Ellerman cpuhw->disabled = 0;
240f2699491SMichael Ellerman ppc_set_pmu_inuse(cpuhw->n_events != 0);
241f2699491SMichael Ellerman
242f2699491SMichael Ellerman if (cpuhw->n_events > 0) {
243f2699491SMichael Ellerman mtpmr(PMRN_PMGC0, PMGC0_PMIE | PMGC0_FCECE);
244f2699491SMichael Ellerman isync();
245f2699491SMichael Ellerman }
246f2699491SMichael Ellerman
247f2699491SMichael Ellerman out:
248f2699491SMichael Ellerman local_irq_restore(flags);
249f2699491SMichael Ellerman }
250f2699491SMichael Ellerman
collect_events(struct perf_event * group,int max_count,struct perf_event * ctrs[])251f2699491SMichael Ellerman static int collect_events(struct perf_event *group, int max_count,
252f2699491SMichael Ellerman struct perf_event *ctrs[])
253f2699491SMichael Ellerman {
254f2699491SMichael Ellerman int n = 0;
255f2699491SMichael Ellerman struct perf_event *event;
256f2699491SMichael Ellerman
257f2699491SMichael Ellerman if (!is_software_event(group)) {
258f2699491SMichael Ellerman if (n >= max_count)
259f2699491SMichael Ellerman return -1;
260f2699491SMichael Ellerman ctrs[n] = group;
261f2699491SMichael Ellerman n++;
262f2699491SMichael Ellerman }
263edb39592SPeter Zijlstra for_each_sibling_event(event, group) {
264f2699491SMichael Ellerman if (!is_software_event(event) &&
265f2699491SMichael Ellerman event->state != PERF_EVENT_STATE_OFF) {
266f2699491SMichael Ellerman if (n >= max_count)
267f2699491SMichael Ellerman return -1;
268f2699491SMichael Ellerman ctrs[n] = event;
269f2699491SMichael Ellerman n++;
270f2699491SMichael Ellerman }
271f2699491SMichael Ellerman }
272f2699491SMichael Ellerman return n;
273f2699491SMichael Ellerman }
274f2699491SMichael Ellerman
275f2699491SMichael Ellerman /* context locked on entry */
fsl_emb_pmu_add(struct perf_event * event,int flags)276f2699491SMichael Ellerman static int fsl_emb_pmu_add(struct perf_event *event, int flags)
277f2699491SMichael Ellerman {
278f2699491SMichael Ellerman struct cpu_hw_events *cpuhw;
279f2699491SMichael Ellerman int ret = -EAGAIN;
280f2699491SMichael Ellerman int num_counters = ppmu->n_counter;
281f2699491SMichael Ellerman u64 val;
282f2699491SMichael Ellerman int i;
283f2699491SMichael Ellerman
284f2699491SMichael Ellerman perf_pmu_disable(event->pmu);
285f2699491SMichael Ellerman cpuhw = &get_cpu_var(cpu_hw_events);
286f2699491SMichael Ellerman
287f2699491SMichael Ellerman if (event->hw.config & FSL_EMB_EVENT_RESTRICTED)
288f2699491SMichael Ellerman num_counters = ppmu->n_restricted;
289f2699491SMichael Ellerman
290f2699491SMichael Ellerman /*
291f2699491SMichael Ellerman * Allocate counters from top-down, so that restricted-capable
292f2699491SMichael Ellerman * counters are kept free as long as possible.
293f2699491SMichael Ellerman */
294f2699491SMichael Ellerman for (i = num_counters - 1; i >= 0; i--) {
295f2699491SMichael Ellerman if (cpuhw->event[i])
296f2699491SMichael Ellerman continue;
297f2699491SMichael Ellerman
298f2699491SMichael Ellerman break;
299f2699491SMichael Ellerman }
300f2699491SMichael Ellerman
301f2699491SMichael Ellerman if (i < 0)
302f2699491SMichael Ellerman goto out;
303f2699491SMichael Ellerman
304f2699491SMichael Ellerman event->hw.idx = i;
305f2699491SMichael Ellerman cpuhw->event[i] = event;
306f2699491SMichael Ellerman ++cpuhw->n_events;
307f2699491SMichael Ellerman
308f2699491SMichael Ellerman val = 0;
309f2699491SMichael Ellerman if (event->hw.sample_period) {
310f2699491SMichael Ellerman s64 left = local64_read(&event->hw.period_left);
311f2699491SMichael Ellerman if (left < 0x80000000L)
312f2699491SMichael Ellerman val = 0x80000000L - left;
313f2699491SMichael Ellerman }
314f2699491SMichael Ellerman local64_set(&event->hw.prev_count, val);
315f2699491SMichael Ellerman
3160d7d9b3aSAlexandru-Cezar Sardan if (unlikely(!(flags & PERF_EF_START))) {
317f2699491SMichael Ellerman event->hw.state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
318f2699491SMichael Ellerman val = 0;
3190d7d9b3aSAlexandru-Cezar Sardan } else {
3200d7d9b3aSAlexandru-Cezar Sardan event->hw.state &= ~(PERF_HES_STOPPED | PERF_HES_UPTODATE);
321f2699491SMichael Ellerman }
322f2699491SMichael Ellerman
323f2699491SMichael Ellerman write_pmc(i, val);
324f2699491SMichael Ellerman perf_event_update_userpage(event);
325f2699491SMichael Ellerman
326f2699491SMichael Ellerman write_pmlcb(i, event->hw.config >> 32);
327f2699491SMichael Ellerman write_pmlca(i, event->hw.config_base);
328f2699491SMichael Ellerman
329f2699491SMichael Ellerman ret = 0;
330f2699491SMichael Ellerman out:
331f2699491SMichael Ellerman put_cpu_var(cpu_hw_events);
332f2699491SMichael Ellerman perf_pmu_enable(event->pmu);
333f2699491SMichael Ellerman return ret;
334f2699491SMichael Ellerman }
335f2699491SMichael Ellerman
336f2699491SMichael Ellerman /* context locked on entry */
fsl_emb_pmu_del(struct perf_event * event,int flags)337f2699491SMichael Ellerman static void fsl_emb_pmu_del(struct perf_event *event, int flags)
338f2699491SMichael Ellerman {
339f2699491SMichael Ellerman struct cpu_hw_events *cpuhw;
340f2699491SMichael Ellerman int i = event->hw.idx;
341f2699491SMichael Ellerman
342f2699491SMichael Ellerman perf_pmu_disable(event->pmu);
343f2699491SMichael Ellerman if (i < 0)
344f2699491SMichael Ellerman goto out;
345f2699491SMichael Ellerman
346f2699491SMichael Ellerman fsl_emb_pmu_read(event);
347f2699491SMichael Ellerman
348f2699491SMichael Ellerman cpuhw = &get_cpu_var(cpu_hw_events);
349f2699491SMichael Ellerman
350f2699491SMichael Ellerman WARN_ON(event != cpuhw->event[event->hw.idx]);
351f2699491SMichael Ellerman
352f2699491SMichael Ellerman write_pmlca(i, 0);
353f2699491SMichael Ellerman write_pmlcb(i, 0);
354f2699491SMichael Ellerman write_pmc(i, 0);
355f2699491SMichael Ellerman
356f2699491SMichael Ellerman cpuhw->event[i] = NULL;
357f2699491SMichael Ellerman event->hw.idx = -1;
358f2699491SMichael Ellerman
359f2699491SMichael Ellerman /*
360f2699491SMichael Ellerman * TODO: if at least one restricted event exists, and we
361f2699491SMichael Ellerman * just freed up a non-restricted-capable counter, and
362f2699491SMichael Ellerman * there is a restricted-capable counter occupied by
363f2699491SMichael Ellerman * a non-restricted event, migrate that event to the
364f2699491SMichael Ellerman * vacated counter.
365f2699491SMichael Ellerman */
366f2699491SMichael Ellerman
367f2699491SMichael Ellerman cpuhw->n_events--;
368f2699491SMichael Ellerman
369f2699491SMichael Ellerman out:
370f2699491SMichael Ellerman perf_pmu_enable(event->pmu);
371f2699491SMichael Ellerman put_cpu_var(cpu_hw_events);
372f2699491SMichael Ellerman }
373f2699491SMichael Ellerman
fsl_emb_pmu_start(struct perf_event * event,int ef_flags)374f2699491SMichael Ellerman static void fsl_emb_pmu_start(struct perf_event *event, int ef_flags)
375f2699491SMichael Ellerman {
376f2699491SMichael Ellerman unsigned long flags;
377d2caa3ceSTom Huynh unsigned long val;
378f2699491SMichael Ellerman s64 left;
379f2699491SMichael Ellerman
380f2699491SMichael Ellerman if (event->hw.idx < 0 || !event->hw.sample_period)
381f2699491SMichael Ellerman return;
382f2699491SMichael Ellerman
383f2699491SMichael Ellerman if (!(event->hw.state & PERF_HES_STOPPED))
384f2699491SMichael Ellerman return;
385f2699491SMichael Ellerman
386f2699491SMichael Ellerman if (ef_flags & PERF_EF_RELOAD)
387f2699491SMichael Ellerman WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE));
388f2699491SMichael Ellerman
389f2699491SMichael Ellerman local_irq_save(flags);
390f2699491SMichael Ellerman perf_pmu_disable(event->pmu);
391f2699491SMichael Ellerman
392f2699491SMichael Ellerman event->hw.state = 0;
393f2699491SMichael Ellerman left = local64_read(&event->hw.period_left);
394d2caa3ceSTom Huynh val = 0;
395d2caa3ceSTom Huynh if (left < 0x80000000L)
396d2caa3ceSTom Huynh val = 0x80000000L - left;
397d2caa3ceSTom Huynh write_pmc(event->hw.idx, val);
398f2699491SMichael Ellerman
399f2699491SMichael Ellerman perf_event_update_userpage(event);
400f2699491SMichael Ellerman perf_pmu_enable(event->pmu);
401f2699491SMichael Ellerman local_irq_restore(flags);
402f2699491SMichael Ellerman }
403f2699491SMichael Ellerman
fsl_emb_pmu_stop(struct perf_event * event,int ef_flags)404f2699491SMichael Ellerman static void fsl_emb_pmu_stop(struct perf_event *event, int ef_flags)
405f2699491SMichael Ellerman {
406f2699491SMichael Ellerman unsigned long flags;
407f2699491SMichael Ellerman
408f2699491SMichael Ellerman if (event->hw.idx < 0 || !event->hw.sample_period)
409f2699491SMichael Ellerman return;
410f2699491SMichael Ellerman
411f2699491SMichael Ellerman if (event->hw.state & PERF_HES_STOPPED)
412f2699491SMichael Ellerman return;
413f2699491SMichael Ellerman
414f2699491SMichael Ellerman local_irq_save(flags);
415f2699491SMichael Ellerman perf_pmu_disable(event->pmu);
416f2699491SMichael Ellerman
417f2699491SMichael Ellerman fsl_emb_pmu_read(event);
418f2699491SMichael Ellerman event->hw.state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
419f2699491SMichael Ellerman write_pmc(event->hw.idx, 0);
420f2699491SMichael Ellerman
421f2699491SMichael Ellerman perf_event_update_userpage(event);
422f2699491SMichael Ellerman perf_pmu_enable(event->pmu);
423f2699491SMichael Ellerman local_irq_restore(flags);
424f2699491SMichael Ellerman }
425f2699491SMichael Ellerman
426f2699491SMichael Ellerman /*
427f2699491SMichael Ellerman * Release the PMU if this is the last perf_event.
428f2699491SMichael Ellerman */
hw_perf_event_destroy(struct perf_event * event)429f2699491SMichael Ellerman static void hw_perf_event_destroy(struct perf_event *event)
430f2699491SMichael Ellerman {
431f2699491SMichael Ellerman if (!atomic_add_unless(&num_events, -1, 1)) {
432f2699491SMichael Ellerman mutex_lock(&pmc_reserve_mutex);
433f2699491SMichael Ellerman if (atomic_dec_return(&num_events) == 0)
434f2699491SMichael Ellerman release_pmc_hardware();
435f2699491SMichael Ellerman mutex_unlock(&pmc_reserve_mutex);
436f2699491SMichael Ellerman }
437f2699491SMichael Ellerman }
438f2699491SMichael Ellerman
439f2699491SMichael Ellerman /*
440f2699491SMichael Ellerman * Translate a generic cache event_id config to a raw event_id code.
441f2699491SMichael Ellerman */
hw_perf_cache_event(u64 config,u64 * eventp)442f2699491SMichael Ellerman static int hw_perf_cache_event(u64 config, u64 *eventp)
443f2699491SMichael Ellerman {
444f2699491SMichael Ellerman unsigned long type, op, result;
445f2699491SMichael Ellerman int ev;
446f2699491SMichael Ellerman
447f2699491SMichael Ellerman if (!ppmu->cache_events)
448f2699491SMichael Ellerman return -EINVAL;
449f2699491SMichael Ellerman
450f2699491SMichael Ellerman /* unpack config */
451f2699491SMichael Ellerman type = config & 0xff;
452f2699491SMichael Ellerman op = (config >> 8) & 0xff;
453f2699491SMichael Ellerman result = (config >> 16) & 0xff;
454f2699491SMichael Ellerman
455f2699491SMichael Ellerman if (type >= PERF_COUNT_HW_CACHE_MAX ||
456f2699491SMichael Ellerman op >= PERF_COUNT_HW_CACHE_OP_MAX ||
457f2699491SMichael Ellerman result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
458f2699491SMichael Ellerman return -EINVAL;
459f2699491SMichael Ellerman
460f2699491SMichael Ellerman ev = (*ppmu->cache_events)[type][op][result];
461f2699491SMichael Ellerman if (ev == 0)
462f2699491SMichael Ellerman return -EOPNOTSUPP;
463f2699491SMichael Ellerman if (ev == -1)
464f2699491SMichael Ellerman return -EINVAL;
465f2699491SMichael Ellerman *eventp = ev;
466f2699491SMichael Ellerman return 0;
467f2699491SMichael Ellerman }
468f2699491SMichael Ellerman
fsl_emb_pmu_event_init(struct perf_event * event)469f2699491SMichael Ellerman static int fsl_emb_pmu_event_init(struct perf_event *event)
470f2699491SMichael Ellerman {
471f2699491SMichael Ellerman u64 ev;
472f2699491SMichael Ellerman struct perf_event *events[MAX_HWEVENTS];
473f2699491SMichael Ellerman int n;
474f2699491SMichael Ellerman int err;
475f2699491SMichael Ellerman int num_restricted;
476f2699491SMichael Ellerman int i;
477f2699491SMichael Ellerman
47896c3c9e7SCatalin Udma if (ppmu->n_counter > MAX_HWEVENTS) {
47996c3c9e7SCatalin Udma WARN(1, "No. of perf counters (%d) is higher than max array size(%d)\n",
48096c3c9e7SCatalin Udma ppmu->n_counter, MAX_HWEVENTS);
48196c3c9e7SCatalin Udma ppmu->n_counter = MAX_HWEVENTS;
48296c3c9e7SCatalin Udma }
48396c3c9e7SCatalin Udma
484f2699491SMichael Ellerman switch (event->attr.type) {
485f2699491SMichael Ellerman case PERF_TYPE_HARDWARE:
486f2699491SMichael Ellerman ev = event->attr.config;
487f2699491SMichael Ellerman if (ev >= ppmu->n_generic || ppmu->generic_events[ev] == 0)
488f2699491SMichael Ellerman return -EOPNOTSUPP;
489f2699491SMichael Ellerman ev = ppmu->generic_events[ev];
490f2699491SMichael Ellerman break;
491f2699491SMichael Ellerman
492f2699491SMichael Ellerman case PERF_TYPE_HW_CACHE:
493f2699491SMichael Ellerman err = hw_perf_cache_event(event->attr.config, &ev);
494f2699491SMichael Ellerman if (err)
495f2699491SMichael Ellerman return err;
496f2699491SMichael Ellerman break;
497f2699491SMichael Ellerman
498f2699491SMichael Ellerman case PERF_TYPE_RAW:
499f2699491SMichael Ellerman ev = event->attr.config;
500f2699491SMichael Ellerman break;
501f2699491SMichael Ellerman
502f2699491SMichael Ellerman default:
503f2699491SMichael Ellerman return -ENOENT;
504f2699491SMichael Ellerman }
505f2699491SMichael Ellerman
506f2699491SMichael Ellerman event->hw.config = ppmu->xlate_event(ev);
507f2699491SMichael Ellerman if (!(event->hw.config & FSL_EMB_EVENT_VALID))
508f2699491SMichael Ellerman return -EINVAL;
509f2699491SMichael Ellerman
510f2699491SMichael Ellerman /*
511f2699491SMichael Ellerman * If this is in a group, check if it can go on with all the
512f2699491SMichael Ellerman * other hardware events in the group. We assume the event
513f2699491SMichael Ellerman * hasn't been linked into its leader's sibling list at this point.
514f2699491SMichael Ellerman */
515f2699491SMichael Ellerman n = 0;
516f2699491SMichael Ellerman if (event->group_leader != event) {
517f2699491SMichael Ellerman n = collect_events(event->group_leader,
518f2699491SMichael Ellerman ppmu->n_counter - 1, events);
519f2699491SMichael Ellerman if (n < 0)
520f2699491SMichael Ellerman return -EINVAL;
521f2699491SMichael Ellerman }
522f2699491SMichael Ellerman
523f2699491SMichael Ellerman if (event->hw.config & FSL_EMB_EVENT_RESTRICTED) {
524f2699491SMichael Ellerman num_restricted = 0;
525f2699491SMichael Ellerman for (i = 0; i < n; i++) {
526f2699491SMichael Ellerman if (events[i]->hw.config & FSL_EMB_EVENT_RESTRICTED)
527f2699491SMichael Ellerman num_restricted++;
528f2699491SMichael Ellerman }
529f2699491SMichael Ellerman
530f2699491SMichael Ellerman if (num_restricted >= ppmu->n_restricted)
531f2699491SMichael Ellerman return -EINVAL;
532f2699491SMichael Ellerman }
533f2699491SMichael Ellerman
534f2699491SMichael Ellerman event->hw.idx = -1;
535f2699491SMichael Ellerman
536f2699491SMichael Ellerman event->hw.config_base = PMLCA_CE | PMLCA_FCM1 |
537f2699491SMichael Ellerman (u32)((ev << 16) & PMLCA_EVENT_MASK);
538f2699491SMichael Ellerman
539f2699491SMichael Ellerman if (event->attr.exclude_user)
540f2699491SMichael Ellerman event->hw.config_base |= PMLCA_FCU;
541f2699491SMichael Ellerman if (event->attr.exclude_kernel)
542f2699491SMichael Ellerman event->hw.config_base |= PMLCA_FCS;
543f2699491SMichael Ellerman if (event->attr.exclude_idle)
544f2699491SMichael Ellerman return -ENOTSUPP;
545f2699491SMichael Ellerman
546f2699491SMichael Ellerman event->hw.last_period = event->hw.sample_period;
547f2699491SMichael Ellerman local64_set(&event->hw.period_left, event->hw.last_period);
548f2699491SMichael Ellerman
549f2699491SMichael Ellerman /*
550f2699491SMichael Ellerman * See if we need to reserve the PMU.
551f2699491SMichael Ellerman * If no events are currently in use, then we have to take a
552f2699491SMichael Ellerman * mutex to ensure that we don't race with another task doing
553f2699491SMichael Ellerman * reserve_pmc_hardware or release_pmc_hardware.
554f2699491SMichael Ellerman */
555f2699491SMichael Ellerman err = 0;
556f2699491SMichael Ellerman if (!atomic_inc_not_zero(&num_events)) {
557f2699491SMichael Ellerman mutex_lock(&pmc_reserve_mutex);
558f2699491SMichael Ellerman if (atomic_read(&num_events) == 0 &&
559f2699491SMichael Ellerman reserve_pmc_hardware(perf_event_interrupt))
560f2699491SMichael Ellerman err = -EBUSY;
561f2699491SMichael Ellerman else
562f2699491SMichael Ellerman atomic_inc(&num_events);
563f2699491SMichael Ellerman mutex_unlock(&pmc_reserve_mutex);
564f2699491SMichael Ellerman
565f2699491SMichael Ellerman mtpmr(PMRN_PMGC0, PMGC0_FAC);
566f2699491SMichael Ellerman isync();
567f2699491SMichael Ellerman }
568f2699491SMichael Ellerman event->destroy = hw_perf_event_destroy;
569f2699491SMichael Ellerman
570f2699491SMichael Ellerman return err;
571f2699491SMichael Ellerman }
572f2699491SMichael Ellerman
573f2699491SMichael Ellerman static struct pmu fsl_emb_pmu = {
574f2699491SMichael Ellerman .pmu_enable = fsl_emb_pmu_enable,
575f2699491SMichael Ellerman .pmu_disable = fsl_emb_pmu_disable,
576f2699491SMichael Ellerman .event_init = fsl_emb_pmu_event_init,
577f2699491SMichael Ellerman .add = fsl_emb_pmu_add,
578f2699491SMichael Ellerman .del = fsl_emb_pmu_del,
579f2699491SMichael Ellerman .start = fsl_emb_pmu_start,
580f2699491SMichael Ellerman .stop = fsl_emb_pmu_stop,
581f2699491SMichael Ellerman .read = fsl_emb_pmu_read,
582f2699491SMichael Ellerman };
583f2699491SMichael Ellerman
584f2699491SMichael Ellerman /*
585f2699491SMichael Ellerman * A counter has overflowed; update its count and record
586f2699491SMichael Ellerman * things if requested. Note that interrupts are hard-disabled
587f2699491SMichael Ellerman * here so there is no possibility of being interrupted.
588f2699491SMichael Ellerman */
record_and_restart(struct perf_event * event,unsigned long val,struct pt_regs * regs)589f2699491SMichael Ellerman static void record_and_restart(struct perf_event *event, unsigned long val,
590f2699491SMichael Ellerman struct pt_regs *regs)
591f2699491SMichael Ellerman {
592f2699491SMichael Ellerman u64 period = event->hw.sample_period;
593f2699491SMichael Ellerman s64 prev, delta, left;
594f2699491SMichael Ellerman int record = 0;
595f2699491SMichael Ellerman
596f2699491SMichael Ellerman if (event->hw.state & PERF_HES_STOPPED) {
597f2699491SMichael Ellerman write_pmc(event->hw.idx, 0);
598f2699491SMichael Ellerman return;
599f2699491SMichael Ellerman }
600f2699491SMichael Ellerman
601f2699491SMichael Ellerman /* we don't have to worry about interrupts here */
602f2699491SMichael Ellerman prev = local64_read(&event->hw.prev_count);
603f2699491SMichael Ellerman delta = (val - prev) & 0xfffffffful;
604f2699491SMichael Ellerman local64_add(delta, &event->count);
605f2699491SMichael Ellerman
606f2699491SMichael Ellerman /*
607f2699491SMichael Ellerman * See if the total period for this event has expired,
608f2699491SMichael Ellerman * and update for the next period.
609f2699491SMichael Ellerman */
610f2699491SMichael Ellerman val = 0;
611f2699491SMichael Ellerman left = local64_read(&event->hw.period_left) - delta;
612f2699491SMichael Ellerman if (period) {
613f2699491SMichael Ellerman if (left <= 0) {
614f2699491SMichael Ellerman left += period;
615f2699491SMichael Ellerman if (left <= 0)
616f2699491SMichael Ellerman left = period;
617f2699491SMichael Ellerman record = 1;
618f2699491SMichael Ellerman event->hw.last_period = event->hw.sample_period;
619f2699491SMichael Ellerman }
620f2699491SMichael Ellerman if (left < 0x80000000LL)
621f2699491SMichael Ellerman val = 0x80000000LL - left;
622f2699491SMichael Ellerman }
623f2699491SMichael Ellerman
624f2699491SMichael Ellerman write_pmc(event->hw.idx, val);
625f2699491SMichael Ellerman local64_set(&event->hw.prev_count, val);
626f2699491SMichael Ellerman local64_set(&event->hw.period_left, left);
627f2699491SMichael Ellerman perf_event_update_userpage(event);
628f2699491SMichael Ellerman
629f2699491SMichael Ellerman /*
630f2699491SMichael Ellerman * Finally record data if requested.
631f2699491SMichael Ellerman */
632f2699491SMichael Ellerman if (record) {
633f2699491SMichael Ellerman struct perf_sample_data data;
634f2699491SMichael Ellerman
635fd0d000bSRobert Richter perf_sample_data_init(&data, 0, event->hw.last_period);
636f2699491SMichael Ellerman
637f2699491SMichael Ellerman if (perf_event_overflow(event, &data, regs))
638f2699491SMichael Ellerman fsl_emb_pmu_stop(event, 0);
639f2699491SMichael Ellerman }
640f2699491SMichael Ellerman }
641f2699491SMichael Ellerman
perf_event_interrupt(struct pt_regs * regs)642f2699491SMichael Ellerman static void perf_event_interrupt(struct pt_regs *regs)
643f2699491SMichael Ellerman {
644f2699491SMichael Ellerman int i;
64569111bacSChristoph Lameter struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events);
646f2699491SMichael Ellerman struct perf_event *event;
647f2699491SMichael Ellerman unsigned long val;
648f2699491SMichael Ellerman
649f2699491SMichael Ellerman for (i = 0; i < ppmu->n_counter; ++i) {
650f2699491SMichael Ellerman event = cpuhw->event[i];
651f2699491SMichael Ellerman
652f2699491SMichael Ellerman val = read_pmc(i);
653f2699491SMichael Ellerman if ((int)val < 0) {
654f2699491SMichael Ellerman if (event) {
655f2699491SMichael Ellerman /* event has overflowed */
656f2699491SMichael Ellerman record_and_restart(event, val, regs);
657f2699491SMichael Ellerman } else {
658f2699491SMichael Ellerman /*
659f2699491SMichael Ellerman * Disabled counter is negative,
660f2699491SMichael Ellerman * reset it just in case.
661f2699491SMichael Ellerman */
662f2699491SMichael Ellerman write_pmc(i, 0);
663f2699491SMichael Ellerman }
664f2699491SMichael Ellerman }
665f2699491SMichael Ellerman }
666f2699491SMichael Ellerman
667f2699491SMichael Ellerman /* PMM will keep counters frozen until we return from the interrupt. */
668f2699491SMichael Ellerman mtmsr(mfmsr() | MSR_PMM);
669f2699491SMichael Ellerman mtpmr(PMRN_PMGC0, PMGC0_PMIE | PMGC0_FCECE);
670f2699491SMichael Ellerman isync();
671f2699491SMichael Ellerman }
672f2699491SMichael Ellerman
fsl_emb_pmu_prepare_cpu(unsigned int cpu)673*34daf445SChristophe Leroy static int fsl_emb_pmu_prepare_cpu(unsigned int cpu)
674f2699491SMichael Ellerman {
675f2699491SMichael Ellerman struct cpu_hw_events *cpuhw = &per_cpu(cpu_hw_events, cpu);
676f2699491SMichael Ellerman
677f2699491SMichael Ellerman memset(cpuhw, 0, sizeof(*cpuhw));
678*34daf445SChristophe Leroy
679*34daf445SChristophe Leroy return 0;
680f2699491SMichael Ellerman }
681f2699491SMichael Ellerman
register_fsl_emb_pmu(struct fsl_emb_pmu * pmu)682f2699491SMichael Ellerman int register_fsl_emb_pmu(struct fsl_emb_pmu *pmu)
683f2699491SMichael Ellerman {
684f2699491SMichael Ellerman if (ppmu)
685f2699491SMichael Ellerman return -EBUSY; /* something's already registered */
686f2699491SMichael Ellerman
687f2699491SMichael Ellerman ppmu = pmu;
688f2699491SMichael Ellerman pr_info("%s performance monitor hardware support registered\n",
689f2699491SMichael Ellerman pmu->name);
690f2699491SMichael Ellerman
691f2699491SMichael Ellerman perf_pmu_register(&fsl_emb_pmu, "cpu", PERF_TYPE_RAW);
692*34daf445SChristophe Leroy cpuhp_setup_state(CPUHP_PERF_POWER, "perf/powerpc:prepare",
693*34daf445SChristophe Leroy fsl_emb_pmu_prepare_cpu, NULL);
694f2699491SMichael Ellerman
695f2699491SMichael Ellerman return 0;
696f2699491SMichael Ellerman }
697