1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright 2014, Michael Ellerman, IBM Corp.
4  */
5 
6 #define _GNU_SOURCE	/* For CPU_ZERO etc. */
7 
8 #include <sched.h>
9 #include <sys/wait.h>
10 #include <setjmp.h>
11 #include <signal.h>
12 #include <stdio.h>
13 #include <stdlib.h>
14 #include <string.h>
15 #include <sys/ioctl.h>
16 
17 #include "trace.h"
18 #include "ebb.h"
19 
20 
21 void (*ebb_user_func)(void);
22 
23 void ebb_hook(void)
24 {
25 	if (ebb_user_func)
26 		ebb_user_func();
27 }
28 
29 struct ebb_state ebb_state;
30 
31 u64 sample_period = 0x40000000ull;
32 
33 void reset_ebb_with_clear_mask(unsigned long mmcr0_clear_mask)
34 {
35 	u64 val;
36 
37 	/* 2) clear MMCR0[PMAO] - docs say BESCR[PMEO] should do this */
38 	/* 3) set MMCR0[PMAE]	- docs say BESCR[PME] should do this */
39 	val = mfspr(SPRN_MMCR0);
40 	mtspr(SPRN_MMCR0, (val & ~mmcr0_clear_mask) | MMCR0_PMAE);
41 
42 	/* 4) clear BESCR[PMEO] */
43 	mtspr(SPRN_BESCRR, BESCR_PMEO);
44 
45 	/* 5) set BESCR[PME] */
46 	mtspr(SPRN_BESCRS, BESCR_PME);
47 
48 	/* 6) rfebb 1 - done in our caller */
49 }
50 
51 void reset_ebb(void)
52 {
53 	reset_ebb_with_clear_mask(MMCR0_PMAO | MMCR0_FC);
54 }
55 
56 /* Called outside of the EBB handler to check MMCR0 is sane */
57 int ebb_check_mmcr0(void)
58 {
59 	u64 val;
60 
61 	val = mfspr(SPRN_MMCR0);
62 	if ((val & (MMCR0_FC | MMCR0_PMAO)) == MMCR0_FC) {
63 		/* It's OK if we see FC & PMAO, but not FC by itself */
64 		printf("Outside of loop, only FC set 0x%llx\n", val);
65 		return 1;
66 	}
67 
68 	return 0;
69 }
70 
71 bool ebb_check_count(int pmc, u64 sample_period, int fudge)
72 {
73 	u64 count, upper, lower;
74 
75 	count = ebb_state.stats.pmc_count[PMC_INDEX(pmc)];
76 
77 	lower = ebb_state.stats.ebb_count * (sample_period - fudge);
78 
79 	if (count < lower) {
80 		printf("PMC%d count (0x%llx) below lower limit 0x%llx (-0x%llx)\n",
81 			pmc, count, lower, lower - count);
82 		return false;
83 	}
84 
85 	upper = ebb_state.stats.ebb_count * (sample_period + fudge);
86 
87 	if (count > upper) {
88 		printf("PMC%d count (0x%llx) above upper limit 0x%llx (+0x%llx)\n",
89 			pmc, count, upper, count - upper);
90 		return false;
91 	}
92 
93 	printf("PMC%d count (0x%llx) is between 0x%llx and 0x%llx delta +0x%llx/-0x%llx\n",
94 		pmc, count, lower, upper, count - lower, upper - count);
95 
96 	return true;
97 }
98 
99 void standard_ebb_callee(void)
100 {
101 	int found, i;
102 	u64 val;
103 
104 	val = mfspr(SPRN_BESCR);
105 	if (!(val & BESCR_PMEO)) {
106 		ebb_state.stats.spurious++;
107 		goto out;
108 	}
109 
110 	ebb_state.stats.ebb_count++;
111 	trace_log_counter(ebb_state.trace, ebb_state.stats.ebb_count);
112 
113 	val = mfspr(SPRN_MMCR0);
114 	trace_log_reg(ebb_state.trace, SPRN_MMCR0, val);
115 
116 	found = 0;
117 	for (i = 1; i <= 6; i++) {
118 		if (ebb_state.pmc_enable[PMC_INDEX(i)])
119 			found += count_pmc(i, sample_period);
120 	}
121 
122 	if (!found)
123 		ebb_state.stats.no_overflow++;
124 
125 out:
126 	reset_ebb();
127 }
128 
129 extern void ebb_handler(void);
130 
131 void setup_ebb_handler(void (*callee)(void))
132 {
133 	u64 entry;
134 
135 #if defined(_CALL_ELF) && _CALL_ELF == 2
136 	entry = (u64)ebb_handler;
137 #else
138 	struct opd
139 	{
140 	    u64 entry;
141 	    u64 toc;
142 	} *opd;
143 
144 	opd = (struct opd *)ebb_handler;
145 	entry = opd->entry;
146 #endif
147 	printf("EBB Handler is at %#llx\n", entry);
148 
149 	ebb_user_func = callee;
150 
151 	/* Ensure ebb_user_func is set before we set the handler */
152 	mb();
153 	mtspr(SPRN_EBBHR, entry);
154 
155 	/* Make sure the handler is set before we return */
156 	mb();
157 }
158 
159 void clear_ebb_stats(void)
160 {
161 	memset(&ebb_state.stats, 0, sizeof(ebb_state.stats));
162 }
163 
164 void dump_summary_ebb_state(void)
165 {
166 	printf("ebb_state:\n"			\
167 	       "  ebb_count    = %d\n"		\
168 	       "  spurious     = %d\n"		\
169 	       "  negative     = %d\n"		\
170 	       "  no_overflow  = %d\n"		\
171 	       "  pmc[1] count = 0x%llx\n"	\
172 	       "  pmc[2] count = 0x%llx\n"	\
173 	       "  pmc[3] count = 0x%llx\n"	\
174 	       "  pmc[4] count = 0x%llx\n"	\
175 	       "  pmc[5] count = 0x%llx\n"	\
176 	       "  pmc[6] count = 0x%llx\n",
177 		ebb_state.stats.ebb_count, ebb_state.stats.spurious,
178 		ebb_state.stats.negative, ebb_state.stats.no_overflow,
179 		ebb_state.stats.pmc_count[0], ebb_state.stats.pmc_count[1],
180 		ebb_state.stats.pmc_count[2], ebb_state.stats.pmc_count[3],
181 		ebb_state.stats.pmc_count[4], ebb_state.stats.pmc_count[5]);
182 }
183 
184 static char *decode_mmcr0(u32 value)
185 {
186 	static char buf[16];
187 
188 	buf[0] = '\0';
189 
190 	if (value & (1 << 31))
191 		strcat(buf, "FC ");
192 	if (value & (1 << 26))
193 		strcat(buf, "PMAE ");
194 	if (value & (1 << 7))
195 		strcat(buf, "PMAO ");
196 
197 	return buf;
198 }
199 
200 static char *decode_bescr(u64 value)
201 {
202 	static char buf[16];
203 
204 	buf[0] = '\0';
205 
206 	if (value & (1ull << 63))
207 		strcat(buf, "GE ");
208 	if (value & (1ull << 32))
209 		strcat(buf, "PMAE ");
210 	if (value & 1)
211 		strcat(buf, "PMAO ");
212 
213 	return buf;
214 }
215 
216 void dump_ebb_hw_state(void)
217 {
218 	u64 bescr;
219 	u32 mmcr0;
220 
221 	mmcr0 = mfspr(SPRN_MMCR0);
222 	bescr = mfspr(SPRN_BESCR);
223 
224 	printf("HW state:\n"		\
225 	       "MMCR0 0x%016x %s\n"	\
226 	       "MMCR2 0x%016lx\n"	\
227 	       "EBBHR 0x%016lx\n"	\
228 	       "BESCR 0x%016llx %s\n"	\
229 	       "PMC1  0x%016lx\n"	\
230 	       "PMC2  0x%016lx\n"	\
231 	       "PMC3  0x%016lx\n"	\
232 	       "PMC4  0x%016lx\n"	\
233 	       "PMC5  0x%016lx\n"	\
234 	       "PMC6  0x%016lx\n"	\
235 	       "SIAR  0x%016lx\n",
236 	       mmcr0, decode_mmcr0(mmcr0), mfspr(SPRN_MMCR2),
237 	       mfspr(SPRN_EBBHR), bescr, decode_bescr(bescr),
238 	       mfspr(SPRN_PMC1), mfspr(SPRN_PMC2), mfspr(SPRN_PMC3),
239 	       mfspr(SPRN_PMC4), mfspr(SPRN_PMC5), mfspr(SPRN_PMC6),
240 	       mfspr(SPRN_SIAR));
241 }
242 
243 void dump_ebb_state(void)
244 {
245 	dump_summary_ebb_state();
246 
247 	dump_ebb_hw_state();
248 
249 	trace_buffer_print(ebb_state.trace);
250 }
251 
252 int count_pmc(int pmc, uint32_t sample_period)
253 {
254 	uint32_t start_value;
255 	u64 val;
256 
257 	/* 0) Read PMC */
258 	start_value = pmc_sample_period(sample_period);
259 
260 	val = read_pmc(pmc);
261 	if (val < start_value)
262 		ebb_state.stats.negative++;
263 	else
264 		ebb_state.stats.pmc_count[PMC_INDEX(pmc)] += val - start_value;
265 
266 	trace_log_reg(ebb_state.trace, SPRN_PMC1 + pmc - 1, val);
267 
268 	/* 1) Reset PMC */
269 	write_pmc(pmc, start_value);
270 
271 	/* Report if we overflowed */
272 	return val >= COUNTER_OVERFLOW;
273 }
274 
275 int ebb_event_enable(struct event *e)
276 {
277 	int rc;
278 
279 	/* Ensure any SPR writes are ordered vs us */
280 	mb();
281 
282 	rc = ioctl(e->fd, PERF_EVENT_IOC_ENABLE);
283 	if (rc)
284 		return rc;
285 
286 	rc = event_read(e);
287 
288 	/* Ditto */
289 	mb();
290 
291 	return rc;
292 }
293 
294 void ebb_freeze_pmcs(void)
295 {
296 	mtspr(SPRN_MMCR0, mfspr(SPRN_MMCR0) | MMCR0_FC);
297 	mb();
298 }
299 
300 void ebb_unfreeze_pmcs(void)
301 {
302 	/* Unfreeze counters */
303 	mtspr(SPRN_MMCR0, mfspr(SPRN_MMCR0) & ~MMCR0_FC);
304 	mb();
305 }
306 
307 void ebb_global_enable(void)
308 {
309 	/* Enable EBBs globally and PMU EBBs */
310 	mtspr(SPRN_BESCR, 0x8000000100000000ull);
311 	mb();
312 }
313 
314 void ebb_global_disable(void)
315 {
316 	/* Disable EBBs & freeze counters, events are still scheduled */
317 	mtspr(SPRN_BESCRR, BESCR_PME);
318 	mb();
319 }
320 
321 bool ebb_is_supported(void)
322 {
323 #ifdef PPC_FEATURE2_EBB
324 	/* EBB requires at least POWER8 */
325 	return have_hwcap2(PPC_FEATURE2_EBB);
326 #else
327 	return false;
328 #endif
329 }
330 
331 void event_ebb_init(struct event *e)
332 {
333 	e->attr.config |= (1ull << 63);
334 }
335 
336 void event_bhrb_init(struct event *e, unsigned ifm)
337 {
338 	e->attr.config |= (1ull << 62) | ((u64)ifm << 60);
339 }
340 
341 void event_leader_ebb_init(struct event *e)
342 {
343 	event_ebb_init(e);
344 
345 	e->attr.exclusive = 1;
346 	e->attr.pinned = 1;
347 }
348 
349 int ebb_child(union pipe read_pipe, union pipe write_pipe)
350 {
351 	struct event event;
352 	uint64_t val;
353 
354 	FAIL_IF(wait_for_parent(read_pipe));
355 
356 	event_init_named(&event, 0x1001e, "cycles");
357 	event_leader_ebb_init(&event);
358 
359 	event.attr.exclude_kernel = 1;
360 	event.attr.exclude_hv = 1;
361 	event.attr.exclude_idle = 1;
362 
363 	FAIL_IF(event_open(&event));
364 
365 	ebb_enable_pmc_counting(1);
366 	setup_ebb_handler(standard_ebb_callee);
367 	ebb_global_enable();
368 
369 	FAIL_IF(event_enable(&event));
370 
371 	if (event_read(&event)) {
372 		/*
373 		 * Some tests expect to fail here, so don't report an error on
374 		 * this line, and return a distinguisable error code. Tell the
375 		 * parent an error happened.
376 		 */
377 		notify_parent_of_error(write_pipe);
378 		return 2;
379 	}
380 
381 	mtspr(SPRN_PMC1, pmc_sample_period(sample_period));
382 
383 	FAIL_IF(notify_parent(write_pipe));
384 	FAIL_IF(wait_for_parent(read_pipe));
385 	FAIL_IF(notify_parent(write_pipe));
386 
387 	while (ebb_state.stats.ebb_count < 20) {
388 		FAIL_IF(core_busy_loop());
389 
390 		/* To try and hit SIGILL case */
391 		val  = mfspr(SPRN_MMCRA);
392 		val |= mfspr(SPRN_MMCR2);
393 		val |= mfspr(SPRN_MMCR0);
394 	}
395 
396 	ebb_global_disable();
397 	ebb_freeze_pmcs();
398 
399 	dump_ebb_state();
400 
401 	event_close(&event);
402 
403 	FAIL_IF(ebb_state.stats.ebb_count == 0);
404 
405 	return 0;
406 }
407 
408 static jmp_buf setjmp_env;
409 
410 static void sigill_handler(int signal)
411 {
412 	printf("Took sigill\n");
413 	longjmp(setjmp_env, 1);
414 }
415 
416 static struct sigaction sigill_action = {
417 	.sa_handler = sigill_handler,
418 };
419 
420 int catch_sigill(void (*func)(void))
421 {
422 	if (sigaction(SIGILL, &sigill_action, NULL)) {
423 		perror("sigaction");
424 		return 1;
425 	}
426 
427 	if (setjmp(setjmp_env) == 0) {
428 		func();
429 		return 1;
430 	}
431 
432 	return 0;
433 }
434 
435 void write_pmc1(void)
436 {
437 	mtspr(SPRN_PMC1, 0);
438 }
439 
440 void write_pmc(int pmc, u64 value)
441 {
442 	switch (pmc) {
443 		case 1: mtspr(SPRN_PMC1, value); break;
444 		case 2: mtspr(SPRN_PMC2, value); break;
445 		case 3: mtspr(SPRN_PMC3, value); break;
446 		case 4: mtspr(SPRN_PMC4, value); break;
447 		case 5: mtspr(SPRN_PMC5, value); break;
448 		case 6: mtspr(SPRN_PMC6, value); break;
449 	}
450 }
451 
452 u64 read_pmc(int pmc)
453 {
454 	switch (pmc) {
455 		case 1: return mfspr(SPRN_PMC1);
456 		case 2: return mfspr(SPRN_PMC2);
457 		case 3: return mfspr(SPRN_PMC3);
458 		case 4: return mfspr(SPRN_PMC4);
459 		case 5: return mfspr(SPRN_PMC5);
460 		case 6: return mfspr(SPRN_PMC6);
461 	}
462 
463 	return 0;
464 }
465 
466 static void term_handler(int signal)
467 {
468 	dump_summary_ebb_state();
469 	dump_ebb_hw_state();
470 	abort();
471 }
472 
473 struct sigaction term_action = {
474 	.sa_handler = term_handler,
475 };
476 
477 static void __attribute__((constructor)) ebb_init(void)
478 {
479 	clear_ebb_stats();
480 
481 	if (sigaction(SIGTERM, &term_action, NULL))
482 		perror("sigaction");
483 
484 	ebb_state.trace = trace_buffer_allocate(1 * 1024 * 1024);
485 }
486