1 /*
2  * Copyright 2014, Michael Ellerman, IBM Corp.
3  * Licensed under GPLv2.
4  */
5 
6 #define _GNU_SOURCE	/* For CPU_ZERO etc. */
7 
8 #include <sched.h>
9 #include <sys/wait.h>
10 #include <setjmp.h>
11 #include <signal.h>
12 #include <stdio.h>
13 #include <stdlib.h>
14 #include <string.h>
15 #include <sys/ioctl.h>
16 #include <linux/auxvec.h>
17 
18 #include "trace.h"
19 #include "reg.h"
20 #include "ebb.h"
21 
22 
23 void (*ebb_user_func)(void);
24 
25 void ebb_hook(void)
26 {
27 	if (ebb_user_func)
28 		ebb_user_func();
29 }
30 
31 struct ebb_state ebb_state;
32 
33 u64 sample_period = 0x40000000ull;
34 
35 void reset_ebb_with_clear_mask(unsigned long mmcr0_clear_mask)
36 {
37 	u64 val;
38 
39 	/* 2) clear MMCR0[PMAO] - docs say BESCR[PMEO] should do this */
40 	/* 3) set MMCR0[PMAE]	- docs say BESCR[PME] should do this */
41 	val = mfspr(SPRN_MMCR0);
42 	mtspr(SPRN_MMCR0, (val & ~mmcr0_clear_mask) | MMCR0_PMAE);
43 
44 	/* 4) clear BESCR[PMEO] */
45 	mtspr(SPRN_BESCRR, BESCR_PMEO);
46 
47 	/* 5) set BESCR[PME] */
48 	mtspr(SPRN_BESCRS, BESCR_PME);
49 
50 	/* 6) rfebb 1 - done in our caller */
51 }
52 
53 void reset_ebb(void)
54 {
55 	reset_ebb_with_clear_mask(MMCR0_PMAO | MMCR0_FC);
56 }
57 
58 /* Called outside of the EBB handler to check MMCR0 is sane */
59 int ebb_check_mmcr0(void)
60 {
61 	u64 val;
62 
63 	val = mfspr(SPRN_MMCR0);
64 	if ((val & (MMCR0_FC | MMCR0_PMAO)) == MMCR0_FC) {
65 		/* It's OK if we see FC & PMAO, but not FC by itself */
66 		printf("Outside of loop, only FC set 0x%llx\n", val);
67 		return 1;
68 	}
69 
70 	return 0;
71 }
72 
73 bool ebb_check_count(int pmc, u64 sample_period, int fudge)
74 {
75 	u64 count, upper, lower;
76 
77 	count = ebb_state.stats.pmc_count[PMC_INDEX(pmc)];
78 
79 	lower = ebb_state.stats.ebb_count * (sample_period - fudge);
80 
81 	if (count < lower) {
82 		printf("PMC%d count (0x%llx) below lower limit 0x%llx (-0x%llx)\n",
83 			pmc, count, lower, lower - count);
84 		return false;
85 	}
86 
87 	upper = ebb_state.stats.ebb_count * (sample_period + fudge);
88 
89 	if (count > upper) {
90 		printf("PMC%d count (0x%llx) above upper limit 0x%llx (+0x%llx)\n",
91 			pmc, count, upper, count - upper);
92 		return false;
93 	}
94 
95 	printf("PMC%d count (0x%llx) is between 0x%llx and 0x%llx delta +0x%llx/-0x%llx\n",
96 		pmc, count, lower, upper, count - lower, upper - count);
97 
98 	return true;
99 }
100 
101 void standard_ebb_callee(void)
102 {
103 	int found, i;
104 	u64 val;
105 
106 	val = mfspr(SPRN_BESCR);
107 	if (!(val & BESCR_PMEO)) {
108 		ebb_state.stats.spurious++;
109 		goto out;
110 	}
111 
112 	ebb_state.stats.ebb_count++;
113 	trace_log_counter(ebb_state.trace, ebb_state.stats.ebb_count);
114 
115 	val = mfspr(SPRN_MMCR0);
116 	trace_log_reg(ebb_state.trace, SPRN_MMCR0, val);
117 
118 	found = 0;
119 	for (i = 1; i <= 6; i++) {
120 		if (ebb_state.pmc_enable[PMC_INDEX(i)])
121 			found += count_pmc(i, sample_period);
122 	}
123 
124 	if (!found)
125 		ebb_state.stats.no_overflow++;
126 
127 out:
128 	reset_ebb();
129 }
130 
131 extern void ebb_handler(void);
132 
133 void setup_ebb_handler(void (*callee)(void))
134 {
135 	u64 entry;
136 
137 #if defined(_CALL_ELF) && _CALL_ELF == 2
138 	entry = (u64)ebb_handler;
139 #else
140 	struct opd
141 	{
142 	    u64 entry;
143 	    u64 toc;
144 	} *opd;
145 
146 	opd = (struct opd *)ebb_handler;
147 	entry = opd->entry;
148 #endif
149 	printf("EBB Handler is at %#llx\n", entry);
150 
151 	ebb_user_func = callee;
152 
153 	/* Ensure ebb_user_func is set before we set the handler */
154 	mb();
155 	mtspr(SPRN_EBBHR, entry);
156 
157 	/* Make sure the handler is set before we return */
158 	mb();
159 }
160 
161 void clear_ebb_stats(void)
162 {
163 	memset(&ebb_state.stats, 0, sizeof(ebb_state.stats));
164 }
165 
166 void dump_summary_ebb_state(void)
167 {
168 	printf("ebb_state:\n"			\
169 	       "  ebb_count    = %d\n"		\
170 	       "  spurious     = %d\n"		\
171 	       "  negative     = %d\n"		\
172 	       "  no_overflow  = %d\n"		\
173 	       "  pmc[1] count = 0x%llx\n"	\
174 	       "  pmc[2] count = 0x%llx\n"	\
175 	       "  pmc[3] count = 0x%llx\n"	\
176 	       "  pmc[4] count = 0x%llx\n"	\
177 	       "  pmc[5] count = 0x%llx\n"	\
178 	       "  pmc[6] count = 0x%llx\n",
179 		ebb_state.stats.ebb_count, ebb_state.stats.spurious,
180 		ebb_state.stats.negative, ebb_state.stats.no_overflow,
181 		ebb_state.stats.pmc_count[0], ebb_state.stats.pmc_count[1],
182 		ebb_state.stats.pmc_count[2], ebb_state.stats.pmc_count[3],
183 		ebb_state.stats.pmc_count[4], ebb_state.stats.pmc_count[5]);
184 }
185 
186 static char *decode_mmcr0(u32 value)
187 {
188 	static char buf[16];
189 
190 	buf[0] = '\0';
191 
192 	if (value & (1 << 31))
193 		strcat(buf, "FC ");
194 	if (value & (1 << 26))
195 		strcat(buf, "PMAE ");
196 	if (value & (1 << 7))
197 		strcat(buf, "PMAO ");
198 
199 	return buf;
200 }
201 
202 static char *decode_bescr(u64 value)
203 {
204 	static char buf[16];
205 
206 	buf[0] = '\0';
207 
208 	if (value & (1ull << 63))
209 		strcat(buf, "GE ");
210 	if (value & (1ull << 32))
211 		strcat(buf, "PMAE ");
212 	if (value & 1)
213 		strcat(buf, "PMAO ");
214 
215 	return buf;
216 }
217 
218 void dump_ebb_hw_state(void)
219 {
220 	u64 bescr;
221 	u32 mmcr0;
222 
223 	mmcr0 = mfspr(SPRN_MMCR0);
224 	bescr = mfspr(SPRN_BESCR);
225 
226 	printf("HW state:\n"		\
227 	       "MMCR0 0x%016x %s\n"	\
228 	       "MMCR2 0x%016lx\n"	\
229 	       "EBBHR 0x%016lx\n"	\
230 	       "BESCR 0x%016llx %s\n"	\
231 	       "PMC1  0x%016lx\n"	\
232 	       "PMC2  0x%016lx\n"	\
233 	       "PMC3  0x%016lx\n"	\
234 	       "PMC4  0x%016lx\n"	\
235 	       "PMC5  0x%016lx\n"	\
236 	       "PMC6  0x%016lx\n"	\
237 	       "SIAR  0x%016lx\n",
238 	       mmcr0, decode_mmcr0(mmcr0), mfspr(SPRN_MMCR2),
239 	       mfspr(SPRN_EBBHR), bescr, decode_bescr(bescr),
240 	       mfspr(SPRN_PMC1), mfspr(SPRN_PMC2), mfspr(SPRN_PMC3),
241 	       mfspr(SPRN_PMC4), mfspr(SPRN_PMC5), mfspr(SPRN_PMC6),
242 	       mfspr(SPRN_SIAR));
243 }
244 
245 void dump_ebb_state(void)
246 {
247 	dump_summary_ebb_state();
248 
249 	dump_ebb_hw_state();
250 
251 	trace_buffer_print(ebb_state.trace);
252 }
253 
254 int count_pmc(int pmc, uint32_t sample_period)
255 {
256 	uint32_t start_value;
257 	u64 val;
258 
259 	/* 0) Read PMC */
260 	start_value = pmc_sample_period(sample_period);
261 
262 	val = read_pmc(pmc);
263 	if (val < start_value)
264 		ebb_state.stats.negative++;
265 	else
266 		ebb_state.stats.pmc_count[PMC_INDEX(pmc)] += val - start_value;
267 
268 	trace_log_reg(ebb_state.trace, SPRN_PMC1 + pmc - 1, val);
269 
270 	/* 1) Reset PMC */
271 	write_pmc(pmc, start_value);
272 
273 	/* Report if we overflowed */
274 	return val >= COUNTER_OVERFLOW;
275 }
276 
277 int ebb_event_enable(struct event *e)
278 {
279 	int rc;
280 
281 	/* Ensure any SPR writes are ordered vs us */
282 	mb();
283 
284 	rc = ioctl(e->fd, PERF_EVENT_IOC_ENABLE);
285 	if (rc)
286 		return rc;
287 
288 	rc = event_read(e);
289 
290 	/* Ditto */
291 	mb();
292 
293 	return rc;
294 }
295 
296 void ebb_freeze_pmcs(void)
297 {
298 	mtspr(SPRN_MMCR0, mfspr(SPRN_MMCR0) | MMCR0_FC);
299 	mb();
300 }
301 
302 void ebb_unfreeze_pmcs(void)
303 {
304 	/* Unfreeze counters */
305 	mtspr(SPRN_MMCR0, mfspr(SPRN_MMCR0) & ~MMCR0_FC);
306 	mb();
307 }
308 
309 void ebb_global_enable(void)
310 {
311 	/* Enable EBBs globally and PMU EBBs */
312 	mtspr(SPRN_BESCR, 0x8000000100000000ull);
313 	mb();
314 }
315 
316 void ebb_global_disable(void)
317 {
318 	/* Disable EBBs & freeze counters, events are still scheduled */
319 	mtspr(SPRN_BESCRR, BESCR_PME);
320 	mb();
321 }
322 
323 bool ebb_is_supported(void)
324 {
325 #ifdef PPC_FEATURE2_EBB
326 	/* EBB requires at least POWER8 */
327 	return ((long)get_auxv_entry(AT_HWCAP2) & PPC_FEATURE2_EBB);
328 #else
329 	return false;
330 #endif
331 }
332 
333 void event_ebb_init(struct event *e)
334 {
335 	e->attr.config |= (1ull << 63);
336 }
337 
338 void event_bhrb_init(struct event *e, unsigned ifm)
339 {
340 	e->attr.config |= (1ull << 62) | ((u64)ifm << 60);
341 }
342 
343 void event_leader_ebb_init(struct event *e)
344 {
345 	event_ebb_init(e);
346 
347 	e->attr.exclusive = 1;
348 	e->attr.pinned = 1;
349 }
350 
351 int ebb_child(union pipe read_pipe, union pipe write_pipe)
352 {
353 	struct event event;
354 	uint64_t val;
355 
356 	FAIL_IF(wait_for_parent(read_pipe));
357 
358 	event_init_named(&event, 0x1001e, "cycles");
359 	event_leader_ebb_init(&event);
360 
361 	event.attr.exclude_kernel = 1;
362 	event.attr.exclude_hv = 1;
363 	event.attr.exclude_idle = 1;
364 
365 	FAIL_IF(event_open(&event));
366 
367 	ebb_enable_pmc_counting(1);
368 	setup_ebb_handler(standard_ebb_callee);
369 	ebb_global_enable();
370 
371 	FAIL_IF(event_enable(&event));
372 
373 	if (event_read(&event)) {
374 		/*
375 		 * Some tests expect to fail here, so don't report an error on
376 		 * this line, and return a distinguisable error code. Tell the
377 		 * parent an error happened.
378 		 */
379 		notify_parent_of_error(write_pipe);
380 		return 2;
381 	}
382 
383 	mtspr(SPRN_PMC1, pmc_sample_period(sample_period));
384 
385 	FAIL_IF(notify_parent(write_pipe));
386 	FAIL_IF(wait_for_parent(read_pipe));
387 	FAIL_IF(notify_parent(write_pipe));
388 
389 	while (ebb_state.stats.ebb_count < 20) {
390 		FAIL_IF(core_busy_loop());
391 
392 		/* To try and hit SIGILL case */
393 		val  = mfspr(SPRN_MMCRA);
394 		val |= mfspr(SPRN_MMCR2);
395 		val |= mfspr(SPRN_MMCR0);
396 	}
397 
398 	ebb_global_disable();
399 	ebb_freeze_pmcs();
400 
401 	count_pmc(1, sample_period);
402 
403 	dump_ebb_state();
404 
405 	event_close(&event);
406 
407 	FAIL_IF(ebb_state.stats.ebb_count == 0);
408 
409 	return 0;
410 }
411 
412 static jmp_buf setjmp_env;
413 
414 static void sigill_handler(int signal)
415 {
416 	printf("Took sigill\n");
417 	longjmp(setjmp_env, 1);
418 }
419 
420 static struct sigaction sigill_action = {
421 	.sa_handler = sigill_handler,
422 };
423 
424 int catch_sigill(void (*func)(void))
425 {
426 	if (sigaction(SIGILL, &sigill_action, NULL)) {
427 		perror("sigaction");
428 		return 1;
429 	}
430 
431 	if (setjmp(setjmp_env) == 0) {
432 		func();
433 		return 1;
434 	}
435 
436 	return 0;
437 }
438 
439 void write_pmc1(void)
440 {
441 	mtspr(SPRN_PMC1, 0);
442 }
443 
444 void write_pmc(int pmc, u64 value)
445 {
446 	switch (pmc) {
447 		case 1: mtspr(SPRN_PMC1, value); break;
448 		case 2: mtspr(SPRN_PMC2, value); break;
449 		case 3: mtspr(SPRN_PMC3, value); break;
450 		case 4: mtspr(SPRN_PMC4, value); break;
451 		case 5: mtspr(SPRN_PMC5, value); break;
452 		case 6: mtspr(SPRN_PMC6, value); break;
453 	}
454 }
455 
456 u64 read_pmc(int pmc)
457 {
458 	switch (pmc) {
459 		case 1: return mfspr(SPRN_PMC1);
460 		case 2: return mfspr(SPRN_PMC2);
461 		case 3: return mfspr(SPRN_PMC3);
462 		case 4: return mfspr(SPRN_PMC4);
463 		case 5: return mfspr(SPRN_PMC5);
464 		case 6: return mfspr(SPRN_PMC6);
465 	}
466 
467 	return 0;
468 }
469 
470 static void term_handler(int signal)
471 {
472 	dump_summary_ebb_state();
473 	dump_ebb_hw_state();
474 	abort();
475 }
476 
477 struct sigaction term_action = {
478 	.sa_handler = term_handler,
479 };
480 
481 static void __attribute__((constructor)) ebb_init(void)
482 {
483 	clear_ebb_stats();
484 
485 	if (sigaction(SIGTERM, &term_action, NULL))
486 		perror("sigaction");
487 
488 	ebb_state.trace = trace_buffer_allocate(1 * 1024 * 1024);
489 }
490