1 /* 2 * Copyright 2014, Michael Ellerman, IBM Corp. 3 * Licensed under GPLv2. 4 */ 5 6 #define _GNU_SOURCE /* For CPU_ZERO etc. */ 7 8 #include <sched.h> 9 #include <sys/wait.h> 10 #include <setjmp.h> 11 #include <signal.h> 12 #include <stdio.h> 13 #include <stdlib.h> 14 #include <string.h> 15 #include <sys/ioctl.h> 16 17 #include "trace.h" 18 #include "reg.h" 19 #include "ebb.h" 20 21 22 void (*ebb_user_func)(void); 23 24 void ebb_hook(void) 25 { 26 if (ebb_user_func) 27 ebb_user_func(); 28 } 29 30 struct ebb_state ebb_state; 31 32 u64 sample_period = 0x40000000ull; 33 34 void reset_ebb_with_clear_mask(unsigned long mmcr0_clear_mask) 35 { 36 u64 val; 37 38 /* 2) clear MMCR0[PMAO] - docs say BESCR[PMEO] should do this */ 39 /* 3) set MMCR0[PMAE] - docs say BESCR[PME] should do this */ 40 val = mfspr(SPRN_MMCR0); 41 mtspr(SPRN_MMCR0, (val & ~mmcr0_clear_mask) | MMCR0_PMAE); 42 43 /* 4) clear BESCR[PMEO] */ 44 mtspr(SPRN_BESCRR, BESCR_PMEO); 45 46 /* 5) set BESCR[PME] */ 47 mtspr(SPRN_BESCRS, BESCR_PME); 48 49 /* 6) rfebb 1 - done in our caller */ 50 } 51 52 void reset_ebb(void) 53 { 54 reset_ebb_with_clear_mask(MMCR0_PMAO | MMCR0_FC); 55 } 56 57 /* Called outside of the EBB handler to check MMCR0 is sane */ 58 int ebb_check_mmcr0(void) 59 { 60 u64 val; 61 62 val = mfspr(SPRN_MMCR0); 63 if ((val & (MMCR0_FC | MMCR0_PMAO)) == MMCR0_FC) { 64 /* It's OK if we see FC & PMAO, but not FC by itself */ 65 printf("Outside of loop, only FC set 0x%llx\n", val); 66 return 1; 67 } 68 69 return 0; 70 } 71 72 bool ebb_check_count(int pmc, u64 sample_period, int fudge) 73 { 74 u64 count, upper, lower; 75 76 count = ebb_state.stats.pmc_count[PMC_INDEX(pmc)]; 77 78 lower = ebb_state.stats.ebb_count * (sample_period - fudge); 79 80 if (count < lower) { 81 printf("PMC%d count (0x%llx) below lower limit 0x%llx (-0x%llx)\n", 82 pmc, count, lower, lower - count); 83 return false; 84 } 85 86 upper = ebb_state.stats.ebb_count * (sample_period + fudge); 87 88 if (count > upper) { 89 printf("PMC%d count (0x%llx) above upper limit 0x%llx (+0x%llx)\n", 90 pmc, count, upper, count - upper); 91 return false; 92 } 93 94 printf("PMC%d count (0x%llx) is between 0x%llx and 0x%llx delta +0x%llx/-0x%llx\n", 95 pmc, count, lower, upper, count - lower, upper - count); 96 97 return true; 98 } 99 100 void standard_ebb_callee(void) 101 { 102 int found, i; 103 u64 val; 104 105 val = mfspr(SPRN_BESCR); 106 if (!(val & BESCR_PMEO)) { 107 ebb_state.stats.spurious++; 108 goto out; 109 } 110 111 ebb_state.stats.ebb_count++; 112 trace_log_counter(ebb_state.trace, ebb_state.stats.ebb_count); 113 114 val = mfspr(SPRN_MMCR0); 115 trace_log_reg(ebb_state.trace, SPRN_MMCR0, val); 116 117 found = 0; 118 for (i = 1; i <= 6; i++) { 119 if (ebb_state.pmc_enable[PMC_INDEX(i)]) 120 found += count_pmc(i, sample_period); 121 } 122 123 if (!found) 124 ebb_state.stats.no_overflow++; 125 126 out: 127 reset_ebb(); 128 } 129 130 extern void ebb_handler(void); 131 132 void setup_ebb_handler(void (*callee)(void)) 133 { 134 u64 entry; 135 136 #if defined(_CALL_ELF) && _CALL_ELF == 2 137 entry = (u64)ebb_handler; 138 #else 139 struct opd 140 { 141 u64 entry; 142 u64 toc; 143 } *opd; 144 145 opd = (struct opd *)ebb_handler; 146 entry = opd->entry; 147 #endif 148 printf("EBB Handler is at %#llx\n", entry); 149 150 ebb_user_func = callee; 151 152 /* Ensure ebb_user_func is set before we set the handler */ 153 mb(); 154 mtspr(SPRN_EBBHR, entry); 155 156 /* Make sure the handler is set before we return */ 157 mb(); 158 } 159 160 void clear_ebb_stats(void) 161 { 162 memset(&ebb_state.stats, 0, sizeof(ebb_state.stats)); 163 } 164 165 void dump_summary_ebb_state(void) 166 { 167 printf("ebb_state:\n" \ 168 " ebb_count = %d\n" \ 169 " spurious = %d\n" \ 170 " negative = %d\n" \ 171 " no_overflow = %d\n" \ 172 " pmc[1] count = 0x%llx\n" \ 173 " pmc[2] count = 0x%llx\n" \ 174 " pmc[3] count = 0x%llx\n" \ 175 " pmc[4] count = 0x%llx\n" \ 176 " pmc[5] count = 0x%llx\n" \ 177 " pmc[6] count = 0x%llx\n", 178 ebb_state.stats.ebb_count, ebb_state.stats.spurious, 179 ebb_state.stats.negative, ebb_state.stats.no_overflow, 180 ebb_state.stats.pmc_count[0], ebb_state.stats.pmc_count[1], 181 ebb_state.stats.pmc_count[2], ebb_state.stats.pmc_count[3], 182 ebb_state.stats.pmc_count[4], ebb_state.stats.pmc_count[5]); 183 } 184 185 static char *decode_mmcr0(u32 value) 186 { 187 static char buf[16]; 188 189 buf[0] = '\0'; 190 191 if (value & (1 << 31)) 192 strcat(buf, "FC "); 193 if (value & (1 << 26)) 194 strcat(buf, "PMAE "); 195 if (value & (1 << 7)) 196 strcat(buf, "PMAO "); 197 198 return buf; 199 } 200 201 static char *decode_bescr(u64 value) 202 { 203 static char buf[16]; 204 205 buf[0] = '\0'; 206 207 if (value & (1ull << 63)) 208 strcat(buf, "GE "); 209 if (value & (1ull << 32)) 210 strcat(buf, "PMAE "); 211 if (value & 1) 212 strcat(buf, "PMAO "); 213 214 return buf; 215 } 216 217 void dump_ebb_hw_state(void) 218 { 219 u64 bescr; 220 u32 mmcr0; 221 222 mmcr0 = mfspr(SPRN_MMCR0); 223 bescr = mfspr(SPRN_BESCR); 224 225 printf("HW state:\n" \ 226 "MMCR0 0x%016x %s\n" \ 227 "MMCR2 0x%016lx\n" \ 228 "EBBHR 0x%016lx\n" \ 229 "BESCR 0x%016llx %s\n" \ 230 "PMC1 0x%016lx\n" \ 231 "PMC2 0x%016lx\n" \ 232 "PMC3 0x%016lx\n" \ 233 "PMC4 0x%016lx\n" \ 234 "PMC5 0x%016lx\n" \ 235 "PMC6 0x%016lx\n" \ 236 "SIAR 0x%016lx\n", 237 mmcr0, decode_mmcr0(mmcr0), mfspr(SPRN_MMCR2), 238 mfspr(SPRN_EBBHR), bescr, decode_bescr(bescr), 239 mfspr(SPRN_PMC1), mfspr(SPRN_PMC2), mfspr(SPRN_PMC3), 240 mfspr(SPRN_PMC4), mfspr(SPRN_PMC5), mfspr(SPRN_PMC6), 241 mfspr(SPRN_SIAR)); 242 } 243 244 void dump_ebb_state(void) 245 { 246 dump_summary_ebb_state(); 247 248 dump_ebb_hw_state(); 249 250 trace_buffer_print(ebb_state.trace); 251 } 252 253 int count_pmc(int pmc, uint32_t sample_period) 254 { 255 uint32_t start_value; 256 u64 val; 257 258 /* 0) Read PMC */ 259 start_value = pmc_sample_period(sample_period); 260 261 val = read_pmc(pmc); 262 if (val < start_value) 263 ebb_state.stats.negative++; 264 else 265 ebb_state.stats.pmc_count[PMC_INDEX(pmc)] += val - start_value; 266 267 trace_log_reg(ebb_state.trace, SPRN_PMC1 + pmc - 1, val); 268 269 /* 1) Reset PMC */ 270 write_pmc(pmc, start_value); 271 272 /* Report if we overflowed */ 273 return val >= COUNTER_OVERFLOW; 274 } 275 276 int ebb_event_enable(struct event *e) 277 { 278 int rc; 279 280 /* Ensure any SPR writes are ordered vs us */ 281 mb(); 282 283 rc = ioctl(e->fd, PERF_EVENT_IOC_ENABLE); 284 if (rc) 285 return rc; 286 287 rc = event_read(e); 288 289 /* Ditto */ 290 mb(); 291 292 return rc; 293 } 294 295 void ebb_freeze_pmcs(void) 296 { 297 mtspr(SPRN_MMCR0, mfspr(SPRN_MMCR0) | MMCR0_FC); 298 mb(); 299 } 300 301 void ebb_unfreeze_pmcs(void) 302 { 303 /* Unfreeze counters */ 304 mtspr(SPRN_MMCR0, mfspr(SPRN_MMCR0) & ~MMCR0_FC); 305 mb(); 306 } 307 308 void ebb_global_enable(void) 309 { 310 /* Enable EBBs globally and PMU EBBs */ 311 mtspr(SPRN_BESCR, 0x8000000100000000ull); 312 mb(); 313 } 314 315 void ebb_global_disable(void) 316 { 317 /* Disable EBBs & freeze counters, events are still scheduled */ 318 mtspr(SPRN_BESCRR, BESCR_PME); 319 mb(); 320 } 321 322 bool ebb_is_supported(void) 323 { 324 #ifdef PPC_FEATURE2_EBB 325 /* EBB requires at least POWER8 */ 326 return have_hwcap2(PPC_FEATURE2_EBB); 327 #else 328 return false; 329 #endif 330 } 331 332 void event_ebb_init(struct event *e) 333 { 334 e->attr.config |= (1ull << 63); 335 } 336 337 void event_bhrb_init(struct event *e, unsigned ifm) 338 { 339 e->attr.config |= (1ull << 62) | ((u64)ifm << 60); 340 } 341 342 void event_leader_ebb_init(struct event *e) 343 { 344 event_ebb_init(e); 345 346 e->attr.exclusive = 1; 347 e->attr.pinned = 1; 348 } 349 350 int ebb_child(union pipe read_pipe, union pipe write_pipe) 351 { 352 struct event event; 353 uint64_t val; 354 355 FAIL_IF(wait_for_parent(read_pipe)); 356 357 event_init_named(&event, 0x1001e, "cycles"); 358 event_leader_ebb_init(&event); 359 360 event.attr.exclude_kernel = 1; 361 event.attr.exclude_hv = 1; 362 event.attr.exclude_idle = 1; 363 364 FAIL_IF(event_open(&event)); 365 366 ebb_enable_pmc_counting(1); 367 setup_ebb_handler(standard_ebb_callee); 368 ebb_global_enable(); 369 370 FAIL_IF(event_enable(&event)); 371 372 if (event_read(&event)) { 373 /* 374 * Some tests expect to fail here, so don't report an error on 375 * this line, and return a distinguisable error code. Tell the 376 * parent an error happened. 377 */ 378 notify_parent_of_error(write_pipe); 379 return 2; 380 } 381 382 mtspr(SPRN_PMC1, pmc_sample_period(sample_period)); 383 384 FAIL_IF(notify_parent(write_pipe)); 385 FAIL_IF(wait_for_parent(read_pipe)); 386 FAIL_IF(notify_parent(write_pipe)); 387 388 while (ebb_state.stats.ebb_count < 20) { 389 FAIL_IF(core_busy_loop()); 390 391 /* To try and hit SIGILL case */ 392 val = mfspr(SPRN_MMCRA); 393 val |= mfspr(SPRN_MMCR2); 394 val |= mfspr(SPRN_MMCR0); 395 } 396 397 ebb_global_disable(); 398 ebb_freeze_pmcs(); 399 400 count_pmc(1, sample_period); 401 402 dump_ebb_state(); 403 404 event_close(&event); 405 406 FAIL_IF(ebb_state.stats.ebb_count == 0); 407 408 return 0; 409 } 410 411 static jmp_buf setjmp_env; 412 413 static void sigill_handler(int signal) 414 { 415 printf("Took sigill\n"); 416 longjmp(setjmp_env, 1); 417 } 418 419 static struct sigaction sigill_action = { 420 .sa_handler = sigill_handler, 421 }; 422 423 int catch_sigill(void (*func)(void)) 424 { 425 if (sigaction(SIGILL, &sigill_action, NULL)) { 426 perror("sigaction"); 427 return 1; 428 } 429 430 if (setjmp(setjmp_env) == 0) { 431 func(); 432 return 1; 433 } 434 435 return 0; 436 } 437 438 void write_pmc1(void) 439 { 440 mtspr(SPRN_PMC1, 0); 441 } 442 443 void write_pmc(int pmc, u64 value) 444 { 445 switch (pmc) { 446 case 1: mtspr(SPRN_PMC1, value); break; 447 case 2: mtspr(SPRN_PMC2, value); break; 448 case 3: mtspr(SPRN_PMC3, value); break; 449 case 4: mtspr(SPRN_PMC4, value); break; 450 case 5: mtspr(SPRN_PMC5, value); break; 451 case 6: mtspr(SPRN_PMC6, value); break; 452 } 453 } 454 455 u64 read_pmc(int pmc) 456 { 457 switch (pmc) { 458 case 1: return mfspr(SPRN_PMC1); 459 case 2: return mfspr(SPRN_PMC2); 460 case 3: return mfspr(SPRN_PMC3); 461 case 4: return mfspr(SPRN_PMC4); 462 case 5: return mfspr(SPRN_PMC5); 463 case 6: return mfspr(SPRN_PMC6); 464 } 465 466 return 0; 467 } 468 469 static void term_handler(int signal) 470 { 471 dump_summary_ebb_state(); 472 dump_ebb_hw_state(); 473 abort(); 474 } 475 476 struct sigaction term_action = { 477 .sa_handler = term_handler, 478 }; 479 480 static void __attribute__((constructor)) ebb_init(void) 481 { 482 clear_ebb_stats(); 483 484 if (sigaction(SIGTERM, &term_action, NULL)) 485 perror("sigaction"); 486 487 ebb_state.trace = trace_buffer_allocate(1 * 1024 * 1024); 488 } 489