1 /* 2 * Copyright 2014, Michael Ellerman, IBM Corp. 3 * Licensed under GPLv2. 4 */ 5 6 #define _GNU_SOURCE /* For CPU_ZERO etc. */ 7 8 #include <sched.h> 9 #include <sys/wait.h> 10 #include <setjmp.h> 11 #include <signal.h> 12 #include <stdio.h> 13 #include <stdlib.h> 14 #include <string.h> 15 #include <sys/ioctl.h> 16 17 #include "trace.h" 18 #include "reg.h" 19 #include "ebb.h" 20 21 22 void (*ebb_user_func)(void); 23 24 void ebb_hook(void) 25 { 26 if (ebb_user_func) 27 ebb_user_func(); 28 } 29 30 struct ebb_state ebb_state; 31 32 u64 sample_period = 0x40000000ull; 33 34 void reset_ebb_with_clear_mask(unsigned long mmcr0_clear_mask) 35 { 36 u64 val; 37 38 /* 2) clear MMCR0[PMAO] - docs say BESCR[PMEO] should do this */ 39 /* 3) set MMCR0[PMAE] - docs say BESCR[PME] should do this */ 40 val = mfspr(SPRN_MMCR0); 41 mtspr(SPRN_MMCR0, (val & ~mmcr0_clear_mask) | MMCR0_PMAE); 42 43 /* 4) clear BESCR[PMEO] */ 44 mtspr(SPRN_BESCRR, BESCR_PMEO); 45 46 /* 5) set BESCR[PME] */ 47 mtspr(SPRN_BESCRS, BESCR_PME); 48 49 /* 6) rfebb 1 - done in our caller */ 50 } 51 52 void reset_ebb(void) 53 { 54 reset_ebb_with_clear_mask(MMCR0_PMAO | MMCR0_FC); 55 } 56 57 /* Called outside of the EBB handler to check MMCR0 is sane */ 58 int ebb_check_mmcr0(void) 59 { 60 u64 val; 61 62 val = mfspr(SPRN_MMCR0); 63 if ((val & (MMCR0_FC | MMCR0_PMAO)) == MMCR0_FC) { 64 /* It's OK if we see FC & PMAO, but not FC by itself */ 65 printf("Outside of loop, only FC set 0x%llx\n", val); 66 return 1; 67 } 68 69 return 0; 70 } 71 72 bool ebb_check_count(int pmc, u64 sample_period, int fudge) 73 { 74 u64 count, upper, lower; 75 76 count = ebb_state.stats.pmc_count[PMC_INDEX(pmc)]; 77 78 lower = ebb_state.stats.ebb_count * (sample_period - fudge); 79 80 if (count < lower) { 81 printf("PMC%d count (0x%llx) below lower limit 0x%llx (-0x%llx)\n", 82 pmc, count, lower, lower - count); 83 return false; 84 } 85 86 upper = ebb_state.stats.ebb_count * (sample_period + fudge); 87 88 if (count > upper) { 89 printf("PMC%d count (0x%llx) above upper limit 0x%llx (+0x%llx)\n", 90 pmc, count, upper, count - upper); 91 return false; 92 } 93 94 printf("PMC%d count (0x%llx) is between 0x%llx and 0x%llx delta +0x%llx/-0x%llx\n", 95 pmc, count, lower, upper, count - lower, upper - count); 96 97 return true; 98 } 99 100 void standard_ebb_callee(void) 101 { 102 int found, i; 103 u64 val; 104 105 val = mfspr(SPRN_BESCR); 106 if (!(val & BESCR_PMEO)) { 107 ebb_state.stats.spurious++; 108 goto out; 109 } 110 111 ebb_state.stats.ebb_count++; 112 trace_log_counter(ebb_state.trace, ebb_state.stats.ebb_count); 113 114 val = mfspr(SPRN_MMCR0); 115 trace_log_reg(ebb_state.trace, SPRN_MMCR0, val); 116 117 found = 0; 118 for (i = 1; i <= 6; i++) { 119 if (ebb_state.pmc_enable[PMC_INDEX(i)]) 120 found += count_pmc(i, sample_period); 121 } 122 123 if (!found) 124 ebb_state.stats.no_overflow++; 125 126 out: 127 reset_ebb(); 128 } 129 130 extern void ebb_handler(void); 131 132 void setup_ebb_handler(void (*callee)(void)) 133 { 134 u64 entry; 135 136 #if defined(_CALL_ELF) && _CALL_ELF == 2 137 entry = (u64)ebb_handler; 138 #else 139 struct opd 140 { 141 u64 entry; 142 u64 toc; 143 } *opd; 144 145 opd = (struct opd *)ebb_handler; 146 entry = opd->entry; 147 #endif 148 printf("EBB Handler is at %#llx\n", entry); 149 150 ebb_user_func = callee; 151 152 /* Ensure ebb_user_func is set before we set the handler */ 153 mb(); 154 mtspr(SPRN_EBBHR, entry); 155 156 /* Make sure the handler is set before we return */ 157 mb(); 158 } 159 160 void clear_ebb_stats(void) 161 { 162 memset(&ebb_state.stats, 0, sizeof(ebb_state.stats)); 163 } 164 165 void dump_summary_ebb_state(void) 166 { 167 printf("ebb_state:\n" \ 168 " ebb_count = %d\n" \ 169 " spurious = %d\n" \ 170 " negative = %d\n" \ 171 " no_overflow = %d\n" \ 172 " pmc[1] count = 0x%llx\n" \ 173 " pmc[2] count = 0x%llx\n" \ 174 " pmc[3] count = 0x%llx\n" \ 175 " pmc[4] count = 0x%llx\n" \ 176 " pmc[5] count = 0x%llx\n" \ 177 " pmc[6] count = 0x%llx\n", 178 ebb_state.stats.ebb_count, ebb_state.stats.spurious, 179 ebb_state.stats.negative, ebb_state.stats.no_overflow, 180 ebb_state.stats.pmc_count[0], ebb_state.stats.pmc_count[1], 181 ebb_state.stats.pmc_count[2], ebb_state.stats.pmc_count[3], 182 ebb_state.stats.pmc_count[4], ebb_state.stats.pmc_count[5]); 183 } 184 185 static char *decode_mmcr0(u32 value) 186 { 187 static char buf[16]; 188 189 buf[0] = '\0'; 190 191 if (value & (1 << 31)) 192 strcat(buf, "FC "); 193 if (value & (1 << 26)) 194 strcat(buf, "PMAE "); 195 if (value & (1 << 7)) 196 strcat(buf, "PMAO "); 197 198 return buf; 199 } 200 201 static char *decode_bescr(u64 value) 202 { 203 static char buf[16]; 204 205 buf[0] = '\0'; 206 207 if (value & (1ull << 63)) 208 strcat(buf, "GE "); 209 if (value & (1ull << 32)) 210 strcat(buf, "PMAE "); 211 if (value & 1) 212 strcat(buf, "PMAO "); 213 214 return buf; 215 } 216 217 void dump_ebb_hw_state(void) 218 { 219 u64 bescr; 220 u32 mmcr0; 221 222 mmcr0 = mfspr(SPRN_MMCR0); 223 bescr = mfspr(SPRN_BESCR); 224 225 printf("HW state:\n" \ 226 "MMCR0 0x%016x %s\n" \ 227 "EBBHR 0x%016lx\n" \ 228 "BESCR 0x%016llx %s\n" \ 229 "PMC1 0x%016lx\n" \ 230 "PMC2 0x%016lx\n" \ 231 "PMC3 0x%016lx\n" \ 232 "PMC4 0x%016lx\n" \ 233 "PMC5 0x%016lx\n" \ 234 "PMC6 0x%016lx\n" \ 235 "SIAR 0x%016lx\n", 236 mmcr0, decode_mmcr0(mmcr0), mfspr(SPRN_EBBHR), bescr, 237 decode_bescr(bescr), mfspr(SPRN_PMC1), mfspr(SPRN_PMC2), 238 mfspr(SPRN_PMC3), mfspr(SPRN_PMC4), mfspr(SPRN_PMC5), 239 mfspr(SPRN_PMC6), mfspr(SPRN_SIAR)); 240 } 241 242 void dump_ebb_state(void) 243 { 244 dump_summary_ebb_state(); 245 246 dump_ebb_hw_state(); 247 248 trace_buffer_print(ebb_state.trace); 249 } 250 251 int count_pmc(int pmc, uint32_t sample_period) 252 { 253 uint32_t start_value; 254 u64 val; 255 256 /* 0) Read PMC */ 257 start_value = pmc_sample_period(sample_period); 258 259 val = read_pmc(pmc); 260 if (val < start_value) 261 ebb_state.stats.negative++; 262 else 263 ebb_state.stats.pmc_count[PMC_INDEX(pmc)] += val - start_value; 264 265 trace_log_reg(ebb_state.trace, SPRN_PMC1 + pmc - 1, val); 266 267 /* 1) Reset PMC */ 268 write_pmc(pmc, start_value); 269 270 /* Report if we overflowed */ 271 return val >= COUNTER_OVERFLOW; 272 } 273 274 int ebb_event_enable(struct event *e) 275 { 276 int rc; 277 278 /* Ensure any SPR writes are ordered vs us */ 279 mb(); 280 281 rc = ioctl(e->fd, PERF_EVENT_IOC_ENABLE); 282 if (rc) 283 return rc; 284 285 rc = event_read(e); 286 287 /* Ditto */ 288 mb(); 289 290 return rc; 291 } 292 293 void ebb_freeze_pmcs(void) 294 { 295 mtspr(SPRN_MMCR0, mfspr(SPRN_MMCR0) | MMCR0_FC); 296 mb(); 297 } 298 299 void ebb_unfreeze_pmcs(void) 300 { 301 /* Unfreeze counters */ 302 mtspr(SPRN_MMCR0, mfspr(SPRN_MMCR0) & ~MMCR0_FC); 303 mb(); 304 } 305 306 void ebb_global_enable(void) 307 { 308 /* Enable EBBs globally and PMU EBBs */ 309 mtspr(SPRN_BESCR, 0x8000000100000000ull); 310 mb(); 311 } 312 313 void ebb_global_disable(void) 314 { 315 /* Disable EBBs & freeze counters, events are still scheduled */ 316 mtspr(SPRN_BESCRR, BESCR_PME); 317 mb(); 318 } 319 320 void event_ebb_init(struct event *e) 321 { 322 e->attr.config |= (1ull << 63); 323 } 324 325 void event_bhrb_init(struct event *e, unsigned ifm) 326 { 327 e->attr.config |= (1ull << 62) | ((u64)ifm << 60); 328 } 329 330 void event_leader_ebb_init(struct event *e) 331 { 332 event_ebb_init(e); 333 334 e->attr.exclusive = 1; 335 e->attr.pinned = 1; 336 } 337 338 int core_busy_loop(void) 339 { 340 int rc; 341 342 asm volatile ( 343 "li 3, 0x3030\n" 344 "std 3, -96(1)\n" 345 "li 4, 0x4040\n" 346 "std 4, -104(1)\n" 347 "li 5, 0x5050\n" 348 "std 5, -112(1)\n" 349 "li 6, 0x6060\n" 350 "std 6, -120(1)\n" 351 "li 7, 0x7070\n" 352 "std 7, -128(1)\n" 353 "li 8, 0x0808\n" 354 "std 8, -136(1)\n" 355 "li 9, 0x0909\n" 356 "std 9, -144(1)\n" 357 "li 10, 0x1010\n" 358 "std 10, -152(1)\n" 359 "li 11, 0x1111\n" 360 "std 11, -160(1)\n" 361 "li 14, 0x1414\n" 362 "std 14, -168(1)\n" 363 "li 15, 0x1515\n" 364 "std 15, -176(1)\n" 365 "li 16, 0x1616\n" 366 "std 16, -184(1)\n" 367 "li 17, 0x1717\n" 368 "std 17, -192(1)\n" 369 "li 18, 0x1818\n" 370 "std 18, -200(1)\n" 371 "li 19, 0x1919\n" 372 "std 19, -208(1)\n" 373 "li 20, 0x2020\n" 374 "std 20, -216(1)\n" 375 "li 21, 0x2121\n" 376 "std 21, -224(1)\n" 377 "li 22, 0x2222\n" 378 "std 22, -232(1)\n" 379 "li 23, 0x2323\n" 380 "std 23, -240(1)\n" 381 "li 24, 0x2424\n" 382 "std 24, -248(1)\n" 383 "li 25, 0x2525\n" 384 "std 25, -256(1)\n" 385 "li 26, 0x2626\n" 386 "std 26, -264(1)\n" 387 "li 27, 0x2727\n" 388 "std 27, -272(1)\n" 389 "li 28, 0x2828\n" 390 "std 28, -280(1)\n" 391 "li 29, 0x2929\n" 392 "std 29, -288(1)\n" 393 "li 30, 0x3030\n" 394 "li 31, 0x3131\n" 395 396 "li 3, 0\n" 397 "0: " 398 "addi 3, 3, 1\n" 399 "cmpwi 3, 100\n" 400 "blt 0b\n" 401 402 /* Return 1 (fail) unless we get through all the checks */ 403 "li 0, 1\n" 404 405 /* Check none of our registers have been corrupted */ 406 "cmpwi 4, 0x4040\n" 407 "bne 1f\n" 408 "cmpwi 5, 0x5050\n" 409 "bne 1f\n" 410 "cmpwi 6, 0x6060\n" 411 "bne 1f\n" 412 "cmpwi 7, 0x7070\n" 413 "bne 1f\n" 414 "cmpwi 8, 0x0808\n" 415 "bne 1f\n" 416 "cmpwi 9, 0x0909\n" 417 "bne 1f\n" 418 "cmpwi 10, 0x1010\n" 419 "bne 1f\n" 420 "cmpwi 11, 0x1111\n" 421 "bne 1f\n" 422 "cmpwi 14, 0x1414\n" 423 "bne 1f\n" 424 "cmpwi 15, 0x1515\n" 425 "bne 1f\n" 426 "cmpwi 16, 0x1616\n" 427 "bne 1f\n" 428 "cmpwi 17, 0x1717\n" 429 "bne 1f\n" 430 "cmpwi 18, 0x1818\n" 431 "bne 1f\n" 432 "cmpwi 19, 0x1919\n" 433 "bne 1f\n" 434 "cmpwi 20, 0x2020\n" 435 "bne 1f\n" 436 "cmpwi 21, 0x2121\n" 437 "bne 1f\n" 438 "cmpwi 22, 0x2222\n" 439 "bne 1f\n" 440 "cmpwi 23, 0x2323\n" 441 "bne 1f\n" 442 "cmpwi 24, 0x2424\n" 443 "bne 1f\n" 444 "cmpwi 25, 0x2525\n" 445 "bne 1f\n" 446 "cmpwi 26, 0x2626\n" 447 "bne 1f\n" 448 "cmpwi 27, 0x2727\n" 449 "bne 1f\n" 450 "cmpwi 28, 0x2828\n" 451 "bne 1f\n" 452 "cmpwi 29, 0x2929\n" 453 "bne 1f\n" 454 "cmpwi 30, 0x3030\n" 455 "bne 1f\n" 456 "cmpwi 31, 0x3131\n" 457 "bne 1f\n" 458 459 /* Load junk into all our registers before we reload them from the stack. */ 460 "li 3, 0xde\n" 461 "li 4, 0xad\n" 462 "li 5, 0xbe\n" 463 "li 6, 0xef\n" 464 "li 7, 0xde\n" 465 "li 8, 0xad\n" 466 "li 9, 0xbe\n" 467 "li 10, 0xef\n" 468 "li 11, 0xde\n" 469 "li 14, 0xad\n" 470 "li 15, 0xbe\n" 471 "li 16, 0xef\n" 472 "li 17, 0xde\n" 473 "li 18, 0xad\n" 474 "li 19, 0xbe\n" 475 "li 20, 0xef\n" 476 "li 21, 0xde\n" 477 "li 22, 0xad\n" 478 "li 23, 0xbe\n" 479 "li 24, 0xef\n" 480 "li 25, 0xde\n" 481 "li 26, 0xad\n" 482 "li 27, 0xbe\n" 483 "li 28, 0xef\n" 484 "li 29, 0xdd\n" 485 486 "ld 3, -96(1)\n" 487 "cmpwi 3, 0x3030\n" 488 "bne 1f\n" 489 "ld 4, -104(1)\n" 490 "cmpwi 4, 0x4040\n" 491 "bne 1f\n" 492 "ld 5, -112(1)\n" 493 "cmpwi 5, 0x5050\n" 494 "bne 1f\n" 495 "ld 6, -120(1)\n" 496 "cmpwi 6, 0x6060\n" 497 "bne 1f\n" 498 "ld 7, -128(1)\n" 499 "cmpwi 7, 0x7070\n" 500 "bne 1f\n" 501 "ld 8, -136(1)\n" 502 "cmpwi 8, 0x0808\n" 503 "bne 1f\n" 504 "ld 9, -144(1)\n" 505 "cmpwi 9, 0x0909\n" 506 "bne 1f\n" 507 "ld 10, -152(1)\n" 508 "cmpwi 10, 0x1010\n" 509 "bne 1f\n" 510 "ld 11, -160(1)\n" 511 "cmpwi 11, 0x1111\n" 512 "bne 1f\n" 513 "ld 14, -168(1)\n" 514 "cmpwi 14, 0x1414\n" 515 "bne 1f\n" 516 "ld 15, -176(1)\n" 517 "cmpwi 15, 0x1515\n" 518 "bne 1f\n" 519 "ld 16, -184(1)\n" 520 "cmpwi 16, 0x1616\n" 521 "bne 1f\n" 522 "ld 17, -192(1)\n" 523 "cmpwi 17, 0x1717\n" 524 "bne 1f\n" 525 "ld 18, -200(1)\n" 526 "cmpwi 18, 0x1818\n" 527 "bne 1f\n" 528 "ld 19, -208(1)\n" 529 "cmpwi 19, 0x1919\n" 530 "bne 1f\n" 531 "ld 20, -216(1)\n" 532 "cmpwi 20, 0x2020\n" 533 "bne 1f\n" 534 "ld 21, -224(1)\n" 535 "cmpwi 21, 0x2121\n" 536 "bne 1f\n" 537 "ld 22, -232(1)\n" 538 "cmpwi 22, 0x2222\n" 539 "bne 1f\n" 540 "ld 23, -240(1)\n" 541 "cmpwi 23, 0x2323\n" 542 "bne 1f\n" 543 "ld 24, -248(1)\n" 544 "cmpwi 24, 0x2424\n" 545 "bne 1f\n" 546 "ld 25, -256(1)\n" 547 "cmpwi 25, 0x2525\n" 548 "bne 1f\n" 549 "ld 26, -264(1)\n" 550 "cmpwi 26, 0x2626\n" 551 "bne 1f\n" 552 "ld 27, -272(1)\n" 553 "cmpwi 27, 0x2727\n" 554 "bne 1f\n" 555 "ld 28, -280(1)\n" 556 "cmpwi 28, 0x2828\n" 557 "bne 1f\n" 558 "ld 29, -288(1)\n" 559 "cmpwi 29, 0x2929\n" 560 "bne 1f\n" 561 562 /* Load 0 (success) to return */ 563 "li 0, 0\n" 564 565 "1: mr %0, 0\n" 566 567 : "=r" (rc) 568 : /* no inputs */ 569 : "3", "4", "5", "6", "7", "8", "9", "10", "11", "14", 570 "15", "16", "17", "18", "19", "20", "21", "22", "23", 571 "24", "25", "26", "27", "28", "29", "30", "31", 572 "memory" 573 ); 574 575 return rc; 576 } 577 578 int core_busy_loop_with_freeze(void) 579 { 580 int rc; 581 582 mtspr(SPRN_MMCR0, mfspr(SPRN_MMCR0) & ~MMCR0_FC); 583 rc = core_busy_loop(); 584 mtspr(SPRN_MMCR0, mfspr(SPRN_MMCR0) | MMCR0_FC); 585 586 return rc; 587 } 588 589 int ebb_child(union pipe read_pipe, union pipe write_pipe) 590 { 591 struct event event; 592 uint64_t val; 593 594 FAIL_IF(wait_for_parent(read_pipe)); 595 596 event_init_named(&event, 0x1001e, "cycles"); 597 event_leader_ebb_init(&event); 598 599 event.attr.exclude_kernel = 1; 600 event.attr.exclude_hv = 1; 601 event.attr.exclude_idle = 1; 602 603 FAIL_IF(event_open(&event)); 604 605 ebb_enable_pmc_counting(1); 606 setup_ebb_handler(standard_ebb_callee); 607 ebb_global_enable(); 608 609 FAIL_IF(event_enable(&event)); 610 611 if (event_read(&event)) { 612 /* 613 * Some tests expect to fail here, so don't report an error on 614 * this line, and return a distinguisable error code. Tell the 615 * parent an error happened. 616 */ 617 notify_parent_of_error(write_pipe); 618 return 2; 619 } 620 621 mtspr(SPRN_PMC1, pmc_sample_period(sample_period)); 622 623 FAIL_IF(notify_parent(write_pipe)); 624 FAIL_IF(wait_for_parent(read_pipe)); 625 FAIL_IF(notify_parent(write_pipe)); 626 627 while (ebb_state.stats.ebb_count < 20) { 628 FAIL_IF(core_busy_loop()); 629 630 /* To try and hit SIGILL case */ 631 val = mfspr(SPRN_MMCRA); 632 val |= mfspr(SPRN_MMCR2); 633 val |= mfspr(SPRN_MMCR0); 634 } 635 636 ebb_global_disable(); 637 ebb_freeze_pmcs(); 638 639 count_pmc(1, sample_period); 640 641 dump_ebb_state(); 642 643 event_close(&event); 644 645 FAIL_IF(ebb_state.stats.ebb_count == 0); 646 647 return 0; 648 } 649 650 static jmp_buf setjmp_env; 651 652 static void sigill_handler(int signal) 653 { 654 printf("Took sigill\n"); 655 longjmp(setjmp_env, 1); 656 } 657 658 static struct sigaction sigill_action = { 659 .sa_handler = sigill_handler, 660 }; 661 662 int catch_sigill(void (*func)(void)) 663 { 664 if (sigaction(SIGILL, &sigill_action, NULL)) { 665 perror("sigaction"); 666 return 1; 667 } 668 669 if (setjmp(setjmp_env) == 0) { 670 func(); 671 return 1; 672 } 673 674 return 0; 675 } 676 677 void write_pmc1(void) 678 { 679 mtspr(SPRN_PMC1, 0); 680 } 681 682 void write_pmc(int pmc, u64 value) 683 { 684 switch (pmc) { 685 case 1: mtspr(SPRN_PMC1, value); break; 686 case 2: mtspr(SPRN_PMC2, value); break; 687 case 3: mtspr(SPRN_PMC3, value); break; 688 case 4: mtspr(SPRN_PMC4, value); break; 689 case 5: mtspr(SPRN_PMC5, value); break; 690 case 6: mtspr(SPRN_PMC6, value); break; 691 } 692 } 693 694 u64 read_pmc(int pmc) 695 { 696 switch (pmc) { 697 case 1: return mfspr(SPRN_PMC1); 698 case 2: return mfspr(SPRN_PMC2); 699 case 3: return mfspr(SPRN_PMC3); 700 case 4: return mfspr(SPRN_PMC4); 701 case 5: return mfspr(SPRN_PMC5); 702 case 6: return mfspr(SPRN_PMC6); 703 } 704 705 return 0; 706 } 707 708 static void term_handler(int signal) 709 { 710 dump_summary_ebb_state(); 711 dump_ebb_hw_state(); 712 abort(); 713 } 714 715 struct sigaction term_action = { 716 .sa_handler = term_handler, 717 }; 718 719 static void __attribute__((constructor)) ebb_init(void) 720 { 721 clear_ebb_stats(); 722 723 if (sigaction(SIGTERM, &term_action, NULL)) 724 perror("sigaction"); 725 726 ebb_state.trace = trace_buffer_allocate(1 * 1024 * 1024); 727 } 728