1 /* 2 * Copyright 2014, Michael Ellerman, IBM Corp. 3 * Licensed under GPLv2. 4 */ 5 6 #define _GNU_SOURCE /* For CPU_ZERO etc. */ 7 8 #include <sched.h> 9 #include <sys/wait.h> 10 #include <setjmp.h> 11 #include <signal.h> 12 #include <stdio.h> 13 #include <stdlib.h> 14 #include <string.h> 15 #include <sys/ioctl.h> 16 17 #include "trace.h" 18 #include "ebb.h" 19 20 21 void (*ebb_user_func)(void); 22 23 void ebb_hook(void) 24 { 25 if (ebb_user_func) 26 ebb_user_func(); 27 } 28 29 struct ebb_state ebb_state; 30 31 u64 sample_period = 0x40000000ull; 32 33 void reset_ebb_with_clear_mask(unsigned long mmcr0_clear_mask) 34 { 35 u64 val; 36 37 /* 2) clear MMCR0[PMAO] - docs say BESCR[PMEO] should do this */ 38 /* 3) set MMCR0[PMAE] - docs say BESCR[PME] should do this */ 39 val = mfspr(SPRN_MMCR0); 40 mtspr(SPRN_MMCR0, (val & ~mmcr0_clear_mask) | MMCR0_PMAE); 41 42 /* 4) clear BESCR[PMEO] */ 43 mtspr(SPRN_BESCRR, BESCR_PMEO); 44 45 /* 5) set BESCR[PME] */ 46 mtspr(SPRN_BESCRS, BESCR_PME); 47 48 /* 6) rfebb 1 - done in our caller */ 49 } 50 51 void reset_ebb(void) 52 { 53 reset_ebb_with_clear_mask(MMCR0_PMAO | MMCR0_FC); 54 } 55 56 /* Called outside of the EBB handler to check MMCR0 is sane */ 57 int ebb_check_mmcr0(void) 58 { 59 u64 val; 60 61 val = mfspr(SPRN_MMCR0); 62 if ((val & (MMCR0_FC | MMCR0_PMAO)) == MMCR0_FC) { 63 /* It's OK if we see FC & PMAO, but not FC by itself */ 64 printf("Outside of loop, only FC set 0x%llx\n", val); 65 return 1; 66 } 67 68 return 0; 69 } 70 71 bool ebb_check_count(int pmc, u64 sample_period, int fudge) 72 { 73 u64 count, upper, lower; 74 75 count = ebb_state.stats.pmc_count[PMC_INDEX(pmc)]; 76 77 lower = ebb_state.stats.ebb_count * (sample_period - fudge); 78 79 if (count < lower) { 80 printf("PMC%d count (0x%llx) below lower limit 0x%llx (-0x%llx)\n", 81 pmc, count, lower, lower - count); 82 return false; 83 } 84 85 upper = ebb_state.stats.ebb_count * (sample_period + fudge); 86 87 if (count > upper) { 88 printf("PMC%d count (0x%llx) above upper limit 0x%llx (+0x%llx)\n", 89 pmc, count, upper, count - upper); 90 return false; 91 } 92 93 printf("PMC%d count (0x%llx) is between 0x%llx and 0x%llx delta +0x%llx/-0x%llx\n", 94 pmc, count, lower, upper, count - lower, upper - count); 95 96 return true; 97 } 98 99 void standard_ebb_callee(void) 100 { 101 int found, i; 102 u64 val; 103 104 val = mfspr(SPRN_BESCR); 105 if (!(val & BESCR_PMEO)) { 106 ebb_state.stats.spurious++; 107 goto out; 108 } 109 110 ebb_state.stats.ebb_count++; 111 trace_log_counter(ebb_state.trace, ebb_state.stats.ebb_count); 112 113 val = mfspr(SPRN_MMCR0); 114 trace_log_reg(ebb_state.trace, SPRN_MMCR0, val); 115 116 found = 0; 117 for (i = 1; i <= 6; i++) { 118 if (ebb_state.pmc_enable[PMC_INDEX(i)]) 119 found += count_pmc(i, sample_period); 120 } 121 122 if (!found) 123 ebb_state.stats.no_overflow++; 124 125 out: 126 reset_ebb(); 127 } 128 129 extern void ebb_handler(void); 130 131 void setup_ebb_handler(void (*callee)(void)) 132 { 133 u64 entry; 134 135 #if defined(_CALL_ELF) && _CALL_ELF == 2 136 entry = (u64)ebb_handler; 137 #else 138 struct opd 139 { 140 u64 entry; 141 u64 toc; 142 } *opd; 143 144 opd = (struct opd *)ebb_handler; 145 entry = opd->entry; 146 #endif 147 printf("EBB Handler is at %#llx\n", entry); 148 149 ebb_user_func = callee; 150 151 /* Ensure ebb_user_func is set before we set the handler */ 152 mb(); 153 mtspr(SPRN_EBBHR, entry); 154 155 /* Make sure the handler is set before we return */ 156 mb(); 157 } 158 159 void clear_ebb_stats(void) 160 { 161 memset(&ebb_state.stats, 0, sizeof(ebb_state.stats)); 162 } 163 164 void dump_summary_ebb_state(void) 165 { 166 printf("ebb_state:\n" \ 167 " ebb_count = %d\n" \ 168 " spurious = %d\n" \ 169 " negative = %d\n" \ 170 " no_overflow = %d\n" \ 171 " pmc[1] count = 0x%llx\n" \ 172 " pmc[2] count = 0x%llx\n" \ 173 " pmc[3] count = 0x%llx\n" \ 174 " pmc[4] count = 0x%llx\n" \ 175 " pmc[5] count = 0x%llx\n" \ 176 " pmc[6] count = 0x%llx\n", 177 ebb_state.stats.ebb_count, ebb_state.stats.spurious, 178 ebb_state.stats.negative, ebb_state.stats.no_overflow, 179 ebb_state.stats.pmc_count[0], ebb_state.stats.pmc_count[1], 180 ebb_state.stats.pmc_count[2], ebb_state.stats.pmc_count[3], 181 ebb_state.stats.pmc_count[4], ebb_state.stats.pmc_count[5]); 182 } 183 184 static char *decode_mmcr0(u32 value) 185 { 186 static char buf[16]; 187 188 buf[0] = '\0'; 189 190 if (value & (1 << 31)) 191 strcat(buf, "FC "); 192 if (value & (1 << 26)) 193 strcat(buf, "PMAE "); 194 if (value & (1 << 7)) 195 strcat(buf, "PMAO "); 196 197 return buf; 198 } 199 200 static char *decode_bescr(u64 value) 201 { 202 static char buf[16]; 203 204 buf[0] = '\0'; 205 206 if (value & (1ull << 63)) 207 strcat(buf, "GE "); 208 if (value & (1ull << 32)) 209 strcat(buf, "PMAE "); 210 if (value & 1) 211 strcat(buf, "PMAO "); 212 213 return buf; 214 } 215 216 void dump_ebb_hw_state(void) 217 { 218 u64 bescr; 219 u32 mmcr0; 220 221 mmcr0 = mfspr(SPRN_MMCR0); 222 bescr = mfspr(SPRN_BESCR); 223 224 printf("HW state:\n" \ 225 "MMCR0 0x%016x %s\n" \ 226 "MMCR2 0x%016lx\n" \ 227 "EBBHR 0x%016lx\n" \ 228 "BESCR 0x%016llx %s\n" \ 229 "PMC1 0x%016lx\n" \ 230 "PMC2 0x%016lx\n" \ 231 "PMC3 0x%016lx\n" \ 232 "PMC4 0x%016lx\n" \ 233 "PMC5 0x%016lx\n" \ 234 "PMC6 0x%016lx\n" \ 235 "SIAR 0x%016lx\n", 236 mmcr0, decode_mmcr0(mmcr0), mfspr(SPRN_MMCR2), 237 mfspr(SPRN_EBBHR), bescr, decode_bescr(bescr), 238 mfspr(SPRN_PMC1), mfspr(SPRN_PMC2), mfspr(SPRN_PMC3), 239 mfspr(SPRN_PMC4), mfspr(SPRN_PMC5), mfspr(SPRN_PMC6), 240 mfspr(SPRN_SIAR)); 241 } 242 243 void dump_ebb_state(void) 244 { 245 dump_summary_ebb_state(); 246 247 dump_ebb_hw_state(); 248 249 trace_buffer_print(ebb_state.trace); 250 } 251 252 int count_pmc(int pmc, uint32_t sample_period) 253 { 254 uint32_t start_value; 255 u64 val; 256 257 /* 0) Read PMC */ 258 start_value = pmc_sample_period(sample_period); 259 260 val = read_pmc(pmc); 261 if (val < start_value) 262 ebb_state.stats.negative++; 263 else 264 ebb_state.stats.pmc_count[PMC_INDEX(pmc)] += val - start_value; 265 266 trace_log_reg(ebb_state.trace, SPRN_PMC1 + pmc - 1, val); 267 268 /* 1) Reset PMC */ 269 write_pmc(pmc, start_value); 270 271 /* Report if we overflowed */ 272 return val >= COUNTER_OVERFLOW; 273 } 274 275 int ebb_event_enable(struct event *e) 276 { 277 int rc; 278 279 /* Ensure any SPR writes are ordered vs us */ 280 mb(); 281 282 rc = ioctl(e->fd, PERF_EVENT_IOC_ENABLE); 283 if (rc) 284 return rc; 285 286 rc = event_read(e); 287 288 /* Ditto */ 289 mb(); 290 291 return rc; 292 } 293 294 void ebb_freeze_pmcs(void) 295 { 296 mtspr(SPRN_MMCR0, mfspr(SPRN_MMCR0) | MMCR0_FC); 297 mb(); 298 } 299 300 void ebb_unfreeze_pmcs(void) 301 { 302 /* Unfreeze counters */ 303 mtspr(SPRN_MMCR0, mfspr(SPRN_MMCR0) & ~MMCR0_FC); 304 mb(); 305 } 306 307 void ebb_global_enable(void) 308 { 309 /* Enable EBBs globally and PMU EBBs */ 310 mtspr(SPRN_BESCR, 0x8000000100000000ull); 311 mb(); 312 } 313 314 void ebb_global_disable(void) 315 { 316 /* Disable EBBs & freeze counters, events are still scheduled */ 317 mtspr(SPRN_BESCRR, BESCR_PME); 318 mb(); 319 } 320 321 bool ebb_is_supported(void) 322 { 323 #ifdef PPC_FEATURE2_EBB 324 /* EBB requires at least POWER8 */ 325 return have_hwcap2(PPC_FEATURE2_EBB); 326 #else 327 return false; 328 #endif 329 } 330 331 void event_ebb_init(struct event *e) 332 { 333 e->attr.config |= (1ull << 63); 334 } 335 336 void event_bhrb_init(struct event *e, unsigned ifm) 337 { 338 e->attr.config |= (1ull << 62) | ((u64)ifm << 60); 339 } 340 341 void event_leader_ebb_init(struct event *e) 342 { 343 event_ebb_init(e); 344 345 e->attr.exclusive = 1; 346 e->attr.pinned = 1; 347 } 348 349 int ebb_child(union pipe read_pipe, union pipe write_pipe) 350 { 351 struct event event; 352 uint64_t val; 353 354 FAIL_IF(wait_for_parent(read_pipe)); 355 356 event_init_named(&event, 0x1001e, "cycles"); 357 event_leader_ebb_init(&event); 358 359 event.attr.exclude_kernel = 1; 360 event.attr.exclude_hv = 1; 361 event.attr.exclude_idle = 1; 362 363 FAIL_IF(event_open(&event)); 364 365 ebb_enable_pmc_counting(1); 366 setup_ebb_handler(standard_ebb_callee); 367 ebb_global_enable(); 368 369 FAIL_IF(event_enable(&event)); 370 371 if (event_read(&event)) { 372 /* 373 * Some tests expect to fail here, so don't report an error on 374 * this line, and return a distinguisable error code. Tell the 375 * parent an error happened. 376 */ 377 notify_parent_of_error(write_pipe); 378 return 2; 379 } 380 381 mtspr(SPRN_PMC1, pmc_sample_period(sample_period)); 382 383 FAIL_IF(notify_parent(write_pipe)); 384 FAIL_IF(wait_for_parent(read_pipe)); 385 FAIL_IF(notify_parent(write_pipe)); 386 387 while (ebb_state.stats.ebb_count < 20) { 388 FAIL_IF(core_busy_loop()); 389 390 /* To try and hit SIGILL case */ 391 val = mfspr(SPRN_MMCRA); 392 val |= mfspr(SPRN_MMCR2); 393 val |= mfspr(SPRN_MMCR0); 394 } 395 396 ebb_global_disable(); 397 ebb_freeze_pmcs(); 398 399 count_pmc(1, sample_period); 400 401 dump_ebb_state(); 402 403 event_close(&event); 404 405 FAIL_IF(ebb_state.stats.ebb_count == 0); 406 407 return 0; 408 } 409 410 static jmp_buf setjmp_env; 411 412 static void sigill_handler(int signal) 413 { 414 printf("Took sigill\n"); 415 longjmp(setjmp_env, 1); 416 } 417 418 static struct sigaction sigill_action = { 419 .sa_handler = sigill_handler, 420 }; 421 422 int catch_sigill(void (*func)(void)) 423 { 424 if (sigaction(SIGILL, &sigill_action, NULL)) { 425 perror("sigaction"); 426 return 1; 427 } 428 429 if (setjmp(setjmp_env) == 0) { 430 func(); 431 return 1; 432 } 433 434 return 0; 435 } 436 437 void write_pmc1(void) 438 { 439 mtspr(SPRN_PMC1, 0); 440 } 441 442 void write_pmc(int pmc, u64 value) 443 { 444 switch (pmc) { 445 case 1: mtspr(SPRN_PMC1, value); break; 446 case 2: mtspr(SPRN_PMC2, value); break; 447 case 3: mtspr(SPRN_PMC3, value); break; 448 case 4: mtspr(SPRN_PMC4, value); break; 449 case 5: mtspr(SPRN_PMC5, value); break; 450 case 6: mtspr(SPRN_PMC6, value); break; 451 } 452 } 453 454 u64 read_pmc(int pmc) 455 { 456 switch (pmc) { 457 case 1: return mfspr(SPRN_PMC1); 458 case 2: return mfspr(SPRN_PMC2); 459 case 3: return mfspr(SPRN_PMC3); 460 case 4: return mfspr(SPRN_PMC4); 461 case 5: return mfspr(SPRN_PMC5); 462 case 6: return mfspr(SPRN_PMC6); 463 } 464 465 return 0; 466 } 467 468 static void term_handler(int signal) 469 { 470 dump_summary_ebb_state(); 471 dump_ebb_hw_state(); 472 abort(); 473 } 474 475 struct sigaction term_action = { 476 .sa_handler = term_handler, 477 }; 478 479 static void __attribute__((constructor)) ebb_init(void) 480 { 481 clear_ebb_stats(); 482 483 if (sigaction(SIGTERM, &term_action, NULL)) 484 perror("sigaction"); 485 486 ebb_state.trace = trace_buffer_allocate(1 * 1024 * 1024); 487 } 488