1 #if defined(CONFIG_CPU_MIPS32) || defined(CONFIG_CPU_MIPS64) || \ 2 defined(CONFIG_CPU_R10000) || defined(CONFIG_CPU_SB1) 3 4 #define M_CONFIG1_PC (1 << 4) 5 6 #define M_PERFCTL_EXL (1UL << 0) 7 #define M_PERFCTL_KERNEL (1UL << 1) 8 #define M_PERFCTL_SUPERVISOR (1UL << 2) 9 #define M_PERFCTL_USER (1UL << 3) 10 #define M_PERFCTL_INTERRUPT_ENABLE (1UL << 4) 11 #define M_PERFCTL_EVENT(event) (((event) & 0x3ff) << 5) 12 #define M_PERFCTL_VPEID(vpe) ((vpe) << 16) 13 #define M_PERFCTL_MT_EN(filter) ((filter) << 20) 14 #define M_TC_EN_ALL M_PERFCTL_MT_EN(0) 15 #define M_TC_EN_VPE M_PERFCTL_MT_EN(1) 16 #define M_TC_EN_TC M_PERFCTL_MT_EN(2) 17 #define M_PERFCTL_TCID(tcid) ((tcid) << 22) 18 #define M_PERFCTL_WIDE (1UL << 30) 19 #define M_PERFCTL_MORE (1UL << 31) 20 21 #define M_PERFCTL_COUNT_EVENT_WHENEVER (M_PERFCTL_EXL | \ 22 M_PERFCTL_KERNEL | \ 23 M_PERFCTL_USER | \ 24 M_PERFCTL_SUPERVISOR | \ 25 M_PERFCTL_INTERRUPT_ENABLE) 26 27 #ifdef CONFIG_MIPS_MT_SMP 28 #define M_PERFCTL_CONFIG_MASK 0x3fff801f 29 #else 30 #define M_PERFCTL_CONFIG_MASK 0x1f 31 #endif 32 #define M_PERFCTL_EVENT_MASK 0xfe0 33 34 #define M_COUNTER_OVERFLOW (1UL << 31) 35 36 #ifdef CONFIG_MIPS_MT_SMP 37 static int cpu_has_mipsmt_pertccounters; 38 39 /* 40 * FIXME: For VSMP, vpe_id() is redefined for Perf-events, because 41 * cpu_data[cpuid].vpe_id reports 0 for _both_ CPUs. 42 */ 43 #if defined(CONFIG_HW_PERF_EVENTS) 44 #define vpe_id() (cpu_has_mipsmt_pertccounters ? \ 45 0 : smp_processor_id()) 46 #else 47 #define vpe_id() (cpu_has_mipsmt_pertccounters ? \ 48 0 : cpu_data[smp_processor_id()].vpe_id) 49 #endif 50 51 /* Copied from op_model_mipsxx.c */ 52 static inline unsigned int vpe_shift(void) 53 { 54 if (num_possible_cpus() > 1) 55 return 1; 56 57 return 0; 58 } 59 #else /* !CONFIG_MIPS_MT_SMP */ 60 #define vpe_id() 0 61 62 static inline unsigned int vpe_shift(void) 63 { 64 return 0; 65 } 66 #endif /* CONFIG_MIPS_MT_SMP */ 67 68 static inline unsigned int 69 counters_total_to_per_cpu(unsigned int counters) 70 { 71 return counters >> vpe_shift(); 72 } 73 74 static inline unsigned int 75 counters_per_cpu_to_total(unsigned int counters) 76 { 77 return counters << vpe_shift(); 78 } 79 80 #define __define_perf_accessors(r, n, np) \ 81 \ 82 static inline unsigned int r_c0_ ## r ## n(void) \ 83 { \ 84 unsigned int cpu = vpe_id(); \ 85 \ 86 switch (cpu) { \ 87 case 0: \ 88 return read_c0_ ## r ## n(); \ 89 case 1: \ 90 return read_c0_ ## r ## np(); \ 91 default: \ 92 BUG(); \ 93 } \ 94 return 0; \ 95 } \ 96 \ 97 static inline void w_c0_ ## r ## n(unsigned int value) \ 98 { \ 99 unsigned int cpu = vpe_id(); \ 100 \ 101 switch (cpu) { \ 102 case 0: \ 103 write_c0_ ## r ## n(value); \ 104 return; \ 105 case 1: \ 106 write_c0_ ## r ## np(value); \ 107 return; \ 108 default: \ 109 BUG(); \ 110 } \ 111 return; \ 112 } \ 113 114 __define_perf_accessors(perfcntr, 0, 2) 115 __define_perf_accessors(perfcntr, 1, 3) 116 __define_perf_accessors(perfcntr, 2, 0) 117 __define_perf_accessors(perfcntr, 3, 1) 118 119 __define_perf_accessors(perfctrl, 0, 2) 120 __define_perf_accessors(perfctrl, 1, 3) 121 __define_perf_accessors(perfctrl, 2, 0) 122 __define_perf_accessors(perfctrl, 3, 1) 123 124 static inline int __n_counters(void) 125 { 126 if (!(read_c0_config1() & M_CONFIG1_PC)) 127 return 0; 128 if (!(read_c0_perfctrl0() & M_PERFCTL_MORE)) 129 return 1; 130 if (!(read_c0_perfctrl1() & M_PERFCTL_MORE)) 131 return 2; 132 if (!(read_c0_perfctrl2() & M_PERFCTL_MORE)) 133 return 3; 134 135 return 4; 136 } 137 138 static inline int n_counters(void) 139 { 140 int counters; 141 142 switch (current_cpu_type()) { 143 case CPU_R10000: 144 counters = 2; 145 break; 146 147 case CPU_R12000: 148 case CPU_R14000: 149 counters = 4; 150 break; 151 152 default: 153 counters = __n_counters(); 154 } 155 156 return counters; 157 } 158 159 static void reset_counters(void *arg) 160 { 161 int counters = (int)(long)arg; 162 switch (counters) { 163 case 4: 164 w_c0_perfctrl3(0); 165 w_c0_perfcntr3(0); 166 case 3: 167 w_c0_perfctrl2(0); 168 w_c0_perfcntr2(0); 169 case 2: 170 w_c0_perfctrl1(0); 171 w_c0_perfcntr1(0); 172 case 1: 173 w_c0_perfctrl0(0); 174 w_c0_perfcntr0(0); 175 } 176 } 177 178 static inline u64 179 mipsxx_pmu_read_counter(unsigned int idx) 180 { 181 switch (idx) { 182 case 0: 183 return r_c0_perfcntr0(); 184 case 1: 185 return r_c0_perfcntr1(); 186 case 2: 187 return r_c0_perfcntr2(); 188 case 3: 189 return r_c0_perfcntr3(); 190 default: 191 WARN_ONCE(1, "Invalid performance counter number (%d)\n", idx); 192 return 0; 193 } 194 } 195 196 static inline void 197 mipsxx_pmu_write_counter(unsigned int idx, u64 val) 198 { 199 switch (idx) { 200 case 0: 201 w_c0_perfcntr0(val); 202 return; 203 case 1: 204 w_c0_perfcntr1(val); 205 return; 206 case 2: 207 w_c0_perfcntr2(val); 208 return; 209 case 3: 210 w_c0_perfcntr3(val); 211 return; 212 } 213 } 214 215 static inline unsigned int 216 mipsxx_pmu_read_control(unsigned int idx) 217 { 218 switch (idx) { 219 case 0: 220 return r_c0_perfctrl0(); 221 case 1: 222 return r_c0_perfctrl1(); 223 case 2: 224 return r_c0_perfctrl2(); 225 case 3: 226 return r_c0_perfctrl3(); 227 default: 228 WARN_ONCE(1, "Invalid performance counter number (%d)\n", idx); 229 return 0; 230 } 231 } 232 233 static inline void 234 mipsxx_pmu_write_control(unsigned int idx, unsigned int val) 235 { 236 switch (idx) { 237 case 0: 238 w_c0_perfctrl0(val); 239 return; 240 case 1: 241 w_c0_perfctrl1(val); 242 return; 243 case 2: 244 w_c0_perfctrl2(val); 245 return; 246 case 3: 247 w_c0_perfctrl3(val); 248 return; 249 } 250 } 251 252 #ifdef CONFIG_MIPS_MT_SMP 253 static DEFINE_RWLOCK(pmuint_rwlock); 254 #endif 255 256 /* 24K/34K/1004K cores can share the same event map. */ 257 static const struct mips_perf_event mipsxxcore_event_map 258 [PERF_COUNT_HW_MAX] = { 259 [PERF_COUNT_HW_CPU_CYCLES] = { 0x00, CNTR_EVEN | CNTR_ODD, P }, 260 [PERF_COUNT_HW_INSTRUCTIONS] = { 0x01, CNTR_EVEN | CNTR_ODD, T }, 261 [PERF_COUNT_HW_CACHE_REFERENCES] = { UNSUPPORTED_PERF_EVENT_ID }, 262 [PERF_COUNT_HW_CACHE_MISSES] = { UNSUPPORTED_PERF_EVENT_ID }, 263 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = { 0x02, CNTR_EVEN, T }, 264 [PERF_COUNT_HW_BRANCH_MISSES] = { 0x02, CNTR_ODD, T }, 265 [PERF_COUNT_HW_BUS_CYCLES] = { UNSUPPORTED_PERF_EVENT_ID }, 266 }; 267 268 /* 74K core has different branch event code. */ 269 static const struct mips_perf_event mipsxx74Kcore_event_map 270 [PERF_COUNT_HW_MAX] = { 271 [PERF_COUNT_HW_CPU_CYCLES] = { 0x00, CNTR_EVEN | CNTR_ODD, P }, 272 [PERF_COUNT_HW_INSTRUCTIONS] = { 0x01, CNTR_EVEN | CNTR_ODD, T }, 273 [PERF_COUNT_HW_CACHE_REFERENCES] = { UNSUPPORTED_PERF_EVENT_ID }, 274 [PERF_COUNT_HW_CACHE_MISSES] = { UNSUPPORTED_PERF_EVENT_ID }, 275 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = { 0x27, CNTR_EVEN, T }, 276 [PERF_COUNT_HW_BRANCH_MISSES] = { 0x27, CNTR_ODD, T }, 277 [PERF_COUNT_HW_BUS_CYCLES] = { UNSUPPORTED_PERF_EVENT_ID }, 278 }; 279 280 /* 24K/34K/1004K cores can share the same cache event map. */ 281 static const struct mips_perf_event mipsxxcore_cache_map 282 [PERF_COUNT_HW_CACHE_MAX] 283 [PERF_COUNT_HW_CACHE_OP_MAX] 284 [PERF_COUNT_HW_CACHE_RESULT_MAX] = { 285 [C(L1D)] = { 286 /* 287 * Like some other architectures (e.g. ARM), the performance 288 * counters don't differentiate between read and write 289 * accesses/misses, so this isn't strictly correct, but it's the 290 * best we can do. Writes and reads get combined. 291 */ 292 [C(OP_READ)] = { 293 [C(RESULT_ACCESS)] = { 0x0a, CNTR_EVEN, T }, 294 [C(RESULT_MISS)] = { 0x0b, CNTR_EVEN | CNTR_ODD, T }, 295 }, 296 [C(OP_WRITE)] = { 297 [C(RESULT_ACCESS)] = { 0x0a, CNTR_EVEN, T }, 298 [C(RESULT_MISS)] = { 0x0b, CNTR_EVEN | CNTR_ODD, T }, 299 }, 300 [C(OP_PREFETCH)] = { 301 [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID }, 302 [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, 303 }, 304 }, 305 [C(L1I)] = { 306 [C(OP_READ)] = { 307 [C(RESULT_ACCESS)] = { 0x09, CNTR_EVEN, T }, 308 [C(RESULT_MISS)] = { 0x09, CNTR_ODD, T }, 309 }, 310 [C(OP_WRITE)] = { 311 [C(RESULT_ACCESS)] = { 0x09, CNTR_EVEN, T }, 312 [C(RESULT_MISS)] = { 0x09, CNTR_ODD, T }, 313 }, 314 [C(OP_PREFETCH)] = { 315 [C(RESULT_ACCESS)] = { 0x14, CNTR_EVEN, T }, 316 /* 317 * Note that MIPS has only "hit" events countable for 318 * the prefetch operation. 319 */ 320 [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, 321 }, 322 }, 323 [C(LL)] = { 324 [C(OP_READ)] = { 325 [C(RESULT_ACCESS)] = { 0x15, CNTR_ODD, P }, 326 [C(RESULT_MISS)] = { 0x16, CNTR_EVEN, P }, 327 }, 328 [C(OP_WRITE)] = { 329 [C(RESULT_ACCESS)] = { 0x15, CNTR_ODD, P }, 330 [C(RESULT_MISS)] = { 0x16, CNTR_EVEN, P }, 331 }, 332 [C(OP_PREFETCH)] = { 333 [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID }, 334 [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, 335 }, 336 }, 337 [C(DTLB)] = { 338 [C(OP_READ)] = { 339 [C(RESULT_ACCESS)] = { 0x06, CNTR_EVEN, T }, 340 [C(RESULT_MISS)] = { 0x06, CNTR_ODD, T }, 341 }, 342 [C(OP_WRITE)] = { 343 [C(RESULT_ACCESS)] = { 0x06, CNTR_EVEN, T }, 344 [C(RESULT_MISS)] = { 0x06, CNTR_ODD, T }, 345 }, 346 [C(OP_PREFETCH)] = { 347 [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID }, 348 [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, 349 }, 350 }, 351 [C(ITLB)] = { 352 [C(OP_READ)] = { 353 [C(RESULT_ACCESS)] = { 0x05, CNTR_EVEN, T }, 354 [C(RESULT_MISS)] = { 0x05, CNTR_ODD, T }, 355 }, 356 [C(OP_WRITE)] = { 357 [C(RESULT_ACCESS)] = { 0x05, CNTR_EVEN, T }, 358 [C(RESULT_MISS)] = { 0x05, CNTR_ODD, T }, 359 }, 360 [C(OP_PREFETCH)] = { 361 [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID }, 362 [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, 363 }, 364 }, 365 [C(BPU)] = { 366 /* Using the same code for *HW_BRANCH* */ 367 [C(OP_READ)] = { 368 [C(RESULT_ACCESS)] = { 0x02, CNTR_EVEN, T }, 369 [C(RESULT_MISS)] = { 0x02, CNTR_ODD, T }, 370 }, 371 [C(OP_WRITE)] = { 372 [C(RESULT_ACCESS)] = { 0x02, CNTR_EVEN, T }, 373 [C(RESULT_MISS)] = { 0x02, CNTR_ODD, T }, 374 }, 375 [C(OP_PREFETCH)] = { 376 [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID }, 377 [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, 378 }, 379 }, 380 }; 381 382 /* 74K core has completely different cache event map. */ 383 static const struct mips_perf_event mipsxx74Kcore_cache_map 384 [PERF_COUNT_HW_CACHE_MAX] 385 [PERF_COUNT_HW_CACHE_OP_MAX] 386 [PERF_COUNT_HW_CACHE_RESULT_MAX] = { 387 [C(L1D)] = { 388 /* 389 * Like some other architectures (e.g. ARM), the performance 390 * counters don't differentiate between read and write 391 * accesses/misses, so this isn't strictly correct, but it's the 392 * best we can do. Writes and reads get combined. 393 */ 394 [C(OP_READ)] = { 395 [C(RESULT_ACCESS)] = { 0x17, CNTR_ODD, T }, 396 [C(RESULT_MISS)] = { 0x18, CNTR_ODD, T }, 397 }, 398 [C(OP_WRITE)] = { 399 [C(RESULT_ACCESS)] = { 0x17, CNTR_ODD, T }, 400 [C(RESULT_MISS)] = { 0x18, CNTR_ODD, T }, 401 }, 402 [C(OP_PREFETCH)] = { 403 [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID }, 404 [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, 405 }, 406 }, 407 [C(L1I)] = { 408 [C(OP_READ)] = { 409 [C(RESULT_ACCESS)] = { 0x06, CNTR_EVEN, T }, 410 [C(RESULT_MISS)] = { 0x06, CNTR_ODD, T }, 411 }, 412 [C(OP_WRITE)] = { 413 [C(RESULT_ACCESS)] = { 0x06, CNTR_EVEN, T }, 414 [C(RESULT_MISS)] = { 0x06, CNTR_ODD, T }, 415 }, 416 [C(OP_PREFETCH)] = { 417 [C(RESULT_ACCESS)] = { 0x34, CNTR_EVEN, T }, 418 /* 419 * Note that MIPS has only "hit" events countable for 420 * the prefetch operation. 421 */ 422 [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, 423 }, 424 }, 425 [C(LL)] = { 426 [C(OP_READ)] = { 427 [C(RESULT_ACCESS)] = { 0x1c, CNTR_ODD, P }, 428 [C(RESULT_MISS)] = { 0x1d, CNTR_EVEN | CNTR_ODD, P }, 429 }, 430 [C(OP_WRITE)] = { 431 [C(RESULT_ACCESS)] = { 0x1c, CNTR_ODD, P }, 432 [C(RESULT_MISS)] = { 0x1d, CNTR_EVEN | CNTR_ODD, P }, 433 }, 434 [C(OP_PREFETCH)] = { 435 [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID }, 436 [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, 437 }, 438 }, 439 [C(DTLB)] = { 440 /* 74K core does not have specific DTLB events. */ 441 [C(OP_READ)] = { 442 [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID }, 443 [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, 444 }, 445 [C(OP_WRITE)] = { 446 [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID }, 447 [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, 448 }, 449 [C(OP_PREFETCH)] = { 450 [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID }, 451 [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, 452 }, 453 }, 454 [C(ITLB)] = { 455 [C(OP_READ)] = { 456 [C(RESULT_ACCESS)] = { 0x04, CNTR_EVEN, T }, 457 [C(RESULT_MISS)] = { 0x04, CNTR_ODD, T }, 458 }, 459 [C(OP_WRITE)] = { 460 [C(RESULT_ACCESS)] = { 0x04, CNTR_EVEN, T }, 461 [C(RESULT_MISS)] = { 0x04, CNTR_ODD, T }, 462 }, 463 [C(OP_PREFETCH)] = { 464 [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID }, 465 [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, 466 }, 467 }, 468 [C(BPU)] = { 469 /* Using the same code for *HW_BRANCH* */ 470 [C(OP_READ)] = { 471 [C(RESULT_ACCESS)] = { 0x27, CNTR_EVEN, T }, 472 [C(RESULT_MISS)] = { 0x27, CNTR_ODD, T }, 473 }, 474 [C(OP_WRITE)] = { 475 [C(RESULT_ACCESS)] = { 0x27, CNTR_EVEN, T }, 476 [C(RESULT_MISS)] = { 0x27, CNTR_ODD, T }, 477 }, 478 [C(OP_PREFETCH)] = { 479 [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID }, 480 [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, 481 }, 482 }, 483 }; 484 485 #ifdef CONFIG_MIPS_MT_SMP 486 static void 487 check_and_calc_range(struct perf_event *event, 488 const struct mips_perf_event *pev) 489 { 490 struct hw_perf_event *hwc = &event->hw; 491 492 if (event->cpu >= 0) { 493 if (pev->range > V) { 494 /* 495 * The user selected an event that is processor 496 * wide, while expecting it to be VPE wide. 497 */ 498 hwc->config_base |= M_TC_EN_ALL; 499 } else { 500 /* 501 * FIXME: cpu_data[event->cpu].vpe_id reports 0 502 * for both CPUs. 503 */ 504 hwc->config_base |= M_PERFCTL_VPEID(event->cpu); 505 hwc->config_base |= M_TC_EN_VPE; 506 } 507 } else 508 hwc->config_base |= M_TC_EN_ALL; 509 } 510 #else 511 static void 512 check_and_calc_range(struct perf_event *event, 513 const struct mips_perf_event *pev) 514 { 515 } 516 #endif 517 518 static int __hw_perf_event_init(struct perf_event *event) 519 { 520 struct perf_event_attr *attr = &event->attr; 521 struct hw_perf_event *hwc = &event->hw; 522 const struct mips_perf_event *pev; 523 int err; 524 525 /* Returning MIPS event descriptor for generic perf event. */ 526 if (PERF_TYPE_HARDWARE == event->attr.type) { 527 if (event->attr.config >= PERF_COUNT_HW_MAX) 528 return -EINVAL; 529 pev = mipspmu_map_general_event(event->attr.config); 530 } else if (PERF_TYPE_HW_CACHE == event->attr.type) { 531 pev = mipspmu_map_cache_event(event->attr.config); 532 } else if (PERF_TYPE_RAW == event->attr.type) { 533 /* We are working on the global raw event. */ 534 mutex_lock(&raw_event_mutex); 535 pev = mipspmu->map_raw_event(event->attr.config); 536 } else { 537 /* The event type is not (yet) supported. */ 538 return -EOPNOTSUPP; 539 } 540 541 if (IS_ERR(pev)) { 542 if (PERF_TYPE_RAW == event->attr.type) 543 mutex_unlock(&raw_event_mutex); 544 return PTR_ERR(pev); 545 } 546 547 /* 548 * We allow max flexibility on how each individual counter shared 549 * by the single CPU operates (the mode exclusion and the range). 550 */ 551 hwc->config_base = M_PERFCTL_INTERRUPT_ENABLE; 552 553 /* Calculate range bits and validate it. */ 554 if (num_possible_cpus() > 1) 555 check_and_calc_range(event, pev); 556 557 hwc->event_base = mipspmu_perf_event_encode(pev); 558 if (PERF_TYPE_RAW == event->attr.type) 559 mutex_unlock(&raw_event_mutex); 560 561 if (!attr->exclude_user) 562 hwc->config_base |= M_PERFCTL_USER; 563 if (!attr->exclude_kernel) { 564 hwc->config_base |= M_PERFCTL_KERNEL; 565 /* MIPS kernel mode: KSU == 00b || EXL == 1 || ERL == 1 */ 566 hwc->config_base |= M_PERFCTL_EXL; 567 } 568 if (!attr->exclude_hv) 569 hwc->config_base |= M_PERFCTL_SUPERVISOR; 570 571 hwc->config_base &= M_PERFCTL_CONFIG_MASK; 572 /* 573 * The event can belong to another cpu. We do not assign a local 574 * counter for it for now. 575 */ 576 hwc->idx = -1; 577 hwc->config = 0; 578 579 if (!hwc->sample_period) { 580 hwc->sample_period = MAX_PERIOD; 581 hwc->last_period = hwc->sample_period; 582 local64_set(&hwc->period_left, hwc->sample_period); 583 } 584 585 err = 0; 586 if (event->group_leader != event) { 587 err = validate_group(event); 588 if (err) 589 return -EINVAL; 590 } 591 592 event->destroy = hw_perf_event_destroy; 593 594 return err; 595 } 596 597 static void pause_local_counters(void) 598 { 599 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); 600 int counters = mipspmu->num_counters; 601 unsigned long flags; 602 603 local_irq_save(flags); 604 switch (counters) { 605 case 4: 606 cpuc->saved_ctrl[3] = r_c0_perfctrl3(); 607 w_c0_perfctrl3(cpuc->saved_ctrl[3] & 608 ~M_PERFCTL_COUNT_EVENT_WHENEVER); 609 case 3: 610 cpuc->saved_ctrl[2] = r_c0_perfctrl2(); 611 w_c0_perfctrl2(cpuc->saved_ctrl[2] & 612 ~M_PERFCTL_COUNT_EVENT_WHENEVER); 613 case 2: 614 cpuc->saved_ctrl[1] = r_c0_perfctrl1(); 615 w_c0_perfctrl1(cpuc->saved_ctrl[1] & 616 ~M_PERFCTL_COUNT_EVENT_WHENEVER); 617 case 1: 618 cpuc->saved_ctrl[0] = r_c0_perfctrl0(); 619 w_c0_perfctrl0(cpuc->saved_ctrl[0] & 620 ~M_PERFCTL_COUNT_EVENT_WHENEVER); 621 } 622 local_irq_restore(flags); 623 } 624 625 static void resume_local_counters(void) 626 { 627 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); 628 int counters = mipspmu->num_counters; 629 unsigned long flags; 630 631 local_irq_save(flags); 632 switch (counters) { 633 case 4: 634 w_c0_perfctrl3(cpuc->saved_ctrl[3]); 635 case 3: 636 w_c0_perfctrl2(cpuc->saved_ctrl[2]); 637 case 2: 638 w_c0_perfctrl1(cpuc->saved_ctrl[1]); 639 case 1: 640 w_c0_perfctrl0(cpuc->saved_ctrl[0]); 641 } 642 local_irq_restore(flags); 643 } 644 645 static int mipsxx_pmu_handle_shared_irq(void) 646 { 647 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); 648 struct perf_sample_data data; 649 unsigned int counters = mipspmu->num_counters; 650 unsigned int counter; 651 int handled = IRQ_NONE; 652 struct pt_regs *regs; 653 654 if (cpu_has_mips_r2 && !(read_c0_cause() & (1 << 26))) 655 return handled; 656 657 /* 658 * First we pause the local counters, so that when we are locked 659 * here, the counters are all paused. When it gets locked due to 660 * perf_disable(), the timer interrupt handler will be delayed. 661 * 662 * See also mipsxx_pmu_start(). 663 */ 664 pause_local_counters(); 665 #ifdef CONFIG_MIPS_MT_SMP 666 read_lock(&pmuint_rwlock); 667 #endif 668 669 regs = get_irq_regs(); 670 671 perf_sample_data_init(&data, 0); 672 673 switch (counters) { 674 #define HANDLE_COUNTER(n) \ 675 case n + 1: \ 676 if (test_bit(n, cpuc->used_mask)) { \ 677 counter = r_c0_perfcntr ## n(); \ 678 if (counter & M_COUNTER_OVERFLOW) { \ 679 w_c0_perfcntr ## n(counter & \ 680 VALID_COUNT); \ 681 if (test_and_change_bit(n, cpuc->msbs)) \ 682 handle_associated_event(cpuc, \ 683 n, &data, regs); \ 684 handled = IRQ_HANDLED; \ 685 } \ 686 } 687 HANDLE_COUNTER(3) 688 HANDLE_COUNTER(2) 689 HANDLE_COUNTER(1) 690 HANDLE_COUNTER(0) 691 } 692 693 /* 694 * Do all the work for the pending perf events. We can do this 695 * in here because the performance counter interrupt is a regular 696 * interrupt, not NMI. 697 */ 698 if (handled == IRQ_HANDLED) 699 perf_event_do_pending(); 700 701 #ifdef CONFIG_MIPS_MT_SMP 702 read_unlock(&pmuint_rwlock); 703 #endif 704 resume_local_counters(); 705 return handled; 706 } 707 708 static irqreturn_t 709 mipsxx_pmu_handle_irq(int irq, void *dev) 710 { 711 return mipsxx_pmu_handle_shared_irq(); 712 } 713 714 static void mipsxx_pmu_start(void) 715 { 716 #ifdef CONFIG_MIPS_MT_SMP 717 write_unlock(&pmuint_rwlock); 718 #endif 719 resume_local_counters(); 720 } 721 722 /* 723 * MIPS performance counters can be per-TC. The control registers can 724 * not be directly accessed accross CPUs. Hence if we want to do global 725 * control, we need cross CPU calls. on_each_cpu() can help us, but we 726 * can not make sure this function is called with interrupts enabled. So 727 * here we pause local counters and then grab a rwlock and leave the 728 * counters on other CPUs alone. If any counter interrupt raises while 729 * we own the write lock, simply pause local counters on that CPU and 730 * spin in the handler. Also we know we won't be switched to another 731 * CPU after pausing local counters and before grabbing the lock. 732 */ 733 static void mipsxx_pmu_stop(void) 734 { 735 pause_local_counters(); 736 #ifdef CONFIG_MIPS_MT_SMP 737 write_lock(&pmuint_rwlock); 738 #endif 739 } 740 741 static int 742 mipsxx_pmu_alloc_counter(struct cpu_hw_events *cpuc, 743 struct hw_perf_event *hwc) 744 { 745 int i; 746 747 /* 748 * We only need to care the counter mask. The range has been 749 * checked definitely. 750 */ 751 unsigned long cntr_mask = (hwc->event_base >> 8) & 0xffff; 752 753 for (i = mipspmu->num_counters - 1; i >= 0; i--) { 754 /* 755 * Note that some MIPS perf events can be counted by both 756 * even and odd counters, wheresas many other are only by 757 * even _or_ odd counters. This introduces an issue that 758 * when the former kind of event takes the counter the 759 * latter kind of event wants to use, then the "counter 760 * allocation" for the latter event will fail. In fact if 761 * they can be dynamically swapped, they both feel happy. 762 * But here we leave this issue alone for now. 763 */ 764 if (test_bit(i, &cntr_mask) && 765 !test_and_set_bit(i, cpuc->used_mask)) 766 return i; 767 } 768 769 return -EAGAIN; 770 } 771 772 static void 773 mipsxx_pmu_enable_event(struct hw_perf_event *evt, int idx) 774 { 775 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); 776 unsigned long flags; 777 778 WARN_ON(idx < 0 || idx >= mipspmu->num_counters); 779 780 local_irq_save(flags); 781 cpuc->saved_ctrl[idx] = M_PERFCTL_EVENT(evt->event_base & 0xff) | 782 (evt->config_base & M_PERFCTL_CONFIG_MASK) | 783 /* Make sure interrupt enabled. */ 784 M_PERFCTL_INTERRUPT_ENABLE; 785 /* 786 * We do not actually let the counter run. Leave it until start(). 787 */ 788 local_irq_restore(flags); 789 } 790 791 static void 792 mipsxx_pmu_disable_event(int idx) 793 { 794 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); 795 unsigned long flags; 796 797 WARN_ON(idx < 0 || idx >= mipspmu->num_counters); 798 799 local_irq_save(flags); 800 cpuc->saved_ctrl[idx] = mipsxx_pmu_read_control(idx) & 801 ~M_PERFCTL_COUNT_EVENT_WHENEVER; 802 mipsxx_pmu_write_control(idx, cpuc->saved_ctrl[idx]); 803 local_irq_restore(flags); 804 } 805 806 /* 24K */ 807 #define IS_UNSUPPORTED_24K_EVENT(r, b) \ 808 ((b) == 12 || (r) == 151 || (r) == 152 || (b) == 26 || \ 809 (b) == 27 || (r) == 28 || (r) == 158 || (b) == 31 || \ 810 (b) == 32 || (b) == 34 || (b) == 36 || (r) == 168 || \ 811 (r) == 172 || (b) == 47 || ((b) >= 56 && (b) <= 63) || \ 812 ((b) >= 68 && (b) <= 127)) 813 #define IS_BOTH_COUNTERS_24K_EVENT(b) \ 814 ((b) == 0 || (b) == 1 || (b) == 11) 815 816 /* 34K */ 817 #define IS_UNSUPPORTED_34K_EVENT(r, b) \ 818 ((b) == 12 || (r) == 27 || (r) == 158 || (b) == 36 || \ 819 (b) == 38 || (r) == 175 || ((b) >= 56 && (b) <= 63) || \ 820 ((b) >= 68 && (b) <= 127)) 821 #define IS_BOTH_COUNTERS_34K_EVENT(b) \ 822 ((b) == 0 || (b) == 1 || (b) == 11) 823 #ifdef CONFIG_MIPS_MT_SMP 824 #define IS_RANGE_P_34K_EVENT(r, b) \ 825 ((b) == 0 || (r) == 18 || (b) == 21 || (b) == 22 || \ 826 (b) == 25 || (b) == 39 || (r) == 44 || (r) == 174 || \ 827 (r) == 176 || ((b) >= 50 && (b) <= 55) || \ 828 ((b) >= 64 && (b) <= 67)) 829 #define IS_RANGE_V_34K_EVENT(r) ((r) == 47) 830 #endif 831 832 /* 74K */ 833 #define IS_UNSUPPORTED_74K_EVENT(r, b) \ 834 ((r) == 5 || ((r) >= 135 && (r) <= 137) || \ 835 ((b) >= 10 && (b) <= 12) || (b) == 22 || (b) == 27 || \ 836 (b) == 33 || (b) == 34 || ((b) >= 47 && (b) <= 49) || \ 837 (r) == 178 || (b) == 55 || (b) == 57 || (b) == 60 || \ 838 (b) == 61 || (r) == 62 || (r) == 191 || \ 839 ((b) >= 64 && (b) <= 127)) 840 #define IS_BOTH_COUNTERS_74K_EVENT(b) \ 841 ((b) == 0 || (b) == 1) 842 843 /* 1004K */ 844 #define IS_UNSUPPORTED_1004K_EVENT(r, b) \ 845 ((b) == 12 || (r) == 27 || (r) == 158 || (b) == 38 || \ 846 (r) == 175 || (b) == 63 || ((b) >= 68 && (b) <= 127)) 847 #define IS_BOTH_COUNTERS_1004K_EVENT(b) \ 848 ((b) == 0 || (b) == 1 || (b) == 11) 849 #ifdef CONFIG_MIPS_MT_SMP 850 #define IS_RANGE_P_1004K_EVENT(r, b) \ 851 ((b) == 0 || (r) == 18 || (b) == 21 || (b) == 22 || \ 852 (b) == 25 || (b) == 36 || (b) == 39 || (r) == 44 || \ 853 (r) == 174 || (r) == 176 || ((b) >= 50 && (b) <= 59) || \ 854 (r) == 188 || (b) == 61 || (b) == 62 || \ 855 ((b) >= 64 && (b) <= 67)) 856 #define IS_RANGE_V_1004K_EVENT(r) ((r) == 47) 857 #endif 858 859 /* 860 * User can use 0-255 raw events, where 0-127 for the events of even 861 * counters, and 128-255 for odd counters. Note that bit 7 is used to 862 * indicate the parity. So, for example, when user wants to take the 863 * Event Num of 15 for odd counters (by referring to the user manual), 864 * then 128 needs to be added to 15 as the input for the event config, 865 * i.e., 143 (0x8F) to be used. 866 */ 867 static const struct mips_perf_event * 868 mipsxx_pmu_map_raw_event(u64 config) 869 { 870 unsigned int raw_id = config & 0xff; 871 unsigned int base_id = raw_id & 0x7f; 872 873 switch (current_cpu_type()) { 874 case CPU_24K: 875 if (IS_UNSUPPORTED_24K_EVENT(raw_id, base_id)) 876 return ERR_PTR(-EOPNOTSUPP); 877 raw_event.event_id = base_id; 878 if (IS_BOTH_COUNTERS_24K_EVENT(base_id)) 879 raw_event.cntr_mask = CNTR_EVEN | CNTR_ODD; 880 else 881 raw_event.cntr_mask = 882 raw_id > 127 ? CNTR_ODD : CNTR_EVEN; 883 #ifdef CONFIG_MIPS_MT_SMP 884 /* 885 * This is actually doing nothing. Non-multithreading 886 * CPUs will not check and calculate the range. 887 */ 888 raw_event.range = P; 889 #endif 890 break; 891 case CPU_34K: 892 if (IS_UNSUPPORTED_34K_EVENT(raw_id, base_id)) 893 return ERR_PTR(-EOPNOTSUPP); 894 raw_event.event_id = base_id; 895 if (IS_BOTH_COUNTERS_34K_EVENT(base_id)) 896 raw_event.cntr_mask = CNTR_EVEN | CNTR_ODD; 897 else 898 raw_event.cntr_mask = 899 raw_id > 127 ? CNTR_ODD : CNTR_EVEN; 900 #ifdef CONFIG_MIPS_MT_SMP 901 if (IS_RANGE_P_34K_EVENT(raw_id, base_id)) 902 raw_event.range = P; 903 else if (unlikely(IS_RANGE_V_34K_EVENT(raw_id))) 904 raw_event.range = V; 905 else 906 raw_event.range = T; 907 #endif 908 break; 909 case CPU_74K: 910 if (IS_UNSUPPORTED_74K_EVENT(raw_id, base_id)) 911 return ERR_PTR(-EOPNOTSUPP); 912 raw_event.event_id = base_id; 913 if (IS_BOTH_COUNTERS_74K_EVENT(base_id)) 914 raw_event.cntr_mask = CNTR_EVEN | CNTR_ODD; 915 else 916 raw_event.cntr_mask = 917 raw_id > 127 ? CNTR_ODD : CNTR_EVEN; 918 #ifdef CONFIG_MIPS_MT_SMP 919 raw_event.range = P; 920 #endif 921 break; 922 case CPU_1004K: 923 if (IS_UNSUPPORTED_1004K_EVENT(raw_id, base_id)) 924 return ERR_PTR(-EOPNOTSUPP); 925 raw_event.event_id = base_id; 926 if (IS_BOTH_COUNTERS_1004K_EVENT(base_id)) 927 raw_event.cntr_mask = CNTR_EVEN | CNTR_ODD; 928 else 929 raw_event.cntr_mask = 930 raw_id > 127 ? CNTR_ODD : CNTR_EVEN; 931 #ifdef CONFIG_MIPS_MT_SMP 932 if (IS_RANGE_P_1004K_EVENT(raw_id, base_id)) 933 raw_event.range = P; 934 else if (unlikely(IS_RANGE_V_1004K_EVENT(raw_id))) 935 raw_event.range = V; 936 else 937 raw_event.range = T; 938 #endif 939 break; 940 } 941 942 return &raw_event; 943 } 944 945 static struct mips_pmu mipsxxcore_pmu = { 946 .handle_irq = mipsxx_pmu_handle_irq, 947 .handle_shared_irq = mipsxx_pmu_handle_shared_irq, 948 .start = mipsxx_pmu_start, 949 .stop = mipsxx_pmu_stop, 950 .alloc_counter = mipsxx_pmu_alloc_counter, 951 .read_counter = mipsxx_pmu_read_counter, 952 .write_counter = mipsxx_pmu_write_counter, 953 .enable_event = mipsxx_pmu_enable_event, 954 .disable_event = mipsxx_pmu_disable_event, 955 .map_raw_event = mipsxx_pmu_map_raw_event, 956 .general_event_map = &mipsxxcore_event_map, 957 .cache_event_map = &mipsxxcore_cache_map, 958 }; 959 960 static struct mips_pmu mipsxx74Kcore_pmu = { 961 .handle_irq = mipsxx_pmu_handle_irq, 962 .handle_shared_irq = mipsxx_pmu_handle_shared_irq, 963 .start = mipsxx_pmu_start, 964 .stop = mipsxx_pmu_stop, 965 .alloc_counter = mipsxx_pmu_alloc_counter, 966 .read_counter = mipsxx_pmu_read_counter, 967 .write_counter = mipsxx_pmu_write_counter, 968 .enable_event = mipsxx_pmu_enable_event, 969 .disable_event = mipsxx_pmu_disable_event, 970 .map_raw_event = mipsxx_pmu_map_raw_event, 971 .general_event_map = &mipsxx74Kcore_event_map, 972 .cache_event_map = &mipsxx74Kcore_cache_map, 973 }; 974 975 static int __init 976 init_hw_perf_events(void) 977 { 978 int counters, irq; 979 980 pr_info("Performance counters: "); 981 982 counters = n_counters(); 983 if (counters == 0) { 984 pr_cont("No available PMU.\n"); 985 return -ENODEV; 986 } 987 988 #ifdef CONFIG_MIPS_MT_SMP 989 cpu_has_mipsmt_pertccounters = read_c0_config7() & (1<<19); 990 if (!cpu_has_mipsmt_pertccounters) 991 counters = counters_total_to_per_cpu(counters); 992 #endif 993 994 #ifdef MSC01E_INT_BASE 995 if (cpu_has_veic) { 996 /* 997 * Using platform specific interrupt controller defines. 998 */ 999 irq = MSC01E_INT_BASE + MSC01E_INT_PERFCTR; 1000 } else { 1001 #endif 1002 if (cp0_perfcount_irq >= 0) 1003 irq = MIPS_CPU_IRQ_BASE + cp0_perfcount_irq; 1004 else 1005 irq = -1; 1006 #ifdef MSC01E_INT_BASE 1007 } 1008 #endif 1009 1010 on_each_cpu(reset_counters, (void *)(long)counters, 1); 1011 1012 switch (current_cpu_type()) { 1013 case CPU_24K: 1014 mipsxxcore_pmu.name = "mips/24K"; 1015 mipsxxcore_pmu.num_counters = counters; 1016 mipsxxcore_pmu.irq = irq; 1017 mipspmu = &mipsxxcore_pmu; 1018 break; 1019 case CPU_34K: 1020 mipsxxcore_pmu.name = "mips/34K"; 1021 mipsxxcore_pmu.num_counters = counters; 1022 mipsxxcore_pmu.irq = irq; 1023 mipspmu = &mipsxxcore_pmu; 1024 break; 1025 case CPU_74K: 1026 mipsxx74Kcore_pmu.name = "mips/74K"; 1027 mipsxx74Kcore_pmu.num_counters = counters; 1028 mipsxx74Kcore_pmu.irq = irq; 1029 mipspmu = &mipsxx74Kcore_pmu; 1030 break; 1031 case CPU_1004K: 1032 mipsxxcore_pmu.name = "mips/1004K"; 1033 mipsxxcore_pmu.num_counters = counters; 1034 mipsxxcore_pmu.irq = irq; 1035 mipspmu = &mipsxxcore_pmu; 1036 break; 1037 default: 1038 pr_cont("Either hardware does not support performance " 1039 "counters, or not yet implemented.\n"); 1040 return -ENODEV; 1041 } 1042 1043 if (mipspmu) 1044 pr_cont("%s PMU enabled, %d counters available to each " 1045 "CPU, irq %d%s\n", mipspmu->name, counters, irq, 1046 irq < 0 ? " (share with timer interrupt)" : ""); 1047 1048 return 0; 1049 } 1050 early_initcall(init_hw_perf_events); 1051 1052 #endif /* defined(CONFIG_CPU_MIPS32)... */ 1053