1 #if defined(CONFIG_CPU_MIPS32) || defined(CONFIG_CPU_MIPS64) || \ 2 defined(CONFIG_CPU_R10000) || defined(CONFIG_CPU_SB1) 3 4 #define M_CONFIG1_PC (1 << 4) 5 6 #define M_PERFCTL_EXL (1UL << 0) 7 #define M_PERFCTL_KERNEL (1UL << 1) 8 #define M_PERFCTL_SUPERVISOR (1UL << 2) 9 #define M_PERFCTL_USER (1UL << 3) 10 #define M_PERFCTL_INTERRUPT_ENABLE (1UL << 4) 11 #define M_PERFCTL_EVENT(event) (((event) & 0x3ff) << 5) 12 #define M_PERFCTL_VPEID(vpe) ((vpe) << 16) 13 #define M_PERFCTL_MT_EN(filter) ((filter) << 20) 14 #define M_TC_EN_ALL M_PERFCTL_MT_EN(0) 15 #define M_TC_EN_VPE M_PERFCTL_MT_EN(1) 16 #define M_TC_EN_TC M_PERFCTL_MT_EN(2) 17 #define M_PERFCTL_TCID(tcid) ((tcid) << 22) 18 #define M_PERFCTL_WIDE (1UL << 30) 19 #define M_PERFCTL_MORE (1UL << 31) 20 21 #define M_PERFCTL_COUNT_EVENT_WHENEVER (M_PERFCTL_EXL | \ 22 M_PERFCTL_KERNEL | \ 23 M_PERFCTL_USER | \ 24 M_PERFCTL_SUPERVISOR | \ 25 M_PERFCTL_INTERRUPT_ENABLE) 26 27 #ifdef CONFIG_MIPS_MT_SMP 28 #define M_PERFCTL_CONFIG_MASK 0x3fff801f 29 #else 30 #define M_PERFCTL_CONFIG_MASK 0x1f 31 #endif 32 #define M_PERFCTL_EVENT_MASK 0xfe0 33 34 #define M_COUNTER_OVERFLOW (1UL << 31) 35 36 #ifdef CONFIG_MIPS_MT_SMP 37 static int cpu_has_mipsmt_pertccounters; 38 39 /* 40 * FIXME: For VSMP, vpe_id() is redefined for Perf-events, because 41 * cpu_data[cpuid].vpe_id reports 0 for _both_ CPUs. 42 */ 43 #if defined(CONFIG_HW_PERF_EVENTS) 44 #define vpe_id() (cpu_has_mipsmt_pertccounters ? \ 45 0 : smp_processor_id()) 46 #else 47 #define vpe_id() (cpu_has_mipsmt_pertccounters ? \ 48 0 : cpu_data[smp_processor_id()].vpe_id) 49 #endif 50 51 /* Copied from op_model_mipsxx.c */ 52 static inline unsigned int vpe_shift(void) 53 { 54 if (num_possible_cpus() > 1) 55 return 1; 56 57 return 0; 58 } 59 #else /* !CONFIG_MIPS_MT_SMP */ 60 #define vpe_id() 0 61 62 static inline unsigned int vpe_shift(void) 63 { 64 return 0; 65 } 66 #endif /* CONFIG_MIPS_MT_SMP */ 67 68 static inline unsigned int 69 counters_total_to_per_cpu(unsigned int counters) 70 { 71 return counters >> vpe_shift(); 72 } 73 74 static inline unsigned int 75 counters_per_cpu_to_total(unsigned int counters) 76 { 77 return counters << vpe_shift(); 78 } 79 80 #define __define_perf_accessors(r, n, np) \ 81 \ 82 static inline unsigned int r_c0_ ## r ## n(void) \ 83 { \ 84 unsigned int cpu = vpe_id(); \ 85 \ 86 switch (cpu) { \ 87 case 0: \ 88 return read_c0_ ## r ## n(); \ 89 case 1: \ 90 return read_c0_ ## r ## np(); \ 91 default: \ 92 BUG(); \ 93 } \ 94 return 0; \ 95 } \ 96 \ 97 static inline void w_c0_ ## r ## n(unsigned int value) \ 98 { \ 99 unsigned int cpu = vpe_id(); \ 100 \ 101 switch (cpu) { \ 102 case 0: \ 103 write_c0_ ## r ## n(value); \ 104 return; \ 105 case 1: \ 106 write_c0_ ## r ## np(value); \ 107 return; \ 108 default: \ 109 BUG(); \ 110 } \ 111 return; \ 112 } \ 113 114 __define_perf_accessors(perfcntr, 0, 2) 115 __define_perf_accessors(perfcntr, 1, 3) 116 __define_perf_accessors(perfcntr, 2, 0) 117 __define_perf_accessors(perfcntr, 3, 1) 118 119 __define_perf_accessors(perfctrl, 0, 2) 120 __define_perf_accessors(perfctrl, 1, 3) 121 __define_perf_accessors(perfctrl, 2, 0) 122 __define_perf_accessors(perfctrl, 3, 1) 123 124 static inline int __n_counters(void) 125 { 126 if (!(read_c0_config1() & M_CONFIG1_PC)) 127 return 0; 128 if (!(read_c0_perfctrl0() & M_PERFCTL_MORE)) 129 return 1; 130 if (!(read_c0_perfctrl1() & M_PERFCTL_MORE)) 131 return 2; 132 if (!(read_c0_perfctrl2() & M_PERFCTL_MORE)) 133 return 3; 134 135 return 4; 136 } 137 138 static inline int n_counters(void) 139 { 140 int counters; 141 142 switch (current_cpu_type()) { 143 case CPU_R10000: 144 counters = 2; 145 break; 146 147 case CPU_R12000: 148 case CPU_R14000: 149 counters = 4; 150 break; 151 152 default: 153 counters = __n_counters(); 154 } 155 156 return counters; 157 } 158 159 static void reset_counters(void *arg) 160 { 161 int counters = (int)(long)arg; 162 switch (counters) { 163 case 4: 164 w_c0_perfctrl3(0); 165 w_c0_perfcntr3(0); 166 case 3: 167 w_c0_perfctrl2(0); 168 w_c0_perfcntr2(0); 169 case 2: 170 w_c0_perfctrl1(0); 171 w_c0_perfcntr1(0); 172 case 1: 173 w_c0_perfctrl0(0); 174 w_c0_perfcntr0(0); 175 } 176 } 177 178 static inline u64 179 mipsxx_pmu_read_counter(unsigned int idx) 180 { 181 switch (idx) { 182 case 0: 183 return r_c0_perfcntr0(); 184 case 1: 185 return r_c0_perfcntr1(); 186 case 2: 187 return r_c0_perfcntr2(); 188 case 3: 189 return r_c0_perfcntr3(); 190 default: 191 WARN_ONCE(1, "Invalid performance counter number (%d)\n", idx); 192 return 0; 193 } 194 } 195 196 static inline void 197 mipsxx_pmu_write_counter(unsigned int idx, u64 val) 198 { 199 switch (idx) { 200 case 0: 201 w_c0_perfcntr0(val); 202 return; 203 case 1: 204 w_c0_perfcntr1(val); 205 return; 206 case 2: 207 w_c0_perfcntr2(val); 208 return; 209 case 3: 210 w_c0_perfcntr3(val); 211 return; 212 } 213 } 214 215 static inline unsigned int 216 mipsxx_pmu_read_control(unsigned int idx) 217 { 218 switch (idx) { 219 case 0: 220 return r_c0_perfctrl0(); 221 case 1: 222 return r_c0_perfctrl1(); 223 case 2: 224 return r_c0_perfctrl2(); 225 case 3: 226 return r_c0_perfctrl3(); 227 default: 228 WARN_ONCE(1, "Invalid performance counter number (%d)\n", idx); 229 return 0; 230 } 231 } 232 233 static inline void 234 mipsxx_pmu_write_control(unsigned int idx, unsigned int val) 235 { 236 switch (idx) { 237 case 0: 238 w_c0_perfctrl0(val); 239 return; 240 case 1: 241 w_c0_perfctrl1(val); 242 return; 243 case 2: 244 w_c0_perfctrl2(val); 245 return; 246 case 3: 247 w_c0_perfctrl3(val); 248 return; 249 } 250 } 251 252 #ifdef CONFIG_MIPS_MT_SMP 253 static DEFINE_RWLOCK(pmuint_rwlock); 254 #endif 255 256 /* 24K/34K/1004K cores can share the same event map. */ 257 static const struct mips_perf_event mipsxxcore_event_map 258 [PERF_COUNT_HW_MAX] = { 259 [PERF_COUNT_HW_CPU_CYCLES] = { 0x00, CNTR_EVEN | CNTR_ODD, P }, 260 [PERF_COUNT_HW_INSTRUCTIONS] = { 0x01, CNTR_EVEN | CNTR_ODD, T }, 261 [PERF_COUNT_HW_CACHE_REFERENCES] = { UNSUPPORTED_PERF_EVENT_ID }, 262 [PERF_COUNT_HW_CACHE_MISSES] = { UNSUPPORTED_PERF_EVENT_ID }, 263 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = { 0x02, CNTR_EVEN, T }, 264 [PERF_COUNT_HW_BRANCH_MISSES] = { 0x02, CNTR_ODD, T }, 265 [PERF_COUNT_HW_BUS_CYCLES] = { UNSUPPORTED_PERF_EVENT_ID }, 266 }; 267 268 /* 74K core has different branch event code. */ 269 static const struct mips_perf_event mipsxx74Kcore_event_map 270 [PERF_COUNT_HW_MAX] = { 271 [PERF_COUNT_HW_CPU_CYCLES] = { 0x00, CNTR_EVEN | CNTR_ODD, P }, 272 [PERF_COUNT_HW_INSTRUCTIONS] = { 0x01, CNTR_EVEN | CNTR_ODD, T }, 273 [PERF_COUNT_HW_CACHE_REFERENCES] = { UNSUPPORTED_PERF_EVENT_ID }, 274 [PERF_COUNT_HW_CACHE_MISSES] = { UNSUPPORTED_PERF_EVENT_ID }, 275 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = { 0x27, CNTR_EVEN, T }, 276 [PERF_COUNT_HW_BRANCH_MISSES] = { 0x27, CNTR_ODD, T }, 277 [PERF_COUNT_HW_BUS_CYCLES] = { UNSUPPORTED_PERF_EVENT_ID }, 278 }; 279 280 /* 24K/34K/1004K cores can share the same cache event map. */ 281 static const struct mips_perf_event mipsxxcore_cache_map 282 [PERF_COUNT_HW_CACHE_MAX] 283 [PERF_COUNT_HW_CACHE_OP_MAX] 284 [PERF_COUNT_HW_CACHE_RESULT_MAX] = { 285 [C(L1D)] = { 286 /* 287 * Like some other architectures (e.g. ARM), the performance 288 * counters don't differentiate between read and write 289 * accesses/misses, so this isn't strictly correct, but it's the 290 * best we can do. Writes and reads get combined. 291 */ 292 [C(OP_READ)] = { 293 [C(RESULT_ACCESS)] = { 0x0a, CNTR_EVEN, T }, 294 [C(RESULT_MISS)] = { 0x0b, CNTR_EVEN | CNTR_ODD, T }, 295 }, 296 [C(OP_WRITE)] = { 297 [C(RESULT_ACCESS)] = { 0x0a, CNTR_EVEN, T }, 298 [C(RESULT_MISS)] = { 0x0b, CNTR_EVEN | CNTR_ODD, T }, 299 }, 300 [C(OP_PREFETCH)] = { 301 [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID }, 302 [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, 303 }, 304 }, 305 [C(L1I)] = { 306 [C(OP_READ)] = { 307 [C(RESULT_ACCESS)] = { 0x09, CNTR_EVEN, T }, 308 [C(RESULT_MISS)] = { 0x09, CNTR_ODD, T }, 309 }, 310 [C(OP_WRITE)] = { 311 [C(RESULT_ACCESS)] = { 0x09, CNTR_EVEN, T }, 312 [C(RESULT_MISS)] = { 0x09, CNTR_ODD, T }, 313 }, 314 [C(OP_PREFETCH)] = { 315 [C(RESULT_ACCESS)] = { 0x14, CNTR_EVEN, T }, 316 /* 317 * Note that MIPS has only "hit" events countable for 318 * the prefetch operation. 319 */ 320 [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, 321 }, 322 }, 323 [C(LL)] = { 324 [C(OP_READ)] = { 325 [C(RESULT_ACCESS)] = { 0x15, CNTR_ODD, P }, 326 [C(RESULT_MISS)] = { 0x16, CNTR_EVEN, P }, 327 }, 328 [C(OP_WRITE)] = { 329 [C(RESULT_ACCESS)] = { 0x15, CNTR_ODD, P }, 330 [C(RESULT_MISS)] = { 0x16, CNTR_EVEN, P }, 331 }, 332 [C(OP_PREFETCH)] = { 333 [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID }, 334 [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, 335 }, 336 }, 337 [C(DTLB)] = { 338 [C(OP_READ)] = { 339 [C(RESULT_ACCESS)] = { 0x06, CNTR_EVEN, T }, 340 [C(RESULT_MISS)] = { 0x06, CNTR_ODD, T }, 341 }, 342 [C(OP_WRITE)] = { 343 [C(RESULT_ACCESS)] = { 0x06, CNTR_EVEN, T }, 344 [C(RESULT_MISS)] = { 0x06, CNTR_ODD, T }, 345 }, 346 [C(OP_PREFETCH)] = { 347 [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID }, 348 [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, 349 }, 350 }, 351 [C(ITLB)] = { 352 [C(OP_READ)] = { 353 [C(RESULT_ACCESS)] = { 0x05, CNTR_EVEN, T }, 354 [C(RESULT_MISS)] = { 0x05, CNTR_ODD, T }, 355 }, 356 [C(OP_WRITE)] = { 357 [C(RESULT_ACCESS)] = { 0x05, CNTR_EVEN, T }, 358 [C(RESULT_MISS)] = { 0x05, CNTR_ODD, T }, 359 }, 360 [C(OP_PREFETCH)] = { 361 [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID }, 362 [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, 363 }, 364 }, 365 [C(BPU)] = { 366 /* Using the same code for *HW_BRANCH* */ 367 [C(OP_READ)] = { 368 [C(RESULT_ACCESS)] = { 0x02, CNTR_EVEN, T }, 369 [C(RESULT_MISS)] = { 0x02, CNTR_ODD, T }, 370 }, 371 [C(OP_WRITE)] = { 372 [C(RESULT_ACCESS)] = { 0x02, CNTR_EVEN, T }, 373 [C(RESULT_MISS)] = { 0x02, CNTR_ODD, T }, 374 }, 375 [C(OP_PREFETCH)] = { 376 [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID }, 377 [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, 378 }, 379 }, 380 [C(NODE)] = { 381 [C(OP_READ)] = { 382 [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID }, 383 [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, 384 }, 385 [C(OP_WRITE)] = { 386 [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID }, 387 [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, 388 }, 389 [C(OP_PREFETCH)] = { 390 [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID }, 391 [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, 392 }, 393 }, 394 }; 395 396 /* 74K core has completely different cache event map. */ 397 static const struct mips_perf_event mipsxx74Kcore_cache_map 398 [PERF_COUNT_HW_CACHE_MAX] 399 [PERF_COUNT_HW_CACHE_OP_MAX] 400 [PERF_COUNT_HW_CACHE_RESULT_MAX] = { 401 [C(L1D)] = { 402 /* 403 * Like some other architectures (e.g. ARM), the performance 404 * counters don't differentiate between read and write 405 * accesses/misses, so this isn't strictly correct, but it's the 406 * best we can do. Writes and reads get combined. 407 */ 408 [C(OP_READ)] = { 409 [C(RESULT_ACCESS)] = { 0x17, CNTR_ODD, T }, 410 [C(RESULT_MISS)] = { 0x18, CNTR_ODD, T }, 411 }, 412 [C(OP_WRITE)] = { 413 [C(RESULT_ACCESS)] = { 0x17, CNTR_ODD, T }, 414 [C(RESULT_MISS)] = { 0x18, CNTR_ODD, T }, 415 }, 416 [C(OP_PREFETCH)] = { 417 [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID }, 418 [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, 419 }, 420 }, 421 [C(L1I)] = { 422 [C(OP_READ)] = { 423 [C(RESULT_ACCESS)] = { 0x06, CNTR_EVEN, T }, 424 [C(RESULT_MISS)] = { 0x06, CNTR_ODD, T }, 425 }, 426 [C(OP_WRITE)] = { 427 [C(RESULT_ACCESS)] = { 0x06, CNTR_EVEN, T }, 428 [C(RESULT_MISS)] = { 0x06, CNTR_ODD, T }, 429 }, 430 [C(OP_PREFETCH)] = { 431 [C(RESULT_ACCESS)] = { 0x34, CNTR_EVEN, T }, 432 /* 433 * Note that MIPS has only "hit" events countable for 434 * the prefetch operation. 435 */ 436 [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, 437 }, 438 }, 439 [C(LL)] = { 440 [C(OP_READ)] = { 441 [C(RESULT_ACCESS)] = { 0x1c, CNTR_ODD, P }, 442 [C(RESULT_MISS)] = { 0x1d, CNTR_EVEN | CNTR_ODD, P }, 443 }, 444 [C(OP_WRITE)] = { 445 [C(RESULT_ACCESS)] = { 0x1c, CNTR_ODD, P }, 446 [C(RESULT_MISS)] = { 0x1d, CNTR_EVEN | CNTR_ODD, P }, 447 }, 448 [C(OP_PREFETCH)] = { 449 [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID }, 450 [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, 451 }, 452 }, 453 [C(DTLB)] = { 454 /* 74K core does not have specific DTLB events. */ 455 [C(OP_READ)] = { 456 [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID }, 457 [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, 458 }, 459 [C(OP_WRITE)] = { 460 [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID }, 461 [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, 462 }, 463 [C(OP_PREFETCH)] = { 464 [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID }, 465 [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, 466 }, 467 }, 468 [C(ITLB)] = { 469 [C(OP_READ)] = { 470 [C(RESULT_ACCESS)] = { 0x04, CNTR_EVEN, T }, 471 [C(RESULT_MISS)] = { 0x04, CNTR_ODD, T }, 472 }, 473 [C(OP_WRITE)] = { 474 [C(RESULT_ACCESS)] = { 0x04, CNTR_EVEN, T }, 475 [C(RESULT_MISS)] = { 0x04, CNTR_ODD, T }, 476 }, 477 [C(OP_PREFETCH)] = { 478 [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID }, 479 [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, 480 }, 481 }, 482 [C(BPU)] = { 483 /* Using the same code for *HW_BRANCH* */ 484 [C(OP_READ)] = { 485 [C(RESULT_ACCESS)] = { 0x27, CNTR_EVEN, T }, 486 [C(RESULT_MISS)] = { 0x27, CNTR_ODD, T }, 487 }, 488 [C(OP_WRITE)] = { 489 [C(RESULT_ACCESS)] = { 0x27, CNTR_EVEN, T }, 490 [C(RESULT_MISS)] = { 0x27, CNTR_ODD, T }, 491 }, 492 [C(OP_PREFETCH)] = { 493 [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID }, 494 [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, 495 }, 496 }, 497 [C(NODE)] = { 498 [C(OP_READ)] = { 499 [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID }, 500 [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, 501 }, 502 [C(OP_WRITE)] = { 503 [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID }, 504 [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, 505 }, 506 [C(OP_PREFETCH)] = { 507 [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID }, 508 [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, 509 }, 510 }, 511 }; 512 513 #ifdef CONFIG_MIPS_MT_SMP 514 static void 515 check_and_calc_range(struct perf_event *event, 516 const struct mips_perf_event *pev) 517 { 518 struct hw_perf_event *hwc = &event->hw; 519 520 if (event->cpu >= 0) { 521 if (pev->range > V) { 522 /* 523 * The user selected an event that is processor 524 * wide, while expecting it to be VPE wide. 525 */ 526 hwc->config_base |= M_TC_EN_ALL; 527 } else { 528 /* 529 * FIXME: cpu_data[event->cpu].vpe_id reports 0 530 * for both CPUs. 531 */ 532 hwc->config_base |= M_PERFCTL_VPEID(event->cpu); 533 hwc->config_base |= M_TC_EN_VPE; 534 } 535 } else 536 hwc->config_base |= M_TC_EN_ALL; 537 } 538 #else 539 static void 540 check_and_calc_range(struct perf_event *event, 541 const struct mips_perf_event *pev) 542 { 543 } 544 #endif 545 546 static int __hw_perf_event_init(struct perf_event *event) 547 { 548 struct perf_event_attr *attr = &event->attr; 549 struct hw_perf_event *hwc = &event->hw; 550 const struct mips_perf_event *pev; 551 int err; 552 553 /* Returning MIPS event descriptor for generic perf event. */ 554 if (PERF_TYPE_HARDWARE == event->attr.type) { 555 if (event->attr.config >= PERF_COUNT_HW_MAX) 556 return -EINVAL; 557 pev = mipspmu_map_general_event(event->attr.config); 558 } else if (PERF_TYPE_HW_CACHE == event->attr.type) { 559 pev = mipspmu_map_cache_event(event->attr.config); 560 } else if (PERF_TYPE_RAW == event->attr.type) { 561 /* We are working on the global raw event. */ 562 mutex_lock(&raw_event_mutex); 563 pev = mipspmu->map_raw_event(event->attr.config); 564 } else { 565 /* The event type is not (yet) supported. */ 566 return -EOPNOTSUPP; 567 } 568 569 if (IS_ERR(pev)) { 570 if (PERF_TYPE_RAW == event->attr.type) 571 mutex_unlock(&raw_event_mutex); 572 return PTR_ERR(pev); 573 } 574 575 /* 576 * We allow max flexibility on how each individual counter shared 577 * by the single CPU operates (the mode exclusion and the range). 578 */ 579 hwc->config_base = M_PERFCTL_INTERRUPT_ENABLE; 580 581 /* Calculate range bits and validate it. */ 582 if (num_possible_cpus() > 1) 583 check_and_calc_range(event, pev); 584 585 hwc->event_base = mipspmu_perf_event_encode(pev); 586 if (PERF_TYPE_RAW == event->attr.type) 587 mutex_unlock(&raw_event_mutex); 588 589 if (!attr->exclude_user) 590 hwc->config_base |= M_PERFCTL_USER; 591 if (!attr->exclude_kernel) { 592 hwc->config_base |= M_PERFCTL_KERNEL; 593 /* MIPS kernel mode: KSU == 00b || EXL == 1 || ERL == 1 */ 594 hwc->config_base |= M_PERFCTL_EXL; 595 } 596 if (!attr->exclude_hv) 597 hwc->config_base |= M_PERFCTL_SUPERVISOR; 598 599 hwc->config_base &= M_PERFCTL_CONFIG_MASK; 600 /* 601 * The event can belong to another cpu. We do not assign a local 602 * counter for it for now. 603 */ 604 hwc->idx = -1; 605 hwc->config = 0; 606 607 if (!hwc->sample_period) { 608 hwc->sample_period = MAX_PERIOD; 609 hwc->last_period = hwc->sample_period; 610 local64_set(&hwc->period_left, hwc->sample_period); 611 } 612 613 err = 0; 614 if (event->group_leader != event) { 615 err = validate_group(event); 616 if (err) 617 return -EINVAL; 618 } 619 620 event->destroy = hw_perf_event_destroy; 621 622 return err; 623 } 624 625 static void pause_local_counters(void) 626 { 627 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); 628 int counters = mipspmu->num_counters; 629 unsigned long flags; 630 631 local_irq_save(flags); 632 switch (counters) { 633 case 4: 634 cpuc->saved_ctrl[3] = r_c0_perfctrl3(); 635 w_c0_perfctrl3(cpuc->saved_ctrl[3] & 636 ~M_PERFCTL_COUNT_EVENT_WHENEVER); 637 case 3: 638 cpuc->saved_ctrl[2] = r_c0_perfctrl2(); 639 w_c0_perfctrl2(cpuc->saved_ctrl[2] & 640 ~M_PERFCTL_COUNT_EVENT_WHENEVER); 641 case 2: 642 cpuc->saved_ctrl[1] = r_c0_perfctrl1(); 643 w_c0_perfctrl1(cpuc->saved_ctrl[1] & 644 ~M_PERFCTL_COUNT_EVENT_WHENEVER); 645 case 1: 646 cpuc->saved_ctrl[0] = r_c0_perfctrl0(); 647 w_c0_perfctrl0(cpuc->saved_ctrl[0] & 648 ~M_PERFCTL_COUNT_EVENT_WHENEVER); 649 } 650 local_irq_restore(flags); 651 } 652 653 static void resume_local_counters(void) 654 { 655 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); 656 int counters = mipspmu->num_counters; 657 unsigned long flags; 658 659 local_irq_save(flags); 660 switch (counters) { 661 case 4: 662 w_c0_perfctrl3(cpuc->saved_ctrl[3]); 663 case 3: 664 w_c0_perfctrl2(cpuc->saved_ctrl[2]); 665 case 2: 666 w_c0_perfctrl1(cpuc->saved_ctrl[1]); 667 case 1: 668 w_c0_perfctrl0(cpuc->saved_ctrl[0]); 669 } 670 local_irq_restore(flags); 671 } 672 673 static int mipsxx_pmu_handle_shared_irq(void) 674 { 675 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); 676 struct perf_sample_data data; 677 unsigned int counters = mipspmu->num_counters; 678 unsigned int counter; 679 int handled = IRQ_NONE; 680 struct pt_regs *regs; 681 682 if (cpu_has_mips_r2 && !(read_c0_cause() & (1 << 26))) 683 return handled; 684 685 /* 686 * First we pause the local counters, so that when we are locked 687 * here, the counters are all paused. When it gets locked due to 688 * perf_disable(), the timer interrupt handler will be delayed. 689 * 690 * See also mipsxx_pmu_start(). 691 */ 692 pause_local_counters(); 693 #ifdef CONFIG_MIPS_MT_SMP 694 read_lock(&pmuint_rwlock); 695 #endif 696 697 regs = get_irq_regs(); 698 699 perf_sample_data_init(&data, 0); 700 701 switch (counters) { 702 #define HANDLE_COUNTER(n) \ 703 case n + 1: \ 704 if (test_bit(n, cpuc->used_mask)) { \ 705 counter = r_c0_perfcntr ## n(); \ 706 if (counter & M_COUNTER_OVERFLOW) { \ 707 w_c0_perfcntr ## n(counter & \ 708 VALID_COUNT); \ 709 if (test_and_change_bit(n, cpuc->msbs)) \ 710 handle_associated_event(cpuc, \ 711 n, &data, regs); \ 712 handled = IRQ_HANDLED; \ 713 } \ 714 } 715 HANDLE_COUNTER(3) 716 HANDLE_COUNTER(2) 717 HANDLE_COUNTER(1) 718 HANDLE_COUNTER(0) 719 } 720 721 /* 722 * Do all the work for the pending perf events. We can do this 723 * in here because the performance counter interrupt is a regular 724 * interrupt, not NMI. 725 */ 726 if (handled == IRQ_HANDLED) 727 irq_work_run(); 728 729 #ifdef CONFIG_MIPS_MT_SMP 730 read_unlock(&pmuint_rwlock); 731 #endif 732 resume_local_counters(); 733 return handled; 734 } 735 736 static irqreturn_t 737 mipsxx_pmu_handle_irq(int irq, void *dev) 738 { 739 return mipsxx_pmu_handle_shared_irq(); 740 } 741 742 static void mipsxx_pmu_start(void) 743 { 744 #ifdef CONFIG_MIPS_MT_SMP 745 write_unlock(&pmuint_rwlock); 746 #endif 747 resume_local_counters(); 748 } 749 750 /* 751 * MIPS performance counters can be per-TC. The control registers can 752 * not be directly accessed across CPUs. Hence if we want to do global 753 * control, we need cross CPU calls. on_each_cpu() can help us, but we 754 * can not make sure this function is called with interrupts enabled. So 755 * here we pause local counters and then grab a rwlock and leave the 756 * counters on other CPUs alone. If any counter interrupt raises while 757 * we own the write lock, simply pause local counters on that CPU and 758 * spin in the handler. Also we know we won't be switched to another 759 * CPU after pausing local counters and before grabbing the lock. 760 */ 761 static void mipsxx_pmu_stop(void) 762 { 763 pause_local_counters(); 764 #ifdef CONFIG_MIPS_MT_SMP 765 write_lock(&pmuint_rwlock); 766 #endif 767 } 768 769 static int 770 mipsxx_pmu_alloc_counter(struct cpu_hw_events *cpuc, 771 struct hw_perf_event *hwc) 772 { 773 int i; 774 775 /* 776 * We only need to care the counter mask. The range has been 777 * checked definitely. 778 */ 779 unsigned long cntr_mask = (hwc->event_base >> 8) & 0xffff; 780 781 for (i = mipspmu->num_counters - 1; i >= 0; i--) { 782 /* 783 * Note that some MIPS perf events can be counted by both 784 * even and odd counters, wheresas many other are only by 785 * even _or_ odd counters. This introduces an issue that 786 * when the former kind of event takes the counter the 787 * latter kind of event wants to use, then the "counter 788 * allocation" for the latter event will fail. In fact if 789 * they can be dynamically swapped, they both feel happy. 790 * But here we leave this issue alone for now. 791 */ 792 if (test_bit(i, &cntr_mask) && 793 !test_and_set_bit(i, cpuc->used_mask)) 794 return i; 795 } 796 797 return -EAGAIN; 798 } 799 800 static void 801 mipsxx_pmu_enable_event(struct hw_perf_event *evt, int idx) 802 { 803 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); 804 unsigned long flags; 805 806 WARN_ON(idx < 0 || idx >= mipspmu->num_counters); 807 808 local_irq_save(flags); 809 cpuc->saved_ctrl[idx] = M_PERFCTL_EVENT(evt->event_base & 0xff) | 810 (evt->config_base & M_PERFCTL_CONFIG_MASK) | 811 /* Make sure interrupt enabled. */ 812 M_PERFCTL_INTERRUPT_ENABLE; 813 /* 814 * We do not actually let the counter run. Leave it until start(). 815 */ 816 local_irq_restore(flags); 817 } 818 819 static void 820 mipsxx_pmu_disable_event(int idx) 821 { 822 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); 823 unsigned long flags; 824 825 WARN_ON(idx < 0 || idx >= mipspmu->num_counters); 826 827 local_irq_save(flags); 828 cpuc->saved_ctrl[idx] = mipsxx_pmu_read_control(idx) & 829 ~M_PERFCTL_COUNT_EVENT_WHENEVER; 830 mipsxx_pmu_write_control(idx, cpuc->saved_ctrl[idx]); 831 local_irq_restore(flags); 832 } 833 834 /* 24K */ 835 #define IS_UNSUPPORTED_24K_EVENT(r, b) \ 836 ((b) == 12 || (r) == 151 || (r) == 152 || (b) == 26 || \ 837 (b) == 27 || (r) == 28 || (r) == 158 || (b) == 31 || \ 838 (b) == 32 || (b) == 34 || (b) == 36 || (r) == 168 || \ 839 (r) == 172 || (b) == 47 || ((b) >= 56 && (b) <= 63) || \ 840 ((b) >= 68 && (b) <= 127)) 841 #define IS_BOTH_COUNTERS_24K_EVENT(b) \ 842 ((b) == 0 || (b) == 1 || (b) == 11) 843 844 /* 34K */ 845 #define IS_UNSUPPORTED_34K_EVENT(r, b) \ 846 ((b) == 12 || (r) == 27 || (r) == 158 || (b) == 36 || \ 847 (b) == 38 || (r) == 175 || ((b) >= 56 && (b) <= 63) || \ 848 ((b) >= 68 && (b) <= 127)) 849 #define IS_BOTH_COUNTERS_34K_EVENT(b) \ 850 ((b) == 0 || (b) == 1 || (b) == 11) 851 #ifdef CONFIG_MIPS_MT_SMP 852 #define IS_RANGE_P_34K_EVENT(r, b) \ 853 ((b) == 0 || (r) == 18 || (b) == 21 || (b) == 22 || \ 854 (b) == 25 || (b) == 39 || (r) == 44 || (r) == 174 || \ 855 (r) == 176 || ((b) >= 50 && (b) <= 55) || \ 856 ((b) >= 64 && (b) <= 67)) 857 #define IS_RANGE_V_34K_EVENT(r) ((r) == 47) 858 #endif 859 860 /* 74K */ 861 #define IS_UNSUPPORTED_74K_EVENT(r, b) \ 862 ((r) == 5 || ((r) >= 135 && (r) <= 137) || \ 863 ((b) >= 10 && (b) <= 12) || (b) == 22 || (b) == 27 || \ 864 (b) == 33 || (b) == 34 || ((b) >= 47 && (b) <= 49) || \ 865 (r) == 178 || (b) == 55 || (b) == 57 || (b) == 60 || \ 866 (b) == 61 || (r) == 62 || (r) == 191 || \ 867 ((b) >= 64 && (b) <= 127)) 868 #define IS_BOTH_COUNTERS_74K_EVENT(b) \ 869 ((b) == 0 || (b) == 1) 870 871 /* 1004K */ 872 #define IS_UNSUPPORTED_1004K_EVENT(r, b) \ 873 ((b) == 12 || (r) == 27 || (r) == 158 || (b) == 38 || \ 874 (r) == 175 || (b) == 63 || ((b) >= 68 && (b) <= 127)) 875 #define IS_BOTH_COUNTERS_1004K_EVENT(b) \ 876 ((b) == 0 || (b) == 1 || (b) == 11) 877 #ifdef CONFIG_MIPS_MT_SMP 878 #define IS_RANGE_P_1004K_EVENT(r, b) \ 879 ((b) == 0 || (r) == 18 || (b) == 21 || (b) == 22 || \ 880 (b) == 25 || (b) == 36 || (b) == 39 || (r) == 44 || \ 881 (r) == 174 || (r) == 176 || ((b) >= 50 && (b) <= 59) || \ 882 (r) == 188 || (b) == 61 || (b) == 62 || \ 883 ((b) >= 64 && (b) <= 67)) 884 #define IS_RANGE_V_1004K_EVENT(r) ((r) == 47) 885 #endif 886 887 /* 888 * User can use 0-255 raw events, where 0-127 for the events of even 889 * counters, and 128-255 for odd counters. Note that bit 7 is used to 890 * indicate the parity. So, for example, when user wants to take the 891 * Event Num of 15 for odd counters (by referring to the user manual), 892 * then 128 needs to be added to 15 as the input for the event config, 893 * i.e., 143 (0x8F) to be used. 894 */ 895 static const struct mips_perf_event * 896 mipsxx_pmu_map_raw_event(u64 config) 897 { 898 unsigned int raw_id = config & 0xff; 899 unsigned int base_id = raw_id & 0x7f; 900 901 switch (current_cpu_type()) { 902 case CPU_24K: 903 if (IS_UNSUPPORTED_24K_EVENT(raw_id, base_id)) 904 return ERR_PTR(-EOPNOTSUPP); 905 raw_event.event_id = base_id; 906 if (IS_BOTH_COUNTERS_24K_EVENT(base_id)) 907 raw_event.cntr_mask = CNTR_EVEN | CNTR_ODD; 908 else 909 raw_event.cntr_mask = 910 raw_id > 127 ? CNTR_ODD : CNTR_EVEN; 911 #ifdef CONFIG_MIPS_MT_SMP 912 /* 913 * This is actually doing nothing. Non-multithreading 914 * CPUs will not check and calculate the range. 915 */ 916 raw_event.range = P; 917 #endif 918 break; 919 case CPU_34K: 920 if (IS_UNSUPPORTED_34K_EVENT(raw_id, base_id)) 921 return ERR_PTR(-EOPNOTSUPP); 922 raw_event.event_id = base_id; 923 if (IS_BOTH_COUNTERS_34K_EVENT(base_id)) 924 raw_event.cntr_mask = CNTR_EVEN | CNTR_ODD; 925 else 926 raw_event.cntr_mask = 927 raw_id > 127 ? CNTR_ODD : CNTR_EVEN; 928 #ifdef CONFIG_MIPS_MT_SMP 929 if (IS_RANGE_P_34K_EVENT(raw_id, base_id)) 930 raw_event.range = P; 931 else if (unlikely(IS_RANGE_V_34K_EVENT(raw_id))) 932 raw_event.range = V; 933 else 934 raw_event.range = T; 935 #endif 936 break; 937 case CPU_74K: 938 if (IS_UNSUPPORTED_74K_EVENT(raw_id, base_id)) 939 return ERR_PTR(-EOPNOTSUPP); 940 raw_event.event_id = base_id; 941 if (IS_BOTH_COUNTERS_74K_EVENT(base_id)) 942 raw_event.cntr_mask = CNTR_EVEN | CNTR_ODD; 943 else 944 raw_event.cntr_mask = 945 raw_id > 127 ? CNTR_ODD : CNTR_EVEN; 946 #ifdef CONFIG_MIPS_MT_SMP 947 raw_event.range = P; 948 #endif 949 break; 950 case CPU_1004K: 951 if (IS_UNSUPPORTED_1004K_EVENT(raw_id, base_id)) 952 return ERR_PTR(-EOPNOTSUPP); 953 raw_event.event_id = base_id; 954 if (IS_BOTH_COUNTERS_1004K_EVENT(base_id)) 955 raw_event.cntr_mask = CNTR_EVEN | CNTR_ODD; 956 else 957 raw_event.cntr_mask = 958 raw_id > 127 ? CNTR_ODD : CNTR_EVEN; 959 #ifdef CONFIG_MIPS_MT_SMP 960 if (IS_RANGE_P_1004K_EVENT(raw_id, base_id)) 961 raw_event.range = P; 962 else if (unlikely(IS_RANGE_V_1004K_EVENT(raw_id))) 963 raw_event.range = V; 964 else 965 raw_event.range = T; 966 #endif 967 break; 968 } 969 970 return &raw_event; 971 } 972 973 static struct mips_pmu mipsxxcore_pmu = { 974 .handle_irq = mipsxx_pmu_handle_irq, 975 .handle_shared_irq = mipsxx_pmu_handle_shared_irq, 976 .start = mipsxx_pmu_start, 977 .stop = mipsxx_pmu_stop, 978 .alloc_counter = mipsxx_pmu_alloc_counter, 979 .read_counter = mipsxx_pmu_read_counter, 980 .write_counter = mipsxx_pmu_write_counter, 981 .enable_event = mipsxx_pmu_enable_event, 982 .disable_event = mipsxx_pmu_disable_event, 983 .map_raw_event = mipsxx_pmu_map_raw_event, 984 .general_event_map = &mipsxxcore_event_map, 985 .cache_event_map = &mipsxxcore_cache_map, 986 }; 987 988 static struct mips_pmu mipsxx74Kcore_pmu = { 989 .handle_irq = mipsxx_pmu_handle_irq, 990 .handle_shared_irq = mipsxx_pmu_handle_shared_irq, 991 .start = mipsxx_pmu_start, 992 .stop = mipsxx_pmu_stop, 993 .alloc_counter = mipsxx_pmu_alloc_counter, 994 .read_counter = mipsxx_pmu_read_counter, 995 .write_counter = mipsxx_pmu_write_counter, 996 .enable_event = mipsxx_pmu_enable_event, 997 .disable_event = mipsxx_pmu_disable_event, 998 .map_raw_event = mipsxx_pmu_map_raw_event, 999 .general_event_map = &mipsxx74Kcore_event_map, 1000 .cache_event_map = &mipsxx74Kcore_cache_map, 1001 }; 1002 1003 static int __init 1004 init_hw_perf_events(void) 1005 { 1006 int counters, irq; 1007 1008 pr_info("Performance counters: "); 1009 1010 counters = n_counters(); 1011 if (counters == 0) { 1012 pr_cont("No available PMU.\n"); 1013 return -ENODEV; 1014 } 1015 1016 #ifdef CONFIG_MIPS_MT_SMP 1017 cpu_has_mipsmt_pertccounters = read_c0_config7() & (1<<19); 1018 if (!cpu_has_mipsmt_pertccounters) 1019 counters = counters_total_to_per_cpu(counters); 1020 #endif 1021 1022 #ifdef MSC01E_INT_BASE 1023 if (cpu_has_veic) { 1024 /* 1025 * Using platform specific interrupt controller defines. 1026 */ 1027 irq = MSC01E_INT_BASE + MSC01E_INT_PERFCTR; 1028 } else { 1029 #endif 1030 if (cp0_perfcount_irq >= 0) 1031 irq = MIPS_CPU_IRQ_BASE + cp0_perfcount_irq; 1032 else 1033 irq = -1; 1034 #ifdef MSC01E_INT_BASE 1035 } 1036 #endif 1037 1038 on_each_cpu(reset_counters, (void *)(long)counters, 1); 1039 1040 switch (current_cpu_type()) { 1041 case CPU_24K: 1042 mipsxxcore_pmu.name = "mips/24K"; 1043 mipsxxcore_pmu.num_counters = counters; 1044 mipsxxcore_pmu.irq = irq; 1045 mipspmu = &mipsxxcore_pmu; 1046 break; 1047 case CPU_34K: 1048 mipsxxcore_pmu.name = "mips/34K"; 1049 mipsxxcore_pmu.num_counters = counters; 1050 mipsxxcore_pmu.irq = irq; 1051 mipspmu = &mipsxxcore_pmu; 1052 break; 1053 case CPU_74K: 1054 mipsxx74Kcore_pmu.name = "mips/74K"; 1055 mipsxx74Kcore_pmu.num_counters = counters; 1056 mipsxx74Kcore_pmu.irq = irq; 1057 mipspmu = &mipsxx74Kcore_pmu; 1058 break; 1059 case CPU_1004K: 1060 mipsxxcore_pmu.name = "mips/1004K"; 1061 mipsxxcore_pmu.num_counters = counters; 1062 mipsxxcore_pmu.irq = irq; 1063 mipspmu = &mipsxxcore_pmu; 1064 break; 1065 default: 1066 pr_cont("Either hardware does not support performance " 1067 "counters, or not yet implemented.\n"); 1068 return -ENODEV; 1069 } 1070 1071 if (mipspmu) 1072 pr_cont("%s PMU enabled, %d counters available to each " 1073 "CPU, irq %d%s\n", mipspmu->name, counters, irq, 1074 irq < 0 ? " (share with timer interrupt)" : ""); 1075 1076 perf_pmu_register(&pmu, "cpu", PERF_TYPE_RAW); 1077 1078 return 0; 1079 } 1080 early_initcall(init_hw_perf_events); 1081 1082 #endif /* defined(CONFIG_CPU_MIPS32)... */ 1083