1 /* 2 * Performance event support - Freescale Embedded Performance Monitor 3 * 4 * Copyright 2008-2009 Paul Mackerras, IBM Corporation. 5 * Copyright 2010 Freescale Semiconductor, Inc. 6 * 7 * This program is free software; you can redistribute it and/or 8 * modify it under the terms of the GNU General Public License 9 * as published by the Free Software Foundation; either version 10 * 2 of the License, or (at your option) any later version. 11 */ 12 #include <linux/kernel.h> 13 #include <linux/sched.h> 14 #include <linux/perf_event.h> 15 #include <linux/percpu.h> 16 #include <linux/hardirq.h> 17 #include <asm/reg_fsl_emb.h> 18 #include <asm/pmc.h> 19 #include <asm/machdep.h> 20 #include <asm/firmware.h> 21 #include <asm/ptrace.h> 22 23 struct cpu_hw_events { 24 int n_events; 25 int disabled; 26 u8 pmcs_enabled; 27 struct perf_event *event[MAX_HWEVENTS]; 28 }; 29 static DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events); 30 31 static struct fsl_emb_pmu *ppmu; 32 33 /* Number of perf_events counting hardware events */ 34 static atomic_t num_events; 35 /* Used to avoid races in calling reserve/release_pmc_hardware */ 36 static DEFINE_MUTEX(pmc_reserve_mutex); 37 38 /* 39 * If interrupts were soft-disabled when a PMU interrupt occurs, treat 40 * it as an NMI. 41 */ 42 static inline int perf_intr_is_nmi(struct pt_regs *regs) 43 { 44 #ifdef __powerpc64__ 45 return !regs->softe; 46 #else 47 return 0; 48 #endif 49 } 50 51 static void perf_event_interrupt(struct pt_regs *regs); 52 53 /* 54 * Read one performance monitor counter (PMC). 55 */ 56 static unsigned long read_pmc(int idx) 57 { 58 unsigned long val; 59 60 switch (idx) { 61 case 0: 62 val = mfpmr(PMRN_PMC0); 63 break; 64 case 1: 65 val = mfpmr(PMRN_PMC1); 66 break; 67 case 2: 68 val = mfpmr(PMRN_PMC2); 69 break; 70 case 3: 71 val = mfpmr(PMRN_PMC3); 72 break; 73 case 4: 74 val = mfpmr(PMRN_PMC4); 75 break; 76 case 5: 77 val = mfpmr(PMRN_PMC5); 78 break; 79 default: 80 printk(KERN_ERR "oops trying to read PMC%d\n", idx); 81 val = 0; 82 } 83 return val; 84 } 85 86 /* 87 * Write one PMC. 88 */ 89 static void write_pmc(int idx, unsigned long val) 90 { 91 switch (idx) { 92 case 0: 93 mtpmr(PMRN_PMC0, val); 94 break; 95 case 1: 96 mtpmr(PMRN_PMC1, val); 97 break; 98 case 2: 99 mtpmr(PMRN_PMC2, val); 100 break; 101 case 3: 102 mtpmr(PMRN_PMC3, val); 103 break; 104 case 4: 105 mtpmr(PMRN_PMC4, val); 106 break; 107 case 5: 108 mtpmr(PMRN_PMC5, val); 109 break; 110 default: 111 printk(KERN_ERR "oops trying to write PMC%d\n", idx); 112 } 113 114 isync(); 115 } 116 117 /* 118 * Write one local control A register 119 */ 120 static void write_pmlca(int idx, unsigned long val) 121 { 122 switch (idx) { 123 case 0: 124 mtpmr(PMRN_PMLCA0, val); 125 break; 126 case 1: 127 mtpmr(PMRN_PMLCA1, val); 128 break; 129 case 2: 130 mtpmr(PMRN_PMLCA2, val); 131 break; 132 case 3: 133 mtpmr(PMRN_PMLCA3, val); 134 break; 135 case 4: 136 mtpmr(PMRN_PMLCA4, val); 137 break; 138 case 5: 139 mtpmr(PMRN_PMLCA5, val); 140 break; 141 default: 142 printk(KERN_ERR "oops trying to write PMLCA%d\n", idx); 143 } 144 145 isync(); 146 } 147 148 /* 149 * Write one local control B register 150 */ 151 static void write_pmlcb(int idx, unsigned long val) 152 { 153 switch (idx) { 154 case 0: 155 mtpmr(PMRN_PMLCB0, val); 156 break; 157 case 1: 158 mtpmr(PMRN_PMLCB1, val); 159 break; 160 case 2: 161 mtpmr(PMRN_PMLCB2, val); 162 break; 163 case 3: 164 mtpmr(PMRN_PMLCB3, val); 165 break; 166 case 4: 167 mtpmr(PMRN_PMLCB4, val); 168 break; 169 case 5: 170 mtpmr(PMRN_PMLCB5, val); 171 break; 172 default: 173 printk(KERN_ERR "oops trying to write PMLCB%d\n", idx); 174 } 175 176 isync(); 177 } 178 179 static void fsl_emb_pmu_read(struct perf_event *event) 180 { 181 s64 val, delta, prev; 182 183 if (event->hw.state & PERF_HES_STOPPED) 184 return; 185 186 /* 187 * Performance monitor interrupts come even when interrupts 188 * are soft-disabled, as long as interrupts are hard-enabled. 189 * Therefore we treat them like NMIs. 190 */ 191 do { 192 prev = local64_read(&event->hw.prev_count); 193 barrier(); 194 val = read_pmc(event->hw.idx); 195 } while (local64_cmpxchg(&event->hw.prev_count, prev, val) != prev); 196 197 /* The counters are only 32 bits wide */ 198 delta = (val - prev) & 0xfffffffful; 199 local64_add(delta, &event->count); 200 local64_sub(delta, &event->hw.period_left); 201 } 202 203 /* 204 * Disable all events to prevent PMU interrupts and to allow 205 * events to be added or removed. 206 */ 207 static void fsl_emb_pmu_disable(struct pmu *pmu) 208 { 209 struct cpu_hw_events *cpuhw; 210 unsigned long flags; 211 212 local_irq_save(flags); 213 cpuhw = &__get_cpu_var(cpu_hw_events); 214 215 if (!cpuhw->disabled) { 216 cpuhw->disabled = 1; 217 218 /* 219 * Check if we ever enabled the PMU on this cpu. 220 */ 221 if (!cpuhw->pmcs_enabled) { 222 ppc_enable_pmcs(); 223 cpuhw->pmcs_enabled = 1; 224 } 225 226 if (atomic_read(&num_events)) { 227 /* 228 * Set the 'freeze all counters' bit, and disable 229 * interrupts. The barrier is to make sure the 230 * mtpmr has been executed and the PMU has frozen 231 * the events before we return. 232 */ 233 234 mtpmr(PMRN_PMGC0, PMGC0_FAC); 235 isync(); 236 } 237 } 238 local_irq_restore(flags); 239 } 240 241 /* 242 * Re-enable all events if disable == 0. 243 * If we were previously disabled and events were added, then 244 * put the new config on the PMU. 245 */ 246 static void fsl_emb_pmu_enable(struct pmu *pmu) 247 { 248 struct cpu_hw_events *cpuhw; 249 unsigned long flags; 250 251 local_irq_save(flags); 252 cpuhw = &__get_cpu_var(cpu_hw_events); 253 if (!cpuhw->disabled) 254 goto out; 255 256 cpuhw->disabled = 0; 257 ppc_set_pmu_inuse(cpuhw->n_events != 0); 258 259 if (cpuhw->n_events > 0) { 260 mtpmr(PMRN_PMGC0, PMGC0_PMIE | PMGC0_FCECE); 261 isync(); 262 } 263 264 out: 265 local_irq_restore(flags); 266 } 267 268 static int collect_events(struct perf_event *group, int max_count, 269 struct perf_event *ctrs[]) 270 { 271 int n = 0; 272 struct perf_event *event; 273 274 if (!is_software_event(group)) { 275 if (n >= max_count) 276 return -1; 277 ctrs[n] = group; 278 n++; 279 } 280 list_for_each_entry(event, &group->sibling_list, group_entry) { 281 if (!is_software_event(event) && 282 event->state != PERF_EVENT_STATE_OFF) { 283 if (n >= max_count) 284 return -1; 285 ctrs[n] = event; 286 n++; 287 } 288 } 289 return n; 290 } 291 292 /* context locked on entry */ 293 static int fsl_emb_pmu_add(struct perf_event *event, int flags) 294 { 295 struct cpu_hw_events *cpuhw; 296 int ret = -EAGAIN; 297 int num_counters = ppmu->n_counter; 298 u64 val; 299 int i; 300 301 perf_pmu_disable(event->pmu); 302 cpuhw = &get_cpu_var(cpu_hw_events); 303 304 if (event->hw.config & FSL_EMB_EVENT_RESTRICTED) 305 num_counters = ppmu->n_restricted; 306 307 /* 308 * Allocate counters from top-down, so that restricted-capable 309 * counters are kept free as long as possible. 310 */ 311 for (i = num_counters - 1; i >= 0; i--) { 312 if (cpuhw->event[i]) 313 continue; 314 315 break; 316 } 317 318 if (i < 0) 319 goto out; 320 321 event->hw.idx = i; 322 cpuhw->event[i] = event; 323 ++cpuhw->n_events; 324 325 val = 0; 326 if (event->hw.sample_period) { 327 s64 left = local64_read(&event->hw.period_left); 328 if (left < 0x80000000L) 329 val = 0x80000000L - left; 330 } 331 local64_set(&event->hw.prev_count, val); 332 333 if (!(flags & PERF_EF_START)) { 334 event->hw.state = PERF_HES_STOPPED | PERF_HES_UPTODATE; 335 val = 0; 336 } 337 338 write_pmc(i, val); 339 perf_event_update_userpage(event); 340 341 write_pmlcb(i, event->hw.config >> 32); 342 write_pmlca(i, event->hw.config_base); 343 344 ret = 0; 345 out: 346 put_cpu_var(cpu_hw_events); 347 perf_pmu_enable(event->pmu); 348 return ret; 349 } 350 351 /* context locked on entry */ 352 static void fsl_emb_pmu_del(struct perf_event *event, int flags) 353 { 354 struct cpu_hw_events *cpuhw; 355 int i = event->hw.idx; 356 357 perf_pmu_disable(event->pmu); 358 if (i < 0) 359 goto out; 360 361 fsl_emb_pmu_read(event); 362 363 cpuhw = &get_cpu_var(cpu_hw_events); 364 365 WARN_ON(event != cpuhw->event[event->hw.idx]); 366 367 write_pmlca(i, 0); 368 write_pmlcb(i, 0); 369 write_pmc(i, 0); 370 371 cpuhw->event[i] = NULL; 372 event->hw.idx = -1; 373 374 /* 375 * TODO: if at least one restricted event exists, and we 376 * just freed up a non-restricted-capable counter, and 377 * there is a restricted-capable counter occupied by 378 * a non-restricted event, migrate that event to the 379 * vacated counter. 380 */ 381 382 cpuhw->n_events--; 383 384 out: 385 perf_pmu_enable(event->pmu); 386 put_cpu_var(cpu_hw_events); 387 } 388 389 static void fsl_emb_pmu_start(struct perf_event *event, int ef_flags) 390 { 391 unsigned long flags; 392 s64 left; 393 394 if (event->hw.idx < 0 || !event->hw.sample_period) 395 return; 396 397 if (!(event->hw.state & PERF_HES_STOPPED)) 398 return; 399 400 if (ef_flags & PERF_EF_RELOAD) 401 WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE)); 402 403 local_irq_save(flags); 404 perf_pmu_disable(event->pmu); 405 406 event->hw.state = 0; 407 left = local64_read(&event->hw.period_left); 408 write_pmc(event->hw.idx, left); 409 410 perf_event_update_userpage(event); 411 perf_pmu_enable(event->pmu); 412 local_irq_restore(flags); 413 } 414 415 static void fsl_emb_pmu_stop(struct perf_event *event, int ef_flags) 416 { 417 unsigned long flags; 418 419 if (event->hw.idx < 0 || !event->hw.sample_period) 420 return; 421 422 if (event->hw.state & PERF_HES_STOPPED) 423 return; 424 425 local_irq_save(flags); 426 perf_pmu_disable(event->pmu); 427 428 fsl_emb_pmu_read(event); 429 event->hw.state |= PERF_HES_STOPPED | PERF_HES_UPTODATE; 430 write_pmc(event->hw.idx, 0); 431 432 perf_event_update_userpage(event); 433 perf_pmu_enable(event->pmu); 434 local_irq_restore(flags); 435 } 436 437 /* 438 * Release the PMU if this is the last perf_event. 439 */ 440 static void hw_perf_event_destroy(struct perf_event *event) 441 { 442 if (!atomic_add_unless(&num_events, -1, 1)) { 443 mutex_lock(&pmc_reserve_mutex); 444 if (atomic_dec_return(&num_events) == 0) 445 release_pmc_hardware(); 446 mutex_unlock(&pmc_reserve_mutex); 447 } 448 } 449 450 /* 451 * Translate a generic cache event_id config to a raw event_id code. 452 */ 453 static int hw_perf_cache_event(u64 config, u64 *eventp) 454 { 455 unsigned long type, op, result; 456 int ev; 457 458 if (!ppmu->cache_events) 459 return -EINVAL; 460 461 /* unpack config */ 462 type = config & 0xff; 463 op = (config >> 8) & 0xff; 464 result = (config >> 16) & 0xff; 465 466 if (type >= PERF_COUNT_HW_CACHE_MAX || 467 op >= PERF_COUNT_HW_CACHE_OP_MAX || 468 result >= PERF_COUNT_HW_CACHE_RESULT_MAX) 469 return -EINVAL; 470 471 ev = (*ppmu->cache_events)[type][op][result]; 472 if (ev == 0) 473 return -EOPNOTSUPP; 474 if (ev == -1) 475 return -EINVAL; 476 *eventp = ev; 477 return 0; 478 } 479 480 static int fsl_emb_pmu_event_init(struct perf_event *event) 481 { 482 u64 ev; 483 struct perf_event *events[MAX_HWEVENTS]; 484 int n; 485 int err; 486 int num_restricted; 487 int i; 488 489 if (ppmu->n_counter > MAX_HWEVENTS) { 490 WARN(1, "No. of perf counters (%d) is higher than max array size(%d)\n", 491 ppmu->n_counter, MAX_HWEVENTS); 492 ppmu->n_counter = MAX_HWEVENTS; 493 } 494 495 switch (event->attr.type) { 496 case PERF_TYPE_HARDWARE: 497 ev = event->attr.config; 498 if (ev >= ppmu->n_generic || ppmu->generic_events[ev] == 0) 499 return -EOPNOTSUPP; 500 ev = ppmu->generic_events[ev]; 501 break; 502 503 case PERF_TYPE_HW_CACHE: 504 err = hw_perf_cache_event(event->attr.config, &ev); 505 if (err) 506 return err; 507 break; 508 509 case PERF_TYPE_RAW: 510 ev = event->attr.config; 511 break; 512 513 default: 514 return -ENOENT; 515 } 516 517 event->hw.config = ppmu->xlate_event(ev); 518 if (!(event->hw.config & FSL_EMB_EVENT_VALID)) 519 return -EINVAL; 520 521 /* 522 * If this is in a group, check if it can go on with all the 523 * other hardware events in the group. We assume the event 524 * hasn't been linked into its leader's sibling list at this point. 525 */ 526 n = 0; 527 if (event->group_leader != event) { 528 n = collect_events(event->group_leader, 529 ppmu->n_counter - 1, events); 530 if (n < 0) 531 return -EINVAL; 532 } 533 534 if (event->hw.config & FSL_EMB_EVENT_RESTRICTED) { 535 num_restricted = 0; 536 for (i = 0; i < n; i++) { 537 if (events[i]->hw.config & FSL_EMB_EVENT_RESTRICTED) 538 num_restricted++; 539 } 540 541 if (num_restricted >= ppmu->n_restricted) 542 return -EINVAL; 543 } 544 545 event->hw.idx = -1; 546 547 event->hw.config_base = PMLCA_CE | PMLCA_FCM1 | 548 (u32)((ev << 16) & PMLCA_EVENT_MASK); 549 550 if (event->attr.exclude_user) 551 event->hw.config_base |= PMLCA_FCU; 552 if (event->attr.exclude_kernel) 553 event->hw.config_base |= PMLCA_FCS; 554 if (event->attr.exclude_idle) 555 return -ENOTSUPP; 556 557 event->hw.last_period = event->hw.sample_period; 558 local64_set(&event->hw.period_left, event->hw.last_period); 559 560 /* 561 * See if we need to reserve the PMU. 562 * If no events are currently in use, then we have to take a 563 * mutex to ensure that we don't race with another task doing 564 * reserve_pmc_hardware or release_pmc_hardware. 565 */ 566 err = 0; 567 if (!atomic_inc_not_zero(&num_events)) { 568 mutex_lock(&pmc_reserve_mutex); 569 if (atomic_read(&num_events) == 0 && 570 reserve_pmc_hardware(perf_event_interrupt)) 571 err = -EBUSY; 572 else 573 atomic_inc(&num_events); 574 mutex_unlock(&pmc_reserve_mutex); 575 576 mtpmr(PMRN_PMGC0, PMGC0_FAC); 577 isync(); 578 } 579 event->destroy = hw_perf_event_destroy; 580 581 return err; 582 } 583 584 static struct pmu fsl_emb_pmu = { 585 .pmu_enable = fsl_emb_pmu_enable, 586 .pmu_disable = fsl_emb_pmu_disable, 587 .event_init = fsl_emb_pmu_event_init, 588 .add = fsl_emb_pmu_add, 589 .del = fsl_emb_pmu_del, 590 .start = fsl_emb_pmu_start, 591 .stop = fsl_emb_pmu_stop, 592 .read = fsl_emb_pmu_read, 593 }; 594 595 /* 596 * A counter has overflowed; update its count and record 597 * things if requested. Note that interrupts are hard-disabled 598 * here so there is no possibility of being interrupted. 599 */ 600 static void record_and_restart(struct perf_event *event, unsigned long val, 601 struct pt_regs *regs) 602 { 603 u64 period = event->hw.sample_period; 604 s64 prev, delta, left; 605 int record = 0; 606 607 if (event->hw.state & PERF_HES_STOPPED) { 608 write_pmc(event->hw.idx, 0); 609 return; 610 } 611 612 /* we don't have to worry about interrupts here */ 613 prev = local64_read(&event->hw.prev_count); 614 delta = (val - prev) & 0xfffffffful; 615 local64_add(delta, &event->count); 616 617 /* 618 * See if the total period for this event has expired, 619 * and update for the next period. 620 */ 621 val = 0; 622 left = local64_read(&event->hw.period_left) - delta; 623 if (period) { 624 if (left <= 0) { 625 left += period; 626 if (left <= 0) 627 left = period; 628 record = 1; 629 event->hw.last_period = event->hw.sample_period; 630 } 631 if (left < 0x80000000LL) 632 val = 0x80000000LL - left; 633 } 634 635 write_pmc(event->hw.idx, val); 636 local64_set(&event->hw.prev_count, val); 637 local64_set(&event->hw.period_left, left); 638 perf_event_update_userpage(event); 639 640 /* 641 * Finally record data if requested. 642 */ 643 if (record) { 644 struct perf_sample_data data; 645 646 perf_sample_data_init(&data, 0, event->hw.last_period); 647 648 if (perf_event_overflow(event, &data, regs)) 649 fsl_emb_pmu_stop(event, 0); 650 } 651 } 652 653 static void perf_event_interrupt(struct pt_regs *regs) 654 { 655 int i; 656 struct cpu_hw_events *cpuhw = &__get_cpu_var(cpu_hw_events); 657 struct perf_event *event; 658 unsigned long val; 659 int found = 0; 660 int nmi; 661 662 nmi = perf_intr_is_nmi(regs); 663 if (nmi) 664 nmi_enter(); 665 else 666 irq_enter(); 667 668 for (i = 0; i < ppmu->n_counter; ++i) { 669 event = cpuhw->event[i]; 670 671 val = read_pmc(i); 672 if ((int)val < 0) { 673 if (event) { 674 /* event has overflowed */ 675 found = 1; 676 record_and_restart(event, val, regs); 677 } else { 678 /* 679 * Disabled counter is negative, 680 * reset it just in case. 681 */ 682 write_pmc(i, 0); 683 } 684 } 685 } 686 687 /* PMM will keep counters frozen until we return from the interrupt. */ 688 mtmsr(mfmsr() | MSR_PMM); 689 mtpmr(PMRN_PMGC0, PMGC0_PMIE | PMGC0_FCECE); 690 isync(); 691 692 if (nmi) 693 nmi_exit(); 694 else 695 irq_exit(); 696 } 697 698 void hw_perf_event_setup(int cpu) 699 { 700 struct cpu_hw_events *cpuhw = &per_cpu(cpu_hw_events, cpu); 701 702 memset(cpuhw, 0, sizeof(*cpuhw)); 703 } 704 705 int register_fsl_emb_pmu(struct fsl_emb_pmu *pmu) 706 { 707 if (ppmu) 708 return -EBUSY; /* something's already registered */ 709 710 ppmu = pmu; 711 pr_info("%s performance monitor hardware support registered\n", 712 pmu->name); 713 714 perf_pmu_register(&fsl_emb_pmu, "cpu", PERF_TYPE_RAW); 715 716 return 0; 717 } 718