1 #include <linux/perf_event.h> 2 #include <linux/export.h> 3 #include <linux/types.h> 4 #include <linux/init.h> 5 #include <linux/slab.h> 6 #include <linux/delay.h> 7 #include <asm/apicdef.h> 8 #include <asm/nmi.h> 9 10 #include "../perf_event.h" 11 12 static DEFINE_PER_CPU(unsigned int, perf_nmi_counter); 13 14 static __initconst const u64 amd_hw_cache_event_ids 15 [PERF_COUNT_HW_CACHE_MAX] 16 [PERF_COUNT_HW_CACHE_OP_MAX] 17 [PERF_COUNT_HW_CACHE_RESULT_MAX] = 18 { 19 [ C(L1D) ] = { 20 [ C(OP_READ) ] = { 21 [ C(RESULT_ACCESS) ] = 0x0040, /* Data Cache Accesses */ 22 [ C(RESULT_MISS) ] = 0x0141, /* Data Cache Misses */ 23 }, 24 [ C(OP_WRITE) ] = { 25 [ C(RESULT_ACCESS) ] = 0, 26 [ C(RESULT_MISS) ] = 0, 27 }, 28 [ C(OP_PREFETCH) ] = { 29 [ C(RESULT_ACCESS) ] = 0x0267, /* Data Prefetcher :attempts */ 30 [ C(RESULT_MISS) ] = 0x0167, /* Data Prefetcher :cancelled */ 31 }, 32 }, 33 [ C(L1I ) ] = { 34 [ C(OP_READ) ] = { 35 [ C(RESULT_ACCESS) ] = 0x0080, /* Instruction cache fetches */ 36 [ C(RESULT_MISS) ] = 0x0081, /* Instruction cache misses */ 37 }, 38 [ C(OP_WRITE) ] = { 39 [ C(RESULT_ACCESS) ] = -1, 40 [ C(RESULT_MISS) ] = -1, 41 }, 42 [ C(OP_PREFETCH) ] = { 43 [ C(RESULT_ACCESS) ] = 0x014B, /* Prefetch Instructions :Load */ 44 [ C(RESULT_MISS) ] = 0, 45 }, 46 }, 47 [ C(LL ) ] = { 48 [ C(OP_READ) ] = { 49 [ C(RESULT_ACCESS) ] = 0x037D, /* Requests to L2 Cache :IC+DC */ 50 [ C(RESULT_MISS) ] = 0x037E, /* L2 Cache Misses : IC+DC */ 51 }, 52 [ C(OP_WRITE) ] = { 53 [ C(RESULT_ACCESS) ] = 0x017F, /* L2 Fill/Writeback */ 54 [ C(RESULT_MISS) ] = 0, 55 }, 56 [ C(OP_PREFETCH) ] = { 57 [ C(RESULT_ACCESS) ] = 0, 58 [ C(RESULT_MISS) ] = 0, 59 }, 60 }, 61 [ C(DTLB) ] = { 62 [ C(OP_READ) ] = { 63 [ C(RESULT_ACCESS) ] = 0x0040, /* Data Cache Accesses */ 64 [ C(RESULT_MISS) ] = 0x0746, /* L1_DTLB_AND_L2_DLTB_MISS.ALL */ 65 }, 66 [ C(OP_WRITE) ] = { 67 [ C(RESULT_ACCESS) ] = 0, 68 [ C(RESULT_MISS) ] = 0, 69 }, 70 [ C(OP_PREFETCH) ] = { 71 [ C(RESULT_ACCESS) ] = 0, 72 [ C(RESULT_MISS) ] = 0, 73 }, 74 }, 75 [ C(ITLB) ] = { 76 [ C(OP_READ) ] = { 77 [ C(RESULT_ACCESS) ] = 0x0080, /* Instruction fecthes */ 78 [ C(RESULT_MISS) ] = 0x0385, /* L1_ITLB_AND_L2_ITLB_MISS.ALL */ 79 }, 80 [ C(OP_WRITE) ] = { 81 [ C(RESULT_ACCESS) ] = -1, 82 [ C(RESULT_MISS) ] = -1, 83 }, 84 [ C(OP_PREFETCH) ] = { 85 [ C(RESULT_ACCESS) ] = -1, 86 [ C(RESULT_MISS) ] = -1, 87 }, 88 }, 89 [ C(BPU ) ] = { 90 [ C(OP_READ) ] = { 91 [ C(RESULT_ACCESS) ] = 0x00c2, /* Retired Branch Instr. */ 92 [ C(RESULT_MISS) ] = 0x00c3, /* Retired Mispredicted BI */ 93 }, 94 [ C(OP_WRITE) ] = { 95 [ C(RESULT_ACCESS) ] = -1, 96 [ C(RESULT_MISS) ] = -1, 97 }, 98 [ C(OP_PREFETCH) ] = { 99 [ C(RESULT_ACCESS) ] = -1, 100 [ C(RESULT_MISS) ] = -1, 101 }, 102 }, 103 [ C(NODE) ] = { 104 [ C(OP_READ) ] = { 105 [ C(RESULT_ACCESS) ] = 0xb8e9, /* CPU Request to Memory, l+r */ 106 [ C(RESULT_MISS) ] = 0x98e9, /* CPU Request to Memory, r */ 107 }, 108 [ C(OP_WRITE) ] = { 109 [ C(RESULT_ACCESS) ] = -1, 110 [ C(RESULT_MISS) ] = -1, 111 }, 112 [ C(OP_PREFETCH) ] = { 113 [ C(RESULT_ACCESS) ] = -1, 114 [ C(RESULT_MISS) ] = -1, 115 }, 116 }, 117 }; 118 119 /* 120 * AMD Performance Monitor K7 and later, up to and including Family 16h: 121 */ 122 static const u64 amd_perfmon_event_map[PERF_COUNT_HW_MAX] = 123 { 124 [PERF_COUNT_HW_CPU_CYCLES] = 0x0076, 125 [PERF_COUNT_HW_INSTRUCTIONS] = 0x00c0, 126 [PERF_COUNT_HW_CACHE_REFERENCES] = 0x077d, 127 [PERF_COUNT_HW_CACHE_MISSES] = 0x077e, 128 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = 0x00c2, 129 [PERF_COUNT_HW_BRANCH_MISSES] = 0x00c3, 130 [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = 0x00d0, /* "Decoder empty" event */ 131 [PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = 0x00d1, /* "Dispatch stalls" event */ 132 }; 133 134 /* 135 * AMD Performance Monitor Family 17h and later: 136 */ 137 static const u64 amd_f17h_perfmon_event_map[PERF_COUNT_HW_MAX] = 138 { 139 [PERF_COUNT_HW_CPU_CYCLES] = 0x0076, 140 [PERF_COUNT_HW_INSTRUCTIONS] = 0x00c0, 141 [PERF_COUNT_HW_CACHE_REFERENCES] = 0xff60, 142 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = 0x00c2, 143 [PERF_COUNT_HW_BRANCH_MISSES] = 0x00c3, 144 [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = 0x0287, 145 [PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = 0x0187, 146 }; 147 148 static u64 amd_pmu_event_map(int hw_event) 149 { 150 if (boot_cpu_data.x86 >= 0x17) 151 return amd_f17h_perfmon_event_map[hw_event]; 152 153 return amd_perfmon_event_map[hw_event]; 154 } 155 156 /* 157 * Previously calculated offsets 158 */ 159 static unsigned int event_offsets[X86_PMC_IDX_MAX] __read_mostly; 160 static unsigned int count_offsets[X86_PMC_IDX_MAX] __read_mostly; 161 162 /* 163 * Legacy CPUs: 164 * 4 counters starting at 0xc0010000 each offset by 1 165 * 166 * CPUs with core performance counter extensions: 167 * 6 counters starting at 0xc0010200 each offset by 2 168 */ 169 static inline int amd_pmu_addr_offset(int index, bool eventsel) 170 { 171 int offset; 172 173 if (!index) 174 return index; 175 176 if (eventsel) 177 offset = event_offsets[index]; 178 else 179 offset = count_offsets[index]; 180 181 if (offset) 182 return offset; 183 184 if (!boot_cpu_has(X86_FEATURE_PERFCTR_CORE)) 185 offset = index; 186 else 187 offset = index << 1; 188 189 if (eventsel) 190 event_offsets[index] = offset; 191 else 192 count_offsets[index] = offset; 193 194 return offset; 195 } 196 197 static int amd_core_hw_config(struct perf_event *event) 198 { 199 if (event->attr.exclude_host && event->attr.exclude_guest) 200 /* 201 * When HO == GO == 1 the hardware treats that as GO == HO == 0 202 * and will count in both modes. We don't want to count in that 203 * case so we emulate no-counting by setting US = OS = 0. 204 */ 205 event->hw.config &= ~(ARCH_PERFMON_EVENTSEL_USR | 206 ARCH_PERFMON_EVENTSEL_OS); 207 else if (event->attr.exclude_host) 208 event->hw.config |= AMD64_EVENTSEL_GUESTONLY; 209 else if (event->attr.exclude_guest) 210 event->hw.config |= AMD64_EVENTSEL_HOSTONLY; 211 212 return 0; 213 } 214 215 /* 216 * AMD64 events are detected based on their event codes. 217 */ 218 static inline unsigned int amd_get_event_code(struct hw_perf_event *hwc) 219 { 220 return ((hwc->config >> 24) & 0x0f00) | (hwc->config & 0x00ff); 221 } 222 223 static inline int amd_is_nb_event(struct hw_perf_event *hwc) 224 { 225 return (hwc->config & 0xe0) == 0xe0; 226 } 227 228 static inline int amd_has_nb(struct cpu_hw_events *cpuc) 229 { 230 struct amd_nb *nb = cpuc->amd_nb; 231 232 return nb && nb->nb_id != -1; 233 } 234 235 static int amd_pmu_hw_config(struct perf_event *event) 236 { 237 int ret; 238 239 /* pass precise event sampling to ibs: */ 240 if (event->attr.precise_ip && get_ibs_caps()) 241 return -ENOENT; 242 243 if (has_branch_stack(event)) 244 return -EOPNOTSUPP; 245 246 ret = x86_pmu_hw_config(event); 247 if (ret) 248 return ret; 249 250 if (event->attr.type == PERF_TYPE_RAW) 251 event->hw.config |= event->attr.config & AMD64_RAW_EVENT_MASK; 252 253 return amd_core_hw_config(event); 254 } 255 256 static void __amd_put_nb_event_constraints(struct cpu_hw_events *cpuc, 257 struct perf_event *event) 258 { 259 struct amd_nb *nb = cpuc->amd_nb; 260 int i; 261 262 /* 263 * need to scan whole list because event may not have 264 * been assigned during scheduling 265 * 266 * no race condition possible because event can only 267 * be removed on one CPU at a time AND PMU is disabled 268 * when we come here 269 */ 270 for (i = 0; i < x86_pmu.num_counters; i++) { 271 if (cmpxchg(nb->owners + i, event, NULL) == event) 272 break; 273 } 274 } 275 276 /* 277 * AMD64 NorthBridge events need special treatment because 278 * counter access needs to be synchronized across all cores 279 * of a package. Refer to BKDG section 3.12 280 * 281 * NB events are events measuring L3 cache, Hypertransport 282 * traffic. They are identified by an event code >= 0xe00. 283 * They measure events on the NorthBride which is shared 284 * by all cores on a package. NB events are counted on a 285 * shared set of counters. When a NB event is programmed 286 * in a counter, the data actually comes from a shared 287 * counter. Thus, access to those counters needs to be 288 * synchronized. 289 * 290 * We implement the synchronization such that no two cores 291 * can be measuring NB events using the same counters. Thus, 292 * we maintain a per-NB allocation table. The available slot 293 * is propagated using the event_constraint structure. 294 * 295 * We provide only one choice for each NB event based on 296 * the fact that only NB events have restrictions. Consequently, 297 * if a counter is available, there is a guarantee the NB event 298 * will be assigned to it. If no slot is available, an empty 299 * constraint is returned and scheduling will eventually fail 300 * for this event. 301 * 302 * Note that all cores attached the same NB compete for the same 303 * counters to host NB events, this is why we use atomic ops. Some 304 * multi-chip CPUs may have more than one NB. 305 * 306 * Given that resources are allocated (cmpxchg), they must be 307 * eventually freed for others to use. This is accomplished by 308 * calling __amd_put_nb_event_constraints() 309 * 310 * Non NB events are not impacted by this restriction. 311 */ 312 static struct event_constraint * 313 __amd_get_nb_event_constraints(struct cpu_hw_events *cpuc, struct perf_event *event, 314 struct event_constraint *c) 315 { 316 struct hw_perf_event *hwc = &event->hw; 317 struct amd_nb *nb = cpuc->amd_nb; 318 struct perf_event *old; 319 int idx, new = -1; 320 321 if (!c) 322 c = &unconstrained; 323 324 if (cpuc->is_fake) 325 return c; 326 327 /* 328 * detect if already present, if so reuse 329 * 330 * cannot merge with actual allocation 331 * because of possible holes 332 * 333 * event can already be present yet not assigned (in hwc->idx) 334 * because of successive calls to x86_schedule_events() from 335 * hw_perf_group_sched_in() without hw_perf_enable() 336 */ 337 for_each_set_bit(idx, c->idxmsk, x86_pmu.num_counters) { 338 if (new == -1 || hwc->idx == idx) 339 /* assign free slot, prefer hwc->idx */ 340 old = cmpxchg(nb->owners + idx, NULL, event); 341 else if (nb->owners[idx] == event) 342 /* event already present */ 343 old = event; 344 else 345 continue; 346 347 if (old && old != event) 348 continue; 349 350 /* reassign to this slot */ 351 if (new != -1) 352 cmpxchg(nb->owners + new, event, NULL); 353 new = idx; 354 355 /* already present, reuse */ 356 if (old == event) 357 break; 358 } 359 360 if (new == -1) 361 return &emptyconstraint; 362 363 return &nb->event_constraints[new]; 364 } 365 366 static struct amd_nb *amd_alloc_nb(int cpu) 367 { 368 struct amd_nb *nb; 369 int i; 370 371 nb = kzalloc_node(sizeof(struct amd_nb), GFP_KERNEL, cpu_to_node(cpu)); 372 if (!nb) 373 return NULL; 374 375 nb->nb_id = -1; 376 377 /* 378 * initialize all possible NB constraints 379 */ 380 for (i = 0; i < x86_pmu.num_counters; i++) { 381 __set_bit(i, nb->event_constraints[i].idxmsk); 382 nb->event_constraints[i].weight = 1; 383 } 384 return nb; 385 } 386 387 static int amd_pmu_cpu_prepare(int cpu) 388 { 389 struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu); 390 391 WARN_ON_ONCE(cpuc->amd_nb); 392 393 if (!x86_pmu.amd_nb_constraints) 394 return 0; 395 396 cpuc->amd_nb = amd_alloc_nb(cpu); 397 if (!cpuc->amd_nb) 398 return -ENOMEM; 399 400 return 0; 401 } 402 403 static void amd_pmu_cpu_starting(int cpu) 404 { 405 struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu); 406 void **onln = &cpuc->kfree_on_online[X86_PERF_KFREE_SHARED]; 407 struct amd_nb *nb; 408 int i, nb_id; 409 410 cpuc->perf_ctr_virt_mask = AMD64_EVENTSEL_HOSTONLY; 411 412 if (!x86_pmu.amd_nb_constraints) 413 return; 414 415 nb_id = amd_get_nb_id(cpu); 416 WARN_ON_ONCE(nb_id == BAD_APICID); 417 418 for_each_online_cpu(i) { 419 nb = per_cpu(cpu_hw_events, i).amd_nb; 420 if (WARN_ON_ONCE(!nb)) 421 continue; 422 423 if (nb->nb_id == nb_id) { 424 *onln = cpuc->amd_nb; 425 cpuc->amd_nb = nb; 426 break; 427 } 428 } 429 430 cpuc->amd_nb->nb_id = nb_id; 431 cpuc->amd_nb->refcnt++; 432 } 433 434 static void amd_pmu_cpu_dead(int cpu) 435 { 436 struct cpu_hw_events *cpuhw; 437 438 if (!x86_pmu.amd_nb_constraints) 439 return; 440 441 cpuhw = &per_cpu(cpu_hw_events, cpu); 442 443 if (cpuhw->amd_nb) { 444 struct amd_nb *nb = cpuhw->amd_nb; 445 446 if (nb->nb_id == -1 || --nb->refcnt == 0) 447 kfree(nb); 448 449 cpuhw->amd_nb = NULL; 450 } 451 } 452 453 /* 454 * When a PMC counter overflows, an NMI is used to process the event and 455 * reset the counter. NMI latency can result in the counter being updated 456 * before the NMI can run, which can result in what appear to be spurious 457 * NMIs. This function is intended to wait for the NMI to run and reset 458 * the counter to avoid possible unhandled NMI messages. 459 */ 460 #define OVERFLOW_WAIT_COUNT 50 461 462 static void amd_pmu_wait_on_overflow(int idx) 463 { 464 unsigned int i; 465 u64 counter; 466 467 /* 468 * Wait for the counter to be reset if it has overflowed. This loop 469 * should exit very, very quickly, but just in case, don't wait 470 * forever... 471 */ 472 for (i = 0; i < OVERFLOW_WAIT_COUNT; i++) { 473 rdmsrl(x86_pmu_event_addr(idx), counter); 474 if (counter & (1ULL << (x86_pmu.cntval_bits - 1))) 475 break; 476 477 /* Might be in IRQ context, so can't sleep */ 478 udelay(1); 479 } 480 } 481 482 static void amd_pmu_disable_all(void) 483 { 484 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 485 int idx; 486 487 x86_pmu_disable_all(); 488 489 /* 490 * This shouldn't be called from NMI context, but add a safeguard here 491 * to return, since if we're in NMI context we can't wait for an NMI 492 * to reset an overflowed counter value. 493 */ 494 if (in_nmi()) 495 return; 496 497 /* 498 * Check each counter for overflow and wait for it to be reset by the 499 * NMI if it has overflowed. This relies on the fact that all active 500 * counters are always enabled when this function is caled and 501 * ARCH_PERFMON_EVENTSEL_INT is always set. 502 */ 503 for (idx = 0; idx < x86_pmu.num_counters; idx++) { 504 if (!test_bit(idx, cpuc->active_mask)) 505 continue; 506 507 amd_pmu_wait_on_overflow(idx); 508 } 509 } 510 511 static void amd_pmu_disable_event(struct perf_event *event) 512 { 513 x86_pmu_disable_event(event); 514 515 /* 516 * This can be called from NMI context (via x86_pmu_stop). The counter 517 * may have overflowed, but either way, we'll never see it get reset 518 * by the NMI if we're already in the NMI. And the NMI latency support 519 * below will take care of any pending NMI that might have been 520 * generated by the overflow. 521 */ 522 if (in_nmi()) 523 return; 524 525 amd_pmu_wait_on_overflow(event->hw.idx); 526 } 527 528 /* 529 * Because of NMI latency, if multiple PMC counters are active or other sources 530 * of NMIs are received, the perf NMI handler can handle one or more overflowed 531 * PMC counters outside of the NMI associated with the PMC overflow. If the NMI 532 * doesn't arrive at the LAPIC in time to become a pending NMI, then the kernel 533 * back-to-back NMI support won't be active. This PMC handler needs to take into 534 * account that this can occur, otherwise this could result in unknown NMI 535 * messages being issued. Examples of this is PMC overflow while in the NMI 536 * handler when multiple PMCs are active or PMC overflow while handling some 537 * other source of an NMI. 538 * 539 * Attempt to mitigate this by using the number of active PMCs to determine 540 * whether to return NMI_HANDLED if the perf NMI handler did not handle/reset 541 * any PMCs. The per-CPU perf_nmi_counter variable is set to a minimum of the 542 * number of active PMCs or 2. The value of 2 is used in case an NMI does not 543 * arrive at the LAPIC in time to be collapsed into an already pending NMI. 544 */ 545 static int amd_pmu_handle_irq(struct pt_regs *regs) 546 { 547 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 548 int active, handled; 549 550 /* 551 * Obtain the active count before calling x86_pmu_handle_irq() since 552 * it is possible that x86_pmu_handle_irq() may make a counter 553 * inactive (through x86_pmu_stop). 554 */ 555 active = __bitmap_weight(cpuc->active_mask, X86_PMC_IDX_MAX); 556 557 /* Process any counter overflows */ 558 handled = x86_pmu_handle_irq(regs); 559 560 /* 561 * If a counter was handled, record the number of possible remaining 562 * NMIs that can occur. 563 */ 564 if (handled) { 565 this_cpu_write(perf_nmi_counter, 566 min_t(unsigned int, 2, active)); 567 568 return handled; 569 } 570 571 if (!this_cpu_read(perf_nmi_counter)) 572 return NMI_DONE; 573 574 this_cpu_dec(perf_nmi_counter); 575 576 return NMI_HANDLED; 577 } 578 579 static struct event_constraint * 580 amd_get_event_constraints(struct cpu_hw_events *cpuc, int idx, 581 struct perf_event *event) 582 { 583 /* 584 * if not NB event or no NB, then no constraints 585 */ 586 if (!(amd_has_nb(cpuc) && amd_is_nb_event(&event->hw))) 587 return &unconstrained; 588 589 return __amd_get_nb_event_constraints(cpuc, event, NULL); 590 } 591 592 static void amd_put_event_constraints(struct cpu_hw_events *cpuc, 593 struct perf_event *event) 594 { 595 if (amd_has_nb(cpuc) && amd_is_nb_event(&event->hw)) 596 __amd_put_nb_event_constraints(cpuc, event); 597 } 598 599 PMU_FORMAT_ATTR(event, "config:0-7,32-35"); 600 PMU_FORMAT_ATTR(umask, "config:8-15" ); 601 PMU_FORMAT_ATTR(edge, "config:18" ); 602 PMU_FORMAT_ATTR(inv, "config:23" ); 603 PMU_FORMAT_ATTR(cmask, "config:24-31" ); 604 605 static struct attribute *amd_format_attr[] = { 606 &format_attr_event.attr, 607 &format_attr_umask.attr, 608 &format_attr_edge.attr, 609 &format_attr_inv.attr, 610 &format_attr_cmask.attr, 611 NULL, 612 }; 613 614 /* AMD Family 15h */ 615 616 #define AMD_EVENT_TYPE_MASK 0x000000F0ULL 617 618 #define AMD_EVENT_FP 0x00000000ULL ... 0x00000010ULL 619 #define AMD_EVENT_LS 0x00000020ULL ... 0x00000030ULL 620 #define AMD_EVENT_DC 0x00000040ULL ... 0x00000050ULL 621 #define AMD_EVENT_CU 0x00000060ULL ... 0x00000070ULL 622 #define AMD_EVENT_IC_DE 0x00000080ULL ... 0x00000090ULL 623 #define AMD_EVENT_EX_LS 0x000000C0ULL 624 #define AMD_EVENT_DE 0x000000D0ULL 625 #define AMD_EVENT_NB 0x000000E0ULL ... 0x000000F0ULL 626 627 /* 628 * AMD family 15h event code/PMC mappings: 629 * 630 * type = event_code & 0x0F0: 631 * 632 * 0x000 FP PERF_CTL[5:3] 633 * 0x010 FP PERF_CTL[5:3] 634 * 0x020 LS PERF_CTL[5:0] 635 * 0x030 LS PERF_CTL[5:0] 636 * 0x040 DC PERF_CTL[5:0] 637 * 0x050 DC PERF_CTL[5:0] 638 * 0x060 CU PERF_CTL[2:0] 639 * 0x070 CU PERF_CTL[2:0] 640 * 0x080 IC/DE PERF_CTL[2:0] 641 * 0x090 IC/DE PERF_CTL[2:0] 642 * 0x0A0 --- 643 * 0x0B0 --- 644 * 0x0C0 EX/LS PERF_CTL[5:0] 645 * 0x0D0 DE PERF_CTL[2:0] 646 * 0x0E0 NB NB_PERF_CTL[3:0] 647 * 0x0F0 NB NB_PERF_CTL[3:0] 648 * 649 * Exceptions: 650 * 651 * 0x000 FP PERF_CTL[3], PERF_CTL[5:3] (*) 652 * 0x003 FP PERF_CTL[3] 653 * 0x004 FP PERF_CTL[3], PERF_CTL[5:3] (*) 654 * 0x00B FP PERF_CTL[3] 655 * 0x00D FP PERF_CTL[3] 656 * 0x023 DE PERF_CTL[2:0] 657 * 0x02D LS PERF_CTL[3] 658 * 0x02E LS PERF_CTL[3,0] 659 * 0x031 LS PERF_CTL[2:0] (**) 660 * 0x043 CU PERF_CTL[2:0] 661 * 0x045 CU PERF_CTL[2:0] 662 * 0x046 CU PERF_CTL[2:0] 663 * 0x054 CU PERF_CTL[2:0] 664 * 0x055 CU PERF_CTL[2:0] 665 * 0x08F IC PERF_CTL[0] 666 * 0x187 DE PERF_CTL[0] 667 * 0x188 DE PERF_CTL[0] 668 * 0x0DB EX PERF_CTL[5:0] 669 * 0x0DC LS PERF_CTL[5:0] 670 * 0x0DD LS PERF_CTL[5:0] 671 * 0x0DE LS PERF_CTL[5:0] 672 * 0x0DF LS PERF_CTL[5:0] 673 * 0x1C0 EX PERF_CTL[5:3] 674 * 0x1D6 EX PERF_CTL[5:0] 675 * 0x1D8 EX PERF_CTL[5:0] 676 * 677 * (*) depending on the umask all FPU counters may be used 678 * (**) only one unitmask enabled at a time 679 */ 680 681 static struct event_constraint amd_f15_PMC0 = EVENT_CONSTRAINT(0, 0x01, 0); 682 static struct event_constraint amd_f15_PMC20 = EVENT_CONSTRAINT(0, 0x07, 0); 683 static struct event_constraint amd_f15_PMC3 = EVENT_CONSTRAINT(0, 0x08, 0); 684 static struct event_constraint amd_f15_PMC30 = EVENT_CONSTRAINT_OVERLAP(0, 0x09, 0); 685 static struct event_constraint amd_f15_PMC50 = EVENT_CONSTRAINT(0, 0x3F, 0); 686 static struct event_constraint amd_f15_PMC53 = EVENT_CONSTRAINT(0, 0x38, 0); 687 688 static struct event_constraint * 689 amd_get_event_constraints_f15h(struct cpu_hw_events *cpuc, int idx, 690 struct perf_event *event) 691 { 692 struct hw_perf_event *hwc = &event->hw; 693 unsigned int event_code = amd_get_event_code(hwc); 694 695 switch (event_code & AMD_EVENT_TYPE_MASK) { 696 case AMD_EVENT_FP: 697 switch (event_code) { 698 case 0x000: 699 if (!(hwc->config & 0x0000F000ULL)) 700 break; 701 if (!(hwc->config & 0x00000F00ULL)) 702 break; 703 return &amd_f15_PMC3; 704 case 0x004: 705 if (hweight_long(hwc->config & ARCH_PERFMON_EVENTSEL_UMASK) <= 1) 706 break; 707 return &amd_f15_PMC3; 708 case 0x003: 709 case 0x00B: 710 case 0x00D: 711 return &amd_f15_PMC3; 712 } 713 return &amd_f15_PMC53; 714 case AMD_EVENT_LS: 715 case AMD_EVENT_DC: 716 case AMD_EVENT_EX_LS: 717 switch (event_code) { 718 case 0x023: 719 case 0x043: 720 case 0x045: 721 case 0x046: 722 case 0x054: 723 case 0x055: 724 return &amd_f15_PMC20; 725 case 0x02D: 726 return &amd_f15_PMC3; 727 case 0x02E: 728 return &amd_f15_PMC30; 729 case 0x031: 730 if (hweight_long(hwc->config & ARCH_PERFMON_EVENTSEL_UMASK) <= 1) 731 return &amd_f15_PMC20; 732 return &emptyconstraint; 733 case 0x1C0: 734 return &amd_f15_PMC53; 735 default: 736 return &amd_f15_PMC50; 737 } 738 case AMD_EVENT_CU: 739 case AMD_EVENT_IC_DE: 740 case AMD_EVENT_DE: 741 switch (event_code) { 742 case 0x08F: 743 case 0x187: 744 case 0x188: 745 return &amd_f15_PMC0; 746 case 0x0DB ... 0x0DF: 747 case 0x1D6: 748 case 0x1D8: 749 return &amd_f15_PMC50; 750 default: 751 return &amd_f15_PMC20; 752 } 753 case AMD_EVENT_NB: 754 /* moved to uncore.c */ 755 return &emptyconstraint; 756 default: 757 return &emptyconstraint; 758 } 759 } 760 761 static ssize_t amd_event_sysfs_show(char *page, u64 config) 762 { 763 u64 event = (config & ARCH_PERFMON_EVENTSEL_EVENT) | 764 (config & AMD64_EVENTSEL_EVENT) >> 24; 765 766 return x86_event_sysfs_show(page, config, event); 767 } 768 769 static __initconst const struct x86_pmu amd_pmu = { 770 .name = "AMD", 771 .handle_irq = amd_pmu_handle_irq, 772 .disable_all = amd_pmu_disable_all, 773 .enable_all = x86_pmu_enable_all, 774 .enable = x86_pmu_enable_event, 775 .disable = amd_pmu_disable_event, 776 .hw_config = amd_pmu_hw_config, 777 .schedule_events = x86_schedule_events, 778 .eventsel = MSR_K7_EVNTSEL0, 779 .perfctr = MSR_K7_PERFCTR0, 780 .addr_offset = amd_pmu_addr_offset, 781 .event_map = amd_pmu_event_map, 782 .max_events = ARRAY_SIZE(amd_perfmon_event_map), 783 .num_counters = AMD64_NUM_COUNTERS, 784 .cntval_bits = 48, 785 .cntval_mask = (1ULL << 48) - 1, 786 .apic = 1, 787 /* use highest bit to detect overflow */ 788 .max_period = (1ULL << 47) - 1, 789 .get_event_constraints = amd_get_event_constraints, 790 .put_event_constraints = amd_put_event_constraints, 791 792 .format_attrs = amd_format_attr, 793 .events_sysfs_show = amd_event_sysfs_show, 794 795 .cpu_prepare = amd_pmu_cpu_prepare, 796 .cpu_starting = amd_pmu_cpu_starting, 797 .cpu_dead = amd_pmu_cpu_dead, 798 799 .amd_nb_constraints = 1, 800 }; 801 802 static int __init amd_core_pmu_init(void) 803 { 804 if (!boot_cpu_has(X86_FEATURE_PERFCTR_CORE)) 805 return 0; 806 807 switch (boot_cpu_data.x86) { 808 case 0x15: 809 pr_cont("Fam15h "); 810 x86_pmu.get_event_constraints = amd_get_event_constraints_f15h; 811 break; 812 case 0x17: 813 pr_cont("Fam17h "); 814 /* 815 * In family 17h, there are no event constraints in the PMC hardware. 816 * We fallback to using default amd_get_event_constraints. 817 */ 818 break; 819 case 0x18: 820 pr_cont("Fam18h "); 821 /* Using default amd_get_event_constraints. */ 822 break; 823 default: 824 pr_err("core perfctr but no constraints; unknown hardware!\n"); 825 return -ENODEV; 826 } 827 828 /* 829 * If core performance counter extensions exists, we must use 830 * MSR_F15H_PERF_CTL/MSR_F15H_PERF_CTR msrs. See also 831 * amd_pmu_addr_offset(). 832 */ 833 x86_pmu.eventsel = MSR_F15H_PERF_CTL; 834 x86_pmu.perfctr = MSR_F15H_PERF_CTR; 835 x86_pmu.num_counters = AMD64_NUM_COUNTERS_CORE; 836 /* 837 * AMD Core perfctr has separate MSRs for the NB events, see 838 * the amd/uncore.c driver. 839 */ 840 x86_pmu.amd_nb_constraints = 0; 841 842 pr_cont("core perfctr, "); 843 return 0; 844 } 845 846 __init int amd_pmu_init(void) 847 { 848 int ret; 849 850 /* Performance-monitoring supported from K7 and later: */ 851 if (boot_cpu_data.x86 < 6) 852 return -ENODEV; 853 854 x86_pmu = amd_pmu; 855 856 ret = amd_core_pmu_init(); 857 if (ret) 858 return ret; 859 860 if (num_possible_cpus() == 1) { 861 /* 862 * No point in allocating data structures to serialize 863 * against other CPUs, when there is only the one CPU. 864 */ 865 x86_pmu.amd_nb_constraints = 0; 866 } 867 868 /* Events are common for all AMDs */ 869 memcpy(hw_cache_event_ids, amd_hw_cache_event_ids, 870 sizeof(hw_cache_event_ids)); 871 872 return 0; 873 } 874 875 void amd_pmu_enable_virt(void) 876 { 877 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 878 879 cpuc->perf_ctr_virt_mask = 0; 880 881 /* Reload all events */ 882 amd_pmu_disable_all(); 883 x86_pmu_enable_all(0); 884 } 885 EXPORT_SYMBOL_GPL(amd_pmu_enable_virt); 886 887 void amd_pmu_disable_virt(void) 888 { 889 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 890 891 /* 892 * We only mask out the Host-only bit so that host-only counting works 893 * when SVM is disabled. If someone sets up a guest-only counter when 894 * SVM is disabled the Guest-only bits still gets set and the counter 895 * will not count anything. 896 */ 897 cpuc->perf_ctr_virt_mask = AMD64_EVENTSEL_HOSTONLY; 898 899 /* Reload all events */ 900 amd_pmu_disable_all(); 901 x86_pmu_enable_all(0); 902 } 903 EXPORT_SYMBOL_GPL(amd_pmu_disable_virt); 904