1 /* 2 * Performance events - AMD IBS 3 * 4 * Copyright (C) 2011 Advanced Micro Devices, Inc., Robert Richter 5 * 6 * For licencing details see kernel-base/COPYING 7 */ 8 9 #include <linux/perf_event.h> 10 #include <linux/init.h> 11 #include <linux/export.h> 12 #include <linux/pci.h> 13 #include <linux/ptrace.h> 14 #include <linux/syscore_ops.h> 15 #include <linux/sched/clock.h> 16 17 #include <asm/apic.h> 18 19 #include "../perf_event.h" 20 21 static u32 ibs_caps; 22 23 #if defined(CONFIG_PERF_EVENTS) && defined(CONFIG_CPU_SUP_AMD) 24 25 #include <linux/kprobes.h> 26 #include <linux/hardirq.h> 27 28 #include <asm/nmi.h> 29 #include <asm/amd-ibs.h> 30 31 #define IBS_FETCH_CONFIG_MASK (IBS_FETCH_RAND_EN | IBS_FETCH_MAX_CNT) 32 #define IBS_OP_CONFIG_MASK IBS_OP_MAX_CNT 33 34 35 /* 36 * IBS states: 37 * 38 * ENABLED; tracks the pmu::add(), pmu::del() state, when set the counter is taken 39 * and any further add()s must fail. 40 * 41 * STARTED/STOPPING/STOPPED; deal with pmu::start(), pmu::stop() state but are 42 * complicated by the fact that the IBS hardware can send late NMIs (ie. after 43 * we've cleared the EN bit). 44 * 45 * In order to consume these late NMIs we have the STOPPED state, any NMI that 46 * happens after we've cleared the EN state will clear this bit and report the 47 * NMI handled (this is fundamentally racy in the face or multiple NMI sources, 48 * someone else can consume our BIT and our NMI will go unhandled). 49 * 50 * And since we cannot set/clear this separate bit together with the EN bit, 51 * there are races; if we cleared STARTED early, an NMI could land in 52 * between clearing STARTED and clearing the EN bit (in fact multiple NMIs 53 * could happen if the period is small enough), and consume our STOPPED bit 54 * and trigger streams of unhandled NMIs. 55 * 56 * If, however, we clear STARTED late, an NMI can hit between clearing the 57 * EN bit and clearing STARTED, still see STARTED set and process the event. 58 * If this event will have the VALID bit clear, we bail properly, but this 59 * is not a given. With VALID set we can end up calling pmu::stop() again 60 * (the throttle logic) and trigger the WARNs in there. 61 * 62 * So what we do is set STOPPING before clearing EN to avoid the pmu::stop() 63 * nesting, and clear STARTED late, so that we have a well defined state over 64 * the clearing of the EN bit. 65 * 66 * XXX: we could probably be using !atomic bitops for all this. 67 */ 68 69 enum ibs_states { 70 IBS_ENABLED = 0, 71 IBS_STARTED = 1, 72 IBS_STOPPING = 2, 73 IBS_STOPPED = 3, 74 75 IBS_MAX_STATES, 76 }; 77 78 struct cpu_perf_ibs { 79 struct perf_event *event; 80 unsigned long state[BITS_TO_LONGS(IBS_MAX_STATES)]; 81 }; 82 83 struct perf_ibs { 84 struct pmu pmu; 85 unsigned int msr; 86 u64 config_mask; 87 u64 cnt_mask; 88 u64 enable_mask; 89 u64 valid_mask; 90 u64 max_period; 91 unsigned long offset_mask[1]; 92 int offset_max; 93 unsigned int fetch_count_reset_broken : 1; 94 unsigned int fetch_ignore_if_zero_rip : 1; 95 struct cpu_perf_ibs __percpu *pcpu; 96 97 u64 (*get_count)(u64 config); 98 }; 99 100 static int 101 perf_event_set_period(struct hw_perf_event *hwc, u64 min, u64 max, u64 *hw_period) 102 { 103 s64 left = local64_read(&hwc->period_left); 104 s64 period = hwc->sample_period; 105 int overflow = 0; 106 107 /* 108 * If we are way outside a reasonable range then just skip forward: 109 */ 110 if (unlikely(left <= -period)) { 111 left = period; 112 local64_set(&hwc->period_left, left); 113 hwc->last_period = period; 114 overflow = 1; 115 } 116 117 if (unlikely(left < (s64)min)) { 118 left += period; 119 local64_set(&hwc->period_left, left); 120 hwc->last_period = period; 121 overflow = 1; 122 } 123 124 /* 125 * If the hw period that triggers the sw overflow is too short 126 * we might hit the irq handler. This biases the results. 127 * Thus we shorten the next-to-last period and set the last 128 * period to the max period. 129 */ 130 if (left > max) { 131 left -= max; 132 if (left > max) 133 left = max; 134 else if (left < min) 135 left = min; 136 } 137 138 *hw_period = (u64)left; 139 140 return overflow; 141 } 142 143 static int 144 perf_event_try_update(struct perf_event *event, u64 new_raw_count, int width) 145 { 146 struct hw_perf_event *hwc = &event->hw; 147 int shift = 64 - width; 148 u64 prev_raw_count; 149 u64 delta; 150 151 /* 152 * Careful: an NMI might modify the previous event value. 153 * 154 * Our tactic to handle this is to first atomically read and 155 * exchange a new raw count - then add that new-prev delta 156 * count to the generic event atomically: 157 */ 158 prev_raw_count = local64_read(&hwc->prev_count); 159 if (local64_cmpxchg(&hwc->prev_count, prev_raw_count, 160 new_raw_count) != prev_raw_count) 161 return 0; 162 163 /* 164 * Now we have the new raw value and have updated the prev 165 * timestamp already. We can now calculate the elapsed delta 166 * (event-)time and add that to the generic event. 167 * 168 * Careful, not all hw sign-extends above the physical width 169 * of the count. 170 */ 171 delta = (new_raw_count << shift) - (prev_raw_count << shift); 172 delta >>= shift; 173 174 local64_add(delta, &event->count); 175 local64_sub(delta, &hwc->period_left); 176 177 return 1; 178 } 179 180 static struct perf_ibs perf_ibs_fetch; 181 static struct perf_ibs perf_ibs_op; 182 183 static struct perf_ibs *get_ibs_pmu(int type) 184 { 185 if (perf_ibs_fetch.pmu.type == type) 186 return &perf_ibs_fetch; 187 if (perf_ibs_op.pmu.type == type) 188 return &perf_ibs_op; 189 return NULL; 190 } 191 192 /* 193 * Use IBS for precise event sampling: 194 * 195 * perf record -a -e cpu-cycles:p ... # use ibs op counting cycle count 196 * perf record -a -e r076:p ... # same as -e cpu-cycles:p 197 * perf record -a -e r0C1:p ... # use ibs op counting micro-ops 198 * 199 * IbsOpCntCtl (bit 19) of IBS Execution Control Register (IbsOpCtl, 200 * MSRC001_1033) is used to select either cycle or micro-ops counting 201 * mode. 202 * 203 * The rip of IBS samples has skid 0. Thus, IBS supports precise 204 * levels 1 and 2 and the PERF_EFLAGS_EXACT is set. In rare cases the 205 * rip is invalid when IBS was not able to record the rip correctly. 206 * We clear PERF_EFLAGS_EXACT and take the rip from pt_regs then. 207 * 208 */ 209 static int perf_ibs_precise_event(struct perf_event *event, u64 *config) 210 { 211 switch (event->attr.precise_ip) { 212 case 0: 213 return -ENOENT; 214 case 1: 215 case 2: 216 break; 217 default: 218 return -EOPNOTSUPP; 219 } 220 221 switch (event->attr.type) { 222 case PERF_TYPE_HARDWARE: 223 switch (event->attr.config) { 224 case PERF_COUNT_HW_CPU_CYCLES: 225 *config = 0; 226 return 0; 227 } 228 break; 229 case PERF_TYPE_RAW: 230 switch (event->attr.config) { 231 case 0x0076: 232 *config = 0; 233 return 0; 234 case 0x00C1: 235 *config = IBS_OP_CNT_CTL; 236 return 0; 237 } 238 break; 239 default: 240 return -ENOENT; 241 } 242 243 return -EOPNOTSUPP; 244 } 245 246 static int perf_ibs_init(struct perf_event *event) 247 { 248 struct hw_perf_event *hwc = &event->hw; 249 struct perf_ibs *perf_ibs; 250 u64 max_cnt, config; 251 int ret; 252 253 perf_ibs = get_ibs_pmu(event->attr.type); 254 if (perf_ibs) { 255 config = event->attr.config; 256 } else { 257 perf_ibs = &perf_ibs_op; 258 ret = perf_ibs_precise_event(event, &config); 259 if (ret) 260 return ret; 261 } 262 263 if (event->pmu != &perf_ibs->pmu) 264 return -ENOENT; 265 266 if (config & ~perf_ibs->config_mask) 267 return -EINVAL; 268 269 if (hwc->sample_period) { 270 if (config & perf_ibs->cnt_mask) 271 /* raw max_cnt may not be set */ 272 return -EINVAL; 273 if (!event->attr.sample_freq && hwc->sample_period & 0x0f) 274 /* 275 * lower 4 bits can not be set in ibs max cnt, 276 * but allowing it in case we adjust the 277 * sample period to set a frequency. 278 */ 279 return -EINVAL; 280 hwc->sample_period &= ~0x0FULL; 281 if (!hwc->sample_period) 282 hwc->sample_period = 0x10; 283 } else { 284 max_cnt = config & perf_ibs->cnt_mask; 285 config &= ~perf_ibs->cnt_mask; 286 event->attr.sample_period = max_cnt << 4; 287 hwc->sample_period = event->attr.sample_period; 288 } 289 290 if (!hwc->sample_period) 291 return -EINVAL; 292 293 /* 294 * If we modify hwc->sample_period, we also need to update 295 * hwc->last_period and hwc->period_left. 296 */ 297 hwc->last_period = hwc->sample_period; 298 local64_set(&hwc->period_left, hwc->sample_period); 299 300 hwc->config_base = perf_ibs->msr; 301 hwc->config = config; 302 303 return 0; 304 } 305 306 static int perf_ibs_set_period(struct perf_ibs *perf_ibs, 307 struct hw_perf_event *hwc, u64 *period) 308 { 309 int overflow; 310 311 /* ignore lower 4 bits in min count: */ 312 overflow = perf_event_set_period(hwc, 1<<4, perf_ibs->max_period, period); 313 local64_set(&hwc->prev_count, 0); 314 315 return overflow; 316 } 317 318 static u64 get_ibs_fetch_count(u64 config) 319 { 320 union ibs_fetch_ctl fetch_ctl = (union ibs_fetch_ctl)config; 321 322 return fetch_ctl.fetch_cnt << 4; 323 } 324 325 static u64 get_ibs_op_count(u64 config) 326 { 327 union ibs_op_ctl op_ctl = (union ibs_op_ctl)config; 328 u64 count = 0; 329 330 /* 331 * If the internal 27-bit counter rolled over, the count is MaxCnt 332 * and the lower 7 bits of CurCnt are randomized. 333 * Otherwise CurCnt has the full 27-bit current counter value. 334 */ 335 if (op_ctl.op_val) { 336 count = op_ctl.opmaxcnt << 4; 337 if (ibs_caps & IBS_CAPS_OPCNTEXT) 338 count += op_ctl.opmaxcnt_ext << 20; 339 } else if (ibs_caps & IBS_CAPS_RDWROPCNT) { 340 count = op_ctl.opcurcnt; 341 } 342 343 return count; 344 } 345 346 static void 347 perf_ibs_event_update(struct perf_ibs *perf_ibs, struct perf_event *event, 348 u64 *config) 349 { 350 u64 count = perf_ibs->get_count(*config); 351 352 /* 353 * Set width to 64 since we do not overflow on max width but 354 * instead on max count. In perf_ibs_set_period() we clear 355 * prev count manually on overflow. 356 */ 357 while (!perf_event_try_update(event, count, 64)) { 358 rdmsrl(event->hw.config_base, *config); 359 count = perf_ibs->get_count(*config); 360 } 361 } 362 363 static inline void perf_ibs_enable_event(struct perf_ibs *perf_ibs, 364 struct hw_perf_event *hwc, u64 config) 365 { 366 u64 tmp = hwc->config | config; 367 368 if (perf_ibs->fetch_count_reset_broken) 369 wrmsrl(hwc->config_base, tmp & ~perf_ibs->enable_mask); 370 371 wrmsrl(hwc->config_base, tmp | perf_ibs->enable_mask); 372 } 373 374 /* 375 * Erratum #420 Instruction-Based Sampling Engine May Generate 376 * Interrupt that Cannot Be Cleared: 377 * 378 * Must clear counter mask first, then clear the enable bit. See 379 * Revision Guide for AMD Family 10h Processors, Publication #41322. 380 */ 381 static inline void perf_ibs_disable_event(struct perf_ibs *perf_ibs, 382 struct hw_perf_event *hwc, u64 config) 383 { 384 config &= ~perf_ibs->cnt_mask; 385 if (boot_cpu_data.x86 == 0x10) 386 wrmsrl(hwc->config_base, config); 387 config &= ~perf_ibs->enable_mask; 388 wrmsrl(hwc->config_base, config); 389 } 390 391 /* 392 * We cannot restore the ibs pmu state, so we always needs to update 393 * the event while stopping it and then reset the state when starting 394 * again. Thus, ignoring PERF_EF_RELOAD and PERF_EF_UPDATE flags in 395 * perf_ibs_start()/perf_ibs_stop() and instead always do it. 396 */ 397 static void perf_ibs_start(struct perf_event *event, int flags) 398 { 399 struct hw_perf_event *hwc = &event->hw; 400 struct perf_ibs *perf_ibs = container_of(event->pmu, struct perf_ibs, pmu); 401 struct cpu_perf_ibs *pcpu = this_cpu_ptr(perf_ibs->pcpu); 402 u64 period, config = 0; 403 404 if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED))) 405 return; 406 407 WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE)); 408 hwc->state = 0; 409 410 perf_ibs_set_period(perf_ibs, hwc, &period); 411 if (perf_ibs == &perf_ibs_op && (ibs_caps & IBS_CAPS_OPCNTEXT)) { 412 config |= period & IBS_OP_MAX_CNT_EXT_MASK; 413 period &= ~IBS_OP_MAX_CNT_EXT_MASK; 414 } 415 config |= period >> 4; 416 417 /* 418 * Set STARTED before enabling the hardware, such that a subsequent NMI 419 * must observe it. 420 */ 421 set_bit(IBS_STARTED, pcpu->state); 422 clear_bit(IBS_STOPPING, pcpu->state); 423 perf_ibs_enable_event(perf_ibs, hwc, config); 424 425 perf_event_update_userpage(event); 426 } 427 428 static void perf_ibs_stop(struct perf_event *event, int flags) 429 { 430 struct hw_perf_event *hwc = &event->hw; 431 struct perf_ibs *perf_ibs = container_of(event->pmu, struct perf_ibs, pmu); 432 struct cpu_perf_ibs *pcpu = this_cpu_ptr(perf_ibs->pcpu); 433 u64 config; 434 int stopping; 435 436 if (test_and_set_bit(IBS_STOPPING, pcpu->state)) 437 return; 438 439 stopping = test_bit(IBS_STARTED, pcpu->state); 440 441 if (!stopping && (hwc->state & PERF_HES_UPTODATE)) 442 return; 443 444 rdmsrl(hwc->config_base, config); 445 446 if (stopping) { 447 /* 448 * Set STOPPED before disabling the hardware, such that it 449 * must be visible to NMIs the moment we clear the EN bit, 450 * at which point we can generate an !VALID sample which 451 * we need to consume. 452 */ 453 set_bit(IBS_STOPPED, pcpu->state); 454 perf_ibs_disable_event(perf_ibs, hwc, config); 455 /* 456 * Clear STARTED after disabling the hardware; if it were 457 * cleared before an NMI hitting after the clear but before 458 * clearing the EN bit might think it a spurious NMI and not 459 * handle it. 460 * 461 * Clearing it after, however, creates the problem of the NMI 462 * handler seeing STARTED but not having a valid sample. 463 */ 464 clear_bit(IBS_STARTED, pcpu->state); 465 WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED); 466 hwc->state |= PERF_HES_STOPPED; 467 } 468 469 if (hwc->state & PERF_HES_UPTODATE) 470 return; 471 472 /* 473 * Clear valid bit to not count rollovers on update, rollovers 474 * are only updated in the irq handler. 475 */ 476 config &= ~perf_ibs->valid_mask; 477 478 perf_ibs_event_update(perf_ibs, event, &config); 479 hwc->state |= PERF_HES_UPTODATE; 480 } 481 482 static int perf_ibs_add(struct perf_event *event, int flags) 483 { 484 struct perf_ibs *perf_ibs = container_of(event->pmu, struct perf_ibs, pmu); 485 struct cpu_perf_ibs *pcpu = this_cpu_ptr(perf_ibs->pcpu); 486 487 if (test_and_set_bit(IBS_ENABLED, pcpu->state)) 488 return -ENOSPC; 489 490 event->hw.state = PERF_HES_UPTODATE | PERF_HES_STOPPED; 491 492 pcpu->event = event; 493 494 if (flags & PERF_EF_START) 495 perf_ibs_start(event, PERF_EF_RELOAD); 496 497 return 0; 498 } 499 500 static void perf_ibs_del(struct perf_event *event, int flags) 501 { 502 struct perf_ibs *perf_ibs = container_of(event->pmu, struct perf_ibs, pmu); 503 struct cpu_perf_ibs *pcpu = this_cpu_ptr(perf_ibs->pcpu); 504 505 if (!test_and_clear_bit(IBS_ENABLED, pcpu->state)) 506 return; 507 508 perf_ibs_stop(event, PERF_EF_UPDATE); 509 510 pcpu->event = NULL; 511 512 perf_event_update_userpage(event); 513 } 514 515 static void perf_ibs_read(struct perf_event *event) { } 516 517 /* 518 * We need to initialize with empty group if all attributes in the 519 * group are dynamic. 520 */ 521 static struct attribute *attrs_empty[] = { 522 NULL, 523 }; 524 525 static struct attribute_group empty_format_group = { 526 .name = "format", 527 .attrs = attrs_empty, 528 }; 529 530 static struct attribute_group empty_caps_group = { 531 .name = "caps", 532 .attrs = attrs_empty, 533 }; 534 535 static const struct attribute_group *empty_attr_groups[] = { 536 &empty_format_group, 537 &empty_caps_group, 538 NULL, 539 }; 540 541 PMU_FORMAT_ATTR(rand_en, "config:57"); 542 PMU_FORMAT_ATTR(cnt_ctl, "config:19"); 543 PMU_EVENT_ATTR_STRING(l3missonly, fetch_l3missonly, "config:59"); 544 PMU_EVENT_ATTR_STRING(l3missonly, op_l3missonly, "config:16"); 545 PMU_EVENT_ATTR_STRING(zen4_ibs_extensions, zen4_ibs_extensions, "1"); 546 547 static umode_t 548 zen4_ibs_extensions_is_visible(struct kobject *kobj, struct attribute *attr, int i) 549 { 550 return ibs_caps & IBS_CAPS_ZEN4 ? attr->mode : 0; 551 } 552 553 static struct attribute *rand_en_attrs[] = { 554 &format_attr_rand_en.attr, 555 NULL, 556 }; 557 558 static struct attribute *fetch_l3missonly_attrs[] = { 559 &fetch_l3missonly.attr.attr, 560 NULL, 561 }; 562 563 static struct attribute *zen4_ibs_extensions_attrs[] = { 564 &zen4_ibs_extensions.attr.attr, 565 NULL, 566 }; 567 568 static struct attribute_group group_rand_en = { 569 .name = "format", 570 .attrs = rand_en_attrs, 571 }; 572 573 static struct attribute_group group_fetch_l3missonly = { 574 .name = "format", 575 .attrs = fetch_l3missonly_attrs, 576 .is_visible = zen4_ibs_extensions_is_visible, 577 }; 578 579 static struct attribute_group group_zen4_ibs_extensions = { 580 .name = "caps", 581 .attrs = zen4_ibs_extensions_attrs, 582 .is_visible = zen4_ibs_extensions_is_visible, 583 }; 584 585 static const struct attribute_group *fetch_attr_groups[] = { 586 &group_rand_en, 587 &empty_caps_group, 588 NULL, 589 }; 590 591 static const struct attribute_group *fetch_attr_update[] = { 592 &group_fetch_l3missonly, 593 &group_zen4_ibs_extensions, 594 NULL, 595 }; 596 597 static umode_t 598 cnt_ctl_is_visible(struct kobject *kobj, struct attribute *attr, int i) 599 { 600 return ibs_caps & IBS_CAPS_OPCNT ? attr->mode : 0; 601 } 602 603 static struct attribute *cnt_ctl_attrs[] = { 604 &format_attr_cnt_ctl.attr, 605 NULL, 606 }; 607 608 static struct attribute *op_l3missonly_attrs[] = { 609 &op_l3missonly.attr.attr, 610 NULL, 611 }; 612 613 static struct attribute_group group_cnt_ctl = { 614 .name = "format", 615 .attrs = cnt_ctl_attrs, 616 .is_visible = cnt_ctl_is_visible, 617 }; 618 619 static struct attribute_group group_op_l3missonly = { 620 .name = "format", 621 .attrs = op_l3missonly_attrs, 622 .is_visible = zen4_ibs_extensions_is_visible, 623 }; 624 625 static const struct attribute_group *op_attr_update[] = { 626 &group_cnt_ctl, 627 &group_op_l3missonly, 628 &group_zen4_ibs_extensions, 629 NULL, 630 }; 631 632 static struct perf_ibs perf_ibs_fetch = { 633 .pmu = { 634 .task_ctx_nr = perf_invalid_context, 635 636 .event_init = perf_ibs_init, 637 .add = perf_ibs_add, 638 .del = perf_ibs_del, 639 .start = perf_ibs_start, 640 .stop = perf_ibs_stop, 641 .read = perf_ibs_read, 642 .capabilities = PERF_PMU_CAP_NO_EXCLUDE, 643 }, 644 .msr = MSR_AMD64_IBSFETCHCTL, 645 .config_mask = IBS_FETCH_CONFIG_MASK, 646 .cnt_mask = IBS_FETCH_MAX_CNT, 647 .enable_mask = IBS_FETCH_ENABLE, 648 .valid_mask = IBS_FETCH_VAL, 649 .max_period = IBS_FETCH_MAX_CNT << 4, 650 .offset_mask = { MSR_AMD64_IBSFETCH_REG_MASK }, 651 .offset_max = MSR_AMD64_IBSFETCH_REG_COUNT, 652 653 .get_count = get_ibs_fetch_count, 654 }; 655 656 static struct perf_ibs perf_ibs_op = { 657 .pmu = { 658 .task_ctx_nr = perf_invalid_context, 659 660 .event_init = perf_ibs_init, 661 .add = perf_ibs_add, 662 .del = perf_ibs_del, 663 .start = perf_ibs_start, 664 .stop = perf_ibs_stop, 665 .read = perf_ibs_read, 666 .capabilities = PERF_PMU_CAP_NO_EXCLUDE, 667 }, 668 .msr = MSR_AMD64_IBSOPCTL, 669 .config_mask = IBS_OP_CONFIG_MASK, 670 .cnt_mask = IBS_OP_MAX_CNT | IBS_OP_CUR_CNT | 671 IBS_OP_CUR_CNT_RAND, 672 .enable_mask = IBS_OP_ENABLE, 673 .valid_mask = IBS_OP_VAL, 674 .max_period = IBS_OP_MAX_CNT << 4, 675 .offset_mask = { MSR_AMD64_IBSOP_REG_MASK }, 676 .offset_max = MSR_AMD64_IBSOP_REG_COUNT, 677 678 .get_count = get_ibs_op_count, 679 }; 680 681 static void perf_ibs_get_mem_op(union ibs_op_data3 *op_data3, 682 struct perf_sample_data *data) 683 { 684 union perf_mem_data_src *data_src = &data->data_src; 685 686 data_src->mem_op = PERF_MEM_OP_NA; 687 688 if (op_data3->ld_op) 689 data_src->mem_op = PERF_MEM_OP_LOAD; 690 else if (op_data3->st_op) 691 data_src->mem_op = PERF_MEM_OP_STORE; 692 } 693 694 /* 695 * Processors having CPUID_Fn8000001B_EAX[11] aka IBS_CAPS_ZEN4 has 696 * more fine granular DataSrc encodings. Others have coarse. 697 */ 698 static u8 perf_ibs_data_src(union ibs_op_data2 *op_data2) 699 { 700 if (ibs_caps & IBS_CAPS_ZEN4) 701 return (op_data2->data_src_hi << 3) | op_data2->data_src_lo; 702 703 return op_data2->data_src_lo; 704 } 705 706 static void perf_ibs_get_mem_lvl(union ibs_op_data2 *op_data2, 707 union ibs_op_data3 *op_data3, 708 struct perf_sample_data *data) 709 { 710 union perf_mem_data_src *data_src = &data->data_src; 711 u8 ibs_data_src = perf_ibs_data_src(op_data2); 712 713 data_src->mem_lvl = 0; 714 715 /* 716 * DcMiss, L2Miss, DataSrc, DcMissLat etc. are all invalid for Uncached 717 * memory accesses. So, check DcUcMemAcc bit early. 718 */ 719 if (op_data3->dc_uc_mem_acc && ibs_data_src != IBS_DATA_SRC_EXT_IO) { 720 data_src->mem_lvl = PERF_MEM_LVL_UNC | PERF_MEM_LVL_HIT; 721 return; 722 } 723 724 /* L1 Hit */ 725 if (op_data3->dc_miss == 0) { 726 data_src->mem_lvl = PERF_MEM_LVL_L1 | PERF_MEM_LVL_HIT; 727 return; 728 } 729 730 /* L2 Hit */ 731 if (op_data3->l2_miss == 0) { 732 /* Erratum #1293 */ 733 if (boot_cpu_data.x86 != 0x19 || boot_cpu_data.x86_model > 0xF || 734 !(op_data3->sw_pf || op_data3->dc_miss_no_mab_alloc)) { 735 data_src->mem_lvl = PERF_MEM_LVL_L2 | PERF_MEM_LVL_HIT; 736 return; 737 } 738 } 739 740 /* 741 * OP_DATA2 is valid only for load ops. Skip all checks which 742 * uses OP_DATA2[DataSrc]. 743 */ 744 if (data_src->mem_op != PERF_MEM_OP_LOAD) 745 goto check_mab; 746 747 /* L3 Hit */ 748 if (ibs_caps & IBS_CAPS_ZEN4) { 749 if (ibs_data_src == IBS_DATA_SRC_EXT_LOC_CACHE) { 750 data_src->mem_lvl = PERF_MEM_LVL_L3 | PERF_MEM_LVL_HIT; 751 return; 752 } 753 } else { 754 if (ibs_data_src == IBS_DATA_SRC_LOC_CACHE) { 755 data_src->mem_lvl = PERF_MEM_LVL_L3 | PERF_MEM_LVL_REM_CCE1 | 756 PERF_MEM_LVL_HIT; 757 return; 758 } 759 } 760 761 /* A peer cache in a near CCX */ 762 if (ibs_caps & IBS_CAPS_ZEN4 && 763 ibs_data_src == IBS_DATA_SRC_EXT_NEAR_CCX_CACHE) { 764 data_src->mem_lvl = PERF_MEM_LVL_REM_CCE1 | PERF_MEM_LVL_HIT; 765 return; 766 } 767 768 /* A peer cache in a far CCX */ 769 if (ibs_caps & IBS_CAPS_ZEN4) { 770 if (ibs_data_src == IBS_DATA_SRC_EXT_FAR_CCX_CACHE) { 771 data_src->mem_lvl = PERF_MEM_LVL_REM_CCE2 | PERF_MEM_LVL_HIT; 772 return; 773 } 774 } else { 775 if (ibs_data_src == IBS_DATA_SRC_REM_CACHE) { 776 data_src->mem_lvl = PERF_MEM_LVL_REM_CCE2 | PERF_MEM_LVL_HIT; 777 return; 778 } 779 } 780 781 /* DRAM */ 782 if (ibs_data_src == IBS_DATA_SRC_EXT_DRAM) { 783 if (op_data2->rmt_node == 0) 784 data_src->mem_lvl = PERF_MEM_LVL_LOC_RAM | PERF_MEM_LVL_HIT; 785 else 786 data_src->mem_lvl = PERF_MEM_LVL_REM_RAM1 | PERF_MEM_LVL_HIT; 787 return; 788 } 789 790 /* PMEM */ 791 if (ibs_caps & IBS_CAPS_ZEN4 && ibs_data_src == IBS_DATA_SRC_EXT_PMEM) { 792 data_src->mem_lvl_num = PERF_MEM_LVLNUM_PMEM; 793 if (op_data2->rmt_node) { 794 data_src->mem_remote = PERF_MEM_REMOTE_REMOTE; 795 /* IBS doesn't provide Remote socket detail */ 796 data_src->mem_hops = PERF_MEM_HOPS_1; 797 } 798 return; 799 } 800 801 /* Extension Memory */ 802 if (ibs_caps & IBS_CAPS_ZEN4 && 803 ibs_data_src == IBS_DATA_SRC_EXT_EXT_MEM) { 804 data_src->mem_lvl_num = PERF_MEM_LVLNUM_CXL; 805 if (op_data2->rmt_node) { 806 data_src->mem_remote = PERF_MEM_REMOTE_REMOTE; 807 /* IBS doesn't provide Remote socket detail */ 808 data_src->mem_hops = PERF_MEM_HOPS_1; 809 } 810 return; 811 } 812 813 /* IO */ 814 if (ibs_data_src == IBS_DATA_SRC_EXT_IO) { 815 data_src->mem_lvl = PERF_MEM_LVL_IO; 816 data_src->mem_lvl_num = PERF_MEM_LVLNUM_IO; 817 if (op_data2->rmt_node) { 818 data_src->mem_remote = PERF_MEM_REMOTE_REMOTE; 819 /* IBS doesn't provide Remote socket detail */ 820 data_src->mem_hops = PERF_MEM_HOPS_1; 821 } 822 return; 823 } 824 825 check_mab: 826 /* 827 * MAB (Miss Address Buffer) Hit. MAB keeps track of outstanding 828 * DC misses. However, such data may come from any level in mem 829 * hierarchy. IBS provides detail about both MAB as well as actual 830 * DataSrc simultaneously. Prioritize DataSrc over MAB, i.e. set 831 * MAB only when IBS fails to provide DataSrc. 832 */ 833 if (op_data3->dc_miss_no_mab_alloc) { 834 data_src->mem_lvl = PERF_MEM_LVL_LFB | PERF_MEM_LVL_HIT; 835 return; 836 } 837 838 data_src->mem_lvl = PERF_MEM_LVL_NA; 839 } 840 841 static bool perf_ibs_cache_hit_st_valid(void) 842 { 843 /* 0: Uninitialized, 1: Valid, -1: Invalid */ 844 static int cache_hit_st_valid; 845 846 if (unlikely(!cache_hit_st_valid)) { 847 if (boot_cpu_data.x86 == 0x19 && 848 (boot_cpu_data.x86_model <= 0xF || 849 (boot_cpu_data.x86_model >= 0x20 && 850 boot_cpu_data.x86_model <= 0x5F))) { 851 cache_hit_st_valid = -1; 852 } else { 853 cache_hit_st_valid = 1; 854 } 855 } 856 857 return cache_hit_st_valid == 1; 858 } 859 860 static void perf_ibs_get_mem_snoop(union ibs_op_data2 *op_data2, 861 struct perf_sample_data *data) 862 { 863 union perf_mem_data_src *data_src = &data->data_src; 864 u8 ibs_data_src; 865 866 data_src->mem_snoop = PERF_MEM_SNOOP_NA; 867 868 if (!perf_ibs_cache_hit_st_valid() || 869 data_src->mem_op != PERF_MEM_OP_LOAD || 870 data_src->mem_lvl & PERF_MEM_LVL_L1 || 871 data_src->mem_lvl & PERF_MEM_LVL_L2 || 872 op_data2->cache_hit_st) 873 return; 874 875 ibs_data_src = perf_ibs_data_src(op_data2); 876 877 if (ibs_caps & IBS_CAPS_ZEN4) { 878 if (ibs_data_src == IBS_DATA_SRC_EXT_LOC_CACHE || 879 ibs_data_src == IBS_DATA_SRC_EXT_NEAR_CCX_CACHE || 880 ibs_data_src == IBS_DATA_SRC_EXT_FAR_CCX_CACHE) 881 data_src->mem_snoop = PERF_MEM_SNOOP_HITM; 882 } else if (ibs_data_src == IBS_DATA_SRC_LOC_CACHE) { 883 data_src->mem_snoop = PERF_MEM_SNOOP_HITM; 884 } 885 } 886 887 static void perf_ibs_get_tlb_lvl(union ibs_op_data3 *op_data3, 888 struct perf_sample_data *data) 889 { 890 union perf_mem_data_src *data_src = &data->data_src; 891 892 data_src->mem_dtlb = PERF_MEM_TLB_NA; 893 894 if (!op_data3->dc_lin_addr_valid) 895 return; 896 897 if (!op_data3->dc_l1tlb_miss) { 898 data_src->mem_dtlb = PERF_MEM_TLB_L1 | PERF_MEM_TLB_HIT; 899 return; 900 } 901 902 if (!op_data3->dc_l2tlb_miss) { 903 data_src->mem_dtlb = PERF_MEM_TLB_L2 | PERF_MEM_TLB_HIT; 904 return; 905 } 906 907 data_src->mem_dtlb = PERF_MEM_TLB_L2 | PERF_MEM_TLB_MISS; 908 } 909 910 static void perf_ibs_get_mem_lock(union ibs_op_data3 *op_data3, 911 struct perf_sample_data *data) 912 { 913 union perf_mem_data_src *data_src = &data->data_src; 914 915 data_src->mem_lock = PERF_MEM_LOCK_NA; 916 917 if (op_data3->dc_locked_op) 918 data_src->mem_lock = PERF_MEM_LOCK_LOCKED; 919 } 920 921 #define ibs_op_msr_idx(msr) (msr - MSR_AMD64_IBSOPCTL) 922 923 static void perf_ibs_get_data_src(struct perf_ibs_data *ibs_data, 924 struct perf_sample_data *data, 925 union ibs_op_data2 *op_data2, 926 union ibs_op_data3 *op_data3) 927 { 928 perf_ibs_get_mem_lvl(op_data2, op_data3, data); 929 perf_ibs_get_mem_snoop(op_data2, data); 930 perf_ibs_get_tlb_lvl(op_data3, data); 931 perf_ibs_get_mem_lock(op_data3, data); 932 } 933 934 static __u64 perf_ibs_get_op_data2(struct perf_ibs_data *ibs_data, 935 union ibs_op_data3 *op_data3) 936 { 937 __u64 val = ibs_data->regs[ibs_op_msr_idx(MSR_AMD64_IBSOPDATA2)]; 938 939 /* Erratum #1293 */ 940 if (boot_cpu_data.x86 == 0x19 && boot_cpu_data.x86_model <= 0xF && 941 (op_data3->sw_pf || op_data3->dc_miss_no_mab_alloc)) { 942 /* 943 * OP_DATA2 has only two fields on Zen3: DataSrc and RmtNode. 944 * DataSrc=0 is 'No valid status' and RmtNode is invalid when 945 * DataSrc=0. 946 */ 947 val = 0; 948 } 949 return val; 950 } 951 952 static void perf_ibs_parse_ld_st_data(__u64 sample_type, 953 struct perf_ibs_data *ibs_data, 954 struct perf_sample_data *data) 955 { 956 union ibs_op_data3 op_data3; 957 union ibs_op_data2 op_data2; 958 union ibs_op_data op_data; 959 960 data->data_src.val = PERF_MEM_NA; 961 op_data3.val = ibs_data->regs[ibs_op_msr_idx(MSR_AMD64_IBSOPDATA3)]; 962 963 perf_ibs_get_mem_op(&op_data3, data); 964 if (data->data_src.mem_op != PERF_MEM_OP_LOAD && 965 data->data_src.mem_op != PERF_MEM_OP_STORE) 966 return; 967 968 op_data2.val = perf_ibs_get_op_data2(ibs_data, &op_data3); 969 970 if (sample_type & PERF_SAMPLE_DATA_SRC) { 971 perf_ibs_get_data_src(ibs_data, data, &op_data2, &op_data3); 972 data->sample_flags |= PERF_SAMPLE_DATA_SRC; 973 } 974 975 if (sample_type & PERF_SAMPLE_WEIGHT_TYPE && op_data3.dc_miss && 976 data->data_src.mem_op == PERF_MEM_OP_LOAD) { 977 op_data.val = ibs_data->regs[ibs_op_msr_idx(MSR_AMD64_IBSOPDATA)]; 978 979 if (sample_type & PERF_SAMPLE_WEIGHT_STRUCT) { 980 data->weight.var1_dw = op_data3.dc_miss_lat; 981 data->weight.var2_w = op_data.tag_to_ret_ctr; 982 } else if (sample_type & PERF_SAMPLE_WEIGHT) { 983 data->weight.full = op_data3.dc_miss_lat; 984 } 985 data->sample_flags |= PERF_SAMPLE_WEIGHT_TYPE; 986 } 987 988 if (sample_type & PERF_SAMPLE_ADDR && op_data3.dc_lin_addr_valid) { 989 data->addr = ibs_data->regs[ibs_op_msr_idx(MSR_AMD64_IBSDCLINAD)]; 990 data->sample_flags |= PERF_SAMPLE_ADDR; 991 } 992 993 if (sample_type & PERF_SAMPLE_PHYS_ADDR && op_data3.dc_phy_addr_valid) { 994 data->phys_addr = ibs_data->regs[ibs_op_msr_idx(MSR_AMD64_IBSDCPHYSAD)]; 995 data->sample_flags |= PERF_SAMPLE_PHYS_ADDR; 996 } 997 } 998 999 static int perf_ibs_get_offset_max(struct perf_ibs *perf_ibs, u64 sample_type, 1000 int check_rip) 1001 { 1002 if (sample_type & PERF_SAMPLE_RAW || 1003 (perf_ibs == &perf_ibs_op && 1004 (sample_type & PERF_SAMPLE_DATA_SRC || 1005 sample_type & PERF_SAMPLE_WEIGHT_TYPE || 1006 sample_type & PERF_SAMPLE_ADDR || 1007 sample_type & PERF_SAMPLE_PHYS_ADDR))) 1008 return perf_ibs->offset_max; 1009 else if (check_rip) 1010 return 3; 1011 return 1; 1012 } 1013 1014 static int perf_ibs_handle_irq(struct perf_ibs *perf_ibs, struct pt_regs *iregs) 1015 { 1016 struct cpu_perf_ibs *pcpu = this_cpu_ptr(perf_ibs->pcpu); 1017 struct perf_event *event = pcpu->event; 1018 struct hw_perf_event *hwc; 1019 struct perf_sample_data data; 1020 struct perf_raw_record raw; 1021 struct pt_regs regs; 1022 struct perf_ibs_data ibs_data; 1023 int offset, size, check_rip, offset_max, throttle = 0; 1024 unsigned int msr; 1025 u64 *buf, *config, period, new_config = 0; 1026 1027 if (!test_bit(IBS_STARTED, pcpu->state)) { 1028 fail: 1029 /* 1030 * Catch spurious interrupts after stopping IBS: After 1031 * disabling IBS there could be still incoming NMIs 1032 * with samples that even have the valid bit cleared. 1033 * Mark all this NMIs as handled. 1034 */ 1035 if (test_and_clear_bit(IBS_STOPPED, pcpu->state)) 1036 return 1; 1037 1038 return 0; 1039 } 1040 1041 if (WARN_ON_ONCE(!event)) 1042 goto fail; 1043 1044 hwc = &event->hw; 1045 msr = hwc->config_base; 1046 buf = ibs_data.regs; 1047 rdmsrl(msr, *buf); 1048 if (!(*buf++ & perf_ibs->valid_mask)) 1049 goto fail; 1050 1051 config = &ibs_data.regs[0]; 1052 perf_ibs_event_update(perf_ibs, event, config); 1053 perf_sample_data_init(&data, 0, hwc->last_period); 1054 if (!perf_ibs_set_period(perf_ibs, hwc, &period)) 1055 goto out; /* no sw counter overflow */ 1056 1057 ibs_data.caps = ibs_caps; 1058 size = 1; 1059 offset = 1; 1060 check_rip = (perf_ibs == &perf_ibs_op && (ibs_caps & IBS_CAPS_RIPINVALIDCHK)); 1061 1062 offset_max = perf_ibs_get_offset_max(perf_ibs, event->attr.sample_type, check_rip); 1063 1064 do { 1065 rdmsrl(msr + offset, *buf++); 1066 size++; 1067 offset = find_next_bit(perf_ibs->offset_mask, 1068 perf_ibs->offset_max, 1069 offset + 1); 1070 } while (offset < offset_max); 1071 /* 1072 * Read IbsBrTarget, IbsOpData4, and IbsExtdCtl separately 1073 * depending on their availability. 1074 * Can't add to offset_max as they are staggered 1075 */ 1076 if (event->attr.sample_type & PERF_SAMPLE_RAW) { 1077 if (perf_ibs == &perf_ibs_op) { 1078 if (ibs_caps & IBS_CAPS_BRNTRGT) { 1079 rdmsrl(MSR_AMD64_IBSBRTARGET, *buf++); 1080 size++; 1081 } 1082 if (ibs_caps & IBS_CAPS_OPDATA4) { 1083 rdmsrl(MSR_AMD64_IBSOPDATA4, *buf++); 1084 size++; 1085 } 1086 } 1087 if (perf_ibs == &perf_ibs_fetch && (ibs_caps & IBS_CAPS_FETCHCTLEXTD)) { 1088 rdmsrl(MSR_AMD64_ICIBSEXTDCTL, *buf++); 1089 size++; 1090 } 1091 } 1092 ibs_data.size = sizeof(u64) * size; 1093 1094 regs = *iregs; 1095 if (check_rip && (ibs_data.regs[2] & IBS_RIP_INVALID)) { 1096 regs.flags &= ~PERF_EFLAGS_EXACT; 1097 } else { 1098 /* Workaround for erratum #1197 */ 1099 if (perf_ibs->fetch_ignore_if_zero_rip && !(ibs_data.regs[1])) 1100 goto out; 1101 1102 set_linear_ip(®s, ibs_data.regs[1]); 1103 regs.flags |= PERF_EFLAGS_EXACT; 1104 } 1105 1106 if (event->attr.sample_type & PERF_SAMPLE_RAW) { 1107 raw = (struct perf_raw_record){ 1108 .frag = { 1109 .size = sizeof(u32) + ibs_data.size, 1110 .data = ibs_data.data, 1111 }, 1112 }; 1113 data.raw = &raw; 1114 data.sample_flags |= PERF_SAMPLE_RAW; 1115 } 1116 1117 if (perf_ibs == &perf_ibs_op) 1118 perf_ibs_parse_ld_st_data(event->attr.sample_type, &ibs_data, &data); 1119 1120 /* 1121 * rip recorded by IbsOpRip will not be consistent with rsp and rbp 1122 * recorded as part of interrupt regs. Thus we need to use rip from 1123 * interrupt regs while unwinding call stack. 1124 */ 1125 if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) { 1126 data.callchain = perf_callchain(event, iregs); 1127 data.sample_flags |= PERF_SAMPLE_CALLCHAIN; 1128 } 1129 1130 throttle = perf_event_overflow(event, &data, ®s); 1131 out: 1132 if (throttle) { 1133 perf_ibs_stop(event, 0); 1134 } else { 1135 if (perf_ibs == &perf_ibs_op) { 1136 if (ibs_caps & IBS_CAPS_OPCNTEXT) { 1137 new_config = period & IBS_OP_MAX_CNT_EXT_MASK; 1138 period &= ~IBS_OP_MAX_CNT_EXT_MASK; 1139 } 1140 if ((ibs_caps & IBS_CAPS_RDWROPCNT) && (*config & IBS_OP_CNT_CTL)) 1141 new_config |= *config & IBS_OP_CUR_CNT_RAND; 1142 } 1143 new_config |= period >> 4; 1144 1145 perf_ibs_enable_event(perf_ibs, hwc, new_config); 1146 } 1147 1148 perf_event_update_userpage(event); 1149 1150 return 1; 1151 } 1152 1153 static int 1154 perf_ibs_nmi_handler(unsigned int cmd, struct pt_regs *regs) 1155 { 1156 u64 stamp = sched_clock(); 1157 int handled = 0; 1158 1159 handled += perf_ibs_handle_irq(&perf_ibs_fetch, regs); 1160 handled += perf_ibs_handle_irq(&perf_ibs_op, regs); 1161 1162 if (handled) 1163 inc_irq_stat(apic_perf_irqs); 1164 1165 perf_sample_event_took(sched_clock() - stamp); 1166 1167 return handled; 1168 } 1169 NOKPROBE_SYMBOL(perf_ibs_nmi_handler); 1170 1171 static __init int perf_ibs_pmu_init(struct perf_ibs *perf_ibs, char *name) 1172 { 1173 struct cpu_perf_ibs __percpu *pcpu; 1174 int ret; 1175 1176 pcpu = alloc_percpu(struct cpu_perf_ibs); 1177 if (!pcpu) 1178 return -ENOMEM; 1179 1180 perf_ibs->pcpu = pcpu; 1181 1182 ret = perf_pmu_register(&perf_ibs->pmu, name, -1); 1183 if (ret) { 1184 perf_ibs->pcpu = NULL; 1185 free_percpu(pcpu); 1186 } 1187 1188 return ret; 1189 } 1190 1191 static __init int perf_ibs_fetch_init(void) 1192 { 1193 /* 1194 * Some chips fail to reset the fetch count when it is written; instead 1195 * they need a 0-1 transition of IbsFetchEn. 1196 */ 1197 if (boot_cpu_data.x86 >= 0x16 && boot_cpu_data.x86 <= 0x18) 1198 perf_ibs_fetch.fetch_count_reset_broken = 1; 1199 1200 if (boot_cpu_data.x86 == 0x19 && boot_cpu_data.x86_model < 0x10) 1201 perf_ibs_fetch.fetch_ignore_if_zero_rip = 1; 1202 1203 if (ibs_caps & IBS_CAPS_ZEN4) 1204 perf_ibs_fetch.config_mask |= IBS_FETCH_L3MISSONLY; 1205 1206 perf_ibs_fetch.pmu.attr_groups = fetch_attr_groups; 1207 perf_ibs_fetch.pmu.attr_update = fetch_attr_update; 1208 1209 return perf_ibs_pmu_init(&perf_ibs_fetch, "ibs_fetch"); 1210 } 1211 1212 static __init int perf_ibs_op_init(void) 1213 { 1214 if (ibs_caps & IBS_CAPS_OPCNT) 1215 perf_ibs_op.config_mask |= IBS_OP_CNT_CTL; 1216 1217 if (ibs_caps & IBS_CAPS_OPCNTEXT) { 1218 perf_ibs_op.max_period |= IBS_OP_MAX_CNT_EXT_MASK; 1219 perf_ibs_op.config_mask |= IBS_OP_MAX_CNT_EXT_MASK; 1220 perf_ibs_op.cnt_mask |= IBS_OP_MAX_CNT_EXT_MASK; 1221 } 1222 1223 if (ibs_caps & IBS_CAPS_ZEN4) 1224 perf_ibs_op.config_mask |= IBS_OP_L3MISSONLY; 1225 1226 perf_ibs_op.pmu.attr_groups = empty_attr_groups; 1227 perf_ibs_op.pmu.attr_update = op_attr_update; 1228 1229 return perf_ibs_pmu_init(&perf_ibs_op, "ibs_op"); 1230 } 1231 1232 static __init int perf_event_ibs_init(void) 1233 { 1234 int ret; 1235 1236 ret = perf_ibs_fetch_init(); 1237 if (ret) 1238 return ret; 1239 1240 ret = perf_ibs_op_init(); 1241 if (ret) 1242 goto err_op; 1243 1244 ret = register_nmi_handler(NMI_LOCAL, perf_ibs_nmi_handler, 0, "perf_ibs"); 1245 if (ret) 1246 goto err_nmi; 1247 1248 pr_info("perf: AMD IBS detected (0x%08x)\n", ibs_caps); 1249 return 0; 1250 1251 err_nmi: 1252 perf_pmu_unregister(&perf_ibs_op.pmu); 1253 free_percpu(perf_ibs_op.pcpu); 1254 perf_ibs_op.pcpu = NULL; 1255 err_op: 1256 perf_pmu_unregister(&perf_ibs_fetch.pmu); 1257 free_percpu(perf_ibs_fetch.pcpu); 1258 perf_ibs_fetch.pcpu = NULL; 1259 1260 return ret; 1261 } 1262 1263 #else /* defined(CONFIG_PERF_EVENTS) && defined(CONFIG_CPU_SUP_AMD) */ 1264 1265 static __init int perf_event_ibs_init(void) 1266 { 1267 return 0; 1268 } 1269 1270 #endif 1271 1272 /* IBS - apic initialization, for perf and oprofile */ 1273 1274 static __init u32 __get_ibs_caps(void) 1275 { 1276 u32 caps; 1277 unsigned int max_level; 1278 1279 if (!boot_cpu_has(X86_FEATURE_IBS)) 1280 return 0; 1281 1282 /* check IBS cpuid feature flags */ 1283 max_level = cpuid_eax(0x80000000); 1284 if (max_level < IBS_CPUID_FEATURES) 1285 return IBS_CAPS_DEFAULT; 1286 1287 caps = cpuid_eax(IBS_CPUID_FEATURES); 1288 if (!(caps & IBS_CAPS_AVAIL)) 1289 /* cpuid flags not valid */ 1290 return IBS_CAPS_DEFAULT; 1291 1292 return caps; 1293 } 1294 1295 u32 get_ibs_caps(void) 1296 { 1297 return ibs_caps; 1298 } 1299 1300 EXPORT_SYMBOL(get_ibs_caps); 1301 1302 static inline int get_eilvt(int offset) 1303 { 1304 return !setup_APIC_eilvt(offset, 0, APIC_EILVT_MSG_NMI, 1); 1305 } 1306 1307 static inline int put_eilvt(int offset) 1308 { 1309 return !setup_APIC_eilvt(offset, 0, 0, 1); 1310 } 1311 1312 /* 1313 * Check and reserve APIC extended interrupt LVT offset for IBS if available. 1314 */ 1315 static inline int ibs_eilvt_valid(void) 1316 { 1317 int offset; 1318 u64 val; 1319 int valid = 0; 1320 1321 preempt_disable(); 1322 1323 rdmsrl(MSR_AMD64_IBSCTL, val); 1324 offset = val & IBSCTL_LVT_OFFSET_MASK; 1325 1326 if (!(val & IBSCTL_LVT_OFFSET_VALID)) { 1327 pr_err(FW_BUG "cpu %d, invalid IBS interrupt offset %d (MSR%08X=0x%016llx)\n", 1328 smp_processor_id(), offset, MSR_AMD64_IBSCTL, val); 1329 goto out; 1330 } 1331 1332 if (!get_eilvt(offset)) { 1333 pr_err(FW_BUG "cpu %d, IBS interrupt offset %d not available (MSR%08X=0x%016llx)\n", 1334 smp_processor_id(), offset, MSR_AMD64_IBSCTL, val); 1335 goto out; 1336 } 1337 1338 valid = 1; 1339 out: 1340 preempt_enable(); 1341 1342 return valid; 1343 } 1344 1345 static int setup_ibs_ctl(int ibs_eilvt_off) 1346 { 1347 struct pci_dev *cpu_cfg; 1348 int nodes; 1349 u32 value = 0; 1350 1351 nodes = 0; 1352 cpu_cfg = NULL; 1353 do { 1354 cpu_cfg = pci_get_device(PCI_VENDOR_ID_AMD, 1355 PCI_DEVICE_ID_AMD_10H_NB_MISC, 1356 cpu_cfg); 1357 if (!cpu_cfg) 1358 break; 1359 ++nodes; 1360 pci_write_config_dword(cpu_cfg, IBSCTL, ibs_eilvt_off 1361 | IBSCTL_LVT_OFFSET_VALID); 1362 pci_read_config_dword(cpu_cfg, IBSCTL, &value); 1363 if (value != (ibs_eilvt_off | IBSCTL_LVT_OFFSET_VALID)) { 1364 pci_dev_put(cpu_cfg); 1365 pr_debug("Failed to setup IBS LVT offset, IBSCTL = 0x%08x\n", 1366 value); 1367 return -EINVAL; 1368 } 1369 } while (1); 1370 1371 if (!nodes) { 1372 pr_debug("No CPU node configured for IBS\n"); 1373 return -ENODEV; 1374 } 1375 1376 return 0; 1377 } 1378 1379 /* 1380 * This runs only on the current cpu. We try to find an LVT offset and 1381 * setup the local APIC. For this we must disable preemption. On 1382 * success we initialize all nodes with this offset. This updates then 1383 * the offset in the IBS_CTL per-node msr. The per-core APIC setup of 1384 * the IBS interrupt vector is handled by perf_ibs_cpu_notifier that 1385 * is using the new offset. 1386 */ 1387 static void force_ibs_eilvt_setup(void) 1388 { 1389 int offset; 1390 int ret; 1391 1392 preempt_disable(); 1393 /* find the next free available EILVT entry, skip offset 0 */ 1394 for (offset = 1; offset < APIC_EILVT_NR_MAX; offset++) { 1395 if (get_eilvt(offset)) 1396 break; 1397 } 1398 preempt_enable(); 1399 1400 if (offset == APIC_EILVT_NR_MAX) { 1401 pr_debug("No EILVT entry available\n"); 1402 return; 1403 } 1404 1405 ret = setup_ibs_ctl(offset); 1406 if (ret) 1407 goto out; 1408 1409 if (!ibs_eilvt_valid()) 1410 goto out; 1411 1412 pr_info("LVT offset %d assigned\n", offset); 1413 1414 return; 1415 out: 1416 preempt_disable(); 1417 put_eilvt(offset); 1418 preempt_enable(); 1419 return; 1420 } 1421 1422 static void ibs_eilvt_setup(void) 1423 { 1424 /* 1425 * Force LVT offset assignment for family 10h: The offsets are 1426 * not assigned by the BIOS for this family, so the OS is 1427 * responsible for doing it. If the OS assignment fails, fall 1428 * back to BIOS settings and try to setup this. 1429 */ 1430 if (boot_cpu_data.x86 == 0x10) 1431 force_ibs_eilvt_setup(); 1432 } 1433 1434 static inline int get_ibs_lvt_offset(void) 1435 { 1436 u64 val; 1437 1438 rdmsrl(MSR_AMD64_IBSCTL, val); 1439 if (!(val & IBSCTL_LVT_OFFSET_VALID)) 1440 return -EINVAL; 1441 1442 return val & IBSCTL_LVT_OFFSET_MASK; 1443 } 1444 1445 static void setup_APIC_ibs(void) 1446 { 1447 int offset; 1448 1449 offset = get_ibs_lvt_offset(); 1450 if (offset < 0) 1451 goto failed; 1452 1453 if (!setup_APIC_eilvt(offset, 0, APIC_EILVT_MSG_NMI, 0)) 1454 return; 1455 failed: 1456 pr_warn("perf: IBS APIC setup failed on cpu #%d\n", 1457 smp_processor_id()); 1458 } 1459 1460 static void clear_APIC_ibs(void) 1461 { 1462 int offset; 1463 1464 offset = get_ibs_lvt_offset(); 1465 if (offset >= 0) 1466 setup_APIC_eilvt(offset, 0, APIC_EILVT_MSG_FIX, 1); 1467 } 1468 1469 static int x86_pmu_amd_ibs_starting_cpu(unsigned int cpu) 1470 { 1471 setup_APIC_ibs(); 1472 return 0; 1473 } 1474 1475 #ifdef CONFIG_PM 1476 1477 static int perf_ibs_suspend(void) 1478 { 1479 clear_APIC_ibs(); 1480 return 0; 1481 } 1482 1483 static void perf_ibs_resume(void) 1484 { 1485 ibs_eilvt_setup(); 1486 setup_APIC_ibs(); 1487 } 1488 1489 static struct syscore_ops perf_ibs_syscore_ops = { 1490 .resume = perf_ibs_resume, 1491 .suspend = perf_ibs_suspend, 1492 }; 1493 1494 static void perf_ibs_pm_init(void) 1495 { 1496 register_syscore_ops(&perf_ibs_syscore_ops); 1497 } 1498 1499 #else 1500 1501 static inline void perf_ibs_pm_init(void) { } 1502 1503 #endif 1504 1505 static int x86_pmu_amd_ibs_dying_cpu(unsigned int cpu) 1506 { 1507 clear_APIC_ibs(); 1508 return 0; 1509 } 1510 1511 static __init int amd_ibs_init(void) 1512 { 1513 u32 caps; 1514 1515 caps = __get_ibs_caps(); 1516 if (!caps) 1517 return -ENODEV; /* ibs not supported by the cpu */ 1518 1519 ibs_eilvt_setup(); 1520 1521 if (!ibs_eilvt_valid()) 1522 return -EINVAL; 1523 1524 perf_ibs_pm_init(); 1525 1526 ibs_caps = caps; 1527 /* make ibs_caps visible to other cpus: */ 1528 smp_mb(); 1529 /* 1530 * x86_pmu_amd_ibs_starting_cpu will be called from core on 1531 * all online cpus. 1532 */ 1533 cpuhp_setup_state(CPUHP_AP_PERF_X86_AMD_IBS_STARTING, 1534 "perf/x86/amd/ibs:starting", 1535 x86_pmu_amd_ibs_starting_cpu, 1536 x86_pmu_amd_ibs_dying_cpu); 1537 1538 return perf_event_ibs_init(); 1539 } 1540 1541 /* Since we need the pci subsystem to init ibs we can't do this earlier: */ 1542 device_initcall(amd_ibs_init); 1543