1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Support Intel uncore PerfMon discovery mechanism. 4 * Copyright(c) 2021 Intel Corporation. 5 */ 6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 7 8 #include "uncore.h" 9 #include "uncore_discovery.h" 10 11 static struct rb_root discovery_tables = RB_ROOT; 12 static int num_discovered_types[UNCORE_ACCESS_MAX]; 13 14 static bool has_generic_discovery_table(void) 15 { 16 struct pci_dev *dev; 17 int dvsec; 18 19 dev = pci_get_device(PCI_VENDOR_ID_INTEL, UNCORE_DISCOVERY_TABLE_DEVICE, NULL); 20 if (!dev) 21 return false; 22 23 /* A discovery table device has the unique capability ID. */ 24 dvsec = pci_find_next_ext_capability(dev, 0, UNCORE_EXT_CAP_ID_DISCOVERY); 25 pci_dev_put(dev); 26 if (dvsec) 27 return true; 28 29 return false; 30 } 31 32 static int logical_die_id; 33 34 static int get_device_die_id(struct pci_dev *dev) 35 { 36 int node = pcibus_to_node(dev->bus); 37 38 /* 39 * If the NUMA info is not available, assume that the logical die id is 40 * continuous in the order in which the discovery table devices are 41 * detected. 42 */ 43 if (node < 0) 44 return logical_die_id++; 45 46 return uncore_device_to_die(dev); 47 } 48 49 #define __node_2_type(cur) \ 50 rb_entry((cur), struct intel_uncore_discovery_type, node) 51 52 static inline int __type_cmp(const void *key, const struct rb_node *b) 53 { 54 struct intel_uncore_discovery_type *type_b = __node_2_type(b); 55 const u16 *type_id = key; 56 57 if (type_b->type > *type_id) 58 return -1; 59 else if (type_b->type < *type_id) 60 return 1; 61 62 return 0; 63 } 64 65 static inline struct intel_uncore_discovery_type * 66 search_uncore_discovery_type(u16 type_id) 67 { 68 struct rb_node *node = rb_find(&type_id, &discovery_tables, __type_cmp); 69 70 return (node) ? __node_2_type(node) : NULL; 71 } 72 73 static inline bool __type_less(struct rb_node *a, const struct rb_node *b) 74 { 75 return (__node_2_type(a)->type < __node_2_type(b)->type); 76 } 77 78 static struct intel_uncore_discovery_type * 79 add_uncore_discovery_type(struct uncore_unit_discovery *unit) 80 { 81 struct intel_uncore_discovery_type *type; 82 83 if (unit->access_type >= UNCORE_ACCESS_MAX) { 84 pr_warn("Unsupported access type %d\n", unit->access_type); 85 return NULL; 86 } 87 88 type = kzalloc(sizeof(struct intel_uncore_discovery_type), GFP_KERNEL); 89 if (!type) 90 return NULL; 91 92 type->box_ctrl_die = kcalloc(__uncore_max_dies, sizeof(u64), GFP_KERNEL); 93 if (!type->box_ctrl_die) 94 goto free_type; 95 96 type->access_type = unit->access_type; 97 num_discovered_types[type->access_type]++; 98 type->type = unit->box_type; 99 100 rb_add(&type->node, &discovery_tables, __type_less); 101 102 return type; 103 104 free_type: 105 kfree(type); 106 107 return NULL; 108 109 } 110 111 static struct intel_uncore_discovery_type * 112 get_uncore_discovery_type(struct uncore_unit_discovery *unit) 113 { 114 struct intel_uncore_discovery_type *type; 115 116 type = search_uncore_discovery_type(unit->box_type); 117 if (type) 118 return type; 119 120 return add_uncore_discovery_type(unit); 121 } 122 123 static void 124 uncore_insert_box_info(struct uncore_unit_discovery *unit, 125 int die, bool parsed) 126 { 127 struct intel_uncore_discovery_type *type; 128 unsigned int *box_offset, *ids; 129 int i; 130 131 if (!unit->ctl || !unit->ctl_offset || !unit->ctr_offset) { 132 pr_info("Invalid address is detected for uncore type %d box %d, " 133 "Disable the uncore unit.\n", 134 unit->box_type, unit->box_id); 135 return; 136 } 137 138 if (parsed) { 139 type = search_uncore_discovery_type(unit->box_type); 140 if (!type) { 141 pr_info("A spurious uncore type %d is detected, " 142 "Disable the uncore type.\n", 143 unit->box_type); 144 return; 145 } 146 /* Store the first box of each die */ 147 if (!type->box_ctrl_die[die]) 148 type->box_ctrl_die[die] = unit->ctl; 149 return; 150 } 151 152 type = get_uncore_discovery_type(unit); 153 if (!type) 154 return; 155 156 box_offset = kcalloc(type->num_boxes + 1, sizeof(unsigned int), GFP_KERNEL); 157 if (!box_offset) 158 return; 159 160 ids = kcalloc(type->num_boxes + 1, sizeof(unsigned int), GFP_KERNEL); 161 if (!ids) 162 goto free_box_offset; 163 164 /* Store generic information for the first box */ 165 if (!type->num_boxes) { 166 type->box_ctrl = unit->ctl; 167 type->box_ctrl_die[die] = unit->ctl; 168 type->num_counters = unit->num_regs; 169 type->counter_width = unit->bit_width; 170 type->ctl_offset = unit->ctl_offset; 171 type->ctr_offset = unit->ctr_offset; 172 *ids = unit->box_id; 173 goto end; 174 } 175 176 for (i = 0; i < type->num_boxes; i++) { 177 ids[i] = type->ids[i]; 178 box_offset[i] = type->box_offset[i]; 179 180 if (unit->box_id == ids[i]) { 181 pr_info("Duplicate uncore type %d box ID %d is detected, " 182 "Drop the duplicate uncore unit.\n", 183 unit->box_type, unit->box_id); 184 goto free_ids; 185 } 186 } 187 ids[i] = unit->box_id; 188 box_offset[i] = unit->ctl - type->box_ctrl; 189 kfree(type->ids); 190 kfree(type->box_offset); 191 end: 192 type->ids = ids; 193 type->box_offset = box_offset; 194 type->num_boxes++; 195 return; 196 197 free_ids: 198 kfree(ids); 199 200 free_box_offset: 201 kfree(box_offset); 202 203 } 204 205 static bool 206 uncore_ignore_unit(struct uncore_unit_discovery *unit, int *ignore) 207 { 208 int i; 209 210 if (!ignore) 211 return false; 212 213 for (i = 0; ignore[i] != UNCORE_IGNORE_END ; i++) { 214 if (unit->box_type == ignore[i]) 215 return true; 216 } 217 218 return false; 219 } 220 221 static int parse_discovery_table(struct pci_dev *dev, int die, 222 u32 bar_offset, bool *parsed, 223 int *ignore) 224 { 225 struct uncore_global_discovery global; 226 struct uncore_unit_discovery unit; 227 void __iomem *io_addr; 228 resource_size_t addr; 229 unsigned long size; 230 u32 val; 231 int i; 232 233 pci_read_config_dword(dev, bar_offset, &val); 234 235 if (val & ~PCI_BASE_ADDRESS_MEM_MASK & ~PCI_BASE_ADDRESS_MEM_TYPE_64) 236 return -EINVAL; 237 238 addr = (resource_size_t)(val & PCI_BASE_ADDRESS_MEM_MASK); 239 #ifdef CONFIG_PHYS_ADDR_T_64BIT 240 if ((val & PCI_BASE_ADDRESS_MEM_TYPE_MASK) == PCI_BASE_ADDRESS_MEM_TYPE_64) { 241 u32 val2; 242 243 pci_read_config_dword(dev, bar_offset + 4, &val2); 244 addr |= ((resource_size_t)val2) << 32; 245 } 246 #endif 247 size = UNCORE_DISCOVERY_GLOBAL_MAP_SIZE; 248 io_addr = ioremap(addr, size); 249 if (!io_addr) 250 return -ENOMEM; 251 252 /* Read Global Discovery State */ 253 memcpy_fromio(&global, io_addr, sizeof(struct uncore_global_discovery)); 254 if (uncore_discovery_invalid_unit(global)) { 255 pr_info("Invalid Global Discovery State: 0x%llx 0x%llx 0x%llx\n", 256 global.table1, global.ctl, global.table3); 257 iounmap(io_addr); 258 return -EINVAL; 259 } 260 iounmap(io_addr); 261 262 size = (1 + global.max_units) * global.stride * 8; 263 io_addr = ioremap(addr, size); 264 if (!io_addr) 265 return -ENOMEM; 266 267 /* Parsing Unit Discovery State */ 268 for (i = 0; i < global.max_units; i++) { 269 memcpy_fromio(&unit, io_addr + (i + 1) * (global.stride * 8), 270 sizeof(struct uncore_unit_discovery)); 271 272 if (uncore_discovery_invalid_unit(unit)) 273 continue; 274 275 if (unit.access_type >= UNCORE_ACCESS_MAX) 276 continue; 277 278 if (uncore_ignore_unit(&unit, ignore)) 279 continue; 280 281 uncore_insert_box_info(&unit, die, *parsed); 282 } 283 284 *parsed = true; 285 iounmap(io_addr); 286 return 0; 287 } 288 289 bool intel_uncore_has_discovery_tables(int *ignore) 290 { 291 u32 device, val, entry_id, bar_offset; 292 int die, dvsec = 0, ret = true; 293 struct pci_dev *dev = NULL; 294 bool parsed = false; 295 296 if (has_generic_discovery_table()) 297 device = UNCORE_DISCOVERY_TABLE_DEVICE; 298 else 299 device = PCI_ANY_ID; 300 301 /* 302 * Start a new search and iterates through the list of 303 * the discovery table devices. 304 */ 305 while ((dev = pci_get_device(PCI_VENDOR_ID_INTEL, device, dev)) != NULL) { 306 while ((dvsec = pci_find_next_ext_capability(dev, dvsec, UNCORE_EXT_CAP_ID_DISCOVERY))) { 307 pci_read_config_dword(dev, dvsec + UNCORE_DISCOVERY_DVSEC_OFFSET, &val); 308 entry_id = val & UNCORE_DISCOVERY_DVSEC_ID_MASK; 309 if (entry_id != UNCORE_DISCOVERY_DVSEC_ID_PMON) 310 continue; 311 312 pci_read_config_dword(dev, dvsec + UNCORE_DISCOVERY_DVSEC2_OFFSET, &val); 313 314 if (val & ~UNCORE_DISCOVERY_DVSEC2_BIR_MASK) { 315 ret = false; 316 goto err; 317 } 318 bar_offset = UNCORE_DISCOVERY_BIR_BASE + 319 (val & UNCORE_DISCOVERY_DVSEC2_BIR_MASK) * UNCORE_DISCOVERY_BIR_STEP; 320 321 die = get_device_die_id(dev); 322 if (die < 0) 323 continue; 324 325 parse_discovery_table(dev, die, bar_offset, &parsed, ignore); 326 } 327 } 328 329 /* None of the discovery tables are available */ 330 if (!parsed) 331 ret = false; 332 err: 333 pci_dev_put(dev); 334 335 return ret; 336 } 337 338 void intel_uncore_clear_discovery_tables(void) 339 { 340 struct intel_uncore_discovery_type *type, *next; 341 342 rbtree_postorder_for_each_entry_safe(type, next, &discovery_tables, node) { 343 kfree(type->box_ctrl_die); 344 kfree(type); 345 } 346 } 347 348 DEFINE_UNCORE_FORMAT_ATTR(event, event, "config:0-7"); 349 DEFINE_UNCORE_FORMAT_ATTR(umask, umask, "config:8-15"); 350 DEFINE_UNCORE_FORMAT_ATTR(edge, edge, "config:18"); 351 DEFINE_UNCORE_FORMAT_ATTR(inv, inv, "config:23"); 352 DEFINE_UNCORE_FORMAT_ATTR(thresh, thresh, "config:24-31"); 353 354 static struct attribute *generic_uncore_formats_attr[] = { 355 &format_attr_event.attr, 356 &format_attr_umask.attr, 357 &format_attr_edge.attr, 358 &format_attr_inv.attr, 359 &format_attr_thresh.attr, 360 NULL, 361 }; 362 363 static const struct attribute_group generic_uncore_format_group = { 364 .name = "format", 365 .attrs = generic_uncore_formats_attr, 366 }; 367 368 void intel_generic_uncore_msr_init_box(struct intel_uncore_box *box) 369 { 370 wrmsrl(uncore_msr_box_ctl(box), GENERIC_PMON_BOX_CTL_INT); 371 } 372 373 void intel_generic_uncore_msr_disable_box(struct intel_uncore_box *box) 374 { 375 wrmsrl(uncore_msr_box_ctl(box), GENERIC_PMON_BOX_CTL_FRZ); 376 } 377 378 void intel_generic_uncore_msr_enable_box(struct intel_uncore_box *box) 379 { 380 wrmsrl(uncore_msr_box_ctl(box), 0); 381 } 382 383 static void intel_generic_uncore_msr_enable_event(struct intel_uncore_box *box, 384 struct perf_event *event) 385 { 386 struct hw_perf_event *hwc = &event->hw; 387 388 wrmsrl(hwc->config_base, hwc->config); 389 } 390 391 static void intel_generic_uncore_msr_disable_event(struct intel_uncore_box *box, 392 struct perf_event *event) 393 { 394 struct hw_perf_event *hwc = &event->hw; 395 396 wrmsrl(hwc->config_base, 0); 397 } 398 399 static struct intel_uncore_ops generic_uncore_msr_ops = { 400 .init_box = intel_generic_uncore_msr_init_box, 401 .disable_box = intel_generic_uncore_msr_disable_box, 402 .enable_box = intel_generic_uncore_msr_enable_box, 403 .disable_event = intel_generic_uncore_msr_disable_event, 404 .enable_event = intel_generic_uncore_msr_enable_event, 405 .read_counter = uncore_msr_read_counter, 406 }; 407 408 void intel_generic_uncore_pci_init_box(struct intel_uncore_box *box) 409 { 410 struct pci_dev *pdev = box->pci_dev; 411 int box_ctl = uncore_pci_box_ctl(box); 412 413 __set_bit(UNCORE_BOX_FLAG_CTL_OFFS8, &box->flags); 414 pci_write_config_dword(pdev, box_ctl, GENERIC_PMON_BOX_CTL_INT); 415 } 416 417 void intel_generic_uncore_pci_disable_box(struct intel_uncore_box *box) 418 { 419 struct pci_dev *pdev = box->pci_dev; 420 int box_ctl = uncore_pci_box_ctl(box); 421 422 pci_write_config_dword(pdev, box_ctl, GENERIC_PMON_BOX_CTL_FRZ); 423 } 424 425 void intel_generic_uncore_pci_enable_box(struct intel_uncore_box *box) 426 { 427 struct pci_dev *pdev = box->pci_dev; 428 int box_ctl = uncore_pci_box_ctl(box); 429 430 pci_write_config_dword(pdev, box_ctl, 0); 431 } 432 433 static void intel_generic_uncore_pci_enable_event(struct intel_uncore_box *box, 434 struct perf_event *event) 435 { 436 struct pci_dev *pdev = box->pci_dev; 437 struct hw_perf_event *hwc = &event->hw; 438 439 pci_write_config_dword(pdev, hwc->config_base, hwc->config); 440 } 441 442 void intel_generic_uncore_pci_disable_event(struct intel_uncore_box *box, 443 struct perf_event *event) 444 { 445 struct pci_dev *pdev = box->pci_dev; 446 struct hw_perf_event *hwc = &event->hw; 447 448 pci_write_config_dword(pdev, hwc->config_base, 0); 449 } 450 451 u64 intel_generic_uncore_pci_read_counter(struct intel_uncore_box *box, 452 struct perf_event *event) 453 { 454 struct pci_dev *pdev = box->pci_dev; 455 struct hw_perf_event *hwc = &event->hw; 456 u64 count = 0; 457 458 pci_read_config_dword(pdev, hwc->event_base, (u32 *)&count); 459 pci_read_config_dword(pdev, hwc->event_base + 4, (u32 *)&count + 1); 460 461 return count; 462 } 463 464 static struct intel_uncore_ops generic_uncore_pci_ops = { 465 .init_box = intel_generic_uncore_pci_init_box, 466 .disable_box = intel_generic_uncore_pci_disable_box, 467 .enable_box = intel_generic_uncore_pci_enable_box, 468 .disable_event = intel_generic_uncore_pci_disable_event, 469 .enable_event = intel_generic_uncore_pci_enable_event, 470 .read_counter = intel_generic_uncore_pci_read_counter, 471 }; 472 473 #define UNCORE_GENERIC_MMIO_SIZE 0x4000 474 475 static u64 generic_uncore_mmio_box_ctl(struct intel_uncore_box *box) 476 { 477 struct intel_uncore_type *type = box->pmu->type; 478 479 if (!type->box_ctls || !type->box_ctls[box->dieid] || !type->mmio_offsets) 480 return 0; 481 482 return type->box_ctls[box->dieid] + type->mmio_offsets[box->pmu->pmu_idx]; 483 } 484 485 void intel_generic_uncore_mmio_init_box(struct intel_uncore_box *box) 486 { 487 u64 box_ctl = generic_uncore_mmio_box_ctl(box); 488 struct intel_uncore_type *type = box->pmu->type; 489 resource_size_t addr; 490 491 if (!box_ctl) { 492 pr_warn("Uncore type %d box %d: Invalid box control address.\n", 493 type->type_id, type->box_ids[box->pmu->pmu_idx]); 494 return; 495 } 496 497 addr = box_ctl; 498 box->io_addr = ioremap(addr, UNCORE_GENERIC_MMIO_SIZE); 499 if (!box->io_addr) { 500 pr_warn("Uncore type %d box %d: ioremap error for 0x%llx.\n", 501 type->type_id, type->box_ids[box->pmu->pmu_idx], 502 (unsigned long long)addr); 503 return; 504 } 505 506 writel(GENERIC_PMON_BOX_CTL_INT, box->io_addr); 507 } 508 509 void intel_generic_uncore_mmio_disable_box(struct intel_uncore_box *box) 510 { 511 if (!box->io_addr) 512 return; 513 514 writel(GENERIC_PMON_BOX_CTL_FRZ, box->io_addr); 515 } 516 517 void intel_generic_uncore_mmio_enable_box(struct intel_uncore_box *box) 518 { 519 if (!box->io_addr) 520 return; 521 522 writel(0, box->io_addr); 523 } 524 525 void intel_generic_uncore_mmio_enable_event(struct intel_uncore_box *box, 526 struct perf_event *event) 527 { 528 struct hw_perf_event *hwc = &event->hw; 529 530 if (!box->io_addr) 531 return; 532 533 writel(hwc->config, box->io_addr + hwc->config_base); 534 } 535 536 void intel_generic_uncore_mmio_disable_event(struct intel_uncore_box *box, 537 struct perf_event *event) 538 { 539 struct hw_perf_event *hwc = &event->hw; 540 541 if (!box->io_addr) 542 return; 543 544 writel(0, box->io_addr + hwc->config_base); 545 } 546 547 static struct intel_uncore_ops generic_uncore_mmio_ops = { 548 .init_box = intel_generic_uncore_mmio_init_box, 549 .exit_box = uncore_mmio_exit_box, 550 .disable_box = intel_generic_uncore_mmio_disable_box, 551 .enable_box = intel_generic_uncore_mmio_enable_box, 552 .disable_event = intel_generic_uncore_mmio_disable_event, 553 .enable_event = intel_generic_uncore_mmio_enable_event, 554 .read_counter = uncore_mmio_read_counter, 555 }; 556 557 static bool uncore_update_uncore_type(enum uncore_access_type type_id, 558 struct intel_uncore_type *uncore, 559 struct intel_uncore_discovery_type *type) 560 { 561 uncore->type_id = type->type; 562 uncore->num_boxes = type->num_boxes; 563 uncore->num_counters = type->num_counters; 564 uncore->perf_ctr_bits = type->counter_width; 565 uncore->box_ids = type->ids; 566 567 switch (type_id) { 568 case UNCORE_ACCESS_MSR: 569 uncore->ops = &generic_uncore_msr_ops; 570 uncore->perf_ctr = (unsigned int)type->box_ctrl + type->ctr_offset; 571 uncore->event_ctl = (unsigned int)type->box_ctrl + type->ctl_offset; 572 uncore->box_ctl = (unsigned int)type->box_ctrl; 573 uncore->msr_offsets = type->box_offset; 574 break; 575 case UNCORE_ACCESS_PCI: 576 uncore->ops = &generic_uncore_pci_ops; 577 uncore->perf_ctr = (unsigned int)UNCORE_DISCOVERY_PCI_BOX_CTRL(type->box_ctrl) + type->ctr_offset; 578 uncore->event_ctl = (unsigned int)UNCORE_DISCOVERY_PCI_BOX_CTRL(type->box_ctrl) + type->ctl_offset; 579 uncore->box_ctl = (unsigned int)UNCORE_DISCOVERY_PCI_BOX_CTRL(type->box_ctrl); 580 uncore->box_ctls = type->box_ctrl_die; 581 uncore->pci_offsets = type->box_offset; 582 break; 583 case UNCORE_ACCESS_MMIO: 584 uncore->ops = &generic_uncore_mmio_ops; 585 uncore->perf_ctr = (unsigned int)type->ctr_offset; 586 uncore->event_ctl = (unsigned int)type->ctl_offset; 587 uncore->box_ctl = (unsigned int)type->box_ctrl; 588 uncore->box_ctls = type->box_ctrl_die; 589 uncore->mmio_offsets = type->box_offset; 590 uncore->mmio_map_size = UNCORE_GENERIC_MMIO_SIZE; 591 break; 592 default: 593 return false; 594 } 595 596 return true; 597 } 598 599 struct intel_uncore_type ** 600 intel_uncore_generic_init_uncores(enum uncore_access_type type_id, int num_extra) 601 { 602 struct intel_uncore_discovery_type *type; 603 struct intel_uncore_type **uncores; 604 struct intel_uncore_type *uncore; 605 struct rb_node *node; 606 int i = 0; 607 608 uncores = kcalloc(num_discovered_types[type_id] + num_extra + 1, 609 sizeof(struct intel_uncore_type *), GFP_KERNEL); 610 if (!uncores) 611 return empty_uncore; 612 613 for (node = rb_first(&discovery_tables); node; node = rb_next(node)) { 614 type = rb_entry(node, struct intel_uncore_discovery_type, node); 615 if (type->access_type != type_id) 616 continue; 617 618 uncore = kzalloc(sizeof(struct intel_uncore_type), GFP_KERNEL); 619 if (!uncore) 620 break; 621 622 uncore->event_mask = GENERIC_PMON_RAW_EVENT_MASK; 623 uncore->format_group = &generic_uncore_format_group; 624 625 if (!uncore_update_uncore_type(type_id, uncore, type)) { 626 kfree(uncore); 627 continue; 628 } 629 uncores[i++] = uncore; 630 } 631 632 return uncores; 633 } 634 635 void intel_uncore_generic_uncore_cpu_init(void) 636 { 637 uncore_msr_uncores = intel_uncore_generic_init_uncores(UNCORE_ACCESS_MSR, 0); 638 } 639 640 int intel_uncore_generic_uncore_pci_init(void) 641 { 642 uncore_pci_uncores = intel_uncore_generic_init_uncores(UNCORE_ACCESS_PCI, 0); 643 644 return 0; 645 } 646 647 void intel_uncore_generic_uncore_mmio_init(void) 648 { 649 uncore_mmio_uncores = intel_uncore_generic_init_uncores(UNCORE_ACCESS_MMIO, 0); 650 } 651