1*edae1f06SKan Liang /* SPDX-License-Identifier: GPL-2.0-only */ 2*edae1f06SKan Liang /* 3*edae1f06SKan Liang * Support Intel uncore PerfMon discovery mechanism. 4*edae1f06SKan Liang * Copyright(c) 2021 Intel Corporation. 5*edae1f06SKan Liang */ 6*edae1f06SKan Liang #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 7*edae1f06SKan Liang 8*edae1f06SKan Liang #include "uncore.h" 9*edae1f06SKan Liang #include "uncore_discovery.h" 10*edae1f06SKan Liang 11*edae1f06SKan Liang static struct rb_root discovery_tables = RB_ROOT; 12*edae1f06SKan Liang static int num_discovered_types[UNCORE_ACCESS_MAX]; 13*edae1f06SKan Liang 14*edae1f06SKan Liang static bool has_generic_discovery_table(void) 15*edae1f06SKan Liang { 16*edae1f06SKan Liang struct pci_dev *dev; 17*edae1f06SKan Liang int dvsec; 18*edae1f06SKan Liang 19*edae1f06SKan Liang dev = pci_get_device(PCI_VENDOR_ID_INTEL, UNCORE_DISCOVERY_TABLE_DEVICE, NULL); 20*edae1f06SKan Liang if (!dev) 21*edae1f06SKan Liang return false; 22*edae1f06SKan Liang 23*edae1f06SKan Liang /* A discovery table device has the unique capability ID. */ 24*edae1f06SKan Liang dvsec = pci_find_next_ext_capability(dev, 0, UNCORE_EXT_CAP_ID_DISCOVERY); 25*edae1f06SKan Liang pci_dev_put(dev); 26*edae1f06SKan Liang if (dvsec) 27*edae1f06SKan Liang return true; 28*edae1f06SKan Liang 29*edae1f06SKan Liang return false; 30*edae1f06SKan Liang } 31*edae1f06SKan Liang 32*edae1f06SKan Liang static int logical_die_id; 33*edae1f06SKan Liang 34*edae1f06SKan Liang static int get_device_die_id(struct pci_dev *dev) 35*edae1f06SKan Liang { 36*edae1f06SKan Liang int cpu, node = pcibus_to_node(dev->bus); 37*edae1f06SKan Liang 38*edae1f06SKan Liang /* 39*edae1f06SKan Liang * If the NUMA info is not available, assume that the logical die id is 40*edae1f06SKan Liang * continuous in the order in which the discovery table devices are 41*edae1f06SKan Liang * detected. 42*edae1f06SKan Liang */ 43*edae1f06SKan Liang if (node < 0) 44*edae1f06SKan Liang return logical_die_id++; 45*edae1f06SKan Liang 46*edae1f06SKan Liang for_each_cpu(cpu, cpumask_of_node(node)) { 47*edae1f06SKan Liang struct cpuinfo_x86 *c = &cpu_data(cpu); 48*edae1f06SKan Liang 49*edae1f06SKan Liang if (c->initialized && cpu_to_node(cpu) == node) 50*edae1f06SKan Liang return c->logical_die_id; 51*edae1f06SKan Liang } 52*edae1f06SKan Liang 53*edae1f06SKan Liang /* 54*edae1f06SKan Liang * All CPUs of a node may be offlined. For this case, 55*edae1f06SKan Liang * the PCI and MMIO type of uncore blocks which are 56*edae1f06SKan Liang * enumerated by the device will be unavailable. 57*edae1f06SKan Liang */ 58*edae1f06SKan Liang return -1; 59*edae1f06SKan Liang } 60*edae1f06SKan Liang 61*edae1f06SKan Liang #define __node_2_type(cur) \ 62*edae1f06SKan Liang rb_entry((cur), struct intel_uncore_discovery_type, node) 63*edae1f06SKan Liang 64*edae1f06SKan Liang static inline int __type_cmp(const void *key, const struct rb_node *b) 65*edae1f06SKan Liang { 66*edae1f06SKan Liang struct intel_uncore_discovery_type *type_b = __node_2_type(b); 67*edae1f06SKan Liang const u16 *type_id = key; 68*edae1f06SKan Liang 69*edae1f06SKan Liang if (type_b->type > *type_id) 70*edae1f06SKan Liang return -1; 71*edae1f06SKan Liang else if (type_b->type < *type_id) 72*edae1f06SKan Liang return 1; 73*edae1f06SKan Liang 74*edae1f06SKan Liang return 0; 75*edae1f06SKan Liang } 76*edae1f06SKan Liang 77*edae1f06SKan Liang static inline struct intel_uncore_discovery_type * 78*edae1f06SKan Liang search_uncore_discovery_type(u16 type_id) 79*edae1f06SKan Liang { 80*edae1f06SKan Liang struct rb_node *node = rb_find(&type_id, &discovery_tables, __type_cmp); 81*edae1f06SKan Liang 82*edae1f06SKan Liang return (node) ? __node_2_type(node) : NULL; 83*edae1f06SKan Liang } 84*edae1f06SKan Liang 85*edae1f06SKan Liang static inline bool __type_less(struct rb_node *a, const struct rb_node *b) 86*edae1f06SKan Liang { 87*edae1f06SKan Liang return (__node_2_type(a)->type < __node_2_type(b)->type); 88*edae1f06SKan Liang } 89*edae1f06SKan Liang 90*edae1f06SKan Liang static struct intel_uncore_discovery_type * 91*edae1f06SKan Liang add_uncore_discovery_type(struct uncore_unit_discovery *unit) 92*edae1f06SKan Liang { 93*edae1f06SKan Liang struct intel_uncore_discovery_type *type; 94*edae1f06SKan Liang 95*edae1f06SKan Liang if (unit->access_type >= UNCORE_ACCESS_MAX) { 96*edae1f06SKan Liang pr_warn("Unsupported access type %d\n", unit->access_type); 97*edae1f06SKan Liang return NULL; 98*edae1f06SKan Liang } 99*edae1f06SKan Liang 100*edae1f06SKan Liang type = kzalloc(sizeof(struct intel_uncore_discovery_type), GFP_KERNEL); 101*edae1f06SKan Liang if (!type) 102*edae1f06SKan Liang return NULL; 103*edae1f06SKan Liang 104*edae1f06SKan Liang type->box_ctrl_die = kcalloc(__uncore_max_dies, sizeof(u64), GFP_KERNEL); 105*edae1f06SKan Liang if (!type->box_ctrl_die) 106*edae1f06SKan Liang goto free_type; 107*edae1f06SKan Liang 108*edae1f06SKan Liang type->access_type = unit->access_type; 109*edae1f06SKan Liang num_discovered_types[type->access_type]++; 110*edae1f06SKan Liang type->type = unit->box_type; 111*edae1f06SKan Liang 112*edae1f06SKan Liang rb_add(&type->node, &discovery_tables, __type_less); 113*edae1f06SKan Liang 114*edae1f06SKan Liang return type; 115*edae1f06SKan Liang 116*edae1f06SKan Liang free_type: 117*edae1f06SKan Liang kfree(type); 118*edae1f06SKan Liang 119*edae1f06SKan Liang return NULL; 120*edae1f06SKan Liang 121*edae1f06SKan Liang } 122*edae1f06SKan Liang 123*edae1f06SKan Liang static struct intel_uncore_discovery_type * 124*edae1f06SKan Liang get_uncore_discovery_type(struct uncore_unit_discovery *unit) 125*edae1f06SKan Liang { 126*edae1f06SKan Liang struct intel_uncore_discovery_type *type; 127*edae1f06SKan Liang 128*edae1f06SKan Liang type = search_uncore_discovery_type(unit->box_type); 129*edae1f06SKan Liang if (type) 130*edae1f06SKan Liang return type; 131*edae1f06SKan Liang 132*edae1f06SKan Liang return add_uncore_discovery_type(unit); 133*edae1f06SKan Liang } 134*edae1f06SKan Liang 135*edae1f06SKan Liang static void 136*edae1f06SKan Liang uncore_insert_box_info(struct uncore_unit_discovery *unit, 137*edae1f06SKan Liang int die, bool parsed) 138*edae1f06SKan Liang { 139*edae1f06SKan Liang struct intel_uncore_discovery_type *type; 140*edae1f06SKan Liang unsigned int *box_offset, *ids; 141*edae1f06SKan Liang int i; 142*edae1f06SKan Liang 143*edae1f06SKan Liang if (WARN_ON_ONCE(!unit->ctl || !unit->ctl_offset || !unit->ctr_offset)) 144*edae1f06SKan Liang return; 145*edae1f06SKan Liang 146*edae1f06SKan Liang if (parsed) { 147*edae1f06SKan Liang type = search_uncore_discovery_type(unit->box_type); 148*edae1f06SKan Liang if (WARN_ON_ONCE(!type)) 149*edae1f06SKan Liang return; 150*edae1f06SKan Liang /* Store the first box of each die */ 151*edae1f06SKan Liang if (!type->box_ctrl_die[die]) 152*edae1f06SKan Liang type->box_ctrl_die[die] = unit->ctl; 153*edae1f06SKan Liang return; 154*edae1f06SKan Liang } 155*edae1f06SKan Liang 156*edae1f06SKan Liang type = get_uncore_discovery_type(unit); 157*edae1f06SKan Liang if (!type) 158*edae1f06SKan Liang return; 159*edae1f06SKan Liang 160*edae1f06SKan Liang box_offset = kcalloc(type->num_boxes + 1, sizeof(unsigned int), GFP_KERNEL); 161*edae1f06SKan Liang if (!box_offset) 162*edae1f06SKan Liang return; 163*edae1f06SKan Liang 164*edae1f06SKan Liang ids = kcalloc(type->num_boxes + 1, sizeof(unsigned int), GFP_KERNEL); 165*edae1f06SKan Liang if (!ids) 166*edae1f06SKan Liang goto free_box_offset; 167*edae1f06SKan Liang 168*edae1f06SKan Liang /* Store generic information for the first box */ 169*edae1f06SKan Liang if (!type->num_boxes) { 170*edae1f06SKan Liang type->box_ctrl = unit->ctl; 171*edae1f06SKan Liang type->box_ctrl_die[die] = unit->ctl; 172*edae1f06SKan Liang type->num_counters = unit->num_regs; 173*edae1f06SKan Liang type->counter_width = unit->bit_width; 174*edae1f06SKan Liang type->ctl_offset = unit->ctl_offset; 175*edae1f06SKan Liang type->ctr_offset = unit->ctr_offset; 176*edae1f06SKan Liang *ids = unit->box_id; 177*edae1f06SKan Liang goto end; 178*edae1f06SKan Liang } 179*edae1f06SKan Liang 180*edae1f06SKan Liang for (i = 0; i < type->num_boxes; i++) { 181*edae1f06SKan Liang ids[i] = type->ids[i]; 182*edae1f06SKan Liang box_offset[i] = type->box_offset[i]; 183*edae1f06SKan Liang 184*edae1f06SKan Liang if (WARN_ON_ONCE(unit->box_id == ids[i])) 185*edae1f06SKan Liang goto free_ids; 186*edae1f06SKan Liang } 187*edae1f06SKan Liang ids[i] = unit->box_id; 188*edae1f06SKan Liang box_offset[i] = unit->ctl - type->box_ctrl; 189*edae1f06SKan Liang kfree(type->ids); 190*edae1f06SKan Liang kfree(type->box_offset); 191*edae1f06SKan Liang end: 192*edae1f06SKan Liang type->ids = ids; 193*edae1f06SKan Liang type->box_offset = box_offset; 194*edae1f06SKan Liang type->num_boxes++; 195*edae1f06SKan Liang return; 196*edae1f06SKan Liang 197*edae1f06SKan Liang free_ids: 198*edae1f06SKan Liang kfree(ids); 199*edae1f06SKan Liang 200*edae1f06SKan Liang free_box_offset: 201*edae1f06SKan Liang kfree(box_offset); 202*edae1f06SKan Liang 203*edae1f06SKan Liang } 204*edae1f06SKan Liang 205*edae1f06SKan Liang static int parse_discovery_table(struct pci_dev *dev, int die, 206*edae1f06SKan Liang u32 bar_offset, bool *parsed) 207*edae1f06SKan Liang { 208*edae1f06SKan Liang struct uncore_global_discovery global; 209*edae1f06SKan Liang struct uncore_unit_discovery unit; 210*edae1f06SKan Liang void __iomem *io_addr; 211*edae1f06SKan Liang resource_size_t addr; 212*edae1f06SKan Liang unsigned long size; 213*edae1f06SKan Liang u32 val; 214*edae1f06SKan Liang int i; 215*edae1f06SKan Liang 216*edae1f06SKan Liang pci_read_config_dword(dev, bar_offset, &val); 217*edae1f06SKan Liang 218*edae1f06SKan Liang if (val & UNCORE_DISCOVERY_MASK) 219*edae1f06SKan Liang return -EINVAL; 220*edae1f06SKan Liang 221*edae1f06SKan Liang addr = (resource_size_t)(val & ~UNCORE_DISCOVERY_MASK); 222*edae1f06SKan Liang size = UNCORE_DISCOVERY_GLOBAL_MAP_SIZE; 223*edae1f06SKan Liang io_addr = ioremap(addr, size); 224*edae1f06SKan Liang if (!io_addr) 225*edae1f06SKan Liang return -ENOMEM; 226*edae1f06SKan Liang 227*edae1f06SKan Liang /* Read Global Discovery State */ 228*edae1f06SKan Liang memcpy_fromio(&global, io_addr, sizeof(struct uncore_global_discovery)); 229*edae1f06SKan Liang if (uncore_discovery_invalid_unit(global)) { 230*edae1f06SKan Liang pr_info("Invalid Global Discovery State: 0x%llx 0x%llx 0x%llx\n", 231*edae1f06SKan Liang global.table1, global.ctl, global.table3); 232*edae1f06SKan Liang iounmap(io_addr); 233*edae1f06SKan Liang return -EINVAL; 234*edae1f06SKan Liang } 235*edae1f06SKan Liang iounmap(io_addr); 236*edae1f06SKan Liang 237*edae1f06SKan Liang size = (1 + global.max_units) * global.stride * 8; 238*edae1f06SKan Liang io_addr = ioremap(addr, size); 239*edae1f06SKan Liang if (!io_addr) 240*edae1f06SKan Liang return -ENOMEM; 241*edae1f06SKan Liang 242*edae1f06SKan Liang /* Parsing Unit Discovery State */ 243*edae1f06SKan Liang for (i = 0; i < global.max_units; i++) { 244*edae1f06SKan Liang memcpy_fromio(&unit, io_addr + (i + 1) * (global.stride * 8), 245*edae1f06SKan Liang sizeof(struct uncore_unit_discovery)); 246*edae1f06SKan Liang 247*edae1f06SKan Liang if (uncore_discovery_invalid_unit(unit)) 248*edae1f06SKan Liang continue; 249*edae1f06SKan Liang 250*edae1f06SKan Liang if (unit.access_type >= UNCORE_ACCESS_MAX) 251*edae1f06SKan Liang continue; 252*edae1f06SKan Liang 253*edae1f06SKan Liang uncore_insert_box_info(&unit, die, *parsed); 254*edae1f06SKan Liang } 255*edae1f06SKan Liang 256*edae1f06SKan Liang *parsed = true; 257*edae1f06SKan Liang iounmap(io_addr); 258*edae1f06SKan Liang return 0; 259*edae1f06SKan Liang } 260*edae1f06SKan Liang 261*edae1f06SKan Liang bool intel_uncore_has_discovery_tables(void) 262*edae1f06SKan Liang { 263*edae1f06SKan Liang u32 device, val, entry_id, bar_offset; 264*edae1f06SKan Liang int die, dvsec = 0, ret = true; 265*edae1f06SKan Liang struct pci_dev *dev = NULL; 266*edae1f06SKan Liang bool parsed = false; 267*edae1f06SKan Liang 268*edae1f06SKan Liang if (has_generic_discovery_table()) 269*edae1f06SKan Liang device = UNCORE_DISCOVERY_TABLE_DEVICE; 270*edae1f06SKan Liang else 271*edae1f06SKan Liang device = PCI_ANY_ID; 272*edae1f06SKan Liang 273*edae1f06SKan Liang /* 274*edae1f06SKan Liang * Start a new search and iterates through the list of 275*edae1f06SKan Liang * the discovery table devices. 276*edae1f06SKan Liang */ 277*edae1f06SKan Liang while ((dev = pci_get_device(PCI_VENDOR_ID_INTEL, device, dev)) != NULL) { 278*edae1f06SKan Liang while ((dvsec = pci_find_next_ext_capability(dev, dvsec, UNCORE_EXT_CAP_ID_DISCOVERY))) { 279*edae1f06SKan Liang pci_read_config_dword(dev, dvsec + UNCORE_DISCOVERY_DVSEC_OFFSET, &val); 280*edae1f06SKan Liang entry_id = val & UNCORE_DISCOVERY_DVSEC_ID_MASK; 281*edae1f06SKan Liang if (entry_id != UNCORE_DISCOVERY_DVSEC_ID_PMON) 282*edae1f06SKan Liang continue; 283*edae1f06SKan Liang 284*edae1f06SKan Liang pci_read_config_dword(dev, dvsec + UNCORE_DISCOVERY_DVSEC2_OFFSET, &val); 285*edae1f06SKan Liang 286*edae1f06SKan Liang if (val & ~UNCORE_DISCOVERY_DVSEC2_BIR_MASK) { 287*edae1f06SKan Liang ret = false; 288*edae1f06SKan Liang goto err; 289*edae1f06SKan Liang } 290*edae1f06SKan Liang bar_offset = UNCORE_DISCOVERY_BIR_BASE + 291*edae1f06SKan Liang (val & UNCORE_DISCOVERY_DVSEC2_BIR_MASK) * UNCORE_DISCOVERY_BIR_STEP; 292*edae1f06SKan Liang 293*edae1f06SKan Liang die = get_device_die_id(dev); 294*edae1f06SKan Liang if (die < 0) 295*edae1f06SKan Liang continue; 296*edae1f06SKan Liang 297*edae1f06SKan Liang parse_discovery_table(dev, die, bar_offset, &parsed); 298*edae1f06SKan Liang } 299*edae1f06SKan Liang } 300*edae1f06SKan Liang 301*edae1f06SKan Liang /* None of the discovery tables are available */ 302*edae1f06SKan Liang if (!parsed) 303*edae1f06SKan Liang ret = false; 304*edae1f06SKan Liang err: 305*edae1f06SKan Liang pci_dev_put(dev); 306*edae1f06SKan Liang 307*edae1f06SKan Liang return ret; 308*edae1f06SKan Liang } 309*edae1f06SKan Liang 310*edae1f06SKan Liang void intel_uncore_clear_discovery_tables(void) 311*edae1f06SKan Liang { 312*edae1f06SKan Liang struct intel_uncore_discovery_type *type, *next; 313*edae1f06SKan Liang 314*edae1f06SKan Liang rbtree_postorder_for_each_entry_safe(type, next, &discovery_tables, node) { 315*edae1f06SKan Liang kfree(type->box_ctrl_die); 316*edae1f06SKan Liang kfree(type); 317*edae1f06SKan Liang } 318*edae1f06SKan Liang } 319