1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Cell Broadband Engine Performance Monitor 4 * 5 * (C) Copyright IBM Corporation 2001,2006 6 * 7 * Author: 8 * David Erb (djerb@us.ibm.com) 9 * Kevin Corry (kevcorry@us.ibm.com) 10 */ 11 12 #include <linux/interrupt.h> 13 #include <linux/irqdomain.h> 14 #include <linux/types.h> 15 #include <linux/export.h> 16 #include <asm/io.h> 17 #include <asm/irq_regs.h> 18 #include <asm/machdep.h> 19 #include <asm/pmc.h> 20 #include <asm/reg.h> 21 #include <asm/spu.h> 22 #include <asm/cell-regs.h> 23 24 #include "interrupt.h" 25 26 /* 27 * When writing to write-only mmio addresses, save a shadow copy. All of the 28 * registers are 32-bit, but stored in the upper-half of a 64-bit field in 29 * pmd_regs. 30 */ 31 32 #define WRITE_WO_MMIO(reg, x) \ 33 do { \ 34 u32 _x = (x); \ 35 struct cbe_pmd_regs __iomem *pmd_regs; \ 36 struct cbe_pmd_shadow_regs *shadow_regs; \ 37 pmd_regs = cbe_get_cpu_pmd_regs(cpu); \ 38 shadow_regs = cbe_get_cpu_pmd_shadow_regs(cpu); \ 39 out_be64(&(pmd_regs->reg), (((u64)_x) << 32)); \ 40 shadow_regs->reg = _x; \ 41 } while (0) 42 43 #define READ_SHADOW_REG(val, reg) \ 44 do { \ 45 struct cbe_pmd_shadow_regs *shadow_regs; \ 46 shadow_regs = cbe_get_cpu_pmd_shadow_regs(cpu); \ 47 (val) = shadow_regs->reg; \ 48 } while (0) 49 50 #define READ_MMIO_UPPER32(val, reg) \ 51 do { \ 52 struct cbe_pmd_regs __iomem *pmd_regs; \ 53 pmd_regs = cbe_get_cpu_pmd_regs(cpu); \ 54 (val) = (u32)(in_be64(&pmd_regs->reg) >> 32); \ 55 } while (0) 56 57 /* 58 * Physical counter registers. 59 * Each physical counter can act as one 32-bit counter or two 16-bit counters. 60 */ 61 62 u32 cbe_read_phys_ctr(u32 cpu, u32 phys_ctr) 63 { 64 u32 val_in_latch, val = 0; 65 66 if (phys_ctr < NR_PHYS_CTRS) { 67 READ_SHADOW_REG(val_in_latch, counter_value_in_latch); 68 69 /* Read the latch or the actual counter, whichever is newer. */ 70 if (val_in_latch & (1 << phys_ctr)) { 71 READ_SHADOW_REG(val, pm_ctr[phys_ctr]); 72 } else { 73 READ_MMIO_UPPER32(val, pm_ctr[phys_ctr]); 74 } 75 } 76 77 return val; 78 } 79 EXPORT_SYMBOL_GPL(cbe_read_phys_ctr); 80 81 void cbe_write_phys_ctr(u32 cpu, u32 phys_ctr, u32 val) 82 { 83 struct cbe_pmd_shadow_regs *shadow_regs; 84 u32 pm_ctrl; 85 86 if (phys_ctr < NR_PHYS_CTRS) { 87 /* Writing to a counter only writes to a hardware latch. 88 * The new value is not propagated to the actual counter 89 * until the performance monitor is enabled. 90 */ 91 WRITE_WO_MMIO(pm_ctr[phys_ctr], val); 92 93 pm_ctrl = cbe_read_pm(cpu, pm_control); 94 if (pm_ctrl & CBE_PM_ENABLE_PERF_MON) { 95 /* The counters are already active, so we need to 96 * rewrite the pm_control register to "re-enable" 97 * the PMU. 98 */ 99 cbe_write_pm(cpu, pm_control, pm_ctrl); 100 } else { 101 shadow_regs = cbe_get_cpu_pmd_shadow_regs(cpu); 102 shadow_regs->counter_value_in_latch |= (1 << phys_ctr); 103 } 104 } 105 } 106 EXPORT_SYMBOL_GPL(cbe_write_phys_ctr); 107 108 /* 109 * "Logical" counter registers. 110 * These will read/write 16-bits or 32-bits depending on the 111 * current size of the counter. Counters 4 - 7 are always 16-bit. 112 */ 113 114 u32 cbe_read_ctr(u32 cpu, u32 ctr) 115 { 116 u32 val; 117 u32 phys_ctr = ctr & (NR_PHYS_CTRS - 1); 118 119 val = cbe_read_phys_ctr(cpu, phys_ctr); 120 121 if (cbe_get_ctr_size(cpu, phys_ctr) == 16) 122 val = (ctr < NR_PHYS_CTRS) ? (val >> 16) : (val & 0xffff); 123 124 return val; 125 } 126 EXPORT_SYMBOL_GPL(cbe_read_ctr); 127 128 void cbe_write_ctr(u32 cpu, u32 ctr, u32 val) 129 { 130 u32 phys_ctr; 131 u32 phys_val; 132 133 phys_ctr = ctr & (NR_PHYS_CTRS - 1); 134 135 if (cbe_get_ctr_size(cpu, phys_ctr) == 16) { 136 phys_val = cbe_read_phys_ctr(cpu, phys_ctr); 137 138 if (ctr < NR_PHYS_CTRS) 139 val = (val << 16) | (phys_val & 0xffff); 140 else 141 val = (val & 0xffff) | (phys_val & 0xffff0000); 142 } 143 144 cbe_write_phys_ctr(cpu, phys_ctr, val); 145 } 146 EXPORT_SYMBOL_GPL(cbe_write_ctr); 147 148 /* 149 * Counter-control registers. 150 * Each "logical" counter has a corresponding control register. 151 */ 152 153 u32 cbe_read_pm07_control(u32 cpu, u32 ctr) 154 { 155 u32 pm07_control = 0; 156 157 if (ctr < NR_CTRS) 158 READ_SHADOW_REG(pm07_control, pm07_control[ctr]); 159 160 return pm07_control; 161 } 162 EXPORT_SYMBOL_GPL(cbe_read_pm07_control); 163 164 void cbe_write_pm07_control(u32 cpu, u32 ctr, u32 val) 165 { 166 if (ctr < NR_CTRS) 167 WRITE_WO_MMIO(pm07_control[ctr], val); 168 } 169 EXPORT_SYMBOL_GPL(cbe_write_pm07_control); 170 171 /* 172 * Other PMU control registers. Most of these are write-only. 173 */ 174 175 u32 cbe_read_pm(u32 cpu, enum pm_reg_name reg) 176 { 177 u32 val = 0; 178 179 switch (reg) { 180 case group_control: 181 READ_SHADOW_REG(val, group_control); 182 break; 183 184 case debug_bus_control: 185 READ_SHADOW_REG(val, debug_bus_control); 186 break; 187 188 case trace_address: 189 READ_MMIO_UPPER32(val, trace_address); 190 break; 191 192 case ext_tr_timer: 193 READ_SHADOW_REG(val, ext_tr_timer); 194 break; 195 196 case pm_status: 197 READ_MMIO_UPPER32(val, pm_status); 198 break; 199 200 case pm_control: 201 READ_SHADOW_REG(val, pm_control); 202 break; 203 204 case pm_interval: 205 READ_MMIO_UPPER32(val, pm_interval); 206 break; 207 208 case pm_start_stop: 209 READ_SHADOW_REG(val, pm_start_stop); 210 break; 211 } 212 213 return val; 214 } 215 EXPORT_SYMBOL_GPL(cbe_read_pm); 216 217 void cbe_write_pm(u32 cpu, enum pm_reg_name reg, u32 val) 218 { 219 switch (reg) { 220 case group_control: 221 WRITE_WO_MMIO(group_control, val); 222 break; 223 224 case debug_bus_control: 225 WRITE_WO_MMIO(debug_bus_control, val); 226 break; 227 228 case trace_address: 229 WRITE_WO_MMIO(trace_address, val); 230 break; 231 232 case ext_tr_timer: 233 WRITE_WO_MMIO(ext_tr_timer, val); 234 break; 235 236 case pm_status: 237 WRITE_WO_MMIO(pm_status, val); 238 break; 239 240 case pm_control: 241 WRITE_WO_MMIO(pm_control, val); 242 break; 243 244 case pm_interval: 245 WRITE_WO_MMIO(pm_interval, val); 246 break; 247 248 case pm_start_stop: 249 WRITE_WO_MMIO(pm_start_stop, val); 250 break; 251 } 252 } 253 EXPORT_SYMBOL_GPL(cbe_write_pm); 254 255 /* 256 * Get/set the size of a physical counter to either 16 or 32 bits. 257 */ 258 259 u32 cbe_get_ctr_size(u32 cpu, u32 phys_ctr) 260 { 261 u32 pm_ctrl, size = 0; 262 263 if (phys_ctr < NR_PHYS_CTRS) { 264 pm_ctrl = cbe_read_pm(cpu, pm_control); 265 size = (pm_ctrl & CBE_PM_16BIT_CTR(phys_ctr)) ? 16 : 32; 266 } 267 268 return size; 269 } 270 EXPORT_SYMBOL_GPL(cbe_get_ctr_size); 271 272 void cbe_set_ctr_size(u32 cpu, u32 phys_ctr, u32 ctr_size) 273 { 274 u32 pm_ctrl; 275 276 if (phys_ctr < NR_PHYS_CTRS) { 277 pm_ctrl = cbe_read_pm(cpu, pm_control); 278 switch (ctr_size) { 279 case 16: 280 pm_ctrl |= CBE_PM_16BIT_CTR(phys_ctr); 281 break; 282 283 case 32: 284 pm_ctrl &= ~CBE_PM_16BIT_CTR(phys_ctr); 285 break; 286 } 287 cbe_write_pm(cpu, pm_control, pm_ctrl); 288 } 289 } 290 EXPORT_SYMBOL_GPL(cbe_set_ctr_size); 291 292 /* 293 * Enable/disable the entire performance monitoring unit. 294 * When we enable the PMU, all pending writes to counters get committed. 295 */ 296 297 void cbe_enable_pm(u32 cpu) 298 { 299 struct cbe_pmd_shadow_regs *shadow_regs; 300 u32 pm_ctrl; 301 302 shadow_regs = cbe_get_cpu_pmd_shadow_regs(cpu); 303 shadow_regs->counter_value_in_latch = 0; 304 305 pm_ctrl = cbe_read_pm(cpu, pm_control) | CBE_PM_ENABLE_PERF_MON; 306 cbe_write_pm(cpu, pm_control, pm_ctrl); 307 } 308 EXPORT_SYMBOL_GPL(cbe_enable_pm); 309 310 void cbe_disable_pm(u32 cpu) 311 { 312 u32 pm_ctrl; 313 pm_ctrl = cbe_read_pm(cpu, pm_control) & ~CBE_PM_ENABLE_PERF_MON; 314 cbe_write_pm(cpu, pm_control, pm_ctrl); 315 } 316 EXPORT_SYMBOL_GPL(cbe_disable_pm); 317 318 /* 319 * Reading from the trace_buffer. 320 * The trace buffer is two 64-bit registers. Reading from 321 * the second half automatically increments the trace_address. 322 */ 323 324 void cbe_read_trace_buffer(u32 cpu, u64 *buf) 325 { 326 struct cbe_pmd_regs __iomem *pmd_regs = cbe_get_cpu_pmd_regs(cpu); 327 328 *buf++ = in_be64(&pmd_regs->trace_buffer_0_63); 329 *buf++ = in_be64(&pmd_regs->trace_buffer_64_127); 330 } 331 EXPORT_SYMBOL_GPL(cbe_read_trace_buffer); 332 333 /* 334 * Enabling/disabling interrupts for the entire performance monitoring unit. 335 */ 336 337 u32 cbe_get_and_clear_pm_interrupts(u32 cpu) 338 { 339 /* Reading pm_status clears the interrupt bits. */ 340 return cbe_read_pm(cpu, pm_status); 341 } 342 EXPORT_SYMBOL_GPL(cbe_get_and_clear_pm_interrupts); 343 344 void cbe_enable_pm_interrupts(u32 cpu, u32 thread, u32 mask) 345 { 346 /* Set which node and thread will handle the next interrupt. */ 347 iic_set_interrupt_routing(cpu, thread, 0); 348 349 /* Enable the interrupt bits in the pm_status register. */ 350 if (mask) 351 cbe_write_pm(cpu, pm_status, mask); 352 } 353 EXPORT_SYMBOL_GPL(cbe_enable_pm_interrupts); 354 355 void cbe_disable_pm_interrupts(u32 cpu) 356 { 357 cbe_get_and_clear_pm_interrupts(cpu); 358 cbe_write_pm(cpu, pm_status, 0); 359 } 360 EXPORT_SYMBOL_GPL(cbe_disable_pm_interrupts); 361 362 static irqreturn_t cbe_pm_irq(int irq, void *dev_id) 363 { 364 perf_irq(get_irq_regs()); 365 return IRQ_HANDLED; 366 } 367 368 static int __init cbe_init_pm_irq(void) 369 { 370 unsigned int irq; 371 int rc, node; 372 373 for_each_online_node(node) { 374 irq = irq_create_mapping(NULL, IIC_IRQ_IOEX_PMI | 375 (node << IIC_IRQ_NODE_SHIFT)); 376 if (!irq) { 377 printk("ERROR: Unable to allocate irq for node %d\n", 378 node); 379 return -EINVAL; 380 } 381 382 rc = request_irq(irq, cbe_pm_irq, 383 0, "cbe-pmu-0", NULL); 384 if (rc) { 385 printk("ERROR: Request for irq on node %d failed\n", 386 node); 387 return rc; 388 } 389 } 390 391 return 0; 392 } 393 machine_arch_initcall(cell, cbe_init_pm_irq); 394 395 void cbe_sync_irq(int node) 396 { 397 unsigned int irq; 398 399 irq = irq_find_mapping(NULL, 400 IIC_IRQ_IOEX_PMI 401 | (node << IIC_IRQ_NODE_SHIFT)); 402 403 if (!irq) { 404 printk(KERN_WARNING "ERROR, unable to get existing irq %d " \ 405 "for node %d\n", irq, node); 406 return; 407 } 408 409 synchronize_irq(irq); 410 } 411 EXPORT_SYMBOL_GPL(cbe_sync_irq); 412 413