1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Cell Broadband Engine Performance Monitor 4 * 5 * (C) Copyright IBM Corporation 2001,2006 6 * 7 * Author: 8 * David Erb (djerb@us.ibm.com) 9 * Kevin Corry (kevcorry@us.ibm.com) 10 */ 11 12 #include <linux/interrupt.h> 13 #include <linux/types.h> 14 #include <linux/export.h> 15 #include <asm/io.h> 16 #include <asm/irq_regs.h> 17 #include <asm/machdep.h> 18 #include <asm/pmc.h> 19 #include <asm/reg.h> 20 #include <asm/spu.h> 21 #include <asm/cell-regs.h> 22 23 #include "interrupt.h" 24 25 /* 26 * When writing to write-only mmio addresses, save a shadow copy. All of the 27 * registers are 32-bit, but stored in the upper-half of a 64-bit field in 28 * pmd_regs. 29 */ 30 31 #define WRITE_WO_MMIO(reg, x) \ 32 do { \ 33 u32 _x = (x); \ 34 struct cbe_pmd_regs __iomem *pmd_regs; \ 35 struct cbe_pmd_shadow_regs *shadow_regs; \ 36 pmd_regs = cbe_get_cpu_pmd_regs(cpu); \ 37 shadow_regs = cbe_get_cpu_pmd_shadow_regs(cpu); \ 38 out_be64(&(pmd_regs->reg), (((u64)_x) << 32)); \ 39 shadow_regs->reg = _x; \ 40 } while (0) 41 42 #define READ_SHADOW_REG(val, reg) \ 43 do { \ 44 struct cbe_pmd_shadow_regs *shadow_regs; \ 45 shadow_regs = cbe_get_cpu_pmd_shadow_regs(cpu); \ 46 (val) = shadow_regs->reg; \ 47 } while (0) 48 49 #define READ_MMIO_UPPER32(val, reg) \ 50 do { \ 51 struct cbe_pmd_regs __iomem *pmd_regs; \ 52 pmd_regs = cbe_get_cpu_pmd_regs(cpu); \ 53 (val) = (u32)(in_be64(&pmd_regs->reg) >> 32); \ 54 } while (0) 55 56 /* 57 * Physical counter registers. 58 * Each physical counter can act as one 32-bit counter or two 16-bit counters. 59 */ 60 61 u32 cbe_read_phys_ctr(u32 cpu, u32 phys_ctr) 62 { 63 u32 val_in_latch, val = 0; 64 65 if (phys_ctr < NR_PHYS_CTRS) { 66 READ_SHADOW_REG(val_in_latch, counter_value_in_latch); 67 68 /* Read the latch or the actual counter, whichever is newer. */ 69 if (val_in_latch & (1 << phys_ctr)) { 70 READ_SHADOW_REG(val, pm_ctr[phys_ctr]); 71 } else { 72 READ_MMIO_UPPER32(val, pm_ctr[phys_ctr]); 73 } 74 } 75 76 return val; 77 } 78 EXPORT_SYMBOL_GPL(cbe_read_phys_ctr); 79 80 void cbe_write_phys_ctr(u32 cpu, u32 phys_ctr, u32 val) 81 { 82 struct cbe_pmd_shadow_regs *shadow_regs; 83 u32 pm_ctrl; 84 85 if (phys_ctr < NR_PHYS_CTRS) { 86 /* Writing to a counter only writes to a hardware latch. 87 * The new value is not propagated to the actual counter 88 * until the performance monitor is enabled. 89 */ 90 WRITE_WO_MMIO(pm_ctr[phys_ctr], val); 91 92 pm_ctrl = cbe_read_pm(cpu, pm_control); 93 if (pm_ctrl & CBE_PM_ENABLE_PERF_MON) { 94 /* The counters are already active, so we need to 95 * rewrite the pm_control register to "re-enable" 96 * the PMU. 97 */ 98 cbe_write_pm(cpu, pm_control, pm_ctrl); 99 } else { 100 shadow_regs = cbe_get_cpu_pmd_shadow_regs(cpu); 101 shadow_regs->counter_value_in_latch |= (1 << phys_ctr); 102 } 103 } 104 } 105 EXPORT_SYMBOL_GPL(cbe_write_phys_ctr); 106 107 /* 108 * "Logical" counter registers. 109 * These will read/write 16-bits or 32-bits depending on the 110 * current size of the counter. Counters 4 - 7 are always 16-bit. 111 */ 112 113 u32 cbe_read_ctr(u32 cpu, u32 ctr) 114 { 115 u32 val; 116 u32 phys_ctr = ctr & (NR_PHYS_CTRS - 1); 117 118 val = cbe_read_phys_ctr(cpu, phys_ctr); 119 120 if (cbe_get_ctr_size(cpu, phys_ctr) == 16) 121 val = (ctr < NR_PHYS_CTRS) ? (val >> 16) : (val & 0xffff); 122 123 return val; 124 } 125 EXPORT_SYMBOL_GPL(cbe_read_ctr); 126 127 void cbe_write_ctr(u32 cpu, u32 ctr, u32 val) 128 { 129 u32 phys_ctr; 130 u32 phys_val; 131 132 phys_ctr = ctr & (NR_PHYS_CTRS - 1); 133 134 if (cbe_get_ctr_size(cpu, phys_ctr) == 16) { 135 phys_val = cbe_read_phys_ctr(cpu, phys_ctr); 136 137 if (ctr < NR_PHYS_CTRS) 138 val = (val << 16) | (phys_val & 0xffff); 139 else 140 val = (val & 0xffff) | (phys_val & 0xffff0000); 141 } 142 143 cbe_write_phys_ctr(cpu, phys_ctr, val); 144 } 145 EXPORT_SYMBOL_GPL(cbe_write_ctr); 146 147 /* 148 * Counter-control registers. 149 * Each "logical" counter has a corresponding control register. 150 */ 151 152 u32 cbe_read_pm07_control(u32 cpu, u32 ctr) 153 { 154 u32 pm07_control = 0; 155 156 if (ctr < NR_CTRS) 157 READ_SHADOW_REG(pm07_control, pm07_control[ctr]); 158 159 return pm07_control; 160 } 161 EXPORT_SYMBOL_GPL(cbe_read_pm07_control); 162 163 void cbe_write_pm07_control(u32 cpu, u32 ctr, u32 val) 164 { 165 if (ctr < NR_CTRS) 166 WRITE_WO_MMIO(pm07_control[ctr], val); 167 } 168 EXPORT_SYMBOL_GPL(cbe_write_pm07_control); 169 170 /* 171 * Other PMU control registers. Most of these are write-only. 172 */ 173 174 u32 cbe_read_pm(u32 cpu, enum pm_reg_name reg) 175 { 176 u32 val = 0; 177 178 switch (reg) { 179 case group_control: 180 READ_SHADOW_REG(val, group_control); 181 break; 182 183 case debug_bus_control: 184 READ_SHADOW_REG(val, debug_bus_control); 185 break; 186 187 case trace_address: 188 READ_MMIO_UPPER32(val, trace_address); 189 break; 190 191 case ext_tr_timer: 192 READ_SHADOW_REG(val, ext_tr_timer); 193 break; 194 195 case pm_status: 196 READ_MMIO_UPPER32(val, pm_status); 197 break; 198 199 case pm_control: 200 READ_SHADOW_REG(val, pm_control); 201 break; 202 203 case pm_interval: 204 READ_MMIO_UPPER32(val, pm_interval); 205 break; 206 207 case pm_start_stop: 208 READ_SHADOW_REG(val, pm_start_stop); 209 break; 210 } 211 212 return val; 213 } 214 EXPORT_SYMBOL_GPL(cbe_read_pm); 215 216 void cbe_write_pm(u32 cpu, enum pm_reg_name reg, u32 val) 217 { 218 switch (reg) { 219 case group_control: 220 WRITE_WO_MMIO(group_control, val); 221 break; 222 223 case debug_bus_control: 224 WRITE_WO_MMIO(debug_bus_control, val); 225 break; 226 227 case trace_address: 228 WRITE_WO_MMIO(trace_address, val); 229 break; 230 231 case ext_tr_timer: 232 WRITE_WO_MMIO(ext_tr_timer, val); 233 break; 234 235 case pm_status: 236 WRITE_WO_MMIO(pm_status, val); 237 break; 238 239 case pm_control: 240 WRITE_WO_MMIO(pm_control, val); 241 break; 242 243 case pm_interval: 244 WRITE_WO_MMIO(pm_interval, val); 245 break; 246 247 case pm_start_stop: 248 WRITE_WO_MMIO(pm_start_stop, val); 249 break; 250 } 251 } 252 EXPORT_SYMBOL_GPL(cbe_write_pm); 253 254 /* 255 * Get/set the size of a physical counter to either 16 or 32 bits. 256 */ 257 258 u32 cbe_get_ctr_size(u32 cpu, u32 phys_ctr) 259 { 260 u32 pm_ctrl, size = 0; 261 262 if (phys_ctr < NR_PHYS_CTRS) { 263 pm_ctrl = cbe_read_pm(cpu, pm_control); 264 size = (pm_ctrl & CBE_PM_16BIT_CTR(phys_ctr)) ? 16 : 32; 265 } 266 267 return size; 268 } 269 EXPORT_SYMBOL_GPL(cbe_get_ctr_size); 270 271 void cbe_set_ctr_size(u32 cpu, u32 phys_ctr, u32 ctr_size) 272 { 273 u32 pm_ctrl; 274 275 if (phys_ctr < NR_PHYS_CTRS) { 276 pm_ctrl = cbe_read_pm(cpu, pm_control); 277 switch (ctr_size) { 278 case 16: 279 pm_ctrl |= CBE_PM_16BIT_CTR(phys_ctr); 280 break; 281 282 case 32: 283 pm_ctrl &= ~CBE_PM_16BIT_CTR(phys_ctr); 284 break; 285 } 286 cbe_write_pm(cpu, pm_control, pm_ctrl); 287 } 288 } 289 EXPORT_SYMBOL_GPL(cbe_set_ctr_size); 290 291 /* 292 * Enable/disable the entire performance monitoring unit. 293 * When we enable the PMU, all pending writes to counters get committed. 294 */ 295 296 void cbe_enable_pm(u32 cpu) 297 { 298 struct cbe_pmd_shadow_regs *shadow_regs; 299 u32 pm_ctrl; 300 301 shadow_regs = cbe_get_cpu_pmd_shadow_regs(cpu); 302 shadow_regs->counter_value_in_latch = 0; 303 304 pm_ctrl = cbe_read_pm(cpu, pm_control) | CBE_PM_ENABLE_PERF_MON; 305 cbe_write_pm(cpu, pm_control, pm_ctrl); 306 } 307 EXPORT_SYMBOL_GPL(cbe_enable_pm); 308 309 void cbe_disable_pm(u32 cpu) 310 { 311 u32 pm_ctrl; 312 pm_ctrl = cbe_read_pm(cpu, pm_control) & ~CBE_PM_ENABLE_PERF_MON; 313 cbe_write_pm(cpu, pm_control, pm_ctrl); 314 } 315 EXPORT_SYMBOL_GPL(cbe_disable_pm); 316 317 /* 318 * Reading from the trace_buffer. 319 * The trace buffer is two 64-bit registers. Reading from 320 * the second half automatically increments the trace_address. 321 */ 322 323 void cbe_read_trace_buffer(u32 cpu, u64 *buf) 324 { 325 struct cbe_pmd_regs __iomem *pmd_regs = cbe_get_cpu_pmd_regs(cpu); 326 327 *buf++ = in_be64(&pmd_regs->trace_buffer_0_63); 328 *buf++ = in_be64(&pmd_regs->trace_buffer_64_127); 329 } 330 EXPORT_SYMBOL_GPL(cbe_read_trace_buffer); 331 332 /* 333 * Enabling/disabling interrupts for the entire performance monitoring unit. 334 */ 335 336 u32 cbe_get_and_clear_pm_interrupts(u32 cpu) 337 { 338 /* Reading pm_status clears the interrupt bits. */ 339 return cbe_read_pm(cpu, pm_status); 340 } 341 EXPORT_SYMBOL_GPL(cbe_get_and_clear_pm_interrupts); 342 343 void cbe_enable_pm_interrupts(u32 cpu, u32 thread, u32 mask) 344 { 345 /* Set which node and thread will handle the next interrupt. */ 346 iic_set_interrupt_routing(cpu, thread, 0); 347 348 /* Enable the interrupt bits in the pm_status register. */ 349 if (mask) 350 cbe_write_pm(cpu, pm_status, mask); 351 } 352 EXPORT_SYMBOL_GPL(cbe_enable_pm_interrupts); 353 354 void cbe_disable_pm_interrupts(u32 cpu) 355 { 356 cbe_get_and_clear_pm_interrupts(cpu); 357 cbe_write_pm(cpu, pm_status, 0); 358 } 359 EXPORT_SYMBOL_GPL(cbe_disable_pm_interrupts); 360 361 static irqreturn_t cbe_pm_irq(int irq, void *dev_id) 362 { 363 perf_irq(get_irq_regs()); 364 return IRQ_HANDLED; 365 } 366 367 static int __init cbe_init_pm_irq(void) 368 { 369 unsigned int irq; 370 int rc, node; 371 372 for_each_online_node(node) { 373 irq = irq_create_mapping(NULL, IIC_IRQ_IOEX_PMI | 374 (node << IIC_IRQ_NODE_SHIFT)); 375 if (!irq) { 376 printk("ERROR: Unable to allocate irq for node %d\n", 377 node); 378 return -EINVAL; 379 } 380 381 rc = request_irq(irq, cbe_pm_irq, 382 0, "cbe-pmu-0", NULL); 383 if (rc) { 384 printk("ERROR: Request for irq on node %d failed\n", 385 node); 386 return rc; 387 } 388 } 389 390 return 0; 391 } 392 machine_arch_initcall(cell, cbe_init_pm_irq); 393 394 void cbe_sync_irq(int node) 395 { 396 unsigned int irq; 397 398 irq = irq_find_mapping(NULL, 399 IIC_IRQ_IOEX_PMI 400 | (node << IIC_IRQ_NODE_SHIFT)); 401 402 if (!irq) { 403 printk(KERN_WARNING "ERROR, unable to get existing irq %d " \ 404 "for node %d\n", irq, node); 405 return; 406 } 407 408 synchronize_irq(irq); 409 } 410 EXPORT_SYMBOL_GPL(cbe_sync_irq); 411 412