1 /* 2 * ARMv7 Cortex-A8 and Cortex-A9 Performance Events handling code. 3 * 4 * ARMv7 support: Jean Pihet <jpihet@mvista.com> 5 * 2010 (c) MontaVista Software, LLC. 6 * 7 * Copied from ARMv6 code, with the low level code inspired 8 * by the ARMv7 Oprofile code. 9 * 10 * Cortex-A8 has up to 4 configurable performance counters and 11 * a single cycle counter. 12 * Cortex-A9 has up to 31 configurable performance counters and 13 * a single cycle counter. 14 * 15 * All counters can be enabled/disabled and IRQ masked separately. The cycle 16 * counter and all 4 performance counters together can be reset separately. 17 */ 18 19 #ifdef CONFIG_CPU_V7 20 /* Common ARMv7 event types */ 21 enum armv7_perf_types { 22 ARMV7_PERFCTR_PMNC_SW_INCR = 0x00, 23 ARMV7_PERFCTR_IFETCH_MISS = 0x01, 24 ARMV7_PERFCTR_ITLB_MISS = 0x02, 25 ARMV7_PERFCTR_DCACHE_REFILL = 0x03, 26 ARMV7_PERFCTR_DCACHE_ACCESS = 0x04, 27 ARMV7_PERFCTR_DTLB_REFILL = 0x05, 28 ARMV7_PERFCTR_DREAD = 0x06, 29 ARMV7_PERFCTR_DWRITE = 0x07, 30 31 ARMV7_PERFCTR_EXC_TAKEN = 0x09, 32 ARMV7_PERFCTR_EXC_EXECUTED = 0x0A, 33 ARMV7_PERFCTR_CID_WRITE = 0x0B, 34 /* ARMV7_PERFCTR_PC_WRITE is equivalent to HW_BRANCH_INSTRUCTIONS. 35 * It counts: 36 * - all branch instructions, 37 * - instructions that explicitly write the PC, 38 * - exception generating instructions. 39 */ 40 ARMV7_PERFCTR_PC_WRITE = 0x0C, 41 ARMV7_PERFCTR_PC_IMM_BRANCH = 0x0D, 42 ARMV7_PERFCTR_UNALIGNED_ACCESS = 0x0F, 43 ARMV7_PERFCTR_PC_BRANCH_MIS_PRED = 0x10, 44 ARMV7_PERFCTR_CLOCK_CYCLES = 0x11, 45 46 ARMV7_PERFCTR_PC_BRANCH_MIS_USED = 0x12, 47 48 ARMV7_PERFCTR_CPU_CYCLES = 0xFF 49 }; 50 51 /* ARMv7 Cortex-A8 specific event types */ 52 enum armv7_a8_perf_types { 53 ARMV7_PERFCTR_INSTR_EXECUTED = 0x08, 54 55 ARMV7_PERFCTR_PC_PROC_RETURN = 0x0E, 56 57 ARMV7_PERFCTR_WRITE_BUFFER_FULL = 0x40, 58 ARMV7_PERFCTR_L2_STORE_MERGED = 0x41, 59 ARMV7_PERFCTR_L2_STORE_BUFF = 0x42, 60 ARMV7_PERFCTR_L2_ACCESS = 0x43, 61 ARMV7_PERFCTR_L2_CACH_MISS = 0x44, 62 ARMV7_PERFCTR_AXI_READ_CYCLES = 0x45, 63 ARMV7_PERFCTR_AXI_WRITE_CYCLES = 0x46, 64 ARMV7_PERFCTR_MEMORY_REPLAY = 0x47, 65 ARMV7_PERFCTR_UNALIGNED_ACCESS_REPLAY = 0x48, 66 ARMV7_PERFCTR_L1_DATA_MISS = 0x49, 67 ARMV7_PERFCTR_L1_INST_MISS = 0x4A, 68 ARMV7_PERFCTR_L1_DATA_COLORING = 0x4B, 69 ARMV7_PERFCTR_L1_NEON_DATA = 0x4C, 70 ARMV7_PERFCTR_L1_NEON_CACH_DATA = 0x4D, 71 ARMV7_PERFCTR_L2_NEON = 0x4E, 72 ARMV7_PERFCTR_L2_NEON_HIT = 0x4F, 73 ARMV7_PERFCTR_L1_INST = 0x50, 74 ARMV7_PERFCTR_PC_RETURN_MIS_PRED = 0x51, 75 ARMV7_PERFCTR_PC_BRANCH_FAILED = 0x52, 76 ARMV7_PERFCTR_PC_BRANCH_TAKEN = 0x53, 77 ARMV7_PERFCTR_PC_BRANCH_EXECUTED = 0x54, 78 ARMV7_PERFCTR_OP_EXECUTED = 0x55, 79 ARMV7_PERFCTR_CYCLES_INST_STALL = 0x56, 80 ARMV7_PERFCTR_CYCLES_INST = 0x57, 81 ARMV7_PERFCTR_CYCLES_NEON_DATA_STALL = 0x58, 82 ARMV7_PERFCTR_CYCLES_NEON_INST_STALL = 0x59, 83 ARMV7_PERFCTR_NEON_CYCLES = 0x5A, 84 85 ARMV7_PERFCTR_PMU0_EVENTS = 0x70, 86 ARMV7_PERFCTR_PMU1_EVENTS = 0x71, 87 ARMV7_PERFCTR_PMU_EVENTS = 0x72, 88 }; 89 90 /* ARMv7 Cortex-A9 specific event types */ 91 enum armv7_a9_perf_types { 92 ARMV7_PERFCTR_JAVA_HW_BYTECODE_EXEC = 0x40, 93 ARMV7_PERFCTR_JAVA_SW_BYTECODE_EXEC = 0x41, 94 ARMV7_PERFCTR_JAZELLE_BRANCH_EXEC = 0x42, 95 96 ARMV7_PERFCTR_COHERENT_LINE_MISS = 0x50, 97 ARMV7_PERFCTR_COHERENT_LINE_HIT = 0x51, 98 99 ARMV7_PERFCTR_ICACHE_DEP_STALL_CYCLES = 0x60, 100 ARMV7_PERFCTR_DCACHE_DEP_STALL_CYCLES = 0x61, 101 ARMV7_PERFCTR_TLB_MISS_DEP_STALL_CYCLES = 0x62, 102 ARMV7_PERFCTR_STREX_EXECUTED_PASSED = 0x63, 103 ARMV7_PERFCTR_STREX_EXECUTED_FAILED = 0x64, 104 ARMV7_PERFCTR_DATA_EVICTION = 0x65, 105 ARMV7_PERFCTR_ISSUE_STAGE_NO_INST = 0x66, 106 ARMV7_PERFCTR_ISSUE_STAGE_EMPTY = 0x67, 107 ARMV7_PERFCTR_INST_OUT_OF_RENAME_STAGE = 0x68, 108 109 ARMV7_PERFCTR_PREDICTABLE_FUNCT_RETURNS = 0x6E, 110 111 ARMV7_PERFCTR_MAIN_UNIT_EXECUTED_INST = 0x70, 112 ARMV7_PERFCTR_SECOND_UNIT_EXECUTED_INST = 0x71, 113 ARMV7_PERFCTR_LD_ST_UNIT_EXECUTED_INST = 0x72, 114 ARMV7_PERFCTR_FP_EXECUTED_INST = 0x73, 115 ARMV7_PERFCTR_NEON_EXECUTED_INST = 0x74, 116 117 ARMV7_PERFCTR_PLD_FULL_DEP_STALL_CYCLES = 0x80, 118 ARMV7_PERFCTR_DATA_WR_DEP_STALL_CYCLES = 0x81, 119 ARMV7_PERFCTR_ITLB_MISS_DEP_STALL_CYCLES = 0x82, 120 ARMV7_PERFCTR_DTLB_MISS_DEP_STALL_CYCLES = 0x83, 121 ARMV7_PERFCTR_MICRO_ITLB_MISS_DEP_STALL_CYCLES = 0x84, 122 ARMV7_PERFCTR_MICRO_DTLB_MISS_DEP_STALL_CYCLES = 0x85, 123 ARMV7_PERFCTR_DMB_DEP_STALL_CYCLES = 0x86, 124 125 ARMV7_PERFCTR_INTGR_CLK_ENABLED_CYCLES = 0x8A, 126 ARMV7_PERFCTR_DATA_ENGINE_CLK_EN_CYCLES = 0x8B, 127 128 ARMV7_PERFCTR_ISB_INST = 0x90, 129 ARMV7_PERFCTR_DSB_INST = 0x91, 130 ARMV7_PERFCTR_DMB_INST = 0x92, 131 ARMV7_PERFCTR_EXT_INTERRUPTS = 0x93, 132 133 ARMV7_PERFCTR_PLE_CACHE_LINE_RQST_COMPLETED = 0xA0, 134 ARMV7_PERFCTR_PLE_CACHE_LINE_RQST_SKIPPED = 0xA1, 135 ARMV7_PERFCTR_PLE_FIFO_FLUSH = 0xA2, 136 ARMV7_PERFCTR_PLE_RQST_COMPLETED = 0xA3, 137 ARMV7_PERFCTR_PLE_FIFO_OVERFLOW = 0xA4, 138 ARMV7_PERFCTR_PLE_RQST_PROG = 0xA5 139 }; 140 141 /* 142 * Cortex-A8 HW events mapping 143 * 144 * The hardware events that we support. We do support cache operations but 145 * we have harvard caches and no way to combine instruction and data 146 * accesses/misses in hardware. 147 */ 148 static const unsigned armv7_a8_perf_map[PERF_COUNT_HW_MAX] = { 149 [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES, 150 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED, 151 [PERF_COUNT_HW_CACHE_REFERENCES] = HW_OP_UNSUPPORTED, 152 [PERF_COUNT_HW_CACHE_MISSES] = HW_OP_UNSUPPORTED, 153 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE, 154 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, 155 [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_CLOCK_CYCLES, 156 }; 157 158 static const unsigned armv7_a8_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] 159 [PERF_COUNT_HW_CACHE_OP_MAX] 160 [PERF_COUNT_HW_CACHE_RESULT_MAX] = { 161 [C(L1D)] = { 162 /* 163 * The performance counters don't differentiate between read 164 * and write accesses/misses so this isn't strictly correct, 165 * but it's the best we can do. Writes and reads get 166 * combined. 167 */ 168 [C(OP_READ)] = { 169 [C(RESULT_ACCESS)] = ARMV7_PERFCTR_DCACHE_ACCESS, 170 [C(RESULT_MISS)] = ARMV7_PERFCTR_DCACHE_REFILL, 171 }, 172 [C(OP_WRITE)] = { 173 [C(RESULT_ACCESS)] = ARMV7_PERFCTR_DCACHE_ACCESS, 174 [C(RESULT_MISS)] = ARMV7_PERFCTR_DCACHE_REFILL, 175 }, 176 [C(OP_PREFETCH)] = { 177 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 178 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, 179 }, 180 }, 181 [C(L1I)] = { 182 [C(OP_READ)] = { 183 [C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_INST, 184 [C(RESULT_MISS)] = ARMV7_PERFCTR_L1_INST_MISS, 185 }, 186 [C(OP_WRITE)] = { 187 [C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_INST, 188 [C(RESULT_MISS)] = ARMV7_PERFCTR_L1_INST_MISS, 189 }, 190 [C(OP_PREFETCH)] = { 191 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 192 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, 193 }, 194 }, 195 [C(LL)] = { 196 [C(OP_READ)] = { 197 [C(RESULT_ACCESS)] = ARMV7_PERFCTR_L2_ACCESS, 198 [C(RESULT_MISS)] = ARMV7_PERFCTR_L2_CACH_MISS, 199 }, 200 [C(OP_WRITE)] = { 201 [C(RESULT_ACCESS)] = ARMV7_PERFCTR_L2_ACCESS, 202 [C(RESULT_MISS)] = ARMV7_PERFCTR_L2_CACH_MISS, 203 }, 204 [C(OP_PREFETCH)] = { 205 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 206 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, 207 }, 208 }, 209 [C(DTLB)] = { 210 /* 211 * Only ITLB misses and DTLB refills are supported. 212 * If users want the DTLB refills misses a raw counter 213 * must be used. 214 */ 215 [C(OP_READ)] = { 216 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 217 [C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL, 218 }, 219 [C(OP_WRITE)] = { 220 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 221 [C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL, 222 }, 223 [C(OP_PREFETCH)] = { 224 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 225 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, 226 }, 227 }, 228 [C(ITLB)] = { 229 [C(OP_READ)] = { 230 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 231 [C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_MISS, 232 }, 233 [C(OP_WRITE)] = { 234 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 235 [C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_MISS, 236 }, 237 [C(OP_PREFETCH)] = { 238 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 239 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, 240 }, 241 }, 242 [C(BPU)] = { 243 [C(OP_READ)] = { 244 [C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_WRITE, 245 [C(RESULT_MISS)] 246 = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, 247 }, 248 [C(OP_WRITE)] = { 249 [C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_WRITE, 250 [C(RESULT_MISS)] 251 = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, 252 }, 253 [C(OP_PREFETCH)] = { 254 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 255 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, 256 }, 257 }, 258 }; 259 260 /* 261 * Cortex-A9 HW events mapping 262 */ 263 static const unsigned armv7_a9_perf_map[PERF_COUNT_HW_MAX] = { 264 [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES, 265 [PERF_COUNT_HW_INSTRUCTIONS] = 266 ARMV7_PERFCTR_INST_OUT_OF_RENAME_STAGE, 267 [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_COHERENT_LINE_HIT, 268 [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_COHERENT_LINE_MISS, 269 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE, 270 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, 271 [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_CLOCK_CYCLES, 272 }; 273 274 static const unsigned armv7_a9_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] 275 [PERF_COUNT_HW_CACHE_OP_MAX] 276 [PERF_COUNT_HW_CACHE_RESULT_MAX] = { 277 [C(L1D)] = { 278 /* 279 * The performance counters don't differentiate between read 280 * and write accesses/misses so this isn't strictly correct, 281 * but it's the best we can do. Writes and reads get 282 * combined. 283 */ 284 [C(OP_READ)] = { 285 [C(RESULT_ACCESS)] = ARMV7_PERFCTR_DCACHE_ACCESS, 286 [C(RESULT_MISS)] = ARMV7_PERFCTR_DCACHE_REFILL, 287 }, 288 [C(OP_WRITE)] = { 289 [C(RESULT_ACCESS)] = ARMV7_PERFCTR_DCACHE_ACCESS, 290 [C(RESULT_MISS)] = ARMV7_PERFCTR_DCACHE_REFILL, 291 }, 292 [C(OP_PREFETCH)] = { 293 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 294 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, 295 }, 296 }, 297 [C(L1I)] = { 298 [C(OP_READ)] = { 299 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 300 [C(RESULT_MISS)] = ARMV7_PERFCTR_IFETCH_MISS, 301 }, 302 [C(OP_WRITE)] = { 303 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 304 [C(RESULT_MISS)] = ARMV7_PERFCTR_IFETCH_MISS, 305 }, 306 [C(OP_PREFETCH)] = { 307 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 308 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, 309 }, 310 }, 311 [C(LL)] = { 312 [C(OP_READ)] = { 313 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 314 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, 315 }, 316 [C(OP_WRITE)] = { 317 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 318 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, 319 }, 320 [C(OP_PREFETCH)] = { 321 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 322 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, 323 }, 324 }, 325 [C(DTLB)] = { 326 /* 327 * Only ITLB misses and DTLB refills are supported. 328 * If users want the DTLB refills misses a raw counter 329 * must be used. 330 */ 331 [C(OP_READ)] = { 332 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 333 [C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL, 334 }, 335 [C(OP_WRITE)] = { 336 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 337 [C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL, 338 }, 339 [C(OP_PREFETCH)] = { 340 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 341 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, 342 }, 343 }, 344 [C(ITLB)] = { 345 [C(OP_READ)] = { 346 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 347 [C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_MISS, 348 }, 349 [C(OP_WRITE)] = { 350 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 351 [C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_MISS, 352 }, 353 [C(OP_PREFETCH)] = { 354 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 355 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, 356 }, 357 }, 358 [C(BPU)] = { 359 [C(OP_READ)] = { 360 [C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_WRITE, 361 [C(RESULT_MISS)] 362 = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, 363 }, 364 [C(OP_WRITE)] = { 365 [C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_WRITE, 366 [C(RESULT_MISS)] 367 = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, 368 }, 369 [C(OP_PREFETCH)] = { 370 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 371 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, 372 }, 373 }, 374 }; 375 376 /* 377 * Perf Events counters 378 */ 379 enum armv7_counters { 380 ARMV7_CYCLE_COUNTER = 1, /* Cycle counter */ 381 ARMV7_COUNTER0 = 2, /* First event counter */ 382 }; 383 384 /* 385 * The cycle counter is ARMV7_CYCLE_COUNTER. 386 * The first event counter is ARMV7_COUNTER0. 387 * The last event counter is (ARMV7_COUNTER0 + armpmu->num_events - 1). 388 */ 389 #define ARMV7_COUNTER_LAST (ARMV7_COUNTER0 + armpmu->num_events - 1) 390 391 /* 392 * ARMv7 low level PMNC access 393 */ 394 395 /* 396 * Per-CPU PMNC: config reg 397 */ 398 #define ARMV7_PMNC_E (1 << 0) /* Enable all counters */ 399 #define ARMV7_PMNC_P (1 << 1) /* Reset all counters */ 400 #define ARMV7_PMNC_C (1 << 2) /* Cycle counter reset */ 401 #define ARMV7_PMNC_D (1 << 3) /* CCNT counts every 64th cpu cycle */ 402 #define ARMV7_PMNC_X (1 << 4) /* Export to ETM */ 403 #define ARMV7_PMNC_DP (1 << 5) /* Disable CCNT if non-invasive debug*/ 404 #define ARMV7_PMNC_N_SHIFT 11 /* Number of counters supported */ 405 #define ARMV7_PMNC_N_MASK 0x1f 406 #define ARMV7_PMNC_MASK 0x3f /* Mask for writable bits */ 407 408 /* 409 * Available counters 410 */ 411 #define ARMV7_CNT0 0 /* First event counter */ 412 #define ARMV7_CCNT 31 /* Cycle counter */ 413 414 /* Perf Event to low level counters mapping */ 415 #define ARMV7_EVENT_CNT_TO_CNTx (ARMV7_COUNTER0 - ARMV7_CNT0) 416 417 /* 418 * CNTENS: counters enable reg 419 */ 420 #define ARMV7_CNTENS_P(idx) (1 << (idx - ARMV7_EVENT_CNT_TO_CNTx)) 421 #define ARMV7_CNTENS_C (1 << ARMV7_CCNT) 422 423 /* 424 * CNTENC: counters disable reg 425 */ 426 #define ARMV7_CNTENC_P(idx) (1 << (idx - ARMV7_EVENT_CNT_TO_CNTx)) 427 #define ARMV7_CNTENC_C (1 << ARMV7_CCNT) 428 429 /* 430 * INTENS: counters overflow interrupt enable reg 431 */ 432 #define ARMV7_INTENS_P(idx) (1 << (idx - ARMV7_EVENT_CNT_TO_CNTx)) 433 #define ARMV7_INTENS_C (1 << ARMV7_CCNT) 434 435 /* 436 * INTENC: counters overflow interrupt disable reg 437 */ 438 #define ARMV7_INTENC_P(idx) (1 << (idx - ARMV7_EVENT_CNT_TO_CNTx)) 439 #define ARMV7_INTENC_C (1 << ARMV7_CCNT) 440 441 /* 442 * EVTSEL: Event selection reg 443 */ 444 #define ARMV7_EVTSEL_MASK 0xff /* Mask for writable bits */ 445 446 /* 447 * SELECT: Counter selection reg 448 */ 449 #define ARMV7_SELECT_MASK 0x1f /* Mask for writable bits */ 450 451 /* 452 * FLAG: counters overflow flag status reg 453 */ 454 #define ARMV7_FLAG_P(idx) (1 << (idx - ARMV7_EVENT_CNT_TO_CNTx)) 455 #define ARMV7_FLAG_C (1 << ARMV7_CCNT) 456 #define ARMV7_FLAG_MASK 0xffffffff /* Mask for writable bits */ 457 #define ARMV7_OVERFLOWED_MASK ARMV7_FLAG_MASK 458 459 static inline unsigned long armv7_pmnc_read(void) 460 { 461 u32 val; 462 asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r"(val)); 463 return val; 464 } 465 466 static inline void armv7_pmnc_write(unsigned long val) 467 { 468 val &= ARMV7_PMNC_MASK; 469 isb(); 470 asm volatile("mcr p15, 0, %0, c9, c12, 0" : : "r"(val)); 471 } 472 473 static inline int armv7_pmnc_has_overflowed(unsigned long pmnc) 474 { 475 return pmnc & ARMV7_OVERFLOWED_MASK; 476 } 477 478 static inline int armv7_pmnc_counter_has_overflowed(unsigned long pmnc, 479 enum armv7_counters counter) 480 { 481 int ret = 0; 482 483 if (counter == ARMV7_CYCLE_COUNTER) 484 ret = pmnc & ARMV7_FLAG_C; 485 else if ((counter >= ARMV7_COUNTER0) && (counter <= ARMV7_COUNTER_LAST)) 486 ret = pmnc & ARMV7_FLAG_P(counter); 487 else 488 pr_err("CPU%u checking wrong counter %d overflow status\n", 489 smp_processor_id(), counter); 490 491 return ret; 492 } 493 494 static inline int armv7_pmnc_select_counter(unsigned int idx) 495 { 496 u32 val; 497 498 if ((idx < ARMV7_COUNTER0) || (idx > ARMV7_COUNTER_LAST)) { 499 pr_err("CPU%u selecting wrong PMNC counter" 500 " %d\n", smp_processor_id(), idx); 501 return -1; 502 } 503 504 val = (idx - ARMV7_EVENT_CNT_TO_CNTx) & ARMV7_SELECT_MASK; 505 asm volatile("mcr p15, 0, %0, c9, c12, 5" : : "r" (val)); 506 isb(); 507 508 return idx; 509 } 510 511 static inline u32 armv7pmu_read_counter(int idx) 512 { 513 unsigned long value = 0; 514 515 if (idx == ARMV7_CYCLE_COUNTER) 516 asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (value)); 517 else if ((idx >= ARMV7_COUNTER0) && (idx <= ARMV7_COUNTER_LAST)) { 518 if (armv7_pmnc_select_counter(idx) == idx) 519 asm volatile("mrc p15, 0, %0, c9, c13, 2" 520 : "=r" (value)); 521 } else 522 pr_err("CPU%u reading wrong counter %d\n", 523 smp_processor_id(), idx); 524 525 return value; 526 } 527 528 static inline void armv7pmu_write_counter(int idx, u32 value) 529 { 530 if (idx == ARMV7_CYCLE_COUNTER) 531 asm volatile("mcr p15, 0, %0, c9, c13, 0" : : "r" (value)); 532 else if ((idx >= ARMV7_COUNTER0) && (idx <= ARMV7_COUNTER_LAST)) { 533 if (armv7_pmnc_select_counter(idx) == idx) 534 asm volatile("mcr p15, 0, %0, c9, c13, 2" 535 : : "r" (value)); 536 } else 537 pr_err("CPU%u writing wrong counter %d\n", 538 smp_processor_id(), idx); 539 } 540 541 static inline void armv7_pmnc_write_evtsel(unsigned int idx, u32 val) 542 { 543 if (armv7_pmnc_select_counter(idx) == idx) { 544 val &= ARMV7_EVTSEL_MASK; 545 asm volatile("mcr p15, 0, %0, c9, c13, 1" : : "r" (val)); 546 } 547 } 548 549 static inline u32 armv7_pmnc_enable_counter(unsigned int idx) 550 { 551 u32 val; 552 553 if ((idx != ARMV7_CYCLE_COUNTER) && 554 ((idx < ARMV7_COUNTER0) || (idx > ARMV7_COUNTER_LAST))) { 555 pr_err("CPU%u enabling wrong PMNC counter" 556 " %d\n", smp_processor_id(), idx); 557 return -1; 558 } 559 560 if (idx == ARMV7_CYCLE_COUNTER) 561 val = ARMV7_CNTENS_C; 562 else 563 val = ARMV7_CNTENS_P(idx); 564 565 asm volatile("mcr p15, 0, %0, c9, c12, 1" : : "r" (val)); 566 567 return idx; 568 } 569 570 static inline u32 armv7_pmnc_disable_counter(unsigned int idx) 571 { 572 u32 val; 573 574 575 if ((idx != ARMV7_CYCLE_COUNTER) && 576 ((idx < ARMV7_COUNTER0) || (idx > ARMV7_COUNTER_LAST))) { 577 pr_err("CPU%u disabling wrong PMNC counter" 578 " %d\n", smp_processor_id(), idx); 579 return -1; 580 } 581 582 if (idx == ARMV7_CYCLE_COUNTER) 583 val = ARMV7_CNTENC_C; 584 else 585 val = ARMV7_CNTENC_P(idx); 586 587 asm volatile("mcr p15, 0, %0, c9, c12, 2" : : "r" (val)); 588 589 return idx; 590 } 591 592 static inline u32 armv7_pmnc_enable_intens(unsigned int idx) 593 { 594 u32 val; 595 596 if ((idx != ARMV7_CYCLE_COUNTER) && 597 ((idx < ARMV7_COUNTER0) || (idx > ARMV7_COUNTER_LAST))) { 598 pr_err("CPU%u enabling wrong PMNC counter" 599 " interrupt enable %d\n", smp_processor_id(), idx); 600 return -1; 601 } 602 603 if (idx == ARMV7_CYCLE_COUNTER) 604 val = ARMV7_INTENS_C; 605 else 606 val = ARMV7_INTENS_P(idx); 607 608 asm volatile("mcr p15, 0, %0, c9, c14, 1" : : "r" (val)); 609 610 return idx; 611 } 612 613 static inline u32 armv7_pmnc_disable_intens(unsigned int idx) 614 { 615 u32 val; 616 617 if ((idx != ARMV7_CYCLE_COUNTER) && 618 ((idx < ARMV7_COUNTER0) || (idx > ARMV7_COUNTER_LAST))) { 619 pr_err("CPU%u disabling wrong PMNC counter" 620 " interrupt enable %d\n", smp_processor_id(), idx); 621 return -1; 622 } 623 624 if (idx == ARMV7_CYCLE_COUNTER) 625 val = ARMV7_INTENC_C; 626 else 627 val = ARMV7_INTENC_P(idx); 628 629 asm volatile("mcr p15, 0, %0, c9, c14, 2" : : "r" (val)); 630 631 return idx; 632 } 633 634 static inline u32 armv7_pmnc_getreset_flags(void) 635 { 636 u32 val; 637 638 /* Read */ 639 asm volatile("mrc p15, 0, %0, c9, c12, 3" : "=r" (val)); 640 641 /* Write to clear flags */ 642 val &= ARMV7_FLAG_MASK; 643 asm volatile("mcr p15, 0, %0, c9, c12, 3" : : "r" (val)); 644 645 return val; 646 } 647 648 #ifdef DEBUG 649 static void armv7_pmnc_dump_regs(void) 650 { 651 u32 val; 652 unsigned int cnt; 653 654 printk(KERN_INFO "PMNC registers dump:\n"); 655 656 asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r" (val)); 657 printk(KERN_INFO "PMNC =0x%08x\n", val); 658 659 asm volatile("mrc p15, 0, %0, c9, c12, 1" : "=r" (val)); 660 printk(KERN_INFO "CNTENS=0x%08x\n", val); 661 662 asm volatile("mrc p15, 0, %0, c9, c14, 1" : "=r" (val)); 663 printk(KERN_INFO "INTENS=0x%08x\n", val); 664 665 asm volatile("mrc p15, 0, %0, c9, c12, 3" : "=r" (val)); 666 printk(KERN_INFO "FLAGS =0x%08x\n", val); 667 668 asm volatile("mrc p15, 0, %0, c9, c12, 5" : "=r" (val)); 669 printk(KERN_INFO "SELECT=0x%08x\n", val); 670 671 asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (val)); 672 printk(KERN_INFO "CCNT =0x%08x\n", val); 673 674 for (cnt = ARMV7_COUNTER0; cnt < ARMV7_COUNTER_LAST; cnt++) { 675 armv7_pmnc_select_counter(cnt); 676 asm volatile("mrc p15, 0, %0, c9, c13, 2" : "=r" (val)); 677 printk(KERN_INFO "CNT[%d] count =0x%08x\n", 678 cnt-ARMV7_EVENT_CNT_TO_CNTx, val); 679 asm volatile("mrc p15, 0, %0, c9, c13, 1" : "=r" (val)); 680 printk(KERN_INFO "CNT[%d] evtsel=0x%08x\n", 681 cnt-ARMV7_EVENT_CNT_TO_CNTx, val); 682 } 683 } 684 #endif 685 686 static void armv7pmu_enable_event(struct hw_perf_event *hwc, int idx) 687 { 688 unsigned long flags; 689 690 /* 691 * Enable counter and interrupt, and set the counter to count 692 * the event that we're interested in. 693 */ 694 raw_spin_lock_irqsave(&pmu_lock, flags); 695 696 /* 697 * Disable counter 698 */ 699 armv7_pmnc_disable_counter(idx); 700 701 /* 702 * Set event (if destined for PMNx counters) 703 * We don't need to set the event if it's a cycle count 704 */ 705 if (idx != ARMV7_CYCLE_COUNTER) 706 armv7_pmnc_write_evtsel(idx, hwc->config_base); 707 708 /* 709 * Enable interrupt for this counter 710 */ 711 armv7_pmnc_enable_intens(idx); 712 713 /* 714 * Enable counter 715 */ 716 armv7_pmnc_enable_counter(idx); 717 718 raw_spin_unlock_irqrestore(&pmu_lock, flags); 719 } 720 721 static void armv7pmu_disable_event(struct hw_perf_event *hwc, int idx) 722 { 723 unsigned long flags; 724 725 /* 726 * Disable counter and interrupt 727 */ 728 raw_spin_lock_irqsave(&pmu_lock, flags); 729 730 /* 731 * Disable counter 732 */ 733 armv7_pmnc_disable_counter(idx); 734 735 /* 736 * Disable interrupt for this counter 737 */ 738 armv7_pmnc_disable_intens(idx); 739 740 raw_spin_unlock_irqrestore(&pmu_lock, flags); 741 } 742 743 static irqreturn_t armv7pmu_handle_irq(int irq_num, void *dev) 744 { 745 unsigned long pmnc; 746 struct perf_sample_data data; 747 struct cpu_hw_events *cpuc; 748 struct pt_regs *regs; 749 int idx; 750 751 /* 752 * Get and reset the IRQ flags 753 */ 754 pmnc = armv7_pmnc_getreset_flags(); 755 756 /* 757 * Did an overflow occur? 758 */ 759 if (!armv7_pmnc_has_overflowed(pmnc)) 760 return IRQ_NONE; 761 762 /* 763 * Handle the counter(s) overflow(s) 764 */ 765 regs = get_irq_regs(); 766 767 perf_sample_data_init(&data, 0); 768 769 cpuc = &__get_cpu_var(cpu_hw_events); 770 for (idx = 0; idx <= armpmu->num_events; ++idx) { 771 struct perf_event *event = cpuc->events[idx]; 772 struct hw_perf_event *hwc; 773 774 if (!test_bit(idx, cpuc->active_mask)) 775 continue; 776 777 /* 778 * We have a single interrupt for all counters. Check that 779 * each counter has overflowed before we process it. 780 */ 781 if (!armv7_pmnc_counter_has_overflowed(pmnc, idx)) 782 continue; 783 784 hwc = &event->hw; 785 armpmu_event_update(event, hwc, idx, 1); 786 data.period = event->hw.last_period; 787 if (!armpmu_event_set_period(event, hwc, idx)) 788 continue; 789 790 if (perf_event_overflow(event, 0, &data, regs)) 791 armpmu->disable(hwc, idx); 792 } 793 794 /* 795 * Handle the pending perf events. 796 * 797 * Note: this call *must* be run with interrupts disabled. For 798 * platforms that can have the PMU interrupts raised as an NMI, this 799 * will not work. 800 */ 801 irq_work_run(); 802 803 return IRQ_HANDLED; 804 } 805 806 static void armv7pmu_start(void) 807 { 808 unsigned long flags; 809 810 raw_spin_lock_irqsave(&pmu_lock, flags); 811 /* Enable all counters */ 812 armv7_pmnc_write(armv7_pmnc_read() | ARMV7_PMNC_E); 813 raw_spin_unlock_irqrestore(&pmu_lock, flags); 814 } 815 816 static void armv7pmu_stop(void) 817 { 818 unsigned long flags; 819 820 raw_spin_lock_irqsave(&pmu_lock, flags); 821 /* Disable all counters */ 822 armv7_pmnc_write(armv7_pmnc_read() & ~ARMV7_PMNC_E); 823 raw_spin_unlock_irqrestore(&pmu_lock, flags); 824 } 825 826 static int armv7pmu_get_event_idx(struct cpu_hw_events *cpuc, 827 struct hw_perf_event *event) 828 { 829 int idx; 830 831 /* Always place a cycle counter into the cycle counter. */ 832 if (event->config_base == ARMV7_PERFCTR_CPU_CYCLES) { 833 if (test_and_set_bit(ARMV7_CYCLE_COUNTER, cpuc->used_mask)) 834 return -EAGAIN; 835 836 return ARMV7_CYCLE_COUNTER; 837 } else { 838 /* 839 * For anything other than a cycle counter, try and use 840 * the events counters 841 */ 842 for (idx = ARMV7_COUNTER0; idx <= armpmu->num_events; ++idx) { 843 if (!test_and_set_bit(idx, cpuc->used_mask)) 844 return idx; 845 } 846 847 /* The counters are all in use. */ 848 return -EAGAIN; 849 } 850 } 851 852 static void armv7pmu_reset(void *info) 853 { 854 u32 idx, nb_cnt = armpmu->num_events; 855 856 /* The counter and interrupt enable registers are unknown at reset. */ 857 for (idx = 1; idx < nb_cnt; ++idx) 858 armv7pmu_disable_event(NULL, idx); 859 860 /* Initialize & Reset PMNC: C and P bits */ 861 armv7_pmnc_write(ARMV7_PMNC_P | ARMV7_PMNC_C); 862 } 863 864 static struct arm_pmu armv7pmu = { 865 .handle_irq = armv7pmu_handle_irq, 866 .enable = armv7pmu_enable_event, 867 .disable = armv7pmu_disable_event, 868 .read_counter = armv7pmu_read_counter, 869 .write_counter = armv7pmu_write_counter, 870 .get_event_idx = armv7pmu_get_event_idx, 871 .start = armv7pmu_start, 872 .stop = armv7pmu_stop, 873 .reset = armv7pmu_reset, 874 .raw_event_mask = 0xFF, 875 .max_period = (1LLU << 32) - 1, 876 }; 877 878 static u32 __init armv7_read_num_pmnc_events(void) 879 { 880 u32 nb_cnt; 881 882 /* Read the nb of CNTx counters supported from PMNC */ 883 nb_cnt = (armv7_pmnc_read() >> ARMV7_PMNC_N_SHIFT) & ARMV7_PMNC_N_MASK; 884 885 /* Add the CPU cycles counter and return */ 886 return nb_cnt + 1; 887 } 888 889 static const struct arm_pmu *__init armv7_a8_pmu_init(void) 890 { 891 armv7pmu.id = ARM_PERF_PMU_ID_CA8; 892 armv7pmu.name = "ARMv7 Cortex-A8"; 893 armv7pmu.cache_map = &armv7_a8_perf_cache_map; 894 armv7pmu.event_map = &armv7_a8_perf_map; 895 armv7pmu.num_events = armv7_read_num_pmnc_events(); 896 return &armv7pmu; 897 } 898 899 static const struct arm_pmu *__init armv7_a9_pmu_init(void) 900 { 901 armv7pmu.id = ARM_PERF_PMU_ID_CA9; 902 armv7pmu.name = "ARMv7 Cortex-A9"; 903 armv7pmu.cache_map = &armv7_a9_perf_cache_map; 904 armv7pmu.event_map = &armv7_a9_perf_map; 905 armv7pmu.num_events = armv7_read_num_pmnc_events(); 906 return &armv7pmu; 907 } 908 #else 909 static const struct arm_pmu *__init armv7_a8_pmu_init(void) 910 { 911 return NULL; 912 } 913 914 static const struct arm_pmu *__init armv7_a9_pmu_init(void) 915 { 916 return NULL; 917 } 918 #endif /* CONFIG_CPU_V7 */ 919