1 /* 2 * ARMv7 Cortex-A8 and Cortex-A9 Performance Events handling code. 3 * 4 * ARMv7 support: Jean Pihet <jpihet@mvista.com> 5 * 2010 (c) MontaVista Software, LLC. 6 * 7 * Copied from ARMv6 code, with the low level code inspired 8 * by the ARMv7 Oprofile code. 9 * 10 * Cortex-A8 has up to 4 configurable performance counters and 11 * a single cycle counter. 12 * Cortex-A9 has up to 31 configurable performance counters and 13 * a single cycle counter. 14 * 15 * All counters can be enabled/disabled and IRQ masked separately. The cycle 16 * counter and all 4 performance counters together can be reset separately. 17 */ 18 19 #ifdef CONFIG_CPU_V7 20 /* Common ARMv7 event types */ 21 enum armv7_perf_types { 22 ARMV7_PERFCTR_PMNC_SW_INCR = 0x00, 23 ARMV7_PERFCTR_IFETCH_MISS = 0x01, 24 ARMV7_PERFCTR_ITLB_MISS = 0x02, 25 ARMV7_PERFCTR_DCACHE_REFILL = 0x03, 26 ARMV7_PERFCTR_DCACHE_ACCESS = 0x04, 27 ARMV7_PERFCTR_DTLB_REFILL = 0x05, 28 ARMV7_PERFCTR_DREAD = 0x06, 29 ARMV7_PERFCTR_DWRITE = 0x07, 30 31 ARMV7_PERFCTR_EXC_TAKEN = 0x09, 32 ARMV7_PERFCTR_EXC_EXECUTED = 0x0A, 33 ARMV7_PERFCTR_CID_WRITE = 0x0B, 34 /* ARMV7_PERFCTR_PC_WRITE is equivalent to HW_BRANCH_INSTRUCTIONS. 35 * It counts: 36 * - all branch instructions, 37 * - instructions that explicitly write the PC, 38 * - exception generating instructions. 39 */ 40 ARMV7_PERFCTR_PC_WRITE = 0x0C, 41 ARMV7_PERFCTR_PC_IMM_BRANCH = 0x0D, 42 ARMV7_PERFCTR_UNALIGNED_ACCESS = 0x0F, 43 ARMV7_PERFCTR_PC_BRANCH_MIS_PRED = 0x10, 44 ARMV7_PERFCTR_CLOCK_CYCLES = 0x11, 45 46 ARMV7_PERFCTR_PC_BRANCH_MIS_USED = 0x12, 47 48 ARMV7_PERFCTR_CPU_CYCLES = 0xFF 49 }; 50 51 /* ARMv7 Cortex-A8 specific event types */ 52 enum armv7_a8_perf_types { 53 ARMV7_PERFCTR_INSTR_EXECUTED = 0x08, 54 55 ARMV7_PERFCTR_PC_PROC_RETURN = 0x0E, 56 57 ARMV7_PERFCTR_WRITE_BUFFER_FULL = 0x40, 58 ARMV7_PERFCTR_L2_STORE_MERGED = 0x41, 59 ARMV7_PERFCTR_L2_STORE_BUFF = 0x42, 60 ARMV7_PERFCTR_L2_ACCESS = 0x43, 61 ARMV7_PERFCTR_L2_CACH_MISS = 0x44, 62 ARMV7_PERFCTR_AXI_READ_CYCLES = 0x45, 63 ARMV7_PERFCTR_AXI_WRITE_CYCLES = 0x46, 64 ARMV7_PERFCTR_MEMORY_REPLAY = 0x47, 65 ARMV7_PERFCTR_UNALIGNED_ACCESS_REPLAY = 0x48, 66 ARMV7_PERFCTR_L1_DATA_MISS = 0x49, 67 ARMV7_PERFCTR_L1_INST_MISS = 0x4A, 68 ARMV7_PERFCTR_L1_DATA_COLORING = 0x4B, 69 ARMV7_PERFCTR_L1_NEON_DATA = 0x4C, 70 ARMV7_PERFCTR_L1_NEON_CACH_DATA = 0x4D, 71 ARMV7_PERFCTR_L2_NEON = 0x4E, 72 ARMV7_PERFCTR_L2_NEON_HIT = 0x4F, 73 ARMV7_PERFCTR_L1_INST = 0x50, 74 ARMV7_PERFCTR_PC_RETURN_MIS_PRED = 0x51, 75 ARMV7_PERFCTR_PC_BRANCH_FAILED = 0x52, 76 ARMV7_PERFCTR_PC_BRANCH_TAKEN = 0x53, 77 ARMV7_PERFCTR_PC_BRANCH_EXECUTED = 0x54, 78 ARMV7_PERFCTR_OP_EXECUTED = 0x55, 79 ARMV7_PERFCTR_CYCLES_INST_STALL = 0x56, 80 ARMV7_PERFCTR_CYCLES_INST = 0x57, 81 ARMV7_PERFCTR_CYCLES_NEON_DATA_STALL = 0x58, 82 ARMV7_PERFCTR_CYCLES_NEON_INST_STALL = 0x59, 83 ARMV7_PERFCTR_NEON_CYCLES = 0x5A, 84 85 ARMV7_PERFCTR_PMU0_EVENTS = 0x70, 86 ARMV7_PERFCTR_PMU1_EVENTS = 0x71, 87 ARMV7_PERFCTR_PMU_EVENTS = 0x72, 88 }; 89 90 /* ARMv7 Cortex-A9 specific event types */ 91 enum armv7_a9_perf_types { 92 ARMV7_PERFCTR_JAVA_HW_BYTECODE_EXEC = 0x40, 93 ARMV7_PERFCTR_JAVA_SW_BYTECODE_EXEC = 0x41, 94 ARMV7_PERFCTR_JAZELLE_BRANCH_EXEC = 0x42, 95 96 ARMV7_PERFCTR_COHERENT_LINE_MISS = 0x50, 97 ARMV7_PERFCTR_COHERENT_LINE_HIT = 0x51, 98 99 ARMV7_PERFCTR_ICACHE_DEP_STALL_CYCLES = 0x60, 100 ARMV7_PERFCTR_DCACHE_DEP_STALL_CYCLES = 0x61, 101 ARMV7_PERFCTR_TLB_MISS_DEP_STALL_CYCLES = 0x62, 102 ARMV7_PERFCTR_STREX_EXECUTED_PASSED = 0x63, 103 ARMV7_PERFCTR_STREX_EXECUTED_FAILED = 0x64, 104 ARMV7_PERFCTR_DATA_EVICTION = 0x65, 105 ARMV7_PERFCTR_ISSUE_STAGE_NO_INST = 0x66, 106 ARMV7_PERFCTR_ISSUE_STAGE_EMPTY = 0x67, 107 ARMV7_PERFCTR_INST_OUT_OF_RENAME_STAGE = 0x68, 108 109 ARMV7_PERFCTR_PREDICTABLE_FUNCT_RETURNS = 0x6E, 110 111 ARMV7_PERFCTR_MAIN_UNIT_EXECUTED_INST = 0x70, 112 ARMV7_PERFCTR_SECOND_UNIT_EXECUTED_INST = 0x71, 113 ARMV7_PERFCTR_LD_ST_UNIT_EXECUTED_INST = 0x72, 114 ARMV7_PERFCTR_FP_EXECUTED_INST = 0x73, 115 ARMV7_PERFCTR_NEON_EXECUTED_INST = 0x74, 116 117 ARMV7_PERFCTR_PLD_FULL_DEP_STALL_CYCLES = 0x80, 118 ARMV7_PERFCTR_DATA_WR_DEP_STALL_CYCLES = 0x81, 119 ARMV7_PERFCTR_ITLB_MISS_DEP_STALL_CYCLES = 0x82, 120 ARMV7_PERFCTR_DTLB_MISS_DEP_STALL_CYCLES = 0x83, 121 ARMV7_PERFCTR_MICRO_ITLB_MISS_DEP_STALL_CYCLES = 0x84, 122 ARMV7_PERFCTR_MICRO_DTLB_MISS_DEP_STALL_CYCLES = 0x85, 123 ARMV7_PERFCTR_DMB_DEP_STALL_CYCLES = 0x86, 124 125 ARMV7_PERFCTR_INTGR_CLK_ENABLED_CYCLES = 0x8A, 126 ARMV7_PERFCTR_DATA_ENGINE_CLK_EN_CYCLES = 0x8B, 127 128 ARMV7_PERFCTR_ISB_INST = 0x90, 129 ARMV7_PERFCTR_DSB_INST = 0x91, 130 ARMV7_PERFCTR_DMB_INST = 0x92, 131 ARMV7_PERFCTR_EXT_INTERRUPTS = 0x93, 132 133 ARMV7_PERFCTR_PLE_CACHE_LINE_RQST_COMPLETED = 0xA0, 134 ARMV7_PERFCTR_PLE_CACHE_LINE_RQST_SKIPPED = 0xA1, 135 ARMV7_PERFCTR_PLE_FIFO_FLUSH = 0xA2, 136 ARMV7_PERFCTR_PLE_RQST_COMPLETED = 0xA3, 137 ARMV7_PERFCTR_PLE_FIFO_OVERFLOW = 0xA4, 138 ARMV7_PERFCTR_PLE_RQST_PROG = 0xA5 139 }; 140 141 /* 142 * Cortex-A8 HW events mapping 143 * 144 * The hardware events that we support. We do support cache operations but 145 * we have harvard caches and no way to combine instruction and data 146 * accesses/misses in hardware. 147 */ 148 static const unsigned armv7_a8_perf_map[PERF_COUNT_HW_MAX] = { 149 [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES, 150 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED, 151 [PERF_COUNT_HW_CACHE_REFERENCES] = HW_OP_UNSUPPORTED, 152 [PERF_COUNT_HW_CACHE_MISSES] = HW_OP_UNSUPPORTED, 153 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE, 154 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, 155 [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_CLOCK_CYCLES, 156 }; 157 158 static const unsigned armv7_a8_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] 159 [PERF_COUNT_HW_CACHE_OP_MAX] 160 [PERF_COUNT_HW_CACHE_RESULT_MAX] = { 161 [C(L1D)] = { 162 /* 163 * The performance counters don't differentiate between read 164 * and write accesses/misses so this isn't strictly correct, 165 * but it's the best we can do. Writes and reads get 166 * combined. 167 */ 168 [C(OP_READ)] = { 169 [C(RESULT_ACCESS)] = ARMV7_PERFCTR_DCACHE_ACCESS, 170 [C(RESULT_MISS)] = ARMV7_PERFCTR_DCACHE_REFILL, 171 }, 172 [C(OP_WRITE)] = { 173 [C(RESULT_ACCESS)] = ARMV7_PERFCTR_DCACHE_ACCESS, 174 [C(RESULT_MISS)] = ARMV7_PERFCTR_DCACHE_REFILL, 175 }, 176 [C(OP_PREFETCH)] = { 177 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 178 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, 179 }, 180 }, 181 [C(L1I)] = { 182 [C(OP_READ)] = { 183 [C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_INST, 184 [C(RESULT_MISS)] = ARMV7_PERFCTR_L1_INST_MISS, 185 }, 186 [C(OP_WRITE)] = { 187 [C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_INST, 188 [C(RESULT_MISS)] = ARMV7_PERFCTR_L1_INST_MISS, 189 }, 190 [C(OP_PREFETCH)] = { 191 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 192 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, 193 }, 194 }, 195 [C(LL)] = { 196 [C(OP_READ)] = { 197 [C(RESULT_ACCESS)] = ARMV7_PERFCTR_L2_ACCESS, 198 [C(RESULT_MISS)] = ARMV7_PERFCTR_L2_CACH_MISS, 199 }, 200 [C(OP_WRITE)] = { 201 [C(RESULT_ACCESS)] = ARMV7_PERFCTR_L2_ACCESS, 202 [C(RESULT_MISS)] = ARMV7_PERFCTR_L2_CACH_MISS, 203 }, 204 [C(OP_PREFETCH)] = { 205 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 206 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, 207 }, 208 }, 209 [C(DTLB)] = { 210 /* 211 * Only ITLB misses and DTLB refills are supported. 212 * If users want the DTLB refills misses a raw counter 213 * must be used. 214 */ 215 [C(OP_READ)] = { 216 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 217 [C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL, 218 }, 219 [C(OP_WRITE)] = { 220 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 221 [C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL, 222 }, 223 [C(OP_PREFETCH)] = { 224 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 225 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, 226 }, 227 }, 228 [C(ITLB)] = { 229 [C(OP_READ)] = { 230 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 231 [C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_MISS, 232 }, 233 [C(OP_WRITE)] = { 234 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 235 [C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_MISS, 236 }, 237 [C(OP_PREFETCH)] = { 238 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 239 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, 240 }, 241 }, 242 [C(BPU)] = { 243 [C(OP_READ)] = { 244 [C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_WRITE, 245 [C(RESULT_MISS)] 246 = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, 247 }, 248 [C(OP_WRITE)] = { 249 [C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_WRITE, 250 [C(RESULT_MISS)] 251 = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, 252 }, 253 [C(OP_PREFETCH)] = { 254 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 255 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, 256 }, 257 }, 258 }; 259 260 /* 261 * Cortex-A9 HW events mapping 262 */ 263 static const unsigned armv7_a9_perf_map[PERF_COUNT_HW_MAX] = { 264 [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES, 265 [PERF_COUNT_HW_INSTRUCTIONS] = 266 ARMV7_PERFCTR_INST_OUT_OF_RENAME_STAGE, 267 [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_COHERENT_LINE_HIT, 268 [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_COHERENT_LINE_MISS, 269 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE, 270 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, 271 [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_CLOCK_CYCLES, 272 }; 273 274 static const unsigned armv7_a9_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] 275 [PERF_COUNT_HW_CACHE_OP_MAX] 276 [PERF_COUNT_HW_CACHE_RESULT_MAX] = { 277 [C(L1D)] = { 278 /* 279 * The performance counters don't differentiate between read 280 * and write accesses/misses so this isn't strictly correct, 281 * but it's the best we can do. Writes and reads get 282 * combined. 283 */ 284 [C(OP_READ)] = { 285 [C(RESULT_ACCESS)] = ARMV7_PERFCTR_DCACHE_ACCESS, 286 [C(RESULT_MISS)] = ARMV7_PERFCTR_DCACHE_REFILL, 287 }, 288 [C(OP_WRITE)] = { 289 [C(RESULT_ACCESS)] = ARMV7_PERFCTR_DCACHE_ACCESS, 290 [C(RESULT_MISS)] = ARMV7_PERFCTR_DCACHE_REFILL, 291 }, 292 [C(OP_PREFETCH)] = { 293 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 294 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, 295 }, 296 }, 297 [C(L1I)] = { 298 [C(OP_READ)] = { 299 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 300 [C(RESULT_MISS)] = ARMV7_PERFCTR_IFETCH_MISS, 301 }, 302 [C(OP_WRITE)] = { 303 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 304 [C(RESULT_MISS)] = ARMV7_PERFCTR_IFETCH_MISS, 305 }, 306 [C(OP_PREFETCH)] = { 307 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 308 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, 309 }, 310 }, 311 [C(LL)] = { 312 [C(OP_READ)] = { 313 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 314 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, 315 }, 316 [C(OP_WRITE)] = { 317 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 318 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, 319 }, 320 [C(OP_PREFETCH)] = { 321 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 322 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, 323 }, 324 }, 325 [C(DTLB)] = { 326 /* 327 * Only ITLB misses and DTLB refills are supported. 328 * If users want the DTLB refills misses a raw counter 329 * must be used. 330 */ 331 [C(OP_READ)] = { 332 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 333 [C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL, 334 }, 335 [C(OP_WRITE)] = { 336 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 337 [C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL, 338 }, 339 [C(OP_PREFETCH)] = { 340 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 341 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, 342 }, 343 }, 344 [C(ITLB)] = { 345 [C(OP_READ)] = { 346 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 347 [C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_MISS, 348 }, 349 [C(OP_WRITE)] = { 350 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 351 [C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_MISS, 352 }, 353 [C(OP_PREFETCH)] = { 354 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 355 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, 356 }, 357 }, 358 [C(BPU)] = { 359 [C(OP_READ)] = { 360 [C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_WRITE, 361 [C(RESULT_MISS)] 362 = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, 363 }, 364 [C(OP_WRITE)] = { 365 [C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_WRITE, 366 [C(RESULT_MISS)] 367 = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, 368 }, 369 [C(OP_PREFETCH)] = { 370 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 371 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, 372 }, 373 }, 374 }; 375 376 /* 377 * Perf Events counters 378 */ 379 enum armv7_counters { 380 ARMV7_CYCLE_COUNTER = 1, /* Cycle counter */ 381 ARMV7_COUNTER0 = 2, /* First event counter */ 382 }; 383 384 /* 385 * The cycle counter is ARMV7_CYCLE_COUNTER. 386 * The first event counter is ARMV7_COUNTER0. 387 * The last event counter is (ARMV7_COUNTER0 + armpmu->num_events - 1). 388 */ 389 #define ARMV7_COUNTER_LAST (ARMV7_COUNTER0 + armpmu->num_events - 1) 390 391 /* 392 * ARMv7 low level PMNC access 393 */ 394 395 /* 396 * Per-CPU PMNC: config reg 397 */ 398 #define ARMV7_PMNC_E (1 << 0) /* Enable all counters */ 399 #define ARMV7_PMNC_P (1 << 1) /* Reset all counters */ 400 #define ARMV7_PMNC_C (1 << 2) /* Cycle counter reset */ 401 #define ARMV7_PMNC_D (1 << 3) /* CCNT counts every 64th cpu cycle */ 402 #define ARMV7_PMNC_X (1 << 4) /* Export to ETM */ 403 #define ARMV7_PMNC_DP (1 << 5) /* Disable CCNT if non-invasive debug*/ 404 #define ARMV7_PMNC_N_SHIFT 11 /* Number of counters supported */ 405 #define ARMV7_PMNC_N_MASK 0x1f 406 #define ARMV7_PMNC_MASK 0x3f /* Mask for writable bits */ 407 408 /* 409 * Available counters 410 */ 411 #define ARMV7_CNT0 0 /* First event counter */ 412 #define ARMV7_CCNT 31 /* Cycle counter */ 413 414 /* Perf Event to low level counters mapping */ 415 #define ARMV7_EVENT_CNT_TO_CNTx (ARMV7_COUNTER0 - ARMV7_CNT0) 416 417 /* 418 * CNTENS: counters enable reg 419 */ 420 #define ARMV7_CNTENS_P(idx) (1 << (idx - ARMV7_EVENT_CNT_TO_CNTx)) 421 #define ARMV7_CNTENS_C (1 << ARMV7_CCNT) 422 423 /* 424 * CNTENC: counters disable reg 425 */ 426 #define ARMV7_CNTENC_P(idx) (1 << (idx - ARMV7_EVENT_CNT_TO_CNTx)) 427 #define ARMV7_CNTENC_C (1 << ARMV7_CCNT) 428 429 /* 430 * INTENS: counters overflow interrupt enable reg 431 */ 432 #define ARMV7_INTENS_P(idx) (1 << (idx - ARMV7_EVENT_CNT_TO_CNTx)) 433 #define ARMV7_INTENS_C (1 << ARMV7_CCNT) 434 435 /* 436 * INTENC: counters overflow interrupt disable reg 437 */ 438 #define ARMV7_INTENC_P(idx) (1 << (idx - ARMV7_EVENT_CNT_TO_CNTx)) 439 #define ARMV7_INTENC_C (1 << ARMV7_CCNT) 440 441 /* 442 * EVTSEL: Event selection reg 443 */ 444 #define ARMV7_EVTSEL_MASK 0xff /* Mask for writable bits */ 445 446 /* 447 * SELECT: Counter selection reg 448 */ 449 #define ARMV7_SELECT_MASK 0x1f /* Mask for writable bits */ 450 451 /* 452 * FLAG: counters overflow flag status reg 453 */ 454 #define ARMV7_FLAG_P(idx) (1 << (idx - ARMV7_EVENT_CNT_TO_CNTx)) 455 #define ARMV7_FLAG_C (1 << ARMV7_CCNT) 456 #define ARMV7_FLAG_MASK 0xffffffff /* Mask for writable bits */ 457 #define ARMV7_OVERFLOWED_MASK ARMV7_FLAG_MASK 458 459 static inline unsigned long armv7_pmnc_read(void) 460 { 461 u32 val; 462 asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r"(val)); 463 return val; 464 } 465 466 static inline void armv7_pmnc_write(unsigned long val) 467 { 468 val &= ARMV7_PMNC_MASK; 469 asm volatile("mcr p15, 0, %0, c9, c12, 0" : : "r"(val)); 470 } 471 472 static inline int armv7_pmnc_has_overflowed(unsigned long pmnc) 473 { 474 return pmnc & ARMV7_OVERFLOWED_MASK; 475 } 476 477 static inline int armv7_pmnc_counter_has_overflowed(unsigned long pmnc, 478 enum armv7_counters counter) 479 { 480 int ret = 0; 481 482 if (counter == ARMV7_CYCLE_COUNTER) 483 ret = pmnc & ARMV7_FLAG_C; 484 else if ((counter >= ARMV7_COUNTER0) && (counter <= ARMV7_COUNTER_LAST)) 485 ret = pmnc & ARMV7_FLAG_P(counter); 486 else 487 pr_err("CPU%u checking wrong counter %d overflow status\n", 488 smp_processor_id(), counter); 489 490 return ret; 491 } 492 493 static inline int armv7_pmnc_select_counter(unsigned int idx) 494 { 495 u32 val; 496 497 if ((idx < ARMV7_COUNTER0) || (idx > ARMV7_COUNTER_LAST)) { 498 pr_err("CPU%u selecting wrong PMNC counter" 499 " %d\n", smp_processor_id(), idx); 500 return -1; 501 } 502 503 val = (idx - ARMV7_EVENT_CNT_TO_CNTx) & ARMV7_SELECT_MASK; 504 asm volatile("mcr p15, 0, %0, c9, c12, 5" : : "r" (val)); 505 506 return idx; 507 } 508 509 static inline u32 armv7pmu_read_counter(int idx) 510 { 511 unsigned long value = 0; 512 513 if (idx == ARMV7_CYCLE_COUNTER) 514 asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (value)); 515 else if ((idx >= ARMV7_COUNTER0) && (idx <= ARMV7_COUNTER_LAST)) { 516 if (armv7_pmnc_select_counter(idx) == idx) 517 asm volatile("mrc p15, 0, %0, c9, c13, 2" 518 : "=r" (value)); 519 } else 520 pr_err("CPU%u reading wrong counter %d\n", 521 smp_processor_id(), idx); 522 523 return value; 524 } 525 526 static inline void armv7pmu_write_counter(int idx, u32 value) 527 { 528 if (idx == ARMV7_CYCLE_COUNTER) 529 asm volatile("mcr p15, 0, %0, c9, c13, 0" : : "r" (value)); 530 else if ((idx >= ARMV7_COUNTER0) && (idx <= ARMV7_COUNTER_LAST)) { 531 if (armv7_pmnc_select_counter(idx) == idx) 532 asm volatile("mcr p15, 0, %0, c9, c13, 2" 533 : : "r" (value)); 534 } else 535 pr_err("CPU%u writing wrong counter %d\n", 536 smp_processor_id(), idx); 537 } 538 539 static inline void armv7_pmnc_write_evtsel(unsigned int idx, u32 val) 540 { 541 if (armv7_pmnc_select_counter(idx) == idx) { 542 val &= ARMV7_EVTSEL_MASK; 543 asm volatile("mcr p15, 0, %0, c9, c13, 1" : : "r" (val)); 544 } 545 } 546 547 static inline u32 armv7_pmnc_enable_counter(unsigned int idx) 548 { 549 u32 val; 550 551 if ((idx != ARMV7_CYCLE_COUNTER) && 552 ((idx < ARMV7_COUNTER0) || (idx > ARMV7_COUNTER_LAST))) { 553 pr_err("CPU%u enabling wrong PMNC counter" 554 " %d\n", smp_processor_id(), idx); 555 return -1; 556 } 557 558 if (idx == ARMV7_CYCLE_COUNTER) 559 val = ARMV7_CNTENS_C; 560 else 561 val = ARMV7_CNTENS_P(idx); 562 563 asm volatile("mcr p15, 0, %0, c9, c12, 1" : : "r" (val)); 564 565 return idx; 566 } 567 568 static inline u32 armv7_pmnc_disable_counter(unsigned int idx) 569 { 570 u32 val; 571 572 573 if ((idx != ARMV7_CYCLE_COUNTER) && 574 ((idx < ARMV7_COUNTER0) || (idx > ARMV7_COUNTER_LAST))) { 575 pr_err("CPU%u disabling wrong PMNC counter" 576 " %d\n", smp_processor_id(), idx); 577 return -1; 578 } 579 580 if (idx == ARMV7_CYCLE_COUNTER) 581 val = ARMV7_CNTENC_C; 582 else 583 val = ARMV7_CNTENC_P(idx); 584 585 asm volatile("mcr p15, 0, %0, c9, c12, 2" : : "r" (val)); 586 587 return idx; 588 } 589 590 static inline u32 armv7_pmnc_enable_intens(unsigned int idx) 591 { 592 u32 val; 593 594 if ((idx != ARMV7_CYCLE_COUNTER) && 595 ((idx < ARMV7_COUNTER0) || (idx > ARMV7_COUNTER_LAST))) { 596 pr_err("CPU%u enabling wrong PMNC counter" 597 " interrupt enable %d\n", smp_processor_id(), idx); 598 return -1; 599 } 600 601 if (idx == ARMV7_CYCLE_COUNTER) 602 val = ARMV7_INTENS_C; 603 else 604 val = ARMV7_INTENS_P(idx); 605 606 asm volatile("mcr p15, 0, %0, c9, c14, 1" : : "r" (val)); 607 608 return idx; 609 } 610 611 static inline u32 armv7_pmnc_disable_intens(unsigned int idx) 612 { 613 u32 val; 614 615 if ((idx != ARMV7_CYCLE_COUNTER) && 616 ((idx < ARMV7_COUNTER0) || (idx > ARMV7_COUNTER_LAST))) { 617 pr_err("CPU%u disabling wrong PMNC counter" 618 " interrupt enable %d\n", smp_processor_id(), idx); 619 return -1; 620 } 621 622 if (idx == ARMV7_CYCLE_COUNTER) 623 val = ARMV7_INTENC_C; 624 else 625 val = ARMV7_INTENC_P(idx); 626 627 asm volatile("mcr p15, 0, %0, c9, c14, 2" : : "r" (val)); 628 629 return idx; 630 } 631 632 static inline u32 armv7_pmnc_getreset_flags(void) 633 { 634 u32 val; 635 636 /* Read */ 637 asm volatile("mrc p15, 0, %0, c9, c12, 3" : "=r" (val)); 638 639 /* Write to clear flags */ 640 val &= ARMV7_FLAG_MASK; 641 asm volatile("mcr p15, 0, %0, c9, c12, 3" : : "r" (val)); 642 643 return val; 644 } 645 646 #ifdef DEBUG 647 static void armv7_pmnc_dump_regs(void) 648 { 649 u32 val; 650 unsigned int cnt; 651 652 printk(KERN_INFO "PMNC registers dump:\n"); 653 654 asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r" (val)); 655 printk(KERN_INFO "PMNC =0x%08x\n", val); 656 657 asm volatile("mrc p15, 0, %0, c9, c12, 1" : "=r" (val)); 658 printk(KERN_INFO "CNTENS=0x%08x\n", val); 659 660 asm volatile("mrc p15, 0, %0, c9, c14, 1" : "=r" (val)); 661 printk(KERN_INFO "INTENS=0x%08x\n", val); 662 663 asm volatile("mrc p15, 0, %0, c9, c12, 3" : "=r" (val)); 664 printk(KERN_INFO "FLAGS =0x%08x\n", val); 665 666 asm volatile("mrc p15, 0, %0, c9, c12, 5" : "=r" (val)); 667 printk(KERN_INFO "SELECT=0x%08x\n", val); 668 669 asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (val)); 670 printk(KERN_INFO "CCNT =0x%08x\n", val); 671 672 for (cnt = ARMV7_COUNTER0; cnt < ARMV7_COUNTER_LAST; cnt++) { 673 armv7_pmnc_select_counter(cnt); 674 asm volatile("mrc p15, 0, %0, c9, c13, 2" : "=r" (val)); 675 printk(KERN_INFO "CNT[%d] count =0x%08x\n", 676 cnt-ARMV7_EVENT_CNT_TO_CNTx, val); 677 asm volatile("mrc p15, 0, %0, c9, c13, 1" : "=r" (val)); 678 printk(KERN_INFO "CNT[%d] evtsel=0x%08x\n", 679 cnt-ARMV7_EVENT_CNT_TO_CNTx, val); 680 } 681 } 682 #endif 683 684 static void armv7pmu_enable_event(struct hw_perf_event *hwc, int idx) 685 { 686 unsigned long flags; 687 688 /* 689 * Enable counter and interrupt, and set the counter to count 690 * the event that we're interested in. 691 */ 692 raw_spin_lock_irqsave(&pmu_lock, flags); 693 694 /* 695 * Disable counter 696 */ 697 armv7_pmnc_disable_counter(idx); 698 699 /* 700 * Set event (if destined for PMNx counters) 701 * We don't need to set the event if it's a cycle count 702 */ 703 if (idx != ARMV7_CYCLE_COUNTER) 704 armv7_pmnc_write_evtsel(idx, hwc->config_base); 705 706 /* 707 * Enable interrupt for this counter 708 */ 709 armv7_pmnc_enable_intens(idx); 710 711 /* 712 * Enable counter 713 */ 714 armv7_pmnc_enable_counter(idx); 715 716 raw_spin_unlock_irqrestore(&pmu_lock, flags); 717 } 718 719 static void armv7pmu_disable_event(struct hw_perf_event *hwc, int idx) 720 { 721 unsigned long flags; 722 723 /* 724 * Disable counter and interrupt 725 */ 726 raw_spin_lock_irqsave(&pmu_lock, flags); 727 728 /* 729 * Disable counter 730 */ 731 armv7_pmnc_disable_counter(idx); 732 733 /* 734 * Disable interrupt for this counter 735 */ 736 armv7_pmnc_disable_intens(idx); 737 738 raw_spin_unlock_irqrestore(&pmu_lock, flags); 739 } 740 741 static irqreturn_t armv7pmu_handle_irq(int irq_num, void *dev) 742 { 743 unsigned long pmnc; 744 struct perf_sample_data data; 745 struct cpu_hw_events *cpuc; 746 struct pt_regs *regs; 747 int idx; 748 749 /* 750 * Get and reset the IRQ flags 751 */ 752 pmnc = armv7_pmnc_getreset_flags(); 753 754 /* 755 * Did an overflow occur? 756 */ 757 if (!armv7_pmnc_has_overflowed(pmnc)) 758 return IRQ_NONE; 759 760 /* 761 * Handle the counter(s) overflow(s) 762 */ 763 regs = get_irq_regs(); 764 765 perf_sample_data_init(&data, 0); 766 767 cpuc = &__get_cpu_var(cpu_hw_events); 768 for (idx = 0; idx <= armpmu->num_events; ++idx) { 769 struct perf_event *event = cpuc->events[idx]; 770 struct hw_perf_event *hwc; 771 772 if (!test_bit(idx, cpuc->active_mask)) 773 continue; 774 775 /* 776 * We have a single interrupt for all counters. Check that 777 * each counter has overflowed before we process it. 778 */ 779 if (!armv7_pmnc_counter_has_overflowed(pmnc, idx)) 780 continue; 781 782 hwc = &event->hw; 783 armpmu_event_update(event, hwc, idx); 784 data.period = event->hw.last_period; 785 if (!armpmu_event_set_period(event, hwc, idx)) 786 continue; 787 788 if (perf_event_overflow(event, 0, &data, regs)) 789 armpmu->disable(hwc, idx); 790 } 791 792 /* 793 * Handle the pending perf events. 794 * 795 * Note: this call *must* be run with interrupts disabled. For 796 * platforms that can have the PMU interrupts raised as an NMI, this 797 * will not work. 798 */ 799 irq_work_run(); 800 801 return IRQ_HANDLED; 802 } 803 804 static void armv7pmu_start(void) 805 { 806 unsigned long flags; 807 808 raw_spin_lock_irqsave(&pmu_lock, flags); 809 /* Enable all counters */ 810 armv7_pmnc_write(armv7_pmnc_read() | ARMV7_PMNC_E); 811 raw_spin_unlock_irqrestore(&pmu_lock, flags); 812 } 813 814 static void armv7pmu_stop(void) 815 { 816 unsigned long flags; 817 818 raw_spin_lock_irqsave(&pmu_lock, flags); 819 /* Disable all counters */ 820 armv7_pmnc_write(armv7_pmnc_read() & ~ARMV7_PMNC_E); 821 raw_spin_unlock_irqrestore(&pmu_lock, flags); 822 } 823 824 static int armv7pmu_get_event_idx(struct cpu_hw_events *cpuc, 825 struct hw_perf_event *event) 826 { 827 int idx; 828 829 /* Always place a cycle counter into the cycle counter. */ 830 if (event->config_base == ARMV7_PERFCTR_CPU_CYCLES) { 831 if (test_and_set_bit(ARMV7_CYCLE_COUNTER, cpuc->used_mask)) 832 return -EAGAIN; 833 834 return ARMV7_CYCLE_COUNTER; 835 } else { 836 /* 837 * For anything other than a cycle counter, try and use 838 * the events counters 839 */ 840 for (idx = ARMV7_COUNTER0; idx <= armpmu->num_events; ++idx) { 841 if (!test_and_set_bit(idx, cpuc->used_mask)) 842 return idx; 843 } 844 845 /* The counters are all in use. */ 846 return -EAGAIN; 847 } 848 } 849 850 static struct arm_pmu armv7pmu = { 851 .handle_irq = armv7pmu_handle_irq, 852 .enable = armv7pmu_enable_event, 853 .disable = armv7pmu_disable_event, 854 .read_counter = armv7pmu_read_counter, 855 .write_counter = armv7pmu_write_counter, 856 .get_event_idx = armv7pmu_get_event_idx, 857 .start = armv7pmu_start, 858 .stop = armv7pmu_stop, 859 .raw_event_mask = 0xFF, 860 .max_period = (1LLU << 32) - 1, 861 }; 862 863 static u32 __init armv7_reset_read_pmnc(void) 864 { 865 u32 nb_cnt; 866 867 /* Initialize & Reset PMNC: C and P bits */ 868 armv7_pmnc_write(ARMV7_PMNC_P | ARMV7_PMNC_C); 869 870 /* Read the nb of CNTx counters supported from PMNC */ 871 nb_cnt = (armv7_pmnc_read() >> ARMV7_PMNC_N_SHIFT) & ARMV7_PMNC_N_MASK; 872 873 /* Add the CPU cycles counter and return */ 874 return nb_cnt + 1; 875 } 876 877 static const struct arm_pmu *__init armv7_a8_pmu_init(void) 878 { 879 armv7pmu.id = ARM_PERF_PMU_ID_CA8; 880 armv7pmu.name = "ARMv7 Cortex-A8"; 881 armv7pmu.cache_map = &armv7_a8_perf_cache_map; 882 armv7pmu.event_map = &armv7_a8_perf_map; 883 armv7pmu.num_events = armv7_reset_read_pmnc(); 884 return &armv7pmu; 885 } 886 887 static const struct arm_pmu *__init armv7_a9_pmu_init(void) 888 { 889 armv7pmu.id = ARM_PERF_PMU_ID_CA9; 890 armv7pmu.name = "ARMv7 Cortex-A9"; 891 armv7pmu.cache_map = &armv7_a9_perf_cache_map; 892 armv7pmu.event_map = &armv7_a9_perf_map; 893 armv7pmu.num_events = armv7_reset_read_pmnc(); 894 return &armv7pmu; 895 } 896 #else 897 static const struct arm_pmu *__init armv7_a8_pmu_init(void) 898 { 899 return NULL; 900 } 901 902 static const struct arm_pmu *__init armv7_a9_pmu_init(void) 903 { 904 return NULL; 905 } 906 #endif /* CONFIG_CPU_V7 */ 907