1 /* 2 * ARMv7 Cortex-A8 and Cortex-A9 Performance Events handling code. 3 * 4 * ARMv7 support: Jean Pihet <jpihet@mvista.com> 5 * 2010 (c) MontaVista Software, LLC. 6 * 7 * Copied from ARMv6 code, with the low level code inspired 8 * by the ARMv7 Oprofile code. 9 * 10 * Cortex-A8 has up to 4 configurable performance counters and 11 * a single cycle counter. 12 * Cortex-A9 has up to 31 configurable performance counters and 13 * a single cycle counter. 14 * 15 * All counters can be enabled/disabled and IRQ masked separately. The cycle 16 * counter and all 4 performance counters together can be reset separately. 17 */ 18 19 #ifdef CONFIG_CPU_V7 20 /* Common ARMv7 event types */ 21 enum armv7_perf_types { 22 ARMV7_PERFCTR_PMNC_SW_INCR = 0x00, 23 ARMV7_PERFCTR_IFETCH_MISS = 0x01, 24 ARMV7_PERFCTR_ITLB_MISS = 0x02, 25 ARMV7_PERFCTR_DCACHE_REFILL = 0x03, 26 ARMV7_PERFCTR_DCACHE_ACCESS = 0x04, 27 ARMV7_PERFCTR_DTLB_REFILL = 0x05, 28 ARMV7_PERFCTR_DREAD = 0x06, 29 ARMV7_PERFCTR_DWRITE = 0x07, 30 31 ARMV7_PERFCTR_EXC_TAKEN = 0x09, 32 ARMV7_PERFCTR_EXC_EXECUTED = 0x0A, 33 ARMV7_PERFCTR_CID_WRITE = 0x0B, 34 /* ARMV7_PERFCTR_PC_WRITE is equivalent to HW_BRANCH_INSTRUCTIONS. 35 * It counts: 36 * - all branch instructions, 37 * - instructions that explicitly write the PC, 38 * - exception generating instructions. 39 */ 40 ARMV7_PERFCTR_PC_WRITE = 0x0C, 41 ARMV7_PERFCTR_PC_IMM_BRANCH = 0x0D, 42 ARMV7_PERFCTR_UNALIGNED_ACCESS = 0x0F, 43 ARMV7_PERFCTR_PC_BRANCH_MIS_PRED = 0x10, 44 ARMV7_PERFCTR_CLOCK_CYCLES = 0x11, 45 46 ARMV7_PERFCTR_PC_BRANCH_MIS_USED = 0x12, 47 48 ARMV7_PERFCTR_CPU_CYCLES = 0xFF 49 }; 50 51 /* ARMv7 Cortex-A8 specific event types */ 52 enum armv7_a8_perf_types { 53 ARMV7_PERFCTR_INSTR_EXECUTED = 0x08, 54 55 ARMV7_PERFCTR_PC_PROC_RETURN = 0x0E, 56 57 ARMV7_PERFCTR_WRITE_BUFFER_FULL = 0x40, 58 ARMV7_PERFCTR_L2_STORE_MERGED = 0x41, 59 ARMV7_PERFCTR_L2_STORE_BUFF = 0x42, 60 ARMV7_PERFCTR_L2_ACCESS = 0x43, 61 ARMV7_PERFCTR_L2_CACH_MISS = 0x44, 62 ARMV7_PERFCTR_AXI_READ_CYCLES = 0x45, 63 ARMV7_PERFCTR_AXI_WRITE_CYCLES = 0x46, 64 ARMV7_PERFCTR_MEMORY_REPLAY = 0x47, 65 ARMV7_PERFCTR_UNALIGNED_ACCESS_REPLAY = 0x48, 66 ARMV7_PERFCTR_L1_DATA_MISS = 0x49, 67 ARMV7_PERFCTR_L1_INST_MISS = 0x4A, 68 ARMV7_PERFCTR_L1_DATA_COLORING = 0x4B, 69 ARMV7_PERFCTR_L1_NEON_DATA = 0x4C, 70 ARMV7_PERFCTR_L1_NEON_CACH_DATA = 0x4D, 71 ARMV7_PERFCTR_L2_NEON = 0x4E, 72 ARMV7_PERFCTR_L2_NEON_HIT = 0x4F, 73 ARMV7_PERFCTR_L1_INST = 0x50, 74 ARMV7_PERFCTR_PC_RETURN_MIS_PRED = 0x51, 75 ARMV7_PERFCTR_PC_BRANCH_FAILED = 0x52, 76 ARMV7_PERFCTR_PC_BRANCH_TAKEN = 0x53, 77 ARMV7_PERFCTR_PC_BRANCH_EXECUTED = 0x54, 78 ARMV7_PERFCTR_OP_EXECUTED = 0x55, 79 ARMV7_PERFCTR_CYCLES_INST_STALL = 0x56, 80 ARMV7_PERFCTR_CYCLES_INST = 0x57, 81 ARMV7_PERFCTR_CYCLES_NEON_DATA_STALL = 0x58, 82 ARMV7_PERFCTR_CYCLES_NEON_INST_STALL = 0x59, 83 ARMV7_PERFCTR_NEON_CYCLES = 0x5A, 84 85 ARMV7_PERFCTR_PMU0_EVENTS = 0x70, 86 ARMV7_PERFCTR_PMU1_EVENTS = 0x71, 87 ARMV7_PERFCTR_PMU_EVENTS = 0x72, 88 }; 89 90 /* ARMv7 Cortex-A9 specific event types */ 91 enum armv7_a9_perf_types { 92 ARMV7_PERFCTR_JAVA_HW_BYTECODE_EXEC = 0x40, 93 ARMV7_PERFCTR_JAVA_SW_BYTECODE_EXEC = 0x41, 94 ARMV7_PERFCTR_JAZELLE_BRANCH_EXEC = 0x42, 95 96 ARMV7_PERFCTR_COHERENT_LINE_MISS = 0x50, 97 ARMV7_PERFCTR_COHERENT_LINE_HIT = 0x51, 98 99 ARMV7_PERFCTR_ICACHE_DEP_STALL_CYCLES = 0x60, 100 ARMV7_PERFCTR_DCACHE_DEP_STALL_CYCLES = 0x61, 101 ARMV7_PERFCTR_TLB_MISS_DEP_STALL_CYCLES = 0x62, 102 ARMV7_PERFCTR_STREX_EXECUTED_PASSED = 0x63, 103 ARMV7_PERFCTR_STREX_EXECUTED_FAILED = 0x64, 104 ARMV7_PERFCTR_DATA_EVICTION = 0x65, 105 ARMV7_PERFCTR_ISSUE_STAGE_NO_INST = 0x66, 106 ARMV7_PERFCTR_ISSUE_STAGE_EMPTY = 0x67, 107 ARMV7_PERFCTR_INST_OUT_OF_RENAME_STAGE = 0x68, 108 109 ARMV7_PERFCTR_PREDICTABLE_FUNCT_RETURNS = 0x6E, 110 111 ARMV7_PERFCTR_MAIN_UNIT_EXECUTED_INST = 0x70, 112 ARMV7_PERFCTR_SECOND_UNIT_EXECUTED_INST = 0x71, 113 ARMV7_PERFCTR_LD_ST_UNIT_EXECUTED_INST = 0x72, 114 ARMV7_PERFCTR_FP_EXECUTED_INST = 0x73, 115 ARMV7_PERFCTR_NEON_EXECUTED_INST = 0x74, 116 117 ARMV7_PERFCTR_PLD_FULL_DEP_STALL_CYCLES = 0x80, 118 ARMV7_PERFCTR_DATA_WR_DEP_STALL_CYCLES = 0x81, 119 ARMV7_PERFCTR_ITLB_MISS_DEP_STALL_CYCLES = 0x82, 120 ARMV7_PERFCTR_DTLB_MISS_DEP_STALL_CYCLES = 0x83, 121 ARMV7_PERFCTR_MICRO_ITLB_MISS_DEP_STALL_CYCLES = 0x84, 122 ARMV7_PERFCTR_MICRO_DTLB_MISS_DEP_STALL_CYCLES = 0x85, 123 ARMV7_PERFCTR_DMB_DEP_STALL_CYCLES = 0x86, 124 125 ARMV7_PERFCTR_INTGR_CLK_ENABLED_CYCLES = 0x8A, 126 ARMV7_PERFCTR_DATA_ENGINE_CLK_EN_CYCLES = 0x8B, 127 128 ARMV7_PERFCTR_ISB_INST = 0x90, 129 ARMV7_PERFCTR_DSB_INST = 0x91, 130 ARMV7_PERFCTR_DMB_INST = 0x92, 131 ARMV7_PERFCTR_EXT_INTERRUPTS = 0x93, 132 133 ARMV7_PERFCTR_PLE_CACHE_LINE_RQST_COMPLETED = 0xA0, 134 ARMV7_PERFCTR_PLE_CACHE_LINE_RQST_SKIPPED = 0xA1, 135 ARMV7_PERFCTR_PLE_FIFO_FLUSH = 0xA2, 136 ARMV7_PERFCTR_PLE_RQST_COMPLETED = 0xA3, 137 ARMV7_PERFCTR_PLE_FIFO_OVERFLOW = 0xA4, 138 ARMV7_PERFCTR_PLE_RQST_PROG = 0xA5 139 }; 140 141 /* 142 * Cortex-A8 HW events mapping 143 * 144 * The hardware events that we support. We do support cache operations but 145 * we have harvard caches and no way to combine instruction and data 146 * accesses/misses in hardware. 147 */ 148 static const unsigned armv7_a8_perf_map[PERF_COUNT_HW_MAX] = { 149 [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES, 150 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED, 151 [PERF_COUNT_HW_CACHE_REFERENCES] = HW_OP_UNSUPPORTED, 152 [PERF_COUNT_HW_CACHE_MISSES] = HW_OP_UNSUPPORTED, 153 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE, 154 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, 155 [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_CLOCK_CYCLES, 156 }; 157 158 static const unsigned armv7_a8_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] 159 [PERF_COUNT_HW_CACHE_OP_MAX] 160 [PERF_COUNT_HW_CACHE_RESULT_MAX] = { 161 [C(L1D)] = { 162 /* 163 * The performance counters don't differentiate between read 164 * and write accesses/misses so this isn't strictly correct, 165 * but it's the best we can do. Writes and reads get 166 * combined. 167 */ 168 [C(OP_READ)] = { 169 [C(RESULT_ACCESS)] = ARMV7_PERFCTR_DCACHE_ACCESS, 170 [C(RESULT_MISS)] = ARMV7_PERFCTR_DCACHE_REFILL, 171 }, 172 [C(OP_WRITE)] = { 173 [C(RESULT_ACCESS)] = ARMV7_PERFCTR_DCACHE_ACCESS, 174 [C(RESULT_MISS)] = ARMV7_PERFCTR_DCACHE_REFILL, 175 }, 176 [C(OP_PREFETCH)] = { 177 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 178 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, 179 }, 180 }, 181 [C(L1I)] = { 182 [C(OP_READ)] = { 183 [C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_INST, 184 [C(RESULT_MISS)] = ARMV7_PERFCTR_L1_INST_MISS, 185 }, 186 [C(OP_WRITE)] = { 187 [C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_INST, 188 [C(RESULT_MISS)] = ARMV7_PERFCTR_L1_INST_MISS, 189 }, 190 [C(OP_PREFETCH)] = { 191 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 192 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, 193 }, 194 }, 195 [C(LL)] = { 196 [C(OP_READ)] = { 197 [C(RESULT_ACCESS)] = ARMV7_PERFCTR_L2_ACCESS, 198 [C(RESULT_MISS)] = ARMV7_PERFCTR_L2_CACH_MISS, 199 }, 200 [C(OP_WRITE)] = { 201 [C(RESULT_ACCESS)] = ARMV7_PERFCTR_L2_ACCESS, 202 [C(RESULT_MISS)] = ARMV7_PERFCTR_L2_CACH_MISS, 203 }, 204 [C(OP_PREFETCH)] = { 205 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 206 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, 207 }, 208 }, 209 [C(DTLB)] = { 210 /* 211 * Only ITLB misses and DTLB refills are supported. 212 * If users want the DTLB refills misses a raw counter 213 * must be used. 214 */ 215 [C(OP_READ)] = { 216 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 217 [C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL, 218 }, 219 [C(OP_WRITE)] = { 220 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 221 [C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL, 222 }, 223 [C(OP_PREFETCH)] = { 224 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 225 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, 226 }, 227 }, 228 [C(ITLB)] = { 229 [C(OP_READ)] = { 230 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 231 [C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_MISS, 232 }, 233 [C(OP_WRITE)] = { 234 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 235 [C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_MISS, 236 }, 237 [C(OP_PREFETCH)] = { 238 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 239 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, 240 }, 241 }, 242 [C(BPU)] = { 243 [C(OP_READ)] = { 244 [C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_WRITE, 245 [C(RESULT_MISS)] 246 = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, 247 }, 248 [C(OP_WRITE)] = { 249 [C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_WRITE, 250 [C(RESULT_MISS)] 251 = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, 252 }, 253 [C(OP_PREFETCH)] = { 254 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 255 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, 256 }, 257 }, 258 [C(NODE)] = { 259 [C(OP_READ)] = { 260 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 261 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, 262 }, 263 [C(OP_WRITE)] = { 264 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 265 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, 266 }, 267 [C(OP_PREFETCH)] = { 268 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 269 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, 270 }, 271 }, 272 }; 273 274 /* 275 * Cortex-A9 HW events mapping 276 */ 277 static const unsigned armv7_a9_perf_map[PERF_COUNT_HW_MAX] = { 278 [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES, 279 [PERF_COUNT_HW_INSTRUCTIONS] = 280 ARMV7_PERFCTR_INST_OUT_OF_RENAME_STAGE, 281 [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_COHERENT_LINE_HIT, 282 [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_COHERENT_LINE_MISS, 283 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE, 284 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, 285 [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_CLOCK_CYCLES, 286 }; 287 288 static const unsigned armv7_a9_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] 289 [PERF_COUNT_HW_CACHE_OP_MAX] 290 [PERF_COUNT_HW_CACHE_RESULT_MAX] = { 291 [C(L1D)] = { 292 /* 293 * The performance counters don't differentiate between read 294 * and write accesses/misses so this isn't strictly correct, 295 * but it's the best we can do. Writes and reads get 296 * combined. 297 */ 298 [C(OP_READ)] = { 299 [C(RESULT_ACCESS)] = ARMV7_PERFCTR_DCACHE_ACCESS, 300 [C(RESULT_MISS)] = ARMV7_PERFCTR_DCACHE_REFILL, 301 }, 302 [C(OP_WRITE)] = { 303 [C(RESULT_ACCESS)] = ARMV7_PERFCTR_DCACHE_ACCESS, 304 [C(RESULT_MISS)] = ARMV7_PERFCTR_DCACHE_REFILL, 305 }, 306 [C(OP_PREFETCH)] = { 307 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 308 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, 309 }, 310 }, 311 [C(L1I)] = { 312 [C(OP_READ)] = { 313 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 314 [C(RESULT_MISS)] = ARMV7_PERFCTR_IFETCH_MISS, 315 }, 316 [C(OP_WRITE)] = { 317 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 318 [C(RESULT_MISS)] = ARMV7_PERFCTR_IFETCH_MISS, 319 }, 320 [C(OP_PREFETCH)] = { 321 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 322 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, 323 }, 324 }, 325 [C(LL)] = { 326 [C(OP_READ)] = { 327 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 328 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, 329 }, 330 [C(OP_WRITE)] = { 331 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 332 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, 333 }, 334 [C(OP_PREFETCH)] = { 335 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 336 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, 337 }, 338 }, 339 [C(DTLB)] = { 340 /* 341 * Only ITLB misses and DTLB refills are supported. 342 * If users want the DTLB refills misses a raw counter 343 * must be used. 344 */ 345 [C(OP_READ)] = { 346 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 347 [C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL, 348 }, 349 [C(OP_WRITE)] = { 350 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 351 [C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL, 352 }, 353 [C(OP_PREFETCH)] = { 354 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 355 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, 356 }, 357 }, 358 [C(ITLB)] = { 359 [C(OP_READ)] = { 360 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 361 [C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_MISS, 362 }, 363 [C(OP_WRITE)] = { 364 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 365 [C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_MISS, 366 }, 367 [C(OP_PREFETCH)] = { 368 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 369 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, 370 }, 371 }, 372 [C(BPU)] = { 373 [C(OP_READ)] = { 374 [C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_WRITE, 375 [C(RESULT_MISS)] 376 = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, 377 }, 378 [C(OP_WRITE)] = { 379 [C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_WRITE, 380 [C(RESULT_MISS)] 381 = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, 382 }, 383 [C(OP_PREFETCH)] = { 384 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 385 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, 386 }, 387 }, 388 [C(NODE)] = { 389 [C(OP_READ)] = { 390 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 391 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, 392 }, 393 [C(OP_WRITE)] = { 394 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 395 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, 396 }, 397 [C(OP_PREFETCH)] = { 398 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, 399 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, 400 }, 401 }, 402 }; 403 404 /* 405 * Perf Events counters 406 */ 407 enum armv7_counters { 408 ARMV7_CYCLE_COUNTER = 1, /* Cycle counter */ 409 ARMV7_COUNTER0 = 2, /* First event counter */ 410 }; 411 412 /* 413 * The cycle counter is ARMV7_CYCLE_COUNTER. 414 * The first event counter is ARMV7_COUNTER0. 415 * The last event counter is (ARMV7_COUNTER0 + armpmu->num_events - 1). 416 */ 417 #define ARMV7_COUNTER_LAST (ARMV7_COUNTER0 + armpmu->num_events - 1) 418 419 /* 420 * ARMv7 low level PMNC access 421 */ 422 423 /* 424 * Per-CPU PMNC: config reg 425 */ 426 #define ARMV7_PMNC_E (1 << 0) /* Enable all counters */ 427 #define ARMV7_PMNC_P (1 << 1) /* Reset all counters */ 428 #define ARMV7_PMNC_C (1 << 2) /* Cycle counter reset */ 429 #define ARMV7_PMNC_D (1 << 3) /* CCNT counts every 64th cpu cycle */ 430 #define ARMV7_PMNC_X (1 << 4) /* Export to ETM */ 431 #define ARMV7_PMNC_DP (1 << 5) /* Disable CCNT if non-invasive debug*/ 432 #define ARMV7_PMNC_N_SHIFT 11 /* Number of counters supported */ 433 #define ARMV7_PMNC_N_MASK 0x1f 434 #define ARMV7_PMNC_MASK 0x3f /* Mask for writable bits */ 435 436 /* 437 * Available counters 438 */ 439 #define ARMV7_CNT0 0 /* First event counter */ 440 #define ARMV7_CCNT 31 /* Cycle counter */ 441 442 /* Perf Event to low level counters mapping */ 443 #define ARMV7_EVENT_CNT_TO_CNTx (ARMV7_COUNTER0 - ARMV7_CNT0) 444 445 /* 446 * CNTENS: counters enable reg 447 */ 448 #define ARMV7_CNTENS_P(idx) (1 << (idx - ARMV7_EVENT_CNT_TO_CNTx)) 449 #define ARMV7_CNTENS_C (1 << ARMV7_CCNT) 450 451 /* 452 * CNTENC: counters disable reg 453 */ 454 #define ARMV7_CNTENC_P(idx) (1 << (idx - ARMV7_EVENT_CNT_TO_CNTx)) 455 #define ARMV7_CNTENC_C (1 << ARMV7_CCNT) 456 457 /* 458 * INTENS: counters overflow interrupt enable reg 459 */ 460 #define ARMV7_INTENS_P(idx) (1 << (idx - ARMV7_EVENT_CNT_TO_CNTx)) 461 #define ARMV7_INTENS_C (1 << ARMV7_CCNT) 462 463 /* 464 * INTENC: counters overflow interrupt disable reg 465 */ 466 #define ARMV7_INTENC_P(idx) (1 << (idx - ARMV7_EVENT_CNT_TO_CNTx)) 467 #define ARMV7_INTENC_C (1 << ARMV7_CCNT) 468 469 /* 470 * EVTSEL: Event selection reg 471 */ 472 #define ARMV7_EVTSEL_MASK 0xff /* Mask for writable bits */ 473 474 /* 475 * SELECT: Counter selection reg 476 */ 477 #define ARMV7_SELECT_MASK 0x1f /* Mask for writable bits */ 478 479 /* 480 * FLAG: counters overflow flag status reg 481 */ 482 #define ARMV7_FLAG_P(idx) (1 << (idx - ARMV7_EVENT_CNT_TO_CNTx)) 483 #define ARMV7_FLAG_C (1 << ARMV7_CCNT) 484 #define ARMV7_FLAG_MASK 0xffffffff /* Mask for writable bits */ 485 #define ARMV7_OVERFLOWED_MASK ARMV7_FLAG_MASK 486 487 static inline unsigned long armv7_pmnc_read(void) 488 { 489 u32 val; 490 asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r"(val)); 491 return val; 492 } 493 494 static inline void armv7_pmnc_write(unsigned long val) 495 { 496 val &= ARMV7_PMNC_MASK; 497 isb(); 498 asm volatile("mcr p15, 0, %0, c9, c12, 0" : : "r"(val)); 499 } 500 501 static inline int armv7_pmnc_has_overflowed(unsigned long pmnc) 502 { 503 return pmnc & ARMV7_OVERFLOWED_MASK; 504 } 505 506 static inline int armv7_pmnc_counter_has_overflowed(unsigned long pmnc, 507 enum armv7_counters counter) 508 { 509 int ret = 0; 510 511 if (counter == ARMV7_CYCLE_COUNTER) 512 ret = pmnc & ARMV7_FLAG_C; 513 else if ((counter >= ARMV7_COUNTER0) && (counter <= ARMV7_COUNTER_LAST)) 514 ret = pmnc & ARMV7_FLAG_P(counter); 515 else 516 pr_err("CPU%u checking wrong counter %d overflow status\n", 517 smp_processor_id(), counter); 518 519 return ret; 520 } 521 522 static inline int armv7_pmnc_select_counter(unsigned int idx) 523 { 524 u32 val; 525 526 if ((idx < ARMV7_COUNTER0) || (idx > ARMV7_COUNTER_LAST)) { 527 pr_err("CPU%u selecting wrong PMNC counter" 528 " %d\n", smp_processor_id(), idx); 529 return -1; 530 } 531 532 val = (idx - ARMV7_EVENT_CNT_TO_CNTx) & ARMV7_SELECT_MASK; 533 asm volatile("mcr p15, 0, %0, c9, c12, 5" : : "r" (val)); 534 isb(); 535 536 return idx; 537 } 538 539 static inline u32 armv7pmu_read_counter(int idx) 540 { 541 unsigned long value = 0; 542 543 if (idx == ARMV7_CYCLE_COUNTER) 544 asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (value)); 545 else if ((idx >= ARMV7_COUNTER0) && (idx <= ARMV7_COUNTER_LAST)) { 546 if (armv7_pmnc_select_counter(idx) == idx) 547 asm volatile("mrc p15, 0, %0, c9, c13, 2" 548 : "=r" (value)); 549 } else 550 pr_err("CPU%u reading wrong counter %d\n", 551 smp_processor_id(), idx); 552 553 return value; 554 } 555 556 static inline void armv7pmu_write_counter(int idx, u32 value) 557 { 558 if (idx == ARMV7_CYCLE_COUNTER) 559 asm volatile("mcr p15, 0, %0, c9, c13, 0" : : "r" (value)); 560 else if ((idx >= ARMV7_COUNTER0) && (idx <= ARMV7_COUNTER_LAST)) { 561 if (armv7_pmnc_select_counter(idx) == idx) 562 asm volatile("mcr p15, 0, %0, c9, c13, 2" 563 : : "r" (value)); 564 } else 565 pr_err("CPU%u writing wrong counter %d\n", 566 smp_processor_id(), idx); 567 } 568 569 static inline void armv7_pmnc_write_evtsel(unsigned int idx, u32 val) 570 { 571 if (armv7_pmnc_select_counter(idx) == idx) { 572 val &= ARMV7_EVTSEL_MASK; 573 asm volatile("mcr p15, 0, %0, c9, c13, 1" : : "r" (val)); 574 } 575 } 576 577 static inline u32 armv7_pmnc_enable_counter(unsigned int idx) 578 { 579 u32 val; 580 581 if ((idx != ARMV7_CYCLE_COUNTER) && 582 ((idx < ARMV7_COUNTER0) || (idx > ARMV7_COUNTER_LAST))) { 583 pr_err("CPU%u enabling wrong PMNC counter" 584 " %d\n", smp_processor_id(), idx); 585 return -1; 586 } 587 588 if (idx == ARMV7_CYCLE_COUNTER) 589 val = ARMV7_CNTENS_C; 590 else 591 val = ARMV7_CNTENS_P(idx); 592 593 asm volatile("mcr p15, 0, %0, c9, c12, 1" : : "r" (val)); 594 595 return idx; 596 } 597 598 static inline u32 armv7_pmnc_disable_counter(unsigned int idx) 599 { 600 u32 val; 601 602 603 if ((idx != ARMV7_CYCLE_COUNTER) && 604 ((idx < ARMV7_COUNTER0) || (idx > ARMV7_COUNTER_LAST))) { 605 pr_err("CPU%u disabling wrong PMNC counter" 606 " %d\n", smp_processor_id(), idx); 607 return -1; 608 } 609 610 if (idx == ARMV7_CYCLE_COUNTER) 611 val = ARMV7_CNTENC_C; 612 else 613 val = ARMV7_CNTENC_P(idx); 614 615 asm volatile("mcr p15, 0, %0, c9, c12, 2" : : "r" (val)); 616 617 return idx; 618 } 619 620 static inline u32 armv7_pmnc_enable_intens(unsigned int idx) 621 { 622 u32 val; 623 624 if ((idx != ARMV7_CYCLE_COUNTER) && 625 ((idx < ARMV7_COUNTER0) || (idx > ARMV7_COUNTER_LAST))) { 626 pr_err("CPU%u enabling wrong PMNC counter" 627 " interrupt enable %d\n", smp_processor_id(), idx); 628 return -1; 629 } 630 631 if (idx == ARMV7_CYCLE_COUNTER) 632 val = ARMV7_INTENS_C; 633 else 634 val = ARMV7_INTENS_P(idx); 635 636 asm volatile("mcr p15, 0, %0, c9, c14, 1" : : "r" (val)); 637 638 return idx; 639 } 640 641 static inline u32 armv7_pmnc_disable_intens(unsigned int idx) 642 { 643 u32 val; 644 645 if ((idx != ARMV7_CYCLE_COUNTER) && 646 ((idx < ARMV7_COUNTER0) || (idx > ARMV7_COUNTER_LAST))) { 647 pr_err("CPU%u disabling wrong PMNC counter" 648 " interrupt enable %d\n", smp_processor_id(), idx); 649 return -1; 650 } 651 652 if (idx == ARMV7_CYCLE_COUNTER) 653 val = ARMV7_INTENC_C; 654 else 655 val = ARMV7_INTENC_P(idx); 656 657 asm volatile("mcr p15, 0, %0, c9, c14, 2" : : "r" (val)); 658 659 return idx; 660 } 661 662 static inline u32 armv7_pmnc_getreset_flags(void) 663 { 664 u32 val; 665 666 /* Read */ 667 asm volatile("mrc p15, 0, %0, c9, c12, 3" : "=r" (val)); 668 669 /* Write to clear flags */ 670 val &= ARMV7_FLAG_MASK; 671 asm volatile("mcr p15, 0, %0, c9, c12, 3" : : "r" (val)); 672 673 return val; 674 } 675 676 #ifdef DEBUG 677 static void armv7_pmnc_dump_regs(void) 678 { 679 u32 val; 680 unsigned int cnt; 681 682 printk(KERN_INFO "PMNC registers dump:\n"); 683 684 asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r" (val)); 685 printk(KERN_INFO "PMNC =0x%08x\n", val); 686 687 asm volatile("mrc p15, 0, %0, c9, c12, 1" : "=r" (val)); 688 printk(KERN_INFO "CNTENS=0x%08x\n", val); 689 690 asm volatile("mrc p15, 0, %0, c9, c14, 1" : "=r" (val)); 691 printk(KERN_INFO "INTENS=0x%08x\n", val); 692 693 asm volatile("mrc p15, 0, %0, c9, c12, 3" : "=r" (val)); 694 printk(KERN_INFO "FLAGS =0x%08x\n", val); 695 696 asm volatile("mrc p15, 0, %0, c9, c12, 5" : "=r" (val)); 697 printk(KERN_INFO "SELECT=0x%08x\n", val); 698 699 asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (val)); 700 printk(KERN_INFO "CCNT =0x%08x\n", val); 701 702 for (cnt = ARMV7_COUNTER0; cnt < ARMV7_COUNTER_LAST; cnt++) { 703 armv7_pmnc_select_counter(cnt); 704 asm volatile("mrc p15, 0, %0, c9, c13, 2" : "=r" (val)); 705 printk(KERN_INFO "CNT[%d] count =0x%08x\n", 706 cnt-ARMV7_EVENT_CNT_TO_CNTx, val); 707 asm volatile("mrc p15, 0, %0, c9, c13, 1" : "=r" (val)); 708 printk(KERN_INFO "CNT[%d] evtsel=0x%08x\n", 709 cnt-ARMV7_EVENT_CNT_TO_CNTx, val); 710 } 711 } 712 #endif 713 714 static void armv7pmu_enable_event(struct hw_perf_event *hwc, int idx) 715 { 716 unsigned long flags; 717 718 /* 719 * Enable counter and interrupt, and set the counter to count 720 * the event that we're interested in. 721 */ 722 raw_spin_lock_irqsave(&pmu_lock, flags); 723 724 /* 725 * Disable counter 726 */ 727 armv7_pmnc_disable_counter(idx); 728 729 /* 730 * Set event (if destined for PMNx counters) 731 * We don't need to set the event if it's a cycle count 732 */ 733 if (idx != ARMV7_CYCLE_COUNTER) 734 armv7_pmnc_write_evtsel(idx, hwc->config_base); 735 736 /* 737 * Enable interrupt for this counter 738 */ 739 armv7_pmnc_enable_intens(idx); 740 741 /* 742 * Enable counter 743 */ 744 armv7_pmnc_enable_counter(idx); 745 746 raw_spin_unlock_irqrestore(&pmu_lock, flags); 747 } 748 749 static void armv7pmu_disable_event(struct hw_perf_event *hwc, int idx) 750 { 751 unsigned long flags; 752 753 /* 754 * Disable counter and interrupt 755 */ 756 raw_spin_lock_irqsave(&pmu_lock, flags); 757 758 /* 759 * Disable counter 760 */ 761 armv7_pmnc_disable_counter(idx); 762 763 /* 764 * Disable interrupt for this counter 765 */ 766 armv7_pmnc_disable_intens(idx); 767 768 raw_spin_unlock_irqrestore(&pmu_lock, flags); 769 } 770 771 static irqreturn_t armv7pmu_handle_irq(int irq_num, void *dev) 772 { 773 unsigned long pmnc; 774 struct perf_sample_data data; 775 struct cpu_hw_events *cpuc; 776 struct pt_regs *regs; 777 int idx; 778 779 /* 780 * Get and reset the IRQ flags 781 */ 782 pmnc = armv7_pmnc_getreset_flags(); 783 784 /* 785 * Did an overflow occur? 786 */ 787 if (!armv7_pmnc_has_overflowed(pmnc)) 788 return IRQ_NONE; 789 790 /* 791 * Handle the counter(s) overflow(s) 792 */ 793 regs = get_irq_regs(); 794 795 perf_sample_data_init(&data, 0); 796 797 cpuc = &__get_cpu_var(cpu_hw_events); 798 for (idx = 0; idx <= armpmu->num_events; ++idx) { 799 struct perf_event *event = cpuc->events[idx]; 800 struct hw_perf_event *hwc; 801 802 if (!test_bit(idx, cpuc->active_mask)) 803 continue; 804 805 /* 806 * We have a single interrupt for all counters. Check that 807 * each counter has overflowed before we process it. 808 */ 809 if (!armv7_pmnc_counter_has_overflowed(pmnc, idx)) 810 continue; 811 812 hwc = &event->hw; 813 armpmu_event_update(event, hwc, idx, 1); 814 data.period = event->hw.last_period; 815 if (!armpmu_event_set_period(event, hwc, idx)) 816 continue; 817 818 if (perf_event_overflow(event, &data, regs)) 819 armpmu->disable(hwc, idx); 820 } 821 822 /* 823 * Handle the pending perf events. 824 * 825 * Note: this call *must* be run with interrupts disabled. For 826 * platforms that can have the PMU interrupts raised as an NMI, this 827 * will not work. 828 */ 829 irq_work_run(); 830 831 return IRQ_HANDLED; 832 } 833 834 static void armv7pmu_start(void) 835 { 836 unsigned long flags; 837 838 raw_spin_lock_irqsave(&pmu_lock, flags); 839 /* Enable all counters */ 840 armv7_pmnc_write(armv7_pmnc_read() | ARMV7_PMNC_E); 841 raw_spin_unlock_irqrestore(&pmu_lock, flags); 842 } 843 844 static void armv7pmu_stop(void) 845 { 846 unsigned long flags; 847 848 raw_spin_lock_irqsave(&pmu_lock, flags); 849 /* Disable all counters */ 850 armv7_pmnc_write(armv7_pmnc_read() & ~ARMV7_PMNC_E); 851 raw_spin_unlock_irqrestore(&pmu_lock, flags); 852 } 853 854 static int armv7pmu_get_event_idx(struct cpu_hw_events *cpuc, 855 struct hw_perf_event *event) 856 { 857 int idx; 858 859 /* Always place a cycle counter into the cycle counter. */ 860 if (event->config_base == ARMV7_PERFCTR_CPU_CYCLES) { 861 if (test_and_set_bit(ARMV7_CYCLE_COUNTER, cpuc->used_mask)) 862 return -EAGAIN; 863 864 return ARMV7_CYCLE_COUNTER; 865 } else { 866 /* 867 * For anything other than a cycle counter, try and use 868 * the events counters 869 */ 870 for (idx = ARMV7_COUNTER0; idx <= armpmu->num_events; ++idx) { 871 if (!test_and_set_bit(idx, cpuc->used_mask)) 872 return idx; 873 } 874 875 /* The counters are all in use. */ 876 return -EAGAIN; 877 } 878 } 879 880 static void armv7pmu_reset(void *info) 881 { 882 u32 idx, nb_cnt = armpmu->num_events; 883 884 /* The counter and interrupt enable registers are unknown at reset. */ 885 for (idx = 1; idx < nb_cnt; ++idx) 886 armv7pmu_disable_event(NULL, idx); 887 888 /* Initialize & Reset PMNC: C and P bits */ 889 armv7_pmnc_write(ARMV7_PMNC_P | ARMV7_PMNC_C); 890 } 891 892 static struct arm_pmu armv7pmu = { 893 .handle_irq = armv7pmu_handle_irq, 894 .enable = armv7pmu_enable_event, 895 .disable = armv7pmu_disable_event, 896 .read_counter = armv7pmu_read_counter, 897 .write_counter = armv7pmu_write_counter, 898 .get_event_idx = armv7pmu_get_event_idx, 899 .start = armv7pmu_start, 900 .stop = armv7pmu_stop, 901 .reset = armv7pmu_reset, 902 .raw_event_mask = 0xFF, 903 .max_period = (1LLU << 32) - 1, 904 }; 905 906 static u32 __init armv7_read_num_pmnc_events(void) 907 { 908 u32 nb_cnt; 909 910 /* Read the nb of CNTx counters supported from PMNC */ 911 nb_cnt = (armv7_pmnc_read() >> ARMV7_PMNC_N_SHIFT) & ARMV7_PMNC_N_MASK; 912 913 /* Add the CPU cycles counter and return */ 914 return nb_cnt + 1; 915 } 916 917 static const struct arm_pmu *__init armv7_a8_pmu_init(void) 918 { 919 armv7pmu.id = ARM_PERF_PMU_ID_CA8; 920 armv7pmu.name = "ARMv7 Cortex-A8"; 921 armv7pmu.cache_map = &armv7_a8_perf_cache_map; 922 armv7pmu.event_map = &armv7_a8_perf_map; 923 armv7pmu.num_events = armv7_read_num_pmnc_events(); 924 return &armv7pmu; 925 } 926 927 static const struct arm_pmu *__init armv7_a9_pmu_init(void) 928 { 929 armv7pmu.id = ARM_PERF_PMU_ID_CA9; 930 armv7pmu.name = "ARMv7 Cortex-A9"; 931 armv7pmu.cache_map = &armv7_a9_perf_cache_map; 932 armv7pmu.event_map = &armv7_a9_perf_map; 933 armv7pmu.num_events = armv7_read_num_pmnc_events(); 934 return &armv7pmu; 935 } 936 #else 937 static const struct arm_pmu *__init armv7_a8_pmu_init(void) 938 { 939 return NULL; 940 } 941 942 static const struct arm_pmu *__init armv7_a9_pmu_init(void) 943 { 944 return NULL; 945 } 946 #endif /* CONFIG_CPU_V7 */ 947