1 /* 2 * ARMv7 Cortex-A8 and Cortex-A9 Performance Events handling code. 3 * 4 * ARMv7 support: Jean Pihet <jpihet@mvista.com> 5 * 2010 (c) MontaVista Software, LLC. 6 * 7 * Copied from ARMv6 code, with the low level code inspired 8 * by the ARMv7 Oprofile code. 9 * 10 * Cortex-A8 has up to 4 configurable performance counters and 11 * a single cycle counter. 12 * Cortex-A9 has up to 31 configurable performance counters and 13 * a single cycle counter. 14 * 15 * All counters can be enabled/disabled and IRQ masked separately. The cycle 16 * counter and all 4 performance counters together can be reset separately. 17 */ 18 19 #ifdef CONFIG_CPU_V7 20 21 #include <asm/cp15.h> 22 #include <asm/vfp.h> 23 #include "../vfp/vfpinstr.h" 24 25 /* 26 * Common ARMv7 event types 27 * 28 * Note: An implementation may not be able to count all of these events 29 * but the encodings are considered to be `reserved' in the case that 30 * they are not available. 31 */ 32 enum armv7_perf_types { 33 ARMV7_PERFCTR_PMNC_SW_INCR = 0x00, 34 ARMV7_PERFCTR_L1_ICACHE_REFILL = 0x01, 35 ARMV7_PERFCTR_ITLB_REFILL = 0x02, 36 ARMV7_PERFCTR_L1_DCACHE_REFILL = 0x03, 37 ARMV7_PERFCTR_L1_DCACHE_ACCESS = 0x04, 38 ARMV7_PERFCTR_DTLB_REFILL = 0x05, 39 ARMV7_PERFCTR_MEM_READ = 0x06, 40 ARMV7_PERFCTR_MEM_WRITE = 0x07, 41 ARMV7_PERFCTR_INSTR_EXECUTED = 0x08, 42 ARMV7_PERFCTR_EXC_TAKEN = 0x09, 43 ARMV7_PERFCTR_EXC_EXECUTED = 0x0A, 44 ARMV7_PERFCTR_CID_WRITE = 0x0B, 45 46 /* 47 * ARMV7_PERFCTR_PC_WRITE is equivalent to HW_BRANCH_INSTRUCTIONS. 48 * It counts: 49 * - all (taken) branch instructions, 50 * - instructions that explicitly write the PC, 51 * - exception generating instructions. 52 */ 53 ARMV7_PERFCTR_PC_WRITE = 0x0C, 54 ARMV7_PERFCTR_PC_IMM_BRANCH = 0x0D, 55 ARMV7_PERFCTR_PC_PROC_RETURN = 0x0E, 56 ARMV7_PERFCTR_MEM_UNALIGNED_ACCESS = 0x0F, 57 ARMV7_PERFCTR_PC_BRANCH_MIS_PRED = 0x10, 58 ARMV7_PERFCTR_CLOCK_CYCLES = 0x11, 59 ARMV7_PERFCTR_PC_BRANCH_PRED = 0x12, 60 61 /* These events are defined by the PMUv2 supplement (ARM DDI 0457A). */ 62 ARMV7_PERFCTR_MEM_ACCESS = 0x13, 63 ARMV7_PERFCTR_L1_ICACHE_ACCESS = 0x14, 64 ARMV7_PERFCTR_L1_DCACHE_WB = 0x15, 65 ARMV7_PERFCTR_L2_CACHE_ACCESS = 0x16, 66 ARMV7_PERFCTR_L2_CACHE_REFILL = 0x17, 67 ARMV7_PERFCTR_L2_CACHE_WB = 0x18, 68 ARMV7_PERFCTR_BUS_ACCESS = 0x19, 69 ARMV7_PERFCTR_MEM_ERROR = 0x1A, 70 ARMV7_PERFCTR_INSTR_SPEC = 0x1B, 71 ARMV7_PERFCTR_TTBR_WRITE = 0x1C, 72 ARMV7_PERFCTR_BUS_CYCLES = 0x1D, 73 74 ARMV7_PERFCTR_CPU_CYCLES = 0xFF 75 }; 76 77 /* ARMv7 Cortex-A8 specific event types */ 78 enum armv7_a8_perf_types { 79 ARMV7_A8_PERFCTR_L2_CACHE_ACCESS = 0x43, 80 ARMV7_A8_PERFCTR_L2_CACHE_REFILL = 0x44, 81 ARMV7_A8_PERFCTR_L1_ICACHE_ACCESS = 0x50, 82 ARMV7_A8_PERFCTR_STALL_ISIDE = 0x56, 83 }; 84 85 /* ARMv7 Cortex-A9 specific event types */ 86 enum armv7_a9_perf_types { 87 ARMV7_A9_PERFCTR_INSTR_CORE_RENAME = 0x68, 88 ARMV7_A9_PERFCTR_STALL_ICACHE = 0x60, 89 ARMV7_A9_PERFCTR_STALL_DISPATCH = 0x66, 90 }; 91 92 /* ARMv7 Cortex-A5 specific event types */ 93 enum armv7_a5_perf_types { 94 ARMV7_A5_PERFCTR_PREFETCH_LINEFILL = 0xc2, 95 ARMV7_A5_PERFCTR_PREFETCH_LINEFILL_DROP = 0xc3, 96 }; 97 98 /* ARMv7 Cortex-A15 specific event types */ 99 enum armv7_a15_perf_types { 100 ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_READ = 0x40, 101 ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_WRITE = 0x41, 102 ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_READ = 0x42, 103 ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_WRITE = 0x43, 104 105 ARMV7_A15_PERFCTR_DTLB_REFILL_L1_READ = 0x4C, 106 ARMV7_A15_PERFCTR_DTLB_REFILL_L1_WRITE = 0x4D, 107 108 ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_READ = 0x50, 109 ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_WRITE = 0x51, 110 ARMV7_A15_PERFCTR_L2_CACHE_REFILL_READ = 0x52, 111 ARMV7_A15_PERFCTR_L2_CACHE_REFILL_WRITE = 0x53, 112 113 ARMV7_A15_PERFCTR_PC_WRITE_SPEC = 0x76, 114 }; 115 116 /* ARMv7 Cortex-A12 specific event types */ 117 enum armv7_a12_perf_types { 118 ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_READ = 0x40, 119 ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_WRITE = 0x41, 120 121 ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_READ = 0x50, 122 ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_WRITE = 0x51, 123 124 ARMV7_A12_PERFCTR_PC_WRITE_SPEC = 0x76, 125 126 ARMV7_A12_PERFCTR_PF_TLB_REFILL = 0xe7, 127 }; 128 129 /* ARMv7 Krait specific event types */ 130 enum krait_perf_types { 131 KRAIT_PMRESR0_GROUP0 = 0xcc, 132 KRAIT_PMRESR1_GROUP0 = 0xd0, 133 KRAIT_PMRESR2_GROUP0 = 0xd4, 134 KRAIT_VPMRESR0_GROUP0 = 0xd8, 135 136 KRAIT_PERFCTR_L1_ICACHE_ACCESS = 0x10011, 137 KRAIT_PERFCTR_L1_ICACHE_MISS = 0x10010, 138 139 KRAIT_PERFCTR_L1_ITLB_ACCESS = 0x12222, 140 KRAIT_PERFCTR_L1_DTLB_ACCESS = 0x12210, 141 }; 142 143 /* ARMv7 Scorpion specific event types */ 144 enum scorpion_perf_types { 145 SCORPION_LPM0_GROUP0 = 0x4c, 146 SCORPION_LPM1_GROUP0 = 0x50, 147 SCORPION_LPM2_GROUP0 = 0x54, 148 SCORPION_L2LPM_GROUP0 = 0x58, 149 SCORPION_VLPM_GROUP0 = 0x5c, 150 151 SCORPION_ICACHE_ACCESS = 0x10053, 152 SCORPION_ICACHE_MISS = 0x10052, 153 154 SCORPION_DTLB_ACCESS = 0x12013, 155 SCORPION_DTLB_MISS = 0x12012, 156 157 SCORPION_ITLB_MISS = 0x12021, 158 }; 159 160 /* 161 * Cortex-A8 HW events mapping 162 * 163 * The hardware events that we support. We do support cache operations but 164 * we have harvard caches and no way to combine instruction and data 165 * accesses/misses in hardware. 166 */ 167 static const unsigned armv7_a8_perf_map[PERF_COUNT_HW_MAX] = { 168 PERF_MAP_ALL_UNSUPPORTED, 169 [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES, 170 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED, 171 [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, 172 [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_L1_DCACHE_REFILL, 173 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE, 174 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, 175 [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = ARMV7_A8_PERFCTR_STALL_ISIDE, 176 }; 177 178 static const unsigned armv7_a8_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] 179 [PERF_COUNT_HW_CACHE_OP_MAX] 180 [PERF_COUNT_HW_CACHE_RESULT_MAX] = { 181 PERF_CACHE_MAP_ALL_UNSUPPORTED, 182 183 /* 184 * The performance counters don't differentiate between read and write 185 * accesses/misses so this isn't strictly correct, but it's the best we 186 * can do. Writes and reads get combined. 187 */ 188 [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, 189 [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL, 190 [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, 191 [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL, 192 193 [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_A8_PERFCTR_L1_ICACHE_ACCESS, 194 [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL, 195 196 [C(LL)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_A8_PERFCTR_L2_CACHE_ACCESS, 197 [C(LL)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_A8_PERFCTR_L2_CACHE_REFILL, 198 [C(LL)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_A8_PERFCTR_L2_CACHE_ACCESS, 199 [C(LL)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_A8_PERFCTR_L2_CACHE_REFILL, 200 201 [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL, 202 [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL, 203 204 [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL, 205 [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL, 206 207 [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, 208 [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, 209 [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, 210 [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, 211 }; 212 213 /* 214 * Cortex-A9 HW events mapping 215 */ 216 static const unsigned armv7_a9_perf_map[PERF_COUNT_HW_MAX] = { 217 PERF_MAP_ALL_UNSUPPORTED, 218 [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES, 219 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_A9_PERFCTR_INSTR_CORE_RENAME, 220 [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, 221 [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_L1_DCACHE_REFILL, 222 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE, 223 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, 224 [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = ARMV7_A9_PERFCTR_STALL_ICACHE, 225 [PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = ARMV7_A9_PERFCTR_STALL_DISPATCH, 226 }; 227 228 static const unsigned armv7_a9_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] 229 [PERF_COUNT_HW_CACHE_OP_MAX] 230 [PERF_COUNT_HW_CACHE_RESULT_MAX] = { 231 PERF_CACHE_MAP_ALL_UNSUPPORTED, 232 233 /* 234 * The performance counters don't differentiate between read and write 235 * accesses/misses so this isn't strictly correct, but it's the best we 236 * can do. Writes and reads get combined. 237 */ 238 [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, 239 [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL, 240 [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, 241 [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL, 242 243 [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL, 244 245 [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL, 246 [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL, 247 248 [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL, 249 [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL, 250 251 [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, 252 [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, 253 [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, 254 [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, 255 }; 256 257 /* 258 * Cortex-A5 HW events mapping 259 */ 260 static const unsigned armv7_a5_perf_map[PERF_COUNT_HW_MAX] = { 261 PERF_MAP_ALL_UNSUPPORTED, 262 [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES, 263 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED, 264 [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, 265 [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_L1_DCACHE_REFILL, 266 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE, 267 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, 268 }; 269 270 static const unsigned armv7_a5_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] 271 [PERF_COUNT_HW_CACHE_OP_MAX] 272 [PERF_COUNT_HW_CACHE_RESULT_MAX] = { 273 PERF_CACHE_MAP_ALL_UNSUPPORTED, 274 275 [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, 276 [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL, 277 [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, 278 [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL, 279 [C(L1D)][C(OP_PREFETCH)][C(RESULT_ACCESS)] = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL, 280 [C(L1D)][C(OP_PREFETCH)][C(RESULT_MISS)] = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL_DROP, 281 282 [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_ICACHE_ACCESS, 283 [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL, 284 /* 285 * The prefetch counters don't differentiate between the I side and the 286 * D side. 287 */ 288 [C(L1I)][C(OP_PREFETCH)][C(RESULT_ACCESS)] = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL, 289 [C(L1I)][C(OP_PREFETCH)][C(RESULT_MISS)] = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL_DROP, 290 291 [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL, 292 [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL, 293 294 [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL, 295 [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL, 296 297 [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, 298 [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, 299 [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, 300 [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, 301 }; 302 303 /* 304 * Cortex-A15 HW events mapping 305 */ 306 static const unsigned armv7_a15_perf_map[PERF_COUNT_HW_MAX] = { 307 PERF_MAP_ALL_UNSUPPORTED, 308 [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES, 309 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED, 310 [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, 311 [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_L1_DCACHE_REFILL, 312 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_A15_PERFCTR_PC_WRITE_SPEC, 313 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, 314 [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_BUS_CYCLES, 315 }; 316 317 static const unsigned armv7_a15_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] 318 [PERF_COUNT_HW_CACHE_OP_MAX] 319 [PERF_COUNT_HW_CACHE_RESULT_MAX] = { 320 PERF_CACHE_MAP_ALL_UNSUPPORTED, 321 322 [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_READ, 323 [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_READ, 324 [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_WRITE, 325 [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_WRITE, 326 327 /* 328 * Not all performance counters differentiate between read and write 329 * accesses/misses so we're not always strictly correct, but it's the 330 * best we can do. Writes and reads get combined in these cases. 331 */ 332 [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_ICACHE_ACCESS, 333 [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL, 334 335 [C(LL)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_READ, 336 [C(LL)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_A15_PERFCTR_L2_CACHE_REFILL_READ, 337 [C(LL)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_WRITE, 338 [C(LL)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_A15_PERFCTR_L2_CACHE_REFILL_WRITE, 339 340 [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_A15_PERFCTR_DTLB_REFILL_L1_READ, 341 [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_A15_PERFCTR_DTLB_REFILL_L1_WRITE, 342 343 [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL, 344 [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL, 345 346 [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, 347 [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, 348 [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, 349 [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, 350 }; 351 352 /* 353 * Cortex-A7 HW events mapping 354 */ 355 static const unsigned armv7_a7_perf_map[PERF_COUNT_HW_MAX] = { 356 PERF_MAP_ALL_UNSUPPORTED, 357 [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES, 358 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED, 359 [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, 360 [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_L1_DCACHE_REFILL, 361 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE, 362 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, 363 [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_BUS_CYCLES, 364 }; 365 366 static const unsigned armv7_a7_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] 367 [PERF_COUNT_HW_CACHE_OP_MAX] 368 [PERF_COUNT_HW_CACHE_RESULT_MAX] = { 369 PERF_CACHE_MAP_ALL_UNSUPPORTED, 370 371 /* 372 * The performance counters don't differentiate between read and write 373 * accesses/misses so this isn't strictly correct, but it's the best we 374 * can do. Writes and reads get combined. 375 */ 376 [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, 377 [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL, 378 [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, 379 [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL, 380 381 [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_ICACHE_ACCESS, 382 [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL, 383 384 [C(LL)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L2_CACHE_ACCESS, 385 [C(LL)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L2_CACHE_REFILL, 386 [C(LL)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L2_CACHE_ACCESS, 387 [C(LL)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L2_CACHE_REFILL, 388 389 [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL, 390 [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL, 391 392 [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL, 393 [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL, 394 395 [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, 396 [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, 397 [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, 398 [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, 399 }; 400 401 /* 402 * Cortex-A12 HW events mapping 403 */ 404 static const unsigned armv7_a12_perf_map[PERF_COUNT_HW_MAX] = { 405 PERF_MAP_ALL_UNSUPPORTED, 406 [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES, 407 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED, 408 [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, 409 [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_L1_DCACHE_REFILL, 410 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_A12_PERFCTR_PC_WRITE_SPEC, 411 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, 412 [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_BUS_CYCLES, 413 }; 414 415 static const unsigned armv7_a12_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] 416 [PERF_COUNT_HW_CACHE_OP_MAX] 417 [PERF_COUNT_HW_CACHE_RESULT_MAX] = { 418 PERF_CACHE_MAP_ALL_UNSUPPORTED, 419 420 [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_READ, 421 [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL, 422 [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_WRITE, 423 [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL, 424 425 /* 426 * Not all performance counters differentiate between read and write 427 * accesses/misses so we're not always strictly correct, but it's the 428 * best we can do. Writes and reads get combined in these cases. 429 */ 430 [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_ICACHE_ACCESS, 431 [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL, 432 433 [C(LL)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_READ, 434 [C(LL)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L2_CACHE_REFILL, 435 [C(LL)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_WRITE, 436 [C(LL)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L2_CACHE_REFILL, 437 438 [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL, 439 [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL, 440 [C(DTLB)][C(OP_PREFETCH)][C(RESULT_MISS)] = ARMV7_A12_PERFCTR_PF_TLB_REFILL, 441 442 [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL, 443 [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL, 444 445 [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, 446 [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, 447 [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, 448 [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, 449 }; 450 451 /* 452 * Krait HW events mapping 453 */ 454 static const unsigned krait_perf_map[PERF_COUNT_HW_MAX] = { 455 PERF_MAP_ALL_UNSUPPORTED, 456 [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES, 457 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED, 458 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE, 459 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, 460 [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_CLOCK_CYCLES, 461 }; 462 463 static const unsigned krait_perf_map_no_branch[PERF_COUNT_HW_MAX] = { 464 PERF_MAP_ALL_UNSUPPORTED, 465 [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES, 466 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED, 467 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, 468 [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_CLOCK_CYCLES, 469 }; 470 471 static const unsigned krait_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] 472 [PERF_COUNT_HW_CACHE_OP_MAX] 473 [PERF_COUNT_HW_CACHE_RESULT_MAX] = { 474 PERF_CACHE_MAP_ALL_UNSUPPORTED, 475 476 /* 477 * The performance counters don't differentiate between read and write 478 * accesses/misses so this isn't strictly correct, but it's the best we 479 * can do. Writes and reads get combined. 480 */ 481 [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, 482 [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL, 483 [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, 484 [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL, 485 486 [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = KRAIT_PERFCTR_L1_ICACHE_ACCESS, 487 [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = KRAIT_PERFCTR_L1_ICACHE_MISS, 488 489 [C(DTLB)][C(OP_READ)][C(RESULT_ACCESS)] = KRAIT_PERFCTR_L1_DTLB_ACCESS, 490 [C(DTLB)][C(OP_WRITE)][C(RESULT_ACCESS)] = KRAIT_PERFCTR_L1_DTLB_ACCESS, 491 492 [C(ITLB)][C(OP_READ)][C(RESULT_ACCESS)] = KRAIT_PERFCTR_L1_ITLB_ACCESS, 493 [C(ITLB)][C(OP_WRITE)][C(RESULT_ACCESS)] = KRAIT_PERFCTR_L1_ITLB_ACCESS, 494 495 [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, 496 [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, 497 [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, 498 [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, 499 }; 500 501 /* 502 * Scorpion HW events mapping 503 */ 504 static const unsigned scorpion_perf_map[PERF_COUNT_HW_MAX] = { 505 PERF_MAP_ALL_UNSUPPORTED, 506 [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES, 507 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED, 508 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE, 509 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, 510 [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_CLOCK_CYCLES, 511 }; 512 513 static const unsigned scorpion_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] 514 [PERF_COUNT_HW_CACHE_OP_MAX] 515 [PERF_COUNT_HW_CACHE_RESULT_MAX] = { 516 PERF_CACHE_MAP_ALL_UNSUPPORTED, 517 /* 518 * The performance counters don't differentiate between read and write 519 * accesses/misses so this isn't strictly correct, but it's the best we 520 * can do. Writes and reads get combined. 521 */ 522 [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, 523 [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL, 524 [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, 525 [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL, 526 [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = SCORPION_ICACHE_ACCESS, 527 [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = SCORPION_ICACHE_MISS, 528 /* 529 * Only ITLB misses and DTLB refills are supported. If users want the 530 * DTLB refills misses a raw counter must be used. 531 */ 532 [C(DTLB)][C(OP_READ)][C(RESULT_ACCESS)] = SCORPION_DTLB_ACCESS, 533 [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = SCORPION_DTLB_MISS, 534 [C(DTLB)][C(OP_WRITE)][C(RESULT_ACCESS)] = SCORPION_DTLB_ACCESS, 535 [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = SCORPION_DTLB_MISS, 536 [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = SCORPION_ITLB_MISS, 537 [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = SCORPION_ITLB_MISS, 538 [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, 539 [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, 540 [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, 541 [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, 542 }; 543 544 /* 545 * Perf Events' indices 546 */ 547 #define ARMV7_IDX_CYCLE_COUNTER 0 548 #define ARMV7_IDX_COUNTER0 1 549 #define ARMV7_IDX_COUNTER_LAST(cpu_pmu) \ 550 (ARMV7_IDX_CYCLE_COUNTER + cpu_pmu->num_events - 1) 551 552 #define ARMV7_MAX_COUNTERS 32 553 #define ARMV7_COUNTER_MASK (ARMV7_MAX_COUNTERS - 1) 554 555 /* 556 * ARMv7 low level PMNC access 557 */ 558 559 /* 560 * Perf Event to low level counters mapping 561 */ 562 #define ARMV7_IDX_TO_COUNTER(x) \ 563 (((x) - ARMV7_IDX_COUNTER0) & ARMV7_COUNTER_MASK) 564 565 /* 566 * Per-CPU PMNC: config reg 567 */ 568 #define ARMV7_PMNC_E (1 << 0) /* Enable all counters */ 569 #define ARMV7_PMNC_P (1 << 1) /* Reset all counters */ 570 #define ARMV7_PMNC_C (1 << 2) /* Cycle counter reset */ 571 #define ARMV7_PMNC_D (1 << 3) /* CCNT counts every 64th cpu cycle */ 572 #define ARMV7_PMNC_X (1 << 4) /* Export to ETM */ 573 #define ARMV7_PMNC_DP (1 << 5) /* Disable CCNT if non-invasive debug*/ 574 #define ARMV7_PMNC_N_SHIFT 11 /* Number of counters supported */ 575 #define ARMV7_PMNC_N_MASK 0x1f 576 #define ARMV7_PMNC_MASK 0x3f /* Mask for writable bits */ 577 578 /* 579 * FLAG: counters overflow flag status reg 580 */ 581 #define ARMV7_FLAG_MASK 0xffffffff /* Mask for writable bits */ 582 #define ARMV7_OVERFLOWED_MASK ARMV7_FLAG_MASK 583 584 /* 585 * PMXEVTYPER: Event selection reg 586 */ 587 #define ARMV7_EVTYPE_MASK 0xc80000ff /* Mask for writable bits */ 588 #define ARMV7_EVTYPE_EVENT 0xff /* Mask for EVENT bits */ 589 590 /* 591 * Event filters for PMUv2 592 */ 593 #define ARMV7_EXCLUDE_PL1 (1 << 31) 594 #define ARMV7_EXCLUDE_USER (1 << 30) 595 #define ARMV7_INCLUDE_HYP (1 << 27) 596 597 static inline u32 armv7_pmnc_read(void) 598 { 599 u32 val; 600 asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r"(val)); 601 return val; 602 } 603 604 static inline void armv7_pmnc_write(u32 val) 605 { 606 val &= ARMV7_PMNC_MASK; 607 isb(); 608 asm volatile("mcr p15, 0, %0, c9, c12, 0" : : "r"(val)); 609 } 610 611 static inline int armv7_pmnc_has_overflowed(u32 pmnc) 612 { 613 return pmnc & ARMV7_OVERFLOWED_MASK; 614 } 615 616 static inline int armv7_pmnc_counter_valid(struct arm_pmu *cpu_pmu, int idx) 617 { 618 return idx >= ARMV7_IDX_CYCLE_COUNTER && 619 idx <= ARMV7_IDX_COUNTER_LAST(cpu_pmu); 620 } 621 622 static inline int armv7_pmnc_counter_has_overflowed(u32 pmnc, int idx) 623 { 624 return pmnc & BIT(ARMV7_IDX_TO_COUNTER(idx)); 625 } 626 627 static inline void armv7_pmnc_select_counter(int idx) 628 { 629 u32 counter = ARMV7_IDX_TO_COUNTER(idx); 630 asm volatile("mcr p15, 0, %0, c9, c12, 5" : : "r" (counter)); 631 isb(); 632 } 633 634 static inline u32 armv7pmu_read_counter(struct perf_event *event) 635 { 636 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); 637 struct hw_perf_event *hwc = &event->hw; 638 int idx = hwc->idx; 639 u32 value = 0; 640 641 if (!armv7_pmnc_counter_valid(cpu_pmu, idx)) { 642 pr_err("CPU%u reading wrong counter %d\n", 643 smp_processor_id(), idx); 644 } else if (idx == ARMV7_IDX_CYCLE_COUNTER) { 645 asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (value)); 646 } else { 647 armv7_pmnc_select_counter(idx); 648 asm volatile("mrc p15, 0, %0, c9, c13, 2" : "=r" (value)); 649 } 650 651 return value; 652 } 653 654 static inline void armv7pmu_write_counter(struct perf_event *event, u32 value) 655 { 656 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); 657 struct hw_perf_event *hwc = &event->hw; 658 int idx = hwc->idx; 659 660 if (!armv7_pmnc_counter_valid(cpu_pmu, idx)) { 661 pr_err("CPU%u writing wrong counter %d\n", 662 smp_processor_id(), idx); 663 } else if (idx == ARMV7_IDX_CYCLE_COUNTER) { 664 asm volatile("mcr p15, 0, %0, c9, c13, 0" : : "r" (value)); 665 } else { 666 armv7_pmnc_select_counter(idx); 667 asm volatile("mcr p15, 0, %0, c9, c13, 2" : : "r" (value)); 668 } 669 } 670 671 static inline void armv7_pmnc_write_evtsel(int idx, u32 val) 672 { 673 armv7_pmnc_select_counter(idx); 674 val &= ARMV7_EVTYPE_MASK; 675 asm volatile("mcr p15, 0, %0, c9, c13, 1" : : "r" (val)); 676 } 677 678 static inline void armv7_pmnc_enable_counter(int idx) 679 { 680 u32 counter = ARMV7_IDX_TO_COUNTER(idx); 681 asm volatile("mcr p15, 0, %0, c9, c12, 1" : : "r" (BIT(counter))); 682 } 683 684 static inline void armv7_pmnc_disable_counter(int idx) 685 { 686 u32 counter = ARMV7_IDX_TO_COUNTER(idx); 687 asm volatile("mcr p15, 0, %0, c9, c12, 2" : : "r" (BIT(counter))); 688 } 689 690 static inline void armv7_pmnc_enable_intens(int idx) 691 { 692 u32 counter = ARMV7_IDX_TO_COUNTER(idx); 693 asm volatile("mcr p15, 0, %0, c9, c14, 1" : : "r" (BIT(counter))); 694 } 695 696 static inline void armv7_pmnc_disable_intens(int idx) 697 { 698 u32 counter = ARMV7_IDX_TO_COUNTER(idx); 699 asm volatile("mcr p15, 0, %0, c9, c14, 2" : : "r" (BIT(counter))); 700 isb(); 701 /* Clear the overflow flag in case an interrupt is pending. */ 702 asm volatile("mcr p15, 0, %0, c9, c12, 3" : : "r" (BIT(counter))); 703 isb(); 704 } 705 706 static inline u32 armv7_pmnc_getreset_flags(void) 707 { 708 u32 val; 709 710 /* Read */ 711 asm volatile("mrc p15, 0, %0, c9, c12, 3" : "=r" (val)); 712 713 /* Write to clear flags */ 714 val &= ARMV7_FLAG_MASK; 715 asm volatile("mcr p15, 0, %0, c9, c12, 3" : : "r" (val)); 716 717 return val; 718 } 719 720 #ifdef DEBUG 721 static void armv7_pmnc_dump_regs(struct arm_pmu *cpu_pmu) 722 { 723 u32 val; 724 unsigned int cnt; 725 726 pr_info("PMNC registers dump:\n"); 727 728 asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r" (val)); 729 pr_info("PMNC =0x%08x\n", val); 730 731 asm volatile("mrc p15, 0, %0, c9, c12, 1" : "=r" (val)); 732 pr_info("CNTENS=0x%08x\n", val); 733 734 asm volatile("mrc p15, 0, %0, c9, c14, 1" : "=r" (val)); 735 pr_info("INTENS=0x%08x\n", val); 736 737 asm volatile("mrc p15, 0, %0, c9, c12, 3" : "=r" (val)); 738 pr_info("FLAGS =0x%08x\n", val); 739 740 asm volatile("mrc p15, 0, %0, c9, c12, 5" : "=r" (val)); 741 pr_info("SELECT=0x%08x\n", val); 742 743 asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (val)); 744 pr_info("CCNT =0x%08x\n", val); 745 746 for (cnt = ARMV7_IDX_COUNTER0; 747 cnt <= ARMV7_IDX_COUNTER_LAST(cpu_pmu); cnt++) { 748 armv7_pmnc_select_counter(cnt); 749 asm volatile("mrc p15, 0, %0, c9, c13, 2" : "=r" (val)); 750 pr_info("CNT[%d] count =0x%08x\n", 751 ARMV7_IDX_TO_COUNTER(cnt), val); 752 asm volatile("mrc p15, 0, %0, c9, c13, 1" : "=r" (val)); 753 pr_info("CNT[%d] evtsel=0x%08x\n", 754 ARMV7_IDX_TO_COUNTER(cnt), val); 755 } 756 } 757 #endif 758 759 static void armv7pmu_enable_event(struct perf_event *event) 760 { 761 unsigned long flags; 762 struct hw_perf_event *hwc = &event->hw; 763 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); 764 struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events); 765 int idx = hwc->idx; 766 767 if (!armv7_pmnc_counter_valid(cpu_pmu, idx)) { 768 pr_err("CPU%u enabling wrong PMNC counter IRQ enable %d\n", 769 smp_processor_id(), idx); 770 return; 771 } 772 773 /* 774 * Enable counter and interrupt, and set the counter to count 775 * the event that we're interested in. 776 */ 777 raw_spin_lock_irqsave(&events->pmu_lock, flags); 778 779 /* 780 * Disable counter 781 */ 782 armv7_pmnc_disable_counter(idx); 783 784 /* 785 * Set event (if destined for PMNx counters) 786 * We only need to set the event for the cycle counter if we 787 * have the ability to perform event filtering. 788 */ 789 if (cpu_pmu->set_event_filter || idx != ARMV7_IDX_CYCLE_COUNTER) 790 armv7_pmnc_write_evtsel(idx, hwc->config_base); 791 792 /* 793 * Enable interrupt for this counter 794 */ 795 armv7_pmnc_enable_intens(idx); 796 797 /* 798 * Enable counter 799 */ 800 armv7_pmnc_enable_counter(idx); 801 802 raw_spin_unlock_irqrestore(&events->pmu_lock, flags); 803 } 804 805 static void armv7pmu_disable_event(struct perf_event *event) 806 { 807 unsigned long flags; 808 struct hw_perf_event *hwc = &event->hw; 809 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); 810 struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events); 811 int idx = hwc->idx; 812 813 if (!armv7_pmnc_counter_valid(cpu_pmu, idx)) { 814 pr_err("CPU%u disabling wrong PMNC counter IRQ enable %d\n", 815 smp_processor_id(), idx); 816 return; 817 } 818 819 /* 820 * Disable counter and interrupt 821 */ 822 raw_spin_lock_irqsave(&events->pmu_lock, flags); 823 824 /* 825 * Disable counter 826 */ 827 armv7_pmnc_disable_counter(idx); 828 829 /* 830 * Disable interrupt for this counter 831 */ 832 armv7_pmnc_disable_intens(idx); 833 834 raw_spin_unlock_irqrestore(&events->pmu_lock, flags); 835 } 836 837 static irqreturn_t armv7pmu_handle_irq(int irq_num, void *dev) 838 { 839 u32 pmnc; 840 struct perf_sample_data data; 841 struct arm_pmu *cpu_pmu = (struct arm_pmu *)dev; 842 struct pmu_hw_events *cpuc = this_cpu_ptr(cpu_pmu->hw_events); 843 struct pt_regs *regs; 844 int idx; 845 846 /* 847 * Get and reset the IRQ flags 848 */ 849 pmnc = armv7_pmnc_getreset_flags(); 850 851 /* 852 * Did an overflow occur? 853 */ 854 if (!armv7_pmnc_has_overflowed(pmnc)) 855 return IRQ_NONE; 856 857 /* 858 * Handle the counter(s) overflow(s) 859 */ 860 regs = get_irq_regs(); 861 862 for (idx = 0; idx < cpu_pmu->num_events; ++idx) { 863 struct perf_event *event = cpuc->events[idx]; 864 struct hw_perf_event *hwc; 865 866 /* Ignore if we don't have an event. */ 867 if (!event) 868 continue; 869 870 /* 871 * We have a single interrupt for all counters. Check that 872 * each counter has overflowed before we process it. 873 */ 874 if (!armv7_pmnc_counter_has_overflowed(pmnc, idx)) 875 continue; 876 877 hwc = &event->hw; 878 armpmu_event_update(event); 879 perf_sample_data_init(&data, 0, hwc->last_period); 880 if (!armpmu_event_set_period(event)) 881 continue; 882 883 if (perf_event_overflow(event, &data, regs)) 884 cpu_pmu->disable(event); 885 } 886 887 /* 888 * Handle the pending perf events. 889 * 890 * Note: this call *must* be run with interrupts disabled. For 891 * platforms that can have the PMU interrupts raised as an NMI, this 892 * will not work. 893 */ 894 irq_work_run(); 895 896 return IRQ_HANDLED; 897 } 898 899 static void armv7pmu_start(struct arm_pmu *cpu_pmu) 900 { 901 unsigned long flags; 902 struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events); 903 904 raw_spin_lock_irqsave(&events->pmu_lock, flags); 905 /* Enable all counters */ 906 armv7_pmnc_write(armv7_pmnc_read() | ARMV7_PMNC_E); 907 raw_spin_unlock_irqrestore(&events->pmu_lock, flags); 908 } 909 910 static void armv7pmu_stop(struct arm_pmu *cpu_pmu) 911 { 912 unsigned long flags; 913 struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events); 914 915 raw_spin_lock_irqsave(&events->pmu_lock, flags); 916 /* Disable all counters */ 917 armv7_pmnc_write(armv7_pmnc_read() & ~ARMV7_PMNC_E); 918 raw_spin_unlock_irqrestore(&events->pmu_lock, flags); 919 } 920 921 static int armv7pmu_get_event_idx(struct pmu_hw_events *cpuc, 922 struct perf_event *event) 923 { 924 int idx; 925 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); 926 struct hw_perf_event *hwc = &event->hw; 927 unsigned long evtype = hwc->config_base & ARMV7_EVTYPE_EVENT; 928 929 /* Always place a cycle counter into the cycle counter. */ 930 if (evtype == ARMV7_PERFCTR_CPU_CYCLES) { 931 if (test_and_set_bit(ARMV7_IDX_CYCLE_COUNTER, cpuc->used_mask)) 932 return -EAGAIN; 933 934 return ARMV7_IDX_CYCLE_COUNTER; 935 } 936 937 /* 938 * For anything other than a cycle counter, try and use 939 * the events counters 940 */ 941 for (idx = ARMV7_IDX_COUNTER0; idx < cpu_pmu->num_events; ++idx) { 942 if (!test_and_set_bit(idx, cpuc->used_mask)) 943 return idx; 944 } 945 946 /* The counters are all in use. */ 947 return -EAGAIN; 948 } 949 950 /* 951 * Add an event filter to a given event. This will only work for PMUv2 PMUs. 952 */ 953 static int armv7pmu_set_event_filter(struct hw_perf_event *event, 954 struct perf_event_attr *attr) 955 { 956 unsigned long config_base = 0; 957 958 if (attr->exclude_idle) 959 return -EPERM; 960 if (attr->exclude_user) 961 config_base |= ARMV7_EXCLUDE_USER; 962 if (attr->exclude_kernel) 963 config_base |= ARMV7_EXCLUDE_PL1; 964 if (!attr->exclude_hv) 965 config_base |= ARMV7_INCLUDE_HYP; 966 967 /* 968 * Install the filter into config_base as this is used to 969 * construct the event type. 970 */ 971 event->config_base = config_base; 972 973 return 0; 974 } 975 976 static void armv7pmu_reset(void *info) 977 { 978 struct arm_pmu *cpu_pmu = (struct arm_pmu *)info; 979 u32 idx, nb_cnt = cpu_pmu->num_events; 980 981 /* The counter and interrupt enable registers are unknown at reset. */ 982 for (idx = ARMV7_IDX_CYCLE_COUNTER; idx < nb_cnt; ++idx) { 983 armv7_pmnc_disable_counter(idx); 984 armv7_pmnc_disable_intens(idx); 985 } 986 987 /* Initialize & Reset PMNC: C and P bits */ 988 armv7_pmnc_write(ARMV7_PMNC_P | ARMV7_PMNC_C); 989 } 990 991 static int armv7_a8_map_event(struct perf_event *event) 992 { 993 return armpmu_map_event(event, &armv7_a8_perf_map, 994 &armv7_a8_perf_cache_map, 0xFF); 995 } 996 997 static int armv7_a9_map_event(struct perf_event *event) 998 { 999 return armpmu_map_event(event, &armv7_a9_perf_map, 1000 &armv7_a9_perf_cache_map, 0xFF); 1001 } 1002 1003 static int armv7_a5_map_event(struct perf_event *event) 1004 { 1005 return armpmu_map_event(event, &armv7_a5_perf_map, 1006 &armv7_a5_perf_cache_map, 0xFF); 1007 } 1008 1009 static int armv7_a15_map_event(struct perf_event *event) 1010 { 1011 return armpmu_map_event(event, &armv7_a15_perf_map, 1012 &armv7_a15_perf_cache_map, 0xFF); 1013 } 1014 1015 static int armv7_a7_map_event(struct perf_event *event) 1016 { 1017 return armpmu_map_event(event, &armv7_a7_perf_map, 1018 &armv7_a7_perf_cache_map, 0xFF); 1019 } 1020 1021 static int armv7_a12_map_event(struct perf_event *event) 1022 { 1023 return armpmu_map_event(event, &armv7_a12_perf_map, 1024 &armv7_a12_perf_cache_map, 0xFF); 1025 } 1026 1027 static int krait_map_event(struct perf_event *event) 1028 { 1029 return armpmu_map_event(event, &krait_perf_map, 1030 &krait_perf_cache_map, 0xFFFFF); 1031 } 1032 1033 static int krait_map_event_no_branch(struct perf_event *event) 1034 { 1035 return armpmu_map_event(event, &krait_perf_map_no_branch, 1036 &krait_perf_cache_map, 0xFFFFF); 1037 } 1038 1039 static int scorpion_map_event(struct perf_event *event) 1040 { 1041 return armpmu_map_event(event, &scorpion_perf_map, 1042 &scorpion_perf_cache_map, 0xFFFFF); 1043 } 1044 1045 static void armv7pmu_init(struct arm_pmu *cpu_pmu) 1046 { 1047 cpu_pmu->handle_irq = armv7pmu_handle_irq; 1048 cpu_pmu->enable = armv7pmu_enable_event; 1049 cpu_pmu->disable = armv7pmu_disable_event; 1050 cpu_pmu->read_counter = armv7pmu_read_counter; 1051 cpu_pmu->write_counter = armv7pmu_write_counter; 1052 cpu_pmu->get_event_idx = armv7pmu_get_event_idx; 1053 cpu_pmu->start = armv7pmu_start; 1054 cpu_pmu->stop = armv7pmu_stop; 1055 cpu_pmu->reset = armv7pmu_reset; 1056 cpu_pmu->max_period = (1LLU << 32) - 1; 1057 }; 1058 1059 static u32 armv7_read_num_pmnc_events(void) 1060 { 1061 u32 nb_cnt; 1062 1063 /* Read the nb of CNTx counters supported from PMNC */ 1064 nb_cnt = (armv7_pmnc_read() >> ARMV7_PMNC_N_SHIFT) & ARMV7_PMNC_N_MASK; 1065 1066 /* Add the CPU cycles counter and return */ 1067 return nb_cnt + 1; 1068 } 1069 1070 static int armv7_a8_pmu_init(struct arm_pmu *cpu_pmu) 1071 { 1072 armv7pmu_init(cpu_pmu); 1073 cpu_pmu->name = "armv7_cortex_a8"; 1074 cpu_pmu->map_event = armv7_a8_map_event; 1075 cpu_pmu->num_events = armv7_read_num_pmnc_events(); 1076 return 0; 1077 } 1078 1079 static int armv7_a9_pmu_init(struct arm_pmu *cpu_pmu) 1080 { 1081 armv7pmu_init(cpu_pmu); 1082 cpu_pmu->name = "armv7_cortex_a9"; 1083 cpu_pmu->map_event = armv7_a9_map_event; 1084 cpu_pmu->num_events = armv7_read_num_pmnc_events(); 1085 return 0; 1086 } 1087 1088 static int armv7_a5_pmu_init(struct arm_pmu *cpu_pmu) 1089 { 1090 armv7pmu_init(cpu_pmu); 1091 cpu_pmu->name = "armv7_cortex_a5"; 1092 cpu_pmu->map_event = armv7_a5_map_event; 1093 cpu_pmu->num_events = armv7_read_num_pmnc_events(); 1094 return 0; 1095 } 1096 1097 static int armv7_a15_pmu_init(struct arm_pmu *cpu_pmu) 1098 { 1099 armv7pmu_init(cpu_pmu); 1100 cpu_pmu->name = "armv7_cortex_a15"; 1101 cpu_pmu->map_event = armv7_a15_map_event; 1102 cpu_pmu->num_events = armv7_read_num_pmnc_events(); 1103 cpu_pmu->set_event_filter = armv7pmu_set_event_filter; 1104 return 0; 1105 } 1106 1107 static int armv7_a7_pmu_init(struct arm_pmu *cpu_pmu) 1108 { 1109 armv7pmu_init(cpu_pmu); 1110 cpu_pmu->name = "armv7_cortex_a7"; 1111 cpu_pmu->map_event = armv7_a7_map_event; 1112 cpu_pmu->num_events = armv7_read_num_pmnc_events(); 1113 cpu_pmu->set_event_filter = armv7pmu_set_event_filter; 1114 return 0; 1115 } 1116 1117 static int armv7_a12_pmu_init(struct arm_pmu *cpu_pmu) 1118 { 1119 armv7pmu_init(cpu_pmu); 1120 cpu_pmu->name = "armv7_cortex_a12"; 1121 cpu_pmu->map_event = armv7_a12_map_event; 1122 cpu_pmu->num_events = armv7_read_num_pmnc_events(); 1123 cpu_pmu->set_event_filter = armv7pmu_set_event_filter; 1124 return 0; 1125 } 1126 1127 static int armv7_a17_pmu_init(struct arm_pmu *cpu_pmu) 1128 { 1129 armv7_a12_pmu_init(cpu_pmu); 1130 cpu_pmu->name = "armv7_cortex_a17"; 1131 return 0; 1132 } 1133 1134 /* 1135 * Krait Performance Monitor Region Event Selection Register (PMRESRn) 1136 * 1137 * 31 30 24 16 8 0 1138 * +--------------------------------+ 1139 * PMRESR0 | EN | CC | CC | CC | CC | N = 1, R = 0 1140 * +--------------------------------+ 1141 * PMRESR1 | EN | CC | CC | CC | CC | N = 1, R = 1 1142 * +--------------------------------+ 1143 * PMRESR2 | EN | CC | CC | CC | CC | N = 1, R = 2 1144 * +--------------------------------+ 1145 * VPMRESR0 | EN | CC | CC | CC | CC | N = 2, R = ? 1146 * +--------------------------------+ 1147 * EN | G=3 | G=2 | G=1 | G=0 1148 * 1149 * Event Encoding: 1150 * 1151 * hwc->config_base = 0xNRCCG 1152 * 1153 * N = prefix, 1 for Krait CPU (PMRESRn), 2 for Venum VFP (VPMRESR) 1154 * R = region register 1155 * CC = class of events the group G is choosing from 1156 * G = group or particular event 1157 * 1158 * Example: 0x12021 is a Krait CPU event in PMRESR2's group 1 with code 2 1159 * 1160 * A region (R) corresponds to a piece of the CPU (execution unit, instruction 1161 * unit, etc.) while the event code (CC) corresponds to a particular class of 1162 * events (interrupts for example). An event code is broken down into 1163 * groups (G) that can be mapped into the PMU (irq, fiqs, and irq+fiqs for 1164 * example). 1165 */ 1166 1167 #define KRAIT_EVENT (1 << 16) 1168 #define VENUM_EVENT (2 << 16) 1169 #define KRAIT_EVENT_MASK (KRAIT_EVENT | VENUM_EVENT) 1170 #define PMRESRn_EN BIT(31) 1171 1172 #define EVENT_REGION(event) (((event) >> 12) & 0xf) /* R */ 1173 #define EVENT_GROUP(event) ((event) & 0xf) /* G */ 1174 #define EVENT_CODE(event) (((event) >> 4) & 0xff) /* CC */ 1175 #define EVENT_VENUM(event) (!!(event & VENUM_EVENT)) /* N=2 */ 1176 #define EVENT_CPU(event) (!!(event & KRAIT_EVENT)) /* N=1 */ 1177 1178 static u32 krait_read_pmresrn(int n) 1179 { 1180 u32 val; 1181 1182 switch (n) { 1183 case 0: 1184 asm volatile("mrc p15, 1, %0, c9, c15, 0" : "=r" (val)); 1185 break; 1186 case 1: 1187 asm volatile("mrc p15, 1, %0, c9, c15, 1" : "=r" (val)); 1188 break; 1189 case 2: 1190 asm volatile("mrc p15, 1, %0, c9, c15, 2" : "=r" (val)); 1191 break; 1192 default: 1193 BUG(); /* Should be validated in krait_pmu_get_event_idx() */ 1194 } 1195 1196 return val; 1197 } 1198 1199 static void krait_write_pmresrn(int n, u32 val) 1200 { 1201 switch (n) { 1202 case 0: 1203 asm volatile("mcr p15, 1, %0, c9, c15, 0" : : "r" (val)); 1204 break; 1205 case 1: 1206 asm volatile("mcr p15, 1, %0, c9, c15, 1" : : "r" (val)); 1207 break; 1208 case 2: 1209 asm volatile("mcr p15, 1, %0, c9, c15, 2" : : "r" (val)); 1210 break; 1211 default: 1212 BUG(); /* Should be validated in krait_pmu_get_event_idx() */ 1213 } 1214 } 1215 1216 static u32 venum_read_pmresr(void) 1217 { 1218 u32 val; 1219 asm volatile("mrc p10, 7, %0, c11, c0, 0" : "=r" (val)); 1220 return val; 1221 } 1222 1223 static void venum_write_pmresr(u32 val) 1224 { 1225 asm volatile("mcr p10, 7, %0, c11, c0, 0" : : "r" (val)); 1226 } 1227 1228 static void venum_pre_pmresr(u32 *venum_orig_val, u32 *fp_orig_val) 1229 { 1230 u32 venum_new_val; 1231 u32 fp_new_val; 1232 1233 BUG_ON(preemptible()); 1234 /* CPACR Enable CP10 and CP11 access */ 1235 *venum_orig_val = get_copro_access(); 1236 venum_new_val = *venum_orig_val | CPACC_SVC(10) | CPACC_SVC(11); 1237 set_copro_access(venum_new_val); 1238 1239 /* Enable FPEXC */ 1240 *fp_orig_val = fmrx(FPEXC); 1241 fp_new_val = *fp_orig_val | FPEXC_EN; 1242 fmxr(FPEXC, fp_new_val); 1243 } 1244 1245 static void venum_post_pmresr(u32 venum_orig_val, u32 fp_orig_val) 1246 { 1247 BUG_ON(preemptible()); 1248 /* Restore FPEXC */ 1249 fmxr(FPEXC, fp_orig_val); 1250 isb(); 1251 /* Restore CPACR */ 1252 set_copro_access(venum_orig_val); 1253 } 1254 1255 static u32 krait_get_pmresrn_event(unsigned int region) 1256 { 1257 static const u32 pmresrn_table[] = { KRAIT_PMRESR0_GROUP0, 1258 KRAIT_PMRESR1_GROUP0, 1259 KRAIT_PMRESR2_GROUP0 }; 1260 return pmresrn_table[region]; 1261 } 1262 1263 static void krait_evt_setup(int idx, u32 config_base) 1264 { 1265 u32 val; 1266 u32 mask; 1267 u32 vval, fval; 1268 unsigned int region = EVENT_REGION(config_base); 1269 unsigned int group = EVENT_GROUP(config_base); 1270 unsigned int code = EVENT_CODE(config_base); 1271 unsigned int group_shift; 1272 bool venum_event = EVENT_VENUM(config_base); 1273 1274 group_shift = group * 8; 1275 mask = 0xff << group_shift; 1276 1277 /* Configure evtsel for the region and group */ 1278 if (venum_event) 1279 val = KRAIT_VPMRESR0_GROUP0; 1280 else 1281 val = krait_get_pmresrn_event(region); 1282 val += group; 1283 /* Mix in mode-exclusion bits */ 1284 val |= config_base & (ARMV7_EXCLUDE_USER | ARMV7_EXCLUDE_PL1); 1285 armv7_pmnc_write_evtsel(idx, val); 1286 1287 if (venum_event) { 1288 venum_pre_pmresr(&vval, &fval); 1289 val = venum_read_pmresr(); 1290 val &= ~mask; 1291 val |= code << group_shift; 1292 val |= PMRESRn_EN; 1293 venum_write_pmresr(val); 1294 venum_post_pmresr(vval, fval); 1295 } else { 1296 val = krait_read_pmresrn(region); 1297 val &= ~mask; 1298 val |= code << group_shift; 1299 val |= PMRESRn_EN; 1300 krait_write_pmresrn(region, val); 1301 } 1302 } 1303 1304 static u32 clear_pmresrn_group(u32 val, int group) 1305 { 1306 u32 mask; 1307 int group_shift; 1308 1309 group_shift = group * 8; 1310 mask = 0xff << group_shift; 1311 val &= ~mask; 1312 1313 /* Don't clear enable bit if entire region isn't disabled */ 1314 if (val & ~PMRESRn_EN) 1315 return val |= PMRESRn_EN; 1316 1317 return 0; 1318 } 1319 1320 static void krait_clearpmu(u32 config_base) 1321 { 1322 u32 val; 1323 u32 vval, fval; 1324 unsigned int region = EVENT_REGION(config_base); 1325 unsigned int group = EVENT_GROUP(config_base); 1326 bool venum_event = EVENT_VENUM(config_base); 1327 1328 if (venum_event) { 1329 venum_pre_pmresr(&vval, &fval); 1330 val = venum_read_pmresr(); 1331 val = clear_pmresrn_group(val, group); 1332 venum_write_pmresr(val); 1333 venum_post_pmresr(vval, fval); 1334 } else { 1335 val = krait_read_pmresrn(region); 1336 val = clear_pmresrn_group(val, group); 1337 krait_write_pmresrn(region, val); 1338 } 1339 } 1340 1341 static void krait_pmu_disable_event(struct perf_event *event) 1342 { 1343 unsigned long flags; 1344 struct hw_perf_event *hwc = &event->hw; 1345 int idx = hwc->idx; 1346 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); 1347 struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events); 1348 1349 /* Disable counter and interrupt */ 1350 raw_spin_lock_irqsave(&events->pmu_lock, flags); 1351 1352 /* Disable counter */ 1353 armv7_pmnc_disable_counter(idx); 1354 1355 /* 1356 * Clear pmresr code (if destined for PMNx counters) 1357 */ 1358 if (hwc->config_base & KRAIT_EVENT_MASK) 1359 krait_clearpmu(hwc->config_base); 1360 1361 /* Disable interrupt for this counter */ 1362 armv7_pmnc_disable_intens(idx); 1363 1364 raw_spin_unlock_irqrestore(&events->pmu_lock, flags); 1365 } 1366 1367 static void krait_pmu_enable_event(struct perf_event *event) 1368 { 1369 unsigned long flags; 1370 struct hw_perf_event *hwc = &event->hw; 1371 int idx = hwc->idx; 1372 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); 1373 struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events); 1374 1375 /* 1376 * Enable counter and interrupt, and set the counter to count 1377 * the event that we're interested in. 1378 */ 1379 raw_spin_lock_irqsave(&events->pmu_lock, flags); 1380 1381 /* Disable counter */ 1382 armv7_pmnc_disable_counter(idx); 1383 1384 /* 1385 * Set event (if destined for PMNx counters) 1386 * We set the event for the cycle counter because we 1387 * have the ability to perform event filtering. 1388 */ 1389 if (hwc->config_base & KRAIT_EVENT_MASK) 1390 krait_evt_setup(idx, hwc->config_base); 1391 else 1392 armv7_pmnc_write_evtsel(idx, hwc->config_base); 1393 1394 /* Enable interrupt for this counter */ 1395 armv7_pmnc_enable_intens(idx); 1396 1397 /* Enable counter */ 1398 armv7_pmnc_enable_counter(idx); 1399 1400 raw_spin_unlock_irqrestore(&events->pmu_lock, flags); 1401 } 1402 1403 static void krait_pmu_reset(void *info) 1404 { 1405 u32 vval, fval; 1406 struct arm_pmu *cpu_pmu = info; 1407 u32 idx, nb_cnt = cpu_pmu->num_events; 1408 1409 armv7pmu_reset(info); 1410 1411 /* Clear all pmresrs */ 1412 krait_write_pmresrn(0, 0); 1413 krait_write_pmresrn(1, 0); 1414 krait_write_pmresrn(2, 0); 1415 1416 venum_pre_pmresr(&vval, &fval); 1417 venum_write_pmresr(0); 1418 venum_post_pmresr(vval, fval); 1419 1420 /* Reset PMxEVNCTCR to sane default */ 1421 for (idx = ARMV7_IDX_CYCLE_COUNTER; idx < nb_cnt; ++idx) { 1422 armv7_pmnc_select_counter(idx); 1423 asm volatile("mcr p15, 0, %0, c9, c15, 0" : : "r" (0)); 1424 } 1425 1426 } 1427 1428 static int krait_event_to_bit(struct perf_event *event, unsigned int region, 1429 unsigned int group) 1430 { 1431 int bit; 1432 struct hw_perf_event *hwc = &event->hw; 1433 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); 1434 1435 if (hwc->config_base & VENUM_EVENT) 1436 bit = KRAIT_VPMRESR0_GROUP0; 1437 else 1438 bit = krait_get_pmresrn_event(region); 1439 bit -= krait_get_pmresrn_event(0); 1440 bit += group; 1441 /* 1442 * Lower bits are reserved for use by the counters (see 1443 * armv7pmu_get_event_idx() for more info) 1444 */ 1445 bit += ARMV7_IDX_COUNTER_LAST(cpu_pmu) + 1; 1446 1447 return bit; 1448 } 1449 1450 /* 1451 * We check for column exclusion constraints here. 1452 * Two events cant use the same group within a pmresr register. 1453 */ 1454 static int krait_pmu_get_event_idx(struct pmu_hw_events *cpuc, 1455 struct perf_event *event) 1456 { 1457 int idx; 1458 int bit = -1; 1459 struct hw_perf_event *hwc = &event->hw; 1460 unsigned int region = EVENT_REGION(hwc->config_base); 1461 unsigned int code = EVENT_CODE(hwc->config_base); 1462 unsigned int group = EVENT_GROUP(hwc->config_base); 1463 bool venum_event = EVENT_VENUM(hwc->config_base); 1464 bool krait_event = EVENT_CPU(hwc->config_base); 1465 1466 if (venum_event || krait_event) { 1467 /* Ignore invalid events */ 1468 if (group > 3 || region > 2) 1469 return -EINVAL; 1470 if (venum_event && (code & 0xe0)) 1471 return -EINVAL; 1472 1473 bit = krait_event_to_bit(event, region, group); 1474 if (test_and_set_bit(bit, cpuc->used_mask)) 1475 return -EAGAIN; 1476 } 1477 1478 idx = armv7pmu_get_event_idx(cpuc, event); 1479 if (idx < 0 && bit >= 0) 1480 clear_bit(bit, cpuc->used_mask); 1481 1482 return idx; 1483 } 1484 1485 static void krait_pmu_clear_event_idx(struct pmu_hw_events *cpuc, 1486 struct perf_event *event) 1487 { 1488 int bit; 1489 struct hw_perf_event *hwc = &event->hw; 1490 unsigned int region = EVENT_REGION(hwc->config_base); 1491 unsigned int group = EVENT_GROUP(hwc->config_base); 1492 bool venum_event = EVENT_VENUM(hwc->config_base); 1493 bool krait_event = EVENT_CPU(hwc->config_base); 1494 1495 if (venum_event || krait_event) { 1496 bit = krait_event_to_bit(event, region, group); 1497 clear_bit(bit, cpuc->used_mask); 1498 } 1499 } 1500 1501 static int krait_pmu_init(struct arm_pmu *cpu_pmu) 1502 { 1503 armv7pmu_init(cpu_pmu); 1504 cpu_pmu->name = "armv7_krait"; 1505 /* Some early versions of Krait don't support PC write events */ 1506 if (of_property_read_bool(cpu_pmu->plat_device->dev.of_node, 1507 "qcom,no-pc-write")) 1508 cpu_pmu->map_event = krait_map_event_no_branch; 1509 else 1510 cpu_pmu->map_event = krait_map_event; 1511 cpu_pmu->num_events = armv7_read_num_pmnc_events(); 1512 cpu_pmu->set_event_filter = armv7pmu_set_event_filter; 1513 cpu_pmu->reset = krait_pmu_reset; 1514 cpu_pmu->enable = krait_pmu_enable_event; 1515 cpu_pmu->disable = krait_pmu_disable_event; 1516 cpu_pmu->get_event_idx = krait_pmu_get_event_idx; 1517 cpu_pmu->clear_event_idx = krait_pmu_clear_event_idx; 1518 return 0; 1519 } 1520 1521 /* 1522 * Scorpion Local Performance Monitor Register (LPMn) 1523 * 1524 * 31 30 24 16 8 0 1525 * +--------------------------------+ 1526 * LPM0 | EN | CC | CC | CC | CC | N = 1, R = 0 1527 * +--------------------------------+ 1528 * LPM1 | EN | CC | CC | CC | CC | N = 1, R = 1 1529 * +--------------------------------+ 1530 * LPM2 | EN | CC | CC | CC | CC | N = 1, R = 2 1531 * +--------------------------------+ 1532 * L2LPM | EN | CC | CC | CC | CC | N = 1, R = 3 1533 * +--------------------------------+ 1534 * VLPM | EN | CC | CC | CC | CC | N = 2, R = ? 1535 * +--------------------------------+ 1536 * EN | G=3 | G=2 | G=1 | G=0 1537 * 1538 * 1539 * Event Encoding: 1540 * 1541 * hwc->config_base = 0xNRCCG 1542 * 1543 * N = prefix, 1 for Scorpion CPU (LPMn/L2LPM), 2 for Venum VFP (VLPM) 1544 * R = region register 1545 * CC = class of events the group G is choosing from 1546 * G = group or particular event 1547 * 1548 * Example: 0x12021 is a Scorpion CPU event in LPM2's group 1 with code 2 1549 * 1550 * A region (R) corresponds to a piece of the CPU (execution unit, instruction 1551 * unit, etc.) while the event code (CC) corresponds to a particular class of 1552 * events (interrupts for example). An event code is broken down into 1553 * groups (G) that can be mapped into the PMU (irq, fiqs, and irq+fiqs for 1554 * example). 1555 */ 1556 1557 static u32 scorpion_read_pmresrn(int n) 1558 { 1559 u32 val; 1560 1561 switch (n) { 1562 case 0: 1563 asm volatile("mrc p15, 0, %0, c15, c0, 0" : "=r" (val)); 1564 break; 1565 case 1: 1566 asm volatile("mrc p15, 1, %0, c15, c0, 0" : "=r" (val)); 1567 break; 1568 case 2: 1569 asm volatile("mrc p15, 2, %0, c15, c0, 0" : "=r" (val)); 1570 break; 1571 case 3: 1572 asm volatile("mrc p15, 3, %0, c15, c2, 0" : "=r" (val)); 1573 break; 1574 default: 1575 BUG(); /* Should be validated in scorpion_pmu_get_event_idx() */ 1576 } 1577 1578 return val; 1579 } 1580 1581 static void scorpion_write_pmresrn(int n, u32 val) 1582 { 1583 switch (n) { 1584 case 0: 1585 asm volatile("mcr p15, 0, %0, c15, c0, 0" : : "r" (val)); 1586 break; 1587 case 1: 1588 asm volatile("mcr p15, 1, %0, c15, c0, 0" : : "r" (val)); 1589 break; 1590 case 2: 1591 asm volatile("mcr p15, 2, %0, c15, c0, 0" : : "r" (val)); 1592 break; 1593 case 3: 1594 asm volatile("mcr p15, 3, %0, c15, c2, 0" : : "r" (val)); 1595 break; 1596 default: 1597 BUG(); /* Should be validated in scorpion_pmu_get_event_idx() */ 1598 } 1599 } 1600 1601 static u32 scorpion_get_pmresrn_event(unsigned int region) 1602 { 1603 static const u32 pmresrn_table[] = { SCORPION_LPM0_GROUP0, 1604 SCORPION_LPM1_GROUP0, 1605 SCORPION_LPM2_GROUP0, 1606 SCORPION_L2LPM_GROUP0 }; 1607 return pmresrn_table[region]; 1608 } 1609 1610 static void scorpion_evt_setup(int idx, u32 config_base) 1611 { 1612 u32 val; 1613 u32 mask; 1614 u32 vval, fval; 1615 unsigned int region = EVENT_REGION(config_base); 1616 unsigned int group = EVENT_GROUP(config_base); 1617 unsigned int code = EVENT_CODE(config_base); 1618 unsigned int group_shift; 1619 bool venum_event = EVENT_VENUM(config_base); 1620 1621 group_shift = group * 8; 1622 mask = 0xff << group_shift; 1623 1624 /* Configure evtsel for the region and group */ 1625 if (venum_event) 1626 val = SCORPION_VLPM_GROUP0; 1627 else 1628 val = scorpion_get_pmresrn_event(region); 1629 val += group; 1630 /* Mix in mode-exclusion bits */ 1631 val |= config_base & (ARMV7_EXCLUDE_USER | ARMV7_EXCLUDE_PL1); 1632 armv7_pmnc_write_evtsel(idx, val); 1633 1634 asm volatile("mcr p15, 0, %0, c9, c15, 0" : : "r" (0)); 1635 1636 if (venum_event) { 1637 venum_pre_pmresr(&vval, &fval); 1638 val = venum_read_pmresr(); 1639 val &= ~mask; 1640 val |= code << group_shift; 1641 val |= PMRESRn_EN; 1642 venum_write_pmresr(val); 1643 venum_post_pmresr(vval, fval); 1644 } else { 1645 val = scorpion_read_pmresrn(region); 1646 val &= ~mask; 1647 val |= code << group_shift; 1648 val |= PMRESRn_EN; 1649 scorpion_write_pmresrn(region, val); 1650 } 1651 } 1652 1653 static void scorpion_clearpmu(u32 config_base) 1654 { 1655 u32 val; 1656 u32 vval, fval; 1657 unsigned int region = EVENT_REGION(config_base); 1658 unsigned int group = EVENT_GROUP(config_base); 1659 bool venum_event = EVENT_VENUM(config_base); 1660 1661 if (venum_event) { 1662 venum_pre_pmresr(&vval, &fval); 1663 val = venum_read_pmresr(); 1664 val = clear_pmresrn_group(val, group); 1665 venum_write_pmresr(val); 1666 venum_post_pmresr(vval, fval); 1667 } else { 1668 val = scorpion_read_pmresrn(region); 1669 val = clear_pmresrn_group(val, group); 1670 scorpion_write_pmresrn(region, val); 1671 } 1672 } 1673 1674 static void scorpion_pmu_disable_event(struct perf_event *event) 1675 { 1676 unsigned long flags; 1677 struct hw_perf_event *hwc = &event->hw; 1678 int idx = hwc->idx; 1679 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); 1680 struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events); 1681 1682 /* Disable counter and interrupt */ 1683 raw_spin_lock_irqsave(&events->pmu_lock, flags); 1684 1685 /* Disable counter */ 1686 armv7_pmnc_disable_counter(idx); 1687 1688 /* 1689 * Clear pmresr code (if destined for PMNx counters) 1690 */ 1691 if (hwc->config_base & KRAIT_EVENT_MASK) 1692 scorpion_clearpmu(hwc->config_base); 1693 1694 /* Disable interrupt for this counter */ 1695 armv7_pmnc_disable_intens(idx); 1696 1697 raw_spin_unlock_irqrestore(&events->pmu_lock, flags); 1698 } 1699 1700 static void scorpion_pmu_enable_event(struct perf_event *event) 1701 { 1702 unsigned long flags; 1703 struct hw_perf_event *hwc = &event->hw; 1704 int idx = hwc->idx; 1705 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); 1706 struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events); 1707 1708 /* 1709 * Enable counter and interrupt, and set the counter to count 1710 * the event that we're interested in. 1711 */ 1712 raw_spin_lock_irqsave(&events->pmu_lock, flags); 1713 1714 /* Disable counter */ 1715 armv7_pmnc_disable_counter(idx); 1716 1717 /* 1718 * Set event (if destined for PMNx counters) 1719 * We don't set the event for the cycle counter because we 1720 * don't have the ability to perform event filtering. 1721 */ 1722 if (hwc->config_base & KRAIT_EVENT_MASK) 1723 scorpion_evt_setup(idx, hwc->config_base); 1724 else if (idx != ARMV7_IDX_CYCLE_COUNTER) 1725 armv7_pmnc_write_evtsel(idx, hwc->config_base); 1726 1727 /* Enable interrupt for this counter */ 1728 armv7_pmnc_enable_intens(idx); 1729 1730 /* Enable counter */ 1731 armv7_pmnc_enable_counter(idx); 1732 1733 raw_spin_unlock_irqrestore(&events->pmu_lock, flags); 1734 } 1735 1736 static void scorpion_pmu_reset(void *info) 1737 { 1738 u32 vval, fval; 1739 struct arm_pmu *cpu_pmu = info; 1740 u32 idx, nb_cnt = cpu_pmu->num_events; 1741 1742 armv7pmu_reset(info); 1743 1744 /* Clear all pmresrs */ 1745 scorpion_write_pmresrn(0, 0); 1746 scorpion_write_pmresrn(1, 0); 1747 scorpion_write_pmresrn(2, 0); 1748 scorpion_write_pmresrn(3, 0); 1749 1750 venum_pre_pmresr(&vval, &fval); 1751 venum_write_pmresr(0); 1752 venum_post_pmresr(vval, fval); 1753 1754 /* Reset PMxEVNCTCR to sane default */ 1755 for (idx = ARMV7_IDX_CYCLE_COUNTER; idx < nb_cnt; ++idx) { 1756 armv7_pmnc_select_counter(idx); 1757 asm volatile("mcr p15, 0, %0, c9, c15, 0" : : "r" (0)); 1758 } 1759 } 1760 1761 static int scorpion_event_to_bit(struct perf_event *event, unsigned int region, 1762 unsigned int group) 1763 { 1764 int bit; 1765 struct hw_perf_event *hwc = &event->hw; 1766 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); 1767 1768 if (hwc->config_base & VENUM_EVENT) 1769 bit = SCORPION_VLPM_GROUP0; 1770 else 1771 bit = scorpion_get_pmresrn_event(region); 1772 bit -= scorpion_get_pmresrn_event(0); 1773 bit += group; 1774 /* 1775 * Lower bits are reserved for use by the counters (see 1776 * armv7pmu_get_event_idx() for more info) 1777 */ 1778 bit += ARMV7_IDX_COUNTER_LAST(cpu_pmu) + 1; 1779 1780 return bit; 1781 } 1782 1783 /* 1784 * We check for column exclusion constraints here. 1785 * Two events cant use the same group within a pmresr register. 1786 */ 1787 static int scorpion_pmu_get_event_idx(struct pmu_hw_events *cpuc, 1788 struct perf_event *event) 1789 { 1790 int idx; 1791 int bit = -1; 1792 struct hw_perf_event *hwc = &event->hw; 1793 unsigned int region = EVENT_REGION(hwc->config_base); 1794 unsigned int group = EVENT_GROUP(hwc->config_base); 1795 bool venum_event = EVENT_VENUM(hwc->config_base); 1796 bool scorpion_event = EVENT_CPU(hwc->config_base); 1797 1798 if (venum_event || scorpion_event) { 1799 /* Ignore invalid events */ 1800 if (group > 3 || region > 3) 1801 return -EINVAL; 1802 1803 bit = scorpion_event_to_bit(event, region, group); 1804 if (test_and_set_bit(bit, cpuc->used_mask)) 1805 return -EAGAIN; 1806 } 1807 1808 idx = armv7pmu_get_event_idx(cpuc, event); 1809 if (idx < 0 && bit >= 0) 1810 clear_bit(bit, cpuc->used_mask); 1811 1812 return idx; 1813 } 1814 1815 static void scorpion_pmu_clear_event_idx(struct pmu_hw_events *cpuc, 1816 struct perf_event *event) 1817 { 1818 int bit; 1819 struct hw_perf_event *hwc = &event->hw; 1820 unsigned int region = EVENT_REGION(hwc->config_base); 1821 unsigned int group = EVENT_GROUP(hwc->config_base); 1822 bool venum_event = EVENT_VENUM(hwc->config_base); 1823 bool scorpion_event = EVENT_CPU(hwc->config_base); 1824 1825 if (venum_event || scorpion_event) { 1826 bit = scorpion_event_to_bit(event, region, group); 1827 clear_bit(bit, cpuc->used_mask); 1828 } 1829 } 1830 1831 static int scorpion_pmu_init(struct arm_pmu *cpu_pmu) 1832 { 1833 armv7pmu_init(cpu_pmu); 1834 cpu_pmu->name = "armv7_scorpion"; 1835 cpu_pmu->map_event = scorpion_map_event; 1836 cpu_pmu->num_events = armv7_read_num_pmnc_events(); 1837 cpu_pmu->reset = scorpion_pmu_reset; 1838 cpu_pmu->enable = scorpion_pmu_enable_event; 1839 cpu_pmu->disable = scorpion_pmu_disable_event; 1840 cpu_pmu->get_event_idx = scorpion_pmu_get_event_idx; 1841 cpu_pmu->clear_event_idx = scorpion_pmu_clear_event_idx; 1842 return 0; 1843 } 1844 1845 static int scorpion_mp_pmu_init(struct arm_pmu *cpu_pmu) 1846 { 1847 armv7pmu_init(cpu_pmu); 1848 cpu_pmu->name = "armv7_scorpion_mp"; 1849 cpu_pmu->map_event = scorpion_map_event; 1850 cpu_pmu->num_events = armv7_read_num_pmnc_events(); 1851 cpu_pmu->reset = scorpion_pmu_reset; 1852 cpu_pmu->enable = scorpion_pmu_enable_event; 1853 cpu_pmu->disable = scorpion_pmu_disable_event; 1854 cpu_pmu->get_event_idx = scorpion_pmu_get_event_idx; 1855 cpu_pmu->clear_event_idx = scorpion_pmu_clear_event_idx; 1856 return 0; 1857 } 1858 #else 1859 static inline int armv7_a8_pmu_init(struct arm_pmu *cpu_pmu) 1860 { 1861 return -ENODEV; 1862 } 1863 1864 static inline int armv7_a9_pmu_init(struct arm_pmu *cpu_pmu) 1865 { 1866 return -ENODEV; 1867 } 1868 1869 static inline int armv7_a5_pmu_init(struct arm_pmu *cpu_pmu) 1870 { 1871 return -ENODEV; 1872 } 1873 1874 static inline int armv7_a15_pmu_init(struct arm_pmu *cpu_pmu) 1875 { 1876 return -ENODEV; 1877 } 1878 1879 static inline int armv7_a7_pmu_init(struct arm_pmu *cpu_pmu) 1880 { 1881 return -ENODEV; 1882 } 1883 1884 static inline int armv7_a12_pmu_init(struct arm_pmu *cpu_pmu) 1885 { 1886 return -ENODEV; 1887 } 1888 1889 static inline int armv7_a17_pmu_init(struct arm_pmu *cpu_pmu) 1890 { 1891 return -ENODEV; 1892 } 1893 1894 static inline int krait_pmu_init(struct arm_pmu *cpu_pmu) 1895 { 1896 return -ENODEV; 1897 } 1898 1899 static inline int scorpion_pmu_init(struct arm_pmu *cpu_pmu) 1900 { 1901 return -ENODEV; 1902 } 1903 1904 static inline int scorpion_mp_pmu_init(struct arm_pmu *cpu_pmu) 1905 { 1906 return -ENODEV; 1907 } 1908 #endif /* CONFIG_CPU_V7 */ 1909