1 /* 2 * ARMv7 Cortex-A8 and Cortex-A9 Performance Events handling code. 3 * 4 * ARMv7 support: Jean Pihet <jpihet@mvista.com> 5 * 2010 (c) MontaVista Software, LLC. 6 * 7 * Copied from ARMv6 code, with the low level code inspired 8 * by the ARMv7 Oprofile code. 9 * 10 * Cortex-A8 has up to 4 configurable performance counters and 11 * a single cycle counter. 12 * Cortex-A9 has up to 31 configurable performance counters and 13 * a single cycle counter. 14 * 15 * All counters can be enabled/disabled and IRQ masked separately. The cycle 16 * counter and all 4 performance counters together can be reset separately. 17 */ 18 19 #ifdef CONFIG_CPU_V7 20 21 #include <asm/cp15.h> 22 #include <asm/cputype.h> 23 #include <asm/irq_regs.h> 24 #include <asm/vfp.h> 25 #include "../vfp/vfpinstr.h" 26 27 #include <linux/of.h> 28 #include <linux/perf/arm_pmu.h> 29 #include <linux/platform_device.h> 30 31 /* 32 * Common ARMv7 event types 33 * 34 * Note: An implementation may not be able to count all of these events 35 * but the encodings are considered to be `reserved' in the case that 36 * they are not available. 37 */ 38 enum armv7_perf_types { 39 ARMV7_PERFCTR_PMNC_SW_INCR = 0x00, 40 ARMV7_PERFCTR_L1_ICACHE_REFILL = 0x01, 41 ARMV7_PERFCTR_ITLB_REFILL = 0x02, 42 ARMV7_PERFCTR_L1_DCACHE_REFILL = 0x03, 43 ARMV7_PERFCTR_L1_DCACHE_ACCESS = 0x04, 44 ARMV7_PERFCTR_DTLB_REFILL = 0x05, 45 ARMV7_PERFCTR_MEM_READ = 0x06, 46 ARMV7_PERFCTR_MEM_WRITE = 0x07, 47 ARMV7_PERFCTR_INSTR_EXECUTED = 0x08, 48 ARMV7_PERFCTR_EXC_TAKEN = 0x09, 49 ARMV7_PERFCTR_EXC_EXECUTED = 0x0A, 50 ARMV7_PERFCTR_CID_WRITE = 0x0B, 51 52 /* 53 * ARMV7_PERFCTR_PC_WRITE is equivalent to HW_BRANCH_INSTRUCTIONS. 54 * It counts: 55 * - all (taken) branch instructions, 56 * - instructions that explicitly write the PC, 57 * - exception generating instructions. 58 */ 59 ARMV7_PERFCTR_PC_WRITE = 0x0C, 60 ARMV7_PERFCTR_PC_IMM_BRANCH = 0x0D, 61 ARMV7_PERFCTR_PC_PROC_RETURN = 0x0E, 62 ARMV7_PERFCTR_MEM_UNALIGNED_ACCESS = 0x0F, 63 ARMV7_PERFCTR_PC_BRANCH_MIS_PRED = 0x10, 64 ARMV7_PERFCTR_CLOCK_CYCLES = 0x11, 65 ARMV7_PERFCTR_PC_BRANCH_PRED = 0x12, 66 67 /* These events are defined by the PMUv2 supplement (ARM DDI 0457A). */ 68 ARMV7_PERFCTR_MEM_ACCESS = 0x13, 69 ARMV7_PERFCTR_L1_ICACHE_ACCESS = 0x14, 70 ARMV7_PERFCTR_L1_DCACHE_WB = 0x15, 71 ARMV7_PERFCTR_L2_CACHE_ACCESS = 0x16, 72 ARMV7_PERFCTR_L2_CACHE_REFILL = 0x17, 73 ARMV7_PERFCTR_L2_CACHE_WB = 0x18, 74 ARMV7_PERFCTR_BUS_ACCESS = 0x19, 75 ARMV7_PERFCTR_MEM_ERROR = 0x1A, 76 ARMV7_PERFCTR_INSTR_SPEC = 0x1B, 77 ARMV7_PERFCTR_TTBR_WRITE = 0x1C, 78 ARMV7_PERFCTR_BUS_CYCLES = 0x1D, 79 80 ARMV7_PERFCTR_CPU_CYCLES = 0xFF 81 }; 82 83 /* ARMv7 Cortex-A8 specific event types */ 84 enum armv7_a8_perf_types { 85 ARMV7_A8_PERFCTR_L2_CACHE_ACCESS = 0x43, 86 ARMV7_A8_PERFCTR_L2_CACHE_REFILL = 0x44, 87 ARMV7_A8_PERFCTR_L1_ICACHE_ACCESS = 0x50, 88 ARMV7_A8_PERFCTR_STALL_ISIDE = 0x56, 89 }; 90 91 /* ARMv7 Cortex-A9 specific event types */ 92 enum armv7_a9_perf_types { 93 ARMV7_A9_PERFCTR_INSTR_CORE_RENAME = 0x68, 94 ARMV7_A9_PERFCTR_STALL_ICACHE = 0x60, 95 ARMV7_A9_PERFCTR_STALL_DISPATCH = 0x66, 96 }; 97 98 /* ARMv7 Cortex-A5 specific event types */ 99 enum armv7_a5_perf_types { 100 ARMV7_A5_PERFCTR_PREFETCH_LINEFILL = 0xc2, 101 ARMV7_A5_PERFCTR_PREFETCH_LINEFILL_DROP = 0xc3, 102 }; 103 104 /* ARMv7 Cortex-A15 specific event types */ 105 enum armv7_a15_perf_types { 106 ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_READ = 0x40, 107 ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_WRITE = 0x41, 108 ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_READ = 0x42, 109 ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_WRITE = 0x43, 110 111 ARMV7_A15_PERFCTR_DTLB_REFILL_L1_READ = 0x4C, 112 ARMV7_A15_PERFCTR_DTLB_REFILL_L1_WRITE = 0x4D, 113 114 ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_READ = 0x50, 115 ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_WRITE = 0x51, 116 ARMV7_A15_PERFCTR_L2_CACHE_REFILL_READ = 0x52, 117 ARMV7_A15_PERFCTR_L2_CACHE_REFILL_WRITE = 0x53, 118 119 ARMV7_A15_PERFCTR_PC_WRITE_SPEC = 0x76, 120 }; 121 122 /* ARMv7 Cortex-A12 specific event types */ 123 enum armv7_a12_perf_types { 124 ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_READ = 0x40, 125 ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_WRITE = 0x41, 126 127 ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_READ = 0x50, 128 ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_WRITE = 0x51, 129 130 ARMV7_A12_PERFCTR_PC_WRITE_SPEC = 0x76, 131 132 ARMV7_A12_PERFCTR_PF_TLB_REFILL = 0xe7, 133 }; 134 135 /* ARMv7 Krait specific event types */ 136 enum krait_perf_types { 137 KRAIT_PMRESR0_GROUP0 = 0xcc, 138 KRAIT_PMRESR1_GROUP0 = 0xd0, 139 KRAIT_PMRESR2_GROUP0 = 0xd4, 140 KRAIT_VPMRESR0_GROUP0 = 0xd8, 141 142 KRAIT_PERFCTR_L1_ICACHE_ACCESS = 0x10011, 143 KRAIT_PERFCTR_L1_ICACHE_MISS = 0x10010, 144 145 KRAIT_PERFCTR_L1_ITLB_ACCESS = 0x12222, 146 KRAIT_PERFCTR_L1_DTLB_ACCESS = 0x12210, 147 }; 148 149 /* ARMv7 Scorpion specific event types */ 150 enum scorpion_perf_types { 151 SCORPION_LPM0_GROUP0 = 0x4c, 152 SCORPION_LPM1_GROUP0 = 0x50, 153 SCORPION_LPM2_GROUP0 = 0x54, 154 SCORPION_L2LPM_GROUP0 = 0x58, 155 SCORPION_VLPM_GROUP0 = 0x5c, 156 157 SCORPION_ICACHE_ACCESS = 0x10053, 158 SCORPION_ICACHE_MISS = 0x10052, 159 160 SCORPION_DTLB_ACCESS = 0x12013, 161 SCORPION_DTLB_MISS = 0x12012, 162 163 SCORPION_ITLB_MISS = 0x12021, 164 }; 165 166 /* 167 * Cortex-A8 HW events mapping 168 * 169 * The hardware events that we support. We do support cache operations but 170 * we have harvard caches and no way to combine instruction and data 171 * accesses/misses in hardware. 172 */ 173 static const unsigned armv7_a8_perf_map[PERF_COUNT_HW_MAX] = { 174 PERF_MAP_ALL_UNSUPPORTED, 175 [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES, 176 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED, 177 [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, 178 [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_L1_DCACHE_REFILL, 179 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE, 180 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, 181 [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = ARMV7_A8_PERFCTR_STALL_ISIDE, 182 }; 183 184 static const unsigned armv7_a8_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] 185 [PERF_COUNT_HW_CACHE_OP_MAX] 186 [PERF_COUNT_HW_CACHE_RESULT_MAX] = { 187 PERF_CACHE_MAP_ALL_UNSUPPORTED, 188 189 /* 190 * The performance counters don't differentiate between read and write 191 * accesses/misses so this isn't strictly correct, but it's the best we 192 * can do. Writes and reads get combined. 193 */ 194 [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, 195 [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL, 196 [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, 197 [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL, 198 199 [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_A8_PERFCTR_L1_ICACHE_ACCESS, 200 [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL, 201 202 [C(LL)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_A8_PERFCTR_L2_CACHE_ACCESS, 203 [C(LL)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_A8_PERFCTR_L2_CACHE_REFILL, 204 [C(LL)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_A8_PERFCTR_L2_CACHE_ACCESS, 205 [C(LL)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_A8_PERFCTR_L2_CACHE_REFILL, 206 207 [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL, 208 [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL, 209 210 [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL, 211 [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL, 212 213 [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, 214 [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, 215 [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, 216 [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, 217 }; 218 219 /* 220 * Cortex-A9 HW events mapping 221 */ 222 static const unsigned armv7_a9_perf_map[PERF_COUNT_HW_MAX] = { 223 PERF_MAP_ALL_UNSUPPORTED, 224 [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES, 225 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_A9_PERFCTR_INSTR_CORE_RENAME, 226 [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, 227 [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_L1_DCACHE_REFILL, 228 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE, 229 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, 230 [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = ARMV7_A9_PERFCTR_STALL_ICACHE, 231 [PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = ARMV7_A9_PERFCTR_STALL_DISPATCH, 232 }; 233 234 static const unsigned armv7_a9_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] 235 [PERF_COUNT_HW_CACHE_OP_MAX] 236 [PERF_COUNT_HW_CACHE_RESULT_MAX] = { 237 PERF_CACHE_MAP_ALL_UNSUPPORTED, 238 239 /* 240 * The performance counters don't differentiate between read and write 241 * accesses/misses so this isn't strictly correct, but it's the best we 242 * can do. Writes and reads get combined. 243 */ 244 [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, 245 [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL, 246 [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, 247 [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL, 248 249 [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL, 250 251 [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL, 252 [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL, 253 254 [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL, 255 [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL, 256 257 [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, 258 [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, 259 [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, 260 [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, 261 }; 262 263 /* 264 * Cortex-A5 HW events mapping 265 */ 266 static const unsigned armv7_a5_perf_map[PERF_COUNT_HW_MAX] = { 267 PERF_MAP_ALL_UNSUPPORTED, 268 [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES, 269 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED, 270 [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, 271 [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_L1_DCACHE_REFILL, 272 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE, 273 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, 274 }; 275 276 static const unsigned armv7_a5_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] 277 [PERF_COUNT_HW_CACHE_OP_MAX] 278 [PERF_COUNT_HW_CACHE_RESULT_MAX] = { 279 PERF_CACHE_MAP_ALL_UNSUPPORTED, 280 281 [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, 282 [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL, 283 [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, 284 [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL, 285 [C(L1D)][C(OP_PREFETCH)][C(RESULT_ACCESS)] = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL, 286 [C(L1D)][C(OP_PREFETCH)][C(RESULT_MISS)] = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL_DROP, 287 288 [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_ICACHE_ACCESS, 289 [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL, 290 /* 291 * The prefetch counters don't differentiate between the I side and the 292 * D side. 293 */ 294 [C(L1I)][C(OP_PREFETCH)][C(RESULT_ACCESS)] = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL, 295 [C(L1I)][C(OP_PREFETCH)][C(RESULT_MISS)] = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL_DROP, 296 297 [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL, 298 [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL, 299 300 [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL, 301 [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL, 302 303 [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, 304 [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, 305 [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, 306 [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, 307 }; 308 309 /* 310 * Cortex-A15 HW events mapping 311 */ 312 static const unsigned armv7_a15_perf_map[PERF_COUNT_HW_MAX] = { 313 PERF_MAP_ALL_UNSUPPORTED, 314 [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES, 315 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED, 316 [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, 317 [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_L1_DCACHE_REFILL, 318 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_A15_PERFCTR_PC_WRITE_SPEC, 319 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, 320 [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_BUS_CYCLES, 321 }; 322 323 static const unsigned armv7_a15_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] 324 [PERF_COUNT_HW_CACHE_OP_MAX] 325 [PERF_COUNT_HW_CACHE_RESULT_MAX] = { 326 PERF_CACHE_MAP_ALL_UNSUPPORTED, 327 328 [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_READ, 329 [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_READ, 330 [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_WRITE, 331 [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_WRITE, 332 333 /* 334 * Not all performance counters differentiate between read and write 335 * accesses/misses so we're not always strictly correct, but it's the 336 * best we can do. Writes and reads get combined in these cases. 337 */ 338 [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_ICACHE_ACCESS, 339 [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL, 340 341 [C(LL)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_READ, 342 [C(LL)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_A15_PERFCTR_L2_CACHE_REFILL_READ, 343 [C(LL)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_WRITE, 344 [C(LL)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_A15_PERFCTR_L2_CACHE_REFILL_WRITE, 345 346 [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_A15_PERFCTR_DTLB_REFILL_L1_READ, 347 [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_A15_PERFCTR_DTLB_REFILL_L1_WRITE, 348 349 [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL, 350 [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL, 351 352 [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, 353 [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, 354 [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, 355 [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, 356 }; 357 358 /* 359 * Cortex-A7 HW events mapping 360 */ 361 static const unsigned armv7_a7_perf_map[PERF_COUNT_HW_MAX] = { 362 PERF_MAP_ALL_UNSUPPORTED, 363 [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES, 364 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED, 365 [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, 366 [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_L1_DCACHE_REFILL, 367 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE, 368 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, 369 [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_BUS_CYCLES, 370 }; 371 372 static const unsigned armv7_a7_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] 373 [PERF_COUNT_HW_CACHE_OP_MAX] 374 [PERF_COUNT_HW_CACHE_RESULT_MAX] = { 375 PERF_CACHE_MAP_ALL_UNSUPPORTED, 376 377 /* 378 * The performance counters don't differentiate between read and write 379 * accesses/misses so this isn't strictly correct, but it's the best we 380 * can do. Writes and reads get combined. 381 */ 382 [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, 383 [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL, 384 [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, 385 [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL, 386 387 [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_ICACHE_ACCESS, 388 [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL, 389 390 [C(LL)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L2_CACHE_ACCESS, 391 [C(LL)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L2_CACHE_REFILL, 392 [C(LL)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L2_CACHE_ACCESS, 393 [C(LL)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L2_CACHE_REFILL, 394 395 [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL, 396 [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL, 397 398 [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL, 399 [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL, 400 401 [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, 402 [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, 403 [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, 404 [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, 405 }; 406 407 /* 408 * Cortex-A12 HW events mapping 409 */ 410 static const unsigned armv7_a12_perf_map[PERF_COUNT_HW_MAX] = { 411 PERF_MAP_ALL_UNSUPPORTED, 412 [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES, 413 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED, 414 [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, 415 [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_L1_DCACHE_REFILL, 416 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_A12_PERFCTR_PC_WRITE_SPEC, 417 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, 418 [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_BUS_CYCLES, 419 }; 420 421 static const unsigned armv7_a12_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] 422 [PERF_COUNT_HW_CACHE_OP_MAX] 423 [PERF_COUNT_HW_CACHE_RESULT_MAX] = { 424 PERF_CACHE_MAP_ALL_UNSUPPORTED, 425 426 [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_READ, 427 [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL, 428 [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_WRITE, 429 [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL, 430 431 /* 432 * Not all performance counters differentiate between read and write 433 * accesses/misses so we're not always strictly correct, but it's the 434 * best we can do. Writes and reads get combined in these cases. 435 */ 436 [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_ICACHE_ACCESS, 437 [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL, 438 439 [C(LL)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_READ, 440 [C(LL)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L2_CACHE_REFILL, 441 [C(LL)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_WRITE, 442 [C(LL)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L2_CACHE_REFILL, 443 444 [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL, 445 [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL, 446 [C(DTLB)][C(OP_PREFETCH)][C(RESULT_MISS)] = ARMV7_A12_PERFCTR_PF_TLB_REFILL, 447 448 [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL, 449 [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL, 450 451 [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, 452 [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, 453 [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, 454 [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, 455 }; 456 457 /* 458 * Krait HW events mapping 459 */ 460 static const unsigned krait_perf_map[PERF_COUNT_HW_MAX] = { 461 PERF_MAP_ALL_UNSUPPORTED, 462 [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES, 463 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED, 464 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE, 465 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, 466 [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_CLOCK_CYCLES, 467 }; 468 469 static const unsigned krait_perf_map_no_branch[PERF_COUNT_HW_MAX] = { 470 PERF_MAP_ALL_UNSUPPORTED, 471 [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES, 472 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED, 473 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, 474 [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_CLOCK_CYCLES, 475 }; 476 477 static const unsigned krait_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] 478 [PERF_COUNT_HW_CACHE_OP_MAX] 479 [PERF_COUNT_HW_CACHE_RESULT_MAX] = { 480 PERF_CACHE_MAP_ALL_UNSUPPORTED, 481 482 /* 483 * The performance counters don't differentiate between read and write 484 * accesses/misses so this isn't strictly correct, but it's the best we 485 * can do. Writes and reads get combined. 486 */ 487 [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, 488 [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL, 489 [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, 490 [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL, 491 492 [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = KRAIT_PERFCTR_L1_ICACHE_ACCESS, 493 [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = KRAIT_PERFCTR_L1_ICACHE_MISS, 494 495 [C(DTLB)][C(OP_READ)][C(RESULT_ACCESS)] = KRAIT_PERFCTR_L1_DTLB_ACCESS, 496 [C(DTLB)][C(OP_WRITE)][C(RESULT_ACCESS)] = KRAIT_PERFCTR_L1_DTLB_ACCESS, 497 498 [C(ITLB)][C(OP_READ)][C(RESULT_ACCESS)] = KRAIT_PERFCTR_L1_ITLB_ACCESS, 499 [C(ITLB)][C(OP_WRITE)][C(RESULT_ACCESS)] = KRAIT_PERFCTR_L1_ITLB_ACCESS, 500 501 [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, 502 [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, 503 [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, 504 [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, 505 }; 506 507 /* 508 * Scorpion HW events mapping 509 */ 510 static const unsigned scorpion_perf_map[PERF_COUNT_HW_MAX] = { 511 PERF_MAP_ALL_UNSUPPORTED, 512 [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES, 513 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED, 514 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE, 515 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, 516 [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_CLOCK_CYCLES, 517 }; 518 519 static const unsigned scorpion_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] 520 [PERF_COUNT_HW_CACHE_OP_MAX] 521 [PERF_COUNT_HW_CACHE_RESULT_MAX] = { 522 PERF_CACHE_MAP_ALL_UNSUPPORTED, 523 /* 524 * The performance counters don't differentiate between read and write 525 * accesses/misses so this isn't strictly correct, but it's the best we 526 * can do. Writes and reads get combined. 527 */ 528 [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, 529 [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL, 530 [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, 531 [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL, 532 [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = SCORPION_ICACHE_ACCESS, 533 [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = SCORPION_ICACHE_MISS, 534 /* 535 * Only ITLB misses and DTLB refills are supported. If users want the 536 * DTLB refills misses a raw counter must be used. 537 */ 538 [C(DTLB)][C(OP_READ)][C(RESULT_ACCESS)] = SCORPION_DTLB_ACCESS, 539 [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = SCORPION_DTLB_MISS, 540 [C(DTLB)][C(OP_WRITE)][C(RESULT_ACCESS)] = SCORPION_DTLB_ACCESS, 541 [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = SCORPION_DTLB_MISS, 542 [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = SCORPION_ITLB_MISS, 543 [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = SCORPION_ITLB_MISS, 544 [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, 545 [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, 546 [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, 547 [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, 548 }; 549 550 /* 551 * Perf Events' indices 552 */ 553 #define ARMV7_IDX_CYCLE_COUNTER 0 554 #define ARMV7_IDX_COUNTER0 1 555 #define ARMV7_IDX_COUNTER_LAST(cpu_pmu) \ 556 (ARMV7_IDX_CYCLE_COUNTER + cpu_pmu->num_events - 1) 557 558 #define ARMV7_MAX_COUNTERS 32 559 #define ARMV7_COUNTER_MASK (ARMV7_MAX_COUNTERS - 1) 560 561 /* 562 * ARMv7 low level PMNC access 563 */ 564 565 /* 566 * Perf Event to low level counters mapping 567 */ 568 #define ARMV7_IDX_TO_COUNTER(x) \ 569 (((x) - ARMV7_IDX_COUNTER0) & ARMV7_COUNTER_MASK) 570 571 /* 572 * Per-CPU PMNC: config reg 573 */ 574 #define ARMV7_PMNC_E (1 << 0) /* Enable all counters */ 575 #define ARMV7_PMNC_P (1 << 1) /* Reset all counters */ 576 #define ARMV7_PMNC_C (1 << 2) /* Cycle counter reset */ 577 #define ARMV7_PMNC_D (1 << 3) /* CCNT counts every 64th cpu cycle */ 578 #define ARMV7_PMNC_X (1 << 4) /* Export to ETM */ 579 #define ARMV7_PMNC_DP (1 << 5) /* Disable CCNT if non-invasive debug*/ 580 #define ARMV7_PMNC_N_SHIFT 11 /* Number of counters supported */ 581 #define ARMV7_PMNC_N_MASK 0x1f 582 #define ARMV7_PMNC_MASK 0x3f /* Mask for writable bits */ 583 584 /* 585 * FLAG: counters overflow flag status reg 586 */ 587 #define ARMV7_FLAG_MASK 0xffffffff /* Mask for writable bits */ 588 #define ARMV7_OVERFLOWED_MASK ARMV7_FLAG_MASK 589 590 /* 591 * PMXEVTYPER: Event selection reg 592 */ 593 #define ARMV7_EVTYPE_MASK 0xc80000ff /* Mask for writable bits */ 594 #define ARMV7_EVTYPE_EVENT 0xff /* Mask for EVENT bits */ 595 596 /* 597 * Event filters for PMUv2 598 */ 599 #define ARMV7_EXCLUDE_PL1 (1 << 31) 600 #define ARMV7_EXCLUDE_USER (1 << 30) 601 #define ARMV7_INCLUDE_HYP (1 << 27) 602 603 static inline u32 armv7_pmnc_read(void) 604 { 605 u32 val; 606 asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r"(val)); 607 return val; 608 } 609 610 static inline void armv7_pmnc_write(u32 val) 611 { 612 val &= ARMV7_PMNC_MASK; 613 isb(); 614 asm volatile("mcr p15, 0, %0, c9, c12, 0" : : "r"(val)); 615 } 616 617 static inline int armv7_pmnc_has_overflowed(u32 pmnc) 618 { 619 return pmnc & ARMV7_OVERFLOWED_MASK; 620 } 621 622 static inline int armv7_pmnc_counter_valid(struct arm_pmu *cpu_pmu, int idx) 623 { 624 return idx >= ARMV7_IDX_CYCLE_COUNTER && 625 idx <= ARMV7_IDX_COUNTER_LAST(cpu_pmu); 626 } 627 628 static inline int armv7_pmnc_counter_has_overflowed(u32 pmnc, int idx) 629 { 630 return pmnc & BIT(ARMV7_IDX_TO_COUNTER(idx)); 631 } 632 633 static inline void armv7_pmnc_select_counter(int idx) 634 { 635 u32 counter = ARMV7_IDX_TO_COUNTER(idx); 636 asm volatile("mcr p15, 0, %0, c9, c12, 5" : : "r" (counter)); 637 isb(); 638 } 639 640 static inline u32 armv7pmu_read_counter(struct perf_event *event) 641 { 642 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); 643 struct hw_perf_event *hwc = &event->hw; 644 int idx = hwc->idx; 645 u32 value = 0; 646 647 if (!armv7_pmnc_counter_valid(cpu_pmu, idx)) { 648 pr_err("CPU%u reading wrong counter %d\n", 649 smp_processor_id(), idx); 650 } else if (idx == ARMV7_IDX_CYCLE_COUNTER) { 651 asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (value)); 652 } else { 653 armv7_pmnc_select_counter(idx); 654 asm volatile("mrc p15, 0, %0, c9, c13, 2" : "=r" (value)); 655 } 656 657 return value; 658 } 659 660 static inline void armv7pmu_write_counter(struct perf_event *event, u32 value) 661 { 662 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); 663 struct hw_perf_event *hwc = &event->hw; 664 int idx = hwc->idx; 665 666 if (!armv7_pmnc_counter_valid(cpu_pmu, idx)) { 667 pr_err("CPU%u writing wrong counter %d\n", 668 smp_processor_id(), idx); 669 } else if (idx == ARMV7_IDX_CYCLE_COUNTER) { 670 asm volatile("mcr p15, 0, %0, c9, c13, 0" : : "r" (value)); 671 } else { 672 armv7_pmnc_select_counter(idx); 673 asm volatile("mcr p15, 0, %0, c9, c13, 2" : : "r" (value)); 674 } 675 } 676 677 static inline void armv7_pmnc_write_evtsel(int idx, u32 val) 678 { 679 armv7_pmnc_select_counter(idx); 680 val &= ARMV7_EVTYPE_MASK; 681 asm volatile("mcr p15, 0, %0, c9, c13, 1" : : "r" (val)); 682 } 683 684 static inline void armv7_pmnc_enable_counter(int idx) 685 { 686 u32 counter = ARMV7_IDX_TO_COUNTER(idx); 687 asm volatile("mcr p15, 0, %0, c9, c12, 1" : : "r" (BIT(counter))); 688 } 689 690 static inline void armv7_pmnc_disable_counter(int idx) 691 { 692 u32 counter = ARMV7_IDX_TO_COUNTER(idx); 693 asm volatile("mcr p15, 0, %0, c9, c12, 2" : : "r" (BIT(counter))); 694 } 695 696 static inline void armv7_pmnc_enable_intens(int idx) 697 { 698 u32 counter = ARMV7_IDX_TO_COUNTER(idx); 699 asm volatile("mcr p15, 0, %0, c9, c14, 1" : : "r" (BIT(counter))); 700 } 701 702 static inline void armv7_pmnc_disable_intens(int idx) 703 { 704 u32 counter = ARMV7_IDX_TO_COUNTER(idx); 705 asm volatile("mcr p15, 0, %0, c9, c14, 2" : : "r" (BIT(counter))); 706 isb(); 707 /* Clear the overflow flag in case an interrupt is pending. */ 708 asm volatile("mcr p15, 0, %0, c9, c12, 3" : : "r" (BIT(counter))); 709 isb(); 710 } 711 712 static inline u32 armv7_pmnc_getreset_flags(void) 713 { 714 u32 val; 715 716 /* Read */ 717 asm volatile("mrc p15, 0, %0, c9, c12, 3" : "=r" (val)); 718 719 /* Write to clear flags */ 720 val &= ARMV7_FLAG_MASK; 721 asm volatile("mcr p15, 0, %0, c9, c12, 3" : : "r" (val)); 722 723 return val; 724 } 725 726 #ifdef DEBUG 727 static void armv7_pmnc_dump_regs(struct arm_pmu *cpu_pmu) 728 { 729 u32 val; 730 unsigned int cnt; 731 732 pr_info("PMNC registers dump:\n"); 733 734 asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r" (val)); 735 pr_info("PMNC =0x%08x\n", val); 736 737 asm volatile("mrc p15, 0, %0, c9, c12, 1" : "=r" (val)); 738 pr_info("CNTENS=0x%08x\n", val); 739 740 asm volatile("mrc p15, 0, %0, c9, c14, 1" : "=r" (val)); 741 pr_info("INTENS=0x%08x\n", val); 742 743 asm volatile("mrc p15, 0, %0, c9, c12, 3" : "=r" (val)); 744 pr_info("FLAGS =0x%08x\n", val); 745 746 asm volatile("mrc p15, 0, %0, c9, c12, 5" : "=r" (val)); 747 pr_info("SELECT=0x%08x\n", val); 748 749 asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (val)); 750 pr_info("CCNT =0x%08x\n", val); 751 752 for (cnt = ARMV7_IDX_COUNTER0; 753 cnt <= ARMV7_IDX_COUNTER_LAST(cpu_pmu); cnt++) { 754 armv7_pmnc_select_counter(cnt); 755 asm volatile("mrc p15, 0, %0, c9, c13, 2" : "=r" (val)); 756 pr_info("CNT[%d] count =0x%08x\n", 757 ARMV7_IDX_TO_COUNTER(cnt), val); 758 asm volatile("mrc p15, 0, %0, c9, c13, 1" : "=r" (val)); 759 pr_info("CNT[%d] evtsel=0x%08x\n", 760 ARMV7_IDX_TO_COUNTER(cnt), val); 761 } 762 } 763 #endif 764 765 static void armv7pmu_enable_event(struct perf_event *event) 766 { 767 unsigned long flags; 768 struct hw_perf_event *hwc = &event->hw; 769 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); 770 struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events); 771 int idx = hwc->idx; 772 773 if (!armv7_pmnc_counter_valid(cpu_pmu, idx)) { 774 pr_err("CPU%u enabling wrong PMNC counter IRQ enable %d\n", 775 smp_processor_id(), idx); 776 return; 777 } 778 779 /* 780 * Enable counter and interrupt, and set the counter to count 781 * the event that we're interested in. 782 */ 783 raw_spin_lock_irqsave(&events->pmu_lock, flags); 784 785 /* 786 * Disable counter 787 */ 788 armv7_pmnc_disable_counter(idx); 789 790 /* 791 * Set event (if destined for PMNx counters) 792 * We only need to set the event for the cycle counter if we 793 * have the ability to perform event filtering. 794 */ 795 if (cpu_pmu->set_event_filter || idx != ARMV7_IDX_CYCLE_COUNTER) 796 armv7_pmnc_write_evtsel(idx, hwc->config_base); 797 798 /* 799 * Enable interrupt for this counter 800 */ 801 armv7_pmnc_enable_intens(idx); 802 803 /* 804 * Enable counter 805 */ 806 armv7_pmnc_enable_counter(idx); 807 808 raw_spin_unlock_irqrestore(&events->pmu_lock, flags); 809 } 810 811 static void armv7pmu_disable_event(struct perf_event *event) 812 { 813 unsigned long flags; 814 struct hw_perf_event *hwc = &event->hw; 815 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); 816 struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events); 817 int idx = hwc->idx; 818 819 if (!armv7_pmnc_counter_valid(cpu_pmu, idx)) { 820 pr_err("CPU%u disabling wrong PMNC counter IRQ enable %d\n", 821 smp_processor_id(), idx); 822 return; 823 } 824 825 /* 826 * Disable counter and interrupt 827 */ 828 raw_spin_lock_irqsave(&events->pmu_lock, flags); 829 830 /* 831 * Disable counter 832 */ 833 armv7_pmnc_disable_counter(idx); 834 835 /* 836 * Disable interrupt for this counter 837 */ 838 armv7_pmnc_disable_intens(idx); 839 840 raw_spin_unlock_irqrestore(&events->pmu_lock, flags); 841 } 842 843 static irqreturn_t armv7pmu_handle_irq(int irq_num, void *dev) 844 { 845 u32 pmnc; 846 struct perf_sample_data data; 847 struct arm_pmu *cpu_pmu = (struct arm_pmu *)dev; 848 struct pmu_hw_events *cpuc = this_cpu_ptr(cpu_pmu->hw_events); 849 struct pt_regs *regs; 850 int idx; 851 852 /* 853 * Get and reset the IRQ flags 854 */ 855 pmnc = armv7_pmnc_getreset_flags(); 856 857 /* 858 * Did an overflow occur? 859 */ 860 if (!armv7_pmnc_has_overflowed(pmnc)) 861 return IRQ_NONE; 862 863 /* 864 * Handle the counter(s) overflow(s) 865 */ 866 regs = get_irq_regs(); 867 868 for (idx = 0; idx < cpu_pmu->num_events; ++idx) { 869 struct perf_event *event = cpuc->events[idx]; 870 struct hw_perf_event *hwc; 871 872 /* Ignore if we don't have an event. */ 873 if (!event) 874 continue; 875 876 /* 877 * We have a single interrupt for all counters. Check that 878 * each counter has overflowed before we process it. 879 */ 880 if (!armv7_pmnc_counter_has_overflowed(pmnc, idx)) 881 continue; 882 883 hwc = &event->hw; 884 armpmu_event_update(event); 885 perf_sample_data_init(&data, 0, hwc->last_period); 886 if (!armpmu_event_set_period(event)) 887 continue; 888 889 if (perf_event_overflow(event, &data, regs)) 890 cpu_pmu->disable(event); 891 } 892 893 /* 894 * Handle the pending perf events. 895 * 896 * Note: this call *must* be run with interrupts disabled. For 897 * platforms that can have the PMU interrupts raised as an NMI, this 898 * will not work. 899 */ 900 irq_work_run(); 901 902 return IRQ_HANDLED; 903 } 904 905 static void armv7pmu_start(struct arm_pmu *cpu_pmu) 906 { 907 unsigned long flags; 908 struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events); 909 910 raw_spin_lock_irqsave(&events->pmu_lock, flags); 911 /* Enable all counters */ 912 armv7_pmnc_write(armv7_pmnc_read() | ARMV7_PMNC_E); 913 raw_spin_unlock_irqrestore(&events->pmu_lock, flags); 914 } 915 916 static void armv7pmu_stop(struct arm_pmu *cpu_pmu) 917 { 918 unsigned long flags; 919 struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events); 920 921 raw_spin_lock_irqsave(&events->pmu_lock, flags); 922 /* Disable all counters */ 923 armv7_pmnc_write(armv7_pmnc_read() & ~ARMV7_PMNC_E); 924 raw_spin_unlock_irqrestore(&events->pmu_lock, flags); 925 } 926 927 static int armv7pmu_get_event_idx(struct pmu_hw_events *cpuc, 928 struct perf_event *event) 929 { 930 int idx; 931 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); 932 struct hw_perf_event *hwc = &event->hw; 933 unsigned long evtype = hwc->config_base & ARMV7_EVTYPE_EVENT; 934 935 /* Always place a cycle counter into the cycle counter. */ 936 if (evtype == ARMV7_PERFCTR_CPU_CYCLES) { 937 if (test_and_set_bit(ARMV7_IDX_CYCLE_COUNTER, cpuc->used_mask)) 938 return -EAGAIN; 939 940 return ARMV7_IDX_CYCLE_COUNTER; 941 } 942 943 /* 944 * For anything other than a cycle counter, try and use 945 * the events counters 946 */ 947 for (idx = ARMV7_IDX_COUNTER0; idx < cpu_pmu->num_events; ++idx) { 948 if (!test_and_set_bit(idx, cpuc->used_mask)) 949 return idx; 950 } 951 952 /* The counters are all in use. */ 953 return -EAGAIN; 954 } 955 956 /* 957 * Add an event filter to a given event. This will only work for PMUv2 PMUs. 958 */ 959 static int armv7pmu_set_event_filter(struct hw_perf_event *event, 960 struct perf_event_attr *attr) 961 { 962 unsigned long config_base = 0; 963 964 if (attr->exclude_idle) 965 return -EPERM; 966 if (attr->exclude_user) 967 config_base |= ARMV7_EXCLUDE_USER; 968 if (attr->exclude_kernel) 969 config_base |= ARMV7_EXCLUDE_PL1; 970 if (!attr->exclude_hv) 971 config_base |= ARMV7_INCLUDE_HYP; 972 973 /* 974 * Install the filter into config_base as this is used to 975 * construct the event type. 976 */ 977 event->config_base = config_base; 978 979 return 0; 980 } 981 982 static void armv7pmu_reset(void *info) 983 { 984 struct arm_pmu *cpu_pmu = (struct arm_pmu *)info; 985 u32 idx, nb_cnt = cpu_pmu->num_events; 986 987 /* The counter and interrupt enable registers are unknown at reset. */ 988 for (idx = ARMV7_IDX_CYCLE_COUNTER; idx < nb_cnt; ++idx) { 989 armv7_pmnc_disable_counter(idx); 990 armv7_pmnc_disable_intens(idx); 991 } 992 993 /* Initialize & Reset PMNC: C and P bits */ 994 armv7_pmnc_write(ARMV7_PMNC_P | ARMV7_PMNC_C); 995 } 996 997 static int armv7_a8_map_event(struct perf_event *event) 998 { 999 return armpmu_map_event(event, &armv7_a8_perf_map, 1000 &armv7_a8_perf_cache_map, 0xFF); 1001 } 1002 1003 static int armv7_a9_map_event(struct perf_event *event) 1004 { 1005 return armpmu_map_event(event, &armv7_a9_perf_map, 1006 &armv7_a9_perf_cache_map, 0xFF); 1007 } 1008 1009 static int armv7_a5_map_event(struct perf_event *event) 1010 { 1011 return armpmu_map_event(event, &armv7_a5_perf_map, 1012 &armv7_a5_perf_cache_map, 0xFF); 1013 } 1014 1015 static int armv7_a15_map_event(struct perf_event *event) 1016 { 1017 return armpmu_map_event(event, &armv7_a15_perf_map, 1018 &armv7_a15_perf_cache_map, 0xFF); 1019 } 1020 1021 static int armv7_a7_map_event(struct perf_event *event) 1022 { 1023 return armpmu_map_event(event, &armv7_a7_perf_map, 1024 &armv7_a7_perf_cache_map, 0xFF); 1025 } 1026 1027 static int armv7_a12_map_event(struct perf_event *event) 1028 { 1029 return armpmu_map_event(event, &armv7_a12_perf_map, 1030 &armv7_a12_perf_cache_map, 0xFF); 1031 } 1032 1033 static int krait_map_event(struct perf_event *event) 1034 { 1035 return armpmu_map_event(event, &krait_perf_map, 1036 &krait_perf_cache_map, 0xFFFFF); 1037 } 1038 1039 static int krait_map_event_no_branch(struct perf_event *event) 1040 { 1041 return armpmu_map_event(event, &krait_perf_map_no_branch, 1042 &krait_perf_cache_map, 0xFFFFF); 1043 } 1044 1045 static int scorpion_map_event(struct perf_event *event) 1046 { 1047 return armpmu_map_event(event, &scorpion_perf_map, 1048 &scorpion_perf_cache_map, 0xFFFFF); 1049 } 1050 1051 static void armv7pmu_init(struct arm_pmu *cpu_pmu) 1052 { 1053 cpu_pmu->handle_irq = armv7pmu_handle_irq; 1054 cpu_pmu->enable = armv7pmu_enable_event; 1055 cpu_pmu->disable = armv7pmu_disable_event; 1056 cpu_pmu->read_counter = armv7pmu_read_counter; 1057 cpu_pmu->write_counter = armv7pmu_write_counter; 1058 cpu_pmu->get_event_idx = armv7pmu_get_event_idx; 1059 cpu_pmu->start = armv7pmu_start; 1060 cpu_pmu->stop = armv7pmu_stop; 1061 cpu_pmu->reset = armv7pmu_reset; 1062 cpu_pmu->max_period = (1LLU << 32) - 1; 1063 }; 1064 1065 static void armv7_read_num_pmnc_events(void *info) 1066 { 1067 int *nb_cnt = info; 1068 1069 /* Read the nb of CNTx counters supported from PMNC */ 1070 *nb_cnt = (armv7_pmnc_read() >> ARMV7_PMNC_N_SHIFT) & ARMV7_PMNC_N_MASK; 1071 1072 /* Add the CPU cycles counter */ 1073 *nb_cnt += 1; 1074 } 1075 1076 static int armv7_probe_num_events(struct arm_pmu *arm_pmu) 1077 { 1078 return smp_call_function_any(&arm_pmu->supported_cpus, 1079 armv7_read_num_pmnc_events, 1080 &arm_pmu->num_events, 1); 1081 } 1082 1083 static int armv7_a8_pmu_init(struct arm_pmu *cpu_pmu) 1084 { 1085 armv7pmu_init(cpu_pmu); 1086 cpu_pmu->name = "armv7_cortex_a8"; 1087 cpu_pmu->map_event = armv7_a8_map_event; 1088 return armv7_probe_num_events(cpu_pmu); 1089 } 1090 1091 static int armv7_a9_pmu_init(struct arm_pmu *cpu_pmu) 1092 { 1093 armv7pmu_init(cpu_pmu); 1094 cpu_pmu->name = "armv7_cortex_a9"; 1095 cpu_pmu->map_event = armv7_a9_map_event; 1096 return armv7_probe_num_events(cpu_pmu); 1097 } 1098 1099 static int armv7_a5_pmu_init(struct arm_pmu *cpu_pmu) 1100 { 1101 armv7pmu_init(cpu_pmu); 1102 cpu_pmu->name = "armv7_cortex_a5"; 1103 cpu_pmu->map_event = armv7_a5_map_event; 1104 return armv7_probe_num_events(cpu_pmu); 1105 } 1106 1107 static int armv7_a15_pmu_init(struct arm_pmu *cpu_pmu) 1108 { 1109 armv7pmu_init(cpu_pmu); 1110 cpu_pmu->name = "armv7_cortex_a15"; 1111 cpu_pmu->map_event = armv7_a15_map_event; 1112 cpu_pmu->set_event_filter = armv7pmu_set_event_filter; 1113 return armv7_probe_num_events(cpu_pmu); 1114 } 1115 1116 static int armv7_a7_pmu_init(struct arm_pmu *cpu_pmu) 1117 { 1118 armv7pmu_init(cpu_pmu); 1119 cpu_pmu->name = "armv7_cortex_a7"; 1120 cpu_pmu->map_event = armv7_a7_map_event; 1121 cpu_pmu->set_event_filter = armv7pmu_set_event_filter; 1122 return armv7_probe_num_events(cpu_pmu); 1123 } 1124 1125 static int armv7_a12_pmu_init(struct arm_pmu *cpu_pmu) 1126 { 1127 armv7pmu_init(cpu_pmu); 1128 cpu_pmu->name = "armv7_cortex_a12"; 1129 cpu_pmu->map_event = armv7_a12_map_event; 1130 cpu_pmu->set_event_filter = armv7pmu_set_event_filter; 1131 return armv7_probe_num_events(cpu_pmu); 1132 } 1133 1134 static int armv7_a17_pmu_init(struct arm_pmu *cpu_pmu) 1135 { 1136 int ret = armv7_a12_pmu_init(cpu_pmu); 1137 cpu_pmu->name = "armv7_cortex_a17"; 1138 return ret; 1139 } 1140 1141 /* 1142 * Krait Performance Monitor Region Event Selection Register (PMRESRn) 1143 * 1144 * 31 30 24 16 8 0 1145 * +--------------------------------+ 1146 * PMRESR0 | EN | CC | CC | CC | CC | N = 1, R = 0 1147 * +--------------------------------+ 1148 * PMRESR1 | EN | CC | CC | CC | CC | N = 1, R = 1 1149 * +--------------------------------+ 1150 * PMRESR2 | EN | CC | CC | CC | CC | N = 1, R = 2 1151 * +--------------------------------+ 1152 * VPMRESR0 | EN | CC | CC | CC | CC | N = 2, R = ? 1153 * +--------------------------------+ 1154 * EN | G=3 | G=2 | G=1 | G=0 1155 * 1156 * Event Encoding: 1157 * 1158 * hwc->config_base = 0xNRCCG 1159 * 1160 * N = prefix, 1 for Krait CPU (PMRESRn), 2 for Venum VFP (VPMRESR) 1161 * R = region register 1162 * CC = class of events the group G is choosing from 1163 * G = group or particular event 1164 * 1165 * Example: 0x12021 is a Krait CPU event in PMRESR2's group 1 with code 2 1166 * 1167 * A region (R) corresponds to a piece of the CPU (execution unit, instruction 1168 * unit, etc.) while the event code (CC) corresponds to a particular class of 1169 * events (interrupts for example). An event code is broken down into 1170 * groups (G) that can be mapped into the PMU (irq, fiqs, and irq+fiqs for 1171 * example). 1172 */ 1173 1174 #define KRAIT_EVENT (1 << 16) 1175 #define VENUM_EVENT (2 << 16) 1176 #define KRAIT_EVENT_MASK (KRAIT_EVENT | VENUM_EVENT) 1177 #define PMRESRn_EN BIT(31) 1178 1179 #define EVENT_REGION(event) (((event) >> 12) & 0xf) /* R */ 1180 #define EVENT_GROUP(event) ((event) & 0xf) /* G */ 1181 #define EVENT_CODE(event) (((event) >> 4) & 0xff) /* CC */ 1182 #define EVENT_VENUM(event) (!!(event & VENUM_EVENT)) /* N=2 */ 1183 #define EVENT_CPU(event) (!!(event & KRAIT_EVENT)) /* N=1 */ 1184 1185 static u32 krait_read_pmresrn(int n) 1186 { 1187 u32 val; 1188 1189 switch (n) { 1190 case 0: 1191 asm volatile("mrc p15, 1, %0, c9, c15, 0" : "=r" (val)); 1192 break; 1193 case 1: 1194 asm volatile("mrc p15, 1, %0, c9, c15, 1" : "=r" (val)); 1195 break; 1196 case 2: 1197 asm volatile("mrc p15, 1, %0, c9, c15, 2" : "=r" (val)); 1198 break; 1199 default: 1200 BUG(); /* Should be validated in krait_pmu_get_event_idx() */ 1201 } 1202 1203 return val; 1204 } 1205 1206 static void krait_write_pmresrn(int n, u32 val) 1207 { 1208 switch (n) { 1209 case 0: 1210 asm volatile("mcr p15, 1, %0, c9, c15, 0" : : "r" (val)); 1211 break; 1212 case 1: 1213 asm volatile("mcr p15, 1, %0, c9, c15, 1" : : "r" (val)); 1214 break; 1215 case 2: 1216 asm volatile("mcr p15, 1, %0, c9, c15, 2" : : "r" (val)); 1217 break; 1218 default: 1219 BUG(); /* Should be validated in krait_pmu_get_event_idx() */ 1220 } 1221 } 1222 1223 static u32 venum_read_pmresr(void) 1224 { 1225 u32 val; 1226 asm volatile("mrc p10, 7, %0, c11, c0, 0" : "=r" (val)); 1227 return val; 1228 } 1229 1230 static void venum_write_pmresr(u32 val) 1231 { 1232 asm volatile("mcr p10, 7, %0, c11, c0, 0" : : "r" (val)); 1233 } 1234 1235 static void venum_pre_pmresr(u32 *venum_orig_val, u32 *fp_orig_val) 1236 { 1237 u32 venum_new_val; 1238 u32 fp_new_val; 1239 1240 BUG_ON(preemptible()); 1241 /* CPACR Enable CP10 and CP11 access */ 1242 *venum_orig_val = get_copro_access(); 1243 venum_new_val = *venum_orig_val | CPACC_SVC(10) | CPACC_SVC(11); 1244 set_copro_access(venum_new_val); 1245 1246 /* Enable FPEXC */ 1247 *fp_orig_val = fmrx(FPEXC); 1248 fp_new_val = *fp_orig_val | FPEXC_EN; 1249 fmxr(FPEXC, fp_new_val); 1250 } 1251 1252 static void venum_post_pmresr(u32 venum_orig_val, u32 fp_orig_val) 1253 { 1254 BUG_ON(preemptible()); 1255 /* Restore FPEXC */ 1256 fmxr(FPEXC, fp_orig_val); 1257 isb(); 1258 /* Restore CPACR */ 1259 set_copro_access(venum_orig_val); 1260 } 1261 1262 static u32 krait_get_pmresrn_event(unsigned int region) 1263 { 1264 static const u32 pmresrn_table[] = { KRAIT_PMRESR0_GROUP0, 1265 KRAIT_PMRESR1_GROUP0, 1266 KRAIT_PMRESR2_GROUP0 }; 1267 return pmresrn_table[region]; 1268 } 1269 1270 static void krait_evt_setup(int idx, u32 config_base) 1271 { 1272 u32 val; 1273 u32 mask; 1274 u32 vval, fval; 1275 unsigned int region = EVENT_REGION(config_base); 1276 unsigned int group = EVENT_GROUP(config_base); 1277 unsigned int code = EVENT_CODE(config_base); 1278 unsigned int group_shift; 1279 bool venum_event = EVENT_VENUM(config_base); 1280 1281 group_shift = group * 8; 1282 mask = 0xff << group_shift; 1283 1284 /* Configure evtsel for the region and group */ 1285 if (venum_event) 1286 val = KRAIT_VPMRESR0_GROUP0; 1287 else 1288 val = krait_get_pmresrn_event(region); 1289 val += group; 1290 /* Mix in mode-exclusion bits */ 1291 val |= config_base & (ARMV7_EXCLUDE_USER | ARMV7_EXCLUDE_PL1); 1292 armv7_pmnc_write_evtsel(idx, val); 1293 1294 if (venum_event) { 1295 venum_pre_pmresr(&vval, &fval); 1296 val = venum_read_pmresr(); 1297 val &= ~mask; 1298 val |= code << group_shift; 1299 val |= PMRESRn_EN; 1300 venum_write_pmresr(val); 1301 venum_post_pmresr(vval, fval); 1302 } else { 1303 val = krait_read_pmresrn(region); 1304 val &= ~mask; 1305 val |= code << group_shift; 1306 val |= PMRESRn_EN; 1307 krait_write_pmresrn(region, val); 1308 } 1309 } 1310 1311 static u32 clear_pmresrn_group(u32 val, int group) 1312 { 1313 u32 mask; 1314 int group_shift; 1315 1316 group_shift = group * 8; 1317 mask = 0xff << group_shift; 1318 val &= ~mask; 1319 1320 /* Don't clear enable bit if entire region isn't disabled */ 1321 if (val & ~PMRESRn_EN) 1322 return val |= PMRESRn_EN; 1323 1324 return 0; 1325 } 1326 1327 static void krait_clearpmu(u32 config_base) 1328 { 1329 u32 val; 1330 u32 vval, fval; 1331 unsigned int region = EVENT_REGION(config_base); 1332 unsigned int group = EVENT_GROUP(config_base); 1333 bool venum_event = EVENT_VENUM(config_base); 1334 1335 if (venum_event) { 1336 venum_pre_pmresr(&vval, &fval); 1337 val = venum_read_pmresr(); 1338 val = clear_pmresrn_group(val, group); 1339 venum_write_pmresr(val); 1340 venum_post_pmresr(vval, fval); 1341 } else { 1342 val = krait_read_pmresrn(region); 1343 val = clear_pmresrn_group(val, group); 1344 krait_write_pmresrn(region, val); 1345 } 1346 } 1347 1348 static void krait_pmu_disable_event(struct perf_event *event) 1349 { 1350 unsigned long flags; 1351 struct hw_perf_event *hwc = &event->hw; 1352 int idx = hwc->idx; 1353 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); 1354 struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events); 1355 1356 /* Disable counter and interrupt */ 1357 raw_spin_lock_irqsave(&events->pmu_lock, flags); 1358 1359 /* Disable counter */ 1360 armv7_pmnc_disable_counter(idx); 1361 1362 /* 1363 * Clear pmresr code (if destined for PMNx counters) 1364 */ 1365 if (hwc->config_base & KRAIT_EVENT_MASK) 1366 krait_clearpmu(hwc->config_base); 1367 1368 /* Disable interrupt for this counter */ 1369 armv7_pmnc_disable_intens(idx); 1370 1371 raw_spin_unlock_irqrestore(&events->pmu_lock, flags); 1372 } 1373 1374 static void krait_pmu_enable_event(struct perf_event *event) 1375 { 1376 unsigned long flags; 1377 struct hw_perf_event *hwc = &event->hw; 1378 int idx = hwc->idx; 1379 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); 1380 struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events); 1381 1382 /* 1383 * Enable counter and interrupt, and set the counter to count 1384 * the event that we're interested in. 1385 */ 1386 raw_spin_lock_irqsave(&events->pmu_lock, flags); 1387 1388 /* Disable counter */ 1389 armv7_pmnc_disable_counter(idx); 1390 1391 /* 1392 * Set event (if destined for PMNx counters) 1393 * We set the event for the cycle counter because we 1394 * have the ability to perform event filtering. 1395 */ 1396 if (hwc->config_base & KRAIT_EVENT_MASK) 1397 krait_evt_setup(idx, hwc->config_base); 1398 else 1399 armv7_pmnc_write_evtsel(idx, hwc->config_base); 1400 1401 /* Enable interrupt for this counter */ 1402 armv7_pmnc_enable_intens(idx); 1403 1404 /* Enable counter */ 1405 armv7_pmnc_enable_counter(idx); 1406 1407 raw_spin_unlock_irqrestore(&events->pmu_lock, flags); 1408 } 1409 1410 static void krait_pmu_reset(void *info) 1411 { 1412 u32 vval, fval; 1413 struct arm_pmu *cpu_pmu = info; 1414 u32 idx, nb_cnt = cpu_pmu->num_events; 1415 1416 armv7pmu_reset(info); 1417 1418 /* Clear all pmresrs */ 1419 krait_write_pmresrn(0, 0); 1420 krait_write_pmresrn(1, 0); 1421 krait_write_pmresrn(2, 0); 1422 1423 venum_pre_pmresr(&vval, &fval); 1424 venum_write_pmresr(0); 1425 venum_post_pmresr(vval, fval); 1426 1427 /* Reset PMxEVNCTCR to sane default */ 1428 for (idx = ARMV7_IDX_CYCLE_COUNTER; idx < nb_cnt; ++idx) { 1429 armv7_pmnc_select_counter(idx); 1430 asm volatile("mcr p15, 0, %0, c9, c15, 0" : : "r" (0)); 1431 } 1432 1433 } 1434 1435 static int krait_event_to_bit(struct perf_event *event, unsigned int region, 1436 unsigned int group) 1437 { 1438 int bit; 1439 struct hw_perf_event *hwc = &event->hw; 1440 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); 1441 1442 if (hwc->config_base & VENUM_EVENT) 1443 bit = KRAIT_VPMRESR0_GROUP0; 1444 else 1445 bit = krait_get_pmresrn_event(region); 1446 bit -= krait_get_pmresrn_event(0); 1447 bit += group; 1448 /* 1449 * Lower bits are reserved for use by the counters (see 1450 * armv7pmu_get_event_idx() for more info) 1451 */ 1452 bit += ARMV7_IDX_COUNTER_LAST(cpu_pmu) + 1; 1453 1454 return bit; 1455 } 1456 1457 /* 1458 * We check for column exclusion constraints here. 1459 * Two events cant use the same group within a pmresr register. 1460 */ 1461 static int krait_pmu_get_event_idx(struct pmu_hw_events *cpuc, 1462 struct perf_event *event) 1463 { 1464 int idx; 1465 int bit = -1; 1466 struct hw_perf_event *hwc = &event->hw; 1467 unsigned int region = EVENT_REGION(hwc->config_base); 1468 unsigned int code = EVENT_CODE(hwc->config_base); 1469 unsigned int group = EVENT_GROUP(hwc->config_base); 1470 bool venum_event = EVENT_VENUM(hwc->config_base); 1471 bool krait_event = EVENT_CPU(hwc->config_base); 1472 1473 if (venum_event || krait_event) { 1474 /* Ignore invalid events */ 1475 if (group > 3 || region > 2) 1476 return -EINVAL; 1477 if (venum_event && (code & 0xe0)) 1478 return -EINVAL; 1479 1480 bit = krait_event_to_bit(event, region, group); 1481 if (test_and_set_bit(bit, cpuc->used_mask)) 1482 return -EAGAIN; 1483 } 1484 1485 idx = armv7pmu_get_event_idx(cpuc, event); 1486 if (idx < 0 && bit >= 0) 1487 clear_bit(bit, cpuc->used_mask); 1488 1489 return idx; 1490 } 1491 1492 static void krait_pmu_clear_event_idx(struct pmu_hw_events *cpuc, 1493 struct perf_event *event) 1494 { 1495 int bit; 1496 struct hw_perf_event *hwc = &event->hw; 1497 unsigned int region = EVENT_REGION(hwc->config_base); 1498 unsigned int group = EVENT_GROUP(hwc->config_base); 1499 bool venum_event = EVENT_VENUM(hwc->config_base); 1500 bool krait_event = EVENT_CPU(hwc->config_base); 1501 1502 if (venum_event || krait_event) { 1503 bit = krait_event_to_bit(event, region, group); 1504 clear_bit(bit, cpuc->used_mask); 1505 } 1506 } 1507 1508 static int krait_pmu_init(struct arm_pmu *cpu_pmu) 1509 { 1510 armv7pmu_init(cpu_pmu); 1511 cpu_pmu->name = "armv7_krait"; 1512 /* Some early versions of Krait don't support PC write events */ 1513 if (of_property_read_bool(cpu_pmu->plat_device->dev.of_node, 1514 "qcom,no-pc-write")) 1515 cpu_pmu->map_event = krait_map_event_no_branch; 1516 else 1517 cpu_pmu->map_event = krait_map_event; 1518 cpu_pmu->set_event_filter = armv7pmu_set_event_filter; 1519 cpu_pmu->reset = krait_pmu_reset; 1520 cpu_pmu->enable = krait_pmu_enable_event; 1521 cpu_pmu->disable = krait_pmu_disable_event; 1522 cpu_pmu->get_event_idx = krait_pmu_get_event_idx; 1523 cpu_pmu->clear_event_idx = krait_pmu_clear_event_idx; 1524 return armv7_probe_num_events(cpu_pmu); 1525 } 1526 1527 /* 1528 * Scorpion Local Performance Monitor Register (LPMn) 1529 * 1530 * 31 30 24 16 8 0 1531 * +--------------------------------+ 1532 * LPM0 | EN | CC | CC | CC | CC | N = 1, R = 0 1533 * +--------------------------------+ 1534 * LPM1 | EN | CC | CC | CC | CC | N = 1, R = 1 1535 * +--------------------------------+ 1536 * LPM2 | EN | CC | CC | CC | CC | N = 1, R = 2 1537 * +--------------------------------+ 1538 * L2LPM | EN | CC | CC | CC | CC | N = 1, R = 3 1539 * +--------------------------------+ 1540 * VLPM | EN | CC | CC | CC | CC | N = 2, R = ? 1541 * +--------------------------------+ 1542 * EN | G=3 | G=2 | G=1 | G=0 1543 * 1544 * 1545 * Event Encoding: 1546 * 1547 * hwc->config_base = 0xNRCCG 1548 * 1549 * N = prefix, 1 for Scorpion CPU (LPMn/L2LPM), 2 for Venum VFP (VLPM) 1550 * R = region register 1551 * CC = class of events the group G is choosing from 1552 * G = group or particular event 1553 * 1554 * Example: 0x12021 is a Scorpion CPU event in LPM2's group 1 with code 2 1555 * 1556 * A region (R) corresponds to a piece of the CPU (execution unit, instruction 1557 * unit, etc.) while the event code (CC) corresponds to a particular class of 1558 * events (interrupts for example). An event code is broken down into 1559 * groups (G) that can be mapped into the PMU (irq, fiqs, and irq+fiqs for 1560 * example). 1561 */ 1562 1563 static u32 scorpion_read_pmresrn(int n) 1564 { 1565 u32 val; 1566 1567 switch (n) { 1568 case 0: 1569 asm volatile("mrc p15, 0, %0, c15, c0, 0" : "=r" (val)); 1570 break; 1571 case 1: 1572 asm volatile("mrc p15, 1, %0, c15, c0, 0" : "=r" (val)); 1573 break; 1574 case 2: 1575 asm volatile("mrc p15, 2, %0, c15, c0, 0" : "=r" (val)); 1576 break; 1577 case 3: 1578 asm volatile("mrc p15, 3, %0, c15, c2, 0" : "=r" (val)); 1579 break; 1580 default: 1581 BUG(); /* Should be validated in scorpion_pmu_get_event_idx() */ 1582 } 1583 1584 return val; 1585 } 1586 1587 static void scorpion_write_pmresrn(int n, u32 val) 1588 { 1589 switch (n) { 1590 case 0: 1591 asm volatile("mcr p15, 0, %0, c15, c0, 0" : : "r" (val)); 1592 break; 1593 case 1: 1594 asm volatile("mcr p15, 1, %0, c15, c0, 0" : : "r" (val)); 1595 break; 1596 case 2: 1597 asm volatile("mcr p15, 2, %0, c15, c0, 0" : : "r" (val)); 1598 break; 1599 case 3: 1600 asm volatile("mcr p15, 3, %0, c15, c2, 0" : : "r" (val)); 1601 break; 1602 default: 1603 BUG(); /* Should be validated in scorpion_pmu_get_event_idx() */ 1604 } 1605 } 1606 1607 static u32 scorpion_get_pmresrn_event(unsigned int region) 1608 { 1609 static const u32 pmresrn_table[] = { SCORPION_LPM0_GROUP0, 1610 SCORPION_LPM1_GROUP0, 1611 SCORPION_LPM2_GROUP0, 1612 SCORPION_L2LPM_GROUP0 }; 1613 return pmresrn_table[region]; 1614 } 1615 1616 static void scorpion_evt_setup(int idx, u32 config_base) 1617 { 1618 u32 val; 1619 u32 mask; 1620 u32 vval, fval; 1621 unsigned int region = EVENT_REGION(config_base); 1622 unsigned int group = EVENT_GROUP(config_base); 1623 unsigned int code = EVENT_CODE(config_base); 1624 unsigned int group_shift; 1625 bool venum_event = EVENT_VENUM(config_base); 1626 1627 group_shift = group * 8; 1628 mask = 0xff << group_shift; 1629 1630 /* Configure evtsel for the region and group */ 1631 if (venum_event) 1632 val = SCORPION_VLPM_GROUP0; 1633 else 1634 val = scorpion_get_pmresrn_event(region); 1635 val += group; 1636 /* Mix in mode-exclusion bits */ 1637 val |= config_base & (ARMV7_EXCLUDE_USER | ARMV7_EXCLUDE_PL1); 1638 armv7_pmnc_write_evtsel(idx, val); 1639 1640 asm volatile("mcr p15, 0, %0, c9, c15, 0" : : "r" (0)); 1641 1642 if (venum_event) { 1643 venum_pre_pmresr(&vval, &fval); 1644 val = venum_read_pmresr(); 1645 val &= ~mask; 1646 val |= code << group_shift; 1647 val |= PMRESRn_EN; 1648 venum_write_pmresr(val); 1649 venum_post_pmresr(vval, fval); 1650 } else { 1651 val = scorpion_read_pmresrn(region); 1652 val &= ~mask; 1653 val |= code << group_shift; 1654 val |= PMRESRn_EN; 1655 scorpion_write_pmresrn(region, val); 1656 } 1657 } 1658 1659 static void scorpion_clearpmu(u32 config_base) 1660 { 1661 u32 val; 1662 u32 vval, fval; 1663 unsigned int region = EVENT_REGION(config_base); 1664 unsigned int group = EVENT_GROUP(config_base); 1665 bool venum_event = EVENT_VENUM(config_base); 1666 1667 if (venum_event) { 1668 venum_pre_pmresr(&vval, &fval); 1669 val = venum_read_pmresr(); 1670 val = clear_pmresrn_group(val, group); 1671 venum_write_pmresr(val); 1672 venum_post_pmresr(vval, fval); 1673 } else { 1674 val = scorpion_read_pmresrn(region); 1675 val = clear_pmresrn_group(val, group); 1676 scorpion_write_pmresrn(region, val); 1677 } 1678 } 1679 1680 static void scorpion_pmu_disable_event(struct perf_event *event) 1681 { 1682 unsigned long flags; 1683 struct hw_perf_event *hwc = &event->hw; 1684 int idx = hwc->idx; 1685 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); 1686 struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events); 1687 1688 /* Disable counter and interrupt */ 1689 raw_spin_lock_irqsave(&events->pmu_lock, flags); 1690 1691 /* Disable counter */ 1692 armv7_pmnc_disable_counter(idx); 1693 1694 /* 1695 * Clear pmresr code (if destined for PMNx counters) 1696 */ 1697 if (hwc->config_base & KRAIT_EVENT_MASK) 1698 scorpion_clearpmu(hwc->config_base); 1699 1700 /* Disable interrupt for this counter */ 1701 armv7_pmnc_disable_intens(idx); 1702 1703 raw_spin_unlock_irqrestore(&events->pmu_lock, flags); 1704 } 1705 1706 static void scorpion_pmu_enable_event(struct perf_event *event) 1707 { 1708 unsigned long flags; 1709 struct hw_perf_event *hwc = &event->hw; 1710 int idx = hwc->idx; 1711 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); 1712 struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events); 1713 1714 /* 1715 * Enable counter and interrupt, and set the counter to count 1716 * the event that we're interested in. 1717 */ 1718 raw_spin_lock_irqsave(&events->pmu_lock, flags); 1719 1720 /* Disable counter */ 1721 armv7_pmnc_disable_counter(idx); 1722 1723 /* 1724 * Set event (if destined for PMNx counters) 1725 * We don't set the event for the cycle counter because we 1726 * don't have the ability to perform event filtering. 1727 */ 1728 if (hwc->config_base & KRAIT_EVENT_MASK) 1729 scorpion_evt_setup(idx, hwc->config_base); 1730 else if (idx != ARMV7_IDX_CYCLE_COUNTER) 1731 armv7_pmnc_write_evtsel(idx, hwc->config_base); 1732 1733 /* Enable interrupt for this counter */ 1734 armv7_pmnc_enable_intens(idx); 1735 1736 /* Enable counter */ 1737 armv7_pmnc_enable_counter(idx); 1738 1739 raw_spin_unlock_irqrestore(&events->pmu_lock, flags); 1740 } 1741 1742 static void scorpion_pmu_reset(void *info) 1743 { 1744 u32 vval, fval; 1745 struct arm_pmu *cpu_pmu = info; 1746 u32 idx, nb_cnt = cpu_pmu->num_events; 1747 1748 armv7pmu_reset(info); 1749 1750 /* Clear all pmresrs */ 1751 scorpion_write_pmresrn(0, 0); 1752 scorpion_write_pmresrn(1, 0); 1753 scorpion_write_pmresrn(2, 0); 1754 scorpion_write_pmresrn(3, 0); 1755 1756 venum_pre_pmresr(&vval, &fval); 1757 venum_write_pmresr(0); 1758 venum_post_pmresr(vval, fval); 1759 1760 /* Reset PMxEVNCTCR to sane default */ 1761 for (idx = ARMV7_IDX_CYCLE_COUNTER; idx < nb_cnt; ++idx) { 1762 armv7_pmnc_select_counter(idx); 1763 asm volatile("mcr p15, 0, %0, c9, c15, 0" : : "r" (0)); 1764 } 1765 } 1766 1767 static int scorpion_event_to_bit(struct perf_event *event, unsigned int region, 1768 unsigned int group) 1769 { 1770 int bit; 1771 struct hw_perf_event *hwc = &event->hw; 1772 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); 1773 1774 if (hwc->config_base & VENUM_EVENT) 1775 bit = SCORPION_VLPM_GROUP0; 1776 else 1777 bit = scorpion_get_pmresrn_event(region); 1778 bit -= scorpion_get_pmresrn_event(0); 1779 bit += group; 1780 /* 1781 * Lower bits are reserved for use by the counters (see 1782 * armv7pmu_get_event_idx() for more info) 1783 */ 1784 bit += ARMV7_IDX_COUNTER_LAST(cpu_pmu) + 1; 1785 1786 return bit; 1787 } 1788 1789 /* 1790 * We check for column exclusion constraints here. 1791 * Two events cant use the same group within a pmresr register. 1792 */ 1793 static int scorpion_pmu_get_event_idx(struct pmu_hw_events *cpuc, 1794 struct perf_event *event) 1795 { 1796 int idx; 1797 int bit = -1; 1798 struct hw_perf_event *hwc = &event->hw; 1799 unsigned int region = EVENT_REGION(hwc->config_base); 1800 unsigned int group = EVENT_GROUP(hwc->config_base); 1801 bool venum_event = EVENT_VENUM(hwc->config_base); 1802 bool scorpion_event = EVENT_CPU(hwc->config_base); 1803 1804 if (venum_event || scorpion_event) { 1805 /* Ignore invalid events */ 1806 if (group > 3 || region > 3) 1807 return -EINVAL; 1808 1809 bit = scorpion_event_to_bit(event, region, group); 1810 if (test_and_set_bit(bit, cpuc->used_mask)) 1811 return -EAGAIN; 1812 } 1813 1814 idx = armv7pmu_get_event_idx(cpuc, event); 1815 if (idx < 0 && bit >= 0) 1816 clear_bit(bit, cpuc->used_mask); 1817 1818 return idx; 1819 } 1820 1821 static void scorpion_pmu_clear_event_idx(struct pmu_hw_events *cpuc, 1822 struct perf_event *event) 1823 { 1824 int bit; 1825 struct hw_perf_event *hwc = &event->hw; 1826 unsigned int region = EVENT_REGION(hwc->config_base); 1827 unsigned int group = EVENT_GROUP(hwc->config_base); 1828 bool venum_event = EVENT_VENUM(hwc->config_base); 1829 bool scorpion_event = EVENT_CPU(hwc->config_base); 1830 1831 if (venum_event || scorpion_event) { 1832 bit = scorpion_event_to_bit(event, region, group); 1833 clear_bit(bit, cpuc->used_mask); 1834 } 1835 } 1836 1837 static int scorpion_pmu_init(struct arm_pmu *cpu_pmu) 1838 { 1839 armv7pmu_init(cpu_pmu); 1840 cpu_pmu->name = "armv7_scorpion"; 1841 cpu_pmu->map_event = scorpion_map_event; 1842 cpu_pmu->reset = scorpion_pmu_reset; 1843 cpu_pmu->enable = scorpion_pmu_enable_event; 1844 cpu_pmu->disable = scorpion_pmu_disable_event; 1845 cpu_pmu->get_event_idx = scorpion_pmu_get_event_idx; 1846 cpu_pmu->clear_event_idx = scorpion_pmu_clear_event_idx; 1847 return armv7_probe_num_events(cpu_pmu); 1848 } 1849 1850 static int scorpion_mp_pmu_init(struct arm_pmu *cpu_pmu) 1851 { 1852 armv7pmu_init(cpu_pmu); 1853 cpu_pmu->name = "armv7_scorpion_mp"; 1854 cpu_pmu->map_event = scorpion_map_event; 1855 cpu_pmu->reset = scorpion_pmu_reset; 1856 cpu_pmu->enable = scorpion_pmu_enable_event; 1857 cpu_pmu->disable = scorpion_pmu_disable_event; 1858 cpu_pmu->get_event_idx = scorpion_pmu_get_event_idx; 1859 cpu_pmu->clear_event_idx = scorpion_pmu_clear_event_idx; 1860 return armv7_probe_num_events(cpu_pmu); 1861 } 1862 1863 static const struct of_device_id armv7_pmu_of_device_ids[] = { 1864 {.compatible = "arm,cortex-a17-pmu", .data = armv7_a17_pmu_init}, 1865 {.compatible = "arm,cortex-a15-pmu", .data = armv7_a15_pmu_init}, 1866 {.compatible = "arm,cortex-a12-pmu", .data = armv7_a12_pmu_init}, 1867 {.compatible = "arm,cortex-a9-pmu", .data = armv7_a9_pmu_init}, 1868 {.compatible = "arm,cortex-a8-pmu", .data = armv7_a8_pmu_init}, 1869 {.compatible = "arm,cortex-a7-pmu", .data = armv7_a7_pmu_init}, 1870 {.compatible = "arm,cortex-a5-pmu", .data = armv7_a5_pmu_init}, 1871 {.compatible = "qcom,krait-pmu", .data = krait_pmu_init}, 1872 {.compatible = "qcom,scorpion-pmu", .data = scorpion_pmu_init}, 1873 {.compatible = "qcom,scorpion-mp-pmu", .data = scorpion_mp_pmu_init}, 1874 {}, 1875 }; 1876 1877 static const struct pmu_probe_info armv7_pmu_probe_table[] = { 1878 ARM_PMU_PROBE(ARM_CPU_PART_CORTEX_A8, armv7_a8_pmu_init), 1879 ARM_PMU_PROBE(ARM_CPU_PART_CORTEX_A9, armv7_a9_pmu_init), 1880 { /* sentinel value */ } 1881 }; 1882 1883 1884 static int armv7_pmu_device_probe(struct platform_device *pdev) 1885 { 1886 return arm_pmu_device_probe(pdev, armv7_pmu_of_device_ids, 1887 armv7_pmu_probe_table); 1888 } 1889 1890 static struct platform_driver armv7_pmu_driver = { 1891 .driver = { 1892 .name = "armv7-pmu", 1893 .of_match_table = armv7_pmu_of_device_ids, 1894 }, 1895 .probe = armv7_pmu_device_probe, 1896 }; 1897 1898 static int __init register_armv7_pmu_driver(void) 1899 { 1900 return platform_driver_register(&armv7_pmu_driver); 1901 } 1902 device_initcall(register_armv7_pmu_driver); 1903 #endif /* CONFIG_CPU_V7 */ 1904