1 /* 2 * CCI cache coherent interconnect driver 3 * 4 * Copyright (C) 2013 ARM Ltd. 5 * Author: Lorenzo Pieralisi <lorenzo.pieralisi@arm.com> 6 * 7 * This program is free software; you can redistribute it and/or modify 8 * it under the terms of the GNU General Public License version 2 as 9 * published by the Free Software Foundation. 10 * 11 * This program is distributed "as is" WITHOUT ANY WARRANTY of any 12 * kind, whether express or implied; without even the implied warranty 13 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 * GNU General Public License for more details. 15 */ 16 17 #include <linux/arm-cci.h> 18 #include <linux/io.h> 19 #include <linux/interrupt.h> 20 #include <linux/module.h> 21 #include <linux/of_address.h> 22 #include <linux/of_irq.h> 23 #include <linux/of_platform.h> 24 #include <linux/perf_event.h> 25 #include <linux/platform_device.h> 26 #include <linux/slab.h> 27 #include <linux/spinlock.h> 28 29 #include <asm/cacheflush.h> 30 #include <asm/smp_plat.h> 31 32 static void __iomem *cci_ctrl_base; 33 static unsigned long cci_ctrl_phys; 34 35 #ifdef CONFIG_ARM_CCI400_PORT_CTRL 36 struct cci_nb_ports { 37 unsigned int nb_ace; 38 unsigned int nb_ace_lite; 39 }; 40 41 static const struct cci_nb_ports cci400_ports = { 42 .nb_ace = 2, 43 .nb_ace_lite = 3 44 }; 45 46 #define CCI400_PORTS_DATA (&cci400_ports) 47 #else 48 #define CCI400_PORTS_DATA (NULL) 49 #endif 50 51 static const struct of_device_id arm_cci_matches[] = { 52 #ifdef CONFIG_ARM_CCI400_COMMON 53 {.compatible = "arm,cci-400", .data = CCI400_PORTS_DATA }, 54 #endif 55 #ifdef CONFIG_ARM_CCI5xx_PMU 56 { .compatible = "arm,cci-500", }, 57 { .compatible = "arm,cci-550", }, 58 #endif 59 {}, 60 }; 61 62 #ifdef CONFIG_ARM_CCI_PMU 63 64 #define DRIVER_NAME "ARM-CCI" 65 #define DRIVER_NAME_PMU DRIVER_NAME " PMU" 66 67 #define CCI_PMCR 0x0100 68 #define CCI_PID2 0x0fe8 69 70 #define CCI_PMCR_CEN 0x00000001 71 #define CCI_PMCR_NCNT_MASK 0x0000f800 72 #define CCI_PMCR_NCNT_SHIFT 11 73 74 #define CCI_PID2_REV_MASK 0xf0 75 #define CCI_PID2_REV_SHIFT 4 76 77 #define CCI_PMU_EVT_SEL 0x000 78 #define CCI_PMU_CNTR 0x004 79 #define CCI_PMU_CNTR_CTRL 0x008 80 #define CCI_PMU_OVRFLW 0x00c 81 82 #define CCI_PMU_OVRFLW_FLAG 1 83 84 #define CCI_PMU_CNTR_SIZE(model) ((model)->cntr_size) 85 #define CCI_PMU_CNTR_BASE(model, idx) ((idx) * CCI_PMU_CNTR_SIZE(model)) 86 #define CCI_PMU_CNTR_MASK ((1ULL << 32) -1) 87 #define CCI_PMU_CNTR_LAST(cci_pmu) (cci_pmu->num_cntrs - 1) 88 89 #define CCI_PMU_MAX_HW_CNTRS(model) \ 90 ((model)->num_hw_cntrs + (model)->fixed_hw_cntrs) 91 92 /* Types of interfaces that can generate events */ 93 enum { 94 CCI_IF_SLAVE, 95 CCI_IF_MASTER, 96 #ifdef CONFIG_ARM_CCI5xx_PMU 97 CCI_IF_GLOBAL, 98 #endif 99 CCI_IF_MAX, 100 }; 101 102 struct event_range { 103 u32 min; 104 u32 max; 105 }; 106 107 struct cci_pmu_hw_events { 108 struct perf_event **events; 109 unsigned long *used_mask; 110 raw_spinlock_t pmu_lock; 111 }; 112 113 struct cci_pmu; 114 /* 115 * struct cci_pmu_model: 116 * @fixed_hw_cntrs - Number of fixed event counters 117 * @num_hw_cntrs - Maximum number of programmable event counters 118 * @cntr_size - Size of an event counter mapping 119 */ 120 struct cci_pmu_model { 121 char *name; 122 u32 fixed_hw_cntrs; 123 u32 num_hw_cntrs; 124 u32 cntr_size; 125 struct attribute **format_attrs; 126 struct attribute **event_attrs; 127 struct event_range event_ranges[CCI_IF_MAX]; 128 int (*validate_hw_event)(struct cci_pmu *, unsigned long); 129 int (*get_event_idx)(struct cci_pmu *, struct cci_pmu_hw_events *, unsigned long); 130 void (*write_counters)(struct cci_pmu *, unsigned long *); 131 }; 132 133 static struct cci_pmu_model cci_pmu_models[]; 134 135 struct cci_pmu { 136 void __iomem *base; 137 struct pmu pmu; 138 int nr_irqs; 139 int *irqs; 140 unsigned long active_irqs; 141 const struct cci_pmu_model *model; 142 struct cci_pmu_hw_events hw_events; 143 struct platform_device *plat_device; 144 int num_cntrs; 145 atomic_t active_events; 146 struct mutex reserve_mutex; 147 struct notifier_block cpu_nb; 148 cpumask_t cpus; 149 }; 150 151 #define to_cci_pmu(c) (container_of(c, struct cci_pmu, pmu)) 152 153 enum cci_models { 154 #ifdef CONFIG_ARM_CCI400_PMU 155 CCI400_R0, 156 CCI400_R1, 157 #endif 158 #ifdef CONFIG_ARM_CCI5xx_PMU 159 CCI500_R0, 160 CCI550_R0, 161 #endif 162 CCI_MODEL_MAX 163 }; 164 165 static void pmu_write_counters(struct cci_pmu *cci_pmu, 166 unsigned long *mask); 167 static ssize_t cci_pmu_format_show(struct device *dev, 168 struct device_attribute *attr, char *buf); 169 static ssize_t cci_pmu_event_show(struct device *dev, 170 struct device_attribute *attr, char *buf); 171 172 #define CCI_EXT_ATTR_ENTRY(_name, _func, _config) \ 173 &((struct dev_ext_attribute[]) { \ 174 { __ATTR(_name, S_IRUGO, _func, NULL), (void *)_config } \ 175 })[0].attr.attr 176 177 #define CCI_FORMAT_EXT_ATTR_ENTRY(_name, _config) \ 178 CCI_EXT_ATTR_ENTRY(_name, cci_pmu_format_show, (char *)_config) 179 #define CCI_EVENT_EXT_ATTR_ENTRY(_name, _config) \ 180 CCI_EXT_ATTR_ENTRY(_name, cci_pmu_event_show, (unsigned long)_config) 181 182 /* CCI400 PMU Specific definitions */ 183 184 #ifdef CONFIG_ARM_CCI400_PMU 185 186 /* Port ids */ 187 #define CCI400_PORT_S0 0 188 #define CCI400_PORT_S1 1 189 #define CCI400_PORT_S2 2 190 #define CCI400_PORT_S3 3 191 #define CCI400_PORT_S4 4 192 #define CCI400_PORT_M0 5 193 #define CCI400_PORT_M1 6 194 #define CCI400_PORT_M2 7 195 196 #define CCI400_R1_PX 5 197 198 /* 199 * Instead of an event id to monitor CCI cycles, a dedicated counter is 200 * provided. Use 0xff to represent CCI cycles and hope that no future revisions 201 * make use of this event in hardware. 202 */ 203 enum cci400_perf_events { 204 CCI400_PMU_CYCLES = 0xff 205 }; 206 207 #define CCI400_PMU_CYCLE_CNTR_IDX 0 208 #define CCI400_PMU_CNTR0_IDX 1 209 210 /* 211 * CCI PMU event id is an 8-bit value made of two parts - bits 7:5 for one of 8 212 * ports and bits 4:0 are event codes. There are different event codes 213 * associated with each port type. 214 * 215 * Additionally, the range of events associated with the port types changed 216 * between Rev0 and Rev1. 217 * 218 * The constants below define the range of valid codes for each port type for 219 * the different revisions and are used to validate the event to be monitored. 220 */ 221 222 #define CCI400_PMU_EVENT_MASK 0xffUL 223 #define CCI400_PMU_EVENT_SOURCE_SHIFT 5 224 #define CCI400_PMU_EVENT_SOURCE_MASK 0x7 225 #define CCI400_PMU_EVENT_CODE_SHIFT 0 226 #define CCI400_PMU_EVENT_CODE_MASK 0x1f 227 #define CCI400_PMU_EVENT_SOURCE(event) \ 228 ((event >> CCI400_PMU_EVENT_SOURCE_SHIFT) & \ 229 CCI400_PMU_EVENT_SOURCE_MASK) 230 #define CCI400_PMU_EVENT_CODE(event) \ 231 ((event >> CCI400_PMU_EVENT_CODE_SHIFT) & CCI400_PMU_EVENT_CODE_MASK) 232 233 #define CCI400_R0_SLAVE_PORT_MIN_EV 0x00 234 #define CCI400_R0_SLAVE_PORT_MAX_EV 0x13 235 #define CCI400_R0_MASTER_PORT_MIN_EV 0x14 236 #define CCI400_R0_MASTER_PORT_MAX_EV 0x1a 237 238 #define CCI400_R1_SLAVE_PORT_MIN_EV 0x00 239 #define CCI400_R1_SLAVE_PORT_MAX_EV 0x14 240 #define CCI400_R1_MASTER_PORT_MIN_EV 0x00 241 #define CCI400_R1_MASTER_PORT_MAX_EV 0x11 242 243 #define CCI400_CYCLE_EVENT_EXT_ATTR_ENTRY(_name, _config) \ 244 CCI_EXT_ATTR_ENTRY(_name, cci400_pmu_cycle_event_show, \ 245 (unsigned long)_config) 246 247 static ssize_t cci400_pmu_cycle_event_show(struct device *dev, 248 struct device_attribute *attr, char *buf); 249 250 static struct attribute *cci400_pmu_format_attrs[] = { 251 CCI_FORMAT_EXT_ATTR_ENTRY(event, "config:0-4"), 252 CCI_FORMAT_EXT_ATTR_ENTRY(source, "config:5-7"), 253 NULL 254 }; 255 256 static struct attribute *cci400_r0_pmu_event_attrs[] = { 257 /* Slave events */ 258 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_any, 0x0), 259 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_device, 0x01), 260 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_normal_or_nonshareable, 0x2), 261 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_inner_or_outershareable, 0x3), 262 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_cache_maintenance, 0x4), 263 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_mem_barrier, 0x5), 264 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_sync_barrier, 0x6), 265 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_dvm_msg, 0x7), 266 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_dvm_msg_sync, 0x8), 267 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_stall_tt_full, 0x9), 268 CCI_EVENT_EXT_ATTR_ENTRY(si_r_data_last_hs_snoop, 0xA), 269 CCI_EVENT_EXT_ATTR_ENTRY(si_r_data_stall_rvalids_h_rready_l, 0xB), 270 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_any, 0xC), 271 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_device, 0xD), 272 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_normal_or_nonshareable, 0xE), 273 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_inner_or_outershare_wback_wclean, 0xF), 274 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_write_unique, 0x10), 275 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_write_line_unique, 0x11), 276 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_evict, 0x12), 277 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_stall_tt_full, 0x13), 278 /* Master events */ 279 CCI_EVENT_EXT_ATTR_ENTRY(mi_retry_speculative_fetch, 0x14), 280 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_addr_hazard, 0x15), 281 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_id_hazard, 0x16), 282 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_tt_full, 0x17), 283 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_barrier_hazard, 0x18), 284 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_barrier_hazard, 0x19), 285 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_tt_full, 0x1A), 286 /* Special event for cycles counter */ 287 CCI400_CYCLE_EVENT_EXT_ATTR_ENTRY(cycles, 0xff), 288 NULL 289 }; 290 291 static struct attribute *cci400_r1_pmu_event_attrs[] = { 292 /* Slave events */ 293 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_any, 0x0), 294 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_device, 0x01), 295 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_normal_or_nonshareable, 0x2), 296 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_inner_or_outershareable, 0x3), 297 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_cache_maintenance, 0x4), 298 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_mem_barrier, 0x5), 299 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_sync_barrier, 0x6), 300 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_dvm_msg, 0x7), 301 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_dvm_msg_sync, 0x8), 302 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_stall_tt_full, 0x9), 303 CCI_EVENT_EXT_ATTR_ENTRY(si_r_data_last_hs_snoop, 0xA), 304 CCI_EVENT_EXT_ATTR_ENTRY(si_r_data_stall_rvalids_h_rready_l, 0xB), 305 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_any, 0xC), 306 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_device, 0xD), 307 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_normal_or_nonshareable, 0xE), 308 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_inner_or_outershare_wback_wclean, 0xF), 309 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_write_unique, 0x10), 310 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_write_line_unique, 0x11), 311 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_evict, 0x12), 312 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_stall_tt_full, 0x13), 313 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_stall_slave_id_hazard, 0x14), 314 /* Master events */ 315 CCI_EVENT_EXT_ATTR_ENTRY(mi_retry_speculative_fetch, 0x0), 316 CCI_EVENT_EXT_ATTR_ENTRY(mi_stall_cycle_addr_hazard, 0x1), 317 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_master_id_hazard, 0x2), 318 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_hi_prio_rtq_full, 0x3), 319 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_barrier_hazard, 0x4), 320 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_barrier_hazard, 0x5), 321 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_wtq_full, 0x6), 322 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_low_prio_rtq_full, 0x7), 323 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_mid_prio_rtq_full, 0x8), 324 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_qvn_vn0, 0x9), 325 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_qvn_vn1, 0xA), 326 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_qvn_vn2, 0xB), 327 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_qvn_vn3, 0xC), 328 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_qvn_vn0, 0xD), 329 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_qvn_vn1, 0xE), 330 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_qvn_vn2, 0xF), 331 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_qvn_vn3, 0x10), 332 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_unique_or_line_unique_addr_hazard, 0x11), 333 /* Special event for cycles counter */ 334 CCI400_CYCLE_EVENT_EXT_ATTR_ENTRY(cycles, 0xff), 335 NULL 336 }; 337 338 static ssize_t cci400_pmu_cycle_event_show(struct device *dev, 339 struct device_attribute *attr, char *buf) 340 { 341 struct dev_ext_attribute *eattr = container_of(attr, 342 struct dev_ext_attribute, attr); 343 return snprintf(buf, PAGE_SIZE, "config=0x%lx\n", (unsigned long)eattr->var); 344 } 345 346 static int cci400_get_event_idx(struct cci_pmu *cci_pmu, 347 struct cci_pmu_hw_events *hw, 348 unsigned long cci_event) 349 { 350 int idx; 351 352 /* cycles event idx is fixed */ 353 if (cci_event == CCI400_PMU_CYCLES) { 354 if (test_and_set_bit(CCI400_PMU_CYCLE_CNTR_IDX, hw->used_mask)) 355 return -EAGAIN; 356 357 return CCI400_PMU_CYCLE_CNTR_IDX; 358 } 359 360 for (idx = CCI400_PMU_CNTR0_IDX; idx <= CCI_PMU_CNTR_LAST(cci_pmu); ++idx) 361 if (!test_and_set_bit(idx, hw->used_mask)) 362 return idx; 363 364 /* No counters available */ 365 return -EAGAIN; 366 } 367 368 static int cci400_validate_hw_event(struct cci_pmu *cci_pmu, unsigned long hw_event) 369 { 370 u8 ev_source = CCI400_PMU_EVENT_SOURCE(hw_event); 371 u8 ev_code = CCI400_PMU_EVENT_CODE(hw_event); 372 int if_type; 373 374 if (hw_event & ~CCI400_PMU_EVENT_MASK) 375 return -ENOENT; 376 377 if (hw_event == CCI400_PMU_CYCLES) 378 return hw_event; 379 380 switch (ev_source) { 381 case CCI400_PORT_S0: 382 case CCI400_PORT_S1: 383 case CCI400_PORT_S2: 384 case CCI400_PORT_S3: 385 case CCI400_PORT_S4: 386 /* Slave Interface */ 387 if_type = CCI_IF_SLAVE; 388 break; 389 case CCI400_PORT_M0: 390 case CCI400_PORT_M1: 391 case CCI400_PORT_M2: 392 /* Master Interface */ 393 if_type = CCI_IF_MASTER; 394 break; 395 default: 396 return -ENOENT; 397 } 398 399 if (ev_code >= cci_pmu->model->event_ranges[if_type].min && 400 ev_code <= cci_pmu->model->event_ranges[if_type].max) 401 return hw_event; 402 403 return -ENOENT; 404 } 405 406 static int probe_cci400_revision(void) 407 { 408 int rev; 409 rev = readl_relaxed(cci_ctrl_base + CCI_PID2) & CCI_PID2_REV_MASK; 410 rev >>= CCI_PID2_REV_SHIFT; 411 412 if (rev < CCI400_R1_PX) 413 return CCI400_R0; 414 else 415 return CCI400_R1; 416 } 417 418 static const struct cci_pmu_model *probe_cci_model(struct platform_device *pdev) 419 { 420 if (platform_has_secure_cci_access()) 421 return &cci_pmu_models[probe_cci400_revision()]; 422 return NULL; 423 } 424 #else /* !CONFIG_ARM_CCI400_PMU */ 425 static inline struct cci_pmu_model *probe_cci_model(struct platform_device *pdev) 426 { 427 return NULL; 428 } 429 #endif /* CONFIG_ARM_CCI400_PMU */ 430 431 #ifdef CONFIG_ARM_CCI5xx_PMU 432 433 /* 434 * CCI5xx PMU event id is an 9-bit value made of two parts. 435 * bits [8:5] - Source for the event 436 * bits [4:0] - Event code (specific to type of interface) 437 * 438 * 439 */ 440 441 /* Port ids */ 442 #define CCI5xx_PORT_S0 0x0 443 #define CCI5xx_PORT_S1 0x1 444 #define CCI5xx_PORT_S2 0x2 445 #define CCI5xx_PORT_S3 0x3 446 #define CCI5xx_PORT_S4 0x4 447 #define CCI5xx_PORT_S5 0x5 448 #define CCI5xx_PORT_S6 0x6 449 450 #define CCI5xx_PORT_M0 0x8 451 #define CCI5xx_PORT_M1 0x9 452 #define CCI5xx_PORT_M2 0xa 453 #define CCI5xx_PORT_M3 0xb 454 #define CCI5xx_PORT_M4 0xc 455 #define CCI5xx_PORT_M5 0xd 456 #define CCI5xx_PORT_M6 0xe 457 458 #define CCI5xx_PORT_GLOBAL 0xf 459 460 #define CCI5xx_PMU_EVENT_MASK 0x1ffUL 461 #define CCI5xx_PMU_EVENT_SOURCE_SHIFT 0x5 462 #define CCI5xx_PMU_EVENT_SOURCE_MASK 0xf 463 #define CCI5xx_PMU_EVENT_CODE_SHIFT 0x0 464 #define CCI5xx_PMU_EVENT_CODE_MASK 0x1f 465 466 #define CCI5xx_PMU_EVENT_SOURCE(event) \ 467 ((event >> CCI5xx_PMU_EVENT_SOURCE_SHIFT) & CCI5xx_PMU_EVENT_SOURCE_MASK) 468 #define CCI5xx_PMU_EVENT_CODE(event) \ 469 ((event >> CCI5xx_PMU_EVENT_CODE_SHIFT) & CCI5xx_PMU_EVENT_CODE_MASK) 470 471 #define CCI5xx_SLAVE_PORT_MIN_EV 0x00 472 #define CCI5xx_SLAVE_PORT_MAX_EV 0x1f 473 #define CCI5xx_MASTER_PORT_MIN_EV 0x00 474 #define CCI5xx_MASTER_PORT_MAX_EV 0x06 475 #define CCI5xx_GLOBAL_PORT_MIN_EV 0x00 476 #define CCI5xx_GLOBAL_PORT_MAX_EV 0x0f 477 478 479 #define CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(_name, _config) \ 480 CCI_EXT_ATTR_ENTRY(_name, cci5xx_pmu_global_event_show, \ 481 (unsigned long) _config) 482 483 static ssize_t cci5xx_pmu_global_event_show(struct device *dev, 484 struct device_attribute *attr, char *buf); 485 486 static struct attribute *cci5xx_pmu_format_attrs[] = { 487 CCI_FORMAT_EXT_ATTR_ENTRY(event, "config:0-4"), 488 CCI_FORMAT_EXT_ATTR_ENTRY(source, "config:5-8"), 489 NULL, 490 }; 491 492 static struct attribute *cci5xx_pmu_event_attrs[] = { 493 /* Slave events */ 494 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_arvalid, 0x0), 495 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_dev, 0x1), 496 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_nonshareable, 0x2), 497 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_shareable_non_alloc, 0x3), 498 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_shareable_alloc, 0x4), 499 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_invalidate, 0x5), 500 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_cache_maint, 0x6), 501 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_dvm_msg, 0x7), 502 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_rval, 0x8), 503 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_rlast_snoop, 0x9), 504 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_awalid, 0xA), 505 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_dev, 0xB), 506 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_non_shareable, 0xC), 507 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_share_wb, 0xD), 508 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_share_wlu, 0xE), 509 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_share_wunique, 0xF), 510 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_evict, 0x10), 511 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_wrevict, 0x11), 512 CCI_EVENT_EXT_ATTR_ENTRY(si_w_data_beat, 0x12), 513 CCI_EVENT_EXT_ATTR_ENTRY(si_srq_acvalid, 0x13), 514 CCI_EVENT_EXT_ATTR_ENTRY(si_srq_read, 0x14), 515 CCI_EVENT_EXT_ATTR_ENTRY(si_srq_clean, 0x15), 516 CCI_EVENT_EXT_ATTR_ENTRY(si_srq_data_transfer_low, 0x16), 517 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_stall_arvalid, 0x17), 518 CCI_EVENT_EXT_ATTR_ENTRY(si_r_data_stall, 0x18), 519 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_stall, 0x19), 520 CCI_EVENT_EXT_ATTR_ENTRY(si_w_data_stall, 0x1A), 521 CCI_EVENT_EXT_ATTR_ENTRY(si_w_resp_stall, 0x1B), 522 CCI_EVENT_EXT_ATTR_ENTRY(si_srq_stall, 0x1C), 523 CCI_EVENT_EXT_ATTR_ENTRY(si_s_data_stall, 0x1D), 524 CCI_EVENT_EXT_ATTR_ENTRY(si_rq_stall_ot_limit, 0x1E), 525 CCI_EVENT_EXT_ATTR_ENTRY(si_r_stall_arbit, 0x1F), 526 527 /* Master events */ 528 CCI_EVENT_EXT_ATTR_ENTRY(mi_r_data_beat_any, 0x0), 529 CCI_EVENT_EXT_ATTR_ENTRY(mi_w_data_beat_any, 0x1), 530 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall, 0x2), 531 CCI_EVENT_EXT_ATTR_ENTRY(mi_r_data_stall, 0x3), 532 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall, 0x4), 533 CCI_EVENT_EXT_ATTR_ENTRY(mi_w_data_stall, 0x5), 534 CCI_EVENT_EXT_ATTR_ENTRY(mi_w_resp_stall, 0x6), 535 536 /* Global events */ 537 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_filter_bank_0_1, 0x0), 538 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_filter_bank_2_3, 0x1), 539 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_filter_bank_4_5, 0x2), 540 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_filter_bank_6_7, 0x3), 541 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_miss_filter_bank_0_1, 0x4), 542 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_miss_filter_bank_2_3, 0x5), 543 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_miss_filter_bank_4_5, 0x6), 544 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_miss_filter_bank_6_7, 0x7), 545 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_back_invalidation, 0x8), 546 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_stall_alloc_busy, 0x9), 547 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_stall_tt_full, 0xA), 548 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_wrq, 0xB), 549 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_cd_hs, 0xC), 550 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_rq_stall_addr_hazard, 0xD), 551 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snopp_rq_stall_tt_full, 0xE), 552 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_rq_tzmp1_prot, 0xF), 553 NULL 554 }; 555 556 static ssize_t cci5xx_pmu_global_event_show(struct device *dev, 557 struct device_attribute *attr, char *buf) 558 { 559 struct dev_ext_attribute *eattr = container_of(attr, 560 struct dev_ext_attribute, attr); 561 /* Global events have single fixed source code */ 562 return snprintf(buf, PAGE_SIZE, "event=0x%lx,source=0x%x\n", 563 (unsigned long)eattr->var, CCI5xx_PORT_GLOBAL); 564 } 565 566 /* 567 * CCI500 provides 8 independent event counters that can count 568 * any of the events available. 569 * CCI500 PMU event source ids 570 * 0x0-0x6 - Slave interfaces 571 * 0x8-0xD - Master interfaces 572 * 0xf - Global Events 573 * 0x7,0xe - Reserved 574 */ 575 static int cci500_validate_hw_event(struct cci_pmu *cci_pmu, 576 unsigned long hw_event) 577 { 578 u32 ev_source = CCI5xx_PMU_EVENT_SOURCE(hw_event); 579 u32 ev_code = CCI5xx_PMU_EVENT_CODE(hw_event); 580 int if_type; 581 582 if (hw_event & ~CCI5xx_PMU_EVENT_MASK) 583 return -ENOENT; 584 585 switch (ev_source) { 586 case CCI5xx_PORT_S0: 587 case CCI5xx_PORT_S1: 588 case CCI5xx_PORT_S2: 589 case CCI5xx_PORT_S3: 590 case CCI5xx_PORT_S4: 591 case CCI5xx_PORT_S5: 592 case CCI5xx_PORT_S6: 593 if_type = CCI_IF_SLAVE; 594 break; 595 case CCI5xx_PORT_M0: 596 case CCI5xx_PORT_M1: 597 case CCI5xx_PORT_M2: 598 case CCI5xx_PORT_M3: 599 case CCI5xx_PORT_M4: 600 case CCI5xx_PORT_M5: 601 if_type = CCI_IF_MASTER; 602 break; 603 case CCI5xx_PORT_GLOBAL: 604 if_type = CCI_IF_GLOBAL; 605 break; 606 default: 607 return -ENOENT; 608 } 609 610 if (ev_code >= cci_pmu->model->event_ranges[if_type].min && 611 ev_code <= cci_pmu->model->event_ranges[if_type].max) 612 return hw_event; 613 614 return -ENOENT; 615 } 616 617 /* 618 * CCI550 provides 8 independent event counters that can count 619 * any of the events available. 620 * CCI550 PMU event source ids 621 * 0x0-0x6 - Slave interfaces 622 * 0x8-0xe - Master interfaces 623 * 0xf - Global Events 624 * 0x7 - Reserved 625 */ 626 static int cci550_validate_hw_event(struct cci_pmu *cci_pmu, 627 unsigned long hw_event) 628 { 629 u32 ev_source = CCI5xx_PMU_EVENT_SOURCE(hw_event); 630 u32 ev_code = CCI5xx_PMU_EVENT_CODE(hw_event); 631 int if_type; 632 633 if (hw_event & ~CCI5xx_PMU_EVENT_MASK) 634 return -ENOENT; 635 636 switch (ev_source) { 637 case CCI5xx_PORT_S0: 638 case CCI5xx_PORT_S1: 639 case CCI5xx_PORT_S2: 640 case CCI5xx_PORT_S3: 641 case CCI5xx_PORT_S4: 642 case CCI5xx_PORT_S5: 643 case CCI5xx_PORT_S6: 644 if_type = CCI_IF_SLAVE; 645 break; 646 case CCI5xx_PORT_M0: 647 case CCI5xx_PORT_M1: 648 case CCI5xx_PORT_M2: 649 case CCI5xx_PORT_M3: 650 case CCI5xx_PORT_M4: 651 case CCI5xx_PORT_M5: 652 case CCI5xx_PORT_M6: 653 if_type = CCI_IF_MASTER; 654 break; 655 case CCI5xx_PORT_GLOBAL: 656 if_type = CCI_IF_GLOBAL; 657 break; 658 default: 659 return -ENOENT; 660 } 661 662 if (ev_code >= cci_pmu->model->event_ranges[if_type].min && 663 ev_code <= cci_pmu->model->event_ranges[if_type].max) 664 return hw_event; 665 666 return -ENOENT; 667 } 668 669 #endif /* CONFIG_ARM_CCI5xx_PMU */ 670 671 /* 672 * Program the CCI PMU counters which have PERF_HES_ARCH set 673 * with the event period and mark them ready before we enable 674 * PMU. 675 */ 676 static void cci_pmu_sync_counters(struct cci_pmu *cci_pmu) 677 { 678 int i; 679 struct cci_pmu_hw_events *cci_hw = &cci_pmu->hw_events; 680 681 DECLARE_BITMAP(mask, cci_pmu->num_cntrs); 682 683 bitmap_zero(mask, cci_pmu->num_cntrs); 684 for_each_set_bit(i, cci_pmu->hw_events.used_mask, cci_pmu->num_cntrs) { 685 struct perf_event *event = cci_hw->events[i]; 686 687 if (WARN_ON(!event)) 688 continue; 689 690 /* Leave the events which are not counting */ 691 if (event->hw.state & PERF_HES_STOPPED) 692 continue; 693 if (event->hw.state & PERF_HES_ARCH) { 694 set_bit(i, mask); 695 event->hw.state &= ~PERF_HES_ARCH; 696 } 697 } 698 699 pmu_write_counters(cci_pmu, mask); 700 } 701 702 /* Should be called with cci_pmu->hw_events->pmu_lock held */ 703 static void __cci_pmu_enable_nosync(struct cci_pmu *cci_pmu) 704 { 705 u32 val; 706 707 /* Enable all the PMU counters. */ 708 val = readl_relaxed(cci_ctrl_base + CCI_PMCR) | CCI_PMCR_CEN; 709 writel(val, cci_ctrl_base + CCI_PMCR); 710 } 711 712 /* Should be called with cci_pmu->hw_events->pmu_lock held */ 713 static void __cci_pmu_enable_sync(struct cci_pmu *cci_pmu) 714 { 715 cci_pmu_sync_counters(cci_pmu); 716 __cci_pmu_enable_nosync(cci_pmu); 717 } 718 719 /* Should be called with cci_pmu->hw_events->pmu_lock held */ 720 static void __cci_pmu_disable(void) 721 { 722 u32 val; 723 724 /* Disable all the PMU counters. */ 725 val = readl_relaxed(cci_ctrl_base + CCI_PMCR) & ~CCI_PMCR_CEN; 726 writel(val, cci_ctrl_base + CCI_PMCR); 727 } 728 729 static ssize_t cci_pmu_format_show(struct device *dev, 730 struct device_attribute *attr, char *buf) 731 { 732 struct dev_ext_attribute *eattr = container_of(attr, 733 struct dev_ext_attribute, attr); 734 return snprintf(buf, PAGE_SIZE, "%s\n", (char *)eattr->var); 735 } 736 737 static ssize_t cci_pmu_event_show(struct device *dev, 738 struct device_attribute *attr, char *buf) 739 { 740 struct dev_ext_attribute *eattr = container_of(attr, 741 struct dev_ext_attribute, attr); 742 /* source parameter is mandatory for normal PMU events */ 743 return snprintf(buf, PAGE_SIZE, "source=?,event=0x%lx\n", 744 (unsigned long)eattr->var); 745 } 746 747 static int pmu_is_valid_counter(struct cci_pmu *cci_pmu, int idx) 748 { 749 return 0 <= idx && idx <= CCI_PMU_CNTR_LAST(cci_pmu); 750 } 751 752 static u32 pmu_read_register(struct cci_pmu *cci_pmu, int idx, unsigned int offset) 753 { 754 return readl_relaxed(cci_pmu->base + 755 CCI_PMU_CNTR_BASE(cci_pmu->model, idx) + offset); 756 } 757 758 static void pmu_write_register(struct cci_pmu *cci_pmu, u32 value, 759 int idx, unsigned int offset) 760 { 761 writel_relaxed(value, cci_pmu->base + 762 CCI_PMU_CNTR_BASE(cci_pmu->model, idx) + offset); 763 } 764 765 static void pmu_disable_counter(struct cci_pmu *cci_pmu, int idx) 766 { 767 pmu_write_register(cci_pmu, 0, idx, CCI_PMU_CNTR_CTRL); 768 } 769 770 static void pmu_enable_counter(struct cci_pmu *cci_pmu, int idx) 771 { 772 pmu_write_register(cci_pmu, 1, idx, CCI_PMU_CNTR_CTRL); 773 } 774 775 static bool __maybe_unused 776 pmu_counter_is_enabled(struct cci_pmu *cci_pmu, int idx) 777 { 778 return (pmu_read_register(cci_pmu, idx, CCI_PMU_CNTR_CTRL) & 0x1) != 0; 779 } 780 781 static void pmu_set_event(struct cci_pmu *cci_pmu, int idx, unsigned long event) 782 { 783 pmu_write_register(cci_pmu, event, idx, CCI_PMU_EVT_SEL); 784 } 785 786 /* 787 * For all counters on the CCI-PMU, disable any 'enabled' counters, 788 * saving the changed counters in the mask, so that we can restore 789 * it later using pmu_restore_counters. The mask is private to the 790 * caller. We cannot rely on the used_mask maintained by the CCI_PMU 791 * as it only tells us if the counter is assigned to perf_event or not. 792 * The state of the perf_event cannot be locked by the PMU layer, hence 793 * we check the individual counter status (which can be locked by 794 * cci_pm->hw_events->pmu_lock). 795 * 796 * @mask should be initialised to empty by the caller. 797 */ 798 static void __maybe_unused 799 pmu_save_counters(struct cci_pmu *cci_pmu, unsigned long *mask) 800 { 801 int i; 802 803 for (i = 0; i < cci_pmu->num_cntrs; i++) { 804 if (pmu_counter_is_enabled(cci_pmu, i)) { 805 set_bit(i, mask); 806 pmu_disable_counter(cci_pmu, i); 807 } 808 } 809 } 810 811 /* 812 * Restore the status of the counters. Reversal of the pmu_save_counters(). 813 * For each counter set in the mask, enable the counter back. 814 */ 815 static void __maybe_unused 816 pmu_restore_counters(struct cci_pmu *cci_pmu, unsigned long *mask) 817 { 818 int i; 819 820 for_each_set_bit(i, mask, cci_pmu->num_cntrs) 821 pmu_enable_counter(cci_pmu, i); 822 } 823 824 /* 825 * Returns the number of programmable counters actually implemented 826 * by the cci 827 */ 828 static u32 pmu_get_max_counters(void) 829 { 830 return (readl_relaxed(cci_ctrl_base + CCI_PMCR) & 831 CCI_PMCR_NCNT_MASK) >> CCI_PMCR_NCNT_SHIFT; 832 } 833 834 static int pmu_get_event_idx(struct cci_pmu_hw_events *hw, struct perf_event *event) 835 { 836 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu); 837 unsigned long cci_event = event->hw.config_base; 838 int idx; 839 840 if (cci_pmu->model->get_event_idx) 841 return cci_pmu->model->get_event_idx(cci_pmu, hw, cci_event); 842 843 /* Generic code to find an unused idx from the mask */ 844 for(idx = 0; idx <= CCI_PMU_CNTR_LAST(cci_pmu); idx++) 845 if (!test_and_set_bit(idx, hw->used_mask)) 846 return idx; 847 848 /* No counters available */ 849 return -EAGAIN; 850 } 851 852 static int pmu_map_event(struct perf_event *event) 853 { 854 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu); 855 856 if (event->attr.type < PERF_TYPE_MAX || 857 !cci_pmu->model->validate_hw_event) 858 return -ENOENT; 859 860 return cci_pmu->model->validate_hw_event(cci_pmu, event->attr.config); 861 } 862 863 static int pmu_request_irq(struct cci_pmu *cci_pmu, irq_handler_t handler) 864 { 865 int i; 866 struct platform_device *pmu_device = cci_pmu->plat_device; 867 868 if (unlikely(!pmu_device)) 869 return -ENODEV; 870 871 if (cci_pmu->nr_irqs < 1) { 872 dev_err(&pmu_device->dev, "no irqs for CCI PMUs defined\n"); 873 return -ENODEV; 874 } 875 876 /* 877 * Register all available CCI PMU interrupts. In the interrupt handler 878 * we iterate over the counters checking for interrupt source (the 879 * overflowing counter) and clear it. 880 * 881 * This should allow handling of non-unique interrupt for the counters. 882 */ 883 for (i = 0; i < cci_pmu->nr_irqs; i++) { 884 int err = request_irq(cci_pmu->irqs[i], handler, IRQF_SHARED, 885 "arm-cci-pmu", cci_pmu); 886 if (err) { 887 dev_err(&pmu_device->dev, "unable to request IRQ%d for ARM CCI PMU counters\n", 888 cci_pmu->irqs[i]); 889 return err; 890 } 891 892 set_bit(i, &cci_pmu->active_irqs); 893 } 894 895 return 0; 896 } 897 898 static void pmu_free_irq(struct cci_pmu *cci_pmu) 899 { 900 int i; 901 902 for (i = 0; i < cci_pmu->nr_irqs; i++) { 903 if (!test_and_clear_bit(i, &cci_pmu->active_irqs)) 904 continue; 905 906 free_irq(cci_pmu->irqs[i], cci_pmu); 907 } 908 } 909 910 static u32 pmu_read_counter(struct perf_event *event) 911 { 912 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu); 913 struct hw_perf_event *hw_counter = &event->hw; 914 int idx = hw_counter->idx; 915 u32 value; 916 917 if (unlikely(!pmu_is_valid_counter(cci_pmu, idx))) { 918 dev_err(&cci_pmu->plat_device->dev, "Invalid CCI PMU counter %d\n", idx); 919 return 0; 920 } 921 value = pmu_read_register(cci_pmu, idx, CCI_PMU_CNTR); 922 923 return value; 924 } 925 926 static void pmu_write_counter(struct cci_pmu *cci_pmu, u32 value, int idx) 927 { 928 pmu_write_register(cci_pmu, value, idx, CCI_PMU_CNTR); 929 } 930 931 static void __pmu_write_counters(struct cci_pmu *cci_pmu, unsigned long *mask) 932 { 933 int i; 934 struct cci_pmu_hw_events *cci_hw = &cci_pmu->hw_events; 935 936 for_each_set_bit(i, mask, cci_pmu->num_cntrs) { 937 struct perf_event *event = cci_hw->events[i]; 938 939 if (WARN_ON(!event)) 940 continue; 941 pmu_write_counter(cci_pmu, local64_read(&event->hw.prev_count), i); 942 } 943 } 944 945 static void pmu_write_counters(struct cci_pmu *cci_pmu, unsigned long *mask) 946 { 947 if (cci_pmu->model->write_counters) 948 cci_pmu->model->write_counters(cci_pmu, mask); 949 else 950 __pmu_write_counters(cci_pmu, mask); 951 } 952 953 #ifdef CONFIG_ARM_CCI5xx_PMU 954 955 /* 956 * CCI-500/CCI-550 has advanced power saving policies, which could gate the 957 * clocks to the PMU counters, which makes the writes to them ineffective. 958 * The only way to write to those counters is when the global counters 959 * are enabled and the particular counter is enabled. 960 * 961 * So we do the following : 962 * 963 * 1) Disable all the PMU counters, saving their current state 964 * 2) Enable the global PMU profiling, now that all counters are 965 * disabled. 966 * 967 * For each counter to be programmed, repeat steps 3-7: 968 * 969 * 3) Write an invalid event code to the event control register for the 970 counter, so that the counters are not modified. 971 * 4) Enable the counter control for the counter. 972 * 5) Set the counter value 973 * 6) Disable the counter 974 * 7) Restore the event in the target counter 975 * 976 * 8) Disable the global PMU. 977 * 9) Restore the status of the rest of the counters. 978 * 979 * We choose an event which for CCI-5xx is guaranteed not to count. 980 * We use the highest possible event code (0x1f) for the master interface 0. 981 */ 982 #define CCI5xx_INVALID_EVENT ((CCI5xx_PORT_M0 << CCI5xx_PMU_EVENT_SOURCE_SHIFT) | \ 983 (CCI5xx_PMU_EVENT_CODE_MASK << CCI5xx_PMU_EVENT_CODE_SHIFT)) 984 static void cci5xx_pmu_write_counters(struct cci_pmu *cci_pmu, unsigned long *mask) 985 { 986 int i; 987 DECLARE_BITMAP(saved_mask, cci_pmu->num_cntrs); 988 989 bitmap_zero(saved_mask, cci_pmu->num_cntrs); 990 pmu_save_counters(cci_pmu, saved_mask); 991 992 /* 993 * Now that all the counters are disabled, we can safely turn the PMU on, 994 * without syncing the status of the counters 995 */ 996 __cci_pmu_enable_nosync(cci_pmu); 997 998 for_each_set_bit(i, mask, cci_pmu->num_cntrs) { 999 struct perf_event *event = cci_pmu->hw_events.events[i]; 1000 1001 if (WARN_ON(!event)) 1002 continue; 1003 1004 pmu_set_event(cci_pmu, i, CCI5xx_INVALID_EVENT); 1005 pmu_enable_counter(cci_pmu, i); 1006 pmu_write_counter(cci_pmu, local64_read(&event->hw.prev_count), i); 1007 pmu_disable_counter(cci_pmu, i); 1008 pmu_set_event(cci_pmu, i, event->hw.config_base); 1009 } 1010 1011 __cci_pmu_disable(); 1012 1013 pmu_restore_counters(cci_pmu, saved_mask); 1014 } 1015 1016 #endif /* CONFIG_ARM_CCI5xx_PMU */ 1017 1018 static u64 pmu_event_update(struct perf_event *event) 1019 { 1020 struct hw_perf_event *hwc = &event->hw; 1021 u64 delta, prev_raw_count, new_raw_count; 1022 1023 do { 1024 prev_raw_count = local64_read(&hwc->prev_count); 1025 new_raw_count = pmu_read_counter(event); 1026 } while (local64_cmpxchg(&hwc->prev_count, prev_raw_count, 1027 new_raw_count) != prev_raw_count); 1028 1029 delta = (new_raw_count - prev_raw_count) & CCI_PMU_CNTR_MASK; 1030 1031 local64_add(delta, &event->count); 1032 1033 return new_raw_count; 1034 } 1035 1036 static void pmu_read(struct perf_event *event) 1037 { 1038 pmu_event_update(event); 1039 } 1040 1041 static void pmu_event_set_period(struct perf_event *event) 1042 { 1043 struct hw_perf_event *hwc = &event->hw; 1044 /* 1045 * The CCI PMU counters have a period of 2^32. To account for the 1046 * possiblity of extreme interrupt latency we program for a period of 1047 * half that. Hopefully we can handle the interrupt before another 2^31 1048 * events occur and the counter overtakes its previous value. 1049 */ 1050 u64 val = 1ULL << 31; 1051 local64_set(&hwc->prev_count, val); 1052 1053 /* 1054 * CCI PMU uses PERF_HES_ARCH to keep track of the counters, whose 1055 * values needs to be sync-ed with the s/w state before the PMU is 1056 * enabled. 1057 * Mark this counter for sync. 1058 */ 1059 hwc->state |= PERF_HES_ARCH; 1060 } 1061 1062 static irqreturn_t pmu_handle_irq(int irq_num, void *dev) 1063 { 1064 unsigned long flags; 1065 struct cci_pmu *cci_pmu = dev; 1066 struct cci_pmu_hw_events *events = &cci_pmu->hw_events; 1067 int idx, handled = IRQ_NONE; 1068 1069 raw_spin_lock_irqsave(&events->pmu_lock, flags); 1070 1071 /* Disable the PMU while we walk through the counters */ 1072 __cci_pmu_disable(); 1073 /* 1074 * Iterate over counters and update the corresponding perf events. 1075 * This should work regardless of whether we have per-counter overflow 1076 * interrupt or a combined overflow interrupt. 1077 */ 1078 for (idx = 0; idx <= CCI_PMU_CNTR_LAST(cci_pmu); idx++) { 1079 struct perf_event *event = events->events[idx]; 1080 1081 if (!event) 1082 continue; 1083 1084 /* Did this counter overflow? */ 1085 if (!(pmu_read_register(cci_pmu, idx, CCI_PMU_OVRFLW) & 1086 CCI_PMU_OVRFLW_FLAG)) 1087 continue; 1088 1089 pmu_write_register(cci_pmu, CCI_PMU_OVRFLW_FLAG, idx, 1090 CCI_PMU_OVRFLW); 1091 1092 pmu_event_update(event); 1093 pmu_event_set_period(event); 1094 handled = IRQ_HANDLED; 1095 } 1096 1097 /* Enable the PMU and sync possibly overflowed counters */ 1098 __cci_pmu_enable_sync(cci_pmu); 1099 raw_spin_unlock_irqrestore(&events->pmu_lock, flags); 1100 1101 return IRQ_RETVAL(handled); 1102 } 1103 1104 static int cci_pmu_get_hw(struct cci_pmu *cci_pmu) 1105 { 1106 int ret = pmu_request_irq(cci_pmu, pmu_handle_irq); 1107 if (ret) { 1108 pmu_free_irq(cci_pmu); 1109 return ret; 1110 } 1111 return 0; 1112 } 1113 1114 static void cci_pmu_put_hw(struct cci_pmu *cci_pmu) 1115 { 1116 pmu_free_irq(cci_pmu); 1117 } 1118 1119 static void hw_perf_event_destroy(struct perf_event *event) 1120 { 1121 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu); 1122 atomic_t *active_events = &cci_pmu->active_events; 1123 struct mutex *reserve_mutex = &cci_pmu->reserve_mutex; 1124 1125 if (atomic_dec_and_mutex_lock(active_events, reserve_mutex)) { 1126 cci_pmu_put_hw(cci_pmu); 1127 mutex_unlock(reserve_mutex); 1128 } 1129 } 1130 1131 static void cci_pmu_enable(struct pmu *pmu) 1132 { 1133 struct cci_pmu *cci_pmu = to_cci_pmu(pmu); 1134 struct cci_pmu_hw_events *hw_events = &cci_pmu->hw_events; 1135 int enabled = bitmap_weight(hw_events->used_mask, cci_pmu->num_cntrs); 1136 unsigned long flags; 1137 1138 if (!enabled) 1139 return; 1140 1141 raw_spin_lock_irqsave(&hw_events->pmu_lock, flags); 1142 __cci_pmu_enable_sync(cci_pmu); 1143 raw_spin_unlock_irqrestore(&hw_events->pmu_lock, flags); 1144 1145 } 1146 1147 static void cci_pmu_disable(struct pmu *pmu) 1148 { 1149 struct cci_pmu *cci_pmu = to_cci_pmu(pmu); 1150 struct cci_pmu_hw_events *hw_events = &cci_pmu->hw_events; 1151 unsigned long flags; 1152 1153 raw_spin_lock_irqsave(&hw_events->pmu_lock, flags); 1154 __cci_pmu_disable(); 1155 raw_spin_unlock_irqrestore(&hw_events->pmu_lock, flags); 1156 } 1157 1158 /* 1159 * Check if the idx represents a non-programmable counter. 1160 * All the fixed event counters are mapped before the programmable 1161 * counters. 1162 */ 1163 static bool pmu_fixed_hw_idx(struct cci_pmu *cci_pmu, int idx) 1164 { 1165 return (idx >= 0) && (idx < cci_pmu->model->fixed_hw_cntrs); 1166 } 1167 1168 static void cci_pmu_start(struct perf_event *event, int pmu_flags) 1169 { 1170 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu); 1171 struct cci_pmu_hw_events *hw_events = &cci_pmu->hw_events; 1172 struct hw_perf_event *hwc = &event->hw; 1173 int idx = hwc->idx; 1174 unsigned long flags; 1175 1176 /* 1177 * To handle interrupt latency, we always reprogram the period 1178 * regardlesss of PERF_EF_RELOAD. 1179 */ 1180 if (pmu_flags & PERF_EF_RELOAD) 1181 WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE)); 1182 1183 hwc->state = 0; 1184 1185 if (unlikely(!pmu_is_valid_counter(cci_pmu, idx))) { 1186 dev_err(&cci_pmu->plat_device->dev, "Invalid CCI PMU counter %d\n", idx); 1187 return; 1188 } 1189 1190 raw_spin_lock_irqsave(&hw_events->pmu_lock, flags); 1191 1192 /* Configure the counter unless you are counting a fixed event */ 1193 if (!pmu_fixed_hw_idx(cci_pmu, idx)) 1194 pmu_set_event(cci_pmu, idx, hwc->config_base); 1195 1196 pmu_event_set_period(event); 1197 pmu_enable_counter(cci_pmu, idx); 1198 1199 raw_spin_unlock_irqrestore(&hw_events->pmu_lock, flags); 1200 } 1201 1202 static void cci_pmu_stop(struct perf_event *event, int pmu_flags) 1203 { 1204 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu); 1205 struct hw_perf_event *hwc = &event->hw; 1206 int idx = hwc->idx; 1207 1208 if (hwc->state & PERF_HES_STOPPED) 1209 return; 1210 1211 if (unlikely(!pmu_is_valid_counter(cci_pmu, idx))) { 1212 dev_err(&cci_pmu->plat_device->dev, "Invalid CCI PMU counter %d\n", idx); 1213 return; 1214 } 1215 1216 /* 1217 * We always reprogram the counter, so ignore PERF_EF_UPDATE. See 1218 * cci_pmu_start() 1219 */ 1220 pmu_disable_counter(cci_pmu, idx); 1221 pmu_event_update(event); 1222 hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE; 1223 } 1224 1225 static int cci_pmu_add(struct perf_event *event, int flags) 1226 { 1227 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu); 1228 struct cci_pmu_hw_events *hw_events = &cci_pmu->hw_events; 1229 struct hw_perf_event *hwc = &event->hw; 1230 int idx; 1231 int err = 0; 1232 1233 perf_pmu_disable(event->pmu); 1234 1235 /* If we don't have a space for the counter then finish early. */ 1236 idx = pmu_get_event_idx(hw_events, event); 1237 if (idx < 0) { 1238 err = idx; 1239 goto out; 1240 } 1241 1242 event->hw.idx = idx; 1243 hw_events->events[idx] = event; 1244 1245 hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE; 1246 if (flags & PERF_EF_START) 1247 cci_pmu_start(event, PERF_EF_RELOAD); 1248 1249 /* Propagate our changes to the userspace mapping. */ 1250 perf_event_update_userpage(event); 1251 1252 out: 1253 perf_pmu_enable(event->pmu); 1254 return err; 1255 } 1256 1257 static void cci_pmu_del(struct perf_event *event, int flags) 1258 { 1259 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu); 1260 struct cci_pmu_hw_events *hw_events = &cci_pmu->hw_events; 1261 struct hw_perf_event *hwc = &event->hw; 1262 int idx = hwc->idx; 1263 1264 cci_pmu_stop(event, PERF_EF_UPDATE); 1265 hw_events->events[idx] = NULL; 1266 clear_bit(idx, hw_events->used_mask); 1267 1268 perf_event_update_userpage(event); 1269 } 1270 1271 static int 1272 validate_event(struct pmu *cci_pmu, 1273 struct cci_pmu_hw_events *hw_events, 1274 struct perf_event *event) 1275 { 1276 if (is_software_event(event)) 1277 return 1; 1278 1279 /* 1280 * Reject groups spanning multiple HW PMUs (e.g. CPU + CCI). The 1281 * core perf code won't check that the pmu->ctx == leader->ctx 1282 * until after pmu->event_init(event). 1283 */ 1284 if (event->pmu != cci_pmu) 1285 return 0; 1286 1287 if (event->state < PERF_EVENT_STATE_OFF) 1288 return 1; 1289 1290 if (event->state == PERF_EVENT_STATE_OFF && !event->attr.enable_on_exec) 1291 return 1; 1292 1293 return pmu_get_event_idx(hw_events, event) >= 0; 1294 } 1295 1296 static int 1297 validate_group(struct perf_event *event) 1298 { 1299 struct perf_event *sibling, *leader = event->group_leader; 1300 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu); 1301 unsigned long mask[BITS_TO_LONGS(cci_pmu->num_cntrs)]; 1302 struct cci_pmu_hw_events fake_pmu = { 1303 /* 1304 * Initialise the fake PMU. We only need to populate the 1305 * used_mask for the purposes of validation. 1306 */ 1307 .used_mask = mask, 1308 }; 1309 memset(mask, 0, BITS_TO_LONGS(cci_pmu->num_cntrs) * sizeof(unsigned long)); 1310 1311 if (!validate_event(event->pmu, &fake_pmu, leader)) 1312 return -EINVAL; 1313 1314 list_for_each_entry(sibling, &leader->sibling_list, group_entry) { 1315 if (!validate_event(event->pmu, &fake_pmu, sibling)) 1316 return -EINVAL; 1317 } 1318 1319 if (!validate_event(event->pmu, &fake_pmu, event)) 1320 return -EINVAL; 1321 1322 return 0; 1323 } 1324 1325 static int 1326 __hw_perf_event_init(struct perf_event *event) 1327 { 1328 struct hw_perf_event *hwc = &event->hw; 1329 int mapping; 1330 1331 mapping = pmu_map_event(event); 1332 1333 if (mapping < 0) { 1334 pr_debug("event %x:%llx not supported\n", event->attr.type, 1335 event->attr.config); 1336 return mapping; 1337 } 1338 1339 /* 1340 * We don't assign an index until we actually place the event onto 1341 * hardware. Use -1 to signify that we haven't decided where to put it 1342 * yet. 1343 */ 1344 hwc->idx = -1; 1345 hwc->config_base = 0; 1346 hwc->config = 0; 1347 hwc->event_base = 0; 1348 1349 /* 1350 * Store the event encoding into the config_base field. 1351 */ 1352 hwc->config_base |= (unsigned long)mapping; 1353 1354 /* 1355 * Limit the sample_period to half of the counter width. That way, the 1356 * new counter value is far less likely to overtake the previous one 1357 * unless you have some serious IRQ latency issues. 1358 */ 1359 hwc->sample_period = CCI_PMU_CNTR_MASK >> 1; 1360 hwc->last_period = hwc->sample_period; 1361 local64_set(&hwc->period_left, hwc->sample_period); 1362 1363 if (event->group_leader != event) { 1364 if (validate_group(event) != 0) 1365 return -EINVAL; 1366 } 1367 1368 return 0; 1369 } 1370 1371 static int cci_pmu_event_init(struct perf_event *event) 1372 { 1373 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu); 1374 atomic_t *active_events = &cci_pmu->active_events; 1375 int err = 0; 1376 int cpu; 1377 1378 if (event->attr.type != event->pmu->type) 1379 return -ENOENT; 1380 1381 /* Shared by all CPUs, no meaningful state to sample */ 1382 if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK) 1383 return -EOPNOTSUPP; 1384 1385 /* We have no filtering of any kind */ 1386 if (event->attr.exclude_user || 1387 event->attr.exclude_kernel || 1388 event->attr.exclude_hv || 1389 event->attr.exclude_idle || 1390 event->attr.exclude_host || 1391 event->attr.exclude_guest) 1392 return -EINVAL; 1393 1394 /* 1395 * Following the example set by other "uncore" PMUs, we accept any CPU 1396 * and rewrite its affinity dynamically rather than having perf core 1397 * handle cpu == -1 and pid == -1 for this case. 1398 * 1399 * The perf core will pin online CPUs for the duration of this call and 1400 * the event being installed into its context, so the PMU's CPU can't 1401 * change under our feet. 1402 */ 1403 cpu = cpumask_first(&cci_pmu->cpus); 1404 if (event->cpu < 0 || cpu < 0) 1405 return -EINVAL; 1406 event->cpu = cpu; 1407 1408 event->destroy = hw_perf_event_destroy; 1409 if (!atomic_inc_not_zero(active_events)) { 1410 mutex_lock(&cci_pmu->reserve_mutex); 1411 if (atomic_read(active_events) == 0) 1412 err = cci_pmu_get_hw(cci_pmu); 1413 if (!err) 1414 atomic_inc(active_events); 1415 mutex_unlock(&cci_pmu->reserve_mutex); 1416 } 1417 if (err) 1418 return err; 1419 1420 err = __hw_perf_event_init(event); 1421 if (err) 1422 hw_perf_event_destroy(event); 1423 1424 return err; 1425 } 1426 1427 static ssize_t pmu_cpumask_attr_show(struct device *dev, 1428 struct device_attribute *attr, char *buf) 1429 { 1430 struct pmu *pmu = dev_get_drvdata(dev); 1431 struct cci_pmu *cci_pmu = to_cci_pmu(pmu); 1432 1433 int n = scnprintf(buf, PAGE_SIZE - 1, "%*pbl", 1434 cpumask_pr_args(&cci_pmu->cpus)); 1435 buf[n++] = '\n'; 1436 buf[n] = '\0'; 1437 return n; 1438 } 1439 1440 static struct device_attribute pmu_cpumask_attr = 1441 __ATTR(cpumask, S_IRUGO, pmu_cpumask_attr_show, NULL); 1442 1443 static struct attribute *pmu_attrs[] = { 1444 &pmu_cpumask_attr.attr, 1445 NULL, 1446 }; 1447 1448 static struct attribute_group pmu_attr_group = { 1449 .attrs = pmu_attrs, 1450 }; 1451 1452 static struct attribute_group pmu_format_attr_group = { 1453 .name = "format", 1454 .attrs = NULL, /* Filled in cci_pmu_init_attrs */ 1455 }; 1456 1457 static struct attribute_group pmu_event_attr_group = { 1458 .name = "events", 1459 .attrs = NULL, /* Filled in cci_pmu_init_attrs */ 1460 }; 1461 1462 static const struct attribute_group *pmu_attr_groups[] = { 1463 &pmu_attr_group, 1464 &pmu_format_attr_group, 1465 &pmu_event_attr_group, 1466 NULL 1467 }; 1468 1469 static int cci_pmu_init(struct cci_pmu *cci_pmu, struct platform_device *pdev) 1470 { 1471 const struct cci_pmu_model *model = cci_pmu->model; 1472 char *name = model->name; 1473 u32 num_cntrs; 1474 1475 pmu_event_attr_group.attrs = model->event_attrs; 1476 pmu_format_attr_group.attrs = model->format_attrs; 1477 1478 cci_pmu->pmu = (struct pmu) { 1479 .name = cci_pmu->model->name, 1480 .task_ctx_nr = perf_invalid_context, 1481 .pmu_enable = cci_pmu_enable, 1482 .pmu_disable = cci_pmu_disable, 1483 .event_init = cci_pmu_event_init, 1484 .add = cci_pmu_add, 1485 .del = cci_pmu_del, 1486 .start = cci_pmu_start, 1487 .stop = cci_pmu_stop, 1488 .read = pmu_read, 1489 .attr_groups = pmu_attr_groups, 1490 }; 1491 1492 cci_pmu->plat_device = pdev; 1493 num_cntrs = pmu_get_max_counters(); 1494 if (num_cntrs > cci_pmu->model->num_hw_cntrs) { 1495 dev_warn(&pdev->dev, 1496 "PMU implements more counters(%d) than supported by" 1497 " the model(%d), truncated.", 1498 num_cntrs, cci_pmu->model->num_hw_cntrs); 1499 num_cntrs = cci_pmu->model->num_hw_cntrs; 1500 } 1501 cci_pmu->num_cntrs = num_cntrs + cci_pmu->model->fixed_hw_cntrs; 1502 1503 return perf_pmu_register(&cci_pmu->pmu, name, -1); 1504 } 1505 1506 static int cci_pmu_cpu_notifier(struct notifier_block *self, 1507 unsigned long action, void *hcpu) 1508 { 1509 struct cci_pmu *cci_pmu = container_of(self, 1510 struct cci_pmu, cpu_nb); 1511 unsigned int cpu = (long)hcpu; 1512 unsigned int target; 1513 1514 switch (action & ~CPU_TASKS_FROZEN) { 1515 case CPU_DOWN_PREPARE: 1516 if (!cpumask_test_and_clear_cpu(cpu, &cci_pmu->cpus)) 1517 break; 1518 target = cpumask_any_but(cpu_online_mask, cpu); 1519 if (target >= nr_cpu_ids) // UP, last CPU 1520 break; 1521 /* 1522 * TODO: migrate context once core races on event->ctx have 1523 * been fixed. 1524 */ 1525 cpumask_set_cpu(target, &cci_pmu->cpus); 1526 default: 1527 break; 1528 } 1529 1530 return NOTIFY_OK; 1531 } 1532 1533 static struct cci_pmu_model cci_pmu_models[] = { 1534 #ifdef CONFIG_ARM_CCI400_PMU 1535 [CCI400_R0] = { 1536 .name = "CCI_400", 1537 .fixed_hw_cntrs = 1, /* Cycle counter */ 1538 .num_hw_cntrs = 4, 1539 .cntr_size = SZ_4K, 1540 .format_attrs = cci400_pmu_format_attrs, 1541 .event_attrs = cci400_r0_pmu_event_attrs, 1542 .event_ranges = { 1543 [CCI_IF_SLAVE] = { 1544 CCI400_R0_SLAVE_PORT_MIN_EV, 1545 CCI400_R0_SLAVE_PORT_MAX_EV, 1546 }, 1547 [CCI_IF_MASTER] = { 1548 CCI400_R0_MASTER_PORT_MIN_EV, 1549 CCI400_R0_MASTER_PORT_MAX_EV, 1550 }, 1551 }, 1552 .validate_hw_event = cci400_validate_hw_event, 1553 .get_event_idx = cci400_get_event_idx, 1554 }, 1555 [CCI400_R1] = { 1556 .name = "CCI_400_r1", 1557 .fixed_hw_cntrs = 1, /* Cycle counter */ 1558 .num_hw_cntrs = 4, 1559 .cntr_size = SZ_4K, 1560 .format_attrs = cci400_pmu_format_attrs, 1561 .event_attrs = cci400_r1_pmu_event_attrs, 1562 .event_ranges = { 1563 [CCI_IF_SLAVE] = { 1564 CCI400_R1_SLAVE_PORT_MIN_EV, 1565 CCI400_R1_SLAVE_PORT_MAX_EV, 1566 }, 1567 [CCI_IF_MASTER] = { 1568 CCI400_R1_MASTER_PORT_MIN_EV, 1569 CCI400_R1_MASTER_PORT_MAX_EV, 1570 }, 1571 }, 1572 .validate_hw_event = cci400_validate_hw_event, 1573 .get_event_idx = cci400_get_event_idx, 1574 }, 1575 #endif 1576 #ifdef CONFIG_ARM_CCI5xx_PMU 1577 [CCI500_R0] = { 1578 .name = "CCI_500", 1579 .fixed_hw_cntrs = 0, 1580 .num_hw_cntrs = 8, 1581 .cntr_size = SZ_64K, 1582 .format_attrs = cci5xx_pmu_format_attrs, 1583 .event_attrs = cci5xx_pmu_event_attrs, 1584 .event_ranges = { 1585 [CCI_IF_SLAVE] = { 1586 CCI5xx_SLAVE_PORT_MIN_EV, 1587 CCI5xx_SLAVE_PORT_MAX_EV, 1588 }, 1589 [CCI_IF_MASTER] = { 1590 CCI5xx_MASTER_PORT_MIN_EV, 1591 CCI5xx_MASTER_PORT_MAX_EV, 1592 }, 1593 [CCI_IF_GLOBAL] = { 1594 CCI5xx_GLOBAL_PORT_MIN_EV, 1595 CCI5xx_GLOBAL_PORT_MAX_EV, 1596 }, 1597 }, 1598 .validate_hw_event = cci500_validate_hw_event, 1599 .write_counters = cci5xx_pmu_write_counters, 1600 }, 1601 [CCI550_R0] = { 1602 .name = "CCI_550", 1603 .fixed_hw_cntrs = 0, 1604 .num_hw_cntrs = 8, 1605 .cntr_size = SZ_64K, 1606 .format_attrs = cci5xx_pmu_format_attrs, 1607 .event_attrs = cci5xx_pmu_event_attrs, 1608 .event_ranges = { 1609 [CCI_IF_SLAVE] = { 1610 CCI5xx_SLAVE_PORT_MIN_EV, 1611 CCI5xx_SLAVE_PORT_MAX_EV, 1612 }, 1613 [CCI_IF_MASTER] = { 1614 CCI5xx_MASTER_PORT_MIN_EV, 1615 CCI5xx_MASTER_PORT_MAX_EV, 1616 }, 1617 [CCI_IF_GLOBAL] = { 1618 CCI5xx_GLOBAL_PORT_MIN_EV, 1619 CCI5xx_GLOBAL_PORT_MAX_EV, 1620 }, 1621 }, 1622 .validate_hw_event = cci550_validate_hw_event, 1623 .write_counters = cci5xx_pmu_write_counters, 1624 }, 1625 #endif 1626 }; 1627 1628 static const struct of_device_id arm_cci_pmu_matches[] = { 1629 #ifdef CONFIG_ARM_CCI400_PMU 1630 { 1631 .compatible = "arm,cci-400-pmu", 1632 .data = NULL, 1633 }, 1634 { 1635 .compatible = "arm,cci-400-pmu,r0", 1636 .data = &cci_pmu_models[CCI400_R0], 1637 }, 1638 { 1639 .compatible = "arm,cci-400-pmu,r1", 1640 .data = &cci_pmu_models[CCI400_R1], 1641 }, 1642 #endif 1643 #ifdef CONFIG_ARM_CCI5xx_PMU 1644 { 1645 .compatible = "arm,cci-500-pmu,r0", 1646 .data = &cci_pmu_models[CCI500_R0], 1647 }, 1648 { 1649 .compatible = "arm,cci-550-pmu,r0", 1650 .data = &cci_pmu_models[CCI550_R0], 1651 }, 1652 #endif 1653 {}, 1654 }; 1655 1656 static inline const struct cci_pmu_model *get_cci_model(struct platform_device *pdev) 1657 { 1658 const struct of_device_id *match = of_match_node(arm_cci_pmu_matches, 1659 pdev->dev.of_node); 1660 if (!match) 1661 return NULL; 1662 if (match->data) 1663 return match->data; 1664 1665 dev_warn(&pdev->dev, "DEPRECATED compatible property," 1666 "requires secure access to CCI registers"); 1667 return probe_cci_model(pdev); 1668 } 1669 1670 static bool is_duplicate_irq(int irq, int *irqs, int nr_irqs) 1671 { 1672 int i; 1673 1674 for (i = 0; i < nr_irqs; i++) 1675 if (irq == irqs[i]) 1676 return true; 1677 1678 return false; 1679 } 1680 1681 static struct cci_pmu *cci_pmu_alloc(struct platform_device *pdev) 1682 { 1683 struct cci_pmu *cci_pmu; 1684 const struct cci_pmu_model *model; 1685 1686 /* 1687 * All allocations are devm_* hence we don't have to free 1688 * them explicitly on an error, as it would end up in driver 1689 * detach. 1690 */ 1691 model = get_cci_model(pdev); 1692 if (!model) { 1693 dev_warn(&pdev->dev, "CCI PMU version not supported\n"); 1694 return ERR_PTR(-ENODEV); 1695 } 1696 1697 cci_pmu = devm_kzalloc(&pdev->dev, sizeof(*cci_pmu), GFP_KERNEL); 1698 if (!cci_pmu) 1699 return ERR_PTR(-ENOMEM); 1700 1701 cci_pmu->model = model; 1702 cci_pmu->irqs = devm_kcalloc(&pdev->dev, CCI_PMU_MAX_HW_CNTRS(model), 1703 sizeof(*cci_pmu->irqs), GFP_KERNEL); 1704 if (!cci_pmu->irqs) 1705 return ERR_PTR(-ENOMEM); 1706 cci_pmu->hw_events.events = devm_kcalloc(&pdev->dev, 1707 CCI_PMU_MAX_HW_CNTRS(model), 1708 sizeof(*cci_pmu->hw_events.events), 1709 GFP_KERNEL); 1710 if (!cci_pmu->hw_events.events) 1711 return ERR_PTR(-ENOMEM); 1712 cci_pmu->hw_events.used_mask = devm_kcalloc(&pdev->dev, 1713 BITS_TO_LONGS(CCI_PMU_MAX_HW_CNTRS(model)), 1714 sizeof(*cci_pmu->hw_events.used_mask), 1715 GFP_KERNEL); 1716 if (!cci_pmu->hw_events.used_mask) 1717 return ERR_PTR(-ENOMEM); 1718 1719 return cci_pmu; 1720 } 1721 1722 1723 static int cci_pmu_probe(struct platform_device *pdev) 1724 { 1725 struct resource *res; 1726 struct cci_pmu *cci_pmu; 1727 int i, ret, irq; 1728 1729 cci_pmu = cci_pmu_alloc(pdev); 1730 if (IS_ERR(cci_pmu)) 1731 return PTR_ERR(cci_pmu); 1732 1733 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 1734 cci_pmu->base = devm_ioremap_resource(&pdev->dev, res); 1735 if (IS_ERR(cci_pmu->base)) 1736 return -ENOMEM; 1737 1738 /* 1739 * CCI PMU has one overflow interrupt per counter; but some may be tied 1740 * together to a common interrupt. 1741 */ 1742 cci_pmu->nr_irqs = 0; 1743 for (i = 0; i < CCI_PMU_MAX_HW_CNTRS(cci_pmu->model); i++) { 1744 irq = platform_get_irq(pdev, i); 1745 if (irq < 0) 1746 break; 1747 1748 if (is_duplicate_irq(irq, cci_pmu->irqs, cci_pmu->nr_irqs)) 1749 continue; 1750 1751 cci_pmu->irqs[cci_pmu->nr_irqs++] = irq; 1752 } 1753 1754 /* 1755 * Ensure that the device tree has as many interrupts as the number 1756 * of counters. 1757 */ 1758 if (i < CCI_PMU_MAX_HW_CNTRS(cci_pmu->model)) { 1759 dev_warn(&pdev->dev, "In-correct number of interrupts: %d, should be %d\n", 1760 i, CCI_PMU_MAX_HW_CNTRS(cci_pmu->model)); 1761 return -EINVAL; 1762 } 1763 1764 raw_spin_lock_init(&cci_pmu->hw_events.pmu_lock); 1765 mutex_init(&cci_pmu->reserve_mutex); 1766 atomic_set(&cci_pmu->active_events, 0); 1767 cpumask_set_cpu(smp_processor_id(), &cci_pmu->cpus); 1768 1769 cci_pmu->cpu_nb = (struct notifier_block) { 1770 .notifier_call = cci_pmu_cpu_notifier, 1771 /* 1772 * to migrate uncore events, our notifier should be executed 1773 * before perf core's notifier. 1774 */ 1775 .priority = CPU_PRI_PERF + 1, 1776 }; 1777 1778 ret = register_cpu_notifier(&cci_pmu->cpu_nb); 1779 if (ret) 1780 return ret; 1781 1782 ret = cci_pmu_init(cci_pmu, pdev); 1783 if (ret) { 1784 unregister_cpu_notifier(&cci_pmu->cpu_nb); 1785 return ret; 1786 } 1787 1788 pr_info("ARM %s PMU driver probed", cci_pmu->model->name); 1789 return 0; 1790 } 1791 1792 static int cci_platform_probe(struct platform_device *pdev) 1793 { 1794 if (!cci_probed()) 1795 return -ENODEV; 1796 1797 return of_platform_populate(pdev->dev.of_node, NULL, NULL, &pdev->dev); 1798 } 1799 1800 static struct platform_driver cci_pmu_driver = { 1801 .driver = { 1802 .name = DRIVER_NAME_PMU, 1803 .of_match_table = arm_cci_pmu_matches, 1804 }, 1805 .probe = cci_pmu_probe, 1806 }; 1807 1808 static struct platform_driver cci_platform_driver = { 1809 .driver = { 1810 .name = DRIVER_NAME, 1811 .of_match_table = arm_cci_matches, 1812 }, 1813 .probe = cci_platform_probe, 1814 }; 1815 1816 static int __init cci_platform_init(void) 1817 { 1818 int ret; 1819 1820 ret = platform_driver_register(&cci_pmu_driver); 1821 if (ret) 1822 return ret; 1823 1824 return platform_driver_register(&cci_platform_driver); 1825 } 1826 1827 #else /* !CONFIG_ARM_CCI_PMU */ 1828 1829 static int __init cci_platform_init(void) 1830 { 1831 return 0; 1832 } 1833 1834 #endif /* CONFIG_ARM_CCI_PMU */ 1835 1836 #ifdef CONFIG_ARM_CCI400_PORT_CTRL 1837 1838 #define CCI_PORT_CTRL 0x0 1839 #define CCI_CTRL_STATUS 0xc 1840 1841 #define CCI_ENABLE_SNOOP_REQ 0x1 1842 #define CCI_ENABLE_DVM_REQ 0x2 1843 #define CCI_ENABLE_REQ (CCI_ENABLE_SNOOP_REQ | CCI_ENABLE_DVM_REQ) 1844 1845 enum cci_ace_port_type { 1846 ACE_INVALID_PORT = 0x0, 1847 ACE_PORT, 1848 ACE_LITE_PORT, 1849 }; 1850 1851 struct cci_ace_port { 1852 void __iomem *base; 1853 unsigned long phys; 1854 enum cci_ace_port_type type; 1855 struct device_node *dn; 1856 }; 1857 1858 static struct cci_ace_port *ports; 1859 static unsigned int nb_cci_ports; 1860 1861 struct cpu_port { 1862 u64 mpidr; 1863 u32 port; 1864 }; 1865 1866 /* 1867 * Use the port MSB as valid flag, shift can be made dynamic 1868 * by computing number of bits required for port indexes. 1869 * Code disabling CCI cpu ports runs with D-cache invalidated 1870 * and SCTLR bit clear so data accesses must be kept to a minimum 1871 * to improve performance; for now shift is left static to 1872 * avoid one more data access while disabling the CCI port. 1873 */ 1874 #define PORT_VALID_SHIFT 31 1875 #define PORT_VALID (0x1 << PORT_VALID_SHIFT) 1876 1877 static inline void init_cpu_port(struct cpu_port *port, u32 index, u64 mpidr) 1878 { 1879 port->port = PORT_VALID | index; 1880 port->mpidr = mpidr; 1881 } 1882 1883 static inline bool cpu_port_is_valid(struct cpu_port *port) 1884 { 1885 return !!(port->port & PORT_VALID); 1886 } 1887 1888 static inline bool cpu_port_match(struct cpu_port *port, u64 mpidr) 1889 { 1890 return port->mpidr == (mpidr & MPIDR_HWID_BITMASK); 1891 } 1892 1893 static struct cpu_port cpu_port[NR_CPUS]; 1894 1895 /** 1896 * __cci_ace_get_port - Function to retrieve the port index connected to 1897 * a cpu or device. 1898 * 1899 * @dn: device node of the device to look-up 1900 * @type: port type 1901 * 1902 * Return value: 1903 * - CCI port index if success 1904 * - -ENODEV if failure 1905 */ 1906 static int __cci_ace_get_port(struct device_node *dn, int type) 1907 { 1908 int i; 1909 bool ace_match; 1910 struct device_node *cci_portn; 1911 1912 cci_portn = of_parse_phandle(dn, "cci-control-port", 0); 1913 for (i = 0; i < nb_cci_ports; i++) { 1914 ace_match = ports[i].type == type; 1915 if (ace_match && cci_portn == ports[i].dn) 1916 return i; 1917 } 1918 return -ENODEV; 1919 } 1920 1921 int cci_ace_get_port(struct device_node *dn) 1922 { 1923 return __cci_ace_get_port(dn, ACE_LITE_PORT); 1924 } 1925 EXPORT_SYMBOL_GPL(cci_ace_get_port); 1926 1927 static void cci_ace_init_ports(void) 1928 { 1929 int port, cpu; 1930 struct device_node *cpun; 1931 1932 /* 1933 * Port index look-up speeds up the function disabling ports by CPU, 1934 * since the logical to port index mapping is done once and does 1935 * not change after system boot. 1936 * The stashed index array is initialized for all possible CPUs 1937 * at probe time. 1938 */ 1939 for_each_possible_cpu(cpu) { 1940 /* too early to use cpu->of_node */ 1941 cpun = of_get_cpu_node(cpu, NULL); 1942 1943 if (WARN(!cpun, "Missing cpu device node\n")) 1944 continue; 1945 1946 port = __cci_ace_get_port(cpun, ACE_PORT); 1947 if (port < 0) 1948 continue; 1949 1950 init_cpu_port(&cpu_port[cpu], port, cpu_logical_map(cpu)); 1951 } 1952 1953 for_each_possible_cpu(cpu) { 1954 WARN(!cpu_port_is_valid(&cpu_port[cpu]), 1955 "CPU %u does not have an associated CCI port\n", 1956 cpu); 1957 } 1958 } 1959 /* 1960 * Functions to enable/disable a CCI interconnect slave port 1961 * 1962 * They are called by low-level power management code to disable slave 1963 * interfaces snoops and DVM broadcast. 1964 * Since they may execute with cache data allocation disabled and 1965 * after the caches have been cleaned and invalidated the functions provide 1966 * no explicit locking since they may run with D-cache disabled, so normal 1967 * cacheable kernel locks based on ldrex/strex may not work. 1968 * Locking has to be provided by BSP implementations to ensure proper 1969 * operations. 1970 */ 1971 1972 /** 1973 * cci_port_control() - function to control a CCI port 1974 * 1975 * @port: index of the port to setup 1976 * @enable: if true enables the port, if false disables it 1977 */ 1978 static void notrace cci_port_control(unsigned int port, bool enable) 1979 { 1980 void __iomem *base = ports[port].base; 1981 1982 writel_relaxed(enable ? CCI_ENABLE_REQ : 0, base + CCI_PORT_CTRL); 1983 /* 1984 * This function is called from power down procedures 1985 * and must not execute any instruction that might 1986 * cause the processor to be put in a quiescent state 1987 * (eg wfi). Hence, cpu_relax() can not be added to this 1988 * read loop to optimize power, since it might hide possibly 1989 * disruptive operations. 1990 */ 1991 while (readl_relaxed(cci_ctrl_base + CCI_CTRL_STATUS) & 0x1) 1992 ; 1993 } 1994 1995 /** 1996 * cci_disable_port_by_cpu() - function to disable a CCI port by CPU 1997 * reference 1998 * 1999 * @mpidr: mpidr of the CPU whose CCI port should be disabled 2000 * 2001 * Disabling a CCI port for a CPU implies disabling the CCI port 2002 * controlling that CPU cluster. Code disabling CPU CCI ports 2003 * must make sure that the CPU running the code is the last active CPU 2004 * in the cluster ie all other CPUs are quiescent in a low power state. 2005 * 2006 * Return: 2007 * 0 on success 2008 * -ENODEV on port look-up failure 2009 */ 2010 int notrace cci_disable_port_by_cpu(u64 mpidr) 2011 { 2012 int cpu; 2013 bool is_valid; 2014 for (cpu = 0; cpu < nr_cpu_ids; cpu++) { 2015 is_valid = cpu_port_is_valid(&cpu_port[cpu]); 2016 if (is_valid && cpu_port_match(&cpu_port[cpu], mpidr)) { 2017 cci_port_control(cpu_port[cpu].port, false); 2018 return 0; 2019 } 2020 } 2021 return -ENODEV; 2022 } 2023 EXPORT_SYMBOL_GPL(cci_disable_port_by_cpu); 2024 2025 /** 2026 * cci_enable_port_for_self() - enable a CCI port for calling CPU 2027 * 2028 * Enabling a CCI port for the calling CPU implies enabling the CCI 2029 * port controlling that CPU's cluster. Caller must make sure that the 2030 * CPU running the code is the first active CPU in the cluster and all 2031 * other CPUs are quiescent in a low power state or waiting for this CPU 2032 * to complete the CCI initialization. 2033 * 2034 * Because this is called when the MMU is still off and with no stack, 2035 * the code must be position independent and ideally rely on callee 2036 * clobbered registers only. To achieve this we must code this function 2037 * entirely in assembler. 2038 * 2039 * On success this returns with the proper CCI port enabled. In case of 2040 * any failure this never returns as the inability to enable the CCI is 2041 * fatal and there is no possible recovery at this stage. 2042 */ 2043 asmlinkage void __naked cci_enable_port_for_self(void) 2044 { 2045 asm volatile ("\n" 2046 " .arch armv7-a\n" 2047 " mrc p15, 0, r0, c0, c0, 5 @ get MPIDR value \n" 2048 " and r0, r0, #"__stringify(MPIDR_HWID_BITMASK)" \n" 2049 " adr r1, 5f \n" 2050 " ldr r2, [r1] \n" 2051 " add r1, r1, r2 @ &cpu_port \n" 2052 " add ip, r1, %[sizeof_cpu_port] \n" 2053 2054 /* Loop over the cpu_port array looking for a matching MPIDR */ 2055 "1: ldr r2, [r1, %[offsetof_cpu_port_mpidr_lsb]] \n" 2056 " cmp r2, r0 @ compare MPIDR \n" 2057 " bne 2f \n" 2058 2059 /* Found a match, now test port validity */ 2060 " ldr r3, [r1, %[offsetof_cpu_port_port]] \n" 2061 " tst r3, #"__stringify(PORT_VALID)" \n" 2062 " bne 3f \n" 2063 2064 /* no match, loop with the next cpu_port entry */ 2065 "2: add r1, r1, %[sizeof_struct_cpu_port] \n" 2066 " cmp r1, ip @ done? \n" 2067 " blo 1b \n" 2068 2069 /* CCI port not found -- cheaply try to stall this CPU */ 2070 "cci_port_not_found: \n" 2071 " wfi \n" 2072 " wfe \n" 2073 " b cci_port_not_found \n" 2074 2075 /* Use matched port index to look up the corresponding ports entry */ 2076 "3: bic r3, r3, #"__stringify(PORT_VALID)" \n" 2077 " adr r0, 6f \n" 2078 " ldmia r0, {r1, r2} \n" 2079 " sub r1, r1, r0 @ virt - phys \n" 2080 " ldr r0, [r0, r2] @ *(&ports) \n" 2081 " mov r2, %[sizeof_struct_ace_port] \n" 2082 " mla r0, r2, r3, r0 @ &ports[index] \n" 2083 " sub r0, r0, r1 @ virt_to_phys() \n" 2084 2085 /* Enable the CCI port */ 2086 " ldr r0, [r0, %[offsetof_port_phys]] \n" 2087 " mov r3, %[cci_enable_req]\n" 2088 " str r3, [r0, #"__stringify(CCI_PORT_CTRL)"] \n" 2089 2090 /* poll the status reg for completion */ 2091 " adr r1, 7f \n" 2092 " ldr r0, [r1] \n" 2093 " ldr r0, [r0, r1] @ cci_ctrl_base \n" 2094 "4: ldr r1, [r0, #"__stringify(CCI_CTRL_STATUS)"] \n" 2095 " tst r1, %[cci_control_status_bits] \n" 2096 " bne 4b \n" 2097 2098 " mov r0, #0 \n" 2099 " bx lr \n" 2100 2101 " .align 2 \n" 2102 "5: .word cpu_port - . \n" 2103 "6: .word . \n" 2104 " .word ports - 6b \n" 2105 "7: .word cci_ctrl_phys - . \n" 2106 : : 2107 [sizeof_cpu_port] "i" (sizeof(cpu_port)), 2108 [cci_enable_req] "i" cpu_to_le32(CCI_ENABLE_REQ), 2109 [cci_control_status_bits] "i" cpu_to_le32(1), 2110 #ifndef __ARMEB__ 2111 [offsetof_cpu_port_mpidr_lsb] "i" (offsetof(struct cpu_port, mpidr)), 2112 #else 2113 [offsetof_cpu_port_mpidr_lsb] "i" (offsetof(struct cpu_port, mpidr)+4), 2114 #endif 2115 [offsetof_cpu_port_port] "i" (offsetof(struct cpu_port, port)), 2116 [sizeof_struct_cpu_port] "i" (sizeof(struct cpu_port)), 2117 [sizeof_struct_ace_port] "i" (sizeof(struct cci_ace_port)), 2118 [offsetof_port_phys] "i" (offsetof(struct cci_ace_port, phys)) ); 2119 2120 unreachable(); 2121 } 2122 2123 /** 2124 * __cci_control_port_by_device() - function to control a CCI port by device 2125 * reference 2126 * 2127 * @dn: device node pointer of the device whose CCI port should be 2128 * controlled 2129 * @enable: if true enables the port, if false disables it 2130 * 2131 * Return: 2132 * 0 on success 2133 * -ENODEV on port look-up failure 2134 */ 2135 int notrace __cci_control_port_by_device(struct device_node *dn, bool enable) 2136 { 2137 int port; 2138 2139 if (!dn) 2140 return -ENODEV; 2141 2142 port = __cci_ace_get_port(dn, ACE_LITE_PORT); 2143 if (WARN_ONCE(port < 0, "node %s ACE lite port look-up failure\n", 2144 dn->full_name)) 2145 return -ENODEV; 2146 cci_port_control(port, enable); 2147 return 0; 2148 } 2149 EXPORT_SYMBOL_GPL(__cci_control_port_by_device); 2150 2151 /** 2152 * __cci_control_port_by_index() - function to control a CCI port by port index 2153 * 2154 * @port: port index previously retrieved with cci_ace_get_port() 2155 * @enable: if true enables the port, if false disables it 2156 * 2157 * Return: 2158 * 0 on success 2159 * -ENODEV on port index out of range 2160 * -EPERM if operation carried out on an ACE PORT 2161 */ 2162 int notrace __cci_control_port_by_index(u32 port, bool enable) 2163 { 2164 if (port >= nb_cci_ports || ports[port].type == ACE_INVALID_PORT) 2165 return -ENODEV; 2166 /* 2167 * CCI control for ports connected to CPUS is extremely fragile 2168 * and must be made to go through a specific and controlled 2169 * interface (ie cci_disable_port_by_cpu(); control by general purpose 2170 * indexing is therefore disabled for ACE ports. 2171 */ 2172 if (ports[port].type == ACE_PORT) 2173 return -EPERM; 2174 2175 cci_port_control(port, enable); 2176 return 0; 2177 } 2178 EXPORT_SYMBOL_GPL(__cci_control_port_by_index); 2179 2180 static const struct of_device_id arm_cci_ctrl_if_matches[] = { 2181 {.compatible = "arm,cci-400-ctrl-if", }, 2182 {}, 2183 }; 2184 2185 static int cci_probe_ports(struct device_node *np) 2186 { 2187 struct cci_nb_ports const *cci_config; 2188 int ret, i, nb_ace = 0, nb_ace_lite = 0; 2189 struct device_node *cp; 2190 struct resource res; 2191 const char *match_str; 2192 bool is_ace; 2193 2194 2195 cci_config = of_match_node(arm_cci_matches, np)->data; 2196 if (!cci_config) 2197 return -ENODEV; 2198 2199 nb_cci_ports = cci_config->nb_ace + cci_config->nb_ace_lite; 2200 2201 ports = kcalloc(nb_cci_ports, sizeof(*ports), GFP_KERNEL); 2202 if (!ports) 2203 return -ENOMEM; 2204 2205 for_each_child_of_node(np, cp) { 2206 if (!of_match_node(arm_cci_ctrl_if_matches, cp)) 2207 continue; 2208 2209 i = nb_ace + nb_ace_lite; 2210 2211 if (i >= nb_cci_ports) 2212 break; 2213 2214 if (of_property_read_string(cp, "interface-type", 2215 &match_str)) { 2216 WARN(1, "node %s missing interface-type property\n", 2217 cp->full_name); 2218 continue; 2219 } 2220 is_ace = strcmp(match_str, "ace") == 0; 2221 if (!is_ace && strcmp(match_str, "ace-lite")) { 2222 WARN(1, "node %s containing invalid interface-type property, skipping it\n", 2223 cp->full_name); 2224 continue; 2225 } 2226 2227 ret = of_address_to_resource(cp, 0, &res); 2228 if (!ret) { 2229 ports[i].base = ioremap(res.start, resource_size(&res)); 2230 ports[i].phys = res.start; 2231 } 2232 if (ret || !ports[i].base) { 2233 WARN(1, "unable to ioremap CCI port %d\n", i); 2234 continue; 2235 } 2236 2237 if (is_ace) { 2238 if (WARN_ON(nb_ace >= cci_config->nb_ace)) 2239 continue; 2240 ports[i].type = ACE_PORT; 2241 ++nb_ace; 2242 } else { 2243 if (WARN_ON(nb_ace_lite >= cci_config->nb_ace_lite)) 2244 continue; 2245 ports[i].type = ACE_LITE_PORT; 2246 ++nb_ace_lite; 2247 } 2248 ports[i].dn = cp; 2249 } 2250 2251 /* initialize a stashed array of ACE ports to speed-up look-up */ 2252 cci_ace_init_ports(); 2253 2254 /* 2255 * Multi-cluster systems may need this data when non-coherent, during 2256 * cluster power-up/power-down. Make sure it reaches main memory. 2257 */ 2258 sync_cache_w(&cci_ctrl_base); 2259 sync_cache_w(&cci_ctrl_phys); 2260 sync_cache_w(&ports); 2261 sync_cache_w(&cpu_port); 2262 __sync_cache_range_w(ports, sizeof(*ports) * nb_cci_ports); 2263 pr_info("ARM CCI driver probed\n"); 2264 2265 return 0; 2266 } 2267 #else /* !CONFIG_ARM_CCI400_PORT_CTRL */ 2268 static inline int cci_probe_ports(struct device_node *np) 2269 { 2270 return 0; 2271 } 2272 #endif /* CONFIG_ARM_CCI400_PORT_CTRL */ 2273 2274 static int cci_probe(void) 2275 { 2276 int ret; 2277 struct device_node *np; 2278 struct resource res; 2279 2280 np = of_find_matching_node(NULL, arm_cci_matches); 2281 if(!np || !of_device_is_available(np)) 2282 return -ENODEV; 2283 2284 ret = of_address_to_resource(np, 0, &res); 2285 if (!ret) { 2286 cci_ctrl_base = ioremap(res.start, resource_size(&res)); 2287 cci_ctrl_phys = res.start; 2288 } 2289 if (ret || !cci_ctrl_base) { 2290 WARN(1, "unable to ioremap CCI ctrl\n"); 2291 return -ENXIO; 2292 } 2293 2294 return cci_probe_ports(np); 2295 } 2296 2297 static int cci_init_status = -EAGAIN; 2298 static DEFINE_MUTEX(cci_probing); 2299 2300 static int cci_init(void) 2301 { 2302 if (cci_init_status != -EAGAIN) 2303 return cci_init_status; 2304 2305 mutex_lock(&cci_probing); 2306 if (cci_init_status == -EAGAIN) 2307 cci_init_status = cci_probe(); 2308 mutex_unlock(&cci_probing); 2309 return cci_init_status; 2310 } 2311 2312 /* 2313 * To sort out early init calls ordering a helper function is provided to 2314 * check if the CCI driver has beed initialized. Function check if the driver 2315 * has been initialized, if not it calls the init function that probes 2316 * the driver and updates the return value. 2317 */ 2318 bool cci_probed(void) 2319 { 2320 return cci_init() == 0; 2321 } 2322 EXPORT_SYMBOL_GPL(cci_probed); 2323 2324 early_initcall(cci_init); 2325 core_initcall(cci_platform_init); 2326 MODULE_LICENSE("GPL"); 2327 MODULE_DESCRIPTION("ARM CCI support"); 2328