1 /* 2 * CCI cache coherent interconnect driver 3 * 4 * Copyright (C) 2013 ARM Ltd. 5 * Author: Lorenzo Pieralisi <lorenzo.pieralisi@arm.com> 6 * 7 * This program is free software; you can redistribute it and/or modify 8 * it under the terms of the GNU General Public License version 2 as 9 * published by the Free Software Foundation. 10 * 11 * This program is distributed "as is" WITHOUT ANY WARRANTY of any 12 * kind, whether express or implied; without even the implied warranty 13 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 * GNU General Public License for more details. 15 */ 16 17 #include <linux/arm-cci.h> 18 #include <linux/io.h> 19 #include <linux/interrupt.h> 20 #include <linux/module.h> 21 #include <linux/of_address.h> 22 #include <linux/of_irq.h> 23 #include <linux/of_platform.h> 24 #include <linux/perf_event.h> 25 #include <linux/platform_device.h> 26 #include <linux/slab.h> 27 #include <linux/spinlock.h> 28 29 #include <asm/cacheflush.h> 30 #include <asm/smp_plat.h> 31 32 static void __iomem *cci_ctrl_base; 33 static unsigned long cci_ctrl_phys; 34 35 #ifdef CONFIG_ARM_CCI400_PORT_CTRL 36 struct cci_nb_ports { 37 unsigned int nb_ace; 38 unsigned int nb_ace_lite; 39 }; 40 41 static const struct cci_nb_ports cci400_ports = { 42 .nb_ace = 2, 43 .nb_ace_lite = 3 44 }; 45 46 #define CCI400_PORTS_DATA (&cci400_ports) 47 #else 48 #define CCI400_PORTS_DATA (NULL) 49 #endif 50 51 static const struct of_device_id arm_cci_matches[] = { 52 #ifdef CONFIG_ARM_CCI400_COMMON 53 {.compatible = "arm,cci-400", .data = CCI400_PORTS_DATA }, 54 #endif 55 #ifdef CONFIG_ARM_CCI5xx_PMU 56 { .compatible = "arm,cci-500", }, 57 { .compatible = "arm,cci-550", }, 58 #endif 59 {}, 60 }; 61 62 #ifdef CONFIG_ARM_CCI_PMU 63 64 #define DRIVER_NAME "ARM-CCI" 65 #define DRIVER_NAME_PMU DRIVER_NAME " PMU" 66 67 #define CCI_PMCR 0x0100 68 #define CCI_PID2 0x0fe8 69 70 #define CCI_PMCR_CEN 0x00000001 71 #define CCI_PMCR_NCNT_MASK 0x0000f800 72 #define CCI_PMCR_NCNT_SHIFT 11 73 74 #define CCI_PID2_REV_MASK 0xf0 75 #define CCI_PID2_REV_SHIFT 4 76 77 #define CCI_PMU_EVT_SEL 0x000 78 #define CCI_PMU_CNTR 0x004 79 #define CCI_PMU_CNTR_CTRL 0x008 80 #define CCI_PMU_OVRFLW 0x00c 81 82 #define CCI_PMU_OVRFLW_FLAG 1 83 84 #define CCI_PMU_CNTR_SIZE(model) ((model)->cntr_size) 85 #define CCI_PMU_CNTR_BASE(model, idx) ((idx) * CCI_PMU_CNTR_SIZE(model)) 86 #define CCI_PMU_CNTR_MASK ((1ULL << 32) -1) 87 #define CCI_PMU_CNTR_LAST(cci_pmu) (cci_pmu->num_cntrs - 1) 88 89 #define CCI_PMU_MAX_HW_CNTRS(model) \ 90 ((model)->num_hw_cntrs + (model)->fixed_hw_cntrs) 91 92 /* Types of interfaces that can generate events */ 93 enum { 94 CCI_IF_SLAVE, 95 CCI_IF_MASTER, 96 #ifdef CONFIG_ARM_CCI5xx_PMU 97 CCI_IF_GLOBAL, 98 #endif 99 CCI_IF_MAX, 100 }; 101 102 struct event_range { 103 u32 min; 104 u32 max; 105 }; 106 107 struct cci_pmu_hw_events { 108 struct perf_event **events; 109 unsigned long *used_mask; 110 raw_spinlock_t pmu_lock; 111 }; 112 113 struct cci_pmu; 114 /* 115 * struct cci_pmu_model: 116 * @fixed_hw_cntrs - Number of fixed event counters 117 * @num_hw_cntrs - Maximum number of programmable event counters 118 * @cntr_size - Size of an event counter mapping 119 */ 120 struct cci_pmu_model { 121 char *name; 122 u32 fixed_hw_cntrs; 123 u32 num_hw_cntrs; 124 u32 cntr_size; 125 struct attribute **format_attrs; 126 struct attribute **event_attrs; 127 struct event_range event_ranges[CCI_IF_MAX]; 128 int (*validate_hw_event)(struct cci_pmu *, unsigned long); 129 int (*get_event_idx)(struct cci_pmu *, struct cci_pmu_hw_events *, unsigned long); 130 void (*write_counters)(struct cci_pmu *, unsigned long *); 131 }; 132 133 static struct cci_pmu_model cci_pmu_models[]; 134 135 struct cci_pmu { 136 void __iomem *base; 137 struct pmu pmu; 138 int nr_irqs; 139 int *irqs; 140 unsigned long active_irqs; 141 const struct cci_pmu_model *model; 142 struct cci_pmu_hw_events hw_events; 143 struct platform_device *plat_device; 144 int num_cntrs; 145 atomic_t active_events; 146 struct mutex reserve_mutex; 147 struct list_head entry; 148 cpumask_t cpus; 149 }; 150 151 #define to_cci_pmu(c) (container_of(c, struct cci_pmu, pmu)) 152 153 static DEFINE_MUTEX(cci_pmu_mutex); 154 static LIST_HEAD(cci_pmu_list); 155 156 enum cci_models { 157 #ifdef CONFIG_ARM_CCI400_PMU 158 CCI400_R0, 159 CCI400_R1, 160 #endif 161 #ifdef CONFIG_ARM_CCI5xx_PMU 162 CCI500_R0, 163 CCI550_R0, 164 #endif 165 CCI_MODEL_MAX 166 }; 167 168 static void pmu_write_counters(struct cci_pmu *cci_pmu, 169 unsigned long *mask); 170 static ssize_t cci_pmu_format_show(struct device *dev, 171 struct device_attribute *attr, char *buf); 172 static ssize_t cci_pmu_event_show(struct device *dev, 173 struct device_attribute *attr, char *buf); 174 175 #define CCI_EXT_ATTR_ENTRY(_name, _func, _config) \ 176 &((struct dev_ext_attribute[]) { \ 177 { __ATTR(_name, S_IRUGO, _func, NULL), (void *)_config } \ 178 })[0].attr.attr 179 180 #define CCI_FORMAT_EXT_ATTR_ENTRY(_name, _config) \ 181 CCI_EXT_ATTR_ENTRY(_name, cci_pmu_format_show, (char *)_config) 182 #define CCI_EVENT_EXT_ATTR_ENTRY(_name, _config) \ 183 CCI_EXT_ATTR_ENTRY(_name, cci_pmu_event_show, (unsigned long)_config) 184 185 /* CCI400 PMU Specific definitions */ 186 187 #ifdef CONFIG_ARM_CCI400_PMU 188 189 /* Port ids */ 190 #define CCI400_PORT_S0 0 191 #define CCI400_PORT_S1 1 192 #define CCI400_PORT_S2 2 193 #define CCI400_PORT_S3 3 194 #define CCI400_PORT_S4 4 195 #define CCI400_PORT_M0 5 196 #define CCI400_PORT_M1 6 197 #define CCI400_PORT_M2 7 198 199 #define CCI400_R1_PX 5 200 201 /* 202 * Instead of an event id to monitor CCI cycles, a dedicated counter is 203 * provided. Use 0xff to represent CCI cycles and hope that no future revisions 204 * make use of this event in hardware. 205 */ 206 enum cci400_perf_events { 207 CCI400_PMU_CYCLES = 0xff 208 }; 209 210 #define CCI400_PMU_CYCLE_CNTR_IDX 0 211 #define CCI400_PMU_CNTR0_IDX 1 212 213 /* 214 * CCI PMU event id is an 8-bit value made of two parts - bits 7:5 for one of 8 215 * ports and bits 4:0 are event codes. There are different event codes 216 * associated with each port type. 217 * 218 * Additionally, the range of events associated with the port types changed 219 * between Rev0 and Rev1. 220 * 221 * The constants below define the range of valid codes for each port type for 222 * the different revisions and are used to validate the event to be monitored. 223 */ 224 225 #define CCI400_PMU_EVENT_MASK 0xffUL 226 #define CCI400_PMU_EVENT_SOURCE_SHIFT 5 227 #define CCI400_PMU_EVENT_SOURCE_MASK 0x7 228 #define CCI400_PMU_EVENT_CODE_SHIFT 0 229 #define CCI400_PMU_EVENT_CODE_MASK 0x1f 230 #define CCI400_PMU_EVENT_SOURCE(event) \ 231 ((event >> CCI400_PMU_EVENT_SOURCE_SHIFT) & \ 232 CCI400_PMU_EVENT_SOURCE_MASK) 233 #define CCI400_PMU_EVENT_CODE(event) \ 234 ((event >> CCI400_PMU_EVENT_CODE_SHIFT) & CCI400_PMU_EVENT_CODE_MASK) 235 236 #define CCI400_R0_SLAVE_PORT_MIN_EV 0x00 237 #define CCI400_R0_SLAVE_PORT_MAX_EV 0x13 238 #define CCI400_R0_MASTER_PORT_MIN_EV 0x14 239 #define CCI400_R0_MASTER_PORT_MAX_EV 0x1a 240 241 #define CCI400_R1_SLAVE_PORT_MIN_EV 0x00 242 #define CCI400_R1_SLAVE_PORT_MAX_EV 0x14 243 #define CCI400_R1_MASTER_PORT_MIN_EV 0x00 244 #define CCI400_R1_MASTER_PORT_MAX_EV 0x11 245 246 #define CCI400_CYCLE_EVENT_EXT_ATTR_ENTRY(_name, _config) \ 247 CCI_EXT_ATTR_ENTRY(_name, cci400_pmu_cycle_event_show, \ 248 (unsigned long)_config) 249 250 static ssize_t cci400_pmu_cycle_event_show(struct device *dev, 251 struct device_attribute *attr, char *buf); 252 253 static struct attribute *cci400_pmu_format_attrs[] = { 254 CCI_FORMAT_EXT_ATTR_ENTRY(event, "config:0-4"), 255 CCI_FORMAT_EXT_ATTR_ENTRY(source, "config:5-7"), 256 NULL 257 }; 258 259 static struct attribute *cci400_r0_pmu_event_attrs[] = { 260 /* Slave events */ 261 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_any, 0x0), 262 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_device, 0x01), 263 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_normal_or_nonshareable, 0x2), 264 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_inner_or_outershareable, 0x3), 265 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_cache_maintenance, 0x4), 266 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_mem_barrier, 0x5), 267 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_sync_barrier, 0x6), 268 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_dvm_msg, 0x7), 269 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_dvm_msg_sync, 0x8), 270 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_stall_tt_full, 0x9), 271 CCI_EVENT_EXT_ATTR_ENTRY(si_r_data_last_hs_snoop, 0xA), 272 CCI_EVENT_EXT_ATTR_ENTRY(si_r_data_stall_rvalids_h_rready_l, 0xB), 273 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_any, 0xC), 274 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_device, 0xD), 275 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_normal_or_nonshareable, 0xE), 276 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_inner_or_outershare_wback_wclean, 0xF), 277 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_write_unique, 0x10), 278 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_write_line_unique, 0x11), 279 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_evict, 0x12), 280 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_stall_tt_full, 0x13), 281 /* Master events */ 282 CCI_EVENT_EXT_ATTR_ENTRY(mi_retry_speculative_fetch, 0x14), 283 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_addr_hazard, 0x15), 284 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_id_hazard, 0x16), 285 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_tt_full, 0x17), 286 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_barrier_hazard, 0x18), 287 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_barrier_hazard, 0x19), 288 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_tt_full, 0x1A), 289 /* Special event for cycles counter */ 290 CCI400_CYCLE_EVENT_EXT_ATTR_ENTRY(cycles, 0xff), 291 NULL 292 }; 293 294 static struct attribute *cci400_r1_pmu_event_attrs[] = { 295 /* Slave events */ 296 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_any, 0x0), 297 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_device, 0x01), 298 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_normal_or_nonshareable, 0x2), 299 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_inner_or_outershareable, 0x3), 300 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_cache_maintenance, 0x4), 301 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_mem_barrier, 0x5), 302 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_sync_barrier, 0x6), 303 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_dvm_msg, 0x7), 304 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_dvm_msg_sync, 0x8), 305 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_stall_tt_full, 0x9), 306 CCI_EVENT_EXT_ATTR_ENTRY(si_r_data_last_hs_snoop, 0xA), 307 CCI_EVENT_EXT_ATTR_ENTRY(si_r_data_stall_rvalids_h_rready_l, 0xB), 308 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_any, 0xC), 309 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_device, 0xD), 310 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_normal_or_nonshareable, 0xE), 311 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_inner_or_outershare_wback_wclean, 0xF), 312 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_write_unique, 0x10), 313 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_write_line_unique, 0x11), 314 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_evict, 0x12), 315 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_stall_tt_full, 0x13), 316 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_stall_slave_id_hazard, 0x14), 317 /* Master events */ 318 CCI_EVENT_EXT_ATTR_ENTRY(mi_retry_speculative_fetch, 0x0), 319 CCI_EVENT_EXT_ATTR_ENTRY(mi_stall_cycle_addr_hazard, 0x1), 320 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_master_id_hazard, 0x2), 321 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_hi_prio_rtq_full, 0x3), 322 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_barrier_hazard, 0x4), 323 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_barrier_hazard, 0x5), 324 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_wtq_full, 0x6), 325 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_low_prio_rtq_full, 0x7), 326 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_mid_prio_rtq_full, 0x8), 327 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_qvn_vn0, 0x9), 328 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_qvn_vn1, 0xA), 329 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_qvn_vn2, 0xB), 330 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_qvn_vn3, 0xC), 331 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_qvn_vn0, 0xD), 332 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_qvn_vn1, 0xE), 333 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_qvn_vn2, 0xF), 334 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_qvn_vn3, 0x10), 335 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_unique_or_line_unique_addr_hazard, 0x11), 336 /* Special event for cycles counter */ 337 CCI400_CYCLE_EVENT_EXT_ATTR_ENTRY(cycles, 0xff), 338 NULL 339 }; 340 341 static ssize_t cci400_pmu_cycle_event_show(struct device *dev, 342 struct device_attribute *attr, char *buf) 343 { 344 struct dev_ext_attribute *eattr = container_of(attr, 345 struct dev_ext_attribute, attr); 346 return snprintf(buf, PAGE_SIZE, "config=0x%lx\n", (unsigned long)eattr->var); 347 } 348 349 static int cci400_get_event_idx(struct cci_pmu *cci_pmu, 350 struct cci_pmu_hw_events *hw, 351 unsigned long cci_event) 352 { 353 int idx; 354 355 /* cycles event idx is fixed */ 356 if (cci_event == CCI400_PMU_CYCLES) { 357 if (test_and_set_bit(CCI400_PMU_CYCLE_CNTR_IDX, hw->used_mask)) 358 return -EAGAIN; 359 360 return CCI400_PMU_CYCLE_CNTR_IDX; 361 } 362 363 for (idx = CCI400_PMU_CNTR0_IDX; idx <= CCI_PMU_CNTR_LAST(cci_pmu); ++idx) 364 if (!test_and_set_bit(idx, hw->used_mask)) 365 return idx; 366 367 /* No counters available */ 368 return -EAGAIN; 369 } 370 371 static int cci400_validate_hw_event(struct cci_pmu *cci_pmu, unsigned long hw_event) 372 { 373 u8 ev_source = CCI400_PMU_EVENT_SOURCE(hw_event); 374 u8 ev_code = CCI400_PMU_EVENT_CODE(hw_event); 375 int if_type; 376 377 if (hw_event & ~CCI400_PMU_EVENT_MASK) 378 return -ENOENT; 379 380 if (hw_event == CCI400_PMU_CYCLES) 381 return hw_event; 382 383 switch (ev_source) { 384 case CCI400_PORT_S0: 385 case CCI400_PORT_S1: 386 case CCI400_PORT_S2: 387 case CCI400_PORT_S3: 388 case CCI400_PORT_S4: 389 /* Slave Interface */ 390 if_type = CCI_IF_SLAVE; 391 break; 392 case CCI400_PORT_M0: 393 case CCI400_PORT_M1: 394 case CCI400_PORT_M2: 395 /* Master Interface */ 396 if_type = CCI_IF_MASTER; 397 break; 398 default: 399 return -ENOENT; 400 } 401 402 if (ev_code >= cci_pmu->model->event_ranges[if_type].min && 403 ev_code <= cci_pmu->model->event_ranges[if_type].max) 404 return hw_event; 405 406 return -ENOENT; 407 } 408 409 static int probe_cci400_revision(void) 410 { 411 int rev; 412 rev = readl_relaxed(cci_ctrl_base + CCI_PID2) & CCI_PID2_REV_MASK; 413 rev >>= CCI_PID2_REV_SHIFT; 414 415 if (rev < CCI400_R1_PX) 416 return CCI400_R0; 417 else 418 return CCI400_R1; 419 } 420 421 static const struct cci_pmu_model *probe_cci_model(struct platform_device *pdev) 422 { 423 if (platform_has_secure_cci_access()) 424 return &cci_pmu_models[probe_cci400_revision()]; 425 return NULL; 426 } 427 #else /* !CONFIG_ARM_CCI400_PMU */ 428 static inline struct cci_pmu_model *probe_cci_model(struct platform_device *pdev) 429 { 430 return NULL; 431 } 432 #endif /* CONFIG_ARM_CCI400_PMU */ 433 434 #ifdef CONFIG_ARM_CCI5xx_PMU 435 436 /* 437 * CCI5xx PMU event id is an 9-bit value made of two parts. 438 * bits [8:5] - Source for the event 439 * bits [4:0] - Event code (specific to type of interface) 440 * 441 * 442 */ 443 444 /* Port ids */ 445 #define CCI5xx_PORT_S0 0x0 446 #define CCI5xx_PORT_S1 0x1 447 #define CCI5xx_PORT_S2 0x2 448 #define CCI5xx_PORT_S3 0x3 449 #define CCI5xx_PORT_S4 0x4 450 #define CCI5xx_PORT_S5 0x5 451 #define CCI5xx_PORT_S6 0x6 452 453 #define CCI5xx_PORT_M0 0x8 454 #define CCI5xx_PORT_M1 0x9 455 #define CCI5xx_PORT_M2 0xa 456 #define CCI5xx_PORT_M3 0xb 457 #define CCI5xx_PORT_M4 0xc 458 #define CCI5xx_PORT_M5 0xd 459 #define CCI5xx_PORT_M6 0xe 460 461 #define CCI5xx_PORT_GLOBAL 0xf 462 463 #define CCI5xx_PMU_EVENT_MASK 0x1ffUL 464 #define CCI5xx_PMU_EVENT_SOURCE_SHIFT 0x5 465 #define CCI5xx_PMU_EVENT_SOURCE_MASK 0xf 466 #define CCI5xx_PMU_EVENT_CODE_SHIFT 0x0 467 #define CCI5xx_PMU_EVENT_CODE_MASK 0x1f 468 469 #define CCI5xx_PMU_EVENT_SOURCE(event) \ 470 ((event >> CCI5xx_PMU_EVENT_SOURCE_SHIFT) & CCI5xx_PMU_EVENT_SOURCE_MASK) 471 #define CCI5xx_PMU_EVENT_CODE(event) \ 472 ((event >> CCI5xx_PMU_EVENT_CODE_SHIFT) & CCI5xx_PMU_EVENT_CODE_MASK) 473 474 #define CCI5xx_SLAVE_PORT_MIN_EV 0x00 475 #define CCI5xx_SLAVE_PORT_MAX_EV 0x1f 476 #define CCI5xx_MASTER_PORT_MIN_EV 0x00 477 #define CCI5xx_MASTER_PORT_MAX_EV 0x06 478 #define CCI5xx_GLOBAL_PORT_MIN_EV 0x00 479 #define CCI5xx_GLOBAL_PORT_MAX_EV 0x0f 480 481 482 #define CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(_name, _config) \ 483 CCI_EXT_ATTR_ENTRY(_name, cci5xx_pmu_global_event_show, \ 484 (unsigned long) _config) 485 486 static ssize_t cci5xx_pmu_global_event_show(struct device *dev, 487 struct device_attribute *attr, char *buf); 488 489 static struct attribute *cci5xx_pmu_format_attrs[] = { 490 CCI_FORMAT_EXT_ATTR_ENTRY(event, "config:0-4"), 491 CCI_FORMAT_EXT_ATTR_ENTRY(source, "config:5-8"), 492 NULL, 493 }; 494 495 static struct attribute *cci5xx_pmu_event_attrs[] = { 496 /* Slave events */ 497 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_arvalid, 0x0), 498 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_dev, 0x1), 499 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_nonshareable, 0x2), 500 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_shareable_non_alloc, 0x3), 501 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_shareable_alloc, 0x4), 502 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_invalidate, 0x5), 503 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_cache_maint, 0x6), 504 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_dvm_msg, 0x7), 505 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_rval, 0x8), 506 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_rlast_snoop, 0x9), 507 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_awalid, 0xA), 508 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_dev, 0xB), 509 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_non_shareable, 0xC), 510 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_share_wb, 0xD), 511 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_share_wlu, 0xE), 512 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_share_wunique, 0xF), 513 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_evict, 0x10), 514 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_wrevict, 0x11), 515 CCI_EVENT_EXT_ATTR_ENTRY(si_w_data_beat, 0x12), 516 CCI_EVENT_EXT_ATTR_ENTRY(si_srq_acvalid, 0x13), 517 CCI_EVENT_EXT_ATTR_ENTRY(si_srq_read, 0x14), 518 CCI_EVENT_EXT_ATTR_ENTRY(si_srq_clean, 0x15), 519 CCI_EVENT_EXT_ATTR_ENTRY(si_srq_data_transfer_low, 0x16), 520 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_stall_arvalid, 0x17), 521 CCI_EVENT_EXT_ATTR_ENTRY(si_r_data_stall, 0x18), 522 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_stall, 0x19), 523 CCI_EVENT_EXT_ATTR_ENTRY(si_w_data_stall, 0x1A), 524 CCI_EVENT_EXT_ATTR_ENTRY(si_w_resp_stall, 0x1B), 525 CCI_EVENT_EXT_ATTR_ENTRY(si_srq_stall, 0x1C), 526 CCI_EVENT_EXT_ATTR_ENTRY(si_s_data_stall, 0x1D), 527 CCI_EVENT_EXT_ATTR_ENTRY(si_rq_stall_ot_limit, 0x1E), 528 CCI_EVENT_EXT_ATTR_ENTRY(si_r_stall_arbit, 0x1F), 529 530 /* Master events */ 531 CCI_EVENT_EXT_ATTR_ENTRY(mi_r_data_beat_any, 0x0), 532 CCI_EVENT_EXT_ATTR_ENTRY(mi_w_data_beat_any, 0x1), 533 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall, 0x2), 534 CCI_EVENT_EXT_ATTR_ENTRY(mi_r_data_stall, 0x3), 535 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall, 0x4), 536 CCI_EVENT_EXT_ATTR_ENTRY(mi_w_data_stall, 0x5), 537 CCI_EVENT_EXT_ATTR_ENTRY(mi_w_resp_stall, 0x6), 538 539 /* Global events */ 540 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_filter_bank_0_1, 0x0), 541 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_filter_bank_2_3, 0x1), 542 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_filter_bank_4_5, 0x2), 543 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_filter_bank_6_7, 0x3), 544 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_miss_filter_bank_0_1, 0x4), 545 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_miss_filter_bank_2_3, 0x5), 546 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_miss_filter_bank_4_5, 0x6), 547 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_miss_filter_bank_6_7, 0x7), 548 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_back_invalidation, 0x8), 549 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_stall_alloc_busy, 0x9), 550 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_stall_tt_full, 0xA), 551 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_wrq, 0xB), 552 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_cd_hs, 0xC), 553 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_rq_stall_addr_hazard, 0xD), 554 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snopp_rq_stall_tt_full, 0xE), 555 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_rq_tzmp1_prot, 0xF), 556 NULL 557 }; 558 559 static ssize_t cci5xx_pmu_global_event_show(struct device *dev, 560 struct device_attribute *attr, char *buf) 561 { 562 struct dev_ext_attribute *eattr = container_of(attr, 563 struct dev_ext_attribute, attr); 564 /* Global events have single fixed source code */ 565 return snprintf(buf, PAGE_SIZE, "event=0x%lx,source=0x%x\n", 566 (unsigned long)eattr->var, CCI5xx_PORT_GLOBAL); 567 } 568 569 /* 570 * CCI500 provides 8 independent event counters that can count 571 * any of the events available. 572 * CCI500 PMU event source ids 573 * 0x0-0x6 - Slave interfaces 574 * 0x8-0xD - Master interfaces 575 * 0xf - Global Events 576 * 0x7,0xe - Reserved 577 */ 578 static int cci500_validate_hw_event(struct cci_pmu *cci_pmu, 579 unsigned long hw_event) 580 { 581 u32 ev_source = CCI5xx_PMU_EVENT_SOURCE(hw_event); 582 u32 ev_code = CCI5xx_PMU_EVENT_CODE(hw_event); 583 int if_type; 584 585 if (hw_event & ~CCI5xx_PMU_EVENT_MASK) 586 return -ENOENT; 587 588 switch (ev_source) { 589 case CCI5xx_PORT_S0: 590 case CCI5xx_PORT_S1: 591 case CCI5xx_PORT_S2: 592 case CCI5xx_PORT_S3: 593 case CCI5xx_PORT_S4: 594 case CCI5xx_PORT_S5: 595 case CCI5xx_PORT_S6: 596 if_type = CCI_IF_SLAVE; 597 break; 598 case CCI5xx_PORT_M0: 599 case CCI5xx_PORT_M1: 600 case CCI5xx_PORT_M2: 601 case CCI5xx_PORT_M3: 602 case CCI5xx_PORT_M4: 603 case CCI5xx_PORT_M5: 604 if_type = CCI_IF_MASTER; 605 break; 606 case CCI5xx_PORT_GLOBAL: 607 if_type = CCI_IF_GLOBAL; 608 break; 609 default: 610 return -ENOENT; 611 } 612 613 if (ev_code >= cci_pmu->model->event_ranges[if_type].min && 614 ev_code <= cci_pmu->model->event_ranges[if_type].max) 615 return hw_event; 616 617 return -ENOENT; 618 } 619 620 /* 621 * CCI550 provides 8 independent event counters that can count 622 * any of the events available. 623 * CCI550 PMU event source ids 624 * 0x0-0x6 - Slave interfaces 625 * 0x8-0xe - Master interfaces 626 * 0xf - Global Events 627 * 0x7 - Reserved 628 */ 629 static int cci550_validate_hw_event(struct cci_pmu *cci_pmu, 630 unsigned long hw_event) 631 { 632 u32 ev_source = CCI5xx_PMU_EVENT_SOURCE(hw_event); 633 u32 ev_code = CCI5xx_PMU_EVENT_CODE(hw_event); 634 int if_type; 635 636 if (hw_event & ~CCI5xx_PMU_EVENT_MASK) 637 return -ENOENT; 638 639 switch (ev_source) { 640 case CCI5xx_PORT_S0: 641 case CCI5xx_PORT_S1: 642 case CCI5xx_PORT_S2: 643 case CCI5xx_PORT_S3: 644 case CCI5xx_PORT_S4: 645 case CCI5xx_PORT_S5: 646 case CCI5xx_PORT_S6: 647 if_type = CCI_IF_SLAVE; 648 break; 649 case CCI5xx_PORT_M0: 650 case CCI5xx_PORT_M1: 651 case CCI5xx_PORT_M2: 652 case CCI5xx_PORT_M3: 653 case CCI5xx_PORT_M4: 654 case CCI5xx_PORT_M5: 655 case CCI5xx_PORT_M6: 656 if_type = CCI_IF_MASTER; 657 break; 658 case CCI5xx_PORT_GLOBAL: 659 if_type = CCI_IF_GLOBAL; 660 break; 661 default: 662 return -ENOENT; 663 } 664 665 if (ev_code >= cci_pmu->model->event_ranges[if_type].min && 666 ev_code <= cci_pmu->model->event_ranges[if_type].max) 667 return hw_event; 668 669 return -ENOENT; 670 } 671 672 #endif /* CONFIG_ARM_CCI5xx_PMU */ 673 674 /* 675 * Program the CCI PMU counters which have PERF_HES_ARCH set 676 * with the event period and mark them ready before we enable 677 * PMU. 678 */ 679 static void cci_pmu_sync_counters(struct cci_pmu *cci_pmu) 680 { 681 int i; 682 struct cci_pmu_hw_events *cci_hw = &cci_pmu->hw_events; 683 684 DECLARE_BITMAP(mask, cci_pmu->num_cntrs); 685 686 bitmap_zero(mask, cci_pmu->num_cntrs); 687 for_each_set_bit(i, cci_pmu->hw_events.used_mask, cci_pmu->num_cntrs) { 688 struct perf_event *event = cci_hw->events[i]; 689 690 if (WARN_ON(!event)) 691 continue; 692 693 /* Leave the events which are not counting */ 694 if (event->hw.state & PERF_HES_STOPPED) 695 continue; 696 if (event->hw.state & PERF_HES_ARCH) { 697 set_bit(i, mask); 698 event->hw.state &= ~PERF_HES_ARCH; 699 } 700 } 701 702 pmu_write_counters(cci_pmu, mask); 703 } 704 705 /* Should be called with cci_pmu->hw_events->pmu_lock held */ 706 static void __cci_pmu_enable_nosync(struct cci_pmu *cci_pmu) 707 { 708 u32 val; 709 710 /* Enable all the PMU counters. */ 711 val = readl_relaxed(cci_ctrl_base + CCI_PMCR) | CCI_PMCR_CEN; 712 writel(val, cci_ctrl_base + CCI_PMCR); 713 } 714 715 /* Should be called with cci_pmu->hw_events->pmu_lock held */ 716 static void __cci_pmu_enable_sync(struct cci_pmu *cci_pmu) 717 { 718 cci_pmu_sync_counters(cci_pmu); 719 __cci_pmu_enable_nosync(cci_pmu); 720 } 721 722 /* Should be called with cci_pmu->hw_events->pmu_lock held */ 723 static void __cci_pmu_disable(void) 724 { 725 u32 val; 726 727 /* Disable all the PMU counters. */ 728 val = readl_relaxed(cci_ctrl_base + CCI_PMCR) & ~CCI_PMCR_CEN; 729 writel(val, cci_ctrl_base + CCI_PMCR); 730 } 731 732 static ssize_t cci_pmu_format_show(struct device *dev, 733 struct device_attribute *attr, char *buf) 734 { 735 struct dev_ext_attribute *eattr = container_of(attr, 736 struct dev_ext_attribute, attr); 737 return snprintf(buf, PAGE_SIZE, "%s\n", (char *)eattr->var); 738 } 739 740 static ssize_t cci_pmu_event_show(struct device *dev, 741 struct device_attribute *attr, char *buf) 742 { 743 struct dev_ext_attribute *eattr = container_of(attr, 744 struct dev_ext_attribute, attr); 745 /* source parameter is mandatory for normal PMU events */ 746 return snprintf(buf, PAGE_SIZE, "source=?,event=0x%lx\n", 747 (unsigned long)eattr->var); 748 } 749 750 static int pmu_is_valid_counter(struct cci_pmu *cci_pmu, int idx) 751 { 752 return 0 <= idx && idx <= CCI_PMU_CNTR_LAST(cci_pmu); 753 } 754 755 static u32 pmu_read_register(struct cci_pmu *cci_pmu, int idx, unsigned int offset) 756 { 757 return readl_relaxed(cci_pmu->base + 758 CCI_PMU_CNTR_BASE(cci_pmu->model, idx) + offset); 759 } 760 761 static void pmu_write_register(struct cci_pmu *cci_pmu, u32 value, 762 int idx, unsigned int offset) 763 { 764 writel_relaxed(value, cci_pmu->base + 765 CCI_PMU_CNTR_BASE(cci_pmu->model, idx) + offset); 766 } 767 768 static void pmu_disable_counter(struct cci_pmu *cci_pmu, int idx) 769 { 770 pmu_write_register(cci_pmu, 0, idx, CCI_PMU_CNTR_CTRL); 771 } 772 773 static void pmu_enable_counter(struct cci_pmu *cci_pmu, int idx) 774 { 775 pmu_write_register(cci_pmu, 1, idx, CCI_PMU_CNTR_CTRL); 776 } 777 778 static bool __maybe_unused 779 pmu_counter_is_enabled(struct cci_pmu *cci_pmu, int idx) 780 { 781 return (pmu_read_register(cci_pmu, idx, CCI_PMU_CNTR_CTRL) & 0x1) != 0; 782 } 783 784 static void pmu_set_event(struct cci_pmu *cci_pmu, int idx, unsigned long event) 785 { 786 pmu_write_register(cci_pmu, event, idx, CCI_PMU_EVT_SEL); 787 } 788 789 /* 790 * For all counters on the CCI-PMU, disable any 'enabled' counters, 791 * saving the changed counters in the mask, so that we can restore 792 * it later using pmu_restore_counters. The mask is private to the 793 * caller. We cannot rely on the used_mask maintained by the CCI_PMU 794 * as it only tells us if the counter is assigned to perf_event or not. 795 * The state of the perf_event cannot be locked by the PMU layer, hence 796 * we check the individual counter status (which can be locked by 797 * cci_pm->hw_events->pmu_lock). 798 * 799 * @mask should be initialised to empty by the caller. 800 */ 801 static void __maybe_unused 802 pmu_save_counters(struct cci_pmu *cci_pmu, unsigned long *mask) 803 { 804 int i; 805 806 for (i = 0; i < cci_pmu->num_cntrs; i++) { 807 if (pmu_counter_is_enabled(cci_pmu, i)) { 808 set_bit(i, mask); 809 pmu_disable_counter(cci_pmu, i); 810 } 811 } 812 } 813 814 /* 815 * Restore the status of the counters. Reversal of the pmu_save_counters(). 816 * For each counter set in the mask, enable the counter back. 817 */ 818 static void __maybe_unused 819 pmu_restore_counters(struct cci_pmu *cci_pmu, unsigned long *mask) 820 { 821 int i; 822 823 for_each_set_bit(i, mask, cci_pmu->num_cntrs) 824 pmu_enable_counter(cci_pmu, i); 825 } 826 827 /* 828 * Returns the number of programmable counters actually implemented 829 * by the cci 830 */ 831 static u32 pmu_get_max_counters(void) 832 { 833 return (readl_relaxed(cci_ctrl_base + CCI_PMCR) & 834 CCI_PMCR_NCNT_MASK) >> CCI_PMCR_NCNT_SHIFT; 835 } 836 837 static int pmu_get_event_idx(struct cci_pmu_hw_events *hw, struct perf_event *event) 838 { 839 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu); 840 unsigned long cci_event = event->hw.config_base; 841 int idx; 842 843 if (cci_pmu->model->get_event_idx) 844 return cci_pmu->model->get_event_idx(cci_pmu, hw, cci_event); 845 846 /* Generic code to find an unused idx from the mask */ 847 for(idx = 0; idx <= CCI_PMU_CNTR_LAST(cci_pmu); idx++) 848 if (!test_and_set_bit(idx, hw->used_mask)) 849 return idx; 850 851 /* No counters available */ 852 return -EAGAIN; 853 } 854 855 static int pmu_map_event(struct perf_event *event) 856 { 857 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu); 858 859 if (event->attr.type < PERF_TYPE_MAX || 860 !cci_pmu->model->validate_hw_event) 861 return -ENOENT; 862 863 return cci_pmu->model->validate_hw_event(cci_pmu, event->attr.config); 864 } 865 866 static int pmu_request_irq(struct cci_pmu *cci_pmu, irq_handler_t handler) 867 { 868 int i; 869 struct platform_device *pmu_device = cci_pmu->plat_device; 870 871 if (unlikely(!pmu_device)) 872 return -ENODEV; 873 874 if (cci_pmu->nr_irqs < 1) { 875 dev_err(&pmu_device->dev, "no irqs for CCI PMUs defined\n"); 876 return -ENODEV; 877 } 878 879 /* 880 * Register all available CCI PMU interrupts. In the interrupt handler 881 * we iterate over the counters checking for interrupt source (the 882 * overflowing counter) and clear it. 883 * 884 * This should allow handling of non-unique interrupt for the counters. 885 */ 886 for (i = 0; i < cci_pmu->nr_irqs; i++) { 887 int err = request_irq(cci_pmu->irqs[i], handler, IRQF_SHARED, 888 "arm-cci-pmu", cci_pmu); 889 if (err) { 890 dev_err(&pmu_device->dev, "unable to request IRQ%d for ARM CCI PMU counters\n", 891 cci_pmu->irqs[i]); 892 return err; 893 } 894 895 set_bit(i, &cci_pmu->active_irqs); 896 } 897 898 return 0; 899 } 900 901 static void pmu_free_irq(struct cci_pmu *cci_pmu) 902 { 903 int i; 904 905 for (i = 0; i < cci_pmu->nr_irqs; i++) { 906 if (!test_and_clear_bit(i, &cci_pmu->active_irqs)) 907 continue; 908 909 free_irq(cci_pmu->irqs[i], cci_pmu); 910 } 911 } 912 913 static u32 pmu_read_counter(struct perf_event *event) 914 { 915 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu); 916 struct hw_perf_event *hw_counter = &event->hw; 917 int idx = hw_counter->idx; 918 u32 value; 919 920 if (unlikely(!pmu_is_valid_counter(cci_pmu, idx))) { 921 dev_err(&cci_pmu->plat_device->dev, "Invalid CCI PMU counter %d\n", idx); 922 return 0; 923 } 924 value = pmu_read_register(cci_pmu, idx, CCI_PMU_CNTR); 925 926 return value; 927 } 928 929 static void pmu_write_counter(struct cci_pmu *cci_pmu, u32 value, int idx) 930 { 931 pmu_write_register(cci_pmu, value, idx, CCI_PMU_CNTR); 932 } 933 934 static void __pmu_write_counters(struct cci_pmu *cci_pmu, unsigned long *mask) 935 { 936 int i; 937 struct cci_pmu_hw_events *cci_hw = &cci_pmu->hw_events; 938 939 for_each_set_bit(i, mask, cci_pmu->num_cntrs) { 940 struct perf_event *event = cci_hw->events[i]; 941 942 if (WARN_ON(!event)) 943 continue; 944 pmu_write_counter(cci_pmu, local64_read(&event->hw.prev_count), i); 945 } 946 } 947 948 static void pmu_write_counters(struct cci_pmu *cci_pmu, unsigned long *mask) 949 { 950 if (cci_pmu->model->write_counters) 951 cci_pmu->model->write_counters(cci_pmu, mask); 952 else 953 __pmu_write_counters(cci_pmu, mask); 954 } 955 956 #ifdef CONFIG_ARM_CCI5xx_PMU 957 958 /* 959 * CCI-500/CCI-550 has advanced power saving policies, which could gate the 960 * clocks to the PMU counters, which makes the writes to them ineffective. 961 * The only way to write to those counters is when the global counters 962 * are enabled and the particular counter is enabled. 963 * 964 * So we do the following : 965 * 966 * 1) Disable all the PMU counters, saving their current state 967 * 2) Enable the global PMU profiling, now that all counters are 968 * disabled. 969 * 970 * For each counter to be programmed, repeat steps 3-7: 971 * 972 * 3) Write an invalid event code to the event control register for the 973 counter, so that the counters are not modified. 974 * 4) Enable the counter control for the counter. 975 * 5) Set the counter value 976 * 6) Disable the counter 977 * 7) Restore the event in the target counter 978 * 979 * 8) Disable the global PMU. 980 * 9) Restore the status of the rest of the counters. 981 * 982 * We choose an event which for CCI-5xx is guaranteed not to count. 983 * We use the highest possible event code (0x1f) for the master interface 0. 984 */ 985 #define CCI5xx_INVALID_EVENT ((CCI5xx_PORT_M0 << CCI5xx_PMU_EVENT_SOURCE_SHIFT) | \ 986 (CCI5xx_PMU_EVENT_CODE_MASK << CCI5xx_PMU_EVENT_CODE_SHIFT)) 987 static void cci5xx_pmu_write_counters(struct cci_pmu *cci_pmu, unsigned long *mask) 988 { 989 int i; 990 DECLARE_BITMAP(saved_mask, cci_pmu->num_cntrs); 991 992 bitmap_zero(saved_mask, cci_pmu->num_cntrs); 993 pmu_save_counters(cci_pmu, saved_mask); 994 995 /* 996 * Now that all the counters are disabled, we can safely turn the PMU on, 997 * without syncing the status of the counters 998 */ 999 __cci_pmu_enable_nosync(cci_pmu); 1000 1001 for_each_set_bit(i, mask, cci_pmu->num_cntrs) { 1002 struct perf_event *event = cci_pmu->hw_events.events[i]; 1003 1004 if (WARN_ON(!event)) 1005 continue; 1006 1007 pmu_set_event(cci_pmu, i, CCI5xx_INVALID_EVENT); 1008 pmu_enable_counter(cci_pmu, i); 1009 pmu_write_counter(cci_pmu, local64_read(&event->hw.prev_count), i); 1010 pmu_disable_counter(cci_pmu, i); 1011 pmu_set_event(cci_pmu, i, event->hw.config_base); 1012 } 1013 1014 __cci_pmu_disable(); 1015 1016 pmu_restore_counters(cci_pmu, saved_mask); 1017 } 1018 1019 #endif /* CONFIG_ARM_CCI5xx_PMU */ 1020 1021 static u64 pmu_event_update(struct perf_event *event) 1022 { 1023 struct hw_perf_event *hwc = &event->hw; 1024 u64 delta, prev_raw_count, new_raw_count; 1025 1026 do { 1027 prev_raw_count = local64_read(&hwc->prev_count); 1028 new_raw_count = pmu_read_counter(event); 1029 } while (local64_cmpxchg(&hwc->prev_count, prev_raw_count, 1030 new_raw_count) != prev_raw_count); 1031 1032 delta = (new_raw_count - prev_raw_count) & CCI_PMU_CNTR_MASK; 1033 1034 local64_add(delta, &event->count); 1035 1036 return new_raw_count; 1037 } 1038 1039 static void pmu_read(struct perf_event *event) 1040 { 1041 pmu_event_update(event); 1042 } 1043 1044 static void pmu_event_set_period(struct perf_event *event) 1045 { 1046 struct hw_perf_event *hwc = &event->hw; 1047 /* 1048 * The CCI PMU counters have a period of 2^32. To account for the 1049 * possiblity of extreme interrupt latency we program for a period of 1050 * half that. Hopefully we can handle the interrupt before another 2^31 1051 * events occur and the counter overtakes its previous value. 1052 */ 1053 u64 val = 1ULL << 31; 1054 local64_set(&hwc->prev_count, val); 1055 1056 /* 1057 * CCI PMU uses PERF_HES_ARCH to keep track of the counters, whose 1058 * values needs to be sync-ed with the s/w state before the PMU is 1059 * enabled. 1060 * Mark this counter for sync. 1061 */ 1062 hwc->state |= PERF_HES_ARCH; 1063 } 1064 1065 static irqreturn_t pmu_handle_irq(int irq_num, void *dev) 1066 { 1067 unsigned long flags; 1068 struct cci_pmu *cci_pmu = dev; 1069 struct cci_pmu_hw_events *events = &cci_pmu->hw_events; 1070 int idx, handled = IRQ_NONE; 1071 1072 raw_spin_lock_irqsave(&events->pmu_lock, flags); 1073 1074 /* Disable the PMU while we walk through the counters */ 1075 __cci_pmu_disable(); 1076 /* 1077 * Iterate over counters and update the corresponding perf events. 1078 * This should work regardless of whether we have per-counter overflow 1079 * interrupt or a combined overflow interrupt. 1080 */ 1081 for (idx = 0; idx <= CCI_PMU_CNTR_LAST(cci_pmu); idx++) { 1082 struct perf_event *event = events->events[idx]; 1083 1084 if (!event) 1085 continue; 1086 1087 /* Did this counter overflow? */ 1088 if (!(pmu_read_register(cci_pmu, idx, CCI_PMU_OVRFLW) & 1089 CCI_PMU_OVRFLW_FLAG)) 1090 continue; 1091 1092 pmu_write_register(cci_pmu, CCI_PMU_OVRFLW_FLAG, idx, 1093 CCI_PMU_OVRFLW); 1094 1095 pmu_event_update(event); 1096 pmu_event_set_period(event); 1097 handled = IRQ_HANDLED; 1098 } 1099 1100 /* Enable the PMU and sync possibly overflowed counters */ 1101 __cci_pmu_enable_sync(cci_pmu); 1102 raw_spin_unlock_irqrestore(&events->pmu_lock, flags); 1103 1104 return IRQ_RETVAL(handled); 1105 } 1106 1107 static int cci_pmu_get_hw(struct cci_pmu *cci_pmu) 1108 { 1109 int ret = pmu_request_irq(cci_pmu, pmu_handle_irq); 1110 if (ret) { 1111 pmu_free_irq(cci_pmu); 1112 return ret; 1113 } 1114 return 0; 1115 } 1116 1117 static void cci_pmu_put_hw(struct cci_pmu *cci_pmu) 1118 { 1119 pmu_free_irq(cci_pmu); 1120 } 1121 1122 static void hw_perf_event_destroy(struct perf_event *event) 1123 { 1124 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu); 1125 atomic_t *active_events = &cci_pmu->active_events; 1126 struct mutex *reserve_mutex = &cci_pmu->reserve_mutex; 1127 1128 if (atomic_dec_and_mutex_lock(active_events, reserve_mutex)) { 1129 cci_pmu_put_hw(cci_pmu); 1130 mutex_unlock(reserve_mutex); 1131 } 1132 } 1133 1134 static void cci_pmu_enable(struct pmu *pmu) 1135 { 1136 struct cci_pmu *cci_pmu = to_cci_pmu(pmu); 1137 struct cci_pmu_hw_events *hw_events = &cci_pmu->hw_events; 1138 int enabled = bitmap_weight(hw_events->used_mask, cci_pmu->num_cntrs); 1139 unsigned long flags; 1140 1141 if (!enabled) 1142 return; 1143 1144 raw_spin_lock_irqsave(&hw_events->pmu_lock, flags); 1145 __cci_pmu_enable_sync(cci_pmu); 1146 raw_spin_unlock_irqrestore(&hw_events->pmu_lock, flags); 1147 1148 } 1149 1150 static void cci_pmu_disable(struct pmu *pmu) 1151 { 1152 struct cci_pmu *cci_pmu = to_cci_pmu(pmu); 1153 struct cci_pmu_hw_events *hw_events = &cci_pmu->hw_events; 1154 unsigned long flags; 1155 1156 raw_spin_lock_irqsave(&hw_events->pmu_lock, flags); 1157 __cci_pmu_disable(); 1158 raw_spin_unlock_irqrestore(&hw_events->pmu_lock, flags); 1159 } 1160 1161 /* 1162 * Check if the idx represents a non-programmable counter. 1163 * All the fixed event counters are mapped before the programmable 1164 * counters. 1165 */ 1166 static bool pmu_fixed_hw_idx(struct cci_pmu *cci_pmu, int idx) 1167 { 1168 return (idx >= 0) && (idx < cci_pmu->model->fixed_hw_cntrs); 1169 } 1170 1171 static void cci_pmu_start(struct perf_event *event, int pmu_flags) 1172 { 1173 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu); 1174 struct cci_pmu_hw_events *hw_events = &cci_pmu->hw_events; 1175 struct hw_perf_event *hwc = &event->hw; 1176 int idx = hwc->idx; 1177 unsigned long flags; 1178 1179 /* 1180 * To handle interrupt latency, we always reprogram the period 1181 * regardlesss of PERF_EF_RELOAD. 1182 */ 1183 if (pmu_flags & PERF_EF_RELOAD) 1184 WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE)); 1185 1186 hwc->state = 0; 1187 1188 if (unlikely(!pmu_is_valid_counter(cci_pmu, idx))) { 1189 dev_err(&cci_pmu->plat_device->dev, "Invalid CCI PMU counter %d\n", idx); 1190 return; 1191 } 1192 1193 raw_spin_lock_irqsave(&hw_events->pmu_lock, flags); 1194 1195 /* Configure the counter unless you are counting a fixed event */ 1196 if (!pmu_fixed_hw_idx(cci_pmu, idx)) 1197 pmu_set_event(cci_pmu, idx, hwc->config_base); 1198 1199 pmu_event_set_period(event); 1200 pmu_enable_counter(cci_pmu, idx); 1201 1202 raw_spin_unlock_irqrestore(&hw_events->pmu_lock, flags); 1203 } 1204 1205 static void cci_pmu_stop(struct perf_event *event, int pmu_flags) 1206 { 1207 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu); 1208 struct hw_perf_event *hwc = &event->hw; 1209 int idx = hwc->idx; 1210 1211 if (hwc->state & PERF_HES_STOPPED) 1212 return; 1213 1214 if (unlikely(!pmu_is_valid_counter(cci_pmu, idx))) { 1215 dev_err(&cci_pmu->plat_device->dev, "Invalid CCI PMU counter %d\n", idx); 1216 return; 1217 } 1218 1219 /* 1220 * We always reprogram the counter, so ignore PERF_EF_UPDATE. See 1221 * cci_pmu_start() 1222 */ 1223 pmu_disable_counter(cci_pmu, idx); 1224 pmu_event_update(event); 1225 hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE; 1226 } 1227 1228 static int cci_pmu_add(struct perf_event *event, int flags) 1229 { 1230 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu); 1231 struct cci_pmu_hw_events *hw_events = &cci_pmu->hw_events; 1232 struct hw_perf_event *hwc = &event->hw; 1233 int idx; 1234 int err = 0; 1235 1236 perf_pmu_disable(event->pmu); 1237 1238 /* If we don't have a space for the counter then finish early. */ 1239 idx = pmu_get_event_idx(hw_events, event); 1240 if (idx < 0) { 1241 err = idx; 1242 goto out; 1243 } 1244 1245 event->hw.idx = idx; 1246 hw_events->events[idx] = event; 1247 1248 hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE; 1249 if (flags & PERF_EF_START) 1250 cci_pmu_start(event, PERF_EF_RELOAD); 1251 1252 /* Propagate our changes to the userspace mapping. */ 1253 perf_event_update_userpage(event); 1254 1255 out: 1256 perf_pmu_enable(event->pmu); 1257 return err; 1258 } 1259 1260 static void cci_pmu_del(struct perf_event *event, int flags) 1261 { 1262 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu); 1263 struct cci_pmu_hw_events *hw_events = &cci_pmu->hw_events; 1264 struct hw_perf_event *hwc = &event->hw; 1265 int idx = hwc->idx; 1266 1267 cci_pmu_stop(event, PERF_EF_UPDATE); 1268 hw_events->events[idx] = NULL; 1269 clear_bit(idx, hw_events->used_mask); 1270 1271 perf_event_update_userpage(event); 1272 } 1273 1274 static int 1275 validate_event(struct pmu *cci_pmu, 1276 struct cci_pmu_hw_events *hw_events, 1277 struct perf_event *event) 1278 { 1279 if (is_software_event(event)) 1280 return 1; 1281 1282 /* 1283 * Reject groups spanning multiple HW PMUs (e.g. CPU + CCI). The 1284 * core perf code won't check that the pmu->ctx == leader->ctx 1285 * until after pmu->event_init(event). 1286 */ 1287 if (event->pmu != cci_pmu) 1288 return 0; 1289 1290 if (event->state < PERF_EVENT_STATE_OFF) 1291 return 1; 1292 1293 if (event->state == PERF_EVENT_STATE_OFF && !event->attr.enable_on_exec) 1294 return 1; 1295 1296 return pmu_get_event_idx(hw_events, event) >= 0; 1297 } 1298 1299 static int 1300 validate_group(struct perf_event *event) 1301 { 1302 struct perf_event *sibling, *leader = event->group_leader; 1303 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu); 1304 unsigned long mask[BITS_TO_LONGS(cci_pmu->num_cntrs)]; 1305 struct cci_pmu_hw_events fake_pmu = { 1306 /* 1307 * Initialise the fake PMU. We only need to populate the 1308 * used_mask for the purposes of validation. 1309 */ 1310 .used_mask = mask, 1311 }; 1312 memset(mask, 0, BITS_TO_LONGS(cci_pmu->num_cntrs) * sizeof(unsigned long)); 1313 1314 if (!validate_event(event->pmu, &fake_pmu, leader)) 1315 return -EINVAL; 1316 1317 list_for_each_entry(sibling, &leader->sibling_list, group_entry) { 1318 if (!validate_event(event->pmu, &fake_pmu, sibling)) 1319 return -EINVAL; 1320 } 1321 1322 if (!validate_event(event->pmu, &fake_pmu, event)) 1323 return -EINVAL; 1324 1325 return 0; 1326 } 1327 1328 static int 1329 __hw_perf_event_init(struct perf_event *event) 1330 { 1331 struct hw_perf_event *hwc = &event->hw; 1332 int mapping; 1333 1334 mapping = pmu_map_event(event); 1335 1336 if (mapping < 0) { 1337 pr_debug("event %x:%llx not supported\n", event->attr.type, 1338 event->attr.config); 1339 return mapping; 1340 } 1341 1342 /* 1343 * We don't assign an index until we actually place the event onto 1344 * hardware. Use -1 to signify that we haven't decided where to put it 1345 * yet. 1346 */ 1347 hwc->idx = -1; 1348 hwc->config_base = 0; 1349 hwc->config = 0; 1350 hwc->event_base = 0; 1351 1352 /* 1353 * Store the event encoding into the config_base field. 1354 */ 1355 hwc->config_base |= (unsigned long)mapping; 1356 1357 /* 1358 * Limit the sample_period to half of the counter width. That way, the 1359 * new counter value is far less likely to overtake the previous one 1360 * unless you have some serious IRQ latency issues. 1361 */ 1362 hwc->sample_period = CCI_PMU_CNTR_MASK >> 1; 1363 hwc->last_period = hwc->sample_period; 1364 local64_set(&hwc->period_left, hwc->sample_period); 1365 1366 if (event->group_leader != event) { 1367 if (validate_group(event) != 0) 1368 return -EINVAL; 1369 } 1370 1371 return 0; 1372 } 1373 1374 static int cci_pmu_event_init(struct perf_event *event) 1375 { 1376 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu); 1377 atomic_t *active_events = &cci_pmu->active_events; 1378 int err = 0; 1379 int cpu; 1380 1381 if (event->attr.type != event->pmu->type) 1382 return -ENOENT; 1383 1384 /* Shared by all CPUs, no meaningful state to sample */ 1385 if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK) 1386 return -EOPNOTSUPP; 1387 1388 /* We have no filtering of any kind */ 1389 if (event->attr.exclude_user || 1390 event->attr.exclude_kernel || 1391 event->attr.exclude_hv || 1392 event->attr.exclude_idle || 1393 event->attr.exclude_host || 1394 event->attr.exclude_guest) 1395 return -EINVAL; 1396 1397 /* 1398 * Following the example set by other "uncore" PMUs, we accept any CPU 1399 * and rewrite its affinity dynamically rather than having perf core 1400 * handle cpu == -1 and pid == -1 for this case. 1401 * 1402 * The perf core will pin online CPUs for the duration of this call and 1403 * the event being installed into its context, so the PMU's CPU can't 1404 * change under our feet. 1405 */ 1406 cpu = cpumask_first(&cci_pmu->cpus); 1407 if (event->cpu < 0 || cpu < 0) 1408 return -EINVAL; 1409 event->cpu = cpu; 1410 1411 event->destroy = hw_perf_event_destroy; 1412 if (!atomic_inc_not_zero(active_events)) { 1413 mutex_lock(&cci_pmu->reserve_mutex); 1414 if (atomic_read(active_events) == 0) 1415 err = cci_pmu_get_hw(cci_pmu); 1416 if (!err) 1417 atomic_inc(active_events); 1418 mutex_unlock(&cci_pmu->reserve_mutex); 1419 } 1420 if (err) 1421 return err; 1422 1423 err = __hw_perf_event_init(event); 1424 if (err) 1425 hw_perf_event_destroy(event); 1426 1427 return err; 1428 } 1429 1430 static ssize_t pmu_cpumask_attr_show(struct device *dev, 1431 struct device_attribute *attr, char *buf) 1432 { 1433 struct pmu *pmu = dev_get_drvdata(dev); 1434 struct cci_pmu *cci_pmu = to_cci_pmu(pmu); 1435 1436 int n = scnprintf(buf, PAGE_SIZE - 1, "%*pbl", 1437 cpumask_pr_args(&cci_pmu->cpus)); 1438 buf[n++] = '\n'; 1439 buf[n] = '\0'; 1440 return n; 1441 } 1442 1443 static struct device_attribute pmu_cpumask_attr = 1444 __ATTR(cpumask, S_IRUGO, pmu_cpumask_attr_show, NULL); 1445 1446 static struct attribute *pmu_attrs[] = { 1447 &pmu_cpumask_attr.attr, 1448 NULL, 1449 }; 1450 1451 static struct attribute_group pmu_attr_group = { 1452 .attrs = pmu_attrs, 1453 }; 1454 1455 static struct attribute_group pmu_format_attr_group = { 1456 .name = "format", 1457 .attrs = NULL, /* Filled in cci_pmu_init_attrs */ 1458 }; 1459 1460 static struct attribute_group pmu_event_attr_group = { 1461 .name = "events", 1462 .attrs = NULL, /* Filled in cci_pmu_init_attrs */ 1463 }; 1464 1465 static const struct attribute_group *pmu_attr_groups[] = { 1466 &pmu_attr_group, 1467 &pmu_format_attr_group, 1468 &pmu_event_attr_group, 1469 NULL 1470 }; 1471 1472 static int cci_pmu_init(struct cci_pmu *cci_pmu, struct platform_device *pdev) 1473 { 1474 const struct cci_pmu_model *model = cci_pmu->model; 1475 char *name = model->name; 1476 u32 num_cntrs; 1477 1478 pmu_event_attr_group.attrs = model->event_attrs; 1479 pmu_format_attr_group.attrs = model->format_attrs; 1480 1481 cci_pmu->pmu = (struct pmu) { 1482 .name = cci_pmu->model->name, 1483 .task_ctx_nr = perf_invalid_context, 1484 .pmu_enable = cci_pmu_enable, 1485 .pmu_disable = cci_pmu_disable, 1486 .event_init = cci_pmu_event_init, 1487 .add = cci_pmu_add, 1488 .del = cci_pmu_del, 1489 .start = cci_pmu_start, 1490 .stop = cci_pmu_stop, 1491 .read = pmu_read, 1492 .attr_groups = pmu_attr_groups, 1493 }; 1494 1495 cci_pmu->plat_device = pdev; 1496 num_cntrs = pmu_get_max_counters(); 1497 if (num_cntrs > cci_pmu->model->num_hw_cntrs) { 1498 dev_warn(&pdev->dev, 1499 "PMU implements more counters(%d) than supported by" 1500 " the model(%d), truncated.", 1501 num_cntrs, cci_pmu->model->num_hw_cntrs); 1502 num_cntrs = cci_pmu->model->num_hw_cntrs; 1503 } 1504 cci_pmu->num_cntrs = num_cntrs + cci_pmu->model->fixed_hw_cntrs; 1505 1506 return perf_pmu_register(&cci_pmu->pmu, name, -1); 1507 } 1508 1509 static int cci_pmu_offline_cpu(unsigned int cpu) 1510 { 1511 struct cci_pmu *cci_pmu; 1512 unsigned int target; 1513 1514 mutex_lock(&cci_pmu_mutex); 1515 list_for_each_entry(cci_pmu, &cci_pmu_list, entry) { 1516 if (!cpumask_test_and_clear_cpu(cpu, &cci_pmu->cpus)) 1517 continue; 1518 target = cpumask_any_but(cpu_online_mask, cpu); 1519 if (target >= nr_cpu_ids) 1520 continue; 1521 /* 1522 * TODO: migrate context once core races on event->ctx have 1523 * been fixed. 1524 */ 1525 cpumask_set_cpu(target, &cci_pmu->cpus); 1526 } 1527 mutex_unlock(&cci_pmu_mutex); 1528 return 0; 1529 } 1530 1531 static struct cci_pmu_model cci_pmu_models[] = { 1532 #ifdef CONFIG_ARM_CCI400_PMU 1533 [CCI400_R0] = { 1534 .name = "CCI_400", 1535 .fixed_hw_cntrs = 1, /* Cycle counter */ 1536 .num_hw_cntrs = 4, 1537 .cntr_size = SZ_4K, 1538 .format_attrs = cci400_pmu_format_attrs, 1539 .event_attrs = cci400_r0_pmu_event_attrs, 1540 .event_ranges = { 1541 [CCI_IF_SLAVE] = { 1542 CCI400_R0_SLAVE_PORT_MIN_EV, 1543 CCI400_R0_SLAVE_PORT_MAX_EV, 1544 }, 1545 [CCI_IF_MASTER] = { 1546 CCI400_R0_MASTER_PORT_MIN_EV, 1547 CCI400_R0_MASTER_PORT_MAX_EV, 1548 }, 1549 }, 1550 .validate_hw_event = cci400_validate_hw_event, 1551 .get_event_idx = cci400_get_event_idx, 1552 }, 1553 [CCI400_R1] = { 1554 .name = "CCI_400_r1", 1555 .fixed_hw_cntrs = 1, /* Cycle counter */ 1556 .num_hw_cntrs = 4, 1557 .cntr_size = SZ_4K, 1558 .format_attrs = cci400_pmu_format_attrs, 1559 .event_attrs = cci400_r1_pmu_event_attrs, 1560 .event_ranges = { 1561 [CCI_IF_SLAVE] = { 1562 CCI400_R1_SLAVE_PORT_MIN_EV, 1563 CCI400_R1_SLAVE_PORT_MAX_EV, 1564 }, 1565 [CCI_IF_MASTER] = { 1566 CCI400_R1_MASTER_PORT_MIN_EV, 1567 CCI400_R1_MASTER_PORT_MAX_EV, 1568 }, 1569 }, 1570 .validate_hw_event = cci400_validate_hw_event, 1571 .get_event_idx = cci400_get_event_idx, 1572 }, 1573 #endif 1574 #ifdef CONFIG_ARM_CCI5xx_PMU 1575 [CCI500_R0] = { 1576 .name = "CCI_500", 1577 .fixed_hw_cntrs = 0, 1578 .num_hw_cntrs = 8, 1579 .cntr_size = SZ_64K, 1580 .format_attrs = cci5xx_pmu_format_attrs, 1581 .event_attrs = cci5xx_pmu_event_attrs, 1582 .event_ranges = { 1583 [CCI_IF_SLAVE] = { 1584 CCI5xx_SLAVE_PORT_MIN_EV, 1585 CCI5xx_SLAVE_PORT_MAX_EV, 1586 }, 1587 [CCI_IF_MASTER] = { 1588 CCI5xx_MASTER_PORT_MIN_EV, 1589 CCI5xx_MASTER_PORT_MAX_EV, 1590 }, 1591 [CCI_IF_GLOBAL] = { 1592 CCI5xx_GLOBAL_PORT_MIN_EV, 1593 CCI5xx_GLOBAL_PORT_MAX_EV, 1594 }, 1595 }, 1596 .validate_hw_event = cci500_validate_hw_event, 1597 .write_counters = cci5xx_pmu_write_counters, 1598 }, 1599 [CCI550_R0] = { 1600 .name = "CCI_550", 1601 .fixed_hw_cntrs = 0, 1602 .num_hw_cntrs = 8, 1603 .cntr_size = SZ_64K, 1604 .format_attrs = cci5xx_pmu_format_attrs, 1605 .event_attrs = cci5xx_pmu_event_attrs, 1606 .event_ranges = { 1607 [CCI_IF_SLAVE] = { 1608 CCI5xx_SLAVE_PORT_MIN_EV, 1609 CCI5xx_SLAVE_PORT_MAX_EV, 1610 }, 1611 [CCI_IF_MASTER] = { 1612 CCI5xx_MASTER_PORT_MIN_EV, 1613 CCI5xx_MASTER_PORT_MAX_EV, 1614 }, 1615 [CCI_IF_GLOBAL] = { 1616 CCI5xx_GLOBAL_PORT_MIN_EV, 1617 CCI5xx_GLOBAL_PORT_MAX_EV, 1618 }, 1619 }, 1620 .validate_hw_event = cci550_validate_hw_event, 1621 .write_counters = cci5xx_pmu_write_counters, 1622 }, 1623 #endif 1624 }; 1625 1626 static const struct of_device_id arm_cci_pmu_matches[] = { 1627 #ifdef CONFIG_ARM_CCI400_PMU 1628 { 1629 .compatible = "arm,cci-400-pmu", 1630 .data = NULL, 1631 }, 1632 { 1633 .compatible = "arm,cci-400-pmu,r0", 1634 .data = &cci_pmu_models[CCI400_R0], 1635 }, 1636 { 1637 .compatible = "arm,cci-400-pmu,r1", 1638 .data = &cci_pmu_models[CCI400_R1], 1639 }, 1640 #endif 1641 #ifdef CONFIG_ARM_CCI5xx_PMU 1642 { 1643 .compatible = "arm,cci-500-pmu,r0", 1644 .data = &cci_pmu_models[CCI500_R0], 1645 }, 1646 { 1647 .compatible = "arm,cci-550-pmu,r0", 1648 .data = &cci_pmu_models[CCI550_R0], 1649 }, 1650 #endif 1651 {}, 1652 }; 1653 1654 static inline const struct cci_pmu_model *get_cci_model(struct platform_device *pdev) 1655 { 1656 const struct of_device_id *match = of_match_node(arm_cci_pmu_matches, 1657 pdev->dev.of_node); 1658 if (!match) 1659 return NULL; 1660 if (match->data) 1661 return match->data; 1662 1663 dev_warn(&pdev->dev, "DEPRECATED compatible property," 1664 "requires secure access to CCI registers"); 1665 return probe_cci_model(pdev); 1666 } 1667 1668 static bool is_duplicate_irq(int irq, int *irqs, int nr_irqs) 1669 { 1670 int i; 1671 1672 for (i = 0; i < nr_irqs; i++) 1673 if (irq == irqs[i]) 1674 return true; 1675 1676 return false; 1677 } 1678 1679 static struct cci_pmu *cci_pmu_alloc(struct platform_device *pdev) 1680 { 1681 struct cci_pmu *cci_pmu; 1682 const struct cci_pmu_model *model; 1683 1684 /* 1685 * All allocations are devm_* hence we don't have to free 1686 * them explicitly on an error, as it would end up in driver 1687 * detach. 1688 */ 1689 model = get_cci_model(pdev); 1690 if (!model) { 1691 dev_warn(&pdev->dev, "CCI PMU version not supported\n"); 1692 return ERR_PTR(-ENODEV); 1693 } 1694 1695 cci_pmu = devm_kzalloc(&pdev->dev, sizeof(*cci_pmu), GFP_KERNEL); 1696 if (!cci_pmu) 1697 return ERR_PTR(-ENOMEM); 1698 1699 cci_pmu->model = model; 1700 cci_pmu->irqs = devm_kcalloc(&pdev->dev, CCI_PMU_MAX_HW_CNTRS(model), 1701 sizeof(*cci_pmu->irqs), GFP_KERNEL); 1702 if (!cci_pmu->irqs) 1703 return ERR_PTR(-ENOMEM); 1704 cci_pmu->hw_events.events = devm_kcalloc(&pdev->dev, 1705 CCI_PMU_MAX_HW_CNTRS(model), 1706 sizeof(*cci_pmu->hw_events.events), 1707 GFP_KERNEL); 1708 if (!cci_pmu->hw_events.events) 1709 return ERR_PTR(-ENOMEM); 1710 cci_pmu->hw_events.used_mask = devm_kcalloc(&pdev->dev, 1711 BITS_TO_LONGS(CCI_PMU_MAX_HW_CNTRS(model)), 1712 sizeof(*cci_pmu->hw_events.used_mask), 1713 GFP_KERNEL); 1714 if (!cci_pmu->hw_events.used_mask) 1715 return ERR_PTR(-ENOMEM); 1716 1717 return cci_pmu; 1718 } 1719 1720 1721 static int cci_pmu_probe(struct platform_device *pdev) 1722 { 1723 struct resource *res; 1724 struct cci_pmu *cci_pmu; 1725 int i, ret, irq; 1726 1727 cci_pmu = cci_pmu_alloc(pdev); 1728 if (IS_ERR(cci_pmu)) 1729 return PTR_ERR(cci_pmu); 1730 1731 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 1732 cci_pmu->base = devm_ioremap_resource(&pdev->dev, res); 1733 if (IS_ERR(cci_pmu->base)) 1734 return -ENOMEM; 1735 1736 /* 1737 * CCI PMU has one overflow interrupt per counter; but some may be tied 1738 * together to a common interrupt. 1739 */ 1740 cci_pmu->nr_irqs = 0; 1741 for (i = 0; i < CCI_PMU_MAX_HW_CNTRS(cci_pmu->model); i++) { 1742 irq = platform_get_irq(pdev, i); 1743 if (irq < 0) 1744 break; 1745 1746 if (is_duplicate_irq(irq, cci_pmu->irqs, cci_pmu->nr_irqs)) 1747 continue; 1748 1749 cci_pmu->irqs[cci_pmu->nr_irqs++] = irq; 1750 } 1751 1752 /* 1753 * Ensure that the device tree has as many interrupts as the number 1754 * of counters. 1755 */ 1756 if (i < CCI_PMU_MAX_HW_CNTRS(cci_pmu->model)) { 1757 dev_warn(&pdev->dev, "In-correct number of interrupts: %d, should be %d\n", 1758 i, CCI_PMU_MAX_HW_CNTRS(cci_pmu->model)); 1759 return -EINVAL; 1760 } 1761 1762 raw_spin_lock_init(&cci_pmu->hw_events.pmu_lock); 1763 mutex_init(&cci_pmu->reserve_mutex); 1764 atomic_set(&cci_pmu->active_events, 0); 1765 cpumask_set_cpu(smp_processor_id(), &cci_pmu->cpus); 1766 1767 ret = cci_pmu_init(cci_pmu, pdev); 1768 if (ret) 1769 return ret; 1770 1771 mutex_lock(&cci_pmu_mutex); 1772 list_add(&cci_pmu->entry, &cci_pmu_list); 1773 mutex_unlock(&cci_pmu_mutex); 1774 1775 pr_info("ARM %s PMU driver probed", cci_pmu->model->name); 1776 return 0; 1777 } 1778 1779 static int cci_platform_probe(struct platform_device *pdev) 1780 { 1781 if (!cci_probed()) 1782 return -ENODEV; 1783 1784 return of_platform_populate(pdev->dev.of_node, NULL, NULL, &pdev->dev); 1785 } 1786 1787 static struct platform_driver cci_pmu_driver = { 1788 .driver = { 1789 .name = DRIVER_NAME_PMU, 1790 .of_match_table = arm_cci_pmu_matches, 1791 }, 1792 .probe = cci_pmu_probe, 1793 }; 1794 1795 static struct platform_driver cci_platform_driver = { 1796 .driver = { 1797 .name = DRIVER_NAME, 1798 .of_match_table = arm_cci_matches, 1799 }, 1800 .probe = cci_platform_probe, 1801 }; 1802 1803 static int __init cci_platform_init(void) 1804 { 1805 int ret; 1806 1807 ret = cpuhp_setup_state_nocalls(CPUHP_AP_PERF_ARM_CCI_ONLINE, 1808 "AP_PERF_ARM_CCI_ONLINE", NULL, 1809 cci_pmu_offline_cpu); 1810 if (ret) 1811 return ret; 1812 1813 ret = platform_driver_register(&cci_pmu_driver); 1814 if (ret) 1815 return ret; 1816 1817 return platform_driver_register(&cci_platform_driver); 1818 } 1819 1820 #else /* !CONFIG_ARM_CCI_PMU */ 1821 1822 static int __init cci_platform_init(void) 1823 { 1824 return 0; 1825 } 1826 1827 #endif /* CONFIG_ARM_CCI_PMU */ 1828 1829 #ifdef CONFIG_ARM_CCI400_PORT_CTRL 1830 1831 #define CCI_PORT_CTRL 0x0 1832 #define CCI_CTRL_STATUS 0xc 1833 1834 #define CCI_ENABLE_SNOOP_REQ 0x1 1835 #define CCI_ENABLE_DVM_REQ 0x2 1836 #define CCI_ENABLE_REQ (CCI_ENABLE_SNOOP_REQ | CCI_ENABLE_DVM_REQ) 1837 1838 enum cci_ace_port_type { 1839 ACE_INVALID_PORT = 0x0, 1840 ACE_PORT, 1841 ACE_LITE_PORT, 1842 }; 1843 1844 struct cci_ace_port { 1845 void __iomem *base; 1846 unsigned long phys; 1847 enum cci_ace_port_type type; 1848 struct device_node *dn; 1849 }; 1850 1851 static struct cci_ace_port *ports; 1852 static unsigned int nb_cci_ports; 1853 1854 struct cpu_port { 1855 u64 mpidr; 1856 u32 port; 1857 }; 1858 1859 /* 1860 * Use the port MSB as valid flag, shift can be made dynamic 1861 * by computing number of bits required for port indexes. 1862 * Code disabling CCI cpu ports runs with D-cache invalidated 1863 * and SCTLR bit clear so data accesses must be kept to a minimum 1864 * to improve performance; for now shift is left static to 1865 * avoid one more data access while disabling the CCI port. 1866 */ 1867 #define PORT_VALID_SHIFT 31 1868 #define PORT_VALID (0x1 << PORT_VALID_SHIFT) 1869 1870 static inline void init_cpu_port(struct cpu_port *port, u32 index, u64 mpidr) 1871 { 1872 port->port = PORT_VALID | index; 1873 port->mpidr = mpidr; 1874 } 1875 1876 static inline bool cpu_port_is_valid(struct cpu_port *port) 1877 { 1878 return !!(port->port & PORT_VALID); 1879 } 1880 1881 static inline bool cpu_port_match(struct cpu_port *port, u64 mpidr) 1882 { 1883 return port->mpidr == (mpidr & MPIDR_HWID_BITMASK); 1884 } 1885 1886 static struct cpu_port cpu_port[NR_CPUS]; 1887 1888 /** 1889 * __cci_ace_get_port - Function to retrieve the port index connected to 1890 * a cpu or device. 1891 * 1892 * @dn: device node of the device to look-up 1893 * @type: port type 1894 * 1895 * Return value: 1896 * - CCI port index if success 1897 * - -ENODEV if failure 1898 */ 1899 static int __cci_ace_get_port(struct device_node *dn, int type) 1900 { 1901 int i; 1902 bool ace_match; 1903 struct device_node *cci_portn; 1904 1905 cci_portn = of_parse_phandle(dn, "cci-control-port", 0); 1906 for (i = 0; i < nb_cci_ports; i++) { 1907 ace_match = ports[i].type == type; 1908 if (ace_match && cci_portn == ports[i].dn) 1909 return i; 1910 } 1911 return -ENODEV; 1912 } 1913 1914 int cci_ace_get_port(struct device_node *dn) 1915 { 1916 return __cci_ace_get_port(dn, ACE_LITE_PORT); 1917 } 1918 EXPORT_SYMBOL_GPL(cci_ace_get_port); 1919 1920 static void cci_ace_init_ports(void) 1921 { 1922 int port, cpu; 1923 struct device_node *cpun; 1924 1925 /* 1926 * Port index look-up speeds up the function disabling ports by CPU, 1927 * since the logical to port index mapping is done once and does 1928 * not change after system boot. 1929 * The stashed index array is initialized for all possible CPUs 1930 * at probe time. 1931 */ 1932 for_each_possible_cpu(cpu) { 1933 /* too early to use cpu->of_node */ 1934 cpun = of_get_cpu_node(cpu, NULL); 1935 1936 if (WARN(!cpun, "Missing cpu device node\n")) 1937 continue; 1938 1939 port = __cci_ace_get_port(cpun, ACE_PORT); 1940 if (port < 0) 1941 continue; 1942 1943 init_cpu_port(&cpu_port[cpu], port, cpu_logical_map(cpu)); 1944 } 1945 1946 for_each_possible_cpu(cpu) { 1947 WARN(!cpu_port_is_valid(&cpu_port[cpu]), 1948 "CPU %u does not have an associated CCI port\n", 1949 cpu); 1950 } 1951 } 1952 /* 1953 * Functions to enable/disable a CCI interconnect slave port 1954 * 1955 * They are called by low-level power management code to disable slave 1956 * interfaces snoops and DVM broadcast. 1957 * Since they may execute with cache data allocation disabled and 1958 * after the caches have been cleaned and invalidated the functions provide 1959 * no explicit locking since they may run with D-cache disabled, so normal 1960 * cacheable kernel locks based on ldrex/strex may not work. 1961 * Locking has to be provided by BSP implementations to ensure proper 1962 * operations. 1963 */ 1964 1965 /** 1966 * cci_port_control() - function to control a CCI port 1967 * 1968 * @port: index of the port to setup 1969 * @enable: if true enables the port, if false disables it 1970 */ 1971 static void notrace cci_port_control(unsigned int port, bool enable) 1972 { 1973 void __iomem *base = ports[port].base; 1974 1975 writel_relaxed(enable ? CCI_ENABLE_REQ : 0, base + CCI_PORT_CTRL); 1976 /* 1977 * This function is called from power down procedures 1978 * and must not execute any instruction that might 1979 * cause the processor to be put in a quiescent state 1980 * (eg wfi). Hence, cpu_relax() can not be added to this 1981 * read loop to optimize power, since it might hide possibly 1982 * disruptive operations. 1983 */ 1984 while (readl_relaxed(cci_ctrl_base + CCI_CTRL_STATUS) & 0x1) 1985 ; 1986 } 1987 1988 /** 1989 * cci_disable_port_by_cpu() - function to disable a CCI port by CPU 1990 * reference 1991 * 1992 * @mpidr: mpidr of the CPU whose CCI port should be disabled 1993 * 1994 * Disabling a CCI port for a CPU implies disabling the CCI port 1995 * controlling that CPU cluster. Code disabling CPU CCI ports 1996 * must make sure that the CPU running the code is the last active CPU 1997 * in the cluster ie all other CPUs are quiescent in a low power state. 1998 * 1999 * Return: 2000 * 0 on success 2001 * -ENODEV on port look-up failure 2002 */ 2003 int notrace cci_disable_port_by_cpu(u64 mpidr) 2004 { 2005 int cpu; 2006 bool is_valid; 2007 for (cpu = 0; cpu < nr_cpu_ids; cpu++) { 2008 is_valid = cpu_port_is_valid(&cpu_port[cpu]); 2009 if (is_valid && cpu_port_match(&cpu_port[cpu], mpidr)) { 2010 cci_port_control(cpu_port[cpu].port, false); 2011 return 0; 2012 } 2013 } 2014 return -ENODEV; 2015 } 2016 EXPORT_SYMBOL_GPL(cci_disable_port_by_cpu); 2017 2018 /** 2019 * cci_enable_port_for_self() - enable a CCI port for calling CPU 2020 * 2021 * Enabling a CCI port for the calling CPU implies enabling the CCI 2022 * port controlling that CPU's cluster. Caller must make sure that the 2023 * CPU running the code is the first active CPU in the cluster and all 2024 * other CPUs are quiescent in a low power state or waiting for this CPU 2025 * to complete the CCI initialization. 2026 * 2027 * Because this is called when the MMU is still off and with no stack, 2028 * the code must be position independent and ideally rely on callee 2029 * clobbered registers only. To achieve this we must code this function 2030 * entirely in assembler. 2031 * 2032 * On success this returns with the proper CCI port enabled. In case of 2033 * any failure this never returns as the inability to enable the CCI is 2034 * fatal and there is no possible recovery at this stage. 2035 */ 2036 asmlinkage void __naked cci_enable_port_for_self(void) 2037 { 2038 asm volatile ("\n" 2039 " .arch armv7-a\n" 2040 " mrc p15, 0, r0, c0, c0, 5 @ get MPIDR value \n" 2041 " and r0, r0, #"__stringify(MPIDR_HWID_BITMASK)" \n" 2042 " adr r1, 5f \n" 2043 " ldr r2, [r1] \n" 2044 " add r1, r1, r2 @ &cpu_port \n" 2045 " add ip, r1, %[sizeof_cpu_port] \n" 2046 2047 /* Loop over the cpu_port array looking for a matching MPIDR */ 2048 "1: ldr r2, [r1, %[offsetof_cpu_port_mpidr_lsb]] \n" 2049 " cmp r2, r0 @ compare MPIDR \n" 2050 " bne 2f \n" 2051 2052 /* Found a match, now test port validity */ 2053 " ldr r3, [r1, %[offsetof_cpu_port_port]] \n" 2054 " tst r3, #"__stringify(PORT_VALID)" \n" 2055 " bne 3f \n" 2056 2057 /* no match, loop with the next cpu_port entry */ 2058 "2: add r1, r1, %[sizeof_struct_cpu_port] \n" 2059 " cmp r1, ip @ done? \n" 2060 " blo 1b \n" 2061 2062 /* CCI port not found -- cheaply try to stall this CPU */ 2063 "cci_port_not_found: \n" 2064 " wfi \n" 2065 " wfe \n" 2066 " b cci_port_not_found \n" 2067 2068 /* Use matched port index to look up the corresponding ports entry */ 2069 "3: bic r3, r3, #"__stringify(PORT_VALID)" \n" 2070 " adr r0, 6f \n" 2071 " ldmia r0, {r1, r2} \n" 2072 " sub r1, r1, r0 @ virt - phys \n" 2073 " ldr r0, [r0, r2] @ *(&ports) \n" 2074 " mov r2, %[sizeof_struct_ace_port] \n" 2075 " mla r0, r2, r3, r0 @ &ports[index] \n" 2076 " sub r0, r0, r1 @ virt_to_phys() \n" 2077 2078 /* Enable the CCI port */ 2079 " ldr r0, [r0, %[offsetof_port_phys]] \n" 2080 " mov r3, %[cci_enable_req]\n" 2081 " str r3, [r0, #"__stringify(CCI_PORT_CTRL)"] \n" 2082 2083 /* poll the status reg for completion */ 2084 " adr r1, 7f \n" 2085 " ldr r0, [r1] \n" 2086 " ldr r0, [r0, r1] @ cci_ctrl_base \n" 2087 "4: ldr r1, [r0, #"__stringify(CCI_CTRL_STATUS)"] \n" 2088 " tst r1, %[cci_control_status_bits] \n" 2089 " bne 4b \n" 2090 2091 " mov r0, #0 \n" 2092 " bx lr \n" 2093 2094 " .align 2 \n" 2095 "5: .word cpu_port - . \n" 2096 "6: .word . \n" 2097 " .word ports - 6b \n" 2098 "7: .word cci_ctrl_phys - . \n" 2099 : : 2100 [sizeof_cpu_port] "i" (sizeof(cpu_port)), 2101 [cci_enable_req] "i" cpu_to_le32(CCI_ENABLE_REQ), 2102 [cci_control_status_bits] "i" cpu_to_le32(1), 2103 #ifndef __ARMEB__ 2104 [offsetof_cpu_port_mpidr_lsb] "i" (offsetof(struct cpu_port, mpidr)), 2105 #else 2106 [offsetof_cpu_port_mpidr_lsb] "i" (offsetof(struct cpu_port, mpidr)+4), 2107 #endif 2108 [offsetof_cpu_port_port] "i" (offsetof(struct cpu_port, port)), 2109 [sizeof_struct_cpu_port] "i" (sizeof(struct cpu_port)), 2110 [sizeof_struct_ace_port] "i" (sizeof(struct cci_ace_port)), 2111 [offsetof_port_phys] "i" (offsetof(struct cci_ace_port, phys)) ); 2112 2113 unreachable(); 2114 } 2115 2116 /** 2117 * __cci_control_port_by_device() - function to control a CCI port by device 2118 * reference 2119 * 2120 * @dn: device node pointer of the device whose CCI port should be 2121 * controlled 2122 * @enable: if true enables the port, if false disables it 2123 * 2124 * Return: 2125 * 0 on success 2126 * -ENODEV on port look-up failure 2127 */ 2128 int notrace __cci_control_port_by_device(struct device_node *dn, bool enable) 2129 { 2130 int port; 2131 2132 if (!dn) 2133 return -ENODEV; 2134 2135 port = __cci_ace_get_port(dn, ACE_LITE_PORT); 2136 if (WARN_ONCE(port < 0, "node %s ACE lite port look-up failure\n", 2137 dn->full_name)) 2138 return -ENODEV; 2139 cci_port_control(port, enable); 2140 return 0; 2141 } 2142 EXPORT_SYMBOL_GPL(__cci_control_port_by_device); 2143 2144 /** 2145 * __cci_control_port_by_index() - function to control a CCI port by port index 2146 * 2147 * @port: port index previously retrieved with cci_ace_get_port() 2148 * @enable: if true enables the port, if false disables it 2149 * 2150 * Return: 2151 * 0 on success 2152 * -ENODEV on port index out of range 2153 * -EPERM if operation carried out on an ACE PORT 2154 */ 2155 int notrace __cci_control_port_by_index(u32 port, bool enable) 2156 { 2157 if (port >= nb_cci_ports || ports[port].type == ACE_INVALID_PORT) 2158 return -ENODEV; 2159 /* 2160 * CCI control for ports connected to CPUS is extremely fragile 2161 * and must be made to go through a specific and controlled 2162 * interface (ie cci_disable_port_by_cpu(); control by general purpose 2163 * indexing is therefore disabled for ACE ports. 2164 */ 2165 if (ports[port].type == ACE_PORT) 2166 return -EPERM; 2167 2168 cci_port_control(port, enable); 2169 return 0; 2170 } 2171 EXPORT_SYMBOL_GPL(__cci_control_port_by_index); 2172 2173 static const struct of_device_id arm_cci_ctrl_if_matches[] = { 2174 {.compatible = "arm,cci-400-ctrl-if", }, 2175 {}, 2176 }; 2177 2178 static int cci_probe_ports(struct device_node *np) 2179 { 2180 struct cci_nb_ports const *cci_config; 2181 int ret, i, nb_ace = 0, nb_ace_lite = 0; 2182 struct device_node *cp; 2183 struct resource res; 2184 const char *match_str; 2185 bool is_ace; 2186 2187 2188 cci_config = of_match_node(arm_cci_matches, np)->data; 2189 if (!cci_config) 2190 return -ENODEV; 2191 2192 nb_cci_ports = cci_config->nb_ace + cci_config->nb_ace_lite; 2193 2194 ports = kcalloc(nb_cci_ports, sizeof(*ports), GFP_KERNEL); 2195 if (!ports) 2196 return -ENOMEM; 2197 2198 for_each_child_of_node(np, cp) { 2199 if (!of_match_node(arm_cci_ctrl_if_matches, cp)) 2200 continue; 2201 2202 i = nb_ace + nb_ace_lite; 2203 2204 if (i >= nb_cci_ports) 2205 break; 2206 2207 if (of_property_read_string(cp, "interface-type", 2208 &match_str)) { 2209 WARN(1, "node %s missing interface-type property\n", 2210 cp->full_name); 2211 continue; 2212 } 2213 is_ace = strcmp(match_str, "ace") == 0; 2214 if (!is_ace && strcmp(match_str, "ace-lite")) { 2215 WARN(1, "node %s containing invalid interface-type property, skipping it\n", 2216 cp->full_name); 2217 continue; 2218 } 2219 2220 ret = of_address_to_resource(cp, 0, &res); 2221 if (!ret) { 2222 ports[i].base = ioremap(res.start, resource_size(&res)); 2223 ports[i].phys = res.start; 2224 } 2225 if (ret || !ports[i].base) { 2226 WARN(1, "unable to ioremap CCI port %d\n", i); 2227 continue; 2228 } 2229 2230 if (is_ace) { 2231 if (WARN_ON(nb_ace >= cci_config->nb_ace)) 2232 continue; 2233 ports[i].type = ACE_PORT; 2234 ++nb_ace; 2235 } else { 2236 if (WARN_ON(nb_ace_lite >= cci_config->nb_ace_lite)) 2237 continue; 2238 ports[i].type = ACE_LITE_PORT; 2239 ++nb_ace_lite; 2240 } 2241 ports[i].dn = cp; 2242 } 2243 2244 /* initialize a stashed array of ACE ports to speed-up look-up */ 2245 cci_ace_init_ports(); 2246 2247 /* 2248 * Multi-cluster systems may need this data when non-coherent, during 2249 * cluster power-up/power-down. Make sure it reaches main memory. 2250 */ 2251 sync_cache_w(&cci_ctrl_base); 2252 sync_cache_w(&cci_ctrl_phys); 2253 sync_cache_w(&ports); 2254 sync_cache_w(&cpu_port); 2255 __sync_cache_range_w(ports, sizeof(*ports) * nb_cci_ports); 2256 pr_info("ARM CCI driver probed\n"); 2257 2258 return 0; 2259 } 2260 #else /* !CONFIG_ARM_CCI400_PORT_CTRL */ 2261 static inline int cci_probe_ports(struct device_node *np) 2262 { 2263 return 0; 2264 } 2265 #endif /* CONFIG_ARM_CCI400_PORT_CTRL */ 2266 2267 static int cci_probe(void) 2268 { 2269 int ret; 2270 struct device_node *np; 2271 struct resource res; 2272 2273 np = of_find_matching_node(NULL, arm_cci_matches); 2274 if(!np || !of_device_is_available(np)) 2275 return -ENODEV; 2276 2277 ret = of_address_to_resource(np, 0, &res); 2278 if (!ret) { 2279 cci_ctrl_base = ioremap(res.start, resource_size(&res)); 2280 cci_ctrl_phys = res.start; 2281 } 2282 if (ret || !cci_ctrl_base) { 2283 WARN(1, "unable to ioremap CCI ctrl\n"); 2284 return -ENXIO; 2285 } 2286 2287 return cci_probe_ports(np); 2288 } 2289 2290 static int cci_init_status = -EAGAIN; 2291 static DEFINE_MUTEX(cci_probing); 2292 2293 static int cci_init(void) 2294 { 2295 if (cci_init_status != -EAGAIN) 2296 return cci_init_status; 2297 2298 mutex_lock(&cci_probing); 2299 if (cci_init_status == -EAGAIN) 2300 cci_init_status = cci_probe(); 2301 mutex_unlock(&cci_probing); 2302 return cci_init_status; 2303 } 2304 2305 /* 2306 * To sort out early init calls ordering a helper function is provided to 2307 * check if the CCI driver has beed initialized. Function check if the driver 2308 * has been initialized, if not it calls the init function that probes 2309 * the driver and updates the return value. 2310 */ 2311 bool cci_probed(void) 2312 { 2313 return cci_init() == 0; 2314 } 2315 EXPORT_SYMBOL_GPL(cci_probed); 2316 2317 early_initcall(cci_init); 2318 core_initcall(cci_platform_init); 2319 MODULE_LICENSE("GPL"); 2320 MODULE_DESCRIPTION("ARM CCI support"); 2321