1 /* 2 * CCI cache coherent interconnect driver 3 * 4 * Copyright (C) 2013 ARM Ltd. 5 * Author: Lorenzo Pieralisi <lorenzo.pieralisi@arm.com> 6 * 7 * This program is free software; you can redistribute it and/or modify 8 * it under the terms of the GNU General Public License version 2 as 9 * published by the Free Software Foundation. 10 * 11 * This program is distributed "as is" WITHOUT ANY WARRANTY of any 12 * kind, whether express or implied; without even the implied warranty 13 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 * GNU General Public License for more details. 15 */ 16 17 #include <linux/arm-cci.h> 18 #include <linux/io.h> 19 #include <linux/module.h> 20 #include <linux/of_address.h> 21 #include <linux/of_irq.h> 22 #include <linux/of_platform.h> 23 #include <linux/platform_device.h> 24 #include <linux/slab.h> 25 #include <linux/spinlock.h> 26 27 #include <asm/cacheflush.h> 28 #include <asm/irq_regs.h> 29 #include <asm/pmu.h> 30 #include <asm/smp_plat.h> 31 32 #define DRIVER_NAME "CCI-400" 33 #define DRIVER_NAME_PMU DRIVER_NAME " PMU" 34 35 #define CCI_PORT_CTRL 0x0 36 #define CCI_CTRL_STATUS 0xc 37 38 #define CCI_ENABLE_SNOOP_REQ 0x1 39 #define CCI_ENABLE_DVM_REQ 0x2 40 #define CCI_ENABLE_REQ (CCI_ENABLE_SNOOP_REQ | CCI_ENABLE_DVM_REQ) 41 42 struct cci_nb_ports { 43 unsigned int nb_ace; 44 unsigned int nb_ace_lite; 45 }; 46 47 enum cci_ace_port_type { 48 ACE_INVALID_PORT = 0x0, 49 ACE_PORT, 50 ACE_LITE_PORT, 51 }; 52 53 struct cci_ace_port { 54 void __iomem *base; 55 unsigned long phys; 56 enum cci_ace_port_type type; 57 struct device_node *dn; 58 }; 59 60 static struct cci_ace_port *ports; 61 static unsigned int nb_cci_ports; 62 63 static void __iomem *cci_ctrl_base; 64 static unsigned long cci_ctrl_phys; 65 66 #ifdef CONFIG_HW_PERF_EVENTS 67 68 #define CCI_PMCR 0x0100 69 #define CCI_PID2 0x0fe8 70 71 #define CCI_PMCR_CEN 0x00000001 72 #define CCI_PMCR_NCNT_MASK 0x0000f800 73 #define CCI_PMCR_NCNT_SHIFT 11 74 75 #define CCI_PID2_REV_MASK 0xf0 76 #define CCI_PID2_REV_SHIFT 4 77 78 /* Port ids */ 79 #define CCI_PORT_S0 0 80 #define CCI_PORT_S1 1 81 #define CCI_PORT_S2 2 82 #define CCI_PORT_S3 3 83 #define CCI_PORT_S4 4 84 #define CCI_PORT_M0 5 85 #define CCI_PORT_M1 6 86 #define CCI_PORT_M2 7 87 88 #define CCI_REV_R0 0 89 #define CCI_REV_R1 1 90 #define CCI_REV_R1_PX 5 91 92 #define CCI_PMU_EVT_SEL 0x000 93 #define CCI_PMU_CNTR 0x004 94 #define CCI_PMU_CNTR_CTRL 0x008 95 #define CCI_PMU_OVRFLW 0x00c 96 97 #define CCI_PMU_OVRFLW_FLAG 1 98 99 #define CCI_PMU_CNTR_BASE(idx) ((idx) * SZ_4K) 100 101 /* 102 * Instead of an event id to monitor CCI cycles, a dedicated counter is 103 * provided. Use 0xff to represent CCI cycles and hope that no future revisions 104 * make use of this event in hardware. 105 */ 106 enum cci400_perf_events { 107 CCI_PMU_CYCLES = 0xff 108 }; 109 110 #define CCI_PMU_EVENT_MASK 0xff 111 #define CCI_PMU_EVENT_SOURCE(event) ((event >> 5) & 0x7) 112 #define CCI_PMU_EVENT_CODE(event) (event & 0x1f) 113 114 #define CCI_PMU_MAX_HW_EVENTS 5 /* CCI PMU has 4 counters + 1 cycle counter */ 115 116 #define CCI_PMU_CYCLE_CNTR_IDX 0 117 #define CCI_PMU_CNTR0_IDX 1 118 #define CCI_PMU_CNTR_LAST(cci_pmu) (CCI_PMU_CYCLE_CNTR_IDX + cci_pmu->num_events - 1) 119 120 /* 121 * CCI PMU event id is an 8-bit value made of two parts - bits 7:5 for one of 8 122 * ports and bits 4:0 are event codes. There are different event codes 123 * associated with each port type. 124 * 125 * Additionally, the range of events associated with the port types changed 126 * between Rev0 and Rev1. 127 * 128 * The constants below define the range of valid codes for each port type for 129 * the different revisions and are used to validate the event to be monitored. 130 */ 131 132 #define CCI_REV_R0_SLAVE_PORT_MIN_EV 0x00 133 #define CCI_REV_R0_SLAVE_PORT_MAX_EV 0x13 134 #define CCI_REV_R0_MASTER_PORT_MIN_EV 0x14 135 #define CCI_REV_R0_MASTER_PORT_MAX_EV 0x1a 136 137 #define CCI_REV_R1_SLAVE_PORT_MIN_EV 0x00 138 #define CCI_REV_R1_SLAVE_PORT_MAX_EV 0x14 139 #define CCI_REV_R1_MASTER_PORT_MIN_EV 0x00 140 #define CCI_REV_R1_MASTER_PORT_MAX_EV 0x11 141 142 struct pmu_port_event_ranges { 143 u8 slave_min; 144 u8 slave_max; 145 u8 master_min; 146 u8 master_max; 147 }; 148 149 static struct pmu_port_event_ranges port_event_range[] = { 150 [CCI_REV_R0] = { 151 .slave_min = CCI_REV_R0_SLAVE_PORT_MIN_EV, 152 .slave_max = CCI_REV_R0_SLAVE_PORT_MAX_EV, 153 .master_min = CCI_REV_R0_MASTER_PORT_MIN_EV, 154 .master_max = CCI_REV_R0_MASTER_PORT_MAX_EV, 155 }, 156 [CCI_REV_R1] = { 157 .slave_min = CCI_REV_R1_SLAVE_PORT_MIN_EV, 158 .slave_max = CCI_REV_R1_SLAVE_PORT_MAX_EV, 159 .master_min = CCI_REV_R1_MASTER_PORT_MIN_EV, 160 .master_max = CCI_REV_R1_MASTER_PORT_MAX_EV, 161 }, 162 }; 163 164 /* 165 * Export different PMU names for the different revisions so userspace knows 166 * because the event ids are different 167 */ 168 static char *const pmu_names[] = { 169 [CCI_REV_R0] = "CCI_400", 170 [CCI_REV_R1] = "CCI_400_r1", 171 }; 172 173 struct cci_pmu_drv_data { 174 void __iomem *base; 175 struct arm_pmu *cci_pmu; 176 int nr_irqs; 177 int irqs[CCI_PMU_MAX_HW_EVENTS]; 178 unsigned long active_irqs; 179 struct perf_event *events[CCI_PMU_MAX_HW_EVENTS]; 180 unsigned long used_mask[BITS_TO_LONGS(CCI_PMU_MAX_HW_EVENTS)]; 181 struct pmu_port_event_ranges *port_ranges; 182 struct pmu_hw_events hw_events; 183 }; 184 static struct cci_pmu_drv_data *pmu; 185 186 static bool is_duplicate_irq(int irq, int *irqs, int nr_irqs) 187 { 188 int i; 189 190 for (i = 0; i < nr_irqs; i++) 191 if (irq == irqs[i]) 192 return true; 193 194 return false; 195 } 196 197 static int probe_cci_revision(void) 198 { 199 int rev; 200 rev = readl_relaxed(cci_ctrl_base + CCI_PID2) & CCI_PID2_REV_MASK; 201 rev >>= CCI_PID2_REV_SHIFT; 202 203 if (rev < CCI_REV_R1_PX) 204 return CCI_REV_R0; 205 else 206 return CCI_REV_R1; 207 } 208 209 static struct pmu_port_event_ranges *port_range_by_rev(void) 210 { 211 int rev = probe_cci_revision(); 212 213 return &port_event_range[rev]; 214 } 215 216 static int pmu_is_valid_slave_event(u8 ev_code) 217 { 218 return pmu->port_ranges->slave_min <= ev_code && 219 ev_code <= pmu->port_ranges->slave_max; 220 } 221 222 static int pmu_is_valid_master_event(u8 ev_code) 223 { 224 return pmu->port_ranges->master_min <= ev_code && 225 ev_code <= pmu->port_ranges->master_max; 226 } 227 228 static int pmu_validate_hw_event(u8 hw_event) 229 { 230 u8 ev_source = CCI_PMU_EVENT_SOURCE(hw_event); 231 u8 ev_code = CCI_PMU_EVENT_CODE(hw_event); 232 233 switch (ev_source) { 234 case CCI_PORT_S0: 235 case CCI_PORT_S1: 236 case CCI_PORT_S2: 237 case CCI_PORT_S3: 238 case CCI_PORT_S4: 239 /* Slave Interface */ 240 if (pmu_is_valid_slave_event(ev_code)) 241 return hw_event; 242 break; 243 case CCI_PORT_M0: 244 case CCI_PORT_M1: 245 case CCI_PORT_M2: 246 /* Master Interface */ 247 if (pmu_is_valid_master_event(ev_code)) 248 return hw_event; 249 break; 250 } 251 252 return -ENOENT; 253 } 254 255 static int pmu_is_valid_counter(struct arm_pmu *cci_pmu, int idx) 256 { 257 return CCI_PMU_CYCLE_CNTR_IDX <= idx && 258 idx <= CCI_PMU_CNTR_LAST(cci_pmu); 259 } 260 261 static u32 pmu_read_register(int idx, unsigned int offset) 262 { 263 return readl_relaxed(pmu->base + CCI_PMU_CNTR_BASE(idx) + offset); 264 } 265 266 static void pmu_write_register(u32 value, int idx, unsigned int offset) 267 { 268 return writel_relaxed(value, pmu->base + CCI_PMU_CNTR_BASE(idx) + offset); 269 } 270 271 static void pmu_disable_counter(int idx) 272 { 273 pmu_write_register(0, idx, CCI_PMU_CNTR_CTRL); 274 } 275 276 static void pmu_enable_counter(int idx) 277 { 278 pmu_write_register(1, idx, CCI_PMU_CNTR_CTRL); 279 } 280 281 static void pmu_set_event(int idx, unsigned long event) 282 { 283 event &= CCI_PMU_EVENT_MASK; 284 pmu_write_register(event, idx, CCI_PMU_EVT_SEL); 285 } 286 287 static u32 pmu_get_max_counters(void) 288 { 289 u32 n_cnts = (readl_relaxed(cci_ctrl_base + CCI_PMCR) & 290 CCI_PMCR_NCNT_MASK) >> CCI_PMCR_NCNT_SHIFT; 291 292 /* add 1 for cycle counter */ 293 return n_cnts + 1; 294 } 295 296 static struct pmu_hw_events *pmu_get_hw_events(void) 297 { 298 return &pmu->hw_events; 299 } 300 301 static int pmu_get_event_idx(struct pmu_hw_events *hw, struct perf_event *event) 302 { 303 struct arm_pmu *cci_pmu = to_arm_pmu(event->pmu); 304 struct hw_perf_event *hw_event = &event->hw; 305 unsigned long cci_event = hw_event->config_base & CCI_PMU_EVENT_MASK; 306 int idx; 307 308 if (cci_event == CCI_PMU_CYCLES) { 309 if (test_and_set_bit(CCI_PMU_CYCLE_CNTR_IDX, hw->used_mask)) 310 return -EAGAIN; 311 312 return CCI_PMU_CYCLE_CNTR_IDX; 313 } 314 315 for (idx = CCI_PMU_CNTR0_IDX; idx <= CCI_PMU_CNTR_LAST(cci_pmu); ++idx) 316 if (!test_and_set_bit(idx, hw->used_mask)) 317 return idx; 318 319 /* No counters available */ 320 return -EAGAIN; 321 } 322 323 static int pmu_map_event(struct perf_event *event) 324 { 325 int mapping; 326 u8 config = event->attr.config & CCI_PMU_EVENT_MASK; 327 328 if (event->attr.type < PERF_TYPE_MAX) 329 return -ENOENT; 330 331 if (config == CCI_PMU_CYCLES) 332 mapping = config; 333 else 334 mapping = pmu_validate_hw_event(config); 335 336 return mapping; 337 } 338 339 static int pmu_request_irq(struct arm_pmu *cci_pmu, irq_handler_t handler) 340 { 341 int i; 342 struct platform_device *pmu_device = cci_pmu->plat_device; 343 344 if (unlikely(!pmu_device)) 345 return -ENODEV; 346 347 if (pmu->nr_irqs < 1) { 348 dev_err(&pmu_device->dev, "no irqs for CCI PMUs defined\n"); 349 return -ENODEV; 350 } 351 352 /* 353 * Register all available CCI PMU interrupts. In the interrupt handler 354 * we iterate over the counters checking for interrupt source (the 355 * overflowing counter) and clear it. 356 * 357 * This should allow handling of non-unique interrupt for the counters. 358 */ 359 for (i = 0; i < pmu->nr_irqs; i++) { 360 int err = request_irq(pmu->irqs[i], handler, IRQF_SHARED, 361 "arm-cci-pmu", cci_pmu); 362 if (err) { 363 dev_err(&pmu_device->dev, "unable to request IRQ%d for ARM CCI PMU counters\n", 364 pmu->irqs[i]); 365 return err; 366 } 367 368 set_bit(i, &pmu->active_irqs); 369 } 370 371 return 0; 372 } 373 374 static irqreturn_t pmu_handle_irq(int irq_num, void *dev) 375 { 376 unsigned long flags; 377 struct arm_pmu *cci_pmu = (struct arm_pmu *)dev; 378 struct pmu_hw_events *events = cci_pmu->get_hw_events(); 379 struct perf_sample_data data; 380 struct pt_regs *regs; 381 int idx, handled = IRQ_NONE; 382 383 raw_spin_lock_irqsave(&events->pmu_lock, flags); 384 regs = get_irq_regs(); 385 /* 386 * Iterate over counters and update the corresponding perf events. 387 * This should work regardless of whether we have per-counter overflow 388 * interrupt or a combined overflow interrupt. 389 */ 390 for (idx = CCI_PMU_CYCLE_CNTR_IDX; idx <= CCI_PMU_CNTR_LAST(cci_pmu); idx++) { 391 struct perf_event *event = events->events[idx]; 392 struct hw_perf_event *hw_counter; 393 394 if (!event) 395 continue; 396 397 hw_counter = &event->hw; 398 399 /* Did this counter overflow? */ 400 if (!pmu_read_register(idx, CCI_PMU_OVRFLW) & CCI_PMU_OVRFLW_FLAG) 401 continue; 402 403 pmu_write_register(CCI_PMU_OVRFLW_FLAG, idx, CCI_PMU_OVRFLW); 404 405 handled = IRQ_HANDLED; 406 407 armpmu_event_update(event); 408 perf_sample_data_init(&data, 0, hw_counter->last_period); 409 if (!armpmu_event_set_period(event)) 410 continue; 411 412 if (perf_event_overflow(event, &data, regs)) 413 cci_pmu->disable(event); 414 } 415 raw_spin_unlock_irqrestore(&events->pmu_lock, flags); 416 417 return IRQ_RETVAL(handled); 418 } 419 420 static void pmu_free_irq(struct arm_pmu *cci_pmu) 421 { 422 int i; 423 424 for (i = 0; i < pmu->nr_irqs; i++) { 425 if (!test_and_clear_bit(i, &pmu->active_irqs)) 426 continue; 427 428 free_irq(pmu->irqs[i], cci_pmu); 429 } 430 } 431 432 static void pmu_enable_event(struct perf_event *event) 433 { 434 unsigned long flags; 435 struct arm_pmu *cci_pmu = to_arm_pmu(event->pmu); 436 struct pmu_hw_events *events = cci_pmu->get_hw_events(); 437 struct hw_perf_event *hw_counter = &event->hw; 438 int idx = hw_counter->idx; 439 440 if (unlikely(!pmu_is_valid_counter(cci_pmu, idx))) { 441 dev_err(&cci_pmu->plat_device->dev, "Invalid CCI PMU counter %d\n", idx); 442 return; 443 } 444 445 raw_spin_lock_irqsave(&events->pmu_lock, flags); 446 447 /* Configure the event to count, unless you are counting cycles */ 448 if (idx != CCI_PMU_CYCLE_CNTR_IDX) 449 pmu_set_event(idx, hw_counter->config_base); 450 451 pmu_enable_counter(idx); 452 453 raw_spin_unlock_irqrestore(&events->pmu_lock, flags); 454 } 455 456 static void pmu_disable_event(struct perf_event *event) 457 { 458 struct arm_pmu *cci_pmu = to_arm_pmu(event->pmu); 459 struct hw_perf_event *hw_counter = &event->hw; 460 int idx = hw_counter->idx; 461 462 if (unlikely(!pmu_is_valid_counter(cci_pmu, idx))) { 463 dev_err(&cci_pmu->plat_device->dev, "Invalid CCI PMU counter %d\n", idx); 464 return; 465 } 466 467 pmu_disable_counter(idx); 468 } 469 470 static void pmu_start(struct arm_pmu *cci_pmu) 471 { 472 u32 val; 473 unsigned long flags; 474 struct pmu_hw_events *events = cci_pmu->get_hw_events(); 475 476 raw_spin_lock_irqsave(&events->pmu_lock, flags); 477 478 /* Enable all the PMU counters. */ 479 val = readl_relaxed(cci_ctrl_base + CCI_PMCR) | CCI_PMCR_CEN; 480 writel(val, cci_ctrl_base + CCI_PMCR); 481 482 raw_spin_unlock_irqrestore(&events->pmu_lock, flags); 483 } 484 485 static void pmu_stop(struct arm_pmu *cci_pmu) 486 { 487 u32 val; 488 unsigned long flags; 489 struct pmu_hw_events *events = cci_pmu->get_hw_events(); 490 491 raw_spin_lock_irqsave(&events->pmu_lock, flags); 492 493 /* Disable all the PMU counters. */ 494 val = readl_relaxed(cci_ctrl_base + CCI_PMCR) & ~CCI_PMCR_CEN; 495 writel(val, cci_ctrl_base + CCI_PMCR); 496 497 raw_spin_unlock_irqrestore(&events->pmu_lock, flags); 498 } 499 500 static u32 pmu_read_counter(struct perf_event *event) 501 { 502 struct arm_pmu *cci_pmu = to_arm_pmu(event->pmu); 503 struct hw_perf_event *hw_counter = &event->hw; 504 int idx = hw_counter->idx; 505 u32 value; 506 507 if (unlikely(!pmu_is_valid_counter(cci_pmu, idx))) { 508 dev_err(&cci_pmu->plat_device->dev, "Invalid CCI PMU counter %d\n", idx); 509 return 0; 510 } 511 value = pmu_read_register(idx, CCI_PMU_CNTR); 512 513 return value; 514 } 515 516 static void pmu_write_counter(struct perf_event *event, u32 value) 517 { 518 struct arm_pmu *cci_pmu = to_arm_pmu(event->pmu); 519 struct hw_perf_event *hw_counter = &event->hw; 520 int idx = hw_counter->idx; 521 522 if (unlikely(!pmu_is_valid_counter(cci_pmu, idx))) 523 dev_err(&cci_pmu->plat_device->dev, "Invalid CCI PMU counter %d\n", idx); 524 else 525 pmu_write_register(value, idx, CCI_PMU_CNTR); 526 } 527 528 static int cci_pmu_init(struct arm_pmu *cci_pmu, struct platform_device *pdev) 529 { 530 *cci_pmu = (struct arm_pmu){ 531 .name = pmu_names[probe_cci_revision()], 532 .max_period = (1LLU << 32) - 1, 533 .get_hw_events = pmu_get_hw_events, 534 .get_event_idx = pmu_get_event_idx, 535 .map_event = pmu_map_event, 536 .request_irq = pmu_request_irq, 537 .handle_irq = pmu_handle_irq, 538 .free_irq = pmu_free_irq, 539 .enable = pmu_enable_event, 540 .disable = pmu_disable_event, 541 .start = pmu_start, 542 .stop = pmu_stop, 543 .read_counter = pmu_read_counter, 544 .write_counter = pmu_write_counter, 545 }; 546 547 cci_pmu->plat_device = pdev; 548 cci_pmu->num_events = pmu_get_max_counters(); 549 550 return armpmu_register(cci_pmu, -1); 551 } 552 553 static const struct of_device_id arm_cci_pmu_matches[] = { 554 { 555 .compatible = "arm,cci-400-pmu", 556 }, 557 {}, 558 }; 559 560 static int cci_pmu_probe(struct platform_device *pdev) 561 { 562 struct resource *res; 563 int i, ret, irq; 564 565 pmu = devm_kzalloc(&pdev->dev, sizeof(*pmu), GFP_KERNEL); 566 if (!pmu) 567 return -ENOMEM; 568 569 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 570 pmu->base = devm_ioremap_resource(&pdev->dev, res); 571 if (IS_ERR(pmu->base)) 572 return -ENOMEM; 573 574 /* 575 * CCI PMU has 5 overflow signals - one per counter; but some may be tied 576 * together to a common interrupt. 577 */ 578 pmu->nr_irqs = 0; 579 for (i = 0; i < CCI_PMU_MAX_HW_EVENTS; i++) { 580 irq = platform_get_irq(pdev, i); 581 if (irq < 0) 582 break; 583 584 if (is_duplicate_irq(irq, pmu->irqs, pmu->nr_irqs)) 585 continue; 586 587 pmu->irqs[pmu->nr_irqs++] = irq; 588 } 589 590 /* 591 * Ensure that the device tree has as many interrupts as the number 592 * of counters. 593 */ 594 if (i < CCI_PMU_MAX_HW_EVENTS) { 595 dev_warn(&pdev->dev, "In-correct number of interrupts: %d, should be %d\n", 596 i, CCI_PMU_MAX_HW_EVENTS); 597 return -EINVAL; 598 } 599 600 pmu->port_ranges = port_range_by_rev(); 601 if (!pmu->port_ranges) { 602 dev_warn(&pdev->dev, "CCI PMU version not supported\n"); 603 return -EINVAL; 604 } 605 606 pmu->cci_pmu = devm_kzalloc(&pdev->dev, sizeof(*(pmu->cci_pmu)), GFP_KERNEL); 607 if (!pmu->cci_pmu) 608 return -ENOMEM; 609 610 pmu->hw_events.events = pmu->events; 611 pmu->hw_events.used_mask = pmu->used_mask; 612 raw_spin_lock_init(&pmu->hw_events.pmu_lock); 613 614 ret = cci_pmu_init(pmu->cci_pmu, pdev); 615 if (ret) 616 return ret; 617 618 return 0; 619 } 620 621 static int cci_platform_probe(struct platform_device *pdev) 622 { 623 if (!cci_probed()) 624 return -ENODEV; 625 626 return of_platform_populate(pdev->dev.of_node, NULL, NULL, &pdev->dev); 627 } 628 629 #endif /* CONFIG_HW_PERF_EVENTS */ 630 631 struct cpu_port { 632 u64 mpidr; 633 u32 port; 634 }; 635 636 /* 637 * Use the port MSB as valid flag, shift can be made dynamic 638 * by computing number of bits required for port indexes. 639 * Code disabling CCI cpu ports runs with D-cache invalidated 640 * and SCTLR bit clear so data accesses must be kept to a minimum 641 * to improve performance; for now shift is left static to 642 * avoid one more data access while disabling the CCI port. 643 */ 644 #define PORT_VALID_SHIFT 31 645 #define PORT_VALID (0x1 << PORT_VALID_SHIFT) 646 647 static inline void init_cpu_port(struct cpu_port *port, u32 index, u64 mpidr) 648 { 649 port->port = PORT_VALID | index; 650 port->mpidr = mpidr; 651 } 652 653 static inline bool cpu_port_is_valid(struct cpu_port *port) 654 { 655 return !!(port->port & PORT_VALID); 656 } 657 658 static inline bool cpu_port_match(struct cpu_port *port, u64 mpidr) 659 { 660 return port->mpidr == (mpidr & MPIDR_HWID_BITMASK); 661 } 662 663 static struct cpu_port cpu_port[NR_CPUS]; 664 665 /** 666 * __cci_ace_get_port - Function to retrieve the port index connected to 667 * a cpu or device. 668 * 669 * @dn: device node of the device to look-up 670 * @type: port type 671 * 672 * Return value: 673 * - CCI port index if success 674 * - -ENODEV if failure 675 */ 676 static int __cci_ace_get_port(struct device_node *dn, int type) 677 { 678 int i; 679 bool ace_match; 680 struct device_node *cci_portn; 681 682 cci_portn = of_parse_phandle(dn, "cci-control-port", 0); 683 for (i = 0; i < nb_cci_ports; i++) { 684 ace_match = ports[i].type == type; 685 if (ace_match && cci_portn == ports[i].dn) 686 return i; 687 } 688 return -ENODEV; 689 } 690 691 int cci_ace_get_port(struct device_node *dn) 692 { 693 return __cci_ace_get_port(dn, ACE_LITE_PORT); 694 } 695 EXPORT_SYMBOL_GPL(cci_ace_get_port); 696 697 static void cci_ace_init_ports(void) 698 { 699 int port, cpu; 700 struct device_node *cpun; 701 702 /* 703 * Port index look-up speeds up the function disabling ports by CPU, 704 * since the logical to port index mapping is done once and does 705 * not change after system boot. 706 * The stashed index array is initialized for all possible CPUs 707 * at probe time. 708 */ 709 for_each_possible_cpu(cpu) { 710 /* too early to use cpu->of_node */ 711 cpun = of_get_cpu_node(cpu, NULL); 712 713 if (WARN(!cpun, "Missing cpu device node\n")) 714 continue; 715 716 port = __cci_ace_get_port(cpun, ACE_PORT); 717 if (port < 0) 718 continue; 719 720 init_cpu_port(&cpu_port[cpu], port, cpu_logical_map(cpu)); 721 } 722 723 for_each_possible_cpu(cpu) { 724 WARN(!cpu_port_is_valid(&cpu_port[cpu]), 725 "CPU %u does not have an associated CCI port\n", 726 cpu); 727 } 728 } 729 /* 730 * Functions to enable/disable a CCI interconnect slave port 731 * 732 * They are called by low-level power management code to disable slave 733 * interfaces snoops and DVM broadcast. 734 * Since they may execute with cache data allocation disabled and 735 * after the caches have been cleaned and invalidated the functions provide 736 * no explicit locking since they may run with D-cache disabled, so normal 737 * cacheable kernel locks based on ldrex/strex may not work. 738 * Locking has to be provided by BSP implementations to ensure proper 739 * operations. 740 */ 741 742 /** 743 * cci_port_control() - function to control a CCI port 744 * 745 * @port: index of the port to setup 746 * @enable: if true enables the port, if false disables it 747 */ 748 static void notrace cci_port_control(unsigned int port, bool enable) 749 { 750 void __iomem *base = ports[port].base; 751 752 writel_relaxed(enable ? CCI_ENABLE_REQ : 0, base + CCI_PORT_CTRL); 753 /* 754 * This function is called from power down procedures 755 * and must not execute any instruction that might 756 * cause the processor to be put in a quiescent state 757 * (eg wfi). Hence, cpu_relax() can not be added to this 758 * read loop to optimize power, since it might hide possibly 759 * disruptive operations. 760 */ 761 while (readl_relaxed(cci_ctrl_base + CCI_CTRL_STATUS) & 0x1) 762 ; 763 } 764 765 /** 766 * cci_disable_port_by_cpu() - function to disable a CCI port by CPU 767 * reference 768 * 769 * @mpidr: mpidr of the CPU whose CCI port should be disabled 770 * 771 * Disabling a CCI port for a CPU implies disabling the CCI port 772 * controlling that CPU cluster. Code disabling CPU CCI ports 773 * must make sure that the CPU running the code is the last active CPU 774 * in the cluster ie all other CPUs are quiescent in a low power state. 775 * 776 * Return: 777 * 0 on success 778 * -ENODEV on port look-up failure 779 */ 780 int notrace cci_disable_port_by_cpu(u64 mpidr) 781 { 782 int cpu; 783 bool is_valid; 784 for (cpu = 0; cpu < nr_cpu_ids; cpu++) { 785 is_valid = cpu_port_is_valid(&cpu_port[cpu]); 786 if (is_valid && cpu_port_match(&cpu_port[cpu], mpidr)) { 787 cci_port_control(cpu_port[cpu].port, false); 788 return 0; 789 } 790 } 791 return -ENODEV; 792 } 793 EXPORT_SYMBOL_GPL(cci_disable_port_by_cpu); 794 795 /** 796 * cci_enable_port_for_self() - enable a CCI port for calling CPU 797 * 798 * Enabling a CCI port for the calling CPU implies enabling the CCI 799 * port controlling that CPU's cluster. Caller must make sure that the 800 * CPU running the code is the first active CPU in the cluster and all 801 * other CPUs are quiescent in a low power state or waiting for this CPU 802 * to complete the CCI initialization. 803 * 804 * Because this is called when the MMU is still off and with no stack, 805 * the code must be position independent and ideally rely on callee 806 * clobbered registers only. To achieve this we must code this function 807 * entirely in assembler. 808 * 809 * On success this returns with the proper CCI port enabled. In case of 810 * any failure this never returns as the inability to enable the CCI is 811 * fatal and there is no possible recovery at this stage. 812 */ 813 asmlinkage void __naked cci_enable_port_for_self(void) 814 { 815 asm volatile ("\n" 816 " .arch armv7-a\n" 817 " mrc p15, 0, r0, c0, c0, 5 @ get MPIDR value \n" 818 " and r0, r0, #"__stringify(MPIDR_HWID_BITMASK)" \n" 819 " adr r1, 5f \n" 820 " ldr r2, [r1] \n" 821 " add r1, r1, r2 @ &cpu_port \n" 822 " add ip, r1, %[sizeof_cpu_port] \n" 823 824 /* Loop over the cpu_port array looking for a matching MPIDR */ 825 "1: ldr r2, [r1, %[offsetof_cpu_port_mpidr_lsb]] \n" 826 " cmp r2, r0 @ compare MPIDR \n" 827 " bne 2f \n" 828 829 /* Found a match, now test port validity */ 830 " ldr r3, [r1, %[offsetof_cpu_port_port]] \n" 831 " tst r3, #"__stringify(PORT_VALID)" \n" 832 " bne 3f \n" 833 834 /* no match, loop with the next cpu_port entry */ 835 "2: add r1, r1, %[sizeof_struct_cpu_port] \n" 836 " cmp r1, ip @ done? \n" 837 " blo 1b \n" 838 839 /* CCI port not found -- cheaply try to stall this CPU */ 840 "cci_port_not_found: \n" 841 " wfi \n" 842 " wfe \n" 843 " b cci_port_not_found \n" 844 845 /* Use matched port index to look up the corresponding ports entry */ 846 "3: bic r3, r3, #"__stringify(PORT_VALID)" \n" 847 " adr r0, 6f \n" 848 " ldmia r0, {r1, r2} \n" 849 " sub r1, r1, r0 @ virt - phys \n" 850 " ldr r0, [r0, r2] @ *(&ports) \n" 851 " mov r2, %[sizeof_struct_ace_port] \n" 852 " mla r0, r2, r3, r0 @ &ports[index] \n" 853 " sub r0, r0, r1 @ virt_to_phys() \n" 854 855 /* Enable the CCI port */ 856 " ldr r0, [r0, %[offsetof_port_phys]] \n" 857 " mov r3, %[cci_enable_req]\n" 858 " str r3, [r0, #"__stringify(CCI_PORT_CTRL)"] \n" 859 860 /* poll the status reg for completion */ 861 " adr r1, 7f \n" 862 " ldr r0, [r1] \n" 863 " ldr r0, [r0, r1] @ cci_ctrl_base \n" 864 "4: ldr r1, [r0, #"__stringify(CCI_CTRL_STATUS)"] \n" 865 " tst r1, %[cci_control_status_bits] \n" 866 " bne 4b \n" 867 868 " mov r0, #0 \n" 869 " bx lr \n" 870 871 " .align 2 \n" 872 "5: .word cpu_port - . \n" 873 "6: .word . \n" 874 " .word ports - 6b \n" 875 "7: .word cci_ctrl_phys - . \n" 876 : : 877 [sizeof_cpu_port] "i" (sizeof(cpu_port)), 878 [cci_enable_req] "i" cpu_to_le32(CCI_ENABLE_REQ), 879 [cci_control_status_bits] "i" cpu_to_le32(1), 880 #ifndef __ARMEB__ 881 [offsetof_cpu_port_mpidr_lsb] "i" (offsetof(struct cpu_port, mpidr)), 882 #else 883 [offsetof_cpu_port_mpidr_lsb] "i" (offsetof(struct cpu_port, mpidr)+4), 884 #endif 885 [offsetof_cpu_port_port] "i" (offsetof(struct cpu_port, port)), 886 [sizeof_struct_cpu_port] "i" (sizeof(struct cpu_port)), 887 [sizeof_struct_ace_port] "i" (sizeof(struct cci_ace_port)), 888 [offsetof_port_phys] "i" (offsetof(struct cci_ace_port, phys)) ); 889 890 unreachable(); 891 } 892 893 /** 894 * __cci_control_port_by_device() - function to control a CCI port by device 895 * reference 896 * 897 * @dn: device node pointer of the device whose CCI port should be 898 * controlled 899 * @enable: if true enables the port, if false disables it 900 * 901 * Return: 902 * 0 on success 903 * -ENODEV on port look-up failure 904 */ 905 int notrace __cci_control_port_by_device(struct device_node *dn, bool enable) 906 { 907 int port; 908 909 if (!dn) 910 return -ENODEV; 911 912 port = __cci_ace_get_port(dn, ACE_LITE_PORT); 913 if (WARN_ONCE(port < 0, "node %s ACE lite port look-up failure\n", 914 dn->full_name)) 915 return -ENODEV; 916 cci_port_control(port, enable); 917 return 0; 918 } 919 EXPORT_SYMBOL_GPL(__cci_control_port_by_device); 920 921 /** 922 * __cci_control_port_by_index() - function to control a CCI port by port index 923 * 924 * @port: port index previously retrieved with cci_ace_get_port() 925 * @enable: if true enables the port, if false disables it 926 * 927 * Return: 928 * 0 on success 929 * -ENODEV on port index out of range 930 * -EPERM if operation carried out on an ACE PORT 931 */ 932 int notrace __cci_control_port_by_index(u32 port, bool enable) 933 { 934 if (port >= nb_cci_ports || ports[port].type == ACE_INVALID_PORT) 935 return -ENODEV; 936 /* 937 * CCI control for ports connected to CPUS is extremely fragile 938 * and must be made to go through a specific and controlled 939 * interface (ie cci_disable_port_by_cpu(); control by general purpose 940 * indexing is therefore disabled for ACE ports. 941 */ 942 if (ports[port].type == ACE_PORT) 943 return -EPERM; 944 945 cci_port_control(port, enable); 946 return 0; 947 } 948 EXPORT_SYMBOL_GPL(__cci_control_port_by_index); 949 950 static const struct cci_nb_ports cci400_ports = { 951 .nb_ace = 2, 952 .nb_ace_lite = 3 953 }; 954 955 static const struct of_device_id arm_cci_matches[] = { 956 {.compatible = "arm,cci-400", .data = &cci400_ports }, 957 {}, 958 }; 959 960 static const struct of_device_id arm_cci_ctrl_if_matches[] = { 961 {.compatible = "arm,cci-400-ctrl-if", }, 962 {}, 963 }; 964 965 static int cci_probe(void) 966 { 967 struct cci_nb_ports const *cci_config; 968 int ret, i, nb_ace = 0, nb_ace_lite = 0; 969 struct device_node *np, *cp; 970 struct resource res; 971 const char *match_str; 972 bool is_ace; 973 974 np = of_find_matching_node(NULL, arm_cci_matches); 975 if (!np) 976 return -ENODEV; 977 978 cci_config = of_match_node(arm_cci_matches, np)->data; 979 if (!cci_config) 980 return -ENODEV; 981 982 nb_cci_ports = cci_config->nb_ace + cci_config->nb_ace_lite; 983 984 ports = kcalloc(nb_cci_ports, sizeof(*ports), GFP_KERNEL); 985 if (!ports) 986 return -ENOMEM; 987 988 ret = of_address_to_resource(np, 0, &res); 989 if (!ret) { 990 cci_ctrl_base = ioremap(res.start, resource_size(&res)); 991 cci_ctrl_phys = res.start; 992 } 993 if (ret || !cci_ctrl_base) { 994 WARN(1, "unable to ioremap CCI ctrl\n"); 995 ret = -ENXIO; 996 goto memalloc_err; 997 } 998 999 for_each_child_of_node(np, cp) { 1000 if (!of_match_node(arm_cci_ctrl_if_matches, cp)) 1001 continue; 1002 1003 i = nb_ace + nb_ace_lite; 1004 1005 if (i >= nb_cci_ports) 1006 break; 1007 1008 if (of_property_read_string(cp, "interface-type", 1009 &match_str)) { 1010 WARN(1, "node %s missing interface-type property\n", 1011 cp->full_name); 1012 continue; 1013 } 1014 is_ace = strcmp(match_str, "ace") == 0; 1015 if (!is_ace && strcmp(match_str, "ace-lite")) { 1016 WARN(1, "node %s containing invalid interface-type property, skipping it\n", 1017 cp->full_name); 1018 continue; 1019 } 1020 1021 ret = of_address_to_resource(cp, 0, &res); 1022 if (!ret) { 1023 ports[i].base = ioremap(res.start, resource_size(&res)); 1024 ports[i].phys = res.start; 1025 } 1026 if (ret || !ports[i].base) { 1027 WARN(1, "unable to ioremap CCI port %d\n", i); 1028 continue; 1029 } 1030 1031 if (is_ace) { 1032 if (WARN_ON(nb_ace >= cci_config->nb_ace)) 1033 continue; 1034 ports[i].type = ACE_PORT; 1035 ++nb_ace; 1036 } else { 1037 if (WARN_ON(nb_ace_lite >= cci_config->nb_ace_lite)) 1038 continue; 1039 ports[i].type = ACE_LITE_PORT; 1040 ++nb_ace_lite; 1041 } 1042 ports[i].dn = cp; 1043 } 1044 1045 /* initialize a stashed array of ACE ports to speed-up look-up */ 1046 cci_ace_init_ports(); 1047 1048 /* 1049 * Multi-cluster systems may need this data when non-coherent, during 1050 * cluster power-up/power-down. Make sure it reaches main memory. 1051 */ 1052 sync_cache_w(&cci_ctrl_base); 1053 sync_cache_w(&cci_ctrl_phys); 1054 sync_cache_w(&ports); 1055 sync_cache_w(&cpu_port); 1056 __sync_cache_range_w(ports, sizeof(*ports) * nb_cci_ports); 1057 pr_info("ARM CCI driver probed\n"); 1058 return 0; 1059 1060 memalloc_err: 1061 1062 kfree(ports); 1063 return ret; 1064 } 1065 1066 static int cci_init_status = -EAGAIN; 1067 static DEFINE_MUTEX(cci_probing); 1068 1069 static int cci_init(void) 1070 { 1071 if (cci_init_status != -EAGAIN) 1072 return cci_init_status; 1073 1074 mutex_lock(&cci_probing); 1075 if (cci_init_status == -EAGAIN) 1076 cci_init_status = cci_probe(); 1077 mutex_unlock(&cci_probing); 1078 return cci_init_status; 1079 } 1080 1081 #ifdef CONFIG_HW_PERF_EVENTS 1082 static struct platform_driver cci_pmu_driver = { 1083 .driver = { 1084 .name = DRIVER_NAME_PMU, 1085 .of_match_table = arm_cci_pmu_matches, 1086 }, 1087 .probe = cci_pmu_probe, 1088 }; 1089 1090 static struct platform_driver cci_platform_driver = { 1091 .driver = { 1092 .name = DRIVER_NAME, 1093 .of_match_table = arm_cci_matches, 1094 }, 1095 .probe = cci_platform_probe, 1096 }; 1097 1098 static int __init cci_platform_init(void) 1099 { 1100 int ret; 1101 1102 ret = platform_driver_register(&cci_pmu_driver); 1103 if (ret) 1104 return ret; 1105 1106 return platform_driver_register(&cci_platform_driver); 1107 } 1108 1109 #else 1110 1111 static int __init cci_platform_init(void) 1112 { 1113 return 0; 1114 } 1115 1116 #endif 1117 /* 1118 * To sort out early init calls ordering a helper function is provided to 1119 * check if the CCI driver has beed initialized. Function check if the driver 1120 * has been initialized, if not it calls the init function that probes 1121 * the driver and updates the return value. 1122 */ 1123 bool cci_probed(void) 1124 { 1125 return cci_init() == 0; 1126 } 1127 EXPORT_SYMBOL_GPL(cci_probed); 1128 1129 early_initcall(cci_init); 1130 core_initcall(cci_platform_init); 1131 MODULE_LICENSE("GPL"); 1132 MODULE_DESCRIPTION("ARM CCI support"); 1133