1 /* 2 * arch/arm/mm/cache-l2x0.c - L210/L220/L310 cache controller support 3 * 4 * Copyright (C) 2007 ARM Limited 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 * 10 * This program is distributed in the hope that it will be useful, 11 * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 * GNU General Public License for more details. 14 * 15 * You should have received a copy of the GNU General Public License 16 * along with this program; if not, write to the Free Software 17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 18 */ 19 #include <linux/cpu.h> 20 #include <linux/err.h> 21 #include <linux/init.h> 22 #include <linux/smp.h> 23 #include <linux/spinlock.h> 24 #include <linux/log2.h> 25 #include <linux/io.h> 26 #include <linux/of.h> 27 #include <linux/of_address.h> 28 29 #include <asm/cacheflush.h> 30 #include <asm/cp15.h> 31 #include <asm/cputype.h> 32 #include <asm/hardware/cache-l2x0.h> 33 #include "cache-tauros3.h" 34 #include "cache-aurora-l2.h" 35 36 struct l2c_init_data { 37 const char *type; 38 unsigned way_size_0; 39 unsigned num_lock; 40 void (*of_parse)(const struct device_node *, u32 *, u32 *); 41 void (*enable)(void __iomem *, unsigned); 42 void (*fixup)(void __iomem *, u32, struct outer_cache_fns *); 43 void (*save)(void __iomem *); 44 void (*configure)(void __iomem *); 45 void (*unlock)(void __iomem *, unsigned); 46 struct outer_cache_fns outer_cache; 47 }; 48 49 #define CACHE_LINE_SIZE 32 50 51 static void __iomem *l2x0_base; 52 static const struct l2c_init_data *l2x0_data; 53 static DEFINE_RAW_SPINLOCK(l2x0_lock); 54 static u32 l2x0_way_mask; /* Bitmask of active ways */ 55 static u32 l2x0_size; 56 static unsigned long sync_reg_offset = L2X0_CACHE_SYNC; 57 58 struct l2x0_regs l2x0_saved_regs; 59 60 /* 61 * Common code for all cache controllers. 62 */ 63 static inline void l2c_wait_mask(void __iomem *reg, unsigned long mask) 64 { 65 /* wait for cache operation by line or way to complete */ 66 while (readl_relaxed(reg) & mask) 67 cpu_relax(); 68 } 69 70 /* 71 * By default, we write directly to secure registers. Platforms must 72 * override this if they are running non-secure. 73 */ 74 static void l2c_write_sec(unsigned long val, void __iomem *base, unsigned reg) 75 { 76 if (val == readl_relaxed(base + reg)) 77 return; 78 if (outer_cache.write_sec) 79 outer_cache.write_sec(val, reg); 80 else 81 writel_relaxed(val, base + reg); 82 } 83 84 /* 85 * This should only be called when we have a requirement that the 86 * register be written due to a work-around, as platforms running 87 * in non-secure mode may not be able to access this register. 88 */ 89 static inline void l2c_set_debug(void __iomem *base, unsigned long val) 90 { 91 l2c_write_sec(val, base, L2X0_DEBUG_CTRL); 92 } 93 94 static void __l2c_op_way(void __iomem *reg) 95 { 96 writel_relaxed(l2x0_way_mask, reg); 97 l2c_wait_mask(reg, l2x0_way_mask); 98 } 99 100 static inline void l2c_unlock(void __iomem *base, unsigned num) 101 { 102 unsigned i; 103 104 for (i = 0; i < num; i++) { 105 writel_relaxed(0, base + L2X0_LOCKDOWN_WAY_D_BASE + 106 i * L2X0_LOCKDOWN_STRIDE); 107 writel_relaxed(0, base + L2X0_LOCKDOWN_WAY_I_BASE + 108 i * L2X0_LOCKDOWN_STRIDE); 109 } 110 } 111 112 static void l2c_configure(void __iomem *base) 113 { 114 l2c_write_sec(l2x0_saved_regs.aux_ctrl, base, L2X0_AUX_CTRL); 115 } 116 117 /* 118 * Enable the L2 cache controller. This function must only be 119 * called when the cache controller is known to be disabled. 120 */ 121 static void l2c_enable(void __iomem *base, unsigned num_lock) 122 { 123 unsigned long flags; 124 125 if (outer_cache.configure) 126 outer_cache.configure(&l2x0_saved_regs); 127 else 128 l2x0_data->configure(base); 129 130 l2x0_data->unlock(base, num_lock); 131 132 local_irq_save(flags); 133 __l2c_op_way(base + L2X0_INV_WAY); 134 writel_relaxed(0, base + sync_reg_offset); 135 l2c_wait_mask(base + sync_reg_offset, 1); 136 local_irq_restore(flags); 137 138 l2c_write_sec(L2X0_CTRL_EN, base, L2X0_CTRL); 139 } 140 141 static void l2c_disable(void) 142 { 143 void __iomem *base = l2x0_base; 144 145 outer_cache.flush_all(); 146 l2c_write_sec(0, base, L2X0_CTRL); 147 dsb(st); 148 } 149 150 static void l2c_save(void __iomem *base) 151 { 152 l2x0_saved_regs.aux_ctrl = readl_relaxed(l2x0_base + L2X0_AUX_CTRL); 153 } 154 155 static void l2c_resume(void) 156 { 157 void __iomem *base = l2x0_base; 158 159 /* Do not touch the controller if already enabled. */ 160 if (!(readl_relaxed(base + L2X0_CTRL) & L2X0_CTRL_EN)) 161 l2c_enable(base, l2x0_data->num_lock); 162 } 163 164 /* 165 * L2C-210 specific code. 166 * 167 * The L2C-2x0 PA, set/way and sync operations are atomic, but we must 168 * ensure that no background operation is running. The way operations 169 * are all background tasks. 170 * 171 * While a background operation is in progress, any new operation is 172 * ignored (unspecified whether this causes an error.) Thankfully, not 173 * used on SMP. 174 * 175 * Never has a different sync register other than L2X0_CACHE_SYNC, but 176 * we use sync_reg_offset here so we can share some of this with L2C-310. 177 */ 178 static void __l2c210_cache_sync(void __iomem *base) 179 { 180 writel_relaxed(0, base + sync_reg_offset); 181 } 182 183 static void __l2c210_op_pa_range(void __iomem *reg, unsigned long start, 184 unsigned long end) 185 { 186 while (start < end) { 187 writel_relaxed(start, reg); 188 start += CACHE_LINE_SIZE; 189 } 190 } 191 192 static void l2c210_inv_range(unsigned long start, unsigned long end) 193 { 194 void __iomem *base = l2x0_base; 195 196 if (start & (CACHE_LINE_SIZE - 1)) { 197 start &= ~(CACHE_LINE_SIZE - 1); 198 writel_relaxed(start, base + L2X0_CLEAN_INV_LINE_PA); 199 start += CACHE_LINE_SIZE; 200 } 201 202 if (end & (CACHE_LINE_SIZE - 1)) { 203 end &= ~(CACHE_LINE_SIZE - 1); 204 writel_relaxed(end, base + L2X0_CLEAN_INV_LINE_PA); 205 } 206 207 __l2c210_op_pa_range(base + L2X0_INV_LINE_PA, start, end); 208 __l2c210_cache_sync(base); 209 } 210 211 static void l2c210_clean_range(unsigned long start, unsigned long end) 212 { 213 void __iomem *base = l2x0_base; 214 215 start &= ~(CACHE_LINE_SIZE - 1); 216 __l2c210_op_pa_range(base + L2X0_CLEAN_LINE_PA, start, end); 217 __l2c210_cache_sync(base); 218 } 219 220 static void l2c210_flush_range(unsigned long start, unsigned long end) 221 { 222 void __iomem *base = l2x0_base; 223 224 start &= ~(CACHE_LINE_SIZE - 1); 225 __l2c210_op_pa_range(base + L2X0_CLEAN_INV_LINE_PA, start, end); 226 __l2c210_cache_sync(base); 227 } 228 229 static void l2c210_flush_all(void) 230 { 231 void __iomem *base = l2x0_base; 232 233 BUG_ON(!irqs_disabled()); 234 235 __l2c_op_way(base + L2X0_CLEAN_INV_WAY); 236 __l2c210_cache_sync(base); 237 } 238 239 static void l2c210_sync(void) 240 { 241 __l2c210_cache_sync(l2x0_base); 242 } 243 244 static const struct l2c_init_data l2c210_data __initconst = { 245 .type = "L2C-210", 246 .way_size_0 = SZ_8K, 247 .num_lock = 1, 248 .enable = l2c_enable, 249 .save = l2c_save, 250 .configure = l2c_configure, 251 .unlock = l2c_unlock, 252 .outer_cache = { 253 .inv_range = l2c210_inv_range, 254 .clean_range = l2c210_clean_range, 255 .flush_range = l2c210_flush_range, 256 .flush_all = l2c210_flush_all, 257 .disable = l2c_disable, 258 .sync = l2c210_sync, 259 .resume = l2c_resume, 260 }, 261 }; 262 263 /* 264 * L2C-220 specific code. 265 * 266 * All operations are background operations: they have to be waited for. 267 * Conflicting requests generate a slave error (which will cause an 268 * imprecise abort.) Never uses sync_reg_offset, so we hard-code the 269 * sync register here. 270 * 271 * However, we can re-use the l2c210_resume call. 272 */ 273 static inline void __l2c220_cache_sync(void __iomem *base) 274 { 275 writel_relaxed(0, base + L2X0_CACHE_SYNC); 276 l2c_wait_mask(base + L2X0_CACHE_SYNC, 1); 277 } 278 279 static void l2c220_op_way(void __iomem *base, unsigned reg) 280 { 281 unsigned long flags; 282 283 raw_spin_lock_irqsave(&l2x0_lock, flags); 284 __l2c_op_way(base + reg); 285 __l2c220_cache_sync(base); 286 raw_spin_unlock_irqrestore(&l2x0_lock, flags); 287 } 288 289 static unsigned long l2c220_op_pa_range(void __iomem *reg, unsigned long start, 290 unsigned long end, unsigned long flags) 291 { 292 raw_spinlock_t *lock = &l2x0_lock; 293 294 while (start < end) { 295 unsigned long blk_end = start + min(end - start, 4096UL); 296 297 while (start < blk_end) { 298 l2c_wait_mask(reg, 1); 299 writel_relaxed(start, reg); 300 start += CACHE_LINE_SIZE; 301 } 302 303 if (blk_end < end) { 304 raw_spin_unlock_irqrestore(lock, flags); 305 raw_spin_lock_irqsave(lock, flags); 306 } 307 } 308 309 return flags; 310 } 311 312 static void l2c220_inv_range(unsigned long start, unsigned long end) 313 { 314 void __iomem *base = l2x0_base; 315 unsigned long flags; 316 317 raw_spin_lock_irqsave(&l2x0_lock, flags); 318 if ((start | end) & (CACHE_LINE_SIZE - 1)) { 319 if (start & (CACHE_LINE_SIZE - 1)) { 320 start &= ~(CACHE_LINE_SIZE - 1); 321 writel_relaxed(start, base + L2X0_CLEAN_INV_LINE_PA); 322 start += CACHE_LINE_SIZE; 323 } 324 325 if (end & (CACHE_LINE_SIZE - 1)) { 326 end &= ~(CACHE_LINE_SIZE - 1); 327 l2c_wait_mask(base + L2X0_CLEAN_INV_LINE_PA, 1); 328 writel_relaxed(end, base + L2X0_CLEAN_INV_LINE_PA); 329 } 330 } 331 332 flags = l2c220_op_pa_range(base + L2X0_INV_LINE_PA, 333 start, end, flags); 334 l2c_wait_mask(base + L2X0_INV_LINE_PA, 1); 335 __l2c220_cache_sync(base); 336 raw_spin_unlock_irqrestore(&l2x0_lock, flags); 337 } 338 339 static void l2c220_clean_range(unsigned long start, unsigned long end) 340 { 341 void __iomem *base = l2x0_base; 342 unsigned long flags; 343 344 start &= ~(CACHE_LINE_SIZE - 1); 345 if ((end - start) >= l2x0_size) { 346 l2c220_op_way(base, L2X0_CLEAN_WAY); 347 return; 348 } 349 350 raw_spin_lock_irqsave(&l2x0_lock, flags); 351 flags = l2c220_op_pa_range(base + L2X0_CLEAN_LINE_PA, 352 start, end, flags); 353 l2c_wait_mask(base + L2X0_CLEAN_INV_LINE_PA, 1); 354 __l2c220_cache_sync(base); 355 raw_spin_unlock_irqrestore(&l2x0_lock, flags); 356 } 357 358 static void l2c220_flush_range(unsigned long start, unsigned long end) 359 { 360 void __iomem *base = l2x0_base; 361 unsigned long flags; 362 363 start &= ~(CACHE_LINE_SIZE - 1); 364 if ((end - start) >= l2x0_size) { 365 l2c220_op_way(base, L2X0_CLEAN_INV_WAY); 366 return; 367 } 368 369 raw_spin_lock_irqsave(&l2x0_lock, flags); 370 flags = l2c220_op_pa_range(base + L2X0_CLEAN_INV_LINE_PA, 371 start, end, flags); 372 l2c_wait_mask(base + L2X0_CLEAN_INV_LINE_PA, 1); 373 __l2c220_cache_sync(base); 374 raw_spin_unlock_irqrestore(&l2x0_lock, flags); 375 } 376 377 static void l2c220_flush_all(void) 378 { 379 l2c220_op_way(l2x0_base, L2X0_CLEAN_INV_WAY); 380 } 381 382 static void l2c220_sync(void) 383 { 384 unsigned long flags; 385 386 raw_spin_lock_irqsave(&l2x0_lock, flags); 387 __l2c220_cache_sync(l2x0_base); 388 raw_spin_unlock_irqrestore(&l2x0_lock, flags); 389 } 390 391 static void l2c220_enable(void __iomem *base, unsigned num_lock) 392 { 393 /* 394 * Always enable non-secure access to the lockdown registers - 395 * we write to them as part of the L2C enable sequence so they 396 * need to be accessible. 397 */ 398 l2x0_saved_regs.aux_ctrl |= L220_AUX_CTRL_NS_LOCKDOWN; 399 400 l2c_enable(base, num_lock); 401 } 402 403 static void l2c220_unlock(void __iomem *base, unsigned num_lock) 404 { 405 if (readl_relaxed(base + L2X0_AUX_CTRL) & L220_AUX_CTRL_NS_LOCKDOWN) 406 l2c_unlock(base, num_lock); 407 } 408 409 static const struct l2c_init_data l2c220_data = { 410 .type = "L2C-220", 411 .way_size_0 = SZ_8K, 412 .num_lock = 1, 413 .enable = l2c220_enable, 414 .save = l2c_save, 415 .configure = l2c_configure, 416 .unlock = l2c220_unlock, 417 .outer_cache = { 418 .inv_range = l2c220_inv_range, 419 .clean_range = l2c220_clean_range, 420 .flush_range = l2c220_flush_range, 421 .flush_all = l2c220_flush_all, 422 .disable = l2c_disable, 423 .sync = l2c220_sync, 424 .resume = l2c_resume, 425 }, 426 }; 427 428 /* 429 * L2C-310 specific code. 430 * 431 * Very similar to L2C-210, the PA, set/way and sync operations are atomic, 432 * and the way operations are all background tasks. However, issuing an 433 * operation while a background operation is in progress results in a 434 * SLVERR response. We can reuse: 435 * 436 * __l2c210_cache_sync (using sync_reg_offset) 437 * l2c210_sync 438 * l2c210_inv_range (if 588369 is not applicable) 439 * l2c210_clean_range 440 * l2c210_flush_range (if 588369 is not applicable) 441 * l2c210_flush_all (if 727915 is not applicable) 442 * 443 * Errata: 444 * 588369: PL310 R0P0->R1P0, fixed R2P0. 445 * Affects: all clean+invalidate operations 446 * clean and invalidate skips the invalidate step, so we need to issue 447 * separate operations. We also require the above debug workaround 448 * enclosing this code fragment on affected parts. On unaffected parts, 449 * we must not use this workaround without the debug register writes 450 * to avoid exposing a problem similar to 727915. 451 * 452 * 727915: PL310 R2P0->R3P0, fixed R3P1. 453 * Affects: clean+invalidate by way 454 * clean and invalidate by way runs in the background, and a store can 455 * hit the line between the clean operation and invalidate operation, 456 * resulting in the store being lost. 457 * 458 * 752271: PL310 R3P0->R3P1-50REL0, fixed R3P2. 459 * Affects: 8x64-bit (double fill) line fetches 460 * double fill line fetches can fail to cause dirty data to be evicted 461 * from the cache before the new data overwrites the second line. 462 * 463 * 753970: PL310 R3P0, fixed R3P1. 464 * Affects: sync 465 * prevents merging writes after the sync operation, until another L2C 466 * operation is performed (or a number of other conditions.) 467 * 468 * 769419: PL310 R0P0->R3P1, fixed R3P2. 469 * Affects: store buffer 470 * store buffer is not automatically drained. 471 */ 472 static void l2c310_inv_range_erratum(unsigned long start, unsigned long end) 473 { 474 void __iomem *base = l2x0_base; 475 476 if ((start | end) & (CACHE_LINE_SIZE - 1)) { 477 unsigned long flags; 478 479 /* Erratum 588369 for both clean+invalidate operations */ 480 raw_spin_lock_irqsave(&l2x0_lock, flags); 481 l2c_set_debug(base, 0x03); 482 483 if (start & (CACHE_LINE_SIZE - 1)) { 484 start &= ~(CACHE_LINE_SIZE - 1); 485 writel_relaxed(start, base + L2X0_CLEAN_LINE_PA); 486 writel_relaxed(start, base + L2X0_INV_LINE_PA); 487 start += CACHE_LINE_SIZE; 488 } 489 490 if (end & (CACHE_LINE_SIZE - 1)) { 491 end &= ~(CACHE_LINE_SIZE - 1); 492 writel_relaxed(end, base + L2X0_CLEAN_LINE_PA); 493 writel_relaxed(end, base + L2X0_INV_LINE_PA); 494 } 495 496 l2c_set_debug(base, 0x00); 497 raw_spin_unlock_irqrestore(&l2x0_lock, flags); 498 } 499 500 __l2c210_op_pa_range(base + L2X0_INV_LINE_PA, start, end); 501 __l2c210_cache_sync(base); 502 } 503 504 static void l2c310_flush_range_erratum(unsigned long start, unsigned long end) 505 { 506 raw_spinlock_t *lock = &l2x0_lock; 507 unsigned long flags; 508 void __iomem *base = l2x0_base; 509 510 raw_spin_lock_irqsave(lock, flags); 511 while (start < end) { 512 unsigned long blk_end = start + min(end - start, 4096UL); 513 514 l2c_set_debug(base, 0x03); 515 while (start < blk_end) { 516 writel_relaxed(start, base + L2X0_CLEAN_LINE_PA); 517 writel_relaxed(start, base + L2X0_INV_LINE_PA); 518 start += CACHE_LINE_SIZE; 519 } 520 l2c_set_debug(base, 0x00); 521 522 if (blk_end < end) { 523 raw_spin_unlock_irqrestore(lock, flags); 524 raw_spin_lock_irqsave(lock, flags); 525 } 526 } 527 raw_spin_unlock_irqrestore(lock, flags); 528 __l2c210_cache_sync(base); 529 } 530 531 static void l2c310_flush_all_erratum(void) 532 { 533 void __iomem *base = l2x0_base; 534 unsigned long flags; 535 536 raw_spin_lock_irqsave(&l2x0_lock, flags); 537 l2c_set_debug(base, 0x03); 538 __l2c_op_way(base + L2X0_CLEAN_INV_WAY); 539 l2c_set_debug(base, 0x00); 540 __l2c210_cache_sync(base); 541 raw_spin_unlock_irqrestore(&l2x0_lock, flags); 542 } 543 544 static void __init l2c310_save(void __iomem *base) 545 { 546 unsigned revision; 547 548 l2c_save(base); 549 550 l2x0_saved_regs.tag_latency = readl_relaxed(base + 551 L310_TAG_LATENCY_CTRL); 552 l2x0_saved_regs.data_latency = readl_relaxed(base + 553 L310_DATA_LATENCY_CTRL); 554 l2x0_saved_regs.filter_end = readl_relaxed(base + 555 L310_ADDR_FILTER_END); 556 l2x0_saved_regs.filter_start = readl_relaxed(base + 557 L310_ADDR_FILTER_START); 558 559 revision = readl_relaxed(base + L2X0_CACHE_ID) & 560 L2X0_CACHE_ID_RTL_MASK; 561 562 /* From r2p0, there is Prefetch offset/control register */ 563 if (revision >= L310_CACHE_ID_RTL_R2P0) 564 l2x0_saved_regs.prefetch_ctrl = readl_relaxed(base + 565 L310_PREFETCH_CTRL); 566 567 /* From r3p0, there is Power control register */ 568 if (revision >= L310_CACHE_ID_RTL_R3P0) 569 l2x0_saved_regs.pwr_ctrl = readl_relaxed(base + 570 L310_POWER_CTRL); 571 } 572 573 static void l2c310_configure(void __iomem *base) 574 { 575 unsigned revision; 576 577 l2c_configure(base); 578 579 /* restore pl310 setup */ 580 l2c_write_sec(l2x0_saved_regs.tag_latency, base, 581 L310_TAG_LATENCY_CTRL); 582 l2c_write_sec(l2x0_saved_regs.data_latency, base, 583 L310_DATA_LATENCY_CTRL); 584 l2c_write_sec(l2x0_saved_regs.filter_end, base, 585 L310_ADDR_FILTER_END); 586 l2c_write_sec(l2x0_saved_regs.filter_start, base, 587 L310_ADDR_FILTER_START); 588 589 revision = readl_relaxed(base + L2X0_CACHE_ID) & 590 L2X0_CACHE_ID_RTL_MASK; 591 592 if (revision >= L310_CACHE_ID_RTL_R2P0) 593 l2c_write_sec(l2x0_saved_regs.prefetch_ctrl, base, 594 L310_PREFETCH_CTRL); 595 if (revision >= L310_CACHE_ID_RTL_R3P0) 596 l2c_write_sec(l2x0_saved_regs.pwr_ctrl, base, 597 L310_POWER_CTRL); 598 } 599 600 static int l2c310_starting_cpu(unsigned int cpu) 601 { 602 set_auxcr(get_auxcr() | BIT(3) | BIT(2) | BIT(1)); 603 return 0; 604 } 605 606 static int l2c310_dying_cpu(unsigned int cpu) 607 { 608 set_auxcr(get_auxcr() & ~(BIT(3) | BIT(2) | BIT(1))); 609 return 0; 610 } 611 612 static void __init l2c310_enable(void __iomem *base, unsigned num_lock) 613 { 614 unsigned rev = readl_relaxed(base + L2X0_CACHE_ID) & L2X0_CACHE_ID_RTL_MASK; 615 bool cortex_a9 = read_cpuid_part() == ARM_CPU_PART_CORTEX_A9; 616 u32 aux = l2x0_saved_regs.aux_ctrl; 617 618 if (rev >= L310_CACHE_ID_RTL_R2P0) { 619 if (cortex_a9) { 620 aux |= L310_AUX_CTRL_EARLY_BRESP; 621 pr_info("L2C-310 enabling early BRESP for Cortex-A9\n"); 622 } else if (aux & L310_AUX_CTRL_EARLY_BRESP) { 623 pr_warn("L2C-310 early BRESP only supported with Cortex-A9\n"); 624 aux &= ~L310_AUX_CTRL_EARLY_BRESP; 625 } 626 } 627 628 if (cortex_a9) { 629 u32 aux_cur = readl_relaxed(base + L2X0_AUX_CTRL); 630 u32 acr = get_auxcr(); 631 632 pr_debug("Cortex-A9 ACR=0x%08x\n", acr); 633 634 if (acr & BIT(3) && !(aux_cur & L310_AUX_CTRL_FULL_LINE_ZERO)) 635 pr_err("L2C-310: full line of zeros enabled in Cortex-A9 but not L2C-310 - invalid\n"); 636 637 if (aux & L310_AUX_CTRL_FULL_LINE_ZERO && !(acr & BIT(3))) 638 pr_err("L2C-310: enabling full line of zeros but not enabled in Cortex-A9\n"); 639 640 if (!(aux & L310_AUX_CTRL_FULL_LINE_ZERO) && !outer_cache.write_sec) { 641 aux |= L310_AUX_CTRL_FULL_LINE_ZERO; 642 pr_info("L2C-310 full line of zeros enabled for Cortex-A9\n"); 643 } 644 } else if (aux & (L310_AUX_CTRL_FULL_LINE_ZERO | L310_AUX_CTRL_EARLY_BRESP)) { 645 pr_err("L2C-310: disabling Cortex-A9 specific feature bits\n"); 646 aux &= ~(L310_AUX_CTRL_FULL_LINE_ZERO | L310_AUX_CTRL_EARLY_BRESP); 647 } 648 649 /* 650 * Always enable non-secure access to the lockdown registers - 651 * we write to them as part of the L2C enable sequence so they 652 * need to be accessible. 653 */ 654 l2x0_saved_regs.aux_ctrl = aux | L310_AUX_CTRL_NS_LOCKDOWN; 655 656 l2c_enable(base, num_lock); 657 658 /* Read back resulting AUX_CTRL value as it could have been altered. */ 659 aux = readl_relaxed(base + L2X0_AUX_CTRL); 660 661 if (aux & (L310_AUX_CTRL_DATA_PREFETCH | L310_AUX_CTRL_INSTR_PREFETCH)) { 662 u32 prefetch = readl_relaxed(base + L310_PREFETCH_CTRL); 663 664 pr_info("L2C-310 %s%s prefetch enabled, offset %u lines\n", 665 aux & L310_AUX_CTRL_INSTR_PREFETCH ? "I" : "", 666 aux & L310_AUX_CTRL_DATA_PREFETCH ? "D" : "", 667 1 + (prefetch & L310_PREFETCH_CTRL_OFFSET_MASK)); 668 } 669 670 /* r3p0 or later has power control register */ 671 if (rev >= L310_CACHE_ID_RTL_R3P0) { 672 u32 power_ctrl; 673 674 power_ctrl = readl_relaxed(base + L310_POWER_CTRL); 675 pr_info("L2C-310 dynamic clock gating %sabled, standby mode %sabled\n", 676 power_ctrl & L310_DYNAMIC_CLK_GATING_EN ? "en" : "dis", 677 power_ctrl & L310_STNDBY_MODE_EN ? "en" : "dis"); 678 } 679 680 if (aux & L310_AUX_CTRL_FULL_LINE_ZERO) 681 cpuhp_setup_state(CPUHP_AP_ARM_L2X0_STARTING, 682 "AP_ARM_L2X0_STARTING", l2c310_starting_cpu, 683 l2c310_dying_cpu); 684 } 685 686 static void __init l2c310_fixup(void __iomem *base, u32 cache_id, 687 struct outer_cache_fns *fns) 688 { 689 unsigned revision = cache_id & L2X0_CACHE_ID_RTL_MASK; 690 const char *errata[8]; 691 unsigned n = 0; 692 693 if (IS_ENABLED(CONFIG_PL310_ERRATA_588369) && 694 revision < L310_CACHE_ID_RTL_R2P0 && 695 /* For bcm compatibility */ 696 fns->inv_range == l2c210_inv_range) { 697 fns->inv_range = l2c310_inv_range_erratum; 698 fns->flush_range = l2c310_flush_range_erratum; 699 errata[n++] = "588369"; 700 } 701 702 if (IS_ENABLED(CONFIG_PL310_ERRATA_727915) && 703 revision >= L310_CACHE_ID_RTL_R2P0 && 704 revision < L310_CACHE_ID_RTL_R3P1) { 705 fns->flush_all = l2c310_flush_all_erratum; 706 errata[n++] = "727915"; 707 } 708 709 if (revision >= L310_CACHE_ID_RTL_R3P0 && 710 revision < L310_CACHE_ID_RTL_R3P2) { 711 u32 val = l2x0_saved_regs.prefetch_ctrl; 712 if (val & L310_PREFETCH_CTRL_DBL_LINEFILL) { 713 val &= ~L310_PREFETCH_CTRL_DBL_LINEFILL; 714 l2x0_saved_regs.prefetch_ctrl = val; 715 errata[n++] = "752271"; 716 } 717 } 718 719 if (IS_ENABLED(CONFIG_PL310_ERRATA_753970) && 720 revision == L310_CACHE_ID_RTL_R3P0) { 721 sync_reg_offset = L2X0_DUMMY_REG; 722 errata[n++] = "753970"; 723 } 724 725 if (IS_ENABLED(CONFIG_PL310_ERRATA_769419)) 726 errata[n++] = "769419"; 727 728 if (n) { 729 unsigned i; 730 731 pr_info("L2C-310 errat%s", n > 1 ? "a" : "um"); 732 for (i = 0; i < n; i++) 733 pr_cont(" %s", errata[i]); 734 pr_cont(" enabled\n"); 735 } 736 } 737 738 static void l2c310_disable(void) 739 { 740 /* 741 * If full-line-of-zeros is enabled, we must first disable it in the 742 * Cortex-A9 auxiliary control register before disabling the L2 cache. 743 */ 744 if (l2x0_saved_regs.aux_ctrl & L310_AUX_CTRL_FULL_LINE_ZERO) 745 set_auxcr(get_auxcr() & ~(BIT(3) | BIT(2) | BIT(1))); 746 747 l2c_disable(); 748 } 749 750 static void l2c310_resume(void) 751 { 752 l2c_resume(); 753 754 /* Re-enable full-line-of-zeros for Cortex-A9 */ 755 if (l2x0_saved_regs.aux_ctrl & L310_AUX_CTRL_FULL_LINE_ZERO) 756 set_auxcr(get_auxcr() | BIT(3) | BIT(2) | BIT(1)); 757 } 758 759 static void l2c310_unlock(void __iomem *base, unsigned num_lock) 760 { 761 if (readl_relaxed(base + L2X0_AUX_CTRL) & L310_AUX_CTRL_NS_LOCKDOWN) 762 l2c_unlock(base, num_lock); 763 } 764 765 static const struct l2c_init_data l2c310_init_fns __initconst = { 766 .type = "L2C-310", 767 .way_size_0 = SZ_8K, 768 .num_lock = 8, 769 .enable = l2c310_enable, 770 .fixup = l2c310_fixup, 771 .save = l2c310_save, 772 .configure = l2c310_configure, 773 .unlock = l2c310_unlock, 774 .outer_cache = { 775 .inv_range = l2c210_inv_range, 776 .clean_range = l2c210_clean_range, 777 .flush_range = l2c210_flush_range, 778 .flush_all = l2c210_flush_all, 779 .disable = l2c310_disable, 780 .sync = l2c210_sync, 781 .resume = l2c310_resume, 782 }, 783 }; 784 785 static int __init __l2c_init(const struct l2c_init_data *data, 786 u32 aux_val, u32 aux_mask, u32 cache_id, bool nosync) 787 { 788 struct outer_cache_fns fns; 789 unsigned way_size_bits, ways; 790 u32 aux, old_aux; 791 792 /* 793 * Save the pointer globally so that callbacks which do not receive 794 * context from callers can access the structure. 795 */ 796 l2x0_data = kmemdup(data, sizeof(*data), GFP_KERNEL); 797 if (!l2x0_data) 798 return -ENOMEM; 799 800 /* 801 * Sanity check the aux values. aux_mask is the bits we preserve 802 * from reading the hardware register, and aux_val is the bits we 803 * set. 804 */ 805 if (aux_val & aux_mask) 806 pr_alert("L2C: platform provided aux values permit register corruption.\n"); 807 808 old_aux = aux = readl_relaxed(l2x0_base + L2X0_AUX_CTRL); 809 aux &= aux_mask; 810 aux |= aux_val; 811 812 if (old_aux != aux) 813 pr_warn("L2C: DT/platform modifies aux control register: 0x%08x -> 0x%08x\n", 814 old_aux, aux); 815 816 /* Determine the number of ways */ 817 switch (cache_id & L2X0_CACHE_ID_PART_MASK) { 818 case L2X0_CACHE_ID_PART_L310: 819 if ((aux_val | ~aux_mask) & (L2C_AUX_CTRL_WAY_SIZE_MASK | L310_AUX_CTRL_ASSOCIATIVITY_16)) 820 pr_warn("L2C: DT/platform tries to modify or specify cache size\n"); 821 if (aux & (1 << 16)) 822 ways = 16; 823 else 824 ways = 8; 825 break; 826 827 case L2X0_CACHE_ID_PART_L210: 828 case L2X0_CACHE_ID_PART_L220: 829 ways = (aux >> 13) & 0xf; 830 break; 831 832 case AURORA_CACHE_ID: 833 ways = (aux >> 13) & 0xf; 834 ways = 2 << ((ways + 1) >> 2); 835 break; 836 837 default: 838 /* Assume unknown chips have 8 ways */ 839 ways = 8; 840 break; 841 } 842 843 l2x0_way_mask = (1 << ways) - 1; 844 845 /* 846 * way_size_0 is the size that a way_size value of zero would be 847 * given the calculation: way_size = way_size_0 << way_size_bits. 848 * So, if way_size_bits=0 is reserved, but way_size_bits=1 is 16k, 849 * then way_size_0 would be 8k. 850 * 851 * L2 cache size = number of ways * way size. 852 */ 853 way_size_bits = (aux & L2C_AUX_CTRL_WAY_SIZE_MASK) >> 854 L2C_AUX_CTRL_WAY_SIZE_SHIFT; 855 l2x0_size = ways * (data->way_size_0 << way_size_bits); 856 857 fns = data->outer_cache; 858 fns.write_sec = outer_cache.write_sec; 859 fns.configure = outer_cache.configure; 860 if (data->fixup) 861 data->fixup(l2x0_base, cache_id, &fns); 862 if (nosync) { 863 pr_info("L2C: disabling outer sync\n"); 864 fns.sync = NULL; 865 } 866 867 /* 868 * Check if l2x0 controller is already enabled. If we are booting 869 * in non-secure mode accessing the below registers will fault. 870 */ 871 if (!(readl_relaxed(l2x0_base + L2X0_CTRL) & L2X0_CTRL_EN)) { 872 l2x0_saved_regs.aux_ctrl = aux; 873 874 data->enable(l2x0_base, data->num_lock); 875 } 876 877 outer_cache = fns; 878 879 /* 880 * It is strange to save the register state before initialisation, 881 * but hey, this is what the DT implementations decided to do. 882 */ 883 if (data->save) 884 data->save(l2x0_base); 885 886 /* Re-read it in case some bits are reserved. */ 887 aux = readl_relaxed(l2x0_base + L2X0_AUX_CTRL); 888 889 pr_info("%s cache controller enabled, %d ways, %d kB\n", 890 data->type, ways, l2x0_size >> 10); 891 pr_info("%s: CACHE_ID 0x%08x, AUX_CTRL 0x%08x\n", 892 data->type, cache_id, aux); 893 894 return 0; 895 } 896 897 void __init l2x0_init(void __iomem *base, u32 aux_val, u32 aux_mask) 898 { 899 const struct l2c_init_data *data; 900 u32 cache_id; 901 902 l2x0_base = base; 903 904 cache_id = readl_relaxed(base + L2X0_CACHE_ID); 905 906 switch (cache_id & L2X0_CACHE_ID_PART_MASK) { 907 default: 908 case L2X0_CACHE_ID_PART_L210: 909 data = &l2c210_data; 910 break; 911 912 case L2X0_CACHE_ID_PART_L220: 913 data = &l2c220_data; 914 break; 915 916 case L2X0_CACHE_ID_PART_L310: 917 data = &l2c310_init_fns; 918 break; 919 } 920 921 /* Read back current (default) hardware configuration */ 922 if (data->save) 923 data->save(l2x0_base); 924 925 __l2c_init(data, aux_val, aux_mask, cache_id, false); 926 } 927 928 #ifdef CONFIG_OF 929 static int l2_wt_override; 930 931 /* Aurora don't have the cache ID register available, so we have to 932 * pass it though the device tree */ 933 static u32 cache_id_part_number_from_dt; 934 935 /** 936 * l2x0_cache_size_of_parse() - read cache size parameters from DT 937 * @np: the device tree node for the l2 cache 938 * @aux_val: pointer to machine-supplied auxilary register value, to 939 * be augmented by the call (bits to be set to 1) 940 * @aux_mask: pointer to machine-supplied auxilary register mask, to 941 * be augmented by the call (bits to be set to 0) 942 * @associativity: variable to return the calculated associativity in 943 * @max_way_size: the maximum size in bytes for the cache ways 944 */ 945 static int __init l2x0_cache_size_of_parse(const struct device_node *np, 946 u32 *aux_val, u32 *aux_mask, 947 u32 *associativity, 948 u32 max_way_size) 949 { 950 u32 mask = 0, val = 0; 951 u32 cache_size = 0, sets = 0; 952 u32 way_size_bits = 1; 953 u32 way_size = 0; 954 u32 block_size = 0; 955 u32 line_size = 0; 956 957 of_property_read_u32(np, "cache-size", &cache_size); 958 of_property_read_u32(np, "cache-sets", &sets); 959 of_property_read_u32(np, "cache-block-size", &block_size); 960 of_property_read_u32(np, "cache-line-size", &line_size); 961 962 if (!cache_size || !sets) 963 return -ENODEV; 964 965 /* All these l2 caches have the same line = block size actually */ 966 if (!line_size) { 967 if (block_size) { 968 /* If linesize is not given, it is equal to blocksize */ 969 line_size = block_size; 970 } else { 971 /* Fall back to known size */ 972 pr_warn("L2C OF: no cache block/line size given: " 973 "falling back to default size %d bytes\n", 974 CACHE_LINE_SIZE); 975 line_size = CACHE_LINE_SIZE; 976 } 977 } 978 979 if (line_size != CACHE_LINE_SIZE) 980 pr_warn("L2C OF: DT supplied line size %d bytes does " 981 "not match hardware line size of %d bytes\n", 982 line_size, 983 CACHE_LINE_SIZE); 984 985 /* 986 * Since: 987 * set size = cache size / sets 988 * ways = cache size / (sets * line size) 989 * way size = cache size / (cache size / (sets * line size)) 990 * way size = sets * line size 991 * associativity = ways = cache size / way size 992 */ 993 way_size = sets * line_size; 994 *associativity = cache_size / way_size; 995 996 if (way_size > max_way_size) { 997 pr_err("L2C OF: set size %dKB is too large\n", way_size); 998 return -EINVAL; 999 } 1000 1001 pr_info("L2C OF: override cache size: %d bytes (%dKB)\n", 1002 cache_size, cache_size >> 10); 1003 pr_info("L2C OF: override line size: %d bytes\n", line_size); 1004 pr_info("L2C OF: override way size: %d bytes (%dKB)\n", 1005 way_size, way_size >> 10); 1006 pr_info("L2C OF: override associativity: %d\n", *associativity); 1007 1008 /* 1009 * Calculates the bits 17:19 to set for way size: 1010 * 512KB -> 6, 256KB -> 5, ... 16KB -> 1 1011 */ 1012 way_size_bits = ilog2(way_size >> 10) - 3; 1013 if (way_size_bits < 1 || way_size_bits > 6) { 1014 pr_err("L2C OF: cache way size illegal: %dKB is not mapped\n", 1015 way_size); 1016 return -EINVAL; 1017 } 1018 1019 mask |= L2C_AUX_CTRL_WAY_SIZE_MASK; 1020 val |= (way_size_bits << L2C_AUX_CTRL_WAY_SIZE_SHIFT); 1021 1022 *aux_val &= ~mask; 1023 *aux_val |= val; 1024 *aux_mask &= ~mask; 1025 1026 return 0; 1027 } 1028 1029 static void __init l2x0_of_parse(const struct device_node *np, 1030 u32 *aux_val, u32 *aux_mask) 1031 { 1032 u32 data[2] = { 0, 0 }; 1033 u32 tag = 0; 1034 u32 dirty = 0; 1035 u32 val = 0, mask = 0; 1036 u32 assoc; 1037 int ret; 1038 1039 of_property_read_u32(np, "arm,tag-latency", &tag); 1040 if (tag) { 1041 mask |= L2X0_AUX_CTRL_TAG_LATENCY_MASK; 1042 val |= (tag - 1) << L2X0_AUX_CTRL_TAG_LATENCY_SHIFT; 1043 } 1044 1045 of_property_read_u32_array(np, "arm,data-latency", 1046 data, ARRAY_SIZE(data)); 1047 if (data[0] && data[1]) { 1048 mask |= L2X0_AUX_CTRL_DATA_RD_LATENCY_MASK | 1049 L2X0_AUX_CTRL_DATA_WR_LATENCY_MASK; 1050 val |= ((data[0] - 1) << L2X0_AUX_CTRL_DATA_RD_LATENCY_SHIFT) | 1051 ((data[1] - 1) << L2X0_AUX_CTRL_DATA_WR_LATENCY_SHIFT); 1052 } 1053 1054 of_property_read_u32(np, "arm,dirty-latency", &dirty); 1055 if (dirty) { 1056 mask |= L2X0_AUX_CTRL_DIRTY_LATENCY_MASK; 1057 val |= (dirty - 1) << L2X0_AUX_CTRL_DIRTY_LATENCY_SHIFT; 1058 } 1059 1060 if (of_property_read_bool(np, "arm,parity-enable")) { 1061 mask &= ~L2C_AUX_CTRL_PARITY_ENABLE; 1062 val |= L2C_AUX_CTRL_PARITY_ENABLE; 1063 } else if (of_property_read_bool(np, "arm,parity-disable")) { 1064 mask &= ~L2C_AUX_CTRL_PARITY_ENABLE; 1065 } 1066 1067 if (of_property_read_bool(np, "arm,shared-override")) { 1068 mask &= ~L2C_AUX_CTRL_SHARED_OVERRIDE; 1069 val |= L2C_AUX_CTRL_SHARED_OVERRIDE; 1070 } 1071 1072 ret = l2x0_cache_size_of_parse(np, aux_val, aux_mask, &assoc, SZ_256K); 1073 if (ret) 1074 return; 1075 1076 if (assoc > 8) { 1077 pr_err("l2x0 of: cache setting yield too high associativity\n"); 1078 pr_err("l2x0 of: %d calculated, max 8\n", assoc); 1079 } else { 1080 mask |= L2X0_AUX_CTRL_ASSOC_MASK; 1081 val |= (assoc << L2X0_AUX_CTRL_ASSOC_SHIFT); 1082 } 1083 1084 *aux_val &= ~mask; 1085 *aux_val |= val; 1086 *aux_mask &= ~mask; 1087 } 1088 1089 static const struct l2c_init_data of_l2c210_data __initconst = { 1090 .type = "L2C-210", 1091 .way_size_0 = SZ_8K, 1092 .num_lock = 1, 1093 .of_parse = l2x0_of_parse, 1094 .enable = l2c_enable, 1095 .save = l2c_save, 1096 .configure = l2c_configure, 1097 .unlock = l2c_unlock, 1098 .outer_cache = { 1099 .inv_range = l2c210_inv_range, 1100 .clean_range = l2c210_clean_range, 1101 .flush_range = l2c210_flush_range, 1102 .flush_all = l2c210_flush_all, 1103 .disable = l2c_disable, 1104 .sync = l2c210_sync, 1105 .resume = l2c_resume, 1106 }, 1107 }; 1108 1109 static const struct l2c_init_data of_l2c220_data __initconst = { 1110 .type = "L2C-220", 1111 .way_size_0 = SZ_8K, 1112 .num_lock = 1, 1113 .of_parse = l2x0_of_parse, 1114 .enable = l2c220_enable, 1115 .save = l2c_save, 1116 .configure = l2c_configure, 1117 .unlock = l2c220_unlock, 1118 .outer_cache = { 1119 .inv_range = l2c220_inv_range, 1120 .clean_range = l2c220_clean_range, 1121 .flush_range = l2c220_flush_range, 1122 .flush_all = l2c220_flush_all, 1123 .disable = l2c_disable, 1124 .sync = l2c220_sync, 1125 .resume = l2c_resume, 1126 }, 1127 }; 1128 1129 static void __init l2c310_of_parse(const struct device_node *np, 1130 u32 *aux_val, u32 *aux_mask) 1131 { 1132 u32 data[3] = { 0, 0, 0 }; 1133 u32 tag[3] = { 0, 0, 0 }; 1134 u32 filter[2] = { 0, 0 }; 1135 u32 assoc; 1136 u32 prefetch; 1137 u32 power; 1138 u32 val; 1139 int ret; 1140 1141 of_property_read_u32_array(np, "arm,tag-latency", tag, ARRAY_SIZE(tag)); 1142 if (tag[0] && tag[1] && tag[2]) 1143 l2x0_saved_regs.tag_latency = 1144 L310_LATENCY_CTRL_RD(tag[0] - 1) | 1145 L310_LATENCY_CTRL_WR(tag[1] - 1) | 1146 L310_LATENCY_CTRL_SETUP(tag[2] - 1); 1147 1148 of_property_read_u32_array(np, "arm,data-latency", 1149 data, ARRAY_SIZE(data)); 1150 if (data[0] && data[1] && data[2]) 1151 l2x0_saved_regs.data_latency = 1152 L310_LATENCY_CTRL_RD(data[0] - 1) | 1153 L310_LATENCY_CTRL_WR(data[1] - 1) | 1154 L310_LATENCY_CTRL_SETUP(data[2] - 1); 1155 1156 of_property_read_u32_array(np, "arm,filter-ranges", 1157 filter, ARRAY_SIZE(filter)); 1158 if (filter[1]) { 1159 l2x0_saved_regs.filter_end = 1160 ALIGN(filter[0] + filter[1], SZ_1M); 1161 l2x0_saved_regs.filter_start = (filter[0] & ~(SZ_1M - 1)) 1162 | L310_ADDR_FILTER_EN; 1163 } 1164 1165 ret = l2x0_cache_size_of_parse(np, aux_val, aux_mask, &assoc, SZ_512K); 1166 if (!ret) { 1167 switch (assoc) { 1168 case 16: 1169 *aux_val &= ~L2X0_AUX_CTRL_ASSOC_MASK; 1170 *aux_val |= L310_AUX_CTRL_ASSOCIATIVITY_16; 1171 *aux_mask &= ~L2X0_AUX_CTRL_ASSOC_MASK; 1172 break; 1173 case 8: 1174 *aux_val &= ~L2X0_AUX_CTRL_ASSOC_MASK; 1175 *aux_mask &= ~L2X0_AUX_CTRL_ASSOC_MASK; 1176 break; 1177 default: 1178 pr_err("L2C-310 OF cache associativity %d invalid, only 8 or 16 permitted\n", 1179 assoc); 1180 break; 1181 } 1182 } 1183 1184 if (of_property_read_bool(np, "arm,shared-override")) { 1185 *aux_val |= L2C_AUX_CTRL_SHARED_OVERRIDE; 1186 *aux_mask &= ~L2C_AUX_CTRL_SHARED_OVERRIDE; 1187 } 1188 1189 if (of_property_read_bool(np, "arm,parity-enable")) { 1190 *aux_val |= L2C_AUX_CTRL_PARITY_ENABLE; 1191 *aux_mask &= ~L2C_AUX_CTRL_PARITY_ENABLE; 1192 } else if (of_property_read_bool(np, "arm,parity-disable")) { 1193 *aux_val &= ~L2C_AUX_CTRL_PARITY_ENABLE; 1194 *aux_mask &= ~L2C_AUX_CTRL_PARITY_ENABLE; 1195 } 1196 1197 prefetch = l2x0_saved_regs.prefetch_ctrl; 1198 1199 ret = of_property_read_u32(np, "arm,double-linefill", &val); 1200 if (ret == 0) { 1201 if (val) 1202 prefetch |= L310_PREFETCH_CTRL_DBL_LINEFILL; 1203 else 1204 prefetch &= ~L310_PREFETCH_CTRL_DBL_LINEFILL; 1205 } else if (ret != -EINVAL) { 1206 pr_err("L2C-310 OF arm,double-linefill property value is missing\n"); 1207 } 1208 1209 ret = of_property_read_u32(np, "arm,double-linefill-incr", &val); 1210 if (ret == 0) { 1211 if (val) 1212 prefetch |= L310_PREFETCH_CTRL_DBL_LINEFILL_INCR; 1213 else 1214 prefetch &= ~L310_PREFETCH_CTRL_DBL_LINEFILL_INCR; 1215 } else if (ret != -EINVAL) { 1216 pr_err("L2C-310 OF arm,double-linefill-incr property value is missing\n"); 1217 } 1218 1219 ret = of_property_read_u32(np, "arm,double-linefill-wrap", &val); 1220 if (ret == 0) { 1221 if (!val) 1222 prefetch |= L310_PREFETCH_CTRL_DBL_LINEFILL_WRAP; 1223 else 1224 prefetch &= ~L310_PREFETCH_CTRL_DBL_LINEFILL_WRAP; 1225 } else if (ret != -EINVAL) { 1226 pr_err("L2C-310 OF arm,double-linefill-wrap property value is missing\n"); 1227 } 1228 1229 ret = of_property_read_u32(np, "arm,prefetch-drop", &val); 1230 if (ret == 0) { 1231 if (val) 1232 prefetch |= L310_PREFETCH_CTRL_PREFETCH_DROP; 1233 else 1234 prefetch &= ~L310_PREFETCH_CTRL_PREFETCH_DROP; 1235 } else if (ret != -EINVAL) { 1236 pr_err("L2C-310 OF arm,prefetch-drop property value is missing\n"); 1237 } 1238 1239 ret = of_property_read_u32(np, "arm,prefetch-offset", &val); 1240 if (ret == 0) { 1241 prefetch &= ~L310_PREFETCH_CTRL_OFFSET_MASK; 1242 prefetch |= val & L310_PREFETCH_CTRL_OFFSET_MASK; 1243 } else if (ret != -EINVAL) { 1244 pr_err("L2C-310 OF arm,prefetch-offset property value is missing\n"); 1245 } 1246 1247 ret = of_property_read_u32(np, "prefetch-data", &val); 1248 if (ret == 0) { 1249 if (val) 1250 prefetch |= L310_PREFETCH_CTRL_DATA_PREFETCH; 1251 else 1252 prefetch &= ~L310_PREFETCH_CTRL_DATA_PREFETCH; 1253 } else if (ret != -EINVAL) { 1254 pr_err("L2C-310 OF prefetch-data property value is missing\n"); 1255 } 1256 1257 ret = of_property_read_u32(np, "prefetch-instr", &val); 1258 if (ret == 0) { 1259 if (val) 1260 prefetch |= L310_PREFETCH_CTRL_INSTR_PREFETCH; 1261 else 1262 prefetch &= ~L310_PREFETCH_CTRL_INSTR_PREFETCH; 1263 } else if (ret != -EINVAL) { 1264 pr_err("L2C-310 OF prefetch-instr property value is missing\n"); 1265 } 1266 1267 l2x0_saved_regs.prefetch_ctrl = prefetch; 1268 1269 power = l2x0_saved_regs.pwr_ctrl | 1270 L310_DYNAMIC_CLK_GATING_EN | L310_STNDBY_MODE_EN; 1271 1272 ret = of_property_read_u32(np, "arm,dynamic-clock-gating", &val); 1273 if (!ret) { 1274 if (!val) 1275 power &= ~L310_DYNAMIC_CLK_GATING_EN; 1276 } else if (ret != -EINVAL) { 1277 pr_err("L2C-310 OF dynamic-clock-gating property value is missing or invalid\n"); 1278 } 1279 ret = of_property_read_u32(np, "arm,standby-mode", &val); 1280 if (!ret) { 1281 if (!val) 1282 power &= ~L310_STNDBY_MODE_EN; 1283 } else if (ret != -EINVAL) { 1284 pr_err("L2C-310 OF standby-mode property value is missing or invalid\n"); 1285 } 1286 1287 l2x0_saved_regs.pwr_ctrl = power; 1288 } 1289 1290 static const struct l2c_init_data of_l2c310_data __initconst = { 1291 .type = "L2C-310", 1292 .way_size_0 = SZ_8K, 1293 .num_lock = 8, 1294 .of_parse = l2c310_of_parse, 1295 .enable = l2c310_enable, 1296 .fixup = l2c310_fixup, 1297 .save = l2c310_save, 1298 .configure = l2c310_configure, 1299 .unlock = l2c310_unlock, 1300 .outer_cache = { 1301 .inv_range = l2c210_inv_range, 1302 .clean_range = l2c210_clean_range, 1303 .flush_range = l2c210_flush_range, 1304 .flush_all = l2c210_flush_all, 1305 .disable = l2c310_disable, 1306 .sync = l2c210_sync, 1307 .resume = l2c310_resume, 1308 }, 1309 }; 1310 1311 /* 1312 * This is a variant of the of_l2c310_data with .sync set to 1313 * NULL. Outer sync operations are not needed when the system is I/O 1314 * coherent, and potentially harmful in certain situations (PCIe/PL310 1315 * deadlock on Armada 375/38x due to hardware I/O coherency). The 1316 * other operations are kept because they are infrequent (therefore do 1317 * not cause the deadlock in practice) and needed for secondary CPU 1318 * boot and other power management activities. 1319 */ 1320 static const struct l2c_init_data of_l2c310_coherent_data __initconst = { 1321 .type = "L2C-310 Coherent", 1322 .way_size_0 = SZ_8K, 1323 .num_lock = 8, 1324 .of_parse = l2c310_of_parse, 1325 .enable = l2c310_enable, 1326 .fixup = l2c310_fixup, 1327 .save = l2c310_save, 1328 .configure = l2c310_configure, 1329 .unlock = l2c310_unlock, 1330 .outer_cache = { 1331 .inv_range = l2c210_inv_range, 1332 .clean_range = l2c210_clean_range, 1333 .flush_range = l2c210_flush_range, 1334 .flush_all = l2c210_flush_all, 1335 .disable = l2c310_disable, 1336 .resume = l2c310_resume, 1337 }, 1338 }; 1339 1340 /* 1341 * Note that the end addresses passed to Linux primitives are 1342 * noninclusive, while the hardware cache range operations use 1343 * inclusive start and end addresses. 1344 */ 1345 static unsigned long aurora_range_end(unsigned long start, unsigned long end) 1346 { 1347 /* 1348 * Limit the number of cache lines processed at once, 1349 * since cache range operations stall the CPU pipeline 1350 * until completion. 1351 */ 1352 if (end > start + MAX_RANGE_SIZE) 1353 end = start + MAX_RANGE_SIZE; 1354 1355 /* 1356 * Cache range operations can't straddle a page boundary. 1357 */ 1358 if (end > PAGE_ALIGN(start+1)) 1359 end = PAGE_ALIGN(start+1); 1360 1361 return end; 1362 } 1363 1364 static void aurora_pa_range(unsigned long start, unsigned long end, 1365 unsigned long offset) 1366 { 1367 void __iomem *base = l2x0_base; 1368 unsigned long range_end; 1369 unsigned long flags; 1370 1371 /* 1372 * round start and end adresses up to cache line size 1373 */ 1374 start &= ~(CACHE_LINE_SIZE - 1); 1375 end = ALIGN(end, CACHE_LINE_SIZE); 1376 1377 /* 1378 * perform operation on all full cache lines between 'start' and 'end' 1379 */ 1380 while (start < end) { 1381 range_end = aurora_range_end(start, end); 1382 1383 raw_spin_lock_irqsave(&l2x0_lock, flags); 1384 writel_relaxed(start, base + AURORA_RANGE_BASE_ADDR_REG); 1385 writel_relaxed(range_end - CACHE_LINE_SIZE, base + offset); 1386 raw_spin_unlock_irqrestore(&l2x0_lock, flags); 1387 1388 writel_relaxed(0, base + AURORA_SYNC_REG); 1389 start = range_end; 1390 } 1391 } 1392 static void aurora_inv_range(unsigned long start, unsigned long end) 1393 { 1394 aurora_pa_range(start, end, AURORA_INVAL_RANGE_REG); 1395 } 1396 1397 static void aurora_clean_range(unsigned long start, unsigned long end) 1398 { 1399 /* 1400 * If L2 is forced to WT, the L2 will always be clean and we 1401 * don't need to do anything here. 1402 */ 1403 if (!l2_wt_override) 1404 aurora_pa_range(start, end, AURORA_CLEAN_RANGE_REG); 1405 } 1406 1407 static void aurora_flush_range(unsigned long start, unsigned long end) 1408 { 1409 if (l2_wt_override) 1410 aurora_pa_range(start, end, AURORA_INVAL_RANGE_REG); 1411 else 1412 aurora_pa_range(start, end, AURORA_FLUSH_RANGE_REG); 1413 } 1414 1415 static void aurora_flush_all(void) 1416 { 1417 void __iomem *base = l2x0_base; 1418 unsigned long flags; 1419 1420 /* clean all ways */ 1421 raw_spin_lock_irqsave(&l2x0_lock, flags); 1422 __l2c_op_way(base + L2X0_CLEAN_INV_WAY); 1423 raw_spin_unlock_irqrestore(&l2x0_lock, flags); 1424 1425 writel_relaxed(0, base + AURORA_SYNC_REG); 1426 } 1427 1428 static void aurora_cache_sync(void) 1429 { 1430 writel_relaxed(0, l2x0_base + AURORA_SYNC_REG); 1431 } 1432 1433 static void aurora_disable(void) 1434 { 1435 void __iomem *base = l2x0_base; 1436 unsigned long flags; 1437 1438 raw_spin_lock_irqsave(&l2x0_lock, flags); 1439 __l2c_op_way(base + L2X0_CLEAN_INV_WAY); 1440 writel_relaxed(0, base + AURORA_SYNC_REG); 1441 l2c_write_sec(0, base, L2X0_CTRL); 1442 dsb(st); 1443 raw_spin_unlock_irqrestore(&l2x0_lock, flags); 1444 } 1445 1446 static void aurora_save(void __iomem *base) 1447 { 1448 l2x0_saved_regs.ctrl = readl_relaxed(base + L2X0_CTRL); 1449 l2x0_saved_regs.aux_ctrl = readl_relaxed(base + L2X0_AUX_CTRL); 1450 } 1451 1452 /* 1453 * For Aurora cache in no outer mode, enable via the CP15 coprocessor 1454 * broadcasting of cache commands to L2. 1455 */ 1456 static void __init aurora_enable_no_outer(void __iomem *base, 1457 unsigned num_lock) 1458 { 1459 u32 u; 1460 1461 asm volatile("mrc p15, 1, %0, c15, c2, 0" : "=r" (u)); 1462 u |= AURORA_CTRL_FW; /* Set the FW bit */ 1463 asm volatile("mcr p15, 1, %0, c15, c2, 0" : : "r" (u)); 1464 1465 isb(); 1466 1467 l2c_enable(base, num_lock); 1468 } 1469 1470 static void __init aurora_fixup(void __iomem *base, u32 cache_id, 1471 struct outer_cache_fns *fns) 1472 { 1473 sync_reg_offset = AURORA_SYNC_REG; 1474 } 1475 1476 static void __init aurora_of_parse(const struct device_node *np, 1477 u32 *aux_val, u32 *aux_mask) 1478 { 1479 u32 val = AURORA_ACR_REPLACEMENT_TYPE_SEMIPLRU; 1480 u32 mask = AURORA_ACR_REPLACEMENT_MASK; 1481 1482 of_property_read_u32(np, "cache-id-part", 1483 &cache_id_part_number_from_dt); 1484 1485 /* Determine and save the write policy */ 1486 l2_wt_override = of_property_read_bool(np, "wt-override"); 1487 1488 if (l2_wt_override) { 1489 val |= AURORA_ACR_FORCE_WRITE_THRO_POLICY; 1490 mask |= AURORA_ACR_FORCE_WRITE_POLICY_MASK; 1491 } 1492 1493 *aux_val &= ~mask; 1494 *aux_val |= val; 1495 *aux_mask &= ~mask; 1496 } 1497 1498 static const struct l2c_init_data of_aurora_with_outer_data __initconst = { 1499 .type = "Aurora", 1500 .way_size_0 = SZ_4K, 1501 .num_lock = 4, 1502 .of_parse = aurora_of_parse, 1503 .enable = l2c_enable, 1504 .fixup = aurora_fixup, 1505 .save = aurora_save, 1506 .configure = l2c_configure, 1507 .unlock = l2c_unlock, 1508 .outer_cache = { 1509 .inv_range = aurora_inv_range, 1510 .clean_range = aurora_clean_range, 1511 .flush_range = aurora_flush_range, 1512 .flush_all = aurora_flush_all, 1513 .disable = aurora_disable, 1514 .sync = aurora_cache_sync, 1515 .resume = l2c_resume, 1516 }, 1517 }; 1518 1519 static const struct l2c_init_data of_aurora_no_outer_data __initconst = { 1520 .type = "Aurora", 1521 .way_size_0 = SZ_4K, 1522 .num_lock = 4, 1523 .of_parse = aurora_of_parse, 1524 .enable = aurora_enable_no_outer, 1525 .fixup = aurora_fixup, 1526 .save = aurora_save, 1527 .configure = l2c_configure, 1528 .unlock = l2c_unlock, 1529 .outer_cache = { 1530 .resume = l2c_resume, 1531 }, 1532 }; 1533 1534 /* 1535 * For certain Broadcom SoCs, depending on the address range, different offsets 1536 * need to be added to the address before passing it to L2 for 1537 * invalidation/clean/flush 1538 * 1539 * Section Address Range Offset EMI 1540 * 1 0x00000000 - 0x3FFFFFFF 0x80000000 VC 1541 * 2 0x40000000 - 0xBFFFFFFF 0x40000000 SYS 1542 * 3 0xC0000000 - 0xFFFFFFFF 0x80000000 VC 1543 * 1544 * When the start and end addresses have crossed two different sections, we 1545 * need to break the L2 operation into two, each within its own section. 1546 * For example, if we need to invalidate addresses starts at 0xBFFF0000 and 1547 * ends at 0xC0001000, we need do invalidate 1) 0xBFFF0000 - 0xBFFFFFFF and 2) 1548 * 0xC0000000 - 0xC0001000 1549 * 1550 * Note 1: 1551 * By breaking a single L2 operation into two, we may potentially suffer some 1552 * performance hit, but keep in mind the cross section case is very rare 1553 * 1554 * Note 2: 1555 * We do not need to handle the case when the start address is in 1556 * Section 1 and the end address is in Section 3, since it is not a valid use 1557 * case 1558 * 1559 * Note 3: 1560 * Section 1 in practical terms can no longer be used on rev A2. Because of 1561 * that the code does not need to handle section 1 at all. 1562 * 1563 */ 1564 #define BCM_SYS_EMI_START_ADDR 0x40000000UL 1565 #define BCM_VC_EMI_SEC3_START_ADDR 0xC0000000UL 1566 1567 #define BCM_SYS_EMI_OFFSET 0x40000000UL 1568 #define BCM_VC_EMI_OFFSET 0x80000000UL 1569 1570 static inline int bcm_addr_is_sys_emi(unsigned long addr) 1571 { 1572 return (addr >= BCM_SYS_EMI_START_ADDR) && 1573 (addr < BCM_VC_EMI_SEC3_START_ADDR); 1574 } 1575 1576 static inline unsigned long bcm_l2_phys_addr(unsigned long addr) 1577 { 1578 if (bcm_addr_is_sys_emi(addr)) 1579 return addr + BCM_SYS_EMI_OFFSET; 1580 else 1581 return addr + BCM_VC_EMI_OFFSET; 1582 } 1583 1584 static void bcm_inv_range(unsigned long start, unsigned long end) 1585 { 1586 unsigned long new_start, new_end; 1587 1588 BUG_ON(start < BCM_SYS_EMI_START_ADDR); 1589 1590 if (unlikely(end <= start)) 1591 return; 1592 1593 new_start = bcm_l2_phys_addr(start); 1594 new_end = bcm_l2_phys_addr(end); 1595 1596 /* normal case, no cross section between start and end */ 1597 if (likely(bcm_addr_is_sys_emi(end) || !bcm_addr_is_sys_emi(start))) { 1598 l2c210_inv_range(new_start, new_end); 1599 return; 1600 } 1601 1602 /* They cross sections, so it can only be a cross from section 1603 * 2 to section 3 1604 */ 1605 l2c210_inv_range(new_start, 1606 bcm_l2_phys_addr(BCM_VC_EMI_SEC3_START_ADDR-1)); 1607 l2c210_inv_range(bcm_l2_phys_addr(BCM_VC_EMI_SEC3_START_ADDR), 1608 new_end); 1609 } 1610 1611 static void bcm_clean_range(unsigned long start, unsigned long end) 1612 { 1613 unsigned long new_start, new_end; 1614 1615 BUG_ON(start < BCM_SYS_EMI_START_ADDR); 1616 1617 if (unlikely(end <= start)) 1618 return; 1619 1620 new_start = bcm_l2_phys_addr(start); 1621 new_end = bcm_l2_phys_addr(end); 1622 1623 /* normal case, no cross section between start and end */ 1624 if (likely(bcm_addr_is_sys_emi(end) || !bcm_addr_is_sys_emi(start))) { 1625 l2c210_clean_range(new_start, new_end); 1626 return; 1627 } 1628 1629 /* They cross sections, so it can only be a cross from section 1630 * 2 to section 3 1631 */ 1632 l2c210_clean_range(new_start, 1633 bcm_l2_phys_addr(BCM_VC_EMI_SEC3_START_ADDR-1)); 1634 l2c210_clean_range(bcm_l2_phys_addr(BCM_VC_EMI_SEC3_START_ADDR), 1635 new_end); 1636 } 1637 1638 static void bcm_flush_range(unsigned long start, unsigned long end) 1639 { 1640 unsigned long new_start, new_end; 1641 1642 BUG_ON(start < BCM_SYS_EMI_START_ADDR); 1643 1644 if (unlikely(end <= start)) 1645 return; 1646 1647 if ((end - start) >= l2x0_size) { 1648 outer_cache.flush_all(); 1649 return; 1650 } 1651 1652 new_start = bcm_l2_phys_addr(start); 1653 new_end = bcm_l2_phys_addr(end); 1654 1655 /* normal case, no cross section between start and end */ 1656 if (likely(bcm_addr_is_sys_emi(end) || !bcm_addr_is_sys_emi(start))) { 1657 l2c210_flush_range(new_start, new_end); 1658 return; 1659 } 1660 1661 /* They cross sections, so it can only be a cross from section 1662 * 2 to section 3 1663 */ 1664 l2c210_flush_range(new_start, 1665 bcm_l2_phys_addr(BCM_VC_EMI_SEC3_START_ADDR-1)); 1666 l2c210_flush_range(bcm_l2_phys_addr(BCM_VC_EMI_SEC3_START_ADDR), 1667 new_end); 1668 } 1669 1670 /* Broadcom L2C-310 start from ARMs R3P2 or later, and require no fixups */ 1671 static const struct l2c_init_data of_bcm_l2x0_data __initconst = { 1672 .type = "BCM-L2C-310", 1673 .way_size_0 = SZ_8K, 1674 .num_lock = 8, 1675 .of_parse = l2c310_of_parse, 1676 .enable = l2c310_enable, 1677 .save = l2c310_save, 1678 .configure = l2c310_configure, 1679 .unlock = l2c310_unlock, 1680 .outer_cache = { 1681 .inv_range = bcm_inv_range, 1682 .clean_range = bcm_clean_range, 1683 .flush_range = bcm_flush_range, 1684 .flush_all = l2c210_flush_all, 1685 .disable = l2c310_disable, 1686 .sync = l2c210_sync, 1687 .resume = l2c310_resume, 1688 }, 1689 }; 1690 1691 static void __init tauros3_save(void __iomem *base) 1692 { 1693 l2c_save(base); 1694 1695 l2x0_saved_regs.aux2_ctrl = 1696 readl_relaxed(base + TAUROS3_AUX2_CTRL); 1697 l2x0_saved_regs.prefetch_ctrl = 1698 readl_relaxed(base + L310_PREFETCH_CTRL); 1699 } 1700 1701 static void tauros3_configure(void __iomem *base) 1702 { 1703 l2c_configure(base); 1704 writel_relaxed(l2x0_saved_regs.aux2_ctrl, 1705 base + TAUROS3_AUX2_CTRL); 1706 writel_relaxed(l2x0_saved_regs.prefetch_ctrl, 1707 base + L310_PREFETCH_CTRL); 1708 } 1709 1710 static const struct l2c_init_data of_tauros3_data __initconst = { 1711 .type = "Tauros3", 1712 .way_size_0 = SZ_8K, 1713 .num_lock = 8, 1714 .enable = l2c_enable, 1715 .save = tauros3_save, 1716 .configure = tauros3_configure, 1717 .unlock = l2c_unlock, 1718 /* Tauros3 broadcasts L1 cache operations to L2 */ 1719 .outer_cache = { 1720 .resume = l2c_resume, 1721 }, 1722 }; 1723 1724 #define L2C_ID(name, fns) { .compatible = name, .data = (void *)&fns } 1725 static const struct of_device_id l2x0_ids[] __initconst = { 1726 L2C_ID("arm,l210-cache", of_l2c210_data), 1727 L2C_ID("arm,l220-cache", of_l2c220_data), 1728 L2C_ID("arm,pl310-cache", of_l2c310_data), 1729 L2C_ID("brcm,bcm11351-a2-pl310-cache", of_bcm_l2x0_data), 1730 L2C_ID("marvell,aurora-outer-cache", of_aurora_with_outer_data), 1731 L2C_ID("marvell,aurora-system-cache", of_aurora_no_outer_data), 1732 L2C_ID("marvell,tauros3-cache", of_tauros3_data), 1733 /* Deprecated IDs */ 1734 L2C_ID("bcm,bcm11351-a2-pl310-cache", of_bcm_l2x0_data), 1735 {} 1736 }; 1737 1738 int __init l2x0_of_init(u32 aux_val, u32 aux_mask) 1739 { 1740 const struct l2c_init_data *data; 1741 struct device_node *np; 1742 struct resource res; 1743 u32 cache_id, old_aux; 1744 u32 cache_level = 2; 1745 bool nosync = false; 1746 1747 np = of_find_matching_node(NULL, l2x0_ids); 1748 if (!np) 1749 return -ENODEV; 1750 1751 if (of_address_to_resource(np, 0, &res)) 1752 return -ENODEV; 1753 1754 l2x0_base = ioremap(res.start, resource_size(&res)); 1755 if (!l2x0_base) 1756 return -ENOMEM; 1757 1758 l2x0_saved_regs.phy_base = res.start; 1759 1760 data = of_match_node(l2x0_ids, np)->data; 1761 1762 if (of_device_is_compatible(np, "arm,pl310-cache") && 1763 of_property_read_bool(np, "arm,io-coherent")) 1764 data = &of_l2c310_coherent_data; 1765 1766 old_aux = readl_relaxed(l2x0_base + L2X0_AUX_CTRL); 1767 if (old_aux != ((old_aux & aux_mask) | aux_val)) { 1768 pr_warn("L2C: platform modifies aux control register: 0x%08x -> 0x%08x\n", 1769 old_aux, (old_aux & aux_mask) | aux_val); 1770 } else if (aux_mask != ~0U && aux_val != 0) { 1771 pr_alert("L2C: platform provided aux values match the hardware, so have no effect. Please remove them.\n"); 1772 } 1773 1774 /* All L2 caches are unified, so this property should be specified */ 1775 if (!of_property_read_bool(np, "cache-unified")) 1776 pr_err("L2C: device tree omits to specify unified cache\n"); 1777 1778 if (of_property_read_u32(np, "cache-level", &cache_level)) 1779 pr_err("L2C: device tree omits to specify cache-level\n"); 1780 1781 if (cache_level != 2) 1782 pr_err("L2C: device tree specifies invalid cache level\n"); 1783 1784 nosync = of_property_read_bool(np, "arm,outer-sync-disable"); 1785 1786 /* Read back current (default) hardware configuration */ 1787 if (data->save) 1788 data->save(l2x0_base); 1789 1790 /* L2 configuration can only be changed if the cache is disabled */ 1791 if (!(readl_relaxed(l2x0_base + L2X0_CTRL) & L2X0_CTRL_EN)) 1792 if (data->of_parse) 1793 data->of_parse(np, &aux_val, &aux_mask); 1794 1795 if (cache_id_part_number_from_dt) 1796 cache_id = cache_id_part_number_from_dt; 1797 else 1798 cache_id = readl_relaxed(l2x0_base + L2X0_CACHE_ID); 1799 1800 return __l2c_init(data, aux_val, aux_mask, cache_id, nosync); 1801 } 1802 #endif 1803