1 /* 2 * arch/arm/mm/cache-l2x0.c - L210/L220/L310 cache controller support 3 * 4 * Copyright (C) 2007 ARM Limited 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 * 10 * This program is distributed in the hope that it will be useful, 11 * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 * GNU General Public License for more details. 14 * 15 * You should have received a copy of the GNU General Public License 16 * along with this program; if not, write to the Free Software 17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 18 */ 19 #include <linux/cpu.h> 20 #include <linux/err.h> 21 #include <linux/init.h> 22 #include <linux/smp.h> 23 #include <linux/spinlock.h> 24 #include <linux/log2.h> 25 #include <linux/io.h> 26 #include <linux/of.h> 27 #include <linux/of_address.h> 28 29 #include <asm/cacheflush.h> 30 #include <asm/cp15.h> 31 #include <asm/cputype.h> 32 #include <asm/hardware/cache-l2x0.h> 33 #include "cache-tauros3.h" 34 #include "cache-aurora-l2.h" 35 36 struct l2c_init_data { 37 const char *type; 38 unsigned way_size_0; 39 unsigned num_lock; 40 void (*of_parse)(const struct device_node *, u32 *, u32 *); 41 void (*enable)(void __iomem *, unsigned); 42 void (*fixup)(void __iomem *, u32, struct outer_cache_fns *); 43 void (*save)(void __iomem *); 44 void (*configure)(void __iomem *); 45 void (*unlock)(void __iomem *, unsigned); 46 struct outer_cache_fns outer_cache; 47 }; 48 49 #define CACHE_LINE_SIZE 32 50 51 static void __iomem *l2x0_base; 52 static const struct l2c_init_data *l2x0_data; 53 static DEFINE_RAW_SPINLOCK(l2x0_lock); 54 static u32 l2x0_way_mask; /* Bitmask of active ways */ 55 static u32 l2x0_size; 56 static unsigned long sync_reg_offset = L2X0_CACHE_SYNC; 57 58 struct l2x0_regs l2x0_saved_regs; 59 60 /* 61 * Common code for all cache controllers. 62 */ 63 static inline void l2c_wait_mask(void __iomem *reg, unsigned long mask) 64 { 65 /* wait for cache operation by line or way to complete */ 66 while (readl_relaxed(reg) & mask) 67 cpu_relax(); 68 } 69 70 /* 71 * By default, we write directly to secure registers. Platforms must 72 * override this if they are running non-secure. 73 */ 74 static void l2c_write_sec(unsigned long val, void __iomem *base, unsigned reg) 75 { 76 if (val == readl_relaxed(base + reg)) 77 return; 78 if (outer_cache.write_sec) 79 outer_cache.write_sec(val, reg); 80 else 81 writel_relaxed(val, base + reg); 82 } 83 84 /* 85 * This should only be called when we have a requirement that the 86 * register be written due to a work-around, as platforms running 87 * in non-secure mode may not be able to access this register. 88 */ 89 static inline void l2c_set_debug(void __iomem *base, unsigned long val) 90 { 91 l2c_write_sec(val, base, L2X0_DEBUG_CTRL); 92 } 93 94 static void __l2c_op_way(void __iomem *reg) 95 { 96 writel_relaxed(l2x0_way_mask, reg); 97 l2c_wait_mask(reg, l2x0_way_mask); 98 } 99 100 static inline void l2c_unlock(void __iomem *base, unsigned num) 101 { 102 unsigned i; 103 104 for (i = 0; i < num; i++) { 105 writel_relaxed(0, base + L2X0_LOCKDOWN_WAY_D_BASE + 106 i * L2X0_LOCKDOWN_STRIDE); 107 writel_relaxed(0, base + L2X0_LOCKDOWN_WAY_I_BASE + 108 i * L2X0_LOCKDOWN_STRIDE); 109 } 110 } 111 112 static void l2c_configure(void __iomem *base) 113 { 114 l2c_write_sec(l2x0_saved_regs.aux_ctrl, base, L2X0_AUX_CTRL); 115 } 116 117 /* 118 * Enable the L2 cache controller. This function must only be 119 * called when the cache controller is known to be disabled. 120 */ 121 static void l2c_enable(void __iomem *base, unsigned num_lock) 122 { 123 unsigned long flags; 124 125 if (outer_cache.configure) 126 outer_cache.configure(&l2x0_saved_regs); 127 else 128 l2x0_data->configure(base); 129 130 l2x0_data->unlock(base, num_lock); 131 132 local_irq_save(flags); 133 __l2c_op_way(base + L2X0_INV_WAY); 134 writel_relaxed(0, base + sync_reg_offset); 135 l2c_wait_mask(base + sync_reg_offset, 1); 136 local_irq_restore(flags); 137 138 l2c_write_sec(L2X0_CTRL_EN, base, L2X0_CTRL); 139 } 140 141 static void l2c_disable(void) 142 { 143 void __iomem *base = l2x0_base; 144 145 outer_cache.flush_all(); 146 l2c_write_sec(0, base, L2X0_CTRL); 147 dsb(st); 148 } 149 150 static void l2c_save(void __iomem *base) 151 { 152 l2x0_saved_regs.aux_ctrl = readl_relaxed(l2x0_base + L2X0_AUX_CTRL); 153 } 154 155 static void l2c_resume(void) 156 { 157 void __iomem *base = l2x0_base; 158 159 /* Do not touch the controller if already enabled. */ 160 if (!(readl_relaxed(base + L2X0_CTRL) & L2X0_CTRL_EN)) 161 l2c_enable(base, l2x0_data->num_lock); 162 } 163 164 /* 165 * L2C-210 specific code. 166 * 167 * The L2C-2x0 PA, set/way and sync operations are atomic, but we must 168 * ensure that no background operation is running. The way operations 169 * are all background tasks. 170 * 171 * While a background operation is in progress, any new operation is 172 * ignored (unspecified whether this causes an error.) Thankfully, not 173 * used on SMP. 174 * 175 * Never has a different sync register other than L2X0_CACHE_SYNC, but 176 * we use sync_reg_offset here so we can share some of this with L2C-310. 177 */ 178 static void __l2c210_cache_sync(void __iomem *base) 179 { 180 writel_relaxed(0, base + sync_reg_offset); 181 } 182 183 static void __l2c210_op_pa_range(void __iomem *reg, unsigned long start, 184 unsigned long end) 185 { 186 while (start < end) { 187 writel_relaxed(start, reg); 188 start += CACHE_LINE_SIZE; 189 } 190 } 191 192 static void l2c210_inv_range(unsigned long start, unsigned long end) 193 { 194 void __iomem *base = l2x0_base; 195 196 if (start & (CACHE_LINE_SIZE - 1)) { 197 start &= ~(CACHE_LINE_SIZE - 1); 198 writel_relaxed(start, base + L2X0_CLEAN_INV_LINE_PA); 199 start += CACHE_LINE_SIZE; 200 } 201 202 if (end & (CACHE_LINE_SIZE - 1)) { 203 end &= ~(CACHE_LINE_SIZE - 1); 204 writel_relaxed(end, base + L2X0_CLEAN_INV_LINE_PA); 205 } 206 207 __l2c210_op_pa_range(base + L2X0_INV_LINE_PA, start, end); 208 __l2c210_cache_sync(base); 209 } 210 211 static void l2c210_clean_range(unsigned long start, unsigned long end) 212 { 213 void __iomem *base = l2x0_base; 214 215 start &= ~(CACHE_LINE_SIZE - 1); 216 __l2c210_op_pa_range(base + L2X0_CLEAN_LINE_PA, start, end); 217 __l2c210_cache_sync(base); 218 } 219 220 static void l2c210_flush_range(unsigned long start, unsigned long end) 221 { 222 void __iomem *base = l2x0_base; 223 224 start &= ~(CACHE_LINE_SIZE - 1); 225 __l2c210_op_pa_range(base + L2X0_CLEAN_INV_LINE_PA, start, end); 226 __l2c210_cache_sync(base); 227 } 228 229 static void l2c210_flush_all(void) 230 { 231 void __iomem *base = l2x0_base; 232 233 BUG_ON(!irqs_disabled()); 234 235 __l2c_op_way(base + L2X0_CLEAN_INV_WAY); 236 __l2c210_cache_sync(base); 237 } 238 239 static void l2c210_sync(void) 240 { 241 __l2c210_cache_sync(l2x0_base); 242 } 243 244 static const struct l2c_init_data l2c210_data __initconst = { 245 .type = "L2C-210", 246 .way_size_0 = SZ_8K, 247 .num_lock = 1, 248 .enable = l2c_enable, 249 .save = l2c_save, 250 .configure = l2c_configure, 251 .unlock = l2c_unlock, 252 .outer_cache = { 253 .inv_range = l2c210_inv_range, 254 .clean_range = l2c210_clean_range, 255 .flush_range = l2c210_flush_range, 256 .flush_all = l2c210_flush_all, 257 .disable = l2c_disable, 258 .sync = l2c210_sync, 259 .resume = l2c_resume, 260 }, 261 }; 262 263 /* 264 * L2C-220 specific code. 265 * 266 * All operations are background operations: they have to be waited for. 267 * Conflicting requests generate a slave error (which will cause an 268 * imprecise abort.) Never uses sync_reg_offset, so we hard-code the 269 * sync register here. 270 * 271 * However, we can re-use the l2c210_resume call. 272 */ 273 static inline void __l2c220_cache_sync(void __iomem *base) 274 { 275 writel_relaxed(0, base + L2X0_CACHE_SYNC); 276 l2c_wait_mask(base + L2X0_CACHE_SYNC, 1); 277 } 278 279 static void l2c220_op_way(void __iomem *base, unsigned reg) 280 { 281 unsigned long flags; 282 283 raw_spin_lock_irqsave(&l2x0_lock, flags); 284 __l2c_op_way(base + reg); 285 __l2c220_cache_sync(base); 286 raw_spin_unlock_irqrestore(&l2x0_lock, flags); 287 } 288 289 static unsigned long l2c220_op_pa_range(void __iomem *reg, unsigned long start, 290 unsigned long end, unsigned long flags) 291 { 292 raw_spinlock_t *lock = &l2x0_lock; 293 294 while (start < end) { 295 unsigned long blk_end = start + min(end - start, 4096UL); 296 297 while (start < blk_end) { 298 l2c_wait_mask(reg, 1); 299 writel_relaxed(start, reg); 300 start += CACHE_LINE_SIZE; 301 } 302 303 if (blk_end < end) { 304 raw_spin_unlock_irqrestore(lock, flags); 305 raw_spin_lock_irqsave(lock, flags); 306 } 307 } 308 309 return flags; 310 } 311 312 static void l2c220_inv_range(unsigned long start, unsigned long end) 313 { 314 void __iomem *base = l2x0_base; 315 unsigned long flags; 316 317 raw_spin_lock_irqsave(&l2x0_lock, flags); 318 if ((start | end) & (CACHE_LINE_SIZE - 1)) { 319 if (start & (CACHE_LINE_SIZE - 1)) { 320 start &= ~(CACHE_LINE_SIZE - 1); 321 writel_relaxed(start, base + L2X0_CLEAN_INV_LINE_PA); 322 start += CACHE_LINE_SIZE; 323 } 324 325 if (end & (CACHE_LINE_SIZE - 1)) { 326 end &= ~(CACHE_LINE_SIZE - 1); 327 l2c_wait_mask(base + L2X0_CLEAN_INV_LINE_PA, 1); 328 writel_relaxed(end, base + L2X0_CLEAN_INV_LINE_PA); 329 } 330 } 331 332 flags = l2c220_op_pa_range(base + L2X0_INV_LINE_PA, 333 start, end, flags); 334 l2c_wait_mask(base + L2X0_INV_LINE_PA, 1); 335 __l2c220_cache_sync(base); 336 raw_spin_unlock_irqrestore(&l2x0_lock, flags); 337 } 338 339 static void l2c220_clean_range(unsigned long start, unsigned long end) 340 { 341 void __iomem *base = l2x0_base; 342 unsigned long flags; 343 344 start &= ~(CACHE_LINE_SIZE - 1); 345 if ((end - start) >= l2x0_size) { 346 l2c220_op_way(base, L2X0_CLEAN_WAY); 347 return; 348 } 349 350 raw_spin_lock_irqsave(&l2x0_lock, flags); 351 flags = l2c220_op_pa_range(base + L2X0_CLEAN_LINE_PA, 352 start, end, flags); 353 l2c_wait_mask(base + L2X0_CLEAN_INV_LINE_PA, 1); 354 __l2c220_cache_sync(base); 355 raw_spin_unlock_irqrestore(&l2x0_lock, flags); 356 } 357 358 static void l2c220_flush_range(unsigned long start, unsigned long end) 359 { 360 void __iomem *base = l2x0_base; 361 unsigned long flags; 362 363 start &= ~(CACHE_LINE_SIZE - 1); 364 if ((end - start) >= l2x0_size) { 365 l2c220_op_way(base, L2X0_CLEAN_INV_WAY); 366 return; 367 } 368 369 raw_spin_lock_irqsave(&l2x0_lock, flags); 370 flags = l2c220_op_pa_range(base + L2X0_CLEAN_INV_LINE_PA, 371 start, end, flags); 372 l2c_wait_mask(base + L2X0_CLEAN_INV_LINE_PA, 1); 373 __l2c220_cache_sync(base); 374 raw_spin_unlock_irqrestore(&l2x0_lock, flags); 375 } 376 377 static void l2c220_flush_all(void) 378 { 379 l2c220_op_way(l2x0_base, L2X0_CLEAN_INV_WAY); 380 } 381 382 static void l2c220_sync(void) 383 { 384 unsigned long flags; 385 386 raw_spin_lock_irqsave(&l2x0_lock, flags); 387 __l2c220_cache_sync(l2x0_base); 388 raw_spin_unlock_irqrestore(&l2x0_lock, flags); 389 } 390 391 static void l2c220_enable(void __iomem *base, unsigned num_lock) 392 { 393 /* 394 * Always enable non-secure access to the lockdown registers - 395 * we write to them as part of the L2C enable sequence so they 396 * need to be accessible. 397 */ 398 l2x0_saved_regs.aux_ctrl |= L220_AUX_CTRL_NS_LOCKDOWN; 399 400 l2c_enable(base, num_lock); 401 } 402 403 static void l2c220_unlock(void __iomem *base, unsigned num_lock) 404 { 405 if (readl_relaxed(base + L2X0_AUX_CTRL) & L220_AUX_CTRL_NS_LOCKDOWN) 406 l2c_unlock(base, num_lock); 407 } 408 409 static const struct l2c_init_data l2c220_data = { 410 .type = "L2C-220", 411 .way_size_0 = SZ_8K, 412 .num_lock = 1, 413 .enable = l2c220_enable, 414 .save = l2c_save, 415 .configure = l2c_configure, 416 .unlock = l2c220_unlock, 417 .outer_cache = { 418 .inv_range = l2c220_inv_range, 419 .clean_range = l2c220_clean_range, 420 .flush_range = l2c220_flush_range, 421 .flush_all = l2c220_flush_all, 422 .disable = l2c_disable, 423 .sync = l2c220_sync, 424 .resume = l2c_resume, 425 }, 426 }; 427 428 /* 429 * L2C-310 specific code. 430 * 431 * Very similar to L2C-210, the PA, set/way and sync operations are atomic, 432 * and the way operations are all background tasks. However, issuing an 433 * operation while a background operation is in progress results in a 434 * SLVERR response. We can reuse: 435 * 436 * __l2c210_cache_sync (using sync_reg_offset) 437 * l2c210_sync 438 * l2c210_inv_range (if 588369 is not applicable) 439 * l2c210_clean_range 440 * l2c210_flush_range (if 588369 is not applicable) 441 * l2c210_flush_all (if 727915 is not applicable) 442 * 443 * Errata: 444 * 588369: PL310 R0P0->R1P0, fixed R2P0. 445 * Affects: all clean+invalidate operations 446 * clean and invalidate skips the invalidate step, so we need to issue 447 * separate operations. We also require the above debug workaround 448 * enclosing this code fragment on affected parts. On unaffected parts, 449 * we must not use this workaround without the debug register writes 450 * to avoid exposing a problem similar to 727915. 451 * 452 * 727915: PL310 R2P0->R3P0, fixed R3P1. 453 * Affects: clean+invalidate by way 454 * clean and invalidate by way runs in the background, and a store can 455 * hit the line between the clean operation and invalidate operation, 456 * resulting in the store being lost. 457 * 458 * 752271: PL310 R3P0->R3P1-50REL0, fixed R3P2. 459 * Affects: 8x64-bit (double fill) line fetches 460 * double fill line fetches can fail to cause dirty data to be evicted 461 * from the cache before the new data overwrites the second line. 462 * 463 * 753970: PL310 R3P0, fixed R3P1. 464 * Affects: sync 465 * prevents merging writes after the sync operation, until another L2C 466 * operation is performed (or a number of other conditions.) 467 * 468 * 769419: PL310 R0P0->R3P1, fixed R3P2. 469 * Affects: store buffer 470 * store buffer is not automatically drained. 471 */ 472 static void l2c310_inv_range_erratum(unsigned long start, unsigned long end) 473 { 474 void __iomem *base = l2x0_base; 475 476 if ((start | end) & (CACHE_LINE_SIZE - 1)) { 477 unsigned long flags; 478 479 /* Erratum 588369 for both clean+invalidate operations */ 480 raw_spin_lock_irqsave(&l2x0_lock, flags); 481 l2c_set_debug(base, 0x03); 482 483 if (start & (CACHE_LINE_SIZE - 1)) { 484 start &= ~(CACHE_LINE_SIZE - 1); 485 writel_relaxed(start, base + L2X0_CLEAN_LINE_PA); 486 writel_relaxed(start, base + L2X0_INV_LINE_PA); 487 start += CACHE_LINE_SIZE; 488 } 489 490 if (end & (CACHE_LINE_SIZE - 1)) { 491 end &= ~(CACHE_LINE_SIZE - 1); 492 writel_relaxed(end, base + L2X0_CLEAN_LINE_PA); 493 writel_relaxed(end, base + L2X0_INV_LINE_PA); 494 } 495 496 l2c_set_debug(base, 0x00); 497 raw_spin_unlock_irqrestore(&l2x0_lock, flags); 498 } 499 500 __l2c210_op_pa_range(base + L2X0_INV_LINE_PA, start, end); 501 __l2c210_cache_sync(base); 502 } 503 504 static void l2c310_flush_range_erratum(unsigned long start, unsigned long end) 505 { 506 raw_spinlock_t *lock = &l2x0_lock; 507 unsigned long flags; 508 void __iomem *base = l2x0_base; 509 510 raw_spin_lock_irqsave(lock, flags); 511 while (start < end) { 512 unsigned long blk_end = start + min(end - start, 4096UL); 513 514 l2c_set_debug(base, 0x03); 515 while (start < blk_end) { 516 writel_relaxed(start, base + L2X0_CLEAN_LINE_PA); 517 writel_relaxed(start, base + L2X0_INV_LINE_PA); 518 start += CACHE_LINE_SIZE; 519 } 520 l2c_set_debug(base, 0x00); 521 522 if (blk_end < end) { 523 raw_spin_unlock_irqrestore(lock, flags); 524 raw_spin_lock_irqsave(lock, flags); 525 } 526 } 527 raw_spin_unlock_irqrestore(lock, flags); 528 __l2c210_cache_sync(base); 529 } 530 531 static void l2c310_flush_all_erratum(void) 532 { 533 void __iomem *base = l2x0_base; 534 unsigned long flags; 535 536 raw_spin_lock_irqsave(&l2x0_lock, flags); 537 l2c_set_debug(base, 0x03); 538 __l2c_op_way(base + L2X0_CLEAN_INV_WAY); 539 l2c_set_debug(base, 0x00); 540 __l2c210_cache_sync(base); 541 raw_spin_unlock_irqrestore(&l2x0_lock, flags); 542 } 543 544 static void __init l2c310_save(void __iomem *base) 545 { 546 unsigned revision; 547 548 l2c_save(base); 549 550 l2x0_saved_regs.tag_latency = readl_relaxed(base + 551 L310_TAG_LATENCY_CTRL); 552 l2x0_saved_regs.data_latency = readl_relaxed(base + 553 L310_DATA_LATENCY_CTRL); 554 l2x0_saved_regs.filter_end = readl_relaxed(base + 555 L310_ADDR_FILTER_END); 556 l2x0_saved_regs.filter_start = readl_relaxed(base + 557 L310_ADDR_FILTER_START); 558 559 revision = readl_relaxed(base + L2X0_CACHE_ID) & 560 L2X0_CACHE_ID_RTL_MASK; 561 562 /* From r2p0, there is Prefetch offset/control register */ 563 if (revision >= L310_CACHE_ID_RTL_R2P0) 564 l2x0_saved_regs.prefetch_ctrl = readl_relaxed(base + 565 L310_PREFETCH_CTRL); 566 567 /* From r3p0, there is Power control register */ 568 if (revision >= L310_CACHE_ID_RTL_R3P0) 569 l2x0_saved_regs.pwr_ctrl = readl_relaxed(base + 570 L310_POWER_CTRL); 571 } 572 573 static void l2c310_configure(void __iomem *base) 574 { 575 unsigned revision; 576 577 l2c_configure(base); 578 579 /* restore pl310 setup */ 580 l2c_write_sec(l2x0_saved_regs.tag_latency, base, 581 L310_TAG_LATENCY_CTRL); 582 l2c_write_sec(l2x0_saved_regs.data_latency, base, 583 L310_DATA_LATENCY_CTRL); 584 l2c_write_sec(l2x0_saved_regs.filter_end, base, 585 L310_ADDR_FILTER_END); 586 l2c_write_sec(l2x0_saved_regs.filter_start, base, 587 L310_ADDR_FILTER_START); 588 589 revision = readl_relaxed(base + L2X0_CACHE_ID) & 590 L2X0_CACHE_ID_RTL_MASK; 591 592 if (revision >= L310_CACHE_ID_RTL_R2P0) 593 l2c_write_sec(l2x0_saved_regs.prefetch_ctrl, base, 594 L310_PREFETCH_CTRL); 595 if (revision >= L310_CACHE_ID_RTL_R3P0) 596 l2c_write_sec(l2x0_saved_regs.pwr_ctrl, base, 597 L310_POWER_CTRL); 598 } 599 600 static int l2c310_starting_cpu(unsigned int cpu) 601 { 602 set_auxcr(get_auxcr() | BIT(3) | BIT(2) | BIT(1)); 603 return 0; 604 } 605 606 static int l2c310_dying_cpu(unsigned int cpu) 607 { 608 set_auxcr(get_auxcr() & ~(BIT(3) | BIT(2) | BIT(1))); 609 return 0; 610 } 611 612 static void __init l2c310_enable(void __iomem *base, unsigned num_lock) 613 { 614 unsigned rev = readl_relaxed(base + L2X0_CACHE_ID) & L2X0_CACHE_ID_RTL_MASK; 615 bool cortex_a9 = read_cpuid_part() == ARM_CPU_PART_CORTEX_A9; 616 u32 aux = l2x0_saved_regs.aux_ctrl; 617 618 if (rev >= L310_CACHE_ID_RTL_R2P0) { 619 if (cortex_a9) { 620 aux |= L310_AUX_CTRL_EARLY_BRESP; 621 pr_info("L2C-310 enabling early BRESP for Cortex-A9\n"); 622 } else if (aux & L310_AUX_CTRL_EARLY_BRESP) { 623 pr_warn("L2C-310 early BRESP only supported with Cortex-A9\n"); 624 aux &= ~L310_AUX_CTRL_EARLY_BRESP; 625 } 626 } 627 628 if (cortex_a9) { 629 u32 aux_cur = readl_relaxed(base + L2X0_AUX_CTRL); 630 u32 acr = get_auxcr(); 631 632 pr_debug("Cortex-A9 ACR=0x%08x\n", acr); 633 634 if (acr & BIT(3) && !(aux_cur & L310_AUX_CTRL_FULL_LINE_ZERO)) 635 pr_err("L2C-310: full line of zeros enabled in Cortex-A9 but not L2C-310 - invalid\n"); 636 637 if (aux & L310_AUX_CTRL_FULL_LINE_ZERO && !(acr & BIT(3))) 638 pr_err("L2C-310: enabling full line of zeros but not enabled in Cortex-A9\n"); 639 640 if (!(aux & L310_AUX_CTRL_FULL_LINE_ZERO) && !outer_cache.write_sec) { 641 aux |= L310_AUX_CTRL_FULL_LINE_ZERO; 642 pr_info("L2C-310 full line of zeros enabled for Cortex-A9\n"); 643 } 644 } else if (aux & (L310_AUX_CTRL_FULL_LINE_ZERO | L310_AUX_CTRL_EARLY_BRESP)) { 645 pr_err("L2C-310: disabling Cortex-A9 specific feature bits\n"); 646 aux &= ~(L310_AUX_CTRL_FULL_LINE_ZERO | L310_AUX_CTRL_EARLY_BRESP); 647 } 648 649 /* 650 * Always enable non-secure access to the lockdown registers - 651 * we write to them as part of the L2C enable sequence so they 652 * need to be accessible. 653 */ 654 l2x0_saved_regs.aux_ctrl = aux | L310_AUX_CTRL_NS_LOCKDOWN; 655 656 l2c_enable(base, num_lock); 657 658 /* Read back resulting AUX_CTRL value as it could have been altered. */ 659 aux = readl_relaxed(base + L2X0_AUX_CTRL); 660 661 if (aux & (L310_AUX_CTRL_DATA_PREFETCH | L310_AUX_CTRL_INSTR_PREFETCH)) { 662 u32 prefetch = readl_relaxed(base + L310_PREFETCH_CTRL); 663 664 pr_info("L2C-310 %s%s prefetch enabled, offset %u lines\n", 665 aux & L310_AUX_CTRL_INSTR_PREFETCH ? "I" : "", 666 aux & L310_AUX_CTRL_DATA_PREFETCH ? "D" : "", 667 1 + (prefetch & L310_PREFETCH_CTRL_OFFSET_MASK)); 668 } 669 670 /* r3p0 or later has power control register */ 671 if (rev >= L310_CACHE_ID_RTL_R3P0) { 672 u32 power_ctrl; 673 674 power_ctrl = readl_relaxed(base + L310_POWER_CTRL); 675 pr_info("L2C-310 dynamic clock gating %sabled, standby mode %sabled\n", 676 power_ctrl & L310_DYNAMIC_CLK_GATING_EN ? "en" : "dis", 677 power_ctrl & L310_STNDBY_MODE_EN ? "en" : "dis"); 678 } 679 680 if (aux & L310_AUX_CTRL_FULL_LINE_ZERO) 681 cpuhp_setup_state(CPUHP_AP_ARM_L2X0_STARTING, 682 "AP_ARM_L2X0_STARTING", l2c310_starting_cpu, 683 l2c310_dying_cpu); 684 } 685 686 static void __init l2c310_fixup(void __iomem *base, u32 cache_id, 687 struct outer_cache_fns *fns) 688 { 689 unsigned revision = cache_id & L2X0_CACHE_ID_RTL_MASK; 690 const char *errata[8]; 691 unsigned n = 0; 692 693 if (IS_ENABLED(CONFIG_PL310_ERRATA_588369) && 694 revision < L310_CACHE_ID_RTL_R2P0 && 695 /* For bcm compatibility */ 696 fns->inv_range == l2c210_inv_range) { 697 fns->inv_range = l2c310_inv_range_erratum; 698 fns->flush_range = l2c310_flush_range_erratum; 699 errata[n++] = "588369"; 700 } 701 702 if (IS_ENABLED(CONFIG_PL310_ERRATA_727915) && 703 revision >= L310_CACHE_ID_RTL_R2P0 && 704 revision < L310_CACHE_ID_RTL_R3P1) { 705 fns->flush_all = l2c310_flush_all_erratum; 706 errata[n++] = "727915"; 707 } 708 709 if (revision >= L310_CACHE_ID_RTL_R3P0 && 710 revision < L310_CACHE_ID_RTL_R3P2) { 711 u32 val = l2x0_saved_regs.prefetch_ctrl; 712 /* I don't think bit23 is required here... but iMX6 does so */ 713 if (val & (L310_PREFETCH_CTRL_DBL_LINEFILL | 714 L310_PREFETCH_CTRL_DBL_LINEFILL_INCR)) { 715 val &= ~(L310_PREFETCH_CTRL_DBL_LINEFILL | 716 L310_PREFETCH_CTRL_DBL_LINEFILL_INCR); 717 l2x0_saved_regs.prefetch_ctrl = val; 718 errata[n++] = "752271"; 719 } 720 } 721 722 if (IS_ENABLED(CONFIG_PL310_ERRATA_753970) && 723 revision == L310_CACHE_ID_RTL_R3P0) { 724 sync_reg_offset = L2X0_DUMMY_REG; 725 errata[n++] = "753970"; 726 } 727 728 if (IS_ENABLED(CONFIG_PL310_ERRATA_769419)) 729 errata[n++] = "769419"; 730 731 if (n) { 732 unsigned i; 733 734 pr_info("L2C-310 errat%s", n > 1 ? "a" : "um"); 735 for (i = 0; i < n; i++) 736 pr_cont(" %s", errata[i]); 737 pr_cont(" enabled\n"); 738 } 739 } 740 741 static void l2c310_disable(void) 742 { 743 /* 744 * If full-line-of-zeros is enabled, we must first disable it in the 745 * Cortex-A9 auxiliary control register before disabling the L2 cache. 746 */ 747 if (l2x0_saved_regs.aux_ctrl & L310_AUX_CTRL_FULL_LINE_ZERO) 748 set_auxcr(get_auxcr() & ~(BIT(3) | BIT(2) | BIT(1))); 749 750 l2c_disable(); 751 } 752 753 static void l2c310_resume(void) 754 { 755 l2c_resume(); 756 757 /* Re-enable full-line-of-zeros for Cortex-A9 */ 758 if (l2x0_saved_regs.aux_ctrl & L310_AUX_CTRL_FULL_LINE_ZERO) 759 set_auxcr(get_auxcr() | BIT(3) | BIT(2) | BIT(1)); 760 } 761 762 static void l2c310_unlock(void __iomem *base, unsigned num_lock) 763 { 764 if (readl_relaxed(base + L2X0_AUX_CTRL) & L310_AUX_CTRL_NS_LOCKDOWN) 765 l2c_unlock(base, num_lock); 766 } 767 768 static const struct l2c_init_data l2c310_init_fns __initconst = { 769 .type = "L2C-310", 770 .way_size_0 = SZ_8K, 771 .num_lock = 8, 772 .enable = l2c310_enable, 773 .fixup = l2c310_fixup, 774 .save = l2c310_save, 775 .configure = l2c310_configure, 776 .unlock = l2c310_unlock, 777 .outer_cache = { 778 .inv_range = l2c210_inv_range, 779 .clean_range = l2c210_clean_range, 780 .flush_range = l2c210_flush_range, 781 .flush_all = l2c210_flush_all, 782 .disable = l2c310_disable, 783 .sync = l2c210_sync, 784 .resume = l2c310_resume, 785 }, 786 }; 787 788 static int __init __l2c_init(const struct l2c_init_data *data, 789 u32 aux_val, u32 aux_mask, u32 cache_id, bool nosync) 790 { 791 struct outer_cache_fns fns; 792 unsigned way_size_bits, ways; 793 u32 aux, old_aux; 794 795 /* 796 * Save the pointer globally so that callbacks which do not receive 797 * context from callers can access the structure. 798 */ 799 l2x0_data = kmemdup(data, sizeof(*data), GFP_KERNEL); 800 if (!l2x0_data) 801 return -ENOMEM; 802 803 /* 804 * Sanity check the aux values. aux_mask is the bits we preserve 805 * from reading the hardware register, and aux_val is the bits we 806 * set. 807 */ 808 if (aux_val & aux_mask) 809 pr_alert("L2C: platform provided aux values permit register corruption.\n"); 810 811 old_aux = aux = readl_relaxed(l2x0_base + L2X0_AUX_CTRL); 812 aux &= aux_mask; 813 aux |= aux_val; 814 815 if (old_aux != aux) 816 pr_warn("L2C: DT/platform modifies aux control register: 0x%08x -> 0x%08x\n", 817 old_aux, aux); 818 819 /* Determine the number of ways */ 820 switch (cache_id & L2X0_CACHE_ID_PART_MASK) { 821 case L2X0_CACHE_ID_PART_L310: 822 if ((aux_val | ~aux_mask) & (L2C_AUX_CTRL_WAY_SIZE_MASK | L310_AUX_CTRL_ASSOCIATIVITY_16)) 823 pr_warn("L2C: DT/platform tries to modify or specify cache size\n"); 824 if (aux & (1 << 16)) 825 ways = 16; 826 else 827 ways = 8; 828 break; 829 830 case L2X0_CACHE_ID_PART_L210: 831 case L2X0_CACHE_ID_PART_L220: 832 ways = (aux >> 13) & 0xf; 833 break; 834 835 case AURORA_CACHE_ID: 836 ways = (aux >> 13) & 0xf; 837 ways = 2 << ((ways + 1) >> 2); 838 break; 839 840 default: 841 /* Assume unknown chips have 8 ways */ 842 ways = 8; 843 break; 844 } 845 846 l2x0_way_mask = (1 << ways) - 1; 847 848 /* 849 * way_size_0 is the size that a way_size value of zero would be 850 * given the calculation: way_size = way_size_0 << way_size_bits. 851 * So, if way_size_bits=0 is reserved, but way_size_bits=1 is 16k, 852 * then way_size_0 would be 8k. 853 * 854 * L2 cache size = number of ways * way size. 855 */ 856 way_size_bits = (aux & L2C_AUX_CTRL_WAY_SIZE_MASK) >> 857 L2C_AUX_CTRL_WAY_SIZE_SHIFT; 858 l2x0_size = ways * (data->way_size_0 << way_size_bits); 859 860 fns = data->outer_cache; 861 fns.write_sec = outer_cache.write_sec; 862 fns.configure = outer_cache.configure; 863 if (data->fixup) 864 data->fixup(l2x0_base, cache_id, &fns); 865 if (nosync) { 866 pr_info("L2C: disabling outer sync\n"); 867 fns.sync = NULL; 868 } 869 870 /* 871 * Check if l2x0 controller is already enabled. If we are booting 872 * in non-secure mode accessing the below registers will fault. 873 */ 874 if (!(readl_relaxed(l2x0_base + L2X0_CTRL) & L2X0_CTRL_EN)) { 875 l2x0_saved_regs.aux_ctrl = aux; 876 877 data->enable(l2x0_base, data->num_lock); 878 } 879 880 outer_cache = fns; 881 882 /* 883 * It is strange to save the register state before initialisation, 884 * but hey, this is what the DT implementations decided to do. 885 */ 886 if (data->save) 887 data->save(l2x0_base); 888 889 /* Re-read it in case some bits are reserved. */ 890 aux = readl_relaxed(l2x0_base + L2X0_AUX_CTRL); 891 892 pr_info("%s cache controller enabled, %d ways, %d kB\n", 893 data->type, ways, l2x0_size >> 10); 894 pr_info("%s: CACHE_ID 0x%08x, AUX_CTRL 0x%08x\n", 895 data->type, cache_id, aux); 896 897 return 0; 898 } 899 900 void __init l2x0_init(void __iomem *base, u32 aux_val, u32 aux_mask) 901 { 902 const struct l2c_init_data *data; 903 u32 cache_id; 904 905 l2x0_base = base; 906 907 cache_id = readl_relaxed(base + L2X0_CACHE_ID); 908 909 switch (cache_id & L2X0_CACHE_ID_PART_MASK) { 910 default: 911 case L2X0_CACHE_ID_PART_L210: 912 data = &l2c210_data; 913 break; 914 915 case L2X0_CACHE_ID_PART_L220: 916 data = &l2c220_data; 917 break; 918 919 case L2X0_CACHE_ID_PART_L310: 920 data = &l2c310_init_fns; 921 break; 922 } 923 924 /* Read back current (default) hardware configuration */ 925 if (data->save) 926 data->save(l2x0_base); 927 928 __l2c_init(data, aux_val, aux_mask, cache_id, false); 929 } 930 931 #ifdef CONFIG_OF 932 static int l2_wt_override; 933 934 /* Aurora don't have the cache ID register available, so we have to 935 * pass it though the device tree */ 936 static u32 cache_id_part_number_from_dt; 937 938 /** 939 * l2x0_cache_size_of_parse() - read cache size parameters from DT 940 * @np: the device tree node for the l2 cache 941 * @aux_val: pointer to machine-supplied auxilary register value, to 942 * be augmented by the call (bits to be set to 1) 943 * @aux_mask: pointer to machine-supplied auxilary register mask, to 944 * be augmented by the call (bits to be set to 0) 945 * @associativity: variable to return the calculated associativity in 946 * @max_way_size: the maximum size in bytes for the cache ways 947 */ 948 static int __init l2x0_cache_size_of_parse(const struct device_node *np, 949 u32 *aux_val, u32 *aux_mask, 950 u32 *associativity, 951 u32 max_way_size) 952 { 953 u32 mask = 0, val = 0; 954 u32 cache_size = 0, sets = 0; 955 u32 way_size_bits = 1; 956 u32 way_size = 0; 957 u32 block_size = 0; 958 u32 line_size = 0; 959 960 of_property_read_u32(np, "cache-size", &cache_size); 961 of_property_read_u32(np, "cache-sets", &sets); 962 of_property_read_u32(np, "cache-block-size", &block_size); 963 of_property_read_u32(np, "cache-line-size", &line_size); 964 965 if (!cache_size || !sets) 966 return -ENODEV; 967 968 /* All these l2 caches have the same line = block size actually */ 969 if (!line_size) { 970 if (block_size) { 971 /* If linesize is not given, it is equal to blocksize */ 972 line_size = block_size; 973 } else { 974 /* Fall back to known size */ 975 pr_warn("L2C OF: no cache block/line size given: " 976 "falling back to default size %d bytes\n", 977 CACHE_LINE_SIZE); 978 line_size = CACHE_LINE_SIZE; 979 } 980 } 981 982 if (line_size != CACHE_LINE_SIZE) 983 pr_warn("L2C OF: DT supplied line size %d bytes does " 984 "not match hardware line size of %d bytes\n", 985 line_size, 986 CACHE_LINE_SIZE); 987 988 /* 989 * Since: 990 * set size = cache size / sets 991 * ways = cache size / (sets * line size) 992 * way size = cache size / (cache size / (sets * line size)) 993 * way size = sets * line size 994 * associativity = ways = cache size / way size 995 */ 996 way_size = sets * line_size; 997 *associativity = cache_size / way_size; 998 999 if (way_size > max_way_size) { 1000 pr_err("L2C OF: set size %dKB is too large\n", way_size); 1001 return -EINVAL; 1002 } 1003 1004 pr_info("L2C OF: override cache size: %d bytes (%dKB)\n", 1005 cache_size, cache_size >> 10); 1006 pr_info("L2C OF: override line size: %d bytes\n", line_size); 1007 pr_info("L2C OF: override way size: %d bytes (%dKB)\n", 1008 way_size, way_size >> 10); 1009 pr_info("L2C OF: override associativity: %d\n", *associativity); 1010 1011 /* 1012 * Calculates the bits 17:19 to set for way size: 1013 * 512KB -> 6, 256KB -> 5, ... 16KB -> 1 1014 */ 1015 way_size_bits = ilog2(way_size >> 10) - 3; 1016 if (way_size_bits < 1 || way_size_bits > 6) { 1017 pr_err("L2C OF: cache way size illegal: %dKB is not mapped\n", 1018 way_size); 1019 return -EINVAL; 1020 } 1021 1022 mask |= L2C_AUX_CTRL_WAY_SIZE_MASK; 1023 val |= (way_size_bits << L2C_AUX_CTRL_WAY_SIZE_SHIFT); 1024 1025 *aux_val &= ~mask; 1026 *aux_val |= val; 1027 *aux_mask &= ~mask; 1028 1029 return 0; 1030 } 1031 1032 static void __init l2x0_of_parse(const struct device_node *np, 1033 u32 *aux_val, u32 *aux_mask) 1034 { 1035 u32 data[2] = { 0, 0 }; 1036 u32 tag = 0; 1037 u32 dirty = 0; 1038 u32 val = 0, mask = 0; 1039 u32 assoc; 1040 int ret; 1041 1042 of_property_read_u32(np, "arm,tag-latency", &tag); 1043 if (tag) { 1044 mask |= L2X0_AUX_CTRL_TAG_LATENCY_MASK; 1045 val |= (tag - 1) << L2X0_AUX_CTRL_TAG_LATENCY_SHIFT; 1046 } 1047 1048 of_property_read_u32_array(np, "arm,data-latency", 1049 data, ARRAY_SIZE(data)); 1050 if (data[0] && data[1]) { 1051 mask |= L2X0_AUX_CTRL_DATA_RD_LATENCY_MASK | 1052 L2X0_AUX_CTRL_DATA_WR_LATENCY_MASK; 1053 val |= ((data[0] - 1) << L2X0_AUX_CTRL_DATA_RD_LATENCY_SHIFT) | 1054 ((data[1] - 1) << L2X0_AUX_CTRL_DATA_WR_LATENCY_SHIFT); 1055 } 1056 1057 of_property_read_u32(np, "arm,dirty-latency", &dirty); 1058 if (dirty) { 1059 mask |= L2X0_AUX_CTRL_DIRTY_LATENCY_MASK; 1060 val |= (dirty - 1) << L2X0_AUX_CTRL_DIRTY_LATENCY_SHIFT; 1061 } 1062 1063 if (of_property_read_bool(np, "arm,parity-enable")) { 1064 mask &= ~L2C_AUX_CTRL_PARITY_ENABLE; 1065 val |= L2C_AUX_CTRL_PARITY_ENABLE; 1066 } else if (of_property_read_bool(np, "arm,parity-disable")) { 1067 mask &= ~L2C_AUX_CTRL_PARITY_ENABLE; 1068 } 1069 1070 if (of_property_read_bool(np, "arm,shared-override")) { 1071 mask &= ~L2C_AUX_CTRL_SHARED_OVERRIDE; 1072 val |= L2C_AUX_CTRL_SHARED_OVERRIDE; 1073 } 1074 1075 ret = l2x0_cache_size_of_parse(np, aux_val, aux_mask, &assoc, SZ_256K); 1076 if (ret) 1077 return; 1078 1079 if (assoc > 8) { 1080 pr_err("l2x0 of: cache setting yield too high associativity\n"); 1081 pr_err("l2x0 of: %d calculated, max 8\n", assoc); 1082 } else { 1083 mask |= L2X0_AUX_CTRL_ASSOC_MASK; 1084 val |= (assoc << L2X0_AUX_CTRL_ASSOC_SHIFT); 1085 } 1086 1087 *aux_val &= ~mask; 1088 *aux_val |= val; 1089 *aux_mask &= ~mask; 1090 } 1091 1092 static const struct l2c_init_data of_l2c210_data __initconst = { 1093 .type = "L2C-210", 1094 .way_size_0 = SZ_8K, 1095 .num_lock = 1, 1096 .of_parse = l2x0_of_parse, 1097 .enable = l2c_enable, 1098 .save = l2c_save, 1099 .configure = l2c_configure, 1100 .unlock = l2c_unlock, 1101 .outer_cache = { 1102 .inv_range = l2c210_inv_range, 1103 .clean_range = l2c210_clean_range, 1104 .flush_range = l2c210_flush_range, 1105 .flush_all = l2c210_flush_all, 1106 .disable = l2c_disable, 1107 .sync = l2c210_sync, 1108 .resume = l2c_resume, 1109 }, 1110 }; 1111 1112 static const struct l2c_init_data of_l2c220_data __initconst = { 1113 .type = "L2C-220", 1114 .way_size_0 = SZ_8K, 1115 .num_lock = 1, 1116 .of_parse = l2x0_of_parse, 1117 .enable = l2c220_enable, 1118 .save = l2c_save, 1119 .configure = l2c_configure, 1120 .unlock = l2c220_unlock, 1121 .outer_cache = { 1122 .inv_range = l2c220_inv_range, 1123 .clean_range = l2c220_clean_range, 1124 .flush_range = l2c220_flush_range, 1125 .flush_all = l2c220_flush_all, 1126 .disable = l2c_disable, 1127 .sync = l2c220_sync, 1128 .resume = l2c_resume, 1129 }, 1130 }; 1131 1132 static void __init l2c310_of_parse(const struct device_node *np, 1133 u32 *aux_val, u32 *aux_mask) 1134 { 1135 u32 data[3] = { 0, 0, 0 }; 1136 u32 tag[3] = { 0, 0, 0 }; 1137 u32 filter[2] = { 0, 0 }; 1138 u32 assoc; 1139 u32 prefetch; 1140 u32 power; 1141 u32 val; 1142 int ret; 1143 1144 of_property_read_u32_array(np, "arm,tag-latency", tag, ARRAY_SIZE(tag)); 1145 if (tag[0] && tag[1] && tag[2]) 1146 l2x0_saved_regs.tag_latency = 1147 L310_LATENCY_CTRL_RD(tag[0] - 1) | 1148 L310_LATENCY_CTRL_WR(tag[1] - 1) | 1149 L310_LATENCY_CTRL_SETUP(tag[2] - 1); 1150 1151 of_property_read_u32_array(np, "arm,data-latency", 1152 data, ARRAY_SIZE(data)); 1153 if (data[0] && data[1] && data[2]) 1154 l2x0_saved_regs.data_latency = 1155 L310_LATENCY_CTRL_RD(data[0] - 1) | 1156 L310_LATENCY_CTRL_WR(data[1] - 1) | 1157 L310_LATENCY_CTRL_SETUP(data[2] - 1); 1158 1159 of_property_read_u32_array(np, "arm,filter-ranges", 1160 filter, ARRAY_SIZE(filter)); 1161 if (filter[1]) { 1162 l2x0_saved_regs.filter_end = 1163 ALIGN(filter[0] + filter[1], SZ_1M); 1164 l2x0_saved_regs.filter_start = (filter[0] & ~(SZ_1M - 1)) 1165 | L310_ADDR_FILTER_EN; 1166 } 1167 1168 ret = l2x0_cache_size_of_parse(np, aux_val, aux_mask, &assoc, SZ_512K); 1169 if (!ret) { 1170 switch (assoc) { 1171 case 16: 1172 *aux_val &= ~L2X0_AUX_CTRL_ASSOC_MASK; 1173 *aux_val |= L310_AUX_CTRL_ASSOCIATIVITY_16; 1174 *aux_mask &= ~L2X0_AUX_CTRL_ASSOC_MASK; 1175 break; 1176 case 8: 1177 *aux_val &= ~L2X0_AUX_CTRL_ASSOC_MASK; 1178 *aux_mask &= ~L2X0_AUX_CTRL_ASSOC_MASK; 1179 break; 1180 default: 1181 pr_err("L2C-310 OF cache associativity %d invalid, only 8 or 16 permitted\n", 1182 assoc); 1183 break; 1184 } 1185 } 1186 1187 if (of_property_read_bool(np, "arm,shared-override")) { 1188 *aux_val |= L2C_AUX_CTRL_SHARED_OVERRIDE; 1189 *aux_mask &= ~L2C_AUX_CTRL_SHARED_OVERRIDE; 1190 } 1191 1192 if (of_property_read_bool(np, "arm,parity-enable")) { 1193 *aux_val |= L2C_AUX_CTRL_PARITY_ENABLE; 1194 *aux_mask &= ~L2C_AUX_CTRL_PARITY_ENABLE; 1195 } else if (of_property_read_bool(np, "arm,parity-disable")) { 1196 *aux_val &= ~L2C_AUX_CTRL_PARITY_ENABLE; 1197 *aux_mask &= ~L2C_AUX_CTRL_PARITY_ENABLE; 1198 } 1199 1200 prefetch = l2x0_saved_regs.prefetch_ctrl; 1201 1202 ret = of_property_read_u32(np, "arm,double-linefill", &val); 1203 if (ret == 0) { 1204 if (val) 1205 prefetch |= L310_PREFETCH_CTRL_DBL_LINEFILL; 1206 else 1207 prefetch &= ~L310_PREFETCH_CTRL_DBL_LINEFILL; 1208 } else if (ret != -EINVAL) { 1209 pr_err("L2C-310 OF arm,double-linefill property value is missing\n"); 1210 } 1211 1212 ret = of_property_read_u32(np, "arm,double-linefill-incr", &val); 1213 if (ret == 0) { 1214 if (val) 1215 prefetch |= L310_PREFETCH_CTRL_DBL_LINEFILL_INCR; 1216 else 1217 prefetch &= ~L310_PREFETCH_CTRL_DBL_LINEFILL_INCR; 1218 } else if (ret != -EINVAL) { 1219 pr_err("L2C-310 OF arm,double-linefill-incr property value is missing\n"); 1220 } 1221 1222 ret = of_property_read_u32(np, "arm,double-linefill-wrap", &val); 1223 if (ret == 0) { 1224 if (!val) 1225 prefetch |= L310_PREFETCH_CTRL_DBL_LINEFILL_WRAP; 1226 else 1227 prefetch &= ~L310_PREFETCH_CTRL_DBL_LINEFILL_WRAP; 1228 } else if (ret != -EINVAL) { 1229 pr_err("L2C-310 OF arm,double-linefill-wrap property value is missing\n"); 1230 } 1231 1232 ret = of_property_read_u32(np, "arm,prefetch-drop", &val); 1233 if (ret == 0) { 1234 if (val) 1235 prefetch |= L310_PREFETCH_CTRL_PREFETCH_DROP; 1236 else 1237 prefetch &= ~L310_PREFETCH_CTRL_PREFETCH_DROP; 1238 } else if (ret != -EINVAL) { 1239 pr_err("L2C-310 OF arm,prefetch-drop property value is missing\n"); 1240 } 1241 1242 ret = of_property_read_u32(np, "arm,prefetch-offset", &val); 1243 if (ret == 0) { 1244 prefetch &= ~L310_PREFETCH_CTRL_OFFSET_MASK; 1245 prefetch |= val & L310_PREFETCH_CTRL_OFFSET_MASK; 1246 } else if (ret != -EINVAL) { 1247 pr_err("L2C-310 OF arm,prefetch-offset property value is missing\n"); 1248 } 1249 1250 ret = of_property_read_u32(np, "prefetch-data", &val); 1251 if (ret == 0) { 1252 if (val) 1253 prefetch |= L310_PREFETCH_CTRL_DATA_PREFETCH; 1254 else 1255 prefetch &= ~L310_PREFETCH_CTRL_DATA_PREFETCH; 1256 } else if (ret != -EINVAL) { 1257 pr_err("L2C-310 OF prefetch-data property value is missing\n"); 1258 } 1259 1260 ret = of_property_read_u32(np, "prefetch-instr", &val); 1261 if (ret == 0) { 1262 if (val) 1263 prefetch |= L310_PREFETCH_CTRL_INSTR_PREFETCH; 1264 else 1265 prefetch &= ~L310_PREFETCH_CTRL_INSTR_PREFETCH; 1266 } else if (ret != -EINVAL) { 1267 pr_err("L2C-310 OF prefetch-instr property value is missing\n"); 1268 } 1269 1270 l2x0_saved_regs.prefetch_ctrl = prefetch; 1271 1272 power = l2x0_saved_regs.pwr_ctrl | 1273 L310_DYNAMIC_CLK_GATING_EN | L310_STNDBY_MODE_EN; 1274 1275 ret = of_property_read_u32(np, "arm,dynamic-clock-gating", &val); 1276 if (!ret) { 1277 if (!val) 1278 power &= ~L310_DYNAMIC_CLK_GATING_EN; 1279 } else if (ret != -EINVAL) { 1280 pr_err("L2C-310 OF dynamic-clock-gating property value is missing or invalid\n"); 1281 } 1282 ret = of_property_read_u32(np, "arm,standby-mode", &val); 1283 if (!ret) { 1284 if (!val) 1285 power &= ~L310_STNDBY_MODE_EN; 1286 } else if (ret != -EINVAL) { 1287 pr_err("L2C-310 OF standby-mode property value is missing or invalid\n"); 1288 } 1289 1290 l2x0_saved_regs.pwr_ctrl = power; 1291 } 1292 1293 static const struct l2c_init_data of_l2c310_data __initconst = { 1294 .type = "L2C-310", 1295 .way_size_0 = SZ_8K, 1296 .num_lock = 8, 1297 .of_parse = l2c310_of_parse, 1298 .enable = l2c310_enable, 1299 .fixup = l2c310_fixup, 1300 .save = l2c310_save, 1301 .configure = l2c310_configure, 1302 .unlock = l2c310_unlock, 1303 .outer_cache = { 1304 .inv_range = l2c210_inv_range, 1305 .clean_range = l2c210_clean_range, 1306 .flush_range = l2c210_flush_range, 1307 .flush_all = l2c210_flush_all, 1308 .disable = l2c310_disable, 1309 .sync = l2c210_sync, 1310 .resume = l2c310_resume, 1311 }, 1312 }; 1313 1314 /* 1315 * This is a variant of the of_l2c310_data with .sync set to 1316 * NULL. Outer sync operations are not needed when the system is I/O 1317 * coherent, and potentially harmful in certain situations (PCIe/PL310 1318 * deadlock on Armada 375/38x due to hardware I/O coherency). The 1319 * other operations are kept because they are infrequent (therefore do 1320 * not cause the deadlock in practice) and needed for secondary CPU 1321 * boot and other power management activities. 1322 */ 1323 static const struct l2c_init_data of_l2c310_coherent_data __initconst = { 1324 .type = "L2C-310 Coherent", 1325 .way_size_0 = SZ_8K, 1326 .num_lock = 8, 1327 .of_parse = l2c310_of_parse, 1328 .enable = l2c310_enable, 1329 .fixup = l2c310_fixup, 1330 .save = l2c310_save, 1331 .configure = l2c310_configure, 1332 .unlock = l2c310_unlock, 1333 .outer_cache = { 1334 .inv_range = l2c210_inv_range, 1335 .clean_range = l2c210_clean_range, 1336 .flush_range = l2c210_flush_range, 1337 .flush_all = l2c210_flush_all, 1338 .disable = l2c310_disable, 1339 .resume = l2c310_resume, 1340 }, 1341 }; 1342 1343 /* 1344 * Note that the end addresses passed to Linux primitives are 1345 * noninclusive, while the hardware cache range operations use 1346 * inclusive start and end addresses. 1347 */ 1348 static unsigned long aurora_range_end(unsigned long start, unsigned long end) 1349 { 1350 /* 1351 * Limit the number of cache lines processed at once, 1352 * since cache range operations stall the CPU pipeline 1353 * until completion. 1354 */ 1355 if (end > start + MAX_RANGE_SIZE) 1356 end = start + MAX_RANGE_SIZE; 1357 1358 /* 1359 * Cache range operations can't straddle a page boundary. 1360 */ 1361 if (end > PAGE_ALIGN(start+1)) 1362 end = PAGE_ALIGN(start+1); 1363 1364 return end; 1365 } 1366 1367 static void aurora_pa_range(unsigned long start, unsigned long end, 1368 unsigned long offset) 1369 { 1370 void __iomem *base = l2x0_base; 1371 unsigned long range_end; 1372 unsigned long flags; 1373 1374 /* 1375 * round start and end adresses up to cache line size 1376 */ 1377 start &= ~(CACHE_LINE_SIZE - 1); 1378 end = ALIGN(end, CACHE_LINE_SIZE); 1379 1380 /* 1381 * perform operation on all full cache lines between 'start' and 'end' 1382 */ 1383 while (start < end) { 1384 range_end = aurora_range_end(start, end); 1385 1386 raw_spin_lock_irqsave(&l2x0_lock, flags); 1387 writel_relaxed(start, base + AURORA_RANGE_BASE_ADDR_REG); 1388 writel_relaxed(range_end - CACHE_LINE_SIZE, base + offset); 1389 raw_spin_unlock_irqrestore(&l2x0_lock, flags); 1390 1391 writel_relaxed(0, base + AURORA_SYNC_REG); 1392 start = range_end; 1393 } 1394 } 1395 static void aurora_inv_range(unsigned long start, unsigned long end) 1396 { 1397 aurora_pa_range(start, end, AURORA_INVAL_RANGE_REG); 1398 } 1399 1400 static void aurora_clean_range(unsigned long start, unsigned long end) 1401 { 1402 /* 1403 * If L2 is forced to WT, the L2 will always be clean and we 1404 * don't need to do anything here. 1405 */ 1406 if (!l2_wt_override) 1407 aurora_pa_range(start, end, AURORA_CLEAN_RANGE_REG); 1408 } 1409 1410 static void aurora_flush_range(unsigned long start, unsigned long end) 1411 { 1412 if (l2_wt_override) 1413 aurora_pa_range(start, end, AURORA_INVAL_RANGE_REG); 1414 else 1415 aurora_pa_range(start, end, AURORA_FLUSH_RANGE_REG); 1416 } 1417 1418 static void aurora_flush_all(void) 1419 { 1420 void __iomem *base = l2x0_base; 1421 unsigned long flags; 1422 1423 /* clean all ways */ 1424 raw_spin_lock_irqsave(&l2x0_lock, flags); 1425 __l2c_op_way(base + L2X0_CLEAN_INV_WAY); 1426 raw_spin_unlock_irqrestore(&l2x0_lock, flags); 1427 1428 writel_relaxed(0, base + AURORA_SYNC_REG); 1429 } 1430 1431 static void aurora_cache_sync(void) 1432 { 1433 writel_relaxed(0, l2x0_base + AURORA_SYNC_REG); 1434 } 1435 1436 static void aurora_disable(void) 1437 { 1438 void __iomem *base = l2x0_base; 1439 unsigned long flags; 1440 1441 raw_spin_lock_irqsave(&l2x0_lock, flags); 1442 __l2c_op_way(base + L2X0_CLEAN_INV_WAY); 1443 writel_relaxed(0, base + AURORA_SYNC_REG); 1444 l2c_write_sec(0, base, L2X0_CTRL); 1445 dsb(st); 1446 raw_spin_unlock_irqrestore(&l2x0_lock, flags); 1447 } 1448 1449 static void aurora_save(void __iomem *base) 1450 { 1451 l2x0_saved_regs.ctrl = readl_relaxed(base + L2X0_CTRL); 1452 l2x0_saved_regs.aux_ctrl = readl_relaxed(base + L2X0_AUX_CTRL); 1453 } 1454 1455 /* 1456 * For Aurora cache in no outer mode, enable via the CP15 coprocessor 1457 * broadcasting of cache commands to L2. 1458 */ 1459 static void __init aurora_enable_no_outer(void __iomem *base, 1460 unsigned num_lock) 1461 { 1462 u32 u; 1463 1464 asm volatile("mrc p15, 1, %0, c15, c2, 0" : "=r" (u)); 1465 u |= AURORA_CTRL_FW; /* Set the FW bit */ 1466 asm volatile("mcr p15, 1, %0, c15, c2, 0" : : "r" (u)); 1467 1468 isb(); 1469 1470 l2c_enable(base, num_lock); 1471 } 1472 1473 static void __init aurora_fixup(void __iomem *base, u32 cache_id, 1474 struct outer_cache_fns *fns) 1475 { 1476 sync_reg_offset = AURORA_SYNC_REG; 1477 } 1478 1479 static void __init aurora_of_parse(const struct device_node *np, 1480 u32 *aux_val, u32 *aux_mask) 1481 { 1482 u32 val = AURORA_ACR_REPLACEMENT_TYPE_SEMIPLRU; 1483 u32 mask = AURORA_ACR_REPLACEMENT_MASK; 1484 1485 of_property_read_u32(np, "cache-id-part", 1486 &cache_id_part_number_from_dt); 1487 1488 /* Determine and save the write policy */ 1489 l2_wt_override = of_property_read_bool(np, "wt-override"); 1490 1491 if (l2_wt_override) { 1492 val |= AURORA_ACR_FORCE_WRITE_THRO_POLICY; 1493 mask |= AURORA_ACR_FORCE_WRITE_POLICY_MASK; 1494 } 1495 1496 *aux_val &= ~mask; 1497 *aux_val |= val; 1498 *aux_mask &= ~mask; 1499 } 1500 1501 static const struct l2c_init_data of_aurora_with_outer_data __initconst = { 1502 .type = "Aurora", 1503 .way_size_0 = SZ_4K, 1504 .num_lock = 4, 1505 .of_parse = aurora_of_parse, 1506 .enable = l2c_enable, 1507 .fixup = aurora_fixup, 1508 .save = aurora_save, 1509 .configure = l2c_configure, 1510 .unlock = l2c_unlock, 1511 .outer_cache = { 1512 .inv_range = aurora_inv_range, 1513 .clean_range = aurora_clean_range, 1514 .flush_range = aurora_flush_range, 1515 .flush_all = aurora_flush_all, 1516 .disable = aurora_disable, 1517 .sync = aurora_cache_sync, 1518 .resume = l2c_resume, 1519 }, 1520 }; 1521 1522 static const struct l2c_init_data of_aurora_no_outer_data __initconst = { 1523 .type = "Aurora", 1524 .way_size_0 = SZ_4K, 1525 .num_lock = 4, 1526 .of_parse = aurora_of_parse, 1527 .enable = aurora_enable_no_outer, 1528 .fixup = aurora_fixup, 1529 .save = aurora_save, 1530 .configure = l2c_configure, 1531 .unlock = l2c_unlock, 1532 .outer_cache = { 1533 .resume = l2c_resume, 1534 }, 1535 }; 1536 1537 /* 1538 * For certain Broadcom SoCs, depending on the address range, different offsets 1539 * need to be added to the address before passing it to L2 for 1540 * invalidation/clean/flush 1541 * 1542 * Section Address Range Offset EMI 1543 * 1 0x00000000 - 0x3FFFFFFF 0x80000000 VC 1544 * 2 0x40000000 - 0xBFFFFFFF 0x40000000 SYS 1545 * 3 0xC0000000 - 0xFFFFFFFF 0x80000000 VC 1546 * 1547 * When the start and end addresses have crossed two different sections, we 1548 * need to break the L2 operation into two, each within its own section. 1549 * For example, if we need to invalidate addresses starts at 0xBFFF0000 and 1550 * ends at 0xC0001000, we need do invalidate 1) 0xBFFF0000 - 0xBFFFFFFF and 2) 1551 * 0xC0000000 - 0xC0001000 1552 * 1553 * Note 1: 1554 * By breaking a single L2 operation into two, we may potentially suffer some 1555 * performance hit, but keep in mind the cross section case is very rare 1556 * 1557 * Note 2: 1558 * We do not need to handle the case when the start address is in 1559 * Section 1 and the end address is in Section 3, since it is not a valid use 1560 * case 1561 * 1562 * Note 3: 1563 * Section 1 in practical terms can no longer be used on rev A2. Because of 1564 * that the code does not need to handle section 1 at all. 1565 * 1566 */ 1567 #define BCM_SYS_EMI_START_ADDR 0x40000000UL 1568 #define BCM_VC_EMI_SEC3_START_ADDR 0xC0000000UL 1569 1570 #define BCM_SYS_EMI_OFFSET 0x40000000UL 1571 #define BCM_VC_EMI_OFFSET 0x80000000UL 1572 1573 static inline int bcm_addr_is_sys_emi(unsigned long addr) 1574 { 1575 return (addr >= BCM_SYS_EMI_START_ADDR) && 1576 (addr < BCM_VC_EMI_SEC3_START_ADDR); 1577 } 1578 1579 static inline unsigned long bcm_l2_phys_addr(unsigned long addr) 1580 { 1581 if (bcm_addr_is_sys_emi(addr)) 1582 return addr + BCM_SYS_EMI_OFFSET; 1583 else 1584 return addr + BCM_VC_EMI_OFFSET; 1585 } 1586 1587 static void bcm_inv_range(unsigned long start, unsigned long end) 1588 { 1589 unsigned long new_start, new_end; 1590 1591 BUG_ON(start < BCM_SYS_EMI_START_ADDR); 1592 1593 if (unlikely(end <= start)) 1594 return; 1595 1596 new_start = bcm_l2_phys_addr(start); 1597 new_end = bcm_l2_phys_addr(end); 1598 1599 /* normal case, no cross section between start and end */ 1600 if (likely(bcm_addr_is_sys_emi(end) || !bcm_addr_is_sys_emi(start))) { 1601 l2c210_inv_range(new_start, new_end); 1602 return; 1603 } 1604 1605 /* They cross sections, so it can only be a cross from section 1606 * 2 to section 3 1607 */ 1608 l2c210_inv_range(new_start, 1609 bcm_l2_phys_addr(BCM_VC_EMI_SEC3_START_ADDR-1)); 1610 l2c210_inv_range(bcm_l2_phys_addr(BCM_VC_EMI_SEC3_START_ADDR), 1611 new_end); 1612 } 1613 1614 static void bcm_clean_range(unsigned long start, unsigned long end) 1615 { 1616 unsigned long new_start, new_end; 1617 1618 BUG_ON(start < BCM_SYS_EMI_START_ADDR); 1619 1620 if (unlikely(end <= start)) 1621 return; 1622 1623 new_start = bcm_l2_phys_addr(start); 1624 new_end = bcm_l2_phys_addr(end); 1625 1626 /* normal case, no cross section between start and end */ 1627 if (likely(bcm_addr_is_sys_emi(end) || !bcm_addr_is_sys_emi(start))) { 1628 l2c210_clean_range(new_start, new_end); 1629 return; 1630 } 1631 1632 /* They cross sections, so it can only be a cross from section 1633 * 2 to section 3 1634 */ 1635 l2c210_clean_range(new_start, 1636 bcm_l2_phys_addr(BCM_VC_EMI_SEC3_START_ADDR-1)); 1637 l2c210_clean_range(bcm_l2_phys_addr(BCM_VC_EMI_SEC3_START_ADDR), 1638 new_end); 1639 } 1640 1641 static void bcm_flush_range(unsigned long start, unsigned long end) 1642 { 1643 unsigned long new_start, new_end; 1644 1645 BUG_ON(start < BCM_SYS_EMI_START_ADDR); 1646 1647 if (unlikely(end <= start)) 1648 return; 1649 1650 if ((end - start) >= l2x0_size) { 1651 outer_cache.flush_all(); 1652 return; 1653 } 1654 1655 new_start = bcm_l2_phys_addr(start); 1656 new_end = bcm_l2_phys_addr(end); 1657 1658 /* normal case, no cross section between start and end */ 1659 if (likely(bcm_addr_is_sys_emi(end) || !bcm_addr_is_sys_emi(start))) { 1660 l2c210_flush_range(new_start, new_end); 1661 return; 1662 } 1663 1664 /* They cross sections, so it can only be a cross from section 1665 * 2 to section 3 1666 */ 1667 l2c210_flush_range(new_start, 1668 bcm_l2_phys_addr(BCM_VC_EMI_SEC3_START_ADDR-1)); 1669 l2c210_flush_range(bcm_l2_phys_addr(BCM_VC_EMI_SEC3_START_ADDR), 1670 new_end); 1671 } 1672 1673 /* Broadcom L2C-310 start from ARMs R3P2 or later, and require no fixups */ 1674 static const struct l2c_init_data of_bcm_l2x0_data __initconst = { 1675 .type = "BCM-L2C-310", 1676 .way_size_0 = SZ_8K, 1677 .num_lock = 8, 1678 .of_parse = l2c310_of_parse, 1679 .enable = l2c310_enable, 1680 .save = l2c310_save, 1681 .configure = l2c310_configure, 1682 .unlock = l2c310_unlock, 1683 .outer_cache = { 1684 .inv_range = bcm_inv_range, 1685 .clean_range = bcm_clean_range, 1686 .flush_range = bcm_flush_range, 1687 .flush_all = l2c210_flush_all, 1688 .disable = l2c310_disable, 1689 .sync = l2c210_sync, 1690 .resume = l2c310_resume, 1691 }, 1692 }; 1693 1694 static void __init tauros3_save(void __iomem *base) 1695 { 1696 l2c_save(base); 1697 1698 l2x0_saved_regs.aux2_ctrl = 1699 readl_relaxed(base + TAUROS3_AUX2_CTRL); 1700 l2x0_saved_regs.prefetch_ctrl = 1701 readl_relaxed(base + L310_PREFETCH_CTRL); 1702 } 1703 1704 static void tauros3_configure(void __iomem *base) 1705 { 1706 l2c_configure(base); 1707 writel_relaxed(l2x0_saved_regs.aux2_ctrl, 1708 base + TAUROS3_AUX2_CTRL); 1709 writel_relaxed(l2x0_saved_regs.prefetch_ctrl, 1710 base + L310_PREFETCH_CTRL); 1711 } 1712 1713 static const struct l2c_init_data of_tauros3_data __initconst = { 1714 .type = "Tauros3", 1715 .way_size_0 = SZ_8K, 1716 .num_lock = 8, 1717 .enable = l2c_enable, 1718 .save = tauros3_save, 1719 .configure = tauros3_configure, 1720 .unlock = l2c_unlock, 1721 /* Tauros3 broadcasts L1 cache operations to L2 */ 1722 .outer_cache = { 1723 .resume = l2c_resume, 1724 }, 1725 }; 1726 1727 #define L2C_ID(name, fns) { .compatible = name, .data = (void *)&fns } 1728 static const struct of_device_id l2x0_ids[] __initconst = { 1729 L2C_ID("arm,l210-cache", of_l2c210_data), 1730 L2C_ID("arm,l220-cache", of_l2c220_data), 1731 L2C_ID("arm,pl310-cache", of_l2c310_data), 1732 L2C_ID("brcm,bcm11351-a2-pl310-cache", of_bcm_l2x0_data), 1733 L2C_ID("marvell,aurora-outer-cache", of_aurora_with_outer_data), 1734 L2C_ID("marvell,aurora-system-cache", of_aurora_no_outer_data), 1735 L2C_ID("marvell,tauros3-cache", of_tauros3_data), 1736 /* Deprecated IDs */ 1737 L2C_ID("bcm,bcm11351-a2-pl310-cache", of_bcm_l2x0_data), 1738 {} 1739 }; 1740 1741 int __init l2x0_of_init(u32 aux_val, u32 aux_mask) 1742 { 1743 const struct l2c_init_data *data; 1744 struct device_node *np; 1745 struct resource res; 1746 u32 cache_id, old_aux; 1747 u32 cache_level = 2; 1748 bool nosync = false; 1749 1750 np = of_find_matching_node(NULL, l2x0_ids); 1751 if (!np) 1752 return -ENODEV; 1753 1754 if (of_address_to_resource(np, 0, &res)) 1755 return -ENODEV; 1756 1757 l2x0_base = ioremap(res.start, resource_size(&res)); 1758 if (!l2x0_base) 1759 return -ENOMEM; 1760 1761 l2x0_saved_regs.phy_base = res.start; 1762 1763 data = of_match_node(l2x0_ids, np)->data; 1764 1765 if (of_device_is_compatible(np, "arm,pl310-cache") && 1766 of_property_read_bool(np, "arm,io-coherent")) 1767 data = &of_l2c310_coherent_data; 1768 1769 old_aux = readl_relaxed(l2x0_base + L2X0_AUX_CTRL); 1770 if (old_aux != ((old_aux & aux_mask) | aux_val)) { 1771 pr_warn("L2C: platform modifies aux control register: 0x%08x -> 0x%08x\n", 1772 old_aux, (old_aux & aux_mask) | aux_val); 1773 } else if (aux_mask != ~0U && aux_val != 0) { 1774 pr_alert("L2C: platform provided aux values match the hardware, so have no effect. Please remove them.\n"); 1775 } 1776 1777 /* All L2 caches are unified, so this property should be specified */ 1778 if (!of_property_read_bool(np, "cache-unified")) 1779 pr_err("L2C: device tree omits to specify unified cache\n"); 1780 1781 if (of_property_read_u32(np, "cache-level", &cache_level)) 1782 pr_err("L2C: device tree omits to specify cache-level\n"); 1783 1784 if (cache_level != 2) 1785 pr_err("L2C: device tree specifies invalid cache level\n"); 1786 1787 nosync = of_property_read_bool(np, "arm,outer-sync-disable"); 1788 1789 /* Read back current (default) hardware configuration */ 1790 if (data->save) 1791 data->save(l2x0_base); 1792 1793 /* L2 configuration can only be changed if the cache is disabled */ 1794 if (!(readl_relaxed(l2x0_base + L2X0_CTRL) & L2X0_CTRL_EN)) 1795 if (data->of_parse) 1796 data->of_parse(np, &aux_val, &aux_mask); 1797 1798 if (cache_id_part_number_from_dt) 1799 cache_id = cache_id_part_number_from_dt; 1800 else 1801 cache_id = readl_relaxed(l2x0_base + L2X0_CACHE_ID); 1802 1803 return __l2c_init(data, aux_val, aux_mask, cache_id, nosync); 1804 } 1805 #endif 1806