1 /* 2 * arch/arm/mm/cache-l2x0.c - L210/L220 cache controller support 3 * 4 * Copyright (C) 2007 ARM Limited 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 * 10 * This program is distributed in the hope that it will be useful, 11 * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 * GNU General Public License for more details. 14 * 15 * You should have received a copy of the GNU General Public License 16 * along with this program; if not, write to the Free Software 17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 18 */ 19 #include <linux/err.h> 20 #include <linux/init.h> 21 #include <linux/spinlock.h> 22 #include <linux/io.h> 23 #include <linux/of.h> 24 #include <linux/of_address.h> 25 26 #include <asm/cacheflush.h> 27 #include <asm/hardware/cache-l2x0.h> 28 29 #define CACHE_LINE_SIZE 32 30 31 static void __iomem *l2x0_base; 32 static DEFINE_RAW_SPINLOCK(l2x0_lock); 33 static u32 l2x0_way_mask; /* Bitmask of active ways */ 34 static u32 l2x0_size; 35 static unsigned long sync_reg_offset = L2X0_CACHE_SYNC; 36 37 struct l2x0_regs l2x0_saved_regs; 38 39 struct l2x0_of_data { 40 void (*setup)(const struct device_node *, u32 *, u32 *); 41 void (*save)(void); 42 struct outer_cache_fns outer_cache; 43 }; 44 45 static bool of_init = false; 46 47 static inline void cache_wait_way(void __iomem *reg, unsigned long mask) 48 { 49 /* wait for cache operation by line or way to complete */ 50 while (readl_relaxed(reg) & mask) 51 cpu_relax(); 52 } 53 54 #ifdef CONFIG_CACHE_PL310 55 static inline void cache_wait(void __iomem *reg, unsigned long mask) 56 { 57 /* cache operations by line are atomic on PL310 */ 58 } 59 #else 60 #define cache_wait cache_wait_way 61 #endif 62 63 static inline void cache_sync(void) 64 { 65 void __iomem *base = l2x0_base; 66 67 writel_relaxed(0, base + sync_reg_offset); 68 cache_wait(base + L2X0_CACHE_SYNC, 1); 69 } 70 71 static inline void l2x0_clean_line(unsigned long addr) 72 { 73 void __iomem *base = l2x0_base; 74 cache_wait(base + L2X0_CLEAN_LINE_PA, 1); 75 writel_relaxed(addr, base + L2X0_CLEAN_LINE_PA); 76 } 77 78 static inline void l2x0_inv_line(unsigned long addr) 79 { 80 void __iomem *base = l2x0_base; 81 cache_wait(base + L2X0_INV_LINE_PA, 1); 82 writel_relaxed(addr, base + L2X0_INV_LINE_PA); 83 } 84 85 #if defined(CONFIG_PL310_ERRATA_588369) || defined(CONFIG_PL310_ERRATA_727915) 86 static inline void debug_writel(unsigned long val) 87 { 88 if (outer_cache.set_debug) 89 outer_cache.set_debug(val); 90 } 91 92 static void pl310_set_debug(unsigned long val) 93 { 94 writel_relaxed(val, l2x0_base + L2X0_DEBUG_CTRL); 95 } 96 #else 97 /* Optimised out for non-errata case */ 98 static inline void debug_writel(unsigned long val) 99 { 100 } 101 102 #define pl310_set_debug NULL 103 #endif 104 105 #ifdef CONFIG_PL310_ERRATA_588369 106 static inline void l2x0_flush_line(unsigned long addr) 107 { 108 void __iomem *base = l2x0_base; 109 110 /* Clean by PA followed by Invalidate by PA */ 111 cache_wait(base + L2X0_CLEAN_LINE_PA, 1); 112 writel_relaxed(addr, base + L2X0_CLEAN_LINE_PA); 113 cache_wait(base + L2X0_INV_LINE_PA, 1); 114 writel_relaxed(addr, base + L2X0_INV_LINE_PA); 115 } 116 #else 117 118 static inline void l2x0_flush_line(unsigned long addr) 119 { 120 void __iomem *base = l2x0_base; 121 cache_wait(base + L2X0_CLEAN_INV_LINE_PA, 1); 122 writel_relaxed(addr, base + L2X0_CLEAN_INV_LINE_PA); 123 } 124 #endif 125 126 static void l2x0_cache_sync(void) 127 { 128 unsigned long flags; 129 130 raw_spin_lock_irqsave(&l2x0_lock, flags); 131 cache_sync(); 132 raw_spin_unlock_irqrestore(&l2x0_lock, flags); 133 } 134 135 static void __l2x0_flush_all(void) 136 { 137 debug_writel(0x03); 138 writel_relaxed(l2x0_way_mask, l2x0_base + L2X0_CLEAN_INV_WAY); 139 cache_wait_way(l2x0_base + L2X0_CLEAN_INV_WAY, l2x0_way_mask); 140 cache_sync(); 141 debug_writel(0x00); 142 } 143 144 static void l2x0_flush_all(void) 145 { 146 unsigned long flags; 147 148 /* clean all ways */ 149 raw_spin_lock_irqsave(&l2x0_lock, flags); 150 __l2x0_flush_all(); 151 raw_spin_unlock_irqrestore(&l2x0_lock, flags); 152 } 153 154 static void l2x0_clean_all(void) 155 { 156 unsigned long flags; 157 158 /* clean all ways */ 159 raw_spin_lock_irqsave(&l2x0_lock, flags); 160 writel_relaxed(l2x0_way_mask, l2x0_base + L2X0_CLEAN_WAY); 161 cache_wait_way(l2x0_base + L2X0_CLEAN_WAY, l2x0_way_mask); 162 cache_sync(); 163 raw_spin_unlock_irqrestore(&l2x0_lock, flags); 164 } 165 166 static void l2x0_inv_all(void) 167 { 168 unsigned long flags; 169 170 /* invalidate all ways */ 171 raw_spin_lock_irqsave(&l2x0_lock, flags); 172 /* Invalidating when L2 is enabled is a nono */ 173 BUG_ON(readl(l2x0_base + L2X0_CTRL) & 1); 174 writel_relaxed(l2x0_way_mask, l2x0_base + L2X0_INV_WAY); 175 cache_wait_way(l2x0_base + L2X0_INV_WAY, l2x0_way_mask); 176 cache_sync(); 177 raw_spin_unlock_irqrestore(&l2x0_lock, flags); 178 } 179 180 static void l2x0_inv_range(unsigned long start, unsigned long end) 181 { 182 void __iomem *base = l2x0_base; 183 unsigned long flags; 184 185 raw_spin_lock_irqsave(&l2x0_lock, flags); 186 if (start & (CACHE_LINE_SIZE - 1)) { 187 start &= ~(CACHE_LINE_SIZE - 1); 188 debug_writel(0x03); 189 l2x0_flush_line(start); 190 debug_writel(0x00); 191 start += CACHE_LINE_SIZE; 192 } 193 194 if (end & (CACHE_LINE_SIZE - 1)) { 195 end &= ~(CACHE_LINE_SIZE - 1); 196 debug_writel(0x03); 197 l2x0_flush_line(end); 198 debug_writel(0x00); 199 } 200 201 while (start < end) { 202 unsigned long blk_end = start + min(end - start, 4096UL); 203 204 while (start < blk_end) { 205 l2x0_inv_line(start); 206 start += CACHE_LINE_SIZE; 207 } 208 209 if (blk_end < end) { 210 raw_spin_unlock_irqrestore(&l2x0_lock, flags); 211 raw_spin_lock_irqsave(&l2x0_lock, flags); 212 } 213 } 214 cache_wait(base + L2X0_INV_LINE_PA, 1); 215 cache_sync(); 216 raw_spin_unlock_irqrestore(&l2x0_lock, flags); 217 } 218 219 static void l2x0_clean_range(unsigned long start, unsigned long end) 220 { 221 void __iomem *base = l2x0_base; 222 unsigned long flags; 223 224 if ((end - start) >= l2x0_size) { 225 l2x0_clean_all(); 226 return; 227 } 228 229 raw_spin_lock_irqsave(&l2x0_lock, flags); 230 start &= ~(CACHE_LINE_SIZE - 1); 231 while (start < end) { 232 unsigned long blk_end = start + min(end - start, 4096UL); 233 234 while (start < blk_end) { 235 l2x0_clean_line(start); 236 start += CACHE_LINE_SIZE; 237 } 238 239 if (blk_end < end) { 240 raw_spin_unlock_irqrestore(&l2x0_lock, flags); 241 raw_spin_lock_irqsave(&l2x0_lock, flags); 242 } 243 } 244 cache_wait(base + L2X0_CLEAN_LINE_PA, 1); 245 cache_sync(); 246 raw_spin_unlock_irqrestore(&l2x0_lock, flags); 247 } 248 249 static void l2x0_flush_range(unsigned long start, unsigned long end) 250 { 251 void __iomem *base = l2x0_base; 252 unsigned long flags; 253 254 if ((end - start) >= l2x0_size) { 255 l2x0_flush_all(); 256 return; 257 } 258 259 raw_spin_lock_irqsave(&l2x0_lock, flags); 260 start &= ~(CACHE_LINE_SIZE - 1); 261 while (start < end) { 262 unsigned long blk_end = start + min(end - start, 4096UL); 263 264 debug_writel(0x03); 265 while (start < blk_end) { 266 l2x0_flush_line(start); 267 start += CACHE_LINE_SIZE; 268 } 269 debug_writel(0x00); 270 271 if (blk_end < end) { 272 raw_spin_unlock_irqrestore(&l2x0_lock, flags); 273 raw_spin_lock_irqsave(&l2x0_lock, flags); 274 } 275 } 276 cache_wait(base + L2X0_CLEAN_INV_LINE_PA, 1); 277 cache_sync(); 278 raw_spin_unlock_irqrestore(&l2x0_lock, flags); 279 } 280 281 static void l2x0_disable(void) 282 { 283 unsigned long flags; 284 285 raw_spin_lock_irqsave(&l2x0_lock, flags); 286 __l2x0_flush_all(); 287 writel_relaxed(0, l2x0_base + L2X0_CTRL); 288 dsb(); 289 raw_spin_unlock_irqrestore(&l2x0_lock, flags); 290 } 291 292 static void l2x0_unlock(u32 cache_id) 293 { 294 int lockregs; 295 int i; 296 297 if (cache_id == L2X0_CACHE_ID_PART_L310) 298 lockregs = 8; 299 else 300 /* L210 and unknown types */ 301 lockregs = 1; 302 303 for (i = 0; i < lockregs; i++) { 304 writel_relaxed(0x0, l2x0_base + L2X0_LOCKDOWN_WAY_D_BASE + 305 i * L2X0_LOCKDOWN_STRIDE); 306 writel_relaxed(0x0, l2x0_base + L2X0_LOCKDOWN_WAY_I_BASE + 307 i * L2X0_LOCKDOWN_STRIDE); 308 } 309 } 310 311 void __init l2x0_init(void __iomem *base, u32 aux_val, u32 aux_mask) 312 { 313 u32 aux; 314 u32 cache_id; 315 u32 way_size = 0; 316 int ways; 317 const char *type; 318 319 l2x0_base = base; 320 321 cache_id = readl_relaxed(l2x0_base + L2X0_CACHE_ID); 322 aux = readl_relaxed(l2x0_base + L2X0_AUX_CTRL); 323 324 aux &= aux_mask; 325 aux |= aux_val; 326 327 /* Determine the number of ways */ 328 switch (cache_id & L2X0_CACHE_ID_PART_MASK) { 329 case L2X0_CACHE_ID_PART_L310: 330 if (aux & (1 << 16)) 331 ways = 16; 332 else 333 ways = 8; 334 type = "L310"; 335 #ifdef CONFIG_PL310_ERRATA_753970 336 /* Unmapped register. */ 337 sync_reg_offset = L2X0_DUMMY_REG; 338 #endif 339 outer_cache.set_debug = pl310_set_debug; 340 break; 341 case L2X0_CACHE_ID_PART_L210: 342 ways = (aux >> 13) & 0xf; 343 type = "L210"; 344 break; 345 default: 346 /* Assume unknown chips have 8 ways */ 347 ways = 8; 348 type = "L2x0 series"; 349 break; 350 } 351 352 l2x0_way_mask = (1 << ways) - 1; 353 354 /* 355 * L2 cache Size = Way size * Number of ways 356 */ 357 way_size = (aux & L2X0_AUX_CTRL_WAY_SIZE_MASK) >> 17; 358 way_size = 1 << (way_size + 3); 359 l2x0_size = ways * way_size * SZ_1K; 360 361 /* 362 * Check if l2x0 controller is already enabled. 363 * If you are booting from non-secure mode 364 * accessing the below registers will fault. 365 */ 366 if (!(readl_relaxed(l2x0_base + L2X0_CTRL) & 1)) { 367 /* Make sure that I&D is not locked down when starting */ 368 l2x0_unlock(cache_id); 369 370 /* l2x0 controller is disabled */ 371 writel_relaxed(aux, l2x0_base + L2X0_AUX_CTRL); 372 373 l2x0_inv_all(); 374 375 /* enable L2X0 */ 376 writel_relaxed(1, l2x0_base + L2X0_CTRL); 377 } 378 379 /* Re-read it in case some bits are reserved. */ 380 aux = readl_relaxed(l2x0_base + L2X0_AUX_CTRL); 381 382 /* Save the value for resuming. */ 383 l2x0_saved_regs.aux_ctrl = aux; 384 385 if (!of_init) { 386 outer_cache.inv_range = l2x0_inv_range; 387 outer_cache.clean_range = l2x0_clean_range; 388 outer_cache.flush_range = l2x0_flush_range; 389 outer_cache.sync = l2x0_cache_sync; 390 outer_cache.flush_all = l2x0_flush_all; 391 outer_cache.inv_all = l2x0_inv_all; 392 outer_cache.disable = l2x0_disable; 393 } 394 395 printk(KERN_INFO "%s cache controller enabled\n", type); 396 printk(KERN_INFO "l2x0: %d ways, CACHE_ID 0x%08x, AUX_CTRL 0x%08x, Cache size: %d B\n", 397 ways, cache_id, aux, l2x0_size); 398 } 399 400 #ifdef CONFIG_OF 401 static void __init l2x0_of_setup(const struct device_node *np, 402 u32 *aux_val, u32 *aux_mask) 403 { 404 u32 data[2] = { 0, 0 }; 405 u32 tag = 0; 406 u32 dirty = 0; 407 u32 val = 0, mask = 0; 408 409 of_property_read_u32(np, "arm,tag-latency", &tag); 410 if (tag) { 411 mask |= L2X0_AUX_CTRL_TAG_LATENCY_MASK; 412 val |= (tag - 1) << L2X0_AUX_CTRL_TAG_LATENCY_SHIFT; 413 } 414 415 of_property_read_u32_array(np, "arm,data-latency", 416 data, ARRAY_SIZE(data)); 417 if (data[0] && data[1]) { 418 mask |= L2X0_AUX_CTRL_DATA_RD_LATENCY_MASK | 419 L2X0_AUX_CTRL_DATA_WR_LATENCY_MASK; 420 val |= ((data[0] - 1) << L2X0_AUX_CTRL_DATA_RD_LATENCY_SHIFT) | 421 ((data[1] - 1) << L2X0_AUX_CTRL_DATA_WR_LATENCY_SHIFT); 422 } 423 424 of_property_read_u32(np, "arm,dirty-latency", &dirty); 425 if (dirty) { 426 mask |= L2X0_AUX_CTRL_DIRTY_LATENCY_MASK; 427 val |= (dirty - 1) << L2X0_AUX_CTRL_DIRTY_LATENCY_SHIFT; 428 } 429 430 *aux_val &= ~mask; 431 *aux_val |= val; 432 *aux_mask &= ~mask; 433 } 434 435 static void __init pl310_of_setup(const struct device_node *np, 436 u32 *aux_val, u32 *aux_mask) 437 { 438 u32 data[3] = { 0, 0, 0 }; 439 u32 tag[3] = { 0, 0, 0 }; 440 u32 filter[2] = { 0, 0 }; 441 442 of_property_read_u32_array(np, "arm,tag-latency", tag, ARRAY_SIZE(tag)); 443 if (tag[0] && tag[1] && tag[2]) 444 writel_relaxed( 445 ((tag[0] - 1) << L2X0_LATENCY_CTRL_RD_SHIFT) | 446 ((tag[1] - 1) << L2X0_LATENCY_CTRL_WR_SHIFT) | 447 ((tag[2] - 1) << L2X0_LATENCY_CTRL_SETUP_SHIFT), 448 l2x0_base + L2X0_TAG_LATENCY_CTRL); 449 450 of_property_read_u32_array(np, "arm,data-latency", 451 data, ARRAY_SIZE(data)); 452 if (data[0] && data[1] && data[2]) 453 writel_relaxed( 454 ((data[0] - 1) << L2X0_LATENCY_CTRL_RD_SHIFT) | 455 ((data[1] - 1) << L2X0_LATENCY_CTRL_WR_SHIFT) | 456 ((data[2] - 1) << L2X0_LATENCY_CTRL_SETUP_SHIFT), 457 l2x0_base + L2X0_DATA_LATENCY_CTRL); 458 459 of_property_read_u32_array(np, "arm,filter-ranges", 460 filter, ARRAY_SIZE(filter)); 461 if (filter[1]) { 462 writel_relaxed(ALIGN(filter[0] + filter[1], SZ_1M), 463 l2x0_base + L2X0_ADDR_FILTER_END); 464 writel_relaxed((filter[0] & ~(SZ_1M - 1)) | L2X0_ADDR_FILTER_EN, 465 l2x0_base + L2X0_ADDR_FILTER_START); 466 } 467 } 468 469 static void __init pl310_save(void) 470 { 471 u32 l2x0_revision = readl_relaxed(l2x0_base + L2X0_CACHE_ID) & 472 L2X0_CACHE_ID_RTL_MASK; 473 474 l2x0_saved_regs.tag_latency = readl_relaxed(l2x0_base + 475 L2X0_TAG_LATENCY_CTRL); 476 l2x0_saved_regs.data_latency = readl_relaxed(l2x0_base + 477 L2X0_DATA_LATENCY_CTRL); 478 l2x0_saved_regs.filter_end = readl_relaxed(l2x0_base + 479 L2X0_ADDR_FILTER_END); 480 l2x0_saved_regs.filter_start = readl_relaxed(l2x0_base + 481 L2X0_ADDR_FILTER_START); 482 483 if (l2x0_revision >= L2X0_CACHE_ID_RTL_R2P0) { 484 /* 485 * From r2p0, there is Prefetch offset/control register 486 */ 487 l2x0_saved_regs.prefetch_ctrl = readl_relaxed(l2x0_base + 488 L2X0_PREFETCH_CTRL); 489 /* 490 * From r3p0, there is Power control register 491 */ 492 if (l2x0_revision >= L2X0_CACHE_ID_RTL_R3P0) 493 l2x0_saved_regs.pwr_ctrl = readl_relaxed(l2x0_base + 494 L2X0_POWER_CTRL); 495 } 496 } 497 498 static void l2x0_resume(void) 499 { 500 if (!(readl_relaxed(l2x0_base + L2X0_CTRL) & 1)) { 501 /* restore aux ctrl and enable l2 */ 502 l2x0_unlock(readl_relaxed(l2x0_base + L2X0_CACHE_ID)); 503 504 writel_relaxed(l2x0_saved_regs.aux_ctrl, l2x0_base + 505 L2X0_AUX_CTRL); 506 507 l2x0_inv_all(); 508 509 writel_relaxed(1, l2x0_base + L2X0_CTRL); 510 } 511 } 512 513 static void pl310_resume(void) 514 { 515 u32 l2x0_revision; 516 517 if (!(readl_relaxed(l2x0_base + L2X0_CTRL) & 1)) { 518 /* restore pl310 setup */ 519 writel_relaxed(l2x0_saved_regs.tag_latency, 520 l2x0_base + L2X0_TAG_LATENCY_CTRL); 521 writel_relaxed(l2x0_saved_regs.data_latency, 522 l2x0_base + L2X0_DATA_LATENCY_CTRL); 523 writel_relaxed(l2x0_saved_regs.filter_end, 524 l2x0_base + L2X0_ADDR_FILTER_END); 525 writel_relaxed(l2x0_saved_regs.filter_start, 526 l2x0_base + L2X0_ADDR_FILTER_START); 527 528 l2x0_revision = readl_relaxed(l2x0_base + L2X0_CACHE_ID) & 529 L2X0_CACHE_ID_RTL_MASK; 530 531 if (l2x0_revision >= L2X0_CACHE_ID_RTL_R2P0) { 532 writel_relaxed(l2x0_saved_regs.prefetch_ctrl, 533 l2x0_base + L2X0_PREFETCH_CTRL); 534 if (l2x0_revision >= L2X0_CACHE_ID_RTL_R3P0) 535 writel_relaxed(l2x0_saved_regs.pwr_ctrl, 536 l2x0_base + L2X0_POWER_CTRL); 537 } 538 } 539 540 l2x0_resume(); 541 } 542 543 static const struct l2x0_of_data pl310_data = { 544 .setup = pl310_of_setup, 545 .save = pl310_save, 546 .outer_cache = { 547 .resume = pl310_resume, 548 .inv_range = l2x0_inv_range, 549 .clean_range = l2x0_clean_range, 550 .flush_range = l2x0_flush_range, 551 .sync = l2x0_cache_sync, 552 .flush_all = l2x0_flush_all, 553 .inv_all = l2x0_inv_all, 554 .disable = l2x0_disable, 555 .set_debug = pl310_set_debug, 556 }, 557 }; 558 559 static const struct l2x0_of_data l2x0_data = { 560 .setup = l2x0_of_setup, 561 .save = NULL, 562 .outer_cache = { 563 .resume = l2x0_resume, 564 .inv_range = l2x0_inv_range, 565 .clean_range = l2x0_clean_range, 566 .flush_range = l2x0_flush_range, 567 .sync = l2x0_cache_sync, 568 .flush_all = l2x0_flush_all, 569 .inv_all = l2x0_inv_all, 570 .disable = l2x0_disable, 571 }, 572 }; 573 574 static const struct of_device_id l2x0_ids[] __initconst = { 575 { .compatible = "arm,pl310-cache", .data = (void *)&pl310_data }, 576 { .compatible = "arm,l220-cache", .data = (void *)&l2x0_data }, 577 { .compatible = "arm,l210-cache", .data = (void *)&l2x0_data }, 578 {} 579 }; 580 581 int __init l2x0_of_init(u32 aux_val, u32 aux_mask) 582 { 583 struct device_node *np; 584 const struct l2x0_of_data *data; 585 struct resource res; 586 587 np = of_find_matching_node(NULL, l2x0_ids); 588 if (!np) 589 return -ENODEV; 590 591 if (of_address_to_resource(np, 0, &res)) 592 return -ENODEV; 593 594 l2x0_base = ioremap(res.start, resource_size(&res)); 595 if (!l2x0_base) 596 return -ENOMEM; 597 598 l2x0_saved_regs.phy_base = res.start; 599 600 data = of_match_node(l2x0_ids, np)->data; 601 602 /* L2 configuration can only be changed if the cache is disabled */ 603 if (!(readl_relaxed(l2x0_base + L2X0_CTRL) & 1)) { 604 if (data->setup) 605 data->setup(np, &aux_val, &aux_mask); 606 } 607 608 if (data->save) 609 data->save(); 610 611 of_init = true; 612 l2x0_init(l2x0_base, aux_val, aux_mask); 613 614 memcpy(&outer_cache, &data->outer_cache, sizeof(outer_cache)); 615 616 return 0; 617 } 618 #endif 619