1 /* 2 * This file is subject to the terms and conditions of the GNU General Public 3 * License. See the file "COPYING" in the main directory of this archive 4 * for more details. 5 * 6 * Copyright (C) 1996 David S. Miller (davem@davemloft.net) 7 * Copyright (C) 1997, 1998, 1999, 2000, 2001, 2002 Ralf Baechle (ralf@gnu.org) 8 * Copyright (C) 1999, 2000 Silicon Graphics, Inc. 9 */ 10 #include <linux/hardirq.h> 11 #include <linux/init.h> 12 #include <linux/highmem.h> 13 #include <linux/kernel.h> 14 #include <linux/linkage.h> 15 #include <linux/sched.h> 16 #include <linux/smp.h> 17 #include <linux/mm.h> 18 #include <linux/module.h> 19 #include <linux/bitops.h> 20 21 #include <asm/bcache.h> 22 #include <asm/bootinfo.h> 23 #include <asm/cache.h> 24 #include <asm/cacheops.h> 25 #include <asm/cpu.h> 26 #include <asm/cpu-features.h> 27 #include <asm/io.h> 28 #include <asm/page.h> 29 #include <asm/pgtable.h> 30 #include <asm/r4kcache.h> 31 #include <asm/sections.h> 32 #include <asm/mmu_context.h> 33 #include <asm/war.h> 34 #include <asm/cacheflush.h> /* for run_uncached() */ 35 #include <asm/traps.h> 36 #include <asm/dma-coherence.h> 37 38 /* 39 * Special Variant of smp_call_function for use by cache functions: 40 * 41 * o No return value 42 * o collapses to normal function call on UP kernels 43 * o collapses to normal function call on systems with a single shared 44 * primary cache. 45 * o doesn't disable interrupts on the local CPU 46 */ 47 static inline void r4k_on_each_cpu(void (*func) (void *info), void *info) 48 { 49 preempt_disable(); 50 51 #if !defined(CONFIG_MIPS_MT_SMP) && !defined(CONFIG_MIPS_MT_SMTC) 52 smp_call_function(func, info, 1); 53 #endif 54 func(info); 55 preempt_enable(); 56 } 57 58 #if defined(CONFIG_MIPS_CMP) 59 #define cpu_has_safe_index_cacheops 0 60 #else 61 #define cpu_has_safe_index_cacheops 1 62 #endif 63 64 /* 65 * Must die. 66 */ 67 static unsigned long icache_size __read_mostly; 68 static unsigned long dcache_size __read_mostly; 69 static unsigned long scache_size __read_mostly; 70 71 /* 72 * Dummy cache handling routines for machines without boardcaches 73 */ 74 static void cache_noop(void) {} 75 76 static struct bcache_ops no_sc_ops = { 77 .bc_enable = (void *)cache_noop, 78 .bc_disable = (void *)cache_noop, 79 .bc_wback_inv = (void *)cache_noop, 80 .bc_inv = (void *)cache_noop 81 }; 82 83 struct bcache_ops *bcops = &no_sc_ops; 84 85 #define cpu_is_r4600_v1_x() ((read_c0_prid() & 0xfffffff0) == 0x00002010) 86 #define cpu_is_r4600_v2_x() ((read_c0_prid() & 0xfffffff0) == 0x00002020) 87 88 #define R4600_HIT_CACHEOP_WAR_IMPL \ 89 do { \ 90 if (R4600_V2_HIT_CACHEOP_WAR && cpu_is_r4600_v2_x()) \ 91 *(volatile unsigned long *)CKSEG1; \ 92 if (R4600_V1_HIT_CACHEOP_WAR) \ 93 __asm__ __volatile__("nop;nop;nop;nop"); \ 94 } while (0) 95 96 static void (*r4k_blast_dcache_page)(unsigned long addr); 97 98 static inline void r4k_blast_dcache_page_dc32(unsigned long addr) 99 { 100 R4600_HIT_CACHEOP_WAR_IMPL; 101 blast_dcache32_page(addr); 102 } 103 104 static inline void r4k_blast_dcache_page_dc64(unsigned long addr) 105 { 106 R4600_HIT_CACHEOP_WAR_IMPL; 107 blast_dcache64_page(addr); 108 } 109 110 static void __cpuinit r4k_blast_dcache_page_setup(void) 111 { 112 unsigned long dc_lsize = cpu_dcache_line_size(); 113 114 if (dc_lsize == 0) 115 r4k_blast_dcache_page = (void *)cache_noop; 116 else if (dc_lsize == 16) 117 r4k_blast_dcache_page = blast_dcache16_page; 118 else if (dc_lsize == 32) 119 r4k_blast_dcache_page = r4k_blast_dcache_page_dc32; 120 else if (dc_lsize == 64) 121 r4k_blast_dcache_page = r4k_blast_dcache_page_dc64; 122 } 123 124 static void (* r4k_blast_dcache_page_indexed)(unsigned long addr); 125 126 static void __cpuinit r4k_blast_dcache_page_indexed_setup(void) 127 { 128 unsigned long dc_lsize = cpu_dcache_line_size(); 129 130 if (dc_lsize == 0) 131 r4k_blast_dcache_page_indexed = (void *)cache_noop; 132 else if (dc_lsize == 16) 133 r4k_blast_dcache_page_indexed = blast_dcache16_page_indexed; 134 else if (dc_lsize == 32) 135 r4k_blast_dcache_page_indexed = blast_dcache32_page_indexed; 136 else if (dc_lsize == 64) 137 r4k_blast_dcache_page_indexed = blast_dcache64_page_indexed; 138 } 139 140 void (* r4k_blast_dcache)(void); 141 EXPORT_SYMBOL(r4k_blast_dcache); 142 143 static void __cpuinit r4k_blast_dcache_setup(void) 144 { 145 unsigned long dc_lsize = cpu_dcache_line_size(); 146 147 if (dc_lsize == 0) 148 r4k_blast_dcache = (void *)cache_noop; 149 else if (dc_lsize == 16) 150 r4k_blast_dcache = blast_dcache16; 151 else if (dc_lsize == 32) 152 r4k_blast_dcache = blast_dcache32; 153 else if (dc_lsize == 64) 154 r4k_blast_dcache = blast_dcache64; 155 } 156 157 /* force code alignment (used for TX49XX_ICACHE_INDEX_INV_WAR) */ 158 #define JUMP_TO_ALIGN(order) \ 159 __asm__ __volatile__( \ 160 "b\t1f\n\t" \ 161 ".align\t" #order "\n\t" \ 162 "1:\n\t" \ 163 ) 164 #define CACHE32_UNROLL32_ALIGN JUMP_TO_ALIGN(10) /* 32 * 32 = 1024 */ 165 #define CACHE32_UNROLL32_ALIGN2 JUMP_TO_ALIGN(11) 166 167 static inline void blast_r4600_v1_icache32(void) 168 { 169 unsigned long flags; 170 171 local_irq_save(flags); 172 blast_icache32(); 173 local_irq_restore(flags); 174 } 175 176 static inline void tx49_blast_icache32(void) 177 { 178 unsigned long start = INDEX_BASE; 179 unsigned long end = start + current_cpu_data.icache.waysize; 180 unsigned long ws_inc = 1UL << current_cpu_data.icache.waybit; 181 unsigned long ws_end = current_cpu_data.icache.ways << 182 current_cpu_data.icache.waybit; 183 unsigned long ws, addr; 184 185 CACHE32_UNROLL32_ALIGN2; 186 /* I'm in even chunk. blast odd chunks */ 187 for (ws = 0; ws < ws_end; ws += ws_inc) 188 for (addr = start + 0x400; addr < end; addr += 0x400 * 2) 189 cache32_unroll32(addr|ws, Index_Invalidate_I); 190 CACHE32_UNROLL32_ALIGN; 191 /* I'm in odd chunk. blast even chunks */ 192 for (ws = 0; ws < ws_end; ws += ws_inc) 193 for (addr = start; addr < end; addr += 0x400 * 2) 194 cache32_unroll32(addr|ws, Index_Invalidate_I); 195 } 196 197 static inline void blast_icache32_r4600_v1_page_indexed(unsigned long page) 198 { 199 unsigned long flags; 200 201 local_irq_save(flags); 202 blast_icache32_page_indexed(page); 203 local_irq_restore(flags); 204 } 205 206 static inline void tx49_blast_icache32_page_indexed(unsigned long page) 207 { 208 unsigned long indexmask = current_cpu_data.icache.waysize - 1; 209 unsigned long start = INDEX_BASE + (page & indexmask); 210 unsigned long end = start + PAGE_SIZE; 211 unsigned long ws_inc = 1UL << current_cpu_data.icache.waybit; 212 unsigned long ws_end = current_cpu_data.icache.ways << 213 current_cpu_data.icache.waybit; 214 unsigned long ws, addr; 215 216 CACHE32_UNROLL32_ALIGN2; 217 /* I'm in even chunk. blast odd chunks */ 218 for (ws = 0; ws < ws_end; ws += ws_inc) 219 for (addr = start + 0x400; addr < end; addr += 0x400 * 2) 220 cache32_unroll32(addr|ws, Index_Invalidate_I); 221 CACHE32_UNROLL32_ALIGN; 222 /* I'm in odd chunk. blast even chunks */ 223 for (ws = 0; ws < ws_end; ws += ws_inc) 224 for (addr = start; addr < end; addr += 0x400 * 2) 225 cache32_unroll32(addr|ws, Index_Invalidate_I); 226 } 227 228 static void (* r4k_blast_icache_page)(unsigned long addr); 229 230 static void __cpuinit r4k_blast_icache_page_setup(void) 231 { 232 unsigned long ic_lsize = cpu_icache_line_size(); 233 234 if (ic_lsize == 0) 235 r4k_blast_icache_page = (void *)cache_noop; 236 else if (ic_lsize == 16) 237 r4k_blast_icache_page = blast_icache16_page; 238 else if (ic_lsize == 32) 239 r4k_blast_icache_page = blast_icache32_page; 240 else if (ic_lsize == 64) 241 r4k_blast_icache_page = blast_icache64_page; 242 } 243 244 245 static void (* r4k_blast_icache_page_indexed)(unsigned long addr); 246 247 static void __cpuinit r4k_blast_icache_page_indexed_setup(void) 248 { 249 unsigned long ic_lsize = cpu_icache_line_size(); 250 251 if (ic_lsize == 0) 252 r4k_blast_icache_page_indexed = (void *)cache_noop; 253 else if (ic_lsize == 16) 254 r4k_blast_icache_page_indexed = blast_icache16_page_indexed; 255 else if (ic_lsize == 32) { 256 if (R4600_V1_INDEX_ICACHEOP_WAR && cpu_is_r4600_v1_x()) 257 r4k_blast_icache_page_indexed = 258 blast_icache32_r4600_v1_page_indexed; 259 else if (TX49XX_ICACHE_INDEX_INV_WAR) 260 r4k_blast_icache_page_indexed = 261 tx49_blast_icache32_page_indexed; 262 else 263 r4k_blast_icache_page_indexed = 264 blast_icache32_page_indexed; 265 } else if (ic_lsize == 64) 266 r4k_blast_icache_page_indexed = blast_icache64_page_indexed; 267 } 268 269 void (* r4k_blast_icache)(void); 270 EXPORT_SYMBOL(r4k_blast_icache); 271 272 static void __cpuinit r4k_blast_icache_setup(void) 273 { 274 unsigned long ic_lsize = cpu_icache_line_size(); 275 276 if (ic_lsize == 0) 277 r4k_blast_icache = (void *)cache_noop; 278 else if (ic_lsize == 16) 279 r4k_blast_icache = blast_icache16; 280 else if (ic_lsize == 32) { 281 if (R4600_V1_INDEX_ICACHEOP_WAR && cpu_is_r4600_v1_x()) 282 r4k_blast_icache = blast_r4600_v1_icache32; 283 else if (TX49XX_ICACHE_INDEX_INV_WAR) 284 r4k_blast_icache = tx49_blast_icache32; 285 else 286 r4k_blast_icache = blast_icache32; 287 } else if (ic_lsize == 64) 288 r4k_blast_icache = blast_icache64; 289 } 290 291 static void (* r4k_blast_scache_page)(unsigned long addr); 292 293 static void __cpuinit r4k_blast_scache_page_setup(void) 294 { 295 unsigned long sc_lsize = cpu_scache_line_size(); 296 297 if (scache_size == 0) 298 r4k_blast_scache_page = (void *)cache_noop; 299 else if (sc_lsize == 16) 300 r4k_blast_scache_page = blast_scache16_page; 301 else if (sc_lsize == 32) 302 r4k_blast_scache_page = blast_scache32_page; 303 else if (sc_lsize == 64) 304 r4k_blast_scache_page = blast_scache64_page; 305 else if (sc_lsize == 128) 306 r4k_blast_scache_page = blast_scache128_page; 307 } 308 309 static void (* r4k_blast_scache_page_indexed)(unsigned long addr); 310 311 static void __cpuinit r4k_blast_scache_page_indexed_setup(void) 312 { 313 unsigned long sc_lsize = cpu_scache_line_size(); 314 315 if (scache_size == 0) 316 r4k_blast_scache_page_indexed = (void *)cache_noop; 317 else if (sc_lsize == 16) 318 r4k_blast_scache_page_indexed = blast_scache16_page_indexed; 319 else if (sc_lsize == 32) 320 r4k_blast_scache_page_indexed = blast_scache32_page_indexed; 321 else if (sc_lsize == 64) 322 r4k_blast_scache_page_indexed = blast_scache64_page_indexed; 323 else if (sc_lsize == 128) 324 r4k_blast_scache_page_indexed = blast_scache128_page_indexed; 325 } 326 327 static void (* r4k_blast_scache)(void); 328 329 static void __cpuinit r4k_blast_scache_setup(void) 330 { 331 unsigned long sc_lsize = cpu_scache_line_size(); 332 333 if (scache_size == 0) 334 r4k_blast_scache = (void *)cache_noop; 335 else if (sc_lsize == 16) 336 r4k_blast_scache = blast_scache16; 337 else if (sc_lsize == 32) 338 r4k_blast_scache = blast_scache32; 339 else if (sc_lsize == 64) 340 r4k_blast_scache = blast_scache64; 341 else if (sc_lsize == 128) 342 r4k_blast_scache = blast_scache128; 343 } 344 345 static inline void local_r4k___flush_cache_all(void * args) 346 { 347 #if defined(CONFIG_CPU_LOONGSON2) 348 r4k_blast_scache(); 349 return; 350 #endif 351 r4k_blast_dcache(); 352 r4k_blast_icache(); 353 354 switch (current_cpu_type()) { 355 case CPU_R4000SC: 356 case CPU_R4000MC: 357 case CPU_R4400SC: 358 case CPU_R4400MC: 359 case CPU_R10000: 360 case CPU_R12000: 361 case CPU_R14000: 362 r4k_blast_scache(); 363 } 364 } 365 366 static void r4k___flush_cache_all(void) 367 { 368 r4k_on_each_cpu(local_r4k___flush_cache_all, NULL); 369 } 370 371 static inline int has_valid_asid(const struct mm_struct *mm) 372 { 373 #if defined(CONFIG_MIPS_MT_SMP) || defined(CONFIG_MIPS_MT_SMTC) 374 int i; 375 376 for_each_online_cpu(i) 377 if (cpu_context(i, mm)) 378 return 1; 379 380 return 0; 381 #else 382 return cpu_context(smp_processor_id(), mm); 383 #endif 384 } 385 386 static void r4k__flush_cache_vmap(void) 387 { 388 r4k_blast_dcache(); 389 } 390 391 static void r4k__flush_cache_vunmap(void) 392 { 393 r4k_blast_dcache(); 394 } 395 396 static inline void local_r4k_flush_cache_range(void * args) 397 { 398 struct vm_area_struct *vma = args; 399 int exec = vma->vm_flags & VM_EXEC; 400 401 if (!(has_valid_asid(vma->vm_mm))) 402 return; 403 404 r4k_blast_dcache(); 405 if (exec) 406 r4k_blast_icache(); 407 } 408 409 static void r4k_flush_cache_range(struct vm_area_struct *vma, 410 unsigned long start, unsigned long end) 411 { 412 int exec = vma->vm_flags & VM_EXEC; 413 414 if (cpu_has_dc_aliases || (exec && !cpu_has_ic_fills_f_dc)) 415 r4k_on_each_cpu(local_r4k_flush_cache_range, vma); 416 } 417 418 static inline void local_r4k_flush_cache_mm(void * args) 419 { 420 struct mm_struct *mm = args; 421 422 if (!has_valid_asid(mm)) 423 return; 424 425 /* 426 * Kludge alert. For obscure reasons R4000SC and R4400SC go nuts if we 427 * only flush the primary caches but R10000 and R12000 behave sane ... 428 * R4000SC and R4400SC indexed S-cache ops also invalidate primary 429 * caches, so we can bail out early. 430 */ 431 if (current_cpu_type() == CPU_R4000SC || 432 current_cpu_type() == CPU_R4000MC || 433 current_cpu_type() == CPU_R4400SC || 434 current_cpu_type() == CPU_R4400MC) { 435 r4k_blast_scache(); 436 return; 437 } 438 439 r4k_blast_dcache(); 440 } 441 442 static void r4k_flush_cache_mm(struct mm_struct *mm) 443 { 444 if (!cpu_has_dc_aliases) 445 return; 446 447 r4k_on_each_cpu(local_r4k_flush_cache_mm, mm); 448 } 449 450 struct flush_cache_page_args { 451 struct vm_area_struct *vma; 452 unsigned long addr; 453 unsigned long pfn; 454 }; 455 456 static inline void local_r4k_flush_cache_page(void *args) 457 { 458 struct flush_cache_page_args *fcp_args = args; 459 struct vm_area_struct *vma = fcp_args->vma; 460 unsigned long addr = fcp_args->addr; 461 struct page *page = pfn_to_page(fcp_args->pfn); 462 int exec = vma->vm_flags & VM_EXEC; 463 struct mm_struct *mm = vma->vm_mm; 464 int map_coherent = 0; 465 pgd_t *pgdp; 466 pud_t *pudp; 467 pmd_t *pmdp; 468 pte_t *ptep; 469 void *vaddr; 470 471 /* 472 * If ownes no valid ASID yet, cannot possibly have gotten 473 * this page into the cache. 474 */ 475 if (!has_valid_asid(mm)) 476 return; 477 478 addr &= PAGE_MASK; 479 pgdp = pgd_offset(mm, addr); 480 pudp = pud_offset(pgdp, addr); 481 pmdp = pmd_offset(pudp, addr); 482 ptep = pte_offset(pmdp, addr); 483 484 /* 485 * If the page isn't marked valid, the page cannot possibly be 486 * in the cache. 487 */ 488 if (!(pte_present(*ptep))) 489 return; 490 491 if ((mm == current->active_mm) && (pte_val(*ptep) & _PAGE_VALID)) 492 vaddr = NULL; 493 else { 494 /* 495 * Use kmap_coherent or kmap_atomic to do flushes for 496 * another ASID than the current one. 497 */ 498 map_coherent = (cpu_has_dc_aliases && 499 page_mapped(page) && !Page_dcache_dirty(page)); 500 if (map_coherent) 501 vaddr = kmap_coherent(page, addr); 502 else 503 vaddr = kmap_atomic(page); 504 addr = (unsigned long)vaddr; 505 } 506 507 if (cpu_has_dc_aliases || (exec && !cpu_has_ic_fills_f_dc)) { 508 r4k_blast_dcache_page(addr); 509 if (exec && !cpu_icache_snoops_remote_store) 510 r4k_blast_scache_page(addr); 511 } 512 if (exec) { 513 if (vaddr && cpu_has_vtag_icache && mm == current->active_mm) { 514 int cpu = smp_processor_id(); 515 516 if (cpu_context(cpu, mm) != 0) 517 drop_mmu_context(mm, cpu); 518 } else 519 r4k_blast_icache_page(addr); 520 } 521 522 if (vaddr) { 523 if (map_coherent) 524 kunmap_coherent(); 525 else 526 kunmap_atomic(vaddr); 527 } 528 } 529 530 static void r4k_flush_cache_page(struct vm_area_struct *vma, 531 unsigned long addr, unsigned long pfn) 532 { 533 struct flush_cache_page_args args; 534 535 args.vma = vma; 536 args.addr = addr; 537 args.pfn = pfn; 538 539 r4k_on_each_cpu(local_r4k_flush_cache_page, &args); 540 } 541 542 static inline void local_r4k_flush_data_cache_page(void * addr) 543 { 544 r4k_blast_dcache_page((unsigned long) addr); 545 } 546 547 static void r4k_flush_data_cache_page(unsigned long addr) 548 { 549 if (in_atomic()) 550 local_r4k_flush_data_cache_page((void *)addr); 551 else 552 r4k_on_each_cpu(local_r4k_flush_data_cache_page, (void *) addr); 553 } 554 555 struct flush_icache_range_args { 556 unsigned long start; 557 unsigned long end; 558 }; 559 560 static inline void local_r4k_flush_icache_range(unsigned long start, unsigned long end) 561 { 562 if (!cpu_has_ic_fills_f_dc) { 563 if (end - start >= dcache_size) { 564 r4k_blast_dcache(); 565 } else { 566 R4600_HIT_CACHEOP_WAR_IMPL; 567 protected_blast_dcache_range(start, end); 568 } 569 } 570 571 if (end - start > icache_size) 572 r4k_blast_icache(); 573 else 574 protected_blast_icache_range(start, end); 575 } 576 577 static inline void local_r4k_flush_icache_range_ipi(void *args) 578 { 579 struct flush_icache_range_args *fir_args = args; 580 unsigned long start = fir_args->start; 581 unsigned long end = fir_args->end; 582 583 local_r4k_flush_icache_range(start, end); 584 } 585 586 static void r4k_flush_icache_range(unsigned long start, unsigned long end) 587 { 588 struct flush_icache_range_args args; 589 590 args.start = start; 591 args.end = end; 592 593 r4k_on_each_cpu(local_r4k_flush_icache_range_ipi, &args); 594 instruction_hazard(); 595 } 596 597 #ifdef CONFIG_DMA_NONCOHERENT 598 599 static void r4k_dma_cache_wback_inv(unsigned long addr, unsigned long size) 600 { 601 /* Catch bad driver code */ 602 BUG_ON(size == 0); 603 604 if (cpu_has_inclusive_pcaches) { 605 if (size >= scache_size) 606 r4k_blast_scache(); 607 else 608 blast_scache_range(addr, addr + size); 609 __sync(); 610 return; 611 } 612 613 /* 614 * Either no secondary cache or the available caches don't have the 615 * subset property so we have to flush the primary caches 616 * explicitly 617 */ 618 if (cpu_has_safe_index_cacheops && size >= dcache_size) { 619 r4k_blast_dcache(); 620 } else { 621 R4600_HIT_CACHEOP_WAR_IMPL; 622 blast_dcache_range(addr, addr + size); 623 } 624 625 bc_wback_inv(addr, size); 626 __sync(); 627 } 628 629 static void r4k_dma_cache_inv(unsigned long addr, unsigned long size) 630 { 631 /* Catch bad driver code */ 632 BUG_ON(size == 0); 633 634 if (cpu_has_inclusive_pcaches) { 635 if (size >= scache_size) 636 r4k_blast_scache(); 637 else { 638 /* 639 * There is no clearly documented alignment requirement 640 * for the cache instruction on MIPS processors and 641 * some processors, among them the RM5200 and RM7000 642 * QED processors will throw an address error for cache 643 * hit ops with insufficient alignment. Solved by 644 * aligning the address to cache line size. 645 */ 646 blast_inv_scache_range(addr, addr + size); 647 } 648 __sync(); 649 return; 650 } 651 652 if (cpu_has_safe_index_cacheops && size >= dcache_size) { 653 r4k_blast_dcache(); 654 } else { 655 R4600_HIT_CACHEOP_WAR_IMPL; 656 blast_inv_dcache_range(addr, addr + size); 657 } 658 659 bc_inv(addr, size); 660 __sync(); 661 } 662 #endif /* CONFIG_DMA_NONCOHERENT */ 663 664 /* 665 * While we're protected against bad userland addresses we don't care 666 * very much about what happens in that case. Usually a segmentation 667 * fault will dump the process later on anyway ... 668 */ 669 static void local_r4k_flush_cache_sigtramp(void * arg) 670 { 671 unsigned long ic_lsize = cpu_icache_line_size(); 672 unsigned long dc_lsize = cpu_dcache_line_size(); 673 unsigned long sc_lsize = cpu_scache_line_size(); 674 unsigned long addr = (unsigned long) arg; 675 676 R4600_HIT_CACHEOP_WAR_IMPL; 677 if (dc_lsize) 678 protected_writeback_dcache_line(addr & ~(dc_lsize - 1)); 679 if (!cpu_icache_snoops_remote_store && scache_size) 680 protected_writeback_scache_line(addr & ~(sc_lsize - 1)); 681 if (ic_lsize) 682 protected_flush_icache_line(addr & ~(ic_lsize - 1)); 683 if (MIPS4K_ICACHE_REFILL_WAR) { 684 __asm__ __volatile__ ( 685 ".set push\n\t" 686 ".set noat\n\t" 687 ".set mips3\n\t" 688 #ifdef CONFIG_32BIT 689 "la $at,1f\n\t" 690 #endif 691 #ifdef CONFIG_64BIT 692 "dla $at,1f\n\t" 693 #endif 694 "cache %0,($at)\n\t" 695 "nop; nop; nop\n" 696 "1:\n\t" 697 ".set pop" 698 : 699 : "i" (Hit_Invalidate_I)); 700 } 701 if (MIPS_CACHE_SYNC_WAR) 702 __asm__ __volatile__ ("sync"); 703 } 704 705 static void r4k_flush_cache_sigtramp(unsigned long addr) 706 { 707 r4k_on_each_cpu(local_r4k_flush_cache_sigtramp, (void *) addr); 708 } 709 710 static void r4k_flush_icache_all(void) 711 { 712 if (cpu_has_vtag_icache) 713 r4k_blast_icache(); 714 } 715 716 struct flush_kernel_vmap_range_args { 717 unsigned long vaddr; 718 int size; 719 }; 720 721 static inline void local_r4k_flush_kernel_vmap_range(void *args) 722 { 723 struct flush_kernel_vmap_range_args *vmra = args; 724 unsigned long vaddr = vmra->vaddr; 725 int size = vmra->size; 726 727 /* 728 * Aliases only affect the primary caches so don't bother with 729 * S-caches or T-caches. 730 */ 731 if (cpu_has_safe_index_cacheops && size >= dcache_size) 732 r4k_blast_dcache(); 733 else { 734 R4600_HIT_CACHEOP_WAR_IMPL; 735 blast_dcache_range(vaddr, vaddr + size); 736 } 737 } 738 739 static void r4k_flush_kernel_vmap_range(unsigned long vaddr, int size) 740 { 741 struct flush_kernel_vmap_range_args args; 742 743 args.vaddr = (unsigned long) vaddr; 744 args.size = size; 745 746 r4k_on_each_cpu(local_r4k_flush_kernel_vmap_range, &args); 747 } 748 749 static inline void rm7k_erratum31(void) 750 { 751 const unsigned long ic_lsize = 32; 752 unsigned long addr; 753 754 /* RM7000 erratum #31. The icache is screwed at startup. */ 755 write_c0_taglo(0); 756 write_c0_taghi(0); 757 758 for (addr = INDEX_BASE; addr <= INDEX_BASE + 4096; addr += ic_lsize) { 759 __asm__ __volatile__ ( 760 ".set push\n\t" 761 ".set noreorder\n\t" 762 ".set mips3\n\t" 763 "cache\t%1, 0(%0)\n\t" 764 "cache\t%1, 0x1000(%0)\n\t" 765 "cache\t%1, 0x2000(%0)\n\t" 766 "cache\t%1, 0x3000(%0)\n\t" 767 "cache\t%2, 0(%0)\n\t" 768 "cache\t%2, 0x1000(%0)\n\t" 769 "cache\t%2, 0x2000(%0)\n\t" 770 "cache\t%2, 0x3000(%0)\n\t" 771 "cache\t%1, 0(%0)\n\t" 772 "cache\t%1, 0x1000(%0)\n\t" 773 "cache\t%1, 0x2000(%0)\n\t" 774 "cache\t%1, 0x3000(%0)\n\t" 775 ".set pop\n" 776 : 777 : "r" (addr), "i" (Index_Store_Tag_I), "i" (Fill)); 778 } 779 } 780 781 static inline void alias_74k_erratum(struct cpuinfo_mips *c) 782 { 783 /* 784 * Early versions of the 74K do not update the cache tags on a 785 * vtag miss/ptag hit which can occur in the case of KSEG0/KUSEG 786 * aliases. In this case it is better to treat the cache as always 787 * having aliases. 788 */ 789 if ((c->processor_id & 0xff) <= PRID_REV_ENCODE_332(2, 4, 0)) 790 c->dcache.flags |= MIPS_CACHE_VTAG; 791 if ((c->processor_id & 0xff) == PRID_REV_ENCODE_332(2, 4, 0)) 792 write_c0_config6(read_c0_config6() | MIPS_CONF6_SYND); 793 if (((c->processor_id & 0xff00) == PRID_IMP_1074K) && 794 ((c->processor_id & 0xff) <= PRID_REV_ENCODE_332(1, 1, 0))) { 795 c->dcache.flags |= MIPS_CACHE_VTAG; 796 write_c0_config6(read_c0_config6() | MIPS_CONF6_SYND); 797 } 798 } 799 800 static char *way_string[] __cpuinitdata = { NULL, "direct mapped", "2-way", 801 "3-way", "4-way", "5-way", "6-way", "7-way", "8-way" 802 }; 803 804 static void __cpuinit probe_pcache(void) 805 { 806 struct cpuinfo_mips *c = ¤t_cpu_data; 807 unsigned int config = read_c0_config(); 808 unsigned int prid = read_c0_prid(); 809 unsigned long config1; 810 unsigned int lsize; 811 812 switch (c->cputype) { 813 case CPU_R4600: /* QED style two way caches? */ 814 case CPU_R4700: 815 case CPU_R5000: 816 case CPU_NEVADA: 817 icache_size = 1 << (12 + ((config & CONF_IC) >> 9)); 818 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 819 c->icache.ways = 2; 820 c->icache.waybit = __ffs(icache_size/2); 821 822 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6)); 823 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 824 c->dcache.ways = 2; 825 c->dcache.waybit= __ffs(dcache_size/2); 826 827 c->options |= MIPS_CPU_CACHE_CDEX_P; 828 break; 829 830 case CPU_R5432: 831 case CPU_R5500: 832 icache_size = 1 << (12 + ((config & CONF_IC) >> 9)); 833 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 834 c->icache.ways = 2; 835 c->icache.waybit= 0; 836 837 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6)); 838 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 839 c->dcache.ways = 2; 840 c->dcache.waybit = 0; 841 842 c->options |= MIPS_CPU_CACHE_CDEX_P | MIPS_CPU_PREFETCH; 843 break; 844 845 case CPU_TX49XX: 846 icache_size = 1 << (12 + ((config & CONF_IC) >> 9)); 847 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 848 c->icache.ways = 4; 849 c->icache.waybit= 0; 850 851 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6)); 852 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 853 c->dcache.ways = 4; 854 c->dcache.waybit = 0; 855 856 c->options |= MIPS_CPU_CACHE_CDEX_P; 857 c->options |= MIPS_CPU_PREFETCH; 858 break; 859 860 case CPU_R4000PC: 861 case CPU_R4000SC: 862 case CPU_R4000MC: 863 case CPU_R4400PC: 864 case CPU_R4400SC: 865 case CPU_R4400MC: 866 case CPU_R4300: 867 icache_size = 1 << (12 + ((config & CONF_IC) >> 9)); 868 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 869 c->icache.ways = 1; 870 c->icache.waybit = 0; /* doesn't matter */ 871 872 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6)); 873 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 874 c->dcache.ways = 1; 875 c->dcache.waybit = 0; /* does not matter */ 876 877 c->options |= MIPS_CPU_CACHE_CDEX_P; 878 break; 879 880 case CPU_R10000: 881 case CPU_R12000: 882 case CPU_R14000: 883 icache_size = 1 << (12 + ((config & R10K_CONF_IC) >> 29)); 884 c->icache.linesz = 64; 885 c->icache.ways = 2; 886 c->icache.waybit = 0; 887 888 dcache_size = 1 << (12 + ((config & R10K_CONF_DC) >> 26)); 889 c->dcache.linesz = 32; 890 c->dcache.ways = 2; 891 c->dcache.waybit = 0; 892 893 c->options |= MIPS_CPU_PREFETCH; 894 break; 895 896 case CPU_VR4133: 897 write_c0_config(config & ~VR41_CONF_P4K); 898 case CPU_VR4131: 899 /* Workaround for cache instruction bug of VR4131 */ 900 if (c->processor_id == 0x0c80U || c->processor_id == 0x0c81U || 901 c->processor_id == 0x0c82U) { 902 config |= 0x00400000U; 903 if (c->processor_id == 0x0c80U) 904 config |= VR41_CONF_BP; 905 write_c0_config(config); 906 } else 907 c->options |= MIPS_CPU_CACHE_CDEX_P; 908 909 icache_size = 1 << (10 + ((config & CONF_IC) >> 9)); 910 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 911 c->icache.ways = 2; 912 c->icache.waybit = __ffs(icache_size/2); 913 914 dcache_size = 1 << (10 + ((config & CONF_DC) >> 6)); 915 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 916 c->dcache.ways = 2; 917 c->dcache.waybit = __ffs(dcache_size/2); 918 break; 919 920 case CPU_VR41XX: 921 case CPU_VR4111: 922 case CPU_VR4121: 923 case CPU_VR4122: 924 case CPU_VR4181: 925 case CPU_VR4181A: 926 icache_size = 1 << (10 + ((config & CONF_IC) >> 9)); 927 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 928 c->icache.ways = 1; 929 c->icache.waybit = 0; /* doesn't matter */ 930 931 dcache_size = 1 << (10 + ((config & CONF_DC) >> 6)); 932 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 933 c->dcache.ways = 1; 934 c->dcache.waybit = 0; /* does not matter */ 935 936 c->options |= MIPS_CPU_CACHE_CDEX_P; 937 break; 938 939 case CPU_RM7000: 940 rm7k_erratum31(); 941 942 icache_size = 1 << (12 + ((config & CONF_IC) >> 9)); 943 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 944 c->icache.ways = 4; 945 c->icache.waybit = __ffs(icache_size / c->icache.ways); 946 947 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6)); 948 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 949 c->dcache.ways = 4; 950 c->dcache.waybit = __ffs(dcache_size / c->dcache.ways); 951 952 c->options |= MIPS_CPU_CACHE_CDEX_P; 953 c->options |= MIPS_CPU_PREFETCH; 954 break; 955 956 case CPU_LOONGSON2: 957 icache_size = 1 << (12 + ((config & CONF_IC) >> 9)); 958 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 959 if (prid & 0x3) 960 c->icache.ways = 4; 961 else 962 c->icache.ways = 2; 963 c->icache.waybit = 0; 964 965 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6)); 966 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 967 if (prid & 0x3) 968 c->dcache.ways = 4; 969 else 970 c->dcache.ways = 2; 971 c->dcache.waybit = 0; 972 break; 973 974 default: 975 if (!(config & MIPS_CONF_M)) 976 panic("Don't know how to probe P-caches on this cpu."); 977 978 /* 979 * So we seem to be a MIPS32 or MIPS64 CPU 980 * So let's probe the I-cache ... 981 */ 982 config1 = read_c0_config1(); 983 984 if ((lsize = ((config1 >> 19) & 7))) 985 c->icache.linesz = 2 << lsize; 986 else 987 c->icache.linesz = lsize; 988 c->icache.sets = 32 << (((config1 >> 22) + 1) & 7); 989 c->icache.ways = 1 + ((config1 >> 16) & 7); 990 991 icache_size = c->icache.sets * 992 c->icache.ways * 993 c->icache.linesz; 994 c->icache.waybit = __ffs(icache_size/c->icache.ways); 995 996 if (config & 0x8) /* VI bit */ 997 c->icache.flags |= MIPS_CACHE_VTAG; 998 999 /* 1000 * Now probe the MIPS32 / MIPS64 data cache. 1001 */ 1002 c->dcache.flags = 0; 1003 1004 if ((lsize = ((config1 >> 10) & 7))) 1005 c->dcache.linesz = 2 << lsize; 1006 else 1007 c->dcache.linesz= lsize; 1008 c->dcache.sets = 32 << (((config1 >> 13) + 1) & 7); 1009 c->dcache.ways = 1 + ((config1 >> 7) & 7); 1010 1011 dcache_size = c->dcache.sets * 1012 c->dcache.ways * 1013 c->dcache.linesz; 1014 c->dcache.waybit = __ffs(dcache_size/c->dcache.ways); 1015 1016 c->options |= MIPS_CPU_PREFETCH; 1017 break; 1018 } 1019 1020 /* 1021 * Processor configuration sanity check for the R4000SC erratum 1022 * #5. With page sizes larger than 32kB there is no possibility 1023 * to get a VCE exception anymore so we don't care about this 1024 * misconfiguration. The case is rather theoretical anyway; 1025 * presumably no vendor is shipping his hardware in the "bad" 1026 * configuration. 1027 */ 1028 if ((prid & 0xff00) == PRID_IMP_R4000 && (prid & 0xff) < 0x40 && 1029 !(config & CONF_SC) && c->icache.linesz != 16 && 1030 PAGE_SIZE <= 0x8000) 1031 panic("Improper R4000SC processor configuration detected"); 1032 1033 /* compute a couple of other cache variables */ 1034 c->icache.waysize = icache_size / c->icache.ways; 1035 c->dcache.waysize = dcache_size / c->dcache.ways; 1036 1037 c->icache.sets = c->icache.linesz ? 1038 icache_size / (c->icache.linesz * c->icache.ways) : 0; 1039 c->dcache.sets = c->dcache.linesz ? 1040 dcache_size / (c->dcache.linesz * c->dcache.ways) : 0; 1041 1042 /* 1043 * R10000 and R12000 P-caches are odd in a positive way. They're 32kB 1044 * 2-way virtually indexed so normally would suffer from aliases. So 1045 * normally they'd suffer from aliases but magic in the hardware deals 1046 * with that for us so we don't need to take care ourselves. 1047 */ 1048 switch (c->cputype) { 1049 case CPU_20KC: 1050 case CPU_25KF: 1051 case CPU_SB1: 1052 case CPU_SB1A: 1053 case CPU_XLR: 1054 c->dcache.flags |= MIPS_CACHE_PINDEX; 1055 break; 1056 1057 case CPU_R10000: 1058 case CPU_R12000: 1059 case CPU_R14000: 1060 break; 1061 1062 case CPU_M14KC: 1063 case CPU_M14KEC: 1064 case CPU_24K: 1065 case CPU_34K: 1066 case CPU_74K: 1067 case CPU_1004K: 1068 if (c->cputype == CPU_74K) 1069 alias_74k_erratum(c); 1070 if ((read_c0_config7() & (1 << 16))) { 1071 /* effectively physically indexed dcache, 1072 thus no virtual aliases. */ 1073 c->dcache.flags |= MIPS_CACHE_PINDEX; 1074 break; 1075 } 1076 default: 1077 if (c->dcache.waysize > PAGE_SIZE) 1078 c->dcache.flags |= MIPS_CACHE_ALIASES; 1079 } 1080 1081 switch (c->cputype) { 1082 case CPU_20KC: 1083 /* 1084 * Some older 20Kc chips doesn't have the 'VI' bit in 1085 * the config register. 1086 */ 1087 c->icache.flags |= MIPS_CACHE_VTAG; 1088 break; 1089 1090 case CPU_ALCHEMY: 1091 c->icache.flags |= MIPS_CACHE_IC_F_DC; 1092 break; 1093 } 1094 1095 #ifdef CONFIG_CPU_LOONGSON2 1096 /* 1097 * LOONGSON2 has 4 way icache, but when using indexed cache op, 1098 * one op will act on all 4 ways 1099 */ 1100 c->icache.ways = 1; 1101 #endif 1102 1103 printk("Primary instruction cache %ldkB, %s, %s, linesize %d bytes.\n", 1104 icache_size >> 10, 1105 c->icache.flags & MIPS_CACHE_VTAG ? "VIVT" : "VIPT", 1106 way_string[c->icache.ways], c->icache.linesz); 1107 1108 printk("Primary data cache %ldkB, %s, %s, %s, linesize %d bytes\n", 1109 dcache_size >> 10, way_string[c->dcache.ways], 1110 (c->dcache.flags & MIPS_CACHE_PINDEX) ? "PIPT" : "VIPT", 1111 (c->dcache.flags & MIPS_CACHE_ALIASES) ? 1112 "cache aliases" : "no aliases", 1113 c->dcache.linesz); 1114 } 1115 1116 /* 1117 * If you even _breathe_ on this function, look at the gcc output and make sure 1118 * it does not pop things on and off the stack for the cache sizing loop that 1119 * executes in KSEG1 space or else you will crash and burn badly. You have 1120 * been warned. 1121 */ 1122 static int __cpuinit probe_scache(void) 1123 { 1124 unsigned long flags, addr, begin, end, pow2; 1125 unsigned int config = read_c0_config(); 1126 struct cpuinfo_mips *c = ¤t_cpu_data; 1127 1128 if (config & CONF_SC) 1129 return 0; 1130 1131 begin = (unsigned long) &_stext; 1132 begin &= ~((4 * 1024 * 1024) - 1); 1133 end = begin + (4 * 1024 * 1024); 1134 1135 /* 1136 * This is such a bitch, you'd think they would make it easy to do 1137 * this. Away you daemons of stupidity! 1138 */ 1139 local_irq_save(flags); 1140 1141 /* Fill each size-multiple cache line with a valid tag. */ 1142 pow2 = (64 * 1024); 1143 for (addr = begin; addr < end; addr = (begin + pow2)) { 1144 unsigned long *p = (unsigned long *) addr; 1145 __asm__ __volatile__("nop" : : "r" (*p)); /* whee... */ 1146 pow2 <<= 1; 1147 } 1148 1149 /* Load first line with zero (therefore invalid) tag. */ 1150 write_c0_taglo(0); 1151 write_c0_taghi(0); 1152 __asm__ __volatile__("nop; nop; nop; nop;"); /* avoid the hazard */ 1153 cache_op(Index_Store_Tag_I, begin); 1154 cache_op(Index_Store_Tag_D, begin); 1155 cache_op(Index_Store_Tag_SD, begin); 1156 1157 /* Now search for the wrap around point. */ 1158 pow2 = (128 * 1024); 1159 for (addr = begin + (128 * 1024); addr < end; addr = begin + pow2) { 1160 cache_op(Index_Load_Tag_SD, addr); 1161 __asm__ __volatile__("nop; nop; nop; nop;"); /* hazard... */ 1162 if (!read_c0_taglo()) 1163 break; 1164 pow2 <<= 1; 1165 } 1166 local_irq_restore(flags); 1167 addr -= begin; 1168 1169 scache_size = addr; 1170 c->scache.linesz = 16 << ((config & R4K_CONF_SB) >> 22); 1171 c->scache.ways = 1; 1172 c->dcache.waybit = 0; /* does not matter */ 1173 1174 return 1; 1175 } 1176 1177 #if defined(CONFIG_CPU_LOONGSON2) 1178 static void __init loongson2_sc_init(void) 1179 { 1180 struct cpuinfo_mips *c = ¤t_cpu_data; 1181 1182 scache_size = 512*1024; 1183 c->scache.linesz = 32; 1184 c->scache.ways = 4; 1185 c->scache.waybit = 0; 1186 c->scache.waysize = scache_size / (c->scache.ways); 1187 c->scache.sets = scache_size / (c->scache.linesz * c->scache.ways); 1188 pr_info("Unified secondary cache %ldkB %s, linesize %d bytes.\n", 1189 scache_size >> 10, way_string[c->scache.ways], c->scache.linesz); 1190 1191 c->options |= MIPS_CPU_INCLUSIVE_CACHES; 1192 } 1193 #endif 1194 1195 extern int r5k_sc_init(void); 1196 extern int rm7k_sc_init(void); 1197 extern int mips_sc_init(void); 1198 1199 static void __cpuinit setup_scache(void) 1200 { 1201 struct cpuinfo_mips *c = ¤t_cpu_data; 1202 unsigned int config = read_c0_config(); 1203 int sc_present = 0; 1204 1205 /* 1206 * Do the probing thing on R4000SC and R4400SC processors. Other 1207 * processors don't have a S-cache that would be relevant to the 1208 * Linux memory management. 1209 */ 1210 switch (c->cputype) { 1211 case CPU_R4000SC: 1212 case CPU_R4000MC: 1213 case CPU_R4400SC: 1214 case CPU_R4400MC: 1215 sc_present = run_uncached(probe_scache); 1216 if (sc_present) 1217 c->options |= MIPS_CPU_CACHE_CDEX_S; 1218 break; 1219 1220 case CPU_R10000: 1221 case CPU_R12000: 1222 case CPU_R14000: 1223 scache_size = 0x80000 << ((config & R10K_CONF_SS) >> 16); 1224 c->scache.linesz = 64 << ((config >> 13) & 1); 1225 c->scache.ways = 2; 1226 c->scache.waybit= 0; 1227 sc_present = 1; 1228 break; 1229 1230 case CPU_R5000: 1231 case CPU_NEVADA: 1232 #ifdef CONFIG_R5000_CPU_SCACHE 1233 r5k_sc_init(); 1234 #endif 1235 return; 1236 1237 case CPU_RM7000: 1238 #ifdef CONFIG_RM7000_CPU_SCACHE 1239 rm7k_sc_init(); 1240 #endif 1241 return; 1242 1243 #if defined(CONFIG_CPU_LOONGSON2) 1244 case CPU_LOONGSON2: 1245 loongson2_sc_init(); 1246 return; 1247 #endif 1248 case CPU_XLP: 1249 /* don't need to worry about L2, fully coherent */ 1250 return; 1251 1252 default: 1253 if (c->isa_level & (MIPS_CPU_ISA_M32R1 | MIPS_CPU_ISA_M32R2 | 1254 MIPS_CPU_ISA_M64R1 | MIPS_CPU_ISA_M64R2)) { 1255 #ifdef CONFIG_MIPS_CPU_SCACHE 1256 if (mips_sc_init ()) { 1257 scache_size = c->scache.ways * c->scache.sets * c->scache.linesz; 1258 printk("MIPS secondary cache %ldkB, %s, linesize %d bytes.\n", 1259 scache_size >> 10, 1260 way_string[c->scache.ways], c->scache.linesz); 1261 } 1262 #else 1263 if (!(c->scache.flags & MIPS_CACHE_NOT_PRESENT)) 1264 panic("Dunno how to handle MIPS32 / MIPS64 second level cache"); 1265 #endif 1266 return; 1267 } 1268 sc_present = 0; 1269 } 1270 1271 if (!sc_present) 1272 return; 1273 1274 /* compute a couple of other cache variables */ 1275 c->scache.waysize = scache_size / c->scache.ways; 1276 1277 c->scache.sets = scache_size / (c->scache.linesz * c->scache.ways); 1278 1279 printk("Unified secondary cache %ldkB %s, linesize %d bytes.\n", 1280 scache_size >> 10, way_string[c->scache.ways], c->scache.linesz); 1281 1282 c->options |= MIPS_CPU_INCLUSIVE_CACHES; 1283 } 1284 1285 void au1x00_fixup_config_od(void) 1286 { 1287 /* 1288 * c0_config.od (bit 19) was write only (and read as 0) 1289 * on the early revisions of Alchemy SOCs. It disables the bus 1290 * transaction overlapping and needs to be set to fix various errata. 1291 */ 1292 switch (read_c0_prid()) { 1293 case 0x00030100: /* Au1000 DA */ 1294 case 0x00030201: /* Au1000 HA */ 1295 case 0x00030202: /* Au1000 HB */ 1296 case 0x01030200: /* Au1500 AB */ 1297 /* 1298 * Au1100 errata actually keeps silence about this bit, so we set it 1299 * just in case for those revisions that require it to be set according 1300 * to the (now gone) cpu table. 1301 */ 1302 case 0x02030200: /* Au1100 AB */ 1303 case 0x02030201: /* Au1100 BA */ 1304 case 0x02030202: /* Au1100 BC */ 1305 set_c0_config(1 << 19); 1306 break; 1307 } 1308 } 1309 1310 /* CP0 hazard avoidance. */ 1311 #define NXP_BARRIER() \ 1312 __asm__ __volatile__( \ 1313 ".set noreorder\n\t" \ 1314 "nop; nop; nop; nop; nop; nop;\n\t" \ 1315 ".set reorder\n\t") 1316 1317 static void nxp_pr4450_fixup_config(void) 1318 { 1319 unsigned long config0; 1320 1321 config0 = read_c0_config(); 1322 1323 /* clear all three cache coherency fields */ 1324 config0 &= ~(0x7 | (7 << 25) | (7 << 28)); 1325 config0 |= (((_page_cachable_default >> _CACHE_SHIFT) << 0) | 1326 ((_page_cachable_default >> _CACHE_SHIFT) << 25) | 1327 ((_page_cachable_default >> _CACHE_SHIFT) << 28)); 1328 write_c0_config(config0); 1329 NXP_BARRIER(); 1330 } 1331 1332 static int __cpuinitdata cca = -1; 1333 1334 static int __init cca_setup(char *str) 1335 { 1336 get_option(&str, &cca); 1337 1338 return 0; 1339 } 1340 1341 early_param("cca", cca_setup); 1342 1343 static void __cpuinit coherency_setup(void) 1344 { 1345 if (cca < 0 || cca > 7) 1346 cca = read_c0_config() & CONF_CM_CMASK; 1347 _page_cachable_default = cca << _CACHE_SHIFT; 1348 1349 pr_debug("Using cache attribute %d\n", cca); 1350 change_c0_config(CONF_CM_CMASK, cca); 1351 1352 /* 1353 * c0_status.cu=0 specifies that updates by the sc instruction use 1354 * the coherency mode specified by the TLB; 1 means cachable 1355 * coherent update on write will be used. Not all processors have 1356 * this bit and; some wire it to zero, others like Toshiba had the 1357 * silly idea of putting something else there ... 1358 */ 1359 switch (current_cpu_type()) { 1360 case CPU_R4000PC: 1361 case CPU_R4000SC: 1362 case CPU_R4000MC: 1363 case CPU_R4400PC: 1364 case CPU_R4400SC: 1365 case CPU_R4400MC: 1366 clear_c0_config(CONF_CU); 1367 break; 1368 /* 1369 * We need to catch the early Alchemy SOCs with 1370 * the write-only co_config.od bit and set it back to one on: 1371 * Au1000 rev DA, HA, HB; Au1100 AB, BA, BC, Au1500 AB 1372 */ 1373 case CPU_ALCHEMY: 1374 au1x00_fixup_config_od(); 1375 break; 1376 1377 case PRID_IMP_PR4450: 1378 nxp_pr4450_fixup_config(); 1379 break; 1380 } 1381 } 1382 1383 static void __cpuinit r4k_cache_error_setup(void) 1384 { 1385 extern char __weak except_vec2_generic; 1386 extern char __weak except_vec2_sb1; 1387 struct cpuinfo_mips *c = ¤t_cpu_data; 1388 1389 switch (c->cputype) { 1390 case CPU_SB1: 1391 case CPU_SB1A: 1392 set_uncached_handler(0x100, &except_vec2_sb1, 0x80); 1393 break; 1394 1395 default: 1396 set_uncached_handler(0x100, &except_vec2_generic, 0x80); 1397 break; 1398 } 1399 } 1400 1401 void __cpuinit r4k_cache_init(void) 1402 { 1403 extern void build_clear_page(void); 1404 extern void build_copy_page(void); 1405 struct cpuinfo_mips *c = ¤t_cpu_data; 1406 1407 probe_pcache(); 1408 setup_scache(); 1409 1410 r4k_blast_dcache_page_setup(); 1411 r4k_blast_dcache_page_indexed_setup(); 1412 r4k_blast_dcache_setup(); 1413 r4k_blast_icache_page_setup(); 1414 r4k_blast_icache_page_indexed_setup(); 1415 r4k_blast_icache_setup(); 1416 r4k_blast_scache_page_setup(); 1417 r4k_blast_scache_page_indexed_setup(); 1418 r4k_blast_scache_setup(); 1419 1420 /* 1421 * Some MIPS32 and MIPS64 processors have physically indexed caches. 1422 * This code supports virtually indexed processors and will be 1423 * unnecessarily inefficient on physically indexed processors. 1424 */ 1425 if (c->dcache.linesz) 1426 shm_align_mask = max_t( unsigned long, 1427 c->dcache.sets * c->dcache.linesz - 1, 1428 PAGE_SIZE - 1); 1429 else 1430 shm_align_mask = PAGE_SIZE-1; 1431 1432 __flush_cache_vmap = r4k__flush_cache_vmap; 1433 __flush_cache_vunmap = r4k__flush_cache_vunmap; 1434 1435 flush_cache_all = cache_noop; 1436 __flush_cache_all = r4k___flush_cache_all; 1437 flush_cache_mm = r4k_flush_cache_mm; 1438 flush_cache_page = r4k_flush_cache_page; 1439 flush_cache_range = r4k_flush_cache_range; 1440 1441 __flush_kernel_vmap_range = r4k_flush_kernel_vmap_range; 1442 1443 flush_cache_sigtramp = r4k_flush_cache_sigtramp; 1444 flush_icache_all = r4k_flush_icache_all; 1445 local_flush_data_cache_page = local_r4k_flush_data_cache_page; 1446 flush_data_cache_page = r4k_flush_data_cache_page; 1447 flush_icache_range = r4k_flush_icache_range; 1448 local_flush_icache_range = local_r4k_flush_icache_range; 1449 1450 #if defined(CONFIG_DMA_NONCOHERENT) 1451 if (coherentio) { 1452 _dma_cache_wback_inv = (void *)cache_noop; 1453 _dma_cache_wback = (void *)cache_noop; 1454 _dma_cache_inv = (void *)cache_noop; 1455 } else { 1456 _dma_cache_wback_inv = r4k_dma_cache_wback_inv; 1457 _dma_cache_wback = r4k_dma_cache_wback_inv; 1458 _dma_cache_inv = r4k_dma_cache_inv; 1459 } 1460 #endif 1461 1462 build_clear_page(); 1463 build_copy_page(); 1464 1465 /* 1466 * We want to run CMP kernels on core with and without coherent 1467 * caches. Therefore, do not use CONFIG_MIPS_CMP to decide whether 1468 * or not to flush caches. 1469 */ 1470 local_r4k___flush_cache_all(NULL); 1471 1472 coherency_setup(); 1473 board_cache_error_setup = r4k_cache_error_setup; 1474 } 1475