1 /* 2 * This file is subject to the terms and conditions of the GNU General Public 3 * License. See the file "COPYING" in the main directory of this archive 4 * for more details. 5 * 6 * Copyright (C) 1996 David S. Miller (davem@davemloft.net) 7 * Copyright (C) 1997, 1998, 1999, 2000, 2001, 2002 Ralf Baechle (ralf@gnu.org) 8 * Copyright (C) 1999, 2000 Silicon Graphics, Inc. 9 */ 10 #include <linux/hardirq.h> 11 #include <linux/init.h> 12 #include <linux/highmem.h> 13 #include <linux/kernel.h> 14 #include <linux/linkage.h> 15 #include <linux/sched.h> 16 #include <linux/smp.h> 17 #include <linux/mm.h> 18 #include <linux/module.h> 19 #include <linux/bitops.h> 20 21 #include <asm/bcache.h> 22 #include <asm/bootinfo.h> 23 #include <asm/cache.h> 24 #include <asm/cacheops.h> 25 #include <asm/cpu.h> 26 #include <asm/cpu-features.h> 27 #include <asm/io.h> 28 #include <asm/page.h> 29 #include <asm/pgtable.h> 30 #include <asm/r4kcache.h> 31 #include <asm/sections.h> 32 #include <asm/system.h> 33 #include <asm/mmu_context.h> 34 #include <asm/war.h> 35 #include <asm/cacheflush.h> /* for run_uncached() */ 36 37 38 /* 39 * Special Variant of smp_call_function for use by cache functions: 40 * 41 * o No return value 42 * o collapses to normal function call on UP kernels 43 * o collapses to normal function call on systems with a single shared 44 * primary cache. 45 * o doesn't disable interrupts on the local CPU 46 */ 47 static inline void r4k_on_each_cpu(void (*func) (void *info), void *info) 48 { 49 preempt_disable(); 50 51 #if !defined(CONFIG_MIPS_MT_SMP) && !defined(CONFIG_MIPS_MT_SMTC) 52 smp_call_function(func, info, 1); 53 #endif 54 func(info); 55 preempt_enable(); 56 } 57 58 #if defined(CONFIG_MIPS_CMP) 59 #define cpu_has_safe_index_cacheops 0 60 #else 61 #define cpu_has_safe_index_cacheops 1 62 #endif 63 64 /* 65 * Must die. 66 */ 67 static unsigned long icache_size __read_mostly; 68 static unsigned long dcache_size __read_mostly; 69 static unsigned long scache_size __read_mostly; 70 71 /* 72 * Dummy cache handling routines for machines without boardcaches 73 */ 74 static void cache_noop(void) {} 75 76 static struct bcache_ops no_sc_ops = { 77 .bc_enable = (void *)cache_noop, 78 .bc_disable = (void *)cache_noop, 79 .bc_wback_inv = (void *)cache_noop, 80 .bc_inv = (void *)cache_noop 81 }; 82 83 struct bcache_ops *bcops = &no_sc_ops; 84 85 #define cpu_is_r4600_v1_x() ((read_c0_prid() & 0xfffffff0) == 0x00002010) 86 #define cpu_is_r4600_v2_x() ((read_c0_prid() & 0xfffffff0) == 0x00002020) 87 88 #define R4600_HIT_CACHEOP_WAR_IMPL \ 89 do { \ 90 if (R4600_V2_HIT_CACHEOP_WAR && cpu_is_r4600_v2_x()) \ 91 *(volatile unsigned long *)CKSEG1; \ 92 if (R4600_V1_HIT_CACHEOP_WAR) \ 93 __asm__ __volatile__("nop;nop;nop;nop"); \ 94 } while (0) 95 96 static void (*r4k_blast_dcache_page)(unsigned long addr); 97 98 static inline void r4k_blast_dcache_page_dc32(unsigned long addr) 99 { 100 R4600_HIT_CACHEOP_WAR_IMPL; 101 blast_dcache32_page(addr); 102 } 103 104 static inline void r4k_blast_dcache_page_dc64(unsigned long addr) 105 { 106 R4600_HIT_CACHEOP_WAR_IMPL; 107 blast_dcache64_page(addr); 108 } 109 110 static void __cpuinit r4k_blast_dcache_page_setup(void) 111 { 112 unsigned long dc_lsize = cpu_dcache_line_size(); 113 114 if (dc_lsize == 0) 115 r4k_blast_dcache_page = (void *)cache_noop; 116 else if (dc_lsize == 16) 117 r4k_blast_dcache_page = blast_dcache16_page; 118 else if (dc_lsize == 32) 119 r4k_blast_dcache_page = r4k_blast_dcache_page_dc32; 120 else if (dc_lsize == 64) 121 r4k_blast_dcache_page = r4k_blast_dcache_page_dc64; 122 } 123 124 static void (* r4k_blast_dcache_page_indexed)(unsigned long addr); 125 126 static void __cpuinit r4k_blast_dcache_page_indexed_setup(void) 127 { 128 unsigned long dc_lsize = cpu_dcache_line_size(); 129 130 if (dc_lsize == 0) 131 r4k_blast_dcache_page_indexed = (void *)cache_noop; 132 else if (dc_lsize == 16) 133 r4k_blast_dcache_page_indexed = blast_dcache16_page_indexed; 134 else if (dc_lsize == 32) 135 r4k_blast_dcache_page_indexed = blast_dcache32_page_indexed; 136 else if (dc_lsize == 64) 137 r4k_blast_dcache_page_indexed = blast_dcache64_page_indexed; 138 } 139 140 static void (* r4k_blast_dcache)(void); 141 142 static void __cpuinit r4k_blast_dcache_setup(void) 143 { 144 unsigned long dc_lsize = cpu_dcache_line_size(); 145 146 if (dc_lsize == 0) 147 r4k_blast_dcache = (void *)cache_noop; 148 else if (dc_lsize == 16) 149 r4k_blast_dcache = blast_dcache16; 150 else if (dc_lsize == 32) 151 r4k_blast_dcache = blast_dcache32; 152 else if (dc_lsize == 64) 153 r4k_blast_dcache = blast_dcache64; 154 } 155 156 /* force code alignment (used for TX49XX_ICACHE_INDEX_INV_WAR) */ 157 #define JUMP_TO_ALIGN(order) \ 158 __asm__ __volatile__( \ 159 "b\t1f\n\t" \ 160 ".align\t" #order "\n\t" \ 161 "1:\n\t" \ 162 ) 163 #define CACHE32_UNROLL32_ALIGN JUMP_TO_ALIGN(10) /* 32 * 32 = 1024 */ 164 #define CACHE32_UNROLL32_ALIGN2 JUMP_TO_ALIGN(11) 165 166 static inline void blast_r4600_v1_icache32(void) 167 { 168 unsigned long flags; 169 170 local_irq_save(flags); 171 blast_icache32(); 172 local_irq_restore(flags); 173 } 174 175 static inline void tx49_blast_icache32(void) 176 { 177 unsigned long start = INDEX_BASE; 178 unsigned long end = start + current_cpu_data.icache.waysize; 179 unsigned long ws_inc = 1UL << current_cpu_data.icache.waybit; 180 unsigned long ws_end = current_cpu_data.icache.ways << 181 current_cpu_data.icache.waybit; 182 unsigned long ws, addr; 183 184 CACHE32_UNROLL32_ALIGN2; 185 /* I'm in even chunk. blast odd chunks */ 186 for (ws = 0; ws < ws_end; ws += ws_inc) 187 for (addr = start + 0x400; addr < end; addr += 0x400 * 2) 188 cache32_unroll32(addr|ws, Index_Invalidate_I); 189 CACHE32_UNROLL32_ALIGN; 190 /* I'm in odd chunk. blast even chunks */ 191 for (ws = 0; ws < ws_end; ws += ws_inc) 192 for (addr = start; addr < end; addr += 0x400 * 2) 193 cache32_unroll32(addr|ws, Index_Invalidate_I); 194 } 195 196 static inline void blast_icache32_r4600_v1_page_indexed(unsigned long page) 197 { 198 unsigned long flags; 199 200 local_irq_save(flags); 201 blast_icache32_page_indexed(page); 202 local_irq_restore(flags); 203 } 204 205 static inline void tx49_blast_icache32_page_indexed(unsigned long page) 206 { 207 unsigned long indexmask = current_cpu_data.icache.waysize - 1; 208 unsigned long start = INDEX_BASE + (page & indexmask); 209 unsigned long end = start + PAGE_SIZE; 210 unsigned long ws_inc = 1UL << current_cpu_data.icache.waybit; 211 unsigned long ws_end = current_cpu_data.icache.ways << 212 current_cpu_data.icache.waybit; 213 unsigned long ws, addr; 214 215 CACHE32_UNROLL32_ALIGN2; 216 /* I'm in even chunk. blast odd chunks */ 217 for (ws = 0; ws < ws_end; ws += ws_inc) 218 for (addr = start + 0x400; addr < end; addr += 0x400 * 2) 219 cache32_unroll32(addr|ws, Index_Invalidate_I); 220 CACHE32_UNROLL32_ALIGN; 221 /* I'm in odd chunk. blast even chunks */ 222 for (ws = 0; ws < ws_end; ws += ws_inc) 223 for (addr = start; addr < end; addr += 0x400 * 2) 224 cache32_unroll32(addr|ws, Index_Invalidate_I); 225 } 226 227 static void (* r4k_blast_icache_page)(unsigned long addr); 228 229 static void __cpuinit r4k_blast_icache_page_setup(void) 230 { 231 unsigned long ic_lsize = cpu_icache_line_size(); 232 233 if (ic_lsize == 0) 234 r4k_blast_icache_page = (void *)cache_noop; 235 else if (ic_lsize == 16) 236 r4k_blast_icache_page = blast_icache16_page; 237 else if (ic_lsize == 32) 238 r4k_blast_icache_page = blast_icache32_page; 239 else if (ic_lsize == 64) 240 r4k_blast_icache_page = blast_icache64_page; 241 } 242 243 244 static void (* r4k_blast_icache_page_indexed)(unsigned long addr); 245 246 static void __cpuinit r4k_blast_icache_page_indexed_setup(void) 247 { 248 unsigned long ic_lsize = cpu_icache_line_size(); 249 250 if (ic_lsize == 0) 251 r4k_blast_icache_page_indexed = (void *)cache_noop; 252 else if (ic_lsize == 16) 253 r4k_blast_icache_page_indexed = blast_icache16_page_indexed; 254 else if (ic_lsize == 32) { 255 if (R4600_V1_INDEX_ICACHEOP_WAR && cpu_is_r4600_v1_x()) 256 r4k_blast_icache_page_indexed = 257 blast_icache32_r4600_v1_page_indexed; 258 else if (TX49XX_ICACHE_INDEX_INV_WAR) 259 r4k_blast_icache_page_indexed = 260 tx49_blast_icache32_page_indexed; 261 else 262 r4k_blast_icache_page_indexed = 263 blast_icache32_page_indexed; 264 } else if (ic_lsize == 64) 265 r4k_blast_icache_page_indexed = blast_icache64_page_indexed; 266 } 267 268 static void (* r4k_blast_icache)(void); 269 270 static void __cpuinit r4k_blast_icache_setup(void) 271 { 272 unsigned long ic_lsize = cpu_icache_line_size(); 273 274 if (ic_lsize == 0) 275 r4k_blast_icache = (void *)cache_noop; 276 else if (ic_lsize == 16) 277 r4k_blast_icache = blast_icache16; 278 else if (ic_lsize == 32) { 279 if (R4600_V1_INDEX_ICACHEOP_WAR && cpu_is_r4600_v1_x()) 280 r4k_blast_icache = blast_r4600_v1_icache32; 281 else if (TX49XX_ICACHE_INDEX_INV_WAR) 282 r4k_blast_icache = tx49_blast_icache32; 283 else 284 r4k_blast_icache = blast_icache32; 285 } else if (ic_lsize == 64) 286 r4k_blast_icache = blast_icache64; 287 } 288 289 static void (* r4k_blast_scache_page)(unsigned long addr); 290 291 static void __cpuinit r4k_blast_scache_page_setup(void) 292 { 293 unsigned long sc_lsize = cpu_scache_line_size(); 294 295 if (scache_size == 0) 296 r4k_blast_scache_page = (void *)cache_noop; 297 else if (sc_lsize == 16) 298 r4k_blast_scache_page = blast_scache16_page; 299 else if (sc_lsize == 32) 300 r4k_blast_scache_page = blast_scache32_page; 301 else if (sc_lsize == 64) 302 r4k_blast_scache_page = blast_scache64_page; 303 else if (sc_lsize == 128) 304 r4k_blast_scache_page = blast_scache128_page; 305 } 306 307 static void (* r4k_blast_scache_page_indexed)(unsigned long addr); 308 309 static void __cpuinit r4k_blast_scache_page_indexed_setup(void) 310 { 311 unsigned long sc_lsize = cpu_scache_line_size(); 312 313 if (scache_size == 0) 314 r4k_blast_scache_page_indexed = (void *)cache_noop; 315 else if (sc_lsize == 16) 316 r4k_blast_scache_page_indexed = blast_scache16_page_indexed; 317 else if (sc_lsize == 32) 318 r4k_blast_scache_page_indexed = blast_scache32_page_indexed; 319 else if (sc_lsize == 64) 320 r4k_blast_scache_page_indexed = blast_scache64_page_indexed; 321 else if (sc_lsize == 128) 322 r4k_blast_scache_page_indexed = blast_scache128_page_indexed; 323 } 324 325 static void (* r4k_blast_scache)(void); 326 327 static void __cpuinit r4k_blast_scache_setup(void) 328 { 329 unsigned long sc_lsize = cpu_scache_line_size(); 330 331 if (scache_size == 0) 332 r4k_blast_scache = (void *)cache_noop; 333 else if (sc_lsize == 16) 334 r4k_blast_scache = blast_scache16; 335 else if (sc_lsize == 32) 336 r4k_blast_scache = blast_scache32; 337 else if (sc_lsize == 64) 338 r4k_blast_scache = blast_scache64; 339 else if (sc_lsize == 128) 340 r4k_blast_scache = blast_scache128; 341 } 342 343 static inline void local_r4k___flush_cache_all(void * args) 344 { 345 #if defined(CONFIG_CPU_LOONGSON2) 346 r4k_blast_scache(); 347 return; 348 #endif 349 r4k_blast_dcache(); 350 r4k_blast_icache(); 351 352 switch (current_cpu_type()) { 353 case CPU_R4000SC: 354 case CPU_R4000MC: 355 case CPU_R4400SC: 356 case CPU_R4400MC: 357 case CPU_R10000: 358 case CPU_R12000: 359 case CPU_R14000: 360 r4k_blast_scache(); 361 } 362 } 363 364 static void r4k___flush_cache_all(void) 365 { 366 r4k_on_each_cpu(local_r4k___flush_cache_all, NULL); 367 } 368 369 static inline int has_valid_asid(const struct mm_struct *mm) 370 { 371 #if defined(CONFIG_MIPS_MT_SMP) || defined(CONFIG_MIPS_MT_SMTC) 372 int i; 373 374 for_each_online_cpu(i) 375 if (cpu_context(i, mm)) 376 return 1; 377 378 return 0; 379 #else 380 return cpu_context(smp_processor_id(), mm); 381 #endif 382 } 383 384 static void r4k__flush_cache_vmap(void) 385 { 386 r4k_blast_dcache(); 387 } 388 389 static void r4k__flush_cache_vunmap(void) 390 { 391 r4k_blast_dcache(); 392 } 393 394 static inline void local_r4k_flush_cache_range(void * args) 395 { 396 struct vm_area_struct *vma = args; 397 int exec = vma->vm_flags & VM_EXEC; 398 399 if (!(has_valid_asid(vma->vm_mm))) 400 return; 401 402 r4k_blast_dcache(); 403 if (exec) 404 r4k_blast_icache(); 405 } 406 407 static void r4k_flush_cache_range(struct vm_area_struct *vma, 408 unsigned long start, unsigned long end) 409 { 410 int exec = vma->vm_flags & VM_EXEC; 411 412 if (cpu_has_dc_aliases || (exec && !cpu_has_ic_fills_f_dc)) 413 r4k_on_each_cpu(local_r4k_flush_cache_range, vma); 414 } 415 416 static inline void local_r4k_flush_cache_mm(void * args) 417 { 418 struct mm_struct *mm = args; 419 420 if (!has_valid_asid(mm)) 421 return; 422 423 /* 424 * Kludge alert. For obscure reasons R4000SC and R4400SC go nuts if we 425 * only flush the primary caches but R10000 and R12000 behave sane ... 426 * R4000SC and R4400SC indexed S-cache ops also invalidate primary 427 * caches, so we can bail out early. 428 */ 429 if (current_cpu_type() == CPU_R4000SC || 430 current_cpu_type() == CPU_R4000MC || 431 current_cpu_type() == CPU_R4400SC || 432 current_cpu_type() == CPU_R4400MC) { 433 r4k_blast_scache(); 434 return; 435 } 436 437 r4k_blast_dcache(); 438 } 439 440 static void r4k_flush_cache_mm(struct mm_struct *mm) 441 { 442 if (!cpu_has_dc_aliases) 443 return; 444 445 r4k_on_each_cpu(local_r4k_flush_cache_mm, mm); 446 } 447 448 struct flush_cache_page_args { 449 struct vm_area_struct *vma; 450 unsigned long addr; 451 unsigned long pfn; 452 }; 453 454 static inline void local_r4k_flush_cache_page(void *args) 455 { 456 struct flush_cache_page_args *fcp_args = args; 457 struct vm_area_struct *vma = fcp_args->vma; 458 unsigned long addr = fcp_args->addr; 459 struct page *page = pfn_to_page(fcp_args->pfn); 460 int exec = vma->vm_flags & VM_EXEC; 461 struct mm_struct *mm = vma->vm_mm; 462 int map_coherent = 0; 463 pgd_t *pgdp; 464 pud_t *pudp; 465 pmd_t *pmdp; 466 pte_t *ptep; 467 void *vaddr; 468 469 /* 470 * If ownes no valid ASID yet, cannot possibly have gotten 471 * this page into the cache. 472 */ 473 if (!has_valid_asid(mm)) 474 return; 475 476 addr &= PAGE_MASK; 477 pgdp = pgd_offset(mm, addr); 478 pudp = pud_offset(pgdp, addr); 479 pmdp = pmd_offset(pudp, addr); 480 ptep = pte_offset(pmdp, addr); 481 482 /* 483 * If the page isn't marked valid, the page cannot possibly be 484 * in the cache. 485 */ 486 if (!(pte_present(*ptep))) 487 return; 488 489 if ((mm == current->active_mm) && (pte_val(*ptep) & _PAGE_VALID)) 490 vaddr = NULL; 491 else { 492 /* 493 * Use kmap_coherent or kmap_atomic to do flushes for 494 * another ASID than the current one. 495 */ 496 map_coherent = (cpu_has_dc_aliases && 497 page_mapped(page) && !Page_dcache_dirty(page)); 498 if (map_coherent) 499 vaddr = kmap_coherent(page, addr); 500 else 501 vaddr = kmap_atomic(page, KM_USER0); 502 addr = (unsigned long)vaddr; 503 } 504 505 if (cpu_has_dc_aliases || (exec && !cpu_has_ic_fills_f_dc)) { 506 r4k_blast_dcache_page(addr); 507 if (exec && !cpu_icache_snoops_remote_store) 508 r4k_blast_scache_page(addr); 509 } 510 if (exec) { 511 if (vaddr && cpu_has_vtag_icache && mm == current->active_mm) { 512 int cpu = smp_processor_id(); 513 514 if (cpu_context(cpu, mm) != 0) 515 drop_mmu_context(mm, cpu); 516 } else 517 r4k_blast_icache_page(addr); 518 } 519 520 if (vaddr) { 521 if (map_coherent) 522 kunmap_coherent(); 523 else 524 kunmap_atomic(vaddr, KM_USER0); 525 } 526 } 527 528 static void r4k_flush_cache_page(struct vm_area_struct *vma, 529 unsigned long addr, unsigned long pfn) 530 { 531 struct flush_cache_page_args args; 532 533 args.vma = vma; 534 args.addr = addr; 535 args.pfn = pfn; 536 537 r4k_on_each_cpu(local_r4k_flush_cache_page, &args); 538 } 539 540 static inline void local_r4k_flush_data_cache_page(void * addr) 541 { 542 r4k_blast_dcache_page((unsigned long) addr); 543 } 544 545 static void r4k_flush_data_cache_page(unsigned long addr) 546 { 547 if (in_atomic()) 548 local_r4k_flush_data_cache_page((void *)addr); 549 else 550 r4k_on_each_cpu(local_r4k_flush_data_cache_page, (void *) addr); 551 } 552 553 struct flush_icache_range_args { 554 unsigned long start; 555 unsigned long end; 556 }; 557 558 static inline void local_r4k_flush_icache_range(unsigned long start, unsigned long end) 559 { 560 if (!cpu_has_ic_fills_f_dc) { 561 if (end - start >= dcache_size) { 562 r4k_blast_dcache(); 563 } else { 564 R4600_HIT_CACHEOP_WAR_IMPL; 565 protected_blast_dcache_range(start, end); 566 } 567 } 568 569 if (end - start > icache_size) 570 r4k_blast_icache(); 571 else 572 protected_blast_icache_range(start, end); 573 } 574 575 static inline void local_r4k_flush_icache_range_ipi(void *args) 576 { 577 struct flush_icache_range_args *fir_args = args; 578 unsigned long start = fir_args->start; 579 unsigned long end = fir_args->end; 580 581 local_r4k_flush_icache_range(start, end); 582 } 583 584 static void r4k_flush_icache_range(unsigned long start, unsigned long end) 585 { 586 struct flush_icache_range_args args; 587 588 args.start = start; 589 args.end = end; 590 591 r4k_on_each_cpu(local_r4k_flush_icache_range_ipi, &args); 592 instruction_hazard(); 593 } 594 595 #ifdef CONFIG_DMA_NONCOHERENT 596 597 static void r4k_dma_cache_wback_inv(unsigned long addr, unsigned long size) 598 { 599 /* Catch bad driver code */ 600 BUG_ON(size == 0); 601 602 if (cpu_has_inclusive_pcaches) { 603 if (size >= scache_size) 604 r4k_blast_scache(); 605 else 606 blast_scache_range(addr, addr + size); 607 __sync(); 608 return; 609 } 610 611 /* 612 * Either no secondary cache or the available caches don't have the 613 * subset property so we have to flush the primary caches 614 * explicitly 615 */ 616 if (cpu_has_safe_index_cacheops && size >= dcache_size) { 617 r4k_blast_dcache(); 618 } else { 619 R4600_HIT_CACHEOP_WAR_IMPL; 620 blast_dcache_range(addr, addr + size); 621 } 622 623 bc_wback_inv(addr, size); 624 __sync(); 625 } 626 627 static void r4k_dma_cache_inv(unsigned long addr, unsigned long size) 628 { 629 /* Catch bad driver code */ 630 BUG_ON(size == 0); 631 632 if (cpu_has_inclusive_pcaches) { 633 if (size >= scache_size) 634 r4k_blast_scache(); 635 else { 636 unsigned long lsize = cpu_scache_line_size(); 637 unsigned long almask = ~(lsize - 1); 638 639 /* 640 * There is no clearly documented alignment requirement 641 * for the cache instruction on MIPS processors and 642 * some processors, among them the RM5200 and RM7000 643 * QED processors will throw an address error for cache 644 * hit ops with insufficient alignment. Solved by 645 * aligning the address to cache line size. 646 */ 647 cache_op(Hit_Writeback_Inv_SD, addr & almask); 648 cache_op(Hit_Writeback_Inv_SD, 649 (addr + size - 1) & almask); 650 blast_inv_scache_range(addr, addr + size); 651 } 652 __sync(); 653 return; 654 } 655 656 if (cpu_has_safe_index_cacheops && size >= dcache_size) { 657 r4k_blast_dcache(); 658 } else { 659 unsigned long lsize = cpu_dcache_line_size(); 660 unsigned long almask = ~(lsize - 1); 661 662 R4600_HIT_CACHEOP_WAR_IMPL; 663 cache_op(Hit_Writeback_Inv_D, addr & almask); 664 cache_op(Hit_Writeback_Inv_D, (addr + size - 1) & almask); 665 blast_inv_dcache_range(addr, addr + size); 666 } 667 668 bc_inv(addr, size); 669 __sync(); 670 } 671 #endif /* CONFIG_DMA_NONCOHERENT */ 672 673 /* 674 * While we're protected against bad userland addresses we don't care 675 * very much about what happens in that case. Usually a segmentation 676 * fault will dump the process later on anyway ... 677 */ 678 static void local_r4k_flush_cache_sigtramp(void * arg) 679 { 680 unsigned long ic_lsize = cpu_icache_line_size(); 681 unsigned long dc_lsize = cpu_dcache_line_size(); 682 unsigned long sc_lsize = cpu_scache_line_size(); 683 unsigned long addr = (unsigned long) arg; 684 685 R4600_HIT_CACHEOP_WAR_IMPL; 686 if (dc_lsize) 687 protected_writeback_dcache_line(addr & ~(dc_lsize - 1)); 688 if (!cpu_icache_snoops_remote_store && scache_size) 689 protected_writeback_scache_line(addr & ~(sc_lsize - 1)); 690 if (ic_lsize) 691 protected_flush_icache_line(addr & ~(ic_lsize - 1)); 692 if (MIPS4K_ICACHE_REFILL_WAR) { 693 __asm__ __volatile__ ( 694 ".set push\n\t" 695 ".set noat\n\t" 696 ".set mips3\n\t" 697 #ifdef CONFIG_32BIT 698 "la $at,1f\n\t" 699 #endif 700 #ifdef CONFIG_64BIT 701 "dla $at,1f\n\t" 702 #endif 703 "cache %0,($at)\n\t" 704 "nop; nop; nop\n" 705 "1:\n\t" 706 ".set pop" 707 : 708 : "i" (Hit_Invalidate_I)); 709 } 710 if (MIPS_CACHE_SYNC_WAR) 711 __asm__ __volatile__ ("sync"); 712 } 713 714 static void r4k_flush_cache_sigtramp(unsigned long addr) 715 { 716 r4k_on_each_cpu(local_r4k_flush_cache_sigtramp, (void *) addr); 717 } 718 719 static void r4k_flush_icache_all(void) 720 { 721 if (cpu_has_vtag_icache) 722 r4k_blast_icache(); 723 } 724 725 struct flush_kernel_vmap_range_args { 726 unsigned long vaddr; 727 int size; 728 }; 729 730 static inline void local_r4k_flush_kernel_vmap_range(void *args) 731 { 732 struct flush_kernel_vmap_range_args *vmra = args; 733 unsigned long vaddr = vmra->vaddr; 734 int size = vmra->size; 735 736 /* 737 * Aliases only affect the primary caches so don't bother with 738 * S-caches or T-caches. 739 */ 740 if (cpu_has_safe_index_cacheops && size >= dcache_size) 741 r4k_blast_dcache(); 742 else { 743 R4600_HIT_CACHEOP_WAR_IMPL; 744 blast_dcache_range(vaddr, vaddr + size); 745 } 746 } 747 748 static void r4k_flush_kernel_vmap_range(unsigned long vaddr, int size) 749 { 750 struct flush_kernel_vmap_range_args args; 751 752 args.vaddr = (unsigned long) vaddr; 753 args.size = size; 754 755 r4k_on_each_cpu(local_r4k_flush_kernel_vmap_range, &args); 756 } 757 758 static inline void rm7k_erratum31(void) 759 { 760 const unsigned long ic_lsize = 32; 761 unsigned long addr; 762 763 /* RM7000 erratum #31. The icache is screwed at startup. */ 764 write_c0_taglo(0); 765 write_c0_taghi(0); 766 767 for (addr = INDEX_BASE; addr <= INDEX_BASE + 4096; addr += ic_lsize) { 768 __asm__ __volatile__ ( 769 ".set push\n\t" 770 ".set noreorder\n\t" 771 ".set mips3\n\t" 772 "cache\t%1, 0(%0)\n\t" 773 "cache\t%1, 0x1000(%0)\n\t" 774 "cache\t%1, 0x2000(%0)\n\t" 775 "cache\t%1, 0x3000(%0)\n\t" 776 "cache\t%2, 0(%0)\n\t" 777 "cache\t%2, 0x1000(%0)\n\t" 778 "cache\t%2, 0x2000(%0)\n\t" 779 "cache\t%2, 0x3000(%0)\n\t" 780 "cache\t%1, 0(%0)\n\t" 781 "cache\t%1, 0x1000(%0)\n\t" 782 "cache\t%1, 0x2000(%0)\n\t" 783 "cache\t%1, 0x3000(%0)\n\t" 784 ".set pop\n" 785 : 786 : "r" (addr), "i" (Index_Store_Tag_I), "i" (Fill)); 787 } 788 } 789 790 static char *way_string[] __cpuinitdata = { NULL, "direct mapped", "2-way", 791 "3-way", "4-way", "5-way", "6-way", "7-way", "8-way" 792 }; 793 794 static void __cpuinit probe_pcache(void) 795 { 796 struct cpuinfo_mips *c = ¤t_cpu_data; 797 unsigned int config = read_c0_config(); 798 unsigned int prid = read_c0_prid(); 799 unsigned long config1; 800 unsigned int lsize; 801 802 switch (c->cputype) { 803 case CPU_R4600: /* QED style two way caches? */ 804 case CPU_R4700: 805 case CPU_R5000: 806 case CPU_NEVADA: 807 icache_size = 1 << (12 + ((config & CONF_IC) >> 9)); 808 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 809 c->icache.ways = 2; 810 c->icache.waybit = __ffs(icache_size/2); 811 812 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6)); 813 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 814 c->dcache.ways = 2; 815 c->dcache.waybit= __ffs(dcache_size/2); 816 817 c->options |= MIPS_CPU_CACHE_CDEX_P; 818 break; 819 820 case CPU_R5432: 821 case CPU_R5500: 822 icache_size = 1 << (12 + ((config & CONF_IC) >> 9)); 823 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 824 c->icache.ways = 2; 825 c->icache.waybit= 0; 826 827 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6)); 828 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 829 c->dcache.ways = 2; 830 c->dcache.waybit = 0; 831 832 c->options |= MIPS_CPU_CACHE_CDEX_P | MIPS_CPU_PREFETCH; 833 break; 834 835 case CPU_TX49XX: 836 icache_size = 1 << (12 + ((config & CONF_IC) >> 9)); 837 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 838 c->icache.ways = 4; 839 c->icache.waybit= 0; 840 841 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6)); 842 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 843 c->dcache.ways = 4; 844 c->dcache.waybit = 0; 845 846 c->options |= MIPS_CPU_CACHE_CDEX_P; 847 c->options |= MIPS_CPU_PREFETCH; 848 break; 849 850 case CPU_R4000PC: 851 case CPU_R4000SC: 852 case CPU_R4000MC: 853 case CPU_R4400PC: 854 case CPU_R4400SC: 855 case CPU_R4400MC: 856 case CPU_R4300: 857 icache_size = 1 << (12 + ((config & CONF_IC) >> 9)); 858 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 859 c->icache.ways = 1; 860 c->icache.waybit = 0; /* doesn't matter */ 861 862 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6)); 863 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 864 c->dcache.ways = 1; 865 c->dcache.waybit = 0; /* does not matter */ 866 867 c->options |= MIPS_CPU_CACHE_CDEX_P; 868 break; 869 870 case CPU_R10000: 871 case CPU_R12000: 872 case CPU_R14000: 873 icache_size = 1 << (12 + ((config & R10K_CONF_IC) >> 29)); 874 c->icache.linesz = 64; 875 c->icache.ways = 2; 876 c->icache.waybit = 0; 877 878 dcache_size = 1 << (12 + ((config & R10K_CONF_DC) >> 26)); 879 c->dcache.linesz = 32; 880 c->dcache.ways = 2; 881 c->dcache.waybit = 0; 882 883 c->options |= MIPS_CPU_PREFETCH; 884 break; 885 886 case CPU_VR4133: 887 write_c0_config(config & ~VR41_CONF_P4K); 888 case CPU_VR4131: 889 /* Workaround for cache instruction bug of VR4131 */ 890 if (c->processor_id == 0x0c80U || c->processor_id == 0x0c81U || 891 c->processor_id == 0x0c82U) { 892 config |= 0x00400000U; 893 if (c->processor_id == 0x0c80U) 894 config |= VR41_CONF_BP; 895 write_c0_config(config); 896 } else 897 c->options |= MIPS_CPU_CACHE_CDEX_P; 898 899 icache_size = 1 << (10 + ((config & CONF_IC) >> 9)); 900 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 901 c->icache.ways = 2; 902 c->icache.waybit = __ffs(icache_size/2); 903 904 dcache_size = 1 << (10 + ((config & CONF_DC) >> 6)); 905 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 906 c->dcache.ways = 2; 907 c->dcache.waybit = __ffs(dcache_size/2); 908 break; 909 910 case CPU_VR41XX: 911 case CPU_VR4111: 912 case CPU_VR4121: 913 case CPU_VR4122: 914 case CPU_VR4181: 915 case CPU_VR4181A: 916 icache_size = 1 << (10 + ((config & CONF_IC) >> 9)); 917 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 918 c->icache.ways = 1; 919 c->icache.waybit = 0; /* doesn't matter */ 920 921 dcache_size = 1 << (10 + ((config & CONF_DC) >> 6)); 922 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 923 c->dcache.ways = 1; 924 c->dcache.waybit = 0; /* does not matter */ 925 926 c->options |= MIPS_CPU_CACHE_CDEX_P; 927 break; 928 929 case CPU_RM7000: 930 rm7k_erratum31(); 931 932 case CPU_RM9000: 933 icache_size = 1 << (12 + ((config & CONF_IC) >> 9)); 934 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 935 c->icache.ways = 4; 936 c->icache.waybit = __ffs(icache_size / c->icache.ways); 937 938 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6)); 939 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 940 c->dcache.ways = 4; 941 c->dcache.waybit = __ffs(dcache_size / c->dcache.ways); 942 943 #if !defined(CONFIG_SMP) || !defined(RM9000_CDEX_SMP_WAR) 944 c->options |= MIPS_CPU_CACHE_CDEX_P; 945 #endif 946 c->options |= MIPS_CPU_PREFETCH; 947 break; 948 949 case CPU_LOONGSON2: 950 icache_size = 1 << (12 + ((config & CONF_IC) >> 9)); 951 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 952 if (prid & 0x3) 953 c->icache.ways = 4; 954 else 955 c->icache.ways = 2; 956 c->icache.waybit = 0; 957 958 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6)); 959 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 960 if (prid & 0x3) 961 c->dcache.ways = 4; 962 else 963 c->dcache.ways = 2; 964 c->dcache.waybit = 0; 965 break; 966 967 default: 968 if (!(config & MIPS_CONF_M)) 969 panic("Don't know how to probe P-caches on this cpu."); 970 971 /* 972 * So we seem to be a MIPS32 or MIPS64 CPU 973 * So let's probe the I-cache ... 974 */ 975 config1 = read_c0_config1(); 976 977 if ((lsize = ((config1 >> 19) & 7))) 978 c->icache.linesz = 2 << lsize; 979 else 980 c->icache.linesz = lsize; 981 c->icache.sets = 64 << ((config1 >> 22) & 7); 982 c->icache.ways = 1 + ((config1 >> 16) & 7); 983 984 icache_size = c->icache.sets * 985 c->icache.ways * 986 c->icache.linesz; 987 c->icache.waybit = __ffs(icache_size/c->icache.ways); 988 989 if (config & 0x8) /* VI bit */ 990 c->icache.flags |= MIPS_CACHE_VTAG; 991 992 /* 993 * Now probe the MIPS32 / MIPS64 data cache. 994 */ 995 c->dcache.flags = 0; 996 997 if ((lsize = ((config1 >> 10) & 7))) 998 c->dcache.linesz = 2 << lsize; 999 else 1000 c->dcache.linesz= lsize; 1001 c->dcache.sets = 64 << ((config1 >> 13) & 7); 1002 c->dcache.ways = 1 + ((config1 >> 7) & 7); 1003 1004 dcache_size = c->dcache.sets * 1005 c->dcache.ways * 1006 c->dcache.linesz; 1007 c->dcache.waybit = __ffs(dcache_size/c->dcache.ways); 1008 1009 c->options |= MIPS_CPU_PREFETCH; 1010 break; 1011 } 1012 1013 /* 1014 * Processor configuration sanity check for the R4000SC erratum 1015 * #5. With page sizes larger than 32kB there is no possibility 1016 * to get a VCE exception anymore so we don't care about this 1017 * misconfiguration. The case is rather theoretical anyway; 1018 * presumably no vendor is shipping his hardware in the "bad" 1019 * configuration. 1020 */ 1021 if ((prid & 0xff00) == PRID_IMP_R4000 && (prid & 0xff) < 0x40 && 1022 !(config & CONF_SC) && c->icache.linesz != 16 && 1023 PAGE_SIZE <= 0x8000) 1024 panic("Improper R4000SC processor configuration detected"); 1025 1026 /* compute a couple of other cache variables */ 1027 c->icache.waysize = icache_size / c->icache.ways; 1028 c->dcache.waysize = dcache_size / c->dcache.ways; 1029 1030 c->icache.sets = c->icache.linesz ? 1031 icache_size / (c->icache.linesz * c->icache.ways) : 0; 1032 c->dcache.sets = c->dcache.linesz ? 1033 dcache_size / (c->dcache.linesz * c->dcache.ways) : 0; 1034 1035 /* 1036 * R10000 and R12000 P-caches are odd in a positive way. They're 32kB 1037 * 2-way virtually indexed so normally would suffer from aliases. So 1038 * normally they'd suffer from aliases but magic in the hardware deals 1039 * with that for us so we don't need to take care ourselves. 1040 */ 1041 switch (c->cputype) { 1042 case CPU_20KC: 1043 case CPU_25KF: 1044 case CPU_SB1: 1045 case CPU_SB1A: 1046 case CPU_XLR: 1047 c->dcache.flags |= MIPS_CACHE_PINDEX; 1048 break; 1049 1050 case CPU_R10000: 1051 case CPU_R12000: 1052 case CPU_R14000: 1053 break; 1054 1055 case CPU_24K: 1056 case CPU_34K: 1057 case CPU_74K: 1058 case CPU_1004K: 1059 if ((read_c0_config7() & (1 << 16))) { 1060 /* effectively physically indexed dcache, 1061 thus no virtual aliases. */ 1062 c->dcache.flags |= MIPS_CACHE_PINDEX; 1063 break; 1064 } 1065 default: 1066 if (c->dcache.waysize > PAGE_SIZE) 1067 c->dcache.flags |= MIPS_CACHE_ALIASES; 1068 } 1069 1070 switch (c->cputype) { 1071 case CPU_20KC: 1072 /* 1073 * Some older 20Kc chips doesn't have the 'VI' bit in 1074 * the config register. 1075 */ 1076 c->icache.flags |= MIPS_CACHE_VTAG; 1077 break; 1078 1079 case CPU_ALCHEMY: 1080 c->icache.flags |= MIPS_CACHE_IC_F_DC; 1081 break; 1082 } 1083 1084 #ifdef CONFIG_CPU_LOONGSON2 1085 /* 1086 * LOONGSON2 has 4 way icache, but when using indexed cache op, 1087 * one op will act on all 4 ways 1088 */ 1089 c->icache.ways = 1; 1090 #endif 1091 1092 printk("Primary instruction cache %ldkB, %s, %s, linesize %d bytes.\n", 1093 icache_size >> 10, 1094 c->icache.flags & MIPS_CACHE_VTAG ? "VIVT" : "VIPT", 1095 way_string[c->icache.ways], c->icache.linesz); 1096 1097 printk("Primary data cache %ldkB, %s, %s, %s, linesize %d bytes\n", 1098 dcache_size >> 10, way_string[c->dcache.ways], 1099 (c->dcache.flags & MIPS_CACHE_PINDEX) ? "PIPT" : "VIPT", 1100 (c->dcache.flags & MIPS_CACHE_ALIASES) ? 1101 "cache aliases" : "no aliases", 1102 c->dcache.linesz); 1103 } 1104 1105 /* 1106 * If you even _breathe_ on this function, look at the gcc output and make sure 1107 * it does not pop things on and off the stack for the cache sizing loop that 1108 * executes in KSEG1 space or else you will crash and burn badly. You have 1109 * been warned. 1110 */ 1111 static int __cpuinit probe_scache(void) 1112 { 1113 unsigned long flags, addr, begin, end, pow2; 1114 unsigned int config = read_c0_config(); 1115 struct cpuinfo_mips *c = ¤t_cpu_data; 1116 1117 if (config & CONF_SC) 1118 return 0; 1119 1120 begin = (unsigned long) &_stext; 1121 begin &= ~((4 * 1024 * 1024) - 1); 1122 end = begin + (4 * 1024 * 1024); 1123 1124 /* 1125 * This is such a bitch, you'd think they would make it easy to do 1126 * this. Away you daemons of stupidity! 1127 */ 1128 local_irq_save(flags); 1129 1130 /* Fill each size-multiple cache line with a valid tag. */ 1131 pow2 = (64 * 1024); 1132 for (addr = begin; addr < end; addr = (begin + pow2)) { 1133 unsigned long *p = (unsigned long *) addr; 1134 __asm__ __volatile__("nop" : : "r" (*p)); /* whee... */ 1135 pow2 <<= 1; 1136 } 1137 1138 /* Load first line with zero (therefore invalid) tag. */ 1139 write_c0_taglo(0); 1140 write_c0_taghi(0); 1141 __asm__ __volatile__("nop; nop; nop; nop;"); /* avoid the hazard */ 1142 cache_op(Index_Store_Tag_I, begin); 1143 cache_op(Index_Store_Tag_D, begin); 1144 cache_op(Index_Store_Tag_SD, begin); 1145 1146 /* Now search for the wrap around point. */ 1147 pow2 = (128 * 1024); 1148 for (addr = begin + (128 * 1024); addr < end; addr = begin + pow2) { 1149 cache_op(Index_Load_Tag_SD, addr); 1150 __asm__ __volatile__("nop; nop; nop; nop;"); /* hazard... */ 1151 if (!read_c0_taglo()) 1152 break; 1153 pow2 <<= 1; 1154 } 1155 local_irq_restore(flags); 1156 addr -= begin; 1157 1158 scache_size = addr; 1159 c->scache.linesz = 16 << ((config & R4K_CONF_SB) >> 22); 1160 c->scache.ways = 1; 1161 c->dcache.waybit = 0; /* does not matter */ 1162 1163 return 1; 1164 } 1165 1166 #if defined(CONFIG_CPU_LOONGSON2) 1167 static void __init loongson2_sc_init(void) 1168 { 1169 struct cpuinfo_mips *c = ¤t_cpu_data; 1170 1171 scache_size = 512*1024; 1172 c->scache.linesz = 32; 1173 c->scache.ways = 4; 1174 c->scache.waybit = 0; 1175 c->scache.waysize = scache_size / (c->scache.ways); 1176 c->scache.sets = scache_size / (c->scache.linesz * c->scache.ways); 1177 pr_info("Unified secondary cache %ldkB %s, linesize %d bytes.\n", 1178 scache_size >> 10, way_string[c->scache.ways], c->scache.linesz); 1179 1180 c->options |= MIPS_CPU_INCLUSIVE_CACHES; 1181 } 1182 #endif 1183 1184 extern int r5k_sc_init(void); 1185 extern int rm7k_sc_init(void); 1186 extern int mips_sc_init(void); 1187 1188 static void __cpuinit setup_scache(void) 1189 { 1190 struct cpuinfo_mips *c = ¤t_cpu_data; 1191 unsigned int config = read_c0_config(); 1192 int sc_present = 0; 1193 1194 /* 1195 * Do the probing thing on R4000SC and R4400SC processors. Other 1196 * processors don't have a S-cache that would be relevant to the 1197 * Linux memory management. 1198 */ 1199 switch (c->cputype) { 1200 case CPU_R4000SC: 1201 case CPU_R4000MC: 1202 case CPU_R4400SC: 1203 case CPU_R4400MC: 1204 sc_present = run_uncached(probe_scache); 1205 if (sc_present) 1206 c->options |= MIPS_CPU_CACHE_CDEX_S; 1207 break; 1208 1209 case CPU_R10000: 1210 case CPU_R12000: 1211 case CPU_R14000: 1212 scache_size = 0x80000 << ((config & R10K_CONF_SS) >> 16); 1213 c->scache.linesz = 64 << ((config >> 13) & 1); 1214 c->scache.ways = 2; 1215 c->scache.waybit= 0; 1216 sc_present = 1; 1217 break; 1218 1219 case CPU_R5000: 1220 case CPU_NEVADA: 1221 #ifdef CONFIG_R5000_CPU_SCACHE 1222 r5k_sc_init(); 1223 #endif 1224 return; 1225 1226 case CPU_RM7000: 1227 case CPU_RM9000: 1228 #ifdef CONFIG_RM7000_CPU_SCACHE 1229 rm7k_sc_init(); 1230 #endif 1231 return; 1232 1233 #if defined(CONFIG_CPU_LOONGSON2) 1234 case CPU_LOONGSON2: 1235 loongson2_sc_init(); 1236 return; 1237 #endif 1238 1239 default: 1240 if (c->isa_level == MIPS_CPU_ISA_M32R1 || 1241 c->isa_level == MIPS_CPU_ISA_M32R2 || 1242 c->isa_level == MIPS_CPU_ISA_M64R1 || 1243 c->isa_level == MIPS_CPU_ISA_M64R2) { 1244 #ifdef CONFIG_MIPS_CPU_SCACHE 1245 if (mips_sc_init ()) { 1246 scache_size = c->scache.ways * c->scache.sets * c->scache.linesz; 1247 printk("MIPS secondary cache %ldkB, %s, linesize %d bytes.\n", 1248 scache_size >> 10, 1249 way_string[c->scache.ways], c->scache.linesz); 1250 } 1251 #else 1252 if (!(c->scache.flags & MIPS_CACHE_NOT_PRESENT)) 1253 panic("Dunno how to handle MIPS32 / MIPS64 second level cache"); 1254 #endif 1255 return; 1256 } 1257 sc_present = 0; 1258 } 1259 1260 if (!sc_present) 1261 return; 1262 1263 /* compute a couple of other cache variables */ 1264 c->scache.waysize = scache_size / c->scache.ways; 1265 1266 c->scache.sets = scache_size / (c->scache.linesz * c->scache.ways); 1267 1268 printk("Unified secondary cache %ldkB %s, linesize %d bytes.\n", 1269 scache_size >> 10, way_string[c->scache.ways], c->scache.linesz); 1270 1271 c->options |= MIPS_CPU_INCLUSIVE_CACHES; 1272 } 1273 1274 void au1x00_fixup_config_od(void) 1275 { 1276 /* 1277 * c0_config.od (bit 19) was write only (and read as 0) 1278 * on the early revisions of Alchemy SOCs. It disables the bus 1279 * transaction overlapping and needs to be set to fix various errata. 1280 */ 1281 switch (read_c0_prid()) { 1282 case 0x00030100: /* Au1000 DA */ 1283 case 0x00030201: /* Au1000 HA */ 1284 case 0x00030202: /* Au1000 HB */ 1285 case 0x01030200: /* Au1500 AB */ 1286 /* 1287 * Au1100 errata actually keeps silence about this bit, so we set it 1288 * just in case for those revisions that require it to be set according 1289 * to the (now gone) cpu table. 1290 */ 1291 case 0x02030200: /* Au1100 AB */ 1292 case 0x02030201: /* Au1100 BA */ 1293 case 0x02030202: /* Au1100 BC */ 1294 set_c0_config(1 << 19); 1295 break; 1296 } 1297 } 1298 1299 /* CP0 hazard avoidance. */ 1300 #define NXP_BARRIER() \ 1301 __asm__ __volatile__( \ 1302 ".set noreorder\n\t" \ 1303 "nop; nop; nop; nop; nop; nop;\n\t" \ 1304 ".set reorder\n\t") 1305 1306 static void nxp_pr4450_fixup_config(void) 1307 { 1308 unsigned long config0; 1309 1310 config0 = read_c0_config(); 1311 1312 /* clear all three cache coherency fields */ 1313 config0 &= ~(0x7 | (7 << 25) | (7 << 28)); 1314 config0 |= (((_page_cachable_default >> _CACHE_SHIFT) << 0) | 1315 ((_page_cachable_default >> _CACHE_SHIFT) << 25) | 1316 ((_page_cachable_default >> _CACHE_SHIFT) << 28)); 1317 write_c0_config(config0); 1318 NXP_BARRIER(); 1319 } 1320 1321 static int __cpuinitdata cca = -1; 1322 1323 static int __init cca_setup(char *str) 1324 { 1325 get_option(&str, &cca); 1326 1327 return 1; 1328 } 1329 1330 __setup("cca=", cca_setup); 1331 1332 static void __cpuinit coherency_setup(void) 1333 { 1334 if (cca < 0 || cca > 7) 1335 cca = read_c0_config() & CONF_CM_CMASK; 1336 _page_cachable_default = cca << _CACHE_SHIFT; 1337 1338 pr_debug("Using cache attribute %d\n", cca); 1339 change_c0_config(CONF_CM_CMASK, cca); 1340 1341 /* 1342 * c0_status.cu=0 specifies that updates by the sc instruction use 1343 * the coherency mode specified by the TLB; 1 means cachable 1344 * coherent update on write will be used. Not all processors have 1345 * this bit and; some wire it to zero, others like Toshiba had the 1346 * silly idea of putting something else there ... 1347 */ 1348 switch (current_cpu_type()) { 1349 case CPU_R4000PC: 1350 case CPU_R4000SC: 1351 case CPU_R4000MC: 1352 case CPU_R4400PC: 1353 case CPU_R4400SC: 1354 case CPU_R4400MC: 1355 clear_c0_config(CONF_CU); 1356 break; 1357 /* 1358 * We need to catch the early Alchemy SOCs with 1359 * the write-only co_config.od bit and set it back to one on: 1360 * Au1000 rev DA, HA, HB; Au1100 AB, BA, BC, Au1500 AB 1361 */ 1362 case CPU_ALCHEMY: 1363 au1x00_fixup_config_od(); 1364 break; 1365 1366 case PRID_IMP_PR4450: 1367 nxp_pr4450_fixup_config(); 1368 break; 1369 } 1370 } 1371 1372 #if defined(CONFIG_DMA_NONCOHERENT) 1373 1374 static int __cpuinitdata coherentio; 1375 1376 static int __init setcoherentio(char *str) 1377 { 1378 coherentio = 1; 1379 1380 return 1; 1381 } 1382 1383 __setup("coherentio", setcoherentio); 1384 #endif 1385 1386 void __cpuinit r4k_cache_init(void) 1387 { 1388 extern void build_clear_page(void); 1389 extern void build_copy_page(void); 1390 extern char __weak except_vec2_generic; 1391 extern char __weak except_vec2_sb1; 1392 struct cpuinfo_mips *c = ¤t_cpu_data; 1393 1394 switch (c->cputype) { 1395 case CPU_SB1: 1396 case CPU_SB1A: 1397 set_uncached_handler(0x100, &except_vec2_sb1, 0x80); 1398 break; 1399 1400 default: 1401 set_uncached_handler(0x100, &except_vec2_generic, 0x80); 1402 break; 1403 } 1404 1405 probe_pcache(); 1406 setup_scache(); 1407 1408 r4k_blast_dcache_page_setup(); 1409 r4k_blast_dcache_page_indexed_setup(); 1410 r4k_blast_dcache_setup(); 1411 r4k_blast_icache_page_setup(); 1412 r4k_blast_icache_page_indexed_setup(); 1413 r4k_blast_icache_setup(); 1414 r4k_blast_scache_page_setup(); 1415 r4k_blast_scache_page_indexed_setup(); 1416 r4k_blast_scache_setup(); 1417 1418 /* 1419 * Some MIPS32 and MIPS64 processors have physically indexed caches. 1420 * This code supports virtually indexed processors and will be 1421 * unnecessarily inefficient on physically indexed processors. 1422 */ 1423 if (c->dcache.linesz) 1424 shm_align_mask = max_t( unsigned long, 1425 c->dcache.sets * c->dcache.linesz - 1, 1426 PAGE_SIZE - 1); 1427 else 1428 shm_align_mask = PAGE_SIZE-1; 1429 1430 __flush_cache_vmap = r4k__flush_cache_vmap; 1431 __flush_cache_vunmap = r4k__flush_cache_vunmap; 1432 1433 flush_cache_all = cache_noop; 1434 __flush_cache_all = r4k___flush_cache_all; 1435 flush_cache_mm = r4k_flush_cache_mm; 1436 flush_cache_page = r4k_flush_cache_page; 1437 flush_cache_range = r4k_flush_cache_range; 1438 1439 __flush_kernel_vmap_range = r4k_flush_kernel_vmap_range; 1440 1441 flush_cache_sigtramp = r4k_flush_cache_sigtramp; 1442 flush_icache_all = r4k_flush_icache_all; 1443 local_flush_data_cache_page = local_r4k_flush_data_cache_page; 1444 flush_data_cache_page = r4k_flush_data_cache_page; 1445 flush_icache_range = r4k_flush_icache_range; 1446 local_flush_icache_range = local_r4k_flush_icache_range; 1447 1448 #if defined(CONFIG_DMA_NONCOHERENT) 1449 if (coherentio) { 1450 _dma_cache_wback_inv = (void *)cache_noop; 1451 _dma_cache_wback = (void *)cache_noop; 1452 _dma_cache_inv = (void *)cache_noop; 1453 } else { 1454 _dma_cache_wback_inv = r4k_dma_cache_wback_inv; 1455 _dma_cache_wback = r4k_dma_cache_wback_inv; 1456 _dma_cache_inv = r4k_dma_cache_inv; 1457 } 1458 #endif 1459 1460 build_clear_page(); 1461 build_copy_page(); 1462 #if !defined(CONFIG_MIPS_CMP) 1463 local_r4k___flush_cache_all(NULL); 1464 #endif 1465 coherency_setup(); 1466 } 1467