1 /* 2 * This file is subject to the terms and conditions of the GNU General Public 3 * License. See the file "COPYING" in the main directory of this archive 4 * for more details. 5 * 6 * Copyright (C) 1996 David S. Miller (dm@engr.sgi.com) 7 * Copyright (C) 1997, 1998, 1999, 2000, 2001, 2002 Ralf Baechle (ralf@gnu.org) 8 * Copyright (C) 1999, 2000 Silicon Graphics, Inc. 9 */ 10 #include <linux/hardirq.h> 11 #include <linux/init.h> 12 #include <linux/highmem.h> 13 #include <linux/kernel.h> 14 #include <linux/linkage.h> 15 #include <linux/sched.h> 16 #include <linux/smp.h> 17 #include <linux/mm.h> 18 #include <linux/module.h> 19 #include <linux/bitops.h> 20 21 #include <asm/bcache.h> 22 #include <asm/bootinfo.h> 23 #include <asm/cache.h> 24 #include <asm/cacheops.h> 25 #include <asm/cpu.h> 26 #include <asm/cpu-features.h> 27 #include <asm/io.h> 28 #include <asm/page.h> 29 #include <asm/pgtable.h> 30 #include <asm/r4kcache.h> 31 #include <asm/sections.h> 32 #include <asm/system.h> 33 #include <asm/mmu_context.h> 34 #include <asm/war.h> 35 #include <asm/cacheflush.h> /* for run_uncached() */ 36 37 38 /* 39 * Special Variant of smp_call_function for use by cache functions: 40 * 41 * o No return value 42 * o collapses to normal function call on UP kernels 43 * o collapses to normal function call on systems with a single shared 44 * primary cache. 45 */ 46 static inline void r4k_on_each_cpu(void (*func) (void *info), void *info, 47 int wait) 48 { 49 preempt_disable(); 50 51 #if !defined(CONFIG_MIPS_MT_SMP) && !defined(CONFIG_MIPS_MT_SMTC) 52 smp_call_function(func, info, wait); 53 #endif 54 func(info); 55 preempt_enable(); 56 } 57 58 #if defined(CONFIG_MIPS_CMP) 59 #define cpu_has_safe_index_cacheops 0 60 #else 61 #define cpu_has_safe_index_cacheops 1 62 #endif 63 64 /* 65 * Must die. 66 */ 67 static unsigned long icache_size __read_mostly; 68 static unsigned long dcache_size __read_mostly; 69 static unsigned long scache_size __read_mostly; 70 71 /* 72 * Dummy cache handling routines for machines without boardcaches 73 */ 74 static void cache_noop(void) {} 75 76 static struct bcache_ops no_sc_ops = { 77 .bc_enable = (void *)cache_noop, 78 .bc_disable = (void *)cache_noop, 79 .bc_wback_inv = (void *)cache_noop, 80 .bc_inv = (void *)cache_noop 81 }; 82 83 struct bcache_ops *bcops = &no_sc_ops; 84 85 #define cpu_is_r4600_v1_x() ((read_c0_prid() & 0xfffffff0) == 0x00002010) 86 #define cpu_is_r4600_v2_x() ((read_c0_prid() & 0xfffffff0) == 0x00002020) 87 88 #define R4600_HIT_CACHEOP_WAR_IMPL \ 89 do { \ 90 if (R4600_V2_HIT_CACHEOP_WAR && cpu_is_r4600_v2_x()) \ 91 *(volatile unsigned long *)CKSEG1; \ 92 if (R4600_V1_HIT_CACHEOP_WAR) \ 93 __asm__ __volatile__("nop;nop;nop;nop"); \ 94 } while (0) 95 96 static void (*r4k_blast_dcache_page)(unsigned long addr); 97 98 static inline void r4k_blast_dcache_page_dc32(unsigned long addr) 99 { 100 R4600_HIT_CACHEOP_WAR_IMPL; 101 blast_dcache32_page(addr); 102 } 103 104 static inline void r4k_blast_dcache_page_dc64(unsigned long addr) 105 { 106 R4600_HIT_CACHEOP_WAR_IMPL; 107 blast_dcache64_page(addr); 108 } 109 110 static void __cpuinit r4k_blast_dcache_page_setup(void) 111 { 112 unsigned long dc_lsize = cpu_dcache_line_size(); 113 114 if (dc_lsize == 0) 115 r4k_blast_dcache_page = (void *)cache_noop; 116 else if (dc_lsize == 16) 117 r4k_blast_dcache_page = blast_dcache16_page; 118 else if (dc_lsize == 32) 119 r4k_blast_dcache_page = r4k_blast_dcache_page_dc32; 120 else if (dc_lsize == 64) 121 r4k_blast_dcache_page = r4k_blast_dcache_page_dc64; 122 } 123 124 static void (* r4k_blast_dcache_page_indexed)(unsigned long addr); 125 126 static void __cpuinit r4k_blast_dcache_page_indexed_setup(void) 127 { 128 unsigned long dc_lsize = cpu_dcache_line_size(); 129 130 if (dc_lsize == 0) 131 r4k_blast_dcache_page_indexed = (void *)cache_noop; 132 else if (dc_lsize == 16) 133 r4k_blast_dcache_page_indexed = blast_dcache16_page_indexed; 134 else if (dc_lsize == 32) 135 r4k_blast_dcache_page_indexed = blast_dcache32_page_indexed; 136 else if (dc_lsize == 64) 137 r4k_blast_dcache_page_indexed = blast_dcache64_page_indexed; 138 } 139 140 static void (* r4k_blast_dcache)(void); 141 142 static void __cpuinit r4k_blast_dcache_setup(void) 143 { 144 unsigned long dc_lsize = cpu_dcache_line_size(); 145 146 if (dc_lsize == 0) 147 r4k_blast_dcache = (void *)cache_noop; 148 else if (dc_lsize == 16) 149 r4k_blast_dcache = blast_dcache16; 150 else if (dc_lsize == 32) 151 r4k_blast_dcache = blast_dcache32; 152 else if (dc_lsize == 64) 153 r4k_blast_dcache = blast_dcache64; 154 } 155 156 /* force code alignment (used for TX49XX_ICACHE_INDEX_INV_WAR) */ 157 #define JUMP_TO_ALIGN(order) \ 158 __asm__ __volatile__( \ 159 "b\t1f\n\t" \ 160 ".align\t" #order "\n\t" \ 161 "1:\n\t" \ 162 ) 163 #define CACHE32_UNROLL32_ALIGN JUMP_TO_ALIGN(10) /* 32 * 32 = 1024 */ 164 #define CACHE32_UNROLL32_ALIGN2 JUMP_TO_ALIGN(11) 165 166 static inline void blast_r4600_v1_icache32(void) 167 { 168 unsigned long flags; 169 170 local_irq_save(flags); 171 blast_icache32(); 172 local_irq_restore(flags); 173 } 174 175 static inline void tx49_blast_icache32(void) 176 { 177 unsigned long start = INDEX_BASE; 178 unsigned long end = start + current_cpu_data.icache.waysize; 179 unsigned long ws_inc = 1UL << current_cpu_data.icache.waybit; 180 unsigned long ws_end = current_cpu_data.icache.ways << 181 current_cpu_data.icache.waybit; 182 unsigned long ws, addr; 183 184 CACHE32_UNROLL32_ALIGN2; 185 /* I'm in even chunk. blast odd chunks */ 186 for (ws = 0; ws < ws_end; ws += ws_inc) 187 for (addr = start + 0x400; addr < end; addr += 0x400 * 2) 188 cache32_unroll32(addr|ws, Index_Invalidate_I); 189 CACHE32_UNROLL32_ALIGN; 190 /* I'm in odd chunk. blast even chunks */ 191 for (ws = 0; ws < ws_end; ws += ws_inc) 192 for (addr = start; addr < end; addr += 0x400 * 2) 193 cache32_unroll32(addr|ws, Index_Invalidate_I); 194 } 195 196 static inline void blast_icache32_r4600_v1_page_indexed(unsigned long page) 197 { 198 unsigned long flags; 199 200 local_irq_save(flags); 201 blast_icache32_page_indexed(page); 202 local_irq_restore(flags); 203 } 204 205 static inline void tx49_blast_icache32_page_indexed(unsigned long page) 206 { 207 unsigned long indexmask = current_cpu_data.icache.waysize - 1; 208 unsigned long start = INDEX_BASE + (page & indexmask); 209 unsigned long end = start + PAGE_SIZE; 210 unsigned long ws_inc = 1UL << current_cpu_data.icache.waybit; 211 unsigned long ws_end = current_cpu_data.icache.ways << 212 current_cpu_data.icache.waybit; 213 unsigned long ws, addr; 214 215 CACHE32_UNROLL32_ALIGN2; 216 /* I'm in even chunk. blast odd chunks */ 217 for (ws = 0; ws < ws_end; ws += ws_inc) 218 for (addr = start + 0x400; addr < end; addr += 0x400 * 2) 219 cache32_unroll32(addr|ws, Index_Invalidate_I); 220 CACHE32_UNROLL32_ALIGN; 221 /* I'm in odd chunk. blast even chunks */ 222 for (ws = 0; ws < ws_end; ws += ws_inc) 223 for (addr = start; addr < end; addr += 0x400 * 2) 224 cache32_unroll32(addr|ws, Index_Invalidate_I); 225 } 226 227 static void (* r4k_blast_icache_page)(unsigned long addr); 228 229 static void __cpuinit r4k_blast_icache_page_setup(void) 230 { 231 unsigned long ic_lsize = cpu_icache_line_size(); 232 233 if (ic_lsize == 0) 234 r4k_blast_icache_page = (void *)cache_noop; 235 else if (ic_lsize == 16) 236 r4k_blast_icache_page = blast_icache16_page; 237 else if (ic_lsize == 32) 238 r4k_blast_icache_page = blast_icache32_page; 239 else if (ic_lsize == 64) 240 r4k_blast_icache_page = blast_icache64_page; 241 } 242 243 244 static void (* r4k_blast_icache_page_indexed)(unsigned long addr); 245 246 static void __cpuinit r4k_blast_icache_page_indexed_setup(void) 247 { 248 unsigned long ic_lsize = cpu_icache_line_size(); 249 250 if (ic_lsize == 0) 251 r4k_blast_icache_page_indexed = (void *)cache_noop; 252 else if (ic_lsize == 16) 253 r4k_blast_icache_page_indexed = blast_icache16_page_indexed; 254 else if (ic_lsize == 32) { 255 if (R4600_V1_INDEX_ICACHEOP_WAR && cpu_is_r4600_v1_x()) 256 r4k_blast_icache_page_indexed = 257 blast_icache32_r4600_v1_page_indexed; 258 else if (TX49XX_ICACHE_INDEX_INV_WAR) 259 r4k_blast_icache_page_indexed = 260 tx49_blast_icache32_page_indexed; 261 else 262 r4k_blast_icache_page_indexed = 263 blast_icache32_page_indexed; 264 } else if (ic_lsize == 64) 265 r4k_blast_icache_page_indexed = blast_icache64_page_indexed; 266 } 267 268 static void (* r4k_blast_icache)(void); 269 270 static void __cpuinit r4k_blast_icache_setup(void) 271 { 272 unsigned long ic_lsize = cpu_icache_line_size(); 273 274 if (ic_lsize == 0) 275 r4k_blast_icache = (void *)cache_noop; 276 else if (ic_lsize == 16) 277 r4k_blast_icache = blast_icache16; 278 else if (ic_lsize == 32) { 279 if (R4600_V1_INDEX_ICACHEOP_WAR && cpu_is_r4600_v1_x()) 280 r4k_blast_icache = blast_r4600_v1_icache32; 281 else if (TX49XX_ICACHE_INDEX_INV_WAR) 282 r4k_blast_icache = tx49_blast_icache32; 283 else 284 r4k_blast_icache = blast_icache32; 285 } else if (ic_lsize == 64) 286 r4k_blast_icache = blast_icache64; 287 } 288 289 static void (* r4k_blast_scache_page)(unsigned long addr); 290 291 static void __cpuinit r4k_blast_scache_page_setup(void) 292 { 293 unsigned long sc_lsize = cpu_scache_line_size(); 294 295 if (scache_size == 0) 296 r4k_blast_scache_page = (void *)cache_noop; 297 else if (sc_lsize == 16) 298 r4k_blast_scache_page = blast_scache16_page; 299 else if (sc_lsize == 32) 300 r4k_blast_scache_page = blast_scache32_page; 301 else if (sc_lsize == 64) 302 r4k_blast_scache_page = blast_scache64_page; 303 else if (sc_lsize == 128) 304 r4k_blast_scache_page = blast_scache128_page; 305 } 306 307 static void (* r4k_blast_scache_page_indexed)(unsigned long addr); 308 309 static void __cpuinit r4k_blast_scache_page_indexed_setup(void) 310 { 311 unsigned long sc_lsize = cpu_scache_line_size(); 312 313 if (scache_size == 0) 314 r4k_blast_scache_page_indexed = (void *)cache_noop; 315 else if (sc_lsize == 16) 316 r4k_blast_scache_page_indexed = blast_scache16_page_indexed; 317 else if (sc_lsize == 32) 318 r4k_blast_scache_page_indexed = blast_scache32_page_indexed; 319 else if (sc_lsize == 64) 320 r4k_blast_scache_page_indexed = blast_scache64_page_indexed; 321 else if (sc_lsize == 128) 322 r4k_blast_scache_page_indexed = blast_scache128_page_indexed; 323 } 324 325 static void (* r4k_blast_scache)(void); 326 327 static void __cpuinit r4k_blast_scache_setup(void) 328 { 329 unsigned long sc_lsize = cpu_scache_line_size(); 330 331 if (scache_size == 0) 332 r4k_blast_scache = (void *)cache_noop; 333 else if (sc_lsize == 16) 334 r4k_blast_scache = blast_scache16; 335 else if (sc_lsize == 32) 336 r4k_blast_scache = blast_scache32; 337 else if (sc_lsize == 64) 338 r4k_blast_scache = blast_scache64; 339 else if (sc_lsize == 128) 340 r4k_blast_scache = blast_scache128; 341 } 342 343 static inline void local_r4k___flush_cache_all(void * args) 344 { 345 #if defined(CONFIG_CPU_LOONGSON2) 346 r4k_blast_scache(); 347 return; 348 #endif 349 r4k_blast_dcache(); 350 r4k_blast_icache(); 351 352 switch (current_cpu_type()) { 353 case CPU_R4000SC: 354 case CPU_R4000MC: 355 case CPU_R4400SC: 356 case CPU_R4400MC: 357 case CPU_R10000: 358 case CPU_R12000: 359 case CPU_R14000: 360 r4k_blast_scache(); 361 } 362 } 363 364 static void r4k___flush_cache_all(void) 365 { 366 r4k_on_each_cpu(local_r4k___flush_cache_all, NULL, 1); 367 } 368 369 static inline int has_valid_asid(const struct mm_struct *mm) 370 { 371 #if defined(CONFIG_MIPS_MT_SMP) || defined(CONFIG_MIPS_MT_SMTC) 372 int i; 373 374 for_each_online_cpu(i) 375 if (cpu_context(i, mm)) 376 return 1; 377 378 return 0; 379 #else 380 return cpu_context(smp_processor_id(), mm); 381 #endif 382 } 383 384 static void r4k__flush_cache_vmap(void) 385 { 386 r4k_blast_dcache(); 387 } 388 389 static void r4k__flush_cache_vunmap(void) 390 { 391 r4k_blast_dcache(); 392 } 393 394 static inline void local_r4k_flush_cache_range(void * args) 395 { 396 struct vm_area_struct *vma = args; 397 int exec = vma->vm_flags & VM_EXEC; 398 399 if (!(has_valid_asid(vma->vm_mm))) 400 return; 401 402 r4k_blast_dcache(); 403 if (exec) 404 r4k_blast_icache(); 405 } 406 407 static void r4k_flush_cache_range(struct vm_area_struct *vma, 408 unsigned long start, unsigned long end) 409 { 410 int exec = vma->vm_flags & VM_EXEC; 411 412 if (cpu_has_dc_aliases || (exec && !cpu_has_ic_fills_f_dc)) 413 r4k_on_each_cpu(local_r4k_flush_cache_range, vma, 1); 414 } 415 416 static inline void local_r4k_flush_cache_mm(void * args) 417 { 418 struct mm_struct *mm = args; 419 420 if (!has_valid_asid(mm)) 421 return; 422 423 /* 424 * Kludge alert. For obscure reasons R4000SC and R4400SC go nuts if we 425 * only flush the primary caches but R10000 and R12000 behave sane ... 426 * R4000SC and R4400SC indexed S-cache ops also invalidate primary 427 * caches, so we can bail out early. 428 */ 429 if (current_cpu_type() == CPU_R4000SC || 430 current_cpu_type() == CPU_R4000MC || 431 current_cpu_type() == CPU_R4400SC || 432 current_cpu_type() == CPU_R4400MC) { 433 r4k_blast_scache(); 434 return; 435 } 436 437 r4k_blast_dcache(); 438 } 439 440 static void r4k_flush_cache_mm(struct mm_struct *mm) 441 { 442 if (!cpu_has_dc_aliases) 443 return; 444 445 r4k_on_each_cpu(local_r4k_flush_cache_mm, mm, 1); 446 } 447 448 struct flush_cache_page_args { 449 struct vm_area_struct *vma; 450 unsigned long addr; 451 unsigned long pfn; 452 }; 453 454 static inline void local_r4k_flush_cache_page(void *args) 455 { 456 struct flush_cache_page_args *fcp_args = args; 457 struct vm_area_struct *vma = fcp_args->vma; 458 unsigned long addr = fcp_args->addr; 459 struct page *page = pfn_to_page(fcp_args->pfn); 460 int exec = vma->vm_flags & VM_EXEC; 461 struct mm_struct *mm = vma->vm_mm; 462 int map_coherent = 0; 463 pgd_t *pgdp; 464 pud_t *pudp; 465 pmd_t *pmdp; 466 pte_t *ptep; 467 void *vaddr; 468 469 /* 470 * If ownes no valid ASID yet, cannot possibly have gotten 471 * this page into the cache. 472 */ 473 if (!has_valid_asid(mm)) 474 return; 475 476 addr &= PAGE_MASK; 477 pgdp = pgd_offset(mm, addr); 478 pudp = pud_offset(pgdp, addr); 479 pmdp = pmd_offset(pudp, addr); 480 ptep = pte_offset(pmdp, addr); 481 482 /* 483 * If the page isn't marked valid, the page cannot possibly be 484 * in the cache. 485 */ 486 if (!(pte_present(*ptep))) 487 return; 488 489 if ((mm == current->active_mm) && (pte_val(*ptep) & _PAGE_VALID)) 490 vaddr = NULL; 491 else { 492 /* 493 * Use kmap_coherent or kmap_atomic to do flushes for 494 * another ASID than the current one. 495 */ 496 map_coherent = (cpu_has_dc_aliases && 497 page_mapped(page) && !Page_dcache_dirty(page)); 498 if (map_coherent) 499 vaddr = kmap_coherent(page, addr); 500 else 501 vaddr = kmap_atomic(page, KM_USER0); 502 addr = (unsigned long)vaddr; 503 } 504 505 if (cpu_has_dc_aliases || (exec && !cpu_has_ic_fills_f_dc)) { 506 r4k_blast_dcache_page(addr); 507 if (exec && !cpu_icache_snoops_remote_store) 508 r4k_blast_scache_page(addr); 509 } 510 if (exec) { 511 if (vaddr && cpu_has_vtag_icache && mm == current->active_mm) { 512 int cpu = smp_processor_id(); 513 514 if (cpu_context(cpu, mm) != 0) 515 drop_mmu_context(mm, cpu); 516 } else 517 r4k_blast_icache_page(addr); 518 } 519 520 if (vaddr) { 521 if (map_coherent) 522 kunmap_coherent(); 523 else 524 kunmap_atomic(vaddr, KM_USER0); 525 } 526 } 527 528 static void r4k_flush_cache_page(struct vm_area_struct *vma, 529 unsigned long addr, unsigned long pfn) 530 { 531 struct flush_cache_page_args args; 532 533 args.vma = vma; 534 args.addr = addr; 535 args.pfn = pfn; 536 537 r4k_on_each_cpu(local_r4k_flush_cache_page, &args, 1); 538 } 539 540 static inline void local_r4k_flush_data_cache_page(void * addr) 541 { 542 r4k_blast_dcache_page((unsigned long) addr); 543 } 544 545 static void r4k_flush_data_cache_page(unsigned long addr) 546 { 547 if (in_atomic()) 548 local_r4k_flush_data_cache_page((void *)addr); 549 else 550 r4k_on_each_cpu(local_r4k_flush_data_cache_page, (void *) addr, 551 1); 552 } 553 554 struct flush_icache_range_args { 555 unsigned long start; 556 unsigned long end; 557 }; 558 559 static inline void local_r4k_flush_icache_range(unsigned long start, unsigned long end) 560 { 561 if (!cpu_has_ic_fills_f_dc) { 562 if (end - start >= dcache_size) { 563 r4k_blast_dcache(); 564 } else { 565 R4600_HIT_CACHEOP_WAR_IMPL; 566 protected_blast_dcache_range(start, end); 567 } 568 } 569 570 if (end - start > icache_size) 571 r4k_blast_icache(); 572 else 573 protected_blast_icache_range(start, end); 574 } 575 576 static inline void local_r4k_flush_icache_range_ipi(void *args) 577 { 578 struct flush_icache_range_args *fir_args = args; 579 unsigned long start = fir_args->start; 580 unsigned long end = fir_args->end; 581 582 local_r4k_flush_icache_range(start, end); 583 } 584 585 static void r4k_flush_icache_range(unsigned long start, unsigned long end) 586 { 587 struct flush_icache_range_args args; 588 589 args.start = start; 590 args.end = end; 591 592 r4k_on_each_cpu(local_r4k_flush_icache_range_ipi, &args, 1); 593 instruction_hazard(); 594 } 595 596 #ifdef CONFIG_DMA_NONCOHERENT 597 598 static void r4k_dma_cache_wback_inv(unsigned long addr, unsigned long size) 599 { 600 /* Catch bad driver code */ 601 BUG_ON(size == 0); 602 603 if (cpu_has_inclusive_pcaches) { 604 if (size >= scache_size) 605 r4k_blast_scache(); 606 else 607 blast_scache_range(addr, addr + size); 608 return; 609 } 610 611 /* 612 * Either no secondary cache or the available caches don't have the 613 * subset property so we have to flush the primary caches 614 * explicitly 615 */ 616 if (cpu_has_safe_index_cacheops && size >= dcache_size) { 617 r4k_blast_dcache(); 618 } else { 619 R4600_HIT_CACHEOP_WAR_IMPL; 620 blast_dcache_range(addr, addr + size); 621 } 622 623 bc_wback_inv(addr, size); 624 } 625 626 static void r4k_dma_cache_inv(unsigned long addr, unsigned long size) 627 { 628 /* Catch bad driver code */ 629 BUG_ON(size == 0); 630 631 if (cpu_has_inclusive_pcaches) { 632 if (size >= scache_size) 633 r4k_blast_scache(); 634 else { 635 unsigned long lsize = cpu_scache_line_size(); 636 unsigned long almask = ~(lsize - 1); 637 638 /* 639 * There is no clearly documented alignment requirement 640 * for the cache instruction on MIPS processors and 641 * some processors, among them the RM5200 and RM7000 642 * QED processors will throw an address error for cache 643 * hit ops with insufficient alignment. Solved by 644 * aligning the address to cache line size. 645 */ 646 cache_op(Hit_Writeback_Inv_SD, addr & almask); 647 cache_op(Hit_Writeback_Inv_SD, 648 (addr + size - 1) & almask); 649 blast_inv_scache_range(addr, addr + size); 650 } 651 return; 652 } 653 654 if (cpu_has_safe_index_cacheops && size >= dcache_size) { 655 r4k_blast_dcache(); 656 } else { 657 unsigned long lsize = cpu_dcache_line_size(); 658 unsigned long almask = ~(lsize - 1); 659 660 R4600_HIT_CACHEOP_WAR_IMPL; 661 cache_op(Hit_Writeback_Inv_D, addr & almask); 662 cache_op(Hit_Writeback_Inv_D, (addr + size - 1) & almask); 663 blast_inv_dcache_range(addr, addr + size); 664 } 665 666 bc_inv(addr, size); 667 } 668 #endif /* CONFIG_DMA_NONCOHERENT */ 669 670 /* 671 * While we're protected against bad userland addresses we don't care 672 * very much about what happens in that case. Usually a segmentation 673 * fault will dump the process later on anyway ... 674 */ 675 static void local_r4k_flush_cache_sigtramp(void * arg) 676 { 677 unsigned long ic_lsize = cpu_icache_line_size(); 678 unsigned long dc_lsize = cpu_dcache_line_size(); 679 unsigned long sc_lsize = cpu_scache_line_size(); 680 unsigned long addr = (unsigned long) arg; 681 682 R4600_HIT_CACHEOP_WAR_IMPL; 683 if (dc_lsize) 684 protected_writeback_dcache_line(addr & ~(dc_lsize - 1)); 685 if (!cpu_icache_snoops_remote_store && scache_size) 686 protected_writeback_scache_line(addr & ~(sc_lsize - 1)); 687 if (ic_lsize) 688 protected_flush_icache_line(addr & ~(ic_lsize - 1)); 689 if (MIPS4K_ICACHE_REFILL_WAR) { 690 __asm__ __volatile__ ( 691 ".set push\n\t" 692 ".set noat\n\t" 693 ".set mips3\n\t" 694 #ifdef CONFIG_32BIT 695 "la $at,1f\n\t" 696 #endif 697 #ifdef CONFIG_64BIT 698 "dla $at,1f\n\t" 699 #endif 700 "cache %0,($at)\n\t" 701 "nop; nop; nop\n" 702 "1:\n\t" 703 ".set pop" 704 : 705 : "i" (Hit_Invalidate_I)); 706 } 707 if (MIPS_CACHE_SYNC_WAR) 708 __asm__ __volatile__ ("sync"); 709 } 710 711 static void r4k_flush_cache_sigtramp(unsigned long addr) 712 { 713 r4k_on_each_cpu(local_r4k_flush_cache_sigtramp, (void *) addr, 1); 714 } 715 716 static void r4k_flush_icache_all(void) 717 { 718 if (cpu_has_vtag_icache) 719 r4k_blast_icache(); 720 } 721 722 static inline void rm7k_erratum31(void) 723 { 724 const unsigned long ic_lsize = 32; 725 unsigned long addr; 726 727 /* RM7000 erratum #31. The icache is screwed at startup. */ 728 write_c0_taglo(0); 729 write_c0_taghi(0); 730 731 for (addr = INDEX_BASE; addr <= INDEX_BASE + 4096; addr += ic_lsize) { 732 __asm__ __volatile__ ( 733 ".set push\n\t" 734 ".set noreorder\n\t" 735 ".set mips3\n\t" 736 "cache\t%1, 0(%0)\n\t" 737 "cache\t%1, 0x1000(%0)\n\t" 738 "cache\t%1, 0x2000(%0)\n\t" 739 "cache\t%1, 0x3000(%0)\n\t" 740 "cache\t%2, 0(%0)\n\t" 741 "cache\t%2, 0x1000(%0)\n\t" 742 "cache\t%2, 0x2000(%0)\n\t" 743 "cache\t%2, 0x3000(%0)\n\t" 744 "cache\t%1, 0(%0)\n\t" 745 "cache\t%1, 0x1000(%0)\n\t" 746 "cache\t%1, 0x2000(%0)\n\t" 747 "cache\t%1, 0x3000(%0)\n\t" 748 ".set pop\n" 749 : 750 : "r" (addr), "i" (Index_Store_Tag_I), "i" (Fill)); 751 } 752 } 753 754 static char *way_string[] __cpuinitdata = { NULL, "direct mapped", "2-way", 755 "3-way", "4-way", "5-way", "6-way", "7-way", "8-way" 756 }; 757 758 static void __cpuinit probe_pcache(void) 759 { 760 struct cpuinfo_mips *c = ¤t_cpu_data; 761 unsigned int config = read_c0_config(); 762 unsigned int prid = read_c0_prid(); 763 unsigned long config1; 764 unsigned int lsize; 765 766 switch (c->cputype) { 767 case CPU_R4600: /* QED style two way caches? */ 768 case CPU_R4700: 769 case CPU_R5000: 770 case CPU_NEVADA: 771 icache_size = 1 << (12 + ((config & CONF_IC) >> 9)); 772 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 773 c->icache.ways = 2; 774 c->icache.waybit = __ffs(icache_size/2); 775 776 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6)); 777 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 778 c->dcache.ways = 2; 779 c->dcache.waybit= __ffs(dcache_size/2); 780 781 c->options |= MIPS_CPU_CACHE_CDEX_P; 782 break; 783 784 case CPU_R5432: 785 case CPU_R5500: 786 icache_size = 1 << (12 + ((config & CONF_IC) >> 9)); 787 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 788 c->icache.ways = 2; 789 c->icache.waybit= 0; 790 791 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6)); 792 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 793 c->dcache.ways = 2; 794 c->dcache.waybit = 0; 795 796 c->options |= MIPS_CPU_CACHE_CDEX_P | MIPS_CPU_PREFETCH; 797 break; 798 799 case CPU_TX49XX: 800 icache_size = 1 << (12 + ((config & CONF_IC) >> 9)); 801 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 802 c->icache.ways = 4; 803 c->icache.waybit= 0; 804 805 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6)); 806 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 807 c->dcache.ways = 4; 808 c->dcache.waybit = 0; 809 810 c->options |= MIPS_CPU_CACHE_CDEX_P; 811 c->options |= MIPS_CPU_PREFETCH; 812 break; 813 814 case CPU_R4000PC: 815 case CPU_R4000SC: 816 case CPU_R4000MC: 817 case CPU_R4400PC: 818 case CPU_R4400SC: 819 case CPU_R4400MC: 820 case CPU_R4300: 821 icache_size = 1 << (12 + ((config & CONF_IC) >> 9)); 822 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 823 c->icache.ways = 1; 824 c->icache.waybit = 0; /* doesn't matter */ 825 826 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6)); 827 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 828 c->dcache.ways = 1; 829 c->dcache.waybit = 0; /* does not matter */ 830 831 c->options |= MIPS_CPU_CACHE_CDEX_P; 832 break; 833 834 case CPU_R10000: 835 case CPU_R12000: 836 case CPU_R14000: 837 icache_size = 1 << (12 + ((config & R10K_CONF_IC) >> 29)); 838 c->icache.linesz = 64; 839 c->icache.ways = 2; 840 c->icache.waybit = 0; 841 842 dcache_size = 1 << (12 + ((config & R10K_CONF_DC) >> 26)); 843 c->dcache.linesz = 32; 844 c->dcache.ways = 2; 845 c->dcache.waybit = 0; 846 847 c->options |= MIPS_CPU_PREFETCH; 848 break; 849 850 case CPU_VR4133: 851 write_c0_config(config & ~VR41_CONF_P4K); 852 case CPU_VR4131: 853 /* Workaround for cache instruction bug of VR4131 */ 854 if (c->processor_id == 0x0c80U || c->processor_id == 0x0c81U || 855 c->processor_id == 0x0c82U) { 856 config |= 0x00400000U; 857 if (c->processor_id == 0x0c80U) 858 config |= VR41_CONF_BP; 859 write_c0_config(config); 860 } else 861 c->options |= MIPS_CPU_CACHE_CDEX_P; 862 863 icache_size = 1 << (10 + ((config & CONF_IC) >> 9)); 864 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 865 c->icache.ways = 2; 866 c->icache.waybit = __ffs(icache_size/2); 867 868 dcache_size = 1 << (10 + ((config & CONF_DC) >> 6)); 869 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 870 c->dcache.ways = 2; 871 c->dcache.waybit = __ffs(dcache_size/2); 872 break; 873 874 case CPU_VR41XX: 875 case CPU_VR4111: 876 case CPU_VR4121: 877 case CPU_VR4122: 878 case CPU_VR4181: 879 case CPU_VR4181A: 880 icache_size = 1 << (10 + ((config & CONF_IC) >> 9)); 881 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 882 c->icache.ways = 1; 883 c->icache.waybit = 0; /* doesn't matter */ 884 885 dcache_size = 1 << (10 + ((config & CONF_DC) >> 6)); 886 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 887 c->dcache.ways = 1; 888 c->dcache.waybit = 0; /* does not matter */ 889 890 c->options |= MIPS_CPU_CACHE_CDEX_P; 891 break; 892 893 case CPU_RM7000: 894 rm7k_erratum31(); 895 896 case CPU_RM9000: 897 icache_size = 1 << (12 + ((config & CONF_IC) >> 9)); 898 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 899 c->icache.ways = 4; 900 c->icache.waybit = __ffs(icache_size / c->icache.ways); 901 902 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6)); 903 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 904 c->dcache.ways = 4; 905 c->dcache.waybit = __ffs(dcache_size / c->dcache.ways); 906 907 #if !defined(CONFIG_SMP) || !defined(RM9000_CDEX_SMP_WAR) 908 c->options |= MIPS_CPU_CACHE_CDEX_P; 909 #endif 910 c->options |= MIPS_CPU_PREFETCH; 911 break; 912 913 case CPU_LOONGSON2: 914 icache_size = 1 << (12 + ((config & CONF_IC) >> 9)); 915 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 916 if (prid & 0x3) 917 c->icache.ways = 4; 918 else 919 c->icache.ways = 2; 920 c->icache.waybit = 0; 921 922 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6)); 923 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 924 if (prid & 0x3) 925 c->dcache.ways = 4; 926 else 927 c->dcache.ways = 2; 928 c->dcache.waybit = 0; 929 break; 930 931 default: 932 if (!(config & MIPS_CONF_M)) 933 panic("Don't know how to probe P-caches on this cpu."); 934 935 /* 936 * So we seem to be a MIPS32 or MIPS64 CPU 937 * So let's probe the I-cache ... 938 */ 939 config1 = read_c0_config1(); 940 941 if ((lsize = ((config1 >> 19) & 7))) 942 c->icache.linesz = 2 << lsize; 943 else 944 c->icache.linesz = lsize; 945 c->icache.sets = 64 << ((config1 >> 22) & 7); 946 c->icache.ways = 1 + ((config1 >> 16) & 7); 947 948 icache_size = c->icache.sets * 949 c->icache.ways * 950 c->icache.linesz; 951 c->icache.waybit = __ffs(icache_size/c->icache.ways); 952 953 if (config & 0x8) /* VI bit */ 954 c->icache.flags |= MIPS_CACHE_VTAG; 955 956 /* 957 * Now probe the MIPS32 / MIPS64 data cache. 958 */ 959 c->dcache.flags = 0; 960 961 if ((lsize = ((config1 >> 10) & 7))) 962 c->dcache.linesz = 2 << lsize; 963 else 964 c->dcache.linesz= lsize; 965 c->dcache.sets = 64 << ((config1 >> 13) & 7); 966 c->dcache.ways = 1 + ((config1 >> 7) & 7); 967 968 dcache_size = c->dcache.sets * 969 c->dcache.ways * 970 c->dcache.linesz; 971 c->dcache.waybit = __ffs(dcache_size/c->dcache.ways); 972 973 c->options |= MIPS_CPU_PREFETCH; 974 break; 975 } 976 977 /* 978 * Processor configuration sanity check for the R4000SC erratum 979 * #5. With page sizes larger than 32kB there is no possibility 980 * to get a VCE exception anymore so we don't care about this 981 * misconfiguration. The case is rather theoretical anyway; 982 * presumably no vendor is shipping his hardware in the "bad" 983 * configuration. 984 */ 985 if ((prid & 0xff00) == PRID_IMP_R4000 && (prid & 0xff) < 0x40 && 986 !(config & CONF_SC) && c->icache.linesz != 16 && 987 PAGE_SIZE <= 0x8000) 988 panic("Improper R4000SC processor configuration detected"); 989 990 /* compute a couple of other cache variables */ 991 c->icache.waysize = icache_size / c->icache.ways; 992 c->dcache.waysize = dcache_size / c->dcache.ways; 993 994 c->icache.sets = c->icache.linesz ? 995 icache_size / (c->icache.linesz * c->icache.ways) : 0; 996 c->dcache.sets = c->dcache.linesz ? 997 dcache_size / (c->dcache.linesz * c->dcache.ways) : 0; 998 999 /* 1000 * R10000 and R12000 P-caches are odd in a positive way. They're 32kB 1001 * 2-way virtually indexed so normally would suffer from aliases. So 1002 * normally they'd suffer from aliases but magic in the hardware deals 1003 * with that for us so we don't need to take care ourselves. 1004 */ 1005 switch (c->cputype) { 1006 case CPU_20KC: 1007 case CPU_25KF: 1008 case CPU_SB1: 1009 case CPU_SB1A: 1010 c->dcache.flags |= MIPS_CACHE_PINDEX; 1011 break; 1012 1013 case CPU_R10000: 1014 case CPU_R12000: 1015 case CPU_R14000: 1016 break; 1017 1018 case CPU_24K: 1019 case CPU_34K: 1020 case CPU_74K: 1021 case CPU_1004K: 1022 if ((read_c0_config7() & (1 << 16))) { 1023 /* effectively physically indexed dcache, 1024 thus no virtual aliases. */ 1025 c->dcache.flags |= MIPS_CACHE_PINDEX; 1026 break; 1027 } 1028 default: 1029 if (c->dcache.waysize > PAGE_SIZE) 1030 c->dcache.flags |= MIPS_CACHE_ALIASES; 1031 } 1032 1033 switch (c->cputype) { 1034 case CPU_20KC: 1035 /* 1036 * Some older 20Kc chips doesn't have the 'VI' bit in 1037 * the config register. 1038 */ 1039 c->icache.flags |= MIPS_CACHE_VTAG; 1040 break; 1041 1042 case CPU_ALCHEMY: 1043 c->icache.flags |= MIPS_CACHE_IC_F_DC; 1044 break; 1045 } 1046 1047 #ifdef CONFIG_CPU_LOONGSON2 1048 /* 1049 * LOONGSON2 has 4 way icache, but when using indexed cache op, 1050 * one op will act on all 4 ways 1051 */ 1052 c->icache.ways = 1; 1053 #endif 1054 1055 printk("Primary instruction cache %ldkB, %s, %s, linesize %d bytes.\n", 1056 icache_size >> 10, 1057 c->icache.flags & MIPS_CACHE_VTAG ? "VIVT" : "VIPT", 1058 way_string[c->icache.ways], c->icache.linesz); 1059 1060 printk("Primary data cache %ldkB, %s, %s, %s, linesize %d bytes\n", 1061 dcache_size >> 10, way_string[c->dcache.ways], 1062 (c->dcache.flags & MIPS_CACHE_PINDEX) ? "PIPT" : "VIPT", 1063 (c->dcache.flags & MIPS_CACHE_ALIASES) ? 1064 "cache aliases" : "no aliases", 1065 c->dcache.linesz); 1066 } 1067 1068 /* 1069 * If you even _breathe_ on this function, look at the gcc output and make sure 1070 * it does not pop things on and off the stack for the cache sizing loop that 1071 * executes in KSEG1 space or else you will crash and burn badly. You have 1072 * been warned. 1073 */ 1074 static int __cpuinit probe_scache(void) 1075 { 1076 unsigned long flags, addr, begin, end, pow2; 1077 unsigned int config = read_c0_config(); 1078 struct cpuinfo_mips *c = ¤t_cpu_data; 1079 int tmp; 1080 1081 if (config & CONF_SC) 1082 return 0; 1083 1084 begin = (unsigned long) &_stext; 1085 begin &= ~((4 * 1024 * 1024) - 1); 1086 end = begin + (4 * 1024 * 1024); 1087 1088 /* 1089 * This is such a bitch, you'd think they would make it easy to do 1090 * this. Away you daemons of stupidity! 1091 */ 1092 local_irq_save(flags); 1093 1094 /* Fill each size-multiple cache line with a valid tag. */ 1095 pow2 = (64 * 1024); 1096 for (addr = begin; addr < end; addr = (begin + pow2)) { 1097 unsigned long *p = (unsigned long *) addr; 1098 __asm__ __volatile__("nop" : : "r" (*p)); /* whee... */ 1099 pow2 <<= 1; 1100 } 1101 1102 /* Load first line with zero (therefore invalid) tag. */ 1103 write_c0_taglo(0); 1104 write_c0_taghi(0); 1105 __asm__ __volatile__("nop; nop; nop; nop;"); /* avoid the hazard */ 1106 cache_op(Index_Store_Tag_I, begin); 1107 cache_op(Index_Store_Tag_D, begin); 1108 cache_op(Index_Store_Tag_SD, begin); 1109 1110 /* Now search for the wrap around point. */ 1111 pow2 = (128 * 1024); 1112 tmp = 0; 1113 for (addr = begin + (128 * 1024); addr < end; addr = begin + pow2) { 1114 cache_op(Index_Load_Tag_SD, addr); 1115 __asm__ __volatile__("nop; nop; nop; nop;"); /* hazard... */ 1116 if (!read_c0_taglo()) 1117 break; 1118 pow2 <<= 1; 1119 } 1120 local_irq_restore(flags); 1121 addr -= begin; 1122 1123 scache_size = addr; 1124 c->scache.linesz = 16 << ((config & R4K_CONF_SB) >> 22); 1125 c->scache.ways = 1; 1126 c->dcache.waybit = 0; /* does not matter */ 1127 1128 return 1; 1129 } 1130 1131 #if defined(CONFIG_CPU_LOONGSON2) 1132 static void __init loongson2_sc_init(void) 1133 { 1134 struct cpuinfo_mips *c = ¤t_cpu_data; 1135 1136 scache_size = 512*1024; 1137 c->scache.linesz = 32; 1138 c->scache.ways = 4; 1139 c->scache.waybit = 0; 1140 c->scache.waysize = scache_size / (c->scache.ways); 1141 c->scache.sets = scache_size / (c->scache.linesz * c->scache.ways); 1142 pr_info("Unified secondary cache %ldkB %s, linesize %d bytes.\n", 1143 scache_size >> 10, way_string[c->scache.ways], c->scache.linesz); 1144 1145 c->options |= MIPS_CPU_INCLUSIVE_CACHES; 1146 } 1147 #endif 1148 1149 extern int r5k_sc_init(void); 1150 extern int rm7k_sc_init(void); 1151 extern int mips_sc_init(void); 1152 1153 static void __cpuinit setup_scache(void) 1154 { 1155 struct cpuinfo_mips *c = ¤t_cpu_data; 1156 unsigned int config = read_c0_config(); 1157 int sc_present = 0; 1158 1159 /* 1160 * Do the probing thing on R4000SC and R4400SC processors. Other 1161 * processors don't have a S-cache that would be relevant to the 1162 * Linux memory management. 1163 */ 1164 switch (c->cputype) { 1165 case CPU_R4000SC: 1166 case CPU_R4000MC: 1167 case CPU_R4400SC: 1168 case CPU_R4400MC: 1169 sc_present = run_uncached(probe_scache); 1170 if (sc_present) 1171 c->options |= MIPS_CPU_CACHE_CDEX_S; 1172 break; 1173 1174 case CPU_R10000: 1175 case CPU_R12000: 1176 case CPU_R14000: 1177 scache_size = 0x80000 << ((config & R10K_CONF_SS) >> 16); 1178 c->scache.linesz = 64 << ((config >> 13) & 1); 1179 c->scache.ways = 2; 1180 c->scache.waybit= 0; 1181 sc_present = 1; 1182 break; 1183 1184 case CPU_R5000: 1185 case CPU_NEVADA: 1186 #ifdef CONFIG_R5000_CPU_SCACHE 1187 r5k_sc_init(); 1188 #endif 1189 return; 1190 1191 case CPU_RM7000: 1192 case CPU_RM9000: 1193 #ifdef CONFIG_RM7000_CPU_SCACHE 1194 rm7k_sc_init(); 1195 #endif 1196 return; 1197 1198 #if defined(CONFIG_CPU_LOONGSON2) 1199 case CPU_LOONGSON2: 1200 loongson2_sc_init(); 1201 return; 1202 #endif 1203 1204 default: 1205 if (c->isa_level == MIPS_CPU_ISA_M32R1 || 1206 c->isa_level == MIPS_CPU_ISA_M32R2 || 1207 c->isa_level == MIPS_CPU_ISA_M64R1 || 1208 c->isa_level == MIPS_CPU_ISA_M64R2) { 1209 #ifdef CONFIG_MIPS_CPU_SCACHE 1210 if (mips_sc_init ()) { 1211 scache_size = c->scache.ways * c->scache.sets * c->scache.linesz; 1212 printk("MIPS secondary cache %ldkB, %s, linesize %d bytes.\n", 1213 scache_size >> 10, 1214 way_string[c->scache.ways], c->scache.linesz); 1215 } 1216 #else 1217 if (!(c->scache.flags & MIPS_CACHE_NOT_PRESENT)) 1218 panic("Dunno how to handle MIPS32 / MIPS64 second level cache"); 1219 #endif 1220 return; 1221 } 1222 sc_present = 0; 1223 } 1224 1225 if (!sc_present) 1226 return; 1227 1228 /* compute a couple of other cache variables */ 1229 c->scache.waysize = scache_size / c->scache.ways; 1230 1231 c->scache.sets = scache_size / (c->scache.linesz * c->scache.ways); 1232 1233 printk("Unified secondary cache %ldkB %s, linesize %d bytes.\n", 1234 scache_size >> 10, way_string[c->scache.ways], c->scache.linesz); 1235 1236 c->options |= MIPS_CPU_INCLUSIVE_CACHES; 1237 } 1238 1239 void au1x00_fixup_config_od(void) 1240 { 1241 /* 1242 * c0_config.od (bit 19) was write only (and read as 0) 1243 * on the early revisions of Alchemy SOCs. It disables the bus 1244 * transaction overlapping and needs to be set to fix various errata. 1245 */ 1246 switch (read_c0_prid()) { 1247 case 0x00030100: /* Au1000 DA */ 1248 case 0x00030201: /* Au1000 HA */ 1249 case 0x00030202: /* Au1000 HB */ 1250 case 0x01030200: /* Au1500 AB */ 1251 /* 1252 * Au1100 errata actually keeps silence about this bit, so we set it 1253 * just in case for those revisions that require it to be set according 1254 * to the (now gone) cpu table. 1255 */ 1256 case 0x02030200: /* Au1100 AB */ 1257 case 0x02030201: /* Au1100 BA */ 1258 case 0x02030202: /* Au1100 BC */ 1259 set_c0_config(1 << 19); 1260 break; 1261 } 1262 } 1263 1264 /* CP0 hazard avoidance. */ 1265 #define NXP_BARRIER() \ 1266 __asm__ __volatile__( \ 1267 ".set noreorder\n\t" \ 1268 "nop; nop; nop; nop; nop; nop;\n\t" \ 1269 ".set reorder\n\t") 1270 1271 static void nxp_pr4450_fixup_config(void) 1272 { 1273 unsigned long config0; 1274 1275 config0 = read_c0_config(); 1276 1277 /* clear all three cache coherency fields */ 1278 config0 &= ~(0x7 | (7 << 25) | (7 << 28)); 1279 config0 |= (((_page_cachable_default >> _CACHE_SHIFT) << 0) | 1280 ((_page_cachable_default >> _CACHE_SHIFT) << 25) | 1281 ((_page_cachable_default >> _CACHE_SHIFT) << 28)); 1282 write_c0_config(config0); 1283 NXP_BARRIER(); 1284 } 1285 1286 static int __cpuinitdata cca = -1; 1287 1288 static int __init cca_setup(char *str) 1289 { 1290 get_option(&str, &cca); 1291 1292 return 1; 1293 } 1294 1295 __setup("cca=", cca_setup); 1296 1297 static void __cpuinit coherency_setup(void) 1298 { 1299 if (cca < 0 || cca > 7) 1300 cca = read_c0_config() & CONF_CM_CMASK; 1301 _page_cachable_default = cca << _CACHE_SHIFT; 1302 1303 pr_debug("Using cache attribute %d\n", cca); 1304 change_c0_config(CONF_CM_CMASK, cca); 1305 1306 /* 1307 * c0_status.cu=0 specifies that updates by the sc instruction use 1308 * the coherency mode specified by the TLB; 1 means cachable 1309 * coherent update on write will be used. Not all processors have 1310 * this bit and; some wire it to zero, others like Toshiba had the 1311 * silly idea of putting something else there ... 1312 */ 1313 switch (current_cpu_type()) { 1314 case CPU_R4000PC: 1315 case CPU_R4000SC: 1316 case CPU_R4000MC: 1317 case CPU_R4400PC: 1318 case CPU_R4400SC: 1319 case CPU_R4400MC: 1320 clear_c0_config(CONF_CU); 1321 break; 1322 /* 1323 * We need to catch the early Alchemy SOCs with 1324 * the write-only co_config.od bit and set it back to one on: 1325 * Au1000 rev DA, HA, HB; Au1100 AB, BA, BC, Au1500 AB 1326 */ 1327 case CPU_ALCHEMY: 1328 au1x00_fixup_config_od(); 1329 break; 1330 1331 case PRID_IMP_PR4450: 1332 nxp_pr4450_fixup_config(); 1333 break; 1334 } 1335 } 1336 1337 #if defined(CONFIG_DMA_NONCOHERENT) 1338 1339 static int __cpuinitdata coherentio; 1340 1341 static int __init setcoherentio(char *str) 1342 { 1343 coherentio = 1; 1344 1345 return 1; 1346 } 1347 1348 __setup("coherentio", setcoherentio); 1349 #endif 1350 1351 void __cpuinit r4k_cache_init(void) 1352 { 1353 extern void build_clear_page(void); 1354 extern void build_copy_page(void); 1355 extern char __weak except_vec2_generic; 1356 extern char __weak except_vec2_sb1; 1357 struct cpuinfo_mips *c = ¤t_cpu_data; 1358 1359 switch (c->cputype) { 1360 case CPU_SB1: 1361 case CPU_SB1A: 1362 set_uncached_handler(0x100, &except_vec2_sb1, 0x80); 1363 break; 1364 1365 default: 1366 set_uncached_handler(0x100, &except_vec2_generic, 0x80); 1367 break; 1368 } 1369 1370 probe_pcache(); 1371 setup_scache(); 1372 1373 r4k_blast_dcache_page_setup(); 1374 r4k_blast_dcache_page_indexed_setup(); 1375 r4k_blast_dcache_setup(); 1376 r4k_blast_icache_page_setup(); 1377 r4k_blast_icache_page_indexed_setup(); 1378 r4k_blast_icache_setup(); 1379 r4k_blast_scache_page_setup(); 1380 r4k_blast_scache_page_indexed_setup(); 1381 r4k_blast_scache_setup(); 1382 1383 /* 1384 * Some MIPS32 and MIPS64 processors have physically indexed caches. 1385 * This code supports virtually indexed processors and will be 1386 * unnecessarily inefficient on physically indexed processors. 1387 */ 1388 if (c->dcache.linesz) 1389 shm_align_mask = max_t( unsigned long, 1390 c->dcache.sets * c->dcache.linesz - 1, 1391 PAGE_SIZE - 1); 1392 else 1393 shm_align_mask = PAGE_SIZE-1; 1394 1395 __flush_cache_vmap = r4k__flush_cache_vmap; 1396 __flush_cache_vunmap = r4k__flush_cache_vunmap; 1397 1398 flush_cache_all = cache_noop; 1399 __flush_cache_all = r4k___flush_cache_all; 1400 flush_cache_mm = r4k_flush_cache_mm; 1401 flush_cache_page = r4k_flush_cache_page; 1402 flush_cache_range = r4k_flush_cache_range; 1403 1404 flush_cache_sigtramp = r4k_flush_cache_sigtramp; 1405 flush_icache_all = r4k_flush_icache_all; 1406 local_flush_data_cache_page = local_r4k_flush_data_cache_page; 1407 flush_data_cache_page = r4k_flush_data_cache_page; 1408 flush_icache_range = r4k_flush_icache_range; 1409 local_flush_icache_range = local_r4k_flush_icache_range; 1410 1411 #if defined(CONFIG_DMA_NONCOHERENT) 1412 if (coherentio) { 1413 _dma_cache_wback_inv = (void *)cache_noop; 1414 _dma_cache_wback = (void *)cache_noop; 1415 _dma_cache_inv = (void *)cache_noop; 1416 } else { 1417 _dma_cache_wback_inv = r4k_dma_cache_wback_inv; 1418 _dma_cache_wback = r4k_dma_cache_wback_inv; 1419 _dma_cache_inv = r4k_dma_cache_inv; 1420 } 1421 #endif 1422 1423 build_clear_page(); 1424 build_copy_page(); 1425 #if !defined(CONFIG_MIPS_CMP) 1426 local_r4k___flush_cache_all(NULL); 1427 #endif 1428 coherency_setup(); 1429 } 1430