1 /* 2 * This file is subject to the terms and conditions of the GNU General Public 3 * License. See the file "COPYING" in the main directory of this archive 4 * for more details. 5 * 6 * Copyright (C) 1996 David S. Miller (davem@davemloft.net) 7 * Copyright (C) 1997, 1998, 1999, 2000, 2001, 2002 Ralf Baechle (ralf@gnu.org) 8 * Copyright (C) 1999, 2000 Silicon Graphics, Inc. 9 */ 10 #include <linux/hardirq.h> 11 #include <linux/init.h> 12 #include <linux/highmem.h> 13 #include <linux/kernel.h> 14 #include <linux/linkage.h> 15 #include <linux/preempt.h> 16 #include <linux/sched.h> 17 #include <linux/smp.h> 18 #include <linux/mm.h> 19 #include <linux/module.h> 20 #include <linux/bitops.h> 21 22 #include <asm/bcache.h> 23 #include <asm/bootinfo.h> 24 #include <asm/cache.h> 25 #include <asm/cacheops.h> 26 #include <asm/cpu.h> 27 #include <asm/cpu-features.h> 28 #include <asm/cpu-type.h> 29 #include <asm/io.h> 30 #include <asm/page.h> 31 #include <asm/pgtable.h> 32 #include <asm/r4kcache.h> 33 #include <asm/sections.h> 34 #include <asm/mmu_context.h> 35 #include <asm/war.h> 36 #include <asm/cacheflush.h> /* for run_uncached() */ 37 #include <asm/traps.h> 38 #include <asm/dma-coherence.h> 39 40 /* 41 * Special Variant of smp_call_function for use by cache functions: 42 * 43 * o No return value 44 * o collapses to normal function call on UP kernels 45 * o collapses to normal function call on systems with a single shared 46 * primary cache. 47 * o doesn't disable interrupts on the local CPU 48 */ 49 static inline void r4k_on_each_cpu(void (*func) (void *info), void *info) 50 { 51 preempt_disable(); 52 53 #if !defined(CONFIG_MIPS_MT_SMP) && !defined(CONFIG_MIPS_MT_SMTC) 54 smp_call_function(func, info, 1); 55 #endif 56 func(info); 57 preempt_enable(); 58 } 59 60 #if defined(CONFIG_MIPS_CMP) || defined(CONFIG_MIPS_CPS) 61 #define cpu_has_safe_index_cacheops 0 62 #else 63 #define cpu_has_safe_index_cacheops 1 64 #endif 65 66 /* 67 * Must die. 68 */ 69 static unsigned long icache_size __read_mostly; 70 static unsigned long dcache_size __read_mostly; 71 static unsigned long scache_size __read_mostly; 72 73 /* 74 * Dummy cache handling routines for machines without boardcaches 75 */ 76 static void cache_noop(void) {} 77 78 static struct bcache_ops no_sc_ops = { 79 .bc_enable = (void *)cache_noop, 80 .bc_disable = (void *)cache_noop, 81 .bc_wback_inv = (void *)cache_noop, 82 .bc_inv = (void *)cache_noop 83 }; 84 85 struct bcache_ops *bcops = &no_sc_ops; 86 87 #define cpu_is_r4600_v1_x() ((read_c0_prid() & 0xfffffff0) == 0x00002010) 88 #define cpu_is_r4600_v2_x() ((read_c0_prid() & 0xfffffff0) == 0x00002020) 89 90 #define R4600_HIT_CACHEOP_WAR_IMPL \ 91 do { \ 92 if (R4600_V2_HIT_CACHEOP_WAR && cpu_is_r4600_v2_x()) \ 93 *(volatile unsigned long *)CKSEG1; \ 94 if (R4600_V1_HIT_CACHEOP_WAR) \ 95 __asm__ __volatile__("nop;nop;nop;nop"); \ 96 } while (0) 97 98 static void (*r4k_blast_dcache_page)(unsigned long addr); 99 100 static inline void r4k_blast_dcache_page_dc32(unsigned long addr) 101 { 102 R4600_HIT_CACHEOP_WAR_IMPL; 103 blast_dcache32_page(addr); 104 } 105 106 static inline void r4k_blast_dcache_page_dc64(unsigned long addr) 107 { 108 R4600_HIT_CACHEOP_WAR_IMPL; 109 blast_dcache64_page(addr); 110 } 111 112 static void r4k_blast_dcache_page_setup(void) 113 { 114 unsigned long dc_lsize = cpu_dcache_line_size(); 115 116 if (dc_lsize == 0) 117 r4k_blast_dcache_page = (void *)cache_noop; 118 else if (dc_lsize == 16) 119 r4k_blast_dcache_page = blast_dcache16_page; 120 else if (dc_lsize == 32) 121 r4k_blast_dcache_page = r4k_blast_dcache_page_dc32; 122 else if (dc_lsize == 64) 123 r4k_blast_dcache_page = r4k_blast_dcache_page_dc64; 124 } 125 126 #ifndef CONFIG_EVA 127 #define r4k_blast_dcache_user_page r4k_blast_dcache_page 128 #else 129 130 static void (*r4k_blast_dcache_user_page)(unsigned long addr); 131 132 static void r4k_blast_dcache_user_page_setup(void) 133 { 134 unsigned long dc_lsize = cpu_dcache_line_size(); 135 136 if (dc_lsize == 0) 137 r4k_blast_dcache_user_page = (void *)cache_noop; 138 else if (dc_lsize == 16) 139 r4k_blast_dcache_user_page = blast_dcache16_user_page; 140 else if (dc_lsize == 32) 141 r4k_blast_dcache_user_page = blast_dcache32_user_page; 142 else if (dc_lsize == 64) 143 r4k_blast_dcache_user_page = blast_dcache64_user_page; 144 } 145 146 #endif 147 148 static void (* r4k_blast_dcache_page_indexed)(unsigned long addr); 149 150 static void r4k_blast_dcache_page_indexed_setup(void) 151 { 152 unsigned long dc_lsize = cpu_dcache_line_size(); 153 154 if (dc_lsize == 0) 155 r4k_blast_dcache_page_indexed = (void *)cache_noop; 156 else if (dc_lsize == 16) 157 r4k_blast_dcache_page_indexed = blast_dcache16_page_indexed; 158 else if (dc_lsize == 32) 159 r4k_blast_dcache_page_indexed = blast_dcache32_page_indexed; 160 else if (dc_lsize == 64) 161 r4k_blast_dcache_page_indexed = blast_dcache64_page_indexed; 162 } 163 164 void (* r4k_blast_dcache)(void); 165 EXPORT_SYMBOL(r4k_blast_dcache); 166 167 static void r4k_blast_dcache_setup(void) 168 { 169 unsigned long dc_lsize = cpu_dcache_line_size(); 170 171 if (dc_lsize == 0) 172 r4k_blast_dcache = (void *)cache_noop; 173 else if (dc_lsize == 16) 174 r4k_blast_dcache = blast_dcache16; 175 else if (dc_lsize == 32) 176 r4k_blast_dcache = blast_dcache32; 177 else if (dc_lsize == 64) 178 r4k_blast_dcache = blast_dcache64; 179 } 180 181 /* force code alignment (used for TX49XX_ICACHE_INDEX_INV_WAR) */ 182 #define JUMP_TO_ALIGN(order) \ 183 __asm__ __volatile__( \ 184 "b\t1f\n\t" \ 185 ".align\t" #order "\n\t" \ 186 "1:\n\t" \ 187 ) 188 #define CACHE32_UNROLL32_ALIGN JUMP_TO_ALIGN(10) /* 32 * 32 = 1024 */ 189 #define CACHE32_UNROLL32_ALIGN2 JUMP_TO_ALIGN(11) 190 191 static inline void blast_r4600_v1_icache32(void) 192 { 193 unsigned long flags; 194 195 local_irq_save(flags); 196 blast_icache32(); 197 local_irq_restore(flags); 198 } 199 200 static inline void tx49_blast_icache32(void) 201 { 202 unsigned long start = INDEX_BASE; 203 unsigned long end = start + current_cpu_data.icache.waysize; 204 unsigned long ws_inc = 1UL << current_cpu_data.icache.waybit; 205 unsigned long ws_end = current_cpu_data.icache.ways << 206 current_cpu_data.icache.waybit; 207 unsigned long ws, addr; 208 209 CACHE32_UNROLL32_ALIGN2; 210 /* I'm in even chunk. blast odd chunks */ 211 for (ws = 0; ws < ws_end; ws += ws_inc) 212 for (addr = start + 0x400; addr < end; addr += 0x400 * 2) 213 cache32_unroll32(addr|ws, Index_Invalidate_I); 214 CACHE32_UNROLL32_ALIGN; 215 /* I'm in odd chunk. blast even chunks */ 216 for (ws = 0; ws < ws_end; ws += ws_inc) 217 for (addr = start; addr < end; addr += 0x400 * 2) 218 cache32_unroll32(addr|ws, Index_Invalidate_I); 219 } 220 221 static inline void blast_icache32_r4600_v1_page_indexed(unsigned long page) 222 { 223 unsigned long flags; 224 225 local_irq_save(flags); 226 blast_icache32_page_indexed(page); 227 local_irq_restore(flags); 228 } 229 230 static inline void tx49_blast_icache32_page_indexed(unsigned long page) 231 { 232 unsigned long indexmask = current_cpu_data.icache.waysize - 1; 233 unsigned long start = INDEX_BASE + (page & indexmask); 234 unsigned long end = start + PAGE_SIZE; 235 unsigned long ws_inc = 1UL << current_cpu_data.icache.waybit; 236 unsigned long ws_end = current_cpu_data.icache.ways << 237 current_cpu_data.icache.waybit; 238 unsigned long ws, addr; 239 240 CACHE32_UNROLL32_ALIGN2; 241 /* I'm in even chunk. blast odd chunks */ 242 for (ws = 0; ws < ws_end; ws += ws_inc) 243 for (addr = start + 0x400; addr < end; addr += 0x400 * 2) 244 cache32_unroll32(addr|ws, Index_Invalidate_I); 245 CACHE32_UNROLL32_ALIGN; 246 /* I'm in odd chunk. blast even chunks */ 247 for (ws = 0; ws < ws_end; ws += ws_inc) 248 for (addr = start; addr < end; addr += 0x400 * 2) 249 cache32_unroll32(addr|ws, Index_Invalidate_I); 250 } 251 252 static void (* r4k_blast_icache_page)(unsigned long addr); 253 254 static void r4k_blast_icache_page_setup(void) 255 { 256 unsigned long ic_lsize = cpu_icache_line_size(); 257 258 if (ic_lsize == 0) 259 r4k_blast_icache_page = (void *)cache_noop; 260 else if (ic_lsize == 16) 261 r4k_blast_icache_page = blast_icache16_page; 262 else if (ic_lsize == 32 && current_cpu_type() == CPU_LOONGSON2) 263 r4k_blast_icache_page = loongson2_blast_icache32_page; 264 else if (ic_lsize == 32) 265 r4k_blast_icache_page = blast_icache32_page; 266 else if (ic_lsize == 64) 267 r4k_blast_icache_page = blast_icache64_page; 268 } 269 270 #ifndef CONFIG_EVA 271 #define r4k_blast_icache_user_page r4k_blast_icache_page 272 #else 273 274 static void (*r4k_blast_icache_user_page)(unsigned long addr); 275 276 static void __cpuinit r4k_blast_icache_user_page_setup(void) 277 { 278 unsigned long ic_lsize = cpu_icache_line_size(); 279 280 if (ic_lsize == 0) 281 r4k_blast_icache_user_page = (void *)cache_noop; 282 else if (ic_lsize == 16) 283 r4k_blast_icache_user_page = blast_icache16_user_page; 284 else if (ic_lsize == 32) 285 r4k_blast_icache_user_page = blast_icache32_user_page; 286 else if (ic_lsize == 64) 287 r4k_blast_icache_user_page = blast_icache64_user_page; 288 } 289 290 #endif 291 292 static void (* r4k_blast_icache_page_indexed)(unsigned long addr); 293 294 static void r4k_blast_icache_page_indexed_setup(void) 295 { 296 unsigned long ic_lsize = cpu_icache_line_size(); 297 298 if (ic_lsize == 0) 299 r4k_blast_icache_page_indexed = (void *)cache_noop; 300 else if (ic_lsize == 16) 301 r4k_blast_icache_page_indexed = blast_icache16_page_indexed; 302 else if (ic_lsize == 32) { 303 if (R4600_V1_INDEX_ICACHEOP_WAR && cpu_is_r4600_v1_x()) 304 r4k_blast_icache_page_indexed = 305 blast_icache32_r4600_v1_page_indexed; 306 else if (TX49XX_ICACHE_INDEX_INV_WAR) 307 r4k_blast_icache_page_indexed = 308 tx49_blast_icache32_page_indexed; 309 else if (current_cpu_type() == CPU_LOONGSON2) 310 r4k_blast_icache_page_indexed = 311 loongson2_blast_icache32_page_indexed; 312 else 313 r4k_blast_icache_page_indexed = 314 blast_icache32_page_indexed; 315 } else if (ic_lsize == 64) 316 r4k_blast_icache_page_indexed = blast_icache64_page_indexed; 317 } 318 319 void (* r4k_blast_icache)(void); 320 EXPORT_SYMBOL(r4k_blast_icache); 321 322 static void r4k_blast_icache_setup(void) 323 { 324 unsigned long ic_lsize = cpu_icache_line_size(); 325 326 if (ic_lsize == 0) 327 r4k_blast_icache = (void *)cache_noop; 328 else if (ic_lsize == 16) 329 r4k_blast_icache = blast_icache16; 330 else if (ic_lsize == 32) { 331 if (R4600_V1_INDEX_ICACHEOP_WAR && cpu_is_r4600_v1_x()) 332 r4k_blast_icache = blast_r4600_v1_icache32; 333 else if (TX49XX_ICACHE_INDEX_INV_WAR) 334 r4k_blast_icache = tx49_blast_icache32; 335 else if (current_cpu_type() == CPU_LOONGSON2) 336 r4k_blast_icache = loongson2_blast_icache32; 337 else 338 r4k_blast_icache = blast_icache32; 339 } else if (ic_lsize == 64) 340 r4k_blast_icache = blast_icache64; 341 } 342 343 static void (* r4k_blast_scache_page)(unsigned long addr); 344 345 static void r4k_blast_scache_page_setup(void) 346 { 347 unsigned long sc_lsize = cpu_scache_line_size(); 348 349 if (scache_size == 0) 350 r4k_blast_scache_page = (void *)cache_noop; 351 else if (sc_lsize == 16) 352 r4k_blast_scache_page = blast_scache16_page; 353 else if (sc_lsize == 32) 354 r4k_blast_scache_page = blast_scache32_page; 355 else if (sc_lsize == 64) 356 r4k_blast_scache_page = blast_scache64_page; 357 else if (sc_lsize == 128) 358 r4k_blast_scache_page = blast_scache128_page; 359 } 360 361 static void (* r4k_blast_scache_page_indexed)(unsigned long addr); 362 363 static void r4k_blast_scache_page_indexed_setup(void) 364 { 365 unsigned long sc_lsize = cpu_scache_line_size(); 366 367 if (scache_size == 0) 368 r4k_blast_scache_page_indexed = (void *)cache_noop; 369 else if (sc_lsize == 16) 370 r4k_blast_scache_page_indexed = blast_scache16_page_indexed; 371 else if (sc_lsize == 32) 372 r4k_blast_scache_page_indexed = blast_scache32_page_indexed; 373 else if (sc_lsize == 64) 374 r4k_blast_scache_page_indexed = blast_scache64_page_indexed; 375 else if (sc_lsize == 128) 376 r4k_blast_scache_page_indexed = blast_scache128_page_indexed; 377 } 378 379 static void (* r4k_blast_scache)(void); 380 381 static void r4k_blast_scache_setup(void) 382 { 383 unsigned long sc_lsize = cpu_scache_line_size(); 384 385 if (scache_size == 0) 386 r4k_blast_scache = (void *)cache_noop; 387 else if (sc_lsize == 16) 388 r4k_blast_scache = blast_scache16; 389 else if (sc_lsize == 32) 390 r4k_blast_scache = blast_scache32; 391 else if (sc_lsize == 64) 392 r4k_blast_scache = blast_scache64; 393 else if (sc_lsize == 128) 394 r4k_blast_scache = blast_scache128; 395 } 396 397 static inline void local_r4k___flush_cache_all(void * args) 398 { 399 switch (current_cpu_type()) { 400 case CPU_LOONGSON2: 401 case CPU_LOONGSON3: 402 case CPU_R4000SC: 403 case CPU_R4000MC: 404 case CPU_R4400SC: 405 case CPU_R4400MC: 406 case CPU_R10000: 407 case CPU_R12000: 408 case CPU_R14000: 409 /* 410 * These caches are inclusive caches, that is, if something 411 * is not cached in the S-cache, we know it also won't be 412 * in one of the primary caches. 413 */ 414 r4k_blast_scache(); 415 break; 416 417 default: 418 r4k_blast_dcache(); 419 r4k_blast_icache(); 420 break; 421 } 422 } 423 424 static void r4k___flush_cache_all(void) 425 { 426 r4k_on_each_cpu(local_r4k___flush_cache_all, NULL); 427 } 428 429 static inline int has_valid_asid(const struct mm_struct *mm) 430 { 431 #if defined(CONFIG_MIPS_MT_SMP) || defined(CONFIG_MIPS_MT_SMTC) 432 int i; 433 434 for_each_online_cpu(i) 435 if (cpu_context(i, mm)) 436 return 1; 437 438 return 0; 439 #else 440 return cpu_context(smp_processor_id(), mm); 441 #endif 442 } 443 444 static void r4k__flush_cache_vmap(void) 445 { 446 r4k_blast_dcache(); 447 } 448 449 static void r4k__flush_cache_vunmap(void) 450 { 451 r4k_blast_dcache(); 452 } 453 454 static inline void local_r4k_flush_cache_range(void * args) 455 { 456 struct vm_area_struct *vma = args; 457 int exec = vma->vm_flags & VM_EXEC; 458 459 if (!(has_valid_asid(vma->vm_mm))) 460 return; 461 462 r4k_blast_dcache(); 463 if (exec) 464 r4k_blast_icache(); 465 } 466 467 static void r4k_flush_cache_range(struct vm_area_struct *vma, 468 unsigned long start, unsigned long end) 469 { 470 int exec = vma->vm_flags & VM_EXEC; 471 472 if (cpu_has_dc_aliases || (exec && !cpu_has_ic_fills_f_dc)) 473 r4k_on_each_cpu(local_r4k_flush_cache_range, vma); 474 } 475 476 static inline void local_r4k_flush_cache_mm(void * args) 477 { 478 struct mm_struct *mm = args; 479 480 if (!has_valid_asid(mm)) 481 return; 482 483 /* 484 * Kludge alert. For obscure reasons R4000SC and R4400SC go nuts if we 485 * only flush the primary caches but R10000 and R12000 behave sane ... 486 * R4000SC and R4400SC indexed S-cache ops also invalidate primary 487 * caches, so we can bail out early. 488 */ 489 if (current_cpu_type() == CPU_R4000SC || 490 current_cpu_type() == CPU_R4000MC || 491 current_cpu_type() == CPU_R4400SC || 492 current_cpu_type() == CPU_R4400MC) { 493 r4k_blast_scache(); 494 return; 495 } 496 497 r4k_blast_dcache(); 498 } 499 500 static void r4k_flush_cache_mm(struct mm_struct *mm) 501 { 502 if (!cpu_has_dc_aliases) 503 return; 504 505 r4k_on_each_cpu(local_r4k_flush_cache_mm, mm); 506 } 507 508 struct flush_cache_page_args { 509 struct vm_area_struct *vma; 510 unsigned long addr; 511 unsigned long pfn; 512 }; 513 514 static inline void local_r4k_flush_cache_page(void *args) 515 { 516 struct flush_cache_page_args *fcp_args = args; 517 struct vm_area_struct *vma = fcp_args->vma; 518 unsigned long addr = fcp_args->addr; 519 struct page *page = pfn_to_page(fcp_args->pfn); 520 int exec = vma->vm_flags & VM_EXEC; 521 struct mm_struct *mm = vma->vm_mm; 522 int map_coherent = 0; 523 pgd_t *pgdp; 524 pud_t *pudp; 525 pmd_t *pmdp; 526 pte_t *ptep; 527 void *vaddr; 528 529 /* 530 * If ownes no valid ASID yet, cannot possibly have gotten 531 * this page into the cache. 532 */ 533 if (!has_valid_asid(mm)) 534 return; 535 536 addr &= PAGE_MASK; 537 pgdp = pgd_offset(mm, addr); 538 pudp = pud_offset(pgdp, addr); 539 pmdp = pmd_offset(pudp, addr); 540 ptep = pte_offset(pmdp, addr); 541 542 /* 543 * If the page isn't marked valid, the page cannot possibly be 544 * in the cache. 545 */ 546 if (!(pte_present(*ptep))) 547 return; 548 549 if ((mm == current->active_mm) && (pte_val(*ptep) & _PAGE_VALID)) 550 vaddr = NULL; 551 else { 552 /* 553 * Use kmap_coherent or kmap_atomic to do flushes for 554 * another ASID than the current one. 555 */ 556 map_coherent = (cpu_has_dc_aliases && 557 page_mapped(page) && !Page_dcache_dirty(page)); 558 if (map_coherent) 559 vaddr = kmap_coherent(page, addr); 560 else 561 vaddr = kmap_atomic(page); 562 addr = (unsigned long)vaddr; 563 } 564 565 if (cpu_has_dc_aliases || (exec && !cpu_has_ic_fills_f_dc)) { 566 vaddr ? r4k_blast_dcache_page(addr) : 567 r4k_blast_dcache_user_page(addr); 568 if (exec && !cpu_icache_snoops_remote_store) 569 r4k_blast_scache_page(addr); 570 } 571 if (exec) { 572 if (vaddr && cpu_has_vtag_icache && mm == current->active_mm) { 573 int cpu = smp_processor_id(); 574 575 if (cpu_context(cpu, mm) != 0) 576 drop_mmu_context(mm, cpu); 577 } else 578 vaddr ? r4k_blast_icache_page(addr) : 579 r4k_blast_icache_user_page(addr); 580 } 581 582 if (vaddr) { 583 if (map_coherent) 584 kunmap_coherent(); 585 else 586 kunmap_atomic(vaddr); 587 } 588 } 589 590 static void r4k_flush_cache_page(struct vm_area_struct *vma, 591 unsigned long addr, unsigned long pfn) 592 { 593 struct flush_cache_page_args args; 594 595 args.vma = vma; 596 args.addr = addr; 597 args.pfn = pfn; 598 599 r4k_on_each_cpu(local_r4k_flush_cache_page, &args); 600 } 601 602 static inline void local_r4k_flush_data_cache_page(void * addr) 603 { 604 r4k_blast_dcache_page((unsigned long) addr); 605 } 606 607 static void r4k_flush_data_cache_page(unsigned long addr) 608 { 609 if (in_atomic()) 610 local_r4k_flush_data_cache_page((void *)addr); 611 else 612 r4k_on_each_cpu(local_r4k_flush_data_cache_page, (void *) addr); 613 } 614 615 struct flush_icache_range_args { 616 unsigned long start; 617 unsigned long end; 618 }; 619 620 static inline void local_r4k_flush_icache_range(unsigned long start, unsigned long end) 621 { 622 if (!cpu_has_ic_fills_f_dc) { 623 if (end - start >= dcache_size) { 624 r4k_blast_dcache(); 625 } else { 626 R4600_HIT_CACHEOP_WAR_IMPL; 627 protected_blast_dcache_range(start, end); 628 } 629 } 630 631 if (end - start > icache_size) 632 r4k_blast_icache(); 633 else { 634 switch (boot_cpu_type()) { 635 case CPU_LOONGSON2: 636 protected_loongson2_blast_icache_range(start, end); 637 break; 638 639 default: 640 protected_blast_icache_range(start, end); 641 break; 642 } 643 } 644 #ifdef CONFIG_EVA 645 /* 646 * Due to all possible segment mappings, there might cache aliases 647 * caused by the bootloader being in non-EVA mode, and the CPU switching 648 * to EVA during early kernel init. It's best to flush the scache 649 * to avoid having secondary cores fetching stale data and lead to 650 * kernel crashes. 651 */ 652 bc_wback_inv(start, (end - start)); 653 __sync(); 654 #endif 655 } 656 657 static inline void local_r4k_flush_icache_range_ipi(void *args) 658 { 659 struct flush_icache_range_args *fir_args = args; 660 unsigned long start = fir_args->start; 661 unsigned long end = fir_args->end; 662 663 local_r4k_flush_icache_range(start, end); 664 } 665 666 static void r4k_flush_icache_range(unsigned long start, unsigned long end) 667 { 668 struct flush_icache_range_args args; 669 670 args.start = start; 671 args.end = end; 672 673 r4k_on_each_cpu(local_r4k_flush_icache_range_ipi, &args); 674 instruction_hazard(); 675 } 676 677 #if defined(CONFIG_DMA_NONCOHERENT) || defined(CONFIG_DMA_MAYBE_COHERENT) 678 679 static void r4k_dma_cache_wback_inv(unsigned long addr, unsigned long size) 680 { 681 /* Catch bad driver code */ 682 BUG_ON(size == 0); 683 684 preempt_disable(); 685 if (cpu_has_inclusive_pcaches) { 686 if (size >= scache_size) 687 r4k_blast_scache(); 688 else 689 blast_scache_range(addr, addr + size); 690 preempt_enable(); 691 __sync(); 692 return; 693 } 694 695 /* 696 * Either no secondary cache or the available caches don't have the 697 * subset property so we have to flush the primary caches 698 * explicitly 699 */ 700 if (cpu_has_safe_index_cacheops && size >= dcache_size) { 701 r4k_blast_dcache(); 702 } else { 703 R4600_HIT_CACHEOP_WAR_IMPL; 704 blast_dcache_range(addr, addr + size); 705 } 706 preempt_enable(); 707 708 bc_wback_inv(addr, size); 709 __sync(); 710 } 711 712 static void r4k_dma_cache_inv(unsigned long addr, unsigned long size) 713 { 714 /* Catch bad driver code */ 715 BUG_ON(size == 0); 716 717 preempt_disable(); 718 if (cpu_has_inclusive_pcaches) { 719 if (size >= scache_size) 720 r4k_blast_scache(); 721 else { 722 /* 723 * There is no clearly documented alignment requirement 724 * for the cache instruction on MIPS processors and 725 * some processors, among them the RM5200 and RM7000 726 * QED processors will throw an address error for cache 727 * hit ops with insufficient alignment. Solved by 728 * aligning the address to cache line size. 729 */ 730 blast_inv_scache_range(addr, addr + size); 731 } 732 preempt_enable(); 733 __sync(); 734 return; 735 } 736 737 if (cpu_has_safe_index_cacheops && size >= dcache_size) { 738 r4k_blast_dcache(); 739 } else { 740 R4600_HIT_CACHEOP_WAR_IMPL; 741 blast_inv_dcache_range(addr, addr + size); 742 } 743 preempt_enable(); 744 745 bc_inv(addr, size); 746 __sync(); 747 } 748 #endif /* CONFIG_DMA_NONCOHERENT || CONFIG_DMA_MAYBE_COHERENT */ 749 750 /* 751 * While we're protected against bad userland addresses we don't care 752 * very much about what happens in that case. Usually a segmentation 753 * fault will dump the process later on anyway ... 754 */ 755 static void local_r4k_flush_cache_sigtramp(void * arg) 756 { 757 unsigned long ic_lsize = cpu_icache_line_size(); 758 unsigned long dc_lsize = cpu_dcache_line_size(); 759 unsigned long sc_lsize = cpu_scache_line_size(); 760 unsigned long addr = (unsigned long) arg; 761 762 R4600_HIT_CACHEOP_WAR_IMPL; 763 if (dc_lsize) 764 protected_writeback_dcache_line(addr & ~(dc_lsize - 1)); 765 if (!cpu_icache_snoops_remote_store && scache_size) 766 protected_writeback_scache_line(addr & ~(sc_lsize - 1)); 767 if (ic_lsize) 768 protected_flush_icache_line(addr & ~(ic_lsize - 1)); 769 if (MIPS4K_ICACHE_REFILL_WAR) { 770 __asm__ __volatile__ ( 771 ".set push\n\t" 772 ".set noat\n\t" 773 ".set mips3\n\t" 774 #ifdef CONFIG_32BIT 775 "la $at,1f\n\t" 776 #endif 777 #ifdef CONFIG_64BIT 778 "dla $at,1f\n\t" 779 #endif 780 "cache %0,($at)\n\t" 781 "nop; nop; nop\n" 782 "1:\n\t" 783 ".set pop" 784 : 785 : "i" (Hit_Invalidate_I)); 786 } 787 if (MIPS_CACHE_SYNC_WAR) 788 __asm__ __volatile__ ("sync"); 789 } 790 791 static void r4k_flush_cache_sigtramp(unsigned long addr) 792 { 793 r4k_on_each_cpu(local_r4k_flush_cache_sigtramp, (void *) addr); 794 } 795 796 static void r4k_flush_icache_all(void) 797 { 798 if (cpu_has_vtag_icache) 799 r4k_blast_icache(); 800 } 801 802 struct flush_kernel_vmap_range_args { 803 unsigned long vaddr; 804 int size; 805 }; 806 807 static inline void local_r4k_flush_kernel_vmap_range(void *args) 808 { 809 struct flush_kernel_vmap_range_args *vmra = args; 810 unsigned long vaddr = vmra->vaddr; 811 int size = vmra->size; 812 813 /* 814 * Aliases only affect the primary caches so don't bother with 815 * S-caches or T-caches. 816 */ 817 if (cpu_has_safe_index_cacheops && size >= dcache_size) 818 r4k_blast_dcache(); 819 else { 820 R4600_HIT_CACHEOP_WAR_IMPL; 821 blast_dcache_range(vaddr, vaddr + size); 822 } 823 } 824 825 static void r4k_flush_kernel_vmap_range(unsigned long vaddr, int size) 826 { 827 struct flush_kernel_vmap_range_args args; 828 829 args.vaddr = (unsigned long) vaddr; 830 args.size = size; 831 832 r4k_on_each_cpu(local_r4k_flush_kernel_vmap_range, &args); 833 } 834 835 static inline void rm7k_erratum31(void) 836 { 837 const unsigned long ic_lsize = 32; 838 unsigned long addr; 839 840 /* RM7000 erratum #31. The icache is screwed at startup. */ 841 write_c0_taglo(0); 842 write_c0_taghi(0); 843 844 for (addr = INDEX_BASE; addr <= INDEX_BASE + 4096; addr += ic_lsize) { 845 __asm__ __volatile__ ( 846 ".set push\n\t" 847 ".set noreorder\n\t" 848 ".set mips3\n\t" 849 "cache\t%1, 0(%0)\n\t" 850 "cache\t%1, 0x1000(%0)\n\t" 851 "cache\t%1, 0x2000(%0)\n\t" 852 "cache\t%1, 0x3000(%0)\n\t" 853 "cache\t%2, 0(%0)\n\t" 854 "cache\t%2, 0x1000(%0)\n\t" 855 "cache\t%2, 0x2000(%0)\n\t" 856 "cache\t%2, 0x3000(%0)\n\t" 857 "cache\t%1, 0(%0)\n\t" 858 "cache\t%1, 0x1000(%0)\n\t" 859 "cache\t%1, 0x2000(%0)\n\t" 860 "cache\t%1, 0x3000(%0)\n\t" 861 ".set pop\n" 862 : 863 : "r" (addr), "i" (Index_Store_Tag_I), "i" (Fill)); 864 } 865 } 866 867 static inline void alias_74k_erratum(struct cpuinfo_mips *c) 868 { 869 unsigned int imp = c->processor_id & PRID_IMP_MASK; 870 unsigned int rev = c->processor_id & PRID_REV_MASK; 871 872 /* 873 * Early versions of the 74K do not update the cache tags on a 874 * vtag miss/ptag hit which can occur in the case of KSEG0/KUSEG 875 * aliases. In this case it is better to treat the cache as always 876 * having aliases. 877 */ 878 switch (imp) { 879 case PRID_IMP_74K: 880 if (rev <= PRID_REV_ENCODE_332(2, 4, 0)) 881 c->dcache.flags |= MIPS_CACHE_VTAG; 882 if (rev == PRID_REV_ENCODE_332(2, 4, 0)) 883 write_c0_config6(read_c0_config6() | MIPS_CONF6_SYND); 884 break; 885 case PRID_IMP_1074K: 886 if (rev <= PRID_REV_ENCODE_332(1, 1, 0)) { 887 c->dcache.flags |= MIPS_CACHE_VTAG; 888 write_c0_config6(read_c0_config6() | MIPS_CONF6_SYND); 889 } 890 break; 891 default: 892 BUG(); 893 } 894 } 895 896 static char *way_string[] = { NULL, "direct mapped", "2-way", 897 "3-way", "4-way", "5-way", "6-way", "7-way", "8-way" 898 }; 899 900 static void probe_pcache(void) 901 { 902 struct cpuinfo_mips *c = ¤t_cpu_data; 903 unsigned int config = read_c0_config(); 904 unsigned int prid = read_c0_prid(); 905 unsigned long config1; 906 unsigned int lsize; 907 908 switch (current_cpu_type()) { 909 case CPU_R4600: /* QED style two way caches? */ 910 case CPU_R4700: 911 case CPU_R5000: 912 case CPU_NEVADA: 913 icache_size = 1 << (12 + ((config & CONF_IC) >> 9)); 914 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 915 c->icache.ways = 2; 916 c->icache.waybit = __ffs(icache_size/2); 917 918 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6)); 919 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 920 c->dcache.ways = 2; 921 c->dcache.waybit= __ffs(dcache_size/2); 922 923 c->options |= MIPS_CPU_CACHE_CDEX_P; 924 break; 925 926 case CPU_R5432: 927 case CPU_R5500: 928 icache_size = 1 << (12 + ((config & CONF_IC) >> 9)); 929 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 930 c->icache.ways = 2; 931 c->icache.waybit= 0; 932 933 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6)); 934 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 935 c->dcache.ways = 2; 936 c->dcache.waybit = 0; 937 938 c->options |= MIPS_CPU_CACHE_CDEX_P | MIPS_CPU_PREFETCH; 939 break; 940 941 case CPU_TX49XX: 942 icache_size = 1 << (12 + ((config & CONF_IC) >> 9)); 943 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 944 c->icache.ways = 4; 945 c->icache.waybit= 0; 946 947 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6)); 948 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 949 c->dcache.ways = 4; 950 c->dcache.waybit = 0; 951 952 c->options |= MIPS_CPU_CACHE_CDEX_P; 953 c->options |= MIPS_CPU_PREFETCH; 954 break; 955 956 case CPU_R4000PC: 957 case CPU_R4000SC: 958 case CPU_R4000MC: 959 case CPU_R4400PC: 960 case CPU_R4400SC: 961 case CPU_R4400MC: 962 case CPU_R4300: 963 icache_size = 1 << (12 + ((config & CONF_IC) >> 9)); 964 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 965 c->icache.ways = 1; 966 c->icache.waybit = 0; /* doesn't matter */ 967 968 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6)); 969 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 970 c->dcache.ways = 1; 971 c->dcache.waybit = 0; /* does not matter */ 972 973 c->options |= MIPS_CPU_CACHE_CDEX_P; 974 break; 975 976 case CPU_R10000: 977 case CPU_R12000: 978 case CPU_R14000: 979 icache_size = 1 << (12 + ((config & R10K_CONF_IC) >> 29)); 980 c->icache.linesz = 64; 981 c->icache.ways = 2; 982 c->icache.waybit = 0; 983 984 dcache_size = 1 << (12 + ((config & R10K_CONF_DC) >> 26)); 985 c->dcache.linesz = 32; 986 c->dcache.ways = 2; 987 c->dcache.waybit = 0; 988 989 c->options |= MIPS_CPU_PREFETCH; 990 break; 991 992 case CPU_VR4133: 993 write_c0_config(config & ~VR41_CONF_P4K); 994 case CPU_VR4131: 995 /* Workaround for cache instruction bug of VR4131 */ 996 if (c->processor_id == 0x0c80U || c->processor_id == 0x0c81U || 997 c->processor_id == 0x0c82U) { 998 config |= 0x00400000U; 999 if (c->processor_id == 0x0c80U) 1000 config |= VR41_CONF_BP; 1001 write_c0_config(config); 1002 } else 1003 c->options |= MIPS_CPU_CACHE_CDEX_P; 1004 1005 icache_size = 1 << (10 + ((config & CONF_IC) >> 9)); 1006 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 1007 c->icache.ways = 2; 1008 c->icache.waybit = __ffs(icache_size/2); 1009 1010 dcache_size = 1 << (10 + ((config & CONF_DC) >> 6)); 1011 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 1012 c->dcache.ways = 2; 1013 c->dcache.waybit = __ffs(dcache_size/2); 1014 break; 1015 1016 case CPU_VR41XX: 1017 case CPU_VR4111: 1018 case CPU_VR4121: 1019 case CPU_VR4122: 1020 case CPU_VR4181: 1021 case CPU_VR4181A: 1022 icache_size = 1 << (10 + ((config & CONF_IC) >> 9)); 1023 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 1024 c->icache.ways = 1; 1025 c->icache.waybit = 0; /* doesn't matter */ 1026 1027 dcache_size = 1 << (10 + ((config & CONF_DC) >> 6)); 1028 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 1029 c->dcache.ways = 1; 1030 c->dcache.waybit = 0; /* does not matter */ 1031 1032 c->options |= MIPS_CPU_CACHE_CDEX_P; 1033 break; 1034 1035 case CPU_RM7000: 1036 rm7k_erratum31(); 1037 1038 icache_size = 1 << (12 + ((config & CONF_IC) >> 9)); 1039 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 1040 c->icache.ways = 4; 1041 c->icache.waybit = __ffs(icache_size / c->icache.ways); 1042 1043 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6)); 1044 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 1045 c->dcache.ways = 4; 1046 c->dcache.waybit = __ffs(dcache_size / c->dcache.ways); 1047 1048 c->options |= MIPS_CPU_CACHE_CDEX_P; 1049 c->options |= MIPS_CPU_PREFETCH; 1050 break; 1051 1052 case CPU_LOONGSON2: 1053 icache_size = 1 << (12 + ((config & CONF_IC) >> 9)); 1054 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 1055 if (prid & 0x3) 1056 c->icache.ways = 4; 1057 else 1058 c->icache.ways = 2; 1059 c->icache.waybit = 0; 1060 1061 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6)); 1062 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 1063 if (prid & 0x3) 1064 c->dcache.ways = 4; 1065 else 1066 c->dcache.ways = 2; 1067 c->dcache.waybit = 0; 1068 break; 1069 1070 case CPU_LOONGSON3: 1071 config1 = read_c0_config1(); 1072 lsize = (config1 >> 19) & 7; 1073 if (lsize) 1074 c->icache.linesz = 2 << lsize; 1075 else 1076 c->icache.linesz = 0; 1077 c->icache.sets = 64 << ((config1 >> 22) & 7); 1078 c->icache.ways = 1 + ((config1 >> 16) & 7); 1079 icache_size = c->icache.sets * 1080 c->icache.ways * 1081 c->icache.linesz; 1082 c->icache.waybit = 0; 1083 1084 lsize = (config1 >> 10) & 7; 1085 if (lsize) 1086 c->dcache.linesz = 2 << lsize; 1087 else 1088 c->dcache.linesz = 0; 1089 c->dcache.sets = 64 << ((config1 >> 13) & 7); 1090 c->dcache.ways = 1 + ((config1 >> 7) & 7); 1091 dcache_size = c->dcache.sets * 1092 c->dcache.ways * 1093 c->dcache.linesz; 1094 c->dcache.waybit = 0; 1095 break; 1096 1097 default: 1098 if (!(config & MIPS_CONF_M)) 1099 panic("Don't know how to probe P-caches on this cpu."); 1100 1101 /* 1102 * So we seem to be a MIPS32 or MIPS64 CPU 1103 * So let's probe the I-cache ... 1104 */ 1105 config1 = read_c0_config1(); 1106 1107 lsize = (config1 >> 19) & 7; 1108 1109 /* IL == 7 is reserved */ 1110 if (lsize == 7) 1111 panic("Invalid icache line size"); 1112 1113 c->icache.linesz = lsize ? 2 << lsize : 0; 1114 1115 c->icache.sets = 32 << (((config1 >> 22) + 1) & 7); 1116 c->icache.ways = 1 + ((config1 >> 16) & 7); 1117 1118 icache_size = c->icache.sets * 1119 c->icache.ways * 1120 c->icache.linesz; 1121 c->icache.waybit = __ffs(icache_size/c->icache.ways); 1122 1123 if (config & 0x8) /* VI bit */ 1124 c->icache.flags |= MIPS_CACHE_VTAG; 1125 1126 /* 1127 * Now probe the MIPS32 / MIPS64 data cache. 1128 */ 1129 c->dcache.flags = 0; 1130 1131 lsize = (config1 >> 10) & 7; 1132 1133 /* DL == 7 is reserved */ 1134 if (lsize == 7) 1135 panic("Invalid dcache line size"); 1136 1137 c->dcache.linesz = lsize ? 2 << lsize : 0; 1138 1139 c->dcache.sets = 32 << (((config1 >> 13) + 1) & 7); 1140 c->dcache.ways = 1 + ((config1 >> 7) & 7); 1141 1142 dcache_size = c->dcache.sets * 1143 c->dcache.ways * 1144 c->dcache.linesz; 1145 c->dcache.waybit = __ffs(dcache_size/c->dcache.ways); 1146 1147 c->options |= MIPS_CPU_PREFETCH; 1148 break; 1149 } 1150 1151 /* 1152 * Processor configuration sanity check for the R4000SC erratum 1153 * #5. With page sizes larger than 32kB there is no possibility 1154 * to get a VCE exception anymore so we don't care about this 1155 * misconfiguration. The case is rather theoretical anyway; 1156 * presumably no vendor is shipping his hardware in the "bad" 1157 * configuration. 1158 */ 1159 if ((prid & PRID_IMP_MASK) == PRID_IMP_R4000 && 1160 (prid & PRID_REV_MASK) < PRID_REV_R4400 && 1161 !(config & CONF_SC) && c->icache.linesz != 16 && 1162 PAGE_SIZE <= 0x8000) 1163 panic("Improper R4000SC processor configuration detected"); 1164 1165 /* compute a couple of other cache variables */ 1166 c->icache.waysize = icache_size / c->icache.ways; 1167 c->dcache.waysize = dcache_size / c->dcache.ways; 1168 1169 c->icache.sets = c->icache.linesz ? 1170 icache_size / (c->icache.linesz * c->icache.ways) : 0; 1171 c->dcache.sets = c->dcache.linesz ? 1172 dcache_size / (c->dcache.linesz * c->dcache.ways) : 0; 1173 1174 /* 1175 * R10000 and R12000 P-caches are odd in a positive way. They're 32kB 1176 * 2-way virtually indexed so normally would suffer from aliases. So 1177 * normally they'd suffer from aliases but magic in the hardware deals 1178 * with that for us so we don't need to take care ourselves. 1179 */ 1180 switch (current_cpu_type()) { 1181 case CPU_20KC: 1182 case CPU_25KF: 1183 case CPU_SB1: 1184 case CPU_SB1A: 1185 case CPU_XLR: 1186 c->dcache.flags |= MIPS_CACHE_PINDEX; 1187 break; 1188 1189 case CPU_R10000: 1190 case CPU_R12000: 1191 case CPU_R14000: 1192 break; 1193 1194 case CPU_M14KC: 1195 case CPU_M14KEC: 1196 case CPU_24K: 1197 case CPU_34K: 1198 case CPU_74K: 1199 case CPU_1004K: 1200 case CPU_1074K: 1201 case CPU_INTERAPTIV: 1202 case CPU_P5600: 1203 case CPU_PROAPTIV: 1204 case CPU_M5150: 1205 if ((c->cputype == CPU_74K) || (c->cputype == CPU_1074K)) 1206 alias_74k_erratum(c); 1207 if (!(read_c0_config7() & MIPS_CONF7_IAR) && 1208 (c->icache.waysize > PAGE_SIZE)) 1209 c->icache.flags |= MIPS_CACHE_ALIASES; 1210 if (read_c0_config7() & MIPS_CONF7_AR) { 1211 /* 1212 * Effectively physically indexed dcache, 1213 * thus no virtual aliases. 1214 */ 1215 c->dcache.flags |= MIPS_CACHE_PINDEX; 1216 break; 1217 } 1218 default: 1219 if (c->dcache.waysize > PAGE_SIZE) 1220 c->dcache.flags |= MIPS_CACHE_ALIASES; 1221 } 1222 1223 switch (current_cpu_type()) { 1224 case CPU_20KC: 1225 /* 1226 * Some older 20Kc chips doesn't have the 'VI' bit in 1227 * the config register. 1228 */ 1229 c->icache.flags |= MIPS_CACHE_VTAG; 1230 break; 1231 1232 case CPU_ALCHEMY: 1233 c->icache.flags |= MIPS_CACHE_IC_F_DC; 1234 break; 1235 1236 case CPU_LOONGSON2: 1237 /* 1238 * LOONGSON2 has 4 way icache, but when using indexed cache op, 1239 * one op will act on all 4 ways 1240 */ 1241 c->icache.ways = 1; 1242 } 1243 1244 printk("Primary instruction cache %ldkB, %s, %s, linesize %d bytes.\n", 1245 icache_size >> 10, 1246 c->icache.flags & MIPS_CACHE_VTAG ? "VIVT" : "VIPT", 1247 way_string[c->icache.ways], c->icache.linesz); 1248 1249 printk("Primary data cache %ldkB, %s, %s, %s, linesize %d bytes\n", 1250 dcache_size >> 10, way_string[c->dcache.ways], 1251 (c->dcache.flags & MIPS_CACHE_PINDEX) ? "PIPT" : "VIPT", 1252 (c->dcache.flags & MIPS_CACHE_ALIASES) ? 1253 "cache aliases" : "no aliases", 1254 c->dcache.linesz); 1255 } 1256 1257 /* 1258 * If you even _breathe_ on this function, look at the gcc output and make sure 1259 * it does not pop things on and off the stack for the cache sizing loop that 1260 * executes in KSEG1 space or else you will crash and burn badly. You have 1261 * been warned. 1262 */ 1263 static int probe_scache(void) 1264 { 1265 unsigned long flags, addr, begin, end, pow2; 1266 unsigned int config = read_c0_config(); 1267 struct cpuinfo_mips *c = ¤t_cpu_data; 1268 1269 if (config & CONF_SC) 1270 return 0; 1271 1272 begin = (unsigned long) &_stext; 1273 begin &= ~((4 * 1024 * 1024) - 1); 1274 end = begin + (4 * 1024 * 1024); 1275 1276 /* 1277 * This is such a bitch, you'd think they would make it easy to do 1278 * this. Away you daemons of stupidity! 1279 */ 1280 local_irq_save(flags); 1281 1282 /* Fill each size-multiple cache line with a valid tag. */ 1283 pow2 = (64 * 1024); 1284 for (addr = begin; addr < end; addr = (begin + pow2)) { 1285 unsigned long *p = (unsigned long *) addr; 1286 __asm__ __volatile__("nop" : : "r" (*p)); /* whee... */ 1287 pow2 <<= 1; 1288 } 1289 1290 /* Load first line with zero (therefore invalid) tag. */ 1291 write_c0_taglo(0); 1292 write_c0_taghi(0); 1293 __asm__ __volatile__("nop; nop; nop; nop;"); /* avoid the hazard */ 1294 cache_op(Index_Store_Tag_I, begin); 1295 cache_op(Index_Store_Tag_D, begin); 1296 cache_op(Index_Store_Tag_SD, begin); 1297 1298 /* Now search for the wrap around point. */ 1299 pow2 = (128 * 1024); 1300 for (addr = begin + (128 * 1024); addr < end; addr = begin + pow2) { 1301 cache_op(Index_Load_Tag_SD, addr); 1302 __asm__ __volatile__("nop; nop; nop; nop;"); /* hazard... */ 1303 if (!read_c0_taglo()) 1304 break; 1305 pow2 <<= 1; 1306 } 1307 local_irq_restore(flags); 1308 addr -= begin; 1309 1310 scache_size = addr; 1311 c->scache.linesz = 16 << ((config & R4K_CONF_SB) >> 22); 1312 c->scache.ways = 1; 1313 c->dcache.waybit = 0; /* does not matter */ 1314 1315 return 1; 1316 } 1317 1318 static void __init loongson2_sc_init(void) 1319 { 1320 struct cpuinfo_mips *c = ¤t_cpu_data; 1321 1322 scache_size = 512*1024; 1323 c->scache.linesz = 32; 1324 c->scache.ways = 4; 1325 c->scache.waybit = 0; 1326 c->scache.waysize = scache_size / (c->scache.ways); 1327 c->scache.sets = scache_size / (c->scache.linesz * c->scache.ways); 1328 pr_info("Unified secondary cache %ldkB %s, linesize %d bytes.\n", 1329 scache_size >> 10, way_string[c->scache.ways], c->scache.linesz); 1330 1331 c->options |= MIPS_CPU_INCLUSIVE_CACHES; 1332 } 1333 1334 static void __init loongson3_sc_init(void) 1335 { 1336 struct cpuinfo_mips *c = ¤t_cpu_data; 1337 unsigned int config2, lsize; 1338 1339 config2 = read_c0_config2(); 1340 lsize = (config2 >> 4) & 15; 1341 if (lsize) 1342 c->scache.linesz = 2 << lsize; 1343 else 1344 c->scache.linesz = 0; 1345 c->scache.sets = 64 << ((config2 >> 8) & 15); 1346 c->scache.ways = 1 + (config2 & 15); 1347 1348 scache_size = c->scache.sets * 1349 c->scache.ways * 1350 c->scache.linesz; 1351 /* Loongson-3 has 4 cores, 1MB scache for each. scaches are shared */ 1352 scache_size *= 4; 1353 c->scache.waybit = 0; 1354 pr_info("Unified secondary cache %ldkB %s, linesize %d bytes.\n", 1355 scache_size >> 10, way_string[c->scache.ways], c->scache.linesz); 1356 if (scache_size) 1357 c->options |= MIPS_CPU_INCLUSIVE_CACHES; 1358 return; 1359 } 1360 1361 extern int r5k_sc_init(void); 1362 extern int rm7k_sc_init(void); 1363 extern int mips_sc_init(void); 1364 1365 static void setup_scache(void) 1366 { 1367 struct cpuinfo_mips *c = ¤t_cpu_data; 1368 unsigned int config = read_c0_config(); 1369 int sc_present = 0; 1370 1371 /* 1372 * Do the probing thing on R4000SC and R4400SC processors. Other 1373 * processors don't have a S-cache that would be relevant to the 1374 * Linux memory management. 1375 */ 1376 switch (current_cpu_type()) { 1377 case CPU_R4000SC: 1378 case CPU_R4000MC: 1379 case CPU_R4400SC: 1380 case CPU_R4400MC: 1381 sc_present = run_uncached(probe_scache); 1382 if (sc_present) 1383 c->options |= MIPS_CPU_CACHE_CDEX_S; 1384 break; 1385 1386 case CPU_R10000: 1387 case CPU_R12000: 1388 case CPU_R14000: 1389 scache_size = 0x80000 << ((config & R10K_CONF_SS) >> 16); 1390 c->scache.linesz = 64 << ((config >> 13) & 1); 1391 c->scache.ways = 2; 1392 c->scache.waybit= 0; 1393 sc_present = 1; 1394 break; 1395 1396 case CPU_R5000: 1397 case CPU_NEVADA: 1398 #ifdef CONFIG_R5000_CPU_SCACHE 1399 r5k_sc_init(); 1400 #endif 1401 return; 1402 1403 case CPU_RM7000: 1404 #ifdef CONFIG_RM7000_CPU_SCACHE 1405 rm7k_sc_init(); 1406 #endif 1407 return; 1408 1409 case CPU_LOONGSON2: 1410 loongson2_sc_init(); 1411 return; 1412 1413 case CPU_LOONGSON3: 1414 loongson3_sc_init(); 1415 return; 1416 1417 case CPU_XLP: 1418 /* don't need to worry about L2, fully coherent */ 1419 return; 1420 1421 default: 1422 if (c->isa_level & (MIPS_CPU_ISA_M32R1 | MIPS_CPU_ISA_M32R2 | 1423 MIPS_CPU_ISA_M64R1 | MIPS_CPU_ISA_M64R2)) { 1424 #ifdef CONFIG_MIPS_CPU_SCACHE 1425 if (mips_sc_init ()) { 1426 scache_size = c->scache.ways * c->scache.sets * c->scache.linesz; 1427 printk("MIPS secondary cache %ldkB, %s, linesize %d bytes.\n", 1428 scache_size >> 10, 1429 way_string[c->scache.ways], c->scache.linesz); 1430 } 1431 #else 1432 if (!(c->scache.flags & MIPS_CACHE_NOT_PRESENT)) 1433 panic("Dunno how to handle MIPS32 / MIPS64 second level cache"); 1434 #endif 1435 return; 1436 } 1437 sc_present = 0; 1438 } 1439 1440 if (!sc_present) 1441 return; 1442 1443 /* compute a couple of other cache variables */ 1444 c->scache.waysize = scache_size / c->scache.ways; 1445 1446 c->scache.sets = scache_size / (c->scache.linesz * c->scache.ways); 1447 1448 printk("Unified secondary cache %ldkB %s, linesize %d bytes.\n", 1449 scache_size >> 10, way_string[c->scache.ways], c->scache.linesz); 1450 1451 c->options |= MIPS_CPU_INCLUSIVE_CACHES; 1452 } 1453 1454 void au1x00_fixup_config_od(void) 1455 { 1456 /* 1457 * c0_config.od (bit 19) was write only (and read as 0) 1458 * on the early revisions of Alchemy SOCs. It disables the bus 1459 * transaction overlapping and needs to be set to fix various errata. 1460 */ 1461 switch (read_c0_prid()) { 1462 case 0x00030100: /* Au1000 DA */ 1463 case 0x00030201: /* Au1000 HA */ 1464 case 0x00030202: /* Au1000 HB */ 1465 case 0x01030200: /* Au1500 AB */ 1466 /* 1467 * Au1100 errata actually keeps silence about this bit, so we set it 1468 * just in case for those revisions that require it to be set according 1469 * to the (now gone) cpu table. 1470 */ 1471 case 0x02030200: /* Au1100 AB */ 1472 case 0x02030201: /* Au1100 BA */ 1473 case 0x02030202: /* Au1100 BC */ 1474 set_c0_config(1 << 19); 1475 break; 1476 } 1477 } 1478 1479 /* CP0 hazard avoidance. */ 1480 #define NXP_BARRIER() \ 1481 __asm__ __volatile__( \ 1482 ".set noreorder\n\t" \ 1483 "nop; nop; nop; nop; nop; nop;\n\t" \ 1484 ".set reorder\n\t") 1485 1486 static void nxp_pr4450_fixup_config(void) 1487 { 1488 unsigned long config0; 1489 1490 config0 = read_c0_config(); 1491 1492 /* clear all three cache coherency fields */ 1493 config0 &= ~(0x7 | (7 << 25) | (7 << 28)); 1494 config0 |= (((_page_cachable_default >> _CACHE_SHIFT) << 0) | 1495 ((_page_cachable_default >> _CACHE_SHIFT) << 25) | 1496 ((_page_cachable_default >> _CACHE_SHIFT) << 28)); 1497 write_c0_config(config0); 1498 NXP_BARRIER(); 1499 } 1500 1501 static int cca = -1; 1502 1503 static int __init cca_setup(char *str) 1504 { 1505 get_option(&str, &cca); 1506 1507 return 0; 1508 } 1509 1510 early_param("cca", cca_setup); 1511 1512 static void coherency_setup(void) 1513 { 1514 if (cca < 0 || cca > 7) 1515 cca = read_c0_config() & CONF_CM_CMASK; 1516 _page_cachable_default = cca << _CACHE_SHIFT; 1517 1518 pr_debug("Using cache attribute %d\n", cca); 1519 change_c0_config(CONF_CM_CMASK, cca); 1520 1521 /* 1522 * c0_status.cu=0 specifies that updates by the sc instruction use 1523 * the coherency mode specified by the TLB; 1 means cachable 1524 * coherent update on write will be used. Not all processors have 1525 * this bit and; some wire it to zero, others like Toshiba had the 1526 * silly idea of putting something else there ... 1527 */ 1528 switch (current_cpu_type()) { 1529 case CPU_R4000PC: 1530 case CPU_R4000SC: 1531 case CPU_R4000MC: 1532 case CPU_R4400PC: 1533 case CPU_R4400SC: 1534 case CPU_R4400MC: 1535 clear_c0_config(CONF_CU); 1536 break; 1537 /* 1538 * We need to catch the early Alchemy SOCs with 1539 * the write-only co_config.od bit and set it back to one on: 1540 * Au1000 rev DA, HA, HB; Au1100 AB, BA, BC, Au1500 AB 1541 */ 1542 case CPU_ALCHEMY: 1543 au1x00_fixup_config_od(); 1544 break; 1545 1546 case PRID_IMP_PR4450: 1547 nxp_pr4450_fixup_config(); 1548 break; 1549 } 1550 } 1551 1552 static void r4k_cache_error_setup(void) 1553 { 1554 extern char __weak except_vec2_generic; 1555 extern char __weak except_vec2_sb1; 1556 1557 switch (current_cpu_type()) { 1558 case CPU_SB1: 1559 case CPU_SB1A: 1560 set_uncached_handler(0x100, &except_vec2_sb1, 0x80); 1561 break; 1562 1563 default: 1564 set_uncached_handler(0x100, &except_vec2_generic, 0x80); 1565 break; 1566 } 1567 } 1568 1569 void r4k_cache_init(void) 1570 { 1571 extern void build_clear_page(void); 1572 extern void build_copy_page(void); 1573 struct cpuinfo_mips *c = ¤t_cpu_data; 1574 1575 probe_pcache(); 1576 setup_scache(); 1577 1578 r4k_blast_dcache_page_setup(); 1579 r4k_blast_dcache_page_indexed_setup(); 1580 r4k_blast_dcache_setup(); 1581 r4k_blast_icache_page_setup(); 1582 r4k_blast_icache_page_indexed_setup(); 1583 r4k_blast_icache_setup(); 1584 r4k_blast_scache_page_setup(); 1585 r4k_blast_scache_page_indexed_setup(); 1586 r4k_blast_scache_setup(); 1587 #ifdef CONFIG_EVA 1588 r4k_blast_dcache_user_page_setup(); 1589 r4k_blast_icache_user_page_setup(); 1590 #endif 1591 1592 /* 1593 * Some MIPS32 and MIPS64 processors have physically indexed caches. 1594 * This code supports virtually indexed processors and will be 1595 * unnecessarily inefficient on physically indexed processors. 1596 */ 1597 if (c->dcache.linesz) 1598 shm_align_mask = max_t( unsigned long, 1599 c->dcache.sets * c->dcache.linesz - 1, 1600 PAGE_SIZE - 1); 1601 else 1602 shm_align_mask = PAGE_SIZE-1; 1603 1604 __flush_cache_vmap = r4k__flush_cache_vmap; 1605 __flush_cache_vunmap = r4k__flush_cache_vunmap; 1606 1607 flush_cache_all = cache_noop; 1608 __flush_cache_all = r4k___flush_cache_all; 1609 flush_cache_mm = r4k_flush_cache_mm; 1610 flush_cache_page = r4k_flush_cache_page; 1611 flush_cache_range = r4k_flush_cache_range; 1612 1613 __flush_kernel_vmap_range = r4k_flush_kernel_vmap_range; 1614 1615 flush_cache_sigtramp = r4k_flush_cache_sigtramp; 1616 flush_icache_all = r4k_flush_icache_all; 1617 local_flush_data_cache_page = local_r4k_flush_data_cache_page; 1618 flush_data_cache_page = r4k_flush_data_cache_page; 1619 flush_icache_range = r4k_flush_icache_range; 1620 local_flush_icache_range = local_r4k_flush_icache_range; 1621 1622 #if defined(CONFIG_DMA_NONCOHERENT) || defined(CONFIG_DMA_MAYBE_COHERENT) 1623 if (coherentio) { 1624 _dma_cache_wback_inv = (void *)cache_noop; 1625 _dma_cache_wback = (void *)cache_noop; 1626 _dma_cache_inv = (void *)cache_noop; 1627 } else { 1628 _dma_cache_wback_inv = r4k_dma_cache_wback_inv; 1629 _dma_cache_wback = r4k_dma_cache_wback_inv; 1630 _dma_cache_inv = r4k_dma_cache_inv; 1631 } 1632 #endif 1633 1634 build_clear_page(); 1635 build_copy_page(); 1636 1637 /* 1638 * We want to run CMP kernels on core with and without coherent 1639 * caches. Therefore, do not use CONFIG_MIPS_CMP to decide whether 1640 * or not to flush caches. 1641 */ 1642 local_r4k___flush_cache_all(NULL); 1643 1644 coherency_setup(); 1645 board_cache_error_setup = r4k_cache_error_setup; 1646 } 1647