1 /* $Id: cache.c,v 1.4 2000/01/25 00:11:38 prumpf Exp $ 2 * 3 * This file is subject to the terms and conditions of the GNU General Public 4 * License. See the file "COPYING" in the main directory of this archive 5 * for more details. 6 * 7 * Copyright (C) 1999-2006 Helge Deller <deller@gmx.de> (07-13-1999) 8 * Copyright (C) 1999 SuSE GmbH Nuernberg 9 * Copyright (C) 2000 Philipp Rumpf (prumpf@tux.org) 10 * 11 * Cache and TLB management 12 * 13 */ 14 15 #include <linux/init.h> 16 #include <linux/kernel.h> 17 #include <linux/mm.h> 18 #include <linux/module.h> 19 #include <linux/seq_file.h> 20 #include <linux/pagemap.h> 21 #include <linux/sched.h> 22 #include <asm/pdc.h> 23 #include <asm/cache.h> 24 #include <asm/cacheflush.h> 25 #include <asm/tlbflush.h> 26 #include <asm/system.h> 27 #include <asm/page.h> 28 #include <asm/pgalloc.h> 29 #include <asm/processor.h> 30 #include <asm/sections.h> 31 32 int split_tlb __read_mostly; 33 int dcache_stride __read_mostly; 34 int icache_stride __read_mostly; 35 EXPORT_SYMBOL(dcache_stride); 36 37 38 /* On some machines (e.g. ones with the Merced bus), there can be 39 * only a single PxTLB broadcast at a time; this must be guaranteed 40 * by software. We put a spinlock around all TLB flushes to 41 * ensure this. 42 */ 43 DEFINE_SPINLOCK(pa_tlb_lock); 44 45 struct pdc_cache_info cache_info __read_mostly; 46 #ifndef CONFIG_PA20 47 static struct pdc_btlb_info btlb_info __read_mostly; 48 #endif 49 50 #ifdef CONFIG_SMP 51 void 52 flush_data_cache(void) 53 { 54 on_each_cpu(flush_data_cache_local, NULL, 1, 1); 55 } 56 void 57 flush_instruction_cache(void) 58 { 59 on_each_cpu(flush_instruction_cache_local, NULL, 1, 1); 60 } 61 #endif 62 63 void 64 flush_cache_all_local(void) 65 { 66 flush_instruction_cache_local(NULL); 67 flush_data_cache_local(NULL); 68 } 69 EXPORT_SYMBOL(flush_cache_all_local); 70 71 void 72 update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t pte) 73 { 74 struct page *page = pte_page(pte); 75 76 if (pfn_valid(page_to_pfn(page)) && page_mapping(page) && 77 test_bit(PG_dcache_dirty, &page->flags)) { 78 79 flush_kernel_dcache_page(page); 80 clear_bit(PG_dcache_dirty, &page->flags); 81 } else if (parisc_requires_coherency()) 82 flush_kernel_dcache_page(page); 83 } 84 85 void 86 show_cache_info(struct seq_file *m) 87 { 88 char buf[32]; 89 90 seq_printf(m, "I-cache\t\t: %ld KB\n", 91 cache_info.ic_size/1024 ); 92 if (cache_info.dc_loop != 1) 93 snprintf(buf, 32, "%lu-way associative", cache_info.dc_loop); 94 seq_printf(m, "D-cache\t\t: %ld KB (%s%s, %s)\n", 95 cache_info.dc_size/1024, 96 (cache_info.dc_conf.cc_wt ? "WT":"WB"), 97 (cache_info.dc_conf.cc_sh ? ", shared I/D":""), 98 ((cache_info.dc_loop == 1) ? "direct mapped" : buf)); 99 seq_printf(m, "ITLB entries\t: %ld\n" "DTLB entries\t: %ld%s\n", 100 cache_info.it_size, 101 cache_info.dt_size, 102 cache_info.dt_conf.tc_sh ? " - shared with ITLB":"" 103 ); 104 105 #ifndef CONFIG_PA20 106 /* BTLB - Block TLB */ 107 if (btlb_info.max_size==0) { 108 seq_printf(m, "BTLB\t\t: not supported\n" ); 109 } else { 110 seq_printf(m, 111 "BTLB fixed\t: max. %d pages, pagesize=%d (%dMB)\n" 112 "BTLB fix-entr.\t: %d instruction, %d data (%d combined)\n" 113 "BTLB var-entr.\t: %d instruction, %d data (%d combined)\n", 114 btlb_info.max_size, (int)4096, 115 btlb_info.max_size>>8, 116 btlb_info.fixed_range_info.num_i, 117 btlb_info.fixed_range_info.num_d, 118 btlb_info.fixed_range_info.num_comb, 119 btlb_info.variable_range_info.num_i, 120 btlb_info.variable_range_info.num_d, 121 btlb_info.variable_range_info.num_comb 122 ); 123 } 124 #endif 125 } 126 127 void __init 128 parisc_cache_init(void) 129 { 130 if (pdc_cache_info(&cache_info) < 0) 131 panic("parisc_cache_init: pdc_cache_info failed"); 132 133 #if 0 134 printk("ic_size %lx dc_size %lx it_size %lx\n", 135 cache_info.ic_size, 136 cache_info.dc_size, 137 cache_info.it_size); 138 139 printk("DC base 0x%lx stride 0x%lx count 0x%lx loop 0x%lx\n", 140 cache_info.dc_base, 141 cache_info.dc_stride, 142 cache_info.dc_count, 143 cache_info.dc_loop); 144 145 printk("dc_conf = 0x%lx alias %d blk %d line %d shift %d\n", 146 *(unsigned long *) (&cache_info.dc_conf), 147 cache_info.dc_conf.cc_alias, 148 cache_info.dc_conf.cc_block, 149 cache_info.dc_conf.cc_line, 150 cache_info.dc_conf.cc_shift); 151 printk(" wt %d sh %d cst %d hv %d\n", 152 cache_info.dc_conf.cc_wt, 153 cache_info.dc_conf.cc_sh, 154 cache_info.dc_conf.cc_cst, 155 cache_info.dc_conf.cc_hv); 156 157 printk("IC base 0x%lx stride 0x%lx count 0x%lx loop 0x%lx\n", 158 cache_info.ic_base, 159 cache_info.ic_stride, 160 cache_info.ic_count, 161 cache_info.ic_loop); 162 163 printk("ic_conf = 0x%lx alias %d blk %d line %d shift %d\n", 164 *(unsigned long *) (&cache_info.ic_conf), 165 cache_info.ic_conf.cc_alias, 166 cache_info.ic_conf.cc_block, 167 cache_info.ic_conf.cc_line, 168 cache_info.ic_conf.cc_shift); 169 printk(" wt %d sh %d cst %d hv %d\n", 170 cache_info.ic_conf.cc_wt, 171 cache_info.ic_conf.cc_sh, 172 cache_info.ic_conf.cc_cst, 173 cache_info.ic_conf.cc_hv); 174 175 printk("D-TLB conf: sh %d page %d cst %d aid %d pad1 %d \n", 176 cache_info.dt_conf.tc_sh, 177 cache_info.dt_conf.tc_page, 178 cache_info.dt_conf.tc_cst, 179 cache_info.dt_conf.tc_aid, 180 cache_info.dt_conf.tc_pad1); 181 182 printk("I-TLB conf: sh %d page %d cst %d aid %d pad1 %d \n", 183 cache_info.it_conf.tc_sh, 184 cache_info.it_conf.tc_page, 185 cache_info.it_conf.tc_cst, 186 cache_info.it_conf.tc_aid, 187 cache_info.it_conf.tc_pad1); 188 #endif 189 190 split_tlb = 0; 191 if (cache_info.dt_conf.tc_sh == 0 || cache_info.dt_conf.tc_sh == 2) { 192 if (cache_info.dt_conf.tc_sh == 2) 193 printk(KERN_WARNING "Unexpected TLB configuration. " 194 "Will flush I/D separately (could be optimized).\n"); 195 196 split_tlb = 1; 197 } 198 199 /* "New and Improved" version from Jim Hull 200 * (1 << (cc_block-1)) * (cc_line << (4 + cnf.cc_shift)) 201 * The following CAFL_STRIDE is an optimized version, see 202 * http://lists.parisc-linux.org/pipermail/parisc-linux/2004-June/023625.html 203 * http://lists.parisc-linux.org/pipermail/parisc-linux/2004-June/023671.html 204 */ 205 #define CAFL_STRIDE(cnf) (cnf.cc_line << (3 + cnf.cc_block + cnf.cc_shift)) 206 dcache_stride = CAFL_STRIDE(cache_info.dc_conf); 207 icache_stride = CAFL_STRIDE(cache_info.ic_conf); 208 #undef CAFL_STRIDE 209 210 #ifndef CONFIG_PA20 211 if (pdc_btlb_info(&btlb_info) < 0) { 212 memset(&btlb_info, 0, sizeof btlb_info); 213 } 214 #endif 215 216 if ((boot_cpu_data.pdc.capabilities & PDC_MODEL_NVA_MASK) == 217 PDC_MODEL_NVA_UNSUPPORTED) { 218 printk(KERN_WARNING "parisc_cache_init: Only equivalent aliasing supported!\n"); 219 #if 0 220 panic("SMP kernel required to avoid non-equivalent aliasing"); 221 #endif 222 } 223 } 224 225 void disable_sr_hashing(void) 226 { 227 int srhash_type, retval; 228 unsigned long space_bits; 229 230 switch (boot_cpu_data.cpu_type) { 231 case pcx: /* We shouldn't get this far. setup.c should prevent it. */ 232 BUG(); 233 return; 234 235 case pcxs: 236 case pcxt: 237 case pcxt_: 238 srhash_type = SRHASH_PCXST; 239 break; 240 241 case pcxl: 242 srhash_type = SRHASH_PCXL; 243 break; 244 245 case pcxl2: /* pcxl2 doesn't support space register hashing */ 246 return; 247 248 default: /* Currently all PA2.0 machines use the same ins. sequence */ 249 srhash_type = SRHASH_PA20; 250 break; 251 } 252 253 disable_sr_hashing_asm(srhash_type); 254 255 retval = pdc_spaceid_bits(&space_bits); 256 /* If this procedure isn't implemented, don't panic. */ 257 if (retval < 0 && retval != PDC_BAD_OPTION) 258 panic("pdc_spaceid_bits call failed.\n"); 259 if (space_bits != 0) 260 panic("SpaceID hashing is still on!\n"); 261 } 262 263 /* Simple function to work out if we have an existing address translation 264 * for a user space vma. */ 265 static inline int translation_exists(struct vm_area_struct *vma, 266 unsigned long addr, unsigned long pfn) 267 { 268 pgd_t *pgd = pgd_offset(vma->vm_mm, addr); 269 pmd_t *pmd; 270 pte_t pte; 271 272 if(pgd_none(*pgd)) 273 return 0; 274 275 pmd = pmd_offset(pgd, addr); 276 if(pmd_none(*pmd) || pmd_bad(*pmd)) 277 return 0; 278 279 /* We cannot take the pte lock here: flush_cache_page is usually 280 * called with pte lock already held. Whereas flush_dcache_page 281 * takes flush_dcache_mmap_lock, which is lower in the hierarchy: 282 * the vma itself is secure, but the pte might come or go racily. 283 */ 284 pte = *pte_offset_map(pmd, addr); 285 /* But pte_unmap() does nothing on this architecture */ 286 287 /* Filter out coincidental file entries and swap entries */ 288 if (!(pte_val(pte) & (_PAGE_FLUSH|_PAGE_PRESENT))) 289 return 0; 290 291 return pte_pfn(pte) == pfn; 292 } 293 294 /* Private function to flush a page from the cache of a non-current 295 * process. cr25 contains the Page Directory of the current user 296 * process; we're going to hijack both it and the user space %sr3 to 297 * temporarily make the non-current process current. We have to do 298 * this because cache flushing may cause a non-access tlb miss which 299 * the handlers have to fill in from the pgd of the non-current 300 * process. */ 301 static inline void 302 flush_user_cache_page_non_current(struct vm_area_struct *vma, 303 unsigned long vmaddr) 304 { 305 /* save the current process space and pgd */ 306 unsigned long space = mfsp(3), pgd = mfctl(25); 307 308 /* we don't mind taking interrupts since they may not 309 * do anything with user space, but we can't 310 * be preempted here */ 311 preempt_disable(); 312 313 /* make us current */ 314 mtctl(__pa(vma->vm_mm->pgd), 25); 315 mtsp(vma->vm_mm->context, 3); 316 317 flush_user_dcache_page(vmaddr); 318 if(vma->vm_flags & VM_EXEC) 319 flush_user_icache_page(vmaddr); 320 321 /* put the old current process back */ 322 mtsp(space, 3); 323 mtctl(pgd, 25); 324 preempt_enable(); 325 } 326 327 328 static inline void 329 __flush_cache_page(struct vm_area_struct *vma, unsigned long vmaddr) 330 { 331 if (likely(vma->vm_mm->context == mfsp(3))) { 332 flush_user_dcache_page(vmaddr); 333 if (vma->vm_flags & VM_EXEC) 334 flush_user_icache_page(vmaddr); 335 } else { 336 flush_user_cache_page_non_current(vma, vmaddr); 337 } 338 } 339 340 void flush_dcache_page(struct page *page) 341 { 342 struct address_space *mapping = page_mapping(page); 343 struct vm_area_struct *mpnt; 344 struct prio_tree_iter iter; 345 unsigned long offset; 346 unsigned long addr; 347 pgoff_t pgoff; 348 unsigned long pfn = page_to_pfn(page); 349 350 351 if (mapping && !mapping_mapped(mapping)) { 352 set_bit(PG_dcache_dirty, &page->flags); 353 return; 354 } 355 356 flush_kernel_dcache_page(page); 357 358 if (!mapping) 359 return; 360 361 pgoff = page->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT); 362 363 /* We have carefully arranged in arch_get_unmapped_area() that 364 * *any* mappings of a file are always congruently mapped (whether 365 * declared as MAP_PRIVATE or MAP_SHARED), so we only need 366 * to flush one address here for them all to become coherent */ 367 368 flush_dcache_mmap_lock(mapping); 369 vma_prio_tree_foreach(mpnt, &iter, &mapping->i_mmap, pgoff, pgoff) { 370 offset = (pgoff - mpnt->vm_pgoff) << PAGE_SHIFT; 371 addr = mpnt->vm_start + offset; 372 373 /* Flush instructions produce non access tlb misses. 374 * On PA, we nullify these instructions rather than 375 * taking a page fault if the pte doesn't exist. 376 * This is just for speed. If the page translation 377 * isn't there, there's no point exciting the 378 * nadtlb handler into a nullification frenzy. 379 * 380 * Make sure we really have this page: the private 381 * mappings may cover this area but have COW'd this 382 * particular page. 383 */ 384 if (translation_exists(mpnt, addr, pfn)) { 385 __flush_cache_page(mpnt, addr); 386 break; 387 } 388 } 389 flush_dcache_mmap_unlock(mapping); 390 } 391 EXPORT_SYMBOL(flush_dcache_page); 392 393 /* Defined in arch/parisc/kernel/pacache.S */ 394 EXPORT_SYMBOL(flush_kernel_dcache_range_asm); 395 EXPORT_SYMBOL(flush_kernel_dcache_page_asm); 396 EXPORT_SYMBOL(flush_data_cache_local); 397 EXPORT_SYMBOL(flush_kernel_icache_range_asm); 398 399 void clear_user_page_asm(void *page, unsigned long vaddr) 400 { 401 /* This function is implemented in assembly in pacache.S */ 402 extern void __clear_user_page_asm(void *page, unsigned long vaddr); 403 404 purge_tlb_start(); 405 __clear_user_page_asm(page, vaddr); 406 purge_tlb_end(); 407 } 408 409 #define FLUSH_THRESHOLD 0x80000 /* 0.5MB */ 410 int parisc_cache_flush_threshold __read_mostly = FLUSH_THRESHOLD; 411 412 void __init parisc_setup_cache_timing(void) 413 { 414 unsigned long rangetime, alltime; 415 unsigned long size; 416 417 alltime = mfctl(16); 418 flush_data_cache(); 419 alltime = mfctl(16) - alltime; 420 421 size = (unsigned long)(_end - _text); 422 rangetime = mfctl(16); 423 flush_kernel_dcache_range((unsigned long)_text, size); 424 rangetime = mfctl(16) - rangetime; 425 426 printk(KERN_DEBUG "Whole cache flush %lu cycles, flushing %lu bytes %lu cycles\n", 427 alltime, size, rangetime); 428 429 /* Racy, but if we see an intermediate value, it's ok too... */ 430 parisc_cache_flush_threshold = size * alltime / rangetime; 431 432 parisc_cache_flush_threshold = (parisc_cache_flush_threshold + L1_CACHE_BYTES - 1) &~ (L1_CACHE_BYTES - 1); 433 if (!parisc_cache_flush_threshold) 434 parisc_cache_flush_threshold = FLUSH_THRESHOLD; 435 436 if (parisc_cache_flush_threshold > cache_info.dc_size) 437 parisc_cache_flush_threshold = cache_info.dc_size; 438 439 printk(KERN_INFO "Setting cache flush threshold to %x (%d CPUs online)\n", parisc_cache_flush_threshold, num_online_cpus()); 440 } 441 442 extern void purge_kernel_dcache_page(unsigned long); 443 extern void clear_user_page_asm(void *page, unsigned long vaddr); 444 445 void clear_user_page(void *page, unsigned long vaddr, struct page *pg) 446 { 447 purge_kernel_dcache_page((unsigned long)page); 448 purge_tlb_start(); 449 pdtlb_kernel(page); 450 purge_tlb_end(); 451 clear_user_page_asm(page, vaddr); 452 } 453 EXPORT_SYMBOL(clear_user_page); 454 455 void flush_kernel_dcache_page_addr(void *addr) 456 { 457 flush_kernel_dcache_page_asm(addr); 458 purge_tlb_start(); 459 pdtlb_kernel(addr); 460 purge_tlb_end(); 461 } 462 EXPORT_SYMBOL(flush_kernel_dcache_page_addr); 463 464 void copy_user_page(void *vto, void *vfrom, unsigned long vaddr, 465 struct page *pg) 466 { 467 /* no coherency needed (all in kmap/kunmap) */ 468 copy_user_page_asm(vto, vfrom); 469 if (!parisc_requires_coherency()) 470 flush_kernel_dcache_page_asm(vto); 471 } 472 EXPORT_SYMBOL(copy_user_page); 473 474 #ifdef CONFIG_PA8X00 475 476 void kunmap_parisc(void *addr) 477 { 478 if (parisc_requires_coherency()) 479 flush_kernel_dcache_page_addr(addr); 480 } 481 EXPORT_SYMBOL(kunmap_parisc); 482 #endif 483 484 void __flush_tlb_range(unsigned long sid, unsigned long start, 485 unsigned long end) 486 { 487 unsigned long npages; 488 489 npages = ((end - (start & PAGE_MASK)) + (PAGE_SIZE - 1)) >> PAGE_SHIFT; 490 if (npages >= 512) /* 2MB of space: arbitrary, should be tuned */ 491 flush_tlb_all(); 492 else { 493 mtsp(sid, 1); 494 purge_tlb_start(); 495 if (split_tlb) { 496 while (npages--) { 497 pdtlb(start); 498 pitlb(start); 499 start += PAGE_SIZE; 500 } 501 } else { 502 while (npages--) { 503 pdtlb(start); 504 start += PAGE_SIZE; 505 } 506 } 507 purge_tlb_end(); 508 } 509 } 510 511 static void cacheflush_h_tmp_function(void *dummy) 512 { 513 flush_cache_all_local(); 514 } 515 516 void flush_cache_all(void) 517 { 518 on_each_cpu(cacheflush_h_tmp_function, NULL, 1, 1); 519 } 520 521 void flush_cache_mm(struct mm_struct *mm) 522 { 523 #ifdef CONFIG_SMP 524 flush_cache_all(); 525 #else 526 flush_cache_all_local(); 527 #endif 528 } 529 530 void 531 flush_user_dcache_range(unsigned long start, unsigned long end) 532 { 533 if ((end - start) < parisc_cache_flush_threshold) 534 flush_user_dcache_range_asm(start,end); 535 else 536 flush_data_cache(); 537 } 538 539 void 540 flush_user_icache_range(unsigned long start, unsigned long end) 541 { 542 if ((end - start) < parisc_cache_flush_threshold) 543 flush_user_icache_range_asm(start,end); 544 else 545 flush_instruction_cache(); 546 } 547 548 549 void flush_cache_range(struct vm_area_struct *vma, 550 unsigned long start, unsigned long end) 551 { 552 int sr3; 553 554 if (!vma->vm_mm->context) { 555 BUG(); 556 return; 557 } 558 559 sr3 = mfsp(3); 560 if (vma->vm_mm->context == sr3) { 561 flush_user_dcache_range(start,end); 562 flush_user_icache_range(start,end); 563 } else { 564 flush_cache_all(); 565 } 566 } 567 568 void 569 flush_cache_page(struct vm_area_struct *vma, unsigned long vmaddr, unsigned long pfn) 570 { 571 BUG_ON(!vma->vm_mm->context); 572 573 if (likely(translation_exists(vma, vmaddr, pfn))) 574 __flush_cache_page(vma, vmaddr); 575 576 } 577