1 /* 2 * This file is subject to the terms and conditions of the GNU General Public 3 * License. See the file "COPYING" in the main directory of this archive 4 * for more details. 5 * 6 * Copyright (C) 1996 David S. Miller (davem@davemloft.net) 7 * Copyright (C) 1997, 1998, 1999, 2000 Ralf Baechle ralf@gnu.org 8 * Carsten Langgaard, carstenl@mips.com 9 * Copyright (C) 2002 MIPS Technologies, Inc. All rights reserved. 10 */ 11 #include <linux/cpu_pm.h> 12 #include <linux/init.h> 13 #include <linux/sched.h> 14 #include <linux/smp.h> 15 #include <linux/mm.h> 16 #include <linux/hugetlb.h> 17 #include <linux/module.h> 18 19 #include <asm/cpu.h> 20 #include <asm/cpu-type.h> 21 #include <asm/bootinfo.h> 22 #include <asm/mmu_context.h> 23 #include <asm/pgtable.h> 24 #include <asm/tlb.h> 25 #include <asm/tlbmisc.h> 26 27 extern void build_tlb_refill_handler(void); 28 29 /* 30 * LOONGSON2/3 has a 4 entry itlb which is a subset of dtlb, 31 * unfortunately, itlb is not totally transparent to software. 32 */ 33 static inline void flush_itlb(void) 34 { 35 switch (current_cpu_type()) { 36 case CPU_LOONGSON2: 37 case CPU_LOONGSON3: 38 write_c0_diag(4); 39 break; 40 default: 41 break; 42 } 43 } 44 45 static inline void flush_itlb_vm(struct vm_area_struct *vma) 46 { 47 if (vma->vm_flags & VM_EXEC) 48 flush_itlb(); 49 } 50 51 void local_flush_tlb_all(void) 52 { 53 unsigned long flags; 54 unsigned long old_ctx; 55 int entry, ftlbhighset; 56 57 local_irq_save(flags); 58 /* Save old context and create impossible VPN2 value */ 59 old_ctx = read_c0_entryhi(); 60 htw_stop(); 61 write_c0_entrylo0(0); 62 write_c0_entrylo1(0); 63 64 entry = read_c0_wired(); 65 66 /* Blast 'em all away. */ 67 if (cpu_has_tlbinv) { 68 if (current_cpu_data.tlbsizevtlb) { 69 write_c0_index(0); 70 mtc0_tlbw_hazard(); 71 tlbinvf(); /* invalidate VTLB */ 72 } 73 ftlbhighset = current_cpu_data.tlbsizevtlb + 74 current_cpu_data.tlbsizeftlbsets; 75 for (entry = current_cpu_data.tlbsizevtlb; 76 entry < ftlbhighset; 77 entry++) { 78 write_c0_index(entry); 79 mtc0_tlbw_hazard(); 80 tlbinvf(); /* invalidate one FTLB set */ 81 } 82 } else { 83 while (entry < current_cpu_data.tlbsize) { 84 /* Make sure all entries differ. */ 85 write_c0_entryhi(UNIQUE_ENTRYHI(entry)); 86 write_c0_index(entry); 87 mtc0_tlbw_hazard(); 88 tlb_write_indexed(); 89 entry++; 90 } 91 } 92 tlbw_use_hazard(); 93 write_c0_entryhi(old_ctx); 94 htw_start(); 95 flush_itlb(); 96 local_irq_restore(flags); 97 } 98 EXPORT_SYMBOL(local_flush_tlb_all); 99 100 /* All entries common to a mm share an asid. To effectively flush 101 these entries, we just bump the asid. */ 102 void local_flush_tlb_mm(struct mm_struct *mm) 103 { 104 int cpu; 105 106 preempt_disable(); 107 108 cpu = smp_processor_id(); 109 110 if (cpu_context(cpu, mm) != 0) { 111 drop_mmu_context(mm, cpu); 112 } 113 114 preempt_enable(); 115 } 116 117 void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, 118 unsigned long end) 119 { 120 struct mm_struct *mm = vma->vm_mm; 121 int cpu = smp_processor_id(); 122 123 if (cpu_context(cpu, mm) != 0) { 124 unsigned long size, flags; 125 126 local_irq_save(flags); 127 start = round_down(start, PAGE_SIZE << 1); 128 end = round_up(end, PAGE_SIZE << 1); 129 size = (end - start) >> (PAGE_SHIFT + 1); 130 if (size <= (current_cpu_data.tlbsizeftlbsets ? 131 current_cpu_data.tlbsize / 8 : 132 current_cpu_data.tlbsize / 2)) { 133 int oldpid = read_c0_entryhi(); 134 int newpid = cpu_asid(cpu, mm); 135 136 htw_stop(); 137 while (start < end) { 138 int idx; 139 140 write_c0_entryhi(start | newpid); 141 start += (PAGE_SIZE << 1); 142 mtc0_tlbw_hazard(); 143 tlb_probe(); 144 tlb_probe_hazard(); 145 idx = read_c0_index(); 146 write_c0_entrylo0(0); 147 write_c0_entrylo1(0); 148 if (idx < 0) 149 continue; 150 /* Make sure all entries differ. */ 151 write_c0_entryhi(UNIQUE_ENTRYHI(idx)); 152 mtc0_tlbw_hazard(); 153 tlb_write_indexed(); 154 } 155 tlbw_use_hazard(); 156 write_c0_entryhi(oldpid); 157 htw_start(); 158 } else { 159 drop_mmu_context(mm, cpu); 160 } 161 flush_itlb(); 162 local_irq_restore(flags); 163 } 164 } 165 166 void local_flush_tlb_kernel_range(unsigned long start, unsigned long end) 167 { 168 unsigned long size, flags; 169 170 local_irq_save(flags); 171 size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT; 172 size = (size + 1) >> 1; 173 if (size <= (current_cpu_data.tlbsizeftlbsets ? 174 current_cpu_data.tlbsize / 8 : 175 current_cpu_data.tlbsize / 2)) { 176 int pid = read_c0_entryhi(); 177 178 start &= (PAGE_MASK << 1); 179 end += ((PAGE_SIZE << 1) - 1); 180 end &= (PAGE_MASK << 1); 181 htw_stop(); 182 183 while (start < end) { 184 int idx; 185 186 write_c0_entryhi(start); 187 start += (PAGE_SIZE << 1); 188 mtc0_tlbw_hazard(); 189 tlb_probe(); 190 tlb_probe_hazard(); 191 idx = read_c0_index(); 192 write_c0_entrylo0(0); 193 write_c0_entrylo1(0); 194 if (idx < 0) 195 continue; 196 /* Make sure all entries differ. */ 197 write_c0_entryhi(UNIQUE_ENTRYHI(idx)); 198 mtc0_tlbw_hazard(); 199 tlb_write_indexed(); 200 } 201 tlbw_use_hazard(); 202 write_c0_entryhi(pid); 203 htw_start(); 204 } else { 205 local_flush_tlb_all(); 206 } 207 flush_itlb(); 208 local_irq_restore(flags); 209 } 210 211 void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page) 212 { 213 int cpu = smp_processor_id(); 214 215 if (cpu_context(cpu, vma->vm_mm) != 0) { 216 unsigned long flags; 217 int oldpid, newpid, idx; 218 219 newpid = cpu_asid(cpu, vma->vm_mm); 220 page &= (PAGE_MASK << 1); 221 local_irq_save(flags); 222 oldpid = read_c0_entryhi(); 223 htw_stop(); 224 write_c0_entryhi(page | newpid); 225 mtc0_tlbw_hazard(); 226 tlb_probe(); 227 tlb_probe_hazard(); 228 idx = read_c0_index(); 229 write_c0_entrylo0(0); 230 write_c0_entrylo1(0); 231 if (idx < 0) 232 goto finish; 233 /* Make sure all entries differ. */ 234 write_c0_entryhi(UNIQUE_ENTRYHI(idx)); 235 mtc0_tlbw_hazard(); 236 tlb_write_indexed(); 237 tlbw_use_hazard(); 238 239 finish: 240 write_c0_entryhi(oldpid); 241 htw_start(); 242 flush_itlb_vm(vma); 243 local_irq_restore(flags); 244 } 245 } 246 247 /* 248 * This one is only used for pages with the global bit set so we don't care 249 * much about the ASID. 250 */ 251 void local_flush_tlb_one(unsigned long page) 252 { 253 unsigned long flags; 254 int oldpid, idx; 255 256 local_irq_save(flags); 257 oldpid = read_c0_entryhi(); 258 htw_stop(); 259 page &= (PAGE_MASK << 1); 260 write_c0_entryhi(page); 261 mtc0_tlbw_hazard(); 262 tlb_probe(); 263 tlb_probe_hazard(); 264 idx = read_c0_index(); 265 write_c0_entrylo0(0); 266 write_c0_entrylo1(0); 267 if (idx >= 0) { 268 /* Make sure all entries differ. */ 269 write_c0_entryhi(UNIQUE_ENTRYHI(idx)); 270 mtc0_tlbw_hazard(); 271 tlb_write_indexed(); 272 tlbw_use_hazard(); 273 } 274 write_c0_entryhi(oldpid); 275 htw_start(); 276 flush_itlb(); 277 local_irq_restore(flags); 278 } 279 280 /* 281 * We will need multiple versions of update_mmu_cache(), one that just 282 * updates the TLB with the new pte(s), and another which also checks 283 * for the R4k "end of page" hardware bug and does the needy. 284 */ 285 void __update_tlb(struct vm_area_struct * vma, unsigned long address, pte_t pte) 286 { 287 unsigned long flags; 288 pgd_t *pgdp; 289 pud_t *pudp; 290 pmd_t *pmdp; 291 pte_t *ptep; 292 int idx, pid; 293 294 /* 295 * Handle debugger faulting in for debugee. 296 */ 297 if (current->active_mm != vma->vm_mm) 298 return; 299 300 local_irq_save(flags); 301 302 pid = read_c0_entryhi() & ASID_MASK; 303 address &= (PAGE_MASK << 1); 304 write_c0_entryhi(address | pid); 305 pgdp = pgd_offset(vma->vm_mm, address); 306 mtc0_tlbw_hazard(); 307 tlb_probe(); 308 tlb_probe_hazard(); 309 pudp = pud_offset(pgdp, address); 310 pmdp = pmd_offset(pudp, address); 311 idx = read_c0_index(); 312 #ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT 313 /* this could be a huge page */ 314 if (pmd_huge(*pmdp)) { 315 unsigned long lo; 316 write_c0_pagemask(PM_HUGE_MASK); 317 ptep = (pte_t *)pmdp; 318 lo = pte_to_entrylo(pte_val(*ptep)); 319 write_c0_entrylo0(lo); 320 write_c0_entrylo1(lo + (HPAGE_SIZE >> 7)); 321 322 mtc0_tlbw_hazard(); 323 if (idx < 0) 324 tlb_write_random(); 325 else 326 tlb_write_indexed(); 327 tlbw_use_hazard(); 328 write_c0_pagemask(PM_DEFAULT_MASK); 329 } else 330 #endif 331 { 332 ptep = pte_offset_map(pmdp, address); 333 334 #if defined(CONFIG_64BIT_PHYS_ADDR) && defined(CONFIG_CPU_MIPS32) 335 write_c0_entrylo0(ptep->pte_high); 336 ptep++; 337 write_c0_entrylo1(ptep->pte_high); 338 #else 339 write_c0_entrylo0(pte_to_entrylo(pte_val(*ptep++))); 340 write_c0_entrylo1(pte_to_entrylo(pte_val(*ptep))); 341 #endif 342 mtc0_tlbw_hazard(); 343 if (idx < 0) 344 tlb_write_random(); 345 else 346 tlb_write_indexed(); 347 } 348 tlbw_use_hazard(); 349 flush_itlb_vm(vma); 350 local_irq_restore(flags); 351 } 352 353 void add_wired_entry(unsigned long entrylo0, unsigned long entrylo1, 354 unsigned long entryhi, unsigned long pagemask) 355 { 356 unsigned long flags; 357 unsigned long wired; 358 unsigned long old_pagemask; 359 unsigned long old_ctx; 360 361 local_irq_save(flags); 362 /* Save old context and create impossible VPN2 value */ 363 old_ctx = read_c0_entryhi(); 364 htw_stop(); 365 old_pagemask = read_c0_pagemask(); 366 wired = read_c0_wired(); 367 write_c0_wired(wired + 1); 368 write_c0_index(wired); 369 tlbw_use_hazard(); /* What is the hazard here? */ 370 write_c0_pagemask(pagemask); 371 write_c0_entryhi(entryhi); 372 write_c0_entrylo0(entrylo0); 373 write_c0_entrylo1(entrylo1); 374 mtc0_tlbw_hazard(); 375 tlb_write_indexed(); 376 tlbw_use_hazard(); 377 378 write_c0_entryhi(old_ctx); 379 tlbw_use_hazard(); /* What is the hazard here? */ 380 htw_start(); 381 write_c0_pagemask(old_pagemask); 382 local_flush_tlb_all(); 383 local_irq_restore(flags); 384 } 385 386 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 387 388 int __init has_transparent_hugepage(void) 389 { 390 unsigned int mask; 391 unsigned long flags; 392 393 local_irq_save(flags); 394 write_c0_pagemask(PM_HUGE_MASK); 395 back_to_back_c0_hazard(); 396 mask = read_c0_pagemask(); 397 write_c0_pagemask(PM_DEFAULT_MASK); 398 399 local_irq_restore(flags); 400 401 return mask == PM_HUGE_MASK; 402 } 403 404 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 405 406 /* 407 * Used for loading TLB entries before trap_init() has started, when we 408 * don't actually want to add a wired entry which remains throughout the 409 * lifetime of the system 410 */ 411 412 int temp_tlb_entry __cpuinitdata; 413 414 __init int add_temporary_entry(unsigned long entrylo0, unsigned long entrylo1, 415 unsigned long entryhi, unsigned long pagemask) 416 { 417 int ret = 0; 418 unsigned long flags; 419 unsigned long wired; 420 unsigned long old_pagemask; 421 unsigned long old_ctx; 422 423 local_irq_save(flags); 424 /* Save old context and create impossible VPN2 value */ 425 old_ctx = read_c0_entryhi(); 426 old_pagemask = read_c0_pagemask(); 427 wired = read_c0_wired(); 428 if (--temp_tlb_entry < wired) { 429 printk(KERN_WARNING 430 "No TLB space left for add_temporary_entry\n"); 431 ret = -ENOSPC; 432 goto out; 433 } 434 435 write_c0_index(temp_tlb_entry); 436 write_c0_pagemask(pagemask); 437 write_c0_entryhi(entryhi); 438 write_c0_entrylo0(entrylo0); 439 write_c0_entrylo1(entrylo1); 440 mtc0_tlbw_hazard(); 441 tlb_write_indexed(); 442 tlbw_use_hazard(); 443 444 write_c0_entryhi(old_ctx); 445 write_c0_pagemask(old_pagemask); 446 out: 447 local_irq_restore(flags); 448 return ret; 449 } 450 451 static int ntlb; 452 static int __init set_ntlb(char *str) 453 { 454 get_option(&str, &ntlb); 455 return 1; 456 } 457 458 __setup("ntlb=", set_ntlb); 459 460 /* 461 * Configure TLB (for init or after a CPU has been powered off). 462 */ 463 static void r4k_tlb_configure(void) 464 { 465 /* 466 * You should never change this register: 467 * - On R4600 1.7 the tlbp never hits for pages smaller than 468 * the value in the c0_pagemask register. 469 * - The entire mm handling assumes the c0_pagemask register to 470 * be set to fixed-size pages. 471 */ 472 write_c0_pagemask(PM_DEFAULT_MASK); 473 write_c0_wired(0); 474 if (current_cpu_type() == CPU_R10000 || 475 current_cpu_type() == CPU_R12000 || 476 current_cpu_type() == CPU_R14000) 477 write_c0_framemask(0); 478 479 if (cpu_has_rixi) { 480 /* 481 * Enable the no read, no exec bits, and enable large virtual 482 * address. 483 */ 484 u32 pg = PG_RIE | PG_XIE; 485 #ifdef CONFIG_64BIT 486 pg |= PG_ELPA; 487 #endif 488 write_c0_pagegrain(pg); 489 } 490 491 temp_tlb_entry = current_cpu_data.tlbsize - 1; 492 493 /* From this point on the ARC firmware is dead. */ 494 local_flush_tlb_all(); 495 496 /* Did I tell you that ARC SUCKS? */ 497 } 498 499 void tlb_init(void) 500 { 501 r4k_tlb_configure(); 502 503 if (ntlb) { 504 if (ntlb > 1 && ntlb <= current_cpu_data.tlbsize) { 505 int wired = current_cpu_data.tlbsize - ntlb; 506 write_c0_wired(wired); 507 write_c0_index(wired-1); 508 printk("Restricting TLB to %d entries\n", ntlb); 509 } else 510 printk("Ignoring invalid argument ntlb=%d\n", ntlb); 511 } 512 513 build_tlb_refill_handler(); 514 } 515 516 static int r4k_tlb_pm_notifier(struct notifier_block *self, unsigned long cmd, 517 void *v) 518 { 519 switch (cmd) { 520 case CPU_PM_ENTER_FAILED: 521 case CPU_PM_EXIT: 522 r4k_tlb_configure(); 523 break; 524 } 525 526 return NOTIFY_OK; 527 } 528 529 static struct notifier_block r4k_tlb_pm_notifier_block = { 530 .notifier_call = r4k_tlb_pm_notifier, 531 }; 532 533 static int __init r4k_tlb_init_pm(void) 534 { 535 return cpu_pm_register_notifier(&r4k_tlb_pm_notifier_block); 536 } 537 arch_initcall(r4k_tlb_init_pm); 538