1 /* 2 * This file is subject to the terms and conditions of the GNU General Public 3 * License. See the file "COPYING" in the main directory of this archive 4 * for more details. 5 * 6 * Copyright (C) 1996 David S. Miller (dm@engr.sgi.com) 7 * Copyright (C) 1997, 1998, 1999, 2000 Ralf Baechle ralf@gnu.org 8 * Carsten Langgaard, carstenl@mips.com 9 * Copyright (C) 2002 MIPS Technologies, Inc. All rights reserved. 10 */ 11 #include <linux/init.h> 12 #include <linux/sched.h> 13 #include <linux/mm.h> 14 15 #include <asm/cpu.h> 16 #include <asm/bootinfo.h> 17 #include <asm/mmu_context.h> 18 #include <asm/pgtable.h> 19 #include <asm/system.h> 20 21 extern void build_tlb_refill_handler(void); 22 23 /* 24 * Make sure all entries differ. If they're not different 25 * MIPS32 will take revenge ... 26 */ 27 #define UNIQUE_ENTRYHI(idx) (CKSEG0 + ((idx) << (PAGE_SHIFT + 1))) 28 29 /* Atomicity and interruptability */ 30 #ifdef CONFIG_MIPS_MT_SMTC 31 32 #include <asm/smtc.h> 33 #include <asm/mipsmtregs.h> 34 35 #define ENTER_CRITICAL(flags) \ 36 { \ 37 unsigned int mvpflags; \ 38 local_irq_save(flags);\ 39 mvpflags = dvpe() 40 #define EXIT_CRITICAL(flags) \ 41 evpe(mvpflags); \ 42 local_irq_restore(flags); \ 43 } 44 #else 45 46 #define ENTER_CRITICAL(flags) local_irq_save(flags) 47 #define EXIT_CRITICAL(flags) local_irq_restore(flags) 48 49 #endif /* CONFIG_MIPS_MT_SMTC */ 50 51 #if defined(CONFIG_CPU_LOONGSON2) 52 /* 53 * LOONGSON2 has a 4 entry itlb which is a subset of dtlb, 54 * unfortrunately, itlb is not totally transparent to software. 55 */ 56 #define FLUSH_ITLB write_c0_diag(4); 57 58 #define FLUSH_ITLB_VM(vma) { if ((vma)->vm_flags & VM_EXEC) write_c0_diag(4); } 59 60 #else 61 62 #define FLUSH_ITLB 63 #define FLUSH_ITLB_VM(vma) 64 65 #endif 66 67 void local_flush_tlb_all(void) 68 { 69 unsigned long flags; 70 unsigned long old_ctx; 71 int entry; 72 73 ENTER_CRITICAL(flags); 74 /* Save old context and create impossible VPN2 value */ 75 old_ctx = read_c0_entryhi(); 76 write_c0_entrylo0(0); 77 write_c0_entrylo1(0); 78 79 entry = read_c0_wired(); 80 81 /* Blast 'em all away. */ 82 while (entry < current_cpu_data.tlbsize) { 83 /* Make sure all entries differ. */ 84 write_c0_entryhi(UNIQUE_ENTRYHI(entry)); 85 write_c0_index(entry); 86 mtc0_tlbw_hazard(); 87 tlb_write_indexed(); 88 entry++; 89 } 90 tlbw_use_hazard(); 91 write_c0_entryhi(old_ctx); 92 FLUSH_ITLB; 93 EXIT_CRITICAL(flags); 94 } 95 96 /* All entries common to a mm share an asid. To effectively flush 97 these entries, we just bump the asid. */ 98 void local_flush_tlb_mm(struct mm_struct *mm) 99 { 100 int cpu; 101 102 preempt_disable(); 103 104 cpu = smp_processor_id(); 105 106 if (cpu_context(cpu, mm) != 0) { 107 drop_mmu_context(mm, cpu); 108 } 109 110 preempt_enable(); 111 } 112 113 void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, 114 unsigned long end) 115 { 116 struct mm_struct *mm = vma->vm_mm; 117 int cpu = smp_processor_id(); 118 119 if (cpu_context(cpu, mm) != 0) { 120 unsigned long flags; 121 int size; 122 123 ENTER_CRITICAL(flags); 124 size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT; 125 size = (size + 1) >> 1; 126 if (size <= current_cpu_data.tlbsize/2) { 127 int oldpid = read_c0_entryhi(); 128 int newpid = cpu_asid(cpu, mm); 129 130 start &= (PAGE_MASK << 1); 131 end += ((PAGE_SIZE << 1) - 1); 132 end &= (PAGE_MASK << 1); 133 while (start < end) { 134 int idx; 135 136 write_c0_entryhi(start | newpid); 137 start += (PAGE_SIZE << 1); 138 mtc0_tlbw_hazard(); 139 tlb_probe(); 140 tlb_probe_hazard(); 141 idx = read_c0_index(); 142 write_c0_entrylo0(0); 143 write_c0_entrylo1(0); 144 if (idx < 0) 145 continue; 146 /* Make sure all entries differ. */ 147 write_c0_entryhi(UNIQUE_ENTRYHI(idx)); 148 mtc0_tlbw_hazard(); 149 tlb_write_indexed(); 150 } 151 tlbw_use_hazard(); 152 write_c0_entryhi(oldpid); 153 } else { 154 drop_mmu_context(mm, cpu); 155 } 156 FLUSH_ITLB; 157 EXIT_CRITICAL(flags); 158 } 159 } 160 161 void local_flush_tlb_kernel_range(unsigned long start, unsigned long end) 162 { 163 unsigned long flags; 164 int size; 165 166 ENTER_CRITICAL(flags); 167 size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT; 168 size = (size + 1) >> 1; 169 if (size <= current_cpu_data.tlbsize / 2) { 170 int pid = read_c0_entryhi(); 171 172 start &= (PAGE_MASK << 1); 173 end += ((PAGE_SIZE << 1) - 1); 174 end &= (PAGE_MASK << 1); 175 176 while (start < end) { 177 int idx; 178 179 write_c0_entryhi(start); 180 start += (PAGE_SIZE << 1); 181 mtc0_tlbw_hazard(); 182 tlb_probe(); 183 tlb_probe_hazard(); 184 idx = read_c0_index(); 185 write_c0_entrylo0(0); 186 write_c0_entrylo1(0); 187 if (idx < 0) 188 continue; 189 /* Make sure all entries differ. */ 190 write_c0_entryhi(UNIQUE_ENTRYHI(idx)); 191 mtc0_tlbw_hazard(); 192 tlb_write_indexed(); 193 } 194 tlbw_use_hazard(); 195 write_c0_entryhi(pid); 196 } else { 197 local_flush_tlb_all(); 198 } 199 FLUSH_ITLB; 200 EXIT_CRITICAL(flags); 201 } 202 203 void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page) 204 { 205 int cpu = smp_processor_id(); 206 207 if (cpu_context(cpu, vma->vm_mm) != 0) { 208 unsigned long flags; 209 int oldpid, newpid, idx; 210 211 newpid = cpu_asid(cpu, vma->vm_mm); 212 page &= (PAGE_MASK << 1); 213 ENTER_CRITICAL(flags); 214 oldpid = read_c0_entryhi(); 215 write_c0_entryhi(page | newpid); 216 mtc0_tlbw_hazard(); 217 tlb_probe(); 218 tlb_probe_hazard(); 219 idx = read_c0_index(); 220 write_c0_entrylo0(0); 221 write_c0_entrylo1(0); 222 if (idx < 0) 223 goto finish; 224 /* Make sure all entries differ. */ 225 write_c0_entryhi(UNIQUE_ENTRYHI(idx)); 226 mtc0_tlbw_hazard(); 227 tlb_write_indexed(); 228 tlbw_use_hazard(); 229 230 finish: 231 write_c0_entryhi(oldpid); 232 FLUSH_ITLB_VM(vma); 233 EXIT_CRITICAL(flags); 234 } 235 } 236 237 /* 238 * This one is only used for pages with the global bit set so we don't care 239 * much about the ASID. 240 */ 241 void local_flush_tlb_one(unsigned long page) 242 { 243 unsigned long flags; 244 int oldpid, idx; 245 246 ENTER_CRITICAL(flags); 247 oldpid = read_c0_entryhi(); 248 page &= (PAGE_MASK << 1); 249 write_c0_entryhi(page); 250 mtc0_tlbw_hazard(); 251 tlb_probe(); 252 tlb_probe_hazard(); 253 idx = read_c0_index(); 254 write_c0_entrylo0(0); 255 write_c0_entrylo1(0); 256 if (idx >= 0) { 257 /* Make sure all entries differ. */ 258 write_c0_entryhi(UNIQUE_ENTRYHI(idx)); 259 mtc0_tlbw_hazard(); 260 tlb_write_indexed(); 261 tlbw_use_hazard(); 262 } 263 write_c0_entryhi(oldpid); 264 FLUSH_ITLB; 265 EXIT_CRITICAL(flags); 266 } 267 268 /* 269 * We will need multiple versions of update_mmu_cache(), one that just 270 * updates the TLB with the new pte(s), and another which also checks 271 * for the R4k "end of page" hardware bug and does the needy. 272 */ 273 void __update_tlb(struct vm_area_struct * vma, unsigned long address, pte_t pte) 274 { 275 unsigned long flags; 276 pgd_t *pgdp; 277 pud_t *pudp; 278 pmd_t *pmdp; 279 pte_t *ptep; 280 int idx, pid; 281 282 /* 283 * Handle debugger faulting in for debugee. 284 */ 285 if (current->active_mm != vma->vm_mm) 286 return; 287 288 ENTER_CRITICAL(flags); 289 290 pid = read_c0_entryhi() & ASID_MASK; 291 address &= (PAGE_MASK << 1); 292 write_c0_entryhi(address | pid); 293 pgdp = pgd_offset(vma->vm_mm, address); 294 mtc0_tlbw_hazard(); 295 tlb_probe(); 296 tlb_probe_hazard(); 297 pudp = pud_offset(pgdp, address); 298 pmdp = pmd_offset(pudp, address); 299 idx = read_c0_index(); 300 ptep = pte_offset_map(pmdp, address); 301 302 #if defined(CONFIG_64BIT_PHYS_ADDR) && defined(CONFIG_CPU_MIPS32) 303 write_c0_entrylo0(ptep->pte_high); 304 ptep++; 305 write_c0_entrylo1(ptep->pte_high); 306 #else 307 write_c0_entrylo0(pte_val(*ptep++) >> 6); 308 write_c0_entrylo1(pte_val(*ptep) >> 6); 309 #endif 310 mtc0_tlbw_hazard(); 311 if (idx < 0) 312 tlb_write_random(); 313 else 314 tlb_write_indexed(); 315 tlbw_use_hazard(); 316 FLUSH_ITLB_VM(vma); 317 EXIT_CRITICAL(flags); 318 } 319 320 #if 0 321 static void r4k_update_mmu_cache_hwbug(struct vm_area_struct * vma, 322 unsigned long address, pte_t pte) 323 { 324 unsigned long flags; 325 unsigned int asid; 326 pgd_t *pgdp; 327 pmd_t *pmdp; 328 pte_t *ptep; 329 int idx; 330 331 ENTER_CRITICAL(flags); 332 address &= (PAGE_MASK << 1); 333 asid = read_c0_entryhi() & ASID_MASK; 334 write_c0_entryhi(address | asid); 335 pgdp = pgd_offset(vma->vm_mm, address); 336 mtc0_tlbw_hazard(); 337 tlb_probe(); 338 tlb_probe_hazard(); 339 pmdp = pmd_offset(pgdp, address); 340 idx = read_c0_index(); 341 ptep = pte_offset_map(pmdp, address); 342 write_c0_entrylo0(pte_val(*ptep++) >> 6); 343 write_c0_entrylo1(pte_val(*ptep) >> 6); 344 mtc0_tlbw_hazard(); 345 if (idx < 0) 346 tlb_write_random(); 347 else 348 tlb_write_indexed(); 349 tlbw_use_hazard(); 350 EXIT_CRITICAL(flags); 351 } 352 #endif 353 354 void __init add_wired_entry(unsigned long entrylo0, unsigned long entrylo1, 355 unsigned long entryhi, unsigned long pagemask) 356 { 357 unsigned long flags; 358 unsigned long wired; 359 unsigned long old_pagemask; 360 unsigned long old_ctx; 361 362 ENTER_CRITICAL(flags); 363 /* Save old context and create impossible VPN2 value */ 364 old_ctx = read_c0_entryhi(); 365 old_pagemask = read_c0_pagemask(); 366 wired = read_c0_wired(); 367 write_c0_wired(wired + 1); 368 write_c0_index(wired); 369 tlbw_use_hazard(); /* What is the hazard here? */ 370 write_c0_pagemask(pagemask); 371 write_c0_entryhi(entryhi); 372 write_c0_entrylo0(entrylo0); 373 write_c0_entrylo1(entrylo1); 374 mtc0_tlbw_hazard(); 375 tlb_write_indexed(); 376 tlbw_use_hazard(); 377 378 write_c0_entryhi(old_ctx); 379 tlbw_use_hazard(); /* What is the hazard here? */ 380 write_c0_pagemask(old_pagemask); 381 local_flush_tlb_all(); 382 EXIT_CRITICAL(flags); 383 } 384 385 /* 386 * Used for loading TLB entries before trap_init() has started, when we 387 * don't actually want to add a wired entry which remains throughout the 388 * lifetime of the system 389 */ 390 391 static int temp_tlb_entry __cpuinitdata; 392 393 __init int add_temporary_entry(unsigned long entrylo0, unsigned long entrylo1, 394 unsigned long entryhi, unsigned long pagemask) 395 { 396 int ret = 0; 397 unsigned long flags; 398 unsigned long wired; 399 unsigned long old_pagemask; 400 unsigned long old_ctx; 401 402 ENTER_CRITICAL(flags); 403 /* Save old context and create impossible VPN2 value */ 404 old_ctx = read_c0_entryhi(); 405 old_pagemask = read_c0_pagemask(); 406 wired = read_c0_wired(); 407 if (--temp_tlb_entry < wired) { 408 printk(KERN_WARNING 409 "No TLB space left for add_temporary_entry\n"); 410 ret = -ENOSPC; 411 goto out; 412 } 413 414 write_c0_index(temp_tlb_entry); 415 write_c0_pagemask(pagemask); 416 write_c0_entryhi(entryhi); 417 write_c0_entrylo0(entrylo0); 418 write_c0_entrylo1(entrylo1); 419 mtc0_tlbw_hazard(); 420 tlb_write_indexed(); 421 tlbw_use_hazard(); 422 423 write_c0_entryhi(old_ctx); 424 write_c0_pagemask(old_pagemask); 425 out: 426 EXIT_CRITICAL(flags); 427 return ret; 428 } 429 430 static void __cpuinit probe_tlb(unsigned long config) 431 { 432 struct cpuinfo_mips *c = ¤t_cpu_data; 433 unsigned int reg; 434 435 /* 436 * If this isn't a MIPS32 / MIPS64 compliant CPU. Config 1 register 437 * is not supported, we assume R4k style. Cpu probing already figured 438 * out the number of tlb entries. 439 */ 440 if ((c->processor_id & 0xff0000) == PRID_COMP_LEGACY) 441 return; 442 #ifdef CONFIG_MIPS_MT_SMTC 443 /* 444 * If TLB is shared in SMTC system, total size already 445 * has been calculated and written into cpu_data tlbsize 446 */ 447 if((smtc_status & SMTC_TLB_SHARED) == SMTC_TLB_SHARED) 448 return; 449 #endif /* CONFIG_MIPS_MT_SMTC */ 450 451 reg = read_c0_config1(); 452 if (!((config >> 7) & 3)) 453 panic("No TLB present"); 454 455 c->tlbsize = ((reg >> 25) & 0x3f) + 1; 456 } 457 458 static int __cpuinitdata ntlb = 0; 459 static int __init set_ntlb(char *str) 460 { 461 get_option(&str, &ntlb); 462 return 1; 463 } 464 465 __setup("ntlb=", set_ntlb); 466 467 void __cpuinit tlb_init(void) 468 { 469 unsigned int config = read_c0_config(); 470 471 /* 472 * You should never change this register: 473 * - On R4600 1.7 the tlbp never hits for pages smaller than 474 * the value in the c0_pagemask register. 475 * - The entire mm handling assumes the c0_pagemask register to 476 * be set to fixed-size pages. 477 */ 478 probe_tlb(config); 479 write_c0_pagemask(PM_DEFAULT_MASK); 480 write_c0_wired(0); 481 write_c0_framemask(0); 482 temp_tlb_entry = current_cpu_data.tlbsize - 1; 483 484 /* From this point on the ARC firmware is dead. */ 485 local_flush_tlb_all(); 486 487 /* Did I tell you that ARC SUCKS? */ 488 489 if (ntlb) { 490 if (ntlb > 1 && ntlb <= current_cpu_data.tlbsize) { 491 int wired = current_cpu_data.tlbsize - ntlb; 492 write_c0_wired(wired); 493 write_c0_index(wired-1); 494 printk("Restricting TLB to %d entries\n", ntlb); 495 } else 496 printk("Ignoring invalid argument ntlb=%d\n", ntlb); 497 } 498 499 build_tlb_refill_handler(); 500 } 501