1 /* 2 * This file is subject to the terms and conditions of the GNU General Public 3 * License. See the file "COPYING" in the main directory of this archive 4 * for more details. 5 * 6 * Copyright (C) 1996 David S. Miller (davem@davemloft.net) 7 * Copyright (C) 1997, 1998, 1999, 2000 Ralf Baechle ralf@gnu.org 8 * Carsten Langgaard, carstenl@mips.com 9 * Copyright (C) 2002 MIPS Technologies, Inc. All rights reserved. 10 */ 11 #include <linux/init.h> 12 #include <linux/sched.h> 13 #include <linux/smp.h> 14 #include <linux/mm.h> 15 #include <linux/hugetlb.h> 16 17 #include <asm/cpu.h> 18 #include <asm/bootinfo.h> 19 #include <asm/mmu_context.h> 20 #include <asm/pgtable.h> 21 #include <asm/tlbmisc.h> 22 23 extern void build_tlb_refill_handler(void); 24 25 /* 26 * Make sure all entries differ. If they're not different 27 * MIPS32 will take revenge ... 28 */ 29 #define UNIQUE_ENTRYHI(idx) (CKSEG0 + ((idx) << (PAGE_SHIFT + 1))) 30 31 /* Atomicity and interruptability */ 32 #ifdef CONFIG_MIPS_MT_SMTC 33 34 #include <asm/smtc.h> 35 #include <asm/mipsmtregs.h> 36 37 #define ENTER_CRITICAL(flags) \ 38 { \ 39 unsigned int mvpflags; \ 40 local_irq_save(flags);\ 41 mvpflags = dvpe() 42 #define EXIT_CRITICAL(flags) \ 43 evpe(mvpflags); \ 44 local_irq_restore(flags); \ 45 } 46 #else 47 48 #define ENTER_CRITICAL(flags) local_irq_save(flags) 49 #define EXIT_CRITICAL(flags) local_irq_restore(flags) 50 51 #endif /* CONFIG_MIPS_MT_SMTC */ 52 53 #if defined(CONFIG_CPU_LOONGSON2) 54 /* 55 * LOONGSON2 has a 4 entry itlb which is a subset of dtlb, 56 * unfortrunately, itlb is not totally transparent to software. 57 */ 58 #define FLUSH_ITLB write_c0_diag(4); 59 60 #define FLUSH_ITLB_VM(vma) { if ((vma)->vm_flags & VM_EXEC) write_c0_diag(4); } 61 62 #else 63 64 #define FLUSH_ITLB 65 #define FLUSH_ITLB_VM(vma) 66 67 #endif 68 69 void local_flush_tlb_all(void) 70 { 71 unsigned long flags; 72 unsigned long old_ctx; 73 int entry; 74 75 ENTER_CRITICAL(flags); 76 /* Save old context and create impossible VPN2 value */ 77 old_ctx = read_c0_entryhi(); 78 write_c0_entrylo0(0); 79 write_c0_entrylo1(0); 80 81 entry = read_c0_wired(); 82 83 /* Blast 'em all away. */ 84 while (entry < current_cpu_data.tlbsize) { 85 /* Make sure all entries differ. */ 86 write_c0_entryhi(UNIQUE_ENTRYHI(entry)); 87 write_c0_index(entry); 88 mtc0_tlbw_hazard(); 89 tlb_write_indexed(); 90 entry++; 91 } 92 tlbw_use_hazard(); 93 write_c0_entryhi(old_ctx); 94 FLUSH_ITLB; 95 EXIT_CRITICAL(flags); 96 } 97 98 /* All entries common to a mm share an asid. To effectively flush 99 these entries, we just bump the asid. */ 100 void local_flush_tlb_mm(struct mm_struct *mm) 101 { 102 int cpu; 103 104 preempt_disable(); 105 106 cpu = smp_processor_id(); 107 108 if (cpu_context(cpu, mm) != 0) { 109 drop_mmu_context(mm, cpu); 110 } 111 112 preempt_enable(); 113 } 114 115 void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, 116 unsigned long end) 117 { 118 struct mm_struct *mm = vma->vm_mm; 119 int cpu = smp_processor_id(); 120 121 if (cpu_context(cpu, mm) != 0) { 122 unsigned long size, flags; 123 124 ENTER_CRITICAL(flags); 125 start = round_down(start, PAGE_SIZE << 1); 126 end = round_up(end, PAGE_SIZE << 1); 127 size = (end - start) >> (PAGE_SHIFT + 1); 128 if (size <= current_cpu_data.tlbsize/2) { 129 int oldpid = read_c0_entryhi(); 130 int newpid = cpu_asid(cpu, mm); 131 132 while (start < end) { 133 int idx; 134 135 write_c0_entryhi(start | newpid); 136 start += (PAGE_SIZE << 1); 137 mtc0_tlbw_hazard(); 138 tlb_probe(); 139 tlb_probe_hazard(); 140 idx = read_c0_index(); 141 write_c0_entrylo0(0); 142 write_c0_entrylo1(0); 143 if (idx < 0) 144 continue; 145 /* Make sure all entries differ. */ 146 write_c0_entryhi(UNIQUE_ENTRYHI(idx)); 147 mtc0_tlbw_hazard(); 148 tlb_write_indexed(); 149 } 150 tlbw_use_hazard(); 151 write_c0_entryhi(oldpid); 152 } else { 153 drop_mmu_context(mm, cpu); 154 } 155 FLUSH_ITLB; 156 EXIT_CRITICAL(flags); 157 } 158 } 159 160 void local_flush_tlb_kernel_range(unsigned long start, unsigned long end) 161 { 162 unsigned long size, flags; 163 164 ENTER_CRITICAL(flags); 165 size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT; 166 size = (size + 1) >> 1; 167 if (size <= current_cpu_data.tlbsize / 2) { 168 int pid = read_c0_entryhi(); 169 170 start &= (PAGE_MASK << 1); 171 end += ((PAGE_SIZE << 1) - 1); 172 end &= (PAGE_MASK << 1); 173 174 while (start < end) { 175 int idx; 176 177 write_c0_entryhi(start); 178 start += (PAGE_SIZE << 1); 179 mtc0_tlbw_hazard(); 180 tlb_probe(); 181 tlb_probe_hazard(); 182 idx = read_c0_index(); 183 write_c0_entrylo0(0); 184 write_c0_entrylo1(0); 185 if (idx < 0) 186 continue; 187 /* Make sure all entries differ. */ 188 write_c0_entryhi(UNIQUE_ENTRYHI(idx)); 189 mtc0_tlbw_hazard(); 190 tlb_write_indexed(); 191 } 192 tlbw_use_hazard(); 193 write_c0_entryhi(pid); 194 } else { 195 local_flush_tlb_all(); 196 } 197 FLUSH_ITLB; 198 EXIT_CRITICAL(flags); 199 } 200 201 void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page) 202 { 203 int cpu = smp_processor_id(); 204 205 if (cpu_context(cpu, vma->vm_mm) != 0) { 206 unsigned long flags; 207 int oldpid, newpid, idx; 208 209 newpid = cpu_asid(cpu, vma->vm_mm); 210 page &= (PAGE_MASK << 1); 211 ENTER_CRITICAL(flags); 212 oldpid = read_c0_entryhi(); 213 write_c0_entryhi(page | newpid); 214 mtc0_tlbw_hazard(); 215 tlb_probe(); 216 tlb_probe_hazard(); 217 idx = read_c0_index(); 218 write_c0_entrylo0(0); 219 write_c0_entrylo1(0); 220 if (idx < 0) 221 goto finish; 222 /* Make sure all entries differ. */ 223 write_c0_entryhi(UNIQUE_ENTRYHI(idx)); 224 mtc0_tlbw_hazard(); 225 tlb_write_indexed(); 226 tlbw_use_hazard(); 227 228 finish: 229 write_c0_entryhi(oldpid); 230 FLUSH_ITLB_VM(vma); 231 EXIT_CRITICAL(flags); 232 } 233 } 234 235 /* 236 * This one is only used for pages with the global bit set so we don't care 237 * much about the ASID. 238 */ 239 void local_flush_tlb_one(unsigned long page) 240 { 241 unsigned long flags; 242 int oldpid, idx; 243 244 ENTER_CRITICAL(flags); 245 oldpid = read_c0_entryhi(); 246 page &= (PAGE_MASK << 1); 247 write_c0_entryhi(page); 248 mtc0_tlbw_hazard(); 249 tlb_probe(); 250 tlb_probe_hazard(); 251 idx = read_c0_index(); 252 write_c0_entrylo0(0); 253 write_c0_entrylo1(0); 254 if (idx >= 0) { 255 /* Make sure all entries differ. */ 256 write_c0_entryhi(UNIQUE_ENTRYHI(idx)); 257 mtc0_tlbw_hazard(); 258 tlb_write_indexed(); 259 tlbw_use_hazard(); 260 } 261 write_c0_entryhi(oldpid); 262 FLUSH_ITLB; 263 EXIT_CRITICAL(flags); 264 } 265 266 /* 267 * We will need multiple versions of update_mmu_cache(), one that just 268 * updates the TLB with the new pte(s), and another which also checks 269 * for the R4k "end of page" hardware bug and does the needy. 270 */ 271 void __update_tlb(struct vm_area_struct * vma, unsigned long address, pte_t pte) 272 { 273 unsigned long flags; 274 pgd_t *pgdp; 275 pud_t *pudp; 276 pmd_t *pmdp; 277 pte_t *ptep; 278 int idx, pid; 279 280 /* 281 * Handle debugger faulting in for debugee. 282 */ 283 if (current->active_mm != vma->vm_mm) 284 return; 285 286 ENTER_CRITICAL(flags); 287 288 pid = read_c0_entryhi() & ASID_MASK; 289 address &= (PAGE_MASK << 1); 290 write_c0_entryhi(address | pid); 291 pgdp = pgd_offset(vma->vm_mm, address); 292 mtc0_tlbw_hazard(); 293 tlb_probe(); 294 tlb_probe_hazard(); 295 pudp = pud_offset(pgdp, address); 296 pmdp = pmd_offset(pudp, address); 297 idx = read_c0_index(); 298 #ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT 299 /* this could be a huge page */ 300 if (pmd_huge(*pmdp)) { 301 unsigned long lo; 302 write_c0_pagemask(PM_HUGE_MASK); 303 ptep = (pte_t *)pmdp; 304 lo = pte_to_entrylo(pte_val(*ptep)); 305 write_c0_entrylo0(lo); 306 write_c0_entrylo1(lo + (HPAGE_SIZE >> 7)); 307 308 mtc0_tlbw_hazard(); 309 if (idx < 0) 310 tlb_write_random(); 311 else 312 tlb_write_indexed(); 313 tlbw_use_hazard(); 314 write_c0_pagemask(PM_DEFAULT_MASK); 315 } else 316 #endif 317 { 318 ptep = pte_offset_map(pmdp, address); 319 320 #if defined(CONFIG_64BIT_PHYS_ADDR) && defined(CONFIG_CPU_MIPS32) 321 write_c0_entrylo0(ptep->pte_high); 322 ptep++; 323 write_c0_entrylo1(ptep->pte_high); 324 #else 325 write_c0_entrylo0(pte_to_entrylo(pte_val(*ptep++))); 326 write_c0_entrylo1(pte_to_entrylo(pte_val(*ptep))); 327 #endif 328 mtc0_tlbw_hazard(); 329 if (idx < 0) 330 tlb_write_random(); 331 else 332 tlb_write_indexed(); 333 } 334 tlbw_use_hazard(); 335 FLUSH_ITLB_VM(vma); 336 EXIT_CRITICAL(flags); 337 } 338 339 void add_wired_entry(unsigned long entrylo0, unsigned long entrylo1, 340 unsigned long entryhi, unsigned long pagemask) 341 { 342 unsigned long flags; 343 unsigned long wired; 344 unsigned long old_pagemask; 345 unsigned long old_ctx; 346 347 ENTER_CRITICAL(flags); 348 /* Save old context and create impossible VPN2 value */ 349 old_ctx = read_c0_entryhi(); 350 old_pagemask = read_c0_pagemask(); 351 wired = read_c0_wired(); 352 write_c0_wired(wired + 1); 353 write_c0_index(wired); 354 tlbw_use_hazard(); /* What is the hazard here? */ 355 write_c0_pagemask(pagemask); 356 write_c0_entryhi(entryhi); 357 write_c0_entrylo0(entrylo0); 358 write_c0_entrylo1(entrylo1); 359 mtc0_tlbw_hazard(); 360 tlb_write_indexed(); 361 tlbw_use_hazard(); 362 363 write_c0_entryhi(old_ctx); 364 tlbw_use_hazard(); /* What is the hazard here? */ 365 write_c0_pagemask(old_pagemask); 366 local_flush_tlb_all(); 367 EXIT_CRITICAL(flags); 368 } 369 370 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 371 372 int __init has_transparent_hugepage(void) 373 { 374 unsigned int mask; 375 unsigned long flags; 376 377 ENTER_CRITICAL(flags); 378 write_c0_pagemask(PM_HUGE_MASK); 379 back_to_back_c0_hazard(); 380 mask = read_c0_pagemask(); 381 write_c0_pagemask(PM_DEFAULT_MASK); 382 383 EXIT_CRITICAL(flags); 384 385 return mask == PM_HUGE_MASK; 386 } 387 388 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 389 390 static int __cpuinitdata ntlb; 391 static int __init set_ntlb(char *str) 392 { 393 get_option(&str, &ntlb); 394 return 1; 395 } 396 397 __setup("ntlb=", set_ntlb); 398 399 void __cpuinit tlb_init(void) 400 { 401 /* 402 * You should never change this register: 403 * - On R4600 1.7 the tlbp never hits for pages smaller than 404 * the value in the c0_pagemask register. 405 * - The entire mm handling assumes the c0_pagemask register to 406 * be set to fixed-size pages. 407 */ 408 write_c0_pagemask(PM_DEFAULT_MASK); 409 write_c0_wired(0); 410 if (current_cpu_type() == CPU_R10000 || 411 current_cpu_type() == CPU_R12000 || 412 current_cpu_type() == CPU_R14000) 413 write_c0_framemask(0); 414 415 if (cpu_has_rixi) { 416 /* 417 * Enable the no read, no exec bits, and enable large virtual 418 * address. 419 */ 420 u32 pg = PG_RIE | PG_XIE; 421 #ifdef CONFIG_64BIT 422 pg |= PG_ELPA; 423 #endif 424 write_c0_pagegrain(pg); 425 } 426 427 /* From this point on the ARC firmware is dead. */ 428 local_flush_tlb_all(); 429 430 /* Did I tell you that ARC SUCKS? */ 431 432 if (ntlb) { 433 if (ntlb > 1 && ntlb <= current_cpu_data.tlbsize) { 434 int wired = current_cpu_data.tlbsize - ntlb; 435 write_c0_wired(wired); 436 write_c0_index(wired-1); 437 printk("Restricting TLB to %d entries\n", ntlb); 438 } else 439 printk("Ignoring invalid argument ntlb=%d\n", ntlb); 440 } 441 442 build_tlb_refill_handler(); 443 } 444