1 /* 2 * This file is subject to the terms and conditions of the GNU General Public 3 * License. See the file "COPYING" in the main directory of this archive 4 * for more details. 5 * 6 * Copyright (C) 1996 David S. Miller (davem@davemloft.net) 7 * Copyright (C) 1997, 1998, 1999, 2000 Ralf Baechle ralf@gnu.org 8 * Carsten Langgaard, carstenl@mips.com 9 * Copyright (C) 2002 MIPS Technologies, Inc. All rights reserved. 10 */ 11 #include <linux/init.h> 12 #include <linux/sched.h> 13 #include <linux/smp.h> 14 #include <linux/mm.h> 15 #include <linux/hugetlb.h> 16 17 #include <asm/cpu.h> 18 #include <asm/bootinfo.h> 19 #include <asm/mmu_context.h> 20 #include <asm/pgtable.h> 21 #include <asm/tlbmisc.h> 22 23 extern void build_tlb_refill_handler(void); 24 25 /* 26 * Make sure all entries differ. If they're not different 27 * MIPS32 will take revenge ... 28 */ 29 #define UNIQUE_ENTRYHI(idx) (CKSEG0 + ((idx) << (PAGE_SHIFT + 1))) 30 31 /* Atomicity and interruptability */ 32 #ifdef CONFIG_MIPS_MT_SMTC 33 34 #include <asm/smtc.h> 35 #include <asm/mipsmtregs.h> 36 37 #define ENTER_CRITICAL(flags) \ 38 { \ 39 unsigned int mvpflags; \ 40 local_irq_save(flags);\ 41 mvpflags = dvpe() 42 #define EXIT_CRITICAL(flags) \ 43 evpe(mvpflags); \ 44 local_irq_restore(flags); \ 45 } 46 #else 47 48 #define ENTER_CRITICAL(flags) local_irq_save(flags) 49 #define EXIT_CRITICAL(flags) local_irq_restore(flags) 50 51 #endif /* CONFIG_MIPS_MT_SMTC */ 52 53 #if defined(CONFIG_CPU_LOONGSON2) 54 /* 55 * LOONGSON2 has a 4 entry itlb which is a subset of dtlb, 56 * unfortrunately, itlb is not totally transparent to software. 57 */ 58 #define FLUSH_ITLB write_c0_diag(4); 59 60 #define FLUSH_ITLB_VM(vma) { if ((vma)->vm_flags & VM_EXEC) write_c0_diag(4); } 61 62 #else 63 64 #define FLUSH_ITLB 65 #define FLUSH_ITLB_VM(vma) 66 67 #endif 68 69 void local_flush_tlb_all(void) 70 { 71 unsigned long flags; 72 unsigned long old_ctx; 73 int entry; 74 75 ENTER_CRITICAL(flags); 76 /* Save old context and create impossible VPN2 value */ 77 old_ctx = read_c0_entryhi(); 78 write_c0_entrylo0(0); 79 write_c0_entrylo1(0); 80 81 entry = read_c0_wired(); 82 83 /* Blast 'em all away. */ 84 while (entry < current_cpu_data.tlbsize) { 85 /* Make sure all entries differ. */ 86 write_c0_entryhi(UNIQUE_ENTRYHI(entry)); 87 write_c0_index(entry); 88 mtc0_tlbw_hazard(); 89 tlb_write_indexed(); 90 entry++; 91 } 92 tlbw_use_hazard(); 93 write_c0_entryhi(old_ctx); 94 FLUSH_ITLB; 95 EXIT_CRITICAL(flags); 96 } 97 98 /* All entries common to a mm share an asid. To effectively flush 99 these entries, we just bump the asid. */ 100 void local_flush_tlb_mm(struct mm_struct *mm) 101 { 102 int cpu; 103 104 preempt_disable(); 105 106 cpu = smp_processor_id(); 107 108 if (cpu_context(cpu, mm) != 0) { 109 drop_mmu_context(mm, cpu); 110 } 111 112 preempt_enable(); 113 } 114 115 void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, 116 unsigned long end) 117 { 118 struct mm_struct *mm = vma->vm_mm; 119 int cpu = smp_processor_id(); 120 121 if (cpu_context(cpu, mm) != 0) { 122 unsigned long size, flags; 123 int huge = is_vm_hugetlb_page(vma); 124 125 ENTER_CRITICAL(flags); 126 if (huge) { 127 start = round_down(start, HPAGE_SIZE); 128 end = round_up(end, HPAGE_SIZE); 129 size = (end - start) >> HPAGE_SHIFT; 130 } else { 131 start = round_down(start, PAGE_SIZE << 1); 132 end = round_up(end, PAGE_SIZE << 1); 133 size = (end - start) >> (PAGE_SHIFT + 1); 134 } 135 if (size <= current_cpu_data.tlbsize/2) { 136 int oldpid = read_c0_entryhi(); 137 int newpid = cpu_asid(cpu, mm); 138 139 while (start < end) { 140 int idx; 141 142 write_c0_entryhi(start | newpid); 143 if (huge) 144 start += HPAGE_SIZE; 145 else 146 start += (PAGE_SIZE << 1); 147 mtc0_tlbw_hazard(); 148 tlb_probe(); 149 tlb_probe_hazard(); 150 idx = read_c0_index(); 151 write_c0_entrylo0(0); 152 write_c0_entrylo1(0); 153 if (idx < 0) 154 continue; 155 /* Make sure all entries differ. */ 156 write_c0_entryhi(UNIQUE_ENTRYHI(idx)); 157 mtc0_tlbw_hazard(); 158 tlb_write_indexed(); 159 } 160 tlbw_use_hazard(); 161 write_c0_entryhi(oldpid); 162 } else { 163 drop_mmu_context(mm, cpu); 164 } 165 FLUSH_ITLB; 166 EXIT_CRITICAL(flags); 167 } 168 } 169 170 void local_flush_tlb_kernel_range(unsigned long start, unsigned long end) 171 { 172 unsigned long size, flags; 173 174 ENTER_CRITICAL(flags); 175 size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT; 176 size = (size + 1) >> 1; 177 if (size <= current_cpu_data.tlbsize / 2) { 178 int pid = read_c0_entryhi(); 179 180 start &= (PAGE_MASK << 1); 181 end += ((PAGE_SIZE << 1) - 1); 182 end &= (PAGE_MASK << 1); 183 184 while (start < end) { 185 int idx; 186 187 write_c0_entryhi(start); 188 start += (PAGE_SIZE << 1); 189 mtc0_tlbw_hazard(); 190 tlb_probe(); 191 tlb_probe_hazard(); 192 idx = read_c0_index(); 193 write_c0_entrylo0(0); 194 write_c0_entrylo1(0); 195 if (idx < 0) 196 continue; 197 /* Make sure all entries differ. */ 198 write_c0_entryhi(UNIQUE_ENTRYHI(idx)); 199 mtc0_tlbw_hazard(); 200 tlb_write_indexed(); 201 } 202 tlbw_use_hazard(); 203 write_c0_entryhi(pid); 204 } else { 205 local_flush_tlb_all(); 206 } 207 FLUSH_ITLB; 208 EXIT_CRITICAL(flags); 209 } 210 211 void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page) 212 { 213 int cpu = smp_processor_id(); 214 215 if (cpu_context(cpu, vma->vm_mm) != 0) { 216 unsigned long flags; 217 int oldpid, newpid, idx; 218 219 newpid = cpu_asid(cpu, vma->vm_mm); 220 page &= (PAGE_MASK << 1); 221 ENTER_CRITICAL(flags); 222 oldpid = read_c0_entryhi(); 223 write_c0_entryhi(page | newpid); 224 mtc0_tlbw_hazard(); 225 tlb_probe(); 226 tlb_probe_hazard(); 227 idx = read_c0_index(); 228 write_c0_entrylo0(0); 229 write_c0_entrylo1(0); 230 if (idx < 0) 231 goto finish; 232 /* Make sure all entries differ. */ 233 write_c0_entryhi(UNIQUE_ENTRYHI(idx)); 234 mtc0_tlbw_hazard(); 235 tlb_write_indexed(); 236 tlbw_use_hazard(); 237 238 finish: 239 write_c0_entryhi(oldpid); 240 FLUSH_ITLB_VM(vma); 241 EXIT_CRITICAL(flags); 242 } 243 } 244 245 /* 246 * This one is only used for pages with the global bit set so we don't care 247 * much about the ASID. 248 */ 249 void local_flush_tlb_one(unsigned long page) 250 { 251 unsigned long flags; 252 int oldpid, idx; 253 254 ENTER_CRITICAL(flags); 255 oldpid = read_c0_entryhi(); 256 page &= (PAGE_MASK << 1); 257 write_c0_entryhi(page); 258 mtc0_tlbw_hazard(); 259 tlb_probe(); 260 tlb_probe_hazard(); 261 idx = read_c0_index(); 262 write_c0_entrylo0(0); 263 write_c0_entrylo1(0); 264 if (idx >= 0) { 265 /* Make sure all entries differ. */ 266 write_c0_entryhi(UNIQUE_ENTRYHI(idx)); 267 mtc0_tlbw_hazard(); 268 tlb_write_indexed(); 269 tlbw_use_hazard(); 270 } 271 write_c0_entryhi(oldpid); 272 FLUSH_ITLB; 273 EXIT_CRITICAL(flags); 274 } 275 276 /* 277 * We will need multiple versions of update_mmu_cache(), one that just 278 * updates the TLB with the new pte(s), and another which also checks 279 * for the R4k "end of page" hardware bug and does the needy. 280 */ 281 void __update_tlb(struct vm_area_struct * vma, unsigned long address, pte_t pte) 282 { 283 unsigned long flags; 284 pgd_t *pgdp; 285 pud_t *pudp; 286 pmd_t *pmdp; 287 pte_t *ptep; 288 int idx, pid; 289 290 /* 291 * Handle debugger faulting in for debugee. 292 */ 293 if (current->active_mm != vma->vm_mm) 294 return; 295 296 ENTER_CRITICAL(flags); 297 298 pid = read_c0_entryhi() & ASID_MASK; 299 address &= (PAGE_MASK << 1); 300 write_c0_entryhi(address | pid); 301 pgdp = pgd_offset(vma->vm_mm, address); 302 mtc0_tlbw_hazard(); 303 tlb_probe(); 304 tlb_probe_hazard(); 305 pudp = pud_offset(pgdp, address); 306 pmdp = pmd_offset(pudp, address); 307 idx = read_c0_index(); 308 #ifdef CONFIG_HUGETLB_PAGE 309 /* this could be a huge page */ 310 if (pmd_huge(*pmdp)) { 311 unsigned long lo; 312 write_c0_pagemask(PM_HUGE_MASK); 313 ptep = (pte_t *)pmdp; 314 lo = pte_to_entrylo(pte_val(*ptep)); 315 write_c0_entrylo0(lo); 316 write_c0_entrylo1(lo + (HPAGE_SIZE >> 7)); 317 318 mtc0_tlbw_hazard(); 319 if (idx < 0) 320 tlb_write_random(); 321 else 322 tlb_write_indexed(); 323 tlbw_use_hazard(); 324 write_c0_pagemask(PM_DEFAULT_MASK); 325 } else 326 #endif 327 { 328 ptep = pte_offset_map(pmdp, address); 329 330 #if defined(CONFIG_64BIT_PHYS_ADDR) && defined(CONFIG_CPU_MIPS32) 331 write_c0_entrylo0(ptep->pte_high); 332 ptep++; 333 write_c0_entrylo1(ptep->pte_high); 334 #else 335 write_c0_entrylo0(pte_to_entrylo(pte_val(*ptep++))); 336 write_c0_entrylo1(pte_to_entrylo(pte_val(*ptep))); 337 #endif 338 mtc0_tlbw_hazard(); 339 if (idx < 0) 340 tlb_write_random(); 341 else 342 tlb_write_indexed(); 343 } 344 tlbw_use_hazard(); 345 FLUSH_ITLB_VM(vma); 346 EXIT_CRITICAL(flags); 347 } 348 349 void add_wired_entry(unsigned long entrylo0, unsigned long entrylo1, 350 unsigned long entryhi, unsigned long pagemask) 351 { 352 unsigned long flags; 353 unsigned long wired; 354 unsigned long old_pagemask; 355 unsigned long old_ctx; 356 357 ENTER_CRITICAL(flags); 358 /* Save old context and create impossible VPN2 value */ 359 old_ctx = read_c0_entryhi(); 360 old_pagemask = read_c0_pagemask(); 361 wired = read_c0_wired(); 362 write_c0_wired(wired + 1); 363 write_c0_index(wired); 364 tlbw_use_hazard(); /* What is the hazard here? */ 365 write_c0_pagemask(pagemask); 366 write_c0_entryhi(entryhi); 367 write_c0_entrylo0(entrylo0); 368 write_c0_entrylo1(entrylo1); 369 mtc0_tlbw_hazard(); 370 tlb_write_indexed(); 371 tlbw_use_hazard(); 372 373 write_c0_entryhi(old_ctx); 374 tlbw_use_hazard(); /* What is the hazard here? */ 375 write_c0_pagemask(old_pagemask); 376 local_flush_tlb_all(); 377 EXIT_CRITICAL(flags); 378 } 379 380 static int __cpuinitdata ntlb; 381 static int __init set_ntlb(char *str) 382 { 383 get_option(&str, &ntlb); 384 return 1; 385 } 386 387 __setup("ntlb=", set_ntlb); 388 389 void __cpuinit tlb_init(void) 390 { 391 /* 392 * You should never change this register: 393 * - On R4600 1.7 the tlbp never hits for pages smaller than 394 * the value in the c0_pagemask register. 395 * - The entire mm handling assumes the c0_pagemask register to 396 * be set to fixed-size pages. 397 */ 398 write_c0_pagemask(PM_DEFAULT_MASK); 399 write_c0_wired(0); 400 if (current_cpu_type() == CPU_R10000 || 401 current_cpu_type() == CPU_R12000 || 402 current_cpu_type() == CPU_R14000) 403 write_c0_framemask(0); 404 405 if (cpu_has_rixi) { 406 /* 407 * Enable the no read, no exec bits, and enable large virtual 408 * address. 409 */ 410 u32 pg = PG_RIE | PG_XIE; 411 #ifdef CONFIG_64BIT 412 pg |= PG_ELPA; 413 #endif 414 write_c0_pagegrain(pg); 415 } 416 417 /* From this point on the ARC firmware is dead. */ 418 local_flush_tlb_all(); 419 420 /* Did I tell you that ARC SUCKS? */ 421 422 if (ntlb) { 423 if (ntlb > 1 && ntlb <= current_cpu_data.tlbsize) { 424 int wired = current_cpu_data.tlbsize - ntlb; 425 write_c0_wired(wired); 426 write_c0_index(wired-1); 427 printk("Restricting TLB to %d entries\n", ntlb); 428 } else 429 printk("Ignoring invalid argument ntlb=%d\n", ntlb); 430 } 431 432 build_tlb_refill_handler(); 433 } 434