1 /* 2 * This file is subject to the terms and conditions of the GNU General Public 3 * License. See the file "COPYING" in the main directory of this archive 4 * for more details. 5 * 6 * Synthesize TLB refill handlers at runtime. 7 * 8 * Copyright (C) 2004, 2005, 2006, 2008 Thiemo Seufer 9 * Copyright (C) 2005, 2007, 2008, 2009 Maciej W. Rozycki 10 * Copyright (C) 2006 Ralf Baechle (ralf@linux-mips.org) 11 * Copyright (C) 2008, 2009 Cavium Networks, Inc. 12 * 13 * ... and the days got worse and worse and now you see 14 * I've gone completly out of my mind. 15 * 16 * They're coming to take me a away haha 17 * they're coming to take me a away hoho hihi haha 18 * to the funny farm where code is beautiful all the time ... 19 * 20 * (Condolences to Napoleon XIV) 21 */ 22 23 #include <linux/bug.h> 24 #include <linux/kernel.h> 25 #include <linux/types.h> 26 #include <linux/smp.h> 27 #include <linux/string.h> 28 #include <linux/init.h> 29 30 #include <asm/mmu_context.h> 31 #include <asm/war.h> 32 #include <asm/uasm.h> 33 34 /* 35 * TLB load/store/modify handlers. 36 * 37 * Only the fastpath gets synthesized at runtime, the slowpath for 38 * do_page_fault remains normal asm. 39 */ 40 extern void tlb_do_page_fault_0(void); 41 extern void tlb_do_page_fault_1(void); 42 43 44 static inline int r45k_bvahwbug(void) 45 { 46 /* XXX: We should probe for the presence of this bug, but we don't. */ 47 return 0; 48 } 49 50 static inline int r4k_250MHZhwbug(void) 51 { 52 /* XXX: We should probe for the presence of this bug, but we don't. */ 53 return 0; 54 } 55 56 static inline int __maybe_unused bcm1250_m3_war(void) 57 { 58 return BCM1250_M3_WAR; 59 } 60 61 static inline int __maybe_unused r10000_llsc_war(void) 62 { 63 return R10000_LLSC_WAR; 64 } 65 66 /* 67 * Found by experiment: At least some revisions of the 4kc throw under 68 * some circumstances a machine check exception, triggered by invalid 69 * values in the index register. Delaying the tlbp instruction until 70 * after the next branch, plus adding an additional nop in front of 71 * tlbwi/tlbwr avoids the invalid index register values. Nobody knows 72 * why; it's not an issue caused by the core RTL. 73 * 74 */ 75 static int __cpuinit m4kc_tlbp_war(void) 76 { 77 return (current_cpu_data.processor_id & 0xffff00) == 78 (PRID_COMP_MIPS | PRID_IMP_4KC); 79 } 80 81 /* Handle labels (which must be positive integers). */ 82 enum label_id { 83 label_second_part = 1, 84 label_leave, 85 label_vmalloc, 86 label_vmalloc_done, 87 label_tlbw_hazard, 88 label_split, 89 label_tlbl_goaround1, 90 label_tlbl_goaround2, 91 label_nopage_tlbl, 92 label_nopage_tlbs, 93 label_nopage_tlbm, 94 label_smp_pgtable_change, 95 label_r3000_write_probe_fail, 96 label_large_segbits_fault, 97 #ifdef CONFIG_HUGETLB_PAGE 98 label_tlb_huge_update, 99 #endif 100 }; 101 102 UASM_L_LA(_second_part) 103 UASM_L_LA(_leave) 104 UASM_L_LA(_vmalloc) 105 UASM_L_LA(_vmalloc_done) 106 UASM_L_LA(_tlbw_hazard) 107 UASM_L_LA(_split) 108 UASM_L_LA(_tlbl_goaround1) 109 UASM_L_LA(_tlbl_goaround2) 110 UASM_L_LA(_nopage_tlbl) 111 UASM_L_LA(_nopage_tlbs) 112 UASM_L_LA(_nopage_tlbm) 113 UASM_L_LA(_smp_pgtable_change) 114 UASM_L_LA(_r3000_write_probe_fail) 115 UASM_L_LA(_large_segbits_fault) 116 #ifdef CONFIG_HUGETLB_PAGE 117 UASM_L_LA(_tlb_huge_update) 118 #endif 119 120 /* 121 * For debug purposes. 122 */ 123 static inline void dump_handler(const u32 *handler, int count) 124 { 125 int i; 126 127 pr_debug("\t.set push\n"); 128 pr_debug("\t.set noreorder\n"); 129 130 for (i = 0; i < count; i++) 131 pr_debug("\t%p\t.word 0x%08x\n", &handler[i], handler[i]); 132 133 pr_debug("\t.set pop\n"); 134 } 135 136 /* The only general purpose registers allowed in TLB handlers. */ 137 #define K0 26 138 #define K1 27 139 140 /* Some CP0 registers */ 141 #define C0_INDEX 0, 0 142 #define C0_ENTRYLO0 2, 0 143 #define C0_TCBIND 2, 2 144 #define C0_ENTRYLO1 3, 0 145 #define C0_CONTEXT 4, 0 146 #define C0_PAGEMASK 5, 0 147 #define C0_BADVADDR 8, 0 148 #define C0_ENTRYHI 10, 0 149 #define C0_EPC 14, 0 150 #define C0_XCONTEXT 20, 0 151 152 #ifdef CONFIG_64BIT 153 # define GET_CONTEXT(buf, reg) UASM_i_MFC0(buf, reg, C0_XCONTEXT) 154 #else 155 # define GET_CONTEXT(buf, reg) UASM_i_MFC0(buf, reg, C0_CONTEXT) 156 #endif 157 158 /* The worst case length of the handler is around 18 instructions for 159 * R3000-style TLBs and up to 63 instructions for R4000-style TLBs. 160 * Maximum space available is 32 instructions for R3000 and 64 161 * instructions for R4000. 162 * 163 * We deliberately chose a buffer size of 128, so we won't scribble 164 * over anything important on overflow before we panic. 165 */ 166 static u32 tlb_handler[128] __cpuinitdata; 167 168 /* simply assume worst case size for labels and relocs */ 169 static struct uasm_label labels[128] __cpuinitdata; 170 static struct uasm_reloc relocs[128] __cpuinitdata; 171 172 #ifdef CONFIG_64BIT 173 static int check_for_high_segbits __cpuinitdata; 174 #endif 175 176 #ifndef CONFIG_MIPS_PGD_C0_CONTEXT 177 /* 178 * CONFIG_MIPS_PGD_C0_CONTEXT implies 64 bit and lack of pgd_current, 179 * we cannot do r3000 under these circumstances. 180 */ 181 182 /* 183 * The R3000 TLB handler is simple. 184 */ 185 static void __cpuinit build_r3000_tlb_refill_handler(void) 186 { 187 long pgdc = (long)pgd_current; 188 u32 *p; 189 190 memset(tlb_handler, 0, sizeof(tlb_handler)); 191 p = tlb_handler; 192 193 uasm_i_mfc0(&p, K0, C0_BADVADDR); 194 uasm_i_lui(&p, K1, uasm_rel_hi(pgdc)); /* cp0 delay */ 195 uasm_i_lw(&p, K1, uasm_rel_lo(pgdc), K1); 196 uasm_i_srl(&p, K0, K0, 22); /* load delay */ 197 uasm_i_sll(&p, K0, K0, 2); 198 uasm_i_addu(&p, K1, K1, K0); 199 uasm_i_mfc0(&p, K0, C0_CONTEXT); 200 uasm_i_lw(&p, K1, 0, K1); /* cp0 delay */ 201 uasm_i_andi(&p, K0, K0, 0xffc); /* load delay */ 202 uasm_i_addu(&p, K1, K1, K0); 203 uasm_i_lw(&p, K0, 0, K1); 204 uasm_i_nop(&p); /* load delay */ 205 uasm_i_mtc0(&p, K0, C0_ENTRYLO0); 206 uasm_i_mfc0(&p, K1, C0_EPC); /* cp0 delay */ 207 uasm_i_tlbwr(&p); /* cp0 delay */ 208 uasm_i_jr(&p, K1); 209 uasm_i_rfe(&p); /* branch delay */ 210 211 if (p > tlb_handler + 32) 212 panic("TLB refill handler space exceeded"); 213 214 pr_debug("Wrote TLB refill handler (%u instructions).\n", 215 (unsigned int)(p - tlb_handler)); 216 217 memcpy((void *)ebase, tlb_handler, 0x80); 218 219 dump_handler((u32 *)ebase, 32); 220 } 221 #endif /* CONFIG_MIPS_PGD_C0_CONTEXT */ 222 223 /* 224 * The R4000 TLB handler is much more complicated. We have two 225 * consecutive handler areas with 32 instructions space each. 226 * Since they aren't used at the same time, we can overflow in the 227 * other one.To keep things simple, we first assume linear space, 228 * then we relocate it to the final handler layout as needed. 229 */ 230 static u32 final_handler[64] __cpuinitdata; 231 232 /* 233 * Hazards 234 * 235 * From the IDT errata for the QED RM5230 (Nevada), processor revision 1.0: 236 * 2. A timing hazard exists for the TLBP instruction. 237 * 238 * stalling_instruction 239 * TLBP 240 * 241 * The JTLB is being read for the TLBP throughout the stall generated by the 242 * previous instruction. This is not really correct as the stalling instruction 243 * can modify the address used to access the JTLB. The failure symptom is that 244 * the TLBP instruction will use an address created for the stalling instruction 245 * and not the address held in C0_ENHI and thus report the wrong results. 246 * 247 * The software work-around is to not allow the instruction preceding the TLBP 248 * to stall - make it an NOP or some other instruction guaranteed not to stall. 249 * 250 * Errata 2 will not be fixed. This errata is also on the R5000. 251 * 252 * As if we MIPS hackers wouldn't know how to nop pipelines happy ... 253 */ 254 static void __cpuinit __maybe_unused build_tlb_probe_entry(u32 **p) 255 { 256 switch (current_cpu_type()) { 257 /* Found by experiment: R4600 v2.0/R4700 needs this, too. */ 258 case CPU_R4600: 259 case CPU_R4700: 260 case CPU_R5000: 261 case CPU_R5000A: 262 case CPU_NEVADA: 263 uasm_i_nop(p); 264 uasm_i_tlbp(p); 265 break; 266 267 default: 268 uasm_i_tlbp(p); 269 break; 270 } 271 } 272 273 /* 274 * Write random or indexed TLB entry, and care about the hazards from 275 * the preceeding mtc0 and for the following eret. 276 */ 277 enum tlb_write_entry { tlb_random, tlb_indexed }; 278 279 static void __cpuinit build_tlb_write_entry(u32 **p, struct uasm_label **l, 280 struct uasm_reloc **r, 281 enum tlb_write_entry wmode) 282 { 283 void(*tlbw)(u32 **) = NULL; 284 285 switch (wmode) { 286 case tlb_random: tlbw = uasm_i_tlbwr; break; 287 case tlb_indexed: tlbw = uasm_i_tlbwi; break; 288 } 289 290 if (cpu_has_mips_r2) { 291 if (cpu_has_mips_r2_exec_hazard) 292 uasm_i_ehb(p); 293 tlbw(p); 294 return; 295 } 296 297 switch (current_cpu_type()) { 298 case CPU_R4000PC: 299 case CPU_R4000SC: 300 case CPU_R4000MC: 301 case CPU_R4400PC: 302 case CPU_R4400SC: 303 case CPU_R4400MC: 304 /* 305 * This branch uses up a mtc0 hazard nop slot and saves 306 * two nops after the tlbw instruction. 307 */ 308 uasm_il_bgezl(p, r, 0, label_tlbw_hazard); 309 tlbw(p); 310 uasm_l_tlbw_hazard(l, *p); 311 uasm_i_nop(p); 312 break; 313 314 case CPU_R4600: 315 case CPU_R4700: 316 case CPU_R5000: 317 case CPU_R5000A: 318 uasm_i_nop(p); 319 tlbw(p); 320 uasm_i_nop(p); 321 break; 322 323 case CPU_R4300: 324 case CPU_5KC: 325 case CPU_TX49XX: 326 case CPU_PR4450: 327 uasm_i_nop(p); 328 tlbw(p); 329 break; 330 331 case CPU_R10000: 332 case CPU_R12000: 333 case CPU_R14000: 334 case CPU_4KC: 335 case CPU_4KEC: 336 case CPU_SB1: 337 case CPU_SB1A: 338 case CPU_4KSC: 339 case CPU_20KC: 340 case CPU_25KF: 341 case CPU_BCM3302: 342 case CPU_BCM4710: 343 case CPU_LOONGSON2: 344 case CPU_BCM6338: 345 case CPU_BCM6345: 346 case CPU_BCM6348: 347 case CPU_BCM6358: 348 case CPU_R5500: 349 if (m4kc_tlbp_war()) 350 uasm_i_nop(p); 351 case CPU_ALCHEMY: 352 tlbw(p); 353 break; 354 355 case CPU_NEVADA: 356 uasm_i_nop(p); /* QED specifies 2 nops hazard */ 357 /* 358 * This branch uses up a mtc0 hazard nop slot and saves 359 * a nop after the tlbw instruction. 360 */ 361 uasm_il_bgezl(p, r, 0, label_tlbw_hazard); 362 tlbw(p); 363 uasm_l_tlbw_hazard(l, *p); 364 break; 365 366 case CPU_RM7000: 367 uasm_i_nop(p); 368 uasm_i_nop(p); 369 uasm_i_nop(p); 370 uasm_i_nop(p); 371 tlbw(p); 372 break; 373 374 case CPU_RM9000: 375 /* 376 * When the JTLB is updated by tlbwi or tlbwr, a subsequent 377 * use of the JTLB for instructions should not occur for 4 378 * cpu cycles and use for data translations should not occur 379 * for 3 cpu cycles. 380 */ 381 uasm_i_ssnop(p); 382 uasm_i_ssnop(p); 383 uasm_i_ssnop(p); 384 uasm_i_ssnop(p); 385 tlbw(p); 386 uasm_i_ssnop(p); 387 uasm_i_ssnop(p); 388 uasm_i_ssnop(p); 389 uasm_i_ssnop(p); 390 break; 391 392 case CPU_VR4111: 393 case CPU_VR4121: 394 case CPU_VR4122: 395 case CPU_VR4181: 396 case CPU_VR4181A: 397 uasm_i_nop(p); 398 uasm_i_nop(p); 399 tlbw(p); 400 uasm_i_nop(p); 401 uasm_i_nop(p); 402 break; 403 404 case CPU_VR4131: 405 case CPU_VR4133: 406 case CPU_R5432: 407 uasm_i_nop(p); 408 uasm_i_nop(p); 409 tlbw(p); 410 break; 411 412 case CPU_JZRISC: 413 tlbw(p); 414 uasm_i_nop(p); 415 break; 416 417 default: 418 panic("No TLB refill handler yet (CPU type: %d)", 419 current_cpu_data.cputype); 420 break; 421 } 422 } 423 424 static __cpuinit __maybe_unused void build_convert_pte_to_entrylo(u32 **p, 425 unsigned int reg) 426 { 427 if (kernel_uses_smartmips_rixi) { 428 UASM_i_SRL(p, reg, reg, ilog2(_PAGE_NO_EXEC)); 429 UASM_i_ROTR(p, reg, reg, ilog2(_PAGE_GLOBAL) - ilog2(_PAGE_NO_EXEC)); 430 } else { 431 #ifdef CONFIG_64BIT_PHYS_ADDR 432 uasm_i_dsrl_safe(p, reg, reg, ilog2(_PAGE_GLOBAL)); 433 #else 434 UASM_i_SRL(p, reg, reg, ilog2(_PAGE_GLOBAL)); 435 #endif 436 } 437 } 438 439 #ifdef CONFIG_HUGETLB_PAGE 440 441 static __cpuinit void build_restore_pagemask(u32 **p, 442 struct uasm_reloc **r, 443 unsigned int tmp, 444 enum label_id lid) 445 { 446 /* Reset default page size */ 447 if (PM_DEFAULT_MASK >> 16) { 448 uasm_i_lui(p, tmp, PM_DEFAULT_MASK >> 16); 449 uasm_i_ori(p, tmp, tmp, PM_DEFAULT_MASK & 0xffff); 450 uasm_il_b(p, r, lid); 451 uasm_i_mtc0(p, tmp, C0_PAGEMASK); 452 } else if (PM_DEFAULT_MASK) { 453 uasm_i_ori(p, tmp, 0, PM_DEFAULT_MASK); 454 uasm_il_b(p, r, lid); 455 uasm_i_mtc0(p, tmp, C0_PAGEMASK); 456 } else { 457 uasm_il_b(p, r, lid); 458 uasm_i_mtc0(p, 0, C0_PAGEMASK); 459 } 460 } 461 462 static __cpuinit void build_huge_tlb_write_entry(u32 **p, 463 struct uasm_label **l, 464 struct uasm_reloc **r, 465 unsigned int tmp, 466 enum tlb_write_entry wmode) 467 { 468 /* Set huge page tlb entry size */ 469 uasm_i_lui(p, tmp, PM_HUGE_MASK >> 16); 470 uasm_i_ori(p, tmp, tmp, PM_HUGE_MASK & 0xffff); 471 uasm_i_mtc0(p, tmp, C0_PAGEMASK); 472 473 build_tlb_write_entry(p, l, r, wmode); 474 475 build_restore_pagemask(p, r, tmp, label_leave); 476 } 477 478 /* 479 * Check if Huge PTE is present, if so then jump to LABEL. 480 */ 481 static void __cpuinit 482 build_is_huge_pte(u32 **p, struct uasm_reloc **r, unsigned int tmp, 483 unsigned int pmd, int lid) 484 { 485 UASM_i_LW(p, tmp, 0, pmd); 486 uasm_i_andi(p, tmp, tmp, _PAGE_HUGE); 487 uasm_il_bnez(p, r, tmp, lid); 488 } 489 490 static __cpuinit void build_huge_update_entries(u32 **p, 491 unsigned int pte, 492 unsigned int tmp) 493 { 494 int small_sequence; 495 496 /* 497 * A huge PTE describes an area the size of the 498 * configured huge page size. This is twice the 499 * of the large TLB entry size we intend to use. 500 * A TLB entry half the size of the configured 501 * huge page size is configured into entrylo0 502 * and entrylo1 to cover the contiguous huge PTE 503 * address space. 504 */ 505 small_sequence = (HPAGE_SIZE >> 7) < 0x10000; 506 507 /* We can clobber tmp. It isn't used after this.*/ 508 if (!small_sequence) 509 uasm_i_lui(p, tmp, HPAGE_SIZE >> (7 + 16)); 510 511 build_convert_pte_to_entrylo(p, pte); 512 UASM_i_MTC0(p, pte, C0_ENTRYLO0); /* load it */ 513 /* convert to entrylo1 */ 514 if (small_sequence) 515 UASM_i_ADDIU(p, pte, pte, HPAGE_SIZE >> 7); 516 else 517 UASM_i_ADDU(p, pte, pte, tmp); 518 519 UASM_i_MTC0(p, pte, C0_ENTRYLO1); /* load it */ 520 } 521 522 static __cpuinit void build_huge_handler_tail(u32 **p, 523 struct uasm_reloc **r, 524 struct uasm_label **l, 525 unsigned int pte, 526 unsigned int ptr) 527 { 528 #ifdef CONFIG_SMP 529 UASM_i_SC(p, pte, 0, ptr); 530 uasm_il_beqz(p, r, pte, label_tlb_huge_update); 531 UASM_i_LW(p, pte, 0, ptr); /* Needed because SC killed our PTE */ 532 #else 533 UASM_i_SW(p, pte, 0, ptr); 534 #endif 535 build_huge_update_entries(p, pte, ptr); 536 build_huge_tlb_write_entry(p, l, r, pte, tlb_indexed); 537 } 538 #endif /* CONFIG_HUGETLB_PAGE */ 539 540 #ifdef CONFIG_64BIT 541 /* 542 * TMP and PTR are scratch. 543 * TMP will be clobbered, PTR will hold the pmd entry. 544 */ 545 static void __cpuinit 546 build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r, 547 unsigned int tmp, unsigned int ptr) 548 { 549 #ifndef CONFIG_MIPS_PGD_C0_CONTEXT 550 long pgdc = (long)pgd_current; 551 #endif 552 /* 553 * The vmalloc handling is not in the hotpath. 554 */ 555 uasm_i_dmfc0(p, tmp, C0_BADVADDR); 556 557 if (check_for_high_segbits) { 558 /* 559 * The kernel currently implicitely assumes that the 560 * MIPS SEGBITS parameter for the processor is 561 * (PGDIR_SHIFT+PGDIR_BITS) or less, and will never 562 * allocate virtual addresses outside the maximum 563 * range for SEGBITS = (PGDIR_SHIFT+PGDIR_BITS). But 564 * that doesn't prevent user code from accessing the 565 * higher xuseg addresses. Here, we make sure that 566 * everything but the lower xuseg addresses goes down 567 * the module_alloc/vmalloc path. 568 */ 569 uasm_i_dsrl_safe(p, ptr, tmp, PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3); 570 uasm_il_bnez(p, r, ptr, label_vmalloc); 571 } else { 572 uasm_il_bltz(p, r, tmp, label_vmalloc); 573 } 574 /* No uasm_i_nop needed here, since the next insn doesn't touch TMP. */ 575 576 #ifdef CONFIG_MIPS_PGD_C0_CONTEXT 577 /* 578 * &pgd << 11 stored in CONTEXT [23..63]. 579 */ 580 UASM_i_MFC0(p, ptr, C0_CONTEXT); 581 uasm_i_dins(p, ptr, 0, 0, 23); /* Clear lower 23 bits of context. */ 582 uasm_i_ori(p, ptr, ptr, 0x540); /* 1 0 1 0 1 << 6 xkphys cached */ 583 uasm_i_drotr(p, ptr, ptr, 11); 584 #elif defined(CONFIG_SMP) 585 # ifdef CONFIG_MIPS_MT_SMTC 586 /* 587 * SMTC uses TCBind value as "CPU" index 588 */ 589 uasm_i_mfc0(p, ptr, C0_TCBIND); 590 uasm_i_dsrl_safe(p, ptr, ptr, 19); 591 # else 592 /* 593 * 64 bit SMP running in XKPHYS has smp_processor_id() << 3 594 * stored in CONTEXT. 595 */ 596 uasm_i_dmfc0(p, ptr, C0_CONTEXT); 597 uasm_i_dsrl_safe(p, ptr, ptr, 23); 598 # endif 599 UASM_i_LA_mostly(p, tmp, pgdc); 600 uasm_i_daddu(p, ptr, ptr, tmp); 601 uasm_i_dmfc0(p, tmp, C0_BADVADDR); 602 uasm_i_ld(p, ptr, uasm_rel_lo(pgdc), ptr); 603 #else 604 UASM_i_LA_mostly(p, ptr, pgdc); 605 uasm_i_ld(p, ptr, uasm_rel_lo(pgdc), ptr); 606 #endif 607 608 uasm_l_vmalloc_done(l, *p); 609 610 /* get pgd offset in bytes */ 611 uasm_i_dsrl_safe(p, tmp, tmp, PGDIR_SHIFT - 3); 612 613 uasm_i_andi(p, tmp, tmp, (PTRS_PER_PGD - 1)<<3); 614 uasm_i_daddu(p, ptr, ptr, tmp); /* add in pgd offset */ 615 #ifndef __PAGETABLE_PMD_FOLDED 616 uasm_i_dmfc0(p, tmp, C0_BADVADDR); /* get faulting address */ 617 uasm_i_ld(p, ptr, 0, ptr); /* get pmd pointer */ 618 uasm_i_dsrl_safe(p, tmp, tmp, PMD_SHIFT-3); /* get pmd offset in bytes */ 619 uasm_i_andi(p, tmp, tmp, (PTRS_PER_PMD - 1)<<3); 620 uasm_i_daddu(p, ptr, ptr, tmp); /* add in pmd offset */ 621 #endif 622 } 623 624 enum vmalloc64_mode {not_refill, refill}; 625 /* 626 * BVADDR is the faulting address, PTR is scratch. 627 * PTR will hold the pgd for vmalloc. 628 */ 629 static void __cpuinit 630 build_get_pgd_vmalloc64(u32 **p, struct uasm_label **l, struct uasm_reloc **r, 631 unsigned int bvaddr, unsigned int ptr, 632 enum vmalloc64_mode mode) 633 { 634 long swpd = (long)swapper_pg_dir; 635 int single_insn_swpd; 636 int did_vmalloc_branch = 0; 637 638 single_insn_swpd = uasm_in_compat_space_p(swpd) && !uasm_rel_lo(swpd); 639 640 uasm_l_vmalloc(l, *p); 641 642 if (mode == refill && check_for_high_segbits) { 643 if (single_insn_swpd) { 644 uasm_il_bltz(p, r, bvaddr, label_vmalloc_done); 645 uasm_i_lui(p, ptr, uasm_rel_hi(swpd)); 646 did_vmalloc_branch = 1; 647 /* fall through */ 648 } else { 649 uasm_il_bgez(p, r, bvaddr, label_large_segbits_fault); 650 } 651 } 652 if (!did_vmalloc_branch) { 653 if (uasm_in_compat_space_p(swpd) && !uasm_rel_lo(swpd)) { 654 uasm_il_b(p, r, label_vmalloc_done); 655 uasm_i_lui(p, ptr, uasm_rel_hi(swpd)); 656 } else { 657 UASM_i_LA_mostly(p, ptr, swpd); 658 uasm_il_b(p, r, label_vmalloc_done); 659 if (uasm_in_compat_space_p(swpd)) 660 uasm_i_addiu(p, ptr, ptr, uasm_rel_lo(swpd)); 661 else 662 uasm_i_daddiu(p, ptr, ptr, uasm_rel_lo(swpd)); 663 } 664 } 665 if (mode == refill && check_for_high_segbits) { 666 uasm_l_large_segbits_fault(l, *p); 667 /* 668 * We get here if we are an xsseg address, or if we are 669 * an xuseg address above (PGDIR_SHIFT+PGDIR_BITS) boundary. 670 * 671 * Ignoring xsseg (assume disabled so would generate 672 * (address errors?), the only remaining possibility 673 * is the upper xuseg addresses. On processors with 674 * TLB_SEGBITS <= PGDIR_SHIFT+PGDIR_BITS, these 675 * addresses would have taken an address error. We try 676 * to mimic that here by taking a load/istream page 677 * fault. 678 */ 679 UASM_i_LA(p, ptr, (unsigned long)tlb_do_page_fault_0); 680 uasm_i_jr(p, ptr); 681 uasm_i_nop(p); 682 } 683 } 684 685 #else /* !CONFIG_64BIT */ 686 687 /* 688 * TMP and PTR are scratch. 689 * TMP will be clobbered, PTR will hold the pgd entry. 690 */ 691 static void __cpuinit __maybe_unused 692 build_get_pgde32(u32 **p, unsigned int tmp, unsigned int ptr) 693 { 694 long pgdc = (long)pgd_current; 695 696 /* 32 bit SMP has smp_processor_id() stored in CONTEXT. */ 697 #ifdef CONFIG_SMP 698 #ifdef CONFIG_MIPS_MT_SMTC 699 /* 700 * SMTC uses TCBind value as "CPU" index 701 */ 702 uasm_i_mfc0(p, ptr, C0_TCBIND); 703 UASM_i_LA_mostly(p, tmp, pgdc); 704 uasm_i_srl(p, ptr, ptr, 19); 705 #else 706 /* 707 * smp_processor_id() << 3 is stored in CONTEXT. 708 */ 709 uasm_i_mfc0(p, ptr, C0_CONTEXT); 710 UASM_i_LA_mostly(p, tmp, pgdc); 711 uasm_i_srl(p, ptr, ptr, 23); 712 #endif 713 uasm_i_addu(p, ptr, tmp, ptr); 714 #else 715 UASM_i_LA_mostly(p, ptr, pgdc); 716 #endif 717 uasm_i_mfc0(p, tmp, C0_BADVADDR); /* get faulting address */ 718 uasm_i_lw(p, ptr, uasm_rel_lo(pgdc), ptr); 719 uasm_i_srl(p, tmp, tmp, PGDIR_SHIFT); /* get pgd only bits */ 720 uasm_i_sll(p, tmp, tmp, PGD_T_LOG2); 721 uasm_i_addu(p, ptr, ptr, tmp); /* add in pgd offset */ 722 } 723 724 #endif /* !CONFIG_64BIT */ 725 726 static void __cpuinit build_adjust_context(u32 **p, unsigned int ctx) 727 { 728 unsigned int shift = 4 - (PTE_T_LOG2 + 1) + PAGE_SHIFT - 12; 729 unsigned int mask = (PTRS_PER_PTE / 2 - 1) << (PTE_T_LOG2 + 1); 730 731 switch (current_cpu_type()) { 732 case CPU_VR41XX: 733 case CPU_VR4111: 734 case CPU_VR4121: 735 case CPU_VR4122: 736 case CPU_VR4131: 737 case CPU_VR4181: 738 case CPU_VR4181A: 739 case CPU_VR4133: 740 shift += 2; 741 break; 742 743 default: 744 break; 745 } 746 747 if (shift) 748 UASM_i_SRL(p, ctx, ctx, shift); 749 uasm_i_andi(p, ctx, ctx, mask); 750 } 751 752 static void __cpuinit build_get_ptep(u32 **p, unsigned int tmp, unsigned int ptr) 753 { 754 /* 755 * Bug workaround for the Nevada. It seems as if under certain 756 * circumstances the move from cp0_context might produce a 757 * bogus result when the mfc0 instruction and its consumer are 758 * in a different cacheline or a load instruction, probably any 759 * memory reference, is between them. 760 */ 761 switch (current_cpu_type()) { 762 case CPU_NEVADA: 763 UASM_i_LW(p, ptr, 0, ptr); 764 GET_CONTEXT(p, tmp); /* get context reg */ 765 break; 766 767 default: 768 GET_CONTEXT(p, tmp); /* get context reg */ 769 UASM_i_LW(p, ptr, 0, ptr); 770 break; 771 } 772 773 build_adjust_context(p, tmp); 774 UASM_i_ADDU(p, ptr, ptr, tmp); /* add in offset */ 775 } 776 777 static void __cpuinit build_update_entries(u32 **p, unsigned int tmp, 778 unsigned int ptep) 779 { 780 /* 781 * 64bit address support (36bit on a 32bit CPU) in a 32bit 782 * Kernel is a special case. Only a few CPUs use it. 783 */ 784 #ifdef CONFIG_64BIT_PHYS_ADDR 785 if (cpu_has_64bits) { 786 uasm_i_ld(p, tmp, 0, ptep); /* get even pte */ 787 uasm_i_ld(p, ptep, sizeof(pte_t), ptep); /* get odd pte */ 788 if (kernel_uses_smartmips_rixi) { 789 UASM_i_SRL(p, tmp, tmp, ilog2(_PAGE_NO_EXEC)); 790 UASM_i_SRL(p, ptep, ptep, ilog2(_PAGE_NO_EXEC)); 791 UASM_i_ROTR(p, tmp, tmp, ilog2(_PAGE_GLOBAL) - ilog2(_PAGE_NO_EXEC)); 792 UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */ 793 UASM_i_ROTR(p, ptep, ptep, ilog2(_PAGE_GLOBAL) - ilog2(_PAGE_NO_EXEC)); 794 } else { 795 uasm_i_dsrl_safe(p, tmp, tmp, ilog2(_PAGE_GLOBAL)); /* convert to entrylo0 */ 796 UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */ 797 uasm_i_dsrl_safe(p, ptep, ptep, ilog2(_PAGE_GLOBAL)); /* convert to entrylo1 */ 798 } 799 UASM_i_MTC0(p, ptep, C0_ENTRYLO1); /* load it */ 800 } else { 801 int pte_off_even = sizeof(pte_t) / 2; 802 int pte_off_odd = pte_off_even + sizeof(pte_t); 803 804 /* The pte entries are pre-shifted */ 805 uasm_i_lw(p, tmp, pte_off_even, ptep); /* get even pte */ 806 UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */ 807 uasm_i_lw(p, ptep, pte_off_odd, ptep); /* get odd pte */ 808 UASM_i_MTC0(p, ptep, C0_ENTRYLO1); /* load it */ 809 } 810 #else 811 UASM_i_LW(p, tmp, 0, ptep); /* get even pte */ 812 UASM_i_LW(p, ptep, sizeof(pte_t), ptep); /* get odd pte */ 813 if (r45k_bvahwbug()) 814 build_tlb_probe_entry(p); 815 if (kernel_uses_smartmips_rixi) { 816 UASM_i_SRL(p, tmp, tmp, ilog2(_PAGE_NO_EXEC)); 817 UASM_i_SRL(p, ptep, ptep, ilog2(_PAGE_NO_EXEC)); 818 UASM_i_ROTR(p, tmp, tmp, ilog2(_PAGE_GLOBAL) - ilog2(_PAGE_NO_EXEC)); 819 if (r4k_250MHZhwbug()) 820 UASM_i_MTC0(p, 0, C0_ENTRYLO0); 821 UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */ 822 UASM_i_ROTR(p, ptep, ptep, ilog2(_PAGE_GLOBAL) - ilog2(_PAGE_NO_EXEC)); 823 } else { 824 UASM_i_SRL(p, tmp, tmp, ilog2(_PAGE_GLOBAL)); /* convert to entrylo0 */ 825 if (r4k_250MHZhwbug()) 826 UASM_i_MTC0(p, 0, C0_ENTRYLO0); 827 UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */ 828 UASM_i_SRL(p, ptep, ptep, ilog2(_PAGE_GLOBAL)); /* convert to entrylo1 */ 829 if (r45k_bvahwbug()) 830 uasm_i_mfc0(p, tmp, C0_INDEX); 831 } 832 if (r4k_250MHZhwbug()) 833 UASM_i_MTC0(p, 0, C0_ENTRYLO1); 834 UASM_i_MTC0(p, ptep, C0_ENTRYLO1); /* load it */ 835 #endif 836 } 837 838 /* 839 * For a 64-bit kernel, we are using the 64-bit XTLB refill exception 840 * because EXL == 0. If we wrap, we can also use the 32 instruction 841 * slots before the XTLB refill exception handler which belong to the 842 * unused TLB refill exception. 843 */ 844 #define MIPS64_REFILL_INSNS 32 845 846 static void __cpuinit build_r4000_tlb_refill_handler(void) 847 { 848 u32 *p = tlb_handler; 849 struct uasm_label *l = labels; 850 struct uasm_reloc *r = relocs; 851 u32 *f; 852 unsigned int final_len; 853 854 memset(tlb_handler, 0, sizeof(tlb_handler)); 855 memset(labels, 0, sizeof(labels)); 856 memset(relocs, 0, sizeof(relocs)); 857 memset(final_handler, 0, sizeof(final_handler)); 858 859 /* 860 * create the plain linear handler 861 */ 862 if (bcm1250_m3_war()) { 863 unsigned int segbits = 44; 864 865 uasm_i_dmfc0(&p, K0, C0_BADVADDR); 866 uasm_i_dmfc0(&p, K1, C0_ENTRYHI); 867 uasm_i_xor(&p, K0, K0, K1); 868 uasm_i_dsrl_safe(&p, K1, K0, 62); 869 uasm_i_dsrl_safe(&p, K0, K0, 12 + 1); 870 uasm_i_dsll_safe(&p, K0, K0, 64 + 12 + 1 - segbits); 871 uasm_i_or(&p, K0, K0, K1); 872 uasm_il_bnez(&p, &r, K0, label_leave); 873 /* No need for uasm_i_nop */ 874 } 875 876 #ifdef CONFIG_64BIT 877 build_get_pmde64(&p, &l, &r, K0, K1); /* get pmd in K1 */ 878 #else 879 build_get_pgde32(&p, K0, K1); /* get pgd in K1 */ 880 #endif 881 882 #ifdef CONFIG_HUGETLB_PAGE 883 build_is_huge_pte(&p, &r, K0, K1, label_tlb_huge_update); 884 #endif 885 886 build_get_ptep(&p, K0, K1); 887 build_update_entries(&p, K0, K1); 888 build_tlb_write_entry(&p, &l, &r, tlb_random); 889 uasm_l_leave(&l, p); 890 uasm_i_eret(&p); /* return from trap */ 891 892 #ifdef CONFIG_HUGETLB_PAGE 893 uasm_l_tlb_huge_update(&l, p); 894 UASM_i_LW(&p, K0, 0, K1); 895 build_huge_update_entries(&p, K0, K1); 896 build_huge_tlb_write_entry(&p, &l, &r, K0, tlb_random); 897 #endif 898 899 #ifdef CONFIG_64BIT 900 build_get_pgd_vmalloc64(&p, &l, &r, K0, K1, refill); 901 #endif 902 903 /* 904 * Overflow check: For the 64bit handler, we need at least one 905 * free instruction slot for the wrap-around branch. In worst 906 * case, if the intended insertion point is a delay slot, we 907 * need three, with the second nop'ed and the third being 908 * unused. 909 */ 910 /* Loongson2 ebase is different than r4k, we have more space */ 911 #if defined(CONFIG_32BIT) || defined(CONFIG_CPU_LOONGSON2) 912 if ((p - tlb_handler) > 64) 913 panic("TLB refill handler space exceeded"); 914 #else 915 if (((p - tlb_handler) > (MIPS64_REFILL_INSNS * 2) - 1) 916 || (((p - tlb_handler) > (MIPS64_REFILL_INSNS * 2) - 3) 917 && uasm_insn_has_bdelay(relocs, 918 tlb_handler + MIPS64_REFILL_INSNS - 3))) 919 panic("TLB refill handler space exceeded"); 920 #endif 921 922 /* 923 * Now fold the handler in the TLB refill handler space. 924 */ 925 #if defined(CONFIG_32BIT) || defined(CONFIG_CPU_LOONGSON2) 926 f = final_handler; 927 /* Simplest case, just copy the handler. */ 928 uasm_copy_handler(relocs, labels, tlb_handler, p, f); 929 final_len = p - tlb_handler; 930 #else /* CONFIG_64BIT */ 931 f = final_handler + MIPS64_REFILL_INSNS; 932 if ((p - tlb_handler) <= MIPS64_REFILL_INSNS) { 933 /* Just copy the handler. */ 934 uasm_copy_handler(relocs, labels, tlb_handler, p, f); 935 final_len = p - tlb_handler; 936 } else { 937 #if defined(CONFIG_HUGETLB_PAGE) 938 const enum label_id ls = label_tlb_huge_update; 939 #else 940 const enum label_id ls = label_vmalloc; 941 #endif 942 u32 *split; 943 int ov = 0; 944 int i; 945 946 for (i = 0; i < ARRAY_SIZE(labels) && labels[i].lab != ls; i++) 947 ; 948 BUG_ON(i == ARRAY_SIZE(labels)); 949 split = labels[i].addr; 950 951 /* 952 * See if we have overflown one way or the other. 953 */ 954 if (split > tlb_handler + MIPS64_REFILL_INSNS || 955 split < p - MIPS64_REFILL_INSNS) 956 ov = 1; 957 958 if (ov) { 959 /* 960 * Split two instructions before the end. One 961 * for the branch and one for the instruction 962 * in the delay slot. 963 */ 964 split = tlb_handler + MIPS64_REFILL_INSNS - 2; 965 966 /* 967 * If the branch would fall in a delay slot, 968 * we must back up an additional instruction 969 * so that it is no longer in a delay slot. 970 */ 971 if (uasm_insn_has_bdelay(relocs, split - 1)) 972 split--; 973 } 974 /* Copy first part of the handler. */ 975 uasm_copy_handler(relocs, labels, tlb_handler, split, f); 976 f += split - tlb_handler; 977 978 if (ov) { 979 /* Insert branch. */ 980 uasm_l_split(&l, final_handler); 981 uasm_il_b(&f, &r, label_split); 982 if (uasm_insn_has_bdelay(relocs, split)) 983 uasm_i_nop(&f); 984 else { 985 uasm_copy_handler(relocs, labels, 986 split, split + 1, f); 987 uasm_move_labels(labels, f, f + 1, -1); 988 f++; 989 split++; 990 } 991 } 992 993 /* Copy the rest of the handler. */ 994 uasm_copy_handler(relocs, labels, split, p, final_handler); 995 final_len = (f - (final_handler + MIPS64_REFILL_INSNS)) + 996 (p - split); 997 } 998 #endif /* CONFIG_64BIT */ 999 1000 uasm_resolve_relocs(relocs, labels); 1001 pr_debug("Wrote TLB refill handler (%u instructions).\n", 1002 final_len); 1003 1004 memcpy((void *)ebase, final_handler, 0x100); 1005 1006 dump_handler((u32 *)ebase, 64); 1007 } 1008 1009 /* 1010 * 128 instructions for the fastpath handler is generous and should 1011 * never be exceeded. 1012 */ 1013 #define FASTPATH_SIZE 128 1014 1015 u32 handle_tlbl[FASTPATH_SIZE] __cacheline_aligned; 1016 u32 handle_tlbs[FASTPATH_SIZE] __cacheline_aligned; 1017 u32 handle_tlbm[FASTPATH_SIZE] __cacheline_aligned; 1018 1019 static void __cpuinit 1020 iPTE_LW(u32 **p, unsigned int pte, unsigned int ptr) 1021 { 1022 #ifdef CONFIG_SMP 1023 # ifdef CONFIG_64BIT_PHYS_ADDR 1024 if (cpu_has_64bits) 1025 uasm_i_lld(p, pte, 0, ptr); 1026 else 1027 # endif 1028 UASM_i_LL(p, pte, 0, ptr); 1029 #else 1030 # ifdef CONFIG_64BIT_PHYS_ADDR 1031 if (cpu_has_64bits) 1032 uasm_i_ld(p, pte, 0, ptr); 1033 else 1034 # endif 1035 UASM_i_LW(p, pte, 0, ptr); 1036 #endif 1037 } 1038 1039 static void __cpuinit 1040 iPTE_SW(u32 **p, struct uasm_reloc **r, unsigned int pte, unsigned int ptr, 1041 unsigned int mode) 1042 { 1043 #ifdef CONFIG_64BIT_PHYS_ADDR 1044 unsigned int hwmode = mode & (_PAGE_VALID | _PAGE_DIRTY); 1045 #endif 1046 1047 uasm_i_ori(p, pte, pte, mode); 1048 #ifdef CONFIG_SMP 1049 # ifdef CONFIG_64BIT_PHYS_ADDR 1050 if (cpu_has_64bits) 1051 uasm_i_scd(p, pte, 0, ptr); 1052 else 1053 # endif 1054 UASM_i_SC(p, pte, 0, ptr); 1055 1056 if (r10000_llsc_war()) 1057 uasm_il_beqzl(p, r, pte, label_smp_pgtable_change); 1058 else 1059 uasm_il_beqz(p, r, pte, label_smp_pgtable_change); 1060 1061 # ifdef CONFIG_64BIT_PHYS_ADDR 1062 if (!cpu_has_64bits) { 1063 /* no uasm_i_nop needed */ 1064 uasm_i_ll(p, pte, sizeof(pte_t) / 2, ptr); 1065 uasm_i_ori(p, pte, pte, hwmode); 1066 uasm_i_sc(p, pte, sizeof(pte_t) / 2, ptr); 1067 uasm_il_beqz(p, r, pte, label_smp_pgtable_change); 1068 /* no uasm_i_nop needed */ 1069 uasm_i_lw(p, pte, 0, ptr); 1070 } else 1071 uasm_i_nop(p); 1072 # else 1073 uasm_i_nop(p); 1074 # endif 1075 #else 1076 # ifdef CONFIG_64BIT_PHYS_ADDR 1077 if (cpu_has_64bits) 1078 uasm_i_sd(p, pte, 0, ptr); 1079 else 1080 # endif 1081 UASM_i_SW(p, pte, 0, ptr); 1082 1083 # ifdef CONFIG_64BIT_PHYS_ADDR 1084 if (!cpu_has_64bits) { 1085 uasm_i_lw(p, pte, sizeof(pte_t) / 2, ptr); 1086 uasm_i_ori(p, pte, pte, hwmode); 1087 uasm_i_sw(p, pte, sizeof(pte_t) / 2, ptr); 1088 uasm_i_lw(p, pte, 0, ptr); 1089 } 1090 # endif 1091 #endif 1092 } 1093 1094 /* 1095 * Check if PTE is present, if not then jump to LABEL. PTR points to 1096 * the page table where this PTE is located, PTE will be re-loaded 1097 * with it's original value. 1098 */ 1099 static void __cpuinit 1100 build_pte_present(u32 **p, struct uasm_reloc **r, 1101 unsigned int pte, unsigned int ptr, enum label_id lid) 1102 { 1103 if (kernel_uses_smartmips_rixi) { 1104 uasm_i_andi(p, pte, pte, _PAGE_PRESENT); 1105 uasm_il_beqz(p, r, pte, lid); 1106 } else { 1107 uasm_i_andi(p, pte, pte, _PAGE_PRESENT | _PAGE_READ); 1108 uasm_i_xori(p, pte, pte, _PAGE_PRESENT | _PAGE_READ); 1109 uasm_il_bnez(p, r, pte, lid); 1110 } 1111 iPTE_LW(p, pte, ptr); 1112 } 1113 1114 /* Make PTE valid, store result in PTR. */ 1115 static void __cpuinit 1116 build_make_valid(u32 **p, struct uasm_reloc **r, unsigned int pte, 1117 unsigned int ptr) 1118 { 1119 unsigned int mode = _PAGE_VALID | _PAGE_ACCESSED; 1120 1121 iPTE_SW(p, r, pte, ptr, mode); 1122 } 1123 1124 /* 1125 * Check if PTE can be written to, if not branch to LABEL. Regardless 1126 * restore PTE with value from PTR when done. 1127 */ 1128 static void __cpuinit 1129 build_pte_writable(u32 **p, struct uasm_reloc **r, 1130 unsigned int pte, unsigned int ptr, enum label_id lid) 1131 { 1132 uasm_i_andi(p, pte, pte, _PAGE_PRESENT | _PAGE_WRITE); 1133 uasm_i_xori(p, pte, pte, _PAGE_PRESENT | _PAGE_WRITE); 1134 uasm_il_bnez(p, r, pte, lid); 1135 iPTE_LW(p, pte, ptr); 1136 } 1137 1138 /* Make PTE writable, update software status bits as well, then store 1139 * at PTR. 1140 */ 1141 static void __cpuinit 1142 build_make_write(u32 **p, struct uasm_reloc **r, unsigned int pte, 1143 unsigned int ptr) 1144 { 1145 unsigned int mode = (_PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID 1146 | _PAGE_DIRTY); 1147 1148 iPTE_SW(p, r, pte, ptr, mode); 1149 } 1150 1151 /* 1152 * Check if PTE can be modified, if not branch to LABEL. Regardless 1153 * restore PTE with value from PTR when done. 1154 */ 1155 static void __cpuinit 1156 build_pte_modifiable(u32 **p, struct uasm_reloc **r, 1157 unsigned int pte, unsigned int ptr, enum label_id lid) 1158 { 1159 uasm_i_andi(p, pte, pte, _PAGE_WRITE); 1160 uasm_il_beqz(p, r, pte, lid); 1161 iPTE_LW(p, pte, ptr); 1162 } 1163 1164 #ifndef CONFIG_MIPS_PGD_C0_CONTEXT 1165 /* 1166 * R3000 style TLB load/store/modify handlers. 1167 */ 1168 1169 /* 1170 * This places the pte into ENTRYLO0 and writes it with tlbwi. 1171 * Then it returns. 1172 */ 1173 static void __cpuinit 1174 build_r3000_pte_reload_tlbwi(u32 **p, unsigned int pte, unsigned int tmp) 1175 { 1176 uasm_i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */ 1177 uasm_i_mfc0(p, tmp, C0_EPC); /* cp0 delay */ 1178 uasm_i_tlbwi(p); 1179 uasm_i_jr(p, tmp); 1180 uasm_i_rfe(p); /* branch delay */ 1181 } 1182 1183 /* 1184 * This places the pte into ENTRYLO0 and writes it with tlbwi 1185 * or tlbwr as appropriate. This is because the index register 1186 * may have the probe fail bit set as a result of a trap on a 1187 * kseg2 access, i.e. without refill. Then it returns. 1188 */ 1189 static void __cpuinit 1190 build_r3000_tlb_reload_write(u32 **p, struct uasm_label **l, 1191 struct uasm_reloc **r, unsigned int pte, 1192 unsigned int tmp) 1193 { 1194 uasm_i_mfc0(p, tmp, C0_INDEX); 1195 uasm_i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */ 1196 uasm_il_bltz(p, r, tmp, label_r3000_write_probe_fail); /* cp0 delay */ 1197 uasm_i_mfc0(p, tmp, C0_EPC); /* branch delay */ 1198 uasm_i_tlbwi(p); /* cp0 delay */ 1199 uasm_i_jr(p, tmp); 1200 uasm_i_rfe(p); /* branch delay */ 1201 uasm_l_r3000_write_probe_fail(l, *p); 1202 uasm_i_tlbwr(p); /* cp0 delay */ 1203 uasm_i_jr(p, tmp); 1204 uasm_i_rfe(p); /* branch delay */ 1205 } 1206 1207 static void __cpuinit 1208 build_r3000_tlbchange_handler_head(u32 **p, unsigned int pte, 1209 unsigned int ptr) 1210 { 1211 long pgdc = (long)pgd_current; 1212 1213 uasm_i_mfc0(p, pte, C0_BADVADDR); 1214 uasm_i_lui(p, ptr, uasm_rel_hi(pgdc)); /* cp0 delay */ 1215 uasm_i_lw(p, ptr, uasm_rel_lo(pgdc), ptr); 1216 uasm_i_srl(p, pte, pte, 22); /* load delay */ 1217 uasm_i_sll(p, pte, pte, 2); 1218 uasm_i_addu(p, ptr, ptr, pte); 1219 uasm_i_mfc0(p, pte, C0_CONTEXT); 1220 uasm_i_lw(p, ptr, 0, ptr); /* cp0 delay */ 1221 uasm_i_andi(p, pte, pte, 0xffc); /* load delay */ 1222 uasm_i_addu(p, ptr, ptr, pte); 1223 uasm_i_lw(p, pte, 0, ptr); 1224 uasm_i_tlbp(p); /* load delay */ 1225 } 1226 1227 static void __cpuinit build_r3000_tlb_load_handler(void) 1228 { 1229 u32 *p = handle_tlbl; 1230 struct uasm_label *l = labels; 1231 struct uasm_reloc *r = relocs; 1232 1233 memset(handle_tlbl, 0, sizeof(handle_tlbl)); 1234 memset(labels, 0, sizeof(labels)); 1235 memset(relocs, 0, sizeof(relocs)); 1236 1237 build_r3000_tlbchange_handler_head(&p, K0, K1); 1238 build_pte_present(&p, &r, K0, K1, label_nopage_tlbl); 1239 uasm_i_nop(&p); /* load delay */ 1240 build_make_valid(&p, &r, K0, K1); 1241 build_r3000_tlb_reload_write(&p, &l, &r, K0, K1); 1242 1243 uasm_l_nopage_tlbl(&l, p); 1244 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff); 1245 uasm_i_nop(&p); 1246 1247 if ((p - handle_tlbl) > FASTPATH_SIZE) 1248 panic("TLB load handler fastpath space exceeded"); 1249 1250 uasm_resolve_relocs(relocs, labels); 1251 pr_debug("Wrote TLB load handler fastpath (%u instructions).\n", 1252 (unsigned int)(p - handle_tlbl)); 1253 1254 dump_handler(handle_tlbl, ARRAY_SIZE(handle_tlbl)); 1255 } 1256 1257 static void __cpuinit build_r3000_tlb_store_handler(void) 1258 { 1259 u32 *p = handle_tlbs; 1260 struct uasm_label *l = labels; 1261 struct uasm_reloc *r = relocs; 1262 1263 memset(handle_tlbs, 0, sizeof(handle_tlbs)); 1264 memset(labels, 0, sizeof(labels)); 1265 memset(relocs, 0, sizeof(relocs)); 1266 1267 build_r3000_tlbchange_handler_head(&p, K0, K1); 1268 build_pte_writable(&p, &r, K0, K1, label_nopage_tlbs); 1269 uasm_i_nop(&p); /* load delay */ 1270 build_make_write(&p, &r, K0, K1); 1271 build_r3000_tlb_reload_write(&p, &l, &r, K0, K1); 1272 1273 uasm_l_nopage_tlbs(&l, p); 1274 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); 1275 uasm_i_nop(&p); 1276 1277 if ((p - handle_tlbs) > FASTPATH_SIZE) 1278 panic("TLB store handler fastpath space exceeded"); 1279 1280 uasm_resolve_relocs(relocs, labels); 1281 pr_debug("Wrote TLB store handler fastpath (%u instructions).\n", 1282 (unsigned int)(p - handle_tlbs)); 1283 1284 dump_handler(handle_tlbs, ARRAY_SIZE(handle_tlbs)); 1285 } 1286 1287 static void __cpuinit build_r3000_tlb_modify_handler(void) 1288 { 1289 u32 *p = handle_tlbm; 1290 struct uasm_label *l = labels; 1291 struct uasm_reloc *r = relocs; 1292 1293 memset(handle_tlbm, 0, sizeof(handle_tlbm)); 1294 memset(labels, 0, sizeof(labels)); 1295 memset(relocs, 0, sizeof(relocs)); 1296 1297 build_r3000_tlbchange_handler_head(&p, K0, K1); 1298 build_pte_modifiable(&p, &r, K0, K1, label_nopage_tlbm); 1299 uasm_i_nop(&p); /* load delay */ 1300 build_make_write(&p, &r, K0, K1); 1301 build_r3000_pte_reload_tlbwi(&p, K0, K1); 1302 1303 uasm_l_nopage_tlbm(&l, p); 1304 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); 1305 uasm_i_nop(&p); 1306 1307 if ((p - handle_tlbm) > FASTPATH_SIZE) 1308 panic("TLB modify handler fastpath space exceeded"); 1309 1310 uasm_resolve_relocs(relocs, labels); 1311 pr_debug("Wrote TLB modify handler fastpath (%u instructions).\n", 1312 (unsigned int)(p - handle_tlbm)); 1313 1314 dump_handler(handle_tlbm, ARRAY_SIZE(handle_tlbm)); 1315 } 1316 #endif /* CONFIG_MIPS_PGD_C0_CONTEXT */ 1317 1318 /* 1319 * R4000 style TLB load/store/modify handlers. 1320 */ 1321 static void __cpuinit 1322 build_r4000_tlbchange_handler_head(u32 **p, struct uasm_label **l, 1323 struct uasm_reloc **r, unsigned int pte, 1324 unsigned int ptr) 1325 { 1326 #ifdef CONFIG_64BIT 1327 build_get_pmde64(p, l, r, pte, ptr); /* get pmd in ptr */ 1328 #else 1329 build_get_pgde32(p, pte, ptr); /* get pgd in ptr */ 1330 #endif 1331 1332 #ifdef CONFIG_HUGETLB_PAGE 1333 /* 1334 * For huge tlb entries, pmd doesn't contain an address but 1335 * instead contains the tlb pte. Check the PAGE_HUGE bit and 1336 * see if we need to jump to huge tlb processing. 1337 */ 1338 build_is_huge_pte(p, r, pte, ptr, label_tlb_huge_update); 1339 #endif 1340 1341 UASM_i_MFC0(p, pte, C0_BADVADDR); 1342 UASM_i_LW(p, ptr, 0, ptr); 1343 UASM_i_SRL(p, pte, pte, PAGE_SHIFT + PTE_ORDER - PTE_T_LOG2); 1344 uasm_i_andi(p, pte, pte, (PTRS_PER_PTE - 1) << PTE_T_LOG2); 1345 UASM_i_ADDU(p, ptr, ptr, pte); 1346 1347 #ifdef CONFIG_SMP 1348 uasm_l_smp_pgtable_change(l, *p); 1349 #endif 1350 iPTE_LW(p, pte, ptr); /* get even pte */ 1351 if (!m4kc_tlbp_war()) 1352 build_tlb_probe_entry(p); 1353 } 1354 1355 static void __cpuinit 1356 build_r4000_tlbchange_handler_tail(u32 **p, struct uasm_label **l, 1357 struct uasm_reloc **r, unsigned int tmp, 1358 unsigned int ptr) 1359 { 1360 uasm_i_ori(p, ptr, ptr, sizeof(pte_t)); 1361 uasm_i_xori(p, ptr, ptr, sizeof(pte_t)); 1362 build_update_entries(p, tmp, ptr); 1363 build_tlb_write_entry(p, l, r, tlb_indexed); 1364 uasm_l_leave(l, *p); 1365 uasm_i_eret(p); /* return from trap */ 1366 1367 #ifdef CONFIG_64BIT 1368 build_get_pgd_vmalloc64(p, l, r, tmp, ptr, not_refill); 1369 #endif 1370 } 1371 1372 static void __cpuinit build_r4000_tlb_load_handler(void) 1373 { 1374 u32 *p = handle_tlbl; 1375 struct uasm_label *l = labels; 1376 struct uasm_reloc *r = relocs; 1377 1378 memset(handle_tlbl, 0, sizeof(handle_tlbl)); 1379 memset(labels, 0, sizeof(labels)); 1380 memset(relocs, 0, sizeof(relocs)); 1381 1382 if (bcm1250_m3_war()) { 1383 unsigned int segbits = 44; 1384 1385 uasm_i_dmfc0(&p, K0, C0_BADVADDR); 1386 uasm_i_dmfc0(&p, K1, C0_ENTRYHI); 1387 uasm_i_xor(&p, K0, K0, K1); 1388 uasm_i_dsrl_safe(&p, K1, K0, 62); 1389 uasm_i_dsrl_safe(&p, K0, K0, 12 + 1); 1390 uasm_i_dsll_safe(&p, K0, K0, 64 + 12 + 1 - segbits); 1391 uasm_i_or(&p, K0, K0, K1); 1392 uasm_il_bnez(&p, &r, K0, label_leave); 1393 /* No need for uasm_i_nop */ 1394 } 1395 1396 build_r4000_tlbchange_handler_head(&p, &l, &r, K0, K1); 1397 build_pte_present(&p, &r, K0, K1, label_nopage_tlbl); 1398 if (m4kc_tlbp_war()) 1399 build_tlb_probe_entry(&p); 1400 1401 if (kernel_uses_smartmips_rixi) { 1402 /* 1403 * If the page is not _PAGE_VALID, RI or XI could not 1404 * have triggered it. Skip the expensive test.. 1405 */ 1406 uasm_i_andi(&p, K0, K0, _PAGE_VALID); 1407 uasm_il_beqz(&p, &r, K0, label_tlbl_goaround1); 1408 uasm_i_nop(&p); 1409 1410 uasm_i_tlbr(&p); 1411 /* Examine entrylo 0 or 1 based on ptr. */ 1412 uasm_i_andi(&p, K0, K1, sizeof(pte_t)); 1413 uasm_i_beqz(&p, K0, 8); 1414 1415 UASM_i_MFC0(&p, K0, C0_ENTRYLO0); /* load it in the delay slot*/ 1416 UASM_i_MFC0(&p, K0, C0_ENTRYLO1); /* load it if ptr is odd */ 1417 /* 1418 * If the entryLo (now in K0) is valid (bit 1), RI or 1419 * XI must have triggered it. 1420 */ 1421 uasm_i_andi(&p, K0, K0, 2); 1422 uasm_il_bnez(&p, &r, K0, label_nopage_tlbl); 1423 1424 uasm_l_tlbl_goaround1(&l, p); 1425 /* Reload the PTE value */ 1426 iPTE_LW(&p, K0, K1); 1427 } 1428 build_make_valid(&p, &r, K0, K1); 1429 build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1); 1430 1431 #ifdef CONFIG_HUGETLB_PAGE 1432 /* 1433 * This is the entry point when build_r4000_tlbchange_handler_head 1434 * spots a huge page. 1435 */ 1436 uasm_l_tlb_huge_update(&l, p); 1437 iPTE_LW(&p, K0, K1); 1438 build_pte_present(&p, &r, K0, K1, label_nopage_tlbl); 1439 build_tlb_probe_entry(&p); 1440 1441 if (kernel_uses_smartmips_rixi) { 1442 /* 1443 * If the page is not _PAGE_VALID, RI or XI could not 1444 * have triggered it. Skip the expensive test.. 1445 */ 1446 uasm_i_andi(&p, K0, K0, _PAGE_VALID); 1447 uasm_il_beqz(&p, &r, K0, label_tlbl_goaround2); 1448 uasm_i_nop(&p); 1449 1450 uasm_i_tlbr(&p); 1451 /* Examine entrylo 0 or 1 based on ptr. */ 1452 uasm_i_andi(&p, K0, K1, sizeof(pte_t)); 1453 uasm_i_beqz(&p, K0, 8); 1454 1455 UASM_i_MFC0(&p, K0, C0_ENTRYLO0); /* load it in the delay slot*/ 1456 UASM_i_MFC0(&p, K0, C0_ENTRYLO1); /* load it if ptr is odd */ 1457 /* 1458 * If the entryLo (now in K0) is valid (bit 1), RI or 1459 * XI must have triggered it. 1460 */ 1461 uasm_i_andi(&p, K0, K0, 2); 1462 uasm_il_beqz(&p, &r, K0, label_tlbl_goaround2); 1463 /* Reload the PTE value */ 1464 iPTE_LW(&p, K0, K1); 1465 1466 /* 1467 * We clobbered C0_PAGEMASK, restore it. On the other branch 1468 * it is restored in build_huge_tlb_write_entry. 1469 */ 1470 build_restore_pagemask(&p, &r, K0, label_nopage_tlbl); 1471 1472 uasm_l_tlbl_goaround2(&l, p); 1473 } 1474 uasm_i_ori(&p, K0, K0, (_PAGE_ACCESSED | _PAGE_VALID)); 1475 build_huge_handler_tail(&p, &r, &l, K0, K1); 1476 #endif 1477 1478 uasm_l_nopage_tlbl(&l, p); 1479 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff); 1480 uasm_i_nop(&p); 1481 1482 if ((p - handle_tlbl) > FASTPATH_SIZE) 1483 panic("TLB load handler fastpath space exceeded"); 1484 1485 uasm_resolve_relocs(relocs, labels); 1486 pr_debug("Wrote TLB load handler fastpath (%u instructions).\n", 1487 (unsigned int)(p - handle_tlbl)); 1488 1489 dump_handler(handle_tlbl, ARRAY_SIZE(handle_tlbl)); 1490 } 1491 1492 static void __cpuinit build_r4000_tlb_store_handler(void) 1493 { 1494 u32 *p = handle_tlbs; 1495 struct uasm_label *l = labels; 1496 struct uasm_reloc *r = relocs; 1497 1498 memset(handle_tlbs, 0, sizeof(handle_tlbs)); 1499 memset(labels, 0, sizeof(labels)); 1500 memset(relocs, 0, sizeof(relocs)); 1501 1502 build_r4000_tlbchange_handler_head(&p, &l, &r, K0, K1); 1503 build_pte_writable(&p, &r, K0, K1, label_nopage_tlbs); 1504 if (m4kc_tlbp_war()) 1505 build_tlb_probe_entry(&p); 1506 build_make_write(&p, &r, K0, K1); 1507 build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1); 1508 1509 #ifdef CONFIG_HUGETLB_PAGE 1510 /* 1511 * This is the entry point when 1512 * build_r4000_tlbchange_handler_head spots a huge page. 1513 */ 1514 uasm_l_tlb_huge_update(&l, p); 1515 iPTE_LW(&p, K0, K1); 1516 build_pte_writable(&p, &r, K0, K1, label_nopage_tlbs); 1517 build_tlb_probe_entry(&p); 1518 uasm_i_ori(&p, K0, K0, 1519 _PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID | _PAGE_DIRTY); 1520 build_huge_handler_tail(&p, &r, &l, K0, K1); 1521 #endif 1522 1523 uasm_l_nopage_tlbs(&l, p); 1524 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); 1525 uasm_i_nop(&p); 1526 1527 if ((p - handle_tlbs) > FASTPATH_SIZE) 1528 panic("TLB store handler fastpath space exceeded"); 1529 1530 uasm_resolve_relocs(relocs, labels); 1531 pr_debug("Wrote TLB store handler fastpath (%u instructions).\n", 1532 (unsigned int)(p - handle_tlbs)); 1533 1534 dump_handler(handle_tlbs, ARRAY_SIZE(handle_tlbs)); 1535 } 1536 1537 static void __cpuinit build_r4000_tlb_modify_handler(void) 1538 { 1539 u32 *p = handle_tlbm; 1540 struct uasm_label *l = labels; 1541 struct uasm_reloc *r = relocs; 1542 1543 memset(handle_tlbm, 0, sizeof(handle_tlbm)); 1544 memset(labels, 0, sizeof(labels)); 1545 memset(relocs, 0, sizeof(relocs)); 1546 1547 build_r4000_tlbchange_handler_head(&p, &l, &r, K0, K1); 1548 build_pte_modifiable(&p, &r, K0, K1, label_nopage_tlbm); 1549 if (m4kc_tlbp_war()) 1550 build_tlb_probe_entry(&p); 1551 /* Present and writable bits set, set accessed and dirty bits. */ 1552 build_make_write(&p, &r, K0, K1); 1553 build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1); 1554 1555 #ifdef CONFIG_HUGETLB_PAGE 1556 /* 1557 * This is the entry point when 1558 * build_r4000_tlbchange_handler_head spots a huge page. 1559 */ 1560 uasm_l_tlb_huge_update(&l, p); 1561 iPTE_LW(&p, K0, K1); 1562 build_pte_modifiable(&p, &r, K0, K1, label_nopage_tlbm); 1563 build_tlb_probe_entry(&p); 1564 uasm_i_ori(&p, K0, K0, 1565 _PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID | _PAGE_DIRTY); 1566 build_huge_handler_tail(&p, &r, &l, K0, K1); 1567 #endif 1568 1569 uasm_l_nopage_tlbm(&l, p); 1570 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); 1571 uasm_i_nop(&p); 1572 1573 if ((p - handle_tlbm) > FASTPATH_SIZE) 1574 panic("TLB modify handler fastpath space exceeded"); 1575 1576 uasm_resolve_relocs(relocs, labels); 1577 pr_debug("Wrote TLB modify handler fastpath (%u instructions).\n", 1578 (unsigned int)(p - handle_tlbm)); 1579 1580 dump_handler(handle_tlbm, ARRAY_SIZE(handle_tlbm)); 1581 } 1582 1583 void __cpuinit build_tlb_refill_handler(void) 1584 { 1585 /* 1586 * The refill handler is generated per-CPU, multi-node systems 1587 * may have local storage for it. The other handlers are only 1588 * needed once. 1589 */ 1590 static int run_once = 0; 1591 1592 #ifdef CONFIG_64BIT 1593 check_for_high_segbits = current_cpu_data.vmbits > (PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3); 1594 #endif 1595 1596 switch (current_cpu_type()) { 1597 case CPU_R2000: 1598 case CPU_R3000: 1599 case CPU_R3000A: 1600 case CPU_R3081E: 1601 case CPU_TX3912: 1602 case CPU_TX3922: 1603 case CPU_TX3927: 1604 #ifndef CONFIG_MIPS_PGD_C0_CONTEXT 1605 build_r3000_tlb_refill_handler(); 1606 if (!run_once) { 1607 build_r3000_tlb_load_handler(); 1608 build_r3000_tlb_store_handler(); 1609 build_r3000_tlb_modify_handler(); 1610 run_once++; 1611 } 1612 #else 1613 panic("No R3000 TLB refill handler"); 1614 #endif 1615 break; 1616 1617 case CPU_R6000: 1618 case CPU_R6000A: 1619 panic("No R6000 TLB refill handler yet"); 1620 break; 1621 1622 case CPU_R8000: 1623 panic("No R8000 TLB refill handler yet"); 1624 break; 1625 1626 default: 1627 build_r4000_tlb_refill_handler(); 1628 if (!run_once) { 1629 build_r4000_tlb_load_handler(); 1630 build_r4000_tlb_store_handler(); 1631 build_r4000_tlb_modify_handler(); 1632 run_once++; 1633 } 1634 } 1635 } 1636 1637 void __cpuinit flush_tlb_handlers(void) 1638 { 1639 local_flush_icache_range((unsigned long)handle_tlbl, 1640 (unsigned long)handle_tlbl + sizeof(handle_tlbl)); 1641 local_flush_icache_range((unsigned long)handle_tlbs, 1642 (unsigned long)handle_tlbs + sizeof(handle_tlbs)); 1643 local_flush_icache_range((unsigned long)handle_tlbm, 1644 (unsigned long)handle_tlbm + sizeof(handle_tlbm)); 1645 } 1646