1 /* 2 * This file is subject to the terms and conditions of the GNU General Public 3 * License. See the file "COPYING" in the main directory of this archive 4 * for more details. 5 * 6 * Synthesize TLB refill handlers at runtime. 7 * 8 * Copyright (C) 2004, 2005, 2006, 2008 Thiemo Seufer 9 * Copyright (C) 2005, 2007, 2008, 2009 Maciej W. Rozycki 10 * Copyright (C) 2006 Ralf Baechle (ralf@linux-mips.org) 11 * Copyright (C) 2008, 2009 Cavium Networks, Inc. 12 * 13 * ... and the days got worse and worse and now you see 14 * I've gone completly out of my mind. 15 * 16 * They're coming to take me a away haha 17 * they're coming to take me a away hoho hihi haha 18 * to the funny farm where code is beautiful all the time ... 19 * 20 * (Condolences to Napoleon XIV) 21 */ 22 23 #include <linux/bug.h> 24 #include <linux/kernel.h> 25 #include <linux/types.h> 26 #include <linux/smp.h> 27 #include <linux/string.h> 28 #include <linux/init.h> 29 30 #include <asm/mmu_context.h> 31 #include <asm/war.h> 32 #include <asm/uasm.h> 33 34 /* 35 * TLB load/store/modify handlers. 36 * 37 * Only the fastpath gets synthesized at runtime, the slowpath for 38 * do_page_fault remains normal asm. 39 */ 40 extern void tlb_do_page_fault_0(void); 41 extern void tlb_do_page_fault_1(void); 42 43 44 static inline int r45k_bvahwbug(void) 45 { 46 /* XXX: We should probe for the presence of this bug, but we don't. */ 47 return 0; 48 } 49 50 static inline int r4k_250MHZhwbug(void) 51 { 52 /* XXX: We should probe for the presence of this bug, but we don't. */ 53 return 0; 54 } 55 56 static inline int __maybe_unused bcm1250_m3_war(void) 57 { 58 return BCM1250_M3_WAR; 59 } 60 61 static inline int __maybe_unused r10000_llsc_war(void) 62 { 63 return R10000_LLSC_WAR; 64 } 65 66 /* 67 * Found by experiment: At least some revisions of the 4kc throw under 68 * some circumstances a machine check exception, triggered by invalid 69 * values in the index register. Delaying the tlbp instruction until 70 * after the next branch, plus adding an additional nop in front of 71 * tlbwi/tlbwr avoids the invalid index register values. Nobody knows 72 * why; it's not an issue caused by the core RTL. 73 * 74 */ 75 static int __cpuinit m4kc_tlbp_war(void) 76 { 77 return (current_cpu_data.processor_id & 0xffff00) == 78 (PRID_COMP_MIPS | PRID_IMP_4KC); 79 } 80 81 /* Handle labels (which must be positive integers). */ 82 enum label_id { 83 label_second_part = 1, 84 label_leave, 85 label_vmalloc, 86 label_vmalloc_done, 87 label_tlbw_hazard, 88 label_split, 89 label_tlbl_goaround1, 90 label_tlbl_goaround2, 91 label_nopage_tlbl, 92 label_nopage_tlbs, 93 label_nopage_tlbm, 94 label_smp_pgtable_change, 95 label_r3000_write_probe_fail, 96 label_large_segbits_fault, 97 #ifdef CONFIG_HUGETLB_PAGE 98 label_tlb_huge_update, 99 #endif 100 }; 101 102 UASM_L_LA(_second_part) 103 UASM_L_LA(_leave) 104 UASM_L_LA(_vmalloc) 105 UASM_L_LA(_vmalloc_done) 106 UASM_L_LA(_tlbw_hazard) 107 UASM_L_LA(_split) 108 UASM_L_LA(_tlbl_goaround1) 109 UASM_L_LA(_tlbl_goaround2) 110 UASM_L_LA(_nopage_tlbl) 111 UASM_L_LA(_nopage_tlbs) 112 UASM_L_LA(_nopage_tlbm) 113 UASM_L_LA(_smp_pgtable_change) 114 UASM_L_LA(_r3000_write_probe_fail) 115 UASM_L_LA(_large_segbits_fault) 116 #ifdef CONFIG_HUGETLB_PAGE 117 UASM_L_LA(_tlb_huge_update) 118 #endif 119 120 /* 121 * For debug purposes. 122 */ 123 static inline void dump_handler(const u32 *handler, int count) 124 { 125 int i; 126 127 pr_debug("\t.set push\n"); 128 pr_debug("\t.set noreorder\n"); 129 130 for (i = 0; i < count; i++) 131 pr_debug("\t%p\t.word 0x%08x\n", &handler[i], handler[i]); 132 133 pr_debug("\t.set pop\n"); 134 } 135 136 /* The only general purpose registers allowed in TLB handlers. */ 137 #define K0 26 138 #define K1 27 139 140 /* Some CP0 registers */ 141 #define C0_INDEX 0, 0 142 #define C0_ENTRYLO0 2, 0 143 #define C0_TCBIND 2, 2 144 #define C0_ENTRYLO1 3, 0 145 #define C0_CONTEXT 4, 0 146 #define C0_PAGEMASK 5, 0 147 #define C0_BADVADDR 8, 0 148 #define C0_ENTRYHI 10, 0 149 #define C0_EPC 14, 0 150 #define C0_XCONTEXT 20, 0 151 152 #ifdef CONFIG_64BIT 153 # define GET_CONTEXT(buf, reg) UASM_i_MFC0(buf, reg, C0_XCONTEXT) 154 #else 155 # define GET_CONTEXT(buf, reg) UASM_i_MFC0(buf, reg, C0_CONTEXT) 156 #endif 157 158 /* The worst case length of the handler is around 18 instructions for 159 * R3000-style TLBs and up to 63 instructions for R4000-style TLBs. 160 * Maximum space available is 32 instructions for R3000 and 64 161 * instructions for R4000. 162 * 163 * We deliberately chose a buffer size of 128, so we won't scribble 164 * over anything important on overflow before we panic. 165 */ 166 static u32 tlb_handler[128] __cpuinitdata; 167 168 /* simply assume worst case size for labels and relocs */ 169 static struct uasm_label labels[128] __cpuinitdata; 170 static struct uasm_reloc relocs[128] __cpuinitdata; 171 172 #ifdef CONFIG_64BIT 173 static int check_for_high_segbits __cpuinitdata; 174 #endif 175 176 #ifndef CONFIG_MIPS_PGD_C0_CONTEXT 177 /* 178 * CONFIG_MIPS_PGD_C0_CONTEXT implies 64 bit and lack of pgd_current, 179 * we cannot do r3000 under these circumstances. 180 */ 181 182 /* 183 * The R3000 TLB handler is simple. 184 */ 185 static void __cpuinit build_r3000_tlb_refill_handler(void) 186 { 187 long pgdc = (long)pgd_current; 188 u32 *p; 189 190 memset(tlb_handler, 0, sizeof(tlb_handler)); 191 p = tlb_handler; 192 193 uasm_i_mfc0(&p, K0, C0_BADVADDR); 194 uasm_i_lui(&p, K1, uasm_rel_hi(pgdc)); /* cp0 delay */ 195 uasm_i_lw(&p, K1, uasm_rel_lo(pgdc), K1); 196 uasm_i_srl(&p, K0, K0, 22); /* load delay */ 197 uasm_i_sll(&p, K0, K0, 2); 198 uasm_i_addu(&p, K1, K1, K0); 199 uasm_i_mfc0(&p, K0, C0_CONTEXT); 200 uasm_i_lw(&p, K1, 0, K1); /* cp0 delay */ 201 uasm_i_andi(&p, K0, K0, 0xffc); /* load delay */ 202 uasm_i_addu(&p, K1, K1, K0); 203 uasm_i_lw(&p, K0, 0, K1); 204 uasm_i_nop(&p); /* load delay */ 205 uasm_i_mtc0(&p, K0, C0_ENTRYLO0); 206 uasm_i_mfc0(&p, K1, C0_EPC); /* cp0 delay */ 207 uasm_i_tlbwr(&p); /* cp0 delay */ 208 uasm_i_jr(&p, K1); 209 uasm_i_rfe(&p); /* branch delay */ 210 211 if (p > tlb_handler + 32) 212 panic("TLB refill handler space exceeded"); 213 214 pr_debug("Wrote TLB refill handler (%u instructions).\n", 215 (unsigned int)(p - tlb_handler)); 216 217 memcpy((void *)ebase, tlb_handler, 0x80); 218 219 dump_handler((u32 *)ebase, 32); 220 } 221 #endif /* CONFIG_MIPS_PGD_C0_CONTEXT */ 222 223 /* 224 * The R4000 TLB handler is much more complicated. We have two 225 * consecutive handler areas with 32 instructions space each. 226 * Since they aren't used at the same time, we can overflow in the 227 * other one.To keep things simple, we first assume linear space, 228 * then we relocate it to the final handler layout as needed. 229 */ 230 static u32 final_handler[64] __cpuinitdata; 231 232 /* 233 * Hazards 234 * 235 * From the IDT errata for the QED RM5230 (Nevada), processor revision 1.0: 236 * 2. A timing hazard exists for the TLBP instruction. 237 * 238 * stalling_instruction 239 * TLBP 240 * 241 * The JTLB is being read for the TLBP throughout the stall generated by the 242 * previous instruction. This is not really correct as the stalling instruction 243 * can modify the address used to access the JTLB. The failure symptom is that 244 * the TLBP instruction will use an address created for the stalling instruction 245 * and not the address held in C0_ENHI and thus report the wrong results. 246 * 247 * The software work-around is to not allow the instruction preceding the TLBP 248 * to stall - make it an NOP or some other instruction guaranteed not to stall. 249 * 250 * Errata 2 will not be fixed. This errata is also on the R5000. 251 * 252 * As if we MIPS hackers wouldn't know how to nop pipelines happy ... 253 */ 254 static void __cpuinit __maybe_unused build_tlb_probe_entry(u32 **p) 255 { 256 switch (current_cpu_type()) { 257 /* Found by experiment: R4600 v2.0/R4700 needs this, too. */ 258 case CPU_R4600: 259 case CPU_R4700: 260 case CPU_R5000: 261 case CPU_R5000A: 262 case CPU_NEVADA: 263 uasm_i_nop(p); 264 uasm_i_tlbp(p); 265 break; 266 267 default: 268 uasm_i_tlbp(p); 269 break; 270 } 271 } 272 273 /* 274 * Write random or indexed TLB entry, and care about the hazards from 275 * the preceeding mtc0 and for the following eret. 276 */ 277 enum tlb_write_entry { tlb_random, tlb_indexed }; 278 279 static void __cpuinit build_tlb_write_entry(u32 **p, struct uasm_label **l, 280 struct uasm_reloc **r, 281 enum tlb_write_entry wmode) 282 { 283 void(*tlbw)(u32 **) = NULL; 284 285 switch (wmode) { 286 case tlb_random: tlbw = uasm_i_tlbwr; break; 287 case tlb_indexed: tlbw = uasm_i_tlbwi; break; 288 } 289 290 if (cpu_has_mips_r2) { 291 if (cpu_has_mips_r2_exec_hazard) 292 uasm_i_ehb(p); 293 tlbw(p); 294 return; 295 } 296 297 switch (current_cpu_type()) { 298 case CPU_R4000PC: 299 case CPU_R4000SC: 300 case CPU_R4000MC: 301 case CPU_R4400PC: 302 case CPU_R4400SC: 303 case CPU_R4400MC: 304 /* 305 * This branch uses up a mtc0 hazard nop slot and saves 306 * two nops after the tlbw instruction. 307 */ 308 uasm_il_bgezl(p, r, 0, label_tlbw_hazard); 309 tlbw(p); 310 uasm_l_tlbw_hazard(l, *p); 311 uasm_i_nop(p); 312 break; 313 314 case CPU_R4600: 315 case CPU_R4700: 316 case CPU_R5000: 317 case CPU_R5000A: 318 uasm_i_nop(p); 319 tlbw(p); 320 uasm_i_nop(p); 321 break; 322 323 case CPU_R4300: 324 case CPU_5KC: 325 case CPU_TX49XX: 326 case CPU_PR4450: 327 uasm_i_nop(p); 328 tlbw(p); 329 break; 330 331 case CPU_R10000: 332 case CPU_R12000: 333 case CPU_R14000: 334 case CPU_4KC: 335 case CPU_4KEC: 336 case CPU_SB1: 337 case CPU_SB1A: 338 case CPU_4KSC: 339 case CPU_20KC: 340 case CPU_25KF: 341 case CPU_BCM3302: 342 case CPU_BCM4710: 343 case CPU_LOONGSON2: 344 case CPU_BCM6338: 345 case CPU_BCM6345: 346 case CPU_BCM6348: 347 case CPU_BCM6358: 348 case CPU_R5500: 349 if (m4kc_tlbp_war()) 350 uasm_i_nop(p); 351 case CPU_ALCHEMY: 352 tlbw(p); 353 break; 354 355 case CPU_NEVADA: 356 uasm_i_nop(p); /* QED specifies 2 nops hazard */ 357 /* 358 * This branch uses up a mtc0 hazard nop slot and saves 359 * a nop after the tlbw instruction. 360 */ 361 uasm_il_bgezl(p, r, 0, label_tlbw_hazard); 362 tlbw(p); 363 uasm_l_tlbw_hazard(l, *p); 364 break; 365 366 case CPU_RM7000: 367 uasm_i_nop(p); 368 uasm_i_nop(p); 369 uasm_i_nop(p); 370 uasm_i_nop(p); 371 tlbw(p); 372 break; 373 374 case CPU_RM9000: 375 /* 376 * When the JTLB is updated by tlbwi or tlbwr, a subsequent 377 * use of the JTLB for instructions should not occur for 4 378 * cpu cycles and use for data translations should not occur 379 * for 3 cpu cycles. 380 */ 381 uasm_i_ssnop(p); 382 uasm_i_ssnop(p); 383 uasm_i_ssnop(p); 384 uasm_i_ssnop(p); 385 tlbw(p); 386 uasm_i_ssnop(p); 387 uasm_i_ssnop(p); 388 uasm_i_ssnop(p); 389 uasm_i_ssnop(p); 390 break; 391 392 case CPU_VR4111: 393 case CPU_VR4121: 394 case CPU_VR4122: 395 case CPU_VR4181: 396 case CPU_VR4181A: 397 uasm_i_nop(p); 398 uasm_i_nop(p); 399 tlbw(p); 400 uasm_i_nop(p); 401 uasm_i_nop(p); 402 break; 403 404 case CPU_VR4131: 405 case CPU_VR4133: 406 case CPU_R5432: 407 uasm_i_nop(p); 408 uasm_i_nop(p); 409 tlbw(p); 410 break; 411 412 default: 413 panic("No TLB refill handler yet (CPU type: %d)", 414 current_cpu_data.cputype); 415 break; 416 } 417 } 418 419 static __cpuinit __maybe_unused void build_convert_pte_to_entrylo(u32 **p, 420 unsigned int reg) 421 { 422 if (kernel_uses_smartmips_rixi) { 423 UASM_i_SRL(p, reg, reg, ilog2(_PAGE_NO_EXEC)); 424 UASM_i_ROTR(p, reg, reg, ilog2(_PAGE_GLOBAL) - ilog2(_PAGE_NO_EXEC)); 425 } else { 426 #ifdef CONFIG_64BIT_PHYS_ADDR 427 uasm_i_dsrl_safe(p, reg, reg, ilog2(_PAGE_GLOBAL)); 428 #else 429 UASM_i_SRL(p, reg, reg, ilog2(_PAGE_GLOBAL)); 430 #endif 431 } 432 } 433 434 #ifdef CONFIG_HUGETLB_PAGE 435 436 static __cpuinit void build_restore_pagemask(u32 **p, 437 struct uasm_reloc **r, 438 unsigned int tmp, 439 enum label_id lid) 440 { 441 /* Reset default page size */ 442 if (PM_DEFAULT_MASK >> 16) { 443 uasm_i_lui(p, tmp, PM_DEFAULT_MASK >> 16); 444 uasm_i_ori(p, tmp, tmp, PM_DEFAULT_MASK & 0xffff); 445 uasm_il_b(p, r, lid); 446 uasm_i_mtc0(p, tmp, C0_PAGEMASK); 447 } else if (PM_DEFAULT_MASK) { 448 uasm_i_ori(p, tmp, 0, PM_DEFAULT_MASK); 449 uasm_il_b(p, r, lid); 450 uasm_i_mtc0(p, tmp, C0_PAGEMASK); 451 } else { 452 uasm_il_b(p, r, lid); 453 uasm_i_mtc0(p, 0, C0_PAGEMASK); 454 } 455 } 456 457 static __cpuinit void build_huge_tlb_write_entry(u32 **p, 458 struct uasm_label **l, 459 struct uasm_reloc **r, 460 unsigned int tmp, 461 enum tlb_write_entry wmode) 462 { 463 /* Set huge page tlb entry size */ 464 uasm_i_lui(p, tmp, PM_HUGE_MASK >> 16); 465 uasm_i_ori(p, tmp, tmp, PM_HUGE_MASK & 0xffff); 466 uasm_i_mtc0(p, tmp, C0_PAGEMASK); 467 468 build_tlb_write_entry(p, l, r, wmode); 469 470 build_restore_pagemask(p, r, tmp, label_leave); 471 } 472 473 /* 474 * Check if Huge PTE is present, if so then jump to LABEL. 475 */ 476 static void __cpuinit 477 build_is_huge_pte(u32 **p, struct uasm_reloc **r, unsigned int tmp, 478 unsigned int pmd, int lid) 479 { 480 UASM_i_LW(p, tmp, 0, pmd); 481 uasm_i_andi(p, tmp, tmp, _PAGE_HUGE); 482 uasm_il_bnez(p, r, tmp, lid); 483 } 484 485 static __cpuinit void build_huge_update_entries(u32 **p, 486 unsigned int pte, 487 unsigned int tmp) 488 { 489 int small_sequence; 490 491 /* 492 * A huge PTE describes an area the size of the 493 * configured huge page size. This is twice the 494 * of the large TLB entry size we intend to use. 495 * A TLB entry half the size of the configured 496 * huge page size is configured into entrylo0 497 * and entrylo1 to cover the contiguous huge PTE 498 * address space. 499 */ 500 small_sequence = (HPAGE_SIZE >> 7) < 0x10000; 501 502 /* We can clobber tmp. It isn't used after this.*/ 503 if (!small_sequence) 504 uasm_i_lui(p, tmp, HPAGE_SIZE >> (7 + 16)); 505 506 build_convert_pte_to_entrylo(p, pte); 507 UASM_i_MTC0(p, pte, C0_ENTRYLO0); /* load it */ 508 /* convert to entrylo1 */ 509 if (small_sequence) 510 UASM_i_ADDIU(p, pte, pte, HPAGE_SIZE >> 7); 511 else 512 UASM_i_ADDU(p, pte, pte, tmp); 513 514 UASM_i_MTC0(p, pte, C0_ENTRYLO1); /* load it */ 515 } 516 517 static __cpuinit void build_huge_handler_tail(u32 **p, 518 struct uasm_reloc **r, 519 struct uasm_label **l, 520 unsigned int pte, 521 unsigned int ptr) 522 { 523 #ifdef CONFIG_SMP 524 UASM_i_SC(p, pte, 0, ptr); 525 uasm_il_beqz(p, r, pte, label_tlb_huge_update); 526 UASM_i_LW(p, pte, 0, ptr); /* Needed because SC killed our PTE */ 527 #else 528 UASM_i_SW(p, pte, 0, ptr); 529 #endif 530 build_huge_update_entries(p, pte, ptr); 531 build_huge_tlb_write_entry(p, l, r, pte, tlb_indexed); 532 } 533 #endif /* CONFIG_HUGETLB_PAGE */ 534 535 #ifdef CONFIG_64BIT 536 /* 537 * TMP and PTR are scratch. 538 * TMP will be clobbered, PTR will hold the pmd entry. 539 */ 540 static void __cpuinit 541 build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r, 542 unsigned int tmp, unsigned int ptr) 543 { 544 #ifndef CONFIG_MIPS_PGD_C0_CONTEXT 545 long pgdc = (long)pgd_current; 546 #endif 547 /* 548 * The vmalloc handling is not in the hotpath. 549 */ 550 uasm_i_dmfc0(p, tmp, C0_BADVADDR); 551 552 if (check_for_high_segbits) { 553 /* 554 * The kernel currently implicitely assumes that the 555 * MIPS SEGBITS parameter for the processor is 556 * (PGDIR_SHIFT+PGDIR_BITS) or less, and will never 557 * allocate virtual addresses outside the maximum 558 * range for SEGBITS = (PGDIR_SHIFT+PGDIR_BITS). But 559 * that doesn't prevent user code from accessing the 560 * higher xuseg addresses. Here, we make sure that 561 * everything but the lower xuseg addresses goes down 562 * the module_alloc/vmalloc path. 563 */ 564 uasm_i_dsrl_safe(p, ptr, tmp, PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3); 565 uasm_il_bnez(p, r, ptr, label_vmalloc); 566 } else { 567 uasm_il_bltz(p, r, tmp, label_vmalloc); 568 } 569 /* No uasm_i_nop needed here, since the next insn doesn't touch TMP. */ 570 571 #ifdef CONFIG_MIPS_PGD_C0_CONTEXT 572 /* 573 * &pgd << 11 stored in CONTEXT [23..63]. 574 */ 575 UASM_i_MFC0(p, ptr, C0_CONTEXT); 576 uasm_i_dins(p, ptr, 0, 0, 23); /* Clear lower 23 bits of context. */ 577 uasm_i_ori(p, ptr, ptr, 0x540); /* 1 0 1 0 1 << 6 xkphys cached */ 578 uasm_i_drotr(p, ptr, ptr, 11); 579 #elif defined(CONFIG_SMP) 580 # ifdef CONFIG_MIPS_MT_SMTC 581 /* 582 * SMTC uses TCBind value as "CPU" index 583 */ 584 uasm_i_mfc0(p, ptr, C0_TCBIND); 585 uasm_i_dsrl_safe(p, ptr, ptr, 19); 586 # else 587 /* 588 * 64 bit SMP running in XKPHYS has smp_processor_id() << 3 589 * stored in CONTEXT. 590 */ 591 uasm_i_dmfc0(p, ptr, C0_CONTEXT); 592 uasm_i_dsrl_safe(p, ptr, ptr, 23); 593 # endif 594 UASM_i_LA_mostly(p, tmp, pgdc); 595 uasm_i_daddu(p, ptr, ptr, tmp); 596 uasm_i_dmfc0(p, tmp, C0_BADVADDR); 597 uasm_i_ld(p, ptr, uasm_rel_lo(pgdc), ptr); 598 #else 599 UASM_i_LA_mostly(p, ptr, pgdc); 600 uasm_i_ld(p, ptr, uasm_rel_lo(pgdc), ptr); 601 #endif 602 603 uasm_l_vmalloc_done(l, *p); 604 605 /* get pgd offset in bytes */ 606 uasm_i_dsrl_safe(p, tmp, tmp, PGDIR_SHIFT - 3); 607 608 uasm_i_andi(p, tmp, tmp, (PTRS_PER_PGD - 1)<<3); 609 uasm_i_daddu(p, ptr, ptr, tmp); /* add in pgd offset */ 610 #ifndef __PAGETABLE_PMD_FOLDED 611 uasm_i_dmfc0(p, tmp, C0_BADVADDR); /* get faulting address */ 612 uasm_i_ld(p, ptr, 0, ptr); /* get pmd pointer */ 613 uasm_i_dsrl_safe(p, tmp, tmp, PMD_SHIFT-3); /* get pmd offset in bytes */ 614 uasm_i_andi(p, tmp, tmp, (PTRS_PER_PMD - 1)<<3); 615 uasm_i_daddu(p, ptr, ptr, tmp); /* add in pmd offset */ 616 #endif 617 } 618 619 enum vmalloc64_mode {not_refill, refill}; 620 /* 621 * BVADDR is the faulting address, PTR is scratch. 622 * PTR will hold the pgd for vmalloc. 623 */ 624 static void __cpuinit 625 build_get_pgd_vmalloc64(u32 **p, struct uasm_label **l, struct uasm_reloc **r, 626 unsigned int bvaddr, unsigned int ptr, 627 enum vmalloc64_mode mode) 628 { 629 long swpd = (long)swapper_pg_dir; 630 int single_insn_swpd; 631 int did_vmalloc_branch = 0; 632 633 single_insn_swpd = uasm_in_compat_space_p(swpd) && !uasm_rel_lo(swpd); 634 635 uasm_l_vmalloc(l, *p); 636 637 if (mode == refill && check_for_high_segbits) { 638 if (single_insn_swpd) { 639 uasm_il_bltz(p, r, bvaddr, label_vmalloc_done); 640 uasm_i_lui(p, ptr, uasm_rel_hi(swpd)); 641 did_vmalloc_branch = 1; 642 /* fall through */ 643 } else { 644 uasm_il_bgez(p, r, bvaddr, label_large_segbits_fault); 645 } 646 } 647 if (!did_vmalloc_branch) { 648 if (uasm_in_compat_space_p(swpd) && !uasm_rel_lo(swpd)) { 649 uasm_il_b(p, r, label_vmalloc_done); 650 uasm_i_lui(p, ptr, uasm_rel_hi(swpd)); 651 } else { 652 UASM_i_LA_mostly(p, ptr, swpd); 653 uasm_il_b(p, r, label_vmalloc_done); 654 if (uasm_in_compat_space_p(swpd)) 655 uasm_i_addiu(p, ptr, ptr, uasm_rel_lo(swpd)); 656 else 657 uasm_i_daddiu(p, ptr, ptr, uasm_rel_lo(swpd)); 658 } 659 } 660 if (mode == refill && check_for_high_segbits) { 661 uasm_l_large_segbits_fault(l, *p); 662 /* 663 * We get here if we are an xsseg address, or if we are 664 * an xuseg address above (PGDIR_SHIFT+PGDIR_BITS) boundary. 665 * 666 * Ignoring xsseg (assume disabled so would generate 667 * (address errors?), the only remaining possibility 668 * is the upper xuseg addresses. On processors with 669 * TLB_SEGBITS <= PGDIR_SHIFT+PGDIR_BITS, these 670 * addresses would have taken an address error. We try 671 * to mimic that here by taking a load/istream page 672 * fault. 673 */ 674 UASM_i_LA(p, ptr, (unsigned long)tlb_do_page_fault_0); 675 uasm_i_jr(p, ptr); 676 uasm_i_nop(p); 677 } 678 } 679 680 #else /* !CONFIG_64BIT */ 681 682 /* 683 * TMP and PTR are scratch. 684 * TMP will be clobbered, PTR will hold the pgd entry. 685 */ 686 static void __cpuinit __maybe_unused 687 build_get_pgde32(u32 **p, unsigned int tmp, unsigned int ptr) 688 { 689 long pgdc = (long)pgd_current; 690 691 /* 32 bit SMP has smp_processor_id() stored in CONTEXT. */ 692 #ifdef CONFIG_SMP 693 #ifdef CONFIG_MIPS_MT_SMTC 694 /* 695 * SMTC uses TCBind value as "CPU" index 696 */ 697 uasm_i_mfc0(p, ptr, C0_TCBIND); 698 UASM_i_LA_mostly(p, tmp, pgdc); 699 uasm_i_srl(p, ptr, ptr, 19); 700 #else 701 /* 702 * smp_processor_id() << 3 is stored in CONTEXT. 703 */ 704 uasm_i_mfc0(p, ptr, C0_CONTEXT); 705 UASM_i_LA_mostly(p, tmp, pgdc); 706 uasm_i_srl(p, ptr, ptr, 23); 707 #endif 708 uasm_i_addu(p, ptr, tmp, ptr); 709 #else 710 UASM_i_LA_mostly(p, ptr, pgdc); 711 #endif 712 uasm_i_mfc0(p, tmp, C0_BADVADDR); /* get faulting address */ 713 uasm_i_lw(p, ptr, uasm_rel_lo(pgdc), ptr); 714 uasm_i_srl(p, tmp, tmp, PGDIR_SHIFT); /* get pgd only bits */ 715 uasm_i_sll(p, tmp, tmp, PGD_T_LOG2); 716 uasm_i_addu(p, ptr, ptr, tmp); /* add in pgd offset */ 717 } 718 719 #endif /* !CONFIG_64BIT */ 720 721 static void __cpuinit build_adjust_context(u32 **p, unsigned int ctx) 722 { 723 unsigned int shift = 4 - (PTE_T_LOG2 + 1) + PAGE_SHIFT - 12; 724 unsigned int mask = (PTRS_PER_PTE / 2 - 1) << (PTE_T_LOG2 + 1); 725 726 switch (current_cpu_type()) { 727 case CPU_VR41XX: 728 case CPU_VR4111: 729 case CPU_VR4121: 730 case CPU_VR4122: 731 case CPU_VR4131: 732 case CPU_VR4181: 733 case CPU_VR4181A: 734 case CPU_VR4133: 735 shift += 2; 736 break; 737 738 default: 739 break; 740 } 741 742 if (shift) 743 UASM_i_SRL(p, ctx, ctx, shift); 744 uasm_i_andi(p, ctx, ctx, mask); 745 } 746 747 static void __cpuinit build_get_ptep(u32 **p, unsigned int tmp, unsigned int ptr) 748 { 749 /* 750 * Bug workaround for the Nevada. It seems as if under certain 751 * circumstances the move from cp0_context might produce a 752 * bogus result when the mfc0 instruction and its consumer are 753 * in a different cacheline or a load instruction, probably any 754 * memory reference, is between them. 755 */ 756 switch (current_cpu_type()) { 757 case CPU_NEVADA: 758 UASM_i_LW(p, ptr, 0, ptr); 759 GET_CONTEXT(p, tmp); /* get context reg */ 760 break; 761 762 default: 763 GET_CONTEXT(p, tmp); /* get context reg */ 764 UASM_i_LW(p, ptr, 0, ptr); 765 break; 766 } 767 768 build_adjust_context(p, tmp); 769 UASM_i_ADDU(p, ptr, ptr, tmp); /* add in offset */ 770 } 771 772 static void __cpuinit build_update_entries(u32 **p, unsigned int tmp, 773 unsigned int ptep) 774 { 775 /* 776 * 64bit address support (36bit on a 32bit CPU) in a 32bit 777 * Kernel is a special case. Only a few CPUs use it. 778 */ 779 #ifdef CONFIG_64BIT_PHYS_ADDR 780 if (cpu_has_64bits) { 781 uasm_i_ld(p, tmp, 0, ptep); /* get even pte */ 782 uasm_i_ld(p, ptep, sizeof(pte_t), ptep); /* get odd pte */ 783 if (kernel_uses_smartmips_rixi) { 784 UASM_i_SRL(p, tmp, tmp, ilog2(_PAGE_NO_EXEC)); 785 UASM_i_SRL(p, ptep, ptep, ilog2(_PAGE_NO_EXEC)); 786 UASM_i_ROTR(p, tmp, tmp, ilog2(_PAGE_GLOBAL) - ilog2(_PAGE_NO_EXEC)); 787 UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */ 788 UASM_i_ROTR(p, ptep, ptep, ilog2(_PAGE_GLOBAL) - ilog2(_PAGE_NO_EXEC)); 789 } else { 790 uasm_i_dsrl_safe(p, tmp, tmp, ilog2(_PAGE_GLOBAL)); /* convert to entrylo0 */ 791 UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */ 792 uasm_i_dsrl_safe(p, ptep, ptep, ilog2(_PAGE_GLOBAL)); /* convert to entrylo1 */ 793 } 794 UASM_i_MTC0(p, ptep, C0_ENTRYLO1); /* load it */ 795 } else { 796 int pte_off_even = sizeof(pte_t) / 2; 797 int pte_off_odd = pte_off_even + sizeof(pte_t); 798 799 /* The pte entries are pre-shifted */ 800 uasm_i_lw(p, tmp, pte_off_even, ptep); /* get even pte */ 801 UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */ 802 uasm_i_lw(p, ptep, pte_off_odd, ptep); /* get odd pte */ 803 UASM_i_MTC0(p, ptep, C0_ENTRYLO1); /* load it */ 804 } 805 #else 806 UASM_i_LW(p, tmp, 0, ptep); /* get even pte */ 807 UASM_i_LW(p, ptep, sizeof(pte_t), ptep); /* get odd pte */ 808 if (r45k_bvahwbug()) 809 build_tlb_probe_entry(p); 810 if (kernel_uses_smartmips_rixi) { 811 UASM_i_SRL(p, tmp, tmp, ilog2(_PAGE_NO_EXEC)); 812 UASM_i_SRL(p, ptep, ptep, ilog2(_PAGE_NO_EXEC)); 813 UASM_i_ROTR(p, tmp, tmp, ilog2(_PAGE_GLOBAL) - ilog2(_PAGE_NO_EXEC)); 814 if (r4k_250MHZhwbug()) 815 UASM_i_MTC0(p, 0, C0_ENTRYLO0); 816 UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */ 817 UASM_i_ROTR(p, ptep, ptep, ilog2(_PAGE_GLOBAL) - ilog2(_PAGE_NO_EXEC)); 818 } else { 819 UASM_i_SRL(p, tmp, tmp, ilog2(_PAGE_GLOBAL)); /* convert to entrylo0 */ 820 if (r4k_250MHZhwbug()) 821 UASM_i_MTC0(p, 0, C0_ENTRYLO0); 822 UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */ 823 UASM_i_SRL(p, ptep, ptep, ilog2(_PAGE_GLOBAL)); /* convert to entrylo1 */ 824 if (r45k_bvahwbug()) 825 uasm_i_mfc0(p, tmp, C0_INDEX); 826 } 827 if (r4k_250MHZhwbug()) 828 UASM_i_MTC0(p, 0, C0_ENTRYLO1); 829 UASM_i_MTC0(p, ptep, C0_ENTRYLO1); /* load it */ 830 #endif 831 } 832 833 /* 834 * For a 64-bit kernel, we are using the 64-bit XTLB refill exception 835 * because EXL == 0. If we wrap, we can also use the 32 instruction 836 * slots before the XTLB refill exception handler which belong to the 837 * unused TLB refill exception. 838 */ 839 #define MIPS64_REFILL_INSNS 32 840 841 static void __cpuinit build_r4000_tlb_refill_handler(void) 842 { 843 u32 *p = tlb_handler; 844 struct uasm_label *l = labels; 845 struct uasm_reloc *r = relocs; 846 u32 *f; 847 unsigned int final_len; 848 849 memset(tlb_handler, 0, sizeof(tlb_handler)); 850 memset(labels, 0, sizeof(labels)); 851 memset(relocs, 0, sizeof(relocs)); 852 memset(final_handler, 0, sizeof(final_handler)); 853 854 /* 855 * create the plain linear handler 856 */ 857 if (bcm1250_m3_war()) { 858 unsigned int segbits = 44; 859 860 uasm_i_dmfc0(&p, K0, C0_BADVADDR); 861 uasm_i_dmfc0(&p, K1, C0_ENTRYHI); 862 uasm_i_xor(&p, K0, K0, K1); 863 uasm_i_dsrl_safe(&p, K1, K0, 62); 864 uasm_i_dsrl_safe(&p, K0, K0, 12 + 1); 865 uasm_i_dsll_safe(&p, K0, K0, 64 + 12 + 1 - segbits); 866 uasm_i_or(&p, K0, K0, K1); 867 uasm_il_bnez(&p, &r, K0, label_leave); 868 /* No need for uasm_i_nop */ 869 } 870 871 #ifdef CONFIG_64BIT 872 build_get_pmde64(&p, &l, &r, K0, K1); /* get pmd in K1 */ 873 #else 874 build_get_pgde32(&p, K0, K1); /* get pgd in K1 */ 875 #endif 876 877 #ifdef CONFIG_HUGETLB_PAGE 878 build_is_huge_pte(&p, &r, K0, K1, label_tlb_huge_update); 879 #endif 880 881 build_get_ptep(&p, K0, K1); 882 build_update_entries(&p, K0, K1); 883 build_tlb_write_entry(&p, &l, &r, tlb_random); 884 uasm_l_leave(&l, p); 885 uasm_i_eret(&p); /* return from trap */ 886 887 #ifdef CONFIG_HUGETLB_PAGE 888 uasm_l_tlb_huge_update(&l, p); 889 UASM_i_LW(&p, K0, 0, K1); 890 build_huge_update_entries(&p, K0, K1); 891 build_huge_tlb_write_entry(&p, &l, &r, K0, tlb_random); 892 #endif 893 894 #ifdef CONFIG_64BIT 895 build_get_pgd_vmalloc64(&p, &l, &r, K0, K1, refill); 896 #endif 897 898 /* 899 * Overflow check: For the 64bit handler, we need at least one 900 * free instruction slot for the wrap-around branch. In worst 901 * case, if the intended insertion point is a delay slot, we 902 * need three, with the second nop'ed and the third being 903 * unused. 904 */ 905 /* Loongson2 ebase is different than r4k, we have more space */ 906 #if defined(CONFIG_32BIT) || defined(CONFIG_CPU_LOONGSON2) 907 if ((p - tlb_handler) > 64) 908 panic("TLB refill handler space exceeded"); 909 #else 910 if (((p - tlb_handler) > (MIPS64_REFILL_INSNS * 2) - 1) 911 || (((p - tlb_handler) > (MIPS64_REFILL_INSNS * 2) - 3) 912 && uasm_insn_has_bdelay(relocs, 913 tlb_handler + MIPS64_REFILL_INSNS - 3))) 914 panic("TLB refill handler space exceeded"); 915 #endif 916 917 /* 918 * Now fold the handler in the TLB refill handler space. 919 */ 920 #if defined(CONFIG_32BIT) || defined(CONFIG_CPU_LOONGSON2) 921 f = final_handler; 922 /* Simplest case, just copy the handler. */ 923 uasm_copy_handler(relocs, labels, tlb_handler, p, f); 924 final_len = p - tlb_handler; 925 #else /* CONFIG_64BIT */ 926 f = final_handler + MIPS64_REFILL_INSNS; 927 if ((p - tlb_handler) <= MIPS64_REFILL_INSNS) { 928 /* Just copy the handler. */ 929 uasm_copy_handler(relocs, labels, tlb_handler, p, f); 930 final_len = p - tlb_handler; 931 } else { 932 #if defined(CONFIG_HUGETLB_PAGE) 933 const enum label_id ls = label_tlb_huge_update; 934 #else 935 const enum label_id ls = label_vmalloc; 936 #endif 937 u32 *split; 938 int ov = 0; 939 int i; 940 941 for (i = 0; i < ARRAY_SIZE(labels) && labels[i].lab != ls; i++) 942 ; 943 BUG_ON(i == ARRAY_SIZE(labels)); 944 split = labels[i].addr; 945 946 /* 947 * See if we have overflown one way or the other. 948 */ 949 if (split > tlb_handler + MIPS64_REFILL_INSNS || 950 split < p - MIPS64_REFILL_INSNS) 951 ov = 1; 952 953 if (ov) { 954 /* 955 * Split two instructions before the end. One 956 * for the branch and one for the instruction 957 * in the delay slot. 958 */ 959 split = tlb_handler + MIPS64_REFILL_INSNS - 2; 960 961 /* 962 * If the branch would fall in a delay slot, 963 * we must back up an additional instruction 964 * so that it is no longer in a delay slot. 965 */ 966 if (uasm_insn_has_bdelay(relocs, split - 1)) 967 split--; 968 } 969 /* Copy first part of the handler. */ 970 uasm_copy_handler(relocs, labels, tlb_handler, split, f); 971 f += split - tlb_handler; 972 973 if (ov) { 974 /* Insert branch. */ 975 uasm_l_split(&l, final_handler); 976 uasm_il_b(&f, &r, label_split); 977 if (uasm_insn_has_bdelay(relocs, split)) 978 uasm_i_nop(&f); 979 else { 980 uasm_copy_handler(relocs, labels, 981 split, split + 1, f); 982 uasm_move_labels(labels, f, f + 1, -1); 983 f++; 984 split++; 985 } 986 } 987 988 /* Copy the rest of the handler. */ 989 uasm_copy_handler(relocs, labels, split, p, final_handler); 990 final_len = (f - (final_handler + MIPS64_REFILL_INSNS)) + 991 (p - split); 992 } 993 #endif /* CONFIG_64BIT */ 994 995 uasm_resolve_relocs(relocs, labels); 996 pr_debug("Wrote TLB refill handler (%u instructions).\n", 997 final_len); 998 999 memcpy((void *)ebase, final_handler, 0x100); 1000 1001 dump_handler((u32 *)ebase, 64); 1002 } 1003 1004 /* 1005 * 128 instructions for the fastpath handler is generous and should 1006 * never be exceeded. 1007 */ 1008 #define FASTPATH_SIZE 128 1009 1010 u32 handle_tlbl[FASTPATH_SIZE] __cacheline_aligned; 1011 u32 handle_tlbs[FASTPATH_SIZE] __cacheline_aligned; 1012 u32 handle_tlbm[FASTPATH_SIZE] __cacheline_aligned; 1013 1014 static void __cpuinit 1015 iPTE_LW(u32 **p, unsigned int pte, unsigned int ptr) 1016 { 1017 #ifdef CONFIG_SMP 1018 # ifdef CONFIG_64BIT_PHYS_ADDR 1019 if (cpu_has_64bits) 1020 uasm_i_lld(p, pte, 0, ptr); 1021 else 1022 # endif 1023 UASM_i_LL(p, pte, 0, ptr); 1024 #else 1025 # ifdef CONFIG_64BIT_PHYS_ADDR 1026 if (cpu_has_64bits) 1027 uasm_i_ld(p, pte, 0, ptr); 1028 else 1029 # endif 1030 UASM_i_LW(p, pte, 0, ptr); 1031 #endif 1032 } 1033 1034 static void __cpuinit 1035 iPTE_SW(u32 **p, struct uasm_reloc **r, unsigned int pte, unsigned int ptr, 1036 unsigned int mode) 1037 { 1038 #ifdef CONFIG_64BIT_PHYS_ADDR 1039 unsigned int hwmode = mode & (_PAGE_VALID | _PAGE_DIRTY); 1040 #endif 1041 1042 uasm_i_ori(p, pte, pte, mode); 1043 #ifdef CONFIG_SMP 1044 # ifdef CONFIG_64BIT_PHYS_ADDR 1045 if (cpu_has_64bits) 1046 uasm_i_scd(p, pte, 0, ptr); 1047 else 1048 # endif 1049 UASM_i_SC(p, pte, 0, ptr); 1050 1051 if (r10000_llsc_war()) 1052 uasm_il_beqzl(p, r, pte, label_smp_pgtable_change); 1053 else 1054 uasm_il_beqz(p, r, pte, label_smp_pgtable_change); 1055 1056 # ifdef CONFIG_64BIT_PHYS_ADDR 1057 if (!cpu_has_64bits) { 1058 /* no uasm_i_nop needed */ 1059 uasm_i_ll(p, pte, sizeof(pte_t) / 2, ptr); 1060 uasm_i_ori(p, pte, pte, hwmode); 1061 uasm_i_sc(p, pte, sizeof(pte_t) / 2, ptr); 1062 uasm_il_beqz(p, r, pte, label_smp_pgtable_change); 1063 /* no uasm_i_nop needed */ 1064 uasm_i_lw(p, pte, 0, ptr); 1065 } else 1066 uasm_i_nop(p); 1067 # else 1068 uasm_i_nop(p); 1069 # endif 1070 #else 1071 # ifdef CONFIG_64BIT_PHYS_ADDR 1072 if (cpu_has_64bits) 1073 uasm_i_sd(p, pte, 0, ptr); 1074 else 1075 # endif 1076 UASM_i_SW(p, pte, 0, ptr); 1077 1078 # ifdef CONFIG_64BIT_PHYS_ADDR 1079 if (!cpu_has_64bits) { 1080 uasm_i_lw(p, pte, sizeof(pte_t) / 2, ptr); 1081 uasm_i_ori(p, pte, pte, hwmode); 1082 uasm_i_sw(p, pte, sizeof(pte_t) / 2, ptr); 1083 uasm_i_lw(p, pte, 0, ptr); 1084 } 1085 # endif 1086 #endif 1087 } 1088 1089 /* 1090 * Check if PTE is present, if not then jump to LABEL. PTR points to 1091 * the page table where this PTE is located, PTE will be re-loaded 1092 * with it's original value. 1093 */ 1094 static void __cpuinit 1095 build_pte_present(u32 **p, struct uasm_reloc **r, 1096 unsigned int pte, unsigned int ptr, enum label_id lid) 1097 { 1098 if (kernel_uses_smartmips_rixi) { 1099 uasm_i_andi(p, pte, pte, _PAGE_PRESENT); 1100 uasm_il_beqz(p, r, pte, lid); 1101 } else { 1102 uasm_i_andi(p, pte, pte, _PAGE_PRESENT | _PAGE_READ); 1103 uasm_i_xori(p, pte, pte, _PAGE_PRESENT | _PAGE_READ); 1104 uasm_il_bnez(p, r, pte, lid); 1105 } 1106 iPTE_LW(p, pte, ptr); 1107 } 1108 1109 /* Make PTE valid, store result in PTR. */ 1110 static void __cpuinit 1111 build_make_valid(u32 **p, struct uasm_reloc **r, unsigned int pte, 1112 unsigned int ptr) 1113 { 1114 unsigned int mode = _PAGE_VALID | _PAGE_ACCESSED; 1115 1116 iPTE_SW(p, r, pte, ptr, mode); 1117 } 1118 1119 /* 1120 * Check if PTE can be written to, if not branch to LABEL. Regardless 1121 * restore PTE with value from PTR when done. 1122 */ 1123 static void __cpuinit 1124 build_pte_writable(u32 **p, struct uasm_reloc **r, 1125 unsigned int pte, unsigned int ptr, enum label_id lid) 1126 { 1127 uasm_i_andi(p, pte, pte, _PAGE_PRESENT | _PAGE_WRITE); 1128 uasm_i_xori(p, pte, pte, _PAGE_PRESENT | _PAGE_WRITE); 1129 uasm_il_bnez(p, r, pte, lid); 1130 iPTE_LW(p, pte, ptr); 1131 } 1132 1133 /* Make PTE writable, update software status bits as well, then store 1134 * at PTR. 1135 */ 1136 static void __cpuinit 1137 build_make_write(u32 **p, struct uasm_reloc **r, unsigned int pte, 1138 unsigned int ptr) 1139 { 1140 unsigned int mode = (_PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID 1141 | _PAGE_DIRTY); 1142 1143 iPTE_SW(p, r, pte, ptr, mode); 1144 } 1145 1146 /* 1147 * Check if PTE can be modified, if not branch to LABEL. Regardless 1148 * restore PTE with value from PTR when done. 1149 */ 1150 static void __cpuinit 1151 build_pte_modifiable(u32 **p, struct uasm_reloc **r, 1152 unsigned int pte, unsigned int ptr, enum label_id lid) 1153 { 1154 uasm_i_andi(p, pte, pte, _PAGE_WRITE); 1155 uasm_il_beqz(p, r, pte, lid); 1156 iPTE_LW(p, pte, ptr); 1157 } 1158 1159 #ifndef CONFIG_MIPS_PGD_C0_CONTEXT 1160 /* 1161 * R3000 style TLB load/store/modify handlers. 1162 */ 1163 1164 /* 1165 * This places the pte into ENTRYLO0 and writes it with tlbwi. 1166 * Then it returns. 1167 */ 1168 static void __cpuinit 1169 build_r3000_pte_reload_tlbwi(u32 **p, unsigned int pte, unsigned int tmp) 1170 { 1171 uasm_i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */ 1172 uasm_i_mfc0(p, tmp, C0_EPC); /* cp0 delay */ 1173 uasm_i_tlbwi(p); 1174 uasm_i_jr(p, tmp); 1175 uasm_i_rfe(p); /* branch delay */ 1176 } 1177 1178 /* 1179 * This places the pte into ENTRYLO0 and writes it with tlbwi 1180 * or tlbwr as appropriate. This is because the index register 1181 * may have the probe fail bit set as a result of a trap on a 1182 * kseg2 access, i.e. without refill. Then it returns. 1183 */ 1184 static void __cpuinit 1185 build_r3000_tlb_reload_write(u32 **p, struct uasm_label **l, 1186 struct uasm_reloc **r, unsigned int pte, 1187 unsigned int tmp) 1188 { 1189 uasm_i_mfc0(p, tmp, C0_INDEX); 1190 uasm_i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */ 1191 uasm_il_bltz(p, r, tmp, label_r3000_write_probe_fail); /* cp0 delay */ 1192 uasm_i_mfc0(p, tmp, C0_EPC); /* branch delay */ 1193 uasm_i_tlbwi(p); /* cp0 delay */ 1194 uasm_i_jr(p, tmp); 1195 uasm_i_rfe(p); /* branch delay */ 1196 uasm_l_r3000_write_probe_fail(l, *p); 1197 uasm_i_tlbwr(p); /* cp0 delay */ 1198 uasm_i_jr(p, tmp); 1199 uasm_i_rfe(p); /* branch delay */ 1200 } 1201 1202 static void __cpuinit 1203 build_r3000_tlbchange_handler_head(u32 **p, unsigned int pte, 1204 unsigned int ptr) 1205 { 1206 long pgdc = (long)pgd_current; 1207 1208 uasm_i_mfc0(p, pte, C0_BADVADDR); 1209 uasm_i_lui(p, ptr, uasm_rel_hi(pgdc)); /* cp0 delay */ 1210 uasm_i_lw(p, ptr, uasm_rel_lo(pgdc), ptr); 1211 uasm_i_srl(p, pte, pte, 22); /* load delay */ 1212 uasm_i_sll(p, pte, pte, 2); 1213 uasm_i_addu(p, ptr, ptr, pte); 1214 uasm_i_mfc0(p, pte, C0_CONTEXT); 1215 uasm_i_lw(p, ptr, 0, ptr); /* cp0 delay */ 1216 uasm_i_andi(p, pte, pte, 0xffc); /* load delay */ 1217 uasm_i_addu(p, ptr, ptr, pte); 1218 uasm_i_lw(p, pte, 0, ptr); 1219 uasm_i_tlbp(p); /* load delay */ 1220 } 1221 1222 static void __cpuinit build_r3000_tlb_load_handler(void) 1223 { 1224 u32 *p = handle_tlbl; 1225 struct uasm_label *l = labels; 1226 struct uasm_reloc *r = relocs; 1227 1228 memset(handle_tlbl, 0, sizeof(handle_tlbl)); 1229 memset(labels, 0, sizeof(labels)); 1230 memset(relocs, 0, sizeof(relocs)); 1231 1232 build_r3000_tlbchange_handler_head(&p, K0, K1); 1233 build_pte_present(&p, &r, K0, K1, label_nopage_tlbl); 1234 uasm_i_nop(&p); /* load delay */ 1235 build_make_valid(&p, &r, K0, K1); 1236 build_r3000_tlb_reload_write(&p, &l, &r, K0, K1); 1237 1238 uasm_l_nopage_tlbl(&l, p); 1239 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff); 1240 uasm_i_nop(&p); 1241 1242 if ((p - handle_tlbl) > FASTPATH_SIZE) 1243 panic("TLB load handler fastpath space exceeded"); 1244 1245 uasm_resolve_relocs(relocs, labels); 1246 pr_debug("Wrote TLB load handler fastpath (%u instructions).\n", 1247 (unsigned int)(p - handle_tlbl)); 1248 1249 dump_handler(handle_tlbl, ARRAY_SIZE(handle_tlbl)); 1250 } 1251 1252 static void __cpuinit build_r3000_tlb_store_handler(void) 1253 { 1254 u32 *p = handle_tlbs; 1255 struct uasm_label *l = labels; 1256 struct uasm_reloc *r = relocs; 1257 1258 memset(handle_tlbs, 0, sizeof(handle_tlbs)); 1259 memset(labels, 0, sizeof(labels)); 1260 memset(relocs, 0, sizeof(relocs)); 1261 1262 build_r3000_tlbchange_handler_head(&p, K0, K1); 1263 build_pte_writable(&p, &r, K0, K1, label_nopage_tlbs); 1264 uasm_i_nop(&p); /* load delay */ 1265 build_make_write(&p, &r, K0, K1); 1266 build_r3000_tlb_reload_write(&p, &l, &r, K0, K1); 1267 1268 uasm_l_nopage_tlbs(&l, p); 1269 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); 1270 uasm_i_nop(&p); 1271 1272 if ((p - handle_tlbs) > FASTPATH_SIZE) 1273 panic("TLB store handler fastpath space exceeded"); 1274 1275 uasm_resolve_relocs(relocs, labels); 1276 pr_debug("Wrote TLB store handler fastpath (%u instructions).\n", 1277 (unsigned int)(p - handle_tlbs)); 1278 1279 dump_handler(handle_tlbs, ARRAY_SIZE(handle_tlbs)); 1280 } 1281 1282 static void __cpuinit build_r3000_tlb_modify_handler(void) 1283 { 1284 u32 *p = handle_tlbm; 1285 struct uasm_label *l = labels; 1286 struct uasm_reloc *r = relocs; 1287 1288 memset(handle_tlbm, 0, sizeof(handle_tlbm)); 1289 memset(labels, 0, sizeof(labels)); 1290 memset(relocs, 0, sizeof(relocs)); 1291 1292 build_r3000_tlbchange_handler_head(&p, K0, K1); 1293 build_pte_modifiable(&p, &r, K0, K1, label_nopage_tlbm); 1294 uasm_i_nop(&p); /* load delay */ 1295 build_make_write(&p, &r, K0, K1); 1296 build_r3000_pte_reload_tlbwi(&p, K0, K1); 1297 1298 uasm_l_nopage_tlbm(&l, p); 1299 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); 1300 uasm_i_nop(&p); 1301 1302 if ((p - handle_tlbm) > FASTPATH_SIZE) 1303 panic("TLB modify handler fastpath space exceeded"); 1304 1305 uasm_resolve_relocs(relocs, labels); 1306 pr_debug("Wrote TLB modify handler fastpath (%u instructions).\n", 1307 (unsigned int)(p - handle_tlbm)); 1308 1309 dump_handler(handle_tlbm, ARRAY_SIZE(handle_tlbm)); 1310 } 1311 #endif /* CONFIG_MIPS_PGD_C0_CONTEXT */ 1312 1313 /* 1314 * R4000 style TLB load/store/modify handlers. 1315 */ 1316 static void __cpuinit 1317 build_r4000_tlbchange_handler_head(u32 **p, struct uasm_label **l, 1318 struct uasm_reloc **r, unsigned int pte, 1319 unsigned int ptr) 1320 { 1321 #ifdef CONFIG_64BIT 1322 build_get_pmde64(p, l, r, pte, ptr); /* get pmd in ptr */ 1323 #else 1324 build_get_pgde32(p, pte, ptr); /* get pgd in ptr */ 1325 #endif 1326 1327 #ifdef CONFIG_HUGETLB_PAGE 1328 /* 1329 * For huge tlb entries, pmd doesn't contain an address but 1330 * instead contains the tlb pte. Check the PAGE_HUGE bit and 1331 * see if we need to jump to huge tlb processing. 1332 */ 1333 build_is_huge_pte(p, r, pte, ptr, label_tlb_huge_update); 1334 #endif 1335 1336 UASM_i_MFC0(p, pte, C0_BADVADDR); 1337 UASM_i_LW(p, ptr, 0, ptr); 1338 UASM_i_SRL(p, pte, pte, PAGE_SHIFT + PTE_ORDER - PTE_T_LOG2); 1339 uasm_i_andi(p, pte, pte, (PTRS_PER_PTE - 1) << PTE_T_LOG2); 1340 UASM_i_ADDU(p, ptr, ptr, pte); 1341 1342 #ifdef CONFIG_SMP 1343 uasm_l_smp_pgtable_change(l, *p); 1344 #endif 1345 iPTE_LW(p, pte, ptr); /* get even pte */ 1346 if (!m4kc_tlbp_war()) 1347 build_tlb_probe_entry(p); 1348 } 1349 1350 static void __cpuinit 1351 build_r4000_tlbchange_handler_tail(u32 **p, struct uasm_label **l, 1352 struct uasm_reloc **r, unsigned int tmp, 1353 unsigned int ptr) 1354 { 1355 uasm_i_ori(p, ptr, ptr, sizeof(pte_t)); 1356 uasm_i_xori(p, ptr, ptr, sizeof(pte_t)); 1357 build_update_entries(p, tmp, ptr); 1358 build_tlb_write_entry(p, l, r, tlb_indexed); 1359 uasm_l_leave(l, *p); 1360 uasm_i_eret(p); /* return from trap */ 1361 1362 #ifdef CONFIG_64BIT 1363 build_get_pgd_vmalloc64(p, l, r, tmp, ptr, not_refill); 1364 #endif 1365 } 1366 1367 static void __cpuinit build_r4000_tlb_load_handler(void) 1368 { 1369 u32 *p = handle_tlbl; 1370 struct uasm_label *l = labels; 1371 struct uasm_reloc *r = relocs; 1372 1373 memset(handle_tlbl, 0, sizeof(handle_tlbl)); 1374 memset(labels, 0, sizeof(labels)); 1375 memset(relocs, 0, sizeof(relocs)); 1376 1377 if (bcm1250_m3_war()) { 1378 unsigned int segbits = 44; 1379 1380 uasm_i_dmfc0(&p, K0, C0_BADVADDR); 1381 uasm_i_dmfc0(&p, K1, C0_ENTRYHI); 1382 uasm_i_xor(&p, K0, K0, K1); 1383 uasm_i_dsrl_safe(&p, K1, K0, 62); 1384 uasm_i_dsrl_safe(&p, K0, K0, 12 + 1); 1385 uasm_i_dsll_safe(&p, K0, K0, 64 + 12 + 1 - segbits); 1386 uasm_i_or(&p, K0, K0, K1); 1387 uasm_il_bnez(&p, &r, K0, label_leave); 1388 /* No need for uasm_i_nop */ 1389 } 1390 1391 build_r4000_tlbchange_handler_head(&p, &l, &r, K0, K1); 1392 build_pte_present(&p, &r, K0, K1, label_nopage_tlbl); 1393 if (m4kc_tlbp_war()) 1394 build_tlb_probe_entry(&p); 1395 1396 if (kernel_uses_smartmips_rixi) { 1397 /* 1398 * If the page is not _PAGE_VALID, RI or XI could not 1399 * have triggered it. Skip the expensive test.. 1400 */ 1401 uasm_i_andi(&p, K0, K0, _PAGE_VALID); 1402 uasm_il_beqz(&p, &r, K0, label_tlbl_goaround1); 1403 uasm_i_nop(&p); 1404 1405 uasm_i_tlbr(&p); 1406 /* Examine entrylo 0 or 1 based on ptr. */ 1407 uasm_i_andi(&p, K0, K1, sizeof(pte_t)); 1408 uasm_i_beqz(&p, K0, 8); 1409 1410 UASM_i_MFC0(&p, K0, C0_ENTRYLO0); /* load it in the delay slot*/ 1411 UASM_i_MFC0(&p, K0, C0_ENTRYLO1); /* load it if ptr is odd */ 1412 /* 1413 * If the entryLo (now in K0) is valid (bit 1), RI or 1414 * XI must have triggered it. 1415 */ 1416 uasm_i_andi(&p, K0, K0, 2); 1417 uasm_il_bnez(&p, &r, K0, label_nopage_tlbl); 1418 1419 uasm_l_tlbl_goaround1(&l, p); 1420 /* Reload the PTE value */ 1421 iPTE_LW(&p, K0, K1); 1422 } 1423 build_make_valid(&p, &r, K0, K1); 1424 build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1); 1425 1426 #ifdef CONFIG_HUGETLB_PAGE 1427 /* 1428 * This is the entry point when build_r4000_tlbchange_handler_head 1429 * spots a huge page. 1430 */ 1431 uasm_l_tlb_huge_update(&l, p); 1432 iPTE_LW(&p, K0, K1); 1433 build_pte_present(&p, &r, K0, K1, label_nopage_tlbl); 1434 build_tlb_probe_entry(&p); 1435 1436 if (kernel_uses_smartmips_rixi) { 1437 /* 1438 * If the page is not _PAGE_VALID, RI or XI could not 1439 * have triggered it. Skip the expensive test.. 1440 */ 1441 uasm_i_andi(&p, K0, K0, _PAGE_VALID); 1442 uasm_il_beqz(&p, &r, K0, label_tlbl_goaround2); 1443 uasm_i_nop(&p); 1444 1445 uasm_i_tlbr(&p); 1446 /* Examine entrylo 0 or 1 based on ptr. */ 1447 uasm_i_andi(&p, K0, K1, sizeof(pte_t)); 1448 uasm_i_beqz(&p, K0, 8); 1449 1450 UASM_i_MFC0(&p, K0, C0_ENTRYLO0); /* load it in the delay slot*/ 1451 UASM_i_MFC0(&p, K0, C0_ENTRYLO1); /* load it if ptr is odd */ 1452 /* 1453 * If the entryLo (now in K0) is valid (bit 1), RI or 1454 * XI must have triggered it. 1455 */ 1456 uasm_i_andi(&p, K0, K0, 2); 1457 uasm_il_beqz(&p, &r, K0, label_tlbl_goaround2); 1458 /* Reload the PTE value */ 1459 iPTE_LW(&p, K0, K1); 1460 1461 /* 1462 * We clobbered C0_PAGEMASK, restore it. On the other branch 1463 * it is restored in build_huge_tlb_write_entry. 1464 */ 1465 build_restore_pagemask(&p, &r, K0, label_nopage_tlbl); 1466 1467 uasm_l_tlbl_goaround2(&l, p); 1468 } 1469 uasm_i_ori(&p, K0, K0, (_PAGE_ACCESSED | _PAGE_VALID)); 1470 build_huge_handler_tail(&p, &r, &l, K0, K1); 1471 #endif 1472 1473 uasm_l_nopage_tlbl(&l, p); 1474 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff); 1475 uasm_i_nop(&p); 1476 1477 if ((p - handle_tlbl) > FASTPATH_SIZE) 1478 panic("TLB load handler fastpath space exceeded"); 1479 1480 uasm_resolve_relocs(relocs, labels); 1481 pr_debug("Wrote TLB load handler fastpath (%u instructions).\n", 1482 (unsigned int)(p - handle_tlbl)); 1483 1484 dump_handler(handle_tlbl, ARRAY_SIZE(handle_tlbl)); 1485 } 1486 1487 static void __cpuinit build_r4000_tlb_store_handler(void) 1488 { 1489 u32 *p = handle_tlbs; 1490 struct uasm_label *l = labels; 1491 struct uasm_reloc *r = relocs; 1492 1493 memset(handle_tlbs, 0, sizeof(handle_tlbs)); 1494 memset(labels, 0, sizeof(labels)); 1495 memset(relocs, 0, sizeof(relocs)); 1496 1497 build_r4000_tlbchange_handler_head(&p, &l, &r, K0, K1); 1498 build_pte_writable(&p, &r, K0, K1, label_nopage_tlbs); 1499 if (m4kc_tlbp_war()) 1500 build_tlb_probe_entry(&p); 1501 build_make_write(&p, &r, K0, K1); 1502 build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1); 1503 1504 #ifdef CONFIG_HUGETLB_PAGE 1505 /* 1506 * This is the entry point when 1507 * build_r4000_tlbchange_handler_head spots a huge page. 1508 */ 1509 uasm_l_tlb_huge_update(&l, p); 1510 iPTE_LW(&p, K0, K1); 1511 build_pte_writable(&p, &r, K0, K1, label_nopage_tlbs); 1512 build_tlb_probe_entry(&p); 1513 uasm_i_ori(&p, K0, K0, 1514 _PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID | _PAGE_DIRTY); 1515 build_huge_handler_tail(&p, &r, &l, K0, K1); 1516 #endif 1517 1518 uasm_l_nopage_tlbs(&l, p); 1519 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); 1520 uasm_i_nop(&p); 1521 1522 if ((p - handle_tlbs) > FASTPATH_SIZE) 1523 panic("TLB store handler fastpath space exceeded"); 1524 1525 uasm_resolve_relocs(relocs, labels); 1526 pr_debug("Wrote TLB store handler fastpath (%u instructions).\n", 1527 (unsigned int)(p - handle_tlbs)); 1528 1529 dump_handler(handle_tlbs, ARRAY_SIZE(handle_tlbs)); 1530 } 1531 1532 static void __cpuinit build_r4000_tlb_modify_handler(void) 1533 { 1534 u32 *p = handle_tlbm; 1535 struct uasm_label *l = labels; 1536 struct uasm_reloc *r = relocs; 1537 1538 memset(handle_tlbm, 0, sizeof(handle_tlbm)); 1539 memset(labels, 0, sizeof(labels)); 1540 memset(relocs, 0, sizeof(relocs)); 1541 1542 build_r4000_tlbchange_handler_head(&p, &l, &r, K0, K1); 1543 build_pte_modifiable(&p, &r, K0, K1, label_nopage_tlbm); 1544 if (m4kc_tlbp_war()) 1545 build_tlb_probe_entry(&p); 1546 /* Present and writable bits set, set accessed and dirty bits. */ 1547 build_make_write(&p, &r, K0, K1); 1548 build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1); 1549 1550 #ifdef CONFIG_HUGETLB_PAGE 1551 /* 1552 * This is the entry point when 1553 * build_r4000_tlbchange_handler_head spots a huge page. 1554 */ 1555 uasm_l_tlb_huge_update(&l, p); 1556 iPTE_LW(&p, K0, K1); 1557 build_pte_modifiable(&p, &r, K0, K1, label_nopage_tlbm); 1558 build_tlb_probe_entry(&p); 1559 uasm_i_ori(&p, K0, K0, 1560 _PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID | _PAGE_DIRTY); 1561 build_huge_handler_tail(&p, &r, &l, K0, K1); 1562 #endif 1563 1564 uasm_l_nopage_tlbm(&l, p); 1565 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); 1566 uasm_i_nop(&p); 1567 1568 if ((p - handle_tlbm) > FASTPATH_SIZE) 1569 panic("TLB modify handler fastpath space exceeded"); 1570 1571 uasm_resolve_relocs(relocs, labels); 1572 pr_debug("Wrote TLB modify handler fastpath (%u instructions).\n", 1573 (unsigned int)(p - handle_tlbm)); 1574 1575 dump_handler(handle_tlbm, ARRAY_SIZE(handle_tlbm)); 1576 } 1577 1578 void __cpuinit build_tlb_refill_handler(void) 1579 { 1580 /* 1581 * The refill handler is generated per-CPU, multi-node systems 1582 * may have local storage for it. The other handlers are only 1583 * needed once. 1584 */ 1585 static int run_once = 0; 1586 1587 #ifdef CONFIG_64BIT 1588 check_for_high_segbits = current_cpu_data.vmbits > (PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3); 1589 #endif 1590 1591 switch (current_cpu_type()) { 1592 case CPU_R2000: 1593 case CPU_R3000: 1594 case CPU_R3000A: 1595 case CPU_R3081E: 1596 case CPU_TX3912: 1597 case CPU_TX3922: 1598 case CPU_TX3927: 1599 #ifndef CONFIG_MIPS_PGD_C0_CONTEXT 1600 build_r3000_tlb_refill_handler(); 1601 if (!run_once) { 1602 build_r3000_tlb_load_handler(); 1603 build_r3000_tlb_store_handler(); 1604 build_r3000_tlb_modify_handler(); 1605 run_once++; 1606 } 1607 #else 1608 panic("No R3000 TLB refill handler"); 1609 #endif 1610 break; 1611 1612 case CPU_R6000: 1613 case CPU_R6000A: 1614 panic("No R6000 TLB refill handler yet"); 1615 break; 1616 1617 case CPU_R8000: 1618 panic("No R8000 TLB refill handler yet"); 1619 break; 1620 1621 default: 1622 build_r4000_tlb_refill_handler(); 1623 if (!run_once) { 1624 build_r4000_tlb_load_handler(); 1625 build_r4000_tlb_store_handler(); 1626 build_r4000_tlb_modify_handler(); 1627 run_once++; 1628 } 1629 } 1630 } 1631 1632 void __cpuinit flush_tlb_handlers(void) 1633 { 1634 local_flush_icache_range((unsigned long)handle_tlbl, 1635 (unsigned long)handle_tlbl + sizeof(handle_tlbl)); 1636 local_flush_icache_range((unsigned long)handle_tlbs, 1637 (unsigned long)handle_tlbs + sizeof(handle_tlbs)); 1638 local_flush_icache_range((unsigned long)handle_tlbm, 1639 (unsigned long)handle_tlbm + sizeof(handle_tlbm)); 1640 } 1641