1 /* 2 * This file is subject to the terms and conditions of the GNU General Public 3 * License. See the file "COPYING" in the main directory of this archive 4 * for more details. 5 * 6 * Synthesize TLB refill handlers at runtime. 7 * 8 * Copyright (C) 2004, 2005, 2006, 2008 Thiemo Seufer 9 * Copyright (C) 2005, 2007, 2008, 2009 Maciej W. Rozycki 10 * Copyright (C) 2006 Ralf Baechle (ralf@linux-mips.org) 11 * Copyright (C) 2008, 2009 Cavium Networks, Inc. 12 * 13 * ... and the days got worse and worse and now you see 14 * I've gone completly out of my mind. 15 * 16 * They're coming to take me a away haha 17 * they're coming to take me a away hoho hihi haha 18 * to the funny farm where code is beautiful all the time ... 19 * 20 * (Condolences to Napoleon XIV) 21 */ 22 23 #include <linux/bug.h> 24 #include <linux/kernel.h> 25 #include <linux/types.h> 26 #include <linux/smp.h> 27 #include <linux/string.h> 28 #include <linux/init.h> 29 #include <linux/cache.h> 30 31 #include <asm/cacheflush.h> 32 #include <asm/pgtable.h> 33 #include <asm/war.h> 34 #include <asm/uasm.h> 35 36 /* 37 * TLB load/store/modify handlers. 38 * 39 * Only the fastpath gets synthesized at runtime, the slowpath for 40 * do_page_fault remains normal asm. 41 */ 42 extern void tlb_do_page_fault_0(void); 43 extern void tlb_do_page_fault_1(void); 44 45 46 static inline int r45k_bvahwbug(void) 47 { 48 /* XXX: We should probe for the presence of this bug, but we don't. */ 49 return 0; 50 } 51 52 static inline int r4k_250MHZhwbug(void) 53 { 54 /* XXX: We should probe for the presence of this bug, but we don't. */ 55 return 0; 56 } 57 58 static inline int __maybe_unused bcm1250_m3_war(void) 59 { 60 return BCM1250_M3_WAR; 61 } 62 63 static inline int __maybe_unused r10000_llsc_war(void) 64 { 65 return R10000_LLSC_WAR; 66 } 67 68 static int use_bbit_insns(void) 69 { 70 switch (current_cpu_type()) { 71 case CPU_CAVIUM_OCTEON: 72 case CPU_CAVIUM_OCTEON_PLUS: 73 case CPU_CAVIUM_OCTEON2: 74 return 1; 75 default: 76 return 0; 77 } 78 } 79 80 static int use_lwx_insns(void) 81 { 82 switch (current_cpu_type()) { 83 case CPU_CAVIUM_OCTEON2: 84 return 1; 85 default: 86 return 0; 87 } 88 } 89 #if defined(CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE) && \ 90 CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE > 0 91 static bool scratchpad_available(void) 92 { 93 return true; 94 } 95 static int scratchpad_offset(int i) 96 { 97 /* 98 * CVMSEG starts at address -32768 and extends for 99 * CAVIUM_OCTEON_CVMSEG_SIZE 128 byte cache lines. 100 */ 101 i += 1; /* Kernel use starts at the top and works down. */ 102 return CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE * 128 - (8 * i) - 32768; 103 } 104 #else 105 static bool scratchpad_available(void) 106 { 107 return false; 108 } 109 static int scratchpad_offset(int i) 110 { 111 BUG(); 112 } 113 #endif 114 /* 115 * Found by experiment: At least some revisions of the 4kc throw under 116 * some circumstances a machine check exception, triggered by invalid 117 * values in the index register. Delaying the tlbp instruction until 118 * after the next branch, plus adding an additional nop in front of 119 * tlbwi/tlbwr avoids the invalid index register values. Nobody knows 120 * why; it's not an issue caused by the core RTL. 121 * 122 */ 123 static int __cpuinit m4kc_tlbp_war(void) 124 { 125 return (current_cpu_data.processor_id & 0xffff00) == 126 (PRID_COMP_MIPS | PRID_IMP_4KC); 127 } 128 129 /* Handle labels (which must be positive integers). */ 130 enum label_id { 131 label_second_part = 1, 132 label_leave, 133 label_vmalloc, 134 label_vmalloc_done, 135 label_tlbw_hazard, 136 label_split, 137 label_tlbl_goaround1, 138 label_tlbl_goaround2, 139 label_nopage_tlbl, 140 label_nopage_tlbs, 141 label_nopage_tlbm, 142 label_smp_pgtable_change, 143 label_r3000_write_probe_fail, 144 label_large_segbits_fault, 145 #ifdef CONFIG_HUGETLB_PAGE 146 label_tlb_huge_update, 147 #endif 148 }; 149 150 UASM_L_LA(_second_part) 151 UASM_L_LA(_leave) 152 UASM_L_LA(_vmalloc) 153 UASM_L_LA(_vmalloc_done) 154 UASM_L_LA(_tlbw_hazard) 155 UASM_L_LA(_split) 156 UASM_L_LA(_tlbl_goaround1) 157 UASM_L_LA(_tlbl_goaround2) 158 UASM_L_LA(_nopage_tlbl) 159 UASM_L_LA(_nopage_tlbs) 160 UASM_L_LA(_nopage_tlbm) 161 UASM_L_LA(_smp_pgtable_change) 162 UASM_L_LA(_r3000_write_probe_fail) 163 UASM_L_LA(_large_segbits_fault) 164 #ifdef CONFIG_HUGETLB_PAGE 165 UASM_L_LA(_tlb_huge_update) 166 #endif 167 168 /* 169 * For debug purposes. 170 */ 171 static inline void dump_handler(const u32 *handler, int count) 172 { 173 int i; 174 175 pr_debug("\t.set push\n"); 176 pr_debug("\t.set noreorder\n"); 177 178 for (i = 0; i < count; i++) 179 pr_debug("\t%p\t.word 0x%08x\n", &handler[i], handler[i]); 180 181 pr_debug("\t.set pop\n"); 182 } 183 184 /* The only general purpose registers allowed in TLB handlers. */ 185 #define K0 26 186 #define K1 27 187 188 /* Some CP0 registers */ 189 #define C0_INDEX 0, 0 190 #define C0_ENTRYLO0 2, 0 191 #define C0_TCBIND 2, 2 192 #define C0_ENTRYLO1 3, 0 193 #define C0_CONTEXT 4, 0 194 #define C0_PAGEMASK 5, 0 195 #define C0_BADVADDR 8, 0 196 #define C0_ENTRYHI 10, 0 197 #define C0_EPC 14, 0 198 #define C0_XCONTEXT 20, 0 199 200 #ifdef CONFIG_64BIT 201 # define GET_CONTEXT(buf, reg) UASM_i_MFC0(buf, reg, C0_XCONTEXT) 202 #else 203 # define GET_CONTEXT(buf, reg) UASM_i_MFC0(buf, reg, C0_CONTEXT) 204 #endif 205 206 /* The worst case length of the handler is around 18 instructions for 207 * R3000-style TLBs and up to 63 instructions for R4000-style TLBs. 208 * Maximum space available is 32 instructions for R3000 and 64 209 * instructions for R4000. 210 * 211 * We deliberately chose a buffer size of 128, so we won't scribble 212 * over anything important on overflow before we panic. 213 */ 214 static u32 tlb_handler[128] __cpuinitdata; 215 216 /* simply assume worst case size for labels and relocs */ 217 static struct uasm_label labels[128] __cpuinitdata; 218 static struct uasm_reloc relocs[128] __cpuinitdata; 219 220 #ifdef CONFIG_64BIT 221 static int check_for_high_segbits __cpuinitdata; 222 #endif 223 224 static int check_for_high_segbits __cpuinitdata; 225 226 static unsigned int kscratch_used_mask __cpuinitdata; 227 228 static int __cpuinit allocate_kscratch(void) 229 { 230 int r; 231 unsigned int a = cpu_data[0].kscratch_mask & ~kscratch_used_mask; 232 233 r = ffs(a); 234 235 if (r == 0) 236 return -1; 237 238 r--; /* make it zero based */ 239 240 kscratch_used_mask |= (1 << r); 241 242 return r; 243 } 244 245 static int scratch_reg __cpuinitdata; 246 static int pgd_reg __cpuinitdata; 247 enum vmalloc64_mode {not_refill, refill_scratch, refill_noscratch}; 248 249 #ifndef CONFIG_MIPS_PGD_C0_CONTEXT 250 251 /* 252 * CONFIG_MIPS_PGD_C0_CONTEXT implies 64 bit and lack of pgd_current, 253 * we cannot do r3000 under these circumstances. 254 * 255 * Declare pgd_current here instead of including mmu_context.h to avoid type 256 * conflicts for tlbmiss_handler_setup_pgd 257 */ 258 extern unsigned long pgd_current[]; 259 260 /* 261 * The R3000 TLB handler is simple. 262 */ 263 static void __cpuinit build_r3000_tlb_refill_handler(void) 264 { 265 long pgdc = (long)pgd_current; 266 u32 *p; 267 268 memset(tlb_handler, 0, sizeof(tlb_handler)); 269 p = tlb_handler; 270 271 uasm_i_mfc0(&p, K0, C0_BADVADDR); 272 uasm_i_lui(&p, K1, uasm_rel_hi(pgdc)); /* cp0 delay */ 273 uasm_i_lw(&p, K1, uasm_rel_lo(pgdc), K1); 274 uasm_i_srl(&p, K0, K0, 22); /* load delay */ 275 uasm_i_sll(&p, K0, K0, 2); 276 uasm_i_addu(&p, K1, K1, K0); 277 uasm_i_mfc0(&p, K0, C0_CONTEXT); 278 uasm_i_lw(&p, K1, 0, K1); /* cp0 delay */ 279 uasm_i_andi(&p, K0, K0, 0xffc); /* load delay */ 280 uasm_i_addu(&p, K1, K1, K0); 281 uasm_i_lw(&p, K0, 0, K1); 282 uasm_i_nop(&p); /* load delay */ 283 uasm_i_mtc0(&p, K0, C0_ENTRYLO0); 284 uasm_i_mfc0(&p, K1, C0_EPC); /* cp0 delay */ 285 uasm_i_tlbwr(&p); /* cp0 delay */ 286 uasm_i_jr(&p, K1); 287 uasm_i_rfe(&p); /* branch delay */ 288 289 if (p > tlb_handler + 32) 290 panic("TLB refill handler space exceeded"); 291 292 pr_debug("Wrote TLB refill handler (%u instructions).\n", 293 (unsigned int)(p - tlb_handler)); 294 295 memcpy((void *)ebase, tlb_handler, 0x80); 296 297 dump_handler((u32 *)ebase, 32); 298 } 299 #endif /* CONFIG_MIPS_PGD_C0_CONTEXT */ 300 301 /* 302 * The R4000 TLB handler is much more complicated. We have two 303 * consecutive handler areas with 32 instructions space each. 304 * Since they aren't used at the same time, we can overflow in the 305 * other one.To keep things simple, we first assume linear space, 306 * then we relocate it to the final handler layout as needed. 307 */ 308 static u32 final_handler[64] __cpuinitdata; 309 310 /* 311 * Hazards 312 * 313 * From the IDT errata for the QED RM5230 (Nevada), processor revision 1.0: 314 * 2. A timing hazard exists for the TLBP instruction. 315 * 316 * stalling_instruction 317 * TLBP 318 * 319 * The JTLB is being read for the TLBP throughout the stall generated by the 320 * previous instruction. This is not really correct as the stalling instruction 321 * can modify the address used to access the JTLB. The failure symptom is that 322 * the TLBP instruction will use an address created for the stalling instruction 323 * and not the address held in C0_ENHI and thus report the wrong results. 324 * 325 * The software work-around is to not allow the instruction preceding the TLBP 326 * to stall - make it an NOP or some other instruction guaranteed not to stall. 327 * 328 * Errata 2 will not be fixed. This errata is also on the R5000. 329 * 330 * As if we MIPS hackers wouldn't know how to nop pipelines happy ... 331 */ 332 static void __cpuinit __maybe_unused build_tlb_probe_entry(u32 **p) 333 { 334 switch (current_cpu_type()) { 335 /* Found by experiment: R4600 v2.0/R4700 needs this, too. */ 336 case CPU_R4600: 337 case CPU_R4700: 338 case CPU_R5000: 339 case CPU_R5000A: 340 case CPU_NEVADA: 341 uasm_i_nop(p); 342 uasm_i_tlbp(p); 343 break; 344 345 default: 346 uasm_i_tlbp(p); 347 break; 348 } 349 } 350 351 /* 352 * Write random or indexed TLB entry, and care about the hazards from 353 * the preceeding mtc0 and for the following eret. 354 */ 355 enum tlb_write_entry { tlb_random, tlb_indexed }; 356 357 static void __cpuinit build_tlb_write_entry(u32 **p, struct uasm_label **l, 358 struct uasm_reloc **r, 359 enum tlb_write_entry wmode) 360 { 361 void(*tlbw)(u32 **) = NULL; 362 363 switch (wmode) { 364 case tlb_random: tlbw = uasm_i_tlbwr; break; 365 case tlb_indexed: tlbw = uasm_i_tlbwi; break; 366 } 367 368 if (cpu_has_mips_r2) { 369 if (cpu_has_mips_r2_exec_hazard) 370 uasm_i_ehb(p); 371 tlbw(p); 372 return; 373 } 374 375 switch (current_cpu_type()) { 376 case CPU_R4000PC: 377 case CPU_R4000SC: 378 case CPU_R4000MC: 379 case CPU_R4400PC: 380 case CPU_R4400SC: 381 case CPU_R4400MC: 382 /* 383 * This branch uses up a mtc0 hazard nop slot and saves 384 * two nops after the tlbw instruction. 385 */ 386 uasm_il_bgezl(p, r, 0, label_tlbw_hazard); 387 tlbw(p); 388 uasm_l_tlbw_hazard(l, *p); 389 uasm_i_nop(p); 390 break; 391 392 case CPU_R4600: 393 case CPU_R4700: 394 case CPU_R5000: 395 case CPU_R5000A: 396 uasm_i_nop(p); 397 tlbw(p); 398 uasm_i_nop(p); 399 break; 400 401 case CPU_R4300: 402 case CPU_5KC: 403 case CPU_TX49XX: 404 case CPU_PR4450: 405 uasm_i_nop(p); 406 tlbw(p); 407 break; 408 409 case CPU_R10000: 410 case CPU_R12000: 411 case CPU_R14000: 412 case CPU_4KC: 413 case CPU_4KEC: 414 case CPU_SB1: 415 case CPU_SB1A: 416 case CPU_4KSC: 417 case CPU_20KC: 418 case CPU_25KF: 419 case CPU_BMIPS32: 420 case CPU_BMIPS3300: 421 case CPU_BMIPS4350: 422 case CPU_BMIPS4380: 423 case CPU_BMIPS5000: 424 case CPU_LOONGSON2: 425 case CPU_R5500: 426 if (m4kc_tlbp_war()) 427 uasm_i_nop(p); 428 case CPU_ALCHEMY: 429 tlbw(p); 430 break; 431 432 case CPU_NEVADA: 433 uasm_i_nop(p); /* QED specifies 2 nops hazard */ 434 /* 435 * This branch uses up a mtc0 hazard nop slot and saves 436 * a nop after the tlbw instruction. 437 */ 438 uasm_il_bgezl(p, r, 0, label_tlbw_hazard); 439 tlbw(p); 440 uasm_l_tlbw_hazard(l, *p); 441 break; 442 443 case CPU_RM7000: 444 uasm_i_nop(p); 445 uasm_i_nop(p); 446 uasm_i_nop(p); 447 uasm_i_nop(p); 448 tlbw(p); 449 break; 450 451 case CPU_RM9000: 452 /* 453 * When the JTLB is updated by tlbwi or tlbwr, a subsequent 454 * use of the JTLB for instructions should not occur for 4 455 * cpu cycles and use for data translations should not occur 456 * for 3 cpu cycles. 457 */ 458 uasm_i_ssnop(p); 459 uasm_i_ssnop(p); 460 uasm_i_ssnop(p); 461 uasm_i_ssnop(p); 462 tlbw(p); 463 uasm_i_ssnop(p); 464 uasm_i_ssnop(p); 465 uasm_i_ssnop(p); 466 uasm_i_ssnop(p); 467 break; 468 469 case CPU_VR4111: 470 case CPU_VR4121: 471 case CPU_VR4122: 472 case CPU_VR4181: 473 case CPU_VR4181A: 474 uasm_i_nop(p); 475 uasm_i_nop(p); 476 tlbw(p); 477 uasm_i_nop(p); 478 uasm_i_nop(p); 479 break; 480 481 case CPU_VR4131: 482 case CPU_VR4133: 483 case CPU_R5432: 484 uasm_i_nop(p); 485 uasm_i_nop(p); 486 tlbw(p); 487 break; 488 489 case CPU_JZRISC: 490 tlbw(p); 491 uasm_i_nop(p); 492 break; 493 494 default: 495 panic("No TLB refill handler yet (CPU type: %d)", 496 current_cpu_data.cputype); 497 break; 498 } 499 } 500 501 static __cpuinit __maybe_unused void build_convert_pte_to_entrylo(u32 **p, 502 unsigned int reg) 503 { 504 if (kernel_uses_smartmips_rixi) { 505 UASM_i_SRL(p, reg, reg, ilog2(_PAGE_NO_EXEC)); 506 UASM_i_ROTR(p, reg, reg, ilog2(_PAGE_GLOBAL) - ilog2(_PAGE_NO_EXEC)); 507 } else { 508 #ifdef CONFIG_64BIT_PHYS_ADDR 509 uasm_i_dsrl_safe(p, reg, reg, ilog2(_PAGE_GLOBAL)); 510 #else 511 UASM_i_SRL(p, reg, reg, ilog2(_PAGE_GLOBAL)); 512 #endif 513 } 514 } 515 516 #ifdef CONFIG_HUGETLB_PAGE 517 518 static __cpuinit void build_restore_pagemask(u32 **p, 519 struct uasm_reloc **r, 520 unsigned int tmp, 521 enum label_id lid, 522 int restore_scratch) 523 { 524 if (restore_scratch) { 525 /* Reset default page size */ 526 if (PM_DEFAULT_MASK >> 16) { 527 uasm_i_lui(p, tmp, PM_DEFAULT_MASK >> 16); 528 uasm_i_ori(p, tmp, tmp, PM_DEFAULT_MASK & 0xffff); 529 uasm_i_mtc0(p, tmp, C0_PAGEMASK); 530 uasm_il_b(p, r, lid); 531 } else if (PM_DEFAULT_MASK) { 532 uasm_i_ori(p, tmp, 0, PM_DEFAULT_MASK); 533 uasm_i_mtc0(p, tmp, C0_PAGEMASK); 534 uasm_il_b(p, r, lid); 535 } else { 536 uasm_i_mtc0(p, 0, C0_PAGEMASK); 537 uasm_il_b(p, r, lid); 538 } 539 if (scratch_reg > 0) 540 UASM_i_MFC0(p, 1, 31, scratch_reg); 541 else 542 UASM_i_LW(p, 1, scratchpad_offset(0), 0); 543 } else { 544 /* Reset default page size */ 545 if (PM_DEFAULT_MASK >> 16) { 546 uasm_i_lui(p, tmp, PM_DEFAULT_MASK >> 16); 547 uasm_i_ori(p, tmp, tmp, PM_DEFAULT_MASK & 0xffff); 548 uasm_il_b(p, r, lid); 549 uasm_i_mtc0(p, tmp, C0_PAGEMASK); 550 } else if (PM_DEFAULT_MASK) { 551 uasm_i_ori(p, tmp, 0, PM_DEFAULT_MASK); 552 uasm_il_b(p, r, lid); 553 uasm_i_mtc0(p, tmp, C0_PAGEMASK); 554 } else { 555 uasm_il_b(p, r, lid); 556 uasm_i_mtc0(p, 0, C0_PAGEMASK); 557 } 558 } 559 } 560 561 static __cpuinit void build_huge_tlb_write_entry(u32 **p, 562 struct uasm_label **l, 563 struct uasm_reloc **r, 564 unsigned int tmp, 565 enum tlb_write_entry wmode, 566 int restore_scratch) 567 { 568 /* Set huge page tlb entry size */ 569 uasm_i_lui(p, tmp, PM_HUGE_MASK >> 16); 570 uasm_i_ori(p, tmp, tmp, PM_HUGE_MASK & 0xffff); 571 uasm_i_mtc0(p, tmp, C0_PAGEMASK); 572 573 build_tlb_write_entry(p, l, r, wmode); 574 575 build_restore_pagemask(p, r, tmp, label_leave, restore_scratch); 576 } 577 578 /* 579 * Check if Huge PTE is present, if so then jump to LABEL. 580 */ 581 static void __cpuinit 582 build_is_huge_pte(u32 **p, struct uasm_reloc **r, unsigned int tmp, 583 unsigned int pmd, int lid) 584 { 585 UASM_i_LW(p, tmp, 0, pmd); 586 if (use_bbit_insns()) { 587 uasm_il_bbit1(p, r, tmp, ilog2(_PAGE_HUGE), lid); 588 } else { 589 uasm_i_andi(p, tmp, tmp, _PAGE_HUGE); 590 uasm_il_bnez(p, r, tmp, lid); 591 } 592 } 593 594 static __cpuinit void build_huge_update_entries(u32 **p, 595 unsigned int pte, 596 unsigned int tmp) 597 { 598 int small_sequence; 599 600 /* 601 * A huge PTE describes an area the size of the 602 * configured huge page size. This is twice the 603 * of the large TLB entry size we intend to use. 604 * A TLB entry half the size of the configured 605 * huge page size is configured into entrylo0 606 * and entrylo1 to cover the contiguous huge PTE 607 * address space. 608 */ 609 small_sequence = (HPAGE_SIZE >> 7) < 0x10000; 610 611 /* We can clobber tmp. It isn't used after this.*/ 612 if (!small_sequence) 613 uasm_i_lui(p, tmp, HPAGE_SIZE >> (7 + 16)); 614 615 build_convert_pte_to_entrylo(p, pte); 616 UASM_i_MTC0(p, pte, C0_ENTRYLO0); /* load it */ 617 /* convert to entrylo1 */ 618 if (small_sequence) 619 UASM_i_ADDIU(p, pte, pte, HPAGE_SIZE >> 7); 620 else 621 UASM_i_ADDU(p, pte, pte, tmp); 622 623 UASM_i_MTC0(p, pte, C0_ENTRYLO1); /* load it */ 624 } 625 626 static __cpuinit void build_huge_handler_tail(u32 **p, 627 struct uasm_reloc **r, 628 struct uasm_label **l, 629 unsigned int pte, 630 unsigned int ptr) 631 { 632 #ifdef CONFIG_SMP 633 UASM_i_SC(p, pte, 0, ptr); 634 uasm_il_beqz(p, r, pte, label_tlb_huge_update); 635 UASM_i_LW(p, pte, 0, ptr); /* Needed because SC killed our PTE */ 636 #else 637 UASM_i_SW(p, pte, 0, ptr); 638 #endif 639 build_huge_update_entries(p, pte, ptr); 640 build_huge_tlb_write_entry(p, l, r, pte, tlb_indexed, 0); 641 } 642 #endif /* CONFIG_HUGETLB_PAGE */ 643 644 #ifdef CONFIG_64BIT 645 /* 646 * TMP and PTR are scratch. 647 * TMP will be clobbered, PTR will hold the pmd entry. 648 */ 649 static void __cpuinit 650 build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r, 651 unsigned int tmp, unsigned int ptr) 652 { 653 #ifndef CONFIG_MIPS_PGD_C0_CONTEXT 654 long pgdc = (long)pgd_current; 655 #endif 656 /* 657 * The vmalloc handling is not in the hotpath. 658 */ 659 uasm_i_dmfc0(p, tmp, C0_BADVADDR); 660 661 if (check_for_high_segbits) { 662 /* 663 * The kernel currently implicitely assumes that the 664 * MIPS SEGBITS parameter for the processor is 665 * (PGDIR_SHIFT+PGDIR_BITS) or less, and will never 666 * allocate virtual addresses outside the maximum 667 * range for SEGBITS = (PGDIR_SHIFT+PGDIR_BITS). But 668 * that doesn't prevent user code from accessing the 669 * higher xuseg addresses. Here, we make sure that 670 * everything but the lower xuseg addresses goes down 671 * the module_alloc/vmalloc path. 672 */ 673 uasm_i_dsrl_safe(p, ptr, tmp, PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3); 674 uasm_il_bnez(p, r, ptr, label_vmalloc); 675 } else { 676 uasm_il_bltz(p, r, tmp, label_vmalloc); 677 } 678 /* No uasm_i_nop needed here, since the next insn doesn't touch TMP. */ 679 680 #ifdef CONFIG_MIPS_PGD_C0_CONTEXT 681 if (pgd_reg != -1) { 682 /* pgd is in pgd_reg */ 683 UASM_i_MFC0(p, ptr, 31, pgd_reg); 684 } else { 685 /* 686 * &pgd << 11 stored in CONTEXT [23..63]. 687 */ 688 UASM_i_MFC0(p, ptr, C0_CONTEXT); 689 690 /* Clear lower 23 bits of context. */ 691 uasm_i_dins(p, ptr, 0, 0, 23); 692 693 /* 1 0 1 0 1 << 6 xkphys cached */ 694 uasm_i_ori(p, ptr, ptr, 0x540); 695 uasm_i_drotr(p, ptr, ptr, 11); 696 } 697 #elif defined(CONFIG_SMP) 698 # ifdef CONFIG_MIPS_MT_SMTC 699 /* 700 * SMTC uses TCBind value as "CPU" index 701 */ 702 uasm_i_mfc0(p, ptr, C0_TCBIND); 703 uasm_i_dsrl_safe(p, ptr, ptr, 19); 704 # else 705 /* 706 * 64 bit SMP running in XKPHYS has smp_processor_id() << 3 707 * stored in CONTEXT. 708 */ 709 uasm_i_dmfc0(p, ptr, C0_CONTEXT); 710 uasm_i_dsrl_safe(p, ptr, ptr, 23); 711 # endif 712 UASM_i_LA_mostly(p, tmp, pgdc); 713 uasm_i_daddu(p, ptr, ptr, tmp); 714 uasm_i_dmfc0(p, tmp, C0_BADVADDR); 715 uasm_i_ld(p, ptr, uasm_rel_lo(pgdc), ptr); 716 #else 717 UASM_i_LA_mostly(p, ptr, pgdc); 718 uasm_i_ld(p, ptr, uasm_rel_lo(pgdc), ptr); 719 #endif 720 721 uasm_l_vmalloc_done(l, *p); 722 723 /* get pgd offset in bytes */ 724 uasm_i_dsrl_safe(p, tmp, tmp, PGDIR_SHIFT - 3); 725 726 uasm_i_andi(p, tmp, tmp, (PTRS_PER_PGD - 1)<<3); 727 uasm_i_daddu(p, ptr, ptr, tmp); /* add in pgd offset */ 728 #ifndef __PAGETABLE_PMD_FOLDED 729 uasm_i_dmfc0(p, tmp, C0_BADVADDR); /* get faulting address */ 730 uasm_i_ld(p, ptr, 0, ptr); /* get pmd pointer */ 731 uasm_i_dsrl_safe(p, tmp, tmp, PMD_SHIFT-3); /* get pmd offset in bytes */ 732 uasm_i_andi(p, tmp, tmp, (PTRS_PER_PMD - 1)<<3); 733 uasm_i_daddu(p, ptr, ptr, tmp); /* add in pmd offset */ 734 #endif 735 } 736 737 /* 738 * BVADDR is the faulting address, PTR is scratch. 739 * PTR will hold the pgd for vmalloc. 740 */ 741 static void __cpuinit 742 build_get_pgd_vmalloc64(u32 **p, struct uasm_label **l, struct uasm_reloc **r, 743 unsigned int bvaddr, unsigned int ptr, 744 enum vmalloc64_mode mode) 745 { 746 long swpd = (long)swapper_pg_dir; 747 int single_insn_swpd; 748 int did_vmalloc_branch = 0; 749 750 single_insn_swpd = uasm_in_compat_space_p(swpd) && !uasm_rel_lo(swpd); 751 752 uasm_l_vmalloc(l, *p); 753 754 if (mode != not_refill && check_for_high_segbits) { 755 if (single_insn_swpd) { 756 uasm_il_bltz(p, r, bvaddr, label_vmalloc_done); 757 uasm_i_lui(p, ptr, uasm_rel_hi(swpd)); 758 did_vmalloc_branch = 1; 759 /* fall through */ 760 } else { 761 uasm_il_bgez(p, r, bvaddr, label_large_segbits_fault); 762 } 763 } 764 if (!did_vmalloc_branch) { 765 if (uasm_in_compat_space_p(swpd) && !uasm_rel_lo(swpd)) { 766 uasm_il_b(p, r, label_vmalloc_done); 767 uasm_i_lui(p, ptr, uasm_rel_hi(swpd)); 768 } else { 769 UASM_i_LA_mostly(p, ptr, swpd); 770 uasm_il_b(p, r, label_vmalloc_done); 771 if (uasm_in_compat_space_p(swpd)) 772 uasm_i_addiu(p, ptr, ptr, uasm_rel_lo(swpd)); 773 else 774 uasm_i_daddiu(p, ptr, ptr, uasm_rel_lo(swpd)); 775 } 776 } 777 if (mode != not_refill && check_for_high_segbits) { 778 uasm_l_large_segbits_fault(l, *p); 779 /* 780 * We get here if we are an xsseg address, or if we are 781 * an xuseg address above (PGDIR_SHIFT+PGDIR_BITS) boundary. 782 * 783 * Ignoring xsseg (assume disabled so would generate 784 * (address errors?), the only remaining possibility 785 * is the upper xuseg addresses. On processors with 786 * TLB_SEGBITS <= PGDIR_SHIFT+PGDIR_BITS, these 787 * addresses would have taken an address error. We try 788 * to mimic that here by taking a load/istream page 789 * fault. 790 */ 791 UASM_i_LA(p, ptr, (unsigned long)tlb_do_page_fault_0); 792 uasm_i_jr(p, ptr); 793 794 if (mode == refill_scratch) { 795 if (scratch_reg > 0) 796 UASM_i_MFC0(p, 1, 31, scratch_reg); 797 else 798 UASM_i_LW(p, 1, scratchpad_offset(0), 0); 799 } else { 800 uasm_i_nop(p); 801 } 802 } 803 } 804 805 #else /* !CONFIG_64BIT */ 806 807 /* 808 * TMP and PTR are scratch. 809 * TMP will be clobbered, PTR will hold the pgd entry. 810 */ 811 static void __cpuinit __maybe_unused 812 build_get_pgde32(u32 **p, unsigned int tmp, unsigned int ptr) 813 { 814 long pgdc = (long)pgd_current; 815 816 /* 32 bit SMP has smp_processor_id() stored in CONTEXT. */ 817 #ifdef CONFIG_SMP 818 #ifdef CONFIG_MIPS_MT_SMTC 819 /* 820 * SMTC uses TCBind value as "CPU" index 821 */ 822 uasm_i_mfc0(p, ptr, C0_TCBIND); 823 UASM_i_LA_mostly(p, tmp, pgdc); 824 uasm_i_srl(p, ptr, ptr, 19); 825 #else 826 /* 827 * smp_processor_id() << 3 is stored in CONTEXT. 828 */ 829 uasm_i_mfc0(p, ptr, C0_CONTEXT); 830 UASM_i_LA_mostly(p, tmp, pgdc); 831 uasm_i_srl(p, ptr, ptr, 23); 832 #endif 833 uasm_i_addu(p, ptr, tmp, ptr); 834 #else 835 UASM_i_LA_mostly(p, ptr, pgdc); 836 #endif 837 uasm_i_mfc0(p, tmp, C0_BADVADDR); /* get faulting address */ 838 uasm_i_lw(p, ptr, uasm_rel_lo(pgdc), ptr); 839 uasm_i_srl(p, tmp, tmp, PGDIR_SHIFT); /* get pgd only bits */ 840 uasm_i_sll(p, tmp, tmp, PGD_T_LOG2); 841 uasm_i_addu(p, ptr, ptr, tmp); /* add in pgd offset */ 842 } 843 844 #endif /* !CONFIG_64BIT */ 845 846 static void __cpuinit build_adjust_context(u32 **p, unsigned int ctx) 847 { 848 unsigned int shift = 4 - (PTE_T_LOG2 + 1) + PAGE_SHIFT - 12; 849 unsigned int mask = (PTRS_PER_PTE / 2 - 1) << (PTE_T_LOG2 + 1); 850 851 switch (current_cpu_type()) { 852 case CPU_VR41XX: 853 case CPU_VR4111: 854 case CPU_VR4121: 855 case CPU_VR4122: 856 case CPU_VR4131: 857 case CPU_VR4181: 858 case CPU_VR4181A: 859 case CPU_VR4133: 860 shift += 2; 861 break; 862 863 default: 864 break; 865 } 866 867 if (shift) 868 UASM_i_SRL(p, ctx, ctx, shift); 869 uasm_i_andi(p, ctx, ctx, mask); 870 } 871 872 static void __cpuinit build_get_ptep(u32 **p, unsigned int tmp, unsigned int ptr) 873 { 874 /* 875 * Bug workaround for the Nevada. It seems as if under certain 876 * circumstances the move from cp0_context might produce a 877 * bogus result when the mfc0 instruction and its consumer are 878 * in a different cacheline or a load instruction, probably any 879 * memory reference, is between them. 880 */ 881 switch (current_cpu_type()) { 882 case CPU_NEVADA: 883 UASM_i_LW(p, ptr, 0, ptr); 884 GET_CONTEXT(p, tmp); /* get context reg */ 885 break; 886 887 default: 888 GET_CONTEXT(p, tmp); /* get context reg */ 889 UASM_i_LW(p, ptr, 0, ptr); 890 break; 891 } 892 893 build_adjust_context(p, tmp); 894 UASM_i_ADDU(p, ptr, ptr, tmp); /* add in offset */ 895 } 896 897 static void __cpuinit build_update_entries(u32 **p, unsigned int tmp, 898 unsigned int ptep) 899 { 900 /* 901 * 64bit address support (36bit on a 32bit CPU) in a 32bit 902 * Kernel is a special case. Only a few CPUs use it. 903 */ 904 #ifdef CONFIG_64BIT_PHYS_ADDR 905 if (cpu_has_64bits) { 906 uasm_i_ld(p, tmp, 0, ptep); /* get even pte */ 907 uasm_i_ld(p, ptep, sizeof(pte_t), ptep); /* get odd pte */ 908 if (kernel_uses_smartmips_rixi) { 909 UASM_i_SRL(p, tmp, tmp, ilog2(_PAGE_NO_EXEC)); 910 UASM_i_SRL(p, ptep, ptep, ilog2(_PAGE_NO_EXEC)); 911 UASM_i_ROTR(p, tmp, tmp, ilog2(_PAGE_GLOBAL) - ilog2(_PAGE_NO_EXEC)); 912 UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */ 913 UASM_i_ROTR(p, ptep, ptep, ilog2(_PAGE_GLOBAL) - ilog2(_PAGE_NO_EXEC)); 914 } else { 915 uasm_i_dsrl_safe(p, tmp, tmp, ilog2(_PAGE_GLOBAL)); /* convert to entrylo0 */ 916 UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */ 917 uasm_i_dsrl_safe(p, ptep, ptep, ilog2(_PAGE_GLOBAL)); /* convert to entrylo1 */ 918 } 919 UASM_i_MTC0(p, ptep, C0_ENTRYLO1); /* load it */ 920 } else { 921 int pte_off_even = sizeof(pte_t) / 2; 922 int pte_off_odd = pte_off_even + sizeof(pte_t); 923 924 /* The pte entries are pre-shifted */ 925 uasm_i_lw(p, tmp, pte_off_even, ptep); /* get even pte */ 926 UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */ 927 uasm_i_lw(p, ptep, pte_off_odd, ptep); /* get odd pte */ 928 UASM_i_MTC0(p, ptep, C0_ENTRYLO1); /* load it */ 929 } 930 #else 931 UASM_i_LW(p, tmp, 0, ptep); /* get even pte */ 932 UASM_i_LW(p, ptep, sizeof(pte_t), ptep); /* get odd pte */ 933 if (r45k_bvahwbug()) 934 build_tlb_probe_entry(p); 935 if (kernel_uses_smartmips_rixi) { 936 UASM_i_SRL(p, tmp, tmp, ilog2(_PAGE_NO_EXEC)); 937 UASM_i_SRL(p, ptep, ptep, ilog2(_PAGE_NO_EXEC)); 938 UASM_i_ROTR(p, tmp, tmp, ilog2(_PAGE_GLOBAL) - ilog2(_PAGE_NO_EXEC)); 939 if (r4k_250MHZhwbug()) 940 UASM_i_MTC0(p, 0, C0_ENTRYLO0); 941 UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */ 942 UASM_i_ROTR(p, ptep, ptep, ilog2(_PAGE_GLOBAL) - ilog2(_PAGE_NO_EXEC)); 943 } else { 944 UASM_i_SRL(p, tmp, tmp, ilog2(_PAGE_GLOBAL)); /* convert to entrylo0 */ 945 if (r4k_250MHZhwbug()) 946 UASM_i_MTC0(p, 0, C0_ENTRYLO0); 947 UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */ 948 UASM_i_SRL(p, ptep, ptep, ilog2(_PAGE_GLOBAL)); /* convert to entrylo1 */ 949 if (r45k_bvahwbug()) 950 uasm_i_mfc0(p, tmp, C0_INDEX); 951 } 952 if (r4k_250MHZhwbug()) 953 UASM_i_MTC0(p, 0, C0_ENTRYLO1); 954 UASM_i_MTC0(p, ptep, C0_ENTRYLO1); /* load it */ 955 #endif 956 } 957 958 struct mips_huge_tlb_info { 959 int huge_pte; 960 int restore_scratch; 961 }; 962 963 static struct mips_huge_tlb_info __cpuinit 964 build_fast_tlb_refill_handler (u32 **p, struct uasm_label **l, 965 struct uasm_reloc **r, unsigned int tmp, 966 unsigned int ptr, int c0_scratch) 967 { 968 struct mips_huge_tlb_info rv; 969 unsigned int even, odd; 970 int vmalloc_branch_delay_filled = 0; 971 const int scratch = 1; /* Our extra working register */ 972 973 rv.huge_pte = scratch; 974 rv.restore_scratch = 0; 975 976 if (check_for_high_segbits) { 977 UASM_i_MFC0(p, tmp, C0_BADVADDR); 978 979 if (pgd_reg != -1) 980 UASM_i_MFC0(p, ptr, 31, pgd_reg); 981 else 982 UASM_i_MFC0(p, ptr, C0_CONTEXT); 983 984 if (c0_scratch >= 0) 985 UASM_i_MTC0(p, scratch, 31, c0_scratch); 986 else 987 UASM_i_SW(p, scratch, scratchpad_offset(0), 0); 988 989 uasm_i_dsrl_safe(p, scratch, tmp, 990 PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3); 991 uasm_il_bnez(p, r, scratch, label_vmalloc); 992 993 if (pgd_reg == -1) { 994 vmalloc_branch_delay_filled = 1; 995 /* Clear lower 23 bits of context. */ 996 uasm_i_dins(p, ptr, 0, 0, 23); 997 } 998 } else { 999 if (pgd_reg != -1) 1000 UASM_i_MFC0(p, ptr, 31, pgd_reg); 1001 else 1002 UASM_i_MFC0(p, ptr, C0_CONTEXT); 1003 1004 UASM_i_MFC0(p, tmp, C0_BADVADDR); 1005 1006 if (c0_scratch >= 0) 1007 UASM_i_MTC0(p, scratch, 31, c0_scratch); 1008 else 1009 UASM_i_SW(p, scratch, scratchpad_offset(0), 0); 1010 1011 if (pgd_reg == -1) 1012 /* Clear lower 23 bits of context. */ 1013 uasm_i_dins(p, ptr, 0, 0, 23); 1014 1015 uasm_il_bltz(p, r, tmp, label_vmalloc); 1016 } 1017 1018 if (pgd_reg == -1) { 1019 vmalloc_branch_delay_filled = 1; 1020 /* 1 0 1 0 1 << 6 xkphys cached */ 1021 uasm_i_ori(p, ptr, ptr, 0x540); 1022 uasm_i_drotr(p, ptr, ptr, 11); 1023 } 1024 1025 #ifdef __PAGETABLE_PMD_FOLDED 1026 #define LOC_PTEP scratch 1027 #else 1028 #define LOC_PTEP ptr 1029 #endif 1030 1031 if (!vmalloc_branch_delay_filled) 1032 /* get pgd offset in bytes */ 1033 uasm_i_dsrl_safe(p, scratch, tmp, PGDIR_SHIFT - 3); 1034 1035 uasm_l_vmalloc_done(l, *p); 1036 1037 /* 1038 * tmp ptr 1039 * fall-through case = badvaddr *pgd_current 1040 * vmalloc case = badvaddr swapper_pg_dir 1041 */ 1042 1043 if (vmalloc_branch_delay_filled) 1044 /* get pgd offset in bytes */ 1045 uasm_i_dsrl_safe(p, scratch, tmp, PGDIR_SHIFT - 3); 1046 1047 #ifdef __PAGETABLE_PMD_FOLDED 1048 GET_CONTEXT(p, tmp); /* get context reg */ 1049 #endif 1050 uasm_i_andi(p, scratch, scratch, (PTRS_PER_PGD - 1) << 3); 1051 1052 if (use_lwx_insns()) { 1053 UASM_i_LWX(p, LOC_PTEP, scratch, ptr); 1054 } else { 1055 uasm_i_daddu(p, ptr, ptr, scratch); /* add in pgd offset */ 1056 uasm_i_ld(p, LOC_PTEP, 0, ptr); /* get pmd pointer */ 1057 } 1058 1059 #ifndef __PAGETABLE_PMD_FOLDED 1060 /* get pmd offset in bytes */ 1061 uasm_i_dsrl_safe(p, scratch, tmp, PMD_SHIFT - 3); 1062 uasm_i_andi(p, scratch, scratch, (PTRS_PER_PMD - 1) << 3); 1063 GET_CONTEXT(p, tmp); /* get context reg */ 1064 1065 if (use_lwx_insns()) { 1066 UASM_i_LWX(p, scratch, scratch, ptr); 1067 } else { 1068 uasm_i_daddu(p, ptr, ptr, scratch); /* add in pmd offset */ 1069 UASM_i_LW(p, scratch, 0, ptr); 1070 } 1071 #endif 1072 /* Adjust the context during the load latency. */ 1073 build_adjust_context(p, tmp); 1074 1075 #ifdef CONFIG_HUGETLB_PAGE 1076 uasm_il_bbit1(p, r, scratch, ilog2(_PAGE_HUGE), label_tlb_huge_update); 1077 /* 1078 * The in the LWX case we don't want to do the load in the 1079 * delay slot. It cannot issue in the same cycle and may be 1080 * speculative and unneeded. 1081 */ 1082 if (use_lwx_insns()) 1083 uasm_i_nop(p); 1084 #endif /* CONFIG_HUGETLB_PAGE */ 1085 1086 1087 /* build_update_entries */ 1088 if (use_lwx_insns()) { 1089 even = ptr; 1090 odd = tmp; 1091 UASM_i_LWX(p, even, scratch, tmp); 1092 UASM_i_ADDIU(p, tmp, tmp, sizeof(pte_t)); 1093 UASM_i_LWX(p, odd, scratch, tmp); 1094 } else { 1095 UASM_i_ADDU(p, ptr, scratch, tmp); /* add in offset */ 1096 even = tmp; 1097 odd = ptr; 1098 UASM_i_LW(p, even, 0, ptr); /* get even pte */ 1099 UASM_i_LW(p, odd, sizeof(pte_t), ptr); /* get odd pte */ 1100 } 1101 if (kernel_uses_smartmips_rixi) { 1102 uasm_i_dsrl_safe(p, even, even, ilog2(_PAGE_NO_EXEC)); 1103 uasm_i_dsrl_safe(p, odd, odd, ilog2(_PAGE_NO_EXEC)); 1104 uasm_i_drotr(p, even, even, 1105 ilog2(_PAGE_GLOBAL) - ilog2(_PAGE_NO_EXEC)); 1106 UASM_i_MTC0(p, even, C0_ENTRYLO0); /* load it */ 1107 uasm_i_drotr(p, odd, odd, 1108 ilog2(_PAGE_GLOBAL) - ilog2(_PAGE_NO_EXEC)); 1109 } else { 1110 uasm_i_dsrl_safe(p, even, even, ilog2(_PAGE_GLOBAL)); 1111 UASM_i_MTC0(p, even, C0_ENTRYLO0); /* load it */ 1112 uasm_i_dsrl_safe(p, odd, odd, ilog2(_PAGE_GLOBAL)); 1113 } 1114 UASM_i_MTC0(p, odd, C0_ENTRYLO1); /* load it */ 1115 1116 if (c0_scratch >= 0) { 1117 UASM_i_MFC0(p, scratch, 31, c0_scratch); 1118 build_tlb_write_entry(p, l, r, tlb_random); 1119 uasm_l_leave(l, *p); 1120 rv.restore_scratch = 1; 1121 } else if (PAGE_SHIFT == 14 || PAGE_SHIFT == 13) { 1122 build_tlb_write_entry(p, l, r, tlb_random); 1123 uasm_l_leave(l, *p); 1124 UASM_i_LW(p, scratch, scratchpad_offset(0), 0); 1125 } else { 1126 UASM_i_LW(p, scratch, scratchpad_offset(0), 0); 1127 build_tlb_write_entry(p, l, r, tlb_random); 1128 uasm_l_leave(l, *p); 1129 rv.restore_scratch = 1; 1130 } 1131 1132 uasm_i_eret(p); /* return from trap */ 1133 1134 return rv; 1135 } 1136 1137 /* 1138 * For a 64-bit kernel, we are using the 64-bit XTLB refill exception 1139 * because EXL == 0. If we wrap, we can also use the 32 instruction 1140 * slots before the XTLB refill exception handler which belong to the 1141 * unused TLB refill exception. 1142 */ 1143 #define MIPS64_REFILL_INSNS 32 1144 1145 static void __cpuinit build_r4000_tlb_refill_handler(void) 1146 { 1147 u32 *p = tlb_handler; 1148 struct uasm_label *l = labels; 1149 struct uasm_reloc *r = relocs; 1150 u32 *f; 1151 unsigned int final_len; 1152 struct mips_huge_tlb_info htlb_info; 1153 enum vmalloc64_mode vmalloc_mode; 1154 1155 memset(tlb_handler, 0, sizeof(tlb_handler)); 1156 memset(labels, 0, sizeof(labels)); 1157 memset(relocs, 0, sizeof(relocs)); 1158 memset(final_handler, 0, sizeof(final_handler)); 1159 1160 if (scratch_reg == 0) 1161 scratch_reg = allocate_kscratch(); 1162 1163 if ((scratch_reg > 0 || scratchpad_available()) && use_bbit_insns()) { 1164 htlb_info = build_fast_tlb_refill_handler(&p, &l, &r, K0, K1, 1165 scratch_reg); 1166 vmalloc_mode = refill_scratch; 1167 } else { 1168 htlb_info.huge_pte = K0; 1169 htlb_info.restore_scratch = 0; 1170 vmalloc_mode = refill_noscratch; 1171 /* 1172 * create the plain linear handler 1173 */ 1174 if (bcm1250_m3_war()) { 1175 unsigned int segbits = 44; 1176 1177 uasm_i_dmfc0(&p, K0, C0_BADVADDR); 1178 uasm_i_dmfc0(&p, K1, C0_ENTRYHI); 1179 uasm_i_xor(&p, K0, K0, K1); 1180 uasm_i_dsrl_safe(&p, K1, K0, 62); 1181 uasm_i_dsrl_safe(&p, K0, K0, 12 + 1); 1182 uasm_i_dsll_safe(&p, K0, K0, 64 + 12 + 1 - segbits); 1183 uasm_i_or(&p, K0, K0, K1); 1184 uasm_il_bnez(&p, &r, K0, label_leave); 1185 /* No need for uasm_i_nop */ 1186 } 1187 1188 #ifdef CONFIG_64BIT 1189 build_get_pmde64(&p, &l, &r, K0, K1); /* get pmd in K1 */ 1190 #else 1191 build_get_pgde32(&p, K0, K1); /* get pgd in K1 */ 1192 #endif 1193 1194 #ifdef CONFIG_HUGETLB_PAGE 1195 build_is_huge_pte(&p, &r, K0, K1, label_tlb_huge_update); 1196 #endif 1197 1198 build_get_ptep(&p, K0, K1); 1199 build_update_entries(&p, K0, K1); 1200 build_tlb_write_entry(&p, &l, &r, tlb_random); 1201 uasm_l_leave(&l, p); 1202 uasm_i_eret(&p); /* return from trap */ 1203 } 1204 #ifdef CONFIG_HUGETLB_PAGE 1205 uasm_l_tlb_huge_update(&l, p); 1206 build_huge_update_entries(&p, htlb_info.huge_pte, K1); 1207 build_huge_tlb_write_entry(&p, &l, &r, K0, tlb_random, 1208 htlb_info.restore_scratch); 1209 #endif 1210 1211 #ifdef CONFIG_64BIT 1212 build_get_pgd_vmalloc64(&p, &l, &r, K0, K1, vmalloc_mode); 1213 #endif 1214 1215 /* 1216 * Overflow check: For the 64bit handler, we need at least one 1217 * free instruction slot for the wrap-around branch. In worst 1218 * case, if the intended insertion point is a delay slot, we 1219 * need three, with the second nop'ed and the third being 1220 * unused. 1221 */ 1222 /* Loongson2 ebase is different than r4k, we have more space */ 1223 #if defined(CONFIG_32BIT) || defined(CONFIG_CPU_LOONGSON2) 1224 if ((p - tlb_handler) > 64) 1225 panic("TLB refill handler space exceeded"); 1226 #else 1227 if (((p - tlb_handler) > (MIPS64_REFILL_INSNS * 2) - 1) 1228 || (((p - tlb_handler) > (MIPS64_REFILL_INSNS * 2) - 3) 1229 && uasm_insn_has_bdelay(relocs, 1230 tlb_handler + MIPS64_REFILL_INSNS - 3))) 1231 panic("TLB refill handler space exceeded"); 1232 #endif 1233 1234 /* 1235 * Now fold the handler in the TLB refill handler space. 1236 */ 1237 #if defined(CONFIG_32BIT) || defined(CONFIG_CPU_LOONGSON2) 1238 f = final_handler; 1239 /* Simplest case, just copy the handler. */ 1240 uasm_copy_handler(relocs, labels, tlb_handler, p, f); 1241 final_len = p - tlb_handler; 1242 #else /* CONFIG_64BIT */ 1243 f = final_handler + MIPS64_REFILL_INSNS; 1244 if ((p - tlb_handler) <= MIPS64_REFILL_INSNS) { 1245 /* Just copy the handler. */ 1246 uasm_copy_handler(relocs, labels, tlb_handler, p, f); 1247 final_len = p - tlb_handler; 1248 } else { 1249 #if defined(CONFIG_HUGETLB_PAGE) 1250 const enum label_id ls = label_tlb_huge_update; 1251 #else 1252 const enum label_id ls = label_vmalloc; 1253 #endif 1254 u32 *split; 1255 int ov = 0; 1256 int i; 1257 1258 for (i = 0; i < ARRAY_SIZE(labels) && labels[i].lab != ls; i++) 1259 ; 1260 BUG_ON(i == ARRAY_SIZE(labels)); 1261 split = labels[i].addr; 1262 1263 /* 1264 * See if we have overflown one way or the other. 1265 */ 1266 if (split > tlb_handler + MIPS64_REFILL_INSNS || 1267 split < p - MIPS64_REFILL_INSNS) 1268 ov = 1; 1269 1270 if (ov) { 1271 /* 1272 * Split two instructions before the end. One 1273 * for the branch and one for the instruction 1274 * in the delay slot. 1275 */ 1276 split = tlb_handler + MIPS64_REFILL_INSNS - 2; 1277 1278 /* 1279 * If the branch would fall in a delay slot, 1280 * we must back up an additional instruction 1281 * so that it is no longer in a delay slot. 1282 */ 1283 if (uasm_insn_has_bdelay(relocs, split - 1)) 1284 split--; 1285 } 1286 /* Copy first part of the handler. */ 1287 uasm_copy_handler(relocs, labels, tlb_handler, split, f); 1288 f += split - tlb_handler; 1289 1290 if (ov) { 1291 /* Insert branch. */ 1292 uasm_l_split(&l, final_handler); 1293 uasm_il_b(&f, &r, label_split); 1294 if (uasm_insn_has_bdelay(relocs, split)) 1295 uasm_i_nop(&f); 1296 else { 1297 uasm_copy_handler(relocs, labels, 1298 split, split + 1, f); 1299 uasm_move_labels(labels, f, f + 1, -1); 1300 f++; 1301 split++; 1302 } 1303 } 1304 1305 /* Copy the rest of the handler. */ 1306 uasm_copy_handler(relocs, labels, split, p, final_handler); 1307 final_len = (f - (final_handler + MIPS64_REFILL_INSNS)) + 1308 (p - split); 1309 } 1310 #endif /* CONFIG_64BIT */ 1311 1312 uasm_resolve_relocs(relocs, labels); 1313 pr_debug("Wrote TLB refill handler (%u instructions).\n", 1314 final_len); 1315 1316 memcpy((void *)ebase, final_handler, 0x100); 1317 1318 dump_handler((u32 *)ebase, 64); 1319 } 1320 1321 /* 1322 * 128 instructions for the fastpath handler is generous and should 1323 * never be exceeded. 1324 */ 1325 #define FASTPATH_SIZE 128 1326 1327 u32 handle_tlbl[FASTPATH_SIZE] __cacheline_aligned; 1328 u32 handle_tlbs[FASTPATH_SIZE] __cacheline_aligned; 1329 u32 handle_tlbm[FASTPATH_SIZE] __cacheline_aligned; 1330 #ifdef CONFIG_MIPS_PGD_C0_CONTEXT 1331 u32 tlbmiss_handler_setup_pgd[16] __cacheline_aligned; 1332 1333 static void __cpuinit build_r4000_setup_pgd(void) 1334 { 1335 const int a0 = 4; 1336 const int a1 = 5; 1337 u32 *p = tlbmiss_handler_setup_pgd; 1338 struct uasm_label *l = labels; 1339 struct uasm_reloc *r = relocs; 1340 1341 memset(tlbmiss_handler_setup_pgd, 0, sizeof(tlbmiss_handler_setup_pgd)); 1342 memset(labels, 0, sizeof(labels)); 1343 memset(relocs, 0, sizeof(relocs)); 1344 1345 pgd_reg = allocate_kscratch(); 1346 1347 if (pgd_reg == -1) { 1348 /* PGD << 11 in c0_Context */ 1349 /* 1350 * If it is a ckseg0 address, convert to a physical 1351 * address. Shifting right by 29 and adding 4 will 1352 * result in zero for these addresses. 1353 * 1354 */ 1355 UASM_i_SRA(&p, a1, a0, 29); 1356 UASM_i_ADDIU(&p, a1, a1, 4); 1357 uasm_il_bnez(&p, &r, a1, label_tlbl_goaround1); 1358 uasm_i_nop(&p); 1359 uasm_i_dinsm(&p, a0, 0, 29, 64 - 29); 1360 uasm_l_tlbl_goaround1(&l, p); 1361 UASM_i_SLL(&p, a0, a0, 11); 1362 uasm_i_jr(&p, 31); 1363 UASM_i_MTC0(&p, a0, C0_CONTEXT); 1364 } else { 1365 /* PGD in c0_KScratch */ 1366 uasm_i_jr(&p, 31); 1367 UASM_i_MTC0(&p, a0, 31, pgd_reg); 1368 } 1369 if (p - tlbmiss_handler_setup_pgd > ARRAY_SIZE(tlbmiss_handler_setup_pgd)) 1370 panic("tlbmiss_handler_setup_pgd space exceeded"); 1371 uasm_resolve_relocs(relocs, labels); 1372 pr_debug("Wrote tlbmiss_handler_setup_pgd (%u instructions).\n", 1373 (unsigned int)(p - tlbmiss_handler_setup_pgd)); 1374 1375 dump_handler(tlbmiss_handler_setup_pgd, 1376 ARRAY_SIZE(tlbmiss_handler_setup_pgd)); 1377 } 1378 #endif 1379 1380 static void __cpuinit 1381 iPTE_LW(u32 **p, unsigned int pte, unsigned int ptr) 1382 { 1383 #ifdef CONFIG_SMP 1384 # ifdef CONFIG_64BIT_PHYS_ADDR 1385 if (cpu_has_64bits) 1386 uasm_i_lld(p, pte, 0, ptr); 1387 else 1388 # endif 1389 UASM_i_LL(p, pte, 0, ptr); 1390 #else 1391 # ifdef CONFIG_64BIT_PHYS_ADDR 1392 if (cpu_has_64bits) 1393 uasm_i_ld(p, pte, 0, ptr); 1394 else 1395 # endif 1396 UASM_i_LW(p, pte, 0, ptr); 1397 #endif 1398 } 1399 1400 static void __cpuinit 1401 iPTE_SW(u32 **p, struct uasm_reloc **r, unsigned int pte, unsigned int ptr, 1402 unsigned int mode) 1403 { 1404 #ifdef CONFIG_64BIT_PHYS_ADDR 1405 unsigned int hwmode = mode & (_PAGE_VALID | _PAGE_DIRTY); 1406 #endif 1407 1408 uasm_i_ori(p, pte, pte, mode); 1409 #ifdef CONFIG_SMP 1410 # ifdef CONFIG_64BIT_PHYS_ADDR 1411 if (cpu_has_64bits) 1412 uasm_i_scd(p, pte, 0, ptr); 1413 else 1414 # endif 1415 UASM_i_SC(p, pte, 0, ptr); 1416 1417 if (r10000_llsc_war()) 1418 uasm_il_beqzl(p, r, pte, label_smp_pgtable_change); 1419 else 1420 uasm_il_beqz(p, r, pte, label_smp_pgtable_change); 1421 1422 # ifdef CONFIG_64BIT_PHYS_ADDR 1423 if (!cpu_has_64bits) { 1424 /* no uasm_i_nop needed */ 1425 uasm_i_ll(p, pte, sizeof(pte_t) / 2, ptr); 1426 uasm_i_ori(p, pte, pte, hwmode); 1427 uasm_i_sc(p, pte, sizeof(pte_t) / 2, ptr); 1428 uasm_il_beqz(p, r, pte, label_smp_pgtable_change); 1429 /* no uasm_i_nop needed */ 1430 uasm_i_lw(p, pte, 0, ptr); 1431 } else 1432 uasm_i_nop(p); 1433 # else 1434 uasm_i_nop(p); 1435 # endif 1436 #else 1437 # ifdef CONFIG_64BIT_PHYS_ADDR 1438 if (cpu_has_64bits) 1439 uasm_i_sd(p, pte, 0, ptr); 1440 else 1441 # endif 1442 UASM_i_SW(p, pte, 0, ptr); 1443 1444 # ifdef CONFIG_64BIT_PHYS_ADDR 1445 if (!cpu_has_64bits) { 1446 uasm_i_lw(p, pte, sizeof(pte_t) / 2, ptr); 1447 uasm_i_ori(p, pte, pte, hwmode); 1448 uasm_i_sw(p, pte, sizeof(pte_t) / 2, ptr); 1449 uasm_i_lw(p, pte, 0, ptr); 1450 } 1451 # endif 1452 #endif 1453 } 1454 1455 /* 1456 * Check if PTE is present, if not then jump to LABEL. PTR points to 1457 * the page table where this PTE is located, PTE will be re-loaded 1458 * with it's original value. 1459 */ 1460 static void __cpuinit 1461 build_pte_present(u32 **p, struct uasm_reloc **r, 1462 unsigned int pte, unsigned int ptr, enum label_id lid) 1463 { 1464 if (kernel_uses_smartmips_rixi) { 1465 if (use_bbit_insns()) { 1466 uasm_il_bbit0(p, r, pte, ilog2(_PAGE_PRESENT), lid); 1467 uasm_i_nop(p); 1468 } else { 1469 uasm_i_andi(p, pte, pte, _PAGE_PRESENT); 1470 uasm_il_beqz(p, r, pte, lid); 1471 iPTE_LW(p, pte, ptr); 1472 } 1473 } else { 1474 uasm_i_andi(p, pte, pte, _PAGE_PRESENT | _PAGE_READ); 1475 uasm_i_xori(p, pte, pte, _PAGE_PRESENT | _PAGE_READ); 1476 uasm_il_bnez(p, r, pte, lid); 1477 iPTE_LW(p, pte, ptr); 1478 } 1479 } 1480 1481 /* Make PTE valid, store result in PTR. */ 1482 static void __cpuinit 1483 build_make_valid(u32 **p, struct uasm_reloc **r, unsigned int pte, 1484 unsigned int ptr) 1485 { 1486 unsigned int mode = _PAGE_VALID | _PAGE_ACCESSED; 1487 1488 iPTE_SW(p, r, pte, ptr, mode); 1489 } 1490 1491 /* 1492 * Check if PTE can be written to, if not branch to LABEL. Regardless 1493 * restore PTE with value from PTR when done. 1494 */ 1495 static void __cpuinit 1496 build_pte_writable(u32 **p, struct uasm_reloc **r, 1497 unsigned int pte, unsigned int ptr, enum label_id lid) 1498 { 1499 if (use_bbit_insns()) { 1500 uasm_il_bbit0(p, r, pte, ilog2(_PAGE_PRESENT), lid); 1501 uasm_i_nop(p); 1502 uasm_il_bbit0(p, r, pte, ilog2(_PAGE_WRITE), lid); 1503 uasm_i_nop(p); 1504 } else { 1505 uasm_i_andi(p, pte, pte, _PAGE_PRESENT | _PAGE_WRITE); 1506 uasm_i_xori(p, pte, pte, _PAGE_PRESENT | _PAGE_WRITE); 1507 uasm_il_bnez(p, r, pte, lid); 1508 iPTE_LW(p, pte, ptr); 1509 } 1510 } 1511 1512 /* Make PTE writable, update software status bits as well, then store 1513 * at PTR. 1514 */ 1515 static void __cpuinit 1516 build_make_write(u32 **p, struct uasm_reloc **r, unsigned int pte, 1517 unsigned int ptr) 1518 { 1519 unsigned int mode = (_PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID 1520 | _PAGE_DIRTY); 1521 1522 iPTE_SW(p, r, pte, ptr, mode); 1523 } 1524 1525 /* 1526 * Check if PTE can be modified, if not branch to LABEL. Regardless 1527 * restore PTE with value from PTR when done. 1528 */ 1529 static void __cpuinit 1530 build_pte_modifiable(u32 **p, struct uasm_reloc **r, 1531 unsigned int pte, unsigned int ptr, enum label_id lid) 1532 { 1533 if (use_bbit_insns()) { 1534 uasm_il_bbit0(p, r, pte, ilog2(_PAGE_WRITE), lid); 1535 uasm_i_nop(p); 1536 } else { 1537 uasm_i_andi(p, pte, pte, _PAGE_WRITE); 1538 uasm_il_beqz(p, r, pte, lid); 1539 iPTE_LW(p, pte, ptr); 1540 } 1541 } 1542 1543 #ifndef CONFIG_MIPS_PGD_C0_CONTEXT 1544 1545 1546 /* 1547 * R3000 style TLB load/store/modify handlers. 1548 */ 1549 1550 /* 1551 * This places the pte into ENTRYLO0 and writes it with tlbwi. 1552 * Then it returns. 1553 */ 1554 static void __cpuinit 1555 build_r3000_pte_reload_tlbwi(u32 **p, unsigned int pte, unsigned int tmp) 1556 { 1557 uasm_i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */ 1558 uasm_i_mfc0(p, tmp, C0_EPC); /* cp0 delay */ 1559 uasm_i_tlbwi(p); 1560 uasm_i_jr(p, tmp); 1561 uasm_i_rfe(p); /* branch delay */ 1562 } 1563 1564 /* 1565 * This places the pte into ENTRYLO0 and writes it with tlbwi 1566 * or tlbwr as appropriate. This is because the index register 1567 * may have the probe fail bit set as a result of a trap on a 1568 * kseg2 access, i.e. without refill. Then it returns. 1569 */ 1570 static void __cpuinit 1571 build_r3000_tlb_reload_write(u32 **p, struct uasm_label **l, 1572 struct uasm_reloc **r, unsigned int pte, 1573 unsigned int tmp) 1574 { 1575 uasm_i_mfc0(p, tmp, C0_INDEX); 1576 uasm_i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */ 1577 uasm_il_bltz(p, r, tmp, label_r3000_write_probe_fail); /* cp0 delay */ 1578 uasm_i_mfc0(p, tmp, C0_EPC); /* branch delay */ 1579 uasm_i_tlbwi(p); /* cp0 delay */ 1580 uasm_i_jr(p, tmp); 1581 uasm_i_rfe(p); /* branch delay */ 1582 uasm_l_r3000_write_probe_fail(l, *p); 1583 uasm_i_tlbwr(p); /* cp0 delay */ 1584 uasm_i_jr(p, tmp); 1585 uasm_i_rfe(p); /* branch delay */ 1586 } 1587 1588 static void __cpuinit 1589 build_r3000_tlbchange_handler_head(u32 **p, unsigned int pte, 1590 unsigned int ptr) 1591 { 1592 long pgdc = (long)pgd_current; 1593 1594 uasm_i_mfc0(p, pte, C0_BADVADDR); 1595 uasm_i_lui(p, ptr, uasm_rel_hi(pgdc)); /* cp0 delay */ 1596 uasm_i_lw(p, ptr, uasm_rel_lo(pgdc), ptr); 1597 uasm_i_srl(p, pte, pte, 22); /* load delay */ 1598 uasm_i_sll(p, pte, pte, 2); 1599 uasm_i_addu(p, ptr, ptr, pte); 1600 uasm_i_mfc0(p, pte, C0_CONTEXT); 1601 uasm_i_lw(p, ptr, 0, ptr); /* cp0 delay */ 1602 uasm_i_andi(p, pte, pte, 0xffc); /* load delay */ 1603 uasm_i_addu(p, ptr, ptr, pte); 1604 uasm_i_lw(p, pte, 0, ptr); 1605 uasm_i_tlbp(p); /* load delay */ 1606 } 1607 1608 static void __cpuinit build_r3000_tlb_load_handler(void) 1609 { 1610 u32 *p = handle_tlbl; 1611 struct uasm_label *l = labels; 1612 struct uasm_reloc *r = relocs; 1613 1614 memset(handle_tlbl, 0, sizeof(handle_tlbl)); 1615 memset(labels, 0, sizeof(labels)); 1616 memset(relocs, 0, sizeof(relocs)); 1617 1618 build_r3000_tlbchange_handler_head(&p, K0, K1); 1619 build_pte_present(&p, &r, K0, K1, label_nopage_tlbl); 1620 uasm_i_nop(&p); /* load delay */ 1621 build_make_valid(&p, &r, K0, K1); 1622 build_r3000_tlb_reload_write(&p, &l, &r, K0, K1); 1623 1624 uasm_l_nopage_tlbl(&l, p); 1625 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff); 1626 uasm_i_nop(&p); 1627 1628 if ((p - handle_tlbl) > FASTPATH_SIZE) 1629 panic("TLB load handler fastpath space exceeded"); 1630 1631 uasm_resolve_relocs(relocs, labels); 1632 pr_debug("Wrote TLB load handler fastpath (%u instructions).\n", 1633 (unsigned int)(p - handle_tlbl)); 1634 1635 dump_handler(handle_tlbl, ARRAY_SIZE(handle_tlbl)); 1636 } 1637 1638 static void __cpuinit build_r3000_tlb_store_handler(void) 1639 { 1640 u32 *p = handle_tlbs; 1641 struct uasm_label *l = labels; 1642 struct uasm_reloc *r = relocs; 1643 1644 memset(handle_tlbs, 0, sizeof(handle_tlbs)); 1645 memset(labels, 0, sizeof(labels)); 1646 memset(relocs, 0, sizeof(relocs)); 1647 1648 build_r3000_tlbchange_handler_head(&p, K0, K1); 1649 build_pte_writable(&p, &r, K0, K1, label_nopage_tlbs); 1650 uasm_i_nop(&p); /* load delay */ 1651 build_make_write(&p, &r, K0, K1); 1652 build_r3000_tlb_reload_write(&p, &l, &r, K0, K1); 1653 1654 uasm_l_nopage_tlbs(&l, p); 1655 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); 1656 uasm_i_nop(&p); 1657 1658 if ((p - handle_tlbs) > FASTPATH_SIZE) 1659 panic("TLB store handler fastpath space exceeded"); 1660 1661 uasm_resolve_relocs(relocs, labels); 1662 pr_debug("Wrote TLB store handler fastpath (%u instructions).\n", 1663 (unsigned int)(p - handle_tlbs)); 1664 1665 dump_handler(handle_tlbs, ARRAY_SIZE(handle_tlbs)); 1666 } 1667 1668 static void __cpuinit build_r3000_tlb_modify_handler(void) 1669 { 1670 u32 *p = handle_tlbm; 1671 struct uasm_label *l = labels; 1672 struct uasm_reloc *r = relocs; 1673 1674 memset(handle_tlbm, 0, sizeof(handle_tlbm)); 1675 memset(labels, 0, sizeof(labels)); 1676 memset(relocs, 0, sizeof(relocs)); 1677 1678 build_r3000_tlbchange_handler_head(&p, K0, K1); 1679 build_pte_modifiable(&p, &r, K0, K1, label_nopage_tlbm); 1680 uasm_i_nop(&p); /* load delay */ 1681 build_make_write(&p, &r, K0, K1); 1682 build_r3000_pte_reload_tlbwi(&p, K0, K1); 1683 1684 uasm_l_nopage_tlbm(&l, p); 1685 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); 1686 uasm_i_nop(&p); 1687 1688 if ((p - handle_tlbm) > FASTPATH_SIZE) 1689 panic("TLB modify handler fastpath space exceeded"); 1690 1691 uasm_resolve_relocs(relocs, labels); 1692 pr_debug("Wrote TLB modify handler fastpath (%u instructions).\n", 1693 (unsigned int)(p - handle_tlbm)); 1694 1695 dump_handler(handle_tlbm, ARRAY_SIZE(handle_tlbm)); 1696 } 1697 #endif /* CONFIG_MIPS_PGD_C0_CONTEXT */ 1698 1699 /* 1700 * R4000 style TLB load/store/modify handlers. 1701 */ 1702 static void __cpuinit 1703 build_r4000_tlbchange_handler_head(u32 **p, struct uasm_label **l, 1704 struct uasm_reloc **r, unsigned int pte, 1705 unsigned int ptr) 1706 { 1707 #ifdef CONFIG_64BIT 1708 build_get_pmde64(p, l, r, pte, ptr); /* get pmd in ptr */ 1709 #else 1710 build_get_pgde32(p, pte, ptr); /* get pgd in ptr */ 1711 #endif 1712 1713 #ifdef CONFIG_HUGETLB_PAGE 1714 /* 1715 * For huge tlb entries, pmd doesn't contain an address but 1716 * instead contains the tlb pte. Check the PAGE_HUGE bit and 1717 * see if we need to jump to huge tlb processing. 1718 */ 1719 build_is_huge_pte(p, r, pte, ptr, label_tlb_huge_update); 1720 #endif 1721 1722 UASM_i_MFC0(p, pte, C0_BADVADDR); 1723 UASM_i_LW(p, ptr, 0, ptr); 1724 UASM_i_SRL(p, pte, pte, PAGE_SHIFT + PTE_ORDER - PTE_T_LOG2); 1725 uasm_i_andi(p, pte, pte, (PTRS_PER_PTE - 1) << PTE_T_LOG2); 1726 UASM_i_ADDU(p, ptr, ptr, pte); 1727 1728 #ifdef CONFIG_SMP 1729 uasm_l_smp_pgtable_change(l, *p); 1730 #endif 1731 iPTE_LW(p, pte, ptr); /* get even pte */ 1732 if (!m4kc_tlbp_war()) 1733 build_tlb_probe_entry(p); 1734 } 1735 1736 static void __cpuinit 1737 build_r4000_tlbchange_handler_tail(u32 **p, struct uasm_label **l, 1738 struct uasm_reloc **r, unsigned int tmp, 1739 unsigned int ptr) 1740 { 1741 uasm_i_ori(p, ptr, ptr, sizeof(pte_t)); 1742 uasm_i_xori(p, ptr, ptr, sizeof(pte_t)); 1743 build_update_entries(p, tmp, ptr); 1744 build_tlb_write_entry(p, l, r, tlb_indexed); 1745 uasm_l_leave(l, *p); 1746 uasm_i_eret(p); /* return from trap */ 1747 1748 #ifdef CONFIG_64BIT 1749 build_get_pgd_vmalloc64(p, l, r, tmp, ptr, not_refill); 1750 #endif 1751 } 1752 1753 static void __cpuinit build_r4000_tlb_load_handler(void) 1754 { 1755 u32 *p = handle_tlbl; 1756 struct uasm_label *l = labels; 1757 struct uasm_reloc *r = relocs; 1758 1759 memset(handle_tlbl, 0, sizeof(handle_tlbl)); 1760 memset(labels, 0, sizeof(labels)); 1761 memset(relocs, 0, sizeof(relocs)); 1762 1763 if (bcm1250_m3_war()) { 1764 unsigned int segbits = 44; 1765 1766 uasm_i_dmfc0(&p, K0, C0_BADVADDR); 1767 uasm_i_dmfc0(&p, K1, C0_ENTRYHI); 1768 uasm_i_xor(&p, K0, K0, K1); 1769 uasm_i_dsrl_safe(&p, K1, K0, 62); 1770 uasm_i_dsrl_safe(&p, K0, K0, 12 + 1); 1771 uasm_i_dsll_safe(&p, K0, K0, 64 + 12 + 1 - segbits); 1772 uasm_i_or(&p, K0, K0, K1); 1773 uasm_il_bnez(&p, &r, K0, label_leave); 1774 /* No need for uasm_i_nop */ 1775 } 1776 1777 build_r4000_tlbchange_handler_head(&p, &l, &r, K0, K1); 1778 build_pte_present(&p, &r, K0, K1, label_nopage_tlbl); 1779 if (m4kc_tlbp_war()) 1780 build_tlb_probe_entry(&p); 1781 1782 if (kernel_uses_smartmips_rixi) { 1783 /* 1784 * If the page is not _PAGE_VALID, RI or XI could not 1785 * have triggered it. Skip the expensive test.. 1786 */ 1787 if (use_bbit_insns()) { 1788 uasm_il_bbit0(&p, &r, K0, ilog2(_PAGE_VALID), 1789 label_tlbl_goaround1); 1790 } else { 1791 uasm_i_andi(&p, K0, K0, _PAGE_VALID); 1792 uasm_il_beqz(&p, &r, K0, label_tlbl_goaround1); 1793 } 1794 uasm_i_nop(&p); 1795 1796 uasm_i_tlbr(&p); 1797 /* Examine entrylo 0 or 1 based on ptr. */ 1798 if (use_bbit_insns()) { 1799 uasm_i_bbit0(&p, K1, ilog2(sizeof(pte_t)), 8); 1800 } else { 1801 uasm_i_andi(&p, K0, K1, sizeof(pte_t)); 1802 uasm_i_beqz(&p, K0, 8); 1803 } 1804 1805 UASM_i_MFC0(&p, K0, C0_ENTRYLO0); /* load it in the delay slot*/ 1806 UASM_i_MFC0(&p, K0, C0_ENTRYLO1); /* load it if ptr is odd */ 1807 /* 1808 * If the entryLo (now in K0) is valid (bit 1), RI or 1809 * XI must have triggered it. 1810 */ 1811 if (use_bbit_insns()) { 1812 uasm_il_bbit1(&p, &r, K0, 1, label_nopage_tlbl); 1813 /* Reload the PTE value */ 1814 iPTE_LW(&p, K0, K1); 1815 uasm_l_tlbl_goaround1(&l, p); 1816 } else { 1817 uasm_i_andi(&p, K0, K0, 2); 1818 uasm_il_bnez(&p, &r, K0, label_nopage_tlbl); 1819 uasm_l_tlbl_goaround1(&l, p); 1820 /* Reload the PTE value */ 1821 iPTE_LW(&p, K0, K1); 1822 } 1823 } 1824 build_make_valid(&p, &r, K0, K1); 1825 build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1); 1826 1827 #ifdef CONFIG_HUGETLB_PAGE 1828 /* 1829 * This is the entry point when build_r4000_tlbchange_handler_head 1830 * spots a huge page. 1831 */ 1832 uasm_l_tlb_huge_update(&l, p); 1833 iPTE_LW(&p, K0, K1); 1834 build_pte_present(&p, &r, K0, K1, label_nopage_tlbl); 1835 build_tlb_probe_entry(&p); 1836 1837 if (kernel_uses_smartmips_rixi) { 1838 /* 1839 * If the page is not _PAGE_VALID, RI or XI could not 1840 * have triggered it. Skip the expensive test.. 1841 */ 1842 if (use_bbit_insns()) { 1843 uasm_il_bbit0(&p, &r, K0, ilog2(_PAGE_VALID), 1844 label_tlbl_goaround2); 1845 } else { 1846 uasm_i_andi(&p, K0, K0, _PAGE_VALID); 1847 uasm_il_beqz(&p, &r, K0, label_tlbl_goaround2); 1848 } 1849 uasm_i_nop(&p); 1850 1851 uasm_i_tlbr(&p); 1852 /* Examine entrylo 0 or 1 based on ptr. */ 1853 if (use_bbit_insns()) { 1854 uasm_i_bbit0(&p, K1, ilog2(sizeof(pte_t)), 8); 1855 } else { 1856 uasm_i_andi(&p, K0, K1, sizeof(pte_t)); 1857 uasm_i_beqz(&p, K0, 8); 1858 } 1859 UASM_i_MFC0(&p, K0, C0_ENTRYLO0); /* load it in the delay slot*/ 1860 UASM_i_MFC0(&p, K0, C0_ENTRYLO1); /* load it if ptr is odd */ 1861 /* 1862 * If the entryLo (now in K0) is valid (bit 1), RI or 1863 * XI must have triggered it. 1864 */ 1865 if (use_bbit_insns()) { 1866 uasm_il_bbit0(&p, &r, K0, 1, label_tlbl_goaround2); 1867 } else { 1868 uasm_i_andi(&p, K0, K0, 2); 1869 uasm_il_beqz(&p, &r, K0, label_tlbl_goaround2); 1870 } 1871 /* Reload the PTE value */ 1872 iPTE_LW(&p, K0, K1); 1873 1874 /* 1875 * We clobbered C0_PAGEMASK, restore it. On the other branch 1876 * it is restored in build_huge_tlb_write_entry. 1877 */ 1878 build_restore_pagemask(&p, &r, K0, label_nopage_tlbl, 0); 1879 1880 uasm_l_tlbl_goaround2(&l, p); 1881 } 1882 uasm_i_ori(&p, K0, K0, (_PAGE_ACCESSED | _PAGE_VALID)); 1883 build_huge_handler_tail(&p, &r, &l, K0, K1); 1884 #endif 1885 1886 uasm_l_nopage_tlbl(&l, p); 1887 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff); 1888 uasm_i_nop(&p); 1889 1890 if ((p - handle_tlbl) > FASTPATH_SIZE) 1891 panic("TLB load handler fastpath space exceeded"); 1892 1893 uasm_resolve_relocs(relocs, labels); 1894 pr_debug("Wrote TLB load handler fastpath (%u instructions).\n", 1895 (unsigned int)(p - handle_tlbl)); 1896 1897 dump_handler(handle_tlbl, ARRAY_SIZE(handle_tlbl)); 1898 } 1899 1900 static void __cpuinit build_r4000_tlb_store_handler(void) 1901 { 1902 u32 *p = handle_tlbs; 1903 struct uasm_label *l = labels; 1904 struct uasm_reloc *r = relocs; 1905 1906 memset(handle_tlbs, 0, sizeof(handle_tlbs)); 1907 memset(labels, 0, sizeof(labels)); 1908 memset(relocs, 0, sizeof(relocs)); 1909 1910 build_r4000_tlbchange_handler_head(&p, &l, &r, K0, K1); 1911 build_pte_writable(&p, &r, K0, K1, label_nopage_tlbs); 1912 if (m4kc_tlbp_war()) 1913 build_tlb_probe_entry(&p); 1914 build_make_write(&p, &r, K0, K1); 1915 build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1); 1916 1917 #ifdef CONFIG_HUGETLB_PAGE 1918 /* 1919 * This is the entry point when 1920 * build_r4000_tlbchange_handler_head spots a huge page. 1921 */ 1922 uasm_l_tlb_huge_update(&l, p); 1923 iPTE_LW(&p, K0, K1); 1924 build_pte_writable(&p, &r, K0, K1, label_nopage_tlbs); 1925 build_tlb_probe_entry(&p); 1926 uasm_i_ori(&p, K0, K0, 1927 _PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID | _PAGE_DIRTY); 1928 build_huge_handler_tail(&p, &r, &l, K0, K1); 1929 #endif 1930 1931 uasm_l_nopage_tlbs(&l, p); 1932 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); 1933 uasm_i_nop(&p); 1934 1935 if ((p - handle_tlbs) > FASTPATH_SIZE) 1936 panic("TLB store handler fastpath space exceeded"); 1937 1938 uasm_resolve_relocs(relocs, labels); 1939 pr_debug("Wrote TLB store handler fastpath (%u instructions).\n", 1940 (unsigned int)(p - handle_tlbs)); 1941 1942 dump_handler(handle_tlbs, ARRAY_SIZE(handle_tlbs)); 1943 } 1944 1945 static void __cpuinit build_r4000_tlb_modify_handler(void) 1946 { 1947 u32 *p = handle_tlbm; 1948 struct uasm_label *l = labels; 1949 struct uasm_reloc *r = relocs; 1950 1951 memset(handle_tlbm, 0, sizeof(handle_tlbm)); 1952 memset(labels, 0, sizeof(labels)); 1953 memset(relocs, 0, sizeof(relocs)); 1954 1955 build_r4000_tlbchange_handler_head(&p, &l, &r, K0, K1); 1956 build_pte_modifiable(&p, &r, K0, K1, label_nopage_tlbm); 1957 if (m4kc_tlbp_war()) 1958 build_tlb_probe_entry(&p); 1959 /* Present and writable bits set, set accessed and dirty bits. */ 1960 build_make_write(&p, &r, K0, K1); 1961 build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1); 1962 1963 #ifdef CONFIG_HUGETLB_PAGE 1964 /* 1965 * This is the entry point when 1966 * build_r4000_tlbchange_handler_head spots a huge page. 1967 */ 1968 uasm_l_tlb_huge_update(&l, p); 1969 iPTE_LW(&p, K0, K1); 1970 build_pte_modifiable(&p, &r, K0, K1, label_nopage_tlbm); 1971 build_tlb_probe_entry(&p); 1972 uasm_i_ori(&p, K0, K0, 1973 _PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID | _PAGE_DIRTY); 1974 build_huge_handler_tail(&p, &r, &l, K0, K1); 1975 #endif 1976 1977 uasm_l_nopage_tlbm(&l, p); 1978 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); 1979 uasm_i_nop(&p); 1980 1981 if ((p - handle_tlbm) > FASTPATH_SIZE) 1982 panic("TLB modify handler fastpath space exceeded"); 1983 1984 uasm_resolve_relocs(relocs, labels); 1985 pr_debug("Wrote TLB modify handler fastpath (%u instructions).\n", 1986 (unsigned int)(p - handle_tlbm)); 1987 1988 dump_handler(handle_tlbm, ARRAY_SIZE(handle_tlbm)); 1989 } 1990 1991 void __cpuinit build_tlb_refill_handler(void) 1992 { 1993 /* 1994 * The refill handler is generated per-CPU, multi-node systems 1995 * may have local storage for it. The other handlers are only 1996 * needed once. 1997 */ 1998 static int run_once = 0; 1999 2000 #ifdef CONFIG_64BIT 2001 check_for_high_segbits = current_cpu_data.vmbits > (PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3); 2002 #endif 2003 2004 switch (current_cpu_type()) { 2005 case CPU_R2000: 2006 case CPU_R3000: 2007 case CPU_R3000A: 2008 case CPU_R3081E: 2009 case CPU_TX3912: 2010 case CPU_TX3922: 2011 case CPU_TX3927: 2012 #ifndef CONFIG_MIPS_PGD_C0_CONTEXT 2013 build_r3000_tlb_refill_handler(); 2014 if (!run_once) { 2015 build_r3000_tlb_load_handler(); 2016 build_r3000_tlb_store_handler(); 2017 build_r3000_tlb_modify_handler(); 2018 run_once++; 2019 } 2020 #else 2021 panic("No R3000 TLB refill handler"); 2022 #endif 2023 break; 2024 2025 case CPU_R6000: 2026 case CPU_R6000A: 2027 panic("No R6000 TLB refill handler yet"); 2028 break; 2029 2030 case CPU_R8000: 2031 panic("No R8000 TLB refill handler yet"); 2032 break; 2033 2034 default: 2035 if (!run_once) { 2036 #ifdef CONFIG_MIPS_PGD_C0_CONTEXT 2037 build_r4000_setup_pgd(); 2038 #endif 2039 build_r4000_tlb_load_handler(); 2040 build_r4000_tlb_store_handler(); 2041 build_r4000_tlb_modify_handler(); 2042 run_once++; 2043 } 2044 build_r4000_tlb_refill_handler(); 2045 } 2046 } 2047 2048 void __cpuinit flush_tlb_handlers(void) 2049 { 2050 local_flush_icache_range((unsigned long)handle_tlbl, 2051 (unsigned long)handle_tlbl + sizeof(handle_tlbl)); 2052 local_flush_icache_range((unsigned long)handle_tlbs, 2053 (unsigned long)handle_tlbs + sizeof(handle_tlbs)); 2054 local_flush_icache_range((unsigned long)handle_tlbm, 2055 (unsigned long)handle_tlbm + sizeof(handle_tlbm)); 2056 #ifdef CONFIG_MIPS_PGD_C0_CONTEXT 2057 local_flush_icache_range((unsigned long)tlbmiss_handler_setup_pgd, 2058 (unsigned long)tlbmiss_handler_setup_pgd + sizeof(handle_tlbm)); 2059 #endif 2060 } 2061