1 /* 2 * This file is subject to the terms and conditions of the GNU General Public 3 * License. See the file "COPYING" in the main directory of this archive 4 * for more details. 5 * 6 * Synthesize TLB refill handlers at runtime. 7 * 8 * Copyright (C) 2004, 2005, 2006, 2008 Thiemo Seufer 9 * Copyright (C) 2005, 2007, 2008, 2009 Maciej W. Rozycki 10 * Copyright (C) 2006 Ralf Baechle (ralf@linux-mips.org) 11 * Copyright (C) 2008, 2009 Cavium Networks, Inc. 12 * Copyright (C) 2011 MIPS Technologies, Inc. 13 * 14 * ... and the days got worse and worse and now you see 15 * I've gone completly out of my mind. 16 * 17 * They're coming to take me a away haha 18 * they're coming to take me a away hoho hihi haha 19 * to the funny farm where code is beautiful all the time ... 20 * 21 * (Condolences to Napoleon XIV) 22 */ 23 24 #include <linux/bug.h> 25 #include <linux/kernel.h> 26 #include <linux/types.h> 27 #include <linux/smp.h> 28 #include <linux/string.h> 29 #include <linux/init.h> 30 #include <linux/cache.h> 31 32 #include <asm/cacheflush.h> 33 #include <asm/cpu-type.h> 34 #include <asm/pgtable.h> 35 #include <asm/war.h> 36 #include <asm/uasm.h> 37 #include <asm/setup.h> 38 39 /* 40 * TLB load/store/modify handlers. 41 * 42 * Only the fastpath gets synthesized at runtime, the slowpath for 43 * do_page_fault remains normal asm. 44 */ 45 extern void tlb_do_page_fault_0(void); 46 extern void tlb_do_page_fault_1(void); 47 48 struct work_registers { 49 int r1; 50 int r2; 51 int r3; 52 }; 53 54 struct tlb_reg_save { 55 unsigned long a; 56 unsigned long b; 57 } ____cacheline_aligned_in_smp; 58 59 static struct tlb_reg_save handler_reg_save[NR_CPUS]; 60 61 static inline int r45k_bvahwbug(void) 62 { 63 /* XXX: We should probe for the presence of this bug, but we don't. */ 64 return 0; 65 } 66 67 static inline int r4k_250MHZhwbug(void) 68 { 69 /* XXX: We should probe for the presence of this bug, but we don't. */ 70 return 0; 71 } 72 73 static inline int __maybe_unused bcm1250_m3_war(void) 74 { 75 return BCM1250_M3_WAR; 76 } 77 78 static inline int __maybe_unused r10000_llsc_war(void) 79 { 80 return R10000_LLSC_WAR; 81 } 82 83 static int use_bbit_insns(void) 84 { 85 switch (current_cpu_type()) { 86 case CPU_CAVIUM_OCTEON: 87 case CPU_CAVIUM_OCTEON_PLUS: 88 case CPU_CAVIUM_OCTEON2: 89 case CPU_CAVIUM_OCTEON3: 90 return 1; 91 default: 92 return 0; 93 } 94 } 95 96 static int use_lwx_insns(void) 97 { 98 switch (current_cpu_type()) { 99 case CPU_CAVIUM_OCTEON2: 100 case CPU_CAVIUM_OCTEON3: 101 return 1; 102 default: 103 return 0; 104 } 105 } 106 #if defined(CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE) && \ 107 CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE > 0 108 static bool scratchpad_available(void) 109 { 110 return true; 111 } 112 static int scratchpad_offset(int i) 113 { 114 /* 115 * CVMSEG starts at address -32768 and extends for 116 * CAVIUM_OCTEON_CVMSEG_SIZE 128 byte cache lines. 117 */ 118 i += 1; /* Kernel use starts at the top and works down. */ 119 return CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE * 128 - (8 * i) - 32768; 120 } 121 #else 122 static bool scratchpad_available(void) 123 { 124 return false; 125 } 126 static int scratchpad_offset(int i) 127 { 128 BUG(); 129 /* Really unreachable, but evidently some GCC want this. */ 130 return 0; 131 } 132 #endif 133 /* 134 * Found by experiment: At least some revisions of the 4kc throw under 135 * some circumstances a machine check exception, triggered by invalid 136 * values in the index register. Delaying the tlbp instruction until 137 * after the next branch, plus adding an additional nop in front of 138 * tlbwi/tlbwr avoids the invalid index register values. Nobody knows 139 * why; it's not an issue caused by the core RTL. 140 * 141 */ 142 static int m4kc_tlbp_war(void) 143 { 144 return (current_cpu_data.processor_id & 0xffff00) == 145 (PRID_COMP_MIPS | PRID_IMP_4KC); 146 } 147 148 /* Handle labels (which must be positive integers). */ 149 enum label_id { 150 label_second_part = 1, 151 label_leave, 152 label_vmalloc, 153 label_vmalloc_done, 154 label_tlbw_hazard_0, 155 label_split = label_tlbw_hazard_0 + 8, 156 label_tlbl_goaround1, 157 label_tlbl_goaround2, 158 label_nopage_tlbl, 159 label_nopage_tlbs, 160 label_nopage_tlbm, 161 label_smp_pgtable_change, 162 label_r3000_write_probe_fail, 163 label_large_segbits_fault, 164 #ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT 165 label_tlb_huge_update, 166 #endif 167 }; 168 169 UASM_L_LA(_second_part) 170 UASM_L_LA(_leave) 171 UASM_L_LA(_vmalloc) 172 UASM_L_LA(_vmalloc_done) 173 /* _tlbw_hazard_x is handled differently. */ 174 UASM_L_LA(_split) 175 UASM_L_LA(_tlbl_goaround1) 176 UASM_L_LA(_tlbl_goaround2) 177 UASM_L_LA(_nopage_tlbl) 178 UASM_L_LA(_nopage_tlbs) 179 UASM_L_LA(_nopage_tlbm) 180 UASM_L_LA(_smp_pgtable_change) 181 UASM_L_LA(_r3000_write_probe_fail) 182 UASM_L_LA(_large_segbits_fault) 183 #ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT 184 UASM_L_LA(_tlb_huge_update) 185 #endif 186 187 static int hazard_instance; 188 189 static void uasm_bgezl_hazard(u32 **p, struct uasm_reloc **r, int instance) 190 { 191 switch (instance) { 192 case 0 ... 7: 193 uasm_il_bgezl(p, r, 0, label_tlbw_hazard_0 + instance); 194 return; 195 default: 196 BUG(); 197 } 198 } 199 200 static void uasm_bgezl_label(struct uasm_label **l, u32 **p, int instance) 201 { 202 switch (instance) { 203 case 0 ... 7: 204 uasm_build_label(l, *p, label_tlbw_hazard_0 + instance); 205 break; 206 default: 207 BUG(); 208 } 209 } 210 211 /* 212 * pgtable bits are assigned dynamically depending on processor feature 213 * and statically based on kernel configuration. This spits out the actual 214 * values the kernel is using. Required to make sense from disassembled 215 * TLB exception handlers. 216 */ 217 static void output_pgtable_bits_defines(void) 218 { 219 #define pr_define(fmt, ...) \ 220 pr_debug("#define " fmt, ##__VA_ARGS__) 221 222 pr_debug("#include <asm/asm.h>\n"); 223 pr_debug("#include <asm/regdef.h>\n"); 224 pr_debug("\n"); 225 226 pr_define("_PAGE_PRESENT_SHIFT %d\n", _PAGE_PRESENT_SHIFT); 227 pr_define("_PAGE_READ_SHIFT %d\n", _PAGE_READ_SHIFT); 228 pr_define("_PAGE_WRITE_SHIFT %d\n", _PAGE_WRITE_SHIFT); 229 pr_define("_PAGE_ACCESSED_SHIFT %d\n", _PAGE_ACCESSED_SHIFT); 230 pr_define("_PAGE_MODIFIED_SHIFT %d\n", _PAGE_MODIFIED_SHIFT); 231 #ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT 232 pr_define("_PAGE_HUGE_SHIFT %d\n", _PAGE_HUGE_SHIFT); 233 pr_define("_PAGE_SPLITTING_SHIFT %d\n", _PAGE_SPLITTING_SHIFT); 234 #endif 235 if (cpu_has_rixi) { 236 #ifdef _PAGE_NO_EXEC_SHIFT 237 pr_define("_PAGE_NO_EXEC_SHIFT %d\n", _PAGE_NO_EXEC_SHIFT); 238 #endif 239 #ifdef _PAGE_NO_READ_SHIFT 240 pr_define("_PAGE_NO_READ_SHIFT %d\n", _PAGE_NO_READ_SHIFT); 241 #endif 242 } 243 pr_define("_PAGE_GLOBAL_SHIFT %d\n", _PAGE_GLOBAL_SHIFT); 244 pr_define("_PAGE_VALID_SHIFT %d\n", _PAGE_VALID_SHIFT); 245 pr_define("_PAGE_DIRTY_SHIFT %d\n", _PAGE_DIRTY_SHIFT); 246 pr_define("_PFN_SHIFT %d\n", _PFN_SHIFT); 247 pr_debug("\n"); 248 } 249 250 static inline void dump_handler(const char *symbol, const u32 *handler, int count) 251 { 252 int i; 253 254 pr_debug("LEAF(%s)\n", symbol); 255 256 pr_debug("\t.set push\n"); 257 pr_debug("\t.set noreorder\n"); 258 259 for (i = 0; i < count; i++) 260 pr_debug("\t.word\t0x%08x\t\t# %p\n", handler[i], &handler[i]); 261 262 pr_debug("\t.set\tpop\n"); 263 264 pr_debug("\tEND(%s)\n", symbol); 265 } 266 267 /* The only general purpose registers allowed in TLB handlers. */ 268 #define K0 26 269 #define K1 27 270 271 /* Some CP0 registers */ 272 #define C0_INDEX 0, 0 273 #define C0_ENTRYLO0 2, 0 274 #define C0_TCBIND 2, 2 275 #define C0_ENTRYLO1 3, 0 276 #define C0_CONTEXT 4, 0 277 #define C0_PAGEMASK 5, 0 278 #define C0_BADVADDR 8, 0 279 #define C0_ENTRYHI 10, 0 280 #define C0_EPC 14, 0 281 #define C0_XCONTEXT 20, 0 282 283 #ifdef CONFIG_64BIT 284 # define GET_CONTEXT(buf, reg) UASM_i_MFC0(buf, reg, C0_XCONTEXT) 285 #else 286 # define GET_CONTEXT(buf, reg) UASM_i_MFC0(buf, reg, C0_CONTEXT) 287 #endif 288 289 /* The worst case length of the handler is around 18 instructions for 290 * R3000-style TLBs and up to 63 instructions for R4000-style TLBs. 291 * Maximum space available is 32 instructions for R3000 and 64 292 * instructions for R4000. 293 * 294 * We deliberately chose a buffer size of 128, so we won't scribble 295 * over anything important on overflow before we panic. 296 */ 297 static u32 tlb_handler[128]; 298 299 /* simply assume worst case size for labels and relocs */ 300 static struct uasm_label labels[128]; 301 static struct uasm_reloc relocs[128]; 302 303 static int check_for_high_segbits; 304 305 static unsigned int kscratch_used_mask; 306 307 static inline int __maybe_unused c0_kscratch(void) 308 { 309 switch (current_cpu_type()) { 310 case CPU_XLP: 311 case CPU_XLR: 312 return 22; 313 default: 314 return 31; 315 } 316 } 317 318 static int allocate_kscratch(void) 319 { 320 int r; 321 unsigned int a = cpu_data[0].kscratch_mask & ~kscratch_used_mask; 322 323 r = ffs(a); 324 325 if (r == 0) 326 return -1; 327 328 r--; /* make it zero based */ 329 330 kscratch_used_mask |= (1 << r); 331 332 return r; 333 } 334 335 static int scratch_reg; 336 static int pgd_reg; 337 enum vmalloc64_mode {not_refill, refill_scratch, refill_noscratch}; 338 339 static struct work_registers build_get_work_registers(u32 **p) 340 { 341 struct work_registers r; 342 343 int smp_processor_id_reg; 344 int smp_processor_id_sel; 345 int smp_processor_id_shift; 346 347 if (scratch_reg >= 0) { 348 /* Save in CPU local C0_KScratch? */ 349 UASM_i_MTC0(p, 1, c0_kscratch(), scratch_reg); 350 r.r1 = K0; 351 r.r2 = K1; 352 r.r3 = 1; 353 return r; 354 } 355 356 if (num_possible_cpus() > 1) { 357 #ifdef CONFIG_MIPS_PGD_C0_CONTEXT 358 smp_processor_id_shift = 51; 359 smp_processor_id_reg = 20; /* XContext */ 360 smp_processor_id_sel = 0; 361 #else 362 # ifdef CONFIG_32BIT 363 smp_processor_id_shift = 25; 364 smp_processor_id_reg = 4; /* Context */ 365 smp_processor_id_sel = 0; 366 # endif 367 # ifdef CONFIG_64BIT 368 smp_processor_id_shift = 26; 369 smp_processor_id_reg = 4; /* Context */ 370 smp_processor_id_sel = 0; 371 # endif 372 #endif 373 /* Get smp_processor_id */ 374 UASM_i_MFC0(p, K0, smp_processor_id_reg, smp_processor_id_sel); 375 UASM_i_SRL_SAFE(p, K0, K0, smp_processor_id_shift); 376 377 /* handler_reg_save index in K0 */ 378 UASM_i_SLL(p, K0, K0, ilog2(sizeof(struct tlb_reg_save))); 379 380 UASM_i_LA(p, K1, (long)&handler_reg_save); 381 UASM_i_ADDU(p, K0, K0, K1); 382 } else { 383 UASM_i_LA(p, K0, (long)&handler_reg_save); 384 } 385 /* K0 now points to save area, save $1 and $2 */ 386 UASM_i_SW(p, 1, offsetof(struct tlb_reg_save, a), K0); 387 UASM_i_SW(p, 2, offsetof(struct tlb_reg_save, b), K0); 388 389 r.r1 = K1; 390 r.r2 = 1; 391 r.r3 = 2; 392 return r; 393 } 394 395 static void build_restore_work_registers(u32 **p) 396 { 397 if (scratch_reg >= 0) { 398 UASM_i_MFC0(p, 1, c0_kscratch(), scratch_reg); 399 return; 400 } 401 /* K0 already points to save area, restore $1 and $2 */ 402 UASM_i_LW(p, 1, offsetof(struct tlb_reg_save, a), K0); 403 UASM_i_LW(p, 2, offsetof(struct tlb_reg_save, b), K0); 404 } 405 406 #ifndef CONFIG_MIPS_PGD_C0_CONTEXT 407 408 /* 409 * CONFIG_MIPS_PGD_C0_CONTEXT implies 64 bit and lack of pgd_current, 410 * we cannot do r3000 under these circumstances. 411 * 412 * Declare pgd_current here instead of including mmu_context.h to avoid type 413 * conflicts for tlbmiss_handler_setup_pgd 414 */ 415 extern unsigned long pgd_current[]; 416 417 /* 418 * The R3000 TLB handler is simple. 419 */ 420 static void build_r3000_tlb_refill_handler(void) 421 { 422 long pgdc = (long)pgd_current; 423 u32 *p; 424 425 memset(tlb_handler, 0, sizeof(tlb_handler)); 426 p = tlb_handler; 427 428 uasm_i_mfc0(&p, K0, C0_BADVADDR); 429 uasm_i_lui(&p, K1, uasm_rel_hi(pgdc)); /* cp0 delay */ 430 uasm_i_lw(&p, K1, uasm_rel_lo(pgdc), K1); 431 uasm_i_srl(&p, K0, K0, 22); /* load delay */ 432 uasm_i_sll(&p, K0, K0, 2); 433 uasm_i_addu(&p, K1, K1, K0); 434 uasm_i_mfc0(&p, K0, C0_CONTEXT); 435 uasm_i_lw(&p, K1, 0, K1); /* cp0 delay */ 436 uasm_i_andi(&p, K0, K0, 0xffc); /* load delay */ 437 uasm_i_addu(&p, K1, K1, K0); 438 uasm_i_lw(&p, K0, 0, K1); 439 uasm_i_nop(&p); /* load delay */ 440 uasm_i_mtc0(&p, K0, C0_ENTRYLO0); 441 uasm_i_mfc0(&p, K1, C0_EPC); /* cp0 delay */ 442 uasm_i_tlbwr(&p); /* cp0 delay */ 443 uasm_i_jr(&p, K1); 444 uasm_i_rfe(&p); /* branch delay */ 445 446 if (p > tlb_handler + 32) 447 panic("TLB refill handler space exceeded"); 448 449 pr_debug("Wrote TLB refill handler (%u instructions).\n", 450 (unsigned int)(p - tlb_handler)); 451 452 memcpy((void *)ebase, tlb_handler, 0x80); 453 454 dump_handler("r3000_tlb_refill", (u32 *)ebase, 32); 455 } 456 #endif /* CONFIG_MIPS_PGD_C0_CONTEXT */ 457 458 /* 459 * The R4000 TLB handler is much more complicated. We have two 460 * consecutive handler areas with 32 instructions space each. 461 * Since they aren't used at the same time, we can overflow in the 462 * other one.To keep things simple, we first assume linear space, 463 * then we relocate it to the final handler layout as needed. 464 */ 465 static u32 final_handler[64]; 466 467 /* 468 * Hazards 469 * 470 * From the IDT errata for the QED RM5230 (Nevada), processor revision 1.0: 471 * 2. A timing hazard exists for the TLBP instruction. 472 * 473 * stalling_instruction 474 * TLBP 475 * 476 * The JTLB is being read for the TLBP throughout the stall generated by the 477 * previous instruction. This is not really correct as the stalling instruction 478 * can modify the address used to access the JTLB. The failure symptom is that 479 * the TLBP instruction will use an address created for the stalling instruction 480 * and not the address held in C0_ENHI and thus report the wrong results. 481 * 482 * The software work-around is to not allow the instruction preceding the TLBP 483 * to stall - make it an NOP or some other instruction guaranteed not to stall. 484 * 485 * Errata 2 will not be fixed. This errata is also on the R5000. 486 * 487 * As if we MIPS hackers wouldn't know how to nop pipelines happy ... 488 */ 489 static void __maybe_unused build_tlb_probe_entry(u32 **p) 490 { 491 switch (current_cpu_type()) { 492 /* Found by experiment: R4600 v2.0/R4700 needs this, too. */ 493 case CPU_R4600: 494 case CPU_R4700: 495 case CPU_R5000: 496 case CPU_NEVADA: 497 uasm_i_nop(p); 498 uasm_i_tlbp(p); 499 break; 500 501 default: 502 uasm_i_tlbp(p); 503 break; 504 } 505 } 506 507 /* 508 * Write random or indexed TLB entry, and care about the hazards from 509 * the preceding mtc0 and for the following eret. 510 */ 511 enum tlb_write_entry { tlb_random, tlb_indexed }; 512 513 static void build_tlb_write_entry(u32 **p, struct uasm_label **l, 514 struct uasm_reloc **r, 515 enum tlb_write_entry wmode) 516 { 517 void(*tlbw)(u32 **) = NULL; 518 519 switch (wmode) { 520 case tlb_random: tlbw = uasm_i_tlbwr; break; 521 case tlb_indexed: tlbw = uasm_i_tlbwi; break; 522 } 523 524 if (cpu_has_mips_r2) { 525 /* 526 * The architecture spec says an ehb is required here, 527 * but a number of cores do not have the hazard and 528 * using an ehb causes an expensive pipeline stall. 529 */ 530 switch (current_cpu_type()) { 531 case CPU_M14KC: 532 case CPU_74K: 533 break; 534 535 default: 536 uasm_i_ehb(p); 537 break; 538 } 539 tlbw(p); 540 return; 541 } 542 543 switch (current_cpu_type()) { 544 case CPU_R4000PC: 545 case CPU_R4000SC: 546 case CPU_R4000MC: 547 case CPU_R4400PC: 548 case CPU_R4400SC: 549 case CPU_R4400MC: 550 /* 551 * This branch uses up a mtc0 hazard nop slot and saves 552 * two nops after the tlbw instruction. 553 */ 554 uasm_bgezl_hazard(p, r, hazard_instance); 555 tlbw(p); 556 uasm_bgezl_label(l, p, hazard_instance); 557 hazard_instance++; 558 uasm_i_nop(p); 559 break; 560 561 case CPU_R4600: 562 case CPU_R4700: 563 uasm_i_nop(p); 564 tlbw(p); 565 uasm_i_nop(p); 566 break; 567 568 case CPU_R5000: 569 case CPU_NEVADA: 570 uasm_i_nop(p); /* QED specifies 2 nops hazard */ 571 uasm_i_nop(p); /* QED specifies 2 nops hazard */ 572 tlbw(p); 573 break; 574 575 case CPU_R4300: 576 case CPU_5KC: 577 case CPU_TX49XX: 578 case CPU_PR4450: 579 case CPU_XLR: 580 uasm_i_nop(p); 581 tlbw(p); 582 break; 583 584 case CPU_R10000: 585 case CPU_R12000: 586 case CPU_R14000: 587 case CPU_4KC: 588 case CPU_4KEC: 589 case CPU_M14KC: 590 case CPU_M14KEC: 591 case CPU_SB1: 592 case CPU_SB1A: 593 case CPU_4KSC: 594 case CPU_20KC: 595 case CPU_25KF: 596 case CPU_BMIPS32: 597 case CPU_BMIPS3300: 598 case CPU_BMIPS4350: 599 case CPU_BMIPS4380: 600 case CPU_BMIPS5000: 601 case CPU_LOONGSON2: 602 case CPU_R5500: 603 if (m4kc_tlbp_war()) 604 uasm_i_nop(p); 605 case CPU_ALCHEMY: 606 tlbw(p); 607 break; 608 609 case CPU_RM7000: 610 uasm_i_nop(p); 611 uasm_i_nop(p); 612 uasm_i_nop(p); 613 uasm_i_nop(p); 614 tlbw(p); 615 break; 616 617 case CPU_VR4111: 618 case CPU_VR4121: 619 case CPU_VR4122: 620 case CPU_VR4181: 621 case CPU_VR4181A: 622 uasm_i_nop(p); 623 uasm_i_nop(p); 624 tlbw(p); 625 uasm_i_nop(p); 626 uasm_i_nop(p); 627 break; 628 629 case CPU_VR4131: 630 case CPU_VR4133: 631 case CPU_R5432: 632 uasm_i_nop(p); 633 uasm_i_nop(p); 634 tlbw(p); 635 break; 636 637 case CPU_JZRISC: 638 tlbw(p); 639 uasm_i_nop(p); 640 break; 641 642 default: 643 panic("No TLB refill handler yet (CPU type: %d)", 644 current_cpu_data.cputype); 645 break; 646 } 647 } 648 649 static __maybe_unused void build_convert_pte_to_entrylo(u32 **p, 650 unsigned int reg) 651 { 652 if (cpu_has_rixi) { 653 UASM_i_ROTR(p, reg, reg, ilog2(_PAGE_GLOBAL)); 654 } else { 655 #ifdef CONFIG_64BIT_PHYS_ADDR 656 uasm_i_dsrl_safe(p, reg, reg, ilog2(_PAGE_GLOBAL)); 657 #else 658 UASM_i_SRL(p, reg, reg, ilog2(_PAGE_GLOBAL)); 659 #endif 660 } 661 } 662 663 #ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT 664 665 static void build_restore_pagemask(u32 **p, struct uasm_reloc **r, 666 unsigned int tmp, enum label_id lid, 667 int restore_scratch) 668 { 669 if (restore_scratch) { 670 /* Reset default page size */ 671 if (PM_DEFAULT_MASK >> 16) { 672 uasm_i_lui(p, tmp, PM_DEFAULT_MASK >> 16); 673 uasm_i_ori(p, tmp, tmp, PM_DEFAULT_MASK & 0xffff); 674 uasm_i_mtc0(p, tmp, C0_PAGEMASK); 675 uasm_il_b(p, r, lid); 676 } else if (PM_DEFAULT_MASK) { 677 uasm_i_ori(p, tmp, 0, PM_DEFAULT_MASK); 678 uasm_i_mtc0(p, tmp, C0_PAGEMASK); 679 uasm_il_b(p, r, lid); 680 } else { 681 uasm_i_mtc0(p, 0, C0_PAGEMASK); 682 uasm_il_b(p, r, lid); 683 } 684 if (scratch_reg >= 0) 685 UASM_i_MFC0(p, 1, c0_kscratch(), scratch_reg); 686 else 687 UASM_i_LW(p, 1, scratchpad_offset(0), 0); 688 } else { 689 /* Reset default page size */ 690 if (PM_DEFAULT_MASK >> 16) { 691 uasm_i_lui(p, tmp, PM_DEFAULT_MASK >> 16); 692 uasm_i_ori(p, tmp, tmp, PM_DEFAULT_MASK & 0xffff); 693 uasm_il_b(p, r, lid); 694 uasm_i_mtc0(p, tmp, C0_PAGEMASK); 695 } else if (PM_DEFAULT_MASK) { 696 uasm_i_ori(p, tmp, 0, PM_DEFAULT_MASK); 697 uasm_il_b(p, r, lid); 698 uasm_i_mtc0(p, tmp, C0_PAGEMASK); 699 } else { 700 uasm_il_b(p, r, lid); 701 uasm_i_mtc0(p, 0, C0_PAGEMASK); 702 } 703 } 704 } 705 706 static void build_huge_tlb_write_entry(u32 **p, struct uasm_label **l, 707 struct uasm_reloc **r, 708 unsigned int tmp, 709 enum tlb_write_entry wmode, 710 int restore_scratch) 711 { 712 /* Set huge page tlb entry size */ 713 uasm_i_lui(p, tmp, PM_HUGE_MASK >> 16); 714 uasm_i_ori(p, tmp, tmp, PM_HUGE_MASK & 0xffff); 715 uasm_i_mtc0(p, tmp, C0_PAGEMASK); 716 717 build_tlb_write_entry(p, l, r, wmode); 718 719 build_restore_pagemask(p, r, tmp, label_leave, restore_scratch); 720 } 721 722 /* 723 * Check if Huge PTE is present, if so then jump to LABEL. 724 */ 725 static void 726 build_is_huge_pte(u32 **p, struct uasm_reloc **r, unsigned int tmp, 727 unsigned int pmd, int lid) 728 { 729 UASM_i_LW(p, tmp, 0, pmd); 730 if (use_bbit_insns()) { 731 uasm_il_bbit1(p, r, tmp, ilog2(_PAGE_HUGE), lid); 732 } else { 733 uasm_i_andi(p, tmp, tmp, _PAGE_HUGE); 734 uasm_il_bnez(p, r, tmp, lid); 735 } 736 } 737 738 static void build_huge_update_entries(u32 **p, unsigned int pte, 739 unsigned int tmp) 740 { 741 int small_sequence; 742 743 /* 744 * A huge PTE describes an area the size of the 745 * configured huge page size. This is twice the 746 * of the large TLB entry size we intend to use. 747 * A TLB entry half the size of the configured 748 * huge page size is configured into entrylo0 749 * and entrylo1 to cover the contiguous huge PTE 750 * address space. 751 */ 752 small_sequence = (HPAGE_SIZE >> 7) < 0x10000; 753 754 /* We can clobber tmp. It isn't used after this.*/ 755 if (!small_sequence) 756 uasm_i_lui(p, tmp, HPAGE_SIZE >> (7 + 16)); 757 758 build_convert_pte_to_entrylo(p, pte); 759 UASM_i_MTC0(p, pte, C0_ENTRYLO0); /* load it */ 760 /* convert to entrylo1 */ 761 if (small_sequence) 762 UASM_i_ADDIU(p, pte, pte, HPAGE_SIZE >> 7); 763 else 764 UASM_i_ADDU(p, pte, pte, tmp); 765 766 UASM_i_MTC0(p, pte, C0_ENTRYLO1); /* load it */ 767 } 768 769 static void build_huge_handler_tail(u32 **p, struct uasm_reloc **r, 770 struct uasm_label **l, 771 unsigned int pte, 772 unsigned int ptr) 773 { 774 #ifdef CONFIG_SMP 775 UASM_i_SC(p, pte, 0, ptr); 776 uasm_il_beqz(p, r, pte, label_tlb_huge_update); 777 UASM_i_LW(p, pte, 0, ptr); /* Needed because SC killed our PTE */ 778 #else 779 UASM_i_SW(p, pte, 0, ptr); 780 #endif 781 build_huge_update_entries(p, pte, ptr); 782 build_huge_tlb_write_entry(p, l, r, pte, tlb_indexed, 0); 783 } 784 #endif /* CONFIG_MIPS_HUGE_TLB_SUPPORT */ 785 786 #ifdef CONFIG_64BIT 787 /* 788 * TMP and PTR are scratch. 789 * TMP will be clobbered, PTR will hold the pmd entry. 790 */ 791 static void 792 build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r, 793 unsigned int tmp, unsigned int ptr) 794 { 795 #ifndef CONFIG_MIPS_PGD_C0_CONTEXT 796 long pgdc = (long)pgd_current; 797 #endif 798 /* 799 * The vmalloc handling is not in the hotpath. 800 */ 801 uasm_i_dmfc0(p, tmp, C0_BADVADDR); 802 803 if (check_for_high_segbits) { 804 /* 805 * The kernel currently implicitely assumes that the 806 * MIPS SEGBITS parameter for the processor is 807 * (PGDIR_SHIFT+PGDIR_BITS) or less, and will never 808 * allocate virtual addresses outside the maximum 809 * range for SEGBITS = (PGDIR_SHIFT+PGDIR_BITS). But 810 * that doesn't prevent user code from accessing the 811 * higher xuseg addresses. Here, we make sure that 812 * everything but the lower xuseg addresses goes down 813 * the module_alloc/vmalloc path. 814 */ 815 uasm_i_dsrl_safe(p, ptr, tmp, PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3); 816 uasm_il_bnez(p, r, ptr, label_vmalloc); 817 } else { 818 uasm_il_bltz(p, r, tmp, label_vmalloc); 819 } 820 /* No uasm_i_nop needed here, since the next insn doesn't touch TMP. */ 821 822 #ifdef CONFIG_MIPS_PGD_C0_CONTEXT 823 if (pgd_reg != -1) { 824 /* pgd is in pgd_reg */ 825 UASM_i_MFC0(p, ptr, c0_kscratch(), pgd_reg); 826 } else { 827 /* 828 * &pgd << 11 stored in CONTEXT [23..63]. 829 */ 830 UASM_i_MFC0(p, ptr, C0_CONTEXT); 831 832 /* Clear lower 23 bits of context. */ 833 uasm_i_dins(p, ptr, 0, 0, 23); 834 835 /* 1 0 1 0 1 << 6 xkphys cached */ 836 uasm_i_ori(p, ptr, ptr, 0x540); 837 uasm_i_drotr(p, ptr, ptr, 11); 838 } 839 #elif defined(CONFIG_SMP) 840 # ifdef CONFIG_MIPS_MT_SMTC 841 /* 842 * SMTC uses TCBind value as "CPU" index 843 */ 844 uasm_i_mfc0(p, ptr, C0_TCBIND); 845 uasm_i_dsrl_safe(p, ptr, ptr, 19); 846 # else 847 /* 848 * 64 bit SMP running in XKPHYS has smp_processor_id() << 3 849 * stored in CONTEXT. 850 */ 851 uasm_i_dmfc0(p, ptr, C0_CONTEXT); 852 uasm_i_dsrl_safe(p, ptr, ptr, 23); 853 # endif 854 UASM_i_LA_mostly(p, tmp, pgdc); 855 uasm_i_daddu(p, ptr, ptr, tmp); 856 uasm_i_dmfc0(p, tmp, C0_BADVADDR); 857 uasm_i_ld(p, ptr, uasm_rel_lo(pgdc), ptr); 858 #else 859 UASM_i_LA_mostly(p, ptr, pgdc); 860 uasm_i_ld(p, ptr, uasm_rel_lo(pgdc), ptr); 861 #endif 862 863 uasm_l_vmalloc_done(l, *p); 864 865 /* get pgd offset in bytes */ 866 uasm_i_dsrl_safe(p, tmp, tmp, PGDIR_SHIFT - 3); 867 868 uasm_i_andi(p, tmp, tmp, (PTRS_PER_PGD - 1)<<3); 869 uasm_i_daddu(p, ptr, ptr, tmp); /* add in pgd offset */ 870 #ifndef __PAGETABLE_PMD_FOLDED 871 uasm_i_dmfc0(p, tmp, C0_BADVADDR); /* get faulting address */ 872 uasm_i_ld(p, ptr, 0, ptr); /* get pmd pointer */ 873 uasm_i_dsrl_safe(p, tmp, tmp, PMD_SHIFT-3); /* get pmd offset in bytes */ 874 uasm_i_andi(p, tmp, tmp, (PTRS_PER_PMD - 1)<<3); 875 uasm_i_daddu(p, ptr, ptr, tmp); /* add in pmd offset */ 876 #endif 877 } 878 879 /* 880 * BVADDR is the faulting address, PTR is scratch. 881 * PTR will hold the pgd for vmalloc. 882 */ 883 static void 884 build_get_pgd_vmalloc64(u32 **p, struct uasm_label **l, struct uasm_reloc **r, 885 unsigned int bvaddr, unsigned int ptr, 886 enum vmalloc64_mode mode) 887 { 888 long swpd = (long)swapper_pg_dir; 889 int single_insn_swpd; 890 int did_vmalloc_branch = 0; 891 892 single_insn_swpd = uasm_in_compat_space_p(swpd) && !uasm_rel_lo(swpd); 893 894 uasm_l_vmalloc(l, *p); 895 896 if (mode != not_refill && check_for_high_segbits) { 897 if (single_insn_swpd) { 898 uasm_il_bltz(p, r, bvaddr, label_vmalloc_done); 899 uasm_i_lui(p, ptr, uasm_rel_hi(swpd)); 900 did_vmalloc_branch = 1; 901 /* fall through */ 902 } else { 903 uasm_il_bgez(p, r, bvaddr, label_large_segbits_fault); 904 } 905 } 906 if (!did_vmalloc_branch) { 907 if (uasm_in_compat_space_p(swpd) && !uasm_rel_lo(swpd)) { 908 uasm_il_b(p, r, label_vmalloc_done); 909 uasm_i_lui(p, ptr, uasm_rel_hi(swpd)); 910 } else { 911 UASM_i_LA_mostly(p, ptr, swpd); 912 uasm_il_b(p, r, label_vmalloc_done); 913 if (uasm_in_compat_space_p(swpd)) 914 uasm_i_addiu(p, ptr, ptr, uasm_rel_lo(swpd)); 915 else 916 uasm_i_daddiu(p, ptr, ptr, uasm_rel_lo(swpd)); 917 } 918 } 919 if (mode != not_refill && check_for_high_segbits) { 920 uasm_l_large_segbits_fault(l, *p); 921 /* 922 * We get here if we are an xsseg address, or if we are 923 * an xuseg address above (PGDIR_SHIFT+PGDIR_BITS) boundary. 924 * 925 * Ignoring xsseg (assume disabled so would generate 926 * (address errors?), the only remaining possibility 927 * is the upper xuseg addresses. On processors with 928 * TLB_SEGBITS <= PGDIR_SHIFT+PGDIR_BITS, these 929 * addresses would have taken an address error. We try 930 * to mimic that here by taking a load/istream page 931 * fault. 932 */ 933 UASM_i_LA(p, ptr, (unsigned long)tlb_do_page_fault_0); 934 uasm_i_jr(p, ptr); 935 936 if (mode == refill_scratch) { 937 if (scratch_reg >= 0) 938 UASM_i_MFC0(p, 1, c0_kscratch(), scratch_reg); 939 else 940 UASM_i_LW(p, 1, scratchpad_offset(0), 0); 941 } else { 942 uasm_i_nop(p); 943 } 944 } 945 } 946 947 #else /* !CONFIG_64BIT */ 948 949 /* 950 * TMP and PTR are scratch. 951 * TMP will be clobbered, PTR will hold the pgd entry. 952 */ 953 static void __maybe_unused 954 build_get_pgde32(u32 **p, unsigned int tmp, unsigned int ptr) 955 { 956 long pgdc = (long)pgd_current; 957 958 /* 32 bit SMP has smp_processor_id() stored in CONTEXT. */ 959 #ifdef CONFIG_SMP 960 #ifdef CONFIG_MIPS_MT_SMTC 961 /* 962 * SMTC uses TCBind value as "CPU" index 963 */ 964 uasm_i_mfc0(p, ptr, C0_TCBIND); 965 UASM_i_LA_mostly(p, tmp, pgdc); 966 uasm_i_srl(p, ptr, ptr, 19); 967 #else 968 /* 969 * smp_processor_id() << 2 is stored in CONTEXT. 970 */ 971 uasm_i_mfc0(p, ptr, C0_CONTEXT); 972 UASM_i_LA_mostly(p, tmp, pgdc); 973 uasm_i_srl(p, ptr, ptr, 23); 974 #endif 975 uasm_i_addu(p, ptr, tmp, ptr); 976 #else 977 UASM_i_LA_mostly(p, ptr, pgdc); 978 #endif 979 uasm_i_mfc0(p, tmp, C0_BADVADDR); /* get faulting address */ 980 uasm_i_lw(p, ptr, uasm_rel_lo(pgdc), ptr); 981 uasm_i_srl(p, tmp, tmp, PGDIR_SHIFT); /* get pgd only bits */ 982 uasm_i_sll(p, tmp, tmp, PGD_T_LOG2); 983 uasm_i_addu(p, ptr, ptr, tmp); /* add in pgd offset */ 984 } 985 986 #endif /* !CONFIG_64BIT */ 987 988 static void build_adjust_context(u32 **p, unsigned int ctx) 989 { 990 unsigned int shift = 4 - (PTE_T_LOG2 + 1) + PAGE_SHIFT - 12; 991 unsigned int mask = (PTRS_PER_PTE / 2 - 1) << (PTE_T_LOG2 + 1); 992 993 switch (current_cpu_type()) { 994 case CPU_VR41XX: 995 case CPU_VR4111: 996 case CPU_VR4121: 997 case CPU_VR4122: 998 case CPU_VR4131: 999 case CPU_VR4181: 1000 case CPU_VR4181A: 1001 case CPU_VR4133: 1002 shift += 2; 1003 break; 1004 1005 default: 1006 break; 1007 } 1008 1009 if (shift) 1010 UASM_i_SRL(p, ctx, ctx, shift); 1011 uasm_i_andi(p, ctx, ctx, mask); 1012 } 1013 1014 static void build_get_ptep(u32 **p, unsigned int tmp, unsigned int ptr) 1015 { 1016 /* 1017 * Bug workaround for the Nevada. It seems as if under certain 1018 * circumstances the move from cp0_context might produce a 1019 * bogus result when the mfc0 instruction and its consumer are 1020 * in a different cacheline or a load instruction, probably any 1021 * memory reference, is between them. 1022 */ 1023 switch (current_cpu_type()) { 1024 case CPU_NEVADA: 1025 UASM_i_LW(p, ptr, 0, ptr); 1026 GET_CONTEXT(p, tmp); /* get context reg */ 1027 break; 1028 1029 default: 1030 GET_CONTEXT(p, tmp); /* get context reg */ 1031 UASM_i_LW(p, ptr, 0, ptr); 1032 break; 1033 } 1034 1035 build_adjust_context(p, tmp); 1036 UASM_i_ADDU(p, ptr, ptr, tmp); /* add in offset */ 1037 } 1038 1039 static void build_update_entries(u32 **p, unsigned int tmp, unsigned int ptep) 1040 { 1041 /* 1042 * 64bit address support (36bit on a 32bit CPU) in a 32bit 1043 * Kernel is a special case. Only a few CPUs use it. 1044 */ 1045 #ifdef CONFIG_64BIT_PHYS_ADDR 1046 if (cpu_has_64bits) { 1047 uasm_i_ld(p, tmp, 0, ptep); /* get even pte */ 1048 uasm_i_ld(p, ptep, sizeof(pte_t), ptep); /* get odd pte */ 1049 if (cpu_has_rixi) { 1050 UASM_i_ROTR(p, tmp, tmp, ilog2(_PAGE_GLOBAL)); 1051 UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */ 1052 UASM_i_ROTR(p, ptep, ptep, ilog2(_PAGE_GLOBAL)); 1053 } else { 1054 uasm_i_dsrl_safe(p, tmp, tmp, ilog2(_PAGE_GLOBAL)); /* convert to entrylo0 */ 1055 UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */ 1056 uasm_i_dsrl_safe(p, ptep, ptep, ilog2(_PAGE_GLOBAL)); /* convert to entrylo1 */ 1057 } 1058 UASM_i_MTC0(p, ptep, C0_ENTRYLO1); /* load it */ 1059 } else { 1060 int pte_off_even = sizeof(pte_t) / 2; 1061 int pte_off_odd = pte_off_even + sizeof(pte_t); 1062 1063 /* The pte entries are pre-shifted */ 1064 uasm_i_lw(p, tmp, pte_off_even, ptep); /* get even pte */ 1065 UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */ 1066 uasm_i_lw(p, ptep, pte_off_odd, ptep); /* get odd pte */ 1067 UASM_i_MTC0(p, ptep, C0_ENTRYLO1); /* load it */ 1068 } 1069 #else 1070 UASM_i_LW(p, tmp, 0, ptep); /* get even pte */ 1071 UASM_i_LW(p, ptep, sizeof(pte_t), ptep); /* get odd pte */ 1072 if (r45k_bvahwbug()) 1073 build_tlb_probe_entry(p); 1074 if (cpu_has_rixi) { 1075 UASM_i_ROTR(p, tmp, tmp, ilog2(_PAGE_GLOBAL)); 1076 if (r4k_250MHZhwbug()) 1077 UASM_i_MTC0(p, 0, C0_ENTRYLO0); 1078 UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */ 1079 UASM_i_ROTR(p, ptep, ptep, ilog2(_PAGE_GLOBAL)); 1080 } else { 1081 UASM_i_SRL(p, tmp, tmp, ilog2(_PAGE_GLOBAL)); /* convert to entrylo0 */ 1082 if (r4k_250MHZhwbug()) 1083 UASM_i_MTC0(p, 0, C0_ENTRYLO0); 1084 UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */ 1085 UASM_i_SRL(p, ptep, ptep, ilog2(_PAGE_GLOBAL)); /* convert to entrylo1 */ 1086 if (r45k_bvahwbug()) 1087 uasm_i_mfc0(p, tmp, C0_INDEX); 1088 } 1089 if (r4k_250MHZhwbug()) 1090 UASM_i_MTC0(p, 0, C0_ENTRYLO1); 1091 UASM_i_MTC0(p, ptep, C0_ENTRYLO1); /* load it */ 1092 #endif 1093 } 1094 1095 struct mips_huge_tlb_info { 1096 int huge_pte; 1097 int restore_scratch; 1098 }; 1099 1100 static struct mips_huge_tlb_info 1101 build_fast_tlb_refill_handler (u32 **p, struct uasm_label **l, 1102 struct uasm_reloc **r, unsigned int tmp, 1103 unsigned int ptr, int c0_scratch_reg) 1104 { 1105 struct mips_huge_tlb_info rv; 1106 unsigned int even, odd; 1107 int vmalloc_branch_delay_filled = 0; 1108 const int scratch = 1; /* Our extra working register */ 1109 1110 rv.huge_pte = scratch; 1111 rv.restore_scratch = 0; 1112 1113 if (check_for_high_segbits) { 1114 UASM_i_MFC0(p, tmp, C0_BADVADDR); 1115 1116 if (pgd_reg != -1) 1117 UASM_i_MFC0(p, ptr, c0_kscratch(), pgd_reg); 1118 else 1119 UASM_i_MFC0(p, ptr, C0_CONTEXT); 1120 1121 if (c0_scratch_reg >= 0) 1122 UASM_i_MTC0(p, scratch, c0_kscratch(), c0_scratch_reg); 1123 else 1124 UASM_i_SW(p, scratch, scratchpad_offset(0), 0); 1125 1126 uasm_i_dsrl_safe(p, scratch, tmp, 1127 PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3); 1128 uasm_il_bnez(p, r, scratch, label_vmalloc); 1129 1130 if (pgd_reg == -1) { 1131 vmalloc_branch_delay_filled = 1; 1132 /* Clear lower 23 bits of context. */ 1133 uasm_i_dins(p, ptr, 0, 0, 23); 1134 } 1135 } else { 1136 if (pgd_reg != -1) 1137 UASM_i_MFC0(p, ptr, c0_kscratch(), pgd_reg); 1138 else 1139 UASM_i_MFC0(p, ptr, C0_CONTEXT); 1140 1141 UASM_i_MFC0(p, tmp, C0_BADVADDR); 1142 1143 if (c0_scratch_reg >= 0) 1144 UASM_i_MTC0(p, scratch, c0_kscratch(), c0_scratch_reg); 1145 else 1146 UASM_i_SW(p, scratch, scratchpad_offset(0), 0); 1147 1148 if (pgd_reg == -1) 1149 /* Clear lower 23 bits of context. */ 1150 uasm_i_dins(p, ptr, 0, 0, 23); 1151 1152 uasm_il_bltz(p, r, tmp, label_vmalloc); 1153 } 1154 1155 if (pgd_reg == -1) { 1156 vmalloc_branch_delay_filled = 1; 1157 /* 1 0 1 0 1 << 6 xkphys cached */ 1158 uasm_i_ori(p, ptr, ptr, 0x540); 1159 uasm_i_drotr(p, ptr, ptr, 11); 1160 } 1161 1162 #ifdef __PAGETABLE_PMD_FOLDED 1163 #define LOC_PTEP scratch 1164 #else 1165 #define LOC_PTEP ptr 1166 #endif 1167 1168 if (!vmalloc_branch_delay_filled) 1169 /* get pgd offset in bytes */ 1170 uasm_i_dsrl_safe(p, scratch, tmp, PGDIR_SHIFT - 3); 1171 1172 uasm_l_vmalloc_done(l, *p); 1173 1174 /* 1175 * tmp ptr 1176 * fall-through case = badvaddr *pgd_current 1177 * vmalloc case = badvaddr swapper_pg_dir 1178 */ 1179 1180 if (vmalloc_branch_delay_filled) 1181 /* get pgd offset in bytes */ 1182 uasm_i_dsrl_safe(p, scratch, tmp, PGDIR_SHIFT - 3); 1183 1184 #ifdef __PAGETABLE_PMD_FOLDED 1185 GET_CONTEXT(p, tmp); /* get context reg */ 1186 #endif 1187 uasm_i_andi(p, scratch, scratch, (PTRS_PER_PGD - 1) << 3); 1188 1189 if (use_lwx_insns()) { 1190 UASM_i_LWX(p, LOC_PTEP, scratch, ptr); 1191 } else { 1192 uasm_i_daddu(p, ptr, ptr, scratch); /* add in pgd offset */ 1193 uasm_i_ld(p, LOC_PTEP, 0, ptr); /* get pmd pointer */ 1194 } 1195 1196 #ifndef __PAGETABLE_PMD_FOLDED 1197 /* get pmd offset in bytes */ 1198 uasm_i_dsrl_safe(p, scratch, tmp, PMD_SHIFT - 3); 1199 uasm_i_andi(p, scratch, scratch, (PTRS_PER_PMD - 1) << 3); 1200 GET_CONTEXT(p, tmp); /* get context reg */ 1201 1202 if (use_lwx_insns()) { 1203 UASM_i_LWX(p, scratch, scratch, ptr); 1204 } else { 1205 uasm_i_daddu(p, ptr, ptr, scratch); /* add in pmd offset */ 1206 UASM_i_LW(p, scratch, 0, ptr); 1207 } 1208 #endif 1209 /* Adjust the context during the load latency. */ 1210 build_adjust_context(p, tmp); 1211 1212 #ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT 1213 uasm_il_bbit1(p, r, scratch, ilog2(_PAGE_HUGE), label_tlb_huge_update); 1214 /* 1215 * The in the LWX case we don't want to do the load in the 1216 * delay slot. It cannot issue in the same cycle and may be 1217 * speculative and unneeded. 1218 */ 1219 if (use_lwx_insns()) 1220 uasm_i_nop(p); 1221 #endif /* CONFIG_MIPS_HUGE_TLB_SUPPORT */ 1222 1223 1224 /* build_update_entries */ 1225 if (use_lwx_insns()) { 1226 even = ptr; 1227 odd = tmp; 1228 UASM_i_LWX(p, even, scratch, tmp); 1229 UASM_i_ADDIU(p, tmp, tmp, sizeof(pte_t)); 1230 UASM_i_LWX(p, odd, scratch, tmp); 1231 } else { 1232 UASM_i_ADDU(p, ptr, scratch, tmp); /* add in offset */ 1233 even = tmp; 1234 odd = ptr; 1235 UASM_i_LW(p, even, 0, ptr); /* get even pte */ 1236 UASM_i_LW(p, odd, sizeof(pte_t), ptr); /* get odd pte */ 1237 } 1238 if (cpu_has_rixi) { 1239 uasm_i_drotr(p, even, even, ilog2(_PAGE_GLOBAL)); 1240 UASM_i_MTC0(p, even, C0_ENTRYLO0); /* load it */ 1241 uasm_i_drotr(p, odd, odd, ilog2(_PAGE_GLOBAL)); 1242 } else { 1243 uasm_i_dsrl_safe(p, even, even, ilog2(_PAGE_GLOBAL)); 1244 UASM_i_MTC0(p, even, C0_ENTRYLO0); /* load it */ 1245 uasm_i_dsrl_safe(p, odd, odd, ilog2(_PAGE_GLOBAL)); 1246 } 1247 UASM_i_MTC0(p, odd, C0_ENTRYLO1); /* load it */ 1248 1249 if (c0_scratch_reg >= 0) { 1250 UASM_i_MFC0(p, scratch, c0_kscratch(), c0_scratch_reg); 1251 build_tlb_write_entry(p, l, r, tlb_random); 1252 uasm_l_leave(l, *p); 1253 rv.restore_scratch = 1; 1254 } else if (PAGE_SHIFT == 14 || PAGE_SHIFT == 13) { 1255 build_tlb_write_entry(p, l, r, tlb_random); 1256 uasm_l_leave(l, *p); 1257 UASM_i_LW(p, scratch, scratchpad_offset(0), 0); 1258 } else { 1259 UASM_i_LW(p, scratch, scratchpad_offset(0), 0); 1260 build_tlb_write_entry(p, l, r, tlb_random); 1261 uasm_l_leave(l, *p); 1262 rv.restore_scratch = 1; 1263 } 1264 1265 uasm_i_eret(p); /* return from trap */ 1266 1267 return rv; 1268 } 1269 1270 /* 1271 * For a 64-bit kernel, we are using the 64-bit XTLB refill exception 1272 * because EXL == 0. If we wrap, we can also use the 32 instruction 1273 * slots before the XTLB refill exception handler which belong to the 1274 * unused TLB refill exception. 1275 */ 1276 #define MIPS64_REFILL_INSNS 32 1277 1278 static void build_r4000_tlb_refill_handler(void) 1279 { 1280 u32 *p = tlb_handler; 1281 struct uasm_label *l = labels; 1282 struct uasm_reloc *r = relocs; 1283 u32 *f; 1284 unsigned int final_len; 1285 struct mips_huge_tlb_info htlb_info __maybe_unused; 1286 enum vmalloc64_mode vmalloc_mode __maybe_unused; 1287 1288 memset(tlb_handler, 0, sizeof(tlb_handler)); 1289 memset(labels, 0, sizeof(labels)); 1290 memset(relocs, 0, sizeof(relocs)); 1291 memset(final_handler, 0, sizeof(final_handler)); 1292 1293 if ((scratch_reg >= 0 || scratchpad_available()) && use_bbit_insns()) { 1294 htlb_info = build_fast_tlb_refill_handler(&p, &l, &r, K0, K1, 1295 scratch_reg); 1296 vmalloc_mode = refill_scratch; 1297 } else { 1298 htlb_info.huge_pte = K0; 1299 htlb_info.restore_scratch = 0; 1300 vmalloc_mode = refill_noscratch; 1301 /* 1302 * create the plain linear handler 1303 */ 1304 if (bcm1250_m3_war()) { 1305 unsigned int segbits = 44; 1306 1307 uasm_i_dmfc0(&p, K0, C0_BADVADDR); 1308 uasm_i_dmfc0(&p, K1, C0_ENTRYHI); 1309 uasm_i_xor(&p, K0, K0, K1); 1310 uasm_i_dsrl_safe(&p, K1, K0, 62); 1311 uasm_i_dsrl_safe(&p, K0, K0, 12 + 1); 1312 uasm_i_dsll_safe(&p, K0, K0, 64 + 12 + 1 - segbits); 1313 uasm_i_or(&p, K0, K0, K1); 1314 uasm_il_bnez(&p, &r, K0, label_leave); 1315 /* No need for uasm_i_nop */ 1316 } 1317 1318 #ifdef CONFIG_64BIT 1319 build_get_pmde64(&p, &l, &r, K0, K1); /* get pmd in K1 */ 1320 #else 1321 build_get_pgde32(&p, K0, K1); /* get pgd in K1 */ 1322 #endif 1323 1324 #ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT 1325 build_is_huge_pte(&p, &r, K0, K1, label_tlb_huge_update); 1326 #endif 1327 1328 build_get_ptep(&p, K0, K1); 1329 build_update_entries(&p, K0, K1); 1330 build_tlb_write_entry(&p, &l, &r, tlb_random); 1331 uasm_l_leave(&l, p); 1332 uasm_i_eret(&p); /* return from trap */ 1333 } 1334 #ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT 1335 uasm_l_tlb_huge_update(&l, p); 1336 build_huge_update_entries(&p, htlb_info.huge_pte, K1); 1337 build_huge_tlb_write_entry(&p, &l, &r, K0, tlb_random, 1338 htlb_info.restore_scratch); 1339 #endif 1340 1341 #ifdef CONFIG_64BIT 1342 build_get_pgd_vmalloc64(&p, &l, &r, K0, K1, vmalloc_mode); 1343 #endif 1344 1345 /* 1346 * Overflow check: For the 64bit handler, we need at least one 1347 * free instruction slot for the wrap-around branch. In worst 1348 * case, if the intended insertion point is a delay slot, we 1349 * need three, with the second nop'ed and the third being 1350 * unused. 1351 */ 1352 /* Loongson2 ebase is different than r4k, we have more space */ 1353 #if defined(CONFIG_32BIT) || defined(CONFIG_CPU_LOONGSON2) 1354 if ((p - tlb_handler) > 64) 1355 panic("TLB refill handler space exceeded"); 1356 #else 1357 if (((p - tlb_handler) > (MIPS64_REFILL_INSNS * 2) - 1) 1358 || (((p - tlb_handler) > (MIPS64_REFILL_INSNS * 2) - 3) 1359 && uasm_insn_has_bdelay(relocs, 1360 tlb_handler + MIPS64_REFILL_INSNS - 3))) 1361 panic("TLB refill handler space exceeded"); 1362 #endif 1363 1364 /* 1365 * Now fold the handler in the TLB refill handler space. 1366 */ 1367 #if defined(CONFIG_32BIT) || defined(CONFIG_CPU_LOONGSON2) 1368 f = final_handler; 1369 /* Simplest case, just copy the handler. */ 1370 uasm_copy_handler(relocs, labels, tlb_handler, p, f); 1371 final_len = p - tlb_handler; 1372 #else /* CONFIG_64BIT */ 1373 f = final_handler + MIPS64_REFILL_INSNS; 1374 if ((p - tlb_handler) <= MIPS64_REFILL_INSNS) { 1375 /* Just copy the handler. */ 1376 uasm_copy_handler(relocs, labels, tlb_handler, p, f); 1377 final_len = p - tlb_handler; 1378 } else { 1379 #ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT 1380 const enum label_id ls = label_tlb_huge_update; 1381 #else 1382 const enum label_id ls = label_vmalloc; 1383 #endif 1384 u32 *split; 1385 int ov = 0; 1386 int i; 1387 1388 for (i = 0; i < ARRAY_SIZE(labels) && labels[i].lab != ls; i++) 1389 ; 1390 BUG_ON(i == ARRAY_SIZE(labels)); 1391 split = labels[i].addr; 1392 1393 /* 1394 * See if we have overflown one way or the other. 1395 */ 1396 if (split > tlb_handler + MIPS64_REFILL_INSNS || 1397 split < p - MIPS64_REFILL_INSNS) 1398 ov = 1; 1399 1400 if (ov) { 1401 /* 1402 * Split two instructions before the end. One 1403 * for the branch and one for the instruction 1404 * in the delay slot. 1405 */ 1406 split = tlb_handler + MIPS64_REFILL_INSNS - 2; 1407 1408 /* 1409 * If the branch would fall in a delay slot, 1410 * we must back up an additional instruction 1411 * so that it is no longer in a delay slot. 1412 */ 1413 if (uasm_insn_has_bdelay(relocs, split - 1)) 1414 split--; 1415 } 1416 /* Copy first part of the handler. */ 1417 uasm_copy_handler(relocs, labels, tlb_handler, split, f); 1418 f += split - tlb_handler; 1419 1420 if (ov) { 1421 /* Insert branch. */ 1422 uasm_l_split(&l, final_handler); 1423 uasm_il_b(&f, &r, label_split); 1424 if (uasm_insn_has_bdelay(relocs, split)) 1425 uasm_i_nop(&f); 1426 else { 1427 uasm_copy_handler(relocs, labels, 1428 split, split + 1, f); 1429 uasm_move_labels(labels, f, f + 1, -1); 1430 f++; 1431 split++; 1432 } 1433 } 1434 1435 /* Copy the rest of the handler. */ 1436 uasm_copy_handler(relocs, labels, split, p, final_handler); 1437 final_len = (f - (final_handler + MIPS64_REFILL_INSNS)) + 1438 (p - split); 1439 } 1440 #endif /* CONFIG_64BIT */ 1441 1442 uasm_resolve_relocs(relocs, labels); 1443 pr_debug("Wrote TLB refill handler (%u instructions).\n", 1444 final_len); 1445 1446 memcpy((void *)ebase, final_handler, 0x100); 1447 1448 dump_handler("r4000_tlb_refill", (u32 *)ebase, 64); 1449 } 1450 1451 extern u32 handle_tlbl[], handle_tlbl_end[]; 1452 extern u32 handle_tlbs[], handle_tlbs_end[]; 1453 extern u32 handle_tlbm[], handle_tlbm_end[]; 1454 1455 #ifdef CONFIG_MIPS_PGD_C0_CONTEXT 1456 extern u32 tlbmiss_handler_setup_pgd[], tlbmiss_handler_setup_pgd_end[]; 1457 1458 static void build_r4000_setup_pgd(void) 1459 { 1460 const int a0 = 4; 1461 const int a1 = 5; 1462 u32 *p = tlbmiss_handler_setup_pgd; 1463 const int tlbmiss_handler_setup_pgd_size = 1464 tlbmiss_handler_setup_pgd_end - tlbmiss_handler_setup_pgd; 1465 struct uasm_label *l = labels; 1466 struct uasm_reloc *r = relocs; 1467 1468 memset(tlbmiss_handler_setup_pgd, 0, tlbmiss_handler_setup_pgd_size * 1469 sizeof(tlbmiss_handler_setup_pgd[0])); 1470 memset(labels, 0, sizeof(labels)); 1471 memset(relocs, 0, sizeof(relocs)); 1472 1473 pgd_reg = allocate_kscratch(); 1474 1475 if (pgd_reg == -1) { 1476 /* PGD << 11 in c0_Context */ 1477 /* 1478 * If it is a ckseg0 address, convert to a physical 1479 * address. Shifting right by 29 and adding 4 will 1480 * result in zero for these addresses. 1481 * 1482 */ 1483 UASM_i_SRA(&p, a1, a0, 29); 1484 UASM_i_ADDIU(&p, a1, a1, 4); 1485 uasm_il_bnez(&p, &r, a1, label_tlbl_goaround1); 1486 uasm_i_nop(&p); 1487 uasm_i_dinsm(&p, a0, 0, 29, 64 - 29); 1488 uasm_l_tlbl_goaround1(&l, p); 1489 UASM_i_SLL(&p, a0, a0, 11); 1490 uasm_i_jr(&p, 31); 1491 UASM_i_MTC0(&p, a0, C0_CONTEXT); 1492 } else { 1493 /* PGD in c0_KScratch */ 1494 uasm_i_jr(&p, 31); 1495 UASM_i_MTC0(&p, a0, c0_kscratch(), pgd_reg); 1496 } 1497 if (p >= tlbmiss_handler_setup_pgd_end) 1498 panic("tlbmiss_handler_setup_pgd space exceeded"); 1499 1500 uasm_resolve_relocs(relocs, labels); 1501 pr_debug("Wrote tlbmiss_handler_setup_pgd (%u instructions).\n", 1502 (unsigned int)(p - tlbmiss_handler_setup_pgd)); 1503 1504 dump_handler("tlbmiss_handler", tlbmiss_handler_setup_pgd, 1505 tlbmiss_handler_setup_pgd_size); 1506 } 1507 #endif 1508 1509 static void 1510 iPTE_LW(u32 **p, unsigned int pte, unsigned int ptr) 1511 { 1512 #ifdef CONFIG_SMP 1513 # ifdef CONFIG_64BIT_PHYS_ADDR 1514 if (cpu_has_64bits) 1515 uasm_i_lld(p, pte, 0, ptr); 1516 else 1517 # endif 1518 UASM_i_LL(p, pte, 0, ptr); 1519 #else 1520 # ifdef CONFIG_64BIT_PHYS_ADDR 1521 if (cpu_has_64bits) 1522 uasm_i_ld(p, pte, 0, ptr); 1523 else 1524 # endif 1525 UASM_i_LW(p, pte, 0, ptr); 1526 #endif 1527 } 1528 1529 static void 1530 iPTE_SW(u32 **p, struct uasm_reloc **r, unsigned int pte, unsigned int ptr, 1531 unsigned int mode) 1532 { 1533 #ifdef CONFIG_64BIT_PHYS_ADDR 1534 unsigned int hwmode = mode & (_PAGE_VALID | _PAGE_DIRTY); 1535 #endif 1536 1537 uasm_i_ori(p, pte, pte, mode); 1538 #ifdef CONFIG_SMP 1539 # ifdef CONFIG_64BIT_PHYS_ADDR 1540 if (cpu_has_64bits) 1541 uasm_i_scd(p, pte, 0, ptr); 1542 else 1543 # endif 1544 UASM_i_SC(p, pte, 0, ptr); 1545 1546 if (r10000_llsc_war()) 1547 uasm_il_beqzl(p, r, pte, label_smp_pgtable_change); 1548 else 1549 uasm_il_beqz(p, r, pte, label_smp_pgtable_change); 1550 1551 # ifdef CONFIG_64BIT_PHYS_ADDR 1552 if (!cpu_has_64bits) { 1553 /* no uasm_i_nop needed */ 1554 uasm_i_ll(p, pte, sizeof(pte_t) / 2, ptr); 1555 uasm_i_ori(p, pte, pte, hwmode); 1556 uasm_i_sc(p, pte, sizeof(pte_t) / 2, ptr); 1557 uasm_il_beqz(p, r, pte, label_smp_pgtable_change); 1558 /* no uasm_i_nop needed */ 1559 uasm_i_lw(p, pte, 0, ptr); 1560 } else 1561 uasm_i_nop(p); 1562 # else 1563 uasm_i_nop(p); 1564 # endif 1565 #else 1566 # ifdef CONFIG_64BIT_PHYS_ADDR 1567 if (cpu_has_64bits) 1568 uasm_i_sd(p, pte, 0, ptr); 1569 else 1570 # endif 1571 UASM_i_SW(p, pte, 0, ptr); 1572 1573 # ifdef CONFIG_64BIT_PHYS_ADDR 1574 if (!cpu_has_64bits) { 1575 uasm_i_lw(p, pte, sizeof(pte_t) / 2, ptr); 1576 uasm_i_ori(p, pte, pte, hwmode); 1577 uasm_i_sw(p, pte, sizeof(pte_t) / 2, ptr); 1578 uasm_i_lw(p, pte, 0, ptr); 1579 } 1580 # endif 1581 #endif 1582 } 1583 1584 /* 1585 * Check if PTE is present, if not then jump to LABEL. PTR points to 1586 * the page table where this PTE is located, PTE will be re-loaded 1587 * with it's original value. 1588 */ 1589 static void 1590 build_pte_present(u32 **p, struct uasm_reloc **r, 1591 int pte, int ptr, int scratch, enum label_id lid) 1592 { 1593 int t = scratch >= 0 ? scratch : pte; 1594 1595 if (cpu_has_rixi) { 1596 if (use_bbit_insns()) { 1597 uasm_il_bbit0(p, r, pte, ilog2(_PAGE_PRESENT), lid); 1598 uasm_i_nop(p); 1599 } else { 1600 uasm_i_andi(p, t, pte, _PAGE_PRESENT); 1601 uasm_il_beqz(p, r, t, lid); 1602 if (pte == t) 1603 /* You lose the SMP race :-(*/ 1604 iPTE_LW(p, pte, ptr); 1605 } 1606 } else { 1607 uasm_i_andi(p, t, pte, _PAGE_PRESENT | _PAGE_READ); 1608 uasm_i_xori(p, t, t, _PAGE_PRESENT | _PAGE_READ); 1609 uasm_il_bnez(p, r, t, lid); 1610 if (pte == t) 1611 /* You lose the SMP race :-(*/ 1612 iPTE_LW(p, pte, ptr); 1613 } 1614 } 1615 1616 /* Make PTE valid, store result in PTR. */ 1617 static void 1618 build_make_valid(u32 **p, struct uasm_reloc **r, unsigned int pte, 1619 unsigned int ptr) 1620 { 1621 unsigned int mode = _PAGE_VALID | _PAGE_ACCESSED; 1622 1623 iPTE_SW(p, r, pte, ptr, mode); 1624 } 1625 1626 /* 1627 * Check if PTE can be written to, if not branch to LABEL. Regardless 1628 * restore PTE with value from PTR when done. 1629 */ 1630 static void 1631 build_pte_writable(u32 **p, struct uasm_reloc **r, 1632 unsigned int pte, unsigned int ptr, int scratch, 1633 enum label_id lid) 1634 { 1635 int t = scratch >= 0 ? scratch : pte; 1636 1637 uasm_i_andi(p, t, pte, _PAGE_PRESENT | _PAGE_WRITE); 1638 uasm_i_xori(p, t, t, _PAGE_PRESENT | _PAGE_WRITE); 1639 uasm_il_bnez(p, r, t, lid); 1640 if (pte == t) 1641 /* You lose the SMP race :-(*/ 1642 iPTE_LW(p, pte, ptr); 1643 else 1644 uasm_i_nop(p); 1645 } 1646 1647 /* Make PTE writable, update software status bits as well, then store 1648 * at PTR. 1649 */ 1650 static void 1651 build_make_write(u32 **p, struct uasm_reloc **r, unsigned int pte, 1652 unsigned int ptr) 1653 { 1654 unsigned int mode = (_PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID 1655 | _PAGE_DIRTY); 1656 1657 iPTE_SW(p, r, pte, ptr, mode); 1658 } 1659 1660 /* 1661 * Check if PTE can be modified, if not branch to LABEL. Regardless 1662 * restore PTE with value from PTR when done. 1663 */ 1664 static void 1665 build_pte_modifiable(u32 **p, struct uasm_reloc **r, 1666 unsigned int pte, unsigned int ptr, int scratch, 1667 enum label_id lid) 1668 { 1669 if (use_bbit_insns()) { 1670 uasm_il_bbit0(p, r, pte, ilog2(_PAGE_WRITE), lid); 1671 uasm_i_nop(p); 1672 } else { 1673 int t = scratch >= 0 ? scratch : pte; 1674 uasm_i_andi(p, t, pte, _PAGE_WRITE); 1675 uasm_il_beqz(p, r, t, lid); 1676 if (pte == t) 1677 /* You lose the SMP race :-(*/ 1678 iPTE_LW(p, pte, ptr); 1679 } 1680 } 1681 1682 #ifndef CONFIG_MIPS_PGD_C0_CONTEXT 1683 1684 1685 /* 1686 * R3000 style TLB load/store/modify handlers. 1687 */ 1688 1689 /* 1690 * This places the pte into ENTRYLO0 and writes it with tlbwi. 1691 * Then it returns. 1692 */ 1693 static void 1694 build_r3000_pte_reload_tlbwi(u32 **p, unsigned int pte, unsigned int tmp) 1695 { 1696 uasm_i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */ 1697 uasm_i_mfc0(p, tmp, C0_EPC); /* cp0 delay */ 1698 uasm_i_tlbwi(p); 1699 uasm_i_jr(p, tmp); 1700 uasm_i_rfe(p); /* branch delay */ 1701 } 1702 1703 /* 1704 * This places the pte into ENTRYLO0 and writes it with tlbwi 1705 * or tlbwr as appropriate. This is because the index register 1706 * may have the probe fail bit set as a result of a trap on a 1707 * kseg2 access, i.e. without refill. Then it returns. 1708 */ 1709 static void 1710 build_r3000_tlb_reload_write(u32 **p, struct uasm_label **l, 1711 struct uasm_reloc **r, unsigned int pte, 1712 unsigned int tmp) 1713 { 1714 uasm_i_mfc0(p, tmp, C0_INDEX); 1715 uasm_i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */ 1716 uasm_il_bltz(p, r, tmp, label_r3000_write_probe_fail); /* cp0 delay */ 1717 uasm_i_mfc0(p, tmp, C0_EPC); /* branch delay */ 1718 uasm_i_tlbwi(p); /* cp0 delay */ 1719 uasm_i_jr(p, tmp); 1720 uasm_i_rfe(p); /* branch delay */ 1721 uasm_l_r3000_write_probe_fail(l, *p); 1722 uasm_i_tlbwr(p); /* cp0 delay */ 1723 uasm_i_jr(p, tmp); 1724 uasm_i_rfe(p); /* branch delay */ 1725 } 1726 1727 static void 1728 build_r3000_tlbchange_handler_head(u32 **p, unsigned int pte, 1729 unsigned int ptr) 1730 { 1731 long pgdc = (long)pgd_current; 1732 1733 uasm_i_mfc0(p, pte, C0_BADVADDR); 1734 uasm_i_lui(p, ptr, uasm_rel_hi(pgdc)); /* cp0 delay */ 1735 uasm_i_lw(p, ptr, uasm_rel_lo(pgdc), ptr); 1736 uasm_i_srl(p, pte, pte, 22); /* load delay */ 1737 uasm_i_sll(p, pte, pte, 2); 1738 uasm_i_addu(p, ptr, ptr, pte); 1739 uasm_i_mfc0(p, pte, C0_CONTEXT); 1740 uasm_i_lw(p, ptr, 0, ptr); /* cp0 delay */ 1741 uasm_i_andi(p, pte, pte, 0xffc); /* load delay */ 1742 uasm_i_addu(p, ptr, ptr, pte); 1743 uasm_i_lw(p, pte, 0, ptr); 1744 uasm_i_tlbp(p); /* load delay */ 1745 } 1746 1747 static void build_r3000_tlb_load_handler(void) 1748 { 1749 u32 *p = handle_tlbl; 1750 const int handle_tlbl_size = handle_tlbl_end - handle_tlbl; 1751 struct uasm_label *l = labels; 1752 struct uasm_reloc *r = relocs; 1753 1754 memset(handle_tlbl, 0, handle_tlbl_size * sizeof(handle_tlbl[0])); 1755 memset(labels, 0, sizeof(labels)); 1756 memset(relocs, 0, sizeof(relocs)); 1757 1758 build_r3000_tlbchange_handler_head(&p, K0, K1); 1759 build_pte_present(&p, &r, K0, K1, -1, label_nopage_tlbl); 1760 uasm_i_nop(&p); /* load delay */ 1761 build_make_valid(&p, &r, K0, K1); 1762 build_r3000_tlb_reload_write(&p, &l, &r, K0, K1); 1763 1764 uasm_l_nopage_tlbl(&l, p); 1765 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff); 1766 uasm_i_nop(&p); 1767 1768 if (p >= handle_tlbl_end) 1769 panic("TLB load handler fastpath space exceeded"); 1770 1771 uasm_resolve_relocs(relocs, labels); 1772 pr_debug("Wrote TLB load handler fastpath (%u instructions).\n", 1773 (unsigned int)(p - handle_tlbl)); 1774 1775 dump_handler("r3000_tlb_load", handle_tlbl, handle_tlbl_size); 1776 } 1777 1778 static void build_r3000_tlb_store_handler(void) 1779 { 1780 u32 *p = handle_tlbs; 1781 const int handle_tlbs_size = handle_tlbs_end - handle_tlbs; 1782 struct uasm_label *l = labels; 1783 struct uasm_reloc *r = relocs; 1784 1785 memset(handle_tlbs, 0, handle_tlbs_size * sizeof(handle_tlbs[0])); 1786 memset(labels, 0, sizeof(labels)); 1787 memset(relocs, 0, sizeof(relocs)); 1788 1789 build_r3000_tlbchange_handler_head(&p, K0, K1); 1790 build_pte_writable(&p, &r, K0, K1, -1, label_nopage_tlbs); 1791 uasm_i_nop(&p); /* load delay */ 1792 build_make_write(&p, &r, K0, K1); 1793 build_r3000_tlb_reload_write(&p, &l, &r, K0, K1); 1794 1795 uasm_l_nopage_tlbs(&l, p); 1796 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); 1797 uasm_i_nop(&p); 1798 1799 if (p >= handle_tlbs_end) 1800 panic("TLB store handler fastpath space exceeded"); 1801 1802 uasm_resolve_relocs(relocs, labels); 1803 pr_debug("Wrote TLB store handler fastpath (%u instructions).\n", 1804 (unsigned int)(p - handle_tlbs)); 1805 1806 dump_handler("r3000_tlb_store", handle_tlbs, handle_tlbs_size); 1807 } 1808 1809 static void build_r3000_tlb_modify_handler(void) 1810 { 1811 u32 *p = handle_tlbm; 1812 const int handle_tlbm_size = handle_tlbm_end - handle_tlbm; 1813 struct uasm_label *l = labels; 1814 struct uasm_reloc *r = relocs; 1815 1816 memset(handle_tlbm, 0, handle_tlbm_size * sizeof(handle_tlbm[0])); 1817 memset(labels, 0, sizeof(labels)); 1818 memset(relocs, 0, sizeof(relocs)); 1819 1820 build_r3000_tlbchange_handler_head(&p, K0, K1); 1821 build_pte_modifiable(&p, &r, K0, K1, -1, label_nopage_tlbm); 1822 uasm_i_nop(&p); /* load delay */ 1823 build_make_write(&p, &r, K0, K1); 1824 build_r3000_pte_reload_tlbwi(&p, K0, K1); 1825 1826 uasm_l_nopage_tlbm(&l, p); 1827 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); 1828 uasm_i_nop(&p); 1829 1830 if (p >= handle_tlbm_end) 1831 panic("TLB modify handler fastpath space exceeded"); 1832 1833 uasm_resolve_relocs(relocs, labels); 1834 pr_debug("Wrote TLB modify handler fastpath (%u instructions).\n", 1835 (unsigned int)(p - handle_tlbm)); 1836 1837 dump_handler("r3000_tlb_modify", handle_tlbm, handle_tlbm_size); 1838 } 1839 #endif /* CONFIG_MIPS_PGD_C0_CONTEXT */ 1840 1841 /* 1842 * R4000 style TLB load/store/modify handlers. 1843 */ 1844 static struct work_registers 1845 build_r4000_tlbchange_handler_head(u32 **p, struct uasm_label **l, 1846 struct uasm_reloc **r) 1847 { 1848 struct work_registers wr = build_get_work_registers(p); 1849 1850 #ifdef CONFIG_64BIT 1851 build_get_pmde64(p, l, r, wr.r1, wr.r2); /* get pmd in ptr */ 1852 #else 1853 build_get_pgde32(p, wr.r1, wr.r2); /* get pgd in ptr */ 1854 #endif 1855 1856 #ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT 1857 /* 1858 * For huge tlb entries, pmd doesn't contain an address but 1859 * instead contains the tlb pte. Check the PAGE_HUGE bit and 1860 * see if we need to jump to huge tlb processing. 1861 */ 1862 build_is_huge_pte(p, r, wr.r1, wr.r2, label_tlb_huge_update); 1863 #endif 1864 1865 UASM_i_MFC0(p, wr.r1, C0_BADVADDR); 1866 UASM_i_LW(p, wr.r2, 0, wr.r2); 1867 UASM_i_SRL(p, wr.r1, wr.r1, PAGE_SHIFT + PTE_ORDER - PTE_T_LOG2); 1868 uasm_i_andi(p, wr.r1, wr.r1, (PTRS_PER_PTE - 1) << PTE_T_LOG2); 1869 UASM_i_ADDU(p, wr.r2, wr.r2, wr.r1); 1870 1871 #ifdef CONFIG_SMP 1872 uasm_l_smp_pgtable_change(l, *p); 1873 #endif 1874 iPTE_LW(p, wr.r1, wr.r2); /* get even pte */ 1875 if (!m4kc_tlbp_war()) 1876 build_tlb_probe_entry(p); 1877 return wr; 1878 } 1879 1880 static void 1881 build_r4000_tlbchange_handler_tail(u32 **p, struct uasm_label **l, 1882 struct uasm_reloc **r, unsigned int tmp, 1883 unsigned int ptr) 1884 { 1885 uasm_i_ori(p, ptr, ptr, sizeof(pte_t)); 1886 uasm_i_xori(p, ptr, ptr, sizeof(pte_t)); 1887 build_update_entries(p, tmp, ptr); 1888 build_tlb_write_entry(p, l, r, tlb_indexed); 1889 uasm_l_leave(l, *p); 1890 build_restore_work_registers(p); 1891 uasm_i_eret(p); /* return from trap */ 1892 1893 #ifdef CONFIG_64BIT 1894 build_get_pgd_vmalloc64(p, l, r, tmp, ptr, not_refill); 1895 #endif 1896 } 1897 1898 static void build_r4000_tlb_load_handler(void) 1899 { 1900 u32 *p = handle_tlbl; 1901 const int handle_tlbl_size = handle_tlbl_end - handle_tlbl; 1902 struct uasm_label *l = labels; 1903 struct uasm_reloc *r = relocs; 1904 struct work_registers wr; 1905 1906 memset(handle_tlbl, 0, handle_tlbl_size * sizeof(handle_tlbl[0])); 1907 memset(labels, 0, sizeof(labels)); 1908 memset(relocs, 0, sizeof(relocs)); 1909 1910 if (bcm1250_m3_war()) { 1911 unsigned int segbits = 44; 1912 1913 uasm_i_dmfc0(&p, K0, C0_BADVADDR); 1914 uasm_i_dmfc0(&p, K1, C0_ENTRYHI); 1915 uasm_i_xor(&p, K0, K0, K1); 1916 uasm_i_dsrl_safe(&p, K1, K0, 62); 1917 uasm_i_dsrl_safe(&p, K0, K0, 12 + 1); 1918 uasm_i_dsll_safe(&p, K0, K0, 64 + 12 + 1 - segbits); 1919 uasm_i_or(&p, K0, K0, K1); 1920 uasm_il_bnez(&p, &r, K0, label_leave); 1921 /* No need for uasm_i_nop */ 1922 } 1923 1924 wr = build_r4000_tlbchange_handler_head(&p, &l, &r); 1925 build_pte_present(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbl); 1926 if (m4kc_tlbp_war()) 1927 build_tlb_probe_entry(&p); 1928 1929 if (cpu_has_rixi) { 1930 /* 1931 * If the page is not _PAGE_VALID, RI or XI could not 1932 * have triggered it. Skip the expensive test.. 1933 */ 1934 if (use_bbit_insns()) { 1935 uasm_il_bbit0(&p, &r, wr.r1, ilog2(_PAGE_VALID), 1936 label_tlbl_goaround1); 1937 } else { 1938 uasm_i_andi(&p, wr.r3, wr.r1, _PAGE_VALID); 1939 uasm_il_beqz(&p, &r, wr.r3, label_tlbl_goaround1); 1940 } 1941 uasm_i_nop(&p); 1942 1943 uasm_i_tlbr(&p); 1944 1945 switch (current_cpu_type()) { 1946 default: 1947 if (cpu_has_mips_r2) { 1948 uasm_i_ehb(&p); 1949 1950 case CPU_CAVIUM_OCTEON: 1951 case CPU_CAVIUM_OCTEON_PLUS: 1952 case CPU_CAVIUM_OCTEON2: 1953 break; 1954 } 1955 } 1956 1957 /* Examine entrylo 0 or 1 based on ptr. */ 1958 if (use_bbit_insns()) { 1959 uasm_i_bbit0(&p, wr.r2, ilog2(sizeof(pte_t)), 8); 1960 } else { 1961 uasm_i_andi(&p, wr.r3, wr.r2, sizeof(pte_t)); 1962 uasm_i_beqz(&p, wr.r3, 8); 1963 } 1964 /* load it in the delay slot*/ 1965 UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO0); 1966 /* load it if ptr is odd */ 1967 UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO1); 1968 /* 1969 * If the entryLo (now in wr.r3) is valid (bit 1), RI or 1970 * XI must have triggered it. 1971 */ 1972 if (use_bbit_insns()) { 1973 uasm_il_bbit1(&p, &r, wr.r3, 1, label_nopage_tlbl); 1974 uasm_i_nop(&p); 1975 uasm_l_tlbl_goaround1(&l, p); 1976 } else { 1977 uasm_i_andi(&p, wr.r3, wr.r3, 2); 1978 uasm_il_bnez(&p, &r, wr.r3, label_nopage_tlbl); 1979 uasm_i_nop(&p); 1980 } 1981 uasm_l_tlbl_goaround1(&l, p); 1982 } 1983 build_make_valid(&p, &r, wr.r1, wr.r2); 1984 build_r4000_tlbchange_handler_tail(&p, &l, &r, wr.r1, wr.r2); 1985 1986 #ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT 1987 /* 1988 * This is the entry point when build_r4000_tlbchange_handler_head 1989 * spots a huge page. 1990 */ 1991 uasm_l_tlb_huge_update(&l, p); 1992 iPTE_LW(&p, wr.r1, wr.r2); 1993 build_pte_present(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbl); 1994 build_tlb_probe_entry(&p); 1995 1996 if (cpu_has_rixi) { 1997 /* 1998 * If the page is not _PAGE_VALID, RI or XI could not 1999 * have triggered it. Skip the expensive test.. 2000 */ 2001 if (use_bbit_insns()) { 2002 uasm_il_bbit0(&p, &r, wr.r1, ilog2(_PAGE_VALID), 2003 label_tlbl_goaround2); 2004 } else { 2005 uasm_i_andi(&p, wr.r3, wr.r1, _PAGE_VALID); 2006 uasm_il_beqz(&p, &r, wr.r3, label_tlbl_goaround2); 2007 } 2008 uasm_i_nop(&p); 2009 2010 uasm_i_tlbr(&p); 2011 2012 switch (current_cpu_type()) { 2013 default: 2014 if (cpu_has_mips_r2) { 2015 uasm_i_ehb(&p); 2016 2017 case CPU_CAVIUM_OCTEON: 2018 case CPU_CAVIUM_OCTEON_PLUS: 2019 case CPU_CAVIUM_OCTEON2: 2020 break; 2021 } 2022 } 2023 2024 /* Examine entrylo 0 or 1 based on ptr. */ 2025 if (use_bbit_insns()) { 2026 uasm_i_bbit0(&p, wr.r2, ilog2(sizeof(pte_t)), 8); 2027 } else { 2028 uasm_i_andi(&p, wr.r3, wr.r2, sizeof(pte_t)); 2029 uasm_i_beqz(&p, wr.r3, 8); 2030 } 2031 /* load it in the delay slot*/ 2032 UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO0); 2033 /* load it if ptr is odd */ 2034 UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO1); 2035 /* 2036 * If the entryLo (now in wr.r3) is valid (bit 1), RI or 2037 * XI must have triggered it. 2038 */ 2039 if (use_bbit_insns()) { 2040 uasm_il_bbit0(&p, &r, wr.r3, 1, label_tlbl_goaround2); 2041 } else { 2042 uasm_i_andi(&p, wr.r3, wr.r3, 2); 2043 uasm_il_beqz(&p, &r, wr.r3, label_tlbl_goaround2); 2044 } 2045 if (PM_DEFAULT_MASK == 0) 2046 uasm_i_nop(&p); 2047 /* 2048 * We clobbered C0_PAGEMASK, restore it. On the other branch 2049 * it is restored in build_huge_tlb_write_entry. 2050 */ 2051 build_restore_pagemask(&p, &r, wr.r3, label_nopage_tlbl, 0); 2052 2053 uasm_l_tlbl_goaround2(&l, p); 2054 } 2055 uasm_i_ori(&p, wr.r1, wr.r1, (_PAGE_ACCESSED | _PAGE_VALID)); 2056 build_huge_handler_tail(&p, &r, &l, wr.r1, wr.r2); 2057 #endif 2058 2059 uasm_l_nopage_tlbl(&l, p); 2060 build_restore_work_registers(&p); 2061 #ifdef CONFIG_CPU_MICROMIPS 2062 if ((unsigned long)tlb_do_page_fault_0 & 1) { 2063 uasm_i_lui(&p, K0, uasm_rel_hi((long)tlb_do_page_fault_0)); 2064 uasm_i_addiu(&p, K0, K0, uasm_rel_lo((long)tlb_do_page_fault_0)); 2065 uasm_i_jr(&p, K0); 2066 } else 2067 #endif 2068 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff); 2069 uasm_i_nop(&p); 2070 2071 if (p >= handle_tlbl_end) 2072 panic("TLB load handler fastpath space exceeded"); 2073 2074 uasm_resolve_relocs(relocs, labels); 2075 pr_debug("Wrote TLB load handler fastpath (%u instructions).\n", 2076 (unsigned int)(p - handle_tlbl)); 2077 2078 dump_handler("r4000_tlb_load", handle_tlbl, handle_tlbl_size); 2079 } 2080 2081 static void build_r4000_tlb_store_handler(void) 2082 { 2083 u32 *p = handle_tlbs; 2084 const int handle_tlbs_size = handle_tlbs_end - handle_tlbs; 2085 struct uasm_label *l = labels; 2086 struct uasm_reloc *r = relocs; 2087 struct work_registers wr; 2088 2089 memset(handle_tlbs, 0, handle_tlbs_size * sizeof(handle_tlbs[0])); 2090 memset(labels, 0, sizeof(labels)); 2091 memset(relocs, 0, sizeof(relocs)); 2092 2093 wr = build_r4000_tlbchange_handler_head(&p, &l, &r); 2094 build_pte_writable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbs); 2095 if (m4kc_tlbp_war()) 2096 build_tlb_probe_entry(&p); 2097 build_make_write(&p, &r, wr.r1, wr.r2); 2098 build_r4000_tlbchange_handler_tail(&p, &l, &r, wr.r1, wr.r2); 2099 2100 #ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT 2101 /* 2102 * This is the entry point when 2103 * build_r4000_tlbchange_handler_head spots a huge page. 2104 */ 2105 uasm_l_tlb_huge_update(&l, p); 2106 iPTE_LW(&p, wr.r1, wr.r2); 2107 build_pte_writable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbs); 2108 build_tlb_probe_entry(&p); 2109 uasm_i_ori(&p, wr.r1, wr.r1, 2110 _PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID | _PAGE_DIRTY); 2111 build_huge_handler_tail(&p, &r, &l, wr.r1, wr.r2); 2112 #endif 2113 2114 uasm_l_nopage_tlbs(&l, p); 2115 build_restore_work_registers(&p); 2116 #ifdef CONFIG_CPU_MICROMIPS 2117 if ((unsigned long)tlb_do_page_fault_1 & 1) { 2118 uasm_i_lui(&p, K0, uasm_rel_hi((long)tlb_do_page_fault_1)); 2119 uasm_i_addiu(&p, K0, K0, uasm_rel_lo((long)tlb_do_page_fault_1)); 2120 uasm_i_jr(&p, K0); 2121 } else 2122 #endif 2123 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); 2124 uasm_i_nop(&p); 2125 2126 if (p >= handle_tlbs_end) 2127 panic("TLB store handler fastpath space exceeded"); 2128 2129 uasm_resolve_relocs(relocs, labels); 2130 pr_debug("Wrote TLB store handler fastpath (%u instructions).\n", 2131 (unsigned int)(p - handle_tlbs)); 2132 2133 dump_handler("r4000_tlb_store", handle_tlbs, handle_tlbs_size); 2134 } 2135 2136 static void build_r4000_tlb_modify_handler(void) 2137 { 2138 u32 *p = handle_tlbm; 2139 const int handle_tlbm_size = handle_tlbm_end - handle_tlbm; 2140 struct uasm_label *l = labels; 2141 struct uasm_reloc *r = relocs; 2142 struct work_registers wr; 2143 2144 memset(handle_tlbm, 0, handle_tlbm_size * sizeof(handle_tlbm[0])); 2145 memset(labels, 0, sizeof(labels)); 2146 memset(relocs, 0, sizeof(relocs)); 2147 2148 wr = build_r4000_tlbchange_handler_head(&p, &l, &r); 2149 build_pte_modifiable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbm); 2150 if (m4kc_tlbp_war()) 2151 build_tlb_probe_entry(&p); 2152 /* Present and writable bits set, set accessed and dirty bits. */ 2153 build_make_write(&p, &r, wr.r1, wr.r2); 2154 build_r4000_tlbchange_handler_tail(&p, &l, &r, wr.r1, wr.r2); 2155 2156 #ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT 2157 /* 2158 * This is the entry point when 2159 * build_r4000_tlbchange_handler_head spots a huge page. 2160 */ 2161 uasm_l_tlb_huge_update(&l, p); 2162 iPTE_LW(&p, wr.r1, wr.r2); 2163 build_pte_modifiable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbm); 2164 build_tlb_probe_entry(&p); 2165 uasm_i_ori(&p, wr.r1, wr.r1, 2166 _PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID | _PAGE_DIRTY); 2167 build_huge_handler_tail(&p, &r, &l, wr.r1, wr.r2); 2168 #endif 2169 2170 uasm_l_nopage_tlbm(&l, p); 2171 build_restore_work_registers(&p); 2172 #ifdef CONFIG_CPU_MICROMIPS 2173 if ((unsigned long)tlb_do_page_fault_1 & 1) { 2174 uasm_i_lui(&p, K0, uasm_rel_hi((long)tlb_do_page_fault_1)); 2175 uasm_i_addiu(&p, K0, K0, uasm_rel_lo((long)tlb_do_page_fault_1)); 2176 uasm_i_jr(&p, K0); 2177 } else 2178 #endif 2179 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); 2180 uasm_i_nop(&p); 2181 2182 if (p >= handle_tlbm_end) 2183 panic("TLB modify handler fastpath space exceeded"); 2184 2185 uasm_resolve_relocs(relocs, labels); 2186 pr_debug("Wrote TLB modify handler fastpath (%u instructions).\n", 2187 (unsigned int)(p - handle_tlbm)); 2188 2189 dump_handler("r4000_tlb_modify", handle_tlbm, handle_tlbm_size); 2190 } 2191 2192 static void flush_tlb_handlers(void) 2193 { 2194 local_flush_icache_range((unsigned long)handle_tlbl, 2195 (unsigned long)handle_tlbl_end); 2196 local_flush_icache_range((unsigned long)handle_tlbs, 2197 (unsigned long)handle_tlbs_end); 2198 local_flush_icache_range((unsigned long)handle_tlbm, 2199 (unsigned long)handle_tlbm_end); 2200 #ifdef CONFIG_MIPS_PGD_C0_CONTEXT 2201 local_flush_icache_range((unsigned long)tlbmiss_handler_setup_pgd, 2202 (unsigned long)tlbmiss_handler_setup_pgd_end); 2203 #endif 2204 } 2205 2206 void build_tlb_refill_handler(void) 2207 { 2208 /* 2209 * The refill handler is generated per-CPU, multi-node systems 2210 * may have local storage for it. The other handlers are only 2211 * needed once. 2212 */ 2213 static int run_once = 0; 2214 2215 output_pgtable_bits_defines(); 2216 2217 #ifdef CONFIG_64BIT 2218 check_for_high_segbits = current_cpu_data.vmbits > (PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3); 2219 #endif 2220 2221 switch (current_cpu_type()) { 2222 case CPU_R2000: 2223 case CPU_R3000: 2224 case CPU_R3000A: 2225 case CPU_R3081E: 2226 case CPU_TX3912: 2227 case CPU_TX3922: 2228 case CPU_TX3927: 2229 #ifndef CONFIG_MIPS_PGD_C0_CONTEXT 2230 if (cpu_has_local_ebase) 2231 build_r3000_tlb_refill_handler(); 2232 if (!run_once) { 2233 if (!cpu_has_local_ebase) 2234 build_r3000_tlb_refill_handler(); 2235 build_r3000_tlb_load_handler(); 2236 build_r3000_tlb_store_handler(); 2237 build_r3000_tlb_modify_handler(); 2238 flush_tlb_handlers(); 2239 run_once++; 2240 } 2241 #else 2242 panic("No R3000 TLB refill handler"); 2243 #endif 2244 break; 2245 2246 case CPU_R6000: 2247 case CPU_R6000A: 2248 panic("No R6000 TLB refill handler yet"); 2249 break; 2250 2251 case CPU_R8000: 2252 panic("No R8000 TLB refill handler yet"); 2253 break; 2254 2255 default: 2256 if (!run_once) { 2257 scratch_reg = allocate_kscratch(); 2258 #ifdef CONFIG_MIPS_PGD_C0_CONTEXT 2259 build_r4000_setup_pgd(); 2260 #endif 2261 build_r4000_tlb_load_handler(); 2262 build_r4000_tlb_store_handler(); 2263 build_r4000_tlb_modify_handler(); 2264 if (!cpu_has_local_ebase) 2265 build_r4000_tlb_refill_handler(); 2266 flush_tlb_handlers(); 2267 run_once++; 2268 } 2269 if (cpu_has_local_ebase) 2270 build_r4000_tlb_refill_handler(); 2271 } 2272 } 2273