1 /* 2 * This file is subject to the terms and conditions of the GNU General Public 3 * License. See the file "COPYING" in the main directory of this archive 4 * for more details. 5 * 6 * Synthesize TLB refill handlers at runtime. 7 * 8 * Copyright (C) 2004, 2005, 2006, 2008 Thiemo Seufer 9 * Copyright (C) 2005, 2007, 2008, 2009 Maciej W. Rozycki 10 * Copyright (C) 2006 Ralf Baechle (ralf@linux-mips.org) 11 * Copyright (C) 2008, 2009 Cavium Networks, Inc. 12 * Copyright (C) 2011 MIPS Technologies, Inc. 13 * 14 * ... and the days got worse and worse and now you see 15 * I've gone completely out of my mind. 16 * 17 * They're coming to take me a away haha 18 * they're coming to take me a away hoho hihi haha 19 * to the funny farm where code is beautiful all the time ... 20 * 21 * (Condolences to Napoleon XIV) 22 */ 23 24 #include <linux/bug.h> 25 #include <linux/export.h> 26 #include <linux/kernel.h> 27 #include <linux/types.h> 28 #include <linux/smp.h> 29 #include <linux/string.h> 30 #include <linux/cache.h> 31 #include <linux/pgtable.h> 32 33 #include <asm/cacheflush.h> 34 #include <asm/cpu-type.h> 35 #include <asm/mmu_context.h> 36 #include <asm/war.h> 37 #include <asm/uasm.h> 38 #include <asm/setup.h> 39 #include <asm/tlbex.h> 40 41 static int mips_xpa_disabled; 42 43 static int __init xpa_disable(char *s) 44 { 45 mips_xpa_disabled = 1; 46 47 return 1; 48 } 49 50 __setup("noxpa", xpa_disable); 51 52 /* 53 * TLB load/store/modify handlers. 54 * 55 * Only the fastpath gets synthesized at runtime, the slowpath for 56 * do_page_fault remains normal asm. 57 */ 58 extern void tlb_do_page_fault_0(void); 59 extern void tlb_do_page_fault_1(void); 60 61 struct work_registers { 62 int r1; 63 int r2; 64 int r3; 65 }; 66 67 struct tlb_reg_save { 68 unsigned long a; 69 unsigned long b; 70 } ____cacheline_aligned_in_smp; 71 72 static struct tlb_reg_save handler_reg_save[NR_CPUS]; 73 74 static inline int r45k_bvahwbug(void) 75 { 76 /* XXX: We should probe for the presence of this bug, but we don't. */ 77 return 0; 78 } 79 80 static inline int r4k_250MHZhwbug(void) 81 { 82 /* XXX: We should probe for the presence of this bug, but we don't. */ 83 return 0; 84 } 85 86 extern int sb1250_m3_workaround_needed(void); 87 88 static inline int __maybe_unused bcm1250_m3_war(void) 89 { 90 if (IS_ENABLED(CONFIG_SB1_PASS_2_WORKAROUNDS)) 91 return sb1250_m3_workaround_needed(); 92 return 0; 93 } 94 95 static inline int __maybe_unused r10000_llsc_war(void) 96 { 97 return IS_ENABLED(CONFIG_WAR_R10000_LLSC); 98 } 99 100 static int use_bbit_insns(void) 101 { 102 switch (current_cpu_type()) { 103 case CPU_CAVIUM_OCTEON: 104 case CPU_CAVIUM_OCTEON_PLUS: 105 case CPU_CAVIUM_OCTEON2: 106 case CPU_CAVIUM_OCTEON3: 107 return 1; 108 default: 109 return 0; 110 } 111 } 112 113 static int use_lwx_insns(void) 114 { 115 switch (current_cpu_type()) { 116 case CPU_CAVIUM_OCTEON2: 117 case CPU_CAVIUM_OCTEON3: 118 return 1; 119 default: 120 return 0; 121 } 122 } 123 #if defined(CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE) && \ 124 CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE > 0 125 static bool scratchpad_available(void) 126 { 127 return true; 128 } 129 static int scratchpad_offset(int i) 130 { 131 /* 132 * CVMSEG starts at address -32768 and extends for 133 * CAVIUM_OCTEON_CVMSEG_SIZE 128 byte cache lines. 134 */ 135 i += 1; /* Kernel use starts at the top and works down. */ 136 return CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE * 128 - (8 * i) - 32768; 137 } 138 #else 139 static bool scratchpad_available(void) 140 { 141 return false; 142 } 143 static int scratchpad_offset(int i) 144 { 145 BUG(); 146 /* Really unreachable, but evidently some GCC want this. */ 147 return 0; 148 } 149 #endif 150 /* 151 * Found by experiment: At least some revisions of the 4kc throw under 152 * some circumstances a machine check exception, triggered by invalid 153 * values in the index register. Delaying the tlbp instruction until 154 * after the next branch, plus adding an additional nop in front of 155 * tlbwi/tlbwr avoids the invalid index register values. Nobody knows 156 * why; it's not an issue caused by the core RTL. 157 * 158 */ 159 static int m4kc_tlbp_war(void) 160 { 161 return current_cpu_type() == CPU_4KC; 162 } 163 164 /* Handle labels (which must be positive integers). */ 165 enum label_id { 166 label_second_part = 1, 167 label_leave, 168 label_vmalloc, 169 label_vmalloc_done, 170 label_tlbw_hazard_0, 171 label_split = label_tlbw_hazard_0 + 8, 172 label_tlbl_goaround1, 173 label_tlbl_goaround2, 174 label_nopage_tlbl, 175 label_nopage_tlbs, 176 label_nopage_tlbm, 177 label_smp_pgtable_change, 178 label_r3000_write_probe_fail, 179 label_large_segbits_fault, 180 #ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT 181 label_tlb_huge_update, 182 #endif 183 }; 184 185 UASM_L_LA(_second_part) 186 UASM_L_LA(_leave) 187 UASM_L_LA(_vmalloc) 188 UASM_L_LA(_vmalloc_done) 189 /* _tlbw_hazard_x is handled differently. */ 190 UASM_L_LA(_split) 191 UASM_L_LA(_tlbl_goaround1) 192 UASM_L_LA(_tlbl_goaround2) 193 UASM_L_LA(_nopage_tlbl) 194 UASM_L_LA(_nopage_tlbs) 195 UASM_L_LA(_nopage_tlbm) 196 UASM_L_LA(_smp_pgtable_change) 197 UASM_L_LA(_r3000_write_probe_fail) 198 UASM_L_LA(_large_segbits_fault) 199 #ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT 200 UASM_L_LA(_tlb_huge_update) 201 #endif 202 203 static int hazard_instance; 204 205 static void uasm_bgezl_hazard(u32 **p, struct uasm_reloc **r, int instance) 206 { 207 switch (instance) { 208 case 0 ... 7: 209 uasm_il_bgezl(p, r, 0, label_tlbw_hazard_0 + instance); 210 return; 211 default: 212 BUG(); 213 } 214 } 215 216 static void uasm_bgezl_label(struct uasm_label **l, u32 **p, int instance) 217 { 218 switch (instance) { 219 case 0 ... 7: 220 uasm_build_label(l, *p, label_tlbw_hazard_0 + instance); 221 break; 222 default: 223 BUG(); 224 } 225 } 226 227 /* 228 * pgtable bits are assigned dynamically depending on processor feature 229 * and statically based on kernel configuration. This spits out the actual 230 * values the kernel is using. Required to make sense from disassembled 231 * TLB exception handlers. 232 */ 233 static void output_pgtable_bits_defines(void) 234 { 235 #define pr_define(fmt, ...) \ 236 pr_debug("#define " fmt, ##__VA_ARGS__) 237 238 pr_debug("#include <asm/asm.h>\n"); 239 pr_debug("#include <asm/regdef.h>\n"); 240 pr_debug("\n"); 241 242 pr_define("_PAGE_PRESENT_SHIFT %d\n", _PAGE_PRESENT_SHIFT); 243 pr_define("_PAGE_NO_READ_SHIFT %d\n", _PAGE_NO_READ_SHIFT); 244 pr_define("_PAGE_WRITE_SHIFT %d\n", _PAGE_WRITE_SHIFT); 245 pr_define("_PAGE_ACCESSED_SHIFT %d\n", _PAGE_ACCESSED_SHIFT); 246 pr_define("_PAGE_MODIFIED_SHIFT %d\n", _PAGE_MODIFIED_SHIFT); 247 #ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT 248 pr_define("_PAGE_HUGE_SHIFT %d\n", _PAGE_HUGE_SHIFT); 249 #endif 250 #ifdef _PAGE_NO_EXEC_SHIFT 251 if (cpu_has_rixi) 252 pr_define("_PAGE_NO_EXEC_SHIFT %d\n", _PAGE_NO_EXEC_SHIFT); 253 #endif 254 pr_define("_PAGE_GLOBAL_SHIFT %d\n", _PAGE_GLOBAL_SHIFT); 255 pr_define("_PAGE_VALID_SHIFT %d\n", _PAGE_VALID_SHIFT); 256 pr_define("_PAGE_DIRTY_SHIFT %d\n", _PAGE_DIRTY_SHIFT); 257 pr_define("_PFN_SHIFT %d\n", _PFN_SHIFT); 258 pr_debug("\n"); 259 } 260 261 static inline void dump_handler(const char *symbol, const void *start, const void *end) 262 { 263 unsigned int count = (end - start) / sizeof(u32); 264 const u32 *handler = start; 265 int i; 266 267 pr_debug("LEAF(%s)\n", symbol); 268 269 pr_debug("\t.set push\n"); 270 pr_debug("\t.set noreorder\n"); 271 272 for (i = 0; i < count; i++) 273 pr_debug("\t.word\t0x%08x\t\t# %p\n", handler[i], &handler[i]); 274 275 pr_debug("\t.set\tpop\n"); 276 277 pr_debug("\tEND(%s)\n", symbol); 278 } 279 280 /* The only general purpose registers allowed in TLB handlers. */ 281 #define K0 26 282 #define K1 27 283 284 /* Some CP0 registers */ 285 #define C0_INDEX 0, 0 286 #define C0_ENTRYLO0 2, 0 287 #define C0_TCBIND 2, 2 288 #define C0_ENTRYLO1 3, 0 289 #define C0_CONTEXT 4, 0 290 #define C0_PAGEMASK 5, 0 291 #define C0_PWBASE 5, 5 292 #define C0_PWFIELD 5, 6 293 #define C0_PWSIZE 5, 7 294 #define C0_PWCTL 6, 6 295 #define C0_BADVADDR 8, 0 296 #define C0_PGD 9, 7 297 #define C0_ENTRYHI 10, 0 298 #define C0_EPC 14, 0 299 #define C0_XCONTEXT 20, 0 300 301 #ifdef CONFIG_64BIT 302 # define GET_CONTEXT(buf, reg) UASM_i_MFC0(buf, reg, C0_XCONTEXT) 303 #else 304 # define GET_CONTEXT(buf, reg) UASM_i_MFC0(buf, reg, C0_CONTEXT) 305 #endif 306 307 /* The worst case length of the handler is around 18 instructions for 308 * R3000-style TLBs and up to 63 instructions for R4000-style TLBs. 309 * Maximum space available is 32 instructions for R3000 and 64 310 * instructions for R4000. 311 * 312 * We deliberately chose a buffer size of 128, so we won't scribble 313 * over anything important on overflow before we panic. 314 */ 315 static u32 tlb_handler[128]; 316 317 /* simply assume worst case size for labels and relocs */ 318 static struct uasm_label labels[128]; 319 static struct uasm_reloc relocs[128]; 320 321 static int check_for_high_segbits; 322 static bool fill_includes_sw_bits; 323 324 static unsigned int kscratch_used_mask; 325 326 static inline int __maybe_unused c0_kscratch(void) 327 { 328 switch (current_cpu_type()) { 329 case CPU_XLP: 330 case CPU_XLR: 331 return 22; 332 default: 333 return 31; 334 } 335 } 336 337 static int allocate_kscratch(void) 338 { 339 int r; 340 unsigned int a = cpu_data[0].kscratch_mask & ~kscratch_used_mask; 341 342 r = ffs(a); 343 344 if (r == 0) 345 return -1; 346 347 r--; /* make it zero based */ 348 349 kscratch_used_mask |= (1 << r); 350 351 return r; 352 } 353 354 static int scratch_reg; 355 int pgd_reg; 356 EXPORT_SYMBOL_GPL(pgd_reg); 357 enum vmalloc64_mode {not_refill, refill_scratch, refill_noscratch}; 358 359 static struct work_registers build_get_work_registers(u32 **p) 360 { 361 struct work_registers r; 362 363 if (scratch_reg >= 0) { 364 /* Save in CPU local C0_KScratch? */ 365 UASM_i_MTC0(p, 1, c0_kscratch(), scratch_reg); 366 r.r1 = K0; 367 r.r2 = K1; 368 r.r3 = 1; 369 return r; 370 } 371 372 if (num_possible_cpus() > 1) { 373 /* Get smp_processor_id */ 374 UASM_i_CPUID_MFC0(p, K0, SMP_CPUID_REG); 375 UASM_i_SRL_SAFE(p, K0, K0, SMP_CPUID_REGSHIFT); 376 377 /* handler_reg_save index in K0 */ 378 UASM_i_SLL(p, K0, K0, ilog2(sizeof(struct tlb_reg_save))); 379 380 UASM_i_LA(p, K1, (long)&handler_reg_save); 381 UASM_i_ADDU(p, K0, K0, K1); 382 } else { 383 UASM_i_LA(p, K0, (long)&handler_reg_save); 384 } 385 /* K0 now points to save area, save $1 and $2 */ 386 UASM_i_SW(p, 1, offsetof(struct tlb_reg_save, a), K0); 387 UASM_i_SW(p, 2, offsetof(struct tlb_reg_save, b), K0); 388 389 r.r1 = K1; 390 r.r2 = 1; 391 r.r3 = 2; 392 return r; 393 } 394 395 static void build_restore_work_registers(u32 **p) 396 { 397 if (scratch_reg >= 0) { 398 uasm_i_ehb(p); 399 UASM_i_MFC0(p, 1, c0_kscratch(), scratch_reg); 400 return; 401 } 402 /* K0 already points to save area, restore $1 and $2 */ 403 UASM_i_LW(p, 1, offsetof(struct tlb_reg_save, a), K0); 404 UASM_i_LW(p, 2, offsetof(struct tlb_reg_save, b), K0); 405 } 406 407 #ifndef CONFIG_MIPS_PGD_C0_CONTEXT 408 409 /* 410 * CONFIG_MIPS_PGD_C0_CONTEXT implies 64 bit and lack of pgd_current, 411 * we cannot do r3000 under these circumstances. 412 * 413 * The R3000 TLB handler is simple. 414 */ 415 static void build_r3000_tlb_refill_handler(void) 416 { 417 long pgdc = (long)pgd_current; 418 u32 *p; 419 420 memset(tlb_handler, 0, sizeof(tlb_handler)); 421 p = tlb_handler; 422 423 uasm_i_mfc0(&p, K0, C0_BADVADDR); 424 uasm_i_lui(&p, K1, uasm_rel_hi(pgdc)); /* cp0 delay */ 425 uasm_i_lw(&p, K1, uasm_rel_lo(pgdc), K1); 426 uasm_i_srl(&p, K0, K0, 22); /* load delay */ 427 uasm_i_sll(&p, K0, K0, 2); 428 uasm_i_addu(&p, K1, K1, K0); 429 uasm_i_mfc0(&p, K0, C0_CONTEXT); 430 uasm_i_lw(&p, K1, 0, K1); /* cp0 delay */ 431 uasm_i_andi(&p, K0, K0, 0xffc); /* load delay */ 432 uasm_i_addu(&p, K1, K1, K0); 433 uasm_i_lw(&p, K0, 0, K1); 434 uasm_i_nop(&p); /* load delay */ 435 uasm_i_mtc0(&p, K0, C0_ENTRYLO0); 436 uasm_i_mfc0(&p, K1, C0_EPC); /* cp0 delay */ 437 uasm_i_tlbwr(&p); /* cp0 delay */ 438 uasm_i_jr(&p, K1); 439 uasm_i_rfe(&p); /* branch delay */ 440 441 if (p > tlb_handler + 32) 442 panic("TLB refill handler space exceeded"); 443 444 pr_debug("Wrote TLB refill handler (%u instructions).\n", 445 (unsigned int)(p - tlb_handler)); 446 447 memcpy((void *)ebase, tlb_handler, 0x80); 448 local_flush_icache_range(ebase, ebase + 0x80); 449 dump_handler("r3000_tlb_refill", (u32 *)ebase, (u32 *)(ebase + 0x80)); 450 } 451 #endif /* CONFIG_MIPS_PGD_C0_CONTEXT */ 452 453 /* 454 * The R4000 TLB handler is much more complicated. We have two 455 * consecutive handler areas with 32 instructions space each. 456 * Since they aren't used at the same time, we can overflow in the 457 * other one.To keep things simple, we first assume linear space, 458 * then we relocate it to the final handler layout as needed. 459 */ 460 static u32 final_handler[64]; 461 462 /* 463 * Hazards 464 * 465 * From the IDT errata for the QED RM5230 (Nevada), processor revision 1.0: 466 * 2. A timing hazard exists for the TLBP instruction. 467 * 468 * stalling_instruction 469 * TLBP 470 * 471 * The JTLB is being read for the TLBP throughout the stall generated by the 472 * previous instruction. This is not really correct as the stalling instruction 473 * can modify the address used to access the JTLB. The failure symptom is that 474 * the TLBP instruction will use an address created for the stalling instruction 475 * and not the address held in C0_ENHI and thus report the wrong results. 476 * 477 * The software work-around is to not allow the instruction preceding the TLBP 478 * to stall - make it an NOP or some other instruction guaranteed not to stall. 479 * 480 * Errata 2 will not be fixed. This errata is also on the R5000. 481 * 482 * As if we MIPS hackers wouldn't know how to nop pipelines happy ... 483 */ 484 static void __maybe_unused build_tlb_probe_entry(u32 **p) 485 { 486 switch (current_cpu_type()) { 487 /* Found by experiment: R4600 v2.0/R4700 needs this, too. */ 488 case CPU_R4600: 489 case CPU_R4700: 490 case CPU_R5000: 491 case CPU_NEVADA: 492 uasm_i_nop(p); 493 uasm_i_tlbp(p); 494 break; 495 496 default: 497 uasm_i_tlbp(p); 498 break; 499 } 500 } 501 502 void build_tlb_write_entry(u32 **p, struct uasm_label **l, 503 struct uasm_reloc **r, 504 enum tlb_write_entry wmode) 505 { 506 void(*tlbw)(u32 **) = NULL; 507 508 switch (wmode) { 509 case tlb_random: tlbw = uasm_i_tlbwr; break; 510 case tlb_indexed: tlbw = uasm_i_tlbwi; break; 511 } 512 513 if (cpu_has_mips_r2_r6) { 514 if (cpu_has_mips_r2_exec_hazard) 515 uasm_i_ehb(p); 516 tlbw(p); 517 return; 518 } 519 520 switch (current_cpu_type()) { 521 case CPU_R4000PC: 522 case CPU_R4000SC: 523 case CPU_R4000MC: 524 case CPU_R4400PC: 525 case CPU_R4400SC: 526 case CPU_R4400MC: 527 /* 528 * This branch uses up a mtc0 hazard nop slot and saves 529 * two nops after the tlbw instruction. 530 */ 531 uasm_bgezl_hazard(p, r, hazard_instance); 532 tlbw(p); 533 uasm_bgezl_label(l, p, hazard_instance); 534 hazard_instance++; 535 uasm_i_nop(p); 536 break; 537 538 case CPU_R4600: 539 case CPU_R4700: 540 uasm_i_nop(p); 541 tlbw(p); 542 uasm_i_nop(p); 543 break; 544 545 case CPU_R5000: 546 case CPU_NEVADA: 547 uasm_i_nop(p); /* QED specifies 2 nops hazard */ 548 uasm_i_nop(p); /* QED specifies 2 nops hazard */ 549 tlbw(p); 550 break; 551 552 case CPU_5KC: 553 case CPU_TX49XX: 554 case CPU_PR4450: 555 case CPU_XLR: 556 uasm_i_nop(p); 557 tlbw(p); 558 break; 559 560 case CPU_R10000: 561 case CPU_R12000: 562 case CPU_R14000: 563 case CPU_R16000: 564 case CPU_4KC: 565 case CPU_4KEC: 566 case CPU_M14KC: 567 case CPU_M14KEC: 568 case CPU_SB1: 569 case CPU_SB1A: 570 case CPU_4KSC: 571 case CPU_20KC: 572 case CPU_25KF: 573 case CPU_BMIPS32: 574 case CPU_BMIPS3300: 575 case CPU_BMIPS4350: 576 case CPU_BMIPS4380: 577 case CPU_BMIPS5000: 578 case CPU_LOONGSON2EF: 579 case CPU_LOONGSON64: 580 case CPU_R5500: 581 if (m4kc_tlbp_war()) 582 uasm_i_nop(p); 583 fallthrough; 584 case CPU_ALCHEMY: 585 tlbw(p); 586 break; 587 588 case CPU_RM7000: 589 uasm_i_nop(p); 590 uasm_i_nop(p); 591 uasm_i_nop(p); 592 uasm_i_nop(p); 593 tlbw(p); 594 break; 595 596 case CPU_VR4111: 597 case CPU_VR4121: 598 case CPU_VR4122: 599 case CPU_VR4181: 600 case CPU_VR4181A: 601 uasm_i_nop(p); 602 uasm_i_nop(p); 603 tlbw(p); 604 uasm_i_nop(p); 605 uasm_i_nop(p); 606 break; 607 608 case CPU_VR4131: 609 case CPU_VR4133: 610 uasm_i_nop(p); 611 uasm_i_nop(p); 612 tlbw(p); 613 break; 614 615 case CPU_XBURST: 616 tlbw(p); 617 uasm_i_nop(p); 618 break; 619 620 default: 621 panic("No TLB refill handler yet (CPU type: %d)", 622 current_cpu_type()); 623 break; 624 } 625 } 626 EXPORT_SYMBOL_GPL(build_tlb_write_entry); 627 628 static __maybe_unused void build_convert_pte_to_entrylo(u32 **p, 629 unsigned int reg) 630 { 631 if (_PAGE_GLOBAL_SHIFT == 0) { 632 /* pte_t is already in EntryLo format */ 633 return; 634 } 635 636 if (cpu_has_rixi && !!_PAGE_NO_EXEC) { 637 if (fill_includes_sw_bits) { 638 UASM_i_ROTR(p, reg, reg, ilog2(_PAGE_GLOBAL)); 639 } else { 640 UASM_i_SRL(p, reg, reg, ilog2(_PAGE_NO_EXEC)); 641 UASM_i_ROTR(p, reg, reg, 642 ilog2(_PAGE_GLOBAL) - ilog2(_PAGE_NO_EXEC)); 643 } 644 } else { 645 #ifdef CONFIG_PHYS_ADDR_T_64BIT 646 uasm_i_dsrl_safe(p, reg, reg, ilog2(_PAGE_GLOBAL)); 647 #else 648 UASM_i_SRL(p, reg, reg, ilog2(_PAGE_GLOBAL)); 649 #endif 650 } 651 } 652 653 #ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT 654 655 static void build_restore_pagemask(u32 **p, struct uasm_reloc **r, 656 unsigned int tmp, enum label_id lid, 657 int restore_scratch) 658 { 659 if (restore_scratch) { 660 /* 661 * Ensure the MFC0 below observes the value written to the 662 * KScratch register by the prior MTC0. 663 */ 664 if (scratch_reg >= 0) 665 uasm_i_ehb(p); 666 667 /* Reset default page size */ 668 if (PM_DEFAULT_MASK >> 16) { 669 uasm_i_lui(p, tmp, PM_DEFAULT_MASK >> 16); 670 uasm_i_ori(p, tmp, tmp, PM_DEFAULT_MASK & 0xffff); 671 uasm_i_mtc0(p, tmp, C0_PAGEMASK); 672 uasm_il_b(p, r, lid); 673 } else if (PM_DEFAULT_MASK) { 674 uasm_i_ori(p, tmp, 0, PM_DEFAULT_MASK); 675 uasm_i_mtc0(p, tmp, C0_PAGEMASK); 676 uasm_il_b(p, r, lid); 677 } else { 678 uasm_i_mtc0(p, 0, C0_PAGEMASK); 679 uasm_il_b(p, r, lid); 680 } 681 if (scratch_reg >= 0) 682 UASM_i_MFC0(p, 1, c0_kscratch(), scratch_reg); 683 else 684 UASM_i_LW(p, 1, scratchpad_offset(0), 0); 685 } else { 686 /* Reset default page size */ 687 if (PM_DEFAULT_MASK >> 16) { 688 uasm_i_lui(p, tmp, PM_DEFAULT_MASK >> 16); 689 uasm_i_ori(p, tmp, tmp, PM_DEFAULT_MASK & 0xffff); 690 uasm_il_b(p, r, lid); 691 uasm_i_mtc0(p, tmp, C0_PAGEMASK); 692 } else if (PM_DEFAULT_MASK) { 693 uasm_i_ori(p, tmp, 0, PM_DEFAULT_MASK); 694 uasm_il_b(p, r, lid); 695 uasm_i_mtc0(p, tmp, C0_PAGEMASK); 696 } else { 697 uasm_il_b(p, r, lid); 698 uasm_i_mtc0(p, 0, C0_PAGEMASK); 699 } 700 } 701 } 702 703 static void build_huge_tlb_write_entry(u32 **p, struct uasm_label **l, 704 struct uasm_reloc **r, 705 unsigned int tmp, 706 enum tlb_write_entry wmode, 707 int restore_scratch) 708 { 709 /* Set huge page tlb entry size */ 710 uasm_i_lui(p, tmp, PM_HUGE_MASK >> 16); 711 uasm_i_ori(p, tmp, tmp, PM_HUGE_MASK & 0xffff); 712 uasm_i_mtc0(p, tmp, C0_PAGEMASK); 713 714 build_tlb_write_entry(p, l, r, wmode); 715 716 build_restore_pagemask(p, r, tmp, label_leave, restore_scratch); 717 } 718 719 /* 720 * Check if Huge PTE is present, if so then jump to LABEL. 721 */ 722 static void 723 build_is_huge_pte(u32 **p, struct uasm_reloc **r, unsigned int tmp, 724 unsigned int pmd, int lid) 725 { 726 UASM_i_LW(p, tmp, 0, pmd); 727 if (use_bbit_insns()) { 728 uasm_il_bbit1(p, r, tmp, ilog2(_PAGE_HUGE), lid); 729 } else { 730 uasm_i_andi(p, tmp, tmp, _PAGE_HUGE); 731 uasm_il_bnez(p, r, tmp, lid); 732 } 733 } 734 735 static void build_huge_update_entries(u32 **p, unsigned int pte, 736 unsigned int tmp) 737 { 738 int small_sequence; 739 740 /* 741 * A huge PTE describes an area the size of the 742 * configured huge page size. This is twice the 743 * of the large TLB entry size we intend to use. 744 * A TLB entry half the size of the configured 745 * huge page size is configured into entrylo0 746 * and entrylo1 to cover the contiguous huge PTE 747 * address space. 748 */ 749 small_sequence = (HPAGE_SIZE >> 7) < 0x10000; 750 751 /* We can clobber tmp. It isn't used after this.*/ 752 if (!small_sequence) 753 uasm_i_lui(p, tmp, HPAGE_SIZE >> (7 + 16)); 754 755 build_convert_pte_to_entrylo(p, pte); 756 UASM_i_MTC0(p, pte, C0_ENTRYLO0); /* load it */ 757 /* convert to entrylo1 */ 758 if (small_sequence) 759 UASM_i_ADDIU(p, pte, pte, HPAGE_SIZE >> 7); 760 else 761 UASM_i_ADDU(p, pte, pte, tmp); 762 763 UASM_i_MTC0(p, pte, C0_ENTRYLO1); /* load it */ 764 } 765 766 static void build_huge_handler_tail(u32 **p, struct uasm_reloc **r, 767 struct uasm_label **l, 768 unsigned int pte, 769 unsigned int ptr, 770 unsigned int flush) 771 { 772 #ifdef CONFIG_SMP 773 UASM_i_SC(p, pte, 0, ptr); 774 uasm_il_beqz(p, r, pte, label_tlb_huge_update); 775 UASM_i_LW(p, pte, 0, ptr); /* Needed because SC killed our PTE */ 776 #else 777 UASM_i_SW(p, pte, 0, ptr); 778 #endif 779 if (cpu_has_ftlb && flush) { 780 BUG_ON(!cpu_has_tlbinv); 781 782 UASM_i_MFC0(p, ptr, C0_ENTRYHI); 783 uasm_i_ori(p, ptr, ptr, MIPS_ENTRYHI_EHINV); 784 UASM_i_MTC0(p, ptr, C0_ENTRYHI); 785 build_tlb_write_entry(p, l, r, tlb_indexed); 786 787 uasm_i_xori(p, ptr, ptr, MIPS_ENTRYHI_EHINV); 788 UASM_i_MTC0(p, ptr, C0_ENTRYHI); 789 build_huge_update_entries(p, pte, ptr); 790 build_huge_tlb_write_entry(p, l, r, pte, tlb_random, 0); 791 792 return; 793 } 794 795 build_huge_update_entries(p, pte, ptr); 796 build_huge_tlb_write_entry(p, l, r, pte, tlb_indexed, 0); 797 } 798 #endif /* CONFIG_MIPS_HUGE_TLB_SUPPORT */ 799 800 #ifdef CONFIG_64BIT 801 /* 802 * TMP and PTR are scratch. 803 * TMP will be clobbered, PTR will hold the pmd entry. 804 */ 805 void build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r, 806 unsigned int tmp, unsigned int ptr) 807 { 808 #ifndef CONFIG_MIPS_PGD_C0_CONTEXT 809 long pgdc = (long)pgd_current; 810 #endif 811 /* 812 * The vmalloc handling is not in the hotpath. 813 */ 814 uasm_i_dmfc0(p, tmp, C0_BADVADDR); 815 816 if (check_for_high_segbits) { 817 /* 818 * The kernel currently implicitely assumes that the 819 * MIPS SEGBITS parameter for the processor is 820 * (PGDIR_SHIFT+PGDIR_BITS) or less, and will never 821 * allocate virtual addresses outside the maximum 822 * range for SEGBITS = (PGDIR_SHIFT+PGDIR_BITS). But 823 * that doesn't prevent user code from accessing the 824 * higher xuseg addresses. Here, we make sure that 825 * everything but the lower xuseg addresses goes down 826 * the module_alloc/vmalloc path. 827 */ 828 uasm_i_dsrl_safe(p, ptr, tmp, PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3); 829 uasm_il_bnez(p, r, ptr, label_vmalloc); 830 } else { 831 uasm_il_bltz(p, r, tmp, label_vmalloc); 832 } 833 /* No uasm_i_nop needed here, since the next insn doesn't touch TMP. */ 834 835 if (pgd_reg != -1) { 836 /* pgd is in pgd_reg */ 837 if (cpu_has_ldpte) 838 UASM_i_MFC0(p, ptr, C0_PWBASE); 839 else 840 UASM_i_MFC0(p, ptr, c0_kscratch(), pgd_reg); 841 } else { 842 #if defined(CONFIG_MIPS_PGD_C0_CONTEXT) 843 /* 844 * &pgd << 11 stored in CONTEXT [23..63]. 845 */ 846 UASM_i_MFC0(p, ptr, C0_CONTEXT); 847 848 /* Clear lower 23 bits of context. */ 849 uasm_i_dins(p, ptr, 0, 0, 23); 850 851 /* 1 0 1 0 1 << 6 xkphys cached */ 852 uasm_i_ori(p, ptr, ptr, 0x540); 853 uasm_i_drotr(p, ptr, ptr, 11); 854 #elif defined(CONFIG_SMP) 855 UASM_i_CPUID_MFC0(p, ptr, SMP_CPUID_REG); 856 uasm_i_dsrl_safe(p, ptr, ptr, SMP_CPUID_PTRSHIFT); 857 UASM_i_LA_mostly(p, tmp, pgdc); 858 uasm_i_daddu(p, ptr, ptr, tmp); 859 uasm_i_dmfc0(p, tmp, C0_BADVADDR); 860 uasm_i_ld(p, ptr, uasm_rel_lo(pgdc), ptr); 861 #else 862 UASM_i_LA_mostly(p, ptr, pgdc); 863 uasm_i_ld(p, ptr, uasm_rel_lo(pgdc), ptr); 864 #endif 865 } 866 867 uasm_l_vmalloc_done(l, *p); 868 869 /* get pgd offset in bytes */ 870 uasm_i_dsrl_safe(p, tmp, tmp, PGDIR_SHIFT - 3); 871 872 uasm_i_andi(p, tmp, tmp, (PTRS_PER_PGD - 1)<<3); 873 uasm_i_daddu(p, ptr, ptr, tmp); /* add in pgd offset */ 874 #ifndef __PAGETABLE_PUD_FOLDED 875 uasm_i_dmfc0(p, tmp, C0_BADVADDR); /* get faulting address */ 876 uasm_i_ld(p, ptr, 0, ptr); /* get pud pointer */ 877 uasm_i_dsrl_safe(p, tmp, tmp, PUD_SHIFT - 3); /* get pud offset in bytes */ 878 uasm_i_andi(p, tmp, tmp, (PTRS_PER_PUD - 1) << 3); 879 uasm_i_daddu(p, ptr, ptr, tmp); /* add in pud offset */ 880 #endif 881 #ifndef __PAGETABLE_PMD_FOLDED 882 uasm_i_dmfc0(p, tmp, C0_BADVADDR); /* get faulting address */ 883 uasm_i_ld(p, ptr, 0, ptr); /* get pmd pointer */ 884 uasm_i_dsrl_safe(p, tmp, tmp, PMD_SHIFT-3); /* get pmd offset in bytes */ 885 uasm_i_andi(p, tmp, tmp, (PTRS_PER_PMD - 1)<<3); 886 uasm_i_daddu(p, ptr, ptr, tmp); /* add in pmd offset */ 887 #endif 888 } 889 EXPORT_SYMBOL_GPL(build_get_pmde64); 890 891 /* 892 * BVADDR is the faulting address, PTR is scratch. 893 * PTR will hold the pgd for vmalloc. 894 */ 895 static void 896 build_get_pgd_vmalloc64(u32 **p, struct uasm_label **l, struct uasm_reloc **r, 897 unsigned int bvaddr, unsigned int ptr, 898 enum vmalloc64_mode mode) 899 { 900 long swpd = (long)swapper_pg_dir; 901 int single_insn_swpd; 902 int did_vmalloc_branch = 0; 903 904 single_insn_swpd = uasm_in_compat_space_p(swpd) && !uasm_rel_lo(swpd); 905 906 uasm_l_vmalloc(l, *p); 907 908 if (mode != not_refill && check_for_high_segbits) { 909 if (single_insn_swpd) { 910 uasm_il_bltz(p, r, bvaddr, label_vmalloc_done); 911 uasm_i_lui(p, ptr, uasm_rel_hi(swpd)); 912 did_vmalloc_branch = 1; 913 /* fall through */ 914 } else { 915 uasm_il_bgez(p, r, bvaddr, label_large_segbits_fault); 916 } 917 } 918 if (!did_vmalloc_branch) { 919 if (single_insn_swpd) { 920 uasm_il_b(p, r, label_vmalloc_done); 921 uasm_i_lui(p, ptr, uasm_rel_hi(swpd)); 922 } else { 923 UASM_i_LA_mostly(p, ptr, swpd); 924 uasm_il_b(p, r, label_vmalloc_done); 925 if (uasm_in_compat_space_p(swpd)) 926 uasm_i_addiu(p, ptr, ptr, uasm_rel_lo(swpd)); 927 else 928 uasm_i_daddiu(p, ptr, ptr, uasm_rel_lo(swpd)); 929 } 930 } 931 if (mode != not_refill && check_for_high_segbits) { 932 uasm_l_large_segbits_fault(l, *p); 933 934 if (mode == refill_scratch && scratch_reg >= 0) 935 uasm_i_ehb(p); 936 937 /* 938 * We get here if we are an xsseg address, or if we are 939 * an xuseg address above (PGDIR_SHIFT+PGDIR_BITS) boundary. 940 * 941 * Ignoring xsseg (assume disabled so would generate 942 * (address errors?), the only remaining possibility 943 * is the upper xuseg addresses. On processors with 944 * TLB_SEGBITS <= PGDIR_SHIFT+PGDIR_BITS, these 945 * addresses would have taken an address error. We try 946 * to mimic that here by taking a load/istream page 947 * fault. 948 */ 949 if (IS_ENABLED(CONFIG_CPU_LOONGSON3_WORKAROUNDS)) 950 uasm_i_sync(p, 0); 951 UASM_i_LA(p, ptr, (unsigned long)tlb_do_page_fault_0); 952 uasm_i_jr(p, ptr); 953 954 if (mode == refill_scratch) { 955 if (scratch_reg >= 0) 956 UASM_i_MFC0(p, 1, c0_kscratch(), scratch_reg); 957 else 958 UASM_i_LW(p, 1, scratchpad_offset(0), 0); 959 } else { 960 uasm_i_nop(p); 961 } 962 } 963 } 964 965 #else /* !CONFIG_64BIT */ 966 967 /* 968 * TMP and PTR are scratch. 969 * TMP will be clobbered, PTR will hold the pgd entry. 970 */ 971 void build_get_pgde32(u32 **p, unsigned int tmp, unsigned int ptr) 972 { 973 if (pgd_reg != -1) { 974 /* pgd is in pgd_reg */ 975 uasm_i_mfc0(p, ptr, c0_kscratch(), pgd_reg); 976 uasm_i_mfc0(p, tmp, C0_BADVADDR); /* get faulting address */ 977 } else { 978 long pgdc = (long)pgd_current; 979 980 /* 32 bit SMP has smp_processor_id() stored in CONTEXT. */ 981 #ifdef CONFIG_SMP 982 uasm_i_mfc0(p, ptr, SMP_CPUID_REG); 983 UASM_i_LA_mostly(p, tmp, pgdc); 984 uasm_i_srl(p, ptr, ptr, SMP_CPUID_PTRSHIFT); 985 uasm_i_addu(p, ptr, tmp, ptr); 986 #else 987 UASM_i_LA_mostly(p, ptr, pgdc); 988 #endif 989 uasm_i_mfc0(p, tmp, C0_BADVADDR); /* get faulting address */ 990 uasm_i_lw(p, ptr, uasm_rel_lo(pgdc), ptr); 991 } 992 uasm_i_srl(p, tmp, tmp, PGDIR_SHIFT); /* get pgd only bits */ 993 uasm_i_sll(p, tmp, tmp, PGD_T_LOG2); 994 uasm_i_addu(p, ptr, ptr, tmp); /* add in pgd offset */ 995 } 996 EXPORT_SYMBOL_GPL(build_get_pgde32); 997 998 #endif /* !CONFIG_64BIT */ 999 1000 static void build_adjust_context(u32 **p, unsigned int ctx) 1001 { 1002 unsigned int shift = 4 - (PTE_T_LOG2 + 1) + PAGE_SHIFT - 12; 1003 unsigned int mask = (PTRS_PER_PTE / 2 - 1) << (PTE_T_LOG2 + 1); 1004 1005 switch (current_cpu_type()) { 1006 case CPU_VR41XX: 1007 case CPU_VR4111: 1008 case CPU_VR4121: 1009 case CPU_VR4122: 1010 case CPU_VR4131: 1011 case CPU_VR4181: 1012 case CPU_VR4181A: 1013 case CPU_VR4133: 1014 shift += 2; 1015 break; 1016 1017 default: 1018 break; 1019 } 1020 1021 if (shift) 1022 UASM_i_SRL(p, ctx, ctx, shift); 1023 uasm_i_andi(p, ctx, ctx, mask); 1024 } 1025 1026 void build_get_ptep(u32 **p, unsigned int tmp, unsigned int ptr) 1027 { 1028 /* 1029 * Bug workaround for the Nevada. It seems as if under certain 1030 * circumstances the move from cp0_context might produce a 1031 * bogus result when the mfc0 instruction and its consumer are 1032 * in a different cacheline or a load instruction, probably any 1033 * memory reference, is between them. 1034 */ 1035 switch (current_cpu_type()) { 1036 case CPU_NEVADA: 1037 UASM_i_LW(p, ptr, 0, ptr); 1038 GET_CONTEXT(p, tmp); /* get context reg */ 1039 break; 1040 1041 default: 1042 GET_CONTEXT(p, tmp); /* get context reg */ 1043 UASM_i_LW(p, ptr, 0, ptr); 1044 break; 1045 } 1046 1047 build_adjust_context(p, tmp); 1048 UASM_i_ADDU(p, ptr, ptr, tmp); /* add in offset */ 1049 } 1050 EXPORT_SYMBOL_GPL(build_get_ptep); 1051 1052 void build_update_entries(u32 **p, unsigned int tmp, unsigned int ptep) 1053 { 1054 int pte_off_even = 0; 1055 int pte_off_odd = sizeof(pte_t); 1056 1057 #if defined(CONFIG_CPU_MIPS32) && defined(CONFIG_PHYS_ADDR_T_64BIT) 1058 /* The low 32 bits of EntryLo is stored in pte_high */ 1059 pte_off_even += offsetof(pte_t, pte_high); 1060 pte_off_odd += offsetof(pte_t, pte_high); 1061 #endif 1062 1063 if (IS_ENABLED(CONFIG_XPA)) { 1064 uasm_i_lw(p, tmp, pte_off_even, ptep); /* even pte */ 1065 UASM_i_ROTR(p, tmp, tmp, ilog2(_PAGE_GLOBAL)); 1066 UASM_i_MTC0(p, tmp, C0_ENTRYLO0); 1067 1068 if (cpu_has_xpa && !mips_xpa_disabled) { 1069 uasm_i_lw(p, tmp, 0, ptep); 1070 uasm_i_ext(p, tmp, tmp, 0, 24); 1071 uasm_i_mthc0(p, tmp, C0_ENTRYLO0); 1072 } 1073 1074 uasm_i_lw(p, tmp, pte_off_odd, ptep); /* odd pte */ 1075 UASM_i_ROTR(p, tmp, tmp, ilog2(_PAGE_GLOBAL)); 1076 UASM_i_MTC0(p, tmp, C0_ENTRYLO1); 1077 1078 if (cpu_has_xpa && !mips_xpa_disabled) { 1079 uasm_i_lw(p, tmp, sizeof(pte_t), ptep); 1080 uasm_i_ext(p, tmp, tmp, 0, 24); 1081 uasm_i_mthc0(p, tmp, C0_ENTRYLO1); 1082 } 1083 return; 1084 } 1085 1086 UASM_i_LW(p, tmp, pte_off_even, ptep); /* get even pte */ 1087 UASM_i_LW(p, ptep, pte_off_odd, ptep); /* get odd pte */ 1088 if (r45k_bvahwbug()) 1089 build_tlb_probe_entry(p); 1090 build_convert_pte_to_entrylo(p, tmp); 1091 if (r4k_250MHZhwbug()) 1092 UASM_i_MTC0(p, 0, C0_ENTRYLO0); 1093 UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */ 1094 build_convert_pte_to_entrylo(p, ptep); 1095 if (r45k_bvahwbug()) 1096 uasm_i_mfc0(p, tmp, C0_INDEX); 1097 if (r4k_250MHZhwbug()) 1098 UASM_i_MTC0(p, 0, C0_ENTRYLO1); 1099 UASM_i_MTC0(p, ptep, C0_ENTRYLO1); /* load it */ 1100 } 1101 EXPORT_SYMBOL_GPL(build_update_entries); 1102 1103 struct mips_huge_tlb_info { 1104 int huge_pte; 1105 int restore_scratch; 1106 bool need_reload_pte; 1107 }; 1108 1109 static struct mips_huge_tlb_info 1110 build_fast_tlb_refill_handler (u32 **p, struct uasm_label **l, 1111 struct uasm_reloc **r, unsigned int tmp, 1112 unsigned int ptr, int c0_scratch_reg) 1113 { 1114 struct mips_huge_tlb_info rv; 1115 unsigned int even, odd; 1116 int vmalloc_branch_delay_filled = 0; 1117 const int scratch = 1; /* Our extra working register */ 1118 1119 rv.huge_pte = scratch; 1120 rv.restore_scratch = 0; 1121 rv.need_reload_pte = false; 1122 1123 if (check_for_high_segbits) { 1124 UASM_i_MFC0(p, tmp, C0_BADVADDR); 1125 1126 if (pgd_reg != -1) 1127 UASM_i_MFC0(p, ptr, c0_kscratch(), pgd_reg); 1128 else 1129 UASM_i_MFC0(p, ptr, C0_CONTEXT); 1130 1131 if (c0_scratch_reg >= 0) 1132 UASM_i_MTC0(p, scratch, c0_kscratch(), c0_scratch_reg); 1133 else 1134 UASM_i_SW(p, scratch, scratchpad_offset(0), 0); 1135 1136 uasm_i_dsrl_safe(p, scratch, tmp, 1137 PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3); 1138 uasm_il_bnez(p, r, scratch, label_vmalloc); 1139 1140 if (pgd_reg == -1) { 1141 vmalloc_branch_delay_filled = 1; 1142 /* Clear lower 23 bits of context. */ 1143 uasm_i_dins(p, ptr, 0, 0, 23); 1144 } 1145 } else { 1146 if (pgd_reg != -1) 1147 UASM_i_MFC0(p, ptr, c0_kscratch(), pgd_reg); 1148 else 1149 UASM_i_MFC0(p, ptr, C0_CONTEXT); 1150 1151 UASM_i_MFC0(p, tmp, C0_BADVADDR); 1152 1153 if (c0_scratch_reg >= 0) 1154 UASM_i_MTC0(p, scratch, c0_kscratch(), c0_scratch_reg); 1155 else 1156 UASM_i_SW(p, scratch, scratchpad_offset(0), 0); 1157 1158 if (pgd_reg == -1) 1159 /* Clear lower 23 bits of context. */ 1160 uasm_i_dins(p, ptr, 0, 0, 23); 1161 1162 uasm_il_bltz(p, r, tmp, label_vmalloc); 1163 } 1164 1165 if (pgd_reg == -1) { 1166 vmalloc_branch_delay_filled = 1; 1167 /* 1 0 1 0 1 << 6 xkphys cached */ 1168 uasm_i_ori(p, ptr, ptr, 0x540); 1169 uasm_i_drotr(p, ptr, ptr, 11); 1170 } 1171 1172 #ifdef __PAGETABLE_PMD_FOLDED 1173 #define LOC_PTEP scratch 1174 #else 1175 #define LOC_PTEP ptr 1176 #endif 1177 1178 if (!vmalloc_branch_delay_filled) 1179 /* get pgd offset in bytes */ 1180 uasm_i_dsrl_safe(p, scratch, tmp, PGDIR_SHIFT - 3); 1181 1182 uasm_l_vmalloc_done(l, *p); 1183 1184 /* 1185 * tmp ptr 1186 * fall-through case = badvaddr *pgd_current 1187 * vmalloc case = badvaddr swapper_pg_dir 1188 */ 1189 1190 if (vmalloc_branch_delay_filled) 1191 /* get pgd offset in bytes */ 1192 uasm_i_dsrl_safe(p, scratch, tmp, PGDIR_SHIFT - 3); 1193 1194 #ifdef __PAGETABLE_PMD_FOLDED 1195 GET_CONTEXT(p, tmp); /* get context reg */ 1196 #endif 1197 uasm_i_andi(p, scratch, scratch, (PTRS_PER_PGD - 1) << 3); 1198 1199 if (use_lwx_insns()) { 1200 UASM_i_LWX(p, LOC_PTEP, scratch, ptr); 1201 } else { 1202 uasm_i_daddu(p, ptr, ptr, scratch); /* add in pgd offset */ 1203 uasm_i_ld(p, LOC_PTEP, 0, ptr); /* get pmd pointer */ 1204 } 1205 1206 #ifndef __PAGETABLE_PUD_FOLDED 1207 /* get pud offset in bytes */ 1208 uasm_i_dsrl_safe(p, scratch, tmp, PUD_SHIFT - 3); 1209 uasm_i_andi(p, scratch, scratch, (PTRS_PER_PUD - 1) << 3); 1210 1211 if (use_lwx_insns()) { 1212 UASM_i_LWX(p, ptr, scratch, ptr); 1213 } else { 1214 uasm_i_daddu(p, ptr, ptr, scratch); /* add in pmd offset */ 1215 UASM_i_LW(p, ptr, 0, ptr); 1216 } 1217 /* ptr contains a pointer to PMD entry */ 1218 /* tmp contains the address */ 1219 #endif 1220 1221 #ifndef __PAGETABLE_PMD_FOLDED 1222 /* get pmd offset in bytes */ 1223 uasm_i_dsrl_safe(p, scratch, tmp, PMD_SHIFT - 3); 1224 uasm_i_andi(p, scratch, scratch, (PTRS_PER_PMD - 1) << 3); 1225 GET_CONTEXT(p, tmp); /* get context reg */ 1226 1227 if (use_lwx_insns()) { 1228 UASM_i_LWX(p, scratch, scratch, ptr); 1229 } else { 1230 uasm_i_daddu(p, ptr, ptr, scratch); /* add in pmd offset */ 1231 UASM_i_LW(p, scratch, 0, ptr); 1232 } 1233 #endif 1234 /* Adjust the context during the load latency. */ 1235 build_adjust_context(p, tmp); 1236 1237 #ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT 1238 uasm_il_bbit1(p, r, scratch, ilog2(_PAGE_HUGE), label_tlb_huge_update); 1239 /* 1240 * The in the LWX case we don't want to do the load in the 1241 * delay slot. It cannot issue in the same cycle and may be 1242 * speculative and unneeded. 1243 */ 1244 if (use_lwx_insns()) 1245 uasm_i_nop(p); 1246 #endif /* CONFIG_MIPS_HUGE_TLB_SUPPORT */ 1247 1248 1249 /* build_update_entries */ 1250 if (use_lwx_insns()) { 1251 even = ptr; 1252 odd = tmp; 1253 UASM_i_LWX(p, even, scratch, tmp); 1254 UASM_i_ADDIU(p, tmp, tmp, sizeof(pte_t)); 1255 UASM_i_LWX(p, odd, scratch, tmp); 1256 } else { 1257 UASM_i_ADDU(p, ptr, scratch, tmp); /* add in offset */ 1258 even = tmp; 1259 odd = ptr; 1260 UASM_i_LW(p, even, 0, ptr); /* get even pte */ 1261 UASM_i_LW(p, odd, sizeof(pte_t), ptr); /* get odd pte */ 1262 } 1263 if (cpu_has_rixi) { 1264 uasm_i_drotr(p, even, even, ilog2(_PAGE_GLOBAL)); 1265 UASM_i_MTC0(p, even, C0_ENTRYLO0); /* load it */ 1266 uasm_i_drotr(p, odd, odd, ilog2(_PAGE_GLOBAL)); 1267 } else { 1268 uasm_i_dsrl_safe(p, even, even, ilog2(_PAGE_GLOBAL)); 1269 UASM_i_MTC0(p, even, C0_ENTRYLO0); /* load it */ 1270 uasm_i_dsrl_safe(p, odd, odd, ilog2(_PAGE_GLOBAL)); 1271 } 1272 UASM_i_MTC0(p, odd, C0_ENTRYLO1); /* load it */ 1273 1274 if (c0_scratch_reg >= 0) { 1275 uasm_i_ehb(p); 1276 UASM_i_MFC0(p, scratch, c0_kscratch(), c0_scratch_reg); 1277 build_tlb_write_entry(p, l, r, tlb_random); 1278 uasm_l_leave(l, *p); 1279 rv.restore_scratch = 1; 1280 } else if (PAGE_SHIFT == 14 || PAGE_SHIFT == 13) { 1281 build_tlb_write_entry(p, l, r, tlb_random); 1282 uasm_l_leave(l, *p); 1283 UASM_i_LW(p, scratch, scratchpad_offset(0), 0); 1284 } else { 1285 UASM_i_LW(p, scratch, scratchpad_offset(0), 0); 1286 build_tlb_write_entry(p, l, r, tlb_random); 1287 uasm_l_leave(l, *p); 1288 rv.restore_scratch = 1; 1289 } 1290 1291 uasm_i_eret(p); /* return from trap */ 1292 1293 return rv; 1294 } 1295 1296 /* 1297 * For a 64-bit kernel, we are using the 64-bit XTLB refill exception 1298 * because EXL == 0. If we wrap, we can also use the 32 instruction 1299 * slots before the XTLB refill exception handler which belong to the 1300 * unused TLB refill exception. 1301 */ 1302 #define MIPS64_REFILL_INSNS 32 1303 1304 static void build_r4000_tlb_refill_handler(void) 1305 { 1306 u32 *p = tlb_handler; 1307 struct uasm_label *l = labels; 1308 struct uasm_reloc *r = relocs; 1309 u32 *f; 1310 unsigned int final_len; 1311 struct mips_huge_tlb_info htlb_info __maybe_unused; 1312 enum vmalloc64_mode vmalloc_mode __maybe_unused; 1313 1314 memset(tlb_handler, 0, sizeof(tlb_handler)); 1315 memset(labels, 0, sizeof(labels)); 1316 memset(relocs, 0, sizeof(relocs)); 1317 memset(final_handler, 0, sizeof(final_handler)); 1318 1319 if (IS_ENABLED(CONFIG_64BIT) && (scratch_reg >= 0 || scratchpad_available()) && use_bbit_insns()) { 1320 htlb_info = build_fast_tlb_refill_handler(&p, &l, &r, K0, K1, 1321 scratch_reg); 1322 vmalloc_mode = refill_scratch; 1323 } else { 1324 htlb_info.huge_pte = K0; 1325 htlb_info.restore_scratch = 0; 1326 htlb_info.need_reload_pte = true; 1327 vmalloc_mode = refill_noscratch; 1328 /* 1329 * create the plain linear handler 1330 */ 1331 if (bcm1250_m3_war()) { 1332 unsigned int segbits = 44; 1333 1334 uasm_i_dmfc0(&p, K0, C0_BADVADDR); 1335 uasm_i_dmfc0(&p, K1, C0_ENTRYHI); 1336 uasm_i_xor(&p, K0, K0, K1); 1337 uasm_i_dsrl_safe(&p, K1, K0, 62); 1338 uasm_i_dsrl_safe(&p, K0, K0, 12 + 1); 1339 uasm_i_dsll_safe(&p, K0, K0, 64 + 12 + 1 - segbits); 1340 uasm_i_or(&p, K0, K0, K1); 1341 uasm_il_bnez(&p, &r, K0, label_leave); 1342 /* No need for uasm_i_nop */ 1343 } 1344 1345 #ifdef CONFIG_64BIT 1346 build_get_pmde64(&p, &l, &r, K0, K1); /* get pmd in K1 */ 1347 #else 1348 build_get_pgde32(&p, K0, K1); /* get pgd in K1 */ 1349 #endif 1350 1351 #ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT 1352 build_is_huge_pte(&p, &r, K0, K1, label_tlb_huge_update); 1353 #endif 1354 1355 build_get_ptep(&p, K0, K1); 1356 build_update_entries(&p, K0, K1); 1357 build_tlb_write_entry(&p, &l, &r, tlb_random); 1358 uasm_l_leave(&l, p); 1359 uasm_i_eret(&p); /* return from trap */ 1360 } 1361 #ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT 1362 uasm_l_tlb_huge_update(&l, p); 1363 if (htlb_info.need_reload_pte) 1364 UASM_i_LW(&p, htlb_info.huge_pte, 0, K1); 1365 build_huge_update_entries(&p, htlb_info.huge_pte, K1); 1366 build_huge_tlb_write_entry(&p, &l, &r, K0, tlb_random, 1367 htlb_info.restore_scratch); 1368 #endif 1369 1370 #ifdef CONFIG_64BIT 1371 build_get_pgd_vmalloc64(&p, &l, &r, K0, K1, vmalloc_mode); 1372 #endif 1373 1374 /* 1375 * Overflow check: For the 64bit handler, we need at least one 1376 * free instruction slot for the wrap-around branch. In worst 1377 * case, if the intended insertion point is a delay slot, we 1378 * need three, with the second nop'ed and the third being 1379 * unused. 1380 */ 1381 switch (boot_cpu_type()) { 1382 default: 1383 if (sizeof(long) == 4) { 1384 case CPU_LOONGSON2EF: 1385 /* Loongson2 ebase is different than r4k, we have more space */ 1386 if ((p - tlb_handler) > 64) 1387 panic("TLB refill handler space exceeded"); 1388 /* 1389 * Now fold the handler in the TLB refill handler space. 1390 */ 1391 f = final_handler; 1392 /* Simplest case, just copy the handler. */ 1393 uasm_copy_handler(relocs, labels, tlb_handler, p, f); 1394 final_len = p - tlb_handler; 1395 break; 1396 } else { 1397 if (((p - tlb_handler) > (MIPS64_REFILL_INSNS * 2) - 1) 1398 || (((p - tlb_handler) > (MIPS64_REFILL_INSNS * 2) - 3) 1399 && uasm_insn_has_bdelay(relocs, 1400 tlb_handler + MIPS64_REFILL_INSNS - 3))) 1401 panic("TLB refill handler space exceeded"); 1402 /* 1403 * Now fold the handler in the TLB refill handler space. 1404 */ 1405 f = final_handler + MIPS64_REFILL_INSNS; 1406 if ((p - tlb_handler) <= MIPS64_REFILL_INSNS) { 1407 /* Just copy the handler. */ 1408 uasm_copy_handler(relocs, labels, tlb_handler, p, f); 1409 final_len = p - tlb_handler; 1410 } else { 1411 #ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT 1412 const enum label_id ls = label_tlb_huge_update; 1413 #else 1414 const enum label_id ls = label_vmalloc; 1415 #endif 1416 u32 *split; 1417 int ov = 0; 1418 int i; 1419 1420 for (i = 0; i < ARRAY_SIZE(labels) && labels[i].lab != ls; i++) 1421 ; 1422 BUG_ON(i == ARRAY_SIZE(labels)); 1423 split = labels[i].addr; 1424 1425 /* 1426 * See if we have overflown one way or the other. 1427 */ 1428 if (split > tlb_handler + MIPS64_REFILL_INSNS || 1429 split < p - MIPS64_REFILL_INSNS) 1430 ov = 1; 1431 1432 if (ov) { 1433 /* 1434 * Split two instructions before the end. One 1435 * for the branch and one for the instruction 1436 * in the delay slot. 1437 */ 1438 split = tlb_handler + MIPS64_REFILL_INSNS - 2; 1439 1440 /* 1441 * If the branch would fall in a delay slot, 1442 * we must back up an additional instruction 1443 * so that it is no longer in a delay slot. 1444 */ 1445 if (uasm_insn_has_bdelay(relocs, split - 1)) 1446 split--; 1447 } 1448 /* Copy first part of the handler. */ 1449 uasm_copy_handler(relocs, labels, tlb_handler, split, f); 1450 f += split - tlb_handler; 1451 1452 if (ov) { 1453 /* Insert branch. */ 1454 uasm_l_split(&l, final_handler); 1455 uasm_il_b(&f, &r, label_split); 1456 if (uasm_insn_has_bdelay(relocs, split)) 1457 uasm_i_nop(&f); 1458 else { 1459 uasm_copy_handler(relocs, labels, 1460 split, split + 1, f); 1461 uasm_move_labels(labels, f, f + 1, -1); 1462 f++; 1463 split++; 1464 } 1465 } 1466 1467 /* Copy the rest of the handler. */ 1468 uasm_copy_handler(relocs, labels, split, p, final_handler); 1469 final_len = (f - (final_handler + MIPS64_REFILL_INSNS)) + 1470 (p - split); 1471 } 1472 } 1473 break; 1474 } 1475 1476 uasm_resolve_relocs(relocs, labels); 1477 pr_debug("Wrote TLB refill handler (%u instructions).\n", 1478 final_len); 1479 1480 memcpy((void *)ebase, final_handler, 0x100); 1481 local_flush_icache_range(ebase, ebase + 0x100); 1482 dump_handler("r4000_tlb_refill", (u32 *)ebase, (u32 *)(ebase + 0x100)); 1483 } 1484 1485 static void setup_pw(void) 1486 { 1487 unsigned int pwctl; 1488 unsigned long pgd_i, pgd_w; 1489 #ifndef __PAGETABLE_PMD_FOLDED 1490 unsigned long pmd_i, pmd_w; 1491 #endif 1492 unsigned long pt_i, pt_w; 1493 unsigned long pte_i, pte_w; 1494 #ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT 1495 unsigned long psn; 1496 1497 psn = ilog2(_PAGE_HUGE); /* bit used to indicate huge page */ 1498 #endif 1499 pgd_i = PGDIR_SHIFT; /* 1st level PGD */ 1500 #ifndef __PAGETABLE_PMD_FOLDED 1501 pgd_w = PGDIR_SHIFT - PMD_SHIFT + PGD_ORDER; 1502 1503 pmd_i = PMD_SHIFT; /* 2nd level PMD */ 1504 pmd_w = PMD_SHIFT - PAGE_SHIFT; 1505 #else 1506 pgd_w = PGDIR_SHIFT - PAGE_SHIFT + PGD_ORDER; 1507 #endif 1508 1509 pt_i = PAGE_SHIFT; /* 3rd level PTE */ 1510 pt_w = PAGE_SHIFT - 3; 1511 1512 pte_i = ilog2(_PAGE_GLOBAL); 1513 pte_w = 0; 1514 pwctl = 1 << 30; /* Set PWDirExt */ 1515 1516 #ifndef __PAGETABLE_PMD_FOLDED 1517 write_c0_pwfield(pgd_i << 24 | pmd_i << 12 | pt_i << 6 | pte_i); 1518 write_c0_pwsize(1 << 30 | pgd_w << 24 | pmd_w << 12 | pt_w << 6 | pte_w); 1519 #else 1520 write_c0_pwfield(pgd_i << 24 | pt_i << 6 | pte_i); 1521 write_c0_pwsize(1 << 30 | pgd_w << 24 | pt_w << 6 | pte_w); 1522 #endif 1523 1524 #ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT 1525 pwctl |= (1 << 6 | psn); 1526 #endif 1527 write_c0_pwctl(pwctl); 1528 write_c0_kpgd((long)swapper_pg_dir); 1529 kscratch_used_mask |= (1 << 7); /* KScratch6 is used for KPGD */ 1530 } 1531 1532 static void build_loongson3_tlb_refill_handler(void) 1533 { 1534 u32 *p = tlb_handler; 1535 struct uasm_label *l = labels; 1536 struct uasm_reloc *r = relocs; 1537 1538 memset(labels, 0, sizeof(labels)); 1539 memset(relocs, 0, sizeof(relocs)); 1540 memset(tlb_handler, 0, sizeof(tlb_handler)); 1541 1542 if (check_for_high_segbits) { 1543 uasm_i_dmfc0(&p, K0, C0_BADVADDR); 1544 uasm_i_dsrl_safe(&p, K1, K0, PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3); 1545 uasm_il_beqz(&p, &r, K1, label_vmalloc); 1546 uasm_i_nop(&p); 1547 1548 uasm_il_bgez(&p, &r, K0, label_large_segbits_fault); 1549 uasm_i_nop(&p); 1550 uasm_l_vmalloc(&l, p); 1551 } 1552 1553 uasm_i_dmfc0(&p, K1, C0_PGD); 1554 1555 uasm_i_lddir(&p, K0, K1, 3); /* global page dir */ 1556 #ifndef __PAGETABLE_PMD_FOLDED 1557 uasm_i_lddir(&p, K1, K0, 1); /* middle page dir */ 1558 #endif 1559 uasm_i_ldpte(&p, K1, 0); /* even */ 1560 uasm_i_ldpte(&p, K1, 1); /* odd */ 1561 uasm_i_tlbwr(&p); 1562 1563 /* restore page mask */ 1564 if (PM_DEFAULT_MASK >> 16) { 1565 uasm_i_lui(&p, K0, PM_DEFAULT_MASK >> 16); 1566 uasm_i_ori(&p, K0, K0, PM_DEFAULT_MASK & 0xffff); 1567 uasm_i_mtc0(&p, K0, C0_PAGEMASK); 1568 } else if (PM_DEFAULT_MASK) { 1569 uasm_i_ori(&p, K0, 0, PM_DEFAULT_MASK); 1570 uasm_i_mtc0(&p, K0, C0_PAGEMASK); 1571 } else { 1572 uasm_i_mtc0(&p, 0, C0_PAGEMASK); 1573 } 1574 1575 uasm_i_eret(&p); 1576 1577 if (check_for_high_segbits) { 1578 uasm_l_large_segbits_fault(&l, p); 1579 UASM_i_LA(&p, K1, (unsigned long)tlb_do_page_fault_0); 1580 uasm_i_jr(&p, K1); 1581 uasm_i_nop(&p); 1582 } 1583 1584 uasm_resolve_relocs(relocs, labels); 1585 memcpy((void *)(ebase + 0x80), tlb_handler, 0x80); 1586 local_flush_icache_range(ebase + 0x80, ebase + 0x100); 1587 dump_handler("loongson3_tlb_refill", 1588 (u32 *)(ebase + 0x80), (u32 *)(ebase + 0x100)); 1589 } 1590 1591 static void build_setup_pgd(void) 1592 { 1593 const int a0 = 4; 1594 const int __maybe_unused a1 = 5; 1595 const int __maybe_unused a2 = 6; 1596 u32 *p = (u32 *)msk_isa16_mode((ulong)tlbmiss_handler_setup_pgd); 1597 #ifndef CONFIG_MIPS_PGD_C0_CONTEXT 1598 long pgdc = (long)pgd_current; 1599 #endif 1600 1601 memset(p, 0, tlbmiss_handler_setup_pgd_end - (char *)p); 1602 memset(labels, 0, sizeof(labels)); 1603 memset(relocs, 0, sizeof(relocs)); 1604 pgd_reg = allocate_kscratch(); 1605 #ifdef CONFIG_MIPS_PGD_C0_CONTEXT 1606 if (pgd_reg == -1) { 1607 struct uasm_label *l = labels; 1608 struct uasm_reloc *r = relocs; 1609 1610 /* PGD << 11 in c0_Context */ 1611 /* 1612 * If it is a ckseg0 address, convert to a physical 1613 * address. Shifting right by 29 and adding 4 will 1614 * result in zero for these addresses. 1615 * 1616 */ 1617 UASM_i_SRA(&p, a1, a0, 29); 1618 UASM_i_ADDIU(&p, a1, a1, 4); 1619 uasm_il_bnez(&p, &r, a1, label_tlbl_goaround1); 1620 uasm_i_nop(&p); 1621 uasm_i_dinsm(&p, a0, 0, 29, 64 - 29); 1622 uasm_l_tlbl_goaround1(&l, p); 1623 UASM_i_SLL(&p, a0, a0, 11); 1624 UASM_i_MTC0(&p, a0, C0_CONTEXT); 1625 uasm_i_jr(&p, 31); 1626 uasm_i_ehb(&p); 1627 } else { 1628 /* PGD in c0_KScratch */ 1629 if (cpu_has_ldpte) 1630 UASM_i_MTC0(&p, a0, C0_PWBASE); 1631 else 1632 UASM_i_MTC0(&p, a0, c0_kscratch(), pgd_reg); 1633 uasm_i_jr(&p, 31); 1634 uasm_i_ehb(&p); 1635 } 1636 #else 1637 #ifdef CONFIG_SMP 1638 /* Save PGD to pgd_current[smp_processor_id()] */ 1639 UASM_i_CPUID_MFC0(&p, a1, SMP_CPUID_REG); 1640 UASM_i_SRL_SAFE(&p, a1, a1, SMP_CPUID_PTRSHIFT); 1641 UASM_i_LA_mostly(&p, a2, pgdc); 1642 UASM_i_ADDU(&p, a2, a2, a1); 1643 UASM_i_SW(&p, a0, uasm_rel_lo(pgdc), a2); 1644 #else 1645 UASM_i_LA_mostly(&p, a2, pgdc); 1646 UASM_i_SW(&p, a0, uasm_rel_lo(pgdc), a2); 1647 #endif /* SMP */ 1648 1649 /* if pgd_reg is allocated, save PGD also to scratch register */ 1650 if (pgd_reg != -1) { 1651 UASM_i_MTC0(&p, a0, c0_kscratch(), pgd_reg); 1652 uasm_i_jr(&p, 31); 1653 uasm_i_ehb(&p); 1654 } else { 1655 uasm_i_jr(&p, 31); 1656 uasm_i_nop(&p); 1657 } 1658 #endif 1659 if (p >= (u32 *)tlbmiss_handler_setup_pgd_end) 1660 panic("tlbmiss_handler_setup_pgd space exceeded"); 1661 1662 uasm_resolve_relocs(relocs, labels); 1663 pr_debug("Wrote tlbmiss_handler_setup_pgd (%u instructions).\n", 1664 (unsigned int)(p - (u32 *)tlbmiss_handler_setup_pgd)); 1665 1666 dump_handler("tlbmiss_handler", tlbmiss_handler_setup_pgd, 1667 tlbmiss_handler_setup_pgd_end); 1668 } 1669 1670 static void 1671 iPTE_LW(u32 **p, unsigned int pte, unsigned int ptr) 1672 { 1673 #ifdef CONFIG_SMP 1674 if (IS_ENABLED(CONFIG_CPU_LOONGSON3_WORKAROUNDS)) 1675 uasm_i_sync(p, 0); 1676 # ifdef CONFIG_PHYS_ADDR_T_64BIT 1677 if (cpu_has_64bits) 1678 uasm_i_lld(p, pte, 0, ptr); 1679 else 1680 # endif 1681 UASM_i_LL(p, pte, 0, ptr); 1682 #else 1683 # ifdef CONFIG_PHYS_ADDR_T_64BIT 1684 if (cpu_has_64bits) 1685 uasm_i_ld(p, pte, 0, ptr); 1686 else 1687 # endif 1688 UASM_i_LW(p, pte, 0, ptr); 1689 #endif 1690 } 1691 1692 static void 1693 iPTE_SW(u32 **p, struct uasm_reloc **r, unsigned int pte, unsigned int ptr, 1694 unsigned int mode, unsigned int scratch) 1695 { 1696 unsigned int hwmode = mode & (_PAGE_VALID | _PAGE_DIRTY); 1697 unsigned int swmode = mode & ~hwmode; 1698 1699 if (IS_ENABLED(CONFIG_XPA) && !cpu_has_64bits) { 1700 uasm_i_lui(p, scratch, swmode >> 16); 1701 uasm_i_or(p, pte, pte, scratch); 1702 BUG_ON(swmode & 0xffff); 1703 } else { 1704 uasm_i_ori(p, pte, pte, mode); 1705 } 1706 1707 #ifdef CONFIG_SMP 1708 # ifdef CONFIG_PHYS_ADDR_T_64BIT 1709 if (cpu_has_64bits) 1710 uasm_i_scd(p, pte, 0, ptr); 1711 else 1712 # endif 1713 UASM_i_SC(p, pte, 0, ptr); 1714 1715 if (r10000_llsc_war()) 1716 uasm_il_beqzl(p, r, pte, label_smp_pgtable_change); 1717 else 1718 uasm_il_beqz(p, r, pte, label_smp_pgtable_change); 1719 1720 # ifdef CONFIG_PHYS_ADDR_T_64BIT 1721 if (!cpu_has_64bits) { 1722 /* no uasm_i_nop needed */ 1723 uasm_i_ll(p, pte, sizeof(pte_t) / 2, ptr); 1724 uasm_i_ori(p, pte, pte, hwmode); 1725 BUG_ON(hwmode & ~0xffff); 1726 uasm_i_sc(p, pte, sizeof(pte_t) / 2, ptr); 1727 uasm_il_beqz(p, r, pte, label_smp_pgtable_change); 1728 /* no uasm_i_nop needed */ 1729 uasm_i_lw(p, pte, 0, ptr); 1730 } else 1731 uasm_i_nop(p); 1732 # else 1733 uasm_i_nop(p); 1734 # endif 1735 #else 1736 # ifdef CONFIG_PHYS_ADDR_T_64BIT 1737 if (cpu_has_64bits) 1738 uasm_i_sd(p, pte, 0, ptr); 1739 else 1740 # endif 1741 UASM_i_SW(p, pte, 0, ptr); 1742 1743 # ifdef CONFIG_PHYS_ADDR_T_64BIT 1744 if (!cpu_has_64bits) { 1745 uasm_i_lw(p, pte, sizeof(pte_t) / 2, ptr); 1746 uasm_i_ori(p, pte, pte, hwmode); 1747 BUG_ON(hwmode & ~0xffff); 1748 uasm_i_sw(p, pte, sizeof(pte_t) / 2, ptr); 1749 uasm_i_lw(p, pte, 0, ptr); 1750 } 1751 # endif 1752 #endif 1753 } 1754 1755 /* 1756 * Check if PTE is present, if not then jump to LABEL. PTR points to 1757 * the page table where this PTE is located, PTE will be re-loaded 1758 * with it's original value. 1759 */ 1760 static void 1761 build_pte_present(u32 **p, struct uasm_reloc **r, 1762 int pte, int ptr, int scratch, enum label_id lid) 1763 { 1764 int t = scratch >= 0 ? scratch : pte; 1765 int cur = pte; 1766 1767 if (cpu_has_rixi) { 1768 if (use_bbit_insns()) { 1769 uasm_il_bbit0(p, r, pte, ilog2(_PAGE_PRESENT), lid); 1770 uasm_i_nop(p); 1771 } else { 1772 if (_PAGE_PRESENT_SHIFT) { 1773 uasm_i_srl(p, t, cur, _PAGE_PRESENT_SHIFT); 1774 cur = t; 1775 } 1776 uasm_i_andi(p, t, cur, 1); 1777 uasm_il_beqz(p, r, t, lid); 1778 if (pte == t) 1779 /* You lose the SMP race :-(*/ 1780 iPTE_LW(p, pte, ptr); 1781 } 1782 } else { 1783 if (_PAGE_PRESENT_SHIFT) { 1784 uasm_i_srl(p, t, cur, _PAGE_PRESENT_SHIFT); 1785 cur = t; 1786 } 1787 uasm_i_andi(p, t, cur, 1788 (_PAGE_PRESENT | _PAGE_NO_READ) >> _PAGE_PRESENT_SHIFT); 1789 uasm_i_xori(p, t, t, _PAGE_PRESENT >> _PAGE_PRESENT_SHIFT); 1790 uasm_il_bnez(p, r, t, lid); 1791 if (pte == t) 1792 /* You lose the SMP race :-(*/ 1793 iPTE_LW(p, pte, ptr); 1794 } 1795 } 1796 1797 /* Make PTE valid, store result in PTR. */ 1798 static void 1799 build_make_valid(u32 **p, struct uasm_reloc **r, unsigned int pte, 1800 unsigned int ptr, unsigned int scratch) 1801 { 1802 unsigned int mode = _PAGE_VALID | _PAGE_ACCESSED; 1803 1804 iPTE_SW(p, r, pte, ptr, mode, scratch); 1805 } 1806 1807 /* 1808 * Check if PTE can be written to, if not branch to LABEL. Regardless 1809 * restore PTE with value from PTR when done. 1810 */ 1811 static void 1812 build_pte_writable(u32 **p, struct uasm_reloc **r, 1813 unsigned int pte, unsigned int ptr, int scratch, 1814 enum label_id lid) 1815 { 1816 int t = scratch >= 0 ? scratch : pte; 1817 int cur = pte; 1818 1819 if (_PAGE_PRESENT_SHIFT) { 1820 uasm_i_srl(p, t, cur, _PAGE_PRESENT_SHIFT); 1821 cur = t; 1822 } 1823 uasm_i_andi(p, t, cur, 1824 (_PAGE_PRESENT | _PAGE_WRITE) >> _PAGE_PRESENT_SHIFT); 1825 uasm_i_xori(p, t, t, 1826 (_PAGE_PRESENT | _PAGE_WRITE) >> _PAGE_PRESENT_SHIFT); 1827 uasm_il_bnez(p, r, t, lid); 1828 if (pte == t) 1829 /* You lose the SMP race :-(*/ 1830 iPTE_LW(p, pte, ptr); 1831 else 1832 uasm_i_nop(p); 1833 } 1834 1835 /* Make PTE writable, update software status bits as well, then store 1836 * at PTR. 1837 */ 1838 static void 1839 build_make_write(u32 **p, struct uasm_reloc **r, unsigned int pte, 1840 unsigned int ptr, unsigned int scratch) 1841 { 1842 unsigned int mode = (_PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID 1843 | _PAGE_DIRTY); 1844 1845 iPTE_SW(p, r, pte, ptr, mode, scratch); 1846 } 1847 1848 /* 1849 * Check if PTE can be modified, if not branch to LABEL. Regardless 1850 * restore PTE with value from PTR when done. 1851 */ 1852 static void 1853 build_pte_modifiable(u32 **p, struct uasm_reloc **r, 1854 unsigned int pte, unsigned int ptr, int scratch, 1855 enum label_id lid) 1856 { 1857 if (use_bbit_insns()) { 1858 uasm_il_bbit0(p, r, pte, ilog2(_PAGE_WRITE), lid); 1859 uasm_i_nop(p); 1860 } else { 1861 int t = scratch >= 0 ? scratch : pte; 1862 uasm_i_srl(p, t, pte, _PAGE_WRITE_SHIFT); 1863 uasm_i_andi(p, t, t, 1); 1864 uasm_il_beqz(p, r, t, lid); 1865 if (pte == t) 1866 /* You lose the SMP race :-(*/ 1867 iPTE_LW(p, pte, ptr); 1868 } 1869 } 1870 1871 #ifndef CONFIG_MIPS_PGD_C0_CONTEXT 1872 1873 1874 /* 1875 * R3000 style TLB load/store/modify handlers. 1876 */ 1877 1878 /* 1879 * This places the pte into ENTRYLO0 and writes it with tlbwi. 1880 * Then it returns. 1881 */ 1882 static void 1883 build_r3000_pte_reload_tlbwi(u32 **p, unsigned int pte, unsigned int tmp) 1884 { 1885 uasm_i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */ 1886 uasm_i_mfc0(p, tmp, C0_EPC); /* cp0 delay */ 1887 uasm_i_tlbwi(p); 1888 uasm_i_jr(p, tmp); 1889 uasm_i_rfe(p); /* branch delay */ 1890 } 1891 1892 /* 1893 * This places the pte into ENTRYLO0 and writes it with tlbwi 1894 * or tlbwr as appropriate. This is because the index register 1895 * may have the probe fail bit set as a result of a trap on a 1896 * kseg2 access, i.e. without refill. Then it returns. 1897 */ 1898 static void 1899 build_r3000_tlb_reload_write(u32 **p, struct uasm_label **l, 1900 struct uasm_reloc **r, unsigned int pte, 1901 unsigned int tmp) 1902 { 1903 uasm_i_mfc0(p, tmp, C0_INDEX); 1904 uasm_i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */ 1905 uasm_il_bltz(p, r, tmp, label_r3000_write_probe_fail); /* cp0 delay */ 1906 uasm_i_mfc0(p, tmp, C0_EPC); /* branch delay */ 1907 uasm_i_tlbwi(p); /* cp0 delay */ 1908 uasm_i_jr(p, tmp); 1909 uasm_i_rfe(p); /* branch delay */ 1910 uasm_l_r3000_write_probe_fail(l, *p); 1911 uasm_i_tlbwr(p); /* cp0 delay */ 1912 uasm_i_jr(p, tmp); 1913 uasm_i_rfe(p); /* branch delay */ 1914 } 1915 1916 static void 1917 build_r3000_tlbchange_handler_head(u32 **p, unsigned int pte, 1918 unsigned int ptr) 1919 { 1920 long pgdc = (long)pgd_current; 1921 1922 uasm_i_mfc0(p, pte, C0_BADVADDR); 1923 uasm_i_lui(p, ptr, uasm_rel_hi(pgdc)); /* cp0 delay */ 1924 uasm_i_lw(p, ptr, uasm_rel_lo(pgdc), ptr); 1925 uasm_i_srl(p, pte, pte, 22); /* load delay */ 1926 uasm_i_sll(p, pte, pte, 2); 1927 uasm_i_addu(p, ptr, ptr, pte); 1928 uasm_i_mfc0(p, pte, C0_CONTEXT); 1929 uasm_i_lw(p, ptr, 0, ptr); /* cp0 delay */ 1930 uasm_i_andi(p, pte, pte, 0xffc); /* load delay */ 1931 uasm_i_addu(p, ptr, ptr, pte); 1932 uasm_i_lw(p, pte, 0, ptr); 1933 uasm_i_tlbp(p); /* load delay */ 1934 } 1935 1936 static void build_r3000_tlb_load_handler(void) 1937 { 1938 u32 *p = (u32 *)handle_tlbl; 1939 struct uasm_label *l = labels; 1940 struct uasm_reloc *r = relocs; 1941 1942 memset(p, 0, handle_tlbl_end - (char *)p); 1943 memset(labels, 0, sizeof(labels)); 1944 memset(relocs, 0, sizeof(relocs)); 1945 1946 build_r3000_tlbchange_handler_head(&p, K0, K1); 1947 build_pte_present(&p, &r, K0, K1, -1, label_nopage_tlbl); 1948 uasm_i_nop(&p); /* load delay */ 1949 build_make_valid(&p, &r, K0, K1, -1); 1950 build_r3000_tlb_reload_write(&p, &l, &r, K0, K1); 1951 1952 uasm_l_nopage_tlbl(&l, p); 1953 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff); 1954 uasm_i_nop(&p); 1955 1956 if (p >= (u32 *)handle_tlbl_end) 1957 panic("TLB load handler fastpath space exceeded"); 1958 1959 uasm_resolve_relocs(relocs, labels); 1960 pr_debug("Wrote TLB load handler fastpath (%u instructions).\n", 1961 (unsigned int)(p - (u32 *)handle_tlbl)); 1962 1963 dump_handler("r3000_tlb_load", handle_tlbl, handle_tlbl_end); 1964 } 1965 1966 static void build_r3000_tlb_store_handler(void) 1967 { 1968 u32 *p = (u32 *)handle_tlbs; 1969 struct uasm_label *l = labels; 1970 struct uasm_reloc *r = relocs; 1971 1972 memset(p, 0, handle_tlbs_end - (char *)p); 1973 memset(labels, 0, sizeof(labels)); 1974 memset(relocs, 0, sizeof(relocs)); 1975 1976 build_r3000_tlbchange_handler_head(&p, K0, K1); 1977 build_pte_writable(&p, &r, K0, K1, -1, label_nopage_tlbs); 1978 uasm_i_nop(&p); /* load delay */ 1979 build_make_write(&p, &r, K0, K1, -1); 1980 build_r3000_tlb_reload_write(&p, &l, &r, K0, K1); 1981 1982 uasm_l_nopage_tlbs(&l, p); 1983 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); 1984 uasm_i_nop(&p); 1985 1986 if (p >= (u32 *)handle_tlbs_end) 1987 panic("TLB store handler fastpath space exceeded"); 1988 1989 uasm_resolve_relocs(relocs, labels); 1990 pr_debug("Wrote TLB store handler fastpath (%u instructions).\n", 1991 (unsigned int)(p - (u32 *)handle_tlbs)); 1992 1993 dump_handler("r3000_tlb_store", handle_tlbs, handle_tlbs_end); 1994 } 1995 1996 static void build_r3000_tlb_modify_handler(void) 1997 { 1998 u32 *p = (u32 *)handle_tlbm; 1999 struct uasm_label *l = labels; 2000 struct uasm_reloc *r = relocs; 2001 2002 memset(p, 0, handle_tlbm_end - (char *)p); 2003 memset(labels, 0, sizeof(labels)); 2004 memset(relocs, 0, sizeof(relocs)); 2005 2006 build_r3000_tlbchange_handler_head(&p, K0, K1); 2007 build_pte_modifiable(&p, &r, K0, K1, -1, label_nopage_tlbm); 2008 uasm_i_nop(&p); /* load delay */ 2009 build_make_write(&p, &r, K0, K1, -1); 2010 build_r3000_pte_reload_tlbwi(&p, K0, K1); 2011 2012 uasm_l_nopage_tlbm(&l, p); 2013 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); 2014 uasm_i_nop(&p); 2015 2016 if (p >= (u32 *)handle_tlbm_end) 2017 panic("TLB modify handler fastpath space exceeded"); 2018 2019 uasm_resolve_relocs(relocs, labels); 2020 pr_debug("Wrote TLB modify handler fastpath (%u instructions).\n", 2021 (unsigned int)(p - (u32 *)handle_tlbm)); 2022 2023 dump_handler("r3000_tlb_modify", handle_tlbm, handle_tlbm_end); 2024 } 2025 #endif /* CONFIG_MIPS_PGD_C0_CONTEXT */ 2026 2027 static bool cpu_has_tlbex_tlbp_race(void) 2028 { 2029 /* 2030 * When a Hardware Table Walker is running it can replace TLB entries 2031 * at any time, leading to a race between it & the CPU. 2032 */ 2033 if (cpu_has_htw) 2034 return true; 2035 2036 /* 2037 * If the CPU shares FTLB RAM with its siblings then our entry may be 2038 * replaced at any time by a sibling performing a write to the FTLB. 2039 */ 2040 if (cpu_has_shared_ftlb_ram) 2041 return true; 2042 2043 /* In all other cases there ought to be no race condition to handle */ 2044 return false; 2045 } 2046 2047 /* 2048 * R4000 style TLB load/store/modify handlers. 2049 */ 2050 static struct work_registers 2051 build_r4000_tlbchange_handler_head(u32 **p, struct uasm_label **l, 2052 struct uasm_reloc **r) 2053 { 2054 struct work_registers wr = build_get_work_registers(p); 2055 2056 #ifdef CONFIG_64BIT 2057 build_get_pmde64(p, l, r, wr.r1, wr.r2); /* get pmd in ptr */ 2058 #else 2059 build_get_pgde32(p, wr.r1, wr.r2); /* get pgd in ptr */ 2060 #endif 2061 2062 #ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT 2063 /* 2064 * For huge tlb entries, pmd doesn't contain an address but 2065 * instead contains the tlb pte. Check the PAGE_HUGE bit and 2066 * see if we need to jump to huge tlb processing. 2067 */ 2068 build_is_huge_pte(p, r, wr.r1, wr.r2, label_tlb_huge_update); 2069 #endif 2070 2071 UASM_i_MFC0(p, wr.r1, C0_BADVADDR); 2072 UASM_i_LW(p, wr.r2, 0, wr.r2); 2073 UASM_i_SRL(p, wr.r1, wr.r1, PAGE_SHIFT + PTE_ORDER - PTE_T_LOG2); 2074 uasm_i_andi(p, wr.r1, wr.r1, (PTRS_PER_PTE - 1) << PTE_T_LOG2); 2075 UASM_i_ADDU(p, wr.r2, wr.r2, wr.r1); 2076 2077 #ifdef CONFIG_SMP 2078 uasm_l_smp_pgtable_change(l, *p); 2079 #endif 2080 iPTE_LW(p, wr.r1, wr.r2); /* get even pte */ 2081 if (!m4kc_tlbp_war()) { 2082 build_tlb_probe_entry(p); 2083 if (cpu_has_tlbex_tlbp_race()) { 2084 /* race condition happens, leaving */ 2085 uasm_i_ehb(p); 2086 uasm_i_mfc0(p, wr.r3, C0_INDEX); 2087 uasm_il_bltz(p, r, wr.r3, label_leave); 2088 uasm_i_nop(p); 2089 } 2090 } 2091 return wr; 2092 } 2093 2094 static void 2095 build_r4000_tlbchange_handler_tail(u32 **p, struct uasm_label **l, 2096 struct uasm_reloc **r, unsigned int tmp, 2097 unsigned int ptr) 2098 { 2099 uasm_i_ori(p, ptr, ptr, sizeof(pte_t)); 2100 uasm_i_xori(p, ptr, ptr, sizeof(pte_t)); 2101 build_update_entries(p, tmp, ptr); 2102 build_tlb_write_entry(p, l, r, tlb_indexed); 2103 uasm_l_leave(l, *p); 2104 build_restore_work_registers(p); 2105 uasm_i_eret(p); /* return from trap */ 2106 2107 #ifdef CONFIG_64BIT 2108 build_get_pgd_vmalloc64(p, l, r, tmp, ptr, not_refill); 2109 #endif 2110 } 2111 2112 static void build_r4000_tlb_load_handler(void) 2113 { 2114 u32 *p = (u32 *)msk_isa16_mode((ulong)handle_tlbl); 2115 struct uasm_label *l = labels; 2116 struct uasm_reloc *r = relocs; 2117 struct work_registers wr; 2118 2119 memset(p, 0, handle_tlbl_end - (char *)p); 2120 memset(labels, 0, sizeof(labels)); 2121 memset(relocs, 0, sizeof(relocs)); 2122 2123 if (bcm1250_m3_war()) { 2124 unsigned int segbits = 44; 2125 2126 uasm_i_dmfc0(&p, K0, C0_BADVADDR); 2127 uasm_i_dmfc0(&p, K1, C0_ENTRYHI); 2128 uasm_i_xor(&p, K0, K0, K1); 2129 uasm_i_dsrl_safe(&p, K1, K0, 62); 2130 uasm_i_dsrl_safe(&p, K0, K0, 12 + 1); 2131 uasm_i_dsll_safe(&p, K0, K0, 64 + 12 + 1 - segbits); 2132 uasm_i_or(&p, K0, K0, K1); 2133 uasm_il_bnez(&p, &r, K0, label_leave); 2134 /* No need for uasm_i_nop */ 2135 } 2136 2137 wr = build_r4000_tlbchange_handler_head(&p, &l, &r); 2138 build_pte_present(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbl); 2139 if (m4kc_tlbp_war()) 2140 build_tlb_probe_entry(&p); 2141 2142 if (cpu_has_rixi && !cpu_has_rixiex) { 2143 /* 2144 * If the page is not _PAGE_VALID, RI or XI could not 2145 * have triggered it. Skip the expensive test.. 2146 */ 2147 if (use_bbit_insns()) { 2148 uasm_il_bbit0(&p, &r, wr.r1, ilog2(_PAGE_VALID), 2149 label_tlbl_goaround1); 2150 } else { 2151 uasm_i_andi(&p, wr.r3, wr.r1, _PAGE_VALID); 2152 uasm_il_beqz(&p, &r, wr.r3, label_tlbl_goaround1); 2153 } 2154 uasm_i_nop(&p); 2155 2156 /* 2157 * Warn if something may race with us & replace the TLB entry 2158 * before we read it here. Everything with such races should 2159 * also have dedicated RiXi exception handlers, so this 2160 * shouldn't be hit. 2161 */ 2162 WARN(cpu_has_tlbex_tlbp_race(), "Unhandled race in RiXi path"); 2163 2164 uasm_i_tlbr(&p); 2165 2166 switch (current_cpu_type()) { 2167 default: 2168 if (cpu_has_mips_r2_exec_hazard) { 2169 uasm_i_ehb(&p); 2170 2171 case CPU_CAVIUM_OCTEON: 2172 case CPU_CAVIUM_OCTEON_PLUS: 2173 case CPU_CAVIUM_OCTEON2: 2174 break; 2175 } 2176 } 2177 2178 /* Examine entrylo 0 or 1 based on ptr. */ 2179 if (use_bbit_insns()) { 2180 uasm_i_bbit0(&p, wr.r2, ilog2(sizeof(pte_t)), 8); 2181 } else { 2182 uasm_i_andi(&p, wr.r3, wr.r2, sizeof(pte_t)); 2183 uasm_i_beqz(&p, wr.r3, 8); 2184 } 2185 /* load it in the delay slot*/ 2186 UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO0); 2187 /* load it if ptr is odd */ 2188 UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO1); 2189 /* 2190 * If the entryLo (now in wr.r3) is valid (bit 1), RI or 2191 * XI must have triggered it. 2192 */ 2193 if (use_bbit_insns()) { 2194 uasm_il_bbit1(&p, &r, wr.r3, 1, label_nopage_tlbl); 2195 uasm_i_nop(&p); 2196 uasm_l_tlbl_goaround1(&l, p); 2197 } else { 2198 uasm_i_andi(&p, wr.r3, wr.r3, 2); 2199 uasm_il_bnez(&p, &r, wr.r3, label_nopage_tlbl); 2200 uasm_i_nop(&p); 2201 } 2202 uasm_l_tlbl_goaround1(&l, p); 2203 } 2204 build_make_valid(&p, &r, wr.r1, wr.r2, wr.r3); 2205 build_r4000_tlbchange_handler_tail(&p, &l, &r, wr.r1, wr.r2); 2206 2207 #ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT 2208 /* 2209 * This is the entry point when build_r4000_tlbchange_handler_head 2210 * spots a huge page. 2211 */ 2212 uasm_l_tlb_huge_update(&l, p); 2213 iPTE_LW(&p, wr.r1, wr.r2); 2214 build_pte_present(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbl); 2215 build_tlb_probe_entry(&p); 2216 2217 if (cpu_has_rixi && !cpu_has_rixiex) { 2218 /* 2219 * If the page is not _PAGE_VALID, RI or XI could not 2220 * have triggered it. Skip the expensive test.. 2221 */ 2222 if (use_bbit_insns()) { 2223 uasm_il_bbit0(&p, &r, wr.r1, ilog2(_PAGE_VALID), 2224 label_tlbl_goaround2); 2225 } else { 2226 uasm_i_andi(&p, wr.r3, wr.r1, _PAGE_VALID); 2227 uasm_il_beqz(&p, &r, wr.r3, label_tlbl_goaround2); 2228 } 2229 uasm_i_nop(&p); 2230 2231 /* 2232 * Warn if something may race with us & replace the TLB entry 2233 * before we read it here. Everything with such races should 2234 * also have dedicated RiXi exception handlers, so this 2235 * shouldn't be hit. 2236 */ 2237 WARN(cpu_has_tlbex_tlbp_race(), "Unhandled race in RiXi path"); 2238 2239 uasm_i_tlbr(&p); 2240 2241 switch (current_cpu_type()) { 2242 default: 2243 if (cpu_has_mips_r2_exec_hazard) { 2244 uasm_i_ehb(&p); 2245 2246 case CPU_CAVIUM_OCTEON: 2247 case CPU_CAVIUM_OCTEON_PLUS: 2248 case CPU_CAVIUM_OCTEON2: 2249 break; 2250 } 2251 } 2252 2253 /* Examine entrylo 0 or 1 based on ptr. */ 2254 if (use_bbit_insns()) { 2255 uasm_i_bbit0(&p, wr.r2, ilog2(sizeof(pte_t)), 8); 2256 } else { 2257 uasm_i_andi(&p, wr.r3, wr.r2, sizeof(pte_t)); 2258 uasm_i_beqz(&p, wr.r3, 8); 2259 } 2260 /* load it in the delay slot*/ 2261 UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO0); 2262 /* load it if ptr is odd */ 2263 UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO1); 2264 /* 2265 * If the entryLo (now in wr.r3) is valid (bit 1), RI or 2266 * XI must have triggered it. 2267 */ 2268 if (use_bbit_insns()) { 2269 uasm_il_bbit0(&p, &r, wr.r3, 1, label_tlbl_goaround2); 2270 } else { 2271 uasm_i_andi(&p, wr.r3, wr.r3, 2); 2272 uasm_il_beqz(&p, &r, wr.r3, label_tlbl_goaround2); 2273 } 2274 if (PM_DEFAULT_MASK == 0) 2275 uasm_i_nop(&p); 2276 /* 2277 * We clobbered C0_PAGEMASK, restore it. On the other branch 2278 * it is restored in build_huge_tlb_write_entry. 2279 */ 2280 build_restore_pagemask(&p, &r, wr.r3, label_nopage_tlbl, 0); 2281 2282 uasm_l_tlbl_goaround2(&l, p); 2283 } 2284 uasm_i_ori(&p, wr.r1, wr.r1, (_PAGE_ACCESSED | _PAGE_VALID)); 2285 build_huge_handler_tail(&p, &r, &l, wr.r1, wr.r2, 1); 2286 #endif 2287 2288 uasm_l_nopage_tlbl(&l, p); 2289 if (IS_ENABLED(CONFIG_CPU_LOONGSON3_WORKAROUNDS)) 2290 uasm_i_sync(&p, 0); 2291 build_restore_work_registers(&p); 2292 #ifdef CONFIG_CPU_MICROMIPS 2293 if ((unsigned long)tlb_do_page_fault_0 & 1) { 2294 uasm_i_lui(&p, K0, uasm_rel_hi((long)tlb_do_page_fault_0)); 2295 uasm_i_addiu(&p, K0, K0, uasm_rel_lo((long)tlb_do_page_fault_0)); 2296 uasm_i_jr(&p, K0); 2297 } else 2298 #endif 2299 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff); 2300 uasm_i_nop(&p); 2301 2302 if (p >= (u32 *)handle_tlbl_end) 2303 panic("TLB load handler fastpath space exceeded"); 2304 2305 uasm_resolve_relocs(relocs, labels); 2306 pr_debug("Wrote TLB load handler fastpath (%u instructions).\n", 2307 (unsigned int)(p - (u32 *)handle_tlbl)); 2308 2309 dump_handler("r4000_tlb_load", handle_tlbl, handle_tlbl_end); 2310 } 2311 2312 static void build_r4000_tlb_store_handler(void) 2313 { 2314 u32 *p = (u32 *)msk_isa16_mode((ulong)handle_tlbs); 2315 struct uasm_label *l = labels; 2316 struct uasm_reloc *r = relocs; 2317 struct work_registers wr; 2318 2319 memset(p, 0, handle_tlbs_end - (char *)p); 2320 memset(labels, 0, sizeof(labels)); 2321 memset(relocs, 0, sizeof(relocs)); 2322 2323 wr = build_r4000_tlbchange_handler_head(&p, &l, &r); 2324 build_pte_writable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbs); 2325 if (m4kc_tlbp_war()) 2326 build_tlb_probe_entry(&p); 2327 build_make_write(&p, &r, wr.r1, wr.r2, wr.r3); 2328 build_r4000_tlbchange_handler_tail(&p, &l, &r, wr.r1, wr.r2); 2329 2330 #ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT 2331 /* 2332 * This is the entry point when 2333 * build_r4000_tlbchange_handler_head spots a huge page. 2334 */ 2335 uasm_l_tlb_huge_update(&l, p); 2336 iPTE_LW(&p, wr.r1, wr.r2); 2337 build_pte_writable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbs); 2338 build_tlb_probe_entry(&p); 2339 uasm_i_ori(&p, wr.r1, wr.r1, 2340 _PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID | _PAGE_DIRTY); 2341 build_huge_handler_tail(&p, &r, &l, wr.r1, wr.r2, 1); 2342 #endif 2343 2344 uasm_l_nopage_tlbs(&l, p); 2345 if (IS_ENABLED(CONFIG_CPU_LOONGSON3_WORKAROUNDS)) 2346 uasm_i_sync(&p, 0); 2347 build_restore_work_registers(&p); 2348 #ifdef CONFIG_CPU_MICROMIPS 2349 if ((unsigned long)tlb_do_page_fault_1 & 1) { 2350 uasm_i_lui(&p, K0, uasm_rel_hi((long)tlb_do_page_fault_1)); 2351 uasm_i_addiu(&p, K0, K0, uasm_rel_lo((long)tlb_do_page_fault_1)); 2352 uasm_i_jr(&p, K0); 2353 } else 2354 #endif 2355 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); 2356 uasm_i_nop(&p); 2357 2358 if (p >= (u32 *)handle_tlbs_end) 2359 panic("TLB store handler fastpath space exceeded"); 2360 2361 uasm_resolve_relocs(relocs, labels); 2362 pr_debug("Wrote TLB store handler fastpath (%u instructions).\n", 2363 (unsigned int)(p - (u32 *)handle_tlbs)); 2364 2365 dump_handler("r4000_tlb_store", handle_tlbs, handle_tlbs_end); 2366 } 2367 2368 static void build_r4000_tlb_modify_handler(void) 2369 { 2370 u32 *p = (u32 *)msk_isa16_mode((ulong)handle_tlbm); 2371 struct uasm_label *l = labels; 2372 struct uasm_reloc *r = relocs; 2373 struct work_registers wr; 2374 2375 memset(p, 0, handle_tlbm_end - (char *)p); 2376 memset(labels, 0, sizeof(labels)); 2377 memset(relocs, 0, sizeof(relocs)); 2378 2379 wr = build_r4000_tlbchange_handler_head(&p, &l, &r); 2380 build_pte_modifiable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbm); 2381 if (m4kc_tlbp_war()) 2382 build_tlb_probe_entry(&p); 2383 /* Present and writable bits set, set accessed and dirty bits. */ 2384 build_make_write(&p, &r, wr.r1, wr.r2, wr.r3); 2385 build_r4000_tlbchange_handler_tail(&p, &l, &r, wr.r1, wr.r2); 2386 2387 #ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT 2388 /* 2389 * This is the entry point when 2390 * build_r4000_tlbchange_handler_head spots a huge page. 2391 */ 2392 uasm_l_tlb_huge_update(&l, p); 2393 iPTE_LW(&p, wr.r1, wr.r2); 2394 build_pte_modifiable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbm); 2395 build_tlb_probe_entry(&p); 2396 uasm_i_ori(&p, wr.r1, wr.r1, 2397 _PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID | _PAGE_DIRTY); 2398 build_huge_handler_tail(&p, &r, &l, wr.r1, wr.r2, 0); 2399 #endif 2400 2401 uasm_l_nopage_tlbm(&l, p); 2402 if (IS_ENABLED(CONFIG_CPU_LOONGSON3_WORKAROUNDS)) 2403 uasm_i_sync(&p, 0); 2404 build_restore_work_registers(&p); 2405 #ifdef CONFIG_CPU_MICROMIPS 2406 if ((unsigned long)tlb_do_page_fault_1 & 1) { 2407 uasm_i_lui(&p, K0, uasm_rel_hi((long)tlb_do_page_fault_1)); 2408 uasm_i_addiu(&p, K0, K0, uasm_rel_lo((long)tlb_do_page_fault_1)); 2409 uasm_i_jr(&p, K0); 2410 } else 2411 #endif 2412 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); 2413 uasm_i_nop(&p); 2414 2415 if (p >= (u32 *)handle_tlbm_end) 2416 panic("TLB modify handler fastpath space exceeded"); 2417 2418 uasm_resolve_relocs(relocs, labels); 2419 pr_debug("Wrote TLB modify handler fastpath (%u instructions).\n", 2420 (unsigned int)(p - (u32 *)handle_tlbm)); 2421 2422 dump_handler("r4000_tlb_modify", handle_tlbm, handle_tlbm_end); 2423 } 2424 2425 static void flush_tlb_handlers(void) 2426 { 2427 local_flush_icache_range((unsigned long)handle_tlbl, 2428 (unsigned long)handle_tlbl_end); 2429 local_flush_icache_range((unsigned long)handle_tlbs, 2430 (unsigned long)handle_tlbs_end); 2431 local_flush_icache_range((unsigned long)handle_tlbm, 2432 (unsigned long)handle_tlbm_end); 2433 local_flush_icache_range((unsigned long)tlbmiss_handler_setup_pgd, 2434 (unsigned long)tlbmiss_handler_setup_pgd_end); 2435 } 2436 2437 static void print_htw_config(void) 2438 { 2439 unsigned long config; 2440 unsigned int pwctl; 2441 const int field = 2 * sizeof(unsigned long); 2442 2443 config = read_c0_pwfield(); 2444 pr_debug("PWField (0x%0*lx): GDI: 0x%02lx UDI: 0x%02lx MDI: 0x%02lx PTI: 0x%02lx PTEI: 0x%02lx\n", 2445 field, config, 2446 (config & MIPS_PWFIELD_GDI_MASK) >> MIPS_PWFIELD_GDI_SHIFT, 2447 (config & MIPS_PWFIELD_UDI_MASK) >> MIPS_PWFIELD_UDI_SHIFT, 2448 (config & MIPS_PWFIELD_MDI_MASK) >> MIPS_PWFIELD_MDI_SHIFT, 2449 (config & MIPS_PWFIELD_PTI_MASK) >> MIPS_PWFIELD_PTI_SHIFT, 2450 (config & MIPS_PWFIELD_PTEI_MASK) >> MIPS_PWFIELD_PTEI_SHIFT); 2451 2452 config = read_c0_pwsize(); 2453 pr_debug("PWSize (0x%0*lx): PS: 0x%lx GDW: 0x%02lx UDW: 0x%02lx MDW: 0x%02lx PTW: 0x%02lx PTEW: 0x%02lx\n", 2454 field, config, 2455 (config & MIPS_PWSIZE_PS_MASK) >> MIPS_PWSIZE_PS_SHIFT, 2456 (config & MIPS_PWSIZE_GDW_MASK) >> MIPS_PWSIZE_GDW_SHIFT, 2457 (config & MIPS_PWSIZE_UDW_MASK) >> MIPS_PWSIZE_UDW_SHIFT, 2458 (config & MIPS_PWSIZE_MDW_MASK) >> MIPS_PWSIZE_MDW_SHIFT, 2459 (config & MIPS_PWSIZE_PTW_MASK) >> MIPS_PWSIZE_PTW_SHIFT, 2460 (config & MIPS_PWSIZE_PTEW_MASK) >> MIPS_PWSIZE_PTEW_SHIFT); 2461 2462 pwctl = read_c0_pwctl(); 2463 pr_debug("PWCtl (0x%x): PWEn: 0x%x XK: 0x%x XS: 0x%x XU: 0x%x DPH: 0x%x HugePg: 0x%x Psn: 0x%x\n", 2464 pwctl, 2465 (pwctl & MIPS_PWCTL_PWEN_MASK) >> MIPS_PWCTL_PWEN_SHIFT, 2466 (pwctl & MIPS_PWCTL_XK_MASK) >> MIPS_PWCTL_XK_SHIFT, 2467 (pwctl & MIPS_PWCTL_XS_MASK) >> MIPS_PWCTL_XS_SHIFT, 2468 (pwctl & MIPS_PWCTL_XU_MASK) >> MIPS_PWCTL_XU_SHIFT, 2469 (pwctl & MIPS_PWCTL_DPH_MASK) >> MIPS_PWCTL_DPH_SHIFT, 2470 (pwctl & MIPS_PWCTL_HUGEPG_MASK) >> MIPS_PWCTL_HUGEPG_SHIFT, 2471 (pwctl & MIPS_PWCTL_PSN_MASK) >> MIPS_PWCTL_PSN_SHIFT); 2472 } 2473 2474 static void config_htw_params(void) 2475 { 2476 unsigned long pwfield, pwsize, ptei; 2477 unsigned int config; 2478 2479 /* 2480 * We are using 2-level page tables, so we only need to 2481 * setup GDW and PTW appropriately. UDW and MDW will remain 0. 2482 * The default value of GDI/UDI/MDI/PTI is 0xc. It is illegal to 2483 * write values less than 0xc in these fields because the entire 2484 * write will be dropped. As a result of which, we must preserve 2485 * the original reset values and overwrite only what we really want. 2486 */ 2487 2488 pwfield = read_c0_pwfield(); 2489 /* re-initialize the GDI field */ 2490 pwfield &= ~MIPS_PWFIELD_GDI_MASK; 2491 pwfield |= PGDIR_SHIFT << MIPS_PWFIELD_GDI_SHIFT; 2492 /* re-initialize the PTI field including the even/odd bit */ 2493 pwfield &= ~MIPS_PWFIELD_PTI_MASK; 2494 pwfield |= PAGE_SHIFT << MIPS_PWFIELD_PTI_SHIFT; 2495 if (CONFIG_PGTABLE_LEVELS >= 3) { 2496 pwfield &= ~MIPS_PWFIELD_MDI_MASK; 2497 pwfield |= PMD_SHIFT << MIPS_PWFIELD_MDI_SHIFT; 2498 } 2499 /* Set the PTEI right shift */ 2500 ptei = _PAGE_GLOBAL_SHIFT << MIPS_PWFIELD_PTEI_SHIFT; 2501 pwfield |= ptei; 2502 write_c0_pwfield(pwfield); 2503 /* Check whether the PTEI value is supported */ 2504 back_to_back_c0_hazard(); 2505 pwfield = read_c0_pwfield(); 2506 if (((pwfield & MIPS_PWFIELD_PTEI_MASK) << MIPS_PWFIELD_PTEI_SHIFT) 2507 != ptei) { 2508 pr_warn("Unsupported PTEI field value: 0x%lx. HTW will not be enabled", 2509 ptei); 2510 /* 2511 * Drop option to avoid HTW being enabled via another path 2512 * (eg htw_reset()) 2513 */ 2514 current_cpu_data.options &= ~MIPS_CPU_HTW; 2515 return; 2516 } 2517 2518 pwsize = ilog2(PTRS_PER_PGD) << MIPS_PWSIZE_GDW_SHIFT; 2519 pwsize |= ilog2(PTRS_PER_PTE) << MIPS_PWSIZE_PTW_SHIFT; 2520 if (CONFIG_PGTABLE_LEVELS >= 3) 2521 pwsize |= ilog2(PTRS_PER_PMD) << MIPS_PWSIZE_MDW_SHIFT; 2522 2523 /* Set pointer size to size of directory pointers */ 2524 if (IS_ENABLED(CONFIG_64BIT)) 2525 pwsize |= MIPS_PWSIZE_PS_MASK; 2526 /* PTEs may be multiple pointers long (e.g. with XPA) */ 2527 pwsize |= ((PTE_T_LOG2 - PGD_T_LOG2) << MIPS_PWSIZE_PTEW_SHIFT) 2528 & MIPS_PWSIZE_PTEW_MASK; 2529 2530 write_c0_pwsize(pwsize); 2531 2532 /* Make sure everything is set before we enable the HTW */ 2533 back_to_back_c0_hazard(); 2534 2535 /* 2536 * Enable HTW (and only for XUSeg on 64-bit), and disable the rest of 2537 * the pwctl fields. 2538 */ 2539 config = 1 << MIPS_PWCTL_PWEN_SHIFT; 2540 if (IS_ENABLED(CONFIG_64BIT)) 2541 config |= MIPS_PWCTL_XU_MASK; 2542 write_c0_pwctl(config); 2543 pr_info("Hardware Page Table Walker enabled\n"); 2544 2545 print_htw_config(); 2546 } 2547 2548 static void config_xpa_params(void) 2549 { 2550 #ifdef CONFIG_XPA 2551 unsigned int pagegrain; 2552 2553 if (mips_xpa_disabled) { 2554 pr_info("Extended Physical Addressing (XPA) disabled\n"); 2555 return; 2556 } 2557 2558 pagegrain = read_c0_pagegrain(); 2559 write_c0_pagegrain(pagegrain | PG_ELPA); 2560 back_to_back_c0_hazard(); 2561 pagegrain = read_c0_pagegrain(); 2562 2563 if (pagegrain & PG_ELPA) 2564 pr_info("Extended Physical Addressing (XPA) enabled\n"); 2565 else 2566 panic("Extended Physical Addressing (XPA) disabled"); 2567 #endif 2568 } 2569 2570 static void check_pabits(void) 2571 { 2572 unsigned long entry; 2573 unsigned pabits, fillbits; 2574 2575 if (!cpu_has_rixi || !_PAGE_NO_EXEC) { 2576 /* 2577 * We'll only be making use of the fact that we can rotate bits 2578 * into the fill if the CPU supports RIXI, so don't bother 2579 * probing this for CPUs which don't. 2580 */ 2581 return; 2582 } 2583 2584 write_c0_entrylo0(~0ul); 2585 back_to_back_c0_hazard(); 2586 entry = read_c0_entrylo0(); 2587 2588 /* clear all non-PFN bits */ 2589 entry &= ~((1 << MIPS_ENTRYLO_PFN_SHIFT) - 1); 2590 entry &= ~(MIPS_ENTRYLO_RI | MIPS_ENTRYLO_XI); 2591 2592 /* find a lower bound on PABITS, and upper bound on fill bits */ 2593 pabits = fls_long(entry) + 6; 2594 fillbits = max_t(int, (int)BITS_PER_LONG - pabits, 0); 2595 2596 /* minus the RI & XI bits */ 2597 fillbits -= min_t(unsigned, fillbits, 2); 2598 2599 if (fillbits >= ilog2(_PAGE_NO_EXEC)) 2600 fill_includes_sw_bits = true; 2601 2602 pr_debug("Entry* registers contain %u fill bits\n", fillbits); 2603 } 2604 2605 void build_tlb_refill_handler(void) 2606 { 2607 /* 2608 * The refill handler is generated per-CPU, multi-node systems 2609 * may have local storage for it. The other handlers are only 2610 * needed once. 2611 */ 2612 static int run_once = 0; 2613 2614 if (IS_ENABLED(CONFIG_XPA) && !cpu_has_rixi) 2615 panic("Kernels supporting XPA currently require CPUs with RIXI"); 2616 2617 output_pgtable_bits_defines(); 2618 check_pabits(); 2619 2620 #ifdef CONFIG_64BIT 2621 check_for_high_segbits = current_cpu_data.vmbits > (PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3); 2622 #endif 2623 2624 if (cpu_has_3kex) { 2625 #ifndef CONFIG_MIPS_PGD_C0_CONTEXT 2626 if (!run_once) { 2627 build_setup_pgd(); 2628 build_r3000_tlb_refill_handler(); 2629 build_r3000_tlb_load_handler(); 2630 build_r3000_tlb_store_handler(); 2631 build_r3000_tlb_modify_handler(); 2632 flush_tlb_handlers(); 2633 run_once++; 2634 } 2635 #else 2636 panic("No R3000 TLB refill handler"); 2637 #endif 2638 return; 2639 } 2640 2641 if (cpu_has_ldpte) 2642 setup_pw(); 2643 2644 if (!run_once) { 2645 scratch_reg = allocate_kscratch(); 2646 build_setup_pgd(); 2647 build_r4000_tlb_load_handler(); 2648 build_r4000_tlb_store_handler(); 2649 build_r4000_tlb_modify_handler(); 2650 if (cpu_has_ldpte) 2651 build_loongson3_tlb_refill_handler(); 2652 else 2653 build_r4000_tlb_refill_handler(); 2654 flush_tlb_handlers(); 2655 run_once++; 2656 } 2657 if (cpu_has_xpa) 2658 config_xpa_params(); 2659 if (cpu_has_htw) 2660 config_htw_params(); 2661 } 2662