1 /* 2 * This file is subject to the terms and conditions of the GNU General Public 3 * License. See the file "COPYING" in the main directory of this archive 4 * for more details. 5 * 6 * Synthesize TLB refill handlers at runtime. 7 * 8 * Copyright (C) 2004, 2005, 2006, 2008 Thiemo Seufer 9 * Copyright (C) 2005, 2007, 2008, 2009 Maciej W. Rozycki 10 * Copyright (C) 2006 Ralf Baechle (ralf@linux-mips.org) 11 * Copyright (C) 2008, 2009 Cavium Networks, Inc. 12 * Copyright (C) 2011 MIPS Technologies, Inc. 13 * 14 * ... and the days got worse and worse and now you see 15 * I've gone completly out of my mind. 16 * 17 * They're coming to take me a away haha 18 * they're coming to take me a away hoho hihi haha 19 * to the funny farm where code is beautiful all the time ... 20 * 21 * (Condolences to Napoleon XIV) 22 */ 23 24 #include <linux/bug.h> 25 #include <linux/kernel.h> 26 #include <linux/types.h> 27 #include <linux/smp.h> 28 #include <linux/string.h> 29 #include <linux/init.h> 30 #include <linux/cache.h> 31 32 #include <asm/cacheflush.h> 33 #include <asm/pgtable.h> 34 #include <asm/war.h> 35 #include <asm/uasm.h> 36 #include <asm/setup.h> 37 38 /* 39 * TLB load/store/modify handlers. 40 * 41 * Only the fastpath gets synthesized at runtime, the slowpath for 42 * do_page_fault remains normal asm. 43 */ 44 extern void tlb_do_page_fault_0(void); 45 extern void tlb_do_page_fault_1(void); 46 47 struct work_registers { 48 int r1; 49 int r2; 50 int r3; 51 }; 52 53 struct tlb_reg_save { 54 unsigned long a; 55 unsigned long b; 56 } ____cacheline_aligned_in_smp; 57 58 static struct tlb_reg_save handler_reg_save[NR_CPUS]; 59 60 static inline int r45k_bvahwbug(void) 61 { 62 /* XXX: We should probe for the presence of this bug, but we don't. */ 63 return 0; 64 } 65 66 static inline int r4k_250MHZhwbug(void) 67 { 68 /* XXX: We should probe for the presence of this bug, but we don't. */ 69 return 0; 70 } 71 72 static inline int __maybe_unused bcm1250_m3_war(void) 73 { 74 return BCM1250_M3_WAR; 75 } 76 77 static inline int __maybe_unused r10000_llsc_war(void) 78 { 79 return R10000_LLSC_WAR; 80 } 81 82 static int use_bbit_insns(void) 83 { 84 switch (current_cpu_type()) { 85 case CPU_CAVIUM_OCTEON: 86 case CPU_CAVIUM_OCTEON_PLUS: 87 case CPU_CAVIUM_OCTEON2: 88 return 1; 89 default: 90 return 0; 91 } 92 } 93 94 static int use_lwx_insns(void) 95 { 96 switch (current_cpu_type()) { 97 case CPU_CAVIUM_OCTEON2: 98 return 1; 99 default: 100 return 0; 101 } 102 } 103 #if defined(CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE) && \ 104 CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE > 0 105 static bool scratchpad_available(void) 106 { 107 return true; 108 } 109 static int scratchpad_offset(int i) 110 { 111 /* 112 * CVMSEG starts at address -32768 and extends for 113 * CAVIUM_OCTEON_CVMSEG_SIZE 128 byte cache lines. 114 */ 115 i += 1; /* Kernel use starts at the top and works down. */ 116 return CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE * 128 - (8 * i) - 32768; 117 } 118 #else 119 static bool scratchpad_available(void) 120 { 121 return false; 122 } 123 static int scratchpad_offset(int i) 124 { 125 BUG(); 126 /* Really unreachable, but evidently some GCC want this. */ 127 return 0; 128 } 129 #endif 130 /* 131 * Found by experiment: At least some revisions of the 4kc throw under 132 * some circumstances a machine check exception, triggered by invalid 133 * values in the index register. Delaying the tlbp instruction until 134 * after the next branch, plus adding an additional nop in front of 135 * tlbwi/tlbwr avoids the invalid index register values. Nobody knows 136 * why; it's not an issue caused by the core RTL. 137 * 138 */ 139 static int __cpuinit m4kc_tlbp_war(void) 140 { 141 return (current_cpu_data.processor_id & 0xffff00) == 142 (PRID_COMP_MIPS | PRID_IMP_4KC); 143 } 144 145 /* Handle labels (which must be positive integers). */ 146 enum label_id { 147 label_second_part = 1, 148 label_leave, 149 label_vmalloc, 150 label_vmalloc_done, 151 label_tlbw_hazard, 152 label_split, 153 label_tlbl_goaround1, 154 label_tlbl_goaround2, 155 label_nopage_tlbl, 156 label_nopage_tlbs, 157 label_nopage_tlbm, 158 label_smp_pgtable_change, 159 label_r3000_write_probe_fail, 160 label_large_segbits_fault, 161 #ifdef CONFIG_HUGETLB_PAGE 162 label_tlb_huge_update, 163 #endif 164 }; 165 166 UASM_L_LA(_second_part) 167 UASM_L_LA(_leave) 168 UASM_L_LA(_vmalloc) 169 UASM_L_LA(_vmalloc_done) 170 UASM_L_LA(_tlbw_hazard) 171 UASM_L_LA(_split) 172 UASM_L_LA(_tlbl_goaround1) 173 UASM_L_LA(_tlbl_goaround2) 174 UASM_L_LA(_nopage_tlbl) 175 UASM_L_LA(_nopage_tlbs) 176 UASM_L_LA(_nopage_tlbm) 177 UASM_L_LA(_smp_pgtable_change) 178 UASM_L_LA(_r3000_write_probe_fail) 179 UASM_L_LA(_large_segbits_fault) 180 #ifdef CONFIG_HUGETLB_PAGE 181 UASM_L_LA(_tlb_huge_update) 182 #endif 183 184 /* 185 * For debug purposes. 186 */ 187 static inline void dump_handler(const u32 *handler, int count) 188 { 189 int i; 190 191 pr_debug("\t.set push\n"); 192 pr_debug("\t.set noreorder\n"); 193 194 for (i = 0; i < count; i++) 195 pr_debug("\t%p\t.word 0x%08x\n", &handler[i], handler[i]); 196 197 pr_debug("\t.set pop\n"); 198 } 199 200 /* The only general purpose registers allowed in TLB handlers. */ 201 #define K0 26 202 #define K1 27 203 204 /* Some CP0 registers */ 205 #define C0_INDEX 0, 0 206 #define C0_ENTRYLO0 2, 0 207 #define C0_TCBIND 2, 2 208 #define C0_ENTRYLO1 3, 0 209 #define C0_CONTEXT 4, 0 210 #define C0_PAGEMASK 5, 0 211 #define C0_BADVADDR 8, 0 212 #define C0_ENTRYHI 10, 0 213 #define C0_EPC 14, 0 214 #define C0_XCONTEXT 20, 0 215 216 #ifdef CONFIG_64BIT 217 # define GET_CONTEXT(buf, reg) UASM_i_MFC0(buf, reg, C0_XCONTEXT) 218 #else 219 # define GET_CONTEXT(buf, reg) UASM_i_MFC0(buf, reg, C0_CONTEXT) 220 #endif 221 222 /* The worst case length of the handler is around 18 instructions for 223 * R3000-style TLBs and up to 63 instructions for R4000-style TLBs. 224 * Maximum space available is 32 instructions for R3000 and 64 225 * instructions for R4000. 226 * 227 * We deliberately chose a buffer size of 128, so we won't scribble 228 * over anything important on overflow before we panic. 229 */ 230 static u32 tlb_handler[128] __cpuinitdata; 231 232 /* simply assume worst case size for labels and relocs */ 233 static struct uasm_label labels[128] __cpuinitdata; 234 static struct uasm_reloc relocs[128] __cpuinitdata; 235 236 #ifdef CONFIG_64BIT 237 static int check_for_high_segbits __cpuinitdata; 238 #endif 239 240 static int check_for_high_segbits __cpuinitdata; 241 242 static unsigned int kscratch_used_mask __cpuinitdata; 243 244 static int __cpuinit allocate_kscratch(void) 245 { 246 int r; 247 unsigned int a = cpu_data[0].kscratch_mask & ~kscratch_used_mask; 248 249 r = ffs(a); 250 251 if (r == 0) 252 return -1; 253 254 r--; /* make it zero based */ 255 256 kscratch_used_mask |= (1 << r); 257 258 return r; 259 } 260 261 static int scratch_reg __cpuinitdata; 262 static int pgd_reg __cpuinitdata; 263 enum vmalloc64_mode {not_refill, refill_scratch, refill_noscratch}; 264 265 static struct work_registers __cpuinit build_get_work_registers(u32 **p) 266 { 267 struct work_registers r; 268 269 int smp_processor_id_reg; 270 int smp_processor_id_sel; 271 int smp_processor_id_shift; 272 273 if (scratch_reg > 0) { 274 /* Save in CPU local C0_KScratch? */ 275 UASM_i_MTC0(p, 1, 31, scratch_reg); 276 r.r1 = K0; 277 r.r2 = K1; 278 r.r3 = 1; 279 return r; 280 } 281 282 if (num_possible_cpus() > 1) { 283 #ifdef CONFIG_MIPS_PGD_C0_CONTEXT 284 smp_processor_id_shift = 51; 285 smp_processor_id_reg = 20; /* XContext */ 286 smp_processor_id_sel = 0; 287 #else 288 # ifdef CONFIG_32BIT 289 smp_processor_id_shift = 25; 290 smp_processor_id_reg = 4; /* Context */ 291 smp_processor_id_sel = 0; 292 # endif 293 # ifdef CONFIG_64BIT 294 smp_processor_id_shift = 26; 295 smp_processor_id_reg = 4; /* Context */ 296 smp_processor_id_sel = 0; 297 # endif 298 #endif 299 /* Get smp_processor_id */ 300 UASM_i_MFC0(p, K0, smp_processor_id_reg, smp_processor_id_sel); 301 UASM_i_SRL_SAFE(p, K0, K0, smp_processor_id_shift); 302 303 /* handler_reg_save index in K0 */ 304 UASM_i_SLL(p, K0, K0, ilog2(sizeof(struct tlb_reg_save))); 305 306 UASM_i_LA(p, K1, (long)&handler_reg_save); 307 UASM_i_ADDU(p, K0, K0, K1); 308 } else { 309 UASM_i_LA(p, K0, (long)&handler_reg_save); 310 } 311 /* K0 now points to save area, save $1 and $2 */ 312 UASM_i_SW(p, 1, offsetof(struct tlb_reg_save, a), K0); 313 UASM_i_SW(p, 2, offsetof(struct tlb_reg_save, b), K0); 314 315 r.r1 = K1; 316 r.r2 = 1; 317 r.r3 = 2; 318 return r; 319 } 320 321 static void __cpuinit build_restore_work_registers(u32 **p) 322 { 323 if (scratch_reg > 0) { 324 UASM_i_MFC0(p, 1, 31, scratch_reg); 325 return; 326 } 327 /* K0 already points to save area, restore $1 and $2 */ 328 UASM_i_LW(p, 1, offsetof(struct tlb_reg_save, a), K0); 329 UASM_i_LW(p, 2, offsetof(struct tlb_reg_save, b), K0); 330 } 331 332 #ifndef CONFIG_MIPS_PGD_C0_CONTEXT 333 334 /* 335 * CONFIG_MIPS_PGD_C0_CONTEXT implies 64 bit and lack of pgd_current, 336 * we cannot do r3000 under these circumstances. 337 * 338 * Declare pgd_current here instead of including mmu_context.h to avoid type 339 * conflicts for tlbmiss_handler_setup_pgd 340 */ 341 extern unsigned long pgd_current[]; 342 343 /* 344 * The R3000 TLB handler is simple. 345 */ 346 static void __cpuinit build_r3000_tlb_refill_handler(void) 347 { 348 long pgdc = (long)pgd_current; 349 u32 *p; 350 351 memset(tlb_handler, 0, sizeof(tlb_handler)); 352 p = tlb_handler; 353 354 uasm_i_mfc0(&p, K0, C0_BADVADDR); 355 uasm_i_lui(&p, K1, uasm_rel_hi(pgdc)); /* cp0 delay */ 356 uasm_i_lw(&p, K1, uasm_rel_lo(pgdc), K1); 357 uasm_i_srl(&p, K0, K0, 22); /* load delay */ 358 uasm_i_sll(&p, K0, K0, 2); 359 uasm_i_addu(&p, K1, K1, K0); 360 uasm_i_mfc0(&p, K0, C0_CONTEXT); 361 uasm_i_lw(&p, K1, 0, K1); /* cp0 delay */ 362 uasm_i_andi(&p, K0, K0, 0xffc); /* load delay */ 363 uasm_i_addu(&p, K1, K1, K0); 364 uasm_i_lw(&p, K0, 0, K1); 365 uasm_i_nop(&p); /* load delay */ 366 uasm_i_mtc0(&p, K0, C0_ENTRYLO0); 367 uasm_i_mfc0(&p, K1, C0_EPC); /* cp0 delay */ 368 uasm_i_tlbwr(&p); /* cp0 delay */ 369 uasm_i_jr(&p, K1); 370 uasm_i_rfe(&p); /* branch delay */ 371 372 if (p > tlb_handler + 32) 373 panic("TLB refill handler space exceeded"); 374 375 pr_debug("Wrote TLB refill handler (%u instructions).\n", 376 (unsigned int)(p - tlb_handler)); 377 378 memcpy((void *)ebase, tlb_handler, 0x80); 379 380 dump_handler((u32 *)ebase, 32); 381 } 382 #endif /* CONFIG_MIPS_PGD_C0_CONTEXT */ 383 384 /* 385 * The R4000 TLB handler is much more complicated. We have two 386 * consecutive handler areas with 32 instructions space each. 387 * Since they aren't used at the same time, we can overflow in the 388 * other one.To keep things simple, we first assume linear space, 389 * then we relocate it to the final handler layout as needed. 390 */ 391 static u32 final_handler[64] __cpuinitdata; 392 393 /* 394 * Hazards 395 * 396 * From the IDT errata for the QED RM5230 (Nevada), processor revision 1.0: 397 * 2. A timing hazard exists for the TLBP instruction. 398 * 399 * stalling_instruction 400 * TLBP 401 * 402 * The JTLB is being read for the TLBP throughout the stall generated by the 403 * previous instruction. This is not really correct as the stalling instruction 404 * can modify the address used to access the JTLB. The failure symptom is that 405 * the TLBP instruction will use an address created for the stalling instruction 406 * and not the address held in C0_ENHI and thus report the wrong results. 407 * 408 * The software work-around is to not allow the instruction preceding the TLBP 409 * to stall - make it an NOP or some other instruction guaranteed not to stall. 410 * 411 * Errata 2 will not be fixed. This errata is also on the R5000. 412 * 413 * As if we MIPS hackers wouldn't know how to nop pipelines happy ... 414 */ 415 static void __cpuinit __maybe_unused build_tlb_probe_entry(u32 **p) 416 { 417 switch (current_cpu_type()) { 418 /* Found by experiment: R4600 v2.0/R4700 needs this, too. */ 419 case CPU_R4600: 420 case CPU_R4700: 421 case CPU_R5000: 422 case CPU_R5000A: 423 case CPU_NEVADA: 424 uasm_i_nop(p); 425 uasm_i_tlbp(p); 426 break; 427 428 default: 429 uasm_i_tlbp(p); 430 break; 431 } 432 } 433 434 /* 435 * Write random or indexed TLB entry, and care about the hazards from 436 * the preceding mtc0 and for the following eret. 437 */ 438 enum tlb_write_entry { tlb_random, tlb_indexed }; 439 440 static void __cpuinit build_tlb_write_entry(u32 **p, struct uasm_label **l, 441 struct uasm_reloc **r, 442 enum tlb_write_entry wmode) 443 { 444 void(*tlbw)(u32 **) = NULL; 445 446 switch (wmode) { 447 case tlb_random: tlbw = uasm_i_tlbwr; break; 448 case tlb_indexed: tlbw = uasm_i_tlbwi; break; 449 } 450 451 if (cpu_has_mips_r2) { 452 if (cpu_has_mips_r2_exec_hazard) 453 uasm_i_ehb(p); 454 tlbw(p); 455 return; 456 } 457 458 switch (current_cpu_type()) { 459 case CPU_R4000PC: 460 case CPU_R4000SC: 461 case CPU_R4000MC: 462 case CPU_R4400PC: 463 case CPU_R4400SC: 464 case CPU_R4400MC: 465 /* 466 * This branch uses up a mtc0 hazard nop slot and saves 467 * two nops after the tlbw instruction. 468 */ 469 uasm_il_bgezl(p, r, 0, label_tlbw_hazard); 470 tlbw(p); 471 uasm_l_tlbw_hazard(l, *p); 472 uasm_i_nop(p); 473 break; 474 475 case CPU_R4600: 476 case CPU_R4700: 477 case CPU_R5000: 478 case CPU_R5000A: 479 uasm_i_nop(p); 480 tlbw(p); 481 uasm_i_nop(p); 482 break; 483 484 case CPU_R4300: 485 case CPU_5KC: 486 case CPU_TX49XX: 487 case CPU_PR4450: 488 case CPU_XLR: 489 uasm_i_nop(p); 490 tlbw(p); 491 break; 492 493 case CPU_R10000: 494 case CPU_R12000: 495 case CPU_R14000: 496 case CPU_4KC: 497 case CPU_4KEC: 498 case CPU_M14KC: 499 case CPU_SB1: 500 case CPU_SB1A: 501 case CPU_4KSC: 502 case CPU_20KC: 503 case CPU_25KF: 504 case CPU_BMIPS32: 505 case CPU_BMIPS3300: 506 case CPU_BMIPS4350: 507 case CPU_BMIPS4380: 508 case CPU_BMIPS5000: 509 case CPU_LOONGSON2: 510 case CPU_R5500: 511 if (m4kc_tlbp_war()) 512 uasm_i_nop(p); 513 case CPU_ALCHEMY: 514 tlbw(p); 515 break; 516 517 case CPU_NEVADA: 518 uasm_i_nop(p); /* QED specifies 2 nops hazard */ 519 /* 520 * This branch uses up a mtc0 hazard nop slot and saves 521 * a nop after the tlbw instruction. 522 */ 523 uasm_il_bgezl(p, r, 0, label_tlbw_hazard); 524 tlbw(p); 525 uasm_l_tlbw_hazard(l, *p); 526 break; 527 528 case CPU_RM7000: 529 uasm_i_nop(p); 530 uasm_i_nop(p); 531 uasm_i_nop(p); 532 uasm_i_nop(p); 533 tlbw(p); 534 break; 535 536 case CPU_RM9000: 537 /* 538 * When the JTLB is updated by tlbwi or tlbwr, a subsequent 539 * use of the JTLB for instructions should not occur for 4 540 * cpu cycles and use for data translations should not occur 541 * for 3 cpu cycles. 542 */ 543 uasm_i_ssnop(p); 544 uasm_i_ssnop(p); 545 uasm_i_ssnop(p); 546 uasm_i_ssnop(p); 547 tlbw(p); 548 uasm_i_ssnop(p); 549 uasm_i_ssnop(p); 550 uasm_i_ssnop(p); 551 uasm_i_ssnop(p); 552 break; 553 554 case CPU_VR4111: 555 case CPU_VR4121: 556 case CPU_VR4122: 557 case CPU_VR4181: 558 case CPU_VR4181A: 559 uasm_i_nop(p); 560 uasm_i_nop(p); 561 tlbw(p); 562 uasm_i_nop(p); 563 uasm_i_nop(p); 564 break; 565 566 case CPU_VR4131: 567 case CPU_VR4133: 568 case CPU_R5432: 569 uasm_i_nop(p); 570 uasm_i_nop(p); 571 tlbw(p); 572 break; 573 574 case CPU_JZRISC: 575 tlbw(p); 576 uasm_i_nop(p); 577 break; 578 579 default: 580 panic("No TLB refill handler yet (CPU type: %d)", 581 current_cpu_data.cputype); 582 break; 583 } 584 } 585 586 static __cpuinit __maybe_unused void build_convert_pte_to_entrylo(u32 **p, 587 unsigned int reg) 588 { 589 if (kernel_uses_smartmips_rixi) { 590 UASM_i_SRL(p, reg, reg, ilog2(_PAGE_NO_EXEC)); 591 UASM_i_ROTR(p, reg, reg, ilog2(_PAGE_GLOBAL) - ilog2(_PAGE_NO_EXEC)); 592 } else { 593 #ifdef CONFIG_64BIT_PHYS_ADDR 594 uasm_i_dsrl_safe(p, reg, reg, ilog2(_PAGE_GLOBAL)); 595 #else 596 UASM_i_SRL(p, reg, reg, ilog2(_PAGE_GLOBAL)); 597 #endif 598 } 599 } 600 601 #ifdef CONFIG_HUGETLB_PAGE 602 603 static __cpuinit void build_restore_pagemask(u32 **p, 604 struct uasm_reloc **r, 605 unsigned int tmp, 606 enum label_id lid, 607 int restore_scratch) 608 { 609 if (restore_scratch) { 610 /* Reset default page size */ 611 if (PM_DEFAULT_MASK >> 16) { 612 uasm_i_lui(p, tmp, PM_DEFAULT_MASK >> 16); 613 uasm_i_ori(p, tmp, tmp, PM_DEFAULT_MASK & 0xffff); 614 uasm_i_mtc0(p, tmp, C0_PAGEMASK); 615 uasm_il_b(p, r, lid); 616 } else if (PM_DEFAULT_MASK) { 617 uasm_i_ori(p, tmp, 0, PM_DEFAULT_MASK); 618 uasm_i_mtc0(p, tmp, C0_PAGEMASK); 619 uasm_il_b(p, r, lid); 620 } else { 621 uasm_i_mtc0(p, 0, C0_PAGEMASK); 622 uasm_il_b(p, r, lid); 623 } 624 if (scratch_reg > 0) 625 UASM_i_MFC0(p, 1, 31, scratch_reg); 626 else 627 UASM_i_LW(p, 1, scratchpad_offset(0), 0); 628 } else { 629 /* Reset default page size */ 630 if (PM_DEFAULT_MASK >> 16) { 631 uasm_i_lui(p, tmp, PM_DEFAULT_MASK >> 16); 632 uasm_i_ori(p, tmp, tmp, PM_DEFAULT_MASK & 0xffff); 633 uasm_il_b(p, r, lid); 634 uasm_i_mtc0(p, tmp, C0_PAGEMASK); 635 } else if (PM_DEFAULT_MASK) { 636 uasm_i_ori(p, tmp, 0, PM_DEFAULT_MASK); 637 uasm_il_b(p, r, lid); 638 uasm_i_mtc0(p, tmp, C0_PAGEMASK); 639 } else { 640 uasm_il_b(p, r, lid); 641 uasm_i_mtc0(p, 0, C0_PAGEMASK); 642 } 643 } 644 } 645 646 static __cpuinit void build_huge_tlb_write_entry(u32 **p, 647 struct uasm_label **l, 648 struct uasm_reloc **r, 649 unsigned int tmp, 650 enum tlb_write_entry wmode, 651 int restore_scratch) 652 { 653 /* Set huge page tlb entry size */ 654 uasm_i_lui(p, tmp, PM_HUGE_MASK >> 16); 655 uasm_i_ori(p, tmp, tmp, PM_HUGE_MASK & 0xffff); 656 uasm_i_mtc0(p, tmp, C0_PAGEMASK); 657 658 build_tlb_write_entry(p, l, r, wmode); 659 660 build_restore_pagemask(p, r, tmp, label_leave, restore_scratch); 661 } 662 663 /* 664 * Check if Huge PTE is present, if so then jump to LABEL. 665 */ 666 static void __cpuinit 667 build_is_huge_pte(u32 **p, struct uasm_reloc **r, unsigned int tmp, 668 unsigned int pmd, int lid) 669 { 670 UASM_i_LW(p, tmp, 0, pmd); 671 if (use_bbit_insns()) { 672 uasm_il_bbit1(p, r, tmp, ilog2(_PAGE_HUGE), lid); 673 } else { 674 uasm_i_andi(p, tmp, tmp, _PAGE_HUGE); 675 uasm_il_bnez(p, r, tmp, lid); 676 } 677 } 678 679 static __cpuinit void build_huge_update_entries(u32 **p, 680 unsigned int pte, 681 unsigned int tmp) 682 { 683 int small_sequence; 684 685 /* 686 * A huge PTE describes an area the size of the 687 * configured huge page size. This is twice the 688 * of the large TLB entry size we intend to use. 689 * A TLB entry half the size of the configured 690 * huge page size is configured into entrylo0 691 * and entrylo1 to cover the contiguous huge PTE 692 * address space. 693 */ 694 small_sequence = (HPAGE_SIZE >> 7) < 0x10000; 695 696 /* We can clobber tmp. It isn't used after this.*/ 697 if (!small_sequence) 698 uasm_i_lui(p, tmp, HPAGE_SIZE >> (7 + 16)); 699 700 build_convert_pte_to_entrylo(p, pte); 701 UASM_i_MTC0(p, pte, C0_ENTRYLO0); /* load it */ 702 /* convert to entrylo1 */ 703 if (small_sequence) 704 UASM_i_ADDIU(p, pte, pte, HPAGE_SIZE >> 7); 705 else 706 UASM_i_ADDU(p, pte, pte, tmp); 707 708 UASM_i_MTC0(p, pte, C0_ENTRYLO1); /* load it */ 709 } 710 711 static __cpuinit void build_huge_handler_tail(u32 **p, 712 struct uasm_reloc **r, 713 struct uasm_label **l, 714 unsigned int pte, 715 unsigned int ptr) 716 { 717 #ifdef CONFIG_SMP 718 UASM_i_SC(p, pte, 0, ptr); 719 uasm_il_beqz(p, r, pte, label_tlb_huge_update); 720 UASM_i_LW(p, pte, 0, ptr); /* Needed because SC killed our PTE */ 721 #else 722 UASM_i_SW(p, pte, 0, ptr); 723 #endif 724 build_huge_update_entries(p, pte, ptr); 725 build_huge_tlb_write_entry(p, l, r, pte, tlb_indexed, 0); 726 } 727 #endif /* CONFIG_HUGETLB_PAGE */ 728 729 #ifdef CONFIG_64BIT 730 /* 731 * TMP and PTR are scratch. 732 * TMP will be clobbered, PTR will hold the pmd entry. 733 */ 734 static void __cpuinit 735 build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r, 736 unsigned int tmp, unsigned int ptr) 737 { 738 #ifndef CONFIG_MIPS_PGD_C0_CONTEXT 739 long pgdc = (long)pgd_current; 740 #endif 741 /* 742 * The vmalloc handling is not in the hotpath. 743 */ 744 uasm_i_dmfc0(p, tmp, C0_BADVADDR); 745 746 if (check_for_high_segbits) { 747 /* 748 * The kernel currently implicitely assumes that the 749 * MIPS SEGBITS parameter for the processor is 750 * (PGDIR_SHIFT+PGDIR_BITS) or less, and will never 751 * allocate virtual addresses outside the maximum 752 * range for SEGBITS = (PGDIR_SHIFT+PGDIR_BITS). But 753 * that doesn't prevent user code from accessing the 754 * higher xuseg addresses. Here, we make sure that 755 * everything but the lower xuseg addresses goes down 756 * the module_alloc/vmalloc path. 757 */ 758 uasm_i_dsrl_safe(p, ptr, tmp, PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3); 759 uasm_il_bnez(p, r, ptr, label_vmalloc); 760 } else { 761 uasm_il_bltz(p, r, tmp, label_vmalloc); 762 } 763 /* No uasm_i_nop needed here, since the next insn doesn't touch TMP. */ 764 765 #ifdef CONFIG_MIPS_PGD_C0_CONTEXT 766 if (pgd_reg != -1) { 767 /* pgd is in pgd_reg */ 768 UASM_i_MFC0(p, ptr, 31, pgd_reg); 769 } else { 770 /* 771 * &pgd << 11 stored in CONTEXT [23..63]. 772 */ 773 UASM_i_MFC0(p, ptr, C0_CONTEXT); 774 775 /* Clear lower 23 bits of context. */ 776 uasm_i_dins(p, ptr, 0, 0, 23); 777 778 /* 1 0 1 0 1 << 6 xkphys cached */ 779 uasm_i_ori(p, ptr, ptr, 0x540); 780 uasm_i_drotr(p, ptr, ptr, 11); 781 } 782 #elif defined(CONFIG_SMP) 783 # ifdef CONFIG_MIPS_MT_SMTC 784 /* 785 * SMTC uses TCBind value as "CPU" index 786 */ 787 uasm_i_mfc0(p, ptr, C0_TCBIND); 788 uasm_i_dsrl_safe(p, ptr, ptr, 19); 789 # else 790 /* 791 * 64 bit SMP running in XKPHYS has smp_processor_id() << 3 792 * stored in CONTEXT. 793 */ 794 uasm_i_dmfc0(p, ptr, C0_CONTEXT); 795 uasm_i_dsrl_safe(p, ptr, ptr, 23); 796 # endif 797 UASM_i_LA_mostly(p, tmp, pgdc); 798 uasm_i_daddu(p, ptr, ptr, tmp); 799 uasm_i_dmfc0(p, tmp, C0_BADVADDR); 800 uasm_i_ld(p, ptr, uasm_rel_lo(pgdc), ptr); 801 #else 802 UASM_i_LA_mostly(p, ptr, pgdc); 803 uasm_i_ld(p, ptr, uasm_rel_lo(pgdc), ptr); 804 #endif 805 806 uasm_l_vmalloc_done(l, *p); 807 808 /* get pgd offset in bytes */ 809 uasm_i_dsrl_safe(p, tmp, tmp, PGDIR_SHIFT - 3); 810 811 uasm_i_andi(p, tmp, tmp, (PTRS_PER_PGD - 1)<<3); 812 uasm_i_daddu(p, ptr, ptr, tmp); /* add in pgd offset */ 813 #ifndef __PAGETABLE_PMD_FOLDED 814 uasm_i_dmfc0(p, tmp, C0_BADVADDR); /* get faulting address */ 815 uasm_i_ld(p, ptr, 0, ptr); /* get pmd pointer */ 816 uasm_i_dsrl_safe(p, tmp, tmp, PMD_SHIFT-3); /* get pmd offset in bytes */ 817 uasm_i_andi(p, tmp, tmp, (PTRS_PER_PMD - 1)<<3); 818 uasm_i_daddu(p, ptr, ptr, tmp); /* add in pmd offset */ 819 #endif 820 } 821 822 /* 823 * BVADDR is the faulting address, PTR is scratch. 824 * PTR will hold the pgd for vmalloc. 825 */ 826 static void __cpuinit 827 build_get_pgd_vmalloc64(u32 **p, struct uasm_label **l, struct uasm_reloc **r, 828 unsigned int bvaddr, unsigned int ptr, 829 enum vmalloc64_mode mode) 830 { 831 long swpd = (long)swapper_pg_dir; 832 int single_insn_swpd; 833 int did_vmalloc_branch = 0; 834 835 single_insn_swpd = uasm_in_compat_space_p(swpd) && !uasm_rel_lo(swpd); 836 837 uasm_l_vmalloc(l, *p); 838 839 if (mode != not_refill && check_for_high_segbits) { 840 if (single_insn_swpd) { 841 uasm_il_bltz(p, r, bvaddr, label_vmalloc_done); 842 uasm_i_lui(p, ptr, uasm_rel_hi(swpd)); 843 did_vmalloc_branch = 1; 844 /* fall through */ 845 } else { 846 uasm_il_bgez(p, r, bvaddr, label_large_segbits_fault); 847 } 848 } 849 if (!did_vmalloc_branch) { 850 if (uasm_in_compat_space_p(swpd) && !uasm_rel_lo(swpd)) { 851 uasm_il_b(p, r, label_vmalloc_done); 852 uasm_i_lui(p, ptr, uasm_rel_hi(swpd)); 853 } else { 854 UASM_i_LA_mostly(p, ptr, swpd); 855 uasm_il_b(p, r, label_vmalloc_done); 856 if (uasm_in_compat_space_p(swpd)) 857 uasm_i_addiu(p, ptr, ptr, uasm_rel_lo(swpd)); 858 else 859 uasm_i_daddiu(p, ptr, ptr, uasm_rel_lo(swpd)); 860 } 861 } 862 if (mode != not_refill && check_for_high_segbits) { 863 uasm_l_large_segbits_fault(l, *p); 864 /* 865 * We get here if we are an xsseg address, or if we are 866 * an xuseg address above (PGDIR_SHIFT+PGDIR_BITS) boundary. 867 * 868 * Ignoring xsseg (assume disabled so would generate 869 * (address errors?), the only remaining possibility 870 * is the upper xuseg addresses. On processors with 871 * TLB_SEGBITS <= PGDIR_SHIFT+PGDIR_BITS, these 872 * addresses would have taken an address error. We try 873 * to mimic that here by taking a load/istream page 874 * fault. 875 */ 876 UASM_i_LA(p, ptr, (unsigned long)tlb_do_page_fault_0); 877 uasm_i_jr(p, ptr); 878 879 if (mode == refill_scratch) { 880 if (scratch_reg > 0) 881 UASM_i_MFC0(p, 1, 31, scratch_reg); 882 else 883 UASM_i_LW(p, 1, scratchpad_offset(0), 0); 884 } else { 885 uasm_i_nop(p); 886 } 887 } 888 } 889 890 #else /* !CONFIG_64BIT */ 891 892 /* 893 * TMP and PTR are scratch. 894 * TMP will be clobbered, PTR will hold the pgd entry. 895 */ 896 static void __cpuinit __maybe_unused 897 build_get_pgde32(u32 **p, unsigned int tmp, unsigned int ptr) 898 { 899 long pgdc = (long)pgd_current; 900 901 /* 32 bit SMP has smp_processor_id() stored in CONTEXT. */ 902 #ifdef CONFIG_SMP 903 #ifdef CONFIG_MIPS_MT_SMTC 904 /* 905 * SMTC uses TCBind value as "CPU" index 906 */ 907 uasm_i_mfc0(p, ptr, C0_TCBIND); 908 UASM_i_LA_mostly(p, tmp, pgdc); 909 uasm_i_srl(p, ptr, ptr, 19); 910 #else 911 /* 912 * smp_processor_id() << 3 is stored in CONTEXT. 913 */ 914 uasm_i_mfc0(p, ptr, C0_CONTEXT); 915 UASM_i_LA_mostly(p, tmp, pgdc); 916 uasm_i_srl(p, ptr, ptr, 23); 917 #endif 918 uasm_i_addu(p, ptr, tmp, ptr); 919 #else 920 UASM_i_LA_mostly(p, ptr, pgdc); 921 #endif 922 uasm_i_mfc0(p, tmp, C0_BADVADDR); /* get faulting address */ 923 uasm_i_lw(p, ptr, uasm_rel_lo(pgdc), ptr); 924 uasm_i_srl(p, tmp, tmp, PGDIR_SHIFT); /* get pgd only bits */ 925 uasm_i_sll(p, tmp, tmp, PGD_T_LOG2); 926 uasm_i_addu(p, ptr, ptr, tmp); /* add in pgd offset */ 927 } 928 929 #endif /* !CONFIG_64BIT */ 930 931 static void __cpuinit build_adjust_context(u32 **p, unsigned int ctx) 932 { 933 unsigned int shift = 4 - (PTE_T_LOG2 + 1) + PAGE_SHIFT - 12; 934 unsigned int mask = (PTRS_PER_PTE / 2 - 1) << (PTE_T_LOG2 + 1); 935 936 switch (current_cpu_type()) { 937 case CPU_VR41XX: 938 case CPU_VR4111: 939 case CPU_VR4121: 940 case CPU_VR4122: 941 case CPU_VR4131: 942 case CPU_VR4181: 943 case CPU_VR4181A: 944 case CPU_VR4133: 945 shift += 2; 946 break; 947 948 default: 949 break; 950 } 951 952 if (shift) 953 UASM_i_SRL(p, ctx, ctx, shift); 954 uasm_i_andi(p, ctx, ctx, mask); 955 } 956 957 static void __cpuinit build_get_ptep(u32 **p, unsigned int tmp, unsigned int ptr) 958 { 959 /* 960 * Bug workaround for the Nevada. It seems as if under certain 961 * circumstances the move from cp0_context might produce a 962 * bogus result when the mfc0 instruction and its consumer are 963 * in a different cacheline or a load instruction, probably any 964 * memory reference, is between them. 965 */ 966 switch (current_cpu_type()) { 967 case CPU_NEVADA: 968 UASM_i_LW(p, ptr, 0, ptr); 969 GET_CONTEXT(p, tmp); /* get context reg */ 970 break; 971 972 default: 973 GET_CONTEXT(p, tmp); /* get context reg */ 974 UASM_i_LW(p, ptr, 0, ptr); 975 break; 976 } 977 978 build_adjust_context(p, tmp); 979 UASM_i_ADDU(p, ptr, ptr, tmp); /* add in offset */ 980 } 981 982 static void __cpuinit build_update_entries(u32 **p, unsigned int tmp, 983 unsigned int ptep) 984 { 985 /* 986 * 64bit address support (36bit on a 32bit CPU) in a 32bit 987 * Kernel is a special case. Only a few CPUs use it. 988 */ 989 #ifdef CONFIG_64BIT_PHYS_ADDR 990 if (cpu_has_64bits) { 991 uasm_i_ld(p, tmp, 0, ptep); /* get even pte */ 992 uasm_i_ld(p, ptep, sizeof(pte_t), ptep); /* get odd pte */ 993 if (kernel_uses_smartmips_rixi) { 994 UASM_i_SRL(p, tmp, tmp, ilog2(_PAGE_NO_EXEC)); 995 UASM_i_SRL(p, ptep, ptep, ilog2(_PAGE_NO_EXEC)); 996 UASM_i_ROTR(p, tmp, tmp, ilog2(_PAGE_GLOBAL) - ilog2(_PAGE_NO_EXEC)); 997 UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */ 998 UASM_i_ROTR(p, ptep, ptep, ilog2(_PAGE_GLOBAL) - ilog2(_PAGE_NO_EXEC)); 999 } else { 1000 uasm_i_dsrl_safe(p, tmp, tmp, ilog2(_PAGE_GLOBAL)); /* convert to entrylo0 */ 1001 UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */ 1002 uasm_i_dsrl_safe(p, ptep, ptep, ilog2(_PAGE_GLOBAL)); /* convert to entrylo1 */ 1003 } 1004 UASM_i_MTC0(p, ptep, C0_ENTRYLO1); /* load it */ 1005 } else { 1006 int pte_off_even = sizeof(pte_t) / 2; 1007 int pte_off_odd = pte_off_even + sizeof(pte_t); 1008 1009 /* The pte entries are pre-shifted */ 1010 uasm_i_lw(p, tmp, pte_off_even, ptep); /* get even pte */ 1011 UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */ 1012 uasm_i_lw(p, ptep, pte_off_odd, ptep); /* get odd pte */ 1013 UASM_i_MTC0(p, ptep, C0_ENTRYLO1); /* load it */ 1014 } 1015 #else 1016 UASM_i_LW(p, tmp, 0, ptep); /* get even pte */ 1017 UASM_i_LW(p, ptep, sizeof(pte_t), ptep); /* get odd pte */ 1018 if (r45k_bvahwbug()) 1019 build_tlb_probe_entry(p); 1020 if (kernel_uses_smartmips_rixi) { 1021 UASM_i_SRL(p, tmp, tmp, ilog2(_PAGE_NO_EXEC)); 1022 UASM_i_SRL(p, ptep, ptep, ilog2(_PAGE_NO_EXEC)); 1023 UASM_i_ROTR(p, tmp, tmp, ilog2(_PAGE_GLOBAL) - ilog2(_PAGE_NO_EXEC)); 1024 if (r4k_250MHZhwbug()) 1025 UASM_i_MTC0(p, 0, C0_ENTRYLO0); 1026 UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */ 1027 UASM_i_ROTR(p, ptep, ptep, ilog2(_PAGE_GLOBAL) - ilog2(_PAGE_NO_EXEC)); 1028 } else { 1029 UASM_i_SRL(p, tmp, tmp, ilog2(_PAGE_GLOBAL)); /* convert to entrylo0 */ 1030 if (r4k_250MHZhwbug()) 1031 UASM_i_MTC0(p, 0, C0_ENTRYLO0); 1032 UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */ 1033 UASM_i_SRL(p, ptep, ptep, ilog2(_PAGE_GLOBAL)); /* convert to entrylo1 */ 1034 if (r45k_bvahwbug()) 1035 uasm_i_mfc0(p, tmp, C0_INDEX); 1036 } 1037 if (r4k_250MHZhwbug()) 1038 UASM_i_MTC0(p, 0, C0_ENTRYLO1); 1039 UASM_i_MTC0(p, ptep, C0_ENTRYLO1); /* load it */ 1040 #endif 1041 } 1042 1043 struct mips_huge_tlb_info { 1044 int huge_pte; 1045 int restore_scratch; 1046 }; 1047 1048 static struct mips_huge_tlb_info __cpuinit 1049 build_fast_tlb_refill_handler (u32 **p, struct uasm_label **l, 1050 struct uasm_reloc **r, unsigned int tmp, 1051 unsigned int ptr, int c0_scratch) 1052 { 1053 struct mips_huge_tlb_info rv; 1054 unsigned int even, odd; 1055 int vmalloc_branch_delay_filled = 0; 1056 const int scratch = 1; /* Our extra working register */ 1057 1058 rv.huge_pte = scratch; 1059 rv.restore_scratch = 0; 1060 1061 if (check_for_high_segbits) { 1062 UASM_i_MFC0(p, tmp, C0_BADVADDR); 1063 1064 if (pgd_reg != -1) 1065 UASM_i_MFC0(p, ptr, 31, pgd_reg); 1066 else 1067 UASM_i_MFC0(p, ptr, C0_CONTEXT); 1068 1069 if (c0_scratch >= 0) 1070 UASM_i_MTC0(p, scratch, 31, c0_scratch); 1071 else 1072 UASM_i_SW(p, scratch, scratchpad_offset(0), 0); 1073 1074 uasm_i_dsrl_safe(p, scratch, tmp, 1075 PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3); 1076 uasm_il_bnez(p, r, scratch, label_vmalloc); 1077 1078 if (pgd_reg == -1) { 1079 vmalloc_branch_delay_filled = 1; 1080 /* Clear lower 23 bits of context. */ 1081 uasm_i_dins(p, ptr, 0, 0, 23); 1082 } 1083 } else { 1084 if (pgd_reg != -1) 1085 UASM_i_MFC0(p, ptr, 31, pgd_reg); 1086 else 1087 UASM_i_MFC0(p, ptr, C0_CONTEXT); 1088 1089 UASM_i_MFC0(p, tmp, C0_BADVADDR); 1090 1091 if (c0_scratch >= 0) 1092 UASM_i_MTC0(p, scratch, 31, c0_scratch); 1093 else 1094 UASM_i_SW(p, scratch, scratchpad_offset(0), 0); 1095 1096 if (pgd_reg == -1) 1097 /* Clear lower 23 bits of context. */ 1098 uasm_i_dins(p, ptr, 0, 0, 23); 1099 1100 uasm_il_bltz(p, r, tmp, label_vmalloc); 1101 } 1102 1103 if (pgd_reg == -1) { 1104 vmalloc_branch_delay_filled = 1; 1105 /* 1 0 1 0 1 << 6 xkphys cached */ 1106 uasm_i_ori(p, ptr, ptr, 0x540); 1107 uasm_i_drotr(p, ptr, ptr, 11); 1108 } 1109 1110 #ifdef __PAGETABLE_PMD_FOLDED 1111 #define LOC_PTEP scratch 1112 #else 1113 #define LOC_PTEP ptr 1114 #endif 1115 1116 if (!vmalloc_branch_delay_filled) 1117 /* get pgd offset in bytes */ 1118 uasm_i_dsrl_safe(p, scratch, tmp, PGDIR_SHIFT - 3); 1119 1120 uasm_l_vmalloc_done(l, *p); 1121 1122 /* 1123 * tmp ptr 1124 * fall-through case = badvaddr *pgd_current 1125 * vmalloc case = badvaddr swapper_pg_dir 1126 */ 1127 1128 if (vmalloc_branch_delay_filled) 1129 /* get pgd offset in bytes */ 1130 uasm_i_dsrl_safe(p, scratch, tmp, PGDIR_SHIFT - 3); 1131 1132 #ifdef __PAGETABLE_PMD_FOLDED 1133 GET_CONTEXT(p, tmp); /* get context reg */ 1134 #endif 1135 uasm_i_andi(p, scratch, scratch, (PTRS_PER_PGD - 1) << 3); 1136 1137 if (use_lwx_insns()) { 1138 UASM_i_LWX(p, LOC_PTEP, scratch, ptr); 1139 } else { 1140 uasm_i_daddu(p, ptr, ptr, scratch); /* add in pgd offset */ 1141 uasm_i_ld(p, LOC_PTEP, 0, ptr); /* get pmd pointer */ 1142 } 1143 1144 #ifndef __PAGETABLE_PMD_FOLDED 1145 /* get pmd offset in bytes */ 1146 uasm_i_dsrl_safe(p, scratch, tmp, PMD_SHIFT - 3); 1147 uasm_i_andi(p, scratch, scratch, (PTRS_PER_PMD - 1) << 3); 1148 GET_CONTEXT(p, tmp); /* get context reg */ 1149 1150 if (use_lwx_insns()) { 1151 UASM_i_LWX(p, scratch, scratch, ptr); 1152 } else { 1153 uasm_i_daddu(p, ptr, ptr, scratch); /* add in pmd offset */ 1154 UASM_i_LW(p, scratch, 0, ptr); 1155 } 1156 #endif 1157 /* Adjust the context during the load latency. */ 1158 build_adjust_context(p, tmp); 1159 1160 #ifdef CONFIG_HUGETLB_PAGE 1161 uasm_il_bbit1(p, r, scratch, ilog2(_PAGE_HUGE), label_tlb_huge_update); 1162 /* 1163 * The in the LWX case we don't want to do the load in the 1164 * delay slot. It cannot issue in the same cycle and may be 1165 * speculative and unneeded. 1166 */ 1167 if (use_lwx_insns()) 1168 uasm_i_nop(p); 1169 #endif /* CONFIG_HUGETLB_PAGE */ 1170 1171 1172 /* build_update_entries */ 1173 if (use_lwx_insns()) { 1174 even = ptr; 1175 odd = tmp; 1176 UASM_i_LWX(p, even, scratch, tmp); 1177 UASM_i_ADDIU(p, tmp, tmp, sizeof(pte_t)); 1178 UASM_i_LWX(p, odd, scratch, tmp); 1179 } else { 1180 UASM_i_ADDU(p, ptr, scratch, tmp); /* add in offset */ 1181 even = tmp; 1182 odd = ptr; 1183 UASM_i_LW(p, even, 0, ptr); /* get even pte */ 1184 UASM_i_LW(p, odd, sizeof(pte_t), ptr); /* get odd pte */ 1185 } 1186 if (kernel_uses_smartmips_rixi) { 1187 uasm_i_dsrl_safe(p, even, even, ilog2(_PAGE_NO_EXEC)); 1188 uasm_i_dsrl_safe(p, odd, odd, ilog2(_PAGE_NO_EXEC)); 1189 uasm_i_drotr(p, even, even, 1190 ilog2(_PAGE_GLOBAL) - ilog2(_PAGE_NO_EXEC)); 1191 UASM_i_MTC0(p, even, C0_ENTRYLO0); /* load it */ 1192 uasm_i_drotr(p, odd, odd, 1193 ilog2(_PAGE_GLOBAL) - ilog2(_PAGE_NO_EXEC)); 1194 } else { 1195 uasm_i_dsrl_safe(p, even, even, ilog2(_PAGE_GLOBAL)); 1196 UASM_i_MTC0(p, even, C0_ENTRYLO0); /* load it */ 1197 uasm_i_dsrl_safe(p, odd, odd, ilog2(_PAGE_GLOBAL)); 1198 } 1199 UASM_i_MTC0(p, odd, C0_ENTRYLO1); /* load it */ 1200 1201 if (c0_scratch >= 0) { 1202 UASM_i_MFC0(p, scratch, 31, c0_scratch); 1203 build_tlb_write_entry(p, l, r, tlb_random); 1204 uasm_l_leave(l, *p); 1205 rv.restore_scratch = 1; 1206 } else if (PAGE_SHIFT == 14 || PAGE_SHIFT == 13) { 1207 build_tlb_write_entry(p, l, r, tlb_random); 1208 uasm_l_leave(l, *p); 1209 UASM_i_LW(p, scratch, scratchpad_offset(0), 0); 1210 } else { 1211 UASM_i_LW(p, scratch, scratchpad_offset(0), 0); 1212 build_tlb_write_entry(p, l, r, tlb_random); 1213 uasm_l_leave(l, *p); 1214 rv.restore_scratch = 1; 1215 } 1216 1217 uasm_i_eret(p); /* return from trap */ 1218 1219 return rv; 1220 } 1221 1222 /* 1223 * For a 64-bit kernel, we are using the 64-bit XTLB refill exception 1224 * because EXL == 0. If we wrap, we can also use the 32 instruction 1225 * slots before the XTLB refill exception handler which belong to the 1226 * unused TLB refill exception. 1227 */ 1228 #define MIPS64_REFILL_INSNS 32 1229 1230 static void __cpuinit build_r4000_tlb_refill_handler(void) 1231 { 1232 u32 *p = tlb_handler; 1233 struct uasm_label *l = labels; 1234 struct uasm_reloc *r = relocs; 1235 u32 *f; 1236 unsigned int final_len; 1237 struct mips_huge_tlb_info htlb_info __maybe_unused; 1238 enum vmalloc64_mode vmalloc_mode __maybe_unused; 1239 1240 memset(tlb_handler, 0, sizeof(tlb_handler)); 1241 memset(labels, 0, sizeof(labels)); 1242 memset(relocs, 0, sizeof(relocs)); 1243 memset(final_handler, 0, sizeof(final_handler)); 1244 1245 if ((scratch_reg > 0 || scratchpad_available()) && use_bbit_insns()) { 1246 htlb_info = build_fast_tlb_refill_handler(&p, &l, &r, K0, K1, 1247 scratch_reg); 1248 vmalloc_mode = refill_scratch; 1249 } else { 1250 htlb_info.huge_pte = K0; 1251 htlb_info.restore_scratch = 0; 1252 vmalloc_mode = refill_noscratch; 1253 /* 1254 * create the plain linear handler 1255 */ 1256 if (bcm1250_m3_war()) { 1257 unsigned int segbits = 44; 1258 1259 uasm_i_dmfc0(&p, K0, C0_BADVADDR); 1260 uasm_i_dmfc0(&p, K1, C0_ENTRYHI); 1261 uasm_i_xor(&p, K0, K0, K1); 1262 uasm_i_dsrl_safe(&p, K1, K0, 62); 1263 uasm_i_dsrl_safe(&p, K0, K0, 12 + 1); 1264 uasm_i_dsll_safe(&p, K0, K0, 64 + 12 + 1 - segbits); 1265 uasm_i_or(&p, K0, K0, K1); 1266 uasm_il_bnez(&p, &r, K0, label_leave); 1267 /* No need for uasm_i_nop */ 1268 } 1269 1270 #ifdef CONFIG_64BIT 1271 build_get_pmde64(&p, &l, &r, K0, K1); /* get pmd in K1 */ 1272 #else 1273 build_get_pgde32(&p, K0, K1); /* get pgd in K1 */ 1274 #endif 1275 1276 #ifdef CONFIG_HUGETLB_PAGE 1277 build_is_huge_pte(&p, &r, K0, K1, label_tlb_huge_update); 1278 #endif 1279 1280 build_get_ptep(&p, K0, K1); 1281 build_update_entries(&p, K0, K1); 1282 build_tlb_write_entry(&p, &l, &r, tlb_random); 1283 uasm_l_leave(&l, p); 1284 uasm_i_eret(&p); /* return from trap */ 1285 } 1286 #ifdef CONFIG_HUGETLB_PAGE 1287 uasm_l_tlb_huge_update(&l, p); 1288 build_huge_update_entries(&p, htlb_info.huge_pte, K1); 1289 build_huge_tlb_write_entry(&p, &l, &r, K0, tlb_random, 1290 htlb_info.restore_scratch); 1291 #endif 1292 1293 #ifdef CONFIG_64BIT 1294 build_get_pgd_vmalloc64(&p, &l, &r, K0, K1, vmalloc_mode); 1295 #endif 1296 1297 /* 1298 * Overflow check: For the 64bit handler, we need at least one 1299 * free instruction slot for the wrap-around branch. In worst 1300 * case, if the intended insertion point is a delay slot, we 1301 * need three, with the second nop'ed and the third being 1302 * unused. 1303 */ 1304 /* Loongson2 ebase is different than r4k, we have more space */ 1305 #if defined(CONFIG_32BIT) || defined(CONFIG_CPU_LOONGSON2) 1306 if ((p - tlb_handler) > 64) 1307 panic("TLB refill handler space exceeded"); 1308 #else 1309 if (((p - tlb_handler) > (MIPS64_REFILL_INSNS * 2) - 1) 1310 || (((p - tlb_handler) > (MIPS64_REFILL_INSNS * 2) - 3) 1311 && uasm_insn_has_bdelay(relocs, 1312 tlb_handler + MIPS64_REFILL_INSNS - 3))) 1313 panic("TLB refill handler space exceeded"); 1314 #endif 1315 1316 /* 1317 * Now fold the handler in the TLB refill handler space. 1318 */ 1319 #if defined(CONFIG_32BIT) || defined(CONFIG_CPU_LOONGSON2) 1320 f = final_handler; 1321 /* Simplest case, just copy the handler. */ 1322 uasm_copy_handler(relocs, labels, tlb_handler, p, f); 1323 final_len = p - tlb_handler; 1324 #else /* CONFIG_64BIT */ 1325 f = final_handler + MIPS64_REFILL_INSNS; 1326 if ((p - tlb_handler) <= MIPS64_REFILL_INSNS) { 1327 /* Just copy the handler. */ 1328 uasm_copy_handler(relocs, labels, tlb_handler, p, f); 1329 final_len = p - tlb_handler; 1330 } else { 1331 #if defined(CONFIG_HUGETLB_PAGE) 1332 const enum label_id ls = label_tlb_huge_update; 1333 #else 1334 const enum label_id ls = label_vmalloc; 1335 #endif 1336 u32 *split; 1337 int ov = 0; 1338 int i; 1339 1340 for (i = 0; i < ARRAY_SIZE(labels) && labels[i].lab != ls; i++) 1341 ; 1342 BUG_ON(i == ARRAY_SIZE(labels)); 1343 split = labels[i].addr; 1344 1345 /* 1346 * See if we have overflown one way or the other. 1347 */ 1348 if (split > tlb_handler + MIPS64_REFILL_INSNS || 1349 split < p - MIPS64_REFILL_INSNS) 1350 ov = 1; 1351 1352 if (ov) { 1353 /* 1354 * Split two instructions before the end. One 1355 * for the branch and one for the instruction 1356 * in the delay slot. 1357 */ 1358 split = tlb_handler + MIPS64_REFILL_INSNS - 2; 1359 1360 /* 1361 * If the branch would fall in a delay slot, 1362 * we must back up an additional instruction 1363 * so that it is no longer in a delay slot. 1364 */ 1365 if (uasm_insn_has_bdelay(relocs, split - 1)) 1366 split--; 1367 } 1368 /* Copy first part of the handler. */ 1369 uasm_copy_handler(relocs, labels, tlb_handler, split, f); 1370 f += split - tlb_handler; 1371 1372 if (ov) { 1373 /* Insert branch. */ 1374 uasm_l_split(&l, final_handler); 1375 uasm_il_b(&f, &r, label_split); 1376 if (uasm_insn_has_bdelay(relocs, split)) 1377 uasm_i_nop(&f); 1378 else { 1379 uasm_copy_handler(relocs, labels, 1380 split, split + 1, f); 1381 uasm_move_labels(labels, f, f + 1, -1); 1382 f++; 1383 split++; 1384 } 1385 } 1386 1387 /* Copy the rest of the handler. */ 1388 uasm_copy_handler(relocs, labels, split, p, final_handler); 1389 final_len = (f - (final_handler + MIPS64_REFILL_INSNS)) + 1390 (p - split); 1391 } 1392 #endif /* CONFIG_64BIT */ 1393 1394 uasm_resolve_relocs(relocs, labels); 1395 pr_debug("Wrote TLB refill handler (%u instructions).\n", 1396 final_len); 1397 1398 memcpy((void *)ebase, final_handler, 0x100); 1399 1400 dump_handler((u32 *)ebase, 64); 1401 } 1402 1403 /* 1404 * 128 instructions for the fastpath handler is generous and should 1405 * never be exceeded. 1406 */ 1407 #define FASTPATH_SIZE 128 1408 1409 u32 handle_tlbl[FASTPATH_SIZE] __cacheline_aligned; 1410 u32 handle_tlbs[FASTPATH_SIZE] __cacheline_aligned; 1411 u32 handle_tlbm[FASTPATH_SIZE] __cacheline_aligned; 1412 #ifdef CONFIG_MIPS_PGD_C0_CONTEXT 1413 u32 tlbmiss_handler_setup_pgd[16] __cacheline_aligned; 1414 1415 static void __cpuinit build_r4000_setup_pgd(void) 1416 { 1417 const int a0 = 4; 1418 const int a1 = 5; 1419 u32 *p = tlbmiss_handler_setup_pgd; 1420 struct uasm_label *l = labels; 1421 struct uasm_reloc *r = relocs; 1422 1423 memset(tlbmiss_handler_setup_pgd, 0, sizeof(tlbmiss_handler_setup_pgd)); 1424 memset(labels, 0, sizeof(labels)); 1425 memset(relocs, 0, sizeof(relocs)); 1426 1427 pgd_reg = allocate_kscratch(); 1428 1429 if (pgd_reg == -1) { 1430 /* PGD << 11 in c0_Context */ 1431 /* 1432 * If it is a ckseg0 address, convert to a physical 1433 * address. Shifting right by 29 and adding 4 will 1434 * result in zero for these addresses. 1435 * 1436 */ 1437 UASM_i_SRA(&p, a1, a0, 29); 1438 UASM_i_ADDIU(&p, a1, a1, 4); 1439 uasm_il_bnez(&p, &r, a1, label_tlbl_goaround1); 1440 uasm_i_nop(&p); 1441 uasm_i_dinsm(&p, a0, 0, 29, 64 - 29); 1442 uasm_l_tlbl_goaround1(&l, p); 1443 UASM_i_SLL(&p, a0, a0, 11); 1444 uasm_i_jr(&p, 31); 1445 UASM_i_MTC0(&p, a0, C0_CONTEXT); 1446 } else { 1447 /* PGD in c0_KScratch */ 1448 uasm_i_jr(&p, 31); 1449 UASM_i_MTC0(&p, a0, 31, pgd_reg); 1450 } 1451 if (p - tlbmiss_handler_setup_pgd > ARRAY_SIZE(tlbmiss_handler_setup_pgd)) 1452 panic("tlbmiss_handler_setup_pgd space exceeded"); 1453 uasm_resolve_relocs(relocs, labels); 1454 pr_debug("Wrote tlbmiss_handler_setup_pgd (%u instructions).\n", 1455 (unsigned int)(p - tlbmiss_handler_setup_pgd)); 1456 1457 dump_handler(tlbmiss_handler_setup_pgd, 1458 ARRAY_SIZE(tlbmiss_handler_setup_pgd)); 1459 } 1460 #endif 1461 1462 static void __cpuinit 1463 iPTE_LW(u32 **p, unsigned int pte, unsigned int ptr) 1464 { 1465 #ifdef CONFIG_SMP 1466 # ifdef CONFIG_64BIT_PHYS_ADDR 1467 if (cpu_has_64bits) 1468 uasm_i_lld(p, pte, 0, ptr); 1469 else 1470 # endif 1471 UASM_i_LL(p, pte, 0, ptr); 1472 #else 1473 # ifdef CONFIG_64BIT_PHYS_ADDR 1474 if (cpu_has_64bits) 1475 uasm_i_ld(p, pte, 0, ptr); 1476 else 1477 # endif 1478 UASM_i_LW(p, pte, 0, ptr); 1479 #endif 1480 } 1481 1482 static void __cpuinit 1483 iPTE_SW(u32 **p, struct uasm_reloc **r, unsigned int pte, unsigned int ptr, 1484 unsigned int mode) 1485 { 1486 #ifdef CONFIG_64BIT_PHYS_ADDR 1487 unsigned int hwmode = mode & (_PAGE_VALID | _PAGE_DIRTY); 1488 #endif 1489 1490 uasm_i_ori(p, pte, pte, mode); 1491 #ifdef CONFIG_SMP 1492 # ifdef CONFIG_64BIT_PHYS_ADDR 1493 if (cpu_has_64bits) 1494 uasm_i_scd(p, pte, 0, ptr); 1495 else 1496 # endif 1497 UASM_i_SC(p, pte, 0, ptr); 1498 1499 if (r10000_llsc_war()) 1500 uasm_il_beqzl(p, r, pte, label_smp_pgtable_change); 1501 else 1502 uasm_il_beqz(p, r, pte, label_smp_pgtable_change); 1503 1504 # ifdef CONFIG_64BIT_PHYS_ADDR 1505 if (!cpu_has_64bits) { 1506 /* no uasm_i_nop needed */ 1507 uasm_i_ll(p, pte, sizeof(pte_t) / 2, ptr); 1508 uasm_i_ori(p, pte, pte, hwmode); 1509 uasm_i_sc(p, pte, sizeof(pte_t) / 2, ptr); 1510 uasm_il_beqz(p, r, pte, label_smp_pgtable_change); 1511 /* no uasm_i_nop needed */ 1512 uasm_i_lw(p, pte, 0, ptr); 1513 } else 1514 uasm_i_nop(p); 1515 # else 1516 uasm_i_nop(p); 1517 # endif 1518 #else 1519 # ifdef CONFIG_64BIT_PHYS_ADDR 1520 if (cpu_has_64bits) 1521 uasm_i_sd(p, pte, 0, ptr); 1522 else 1523 # endif 1524 UASM_i_SW(p, pte, 0, ptr); 1525 1526 # ifdef CONFIG_64BIT_PHYS_ADDR 1527 if (!cpu_has_64bits) { 1528 uasm_i_lw(p, pte, sizeof(pte_t) / 2, ptr); 1529 uasm_i_ori(p, pte, pte, hwmode); 1530 uasm_i_sw(p, pte, sizeof(pte_t) / 2, ptr); 1531 uasm_i_lw(p, pte, 0, ptr); 1532 } 1533 # endif 1534 #endif 1535 } 1536 1537 /* 1538 * Check if PTE is present, if not then jump to LABEL. PTR points to 1539 * the page table where this PTE is located, PTE will be re-loaded 1540 * with it's original value. 1541 */ 1542 static void __cpuinit 1543 build_pte_present(u32 **p, struct uasm_reloc **r, 1544 int pte, int ptr, int scratch, enum label_id lid) 1545 { 1546 int t = scratch >= 0 ? scratch : pte; 1547 1548 if (kernel_uses_smartmips_rixi) { 1549 if (use_bbit_insns()) { 1550 uasm_il_bbit0(p, r, pte, ilog2(_PAGE_PRESENT), lid); 1551 uasm_i_nop(p); 1552 } else { 1553 uasm_i_andi(p, t, pte, _PAGE_PRESENT); 1554 uasm_il_beqz(p, r, t, lid); 1555 if (pte == t) 1556 /* You lose the SMP race :-(*/ 1557 iPTE_LW(p, pte, ptr); 1558 } 1559 } else { 1560 uasm_i_andi(p, t, pte, _PAGE_PRESENT | _PAGE_READ); 1561 uasm_i_xori(p, t, t, _PAGE_PRESENT | _PAGE_READ); 1562 uasm_il_bnez(p, r, t, lid); 1563 if (pte == t) 1564 /* You lose the SMP race :-(*/ 1565 iPTE_LW(p, pte, ptr); 1566 } 1567 } 1568 1569 /* Make PTE valid, store result in PTR. */ 1570 static void __cpuinit 1571 build_make_valid(u32 **p, struct uasm_reloc **r, unsigned int pte, 1572 unsigned int ptr) 1573 { 1574 unsigned int mode = _PAGE_VALID | _PAGE_ACCESSED; 1575 1576 iPTE_SW(p, r, pte, ptr, mode); 1577 } 1578 1579 /* 1580 * Check if PTE can be written to, if not branch to LABEL. Regardless 1581 * restore PTE with value from PTR when done. 1582 */ 1583 static void __cpuinit 1584 build_pte_writable(u32 **p, struct uasm_reloc **r, 1585 unsigned int pte, unsigned int ptr, int scratch, 1586 enum label_id lid) 1587 { 1588 int t = scratch >= 0 ? scratch : pte; 1589 1590 uasm_i_andi(p, t, pte, _PAGE_PRESENT | _PAGE_WRITE); 1591 uasm_i_xori(p, t, t, _PAGE_PRESENT | _PAGE_WRITE); 1592 uasm_il_bnez(p, r, t, lid); 1593 if (pte == t) 1594 /* You lose the SMP race :-(*/ 1595 iPTE_LW(p, pte, ptr); 1596 else 1597 uasm_i_nop(p); 1598 } 1599 1600 /* Make PTE writable, update software status bits as well, then store 1601 * at PTR. 1602 */ 1603 static void __cpuinit 1604 build_make_write(u32 **p, struct uasm_reloc **r, unsigned int pte, 1605 unsigned int ptr) 1606 { 1607 unsigned int mode = (_PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID 1608 | _PAGE_DIRTY); 1609 1610 iPTE_SW(p, r, pte, ptr, mode); 1611 } 1612 1613 /* 1614 * Check if PTE can be modified, if not branch to LABEL. Regardless 1615 * restore PTE with value from PTR when done. 1616 */ 1617 static void __cpuinit 1618 build_pte_modifiable(u32 **p, struct uasm_reloc **r, 1619 unsigned int pte, unsigned int ptr, int scratch, 1620 enum label_id lid) 1621 { 1622 if (use_bbit_insns()) { 1623 uasm_il_bbit0(p, r, pte, ilog2(_PAGE_WRITE), lid); 1624 uasm_i_nop(p); 1625 } else { 1626 int t = scratch >= 0 ? scratch : pte; 1627 uasm_i_andi(p, t, pte, _PAGE_WRITE); 1628 uasm_il_beqz(p, r, t, lid); 1629 if (pte == t) 1630 /* You lose the SMP race :-(*/ 1631 iPTE_LW(p, pte, ptr); 1632 } 1633 } 1634 1635 #ifndef CONFIG_MIPS_PGD_C0_CONTEXT 1636 1637 1638 /* 1639 * R3000 style TLB load/store/modify handlers. 1640 */ 1641 1642 /* 1643 * This places the pte into ENTRYLO0 and writes it with tlbwi. 1644 * Then it returns. 1645 */ 1646 static void __cpuinit 1647 build_r3000_pte_reload_tlbwi(u32 **p, unsigned int pte, unsigned int tmp) 1648 { 1649 uasm_i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */ 1650 uasm_i_mfc0(p, tmp, C0_EPC); /* cp0 delay */ 1651 uasm_i_tlbwi(p); 1652 uasm_i_jr(p, tmp); 1653 uasm_i_rfe(p); /* branch delay */ 1654 } 1655 1656 /* 1657 * This places the pte into ENTRYLO0 and writes it with tlbwi 1658 * or tlbwr as appropriate. This is because the index register 1659 * may have the probe fail bit set as a result of a trap on a 1660 * kseg2 access, i.e. without refill. Then it returns. 1661 */ 1662 static void __cpuinit 1663 build_r3000_tlb_reload_write(u32 **p, struct uasm_label **l, 1664 struct uasm_reloc **r, unsigned int pte, 1665 unsigned int tmp) 1666 { 1667 uasm_i_mfc0(p, tmp, C0_INDEX); 1668 uasm_i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */ 1669 uasm_il_bltz(p, r, tmp, label_r3000_write_probe_fail); /* cp0 delay */ 1670 uasm_i_mfc0(p, tmp, C0_EPC); /* branch delay */ 1671 uasm_i_tlbwi(p); /* cp0 delay */ 1672 uasm_i_jr(p, tmp); 1673 uasm_i_rfe(p); /* branch delay */ 1674 uasm_l_r3000_write_probe_fail(l, *p); 1675 uasm_i_tlbwr(p); /* cp0 delay */ 1676 uasm_i_jr(p, tmp); 1677 uasm_i_rfe(p); /* branch delay */ 1678 } 1679 1680 static void __cpuinit 1681 build_r3000_tlbchange_handler_head(u32 **p, unsigned int pte, 1682 unsigned int ptr) 1683 { 1684 long pgdc = (long)pgd_current; 1685 1686 uasm_i_mfc0(p, pte, C0_BADVADDR); 1687 uasm_i_lui(p, ptr, uasm_rel_hi(pgdc)); /* cp0 delay */ 1688 uasm_i_lw(p, ptr, uasm_rel_lo(pgdc), ptr); 1689 uasm_i_srl(p, pte, pte, 22); /* load delay */ 1690 uasm_i_sll(p, pte, pte, 2); 1691 uasm_i_addu(p, ptr, ptr, pte); 1692 uasm_i_mfc0(p, pte, C0_CONTEXT); 1693 uasm_i_lw(p, ptr, 0, ptr); /* cp0 delay */ 1694 uasm_i_andi(p, pte, pte, 0xffc); /* load delay */ 1695 uasm_i_addu(p, ptr, ptr, pte); 1696 uasm_i_lw(p, pte, 0, ptr); 1697 uasm_i_tlbp(p); /* load delay */ 1698 } 1699 1700 static void __cpuinit build_r3000_tlb_load_handler(void) 1701 { 1702 u32 *p = handle_tlbl; 1703 struct uasm_label *l = labels; 1704 struct uasm_reloc *r = relocs; 1705 1706 memset(handle_tlbl, 0, sizeof(handle_tlbl)); 1707 memset(labels, 0, sizeof(labels)); 1708 memset(relocs, 0, sizeof(relocs)); 1709 1710 build_r3000_tlbchange_handler_head(&p, K0, K1); 1711 build_pte_present(&p, &r, K0, K1, -1, label_nopage_tlbl); 1712 uasm_i_nop(&p); /* load delay */ 1713 build_make_valid(&p, &r, K0, K1); 1714 build_r3000_tlb_reload_write(&p, &l, &r, K0, K1); 1715 1716 uasm_l_nopage_tlbl(&l, p); 1717 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff); 1718 uasm_i_nop(&p); 1719 1720 if ((p - handle_tlbl) > FASTPATH_SIZE) 1721 panic("TLB load handler fastpath space exceeded"); 1722 1723 uasm_resolve_relocs(relocs, labels); 1724 pr_debug("Wrote TLB load handler fastpath (%u instructions).\n", 1725 (unsigned int)(p - handle_tlbl)); 1726 1727 dump_handler(handle_tlbl, ARRAY_SIZE(handle_tlbl)); 1728 } 1729 1730 static void __cpuinit build_r3000_tlb_store_handler(void) 1731 { 1732 u32 *p = handle_tlbs; 1733 struct uasm_label *l = labels; 1734 struct uasm_reloc *r = relocs; 1735 1736 memset(handle_tlbs, 0, sizeof(handle_tlbs)); 1737 memset(labels, 0, sizeof(labels)); 1738 memset(relocs, 0, sizeof(relocs)); 1739 1740 build_r3000_tlbchange_handler_head(&p, K0, K1); 1741 build_pte_writable(&p, &r, K0, K1, -1, label_nopage_tlbs); 1742 uasm_i_nop(&p); /* load delay */ 1743 build_make_write(&p, &r, K0, K1); 1744 build_r3000_tlb_reload_write(&p, &l, &r, K0, K1); 1745 1746 uasm_l_nopage_tlbs(&l, p); 1747 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); 1748 uasm_i_nop(&p); 1749 1750 if ((p - handle_tlbs) > FASTPATH_SIZE) 1751 panic("TLB store handler fastpath space exceeded"); 1752 1753 uasm_resolve_relocs(relocs, labels); 1754 pr_debug("Wrote TLB store handler fastpath (%u instructions).\n", 1755 (unsigned int)(p - handle_tlbs)); 1756 1757 dump_handler(handle_tlbs, ARRAY_SIZE(handle_tlbs)); 1758 } 1759 1760 static void __cpuinit build_r3000_tlb_modify_handler(void) 1761 { 1762 u32 *p = handle_tlbm; 1763 struct uasm_label *l = labels; 1764 struct uasm_reloc *r = relocs; 1765 1766 memset(handle_tlbm, 0, sizeof(handle_tlbm)); 1767 memset(labels, 0, sizeof(labels)); 1768 memset(relocs, 0, sizeof(relocs)); 1769 1770 build_r3000_tlbchange_handler_head(&p, K0, K1); 1771 build_pte_modifiable(&p, &r, K0, K1, -1, label_nopage_tlbm); 1772 uasm_i_nop(&p); /* load delay */ 1773 build_make_write(&p, &r, K0, K1); 1774 build_r3000_pte_reload_tlbwi(&p, K0, K1); 1775 1776 uasm_l_nopage_tlbm(&l, p); 1777 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); 1778 uasm_i_nop(&p); 1779 1780 if ((p - handle_tlbm) > FASTPATH_SIZE) 1781 panic("TLB modify handler fastpath space exceeded"); 1782 1783 uasm_resolve_relocs(relocs, labels); 1784 pr_debug("Wrote TLB modify handler fastpath (%u instructions).\n", 1785 (unsigned int)(p - handle_tlbm)); 1786 1787 dump_handler(handle_tlbm, ARRAY_SIZE(handle_tlbm)); 1788 } 1789 #endif /* CONFIG_MIPS_PGD_C0_CONTEXT */ 1790 1791 /* 1792 * R4000 style TLB load/store/modify handlers. 1793 */ 1794 static struct work_registers __cpuinit 1795 build_r4000_tlbchange_handler_head(u32 **p, struct uasm_label **l, 1796 struct uasm_reloc **r) 1797 { 1798 struct work_registers wr = build_get_work_registers(p); 1799 1800 #ifdef CONFIG_64BIT 1801 build_get_pmde64(p, l, r, wr.r1, wr.r2); /* get pmd in ptr */ 1802 #else 1803 build_get_pgde32(p, wr.r1, wr.r2); /* get pgd in ptr */ 1804 #endif 1805 1806 #ifdef CONFIG_HUGETLB_PAGE 1807 /* 1808 * For huge tlb entries, pmd doesn't contain an address but 1809 * instead contains the tlb pte. Check the PAGE_HUGE bit and 1810 * see if we need to jump to huge tlb processing. 1811 */ 1812 build_is_huge_pte(p, r, wr.r1, wr.r2, label_tlb_huge_update); 1813 #endif 1814 1815 UASM_i_MFC0(p, wr.r1, C0_BADVADDR); 1816 UASM_i_LW(p, wr.r2, 0, wr.r2); 1817 UASM_i_SRL(p, wr.r1, wr.r1, PAGE_SHIFT + PTE_ORDER - PTE_T_LOG2); 1818 uasm_i_andi(p, wr.r1, wr.r1, (PTRS_PER_PTE - 1) << PTE_T_LOG2); 1819 UASM_i_ADDU(p, wr.r2, wr.r2, wr.r1); 1820 1821 #ifdef CONFIG_SMP 1822 uasm_l_smp_pgtable_change(l, *p); 1823 #endif 1824 iPTE_LW(p, wr.r1, wr.r2); /* get even pte */ 1825 if (!m4kc_tlbp_war()) 1826 build_tlb_probe_entry(p); 1827 return wr; 1828 } 1829 1830 static void __cpuinit 1831 build_r4000_tlbchange_handler_tail(u32 **p, struct uasm_label **l, 1832 struct uasm_reloc **r, unsigned int tmp, 1833 unsigned int ptr) 1834 { 1835 uasm_i_ori(p, ptr, ptr, sizeof(pte_t)); 1836 uasm_i_xori(p, ptr, ptr, sizeof(pte_t)); 1837 build_update_entries(p, tmp, ptr); 1838 build_tlb_write_entry(p, l, r, tlb_indexed); 1839 uasm_l_leave(l, *p); 1840 build_restore_work_registers(p); 1841 uasm_i_eret(p); /* return from trap */ 1842 1843 #ifdef CONFIG_64BIT 1844 build_get_pgd_vmalloc64(p, l, r, tmp, ptr, not_refill); 1845 #endif 1846 } 1847 1848 static void __cpuinit build_r4000_tlb_load_handler(void) 1849 { 1850 u32 *p = handle_tlbl; 1851 struct uasm_label *l = labels; 1852 struct uasm_reloc *r = relocs; 1853 struct work_registers wr; 1854 1855 memset(handle_tlbl, 0, sizeof(handle_tlbl)); 1856 memset(labels, 0, sizeof(labels)); 1857 memset(relocs, 0, sizeof(relocs)); 1858 1859 if (bcm1250_m3_war()) { 1860 unsigned int segbits = 44; 1861 1862 uasm_i_dmfc0(&p, K0, C0_BADVADDR); 1863 uasm_i_dmfc0(&p, K1, C0_ENTRYHI); 1864 uasm_i_xor(&p, K0, K0, K1); 1865 uasm_i_dsrl_safe(&p, K1, K0, 62); 1866 uasm_i_dsrl_safe(&p, K0, K0, 12 + 1); 1867 uasm_i_dsll_safe(&p, K0, K0, 64 + 12 + 1 - segbits); 1868 uasm_i_or(&p, K0, K0, K1); 1869 uasm_il_bnez(&p, &r, K0, label_leave); 1870 /* No need for uasm_i_nop */ 1871 } 1872 1873 wr = build_r4000_tlbchange_handler_head(&p, &l, &r); 1874 build_pte_present(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbl); 1875 if (m4kc_tlbp_war()) 1876 build_tlb_probe_entry(&p); 1877 1878 if (kernel_uses_smartmips_rixi) { 1879 /* 1880 * If the page is not _PAGE_VALID, RI or XI could not 1881 * have triggered it. Skip the expensive test.. 1882 */ 1883 if (use_bbit_insns()) { 1884 uasm_il_bbit0(&p, &r, wr.r1, ilog2(_PAGE_VALID), 1885 label_tlbl_goaround1); 1886 } else { 1887 uasm_i_andi(&p, wr.r3, wr.r1, _PAGE_VALID); 1888 uasm_il_beqz(&p, &r, wr.r3, label_tlbl_goaround1); 1889 } 1890 uasm_i_nop(&p); 1891 1892 uasm_i_tlbr(&p); 1893 /* Examine entrylo 0 or 1 based on ptr. */ 1894 if (use_bbit_insns()) { 1895 uasm_i_bbit0(&p, wr.r2, ilog2(sizeof(pte_t)), 8); 1896 } else { 1897 uasm_i_andi(&p, wr.r3, wr.r2, sizeof(pte_t)); 1898 uasm_i_beqz(&p, wr.r3, 8); 1899 } 1900 /* load it in the delay slot*/ 1901 UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO0); 1902 /* load it if ptr is odd */ 1903 UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO1); 1904 /* 1905 * If the entryLo (now in wr.r3) is valid (bit 1), RI or 1906 * XI must have triggered it. 1907 */ 1908 if (use_bbit_insns()) { 1909 uasm_il_bbit1(&p, &r, wr.r3, 1, label_nopage_tlbl); 1910 uasm_i_nop(&p); 1911 uasm_l_tlbl_goaround1(&l, p); 1912 } else { 1913 uasm_i_andi(&p, wr.r3, wr.r3, 2); 1914 uasm_il_bnez(&p, &r, wr.r3, label_nopage_tlbl); 1915 uasm_i_nop(&p); 1916 } 1917 uasm_l_tlbl_goaround1(&l, p); 1918 } 1919 build_make_valid(&p, &r, wr.r1, wr.r2); 1920 build_r4000_tlbchange_handler_tail(&p, &l, &r, wr.r1, wr.r2); 1921 1922 #ifdef CONFIG_HUGETLB_PAGE 1923 /* 1924 * This is the entry point when build_r4000_tlbchange_handler_head 1925 * spots a huge page. 1926 */ 1927 uasm_l_tlb_huge_update(&l, p); 1928 iPTE_LW(&p, wr.r1, wr.r2); 1929 build_pte_present(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbl); 1930 build_tlb_probe_entry(&p); 1931 1932 if (kernel_uses_smartmips_rixi) { 1933 /* 1934 * If the page is not _PAGE_VALID, RI or XI could not 1935 * have triggered it. Skip the expensive test.. 1936 */ 1937 if (use_bbit_insns()) { 1938 uasm_il_bbit0(&p, &r, wr.r1, ilog2(_PAGE_VALID), 1939 label_tlbl_goaround2); 1940 } else { 1941 uasm_i_andi(&p, wr.r3, wr.r1, _PAGE_VALID); 1942 uasm_il_beqz(&p, &r, wr.r3, label_tlbl_goaround2); 1943 } 1944 uasm_i_nop(&p); 1945 1946 uasm_i_tlbr(&p); 1947 /* Examine entrylo 0 or 1 based on ptr. */ 1948 if (use_bbit_insns()) { 1949 uasm_i_bbit0(&p, wr.r2, ilog2(sizeof(pte_t)), 8); 1950 } else { 1951 uasm_i_andi(&p, wr.r3, wr.r2, sizeof(pte_t)); 1952 uasm_i_beqz(&p, wr.r3, 8); 1953 } 1954 /* load it in the delay slot*/ 1955 UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO0); 1956 /* load it if ptr is odd */ 1957 UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO1); 1958 /* 1959 * If the entryLo (now in wr.r3) is valid (bit 1), RI or 1960 * XI must have triggered it. 1961 */ 1962 if (use_bbit_insns()) { 1963 uasm_il_bbit0(&p, &r, wr.r3, 1, label_tlbl_goaround2); 1964 } else { 1965 uasm_i_andi(&p, wr.r3, wr.r3, 2); 1966 uasm_il_beqz(&p, &r, wr.r3, label_tlbl_goaround2); 1967 } 1968 if (PM_DEFAULT_MASK == 0) 1969 uasm_i_nop(&p); 1970 /* 1971 * We clobbered C0_PAGEMASK, restore it. On the other branch 1972 * it is restored in build_huge_tlb_write_entry. 1973 */ 1974 build_restore_pagemask(&p, &r, wr.r3, label_nopage_tlbl, 0); 1975 1976 uasm_l_tlbl_goaround2(&l, p); 1977 } 1978 uasm_i_ori(&p, wr.r1, wr.r1, (_PAGE_ACCESSED | _PAGE_VALID)); 1979 build_huge_handler_tail(&p, &r, &l, wr.r1, wr.r2); 1980 #endif 1981 1982 uasm_l_nopage_tlbl(&l, p); 1983 build_restore_work_registers(&p); 1984 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff); 1985 uasm_i_nop(&p); 1986 1987 if ((p - handle_tlbl) > FASTPATH_SIZE) 1988 panic("TLB load handler fastpath space exceeded"); 1989 1990 uasm_resolve_relocs(relocs, labels); 1991 pr_debug("Wrote TLB load handler fastpath (%u instructions).\n", 1992 (unsigned int)(p - handle_tlbl)); 1993 1994 dump_handler(handle_tlbl, ARRAY_SIZE(handle_tlbl)); 1995 } 1996 1997 static void __cpuinit build_r4000_tlb_store_handler(void) 1998 { 1999 u32 *p = handle_tlbs; 2000 struct uasm_label *l = labels; 2001 struct uasm_reloc *r = relocs; 2002 struct work_registers wr; 2003 2004 memset(handle_tlbs, 0, sizeof(handle_tlbs)); 2005 memset(labels, 0, sizeof(labels)); 2006 memset(relocs, 0, sizeof(relocs)); 2007 2008 wr = build_r4000_tlbchange_handler_head(&p, &l, &r); 2009 build_pte_writable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbs); 2010 if (m4kc_tlbp_war()) 2011 build_tlb_probe_entry(&p); 2012 build_make_write(&p, &r, wr.r1, wr.r2); 2013 build_r4000_tlbchange_handler_tail(&p, &l, &r, wr.r1, wr.r2); 2014 2015 #ifdef CONFIG_HUGETLB_PAGE 2016 /* 2017 * This is the entry point when 2018 * build_r4000_tlbchange_handler_head spots a huge page. 2019 */ 2020 uasm_l_tlb_huge_update(&l, p); 2021 iPTE_LW(&p, wr.r1, wr.r2); 2022 build_pte_writable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbs); 2023 build_tlb_probe_entry(&p); 2024 uasm_i_ori(&p, wr.r1, wr.r1, 2025 _PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID | _PAGE_DIRTY); 2026 build_huge_handler_tail(&p, &r, &l, wr.r1, wr.r2); 2027 #endif 2028 2029 uasm_l_nopage_tlbs(&l, p); 2030 build_restore_work_registers(&p); 2031 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); 2032 uasm_i_nop(&p); 2033 2034 if ((p - handle_tlbs) > FASTPATH_SIZE) 2035 panic("TLB store handler fastpath space exceeded"); 2036 2037 uasm_resolve_relocs(relocs, labels); 2038 pr_debug("Wrote TLB store handler fastpath (%u instructions).\n", 2039 (unsigned int)(p - handle_tlbs)); 2040 2041 dump_handler(handle_tlbs, ARRAY_SIZE(handle_tlbs)); 2042 } 2043 2044 static void __cpuinit build_r4000_tlb_modify_handler(void) 2045 { 2046 u32 *p = handle_tlbm; 2047 struct uasm_label *l = labels; 2048 struct uasm_reloc *r = relocs; 2049 struct work_registers wr; 2050 2051 memset(handle_tlbm, 0, sizeof(handle_tlbm)); 2052 memset(labels, 0, sizeof(labels)); 2053 memset(relocs, 0, sizeof(relocs)); 2054 2055 wr = build_r4000_tlbchange_handler_head(&p, &l, &r); 2056 build_pte_modifiable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbm); 2057 if (m4kc_tlbp_war()) 2058 build_tlb_probe_entry(&p); 2059 /* Present and writable bits set, set accessed and dirty bits. */ 2060 build_make_write(&p, &r, wr.r1, wr.r2); 2061 build_r4000_tlbchange_handler_tail(&p, &l, &r, wr.r1, wr.r2); 2062 2063 #ifdef CONFIG_HUGETLB_PAGE 2064 /* 2065 * This is the entry point when 2066 * build_r4000_tlbchange_handler_head spots a huge page. 2067 */ 2068 uasm_l_tlb_huge_update(&l, p); 2069 iPTE_LW(&p, wr.r1, wr.r2); 2070 build_pte_modifiable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbm); 2071 build_tlb_probe_entry(&p); 2072 uasm_i_ori(&p, wr.r1, wr.r1, 2073 _PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID | _PAGE_DIRTY); 2074 build_huge_handler_tail(&p, &r, &l, wr.r1, wr.r2); 2075 #endif 2076 2077 uasm_l_nopage_tlbm(&l, p); 2078 build_restore_work_registers(&p); 2079 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); 2080 uasm_i_nop(&p); 2081 2082 if ((p - handle_tlbm) > FASTPATH_SIZE) 2083 panic("TLB modify handler fastpath space exceeded"); 2084 2085 uasm_resolve_relocs(relocs, labels); 2086 pr_debug("Wrote TLB modify handler fastpath (%u instructions).\n", 2087 (unsigned int)(p - handle_tlbm)); 2088 2089 dump_handler(handle_tlbm, ARRAY_SIZE(handle_tlbm)); 2090 } 2091 2092 void __cpuinit build_tlb_refill_handler(void) 2093 { 2094 /* 2095 * The refill handler is generated per-CPU, multi-node systems 2096 * may have local storage for it. The other handlers are only 2097 * needed once. 2098 */ 2099 static int run_once = 0; 2100 2101 #ifdef CONFIG_64BIT 2102 check_for_high_segbits = current_cpu_data.vmbits > (PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3); 2103 #endif 2104 2105 switch (current_cpu_type()) { 2106 case CPU_R2000: 2107 case CPU_R3000: 2108 case CPU_R3000A: 2109 case CPU_R3081E: 2110 case CPU_TX3912: 2111 case CPU_TX3922: 2112 case CPU_TX3927: 2113 #ifndef CONFIG_MIPS_PGD_C0_CONTEXT 2114 build_r3000_tlb_refill_handler(); 2115 if (!run_once) { 2116 build_r3000_tlb_load_handler(); 2117 build_r3000_tlb_store_handler(); 2118 build_r3000_tlb_modify_handler(); 2119 run_once++; 2120 } 2121 #else 2122 panic("No R3000 TLB refill handler"); 2123 #endif 2124 break; 2125 2126 case CPU_R6000: 2127 case CPU_R6000A: 2128 panic("No R6000 TLB refill handler yet"); 2129 break; 2130 2131 case CPU_R8000: 2132 panic("No R8000 TLB refill handler yet"); 2133 break; 2134 2135 default: 2136 if (!run_once) { 2137 scratch_reg = allocate_kscratch(); 2138 #ifdef CONFIG_MIPS_PGD_C0_CONTEXT 2139 build_r4000_setup_pgd(); 2140 #endif 2141 build_r4000_tlb_load_handler(); 2142 build_r4000_tlb_store_handler(); 2143 build_r4000_tlb_modify_handler(); 2144 run_once++; 2145 } 2146 build_r4000_tlb_refill_handler(); 2147 } 2148 } 2149 2150 void __cpuinit flush_tlb_handlers(void) 2151 { 2152 local_flush_icache_range((unsigned long)handle_tlbl, 2153 (unsigned long)handle_tlbl + sizeof(handle_tlbl)); 2154 local_flush_icache_range((unsigned long)handle_tlbs, 2155 (unsigned long)handle_tlbs + sizeof(handle_tlbs)); 2156 local_flush_icache_range((unsigned long)handle_tlbm, 2157 (unsigned long)handle_tlbm + sizeof(handle_tlbm)); 2158 #ifdef CONFIG_MIPS_PGD_C0_CONTEXT 2159 local_flush_icache_range((unsigned long)tlbmiss_handler_setup_pgd, 2160 (unsigned long)tlbmiss_handler_setup_pgd + sizeof(handle_tlbm)); 2161 #endif 2162 } 2163