1 /* 2 * This file is subject to the terms and conditions of the GNU General Public 3 * License. See the file "COPYING" in the main directory of this archive 4 * for more details. 5 * 6 * Synthesize TLB refill handlers at runtime. 7 * 8 * Copyright (C) 2004, 2005, 2006, 2008 Thiemo Seufer 9 * Copyright (C) 2005, 2007 Maciej W. Rozycki 10 * Copyright (C) 2006 Ralf Baechle (ralf@linux-mips.org) 11 * 12 * ... and the days got worse and worse and now you see 13 * I've gone completly out of my mind. 14 * 15 * They're coming to take me a away haha 16 * they're coming to take me a away hoho hihi haha 17 * to the funny farm where code is beautiful all the time ... 18 * 19 * (Condolences to Napoleon XIV) 20 */ 21 22 #include <linux/kernel.h> 23 #include <linux/types.h> 24 #include <linux/string.h> 25 #include <linux/init.h> 26 27 #include <asm/mmu_context.h> 28 #include <asm/war.h> 29 30 #include "uasm.h" 31 32 static inline int r45k_bvahwbug(void) 33 { 34 /* XXX: We should probe for the presence of this bug, but we don't. */ 35 return 0; 36 } 37 38 static inline int r4k_250MHZhwbug(void) 39 { 40 /* XXX: We should probe for the presence of this bug, but we don't. */ 41 return 0; 42 } 43 44 static inline int __maybe_unused bcm1250_m3_war(void) 45 { 46 return BCM1250_M3_WAR; 47 } 48 49 static inline int __maybe_unused r10000_llsc_war(void) 50 { 51 return R10000_LLSC_WAR; 52 } 53 54 /* 55 * Found by experiment: At least some revisions of the 4kc throw under 56 * some circumstances a machine check exception, triggered by invalid 57 * values in the index register. Delaying the tlbp instruction until 58 * after the next branch, plus adding an additional nop in front of 59 * tlbwi/tlbwr avoids the invalid index register values. Nobody knows 60 * why; it's not an issue caused by the core RTL. 61 * 62 */ 63 static int __cpuinit m4kc_tlbp_war(void) 64 { 65 return (current_cpu_data.processor_id & 0xffff00) == 66 (PRID_COMP_MIPS | PRID_IMP_4KC); 67 } 68 69 /* Handle labels (which must be positive integers). */ 70 enum label_id { 71 label_second_part = 1, 72 label_leave, 73 #ifdef MODULE_START 74 label_module_alloc, 75 #endif 76 label_vmalloc, 77 label_vmalloc_done, 78 label_tlbw_hazard, 79 label_split, 80 label_nopage_tlbl, 81 label_nopage_tlbs, 82 label_nopage_tlbm, 83 label_smp_pgtable_change, 84 label_r3000_write_probe_fail, 85 }; 86 87 UASM_L_LA(_second_part) 88 UASM_L_LA(_leave) 89 #ifdef MODULE_START 90 UASM_L_LA(_module_alloc) 91 #endif 92 UASM_L_LA(_vmalloc) 93 UASM_L_LA(_vmalloc_done) 94 UASM_L_LA(_tlbw_hazard) 95 UASM_L_LA(_split) 96 UASM_L_LA(_nopage_tlbl) 97 UASM_L_LA(_nopage_tlbs) 98 UASM_L_LA(_nopage_tlbm) 99 UASM_L_LA(_smp_pgtable_change) 100 UASM_L_LA(_r3000_write_probe_fail) 101 102 /* 103 * For debug purposes. 104 */ 105 static inline void dump_handler(const u32 *handler, int count) 106 { 107 int i; 108 109 pr_debug("\t.set push\n"); 110 pr_debug("\t.set noreorder\n"); 111 112 for (i = 0; i < count; i++) 113 pr_debug("\t%p\t.word 0x%08x\n", &handler[i], handler[i]); 114 115 pr_debug("\t.set pop\n"); 116 } 117 118 /* The only general purpose registers allowed in TLB handlers. */ 119 #define K0 26 120 #define K1 27 121 122 /* Some CP0 registers */ 123 #define C0_INDEX 0, 0 124 #define C0_ENTRYLO0 2, 0 125 #define C0_TCBIND 2, 2 126 #define C0_ENTRYLO1 3, 0 127 #define C0_CONTEXT 4, 0 128 #define C0_BADVADDR 8, 0 129 #define C0_ENTRYHI 10, 0 130 #define C0_EPC 14, 0 131 #define C0_XCONTEXT 20, 0 132 133 #ifdef CONFIG_64BIT 134 # define GET_CONTEXT(buf, reg) UASM_i_MFC0(buf, reg, C0_XCONTEXT) 135 #else 136 # define GET_CONTEXT(buf, reg) UASM_i_MFC0(buf, reg, C0_CONTEXT) 137 #endif 138 139 /* The worst case length of the handler is around 18 instructions for 140 * R3000-style TLBs and up to 63 instructions for R4000-style TLBs. 141 * Maximum space available is 32 instructions for R3000 and 64 142 * instructions for R4000. 143 * 144 * We deliberately chose a buffer size of 128, so we won't scribble 145 * over anything important on overflow before we panic. 146 */ 147 static u32 tlb_handler[128] __cpuinitdata; 148 149 /* simply assume worst case size for labels and relocs */ 150 static struct uasm_label labels[128] __cpuinitdata; 151 static struct uasm_reloc relocs[128] __cpuinitdata; 152 153 /* 154 * The R3000 TLB handler is simple. 155 */ 156 static void __cpuinit build_r3000_tlb_refill_handler(void) 157 { 158 long pgdc = (long)pgd_current; 159 u32 *p; 160 161 memset(tlb_handler, 0, sizeof(tlb_handler)); 162 p = tlb_handler; 163 164 uasm_i_mfc0(&p, K0, C0_BADVADDR); 165 uasm_i_lui(&p, K1, uasm_rel_hi(pgdc)); /* cp0 delay */ 166 uasm_i_lw(&p, K1, uasm_rel_lo(pgdc), K1); 167 uasm_i_srl(&p, K0, K0, 22); /* load delay */ 168 uasm_i_sll(&p, K0, K0, 2); 169 uasm_i_addu(&p, K1, K1, K0); 170 uasm_i_mfc0(&p, K0, C0_CONTEXT); 171 uasm_i_lw(&p, K1, 0, K1); /* cp0 delay */ 172 uasm_i_andi(&p, K0, K0, 0xffc); /* load delay */ 173 uasm_i_addu(&p, K1, K1, K0); 174 uasm_i_lw(&p, K0, 0, K1); 175 uasm_i_nop(&p); /* load delay */ 176 uasm_i_mtc0(&p, K0, C0_ENTRYLO0); 177 uasm_i_mfc0(&p, K1, C0_EPC); /* cp0 delay */ 178 uasm_i_tlbwr(&p); /* cp0 delay */ 179 uasm_i_jr(&p, K1); 180 uasm_i_rfe(&p); /* branch delay */ 181 182 if (p > tlb_handler + 32) 183 panic("TLB refill handler space exceeded"); 184 185 pr_debug("Wrote TLB refill handler (%u instructions).\n", 186 (unsigned int)(p - tlb_handler)); 187 188 memcpy((void *)ebase, tlb_handler, 0x80); 189 190 dump_handler((u32 *)ebase, 32); 191 } 192 193 /* 194 * The R4000 TLB handler is much more complicated. We have two 195 * consecutive handler areas with 32 instructions space each. 196 * Since they aren't used at the same time, we can overflow in the 197 * other one.To keep things simple, we first assume linear space, 198 * then we relocate it to the final handler layout as needed. 199 */ 200 static u32 final_handler[64] __cpuinitdata; 201 202 /* 203 * Hazards 204 * 205 * From the IDT errata for the QED RM5230 (Nevada), processor revision 1.0: 206 * 2. A timing hazard exists for the TLBP instruction. 207 * 208 * stalling_instruction 209 * TLBP 210 * 211 * The JTLB is being read for the TLBP throughout the stall generated by the 212 * previous instruction. This is not really correct as the stalling instruction 213 * can modify the address used to access the JTLB. The failure symptom is that 214 * the TLBP instruction will use an address created for the stalling instruction 215 * and not the address held in C0_ENHI and thus report the wrong results. 216 * 217 * The software work-around is to not allow the instruction preceding the TLBP 218 * to stall - make it an NOP or some other instruction guaranteed not to stall. 219 * 220 * Errata 2 will not be fixed. This errata is also on the R5000. 221 * 222 * As if we MIPS hackers wouldn't know how to nop pipelines happy ... 223 */ 224 static void __cpuinit __maybe_unused build_tlb_probe_entry(u32 **p) 225 { 226 switch (current_cpu_type()) { 227 /* Found by experiment: R4600 v2.0/R4700 needs this, too. */ 228 case CPU_R4600: 229 case CPU_R4700: 230 case CPU_R5000: 231 case CPU_R5000A: 232 case CPU_NEVADA: 233 uasm_i_nop(p); 234 uasm_i_tlbp(p); 235 break; 236 237 default: 238 uasm_i_tlbp(p); 239 break; 240 } 241 } 242 243 /* 244 * Write random or indexed TLB entry, and care about the hazards from 245 * the preceeding mtc0 and for the following eret. 246 */ 247 enum tlb_write_entry { tlb_random, tlb_indexed }; 248 249 static void __cpuinit build_tlb_write_entry(u32 **p, struct uasm_label **l, 250 struct uasm_reloc **r, 251 enum tlb_write_entry wmode) 252 { 253 void(*tlbw)(u32 **) = NULL; 254 255 switch (wmode) { 256 case tlb_random: tlbw = uasm_i_tlbwr; break; 257 case tlb_indexed: tlbw = uasm_i_tlbwi; break; 258 } 259 260 if (cpu_has_mips_r2) { 261 uasm_i_ehb(p); 262 tlbw(p); 263 return; 264 } 265 266 switch (current_cpu_type()) { 267 case CPU_R4000PC: 268 case CPU_R4000SC: 269 case CPU_R4000MC: 270 case CPU_R4400PC: 271 case CPU_R4400SC: 272 case CPU_R4400MC: 273 /* 274 * This branch uses up a mtc0 hazard nop slot and saves 275 * two nops after the tlbw instruction. 276 */ 277 uasm_il_bgezl(p, r, 0, label_tlbw_hazard); 278 tlbw(p); 279 uasm_l_tlbw_hazard(l, *p); 280 uasm_i_nop(p); 281 break; 282 283 case CPU_R4600: 284 case CPU_R4700: 285 case CPU_R5000: 286 case CPU_R5000A: 287 uasm_i_nop(p); 288 tlbw(p); 289 uasm_i_nop(p); 290 break; 291 292 case CPU_R4300: 293 case CPU_5KC: 294 case CPU_TX49XX: 295 case CPU_AU1000: 296 case CPU_AU1100: 297 case CPU_AU1500: 298 case CPU_AU1550: 299 case CPU_AU1200: 300 case CPU_AU1210: 301 case CPU_AU1250: 302 case CPU_PR4450: 303 uasm_i_nop(p); 304 tlbw(p); 305 break; 306 307 case CPU_R10000: 308 case CPU_R12000: 309 case CPU_R14000: 310 case CPU_4KC: 311 case CPU_4KEC: 312 case CPU_SB1: 313 case CPU_SB1A: 314 case CPU_4KSC: 315 case CPU_20KC: 316 case CPU_25KF: 317 case CPU_BCM3302: 318 case CPU_BCM4710: 319 case CPU_LOONGSON2: 320 if (m4kc_tlbp_war()) 321 uasm_i_nop(p); 322 tlbw(p); 323 break; 324 325 case CPU_NEVADA: 326 uasm_i_nop(p); /* QED specifies 2 nops hazard */ 327 /* 328 * This branch uses up a mtc0 hazard nop slot and saves 329 * a nop after the tlbw instruction. 330 */ 331 uasm_il_bgezl(p, r, 0, label_tlbw_hazard); 332 tlbw(p); 333 uasm_l_tlbw_hazard(l, *p); 334 break; 335 336 case CPU_RM7000: 337 uasm_i_nop(p); 338 uasm_i_nop(p); 339 uasm_i_nop(p); 340 uasm_i_nop(p); 341 tlbw(p); 342 break; 343 344 case CPU_RM9000: 345 /* 346 * When the JTLB is updated by tlbwi or tlbwr, a subsequent 347 * use of the JTLB for instructions should not occur for 4 348 * cpu cycles and use for data translations should not occur 349 * for 3 cpu cycles. 350 */ 351 uasm_i_ssnop(p); 352 uasm_i_ssnop(p); 353 uasm_i_ssnop(p); 354 uasm_i_ssnop(p); 355 tlbw(p); 356 uasm_i_ssnop(p); 357 uasm_i_ssnop(p); 358 uasm_i_ssnop(p); 359 uasm_i_ssnop(p); 360 break; 361 362 case CPU_VR4111: 363 case CPU_VR4121: 364 case CPU_VR4122: 365 case CPU_VR4181: 366 case CPU_VR4181A: 367 uasm_i_nop(p); 368 uasm_i_nop(p); 369 tlbw(p); 370 uasm_i_nop(p); 371 uasm_i_nop(p); 372 break; 373 374 case CPU_VR4131: 375 case CPU_VR4133: 376 case CPU_R5432: 377 uasm_i_nop(p); 378 uasm_i_nop(p); 379 tlbw(p); 380 break; 381 382 default: 383 panic("No TLB refill handler yet (CPU type: %d)", 384 current_cpu_data.cputype); 385 break; 386 } 387 } 388 389 #ifdef CONFIG_64BIT 390 /* 391 * TMP and PTR are scratch. 392 * TMP will be clobbered, PTR will hold the pmd entry. 393 */ 394 static void __cpuinit 395 build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r, 396 unsigned int tmp, unsigned int ptr) 397 { 398 long pgdc = (long)pgd_current; 399 400 /* 401 * The vmalloc handling is not in the hotpath. 402 */ 403 uasm_i_dmfc0(p, tmp, C0_BADVADDR); 404 #ifdef MODULE_START 405 uasm_il_bltz(p, r, tmp, label_module_alloc); 406 #else 407 uasm_il_bltz(p, r, tmp, label_vmalloc); 408 #endif 409 /* No uasm_i_nop needed here, since the next insn doesn't touch TMP. */ 410 411 #ifdef CONFIG_SMP 412 # ifdef CONFIG_MIPS_MT_SMTC 413 /* 414 * SMTC uses TCBind value as "CPU" index 415 */ 416 uasm_i_mfc0(p, ptr, C0_TCBIND); 417 uasm_i_dsrl(p, ptr, ptr, 19); 418 # else 419 /* 420 * 64 bit SMP running in XKPHYS has smp_processor_id() << 3 421 * stored in CONTEXT. 422 */ 423 uasm_i_dmfc0(p, ptr, C0_CONTEXT); 424 uasm_i_dsrl(p, ptr, ptr, 23); 425 #endif 426 UASM_i_LA_mostly(p, tmp, pgdc); 427 uasm_i_daddu(p, ptr, ptr, tmp); 428 uasm_i_dmfc0(p, tmp, C0_BADVADDR); 429 uasm_i_ld(p, ptr, uasm_rel_lo(pgdc), ptr); 430 #else 431 UASM_i_LA_mostly(p, ptr, pgdc); 432 uasm_i_ld(p, ptr, uasm_rel_lo(pgdc), ptr); 433 #endif 434 435 uasm_l_vmalloc_done(l, *p); 436 437 if (PGDIR_SHIFT - 3 < 32) /* get pgd offset in bytes */ 438 uasm_i_dsrl(p, tmp, tmp, PGDIR_SHIFT-3); 439 else 440 uasm_i_dsrl32(p, tmp, tmp, PGDIR_SHIFT - 3 - 32); 441 442 uasm_i_andi(p, tmp, tmp, (PTRS_PER_PGD - 1)<<3); 443 uasm_i_daddu(p, ptr, ptr, tmp); /* add in pgd offset */ 444 uasm_i_dmfc0(p, tmp, C0_BADVADDR); /* get faulting address */ 445 uasm_i_ld(p, ptr, 0, ptr); /* get pmd pointer */ 446 uasm_i_dsrl(p, tmp, tmp, PMD_SHIFT-3); /* get pmd offset in bytes */ 447 uasm_i_andi(p, tmp, tmp, (PTRS_PER_PMD - 1)<<3); 448 uasm_i_daddu(p, ptr, ptr, tmp); /* add in pmd offset */ 449 } 450 451 /* 452 * BVADDR is the faulting address, PTR is scratch. 453 * PTR will hold the pgd for vmalloc. 454 */ 455 static void __cpuinit 456 build_get_pgd_vmalloc64(u32 **p, struct uasm_label **l, struct uasm_reloc **r, 457 unsigned int bvaddr, unsigned int ptr) 458 { 459 long swpd = (long)swapper_pg_dir; 460 461 #ifdef MODULE_START 462 long modd = (long)module_pg_dir; 463 464 uasm_l_module_alloc(l, *p); 465 /* 466 * Assumption: 467 * VMALLOC_START >= 0xc000000000000000UL 468 * MODULE_START >= 0xe000000000000000UL 469 */ 470 UASM_i_SLL(p, ptr, bvaddr, 2); 471 uasm_il_bgez(p, r, ptr, label_vmalloc); 472 473 if (uasm_in_compat_space_p(MODULE_START) && 474 !uasm_rel_lo(MODULE_START)) { 475 uasm_i_lui(p, ptr, uasm_rel_hi(MODULE_START)); /* delay slot */ 476 } else { 477 /* unlikely configuration */ 478 uasm_i_nop(p); /* delay slot */ 479 UASM_i_LA(p, ptr, MODULE_START); 480 } 481 uasm_i_dsubu(p, bvaddr, bvaddr, ptr); 482 483 if (uasm_in_compat_space_p(modd) && !uasm_rel_lo(modd)) { 484 uasm_il_b(p, r, label_vmalloc_done); 485 uasm_i_lui(p, ptr, uasm_rel_hi(modd)); 486 } else { 487 UASM_i_LA_mostly(p, ptr, modd); 488 uasm_il_b(p, r, label_vmalloc_done); 489 if (uasm_in_compat_space_p(modd)) 490 uasm_i_addiu(p, ptr, ptr, uasm_rel_lo(modd)); 491 else 492 uasm_i_daddiu(p, ptr, ptr, uasm_rel_lo(modd)); 493 } 494 495 uasm_l_vmalloc(l, *p); 496 if (uasm_in_compat_space_p(MODULE_START) && 497 !uasm_rel_lo(MODULE_START) && 498 MODULE_START << 32 == VMALLOC_START) 499 uasm_i_dsll32(p, ptr, ptr, 0); /* typical case */ 500 else 501 UASM_i_LA(p, ptr, VMALLOC_START); 502 #else 503 uasm_l_vmalloc(l, *p); 504 UASM_i_LA(p, ptr, VMALLOC_START); 505 #endif 506 uasm_i_dsubu(p, bvaddr, bvaddr, ptr); 507 508 if (uasm_in_compat_space_p(swpd) && !uasm_rel_lo(swpd)) { 509 uasm_il_b(p, r, label_vmalloc_done); 510 uasm_i_lui(p, ptr, uasm_rel_hi(swpd)); 511 } else { 512 UASM_i_LA_mostly(p, ptr, swpd); 513 uasm_il_b(p, r, label_vmalloc_done); 514 if (uasm_in_compat_space_p(swpd)) 515 uasm_i_addiu(p, ptr, ptr, uasm_rel_lo(swpd)); 516 else 517 uasm_i_daddiu(p, ptr, ptr, uasm_rel_lo(swpd)); 518 } 519 } 520 521 #else /* !CONFIG_64BIT */ 522 523 /* 524 * TMP and PTR are scratch. 525 * TMP will be clobbered, PTR will hold the pgd entry. 526 */ 527 static void __cpuinit __maybe_unused 528 build_get_pgde32(u32 **p, unsigned int tmp, unsigned int ptr) 529 { 530 long pgdc = (long)pgd_current; 531 532 /* 32 bit SMP has smp_processor_id() stored in CONTEXT. */ 533 #ifdef CONFIG_SMP 534 #ifdef CONFIG_MIPS_MT_SMTC 535 /* 536 * SMTC uses TCBind value as "CPU" index 537 */ 538 uasm_i_mfc0(p, ptr, C0_TCBIND); 539 UASM_i_LA_mostly(p, tmp, pgdc); 540 uasm_i_srl(p, ptr, ptr, 19); 541 #else 542 /* 543 * smp_processor_id() << 3 is stored in CONTEXT. 544 */ 545 uasm_i_mfc0(p, ptr, C0_CONTEXT); 546 UASM_i_LA_mostly(p, tmp, pgdc); 547 uasm_i_srl(p, ptr, ptr, 23); 548 #endif 549 uasm_i_addu(p, ptr, tmp, ptr); 550 #else 551 UASM_i_LA_mostly(p, ptr, pgdc); 552 #endif 553 uasm_i_mfc0(p, tmp, C0_BADVADDR); /* get faulting address */ 554 uasm_i_lw(p, ptr, uasm_rel_lo(pgdc), ptr); 555 uasm_i_srl(p, tmp, tmp, PGDIR_SHIFT); /* get pgd only bits */ 556 uasm_i_sll(p, tmp, tmp, PGD_T_LOG2); 557 uasm_i_addu(p, ptr, ptr, tmp); /* add in pgd offset */ 558 } 559 560 #endif /* !CONFIG_64BIT */ 561 562 static void __cpuinit build_adjust_context(u32 **p, unsigned int ctx) 563 { 564 unsigned int shift = 4 - (PTE_T_LOG2 + 1) + PAGE_SHIFT - 12; 565 unsigned int mask = (PTRS_PER_PTE / 2 - 1) << (PTE_T_LOG2 + 1); 566 567 switch (current_cpu_type()) { 568 case CPU_VR41XX: 569 case CPU_VR4111: 570 case CPU_VR4121: 571 case CPU_VR4122: 572 case CPU_VR4131: 573 case CPU_VR4181: 574 case CPU_VR4181A: 575 case CPU_VR4133: 576 shift += 2; 577 break; 578 579 default: 580 break; 581 } 582 583 if (shift) 584 UASM_i_SRL(p, ctx, ctx, shift); 585 uasm_i_andi(p, ctx, ctx, mask); 586 } 587 588 static void __cpuinit build_get_ptep(u32 **p, unsigned int tmp, unsigned int ptr) 589 { 590 /* 591 * Bug workaround for the Nevada. It seems as if under certain 592 * circumstances the move from cp0_context might produce a 593 * bogus result when the mfc0 instruction and its consumer are 594 * in a different cacheline or a load instruction, probably any 595 * memory reference, is between them. 596 */ 597 switch (current_cpu_type()) { 598 case CPU_NEVADA: 599 UASM_i_LW(p, ptr, 0, ptr); 600 GET_CONTEXT(p, tmp); /* get context reg */ 601 break; 602 603 default: 604 GET_CONTEXT(p, tmp); /* get context reg */ 605 UASM_i_LW(p, ptr, 0, ptr); 606 break; 607 } 608 609 build_adjust_context(p, tmp); 610 UASM_i_ADDU(p, ptr, ptr, tmp); /* add in offset */ 611 } 612 613 static void __cpuinit build_update_entries(u32 **p, unsigned int tmp, 614 unsigned int ptep) 615 { 616 /* 617 * 64bit address support (36bit on a 32bit CPU) in a 32bit 618 * Kernel is a special case. Only a few CPUs use it. 619 */ 620 #ifdef CONFIG_64BIT_PHYS_ADDR 621 if (cpu_has_64bits) { 622 uasm_i_ld(p, tmp, 0, ptep); /* get even pte */ 623 uasm_i_ld(p, ptep, sizeof(pte_t), ptep); /* get odd pte */ 624 uasm_i_dsrl(p, tmp, tmp, 6); /* convert to entrylo0 */ 625 uasm_i_mtc0(p, tmp, C0_ENTRYLO0); /* load it */ 626 uasm_i_dsrl(p, ptep, ptep, 6); /* convert to entrylo1 */ 627 uasm_i_mtc0(p, ptep, C0_ENTRYLO1); /* load it */ 628 } else { 629 int pte_off_even = sizeof(pte_t) / 2; 630 int pte_off_odd = pte_off_even + sizeof(pte_t); 631 632 /* The pte entries are pre-shifted */ 633 uasm_i_lw(p, tmp, pte_off_even, ptep); /* get even pte */ 634 uasm_i_mtc0(p, tmp, C0_ENTRYLO0); /* load it */ 635 uasm_i_lw(p, ptep, pte_off_odd, ptep); /* get odd pte */ 636 uasm_i_mtc0(p, ptep, C0_ENTRYLO1); /* load it */ 637 } 638 #else 639 UASM_i_LW(p, tmp, 0, ptep); /* get even pte */ 640 UASM_i_LW(p, ptep, sizeof(pte_t), ptep); /* get odd pte */ 641 if (r45k_bvahwbug()) 642 build_tlb_probe_entry(p); 643 UASM_i_SRL(p, tmp, tmp, 6); /* convert to entrylo0 */ 644 if (r4k_250MHZhwbug()) 645 uasm_i_mtc0(p, 0, C0_ENTRYLO0); 646 uasm_i_mtc0(p, tmp, C0_ENTRYLO0); /* load it */ 647 UASM_i_SRL(p, ptep, ptep, 6); /* convert to entrylo1 */ 648 if (r45k_bvahwbug()) 649 uasm_i_mfc0(p, tmp, C0_INDEX); 650 if (r4k_250MHZhwbug()) 651 uasm_i_mtc0(p, 0, C0_ENTRYLO1); 652 uasm_i_mtc0(p, ptep, C0_ENTRYLO1); /* load it */ 653 #endif 654 } 655 656 static void __cpuinit build_r4000_tlb_refill_handler(void) 657 { 658 u32 *p = tlb_handler; 659 struct uasm_label *l = labels; 660 struct uasm_reloc *r = relocs; 661 u32 *f; 662 unsigned int final_len; 663 664 memset(tlb_handler, 0, sizeof(tlb_handler)); 665 memset(labels, 0, sizeof(labels)); 666 memset(relocs, 0, sizeof(relocs)); 667 memset(final_handler, 0, sizeof(final_handler)); 668 669 /* 670 * create the plain linear handler 671 */ 672 if (bcm1250_m3_war()) { 673 UASM_i_MFC0(&p, K0, C0_BADVADDR); 674 UASM_i_MFC0(&p, K1, C0_ENTRYHI); 675 uasm_i_xor(&p, K0, K0, K1); 676 UASM_i_SRL(&p, K0, K0, PAGE_SHIFT + 1); 677 uasm_il_bnez(&p, &r, K0, label_leave); 678 /* No need for uasm_i_nop */ 679 } 680 681 #ifdef CONFIG_64BIT 682 build_get_pmde64(&p, &l, &r, K0, K1); /* get pmd in K1 */ 683 #else 684 build_get_pgde32(&p, K0, K1); /* get pgd in K1 */ 685 #endif 686 687 build_get_ptep(&p, K0, K1); 688 build_update_entries(&p, K0, K1); 689 build_tlb_write_entry(&p, &l, &r, tlb_random); 690 uasm_l_leave(&l, p); 691 uasm_i_eret(&p); /* return from trap */ 692 693 #ifdef CONFIG_64BIT 694 build_get_pgd_vmalloc64(&p, &l, &r, K0, K1); 695 #endif 696 697 /* 698 * Overflow check: For the 64bit handler, we need at least one 699 * free instruction slot for the wrap-around branch. In worst 700 * case, if the intended insertion point is a delay slot, we 701 * need three, with the second nop'ed and the third being 702 * unused. 703 */ 704 /* Loongson2 ebase is different than r4k, we have more space */ 705 #if defined(CONFIG_32BIT) || defined(CONFIG_CPU_LOONGSON2) 706 if ((p - tlb_handler) > 64) 707 panic("TLB refill handler space exceeded"); 708 #else 709 if (((p - tlb_handler) > 63) 710 || (((p - tlb_handler) > 61) 711 && uasm_insn_has_bdelay(relocs, tlb_handler + 29))) 712 panic("TLB refill handler space exceeded"); 713 #endif 714 715 /* 716 * Now fold the handler in the TLB refill handler space. 717 */ 718 #if defined(CONFIG_32BIT) || defined(CONFIG_CPU_LOONGSON2) 719 f = final_handler; 720 /* Simplest case, just copy the handler. */ 721 uasm_copy_handler(relocs, labels, tlb_handler, p, f); 722 final_len = p - tlb_handler; 723 #else /* CONFIG_64BIT */ 724 f = final_handler + 32; 725 if ((p - tlb_handler) <= 32) { 726 /* Just copy the handler. */ 727 uasm_copy_handler(relocs, labels, tlb_handler, p, f); 728 final_len = p - tlb_handler; 729 } else { 730 u32 *split = tlb_handler + 30; 731 732 /* 733 * Find the split point. 734 */ 735 if (uasm_insn_has_bdelay(relocs, split - 1)) 736 split--; 737 738 /* Copy first part of the handler. */ 739 uasm_copy_handler(relocs, labels, tlb_handler, split, f); 740 f += split - tlb_handler; 741 742 /* Insert branch. */ 743 uasm_l_split(&l, final_handler); 744 uasm_il_b(&f, &r, label_split); 745 if (uasm_insn_has_bdelay(relocs, split)) 746 uasm_i_nop(&f); 747 else { 748 uasm_copy_handler(relocs, labels, split, split + 1, f); 749 uasm_move_labels(labels, f, f + 1, -1); 750 f++; 751 split++; 752 } 753 754 /* Copy the rest of the handler. */ 755 uasm_copy_handler(relocs, labels, split, p, final_handler); 756 final_len = (f - (final_handler + 32)) + (p - split); 757 } 758 #endif /* CONFIG_64BIT */ 759 760 uasm_resolve_relocs(relocs, labels); 761 pr_debug("Wrote TLB refill handler (%u instructions).\n", 762 final_len); 763 764 memcpy((void *)ebase, final_handler, 0x100); 765 766 dump_handler((u32 *)ebase, 64); 767 } 768 769 /* 770 * TLB load/store/modify handlers. 771 * 772 * Only the fastpath gets synthesized at runtime, the slowpath for 773 * do_page_fault remains normal asm. 774 */ 775 extern void tlb_do_page_fault_0(void); 776 extern void tlb_do_page_fault_1(void); 777 778 /* 779 * 128 instructions for the fastpath handler is generous and should 780 * never be exceeded. 781 */ 782 #define FASTPATH_SIZE 128 783 784 u32 handle_tlbl[FASTPATH_SIZE] __cacheline_aligned; 785 u32 handle_tlbs[FASTPATH_SIZE] __cacheline_aligned; 786 u32 handle_tlbm[FASTPATH_SIZE] __cacheline_aligned; 787 788 static void __cpuinit 789 iPTE_LW(u32 **p, struct uasm_label **l, unsigned int pte, unsigned int ptr) 790 { 791 #ifdef CONFIG_SMP 792 # ifdef CONFIG_64BIT_PHYS_ADDR 793 if (cpu_has_64bits) 794 uasm_i_lld(p, pte, 0, ptr); 795 else 796 # endif 797 UASM_i_LL(p, pte, 0, ptr); 798 #else 799 # ifdef CONFIG_64BIT_PHYS_ADDR 800 if (cpu_has_64bits) 801 uasm_i_ld(p, pte, 0, ptr); 802 else 803 # endif 804 UASM_i_LW(p, pte, 0, ptr); 805 #endif 806 } 807 808 static void __cpuinit 809 iPTE_SW(u32 **p, struct uasm_reloc **r, unsigned int pte, unsigned int ptr, 810 unsigned int mode) 811 { 812 #ifdef CONFIG_64BIT_PHYS_ADDR 813 unsigned int hwmode = mode & (_PAGE_VALID | _PAGE_DIRTY); 814 #endif 815 816 uasm_i_ori(p, pte, pte, mode); 817 #ifdef CONFIG_SMP 818 # ifdef CONFIG_64BIT_PHYS_ADDR 819 if (cpu_has_64bits) 820 uasm_i_scd(p, pte, 0, ptr); 821 else 822 # endif 823 UASM_i_SC(p, pte, 0, ptr); 824 825 if (r10000_llsc_war()) 826 uasm_il_beqzl(p, r, pte, label_smp_pgtable_change); 827 else 828 uasm_il_beqz(p, r, pte, label_smp_pgtable_change); 829 830 # ifdef CONFIG_64BIT_PHYS_ADDR 831 if (!cpu_has_64bits) { 832 /* no uasm_i_nop needed */ 833 uasm_i_ll(p, pte, sizeof(pte_t) / 2, ptr); 834 uasm_i_ori(p, pte, pte, hwmode); 835 uasm_i_sc(p, pte, sizeof(pte_t) / 2, ptr); 836 uasm_il_beqz(p, r, pte, label_smp_pgtable_change); 837 /* no uasm_i_nop needed */ 838 uasm_i_lw(p, pte, 0, ptr); 839 } else 840 uasm_i_nop(p); 841 # else 842 uasm_i_nop(p); 843 # endif 844 #else 845 # ifdef CONFIG_64BIT_PHYS_ADDR 846 if (cpu_has_64bits) 847 uasm_i_sd(p, pte, 0, ptr); 848 else 849 # endif 850 UASM_i_SW(p, pte, 0, ptr); 851 852 # ifdef CONFIG_64BIT_PHYS_ADDR 853 if (!cpu_has_64bits) { 854 uasm_i_lw(p, pte, sizeof(pte_t) / 2, ptr); 855 uasm_i_ori(p, pte, pte, hwmode); 856 uasm_i_sw(p, pte, sizeof(pte_t) / 2, ptr); 857 uasm_i_lw(p, pte, 0, ptr); 858 } 859 # endif 860 #endif 861 } 862 863 /* 864 * Check if PTE is present, if not then jump to LABEL. PTR points to 865 * the page table where this PTE is located, PTE will be re-loaded 866 * with it's original value. 867 */ 868 static void __cpuinit 869 build_pte_present(u32 **p, struct uasm_label **l, struct uasm_reloc **r, 870 unsigned int pte, unsigned int ptr, enum label_id lid) 871 { 872 uasm_i_andi(p, pte, pte, _PAGE_PRESENT | _PAGE_READ); 873 uasm_i_xori(p, pte, pte, _PAGE_PRESENT | _PAGE_READ); 874 uasm_il_bnez(p, r, pte, lid); 875 iPTE_LW(p, l, pte, ptr); 876 } 877 878 /* Make PTE valid, store result in PTR. */ 879 static void __cpuinit 880 build_make_valid(u32 **p, struct uasm_reloc **r, unsigned int pte, 881 unsigned int ptr) 882 { 883 unsigned int mode = _PAGE_VALID | _PAGE_ACCESSED; 884 885 iPTE_SW(p, r, pte, ptr, mode); 886 } 887 888 /* 889 * Check if PTE can be written to, if not branch to LABEL. Regardless 890 * restore PTE with value from PTR when done. 891 */ 892 static void __cpuinit 893 build_pte_writable(u32 **p, struct uasm_label **l, struct uasm_reloc **r, 894 unsigned int pte, unsigned int ptr, enum label_id lid) 895 { 896 uasm_i_andi(p, pte, pte, _PAGE_PRESENT | _PAGE_WRITE); 897 uasm_i_xori(p, pte, pte, _PAGE_PRESENT | _PAGE_WRITE); 898 uasm_il_bnez(p, r, pte, lid); 899 iPTE_LW(p, l, pte, ptr); 900 } 901 902 /* Make PTE writable, update software status bits as well, then store 903 * at PTR. 904 */ 905 static void __cpuinit 906 build_make_write(u32 **p, struct uasm_reloc **r, unsigned int pte, 907 unsigned int ptr) 908 { 909 unsigned int mode = (_PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID 910 | _PAGE_DIRTY); 911 912 iPTE_SW(p, r, pte, ptr, mode); 913 } 914 915 /* 916 * Check if PTE can be modified, if not branch to LABEL. Regardless 917 * restore PTE with value from PTR when done. 918 */ 919 static void __cpuinit 920 build_pte_modifiable(u32 **p, struct uasm_label **l, struct uasm_reloc **r, 921 unsigned int pte, unsigned int ptr, enum label_id lid) 922 { 923 uasm_i_andi(p, pte, pte, _PAGE_WRITE); 924 uasm_il_beqz(p, r, pte, lid); 925 iPTE_LW(p, l, pte, ptr); 926 } 927 928 /* 929 * R3000 style TLB load/store/modify handlers. 930 */ 931 932 /* 933 * This places the pte into ENTRYLO0 and writes it with tlbwi. 934 * Then it returns. 935 */ 936 static void __cpuinit 937 build_r3000_pte_reload_tlbwi(u32 **p, unsigned int pte, unsigned int tmp) 938 { 939 uasm_i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */ 940 uasm_i_mfc0(p, tmp, C0_EPC); /* cp0 delay */ 941 uasm_i_tlbwi(p); 942 uasm_i_jr(p, tmp); 943 uasm_i_rfe(p); /* branch delay */ 944 } 945 946 /* 947 * This places the pte into ENTRYLO0 and writes it with tlbwi 948 * or tlbwr as appropriate. This is because the index register 949 * may have the probe fail bit set as a result of a trap on a 950 * kseg2 access, i.e. without refill. Then it returns. 951 */ 952 static void __cpuinit 953 build_r3000_tlb_reload_write(u32 **p, struct uasm_label **l, 954 struct uasm_reloc **r, unsigned int pte, 955 unsigned int tmp) 956 { 957 uasm_i_mfc0(p, tmp, C0_INDEX); 958 uasm_i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */ 959 uasm_il_bltz(p, r, tmp, label_r3000_write_probe_fail); /* cp0 delay */ 960 uasm_i_mfc0(p, tmp, C0_EPC); /* branch delay */ 961 uasm_i_tlbwi(p); /* cp0 delay */ 962 uasm_i_jr(p, tmp); 963 uasm_i_rfe(p); /* branch delay */ 964 uasm_l_r3000_write_probe_fail(l, *p); 965 uasm_i_tlbwr(p); /* cp0 delay */ 966 uasm_i_jr(p, tmp); 967 uasm_i_rfe(p); /* branch delay */ 968 } 969 970 static void __cpuinit 971 build_r3000_tlbchange_handler_head(u32 **p, unsigned int pte, 972 unsigned int ptr) 973 { 974 long pgdc = (long)pgd_current; 975 976 uasm_i_mfc0(p, pte, C0_BADVADDR); 977 uasm_i_lui(p, ptr, uasm_rel_hi(pgdc)); /* cp0 delay */ 978 uasm_i_lw(p, ptr, uasm_rel_lo(pgdc), ptr); 979 uasm_i_srl(p, pte, pte, 22); /* load delay */ 980 uasm_i_sll(p, pte, pte, 2); 981 uasm_i_addu(p, ptr, ptr, pte); 982 uasm_i_mfc0(p, pte, C0_CONTEXT); 983 uasm_i_lw(p, ptr, 0, ptr); /* cp0 delay */ 984 uasm_i_andi(p, pte, pte, 0xffc); /* load delay */ 985 uasm_i_addu(p, ptr, ptr, pte); 986 uasm_i_lw(p, pte, 0, ptr); 987 uasm_i_tlbp(p); /* load delay */ 988 } 989 990 static void __cpuinit build_r3000_tlb_load_handler(void) 991 { 992 u32 *p = handle_tlbl; 993 struct uasm_label *l = labels; 994 struct uasm_reloc *r = relocs; 995 996 memset(handle_tlbl, 0, sizeof(handle_tlbl)); 997 memset(labels, 0, sizeof(labels)); 998 memset(relocs, 0, sizeof(relocs)); 999 1000 build_r3000_tlbchange_handler_head(&p, K0, K1); 1001 build_pte_present(&p, &l, &r, K0, K1, label_nopage_tlbl); 1002 uasm_i_nop(&p); /* load delay */ 1003 build_make_valid(&p, &r, K0, K1); 1004 build_r3000_tlb_reload_write(&p, &l, &r, K0, K1); 1005 1006 uasm_l_nopage_tlbl(&l, p); 1007 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff); 1008 uasm_i_nop(&p); 1009 1010 if ((p - handle_tlbl) > FASTPATH_SIZE) 1011 panic("TLB load handler fastpath space exceeded"); 1012 1013 uasm_resolve_relocs(relocs, labels); 1014 pr_debug("Wrote TLB load handler fastpath (%u instructions).\n", 1015 (unsigned int)(p - handle_tlbl)); 1016 1017 dump_handler(handle_tlbl, ARRAY_SIZE(handle_tlbl)); 1018 } 1019 1020 static void __cpuinit build_r3000_tlb_store_handler(void) 1021 { 1022 u32 *p = handle_tlbs; 1023 struct uasm_label *l = labels; 1024 struct uasm_reloc *r = relocs; 1025 1026 memset(handle_tlbs, 0, sizeof(handle_tlbs)); 1027 memset(labels, 0, sizeof(labels)); 1028 memset(relocs, 0, sizeof(relocs)); 1029 1030 build_r3000_tlbchange_handler_head(&p, K0, K1); 1031 build_pte_writable(&p, &l, &r, K0, K1, label_nopage_tlbs); 1032 uasm_i_nop(&p); /* load delay */ 1033 build_make_write(&p, &r, K0, K1); 1034 build_r3000_tlb_reload_write(&p, &l, &r, K0, K1); 1035 1036 uasm_l_nopage_tlbs(&l, p); 1037 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); 1038 uasm_i_nop(&p); 1039 1040 if ((p - handle_tlbs) > FASTPATH_SIZE) 1041 panic("TLB store handler fastpath space exceeded"); 1042 1043 uasm_resolve_relocs(relocs, labels); 1044 pr_debug("Wrote TLB store handler fastpath (%u instructions).\n", 1045 (unsigned int)(p - handle_tlbs)); 1046 1047 dump_handler(handle_tlbs, ARRAY_SIZE(handle_tlbs)); 1048 } 1049 1050 static void __cpuinit build_r3000_tlb_modify_handler(void) 1051 { 1052 u32 *p = handle_tlbm; 1053 struct uasm_label *l = labels; 1054 struct uasm_reloc *r = relocs; 1055 1056 memset(handle_tlbm, 0, sizeof(handle_tlbm)); 1057 memset(labels, 0, sizeof(labels)); 1058 memset(relocs, 0, sizeof(relocs)); 1059 1060 build_r3000_tlbchange_handler_head(&p, K0, K1); 1061 build_pte_modifiable(&p, &l, &r, K0, K1, label_nopage_tlbm); 1062 uasm_i_nop(&p); /* load delay */ 1063 build_make_write(&p, &r, K0, K1); 1064 build_r3000_pte_reload_tlbwi(&p, K0, K1); 1065 1066 uasm_l_nopage_tlbm(&l, p); 1067 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); 1068 uasm_i_nop(&p); 1069 1070 if ((p - handle_tlbm) > FASTPATH_SIZE) 1071 panic("TLB modify handler fastpath space exceeded"); 1072 1073 uasm_resolve_relocs(relocs, labels); 1074 pr_debug("Wrote TLB modify handler fastpath (%u instructions).\n", 1075 (unsigned int)(p - handle_tlbm)); 1076 1077 dump_handler(handle_tlbm, ARRAY_SIZE(handle_tlbm)); 1078 } 1079 1080 /* 1081 * R4000 style TLB load/store/modify handlers. 1082 */ 1083 static void __cpuinit 1084 build_r4000_tlbchange_handler_head(u32 **p, struct uasm_label **l, 1085 struct uasm_reloc **r, unsigned int pte, 1086 unsigned int ptr) 1087 { 1088 #ifdef CONFIG_64BIT 1089 build_get_pmde64(p, l, r, pte, ptr); /* get pmd in ptr */ 1090 #else 1091 build_get_pgde32(p, pte, ptr); /* get pgd in ptr */ 1092 #endif 1093 1094 UASM_i_MFC0(p, pte, C0_BADVADDR); 1095 UASM_i_LW(p, ptr, 0, ptr); 1096 UASM_i_SRL(p, pte, pte, PAGE_SHIFT + PTE_ORDER - PTE_T_LOG2); 1097 uasm_i_andi(p, pte, pte, (PTRS_PER_PTE - 1) << PTE_T_LOG2); 1098 UASM_i_ADDU(p, ptr, ptr, pte); 1099 1100 #ifdef CONFIG_SMP 1101 uasm_l_smp_pgtable_change(l, *p); 1102 #endif 1103 iPTE_LW(p, l, pte, ptr); /* get even pte */ 1104 if (!m4kc_tlbp_war()) 1105 build_tlb_probe_entry(p); 1106 } 1107 1108 static void __cpuinit 1109 build_r4000_tlbchange_handler_tail(u32 **p, struct uasm_label **l, 1110 struct uasm_reloc **r, unsigned int tmp, 1111 unsigned int ptr) 1112 { 1113 uasm_i_ori(p, ptr, ptr, sizeof(pte_t)); 1114 uasm_i_xori(p, ptr, ptr, sizeof(pte_t)); 1115 build_update_entries(p, tmp, ptr); 1116 build_tlb_write_entry(p, l, r, tlb_indexed); 1117 uasm_l_leave(l, *p); 1118 uasm_i_eret(p); /* return from trap */ 1119 1120 #ifdef CONFIG_64BIT 1121 build_get_pgd_vmalloc64(p, l, r, tmp, ptr); 1122 #endif 1123 } 1124 1125 static void __cpuinit build_r4000_tlb_load_handler(void) 1126 { 1127 u32 *p = handle_tlbl; 1128 struct uasm_label *l = labels; 1129 struct uasm_reloc *r = relocs; 1130 1131 memset(handle_tlbl, 0, sizeof(handle_tlbl)); 1132 memset(labels, 0, sizeof(labels)); 1133 memset(relocs, 0, sizeof(relocs)); 1134 1135 if (bcm1250_m3_war()) { 1136 UASM_i_MFC0(&p, K0, C0_BADVADDR); 1137 UASM_i_MFC0(&p, K1, C0_ENTRYHI); 1138 uasm_i_xor(&p, K0, K0, K1); 1139 UASM_i_SRL(&p, K0, K0, PAGE_SHIFT + 1); 1140 uasm_il_bnez(&p, &r, K0, label_leave); 1141 /* No need for uasm_i_nop */ 1142 } 1143 1144 build_r4000_tlbchange_handler_head(&p, &l, &r, K0, K1); 1145 build_pte_present(&p, &l, &r, K0, K1, label_nopage_tlbl); 1146 if (m4kc_tlbp_war()) 1147 build_tlb_probe_entry(&p); 1148 build_make_valid(&p, &r, K0, K1); 1149 build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1); 1150 1151 uasm_l_nopage_tlbl(&l, p); 1152 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff); 1153 uasm_i_nop(&p); 1154 1155 if ((p - handle_tlbl) > FASTPATH_SIZE) 1156 panic("TLB load handler fastpath space exceeded"); 1157 1158 uasm_resolve_relocs(relocs, labels); 1159 pr_debug("Wrote TLB load handler fastpath (%u instructions).\n", 1160 (unsigned int)(p - handle_tlbl)); 1161 1162 dump_handler(handle_tlbl, ARRAY_SIZE(handle_tlbl)); 1163 } 1164 1165 static void __cpuinit build_r4000_tlb_store_handler(void) 1166 { 1167 u32 *p = handle_tlbs; 1168 struct uasm_label *l = labels; 1169 struct uasm_reloc *r = relocs; 1170 1171 memset(handle_tlbs, 0, sizeof(handle_tlbs)); 1172 memset(labels, 0, sizeof(labels)); 1173 memset(relocs, 0, sizeof(relocs)); 1174 1175 build_r4000_tlbchange_handler_head(&p, &l, &r, K0, K1); 1176 build_pte_writable(&p, &l, &r, K0, K1, label_nopage_tlbs); 1177 if (m4kc_tlbp_war()) 1178 build_tlb_probe_entry(&p); 1179 build_make_write(&p, &r, K0, K1); 1180 build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1); 1181 1182 uasm_l_nopage_tlbs(&l, p); 1183 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); 1184 uasm_i_nop(&p); 1185 1186 if ((p - handle_tlbs) > FASTPATH_SIZE) 1187 panic("TLB store handler fastpath space exceeded"); 1188 1189 uasm_resolve_relocs(relocs, labels); 1190 pr_debug("Wrote TLB store handler fastpath (%u instructions).\n", 1191 (unsigned int)(p - handle_tlbs)); 1192 1193 dump_handler(handle_tlbs, ARRAY_SIZE(handle_tlbs)); 1194 } 1195 1196 static void __cpuinit build_r4000_tlb_modify_handler(void) 1197 { 1198 u32 *p = handle_tlbm; 1199 struct uasm_label *l = labels; 1200 struct uasm_reloc *r = relocs; 1201 1202 memset(handle_tlbm, 0, sizeof(handle_tlbm)); 1203 memset(labels, 0, sizeof(labels)); 1204 memset(relocs, 0, sizeof(relocs)); 1205 1206 build_r4000_tlbchange_handler_head(&p, &l, &r, K0, K1); 1207 build_pte_modifiable(&p, &l, &r, K0, K1, label_nopage_tlbm); 1208 if (m4kc_tlbp_war()) 1209 build_tlb_probe_entry(&p); 1210 /* Present and writable bits set, set accessed and dirty bits. */ 1211 build_make_write(&p, &r, K0, K1); 1212 build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1); 1213 1214 uasm_l_nopage_tlbm(&l, p); 1215 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); 1216 uasm_i_nop(&p); 1217 1218 if ((p - handle_tlbm) > FASTPATH_SIZE) 1219 panic("TLB modify handler fastpath space exceeded"); 1220 1221 uasm_resolve_relocs(relocs, labels); 1222 pr_debug("Wrote TLB modify handler fastpath (%u instructions).\n", 1223 (unsigned int)(p - handle_tlbm)); 1224 1225 dump_handler(handle_tlbm, ARRAY_SIZE(handle_tlbm)); 1226 } 1227 1228 void __cpuinit build_tlb_refill_handler(void) 1229 { 1230 /* 1231 * The refill handler is generated per-CPU, multi-node systems 1232 * may have local storage for it. The other handlers are only 1233 * needed once. 1234 */ 1235 static int run_once = 0; 1236 1237 switch (current_cpu_type()) { 1238 case CPU_R2000: 1239 case CPU_R3000: 1240 case CPU_R3000A: 1241 case CPU_R3081E: 1242 case CPU_TX3912: 1243 case CPU_TX3922: 1244 case CPU_TX3927: 1245 build_r3000_tlb_refill_handler(); 1246 if (!run_once) { 1247 build_r3000_tlb_load_handler(); 1248 build_r3000_tlb_store_handler(); 1249 build_r3000_tlb_modify_handler(); 1250 run_once++; 1251 } 1252 break; 1253 1254 case CPU_R6000: 1255 case CPU_R6000A: 1256 panic("No R6000 TLB refill handler yet"); 1257 break; 1258 1259 case CPU_R8000: 1260 panic("No R8000 TLB refill handler yet"); 1261 break; 1262 1263 default: 1264 build_r4000_tlb_refill_handler(); 1265 if (!run_once) { 1266 build_r4000_tlb_load_handler(); 1267 build_r4000_tlb_store_handler(); 1268 build_r4000_tlb_modify_handler(); 1269 run_once++; 1270 } 1271 } 1272 } 1273 1274 void __cpuinit flush_tlb_handlers(void) 1275 { 1276 flush_icache_range((unsigned long)handle_tlbl, 1277 (unsigned long)handle_tlbl + sizeof(handle_tlbl)); 1278 flush_icache_range((unsigned long)handle_tlbs, 1279 (unsigned long)handle_tlbs + sizeof(handle_tlbs)); 1280 flush_icache_range((unsigned long)handle_tlbm, 1281 (unsigned long)handle_tlbm + sizeof(handle_tlbm)); 1282 } 1283