1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 /* 3 * Low level TLB miss handlers for Book3E 4 * 5 * Copyright (C) 2008-2009 6 * Ben. Herrenschmidt (benh@kernel.crashing.org), IBM Corp. 7 */ 8 9 #include <linux/pgtable.h> 10 #include <asm/processor.h> 11 #include <asm/reg.h> 12 #include <asm/page.h> 13 #include <asm/mmu.h> 14 #include <asm/ppc_asm.h> 15 #include <asm/asm-offsets.h> 16 #include <asm/cputable.h> 17 #include <asm/exception-64e.h> 18 #include <asm/ppc-opcode.h> 19 #include <asm/kvm_asm.h> 20 #include <asm/kvm_booke_hv_asm.h> 21 #include <asm/feature-fixups.h> 22 23 #define VPTE_PMD_SHIFT (PTE_INDEX_SIZE) 24 #define VPTE_PUD_SHIFT (VPTE_PMD_SHIFT + PMD_INDEX_SIZE) 25 #define VPTE_PGD_SHIFT (VPTE_PUD_SHIFT + PUD_INDEX_SIZE) 26 #define VPTE_INDEX_SIZE (VPTE_PGD_SHIFT + PGD_INDEX_SIZE) 27 28 /********************************************************************** 29 * * 30 * TLB miss handling for Book3E with a bolted linear mapping * 31 * No virtual page table, no nested TLB misses * 32 * * 33 **********************************************************************/ 34 35 /* 36 * Note that, unlike non-bolted handlers, TLB_EXFRAME is not 37 * modified by the TLB miss handlers themselves, since the TLB miss 38 * handler code will not itself cause a recursive TLB miss. 39 * 40 * TLB_EXFRAME will be modified when crit/mc/debug exceptions are 41 * entered/exited. 42 */ 43 .macro tlb_prolog_bolted intnum addr 44 mtspr SPRN_SPRG_GEN_SCRATCH,r12 45 mfspr r12,SPRN_SPRG_TLB_EXFRAME 46 std r13,EX_TLB_R13(r12) 47 std r10,EX_TLB_R10(r12) 48 mfspr r13,SPRN_SPRG_PACA 49 50 mfcr r10 51 std r11,EX_TLB_R11(r12) 52 #ifdef CONFIG_KVM_BOOKE_HV 53 BEGIN_FTR_SECTION 54 mfspr r11, SPRN_SRR1 55 END_FTR_SECTION_IFSET(CPU_FTR_EMB_HV) 56 #endif 57 DO_KVM \intnum, SPRN_SRR1 58 std r16,EX_TLB_R16(r12) 59 mfspr r16,\addr /* get faulting address */ 60 std r14,EX_TLB_R14(r12) 61 ld r14,PACAPGD(r13) 62 std r15,EX_TLB_R15(r12) 63 std r10,EX_TLB_CR(r12) 64 START_BTB_FLUSH_SECTION 65 mfspr r11, SPRN_SRR1 66 andi. r10,r11,MSR_PR 67 beq 1f 68 BTB_FLUSH(r10) 69 1: 70 END_BTB_FLUSH_SECTION 71 std r7,EX_TLB_R7(r12) 72 .endm 73 74 .macro tlb_epilog_bolted 75 ld r14,EX_TLB_CR(r12) 76 ld r7,EX_TLB_R7(r12) 77 ld r10,EX_TLB_R10(r12) 78 ld r11,EX_TLB_R11(r12) 79 ld r13,EX_TLB_R13(r12) 80 mtcr r14 81 ld r14,EX_TLB_R14(r12) 82 ld r15,EX_TLB_R15(r12) 83 ld r16,EX_TLB_R16(r12) 84 mfspr r12,SPRN_SPRG_GEN_SCRATCH 85 .endm 86 87 /* Data TLB miss */ 88 START_EXCEPTION(data_tlb_miss_bolted) 89 tlb_prolog_bolted BOOKE_INTERRUPT_DTLB_MISS SPRN_DEAR 90 91 /* We need _PAGE_PRESENT and _PAGE_ACCESSED set */ 92 93 /* We do the user/kernel test for the PID here along with the RW test 94 */ 95 /* We pre-test some combination of permissions to avoid double 96 * faults: 97 * 98 * We move the ESR:ST bit into the position of _PAGE_BAP_SW in the PTE 99 * ESR_ST is 0x00800000 100 * _PAGE_BAP_SW is 0x00000010 101 * So the shift is >> 19. This tests for supervisor writeability. 102 * If the page happens to be supervisor writeable and not user 103 * writeable, we will take a new fault later, but that should be 104 * a rare enough case. 105 * 106 * We also move ESR_ST in _PAGE_DIRTY position 107 * _PAGE_DIRTY is 0x00001000 so the shift is >> 11 108 * 109 * MAS1 is preset for all we need except for TID that needs to 110 * be cleared for kernel translations 111 */ 112 113 mfspr r11,SPRN_ESR 114 115 srdi r15,r16,60 /* get region */ 116 rldicl. r10,r16,64-PGTABLE_EADDR_SIZE,PGTABLE_EADDR_SIZE+4 117 bne- dtlb_miss_fault_bolted /* Bail if fault addr is invalid */ 118 119 rlwinm r10,r11,32-19,27,27 120 rlwimi r10,r11,32-16,19,19 121 cmpwi r15,0 /* user vs kernel check */ 122 ori r10,r10,_PAGE_PRESENT 123 oris r11,r10,_PAGE_ACCESSED@h 124 125 bne tlb_miss_kernel_bolted 126 127 tlb_miss_user_bolted: 128 #ifdef CONFIG_PPC_KUAP 129 mfspr r10,SPRN_MAS1 130 rlwinm. r10,r10,0,0x3fff0000 131 beq- tlb_miss_fault_bolted /* KUAP fault */ 132 #endif 133 134 tlb_miss_common_bolted: 135 /* 136 * This is the guts of the TLB miss handler for bolted-linear. 137 * We are entered with: 138 * 139 * r16 = faulting address 140 * r15 = crap (free to use) 141 * r14 = page table base 142 * r13 = PACA 143 * r11 = PTE permission mask 144 * r10 = crap (free to use) 145 */ 146 rldicl r15,r16,64-PGDIR_SHIFT+3,64-PGD_INDEX_SIZE-3 147 cmpldi cr0,r14,0 148 clrrdi r15,r15,3 149 beq tlb_miss_fault_bolted /* No PGDIR, bail */ 150 151 ldx r14,r14,r15 /* grab pgd entry */ 152 153 rldicl r15,r16,64-PUD_SHIFT+3,64-PUD_INDEX_SIZE-3 154 clrrdi r15,r15,3 155 cmpdi cr0,r14,0 156 bge tlb_miss_fault_bolted /* Bad pgd entry or hugepage; bail */ 157 ldx r14,r14,r15 /* grab pud entry */ 158 159 rldicl r15,r16,64-PMD_SHIFT+3,64-PMD_INDEX_SIZE-3 160 clrrdi r15,r15,3 161 cmpdi cr0,r14,0 162 bge tlb_miss_fault_bolted 163 ldx r14,r14,r15 /* Grab pmd entry */ 164 165 rldicl r15,r16,64-PAGE_SHIFT+3,64-PTE_INDEX_SIZE-3 166 clrrdi r15,r15,3 167 cmpdi cr0,r14,0 168 bge tlb_miss_fault_bolted 169 ldx r14,r14,r15 /* Grab PTE, normal (!huge) page */ 170 171 /* Check if required permissions are met */ 172 andc. r15,r11,r14 173 rldicr r15,r14,64-(PTE_RPN_SHIFT-PAGE_SHIFT),63-PAGE_SHIFT 174 bne- tlb_miss_fault_bolted 175 176 /* Now we build the MAS: 177 * 178 * MAS 0 : Fully setup with defaults in MAS4 and TLBnCFG 179 * MAS 1 : Almost fully setup 180 * - PID already updated by caller if necessary 181 * - TSIZE need change if !base page size, not 182 * yet implemented for now 183 * MAS 2 : Defaults not useful, need to be redone 184 * MAS 3+7 : Needs to be done 185 */ 186 clrrdi r11,r16,12 /* Clear low crap in EA */ 187 clrldi r15,r15,12 /* Clear crap at the top */ 188 rlwimi r11,r14,32-19,27,31 /* Insert WIMGE */ 189 rlwimi r15,r14,32-8,22,25 /* Move in U bits */ 190 mtspr SPRN_MAS2,r11 191 andi. r11,r14,_PAGE_DIRTY 192 rlwimi r15,r14,32-2,26,31 /* Move in BAP bits */ 193 194 /* Mask out SW and UW if !DIRTY (XXX optimize this !) */ 195 bne 1f 196 li r11,MAS3_SW|MAS3_UW 197 andc r15,r15,r11 198 1: 199 mtspr SPRN_MAS7_MAS3,r15 200 tlbwe 201 202 tlb_miss_done_bolted: 203 tlb_epilog_bolted 204 rfi 205 206 itlb_miss_kernel_bolted: 207 li r11,_PAGE_PRESENT|_PAGE_BAP_SX /* Base perm */ 208 oris r11,r11,_PAGE_ACCESSED@h 209 tlb_miss_kernel_bolted: 210 mfspr r10,SPRN_MAS1 211 ld r14,PACA_KERNELPGD(r13) 212 srdi r15,r16,44 /* get kernel region */ 213 andi. r15,r15,1 /* Check for vmalloc region */ 214 rlwinm r10,r10,0,16,1 /* Clear TID */ 215 mtspr SPRN_MAS1,r10 216 bne+ tlb_miss_common_bolted 217 218 tlb_miss_fault_bolted: 219 /* We need to check if it was an instruction miss */ 220 andi. r10,r11,_PAGE_BAP_UX|_PAGE_BAP_SX 221 bne itlb_miss_fault_bolted 222 dtlb_miss_fault_bolted: 223 tlb_epilog_bolted 224 b exc_data_storage_book3e 225 itlb_miss_fault_bolted: 226 tlb_epilog_bolted 227 b exc_instruction_storage_book3e 228 229 /* Instruction TLB miss */ 230 START_EXCEPTION(instruction_tlb_miss_bolted) 231 tlb_prolog_bolted BOOKE_INTERRUPT_ITLB_MISS SPRN_SRR0 232 233 rldicl. r10,r16,64-PGTABLE_EADDR_SIZE,PGTABLE_EADDR_SIZE+4 234 srdi r15,r16,60 /* get region */ 235 bne- itlb_miss_fault_bolted 236 237 li r11,_PAGE_PRESENT|_PAGE_BAP_UX /* Base perm */ 238 239 /* We do the user/kernel test for the PID here along with the RW test 240 */ 241 242 cmpldi cr0,r15,0 /* Check for user region */ 243 oris r11,r11,_PAGE_ACCESSED@h 244 beq tlb_miss_user_bolted 245 b itlb_miss_kernel_bolted 246 247 /* 248 * TLB miss handling for e6500 and derivatives, using hardware tablewalk. 249 * 250 * Linear mapping is bolted: no virtual page table or nested TLB misses 251 * Indirect entries in TLB1, hardware loads resulting direct entries 252 * into TLB0 253 * No HES or NV hint on TLB1, so we need to do software round-robin 254 * No tlbsrx. so we need a spinlock, and we have to deal 255 * with MAS-damage caused by tlbsx 256 * 4K pages only 257 */ 258 259 START_EXCEPTION(instruction_tlb_miss_e6500) 260 tlb_prolog_bolted BOOKE_INTERRUPT_ITLB_MISS SPRN_SRR0 261 262 ld r11,PACA_TCD_PTR(r13) 263 srdi. r15,r16,60 /* get region */ 264 ori r16,r16,1 265 266 bne tlb_miss_kernel_e6500 /* user/kernel test */ 267 268 b tlb_miss_common_e6500 269 270 START_EXCEPTION(data_tlb_miss_e6500) 271 tlb_prolog_bolted BOOKE_INTERRUPT_DTLB_MISS SPRN_DEAR 272 273 ld r11,PACA_TCD_PTR(r13) 274 srdi. r15,r16,60 /* get region */ 275 rldicr r16,r16,0,62 276 277 bne tlb_miss_kernel_e6500 /* user vs kernel check */ 278 279 /* 280 * This is the guts of the TLB miss handler for e6500 and derivatives. 281 * We are entered with: 282 * 283 * r16 = page of faulting address (low bit 0 if data, 1 if instruction) 284 * r15 = crap (free to use) 285 * r14 = page table base 286 * r13 = PACA 287 * r11 = tlb_per_core ptr 288 * r10 = crap (free to use) 289 * r7 = esel_next 290 */ 291 tlb_miss_common_e6500: 292 crmove cr2*4+2,cr0*4+2 /* cr2.eq != 0 if kernel address */ 293 294 BEGIN_FTR_SECTION /* CPU_FTR_SMT */ 295 /* 296 * Search if we already have an indirect entry for that virtual 297 * address, and if we do, bail out. 298 * 299 * MAS6:IND should be already set based on MAS4 300 */ 301 lhz r10,PACAPACAINDEX(r13) 302 addi r10,r10,1 303 crclr cr1*4+eq /* set cr1.eq = 0 for non-recursive */ 304 1: lbarx r15,0,r11 305 cmpdi r15,0 306 bne 2f 307 stbcx. r10,0,r11 308 bne 1b 309 3: 310 .subsection 1 311 2: cmpd cr1,r15,r10 /* recursive lock due to mcheck/crit/etc? */ 312 beq cr1,3b /* unlock will happen if cr1.eq = 0 */ 313 10: lbz r15,0(r11) 314 cmpdi r15,0 315 bne 10b 316 b 1b 317 .previous 318 END_FTR_SECTION_IFSET(CPU_FTR_SMT) 319 320 lbz r7,TCD_ESEL_NEXT(r11) 321 322 BEGIN_FTR_SECTION /* CPU_FTR_SMT */ 323 /* 324 * Erratum A-008139 says that we can't use tlbwe to change 325 * an indirect entry in any way (including replacing or 326 * invalidating) if the other thread could be in the process 327 * of a lookup. The workaround is to invalidate the entry 328 * with tlbilx before overwriting. 329 */ 330 331 rlwinm r10,r7,16,0xff0000 332 oris r10,r10,MAS0_TLBSEL(1)@h 333 mtspr SPRN_MAS0,r10 334 isync 335 tlbre 336 mfspr r15,SPRN_MAS1 337 andis. r15,r15,MAS1_VALID@h 338 beq 5f 339 340 BEGIN_FTR_SECTION_NESTED(532) 341 mfspr r10,SPRN_MAS8 342 rlwinm r10,r10,0,0x80000fff /* tgs,tlpid -> sgs,slpid */ 343 mtspr SPRN_MAS5,r10 344 END_FTR_SECTION_NESTED(CPU_FTR_EMB_HV,CPU_FTR_EMB_HV,532) 345 346 mfspr r10,SPRN_MAS1 347 rlwinm r15,r10,0,0x3fff0000 /* tid -> spid */ 348 rlwimi r15,r10,20,0x00000003 /* ind,ts -> sind,sas */ 349 mfspr r10,SPRN_MAS6 350 mtspr SPRN_MAS6,r15 351 352 mfspr r15,SPRN_MAS2 353 isync 354 tlbilxva 0,r15 355 isync 356 357 mtspr SPRN_MAS6,r10 358 359 5: 360 BEGIN_FTR_SECTION_NESTED(532) 361 li r10,0 362 mtspr SPRN_MAS8,r10 363 mtspr SPRN_MAS5,r10 364 END_FTR_SECTION_NESTED(CPU_FTR_EMB_HV,CPU_FTR_EMB_HV,532) 365 366 tlbsx 0,r16 367 mfspr r10,SPRN_MAS1 368 andis. r15,r10,MAS1_VALID@h 369 bne tlb_miss_done_e6500 370 FTR_SECTION_ELSE 371 mfspr r10,SPRN_MAS1 372 ALT_FTR_SECTION_END_IFSET(CPU_FTR_SMT) 373 374 oris r10,r10,MAS1_VALID@h 375 beq cr2,4f 376 rlwinm r10,r10,0,16,1 /* Clear TID */ 377 4: mtspr SPRN_MAS1,r10 378 379 /* Now, we need to walk the page tables. First check if we are in 380 * range. 381 */ 382 rldicl. r10,r16,64-PGTABLE_EADDR_SIZE,PGTABLE_EADDR_SIZE+4 383 bne- tlb_miss_fault_e6500 384 385 rldicl r15,r16,64-PGDIR_SHIFT+3,64-PGD_INDEX_SIZE-3 386 cmpldi cr0,r14,0 387 clrrdi r15,r15,3 388 beq- tlb_miss_fault_e6500 /* No PGDIR, bail */ 389 ldx r14,r14,r15 /* grab pgd entry */ 390 391 rldicl r15,r16,64-PUD_SHIFT+3,64-PUD_INDEX_SIZE-3 392 clrrdi r15,r15,3 393 cmpdi cr0,r14,0 394 bge tlb_miss_huge_e6500 /* Bad pgd entry or hugepage; bail */ 395 ldx r14,r14,r15 /* grab pud entry */ 396 397 rldicl r15,r16,64-PMD_SHIFT+3,64-PMD_INDEX_SIZE-3 398 clrrdi r15,r15,3 399 cmpdi cr0,r14,0 400 bge tlb_miss_huge_e6500 401 ldx r14,r14,r15 /* Grab pmd entry */ 402 403 mfspr r10,SPRN_MAS0 404 cmpdi cr0,r14,0 405 bge tlb_miss_huge_e6500 406 407 /* Now we build the MAS for a 2M indirect page: 408 * 409 * MAS 0 : ESEL needs to be filled by software round-robin 410 * MAS 1 : Fully set up 411 * - PID already updated by caller if necessary 412 * - TSIZE for now is base ind page size always 413 * - TID already cleared if necessary 414 * MAS 2 : Default not 2M-aligned, need to be redone 415 * MAS 3+7 : Needs to be done 416 */ 417 418 ori r14,r14,(BOOK3E_PAGESZ_4K << MAS3_SPSIZE_SHIFT) 419 mtspr SPRN_MAS7_MAS3,r14 420 421 clrrdi r15,r16,21 /* make EA 2M-aligned */ 422 mtspr SPRN_MAS2,r15 423 424 tlb_miss_huge_done_e6500: 425 lbz r16,TCD_ESEL_MAX(r11) 426 lbz r14,TCD_ESEL_FIRST(r11) 427 rlwimi r10,r7,16,0x00ff0000 /* insert esel_next into MAS0 */ 428 addi r7,r7,1 /* increment esel_next */ 429 mtspr SPRN_MAS0,r10 430 cmpw r7,r16 431 iseleq r7,r14,r7 /* if next == last use first */ 432 stb r7,TCD_ESEL_NEXT(r11) 433 434 tlbwe 435 436 tlb_miss_done_e6500: 437 .macro tlb_unlock_e6500 438 BEGIN_FTR_SECTION 439 beq cr1,1f /* no unlock if lock was recursively grabbed */ 440 li r15,0 441 isync 442 stb r15,0(r11) 443 1: 444 END_FTR_SECTION_IFSET(CPU_FTR_SMT) 445 .endm 446 447 tlb_unlock_e6500 448 tlb_epilog_bolted 449 rfi 450 451 tlb_miss_huge_e6500: 452 beq tlb_miss_fault_e6500 453 li r10,1 454 andi. r15,r14,HUGEPD_SHIFT_MASK@l /* r15 = psize */ 455 rldimi r14,r10,63,0 /* Set PD_HUGE */ 456 xor r14,r14,r15 /* Clear size bits */ 457 ldx r14,0,r14 458 459 /* 460 * Now we build the MAS for a huge page. 461 * 462 * MAS 0 : ESEL needs to be filled by software round-robin 463 * - can be handled by indirect code 464 * MAS 1 : Need to clear IND and set TSIZE 465 * MAS 2,3+7: Needs to be redone similar to non-tablewalk handler 466 */ 467 468 subi r15,r15,10 /* Convert psize to tsize */ 469 mfspr r10,SPRN_MAS1 470 rlwinm r10,r10,0,~MAS1_IND 471 rlwimi r10,r15,MAS1_TSIZE_SHIFT,MAS1_TSIZE_MASK 472 mtspr SPRN_MAS1,r10 473 474 li r10,-0x400 475 sld r15,r10,r15 /* Generate mask based on size */ 476 and r10,r16,r15 477 rldicr r15,r14,64-(PTE_RPN_SHIFT-PAGE_SHIFT),63-PAGE_SHIFT 478 rlwimi r10,r14,32-19,27,31 /* Insert WIMGE */ 479 clrldi r15,r15,PAGE_SHIFT /* Clear crap at the top */ 480 rlwimi r15,r14,32-8,22,25 /* Move in U bits */ 481 mtspr SPRN_MAS2,r10 482 andi. r10,r14,_PAGE_DIRTY 483 rlwimi r15,r14,32-2,26,31 /* Move in BAP bits */ 484 485 /* Mask out SW and UW if !DIRTY (XXX optimize this !) */ 486 bne 1f 487 li r10,MAS3_SW|MAS3_UW 488 andc r15,r15,r10 489 1: 490 mtspr SPRN_MAS7_MAS3,r15 491 492 mfspr r10,SPRN_MAS0 493 b tlb_miss_huge_done_e6500 494 495 tlb_miss_kernel_e6500: 496 ld r14,PACA_KERNELPGD(r13) 497 srdi r15,r16,44 /* get kernel region */ 498 xoris r15,r15,0xc /* Check for vmalloc region */ 499 cmplwi cr1,r15,1 500 beq+ cr1,tlb_miss_common_e6500 501 502 tlb_miss_fault_e6500: 503 tlb_unlock_e6500 504 /* We need to check if it was an instruction miss */ 505 andi. r16,r16,1 506 bne itlb_miss_fault_e6500 507 dtlb_miss_fault_e6500: 508 tlb_epilog_bolted 509 b exc_data_storage_book3e 510 itlb_miss_fault_e6500: 511 tlb_epilog_bolted 512 b exc_instruction_storage_book3e 513 514 /********************************************************************** 515 * * 516 * TLB miss handling for Book3E with TLB reservation and HES support * 517 * * 518 **********************************************************************/ 519 520 521 /* Data TLB miss */ 522 START_EXCEPTION(data_tlb_miss) 523 TLB_MISS_PROLOG 524 525 /* Now we handle the fault proper. We only save DEAR in normal 526 * fault case since that's the only interesting values here. 527 * We could probably also optimize by not saving SRR0/1 in the 528 * linear mapping case but I'll leave that for later 529 */ 530 mfspr r14,SPRN_ESR 531 mfspr r16,SPRN_DEAR /* get faulting address */ 532 srdi r15,r16,44 /* get region */ 533 xoris r15,r15,0xc 534 cmpldi cr0,r15,0 /* linear mapping ? */ 535 beq tlb_load_linear /* yes -> go to linear map load */ 536 cmpldi cr1,r15,1 /* vmalloc mapping ? */ 537 538 /* The page tables are mapped virtually linear. At this point, though, 539 * we don't know whether we are trying to fault in a first level 540 * virtual address or a virtual page table address. We can get that 541 * from bit 0x1 of the region ID which we have set for a page table 542 */ 543 andis. r10,r15,0x1 544 bne- virt_page_table_tlb_miss 545 546 std r14,EX_TLB_ESR(r12); /* save ESR */ 547 std r16,EX_TLB_DEAR(r12); /* save DEAR */ 548 549 /* We need _PAGE_PRESENT and _PAGE_ACCESSED set */ 550 li r11,_PAGE_PRESENT 551 oris r11,r11,_PAGE_ACCESSED@h 552 553 /* We do the user/kernel test for the PID here along with the RW test 554 */ 555 srdi. r15,r16,60 /* Check for user region */ 556 557 /* We pre-test some combination of permissions to avoid double 558 * faults: 559 * 560 * We move the ESR:ST bit into the position of _PAGE_BAP_SW in the PTE 561 * ESR_ST is 0x00800000 562 * _PAGE_BAP_SW is 0x00000010 563 * So the shift is >> 19. This tests for supervisor writeability. 564 * If the page happens to be supervisor writeable and not user 565 * writeable, we will take a new fault later, but that should be 566 * a rare enough case. 567 * 568 * We also move ESR_ST in _PAGE_DIRTY position 569 * _PAGE_DIRTY is 0x00001000 so the shift is >> 11 570 * 571 * MAS1 is preset for all we need except for TID that needs to 572 * be cleared for kernel translations 573 */ 574 rlwimi r11,r14,32-19,27,27 575 rlwimi r11,r14,32-16,19,19 576 beq normal_tlb_miss_user 577 /* XXX replace the RMW cycles with immediate loads + writes */ 578 1: mfspr r10,SPRN_MAS1 579 rlwinm r10,r10,0,16,1 /* Clear TID */ 580 mtspr SPRN_MAS1,r10 581 beq+ cr1,normal_tlb_miss 582 583 /* We got a crappy address, just fault with whatever DEAR and ESR 584 * are here 585 */ 586 TLB_MISS_EPILOG_ERROR 587 b exc_data_storage_book3e 588 589 /* Instruction TLB miss */ 590 START_EXCEPTION(instruction_tlb_miss) 591 TLB_MISS_PROLOG 592 593 /* If we take a recursive fault, the second level handler may need 594 * to know whether we are handling a data or instruction fault in 595 * order to get to the right store fault handler. We provide that 596 * info by writing a crazy value in ESR in our exception frame 597 */ 598 li r14,-1 /* store to exception frame is done later */ 599 600 /* Now we handle the fault proper. We only save DEAR in the non 601 * linear mapping case since we know the linear mapping case will 602 * not re-enter. We could indeed optimize and also not save SRR0/1 603 * in the linear mapping case but I'll leave that for later 604 * 605 * Faulting address is SRR0 which is already in r16 606 */ 607 srdi r15,r16,44 /* get region */ 608 xoris r15,r15,0xc 609 cmpldi cr0,r15,0 /* linear mapping ? */ 610 beq tlb_load_linear /* yes -> go to linear map load */ 611 cmpldi cr1,r15,1 /* vmalloc mapping ? */ 612 613 /* We do the user/kernel test for the PID here along with the RW test 614 */ 615 li r11,_PAGE_PRESENT|_PAGE_BAP_UX /* Base perm */ 616 oris r11,r11,_PAGE_ACCESSED@h 617 618 srdi. r15,r16,60 /* Check for user region */ 619 std r14,EX_TLB_ESR(r12) /* write crazy -1 to frame */ 620 beq normal_tlb_miss_user 621 622 li r11,_PAGE_PRESENT|_PAGE_BAP_SX /* Base perm */ 623 oris r11,r11,_PAGE_ACCESSED@h 624 /* XXX replace the RMW cycles with immediate loads + writes */ 625 mfspr r10,SPRN_MAS1 626 rlwinm r10,r10,0,16,1 /* Clear TID */ 627 mtspr SPRN_MAS1,r10 628 beq+ cr1,normal_tlb_miss 629 630 /* We got a crappy address, just fault */ 631 TLB_MISS_EPILOG_ERROR 632 b exc_instruction_storage_book3e 633 634 /* 635 * This is the guts of the first-level TLB miss handler for direct 636 * misses. We are entered with: 637 * 638 * r16 = faulting address 639 * r15 = region ID 640 * r14 = crap (free to use) 641 * r13 = PACA 642 * r12 = TLB exception frame in PACA 643 * r11 = PTE permission mask 644 * r10 = crap (free to use) 645 */ 646 normal_tlb_miss_user: 647 #ifdef CONFIG_PPC_KUAP 648 mfspr r14,SPRN_MAS1 649 rlwinm. r14,r14,0,0x3fff0000 650 beq- normal_tlb_miss_access_fault /* KUAP fault */ 651 #endif 652 normal_tlb_miss: 653 /* So we first construct the page table address. We do that by 654 * shifting the bottom of the address (not the region ID) by 655 * PAGE_SHIFT-3, clearing the bottom 3 bits (get a PTE ptr) and 656 * or'ing the fourth high bit. 657 * 658 * NOTE: For 64K pages, we do things slightly differently in 659 * order to handle the weird page table format used by linux 660 */ 661 srdi r15,r16,44 662 oris r10,r15,0x1 663 rldicl r14,r16,64-(PAGE_SHIFT-3),PAGE_SHIFT-3+4 664 sldi r15,r10,44 665 clrrdi r14,r14,19 666 or r10,r15,r14 667 668 ld r14,0(r10) 669 670 finish_normal_tlb_miss: 671 /* Check if required permissions are met */ 672 andc. r15,r11,r14 673 bne- normal_tlb_miss_access_fault 674 675 /* Now we build the MAS: 676 * 677 * MAS 0 : Fully setup with defaults in MAS4 and TLBnCFG 678 * MAS 1 : Almost fully setup 679 * - PID already updated by caller if necessary 680 * - TSIZE need change if !base page size, not 681 * yet implemented for now 682 * MAS 2 : Defaults not useful, need to be redone 683 * MAS 3+7 : Needs to be done 684 * 685 * TODO: mix up code below for better scheduling 686 */ 687 clrrdi r10,r16,12 /* Clear low crap in EA */ 688 rlwimi r10,r14,32-19,27,31 /* Insert WIMGE */ 689 mtspr SPRN_MAS2,r10 690 691 /* Check page size, if not standard, update MAS1 */ 692 rldicl r10,r14,64-8,64-8 693 cmpldi cr0,r10,BOOK3E_PAGESZ_4K 694 beq- 1f 695 mfspr r11,SPRN_MAS1 696 rlwimi r11,r14,31,21,24 697 rlwinm r11,r11,0,21,19 698 mtspr SPRN_MAS1,r11 699 1: 700 /* Move RPN in position */ 701 rldicr r11,r14,64-(PTE_RPN_SHIFT-PAGE_SHIFT),63-PAGE_SHIFT 702 clrldi r15,r11,12 /* Clear crap at the top */ 703 rlwimi r15,r14,32-8,22,25 /* Move in U bits */ 704 rlwimi r15,r14,32-2,26,31 /* Move in BAP bits */ 705 706 /* Mask out SW and UW if !DIRTY (XXX optimize this !) */ 707 andi. r11,r14,_PAGE_DIRTY 708 bne 1f 709 li r11,MAS3_SW|MAS3_UW 710 andc r15,r15,r11 711 1: 712 srdi r16,r15,32 713 mtspr SPRN_MAS3,r15 714 mtspr SPRN_MAS7,r16 715 716 tlbwe 717 718 normal_tlb_miss_done: 719 /* We don't bother with restoring DEAR or ESR since we know we are 720 * level 0 and just going back to userland. They are only needed 721 * if you are going to take an access fault 722 */ 723 TLB_MISS_EPILOG_SUCCESS 724 rfi 725 726 normal_tlb_miss_access_fault: 727 /* We need to check if it was an instruction miss */ 728 andi. r10,r11,_PAGE_BAP_UX 729 bne 1f 730 ld r14,EX_TLB_DEAR(r12) 731 ld r15,EX_TLB_ESR(r12) 732 mtspr SPRN_DEAR,r14 733 mtspr SPRN_ESR,r15 734 TLB_MISS_EPILOG_ERROR 735 b exc_data_storage_book3e 736 1: TLB_MISS_EPILOG_ERROR 737 b exc_instruction_storage_book3e 738 739 740 /* 741 * This is the guts of the second-level TLB miss handler for direct 742 * misses. We are entered with: 743 * 744 * r16 = virtual page table faulting address 745 * r15 = region (top 4 bits of address) 746 * r14 = crap (free to use) 747 * r13 = PACA 748 * r12 = TLB exception frame in PACA 749 * r11 = crap (free to use) 750 * r10 = crap (free to use) 751 * 752 * Note that this should only ever be called as a second level handler 753 * with the current scheme when using SW load. 754 * That means we can always get the original fault DEAR at 755 * EX_TLB_DEAR-EX_TLB_SIZE(r12) 756 * 757 * It can be re-entered by the linear mapping miss handler. However, to 758 * avoid too much complication, it will restart the whole fault at level 759 * 0 so we don't care too much about clobbers 760 * 761 * XXX That code was written back when we couldn't clobber r14. We can now, 762 * so we could probably optimize things a bit 763 */ 764 virt_page_table_tlb_miss: 765 /* Are we hitting a kernel page table ? */ 766 srdi r15,r16,60 767 andi. r10,r15,0x8 768 769 /* The cool thing now is that r10 contains 0 for user and 8 for kernel, 770 * and we happen to have the swapper_pg_dir at offset 8 from the user 771 * pgdir in the PACA :-). 772 */ 773 add r11,r10,r13 774 775 /* If kernel, we need to clear MAS1 TID */ 776 beq 1f 777 /* XXX replace the RMW cycles with immediate loads + writes */ 778 mfspr r10,SPRN_MAS1 779 rlwinm r10,r10,0,16,1 /* Clear TID */ 780 mtspr SPRN_MAS1,r10 781 #ifdef CONFIG_PPC_KUAP 782 b 2f 783 1: 784 mfspr r10,SPRN_MAS1 785 rlwinm. r10,r10,0,0x3fff0000 786 beq- virt_page_table_tlb_miss_fault /* KUAP fault */ 787 2: 788 #else 789 1: 790 #endif 791 792 /* Now, we need to walk the page tables. First check if we are in 793 * range. 794 */ 795 rldicl r10,r16,64-(VPTE_INDEX_SIZE+3),VPTE_INDEX_SIZE+3+4 796 cmpldi r10,0x80 797 bne- virt_page_table_tlb_miss_fault 798 799 /* Get the PGD pointer */ 800 ld r15,PACAPGD(r11) 801 cmpldi cr0,r15,0 802 beq- virt_page_table_tlb_miss_fault 803 804 /* Get to PGD entry */ 805 rldicl r11,r16,64-VPTE_PGD_SHIFT,64-PGD_INDEX_SIZE-3 806 clrrdi r10,r11,3 807 ldx r15,r10,r15 808 cmpdi cr0,r15,0 809 bge virt_page_table_tlb_miss_fault 810 811 /* Get to PUD entry */ 812 rldicl r11,r16,64-VPTE_PUD_SHIFT,64-PUD_INDEX_SIZE-3 813 clrrdi r10,r11,3 814 ldx r15,r10,r15 815 cmpdi cr0,r15,0 816 bge virt_page_table_tlb_miss_fault 817 818 /* Get to PMD entry */ 819 rldicl r11,r16,64-VPTE_PMD_SHIFT,64-PMD_INDEX_SIZE-3 820 clrrdi r10,r11,3 821 ldx r15,r10,r15 822 cmpdi cr0,r15,0 823 bge virt_page_table_tlb_miss_fault 824 825 /* Ok, we're all right, we can now create a kernel translation for 826 * a 4K or 64K page from r16 -> r15. 827 */ 828 /* Now we build the MAS: 829 * 830 * MAS 0 : Fully setup with defaults in MAS4 and TLBnCFG 831 * MAS 1 : Almost fully setup 832 * - PID already updated by caller if necessary 833 * - TSIZE for now is base page size always 834 * MAS 2 : Use defaults 835 * MAS 3+7 : Needs to be done 836 * 837 * So we only do MAS 2 and 3 for now... 838 */ 839 clrldi r11,r15,4 /* remove region ID from RPN */ 840 ori r10,r11,1 /* Or-in SR */ 841 842 srdi r16,r10,32 843 mtspr SPRN_MAS3,r10 844 mtspr SPRN_MAS7,r16 845 846 tlbwe 847 848 /* Return to caller, normal case */ 849 TLB_MISS_EPILOG_SUCCESS 850 rfi 851 852 virt_page_table_tlb_miss_fault: 853 /* If we fault here, things are a little bit tricky. We need to call 854 * either data or instruction store fault, and we need to retrieve 855 * the original fault address and ESR (for data). 856 * 857 * The thing is, we know that in normal circumstances, this is 858 * always called as a second level tlb miss for SW load or as a first 859 * level TLB miss for HW load, so we should be able to peek at the 860 * relevant information in the first exception frame in the PACA. 861 * 862 * However, we do need to double check that, because we may just hit 863 * a stray kernel pointer or a userland attack trying to hit those 864 * areas. If that is the case, we do a data fault. (We can't get here 865 * from an instruction tlb miss anyway). 866 * 867 * Note also that when going to a fault, we must unwind the previous 868 * level as well. Since we are doing that, we don't need to clear or 869 * restore the TLB reservation neither. 870 */ 871 subf r10,r13,r12 872 cmpldi cr0,r10,PACA_EXTLB+EX_TLB_SIZE 873 bne- virt_page_table_tlb_miss_whacko_fault 874 875 /* We dig the original DEAR and ESR from slot 0 */ 876 ld r15,EX_TLB_DEAR+PACA_EXTLB(r13) 877 ld r16,EX_TLB_ESR+PACA_EXTLB(r13) 878 879 /* We check for the "special" ESR value for instruction faults */ 880 cmpdi cr0,r16,-1 881 beq 1f 882 mtspr SPRN_DEAR,r15 883 mtspr SPRN_ESR,r16 884 TLB_MISS_EPILOG_ERROR 885 b exc_data_storage_book3e 886 1: TLB_MISS_EPILOG_ERROR 887 b exc_instruction_storage_book3e 888 889 virt_page_table_tlb_miss_whacko_fault: 890 /* The linear fault will restart everything so ESR and DEAR will 891 * not have been clobbered, let's just fault with what we have 892 */ 893 TLB_MISS_EPILOG_ERROR 894 b exc_data_storage_book3e 895 896 897 /************************************************************** 898 * * 899 * TLB miss handling for Book3E with hw page table support * 900 * * 901 **************************************************************/ 902 903 904 /* Data TLB miss */ 905 START_EXCEPTION(data_tlb_miss_htw) 906 TLB_MISS_PROLOG 907 908 /* Now we handle the fault proper. We only save DEAR in normal 909 * fault case since that's the only interesting values here. 910 * We could probably also optimize by not saving SRR0/1 in the 911 * linear mapping case but I'll leave that for later 912 */ 913 mfspr r14,SPRN_ESR 914 mfspr r16,SPRN_DEAR /* get faulting address */ 915 srdi r11,r16,44 /* get region */ 916 xoris r11,r11,0xc 917 cmpldi cr0,r11,0 /* linear mapping ? */ 918 beq tlb_load_linear /* yes -> go to linear map load */ 919 cmpldi cr1,r11,1 /* vmalloc mapping ? */ 920 921 /* We do the user/kernel test for the PID here along with the RW test 922 */ 923 srdi. r11,r16,60 /* Check for user region */ 924 ld r15,PACAPGD(r13) /* Load user pgdir */ 925 beq htw_tlb_miss 926 927 /* XXX replace the RMW cycles with immediate loads + writes */ 928 1: mfspr r10,SPRN_MAS1 929 rlwinm r10,r10,0,16,1 /* Clear TID */ 930 mtspr SPRN_MAS1,r10 931 ld r15,PACA_KERNELPGD(r13) /* Load kernel pgdir */ 932 beq+ cr1,htw_tlb_miss 933 934 /* We got a crappy address, just fault with whatever DEAR and ESR 935 * are here 936 */ 937 TLB_MISS_EPILOG_ERROR 938 b exc_data_storage_book3e 939 940 /* Instruction TLB miss */ 941 START_EXCEPTION(instruction_tlb_miss_htw) 942 TLB_MISS_PROLOG 943 944 /* If we take a recursive fault, the second level handler may need 945 * to know whether we are handling a data or instruction fault in 946 * order to get to the right store fault handler. We provide that 947 * info by keeping a crazy value for ESR in r14 948 */ 949 li r14,-1 /* store to exception frame is done later */ 950 951 /* Now we handle the fault proper. We only save DEAR in the non 952 * linear mapping case since we know the linear mapping case will 953 * not re-enter. We could indeed optimize and also not save SRR0/1 954 * in the linear mapping case but I'll leave that for later 955 * 956 * Faulting address is SRR0 which is already in r16 957 */ 958 srdi r11,r16,44 /* get region */ 959 xoris r11,r11,0xc 960 cmpldi cr0,r11,0 /* linear mapping ? */ 961 beq tlb_load_linear /* yes -> go to linear map load */ 962 cmpldi cr1,r11,1 /* vmalloc mapping ? */ 963 964 /* We do the user/kernel test for the PID here along with the RW test 965 */ 966 srdi. r11,r16,60 /* Check for user region */ 967 ld r15,PACAPGD(r13) /* Load user pgdir */ 968 beq htw_tlb_miss 969 970 /* XXX replace the RMW cycles with immediate loads + writes */ 971 1: mfspr r10,SPRN_MAS1 972 rlwinm r10,r10,0,16,1 /* Clear TID */ 973 mtspr SPRN_MAS1,r10 974 ld r15,PACA_KERNELPGD(r13) /* Load kernel pgdir */ 975 beq+ htw_tlb_miss 976 977 /* We got a crappy address, just fault */ 978 TLB_MISS_EPILOG_ERROR 979 b exc_instruction_storage_book3e 980 981 982 /* 983 * This is the guts of the second-level TLB miss handler for direct 984 * misses. We are entered with: 985 * 986 * r16 = virtual page table faulting address 987 * r15 = PGD pointer 988 * r14 = ESR 989 * r13 = PACA 990 * r12 = TLB exception frame in PACA 991 * r11 = crap (free to use) 992 * r10 = crap (free to use) 993 * 994 * It can be re-entered by the linear mapping miss handler. However, to 995 * avoid too much complication, it will save/restore things for us 996 */ 997 htw_tlb_miss: 998 #ifdef CONFIG_PPC_KUAP 999 mfspr r10,SPRN_MAS1 1000 rlwinm. r10,r10,0,0x3fff0000 1001 beq- htw_tlb_miss_fault /* KUAP fault */ 1002 #endif 1003 /* Search if we already have a TLB entry for that virtual address, and 1004 * if we do, bail out. 1005 * 1006 * MAS1:IND should be already set based on MAS4 1007 */ 1008 PPC_TLBSRX_DOT(0,R16) 1009 beq htw_tlb_miss_done 1010 1011 /* Now, we need to walk the page tables. First check if we are in 1012 * range. 1013 */ 1014 rldicl. r10,r16,64-PGTABLE_EADDR_SIZE,PGTABLE_EADDR_SIZE+4 1015 bne- htw_tlb_miss_fault 1016 1017 /* Get the PGD pointer */ 1018 cmpldi cr0,r15,0 1019 beq- htw_tlb_miss_fault 1020 1021 /* Get to PGD entry */ 1022 rldicl r11,r16,64-(PGDIR_SHIFT-3),64-PGD_INDEX_SIZE-3 1023 clrrdi r10,r11,3 1024 ldx r15,r10,r15 1025 cmpdi cr0,r15,0 1026 bge htw_tlb_miss_fault 1027 1028 /* Get to PUD entry */ 1029 rldicl r11,r16,64-(PUD_SHIFT-3),64-PUD_INDEX_SIZE-3 1030 clrrdi r10,r11,3 1031 ldx r15,r10,r15 1032 cmpdi cr0,r15,0 1033 bge htw_tlb_miss_fault 1034 1035 /* Get to PMD entry */ 1036 rldicl r11,r16,64-(PMD_SHIFT-3),64-PMD_INDEX_SIZE-3 1037 clrrdi r10,r11,3 1038 ldx r15,r10,r15 1039 cmpdi cr0,r15,0 1040 bge htw_tlb_miss_fault 1041 1042 /* Ok, we're all right, we can now create an indirect entry for 1043 * a 1M or 256M page. 1044 * 1045 * The last trick is now that because we use "half" pages for 1046 * the HTW (1M IND is 2K and 256M IND is 32K) we need to account 1047 * for an added LSB bit to the RPN. For 64K pages, there is no 1048 * problem as we already use 32K arrays (half PTE pages), but for 1049 * 4K page we need to extract a bit from the virtual address and 1050 * insert it into the "PA52" bit of the RPN. 1051 */ 1052 rlwimi r15,r16,32-9,20,20 1053 /* Now we build the MAS: 1054 * 1055 * MAS 0 : Fully setup with defaults in MAS4 and TLBnCFG 1056 * MAS 1 : Almost fully setup 1057 * - PID already updated by caller if necessary 1058 * - TSIZE for now is base ind page size always 1059 * MAS 2 : Use defaults 1060 * MAS 3+7 : Needs to be done 1061 */ 1062 ori r10,r15,(BOOK3E_PAGESZ_4K << MAS3_SPSIZE_SHIFT) 1063 1064 srdi r16,r10,32 1065 mtspr SPRN_MAS3,r10 1066 mtspr SPRN_MAS7,r16 1067 1068 tlbwe 1069 1070 htw_tlb_miss_done: 1071 /* We don't bother with restoring DEAR or ESR since we know we are 1072 * level 0 and just going back to userland. They are only needed 1073 * if you are going to take an access fault 1074 */ 1075 TLB_MISS_EPILOG_SUCCESS 1076 rfi 1077 1078 htw_tlb_miss_fault: 1079 /* We need to check if it was an instruction miss. We know this 1080 * though because r14 would contain -1 1081 */ 1082 cmpdi cr0,r14,-1 1083 beq 1f 1084 mtspr SPRN_DEAR,r16 1085 mtspr SPRN_ESR,r14 1086 TLB_MISS_EPILOG_ERROR 1087 b exc_data_storage_book3e 1088 1: TLB_MISS_EPILOG_ERROR 1089 b exc_instruction_storage_book3e 1090 1091 /* 1092 * This is the guts of "any" level TLB miss handler for kernel linear 1093 * mapping misses. We are entered with: 1094 * 1095 * 1096 * r16 = faulting address 1097 * r15 = crap (free to use) 1098 * r14 = ESR (data) or -1 (instruction) 1099 * r13 = PACA 1100 * r12 = TLB exception frame in PACA 1101 * r11 = crap (free to use) 1102 * r10 = crap (free to use) 1103 * 1104 * In addition we know that we will not re-enter, so in theory, we could 1105 * use a simpler epilog not restoring SRR0/1 etc.. but we'll do that later. 1106 * 1107 * We also need to be careful about MAS registers here & TLB reservation, 1108 * as we know we'll have clobbered them if we interrupt the main TLB miss 1109 * handlers in which case we probably want to do a full restart at level 1110 * 0 rather than saving / restoring the MAS. 1111 * 1112 * Note: If we care about performance of that core, we can easily shuffle 1113 * a few things around 1114 */ 1115 tlb_load_linear: 1116 /* For now, we assume the linear mapping is contiguous and stops at 1117 * linear_map_top. We also assume the size is a multiple of 1G, thus 1118 * we only use 1G pages for now. That might have to be changed in a 1119 * final implementation, especially when dealing with hypervisors 1120 */ 1121 __LOAD_PACA_TOC(r11) 1122 LOAD_REG_ADDR_ALTTOC(r11, r11, linear_map_top) 1123 ld r10,0(r11) 1124 tovirt(10,10) 1125 cmpld cr0,r16,r10 1126 bge tlb_load_linear_fault 1127 1128 /* MAS1 need whole new setup. */ 1129 li r15,(BOOK3E_PAGESZ_1GB<<MAS1_TSIZE_SHIFT) 1130 oris r15,r15,MAS1_VALID@h /* MAS1 needs V and TSIZE */ 1131 mtspr SPRN_MAS1,r15 1132 1133 /* Already somebody there ? */ 1134 PPC_TLBSRX_DOT(0,R16) 1135 beq tlb_load_linear_done 1136 1137 /* Now we build the remaining MAS. MAS0 and 2 should be fine 1138 * with their defaults, which leaves us with MAS 3 and 7. The 1139 * mapping is linear, so we just take the address, clear the 1140 * region bits, and or in the permission bits which are currently 1141 * hard wired 1142 */ 1143 clrrdi r10,r16,30 /* 1G page index */ 1144 clrldi r10,r10,4 /* clear region bits */ 1145 ori r10,r10,MAS3_SR|MAS3_SW|MAS3_SX 1146 1147 srdi r16,r10,32 1148 mtspr SPRN_MAS3,r10 1149 mtspr SPRN_MAS7,r16 1150 1151 tlbwe 1152 1153 tlb_load_linear_done: 1154 /* We use the "error" epilog for success as we do want to 1155 * restore to the initial faulting context, whatever it was. 1156 * We do that because we can't resume a fault within a TLB 1157 * miss handler, due to MAS and TLB reservation being clobbered. 1158 */ 1159 TLB_MISS_EPILOG_ERROR 1160 rfi 1161 1162 tlb_load_linear_fault: 1163 /* We keep the DEAR and ESR around, this shouldn't have happened */ 1164 cmpdi cr0,r14,-1 1165 beq 1f 1166 TLB_MISS_EPILOG_ERROR_SPECIAL 1167 b exc_data_storage_book3e 1168 1: TLB_MISS_EPILOG_ERROR_SPECIAL 1169 b exc_instruction_storage_book3e 1170