1/* SPDX-License-Identifier: GPL-2.0-or-later */ 2/* 3 * Boot code and exception vectors for Book3E processors 4 * 5 * Copyright (C) 2007 Ben. Herrenschmidt (benh@kernel.crashing.org), IBM Corp. 6 */ 7 8#include <linux/threads.h> 9#include <asm/reg.h> 10#include <asm/page.h> 11#include <asm/ppc_asm.h> 12#include <asm/asm-offsets.h> 13#include <asm/cputable.h> 14#include <asm/setup.h> 15#include <asm/thread_info.h> 16#include <asm/reg_a2.h> 17#include <asm/exception-64e.h> 18#include <asm/bug.h> 19#include <asm/irqflags.h> 20#include <asm/ptrace.h> 21#include <asm/ppc-opcode.h> 22#include <asm/mmu.h> 23#include <asm/hw_irq.h> 24#include <asm/kvm_asm.h> 25#include <asm/kvm_booke_hv_asm.h> 26#include <asm/feature-fixups.h> 27#include <asm/context_tracking.h> 28 29/* XXX This will ultimately add space for a special exception save 30 * structure used to save things like SRR0/SRR1, SPRGs, MAS, etc... 31 * when taking special interrupts. For now we don't support that, 32 * special interrupts from within a non-standard level will probably 33 * blow you up 34 */ 35#define SPECIAL_EXC_SRR0 0 36#define SPECIAL_EXC_SRR1 1 37#define SPECIAL_EXC_SPRG_GEN 2 38#define SPECIAL_EXC_SPRG_TLB 3 39#define SPECIAL_EXC_MAS0 4 40#define SPECIAL_EXC_MAS1 5 41#define SPECIAL_EXC_MAS2 6 42#define SPECIAL_EXC_MAS3 7 43#define SPECIAL_EXC_MAS6 8 44#define SPECIAL_EXC_MAS7 9 45#define SPECIAL_EXC_MAS5 10 /* E.HV only */ 46#define SPECIAL_EXC_MAS8 11 /* E.HV only */ 47#define SPECIAL_EXC_IRQHAPPENED 12 48#define SPECIAL_EXC_DEAR 13 49#define SPECIAL_EXC_ESR 14 50#define SPECIAL_EXC_SOFTE 15 51#define SPECIAL_EXC_CSRR0 16 52#define SPECIAL_EXC_CSRR1 17 53/* must be even to keep 16-byte stack alignment */ 54#define SPECIAL_EXC_END 18 55 56#define SPECIAL_EXC_FRAME_SIZE (INT_FRAME_SIZE + SPECIAL_EXC_END * 8) 57#define SPECIAL_EXC_FRAME_OFFS (INT_FRAME_SIZE - 288) 58 59#define SPECIAL_EXC_STORE(reg, name) \ 60 std reg, (SPECIAL_EXC_##name * 8 + SPECIAL_EXC_FRAME_OFFS)(r1) 61 62#define SPECIAL_EXC_LOAD(reg, name) \ 63 ld reg, (SPECIAL_EXC_##name * 8 + SPECIAL_EXC_FRAME_OFFS)(r1) 64 65special_reg_save: 66 /* 67 * We only need (or have stack space) to save this stuff if 68 * we interrupted the kernel. 69 */ 70 ld r3,_MSR(r1) 71 andi. r3,r3,MSR_PR 72 bnelr 73 74 /* 75 * Advance to the next TLB exception frame for handler 76 * types that don't do it automatically. 77 */ 78 LOAD_REG_ADDR(r11,extlb_level_exc) 79 lwz r12,0(r11) 80 mfspr r10,SPRN_SPRG_TLB_EXFRAME 81 add r10,r10,r12 82 mtspr SPRN_SPRG_TLB_EXFRAME,r10 83 84 /* 85 * Save registers needed to allow nesting of certain exceptions 86 * (such as TLB misses) inside special exception levels 87 */ 88 mfspr r10,SPRN_SRR0 89 SPECIAL_EXC_STORE(r10,SRR0) 90 mfspr r10,SPRN_SRR1 91 SPECIAL_EXC_STORE(r10,SRR1) 92 mfspr r10,SPRN_SPRG_GEN_SCRATCH 93 SPECIAL_EXC_STORE(r10,SPRG_GEN) 94 mfspr r10,SPRN_SPRG_TLB_SCRATCH 95 SPECIAL_EXC_STORE(r10,SPRG_TLB) 96 mfspr r10,SPRN_MAS0 97 SPECIAL_EXC_STORE(r10,MAS0) 98 mfspr r10,SPRN_MAS1 99 SPECIAL_EXC_STORE(r10,MAS1) 100 mfspr r10,SPRN_MAS2 101 SPECIAL_EXC_STORE(r10,MAS2) 102 mfspr r10,SPRN_MAS3 103 SPECIAL_EXC_STORE(r10,MAS3) 104 mfspr r10,SPRN_MAS6 105 SPECIAL_EXC_STORE(r10,MAS6) 106 mfspr r10,SPRN_MAS7 107 SPECIAL_EXC_STORE(r10,MAS7) 108BEGIN_FTR_SECTION 109 mfspr r10,SPRN_MAS5 110 SPECIAL_EXC_STORE(r10,MAS5) 111 mfspr r10,SPRN_MAS8 112 SPECIAL_EXC_STORE(r10,MAS8) 113 114 /* MAS5/8 could have inappropriate values if we interrupted KVM code */ 115 li r10,0 116 mtspr SPRN_MAS5,r10 117 mtspr SPRN_MAS8,r10 118END_FTR_SECTION_IFSET(CPU_FTR_EMB_HV) 119 mfspr r10,SPRN_DEAR 120 SPECIAL_EXC_STORE(r10,DEAR) 121 mfspr r10,SPRN_ESR 122 SPECIAL_EXC_STORE(r10,ESR) 123 124 ld r10,_NIP(r1) 125 SPECIAL_EXC_STORE(r10,CSRR0) 126 ld r10,_MSR(r1) 127 SPECIAL_EXC_STORE(r10,CSRR1) 128 129 blr 130 131ret_from_level_except: 132 ld r3,_MSR(r1) 133 andi. r3,r3,MSR_PR 134 beq 1f 135 REST_NVGPRS(r1) 136 b interrupt_return 1371: 138 139 LOAD_REG_ADDR(r11,extlb_level_exc) 140 lwz r12,0(r11) 141 mfspr r10,SPRN_SPRG_TLB_EXFRAME 142 sub r10,r10,r12 143 mtspr SPRN_SPRG_TLB_EXFRAME,r10 144 145 /* 146 * It's possible that the special level exception interrupted a 147 * TLB miss handler, and inserted the same entry that the 148 * interrupted handler was about to insert. On CPUs without TLB 149 * write conditional, this can result in a duplicate TLB entry. 150 * Wipe all non-bolted entries to be safe. 151 * 152 * Note that this doesn't protect against any TLB misses 153 * we may take accessing the stack from here to the end of 154 * the special level exception. It's not clear how we can 155 * reasonably protect against that, but only CPUs with 156 * neither TLB write conditional nor bolted kernel memory 157 * are affected. Do any such CPUs even exist? 158 */ 159 PPC_TLBILX_ALL(0,R0) 160 161 REST_NVGPRS(r1) 162 163 SPECIAL_EXC_LOAD(r10,SRR0) 164 mtspr SPRN_SRR0,r10 165 SPECIAL_EXC_LOAD(r10,SRR1) 166 mtspr SPRN_SRR1,r10 167 SPECIAL_EXC_LOAD(r10,SPRG_GEN) 168 mtspr SPRN_SPRG_GEN_SCRATCH,r10 169 SPECIAL_EXC_LOAD(r10,SPRG_TLB) 170 mtspr SPRN_SPRG_TLB_SCRATCH,r10 171 SPECIAL_EXC_LOAD(r10,MAS0) 172 mtspr SPRN_MAS0,r10 173 SPECIAL_EXC_LOAD(r10,MAS1) 174 mtspr SPRN_MAS1,r10 175 SPECIAL_EXC_LOAD(r10,MAS2) 176 mtspr SPRN_MAS2,r10 177 SPECIAL_EXC_LOAD(r10,MAS3) 178 mtspr SPRN_MAS3,r10 179 SPECIAL_EXC_LOAD(r10,MAS6) 180 mtspr SPRN_MAS6,r10 181 SPECIAL_EXC_LOAD(r10,MAS7) 182 mtspr SPRN_MAS7,r10 183BEGIN_FTR_SECTION 184 SPECIAL_EXC_LOAD(r10,MAS5) 185 mtspr SPRN_MAS5,r10 186 SPECIAL_EXC_LOAD(r10,MAS8) 187 mtspr SPRN_MAS8,r10 188END_FTR_SECTION_IFSET(CPU_FTR_EMB_HV) 189 190 SPECIAL_EXC_LOAD(r10,DEAR) 191 mtspr SPRN_DEAR,r10 192 SPECIAL_EXC_LOAD(r10,ESR) 193 mtspr SPRN_ESR,r10 194 195 stdcx. r0,0,r1 /* to clear the reservation */ 196 197 REST_4GPRS(2, r1) 198 REST_4GPRS(6, r1) 199 200 ld r10,_CTR(r1) 201 ld r11,_XER(r1) 202 mtctr r10 203 mtxer r11 204 205 blr 206 207.macro ret_from_level srr0 srr1 paca_ex scratch 208 bl ret_from_level_except 209 210 ld r10,_LINK(r1) 211 ld r11,_CCR(r1) 212 ld r0,GPR13(r1) 213 mtlr r10 214 mtcr r11 215 216 ld r10,GPR10(r1) 217 ld r11,GPR11(r1) 218 ld r12,GPR12(r1) 219 mtspr \scratch,r0 220 221 std r10,\paca_ex+EX_R10(r13); 222 std r11,\paca_ex+EX_R11(r13); 223 ld r10,_NIP(r1) 224 ld r11,_MSR(r1) 225 ld r0,GPR0(r1) 226 ld r1,GPR1(r1) 227 mtspr \srr0,r10 228 mtspr \srr1,r11 229 ld r10,\paca_ex+EX_R10(r13) 230 ld r11,\paca_ex+EX_R11(r13) 231 mfspr r13,\scratch 232.endm 233 234ret_from_crit_except: 235 ret_from_level SPRN_CSRR0 SPRN_CSRR1 PACA_EXCRIT SPRN_SPRG_CRIT_SCRATCH 236 rfci 237 238ret_from_mc_except: 239 ret_from_level SPRN_MCSRR0 SPRN_MCSRR1 PACA_EXMC SPRN_SPRG_MC_SCRATCH 240 rfmci 241 242/* Exception prolog code for all exceptions */ 243#define EXCEPTION_PROLOG(n, intnum, type, addition) \ 244 mtspr SPRN_SPRG_##type##_SCRATCH,r13; /* get spare registers */ \ 245 mfspr r13,SPRN_SPRG_PACA; /* get PACA */ \ 246 std r10,PACA_EX##type+EX_R10(r13); \ 247 std r11,PACA_EX##type+EX_R11(r13); \ 248 mfcr r10; /* save CR */ \ 249 mfspr r11,SPRN_##type##_SRR1;/* what are we coming from */ \ 250 DO_KVM intnum,SPRN_##type##_SRR1; /* KVM hook */ \ 251 stw r10,PACA_EX##type+EX_CR(r13); /* save old CR in the PACA */ \ 252 addition; /* additional code for that exc. */ \ 253 std r1,PACA_EX##type+EX_R1(r13); /* save old r1 in the PACA */ \ 254 type##_SET_KSTACK; /* get special stack if necessary */\ 255 andi. r10,r11,MSR_PR; /* save stack pointer */ \ 256 beq 1f; /* branch around if supervisor */ \ 257 ld r1,PACAKSAVE(r13); /* get kernel stack coming from usr */\ 2581: type##_BTB_FLUSH \ 259 cmpdi cr1,r1,0; /* check if SP makes sense */ \ 260 bge- cr1,exc_##n##_bad_stack;/* bad stack (TODO: out of line) */ \ 261 mfspr r10,SPRN_##type##_SRR0; /* read SRR0 before touching stack */ 262 263/* Exception type-specific macros */ 264#define GEN_SET_KSTACK \ 265 subi r1,r1,INT_FRAME_SIZE; /* alloc frame on kernel stack */ 266#define SPRN_GEN_SRR0 SPRN_SRR0 267#define SPRN_GEN_SRR1 SPRN_SRR1 268 269#define GDBELL_SET_KSTACK GEN_SET_KSTACK 270#define SPRN_GDBELL_SRR0 SPRN_GSRR0 271#define SPRN_GDBELL_SRR1 SPRN_GSRR1 272 273#define CRIT_SET_KSTACK \ 274 ld r1,PACA_CRIT_STACK(r13); \ 275 subi r1,r1,SPECIAL_EXC_FRAME_SIZE 276#define SPRN_CRIT_SRR0 SPRN_CSRR0 277#define SPRN_CRIT_SRR1 SPRN_CSRR1 278 279#define DBG_SET_KSTACK \ 280 ld r1,PACA_DBG_STACK(r13); \ 281 subi r1,r1,SPECIAL_EXC_FRAME_SIZE 282#define SPRN_DBG_SRR0 SPRN_DSRR0 283#define SPRN_DBG_SRR1 SPRN_DSRR1 284 285#define MC_SET_KSTACK \ 286 ld r1,PACA_MC_STACK(r13); \ 287 subi r1,r1,SPECIAL_EXC_FRAME_SIZE 288#define SPRN_MC_SRR0 SPRN_MCSRR0 289#define SPRN_MC_SRR1 SPRN_MCSRR1 290 291#ifdef CONFIG_PPC_FSL_BOOK3E 292#define GEN_BTB_FLUSH \ 293 START_BTB_FLUSH_SECTION \ 294 beq 1f; \ 295 BTB_FLUSH(r10) \ 296 1: \ 297 END_BTB_FLUSH_SECTION 298 299#define CRIT_BTB_FLUSH \ 300 START_BTB_FLUSH_SECTION \ 301 BTB_FLUSH(r10) \ 302 END_BTB_FLUSH_SECTION 303 304#define DBG_BTB_FLUSH CRIT_BTB_FLUSH 305#define MC_BTB_FLUSH CRIT_BTB_FLUSH 306#define GDBELL_BTB_FLUSH GEN_BTB_FLUSH 307#else 308#define GEN_BTB_FLUSH 309#define CRIT_BTB_FLUSH 310#define DBG_BTB_FLUSH 311#define MC_BTB_FLUSH 312#define GDBELL_BTB_FLUSH 313#endif 314 315#define NORMAL_EXCEPTION_PROLOG(n, intnum, addition) \ 316 EXCEPTION_PROLOG(n, intnum, GEN, addition##_GEN(n)) 317 318#define CRIT_EXCEPTION_PROLOG(n, intnum, addition) \ 319 EXCEPTION_PROLOG(n, intnum, CRIT, addition##_CRIT(n)) 320 321#define DBG_EXCEPTION_PROLOG(n, intnum, addition) \ 322 EXCEPTION_PROLOG(n, intnum, DBG, addition##_DBG(n)) 323 324#define MC_EXCEPTION_PROLOG(n, intnum, addition) \ 325 EXCEPTION_PROLOG(n, intnum, MC, addition##_MC(n)) 326 327#define GDBELL_EXCEPTION_PROLOG(n, intnum, addition) \ 328 EXCEPTION_PROLOG(n, intnum, GDBELL, addition##_GDBELL(n)) 329 330/* Variants of the "addition" argument for the prolog 331 */ 332#define PROLOG_ADDITION_NONE_GEN(n) 333#define PROLOG_ADDITION_NONE_GDBELL(n) 334#define PROLOG_ADDITION_NONE_CRIT(n) 335#define PROLOG_ADDITION_NONE_DBG(n) 336#define PROLOG_ADDITION_NONE_MC(n) 337 338#define PROLOG_ADDITION_MASKABLE_GEN(n) \ 339 lbz r10,PACAIRQSOFTMASK(r13); /* are irqs soft-masked? */ \ 340 andi. r10,r10,IRQS_DISABLED; /* yes -> go out of line */ \ 341 bne masked_interrupt_book3e_##n 342 343/* 344 * Additional regs must be re-loaded from paca before EXCEPTION_COMMON* is 345 * called, because that does SAVE_NVGPRS which must see the original register 346 * values, otherwise the scratch values might be restored when exiting the 347 * interrupt. 348 */ 349#define PROLOG_ADDITION_2REGS_GEN(n) \ 350 std r14,PACA_EXGEN+EX_R14(r13); \ 351 std r15,PACA_EXGEN+EX_R15(r13) 352 353#define PROLOG_ADDITION_1REG_GEN(n) \ 354 std r14,PACA_EXGEN+EX_R14(r13); 355 356#define PROLOG_ADDITION_2REGS_CRIT(n) \ 357 std r14,PACA_EXCRIT+EX_R14(r13); \ 358 std r15,PACA_EXCRIT+EX_R15(r13) 359 360#define PROLOG_ADDITION_2REGS_DBG(n) \ 361 std r14,PACA_EXDBG+EX_R14(r13); \ 362 std r15,PACA_EXDBG+EX_R15(r13) 363 364#define PROLOG_ADDITION_2REGS_MC(n) \ 365 std r14,PACA_EXMC+EX_R14(r13); \ 366 std r15,PACA_EXMC+EX_R15(r13) 367 368 369/* Core exception code for all exceptions except TLB misses. */ 370#define EXCEPTION_COMMON_LVL(n, scratch, excf) \ 371exc_##n##_common: \ 372 std r0,GPR0(r1); /* save r0 in stackframe */ \ 373 std r2,GPR2(r1); /* save r2 in stackframe */ \ 374 SAVE_4GPRS(3, r1); /* save r3 - r6 in stackframe */ \ 375 SAVE_2GPRS(7, r1); /* save r7, r8 in stackframe */ \ 376 std r9,GPR9(r1); /* save r9 in stackframe */ \ 377 std r10,_NIP(r1); /* save SRR0 to stackframe */ \ 378 std r11,_MSR(r1); /* save SRR1 to stackframe */ \ 379 beq 2f; /* if from kernel mode */ \ 3802: ld r3,excf+EX_R10(r13); /* get back r10 */ \ 381 ld r4,excf+EX_R11(r13); /* get back r11 */ \ 382 mfspr r5,scratch; /* get back r13 */ \ 383 std r12,GPR12(r1); /* save r12 in stackframe */ \ 384 ld r2,PACATOC(r13); /* get kernel TOC into r2 */ \ 385 mflr r6; /* save LR in stackframe */ \ 386 mfctr r7; /* save CTR in stackframe */ \ 387 mfspr r8,SPRN_XER; /* save XER in stackframe */ \ 388 ld r9,excf+EX_R1(r13); /* load orig r1 back from PACA */ \ 389 lwz r10,excf+EX_CR(r13); /* load orig CR back from PACA */ \ 390 lbz r11,PACAIRQSOFTMASK(r13); /* get current IRQ softe */ \ 391 ld r12,exception_marker@toc(r2); \ 392 li r0,0; \ 393 std r3,GPR10(r1); /* save r10 to stackframe */ \ 394 std r4,GPR11(r1); /* save r11 to stackframe */ \ 395 std r5,GPR13(r1); /* save it to stackframe */ \ 396 std r6,_LINK(r1); \ 397 std r7,_CTR(r1); \ 398 std r8,_XER(r1); \ 399 li r3,(n); /* regs.trap vector */ \ 400 std r9,0(r1); /* store stack frame back link */ \ 401 std r10,_CCR(r1); /* store orig CR in stackframe */ \ 402 std r9,GPR1(r1); /* store stack frame back link */ \ 403 std r11,SOFTE(r1); /* and save it to stackframe */ \ 404 std r12,STACK_FRAME_OVERHEAD-16(r1); /* mark the frame */ \ 405 std r3,_TRAP(r1); /* set trap number */ \ 406 std r0,RESULT(r1); /* clear regs->result */ \ 407 SAVE_NVGPRS(r1); 408 409#define EXCEPTION_COMMON(n) \ 410 EXCEPTION_COMMON_LVL(n, SPRN_SPRG_GEN_SCRATCH, PACA_EXGEN) 411#define EXCEPTION_COMMON_CRIT(n) \ 412 EXCEPTION_COMMON_LVL(n, SPRN_SPRG_CRIT_SCRATCH, PACA_EXCRIT) 413#define EXCEPTION_COMMON_MC(n) \ 414 EXCEPTION_COMMON_LVL(n, SPRN_SPRG_MC_SCRATCH, PACA_EXMC) 415#define EXCEPTION_COMMON_DBG(n) \ 416 EXCEPTION_COMMON_LVL(n, SPRN_SPRG_DBG_SCRATCH, PACA_EXDBG) 417 418/* XXX FIXME: Restore r14/r15 when necessary */ 419#define BAD_STACK_TRAMPOLINE(n) \ 420exc_##n##_bad_stack: \ 421 li r1,(n); /* get exception number */ \ 422 sth r1,PACA_TRAP_SAVE(r13); /* store trap */ \ 423 b bad_stack_book3e; /* bad stack error */ 424 425/* WARNING: If you change the layout of this stub, make sure you check 426 * the debug exception handler which handles single stepping 427 * into exceptions from userspace, and the MM code in 428 * arch/powerpc/mm/tlb_nohash.c which patches the branch here 429 * and would need to be updated if that branch is moved 430 */ 431#define EXCEPTION_STUB(loc, label) \ 432 . = interrupt_base_book3e + loc; \ 433 nop; /* To make debug interrupts happy */ \ 434 b exc_##label##_book3e; 435 436#define ACK_NONE(r) 437#define ACK_DEC(r) \ 438 lis r,TSR_DIS@h; \ 439 mtspr SPRN_TSR,r 440#define ACK_FIT(r) \ 441 lis r,TSR_FIS@h; \ 442 mtspr SPRN_TSR,r 443 444/* Used by asynchronous interrupt that may happen in the idle loop. 445 * 446 * This check if the thread was in the idle loop, and if yes, returns 447 * to the caller rather than the PC. This is to avoid a race if 448 * interrupts happen before the wait instruction. 449 */ 450#define CHECK_NAPPING() \ 451 ld r11, PACA_THREAD_INFO(r13); \ 452 ld r10,TI_LOCAL_FLAGS(r11); \ 453 andi. r9,r10,_TLF_NAPPING; \ 454 beq+ 1f; \ 455 ld r8,_LINK(r1); \ 456 rlwinm r7,r10,0,~_TLF_NAPPING; \ 457 std r8,_NIP(r1); \ 458 std r7,TI_LOCAL_FLAGS(r11); \ 4591: 460 461 462#define MASKABLE_EXCEPTION(trapnum, intnum, label, hdlr, ack) \ 463 START_EXCEPTION(label); \ 464 NORMAL_EXCEPTION_PROLOG(trapnum, intnum, PROLOG_ADDITION_MASKABLE)\ 465 EXCEPTION_COMMON(trapnum) \ 466 ack(r8); \ 467 CHECK_NAPPING(); \ 468 addi r3,r1,STACK_FRAME_OVERHEAD; \ 469 bl hdlr; \ 470 b interrupt_return 471 472/* This value is used to mark exception frames on the stack. */ 473 .section ".toc","aw" 474exception_marker: 475 .tc ID_EXC_MARKER[TC],STACK_FRAME_REGS_MARKER 476 477 478/* 479 * And here we have the exception vectors ! 480 */ 481 482 .text 483 .balign 0x1000 484 .globl interrupt_base_book3e 485interrupt_base_book3e: /* fake trap */ 486 EXCEPTION_STUB(0x000, machine_check) 487 EXCEPTION_STUB(0x020, critical_input) /* 0x0100 */ 488 EXCEPTION_STUB(0x040, debug_crit) /* 0x0d00 */ 489 EXCEPTION_STUB(0x060, data_storage) /* 0x0300 */ 490 EXCEPTION_STUB(0x080, instruction_storage) /* 0x0400 */ 491 EXCEPTION_STUB(0x0a0, external_input) /* 0x0500 */ 492 EXCEPTION_STUB(0x0c0, alignment) /* 0x0600 */ 493 EXCEPTION_STUB(0x0e0, program) /* 0x0700 */ 494 EXCEPTION_STUB(0x100, fp_unavailable) /* 0x0800 */ 495 EXCEPTION_STUB(0x120, system_call) /* 0x0c00 */ 496 EXCEPTION_STUB(0x140, ap_unavailable) /* 0x0f20 */ 497 EXCEPTION_STUB(0x160, decrementer) /* 0x0900 */ 498 EXCEPTION_STUB(0x180, fixed_interval) /* 0x0980 */ 499 EXCEPTION_STUB(0x1a0, watchdog) /* 0x09f0 */ 500 EXCEPTION_STUB(0x1c0, data_tlb_miss) 501 EXCEPTION_STUB(0x1e0, instruction_tlb_miss) 502 EXCEPTION_STUB(0x200, altivec_unavailable) 503 EXCEPTION_STUB(0x220, altivec_assist) 504 EXCEPTION_STUB(0x260, perfmon) 505 EXCEPTION_STUB(0x280, doorbell) 506 EXCEPTION_STUB(0x2a0, doorbell_crit) 507 EXCEPTION_STUB(0x2c0, guest_doorbell) 508 EXCEPTION_STUB(0x2e0, guest_doorbell_crit) 509 EXCEPTION_STUB(0x300, hypercall) 510 EXCEPTION_STUB(0x320, ehpriv) 511 EXCEPTION_STUB(0x340, lrat_error) 512 513 .globl __end_interrupts 514__end_interrupts: 515 516/* Critical Input Interrupt */ 517 START_EXCEPTION(critical_input); 518 CRIT_EXCEPTION_PROLOG(0x100, BOOKE_INTERRUPT_CRITICAL, 519 PROLOG_ADDITION_NONE) 520 EXCEPTION_COMMON_CRIT(0x100) 521 bl special_reg_save 522 CHECK_NAPPING(); 523 addi r3,r1,STACK_FRAME_OVERHEAD 524 bl unknown_nmi_exception 525 b ret_from_crit_except 526 527/* Machine Check Interrupt */ 528 START_EXCEPTION(machine_check); 529 MC_EXCEPTION_PROLOG(0x000, BOOKE_INTERRUPT_MACHINE_CHECK, 530 PROLOG_ADDITION_NONE) 531 EXCEPTION_COMMON_MC(0x000) 532 bl special_reg_save 533 CHECK_NAPPING(); 534 addi r3,r1,STACK_FRAME_OVERHEAD 535 bl machine_check_exception 536 b ret_from_mc_except 537 538/* Data Storage Interrupt */ 539 START_EXCEPTION(data_storage) 540 NORMAL_EXCEPTION_PROLOG(0x300, BOOKE_INTERRUPT_DATA_STORAGE, 541 PROLOG_ADDITION_2REGS) 542 mfspr r14,SPRN_DEAR 543 mfspr r15,SPRN_ESR 544 std r14,_DAR(r1) 545 std r15,_DSISR(r1) 546 ld r14,PACA_EXGEN+EX_R14(r13) 547 ld r15,PACA_EXGEN+EX_R15(r13) 548 EXCEPTION_COMMON(0x300) 549 b storage_fault_common 550 551/* Instruction Storage Interrupt */ 552 START_EXCEPTION(instruction_storage); 553 NORMAL_EXCEPTION_PROLOG(0x400, BOOKE_INTERRUPT_INST_STORAGE, 554 PROLOG_ADDITION_2REGS) 555 li r15,0 556 mr r14,r10 557 std r14,_DAR(r1) 558 std r15,_DSISR(r1) 559 ld r14,PACA_EXGEN+EX_R14(r13) 560 ld r15,PACA_EXGEN+EX_R15(r13) 561 EXCEPTION_COMMON(0x400) 562 b storage_fault_common 563 564/* External Input Interrupt */ 565 MASKABLE_EXCEPTION(0x500, BOOKE_INTERRUPT_EXTERNAL, 566 external_input, do_IRQ, ACK_NONE) 567 568/* Alignment */ 569 START_EXCEPTION(alignment); 570 NORMAL_EXCEPTION_PROLOG(0x600, BOOKE_INTERRUPT_ALIGNMENT, 571 PROLOG_ADDITION_2REGS) 572 mfspr r14,SPRN_DEAR 573 mfspr r15,SPRN_ESR 574 std r14,_DAR(r1) 575 std r15,_DSISR(r1) 576 ld r14,PACA_EXGEN+EX_R14(r13) 577 ld r15,PACA_EXGEN+EX_R15(r13) 578 EXCEPTION_COMMON(0x600) 579 b alignment_more /* no room, go out of line */ 580 581/* Program Interrupt */ 582 START_EXCEPTION(program); 583 NORMAL_EXCEPTION_PROLOG(0x700, BOOKE_INTERRUPT_PROGRAM, 584 PROLOG_ADDITION_1REG) 585 mfspr r14,SPRN_ESR 586 std r14,_DSISR(r1) 587 ld r14,PACA_EXGEN+EX_R14(r13) 588 EXCEPTION_COMMON(0x700) 589 addi r3,r1,STACK_FRAME_OVERHEAD 590 bl program_check_exception 591 REST_NVGPRS(r1) 592 b interrupt_return 593 594/* Floating Point Unavailable Interrupt */ 595 START_EXCEPTION(fp_unavailable); 596 NORMAL_EXCEPTION_PROLOG(0x800, BOOKE_INTERRUPT_FP_UNAVAIL, 597 PROLOG_ADDITION_NONE) 598 /* we can probably do a shorter exception entry for that one... */ 599 EXCEPTION_COMMON(0x800) 600 ld r12,_MSR(r1) 601 andi. r0,r12,MSR_PR; 602 beq- 1f 603 bl load_up_fpu 604 b fast_interrupt_return 6051: addi r3,r1,STACK_FRAME_OVERHEAD 606 bl kernel_fp_unavailable_exception 607 b interrupt_return 608 609/* Altivec Unavailable Interrupt */ 610 START_EXCEPTION(altivec_unavailable); 611 NORMAL_EXCEPTION_PROLOG(0x200, BOOKE_INTERRUPT_ALTIVEC_UNAVAIL, 612 PROLOG_ADDITION_NONE) 613 /* we can probably do a shorter exception entry for that one... */ 614 EXCEPTION_COMMON(0x200) 615#ifdef CONFIG_ALTIVEC 616BEGIN_FTR_SECTION 617 ld r12,_MSR(r1) 618 andi. r0,r12,MSR_PR; 619 beq- 1f 620 bl load_up_altivec 621 b fast_interrupt_return 6221: 623END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 624#endif 625 addi r3,r1,STACK_FRAME_OVERHEAD 626 bl altivec_unavailable_exception 627 b interrupt_return 628 629/* AltiVec Assist */ 630 START_EXCEPTION(altivec_assist); 631 NORMAL_EXCEPTION_PROLOG(0x220, 632 BOOKE_INTERRUPT_ALTIVEC_ASSIST, 633 PROLOG_ADDITION_NONE) 634 EXCEPTION_COMMON(0x220) 635 addi r3,r1,STACK_FRAME_OVERHEAD 636#ifdef CONFIG_ALTIVEC 637BEGIN_FTR_SECTION 638 bl altivec_assist_exception 639END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 640 REST_NVGPRS(r1) 641#else 642 bl unknown_exception 643#endif 644 b interrupt_return 645 646 647/* Decrementer Interrupt */ 648 MASKABLE_EXCEPTION(0x900, BOOKE_INTERRUPT_DECREMENTER, 649 decrementer, timer_interrupt, ACK_DEC) 650 651/* Fixed Interval Timer Interrupt */ 652 MASKABLE_EXCEPTION(0x980, BOOKE_INTERRUPT_FIT, 653 fixed_interval, unknown_exception, ACK_FIT) 654 655/* Watchdog Timer Interrupt */ 656 START_EXCEPTION(watchdog); 657 CRIT_EXCEPTION_PROLOG(0x9f0, BOOKE_INTERRUPT_WATCHDOG, 658 PROLOG_ADDITION_NONE) 659 EXCEPTION_COMMON_CRIT(0x9f0) 660 bl special_reg_save 661 CHECK_NAPPING(); 662 addi r3,r1,STACK_FRAME_OVERHEAD 663#ifdef CONFIG_BOOKE_WDT 664 bl WatchdogException 665#else 666 bl unknown_nmi_exception 667#endif 668 b ret_from_crit_except 669 670/* System Call Interrupt */ 671 START_EXCEPTION(system_call) 672 mr r9,r13 /* keep a copy of userland r13 */ 673 mfspr r11,SPRN_SRR0 /* get return address */ 674 mfspr r12,SPRN_SRR1 /* get previous MSR */ 675 mfspr r13,SPRN_SPRG_PACA /* get our PACA */ 676 b system_call_common 677 678/* Auxiliary Processor Unavailable Interrupt */ 679 START_EXCEPTION(ap_unavailable); 680 NORMAL_EXCEPTION_PROLOG(0xf20, BOOKE_INTERRUPT_AP_UNAVAIL, 681 PROLOG_ADDITION_NONE) 682 EXCEPTION_COMMON(0xf20) 683 addi r3,r1,STACK_FRAME_OVERHEAD 684 bl unknown_exception 685 b interrupt_return 686 687/* Debug exception as a critical interrupt*/ 688 START_EXCEPTION(debug_crit); 689 CRIT_EXCEPTION_PROLOG(0xd00, BOOKE_INTERRUPT_DEBUG, 690 PROLOG_ADDITION_2REGS) 691 692 /* 693 * If there is a single step or branch-taken exception in an 694 * exception entry sequence, it was probably meant to apply to 695 * the code where the exception occurred (since exception entry 696 * doesn't turn off DE automatically). We simulate the effect 697 * of turning off DE on entry to an exception handler by turning 698 * off DE in the CSRR1 value and clearing the debug status. 699 */ 700 701 mfspr r14,SPRN_DBSR /* check single-step/branch taken */ 702 andis. r15,r14,(DBSR_IC|DBSR_BT)@h 703 beq+ 1f 704 705#ifdef CONFIG_RELOCATABLE 706 ld r15,PACATOC(r13) 707 ld r14,interrupt_base_book3e@got(r15) 708 ld r15,__end_interrupts@got(r15) 709 cmpld cr0,r10,r14 710 cmpld cr1,r10,r15 711#else 712 LOAD_REG_IMMEDIATE_SYM(r14, r15, interrupt_base_book3e) 713 cmpld cr0, r10, r14 714 LOAD_REG_IMMEDIATE_SYM(r14, r15, __end_interrupts) 715 cmpld cr1, r10, r14 716#endif 717 blt+ cr0,1f 718 bge+ cr1,1f 719 720 /* here it looks like we got an inappropriate debug exception. */ 721 lis r14,(DBSR_IC|DBSR_BT)@h /* clear the event */ 722 rlwinm r11,r11,0,~MSR_DE /* clear DE in the CSRR1 value */ 723 mtspr SPRN_DBSR,r14 724 mtspr SPRN_CSRR1,r11 725 lwz r10,PACA_EXCRIT+EX_CR(r13) /* restore registers */ 726 ld r1,PACA_EXCRIT+EX_R1(r13) 727 ld r14,PACA_EXCRIT+EX_R14(r13) 728 ld r15,PACA_EXCRIT+EX_R15(r13) 729 mtcr r10 730 ld r10,PACA_EXCRIT+EX_R10(r13) /* restore registers */ 731 ld r11,PACA_EXCRIT+EX_R11(r13) 732 mfspr r13,SPRN_SPRG_CRIT_SCRATCH 733 rfci 734 735 /* Normal debug exception */ 736 /* XXX We only handle coming from userspace for now since we can't 737 * quite save properly an interrupted kernel state yet 738 */ 7391: andi. r14,r11,MSR_PR; /* check for userspace again */ 740 beq kernel_dbg_exc; /* if from kernel mode */ 741 742 /* Now we mash up things to make it look like we are coming on a 743 * normal exception 744 */ 745 mfspr r14,SPRN_DBSR 746 std r14,_DSISR(r1) 747 ld r14,PACA_EXCRIT+EX_R14(r13) 748 ld r15,PACA_EXCRIT+EX_R15(r13) 749 EXCEPTION_COMMON_CRIT(0xd00) 750 addi r3,r1,STACK_FRAME_OVERHEAD 751 bl DebugException 752 REST_NVGPRS(r1) 753 b interrupt_return 754 755kernel_dbg_exc: 756 b . /* NYI */ 757 758/* Debug exception as a debug interrupt*/ 759 START_EXCEPTION(debug_debug); 760 DBG_EXCEPTION_PROLOG(0xd00, BOOKE_INTERRUPT_DEBUG, 761 PROLOG_ADDITION_2REGS) 762 763 /* 764 * If there is a single step or branch-taken exception in an 765 * exception entry sequence, it was probably meant to apply to 766 * the code where the exception occurred (since exception entry 767 * doesn't turn off DE automatically). We simulate the effect 768 * of turning off DE on entry to an exception handler by turning 769 * off DE in the DSRR1 value and clearing the debug status. 770 */ 771 772 mfspr r14,SPRN_DBSR /* check single-step/branch taken */ 773 andis. r15,r14,(DBSR_IC|DBSR_BT)@h 774 beq+ 1f 775 776#ifdef CONFIG_RELOCATABLE 777 ld r15,PACATOC(r13) 778 ld r14,interrupt_base_book3e@got(r15) 779 ld r15,__end_interrupts@got(r15) 780 cmpld cr0,r10,r14 781 cmpld cr1,r10,r15 782#else 783 LOAD_REG_IMMEDIATE_SYM(r14, r15, interrupt_base_book3e) 784 cmpld cr0, r10, r14 785 LOAD_REG_IMMEDIATE_SYM(r14, r15,__end_interrupts) 786 cmpld cr1, r10, r14 787#endif 788 blt+ cr0,1f 789 bge+ cr1,1f 790 791 /* here it looks like we got an inappropriate debug exception. */ 792 lis r14,(DBSR_IC|DBSR_BT)@h /* clear the event */ 793 rlwinm r11,r11,0,~MSR_DE /* clear DE in the DSRR1 value */ 794 mtspr SPRN_DBSR,r14 795 mtspr SPRN_DSRR1,r11 796 lwz r10,PACA_EXDBG+EX_CR(r13) /* restore registers */ 797 ld r1,PACA_EXDBG+EX_R1(r13) 798 ld r14,PACA_EXDBG+EX_R14(r13) 799 ld r15,PACA_EXDBG+EX_R15(r13) 800 mtcr r10 801 ld r10,PACA_EXDBG+EX_R10(r13) /* restore registers */ 802 ld r11,PACA_EXDBG+EX_R11(r13) 803 mfspr r13,SPRN_SPRG_DBG_SCRATCH 804 rfdi 805 806 /* Normal debug exception */ 807 /* XXX We only handle coming from userspace for now since we can't 808 * quite save properly an interrupted kernel state yet 809 */ 8101: andi. r14,r11,MSR_PR; /* check for userspace again */ 811 beq kernel_dbg_exc; /* if from kernel mode */ 812 813 /* Now we mash up things to make it look like we are coming on a 814 * normal exception 815 */ 816 mfspr r14,SPRN_DBSR 817 std r14,_DSISR(r1) 818 ld r14,PACA_EXDBG+EX_R14(r13) 819 ld r15,PACA_EXDBG+EX_R15(r13) 820 EXCEPTION_COMMON_DBG(0xd08) 821 addi r3,r1,STACK_FRAME_OVERHEAD 822 bl DebugException 823 REST_NVGPRS(r1) 824 b interrupt_return 825 826 START_EXCEPTION(perfmon); 827 NORMAL_EXCEPTION_PROLOG(0x260, BOOKE_INTERRUPT_PERFORMANCE_MONITOR, 828 PROLOG_ADDITION_NONE) 829 EXCEPTION_COMMON(0x260) 830 CHECK_NAPPING() 831 addi r3,r1,STACK_FRAME_OVERHEAD 832 bl performance_monitor_exception 833 b interrupt_return 834 835/* Doorbell interrupt */ 836 MASKABLE_EXCEPTION(0x280, BOOKE_INTERRUPT_DOORBELL, 837 doorbell, doorbell_exception, ACK_NONE) 838 839/* Doorbell critical Interrupt */ 840 START_EXCEPTION(doorbell_crit); 841 CRIT_EXCEPTION_PROLOG(0x2a0, BOOKE_INTERRUPT_DOORBELL_CRITICAL, 842 PROLOG_ADDITION_NONE) 843 EXCEPTION_COMMON_CRIT(0x2a0) 844 bl special_reg_save 845 CHECK_NAPPING(); 846 addi r3,r1,STACK_FRAME_OVERHEAD 847 bl unknown_nmi_exception 848 b ret_from_crit_except 849 850/* 851 * Guest doorbell interrupt 852 * This general exception use GSRRx save/restore registers 853 */ 854 START_EXCEPTION(guest_doorbell); 855 GDBELL_EXCEPTION_PROLOG(0x2c0, BOOKE_INTERRUPT_GUEST_DBELL, 856 PROLOG_ADDITION_NONE) 857 EXCEPTION_COMMON(0x2c0) 858 addi r3,r1,STACK_FRAME_OVERHEAD 859 bl unknown_exception 860 b interrupt_return 861 862/* Guest Doorbell critical Interrupt */ 863 START_EXCEPTION(guest_doorbell_crit); 864 CRIT_EXCEPTION_PROLOG(0x2e0, BOOKE_INTERRUPT_GUEST_DBELL_CRIT, 865 PROLOG_ADDITION_NONE) 866 EXCEPTION_COMMON_CRIT(0x2e0) 867 bl special_reg_save 868 CHECK_NAPPING(); 869 addi r3,r1,STACK_FRAME_OVERHEAD 870 bl unknown_nmi_exception 871 b ret_from_crit_except 872 873/* Hypervisor call */ 874 START_EXCEPTION(hypercall); 875 NORMAL_EXCEPTION_PROLOG(0x310, BOOKE_INTERRUPT_HV_SYSCALL, 876 PROLOG_ADDITION_NONE) 877 EXCEPTION_COMMON(0x310) 878 addi r3,r1,STACK_FRAME_OVERHEAD 879 bl unknown_exception 880 b interrupt_return 881 882/* Embedded Hypervisor priviledged */ 883 START_EXCEPTION(ehpriv); 884 NORMAL_EXCEPTION_PROLOG(0x320, BOOKE_INTERRUPT_HV_PRIV, 885 PROLOG_ADDITION_NONE) 886 EXCEPTION_COMMON(0x320) 887 addi r3,r1,STACK_FRAME_OVERHEAD 888 bl unknown_exception 889 b interrupt_return 890 891/* LRAT Error interrupt */ 892 START_EXCEPTION(lrat_error); 893 NORMAL_EXCEPTION_PROLOG(0x340, BOOKE_INTERRUPT_LRAT_ERROR, 894 PROLOG_ADDITION_NONE) 895 EXCEPTION_COMMON(0x340) 896 addi r3,r1,STACK_FRAME_OVERHEAD 897 bl unknown_exception 898 b interrupt_return 899 900/* 901 * An interrupt came in while soft-disabled; We mark paca->irq_happened 902 * accordingly and if the interrupt is level sensitive, we hard disable 903 * hard disable (full_mask) corresponds to PACA_IRQ_MUST_HARD_MASK, so 904 * keep these in synch. 905 */ 906 907.macro masked_interrupt_book3e paca_irq full_mask 908 lbz r10,PACAIRQHAPPENED(r13) 909 .if \full_mask == 1 910 ori r10,r10,\paca_irq | PACA_IRQ_HARD_DIS 911 .else 912 ori r10,r10,\paca_irq 913 .endif 914 stb r10,PACAIRQHAPPENED(r13) 915 916 .if \full_mask == 1 917 rldicl r10,r11,48,1 /* clear MSR_EE */ 918 rotldi r11,r10,16 919 mtspr SPRN_SRR1,r11 920 .endif 921 922 lwz r11,PACA_EXGEN+EX_CR(r13) 923 mtcr r11 924 ld r10,PACA_EXGEN+EX_R10(r13) 925 ld r11,PACA_EXGEN+EX_R11(r13) 926 mfspr r13,SPRN_SPRG_GEN_SCRATCH 927 rfi 928 b . 929.endm 930 931masked_interrupt_book3e_0x500: 932 masked_interrupt_book3e PACA_IRQ_EE 1 933 934masked_interrupt_book3e_0x900: 935 ACK_DEC(r10); 936 masked_interrupt_book3e PACA_IRQ_DEC 0 937 938masked_interrupt_book3e_0x980: 939 ACK_FIT(r10); 940 masked_interrupt_book3e PACA_IRQ_DEC 0 941 942masked_interrupt_book3e_0x280: 943masked_interrupt_book3e_0x2c0: 944 masked_interrupt_book3e PACA_IRQ_DBELL 0 945 946/* 947 * This is called from 0x300 and 0x400 handlers after the prologs with 948 * r14 and r15 containing the fault address and error code, with the 949 * original values stashed away in the PACA 950 */ 951storage_fault_common: 952 addi r3,r1,STACK_FRAME_OVERHEAD 953 bl do_page_fault 954 b interrupt_return 955 956/* 957 * Alignment exception doesn't fit entirely in the 0x100 bytes so it 958 * continues here. 959 */ 960alignment_more: 961 addi r3,r1,STACK_FRAME_OVERHEAD 962 bl alignment_exception 963 REST_NVGPRS(r1) 964 b interrupt_return 965 966/* 967 * Trampolines used when spotting a bad kernel stack pointer in 968 * the exception entry code. 969 * 970 * TODO: move some bits like SRR0 read to trampoline, pass PACA 971 * index around, etc... to handle crit & mcheck 972 */ 973BAD_STACK_TRAMPOLINE(0x000) 974BAD_STACK_TRAMPOLINE(0x100) 975BAD_STACK_TRAMPOLINE(0x200) 976BAD_STACK_TRAMPOLINE(0x220) 977BAD_STACK_TRAMPOLINE(0x260) 978BAD_STACK_TRAMPOLINE(0x280) 979BAD_STACK_TRAMPOLINE(0x2a0) 980BAD_STACK_TRAMPOLINE(0x2c0) 981BAD_STACK_TRAMPOLINE(0x2e0) 982BAD_STACK_TRAMPOLINE(0x300) 983BAD_STACK_TRAMPOLINE(0x310) 984BAD_STACK_TRAMPOLINE(0x320) 985BAD_STACK_TRAMPOLINE(0x340) 986BAD_STACK_TRAMPOLINE(0x400) 987BAD_STACK_TRAMPOLINE(0x500) 988BAD_STACK_TRAMPOLINE(0x600) 989BAD_STACK_TRAMPOLINE(0x700) 990BAD_STACK_TRAMPOLINE(0x800) 991BAD_STACK_TRAMPOLINE(0x900) 992BAD_STACK_TRAMPOLINE(0x980) 993BAD_STACK_TRAMPOLINE(0x9f0) 994BAD_STACK_TRAMPOLINE(0xa00) 995BAD_STACK_TRAMPOLINE(0xb00) 996BAD_STACK_TRAMPOLINE(0xc00) 997BAD_STACK_TRAMPOLINE(0xd00) 998BAD_STACK_TRAMPOLINE(0xd08) 999BAD_STACK_TRAMPOLINE(0xe00) 1000BAD_STACK_TRAMPOLINE(0xf00) 1001BAD_STACK_TRAMPOLINE(0xf20) 1002 1003 .globl bad_stack_book3e 1004bad_stack_book3e: 1005 /* XXX: Needs to make SPRN_SPRG_GEN depend on exception type */ 1006 mfspr r10,SPRN_SRR0; /* read SRR0 before touching stack */ 1007 ld r1,PACAEMERGSP(r13) 1008 subi r1,r1,64+INT_FRAME_SIZE 1009 std r10,_NIP(r1) 1010 std r11,_MSR(r1) 1011 ld r10,PACA_EXGEN+EX_R1(r13) /* FIXME for crit & mcheck */ 1012 lwz r11,PACA_EXGEN+EX_CR(r13) /* FIXME for crit & mcheck */ 1013 std r10,GPR1(r1) 1014 std r11,_CCR(r1) 1015 mfspr r10,SPRN_DEAR 1016 mfspr r11,SPRN_ESR 1017 std r10,_DAR(r1) 1018 std r11,_DSISR(r1) 1019 std r0,GPR0(r1); /* save r0 in stackframe */ \ 1020 std r2,GPR2(r1); /* save r2 in stackframe */ \ 1021 SAVE_4GPRS(3, r1); /* save r3 - r6 in stackframe */ \ 1022 SAVE_2GPRS(7, r1); /* save r7, r8 in stackframe */ \ 1023 std r9,GPR9(r1); /* save r9 in stackframe */ \ 1024 ld r3,PACA_EXGEN+EX_R10(r13);/* get back r10 */ \ 1025 ld r4,PACA_EXGEN+EX_R11(r13);/* get back r11 */ \ 1026 mfspr r5,SPRN_SPRG_GEN_SCRATCH;/* get back r13 XXX can be wrong */ \ 1027 std r3,GPR10(r1); /* save r10 to stackframe */ \ 1028 std r4,GPR11(r1); /* save r11 to stackframe */ \ 1029 std r12,GPR12(r1); /* save r12 in stackframe */ \ 1030 std r5,GPR13(r1); /* save it to stackframe */ \ 1031 mflr r10 1032 mfctr r11 1033 mfxer r12 1034 std r10,_LINK(r1) 1035 std r11,_CTR(r1) 1036 std r12,_XER(r1) 1037 SAVE_10GPRS(14,r1) 1038 SAVE_8GPRS(24,r1) 1039 lhz r12,PACA_TRAP_SAVE(r13) 1040 std r12,_TRAP(r1) 1041 addi r11,r1,INT_FRAME_SIZE 1042 std r11,0(r1) 1043 li r12,0 1044 std r12,0(r11) 1045 ld r2,PACATOC(r13) 10461: addi r3,r1,STACK_FRAME_OVERHEAD 1047 bl kernel_bad_stack 1048 b 1b 1049 1050/* 1051 * Setup the initial TLB for a core. This current implementation 1052 * assume that whatever we are running off will not conflict with 1053 * the new mapping at PAGE_OFFSET. 1054 */ 1055_GLOBAL(initial_tlb_book3e) 1056 1057 /* Look for the first TLB with IPROT set */ 1058 mfspr r4,SPRN_TLB0CFG 1059 andi. r3,r4,TLBnCFG_IPROT 1060 lis r3,MAS0_TLBSEL(0)@h 1061 bne found_iprot 1062 1063 mfspr r4,SPRN_TLB1CFG 1064 andi. r3,r4,TLBnCFG_IPROT 1065 lis r3,MAS0_TLBSEL(1)@h 1066 bne found_iprot 1067 1068 mfspr r4,SPRN_TLB2CFG 1069 andi. r3,r4,TLBnCFG_IPROT 1070 lis r3,MAS0_TLBSEL(2)@h 1071 bne found_iprot 1072 1073 lis r3,MAS0_TLBSEL(3)@h 1074 mfspr r4,SPRN_TLB3CFG 1075 /* fall through */ 1076 1077found_iprot: 1078 andi. r5,r4,TLBnCFG_HES 1079 bne have_hes 1080 1081 mflr r8 /* save LR */ 1082/* 1. Find the index of the entry we're executing in 1083 * 1084 * r3 = MAS0_TLBSEL (for the iprot array) 1085 * r4 = SPRN_TLBnCFG 1086 */ 1087 bl invstr /* Find our address */ 1088invstr: mflr r6 /* Make it accessible */ 1089 mfmsr r7 1090 rlwinm r5,r7,27,31,31 /* extract MSR[IS] */ 1091 mfspr r7,SPRN_PID 1092 slwi r7,r7,16 1093 or r7,r7,r5 1094 mtspr SPRN_MAS6,r7 1095 tlbsx 0,r6 /* search MSR[IS], SPID=PID */ 1096 1097 mfspr r3,SPRN_MAS0 1098 rlwinm r5,r3,16,20,31 /* Extract MAS0(Entry) */ 1099 1100 mfspr r7,SPRN_MAS1 /* Insure IPROT set */ 1101 oris r7,r7,MAS1_IPROT@h 1102 mtspr SPRN_MAS1,r7 1103 tlbwe 1104 1105/* 2. Invalidate all entries except the entry we're executing in 1106 * 1107 * r3 = MAS0 w/TLBSEL & ESEL for the entry we are running in 1108 * r4 = SPRN_TLBnCFG 1109 * r5 = ESEL of entry we are running in 1110 */ 1111 andi. r4,r4,TLBnCFG_N_ENTRY /* Extract # entries */ 1112 li r6,0 /* Set Entry counter to 0 */ 11131: mr r7,r3 /* Set MAS0(TLBSEL) */ 1114 rlwimi r7,r6,16,4,15 /* Setup MAS0 = TLBSEL | ESEL(r6) */ 1115 mtspr SPRN_MAS0,r7 1116 tlbre 1117 mfspr r7,SPRN_MAS1 1118 rlwinm r7,r7,0,2,31 /* Clear MAS1 Valid and IPROT */ 1119 cmpw r5,r6 1120 beq skpinv /* Dont update the current execution TLB */ 1121 mtspr SPRN_MAS1,r7 1122 tlbwe 1123 isync 1124skpinv: addi r6,r6,1 /* Increment */ 1125 cmpw r6,r4 /* Are we done? */ 1126 bne 1b /* If not, repeat */ 1127 1128 /* Invalidate all TLBs */ 1129 PPC_TLBILX_ALL(0,R0) 1130 sync 1131 isync 1132 1133/* 3. Setup a temp mapping and jump to it 1134 * 1135 * r3 = MAS0 w/TLBSEL & ESEL for the entry we are running in 1136 * r5 = ESEL of entry we are running in 1137 */ 1138 andi. r7,r5,0x1 /* Find an entry not used and is non-zero */ 1139 addi r7,r7,0x1 1140 mr r4,r3 /* Set MAS0(TLBSEL) = 1 */ 1141 mtspr SPRN_MAS0,r4 1142 tlbre 1143 1144 rlwimi r4,r7,16,4,15 /* Setup MAS0 = TLBSEL | ESEL(r7) */ 1145 mtspr SPRN_MAS0,r4 1146 1147 mfspr r7,SPRN_MAS1 1148 xori r6,r7,MAS1_TS /* Setup TMP mapping in the other Address space */ 1149 mtspr SPRN_MAS1,r6 1150 1151 tlbwe 1152 1153 mfmsr r6 1154 xori r6,r6,MSR_IS 1155 mtspr SPRN_SRR1,r6 1156 bl 1f /* Find our address */ 11571: mflr r6 1158 addi r6,r6,(2f - 1b) 1159 mtspr SPRN_SRR0,r6 1160 rfi 11612: 1162 1163/* 4. Clear out PIDs & Search info 1164 * 1165 * r3 = MAS0 w/TLBSEL & ESEL for the entry we started in 1166 * r4 = MAS0 w/TLBSEL & ESEL for the temp mapping 1167 * r5 = MAS3 1168 */ 1169 li r6,0 1170 mtspr SPRN_MAS6,r6 1171 mtspr SPRN_PID,r6 1172 1173/* 5. Invalidate mapping we started in 1174 * 1175 * r3 = MAS0 w/TLBSEL & ESEL for the entry we started in 1176 * r4 = MAS0 w/TLBSEL & ESEL for the temp mapping 1177 * r5 = MAS3 1178 */ 1179 mtspr SPRN_MAS0,r3 1180 tlbre 1181 mfspr r6,SPRN_MAS1 1182 rlwinm r6,r6,0,2,31 /* clear IPROT and VALID */ 1183 mtspr SPRN_MAS1,r6 1184 tlbwe 1185 sync 1186 isync 1187 1188/* 6. Setup KERNELBASE mapping in TLB[0] 1189 * 1190 * r3 = MAS0 w/TLBSEL & ESEL for the entry we started in 1191 * r4 = MAS0 w/TLBSEL & ESEL for the temp mapping 1192 * r5 = MAS3 1193 */ 1194 rlwinm r3,r3,0,16,3 /* clear ESEL */ 1195 mtspr SPRN_MAS0,r3 1196 lis r6,(MAS1_VALID|MAS1_IPROT)@h 1197 ori r6,r6,(MAS1_TSIZE(BOOK3E_PAGESZ_1GB))@l 1198 mtspr SPRN_MAS1,r6 1199 1200 LOAD_REG_IMMEDIATE(r6, PAGE_OFFSET | MAS2_M_IF_NEEDED) 1201 mtspr SPRN_MAS2,r6 1202 1203 rlwinm r5,r5,0,0,25 1204 ori r5,r5,MAS3_SR | MAS3_SW | MAS3_SX 1205 mtspr SPRN_MAS3,r5 1206 li r5,-1 1207 rlwinm r5,r5,0,0,25 1208 1209 tlbwe 1210 1211/* 7. Jump to KERNELBASE mapping 1212 * 1213 * r4 = MAS0 w/TLBSEL & ESEL for the temp mapping 1214 */ 1215 /* Now we branch the new virtual address mapped by this entry */ 1216 bl 1f /* Find our address */ 12171: mflr r6 1218 addi r6,r6,(2f - 1b) 1219 tovirt(r6,r6) 1220 lis r7,MSR_KERNEL@h 1221 ori r7,r7,MSR_KERNEL@l 1222 mtspr SPRN_SRR0,r6 1223 mtspr SPRN_SRR1,r7 1224 rfi /* start execution out of TLB1[0] entry */ 12252: 1226 1227/* 8. Clear out the temp mapping 1228 * 1229 * r4 = MAS0 w/TLBSEL & ESEL for the entry we are running in 1230 */ 1231 mtspr SPRN_MAS0,r4 1232 tlbre 1233 mfspr r5,SPRN_MAS1 1234 rlwinm r5,r5,0,2,31 /* clear IPROT and VALID */ 1235 mtspr SPRN_MAS1,r5 1236 tlbwe 1237 sync 1238 isync 1239 1240 /* We translate LR and return */ 1241 tovirt(r8,r8) 1242 mtlr r8 1243 blr 1244 1245have_hes: 1246 /* Setup MAS 0,1,2,3 and 7 for tlbwe of a 1G entry that maps the 1247 * kernel linear mapping. We also set MAS8 once for all here though 1248 * that will have to be made dependent on whether we are running under 1249 * a hypervisor I suppose. 1250 */ 1251 1252 /* BEWARE, MAGIC 1253 * This code is called as an ordinary function on the boot CPU. But to 1254 * avoid duplication, this code is also used in SCOM bringup of 1255 * secondary CPUs. We read the code between the initial_tlb_code_start 1256 * and initial_tlb_code_end labels one instruction at a time and RAM it 1257 * into the new core via SCOM. That doesn't process branches, so there 1258 * must be none between those two labels. It also means if this code 1259 * ever takes any parameters, the SCOM code must also be updated to 1260 * provide them. 1261 */ 1262 .globl a2_tlbinit_code_start 1263a2_tlbinit_code_start: 1264 1265 ori r11,r3,MAS0_WQ_ALLWAYS 1266 oris r11,r11,MAS0_ESEL(3)@h /* Use way 3: workaround A2 erratum 376 */ 1267 mtspr SPRN_MAS0,r11 1268 lis r3,(MAS1_VALID | MAS1_IPROT)@h 1269 ori r3,r3,BOOK3E_PAGESZ_1GB << MAS1_TSIZE_SHIFT 1270 mtspr SPRN_MAS1,r3 1271 LOAD_REG_IMMEDIATE(r3, PAGE_OFFSET | MAS2_M) 1272 mtspr SPRN_MAS2,r3 1273 li r3,MAS3_SR | MAS3_SW | MAS3_SX 1274 mtspr SPRN_MAS7_MAS3,r3 1275 li r3,0 1276 mtspr SPRN_MAS8,r3 1277 1278 /* Write the TLB entry */ 1279 tlbwe 1280 1281 .globl a2_tlbinit_after_linear_map 1282a2_tlbinit_after_linear_map: 1283 1284 /* Now we branch the new virtual address mapped by this entry */ 1285 LOAD_REG_IMMEDIATE_SYM(r3, r5, 1f) 1286 mtctr r3 1287 bctr 1288 12891: /* We are now running at PAGE_OFFSET, clean the TLB of everything 1290 * else (including IPROTed things left by firmware) 1291 * r4 = TLBnCFG 1292 * r3 = current address (more or less) 1293 */ 1294 1295 li r5,0 1296 mtspr SPRN_MAS6,r5 1297 tlbsx 0,r3 1298 1299 rlwinm r9,r4,0,TLBnCFG_N_ENTRY 1300 rlwinm r10,r4,8,0xff 1301 addi r10,r10,-1 /* Get inner loop mask */ 1302 1303 li r3,1 1304 1305 mfspr r5,SPRN_MAS1 1306 rlwinm r5,r5,0,(~(MAS1_VALID|MAS1_IPROT)) 1307 1308 mfspr r6,SPRN_MAS2 1309 rldicr r6,r6,0,51 /* Extract EPN */ 1310 1311 mfspr r7,SPRN_MAS0 1312 rlwinm r7,r7,0,0xffff0fff /* Clear HES and WQ */ 1313 1314 rlwinm r8,r7,16,0xfff /* Extract ESEL */ 1315 13162: add r4,r3,r8 1317 and r4,r4,r10 1318 1319 rlwimi r7,r4,16,MAS0_ESEL_MASK 1320 1321 mtspr SPRN_MAS0,r7 1322 mtspr SPRN_MAS1,r5 1323 mtspr SPRN_MAS2,r6 1324 tlbwe 1325 1326 addi r3,r3,1 1327 and. r4,r3,r10 1328 1329 bne 3f 1330 addis r6,r6,(1<<30)@h 13313: 1332 cmpw r3,r9 1333 blt 2b 1334 1335 .globl a2_tlbinit_after_iprot_flush 1336a2_tlbinit_after_iprot_flush: 1337 1338 PPC_TLBILX(0,0,R0) 1339 sync 1340 isync 1341 1342 .globl a2_tlbinit_code_end 1343a2_tlbinit_code_end: 1344 1345 /* We translate LR and return */ 1346 mflr r3 1347 tovirt(r3,r3) 1348 mtlr r3 1349 blr 1350 1351/* 1352 * Main entry (boot CPU, thread 0) 1353 * 1354 * We enter here from head_64.S, possibly after the prom_init trampoline 1355 * with r3 and r4 already saved to r31 and 30 respectively and in 64 bits 1356 * mode. Anything else is as it was left by the bootloader 1357 * 1358 * Initial requirements of this port: 1359 * 1360 * - Kernel loaded at 0 physical 1361 * - A good lump of memory mapped 0:0 by UTLB entry 0 1362 * - MSR:IS & MSR:DS set to 0 1363 * 1364 * Note that some of the above requirements will be relaxed in the future 1365 * as the kernel becomes smarter at dealing with different initial conditions 1366 * but for now you have to be careful 1367 */ 1368_GLOBAL(start_initialization_book3e) 1369 mflr r28 1370 1371 /* First, we need to setup some initial TLBs to map the kernel 1372 * text, data and bss at PAGE_OFFSET. We don't have a real mode 1373 * and always use AS 0, so we just set it up to match our link 1374 * address and never use 0 based addresses. 1375 */ 1376 bl initial_tlb_book3e 1377 1378 /* Init global core bits */ 1379 bl init_core_book3e 1380 1381 /* Init per-thread bits */ 1382 bl init_thread_book3e 1383 1384 /* Return to common init code */ 1385 tovirt(r28,r28) 1386 mtlr r28 1387 blr 1388 1389 1390/* 1391 * Secondary core/processor entry 1392 * 1393 * This is entered for thread 0 of a secondary core, all other threads 1394 * are expected to be stopped. It's similar to start_initialization_book3e 1395 * except that it's generally entered from the holding loop in head_64.S 1396 * after CPUs have been gathered by Open Firmware. 1397 * 1398 * We assume we are in 32 bits mode running with whatever TLB entry was 1399 * set for us by the firmware or POR engine. 1400 */ 1401_GLOBAL(book3e_secondary_core_init_tlb_set) 1402 li r4,1 1403 b generic_secondary_smp_init 1404 1405_GLOBAL(book3e_secondary_core_init) 1406 mflr r28 1407 1408 /* Do we need to setup initial TLB entry ? */ 1409 cmplwi r4,0 1410 bne 2f 1411 1412 /* Setup TLB for this core */ 1413 bl initial_tlb_book3e 1414 1415 /* We can return from the above running at a different 1416 * address, so recalculate r2 (TOC) 1417 */ 1418 bl relative_toc 1419 1420 /* Init global core bits */ 14212: bl init_core_book3e 1422 1423 /* Init per-thread bits */ 14243: bl init_thread_book3e 1425 1426 /* Return to common init code at proper virtual address. 1427 * 1428 * Due to various previous assumptions, we know we entered this 1429 * function at either the final PAGE_OFFSET mapping or using a 1430 * 1:1 mapping at 0, so we don't bother doing a complicated check 1431 * here, we just ensure the return address has the right top bits. 1432 * 1433 * Note that if we ever want to be smarter about where we can be 1434 * started from, we have to be careful that by the time we reach 1435 * the code below we may already be running at a different location 1436 * than the one we were called from since initial_tlb_book3e can 1437 * have moved us already. 1438 */ 1439 cmpdi cr0,r28,0 1440 blt 1f 1441 lis r3,PAGE_OFFSET@highest 1442 sldi r3,r3,32 1443 or r28,r28,r3 14441: mtlr r28 1445 blr 1446 1447_GLOBAL(book3e_secondary_thread_init) 1448 mflr r28 1449 b 3b 1450 1451 .globl init_core_book3e 1452init_core_book3e: 1453 /* Establish the interrupt vector base */ 1454 tovirt(r2,r2) 1455 LOAD_REG_ADDR(r3, interrupt_base_book3e) 1456 mtspr SPRN_IVPR,r3 1457 sync 1458 blr 1459 1460init_thread_book3e: 1461 lis r3,(SPRN_EPCR_ICM | SPRN_EPCR_GICM)@h 1462 mtspr SPRN_EPCR,r3 1463 1464 /* Make sure interrupts are off */ 1465 wrteei 0 1466 1467 /* disable all timers and clear out status */ 1468 li r3,0 1469 mtspr SPRN_TCR,r3 1470 mfspr r3,SPRN_TSR 1471 mtspr SPRN_TSR,r3 1472 1473 blr 1474 1475_GLOBAL(__setup_base_ivors) 1476 SET_IVOR(0, 0x020) /* Critical Input */ 1477 SET_IVOR(1, 0x000) /* Machine Check */ 1478 SET_IVOR(2, 0x060) /* Data Storage */ 1479 SET_IVOR(3, 0x080) /* Instruction Storage */ 1480 SET_IVOR(4, 0x0a0) /* External Input */ 1481 SET_IVOR(5, 0x0c0) /* Alignment */ 1482 SET_IVOR(6, 0x0e0) /* Program */ 1483 SET_IVOR(7, 0x100) /* FP Unavailable */ 1484 SET_IVOR(8, 0x120) /* System Call */ 1485 SET_IVOR(9, 0x140) /* Auxiliary Processor Unavailable */ 1486 SET_IVOR(10, 0x160) /* Decrementer */ 1487 SET_IVOR(11, 0x180) /* Fixed Interval Timer */ 1488 SET_IVOR(12, 0x1a0) /* Watchdog Timer */ 1489 SET_IVOR(13, 0x1c0) /* Data TLB Error */ 1490 SET_IVOR(14, 0x1e0) /* Instruction TLB Error */ 1491 SET_IVOR(15, 0x040) /* Debug */ 1492 1493 sync 1494 1495 blr 1496 1497_GLOBAL(setup_altivec_ivors) 1498 SET_IVOR(32, 0x200) /* AltiVec Unavailable */ 1499 SET_IVOR(33, 0x220) /* AltiVec Assist */ 1500 blr 1501 1502_GLOBAL(setup_perfmon_ivor) 1503 SET_IVOR(35, 0x260) /* Performance Monitor */ 1504 blr 1505 1506_GLOBAL(setup_doorbell_ivors) 1507 SET_IVOR(36, 0x280) /* Processor Doorbell */ 1508 SET_IVOR(37, 0x2a0) /* Processor Doorbell Crit */ 1509 blr 1510 1511_GLOBAL(setup_ehv_ivors) 1512 SET_IVOR(40, 0x300) /* Embedded Hypervisor System Call */ 1513 SET_IVOR(41, 0x320) /* Embedded Hypervisor Privilege */ 1514 SET_IVOR(38, 0x2c0) /* Guest Processor Doorbell */ 1515 SET_IVOR(39, 0x2e0) /* Guest Processor Doorbell Crit/MC */ 1516 blr 1517 1518_GLOBAL(setup_lrat_ivor) 1519 SET_IVOR(42, 0x340) /* LRAT Error */ 1520 blr 1521