1/* SPDX-License-Identifier: GPL-2.0 */ 2/* 3 * This file contains the 64-bit "server" PowerPC variant 4 * of the low level exception handling including exception 5 * vectors, exception return, part of the slb and stab 6 * handling and other fixed offset specific things. 7 * 8 * This file is meant to be #included from head_64.S due to 9 * position dependent assembly. 10 * 11 * Most of this originates from head_64.S and thus has the same 12 * copyright history. 13 * 14 */ 15 16#include <asm/hw_irq.h> 17#include <asm/exception-64s.h> 18#include <asm/ptrace.h> 19#include <asm/cpuidle.h> 20#include <asm/head-64.h> 21#include <asm/feature-fixups.h> 22#include <asm/kup.h> 23 24/* PACA save area offsets (exgen, exmc, etc) */ 25#define EX_R9 0 26#define EX_R10 8 27#define EX_R11 16 28#define EX_R12 24 29#define EX_R13 32 30#define EX_DAR 40 31#define EX_DSISR 48 32#define EX_CCR 52 33#define EX_CFAR 56 34#define EX_PPR 64 35#define EX_CTR 72 36.if EX_SIZE != 10 37 .error "EX_SIZE is wrong" 38.endif 39 40/* 41 * Following are fixed section helper macros. 42 * 43 * EXC_REAL_BEGIN/END - real, unrelocated exception vectors 44 * EXC_VIRT_BEGIN/END - virt (AIL), unrelocated exception vectors 45 * TRAMP_REAL_BEGIN - real, unrelocated helpers (virt may call these) 46 * TRAMP_VIRT_BEGIN - virt, unreloc helpers (in practice, real can use) 47 * EXC_COMMON - After switching to virtual, relocated mode. 48 */ 49 50#define EXC_REAL_BEGIN(name, start, size) \ 51 FIXED_SECTION_ENTRY_BEGIN_LOCATION(real_vectors, exc_real_##start##_##name, start, size) 52 53#define EXC_REAL_END(name, start, size) \ 54 FIXED_SECTION_ENTRY_END_LOCATION(real_vectors, exc_real_##start##_##name, start, size) 55 56#define EXC_VIRT_BEGIN(name, start, size) \ 57 FIXED_SECTION_ENTRY_BEGIN_LOCATION(virt_vectors, exc_virt_##start##_##name, start, size) 58 59#define EXC_VIRT_END(name, start, size) \ 60 FIXED_SECTION_ENTRY_END_LOCATION(virt_vectors, exc_virt_##start##_##name, start, size) 61 62#define EXC_COMMON_BEGIN(name) \ 63 USE_TEXT_SECTION(); \ 64 .balign IFETCH_ALIGN_BYTES; \ 65 .global name; \ 66 _ASM_NOKPROBE_SYMBOL(name); \ 67 DEFINE_FIXED_SYMBOL(name); \ 68name: 69 70#define TRAMP_REAL_BEGIN(name) \ 71 FIXED_SECTION_ENTRY_BEGIN(real_trampolines, name) 72 73#define TRAMP_VIRT_BEGIN(name) \ 74 FIXED_SECTION_ENTRY_BEGIN(virt_trampolines, name) 75 76#define EXC_REAL_NONE(start, size) \ 77 FIXED_SECTION_ENTRY_BEGIN_LOCATION(real_vectors, exc_real_##start##_##unused, start, size); \ 78 FIXED_SECTION_ENTRY_END_LOCATION(real_vectors, exc_real_##start##_##unused, start, size) 79 80#define EXC_VIRT_NONE(start, size) \ 81 FIXED_SECTION_ENTRY_BEGIN_LOCATION(virt_vectors, exc_virt_##start##_##unused, start, size); \ 82 FIXED_SECTION_ENTRY_END_LOCATION(virt_vectors, exc_virt_##start##_##unused, start, size) 83 84/* 85 * We're short on space and time in the exception prolog, so we can't 86 * use the normal LOAD_REG_IMMEDIATE macro to load the address of label. 87 * Instead we get the base of the kernel from paca->kernelbase and or in the low 88 * part of label. This requires that the label be within 64KB of kernelbase, and 89 * that kernelbase be 64K aligned. 90 */ 91#define LOAD_HANDLER(reg, label) \ 92 ld reg,PACAKBASE(r13); /* get high part of &label */ \ 93 ori reg,reg,FIXED_SYMBOL_ABS_ADDR(label) 94 95#define __LOAD_HANDLER(reg, label) \ 96 ld reg,PACAKBASE(r13); \ 97 ori reg,reg,(ABS_ADDR(label))@l 98 99/* 100 * Branches from unrelocated code (e.g., interrupts) to labels outside 101 * head-y require >64K offsets. 102 */ 103#define __LOAD_FAR_HANDLER(reg, label) \ 104 ld reg,PACAKBASE(r13); \ 105 ori reg,reg,(ABS_ADDR(label))@l; \ 106 addis reg,reg,(ABS_ADDR(label))@h 107 108/* 109 * Branch to label using its 0xC000 address. This results in instruction 110 * address suitable for MSR[IR]=0 or 1, which allows relocation to be turned 111 * on using mtmsr rather than rfid. 112 * 113 * This could set the 0xc bits for !RELOCATABLE as an immediate, rather than 114 * load KBASE for a slight optimisation. 115 */ 116#define BRANCH_TO_C000(reg, label) \ 117 __LOAD_FAR_HANDLER(reg, label); \ 118 mtctr reg; \ 119 bctr 120 121/* 122 * Interrupt code generation macros 123 */ 124#define IVEC .L_IVEC_\name\() /* Interrupt vector address */ 125#define IHSRR .L_IHSRR_\name\() /* Sets SRR or HSRR registers */ 126#define IHSRR_IF_HVMODE .L_IHSRR_IF_HVMODE_\name\() /* HSRR if HV else SRR */ 127#define IAREA .L_IAREA_\name\() /* PACA save area */ 128#define IVIRT .L_IVIRT_\name\() /* Has virt mode entry point */ 129#define IISIDE .L_IISIDE_\name\() /* Uses SRR0/1 not DAR/DSISR */ 130#define IDAR .L_IDAR_\name\() /* Uses DAR (or SRR0) */ 131#define IDSISR .L_IDSISR_\name\() /* Uses DSISR (or SRR1) */ 132#define ISET_RI .L_ISET_RI_\name\() /* Run common code w/ MSR[RI]=1 */ 133#define IBRANCH_TO_COMMON .L_IBRANCH_TO_COMMON_\name\() /* ENTRY branch to common */ 134#define IREALMODE_COMMON .L_IREALMODE_COMMON_\name\() /* Common runs in realmode */ 135#define IMASK .L_IMASK_\name\() /* IRQ soft-mask bit */ 136#define IKVM_SKIP .L_IKVM_SKIP_\name\() /* Generate KVM skip handler */ 137#define IKVM_REAL .L_IKVM_REAL_\name\() /* Real entry tests KVM */ 138#define __IKVM_REAL(name) .L_IKVM_REAL_ ## name 139#define IKVM_VIRT .L_IKVM_VIRT_\name\() /* Virt entry tests KVM */ 140#define ISTACK .L_ISTACK_\name\() /* Set regular kernel stack */ 141#define __ISTACK(name) .L_ISTACK_ ## name 142#define IRECONCILE .L_IRECONCILE_\name\() /* Do RECONCILE_IRQ_STATE */ 143#define IKUAP .L_IKUAP_\name\() /* Do KUAP lock */ 144 145#define INT_DEFINE_BEGIN(n) \ 146.macro int_define_ ## n name 147 148#define INT_DEFINE_END(n) \ 149.endm ; \ 150int_define_ ## n n ; \ 151do_define_int n 152 153.macro do_define_int name 154 .ifndef IVEC 155 .error "IVEC not defined" 156 .endif 157 .ifndef IHSRR 158 IHSRR=0 159 .endif 160 .ifndef IHSRR_IF_HVMODE 161 IHSRR_IF_HVMODE=0 162 .endif 163 .ifndef IAREA 164 IAREA=PACA_EXGEN 165 .endif 166 .ifndef IVIRT 167 IVIRT=1 168 .endif 169 .ifndef IISIDE 170 IISIDE=0 171 .endif 172 .ifndef IDAR 173 IDAR=0 174 .endif 175 .ifndef IDSISR 176 IDSISR=0 177 .endif 178 .ifndef ISET_RI 179 ISET_RI=1 180 .endif 181 .ifndef IBRANCH_TO_COMMON 182 IBRANCH_TO_COMMON=1 183 .endif 184 .ifndef IREALMODE_COMMON 185 IREALMODE_COMMON=0 186 .else 187 .if ! IBRANCH_TO_COMMON 188 .error "IREALMODE_COMMON=1 but IBRANCH_TO_COMMON=0" 189 .endif 190 .endif 191 .ifndef IMASK 192 IMASK=0 193 .endif 194 .ifndef IKVM_SKIP 195 IKVM_SKIP=0 196 .endif 197 .ifndef IKVM_REAL 198 IKVM_REAL=0 199 .endif 200 .ifndef IKVM_VIRT 201 IKVM_VIRT=0 202 .endif 203 .ifndef ISTACK 204 ISTACK=1 205 .endif 206 .ifndef IRECONCILE 207 IRECONCILE=1 208 .endif 209 .ifndef IKUAP 210 IKUAP=1 211 .endif 212.endm 213 214#ifdef CONFIG_KVM_BOOK3S_64_HANDLER 215#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE 216/* 217 * All interrupts which set HSRR registers, as well as SRESET and MCE and 218 * syscall when invoked with "sc 1" switch to MSR[HV]=1 (HVMODE) to be taken, 219 * so they all generally need to test whether they were taken in guest context. 220 * 221 * Note: SRESET and MCE may also be sent to the guest by the hypervisor, and be 222 * taken with MSR[HV]=0. 223 * 224 * Interrupts which set SRR registers (with the above exceptions) do not 225 * elevate to MSR[HV]=1 mode, though most can be taken when running with 226 * MSR[HV]=1 (e.g., bare metal kernel and userspace). So these interrupts do 227 * not need to test whether a guest is running because they get delivered to 228 * the guest directly, including nested HV KVM guests. 229 * 230 * The exception is PR KVM, where the guest runs with MSR[PR]=1 and the host 231 * runs with MSR[HV]=0, so the host takes all interrupts on behalf of the 232 * guest. PR KVM runs with LPCR[AIL]=0 which causes interrupts to always be 233 * delivered to the real-mode entry point, therefore such interrupts only test 234 * KVM in their real mode handlers, and only when PR KVM is possible. 235 * 236 * Interrupts that are taken in MSR[HV]=0 and escalate to MSR[HV]=1 are always 237 * delivered in real-mode when the MMU is in hash mode because the MMU 238 * registers are not set appropriately to translate host addresses. In nested 239 * radix mode these can be delivered in virt-mode as the host translations are 240 * used implicitly (see: effective LPID, effective PID). 241 */ 242 243/* 244 * If an interrupt is taken while a guest is running, it is immediately routed 245 * to KVM to handle. If both HV and PR KVM arepossible, KVM interrupts go first 246 * to kvmppc_interrupt_hv, which handles the PR guest case. 247 */ 248#define kvmppc_interrupt kvmppc_interrupt_hv 249#else 250#define kvmppc_interrupt kvmppc_interrupt_pr 251#endif 252 253.macro KVMTEST name 254 lbz r10,HSTATE_IN_GUEST(r13) 255 cmpwi r10,0 256 bne \name\()_kvm 257.endm 258 259.macro GEN_KVM name 260 .balign IFETCH_ALIGN_BYTES 261\name\()_kvm: 262 263 .if IKVM_SKIP 264 cmpwi r10,KVM_GUEST_MODE_SKIP 265 beq 89f 266 .else 267BEGIN_FTR_SECTION 268 ld r10,IAREA+EX_CFAR(r13) 269 std r10,HSTATE_CFAR(r13) 270END_FTR_SECTION_IFSET(CPU_FTR_CFAR) 271 .endif 272 273 ld r10,PACA_EXGEN+EX_CTR(r13) 274 mtctr r10 275BEGIN_FTR_SECTION 276 ld r10,IAREA+EX_PPR(r13) 277 std r10,HSTATE_PPR(r13) 278END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 279 ld r11,IAREA+EX_R11(r13) 280 ld r12,IAREA+EX_R12(r13) 281 std r12,HSTATE_SCRATCH0(r13) 282 sldi r12,r9,32 283 ld r9,IAREA+EX_R9(r13) 284 ld r10,IAREA+EX_R10(r13) 285 /* HSRR variants have the 0x2 bit added to their trap number */ 286 .if IHSRR_IF_HVMODE 287 BEGIN_FTR_SECTION 288 ori r12,r12,(IVEC + 0x2) 289 FTR_SECTION_ELSE 290 ori r12,r12,(IVEC) 291 ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) 292 .elseif IHSRR 293 ori r12,r12,(IVEC+ 0x2) 294 .else 295 ori r12,r12,(IVEC) 296 .endif 297 b kvmppc_interrupt 298 299 .if IKVM_SKIP 30089: mtocrf 0x80,r9 301 ld r10,PACA_EXGEN+EX_CTR(r13) 302 mtctr r10 303 ld r9,IAREA+EX_R9(r13) 304 ld r10,IAREA+EX_R10(r13) 305 ld r11,IAREA+EX_R11(r13) 306 ld r12,IAREA+EX_R12(r13) 307 .if IHSRR_IF_HVMODE 308 BEGIN_FTR_SECTION 309 b kvmppc_skip_Hinterrupt 310 FTR_SECTION_ELSE 311 b kvmppc_skip_interrupt 312 ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) 313 .elseif IHSRR 314 b kvmppc_skip_Hinterrupt 315 .else 316 b kvmppc_skip_interrupt 317 .endif 318 .endif 319.endm 320 321#else 322.macro KVMTEST name 323.endm 324.macro GEN_KVM name 325.endm 326#endif 327 328/* 329 * This is the BOOK3S interrupt entry code macro. 330 * 331 * This can result in one of several things happening: 332 * - Branch to the _common handler, relocated, in virtual mode. 333 * These are normal interrupts (synchronous and asynchronous) handled by 334 * the kernel. 335 * - Branch to KVM, relocated but real mode interrupts remain in real mode. 336 * These occur when HSTATE_IN_GUEST is set. The interrupt may be caused by 337 * / intended for host or guest kernel, but KVM must always be involved 338 * because the machine state is set for guest execution. 339 * - Branch to the masked handler, unrelocated. 340 * These occur when maskable asynchronous interrupts are taken with the 341 * irq_soft_mask set. 342 * - Branch to an "early" handler in real mode but relocated. 343 * This is done if early=1. MCE and HMI use these to handle errors in real 344 * mode. 345 * - Fall through and continue executing in real, unrelocated mode. 346 * This is done if early=2. 347 */ 348 349.macro GEN_BRANCH_TO_COMMON name, virt 350 .if IREALMODE_COMMON 351 LOAD_HANDLER(r10, \name\()_common) 352 mtctr r10 353 bctr 354 .else 355 .if \virt 356#ifndef CONFIG_RELOCATABLE 357 b \name\()_common_virt 358#else 359 LOAD_HANDLER(r10, \name\()_common_virt) 360 mtctr r10 361 bctr 362#endif 363 .else 364 LOAD_HANDLER(r10, \name\()_common_real) 365 mtctr r10 366 bctr 367 .endif 368 .endif 369.endm 370 371.macro GEN_INT_ENTRY name, virt, ool=0 372 SET_SCRATCH0(r13) /* save r13 */ 373 GET_PACA(r13) 374 std r9,IAREA+EX_R9(r13) /* save r9 */ 375BEGIN_FTR_SECTION 376 mfspr r9,SPRN_PPR 377END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 378 HMT_MEDIUM 379 std r10,IAREA+EX_R10(r13) /* save r10 - r12 */ 380BEGIN_FTR_SECTION 381 mfspr r10,SPRN_CFAR 382END_FTR_SECTION_IFSET(CPU_FTR_CFAR) 383 .if \ool 384 .if !\virt 385 b tramp_real_\name 386 .pushsection .text 387 TRAMP_REAL_BEGIN(tramp_real_\name) 388 .else 389 b tramp_virt_\name 390 .pushsection .text 391 TRAMP_VIRT_BEGIN(tramp_virt_\name) 392 .endif 393 .endif 394 395BEGIN_FTR_SECTION 396 std r9,IAREA+EX_PPR(r13) 397END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 398BEGIN_FTR_SECTION 399 std r10,IAREA+EX_CFAR(r13) 400END_FTR_SECTION_IFSET(CPU_FTR_CFAR) 401 INTERRUPT_TO_KERNEL 402 mfctr r10 403 std r10,IAREA+EX_CTR(r13) 404 mfcr r9 405 std r11,IAREA+EX_R11(r13) 406 std r12,IAREA+EX_R12(r13) 407 408 /* 409 * DAR/DSISR, SCRATCH0 must be read before setting MSR[RI], 410 * because a d-side MCE will clobber those registers so is 411 * not recoverable if they are live. 412 */ 413 GET_SCRATCH0(r10) 414 std r10,IAREA+EX_R13(r13) 415 .if IDAR && !IISIDE 416 .if IHSRR 417 mfspr r10,SPRN_HDAR 418 .else 419 mfspr r10,SPRN_DAR 420 .endif 421 std r10,IAREA+EX_DAR(r13) 422 .endif 423 .if IDSISR && !IISIDE 424 .if IHSRR 425 mfspr r10,SPRN_HDSISR 426 .else 427 mfspr r10,SPRN_DSISR 428 .endif 429 stw r10,IAREA+EX_DSISR(r13) 430 .endif 431 432 .if IHSRR_IF_HVMODE 433 BEGIN_FTR_SECTION 434 mfspr r11,SPRN_HSRR0 /* save HSRR0 */ 435 mfspr r12,SPRN_HSRR1 /* and HSRR1 */ 436 FTR_SECTION_ELSE 437 mfspr r11,SPRN_SRR0 /* save SRR0 */ 438 mfspr r12,SPRN_SRR1 /* and SRR1 */ 439 ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) 440 .elseif IHSRR 441 mfspr r11,SPRN_HSRR0 /* save HSRR0 */ 442 mfspr r12,SPRN_HSRR1 /* and HSRR1 */ 443 .else 444 mfspr r11,SPRN_SRR0 /* save SRR0 */ 445 mfspr r12,SPRN_SRR1 /* and SRR1 */ 446 .endif 447 448 .if IBRANCH_TO_COMMON 449 GEN_BRANCH_TO_COMMON \name \virt 450 .endif 451 452 .if \ool 453 .popsection 454 .endif 455.endm 456 457/* 458 * __GEN_COMMON_ENTRY is required to receive the branch from interrupt 459 * entry, except in the case of the real-mode handlers which require 460 * __GEN_REALMODE_COMMON_ENTRY. 461 * 462 * This switches to virtual mode and sets MSR[RI]. 463 */ 464.macro __GEN_COMMON_ENTRY name 465DEFINE_FIXED_SYMBOL(\name\()_common_real) 466\name\()_common_real: 467 .if IKVM_REAL 468 KVMTEST \name 469 .endif 470 471 ld r10,PACAKMSR(r13) /* get MSR value for kernel */ 472 /* MSR[RI] is clear iff using SRR regs */ 473 .if IHSRR == EXC_HV_OR_STD 474 BEGIN_FTR_SECTION 475 xori r10,r10,MSR_RI 476 END_FTR_SECTION_IFCLR(CPU_FTR_HVMODE) 477 .elseif ! IHSRR 478 xori r10,r10,MSR_RI 479 .endif 480 mtmsrd r10 481 482 .if IVIRT 483 .if IKVM_VIRT 484 b 1f /* skip the virt test coming from real */ 485 .endif 486 487 .balign IFETCH_ALIGN_BYTES 488DEFINE_FIXED_SYMBOL(\name\()_common_virt) 489\name\()_common_virt: 490 .if IKVM_VIRT 491 KVMTEST \name 4921: 493 .endif 494 .endif /* IVIRT */ 495.endm 496 497/* 498 * Don't switch to virt mode. Used for early MCE and HMI handlers that 499 * want to run in real mode. 500 */ 501.macro __GEN_REALMODE_COMMON_ENTRY name 502DEFINE_FIXED_SYMBOL(\name\()_common_real) 503\name\()_common_real: 504 .if IKVM_REAL 505 KVMTEST \name 506 .endif 507.endm 508 509.macro __GEN_COMMON_BODY name 510 .if IMASK 511 lbz r10,PACAIRQSOFTMASK(r13) 512 andi. r10,r10,IMASK 513 /* Associate vector numbers with bits in paca->irq_happened */ 514 .if IVEC == 0x500 || IVEC == 0xea0 515 li r10,PACA_IRQ_EE 516 .elseif IVEC == 0x900 517 li r10,PACA_IRQ_DEC 518 .elseif IVEC == 0xa00 || IVEC == 0xe80 519 li r10,PACA_IRQ_DBELL 520 .elseif IVEC == 0xe60 521 li r10,PACA_IRQ_HMI 522 .elseif IVEC == 0xf00 523 li r10,PACA_IRQ_PMI 524 .else 525 .abort "Bad maskable vector" 526 .endif 527 528 .if IHSRR_IF_HVMODE 529 BEGIN_FTR_SECTION 530 bne masked_Hinterrupt 531 FTR_SECTION_ELSE 532 bne masked_interrupt 533 ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) 534 .elseif IHSRR 535 bne masked_Hinterrupt 536 .else 537 bne masked_interrupt 538 .endif 539 .endif 540 541 .if ISTACK 542 andi. r10,r12,MSR_PR /* See if coming from user */ 543 mr r10,r1 /* Save r1 */ 544 subi r1,r1,INT_FRAME_SIZE /* alloc frame on kernel stack */ 545 beq- 100f 546 ld r1,PACAKSAVE(r13) /* kernel stack to use */ 547100: tdgei r1,-INT_FRAME_SIZE /* trap if r1 is in userspace */ 548 EMIT_BUG_ENTRY 100b,__FILE__,__LINE__,0 549 .endif 550 551 std r9,_CCR(r1) /* save CR in stackframe */ 552 std r11,_NIP(r1) /* save SRR0 in stackframe */ 553 std r12,_MSR(r1) /* save SRR1 in stackframe */ 554 std r10,0(r1) /* make stack chain pointer */ 555 std r0,GPR0(r1) /* save r0 in stackframe */ 556 std r10,GPR1(r1) /* save r1 in stackframe */ 557 558 .if ISET_RI 559 li r10,MSR_RI 560 mtmsrd r10,1 /* Set MSR_RI */ 561 .endif 562 563 .if ISTACK 564 .if IKUAP 565 kuap_save_amr_and_lock r9, r10, cr1, cr0 566 .endif 567 beq 101f /* if from kernel mode */ 568 ACCOUNT_CPU_USER_ENTRY(r13, r9, r10) 569BEGIN_FTR_SECTION 570 ld r9,IAREA+EX_PPR(r13) /* Read PPR from paca */ 571 std r9,_PPR(r1) 572END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 573101: 574 .else 575 .if IKUAP 576 kuap_save_amr_and_lock r9, r10, cr1 577 .endif 578 .endif 579 580 /* Save original regs values from save area to stack frame. */ 581 ld r9,IAREA+EX_R9(r13) /* move r9, r10 to stackframe */ 582 ld r10,IAREA+EX_R10(r13) 583 std r9,GPR9(r1) 584 std r10,GPR10(r1) 585 ld r9,IAREA+EX_R11(r13) /* move r11 - r13 to stackframe */ 586 ld r10,IAREA+EX_R12(r13) 587 ld r11,IAREA+EX_R13(r13) 588 std r9,GPR11(r1) 589 std r10,GPR12(r1) 590 std r11,GPR13(r1) 591 592 SAVE_NVGPRS(r1) 593 594 .if IDAR 595 .if IISIDE 596 ld r10,_NIP(r1) 597 .else 598 ld r10,IAREA+EX_DAR(r13) 599 .endif 600 std r10,_DAR(r1) 601 .endif 602 603 .if IDSISR 604 .if IISIDE 605 ld r10,_MSR(r1) 606 lis r11,DSISR_SRR1_MATCH_64S@h 607 and r10,r10,r11 608 .else 609 lwz r10,IAREA+EX_DSISR(r13) 610 .endif 611 std r10,_DSISR(r1) 612 .endif 613 614BEGIN_FTR_SECTION 615 ld r10,IAREA+EX_CFAR(r13) 616 std r10,ORIG_GPR3(r1) 617END_FTR_SECTION_IFSET(CPU_FTR_CFAR) 618 ld r10,IAREA+EX_CTR(r13) 619 std r10,_CTR(r1) 620 std r2,GPR2(r1) /* save r2 in stackframe */ 621 SAVE_4GPRS(3, r1) /* save r3 - r6 in stackframe */ 622 SAVE_2GPRS(7, r1) /* save r7, r8 in stackframe */ 623 mflr r9 /* Get LR, later save to stack */ 624 ld r2,PACATOC(r13) /* get kernel TOC into r2 */ 625 std r9,_LINK(r1) 626 lbz r10,PACAIRQSOFTMASK(r13) 627 mfspr r11,SPRN_XER /* save XER in stackframe */ 628 std r10,SOFTE(r1) 629 std r11,_XER(r1) 630 li r9,IVEC 631 std r9,_TRAP(r1) /* set trap number */ 632 li r10,0 633 ld r11,exception_marker@toc(r2) 634 std r10,RESULT(r1) /* clear regs->result */ 635 std r11,STACK_FRAME_OVERHEAD-16(r1) /* mark the frame */ 636 637 .if ISTACK 638 ACCOUNT_STOLEN_TIME 639 .endif 640 641 .if IRECONCILE 642 RECONCILE_IRQ_STATE(r10, r11) 643 .endif 644.endm 645 646/* 647 * On entry r13 points to the paca, r9-r13 are saved in the paca, 648 * r9 contains the saved CR, r11 and r12 contain the saved SRR0 and 649 * SRR1, and relocation is on. 650 * 651 * If stack=0, then the stack is already set in r1, and r1 is saved in r10. 652 * PPR save and CPU accounting is not done for the !stack case (XXX why not?) 653 */ 654.macro GEN_COMMON name 655 __GEN_COMMON_ENTRY \name 656 __GEN_COMMON_BODY \name 657.endm 658 659/* 660 * Restore all registers including H/SRR0/1 saved in a stack frame of a 661 * standard exception. 662 */ 663.macro EXCEPTION_RESTORE_REGS hsrr=0 664 /* Move original SRR0 and SRR1 into the respective regs */ 665 ld r9,_MSR(r1) 666 .if \hsrr 667 mtspr SPRN_HSRR1,r9 668 .else 669 mtspr SPRN_SRR1,r9 670 .endif 671 ld r9,_NIP(r1) 672 .if \hsrr 673 mtspr SPRN_HSRR0,r9 674 .else 675 mtspr SPRN_SRR0,r9 676 .endif 677 ld r9,_CTR(r1) 678 mtctr r9 679 ld r9,_XER(r1) 680 mtxer r9 681 ld r9,_LINK(r1) 682 mtlr r9 683 ld r9,_CCR(r1) 684 mtcr r9 685 REST_8GPRS(2, r1) 686 REST_4GPRS(10, r1) 687 REST_GPR(0, r1) 688 /* restore original r1. */ 689 ld r1,GPR1(r1) 690.endm 691 692#define RUNLATCH_ON \ 693BEGIN_FTR_SECTION \ 694 ld r3, PACA_THREAD_INFO(r13); \ 695 ld r4,TI_LOCAL_FLAGS(r3); \ 696 andi. r0,r4,_TLF_RUNLATCH; \ 697 beql ppc64_runlatch_on_trampoline; \ 698END_FTR_SECTION_IFSET(CPU_FTR_CTRL) 699 700/* 701 * When the idle code in power4_idle puts the CPU into NAP mode, 702 * it has to do so in a loop, and relies on the external interrupt 703 * and decrementer interrupt entry code to get it out of the loop. 704 * It sets the _TLF_NAPPING bit in current_thread_info()->local_flags 705 * to signal that it is in the loop and needs help to get out. 706 */ 707#ifdef CONFIG_PPC_970_NAP 708#define FINISH_NAP \ 709BEGIN_FTR_SECTION \ 710 ld r11, PACA_THREAD_INFO(r13); \ 711 ld r9,TI_LOCAL_FLAGS(r11); \ 712 andi. r10,r9,_TLF_NAPPING; \ 713 bnel power4_fixup_nap; \ 714END_FTR_SECTION_IFSET(CPU_FTR_CAN_NAP) 715#else 716#define FINISH_NAP 717#endif 718 719/* 720 * There are a few constraints to be concerned with. 721 * - Real mode exceptions code/data must be located at their physical location. 722 * - Virtual mode exceptions must be mapped at their 0xc000... location. 723 * - Fixed location code must not call directly beyond the __end_interrupts 724 * area when built with CONFIG_RELOCATABLE. LOAD_HANDLER / bctr sequence 725 * must be used. 726 * - LOAD_HANDLER targets must be within first 64K of physical 0 / 727 * virtual 0xc00... 728 * - Conditional branch targets must be within +/-32K of caller. 729 * 730 * "Virtual exceptions" run with relocation on (MSR_IR=1, MSR_DR=1), and 731 * therefore don't have to run in physically located code or rfid to 732 * virtual mode kernel code. However on relocatable kernels they do have 733 * to branch to KERNELBASE offset because the rest of the kernel (outside 734 * the exception vectors) may be located elsewhere. 735 * 736 * Virtual exceptions correspond with physical, except their entry points 737 * are offset by 0xc000000000000000 and also tend to get an added 0x4000 738 * offset applied. Virtual exceptions are enabled with the Alternate 739 * Interrupt Location (AIL) bit set in the LPCR. However this does not 740 * guarantee they will be delivered virtually. Some conditions (see the ISA) 741 * cause exceptions to be delivered in real mode. 742 * 743 * It's impossible to receive interrupts below 0x300 via AIL. 744 * 745 * KVM: None of the virtual exceptions are from the guest. Anything that 746 * escalated to HV=1 from HV=0 is delivered via real mode handlers. 747 * 748 * 749 * We layout physical memory as follows: 750 * 0x0000 - 0x00ff : Secondary processor spin code 751 * 0x0100 - 0x18ff : Real mode pSeries interrupt vectors 752 * 0x1900 - 0x3fff : Real mode trampolines 753 * 0x4000 - 0x58ff : Relon (IR=1,DR=1) mode pSeries interrupt vectors 754 * 0x5900 - 0x6fff : Relon mode trampolines 755 * 0x7000 - 0x7fff : FWNMI data area 756 * 0x8000 - .... : Common interrupt handlers, remaining early 757 * setup code, rest of kernel. 758 * 759 * We could reclaim 0x4000-0x42ff for real mode trampolines if the space 760 * is necessary. Until then it's more consistent to explicitly put VIRT_NONE 761 * vectors there. 762 */ 763OPEN_FIXED_SECTION(real_vectors, 0x0100, 0x1900) 764OPEN_FIXED_SECTION(real_trampolines, 0x1900, 0x4000) 765OPEN_FIXED_SECTION(virt_vectors, 0x4000, 0x5900) 766OPEN_FIXED_SECTION(virt_trampolines, 0x5900, 0x7000) 767 768#ifdef CONFIG_PPC_POWERNV 769 .globl start_real_trampolines 770 .globl end_real_trampolines 771 .globl start_virt_trampolines 772 .globl end_virt_trampolines 773#endif 774 775#if defined(CONFIG_PPC_PSERIES) || defined(CONFIG_PPC_POWERNV) 776/* 777 * Data area reserved for FWNMI option. 778 * This address (0x7000) is fixed by the RPA. 779 * pseries and powernv need to keep the whole page from 780 * 0x7000 to 0x8000 free for use by the firmware 781 */ 782ZERO_FIXED_SECTION(fwnmi_page, 0x7000, 0x8000) 783OPEN_TEXT_SECTION(0x8000) 784#else 785OPEN_TEXT_SECTION(0x7000) 786#endif 787 788USE_FIXED_SECTION(real_vectors) 789 790/* 791 * This is the start of the interrupt handlers for pSeries 792 * This code runs with relocation off. 793 * Code from here to __end_interrupts gets copied down to real 794 * address 0x100 when we are running a relocatable kernel. 795 * Therefore any relative branches in this section must only 796 * branch to labels in this section. 797 */ 798 .globl __start_interrupts 799__start_interrupts: 800 801/* No virt vectors corresponding with 0x0..0x100 */ 802EXC_VIRT_NONE(0x4000, 0x100) 803 804 805/** 806 * Interrupt 0x100 - System Reset Interrupt (SRESET aka NMI). 807 * This is a non-maskable, asynchronous interrupt always taken in real-mode. 808 * It is caused by: 809 * - Wake from power-saving state, on powernv. 810 * - An NMI from another CPU, triggered by firmware or hypercall. 811 * - As crash/debug signal injected from BMC, firmware or hypervisor. 812 * 813 * Handling: 814 * Power-save wakeup is the only performance critical path, so this is 815 * determined quickly as possible first. In this case volatile registers 816 * can be discarded and SPRs like CFAR don't need to be read. 817 * 818 * If not a powersave wakeup, then it's run as a regular interrupt, however 819 * it uses its own stack and PACA save area to preserve the regular kernel 820 * environment for debugging. 821 * 822 * This interrupt is not maskable, so triggering it when MSR[RI] is clear, 823 * or SCRATCH0 is in use, etc. may cause a crash. It's also not entirely 824 * correct to switch to virtual mode to run the regular interrupt handler 825 * because it might be interrupted when the MMU is in a bad state (e.g., SLB 826 * is clear). 827 * 828 * FWNMI: 829 * PAPR specifies a "fwnmi" facility which sends the sreset to a different 830 * entry point with a different register set up. Some hypervisors will 831 * send the sreset to 0x100 in the guest if it is not fwnmi capable. 832 * 833 * KVM: 834 * Unlike most SRR interrupts, this may be taken by the host while executing 835 * in a guest, so a KVM test is required. KVM will pull the CPU out of guest 836 * mode and then raise the sreset. 837 */ 838INT_DEFINE_BEGIN(system_reset) 839 IVEC=0x100 840 IAREA=PACA_EXNMI 841 IVIRT=0 /* no virt entry point */ 842 /* 843 * MSR_RI is not enabled, because PACA_EXNMI and nmi stack is 844 * being used, so a nested NMI exception would corrupt it. 845 */ 846 ISET_RI=0 847 ISTACK=0 848 IRECONCILE=0 849 IKVM_REAL=1 850INT_DEFINE_END(system_reset) 851 852EXC_REAL_BEGIN(system_reset, 0x100, 0x100) 853#ifdef CONFIG_PPC_P7_NAP 854 /* 855 * If running native on arch 2.06 or later, check if we are waking up 856 * from nap/sleep/winkle, and branch to idle handler. This tests SRR1 857 * bits 46:47. A non-0 value indicates that we are coming from a power 858 * saving state. The idle wakeup handler initially runs in real mode, 859 * but we branch to the 0xc000... address so we can turn on relocation 860 * with mtmsrd later, after SPRs are restored. 861 * 862 * Careful to minimise cost for the fast path (idle wakeup) while 863 * also avoiding clobbering CFAR for the debug path (non-idle). 864 * 865 * For the idle wake case volatile registers can be clobbered, which 866 * is why we use those initially. If it turns out to not be an idle 867 * wake, carefully put everything back the way it was, so we can use 868 * common exception macros to handle it. 869 */ 870BEGIN_FTR_SECTION 871 SET_SCRATCH0(r13) 872 GET_PACA(r13) 873 std r3,PACA_EXNMI+0*8(r13) 874 std r4,PACA_EXNMI+1*8(r13) 875 std r5,PACA_EXNMI+2*8(r13) 876 mfspr r3,SPRN_SRR1 877 mfocrf r4,0x80 878 rlwinm. r5,r3,47-31,30,31 879 bne+ system_reset_idle_wake 880 /* Not powersave wakeup. Restore regs for regular interrupt handler. */ 881 mtocrf 0x80,r4 882 ld r3,PACA_EXNMI+0*8(r13) 883 ld r4,PACA_EXNMI+1*8(r13) 884 ld r5,PACA_EXNMI+2*8(r13) 885 GET_SCRATCH0(r13) 886END_FTR_SECTION_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) 887#endif 888 889 GEN_INT_ENTRY system_reset, virt=0 890 /* 891 * In theory, we should not enable relocation here if it was disabled 892 * in SRR1, because the MMU may not be configured to support it (e.g., 893 * SLB may have been cleared). In practice, there should only be a few 894 * small windows where that's the case, and sreset is considered to 895 * be dangerous anyway. 896 */ 897EXC_REAL_END(system_reset, 0x100, 0x100) 898EXC_VIRT_NONE(0x4100, 0x100) 899 900#ifdef CONFIG_PPC_P7_NAP 901TRAMP_REAL_BEGIN(system_reset_idle_wake) 902 /* We are waking up from idle, so may clobber any volatile register */ 903 cmpwi cr1,r5,2 904 bltlr cr1 /* no state loss, return to idle caller with r3=SRR1 */ 905 BRANCH_TO_C000(r12, DOTSYM(idle_return_gpr_loss)) 906#endif 907 908#ifdef CONFIG_PPC_PSERIES 909/* 910 * Vectors for the FWNMI option. Share common code. 911 */ 912TRAMP_REAL_BEGIN(system_reset_fwnmi) 913 /* XXX: fwnmi guest could run a nested/PR guest, so why no test? */ 914 __IKVM_REAL(system_reset)=0 915 GEN_INT_ENTRY system_reset, virt=0 916 917#endif /* CONFIG_PPC_PSERIES */ 918 919EXC_COMMON_BEGIN(system_reset_common) 920 __GEN_COMMON_ENTRY system_reset 921 /* 922 * Increment paca->in_nmi then enable MSR_RI. SLB or MCE will be able 923 * to recover, but nested NMI will notice in_nmi and not recover 924 * because of the use of the NMI stack. in_nmi reentrancy is tested in 925 * system_reset_exception. 926 */ 927 lhz r10,PACA_IN_NMI(r13) 928 addi r10,r10,1 929 sth r10,PACA_IN_NMI(r13) 930 li r10,MSR_RI 931 mtmsrd r10,1 932 933 mr r10,r1 934 ld r1,PACA_NMI_EMERG_SP(r13) 935 subi r1,r1,INT_FRAME_SIZE 936 __GEN_COMMON_BODY system_reset 937 /* 938 * Set IRQS_ALL_DISABLED unconditionally so irqs_disabled() does 939 * the right thing. We do not want to reconcile because that goes 940 * through irq tracing which we don't want in NMI. 941 * 942 * Save PACAIRQHAPPENED to _DAR (otherwise unused), and set HARD_DIS 943 * as we are running with MSR[EE]=0. 944 */ 945 li r10,IRQS_ALL_DISABLED 946 stb r10,PACAIRQSOFTMASK(r13) 947 lbz r10,PACAIRQHAPPENED(r13) 948 std r10,_DAR(r1) 949 ori r10,r10,PACA_IRQ_HARD_DIS 950 stb r10,PACAIRQHAPPENED(r13) 951 952 addi r3,r1,STACK_FRAME_OVERHEAD 953 bl system_reset_exception 954 955 /* Clear MSR_RI before setting SRR0 and SRR1. */ 956 li r9,0 957 mtmsrd r9,1 958 959 /* 960 * MSR_RI is clear, now we can decrement paca->in_nmi. 961 */ 962 lhz r10,PACA_IN_NMI(r13) 963 subi r10,r10,1 964 sth r10,PACA_IN_NMI(r13) 965 966 /* 967 * Restore soft mask settings. 968 */ 969 ld r10,_DAR(r1) 970 stb r10,PACAIRQHAPPENED(r13) 971 ld r10,SOFTE(r1) 972 stb r10,PACAIRQSOFTMASK(r13) 973 974 kuap_restore_amr r10 975 EXCEPTION_RESTORE_REGS 976 RFI_TO_USER_OR_KERNEL 977 978 GEN_KVM system_reset 979 980 981/** 982 * Interrupt 0x200 - Machine Check Interrupt (MCE). 983 * This is a non-maskable interrupt always taken in real-mode. It can be 984 * synchronous or asynchronous, caused by hardware or software, and it may be 985 * taken in a power-saving state. 986 * 987 * Handling: 988 * Similarly to system reset, this uses its own stack and PACA save area, 989 * the difference is re-entrancy is allowed on the machine check stack. 990 * 991 * machine_check_early is run in real mode, and carefully decodes the 992 * machine check and tries to handle it (e.g., flush the SLB if there was an 993 * error detected there), determines if it was recoverable and logs the 994 * event. 995 * 996 * This early code does not "reconcile" irq soft-mask state like SRESET or 997 * regular interrupts do, so irqs_disabled() among other things may not work 998 * properly (irq disable/enable already doesn't work because irq tracing can 999 * not work in real mode). 1000 * 1001 * Then, depending on the execution context when the interrupt is taken, there 1002 * are 3 main actions: 1003 * - Executing in kernel mode. The event is queued with irq_work, which means 1004 * it is handled when it is next safe to do so (i.e., the kernel has enabled 1005 * interrupts), which could be immediately when the interrupt returns. This 1006 * avoids nasty issues like switching to virtual mode when the MMU is in a 1007 * bad state, or when executing OPAL code. (SRESET is exposed to such issues, 1008 * but it has different priorities). Check to see if the CPU was in power 1009 * save, and return via the wake up code if it was. 1010 * 1011 * - Executing in user mode. machine_check_exception is run like a normal 1012 * interrupt handler, which processes the data generated by the early handler. 1013 * 1014 * - Executing in guest mode. The interrupt is run with its KVM test, and 1015 * branches to KVM to deal with. KVM may queue the event for the host 1016 * to report later. 1017 * 1018 * This interrupt is not maskable, so if it triggers when MSR[RI] is clear, 1019 * or SCRATCH0 is in use, it may cause a crash. 1020 * 1021 * KVM: 1022 * See SRESET. 1023 */ 1024INT_DEFINE_BEGIN(machine_check_early) 1025 IVEC=0x200 1026 IAREA=PACA_EXMC 1027 IVIRT=0 /* no virt entry point */ 1028 IREALMODE_COMMON=1 1029 /* 1030 * MSR_RI is not enabled, because PACA_EXMC is being used, so a 1031 * nested machine check corrupts it. machine_check_common enables 1032 * MSR_RI. 1033 */ 1034 ISET_RI=0 1035 ISTACK=0 1036 IDAR=1 1037 IDSISR=1 1038 IRECONCILE=0 1039 IKUAP=0 /* We don't touch AMR here, we never go to virtual mode */ 1040INT_DEFINE_END(machine_check_early) 1041 1042INT_DEFINE_BEGIN(machine_check) 1043 IVEC=0x200 1044 IAREA=PACA_EXMC 1045 IVIRT=0 /* no virt entry point */ 1046 ISET_RI=0 1047 IDAR=1 1048 IDSISR=1 1049 IKVM_SKIP=1 1050 IKVM_REAL=1 1051INT_DEFINE_END(machine_check) 1052 1053EXC_REAL_BEGIN(machine_check, 0x200, 0x100) 1054 GEN_INT_ENTRY machine_check_early, virt=0 1055EXC_REAL_END(machine_check, 0x200, 0x100) 1056EXC_VIRT_NONE(0x4200, 0x100) 1057 1058#ifdef CONFIG_PPC_PSERIES 1059TRAMP_REAL_BEGIN(machine_check_fwnmi) 1060 /* See comment at machine_check exception, don't turn on RI */ 1061 GEN_INT_ENTRY machine_check_early, virt=0 1062#endif 1063 1064#define MACHINE_CHECK_HANDLER_WINDUP \ 1065 /* Clear MSR_RI before setting SRR0 and SRR1. */\ 1066 li r9,0; \ 1067 mtmsrd r9,1; /* Clear MSR_RI */ \ 1068 /* Decrement paca->in_mce now RI is clear. */ \ 1069 lhz r12,PACA_IN_MCE(r13); \ 1070 subi r12,r12,1; \ 1071 sth r12,PACA_IN_MCE(r13); \ 1072 EXCEPTION_RESTORE_REGS 1073 1074EXC_COMMON_BEGIN(machine_check_early_common) 1075 __GEN_REALMODE_COMMON_ENTRY machine_check_early 1076 1077 /* 1078 * Switch to mc_emergency stack and handle re-entrancy (we limit 1079 * the nested MCE upto level 4 to avoid stack overflow). 1080 * Save MCE registers srr1, srr0, dar and dsisr and then set ME=1 1081 * 1082 * We use paca->in_mce to check whether this is the first entry or 1083 * nested machine check. We increment paca->in_mce to track nested 1084 * machine checks. 1085 * 1086 * If this is the first entry then set stack pointer to 1087 * paca->mc_emergency_sp, otherwise r1 is already pointing to 1088 * stack frame on mc_emergency stack. 1089 * 1090 * NOTE: We are here with MSR_ME=0 (off), which means we risk a 1091 * checkstop if we get another machine check exception before we do 1092 * rfid with MSR_ME=1. 1093 * 1094 * This interrupt can wake directly from idle. If that is the case, 1095 * the machine check is handled then the idle wakeup code is called 1096 * to restore state. 1097 */ 1098 lhz r10,PACA_IN_MCE(r13) 1099 cmpwi r10,0 /* Are we in nested machine check */ 1100 cmpwi cr1,r10,MAX_MCE_DEPTH /* Are we at maximum nesting */ 1101 addi r10,r10,1 /* increment paca->in_mce */ 1102 sth r10,PACA_IN_MCE(r13) 1103 1104 mr r10,r1 /* Save r1 */ 1105 bne 1f 1106 /* First machine check entry */ 1107 ld r1,PACAMCEMERGSP(r13) /* Use MC emergency stack */ 11081: /* Limit nested MCE to level 4 to avoid stack overflow */ 1109 bgt cr1,unrecoverable_mce /* Check if we hit limit of 4 */ 1110 subi r1,r1,INT_FRAME_SIZE /* alloc stack frame */ 1111 1112 __GEN_COMMON_BODY machine_check_early 1113 1114BEGIN_FTR_SECTION 1115 bl enable_machine_check 1116END_FTR_SECTION_IFSET(CPU_FTR_HVMODE) 1117 li r10,MSR_RI 1118 mtmsrd r10,1 1119 1120 addi r3,r1,STACK_FRAME_OVERHEAD 1121 bl machine_check_early 1122 std r3,RESULT(r1) /* Save result */ 1123 ld r12,_MSR(r1) 1124 1125#ifdef CONFIG_PPC_P7_NAP 1126 /* 1127 * Check if thread was in power saving mode. We come here when any 1128 * of the following is true: 1129 * a. thread wasn't in power saving mode 1130 * b. thread was in power saving mode with no state loss, 1131 * supervisor state loss or hypervisor state loss. 1132 * 1133 * Go back to nap/sleep/winkle mode again if (b) is true. 1134 */ 1135BEGIN_FTR_SECTION 1136 rlwinm. r11,r12,47-31,30,31 1137 bne machine_check_idle_common 1138END_FTR_SECTION_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) 1139#endif 1140 1141#ifdef CONFIG_KVM_BOOK3S_64_HANDLER 1142 /* 1143 * Check if we are coming from guest. If yes, then run the normal 1144 * exception handler which will take the 1145 * machine_check_kvm->kvmppc_interrupt branch to deliver the MC event 1146 * to guest. 1147 */ 1148 lbz r11,HSTATE_IN_GUEST(r13) 1149 cmpwi r11,0 /* Check if coming from guest */ 1150 bne mce_deliver /* continue if we are. */ 1151#endif 1152 1153 /* 1154 * Check if we are coming from userspace. If yes, then run the normal 1155 * exception handler which will deliver the MC event to this kernel. 1156 */ 1157 andi. r11,r12,MSR_PR /* See if coming from user. */ 1158 bne mce_deliver /* continue in V mode if we are. */ 1159 1160 /* 1161 * At this point we are coming from kernel context. 1162 * Queue up the MCE event and return from the interrupt. 1163 * But before that, check if this is an un-recoverable exception. 1164 * If yes, then stay on emergency stack and panic. 1165 */ 1166 andi. r11,r12,MSR_RI 1167 beq unrecoverable_mce 1168 1169 /* 1170 * Check if we have successfully handled/recovered from error, if not 1171 * then stay on emergency stack and panic. 1172 */ 1173 ld r3,RESULT(r1) /* Load result */ 1174 cmpdi r3,0 /* see if we handled MCE successfully */ 1175 beq unrecoverable_mce /* if !handled then panic */ 1176 1177 /* 1178 * Return from MC interrupt. 1179 * Queue up the MCE event so that we can log it later, while 1180 * returning from kernel or opal call. 1181 */ 1182 bl machine_check_queue_event 1183 MACHINE_CHECK_HANDLER_WINDUP 1184 RFI_TO_KERNEL 1185 1186mce_deliver: 1187 /* 1188 * This is a host user or guest MCE. Restore all registers, then 1189 * run the "late" handler. For host user, this will run the 1190 * machine_check_exception handler in virtual mode like a normal 1191 * interrupt handler. For guest, this will trigger the KVM test 1192 * and branch to the KVM interrupt similarly to other interrupts. 1193 */ 1194BEGIN_FTR_SECTION 1195 ld r10,ORIG_GPR3(r1) 1196 mtspr SPRN_CFAR,r10 1197END_FTR_SECTION_IFSET(CPU_FTR_CFAR) 1198 MACHINE_CHECK_HANDLER_WINDUP 1199 GEN_INT_ENTRY machine_check, virt=0 1200 1201EXC_COMMON_BEGIN(machine_check_common) 1202 /* 1203 * Machine check is different because we use a different 1204 * save area: PACA_EXMC instead of PACA_EXGEN. 1205 */ 1206 GEN_COMMON machine_check 1207 1208 FINISH_NAP 1209 /* Enable MSR_RI when finished with PACA_EXMC */ 1210 li r10,MSR_RI 1211 mtmsrd r10,1 1212 addi r3,r1,STACK_FRAME_OVERHEAD 1213 bl machine_check_exception 1214 b interrupt_return 1215 1216 GEN_KVM machine_check 1217 1218 1219#ifdef CONFIG_PPC_P7_NAP 1220/* 1221 * This is an idle wakeup. Low level machine check has already been 1222 * done. Queue the event then call the idle code to do the wake up. 1223 */ 1224EXC_COMMON_BEGIN(machine_check_idle_common) 1225 bl machine_check_queue_event 1226 1227 /* 1228 * We have not used any non-volatile GPRs here, and as a rule 1229 * most exception code including machine check does not. 1230 * Therefore PACA_NAPSTATELOST does not need to be set. Idle 1231 * wakeup will restore volatile registers. 1232 * 1233 * Load the original SRR1 into r3 for pnv_powersave_wakeup_mce. 1234 * 1235 * Then decrement MCE nesting after finishing with the stack. 1236 */ 1237 ld r3,_MSR(r1) 1238 ld r4,_LINK(r1) 1239 1240 lhz r11,PACA_IN_MCE(r13) 1241 subi r11,r11,1 1242 sth r11,PACA_IN_MCE(r13) 1243 1244 mtlr r4 1245 rlwinm r10,r3,47-31,30,31 1246 cmpwi cr1,r10,2 1247 bltlr cr1 /* no state loss, return to idle caller */ 1248 b idle_return_gpr_loss 1249#endif 1250 1251EXC_COMMON_BEGIN(unrecoverable_mce) 1252 /* 1253 * We are going down. But there are chances that we might get hit by 1254 * another MCE during panic path and we may run into unstable state 1255 * with no way out. Hence, turn ME bit off while going down, so that 1256 * when another MCE is hit during panic path, system will checkstop 1257 * and hypervisor will get restarted cleanly by SP. 1258 */ 1259BEGIN_FTR_SECTION 1260 li r10,0 /* clear MSR_RI */ 1261 mtmsrd r10,1 1262 bl disable_machine_check 1263END_FTR_SECTION_IFSET(CPU_FTR_HVMODE) 1264 ld r10,PACAKMSR(r13) 1265 li r3,MSR_ME 1266 andc r10,r10,r3 1267 mtmsrd r10 1268 1269 /* Invoke machine_check_exception to print MCE event and panic. */ 1270 addi r3,r1,STACK_FRAME_OVERHEAD 1271 bl machine_check_exception 1272 1273 /* 1274 * We will not reach here. Even if we did, there is no way out. 1275 * Call unrecoverable_exception and die. 1276 */ 1277 addi r3,r1,STACK_FRAME_OVERHEAD 1278 bl unrecoverable_exception 1279 b . 1280 1281 1282/** 1283 * Interrupt 0x300 - Data Storage Interrupt (DSI). 1284 * This is a synchronous interrupt generated due to a data access exception, 1285 * e.g., a load orstore which does not have a valid page table entry with 1286 * permissions. DAWR matches also fault here, as do RC updates, and minor misc 1287 * errors e.g., copy/paste, AMO, certain invalid CI accesses, etc. 1288 * 1289 * Handling: 1290 * - Hash MMU 1291 * Go to do_hash_page first to see if the HPT can be filled from an entry in 1292 * the Linux page table. Hash faults can hit in kernel mode in a fairly 1293 * arbitrary state (e.g., interrupts disabled, locks held) when accessing 1294 * "non-bolted" regions, e.g., vmalloc space. However these should always be 1295 * backed by Linux page tables. 1296 * 1297 * If none is found, do a Linux page fault. Linux page faults can happen in 1298 * kernel mode due to user copy operations of course. 1299 * 1300 * - Radix MMU 1301 * The hardware loads from the Linux page table directly, so a fault goes 1302 * immediately to Linux page fault. 1303 * 1304 * Conditions like DAWR match are handled on the way in to Linux page fault. 1305 */ 1306INT_DEFINE_BEGIN(data_access) 1307 IVEC=0x300 1308 IDAR=1 1309 IDSISR=1 1310#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 1311 IKVM_SKIP=1 1312 IKVM_REAL=1 1313#endif 1314INT_DEFINE_END(data_access) 1315 1316EXC_REAL_BEGIN(data_access, 0x300, 0x80) 1317 GEN_INT_ENTRY data_access, virt=0 1318EXC_REAL_END(data_access, 0x300, 0x80) 1319EXC_VIRT_BEGIN(data_access, 0x4300, 0x80) 1320 GEN_INT_ENTRY data_access, virt=1 1321EXC_VIRT_END(data_access, 0x4300, 0x80) 1322EXC_COMMON_BEGIN(data_access_common) 1323 GEN_COMMON data_access 1324 ld r4,_DAR(r1) 1325 ld r5,_DSISR(r1) 1326BEGIN_MMU_FTR_SECTION 1327 ld r6,_MSR(r1) 1328 li r3,0x300 1329 b do_hash_page /* Try to handle as hpte fault */ 1330MMU_FTR_SECTION_ELSE 1331 b handle_page_fault 1332ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) 1333 1334 GEN_KVM data_access 1335 1336 1337/** 1338 * Interrupt 0x380 - Data Segment Interrupt (DSLB). 1339 * This is a synchronous interrupt in response to an MMU fault missing SLB 1340 * entry for HPT, or an address outside RPT translation range. 1341 * 1342 * Handling: 1343 * - HPT: 1344 * This refills the SLB, or reports an access fault similarly to a bad page 1345 * fault. When coming from user-mode, the SLB handler may access any kernel 1346 * data, though it may itself take a DSLB. When coming from kernel mode, 1347 * recursive faults must be avoided so access is restricted to the kernel 1348 * image text/data, kernel stack, and any data allocated below 1349 * ppc64_bolted_size (first segment). The kernel handler must avoid stomping 1350 * on user-handler data structures. 1351 * 1352 * A dedicated save area EXSLB is used (XXX: but it actually need not be 1353 * these days, we could use EXGEN). 1354 */ 1355INT_DEFINE_BEGIN(data_access_slb) 1356 IVEC=0x380 1357 IAREA=PACA_EXSLB 1358 IRECONCILE=0 1359 IDAR=1 1360#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 1361 IKVM_SKIP=1 1362 IKVM_REAL=1 1363#endif 1364INT_DEFINE_END(data_access_slb) 1365 1366EXC_REAL_BEGIN(data_access_slb, 0x380, 0x80) 1367 GEN_INT_ENTRY data_access_slb, virt=0 1368EXC_REAL_END(data_access_slb, 0x380, 0x80) 1369EXC_VIRT_BEGIN(data_access_slb, 0x4380, 0x80) 1370 GEN_INT_ENTRY data_access_slb, virt=1 1371EXC_VIRT_END(data_access_slb, 0x4380, 0x80) 1372EXC_COMMON_BEGIN(data_access_slb_common) 1373 GEN_COMMON data_access_slb 1374 ld r4,_DAR(r1) 1375 addi r3,r1,STACK_FRAME_OVERHEAD 1376BEGIN_MMU_FTR_SECTION 1377 /* HPT case, do SLB fault */ 1378 bl do_slb_fault 1379 cmpdi r3,0 1380 bne- 1f 1381 b fast_interrupt_return 13821: /* Error case */ 1383MMU_FTR_SECTION_ELSE 1384 /* Radix case, access is outside page table range */ 1385 li r3,-EFAULT 1386ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) 1387 std r3,RESULT(r1) 1388 RECONCILE_IRQ_STATE(r10, r11) 1389 ld r4,_DAR(r1) 1390 ld r5,RESULT(r1) 1391 addi r3,r1,STACK_FRAME_OVERHEAD 1392 bl do_bad_slb_fault 1393 b interrupt_return 1394 1395 GEN_KVM data_access_slb 1396 1397 1398/** 1399 * Interrupt 0x400 - Instruction Storage Interrupt (ISI). 1400 * This is a synchronous interrupt in response to an MMU fault due to an 1401 * instruction fetch. 1402 * 1403 * Handling: 1404 * Similar to DSI, though in response to fetch. The faulting address is found 1405 * in SRR0 (rather than DAR), and status in SRR1 (rather than DSISR). 1406 */ 1407INT_DEFINE_BEGIN(instruction_access) 1408 IVEC=0x400 1409 IISIDE=1 1410 IDAR=1 1411 IDSISR=1 1412#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 1413 IKVM_REAL=1 1414#endif 1415INT_DEFINE_END(instruction_access) 1416 1417EXC_REAL_BEGIN(instruction_access, 0x400, 0x80) 1418 GEN_INT_ENTRY instruction_access, virt=0 1419EXC_REAL_END(instruction_access, 0x400, 0x80) 1420EXC_VIRT_BEGIN(instruction_access, 0x4400, 0x80) 1421 GEN_INT_ENTRY instruction_access, virt=1 1422EXC_VIRT_END(instruction_access, 0x4400, 0x80) 1423EXC_COMMON_BEGIN(instruction_access_common) 1424 GEN_COMMON instruction_access 1425 ld r4,_DAR(r1) 1426 ld r5,_DSISR(r1) 1427BEGIN_MMU_FTR_SECTION 1428 ld r6,_MSR(r1) 1429 li r3,0x400 1430 b do_hash_page /* Try to handle as hpte fault */ 1431MMU_FTR_SECTION_ELSE 1432 b handle_page_fault 1433ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) 1434 1435 GEN_KVM instruction_access 1436 1437 1438/** 1439 * Interrupt 0x480 - Instruction Segment Interrupt (ISLB). 1440 * This is a synchronous interrupt in response to an MMU fault due to an 1441 * instruction fetch. 1442 * 1443 * Handling: 1444 * Similar to DSLB, though in response to fetch. The faulting address is found 1445 * in SRR0 (rather than DAR). 1446 */ 1447INT_DEFINE_BEGIN(instruction_access_slb) 1448 IVEC=0x480 1449 IAREA=PACA_EXSLB 1450 IRECONCILE=0 1451 IISIDE=1 1452 IDAR=1 1453#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 1454 IKVM_REAL=1 1455#endif 1456INT_DEFINE_END(instruction_access_slb) 1457 1458EXC_REAL_BEGIN(instruction_access_slb, 0x480, 0x80) 1459 GEN_INT_ENTRY instruction_access_slb, virt=0 1460EXC_REAL_END(instruction_access_slb, 0x480, 0x80) 1461EXC_VIRT_BEGIN(instruction_access_slb, 0x4480, 0x80) 1462 GEN_INT_ENTRY instruction_access_slb, virt=1 1463EXC_VIRT_END(instruction_access_slb, 0x4480, 0x80) 1464EXC_COMMON_BEGIN(instruction_access_slb_common) 1465 GEN_COMMON instruction_access_slb 1466 ld r4,_DAR(r1) 1467 addi r3,r1,STACK_FRAME_OVERHEAD 1468BEGIN_MMU_FTR_SECTION 1469 /* HPT case, do SLB fault */ 1470 bl do_slb_fault 1471 cmpdi r3,0 1472 bne- 1f 1473 b fast_interrupt_return 14741: /* Error case */ 1475MMU_FTR_SECTION_ELSE 1476 /* Radix case, access is outside page table range */ 1477 li r3,-EFAULT 1478ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) 1479 std r3,RESULT(r1) 1480 RECONCILE_IRQ_STATE(r10, r11) 1481 ld r4,_DAR(r1) 1482 ld r5,RESULT(r1) 1483 addi r3,r1,STACK_FRAME_OVERHEAD 1484 bl do_bad_slb_fault 1485 b interrupt_return 1486 1487 GEN_KVM instruction_access_slb 1488 1489 1490/** 1491 * Interrupt 0x500 - External Interrupt. 1492 * This is an asynchronous maskable interrupt in response to an "external 1493 * exception" from the interrupt controller or hypervisor (e.g., device 1494 * interrupt). It is maskable in hardware by clearing MSR[EE], and 1495 * soft-maskable with IRQS_DISABLED mask (i.e., local_irq_disable()). 1496 * 1497 * When running in HV mode, Linux sets up the LPCR[LPES] bit such that 1498 * interrupts are delivered with HSRR registers, guests use SRRs, which 1499 * reqiures IHSRR_IF_HVMODE. 1500 * 1501 * On bare metal POWER9 and later, Linux sets the LPCR[HVICE] bit such that 1502 * external interrupts are delivered as Hypervisor Virtualization Interrupts 1503 * rather than External Interrupts. 1504 * 1505 * Handling: 1506 * This calls into Linux IRQ handler. NVGPRs are not saved to reduce overhead, 1507 * because registers at the time of the interrupt are not so important as it is 1508 * asynchronous. 1509 * 1510 * If soft masked, the masked handler will note the pending interrupt for 1511 * replay, and clear MSR[EE] in the interrupted context. 1512 */ 1513INT_DEFINE_BEGIN(hardware_interrupt) 1514 IVEC=0x500 1515 IHSRR_IF_HVMODE=1 1516 IMASK=IRQS_DISABLED 1517 IKVM_REAL=1 1518 IKVM_VIRT=1 1519INT_DEFINE_END(hardware_interrupt) 1520 1521EXC_REAL_BEGIN(hardware_interrupt, 0x500, 0x100) 1522 GEN_INT_ENTRY hardware_interrupt, virt=0 1523EXC_REAL_END(hardware_interrupt, 0x500, 0x100) 1524EXC_VIRT_BEGIN(hardware_interrupt, 0x4500, 0x100) 1525 GEN_INT_ENTRY hardware_interrupt, virt=1 1526EXC_VIRT_END(hardware_interrupt, 0x4500, 0x100) 1527EXC_COMMON_BEGIN(hardware_interrupt_common) 1528 GEN_COMMON hardware_interrupt 1529 FINISH_NAP 1530 RUNLATCH_ON 1531 addi r3,r1,STACK_FRAME_OVERHEAD 1532 bl do_IRQ 1533 b interrupt_return 1534 1535 GEN_KVM hardware_interrupt 1536 1537 1538/** 1539 * Interrupt 0x600 - Alignment Interrupt 1540 * This is a synchronous interrupt in response to data alignment fault. 1541 */ 1542INT_DEFINE_BEGIN(alignment) 1543 IVEC=0x600 1544 IDAR=1 1545 IDSISR=1 1546#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 1547 IKVM_REAL=1 1548#endif 1549INT_DEFINE_END(alignment) 1550 1551EXC_REAL_BEGIN(alignment, 0x600, 0x100) 1552 GEN_INT_ENTRY alignment, virt=0 1553EXC_REAL_END(alignment, 0x600, 0x100) 1554EXC_VIRT_BEGIN(alignment, 0x4600, 0x100) 1555 GEN_INT_ENTRY alignment, virt=1 1556EXC_VIRT_END(alignment, 0x4600, 0x100) 1557EXC_COMMON_BEGIN(alignment_common) 1558 GEN_COMMON alignment 1559 addi r3,r1,STACK_FRAME_OVERHEAD 1560 bl alignment_exception 1561 REST_NVGPRS(r1) /* instruction emulation may change GPRs */ 1562 b interrupt_return 1563 1564 GEN_KVM alignment 1565 1566 1567/** 1568 * Interrupt 0x700 - Program Interrupt (program check). 1569 * This is a synchronous interrupt in response to various instruction faults: 1570 * traps, privilege errors, TM errors, floating point exceptions. 1571 * 1572 * Handling: 1573 * This interrupt may use the "emergency stack" in some cases when being taken 1574 * from kernel context, which complicates handling. 1575 */ 1576INT_DEFINE_BEGIN(program_check) 1577 IVEC=0x700 1578#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 1579 IKVM_REAL=1 1580#endif 1581INT_DEFINE_END(program_check) 1582 1583EXC_REAL_BEGIN(program_check, 0x700, 0x100) 1584 GEN_INT_ENTRY program_check, virt=0 1585EXC_REAL_END(program_check, 0x700, 0x100) 1586EXC_VIRT_BEGIN(program_check, 0x4700, 0x100) 1587 GEN_INT_ENTRY program_check, virt=1 1588EXC_VIRT_END(program_check, 0x4700, 0x100) 1589EXC_COMMON_BEGIN(program_check_common) 1590 __GEN_COMMON_ENTRY program_check 1591 1592 /* 1593 * It's possible to receive a TM Bad Thing type program check with 1594 * userspace register values (in particular r1), but with SRR1 reporting 1595 * that we came from the kernel. Normally that would confuse the bad 1596 * stack logic, and we would report a bad kernel stack pointer. Instead 1597 * we switch to the emergency stack if we're taking a TM Bad Thing from 1598 * the kernel. 1599 */ 1600 1601 andi. r10,r12,MSR_PR 1602 bne 2f /* If userspace, go normal path */ 1603 1604 andis. r10,r12,(SRR1_PROGTM)@h 1605 bne 1f /* If TM, emergency */ 1606 1607 cmpdi r1,-INT_FRAME_SIZE /* check if r1 is in userspace */ 1608 blt 2f /* normal path if not */ 1609 1610 /* Use the emergency stack */ 16111: andi. r10,r12,MSR_PR /* Set CR0 correctly for label */ 1612 /* 3 in EXCEPTION_PROLOG_COMMON */ 1613 mr r10,r1 /* Save r1 */ 1614 ld r1,PACAEMERGSP(r13) /* Use emergency stack */ 1615 subi r1,r1,INT_FRAME_SIZE /* alloc stack frame */ 1616 __ISTACK(program_check)=0 1617 __GEN_COMMON_BODY program_check 1618 b 3f 16192: 1620 __ISTACK(program_check)=1 1621 __GEN_COMMON_BODY program_check 16223: 1623 addi r3,r1,STACK_FRAME_OVERHEAD 1624 bl program_check_exception 1625 REST_NVGPRS(r1) /* instruction emulation may change GPRs */ 1626 b interrupt_return 1627 1628 GEN_KVM program_check 1629 1630 1631/* 1632 * Interrupt 0x800 - Floating-Point Unavailable Interrupt. 1633 * This is a synchronous interrupt in response to executing an fp instruction 1634 * with MSR[FP]=0. 1635 * 1636 * Handling: 1637 * This will load FP registers and enable the FP bit if coming from userspace, 1638 * otherwise report a bad kernel use of FP. 1639 */ 1640INT_DEFINE_BEGIN(fp_unavailable) 1641 IVEC=0x800 1642 IRECONCILE=0 1643#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 1644 IKVM_REAL=1 1645#endif 1646INT_DEFINE_END(fp_unavailable) 1647 1648EXC_REAL_BEGIN(fp_unavailable, 0x800, 0x100) 1649 GEN_INT_ENTRY fp_unavailable, virt=0 1650EXC_REAL_END(fp_unavailable, 0x800, 0x100) 1651EXC_VIRT_BEGIN(fp_unavailable, 0x4800, 0x100) 1652 GEN_INT_ENTRY fp_unavailable, virt=1 1653EXC_VIRT_END(fp_unavailable, 0x4800, 0x100) 1654EXC_COMMON_BEGIN(fp_unavailable_common) 1655 GEN_COMMON fp_unavailable 1656 bne 1f /* if from user, just load it up */ 1657 RECONCILE_IRQ_STATE(r10, r11) 1658 addi r3,r1,STACK_FRAME_OVERHEAD 1659 bl kernel_fp_unavailable_exception 16600: trap 1661 EMIT_BUG_ENTRY 0b, __FILE__, __LINE__, 0 16621: 1663#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 1664BEGIN_FTR_SECTION 1665 /* Test if 2 TM state bits are zero. If non-zero (ie. userspace was in 1666 * transaction), go do TM stuff 1667 */ 1668 rldicl. r0, r12, (64-MSR_TS_LG), (64-2) 1669 bne- 2f 1670END_FTR_SECTION_IFSET(CPU_FTR_TM) 1671#endif 1672 bl load_up_fpu 1673 b fast_interrupt_return 1674#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 16752: /* User process was in a transaction */ 1676 RECONCILE_IRQ_STATE(r10, r11) 1677 addi r3,r1,STACK_FRAME_OVERHEAD 1678 bl fp_unavailable_tm 1679 b interrupt_return 1680#endif 1681 1682 GEN_KVM fp_unavailable 1683 1684 1685/** 1686 * Interrupt 0x900 - Decrementer Interrupt. 1687 * This is an asynchronous interrupt in response to a decrementer exception 1688 * (e.g., DEC has wrapped below zero). It is maskable in hardware by clearing 1689 * MSR[EE], and soft-maskable with IRQS_DISABLED mask (i.e., 1690 * local_irq_disable()). 1691 * 1692 * Handling: 1693 * This calls into Linux timer handler. NVGPRs are not saved (see 0x500). 1694 * 1695 * If soft masked, the masked handler will note the pending interrupt for 1696 * replay, and bump the decrementer to a high value, leaving MSR[EE] enabled 1697 * in the interrupted context. 1698 * If PPC_WATCHDOG is configured, the soft masked handler will actually set 1699 * things back up to run soft_nmi_interrupt as a regular interrupt handler 1700 * on the emergency stack. 1701 */ 1702INT_DEFINE_BEGIN(decrementer) 1703 IVEC=0x900 1704 IMASK=IRQS_DISABLED 1705#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 1706 IKVM_REAL=1 1707#endif 1708INT_DEFINE_END(decrementer) 1709 1710EXC_REAL_BEGIN(decrementer, 0x900, 0x80) 1711 GEN_INT_ENTRY decrementer, virt=0 1712EXC_REAL_END(decrementer, 0x900, 0x80) 1713EXC_VIRT_BEGIN(decrementer, 0x4900, 0x80) 1714 GEN_INT_ENTRY decrementer, virt=1 1715EXC_VIRT_END(decrementer, 0x4900, 0x80) 1716EXC_COMMON_BEGIN(decrementer_common) 1717 GEN_COMMON decrementer 1718 FINISH_NAP 1719 RUNLATCH_ON 1720 addi r3,r1,STACK_FRAME_OVERHEAD 1721 bl timer_interrupt 1722 b interrupt_return 1723 1724 GEN_KVM decrementer 1725 1726 1727/** 1728 * Interrupt 0x980 - Hypervisor Decrementer Interrupt. 1729 * This is an asynchronous interrupt, similar to 0x900 but for the HDEC 1730 * register. 1731 * 1732 * Handling: 1733 * Linux does not use this outside KVM where it's used to keep a host timer 1734 * while the guest is given control of DEC. It should normally be caught by 1735 * the KVM test and routed there. 1736 */ 1737INT_DEFINE_BEGIN(hdecrementer) 1738 IVEC=0x980 1739 IHSRR=1 1740 ISTACK=0 1741 IRECONCILE=0 1742 IKVM_REAL=1 1743 IKVM_VIRT=1 1744INT_DEFINE_END(hdecrementer) 1745 1746EXC_REAL_BEGIN(hdecrementer, 0x980, 0x80) 1747 GEN_INT_ENTRY hdecrementer, virt=0 1748EXC_REAL_END(hdecrementer, 0x980, 0x80) 1749EXC_VIRT_BEGIN(hdecrementer, 0x4980, 0x80) 1750 GEN_INT_ENTRY hdecrementer, virt=1 1751EXC_VIRT_END(hdecrementer, 0x4980, 0x80) 1752EXC_COMMON_BEGIN(hdecrementer_common) 1753 __GEN_COMMON_ENTRY hdecrementer 1754 /* 1755 * Hypervisor decrementer interrupts not caught by the KVM test 1756 * shouldn't occur but are sometimes left pending on exit from a KVM 1757 * guest. We don't need to do anything to clear them, as they are 1758 * edge-triggered. 1759 * 1760 * Be careful to avoid touching the kernel stack. 1761 */ 1762 ld r10,PACA_EXGEN+EX_CTR(r13) 1763 mtctr r10 1764 mtcrf 0x80,r9 1765 ld r9,PACA_EXGEN+EX_R9(r13) 1766 ld r10,PACA_EXGEN+EX_R10(r13) 1767 ld r11,PACA_EXGEN+EX_R11(r13) 1768 ld r12,PACA_EXGEN+EX_R12(r13) 1769 ld r13,PACA_EXGEN+EX_R13(r13) 1770 HRFI_TO_KERNEL 1771 1772 GEN_KVM hdecrementer 1773 1774 1775/** 1776 * Interrupt 0xa00 - Directed Privileged Doorbell Interrupt. 1777 * This is an asynchronous interrupt in response to a msgsndp doorbell. 1778 * It is maskable in hardware by clearing MSR[EE], and soft-maskable with 1779 * IRQS_DISABLED mask (i.e., local_irq_disable()). 1780 * 1781 * Handling: 1782 * Guests may use this for IPIs between threads in a core if the 1783 * hypervisor supports it. NVGPRS are not saved (see 0x500). 1784 * 1785 * If soft masked, the masked handler will note the pending interrupt for 1786 * replay, leaving MSR[EE] enabled in the interrupted context because the 1787 * doorbells are edge triggered. 1788 */ 1789INT_DEFINE_BEGIN(doorbell_super) 1790 IVEC=0xa00 1791 IMASK=IRQS_DISABLED 1792#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 1793 IKVM_REAL=1 1794#endif 1795INT_DEFINE_END(doorbell_super) 1796 1797EXC_REAL_BEGIN(doorbell_super, 0xa00, 0x100) 1798 GEN_INT_ENTRY doorbell_super, virt=0 1799EXC_REAL_END(doorbell_super, 0xa00, 0x100) 1800EXC_VIRT_BEGIN(doorbell_super, 0x4a00, 0x100) 1801 GEN_INT_ENTRY doorbell_super, virt=1 1802EXC_VIRT_END(doorbell_super, 0x4a00, 0x100) 1803EXC_COMMON_BEGIN(doorbell_super_common) 1804 GEN_COMMON doorbell_super 1805 FINISH_NAP 1806 RUNLATCH_ON 1807 addi r3,r1,STACK_FRAME_OVERHEAD 1808#ifdef CONFIG_PPC_DOORBELL 1809 bl doorbell_exception 1810#else 1811 bl unknown_exception 1812#endif 1813 b interrupt_return 1814 1815 GEN_KVM doorbell_super 1816 1817 1818EXC_REAL_NONE(0xb00, 0x100) 1819EXC_VIRT_NONE(0x4b00, 0x100) 1820 1821/** 1822 * Interrupt 0xc00 - System Call Interrupt (syscall, hcall). 1823 * This is a synchronous interrupt invoked with the "sc" instruction. The 1824 * system call is invoked with "sc 0" and does not alter the HV bit, so it 1825 * is directed to the currently running OS. The hypercall is invoked with 1826 * "sc 1" and it sets HV=1, so it elevates to hypervisor. 1827 * 1828 * In HPT, sc 1 always goes to 0xc00 real mode. In RADIX, sc 1 can go to 1829 * 0x4c00 virtual mode. 1830 * 1831 * Handling: 1832 * If the KVM test fires then it was due to a hypercall and is accordingly 1833 * routed to KVM. Otherwise this executes a normal Linux system call. 1834 * 1835 * Call convention: 1836 * 1837 * syscall and hypercalls register conventions are documented in 1838 * Documentation/powerpc/syscall64-abi.rst and 1839 * Documentation/powerpc/papr_hcalls.rst respectively. 1840 * 1841 * The intersection of volatile registers that don't contain possible 1842 * inputs is: cr0, xer, ctr. We may use these as scratch regs upon entry 1843 * without saving, though xer is not a good idea to use, as hardware may 1844 * interpret some bits so it may be costly to change them. 1845 */ 1846INT_DEFINE_BEGIN(system_call) 1847 IVEC=0xc00 1848 IKVM_REAL=1 1849 IKVM_VIRT=1 1850INT_DEFINE_END(system_call) 1851 1852.macro SYSTEM_CALL virt 1853#ifdef CONFIG_KVM_BOOK3S_64_HANDLER 1854 /* 1855 * There is a little bit of juggling to get syscall and hcall 1856 * working well. Save r13 in ctr to avoid using SPRG scratch 1857 * register. 1858 * 1859 * Userspace syscalls have already saved the PPR, hcalls must save 1860 * it before setting HMT_MEDIUM. 1861 */ 1862 mtctr r13 1863 GET_PACA(r13) 1864 std r10,PACA_EXGEN+EX_R10(r13) 1865 INTERRUPT_TO_KERNEL 1866 KVMTEST system_call /* uses r10, branch to system_call_kvm */ 1867 mfctr r9 1868#else 1869 mr r9,r13 1870 GET_PACA(r13) 1871 INTERRUPT_TO_KERNEL 1872#endif 1873 1874#ifdef CONFIG_PPC_FAST_ENDIAN_SWITCH 1875BEGIN_FTR_SECTION 1876 cmpdi r0,0x1ebe 1877 beq- 1f 1878END_FTR_SECTION_IFSET(CPU_FTR_REAL_LE) 1879#endif 1880 1881 /* We reach here with PACA in r13, r13 in r9. */ 1882 mfspr r11,SPRN_SRR0 1883 mfspr r12,SPRN_SRR1 1884 1885 HMT_MEDIUM 1886 1887 .if ! \virt 1888 __LOAD_HANDLER(r10, system_call_common) 1889 mtspr SPRN_SRR0,r10 1890 ld r10,PACAKMSR(r13) 1891 mtspr SPRN_SRR1,r10 1892 RFI_TO_KERNEL 1893 b . /* prevent speculative execution */ 1894 .else 1895 li r10,MSR_RI 1896 mtmsrd r10,1 /* Set RI (EE=0) */ 1897#ifdef CONFIG_RELOCATABLE 1898 __LOAD_HANDLER(r10, system_call_common) 1899 mtctr r10 1900 bctr 1901#else 1902 b system_call_common 1903#endif 1904 .endif 1905 1906#ifdef CONFIG_PPC_FAST_ENDIAN_SWITCH 1907 /* Fast LE/BE switch system call */ 19081: mfspr r12,SPRN_SRR1 1909 xori r12,r12,MSR_LE 1910 mtspr SPRN_SRR1,r12 1911 mr r13,r9 1912 RFI_TO_USER /* return to userspace */ 1913 b . /* prevent speculative execution */ 1914#endif 1915.endm 1916 1917EXC_REAL_BEGIN(system_call, 0xc00, 0x100) 1918 SYSTEM_CALL 0 1919EXC_REAL_END(system_call, 0xc00, 0x100) 1920EXC_VIRT_BEGIN(system_call, 0x4c00, 0x100) 1921 SYSTEM_CALL 1 1922EXC_VIRT_END(system_call, 0x4c00, 0x100) 1923 1924#ifdef CONFIG_KVM_BOOK3S_64_HANDLER 1925TRAMP_REAL_BEGIN(system_call_kvm) 1926 /* 1927 * This is a hcall, so register convention is as above, with these 1928 * differences: 1929 * r13 = PACA 1930 * ctr = orig r13 1931 * orig r10 saved in PACA 1932 */ 1933 /* 1934 * Save the PPR (on systems that support it) before changing to 1935 * HMT_MEDIUM. That allows the KVM code to save that value into the 1936 * guest state (it is the guest's PPR value). 1937 */ 1938BEGIN_FTR_SECTION 1939 mfspr r10,SPRN_PPR 1940 std r10,HSTATE_PPR(r13) 1941END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 1942 HMT_MEDIUM 1943 mfctr r10 1944 SET_SCRATCH0(r10) 1945 mfcr r10 1946 std r12,HSTATE_SCRATCH0(r13) 1947 sldi r12,r10,32 1948 ori r12,r12,0xc00 1949#ifdef CONFIG_RELOCATABLE 1950 /* 1951 * Requires __LOAD_FAR_HANDLER beause kvmppc_interrupt lives 1952 * outside the head section. 1953 */ 1954 __LOAD_FAR_HANDLER(r10, kvmppc_interrupt) 1955 mtctr r10 1956 ld r10,PACA_EXGEN+EX_R10(r13) 1957 bctr 1958#else 1959 ld r10,PACA_EXGEN+EX_R10(r13) 1960 b kvmppc_interrupt 1961#endif 1962#endif 1963 1964 1965/** 1966 * Interrupt 0xd00 - Trace Interrupt. 1967 * This is a synchronous interrupt in response to instruction step or 1968 * breakpoint faults. 1969 */ 1970INT_DEFINE_BEGIN(single_step) 1971 IVEC=0xd00 1972#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 1973 IKVM_REAL=1 1974#endif 1975INT_DEFINE_END(single_step) 1976 1977EXC_REAL_BEGIN(single_step, 0xd00, 0x100) 1978 GEN_INT_ENTRY single_step, virt=0 1979EXC_REAL_END(single_step, 0xd00, 0x100) 1980EXC_VIRT_BEGIN(single_step, 0x4d00, 0x100) 1981 GEN_INT_ENTRY single_step, virt=1 1982EXC_VIRT_END(single_step, 0x4d00, 0x100) 1983EXC_COMMON_BEGIN(single_step_common) 1984 GEN_COMMON single_step 1985 addi r3,r1,STACK_FRAME_OVERHEAD 1986 bl single_step_exception 1987 b interrupt_return 1988 1989 GEN_KVM single_step 1990 1991 1992/** 1993 * Interrupt 0xe00 - Hypervisor Data Storage Interrupt (HDSI). 1994 * This is a synchronous interrupt in response to an MMU fault caused by a 1995 * guest data access. 1996 * 1997 * Handling: 1998 * This should always get routed to KVM. In radix MMU mode, this is caused 1999 * by a guest nested radix access that can't be performed due to the 2000 * partition scope page table. In hash mode, this can be caused by guests 2001 * running with translation disabled (virtual real mode) or with VPM enabled. 2002 * KVM will update the page table structures or disallow the access. 2003 */ 2004INT_DEFINE_BEGIN(h_data_storage) 2005 IVEC=0xe00 2006 IHSRR=1 2007 IDAR=1 2008 IDSISR=1 2009 IKVM_SKIP=1 2010 IKVM_REAL=1 2011 IKVM_VIRT=1 2012INT_DEFINE_END(h_data_storage) 2013 2014EXC_REAL_BEGIN(h_data_storage, 0xe00, 0x20) 2015 GEN_INT_ENTRY h_data_storage, virt=0, ool=1 2016EXC_REAL_END(h_data_storage, 0xe00, 0x20) 2017EXC_VIRT_BEGIN(h_data_storage, 0x4e00, 0x20) 2018 GEN_INT_ENTRY h_data_storage, virt=1, ool=1 2019EXC_VIRT_END(h_data_storage, 0x4e00, 0x20) 2020EXC_COMMON_BEGIN(h_data_storage_common) 2021 GEN_COMMON h_data_storage 2022 addi r3,r1,STACK_FRAME_OVERHEAD 2023BEGIN_MMU_FTR_SECTION 2024 ld r4,_DAR(r1) 2025 li r5,SIGSEGV 2026 bl bad_page_fault 2027MMU_FTR_SECTION_ELSE 2028 bl unknown_exception 2029ALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_TYPE_RADIX) 2030 b interrupt_return 2031 2032 GEN_KVM h_data_storage 2033 2034 2035/** 2036 * Interrupt 0xe20 - Hypervisor Instruction Storage Interrupt (HISI). 2037 * This is a synchronous interrupt in response to an MMU fault caused by a 2038 * guest instruction fetch, similar to HDSI. 2039 */ 2040INT_DEFINE_BEGIN(h_instr_storage) 2041 IVEC=0xe20 2042 IHSRR=1 2043 IKVM_REAL=1 2044 IKVM_VIRT=1 2045INT_DEFINE_END(h_instr_storage) 2046 2047EXC_REAL_BEGIN(h_instr_storage, 0xe20, 0x20) 2048 GEN_INT_ENTRY h_instr_storage, virt=0, ool=1 2049EXC_REAL_END(h_instr_storage, 0xe20, 0x20) 2050EXC_VIRT_BEGIN(h_instr_storage, 0x4e20, 0x20) 2051 GEN_INT_ENTRY h_instr_storage, virt=1, ool=1 2052EXC_VIRT_END(h_instr_storage, 0x4e20, 0x20) 2053EXC_COMMON_BEGIN(h_instr_storage_common) 2054 GEN_COMMON h_instr_storage 2055 addi r3,r1,STACK_FRAME_OVERHEAD 2056 bl unknown_exception 2057 b interrupt_return 2058 2059 GEN_KVM h_instr_storage 2060 2061 2062/** 2063 * Interrupt 0xe40 - Hypervisor Emulation Assistance Interrupt. 2064 */ 2065INT_DEFINE_BEGIN(emulation_assist) 2066 IVEC=0xe40 2067 IHSRR=1 2068 IKVM_REAL=1 2069 IKVM_VIRT=1 2070INT_DEFINE_END(emulation_assist) 2071 2072EXC_REAL_BEGIN(emulation_assist, 0xe40, 0x20) 2073 GEN_INT_ENTRY emulation_assist, virt=0, ool=1 2074EXC_REAL_END(emulation_assist, 0xe40, 0x20) 2075EXC_VIRT_BEGIN(emulation_assist, 0x4e40, 0x20) 2076 GEN_INT_ENTRY emulation_assist, virt=1, ool=1 2077EXC_VIRT_END(emulation_assist, 0x4e40, 0x20) 2078EXC_COMMON_BEGIN(emulation_assist_common) 2079 GEN_COMMON emulation_assist 2080 addi r3,r1,STACK_FRAME_OVERHEAD 2081 bl emulation_assist_interrupt 2082 REST_NVGPRS(r1) /* instruction emulation may change GPRs */ 2083 b interrupt_return 2084 2085 GEN_KVM emulation_assist 2086 2087 2088/** 2089 * Interrupt 0xe60 - Hypervisor Maintenance Interrupt (HMI). 2090 * This is an asynchronous interrupt caused by a Hypervisor Maintenance 2091 * Exception. It is always taken in real mode but uses HSRR registers 2092 * unlike SRESET and MCE. 2093 * 2094 * It is maskable in hardware by clearing MSR[EE], and partially soft-maskable 2095 * with IRQS_DISABLED mask (i.e., local_irq_disable()). 2096 * 2097 * Handling: 2098 * This is a special case, this is handled similarly to machine checks, with an 2099 * initial real mode handler that is not soft-masked, which attempts to fix the 2100 * problem. Then a regular handler which is soft-maskable and reports the 2101 * problem. 2102 * 2103 * The emergency stack is used for the early real mode handler. 2104 * 2105 * XXX: unclear why MCE and HMI schemes could not be made common, e.g., 2106 * either use soft-masking for the MCE, or use irq_work for the HMI. 2107 * 2108 * KVM: 2109 * Unlike MCE, this calls into KVM without calling the real mode handler 2110 * first. 2111 */ 2112INT_DEFINE_BEGIN(hmi_exception_early) 2113 IVEC=0xe60 2114 IHSRR=1 2115 IREALMODE_COMMON=1 2116 ISTACK=0 2117 IRECONCILE=0 2118 IKUAP=0 /* We don't touch AMR here, we never go to virtual mode */ 2119 IKVM_REAL=1 2120INT_DEFINE_END(hmi_exception_early) 2121 2122INT_DEFINE_BEGIN(hmi_exception) 2123 IVEC=0xe60 2124 IHSRR=1 2125 IMASK=IRQS_DISABLED 2126 IKVM_REAL=1 2127INT_DEFINE_END(hmi_exception) 2128 2129EXC_REAL_BEGIN(hmi_exception, 0xe60, 0x20) 2130 GEN_INT_ENTRY hmi_exception_early, virt=0, ool=1 2131EXC_REAL_END(hmi_exception, 0xe60, 0x20) 2132EXC_VIRT_NONE(0x4e60, 0x20) 2133 2134EXC_COMMON_BEGIN(hmi_exception_early_common) 2135 __GEN_REALMODE_COMMON_ENTRY hmi_exception_early 2136 2137 mr r10,r1 /* Save r1 */ 2138 ld r1,PACAEMERGSP(r13) /* Use emergency stack for realmode */ 2139 subi r1,r1,INT_FRAME_SIZE /* alloc stack frame */ 2140 2141 __GEN_COMMON_BODY hmi_exception_early 2142 2143 addi r3,r1,STACK_FRAME_OVERHEAD 2144 bl hmi_exception_realmode 2145 cmpdi cr0,r3,0 2146 bne 1f 2147 2148 EXCEPTION_RESTORE_REGS hsrr=1 2149 HRFI_TO_USER_OR_KERNEL 2150 21511: 2152 /* 2153 * Go to virtual mode and pull the HMI event information from 2154 * firmware. 2155 */ 2156 EXCEPTION_RESTORE_REGS hsrr=1 2157 GEN_INT_ENTRY hmi_exception, virt=0 2158 2159 GEN_KVM hmi_exception_early 2160 2161EXC_COMMON_BEGIN(hmi_exception_common) 2162 GEN_COMMON hmi_exception 2163 FINISH_NAP 2164 RUNLATCH_ON 2165 addi r3,r1,STACK_FRAME_OVERHEAD 2166 bl handle_hmi_exception 2167 b interrupt_return 2168 2169 GEN_KVM hmi_exception 2170 2171 2172/** 2173 * Interrupt 0xe80 - Directed Hypervisor Doorbell Interrupt. 2174 * This is an asynchronous interrupt in response to a msgsnd doorbell. 2175 * Similar to the 0xa00 doorbell but for host rather than guest. 2176 */ 2177INT_DEFINE_BEGIN(h_doorbell) 2178 IVEC=0xe80 2179 IHSRR=1 2180 IMASK=IRQS_DISABLED 2181 IKVM_REAL=1 2182 IKVM_VIRT=1 2183INT_DEFINE_END(h_doorbell) 2184 2185EXC_REAL_BEGIN(h_doorbell, 0xe80, 0x20) 2186 GEN_INT_ENTRY h_doorbell, virt=0, ool=1 2187EXC_REAL_END(h_doorbell, 0xe80, 0x20) 2188EXC_VIRT_BEGIN(h_doorbell, 0x4e80, 0x20) 2189 GEN_INT_ENTRY h_doorbell, virt=1, ool=1 2190EXC_VIRT_END(h_doorbell, 0x4e80, 0x20) 2191EXC_COMMON_BEGIN(h_doorbell_common) 2192 GEN_COMMON h_doorbell 2193 FINISH_NAP 2194 RUNLATCH_ON 2195 addi r3,r1,STACK_FRAME_OVERHEAD 2196#ifdef CONFIG_PPC_DOORBELL 2197 bl doorbell_exception 2198#else 2199 bl unknown_exception 2200#endif 2201 b interrupt_return 2202 2203 GEN_KVM h_doorbell 2204 2205 2206/** 2207 * Interrupt 0xea0 - Hypervisor Virtualization Interrupt. 2208 * This is an asynchronous interrupt in response to an "external exception". 2209 * Similar to 0x500 but for host only. 2210 */ 2211INT_DEFINE_BEGIN(h_virt_irq) 2212 IVEC=0xea0 2213 IHSRR=1 2214 IMASK=IRQS_DISABLED 2215 IKVM_REAL=1 2216 IKVM_VIRT=1 2217INT_DEFINE_END(h_virt_irq) 2218 2219EXC_REAL_BEGIN(h_virt_irq, 0xea0, 0x20) 2220 GEN_INT_ENTRY h_virt_irq, virt=0, ool=1 2221EXC_REAL_END(h_virt_irq, 0xea0, 0x20) 2222EXC_VIRT_BEGIN(h_virt_irq, 0x4ea0, 0x20) 2223 GEN_INT_ENTRY h_virt_irq, virt=1, ool=1 2224EXC_VIRT_END(h_virt_irq, 0x4ea0, 0x20) 2225EXC_COMMON_BEGIN(h_virt_irq_common) 2226 GEN_COMMON h_virt_irq 2227 FINISH_NAP 2228 RUNLATCH_ON 2229 addi r3,r1,STACK_FRAME_OVERHEAD 2230 bl do_IRQ 2231 b interrupt_return 2232 2233 GEN_KVM h_virt_irq 2234 2235 2236EXC_REAL_NONE(0xec0, 0x20) 2237EXC_VIRT_NONE(0x4ec0, 0x20) 2238EXC_REAL_NONE(0xee0, 0x20) 2239EXC_VIRT_NONE(0x4ee0, 0x20) 2240 2241 2242/* 2243 * Interrupt 0xf00 - Performance Monitor Interrupt (PMI, PMU). 2244 * This is an asynchronous interrupt in response to a PMU exception. 2245 * It is maskable in hardware by clearing MSR[EE], and soft-maskable with 2246 * IRQS_PMI_DISABLED mask (NOTE: NOT local_irq_disable()). 2247 * 2248 * Handling: 2249 * This calls into the perf subsystem. 2250 * 2251 * Like the watchdog soft-nmi, it appears an NMI interrupt to Linux, in that it 2252 * runs under local_irq_disable. However it may be soft-masked in 2253 * powerpc-specific code. 2254 * 2255 * If soft masked, the masked handler will note the pending interrupt for 2256 * replay, and clear MSR[EE] in the interrupted context. 2257 */ 2258INT_DEFINE_BEGIN(performance_monitor) 2259 IVEC=0xf00 2260 IMASK=IRQS_PMI_DISABLED 2261#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 2262 IKVM_REAL=1 2263#endif 2264INT_DEFINE_END(performance_monitor) 2265 2266EXC_REAL_BEGIN(performance_monitor, 0xf00, 0x20) 2267 GEN_INT_ENTRY performance_monitor, virt=0, ool=1 2268EXC_REAL_END(performance_monitor, 0xf00, 0x20) 2269EXC_VIRT_BEGIN(performance_monitor, 0x4f00, 0x20) 2270 GEN_INT_ENTRY performance_monitor, virt=1, ool=1 2271EXC_VIRT_END(performance_monitor, 0x4f00, 0x20) 2272EXC_COMMON_BEGIN(performance_monitor_common) 2273 GEN_COMMON performance_monitor 2274 FINISH_NAP 2275 RUNLATCH_ON 2276 addi r3,r1,STACK_FRAME_OVERHEAD 2277 bl performance_monitor_exception 2278 b interrupt_return 2279 2280 GEN_KVM performance_monitor 2281 2282 2283/** 2284 * Interrupt 0xf20 - Vector Unavailable Interrupt. 2285 * This is a synchronous interrupt in response to 2286 * executing a vector (or altivec) instruction with MSR[VEC]=0. 2287 * Similar to FP unavailable. 2288 */ 2289INT_DEFINE_BEGIN(altivec_unavailable) 2290 IVEC=0xf20 2291 IRECONCILE=0 2292#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 2293 IKVM_REAL=1 2294#endif 2295INT_DEFINE_END(altivec_unavailable) 2296 2297EXC_REAL_BEGIN(altivec_unavailable, 0xf20, 0x20) 2298 GEN_INT_ENTRY altivec_unavailable, virt=0, ool=1 2299EXC_REAL_END(altivec_unavailable, 0xf20, 0x20) 2300EXC_VIRT_BEGIN(altivec_unavailable, 0x4f20, 0x20) 2301 GEN_INT_ENTRY altivec_unavailable, virt=1, ool=1 2302EXC_VIRT_END(altivec_unavailable, 0x4f20, 0x20) 2303EXC_COMMON_BEGIN(altivec_unavailable_common) 2304 GEN_COMMON altivec_unavailable 2305#ifdef CONFIG_ALTIVEC 2306BEGIN_FTR_SECTION 2307 beq 1f 2308#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 2309 BEGIN_FTR_SECTION_NESTED(69) 2310 /* Test if 2 TM state bits are zero. If non-zero (ie. userspace was in 2311 * transaction), go do TM stuff 2312 */ 2313 rldicl. r0, r12, (64-MSR_TS_LG), (64-2) 2314 bne- 2f 2315 END_FTR_SECTION_NESTED(CPU_FTR_TM, CPU_FTR_TM, 69) 2316#endif 2317 bl load_up_altivec 2318 b fast_interrupt_return 2319#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 23202: /* User process was in a transaction */ 2321 RECONCILE_IRQ_STATE(r10, r11) 2322 addi r3,r1,STACK_FRAME_OVERHEAD 2323 bl altivec_unavailable_tm 2324 b interrupt_return 2325#endif 23261: 2327END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 2328#endif 2329 RECONCILE_IRQ_STATE(r10, r11) 2330 addi r3,r1,STACK_FRAME_OVERHEAD 2331 bl altivec_unavailable_exception 2332 b interrupt_return 2333 2334 GEN_KVM altivec_unavailable 2335 2336 2337/** 2338 * Interrupt 0xf40 - VSX Unavailable Interrupt. 2339 * This is a synchronous interrupt in response to 2340 * executing a VSX instruction with MSR[VSX]=0. 2341 * Similar to FP unavailable. 2342 */ 2343INT_DEFINE_BEGIN(vsx_unavailable) 2344 IVEC=0xf40 2345 IRECONCILE=0 2346#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 2347 IKVM_REAL=1 2348#endif 2349INT_DEFINE_END(vsx_unavailable) 2350 2351EXC_REAL_BEGIN(vsx_unavailable, 0xf40, 0x20) 2352 GEN_INT_ENTRY vsx_unavailable, virt=0, ool=1 2353EXC_REAL_END(vsx_unavailable, 0xf40, 0x20) 2354EXC_VIRT_BEGIN(vsx_unavailable, 0x4f40, 0x20) 2355 GEN_INT_ENTRY vsx_unavailable, virt=1, ool=1 2356EXC_VIRT_END(vsx_unavailable, 0x4f40, 0x20) 2357EXC_COMMON_BEGIN(vsx_unavailable_common) 2358 GEN_COMMON vsx_unavailable 2359#ifdef CONFIG_VSX 2360BEGIN_FTR_SECTION 2361 beq 1f 2362#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 2363 BEGIN_FTR_SECTION_NESTED(69) 2364 /* Test if 2 TM state bits are zero. If non-zero (ie. userspace was in 2365 * transaction), go do TM stuff 2366 */ 2367 rldicl. r0, r12, (64-MSR_TS_LG), (64-2) 2368 bne- 2f 2369 END_FTR_SECTION_NESTED(CPU_FTR_TM, CPU_FTR_TM, 69) 2370#endif 2371 b load_up_vsx 2372#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 23732: /* User process was in a transaction */ 2374 RECONCILE_IRQ_STATE(r10, r11) 2375 addi r3,r1,STACK_FRAME_OVERHEAD 2376 bl vsx_unavailable_tm 2377 b interrupt_return 2378#endif 23791: 2380END_FTR_SECTION_IFSET(CPU_FTR_VSX) 2381#endif 2382 RECONCILE_IRQ_STATE(r10, r11) 2383 addi r3,r1,STACK_FRAME_OVERHEAD 2384 bl vsx_unavailable_exception 2385 b interrupt_return 2386 2387 GEN_KVM vsx_unavailable 2388 2389 2390/** 2391 * Interrupt 0xf60 - Facility Unavailable Interrupt. 2392 * This is a synchronous interrupt in response to 2393 * executing an instruction without access to the facility that can be 2394 * resolved by the OS (e.g., FSCR, MSR). 2395 * Similar to FP unavailable. 2396 */ 2397INT_DEFINE_BEGIN(facility_unavailable) 2398 IVEC=0xf60 2399#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 2400 IKVM_REAL=1 2401#endif 2402INT_DEFINE_END(facility_unavailable) 2403 2404EXC_REAL_BEGIN(facility_unavailable, 0xf60, 0x20) 2405 GEN_INT_ENTRY facility_unavailable, virt=0, ool=1 2406EXC_REAL_END(facility_unavailable, 0xf60, 0x20) 2407EXC_VIRT_BEGIN(facility_unavailable, 0x4f60, 0x20) 2408 GEN_INT_ENTRY facility_unavailable, virt=1, ool=1 2409EXC_VIRT_END(facility_unavailable, 0x4f60, 0x20) 2410EXC_COMMON_BEGIN(facility_unavailable_common) 2411 GEN_COMMON facility_unavailable 2412 addi r3,r1,STACK_FRAME_OVERHEAD 2413 bl facility_unavailable_exception 2414 REST_NVGPRS(r1) /* instruction emulation may change GPRs */ 2415 b interrupt_return 2416 2417 GEN_KVM facility_unavailable 2418 2419 2420/** 2421 * Interrupt 0xf60 - Hypervisor Facility Unavailable Interrupt. 2422 * This is a synchronous interrupt in response to 2423 * executing an instruction without access to the facility that can only 2424 * be resolved in HV mode (e.g., HFSCR). 2425 * Similar to FP unavailable. 2426 */ 2427INT_DEFINE_BEGIN(h_facility_unavailable) 2428 IVEC=0xf80 2429 IHSRR=1 2430 IKVM_REAL=1 2431 IKVM_VIRT=1 2432INT_DEFINE_END(h_facility_unavailable) 2433 2434EXC_REAL_BEGIN(h_facility_unavailable, 0xf80, 0x20) 2435 GEN_INT_ENTRY h_facility_unavailable, virt=0, ool=1 2436EXC_REAL_END(h_facility_unavailable, 0xf80, 0x20) 2437EXC_VIRT_BEGIN(h_facility_unavailable, 0x4f80, 0x20) 2438 GEN_INT_ENTRY h_facility_unavailable, virt=1, ool=1 2439EXC_VIRT_END(h_facility_unavailable, 0x4f80, 0x20) 2440EXC_COMMON_BEGIN(h_facility_unavailable_common) 2441 GEN_COMMON h_facility_unavailable 2442 addi r3,r1,STACK_FRAME_OVERHEAD 2443 bl facility_unavailable_exception 2444 REST_NVGPRS(r1) /* XXX Shouldn't be necessary in practice */ 2445 b interrupt_return 2446 2447 GEN_KVM h_facility_unavailable 2448 2449 2450EXC_REAL_NONE(0xfa0, 0x20) 2451EXC_VIRT_NONE(0x4fa0, 0x20) 2452EXC_REAL_NONE(0xfc0, 0x20) 2453EXC_VIRT_NONE(0x4fc0, 0x20) 2454EXC_REAL_NONE(0xfe0, 0x20) 2455EXC_VIRT_NONE(0x4fe0, 0x20) 2456 2457EXC_REAL_NONE(0x1000, 0x100) 2458EXC_VIRT_NONE(0x5000, 0x100) 2459EXC_REAL_NONE(0x1100, 0x100) 2460EXC_VIRT_NONE(0x5100, 0x100) 2461 2462#ifdef CONFIG_CBE_RAS 2463INT_DEFINE_BEGIN(cbe_system_error) 2464 IVEC=0x1200 2465 IHSRR=1 2466 IKVM_SKIP=1 2467 IKVM_REAL=1 2468INT_DEFINE_END(cbe_system_error) 2469 2470EXC_REAL_BEGIN(cbe_system_error, 0x1200, 0x100) 2471 GEN_INT_ENTRY cbe_system_error, virt=0 2472EXC_REAL_END(cbe_system_error, 0x1200, 0x100) 2473EXC_VIRT_NONE(0x5200, 0x100) 2474EXC_COMMON_BEGIN(cbe_system_error_common) 2475 GEN_COMMON cbe_system_error 2476 addi r3,r1,STACK_FRAME_OVERHEAD 2477 bl cbe_system_error_exception 2478 b interrupt_return 2479 2480 GEN_KVM cbe_system_error 2481 2482#else /* CONFIG_CBE_RAS */ 2483EXC_REAL_NONE(0x1200, 0x100) 2484EXC_VIRT_NONE(0x5200, 0x100) 2485#endif 2486 2487 2488INT_DEFINE_BEGIN(instruction_breakpoint) 2489 IVEC=0x1300 2490#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 2491 IKVM_SKIP=1 2492 IKVM_REAL=1 2493#endif 2494INT_DEFINE_END(instruction_breakpoint) 2495 2496EXC_REAL_BEGIN(instruction_breakpoint, 0x1300, 0x100) 2497 GEN_INT_ENTRY instruction_breakpoint, virt=0 2498EXC_REAL_END(instruction_breakpoint, 0x1300, 0x100) 2499EXC_VIRT_BEGIN(instruction_breakpoint, 0x5300, 0x100) 2500 GEN_INT_ENTRY instruction_breakpoint, virt=1 2501EXC_VIRT_END(instruction_breakpoint, 0x5300, 0x100) 2502EXC_COMMON_BEGIN(instruction_breakpoint_common) 2503 GEN_COMMON instruction_breakpoint 2504 addi r3,r1,STACK_FRAME_OVERHEAD 2505 bl instruction_breakpoint_exception 2506 b interrupt_return 2507 2508 GEN_KVM instruction_breakpoint 2509 2510 2511EXC_REAL_NONE(0x1400, 0x100) 2512EXC_VIRT_NONE(0x5400, 0x100) 2513 2514/** 2515 * Interrupt 0x1500 - Soft Patch Interrupt 2516 * 2517 * Handling: 2518 * This is an implementation specific interrupt which can be used for a 2519 * range of exceptions. 2520 * 2521 * This interrupt handler is unique in that it runs the denormal assist 2522 * code even for guests (and even in guest context) without going to KVM, 2523 * for speed. POWER9 does not raise denorm exceptions, so this special case 2524 * could be phased out in future to reduce special cases. 2525 */ 2526INT_DEFINE_BEGIN(denorm_exception) 2527 IVEC=0x1500 2528 IHSRR=1 2529 IBRANCH_COMMON=0 2530 IKVM_REAL=1 2531INT_DEFINE_END(denorm_exception) 2532 2533EXC_REAL_BEGIN(denorm_exception, 0x1500, 0x100) 2534 GEN_INT_ENTRY denorm_exception, virt=0 2535#ifdef CONFIG_PPC_DENORMALISATION 2536 andis. r10,r12,(HSRR1_DENORM)@h /* denorm? */ 2537 bne+ denorm_assist 2538#endif 2539 GEN_BRANCH_TO_COMMON denorm_exception, virt=0 2540EXC_REAL_END(denorm_exception, 0x1500, 0x100) 2541#ifdef CONFIG_PPC_DENORMALISATION 2542EXC_VIRT_BEGIN(denorm_exception, 0x5500, 0x100) 2543 GEN_INT_ENTRY denorm_exception, virt=1 2544 andis. r10,r12,(HSRR1_DENORM)@h /* denorm? */ 2545 bne+ denorm_assist 2546 GEN_BRANCH_TO_COMMON denorm_exception, virt=1 2547EXC_VIRT_END(denorm_exception, 0x5500, 0x100) 2548#else 2549EXC_VIRT_NONE(0x5500, 0x100) 2550#endif 2551 2552#ifdef CONFIG_PPC_DENORMALISATION 2553TRAMP_REAL_BEGIN(denorm_assist) 2554BEGIN_FTR_SECTION 2555/* 2556 * To denormalise we need to move a copy of the register to itself. 2557 * For POWER6 do that here for all FP regs. 2558 */ 2559 mfmsr r10 2560 ori r10,r10,(MSR_FP|MSR_FE0|MSR_FE1) 2561 xori r10,r10,(MSR_FE0|MSR_FE1) 2562 mtmsrd r10 2563 sync 2564 2565 .Lreg=0 2566 .rept 32 2567 fmr .Lreg,.Lreg 2568 .Lreg=.Lreg+1 2569 .endr 2570 2571FTR_SECTION_ELSE 2572/* 2573 * To denormalise we need to move a copy of the register to itself. 2574 * For POWER7 do that here for the first 32 VSX registers only. 2575 */ 2576 mfmsr r10 2577 oris r10,r10,MSR_VSX@h 2578 mtmsrd r10 2579 sync 2580 2581 .Lreg=0 2582 .rept 32 2583 XVCPSGNDP(.Lreg,.Lreg,.Lreg) 2584 .Lreg=.Lreg+1 2585 .endr 2586 2587ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_206) 2588 2589BEGIN_FTR_SECTION 2590 b denorm_done 2591END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) 2592/* 2593 * To denormalise we need to move a copy of the register to itself. 2594 * For POWER8 we need to do that for all 64 VSX registers 2595 */ 2596 .Lreg=32 2597 .rept 32 2598 XVCPSGNDP(.Lreg,.Lreg,.Lreg) 2599 .Lreg=.Lreg+1 2600 .endr 2601 2602denorm_done: 2603 mfspr r11,SPRN_HSRR0 2604 subi r11,r11,4 2605 mtspr SPRN_HSRR0,r11 2606 mtcrf 0x80,r9 2607 ld r9,PACA_EXGEN+EX_R9(r13) 2608BEGIN_FTR_SECTION 2609 ld r10,PACA_EXGEN+EX_PPR(r13) 2610 mtspr SPRN_PPR,r10 2611END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 2612BEGIN_FTR_SECTION 2613 ld r10,PACA_EXGEN+EX_CFAR(r13) 2614 mtspr SPRN_CFAR,r10 2615END_FTR_SECTION_IFSET(CPU_FTR_CFAR) 2616 ld r10,PACA_EXGEN+EX_R10(r13) 2617 ld r11,PACA_EXGEN+EX_R11(r13) 2618 ld r12,PACA_EXGEN+EX_R12(r13) 2619 ld r13,PACA_EXGEN+EX_R13(r13) 2620 HRFI_TO_UNKNOWN 2621 b . 2622#endif 2623 2624EXC_COMMON_BEGIN(denorm_exception_common) 2625 GEN_COMMON denorm_exception 2626 addi r3,r1,STACK_FRAME_OVERHEAD 2627 bl unknown_exception 2628 b interrupt_return 2629 2630 GEN_KVM denorm_exception 2631 2632 2633#ifdef CONFIG_CBE_RAS 2634INT_DEFINE_BEGIN(cbe_maintenance) 2635 IVEC=0x1600 2636 IHSRR=1 2637 IKVM_SKIP=1 2638 IKVM_REAL=1 2639INT_DEFINE_END(cbe_maintenance) 2640 2641EXC_REAL_BEGIN(cbe_maintenance, 0x1600, 0x100) 2642 GEN_INT_ENTRY cbe_maintenance, virt=0 2643EXC_REAL_END(cbe_maintenance, 0x1600, 0x100) 2644EXC_VIRT_NONE(0x5600, 0x100) 2645EXC_COMMON_BEGIN(cbe_maintenance_common) 2646 GEN_COMMON cbe_maintenance 2647 addi r3,r1,STACK_FRAME_OVERHEAD 2648 bl cbe_maintenance_exception 2649 b interrupt_return 2650 2651 GEN_KVM cbe_maintenance 2652 2653#else /* CONFIG_CBE_RAS */ 2654EXC_REAL_NONE(0x1600, 0x100) 2655EXC_VIRT_NONE(0x5600, 0x100) 2656#endif 2657 2658 2659INT_DEFINE_BEGIN(altivec_assist) 2660 IVEC=0x1700 2661#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 2662 IKVM_REAL=1 2663#endif 2664INT_DEFINE_END(altivec_assist) 2665 2666EXC_REAL_BEGIN(altivec_assist, 0x1700, 0x100) 2667 GEN_INT_ENTRY altivec_assist, virt=0 2668EXC_REAL_END(altivec_assist, 0x1700, 0x100) 2669EXC_VIRT_BEGIN(altivec_assist, 0x5700, 0x100) 2670 GEN_INT_ENTRY altivec_assist, virt=1 2671EXC_VIRT_END(altivec_assist, 0x5700, 0x100) 2672EXC_COMMON_BEGIN(altivec_assist_common) 2673 GEN_COMMON altivec_assist 2674 addi r3,r1,STACK_FRAME_OVERHEAD 2675#ifdef CONFIG_ALTIVEC 2676 bl altivec_assist_exception 2677 REST_NVGPRS(r1) /* instruction emulation may change GPRs */ 2678#else 2679 bl unknown_exception 2680#endif 2681 b interrupt_return 2682 2683 GEN_KVM altivec_assist 2684 2685 2686#ifdef CONFIG_CBE_RAS 2687INT_DEFINE_BEGIN(cbe_thermal) 2688 IVEC=0x1800 2689 IHSRR=1 2690 IKVM_SKIP=1 2691 IKVM_REAL=1 2692INT_DEFINE_END(cbe_thermal) 2693 2694EXC_REAL_BEGIN(cbe_thermal, 0x1800, 0x100) 2695 GEN_INT_ENTRY cbe_thermal, virt=0 2696EXC_REAL_END(cbe_thermal, 0x1800, 0x100) 2697EXC_VIRT_NONE(0x5800, 0x100) 2698EXC_COMMON_BEGIN(cbe_thermal_common) 2699 GEN_COMMON cbe_thermal 2700 addi r3,r1,STACK_FRAME_OVERHEAD 2701 bl cbe_thermal_exception 2702 b interrupt_return 2703 2704 GEN_KVM cbe_thermal 2705 2706#else /* CONFIG_CBE_RAS */ 2707EXC_REAL_NONE(0x1800, 0x100) 2708EXC_VIRT_NONE(0x5800, 0x100) 2709#endif 2710 2711 2712#ifdef CONFIG_PPC_WATCHDOG 2713 2714INT_DEFINE_BEGIN(soft_nmi) 2715 IVEC=0x900 2716 ISTACK=0 2717 IRECONCILE=0 /* Soft-NMI may fire under local_irq_disable */ 2718INT_DEFINE_END(soft_nmi) 2719 2720/* 2721 * Branch to soft_nmi_interrupt using the emergency stack. The emergency 2722 * stack is one that is usable by maskable interrupts so long as MSR_EE 2723 * remains off. It is used for recovery when something has corrupted the 2724 * normal kernel stack, for example. The "soft NMI" must not use the process 2725 * stack because we want irq disabled sections to avoid touching the stack 2726 * at all (other than PMU interrupts), so use the emergency stack for this, 2727 * and run it entirely with interrupts hard disabled. 2728 */ 2729EXC_COMMON_BEGIN(soft_nmi_common) 2730 mfspr r11,SPRN_SRR0 2731 mr r10,r1 2732 ld r1,PACAEMERGSP(r13) 2733 subi r1,r1,INT_FRAME_SIZE 2734 __GEN_COMMON_BODY soft_nmi 2735 2736 /* 2737 * Set IRQS_ALL_DISABLED and save PACAIRQHAPPENED (see 2738 * system_reset_common) 2739 */ 2740 li r10,IRQS_ALL_DISABLED 2741 stb r10,PACAIRQSOFTMASK(r13) 2742 lbz r10,PACAIRQHAPPENED(r13) 2743 std r10,_DAR(r1) 2744 ori r10,r10,PACA_IRQ_HARD_DIS 2745 stb r10,PACAIRQHAPPENED(r13) 2746 2747 addi r3,r1,STACK_FRAME_OVERHEAD 2748 bl soft_nmi_interrupt 2749 2750 /* Clear MSR_RI before setting SRR0 and SRR1. */ 2751 li r9,0 2752 mtmsrd r9,1 2753 2754 /* 2755 * Restore soft mask settings. 2756 */ 2757 ld r10,_DAR(r1) 2758 stb r10,PACAIRQHAPPENED(r13) 2759 ld r10,SOFTE(r1) 2760 stb r10,PACAIRQSOFTMASK(r13) 2761 2762 kuap_restore_amr r10 2763 EXCEPTION_RESTORE_REGS hsrr=0 2764 RFI_TO_KERNEL 2765 2766#endif /* CONFIG_PPC_WATCHDOG */ 2767 2768/* 2769 * An interrupt came in while soft-disabled. We set paca->irq_happened, then: 2770 * - If it was a decrementer interrupt, we bump the dec to max and and return. 2771 * - If it was a doorbell we return immediately since doorbells are edge 2772 * triggered and won't automatically refire. 2773 * - If it was a HMI we return immediately since we handled it in realmode 2774 * and it won't refire. 2775 * - Else it is one of PACA_IRQ_MUST_HARD_MASK, so hard disable and return. 2776 * This is called with r10 containing the value to OR to the paca field. 2777 */ 2778.macro MASKED_INTERRUPT hsrr=0 2779 .if \hsrr 2780masked_Hinterrupt: 2781 .else 2782masked_interrupt: 2783 .endif 2784 lbz r11,PACAIRQHAPPENED(r13) 2785 or r11,r11,r10 2786 stb r11,PACAIRQHAPPENED(r13) 2787 cmpwi r10,PACA_IRQ_DEC 2788 bne 1f 2789 lis r10,0x7fff 2790 ori r10,r10,0xffff 2791 mtspr SPRN_DEC,r10 2792#ifdef CONFIG_PPC_WATCHDOG 2793 b soft_nmi_common 2794#else 2795 b 2f 2796#endif 27971: andi. r10,r10,PACA_IRQ_MUST_HARD_MASK 2798 beq 2f 2799 xori r12,r12,MSR_EE /* clear MSR_EE */ 2800 .if \hsrr 2801 mtspr SPRN_HSRR1,r12 2802 .else 2803 mtspr SPRN_SRR1,r12 2804 .endif 2805 ori r11,r11,PACA_IRQ_HARD_DIS 2806 stb r11,PACAIRQHAPPENED(r13) 28072: /* done */ 2808 ld r10,PACA_EXGEN+EX_CTR(r13) 2809 mtctr r10 2810 mtcrf 0x80,r9 2811 std r1,PACAR1(r13) 2812 ld r9,PACA_EXGEN+EX_R9(r13) 2813 ld r10,PACA_EXGEN+EX_R10(r13) 2814 ld r11,PACA_EXGEN+EX_R11(r13) 2815 ld r12,PACA_EXGEN+EX_R12(r13) 2816 /* returns to kernel where r13 must be set up, so don't restore it */ 2817 .if \hsrr 2818 HRFI_TO_KERNEL 2819 .else 2820 RFI_TO_KERNEL 2821 .endif 2822 b . 2823.endm 2824 2825TRAMP_REAL_BEGIN(stf_barrier_fallback) 2826 std r9,PACA_EXRFI+EX_R9(r13) 2827 std r10,PACA_EXRFI+EX_R10(r13) 2828 sync 2829 ld r9,PACA_EXRFI+EX_R9(r13) 2830 ld r10,PACA_EXRFI+EX_R10(r13) 2831 ori 31,31,0 2832 .rept 14 2833 b 1f 28341: 2835 .endr 2836 blr 2837 2838TRAMP_REAL_BEGIN(rfi_flush_fallback) 2839 SET_SCRATCH0(r13); 2840 GET_PACA(r13); 2841 std r1,PACA_EXRFI+EX_R12(r13) 2842 ld r1,PACAKSAVE(r13) 2843 std r9,PACA_EXRFI+EX_R9(r13) 2844 std r10,PACA_EXRFI+EX_R10(r13) 2845 std r11,PACA_EXRFI+EX_R11(r13) 2846 mfctr r9 2847 ld r10,PACA_RFI_FLUSH_FALLBACK_AREA(r13) 2848 ld r11,PACA_L1D_FLUSH_SIZE(r13) 2849 srdi r11,r11,(7 + 3) /* 128 byte lines, unrolled 8x */ 2850 mtctr r11 2851 DCBT_BOOK3S_STOP_ALL_STREAM_IDS(r11) /* Stop prefetch streams */ 2852 2853 /* order ld/st prior to dcbt stop all streams with flushing */ 2854 sync 2855 2856 /* 2857 * The load adresses are at staggered offsets within cachelines, 2858 * which suits some pipelines better (on others it should not 2859 * hurt). 2860 */ 28611: 2862 ld r11,(0x80 + 8)*0(r10) 2863 ld r11,(0x80 + 8)*1(r10) 2864 ld r11,(0x80 + 8)*2(r10) 2865 ld r11,(0x80 + 8)*3(r10) 2866 ld r11,(0x80 + 8)*4(r10) 2867 ld r11,(0x80 + 8)*5(r10) 2868 ld r11,(0x80 + 8)*6(r10) 2869 ld r11,(0x80 + 8)*7(r10) 2870 addi r10,r10,0x80*8 2871 bdnz 1b 2872 2873 mtctr r9 2874 ld r9,PACA_EXRFI+EX_R9(r13) 2875 ld r10,PACA_EXRFI+EX_R10(r13) 2876 ld r11,PACA_EXRFI+EX_R11(r13) 2877 ld r1,PACA_EXRFI+EX_R12(r13) 2878 GET_SCRATCH0(r13); 2879 rfid 2880 2881TRAMP_REAL_BEGIN(hrfi_flush_fallback) 2882 SET_SCRATCH0(r13); 2883 GET_PACA(r13); 2884 std r1,PACA_EXRFI+EX_R12(r13) 2885 ld r1,PACAKSAVE(r13) 2886 std r9,PACA_EXRFI+EX_R9(r13) 2887 std r10,PACA_EXRFI+EX_R10(r13) 2888 std r11,PACA_EXRFI+EX_R11(r13) 2889 mfctr r9 2890 ld r10,PACA_RFI_FLUSH_FALLBACK_AREA(r13) 2891 ld r11,PACA_L1D_FLUSH_SIZE(r13) 2892 srdi r11,r11,(7 + 3) /* 128 byte lines, unrolled 8x */ 2893 mtctr r11 2894 DCBT_BOOK3S_STOP_ALL_STREAM_IDS(r11) /* Stop prefetch streams */ 2895 2896 /* order ld/st prior to dcbt stop all streams with flushing */ 2897 sync 2898 2899 /* 2900 * The load adresses are at staggered offsets within cachelines, 2901 * which suits some pipelines better (on others it should not 2902 * hurt). 2903 */ 29041: 2905 ld r11,(0x80 + 8)*0(r10) 2906 ld r11,(0x80 + 8)*1(r10) 2907 ld r11,(0x80 + 8)*2(r10) 2908 ld r11,(0x80 + 8)*3(r10) 2909 ld r11,(0x80 + 8)*4(r10) 2910 ld r11,(0x80 + 8)*5(r10) 2911 ld r11,(0x80 + 8)*6(r10) 2912 ld r11,(0x80 + 8)*7(r10) 2913 addi r10,r10,0x80*8 2914 bdnz 1b 2915 2916 mtctr r9 2917 ld r9,PACA_EXRFI+EX_R9(r13) 2918 ld r10,PACA_EXRFI+EX_R10(r13) 2919 ld r11,PACA_EXRFI+EX_R11(r13) 2920 ld r1,PACA_EXRFI+EX_R12(r13) 2921 GET_SCRATCH0(r13); 2922 hrfid 2923 2924USE_TEXT_SECTION() 2925 MASKED_INTERRUPT 2926 MASKED_INTERRUPT hsrr=1 2927 2928#ifdef CONFIG_KVM_BOOK3S_64_HANDLER 2929kvmppc_skip_interrupt: 2930 /* 2931 * Here all GPRs are unchanged from when the interrupt happened 2932 * except for r13, which is saved in SPRG_SCRATCH0. 2933 */ 2934 mfspr r13, SPRN_SRR0 2935 addi r13, r13, 4 2936 mtspr SPRN_SRR0, r13 2937 GET_SCRATCH0(r13) 2938 RFI_TO_KERNEL 2939 b . 2940 2941kvmppc_skip_Hinterrupt: 2942 /* 2943 * Here all GPRs are unchanged from when the interrupt happened 2944 * except for r13, which is saved in SPRG_SCRATCH0. 2945 */ 2946 mfspr r13, SPRN_HSRR0 2947 addi r13, r13, 4 2948 mtspr SPRN_HSRR0, r13 2949 GET_SCRATCH0(r13) 2950 HRFI_TO_KERNEL 2951 b . 2952#endif 2953 2954 /* 2955 * Relocation-on interrupts: A subset of the interrupts can be delivered 2956 * with IR=1/DR=1, if AIL==2 and MSR.HV won't be changed by delivering 2957 * it. Addresses are the same as the original interrupt addresses, but 2958 * offset by 0xc000000000004000. 2959 * It's impossible to receive interrupts below 0x300 via this mechanism. 2960 * KVM: None of these traps are from the guest ; anything that escalated 2961 * to HV=1 from HV=0 is delivered via real mode handlers. 2962 */ 2963 2964 /* 2965 * This uses the standard macro, since the original 0x300 vector 2966 * only has extra guff for STAB-based processors -- which never 2967 * come here. 2968 */ 2969 2970EXC_COMMON_BEGIN(ppc64_runlatch_on_trampoline) 2971 b __ppc64_runlatch_on 2972 2973USE_FIXED_SECTION(virt_trampolines) 2974 /* 2975 * The __end_interrupts marker must be past the out-of-line (OOL) 2976 * handlers, so that they are copied to real address 0x100 when running 2977 * a relocatable kernel. This ensures they can be reached from the short 2978 * trampoline handlers (like 0x4f00, 0x4f20, etc.) which branch 2979 * directly, without using LOAD_HANDLER(). 2980 */ 2981 .align 7 2982 .globl __end_interrupts 2983__end_interrupts: 2984DEFINE_FIXED_SYMBOL(__end_interrupts) 2985 2986#ifdef CONFIG_PPC_970_NAP 2987 /* 2988 * Called by exception entry code if _TLF_NAPPING was set, this clears 2989 * the NAPPING flag, and redirects the exception exit to 2990 * power4_fixup_nap_return. 2991 */ 2992 .globl power4_fixup_nap 2993EXC_COMMON_BEGIN(power4_fixup_nap) 2994 andc r9,r9,r10 2995 std r9,TI_LOCAL_FLAGS(r11) 2996 LOAD_REG_ADDR(r10, power4_idle_nap_return) 2997 std r10,_NIP(r1) 2998 blr 2999 3000power4_idle_nap_return: 3001 blr 3002#endif 3003 3004CLOSE_FIXED_SECTION(real_vectors); 3005CLOSE_FIXED_SECTION(real_trampolines); 3006CLOSE_FIXED_SECTION(virt_vectors); 3007CLOSE_FIXED_SECTION(virt_trampolines); 3008 3009USE_TEXT_SECTION() 3010 3011/* MSR[RI] should be clear because this uses SRR[01] */ 3012enable_machine_check: 3013 mflr r0 3014 bcl 20,31,$+4 30150: mflr r3 3016 addi r3,r3,(1f - 0b) 3017 mtspr SPRN_SRR0,r3 3018 mfmsr r3 3019 ori r3,r3,MSR_ME 3020 mtspr SPRN_SRR1,r3 3021 RFI_TO_KERNEL 30221: mtlr r0 3023 blr 3024 3025/* MSR[RI] should be clear because this uses SRR[01] */ 3026disable_machine_check: 3027 mflr r0 3028 bcl 20,31,$+4 30290: mflr r3 3030 addi r3,r3,(1f - 0b) 3031 mtspr SPRN_SRR0,r3 3032 mfmsr r3 3033 li r4,MSR_ME 3034 andc r3,r3,r4 3035 mtspr SPRN_SRR1,r3 3036 RFI_TO_KERNEL 30371: mtlr r0 3038 blr 3039 3040/* 3041 * Hash table stuff 3042 */ 3043 .balign IFETCH_ALIGN_BYTES 3044do_hash_page: 3045#ifdef CONFIG_PPC_BOOK3S_64 3046 lis r0,(DSISR_BAD_FAULT_64S | DSISR_DABRMATCH | DSISR_KEYFAULT)@h 3047 ori r0,r0,DSISR_BAD_FAULT_64S@l 3048 and. r0,r5,r0 /* weird error? */ 3049 bne- handle_page_fault /* if not, try to insert a HPTE */ 3050 ld r11, PACA_THREAD_INFO(r13) 3051 lwz r0,TI_PREEMPT(r11) /* If we're in an "NMI" */ 3052 andis. r0,r0,NMI_MASK@h /* (i.e. an irq when soft-disabled) */ 3053 bne 77f /* then don't call hash_page now */ 3054 3055 /* 3056 * r3 contains the trap number 3057 * r4 contains the faulting address 3058 * r5 contains dsisr 3059 * r6 msr 3060 * 3061 * at return r3 = 0 for success, 1 for page fault, negative for error 3062 */ 3063 bl __hash_page /* build HPTE if possible */ 3064 cmpdi r3,0 /* see if __hash_page succeeded */ 3065 3066 /* Success */ 3067 beq interrupt_return /* Return from exception on success */ 3068 3069 /* Error */ 3070 blt- 13f 3071 3072 /* Reload DAR/DSISR into r4/r5 for the DABR check below */ 3073 ld r4,_DAR(r1) 3074 ld r5,_DSISR(r1) 3075#endif /* CONFIG_PPC_BOOK3S_64 */ 3076 3077/* Here we have a page fault that hash_page can't handle. */ 3078handle_page_fault: 307911: andis. r0,r5,DSISR_DABRMATCH@h 3080 bne- handle_dabr_fault 3081 addi r3,r1,STACK_FRAME_OVERHEAD 3082 bl do_page_fault 3083 cmpdi r3,0 3084 beq+ interrupt_return 3085 mr r5,r3 3086 addi r3,r1,STACK_FRAME_OVERHEAD 3087 ld r4,_DAR(r1) 3088 bl bad_page_fault 3089 b interrupt_return 3090 3091/* We have a data breakpoint exception - handle it */ 3092handle_dabr_fault: 3093 ld r4,_DAR(r1) 3094 ld r5,_DSISR(r1) 3095 addi r3,r1,STACK_FRAME_OVERHEAD 3096 bl do_break 3097 /* 3098 * do_break() may have changed the NV GPRS while handling a breakpoint. 3099 * If so, we need to restore them with their updated values. 3100 */ 3101 REST_NVGPRS(r1) 3102 b interrupt_return 3103 3104 3105#ifdef CONFIG_PPC_BOOK3S_64 3106/* We have a page fault that hash_page could handle but HV refused 3107 * the PTE insertion 3108 */ 310913: mr r5,r3 3110 addi r3,r1,STACK_FRAME_OVERHEAD 3111 ld r4,_DAR(r1) 3112 bl low_hash_fault 3113 b interrupt_return 3114#endif 3115 3116/* 3117 * We come here as a result of a DSI at a point where we don't want 3118 * to call hash_page, such as when we are accessing memory (possibly 3119 * user memory) inside a PMU interrupt that occurred while interrupts 3120 * were soft-disabled. We want to invoke the exception handler for 3121 * the access, or panic if there isn't a handler. 3122 */ 312377: addi r3,r1,STACK_FRAME_OVERHEAD 3124 li r5,SIGSEGV 3125 bl bad_page_fault 3126 b interrupt_return 3127