1/* SPDX-License-Identifier: GPL-2.0 */ 2/* 3 * This file contains the 64-bit "server" PowerPC variant 4 * of the low level exception handling including exception 5 * vectors, exception return, part of the slb and stab 6 * handling and other fixed offset specific things. 7 * 8 * This file is meant to be #included from head_64.S due to 9 * position dependent assembly. 10 * 11 * Most of this originates from head_64.S and thus has the same 12 * copyright history. 13 * 14 */ 15 16#include <asm/hw_irq.h> 17#include <asm/exception-64s.h> 18#include <asm/ptrace.h> 19#include <asm/cpuidle.h> 20#include <asm/head-64.h> 21#include <asm/feature-fixups.h> 22#include <asm/kup.h> 23 24/* PACA save area offsets (exgen, exmc, etc) */ 25#define EX_R9 0 26#define EX_R10 8 27#define EX_R11 16 28#define EX_R12 24 29#define EX_R13 32 30#define EX_DAR 40 31#define EX_DSISR 48 32#define EX_CCR 52 33#define EX_CFAR 56 34#define EX_PPR 64 35#define EX_CTR 72 36.if EX_SIZE != 10 37 .error "EX_SIZE is wrong" 38.endif 39 40/* 41 * Following are fixed section helper macros. 42 * 43 * EXC_REAL_BEGIN/END - real, unrelocated exception vectors 44 * EXC_VIRT_BEGIN/END - virt (AIL), unrelocated exception vectors 45 * TRAMP_REAL_BEGIN - real, unrelocated helpers (virt may call these) 46 * TRAMP_VIRT_BEGIN - virt, unreloc helpers (in practice, real can use) 47 * EXC_COMMON - After switching to virtual, relocated mode. 48 */ 49 50#define EXC_REAL_BEGIN(name, start, size) \ 51 FIXED_SECTION_ENTRY_BEGIN_LOCATION(real_vectors, exc_real_##start##_##name, start, size) 52 53#define EXC_REAL_END(name, start, size) \ 54 FIXED_SECTION_ENTRY_END_LOCATION(real_vectors, exc_real_##start##_##name, start, size) 55 56#define EXC_VIRT_BEGIN(name, start, size) \ 57 FIXED_SECTION_ENTRY_BEGIN_LOCATION(virt_vectors, exc_virt_##start##_##name, start, size) 58 59#define EXC_VIRT_END(name, start, size) \ 60 FIXED_SECTION_ENTRY_END_LOCATION(virt_vectors, exc_virt_##start##_##name, start, size) 61 62#define EXC_COMMON_BEGIN(name) \ 63 USE_TEXT_SECTION(); \ 64 .balign IFETCH_ALIGN_BYTES; \ 65 .global name; \ 66 _ASM_NOKPROBE_SYMBOL(name); \ 67 DEFINE_FIXED_SYMBOL(name); \ 68name: 69 70#define TRAMP_REAL_BEGIN(name) \ 71 FIXED_SECTION_ENTRY_BEGIN(real_trampolines, name) 72 73#define TRAMP_VIRT_BEGIN(name) \ 74 FIXED_SECTION_ENTRY_BEGIN(virt_trampolines, name) 75 76#define EXC_REAL_NONE(start, size) \ 77 FIXED_SECTION_ENTRY_BEGIN_LOCATION(real_vectors, exc_real_##start##_##unused, start, size); \ 78 FIXED_SECTION_ENTRY_END_LOCATION(real_vectors, exc_real_##start##_##unused, start, size) 79 80#define EXC_VIRT_NONE(start, size) \ 81 FIXED_SECTION_ENTRY_BEGIN_LOCATION(virt_vectors, exc_virt_##start##_##unused, start, size); \ 82 FIXED_SECTION_ENTRY_END_LOCATION(virt_vectors, exc_virt_##start##_##unused, start, size) 83 84/* 85 * We're short on space and time in the exception prolog, so we can't 86 * use the normal LOAD_REG_IMMEDIATE macro to load the address of label. 87 * Instead we get the base of the kernel from paca->kernelbase and or in the low 88 * part of label. This requires that the label be within 64KB of kernelbase, and 89 * that kernelbase be 64K aligned. 90 */ 91#define LOAD_HANDLER(reg, label) \ 92 ld reg,PACAKBASE(r13); /* get high part of &label */ \ 93 ori reg,reg,FIXED_SYMBOL_ABS_ADDR(label) 94 95#define __LOAD_HANDLER(reg, label) \ 96 ld reg,PACAKBASE(r13); \ 97 ori reg,reg,(ABS_ADDR(label))@l 98 99/* 100 * Branches from unrelocated code (e.g., interrupts) to labels outside 101 * head-y require >64K offsets. 102 */ 103#define __LOAD_FAR_HANDLER(reg, label) \ 104 ld reg,PACAKBASE(r13); \ 105 ori reg,reg,(ABS_ADDR(label))@l; \ 106 addis reg,reg,(ABS_ADDR(label))@h 107 108/* 109 * Branch to label using its 0xC000 address. This results in instruction 110 * address suitable for MSR[IR]=0 or 1, which allows relocation to be turned 111 * on using mtmsr rather than rfid. 112 * 113 * This could set the 0xc bits for !RELOCATABLE as an immediate, rather than 114 * load KBASE for a slight optimisation. 115 */ 116#define BRANCH_TO_C000(reg, label) \ 117 __LOAD_FAR_HANDLER(reg, label); \ 118 mtctr reg; \ 119 bctr 120 121/* 122 * Interrupt code generation macros 123 */ 124#define IVEC .L_IVEC_\name\() /* Interrupt vector address */ 125#define IHSRR .L_IHSRR_\name\() /* Sets SRR or HSRR registers */ 126#define IHSRR_IF_HVMODE .L_IHSRR_IF_HVMODE_\name\() /* HSRR if HV else SRR */ 127#define IAREA .L_IAREA_\name\() /* PACA save area */ 128#define IVIRT .L_IVIRT_\name\() /* Has virt mode entry point */ 129#define IISIDE .L_IISIDE_\name\() /* Uses SRR0/1 not DAR/DSISR */ 130#define IDAR .L_IDAR_\name\() /* Uses DAR (or SRR0) */ 131#define IDSISR .L_IDSISR_\name\() /* Uses DSISR (or SRR1) */ 132#define ISET_RI .L_ISET_RI_\name\() /* Run common code w/ MSR[RI]=1 */ 133#define IBRANCH_TO_COMMON .L_IBRANCH_TO_COMMON_\name\() /* ENTRY branch to common */ 134#define IREALMODE_COMMON .L_IREALMODE_COMMON_\name\() /* Common runs in realmode */ 135#define IMASK .L_IMASK_\name\() /* IRQ soft-mask bit */ 136#define IKVM_SKIP .L_IKVM_SKIP_\name\() /* Generate KVM skip handler */ 137#define IKVM_REAL .L_IKVM_REAL_\name\() /* Real entry tests KVM */ 138#define __IKVM_REAL(name) .L_IKVM_REAL_ ## name 139#define IKVM_VIRT .L_IKVM_VIRT_\name\() /* Virt entry tests KVM */ 140#define ISTACK .L_ISTACK_\name\() /* Set regular kernel stack */ 141#define __ISTACK(name) .L_ISTACK_ ## name 142#define IKUAP .L_IKUAP_\name\() /* Do KUAP lock */ 143 144#define INT_DEFINE_BEGIN(n) \ 145.macro int_define_ ## n name 146 147#define INT_DEFINE_END(n) \ 148.endm ; \ 149int_define_ ## n n ; \ 150do_define_int n 151 152.macro do_define_int name 153 .ifndef IVEC 154 .error "IVEC not defined" 155 .endif 156 .ifndef IHSRR 157 IHSRR=0 158 .endif 159 .ifndef IHSRR_IF_HVMODE 160 IHSRR_IF_HVMODE=0 161 .endif 162 .ifndef IAREA 163 IAREA=PACA_EXGEN 164 .endif 165 .ifndef IVIRT 166 IVIRT=1 167 .endif 168 .ifndef IISIDE 169 IISIDE=0 170 .endif 171 .ifndef IDAR 172 IDAR=0 173 .endif 174 .ifndef IDSISR 175 IDSISR=0 176 .endif 177 .ifndef ISET_RI 178 ISET_RI=1 179 .endif 180 .ifndef IBRANCH_TO_COMMON 181 IBRANCH_TO_COMMON=1 182 .endif 183 .ifndef IREALMODE_COMMON 184 IREALMODE_COMMON=0 185 .else 186 .if ! IBRANCH_TO_COMMON 187 .error "IREALMODE_COMMON=1 but IBRANCH_TO_COMMON=0" 188 .endif 189 .endif 190 .ifndef IMASK 191 IMASK=0 192 .endif 193 .ifndef IKVM_SKIP 194 IKVM_SKIP=0 195 .endif 196 .ifndef IKVM_REAL 197 IKVM_REAL=0 198 .endif 199 .ifndef IKVM_VIRT 200 IKVM_VIRT=0 201 .endif 202 .ifndef ISTACK 203 ISTACK=1 204 .endif 205 .ifndef IKUAP 206 IKUAP=1 207 .endif 208.endm 209 210#ifdef CONFIG_KVM_BOOK3S_64_HANDLER 211#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE 212/* 213 * All interrupts which set HSRR registers, as well as SRESET and MCE and 214 * syscall when invoked with "sc 1" switch to MSR[HV]=1 (HVMODE) to be taken, 215 * so they all generally need to test whether they were taken in guest context. 216 * 217 * Note: SRESET and MCE may also be sent to the guest by the hypervisor, and be 218 * taken with MSR[HV]=0. 219 * 220 * Interrupts which set SRR registers (with the above exceptions) do not 221 * elevate to MSR[HV]=1 mode, though most can be taken when running with 222 * MSR[HV]=1 (e.g., bare metal kernel and userspace). So these interrupts do 223 * not need to test whether a guest is running because they get delivered to 224 * the guest directly, including nested HV KVM guests. 225 * 226 * The exception is PR KVM, where the guest runs with MSR[PR]=1 and the host 227 * runs with MSR[HV]=0, so the host takes all interrupts on behalf of the 228 * guest. PR KVM runs with LPCR[AIL]=0 which causes interrupts to always be 229 * delivered to the real-mode entry point, therefore such interrupts only test 230 * KVM in their real mode handlers, and only when PR KVM is possible. 231 * 232 * Interrupts that are taken in MSR[HV]=0 and escalate to MSR[HV]=1 are always 233 * delivered in real-mode when the MMU is in hash mode because the MMU 234 * registers are not set appropriately to translate host addresses. In nested 235 * radix mode these can be delivered in virt-mode as the host translations are 236 * used implicitly (see: effective LPID, effective PID). 237 */ 238 239/* 240 * If an interrupt is taken while a guest is running, it is immediately routed 241 * to KVM to handle. If both HV and PR KVM arepossible, KVM interrupts go first 242 * to kvmppc_interrupt_hv, which handles the PR guest case. 243 */ 244#define kvmppc_interrupt kvmppc_interrupt_hv 245#else 246#define kvmppc_interrupt kvmppc_interrupt_pr 247#endif 248 249.macro KVMTEST name 250 lbz r10,HSTATE_IN_GUEST(r13) 251 cmpwi r10,0 252 bne \name\()_kvm 253.endm 254 255.macro GEN_KVM name 256 .balign IFETCH_ALIGN_BYTES 257\name\()_kvm: 258 259 .if IKVM_SKIP 260 cmpwi r10,KVM_GUEST_MODE_SKIP 261 beq 89f 262 .else 263BEGIN_FTR_SECTION 264 ld r10,IAREA+EX_CFAR(r13) 265 std r10,HSTATE_CFAR(r13) 266END_FTR_SECTION_IFSET(CPU_FTR_CFAR) 267 .endif 268 269 ld r10,IAREA+EX_CTR(r13) 270 mtctr r10 271BEGIN_FTR_SECTION 272 ld r10,IAREA+EX_PPR(r13) 273 std r10,HSTATE_PPR(r13) 274END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 275 ld r11,IAREA+EX_R11(r13) 276 ld r12,IAREA+EX_R12(r13) 277 std r12,HSTATE_SCRATCH0(r13) 278 sldi r12,r9,32 279 ld r9,IAREA+EX_R9(r13) 280 ld r10,IAREA+EX_R10(r13) 281 /* HSRR variants have the 0x2 bit added to their trap number */ 282 .if IHSRR_IF_HVMODE 283 BEGIN_FTR_SECTION 284 ori r12,r12,(IVEC + 0x2) 285 FTR_SECTION_ELSE 286 ori r12,r12,(IVEC) 287 ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) 288 .elseif IHSRR 289 ori r12,r12,(IVEC+ 0x2) 290 .else 291 ori r12,r12,(IVEC) 292 .endif 293 b kvmppc_interrupt 294 295 .if IKVM_SKIP 29689: mtocrf 0x80,r9 297 ld r10,IAREA+EX_CTR(r13) 298 mtctr r10 299 ld r9,IAREA+EX_R9(r13) 300 ld r10,IAREA+EX_R10(r13) 301 ld r11,IAREA+EX_R11(r13) 302 ld r12,IAREA+EX_R12(r13) 303 .if IHSRR_IF_HVMODE 304 BEGIN_FTR_SECTION 305 b kvmppc_skip_Hinterrupt 306 FTR_SECTION_ELSE 307 b kvmppc_skip_interrupt 308 ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) 309 .elseif IHSRR 310 b kvmppc_skip_Hinterrupt 311 .else 312 b kvmppc_skip_interrupt 313 .endif 314 .endif 315.endm 316 317#else 318.macro KVMTEST name 319.endm 320.macro GEN_KVM name 321.endm 322#endif 323 324/* 325 * This is the BOOK3S interrupt entry code macro. 326 * 327 * This can result in one of several things happening: 328 * - Branch to the _common handler, relocated, in virtual mode. 329 * These are normal interrupts (synchronous and asynchronous) handled by 330 * the kernel. 331 * - Branch to KVM, relocated but real mode interrupts remain in real mode. 332 * These occur when HSTATE_IN_GUEST is set. The interrupt may be caused by 333 * / intended for host or guest kernel, but KVM must always be involved 334 * because the machine state is set for guest execution. 335 * - Branch to the masked handler, unrelocated. 336 * These occur when maskable asynchronous interrupts are taken with the 337 * irq_soft_mask set. 338 * - Branch to an "early" handler in real mode but relocated. 339 * This is done if early=1. MCE and HMI use these to handle errors in real 340 * mode. 341 * - Fall through and continue executing in real, unrelocated mode. 342 * This is done if early=2. 343 */ 344 345.macro GEN_BRANCH_TO_COMMON name, virt 346 .if IREALMODE_COMMON 347 LOAD_HANDLER(r10, \name\()_common) 348 mtctr r10 349 bctr 350 .else 351 .if \virt 352#ifndef CONFIG_RELOCATABLE 353 b \name\()_common_virt 354#else 355 LOAD_HANDLER(r10, \name\()_common_virt) 356 mtctr r10 357 bctr 358#endif 359 .else 360 LOAD_HANDLER(r10, \name\()_common_real) 361 mtctr r10 362 bctr 363 .endif 364 .endif 365.endm 366 367.macro GEN_INT_ENTRY name, virt, ool=0 368 SET_SCRATCH0(r13) /* save r13 */ 369 GET_PACA(r13) 370 std r9,IAREA+EX_R9(r13) /* save r9 */ 371BEGIN_FTR_SECTION 372 mfspr r9,SPRN_PPR 373END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 374 HMT_MEDIUM 375 std r10,IAREA+EX_R10(r13) /* save r10 - r12 */ 376BEGIN_FTR_SECTION 377 mfspr r10,SPRN_CFAR 378END_FTR_SECTION_IFSET(CPU_FTR_CFAR) 379 .if \ool 380 .if !\virt 381 b tramp_real_\name 382 .pushsection .text 383 TRAMP_REAL_BEGIN(tramp_real_\name) 384 .else 385 b tramp_virt_\name 386 .pushsection .text 387 TRAMP_VIRT_BEGIN(tramp_virt_\name) 388 .endif 389 .endif 390 391BEGIN_FTR_SECTION 392 std r9,IAREA+EX_PPR(r13) 393END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 394BEGIN_FTR_SECTION 395 std r10,IAREA+EX_CFAR(r13) 396END_FTR_SECTION_IFSET(CPU_FTR_CFAR) 397 INTERRUPT_TO_KERNEL 398 mfctr r10 399 std r10,IAREA+EX_CTR(r13) 400 mfcr r9 401 std r11,IAREA+EX_R11(r13) 402 std r12,IAREA+EX_R12(r13) 403 404 /* 405 * DAR/DSISR, SCRATCH0 must be read before setting MSR[RI], 406 * because a d-side MCE will clobber those registers so is 407 * not recoverable if they are live. 408 */ 409 GET_SCRATCH0(r10) 410 std r10,IAREA+EX_R13(r13) 411 .if IDAR && !IISIDE 412 .if IHSRR 413 mfspr r10,SPRN_HDAR 414 .else 415 mfspr r10,SPRN_DAR 416 .endif 417 std r10,IAREA+EX_DAR(r13) 418 .endif 419 .if IDSISR && !IISIDE 420 .if IHSRR 421 mfspr r10,SPRN_HDSISR 422 .else 423 mfspr r10,SPRN_DSISR 424 .endif 425 stw r10,IAREA+EX_DSISR(r13) 426 .endif 427 428 .if IHSRR_IF_HVMODE 429 BEGIN_FTR_SECTION 430 mfspr r11,SPRN_HSRR0 /* save HSRR0 */ 431 mfspr r12,SPRN_HSRR1 /* and HSRR1 */ 432 FTR_SECTION_ELSE 433 mfspr r11,SPRN_SRR0 /* save SRR0 */ 434 mfspr r12,SPRN_SRR1 /* and SRR1 */ 435 ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) 436 .elseif IHSRR 437 mfspr r11,SPRN_HSRR0 /* save HSRR0 */ 438 mfspr r12,SPRN_HSRR1 /* and HSRR1 */ 439 .else 440 mfspr r11,SPRN_SRR0 /* save SRR0 */ 441 mfspr r12,SPRN_SRR1 /* and SRR1 */ 442 .endif 443 444 .if IBRANCH_TO_COMMON 445 GEN_BRANCH_TO_COMMON \name \virt 446 .endif 447 448 .if \ool 449 .popsection 450 .endif 451.endm 452 453/* 454 * __GEN_COMMON_ENTRY is required to receive the branch from interrupt 455 * entry, except in the case of the real-mode handlers which require 456 * __GEN_REALMODE_COMMON_ENTRY. 457 * 458 * This switches to virtual mode and sets MSR[RI]. 459 */ 460.macro __GEN_COMMON_ENTRY name 461DEFINE_FIXED_SYMBOL(\name\()_common_real) 462\name\()_common_real: 463 .if IKVM_REAL 464 KVMTEST \name 465 .endif 466 467 ld r10,PACAKMSR(r13) /* get MSR value for kernel */ 468 /* MSR[RI] is clear iff using SRR regs */ 469 .if IHSRR == EXC_HV_OR_STD 470 BEGIN_FTR_SECTION 471 xori r10,r10,MSR_RI 472 END_FTR_SECTION_IFCLR(CPU_FTR_HVMODE) 473 .elseif ! IHSRR 474 xori r10,r10,MSR_RI 475 .endif 476 mtmsrd r10 477 478 .if IVIRT 479 .if IKVM_VIRT 480 b 1f /* skip the virt test coming from real */ 481 .endif 482 483 .balign IFETCH_ALIGN_BYTES 484DEFINE_FIXED_SYMBOL(\name\()_common_virt) 485\name\()_common_virt: 486 .if IKVM_VIRT 487 KVMTEST \name 4881: 489 .endif 490 .endif /* IVIRT */ 491.endm 492 493/* 494 * Don't switch to virt mode. Used for early MCE and HMI handlers that 495 * want to run in real mode. 496 */ 497.macro __GEN_REALMODE_COMMON_ENTRY name 498DEFINE_FIXED_SYMBOL(\name\()_common_real) 499\name\()_common_real: 500 .if IKVM_REAL 501 KVMTEST \name 502 .endif 503.endm 504 505.macro __GEN_COMMON_BODY name 506 .if IMASK 507 .if ! ISTACK 508 .error "No support for masked interrupt to use custom stack" 509 .endif 510 511 /* If coming from user, skip soft-mask tests. */ 512 andi. r10,r12,MSR_PR 513 bne 2f 514 515 /* Kernel code running below __end_interrupts is implicitly 516 * soft-masked */ 517 LOAD_HANDLER(r10, __end_interrupts) 518 cmpld r11,r10 519 li r10,IMASK 520 blt- 1f 521 522 /* Test the soft mask state against our interrupt's bit */ 523 lbz r10,PACAIRQSOFTMASK(r13) 5241: andi. r10,r10,IMASK 525 /* Associate vector numbers with bits in paca->irq_happened */ 526 .if IVEC == 0x500 || IVEC == 0xea0 527 li r10,PACA_IRQ_EE 528 .elseif IVEC == 0x900 529 li r10,PACA_IRQ_DEC 530 .elseif IVEC == 0xa00 || IVEC == 0xe80 531 li r10,PACA_IRQ_DBELL 532 .elseif IVEC == 0xe60 533 li r10,PACA_IRQ_HMI 534 .elseif IVEC == 0xf00 535 li r10,PACA_IRQ_PMI 536 .else 537 .abort "Bad maskable vector" 538 .endif 539 540 .if IHSRR_IF_HVMODE 541 BEGIN_FTR_SECTION 542 bne masked_Hinterrupt 543 FTR_SECTION_ELSE 544 bne masked_interrupt 545 ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) 546 .elseif IHSRR 547 bne masked_Hinterrupt 548 .else 549 bne masked_interrupt 550 .endif 551 .endif 552 553 .if ISTACK 554 andi. r10,r12,MSR_PR /* See if coming from user */ 5552: mr r10,r1 /* Save r1 */ 556 subi r1,r1,INT_FRAME_SIZE /* alloc frame on kernel stack */ 557 beq- 100f 558 ld r1,PACAKSAVE(r13) /* kernel stack to use */ 559100: tdgei r1,-INT_FRAME_SIZE /* trap if r1 is in userspace */ 560 EMIT_BUG_ENTRY 100b,__FILE__,__LINE__,0 561 .endif 562 563 std r9,_CCR(r1) /* save CR in stackframe */ 564 std r11,_NIP(r1) /* save SRR0 in stackframe */ 565 std r12,_MSR(r1) /* save SRR1 in stackframe */ 566 std r10,0(r1) /* make stack chain pointer */ 567 std r0,GPR0(r1) /* save r0 in stackframe */ 568 std r10,GPR1(r1) /* save r1 in stackframe */ 569 570 .if ISET_RI 571 li r10,MSR_RI 572 mtmsrd r10,1 /* Set MSR_RI */ 573 .endif 574 575 .if ISTACK 576 .if IKUAP 577 kuap_save_amr_and_lock r9, r10, cr1, cr0 578 .endif 579 beq 101f /* if from kernel mode */ 580BEGIN_FTR_SECTION 581 ld r9,IAREA+EX_PPR(r13) /* Read PPR from paca */ 582 std r9,_PPR(r1) 583END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 584101: 585 .else 586 .if IKUAP 587 kuap_save_amr_and_lock r9, r10, cr1 588 .endif 589 .endif 590 591 /* Save original regs values from save area to stack frame. */ 592 ld r9,IAREA+EX_R9(r13) /* move r9, r10 to stackframe */ 593 ld r10,IAREA+EX_R10(r13) 594 std r9,GPR9(r1) 595 std r10,GPR10(r1) 596 ld r9,IAREA+EX_R11(r13) /* move r11 - r13 to stackframe */ 597 ld r10,IAREA+EX_R12(r13) 598 ld r11,IAREA+EX_R13(r13) 599 std r9,GPR11(r1) 600 std r10,GPR12(r1) 601 std r11,GPR13(r1) 602 603 SAVE_NVGPRS(r1) 604 605 .if IDAR 606 .if IISIDE 607 ld r10,_NIP(r1) 608 .else 609 ld r10,IAREA+EX_DAR(r13) 610 .endif 611 std r10,_DAR(r1) 612 .endif 613 614 .if IDSISR 615 .if IISIDE 616 ld r10,_MSR(r1) 617 lis r11,DSISR_SRR1_MATCH_64S@h 618 and r10,r10,r11 619 .else 620 lwz r10,IAREA+EX_DSISR(r13) 621 .endif 622 std r10,_DSISR(r1) 623 .endif 624 625BEGIN_FTR_SECTION 626 ld r10,IAREA+EX_CFAR(r13) 627 std r10,ORIG_GPR3(r1) 628END_FTR_SECTION_IFSET(CPU_FTR_CFAR) 629 ld r10,IAREA+EX_CTR(r13) 630 std r10,_CTR(r1) 631 std r2,GPR2(r1) /* save r2 in stackframe */ 632 SAVE_4GPRS(3, r1) /* save r3 - r6 in stackframe */ 633 SAVE_2GPRS(7, r1) /* save r7, r8 in stackframe */ 634 mflr r9 /* Get LR, later save to stack */ 635 ld r2,PACATOC(r13) /* get kernel TOC into r2 */ 636 std r9,_LINK(r1) 637 lbz r10,PACAIRQSOFTMASK(r13) 638 mfspr r11,SPRN_XER /* save XER in stackframe */ 639 std r10,SOFTE(r1) 640 std r11,_XER(r1) 641 li r9,IVEC 642 std r9,_TRAP(r1) /* set trap number */ 643 li r10,0 644 ld r11,exception_marker@toc(r2) 645 std r10,RESULT(r1) /* clear regs->result */ 646 std r11,STACK_FRAME_OVERHEAD-16(r1) /* mark the frame */ 647.endm 648 649/* 650 * On entry r13 points to the paca, r9-r13 are saved in the paca, 651 * r9 contains the saved CR, r11 and r12 contain the saved SRR0 and 652 * SRR1, and relocation is on. 653 * 654 * If stack=0, then the stack is already set in r1, and r1 is saved in r10. 655 * PPR save and CPU accounting is not done for the !stack case (XXX why not?) 656 */ 657.macro GEN_COMMON name 658 __GEN_COMMON_ENTRY \name 659 __GEN_COMMON_BODY \name 660.endm 661 662/* 663 * Restore all registers including H/SRR0/1 saved in a stack frame of a 664 * standard exception. 665 */ 666.macro EXCEPTION_RESTORE_REGS hsrr=0 667 /* Move original SRR0 and SRR1 into the respective regs */ 668 ld r9,_MSR(r1) 669 .if \hsrr 670 mtspr SPRN_HSRR1,r9 671 .else 672 mtspr SPRN_SRR1,r9 673 .endif 674 ld r9,_NIP(r1) 675 .if \hsrr 676 mtspr SPRN_HSRR0,r9 677 .else 678 mtspr SPRN_SRR0,r9 679 .endif 680 ld r9,_CTR(r1) 681 mtctr r9 682 ld r9,_XER(r1) 683 mtxer r9 684 ld r9,_LINK(r1) 685 mtlr r9 686 ld r9,_CCR(r1) 687 mtcr r9 688 REST_8GPRS(2, r1) 689 REST_4GPRS(10, r1) 690 REST_GPR(0, r1) 691 /* restore original r1. */ 692 ld r1,GPR1(r1) 693.endm 694 695/* 696 * When the idle code in power4_idle puts the CPU into NAP mode, 697 * it has to do so in a loop, and relies on the external interrupt 698 * and decrementer interrupt entry code to get it out of the loop. 699 * It sets the _TLF_NAPPING bit in current_thread_info()->local_flags 700 * to signal that it is in the loop and needs help to get out. 701 */ 702#ifdef CONFIG_PPC_970_NAP 703#define FINISH_NAP \ 704BEGIN_FTR_SECTION \ 705 ld r11, PACA_THREAD_INFO(r13); \ 706 ld r9,TI_LOCAL_FLAGS(r11); \ 707 andi. r10,r9,_TLF_NAPPING; \ 708 bnel power4_fixup_nap; \ 709END_FTR_SECTION_IFSET(CPU_FTR_CAN_NAP) 710#else 711#define FINISH_NAP 712#endif 713 714/* 715 * There are a few constraints to be concerned with. 716 * - Real mode exceptions code/data must be located at their physical location. 717 * - Virtual mode exceptions must be mapped at their 0xc000... location. 718 * - Fixed location code must not call directly beyond the __end_interrupts 719 * area when built with CONFIG_RELOCATABLE. LOAD_HANDLER / bctr sequence 720 * must be used. 721 * - LOAD_HANDLER targets must be within first 64K of physical 0 / 722 * virtual 0xc00... 723 * - Conditional branch targets must be within +/-32K of caller. 724 * 725 * "Virtual exceptions" run with relocation on (MSR_IR=1, MSR_DR=1), and 726 * therefore don't have to run in physically located code or rfid to 727 * virtual mode kernel code. However on relocatable kernels they do have 728 * to branch to KERNELBASE offset because the rest of the kernel (outside 729 * the exception vectors) may be located elsewhere. 730 * 731 * Virtual exceptions correspond with physical, except their entry points 732 * are offset by 0xc000000000000000 and also tend to get an added 0x4000 733 * offset applied. Virtual exceptions are enabled with the Alternate 734 * Interrupt Location (AIL) bit set in the LPCR. However this does not 735 * guarantee they will be delivered virtually. Some conditions (see the ISA) 736 * cause exceptions to be delivered in real mode. 737 * 738 * The scv instructions are a special case. They get a 0x3000 offset applied. 739 * scv exceptions have unique reentrancy properties, see below. 740 * 741 * It's impossible to receive interrupts below 0x300 via AIL. 742 * 743 * KVM: None of the virtual exceptions are from the guest. Anything that 744 * escalated to HV=1 from HV=0 is delivered via real mode handlers. 745 * 746 * 747 * We layout physical memory as follows: 748 * 0x0000 - 0x00ff : Secondary processor spin code 749 * 0x0100 - 0x18ff : Real mode pSeries interrupt vectors 750 * 0x1900 - 0x2fff : Real mode trampolines 751 * 0x3000 - 0x58ff : Relon (IR=1,DR=1) mode pSeries interrupt vectors 752 * 0x5900 - 0x6fff : Relon mode trampolines 753 * 0x7000 - 0x7fff : FWNMI data area 754 * 0x8000 - .... : Common interrupt handlers, remaining early 755 * setup code, rest of kernel. 756 * 757 * We could reclaim 0x4000-0x42ff for real mode trampolines if the space 758 * is necessary. Until then it's more consistent to explicitly put VIRT_NONE 759 * vectors there. 760 */ 761OPEN_FIXED_SECTION(real_vectors, 0x0100, 0x1900) 762OPEN_FIXED_SECTION(real_trampolines, 0x1900, 0x3000) 763OPEN_FIXED_SECTION(virt_vectors, 0x3000, 0x5900) 764OPEN_FIXED_SECTION(virt_trampolines, 0x5900, 0x7000) 765 766#ifdef CONFIG_PPC_POWERNV 767 .globl start_real_trampolines 768 .globl end_real_trampolines 769 .globl start_virt_trampolines 770 .globl end_virt_trampolines 771#endif 772 773#if defined(CONFIG_PPC_PSERIES) || defined(CONFIG_PPC_POWERNV) 774/* 775 * Data area reserved for FWNMI option. 776 * This address (0x7000) is fixed by the RPA. 777 * pseries and powernv need to keep the whole page from 778 * 0x7000 to 0x8000 free for use by the firmware 779 */ 780ZERO_FIXED_SECTION(fwnmi_page, 0x7000, 0x8000) 781OPEN_TEXT_SECTION(0x8000) 782#else 783OPEN_TEXT_SECTION(0x7000) 784#endif 785 786USE_FIXED_SECTION(real_vectors) 787 788/* 789 * This is the start of the interrupt handlers for pSeries 790 * This code runs with relocation off. 791 * Code from here to __end_interrupts gets copied down to real 792 * address 0x100 when we are running a relocatable kernel. 793 * Therefore any relative branches in this section must only 794 * branch to labels in this section. 795 */ 796 .globl __start_interrupts 797__start_interrupts: 798 799/** 800 * Interrupt 0x3000 - System Call Vectored Interrupt (syscall). 801 * This is a synchronous interrupt invoked with the "scv" instruction. The 802 * system call does not alter the HV bit, so it is directed to the OS. 803 * 804 * Handling: 805 * scv instructions enter the kernel without changing EE, RI, ME, or HV. 806 * In particular, this means we can take a maskable interrupt at any point 807 * in the scv handler, which is unlike any other interrupt. This is solved 808 * by treating the instruction addresses below __end_interrupts as being 809 * soft-masked. 810 * 811 * AIL-0 mode scv exceptions go to 0x17000-0x17fff, but we set AIL-3 and 812 * ensure scv is never executed with relocation off, which means AIL-0 813 * should never happen. 814 * 815 * Before leaving the below __end_interrupts text, at least of the following 816 * must be true: 817 * - MSR[PR]=1 (i.e., return to userspace) 818 * - MSR_EE|MSR_RI is set (no reentrant exceptions) 819 * - Standard kernel environment is set up (stack, paca, etc) 820 * 821 * Call convention: 822 * 823 * syscall register convention is in Documentation/powerpc/syscall64-abi.rst 824 */ 825EXC_VIRT_BEGIN(system_call_vectored, 0x3000, 0x1000) 826 /* SCV 0 */ 827 mr r9,r13 828 GET_PACA(r13) 829 mflr r11 830 mfctr r12 831 li r10,IRQS_ALL_DISABLED 832 stb r10,PACAIRQSOFTMASK(r13) 833#ifdef CONFIG_RELOCATABLE 834 b system_call_vectored_tramp 835#else 836 b system_call_vectored_common 837#endif 838 nop 839 840 /* SCV 1 - 127 */ 841 .rept 127 842 mr r9,r13 843 GET_PACA(r13) 844 mflr r11 845 mfctr r12 846 li r10,IRQS_ALL_DISABLED 847 stb r10,PACAIRQSOFTMASK(r13) 848 li r0,-1 /* cause failure */ 849#ifdef CONFIG_RELOCATABLE 850 b system_call_vectored_sigill_tramp 851#else 852 b system_call_vectored_sigill 853#endif 854 .endr 855EXC_VIRT_END(system_call_vectored, 0x3000, 0x1000) 856 857#ifdef CONFIG_RELOCATABLE 858TRAMP_VIRT_BEGIN(system_call_vectored_tramp) 859 __LOAD_HANDLER(r10, system_call_vectored_common) 860 mtctr r10 861 bctr 862 863TRAMP_VIRT_BEGIN(system_call_vectored_sigill_tramp) 864 __LOAD_HANDLER(r10, system_call_vectored_sigill) 865 mtctr r10 866 bctr 867#endif 868 869 870/* No virt vectors corresponding with 0x0..0x100 */ 871EXC_VIRT_NONE(0x4000, 0x100) 872 873 874/** 875 * Interrupt 0x100 - System Reset Interrupt (SRESET aka NMI). 876 * This is a non-maskable, asynchronous interrupt always taken in real-mode. 877 * It is caused by: 878 * - Wake from power-saving state, on powernv. 879 * - An NMI from another CPU, triggered by firmware or hypercall. 880 * - As crash/debug signal injected from BMC, firmware or hypervisor. 881 * 882 * Handling: 883 * Power-save wakeup is the only performance critical path, so this is 884 * determined quickly as possible first. In this case volatile registers 885 * can be discarded and SPRs like CFAR don't need to be read. 886 * 887 * If not a powersave wakeup, then it's run as a regular interrupt, however 888 * it uses its own stack and PACA save area to preserve the regular kernel 889 * environment for debugging. 890 * 891 * This interrupt is not maskable, so triggering it when MSR[RI] is clear, 892 * or SCRATCH0 is in use, etc. may cause a crash. It's also not entirely 893 * correct to switch to virtual mode to run the regular interrupt handler 894 * because it might be interrupted when the MMU is in a bad state (e.g., SLB 895 * is clear). 896 * 897 * FWNMI: 898 * PAPR specifies a "fwnmi" facility which sends the sreset to a different 899 * entry point with a different register set up. Some hypervisors will 900 * send the sreset to 0x100 in the guest if it is not fwnmi capable. 901 * 902 * KVM: 903 * Unlike most SRR interrupts, this may be taken by the host while executing 904 * in a guest, so a KVM test is required. KVM will pull the CPU out of guest 905 * mode and then raise the sreset. 906 */ 907INT_DEFINE_BEGIN(system_reset) 908 IVEC=0x100 909 IAREA=PACA_EXNMI 910 IVIRT=0 /* no virt entry point */ 911 /* 912 * MSR_RI is not enabled, because PACA_EXNMI and nmi stack is 913 * being used, so a nested NMI exception would corrupt it. 914 */ 915 ISET_RI=0 916 ISTACK=0 917 IKVM_REAL=1 918INT_DEFINE_END(system_reset) 919 920EXC_REAL_BEGIN(system_reset, 0x100, 0x100) 921#ifdef CONFIG_PPC_P7_NAP 922 /* 923 * If running native on arch 2.06 or later, check if we are waking up 924 * from nap/sleep/winkle, and branch to idle handler. This tests SRR1 925 * bits 46:47. A non-0 value indicates that we are coming from a power 926 * saving state. The idle wakeup handler initially runs in real mode, 927 * but we branch to the 0xc000... address so we can turn on relocation 928 * with mtmsrd later, after SPRs are restored. 929 * 930 * Careful to minimise cost for the fast path (idle wakeup) while 931 * also avoiding clobbering CFAR for the debug path (non-idle). 932 * 933 * For the idle wake case volatile registers can be clobbered, which 934 * is why we use those initially. If it turns out to not be an idle 935 * wake, carefully put everything back the way it was, so we can use 936 * common exception macros to handle it. 937 */ 938BEGIN_FTR_SECTION 939 SET_SCRATCH0(r13) 940 GET_PACA(r13) 941 std r3,PACA_EXNMI+0*8(r13) 942 std r4,PACA_EXNMI+1*8(r13) 943 std r5,PACA_EXNMI+2*8(r13) 944 mfspr r3,SPRN_SRR1 945 mfocrf r4,0x80 946 rlwinm. r5,r3,47-31,30,31 947 bne+ system_reset_idle_wake 948 /* Not powersave wakeup. Restore regs for regular interrupt handler. */ 949 mtocrf 0x80,r4 950 ld r3,PACA_EXNMI+0*8(r13) 951 ld r4,PACA_EXNMI+1*8(r13) 952 ld r5,PACA_EXNMI+2*8(r13) 953 GET_SCRATCH0(r13) 954END_FTR_SECTION_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) 955#endif 956 957 GEN_INT_ENTRY system_reset, virt=0 958 /* 959 * In theory, we should not enable relocation here if it was disabled 960 * in SRR1, because the MMU may not be configured to support it (e.g., 961 * SLB may have been cleared). In practice, there should only be a few 962 * small windows where that's the case, and sreset is considered to 963 * be dangerous anyway. 964 */ 965EXC_REAL_END(system_reset, 0x100, 0x100) 966EXC_VIRT_NONE(0x4100, 0x100) 967 968#ifdef CONFIG_PPC_P7_NAP 969TRAMP_REAL_BEGIN(system_reset_idle_wake) 970 /* We are waking up from idle, so may clobber any volatile register */ 971 cmpwi cr1,r5,2 972 bltlr cr1 /* no state loss, return to idle caller with r3=SRR1 */ 973 BRANCH_TO_C000(r12, DOTSYM(idle_return_gpr_loss)) 974#endif 975 976#ifdef CONFIG_PPC_PSERIES 977/* 978 * Vectors for the FWNMI option. Share common code. 979 */ 980TRAMP_REAL_BEGIN(system_reset_fwnmi) 981 GEN_INT_ENTRY system_reset, virt=0 982 983#endif /* CONFIG_PPC_PSERIES */ 984 985EXC_COMMON_BEGIN(system_reset_common) 986 __GEN_COMMON_ENTRY system_reset 987 /* 988 * Increment paca->in_nmi then enable MSR_RI. SLB or MCE will be able 989 * to recover, but nested NMI will notice in_nmi and not recover 990 * because of the use of the NMI stack. in_nmi reentrancy is tested in 991 * system_reset_exception. 992 */ 993 lhz r10,PACA_IN_NMI(r13) 994 addi r10,r10,1 995 sth r10,PACA_IN_NMI(r13) 996 li r10,MSR_RI 997 mtmsrd r10,1 998 999 mr r10,r1 1000 ld r1,PACA_NMI_EMERG_SP(r13) 1001 subi r1,r1,INT_FRAME_SIZE 1002 __GEN_COMMON_BODY system_reset 1003 1004 addi r3,r1,STACK_FRAME_OVERHEAD 1005 bl system_reset_exception 1006 1007 /* Clear MSR_RI before setting SRR0 and SRR1. */ 1008 li r9,0 1009 mtmsrd r9,1 1010 1011 /* 1012 * MSR_RI is clear, now we can decrement paca->in_nmi. 1013 */ 1014 lhz r10,PACA_IN_NMI(r13) 1015 subi r10,r10,1 1016 sth r10,PACA_IN_NMI(r13) 1017 1018 kuap_kernel_restore r9, r10 1019 EXCEPTION_RESTORE_REGS 1020 RFI_TO_USER_OR_KERNEL 1021 1022 GEN_KVM system_reset 1023 1024 1025/** 1026 * Interrupt 0x200 - Machine Check Interrupt (MCE). 1027 * This is a non-maskable interrupt always taken in real-mode. It can be 1028 * synchronous or asynchronous, caused by hardware or software, and it may be 1029 * taken in a power-saving state. 1030 * 1031 * Handling: 1032 * Similarly to system reset, this uses its own stack and PACA save area, 1033 * the difference is re-entrancy is allowed on the machine check stack. 1034 * 1035 * machine_check_early is run in real mode, and carefully decodes the 1036 * machine check and tries to handle it (e.g., flush the SLB if there was an 1037 * error detected there), determines if it was recoverable and logs the 1038 * event. 1039 * 1040 * This early code does not "reconcile" irq soft-mask state like SRESET or 1041 * regular interrupts do, so irqs_disabled() among other things may not work 1042 * properly (irq disable/enable already doesn't work because irq tracing can 1043 * not work in real mode). 1044 * 1045 * Then, depending on the execution context when the interrupt is taken, there 1046 * are 3 main actions: 1047 * - Executing in kernel mode. The event is queued with irq_work, which means 1048 * it is handled when it is next safe to do so (i.e., the kernel has enabled 1049 * interrupts), which could be immediately when the interrupt returns. This 1050 * avoids nasty issues like switching to virtual mode when the MMU is in a 1051 * bad state, or when executing OPAL code. (SRESET is exposed to such issues, 1052 * but it has different priorities). Check to see if the CPU was in power 1053 * save, and return via the wake up code if it was. 1054 * 1055 * - Executing in user mode. machine_check_exception is run like a normal 1056 * interrupt handler, which processes the data generated by the early handler. 1057 * 1058 * - Executing in guest mode. The interrupt is run with its KVM test, and 1059 * branches to KVM to deal with. KVM may queue the event for the host 1060 * to report later. 1061 * 1062 * This interrupt is not maskable, so if it triggers when MSR[RI] is clear, 1063 * or SCRATCH0 is in use, it may cause a crash. 1064 * 1065 * KVM: 1066 * See SRESET. 1067 */ 1068INT_DEFINE_BEGIN(machine_check_early) 1069 IVEC=0x200 1070 IAREA=PACA_EXMC 1071 IVIRT=0 /* no virt entry point */ 1072 IREALMODE_COMMON=1 1073 /* 1074 * MSR_RI is not enabled, because PACA_EXMC is being used, so a 1075 * nested machine check corrupts it. machine_check_common enables 1076 * MSR_RI. 1077 */ 1078 ISET_RI=0 1079 ISTACK=0 1080 IDAR=1 1081 IDSISR=1 1082 IKUAP=0 /* We don't touch AMR here, we never go to virtual mode */ 1083INT_DEFINE_END(machine_check_early) 1084 1085INT_DEFINE_BEGIN(machine_check) 1086 IVEC=0x200 1087 IAREA=PACA_EXMC 1088 IVIRT=0 /* no virt entry point */ 1089 ISET_RI=0 1090 IDAR=1 1091 IDSISR=1 1092 IKVM_SKIP=1 1093 IKVM_REAL=1 1094INT_DEFINE_END(machine_check) 1095 1096EXC_REAL_BEGIN(machine_check, 0x200, 0x100) 1097 GEN_INT_ENTRY machine_check_early, virt=0 1098EXC_REAL_END(machine_check, 0x200, 0x100) 1099EXC_VIRT_NONE(0x4200, 0x100) 1100 1101#ifdef CONFIG_PPC_PSERIES 1102TRAMP_REAL_BEGIN(machine_check_fwnmi) 1103 /* See comment at machine_check exception, don't turn on RI */ 1104 GEN_INT_ENTRY machine_check_early, virt=0 1105#endif 1106 1107#define MACHINE_CHECK_HANDLER_WINDUP \ 1108 /* Clear MSR_RI before setting SRR0 and SRR1. */\ 1109 li r9,0; \ 1110 mtmsrd r9,1; /* Clear MSR_RI */ \ 1111 /* Decrement paca->in_mce now RI is clear. */ \ 1112 lhz r12,PACA_IN_MCE(r13); \ 1113 subi r12,r12,1; \ 1114 sth r12,PACA_IN_MCE(r13); \ 1115 EXCEPTION_RESTORE_REGS 1116 1117EXC_COMMON_BEGIN(machine_check_early_common) 1118 __GEN_REALMODE_COMMON_ENTRY machine_check_early 1119 1120 /* 1121 * Switch to mc_emergency stack and handle re-entrancy (we limit 1122 * the nested MCE upto level 4 to avoid stack overflow). 1123 * Save MCE registers srr1, srr0, dar and dsisr and then set ME=1 1124 * 1125 * We use paca->in_mce to check whether this is the first entry or 1126 * nested machine check. We increment paca->in_mce to track nested 1127 * machine checks. 1128 * 1129 * If this is the first entry then set stack pointer to 1130 * paca->mc_emergency_sp, otherwise r1 is already pointing to 1131 * stack frame on mc_emergency stack. 1132 * 1133 * NOTE: We are here with MSR_ME=0 (off), which means we risk a 1134 * checkstop if we get another machine check exception before we do 1135 * rfid with MSR_ME=1. 1136 * 1137 * This interrupt can wake directly from idle. If that is the case, 1138 * the machine check is handled then the idle wakeup code is called 1139 * to restore state. 1140 */ 1141 lhz r10,PACA_IN_MCE(r13) 1142 cmpwi r10,0 /* Are we in nested machine check */ 1143 cmpwi cr1,r10,MAX_MCE_DEPTH /* Are we at maximum nesting */ 1144 addi r10,r10,1 /* increment paca->in_mce */ 1145 sth r10,PACA_IN_MCE(r13) 1146 1147 mr r10,r1 /* Save r1 */ 1148 bne 1f 1149 /* First machine check entry */ 1150 ld r1,PACAMCEMERGSP(r13) /* Use MC emergency stack */ 11511: /* Limit nested MCE to level 4 to avoid stack overflow */ 1152 bgt cr1,unrecoverable_mce /* Check if we hit limit of 4 */ 1153 subi r1,r1,INT_FRAME_SIZE /* alloc stack frame */ 1154 1155 __GEN_COMMON_BODY machine_check_early 1156 1157BEGIN_FTR_SECTION 1158 bl enable_machine_check 1159END_FTR_SECTION_IFSET(CPU_FTR_HVMODE) 1160 li r10,MSR_RI 1161 mtmsrd r10,1 1162 1163 addi r3,r1,STACK_FRAME_OVERHEAD 1164 bl machine_check_early 1165 std r3,RESULT(r1) /* Save result */ 1166 ld r12,_MSR(r1) 1167 1168#ifdef CONFIG_PPC_P7_NAP 1169 /* 1170 * Check if thread was in power saving mode. We come here when any 1171 * of the following is true: 1172 * a. thread wasn't in power saving mode 1173 * b. thread was in power saving mode with no state loss, 1174 * supervisor state loss or hypervisor state loss. 1175 * 1176 * Go back to nap/sleep/winkle mode again if (b) is true. 1177 */ 1178BEGIN_FTR_SECTION 1179 rlwinm. r11,r12,47-31,30,31 1180 bne machine_check_idle_common 1181END_FTR_SECTION_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) 1182#endif 1183 1184#ifdef CONFIG_KVM_BOOK3S_64_HANDLER 1185 /* 1186 * Check if we are coming from guest. If yes, then run the normal 1187 * exception handler which will take the 1188 * machine_check_kvm->kvmppc_interrupt branch to deliver the MC event 1189 * to guest. 1190 */ 1191 lbz r11,HSTATE_IN_GUEST(r13) 1192 cmpwi r11,0 /* Check if coming from guest */ 1193 bne mce_deliver /* continue if we are. */ 1194#endif 1195 1196 /* 1197 * Check if we are coming from userspace. If yes, then run the normal 1198 * exception handler which will deliver the MC event to this kernel. 1199 */ 1200 andi. r11,r12,MSR_PR /* See if coming from user. */ 1201 bne mce_deliver /* continue in V mode if we are. */ 1202 1203 /* 1204 * At this point we are coming from kernel context. 1205 * Queue up the MCE event and return from the interrupt. 1206 * But before that, check if this is an un-recoverable exception. 1207 * If yes, then stay on emergency stack and panic. 1208 */ 1209 andi. r11,r12,MSR_RI 1210 beq unrecoverable_mce 1211 1212 /* 1213 * Check if we have successfully handled/recovered from error, if not 1214 * then stay on emergency stack and panic. 1215 */ 1216 ld r3,RESULT(r1) /* Load result */ 1217 cmpdi r3,0 /* see if we handled MCE successfully */ 1218 beq unrecoverable_mce /* if !handled then panic */ 1219 1220 /* 1221 * Return from MC interrupt. 1222 * Queue up the MCE event so that we can log it later, while 1223 * returning from kernel or opal call. 1224 */ 1225 bl machine_check_queue_event 1226 MACHINE_CHECK_HANDLER_WINDUP 1227 RFI_TO_KERNEL 1228 1229mce_deliver: 1230 /* 1231 * This is a host user or guest MCE. Restore all registers, then 1232 * run the "late" handler. For host user, this will run the 1233 * machine_check_exception handler in virtual mode like a normal 1234 * interrupt handler. For guest, this will trigger the KVM test 1235 * and branch to the KVM interrupt similarly to other interrupts. 1236 */ 1237BEGIN_FTR_SECTION 1238 ld r10,ORIG_GPR3(r1) 1239 mtspr SPRN_CFAR,r10 1240END_FTR_SECTION_IFSET(CPU_FTR_CFAR) 1241 MACHINE_CHECK_HANDLER_WINDUP 1242 GEN_INT_ENTRY machine_check, virt=0 1243 1244EXC_COMMON_BEGIN(machine_check_common) 1245 /* 1246 * Machine check is different because we use a different 1247 * save area: PACA_EXMC instead of PACA_EXGEN. 1248 */ 1249 GEN_COMMON machine_check 1250 1251 FINISH_NAP 1252 /* Enable MSR_RI when finished with PACA_EXMC */ 1253 li r10,MSR_RI 1254 mtmsrd r10,1 1255 addi r3,r1,STACK_FRAME_OVERHEAD 1256 bl machine_check_exception 1257 b interrupt_return 1258 1259 GEN_KVM machine_check 1260 1261 1262#ifdef CONFIG_PPC_P7_NAP 1263/* 1264 * This is an idle wakeup. Low level machine check has already been 1265 * done. Queue the event then call the idle code to do the wake up. 1266 */ 1267EXC_COMMON_BEGIN(machine_check_idle_common) 1268 bl machine_check_queue_event 1269 1270 /* 1271 * GPR-loss wakeups are relatively straightforward, because the 1272 * idle sleep code has saved all non-volatile registers on its 1273 * own stack, and r1 in PACAR1. 1274 * 1275 * For no-loss wakeups the r1 and lr registers used by the 1276 * early machine check handler have to be restored first. r2 is 1277 * the kernel TOC, so no need to restore it. 1278 * 1279 * Then decrement MCE nesting after finishing with the stack. 1280 */ 1281 ld r3,_MSR(r1) 1282 ld r4,_LINK(r1) 1283 ld r1,GPR1(r1) 1284 1285 lhz r11,PACA_IN_MCE(r13) 1286 subi r11,r11,1 1287 sth r11,PACA_IN_MCE(r13) 1288 1289 mtlr r4 1290 rlwinm r10,r3,47-31,30,31 1291 cmpwi cr1,r10,2 1292 bltlr cr1 /* no state loss, return to idle caller with r3=SRR1 */ 1293 b idle_return_gpr_loss 1294#endif 1295 1296EXC_COMMON_BEGIN(unrecoverable_mce) 1297 /* 1298 * We are going down. But there are chances that we might get hit by 1299 * another MCE during panic path and we may run into unstable state 1300 * with no way out. Hence, turn ME bit off while going down, so that 1301 * when another MCE is hit during panic path, system will checkstop 1302 * and hypervisor will get restarted cleanly by SP. 1303 */ 1304BEGIN_FTR_SECTION 1305 li r10,0 /* clear MSR_RI */ 1306 mtmsrd r10,1 1307 bl disable_machine_check 1308END_FTR_SECTION_IFSET(CPU_FTR_HVMODE) 1309 ld r10,PACAKMSR(r13) 1310 li r3,MSR_ME 1311 andc r10,r10,r3 1312 mtmsrd r10 1313 1314 lhz r12,PACA_IN_MCE(r13) 1315 subi r12,r12,1 1316 sth r12,PACA_IN_MCE(r13) 1317 1318 /* Invoke machine_check_exception to print MCE event and panic. */ 1319 addi r3,r1,STACK_FRAME_OVERHEAD 1320 bl machine_check_exception 1321 1322 /* 1323 * We will not reach here. Even if we did, there is no way out. 1324 * Call unrecoverable_exception and die. 1325 */ 1326 addi r3,r1,STACK_FRAME_OVERHEAD 1327 bl unrecoverable_exception 1328 b . 1329 1330 1331/** 1332 * Interrupt 0x300 - Data Storage Interrupt (DSI). 1333 * This is a synchronous interrupt generated due to a data access exception, 1334 * e.g., a load orstore which does not have a valid page table entry with 1335 * permissions. DAWR matches also fault here, as do RC updates, and minor misc 1336 * errors e.g., copy/paste, AMO, certain invalid CI accesses, etc. 1337 * 1338 * Handling: 1339 * - Hash MMU 1340 * Go to do_hash_fault, which attempts to fill the HPT from an entry in the 1341 * Linux page table. Hash faults can hit in kernel mode in a fairly 1342 * arbitrary state (e.g., interrupts disabled, locks held) when accessing 1343 * "non-bolted" regions, e.g., vmalloc space. However these should always be 1344 * backed by Linux page table entries. 1345 * 1346 * If no entry is found the Linux page fault handler is invoked (by 1347 * do_hash_fault). Linux page faults can happen in kernel mode due to user 1348 * copy operations of course. 1349 * 1350 * KVM: The KVM HDSI handler may perform a load with MSR[DR]=1 in guest 1351 * MMU context, which may cause a DSI in the host, which must go to the 1352 * KVM handler. MSR[IR] is not enabled, so the real-mode handler will 1353 * always be used regardless of AIL setting. 1354 * 1355 * - Radix MMU 1356 * The hardware loads from the Linux page table directly, so a fault goes 1357 * immediately to Linux page fault. 1358 * 1359 * Conditions like DAWR match are handled on the way in to Linux page fault. 1360 */ 1361INT_DEFINE_BEGIN(data_access) 1362 IVEC=0x300 1363 IDAR=1 1364 IDSISR=1 1365 IKVM_SKIP=1 1366 IKVM_REAL=1 1367INT_DEFINE_END(data_access) 1368 1369EXC_REAL_BEGIN(data_access, 0x300, 0x80) 1370 GEN_INT_ENTRY data_access, virt=0 1371EXC_REAL_END(data_access, 0x300, 0x80) 1372EXC_VIRT_BEGIN(data_access, 0x4300, 0x80) 1373 GEN_INT_ENTRY data_access, virt=1 1374EXC_VIRT_END(data_access, 0x4300, 0x80) 1375EXC_COMMON_BEGIN(data_access_common) 1376 GEN_COMMON data_access 1377 ld r4,_DSISR(r1) 1378 addi r3,r1,STACK_FRAME_OVERHEAD 1379 andis. r0,r4,DSISR_DABRMATCH@h 1380 bne- 1f 1381BEGIN_MMU_FTR_SECTION 1382 bl do_hash_fault 1383MMU_FTR_SECTION_ELSE 1384 bl do_page_fault 1385ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) 1386 b interrupt_return 1387 13881: bl do_break 1389 /* 1390 * do_break() may have changed the NV GPRS while handling a breakpoint. 1391 * If so, we need to restore them with their updated values. 1392 */ 1393 REST_NVGPRS(r1) 1394 b interrupt_return 1395 1396 GEN_KVM data_access 1397 1398 1399/** 1400 * Interrupt 0x380 - Data Segment Interrupt (DSLB). 1401 * This is a synchronous interrupt in response to an MMU fault missing SLB 1402 * entry for HPT, or an address outside RPT translation range. 1403 * 1404 * Handling: 1405 * - HPT: 1406 * This refills the SLB, or reports an access fault similarly to a bad page 1407 * fault. When coming from user-mode, the SLB handler may access any kernel 1408 * data, though it may itself take a DSLB. When coming from kernel mode, 1409 * recursive faults must be avoided so access is restricted to the kernel 1410 * image text/data, kernel stack, and any data allocated below 1411 * ppc64_bolted_size (first segment). The kernel handler must avoid stomping 1412 * on user-handler data structures. 1413 * 1414 * KVM: Same as 0x300, DSLB must test for KVM guest. 1415 */ 1416INT_DEFINE_BEGIN(data_access_slb) 1417 IVEC=0x380 1418 IDAR=1 1419 IKVM_SKIP=1 1420 IKVM_REAL=1 1421INT_DEFINE_END(data_access_slb) 1422 1423EXC_REAL_BEGIN(data_access_slb, 0x380, 0x80) 1424 GEN_INT_ENTRY data_access_slb, virt=0 1425EXC_REAL_END(data_access_slb, 0x380, 0x80) 1426EXC_VIRT_BEGIN(data_access_slb, 0x4380, 0x80) 1427 GEN_INT_ENTRY data_access_slb, virt=1 1428EXC_VIRT_END(data_access_slb, 0x4380, 0x80) 1429EXC_COMMON_BEGIN(data_access_slb_common) 1430 GEN_COMMON data_access_slb 1431BEGIN_MMU_FTR_SECTION 1432 /* HPT case, do SLB fault */ 1433 addi r3,r1,STACK_FRAME_OVERHEAD 1434 bl do_slb_fault 1435 cmpdi r3,0 1436 bne- 1f 1437 b fast_interrupt_return 14381: /* Error case */ 1439MMU_FTR_SECTION_ELSE 1440 /* Radix case, access is outside page table range */ 1441 li r3,-EFAULT 1442ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) 1443 std r3,RESULT(r1) 1444 addi r3,r1,STACK_FRAME_OVERHEAD 1445 bl do_bad_slb_fault 1446 b interrupt_return 1447 1448 GEN_KVM data_access_slb 1449 1450 1451/** 1452 * Interrupt 0x400 - Instruction Storage Interrupt (ISI). 1453 * This is a synchronous interrupt in response to an MMU fault due to an 1454 * instruction fetch. 1455 * 1456 * Handling: 1457 * Similar to DSI, though in response to fetch. The faulting address is found 1458 * in SRR0 (rather than DAR), and status in SRR1 (rather than DSISR). 1459 */ 1460INT_DEFINE_BEGIN(instruction_access) 1461 IVEC=0x400 1462 IISIDE=1 1463 IDAR=1 1464 IDSISR=1 1465#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 1466 IKVM_REAL=1 1467#endif 1468INT_DEFINE_END(instruction_access) 1469 1470EXC_REAL_BEGIN(instruction_access, 0x400, 0x80) 1471 GEN_INT_ENTRY instruction_access, virt=0 1472EXC_REAL_END(instruction_access, 0x400, 0x80) 1473EXC_VIRT_BEGIN(instruction_access, 0x4400, 0x80) 1474 GEN_INT_ENTRY instruction_access, virt=1 1475EXC_VIRT_END(instruction_access, 0x4400, 0x80) 1476EXC_COMMON_BEGIN(instruction_access_common) 1477 GEN_COMMON instruction_access 1478 addi r3,r1,STACK_FRAME_OVERHEAD 1479BEGIN_MMU_FTR_SECTION 1480 bl do_hash_fault 1481MMU_FTR_SECTION_ELSE 1482 bl do_page_fault 1483ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) 1484 b interrupt_return 1485 1486 GEN_KVM instruction_access 1487 1488 1489/** 1490 * Interrupt 0x480 - Instruction Segment Interrupt (ISLB). 1491 * This is a synchronous interrupt in response to an MMU fault due to an 1492 * instruction fetch. 1493 * 1494 * Handling: 1495 * Similar to DSLB, though in response to fetch. The faulting address is found 1496 * in SRR0 (rather than DAR). 1497 */ 1498INT_DEFINE_BEGIN(instruction_access_slb) 1499 IVEC=0x480 1500 IISIDE=1 1501 IDAR=1 1502#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 1503 IKVM_REAL=1 1504#endif 1505INT_DEFINE_END(instruction_access_slb) 1506 1507EXC_REAL_BEGIN(instruction_access_slb, 0x480, 0x80) 1508 GEN_INT_ENTRY instruction_access_slb, virt=0 1509EXC_REAL_END(instruction_access_slb, 0x480, 0x80) 1510EXC_VIRT_BEGIN(instruction_access_slb, 0x4480, 0x80) 1511 GEN_INT_ENTRY instruction_access_slb, virt=1 1512EXC_VIRT_END(instruction_access_slb, 0x4480, 0x80) 1513EXC_COMMON_BEGIN(instruction_access_slb_common) 1514 GEN_COMMON instruction_access_slb 1515BEGIN_MMU_FTR_SECTION 1516 /* HPT case, do SLB fault */ 1517 addi r3,r1,STACK_FRAME_OVERHEAD 1518 bl do_slb_fault 1519 cmpdi r3,0 1520 bne- 1f 1521 b fast_interrupt_return 15221: /* Error case */ 1523MMU_FTR_SECTION_ELSE 1524 /* Radix case, access is outside page table range */ 1525 li r3,-EFAULT 1526ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) 1527 std r3,RESULT(r1) 1528 addi r3,r1,STACK_FRAME_OVERHEAD 1529 bl do_bad_slb_fault 1530 b interrupt_return 1531 1532 GEN_KVM instruction_access_slb 1533 1534 1535/** 1536 * Interrupt 0x500 - External Interrupt. 1537 * This is an asynchronous maskable interrupt in response to an "external 1538 * exception" from the interrupt controller or hypervisor (e.g., device 1539 * interrupt). It is maskable in hardware by clearing MSR[EE], and 1540 * soft-maskable with IRQS_DISABLED mask (i.e., local_irq_disable()). 1541 * 1542 * When running in HV mode, Linux sets up the LPCR[LPES] bit such that 1543 * interrupts are delivered with HSRR registers, guests use SRRs, which 1544 * reqiures IHSRR_IF_HVMODE. 1545 * 1546 * On bare metal POWER9 and later, Linux sets the LPCR[HVICE] bit such that 1547 * external interrupts are delivered as Hypervisor Virtualization Interrupts 1548 * rather than External Interrupts. 1549 * 1550 * Handling: 1551 * This calls into Linux IRQ handler. NVGPRs are not saved to reduce overhead, 1552 * because registers at the time of the interrupt are not so important as it is 1553 * asynchronous. 1554 * 1555 * If soft masked, the masked handler will note the pending interrupt for 1556 * replay, and clear MSR[EE] in the interrupted context. 1557 */ 1558INT_DEFINE_BEGIN(hardware_interrupt) 1559 IVEC=0x500 1560 IHSRR_IF_HVMODE=1 1561 IMASK=IRQS_DISABLED 1562 IKVM_REAL=1 1563 IKVM_VIRT=1 1564INT_DEFINE_END(hardware_interrupt) 1565 1566EXC_REAL_BEGIN(hardware_interrupt, 0x500, 0x100) 1567 GEN_INT_ENTRY hardware_interrupt, virt=0 1568EXC_REAL_END(hardware_interrupt, 0x500, 0x100) 1569EXC_VIRT_BEGIN(hardware_interrupt, 0x4500, 0x100) 1570 GEN_INT_ENTRY hardware_interrupt, virt=1 1571EXC_VIRT_END(hardware_interrupt, 0x4500, 0x100) 1572EXC_COMMON_BEGIN(hardware_interrupt_common) 1573 GEN_COMMON hardware_interrupt 1574 FINISH_NAP 1575 addi r3,r1,STACK_FRAME_OVERHEAD 1576 bl do_IRQ 1577 b interrupt_return 1578 1579 GEN_KVM hardware_interrupt 1580 1581 1582/** 1583 * Interrupt 0x600 - Alignment Interrupt 1584 * This is a synchronous interrupt in response to data alignment fault. 1585 */ 1586INT_DEFINE_BEGIN(alignment) 1587 IVEC=0x600 1588 IDAR=1 1589 IDSISR=1 1590#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 1591 IKVM_REAL=1 1592#endif 1593INT_DEFINE_END(alignment) 1594 1595EXC_REAL_BEGIN(alignment, 0x600, 0x100) 1596 GEN_INT_ENTRY alignment, virt=0 1597EXC_REAL_END(alignment, 0x600, 0x100) 1598EXC_VIRT_BEGIN(alignment, 0x4600, 0x100) 1599 GEN_INT_ENTRY alignment, virt=1 1600EXC_VIRT_END(alignment, 0x4600, 0x100) 1601EXC_COMMON_BEGIN(alignment_common) 1602 GEN_COMMON alignment 1603 addi r3,r1,STACK_FRAME_OVERHEAD 1604 bl alignment_exception 1605 REST_NVGPRS(r1) /* instruction emulation may change GPRs */ 1606 b interrupt_return 1607 1608 GEN_KVM alignment 1609 1610 1611/** 1612 * Interrupt 0x700 - Program Interrupt (program check). 1613 * This is a synchronous interrupt in response to various instruction faults: 1614 * traps, privilege errors, TM errors, floating point exceptions. 1615 * 1616 * Handling: 1617 * This interrupt may use the "emergency stack" in some cases when being taken 1618 * from kernel context, which complicates handling. 1619 */ 1620INT_DEFINE_BEGIN(program_check) 1621 IVEC=0x700 1622#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 1623 IKVM_REAL=1 1624#endif 1625INT_DEFINE_END(program_check) 1626 1627EXC_REAL_BEGIN(program_check, 0x700, 0x100) 1628 1629#ifdef CONFIG_CPU_LITTLE_ENDIAN 1630 /* 1631 * There's a short window during boot where although the kernel is 1632 * running little endian, any exceptions will cause the CPU to switch 1633 * back to big endian. For example a WARN() boils down to a trap 1634 * instruction, which will cause a program check, and we end up here but 1635 * with the CPU in big endian mode. The first instruction of the program 1636 * check handler (in GEN_INT_ENTRY below) is an mtsprg, which when 1637 * executed in the wrong endian is an lhzu with a ~3GB displacement from 1638 * r3. The content of r3 is random, so that is a load from some random 1639 * location, and depending on the system can easily lead to a checkstop, 1640 * or an infinitely recursive page fault. 1641 * 1642 * So to handle that case we have a trampoline here that can detect we 1643 * are in the wrong endian and flip us back to the correct endian. We 1644 * can't flip MSR[LE] using mtmsr, so we have to use rfid. That requires 1645 * backing up SRR0/1 as well as a GPR. To do that we use SPRG0/2/3, as 1646 * SPRG1 is already used for the paca. SPRG3 is user readable, but this 1647 * trampoline is only active very early in boot, and SPRG3 will be 1648 * reinitialised in vdso_getcpu_init() before userspace starts. 1649 */ 1650BEGIN_FTR_SECTION 1651 tdi 0,0,0x48 // Trap never, or in reverse endian: b . + 8 1652 b 1f // Skip trampoline if endian is correct 1653 .long 0xa643707d // mtsprg 0, r11 Backup r11 1654 .long 0xa6027a7d // mfsrr0 r11 1655 .long 0xa643727d // mtsprg 2, r11 Backup SRR0 in SPRG2 1656 .long 0xa6027b7d // mfsrr1 r11 1657 .long 0xa643737d // mtsprg 3, r11 Backup SRR1 in SPRG3 1658 .long 0xa600607d // mfmsr r11 1659 .long 0x01006b69 // xori r11, r11, 1 Invert MSR[LE] 1660 .long 0xa6037b7d // mtsrr1 r11 1661 .long 0x34076039 // li r11, 0x734 1662 .long 0xa6037a7d // mtsrr0 r11 1663 .long 0x2400004c // rfid 1664 mfsprg r11, 3 1665 mtsrr1 r11 // Restore SRR1 1666 mfsprg r11, 2 1667 mtsrr0 r11 // Restore SRR0 1668 mfsprg r11, 0 // Restore r11 16691: 1670END_FTR_SECTION(0, 1) // nop out after boot 1671#endif /* CONFIG_CPU_LITTLE_ENDIAN */ 1672 1673 GEN_INT_ENTRY program_check, virt=0 1674EXC_REAL_END(program_check, 0x700, 0x100) 1675EXC_VIRT_BEGIN(program_check, 0x4700, 0x100) 1676 GEN_INT_ENTRY program_check, virt=1 1677EXC_VIRT_END(program_check, 0x4700, 0x100) 1678EXC_COMMON_BEGIN(program_check_common) 1679 __GEN_COMMON_ENTRY program_check 1680 1681 /* 1682 * It's possible to receive a TM Bad Thing type program check with 1683 * userspace register values (in particular r1), but with SRR1 reporting 1684 * that we came from the kernel. Normally that would confuse the bad 1685 * stack logic, and we would report a bad kernel stack pointer. Instead 1686 * we switch to the emergency stack if we're taking a TM Bad Thing from 1687 * the kernel. 1688 */ 1689 1690 andi. r10,r12,MSR_PR 1691 bne 2f /* If userspace, go normal path */ 1692 1693 andis. r10,r12,(SRR1_PROGTM)@h 1694 bne 1f /* If TM, emergency */ 1695 1696 cmpdi r1,-INT_FRAME_SIZE /* check if r1 is in userspace */ 1697 blt 2f /* normal path if not */ 1698 1699 /* Use the emergency stack */ 17001: andi. r10,r12,MSR_PR /* Set CR0 correctly for label */ 1701 /* 3 in EXCEPTION_PROLOG_COMMON */ 1702 mr r10,r1 /* Save r1 */ 1703 ld r1,PACAEMERGSP(r13) /* Use emergency stack */ 1704 subi r1,r1,INT_FRAME_SIZE /* alloc stack frame */ 1705 __ISTACK(program_check)=0 1706 __GEN_COMMON_BODY program_check 1707 b 3f 17082: 1709 __ISTACK(program_check)=1 1710 __GEN_COMMON_BODY program_check 17113: 1712 addi r3,r1,STACK_FRAME_OVERHEAD 1713 bl program_check_exception 1714 REST_NVGPRS(r1) /* instruction emulation may change GPRs */ 1715 b interrupt_return 1716 1717 GEN_KVM program_check 1718 1719 1720/* 1721 * Interrupt 0x800 - Floating-Point Unavailable Interrupt. 1722 * This is a synchronous interrupt in response to executing an fp instruction 1723 * with MSR[FP]=0. 1724 * 1725 * Handling: 1726 * This will load FP registers and enable the FP bit if coming from userspace, 1727 * otherwise report a bad kernel use of FP. 1728 */ 1729INT_DEFINE_BEGIN(fp_unavailable) 1730 IVEC=0x800 1731#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 1732 IKVM_REAL=1 1733#endif 1734INT_DEFINE_END(fp_unavailable) 1735 1736EXC_REAL_BEGIN(fp_unavailable, 0x800, 0x100) 1737 GEN_INT_ENTRY fp_unavailable, virt=0 1738EXC_REAL_END(fp_unavailable, 0x800, 0x100) 1739EXC_VIRT_BEGIN(fp_unavailable, 0x4800, 0x100) 1740 GEN_INT_ENTRY fp_unavailable, virt=1 1741EXC_VIRT_END(fp_unavailable, 0x4800, 0x100) 1742EXC_COMMON_BEGIN(fp_unavailable_common) 1743 GEN_COMMON fp_unavailable 1744 bne 1f /* if from user, just load it up */ 1745 addi r3,r1,STACK_FRAME_OVERHEAD 1746 bl kernel_fp_unavailable_exception 17470: trap 1748 EMIT_BUG_ENTRY 0b, __FILE__, __LINE__, 0 17491: 1750#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 1751BEGIN_FTR_SECTION 1752 /* Test if 2 TM state bits are zero. If non-zero (ie. userspace was in 1753 * transaction), go do TM stuff 1754 */ 1755 rldicl. r0, r12, (64-MSR_TS_LG), (64-2) 1756 bne- 2f 1757END_FTR_SECTION_IFSET(CPU_FTR_TM) 1758#endif 1759 bl load_up_fpu 1760 b fast_interrupt_return 1761#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 17622: /* User process was in a transaction */ 1763 addi r3,r1,STACK_FRAME_OVERHEAD 1764 bl fp_unavailable_tm 1765 b interrupt_return 1766#endif 1767 1768 GEN_KVM fp_unavailable 1769 1770 1771/** 1772 * Interrupt 0x900 - Decrementer Interrupt. 1773 * This is an asynchronous interrupt in response to a decrementer exception 1774 * (e.g., DEC has wrapped below zero). It is maskable in hardware by clearing 1775 * MSR[EE], and soft-maskable with IRQS_DISABLED mask (i.e., 1776 * local_irq_disable()). 1777 * 1778 * Handling: 1779 * This calls into Linux timer handler. NVGPRs are not saved (see 0x500). 1780 * 1781 * If soft masked, the masked handler will note the pending interrupt for 1782 * replay, and bump the decrementer to a high value, leaving MSR[EE] enabled 1783 * in the interrupted context. 1784 * If PPC_WATCHDOG is configured, the soft masked handler will actually set 1785 * things back up to run soft_nmi_interrupt as a regular interrupt handler 1786 * on the emergency stack. 1787 */ 1788INT_DEFINE_BEGIN(decrementer) 1789 IVEC=0x900 1790 IMASK=IRQS_DISABLED 1791#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 1792 IKVM_REAL=1 1793#endif 1794INT_DEFINE_END(decrementer) 1795 1796EXC_REAL_BEGIN(decrementer, 0x900, 0x80) 1797 GEN_INT_ENTRY decrementer, virt=0 1798EXC_REAL_END(decrementer, 0x900, 0x80) 1799EXC_VIRT_BEGIN(decrementer, 0x4900, 0x80) 1800 GEN_INT_ENTRY decrementer, virt=1 1801EXC_VIRT_END(decrementer, 0x4900, 0x80) 1802EXC_COMMON_BEGIN(decrementer_common) 1803 GEN_COMMON decrementer 1804 FINISH_NAP 1805 addi r3,r1,STACK_FRAME_OVERHEAD 1806 bl timer_interrupt 1807 b interrupt_return 1808 1809 GEN_KVM decrementer 1810 1811 1812/** 1813 * Interrupt 0x980 - Hypervisor Decrementer Interrupt. 1814 * This is an asynchronous interrupt, similar to 0x900 but for the HDEC 1815 * register. 1816 * 1817 * Handling: 1818 * Linux does not use this outside KVM where it's used to keep a host timer 1819 * while the guest is given control of DEC. It should normally be caught by 1820 * the KVM test and routed there. 1821 */ 1822INT_DEFINE_BEGIN(hdecrementer) 1823 IVEC=0x980 1824 IHSRR=1 1825 ISTACK=0 1826 IKVM_REAL=1 1827 IKVM_VIRT=1 1828INT_DEFINE_END(hdecrementer) 1829 1830EXC_REAL_BEGIN(hdecrementer, 0x980, 0x80) 1831 GEN_INT_ENTRY hdecrementer, virt=0 1832EXC_REAL_END(hdecrementer, 0x980, 0x80) 1833EXC_VIRT_BEGIN(hdecrementer, 0x4980, 0x80) 1834 GEN_INT_ENTRY hdecrementer, virt=1 1835EXC_VIRT_END(hdecrementer, 0x4980, 0x80) 1836EXC_COMMON_BEGIN(hdecrementer_common) 1837 __GEN_COMMON_ENTRY hdecrementer 1838 /* 1839 * Hypervisor decrementer interrupts not caught by the KVM test 1840 * shouldn't occur but are sometimes left pending on exit from a KVM 1841 * guest. We don't need to do anything to clear them, as they are 1842 * edge-triggered. 1843 * 1844 * Be careful to avoid touching the kernel stack. 1845 */ 1846 ld r10,PACA_EXGEN+EX_CTR(r13) 1847 mtctr r10 1848 mtcrf 0x80,r9 1849 ld r9,PACA_EXGEN+EX_R9(r13) 1850 ld r10,PACA_EXGEN+EX_R10(r13) 1851 ld r11,PACA_EXGEN+EX_R11(r13) 1852 ld r12,PACA_EXGEN+EX_R12(r13) 1853 ld r13,PACA_EXGEN+EX_R13(r13) 1854 HRFI_TO_KERNEL 1855 1856 GEN_KVM hdecrementer 1857 1858 1859/** 1860 * Interrupt 0xa00 - Directed Privileged Doorbell Interrupt. 1861 * This is an asynchronous interrupt in response to a msgsndp doorbell. 1862 * It is maskable in hardware by clearing MSR[EE], and soft-maskable with 1863 * IRQS_DISABLED mask (i.e., local_irq_disable()). 1864 * 1865 * Handling: 1866 * Guests may use this for IPIs between threads in a core if the 1867 * hypervisor supports it. NVGPRS are not saved (see 0x500). 1868 * 1869 * If soft masked, the masked handler will note the pending interrupt for 1870 * replay, leaving MSR[EE] enabled in the interrupted context because the 1871 * doorbells are edge triggered. 1872 */ 1873INT_DEFINE_BEGIN(doorbell_super) 1874 IVEC=0xa00 1875 IMASK=IRQS_DISABLED 1876#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 1877 IKVM_REAL=1 1878#endif 1879INT_DEFINE_END(doorbell_super) 1880 1881EXC_REAL_BEGIN(doorbell_super, 0xa00, 0x100) 1882 GEN_INT_ENTRY doorbell_super, virt=0 1883EXC_REAL_END(doorbell_super, 0xa00, 0x100) 1884EXC_VIRT_BEGIN(doorbell_super, 0x4a00, 0x100) 1885 GEN_INT_ENTRY doorbell_super, virt=1 1886EXC_VIRT_END(doorbell_super, 0x4a00, 0x100) 1887EXC_COMMON_BEGIN(doorbell_super_common) 1888 GEN_COMMON doorbell_super 1889 FINISH_NAP 1890 addi r3,r1,STACK_FRAME_OVERHEAD 1891#ifdef CONFIG_PPC_DOORBELL 1892 bl doorbell_exception 1893#else 1894 bl unknown_async_exception 1895#endif 1896 b interrupt_return 1897 1898 GEN_KVM doorbell_super 1899 1900 1901EXC_REAL_NONE(0xb00, 0x100) 1902EXC_VIRT_NONE(0x4b00, 0x100) 1903 1904/** 1905 * Interrupt 0xc00 - System Call Interrupt (syscall, hcall). 1906 * This is a synchronous interrupt invoked with the "sc" instruction. The 1907 * system call is invoked with "sc 0" and does not alter the HV bit, so it 1908 * is directed to the currently running OS. The hypercall is invoked with 1909 * "sc 1" and it sets HV=1, so it elevates to hypervisor. 1910 * 1911 * In HPT, sc 1 always goes to 0xc00 real mode. In RADIX, sc 1 can go to 1912 * 0x4c00 virtual mode. 1913 * 1914 * Handling: 1915 * If the KVM test fires then it was due to a hypercall and is accordingly 1916 * routed to KVM. Otherwise this executes a normal Linux system call. 1917 * 1918 * Call convention: 1919 * 1920 * syscall and hypercalls register conventions are documented in 1921 * Documentation/powerpc/syscall64-abi.rst and 1922 * Documentation/powerpc/papr_hcalls.rst respectively. 1923 * 1924 * The intersection of volatile registers that don't contain possible 1925 * inputs is: cr0, xer, ctr. We may use these as scratch regs upon entry 1926 * without saving, though xer is not a good idea to use, as hardware may 1927 * interpret some bits so it may be costly to change them. 1928 */ 1929INT_DEFINE_BEGIN(system_call) 1930 IVEC=0xc00 1931 IKVM_REAL=1 1932 IKVM_VIRT=1 1933INT_DEFINE_END(system_call) 1934 1935.macro SYSTEM_CALL virt 1936#ifdef CONFIG_KVM_BOOK3S_64_HANDLER 1937 /* 1938 * There is a little bit of juggling to get syscall and hcall 1939 * working well. Save r13 in ctr to avoid using SPRG scratch 1940 * register. 1941 * 1942 * Userspace syscalls have already saved the PPR, hcalls must save 1943 * it before setting HMT_MEDIUM. 1944 */ 1945 mtctr r13 1946 GET_PACA(r13) 1947 std r10,PACA_EXGEN+EX_R10(r13) 1948 INTERRUPT_TO_KERNEL 1949 KVMTEST system_call /* uses r10, branch to system_call_kvm */ 1950 mfctr r9 1951#else 1952 mr r9,r13 1953 GET_PACA(r13) 1954 INTERRUPT_TO_KERNEL 1955#endif 1956 1957#ifdef CONFIG_PPC_FAST_ENDIAN_SWITCH 1958BEGIN_FTR_SECTION 1959 cmpdi r0,0x1ebe 1960 beq- 1f 1961END_FTR_SECTION_IFSET(CPU_FTR_REAL_LE) 1962#endif 1963 1964 /* We reach here with PACA in r13, r13 in r9. */ 1965 mfspr r11,SPRN_SRR0 1966 mfspr r12,SPRN_SRR1 1967 1968 HMT_MEDIUM 1969 1970 .if ! \virt 1971 __LOAD_HANDLER(r10, system_call_common_real) 1972 mtctr r10 1973 bctr 1974 .else 1975 li r10,MSR_RI 1976 mtmsrd r10,1 /* Set RI (EE=0) */ 1977#ifdef CONFIG_RELOCATABLE 1978 __LOAD_HANDLER(r10, system_call_common) 1979 mtctr r10 1980 bctr 1981#else 1982 b system_call_common 1983#endif 1984 .endif 1985 1986#ifdef CONFIG_PPC_FAST_ENDIAN_SWITCH 1987 /* Fast LE/BE switch system call */ 19881: mfspr r12,SPRN_SRR1 1989 xori r12,r12,MSR_LE 1990 mtspr SPRN_SRR1,r12 1991 mr r13,r9 1992 RFI_TO_USER /* return to userspace */ 1993 b . /* prevent speculative execution */ 1994#endif 1995.endm 1996 1997EXC_REAL_BEGIN(system_call, 0xc00, 0x100) 1998 SYSTEM_CALL 0 1999EXC_REAL_END(system_call, 0xc00, 0x100) 2000EXC_VIRT_BEGIN(system_call, 0x4c00, 0x100) 2001 SYSTEM_CALL 1 2002EXC_VIRT_END(system_call, 0x4c00, 0x100) 2003 2004#ifdef CONFIG_KVM_BOOK3S_64_HANDLER 2005TRAMP_REAL_BEGIN(system_call_kvm) 2006 /* 2007 * This is a hcall, so register convention is as above, with these 2008 * differences: 2009 * r13 = PACA 2010 * ctr = orig r13 2011 * orig r10 saved in PACA 2012 */ 2013 /* 2014 * Save the PPR (on systems that support it) before changing to 2015 * HMT_MEDIUM. That allows the KVM code to save that value into the 2016 * guest state (it is the guest's PPR value). 2017 */ 2018BEGIN_FTR_SECTION 2019 mfspr r10,SPRN_PPR 2020 std r10,HSTATE_PPR(r13) 2021END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 2022 HMT_MEDIUM 2023 mfctr r10 2024 SET_SCRATCH0(r10) 2025 mfcr r10 2026 std r12,HSTATE_SCRATCH0(r13) 2027 sldi r12,r10,32 2028 ori r12,r12,0xc00 2029#ifdef CONFIG_RELOCATABLE 2030 /* 2031 * Requires __LOAD_FAR_HANDLER beause kvmppc_interrupt lives 2032 * outside the head section. 2033 */ 2034 __LOAD_FAR_HANDLER(r10, kvmppc_interrupt) 2035 mtctr r10 2036 ld r10,PACA_EXGEN+EX_R10(r13) 2037 bctr 2038#else 2039 ld r10,PACA_EXGEN+EX_R10(r13) 2040 b kvmppc_interrupt 2041#endif 2042#endif 2043 2044 2045/** 2046 * Interrupt 0xd00 - Trace Interrupt. 2047 * This is a synchronous interrupt in response to instruction step or 2048 * breakpoint faults. 2049 */ 2050INT_DEFINE_BEGIN(single_step) 2051 IVEC=0xd00 2052#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 2053 IKVM_REAL=1 2054#endif 2055INT_DEFINE_END(single_step) 2056 2057EXC_REAL_BEGIN(single_step, 0xd00, 0x100) 2058 GEN_INT_ENTRY single_step, virt=0 2059EXC_REAL_END(single_step, 0xd00, 0x100) 2060EXC_VIRT_BEGIN(single_step, 0x4d00, 0x100) 2061 GEN_INT_ENTRY single_step, virt=1 2062EXC_VIRT_END(single_step, 0x4d00, 0x100) 2063EXC_COMMON_BEGIN(single_step_common) 2064 GEN_COMMON single_step 2065 addi r3,r1,STACK_FRAME_OVERHEAD 2066 bl single_step_exception 2067 b interrupt_return 2068 2069 GEN_KVM single_step 2070 2071 2072/** 2073 * Interrupt 0xe00 - Hypervisor Data Storage Interrupt (HDSI). 2074 * This is a synchronous interrupt in response to an MMU fault caused by a 2075 * guest data access. 2076 * 2077 * Handling: 2078 * This should always get routed to KVM. In radix MMU mode, this is caused 2079 * by a guest nested radix access that can't be performed due to the 2080 * partition scope page table. In hash mode, this can be caused by guests 2081 * running with translation disabled (virtual real mode) or with VPM enabled. 2082 * KVM will update the page table structures or disallow the access. 2083 */ 2084INT_DEFINE_BEGIN(h_data_storage) 2085 IVEC=0xe00 2086 IHSRR=1 2087 IDAR=1 2088 IDSISR=1 2089 IKVM_SKIP=1 2090 IKVM_REAL=1 2091 IKVM_VIRT=1 2092INT_DEFINE_END(h_data_storage) 2093 2094EXC_REAL_BEGIN(h_data_storage, 0xe00, 0x20) 2095 GEN_INT_ENTRY h_data_storage, virt=0, ool=1 2096EXC_REAL_END(h_data_storage, 0xe00, 0x20) 2097EXC_VIRT_BEGIN(h_data_storage, 0x4e00, 0x20) 2098 GEN_INT_ENTRY h_data_storage, virt=1, ool=1 2099EXC_VIRT_END(h_data_storage, 0x4e00, 0x20) 2100EXC_COMMON_BEGIN(h_data_storage_common) 2101 GEN_COMMON h_data_storage 2102 addi r3,r1,STACK_FRAME_OVERHEAD 2103BEGIN_MMU_FTR_SECTION 2104 bl do_bad_page_fault_segv 2105MMU_FTR_SECTION_ELSE 2106 bl unknown_exception 2107ALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_TYPE_RADIX) 2108 b interrupt_return 2109 2110 GEN_KVM h_data_storage 2111 2112 2113/** 2114 * Interrupt 0xe20 - Hypervisor Instruction Storage Interrupt (HISI). 2115 * This is a synchronous interrupt in response to an MMU fault caused by a 2116 * guest instruction fetch, similar to HDSI. 2117 */ 2118INT_DEFINE_BEGIN(h_instr_storage) 2119 IVEC=0xe20 2120 IHSRR=1 2121 IKVM_REAL=1 2122 IKVM_VIRT=1 2123INT_DEFINE_END(h_instr_storage) 2124 2125EXC_REAL_BEGIN(h_instr_storage, 0xe20, 0x20) 2126 GEN_INT_ENTRY h_instr_storage, virt=0, ool=1 2127EXC_REAL_END(h_instr_storage, 0xe20, 0x20) 2128EXC_VIRT_BEGIN(h_instr_storage, 0x4e20, 0x20) 2129 GEN_INT_ENTRY h_instr_storage, virt=1, ool=1 2130EXC_VIRT_END(h_instr_storage, 0x4e20, 0x20) 2131EXC_COMMON_BEGIN(h_instr_storage_common) 2132 GEN_COMMON h_instr_storage 2133 addi r3,r1,STACK_FRAME_OVERHEAD 2134 bl unknown_exception 2135 b interrupt_return 2136 2137 GEN_KVM h_instr_storage 2138 2139 2140/** 2141 * Interrupt 0xe40 - Hypervisor Emulation Assistance Interrupt. 2142 */ 2143INT_DEFINE_BEGIN(emulation_assist) 2144 IVEC=0xe40 2145 IHSRR=1 2146 IKVM_REAL=1 2147 IKVM_VIRT=1 2148INT_DEFINE_END(emulation_assist) 2149 2150EXC_REAL_BEGIN(emulation_assist, 0xe40, 0x20) 2151 GEN_INT_ENTRY emulation_assist, virt=0, ool=1 2152EXC_REAL_END(emulation_assist, 0xe40, 0x20) 2153EXC_VIRT_BEGIN(emulation_assist, 0x4e40, 0x20) 2154 GEN_INT_ENTRY emulation_assist, virt=1, ool=1 2155EXC_VIRT_END(emulation_assist, 0x4e40, 0x20) 2156EXC_COMMON_BEGIN(emulation_assist_common) 2157 GEN_COMMON emulation_assist 2158 addi r3,r1,STACK_FRAME_OVERHEAD 2159 bl emulation_assist_interrupt 2160 REST_NVGPRS(r1) /* instruction emulation may change GPRs */ 2161 b interrupt_return 2162 2163 GEN_KVM emulation_assist 2164 2165 2166/** 2167 * Interrupt 0xe60 - Hypervisor Maintenance Interrupt (HMI). 2168 * This is an asynchronous interrupt caused by a Hypervisor Maintenance 2169 * Exception. It is always taken in real mode but uses HSRR registers 2170 * unlike SRESET and MCE. 2171 * 2172 * It is maskable in hardware by clearing MSR[EE], and partially soft-maskable 2173 * with IRQS_DISABLED mask (i.e., local_irq_disable()). 2174 * 2175 * Handling: 2176 * This is a special case, this is handled similarly to machine checks, with an 2177 * initial real mode handler that is not soft-masked, which attempts to fix the 2178 * problem. Then a regular handler which is soft-maskable and reports the 2179 * problem. 2180 * 2181 * The emergency stack is used for the early real mode handler. 2182 * 2183 * XXX: unclear why MCE and HMI schemes could not be made common, e.g., 2184 * either use soft-masking for the MCE, or use irq_work for the HMI. 2185 * 2186 * KVM: 2187 * Unlike MCE, this calls into KVM without calling the real mode handler 2188 * first. 2189 */ 2190INT_DEFINE_BEGIN(hmi_exception_early) 2191 IVEC=0xe60 2192 IHSRR=1 2193 IREALMODE_COMMON=1 2194 ISTACK=0 2195 IKUAP=0 /* We don't touch AMR here, we never go to virtual mode */ 2196 IKVM_REAL=1 2197INT_DEFINE_END(hmi_exception_early) 2198 2199INT_DEFINE_BEGIN(hmi_exception) 2200 IVEC=0xe60 2201 IHSRR=1 2202 IMASK=IRQS_DISABLED 2203 IKVM_REAL=1 2204INT_DEFINE_END(hmi_exception) 2205 2206EXC_REAL_BEGIN(hmi_exception, 0xe60, 0x20) 2207 GEN_INT_ENTRY hmi_exception_early, virt=0, ool=1 2208EXC_REAL_END(hmi_exception, 0xe60, 0x20) 2209EXC_VIRT_NONE(0x4e60, 0x20) 2210 2211EXC_COMMON_BEGIN(hmi_exception_early_common) 2212 __GEN_REALMODE_COMMON_ENTRY hmi_exception_early 2213 2214 mr r10,r1 /* Save r1 */ 2215 ld r1,PACAEMERGSP(r13) /* Use emergency stack for realmode */ 2216 subi r1,r1,INT_FRAME_SIZE /* alloc stack frame */ 2217 2218 __GEN_COMMON_BODY hmi_exception_early 2219 2220 addi r3,r1,STACK_FRAME_OVERHEAD 2221 bl hmi_exception_realmode 2222 cmpdi cr0,r3,0 2223 bne 1f 2224 2225 EXCEPTION_RESTORE_REGS hsrr=1 2226 HRFI_TO_USER_OR_KERNEL 2227 22281: 2229 /* 2230 * Go to virtual mode and pull the HMI event information from 2231 * firmware. 2232 */ 2233 EXCEPTION_RESTORE_REGS hsrr=1 2234 GEN_INT_ENTRY hmi_exception, virt=0 2235 2236 GEN_KVM hmi_exception_early 2237 2238EXC_COMMON_BEGIN(hmi_exception_common) 2239 GEN_COMMON hmi_exception 2240 FINISH_NAP 2241 addi r3,r1,STACK_FRAME_OVERHEAD 2242 bl handle_hmi_exception 2243 b interrupt_return 2244 2245 GEN_KVM hmi_exception 2246 2247 2248/** 2249 * Interrupt 0xe80 - Directed Hypervisor Doorbell Interrupt. 2250 * This is an asynchronous interrupt in response to a msgsnd doorbell. 2251 * Similar to the 0xa00 doorbell but for host rather than guest. 2252 */ 2253INT_DEFINE_BEGIN(h_doorbell) 2254 IVEC=0xe80 2255 IHSRR=1 2256 IMASK=IRQS_DISABLED 2257 IKVM_REAL=1 2258 IKVM_VIRT=1 2259INT_DEFINE_END(h_doorbell) 2260 2261EXC_REAL_BEGIN(h_doorbell, 0xe80, 0x20) 2262 GEN_INT_ENTRY h_doorbell, virt=0, ool=1 2263EXC_REAL_END(h_doorbell, 0xe80, 0x20) 2264EXC_VIRT_BEGIN(h_doorbell, 0x4e80, 0x20) 2265 GEN_INT_ENTRY h_doorbell, virt=1, ool=1 2266EXC_VIRT_END(h_doorbell, 0x4e80, 0x20) 2267EXC_COMMON_BEGIN(h_doorbell_common) 2268 GEN_COMMON h_doorbell 2269 FINISH_NAP 2270 addi r3,r1,STACK_FRAME_OVERHEAD 2271#ifdef CONFIG_PPC_DOORBELL 2272 bl doorbell_exception 2273#else 2274 bl unknown_async_exception 2275#endif 2276 b interrupt_return 2277 2278 GEN_KVM h_doorbell 2279 2280 2281/** 2282 * Interrupt 0xea0 - Hypervisor Virtualization Interrupt. 2283 * This is an asynchronous interrupt in response to an "external exception". 2284 * Similar to 0x500 but for host only. 2285 */ 2286INT_DEFINE_BEGIN(h_virt_irq) 2287 IVEC=0xea0 2288 IHSRR=1 2289 IMASK=IRQS_DISABLED 2290 IKVM_REAL=1 2291 IKVM_VIRT=1 2292INT_DEFINE_END(h_virt_irq) 2293 2294EXC_REAL_BEGIN(h_virt_irq, 0xea0, 0x20) 2295 GEN_INT_ENTRY h_virt_irq, virt=0, ool=1 2296EXC_REAL_END(h_virt_irq, 0xea0, 0x20) 2297EXC_VIRT_BEGIN(h_virt_irq, 0x4ea0, 0x20) 2298 GEN_INT_ENTRY h_virt_irq, virt=1, ool=1 2299EXC_VIRT_END(h_virt_irq, 0x4ea0, 0x20) 2300EXC_COMMON_BEGIN(h_virt_irq_common) 2301 GEN_COMMON h_virt_irq 2302 FINISH_NAP 2303 addi r3,r1,STACK_FRAME_OVERHEAD 2304 bl do_IRQ 2305 b interrupt_return 2306 2307 GEN_KVM h_virt_irq 2308 2309 2310EXC_REAL_NONE(0xec0, 0x20) 2311EXC_VIRT_NONE(0x4ec0, 0x20) 2312EXC_REAL_NONE(0xee0, 0x20) 2313EXC_VIRT_NONE(0x4ee0, 0x20) 2314 2315 2316/* 2317 * Interrupt 0xf00 - Performance Monitor Interrupt (PMI, PMU). 2318 * This is an asynchronous interrupt in response to a PMU exception. 2319 * It is maskable in hardware by clearing MSR[EE], and soft-maskable with 2320 * IRQS_PMI_DISABLED mask (NOTE: NOT local_irq_disable()). 2321 * 2322 * Handling: 2323 * This calls into the perf subsystem. 2324 * 2325 * Like the watchdog soft-nmi, it appears an NMI interrupt to Linux, in that it 2326 * runs under local_irq_disable. However it may be soft-masked in 2327 * powerpc-specific code. 2328 * 2329 * If soft masked, the masked handler will note the pending interrupt for 2330 * replay, and clear MSR[EE] in the interrupted context. 2331 */ 2332INT_DEFINE_BEGIN(performance_monitor) 2333 IVEC=0xf00 2334 IMASK=IRQS_PMI_DISABLED 2335#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 2336 IKVM_REAL=1 2337#endif 2338INT_DEFINE_END(performance_monitor) 2339 2340EXC_REAL_BEGIN(performance_monitor, 0xf00, 0x20) 2341 GEN_INT_ENTRY performance_monitor, virt=0, ool=1 2342EXC_REAL_END(performance_monitor, 0xf00, 0x20) 2343EXC_VIRT_BEGIN(performance_monitor, 0x4f00, 0x20) 2344 GEN_INT_ENTRY performance_monitor, virt=1, ool=1 2345EXC_VIRT_END(performance_monitor, 0x4f00, 0x20) 2346EXC_COMMON_BEGIN(performance_monitor_common) 2347 GEN_COMMON performance_monitor 2348 FINISH_NAP 2349 addi r3,r1,STACK_FRAME_OVERHEAD 2350 bl performance_monitor_exception 2351 b interrupt_return 2352 2353 GEN_KVM performance_monitor 2354 2355 2356/** 2357 * Interrupt 0xf20 - Vector Unavailable Interrupt. 2358 * This is a synchronous interrupt in response to 2359 * executing a vector (or altivec) instruction with MSR[VEC]=0. 2360 * Similar to FP unavailable. 2361 */ 2362INT_DEFINE_BEGIN(altivec_unavailable) 2363 IVEC=0xf20 2364#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 2365 IKVM_REAL=1 2366#endif 2367INT_DEFINE_END(altivec_unavailable) 2368 2369EXC_REAL_BEGIN(altivec_unavailable, 0xf20, 0x20) 2370 GEN_INT_ENTRY altivec_unavailable, virt=0, ool=1 2371EXC_REAL_END(altivec_unavailable, 0xf20, 0x20) 2372EXC_VIRT_BEGIN(altivec_unavailable, 0x4f20, 0x20) 2373 GEN_INT_ENTRY altivec_unavailable, virt=1, ool=1 2374EXC_VIRT_END(altivec_unavailable, 0x4f20, 0x20) 2375EXC_COMMON_BEGIN(altivec_unavailable_common) 2376 GEN_COMMON altivec_unavailable 2377#ifdef CONFIG_ALTIVEC 2378BEGIN_FTR_SECTION 2379 beq 1f 2380#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 2381 BEGIN_FTR_SECTION_NESTED(69) 2382 /* Test if 2 TM state bits are zero. If non-zero (ie. userspace was in 2383 * transaction), go do TM stuff 2384 */ 2385 rldicl. r0, r12, (64-MSR_TS_LG), (64-2) 2386 bne- 2f 2387 END_FTR_SECTION_NESTED(CPU_FTR_TM, CPU_FTR_TM, 69) 2388#endif 2389 bl load_up_altivec 2390 b fast_interrupt_return 2391#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 23922: /* User process was in a transaction */ 2393 addi r3,r1,STACK_FRAME_OVERHEAD 2394 bl altivec_unavailable_tm 2395 b interrupt_return 2396#endif 23971: 2398END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 2399#endif 2400 addi r3,r1,STACK_FRAME_OVERHEAD 2401 bl altivec_unavailable_exception 2402 b interrupt_return 2403 2404 GEN_KVM altivec_unavailable 2405 2406 2407/** 2408 * Interrupt 0xf40 - VSX Unavailable Interrupt. 2409 * This is a synchronous interrupt in response to 2410 * executing a VSX instruction with MSR[VSX]=0. 2411 * Similar to FP unavailable. 2412 */ 2413INT_DEFINE_BEGIN(vsx_unavailable) 2414 IVEC=0xf40 2415#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 2416 IKVM_REAL=1 2417#endif 2418INT_DEFINE_END(vsx_unavailable) 2419 2420EXC_REAL_BEGIN(vsx_unavailable, 0xf40, 0x20) 2421 GEN_INT_ENTRY vsx_unavailable, virt=0, ool=1 2422EXC_REAL_END(vsx_unavailable, 0xf40, 0x20) 2423EXC_VIRT_BEGIN(vsx_unavailable, 0x4f40, 0x20) 2424 GEN_INT_ENTRY vsx_unavailable, virt=1, ool=1 2425EXC_VIRT_END(vsx_unavailable, 0x4f40, 0x20) 2426EXC_COMMON_BEGIN(vsx_unavailable_common) 2427 GEN_COMMON vsx_unavailable 2428#ifdef CONFIG_VSX 2429BEGIN_FTR_SECTION 2430 beq 1f 2431#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 2432 BEGIN_FTR_SECTION_NESTED(69) 2433 /* Test if 2 TM state bits are zero. If non-zero (ie. userspace was in 2434 * transaction), go do TM stuff 2435 */ 2436 rldicl. r0, r12, (64-MSR_TS_LG), (64-2) 2437 bne- 2f 2438 END_FTR_SECTION_NESTED(CPU_FTR_TM, CPU_FTR_TM, 69) 2439#endif 2440 b load_up_vsx 2441#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 24422: /* User process was in a transaction */ 2443 addi r3,r1,STACK_FRAME_OVERHEAD 2444 bl vsx_unavailable_tm 2445 b interrupt_return 2446#endif 24471: 2448END_FTR_SECTION_IFSET(CPU_FTR_VSX) 2449#endif 2450 addi r3,r1,STACK_FRAME_OVERHEAD 2451 bl vsx_unavailable_exception 2452 b interrupt_return 2453 2454 GEN_KVM vsx_unavailable 2455 2456 2457/** 2458 * Interrupt 0xf60 - Facility Unavailable Interrupt. 2459 * This is a synchronous interrupt in response to 2460 * executing an instruction without access to the facility that can be 2461 * resolved by the OS (e.g., FSCR, MSR). 2462 * Similar to FP unavailable. 2463 */ 2464INT_DEFINE_BEGIN(facility_unavailable) 2465 IVEC=0xf60 2466#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 2467 IKVM_REAL=1 2468#endif 2469INT_DEFINE_END(facility_unavailable) 2470 2471EXC_REAL_BEGIN(facility_unavailable, 0xf60, 0x20) 2472 GEN_INT_ENTRY facility_unavailable, virt=0, ool=1 2473EXC_REAL_END(facility_unavailable, 0xf60, 0x20) 2474EXC_VIRT_BEGIN(facility_unavailable, 0x4f60, 0x20) 2475 GEN_INT_ENTRY facility_unavailable, virt=1, ool=1 2476EXC_VIRT_END(facility_unavailable, 0x4f60, 0x20) 2477EXC_COMMON_BEGIN(facility_unavailable_common) 2478 GEN_COMMON facility_unavailable 2479 addi r3,r1,STACK_FRAME_OVERHEAD 2480 bl facility_unavailable_exception 2481 REST_NVGPRS(r1) /* instruction emulation may change GPRs */ 2482 b interrupt_return 2483 2484 GEN_KVM facility_unavailable 2485 2486 2487/** 2488 * Interrupt 0xf60 - Hypervisor Facility Unavailable Interrupt. 2489 * This is a synchronous interrupt in response to 2490 * executing an instruction without access to the facility that can only 2491 * be resolved in HV mode (e.g., HFSCR). 2492 * Similar to FP unavailable. 2493 */ 2494INT_DEFINE_BEGIN(h_facility_unavailable) 2495 IVEC=0xf80 2496 IHSRR=1 2497 IKVM_REAL=1 2498 IKVM_VIRT=1 2499INT_DEFINE_END(h_facility_unavailable) 2500 2501EXC_REAL_BEGIN(h_facility_unavailable, 0xf80, 0x20) 2502 GEN_INT_ENTRY h_facility_unavailable, virt=0, ool=1 2503EXC_REAL_END(h_facility_unavailable, 0xf80, 0x20) 2504EXC_VIRT_BEGIN(h_facility_unavailable, 0x4f80, 0x20) 2505 GEN_INT_ENTRY h_facility_unavailable, virt=1, ool=1 2506EXC_VIRT_END(h_facility_unavailable, 0x4f80, 0x20) 2507EXC_COMMON_BEGIN(h_facility_unavailable_common) 2508 GEN_COMMON h_facility_unavailable 2509 addi r3,r1,STACK_FRAME_OVERHEAD 2510 bl facility_unavailable_exception 2511 REST_NVGPRS(r1) /* XXX Shouldn't be necessary in practice */ 2512 b interrupt_return 2513 2514 GEN_KVM h_facility_unavailable 2515 2516 2517EXC_REAL_NONE(0xfa0, 0x20) 2518EXC_VIRT_NONE(0x4fa0, 0x20) 2519EXC_REAL_NONE(0xfc0, 0x20) 2520EXC_VIRT_NONE(0x4fc0, 0x20) 2521EXC_REAL_NONE(0xfe0, 0x20) 2522EXC_VIRT_NONE(0x4fe0, 0x20) 2523 2524EXC_REAL_NONE(0x1000, 0x100) 2525EXC_VIRT_NONE(0x5000, 0x100) 2526EXC_REAL_NONE(0x1100, 0x100) 2527EXC_VIRT_NONE(0x5100, 0x100) 2528 2529#ifdef CONFIG_CBE_RAS 2530INT_DEFINE_BEGIN(cbe_system_error) 2531 IVEC=0x1200 2532 IHSRR=1 2533 IKVM_SKIP=1 2534 IKVM_REAL=1 2535INT_DEFINE_END(cbe_system_error) 2536 2537EXC_REAL_BEGIN(cbe_system_error, 0x1200, 0x100) 2538 GEN_INT_ENTRY cbe_system_error, virt=0 2539EXC_REAL_END(cbe_system_error, 0x1200, 0x100) 2540EXC_VIRT_NONE(0x5200, 0x100) 2541EXC_COMMON_BEGIN(cbe_system_error_common) 2542 GEN_COMMON cbe_system_error 2543 addi r3,r1,STACK_FRAME_OVERHEAD 2544 bl cbe_system_error_exception 2545 b interrupt_return 2546 2547 GEN_KVM cbe_system_error 2548 2549#else /* CONFIG_CBE_RAS */ 2550EXC_REAL_NONE(0x1200, 0x100) 2551EXC_VIRT_NONE(0x5200, 0x100) 2552#endif 2553 2554 2555INT_DEFINE_BEGIN(instruction_breakpoint) 2556 IVEC=0x1300 2557#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 2558 IKVM_SKIP=1 2559 IKVM_REAL=1 2560#endif 2561INT_DEFINE_END(instruction_breakpoint) 2562 2563EXC_REAL_BEGIN(instruction_breakpoint, 0x1300, 0x100) 2564 GEN_INT_ENTRY instruction_breakpoint, virt=0 2565EXC_REAL_END(instruction_breakpoint, 0x1300, 0x100) 2566EXC_VIRT_BEGIN(instruction_breakpoint, 0x5300, 0x100) 2567 GEN_INT_ENTRY instruction_breakpoint, virt=1 2568EXC_VIRT_END(instruction_breakpoint, 0x5300, 0x100) 2569EXC_COMMON_BEGIN(instruction_breakpoint_common) 2570 GEN_COMMON instruction_breakpoint 2571 addi r3,r1,STACK_FRAME_OVERHEAD 2572 bl instruction_breakpoint_exception 2573 b interrupt_return 2574 2575 GEN_KVM instruction_breakpoint 2576 2577 2578EXC_REAL_NONE(0x1400, 0x100) 2579EXC_VIRT_NONE(0x5400, 0x100) 2580 2581/** 2582 * Interrupt 0x1500 - Soft Patch Interrupt 2583 * 2584 * Handling: 2585 * This is an implementation specific interrupt which can be used for a 2586 * range of exceptions. 2587 * 2588 * This interrupt handler is unique in that it runs the denormal assist 2589 * code even for guests (and even in guest context) without going to KVM, 2590 * for speed. POWER9 does not raise denorm exceptions, so this special case 2591 * could be phased out in future to reduce special cases. 2592 */ 2593INT_DEFINE_BEGIN(denorm_exception) 2594 IVEC=0x1500 2595 IHSRR=1 2596 IBRANCH_TO_COMMON=0 2597 IKVM_REAL=1 2598INT_DEFINE_END(denorm_exception) 2599 2600EXC_REAL_BEGIN(denorm_exception, 0x1500, 0x100) 2601 GEN_INT_ENTRY denorm_exception, virt=0 2602#ifdef CONFIG_PPC_DENORMALISATION 2603 andis. r10,r12,(HSRR1_DENORM)@h /* denorm? */ 2604 bne+ denorm_assist 2605#endif 2606 GEN_BRANCH_TO_COMMON denorm_exception, virt=0 2607EXC_REAL_END(denorm_exception, 0x1500, 0x100) 2608#ifdef CONFIG_PPC_DENORMALISATION 2609EXC_VIRT_BEGIN(denorm_exception, 0x5500, 0x100) 2610 GEN_INT_ENTRY denorm_exception, virt=1 2611 andis. r10,r12,(HSRR1_DENORM)@h /* denorm? */ 2612 bne+ denorm_assist 2613 GEN_BRANCH_TO_COMMON denorm_exception, virt=1 2614EXC_VIRT_END(denorm_exception, 0x5500, 0x100) 2615#else 2616EXC_VIRT_NONE(0x5500, 0x100) 2617#endif 2618 2619#ifdef CONFIG_PPC_DENORMALISATION 2620TRAMP_REAL_BEGIN(denorm_assist) 2621BEGIN_FTR_SECTION 2622/* 2623 * To denormalise we need to move a copy of the register to itself. 2624 * For POWER6 do that here for all FP regs. 2625 */ 2626 mfmsr r10 2627 ori r10,r10,(MSR_FP|MSR_FE0|MSR_FE1) 2628 xori r10,r10,(MSR_FE0|MSR_FE1) 2629 mtmsrd r10 2630 sync 2631 2632 .Lreg=0 2633 .rept 32 2634 fmr .Lreg,.Lreg 2635 .Lreg=.Lreg+1 2636 .endr 2637 2638FTR_SECTION_ELSE 2639/* 2640 * To denormalise we need to move a copy of the register to itself. 2641 * For POWER7 do that here for the first 32 VSX registers only. 2642 */ 2643 mfmsr r10 2644 oris r10,r10,MSR_VSX@h 2645 mtmsrd r10 2646 sync 2647 2648 .Lreg=0 2649 .rept 32 2650 XVCPSGNDP(.Lreg,.Lreg,.Lreg) 2651 .Lreg=.Lreg+1 2652 .endr 2653 2654ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_206) 2655 2656BEGIN_FTR_SECTION 2657 b denorm_done 2658END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) 2659/* 2660 * To denormalise we need to move a copy of the register to itself. 2661 * For POWER8 we need to do that for all 64 VSX registers 2662 */ 2663 .Lreg=32 2664 .rept 32 2665 XVCPSGNDP(.Lreg,.Lreg,.Lreg) 2666 .Lreg=.Lreg+1 2667 .endr 2668 2669denorm_done: 2670 mfspr r11,SPRN_HSRR0 2671 subi r11,r11,4 2672 mtspr SPRN_HSRR0,r11 2673 mtcrf 0x80,r9 2674 ld r9,PACA_EXGEN+EX_R9(r13) 2675BEGIN_FTR_SECTION 2676 ld r10,PACA_EXGEN+EX_PPR(r13) 2677 mtspr SPRN_PPR,r10 2678END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 2679BEGIN_FTR_SECTION 2680 ld r10,PACA_EXGEN+EX_CFAR(r13) 2681 mtspr SPRN_CFAR,r10 2682END_FTR_SECTION_IFSET(CPU_FTR_CFAR) 2683 ld r10,PACA_EXGEN+EX_R10(r13) 2684 ld r11,PACA_EXGEN+EX_R11(r13) 2685 ld r12,PACA_EXGEN+EX_R12(r13) 2686 ld r13,PACA_EXGEN+EX_R13(r13) 2687 HRFI_TO_UNKNOWN 2688 b . 2689#endif 2690 2691EXC_COMMON_BEGIN(denorm_exception_common) 2692 GEN_COMMON denorm_exception 2693 addi r3,r1,STACK_FRAME_OVERHEAD 2694 bl unknown_exception 2695 b interrupt_return 2696 2697 GEN_KVM denorm_exception 2698 2699 2700#ifdef CONFIG_CBE_RAS 2701INT_DEFINE_BEGIN(cbe_maintenance) 2702 IVEC=0x1600 2703 IHSRR=1 2704 IKVM_SKIP=1 2705 IKVM_REAL=1 2706INT_DEFINE_END(cbe_maintenance) 2707 2708EXC_REAL_BEGIN(cbe_maintenance, 0x1600, 0x100) 2709 GEN_INT_ENTRY cbe_maintenance, virt=0 2710EXC_REAL_END(cbe_maintenance, 0x1600, 0x100) 2711EXC_VIRT_NONE(0x5600, 0x100) 2712EXC_COMMON_BEGIN(cbe_maintenance_common) 2713 GEN_COMMON cbe_maintenance 2714 addi r3,r1,STACK_FRAME_OVERHEAD 2715 bl cbe_maintenance_exception 2716 b interrupt_return 2717 2718 GEN_KVM cbe_maintenance 2719 2720#else /* CONFIG_CBE_RAS */ 2721EXC_REAL_NONE(0x1600, 0x100) 2722EXC_VIRT_NONE(0x5600, 0x100) 2723#endif 2724 2725 2726INT_DEFINE_BEGIN(altivec_assist) 2727 IVEC=0x1700 2728#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 2729 IKVM_REAL=1 2730#endif 2731INT_DEFINE_END(altivec_assist) 2732 2733EXC_REAL_BEGIN(altivec_assist, 0x1700, 0x100) 2734 GEN_INT_ENTRY altivec_assist, virt=0 2735EXC_REAL_END(altivec_assist, 0x1700, 0x100) 2736EXC_VIRT_BEGIN(altivec_assist, 0x5700, 0x100) 2737 GEN_INT_ENTRY altivec_assist, virt=1 2738EXC_VIRT_END(altivec_assist, 0x5700, 0x100) 2739EXC_COMMON_BEGIN(altivec_assist_common) 2740 GEN_COMMON altivec_assist 2741 addi r3,r1,STACK_FRAME_OVERHEAD 2742#ifdef CONFIG_ALTIVEC 2743 bl altivec_assist_exception 2744 REST_NVGPRS(r1) /* instruction emulation may change GPRs */ 2745#else 2746 bl unknown_exception 2747#endif 2748 b interrupt_return 2749 2750 GEN_KVM altivec_assist 2751 2752 2753#ifdef CONFIG_CBE_RAS 2754INT_DEFINE_BEGIN(cbe_thermal) 2755 IVEC=0x1800 2756 IHSRR=1 2757 IKVM_SKIP=1 2758 IKVM_REAL=1 2759INT_DEFINE_END(cbe_thermal) 2760 2761EXC_REAL_BEGIN(cbe_thermal, 0x1800, 0x100) 2762 GEN_INT_ENTRY cbe_thermal, virt=0 2763EXC_REAL_END(cbe_thermal, 0x1800, 0x100) 2764EXC_VIRT_NONE(0x5800, 0x100) 2765EXC_COMMON_BEGIN(cbe_thermal_common) 2766 GEN_COMMON cbe_thermal 2767 addi r3,r1,STACK_FRAME_OVERHEAD 2768 bl cbe_thermal_exception 2769 b interrupt_return 2770 2771 GEN_KVM cbe_thermal 2772 2773#else /* CONFIG_CBE_RAS */ 2774EXC_REAL_NONE(0x1800, 0x100) 2775EXC_VIRT_NONE(0x5800, 0x100) 2776#endif 2777 2778 2779#ifdef CONFIG_PPC_WATCHDOG 2780 2781INT_DEFINE_BEGIN(soft_nmi) 2782 IVEC=0x900 2783 ISTACK=0 2784INT_DEFINE_END(soft_nmi) 2785 2786/* 2787 * Branch to soft_nmi_interrupt using the emergency stack. The emergency 2788 * stack is one that is usable by maskable interrupts so long as MSR_EE 2789 * remains off. It is used for recovery when something has corrupted the 2790 * normal kernel stack, for example. The "soft NMI" must not use the process 2791 * stack because we want irq disabled sections to avoid touching the stack 2792 * at all (other than PMU interrupts), so use the emergency stack for this, 2793 * and run it entirely with interrupts hard disabled. 2794 */ 2795EXC_COMMON_BEGIN(soft_nmi_common) 2796 mfspr r11,SPRN_SRR0 2797 mr r10,r1 2798 ld r1,PACAEMERGSP(r13) 2799 subi r1,r1,INT_FRAME_SIZE 2800 __GEN_COMMON_BODY soft_nmi 2801 2802 addi r3,r1,STACK_FRAME_OVERHEAD 2803 bl soft_nmi_interrupt 2804 2805 /* Clear MSR_RI before setting SRR0 and SRR1. */ 2806 li r9,0 2807 mtmsrd r9,1 2808 2809 kuap_kernel_restore r9, r10 2810 EXCEPTION_RESTORE_REGS hsrr=0 2811 RFI_TO_KERNEL 2812 2813#endif /* CONFIG_PPC_WATCHDOG */ 2814 2815/* 2816 * An interrupt came in while soft-disabled. We set paca->irq_happened, then: 2817 * - If it was a decrementer interrupt, we bump the dec to max and and return. 2818 * - If it was a doorbell we return immediately since doorbells are edge 2819 * triggered and won't automatically refire. 2820 * - If it was a HMI we return immediately since we handled it in realmode 2821 * and it won't refire. 2822 * - Else it is one of PACA_IRQ_MUST_HARD_MASK, so hard disable and return. 2823 * This is called with r10 containing the value to OR to the paca field. 2824 */ 2825.macro MASKED_INTERRUPT hsrr=0 2826 .if \hsrr 2827masked_Hinterrupt: 2828 .else 2829masked_interrupt: 2830 .endif 2831 lbz r11,PACAIRQHAPPENED(r13) 2832 or r11,r11,r10 2833 stb r11,PACAIRQHAPPENED(r13) 2834 cmpwi r10,PACA_IRQ_DEC 2835 bne 1f 2836 lis r10,0x7fff 2837 ori r10,r10,0xffff 2838 mtspr SPRN_DEC,r10 2839#ifdef CONFIG_PPC_WATCHDOG 2840 b soft_nmi_common 2841#else 2842 b 2f 2843#endif 28441: andi. r10,r10,PACA_IRQ_MUST_HARD_MASK 2845 beq 2f 2846 xori r12,r12,MSR_EE /* clear MSR_EE */ 2847 .if \hsrr 2848 mtspr SPRN_HSRR1,r12 2849 .else 2850 mtspr SPRN_SRR1,r12 2851 .endif 2852 ori r11,r11,PACA_IRQ_HARD_DIS 2853 stb r11,PACAIRQHAPPENED(r13) 28542: /* done */ 2855 ld r10,PACA_EXGEN+EX_CTR(r13) 2856 mtctr r10 2857 mtcrf 0x80,r9 2858 std r1,PACAR1(r13) 2859 ld r9,PACA_EXGEN+EX_R9(r13) 2860 ld r10,PACA_EXGEN+EX_R10(r13) 2861 ld r11,PACA_EXGEN+EX_R11(r13) 2862 ld r12,PACA_EXGEN+EX_R12(r13) 2863 ld r13,PACA_EXGEN+EX_R13(r13) 2864 /* May return to masked low address where r13 is not set up */ 2865 .if \hsrr 2866 HRFI_TO_KERNEL 2867 .else 2868 RFI_TO_KERNEL 2869 .endif 2870 b . 2871.endm 2872 2873TRAMP_REAL_BEGIN(stf_barrier_fallback) 2874 std r9,PACA_EXRFI+EX_R9(r13) 2875 std r10,PACA_EXRFI+EX_R10(r13) 2876 sync 2877 ld r9,PACA_EXRFI+EX_R9(r13) 2878 ld r10,PACA_EXRFI+EX_R10(r13) 2879 ori 31,31,0 2880 .rept 14 2881 b 1f 28821: 2883 .endr 2884 blr 2885 2886/* Clobbers r10, r11, ctr */ 2887.macro L1D_DISPLACEMENT_FLUSH 2888 ld r10,PACA_RFI_FLUSH_FALLBACK_AREA(r13) 2889 ld r11,PACA_L1D_FLUSH_SIZE(r13) 2890 srdi r11,r11,(7 + 3) /* 128 byte lines, unrolled 8x */ 2891 mtctr r11 2892 DCBT_BOOK3S_STOP_ALL_STREAM_IDS(r11) /* Stop prefetch streams */ 2893 2894 /* order ld/st prior to dcbt stop all streams with flushing */ 2895 sync 2896 2897 /* 2898 * The load addresses are at staggered offsets within cachelines, 2899 * which suits some pipelines better (on others it should not 2900 * hurt). 2901 */ 29021: 2903 ld r11,(0x80 + 8)*0(r10) 2904 ld r11,(0x80 + 8)*1(r10) 2905 ld r11,(0x80 + 8)*2(r10) 2906 ld r11,(0x80 + 8)*3(r10) 2907 ld r11,(0x80 + 8)*4(r10) 2908 ld r11,(0x80 + 8)*5(r10) 2909 ld r11,(0x80 + 8)*6(r10) 2910 ld r11,(0x80 + 8)*7(r10) 2911 addi r10,r10,0x80*8 2912 bdnz 1b 2913.endm 2914 2915TRAMP_REAL_BEGIN(entry_flush_fallback) 2916 std r9,PACA_EXRFI+EX_R9(r13) 2917 std r10,PACA_EXRFI+EX_R10(r13) 2918 std r11,PACA_EXRFI+EX_R11(r13) 2919 mfctr r9 2920 L1D_DISPLACEMENT_FLUSH 2921 mtctr r9 2922 ld r9,PACA_EXRFI+EX_R9(r13) 2923 ld r10,PACA_EXRFI+EX_R10(r13) 2924 ld r11,PACA_EXRFI+EX_R11(r13) 2925 blr 2926 2927/* 2928 * The SCV entry flush happens with interrupts enabled, so it must disable 2929 * to prevent EXRFI being clobbered by NMIs (e.g., soft_nmi_common). r10 2930 * (containing LR) does not need to be preserved here because scv entry 2931 * puts 0 in the pt_regs, CTR can be clobbered for the same reason. 2932 */ 2933TRAMP_REAL_BEGIN(scv_entry_flush_fallback) 2934 li r10,0 2935 mtmsrd r10,1 2936 lbz r10,PACAIRQHAPPENED(r13) 2937 ori r10,r10,PACA_IRQ_HARD_DIS 2938 stb r10,PACAIRQHAPPENED(r13) 2939 std r11,PACA_EXRFI+EX_R11(r13) 2940 L1D_DISPLACEMENT_FLUSH 2941 ld r11,PACA_EXRFI+EX_R11(r13) 2942 li r10,MSR_RI 2943 mtmsrd r10,1 2944 blr 2945 2946TRAMP_REAL_BEGIN(rfi_flush_fallback) 2947 SET_SCRATCH0(r13); 2948 GET_PACA(r13); 2949 std r1,PACA_EXRFI+EX_R12(r13) 2950 ld r1,PACAKSAVE(r13) 2951 std r9,PACA_EXRFI+EX_R9(r13) 2952 std r10,PACA_EXRFI+EX_R10(r13) 2953 std r11,PACA_EXRFI+EX_R11(r13) 2954 mfctr r9 2955 L1D_DISPLACEMENT_FLUSH 2956 mtctr r9 2957 ld r9,PACA_EXRFI+EX_R9(r13) 2958 ld r10,PACA_EXRFI+EX_R10(r13) 2959 ld r11,PACA_EXRFI+EX_R11(r13) 2960 ld r1,PACA_EXRFI+EX_R12(r13) 2961 GET_SCRATCH0(r13); 2962 rfid 2963 2964TRAMP_REAL_BEGIN(hrfi_flush_fallback) 2965 SET_SCRATCH0(r13); 2966 GET_PACA(r13); 2967 std r1,PACA_EXRFI+EX_R12(r13) 2968 ld r1,PACAKSAVE(r13) 2969 std r9,PACA_EXRFI+EX_R9(r13) 2970 std r10,PACA_EXRFI+EX_R10(r13) 2971 std r11,PACA_EXRFI+EX_R11(r13) 2972 mfctr r9 2973 L1D_DISPLACEMENT_FLUSH 2974 mtctr r9 2975 ld r9,PACA_EXRFI+EX_R9(r13) 2976 ld r10,PACA_EXRFI+EX_R10(r13) 2977 ld r11,PACA_EXRFI+EX_R11(r13) 2978 ld r1,PACA_EXRFI+EX_R12(r13) 2979 GET_SCRATCH0(r13); 2980 hrfid 2981 2982TRAMP_REAL_BEGIN(rfscv_flush_fallback) 2983 /* system call volatile */ 2984 mr r7,r13 2985 GET_PACA(r13); 2986 mr r8,r1 2987 ld r1,PACAKSAVE(r13) 2988 mfctr r9 2989 ld r10,PACA_RFI_FLUSH_FALLBACK_AREA(r13) 2990 ld r11,PACA_L1D_FLUSH_SIZE(r13) 2991 srdi r11,r11,(7 + 3) /* 128 byte lines, unrolled 8x */ 2992 mtctr r11 2993 DCBT_BOOK3S_STOP_ALL_STREAM_IDS(r11) /* Stop prefetch streams */ 2994 2995 /* order ld/st prior to dcbt stop all streams with flushing */ 2996 sync 2997 2998 /* 2999 * The load adresses are at staggered offsets within cachelines, 3000 * which suits some pipelines better (on others it should not 3001 * hurt). 3002 */ 30031: 3004 ld r11,(0x80 + 8)*0(r10) 3005 ld r11,(0x80 + 8)*1(r10) 3006 ld r11,(0x80 + 8)*2(r10) 3007 ld r11,(0x80 + 8)*3(r10) 3008 ld r11,(0x80 + 8)*4(r10) 3009 ld r11,(0x80 + 8)*5(r10) 3010 ld r11,(0x80 + 8)*6(r10) 3011 ld r11,(0x80 + 8)*7(r10) 3012 addi r10,r10,0x80*8 3013 bdnz 1b 3014 3015 mtctr r9 3016 li r9,0 3017 li r10,0 3018 li r11,0 3019 mr r1,r8 3020 mr r13,r7 3021 RFSCV 3022 3023USE_TEXT_SECTION() 3024 3025_GLOBAL(do_uaccess_flush) 3026 UACCESS_FLUSH_FIXUP_SECTION 3027 nop 3028 nop 3029 nop 3030 blr 3031 L1D_DISPLACEMENT_FLUSH 3032 blr 3033_ASM_NOKPROBE_SYMBOL(do_uaccess_flush) 3034EXPORT_SYMBOL(do_uaccess_flush) 3035 3036 3037MASKED_INTERRUPT 3038MASKED_INTERRUPT hsrr=1 3039 3040#ifdef CONFIG_KVM_BOOK3S_64_HANDLER 3041kvmppc_skip_interrupt: 3042 /* 3043 * Here all GPRs are unchanged from when the interrupt happened 3044 * except for r13, which is saved in SPRG_SCRATCH0. 3045 */ 3046 mfspr r13, SPRN_SRR0 3047 addi r13, r13, 4 3048 mtspr SPRN_SRR0, r13 3049 GET_SCRATCH0(r13) 3050 RFI_TO_KERNEL 3051 b . 3052 3053kvmppc_skip_Hinterrupt: 3054 /* 3055 * Here all GPRs are unchanged from when the interrupt happened 3056 * except for r13, which is saved in SPRG_SCRATCH0. 3057 */ 3058 mfspr r13, SPRN_HSRR0 3059 addi r13, r13, 4 3060 mtspr SPRN_HSRR0, r13 3061 GET_SCRATCH0(r13) 3062 HRFI_TO_KERNEL 3063 b . 3064#endif 3065 3066 /* 3067 * Relocation-on interrupts: A subset of the interrupts can be delivered 3068 * with IR=1/DR=1, if AIL==2 and MSR.HV won't be changed by delivering 3069 * it. Addresses are the same as the original interrupt addresses, but 3070 * offset by 0xc000000000004000. 3071 * It's impossible to receive interrupts below 0x300 via this mechanism. 3072 * KVM: None of these traps are from the guest ; anything that escalated 3073 * to HV=1 from HV=0 is delivered via real mode handlers. 3074 */ 3075 3076 /* 3077 * This uses the standard macro, since the original 0x300 vector 3078 * only has extra guff for STAB-based processors -- which never 3079 * come here. 3080 */ 3081 3082USE_FIXED_SECTION(virt_trampolines) 3083 /* 3084 * All code below __end_interrupts is treated as soft-masked. If 3085 * any code runs here with MSR[EE]=1, it must then cope with pending 3086 * soft interrupt being raised (i.e., by ensuring it is replayed). 3087 * 3088 * The __end_interrupts marker must be past the out-of-line (OOL) 3089 * handlers, so that they are copied to real address 0x100 when running 3090 * a relocatable kernel. This ensures they can be reached from the short 3091 * trampoline handlers (like 0x4f00, 0x4f20, etc.) which branch 3092 * directly, without using LOAD_HANDLER(). 3093 */ 3094 .align 7 3095 .globl __end_interrupts 3096__end_interrupts: 3097DEFINE_FIXED_SYMBOL(__end_interrupts) 3098 3099#ifdef CONFIG_PPC_970_NAP 3100 /* 3101 * Called by exception entry code if _TLF_NAPPING was set, this clears 3102 * the NAPPING flag, and redirects the exception exit to 3103 * power4_fixup_nap_return. 3104 */ 3105 .globl power4_fixup_nap 3106EXC_COMMON_BEGIN(power4_fixup_nap) 3107 andc r9,r9,r10 3108 std r9,TI_LOCAL_FLAGS(r11) 3109 LOAD_REG_ADDR(r10, power4_idle_nap_return) 3110 std r10,_NIP(r1) 3111 blr 3112 3113power4_idle_nap_return: 3114 blr 3115#endif 3116 3117CLOSE_FIXED_SECTION(real_vectors); 3118CLOSE_FIXED_SECTION(real_trampolines); 3119CLOSE_FIXED_SECTION(virt_vectors); 3120CLOSE_FIXED_SECTION(virt_trampolines); 3121 3122USE_TEXT_SECTION() 3123 3124/* MSR[RI] should be clear because this uses SRR[01] */ 3125enable_machine_check: 3126 mflr r0 3127 bcl 20,31,$+4 31280: mflr r3 3129 addi r3,r3,(1f - 0b) 3130 mtspr SPRN_SRR0,r3 3131 mfmsr r3 3132 ori r3,r3,MSR_ME 3133 mtspr SPRN_SRR1,r3 3134 RFI_TO_KERNEL 31351: mtlr r0 3136 blr 3137 3138/* MSR[RI] should be clear because this uses SRR[01] */ 3139disable_machine_check: 3140 mflr r0 3141 bcl 20,31,$+4 31420: mflr r3 3143 addi r3,r3,(1f - 0b) 3144 mtspr SPRN_SRR0,r3 3145 mfmsr r3 3146 li r4,MSR_ME 3147 andc r3,r3,r4 3148 mtspr SPRN_SRR1,r3 3149 RFI_TO_KERNEL 31501: mtlr r0 3151 blr 3152