1 /* SPDX-License-Identifier: GPL-2.0 */
3  * This file contains the 64-bit "server" PowerPC variant
18 #include <asm/exception-64s.h>
21 #include <asm/head-64.h>
22 #include <asm/feature-fixups.h>
28  * EXC_REAL_BEGIN/END  - real, unrelocated exception vectors
29  * EXC_VIRT_BEGIN/END  - virt (AIL), unrelocated exception vectors
30  * TRAMP_REAL_BEGIN    - real, unrelocated helpers (virt may call these)
31  * TRAMP_VIRT_BEGIN    - virt, unreloc helpers (in practice, real can use)
32  * EXC_COMMON          - After switching to virtual, relocated mode.
35 #define EXC_REAL_BEGIN(name, start, size)			\  argument
36 	FIXED_SECTION_ENTRY_BEGIN_LOCATION(real_vectors, exc_real_##start##_##name, start, size)
38 #define EXC_REAL_END(name, start, size)				\  argument
39 	FIXED_SECTION_ENTRY_END_LOCATION(real_vectors, exc_real_##start##_##name, start, size)
41 #define EXC_VIRT_BEGIN(name, start, size)			\  argument
42 	FIXED_SECTION_ENTRY_BEGIN_LOCATION(virt_vectors, exc_virt_##start##_##name, start, size)
44 #define EXC_VIRT_END(name, start, size)				\  argument
45 	FIXED_SECTION_ENTRY_END_LOCATION(virt_vectors, exc_virt_##start##_##name, start, size)
47 #define EXC_COMMON_BEGIN(name)					\  argument
50 	.global name;						\
51 	_ASM_NOKPROBE_SYMBOL(name);				\
52 	DEFINE_FIXED_SYMBOL(name, text);			\
53 name:
55 #define TRAMP_REAL_BEGIN(name)					\  argument
56 	FIXED_SECTION_ENTRY_BEGIN(real_trampolines, name)
58 #define TRAMP_VIRT_BEGIN(name)					\  argument
59 	FIXED_SECTION_ENTRY_BEGIN(virt_trampolines, name)
72  * Instead we get the base of the kernel from paca->kernelbase and or in the low
86  * head-y require >64K offsets.
96 #define IVEC		.L_IVEC_\name\()	/* Interrupt vector address */
97 #define IHSRR		.L_IHSRR_\name\()	/* Sets SRR or HSRR registers */
98 #define IHSRR_IF_HVMODE	.L_IHSRR_IF_HVMODE_\name\() /* HSRR if HV else SRR */
99 #define IAREA		.L_IAREA_\name\()	/* PACA save area */
100 #define IVIRT		.L_IVIRT_\name\()	/* Has virt mode entry point */
101 #define IISIDE		.L_IISIDE_\name\()	/* Uses SRR0/1 not DAR/DSISR */
102 #define ICFAR		.L_ICFAR_\name\()	/* Uses CFAR */
103 #define ICFAR_IF_HVMODE	.L_ICFAR_IF_HVMODE_\name\() /* Uses CFAR if HV */
104 #define IDAR		.L_IDAR_\name\()	/* Uses DAR (or SRR0) */
105 #define IDSISR		.L_IDSISR_\name\()	/* Uses DSISR (or SRR1) */
106 #define IBRANCH_TO_COMMON	.L_IBRANCH_TO_COMMON_\name\() /* ENTRY branch to common */
107 #define IREALMODE_COMMON	.L_IREALMODE_COMMON_\name\() /* Common runs in realmode */
108 #define IMASK		.L_IMASK_\name\()	/* IRQ soft-mask bit */
109 #define IKVM_REAL	.L_IKVM_REAL_\name\()	/* Real entry tests KVM */
110 #define __IKVM_REAL(name)	.L_IKVM_REAL_ ## name  argument
111 #define IKVM_VIRT	.L_IKVM_VIRT_\name\()	/* Virt entry tests KVM */
112 #define ISTACK		.L_ISTACK_\name\()	/* Set regular kernel stack */
113 #define __ISTACK(name)	.L_ISTACK_ ## name  argument
114 #define IKUAP		.L_IKUAP_\name\()	/* Do KUAP lock */
115 #define IMSR_R12	.L_IMSR_R12_\name\()	/* Assumes MSR saved to r12 */
118 .macro int_define_ ## n name
125 .macro do_define_int name  argument
203  * delivered to the real-mode entry point, therefore such interrupts only test
207  * delivered in real-mode when the MMU is in hash mode because the MMU
209  * radix mode these can be delivered in virt-mode as the host translations are
218 .macro KVMTEST name handler
242  * - Branch to the _common handler, relocated, in virtual mode.
245  * - Branch to KVM, relocated but real mode interrupts remain in real mode.
249  * - Branch to the masked handler, unrelocated.
252  * - Branch to an "early" handler in real mode but relocated.
255  * - Fall through and continue executing in real, unrelocated mode.
259 .macro GEN_BRANCH_TO_COMMON name, virt
261 	LOAD_HANDLER(r10, \name\()_common)
267 	b	\name\()_common_virt
269 	LOAD_HANDLER(r10, \name\()_common_virt)
274 	LOAD_HANDLER(r10, \name\()_common_real)
281 .macro GEN_INT_ENTRY name, virt, ool=0
307 	b	tramp_real_\name
309 	TRAMP_REAL_BEGIN(tramp_real_\name)
311 	b	tramp_virt_\name
313 	TRAMP_VIRT_BEGIN(tramp_virt_\name)
329 	std	r11,IAREA+EX_R11(r13)		/* save r11 - r12 */
334 	 * because a d-side MCE will clobber those registers so is
373 	GEN_BRANCH_TO_COMMON \name \virt
383  * entry, except in the case of the real-mode handlers which require
388 .macro __GEN_COMMON_ENTRY name  argument
389 DEFINE_FIXED_SYMBOL(\name\()_common_real, text)
390 \name\()_common_real:
392 		KVMTEST \name kvm_interrupt
412 DEFINE_FIXED_SYMBOL(\name\()_common_virt, text)
413 \name\()_common_virt:
415 		KVMTEST \name kvm_interrupt
425 .macro __GEN_REALMODE_COMMON_ENTRY name  argument
426 DEFINE_FIXED_SYMBOL(\name\()_common_real, text)
427 \name\()_common_real:
429 		KVMTEST \name kvm_interrupt
433 .macro __GEN_COMMON_BODY name  argument
439 		/* If coming from user, skip soft-mask tests. */
445 		 * implicitly soft-masked if it is within the regions
459 		beq	1f		/* Not in soft-mask table */
461 		b	2f		/* In soft-mask table, always mask */
466 		/* Associate vector numbers with bits in paca->irq_happened */
498 	beq-	100f
500 100:	tdgei	r1,-INT_FRAME_SIZE	/* trap if r1 is in userspace	*/
547 	ld	r9,IAREA+EX_R11(r13)	/* move r11 - r13 to stackframe	*/
592 	SAVE_GPRS(2, 8, r1)		/* save r2 - r8 in stackframe   */
605 	std	r10,RESULT(r1)		/* clear regs->result		*/
610  * On entry r13 points to the paca, r9-r13 are saved in the paca,
617 .macro GEN_COMMON name  argument
618 	__GEN_COMMON_ENTRY \name
619 	__GEN_COMMON_BODY \name
719  * EARLY_BOOT_FIXUP - Fix real-mode interrupt with wrong endian in early boot.
785  * - Real mode exceptions code/data must be located at their physical location.
786  * - Virtual mode exceptions must be mapped at their 0xc000... location.
787  * - Fixed location code must not call directly beyond the __end_interrupts
790  * - LOAD_HANDLER targets must be within first 64K of physical 0 /
792  * - Conditional branch targets must be within +/-32K of caller.
817  * 0x0000 - 0x00ff : Secondary processor spin code
818  * 0x0100 - 0x18ff : Real mode pSeries interrupt vectors
819  * 0x1900 - 0x2fff : Real mode trampolines
820  * 0x3000 - 0x58ff : Relon (IR=1,DR=1) mode pSeries interrupt vectors
821  * 0x5900 - 0x6fff : Relon mode trampolines
822  * 0x7000 - 0x7fff : FWNMI data area
823  * 0x8000 -   .... : Common interrupt handlers, remaining early
826  * We could reclaim 0x4000-0x42ff for real mode trampolines if the space
869  * Interrupt 0x3000 - System Call Vectored Interrupt (syscall).
877  * by treating the instruction addresses in the handler as being soft-masked,
880  * AIL-0 mode scv exceptions go to 0x17000-0x17fff, but we set AIL-3 and
881  * ensure scv is never executed with relocation off, which means AIL-0
884  * Before leaving the following inside-__end_soft_masked text, at least of the
886  * - MSR[PR]=1 (i.e., return to userspace)
887  * - MSR_EE|MSR_RI is clear (no reentrant exceptions)
888  * - Standard kernel environment is set up (stack, paca, etc)
891  * These interrupts do not elevate HV 0->1, so HV is not involved. PR KVM
896  * syscall register convention is in Documentation/powerpc/syscall64-abi.rst
913 	/* SCV 1 - 127 */
921 	li	r0,-1 /* cause failure */
930 // Treat scv vectors as soft-masked, see comment above.
953  * Interrupt 0x100 - System Reset Interrupt (SRESET aka NMI).
954  * This is a non-maskable, asynchronous interrupt always taken in real-mode.
956  * - Wake from power-saving state, on powernv.
957  * - An NMI from another CPU, triggered by firmware or hypercall.
958  * - As crash/debug signal injected from BMC, firmware or hypervisor.
961  * Power-save wakeup is the only performance critical path, so this is
998 	 * bits 46:47. A non-0 value indicates that we are coming from a power
1004 	 * also avoiding clobbering CFAR for the debug path (non-idle).
1019 	rlwinm.	r5,r3,47-31,30,31
1063 	 * Increment paca->in_nmi. When the interrupt entry wrapper later
1078 	bl	CFUNC(system_reset_exception)
1085 	 * MSR_RI is clear, now we can decrement paca->in_nmi.
1097  * Interrupt 0x200 - Machine Check Interrupt (MCE).
1098  * This is a non-maskable interrupt always taken in real-mode. It can be
1100  * taken in a power-saving state.
1104  * the difference is re-entrancy is allowed on the machine check stack.
1111  * This early code does not "reconcile" irq soft-mask state like SRESET or
1118  * - Executing in kernel mode. The event is queued with irq_work, which means
1126  * - Executing in user mode. machine_check_exception is run like a normal
1129  * - Executing in guest mode. The interrupt is run with its KVM test, and
1175 	/* Decrement paca->in_mce now RI is clear. */	\
1185 	 * Switch to mc_emergency stack and handle re-entrancy (we limit
1189 	 * We use paca->in_mce to check whether this is the first entry or
1190 	 * nested machine check. We increment paca->in_mce to track nested
1194 	 * paca->mc_emergency_sp, otherwise r1 is already pointing to
1208 	addi	r10,r10,1		/* increment paca->in_mce */
1222 	bl	enable_machine_check
1226 	bl	CFUNC(machine_check_early_boot)
1228 	bl	CFUNC(machine_check_early)
1243 	rlwinm.	r11,r12,47-31,30,31
1252 	 * machine_check_kvm->kvm_interrupt branch to deliver the MC event
1270 	 * But before that, check if this is an un-recoverable exception.
1289 	bl	CFUNC(machine_check_queue_event)
1315 	bl	CFUNC(machine_check_exception_async)
1325 	bl	CFUNC(machine_check_queue_event)
1328 	 * GPR-loss wakeups are relatively straightforward, because the
1329 	 * idle sleep code has saved all non-volatile registers on its
1332 	 * For no-loss wakeups the r1 and lr registers used by the
1347 	rlwinm	r10,r3,47-31,30,31
1364 	bl	CFUNC(disable_machine_check)
1381 	bl	CFUNC(machine_check_exception)
1388 	bl	CFUNC(unrecoverable_exception)
1393  * Interrupt 0x300 - Data Storage Interrupt (DSI).
1400  * - Hash MMU
1404  *   "non-bolted" regions, e.g., vmalloc space. However these should always be
1413  *   KVM handler. MSR[IR] is not enabled, so the real-mode handler will
1416  * - Radix MMU
1440 	bne-	1f
1443 	bl	CFUNC(do_hash_fault)
1445 	bl	CFUNC(do_page_fault)
1448 	bl	CFUNC(do_page_fault)
1452 1:	bl	CFUNC(do_break)
1462  * Interrupt 0x380 - Data Segment Interrupt (DSLB).
1467  * - HPT:
1469  *   fault. When coming from user-mode, the SLB handler may access any kernel
1474  *   on user-handler data structures.
1496 	bl	CFUNC(do_slb_fault)
1498 	bne-	1f
1503 	li	r3,-EFAULT
1506 	li	r3,-EFAULT
1510 	bl	CFUNC(do_bad_segment_interrupt)
1515  * Interrupt 0x400 - Instruction Storage Interrupt (ISI).
1544 	bl	CFUNC(do_hash_fault)
1546 	bl	CFUNC(do_page_fault)
1549 	bl	CFUNC(do_page_fault)
1555  * Interrupt 0x480 - Instruction Segment Interrupt (ISLB).
1584 	bl	CFUNC(do_slb_fault)
1586 	bne-	1f
1591 	li	r3,-EFAULT
1594 	li	r3,-EFAULT
1598 	bl	CFUNC(do_bad_segment_interrupt)
1603  * Interrupt 0x500 - External Interrupt.
1607  * soft-maskable with IRQS_DISABLED mask (i.e., local_irq_disable()).
1652 	bl	CFUNC(do_IRQ)
1661  * Interrupt 0x600 - Alignment Interrupt
1682 	bl	CFUNC(alignment_exception)
1688  * Interrupt 0x700 - Program Interrupt (program check).
1728 	cmpdi	r1,-INT_FRAME_SIZE	/* check if r1 is in userspace	*/
1748 	bl	CFUNC(program_check_exception)
1754  * Interrupt 0x800 - Floating-Point Unavailable Interrupt.
1780 	bl	CFUNC(kernel_fp_unavailable_exception)
1786 	/* Test if 2 TM state bits are zero.  If non-zero (ie. userspace was in
1789 	rldicl.	r0, r12, (64-MSR_TS_LG), (64-2)
1790 	bne-	2f
1793 	bl	CFUNC(load_up_fpu)
1798 	bl	CFUNC(fp_unavailable_tm)
1804  * Interrupt 0x900 - Decrementer Interrupt.
1807  * MSR[EE], and soft-maskable with IRQS_DISABLED mask (i.e.,
1842 	bl	CFUNC(timer_interrupt)
1847  * Interrupt 0x980 - Hypervisor Decrementer Interrupt.
1876 	 * edge-triggered.
1894  * Interrupt 0xa00 - Directed Privileged Doorbell Interrupt.
1896  * It is maskable in hardware by clearing MSR[EE], and soft-maskable with
1928 	bl	CFUNC(doorbell_exception)
1930 	bl	CFUNC(unknown_async_exception)
1939  * Interrupt 0xc00 - System Call Interrupt (syscall, hcall).
1955  * Documentation/powerpc/syscall64-abi.rst and
1995 	beq-	1f
2074  * Interrupt 0xd00 - Trace Interrupt.
2094 	bl	CFUNC(single_step_exception)
2099  * Interrupt 0xe00 - Hypervisor Data Storage Interrupt (HDSI).
2129 	bl	CFUNC(do_bad_page_fault_segv)
2131 	bl	CFUNC(unknown_exception)
2137  * Interrupt 0xe20 - Hypervisor Instruction Storage Interrupt (HISI).
2157 	bl	CFUNC(unknown_exception)
2162  * Interrupt 0xe40 - Hypervisor Emulation Assistance Interrupt.
2180 	bl	CFUNC(emulation_assist_interrupt)
2186  * Interrupt 0xe60 - Hypervisor Maintenance Interrupt (HMI).
2191  * It is maskable in hardware by clearing MSR[EE], and partially soft-maskable
2196  * initial real mode handler that is not soft-masked, which attempts to fix the
2197  * problem. Then a regular handler which is soft-maskable and reports the
2203  * either use soft-masking for the MCE, or use irq_work for the HMI.
2240 	bl	CFUNC(hmi_exception_realmode)
2258 	bl	CFUNC(handle_hmi_exception)
2263  * Interrupt 0xe80 - Directed Hypervisor Doorbell Interrupt.
2293 	bl	CFUNC(doorbell_exception)
2295 	bl	CFUNC(unknown_async_exception)
2301  * Interrupt 0xea0 - Hypervisor Virtualization Interrupt.
2328 	bl	CFUNC(do_IRQ)
2339  * Interrupt 0xf00 - Performance Monitor Interrupt (PMI, PMU).
2341  * It is maskable in hardware by clearing MSR[EE], and soft-maskable with
2347  * Like the watchdog soft-nmi, it appears an NMI interrupt to Linux, in that it
2348  * runs under local_irq_disable. However it may be soft-masked in
2349  * powerpc-specific code.
2377 	bl	CFUNC(performance_monitor_exception_async)
2380 	bl	CFUNC(performance_monitor_exception_nmi)
2391  * Interrupt 0xf20 - Vector Unavailable Interrupt.
2417 	/* Test if 2 TM state bits are zero.  If non-zero (ie. userspace was in
2420 	rldicl.	r0, r12, (64-MSR_TS_LG), (64-2)
2421 	bne-	2f
2424 	bl	CFUNC(load_up_altivec)
2429 	bl	CFUNC(altivec_unavailable_tm)
2436 	bl	CFUNC(altivec_unavailable_exception)
2441  * Interrupt 0xf40 - VSX Unavailable Interrupt.
2467 	/* Test if 2 TM state bits are zero.  If non-zero (ie. userspace was in
2470 	rldicl.	r0, r12, (64-MSR_TS_LG), (64-2)
2471 	bne-	2f
2478 	bl	CFUNC(vsx_unavailable_tm)
2485 	bl	CFUNC(vsx_unavailable_exception)
2490  * Interrupt 0xf60 - Facility Unavailable Interrupt.
2512 	bl	CFUNC(facility_unavailable_exception)
2518  * Interrupt 0xf60 - Hypervisor Facility Unavailable Interrupt.
2540 	bl	CFUNC(facility_unavailable_exception)
2571 	bl	CFUNC(cbe_system_error_exception)
2580  * Interrupt 0x1300 - Instruction Address Breakpoint Interrupt.
2582  * 64-bit BookS ISA supported, however the G5 / 970 implements this
2583  * interrupt with a non-architected feature available through the support
2602 	bl	CFUNC(instruction_breakpoint_exception)
2610  * Interrupt 0x1500 - Soft Patch Interrupt
2724 	bl	CFUNC(unknown_exception)
2741 	bl	CFUNC(cbe_maintenance_exception)
2767 	bl	CFUNC(altivec_assist_exception)
2770 	bl	CFUNC(unknown_exception)
2788 	bl	CFUNC(cbe_thermal_exception)
2821 	bl	CFUNC(soft_nmi_interrupt)
2835  * An interrupt came in while soft-disabled. We set paca->irq_happened, then:
2836  * - If it was a decrementer interrupt, we bump the dec to max and return.
2837  * - If it was a doorbell we return immediately since doorbells are edge
2839  * - If it was a HMI we return immediately since we handled it in realmode
2841  * - Else it is one of PACA_IRQ_MUST_HARD_MASK, so hard disable and return.
3107 	 * All code below __end_soft_masked is treated as soft-masked. If
3111 	 * The __end_interrupts marker must be past the out-of-line (OOL)
3134 	addi	r3,r3,(1f - 0b)
3148 	addi	r3,r3,(1f - 0b)