xref: /openbmc/linux/arch/s390/kernel/entry.S (revision 33ea04872da15ea8236f92da6009af5a1b0af641)
1b2441318SGreg Kroah-Hartman/* SPDX-License-Identifier: GPL-2.0 */
24bfc86ceSHeiko Carstens/*
34bfc86ceSHeiko Carstens *    S390 low-level entry points.
44bfc86ceSHeiko Carstens *
54bfc86ceSHeiko Carstens *    Copyright IBM Corp. 1999, 2012
64bfc86ceSHeiko Carstens *    Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com),
74bfc86ceSHeiko Carstens *		 Hartmut Penner (hp@de.ibm.com),
84bfc86ceSHeiko Carstens *		 Denis Joseph Barrow (djbarrow@de.ibm.com,barrow_dj@yahoo.com),
94bfc86ceSHeiko Carstens *		 Heiko Carstens <heiko.carstens@de.ibm.com>
104bfc86ceSHeiko Carstens */
114bfc86ceSHeiko Carstens
124bfc86ceSHeiko Carstens#include <linux/init.h>
134bfc86ceSHeiko Carstens#include <linux/linkage.h>
14b058661aSMartin Schwidefsky#include <asm/alternative-asm.h>
154bfc86ceSHeiko Carstens#include <asm/processor.h>
164bfc86ceSHeiko Carstens#include <asm/cache.h>
173037a52fSMartin Schwidefsky#include <asm/ctl_reg.h>
18dc24b7b4SHendrik Brueckner#include <asm/dwarf.h>
194bfc86ceSHeiko Carstens#include <asm/errno.h>
204bfc86ceSHeiko Carstens#include <asm/ptrace.h>
214bfc86ceSHeiko Carstens#include <asm/thread_info.h>
224bfc86ceSHeiko Carstens#include <asm/asm-offsets.h>
234bfc86ceSHeiko Carstens#include <asm/unistd.h>
244bfc86ceSHeiko Carstens#include <asm/page.h>
254bfc86ceSHeiko Carstens#include <asm/sigp.h>
264bfc86ceSHeiko Carstens#include <asm/irq.h>
279977e886SHendrik Brueckner#include <asm/vx-insn.h>
2883abeffbSHendrik Brueckner#include <asm/setup.h>
2983abeffbSHendrik Brueckner#include <asm/nmi.h>
30711f5df7SAl Viro#include <asm/export.h>
316dd85fbbSMartin Schwidefsky#include <asm/nospec-insn.h>
324bfc86ceSHeiko Carstens
334bfc86ceSHeiko Carstens__PT_R0      =	__PT_GPRS
344bfc86ceSHeiko Carstens__PT_R1      =	__PT_GPRS + 8
354bfc86ceSHeiko Carstens__PT_R2      =	__PT_GPRS + 16
364bfc86ceSHeiko Carstens__PT_R3      =	__PT_GPRS + 24
374bfc86ceSHeiko Carstens__PT_R4      =	__PT_GPRS + 32
384bfc86ceSHeiko Carstens__PT_R5      =	__PT_GPRS + 40
394bfc86ceSHeiko Carstens__PT_R6      =	__PT_GPRS + 48
404bfc86ceSHeiko Carstens__PT_R7      =	__PT_GPRS + 56
414bfc86ceSHeiko Carstens__PT_R8      =	__PT_GPRS + 64
424bfc86ceSHeiko Carstens__PT_R9      =	__PT_GPRS + 72
434bfc86ceSHeiko Carstens__PT_R10     =	__PT_GPRS + 80
444bfc86ceSHeiko Carstens__PT_R11     =	__PT_GPRS + 88
454bfc86ceSHeiko Carstens__PT_R12     =	__PT_GPRS + 96
464bfc86ceSHeiko Carstens__PT_R13     =	__PT_GPRS + 104
474bfc86ceSHeiko Carstens__PT_R14     =	__PT_GPRS + 112
484bfc86ceSHeiko Carstens__PT_R15     =	__PT_GPRS + 120
494bfc86ceSHeiko Carstens
503a890380SHeiko CarstensSTACK_SHIFT = PAGE_SHIFT + THREAD_SIZE_ORDER
514bfc86ceSHeiko CarstensSTACK_SIZE  = 1 << STACK_SHIFT
524bfc86ceSHeiko CarstensSTACK_INIT = STACK_SIZE - STACK_FRAME_OVERHEAD - __PT_SIZE
534bfc86ceSHeiko Carstens
54e5b98199SMartin Schwidefsky_LPP_OFFSET	= __LC_LPP
55e5b98199SMartin Schwidefsky
56ce3dc447SMartin Schwidefsky	.macro	CHECK_STACK savearea
574bfc86ceSHeiko Carstens#ifdef CONFIG_CHECK_STACK
58ce3dc447SMartin Schwidefsky	tml	%r15,STACK_SIZE - CONFIG_STACK_GUARD
594bfc86ceSHeiko Carstens	lghi	%r14,\savearea
604bfc86ceSHeiko Carstens	jz	stack_overflow
614bfc86ceSHeiko Carstens#endif
624bfc86ceSHeiko Carstens	.endm
634bfc86ceSHeiko Carstens
64ce3dc447SMartin Schwidefsky	.macro	CHECK_VMAP_STACK savearea,oklabel
65ce3dc447SMartin Schwidefsky#ifdef CONFIG_VMAP_STACK
66ce3dc447SMartin Schwidefsky	lgr	%r14,%r15
67ce3dc447SMartin Schwidefsky	nill	%r14,0x10000 - STACK_SIZE
68ce3dc447SMartin Schwidefsky	oill	%r14,STACK_INIT
69ce3dc447SMartin Schwidefsky	clg	%r14,__LC_KERNEL_STACK
70ce3dc447SMartin Schwidefsky	je	\oklabel
71ce3dc447SMartin Schwidefsky	clg	%r14,__LC_ASYNC_STACK
72ce3dc447SMartin Schwidefsky	je	\oklabel
73b61b1595SSven Schnelle	clg	%r14,__LC_MCCK_STACK
74b61b1595SSven Schnelle	je	\oklabel
75ce3dc447SMartin Schwidefsky	clg	%r14,__LC_NODAT_STACK
76ce3dc447SMartin Schwidefsky	je	\oklabel
77ce3dc447SMartin Schwidefsky	clg	%r14,__LC_RESTART_STACK
78ce3dc447SMartin Schwidefsky	je	\oklabel
79ce3dc447SMartin Schwidefsky	lghi	%r14,\savearea
80ce3dc447SMartin Schwidefsky	j	stack_overflow
81ce3dc447SMartin Schwidefsky#else
82ce3dc447SMartin Schwidefsky	j	\oklabel
83ce3dc447SMartin Schwidefsky#endif
84ce3dc447SMartin Schwidefsky	.endm
85ce3dc447SMartin Schwidefsky
864bfc86ceSHeiko Carstens	.macro STCK savearea
8778f65709SHeiko Carstens	ALTERNATIVE ".insn	s,0xb2050000,\savearea", \
8878f65709SHeiko Carstens		    ".insn	s,0xb27c0000,\savearea", 25
894bfc86ceSHeiko Carstens	.endm
904bfc86ceSHeiko Carstens
9183abeffbSHendrik Brueckner	/*
9283abeffbSHendrik Brueckner	 * The TSTMSK macro generates a test-under-mask instruction by
9383abeffbSHendrik Brueckner	 * calculating the memory offset for the specified mask value.
9483abeffbSHendrik Brueckner	 * Mask value can be any constant.  The macro shifts the mask
9583abeffbSHendrik Brueckner	 * value to calculate the memory offset for the test-under-mask
9683abeffbSHendrik Brueckner	 * instruction.
9783abeffbSHendrik Brueckner	 */
9883abeffbSHendrik Brueckner	.macro TSTMSK addr, mask, size=8, bytepos=0
9983abeffbSHendrik Brueckner		.if (\bytepos < \size) && (\mask >> 8)
10083abeffbSHendrik Brueckner			.if (\mask & 0xff)
10183abeffbSHendrik Brueckner				.error "Mask exceeds byte boundary"
10283abeffbSHendrik Brueckner			.endif
10383abeffbSHendrik Brueckner			TSTMSK \addr, "(\mask >> 8)", \size, "(\bytepos + 1)"
10483abeffbSHendrik Brueckner			.exitm
10583abeffbSHendrik Brueckner		.endif
10683abeffbSHendrik Brueckner		.ifeq \mask
10783abeffbSHendrik Brueckner			.error "Mask must not be zero"
10883abeffbSHendrik Brueckner		.endif
10983abeffbSHendrik Brueckner		off = \size - \bytepos - 1
11083abeffbSHendrik Brueckner		tm	off+\addr, \mask
11183abeffbSHendrik Brueckner	.endm
11283abeffbSHendrik Brueckner
113d768bd89SMartin Schwidefsky	.macro BPOFF
114b058661aSMartin Schwidefsky	ALTERNATIVE "", ".long 0xb2e8c000", 82
115d768bd89SMartin Schwidefsky	.endm
116d768bd89SMartin Schwidefsky
117d768bd89SMartin Schwidefsky	.macro BPON
118b058661aSMartin Schwidefsky	ALTERNATIVE "", ".long 0xb2e8d000", 82
119d768bd89SMartin Schwidefsky	.endm
120d768bd89SMartin Schwidefsky
1216b73044bSMartin Schwidefsky	.macro BPENTER tif_ptr,tif_mask
122b058661aSMartin Schwidefsky	ALTERNATIVE "TSTMSK \tif_ptr,\tif_mask; jz .+8; .long 0xb2e8d000", \
123b058661aSMartin Schwidefsky		    "", 82
1246b73044bSMartin Schwidefsky	.endm
1256b73044bSMartin Schwidefsky
1266b73044bSMartin Schwidefsky	.macro BPEXIT tif_ptr,tif_mask
1276b73044bSMartin Schwidefsky	TSTMSK	\tif_ptr,\tif_mask
128b058661aSMartin Schwidefsky	ALTERNATIVE "jz .+8;  .long 0xb2e8c000", \
129b058661aSMartin Schwidefsky		    "jnz .+8; .long 0xb2e8d000", 82
1306b73044bSMartin Schwidefsky	.endm
1316b73044bSMartin Schwidefsky
1326dd85fbbSMartin Schwidefsky	GEN_BR_THUNK %r14
133*33ea0487SSven Schnelle	GEN_BR_THUNK %r14,%r13
134f19fbd5eSMartin Schwidefsky
1354bfc86ceSHeiko Carstens	.section .kprobes.text, "ax"
13646210c44SHeiko Carstens.Ldummy:
13746210c44SHeiko Carstens	/*
13856e62a73SSven Schnelle	 * This nop exists only in order to avoid that __bpon starts at
13946210c44SHeiko Carstens	 * the beginning of the kprobes text section. In that case we would
14046210c44SHeiko Carstens	 * have several symbols at the same address. E.g. objdump would take
14146210c44SHeiko Carstens	 * an arbitrary symbol name when disassembling this code.
14256e62a73SSven Schnelle	 * With the added nop in between the __bpon symbol is unique
14346210c44SHeiko Carstens	 * again.
14446210c44SHeiko Carstens	 */
14546210c44SHeiko Carstens	nop	0
1464bfc86ceSHeiko Carstens
147d768bd89SMartin SchwidefskyENTRY(__bpon)
148d768bd89SMartin Schwidefsky	.globl __bpon
149d768bd89SMartin Schwidefsky	BPON
1506dd85fbbSMartin Schwidefsky	BR_EX	%r14
15126a374aeSMartin SchwidefskyENDPROC(__bpon)
152d768bd89SMartin Schwidefsky
1534bfc86ceSHeiko Carstens/*
1544bfc86ceSHeiko Carstens * Scheduler resume function, called by switch_to
1554bfc86ceSHeiko Carstens *  gpr2 = (task_struct *) prev
1564bfc86ceSHeiko Carstens *  gpr3 = (task_struct *) next
1574bfc86ceSHeiko Carstens * Returns:
1584bfc86ceSHeiko Carstens *  gpr2 = prev
1594bfc86ceSHeiko Carstens */
1604bfc86ceSHeiko CarstensENTRY(__switch_to)
1614bfc86ceSHeiko Carstens	stmg	%r6,%r15,__SF_GPRS(%r15)	# store gprs of prev task
1623241d3ebSHeiko Carstens	lghi	%r4,__TASK_stack
1633241d3ebSHeiko Carstens	lghi	%r1,__TASK_thread
1649fed920eSVasily Gorbik	llill	%r5,STACK_INIT
1653241d3ebSHeiko Carstens	stg	%r15,__THREAD_ksp(%r1,%r2)	# store kernel stack of prev
1669fed920eSVasily Gorbik	lg	%r15,0(%r4,%r3)			# start of kernel stack of next
1679fed920eSVasily Gorbik	agr	%r15,%r5			# end of kernel stack of next
1684bfc86ceSHeiko Carstens	stg	%r3,__LC_CURRENT		# store task struct of next
1694bfc86ceSHeiko Carstens	stg	%r15,__LC_KERNEL_STACK		# store end of kernel stack
1703241d3ebSHeiko Carstens	lg	%r15,__THREAD_ksp(%r1,%r3)	# load kernel stack of next
1713241d3ebSHeiko Carstens	aghi	%r3,__TASK_pid
1723241d3ebSHeiko Carstens	mvc	__LC_CURRENT_PID(4,%r0),0(%r3)	# store pid of next
1734bfc86ceSHeiko Carstens	lmg	%r6,%r15,__SF_GPRS(%r15)	# load gprs of next task
174e5b98199SMartin Schwidefsky	ALTERNATIVE "", ".insn s,0xb2800000,_LPP_OFFSET", 40
1756dd85fbbSMartin Schwidefsky	BR_EX	%r14
17626a374aeSMartin SchwidefskyENDPROC(__switch_to)
1774bfc86ceSHeiko Carstens
178d0fc4107SMartin Schwidefsky#if IS_ENABLED(CONFIG_KVM)
179d0fc4107SMartin Schwidefsky/*
180d0fc4107SMartin Schwidefsky * sie64a calling convention:
181d0fc4107SMartin Schwidefsky * %r2 pointer to sie control block
182d0fc4107SMartin Schwidefsky * %r3 guest register save area
183d0fc4107SMartin Schwidefsky */
184d0fc4107SMartin SchwidefskyENTRY(sie64a)
185d0fc4107SMartin Schwidefsky	stmg	%r6,%r14,__SF_GPRS(%r15)	# save kernel registers
1866b73044bSMartin Schwidefsky	lg	%r12,__LC_CURRENT
18792fa7a13SMartin Schwidefsky	stg	%r2,__SF_SIE_CONTROL(%r15)	# save control block pointer
18892fa7a13SMartin Schwidefsky	stg	%r3,__SF_SIE_SAVEAREA(%r15)	# save guest register save area
18992fa7a13SMartin Schwidefsky	xc	__SF_SIE_REASON(8,%r15),__SF_SIE_REASON(%r15) # reason code = 0
19092fa7a13SMartin Schwidefsky	mvc	__SF_SIE_FLAGS(8,%r15),__TI_flags(%r12) # copy thread flags
191d0fc4107SMartin Schwidefsky	lmg	%r0,%r13,0(%r3)			# load guest gprs 0-13
192d0fc4107SMartin Schwidefsky	lg	%r14,__LC_GMAP			# get gmap pointer
193d0fc4107SMartin Schwidefsky	ltgr	%r14,%r14
194d0fc4107SMartin Schwidefsky	jz	.Lsie_gmap
195d0fc4107SMartin Schwidefsky	lctlg	%c1,%c1,__GMAP_ASCE(%r14)	# load primary asce
196d0fc4107SMartin Schwidefsky.Lsie_gmap:
19792fa7a13SMartin Schwidefsky	lg	%r14,__SF_SIE_CONTROL(%r15)	# get control block pointer
198d0fc4107SMartin Schwidefsky	oi	__SIE_PROG0C+3(%r14),1		# we are going into SIE now
199d0fc4107SMartin Schwidefsky	tm	__SIE_PROG20+3(%r14),3		# last exit...
200d0fc4107SMartin Schwidefsky	jnz	.Lsie_skip
20183abeffbSHendrik Brueckner	TSTMSK	__LC_CPU_FLAGS,_CIF_FPU
202d0fc4107SMartin Schwidefsky	jo	.Lsie_skip			# exit if fp/vx regs changed
20392fa7a13SMartin Schwidefsky	BPEXIT	__SF_SIE_FLAGS(%r15),(_TIF_ISOLATE_BP|_TIF_ISOLATE_BP_GUEST)
204c929500dSQingFeng Hao.Lsie_entry:
205d0fc4107SMartin Schwidefsky	sie	0(%r14)
206d768bd89SMartin Schwidefsky	BPOFF
20792fa7a13SMartin Schwidefsky	BPENTER	__SF_SIE_FLAGS(%r15),(_TIF_ISOLATE_BP|_TIF_ISOLATE_BP_GUEST)
208d0fc4107SMartin Schwidefsky.Lsie_skip:
209d0fc4107SMartin Schwidefsky	ni	__SIE_PROG0C+3(%r14),0xfe	# no longer in SIE
21087d59863SHeiko Carstens	lctlg	%c1,%c1,__LC_KERNEL_ASCE	# load primary asce
211d0fc4107SMartin Schwidefsky.Lsie_done:
212d0fc4107SMartin Schwidefsky# some program checks are suppressing. C code (e.g. do_protection_exception)
213c0e7bb38SChristian Borntraeger# will rewind the PSW by the ILC, which is often 4 bytes in case of SIE. There
214c0e7bb38SChristian Borntraeger# are some corner cases (e.g. runtime instrumentation) where ILC is unpredictable.
215c0e7bb38SChristian Borntraeger# Other instructions between sie64a and .Lsie_done should not cause program
216c0e7bb38SChristian Borntraeger# interrupts. So lets use 3 nops as a landing pad for all possible rewinds.
217d0fc4107SMartin Schwidefsky# See also .Lcleanup_sie
218c0e7bb38SChristian Borntraeger.Lrewind_pad6:
219c0e7bb38SChristian Borntraeger	nopr	7
220c0e7bb38SChristian Borntraeger.Lrewind_pad4:
221c0e7bb38SChristian Borntraeger	nopr	7
222c0e7bb38SChristian Borntraeger.Lrewind_pad2:
223c0e7bb38SChristian Borntraeger	nopr	7
224d0fc4107SMartin Schwidefsky	.globl sie_exit
225d0fc4107SMartin Schwidefskysie_exit:
22692fa7a13SMartin Schwidefsky	lg	%r14,__SF_SIE_SAVEAREA(%r15)	# load guest register save area
227d0fc4107SMartin Schwidefsky	stmg	%r0,%r13,0(%r14)		# save guest gprs 0-13
2287041d281SMartin Schwidefsky	xgr	%r0,%r0				# clear guest registers to
2297041d281SMartin Schwidefsky	xgr	%r1,%r1				# prevent speculative use
2307041d281SMartin Schwidefsky	xgr	%r3,%r3
2317041d281SMartin Schwidefsky	xgr	%r4,%r4
2327041d281SMartin Schwidefsky	xgr	%r5,%r5
233d0fc4107SMartin Schwidefsky	lmg	%r6,%r14,__SF_GPRS(%r15)	# restore kernel registers
23492fa7a13SMartin Schwidefsky	lg	%r2,__SF_SIE_REASON(%r15)	# return exit reason code
2356dd85fbbSMartin Schwidefsky	BR_EX	%r14
236d0fc4107SMartin Schwidefsky.Lsie_fault:
237d0fc4107SMartin Schwidefsky	lghi	%r14,-EFAULT
23892fa7a13SMartin Schwidefsky	stg	%r14,__SF_SIE_REASON(%r15)	# set exit reason code
239d0fc4107SMartin Schwidefsky	j	sie_exit
240d0fc4107SMartin Schwidefsky
241c0e7bb38SChristian Borntraeger	EX_TABLE(.Lrewind_pad6,.Lsie_fault)
242c0e7bb38SChristian Borntraeger	EX_TABLE(.Lrewind_pad4,.Lsie_fault)
243c0e7bb38SChristian Borntraeger	EX_TABLE(.Lrewind_pad2,.Lsie_fault)
244d0fc4107SMartin Schwidefsky	EX_TABLE(sie_exit,.Lsie_fault)
24526a374aeSMartin SchwidefskyENDPROC(sie64a)
246711f5df7SAl ViroEXPORT_SYMBOL(sie64a)
247711f5df7SAl ViroEXPORT_SYMBOL(sie_exit)
248d0fc4107SMartin Schwidefsky#endif
249d0fc4107SMartin Schwidefsky
2504bfc86ceSHeiko Carstens/*
2514bfc86ceSHeiko Carstens * SVC interrupt handler routine. System calls are synchronous events and
2527b7735c5SChristian Borntraeger * are entered with interrupts disabled.
2534bfc86ceSHeiko Carstens */
2544bfc86ceSHeiko Carstens
2554bfc86ceSHeiko CarstensENTRY(system_call)
25656e62a73SSven Schnelle	stpt	__LC_SYS_ENTER_TIMER
2574bfc86ceSHeiko Carstens	stmg	%r8,%r15,__LC_SAVE_AREA_SYNC
258d768bd89SMartin Schwidefsky	BPOFF
25956e62a73SSven Schnelle	lghi	%r14,0
2604bfc86ceSHeiko Carstens.Lsysc_per:
26187d59863SHeiko Carstens	lctlg	%c1,%c1,__LC_KERNEL_ASCE
26256e62a73SSven Schnelle	lg	%r12,__LC_CURRENT
2634bfc86ceSHeiko Carstens	lg	%r15,__LC_KERNEL_STACK
2649365965dSHeiko Carstens	xc	__SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
26556e62a73SSven Schnelle	stmg	%r0,%r7,STACK_FRAME_OVERHEAD+__PT_R0(%r15)
26656e62a73SSven Schnelle	BPENTER __TI_flags(%r12),_TIF_ISOLATE_BP
267d3f46896SChristian Borntraeger	# clear user controlled register to prevent speculative use
268d3f46896SChristian Borntraeger	xgr	%r0,%r0
26956e62a73SSven Schnelle	xgr	%r1,%r1
27056e62a73SSven Schnelle	xgr	%r4,%r4
27156e62a73SSven Schnelle	xgr	%r5,%r5
27256e62a73SSven Schnelle	xgr	%r6,%r6
27356e62a73SSven Schnelle	xgr	%r7,%r7
27456e62a73SSven Schnelle	xgr	%r8,%r8
27556e62a73SSven Schnelle	xgr	%r9,%r9
27656e62a73SSven Schnelle	xgr	%r10,%r10
27756e62a73SSven Schnelle	xgr	%r11,%r11
27856e62a73SSven Schnelle	la	%r2,STACK_FRAME_OVERHEAD(%r15)	# pointer to pt_regs
27956e62a73SSven Schnelle	lgr	%r3,%r14
28056e62a73SSven Schnelle	brasl	%r14,__do_syscall
28187d59863SHeiko Carstens	lctlg	%c1,%c1,__LC_USER_ASCE
28256e62a73SSven Schnelle	mvc	__LC_RETURN_PSW(16),STACK_FRAME_OVERHEAD+__PT_PSW(%r15)
2836b73044bSMartin Schwidefsky	BPEXIT __TI_flags(%r12),_TIF_ISOLATE_BP
28456e62a73SSven Schnelle	lmg	%r0,%r15,STACK_FRAME_OVERHEAD+__PT_R0(%r15)
2854bfc86ceSHeiko Carstens	stpt	__LC_EXIT_TIMER
2860b0ed657SSven Schnelle	b	__LC_RETURN_LPSWE
28726a374aeSMartin SchwidefskyENDPROC(system_call)
2884bfc86ceSHeiko Carstens
2894bfc86ceSHeiko Carstens#
2904bfc86ceSHeiko Carstens# a new process exits the kernel with ret_from_fork
2914bfc86ceSHeiko Carstens#
2924bfc86ceSHeiko CarstensENTRY(ret_from_fork)
29356e62a73SSven Schnelle	lgr	%r3,%r11
29456e62a73SSven Schnelle	brasl	%r14,__ret_from_fork
29556e62a73SSven Schnelle	lctlg	%c1,%c1,__LC_USER_ASCE
29656e62a73SSven Schnelle	mvc	__LC_RETURN_PSW(16),STACK_FRAME_OVERHEAD+__PT_PSW(%r15)
29756e62a73SSven Schnelle	BPEXIT __TI_flags(%r12),_TIF_ISOLATE_BP
29856e62a73SSven Schnelle	lmg	%r0,%r15,STACK_FRAME_OVERHEAD+__PT_R0(%r15)
29956e62a73SSven Schnelle	stpt	__LC_EXIT_TIMER
30056e62a73SSven Schnelle	b	__LC_RETURN_LPSWE
30126a374aeSMartin SchwidefskyENDPROC(ret_from_fork)
30226a374aeSMartin Schwidefsky
3034bfc86ceSHeiko Carstens/*
3044bfc86ceSHeiko Carstens * Program check handler routine
3054bfc86ceSHeiko Carstens */
3064bfc86ceSHeiko Carstens
3074bfc86ceSHeiko CarstensENTRY(pgm_check_handler)
30856e62a73SSven Schnelle	stpt	__LC_SYS_ENTER_TIMER
309d768bd89SMartin Schwidefsky	BPOFF
3104bfc86ceSHeiko Carstens	stmg	%r8,%r15,__LC_SAVE_AREA_SYNC
31156e62a73SSven Schnelle	lg	%r12,__LC_CURRENT
31256e62a73SSven Schnelle	lghi	%r10,0
3134bfc86ceSHeiko Carstens	lmg	%r8,%r9,__LC_PGM_OLD_PSW
31487d59863SHeiko Carstens	tmhh	%r8,0x0001		# coming from user space?
31587d59863SHeiko Carstens	jno	.Lpgm_skip_asce
31687d59863SHeiko Carstens	lctlg	%c1,%c1,__LC_KERNEL_ASCE
31756e62a73SSven Schnelle	j	3f			# -> fault in user space
31887d59863SHeiko Carstens.Lpgm_skip_asce:
319d0fc4107SMartin Schwidefsky#if IS_ENABLED(CONFIG_KVM)
3200a5e2ec2SMartin Schwidefsky	# cleanup critical section for program checks in sie64a
321d0fc4107SMartin Schwidefsky	lgr	%r14,%r9
3220b0ed657SSven Schnelle	larl	%r13,.Lsie_gmap
3230b0ed657SSven Schnelle	slgr	%r14,%r13
3240b0ed657SSven Schnelle	lghi	%r13,.Lsie_done - .Lsie_gmap
3250b0ed657SSven Schnelle	clgr	%r14,%r13
3260b38b5e1SSven Schnelle	jhe	1f
32792fa7a13SMartin Schwidefsky	lg	%r14,__SF_SIE_CONTROL(%r15)	# get control block pointer
3280a5e2ec2SMartin Schwidefsky	ni	__SIE_PROG0C+3(%r14),0xfe	# no longer in SIE
32987d59863SHeiko Carstens	lctlg	%c1,%c1,__LC_KERNEL_ASCE	# load primary asce
3300a5e2ec2SMartin Schwidefsky	larl	%r9,sie_exit			# skip forward to sie_exit
33156e62a73SSven Schnelle	lghi	%r10,_PIF_GUEST_FAULT
332d0fc4107SMartin Schwidefsky#endif
3330b38b5e1SSven Schnelle1:	tmhh	%r8,0x4000		# PER bit set in old PSW ?
3340b38b5e1SSven Schnelle	jnz	2f			# -> enabled, can't be a double fault
3354bfc86ceSHeiko Carstens	tm	__LC_PGM_ILC+3,0x80	# check for per exception
3364bfc86ceSHeiko Carstens	jnz	.Lpgm_svcper		# -> single stepped svc
3370b38b5e1SSven Schnelle2:	CHECK_STACK __LC_SAVE_AREA_SYNC
3384bfc86ceSHeiko Carstens	aghi	%r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
33956e62a73SSven Schnelle	# CHECK_VMAP_STACK branches to stack_overflow or 4f
34056e62a73SSven Schnelle	CHECK_VMAP_STACK __LC_SAVE_AREA_SYNC,4f
34156e62a73SSven Schnelle3:	BPENTER __TI_flags(%r12),_TIF_ISOLATE_BP
3424bfc86ceSHeiko Carstens	lg	%r15,__LC_KERNEL_STACK
34356e62a73SSven Schnelle4:	la	%r11,STACK_FRAME_OVERHEAD(%r15)
34456e62a73SSven Schnelle	stg	%r10,__PT_FLAGS(%r11)
34556e62a73SSven Schnelle	xc	__SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
3464bfc86ceSHeiko Carstens	stmg	%r0,%r7,__PT_R0(%r11)
34756e62a73SSven Schnelle	mvc	__PT_R8(64,%r11),__LC_SAVE_AREA_SYNC
34856e62a73SSven Schnelle	stmg	%r8,%r9,__PT_PSW(%r11)
34956e62a73SSven Schnelle
3507041d281SMartin Schwidefsky	# clear user controlled registers to prevent speculative use
3517041d281SMartin Schwidefsky	xgr	%r0,%r0
3527041d281SMartin Schwidefsky	xgr	%r1,%r1
3537041d281SMartin Schwidefsky	xgr	%r3,%r3
3547041d281SMartin Schwidefsky	xgr	%r4,%r4
3557041d281SMartin Schwidefsky	xgr	%r5,%r5
3567041d281SMartin Schwidefsky	xgr	%r6,%r6
3577041d281SMartin Schwidefsky	xgr	%r7,%r7
35856e62a73SSven Schnelle	lgr	%r2,%r11
35956e62a73SSven Schnelle	brasl	%r14,__do_pgm_check
36056e62a73SSven Schnelle	tmhh	%r8,0x0001		# returning to user space?
36156e62a73SSven Schnelle	jno	.Lpgm_exit_kernel
36256e62a73SSven Schnelle	lctlg	%c1,%c1,__LC_USER_ASCE
36356e62a73SSven Schnelle	BPEXIT __TI_flags(%r12),_TIF_ISOLATE_BP
3640cd9b723SHeiko Carstens	stpt	__LC_EXIT_TIMER
36556e62a73SSven Schnelle.Lpgm_exit_kernel:
36656e62a73SSven Schnelle	mvc	__LC_RETURN_PSW(16),STACK_FRAME_OVERHEAD+__PT_PSW(%r15)
36756e62a73SSven Schnelle	lmg	%r0,%r15,STACK_FRAME_OVERHEAD+__PT_R0(%r15)
3680cd9b723SHeiko Carstens	b	__LC_RETURN_LPSWE
3694bfc86ceSHeiko Carstens
3704bfc86ceSHeiko Carstens#
3714bfc86ceSHeiko Carstens# single stepped system call
3724bfc86ceSHeiko Carstens#
3734bfc86ceSHeiko Carstens.Lpgm_svcper:
3744bfc86ceSHeiko Carstens	mvc	__LC_RETURN_PSW(8),__LC_SVC_NEW_PSW
3754bfc86ceSHeiko Carstens	larl	%r14,.Lsysc_per
3764bfc86ceSHeiko Carstens	stg	%r14,__LC_RETURN_PSW+8
37756e62a73SSven Schnelle	lghi	%r14,1
3780b0ed657SSven Schnelle	lpswe	__LC_RETURN_PSW		# branch to .Lsysc_per
37926a374aeSMartin SchwidefskyENDPROC(pgm_check_handler)
3804bfc86ceSHeiko Carstens
3814bfc86ceSHeiko Carstens/*
38256e62a73SSven Schnelle * Interrupt handler macro used for external and IO interrupts.
3834bfc86ceSHeiko Carstens */
38456e62a73SSven Schnelle.macro INT_HANDLER name,lc_old_psw,handler
38556e62a73SSven SchnelleENTRY(\name)
3864bfc86ceSHeiko Carstens	STCK	__LC_INT_CLOCK
38756e62a73SSven Schnelle	stpt	__LC_SYS_ENTER_TIMER
388d768bd89SMartin Schwidefsky	BPOFF
3894bfc86ceSHeiko Carstens	stmg	%r8,%r15,__LC_SAVE_AREA_ASYNC
390d5c352cdSHeiko Carstens	lg	%r12,__LC_CURRENT
39156e62a73SSven Schnelle	lmg	%r8,%r9,\lc_old_psw
392b0d31159SSven Schnelle	tmhh	%r8,0x0001			# interrupting from user ?
393b0d31159SSven Schnelle	jnz	1f
394b0d31159SSven Schnelle#if IS_ENABLED(CONFIG_KVM)
395b0d31159SSven Schnelle	lgr	%r14,%r9
396b0d31159SSven Schnelle	larl	%r13,.Lsie_gmap
397b0d31159SSven Schnelle	slgr	%r14,%r13
398b0d31159SSven Schnelle	lghi	%r13,.Lsie_done - .Lsie_gmap
399b0d31159SSven Schnelle	clgr	%r14,%r13
400b0d31159SSven Schnelle	jhe	0f
401b0d31159SSven Schnelle	lghi	%r11,__LC_SAVE_AREA_ASYNC	# inside critical section, do cleanup
402b0d31159SSven Schnelle	brasl	%r14,.Lcleanup_sie
403b0d31159SSven Schnelle#endif
404b0d31159SSven Schnelle0:	CHECK_STACK __LC_SAVE_AREA_ASYNC
405b0d31159SSven Schnelle	lgr	%r11,%r15
406b0d31159SSven Schnelle	aghi	%r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
407b0d31159SSven Schnelle	stg	%r11,__SF_BACKCHAIN(%r15)
408b0d31159SSven Schnelle	j	2f
409b0d31159SSven Schnelle1:	BPENTER __TI_flags(%r12),_TIF_ISOLATE_BP
410b0d31159SSven Schnelle	lctlg	%c1,%c1,__LC_KERNEL_ASCE
411b0d31159SSven Schnelle	lg	%r15,__LC_KERNEL_STACK
412b0d31159SSven Schnelle	xc	__SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
413b0d31159SSven Schnelle2:	la	%r11,STACK_FRAME_OVERHEAD(%r15)
4144bfc86ceSHeiko Carstens	stmg	%r0,%r7,__PT_R0(%r11)
4157041d281SMartin Schwidefsky	# clear user controlled registers to prevent speculative use
4167041d281SMartin Schwidefsky	xgr	%r0,%r0
4177041d281SMartin Schwidefsky	xgr	%r1,%r1
4187041d281SMartin Schwidefsky	xgr	%r3,%r3
4197041d281SMartin Schwidefsky	xgr	%r4,%r4
4207041d281SMartin Schwidefsky	xgr	%r5,%r5
4217041d281SMartin Schwidefsky	xgr	%r6,%r6
4227041d281SMartin Schwidefsky	xgr	%r7,%r7
4237041d281SMartin Schwidefsky	xgr	%r10,%r10
4244bfc86ceSHeiko Carstens	mvc	__PT_R8(64,%r11),__LC_SAVE_AREA_ASYNC
4254bfc86ceSHeiko Carstens	stmg	%r8,%r9,__PT_PSW(%r11)
42656e62a73SSven Schnelle	tm	%r8,0x0001		# coming from user space?
42756e62a73SSven Schnelle	jno	1f
42887d59863SHeiko Carstens	lctlg	%c1,%c1,__LC_KERNEL_ASCE
42956e62a73SSven Schnelle1:	lgr	%r2,%r11		# pass pointer to pt_regs
43056e62a73SSven Schnelle	brasl	%r14,\handler
4314bfc86ceSHeiko Carstens	mvc	__LC_RETURN_PSW(16),__PT_PSW(%r11)
43256e62a73SSven Schnelle	tmhh	%r8,0x0001		# returning to user ?
43356e62a73SSven Schnelle	jno	2f
43487d59863SHeiko Carstens	lctlg	%c1,%c1,__LC_USER_ASCE
4356b73044bSMartin Schwidefsky	BPEXIT	__TI_flags(%r12),_TIF_ISOLATE_BP
4364bfc86ceSHeiko Carstens	stpt	__LC_EXIT_TIMER
43756e62a73SSven Schnelle2:	lmg	%r0,%r15,__PT_R0(%r11)
4380b0ed657SSven Schnelle	b	__LC_RETURN_LPSWE
43956e62a73SSven SchnelleENDPROC(\name)
44056e62a73SSven Schnelle.endm
4414bfc86ceSHeiko Carstens
44256e62a73SSven SchnelleINT_HANDLER ext_int_handler,__LC_EXT_OLD_PSW,do_ext_irq
44356e62a73SSven SchnelleINT_HANDLER io_int_handler,__LC_IO_OLD_PSW,do_io_irq
4444bfc86ceSHeiko Carstens
4454bfc86ceSHeiko Carstens/*
4460b0ed657SSven Schnelle * Load idle PSW.
4474bfc86ceSHeiko Carstens */
4484bfc86ceSHeiko CarstensENTRY(psw_idle)
4494bfc86ceSHeiko Carstens	stg	%r3,__SF_EMPTY(%r15)
45056e62a73SSven Schnelle	larl	%r1,psw_idle_exit
4514bfc86ceSHeiko Carstens	stg	%r1,__SF_EMPTY+8(%r15)
45272d38b19SMartin Schwidefsky	larl	%r1,smp_cpu_mtid
45372d38b19SMartin Schwidefsky	llgf	%r1,0(%r1)
45472d38b19SMartin Schwidefsky	ltgr	%r1,%r1
45572d38b19SMartin Schwidefsky	jz	.Lpsw_idle_stcctm
45656e62a73SSven Schnelle	.insn	rsy,0xeb0000000017,%r1,5,__MT_CYCLES_ENTER(%r2)
45772d38b19SMartin Schwidefsky.Lpsw_idle_stcctm:
458419123f9SMartin Schwidefsky	oi	__LC_CPU_FLAGS+7,_CIF_ENABLED_WAIT
459d768bd89SMartin Schwidefsky	BPON
4604bfc86ceSHeiko Carstens	STCK	__CLOCK_IDLE_ENTER(%r2)
4614bfc86ceSHeiko Carstens	stpt	__TIMER_IDLE_ENTER(%r2)
4624bfc86ceSHeiko Carstens	lpswe	__SF_EMPTY(%r15)
46356e62a73SSven Schnelle.globl psw_idle_exit
46456e62a73SSven Schnellepsw_idle_exit:
4656dd85fbbSMartin Schwidefsky	BR_EX	%r14
46626a374aeSMartin SchwidefskyENDPROC(psw_idle)
4674bfc86ceSHeiko Carstens
468b5510d9bSHendrik Brueckner/*
4694bfc86ceSHeiko Carstens * Machine check handler routines
4704bfc86ceSHeiko Carstens */
4714bfc86ceSHeiko CarstensENTRY(mcck_int_handler)
4724bfc86ceSHeiko Carstens	STCK	__LC_MCCK_CLOCK
473d768bd89SMartin Schwidefsky	BPOFF
4743037a52fSMartin Schwidefsky	la	%r1,4095		# validate r1
4753037a52fSMartin Schwidefsky	spt	__LC_CPU_TIMER_SAVE_AREA-4095(%r1)	# validate cpu timer
4763037a52fSMartin Schwidefsky	sckc	__LC_CLOCK_COMPARATOR			# validate comparator
4773037a52fSMartin Schwidefsky	lam	%a0,%a15,__LC_AREGS_SAVE_AREA-4095(%r1) # validate acrs
4783037a52fSMartin Schwidefsky	lmg	%r0,%r15,__LC_GPREGS_SAVE_AREA-4095(%r1)# validate gprs
479d5c352cdSHeiko Carstens	lg	%r12,__LC_CURRENT
4804bfc86ceSHeiko Carstens	lmg	%r8,%r9,__LC_MCK_OLD_PSW
48183abeffbSHendrik Brueckner	TSTMSK	__LC_MCCK_CODE,MCCK_CODE_SYSTEM_DAMAGE
4824bfc86ceSHeiko Carstens	jo	.Lmcck_panic		# yes -> rest of mcck code invalid
4833037a52fSMartin Schwidefsky	TSTMSK	__LC_MCCK_CODE,MCCK_CODE_CR_VALID
4843037a52fSMartin Schwidefsky	jno	.Lmcck_panic		# control registers invalid -> panic
4853037a52fSMartin Schwidefsky	la	%r14,4095
4863037a52fSMartin Schwidefsky	lctlg	%c0,%c15,__LC_CREGS_SAVE_AREA-4095(%r14) # validate ctl regs
4873037a52fSMartin Schwidefsky	ptlb
4882a2d7befSVasily Gorbik	lg	%r11,__LC_MCESAD-4095(%r14) # extended machine check save area
4893037a52fSMartin Schwidefsky	nill	%r11,0xfc00		# MCESA_ORIGIN_MASK
4903037a52fSMartin Schwidefsky	TSTMSK	__LC_CREGS_SAVE_AREA+16-4095(%r14),CR2_GUARDED_STORAGE
4913037a52fSMartin Schwidefsky	jno	0f
4923037a52fSMartin Schwidefsky	TSTMSK	__LC_MCCK_CODE,MCCK_CODE_GS_VALID
4933037a52fSMartin Schwidefsky	jno	0f
4943037a52fSMartin Schwidefsky	.insn	 rxy,0xe3000000004d,0,__MCESA_GS_SAVE_AREA(%r11) # LGSC
4953037a52fSMartin Schwidefsky0:	l	%r14,__LC_FP_CREG_SAVE_AREA-4095(%r14)
4963037a52fSMartin Schwidefsky	TSTMSK	__LC_MCCK_CODE,MCCK_CODE_FC_VALID
4973037a52fSMartin Schwidefsky	jo	0f
4983037a52fSMartin Schwidefsky	sr	%r14,%r14
4993037a52fSMartin Schwidefsky0:	sfpc	%r14
5003037a52fSMartin Schwidefsky	TSTMSK	__LC_MACHINE_FLAGS,MACHINE_FLAG_VX
5013037a52fSMartin Schwidefsky	jo	0f
5023037a52fSMartin Schwidefsky	lghi	%r14,__LC_FPREGS_SAVE_AREA
5033037a52fSMartin Schwidefsky	ld	%f0,0(%r14)
5043037a52fSMartin Schwidefsky	ld	%f1,8(%r14)
5053037a52fSMartin Schwidefsky	ld	%f2,16(%r14)
5063037a52fSMartin Schwidefsky	ld	%f3,24(%r14)
5073037a52fSMartin Schwidefsky	ld	%f4,32(%r14)
5083037a52fSMartin Schwidefsky	ld	%f5,40(%r14)
5093037a52fSMartin Schwidefsky	ld	%f6,48(%r14)
5103037a52fSMartin Schwidefsky	ld	%f7,56(%r14)
5113037a52fSMartin Schwidefsky	ld	%f8,64(%r14)
5123037a52fSMartin Schwidefsky	ld	%f9,72(%r14)
5133037a52fSMartin Schwidefsky	ld	%f10,80(%r14)
5143037a52fSMartin Schwidefsky	ld	%f11,88(%r14)
5153037a52fSMartin Schwidefsky	ld	%f12,96(%r14)
5163037a52fSMartin Schwidefsky	ld	%f13,104(%r14)
5173037a52fSMartin Schwidefsky	ld	%f14,112(%r14)
5183037a52fSMartin Schwidefsky	ld	%f15,120(%r14)
5193037a52fSMartin Schwidefsky	j	1f
5203037a52fSMartin Schwidefsky0:	VLM	%v0,%v15,0,%r11
5213037a52fSMartin Schwidefsky	VLM	%v16,%v31,256,%r11
5223037a52fSMartin Schwidefsky1:	lghi	%r14,__LC_CPU_TIMER_SAVE_AREA
5234bfc86ceSHeiko Carstens	mvc	__LC_MCCK_ENTER_TIMER(8),0(%r14)
52483abeffbSHendrik Brueckner	TSTMSK	__LC_MCCK_CODE,MCCK_CODE_CPU_TIMER_VALID
5254bfc86ceSHeiko Carstens	jo	3f
52656e62a73SSven Schnelle	la	%r14,__LC_SYS_ENTER_TIMER
52756e62a73SSven Schnelle	clc	0(8,%r14),__LC_EXIT_TIMER
5284bfc86ceSHeiko Carstens	jl	1f
5294bfc86ceSHeiko Carstens	la	%r14,__LC_EXIT_TIMER
5304bfc86ceSHeiko Carstens1:	clc	0(8,%r14),__LC_LAST_UPDATE_TIMER
5314bfc86ceSHeiko Carstens	jl	2f
5324bfc86ceSHeiko Carstens	la	%r14,__LC_LAST_UPDATE_TIMER
5334bfc86ceSHeiko Carstens2:	spt	0(%r14)
5344bfc86ceSHeiko Carstens	mvc	__LC_MCCK_ENTER_TIMER(8),0(%r14)
5353037a52fSMartin Schwidefsky3:	TSTMSK	__LC_MCCK_CODE,MCCK_CODE_PSW_MWP_VALID
5363037a52fSMartin Schwidefsky	jno	.Lmcck_panic
5373037a52fSMartin Schwidefsky	tmhh	%r8,0x0001		# interrupting from user ?
5383037a52fSMartin Schwidefsky	jnz	4f
5393037a52fSMartin Schwidefsky	TSTMSK	__LC_MCCK_CODE,MCCK_CODE_PSW_IA_VALID
5403037a52fSMartin Schwidefsky	jno	.Lmcck_panic
541ce3dc447SMartin Schwidefsky4:	ssm	__LC_PGM_NEW_PSW	# turn dat on, keep irqs off
542b0d31159SSven Schnelle	tmhh	%r8,0x0001			# interrupting from user ?
543b0d31159SSven Schnelle	jnz	.Lmcck_user
544b0d31159SSven Schnelle#if IS_ENABLED(CONFIG_KVM)
545b0d31159SSven Schnelle	lgr	%r14,%r9
546b0d31159SSven Schnelle	larl	%r13,.Lsie_gmap
547b0d31159SSven Schnelle	slgr	%r14,%r13
548b0d31159SSven Schnelle	lghi	%r13,.Lsie_done - .Lsie_gmap
549b0d31159SSven Schnelle	clgr	%r14,%r13
550b0d31159SSven Schnelle	jhe	.Lmcck_stack
551b0d31159SSven Schnelle	lghi	%r11,__LC_GPREGS_SAVE_AREA+64	# inside critical section, do cleanup
552b0d31159SSven Schnelle	brasl	%r14,.Lcleanup_sie
553b0d31159SSven Schnelle#endif
554b61b1595SSven Schnelle	j	.Lmcck_stack
555b0d31159SSven Schnelle.Lmcck_user:
556b0d31159SSven Schnelle	BPENTER __TI_flags(%r12),_TIF_ISOLATE_BP
557b61b1595SSven Schnelle.Lmcck_stack:
558b61b1595SSven Schnelle	lg	%r15,__LC_MCCK_STACK
5594bfc86ceSHeiko Carstens.Lmcck_skip:
560b61b1595SSven Schnelle	la	%r11,STACK_FRAME_OVERHEAD(%r15)
56126521412SSven Schnelle	stctg	%c1,%c1,__PT_CR1(%r11)
562b61b1595SSven Schnelle	lctlg	%c1,%c1,__LC_KERNEL_ASCE
563b61b1595SSven Schnelle	xc	__SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
5644bfc86ceSHeiko Carstens	lghi	%r14,__LC_GPREGS_SAVE_AREA+64
5654bfc86ceSHeiko Carstens	stmg	%r0,%r7,__PT_R0(%r11)
5667041d281SMartin Schwidefsky	# clear user controlled registers to prevent speculative use
5677041d281SMartin Schwidefsky	xgr	%r0,%r0
5687041d281SMartin Schwidefsky	xgr	%r1,%r1
5697041d281SMartin Schwidefsky	xgr	%r3,%r3
5707041d281SMartin Schwidefsky	xgr	%r4,%r4
5717041d281SMartin Schwidefsky	xgr	%r5,%r5
5727041d281SMartin Schwidefsky	xgr	%r6,%r6
5737041d281SMartin Schwidefsky	xgr	%r7,%r7
5747041d281SMartin Schwidefsky	xgr	%r10,%r10
5754bfc86ceSHeiko Carstens	mvc	__PT_R8(64,%r11),0(%r14)
5764bfc86ceSHeiko Carstens	stmg	%r8,%r9,__PT_PSW(%r11)
5774bfc86ceSHeiko Carstens	xc	__PT_FLAGS(8,%r11),__PT_FLAGS(%r11)
5784bfc86ceSHeiko Carstens	xc	__SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
5794bfc86ceSHeiko Carstens	lgr	%r2,%r11		# pass pointer to pt_regs
5804bfc86ceSHeiko Carstens	brasl	%r14,s390_do_machine_check
5810b0ed657SSven Schnelle	cghi	%r2,0
5820b0ed657SSven Schnelle	je	.Lmcck_return
5834bfc86ceSHeiko Carstens	lg	%r1,__LC_KERNEL_STACK	# switch to kernel stack
5844bfc86ceSHeiko Carstens	mvc	STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11)
5854bfc86ceSHeiko Carstens	xc	__SF_BACKCHAIN(8,%r1),__SF_BACKCHAIN(%r1)
5864bfc86ceSHeiko Carstens	la	%r11,STACK_FRAME_OVERHEAD(%r1)
5874bfc86ceSHeiko Carstens	lgr	%r15,%r1
5884bfc86ceSHeiko Carstens	brasl	%r14,s390_handle_mcck
5894bfc86ceSHeiko Carstens.Lmcck_return:
59087d59863SHeiko Carstens	lctlg	%c1,%c1,__PT_CR1(%r11)
5914bfc86ceSHeiko Carstens	lmg	%r0,%r10,__PT_R0(%r11)
5924bfc86ceSHeiko Carstens	mvc	__LC_RETURN_MCCK_PSW(16),__PT_PSW(%r11) # move return PSW
5934bfc86ceSHeiko Carstens	tm	__LC_RETURN_MCCK_PSW+1,0x01 # returning to user ?
5944bfc86ceSHeiko Carstens	jno	0f
5956b73044bSMartin Schwidefsky	BPEXIT	__TI_flags(%r12),_TIF_ISOLATE_BP
5964bfc86ceSHeiko Carstens	stpt	__LC_EXIT_TIMER
5974bfc86ceSHeiko Carstens0:	lmg	%r11,%r15,__PT_R11(%r11)
5980b38b5e1SSven Schnelle	b	__LC_RETURN_MCCK_LPSWE
5994bfc86ceSHeiko Carstens
6004bfc86ceSHeiko Carstens.Lmcck_panic:
601ce3dc447SMartin Schwidefsky	lg	%r15,__LC_NODAT_STACK
6024bfc86ceSHeiko Carstens	j	.Lmcck_skip
60326a374aeSMartin SchwidefskyENDPROC(mcck_int_handler)
6044bfc86ceSHeiko Carstens
6054bfc86ceSHeiko Carstens#
6064bfc86ceSHeiko Carstens# PSW restart interrupt handler
6074bfc86ceSHeiko Carstens#
6084bfc86ceSHeiko CarstensENTRY(restart_int_handler)
609e5b98199SMartin Schwidefsky	ALTERNATIVE "", ".insn s,0xb2800000,_LPP_OFFSET", 40
610e5b98199SMartin Schwidefsky	stg	%r15,__LC_SAVE_AREA_RESTART
6114bfc86ceSHeiko Carstens	lg	%r15,__LC_RESTART_STACK
612ce3dc447SMartin Schwidefsky	xc	STACK_FRAME_OVERHEAD(__PT_SIZE,%r15),STACK_FRAME_OVERHEAD(%r15)
613ce3dc447SMartin Schwidefsky	stmg	%r0,%r14,STACK_FRAME_OVERHEAD+__PT_R0(%r15)
614ce3dc447SMartin Schwidefsky	mvc	STACK_FRAME_OVERHEAD+__PT_R15(8,%r15),__LC_SAVE_AREA_RESTART
615ce3dc447SMartin Schwidefsky	mvc	STACK_FRAME_OVERHEAD+__PT_PSW(16,%r15),__LC_RST_OLD_PSW
6164bfc86ceSHeiko Carstens	xc	0(STACK_FRAME_OVERHEAD,%r15),0(%r15)
6174bfc86ceSHeiko Carstens	lg	%r1,__LC_RESTART_FN		# load fn, parm & source cpu
6184bfc86ceSHeiko Carstens	lg	%r2,__LC_RESTART_DATA
6194bfc86ceSHeiko Carstens	lg	%r3,__LC_RESTART_SOURCE
6204bfc86ceSHeiko Carstens	ltgr	%r3,%r3				# test source cpu address
6214bfc86ceSHeiko Carstens	jm	1f				# negative -> skip source stop
6224bfc86ceSHeiko Carstens0:	sigp	%r4,%r3,SIGP_SENSE		# sigp sense to source cpu
6234bfc86ceSHeiko Carstens	brc	10,0b				# wait for status stored
6244bfc86ceSHeiko Carstens1:	basr	%r14,%r1			# call function
6254bfc86ceSHeiko Carstens	stap	__SF_EMPTY(%r15)		# store cpu address
6264bfc86ceSHeiko Carstens	llgh	%r3,__SF_EMPTY(%r15)
6274bfc86ceSHeiko Carstens2:	sigp	%r4,%r3,SIGP_STOP		# sigp stop to current cpu
6284bfc86ceSHeiko Carstens	brc	2,2b
6294bfc86ceSHeiko Carstens3:	j	3b
63026a374aeSMartin SchwidefskyENDPROC(restart_int_handler)
6314bfc86ceSHeiko Carstens
6324bfc86ceSHeiko Carstens	.section .kprobes.text, "ax"
6334bfc86ceSHeiko Carstens
634ce3dc447SMartin Schwidefsky#if defined(CONFIG_CHECK_STACK) || defined(CONFIG_VMAP_STACK)
6354bfc86ceSHeiko Carstens/*
6364bfc86ceSHeiko Carstens * The synchronous or the asynchronous stack overflowed. We are dead.
6374bfc86ceSHeiko Carstens * No need to properly save the registers, we are going to panic anyway.
6384bfc86ceSHeiko Carstens * Setup a pt_regs so that show_trace can provide a good call trace.
6394bfc86ceSHeiko Carstens */
64026a374aeSMartin SchwidefskyENTRY(stack_overflow)
641ce3dc447SMartin Schwidefsky	lg	%r15,__LC_NODAT_STACK	# change to panic stack
6424bfc86ceSHeiko Carstens	la	%r11,STACK_FRAME_OVERHEAD(%r15)
6434bfc86ceSHeiko Carstens	stmg	%r0,%r7,__PT_R0(%r11)
6444bfc86ceSHeiko Carstens	stmg	%r8,%r9,__PT_PSW(%r11)
6454bfc86ceSHeiko Carstens	mvc	__PT_R8(64,%r11),0(%r14)
6464bfc86ceSHeiko Carstens	stg	%r10,__PT_ORIG_GPR2(%r11) # store last break to orig_gpr2
6474bfc86ceSHeiko Carstens	xc	__SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
6484bfc86ceSHeiko Carstens	lgr	%r2,%r11		# pass pointer to pt_regs
6494bfc86ceSHeiko Carstens	jg	kernel_stack_overflow
65026a374aeSMartin SchwidefskyENDPROC(stack_overflow)
6514bfc86ceSHeiko Carstens#endif
6524bfc86ceSHeiko Carstens
653d0fc4107SMartin Schwidefsky#if IS_ENABLED(CONFIG_KVM)
654d0fc4107SMartin Schwidefsky.Lcleanup_sie:
655c929500dSQingFeng Hao	cghi	%r11,__LC_SAVE_AREA_ASYNC	#Is this in normal interrupt?
656c929500dSQingFeng Hao	je	1f
6570b0ed657SSven Schnelle	larl	%r13,.Lsie_entry
6580b0ed657SSven Schnelle	slgr	%r9,%r13
6590b0ed657SSven Schnelle	larl	%r13,.Lsie_skip
6600b0ed657SSven Schnelle	clgr	%r9,%r13
661c929500dSQingFeng Hao	jh	1f
662c929500dSQingFeng Hao	oi	__LC_CPU_FLAGS+7, _CIF_MCCK_GUEST
66392fa7a13SMartin Schwidefsky1:	BPENTER	__SF_SIE_FLAGS(%r15),(_TIF_ISOLATE_BP|_TIF_ISOLATE_BP_GUEST)
66492fa7a13SMartin Schwidefsky	lg	%r9,__SF_SIE_CONTROL(%r15)	# get control block pointer
665e22cf8caSChristian Borntraeger	ni	__SIE_PROG0C+3(%r9),0xfe	# no longer in SIE
66687d59863SHeiko Carstens	lctlg	%c1,%c1,__LC_KERNEL_ASCE
667d0fc4107SMartin Schwidefsky	larl	%r9,sie_exit			# skip forward to sie_exit
668*33ea0487SSven Schnelle	BR_EX	%r14,%r13
6694bfc86ceSHeiko Carstens
6704bfc86ceSHeiko Carstens#endif
6714bfc86ceSHeiko Carstens	.section .rodata, "a"
672ff4a742dSGerald Schaefer#define SYSCALL(esame,emu)	.quad __s390x_ ## esame
6734bfc86ceSHeiko Carstens	.globl	sys_call_table
6744bfc86ceSHeiko Carstenssys_call_table:
6754381f9f1SHendrik Brueckner#include "asm/syscall_table.h"
6764bfc86ceSHeiko Carstens#undef SYSCALL
6774bfc86ceSHeiko Carstens
6784bfc86ceSHeiko Carstens#ifdef CONFIG_COMPAT
6794bfc86ceSHeiko Carstens
680ff4a742dSGerald Schaefer#define SYSCALL(esame,emu)	.quad __s390_ ## emu
6814bfc86ceSHeiko Carstens	.globl	sys_call_table_emu
6824bfc86ceSHeiko Carstenssys_call_table_emu:
6834381f9f1SHendrik Brueckner#include "asm/syscall_table.h"
6844bfc86ceSHeiko Carstens#undef SYSCALL
6854bfc86ceSHeiko Carstens#endif
686