xref: /openbmc/linux/arch/s390/kernel/entry.S (revision d35925b34996196d22a4357dc5212ab03af75151)
1b2441318SGreg Kroah-Hartman/* SPDX-License-Identifier: GPL-2.0 */
24bfc86ceSHeiko Carstens/*
34bfc86ceSHeiko Carstens *    S390 low-level entry points.
44bfc86ceSHeiko Carstens *
54bfc86ceSHeiko Carstens *    Copyright IBM Corp. 1999, 2012
64bfc86ceSHeiko Carstens *    Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com),
74bfc86ceSHeiko Carstens *		 Hartmut Penner (hp@de.ibm.com),
84bfc86ceSHeiko Carstens *		 Denis Joseph Barrow (djbarrow@de.ibm.com,barrow_dj@yahoo.com),
94bfc86ceSHeiko Carstens *		 Heiko Carstens <heiko.carstens@de.ibm.com>
104bfc86ceSHeiko Carstens */
114bfc86ceSHeiko Carstens
124bfc86ceSHeiko Carstens#include <linux/init.h>
134bfc86ceSHeiko Carstens#include <linux/linkage.h>
14b058661aSMartin Schwidefsky#include <asm/alternative-asm.h>
154bfc86ceSHeiko Carstens#include <asm/processor.h>
164bfc86ceSHeiko Carstens#include <asm/cache.h>
173037a52fSMartin Schwidefsky#include <asm/ctl_reg.h>
18dc24b7b4SHendrik Brueckner#include <asm/dwarf.h>
194bfc86ceSHeiko Carstens#include <asm/errno.h>
204bfc86ceSHeiko Carstens#include <asm/ptrace.h>
214bfc86ceSHeiko Carstens#include <asm/thread_info.h>
224bfc86ceSHeiko Carstens#include <asm/asm-offsets.h>
234bfc86ceSHeiko Carstens#include <asm/unistd.h>
244bfc86ceSHeiko Carstens#include <asm/page.h>
254bfc86ceSHeiko Carstens#include <asm/sigp.h>
264bfc86ceSHeiko Carstens#include <asm/irq.h>
279977e886SHendrik Brueckner#include <asm/vx-insn.h>
2883abeffbSHendrik Brueckner#include <asm/setup.h>
2983abeffbSHendrik Brueckner#include <asm/nmi.h>
30711f5df7SAl Viro#include <asm/export.h>
316dd85fbbSMartin Schwidefsky#include <asm/nospec-insn.h>
324bfc86ceSHeiko Carstens
334bfc86ceSHeiko Carstens__PT_R0      =	__PT_GPRS
344bfc86ceSHeiko Carstens__PT_R1      =	__PT_GPRS + 8
354bfc86ceSHeiko Carstens__PT_R2      =	__PT_GPRS + 16
364bfc86ceSHeiko Carstens__PT_R3      =	__PT_GPRS + 24
374bfc86ceSHeiko Carstens__PT_R4      =	__PT_GPRS + 32
384bfc86ceSHeiko Carstens__PT_R5      =	__PT_GPRS + 40
394bfc86ceSHeiko Carstens__PT_R6      =	__PT_GPRS + 48
404bfc86ceSHeiko Carstens__PT_R7      =	__PT_GPRS + 56
414bfc86ceSHeiko Carstens__PT_R8      =	__PT_GPRS + 64
424bfc86ceSHeiko Carstens__PT_R9      =	__PT_GPRS + 72
434bfc86ceSHeiko Carstens__PT_R10     =	__PT_GPRS + 80
444bfc86ceSHeiko Carstens__PT_R11     =	__PT_GPRS + 88
454bfc86ceSHeiko Carstens__PT_R12     =	__PT_GPRS + 96
464bfc86ceSHeiko Carstens__PT_R13     =	__PT_GPRS + 104
474bfc86ceSHeiko Carstens__PT_R14     =	__PT_GPRS + 112
484bfc86ceSHeiko Carstens__PT_R15     =	__PT_GPRS + 120
494bfc86ceSHeiko Carstens
503a890380SHeiko CarstensSTACK_SHIFT = PAGE_SHIFT + THREAD_SIZE_ORDER
514bfc86ceSHeiko CarstensSTACK_SIZE  = 1 << STACK_SHIFT
524bfc86ceSHeiko CarstensSTACK_INIT = STACK_SIZE - STACK_FRAME_OVERHEAD - __PT_SIZE
534bfc86ceSHeiko Carstens
54e5b98199SMartin Schwidefsky_LPP_OFFSET	= __LC_LPP
55e5b98199SMartin Schwidefsky
56ce3dc447SMartin Schwidefsky	.macro	CHECK_STACK savearea
574bfc86ceSHeiko Carstens#ifdef CONFIG_CHECK_STACK
58ce3dc447SMartin Schwidefsky	tml	%r15,STACK_SIZE - CONFIG_STACK_GUARD
594bfc86ceSHeiko Carstens	lghi	%r14,\savearea
604bfc86ceSHeiko Carstens	jz	stack_overflow
614bfc86ceSHeiko Carstens#endif
624bfc86ceSHeiko Carstens	.endm
634bfc86ceSHeiko Carstens
64ce3dc447SMartin Schwidefsky	.macro	CHECK_VMAP_STACK savearea,oklabel
65ce3dc447SMartin Schwidefsky#ifdef CONFIG_VMAP_STACK
66ce3dc447SMartin Schwidefsky	lgr	%r14,%r15
67ce3dc447SMartin Schwidefsky	nill	%r14,0x10000 - STACK_SIZE
68ce3dc447SMartin Schwidefsky	oill	%r14,STACK_INIT
69ce3dc447SMartin Schwidefsky	clg	%r14,__LC_KERNEL_STACK
70ce3dc447SMartin Schwidefsky	je	\oklabel
71ce3dc447SMartin Schwidefsky	clg	%r14,__LC_ASYNC_STACK
72ce3dc447SMartin Schwidefsky	je	\oklabel
73b61b1595SSven Schnelle	clg	%r14,__LC_MCCK_STACK
74b61b1595SSven Schnelle	je	\oklabel
75ce3dc447SMartin Schwidefsky	clg	%r14,__LC_NODAT_STACK
76ce3dc447SMartin Schwidefsky	je	\oklabel
77ce3dc447SMartin Schwidefsky	clg	%r14,__LC_RESTART_STACK
78ce3dc447SMartin Schwidefsky	je	\oklabel
79ce3dc447SMartin Schwidefsky	lghi	%r14,\savearea
80ce3dc447SMartin Schwidefsky	j	stack_overflow
81ce3dc447SMartin Schwidefsky#else
82ce3dc447SMartin Schwidefsky	j	\oklabel
83ce3dc447SMartin Schwidefsky#endif
84ce3dc447SMartin Schwidefsky	.endm
85ce3dc447SMartin Schwidefsky
864bfc86ceSHeiko Carstens	.macro STCK savearea
8778f65709SHeiko Carstens	ALTERNATIVE ".insn	s,0xb2050000,\savearea", \
8878f65709SHeiko Carstens		    ".insn	s,0xb27c0000,\savearea", 25
894bfc86ceSHeiko Carstens	.endm
904bfc86ceSHeiko Carstens
9183abeffbSHendrik Brueckner	/*
9283abeffbSHendrik Brueckner	 * The TSTMSK macro generates a test-under-mask instruction by
9383abeffbSHendrik Brueckner	 * calculating the memory offset for the specified mask value.
9483abeffbSHendrik Brueckner	 * Mask value can be any constant.  The macro shifts the mask
9583abeffbSHendrik Brueckner	 * value to calculate the memory offset for the test-under-mask
9683abeffbSHendrik Brueckner	 * instruction.
9783abeffbSHendrik Brueckner	 */
9883abeffbSHendrik Brueckner	.macro TSTMSK addr, mask, size=8, bytepos=0
9983abeffbSHendrik Brueckner		.if (\bytepos < \size) && (\mask >> 8)
10083abeffbSHendrik Brueckner			.if (\mask & 0xff)
10183abeffbSHendrik Brueckner				.error "Mask exceeds byte boundary"
10283abeffbSHendrik Brueckner			.endif
10383abeffbSHendrik Brueckner			TSTMSK \addr, "(\mask >> 8)", \size, "(\bytepos + 1)"
10483abeffbSHendrik Brueckner			.exitm
10583abeffbSHendrik Brueckner		.endif
10683abeffbSHendrik Brueckner		.ifeq \mask
10783abeffbSHendrik Brueckner			.error "Mask must not be zero"
10883abeffbSHendrik Brueckner		.endif
10983abeffbSHendrik Brueckner		off = \size - \bytepos - 1
11083abeffbSHendrik Brueckner		tm	off+\addr, \mask
11183abeffbSHendrik Brueckner	.endm
11283abeffbSHendrik Brueckner
113d768bd89SMartin Schwidefsky	.macro BPOFF
114b058661aSMartin Schwidefsky	ALTERNATIVE "", ".long 0xb2e8c000", 82
115d768bd89SMartin Schwidefsky	.endm
116d768bd89SMartin Schwidefsky
117d768bd89SMartin Schwidefsky	.macro BPON
118b058661aSMartin Schwidefsky	ALTERNATIVE "", ".long 0xb2e8d000", 82
119d768bd89SMartin Schwidefsky	.endm
120d768bd89SMartin Schwidefsky
1216b73044bSMartin Schwidefsky	.macro BPENTER tif_ptr,tif_mask
122b058661aSMartin Schwidefsky	ALTERNATIVE "TSTMSK \tif_ptr,\tif_mask; jz .+8; .long 0xb2e8d000", \
123b058661aSMartin Schwidefsky		    "", 82
1246b73044bSMartin Schwidefsky	.endm
1256b73044bSMartin Schwidefsky
1266b73044bSMartin Schwidefsky	.macro BPEXIT tif_ptr,tif_mask
1276b73044bSMartin Schwidefsky	TSTMSK	\tif_ptr,\tif_mask
128b058661aSMartin Schwidefsky	ALTERNATIVE "jz .+8;  .long 0xb2e8c000", \
129b058661aSMartin Schwidefsky		    "jnz .+8; .long 0xb2e8d000", 82
1306b73044bSMartin Schwidefsky	.endm
1316b73044bSMartin Schwidefsky
132*d35925b3SAlexander Gordeev	/*
133*d35925b3SAlexander Gordeev	 * The CHKSTG macro jumps to the provided label in case the
134*d35925b3SAlexander Gordeev	 * machine check interruption code reports one of unrecoverable
135*d35925b3SAlexander Gordeev	 * storage errors:
136*d35925b3SAlexander Gordeev	 * - Storage error uncorrected
137*d35925b3SAlexander Gordeev	 * - Storage key error uncorrected
138*d35925b3SAlexander Gordeev	 * - Storage degradation with Failing-storage-address validity
139*d35925b3SAlexander Gordeev	 */
140*d35925b3SAlexander Gordeev	.macro CHKSTG errlabel
141*d35925b3SAlexander Gordeev	TSTMSK	__LC_MCCK_CODE,(MCCK_CODE_STG_ERROR|MCCK_CODE_STG_KEY_ERROR)
142*d35925b3SAlexander Gordeev	jnz	\errlabel
143*d35925b3SAlexander Gordeev	TSTMSK	__LC_MCCK_CODE,MCCK_CODE_STG_DEGRAD
144*d35925b3SAlexander Gordeev	jz	oklabel\@
145*d35925b3SAlexander Gordeev	TSTMSK	__LC_MCCK_CODE,MCCK_CODE_STG_FAIL_ADDR
146*d35925b3SAlexander Gordeev	jnz	\errlabel
147*d35925b3SAlexander Gordeevoklabel\@:
148*d35925b3SAlexander Gordeev	.endm
149*d35925b3SAlexander Gordeev
150b5415c8fSAlexander Gordeev#if IS_ENABLED(CONFIG_KVM)
151b5415c8fSAlexander Gordeev	/*
152b5415c8fSAlexander Gordeev	 * The OUTSIDE macro jumps to the provided label in case the value
153b5415c8fSAlexander Gordeev	 * in the provided register is outside of the provided range. The
154b5415c8fSAlexander Gordeev	 * macro is useful for checking whether a PSW stored in a register
155b5415c8fSAlexander Gordeev	 * pair points inside or outside of a block of instructions.
156b5415c8fSAlexander Gordeev	 * @reg: register to check
157b5415c8fSAlexander Gordeev	 * @start: start of the range
158b5415c8fSAlexander Gordeev	 * @end: end of the range
159b5415c8fSAlexander Gordeev	 * @outside_label: jump here if @reg is outside of [@start..@end)
160b5415c8fSAlexander Gordeev	 */
161b5415c8fSAlexander Gordeev	.macro OUTSIDE reg,start,end,outside_label
162b5415c8fSAlexander Gordeev	lgr	%r14,\reg
163b5415c8fSAlexander Gordeev	larl	%r13,\start
164b5415c8fSAlexander Gordeev	slgr	%r14,%r13
165b5415c8fSAlexander Gordeev	lghi	%r13,\end - \start
166b5415c8fSAlexander Gordeev	clgr	%r14,%r13
167b5415c8fSAlexander Gordeev	jhe	\outside_label
168b5415c8fSAlexander Gordeev	.endm
169fbbdfca5SAlexander Gordeev
170fbbdfca5SAlexander Gordeev	.macro SIEEXIT
171fbbdfca5SAlexander Gordeev	lg	%r9,__SF_SIE_CONTROL(%r15)	# get control block pointer
172fbbdfca5SAlexander Gordeev	ni	__SIE_PROG0C+3(%r9),0xfe	# no longer in SIE
173fbbdfca5SAlexander Gordeev	lctlg	%c1,%c1,__LC_KERNEL_ASCE	# load primary asce
174fbbdfca5SAlexander Gordeev	larl	%r9,sie_exit			# skip forward to sie_exit
175fbbdfca5SAlexander Gordeev	.endm
176b5415c8fSAlexander Gordeev#endif
177b5415c8fSAlexander Gordeev
1786dd85fbbSMartin Schwidefsky	GEN_BR_THUNK %r14
17933ea0487SSven Schnelle	GEN_BR_THUNK %r14,%r13
180f19fbd5eSMartin Schwidefsky
1814bfc86ceSHeiko Carstens	.section .kprobes.text, "ax"
18246210c44SHeiko Carstens.Ldummy:
18346210c44SHeiko Carstens	/*
18456e62a73SSven Schnelle	 * This nop exists only in order to avoid that __bpon starts at
18546210c44SHeiko Carstens	 * the beginning of the kprobes text section. In that case we would
18646210c44SHeiko Carstens	 * have several symbols at the same address. E.g. objdump would take
18746210c44SHeiko Carstens	 * an arbitrary symbol name when disassembling this code.
18856e62a73SSven Schnelle	 * With the added nop in between the __bpon symbol is unique
18946210c44SHeiko Carstens	 * again.
19046210c44SHeiko Carstens	 */
19146210c44SHeiko Carstens	nop	0
1924bfc86ceSHeiko Carstens
193d768bd89SMartin SchwidefskyENTRY(__bpon)
194d768bd89SMartin Schwidefsky	.globl __bpon
195d768bd89SMartin Schwidefsky	BPON
1966dd85fbbSMartin Schwidefsky	BR_EX	%r14
19726a374aeSMartin SchwidefskyENDPROC(__bpon)
198d768bd89SMartin Schwidefsky
1994bfc86ceSHeiko Carstens/*
2004bfc86ceSHeiko Carstens * Scheduler resume function, called by switch_to
2014bfc86ceSHeiko Carstens *  gpr2 = (task_struct *) prev
2024bfc86ceSHeiko Carstens *  gpr3 = (task_struct *) next
2034bfc86ceSHeiko Carstens * Returns:
2044bfc86ceSHeiko Carstens *  gpr2 = prev
2054bfc86ceSHeiko Carstens */
2064bfc86ceSHeiko CarstensENTRY(__switch_to)
2074bfc86ceSHeiko Carstens	stmg	%r6,%r15,__SF_GPRS(%r15)	# store gprs of prev task
2083241d3ebSHeiko Carstens	lghi	%r4,__TASK_stack
2093241d3ebSHeiko Carstens	lghi	%r1,__TASK_thread
2109fed920eSVasily Gorbik	llill	%r5,STACK_INIT
2113241d3ebSHeiko Carstens	stg	%r15,__THREAD_ksp(%r1,%r2)	# store kernel stack of prev
2129fed920eSVasily Gorbik	lg	%r15,0(%r4,%r3)			# start of kernel stack of next
2139fed920eSVasily Gorbik	agr	%r15,%r5			# end of kernel stack of next
2144bfc86ceSHeiko Carstens	stg	%r3,__LC_CURRENT		# store task struct of next
2154bfc86ceSHeiko Carstens	stg	%r15,__LC_KERNEL_STACK		# store end of kernel stack
2163241d3ebSHeiko Carstens	lg	%r15,__THREAD_ksp(%r1,%r3)	# load kernel stack of next
2173241d3ebSHeiko Carstens	aghi	%r3,__TASK_pid
2183241d3ebSHeiko Carstens	mvc	__LC_CURRENT_PID(4,%r0),0(%r3)	# store pid of next
2194bfc86ceSHeiko Carstens	lmg	%r6,%r15,__SF_GPRS(%r15)	# load gprs of next task
220e5b98199SMartin Schwidefsky	ALTERNATIVE "", ".insn s,0xb2800000,_LPP_OFFSET", 40
2216dd85fbbSMartin Schwidefsky	BR_EX	%r14
22226a374aeSMartin SchwidefskyENDPROC(__switch_to)
2234bfc86ceSHeiko Carstens
224d0fc4107SMartin Schwidefsky#if IS_ENABLED(CONFIG_KVM)
225d0fc4107SMartin Schwidefsky/*
226d0fc4107SMartin Schwidefsky * sie64a calling convention:
227d0fc4107SMartin Schwidefsky * %r2 pointer to sie control block
228d0fc4107SMartin Schwidefsky * %r3 guest register save area
229d0fc4107SMartin Schwidefsky */
230d0fc4107SMartin SchwidefskyENTRY(sie64a)
231d0fc4107SMartin Schwidefsky	stmg	%r6,%r14,__SF_GPRS(%r15)	# save kernel registers
2326b73044bSMartin Schwidefsky	lg	%r12,__LC_CURRENT
23392fa7a13SMartin Schwidefsky	stg	%r2,__SF_SIE_CONTROL(%r15)	# save control block pointer
23492fa7a13SMartin Schwidefsky	stg	%r3,__SF_SIE_SAVEAREA(%r15)	# save guest register save area
23592fa7a13SMartin Schwidefsky	xc	__SF_SIE_REASON(8,%r15),__SF_SIE_REASON(%r15) # reason code = 0
23692fa7a13SMartin Schwidefsky	mvc	__SF_SIE_FLAGS(8,%r15),__TI_flags(%r12) # copy thread flags
237d0fc4107SMartin Schwidefsky	lmg	%r0,%r13,0(%r3)			# load guest gprs 0-13
238d0fc4107SMartin Schwidefsky	lg	%r14,__LC_GMAP			# get gmap pointer
239d0fc4107SMartin Schwidefsky	ltgr	%r14,%r14
240d0fc4107SMartin Schwidefsky	jz	.Lsie_gmap
241d0fc4107SMartin Schwidefsky	lctlg	%c1,%c1,__GMAP_ASCE(%r14)	# load primary asce
242d0fc4107SMartin Schwidefsky.Lsie_gmap:
24392fa7a13SMartin Schwidefsky	lg	%r14,__SF_SIE_CONTROL(%r15)	# get control block pointer
244d0fc4107SMartin Schwidefsky	oi	__SIE_PROG0C+3(%r14),1		# we are going into SIE now
245d0fc4107SMartin Schwidefsky	tm	__SIE_PROG20+3(%r14),3		# last exit...
246d0fc4107SMartin Schwidefsky	jnz	.Lsie_skip
24783abeffbSHendrik Brueckner	TSTMSK	__LC_CPU_FLAGS,_CIF_FPU
248d0fc4107SMartin Schwidefsky	jo	.Lsie_skip			# exit if fp/vx regs changed
24992fa7a13SMartin Schwidefsky	BPEXIT	__SF_SIE_FLAGS(%r15),(_TIF_ISOLATE_BP|_TIF_ISOLATE_BP_GUEST)
250c929500dSQingFeng Hao.Lsie_entry:
251d0fc4107SMartin Schwidefsky	sie	0(%r14)
252d768bd89SMartin Schwidefsky	BPOFF
25392fa7a13SMartin Schwidefsky	BPENTER	__SF_SIE_FLAGS(%r15),(_TIF_ISOLATE_BP|_TIF_ISOLATE_BP_GUEST)
254d0fc4107SMartin Schwidefsky.Lsie_skip:
255d0fc4107SMartin Schwidefsky	ni	__SIE_PROG0C+3(%r14),0xfe	# no longer in SIE
25687d59863SHeiko Carstens	lctlg	%c1,%c1,__LC_KERNEL_ASCE	# load primary asce
257d0fc4107SMartin Schwidefsky.Lsie_done:
258d0fc4107SMartin Schwidefsky# some program checks are suppressing. C code (e.g. do_protection_exception)
259c0e7bb38SChristian Borntraeger# will rewind the PSW by the ILC, which is often 4 bytes in case of SIE. There
260c0e7bb38SChristian Borntraeger# are some corner cases (e.g. runtime instrumentation) where ILC is unpredictable.
261c0e7bb38SChristian Borntraeger# Other instructions between sie64a and .Lsie_done should not cause program
262c0e7bb38SChristian Borntraeger# interrupts. So lets use 3 nops as a landing pad for all possible rewinds.
263c0e7bb38SChristian Borntraeger.Lrewind_pad6:
264c0e7bb38SChristian Borntraeger	nopr	7
265c0e7bb38SChristian Borntraeger.Lrewind_pad4:
266c0e7bb38SChristian Borntraeger	nopr	7
267c0e7bb38SChristian Borntraeger.Lrewind_pad2:
268c0e7bb38SChristian Borntraeger	nopr	7
269d0fc4107SMartin Schwidefsky	.globl sie_exit
270d0fc4107SMartin Schwidefskysie_exit:
27192fa7a13SMartin Schwidefsky	lg	%r14,__SF_SIE_SAVEAREA(%r15)	# load guest register save area
272d0fc4107SMartin Schwidefsky	stmg	%r0,%r13,0(%r14)		# save guest gprs 0-13
2737041d281SMartin Schwidefsky	xgr	%r0,%r0				# clear guest registers to
2747041d281SMartin Schwidefsky	xgr	%r1,%r1				# prevent speculative use
2757041d281SMartin Schwidefsky	xgr	%r3,%r3
2767041d281SMartin Schwidefsky	xgr	%r4,%r4
2777041d281SMartin Schwidefsky	xgr	%r5,%r5
278d0fc4107SMartin Schwidefsky	lmg	%r6,%r14,__SF_GPRS(%r15)	# restore kernel registers
27992fa7a13SMartin Schwidefsky	lg	%r2,__SF_SIE_REASON(%r15)	# return exit reason code
2806dd85fbbSMartin Schwidefsky	BR_EX	%r14
281d0fc4107SMartin Schwidefsky.Lsie_fault:
282d0fc4107SMartin Schwidefsky	lghi	%r14,-EFAULT
28392fa7a13SMartin Schwidefsky	stg	%r14,__SF_SIE_REASON(%r15)	# set exit reason code
284d0fc4107SMartin Schwidefsky	j	sie_exit
285d0fc4107SMartin Schwidefsky
286c0e7bb38SChristian Borntraeger	EX_TABLE(.Lrewind_pad6,.Lsie_fault)
287c0e7bb38SChristian Borntraeger	EX_TABLE(.Lrewind_pad4,.Lsie_fault)
288c0e7bb38SChristian Borntraeger	EX_TABLE(.Lrewind_pad2,.Lsie_fault)
289d0fc4107SMartin Schwidefsky	EX_TABLE(sie_exit,.Lsie_fault)
29026a374aeSMartin SchwidefskyENDPROC(sie64a)
291711f5df7SAl ViroEXPORT_SYMBOL(sie64a)
292711f5df7SAl ViroEXPORT_SYMBOL(sie_exit)
293d0fc4107SMartin Schwidefsky#endif
294d0fc4107SMartin Schwidefsky
2954bfc86ceSHeiko Carstens/*
2964bfc86ceSHeiko Carstens * SVC interrupt handler routine. System calls are synchronous events and
2977b7735c5SChristian Borntraeger * are entered with interrupts disabled.
2984bfc86ceSHeiko Carstens */
2994bfc86ceSHeiko Carstens
3004bfc86ceSHeiko CarstensENTRY(system_call)
30156e62a73SSven Schnelle	stpt	__LC_SYS_ENTER_TIMER
3024bfc86ceSHeiko Carstens	stmg	%r8,%r15,__LC_SAVE_AREA_SYNC
303d768bd89SMartin Schwidefsky	BPOFF
30456e62a73SSven Schnelle	lghi	%r14,0
3054bfc86ceSHeiko Carstens.Lsysc_per:
30687d59863SHeiko Carstens	lctlg	%c1,%c1,__LC_KERNEL_ASCE
30756e62a73SSven Schnelle	lg	%r12,__LC_CURRENT
3084bfc86ceSHeiko Carstens	lg	%r15,__LC_KERNEL_STACK
3099365965dSHeiko Carstens	xc	__SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
31056e62a73SSven Schnelle	stmg	%r0,%r7,STACK_FRAME_OVERHEAD+__PT_R0(%r15)
31156e62a73SSven Schnelle	BPENTER __TI_flags(%r12),_TIF_ISOLATE_BP
312d3f46896SChristian Borntraeger	# clear user controlled register to prevent speculative use
313d3f46896SChristian Borntraeger	xgr	%r0,%r0
31456e62a73SSven Schnelle	xgr	%r1,%r1
31556e62a73SSven Schnelle	xgr	%r4,%r4
31656e62a73SSven Schnelle	xgr	%r5,%r5
31756e62a73SSven Schnelle	xgr	%r6,%r6
31856e62a73SSven Schnelle	xgr	%r7,%r7
31956e62a73SSven Schnelle	xgr	%r8,%r8
32056e62a73SSven Schnelle	xgr	%r9,%r9
32156e62a73SSven Schnelle	xgr	%r10,%r10
32256e62a73SSven Schnelle	xgr	%r11,%r11
32356e62a73SSven Schnelle	la	%r2,STACK_FRAME_OVERHEAD(%r15)	# pointer to pt_regs
324af9ad822SSven Schnelle	mvc	__PT_R8(64,%r2),__LC_SAVE_AREA_SYNC
32556e62a73SSven Schnelle	lgr	%r3,%r14
32656e62a73SSven Schnelle	brasl	%r14,__do_syscall
32787d59863SHeiko Carstens	lctlg	%c1,%c1,__LC_USER_ASCE
32856e62a73SSven Schnelle	mvc	__LC_RETURN_PSW(16),STACK_FRAME_OVERHEAD+__PT_PSW(%r15)
3296b73044bSMartin Schwidefsky	BPEXIT __TI_flags(%r12),_TIF_ISOLATE_BP
33056e62a73SSven Schnelle	lmg	%r0,%r15,STACK_FRAME_OVERHEAD+__PT_R0(%r15)
3314bfc86ceSHeiko Carstens	stpt	__LC_EXIT_TIMER
3320b0ed657SSven Schnelle	b	__LC_RETURN_LPSWE
33326a374aeSMartin SchwidefskyENDPROC(system_call)
3344bfc86ceSHeiko Carstens
3354bfc86ceSHeiko Carstens#
3364bfc86ceSHeiko Carstens# a new process exits the kernel with ret_from_fork
3374bfc86ceSHeiko Carstens#
3384bfc86ceSHeiko CarstensENTRY(ret_from_fork)
33956e62a73SSven Schnelle	lgr	%r3,%r11
34056e62a73SSven Schnelle	brasl	%r14,__ret_from_fork
34156e62a73SSven Schnelle	lctlg	%c1,%c1,__LC_USER_ASCE
34256e62a73SSven Schnelle	mvc	__LC_RETURN_PSW(16),STACK_FRAME_OVERHEAD+__PT_PSW(%r15)
34356e62a73SSven Schnelle	BPEXIT __TI_flags(%r12),_TIF_ISOLATE_BP
34456e62a73SSven Schnelle	lmg	%r0,%r15,STACK_FRAME_OVERHEAD+__PT_R0(%r15)
34556e62a73SSven Schnelle	stpt	__LC_EXIT_TIMER
34656e62a73SSven Schnelle	b	__LC_RETURN_LPSWE
34726a374aeSMartin SchwidefskyENDPROC(ret_from_fork)
34826a374aeSMartin Schwidefsky
3494bfc86ceSHeiko Carstens/*
3504bfc86ceSHeiko Carstens * Program check handler routine
3514bfc86ceSHeiko Carstens */
3524bfc86ceSHeiko Carstens
3534bfc86ceSHeiko CarstensENTRY(pgm_check_handler)
35456e62a73SSven Schnelle	stpt	__LC_SYS_ENTER_TIMER
355d768bd89SMartin Schwidefsky	BPOFF
3564bfc86ceSHeiko Carstens	stmg	%r8,%r15,__LC_SAVE_AREA_SYNC
35756e62a73SSven Schnelle	lg	%r12,__LC_CURRENT
35856e62a73SSven Schnelle	lghi	%r10,0
3594bfc86ceSHeiko Carstens	lmg	%r8,%r9,__LC_PGM_OLD_PSW
36087d59863SHeiko Carstens	tmhh	%r8,0x0001		# coming from user space?
36187d59863SHeiko Carstens	jno	.Lpgm_skip_asce
36287d59863SHeiko Carstens	lctlg	%c1,%c1,__LC_KERNEL_ASCE
36356e62a73SSven Schnelle	j	3f			# -> fault in user space
36487d59863SHeiko Carstens.Lpgm_skip_asce:
365d0fc4107SMartin Schwidefsky#if IS_ENABLED(CONFIG_KVM)
3660a5e2ec2SMartin Schwidefsky	# cleanup critical section for program checks in sie64a
367b5415c8fSAlexander Gordeev	OUTSIDE	%r9,.Lsie_gmap,.Lsie_done,1f
368fbbdfca5SAlexander Gordeev	SIEEXIT
36956e62a73SSven Schnelle	lghi	%r10,_PIF_GUEST_FAULT
370d0fc4107SMartin Schwidefsky#endif
3710b38b5e1SSven Schnelle1:	tmhh	%r8,0x4000		# PER bit set in old PSW ?
3720b38b5e1SSven Schnelle	jnz	2f			# -> enabled, can't be a double fault
3734bfc86ceSHeiko Carstens	tm	__LC_PGM_ILC+3,0x80	# check for per exception
3744bfc86ceSHeiko Carstens	jnz	.Lpgm_svcper		# -> single stepped svc
3750b38b5e1SSven Schnelle2:	CHECK_STACK __LC_SAVE_AREA_SYNC
3764bfc86ceSHeiko Carstens	aghi	%r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
37756e62a73SSven Schnelle	# CHECK_VMAP_STACK branches to stack_overflow or 4f
37856e62a73SSven Schnelle	CHECK_VMAP_STACK __LC_SAVE_AREA_SYNC,4f
37956e62a73SSven Schnelle3:	BPENTER __TI_flags(%r12),_TIF_ISOLATE_BP
3804bfc86ceSHeiko Carstens	lg	%r15,__LC_KERNEL_STACK
38156e62a73SSven Schnelle4:	la	%r11,STACK_FRAME_OVERHEAD(%r15)
38256e62a73SSven Schnelle	stg	%r10,__PT_FLAGS(%r11)
38356e62a73SSven Schnelle	xc	__SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
3844bfc86ceSHeiko Carstens	stmg	%r0,%r7,__PT_R0(%r11)
38556e62a73SSven Schnelle	mvc	__PT_R8(64,%r11),__LC_SAVE_AREA_SYNC
38656e62a73SSven Schnelle	stmg	%r8,%r9,__PT_PSW(%r11)
38756e62a73SSven Schnelle
3887041d281SMartin Schwidefsky	# clear user controlled registers to prevent speculative use
3897041d281SMartin Schwidefsky	xgr	%r0,%r0
3907041d281SMartin Schwidefsky	xgr	%r1,%r1
3917041d281SMartin Schwidefsky	xgr	%r3,%r3
3927041d281SMartin Schwidefsky	xgr	%r4,%r4
3937041d281SMartin Schwidefsky	xgr	%r5,%r5
3947041d281SMartin Schwidefsky	xgr	%r6,%r6
3957041d281SMartin Schwidefsky	xgr	%r7,%r7
39656e62a73SSven Schnelle	lgr	%r2,%r11
39756e62a73SSven Schnelle	brasl	%r14,__do_pgm_check
39856e62a73SSven Schnelle	tmhh	%r8,0x0001		# returning to user space?
39956e62a73SSven Schnelle	jno	.Lpgm_exit_kernel
40056e62a73SSven Schnelle	lctlg	%c1,%c1,__LC_USER_ASCE
40156e62a73SSven Schnelle	BPEXIT __TI_flags(%r12),_TIF_ISOLATE_BP
4020cd9b723SHeiko Carstens	stpt	__LC_EXIT_TIMER
40356e62a73SSven Schnelle.Lpgm_exit_kernel:
40456e62a73SSven Schnelle	mvc	__LC_RETURN_PSW(16),STACK_FRAME_OVERHEAD+__PT_PSW(%r15)
40556e62a73SSven Schnelle	lmg	%r0,%r15,STACK_FRAME_OVERHEAD+__PT_R0(%r15)
4060cd9b723SHeiko Carstens	b	__LC_RETURN_LPSWE
4074bfc86ceSHeiko Carstens
4084bfc86ceSHeiko Carstens#
4094bfc86ceSHeiko Carstens# single stepped system call
4104bfc86ceSHeiko Carstens#
4114bfc86ceSHeiko Carstens.Lpgm_svcper:
4124bfc86ceSHeiko Carstens	mvc	__LC_RETURN_PSW(8),__LC_SVC_NEW_PSW
4134bfc86ceSHeiko Carstens	larl	%r14,.Lsysc_per
4144bfc86ceSHeiko Carstens	stg	%r14,__LC_RETURN_PSW+8
41556e62a73SSven Schnelle	lghi	%r14,1
4160b0ed657SSven Schnelle	lpswe	__LC_RETURN_PSW		# branch to .Lsysc_per
41726a374aeSMartin SchwidefskyENDPROC(pgm_check_handler)
4184bfc86ceSHeiko Carstens
4194bfc86ceSHeiko Carstens/*
42056e62a73SSven Schnelle * Interrupt handler macro used for external and IO interrupts.
4214bfc86ceSHeiko Carstens */
42256e62a73SSven Schnelle.macro INT_HANDLER name,lc_old_psw,handler
42356e62a73SSven SchnelleENTRY(\name)
4244bfc86ceSHeiko Carstens	STCK	__LC_INT_CLOCK
42556e62a73SSven Schnelle	stpt	__LC_SYS_ENTER_TIMER
426d768bd89SMartin Schwidefsky	BPOFF
4274bfc86ceSHeiko Carstens	stmg	%r8,%r15,__LC_SAVE_AREA_ASYNC
428d5c352cdSHeiko Carstens	lg	%r12,__LC_CURRENT
42956e62a73SSven Schnelle	lmg	%r8,%r9,\lc_old_psw
430b0d31159SSven Schnelle	tmhh	%r8,0x0001			# interrupting from user ?
431b0d31159SSven Schnelle	jnz	1f
432b0d31159SSven Schnelle#if IS_ENABLED(CONFIG_KVM)
433b5415c8fSAlexander Gordeev	OUTSIDE	%r9,.Lsie_gmap,.Lsie_done,0f
434fbbdfca5SAlexander Gordeev	BPENTER	__SF_SIE_FLAGS(%r15),(_TIF_ISOLATE_BP|_TIF_ISOLATE_BP_GUEST)
435fbbdfca5SAlexander Gordeev	SIEEXIT
436b0d31159SSven Schnelle#endif
437b0d31159SSven Schnelle0:	CHECK_STACK __LC_SAVE_AREA_ASYNC
438b0d31159SSven Schnelle	aghi	%r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
439b0d31159SSven Schnelle	j	2f
440b0d31159SSven Schnelle1:	BPENTER __TI_flags(%r12),_TIF_ISOLATE_BP
441b0d31159SSven Schnelle	lctlg	%c1,%c1,__LC_KERNEL_ASCE
442b0d31159SSven Schnelle	lg	%r15,__LC_KERNEL_STACK
443b74e409eSVasily Gorbik2:	xc	__SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
444b74e409eSVasily Gorbik	la	%r11,STACK_FRAME_OVERHEAD(%r15)
4454bfc86ceSHeiko Carstens	stmg	%r0,%r7,__PT_R0(%r11)
4467041d281SMartin Schwidefsky	# clear user controlled registers to prevent speculative use
4477041d281SMartin Schwidefsky	xgr	%r0,%r0
4487041d281SMartin Schwidefsky	xgr	%r1,%r1
4497041d281SMartin Schwidefsky	xgr	%r3,%r3
4507041d281SMartin Schwidefsky	xgr	%r4,%r4
4517041d281SMartin Schwidefsky	xgr	%r5,%r5
4527041d281SMartin Schwidefsky	xgr	%r6,%r6
4537041d281SMartin Schwidefsky	xgr	%r7,%r7
4547041d281SMartin Schwidefsky	xgr	%r10,%r10
455ca1f4d70SSven Schnelle	xc	__PT_FLAGS(8,%r11),__PT_FLAGS(%r11)
4564bfc86ceSHeiko Carstens	mvc	__PT_R8(64,%r11),__LC_SAVE_AREA_ASYNC
4574bfc86ceSHeiko Carstens	stmg	%r8,%r9,__PT_PSW(%r11)
45856e62a73SSven Schnelle	tm	%r8,0x0001		# coming from user space?
45956e62a73SSven Schnelle	jno	1f
46087d59863SHeiko Carstens	lctlg	%c1,%c1,__LC_KERNEL_ASCE
46156e62a73SSven Schnelle1:	lgr	%r2,%r11		# pass pointer to pt_regs
46256e62a73SSven Schnelle	brasl	%r14,\handler
4634bfc86ceSHeiko Carstens	mvc	__LC_RETURN_PSW(16),__PT_PSW(%r11)
46456e62a73SSven Schnelle	tmhh	%r8,0x0001		# returning to user ?
46556e62a73SSven Schnelle	jno	2f
46687d59863SHeiko Carstens	lctlg	%c1,%c1,__LC_USER_ASCE
4676b73044bSMartin Schwidefsky	BPEXIT	__TI_flags(%r12),_TIF_ISOLATE_BP
4684bfc86ceSHeiko Carstens	stpt	__LC_EXIT_TIMER
46956e62a73SSven Schnelle2:	lmg	%r0,%r15,__PT_R0(%r11)
4700b0ed657SSven Schnelle	b	__LC_RETURN_LPSWE
47156e62a73SSven SchnelleENDPROC(\name)
47256e62a73SSven Schnelle.endm
4734bfc86ceSHeiko Carstens
47456e62a73SSven SchnelleINT_HANDLER ext_int_handler,__LC_EXT_OLD_PSW,do_ext_irq
47556e62a73SSven SchnelleINT_HANDLER io_int_handler,__LC_IO_OLD_PSW,do_io_irq
4764bfc86ceSHeiko Carstens
4774bfc86ceSHeiko Carstens/*
4780b0ed657SSven Schnelle * Load idle PSW.
4794bfc86ceSHeiko Carstens */
4804bfc86ceSHeiko CarstensENTRY(psw_idle)
481a994eddbSVasily Gorbik	stg	%r14,(__SF_GPRS+8*8)(%r15)
4824bfc86ceSHeiko Carstens	stg	%r3,__SF_EMPTY(%r15)
48356e62a73SSven Schnelle	larl	%r1,psw_idle_exit
4844bfc86ceSHeiko Carstens	stg	%r1,__SF_EMPTY+8(%r15)
48572d38b19SMartin Schwidefsky	larl	%r1,smp_cpu_mtid
48672d38b19SMartin Schwidefsky	llgf	%r1,0(%r1)
48772d38b19SMartin Schwidefsky	ltgr	%r1,%r1
48872d38b19SMartin Schwidefsky	jz	.Lpsw_idle_stcctm
48956e62a73SSven Schnelle	.insn	rsy,0xeb0000000017,%r1,5,__MT_CYCLES_ENTER(%r2)
49072d38b19SMartin Schwidefsky.Lpsw_idle_stcctm:
491419123f9SMartin Schwidefsky	oi	__LC_CPU_FLAGS+7,_CIF_ENABLED_WAIT
492d768bd89SMartin Schwidefsky	BPON
4934bfc86ceSHeiko Carstens	STCK	__CLOCK_IDLE_ENTER(%r2)
4944bfc86ceSHeiko Carstens	stpt	__TIMER_IDLE_ENTER(%r2)
4954bfc86ceSHeiko Carstens	lpswe	__SF_EMPTY(%r15)
49656e62a73SSven Schnelle.globl psw_idle_exit
49756e62a73SSven Schnellepsw_idle_exit:
4986dd85fbbSMartin Schwidefsky	BR_EX	%r14
49926a374aeSMartin SchwidefskyENDPROC(psw_idle)
5004bfc86ceSHeiko Carstens
501b5510d9bSHendrik Brueckner/*
5024bfc86ceSHeiko Carstens * Machine check handler routines
5034bfc86ceSHeiko Carstens */
5044bfc86ceSHeiko CarstensENTRY(mcck_int_handler)
5054bfc86ceSHeiko Carstens	STCK	__LC_MCCK_CLOCK
506d768bd89SMartin Schwidefsky	BPOFF
5073037a52fSMartin Schwidefsky	la	%r1,4095		# validate r1
5083037a52fSMartin Schwidefsky	spt	__LC_CPU_TIMER_SAVE_AREA-4095(%r1)	# validate cpu timer
5093037a52fSMartin Schwidefsky	sckc	__LC_CLOCK_COMPARATOR			# validate comparator
5103037a52fSMartin Schwidefsky	lam	%a0,%a15,__LC_AREGS_SAVE_AREA-4095(%r1) # validate acrs
5113037a52fSMartin Schwidefsky	lmg	%r0,%r15,__LC_GPREGS_SAVE_AREA-4095(%r1)# validate gprs
512d5c352cdSHeiko Carstens	lg	%r12,__LC_CURRENT
5134bfc86ceSHeiko Carstens	lmg	%r8,%r9,__LC_MCK_OLD_PSW
51483abeffbSHendrik Brueckner	TSTMSK	__LC_MCCK_CODE,MCCK_CODE_SYSTEM_DAMAGE
5154bfc86ceSHeiko Carstens	jo	.Lmcck_panic		# yes -> rest of mcck code invalid
5163037a52fSMartin Schwidefsky	TSTMSK	__LC_MCCK_CODE,MCCK_CODE_CR_VALID
5173037a52fSMartin Schwidefsky	jno	.Lmcck_panic		# control registers invalid -> panic
5183037a52fSMartin Schwidefsky	la	%r14,4095
5193037a52fSMartin Schwidefsky	lctlg	%c0,%c15,__LC_CREGS_SAVE_AREA-4095(%r14) # validate ctl regs
5203037a52fSMartin Schwidefsky	ptlb
5212a2d7befSVasily Gorbik	lg	%r11,__LC_MCESAD-4095(%r14) # extended machine check save area
5223037a52fSMartin Schwidefsky	nill	%r11,0xfc00		# MCESA_ORIGIN_MASK
5233037a52fSMartin Schwidefsky	TSTMSK	__LC_CREGS_SAVE_AREA+16-4095(%r14),CR2_GUARDED_STORAGE
5243037a52fSMartin Schwidefsky	jno	0f
5253037a52fSMartin Schwidefsky	TSTMSK	__LC_MCCK_CODE,MCCK_CODE_GS_VALID
5263037a52fSMartin Schwidefsky	jno	0f
5273037a52fSMartin Schwidefsky	.insn	 rxy,0xe3000000004d,0,__MCESA_GS_SAVE_AREA(%r11) # LGSC
5283037a52fSMartin Schwidefsky0:	l	%r14,__LC_FP_CREG_SAVE_AREA-4095(%r14)
5293037a52fSMartin Schwidefsky	TSTMSK	__LC_MCCK_CODE,MCCK_CODE_FC_VALID
5303037a52fSMartin Schwidefsky	jo	0f
5313037a52fSMartin Schwidefsky	sr	%r14,%r14
5323037a52fSMartin Schwidefsky0:	sfpc	%r14
5333037a52fSMartin Schwidefsky	TSTMSK	__LC_MACHINE_FLAGS,MACHINE_FLAG_VX
5343037a52fSMartin Schwidefsky	jo	0f
5353037a52fSMartin Schwidefsky	lghi	%r14,__LC_FPREGS_SAVE_AREA
5363037a52fSMartin Schwidefsky	ld	%f0,0(%r14)
5373037a52fSMartin Schwidefsky	ld	%f1,8(%r14)
5383037a52fSMartin Schwidefsky	ld	%f2,16(%r14)
5393037a52fSMartin Schwidefsky	ld	%f3,24(%r14)
5403037a52fSMartin Schwidefsky	ld	%f4,32(%r14)
5413037a52fSMartin Schwidefsky	ld	%f5,40(%r14)
5423037a52fSMartin Schwidefsky	ld	%f6,48(%r14)
5433037a52fSMartin Schwidefsky	ld	%f7,56(%r14)
5443037a52fSMartin Schwidefsky	ld	%f8,64(%r14)
5453037a52fSMartin Schwidefsky	ld	%f9,72(%r14)
5463037a52fSMartin Schwidefsky	ld	%f10,80(%r14)
5473037a52fSMartin Schwidefsky	ld	%f11,88(%r14)
5483037a52fSMartin Schwidefsky	ld	%f12,96(%r14)
5493037a52fSMartin Schwidefsky	ld	%f13,104(%r14)
5503037a52fSMartin Schwidefsky	ld	%f14,112(%r14)
5513037a52fSMartin Schwidefsky	ld	%f15,120(%r14)
5523037a52fSMartin Schwidefsky	j	1f
5533037a52fSMartin Schwidefsky0:	VLM	%v0,%v15,0,%r11
5543037a52fSMartin Schwidefsky	VLM	%v16,%v31,256,%r11
5553037a52fSMartin Schwidefsky1:	lghi	%r14,__LC_CPU_TIMER_SAVE_AREA
5564bfc86ceSHeiko Carstens	mvc	__LC_MCCK_ENTER_TIMER(8),0(%r14)
55783abeffbSHendrik Brueckner	TSTMSK	__LC_MCCK_CODE,MCCK_CODE_CPU_TIMER_VALID
5584bfc86ceSHeiko Carstens	jo	3f
55956e62a73SSven Schnelle	la	%r14,__LC_SYS_ENTER_TIMER
56056e62a73SSven Schnelle	clc	0(8,%r14),__LC_EXIT_TIMER
5614bfc86ceSHeiko Carstens	jl	1f
5624bfc86ceSHeiko Carstens	la	%r14,__LC_EXIT_TIMER
5634bfc86ceSHeiko Carstens1:	clc	0(8,%r14),__LC_LAST_UPDATE_TIMER
5644bfc86ceSHeiko Carstens	jl	2f
5654bfc86ceSHeiko Carstens	la	%r14,__LC_LAST_UPDATE_TIMER
5664bfc86ceSHeiko Carstens2:	spt	0(%r14)
5674bfc86ceSHeiko Carstens	mvc	__LC_MCCK_ENTER_TIMER(8),0(%r14)
5683037a52fSMartin Schwidefsky3:	TSTMSK	__LC_MCCK_CODE,MCCK_CODE_PSW_MWP_VALID
5693037a52fSMartin Schwidefsky	jno	.Lmcck_panic
5703037a52fSMartin Schwidefsky	tmhh	%r8,0x0001		# interrupting from user ?
571*d35925b3SAlexander Gordeev	jnz	6f
5723037a52fSMartin Schwidefsky	TSTMSK	__LC_MCCK_CODE,MCCK_CODE_PSW_IA_VALID
5733037a52fSMartin Schwidefsky	jno	.Lmcck_panic
574b0d31159SSven Schnelle#if IS_ENABLED(CONFIG_KVM)
575*d35925b3SAlexander Gordeev	OUTSIDE	%r9,.Lsie_gmap,.Lsie_done,6f
576*d35925b3SAlexander Gordeev	OUTSIDE	%r9,.Lsie_entry,.Lsie_skip,4f
57720232b18SAlexander Gordeev	oi	__LC_CPU_FLAGS+7, _CIF_MCCK_GUEST
578*d35925b3SAlexander Gordeev	j	5f
579*d35925b3SAlexander Gordeev4:	CHKSTG	.Lmcck_panic
580*d35925b3SAlexander Gordeev5:	larl	%r14,.Lstosm_tmp
581*d35925b3SAlexander Gordeev	stosm	0(%r14),0x04		# turn dat on, keep irqs off
582*d35925b3SAlexander Gordeev	BPENTER	__SF_SIE_FLAGS(%r15),(_TIF_ISOLATE_BP|_TIF_ISOLATE_BP_GUEST)
583fbbdfca5SAlexander Gordeev	SIEEXIT
584b61b1595SSven Schnelle	j	.Lmcck_stack
585e2c13d64SAlexander Gordeev#endif
586*d35925b3SAlexander Gordeev6:	CHKSTG	.Lmcck_panic
587*d35925b3SAlexander Gordeev	larl	%r14,.Lstosm_tmp
588*d35925b3SAlexander Gordeev	stosm	0(%r14),0x04		# turn dat on, keep irqs off
589*d35925b3SAlexander Gordeev	tmhh	%r8,0x0001		# interrupting from user ?
590*d35925b3SAlexander Gordeev	jz	.Lmcck_stack
591b0d31159SSven Schnelle	BPENTER __TI_flags(%r12),_TIF_ISOLATE_BP
592b61b1595SSven Schnelle.Lmcck_stack:
593b61b1595SSven Schnelle	lg	%r15,__LC_MCCK_STACK
594b61b1595SSven Schnelle	la	%r11,STACK_FRAME_OVERHEAD(%r15)
59526521412SSven Schnelle	stctg	%c1,%c1,__PT_CR1(%r11)
596b61b1595SSven Schnelle	lctlg	%c1,%c1,__LC_KERNEL_ASCE
597b61b1595SSven Schnelle	xc	__SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
5984bfc86ceSHeiko Carstens	lghi	%r14,__LC_GPREGS_SAVE_AREA+64
5994bfc86ceSHeiko Carstens	stmg	%r0,%r7,__PT_R0(%r11)
6007041d281SMartin Schwidefsky	# clear user controlled registers to prevent speculative use
6017041d281SMartin Schwidefsky	xgr	%r0,%r0
6027041d281SMartin Schwidefsky	xgr	%r1,%r1
6037041d281SMartin Schwidefsky	xgr	%r3,%r3
6047041d281SMartin Schwidefsky	xgr	%r4,%r4
6057041d281SMartin Schwidefsky	xgr	%r5,%r5
6067041d281SMartin Schwidefsky	xgr	%r6,%r6
6077041d281SMartin Schwidefsky	xgr	%r7,%r7
6087041d281SMartin Schwidefsky	xgr	%r10,%r10
6094bfc86ceSHeiko Carstens	mvc	__PT_R8(64,%r11),0(%r14)
6104bfc86ceSHeiko Carstens	stmg	%r8,%r9,__PT_PSW(%r11)
6114bfc86ceSHeiko Carstens	xc	__PT_FLAGS(8,%r11),__PT_FLAGS(%r11)
6124bfc86ceSHeiko Carstens	xc	__SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
6134bfc86ceSHeiko Carstens	lgr	%r2,%r11		# pass pointer to pt_regs
6144bfc86ceSHeiko Carstens	brasl	%r14,s390_do_machine_check
6150b0ed657SSven Schnelle	cghi	%r2,0
6160b0ed657SSven Schnelle	je	.Lmcck_return
6174bfc86ceSHeiko Carstens	lg	%r1,__LC_KERNEL_STACK	# switch to kernel stack
6184bfc86ceSHeiko Carstens	mvc	STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11)
6194bfc86ceSHeiko Carstens	xc	__SF_BACKCHAIN(8,%r1),__SF_BACKCHAIN(%r1)
6204bfc86ceSHeiko Carstens	la	%r11,STACK_FRAME_OVERHEAD(%r1)
6214bfc86ceSHeiko Carstens	lgr	%r15,%r1
6224bfc86ceSHeiko Carstens	brasl	%r14,s390_handle_mcck
6234bfc86ceSHeiko Carstens.Lmcck_return:
62487d59863SHeiko Carstens	lctlg	%c1,%c1,__PT_CR1(%r11)
6254bfc86ceSHeiko Carstens	lmg	%r0,%r10,__PT_R0(%r11)
6264bfc86ceSHeiko Carstens	mvc	__LC_RETURN_MCCK_PSW(16),__PT_PSW(%r11) # move return PSW
6274bfc86ceSHeiko Carstens	tm	__LC_RETURN_MCCK_PSW+1,0x01 # returning to user ?
6284bfc86ceSHeiko Carstens	jno	0f
6296b73044bSMartin Schwidefsky	BPEXIT	__TI_flags(%r12),_TIF_ISOLATE_BP
6304bfc86ceSHeiko Carstens	stpt	__LC_EXIT_TIMER
6314bfc86ceSHeiko Carstens0:	lmg	%r11,%r15,__PT_R11(%r11)
6320b38b5e1SSven Schnelle	b	__LC_RETURN_MCCK_LPSWE
6334bfc86ceSHeiko Carstens
6344bfc86ceSHeiko Carstens.Lmcck_panic:
6357f6dc8d4SAlexander Gordeev	/*
6367f6dc8d4SAlexander Gordeev	 * Iterate over all possible CPU addresses in the range 0..0xffff
6377f6dc8d4SAlexander Gordeev	 * and stop each CPU using signal processor. Use compare and swap
6387f6dc8d4SAlexander Gordeev	 * to allow just one CPU-stopper and prevent concurrent CPUs from
6397f6dc8d4SAlexander Gordeev	 * stopping each other while leaving the others running.
6407f6dc8d4SAlexander Gordeev	 */
6417f6dc8d4SAlexander Gordeev	lhi	%r5,0
6427f6dc8d4SAlexander Gordeev	lhi	%r6,1
6437f6dc8d4SAlexander Gordeev	larl	%r7,.Lstop_lock
6447f6dc8d4SAlexander Gordeev	cs	%r5,%r6,0(%r7)		# single CPU-stopper only
6457f6dc8d4SAlexander Gordeev	jnz	4f
6467f6dc8d4SAlexander Gordeev	larl	%r7,.Lthis_cpu
6477f6dc8d4SAlexander Gordeev	stap	0(%r7)			# this CPU address
6487f6dc8d4SAlexander Gordeev	lh	%r4,0(%r7)
6497f6dc8d4SAlexander Gordeev	nilh	%r4,0
6507f6dc8d4SAlexander Gordeev	lhi	%r0,1
6517f6dc8d4SAlexander Gordeev	sll	%r0,16			# CPU counter
6527f6dc8d4SAlexander Gordeev	lhi	%r3,0			# next CPU address
6537f6dc8d4SAlexander Gordeev0:	cr	%r3,%r4
6547f6dc8d4SAlexander Gordeev	je	2f
6557f6dc8d4SAlexander Gordeev1:	sigp	%r1,%r3,SIGP_STOP	# stop next CPU
6567f6dc8d4SAlexander Gordeev	brc	SIGP_CC_BUSY,1b
6577f6dc8d4SAlexander Gordeev2:	ahi	%r3,1
6587f6dc8d4SAlexander Gordeev	brct	%r0,0b
6597f6dc8d4SAlexander Gordeev3:	sigp	%r1,%r4,SIGP_STOP	# stop this CPU
6607f6dc8d4SAlexander Gordeev	brc	SIGP_CC_BUSY,3b
6617f6dc8d4SAlexander Gordeev4:	j	4b
66226a374aeSMartin SchwidefskyENDPROC(mcck_int_handler)
6634bfc86ceSHeiko Carstens
6644bfc86ceSHeiko Carstens#
6654bfc86ceSHeiko Carstens# PSW restart interrupt handler
6664bfc86ceSHeiko Carstens#
6674bfc86ceSHeiko CarstensENTRY(restart_int_handler)
668e5b98199SMartin Schwidefsky	ALTERNATIVE "", ".insn s,0xb2800000,_LPP_OFFSET", 40
669e5b98199SMartin Schwidefsky	stg	%r15,__LC_SAVE_AREA_RESTART
6704bfc86ceSHeiko Carstens	lg	%r15,__LC_RESTART_STACK
671ce3dc447SMartin Schwidefsky	xc	STACK_FRAME_OVERHEAD(__PT_SIZE,%r15),STACK_FRAME_OVERHEAD(%r15)
672ce3dc447SMartin Schwidefsky	stmg	%r0,%r14,STACK_FRAME_OVERHEAD+__PT_R0(%r15)
673ce3dc447SMartin Schwidefsky	mvc	STACK_FRAME_OVERHEAD+__PT_R15(8,%r15),__LC_SAVE_AREA_RESTART
674ce3dc447SMartin Schwidefsky	mvc	STACK_FRAME_OVERHEAD+__PT_PSW(16,%r15),__LC_RST_OLD_PSW
6754bfc86ceSHeiko Carstens	xc	0(STACK_FRAME_OVERHEAD,%r15),0(%r15)
6764bfc86ceSHeiko Carstens	lg	%r1,__LC_RESTART_FN		# load fn, parm & source cpu
6774bfc86ceSHeiko Carstens	lg	%r2,__LC_RESTART_DATA
6784bfc86ceSHeiko Carstens	lg	%r3,__LC_RESTART_SOURCE
6794bfc86ceSHeiko Carstens	ltgr	%r3,%r3				# test source cpu address
6804bfc86ceSHeiko Carstens	jm	1f				# negative -> skip source stop
6814bfc86ceSHeiko Carstens0:	sigp	%r4,%r3,SIGP_SENSE		# sigp sense to source cpu
6824bfc86ceSHeiko Carstens	brc	10,0b				# wait for status stored
6834bfc86ceSHeiko Carstens1:	basr	%r14,%r1			# call function
6844bfc86ceSHeiko Carstens	stap	__SF_EMPTY(%r15)		# store cpu address
6854bfc86ceSHeiko Carstens	llgh	%r3,__SF_EMPTY(%r15)
6864bfc86ceSHeiko Carstens2:	sigp	%r4,%r3,SIGP_STOP		# sigp stop to current cpu
6874bfc86ceSHeiko Carstens	brc	2,2b
6884bfc86ceSHeiko Carstens3:	j	3b
68926a374aeSMartin SchwidefskyENDPROC(restart_int_handler)
6904bfc86ceSHeiko Carstens
6914bfc86ceSHeiko Carstens	.section .kprobes.text, "ax"
6924bfc86ceSHeiko Carstens
693ce3dc447SMartin Schwidefsky#if defined(CONFIG_CHECK_STACK) || defined(CONFIG_VMAP_STACK)
6944bfc86ceSHeiko Carstens/*
6954bfc86ceSHeiko Carstens * The synchronous or the asynchronous stack overflowed. We are dead.
6964bfc86ceSHeiko Carstens * No need to properly save the registers, we are going to panic anyway.
6974bfc86ceSHeiko Carstens * Setup a pt_regs so that show_trace can provide a good call trace.
6984bfc86ceSHeiko Carstens */
69926a374aeSMartin SchwidefskyENTRY(stack_overflow)
700ce3dc447SMartin Schwidefsky	lg	%r15,__LC_NODAT_STACK	# change to panic stack
7014bfc86ceSHeiko Carstens	la	%r11,STACK_FRAME_OVERHEAD(%r15)
7024bfc86ceSHeiko Carstens	stmg	%r0,%r7,__PT_R0(%r11)
7034bfc86ceSHeiko Carstens	stmg	%r8,%r9,__PT_PSW(%r11)
7044bfc86ceSHeiko Carstens	mvc	__PT_R8(64,%r11),0(%r14)
7054bfc86ceSHeiko Carstens	stg	%r10,__PT_ORIG_GPR2(%r11) # store last break to orig_gpr2
7064bfc86ceSHeiko Carstens	xc	__SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
7074bfc86ceSHeiko Carstens	lgr	%r2,%r11		# pass pointer to pt_regs
7084bfc86ceSHeiko Carstens	jg	kernel_stack_overflow
70926a374aeSMartin SchwidefskyENDPROC(stack_overflow)
7104bfc86ceSHeiko Carstens#endif
7114bfc86ceSHeiko Carstens
7127f6dc8d4SAlexander Gordeev	.section .data, "aw"
7137f6dc8d4SAlexander Gordeev		.align	4
7147f6dc8d4SAlexander Gordeev.Lstop_lock:	.long	0
7157f6dc8d4SAlexander Gordeev.Lthis_cpu:	.short	0
716*d35925b3SAlexander Gordeev.Lstosm_tmp:	.byte	0
7174bfc86ceSHeiko Carstens	.section .rodata, "a"
718ff4a742dSGerald Schaefer#define SYSCALL(esame,emu)	.quad __s390x_ ## esame
7194bfc86ceSHeiko Carstens	.globl	sys_call_table
7204bfc86ceSHeiko Carstenssys_call_table:
7214381f9f1SHendrik Brueckner#include "asm/syscall_table.h"
7224bfc86ceSHeiko Carstens#undef SYSCALL
7234bfc86ceSHeiko Carstens
7244bfc86ceSHeiko Carstens#ifdef CONFIG_COMPAT
7254bfc86ceSHeiko Carstens
726ff4a742dSGerald Schaefer#define SYSCALL(esame,emu)	.quad __s390_ ## emu
7274bfc86ceSHeiko Carstens	.globl	sys_call_table_emu
7284bfc86ceSHeiko Carstenssys_call_table_emu:
7294381f9f1SHendrik Brueckner#include "asm/syscall_table.h"
7304bfc86ceSHeiko Carstens#undef SYSCALL
7314bfc86ceSHeiko Carstens#endif
732