xref: /openbmc/linux/arch/s390/kernel/entry.S (revision 3dcb2223)
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 *    S390 low-level entry points.
4 *
5 *    Copyright IBM Corp. 1999, 2012
6 *    Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com),
7 *		 Hartmut Penner (hp@de.ibm.com),
8 *		 Denis Joseph Barrow (djbarrow@de.ibm.com,barrow_dj@yahoo.com),
9 */
10
11#include <linux/export.h>
12#include <linux/init.h>
13#include <linux/linkage.h>
14#include <asm/asm-extable.h>
15#include <asm/alternative-asm.h>
16#include <asm/processor.h>
17#include <asm/cache.h>
18#include <asm/dwarf.h>
19#include <asm/errno.h>
20#include <asm/ptrace.h>
21#include <asm/thread_info.h>
22#include <asm/asm-offsets.h>
23#include <asm/unistd.h>
24#include <asm/page.h>
25#include <asm/sigp.h>
26#include <asm/irq.h>
27#include <asm/vx-insn.h>
28#include <asm/setup.h>
29#include <asm/nmi.h>
30#include <asm/nospec-insn.h>
31
32_LPP_OFFSET	= __LC_LPP
33
34	.macro STBEAR address
35	ALTERNATIVE "nop", ".insn s,0xb2010000,\address", 193
36	.endm
37
38	.macro LBEAR address
39	ALTERNATIVE "nop", ".insn s,0xb2000000,\address", 193
40	.endm
41
42	.macro LPSWEY address,lpswe
43	ALTERNATIVE "b \lpswe; nopr", ".insn siy,0xeb0000000071,\address,0", 193
44	.endm
45
46	.macro MBEAR reg
47	ALTERNATIVE "brcl 0,0", __stringify(mvc __PT_LAST_BREAK(8,\reg),__LC_LAST_BREAK), 193
48	.endm
49
50	.macro	CHECK_STACK savearea
51#ifdef CONFIG_CHECK_STACK
52	tml	%r15,THREAD_SIZE - CONFIG_STACK_GUARD
53	lghi	%r14,\savearea
54	jz	stack_overflow
55#endif
56	.endm
57
58	.macro	CHECK_VMAP_STACK savearea,oklabel
59#ifdef CONFIG_VMAP_STACK
60	lgr	%r14,%r15
61	nill	%r14,0x10000 - THREAD_SIZE
62	oill	%r14,STACK_INIT_OFFSET
63	clg	%r14,__LC_KERNEL_STACK
64	je	\oklabel
65	clg	%r14,__LC_ASYNC_STACK
66	je	\oklabel
67	clg	%r14,__LC_MCCK_STACK
68	je	\oklabel
69	clg	%r14,__LC_NODAT_STACK
70	je	\oklabel
71	clg	%r14,__LC_RESTART_STACK
72	je	\oklabel
73	lghi	%r14,\savearea
74	j	stack_overflow
75#else
76	j	\oklabel
77#endif
78	.endm
79
80	/*
81	 * The TSTMSK macro generates a test-under-mask instruction by
82	 * calculating the memory offset for the specified mask value.
83	 * Mask value can be any constant.  The macro shifts the mask
84	 * value to calculate the memory offset for the test-under-mask
85	 * instruction.
86	 */
87	.macro TSTMSK addr, mask, size=8, bytepos=0
88		.if (\bytepos < \size) && (\mask >> 8)
89			.if (\mask & 0xff)
90				.error "Mask exceeds byte boundary"
91			.endif
92			TSTMSK \addr, "(\mask >> 8)", \size, "(\bytepos + 1)"
93			.exitm
94		.endif
95		.ifeq \mask
96			.error "Mask must not be zero"
97		.endif
98		off = \size - \bytepos - 1
99		tm	off+\addr, \mask
100	.endm
101
102	.macro BPOFF
103	ALTERNATIVE "nop", ".insn rrf,0xb2e80000,0,0,12,0", 82
104	.endm
105
106	.macro BPON
107	ALTERNATIVE "nop", ".insn rrf,0xb2e80000,0,0,13,0", 82
108	.endm
109
110	.macro BPENTER tif_ptr,tif_mask
111	ALTERNATIVE "TSTMSK \tif_ptr,\tif_mask; jz .+8; .insn rrf,0xb2e80000,0,0,13,0", \
112		    "j .+12; nop; nop", 82
113	.endm
114
115	.macro BPEXIT tif_ptr,tif_mask
116	TSTMSK	\tif_ptr,\tif_mask
117	ALTERNATIVE "jz .+8;  .insn rrf,0xb2e80000,0,0,12,0", \
118		    "jnz .+8; .insn rrf,0xb2e80000,0,0,13,0", 82
119	.endm
120
121#if IS_ENABLED(CONFIG_KVM)
122	/*
123	 * The OUTSIDE macro jumps to the provided label in case the value
124	 * in the provided register is outside of the provided range. The
125	 * macro is useful for checking whether a PSW stored in a register
126	 * pair points inside or outside of a block of instructions.
127	 * @reg: register to check
128	 * @start: start of the range
129	 * @end: end of the range
130	 * @outside_label: jump here if @reg is outside of [@start..@end)
131	 */
132	.macro OUTSIDE reg,start,end,outside_label
133	lgr	%r14,\reg
134	larl	%r13,\start
135	slgr	%r14,%r13
136	clgfrl	%r14,.Lrange_size\@
137	jhe	\outside_label
138	.section .rodata, "a"
139	.balign 4
140.Lrange_size\@:
141	.long	\end - \start
142	.previous
143	.endm
144
145	.macro SIEEXIT
146	lg	%r9,__SF_SIE_CONTROL(%r15)	# get control block pointer
147	ni	__SIE_PROG0C+3(%r9),0xfe	# no longer in SIE
148	lctlg	%c1,%c1,__LC_KERNEL_ASCE	# load primary asce
149	larl	%r9,sie_exit			# skip forward to sie_exit
150	.endm
151#endif
152
153	.macro STACKLEAK_ERASE
154#ifdef CONFIG_GCC_PLUGIN_STACKLEAK
155	brasl	%r14,stackleak_erase_on_task_stack
156#endif
157	.endm
158
159	GEN_BR_THUNK %r14
160
161	.section .kprobes.text, "ax"
162.Ldummy:
163	/*
164	 * The following nop exists only in order to avoid that the next
165	 * symbol starts at the beginning of the kprobes text section.
166	 * In that case there would be several symbols at the same address.
167	 * E.g. objdump would take an arbitrary symbol when disassembling
168	 * the code.
169	 * With the added nop in between this cannot happen.
170	 */
171	nop	0
172
173/*
174 * Scheduler resume function, called by switch_to
175 *  gpr2 = (task_struct *) prev
176 *  gpr3 = (task_struct *) next
177 * Returns:
178 *  gpr2 = prev
179 */
180SYM_FUNC_START(__switch_to)
181	stmg	%r6,%r15,__SF_GPRS(%r15)	# store gprs of prev task
182	lghi	%r4,__TASK_stack
183	lghi	%r1,__TASK_thread
184	llill	%r5,STACK_INIT_OFFSET
185	stg	%r15,__THREAD_ksp(%r1,%r2)	# store kernel stack of prev
186	lg	%r15,0(%r4,%r3)			# start of kernel stack of next
187	agr	%r15,%r5			# end of kernel stack of next
188	stg	%r3,__LC_CURRENT		# store task struct of next
189	stg	%r15,__LC_KERNEL_STACK		# store end of kernel stack
190	lg	%r15,__THREAD_ksp(%r1,%r3)	# load kernel stack of next
191	aghi	%r3,__TASK_pid
192	mvc	__LC_CURRENT_PID(4,%r0),0(%r3)	# store pid of next
193	lmg	%r6,%r15,__SF_GPRS(%r15)	# load gprs of next task
194	ALTERNATIVE "nop", "lpp _LPP_OFFSET", 40
195	BR_EX	%r14
196SYM_FUNC_END(__switch_to)
197
198#if IS_ENABLED(CONFIG_KVM)
199/*
200 * __sie64a calling convention:
201 * %r2 pointer to sie control block phys
202 * %r3 pointer to sie control block virt
203 * %r4 guest register save area
204 */
205SYM_FUNC_START(__sie64a)
206	stmg	%r6,%r14,__SF_GPRS(%r15)	# save kernel registers
207	lg	%r12,__LC_CURRENT
208	stg	%r2,__SF_SIE_CONTROL_PHYS(%r15)	# save sie block physical..
209	stg	%r3,__SF_SIE_CONTROL(%r15)	# ...and virtual addresses
210	stg	%r4,__SF_SIE_SAVEAREA(%r15)	# save guest register save area
211	xc	__SF_SIE_REASON(8,%r15),__SF_SIE_REASON(%r15) # reason code = 0
212	mvc	__SF_SIE_FLAGS(8,%r15),__TI_flags(%r12) # copy thread flags
213	lmg	%r0,%r13,0(%r4)			# load guest gprs 0-13
214	lg	%r14,__LC_GMAP			# get gmap pointer
215	ltgr	%r14,%r14
216	jz	.Lsie_gmap
217	lctlg	%c1,%c1,__GMAP_ASCE(%r14)	# load primary asce
218.Lsie_gmap:
219	lg	%r14,__SF_SIE_CONTROL(%r15)	# get control block pointer
220	oi	__SIE_PROG0C+3(%r14),1		# we are going into SIE now
221	tm	__SIE_PROG20+3(%r14),3		# last exit...
222	jnz	.Lsie_skip
223	TSTMSK	__LC_CPU_FLAGS,_CIF_FPU
224	jo	.Lsie_skip			# exit if fp/vx regs changed
225	lg	%r14,__SF_SIE_CONTROL_PHYS(%r15)	# get sie block phys addr
226	BPEXIT	__SF_SIE_FLAGS(%r15),_TIF_ISOLATE_BP_GUEST
227.Lsie_entry:
228	sie	0(%r14)
229# Let the next instruction be NOP to avoid triggering a machine check
230# and handling it in a guest as result of the instruction execution.
231	nopr	7
232.Lsie_leave:
233	BPOFF
234	BPENTER	__SF_SIE_FLAGS(%r15),_TIF_ISOLATE_BP_GUEST
235.Lsie_skip:
236	lg	%r14,__SF_SIE_CONTROL(%r15)	# get control block pointer
237	ni	__SIE_PROG0C+3(%r14),0xfe	# no longer in SIE
238	lctlg	%c1,%c1,__LC_KERNEL_ASCE	# load primary asce
239.Lsie_done:
240# some program checks are suppressing. C code (e.g. do_protection_exception)
241# will rewind the PSW by the ILC, which is often 4 bytes in case of SIE. There
242# are some corner cases (e.g. runtime instrumentation) where ILC is unpredictable.
243# Other instructions between __sie64a and .Lsie_done should not cause program
244# interrupts. So lets use 3 nops as a landing pad for all possible rewinds.
245.Lrewind_pad6:
246	nopr	7
247.Lrewind_pad4:
248	nopr	7
249.Lrewind_pad2:
250	nopr	7
251SYM_INNER_LABEL(sie_exit, SYM_L_GLOBAL)
252	lg	%r14,__SF_SIE_SAVEAREA(%r15)	# load guest register save area
253	stmg	%r0,%r13,0(%r14)		# save guest gprs 0-13
254	xgr	%r0,%r0				# clear guest registers to
255	xgr	%r1,%r1				# prevent speculative use
256	xgr	%r3,%r3
257	xgr	%r4,%r4
258	xgr	%r5,%r5
259	lmg	%r6,%r14,__SF_GPRS(%r15)	# restore kernel registers
260	lg	%r2,__SF_SIE_REASON(%r15)	# return exit reason code
261	BR_EX	%r14
262.Lsie_fault:
263	lghi	%r14,-EFAULT
264	stg	%r14,__SF_SIE_REASON(%r15)	# set exit reason code
265	j	sie_exit
266
267	EX_TABLE(.Lrewind_pad6,.Lsie_fault)
268	EX_TABLE(.Lrewind_pad4,.Lsie_fault)
269	EX_TABLE(.Lrewind_pad2,.Lsie_fault)
270	EX_TABLE(sie_exit,.Lsie_fault)
271SYM_FUNC_END(__sie64a)
272EXPORT_SYMBOL(__sie64a)
273EXPORT_SYMBOL(sie_exit)
274#endif
275
276/*
277 * SVC interrupt handler routine. System calls are synchronous events and
278 * are entered with interrupts disabled.
279 */
280
281SYM_CODE_START(system_call)
282	stpt	__LC_SYS_ENTER_TIMER
283	stmg	%r8,%r15,__LC_SAVE_AREA_SYNC
284	BPOFF
285	lghi	%r14,0
286.Lsysc_per:
287	STBEAR	__LC_LAST_BREAK
288	lctlg	%c1,%c1,__LC_KERNEL_ASCE
289	lg	%r15,__LC_KERNEL_STACK
290	xc	__SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
291	stmg	%r0,%r7,STACK_FRAME_OVERHEAD+__PT_R0(%r15)
292	# clear user controlled register to prevent speculative use
293	xgr	%r0,%r0
294	xgr	%r1,%r1
295	xgr	%r4,%r4
296	xgr	%r5,%r5
297	xgr	%r6,%r6
298	xgr	%r7,%r7
299	xgr	%r8,%r8
300	xgr	%r9,%r9
301	xgr	%r10,%r10
302	xgr	%r11,%r11
303	la	%r2,STACK_FRAME_OVERHEAD(%r15)	# pointer to pt_regs
304	mvc	__PT_R8(64,%r2),__LC_SAVE_AREA_SYNC
305	MBEAR	%r2
306	lgr	%r3,%r14
307	brasl	%r14,__do_syscall
308	STACKLEAK_ERASE
309	lctlg	%c1,%c1,__LC_USER_ASCE
310	mvc	__LC_RETURN_PSW(16),STACK_FRAME_OVERHEAD+__PT_PSW(%r15)
311	BPON
312	LBEAR	STACK_FRAME_OVERHEAD+__PT_LAST_BREAK(%r15)
313	lmg	%r0,%r15,STACK_FRAME_OVERHEAD+__PT_R0(%r15)
314	stpt	__LC_EXIT_TIMER
315	LPSWEY	__LC_RETURN_PSW,__LC_RETURN_LPSWE
316SYM_CODE_END(system_call)
317
318#
319# a new process exits the kernel with ret_from_fork
320#
321SYM_CODE_START(ret_from_fork)
322	lgr	%r3,%r11
323	brasl	%r14,__ret_from_fork
324	STACKLEAK_ERASE
325	lctlg	%c1,%c1,__LC_USER_ASCE
326	mvc	__LC_RETURN_PSW(16),STACK_FRAME_OVERHEAD+__PT_PSW(%r15)
327	BPON
328	LBEAR	STACK_FRAME_OVERHEAD+__PT_LAST_BREAK(%r15)
329	lmg	%r0,%r15,STACK_FRAME_OVERHEAD+__PT_R0(%r15)
330	stpt	__LC_EXIT_TIMER
331	LPSWEY	__LC_RETURN_PSW,__LC_RETURN_LPSWE
332SYM_CODE_END(ret_from_fork)
333
334/*
335 * Program check handler routine
336 */
337
338SYM_CODE_START(pgm_check_handler)
339	stpt	__LC_SYS_ENTER_TIMER
340	BPOFF
341	stmg	%r8,%r15,__LC_SAVE_AREA_SYNC
342	lghi	%r10,0
343	lmg	%r8,%r9,__LC_PGM_OLD_PSW
344	tmhh	%r8,0x0001		# coming from user space?
345	jno	.Lpgm_skip_asce
346	lctlg	%c1,%c1,__LC_KERNEL_ASCE
347	j	3f			# -> fault in user space
348.Lpgm_skip_asce:
349#if IS_ENABLED(CONFIG_KVM)
350	# cleanup critical section for program checks in __sie64a
351	OUTSIDE	%r9,.Lsie_gmap,.Lsie_done,1f
352	BPENTER	__SF_SIE_FLAGS(%r15),_TIF_ISOLATE_BP_GUEST
353	SIEEXIT
354	lghi	%r10,_PIF_GUEST_FAULT
355#endif
3561:	tmhh	%r8,0x4000		# PER bit set in old PSW ?
357	jnz	2f			# -> enabled, can't be a double fault
358	tm	__LC_PGM_ILC+3,0x80	# check for per exception
359	jnz	.Lpgm_svcper		# -> single stepped svc
3602:	CHECK_STACK __LC_SAVE_AREA_SYNC
361	aghi	%r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
362	# CHECK_VMAP_STACK branches to stack_overflow or 4f
363	CHECK_VMAP_STACK __LC_SAVE_AREA_SYNC,4f
3643:	lg	%r15,__LC_KERNEL_STACK
3654:	la	%r11,STACK_FRAME_OVERHEAD(%r15)
366	stg	%r10,__PT_FLAGS(%r11)
367	xc	__SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
368	stmg	%r0,%r7,__PT_R0(%r11)
369	mvc	__PT_R8(64,%r11),__LC_SAVE_AREA_SYNC
370	mvc	__PT_LAST_BREAK(8,%r11),__LC_PGM_LAST_BREAK
371	stmg	%r8,%r9,__PT_PSW(%r11)
372
373	# clear user controlled registers to prevent speculative use
374	xgr	%r0,%r0
375	xgr	%r1,%r1
376	xgr	%r3,%r3
377	xgr	%r4,%r4
378	xgr	%r5,%r5
379	xgr	%r6,%r6
380	xgr	%r7,%r7
381	lgr	%r2,%r11
382	brasl	%r14,__do_pgm_check
383	tmhh	%r8,0x0001		# returning to user space?
384	jno	.Lpgm_exit_kernel
385	STACKLEAK_ERASE
386	lctlg	%c1,%c1,__LC_USER_ASCE
387	BPON
388	stpt	__LC_EXIT_TIMER
389.Lpgm_exit_kernel:
390	mvc	__LC_RETURN_PSW(16),STACK_FRAME_OVERHEAD+__PT_PSW(%r15)
391	LBEAR	STACK_FRAME_OVERHEAD+__PT_LAST_BREAK(%r15)
392	lmg	%r0,%r15,STACK_FRAME_OVERHEAD+__PT_R0(%r15)
393	LPSWEY	__LC_RETURN_PSW,__LC_RETURN_LPSWE
394
395#
396# single stepped system call
397#
398.Lpgm_svcper:
399	mvc	__LC_RETURN_PSW(8),__LC_SVC_NEW_PSW
400	larl	%r14,.Lsysc_per
401	stg	%r14,__LC_RETURN_PSW+8
402	lghi	%r14,1
403	LBEAR	__LC_PGM_LAST_BREAK
404	LPSWEY	__LC_RETURN_PSW,__LC_RETURN_LPSWE # branch to .Lsysc_per
405SYM_CODE_END(pgm_check_handler)
406
407/*
408 * Interrupt handler macro used for external and IO interrupts.
409 */
410.macro INT_HANDLER name,lc_old_psw,handler
411SYM_CODE_START(\name)
412	stckf	__LC_INT_CLOCK
413	stpt	__LC_SYS_ENTER_TIMER
414	STBEAR	__LC_LAST_BREAK
415	BPOFF
416	stmg	%r8,%r15,__LC_SAVE_AREA_ASYNC
417	lmg	%r8,%r9,\lc_old_psw
418	tmhh	%r8,0x0001			# interrupting from user ?
419	jnz	1f
420#if IS_ENABLED(CONFIG_KVM)
421	OUTSIDE	%r9,.Lsie_gmap,.Lsie_done,0f
422	BPENTER	__SF_SIE_FLAGS(%r15),_TIF_ISOLATE_BP_GUEST
423	SIEEXIT
424#endif
4250:	CHECK_STACK __LC_SAVE_AREA_ASYNC
426	aghi	%r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
427	j	2f
4281:	lctlg	%c1,%c1,__LC_KERNEL_ASCE
429	lg	%r15,__LC_KERNEL_STACK
4302:	xc	__SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
431	la	%r11,STACK_FRAME_OVERHEAD(%r15)
432	stmg	%r0,%r7,__PT_R0(%r11)
433	# clear user controlled registers to prevent speculative use
434	xgr	%r0,%r0
435	xgr	%r1,%r1
436	xgr	%r3,%r3
437	xgr	%r4,%r4
438	xgr	%r5,%r5
439	xgr	%r6,%r6
440	xgr	%r7,%r7
441	xgr	%r10,%r10
442	xc	__PT_FLAGS(8,%r11),__PT_FLAGS(%r11)
443	mvc	__PT_R8(64,%r11),__LC_SAVE_AREA_ASYNC
444	MBEAR	%r11
445	stmg	%r8,%r9,__PT_PSW(%r11)
446	lgr	%r2,%r11		# pass pointer to pt_regs
447	brasl	%r14,\handler
448	mvc	__LC_RETURN_PSW(16),__PT_PSW(%r11)
449	tmhh	%r8,0x0001		# returning to user ?
450	jno	2f
451	STACKLEAK_ERASE
452	lctlg	%c1,%c1,__LC_USER_ASCE
453	BPON
454	stpt	__LC_EXIT_TIMER
4552:	LBEAR	__PT_LAST_BREAK(%r11)
456	lmg	%r0,%r15,__PT_R0(%r11)
457	LPSWEY	__LC_RETURN_PSW,__LC_RETURN_LPSWE
458SYM_CODE_END(\name)
459.endm
460
461INT_HANDLER ext_int_handler,__LC_EXT_OLD_PSW,do_ext_irq
462INT_HANDLER io_int_handler,__LC_IO_OLD_PSW,do_io_irq
463
464/*
465 * Load idle PSW.
466 */
467SYM_FUNC_START(psw_idle)
468	stg	%r14,(__SF_GPRS+8*8)(%r15)
469	stg	%r3,__SF_EMPTY(%r15)
470	larl	%r1,psw_idle_exit
471	stg	%r1,__SF_EMPTY+8(%r15)
472	larl	%r1,smp_cpu_mtid
473	llgf	%r1,0(%r1)
474	ltgr	%r1,%r1
475	jz	.Lpsw_idle_stcctm
476	.insn	rsy,0xeb0000000017,%r1,5,__MT_CYCLES_ENTER(%r2)
477.Lpsw_idle_stcctm:
478	oi	__LC_CPU_FLAGS+7,_CIF_ENABLED_WAIT
479	BPON
480	stckf	__CLOCK_IDLE_ENTER(%r2)
481	stpt	__TIMER_IDLE_ENTER(%r2)
482	lpswe	__SF_EMPTY(%r15)
483SYM_INNER_LABEL(psw_idle_exit, SYM_L_GLOBAL)
484	BR_EX	%r14
485SYM_FUNC_END(psw_idle)
486
487/*
488 * Machine check handler routines
489 */
490SYM_CODE_START(mcck_int_handler)
491	BPOFF
492	la	%r1,4095		# validate r1
493	spt	__LC_CPU_TIMER_SAVE_AREA-4095(%r1)	# validate cpu timer
494	LBEAR	__LC_LAST_BREAK_SAVE_AREA-4095(%r1)		# validate bear
495	lmg	%r0,%r15,__LC_GPREGS_SAVE_AREA		# validate gprs
496	lmg	%r8,%r9,__LC_MCK_OLD_PSW
497	TSTMSK	__LC_MCCK_CODE,MCCK_CODE_SYSTEM_DAMAGE
498	jo	.Lmcck_panic		# yes -> rest of mcck code invalid
499	TSTMSK	__LC_MCCK_CODE,MCCK_CODE_CR_VALID
500	jno	.Lmcck_panic		# control registers invalid -> panic
501	lctlg	%c0,%c15,__LC_CREGS_SAVE_AREA		# validate ctl regs
502	ptlb
503	lghi	%r14,__LC_CPU_TIMER_SAVE_AREA
504	mvc	__LC_MCCK_ENTER_TIMER(8),0(%r14)
505	TSTMSK	__LC_MCCK_CODE,MCCK_CODE_CPU_TIMER_VALID
506	jo	3f
507	la	%r14,__LC_SYS_ENTER_TIMER
508	clc	0(8,%r14),__LC_EXIT_TIMER
509	jl	1f
510	la	%r14,__LC_EXIT_TIMER
5111:	clc	0(8,%r14),__LC_LAST_UPDATE_TIMER
512	jl	2f
513	la	%r14,__LC_LAST_UPDATE_TIMER
5142:	spt	0(%r14)
515	mvc	__LC_MCCK_ENTER_TIMER(8),0(%r14)
5163:	TSTMSK	__LC_MCCK_CODE,MCCK_CODE_PSW_MWP_VALID
517	jno	.Lmcck_panic
518	tmhh	%r8,0x0001		# interrupting from user ?
519	jnz	.Lmcck_user
520	TSTMSK	__LC_MCCK_CODE,MCCK_CODE_PSW_IA_VALID
521	jno	.Lmcck_panic
522#if IS_ENABLED(CONFIG_KVM)
523	OUTSIDE	%r9,.Lsie_gmap,.Lsie_done,.Lmcck_user
524	OUTSIDE	%r9,.Lsie_entry,.Lsie_leave,4f
525	oi	__LC_CPU_FLAGS+7, _CIF_MCCK_GUEST
5264:	BPENTER	__SF_SIE_FLAGS(%r15),_TIF_ISOLATE_BP_GUEST
527	SIEEXIT
528#endif
529.Lmcck_user:
530	lg	%r15,__LC_MCCK_STACK
531	la	%r11,STACK_FRAME_OVERHEAD(%r15)
532	stctg	%c1,%c1,__PT_CR1(%r11)
533	lctlg	%c1,%c1,__LC_KERNEL_ASCE
534	xc	__SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
535	lghi	%r14,__LC_GPREGS_SAVE_AREA+64
536	stmg	%r0,%r7,__PT_R0(%r11)
537	# clear user controlled registers to prevent speculative use
538	xgr	%r0,%r0
539	xgr	%r1,%r1
540	xgr	%r3,%r3
541	xgr	%r4,%r4
542	xgr	%r5,%r5
543	xgr	%r6,%r6
544	xgr	%r7,%r7
545	xgr	%r10,%r10
546	mvc	__PT_R8(64,%r11),0(%r14)
547	stmg	%r8,%r9,__PT_PSW(%r11)
548	xc	__PT_FLAGS(8,%r11),__PT_FLAGS(%r11)
549	xc	__SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
550	lgr	%r2,%r11		# pass pointer to pt_regs
551	brasl	%r14,s390_do_machine_check
552	lctlg	%c1,%c1,__PT_CR1(%r11)
553	lmg	%r0,%r10,__PT_R0(%r11)
554	mvc	__LC_RETURN_MCCK_PSW(16),__PT_PSW(%r11) # move return PSW
555	tm	__LC_RETURN_MCCK_PSW+1,0x01 # returning to user ?
556	jno	0f
557	BPON
558	stpt	__LC_EXIT_TIMER
5590:	ALTERNATIVE "nop", __stringify(lghi %r12,__LC_LAST_BREAK_SAVE_AREA),193
560	LBEAR	0(%r12)
561	lmg	%r11,%r15,__PT_R11(%r11)
562	LPSWEY	__LC_RETURN_MCCK_PSW,__LC_RETURN_MCCK_LPSWE
563
564.Lmcck_panic:
565	/*
566	 * Iterate over all possible CPU addresses in the range 0..0xffff
567	 * and stop each CPU using signal processor. Use compare and swap
568	 * to allow just one CPU-stopper and prevent concurrent CPUs from
569	 * stopping each other while leaving the others running.
570	 */
571	lhi	%r5,0
572	lhi	%r6,1
573	larl	%r7,stop_lock
574	cs	%r5,%r6,0(%r7)		# single CPU-stopper only
575	jnz	4f
576	larl	%r7,this_cpu
577	stap	0(%r7)			# this CPU address
578	lh	%r4,0(%r7)
579	nilh	%r4,0
580	lhi	%r0,1
581	sll	%r0,16			# CPU counter
582	lhi	%r3,0			# next CPU address
5830:	cr	%r3,%r4
584	je	2f
5851:	sigp	%r1,%r3,SIGP_STOP	# stop next CPU
586	brc	SIGP_CC_BUSY,1b
5872:	ahi	%r3,1
588	brct	%r0,0b
5893:	sigp	%r1,%r4,SIGP_STOP	# stop this CPU
590	brc	SIGP_CC_BUSY,3b
5914:	j	4b
592SYM_CODE_END(mcck_int_handler)
593
594SYM_CODE_START(restart_int_handler)
595	ALTERNATIVE "nop", "lpp _LPP_OFFSET", 40
596	stg	%r15,__LC_SAVE_AREA_RESTART
597	TSTMSK	__LC_RESTART_FLAGS,RESTART_FLAG_CTLREGS,4
598	jz	0f
599	lctlg	%c0,%c15,__LC_CREGS_SAVE_AREA
6000:	larl	%r15,daton_psw
601	lpswe	0(%r15)				# turn dat on, keep irqs off
602.Ldaton:
603	lg	%r15,__LC_RESTART_STACK
604	xc	STACK_FRAME_OVERHEAD(__PT_SIZE,%r15),STACK_FRAME_OVERHEAD(%r15)
605	stmg	%r0,%r14,STACK_FRAME_OVERHEAD+__PT_R0(%r15)
606	mvc	STACK_FRAME_OVERHEAD+__PT_R15(8,%r15),__LC_SAVE_AREA_RESTART
607	mvc	STACK_FRAME_OVERHEAD+__PT_PSW(16,%r15),__LC_RST_OLD_PSW
608	xc	0(STACK_FRAME_OVERHEAD,%r15),0(%r15)
609	lg	%r1,__LC_RESTART_FN		# load fn, parm & source cpu
610	lg	%r2,__LC_RESTART_DATA
611	lgf	%r3,__LC_RESTART_SOURCE
612	ltgr	%r3,%r3				# test source cpu address
613	jm	1f				# negative -> skip source stop
6140:	sigp	%r4,%r3,SIGP_SENSE		# sigp sense to source cpu
615	brc	10,0b				# wait for status stored
6161:	basr	%r14,%r1			# call function
617	stap	__SF_EMPTY(%r15)		# store cpu address
618	llgh	%r3,__SF_EMPTY(%r15)
6192:	sigp	%r4,%r3,SIGP_STOP		# sigp stop to current cpu
620	brc	2,2b
6213:	j	3b
622SYM_CODE_END(restart_int_handler)
623
624	.section .kprobes.text, "ax"
625
626#if defined(CONFIG_CHECK_STACK) || defined(CONFIG_VMAP_STACK)
627/*
628 * The synchronous or the asynchronous stack overflowed. We are dead.
629 * No need to properly save the registers, we are going to panic anyway.
630 * Setup a pt_regs so that show_trace can provide a good call trace.
631 */
632SYM_CODE_START(stack_overflow)
633	lg	%r15,__LC_NODAT_STACK	# change to panic stack
634	la	%r11,STACK_FRAME_OVERHEAD(%r15)
635	stmg	%r0,%r7,__PT_R0(%r11)
636	stmg	%r8,%r9,__PT_PSW(%r11)
637	mvc	__PT_R8(64,%r11),0(%r14)
638	stg	%r10,__PT_ORIG_GPR2(%r11) # store last break to orig_gpr2
639	xc	__SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
640	lgr	%r2,%r11		# pass pointer to pt_regs
641	jg	kernel_stack_overflow
642SYM_CODE_END(stack_overflow)
643#endif
644
645	.section .data, "aw"
646	.balign	4
647SYM_DATA_LOCAL(stop_lock,	.long 0)
648SYM_DATA_LOCAL(this_cpu,	.short 0)
649	.balign	8
650SYM_DATA_START_LOCAL(daton_psw)
651	.quad	PSW_KERNEL_BITS
652	.quad	.Ldaton
653SYM_DATA_END(daton_psw)
654
655	.section .rodata, "a"
656	.balign	8
657#define SYSCALL(esame,emu)	.quad __s390x_ ## esame
658SYM_DATA_START(sys_call_table)
659#include "asm/syscall_table.h"
660SYM_DATA_END(sys_call_table)
661#undef SYSCALL
662
663#ifdef CONFIG_COMPAT
664
665#define SYSCALL(esame,emu)	.quad __s390_ ## emu
666SYM_DATA_START(sys_call_table_emu)
667#include "asm/syscall_table.h"
668SYM_DATA_END(sys_call_table_emu)
669#undef SYSCALL
670#endif
671