xref: /openbmc/linux/arch/s390/kernel/entry.S (revision ecc23d0a422a3118fcf6e4f0a46e17a6c2047b02)
1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3  *    S390 low-level entry points.
4  *
5  *    Copyright IBM Corp. 1999, 2012
6  *    Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com),
7  *		 Hartmut Penner (hp@de.ibm.com),
8  *		 Denis Joseph Barrow (djbarrow@de.ibm.com,barrow_dj@yahoo.com),
9  */
10 
11 #include <linux/export.h>
12 #include <linux/init.h>
13 #include <linux/linkage.h>
14 #include <asm/asm-extable.h>
15 #include <asm/alternative-asm.h>
16 #include <asm/processor.h>
17 #include <asm/cache.h>
18 #include <asm/dwarf.h>
19 #include <asm/errno.h>
20 #include <asm/ptrace.h>
21 #include <asm/thread_info.h>
22 #include <asm/asm-offsets.h>
23 #include <asm/unistd.h>
24 #include <asm/page.h>
25 #include <asm/sigp.h>
26 #include <asm/irq.h>
27 #include <asm/vx-insn.h>
28 #include <asm/setup.h>
29 #include <asm/nmi.h>
30 #include <asm/nospec-insn.h>
31 
32 _LPP_OFFSET	= __LC_LPP
33 
34 	.macro STBEAR address
35 	ALTERNATIVE "nop", ".insn s,0xb2010000,\address", 193
36 	.endm
37 
38 	.macro LBEAR address
39 	ALTERNATIVE "nop", ".insn s,0xb2000000,\address", 193
40 	.endm
41 
42 	.macro LPSWEY address,lpswe
43 	ALTERNATIVE "b \lpswe; nopr", ".insn siy,0xeb0000000071,\address,0", 193
44 	.endm
45 
46 	.macro MBEAR reg
47 	ALTERNATIVE "brcl 0,0", __stringify(mvc __PT_LAST_BREAK(8,\reg),__LC_LAST_BREAK), 193
48 	.endm
49 
50 	.macro	CHECK_STACK savearea
51 #ifdef CONFIG_CHECK_STACK
52 	tml	%r15,THREAD_SIZE - CONFIG_STACK_GUARD
53 	lghi	%r14,\savearea
54 	jz	stack_overflow
55 #endif
56 	.endm
57 
58 	.macro	CHECK_VMAP_STACK savearea,oklabel
59 #ifdef CONFIG_VMAP_STACK
60 	lgr	%r14,%r15
61 	nill	%r14,0x10000 - THREAD_SIZE
62 	oill	%r14,STACK_INIT_OFFSET
63 	clg	%r14,__LC_KERNEL_STACK
64 	je	\oklabel
65 	clg	%r14,__LC_ASYNC_STACK
66 	je	\oklabel
67 	clg	%r14,__LC_MCCK_STACK
68 	je	\oklabel
69 	clg	%r14,__LC_NODAT_STACK
70 	je	\oklabel
71 	clg	%r14,__LC_RESTART_STACK
72 	je	\oklabel
73 	lghi	%r14,\savearea
74 	j	stack_overflow
75 #else
76 	j	\oklabel
77 #endif
78 	.endm
79 
80 	/*
81 	 * The TSTMSK macro generates a test-under-mask instruction by
82 	 * calculating the memory offset for the specified mask value.
83 	 * Mask value can be any constant.  The macro shifts the mask
84 	 * value to calculate the memory offset for the test-under-mask
85 	 * instruction.
86 	 */
87 	.macro TSTMSK addr, mask, size=8, bytepos=0
88 		.if (\bytepos < \size) && (\mask >> 8)
89 			.if (\mask & 0xff)
90 				.error "Mask exceeds byte boundary"
91 			.endif
92 			TSTMSK \addr, "(\mask >> 8)", \size, "(\bytepos + 1)"
93 			.exitm
94 		.endif
95 		.ifeq \mask
96 			.error "Mask must not be zero"
97 		.endif
98 		off = \size - \bytepos - 1
99 		tm	off+\addr, \mask
100 	.endm
101 
102 	.macro BPOFF
103 	ALTERNATIVE "nop", ".insn rrf,0xb2e80000,0,0,12,0", 82
104 	.endm
105 
106 	.macro BPON
107 	ALTERNATIVE "nop", ".insn rrf,0xb2e80000,0,0,13,0", 82
108 	.endm
109 
110 	.macro BPENTER tif_ptr,tif_mask
111 	ALTERNATIVE "TSTMSK \tif_ptr,\tif_mask; jz .+8; .insn rrf,0xb2e80000,0,0,13,0", \
112 		    "j .+12; nop; nop", 82
113 	.endm
114 
115 	.macro BPEXIT tif_ptr,tif_mask
116 	TSTMSK	\tif_ptr,\tif_mask
117 	ALTERNATIVE "jz .+8;  .insn rrf,0xb2e80000,0,0,12,0", \
118 		    "jnz .+8; .insn rrf,0xb2e80000,0,0,13,0", 82
119 	.endm
120 
121 #if IS_ENABLED(CONFIG_KVM)
122 	/*
123 	 * The OUTSIDE macro jumps to the provided label in case the value
124 	 * in the provided register is outside of the provided range. The
125 	 * macro is useful for checking whether a PSW stored in a register
126 	 * pair points inside or outside of a block of instructions.
127 	 * @reg: register to check
128 	 * @start: start of the range
129 	 * @end: end of the range
130 	 * @outside_label: jump here if @reg is outside of [@start..@end)
131 	 */
132 	.macro OUTSIDE reg,start,end,outside_label
133 	lgr	%r14,\reg
134 	larl	%r13,\start
135 	slgr	%r14,%r13
136 	clgfrl	%r14,.Lrange_size\@
137 	jhe	\outside_label
138 	.section .rodata, "a"
139 	.balign 4
140 .Lrange_size\@:
141 	.long	\end - \start
142 	.previous
143 	.endm
144 
145 	.macro SIEEXIT
146 	lg	%r9,__SF_SIE_CONTROL(%r15)	# get control block pointer
147 	ni	__SIE_PROG0C+3(%r9),0xfe	# no longer in SIE
148 	lctlg	%c1,%c1,__LC_KERNEL_ASCE	# load primary asce
149 	larl	%r9,sie_exit			# skip forward to sie_exit
150 	.endm
151 #endif
152 
153 	.macro STACKLEAK_ERASE
154 #ifdef CONFIG_GCC_PLUGIN_STACKLEAK
155 	brasl	%r14,stackleak_erase_on_task_stack
156 #endif
157 	.endm
158 
159 	GEN_BR_THUNK %r14
160 
161 	.section .kprobes.text, "ax"
162 .Ldummy:
163 	/*
164 	 * The following nop exists only in order to avoid that the next
165 	 * symbol starts at the beginning of the kprobes text section.
166 	 * In that case there would be several symbols at the same address.
167 	 * E.g. objdump would take an arbitrary symbol when disassembling
168 	 * the code.
169 	 * With the added nop in between this cannot happen.
170 	 */
171 	nop	0
172 
173 /*
174  * Scheduler resume function, called by switch_to
175  *  gpr2 = (task_struct *) prev
176  *  gpr3 = (task_struct *) next
177  * Returns:
178  *  gpr2 = prev
179  */
180 SYM_FUNC_START(__switch_to)
181 	stmg	%r6,%r15,__SF_GPRS(%r15)	# store gprs of prev task
182 	lghi	%r4,__TASK_stack
183 	lghi	%r1,__TASK_thread
184 	llill	%r5,STACK_INIT_OFFSET
185 	stg	%r15,__THREAD_ksp(%r1,%r2)	# store kernel stack of prev
186 	lg	%r15,0(%r4,%r3)			# start of kernel stack of next
187 	agr	%r15,%r5			# end of kernel stack of next
188 	stg	%r3,__LC_CURRENT		# store task struct of next
189 	stg	%r15,__LC_KERNEL_STACK		# store end of kernel stack
190 	lg	%r15,__THREAD_ksp(%r1,%r3)	# load kernel stack of next
191 	aghi	%r3,__TASK_pid
192 	mvc	__LC_CURRENT_PID(4,%r0),0(%r3)	# store pid of next
193 	lmg	%r6,%r15,__SF_GPRS(%r15)	# load gprs of next task
194 	ALTERNATIVE "nop", "lpp _LPP_OFFSET", 40
195 	BR_EX	%r14
196 SYM_FUNC_END(__switch_to)
197 
198 #if IS_ENABLED(CONFIG_KVM)
199 /*
200  * __sie64a calling convention:
201  * %r2 pointer to sie control block phys
202  * %r3 pointer to sie control block virt
203  * %r4 guest register save area
204  */
205 SYM_FUNC_START(__sie64a)
206 	stmg	%r6,%r14,__SF_GPRS(%r15)	# save kernel registers
207 	lg	%r12,__LC_CURRENT
208 	stg	%r2,__SF_SIE_CONTROL_PHYS(%r15)	# save sie block physical..
209 	stg	%r3,__SF_SIE_CONTROL(%r15)	# ...and virtual addresses
210 	stg	%r4,__SF_SIE_SAVEAREA(%r15)	# save guest register save area
211 	xc	__SF_SIE_REASON(8,%r15),__SF_SIE_REASON(%r15) # reason code = 0
212 	mvc	__SF_SIE_FLAGS(8,%r15),__TI_flags(%r12) # copy thread flags
213 	lmg	%r0,%r13,0(%r4)			# load guest gprs 0-13
214 	lg	%r14,__LC_GMAP			# get gmap pointer
215 	ltgr	%r14,%r14
216 	jz	.Lsie_gmap
217 	lctlg	%c1,%c1,__GMAP_ASCE(%r14)	# load primary asce
218 .Lsie_gmap:
219 	lg	%r14,__SF_SIE_CONTROL(%r15)	# get control block pointer
220 	oi	__SIE_PROG0C+3(%r14),1		# we are going into SIE now
221 	tm	__SIE_PROG20+3(%r14),3		# last exit...
222 	jnz	.Lsie_skip
223 	TSTMSK	__LC_CPU_FLAGS,_CIF_FPU
224 	jo	.Lsie_skip			# exit if fp/vx regs changed
225 	lg	%r14,__SF_SIE_CONTROL_PHYS(%r15)	# get sie block phys addr
226 	BPEXIT	__SF_SIE_FLAGS(%r15),_TIF_ISOLATE_BP_GUEST
227 .Lsie_entry:
228 	sie	0(%r14)
229 # Let the next instruction be NOP to avoid triggering a machine check
230 # and handling it in a guest as result of the instruction execution.
231 	nopr	7
232 .Lsie_leave:
233 	BPOFF
234 	BPENTER	__SF_SIE_FLAGS(%r15),_TIF_ISOLATE_BP_GUEST
235 .Lsie_skip:
236 	lg	%r14,__SF_SIE_CONTROL(%r15)	# get control block pointer
237 	ni	__SIE_PROG0C+3(%r14),0xfe	# no longer in SIE
238 	lctlg	%c1,%c1,__LC_KERNEL_ASCE	# load primary asce
239 .Lsie_done:
240 # some program checks are suppressing. C code (e.g. do_protection_exception)
241 # will rewind the PSW by the ILC, which is often 4 bytes in case of SIE. There
242 # are some corner cases (e.g. runtime instrumentation) where ILC is unpredictable.
243 # Other instructions between __sie64a and .Lsie_done should not cause program
244 # interrupts. So lets use 3 nops as a landing pad for all possible rewinds.
245 .Lrewind_pad6:
246 	nopr	7
247 .Lrewind_pad4:
248 	nopr	7
249 .Lrewind_pad2:
250 	nopr	7
251 SYM_INNER_LABEL(sie_exit, SYM_L_GLOBAL)
252 	lg	%r14,__SF_SIE_SAVEAREA(%r15)	# load guest register save area
253 	stmg	%r0,%r13,0(%r14)		# save guest gprs 0-13
254 	xgr	%r0,%r0				# clear guest registers to
255 	xgr	%r1,%r1				# prevent speculative use
256 	xgr	%r3,%r3
257 	xgr	%r4,%r4
258 	xgr	%r5,%r5
259 	lmg	%r6,%r14,__SF_GPRS(%r15)	# restore kernel registers
260 	lg	%r2,__SF_SIE_REASON(%r15)	# return exit reason code
261 	BR_EX	%r14
262 .Lsie_fault:
263 	lghi	%r14,-EFAULT
264 	stg	%r14,__SF_SIE_REASON(%r15)	# set exit reason code
265 	j	sie_exit
266 
267 	EX_TABLE(.Lrewind_pad6,.Lsie_fault)
268 	EX_TABLE(.Lrewind_pad4,.Lsie_fault)
269 	EX_TABLE(.Lrewind_pad2,.Lsie_fault)
270 	EX_TABLE(sie_exit,.Lsie_fault)
271 SYM_FUNC_END(__sie64a)
272 EXPORT_SYMBOL(__sie64a)
273 EXPORT_SYMBOL(sie_exit)
274 #endif
275 
276 /*
277  * SVC interrupt handler routine. System calls are synchronous events and
278  * are entered with interrupts disabled.
279  */
280 
281 SYM_CODE_START(system_call)
282 	stpt	__LC_SYS_ENTER_TIMER
283 	stmg	%r8,%r15,__LC_SAVE_AREA_SYNC
284 	BPOFF
285 	lghi	%r14,0
286 .Lsysc_per:
287 	STBEAR	__LC_LAST_BREAK
288 	lctlg	%c1,%c1,__LC_KERNEL_ASCE
289 	lg	%r15,__LC_KERNEL_STACK
290 	xc	__SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
291 	stmg	%r0,%r7,STACK_FRAME_OVERHEAD+__PT_R0(%r15)
292 	# clear user controlled register to prevent speculative use
293 	xgr	%r0,%r0
294 	xgr	%r1,%r1
295 	xgr	%r4,%r4
296 	xgr	%r5,%r5
297 	xgr	%r6,%r6
298 	xgr	%r7,%r7
299 	xgr	%r8,%r8
300 	xgr	%r9,%r9
301 	xgr	%r10,%r10
302 	xgr	%r11,%r11
303 	la	%r2,STACK_FRAME_OVERHEAD(%r15)	# pointer to pt_regs
304 	mvc	__PT_R8(64,%r2),__LC_SAVE_AREA_SYNC
305 	MBEAR	%r2
306 	lgr	%r3,%r14
307 	brasl	%r14,__do_syscall
308 	STACKLEAK_ERASE
309 	lctlg	%c1,%c1,__LC_USER_ASCE
310 	mvc	__LC_RETURN_PSW(16),STACK_FRAME_OVERHEAD+__PT_PSW(%r15)
311 	BPON
312 	LBEAR	STACK_FRAME_OVERHEAD+__PT_LAST_BREAK(%r15)
313 	lmg	%r0,%r15,STACK_FRAME_OVERHEAD+__PT_R0(%r15)
314 	stpt	__LC_EXIT_TIMER
315 	LPSWEY	__LC_RETURN_PSW,__LC_RETURN_LPSWE
316 SYM_CODE_END(system_call)
317 
318 #
319 # a new process exits the kernel with ret_from_fork
320 #
321 SYM_CODE_START(ret_from_fork)
322 	lgr	%r3,%r11
323 	brasl	%r14,__ret_from_fork
324 	STACKLEAK_ERASE
325 	lctlg	%c1,%c1,__LC_USER_ASCE
326 	mvc	__LC_RETURN_PSW(16),STACK_FRAME_OVERHEAD+__PT_PSW(%r15)
327 	BPON
328 	LBEAR	STACK_FRAME_OVERHEAD+__PT_LAST_BREAK(%r15)
329 	lmg	%r0,%r15,STACK_FRAME_OVERHEAD+__PT_R0(%r15)
330 	stpt	__LC_EXIT_TIMER
331 	LPSWEY	__LC_RETURN_PSW,__LC_RETURN_LPSWE
332 SYM_CODE_END(ret_from_fork)
333 
334 /*
335  * Program check handler routine
336  */
337 
338 SYM_CODE_START(pgm_check_handler)
339 	stpt	__LC_SYS_ENTER_TIMER
340 	BPOFF
341 	stmg	%r8,%r15,__LC_SAVE_AREA_SYNC
342 	lghi	%r10,0
343 	lmg	%r8,%r9,__LC_PGM_OLD_PSW
344 	tmhh	%r8,0x0001		# coming from user space?
345 	jno	.Lpgm_skip_asce
346 	lctlg	%c1,%c1,__LC_KERNEL_ASCE
347 	j	3f			# -> fault in user space
348 .Lpgm_skip_asce:
349 #if IS_ENABLED(CONFIG_KVM)
350 	# cleanup critical section for program checks in __sie64a
351 	OUTSIDE	%r9,.Lsie_gmap,.Lsie_done,1f
352 	BPENTER	__SF_SIE_FLAGS(%r15),_TIF_ISOLATE_BP_GUEST
353 	SIEEXIT
354 	lghi	%r10,_PIF_GUEST_FAULT
355 #endif
356 1:	tmhh	%r8,0x4000		# PER bit set in old PSW ?
357 	jnz	2f			# -> enabled, can't be a double fault
358 	tm	__LC_PGM_ILC+3,0x80	# check for per exception
359 	jnz	.Lpgm_svcper		# -> single stepped svc
360 2:	CHECK_STACK __LC_SAVE_AREA_SYNC
361 	aghi	%r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
362 	# CHECK_VMAP_STACK branches to stack_overflow or 4f
363 	CHECK_VMAP_STACK __LC_SAVE_AREA_SYNC,4f
364 3:	lg	%r15,__LC_KERNEL_STACK
365 4:	la	%r11,STACK_FRAME_OVERHEAD(%r15)
366 	stg	%r10,__PT_FLAGS(%r11)
367 	xc	__SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
368 	stmg	%r0,%r7,__PT_R0(%r11)
369 	mvc	__PT_R8(64,%r11),__LC_SAVE_AREA_SYNC
370 	mvc	__PT_LAST_BREAK(8,%r11),__LC_PGM_LAST_BREAK
371 	stmg	%r8,%r9,__PT_PSW(%r11)
372 
373 	# clear user controlled registers to prevent speculative use
374 	xgr	%r0,%r0
375 	xgr	%r1,%r1
376 	xgr	%r3,%r3
377 	xgr	%r4,%r4
378 	xgr	%r5,%r5
379 	xgr	%r6,%r6
380 	xgr	%r7,%r7
381 	lgr	%r2,%r11
382 	brasl	%r14,__do_pgm_check
383 	tmhh	%r8,0x0001		# returning to user space?
384 	jno	.Lpgm_exit_kernel
385 	STACKLEAK_ERASE
386 	lctlg	%c1,%c1,__LC_USER_ASCE
387 	BPON
388 	stpt	__LC_EXIT_TIMER
389 .Lpgm_exit_kernel:
390 	mvc	__LC_RETURN_PSW(16),STACK_FRAME_OVERHEAD+__PT_PSW(%r15)
391 	LBEAR	STACK_FRAME_OVERHEAD+__PT_LAST_BREAK(%r15)
392 	lmg	%r0,%r15,STACK_FRAME_OVERHEAD+__PT_R0(%r15)
393 	LPSWEY	__LC_RETURN_PSW,__LC_RETURN_LPSWE
394 
395 #
396 # single stepped system call
397 #
398 .Lpgm_svcper:
399 	mvc	__LC_RETURN_PSW(8),__LC_SVC_NEW_PSW
400 	larl	%r14,.Lsysc_per
401 	stg	%r14,__LC_RETURN_PSW+8
402 	lghi	%r14,1
403 	LBEAR	__LC_PGM_LAST_BREAK
404 	LPSWEY	__LC_RETURN_PSW,__LC_RETURN_LPSWE # branch to .Lsysc_per
405 SYM_CODE_END(pgm_check_handler)
406 
407 /*
408  * Interrupt handler macro used for external and IO interrupts.
409  */
410 .macro INT_HANDLER name,lc_old_psw,handler
411 SYM_CODE_START(\name)
412 	stckf	__LC_INT_CLOCK
413 	stpt	__LC_SYS_ENTER_TIMER
414 	STBEAR	__LC_LAST_BREAK
415 	BPOFF
416 	stmg	%r8,%r15,__LC_SAVE_AREA_ASYNC
417 	lmg	%r8,%r9,\lc_old_psw
418 	tmhh	%r8,0x0001			# interrupting from user ?
419 	jnz	1f
420 #if IS_ENABLED(CONFIG_KVM)
421 	OUTSIDE	%r9,.Lsie_gmap,.Lsie_done,0f
422 	BPENTER	__SF_SIE_FLAGS(%r15),_TIF_ISOLATE_BP_GUEST
423 	SIEEXIT
424 #endif
425 0:	CHECK_STACK __LC_SAVE_AREA_ASYNC
426 	aghi	%r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
427 	j	2f
428 1:	lctlg	%c1,%c1,__LC_KERNEL_ASCE
429 	lg	%r15,__LC_KERNEL_STACK
430 2:	xc	__SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
431 	la	%r11,STACK_FRAME_OVERHEAD(%r15)
432 	stmg	%r0,%r7,__PT_R0(%r11)
433 	# clear user controlled registers to prevent speculative use
434 	xgr	%r0,%r0
435 	xgr	%r1,%r1
436 	xgr	%r3,%r3
437 	xgr	%r4,%r4
438 	xgr	%r5,%r5
439 	xgr	%r6,%r6
440 	xgr	%r7,%r7
441 	xgr	%r10,%r10
442 	xc	__PT_FLAGS(8,%r11),__PT_FLAGS(%r11)
443 	mvc	__PT_R8(64,%r11),__LC_SAVE_AREA_ASYNC
444 	MBEAR	%r11
445 	stmg	%r8,%r9,__PT_PSW(%r11)
446 	lgr	%r2,%r11		# pass pointer to pt_regs
447 	brasl	%r14,\handler
448 	mvc	__LC_RETURN_PSW(16),__PT_PSW(%r11)
449 	tmhh	%r8,0x0001		# returning to user ?
450 	jno	2f
451 	STACKLEAK_ERASE
452 	lctlg	%c1,%c1,__LC_USER_ASCE
453 	BPON
454 	stpt	__LC_EXIT_TIMER
455 2:	LBEAR	__PT_LAST_BREAK(%r11)
456 	lmg	%r0,%r15,__PT_R0(%r11)
457 	LPSWEY	__LC_RETURN_PSW,__LC_RETURN_LPSWE
458 SYM_CODE_END(\name)
459 .endm
460 
461 	.section .irqentry.text, "ax"
462 
463 INT_HANDLER ext_int_handler,__LC_EXT_OLD_PSW,do_ext_irq
464 INT_HANDLER io_int_handler,__LC_IO_OLD_PSW,do_io_irq
465 
466 	.section .kprobes.text, "ax"
467 
468 /*
469  * Load idle PSW.
470  */
471 SYM_FUNC_START(psw_idle)
472 	stg	%r14,(__SF_GPRS+8*8)(%r15)
473 	stg	%r3,__SF_EMPTY(%r15)
474 	larl	%r1,psw_idle_exit
475 	stg	%r1,__SF_EMPTY+8(%r15)
476 	larl	%r1,smp_cpu_mtid
477 	llgf	%r1,0(%r1)
478 	ltgr	%r1,%r1
479 	jz	.Lpsw_idle_stcctm
480 	.insn	rsy,0xeb0000000017,%r1,5,__MT_CYCLES_ENTER(%r2)
481 .Lpsw_idle_stcctm:
482 	oi	__LC_CPU_FLAGS+7,_CIF_ENABLED_WAIT
483 	BPON
484 	stckf	__CLOCK_IDLE_ENTER(%r2)
485 	stpt	__TIMER_IDLE_ENTER(%r2)
486 	lpswe	__SF_EMPTY(%r15)
487 SYM_INNER_LABEL(psw_idle_exit, SYM_L_GLOBAL)
488 	BR_EX	%r14
489 SYM_FUNC_END(psw_idle)
490 
491 /*
492  * Machine check handler routines
493  */
494 SYM_CODE_START(mcck_int_handler)
495 	BPOFF
496 	la	%r1,4095		# validate r1
497 	spt	__LC_CPU_TIMER_SAVE_AREA-4095(%r1)	# validate cpu timer
498 	LBEAR	__LC_LAST_BREAK_SAVE_AREA-4095(%r1)		# validate bear
499 	lmg	%r0,%r15,__LC_GPREGS_SAVE_AREA		# validate gprs
500 	lmg	%r8,%r9,__LC_MCK_OLD_PSW
501 	TSTMSK	__LC_MCCK_CODE,MCCK_CODE_SYSTEM_DAMAGE
502 	jo	.Lmcck_panic		# yes -> rest of mcck code invalid
503 	TSTMSK	__LC_MCCK_CODE,MCCK_CODE_CR_VALID
504 	jno	.Lmcck_panic		# control registers invalid -> panic
505 	lctlg	%c0,%c15,__LC_CREGS_SAVE_AREA		# validate ctl regs
506 	ptlb
507 	lghi	%r14,__LC_CPU_TIMER_SAVE_AREA
508 	mvc	__LC_MCCK_ENTER_TIMER(8),0(%r14)
509 	TSTMSK	__LC_MCCK_CODE,MCCK_CODE_CPU_TIMER_VALID
510 	jo	3f
511 	la	%r14,__LC_SYS_ENTER_TIMER
512 	clc	0(8,%r14),__LC_EXIT_TIMER
513 	jl	1f
514 	la	%r14,__LC_EXIT_TIMER
515 1:	clc	0(8,%r14),__LC_LAST_UPDATE_TIMER
516 	jl	2f
517 	la	%r14,__LC_LAST_UPDATE_TIMER
518 2:	spt	0(%r14)
519 	mvc	__LC_MCCK_ENTER_TIMER(8),0(%r14)
520 3:	TSTMSK	__LC_MCCK_CODE,MCCK_CODE_PSW_MWP_VALID
521 	jno	.Lmcck_panic
522 	tmhh	%r8,0x0001		# interrupting from user ?
523 	jnz	.Lmcck_user
524 	TSTMSK	__LC_MCCK_CODE,MCCK_CODE_PSW_IA_VALID
525 	jno	.Lmcck_panic
526 #if IS_ENABLED(CONFIG_KVM)
527 	OUTSIDE	%r9,.Lsie_gmap,.Lsie_done,.Lmcck_user
528 	OUTSIDE	%r9,.Lsie_entry,.Lsie_leave,4f
529 	oi	__LC_CPU_FLAGS+7, _CIF_MCCK_GUEST
530 4:	BPENTER	__SF_SIE_FLAGS(%r15),_TIF_ISOLATE_BP_GUEST
531 	SIEEXIT
532 #endif
533 .Lmcck_user:
534 	lg	%r15,__LC_MCCK_STACK
535 	la	%r11,STACK_FRAME_OVERHEAD(%r15)
536 	stctg	%c1,%c1,__PT_CR1(%r11)
537 	lctlg	%c1,%c1,__LC_KERNEL_ASCE
538 	xc	__SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
539 	lghi	%r14,__LC_GPREGS_SAVE_AREA+64
540 	stmg	%r0,%r7,__PT_R0(%r11)
541 	# clear user controlled registers to prevent speculative use
542 	xgr	%r0,%r0
543 	xgr	%r1,%r1
544 	xgr	%r3,%r3
545 	xgr	%r4,%r4
546 	xgr	%r5,%r5
547 	xgr	%r6,%r6
548 	xgr	%r7,%r7
549 	xgr	%r10,%r10
550 	mvc	__PT_R8(64,%r11),0(%r14)
551 	stmg	%r8,%r9,__PT_PSW(%r11)
552 	xc	__PT_FLAGS(8,%r11),__PT_FLAGS(%r11)
553 	xc	__SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
554 	lgr	%r2,%r11		# pass pointer to pt_regs
555 	brasl	%r14,s390_do_machine_check
556 	lctlg	%c1,%c1,__PT_CR1(%r11)
557 	lmg	%r0,%r10,__PT_R0(%r11)
558 	mvc	__LC_RETURN_MCCK_PSW(16),__PT_PSW(%r11) # move return PSW
559 	tm	__LC_RETURN_MCCK_PSW+1,0x01 # returning to user ?
560 	jno	0f
561 	BPON
562 	stpt	__LC_EXIT_TIMER
563 0:	ALTERNATIVE "nop", __stringify(lghi %r12,__LC_LAST_BREAK_SAVE_AREA),193
564 	LBEAR	0(%r12)
565 	lmg	%r11,%r15,__PT_R11(%r11)
566 	LPSWEY	__LC_RETURN_MCCK_PSW,__LC_RETURN_MCCK_LPSWE
567 
568 .Lmcck_panic:
569 	/*
570 	 * Iterate over all possible CPU addresses in the range 0..0xffff
571 	 * and stop each CPU using signal processor. Use compare and swap
572 	 * to allow just one CPU-stopper and prevent concurrent CPUs from
573 	 * stopping each other while leaving the others running.
574 	 */
575 	lhi	%r5,0
576 	lhi	%r6,1
577 	larl	%r7,stop_lock
578 	cs	%r5,%r6,0(%r7)		# single CPU-stopper only
579 	jnz	4f
580 	larl	%r7,this_cpu
581 	stap	0(%r7)			# this CPU address
582 	lh	%r4,0(%r7)
583 	nilh	%r4,0
584 	lhi	%r0,1
585 	sll	%r0,16			# CPU counter
586 	lhi	%r3,0			# next CPU address
587 0:	cr	%r3,%r4
588 	je	2f
589 1:	sigp	%r1,%r3,SIGP_STOP	# stop next CPU
590 	brc	SIGP_CC_BUSY,1b
591 2:	ahi	%r3,1
592 	brct	%r0,0b
593 3:	sigp	%r1,%r4,SIGP_STOP	# stop this CPU
594 	brc	SIGP_CC_BUSY,3b
595 4:	j	4b
596 SYM_CODE_END(mcck_int_handler)
597 
598 SYM_CODE_START(restart_int_handler)
599 	ALTERNATIVE "nop", "lpp _LPP_OFFSET", 40
600 	stg	%r15,__LC_SAVE_AREA_RESTART
601 	TSTMSK	__LC_RESTART_FLAGS,RESTART_FLAG_CTLREGS,4
602 	jz	0f
603 	lctlg	%c0,%c15,__LC_CREGS_SAVE_AREA
604 0:	larl	%r15,daton_psw
605 	lpswe	0(%r15)				# turn dat on, keep irqs off
606 .Ldaton:
607 	lg	%r15,__LC_RESTART_STACK
608 	xc	STACK_FRAME_OVERHEAD(__PT_SIZE,%r15),STACK_FRAME_OVERHEAD(%r15)
609 	stmg	%r0,%r14,STACK_FRAME_OVERHEAD+__PT_R0(%r15)
610 	mvc	STACK_FRAME_OVERHEAD+__PT_R15(8,%r15),__LC_SAVE_AREA_RESTART
611 	mvc	STACK_FRAME_OVERHEAD+__PT_PSW(16,%r15),__LC_RST_OLD_PSW
612 	xc	0(STACK_FRAME_OVERHEAD,%r15),0(%r15)
613 	lg	%r1,__LC_RESTART_FN		# load fn, parm & source cpu
614 	lg	%r2,__LC_RESTART_DATA
615 	lgf	%r3,__LC_RESTART_SOURCE
616 	ltgr	%r3,%r3				# test source cpu address
617 	jm	1f				# negative -> skip source stop
618 0:	sigp	%r4,%r3,SIGP_SENSE		# sigp sense to source cpu
619 	brc	10,0b				# wait for status stored
620 1:	basr	%r14,%r1			# call function
621 	stap	__SF_EMPTY(%r15)		# store cpu address
622 	llgh	%r3,__SF_EMPTY(%r15)
623 2:	sigp	%r4,%r3,SIGP_STOP		# sigp stop to current cpu
624 	brc	2,2b
625 3:	j	3b
626 SYM_CODE_END(restart_int_handler)
627 
628 	.section .kprobes.text, "ax"
629 
630 #if defined(CONFIG_CHECK_STACK) || defined(CONFIG_VMAP_STACK)
631 /*
632  * The synchronous or the asynchronous stack overflowed. We are dead.
633  * No need to properly save the registers, we are going to panic anyway.
634  * Setup a pt_regs so that show_trace can provide a good call trace.
635  */
636 SYM_CODE_START(stack_overflow)
637 	lg	%r15,__LC_NODAT_STACK	# change to panic stack
638 	la	%r11,STACK_FRAME_OVERHEAD(%r15)
639 	stmg	%r0,%r7,__PT_R0(%r11)
640 	stmg	%r8,%r9,__PT_PSW(%r11)
641 	mvc	__PT_R8(64,%r11),0(%r14)
642 	stg	%r10,__PT_ORIG_GPR2(%r11) # store last break to orig_gpr2
643 	xc	__SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
644 	lgr	%r2,%r11		# pass pointer to pt_regs
645 	jg	kernel_stack_overflow
646 SYM_CODE_END(stack_overflow)
647 #endif
648 
649 	.section .data, "aw"
650 	.balign	4
651 SYM_DATA_LOCAL(stop_lock,	.long 0)
652 SYM_DATA_LOCAL(this_cpu,	.short 0)
653 	.balign	8
654 SYM_DATA_START_LOCAL(daton_psw)
655 	.quad	PSW_KERNEL_BITS
656 	.quad	.Ldaton
657 SYM_DATA_END(daton_psw)
658 
659 	.section .rodata, "a"
660 	.balign	8
661 #define SYSCALL(esame,emu)	.quad __s390x_ ## esame
662 SYM_DATA_START(sys_call_table)
663 #include "asm/syscall_table.h"
664 SYM_DATA_END(sys_call_table)
665 #undef SYSCALL
666 
667 #ifdef CONFIG_COMPAT
668 
669 #define SYSCALL(esame,emu)	.quad __s390_ ## emu
670 SYM_DATA_START(sys_call_table_emu)
671 #include "asm/syscall_table.h"
672 SYM_DATA_END(sys_call_table_emu)
673 #undef SYSCALL
674 #endif
675