1/* SPDX-License-Identifier: GPL-2.0 */ 2/* 3 * S390 low-level entry points. 4 * 5 * Copyright IBM Corp. 1999, 2012 6 * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com), 7 * Hartmut Penner (hp@de.ibm.com), 8 * Denis Joseph Barrow (djbarrow@de.ibm.com,barrow_dj@yahoo.com), 9 * Heiko Carstens <heiko.carstens@de.ibm.com> 10 */ 11 12#include <linux/init.h> 13#include <linux/linkage.h> 14#include <asm/alternative-asm.h> 15#include <asm/processor.h> 16#include <asm/cache.h> 17#include <asm/ctl_reg.h> 18#include <asm/dwarf.h> 19#include <asm/errno.h> 20#include <asm/ptrace.h> 21#include <asm/thread_info.h> 22#include <asm/asm-offsets.h> 23#include <asm/unistd.h> 24#include <asm/page.h> 25#include <asm/sigp.h> 26#include <asm/irq.h> 27#include <asm/vx-insn.h> 28#include <asm/setup.h> 29#include <asm/nmi.h> 30#include <asm/export.h> 31#include <asm/nospec-insn.h> 32 33__PT_R0 = __PT_GPRS 34__PT_R1 = __PT_GPRS + 8 35__PT_R2 = __PT_GPRS + 16 36__PT_R3 = __PT_GPRS + 24 37__PT_R4 = __PT_GPRS + 32 38__PT_R5 = __PT_GPRS + 40 39__PT_R6 = __PT_GPRS + 48 40__PT_R7 = __PT_GPRS + 56 41__PT_R8 = __PT_GPRS + 64 42__PT_R9 = __PT_GPRS + 72 43__PT_R10 = __PT_GPRS + 80 44__PT_R11 = __PT_GPRS + 88 45__PT_R12 = __PT_GPRS + 96 46__PT_R13 = __PT_GPRS + 104 47__PT_R14 = __PT_GPRS + 112 48__PT_R15 = __PT_GPRS + 120 49 50STACK_SHIFT = PAGE_SHIFT + THREAD_SIZE_ORDER 51STACK_SIZE = 1 << STACK_SHIFT 52STACK_INIT = STACK_SIZE - STACK_FRAME_OVERHEAD - __PT_SIZE 53 54_TIF_WORK = (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_NEED_RESCHED | \ 55 _TIF_UPROBE | _TIF_GUARDED_STORAGE | _TIF_PATCH_PENDING | \ 56 _TIF_NOTIFY_SIGNAL) 57_TIF_TRACE = (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | _TIF_SECCOMP | \ 58 _TIF_SYSCALL_TRACEPOINT) 59_CIF_WORK = (_CIF_FPU) 60_PIF_WORK = (_PIF_PER_TRAP | _PIF_SYSCALL_RESTART) 61 62_LPP_OFFSET = __LC_LPP 63 64 .macro TRACE_IRQS_ON 65#ifdef CONFIG_TRACE_IRQFLAGS 66 basr %r2,%r0 67 brasl %r14,trace_hardirqs_on_caller 68#endif 69 .endm 70 71 .macro TRACE_IRQS_OFF 72#ifdef CONFIG_TRACE_IRQFLAGS 73 basr %r2,%r0 74 brasl %r14,trace_hardirqs_off_caller 75#endif 76 .endm 77 78 .macro LOCKDEP_SYS_EXIT 79#ifdef CONFIG_LOCKDEP 80 tm __PT_PSW+1(%r11),0x01 # returning to user ? 81 jz .+10 82 brasl %r14,lockdep_sys_exit 83#endif 84 .endm 85 86 .macro CHECK_STACK savearea 87#ifdef CONFIG_CHECK_STACK 88 tml %r15,STACK_SIZE - CONFIG_STACK_GUARD 89 lghi %r14,\savearea 90 jz stack_overflow 91#endif 92 .endm 93 94 .macro DEBUG_USER_ASCE 95#ifdef CONFIG_DEBUG_USER_ASCE 96 brasl %r14,debug_user_asce 97#endif 98 .endm 99 100 .macro CHECK_VMAP_STACK savearea,oklabel 101#ifdef CONFIG_VMAP_STACK 102 lgr %r14,%r15 103 nill %r14,0x10000 - STACK_SIZE 104 oill %r14,STACK_INIT 105 clg %r14,__LC_KERNEL_STACK 106 je \oklabel 107 clg %r14,__LC_ASYNC_STACK 108 je \oklabel 109 clg %r14,__LC_NODAT_STACK 110 je \oklabel 111 clg %r14,__LC_RESTART_STACK 112 je \oklabel 113 lghi %r14,\savearea 114 j stack_overflow 115#else 116 j \oklabel 117#endif 118 .endm 119 120 .macro SWITCH_ASYNC savearea,timer,clock 121 tmhh %r8,0x0001 # interrupting from user ? 122 jnz 4f 123#if IS_ENABLED(CONFIG_KVM) 124 lgr %r14,%r9 125 larl %r13,.Lsie_gmap 126 slgr %r14,%r13 127 lghi %r13,.Lsie_done - .Lsie_gmap 128 clgr %r14,%r13 129 jhe 0f 130 lghi %r11,\savearea # inside critical section, do cleanup 131 brasl %r14,.Lcleanup_sie 132#endif 1330: larl %r13,.Lpsw_idle_exit 134 cgr %r13,%r9 135 jne 3f 136 137 larl %r1,smp_cpu_mtid 138 llgf %r1,0(%r1) 139 ltgr %r1,%r1 140 jz 2f # no SMT, skip mt_cycles calculation 141 .insn rsy,0xeb0000000017,%r1,5,__SF_EMPTY+80(%r15) 142 larl %r3,mt_cycles 143 ag %r3,__LC_PERCPU_OFFSET 144 la %r4,__SF_EMPTY+16(%r15) 1451: lg %r0,0(%r3) 146 slg %r0,0(%r4) 147 alg %r0,64(%r4) 148 stg %r0,0(%r3) 149 la %r3,8(%r3) 150 la %r4,8(%r4) 151 brct %r1,1b 152 1532: mvc __CLOCK_IDLE_EXIT(8,%r2), \clock 154 mvc __TIMER_IDLE_EXIT(8,%r2), \timer 155 # account system time going idle 156 ni __LC_CPU_FLAGS+7,255-_CIF_ENABLED_WAIT 157 158 lg %r13,__LC_STEAL_TIMER 159 alg %r13,__CLOCK_IDLE_ENTER(%r2) 160 slg %r13,__LC_LAST_UPDATE_CLOCK 161 stg %r13,__LC_STEAL_TIMER 162 163 mvc __LC_LAST_UPDATE_CLOCK(8),__CLOCK_IDLE_EXIT(%r2) 164 165 lg %r13,__LC_SYSTEM_TIMER 166 alg %r13,__LC_LAST_UPDATE_TIMER 167 slg %r13,__TIMER_IDLE_ENTER(%r2) 168 stg %r13,__LC_SYSTEM_TIMER 169 mvc __LC_LAST_UPDATE_TIMER(8),__TIMER_IDLE_EXIT(%r2) 170 171 nihh %r8,0xfcfd # clear wait state and irq bits 1723: lg %r14,__LC_ASYNC_STACK # are we already on the target stack? 173 slgr %r14,%r15 174 srag %r14,%r14,STACK_SHIFT 175 jnz 5f 176 CHECK_STACK \savearea 177 aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE) 178 j 6f 1794: UPDATE_VTIME %r14,%r15,\timer 180 BPENTER __TI_flags(%r12),_TIF_ISOLATE_BP 1815: lg %r15,__LC_ASYNC_STACK # load async stack 1826: la %r11,STACK_FRAME_OVERHEAD(%r15) 183 .endm 184 185 .macro UPDATE_VTIME w1,w2,enter_timer 186 lg \w1,__LC_EXIT_TIMER 187 lg \w2,__LC_LAST_UPDATE_TIMER 188 slg \w1,\enter_timer 189 slg \w2,__LC_EXIT_TIMER 190 alg \w1,__LC_USER_TIMER 191 alg \w2,__LC_SYSTEM_TIMER 192 stg \w1,__LC_USER_TIMER 193 stg \w2,__LC_SYSTEM_TIMER 194 mvc __LC_LAST_UPDATE_TIMER(8),\enter_timer 195 .endm 196 197 .macro RESTORE_SM_CLEAR_PER 198 stg %r8,__LC_RETURN_PSW 199 ni __LC_RETURN_PSW,0xbf 200 ssm __LC_RETURN_PSW 201 .endm 202 203 .macro ENABLE_INTS 204 stosm __SF_EMPTY(%r15),3 205 .endm 206 207 .macro ENABLE_INTS_TRACE 208 TRACE_IRQS_ON 209 ENABLE_INTS 210 .endm 211 212 .macro DISABLE_INTS 213 stnsm __SF_EMPTY(%r15),0xfc 214 .endm 215 216 .macro DISABLE_INTS_TRACE 217 DISABLE_INTS 218 TRACE_IRQS_OFF 219 .endm 220 221 .macro STCK savearea 222#ifdef CONFIG_HAVE_MARCH_Z9_109_FEATURES 223 .insn s,0xb27c0000,\savearea # store clock fast 224#else 225 .insn s,0xb2050000,\savearea # store clock 226#endif 227 .endm 228 229 /* 230 * The TSTMSK macro generates a test-under-mask instruction by 231 * calculating the memory offset for the specified mask value. 232 * Mask value can be any constant. The macro shifts the mask 233 * value to calculate the memory offset for the test-under-mask 234 * instruction. 235 */ 236 .macro TSTMSK addr, mask, size=8, bytepos=0 237 .if (\bytepos < \size) && (\mask >> 8) 238 .if (\mask & 0xff) 239 .error "Mask exceeds byte boundary" 240 .endif 241 TSTMSK \addr, "(\mask >> 8)", \size, "(\bytepos + 1)" 242 .exitm 243 .endif 244 .ifeq \mask 245 .error "Mask must not be zero" 246 .endif 247 off = \size - \bytepos - 1 248 tm off+\addr, \mask 249 .endm 250 251 .macro BPOFF 252 ALTERNATIVE "", ".long 0xb2e8c000", 82 253 .endm 254 255 .macro BPON 256 ALTERNATIVE "", ".long 0xb2e8d000", 82 257 .endm 258 259 .macro BPENTER tif_ptr,tif_mask 260 ALTERNATIVE "TSTMSK \tif_ptr,\tif_mask; jz .+8; .long 0xb2e8d000", \ 261 "", 82 262 .endm 263 264 .macro BPEXIT tif_ptr,tif_mask 265 TSTMSK \tif_ptr,\tif_mask 266 ALTERNATIVE "jz .+8; .long 0xb2e8c000", \ 267 "jnz .+8; .long 0xb2e8d000", 82 268 .endm 269 270 GEN_BR_THUNK %r9 271 GEN_BR_THUNK %r14 272 GEN_BR_THUNK %r14,%r11 273 274 .section .kprobes.text, "ax" 275.Ldummy: 276 /* 277 * This nop exists only in order to avoid that __switch_to starts at 278 * the beginning of the kprobes text section. In that case we would 279 * have several symbols at the same address. E.g. objdump would take 280 * an arbitrary symbol name when disassembling this code. 281 * With the added nop in between the __switch_to symbol is unique 282 * again. 283 */ 284 nop 0 285 286ENTRY(__bpon) 287 .globl __bpon 288 BPON 289 BR_EX %r14 290ENDPROC(__bpon) 291 292/* 293 * Scheduler resume function, called by switch_to 294 * gpr2 = (task_struct *) prev 295 * gpr3 = (task_struct *) next 296 * Returns: 297 * gpr2 = prev 298 */ 299ENTRY(__switch_to) 300 stmg %r6,%r15,__SF_GPRS(%r15) # store gprs of prev task 301 lghi %r4,__TASK_stack 302 lghi %r1,__TASK_thread 303 llill %r5,STACK_INIT 304 stg %r15,__THREAD_ksp(%r1,%r2) # store kernel stack of prev 305 lg %r15,0(%r4,%r3) # start of kernel stack of next 306 agr %r15,%r5 # end of kernel stack of next 307 stg %r3,__LC_CURRENT # store task struct of next 308 stg %r15,__LC_KERNEL_STACK # store end of kernel stack 309 lg %r15,__THREAD_ksp(%r1,%r3) # load kernel stack of next 310 aghi %r3,__TASK_pid 311 mvc __LC_CURRENT_PID(4,%r0),0(%r3) # store pid of next 312 lmg %r6,%r15,__SF_GPRS(%r15) # load gprs of next task 313 ALTERNATIVE "", ".insn s,0xb2800000,_LPP_OFFSET", 40 314 BR_EX %r14 315ENDPROC(__switch_to) 316 317#if IS_ENABLED(CONFIG_KVM) 318/* 319 * sie64a calling convention: 320 * %r2 pointer to sie control block 321 * %r3 guest register save area 322 */ 323ENTRY(sie64a) 324 stmg %r6,%r14,__SF_GPRS(%r15) # save kernel registers 325 lg %r12,__LC_CURRENT 326 stg %r2,__SF_SIE_CONTROL(%r15) # save control block pointer 327 stg %r3,__SF_SIE_SAVEAREA(%r15) # save guest register save area 328 xc __SF_SIE_REASON(8,%r15),__SF_SIE_REASON(%r15) # reason code = 0 329 mvc __SF_SIE_FLAGS(8,%r15),__TI_flags(%r12) # copy thread flags 330 TSTMSK __LC_CPU_FLAGS,_CIF_FPU # load guest fp/vx registers ? 331 jno .Lsie_load_guest_gprs 332 brasl %r14,load_fpu_regs # load guest fp/vx regs 333.Lsie_load_guest_gprs: 334 lmg %r0,%r13,0(%r3) # load guest gprs 0-13 335 lg %r14,__LC_GMAP # get gmap pointer 336 ltgr %r14,%r14 337 jz .Lsie_gmap 338 lctlg %c1,%c1,__GMAP_ASCE(%r14) # load primary asce 339.Lsie_gmap: 340 lg %r14,__SF_SIE_CONTROL(%r15) # get control block pointer 341 oi __SIE_PROG0C+3(%r14),1 # we are going into SIE now 342 tm __SIE_PROG20+3(%r14),3 # last exit... 343 jnz .Lsie_skip 344 TSTMSK __LC_CPU_FLAGS,_CIF_FPU 345 jo .Lsie_skip # exit if fp/vx regs changed 346 BPEXIT __SF_SIE_FLAGS(%r15),(_TIF_ISOLATE_BP|_TIF_ISOLATE_BP_GUEST) 347.Lsie_entry: 348 sie 0(%r14) 349 BPOFF 350 BPENTER __SF_SIE_FLAGS(%r15),(_TIF_ISOLATE_BP|_TIF_ISOLATE_BP_GUEST) 351.Lsie_skip: 352 ni __SIE_PROG0C+3(%r14),0xfe # no longer in SIE 353 lctlg %c1,%c1,__LC_KERNEL_ASCE # load primary asce 354.Lsie_done: 355# some program checks are suppressing. C code (e.g. do_protection_exception) 356# will rewind the PSW by the ILC, which is often 4 bytes in case of SIE. There 357# are some corner cases (e.g. runtime instrumentation) where ILC is unpredictable. 358# Other instructions between sie64a and .Lsie_done should not cause program 359# interrupts. So lets use 3 nops as a landing pad for all possible rewinds. 360# See also .Lcleanup_sie 361.Lrewind_pad6: 362 nopr 7 363.Lrewind_pad4: 364 nopr 7 365.Lrewind_pad2: 366 nopr 7 367 .globl sie_exit 368sie_exit: 369 lg %r14,__SF_SIE_SAVEAREA(%r15) # load guest register save area 370 stmg %r0,%r13,0(%r14) # save guest gprs 0-13 371 xgr %r0,%r0 # clear guest registers to 372 xgr %r1,%r1 # prevent speculative use 373 xgr %r2,%r2 374 xgr %r3,%r3 375 xgr %r4,%r4 376 xgr %r5,%r5 377 lmg %r6,%r14,__SF_GPRS(%r15) # restore kernel registers 378 lg %r2,__SF_SIE_REASON(%r15) # return exit reason code 379 BR_EX %r14 380.Lsie_fault: 381 lghi %r14,-EFAULT 382 stg %r14,__SF_SIE_REASON(%r15) # set exit reason code 383 j sie_exit 384 385 EX_TABLE(.Lrewind_pad6,.Lsie_fault) 386 EX_TABLE(.Lrewind_pad4,.Lsie_fault) 387 EX_TABLE(.Lrewind_pad2,.Lsie_fault) 388 EX_TABLE(sie_exit,.Lsie_fault) 389ENDPROC(sie64a) 390EXPORT_SYMBOL(sie64a) 391EXPORT_SYMBOL(sie_exit) 392#endif 393 394/* 395 * SVC interrupt handler routine. System calls are synchronous events and 396 * are entered with interrupts disabled. 397 */ 398 399ENTRY(system_call) 400 stpt __LC_SYNC_ENTER_TIMER 401 stmg %r8,%r15,__LC_SAVE_AREA_SYNC 402 BPOFF 403 lg %r12,__LC_CURRENT 404 lghi %r14,_PIF_SYSCALL 405.Lsysc_per: 406 lctlg %c1,%c1,__LC_KERNEL_ASCE 407 lghi %r13,__TASK_thread 408 lg %r15,__LC_KERNEL_STACK 409 la %r11,STACK_FRAME_OVERHEAD(%r15) # pointer to pt_regs 410 UPDATE_VTIME %r8,%r9,__LC_SYNC_ENTER_TIMER 411 BPENTER __TI_flags(%r12),_TIF_ISOLATE_BP 412 stmg %r0,%r7,__PT_R0(%r11) 413 mvc __PT_R8(64,%r11),__LC_SAVE_AREA_SYNC 414 mvc __PT_PSW(16,%r11),__LC_SVC_OLD_PSW 415 mvc __PT_INT_CODE(4,%r11),__LC_SVC_ILC 416 stg %r14,__PT_FLAGS(%r11) 417 ENABLE_INTS 418.Lsysc_do_svc: 419 # clear user controlled register to prevent speculative use 420 xgr %r0,%r0 421 # load address of system call table 422 lg %r10,__THREAD_sysc_table(%r13,%r12) 423 llgh %r8,__PT_INT_CODE+2(%r11) 424 slag %r8,%r8,3 # shift and test for svc 0 425 jnz .Lsysc_nr_ok 426 # svc 0: system call number in %r1 427 llgfr %r1,%r1 # clear high word in r1 428 sth %r1,__PT_INT_CODE+2(%r11) 429 cghi %r1,NR_syscalls 430 jnl .Lsysc_nr_ok 431 slag %r8,%r1,3 432.Lsysc_nr_ok: 433 xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) 434 stg %r2,__PT_ORIG_GPR2(%r11) 435 stg %r7,STACK_FRAME_OVERHEAD(%r15) 436 lg %r9,0(%r8,%r10) # get system call add. 437 TSTMSK __TI_flags(%r12),_TIF_TRACE 438 jnz .Lsysc_tracesys 439 BASR_EX %r14,%r9 # call sys_xxxx 440 stg %r2,__PT_R2(%r11) # store return value 441 442.Lsysc_return: 443#ifdef CONFIG_DEBUG_RSEQ 444 lgr %r2,%r11 445 brasl %r14,rseq_syscall 446#endif 447 LOCKDEP_SYS_EXIT 448.Lsysc_tif: 449 DISABLE_INTS 450 TSTMSK __PT_FLAGS(%r11),_PIF_WORK 451 jnz .Lsysc_work 452 TSTMSK __TI_flags(%r12),_TIF_WORK 453 jnz .Lsysc_work # check for work 454 DEBUG_USER_ASCE 455 lctlg %c1,%c1,__LC_USER_ASCE 456 BPEXIT __TI_flags(%r12),_TIF_ISOLATE_BP 457 TSTMSK __LC_CPU_FLAGS, _CIF_FPU 458 jz .Lsysc_skip_fpu 459 brasl %r14,load_fpu_regs 460.Lsysc_skip_fpu: 461 mvc __LC_RETURN_PSW(16),__PT_PSW(%r11) 462 stpt __LC_EXIT_TIMER 463 lmg %r0,%r15,__PT_R0(%r11) 464 b __LC_RETURN_LPSWE 465 466# 467# One of the work bits is on. Find out which one. 468# 469.Lsysc_work: 470 ENABLE_INTS 471 TSTMSK __TI_flags(%r12),_TIF_NEED_RESCHED 472 jo .Lsysc_reschedule 473 TSTMSK __PT_FLAGS(%r11),_PIF_SYSCALL_RESTART 474 jo .Lsysc_syscall_restart 475#ifdef CONFIG_UPROBES 476 TSTMSK __TI_flags(%r12),_TIF_UPROBE 477 jo .Lsysc_uprobe_notify 478#endif 479 TSTMSK __TI_flags(%r12),_TIF_GUARDED_STORAGE 480 jo .Lsysc_guarded_storage 481 TSTMSK __PT_FLAGS(%r11),_PIF_PER_TRAP 482 jo .Lsysc_singlestep 483#ifdef CONFIG_LIVEPATCH 484 TSTMSK __TI_flags(%r12),_TIF_PATCH_PENDING 485 jo .Lsysc_patch_pending # handle live patching just before 486 # signals and possible syscall restart 487#endif 488 TSTMSK __PT_FLAGS(%r11),_PIF_SYSCALL_RESTART 489 jo .Lsysc_syscall_restart 490 TSTMSK __TI_flags(%r12),(_TIF_SIGPENDING|_TIF_NOTIFY_SIGNAL) 491 jnz .Lsysc_sigpending 492 TSTMSK __TI_flags(%r12),_TIF_NOTIFY_RESUME 493 jo .Lsysc_notify_resume 494 j .Lsysc_return 495 496# 497# _TIF_NEED_RESCHED is set, call schedule 498# 499.Lsysc_reschedule: 500 larl %r14,.Lsysc_return 501 jg schedule 502 503# 504# _TIF_SIGPENDING is set, call do_signal 505# 506.Lsysc_sigpending: 507 lgr %r2,%r11 # pass pointer to pt_regs 508 brasl %r14,do_signal 509 TSTMSK __PT_FLAGS(%r11),_PIF_SYSCALL 510 jno .Lsysc_return 511.Lsysc_do_syscall: 512 lghi %r13,__TASK_thread 513 lmg %r2,%r7,__PT_R2(%r11) # load svc arguments 514 lghi %r1,0 # svc 0 returns -ENOSYS 515 j .Lsysc_do_svc 516 517# 518# _TIF_NOTIFY_RESUME is set, call do_notify_resume 519# 520.Lsysc_notify_resume: 521 lgr %r2,%r11 # pass pointer to pt_regs 522 larl %r14,.Lsysc_return 523 jg do_notify_resume 524 525# 526# _TIF_UPROBE is set, call uprobe_notify_resume 527# 528#ifdef CONFIG_UPROBES 529.Lsysc_uprobe_notify: 530 lgr %r2,%r11 # pass pointer to pt_regs 531 larl %r14,.Lsysc_return 532 jg uprobe_notify_resume 533#endif 534 535# 536# _TIF_GUARDED_STORAGE is set, call guarded_storage_load 537# 538.Lsysc_guarded_storage: 539 lgr %r2,%r11 # pass pointer to pt_regs 540 larl %r14,.Lsysc_return 541 jg gs_load_bc_cb 542# 543# _TIF_PATCH_PENDING is set, call klp_update_patch_state 544# 545#ifdef CONFIG_LIVEPATCH 546.Lsysc_patch_pending: 547 lg %r2,__LC_CURRENT # pass pointer to task struct 548 larl %r14,.Lsysc_return 549 jg klp_update_patch_state 550#endif 551 552# 553# _PIF_PER_TRAP is set, call do_per_trap 554# 555.Lsysc_singlestep: 556 ni __PT_FLAGS+7(%r11),255-_PIF_PER_TRAP 557 lgr %r2,%r11 # pass pointer to pt_regs 558 larl %r14,.Lsysc_return 559 jg do_per_trap 560 561# 562# _PIF_SYSCALL_RESTART is set, repeat the current system call 563# 564.Lsysc_syscall_restart: 565 ni __PT_FLAGS+7(%r11),255-_PIF_SYSCALL_RESTART 566 lmg %r1,%r7,__PT_R1(%r11) # load svc arguments 567 lg %r2,__PT_ORIG_GPR2(%r11) 568 j .Lsysc_do_svc 569 570# 571# call tracehook_report_syscall_entry/tracehook_report_syscall_exit before 572# and after the system call 573# 574.Lsysc_tracesys: 575 lgr %r2,%r11 # pass pointer to pt_regs 576 la %r3,0 577 llgh %r0,__PT_INT_CODE+2(%r11) 578 stg %r0,__PT_R2(%r11) 579 brasl %r14,do_syscall_trace_enter 580 lghi %r0,NR_syscalls 581 clgr %r0,%r2 582 jnh .Lsysc_tracenogo 583 sllg %r8,%r2,3 584 lg %r9,0(%r8,%r10) 585 lmg %r3,%r7,__PT_R3(%r11) 586 stg %r7,STACK_FRAME_OVERHEAD(%r15) 587 lg %r2,__PT_ORIG_GPR2(%r11) 588 BASR_EX %r14,%r9 # call sys_xxx 589 stg %r2,__PT_R2(%r11) # store return value 590.Lsysc_tracenogo: 591 TSTMSK __TI_flags(%r12),_TIF_TRACE 592 jz .Lsysc_return 593 lgr %r2,%r11 # pass pointer to pt_regs 594 larl %r14,.Lsysc_return 595 jg do_syscall_trace_exit 596ENDPROC(system_call) 597 598# 599# a new process exits the kernel with ret_from_fork 600# 601ENTRY(ret_from_fork) 602 la %r11,STACK_FRAME_OVERHEAD(%r15) 603 lg %r12,__LC_CURRENT 604 brasl %r14,schedule_tail 605 tm __PT_PSW+1(%r11),0x01 # forking a kernel thread ? 606 jne .Lsysc_tracenogo 607 # it's a kernel thread 608 lmg %r9,%r10,__PT_R9(%r11) # load gprs 609 la %r2,0(%r10) 610 BASR_EX %r14,%r9 611 j .Lsysc_tracenogo 612ENDPROC(ret_from_fork) 613 614ENTRY(kernel_thread_starter) 615 la %r2,0(%r10) 616 BASR_EX %r14,%r9 617 j .Lsysc_tracenogo 618ENDPROC(kernel_thread_starter) 619 620/* 621 * Program check handler routine 622 */ 623 624ENTRY(pgm_check_handler) 625 stpt __LC_SYNC_ENTER_TIMER 626 BPOFF 627 stmg %r8,%r15,__LC_SAVE_AREA_SYNC 628 lg %r10,__LC_LAST_BREAK 629 srag %r11,%r10,12 630 jnz 0f 631 /* if __LC_LAST_BREAK is < 4096, it contains one of 632 * the lpswe addresses in lowcore. Set it to 1 (initial state) 633 * to prevent leaking that address to userspace. 634 */ 635 lghi %r10,1 6360: lg %r12,__LC_CURRENT 637 lghi %r11,0 638 lmg %r8,%r9,__LC_PGM_OLD_PSW 639 tmhh %r8,0x0001 # coming from user space? 640 jno .Lpgm_skip_asce 641 lctlg %c1,%c1,__LC_KERNEL_ASCE 642 j 3f 643.Lpgm_skip_asce: 644#if IS_ENABLED(CONFIG_KVM) 645 # cleanup critical section for program checks in sie64a 646 lgr %r14,%r9 647 larl %r13,.Lsie_gmap 648 slgr %r14,%r13 649 lghi %r13,.Lsie_done - .Lsie_gmap 650 clgr %r14,%r13 651 jhe 1f 652 lg %r14,__SF_SIE_CONTROL(%r15) # get control block pointer 653 ni __SIE_PROG0C+3(%r14),0xfe # no longer in SIE 654 lctlg %c1,%c1,__LC_KERNEL_ASCE # load primary asce 655 larl %r9,sie_exit # skip forward to sie_exit 656 lghi %r11,_PIF_GUEST_FAULT 657#endif 6581: tmhh %r8,0x4000 # PER bit set in old PSW ? 659 jnz 2f # -> enabled, can't be a double fault 660 tm __LC_PGM_ILC+3,0x80 # check for per exception 661 jnz .Lpgm_svcper # -> single stepped svc 6622: CHECK_STACK __LC_SAVE_AREA_SYNC 663 aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE) 664 # CHECK_VMAP_STACK branches to stack_overflow or 5f 665 CHECK_VMAP_STACK __LC_SAVE_AREA_SYNC,5f 6663: UPDATE_VTIME %r14,%r15,__LC_SYNC_ENTER_TIMER 667 BPENTER __TI_flags(%r12),_TIF_ISOLATE_BP 668 lg %r15,__LC_KERNEL_STACK 669 lgr %r14,%r12 670 aghi %r14,__TASK_thread # pointer to thread_struct 671 lghi %r13,__LC_PGM_TDB 672 tm __LC_PGM_ILC+2,0x02 # check for transaction abort 673 jz 4f 674 mvc __THREAD_trap_tdb(256,%r14),0(%r13) 6754: stg %r10,__THREAD_last_break(%r14) 6765: lgr %r13,%r11 677 la %r11,STACK_FRAME_OVERHEAD(%r15) 678 stmg %r0,%r7,__PT_R0(%r11) 679 # clear user controlled registers to prevent speculative use 680 xgr %r0,%r0 681 xgr %r1,%r1 682 xgr %r2,%r2 683 xgr %r3,%r3 684 xgr %r4,%r4 685 xgr %r5,%r5 686 xgr %r6,%r6 687 xgr %r7,%r7 688 mvc __PT_R8(64,%r11),__LC_SAVE_AREA_SYNC 689 stmg %r8,%r9,__PT_PSW(%r11) 690 mvc __PT_INT_CODE(4,%r11),__LC_PGM_ILC 691 mvc __PT_INT_PARM_LONG(8,%r11),__LC_TRANS_EXC_CODE 692 stg %r13,__PT_FLAGS(%r11) 693 stg %r10,__PT_ARGS(%r11) 694 tm __LC_PGM_ILC+3,0x80 # check for per exception 695 jz 6f 696 tmhh %r8,0x0001 # kernel per event ? 697 jz .Lpgm_kprobe 698 oi __PT_FLAGS+7(%r11),_PIF_PER_TRAP 699 mvc __THREAD_per_address(8,%r14),__LC_PER_ADDRESS 700 mvc __THREAD_per_cause(2,%r14),__LC_PER_CODE 701 mvc __THREAD_per_paid(1,%r14),__LC_PER_ACCESS_ID 7026: RESTORE_SM_CLEAR_PER 703 xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) 704 larl %r1,pgm_check_table 705 llgh %r10,__PT_INT_CODE+2(%r11) 706 nill %r10,0x007f 707 sll %r10,3 708 je .Lpgm_return 709 lg %r9,0(%r10,%r1) # load address of handler routine 710 lgr %r2,%r11 # pass pointer to pt_regs 711 BASR_EX %r14,%r9 # branch to interrupt-handler 712.Lpgm_return: 713 LOCKDEP_SYS_EXIT 714 tm __PT_PSW+1(%r11),0x01 # returning to user ? 715 jno .Lpgm_restore 716 TSTMSK __PT_FLAGS(%r11),_PIF_SYSCALL 717 jo .Lsysc_do_syscall 718 j .Lsysc_tif 719.Lpgm_restore: 720 DISABLE_INTS 721 TSTMSK __LC_CPU_FLAGS, _CIF_FPU 722 jz .Lpgm_skip_fpu 723 brasl %r14,load_fpu_regs 724.Lpgm_skip_fpu: 725 mvc __LC_RETURN_PSW(16),__PT_PSW(%r11) 726 stpt __LC_EXIT_TIMER 727 lmg %r0,%r15,__PT_R0(%r11) 728 b __LC_RETURN_LPSWE 729 730# 731# PER event in supervisor state, must be kprobes 732# 733.Lpgm_kprobe: 734 RESTORE_SM_CLEAR_PER 735 xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) 736 lgr %r2,%r11 # pass pointer to pt_regs 737 brasl %r14,do_per_trap 738 j .Lpgm_return 739 740# 741# single stepped system call 742# 743.Lpgm_svcper: 744 mvc __LC_RETURN_PSW(8),__LC_SVC_NEW_PSW 745 larl %r14,.Lsysc_per 746 stg %r14,__LC_RETURN_PSW+8 747 lghi %r14,_PIF_SYSCALL | _PIF_PER_TRAP 748 lpswe __LC_RETURN_PSW # branch to .Lsysc_per 749ENDPROC(pgm_check_handler) 750 751/* 752 * IO interrupt handler routine 753 */ 754ENTRY(io_int_handler) 755 STCK __LC_INT_CLOCK 756 stpt __LC_ASYNC_ENTER_TIMER 757 BPOFF 758 stmg %r8,%r15,__LC_SAVE_AREA_ASYNC 759 lg %r12,__LC_CURRENT 760 lmg %r8,%r9,__LC_IO_OLD_PSW 761 SWITCH_ASYNC __LC_SAVE_AREA_ASYNC,__LC_ASYNC_ENTER_TIMER,__LC_INT_CLOCK 762 stmg %r0,%r7,__PT_R0(%r11) 763 # clear user controlled registers to prevent speculative use 764 xgr %r0,%r0 765 xgr %r1,%r1 766 xgr %r2,%r2 767 xgr %r3,%r3 768 xgr %r4,%r4 769 xgr %r5,%r5 770 xgr %r6,%r6 771 xgr %r7,%r7 772 xgr %r10,%r10 773 mvc __PT_R8(64,%r11),__LC_SAVE_AREA_ASYNC 774 stmg %r8,%r9,__PT_PSW(%r11) 775 tm __PT_PSW+1(%r11),0x01 # coming from user space? 776 jno .Lio_skip_asce 777 lctlg %c1,%c1,__LC_KERNEL_ASCE 778.Lio_skip_asce: 779 mvc __PT_INT_CODE(12,%r11),__LC_SUBCHANNEL_ID 780 xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11) 781 TSTMSK __LC_CPU_FLAGS,_CIF_IGNORE_IRQ 782 jo .Lio_restore 783 TRACE_IRQS_OFF 784 xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) 785.Lio_loop: 786 lgr %r2,%r11 # pass pointer to pt_regs 787 lghi %r3,IO_INTERRUPT 788 tm __PT_INT_CODE+8(%r11),0x80 # adapter interrupt ? 789 jz .Lio_call 790 lghi %r3,THIN_INTERRUPT 791.Lio_call: 792 brasl %r14,do_IRQ 793 TSTMSK __LC_MACHINE_FLAGS,MACHINE_FLAG_LPAR 794 jz .Lio_return 795 tpi 0 796 jz .Lio_return 797 mvc __PT_INT_CODE(12,%r11),__LC_SUBCHANNEL_ID 798 j .Lio_loop 799.Lio_return: 800 LOCKDEP_SYS_EXIT 801 TSTMSK __TI_flags(%r12),_TIF_WORK 802 jnz .Lio_work # there is work to do (signals etc.) 803 TSTMSK __LC_CPU_FLAGS,_CIF_WORK 804 jnz .Lio_work 805.Lio_restore: 806 TRACE_IRQS_ON 807 mvc __LC_RETURN_PSW(16),__PT_PSW(%r11) 808 tm __PT_PSW+1(%r11),0x01 # returning to user ? 809 jno .Lio_exit_kernel 810 DEBUG_USER_ASCE 811 lctlg %c1,%c1,__LC_USER_ASCE 812 BPEXIT __TI_flags(%r12),_TIF_ISOLATE_BP 813 stpt __LC_EXIT_TIMER 814.Lio_exit_kernel: 815 lmg %r0,%r15,__PT_R0(%r11) 816 b __LC_RETURN_LPSWE 817.Lio_done: 818 819# 820# There is work todo, find out in which context we have been interrupted: 821# 1) if we return to user space we can do all _TIF_WORK work 822# 2) if we return to kernel code and kvm is enabled check if we need to 823# modify the psw to leave SIE 824# 3) if we return to kernel code and preemptive scheduling is enabled check 825# the preemption counter and if it is zero call preempt_schedule_irq 826# Before any work can be done, a switch to the kernel stack is required. 827# 828.Lio_work: 829 tm __PT_PSW+1(%r11),0x01 # returning to user ? 830 jo .Lio_work_user # yes -> do resched & signal 831#ifdef CONFIG_PREEMPTION 832 # check for preemptive scheduling 833 icm %r0,15,__LC_PREEMPT_COUNT 834 jnz .Lio_restore # preemption is disabled 835 TSTMSK __TI_flags(%r12),_TIF_NEED_RESCHED 836 jno .Lio_restore 837 # switch to kernel stack 838 lg %r1,__PT_R15(%r11) 839 aghi %r1,-(STACK_FRAME_OVERHEAD + __PT_SIZE) 840 mvc STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11) 841 xc __SF_BACKCHAIN(8,%r1),__SF_BACKCHAIN(%r1) 842 la %r11,STACK_FRAME_OVERHEAD(%r1) 843 lgr %r15,%r1 844 brasl %r14,preempt_schedule_irq 845 j .Lio_return 846#else 847 j .Lio_restore 848#endif 849 850# 851# Need to do work before returning to userspace, switch to kernel stack 852# 853.Lio_work_user: 854 lg %r1,__LC_KERNEL_STACK 855 mvc STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11) 856 xc __SF_BACKCHAIN(8,%r1),__SF_BACKCHAIN(%r1) 857 la %r11,STACK_FRAME_OVERHEAD(%r1) 858 lgr %r15,%r1 859 860# 861# One of the work bits is on. Find out which one. 862# 863 TSTMSK __TI_flags(%r12),_TIF_NEED_RESCHED 864 jo .Lio_reschedule 865#ifdef CONFIG_LIVEPATCH 866 TSTMSK __TI_flags(%r12),_TIF_PATCH_PENDING 867 jo .Lio_patch_pending 868#endif 869 TSTMSK __TI_flags(%r12),(_TIF_SIGPENDING|_TIF_NOTIFY_SIGNAL) 870 jnz .Lio_sigpending 871 TSTMSK __TI_flags(%r12),_TIF_NOTIFY_RESUME 872 jo .Lio_notify_resume 873 TSTMSK __TI_flags(%r12),_TIF_GUARDED_STORAGE 874 jo .Lio_guarded_storage 875 TSTMSK __LC_CPU_FLAGS,_CIF_FPU 876 jo .Lio_vxrs 877 j .Lio_return 878 879# 880# CIF_FPU is set, restore floating-point controls and floating-point registers. 881# 882.Lio_vxrs: 883 larl %r14,.Lio_return 884 jg load_fpu_regs 885 886# 887# _TIF_GUARDED_STORAGE is set, call guarded_storage_load 888# 889.Lio_guarded_storage: 890 ENABLE_INTS_TRACE 891 lgr %r2,%r11 # pass pointer to pt_regs 892 brasl %r14,gs_load_bc_cb 893 DISABLE_INTS_TRACE 894 j .Lio_return 895 896# 897# _TIF_NEED_RESCHED is set, call schedule 898# 899.Lio_reschedule: 900 ENABLE_INTS_TRACE 901 brasl %r14,schedule # call scheduler 902 DISABLE_INTS_TRACE 903 j .Lio_return 904 905# 906# _TIF_PATCH_PENDING is set, call klp_update_patch_state 907# 908#ifdef CONFIG_LIVEPATCH 909.Lio_patch_pending: 910 lg %r2,__LC_CURRENT # pass pointer to task struct 911 larl %r14,.Lio_return 912 jg klp_update_patch_state 913#endif 914 915# 916# _TIF_SIGPENDING or is set, call do_signal 917# 918.Lio_sigpending: 919 ENABLE_INTS_TRACE 920 lgr %r2,%r11 # pass pointer to pt_regs 921 brasl %r14,do_signal 922 DISABLE_INTS_TRACE 923 j .Lio_return 924 925# 926# _TIF_NOTIFY_RESUME or is set, call do_notify_resume 927# 928.Lio_notify_resume: 929 ENABLE_INTS_TRACE 930 lgr %r2,%r11 # pass pointer to pt_regs 931 brasl %r14,do_notify_resume 932 DISABLE_INTS_TRACE 933 j .Lio_return 934ENDPROC(io_int_handler) 935 936/* 937 * External interrupt handler routine 938 */ 939ENTRY(ext_int_handler) 940 STCK __LC_INT_CLOCK 941 stpt __LC_ASYNC_ENTER_TIMER 942 BPOFF 943 stmg %r8,%r15,__LC_SAVE_AREA_ASYNC 944 lg %r12,__LC_CURRENT 945 lmg %r8,%r9,__LC_EXT_OLD_PSW 946 SWITCH_ASYNC __LC_SAVE_AREA_ASYNC,__LC_ASYNC_ENTER_TIMER,__LC_INT_CLOCK 947 stmg %r0,%r7,__PT_R0(%r11) 948 # clear user controlled registers to prevent speculative use 949 xgr %r0,%r0 950 xgr %r1,%r1 951 xgr %r2,%r2 952 xgr %r3,%r3 953 xgr %r4,%r4 954 xgr %r5,%r5 955 xgr %r6,%r6 956 xgr %r7,%r7 957 xgr %r10,%r10 958 mvc __PT_R8(64,%r11),__LC_SAVE_AREA_ASYNC 959 stmg %r8,%r9,__PT_PSW(%r11) 960 tm __PT_PSW+1(%r11),0x01 # coming from user space? 961 jno .Lext_skip_asce 962 lctlg %c1,%c1,__LC_KERNEL_ASCE 963.Lext_skip_asce: 964 lghi %r1,__LC_EXT_PARAMS2 965 mvc __PT_INT_CODE(4,%r11),__LC_EXT_CPU_ADDR 966 mvc __PT_INT_PARM(4,%r11),__LC_EXT_PARAMS 967 mvc __PT_INT_PARM_LONG(8,%r11),0(%r1) 968 xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11) 969 TSTMSK __LC_CPU_FLAGS,_CIF_IGNORE_IRQ 970 jo .Lio_restore 971 TRACE_IRQS_OFF 972 xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) 973 lgr %r2,%r11 # pass pointer to pt_regs 974 lghi %r3,EXT_INTERRUPT 975 brasl %r14,do_IRQ 976 j .Lio_return 977ENDPROC(ext_int_handler) 978 979/* 980 * Load idle PSW. 981 */ 982ENTRY(psw_idle) 983 stg %r3,__SF_EMPTY(%r15) 984 larl %r1,.Lpsw_idle_exit 985 stg %r1,__SF_EMPTY+8(%r15) 986 larl %r1,smp_cpu_mtid 987 llgf %r1,0(%r1) 988 ltgr %r1,%r1 989 jz .Lpsw_idle_stcctm 990 .insn rsy,0xeb0000000017,%r1,5,__SF_EMPTY+16(%r15) 991.Lpsw_idle_stcctm: 992 oi __LC_CPU_FLAGS+7,_CIF_ENABLED_WAIT 993 BPON 994 STCK __CLOCK_IDLE_ENTER(%r2) 995 stpt __TIMER_IDLE_ENTER(%r2) 996 lpswe __SF_EMPTY(%r15) 997.Lpsw_idle_exit: 998 BR_EX %r14 999ENDPROC(psw_idle) 1000 1001/* 1002 * Store floating-point controls and floating-point or vector register 1003 * depending whether the vector facility is available. A critical section 1004 * cleanup assures that the registers are stored even if interrupted for 1005 * some other work. The CIF_FPU flag is set to trigger a lazy restore 1006 * of the register contents at return from io or a system call. 1007 */ 1008ENTRY(save_fpu_regs) 1009 stnsm __SF_EMPTY(%r15),0xfc 1010 lg %r2,__LC_CURRENT 1011 aghi %r2,__TASK_thread 1012 TSTMSK __LC_CPU_FLAGS,_CIF_FPU 1013 jo .Lsave_fpu_regs_exit 1014 stfpc __THREAD_FPU_fpc(%r2) 1015 lg %r3,__THREAD_FPU_regs(%r2) 1016 TSTMSK __LC_MACHINE_FLAGS,MACHINE_FLAG_VX 1017 jz .Lsave_fpu_regs_fp # no -> store FP regs 1018 VSTM %v0,%v15,0,%r3 # vstm 0,15,0(3) 1019 VSTM %v16,%v31,256,%r3 # vstm 16,31,256(3) 1020 j .Lsave_fpu_regs_done # -> set CIF_FPU flag 1021.Lsave_fpu_regs_fp: 1022 std 0,0(%r3) 1023 std 1,8(%r3) 1024 std 2,16(%r3) 1025 std 3,24(%r3) 1026 std 4,32(%r3) 1027 std 5,40(%r3) 1028 std 6,48(%r3) 1029 std 7,56(%r3) 1030 std 8,64(%r3) 1031 std 9,72(%r3) 1032 std 10,80(%r3) 1033 std 11,88(%r3) 1034 std 12,96(%r3) 1035 std 13,104(%r3) 1036 std 14,112(%r3) 1037 std 15,120(%r3) 1038.Lsave_fpu_regs_done: 1039 oi __LC_CPU_FLAGS+7,_CIF_FPU 1040.Lsave_fpu_regs_exit: 1041 ssm __SF_EMPTY(%r15) 1042 BR_EX %r14 1043.Lsave_fpu_regs_end: 1044ENDPROC(save_fpu_regs) 1045EXPORT_SYMBOL(save_fpu_regs) 1046 1047/* 1048 * Load floating-point controls and floating-point or vector registers. 1049 * A critical section cleanup assures that the register contents are 1050 * loaded even if interrupted for some other work. 1051 * 1052 * There are special calling conventions to fit into sysc and io return work: 1053 * %r15: <kernel stack> 1054 * The function requires: 1055 * %r4 1056 */ 1057load_fpu_regs: 1058 stnsm __SF_EMPTY(%r15),0xfc 1059 lg %r4,__LC_CURRENT 1060 aghi %r4,__TASK_thread 1061 TSTMSK __LC_CPU_FLAGS,_CIF_FPU 1062 jno .Lload_fpu_regs_exit 1063 lfpc __THREAD_FPU_fpc(%r4) 1064 TSTMSK __LC_MACHINE_FLAGS,MACHINE_FLAG_VX 1065 lg %r4,__THREAD_FPU_regs(%r4) # %r4 <- reg save area 1066 jz .Lload_fpu_regs_fp # -> no VX, load FP regs 1067 VLM %v0,%v15,0,%r4 1068 VLM %v16,%v31,256,%r4 1069 j .Lload_fpu_regs_done 1070.Lload_fpu_regs_fp: 1071 ld 0,0(%r4) 1072 ld 1,8(%r4) 1073 ld 2,16(%r4) 1074 ld 3,24(%r4) 1075 ld 4,32(%r4) 1076 ld 5,40(%r4) 1077 ld 6,48(%r4) 1078 ld 7,56(%r4) 1079 ld 8,64(%r4) 1080 ld 9,72(%r4) 1081 ld 10,80(%r4) 1082 ld 11,88(%r4) 1083 ld 12,96(%r4) 1084 ld 13,104(%r4) 1085 ld 14,112(%r4) 1086 ld 15,120(%r4) 1087.Lload_fpu_regs_done: 1088 ni __LC_CPU_FLAGS+7,255-_CIF_FPU 1089.Lload_fpu_regs_exit: 1090 ssm __SF_EMPTY(%r15) 1091 BR_EX %r14 1092.Lload_fpu_regs_end: 1093ENDPROC(load_fpu_regs) 1094 1095/* 1096 * Machine check handler routines 1097 */ 1098ENTRY(mcck_int_handler) 1099 STCK __LC_MCCK_CLOCK 1100 BPOFF 1101 la %r1,4095 # validate r1 1102 spt __LC_CPU_TIMER_SAVE_AREA-4095(%r1) # validate cpu timer 1103 sckc __LC_CLOCK_COMPARATOR # validate comparator 1104 lam %a0,%a15,__LC_AREGS_SAVE_AREA-4095(%r1) # validate acrs 1105 lmg %r0,%r15,__LC_GPREGS_SAVE_AREA-4095(%r1)# validate gprs 1106 lg %r12,__LC_CURRENT 1107 lmg %r8,%r9,__LC_MCK_OLD_PSW 1108 TSTMSK __LC_MCCK_CODE,MCCK_CODE_SYSTEM_DAMAGE 1109 jo .Lmcck_panic # yes -> rest of mcck code invalid 1110 TSTMSK __LC_MCCK_CODE,MCCK_CODE_CR_VALID 1111 jno .Lmcck_panic # control registers invalid -> panic 1112 la %r14,4095 1113 lctlg %c0,%c15,__LC_CREGS_SAVE_AREA-4095(%r14) # validate ctl regs 1114 ptlb 1115 lg %r11,__LC_MCESAD-4095(%r14) # extended machine check save area 1116 nill %r11,0xfc00 # MCESA_ORIGIN_MASK 1117 TSTMSK __LC_CREGS_SAVE_AREA+16-4095(%r14),CR2_GUARDED_STORAGE 1118 jno 0f 1119 TSTMSK __LC_MCCK_CODE,MCCK_CODE_GS_VALID 1120 jno 0f 1121 .insn rxy,0xe3000000004d,0,__MCESA_GS_SAVE_AREA(%r11) # LGSC 11220: l %r14,__LC_FP_CREG_SAVE_AREA-4095(%r14) 1123 TSTMSK __LC_MCCK_CODE,MCCK_CODE_FC_VALID 1124 jo 0f 1125 sr %r14,%r14 11260: sfpc %r14 1127 TSTMSK __LC_MACHINE_FLAGS,MACHINE_FLAG_VX 1128 jo 0f 1129 lghi %r14,__LC_FPREGS_SAVE_AREA 1130 ld %f0,0(%r14) 1131 ld %f1,8(%r14) 1132 ld %f2,16(%r14) 1133 ld %f3,24(%r14) 1134 ld %f4,32(%r14) 1135 ld %f5,40(%r14) 1136 ld %f6,48(%r14) 1137 ld %f7,56(%r14) 1138 ld %f8,64(%r14) 1139 ld %f9,72(%r14) 1140 ld %f10,80(%r14) 1141 ld %f11,88(%r14) 1142 ld %f12,96(%r14) 1143 ld %f13,104(%r14) 1144 ld %f14,112(%r14) 1145 ld %f15,120(%r14) 1146 j 1f 11470: VLM %v0,%v15,0,%r11 1148 VLM %v16,%v31,256,%r11 11491: lghi %r14,__LC_CPU_TIMER_SAVE_AREA 1150 mvc __LC_MCCK_ENTER_TIMER(8),0(%r14) 1151 TSTMSK __LC_MCCK_CODE,MCCK_CODE_CPU_TIMER_VALID 1152 jo 3f 1153 la %r14,__LC_SYNC_ENTER_TIMER 1154 clc 0(8,%r14),__LC_ASYNC_ENTER_TIMER 1155 jl 0f 1156 la %r14,__LC_ASYNC_ENTER_TIMER 11570: clc 0(8,%r14),__LC_EXIT_TIMER 1158 jl 1f 1159 la %r14,__LC_EXIT_TIMER 11601: clc 0(8,%r14),__LC_LAST_UPDATE_TIMER 1161 jl 2f 1162 la %r14,__LC_LAST_UPDATE_TIMER 11632: spt 0(%r14) 1164 mvc __LC_MCCK_ENTER_TIMER(8),0(%r14) 11653: TSTMSK __LC_MCCK_CODE,MCCK_CODE_PSW_MWP_VALID 1166 jno .Lmcck_panic 1167 tmhh %r8,0x0001 # interrupting from user ? 1168 jnz 4f 1169 TSTMSK __LC_MCCK_CODE,MCCK_CODE_PSW_IA_VALID 1170 jno .Lmcck_panic 11714: ssm __LC_PGM_NEW_PSW # turn dat on, keep irqs off 1172 SWITCH_ASYNC __LC_GPREGS_SAVE_AREA+64,__LC_MCCK_ENTER_TIMER,__LC_MCCK_CLOCK 1173.Lmcck_skip: 1174 lghi %r14,__LC_GPREGS_SAVE_AREA+64 1175 stmg %r0,%r7,__PT_R0(%r11) 1176 # clear user controlled registers to prevent speculative use 1177 xgr %r0,%r0 1178 xgr %r1,%r1 1179 xgr %r2,%r2 1180 xgr %r3,%r3 1181 xgr %r4,%r4 1182 xgr %r5,%r5 1183 xgr %r6,%r6 1184 xgr %r7,%r7 1185 xgr %r10,%r10 1186 mvc __PT_R8(64,%r11),0(%r14) 1187 stmg %r8,%r9,__PT_PSW(%r11) 1188 la %r14,4095 1189 mvc __PT_CR1(8,%r11),__LC_CREGS_SAVE_AREA-4095+8(%r14) 1190 lctlg %c1,%c1,__LC_KERNEL_ASCE 1191 xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11) 1192 xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) 1193 lgr %r2,%r11 # pass pointer to pt_regs 1194 brasl %r14,s390_do_machine_check 1195 cghi %r2,0 1196 je .Lmcck_return 1197 lg %r1,__LC_KERNEL_STACK # switch to kernel stack 1198 mvc STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11) 1199 xc __SF_BACKCHAIN(8,%r1),__SF_BACKCHAIN(%r1) 1200 la %r11,STACK_FRAME_OVERHEAD(%r1) 1201 lgr %r15,%r1 1202 TRACE_IRQS_OFF 1203 brasl %r14,s390_handle_mcck 1204 TRACE_IRQS_ON 1205.Lmcck_return: 1206 lctlg %c1,%c1,__PT_CR1(%r11) 1207 lmg %r0,%r10,__PT_R0(%r11) 1208 mvc __LC_RETURN_MCCK_PSW(16),__PT_PSW(%r11) # move return PSW 1209 tm __LC_RETURN_MCCK_PSW+1,0x01 # returning to user ? 1210 jno 0f 1211 BPEXIT __TI_flags(%r12),_TIF_ISOLATE_BP 1212 stpt __LC_EXIT_TIMER 12130: lmg %r11,%r15,__PT_R11(%r11) 1214 b __LC_RETURN_MCCK_LPSWE 1215 1216.Lmcck_panic: 1217 lg %r15,__LC_NODAT_STACK 1218 la %r11,STACK_FRAME_OVERHEAD(%r15) 1219 j .Lmcck_skip 1220ENDPROC(mcck_int_handler) 1221 1222# 1223# PSW restart interrupt handler 1224# 1225ENTRY(restart_int_handler) 1226 ALTERNATIVE "", ".insn s,0xb2800000,_LPP_OFFSET", 40 1227 stg %r15,__LC_SAVE_AREA_RESTART 1228 lg %r15,__LC_RESTART_STACK 1229 xc STACK_FRAME_OVERHEAD(__PT_SIZE,%r15),STACK_FRAME_OVERHEAD(%r15) 1230 stmg %r0,%r14,STACK_FRAME_OVERHEAD+__PT_R0(%r15) 1231 mvc STACK_FRAME_OVERHEAD+__PT_R15(8,%r15),__LC_SAVE_AREA_RESTART 1232 mvc STACK_FRAME_OVERHEAD+__PT_PSW(16,%r15),__LC_RST_OLD_PSW 1233 xc 0(STACK_FRAME_OVERHEAD,%r15),0(%r15) 1234 lg %r1,__LC_RESTART_FN # load fn, parm & source cpu 1235 lg %r2,__LC_RESTART_DATA 1236 lg %r3,__LC_RESTART_SOURCE 1237 ltgr %r3,%r3 # test source cpu address 1238 jm 1f # negative -> skip source stop 12390: sigp %r4,%r3,SIGP_SENSE # sigp sense to source cpu 1240 brc 10,0b # wait for status stored 12411: basr %r14,%r1 # call function 1242 stap __SF_EMPTY(%r15) # store cpu address 1243 llgh %r3,__SF_EMPTY(%r15) 12442: sigp %r4,%r3,SIGP_STOP # sigp stop to current cpu 1245 brc 2,2b 12463: j 3b 1247ENDPROC(restart_int_handler) 1248 1249 .section .kprobes.text, "ax" 1250 1251#if defined(CONFIG_CHECK_STACK) || defined(CONFIG_VMAP_STACK) 1252/* 1253 * The synchronous or the asynchronous stack overflowed. We are dead. 1254 * No need to properly save the registers, we are going to panic anyway. 1255 * Setup a pt_regs so that show_trace can provide a good call trace. 1256 */ 1257ENTRY(stack_overflow) 1258 lg %r15,__LC_NODAT_STACK # change to panic stack 1259 la %r11,STACK_FRAME_OVERHEAD(%r15) 1260 stmg %r0,%r7,__PT_R0(%r11) 1261 stmg %r8,%r9,__PT_PSW(%r11) 1262 mvc __PT_R8(64,%r11),0(%r14) 1263 stg %r10,__PT_ORIG_GPR2(%r11) # store last break to orig_gpr2 1264 xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) 1265 lgr %r2,%r11 # pass pointer to pt_regs 1266 jg kernel_stack_overflow 1267ENDPROC(stack_overflow) 1268#endif 1269 1270#if IS_ENABLED(CONFIG_KVM) 1271.Lcleanup_sie: 1272 cghi %r11,__LC_SAVE_AREA_ASYNC #Is this in normal interrupt? 1273 je 1f 1274 larl %r13,.Lsie_entry 1275 slgr %r9,%r13 1276 larl %r13,.Lsie_skip 1277 clgr %r9,%r13 1278 jh 1f 1279 oi __LC_CPU_FLAGS+7, _CIF_MCCK_GUEST 12801: BPENTER __SF_SIE_FLAGS(%r15),(_TIF_ISOLATE_BP|_TIF_ISOLATE_BP_GUEST) 1281 lg %r9,__SF_SIE_CONTROL(%r15) # get control block pointer 1282 ni __SIE_PROG0C+3(%r9),0xfe # no longer in SIE 1283 lctlg %c1,%c1,__LC_KERNEL_ASCE 1284 larl %r9,sie_exit # skip forward to sie_exit 1285 BR_EX %r14,%r11 1286 1287#endif 1288 .section .rodata, "a" 1289#define SYSCALL(esame,emu) .quad __s390x_ ## esame 1290 .globl sys_call_table 1291sys_call_table: 1292#include "asm/syscall_table.h" 1293#undef SYSCALL 1294 1295#ifdef CONFIG_COMPAT 1296 1297#define SYSCALL(esame,emu) .quad __s390_ ## emu 1298 .globl sys_call_table_emu 1299sys_call_table_emu: 1300#include "asm/syscall_table.h" 1301#undef SYSCALL 1302#endif 1303