1/* SPDX-License-Identifier: GPL-2.0 */ 2/* 3 * S390 low-level entry points. 4 * 5 * Copyright IBM Corp. 1999, 2012 6 * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com), 7 * Hartmut Penner (hp@de.ibm.com), 8 * Denis Joseph Barrow (djbarrow@de.ibm.com,barrow_dj@yahoo.com), 9 * Heiko Carstens <heiko.carstens@de.ibm.com> 10 */ 11 12#include <linux/init.h> 13#include <linux/linkage.h> 14#include <asm/alternative-asm.h> 15#include <asm/processor.h> 16#include <asm/cache.h> 17#include <asm/ctl_reg.h> 18#include <asm/dwarf.h> 19#include <asm/errno.h> 20#include <asm/ptrace.h> 21#include <asm/thread_info.h> 22#include <asm/asm-offsets.h> 23#include <asm/unistd.h> 24#include <asm/page.h> 25#include <asm/sigp.h> 26#include <asm/irq.h> 27#include <asm/vx-insn.h> 28#include <asm/setup.h> 29#include <asm/nmi.h> 30#include <asm/export.h> 31#include <asm/nospec-insn.h> 32 33__PT_R0 = __PT_GPRS 34__PT_R1 = __PT_GPRS + 8 35__PT_R2 = __PT_GPRS + 16 36__PT_R3 = __PT_GPRS + 24 37__PT_R4 = __PT_GPRS + 32 38__PT_R5 = __PT_GPRS + 40 39__PT_R6 = __PT_GPRS + 48 40__PT_R7 = __PT_GPRS + 56 41__PT_R8 = __PT_GPRS + 64 42__PT_R9 = __PT_GPRS + 72 43__PT_R10 = __PT_GPRS + 80 44__PT_R11 = __PT_GPRS + 88 45__PT_R12 = __PT_GPRS + 96 46__PT_R13 = __PT_GPRS + 104 47__PT_R14 = __PT_GPRS + 112 48__PT_R15 = __PT_GPRS + 120 49 50STACK_SHIFT = PAGE_SHIFT + THREAD_SIZE_ORDER 51STACK_SIZE = 1 << STACK_SHIFT 52STACK_INIT = STACK_SIZE - STACK_FRAME_OVERHEAD - __PT_SIZE 53 54_TIF_WORK = (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_NEED_RESCHED | \ 55 _TIF_UPROBE | _TIF_GUARDED_STORAGE | _TIF_PATCH_PENDING) 56_TIF_TRACE = (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | _TIF_SECCOMP | \ 57 _TIF_SYSCALL_TRACEPOINT) 58_CIF_WORK = (_CIF_ASCE_PRIMARY | _CIF_ASCE_SECONDARY | _CIF_FPU) 59_PIF_WORK = (_PIF_PER_TRAP | _PIF_SYSCALL_RESTART) 60 61_LPP_OFFSET = __LC_LPP 62 63 .macro TRACE_IRQS_ON 64#ifdef CONFIG_TRACE_IRQFLAGS 65 basr %r2,%r0 66 brasl %r14,trace_hardirqs_on_caller 67#endif 68 .endm 69 70 .macro TRACE_IRQS_OFF 71#ifdef CONFIG_TRACE_IRQFLAGS 72 basr %r2,%r0 73 brasl %r14,trace_hardirqs_off_caller 74#endif 75 .endm 76 77 .macro LOCKDEP_SYS_EXIT 78#ifdef CONFIG_LOCKDEP 79 tm __PT_PSW+1(%r11),0x01 # returning to user ? 80 jz .+10 81 brasl %r14,lockdep_sys_exit 82#endif 83 .endm 84 85 .macro CHECK_STACK savearea 86#ifdef CONFIG_CHECK_STACK 87 tml %r15,STACK_SIZE - CONFIG_STACK_GUARD 88 lghi %r14,\savearea 89 jz stack_overflow 90#endif 91 .endm 92 93 .macro CHECK_VMAP_STACK savearea,oklabel 94#ifdef CONFIG_VMAP_STACK 95 lgr %r14,%r15 96 nill %r14,0x10000 - STACK_SIZE 97 oill %r14,STACK_INIT 98 clg %r14,__LC_KERNEL_STACK 99 je \oklabel 100 clg %r14,__LC_ASYNC_STACK 101 je \oklabel 102 clg %r14,__LC_NODAT_STACK 103 je \oklabel 104 clg %r14,__LC_RESTART_STACK 105 je \oklabel 106 lghi %r14,\savearea 107 j stack_overflow 108#else 109 j \oklabel 110#endif 111 .endm 112 113 .macro SWITCH_ASYNC savearea,timer 114 tmhh %r8,0x0001 # interrupting from user ? 115 jnz 2f 116#if IS_ENABLED(CONFIG_KVM) 117 lgr %r14,%r9 118 larl %r13,.Lsie_gmap 119 slgr %r14,%r13 120 lghi %r13,.Lsie_done - .Lsie_gmap 121 clgr %r14,%r13 122 jhe 0f 123 lghi %r11,\savearea # inside critical section, do cleanup 124 brasl %r14,.Lcleanup_sie 125#endif 1260: larl %r13,.Lpsw_idle_exit 127 cgr %r13,%r9 128 jne 1f 129 130 mvc __CLOCK_IDLE_EXIT(8,%r2), __LC_INT_CLOCK 131 mvc __TIMER_IDLE_EXIT(8,%r2), __LC_ASYNC_ENTER_TIMER 132 # account system time going idle 133 ni __LC_CPU_FLAGS+7,255-_CIF_ENABLED_WAIT 134 135 lg %r13,__LC_STEAL_TIMER 136 alg %r13,__CLOCK_IDLE_ENTER(%r2) 137 slg %r13,__LC_LAST_UPDATE_CLOCK 138 stg %r13,__LC_STEAL_TIMER 139 140 mvc __LC_LAST_UPDATE_CLOCK(8),__CLOCK_IDLE_EXIT(%r2) 141 142 lg %r13,__LC_SYSTEM_TIMER 143 alg %r13,__LC_LAST_UPDATE_TIMER 144 slg %r13,__TIMER_IDLE_ENTER(%r2) 145 stg %r13,__LC_SYSTEM_TIMER 146 mvc __LC_LAST_UPDATE_TIMER(8),__TIMER_IDLE_EXIT(%r2) 147 148 nihh %r8,0xfcfd # clear wait state and irq bits 1491: lg %r14,__LC_ASYNC_STACK # are we already on the target stack? 150 slgr %r14,%r15 151 srag %r14,%r14,STACK_SHIFT 152 jnz 3f 153 CHECK_STACK \savearea 154 aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE) 155 j 4f 1562: UPDATE_VTIME %r14,%r15,\timer 157 BPENTER __TI_flags(%r12),_TIF_ISOLATE_BP 1583: lg %r15,__LC_ASYNC_STACK # load async stack 1594: la %r11,STACK_FRAME_OVERHEAD(%r15) 160 .endm 161 162 .macro UPDATE_VTIME w1,w2,enter_timer 163 lg \w1,__LC_EXIT_TIMER 164 lg \w2,__LC_LAST_UPDATE_TIMER 165 slg \w1,\enter_timer 166 slg \w2,__LC_EXIT_TIMER 167 alg \w1,__LC_USER_TIMER 168 alg \w2,__LC_SYSTEM_TIMER 169 stg \w1,__LC_USER_TIMER 170 stg \w2,__LC_SYSTEM_TIMER 171 mvc __LC_LAST_UPDATE_TIMER(8),\enter_timer 172 .endm 173 174 .macro RESTORE_SM_CLEAR_PER 175 stg %r8,__LC_RETURN_PSW 176 ni __LC_RETURN_PSW,0xbf 177 ssm __LC_RETURN_PSW 178 .endm 179 180 .macro ENABLE_INTS 181 stosm __SF_EMPTY(%r15),3 182 .endm 183 184 .macro ENABLE_INTS_TRACE 185 TRACE_IRQS_ON 186 ENABLE_INTS 187 .endm 188 189 .macro DISABLE_INTS 190 stnsm __SF_EMPTY(%r15),0xfc 191 .endm 192 193 .macro DISABLE_INTS_TRACE 194 DISABLE_INTS 195 TRACE_IRQS_OFF 196 .endm 197 198 .macro STCK savearea 199#ifdef CONFIG_HAVE_MARCH_Z9_109_FEATURES 200 .insn s,0xb27c0000,\savearea # store clock fast 201#else 202 .insn s,0xb2050000,\savearea # store clock 203#endif 204 .endm 205 206 /* 207 * The TSTMSK macro generates a test-under-mask instruction by 208 * calculating the memory offset for the specified mask value. 209 * Mask value can be any constant. The macro shifts the mask 210 * value to calculate the memory offset for the test-under-mask 211 * instruction. 212 */ 213 .macro TSTMSK addr, mask, size=8, bytepos=0 214 .if (\bytepos < \size) && (\mask >> 8) 215 .if (\mask & 0xff) 216 .error "Mask exceeds byte boundary" 217 .endif 218 TSTMSK \addr, "(\mask >> 8)", \size, "(\bytepos + 1)" 219 .exitm 220 .endif 221 .ifeq \mask 222 .error "Mask must not be zero" 223 .endif 224 off = \size - \bytepos - 1 225 tm off+\addr, \mask 226 .endm 227 228 .macro BPOFF 229 ALTERNATIVE "", ".long 0xb2e8c000", 82 230 .endm 231 232 .macro BPON 233 ALTERNATIVE "", ".long 0xb2e8d000", 82 234 .endm 235 236 .macro BPENTER tif_ptr,tif_mask 237 ALTERNATIVE "TSTMSK \tif_ptr,\tif_mask; jz .+8; .long 0xb2e8d000", \ 238 "", 82 239 .endm 240 241 .macro BPEXIT tif_ptr,tif_mask 242 TSTMSK \tif_ptr,\tif_mask 243 ALTERNATIVE "jz .+8; .long 0xb2e8c000", \ 244 "jnz .+8; .long 0xb2e8d000", 82 245 .endm 246 247 GEN_BR_THUNK %r9 248 GEN_BR_THUNK %r14 249 GEN_BR_THUNK %r14,%r11 250 251 .section .kprobes.text, "ax" 252.Ldummy: 253 /* 254 * This nop exists only in order to avoid that __switch_to starts at 255 * the beginning of the kprobes text section. In that case we would 256 * have several symbols at the same address. E.g. objdump would take 257 * an arbitrary symbol name when disassembling this code. 258 * With the added nop in between the __switch_to symbol is unique 259 * again. 260 */ 261 nop 0 262 263ENTRY(__bpon) 264 .globl __bpon 265 BPON 266 BR_EX %r14 267ENDPROC(__bpon) 268 269/* 270 * Scheduler resume function, called by switch_to 271 * gpr2 = (task_struct *) prev 272 * gpr3 = (task_struct *) next 273 * Returns: 274 * gpr2 = prev 275 */ 276ENTRY(__switch_to) 277 stmg %r6,%r15,__SF_GPRS(%r15) # store gprs of prev task 278 lghi %r4,__TASK_stack 279 lghi %r1,__TASK_thread 280 llill %r5,STACK_INIT 281 stg %r15,__THREAD_ksp(%r1,%r2) # store kernel stack of prev 282 lg %r15,0(%r4,%r3) # start of kernel stack of next 283 agr %r15,%r5 # end of kernel stack of next 284 stg %r3,__LC_CURRENT # store task struct of next 285 stg %r15,__LC_KERNEL_STACK # store end of kernel stack 286 lg %r15,__THREAD_ksp(%r1,%r3) # load kernel stack of next 287 aghi %r3,__TASK_pid 288 mvc __LC_CURRENT_PID(4,%r0),0(%r3) # store pid of next 289 lmg %r6,%r15,__SF_GPRS(%r15) # load gprs of next task 290 ALTERNATIVE "", ".insn s,0xb2800000,_LPP_OFFSET", 40 291 BR_EX %r14 292ENDPROC(__switch_to) 293 294#if IS_ENABLED(CONFIG_KVM) 295/* 296 * sie64a calling convention: 297 * %r2 pointer to sie control block 298 * %r3 guest register save area 299 */ 300ENTRY(sie64a) 301 stmg %r6,%r14,__SF_GPRS(%r15) # save kernel registers 302 lg %r12,__LC_CURRENT 303 stg %r2,__SF_SIE_CONTROL(%r15) # save control block pointer 304 stg %r3,__SF_SIE_SAVEAREA(%r15) # save guest register save area 305 xc __SF_SIE_REASON(8,%r15),__SF_SIE_REASON(%r15) # reason code = 0 306 mvc __SF_SIE_FLAGS(8,%r15),__TI_flags(%r12) # copy thread flags 307 TSTMSK __LC_CPU_FLAGS,_CIF_FPU # load guest fp/vx registers ? 308 jno .Lsie_load_guest_gprs 309 brasl %r14,load_fpu_regs # load guest fp/vx regs 310.Lsie_load_guest_gprs: 311 lmg %r0,%r13,0(%r3) # load guest gprs 0-13 312 lg %r14,__LC_GMAP # get gmap pointer 313 ltgr %r14,%r14 314 jz .Lsie_gmap 315 lctlg %c1,%c1,__GMAP_ASCE(%r14) # load primary asce 316.Lsie_gmap: 317 lg %r14,__SF_SIE_CONTROL(%r15) # get control block pointer 318 oi __SIE_PROG0C+3(%r14),1 # we are going into SIE now 319 tm __SIE_PROG20+3(%r14),3 # last exit... 320 jnz .Lsie_skip 321 TSTMSK __LC_CPU_FLAGS,_CIF_FPU 322 jo .Lsie_skip # exit if fp/vx regs changed 323 BPEXIT __SF_SIE_FLAGS(%r15),(_TIF_ISOLATE_BP|_TIF_ISOLATE_BP_GUEST) 324.Lsie_entry: 325 sie 0(%r14) 326 BPOFF 327 BPENTER __SF_SIE_FLAGS(%r15),(_TIF_ISOLATE_BP|_TIF_ISOLATE_BP_GUEST) 328.Lsie_skip: 329 ni __SIE_PROG0C+3(%r14),0xfe # no longer in SIE 330 lctlg %c1,%c1,__LC_USER_ASCE # load primary asce 331.Lsie_done: 332# some program checks are suppressing. C code (e.g. do_protection_exception) 333# will rewind the PSW by the ILC, which is often 4 bytes in case of SIE. There 334# are some corner cases (e.g. runtime instrumentation) where ILC is unpredictable. 335# Other instructions between sie64a and .Lsie_done should not cause program 336# interrupts. So lets use 3 nops as a landing pad for all possible rewinds. 337# See also .Lcleanup_sie 338.Lrewind_pad6: 339 nopr 7 340.Lrewind_pad4: 341 nopr 7 342.Lrewind_pad2: 343 nopr 7 344 .globl sie_exit 345sie_exit: 346 lg %r14,__SF_SIE_SAVEAREA(%r15) # load guest register save area 347 stmg %r0,%r13,0(%r14) # save guest gprs 0-13 348 xgr %r0,%r0 # clear guest registers to 349 xgr %r1,%r1 # prevent speculative use 350 xgr %r2,%r2 351 xgr %r3,%r3 352 xgr %r4,%r4 353 xgr %r5,%r5 354 lmg %r6,%r14,__SF_GPRS(%r15) # restore kernel registers 355 lg %r2,__SF_SIE_REASON(%r15) # return exit reason code 356 BR_EX %r14 357.Lsie_fault: 358 lghi %r14,-EFAULT 359 stg %r14,__SF_SIE_REASON(%r15) # set exit reason code 360 j sie_exit 361 362 EX_TABLE(.Lrewind_pad6,.Lsie_fault) 363 EX_TABLE(.Lrewind_pad4,.Lsie_fault) 364 EX_TABLE(.Lrewind_pad2,.Lsie_fault) 365 EX_TABLE(sie_exit,.Lsie_fault) 366ENDPROC(sie64a) 367EXPORT_SYMBOL(sie64a) 368EXPORT_SYMBOL(sie_exit) 369#endif 370 371/* 372 * SVC interrupt handler routine. System calls are synchronous events and 373 * are entered with interrupts disabled. 374 */ 375 376ENTRY(system_call) 377 stpt __LC_SYNC_ENTER_TIMER 378 stmg %r8,%r15,__LC_SAVE_AREA_SYNC 379 BPOFF 380 lg %r12,__LC_CURRENT 381 lghi %r14,_PIF_SYSCALL 382.Lsysc_per: 383 lghi %r13,__TASK_thread 384 lg %r15,__LC_KERNEL_STACK 385 la %r11,STACK_FRAME_OVERHEAD(%r15) # pointer to pt_regs 386 UPDATE_VTIME %r8,%r9,__LC_SYNC_ENTER_TIMER 387 BPENTER __TI_flags(%r12),_TIF_ISOLATE_BP 388 stmg %r0,%r7,__PT_R0(%r11) 389 mvc __PT_R8(64,%r11),__LC_SAVE_AREA_SYNC 390 mvc __PT_PSW(16,%r11),__LC_SVC_OLD_PSW 391 mvc __PT_INT_CODE(4,%r11),__LC_SVC_ILC 392 stg %r14,__PT_FLAGS(%r11) 393 ENABLE_INTS 394.Lsysc_do_svc: 395 # clear user controlled register to prevent speculative use 396 xgr %r0,%r0 397 # load address of system call table 398 lg %r10,__THREAD_sysc_table(%r13,%r12) 399 llgh %r8,__PT_INT_CODE+2(%r11) 400 slag %r8,%r8,3 # shift and test for svc 0 401 jnz .Lsysc_nr_ok 402 # svc 0: system call number in %r1 403 llgfr %r1,%r1 # clear high word in r1 404 sth %r1,__PT_INT_CODE+2(%r11) 405 cghi %r1,NR_syscalls 406 jnl .Lsysc_nr_ok 407 slag %r8,%r1,3 408.Lsysc_nr_ok: 409 xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) 410 stg %r2,__PT_ORIG_GPR2(%r11) 411 stg %r7,STACK_FRAME_OVERHEAD(%r15) 412 lg %r9,0(%r8,%r10) # get system call add. 413 TSTMSK __TI_flags(%r12),_TIF_TRACE 414 jnz .Lsysc_tracesys 415 BASR_EX %r14,%r9 # call sys_xxxx 416 stg %r2,__PT_R2(%r11) # store return value 417 418.Lsysc_return: 419#ifdef CONFIG_DEBUG_RSEQ 420 lgr %r2,%r11 421 brasl %r14,rseq_syscall 422#endif 423 LOCKDEP_SYS_EXIT 424.Lsysc_tif: 425 TSTMSK __PT_FLAGS(%r11),_PIF_WORK 426 jnz .Lsysc_work 427 TSTMSK __TI_flags(%r12),_TIF_WORK 428 jnz .Lsysc_work # check for work 429 TSTMSK __LC_CPU_FLAGS,(_CIF_WORK-_CIF_FPU) 430 jnz .Lsysc_work 431 BPEXIT __TI_flags(%r12),_TIF_ISOLATE_BP 432.Lsysc_restore: 433 DISABLE_INTS 434 TSTMSK __LC_CPU_FLAGS, _CIF_FPU 435 jz .Lsysc_skip_fpu 436 brasl %r14,load_fpu_regs 437.Lsysc_skip_fpu: 438 mvc __LC_RETURN_PSW(16),__PT_PSW(%r11) 439 stpt __LC_EXIT_TIMER 440 lmg %r0,%r15,__PT_R0(%r11) 441 b __LC_RETURN_LPSWE 442 443# 444# One of the work bits is on. Find out which one. 445# 446.Lsysc_work: 447 TSTMSK __TI_flags(%r12),_TIF_NEED_RESCHED 448 jo .Lsysc_reschedule 449 TSTMSK __PT_FLAGS(%r11),_PIF_SYSCALL_RESTART 450 jo .Lsysc_syscall_restart 451#ifdef CONFIG_UPROBES 452 TSTMSK __TI_flags(%r12),_TIF_UPROBE 453 jo .Lsysc_uprobe_notify 454#endif 455 TSTMSK __TI_flags(%r12),_TIF_GUARDED_STORAGE 456 jo .Lsysc_guarded_storage 457 TSTMSK __PT_FLAGS(%r11),_PIF_PER_TRAP 458 jo .Lsysc_singlestep 459#ifdef CONFIG_LIVEPATCH 460 TSTMSK __TI_flags(%r12),_TIF_PATCH_PENDING 461 jo .Lsysc_patch_pending # handle live patching just before 462 # signals and possible syscall restart 463#endif 464 TSTMSK __PT_FLAGS(%r11),_PIF_SYSCALL_RESTART 465 jo .Lsysc_syscall_restart 466 TSTMSK __TI_flags(%r12),_TIF_SIGPENDING 467 jo .Lsysc_sigpending 468 TSTMSK __TI_flags(%r12),_TIF_NOTIFY_RESUME 469 jo .Lsysc_notify_resume 470 TSTMSK __LC_CPU_FLAGS,(_CIF_ASCE_PRIMARY|_CIF_ASCE_SECONDARY) 471 jnz .Lsysc_asce 472 j .Lsysc_return 473 474# 475# _TIF_NEED_RESCHED is set, call schedule 476# 477.Lsysc_reschedule: 478 larl %r14,.Lsysc_return 479 jg schedule 480 481# 482# _CIF_ASCE_PRIMARY and/or _CIF_ASCE_SECONDARY set, load user space asce 483# 484.Lsysc_asce: 485 ni __LC_CPU_FLAGS+7,255-_CIF_ASCE_SECONDARY 486 lctlg %c7,%c7,__LC_VDSO_ASCE # load secondary asce 487 TSTMSK __LC_CPU_FLAGS,_CIF_ASCE_PRIMARY 488 jz .Lsysc_return 489#ifndef CONFIG_HAVE_MARCH_Z10_FEATURES 490 tm __LC_STFLE_FAC_LIST+3,0x10 # has MVCOS ? 491 jnz .Lsysc_set_fs_fixup 492 ni __LC_CPU_FLAGS+7,255-_CIF_ASCE_PRIMARY 493 lctlg %c1,%c1,__LC_USER_ASCE # load primary asce 494 j .Lsysc_return 495.Lsysc_set_fs_fixup: 496#endif 497 larl %r14,.Lsysc_return 498 jg set_fs_fixup 499 500 501# 502# _TIF_SIGPENDING is set, call do_signal 503# 504.Lsysc_sigpending: 505 lgr %r2,%r11 # pass pointer to pt_regs 506 brasl %r14,do_signal 507 TSTMSK __PT_FLAGS(%r11),_PIF_SYSCALL 508 jno .Lsysc_return 509.Lsysc_do_syscall: 510 lghi %r13,__TASK_thread 511 lmg %r2,%r7,__PT_R2(%r11) # load svc arguments 512 lghi %r1,0 # svc 0 returns -ENOSYS 513 j .Lsysc_do_svc 514 515# 516# _TIF_NOTIFY_RESUME is set, call do_notify_resume 517# 518.Lsysc_notify_resume: 519 lgr %r2,%r11 # pass pointer to pt_regs 520 larl %r14,.Lsysc_return 521 jg do_notify_resume 522 523# 524# _TIF_UPROBE is set, call uprobe_notify_resume 525# 526#ifdef CONFIG_UPROBES 527.Lsysc_uprobe_notify: 528 lgr %r2,%r11 # pass pointer to pt_regs 529 larl %r14,.Lsysc_return 530 jg uprobe_notify_resume 531#endif 532 533# 534# _TIF_GUARDED_STORAGE is set, call guarded_storage_load 535# 536.Lsysc_guarded_storage: 537 lgr %r2,%r11 # pass pointer to pt_regs 538 larl %r14,.Lsysc_return 539 jg gs_load_bc_cb 540# 541# _TIF_PATCH_PENDING is set, call klp_update_patch_state 542# 543#ifdef CONFIG_LIVEPATCH 544.Lsysc_patch_pending: 545 lg %r2,__LC_CURRENT # pass pointer to task struct 546 larl %r14,.Lsysc_return 547 jg klp_update_patch_state 548#endif 549 550# 551# _PIF_PER_TRAP is set, call do_per_trap 552# 553.Lsysc_singlestep: 554 ni __PT_FLAGS+7(%r11),255-_PIF_PER_TRAP 555 lgr %r2,%r11 # pass pointer to pt_regs 556 larl %r14,.Lsysc_return 557 jg do_per_trap 558 559# 560# _PIF_SYSCALL_RESTART is set, repeat the current system call 561# 562.Lsysc_syscall_restart: 563 ni __PT_FLAGS+7(%r11),255-_PIF_SYSCALL_RESTART 564 lmg %r1,%r7,__PT_R1(%r11) # load svc arguments 565 lg %r2,__PT_ORIG_GPR2(%r11) 566 j .Lsysc_do_svc 567 568# 569# call tracehook_report_syscall_entry/tracehook_report_syscall_exit before 570# and after the system call 571# 572.Lsysc_tracesys: 573 lgr %r2,%r11 # pass pointer to pt_regs 574 la %r3,0 575 llgh %r0,__PT_INT_CODE+2(%r11) 576 stg %r0,__PT_R2(%r11) 577 brasl %r14,do_syscall_trace_enter 578 lghi %r0,NR_syscalls 579 clgr %r0,%r2 580 jnh .Lsysc_tracenogo 581 sllg %r8,%r2,3 582 lg %r9,0(%r8,%r10) 583 lmg %r3,%r7,__PT_R3(%r11) 584 stg %r7,STACK_FRAME_OVERHEAD(%r15) 585 lg %r2,__PT_ORIG_GPR2(%r11) 586 BASR_EX %r14,%r9 # call sys_xxx 587 stg %r2,__PT_R2(%r11) # store return value 588.Lsysc_tracenogo: 589 TSTMSK __TI_flags(%r12),_TIF_TRACE 590 jz .Lsysc_return 591 lgr %r2,%r11 # pass pointer to pt_regs 592 larl %r14,.Lsysc_return 593 jg do_syscall_trace_exit 594ENDPROC(system_call) 595 596# 597# a new process exits the kernel with ret_from_fork 598# 599ENTRY(ret_from_fork) 600 la %r11,STACK_FRAME_OVERHEAD(%r15) 601 lg %r12,__LC_CURRENT 602 brasl %r14,schedule_tail 603 tm __PT_PSW+1(%r11),0x01 # forking a kernel thread ? 604 jne .Lsysc_tracenogo 605 # it's a kernel thread 606 lmg %r9,%r10,__PT_R9(%r11) # load gprs 607 la %r2,0(%r10) 608 BASR_EX %r14,%r9 609 j .Lsysc_tracenogo 610ENDPROC(ret_from_fork) 611 612ENTRY(kernel_thread_starter) 613 la %r2,0(%r10) 614 BASR_EX %r14,%r9 615 j .Lsysc_tracenogo 616ENDPROC(kernel_thread_starter) 617 618/* 619 * Program check handler routine 620 */ 621 622ENTRY(pgm_check_handler) 623 stpt __LC_SYNC_ENTER_TIMER 624 BPOFF 625 stmg %r8,%r15,__LC_SAVE_AREA_SYNC 626 lg %r10,__LC_LAST_BREAK 627 srag %r11,%r10,12 628 jnz 0f 629 /* if __LC_LAST_BREAK is < 4096, it contains one of 630 * the lpswe addresses in lowcore. Set it to 1 (initial state) 631 * to prevent leaking that address to userspace. 632 */ 633 lghi %r10,1 6340: lg %r12,__LC_CURRENT 635 lghi %r11,0 636 lmg %r8,%r9,__LC_PGM_OLD_PSW 637 tmhh %r8,0x0001 # test problem state bit 638 jnz 3f # -> fault in user space 639#if IS_ENABLED(CONFIG_KVM) 640 # cleanup critical section for program checks in sie64a 641 lgr %r14,%r9 642 larl %r13,.Lsie_gmap 643 slgr %r14,%r13 644 lghi %r13,.Lsie_done - .Lsie_gmap 645 clgr %r14,%r13 646 jhe 1f 647 lg %r14,__SF_SIE_CONTROL(%r15) # get control block pointer 648 ni __SIE_PROG0C+3(%r14),0xfe # no longer in SIE 649 lctlg %c1,%c1,__LC_USER_ASCE # load primary asce 650 larl %r9,sie_exit # skip forward to sie_exit 651 lghi %r11,_PIF_GUEST_FAULT 652#endif 6531: tmhh %r8,0x4000 # PER bit set in old PSW ? 654 jnz 2f # -> enabled, can't be a double fault 655 tm __LC_PGM_ILC+3,0x80 # check for per exception 656 jnz .Lpgm_svcper # -> single stepped svc 6572: CHECK_STACK __LC_SAVE_AREA_SYNC 658 aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE) 659 # CHECK_VMAP_STACK branches to stack_overflow or 5f 660 CHECK_VMAP_STACK __LC_SAVE_AREA_SYNC,5f 6613: UPDATE_VTIME %r14,%r15,__LC_SYNC_ENTER_TIMER 662 BPENTER __TI_flags(%r12),_TIF_ISOLATE_BP 663 lg %r15,__LC_KERNEL_STACK 664 lgr %r14,%r12 665 aghi %r14,__TASK_thread # pointer to thread_struct 666 lghi %r13,__LC_PGM_TDB 667 tm __LC_PGM_ILC+2,0x02 # check for transaction abort 668 jz 4f 669 mvc __THREAD_trap_tdb(256,%r14),0(%r13) 6704: stg %r10,__THREAD_last_break(%r14) 6715: lgr %r13,%r11 672 la %r11,STACK_FRAME_OVERHEAD(%r15) 673 stmg %r0,%r7,__PT_R0(%r11) 674 # clear user controlled registers to prevent speculative use 675 xgr %r0,%r0 676 xgr %r1,%r1 677 xgr %r2,%r2 678 xgr %r3,%r3 679 xgr %r4,%r4 680 xgr %r5,%r5 681 xgr %r6,%r6 682 xgr %r7,%r7 683 mvc __PT_R8(64,%r11),__LC_SAVE_AREA_SYNC 684 stmg %r8,%r9,__PT_PSW(%r11) 685 mvc __PT_INT_CODE(4,%r11),__LC_PGM_ILC 686 mvc __PT_INT_PARM_LONG(8,%r11),__LC_TRANS_EXC_CODE 687 stg %r13,__PT_FLAGS(%r11) 688 stg %r10,__PT_ARGS(%r11) 689 tm __LC_PGM_ILC+3,0x80 # check for per exception 690 jz 6f 691 tmhh %r8,0x0001 # kernel per event ? 692 jz .Lpgm_kprobe 693 oi __PT_FLAGS+7(%r11),_PIF_PER_TRAP 694 mvc __THREAD_per_address(8,%r14),__LC_PER_ADDRESS 695 mvc __THREAD_per_cause(2,%r14),__LC_PER_CODE 696 mvc __THREAD_per_paid(1,%r14),__LC_PER_ACCESS_ID 6976: RESTORE_SM_CLEAR_PER 698 xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) 699 larl %r1,pgm_check_table 700 llgh %r10,__PT_INT_CODE+2(%r11) 701 nill %r10,0x007f 702 sll %r10,3 703 je .Lpgm_return 704 lg %r9,0(%r10,%r1) # load address of handler routine 705 lgr %r2,%r11 # pass pointer to pt_regs 706 BASR_EX %r14,%r9 # branch to interrupt-handler 707.Lpgm_return: 708 LOCKDEP_SYS_EXIT 709 tm __PT_PSW+1(%r11),0x01 # returning to user ? 710 jno .Lsysc_restore 711 TSTMSK __PT_FLAGS(%r11),_PIF_SYSCALL 712 jo .Lsysc_do_syscall 713 j .Lsysc_tif 714 715# 716# PER event in supervisor state, must be kprobes 717# 718.Lpgm_kprobe: 719 RESTORE_SM_CLEAR_PER 720 xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) 721 lgr %r2,%r11 # pass pointer to pt_regs 722 brasl %r14,do_per_trap 723 j .Lpgm_return 724 725# 726# single stepped system call 727# 728.Lpgm_svcper: 729 mvc __LC_RETURN_PSW(8),__LC_SVC_NEW_PSW 730 larl %r14,.Lsysc_per 731 stg %r14,__LC_RETURN_PSW+8 732 lghi %r14,_PIF_SYSCALL | _PIF_PER_TRAP 733 lpswe __LC_RETURN_PSW # branch to .Lsysc_per 734ENDPROC(pgm_check_handler) 735 736/* 737 * IO interrupt handler routine 738 */ 739ENTRY(io_int_handler) 740 STCK __LC_INT_CLOCK 741 stpt __LC_ASYNC_ENTER_TIMER 742 BPOFF 743 stmg %r8,%r15,__LC_SAVE_AREA_ASYNC 744 lg %r12,__LC_CURRENT 745 lmg %r8,%r9,__LC_IO_OLD_PSW 746 SWITCH_ASYNC __LC_SAVE_AREA_ASYNC,__LC_ASYNC_ENTER_TIMER 747 stmg %r0,%r7,__PT_R0(%r11) 748 # clear user controlled registers to prevent speculative use 749 xgr %r0,%r0 750 xgr %r1,%r1 751 xgr %r2,%r2 752 xgr %r3,%r3 753 xgr %r4,%r4 754 xgr %r5,%r5 755 xgr %r6,%r6 756 xgr %r7,%r7 757 xgr %r10,%r10 758 mvc __PT_R8(64,%r11),__LC_SAVE_AREA_ASYNC 759 stmg %r8,%r9,__PT_PSW(%r11) 760 mvc __PT_INT_CODE(12,%r11),__LC_SUBCHANNEL_ID 761 xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11) 762 TSTMSK __LC_CPU_FLAGS,_CIF_IGNORE_IRQ 763 jo .Lio_restore 764#if IS_ENABLED(CONFIG_TRACE_IRQFLAGS) 765 tmhh %r8,0x300 766 jz 1f 767 TRACE_IRQS_OFF 7681: 769#endif 770 xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) 771.Lio_loop: 772 lgr %r2,%r11 # pass pointer to pt_regs 773 lghi %r3,IO_INTERRUPT 774 tm __PT_INT_CODE+8(%r11),0x80 # adapter interrupt ? 775 jz .Lio_call 776 lghi %r3,THIN_INTERRUPT 777.Lio_call: 778 brasl %r14,do_IRQ 779 TSTMSK __LC_MACHINE_FLAGS,MACHINE_FLAG_LPAR 780 jz .Lio_return 781 tpi 0 782 jz .Lio_return 783 mvc __PT_INT_CODE(12,%r11),__LC_SUBCHANNEL_ID 784 j .Lio_loop 785.Lio_return: 786 LOCKDEP_SYS_EXIT 787 TSTMSK __TI_flags(%r12),_TIF_WORK 788 jnz .Lio_work # there is work to do (signals etc.) 789 TSTMSK __LC_CPU_FLAGS,_CIF_WORK 790 jnz .Lio_work 791.Lio_restore: 792#if IS_ENABLED(CONFIG_TRACE_IRQFLAGS) 793 tm __PT_PSW(%r11),3 794 jno 0f 795 TRACE_IRQS_ON 7960: 797#endif 798 mvc __LC_RETURN_PSW(16),__PT_PSW(%r11) 799 tm __PT_PSW+1(%r11),0x01 # returning to user ? 800 jno .Lio_exit_kernel 801 BPEXIT __TI_flags(%r12),_TIF_ISOLATE_BP 802 stpt __LC_EXIT_TIMER 803.Lio_exit_kernel: 804 lmg %r0,%r15,__PT_R0(%r11) 805 b __LC_RETURN_LPSWE 806.Lio_done: 807 808# 809# There is work todo, find out in which context we have been interrupted: 810# 1) if we return to user space we can do all _TIF_WORK work 811# 2) if we return to kernel code and kvm is enabled check if we need to 812# modify the psw to leave SIE 813# 3) if we return to kernel code and preemptive scheduling is enabled check 814# the preemption counter and if it is zero call preempt_schedule_irq 815# Before any work can be done, a switch to the kernel stack is required. 816# 817.Lio_work: 818 tm __PT_PSW+1(%r11),0x01 # returning to user ? 819 jo .Lio_work_user # yes -> do resched & signal 820#ifdef CONFIG_PREEMPTION 821 # check for preemptive scheduling 822 icm %r0,15,__LC_PREEMPT_COUNT 823 jnz .Lio_restore # preemption is disabled 824 TSTMSK __TI_flags(%r12),_TIF_NEED_RESCHED 825 jno .Lio_restore 826 # switch to kernel stack 827 lg %r1,__PT_R15(%r11) 828 aghi %r1,-(STACK_FRAME_OVERHEAD + __PT_SIZE) 829 mvc STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11) 830 xc __SF_BACKCHAIN(8,%r1),__SF_BACKCHAIN(%r1) 831 la %r11,STACK_FRAME_OVERHEAD(%r1) 832 lgr %r15,%r1 833 brasl %r14,preempt_schedule_irq 834 j .Lio_return 835#else 836 j .Lio_restore 837#endif 838 839# 840# Need to do work before returning to userspace, switch to kernel stack 841# 842.Lio_work_user: 843 lg %r1,__LC_KERNEL_STACK 844 mvc STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11) 845 xc __SF_BACKCHAIN(8,%r1),__SF_BACKCHAIN(%r1) 846 la %r11,STACK_FRAME_OVERHEAD(%r1) 847 lgr %r15,%r1 848 849# 850# One of the work bits is on. Find out which one. 851# 852 TSTMSK __TI_flags(%r12),_TIF_NEED_RESCHED 853 jo .Lio_reschedule 854#ifdef CONFIG_LIVEPATCH 855 TSTMSK __TI_flags(%r12),_TIF_PATCH_PENDING 856 jo .Lio_patch_pending 857#endif 858 TSTMSK __TI_flags(%r12),_TIF_SIGPENDING 859 jo .Lio_sigpending 860 TSTMSK __TI_flags(%r12),_TIF_NOTIFY_RESUME 861 jo .Lio_notify_resume 862 TSTMSK __TI_flags(%r12),_TIF_GUARDED_STORAGE 863 jo .Lio_guarded_storage 864 TSTMSK __LC_CPU_FLAGS,_CIF_FPU 865 jo .Lio_vxrs 866 TSTMSK __LC_CPU_FLAGS,(_CIF_ASCE_PRIMARY|_CIF_ASCE_SECONDARY) 867 jnz .Lio_asce 868 j .Lio_return 869 870# 871# _CIF_ASCE_PRIMARY and/or CIF_ASCE_SECONDARY set, load user space asce 872# 873.Lio_asce: 874 ni __LC_CPU_FLAGS+7,255-_CIF_ASCE_SECONDARY 875 lctlg %c7,%c7,__LC_VDSO_ASCE # load secondary asce 876 TSTMSK __LC_CPU_FLAGS,_CIF_ASCE_PRIMARY 877 jz .Lio_return 878#ifndef CONFIG_HAVE_MARCH_Z10_FEATURES 879 tm __LC_STFLE_FAC_LIST+3,0x10 # has MVCOS ? 880 jnz .Lio_set_fs_fixup 881 ni __LC_CPU_FLAGS+7,255-_CIF_ASCE_PRIMARY 882 lctlg %c1,%c1,__LC_USER_ASCE # load primary asce 883 j .Lio_return 884.Lio_set_fs_fixup: 885#endif 886 larl %r14,.Lio_return 887 jg set_fs_fixup 888 889# 890# CIF_FPU is set, restore floating-point controls and floating-point registers. 891# 892.Lio_vxrs: 893 larl %r14,.Lio_return 894 jg load_fpu_regs 895 896# 897# _TIF_GUARDED_STORAGE is set, call guarded_storage_load 898# 899.Lio_guarded_storage: 900 ENABLE_INTS_TRACE 901 lgr %r2,%r11 # pass pointer to pt_regs 902 brasl %r14,gs_load_bc_cb 903 DISABLE_INTS_TRACE 904 j .Lio_return 905 906# 907# _TIF_NEED_RESCHED is set, call schedule 908# 909.Lio_reschedule: 910 ENABLE_INTS_TRACE 911 brasl %r14,schedule # call scheduler 912 DISABLE_INTS_TRACE 913 j .Lio_return 914 915# 916# _TIF_PATCH_PENDING is set, call klp_update_patch_state 917# 918#ifdef CONFIG_LIVEPATCH 919.Lio_patch_pending: 920 lg %r2,__LC_CURRENT # pass pointer to task struct 921 larl %r14,.Lio_return 922 jg klp_update_patch_state 923#endif 924 925# 926# _TIF_SIGPENDING or is set, call do_signal 927# 928.Lio_sigpending: 929 ENABLE_INTS_TRACE 930 lgr %r2,%r11 # pass pointer to pt_regs 931 brasl %r14,do_signal 932 DISABLE_INTS_TRACE 933 j .Lio_return 934 935# 936# _TIF_NOTIFY_RESUME or is set, call do_notify_resume 937# 938.Lio_notify_resume: 939 ENABLE_INTS_TRACE 940 lgr %r2,%r11 # pass pointer to pt_regs 941 brasl %r14,do_notify_resume 942 DISABLE_INTS_TRACE 943 j .Lio_return 944ENDPROC(io_int_handler) 945 946/* 947 * External interrupt handler routine 948 */ 949ENTRY(ext_int_handler) 950 STCK __LC_INT_CLOCK 951 stpt __LC_ASYNC_ENTER_TIMER 952 BPOFF 953 stmg %r8,%r15,__LC_SAVE_AREA_ASYNC 954 lg %r12,__LC_CURRENT 955 lmg %r8,%r9,__LC_EXT_OLD_PSW 956 SWITCH_ASYNC __LC_SAVE_AREA_ASYNC,__LC_ASYNC_ENTER_TIMER 957 stmg %r0,%r7,__PT_R0(%r11) 958 # clear user controlled registers to prevent speculative use 959 xgr %r0,%r0 960 xgr %r1,%r1 961 xgr %r2,%r2 962 xgr %r3,%r3 963 xgr %r4,%r4 964 xgr %r5,%r5 965 xgr %r6,%r6 966 xgr %r7,%r7 967 xgr %r10,%r10 968 mvc __PT_R8(64,%r11),__LC_SAVE_AREA_ASYNC 969 stmg %r8,%r9,__PT_PSW(%r11) 970 lghi %r1,__LC_EXT_PARAMS2 971 mvc __PT_INT_CODE(4,%r11),__LC_EXT_CPU_ADDR 972 mvc __PT_INT_PARM(4,%r11),__LC_EXT_PARAMS 973 mvc __PT_INT_PARM_LONG(8,%r11),0(%r1) 974 xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11) 975 TSTMSK __LC_CPU_FLAGS,_CIF_IGNORE_IRQ 976 jo .Lio_restore 977#if IS_ENABLED(CONFIG_TRACE_IRQFLAGS) 978 tmhh %r8,0x300 979 jz 1f 980 TRACE_IRQS_OFF 9811: 982#endif 983 xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) 984 lgr %r2,%r11 # pass pointer to pt_regs 985 lghi %r3,EXT_INTERRUPT 986 brasl %r14,do_IRQ 987 j .Lio_return 988ENDPROC(ext_int_handler) 989 990/* 991 * Load idle PSW. 992 */ 993ENTRY(psw_idle) 994 stg %r3,__SF_EMPTY(%r15) 995 larl %r1,.Lpsw_idle_exit 996 stg %r1,__SF_EMPTY+8(%r15) 997 larl %r1,smp_cpu_mtid 998 llgf %r1,0(%r1) 999 ltgr %r1,%r1 1000 jz .Lpsw_idle_stcctm 1001 .insn rsy,0xeb0000000017,%r1,5,__SF_EMPTY+16(%r15) 1002.Lpsw_idle_stcctm: 1003 oi __LC_CPU_FLAGS+7,_CIF_ENABLED_WAIT 1004 BPON 1005 STCK __CLOCK_IDLE_ENTER(%r2) 1006 stpt __TIMER_IDLE_ENTER(%r2) 1007 lpswe __SF_EMPTY(%r15) 1008.Lpsw_idle_exit: 1009 BR_EX %r14 1010ENDPROC(psw_idle) 1011 1012/* 1013 * Store floating-point controls and floating-point or vector register 1014 * depending whether the vector facility is available. A critical section 1015 * cleanup assures that the registers are stored even if interrupted for 1016 * some other work. The CIF_FPU flag is set to trigger a lazy restore 1017 * of the register contents at return from io or a system call. 1018 */ 1019ENTRY(save_fpu_regs) 1020 stnsm __SF_EMPTY(%r15),0xfc 1021 lg %r2,__LC_CURRENT 1022 aghi %r2,__TASK_thread 1023 TSTMSK __LC_CPU_FLAGS,_CIF_FPU 1024 jo .Lsave_fpu_regs_exit 1025 stfpc __THREAD_FPU_fpc(%r2) 1026 lg %r3,__THREAD_FPU_regs(%r2) 1027 TSTMSK __LC_MACHINE_FLAGS,MACHINE_FLAG_VX 1028 jz .Lsave_fpu_regs_fp # no -> store FP regs 1029 VSTM %v0,%v15,0,%r3 # vstm 0,15,0(3) 1030 VSTM %v16,%v31,256,%r3 # vstm 16,31,256(3) 1031 j .Lsave_fpu_regs_done # -> set CIF_FPU flag 1032.Lsave_fpu_regs_fp: 1033 std 0,0(%r3) 1034 std 1,8(%r3) 1035 std 2,16(%r3) 1036 std 3,24(%r3) 1037 std 4,32(%r3) 1038 std 5,40(%r3) 1039 std 6,48(%r3) 1040 std 7,56(%r3) 1041 std 8,64(%r3) 1042 std 9,72(%r3) 1043 std 10,80(%r3) 1044 std 11,88(%r3) 1045 std 12,96(%r3) 1046 std 13,104(%r3) 1047 std 14,112(%r3) 1048 std 15,120(%r3) 1049.Lsave_fpu_regs_done: 1050 oi __LC_CPU_FLAGS+7,_CIF_FPU 1051.Lsave_fpu_regs_exit: 1052 ssm __SF_EMPTY(%r15) 1053 BR_EX %r14 1054.Lsave_fpu_regs_end: 1055ENDPROC(save_fpu_regs) 1056EXPORT_SYMBOL(save_fpu_regs) 1057 1058/* 1059 * Load floating-point controls and floating-point or vector registers. 1060 * A critical section cleanup assures that the register contents are 1061 * loaded even if interrupted for some other work. 1062 * 1063 * There are special calling conventions to fit into sysc and io return work: 1064 * %r15: <kernel stack> 1065 * The function requires: 1066 * %r4 1067 */ 1068load_fpu_regs: 1069 lg %r4,__LC_CURRENT 1070 aghi %r4,__TASK_thread 1071 TSTMSK __LC_CPU_FLAGS,_CIF_FPU 1072 jno .Lload_fpu_regs_exit 1073 lfpc __THREAD_FPU_fpc(%r4) 1074 TSTMSK __LC_MACHINE_FLAGS,MACHINE_FLAG_VX 1075 lg %r4,__THREAD_FPU_regs(%r4) # %r4 <- reg save area 1076 jz .Lload_fpu_regs_fp # -> no VX, load FP regs 1077 VLM %v0,%v15,0,%r4 1078 VLM %v16,%v31,256,%r4 1079 j .Lload_fpu_regs_done 1080.Lload_fpu_regs_fp: 1081 ld 0,0(%r4) 1082 ld 1,8(%r4) 1083 ld 2,16(%r4) 1084 ld 3,24(%r4) 1085 ld 4,32(%r4) 1086 ld 5,40(%r4) 1087 ld 6,48(%r4) 1088 ld 7,56(%r4) 1089 ld 8,64(%r4) 1090 ld 9,72(%r4) 1091 ld 10,80(%r4) 1092 ld 11,88(%r4) 1093 ld 12,96(%r4) 1094 ld 13,104(%r4) 1095 ld 14,112(%r4) 1096 ld 15,120(%r4) 1097.Lload_fpu_regs_done: 1098 ni __LC_CPU_FLAGS+7,255-_CIF_FPU 1099.Lload_fpu_regs_exit: 1100 BR_EX %r14 1101.Lload_fpu_regs_end: 1102ENDPROC(load_fpu_regs) 1103 1104/* 1105 * Machine check handler routines 1106 */ 1107ENTRY(mcck_int_handler) 1108 STCK __LC_MCCK_CLOCK 1109 BPOFF 1110 la %r1,4095 # validate r1 1111 spt __LC_CPU_TIMER_SAVE_AREA-4095(%r1) # validate cpu timer 1112 sckc __LC_CLOCK_COMPARATOR # validate comparator 1113 lam %a0,%a15,__LC_AREGS_SAVE_AREA-4095(%r1) # validate acrs 1114 lmg %r0,%r15,__LC_GPREGS_SAVE_AREA-4095(%r1)# validate gprs 1115 lg %r12,__LC_CURRENT 1116 lmg %r8,%r9,__LC_MCK_OLD_PSW 1117 TSTMSK __LC_MCCK_CODE,MCCK_CODE_SYSTEM_DAMAGE 1118 jo .Lmcck_panic # yes -> rest of mcck code invalid 1119 TSTMSK __LC_MCCK_CODE,MCCK_CODE_CR_VALID 1120 jno .Lmcck_panic # control registers invalid -> panic 1121 la %r14,4095 1122 lctlg %c0,%c15,__LC_CREGS_SAVE_AREA-4095(%r14) # validate ctl regs 1123 ptlb 1124 lg %r11,__LC_MCESAD-4095(%r14) # extended machine check save area 1125 nill %r11,0xfc00 # MCESA_ORIGIN_MASK 1126 TSTMSK __LC_CREGS_SAVE_AREA+16-4095(%r14),CR2_GUARDED_STORAGE 1127 jno 0f 1128 TSTMSK __LC_MCCK_CODE,MCCK_CODE_GS_VALID 1129 jno 0f 1130 .insn rxy,0xe3000000004d,0,__MCESA_GS_SAVE_AREA(%r11) # LGSC 11310: l %r14,__LC_FP_CREG_SAVE_AREA-4095(%r14) 1132 TSTMSK __LC_MCCK_CODE,MCCK_CODE_FC_VALID 1133 jo 0f 1134 sr %r14,%r14 11350: sfpc %r14 1136 TSTMSK __LC_MACHINE_FLAGS,MACHINE_FLAG_VX 1137 jo 0f 1138 lghi %r14,__LC_FPREGS_SAVE_AREA 1139 ld %f0,0(%r14) 1140 ld %f1,8(%r14) 1141 ld %f2,16(%r14) 1142 ld %f3,24(%r14) 1143 ld %f4,32(%r14) 1144 ld %f5,40(%r14) 1145 ld %f6,48(%r14) 1146 ld %f7,56(%r14) 1147 ld %f8,64(%r14) 1148 ld %f9,72(%r14) 1149 ld %f10,80(%r14) 1150 ld %f11,88(%r14) 1151 ld %f12,96(%r14) 1152 ld %f13,104(%r14) 1153 ld %f14,112(%r14) 1154 ld %f15,120(%r14) 1155 j 1f 11560: VLM %v0,%v15,0,%r11 1157 VLM %v16,%v31,256,%r11 11581: lghi %r14,__LC_CPU_TIMER_SAVE_AREA 1159 mvc __LC_MCCK_ENTER_TIMER(8),0(%r14) 1160 TSTMSK __LC_MCCK_CODE,MCCK_CODE_CPU_TIMER_VALID 1161 jo 3f 1162 la %r14,__LC_SYNC_ENTER_TIMER 1163 clc 0(8,%r14),__LC_ASYNC_ENTER_TIMER 1164 jl 0f 1165 la %r14,__LC_ASYNC_ENTER_TIMER 11660: clc 0(8,%r14),__LC_EXIT_TIMER 1167 jl 1f 1168 la %r14,__LC_EXIT_TIMER 11691: clc 0(8,%r14),__LC_LAST_UPDATE_TIMER 1170 jl 2f 1171 la %r14,__LC_LAST_UPDATE_TIMER 11722: spt 0(%r14) 1173 mvc __LC_MCCK_ENTER_TIMER(8),0(%r14) 11743: TSTMSK __LC_MCCK_CODE,MCCK_CODE_PSW_MWP_VALID 1175 jno .Lmcck_panic 1176 tmhh %r8,0x0001 # interrupting from user ? 1177 jnz 4f 1178 TSTMSK __LC_MCCK_CODE,MCCK_CODE_PSW_IA_VALID 1179 jno .Lmcck_panic 11804: ssm __LC_PGM_NEW_PSW # turn dat on, keep irqs off 1181 SWITCH_ASYNC __LC_GPREGS_SAVE_AREA+64,__LC_MCCK_ENTER_TIMER 1182.Lmcck_skip: 1183 lghi %r14,__LC_GPREGS_SAVE_AREA+64 1184 stmg %r0,%r7,__PT_R0(%r11) 1185 # clear user controlled registers to prevent speculative use 1186 xgr %r0,%r0 1187 xgr %r1,%r1 1188 xgr %r2,%r2 1189 xgr %r3,%r3 1190 xgr %r4,%r4 1191 xgr %r5,%r5 1192 xgr %r6,%r6 1193 xgr %r7,%r7 1194 xgr %r10,%r10 1195 mvc __PT_R8(64,%r11),0(%r14) 1196 stmg %r8,%r9,__PT_PSW(%r11) 1197 xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11) 1198 xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) 1199 lgr %r2,%r11 # pass pointer to pt_regs 1200 brasl %r14,s390_do_machine_check 1201 cghi %r2,0 1202 je .Lmcck_return 1203 lg %r1,__LC_KERNEL_STACK # switch to kernel stack 1204 mvc STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11) 1205 xc __SF_BACKCHAIN(8,%r1),__SF_BACKCHAIN(%r1) 1206 la %r11,STACK_FRAME_OVERHEAD(%r1) 1207 lgr %r15,%r1 1208 TRACE_IRQS_OFF 1209 brasl %r14,s390_handle_mcck 1210 TRACE_IRQS_ON 1211.Lmcck_return: 1212 lmg %r0,%r10,__PT_R0(%r11) 1213 mvc __LC_RETURN_MCCK_PSW(16),__PT_PSW(%r11) # move return PSW 1214 tm __LC_RETURN_MCCK_PSW+1,0x01 # returning to user ? 1215 jno 0f 1216 BPEXIT __TI_flags(%r12),_TIF_ISOLATE_BP 1217 stpt __LC_EXIT_TIMER 12180: lmg %r11,%r15,__PT_R11(%r11) 1219 b __LC_RETURN_MCCK_LPSWE 1220 1221.Lmcck_panic: 1222 lg %r15,__LC_NODAT_STACK 1223 la %r11,STACK_FRAME_OVERHEAD(%r15) 1224 j .Lmcck_skip 1225ENDPROC(mcck_int_handler) 1226 1227# 1228# PSW restart interrupt handler 1229# 1230ENTRY(restart_int_handler) 1231 ALTERNATIVE "", ".insn s,0xb2800000,_LPP_OFFSET", 40 1232 stg %r15,__LC_SAVE_AREA_RESTART 1233 lg %r15,__LC_RESTART_STACK 1234 xc STACK_FRAME_OVERHEAD(__PT_SIZE,%r15),STACK_FRAME_OVERHEAD(%r15) 1235 stmg %r0,%r14,STACK_FRAME_OVERHEAD+__PT_R0(%r15) 1236 mvc STACK_FRAME_OVERHEAD+__PT_R15(8,%r15),__LC_SAVE_AREA_RESTART 1237 mvc STACK_FRAME_OVERHEAD+__PT_PSW(16,%r15),__LC_RST_OLD_PSW 1238 xc 0(STACK_FRAME_OVERHEAD,%r15),0(%r15) 1239 lg %r1,__LC_RESTART_FN # load fn, parm & source cpu 1240 lg %r2,__LC_RESTART_DATA 1241 lg %r3,__LC_RESTART_SOURCE 1242 ltgr %r3,%r3 # test source cpu address 1243 jm 1f # negative -> skip source stop 12440: sigp %r4,%r3,SIGP_SENSE # sigp sense to source cpu 1245 brc 10,0b # wait for status stored 12461: basr %r14,%r1 # call function 1247 stap __SF_EMPTY(%r15) # store cpu address 1248 llgh %r3,__SF_EMPTY(%r15) 12492: sigp %r4,%r3,SIGP_STOP # sigp stop to current cpu 1250 brc 2,2b 12513: j 3b 1252ENDPROC(restart_int_handler) 1253 1254 .section .kprobes.text, "ax" 1255 1256#if defined(CONFIG_CHECK_STACK) || defined(CONFIG_VMAP_STACK) 1257/* 1258 * The synchronous or the asynchronous stack overflowed. We are dead. 1259 * No need to properly save the registers, we are going to panic anyway. 1260 * Setup a pt_regs so that show_trace can provide a good call trace. 1261 */ 1262ENTRY(stack_overflow) 1263 lg %r15,__LC_NODAT_STACK # change to panic stack 1264 la %r11,STACK_FRAME_OVERHEAD(%r15) 1265 stmg %r0,%r7,__PT_R0(%r11) 1266 stmg %r8,%r9,__PT_PSW(%r11) 1267 mvc __PT_R8(64,%r11),0(%r14) 1268 stg %r10,__PT_ORIG_GPR2(%r11) # store last break to orig_gpr2 1269 xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) 1270 lgr %r2,%r11 # pass pointer to pt_regs 1271 jg kernel_stack_overflow 1272ENDPROC(stack_overflow) 1273#endif 1274 1275#if IS_ENABLED(CONFIG_KVM) 1276.Lcleanup_sie: 1277 cghi %r11,__LC_SAVE_AREA_ASYNC #Is this in normal interrupt? 1278 je 1f 1279 larl %r13,.Lsie_entry 1280 slgr %r9,%r13 1281 larl %r13,.Lsie_skip 1282 clgr %r9,%r13 1283 jh 1f 1284 oi __LC_CPU_FLAGS+7, _CIF_MCCK_GUEST 12851: BPENTER __SF_SIE_FLAGS(%r15),(_TIF_ISOLATE_BP|_TIF_ISOLATE_BP_GUEST) 1286 lg %r9,__SF_SIE_CONTROL(%r15) # get control block pointer 1287 ni __SIE_PROG0C+3(%r9),0xfe # no longer in SIE 1288 lctlg %c1,%c1,__LC_USER_ASCE # load primary asce 1289 larl %r9,sie_exit # skip forward to sie_exit 1290 BR_EX %r14,%r11 1291 1292#endif 1293 .section .rodata, "a" 1294#define SYSCALL(esame,emu) .quad __s390x_ ## esame 1295 .globl sys_call_table 1296sys_call_table: 1297#include "asm/syscall_table.h" 1298#undef SYSCALL 1299 1300#ifdef CONFIG_COMPAT 1301 1302#define SYSCALL(esame,emu) .quad __s390_ ## emu 1303 .globl sys_call_table_emu 1304sys_call_table_emu: 1305#include "asm/syscall_table.h" 1306#undef SYSCALL 1307#endif 1308