1b2441318SGreg Kroah-Hartman/* SPDX-License-Identifier: GPL-2.0 */ 24bfc86ceSHeiko Carstens/* 34bfc86ceSHeiko Carstens * S390 low-level entry points. 44bfc86ceSHeiko Carstens * 54bfc86ceSHeiko Carstens * Copyright IBM Corp. 1999, 2012 64bfc86ceSHeiko Carstens * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com), 74bfc86ceSHeiko Carstens * Hartmut Penner (hp@de.ibm.com), 84bfc86ceSHeiko Carstens * Denis Joseph Barrow (djbarrow@de.ibm.com,barrow_dj@yahoo.com), 94bfc86ceSHeiko Carstens * Heiko Carstens <heiko.carstens@de.ibm.com> 104bfc86ceSHeiko Carstens */ 114bfc86ceSHeiko Carstens 124bfc86ceSHeiko Carstens#include <linux/init.h> 134bfc86ceSHeiko Carstens#include <linux/linkage.h> 14b058661aSMartin Schwidefsky#include <asm/alternative-asm.h> 154bfc86ceSHeiko Carstens#include <asm/processor.h> 164bfc86ceSHeiko Carstens#include <asm/cache.h> 173037a52fSMartin Schwidefsky#include <asm/ctl_reg.h> 18dc24b7b4SHendrik Brueckner#include <asm/dwarf.h> 194bfc86ceSHeiko Carstens#include <asm/errno.h> 204bfc86ceSHeiko Carstens#include <asm/ptrace.h> 214bfc86ceSHeiko Carstens#include <asm/thread_info.h> 224bfc86ceSHeiko Carstens#include <asm/asm-offsets.h> 234bfc86ceSHeiko Carstens#include <asm/unistd.h> 244bfc86ceSHeiko Carstens#include <asm/page.h> 254bfc86ceSHeiko Carstens#include <asm/sigp.h> 264bfc86ceSHeiko Carstens#include <asm/irq.h> 279977e886SHendrik Brueckner#include <asm/vx-insn.h> 2883abeffbSHendrik Brueckner#include <asm/setup.h> 2983abeffbSHendrik Brueckner#include <asm/nmi.h> 30711f5df7SAl Viro#include <asm/export.h> 316dd85fbbSMartin Schwidefsky#include <asm/nospec-insn.h> 324bfc86ceSHeiko Carstens 334bfc86ceSHeiko Carstens__PT_R0 = __PT_GPRS 344bfc86ceSHeiko Carstens__PT_R1 = __PT_GPRS + 8 354bfc86ceSHeiko Carstens__PT_R2 = __PT_GPRS + 16 364bfc86ceSHeiko Carstens__PT_R3 = __PT_GPRS + 24 374bfc86ceSHeiko Carstens__PT_R4 = __PT_GPRS + 32 384bfc86ceSHeiko Carstens__PT_R5 = __PT_GPRS + 40 394bfc86ceSHeiko Carstens__PT_R6 = __PT_GPRS + 48 404bfc86ceSHeiko Carstens__PT_R7 = __PT_GPRS + 56 414bfc86ceSHeiko Carstens__PT_R8 = __PT_GPRS + 64 424bfc86ceSHeiko Carstens__PT_R9 = __PT_GPRS + 72 434bfc86ceSHeiko Carstens__PT_R10 = __PT_GPRS + 80 444bfc86ceSHeiko Carstens__PT_R11 = __PT_GPRS + 88 454bfc86ceSHeiko Carstens__PT_R12 = __PT_GPRS + 96 464bfc86ceSHeiko Carstens__PT_R13 = __PT_GPRS + 104 474bfc86ceSHeiko Carstens__PT_R14 = __PT_GPRS + 112 484bfc86ceSHeiko Carstens__PT_R15 = __PT_GPRS + 120 494bfc86ceSHeiko Carstens 503a890380SHeiko CarstensSTACK_SHIFT = PAGE_SHIFT + THREAD_SIZE_ORDER 514bfc86ceSHeiko CarstensSTACK_SIZE = 1 << STACK_SHIFT 524bfc86ceSHeiko CarstensSTACK_INIT = STACK_SIZE - STACK_FRAME_OVERHEAD - __PT_SIZE 534bfc86ceSHeiko Carstens 54e5b98199SMartin Schwidefsky_LPP_OFFSET = __LC_LPP 55e5b98199SMartin Schwidefsky 56ce3dc447SMartin Schwidefsky .macro CHECK_STACK savearea 574bfc86ceSHeiko Carstens#ifdef CONFIG_CHECK_STACK 58ce3dc447SMartin Schwidefsky tml %r15,STACK_SIZE - CONFIG_STACK_GUARD 594bfc86ceSHeiko Carstens lghi %r14,\savearea 604bfc86ceSHeiko Carstens jz stack_overflow 614bfc86ceSHeiko Carstens#endif 624bfc86ceSHeiko Carstens .endm 634bfc86ceSHeiko Carstens 64ce3dc447SMartin Schwidefsky .macro CHECK_VMAP_STACK savearea,oklabel 65ce3dc447SMartin Schwidefsky#ifdef CONFIG_VMAP_STACK 66ce3dc447SMartin Schwidefsky lgr %r14,%r15 67ce3dc447SMartin Schwidefsky nill %r14,0x10000 - STACK_SIZE 68ce3dc447SMartin Schwidefsky oill %r14,STACK_INIT 69ce3dc447SMartin Schwidefsky clg %r14,__LC_KERNEL_STACK 70ce3dc447SMartin Schwidefsky je \oklabel 71ce3dc447SMartin Schwidefsky clg %r14,__LC_ASYNC_STACK 72ce3dc447SMartin Schwidefsky je \oklabel 73b61b1595SSven Schnelle clg %r14,__LC_MCCK_STACK 74b61b1595SSven Schnelle je \oklabel 75ce3dc447SMartin Schwidefsky clg %r14,__LC_NODAT_STACK 76ce3dc447SMartin Schwidefsky je \oklabel 77ce3dc447SMartin Schwidefsky clg %r14,__LC_RESTART_STACK 78ce3dc447SMartin Schwidefsky je \oklabel 79ce3dc447SMartin Schwidefsky lghi %r14,\savearea 80ce3dc447SMartin Schwidefsky j stack_overflow 81ce3dc447SMartin Schwidefsky#else 82ce3dc447SMartin Schwidefsky j \oklabel 83ce3dc447SMartin Schwidefsky#endif 84ce3dc447SMartin Schwidefsky .endm 85ce3dc447SMartin Schwidefsky 864bfc86ceSHeiko Carstens .macro STCK savearea 8778f65709SHeiko Carstens ALTERNATIVE ".insn s,0xb2050000,\savearea", \ 8878f65709SHeiko Carstens ".insn s,0xb27c0000,\savearea", 25 894bfc86ceSHeiko Carstens .endm 904bfc86ceSHeiko Carstens 9183abeffbSHendrik Brueckner /* 9283abeffbSHendrik Brueckner * The TSTMSK macro generates a test-under-mask instruction by 9383abeffbSHendrik Brueckner * calculating the memory offset for the specified mask value. 9483abeffbSHendrik Brueckner * Mask value can be any constant. The macro shifts the mask 9583abeffbSHendrik Brueckner * value to calculate the memory offset for the test-under-mask 9683abeffbSHendrik Brueckner * instruction. 9783abeffbSHendrik Brueckner */ 9883abeffbSHendrik Brueckner .macro TSTMSK addr, mask, size=8, bytepos=0 9983abeffbSHendrik Brueckner .if (\bytepos < \size) && (\mask >> 8) 10083abeffbSHendrik Brueckner .if (\mask & 0xff) 10183abeffbSHendrik Brueckner .error "Mask exceeds byte boundary" 10283abeffbSHendrik Brueckner .endif 10383abeffbSHendrik Brueckner TSTMSK \addr, "(\mask >> 8)", \size, "(\bytepos + 1)" 10483abeffbSHendrik Brueckner .exitm 10583abeffbSHendrik Brueckner .endif 10683abeffbSHendrik Brueckner .ifeq \mask 10783abeffbSHendrik Brueckner .error "Mask must not be zero" 10883abeffbSHendrik Brueckner .endif 10983abeffbSHendrik Brueckner off = \size - \bytepos - 1 11083abeffbSHendrik Brueckner tm off+\addr, \mask 11183abeffbSHendrik Brueckner .endm 11283abeffbSHendrik Brueckner 113d768bd89SMartin Schwidefsky .macro BPOFF 114b058661aSMartin Schwidefsky ALTERNATIVE "", ".long 0xb2e8c000", 82 115d768bd89SMartin Schwidefsky .endm 116d768bd89SMartin Schwidefsky 117d768bd89SMartin Schwidefsky .macro BPON 118b058661aSMartin Schwidefsky ALTERNATIVE "", ".long 0xb2e8d000", 82 119d768bd89SMartin Schwidefsky .endm 120d768bd89SMartin Schwidefsky 1216b73044bSMartin Schwidefsky .macro BPENTER tif_ptr,tif_mask 122b058661aSMartin Schwidefsky ALTERNATIVE "TSTMSK \tif_ptr,\tif_mask; jz .+8; .long 0xb2e8d000", \ 123b058661aSMartin Schwidefsky "", 82 1246b73044bSMartin Schwidefsky .endm 1256b73044bSMartin Schwidefsky 1266b73044bSMartin Schwidefsky .macro BPEXIT tif_ptr,tif_mask 1276b73044bSMartin Schwidefsky TSTMSK \tif_ptr,\tif_mask 128b058661aSMartin Schwidefsky ALTERNATIVE "jz .+8; .long 0xb2e8c000", \ 129b058661aSMartin Schwidefsky "jnz .+8; .long 0xb2e8d000", 82 1306b73044bSMartin Schwidefsky .endm 1316b73044bSMartin Schwidefsky 132*d35925b3SAlexander Gordeev /* 133*d35925b3SAlexander Gordeev * The CHKSTG macro jumps to the provided label in case the 134*d35925b3SAlexander Gordeev * machine check interruption code reports one of unrecoverable 135*d35925b3SAlexander Gordeev * storage errors: 136*d35925b3SAlexander Gordeev * - Storage error uncorrected 137*d35925b3SAlexander Gordeev * - Storage key error uncorrected 138*d35925b3SAlexander Gordeev * - Storage degradation with Failing-storage-address validity 139*d35925b3SAlexander Gordeev */ 140*d35925b3SAlexander Gordeev .macro CHKSTG errlabel 141*d35925b3SAlexander Gordeev TSTMSK __LC_MCCK_CODE,(MCCK_CODE_STG_ERROR|MCCK_CODE_STG_KEY_ERROR) 142*d35925b3SAlexander Gordeev jnz \errlabel 143*d35925b3SAlexander Gordeev TSTMSK __LC_MCCK_CODE,MCCK_CODE_STG_DEGRAD 144*d35925b3SAlexander Gordeev jz oklabel\@ 145*d35925b3SAlexander Gordeev TSTMSK __LC_MCCK_CODE,MCCK_CODE_STG_FAIL_ADDR 146*d35925b3SAlexander Gordeev jnz \errlabel 147*d35925b3SAlexander Gordeevoklabel\@: 148*d35925b3SAlexander Gordeev .endm 149*d35925b3SAlexander Gordeev 150b5415c8fSAlexander Gordeev#if IS_ENABLED(CONFIG_KVM) 151b5415c8fSAlexander Gordeev /* 152b5415c8fSAlexander Gordeev * The OUTSIDE macro jumps to the provided label in case the value 153b5415c8fSAlexander Gordeev * in the provided register is outside of the provided range. The 154b5415c8fSAlexander Gordeev * macro is useful for checking whether a PSW stored in a register 155b5415c8fSAlexander Gordeev * pair points inside or outside of a block of instructions. 156b5415c8fSAlexander Gordeev * @reg: register to check 157b5415c8fSAlexander Gordeev * @start: start of the range 158b5415c8fSAlexander Gordeev * @end: end of the range 159b5415c8fSAlexander Gordeev * @outside_label: jump here if @reg is outside of [@start..@end) 160b5415c8fSAlexander Gordeev */ 161b5415c8fSAlexander Gordeev .macro OUTSIDE reg,start,end,outside_label 162b5415c8fSAlexander Gordeev lgr %r14,\reg 163b5415c8fSAlexander Gordeev larl %r13,\start 164b5415c8fSAlexander Gordeev slgr %r14,%r13 165b5415c8fSAlexander Gordeev lghi %r13,\end - \start 166b5415c8fSAlexander Gordeev clgr %r14,%r13 167b5415c8fSAlexander Gordeev jhe \outside_label 168b5415c8fSAlexander Gordeev .endm 169fbbdfca5SAlexander Gordeev 170fbbdfca5SAlexander Gordeev .macro SIEEXIT 171fbbdfca5SAlexander Gordeev lg %r9,__SF_SIE_CONTROL(%r15) # get control block pointer 172fbbdfca5SAlexander Gordeev ni __SIE_PROG0C+3(%r9),0xfe # no longer in SIE 173fbbdfca5SAlexander Gordeev lctlg %c1,%c1,__LC_KERNEL_ASCE # load primary asce 174fbbdfca5SAlexander Gordeev larl %r9,sie_exit # skip forward to sie_exit 175fbbdfca5SAlexander Gordeev .endm 176b5415c8fSAlexander Gordeev#endif 177b5415c8fSAlexander Gordeev 1786dd85fbbSMartin Schwidefsky GEN_BR_THUNK %r14 17933ea0487SSven Schnelle GEN_BR_THUNK %r14,%r13 180f19fbd5eSMartin Schwidefsky 1814bfc86ceSHeiko Carstens .section .kprobes.text, "ax" 18246210c44SHeiko Carstens.Ldummy: 18346210c44SHeiko Carstens /* 18456e62a73SSven Schnelle * This nop exists only in order to avoid that __bpon starts at 18546210c44SHeiko Carstens * the beginning of the kprobes text section. In that case we would 18646210c44SHeiko Carstens * have several symbols at the same address. E.g. objdump would take 18746210c44SHeiko Carstens * an arbitrary symbol name when disassembling this code. 18856e62a73SSven Schnelle * With the added nop in between the __bpon symbol is unique 18946210c44SHeiko Carstens * again. 19046210c44SHeiko Carstens */ 19146210c44SHeiko Carstens nop 0 1924bfc86ceSHeiko Carstens 193d768bd89SMartin SchwidefskyENTRY(__bpon) 194d768bd89SMartin Schwidefsky .globl __bpon 195d768bd89SMartin Schwidefsky BPON 1966dd85fbbSMartin Schwidefsky BR_EX %r14 19726a374aeSMartin SchwidefskyENDPROC(__bpon) 198d768bd89SMartin Schwidefsky 1994bfc86ceSHeiko Carstens/* 2004bfc86ceSHeiko Carstens * Scheduler resume function, called by switch_to 2014bfc86ceSHeiko Carstens * gpr2 = (task_struct *) prev 2024bfc86ceSHeiko Carstens * gpr3 = (task_struct *) next 2034bfc86ceSHeiko Carstens * Returns: 2044bfc86ceSHeiko Carstens * gpr2 = prev 2054bfc86ceSHeiko Carstens */ 2064bfc86ceSHeiko CarstensENTRY(__switch_to) 2074bfc86ceSHeiko Carstens stmg %r6,%r15,__SF_GPRS(%r15) # store gprs of prev task 2083241d3ebSHeiko Carstens lghi %r4,__TASK_stack 2093241d3ebSHeiko Carstens lghi %r1,__TASK_thread 2109fed920eSVasily Gorbik llill %r5,STACK_INIT 2113241d3ebSHeiko Carstens stg %r15,__THREAD_ksp(%r1,%r2) # store kernel stack of prev 2129fed920eSVasily Gorbik lg %r15,0(%r4,%r3) # start of kernel stack of next 2139fed920eSVasily Gorbik agr %r15,%r5 # end of kernel stack of next 2144bfc86ceSHeiko Carstens stg %r3,__LC_CURRENT # store task struct of next 2154bfc86ceSHeiko Carstens stg %r15,__LC_KERNEL_STACK # store end of kernel stack 2163241d3ebSHeiko Carstens lg %r15,__THREAD_ksp(%r1,%r3) # load kernel stack of next 2173241d3ebSHeiko Carstens aghi %r3,__TASK_pid 2183241d3ebSHeiko Carstens mvc __LC_CURRENT_PID(4,%r0),0(%r3) # store pid of next 2194bfc86ceSHeiko Carstens lmg %r6,%r15,__SF_GPRS(%r15) # load gprs of next task 220e5b98199SMartin Schwidefsky ALTERNATIVE "", ".insn s,0xb2800000,_LPP_OFFSET", 40 2216dd85fbbSMartin Schwidefsky BR_EX %r14 22226a374aeSMartin SchwidefskyENDPROC(__switch_to) 2234bfc86ceSHeiko Carstens 224d0fc4107SMartin Schwidefsky#if IS_ENABLED(CONFIG_KVM) 225d0fc4107SMartin Schwidefsky/* 226d0fc4107SMartin Schwidefsky * sie64a calling convention: 227d0fc4107SMartin Schwidefsky * %r2 pointer to sie control block 228d0fc4107SMartin Schwidefsky * %r3 guest register save area 229d0fc4107SMartin Schwidefsky */ 230d0fc4107SMartin SchwidefskyENTRY(sie64a) 231d0fc4107SMartin Schwidefsky stmg %r6,%r14,__SF_GPRS(%r15) # save kernel registers 2326b73044bSMartin Schwidefsky lg %r12,__LC_CURRENT 23392fa7a13SMartin Schwidefsky stg %r2,__SF_SIE_CONTROL(%r15) # save control block pointer 23492fa7a13SMartin Schwidefsky stg %r3,__SF_SIE_SAVEAREA(%r15) # save guest register save area 23592fa7a13SMartin Schwidefsky xc __SF_SIE_REASON(8,%r15),__SF_SIE_REASON(%r15) # reason code = 0 23692fa7a13SMartin Schwidefsky mvc __SF_SIE_FLAGS(8,%r15),__TI_flags(%r12) # copy thread flags 237d0fc4107SMartin Schwidefsky lmg %r0,%r13,0(%r3) # load guest gprs 0-13 238d0fc4107SMartin Schwidefsky lg %r14,__LC_GMAP # get gmap pointer 239d0fc4107SMartin Schwidefsky ltgr %r14,%r14 240d0fc4107SMartin Schwidefsky jz .Lsie_gmap 241d0fc4107SMartin Schwidefsky lctlg %c1,%c1,__GMAP_ASCE(%r14) # load primary asce 242d0fc4107SMartin Schwidefsky.Lsie_gmap: 24392fa7a13SMartin Schwidefsky lg %r14,__SF_SIE_CONTROL(%r15) # get control block pointer 244d0fc4107SMartin Schwidefsky oi __SIE_PROG0C+3(%r14),1 # we are going into SIE now 245d0fc4107SMartin Schwidefsky tm __SIE_PROG20+3(%r14),3 # last exit... 246d0fc4107SMartin Schwidefsky jnz .Lsie_skip 24783abeffbSHendrik Brueckner TSTMSK __LC_CPU_FLAGS,_CIF_FPU 248d0fc4107SMartin Schwidefsky jo .Lsie_skip # exit if fp/vx regs changed 24992fa7a13SMartin Schwidefsky BPEXIT __SF_SIE_FLAGS(%r15),(_TIF_ISOLATE_BP|_TIF_ISOLATE_BP_GUEST) 250c929500dSQingFeng Hao.Lsie_entry: 251d0fc4107SMartin Schwidefsky sie 0(%r14) 252d768bd89SMartin Schwidefsky BPOFF 25392fa7a13SMartin Schwidefsky BPENTER __SF_SIE_FLAGS(%r15),(_TIF_ISOLATE_BP|_TIF_ISOLATE_BP_GUEST) 254d0fc4107SMartin Schwidefsky.Lsie_skip: 255d0fc4107SMartin Schwidefsky ni __SIE_PROG0C+3(%r14),0xfe # no longer in SIE 25687d59863SHeiko Carstens lctlg %c1,%c1,__LC_KERNEL_ASCE # load primary asce 257d0fc4107SMartin Schwidefsky.Lsie_done: 258d0fc4107SMartin Schwidefsky# some program checks are suppressing. C code (e.g. do_protection_exception) 259c0e7bb38SChristian Borntraeger# will rewind the PSW by the ILC, which is often 4 bytes in case of SIE. There 260c0e7bb38SChristian Borntraeger# are some corner cases (e.g. runtime instrumentation) where ILC is unpredictable. 261c0e7bb38SChristian Borntraeger# Other instructions between sie64a and .Lsie_done should not cause program 262c0e7bb38SChristian Borntraeger# interrupts. So lets use 3 nops as a landing pad for all possible rewinds. 263c0e7bb38SChristian Borntraeger.Lrewind_pad6: 264c0e7bb38SChristian Borntraeger nopr 7 265c0e7bb38SChristian Borntraeger.Lrewind_pad4: 266c0e7bb38SChristian Borntraeger nopr 7 267c0e7bb38SChristian Borntraeger.Lrewind_pad2: 268c0e7bb38SChristian Borntraeger nopr 7 269d0fc4107SMartin Schwidefsky .globl sie_exit 270d0fc4107SMartin Schwidefskysie_exit: 27192fa7a13SMartin Schwidefsky lg %r14,__SF_SIE_SAVEAREA(%r15) # load guest register save area 272d0fc4107SMartin Schwidefsky stmg %r0,%r13,0(%r14) # save guest gprs 0-13 2737041d281SMartin Schwidefsky xgr %r0,%r0 # clear guest registers to 2747041d281SMartin Schwidefsky xgr %r1,%r1 # prevent speculative use 2757041d281SMartin Schwidefsky xgr %r3,%r3 2767041d281SMartin Schwidefsky xgr %r4,%r4 2777041d281SMartin Schwidefsky xgr %r5,%r5 278d0fc4107SMartin Schwidefsky lmg %r6,%r14,__SF_GPRS(%r15) # restore kernel registers 27992fa7a13SMartin Schwidefsky lg %r2,__SF_SIE_REASON(%r15) # return exit reason code 2806dd85fbbSMartin Schwidefsky BR_EX %r14 281d0fc4107SMartin Schwidefsky.Lsie_fault: 282d0fc4107SMartin Schwidefsky lghi %r14,-EFAULT 28392fa7a13SMartin Schwidefsky stg %r14,__SF_SIE_REASON(%r15) # set exit reason code 284d0fc4107SMartin Schwidefsky j sie_exit 285d0fc4107SMartin Schwidefsky 286c0e7bb38SChristian Borntraeger EX_TABLE(.Lrewind_pad6,.Lsie_fault) 287c0e7bb38SChristian Borntraeger EX_TABLE(.Lrewind_pad4,.Lsie_fault) 288c0e7bb38SChristian Borntraeger EX_TABLE(.Lrewind_pad2,.Lsie_fault) 289d0fc4107SMartin Schwidefsky EX_TABLE(sie_exit,.Lsie_fault) 29026a374aeSMartin SchwidefskyENDPROC(sie64a) 291711f5df7SAl ViroEXPORT_SYMBOL(sie64a) 292711f5df7SAl ViroEXPORT_SYMBOL(sie_exit) 293d0fc4107SMartin Schwidefsky#endif 294d0fc4107SMartin Schwidefsky 2954bfc86ceSHeiko Carstens/* 2964bfc86ceSHeiko Carstens * SVC interrupt handler routine. System calls are synchronous events and 2977b7735c5SChristian Borntraeger * are entered with interrupts disabled. 2984bfc86ceSHeiko Carstens */ 2994bfc86ceSHeiko Carstens 3004bfc86ceSHeiko CarstensENTRY(system_call) 30156e62a73SSven Schnelle stpt __LC_SYS_ENTER_TIMER 3024bfc86ceSHeiko Carstens stmg %r8,%r15,__LC_SAVE_AREA_SYNC 303d768bd89SMartin Schwidefsky BPOFF 30456e62a73SSven Schnelle lghi %r14,0 3054bfc86ceSHeiko Carstens.Lsysc_per: 30687d59863SHeiko Carstens lctlg %c1,%c1,__LC_KERNEL_ASCE 30756e62a73SSven Schnelle lg %r12,__LC_CURRENT 3084bfc86ceSHeiko Carstens lg %r15,__LC_KERNEL_STACK 3099365965dSHeiko Carstens xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) 31056e62a73SSven Schnelle stmg %r0,%r7,STACK_FRAME_OVERHEAD+__PT_R0(%r15) 31156e62a73SSven Schnelle BPENTER __TI_flags(%r12),_TIF_ISOLATE_BP 312d3f46896SChristian Borntraeger # clear user controlled register to prevent speculative use 313d3f46896SChristian Borntraeger xgr %r0,%r0 31456e62a73SSven Schnelle xgr %r1,%r1 31556e62a73SSven Schnelle xgr %r4,%r4 31656e62a73SSven Schnelle xgr %r5,%r5 31756e62a73SSven Schnelle xgr %r6,%r6 31856e62a73SSven Schnelle xgr %r7,%r7 31956e62a73SSven Schnelle xgr %r8,%r8 32056e62a73SSven Schnelle xgr %r9,%r9 32156e62a73SSven Schnelle xgr %r10,%r10 32256e62a73SSven Schnelle xgr %r11,%r11 32356e62a73SSven Schnelle la %r2,STACK_FRAME_OVERHEAD(%r15) # pointer to pt_regs 324af9ad822SSven Schnelle mvc __PT_R8(64,%r2),__LC_SAVE_AREA_SYNC 32556e62a73SSven Schnelle lgr %r3,%r14 32656e62a73SSven Schnelle brasl %r14,__do_syscall 32787d59863SHeiko Carstens lctlg %c1,%c1,__LC_USER_ASCE 32856e62a73SSven Schnelle mvc __LC_RETURN_PSW(16),STACK_FRAME_OVERHEAD+__PT_PSW(%r15) 3296b73044bSMartin Schwidefsky BPEXIT __TI_flags(%r12),_TIF_ISOLATE_BP 33056e62a73SSven Schnelle lmg %r0,%r15,STACK_FRAME_OVERHEAD+__PT_R0(%r15) 3314bfc86ceSHeiko Carstens stpt __LC_EXIT_TIMER 3320b0ed657SSven Schnelle b __LC_RETURN_LPSWE 33326a374aeSMartin SchwidefskyENDPROC(system_call) 3344bfc86ceSHeiko Carstens 3354bfc86ceSHeiko Carstens# 3364bfc86ceSHeiko Carstens# a new process exits the kernel with ret_from_fork 3374bfc86ceSHeiko Carstens# 3384bfc86ceSHeiko CarstensENTRY(ret_from_fork) 33956e62a73SSven Schnelle lgr %r3,%r11 34056e62a73SSven Schnelle brasl %r14,__ret_from_fork 34156e62a73SSven Schnelle lctlg %c1,%c1,__LC_USER_ASCE 34256e62a73SSven Schnelle mvc __LC_RETURN_PSW(16),STACK_FRAME_OVERHEAD+__PT_PSW(%r15) 34356e62a73SSven Schnelle BPEXIT __TI_flags(%r12),_TIF_ISOLATE_BP 34456e62a73SSven Schnelle lmg %r0,%r15,STACK_FRAME_OVERHEAD+__PT_R0(%r15) 34556e62a73SSven Schnelle stpt __LC_EXIT_TIMER 34656e62a73SSven Schnelle b __LC_RETURN_LPSWE 34726a374aeSMartin SchwidefskyENDPROC(ret_from_fork) 34826a374aeSMartin Schwidefsky 3494bfc86ceSHeiko Carstens/* 3504bfc86ceSHeiko Carstens * Program check handler routine 3514bfc86ceSHeiko Carstens */ 3524bfc86ceSHeiko Carstens 3534bfc86ceSHeiko CarstensENTRY(pgm_check_handler) 35456e62a73SSven Schnelle stpt __LC_SYS_ENTER_TIMER 355d768bd89SMartin Schwidefsky BPOFF 3564bfc86ceSHeiko Carstens stmg %r8,%r15,__LC_SAVE_AREA_SYNC 35756e62a73SSven Schnelle lg %r12,__LC_CURRENT 35856e62a73SSven Schnelle lghi %r10,0 3594bfc86ceSHeiko Carstens lmg %r8,%r9,__LC_PGM_OLD_PSW 36087d59863SHeiko Carstens tmhh %r8,0x0001 # coming from user space? 36187d59863SHeiko Carstens jno .Lpgm_skip_asce 36287d59863SHeiko Carstens lctlg %c1,%c1,__LC_KERNEL_ASCE 36356e62a73SSven Schnelle j 3f # -> fault in user space 36487d59863SHeiko Carstens.Lpgm_skip_asce: 365d0fc4107SMartin Schwidefsky#if IS_ENABLED(CONFIG_KVM) 3660a5e2ec2SMartin Schwidefsky # cleanup critical section for program checks in sie64a 367b5415c8fSAlexander Gordeev OUTSIDE %r9,.Lsie_gmap,.Lsie_done,1f 368fbbdfca5SAlexander Gordeev SIEEXIT 36956e62a73SSven Schnelle lghi %r10,_PIF_GUEST_FAULT 370d0fc4107SMartin Schwidefsky#endif 3710b38b5e1SSven Schnelle1: tmhh %r8,0x4000 # PER bit set in old PSW ? 3720b38b5e1SSven Schnelle jnz 2f # -> enabled, can't be a double fault 3734bfc86ceSHeiko Carstens tm __LC_PGM_ILC+3,0x80 # check for per exception 3744bfc86ceSHeiko Carstens jnz .Lpgm_svcper # -> single stepped svc 3750b38b5e1SSven Schnelle2: CHECK_STACK __LC_SAVE_AREA_SYNC 3764bfc86ceSHeiko Carstens aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE) 37756e62a73SSven Schnelle # CHECK_VMAP_STACK branches to stack_overflow or 4f 37856e62a73SSven Schnelle CHECK_VMAP_STACK __LC_SAVE_AREA_SYNC,4f 37956e62a73SSven Schnelle3: BPENTER __TI_flags(%r12),_TIF_ISOLATE_BP 3804bfc86ceSHeiko Carstens lg %r15,__LC_KERNEL_STACK 38156e62a73SSven Schnelle4: la %r11,STACK_FRAME_OVERHEAD(%r15) 38256e62a73SSven Schnelle stg %r10,__PT_FLAGS(%r11) 38356e62a73SSven Schnelle xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) 3844bfc86ceSHeiko Carstens stmg %r0,%r7,__PT_R0(%r11) 38556e62a73SSven Schnelle mvc __PT_R8(64,%r11),__LC_SAVE_AREA_SYNC 38656e62a73SSven Schnelle stmg %r8,%r9,__PT_PSW(%r11) 38756e62a73SSven Schnelle 3887041d281SMartin Schwidefsky # clear user controlled registers to prevent speculative use 3897041d281SMartin Schwidefsky xgr %r0,%r0 3907041d281SMartin Schwidefsky xgr %r1,%r1 3917041d281SMartin Schwidefsky xgr %r3,%r3 3927041d281SMartin Schwidefsky xgr %r4,%r4 3937041d281SMartin Schwidefsky xgr %r5,%r5 3947041d281SMartin Schwidefsky xgr %r6,%r6 3957041d281SMartin Schwidefsky xgr %r7,%r7 39656e62a73SSven Schnelle lgr %r2,%r11 39756e62a73SSven Schnelle brasl %r14,__do_pgm_check 39856e62a73SSven Schnelle tmhh %r8,0x0001 # returning to user space? 39956e62a73SSven Schnelle jno .Lpgm_exit_kernel 40056e62a73SSven Schnelle lctlg %c1,%c1,__LC_USER_ASCE 40156e62a73SSven Schnelle BPEXIT __TI_flags(%r12),_TIF_ISOLATE_BP 4020cd9b723SHeiko Carstens stpt __LC_EXIT_TIMER 40356e62a73SSven Schnelle.Lpgm_exit_kernel: 40456e62a73SSven Schnelle mvc __LC_RETURN_PSW(16),STACK_FRAME_OVERHEAD+__PT_PSW(%r15) 40556e62a73SSven Schnelle lmg %r0,%r15,STACK_FRAME_OVERHEAD+__PT_R0(%r15) 4060cd9b723SHeiko Carstens b __LC_RETURN_LPSWE 4074bfc86ceSHeiko Carstens 4084bfc86ceSHeiko Carstens# 4094bfc86ceSHeiko Carstens# single stepped system call 4104bfc86ceSHeiko Carstens# 4114bfc86ceSHeiko Carstens.Lpgm_svcper: 4124bfc86ceSHeiko Carstens mvc __LC_RETURN_PSW(8),__LC_SVC_NEW_PSW 4134bfc86ceSHeiko Carstens larl %r14,.Lsysc_per 4144bfc86ceSHeiko Carstens stg %r14,__LC_RETURN_PSW+8 41556e62a73SSven Schnelle lghi %r14,1 4160b0ed657SSven Schnelle lpswe __LC_RETURN_PSW # branch to .Lsysc_per 41726a374aeSMartin SchwidefskyENDPROC(pgm_check_handler) 4184bfc86ceSHeiko Carstens 4194bfc86ceSHeiko Carstens/* 42056e62a73SSven Schnelle * Interrupt handler macro used for external and IO interrupts. 4214bfc86ceSHeiko Carstens */ 42256e62a73SSven Schnelle.macro INT_HANDLER name,lc_old_psw,handler 42356e62a73SSven SchnelleENTRY(\name) 4244bfc86ceSHeiko Carstens STCK __LC_INT_CLOCK 42556e62a73SSven Schnelle stpt __LC_SYS_ENTER_TIMER 426d768bd89SMartin Schwidefsky BPOFF 4274bfc86ceSHeiko Carstens stmg %r8,%r15,__LC_SAVE_AREA_ASYNC 428d5c352cdSHeiko Carstens lg %r12,__LC_CURRENT 42956e62a73SSven Schnelle lmg %r8,%r9,\lc_old_psw 430b0d31159SSven Schnelle tmhh %r8,0x0001 # interrupting from user ? 431b0d31159SSven Schnelle jnz 1f 432b0d31159SSven Schnelle#if IS_ENABLED(CONFIG_KVM) 433b5415c8fSAlexander Gordeev OUTSIDE %r9,.Lsie_gmap,.Lsie_done,0f 434fbbdfca5SAlexander Gordeev BPENTER __SF_SIE_FLAGS(%r15),(_TIF_ISOLATE_BP|_TIF_ISOLATE_BP_GUEST) 435fbbdfca5SAlexander Gordeev SIEEXIT 436b0d31159SSven Schnelle#endif 437b0d31159SSven Schnelle0: CHECK_STACK __LC_SAVE_AREA_ASYNC 438b0d31159SSven Schnelle aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE) 439b0d31159SSven Schnelle j 2f 440b0d31159SSven Schnelle1: BPENTER __TI_flags(%r12),_TIF_ISOLATE_BP 441b0d31159SSven Schnelle lctlg %c1,%c1,__LC_KERNEL_ASCE 442b0d31159SSven Schnelle lg %r15,__LC_KERNEL_STACK 443b74e409eSVasily Gorbik2: xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) 444b74e409eSVasily Gorbik la %r11,STACK_FRAME_OVERHEAD(%r15) 4454bfc86ceSHeiko Carstens stmg %r0,%r7,__PT_R0(%r11) 4467041d281SMartin Schwidefsky # clear user controlled registers to prevent speculative use 4477041d281SMartin Schwidefsky xgr %r0,%r0 4487041d281SMartin Schwidefsky xgr %r1,%r1 4497041d281SMartin Schwidefsky xgr %r3,%r3 4507041d281SMartin Schwidefsky xgr %r4,%r4 4517041d281SMartin Schwidefsky xgr %r5,%r5 4527041d281SMartin Schwidefsky xgr %r6,%r6 4537041d281SMartin Schwidefsky xgr %r7,%r7 4547041d281SMartin Schwidefsky xgr %r10,%r10 455ca1f4d70SSven Schnelle xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11) 4564bfc86ceSHeiko Carstens mvc __PT_R8(64,%r11),__LC_SAVE_AREA_ASYNC 4574bfc86ceSHeiko Carstens stmg %r8,%r9,__PT_PSW(%r11) 45856e62a73SSven Schnelle tm %r8,0x0001 # coming from user space? 45956e62a73SSven Schnelle jno 1f 46087d59863SHeiko Carstens lctlg %c1,%c1,__LC_KERNEL_ASCE 46156e62a73SSven Schnelle1: lgr %r2,%r11 # pass pointer to pt_regs 46256e62a73SSven Schnelle brasl %r14,\handler 4634bfc86ceSHeiko Carstens mvc __LC_RETURN_PSW(16),__PT_PSW(%r11) 46456e62a73SSven Schnelle tmhh %r8,0x0001 # returning to user ? 46556e62a73SSven Schnelle jno 2f 46687d59863SHeiko Carstens lctlg %c1,%c1,__LC_USER_ASCE 4676b73044bSMartin Schwidefsky BPEXIT __TI_flags(%r12),_TIF_ISOLATE_BP 4684bfc86ceSHeiko Carstens stpt __LC_EXIT_TIMER 46956e62a73SSven Schnelle2: lmg %r0,%r15,__PT_R0(%r11) 4700b0ed657SSven Schnelle b __LC_RETURN_LPSWE 47156e62a73SSven SchnelleENDPROC(\name) 47256e62a73SSven Schnelle.endm 4734bfc86ceSHeiko Carstens 47456e62a73SSven SchnelleINT_HANDLER ext_int_handler,__LC_EXT_OLD_PSW,do_ext_irq 47556e62a73SSven SchnelleINT_HANDLER io_int_handler,__LC_IO_OLD_PSW,do_io_irq 4764bfc86ceSHeiko Carstens 4774bfc86ceSHeiko Carstens/* 4780b0ed657SSven Schnelle * Load idle PSW. 4794bfc86ceSHeiko Carstens */ 4804bfc86ceSHeiko CarstensENTRY(psw_idle) 481a994eddbSVasily Gorbik stg %r14,(__SF_GPRS+8*8)(%r15) 4824bfc86ceSHeiko Carstens stg %r3,__SF_EMPTY(%r15) 48356e62a73SSven Schnelle larl %r1,psw_idle_exit 4844bfc86ceSHeiko Carstens stg %r1,__SF_EMPTY+8(%r15) 48572d38b19SMartin Schwidefsky larl %r1,smp_cpu_mtid 48672d38b19SMartin Schwidefsky llgf %r1,0(%r1) 48772d38b19SMartin Schwidefsky ltgr %r1,%r1 48872d38b19SMartin Schwidefsky jz .Lpsw_idle_stcctm 48956e62a73SSven Schnelle .insn rsy,0xeb0000000017,%r1,5,__MT_CYCLES_ENTER(%r2) 49072d38b19SMartin Schwidefsky.Lpsw_idle_stcctm: 491419123f9SMartin Schwidefsky oi __LC_CPU_FLAGS+7,_CIF_ENABLED_WAIT 492d768bd89SMartin Schwidefsky BPON 4934bfc86ceSHeiko Carstens STCK __CLOCK_IDLE_ENTER(%r2) 4944bfc86ceSHeiko Carstens stpt __TIMER_IDLE_ENTER(%r2) 4954bfc86ceSHeiko Carstens lpswe __SF_EMPTY(%r15) 49656e62a73SSven Schnelle.globl psw_idle_exit 49756e62a73SSven Schnellepsw_idle_exit: 4986dd85fbbSMartin Schwidefsky BR_EX %r14 49926a374aeSMartin SchwidefskyENDPROC(psw_idle) 5004bfc86ceSHeiko Carstens 501b5510d9bSHendrik Brueckner/* 5024bfc86ceSHeiko Carstens * Machine check handler routines 5034bfc86ceSHeiko Carstens */ 5044bfc86ceSHeiko CarstensENTRY(mcck_int_handler) 5054bfc86ceSHeiko Carstens STCK __LC_MCCK_CLOCK 506d768bd89SMartin Schwidefsky BPOFF 5073037a52fSMartin Schwidefsky la %r1,4095 # validate r1 5083037a52fSMartin Schwidefsky spt __LC_CPU_TIMER_SAVE_AREA-4095(%r1) # validate cpu timer 5093037a52fSMartin Schwidefsky sckc __LC_CLOCK_COMPARATOR # validate comparator 5103037a52fSMartin Schwidefsky lam %a0,%a15,__LC_AREGS_SAVE_AREA-4095(%r1) # validate acrs 5113037a52fSMartin Schwidefsky lmg %r0,%r15,__LC_GPREGS_SAVE_AREA-4095(%r1)# validate gprs 512d5c352cdSHeiko Carstens lg %r12,__LC_CURRENT 5134bfc86ceSHeiko Carstens lmg %r8,%r9,__LC_MCK_OLD_PSW 51483abeffbSHendrik Brueckner TSTMSK __LC_MCCK_CODE,MCCK_CODE_SYSTEM_DAMAGE 5154bfc86ceSHeiko Carstens jo .Lmcck_panic # yes -> rest of mcck code invalid 5163037a52fSMartin Schwidefsky TSTMSK __LC_MCCK_CODE,MCCK_CODE_CR_VALID 5173037a52fSMartin Schwidefsky jno .Lmcck_panic # control registers invalid -> panic 5183037a52fSMartin Schwidefsky la %r14,4095 5193037a52fSMartin Schwidefsky lctlg %c0,%c15,__LC_CREGS_SAVE_AREA-4095(%r14) # validate ctl regs 5203037a52fSMartin Schwidefsky ptlb 5212a2d7befSVasily Gorbik lg %r11,__LC_MCESAD-4095(%r14) # extended machine check save area 5223037a52fSMartin Schwidefsky nill %r11,0xfc00 # MCESA_ORIGIN_MASK 5233037a52fSMartin Schwidefsky TSTMSK __LC_CREGS_SAVE_AREA+16-4095(%r14),CR2_GUARDED_STORAGE 5243037a52fSMartin Schwidefsky jno 0f 5253037a52fSMartin Schwidefsky TSTMSK __LC_MCCK_CODE,MCCK_CODE_GS_VALID 5263037a52fSMartin Schwidefsky jno 0f 5273037a52fSMartin Schwidefsky .insn rxy,0xe3000000004d,0,__MCESA_GS_SAVE_AREA(%r11) # LGSC 5283037a52fSMartin Schwidefsky0: l %r14,__LC_FP_CREG_SAVE_AREA-4095(%r14) 5293037a52fSMartin Schwidefsky TSTMSK __LC_MCCK_CODE,MCCK_CODE_FC_VALID 5303037a52fSMartin Schwidefsky jo 0f 5313037a52fSMartin Schwidefsky sr %r14,%r14 5323037a52fSMartin Schwidefsky0: sfpc %r14 5333037a52fSMartin Schwidefsky TSTMSK __LC_MACHINE_FLAGS,MACHINE_FLAG_VX 5343037a52fSMartin Schwidefsky jo 0f 5353037a52fSMartin Schwidefsky lghi %r14,__LC_FPREGS_SAVE_AREA 5363037a52fSMartin Schwidefsky ld %f0,0(%r14) 5373037a52fSMartin Schwidefsky ld %f1,8(%r14) 5383037a52fSMartin Schwidefsky ld %f2,16(%r14) 5393037a52fSMartin Schwidefsky ld %f3,24(%r14) 5403037a52fSMartin Schwidefsky ld %f4,32(%r14) 5413037a52fSMartin Schwidefsky ld %f5,40(%r14) 5423037a52fSMartin Schwidefsky ld %f6,48(%r14) 5433037a52fSMartin Schwidefsky ld %f7,56(%r14) 5443037a52fSMartin Schwidefsky ld %f8,64(%r14) 5453037a52fSMartin Schwidefsky ld %f9,72(%r14) 5463037a52fSMartin Schwidefsky ld %f10,80(%r14) 5473037a52fSMartin Schwidefsky ld %f11,88(%r14) 5483037a52fSMartin Schwidefsky ld %f12,96(%r14) 5493037a52fSMartin Schwidefsky ld %f13,104(%r14) 5503037a52fSMartin Schwidefsky ld %f14,112(%r14) 5513037a52fSMartin Schwidefsky ld %f15,120(%r14) 5523037a52fSMartin Schwidefsky j 1f 5533037a52fSMartin Schwidefsky0: VLM %v0,%v15,0,%r11 5543037a52fSMartin Schwidefsky VLM %v16,%v31,256,%r11 5553037a52fSMartin Schwidefsky1: lghi %r14,__LC_CPU_TIMER_SAVE_AREA 5564bfc86ceSHeiko Carstens mvc __LC_MCCK_ENTER_TIMER(8),0(%r14) 55783abeffbSHendrik Brueckner TSTMSK __LC_MCCK_CODE,MCCK_CODE_CPU_TIMER_VALID 5584bfc86ceSHeiko Carstens jo 3f 55956e62a73SSven Schnelle la %r14,__LC_SYS_ENTER_TIMER 56056e62a73SSven Schnelle clc 0(8,%r14),__LC_EXIT_TIMER 5614bfc86ceSHeiko Carstens jl 1f 5624bfc86ceSHeiko Carstens la %r14,__LC_EXIT_TIMER 5634bfc86ceSHeiko Carstens1: clc 0(8,%r14),__LC_LAST_UPDATE_TIMER 5644bfc86ceSHeiko Carstens jl 2f 5654bfc86ceSHeiko Carstens la %r14,__LC_LAST_UPDATE_TIMER 5664bfc86ceSHeiko Carstens2: spt 0(%r14) 5674bfc86ceSHeiko Carstens mvc __LC_MCCK_ENTER_TIMER(8),0(%r14) 5683037a52fSMartin Schwidefsky3: TSTMSK __LC_MCCK_CODE,MCCK_CODE_PSW_MWP_VALID 5693037a52fSMartin Schwidefsky jno .Lmcck_panic 5703037a52fSMartin Schwidefsky tmhh %r8,0x0001 # interrupting from user ? 571*d35925b3SAlexander Gordeev jnz 6f 5723037a52fSMartin Schwidefsky TSTMSK __LC_MCCK_CODE,MCCK_CODE_PSW_IA_VALID 5733037a52fSMartin Schwidefsky jno .Lmcck_panic 574b0d31159SSven Schnelle#if IS_ENABLED(CONFIG_KVM) 575*d35925b3SAlexander Gordeev OUTSIDE %r9,.Lsie_gmap,.Lsie_done,6f 576*d35925b3SAlexander Gordeev OUTSIDE %r9,.Lsie_entry,.Lsie_skip,4f 57720232b18SAlexander Gordeev oi __LC_CPU_FLAGS+7, _CIF_MCCK_GUEST 578*d35925b3SAlexander Gordeev j 5f 579*d35925b3SAlexander Gordeev4: CHKSTG .Lmcck_panic 580*d35925b3SAlexander Gordeev5: larl %r14,.Lstosm_tmp 581*d35925b3SAlexander Gordeev stosm 0(%r14),0x04 # turn dat on, keep irqs off 582*d35925b3SAlexander Gordeev BPENTER __SF_SIE_FLAGS(%r15),(_TIF_ISOLATE_BP|_TIF_ISOLATE_BP_GUEST) 583fbbdfca5SAlexander Gordeev SIEEXIT 584b61b1595SSven Schnelle j .Lmcck_stack 585e2c13d64SAlexander Gordeev#endif 586*d35925b3SAlexander Gordeev6: CHKSTG .Lmcck_panic 587*d35925b3SAlexander Gordeev larl %r14,.Lstosm_tmp 588*d35925b3SAlexander Gordeev stosm 0(%r14),0x04 # turn dat on, keep irqs off 589*d35925b3SAlexander Gordeev tmhh %r8,0x0001 # interrupting from user ? 590*d35925b3SAlexander Gordeev jz .Lmcck_stack 591b0d31159SSven Schnelle BPENTER __TI_flags(%r12),_TIF_ISOLATE_BP 592b61b1595SSven Schnelle.Lmcck_stack: 593b61b1595SSven Schnelle lg %r15,__LC_MCCK_STACK 594b61b1595SSven Schnelle la %r11,STACK_FRAME_OVERHEAD(%r15) 59526521412SSven Schnelle stctg %c1,%c1,__PT_CR1(%r11) 596b61b1595SSven Schnelle lctlg %c1,%c1,__LC_KERNEL_ASCE 597b61b1595SSven Schnelle xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) 5984bfc86ceSHeiko Carstens lghi %r14,__LC_GPREGS_SAVE_AREA+64 5994bfc86ceSHeiko Carstens stmg %r0,%r7,__PT_R0(%r11) 6007041d281SMartin Schwidefsky # clear user controlled registers to prevent speculative use 6017041d281SMartin Schwidefsky xgr %r0,%r0 6027041d281SMartin Schwidefsky xgr %r1,%r1 6037041d281SMartin Schwidefsky xgr %r3,%r3 6047041d281SMartin Schwidefsky xgr %r4,%r4 6057041d281SMartin Schwidefsky xgr %r5,%r5 6067041d281SMartin Schwidefsky xgr %r6,%r6 6077041d281SMartin Schwidefsky xgr %r7,%r7 6087041d281SMartin Schwidefsky xgr %r10,%r10 6094bfc86ceSHeiko Carstens mvc __PT_R8(64,%r11),0(%r14) 6104bfc86ceSHeiko Carstens stmg %r8,%r9,__PT_PSW(%r11) 6114bfc86ceSHeiko Carstens xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11) 6124bfc86ceSHeiko Carstens xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) 6134bfc86ceSHeiko Carstens lgr %r2,%r11 # pass pointer to pt_regs 6144bfc86ceSHeiko Carstens brasl %r14,s390_do_machine_check 6150b0ed657SSven Schnelle cghi %r2,0 6160b0ed657SSven Schnelle je .Lmcck_return 6174bfc86ceSHeiko Carstens lg %r1,__LC_KERNEL_STACK # switch to kernel stack 6184bfc86ceSHeiko Carstens mvc STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11) 6194bfc86ceSHeiko Carstens xc __SF_BACKCHAIN(8,%r1),__SF_BACKCHAIN(%r1) 6204bfc86ceSHeiko Carstens la %r11,STACK_FRAME_OVERHEAD(%r1) 6214bfc86ceSHeiko Carstens lgr %r15,%r1 6224bfc86ceSHeiko Carstens brasl %r14,s390_handle_mcck 6234bfc86ceSHeiko Carstens.Lmcck_return: 62487d59863SHeiko Carstens lctlg %c1,%c1,__PT_CR1(%r11) 6254bfc86ceSHeiko Carstens lmg %r0,%r10,__PT_R0(%r11) 6264bfc86ceSHeiko Carstens mvc __LC_RETURN_MCCK_PSW(16),__PT_PSW(%r11) # move return PSW 6274bfc86ceSHeiko Carstens tm __LC_RETURN_MCCK_PSW+1,0x01 # returning to user ? 6284bfc86ceSHeiko Carstens jno 0f 6296b73044bSMartin Schwidefsky BPEXIT __TI_flags(%r12),_TIF_ISOLATE_BP 6304bfc86ceSHeiko Carstens stpt __LC_EXIT_TIMER 6314bfc86ceSHeiko Carstens0: lmg %r11,%r15,__PT_R11(%r11) 6320b38b5e1SSven Schnelle b __LC_RETURN_MCCK_LPSWE 6334bfc86ceSHeiko Carstens 6344bfc86ceSHeiko Carstens.Lmcck_panic: 6357f6dc8d4SAlexander Gordeev /* 6367f6dc8d4SAlexander Gordeev * Iterate over all possible CPU addresses in the range 0..0xffff 6377f6dc8d4SAlexander Gordeev * and stop each CPU using signal processor. Use compare and swap 6387f6dc8d4SAlexander Gordeev * to allow just one CPU-stopper and prevent concurrent CPUs from 6397f6dc8d4SAlexander Gordeev * stopping each other while leaving the others running. 6407f6dc8d4SAlexander Gordeev */ 6417f6dc8d4SAlexander Gordeev lhi %r5,0 6427f6dc8d4SAlexander Gordeev lhi %r6,1 6437f6dc8d4SAlexander Gordeev larl %r7,.Lstop_lock 6447f6dc8d4SAlexander Gordeev cs %r5,%r6,0(%r7) # single CPU-stopper only 6457f6dc8d4SAlexander Gordeev jnz 4f 6467f6dc8d4SAlexander Gordeev larl %r7,.Lthis_cpu 6477f6dc8d4SAlexander Gordeev stap 0(%r7) # this CPU address 6487f6dc8d4SAlexander Gordeev lh %r4,0(%r7) 6497f6dc8d4SAlexander Gordeev nilh %r4,0 6507f6dc8d4SAlexander Gordeev lhi %r0,1 6517f6dc8d4SAlexander Gordeev sll %r0,16 # CPU counter 6527f6dc8d4SAlexander Gordeev lhi %r3,0 # next CPU address 6537f6dc8d4SAlexander Gordeev0: cr %r3,%r4 6547f6dc8d4SAlexander Gordeev je 2f 6557f6dc8d4SAlexander Gordeev1: sigp %r1,%r3,SIGP_STOP # stop next CPU 6567f6dc8d4SAlexander Gordeev brc SIGP_CC_BUSY,1b 6577f6dc8d4SAlexander Gordeev2: ahi %r3,1 6587f6dc8d4SAlexander Gordeev brct %r0,0b 6597f6dc8d4SAlexander Gordeev3: sigp %r1,%r4,SIGP_STOP # stop this CPU 6607f6dc8d4SAlexander Gordeev brc SIGP_CC_BUSY,3b 6617f6dc8d4SAlexander Gordeev4: j 4b 66226a374aeSMartin SchwidefskyENDPROC(mcck_int_handler) 6634bfc86ceSHeiko Carstens 6644bfc86ceSHeiko Carstens# 6654bfc86ceSHeiko Carstens# PSW restart interrupt handler 6664bfc86ceSHeiko Carstens# 6674bfc86ceSHeiko CarstensENTRY(restart_int_handler) 668e5b98199SMartin Schwidefsky ALTERNATIVE "", ".insn s,0xb2800000,_LPP_OFFSET", 40 669e5b98199SMartin Schwidefsky stg %r15,__LC_SAVE_AREA_RESTART 6704bfc86ceSHeiko Carstens lg %r15,__LC_RESTART_STACK 671ce3dc447SMartin Schwidefsky xc STACK_FRAME_OVERHEAD(__PT_SIZE,%r15),STACK_FRAME_OVERHEAD(%r15) 672ce3dc447SMartin Schwidefsky stmg %r0,%r14,STACK_FRAME_OVERHEAD+__PT_R0(%r15) 673ce3dc447SMartin Schwidefsky mvc STACK_FRAME_OVERHEAD+__PT_R15(8,%r15),__LC_SAVE_AREA_RESTART 674ce3dc447SMartin Schwidefsky mvc STACK_FRAME_OVERHEAD+__PT_PSW(16,%r15),__LC_RST_OLD_PSW 6754bfc86ceSHeiko Carstens xc 0(STACK_FRAME_OVERHEAD,%r15),0(%r15) 6764bfc86ceSHeiko Carstens lg %r1,__LC_RESTART_FN # load fn, parm & source cpu 6774bfc86ceSHeiko Carstens lg %r2,__LC_RESTART_DATA 6784bfc86ceSHeiko Carstens lg %r3,__LC_RESTART_SOURCE 6794bfc86ceSHeiko Carstens ltgr %r3,%r3 # test source cpu address 6804bfc86ceSHeiko Carstens jm 1f # negative -> skip source stop 6814bfc86ceSHeiko Carstens0: sigp %r4,%r3,SIGP_SENSE # sigp sense to source cpu 6824bfc86ceSHeiko Carstens brc 10,0b # wait for status stored 6834bfc86ceSHeiko Carstens1: basr %r14,%r1 # call function 6844bfc86ceSHeiko Carstens stap __SF_EMPTY(%r15) # store cpu address 6854bfc86ceSHeiko Carstens llgh %r3,__SF_EMPTY(%r15) 6864bfc86ceSHeiko Carstens2: sigp %r4,%r3,SIGP_STOP # sigp stop to current cpu 6874bfc86ceSHeiko Carstens brc 2,2b 6884bfc86ceSHeiko Carstens3: j 3b 68926a374aeSMartin SchwidefskyENDPROC(restart_int_handler) 6904bfc86ceSHeiko Carstens 6914bfc86ceSHeiko Carstens .section .kprobes.text, "ax" 6924bfc86ceSHeiko Carstens 693ce3dc447SMartin Schwidefsky#if defined(CONFIG_CHECK_STACK) || defined(CONFIG_VMAP_STACK) 6944bfc86ceSHeiko Carstens/* 6954bfc86ceSHeiko Carstens * The synchronous or the asynchronous stack overflowed. We are dead. 6964bfc86ceSHeiko Carstens * No need to properly save the registers, we are going to panic anyway. 6974bfc86ceSHeiko Carstens * Setup a pt_regs so that show_trace can provide a good call trace. 6984bfc86ceSHeiko Carstens */ 69926a374aeSMartin SchwidefskyENTRY(stack_overflow) 700ce3dc447SMartin Schwidefsky lg %r15,__LC_NODAT_STACK # change to panic stack 7014bfc86ceSHeiko Carstens la %r11,STACK_FRAME_OVERHEAD(%r15) 7024bfc86ceSHeiko Carstens stmg %r0,%r7,__PT_R0(%r11) 7034bfc86ceSHeiko Carstens stmg %r8,%r9,__PT_PSW(%r11) 7044bfc86ceSHeiko Carstens mvc __PT_R8(64,%r11),0(%r14) 7054bfc86ceSHeiko Carstens stg %r10,__PT_ORIG_GPR2(%r11) # store last break to orig_gpr2 7064bfc86ceSHeiko Carstens xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) 7074bfc86ceSHeiko Carstens lgr %r2,%r11 # pass pointer to pt_regs 7084bfc86ceSHeiko Carstens jg kernel_stack_overflow 70926a374aeSMartin SchwidefskyENDPROC(stack_overflow) 7104bfc86ceSHeiko Carstens#endif 7114bfc86ceSHeiko Carstens 7127f6dc8d4SAlexander Gordeev .section .data, "aw" 7137f6dc8d4SAlexander Gordeev .align 4 7147f6dc8d4SAlexander Gordeev.Lstop_lock: .long 0 7157f6dc8d4SAlexander Gordeev.Lthis_cpu: .short 0 716*d35925b3SAlexander Gordeev.Lstosm_tmp: .byte 0 7174bfc86ceSHeiko Carstens .section .rodata, "a" 718ff4a742dSGerald Schaefer#define SYSCALL(esame,emu) .quad __s390x_ ## esame 7194bfc86ceSHeiko Carstens .globl sys_call_table 7204bfc86ceSHeiko Carstenssys_call_table: 7214381f9f1SHendrik Brueckner#include "asm/syscall_table.h" 7224bfc86ceSHeiko Carstens#undef SYSCALL 7234bfc86ceSHeiko Carstens 7244bfc86ceSHeiko Carstens#ifdef CONFIG_COMPAT 7254bfc86ceSHeiko Carstens 726ff4a742dSGerald Schaefer#define SYSCALL(esame,emu) .quad __s390_ ## emu 7274bfc86ceSHeiko Carstens .globl sys_call_table_emu 7284bfc86ceSHeiko Carstenssys_call_table_emu: 7294381f9f1SHendrik Brueckner#include "asm/syscall_table.h" 7304bfc86ceSHeiko Carstens#undef SYSCALL 7314bfc86ceSHeiko Carstens#endif 732