1/* 2 * S390 low-level entry points. 3 * 4 * Copyright IBM Corp. 1999, 2012 5 * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com), 6 * Hartmut Penner (hp@de.ibm.com), 7 * Denis Joseph Barrow (djbarrow@de.ibm.com,barrow_dj@yahoo.com), 8 * Heiko Carstens <heiko.carstens@de.ibm.com> 9 */ 10 11#include <linux/init.h> 12#include <linux/linkage.h> 13#include <asm/cache.h> 14#include <asm/errno.h> 15#include <asm/ptrace.h> 16#include <asm/thread_info.h> 17#include <asm/asm-offsets.h> 18#include <asm/unistd.h> 19#include <asm/page.h> 20#include <asm/sigp.h> 21#include <asm/irq.h> 22 23__PT_R0 = __PT_GPRS 24__PT_R1 = __PT_GPRS + 4 25__PT_R2 = __PT_GPRS + 8 26__PT_R3 = __PT_GPRS + 12 27__PT_R4 = __PT_GPRS + 16 28__PT_R5 = __PT_GPRS + 20 29__PT_R6 = __PT_GPRS + 24 30__PT_R7 = __PT_GPRS + 28 31__PT_R8 = __PT_GPRS + 32 32__PT_R9 = __PT_GPRS + 36 33__PT_R10 = __PT_GPRS + 40 34__PT_R11 = __PT_GPRS + 44 35__PT_R12 = __PT_GPRS + 48 36__PT_R13 = __PT_GPRS + 524 37__PT_R14 = __PT_GPRS + 56 38__PT_R15 = __PT_GPRS + 60 39 40_TIF_WORK_SVC = (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_NEED_RESCHED | \ 41 _TIF_MCCK_PENDING | _TIF_PER_TRAP ) 42_TIF_WORK_INT = (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_NEED_RESCHED | \ 43 _TIF_MCCK_PENDING) 44_TIF_TRACE = (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | _TIF_SECCOMP | \ 45 _TIF_SYSCALL_TRACEPOINT) 46 47STACK_SHIFT = PAGE_SHIFT + THREAD_ORDER 48STACK_SIZE = 1 << STACK_SHIFT 49STACK_INIT = STACK_SIZE - STACK_FRAME_OVERHEAD - __PT_SIZE 50 51#define BASED(name) name-system_call(%r13) 52 53 .macro TRACE_IRQS_ON 54#ifdef CONFIG_TRACE_IRQFLAGS 55 basr %r2,%r0 56 l %r1,BASED(.Lhardirqs_on) 57 basr %r14,%r1 # call trace_hardirqs_on_caller 58#endif 59 .endm 60 61 .macro TRACE_IRQS_OFF 62#ifdef CONFIG_TRACE_IRQFLAGS 63 basr %r2,%r0 64 l %r1,BASED(.Lhardirqs_off) 65 basr %r14,%r1 # call trace_hardirqs_off_caller 66#endif 67 .endm 68 69 .macro LOCKDEP_SYS_EXIT 70#ifdef CONFIG_LOCKDEP 71 tm __PT_PSW+1(%r11),0x01 # returning to user ? 72 jz .+10 73 l %r1,BASED(.Llockdep_sys_exit) 74 basr %r14,%r1 # call lockdep_sys_exit 75#endif 76 .endm 77 78 .macro CHECK_STACK stacksize,savearea 79#ifdef CONFIG_CHECK_STACK 80 tml %r15,\stacksize - CONFIG_STACK_GUARD 81 la %r14,\savearea 82 jz stack_overflow 83#endif 84 .endm 85 86 .macro SWITCH_ASYNC savearea,stack,shift 87 tmh %r8,0x0001 # interrupting from user ? 88 jnz 1f 89 lr %r14,%r9 90 sl %r14,BASED(.Lcritical_start) 91 cl %r14,BASED(.Lcritical_length) 92 jhe 0f 93 la %r11,\savearea # inside critical section, do cleanup 94 bras %r14,cleanup_critical 95 tmh %r8,0x0001 # retest problem state after cleanup 96 jnz 1f 970: l %r14,\stack # are we already on the target stack? 98 slr %r14,%r15 99 sra %r14,\shift 100 jnz 1f 101 CHECK_STACK 1<<\shift,\savearea 102 ahi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE) 103 j 2f 1041: l %r15,\stack # load target stack 1052: la %r11,STACK_FRAME_OVERHEAD(%r15) 106 .endm 107 108 .macro ADD64 high,low,timer 109 al \high,\timer 110 al \low,4+\timer 111 brc 12,.+8 112 ahi \high,1 113 .endm 114 115 .macro SUB64 high,low,timer 116 sl \high,\timer 117 sl \low,4+\timer 118 brc 3,.+8 119 ahi \high,-1 120 .endm 121 122 .macro UPDATE_VTIME high,low,enter_timer 123 lm \high,\low,__LC_EXIT_TIMER 124 SUB64 \high,\low,\enter_timer 125 ADD64 \high,\low,__LC_USER_TIMER 126 stm \high,\low,__LC_USER_TIMER 127 lm \high,\low,__LC_LAST_UPDATE_TIMER 128 SUB64 \high,\low,__LC_EXIT_TIMER 129 ADD64 \high,\low,__LC_SYSTEM_TIMER 130 stm \high,\low,__LC_SYSTEM_TIMER 131 mvc __LC_LAST_UPDATE_TIMER(8),\enter_timer 132 .endm 133 134 .macro REENABLE_IRQS 135 st %r8,__LC_RETURN_PSW 136 ni __LC_RETURN_PSW,0xbf 137 ssm __LC_RETURN_PSW 138 .endm 139 140 .section .kprobes.text, "ax" 141 142/* 143 * Scheduler resume function, called by switch_to 144 * gpr2 = (task_struct *) prev 145 * gpr3 = (task_struct *) next 146 * Returns: 147 * gpr2 = prev 148 */ 149ENTRY(__switch_to) 150 stm %r6,%r15,__SF_GPRS(%r15) # store gprs of prev task 151 st %r15,__THREAD_ksp(%r2) # store kernel stack of prev 152 l %r4,__THREAD_info(%r2) # get thread_info of prev 153 l %r5,__THREAD_info(%r3) # get thread_info of next 154 lr %r15,%r5 155 ahi %r15,STACK_INIT # end of kernel stack of next 156 st %r3,__LC_CURRENT # store task struct of next 157 st %r5,__LC_THREAD_INFO # store thread info of next 158 st %r15,__LC_KERNEL_STACK # store end of kernel stack 159 lctl %c4,%c4,__TASK_pid(%r3) # load pid to control reg. 4 160 mvc __LC_CURRENT_PID(4,%r0),__TASK_pid(%r3) # store pid of next 161 l %r15,__THREAD_ksp(%r3) # load kernel stack of next 162 tm __TI_flags+3(%r4),_TIF_MCCK_PENDING # machine check pending? 163 jz 0f 164 ni __TI_flags+3(%r4),255-_TIF_MCCK_PENDING # clear flag in prev 165 oi __TI_flags+3(%r5),_TIF_MCCK_PENDING # set it in next 1660: lm %r6,%r15,__SF_GPRS(%r15) # load gprs of next task 167 br %r14 168 169__critical_start: 170/* 171 * SVC interrupt handler routine. System calls are synchronous events and 172 * are executed with interrupts enabled. 173 */ 174 175ENTRY(system_call) 176 stpt __LC_SYNC_ENTER_TIMER 177sysc_stm: 178 stm %r8,%r15,__LC_SAVE_AREA_SYNC 179 l %r12,__LC_THREAD_INFO 180 l %r13,__LC_SVC_NEW_PSW+4 181sysc_per: 182 l %r15,__LC_KERNEL_STACK 183 la %r11,STACK_FRAME_OVERHEAD(%r15) # pointer to pt_regs 184sysc_vtime: 185 UPDATE_VTIME %r8,%r9,__LC_SYNC_ENTER_TIMER 186 stm %r0,%r7,__PT_R0(%r11) 187 mvc __PT_R8(32,%r11),__LC_SAVE_AREA_SYNC 188 mvc __PT_PSW(8,%r11),__LC_SVC_OLD_PSW 189 mvc __PT_INT_CODE(4,%r11),__LC_SVC_ILC 190sysc_do_svc: 191 oi __TI_flags+3(%r12),_TIF_SYSCALL 192 l %r10,__TI_sysc_table(%r12) # 31 bit system call table 193 lh %r8,__PT_INT_CODE+2(%r11) 194 sla %r8,2 # shift and test for svc0 195 jnz sysc_nr_ok 196 # svc 0: system call number in %r1 197 cl %r1,BASED(.Lnr_syscalls) 198 jnl sysc_nr_ok 199 sth %r1,__PT_INT_CODE+2(%r11) 200 lr %r8,%r1 201 sla %r8,2 202sysc_nr_ok: 203 xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15) 204 st %r2,__PT_ORIG_GPR2(%r11) 205 st %r7,STACK_FRAME_OVERHEAD(%r15) 206 l %r9,0(%r8,%r10) # get system call addr. 207 tm __TI_flags+2(%r12),_TIF_TRACE >> 8 208 jnz sysc_tracesys 209 basr %r14,%r9 # call sys_xxxx 210 st %r2,__PT_R2(%r11) # store return value 211 212sysc_return: 213 LOCKDEP_SYS_EXIT 214sysc_tif: 215 tm __PT_PSW+1(%r11),0x01 # returning to user ? 216 jno sysc_restore 217 tm __TI_flags+3(%r12),_TIF_WORK_SVC 218 jnz sysc_work # check for work 219 ni __TI_flags+3(%r12),255-_TIF_SYSCALL 220sysc_restore: 221 mvc __LC_RETURN_PSW(8),__PT_PSW(%r11) 222 stpt __LC_EXIT_TIMER 223 lm %r0,%r15,__PT_R0(%r11) 224 lpsw __LC_RETURN_PSW 225sysc_done: 226 227# 228# One of the work bits is on. Find out which one. 229# 230sysc_work: 231 tm __TI_flags+3(%r12),_TIF_MCCK_PENDING 232 jo sysc_mcck_pending 233 tm __TI_flags+3(%r12),_TIF_NEED_RESCHED 234 jo sysc_reschedule 235 tm __TI_flags+3(%r12),_TIF_PER_TRAP 236 jo sysc_singlestep 237 tm __TI_flags+3(%r12),_TIF_SIGPENDING 238 jo sysc_sigpending 239 tm __TI_flags+3(%r12),_TIF_NOTIFY_RESUME 240 jo sysc_notify_resume 241 j sysc_return # beware of critical section cleanup 242 243# 244# _TIF_NEED_RESCHED is set, call schedule 245# 246sysc_reschedule: 247 l %r1,BASED(.Lschedule) 248 la %r14,BASED(sysc_return) 249 br %r1 # call schedule 250 251# 252# _TIF_MCCK_PENDING is set, call handler 253# 254sysc_mcck_pending: 255 l %r1,BASED(.Lhandle_mcck) 256 la %r14,BASED(sysc_return) 257 br %r1 # TIF bit will be cleared by handler 258 259# 260# _TIF_SIGPENDING is set, call do_signal 261# 262sysc_sigpending: 263 lr %r2,%r11 # pass pointer to pt_regs 264 l %r1,BASED(.Ldo_signal) 265 basr %r14,%r1 # call do_signal 266 tm __TI_flags+3(%r12),_TIF_SYSCALL 267 jno sysc_return 268 lm %r2,%r7,__PT_R2(%r11) # load svc arguments 269 l %r10,__TI_sysc_table(%r12) # 31 bit system call table 270 xr %r8,%r8 # svc 0 returns -ENOSYS 271 clc __PT_INT_CODE+2(2,%r11),BASED(.Lnr_syscalls+2) 272 jnl sysc_nr_ok # invalid svc number -> do svc 0 273 lh %r8,__PT_INT_CODE+2(%r11) # load new svc number 274 sla %r8,2 275 j sysc_nr_ok # restart svc 276 277# 278# _TIF_NOTIFY_RESUME is set, call do_notify_resume 279# 280sysc_notify_resume: 281 lr %r2,%r11 # pass pointer to pt_regs 282 l %r1,BASED(.Ldo_notify_resume) 283 la %r14,BASED(sysc_return) 284 br %r1 # call do_notify_resume 285 286# 287# _TIF_PER_TRAP is set, call do_per_trap 288# 289sysc_singlestep: 290 ni __TI_flags+3(%r12),255-_TIF_PER_TRAP 291 lr %r2,%r11 # pass pointer to pt_regs 292 l %r1,BASED(.Ldo_per_trap) 293 la %r14,BASED(sysc_return) 294 br %r1 # call do_per_trap 295 296# 297# call tracehook_report_syscall_entry/tracehook_report_syscall_exit before 298# and after the system call 299# 300sysc_tracesys: 301 l %r1,BASED(.Ltrace_enter) 302 lr %r2,%r11 # pass pointer to pt_regs 303 la %r3,0 304 xr %r0,%r0 305 icm %r0,3,__PT_INT_CODE+2(%r11) 306 st %r0,__PT_R2(%r11) 307 basr %r14,%r1 # call do_syscall_trace_enter 308 cl %r2,BASED(.Lnr_syscalls) 309 jnl sysc_tracenogo 310 lr %r8,%r2 311 sll %r8,2 312 l %r9,0(%r8,%r10) 313sysc_tracego: 314 lm %r3,%r7,__PT_R3(%r11) 315 st %r7,STACK_FRAME_OVERHEAD(%r15) 316 l %r2,__PT_ORIG_GPR2(%r11) 317 basr %r14,%r9 # call sys_xxx 318 st %r2,__PT_R2(%r11) # store return value 319sysc_tracenogo: 320 tm __TI_flags+2(%r12),_TIF_TRACE >> 8 321 jz sysc_return 322 l %r1,BASED(.Ltrace_exit) 323 lr %r2,%r11 # pass pointer to pt_regs 324 la %r14,BASED(sysc_return) 325 br %r1 # call do_syscall_trace_exit 326 327# 328# a new process exits the kernel with ret_from_fork 329# 330ENTRY(ret_from_fork) 331 la %r11,STACK_FRAME_OVERHEAD(%r15) 332 l %r12,__LC_THREAD_INFO 333 l %r13,__LC_SVC_NEW_PSW+4 334 l %r1,BASED(.Lschedule_tail) 335 basr %r14,%r1 # call schedule_tail 336 TRACE_IRQS_ON 337 ssm __LC_SVC_NEW_PSW # reenable interrupts 338 tm __PT_PSW+1(%r11),0x01 # forking a kernel thread ? 339 jne sysc_tracenogo 340 # it's a kernel thread 341 lm %r9,%r10,__PT_R9(%r11) # load gprs 342ENTRY(kernel_thread_starter) 343 la %r2,0(%r10) 344 basr %r14,%r9 345 j sysc_tracenogo 346 347/* 348 * Program check handler routine 349 */ 350 351ENTRY(pgm_check_handler) 352 stpt __LC_SYNC_ENTER_TIMER 353 stm %r8,%r15,__LC_SAVE_AREA_SYNC 354 l %r12,__LC_THREAD_INFO 355 l %r13,__LC_SVC_NEW_PSW+4 356 lm %r8,%r9,__LC_PGM_OLD_PSW 357 tmh %r8,0x0001 # test problem state bit 358 jnz 1f # -> fault in user space 359 tmh %r8,0x4000 # PER bit set in old PSW ? 360 jnz 0f # -> enabled, can't be a double fault 361 tm __LC_PGM_ILC+3,0x80 # check for per exception 362 jnz pgm_svcper # -> single stepped svc 3630: CHECK_STACK STACK_SIZE,__LC_SAVE_AREA_SYNC 364 ahi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE) 365 j 2f 3661: UPDATE_VTIME %r14,%r15,__LC_SYNC_ENTER_TIMER 367 l %r15,__LC_KERNEL_STACK 3682: la %r11,STACK_FRAME_OVERHEAD(%r15) 369 stm %r0,%r7,__PT_R0(%r11) 370 mvc __PT_R8(32,%r11),__LC_SAVE_AREA_SYNC 371 stm %r8,%r9,__PT_PSW(%r11) 372 mvc __PT_INT_CODE(4,%r11),__LC_PGM_ILC 373 mvc __PT_INT_PARM_LONG(4,%r11),__LC_TRANS_EXC_CODE 374 tm __LC_PGM_ILC+3,0x80 # check for per exception 375 jz 0f 376 l %r1,__TI_task(%r12) 377 tmh %r8,0x0001 # kernel per event ? 378 jz pgm_kprobe 379 oi __TI_flags+3(%r12),_TIF_PER_TRAP 380 mvc __THREAD_per_address(4,%r1),__LC_PER_ADDRESS 381 mvc __THREAD_per_cause(2,%r1),__LC_PER_CAUSE 382 mvc __THREAD_per_paid(1,%r1),__LC_PER_PAID 3830: REENABLE_IRQS 384 xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15) 385 l %r1,BASED(.Ljump_table) 386 la %r10,0x7f 387 n %r10,__PT_INT_CODE(%r11) 388 je sysc_return 389 sll %r10,2 390 l %r1,0(%r10,%r1) # load address of handler routine 391 lr %r2,%r11 # pass pointer to pt_regs 392 basr %r14,%r1 # branch to interrupt-handler 393 j sysc_return 394 395# 396# PER event in supervisor state, must be kprobes 397# 398pgm_kprobe: 399 REENABLE_IRQS 400 xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15) 401 l %r1,BASED(.Ldo_per_trap) 402 lr %r2,%r11 # pass pointer to pt_regs 403 basr %r14,%r1 # call do_per_trap 404 j sysc_return 405 406# 407# single stepped system call 408# 409pgm_svcper: 410 oi __TI_flags+3(%r12),_TIF_PER_TRAP 411 mvc __LC_RETURN_PSW(4),__LC_SVC_NEW_PSW 412 mvc __LC_RETURN_PSW+4(4),BASED(.Lsysc_per) 413 lpsw __LC_RETURN_PSW # branch to sysc_per and enable irqs 414 415/* 416 * IO interrupt handler routine 417 */ 418 419ENTRY(io_int_handler) 420 stck __LC_INT_CLOCK 421 stpt __LC_ASYNC_ENTER_TIMER 422 stm %r8,%r15,__LC_SAVE_AREA_ASYNC 423 l %r12,__LC_THREAD_INFO 424 l %r13,__LC_SVC_NEW_PSW+4 425 lm %r8,%r9,__LC_IO_OLD_PSW 426 tmh %r8,0x0001 # interrupting from user ? 427 jz io_skip 428 UPDATE_VTIME %r14,%r15,__LC_ASYNC_ENTER_TIMER 429io_skip: 430 SWITCH_ASYNC __LC_SAVE_AREA_ASYNC,__LC_ASYNC_STACK,STACK_SHIFT 431 stm %r0,%r7,__PT_R0(%r11) 432 mvc __PT_R8(32,%r11),__LC_SAVE_AREA_ASYNC 433 stm %r8,%r9,__PT_PSW(%r11) 434 mvc __PT_INT_CODE(12,%r11),__LC_SUBCHANNEL_ID 435 TRACE_IRQS_OFF 436 xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15) 437io_loop: 438 l %r1,BASED(.Ldo_IRQ) 439 lr %r2,%r11 # pass pointer to pt_regs 440 lhi %r3,IO_INTERRUPT 441 tm __PT_INT_CODE+8(%r11),0x80 # adapter interrupt ? 442 jz io_call 443 lhi %r3,THIN_INTERRUPT 444io_call: 445 basr %r14,%r1 # call do_IRQ 446 tm __LC_MACHINE_FLAGS+2,0x10 # MACHINE_FLAG_LPAR 447 jz io_return 448 tpi 0 449 jz io_return 450 mvc __PT_INT_CODE(12,%r11),__LC_SUBCHANNEL_ID 451 j io_loop 452io_return: 453 LOCKDEP_SYS_EXIT 454 TRACE_IRQS_ON 455io_tif: 456 tm __TI_flags+3(%r12),_TIF_WORK_INT 457 jnz io_work # there is work to do (signals etc.) 458io_restore: 459 mvc __LC_RETURN_PSW(8),__PT_PSW(%r11) 460 stpt __LC_EXIT_TIMER 461 lm %r0,%r15,__PT_R0(%r11) 462 lpsw __LC_RETURN_PSW 463io_done: 464 465# 466# There is work todo, find out in which context we have been interrupted: 467# 1) if we return to user space we can do all _TIF_WORK_INT work 468# 2) if we return to kernel code and preemptive scheduling is enabled check 469# the preemption counter and if it is zero call preempt_schedule_irq 470# Before any work can be done, a switch to the kernel stack is required. 471# 472io_work: 473 tm __PT_PSW+1(%r11),0x01 # returning to user ? 474 jo io_work_user # yes -> do resched & signal 475#ifdef CONFIG_PREEMPT 476 # check for preemptive scheduling 477 icm %r0,15,__TI_precount(%r12) 478 jnz io_restore # preemption disabled 479 tm __TI_flags+3(%r12),_TIF_NEED_RESCHED 480 jno io_restore 481 # switch to kernel stack 482 l %r1,__PT_R15(%r11) 483 ahi %r1,-(STACK_FRAME_OVERHEAD + __PT_SIZE) 484 mvc STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11) 485 xc __SF_BACKCHAIN(4,%r1),__SF_BACKCHAIN(%r1) 486 la %r11,STACK_FRAME_OVERHEAD(%r1) 487 lr %r15,%r1 488 # TRACE_IRQS_ON already done at io_return, call 489 # TRACE_IRQS_OFF to keep things symmetrical 490 TRACE_IRQS_OFF 491 l %r1,BASED(.Lpreempt_irq) 492 basr %r14,%r1 # call preempt_schedule_irq 493 j io_return 494#else 495 j io_restore 496#endif 497 498# 499# Need to do work before returning to userspace, switch to kernel stack 500# 501io_work_user: 502 l %r1,__LC_KERNEL_STACK 503 mvc STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11) 504 xc __SF_BACKCHAIN(4,%r1),__SF_BACKCHAIN(%r1) 505 la %r11,STACK_FRAME_OVERHEAD(%r1) 506 lr %r15,%r1 507 508# 509# One of the work bits is on. Find out which one. 510# Checked are: _TIF_SIGPENDING, _TIF_NOTIFY_RESUME, _TIF_NEED_RESCHED 511# and _TIF_MCCK_PENDING 512# 513io_work_tif: 514 tm __TI_flags+3(%r12),_TIF_MCCK_PENDING 515 jo io_mcck_pending 516 tm __TI_flags+3(%r12),_TIF_NEED_RESCHED 517 jo io_reschedule 518 tm __TI_flags+3(%r12),_TIF_SIGPENDING 519 jo io_sigpending 520 tm __TI_flags+3(%r12),_TIF_NOTIFY_RESUME 521 jo io_notify_resume 522 j io_return # beware of critical section cleanup 523 524# 525# _TIF_MCCK_PENDING is set, call handler 526# 527io_mcck_pending: 528 # TRACE_IRQS_ON already done at io_return 529 l %r1,BASED(.Lhandle_mcck) 530 basr %r14,%r1 # TIF bit will be cleared by handler 531 TRACE_IRQS_OFF 532 j io_return 533 534# 535# _TIF_NEED_RESCHED is set, call schedule 536# 537io_reschedule: 538 # TRACE_IRQS_ON already done at io_return 539 l %r1,BASED(.Lschedule) 540 ssm __LC_SVC_NEW_PSW # reenable interrupts 541 basr %r14,%r1 # call scheduler 542 ssm __LC_PGM_NEW_PSW # disable I/O and ext. interrupts 543 TRACE_IRQS_OFF 544 j io_return 545 546# 547# _TIF_SIGPENDING is set, call do_signal 548# 549io_sigpending: 550 # TRACE_IRQS_ON already done at io_return 551 l %r1,BASED(.Ldo_signal) 552 ssm __LC_SVC_NEW_PSW # reenable interrupts 553 lr %r2,%r11 # pass pointer to pt_regs 554 basr %r14,%r1 # call do_signal 555 ssm __LC_PGM_NEW_PSW # disable I/O and ext. interrupts 556 TRACE_IRQS_OFF 557 j io_return 558 559# 560# _TIF_SIGPENDING is set, call do_signal 561# 562io_notify_resume: 563 # TRACE_IRQS_ON already done at io_return 564 l %r1,BASED(.Ldo_notify_resume) 565 ssm __LC_SVC_NEW_PSW # reenable interrupts 566 lr %r2,%r11 # pass pointer to pt_regs 567 basr %r14,%r1 # call do_notify_resume 568 ssm __LC_PGM_NEW_PSW # disable I/O and ext. interrupts 569 TRACE_IRQS_OFF 570 j io_return 571 572/* 573 * External interrupt handler routine 574 */ 575 576ENTRY(ext_int_handler) 577 stck __LC_INT_CLOCK 578 stpt __LC_ASYNC_ENTER_TIMER 579 stm %r8,%r15,__LC_SAVE_AREA_ASYNC 580 l %r12,__LC_THREAD_INFO 581 l %r13,__LC_SVC_NEW_PSW+4 582 lm %r8,%r9,__LC_EXT_OLD_PSW 583 tmh %r8,0x0001 # interrupting from user ? 584 jz ext_skip 585 UPDATE_VTIME %r14,%r15,__LC_ASYNC_ENTER_TIMER 586ext_skip: 587 SWITCH_ASYNC __LC_SAVE_AREA_ASYNC,__LC_ASYNC_STACK,STACK_SHIFT 588 stm %r0,%r7,__PT_R0(%r11) 589 mvc __PT_R8(32,%r11),__LC_SAVE_AREA_ASYNC 590 stm %r8,%r9,__PT_PSW(%r11) 591 mvc __PT_INT_CODE(4,%r11),__LC_EXT_CPU_ADDR 592 mvc __PT_INT_PARM(4,%r11),__LC_EXT_PARAMS 593 TRACE_IRQS_OFF 594 l %r1,BASED(.Ldo_IRQ) 595 lr %r2,%r11 # pass pointer to pt_regs 596 lhi %r3,EXT_INTERRUPT 597 basr %r14,%r1 # call do_IRQ 598 j io_return 599 600/* 601 * Load idle PSW. The second "half" of this function is in cleanup_idle. 602 */ 603ENTRY(psw_idle) 604 st %r3,__SF_EMPTY(%r15) 605 basr %r1,0 606 la %r1,psw_idle_lpsw+4-.(%r1) 607 st %r1,__SF_EMPTY+4(%r15) 608 oi __SF_EMPTY+4(%r15),0x80 609 stck __CLOCK_IDLE_ENTER(%r2) 610 stpt __TIMER_IDLE_ENTER(%r2) 611psw_idle_lpsw: 612 lpsw __SF_EMPTY(%r15) 613 br %r14 614psw_idle_end: 615 616__critical_end: 617 618/* 619 * Machine check handler routines 620 */ 621 622ENTRY(mcck_int_handler) 623 stck __LC_MCCK_CLOCK 624 spt __LC_CPU_TIMER_SAVE_AREA # revalidate cpu timer 625 lm %r0,%r15,__LC_GPREGS_SAVE_AREA # revalidate gprs 626 l %r12,__LC_THREAD_INFO 627 l %r13,__LC_SVC_NEW_PSW+4 628 lm %r8,%r9,__LC_MCK_OLD_PSW 629 tm __LC_MCCK_CODE,0x80 # system damage? 630 jo mcck_panic # yes -> rest of mcck code invalid 631 la %r14,__LC_CPU_TIMER_SAVE_AREA 632 mvc __LC_MCCK_ENTER_TIMER(8),0(%r14) 633 tm __LC_MCCK_CODE+5,0x02 # stored cpu timer value valid? 634 jo 3f 635 la %r14,__LC_SYNC_ENTER_TIMER 636 clc 0(8,%r14),__LC_ASYNC_ENTER_TIMER 637 jl 0f 638 la %r14,__LC_ASYNC_ENTER_TIMER 6390: clc 0(8,%r14),__LC_EXIT_TIMER 640 jl 1f 641 la %r14,__LC_EXIT_TIMER 6421: clc 0(8,%r14),__LC_LAST_UPDATE_TIMER 643 jl 2f 644 la %r14,__LC_LAST_UPDATE_TIMER 6452: spt 0(%r14) 646 mvc __LC_MCCK_ENTER_TIMER(8),0(%r14) 6473: tm __LC_MCCK_CODE+2,0x09 # mwp + ia of old psw valid? 648 jno mcck_panic # no -> skip cleanup critical 649 tm %r8,0x0001 # interrupting from user ? 650 jz mcck_skip 651 UPDATE_VTIME %r14,%r15,__LC_MCCK_ENTER_TIMER 652mcck_skip: 653 SWITCH_ASYNC __LC_GPREGS_SAVE_AREA+32,__LC_PANIC_STACK,PAGE_SHIFT 654 stm %r0,%r7,__PT_R0(%r11) 655 mvc __PT_R8(32,%r11),__LC_GPREGS_SAVE_AREA+32 656 stm %r8,%r9,__PT_PSW(%r11) 657 xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15) 658 l %r1,BASED(.Ldo_machine_check) 659 lr %r2,%r11 # pass pointer to pt_regs 660 basr %r14,%r1 # call s390_do_machine_check 661 tm __PT_PSW+1(%r11),0x01 # returning to user ? 662 jno mcck_return 663 l %r1,__LC_KERNEL_STACK # switch to kernel stack 664 mvc STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11) 665 xc __SF_BACKCHAIN(4,%r1),__SF_BACKCHAIN(%r1) 666 la %r11,STACK_FRAME_OVERHEAD(%r15) 667 lr %r15,%r1 668 ssm __LC_PGM_NEW_PSW # turn dat on, keep irqs off 669 tm __TI_flags+3(%r12),_TIF_MCCK_PENDING 670 jno mcck_return 671 TRACE_IRQS_OFF 672 l %r1,BASED(.Lhandle_mcck) 673 basr %r14,%r1 # call s390_handle_mcck 674 TRACE_IRQS_ON 675mcck_return: 676 mvc __LC_RETURN_MCCK_PSW(8),__PT_PSW(%r11) # move return PSW 677 tm __LC_RETURN_MCCK_PSW+1,0x01 # returning to user ? 678 jno 0f 679 lm %r0,%r15,__PT_R0(%r11) 680 stpt __LC_EXIT_TIMER 681 lpsw __LC_RETURN_MCCK_PSW 6820: lm %r0,%r15,__PT_R0(%r11) 683 lpsw __LC_RETURN_MCCK_PSW 684 685mcck_panic: 686 l %r14,__LC_PANIC_STACK 687 slr %r14,%r15 688 sra %r14,PAGE_SHIFT 689 jz 0f 690 l %r15,__LC_PANIC_STACK 691 j mcck_skip 6920: ahi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE) 693 j mcck_skip 694 695# 696# PSW restart interrupt handler 697# 698ENTRY(restart_int_handler) 699 st %r15,__LC_SAVE_AREA_RESTART 700 l %r15,__LC_RESTART_STACK 701 ahi %r15,-__PT_SIZE # create pt_regs on stack 702 xc 0(__PT_SIZE,%r15),0(%r15) 703 stm %r0,%r14,__PT_R0(%r15) 704 mvc __PT_R15(4,%r15),__LC_SAVE_AREA_RESTART 705 mvc __PT_PSW(8,%r15),__LC_RST_OLD_PSW # store restart old psw 706 ahi %r15,-STACK_FRAME_OVERHEAD # create stack frame on stack 707 xc 0(STACK_FRAME_OVERHEAD,%r15),0(%r15) 708 l %r1,__LC_RESTART_FN # load fn, parm & source cpu 709 l %r2,__LC_RESTART_DATA 710 l %r3,__LC_RESTART_SOURCE 711 ltr %r3,%r3 # test source cpu address 712 jm 1f # negative -> skip source stop 7130: sigp %r4,%r3,SIGP_SENSE # sigp sense to source cpu 714 brc 10,0b # wait for status stored 7151: basr %r14,%r1 # call function 716 stap __SF_EMPTY(%r15) # store cpu address 717 lh %r3,__SF_EMPTY(%r15) 7182: sigp %r4,%r3,SIGP_STOP # sigp stop to current cpu 719 brc 2,2b 7203: j 3b 721 722 .section .kprobes.text, "ax" 723 724#ifdef CONFIG_CHECK_STACK 725/* 726 * The synchronous or the asynchronous stack overflowed. We are dead. 727 * No need to properly save the registers, we are going to panic anyway. 728 * Setup a pt_regs so that show_trace can provide a good call trace. 729 */ 730stack_overflow: 731 l %r15,__LC_PANIC_STACK # change to panic stack 732 la %r11,STACK_FRAME_OVERHEAD(%r15) 733 stm %r0,%r7,__PT_R0(%r11) 734 stm %r8,%r9,__PT_PSW(%r11) 735 mvc __PT_R8(32,%r11),0(%r14) 736 l %r1,BASED(1f) 737 xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15) 738 lr %r2,%r11 # pass pointer to pt_regs 739 br %r1 # branch to kernel_stack_overflow 7401: .long kernel_stack_overflow 741#endif 742 743cleanup_table: 744 .long system_call + 0x80000000 745 .long sysc_do_svc + 0x80000000 746 .long sysc_tif + 0x80000000 747 .long sysc_restore + 0x80000000 748 .long sysc_done + 0x80000000 749 .long io_tif + 0x80000000 750 .long io_restore + 0x80000000 751 .long io_done + 0x80000000 752 .long psw_idle + 0x80000000 753 .long psw_idle_end + 0x80000000 754 755cleanup_critical: 756 cl %r9,BASED(cleanup_table) # system_call 757 jl 0f 758 cl %r9,BASED(cleanup_table+4) # sysc_do_svc 759 jl cleanup_system_call 760 cl %r9,BASED(cleanup_table+8) # sysc_tif 761 jl 0f 762 cl %r9,BASED(cleanup_table+12) # sysc_restore 763 jl cleanup_sysc_tif 764 cl %r9,BASED(cleanup_table+16) # sysc_done 765 jl cleanup_sysc_restore 766 cl %r9,BASED(cleanup_table+20) # io_tif 767 jl 0f 768 cl %r9,BASED(cleanup_table+24) # io_restore 769 jl cleanup_io_tif 770 cl %r9,BASED(cleanup_table+28) # io_done 771 jl cleanup_io_restore 772 cl %r9,BASED(cleanup_table+32) # psw_idle 773 jl 0f 774 cl %r9,BASED(cleanup_table+36) # psw_idle_end 775 jl cleanup_idle 7760: br %r14 777 778cleanup_system_call: 779 # check if stpt has been executed 780 cl %r9,BASED(cleanup_system_call_insn) 781 jh 0f 782 mvc __LC_SYNC_ENTER_TIMER(8),__LC_ASYNC_ENTER_TIMER 783 chi %r11,__LC_SAVE_AREA_ASYNC 784 je 0f 785 mvc __LC_SYNC_ENTER_TIMER(8),__LC_MCCK_ENTER_TIMER 7860: # check if stm has been executed 787 cl %r9,BASED(cleanup_system_call_insn+4) 788 jh 0f 789 mvc __LC_SAVE_AREA_SYNC(32),0(%r11) 7900: # set up saved registers r12, and r13 791 st %r12,16(%r11) # r12 thread-info pointer 792 st %r13,20(%r11) # r13 literal-pool pointer 793 # check if the user time calculation has been done 794 cl %r9,BASED(cleanup_system_call_insn+8) 795 jh 0f 796 l %r10,__LC_EXIT_TIMER 797 l %r15,__LC_EXIT_TIMER+4 798 SUB64 %r10,%r15,__LC_SYNC_ENTER_TIMER 799 ADD64 %r10,%r15,__LC_USER_TIMER 800 st %r10,__LC_USER_TIMER 801 st %r15,__LC_USER_TIMER+4 8020: # check if the system time calculation has been done 803 cl %r9,BASED(cleanup_system_call_insn+12) 804 jh 0f 805 l %r10,__LC_LAST_UPDATE_TIMER 806 l %r15,__LC_LAST_UPDATE_TIMER+4 807 SUB64 %r10,%r15,__LC_EXIT_TIMER 808 ADD64 %r10,%r15,__LC_SYSTEM_TIMER 809 st %r10,__LC_SYSTEM_TIMER 810 st %r15,__LC_SYSTEM_TIMER+4 8110: # update accounting time stamp 812 mvc __LC_LAST_UPDATE_TIMER(8),__LC_SYNC_ENTER_TIMER 813 # set up saved register 11 814 l %r15,__LC_KERNEL_STACK 815 la %r9,STACK_FRAME_OVERHEAD(%r15) 816 st %r9,12(%r11) # r11 pt_regs pointer 817 # fill pt_regs 818 mvc __PT_R8(32,%r9),__LC_SAVE_AREA_SYNC 819 stm %r0,%r7,__PT_R0(%r9) 820 mvc __PT_PSW(8,%r9),__LC_SVC_OLD_PSW 821 mvc __PT_INT_CODE(4,%r9),__LC_SVC_ILC 822 # setup saved register 15 823 st %r15,28(%r11) # r15 stack pointer 824 # set new psw address and exit 825 l %r9,BASED(cleanup_table+4) # sysc_do_svc + 0x80000000 826 br %r14 827cleanup_system_call_insn: 828 .long system_call + 0x80000000 829 .long sysc_stm + 0x80000000 830 .long sysc_vtime + 0x80000000 + 36 831 .long sysc_vtime + 0x80000000 + 76 832 833cleanup_sysc_tif: 834 l %r9,BASED(cleanup_table+8) # sysc_tif + 0x80000000 835 br %r14 836 837cleanup_sysc_restore: 838 cl %r9,BASED(cleanup_sysc_restore_insn) 839 jhe 0f 840 l %r9,12(%r11) # get saved pointer to pt_regs 841 mvc __LC_RETURN_PSW(8),__PT_PSW(%r9) 842 mvc 0(32,%r11),__PT_R8(%r9) 843 lm %r0,%r7,__PT_R0(%r9) 8440: lm %r8,%r9,__LC_RETURN_PSW 845 br %r14 846cleanup_sysc_restore_insn: 847 .long sysc_done - 4 + 0x80000000 848 849cleanup_io_tif: 850 l %r9,BASED(cleanup_table+20) # io_tif + 0x80000000 851 br %r14 852 853cleanup_io_restore: 854 cl %r9,BASED(cleanup_io_restore_insn) 855 jhe 0f 856 l %r9,12(%r11) # get saved r11 pointer to pt_regs 857 mvc __LC_RETURN_PSW(8),__PT_PSW(%r9) 858 mvc 0(32,%r11),__PT_R8(%r9) 859 lm %r0,%r7,__PT_R0(%r9) 8600: lm %r8,%r9,__LC_RETURN_PSW 861 br %r14 862cleanup_io_restore_insn: 863 .long io_done - 4 + 0x80000000 864 865cleanup_idle: 866 # copy interrupt clock & cpu timer 867 mvc __CLOCK_IDLE_EXIT(8,%r2),__LC_INT_CLOCK 868 mvc __TIMER_IDLE_EXIT(8,%r2),__LC_ASYNC_ENTER_TIMER 869 chi %r11,__LC_SAVE_AREA_ASYNC 870 je 0f 871 mvc __CLOCK_IDLE_EXIT(8,%r2),__LC_MCCK_CLOCK 872 mvc __TIMER_IDLE_EXIT(8,%r2),__LC_MCCK_ENTER_TIMER 8730: # check if stck has been executed 874 cl %r9,BASED(cleanup_idle_insn) 875 jhe 1f 876 mvc __CLOCK_IDLE_ENTER(8,%r2),__CLOCK_IDLE_EXIT(%r2) 877 mvc __TIMER_IDLE_ENTER(8,%r2),__TIMER_IDLE_EXIT(%r3) 8781: # account system time going idle 879 lm %r9,%r10,__LC_STEAL_TIMER 880 ADD64 %r9,%r10,__CLOCK_IDLE_ENTER(%r2) 881 SUB64 %r9,%r10,__LC_LAST_UPDATE_CLOCK 882 stm %r9,%r10,__LC_STEAL_TIMER 883 mvc __LC_LAST_UPDATE_CLOCK(8),__CLOCK_IDLE_EXIT(%r2) 884 lm %r9,%r10,__LC_SYSTEM_TIMER 885 ADD64 %r9,%r10,__LC_LAST_UPDATE_TIMER 886 SUB64 %r9,%r10,__TIMER_IDLE_ENTER(%r2) 887 stm %r9,%r10,__LC_SYSTEM_TIMER 888 mvc __LC_LAST_UPDATE_TIMER(8),__TIMER_IDLE_EXIT(%r2) 889 # prepare return psw 890 n %r8,BASED(cleanup_idle_wait) # clear irq & wait state bits 891 l %r9,24(%r11) # return from psw_idle 892 br %r14 893cleanup_idle_insn: 894 .long psw_idle_lpsw + 0x80000000 895cleanup_idle_wait: 896 .long 0xfcfdffff 897 898/* 899 * Integer constants 900 */ 901 .align 4 902.Lnr_syscalls: 903 .long NR_syscalls 904.Lvtimer_max: 905 .quad 0x7fffffffffffffff 906 907/* 908 * Symbol constants 909 */ 910.Ldo_machine_check: .long s390_do_machine_check 911.Lhandle_mcck: .long s390_handle_mcck 912.Ldo_IRQ: .long do_IRQ 913.Ldo_signal: .long do_signal 914.Ldo_notify_resume: .long do_notify_resume 915.Ldo_per_trap: .long do_per_trap 916.Ljump_table: .long pgm_check_table 917.Lschedule: .long schedule 918#ifdef CONFIG_PREEMPT 919.Lpreempt_irq: .long preempt_schedule_irq 920#endif 921.Ltrace_enter: .long do_syscall_trace_enter 922.Ltrace_exit: .long do_syscall_trace_exit 923.Lschedule_tail: .long schedule_tail 924.Lsysc_per: .long sysc_per + 0x80000000 925#ifdef CONFIG_TRACE_IRQFLAGS 926.Lhardirqs_on: .long trace_hardirqs_on_caller 927.Lhardirqs_off: .long trace_hardirqs_off_caller 928#endif 929#ifdef CONFIG_LOCKDEP 930.Llockdep_sys_exit: .long lockdep_sys_exit 931#endif 932.Lcritical_start: .long __critical_start + 0x80000000 933.Lcritical_length: .long __critical_end - __critical_start 934 935 .section .rodata, "a" 936#define SYSCALL(esa,esame,emu) .long esa 937 .globl sys_call_table 938sys_call_table: 939#include "syscalls.S" 940#undef SYSCALL 941