1/* 2 * arch/s390/kernel/entry.S 3 * S390 low-level entry points. 4 * 5 * Copyright (C) IBM Corp. 1999,2006 6 * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com), 7 * Hartmut Penner (hp@de.ibm.com), 8 * Denis Joseph Barrow (djbarrow@de.ibm.com,barrow_dj@yahoo.com), 9 * Heiko Carstens <heiko.carstens@de.ibm.com> 10 */ 11 12#include <linux/init.h> 13#include <linux/linkage.h> 14#include <asm/cache.h> 15#include <asm/errno.h> 16#include <asm/ptrace.h> 17#include <asm/thread_info.h> 18#include <asm/asm-offsets.h> 19#include <asm/unistd.h> 20#include <asm/page.h> 21 22__PT_R0 = __PT_GPRS 23__PT_R1 = __PT_GPRS + 4 24__PT_R2 = __PT_GPRS + 8 25__PT_R3 = __PT_GPRS + 12 26__PT_R4 = __PT_GPRS + 16 27__PT_R5 = __PT_GPRS + 20 28__PT_R6 = __PT_GPRS + 24 29__PT_R7 = __PT_GPRS + 28 30__PT_R8 = __PT_GPRS + 32 31__PT_R9 = __PT_GPRS + 36 32__PT_R10 = __PT_GPRS + 40 33__PT_R11 = __PT_GPRS + 44 34__PT_R12 = __PT_GPRS + 48 35__PT_R13 = __PT_GPRS + 524 36__PT_R14 = __PT_GPRS + 56 37__PT_R15 = __PT_GPRS + 60 38 39_TIF_WORK_SVC = (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_NEED_RESCHED | \ 40 _TIF_MCCK_PENDING | _TIF_PER_TRAP ) 41_TIF_WORK_INT = (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_NEED_RESCHED | \ 42 _TIF_MCCK_PENDING) 43_TIF_TRACE = (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | _TIF_SECCOMP | \ 44 _TIF_SYSCALL_TRACEPOINT) 45 46STACK_SHIFT = PAGE_SHIFT + THREAD_ORDER 47STACK_SIZE = 1 << STACK_SHIFT 48 49#define BASED(name) name-system_call(%r13) 50 51 .macro TRACE_IRQS_ON 52#ifdef CONFIG_TRACE_IRQFLAGS 53 basr %r2,%r0 54 l %r1,BASED(.Lhardirqs_on) 55 basr %r14,%r1 # call trace_hardirqs_on_caller 56#endif 57 .endm 58 59 .macro TRACE_IRQS_OFF 60#ifdef CONFIG_TRACE_IRQFLAGS 61 basr %r2,%r0 62 l %r1,BASED(.Lhardirqs_off) 63 basr %r14,%r1 # call trace_hardirqs_off_caller 64#endif 65 .endm 66 67 .macro LOCKDEP_SYS_EXIT 68#ifdef CONFIG_LOCKDEP 69 tm __PT_PSW+1(%r11),0x01 # returning to user ? 70 jz .+10 71 l %r1,BASED(.Llockdep_sys_exit) 72 basr %r14,%r1 # call lockdep_sys_exit 73#endif 74 .endm 75 76 .macro CHECK_STACK stacksize,savearea 77#ifdef CONFIG_CHECK_STACK 78 tml %r15,\stacksize - CONFIG_STACK_GUARD 79 la %r14,\savearea 80 jz stack_overflow 81#endif 82 .endm 83 84 .macro SWITCH_ASYNC savearea,stack,shift 85 tmh %r8,0x0001 # interrupting from user ? 86 jnz 1f 87 lr %r14,%r9 88 sl %r14,BASED(.Lcritical_start) 89 cl %r14,BASED(.Lcritical_length) 90 jhe 0f 91 la %r11,\savearea # inside critical section, do cleanup 92 bras %r14,cleanup_critical 93 tmh %r8,0x0001 # retest problem state after cleanup 94 jnz 1f 950: l %r14,\stack # are we already on the target stack? 96 slr %r14,%r15 97 sra %r14,\shift 98 jnz 1f 99 CHECK_STACK 1<<\shift,\savearea 100 j 2f 1011: l %r15,\stack # load target stack 1022: ahi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE) 103 la %r11,STACK_FRAME_OVERHEAD(%r15) 104 .endm 105 106 .macro ADD64 high,low,timer 107 al \high,\timer 108 al \low,\timer+4 109 brc 12,.+8 110 ahi \high,1 111 .endm 112 113 .macro SUB64 high,low,timer 114 sl \high,\timer 115 sl \low,\timer+4 116 brc 3,.+8 117 ahi \high,-1 118 .endm 119 120 .macro UPDATE_VTIME high,low,enter_timer 121 lm \high,\low,__LC_EXIT_TIMER 122 SUB64 \high,\low,\enter_timer 123 ADD64 \high,\low,__LC_USER_TIMER 124 stm \high,\low,__LC_USER_TIMER 125 lm \high,\low,__LC_LAST_UPDATE_TIMER 126 SUB64 \high,\low,__LC_EXIT_TIMER 127 ADD64 \high,\low,__LC_SYSTEM_TIMER 128 stm \high,\low,__LC_SYSTEM_TIMER 129 mvc __LC_LAST_UPDATE_TIMER(8),\enter_timer 130 .endm 131 132 .macro REENABLE_IRQS 133 st %r8,__LC_RETURN_PSW 134 ni __LC_RETURN_PSW,0xbf 135 ssm __LC_RETURN_PSW 136 .endm 137 138 .section .kprobes.text, "ax" 139 140/* 141 * Scheduler resume function, called by switch_to 142 * gpr2 = (task_struct *) prev 143 * gpr3 = (task_struct *) next 144 * Returns: 145 * gpr2 = prev 146 */ 147ENTRY(__switch_to) 148 l %r4,__THREAD_info(%r2) # get thread_info of prev 149 l %r5,__THREAD_info(%r3) # get thread_info of next 150 tm __TI_flags+3(%r4),_TIF_MCCK_PENDING # machine check pending? 151 jz 0f 152 ni __TI_flags+3(%r4),255-_TIF_MCCK_PENDING # clear flag in prev 153 oi __TI_flags+3(%r5),_TIF_MCCK_PENDING # set it in next 1540: stm %r6,%r15,__SF_GPRS(%r15) # store gprs of prev task 155 st %r15,__THREAD_ksp(%r2) # store kernel stack of prev 156 l %r15,__THREAD_ksp(%r3) # load kernel stack of next 157 lctl %c4,%c4,__TASK_pid(%r3) # load pid to control reg. 4 158 lm %r6,%r15,__SF_GPRS(%r15) # load gprs of next task 159 st %r3,__LC_CURRENT # store task struct of next 160 mvc __LC_CURRENT_PID(4,%r0),__TASK_pid(%r3) # store pid of next 161 st %r5,__LC_THREAD_INFO # store thread info of next 162 ahi %r5,STACK_SIZE # end of kernel stack of next 163 st %r5,__LC_KERNEL_STACK # store end of kernel stack 164 br %r14 165 166__critical_start: 167/* 168 * SVC interrupt handler routine. System calls are synchronous events and 169 * are executed with interrupts enabled. 170 */ 171 172ENTRY(system_call) 173 stpt __LC_SYNC_ENTER_TIMER 174sysc_stm: 175 stm %r8,%r15,__LC_SAVE_AREA_SYNC 176 l %r12,__LC_THREAD_INFO 177 l %r13,__LC_SVC_NEW_PSW+4 178sysc_per: 179 l %r15,__LC_KERNEL_STACK 180 ahi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE) 181 la %r11,STACK_FRAME_OVERHEAD(%r15) # pointer to pt_regs 182sysc_vtime: 183 UPDATE_VTIME %r8,%r9,__LC_SYNC_ENTER_TIMER 184 stm %r0,%r7,__PT_R0(%r11) 185 mvc __PT_R8(32,%r11),__LC_SAVE_AREA_SYNC 186 mvc __PT_PSW(8,%r11),__LC_SVC_OLD_PSW 187 mvc __PT_INT_CODE(4,%r11),__LC_SVC_ILC 188sysc_do_svc: 189 oi __TI_flags+3(%r12),_TIF_SYSCALL 190 lh %r8,__PT_INT_CODE+2(%r11) 191 sla %r8,2 # shift and test for svc0 192 jnz sysc_nr_ok 193 # svc 0: system call number in %r1 194 cl %r1,BASED(.Lnr_syscalls) 195 jnl sysc_nr_ok 196 sth %r1,__PT_INT_CODE+2(%r11) 197 lr %r8,%r1 198 sla %r8,2 199sysc_nr_ok: 200 l %r10,BASED(.Lsys_call_table) # 31 bit system call table 201 xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15) 202 st %r2,__PT_ORIG_GPR2(%r11) 203 st %r7,STACK_FRAME_OVERHEAD(%r15) 204 l %r9,0(%r8,%r10) # get system call addr. 205 tm __TI_flags+2(%r12),_TIF_TRACE >> 8 206 jnz sysc_tracesys 207 basr %r14,%r9 # call sys_xxxx 208 st %r2,__PT_R2(%r11) # store return value 209 210sysc_return: 211 LOCKDEP_SYS_EXIT 212sysc_tif: 213 tm __PT_PSW+1(%r11),0x01 # returning to user ? 214 jno sysc_restore 215 tm __TI_flags+3(%r12),_TIF_WORK_SVC 216 jnz sysc_work # check for work 217 ni __TI_flags+3(%r12),255-_TIF_SYSCALL 218sysc_restore: 219 mvc __LC_RETURN_PSW(8),__PT_PSW(%r11) 220 stpt __LC_EXIT_TIMER 221 lm %r0,%r15,__PT_R0(%r11) 222 lpsw __LC_RETURN_PSW 223sysc_done: 224 225# 226# One of the work bits is on. Find out which one. 227# 228sysc_work: 229 tm __TI_flags+3(%r12),_TIF_MCCK_PENDING 230 jo sysc_mcck_pending 231 tm __TI_flags+3(%r12),_TIF_NEED_RESCHED 232 jo sysc_reschedule 233 tm __TI_flags+3(%r12),_TIF_SIGPENDING 234 jo sysc_sigpending 235 tm __TI_flags+3(%r12),_TIF_NOTIFY_RESUME 236 jo sysc_notify_resume 237 tm __TI_flags+3(%r12),_TIF_PER_TRAP 238 jo sysc_singlestep 239 j sysc_return # beware of critical section cleanup 240 241# 242# _TIF_NEED_RESCHED is set, call schedule 243# 244sysc_reschedule: 245 l %r1,BASED(.Lschedule) 246 la %r14,BASED(sysc_return) 247 br %r1 # call schedule 248 249# 250# _TIF_MCCK_PENDING is set, call handler 251# 252sysc_mcck_pending: 253 l %r1,BASED(.Lhandle_mcck) 254 la %r14,BASED(sysc_return) 255 br %r1 # TIF bit will be cleared by handler 256 257# 258# _TIF_SIGPENDING is set, call do_signal 259# 260sysc_sigpending: 261 ni __TI_flags+3(%r12),255-_TIF_PER_TRAP # clear TIF_PER_TRAP 262 lr %r2,%r11 # pass pointer to pt_regs 263 l %r1,BASED(.Ldo_signal) 264 basr %r14,%r1 # call do_signal 265 tm __TI_flags+3(%r12),_TIF_SYSCALL 266 jno sysc_return 267 lm %r2,%r7,__PT_R2(%r11) # load svc arguments 268 xr %r8,%r8 # svc 0 returns -ENOSYS 269 clc __PT_INT_CODE+2(2,%r11),BASED(.Lnr_syscalls+2) 270 jnl sysc_nr_ok # invalid svc number -> do svc 0 271 lh %r8,__PT_INT_CODE+2(%r11) # load new svc number 272 sla %r8,2 273 j sysc_nr_ok # restart svc 274 275# 276# _TIF_NOTIFY_RESUME is set, call do_notify_resume 277# 278sysc_notify_resume: 279 lr %r2,%r11 # pass pointer to pt_regs 280 l %r1,BASED(.Ldo_notify_resume) 281 la %r14,BASED(sysc_return) 282 br %r1 # call do_notify_resume 283 284# 285# _TIF_PER_TRAP is set, call do_per_trap 286# 287sysc_singlestep: 288 ni __TI_flags+3(%r12),255-(_TIF_SYSCALL | _TIF_PER_TRAP) 289 lr %r2,%r11 # pass pointer to pt_regs 290 l %r1,BASED(.Ldo_per_trap) 291 la %r14,BASED(sysc_return) 292 br %r1 # call do_per_trap 293 294# 295# call tracehook_report_syscall_entry/tracehook_report_syscall_exit before 296# and after the system call 297# 298sysc_tracesys: 299 l %r1,BASED(.Ltrace_enter) 300 lr %r2,%r11 # pass pointer to pt_regs 301 la %r3,0 302 xr %r0,%r0 303 icm %r0,3,__PT_INT_CODE+2(%r11) 304 st %r0,__PT_R2(%r11) 305 basr %r14,%r1 # call do_syscall_trace_enter 306 cl %r2,BASED(.Lnr_syscalls) 307 jnl sysc_tracenogo 308 lr %r8,%r2 309 sll %r8,2 310 l %r9,0(%r8,%r10) 311sysc_tracego: 312 lm %r3,%r7,__PT_R3(%r11) 313 st %r7,STACK_FRAME_OVERHEAD(%r15) 314 l %r2,__PT_ORIG_GPR2(%r11) 315 basr %r14,%r9 # call sys_xxx 316 st %r2,__PT_R2(%r11) # store return value 317sysc_tracenogo: 318 tm __TI_flags+2(%r12),_TIF_TRACE >> 8 319 jz sysc_return 320 l %r1,BASED(.Ltrace_exit) 321 lr %r2,%r11 # pass pointer to pt_regs 322 la %r14,BASED(sysc_return) 323 br %r1 # call do_syscall_trace_exit 324 325# 326# a new process exits the kernel with ret_from_fork 327# 328ENTRY(ret_from_fork) 329 la %r11,STACK_FRAME_OVERHEAD(%r15) 330 l %r12,__LC_THREAD_INFO 331 l %r13,__LC_SVC_NEW_PSW+4 332 tm __PT_PSW+1(%r11),0x01 # forking a kernel thread ? 333 jo 0f 334 st %r15,__PT_R15(%r11) # store stack pointer for new kthread 3350: l %r1,BASED(.Lschedule_tail) 336 basr %r14,%r1 # call schedule_tail 337 TRACE_IRQS_ON 338 ssm __LC_SVC_NEW_PSW # reenable interrupts 339 j sysc_tracenogo 340 341# 342# kernel_execve function needs to deal with pt_regs that is not 343# at the usual place 344# 345ENTRY(kernel_execve) 346 stm %r12,%r15,48(%r15) 347 lr %r14,%r15 348 l %r13,__LC_SVC_NEW_PSW+4 349 ahi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE) 350 st %r14,__SF_BACKCHAIN(%r15) 351 la %r12,STACK_FRAME_OVERHEAD(%r15) 352 xc 0(__PT_SIZE,%r12),0(%r12) 353 l %r1,BASED(.Ldo_execve) 354 lr %r5,%r12 355 basr %r14,%r1 # call do_execve 356 ltr %r2,%r2 357 je 0f 358 ahi %r15,(STACK_FRAME_OVERHEAD + __PT_SIZE) 359 lm %r12,%r15,48(%r15) 360 br %r14 361 # execve succeeded. 3620: ssm __LC_PGM_NEW_PSW # disable I/O and ext. interrupts 363 l %r15,__LC_KERNEL_STACK # load ksp 364 ahi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE) 365 la %r11,STACK_FRAME_OVERHEAD(%r15) 366 mvc 0(__PT_SIZE,%r11),0(%r12) # copy pt_regs 367 l %r12,__LC_THREAD_INFO 368 xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15) 369 ssm __LC_SVC_NEW_PSW # reenable interrupts 370 l %r1,BASED(.Lexecve_tail) 371 basr %r14,%r1 # call execve_tail 372 j sysc_return 373 374/* 375 * Program check handler routine 376 */ 377 378ENTRY(pgm_check_handler) 379 stpt __LC_SYNC_ENTER_TIMER 380 stm %r8,%r15,__LC_SAVE_AREA_SYNC 381 l %r12,__LC_THREAD_INFO 382 l %r13,__LC_SVC_NEW_PSW+4 383 lm %r8,%r9,__LC_PGM_OLD_PSW 384 tmh %r8,0x0001 # test problem state bit 385 jnz 1f # -> fault in user space 386 tmh %r8,0x4000 # PER bit set in old PSW ? 387 jnz 0f # -> enabled, can't be a double fault 388 tm __LC_PGM_ILC+3,0x80 # check for per exception 389 jnz pgm_svcper # -> single stepped svc 3900: CHECK_STACK STACK_SIZE,__LC_SAVE_AREA_SYNC 391 j 2f 3921: UPDATE_VTIME %r14,%r15,__LC_SYNC_ENTER_TIMER 393 l %r15,__LC_KERNEL_STACK 3942: ahi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE) 395 la %r11,STACK_FRAME_OVERHEAD(%r15) 396 stm %r0,%r7,__PT_R0(%r11) 397 mvc __PT_R8(32,%r11),__LC_SAVE_AREA_SYNC 398 stm %r8,%r9,__PT_PSW(%r11) 399 mvc __PT_INT_CODE(4,%r11),__LC_PGM_ILC 400 mvc __PT_INT_PARM_LONG(4,%r11),__LC_TRANS_EXC_CODE 401 tm __LC_PGM_ILC+3,0x80 # check for per exception 402 jz 0f 403 l %r1,__TI_task(%r12) 404 tmh %r8,0x0001 # kernel per event ? 405 jz pgm_kprobe 406 oi __TI_flags+3(%r12),_TIF_PER_TRAP 407 mvc __THREAD_per_address(4,%r1),__LC_PER_ADDRESS 408 mvc __THREAD_per_cause(2,%r1),__LC_PER_CAUSE 409 mvc __THREAD_per_paid(1,%r1),__LC_PER_PAID 4100: REENABLE_IRQS 411 xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15) 412 l %r1,BASED(.Ljump_table) 413 la %r10,0x7f 414 n %r10,__PT_INT_CODE(%r11) 415 je sysc_return 416 sll %r10,2 417 l %r1,0(%r10,%r1) # load address of handler routine 418 lr %r2,%r11 # pass pointer to pt_regs 419 basr %r14,%r1 # branch to interrupt-handler 420 j sysc_return 421 422# 423# PER event in supervisor state, must be kprobes 424# 425pgm_kprobe: 426 REENABLE_IRQS 427 xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15) 428 l %r1,BASED(.Ldo_per_trap) 429 lr %r2,%r11 # pass pointer to pt_regs 430 basr %r14,%r1 # call do_per_trap 431 j sysc_return 432 433# 434# single stepped system call 435# 436pgm_svcper: 437 oi __TI_flags+3(%r12),_TIF_PER_TRAP 438 mvc __LC_RETURN_PSW(4),__LC_SVC_NEW_PSW 439 mvc __LC_RETURN_PSW+4(4),BASED(.Lsysc_per) 440 lpsw __LC_RETURN_PSW # branch to sysc_per and enable irqs 441 442/* 443 * IO interrupt handler routine 444 */ 445 446ENTRY(io_int_handler) 447 stck __LC_INT_CLOCK 448 stpt __LC_ASYNC_ENTER_TIMER 449 stm %r8,%r15,__LC_SAVE_AREA_ASYNC 450 l %r12,__LC_THREAD_INFO 451 l %r13,__LC_SVC_NEW_PSW+4 452 lm %r8,%r9,__LC_IO_OLD_PSW 453 tmh %r8,0x0001 # interrupting from user ? 454 jz io_skip 455 UPDATE_VTIME %r14,%r15,__LC_ASYNC_ENTER_TIMER 456io_skip: 457 SWITCH_ASYNC __LC_SAVE_AREA_ASYNC,__LC_ASYNC_STACK,STACK_SHIFT 458 stm %r0,%r7,__PT_R0(%r11) 459 mvc __PT_R8(32,%r11),__LC_SAVE_AREA_ASYNC 460 stm %r8,%r9,__PT_PSW(%r11) 461 TRACE_IRQS_OFF 462 xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15) 463 l %r1,BASED(.Ldo_IRQ) 464 lr %r2,%r11 # pass pointer to pt_regs 465 basr %r14,%r1 # call do_IRQ 466io_return: 467 LOCKDEP_SYS_EXIT 468 TRACE_IRQS_ON 469io_tif: 470 tm __TI_flags+3(%r12),_TIF_WORK_INT 471 jnz io_work # there is work to do (signals etc.) 472io_restore: 473 mvc __LC_RETURN_PSW(8),__PT_PSW(%r11) 474 ni __LC_RETURN_PSW+1,0xfd # clean wait state bit 475 stpt __LC_EXIT_TIMER 476 lm %r0,%r15,__PT_R0(%r11) 477 lpsw __LC_RETURN_PSW 478io_done: 479 480# 481# There is work todo, find out in which context we have been interrupted: 482# 1) if we return to user space we can do all _TIF_WORK_INT work 483# 2) if we return to kernel code and preemptive scheduling is enabled check 484# the preemption counter and if it is zero call preempt_schedule_irq 485# Before any work can be done, a switch to the kernel stack is required. 486# 487io_work: 488 tm __PT_PSW+1(%r11),0x01 # returning to user ? 489 jo io_work_user # yes -> do resched & signal 490#ifdef CONFIG_PREEMPT 491 # check for preemptive scheduling 492 icm %r0,15,__TI_precount(%r12) 493 jnz io_restore # preemption disabled 494 tm __TI_flags+3(%r12),_TIF_NEED_RESCHED 495 jno io_restore 496 # switch to kernel stack 497 l %r1,__PT_R15(%r11) 498 ahi %r1,-(STACK_FRAME_OVERHEAD + __PT_SIZE) 499 mvc STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11) 500 xc __SF_BACKCHAIN(4,%r1),__SF_BACKCHAIN(%r1) 501 la %r11,STACK_FRAME_OVERHEAD(%r1) 502 lr %r15,%r1 503 # TRACE_IRQS_ON already done at io_return, call 504 # TRACE_IRQS_OFF to keep things symmetrical 505 TRACE_IRQS_OFF 506 l %r1,BASED(.Lpreempt_irq) 507 basr %r14,%r1 # call preempt_schedule_irq 508 j io_return 509#else 510 j io_restore 511#endif 512 513# 514# Need to do work before returning to userspace, switch to kernel stack 515# 516io_work_user: 517 l %r1,__LC_KERNEL_STACK 518 ahi %r1,-(STACK_FRAME_OVERHEAD + __PT_SIZE) 519 mvc STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11) 520 xc __SF_BACKCHAIN(4,%r1),__SF_BACKCHAIN(%r1) 521 la %r11,STACK_FRAME_OVERHEAD(%r1) 522 lr %r15,%r1 523 524# 525# One of the work bits is on. Find out which one. 526# Checked are: _TIF_SIGPENDING, _TIF_NOTIFY_RESUME, _TIF_NEED_RESCHED 527# and _TIF_MCCK_PENDING 528# 529io_work_tif: 530 tm __TI_flags+3(%r12),_TIF_MCCK_PENDING 531 jo io_mcck_pending 532 tm __TI_flags+3(%r12),_TIF_NEED_RESCHED 533 jo io_reschedule 534 tm __TI_flags+3(%r12),_TIF_SIGPENDING 535 jo io_sigpending 536 tm __TI_flags+3(%r12),_TIF_NOTIFY_RESUME 537 jo io_notify_resume 538 j io_return # beware of critical section cleanup 539 540# 541# _TIF_MCCK_PENDING is set, call handler 542# 543io_mcck_pending: 544 # TRACE_IRQS_ON already done at io_return 545 l %r1,BASED(.Lhandle_mcck) 546 basr %r14,%r1 # TIF bit will be cleared by handler 547 TRACE_IRQS_OFF 548 j io_return 549 550# 551# _TIF_NEED_RESCHED is set, call schedule 552# 553io_reschedule: 554 # TRACE_IRQS_ON already done at io_return 555 l %r1,BASED(.Lschedule) 556 ssm __LC_SVC_NEW_PSW # reenable interrupts 557 basr %r14,%r1 # call scheduler 558 ssm __LC_PGM_NEW_PSW # disable I/O and ext. interrupts 559 TRACE_IRQS_OFF 560 j io_return 561 562# 563# _TIF_SIGPENDING is set, call do_signal 564# 565io_sigpending: 566 # TRACE_IRQS_ON already done at io_return 567 l %r1,BASED(.Ldo_signal) 568 ssm __LC_SVC_NEW_PSW # reenable interrupts 569 lr %r2,%r11 # pass pointer to pt_regs 570 basr %r14,%r1 # call do_signal 571 ssm __LC_PGM_NEW_PSW # disable I/O and ext. interrupts 572 TRACE_IRQS_OFF 573 j io_return 574 575# 576# _TIF_SIGPENDING is set, call do_signal 577# 578io_notify_resume: 579 # TRACE_IRQS_ON already done at io_return 580 l %r1,BASED(.Ldo_notify_resume) 581 ssm __LC_SVC_NEW_PSW # reenable interrupts 582 lr %r2,%r11 # pass pointer to pt_regs 583 basr %r14,%r1 # call do_notify_resume 584 ssm __LC_PGM_NEW_PSW # disable I/O and ext. interrupts 585 TRACE_IRQS_OFF 586 j io_return 587 588/* 589 * External interrupt handler routine 590 */ 591 592ENTRY(ext_int_handler) 593 stck __LC_INT_CLOCK 594 stpt __LC_ASYNC_ENTER_TIMER 595 stm %r8,%r15,__LC_SAVE_AREA_ASYNC 596 l %r12,__LC_THREAD_INFO 597 l %r13,__LC_SVC_NEW_PSW+4 598 lm %r8,%r9,__LC_EXT_OLD_PSW 599 tmh %r8,0x0001 # interrupting from user ? 600 jz ext_skip 601 UPDATE_VTIME %r14,%r15,__LC_ASYNC_ENTER_TIMER 602ext_skip: 603 SWITCH_ASYNC __LC_SAVE_AREA_ASYNC,__LC_ASYNC_STACK,STACK_SHIFT 604 stm %r0,%r7,__PT_R0(%r11) 605 mvc __PT_R8(32,%r11),__LC_SAVE_AREA_ASYNC 606 stm %r8,%r9,__PT_PSW(%r11) 607 TRACE_IRQS_OFF 608 lr %r2,%r11 # pass pointer to pt_regs 609 l %r3,__LC_CPU_ADDRESS # get cpu address + interruption code 610 l %r4,__LC_EXT_PARAMS # get external parameters 611 l %r1,BASED(.Ldo_extint) 612 basr %r14,%r1 # call do_extint 613 j io_return 614 615__critical_end: 616 617/* 618 * Machine check handler routines 619 */ 620 621ENTRY(mcck_int_handler) 622 stck __LC_MCCK_CLOCK 623 spt __LC_CPU_TIMER_SAVE_AREA # revalidate cpu timer 624 lm %r0,%r15,__LC_GPREGS_SAVE_AREA # revalidate gprs 625 l %r12,__LC_THREAD_INFO 626 l %r13,__LC_SVC_NEW_PSW+4 627 lm %r8,%r9,__LC_MCK_OLD_PSW 628 tm __LC_MCCK_CODE,0x80 # system damage? 629 jo mcck_panic # yes -> rest of mcck code invalid 630 la %r14,__LC_CPU_TIMER_SAVE_AREA 631 mvc __LC_MCCK_ENTER_TIMER(8),0(%r14) 632 tm __LC_MCCK_CODE+5,0x02 # stored cpu timer value valid? 633 jo 3f 634 la %r14,__LC_SYNC_ENTER_TIMER 635 clc 0(8,%r14),__LC_ASYNC_ENTER_TIMER 636 jl 0f 637 la %r14,__LC_ASYNC_ENTER_TIMER 6380: clc 0(8,%r14),__LC_EXIT_TIMER 639 jl 1f 640 la %r14,__LC_EXIT_TIMER 6411: clc 0(8,%r14),__LC_LAST_UPDATE_TIMER 642 jl 2f 643 la %r14,__LC_LAST_UPDATE_TIMER 6442: spt 0(%r14) 645 mvc __LC_MCCK_ENTER_TIMER(8),0(%r14) 6463: tm __LC_MCCK_CODE+2,0x09 # mwp + ia of old psw valid? 647 jno mcck_panic # no -> skip cleanup critical 648 tm %r8,0x0001 # interrupting from user ? 649 jz mcck_skip 650 UPDATE_VTIME %r14,%r15,__LC_MCCK_ENTER_TIMER 651mcck_skip: 652 SWITCH_ASYNC __LC_GPREGS_SAVE_AREA+32,__LC_PANIC_STACK,PAGE_SHIFT 653 mvc __PT_R0(64,%r11),__LC_GPREGS_SAVE_AREA 654 stm %r8,%r9,__PT_PSW(%r11) 655 xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15) 656 l %r1,BASED(.Ldo_machine_check) 657 lr %r2,%r11 # pass pointer to pt_regs 658 basr %r14,%r1 # call s390_do_machine_check 659 tm __PT_PSW+1(%r11),0x01 # returning to user ? 660 jno mcck_return 661 l %r1,__LC_KERNEL_STACK # switch to kernel stack 662 ahi %r1,-(STACK_FRAME_OVERHEAD + __PT_SIZE) 663 mvc STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11) 664 xc __SF_BACKCHAIN(4,%r1),__SF_BACKCHAIN(%r1) 665 la %r11,STACK_FRAME_OVERHEAD(%r15) 666 lr %r15,%r1 667 ssm __LC_PGM_NEW_PSW # turn dat on, keep irqs off 668 tm __TI_flags+3(%r12),_TIF_MCCK_PENDING 669 jno mcck_return 670 TRACE_IRQS_OFF 671 l %r1,BASED(.Lhandle_mcck) 672 basr %r14,%r1 # call s390_handle_mcck 673 TRACE_IRQS_ON 674mcck_return: 675 mvc __LC_RETURN_MCCK_PSW(8),__PT_PSW(%r11) # move return PSW 676 ni __LC_RETURN_MCCK_PSW+1,0xfd # clear wait state bit 677 tm __LC_RETURN_MCCK_PSW+1,0x01 # returning to user ? 678 jno 0f 679 lm %r0,%r15,__PT_R0(%r11) 680 stpt __LC_EXIT_TIMER 681 lpsw __LC_RETURN_MCCK_PSW 6820: lm %r0,%r15,__PT_R0(%r11) 683 lpsw __LC_RETURN_MCCK_PSW 684 685mcck_panic: 686 l %r14,__LC_PANIC_STACK 687 slr %r14,%r15 688 sra %r14,PAGE_SHIFT 689 jz 0f 690 l %r15,__LC_PANIC_STACK 6910: ahi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE) 692 j mcck_skip 693 694/* 695 * Restart interruption handler, kick starter for additional CPUs 696 */ 697#ifdef CONFIG_SMP 698 __CPUINIT 699ENTRY(restart_int_handler) 700 basr %r1,0 701restart_base: 702 spt restart_vtime-restart_base(%r1) 703 stck __LC_LAST_UPDATE_CLOCK 704 mvc __LC_LAST_UPDATE_TIMER(8),restart_vtime-restart_base(%r1) 705 mvc __LC_EXIT_TIMER(8),restart_vtime-restart_base(%r1) 706 l %r15,__LC_GPREGS_SAVE_AREA+60 # load ksp 707 lctl %c0,%c15,__LC_CREGS_SAVE_AREA # get new ctl regs 708 lam %a0,%a15,__LC_AREGS_SAVE_AREA 709 lm %r6,%r15,__SF_GPRS(%r15)# load registers from clone 710 l %r1,__LC_THREAD_INFO 711 mvc __LC_USER_TIMER(8),__TI_user_timer(%r1) 712 mvc __LC_SYSTEM_TIMER(8),__TI_system_timer(%r1) 713 xc __LC_STEAL_TIMER(8),__LC_STEAL_TIMER 714 ssm __LC_PGM_NEW_PSW # turn dat on, keep irqs off 715 basr %r14,0 716 l %r14,restart_addr-.(%r14) 717 basr %r14,%r14 # call start_secondary 718restart_addr: 719 .long start_secondary 720 .align 8 721restart_vtime: 722 .long 0x7fffffff,0xffffffff 723 .previous 724#else 725/* 726 * If we do not run with SMP enabled, let the new CPU crash ... 727 */ 728ENTRY(restart_int_handler) 729 basr %r1,0 730restart_base: 731 lpsw restart_crash-restart_base(%r1) 732 .align 8 733restart_crash: 734 .long 0x000a0000,0x00000000 735restart_go: 736#endif 737 738# 739# PSW restart interrupt handler 740# 741ENTRY(psw_restart_int_handler) 742 st %r15,__LC_SAVE_AREA_RESTART 743 basr %r15,0 7440: l %r15,.Lrestart_stack-0b(%r15) # load restart stack 745 l %r15,0(%r15) 746 ahi %r15,-__PT_SIZE # create pt_regs on stack 747 stm %r0,%r14,__PT_R0(%r15) 748 mvc __PT_R15(4,%r15),__LC_SAVE_AREA_RESTART 749 mvc __PT_PSW(8,%r15),__LC_RST_OLD_PSW # store restart old psw 750 ahi %r15,-STACK_FRAME_OVERHEAD 751 xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15) 752 basr %r14,0 7531: l %r14,.Ldo_restart-1b(%r14) 754 basr %r14,%r14 755 basr %r14,0 # load disabled wait PSW if 7562: lpsw restart_psw_crash-2b(%r14) # do_restart returns 757 .align 4 758.Ldo_restart: 759 .long do_restart 760.Lrestart_stack: 761 .long restart_stack 762 .align 8 763restart_psw_crash: 764 .long 0x000a0000,0x00000000 + restart_psw_crash 765 766 .section .kprobes.text, "ax" 767 768#ifdef CONFIG_CHECK_STACK 769/* 770 * The synchronous or the asynchronous stack overflowed. We are dead. 771 * No need to properly save the registers, we are going to panic anyway. 772 * Setup a pt_regs so that show_trace can provide a good call trace. 773 */ 774stack_overflow: 775 l %r15,__LC_PANIC_STACK # change to panic stack 776 ahi %r15,-__PT_SIZE # create pt_regs 777 stm %r0,%r7,__PT_R0(%r15) 778 stm %r8,%r9,__PT_PSW(%r15) 779 mvc __PT_R8(32,%r11),0(%r14) 780 lr %r15,%r11 781 ahi %r15,-STACK_FRAME_OVERHEAD 782 l %r1,BASED(1f) 783 xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15) 784 lr %r2,%r11 # pass pointer to pt_regs 785 br %r1 # branch to kernel_stack_overflow 7861: .long kernel_stack_overflow 787#endif 788 789cleanup_table: 790 .long system_call + 0x80000000 791 .long sysc_do_svc + 0x80000000 792 .long sysc_tif + 0x80000000 793 .long sysc_restore + 0x80000000 794 .long sysc_done + 0x80000000 795 .long io_tif + 0x80000000 796 .long io_restore + 0x80000000 797 .long io_done + 0x80000000 798 799cleanup_critical: 800 cl %r9,BASED(cleanup_table) # system_call 801 jl 0f 802 cl %r9,BASED(cleanup_table+4) # sysc_do_svc 803 jl cleanup_system_call 804 cl %r9,BASED(cleanup_table+8) # sysc_tif 805 jl 0f 806 cl %r9,BASED(cleanup_table+12) # sysc_restore 807 jl cleanup_sysc_tif 808 cl %r9,BASED(cleanup_table+16) # sysc_done 809 jl cleanup_sysc_restore 810 cl %r9,BASED(cleanup_table+20) # io_tif 811 jl 0f 812 cl %r9,BASED(cleanup_table+24) # io_restore 813 jl cleanup_io_tif 814 cl %r9,BASED(cleanup_table+28) # io_done 815 jl cleanup_io_restore 8160: br %r14 817 818cleanup_system_call: 819 # check if stpt has been executed 820 cl %r9,BASED(cleanup_system_call_insn) 821 jh 0f 822 mvc __LC_SYNC_ENTER_TIMER(8),__LC_ASYNC_ENTER_TIMER 823 chi %r11,__LC_SAVE_AREA_ASYNC 824 je 0f 825 mvc __LC_SYNC_ENTER_TIMER(8),__LC_MCCK_ENTER_TIMER 8260: # check if stm has been executed 827 cl %r9,BASED(cleanup_system_call_insn+4) 828 jh 0f 829 mvc __LC_SAVE_AREA_SYNC(32),0(%r11) 8300: # set up saved registers r12, and r13 831 st %r12,16(%r11) # r12 thread-info pointer 832 st %r13,20(%r11) # r13 literal-pool pointer 833 # check if the user time calculation has been done 834 cl %r9,BASED(cleanup_system_call_insn+8) 835 jh 0f 836 l %r10,__LC_EXIT_TIMER 837 l %r15,__LC_EXIT_TIMER+4 838 SUB64 %r10,%r15,__LC_SYNC_ENTER_TIMER 839 ADD64 %r10,%r15,__LC_USER_TIMER 840 st %r10,__LC_USER_TIMER 841 st %r15,__LC_USER_TIMER+4 8420: # check if the system time calculation has been done 843 cl %r9,BASED(cleanup_system_call_insn+12) 844 jh 0f 845 l %r10,__LC_LAST_UPDATE_TIMER 846 l %r15,__LC_LAST_UPDATE_TIMER+4 847 SUB64 %r10,%r15,__LC_EXIT_TIMER 848 ADD64 %r10,%r15,__LC_SYSTEM_TIMER 849 st %r10,__LC_SYSTEM_TIMER 850 st %r15,__LC_SYSTEM_TIMER+4 8510: # update accounting time stamp 852 mvc __LC_LAST_UPDATE_TIMER(8),__LC_SYNC_ENTER_TIMER 853 # set up saved register 11 854 l %r15,__LC_KERNEL_STACK 855 ahi %r15,-__PT_SIZE 856 st %r15,12(%r11) # r11 pt_regs pointer 857 # fill pt_regs 858 mvc __PT_R8(32,%r15),__LC_SAVE_AREA_SYNC 859 stm %r0,%r7,__PT_R0(%r15) 860 mvc __PT_PSW(8,%r15),__LC_SVC_OLD_PSW 861 mvc __PT_INT_CODE(4,%r15),__LC_SVC_ILC 862 # setup saved register 15 863 ahi %r15,-STACK_FRAME_OVERHEAD 864 st %r15,28(%r11) # r15 stack pointer 865 # set new psw address and exit 866 l %r9,BASED(cleanup_table+4) # sysc_do_svc + 0x80000000 867 br %r14 868cleanup_system_call_insn: 869 .long system_call + 0x80000000 870 .long sysc_stm + 0x80000000 871 .long sysc_vtime + 0x80000000 + 36 872 .long sysc_vtime + 0x80000000 + 76 873 874cleanup_sysc_tif: 875 l %r9,BASED(cleanup_table+8) # sysc_tif + 0x80000000 876 br %r14 877 878cleanup_sysc_restore: 879 cl %r9,BASED(cleanup_sysc_restore_insn) 880 jhe 0f 881 l %r9,12(%r11) # get saved pointer to pt_regs 882 mvc __LC_RETURN_PSW(8),__PT_PSW(%r9) 883 mvc 0(32,%r11),__PT_R8(%r9) 884 lm %r0,%r7,__PT_R0(%r9) 8850: lm %r8,%r9,__LC_RETURN_PSW 886 br %r14 887cleanup_sysc_restore_insn: 888 .long sysc_done - 4 + 0x80000000 889 890cleanup_io_tif: 891 l %r9,BASED(cleanup_table+20) # io_tif + 0x80000000 892 br %r14 893 894cleanup_io_restore: 895 cl %r9,BASED(cleanup_io_restore_insn) 896 jhe 0f 897 l %r9,12(%r11) # get saved r11 pointer to pt_regs 898 mvc __LC_RETURN_PSW(8),__PT_PSW(%r9) 899 ni __LC_RETURN_PSW+1,0xfd # clear wait state bit 900 mvc 0(32,%r11),__PT_R8(%r9) 901 lm %r0,%r7,__PT_R0(%r9) 9020: lm %r8,%r9,__LC_RETURN_PSW 903 br %r14 904cleanup_io_restore_insn: 905 .long io_done - 4 + 0x80000000 906 907/* 908 * Integer constants 909 */ 910 .align 4 911.Lnr_syscalls: .long NR_syscalls 912 913/* 914 * Symbol constants 915 */ 916.Ldo_machine_check: .long s390_do_machine_check 917.Lhandle_mcck: .long s390_handle_mcck 918.Ldo_IRQ: .long do_IRQ 919.Ldo_extint: .long do_extint 920.Ldo_signal: .long do_signal 921.Ldo_notify_resume: .long do_notify_resume 922.Ldo_per_trap: .long do_per_trap 923.Ldo_execve: .long do_execve 924.Lexecve_tail: .long execve_tail 925.Ljump_table: .long pgm_check_table 926.Lschedule: .long schedule 927#ifdef CONFIG_PREEMPT 928.Lpreempt_irq: .long preempt_schedule_irq 929#endif 930.Ltrace_enter: .long do_syscall_trace_enter 931.Ltrace_exit: .long do_syscall_trace_exit 932.Lschedule_tail: .long schedule_tail 933.Lsys_call_table: .long sys_call_table 934.Lsysc_per: .long sysc_per + 0x80000000 935#ifdef CONFIG_TRACE_IRQFLAGS 936.Lhardirqs_on: .long trace_hardirqs_on_caller 937.Lhardirqs_off: .long trace_hardirqs_off_caller 938#endif 939#ifdef CONFIG_LOCKDEP 940.Llockdep_sys_exit: .long lockdep_sys_exit 941#endif 942.Lcritical_start: .long __critical_start + 0x80000000 943.Lcritical_length: .long __critical_end - __critical_start 944 945 .section .rodata, "a" 946#define SYSCALL(esa,esame,emu) .long esa 947 .globl sys_call_table 948sys_call_table: 949#include "syscalls.S" 950#undef SYSCALL 951