1/* 2 * S390 low-level entry points. 3 * 4 * Copyright IBM Corp. 1999, 2012 5 * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com), 6 * Hartmut Penner (hp@de.ibm.com), 7 * Denis Joseph Barrow (djbarrow@de.ibm.com,barrow_dj@yahoo.com), 8 * Heiko Carstens <heiko.carstens@de.ibm.com> 9 */ 10 11#include <linux/init.h> 12#include <linux/linkage.h> 13#include <asm/cache.h> 14#include <asm/errno.h> 15#include <asm/ptrace.h> 16#include <asm/thread_info.h> 17#include <asm/asm-offsets.h> 18#include <asm/unistd.h> 19#include <asm/page.h> 20#include <asm/sigp.h> 21 22__PT_R0 = __PT_GPRS 23__PT_R1 = __PT_GPRS + 4 24__PT_R2 = __PT_GPRS + 8 25__PT_R3 = __PT_GPRS + 12 26__PT_R4 = __PT_GPRS + 16 27__PT_R5 = __PT_GPRS + 20 28__PT_R6 = __PT_GPRS + 24 29__PT_R7 = __PT_GPRS + 28 30__PT_R8 = __PT_GPRS + 32 31__PT_R9 = __PT_GPRS + 36 32__PT_R10 = __PT_GPRS + 40 33__PT_R11 = __PT_GPRS + 44 34__PT_R12 = __PT_GPRS + 48 35__PT_R13 = __PT_GPRS + 524 36__PT_R14 = __PT_GPRS + 56 37__PT_R15 = __PT_GPRS + 60 38 39_TIF_WORK_SVC = (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_NEED_RESCHED | \ 40 _TIF_MCCK_PENDING | _TIF_PER_TRAP ) 41_TIF_WORK_INT = (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_NEED_RESCHED | \ 42 _TIF_MCCK_PENDING) 43_TIF_TRACE = (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | _TIF_SECCOMP | \ 44 _TIF_SYSCALL_TRACEPOINT) 45 46STACK_SHIFT = PAGE_SHIFT + THREAD_ORDER 47STACK_SIZE = 1 << STACK_SHIFT 48STACK_INIT = STACK_SIZE - STACK_FRAME_OVERHEAD - __PT_SIZE 49 50#define BASED(name) name-system_call(%r13) 51 52 .macro TRACE_IRQS_ON 53#ifdef CONFIG_TRACE_IRQFLAGS 54 basr %r2,%r0 55 l %r1,BASED(.Lhardirqs_on) 56 basr %r14,%r1 # call trace_hardirqs_on_caller 57#endif 58 .endm 59 60 .macro TRACE_IRQS_OFF 61#ifdef CONFIG_TRACE_IRQFLAGS 62 basr %r2,%r0 63 l %r1,BASED(.Lhardirqs_off) 64 basr %r14,%r1 # call trace_hardirqs_off_caller 65#endif 66 .endm 67 68 .macro LOCKDEP_SYS_EXIT 69#ifdef CONFIG_LOCKDEP 70 tm __PT_PSW+1(%r11),0x01 # returning to user ? 71 jz .+10 72 l %r1,BASED(.Llockdep_sys_exit) 73 basr %r14,%r1 # call lockdep_sys_exit 74#endif 75 .endm 76 77 .macro CHECK_STACK stacksize,savearea 78#ifdef CONFIG_CHECK_STACK 79 tml %r15,\stacksize - CONFIG_STACK_GUARD 80 la %r14,\savearea 81 jz stack_overflow 82#endif 83 .endm 84 85 .macro SWITCH_ASYNC savearea,stack,shift 86 tmh %r8,0x0001 # interrupting from user ? 87 jnz 1f 88 lr %r14,%r9 89 sl %r14,BASED(.Lcritical_start) 90 cl %r14,BASED(.Lcritical_length) 91 jhe 0f 92 la %r11,\savearea # inside critical section, do cleanup 93 bras %r14,cleanup_critical 94 tmh %r8,0x0001 # retest problem state after cleanup 95 jnz 1f 960: l %r14,\stack # are we already on the target stack? 97 slr %r14,%r15 98 sra %r14,\shift 99 jnz 1f 100 CHECK_STACK 1<<\shift,\savearea 101 ahi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE) 102 j 2f 1031: l %r15,\stack # load target stack 1042: la %r11,STACK_FRAME_OVERHEAD(%r15) 105 .endm 106 107 .macro ADD64 high,low,timer 108 al \high,\timer 109 al \low,4+\timer 110 brc 12,.+8 111 ahi \high,1 112 .endm 113 114 .macro SUB64 high,low,timer 115 sl \high,\timer 116 sl \low,4+\timer 117 brc 3,.+8 118 ahi \high,-1 119 .endm 120 121 .macro UPDATE_VTIME high,low,enter_timer 122 lm \high,\low,__LC_EXIT_TIMER 123 SUB64 \high,\low,\enter_timer 124 ADD64 \high,\low,__LC_USER_TIMER 125 stm \high,\low,__LC_USER_TIMER 126 lm \high,\low,__LC_LAST_UPDATE_TIMER 127 SUB64 \high,\low,__LC_EXIT_TIMER 128 ADD64 \high,\low,__LC_SYSTEM_TIMER 129 stm \high,\low,__LC_SYSTEM_TIMER 130 mvc __LC_LAST_UPDATE_TIMER(8),\enter_timer 131 .endm 132 133 .macro REENABLE_IRQS 134 st %r8,__LC_RETURN_PSW 135 ni __LC_RETURN_PSW,0xbf 136 ssm __LC_RETURN_PSW 137 .endm 138 139 .section .kprobes.text, "ax" 140 141/* 142 * Scheduler resume function, called by switch_to 143 * gpr2 = (task_struct *) prev 144 * gpr3 = (task_struct *) next 145 * Returns: 146 * gpr2 = prev 147 */ 148ENTRY(__switch_to) 149 stm %r6,%r15,__SF_GPRS(%r15) # store gprs of prev task 150 st %r15,__THREAD_ksp(%r2) # store kernel stack of prev 151 l %r4,__THREAD_info(%r2) # get thread_info of prev 152 l %r5,__THREAD_info(%r3) # get thread_info of next 153 lr %r15,%r5 154 ahi %r15,STACK_INIT # end of kernel stack of next 155 st %r3,__LC_CURRENT # store task struct of next 156 st %r5,__LC_THREAD_INFO # store thread info of next 157 st %r15,__LC_KERNEL_STACK # store end of kernel stack 158 lctl %c4,%c4,__TASK_pid(%r3) # load pid to control reg. 4 159 mvc __LC_CURRENT_PID(4,%r0),__TASK_pid(%r3) # store pid of next 160 l %r15,__THREAD_ksp(%r3) # load kernel stack of next 161 tm __TI_flags+3(%r4),_TIF_MCCK_PENDING # machine check pending? 162 jz 0f 163 ni __TI_flags+3(%r4),255-_TIF_MCCK_PENDING # clear flag in prev 164 oi __TI_flags+3(%r5),_TIF_MCCK_PENDING # set it in next 1650: lm %r6,%r15,__SF_GPRS(%r15) # load gprs of next task 166 br %r14 167 168__critical_start: 169/* 170 * SVC interrupt handler routine. System calls are synchronous events and 171 * are executed with interrupts enabled. 172 */ 173 174ENTRY(system_call) 175 stpt __LC_SYNC_ENTER_TIMER 176sysc_stm: 177 stm %r8,%r15,__LC_SAVE_AREA_SYNC 178 l %r12,__LC_THREAD_INFO 179 l %r13,__LC_SVC_NEW_PSW+4 180sysc_per: 181 l %r15,__LC_KERNEL_STACK 182 la %r11,STACK_FRAME_OVERHEAD(%r15) # pointer to pt_regs 183sysc_vtime: 184 UPDATE_VTIME %r8,%r9,__LC_SYNC_ENTER_TIMER 185 stm %r0,%r7,__PT_R0(%r11) 186 mvc __PT_R8(32,%r11),__LC_SAVE_AREA_SYNC 187 mvc __PT_PSW(8,%r11),__LC_SVC_OLD_PSW 188 mvc __PT_INT_CODE(4,%r11),__LC_SVC_ILC 189sysc_do_svc: 190 oi __TI_flags+3(%r12),_TIF_SYSCALL 191 l %r10,__TI_sysc_table(%r12) # 31 bit system call table 192 lh %r8,__PT_INT_CODE+2(%r11) 193 sla %r8,2 # shift and test for svc0 194 jnz sysc_nr_ok 195 # svc 0: system call number in %r1 196 cl %r1,BASED(.Lnr_syscalls) 197 jnl sysc_nr_ok 198 sth %r1,__PT_INT_CODE+2(%r11) 199 lr %r8,%r1 200 sla %r8,2 201sysc_nr_ok: 202 xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15) 203 st %r2,__PT_ORIG_GPR2(%r11) 204 st %r7,STACK_FRAME_OVERHEAD(%r15) 205 l %r9,0(%r8,%r10) # get system call addr. 206 tm __TI_flags+2(%r12),_TIF_TRACE >> 8 207 jnz sysc_tracesys 208 basr %r14,%r9 # call sys_xxxx 209 st %r2,__PT_R2(%r11) # store return value 210 211sysc_return: 212 LOCKDEP_SYS_EXIT 213sysc_tif: 214 tm __PT_PSW+1(%r11),0x01 # returning to user ? 215 jno sysc_restore 216 tm __TI_flags+3(%r12),_TIF_WORK_SVC 217 jnz sysc_work # check for work 218 ni __TI_flags+3(%r12),255-_TIF_SYSCALL 219sysc_restore: 220 mvc __LC_RETURN_PSW(8),__PT_PSW(%r11) 221 stpt __LC_EXIT_TIMER 222 lm %r0,%r15,__PT_R0(%r11) 223 lpsw __LC_RETURN_PSW 224sysc_done: 225 226# 227# One of the work bits is on. Find out which one. 228# 229sysc_work: 230 tm __TI_flags+3(%r12),_TIF_MCCK_PENDING 231 jo sysc_mcck_pending 232 tm __TI_flags+3(%r12),_TIF_NEED_RESCHED 233 jo sysc_reschedule 234 tm __TI_flags+3(%r12),_TIF_PER_TRAP 235 jo sysc_singlestep 236 tm __TI_flags+3(%r12),_TIF_SIGPENDING 237 jo sysc_sigpending 238 tm __TI_flags+3(%r12),_TIF_NOTIFY_RESUME 239 jo sysc_notify_resume 240 j sysc_return # beware of critical section cleanup 241 242# 243# _TIF_NEED_RESCHED is set, call schedule 244# 245sysc_reschedule: 246 l %r1,BASED(.Lschedule) 247 la %r14,BASED(sysc_return) 248 br %r1 # call schedule 249 250# 251# _TIF_MCCK_PENDING is set, call handler 252# 253sysc_mcck_pending: 254 l %r1,BASED(.Lhandle_mcck) 255 la %r14,BASED(sysc_return) 256 br %r1 # TIF bit will be cleared by handler 257 258# 259# _TIF_SIGPENDING is set, call do_signal 260# 261sysc_sigpending: 262 lr %r2,%r11 # pass pointer to pt_regs 263 l %r1,BASED(.Ldo_signal) 264 basr %r14,%r1 # call do_signal 265 tm __TI_flags+3(%r12),_TIF_SYSCALL 266 jno sysc_return 267 lm %r2,%r7,__PT_R2(%r11) # load svc arguments 268 xr %r8,%r8 # svc 0 returns -ENOSYS 269 clc __PT_INT_CODE+2(2,%r11),BASED(.Lnr_syscalls+2) 270 jnl sysc_nr_ok # invalid svc number -> do svc 0 271 lh %r8,__PT_INT_CODE+2(%r11) # load new svc number 272 sla %r8,2 273 j sysc_nr_ok # restart svc 274 275# 276# _TIF_NOTIFY_RESUME is set, call do_notify_resume 277# 278sysc_notify_resume: 279 lr %r2,%r11 # pass pointer to pt_regs 280 l %r1,BASED(.Ldo_notify_resume) 281 la %r14,BASED(sysc_return) 282 br %r1 # call do_notify_resume 283 284# 285# _TIF_PER_TRAP is set, call do_per_trap 286# 287sysc_singlestep: 288 ni __TI_flags+3(%r12),255-_TIF_PER_TRAP 289 lr %r2,%r11 # pass pointer to pt_regs 290 l %r1,BASED(.Ldo_per_trap) 291 la %r14,BASED(sysc_return) 292 br %r1 # call do_per_trap 293 294# 295# call tracehook_report_syscall_entry/tracehook_report_syscall_exit before 296# and after the system call 297# 298sysc_tracesys: 299 l %r1,BASED(.Ltrace_enter) 300 lr %r2,%r11 # pass pointer to pt_regs 301 la %r3,0 302 xr %r0,%r0 303 icm %r0,3,__PT_INT_CODE+2(%r11) 304 st %r0,__PT_R2(%r11) 305 basr %r14,%r1 # call do_syscall_trace_enter 306 cl %r2,BASED(.Lnr_syscalls) 307 jnl sysc_tracenogo 308 lr %r8,%r2 309 sll %r8,2 310 l %r9,0(%r8,%r10) 311sysc_tracego: 312 lm %r3,%r7,__PT_R3(%r11) 313 st %r7,STACK_FRAME_OVERHEAD(%r15) 314 l %r2,__PT_ORIG_GPR2(%r11) 315 basr %r14,%r9 # call sys_xxx 316 st %r2,__PT_R2(%r11) # store return value 317sysc_tracenogo: 318 tm __TI_flags+2(%r12),_TIF_TRACE >> 8 319 jz sysc_return 320 l %r1,BASED(.Ltrace_exit) 321 lr %r2,%r11 # pass pointer to pt_regs 322 la %r14,BASED(sysc_return) 323 br %r1 # call do_syscall_trace_exit 324 325# 326# a new process exits the kernel with ret_from_fork 327# 328ENTRY(ret_from_fork) 329 la %r11,STACK_FRAME_OVERHEAD(%r15) 330 l %r12,__LC_THREAD_INFO 331 l %r13,__LC_SVC_NEW_PSW+4 332 l %r1,BASED(.Lschedule_tail) 333 basr %r14,%r1 # call schedule_tail 334 TRACE_IRQS_ON 335 ssm __LC_SVC_NEW_PSW # reenable interrupts 336 tm __PT_PSW+1(%r11),0x01 # forking a kernel thread ? 337 jne sysc_tracenogo 338 # it's a kernel thread 339 lm %r9,%r10,__PT_R9(%r11) # load gprs 340ENTRY(kernel_thread_starter) 341 la %r2,0(%r10) 342 basr %r14,%r9 343 j sysc_tracenogo 344 345/* 346 * Program check handler routine 347 */ 348 349ENTRY(pgm_check_handler) 350 stpt __LC_SYNC_ENTER_TIMER 351 stm %r8,%r15,__LC_SAVE_AREA_SYNC 352 l %r12,__LC_THREAD_INFO 353 l %r13,__LC_SVC_NEW_PSW+4 354 lm %r8,%r9,__LC_PGM_OLD_PSW 355 tmh %r8,0x0001 # test problem state bit 356 jnz 1f # -> fault in user space 357 tmh %r8,0x4000 # PER bit set in old PSW ? 358 jnz 0f # -> enabled, can't be a double fault 359 tm __LC_PGM_ILC+3,0x80 # check for per exception 360 jnz pgm_svcper # -> single stepped svc 3610: CHECK_STACK STACK_SIZE,__LC_SAVE_AREA_SYNC 362 ahi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE) 363 j 2f 3641: UPDATE_VTIME %r14,%r15,__LC_SYNC_ENTER_TIMER 365 l %r15,__LC_KERNEL_STACK 3662: la %r11,STACK_FRAME_OVERHEAD(%r15) 367 stm %r0,%r7,__PT_R0(%r11) 368 mvc __PT_R8(32,%r11),__LC_SAVE_AREA_SYNC 369 stm %r8,%r9,__PT_PSW(%r11) 370 mvc __PT_INT_CODE(4,%r11),__LC_PGM_ILC 371 mvc __PT_INT_PARM_LONG(4,%r11),__LC_TRANS_EXC_CODE 372 tm __LC_PGM_ILC+3,0x80 # check for per exception 373 jz 0f 374 l %r1,__TI_task(%r12) 375 tmh %r8,0x0001 # kernel per event ? 376 jz pgm_kprobe 377 oi __TI_flags+3(%r12),_TIF_PER_TRAP 378 mvc __THREAD_per_address(4,%r1),__LC_PER_ADDRESS 379 mvc __THREAD_per_cause(2,%r1),__LC_PER_CAUSE 380 mvc __THREAD_per_paid(1,%r1),__LC_PER_PAID 3810: REENABLE_IRQS 382 xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15) 383 l %r1,BASED(.Ljump_table) 384 la %r10,0x7f 385 n %r10,__PT_INT_CODE(%r11) 386 je sysc_return 387 sll %r10,2 388 l %r1,0(%r10,%r1) # load address of handler routine 389 lr %r2,%r11 # pass pointer to pt_regs 390 basr %r14,%r1 # branch to interrupt-handler 391 j sysc_return 392 393# 394# PER event in supervisor state, must be kprobes 395# 396pgm_kprobe: 397 REENABLE_IRQS 398 xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15) 399 l %r1,BASED(.Ldo_per_trap) 400 lr %r2,%r11 # pass pointer to pt_regs 401 basr %r14,%r1 # call do_per_trap 402 j sysc_return 403 404# 405# single stepped system call 406# 407pgm_svcper: 408 oi __TI_flags+3(%r12),_TIF_PER_TRAP 409 mvc __LC_RETURN_PSW(4),__LC_SVC_NEW_PSW 410 mvc __LC_RETURN_PSW+4(4),BASED(.Lsysc_per) 411 lpsw __LC_RETURN_PSW # branch to sysc_per and enable irqs 412 413/* 414 * IO interrupt handler routine 415 */ 416 417ENTRY(io_int_handler) 418 stck __LC_INT_CLOCK 419 stpt __LC_ASYNC_ENTER_TIMER 420 stm %r8,%r15,__LC_SAVE_AREA_ASYNC 421 l %r12,__LC_THREAD_INFO 422 l %r13,__LC_SVC_NEW_PSW+4 423 lm %r8,%r9,__LC_IO_OLD_PSW 424 tmh %r8,0x0001 # interrupting from user ? 425 jz io_skip 426 UPDATE_VTIME %r14,%r15,__LC_ASYNC_ENTER_TIMER 427io_skip: 428 SWITCH_ASYNC __LC_SAVE_AREA_ASYNC,__LC_ASYNC_STACK,STACK_SHIFT 429 stm %r0,%r7,__PT_R0(%r11) 430 mvc __PT_R8(32,%r11),__LC_SAVE_AREA_ASYNC 431 stm %r8,%r9,__PT_PSW(%r11) 432 mvc __PT_INT_CODE(12,%r11),__LC_SUBCHANNEL_ID 433 TRACE_IRQS_OFF 434 xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15) 435io_loop: 436 l %r1,BASED(.Ldo_IRQ) 437 lr %r2,%r11 # pass pointer to pt_regs 438 basr %r14,%r1 # call do_IRQ 439 tm __LC_MACHINE_FLAGS+2,0x10 # MACHINE_FLAG_LPAR 440 jz io_return 441 tpi 0 442 jz io_return 443 mvc __PT_INT_CODE(12,%r11),__LC_SUBCHANNEL_ID 444 j io_loop 445io_return: 446 LOCKDEP_SYS_EXIT 447 TRACE_IRQS_ON 448io_tif: 449 tm __TI_flags+3(%r12),_TIF_WORK_INT 450 jnz io_work # there is work to do (signals etc.) 451io_restore: 452 mvc __LC_RETURN_PSW(8),__PT_PSW(%r11) 453 stpt __LC_EXIT_TIMER 454 lm %r0,%r15,__PT_R0(%r11) 455 lpsw __LC_RETURN_PSW 456io_done: 457 458# 459# There is work todo, find out in which context we have been interrupted: 460# 1) if we return to user space we can do all _TIF_WORK_INT work 461# 2) if we return to kernel code and preemptive scheduling is enabled check 462# the preemption counter and if it is zero call preempt_schedule_irq 463# Before any work can be done, a switch to the kernel stack is required. 464# 465io_work: 466 tm __PT_PSW+1(%r11),0x01 # returning to user ? 467 jo io_work_user # yes -> do resched & signal 468#ifdef CONFIG_PREEMPT 469 # check for preemptive scheduling 470 icm %r0,15,__TI_precount(%r12) 471 jnz io_restore # preemption disabled 472 tm __TI_flags+3(%r12),_TIF_NEED_RESCHED 473 jno io_restore 474 # switch to kernel stack 475 l %r1,__PT_R15(%r11) 476 ahi %r1,-(STACK_FRAME_OVERHEAD + __PT_SIZE) 477 mvc STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11) 478 xc __SF_BACKCHAIN(4,%r1),__SF_BACKCHAIN(%r1) 479 la %r11,STACK_FRAME_OVERHEAD(%r1) 480 lr %r15,%r1 481 # TRACE_IRQS_ON already done at io_return, call 482 # TRACE_IRQS_OFF to keep things symmetrical 483 TRACE_IRQS_OFF 484 l %r1,BASED(.Lpreempt_irq) 485 basr %r14,%r1 # call preempt_schedule_irq 486 j io_return 487#else 488 j io_restore 489#endif 490 491# 492# Need to do work before returning to userspace, switch to kernel stack 493# 494io_work_user: 495 l %r1,__LC_KERNEL_STACK 496 mvc STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11) 497 xc __SF_BACKCHAIN(4,%r1),__SF_BACKCHAIN(%r1) 498 la %r11,STACK_FRAME_OVERHEAD(%r1) 499 lr %r15,%r1 500 501# 502# One of the work bits is on. Find out which one. 503# Checked are: _TIF_SIGPENDING, _TIF_NOTIFY_RESUME, _TIF_NEED_RESCHED 504# and _TIF_MCCK_PENDING 505# 506io_work_tif: 507 tm __TI_flags+3(%r12),_TIF_MCCK_PENDING 508 jo io_mcck_pending 509 tm __TI_flags+3(%r12),_TIF_NEED_RESCHED 510 jo io_reschedule 511 tm __TI_flags+3(%r12),_TIF_SIGPENDING 512 jo io_sigpending 513 tm __TI_flags+3(%r12),_TIF_NOTIFY_RESUME 514 jo io_notify_resume 515 j io_return # beware of critical section cleanup 516 517# 518# _TIF_MCCK_PENDING is set, call handler 519# 520io_mcck_pending: 521 # TRACE_IRQS_ON already done at io_return 522 l %r1,BASED(.Lhandle_mcck) 523 basr %r14,%r1 # TIF bit will be cleared by handler 524 TRACE_IRQS_OFF 525 j io_return 526 527# 528# _TIF_NEED_RESCHED is set, call schedule 529# 530io_reschedule: 531 # TRACE_IRQS_ON already done at io_return 532 l %r1,BASED(.Lschedule) 533 ssm __LC_SVC_NEW_PSW # reenable interrupts 534 basr %r14,%r1 # call scheduler 535 ssm __LC_PGM_NEW_PSW # disable I/O and ext. interrupts 536 TRACE_IRQS_OFF 537 j io_return 538 539# 540# _TIF_SIGPENDING is set, call do_signal 541# 542io_sigpending: 543 # TRACE_IRQS_ON already done at io_return 544 l %r1,BASED(.Ldo_signal) 545 ssm __LC_SVC_NEW_PSW # reenable interrupts 546 lr %r2,%r11 # pass pointer to pt_regs 547 basr %r14,%r1 # call do_signal 548 ssm __LC_PGM_NEW_PSW # disable I/O and ext. interrupts 549 TRACE_IRQS_OFF 550 j io_return 551 552# 553# _TIF_SIGPENDING is set, call do_signal 554# 555io_notify_resume: 556 # TRACE_IRQS_ON already done at io_return 557 l %r1,BASED(.Ldo_notify_resume) 558 ssm __LC_SVC_NEW_PSW # reenable interrupts 559 lr %r2,%r11 # pass pointer to pt_regs 560 basr %r14,%r1 # call do_notify_resume 561 ssm __LC_PGM_NEW_PSW # disable I/O and ext. interrupts 562 TRACE_IRQS_OFF 563 j io_return 564 565/* 566 * External interrupt handler routine 567 */ 568 569ENTRY(ext_int_handler) 570 stck __LC_INT_CLOCK 571 stpt __LC_ASYNC_ENTER_TIMER 572 stm %r8,%r15,__LC_SAVE_AREA_ASYNC 573 l %r12,__LC_THREAD_INFO 574 l %r13,__LC_SVC_NEW_PSW+4 575 lm %r8,%r9,__LC_EXT_OLD_PSW 576 tmh %r8,0x0001 # interrupting from user ? 577 jz ext_skip 578 UPDATE_VTIME %r14,%r15,__LC_ASYNC_ENTER_TIMER 579ext_skip: 580 SWITCH_ASYNC __LC_SAVE_AREA_ASYNC,__LC_ASYNC_STACK,STACK_SHIFT 581 stm %r0,%r7,__PT_R0(%r11) 582 mvc __PT_R8(32,%r11),__LC_SAVE_AREA_ASYNC 583 stm %r8,%r9,__PT_PSW(%r11) 584 mvc __PT_INT_CODE(4,%r11),__LC_EXT_CPU_ADDR 585 mvc __PT_INT_PARM(4,%r11),__LC_EXT_PARAMS 586 TRACE_IRQS_OFF 587 lr %r2,%r11 # pass pointer to pt_regs 588 l %r1,BASED(.Ldo_extint) 589 basr %r14,%r1 # call do_extint 590 j io_return 591 592/* 593 * Load idle PSW. The second "half" of this function is in cleanup_idle. 594 */ 595ENTRY(psw_idle) 596 st %r3,__SF_EMPTY(%r15) 597 basr %r1,0 598 la %r1,psw_idle_lpsw+4-.(%r1) 599 st %r1,__SF_EMPTY+4(%r15) 600 oi __SF_EMPTY+4(%r15),0x80 601 stck __CLOCK_IDLE_ENTER(%r2) 602 stpt __TIMER_IDLE_ENTER(%r2) 603psw_idle_lpsw: 604 lpsw __SF_EMPTY(%r15) 605 br %r14 606psw_idle_end: 607 608__critical_end: 609 610/* 611 * Machine check handler routines 612 */ 613 614ENTRY(mcck_int_handler) 615 stck __LC_MCCK_CLOCK 616 spt __LC_CPU_TIMER_SAVE_AREA # revalidate cpu timer 617 lm %r0,%r15,__LC_GPREGS_SAVE_AREA # revalidate gprs 618 l %r12,__LC_THREAD_INFO 619 l %r13,__LC_SVC_NEW_PSW+4 620 lm %r8,%r9,__LC_MCK_OLD_PSW 621 tm __LC_MCCK_CODE,0x80 # system damage? 622 jo mcck_panic # yes -> rest of mcck code invalid 623 la %r14,__LC_CPU_TIMER_SAVE_AREA 624 mvc __LC_MCCK_ENTER_TIMER(8),0(%r14) 625 tm __LC_MCCK_CODE+5,0x02 # stored cpu timer value valid? 626 jo 3f 627 la %r14,__LC_SYNC_ENTER_TIMER 628 clc 0(8,%r14),__LC_ASYNC_ENTER_TIMER 629 jl 0f 630 la %r14,__LC_ASYNC_ENTER_TIMER 6310: clc 0(8,%r14),__LC_EXIT_TIMER 632 jl 1f 633 la %r14,__LC_EXIT_TIMER 6341: clc 0(8,%r14),__LC_LAST_UPDATE_TIMER 635 jl 2f 636 la %r14,__LC_LAST_UPDATE_TIMER 6372: spt 0(%r14) 638 mvc __LC_MCCK_ENTER_TIMER(8),0(%r14) 6393: tm __LC_MCCK_CODE+2,0x09 # mwp + ia of old psw valid? 640 jno mcck_panic # no -> skip cleanup critical 641 tm %r8,0x0001 # interrupting from user ? 642 jz mcck_skip 643 UPDATE_VTIME %r14,%r15,__LC_MCCK_ENTER_TIMER 644mcck_skip: 645 SWITCH_ASYNC __LC_GPREGS_SAVE_AREA+32,__LC_PANIC_STACK,PAGE_SHIFT 646 stm %r0,%r7,__PT_R0(%r11) 647 mvc __PT_R8(32,%r11),__LC_GPREGS_SAVE_AREA+32 648 stm %r8,%r9,__PT_PSW(%r11) 649 xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15) 650 l %r1,BASED(.Ldo_machine_check) 651 lr %r2,%r11 # pass pointer to pt_regs 652 basr %r14,%r1 # call s390_do_machine_check 653 tm __PT_PSW+1(%r11),0x01 # returning to user ? 654 jno mcck_return 655 l %r1,__LC_KERNEL_STACK # switch to kernel stack 656 mvc STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11) 657 xc __SF_BACKCHAIN(4,%r1),__SF_BACKCHAIN(%r1) 658 la %r11,STACK_FRAME_OVERHEAD(%r15) 659 lr %r15,%r1 660 ssm __LC_PGM_NEW_PSW # turn dat on, keep irqs off 661 tm __TI_flags+3(%r12),_TIF_MCCK_PENDING 662 jno mcck_return 663 TRACE_IRQS_OFF 664 l %r1,BASED(.Lhandle_mcck) 665 basr %r14,%r1 # call s390_handle_mcck 666 TRACE_IRQS_ON 667mcck_return: 668 mvc __LC_RETURN_MCCK_PSW(8),__PT_PSW(%r11) # move return PSW 669 tm __LC_RETURN_MCCK_PSW+1,0x01 # returning to user ? 670 jno 0f 671 lm %r0,%r15,__PT_R0(%r11) 672 stpt __LC_EXIT_TIMER 673 lpsw __LC_RETURN_MCCK_PSW 6740: lm %r0,%r15,__PT_R0(%r11) 675 lpsw __LC_RETURN_MCCK_PSW 676 677mcck_panic: 678 l %r14,__LC_PANIC_STACK 679 slr %r14,%r15 680 sra %r14,PAGE_SHIFT 681 jz 0f 682 l %r15,__LC_PANIC_STACK 683 j mcck_skip 6840: ahi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE) 685 j mcck_skip 686 687# 688# PSW restart interrupt handler 689# 690ENTRY(restart_int_handler) 691 st %r15,__LC_SAVE_AREA_RESTART 692 l %r15,__LC_RESTART_STACK 693 ahi %r15,-__PT_SIZE # create pt_regs on stack 694 xc 0(__PT_SIZE,%r15),0(%r15) 695 stm %r0,%r14,__PT_R0(%r15) 696 mvc __PT_R15(4,%r15),__LC_SAVE_AREA_RESTART 697 mvc __PT_PSW(8,%r15),__LC_RST_OLD_PSW # store restart old psw 698 ahi %r15,-STACK_FRAME_OVERHEAD # create stack frame on stack 699 xc 0(STACK_FRAME_OVERHEAD,%r15),0(%r15) 700 l %r1,__LC_RESTART_FN # load fn, parm & source cpu 701 l %r2,__LC_RESTART_DATA 702 l %r3,__LC_RESTART_SOURCE 703 ltr %r3,%r3 # test source cpu address 704 jm 1f # negative -> skip source stop 7050: sigp %r4,%r3,SIGP_SENSE # sigp sense to source cpu 706 brc 10,0b # wait for status stored 7071: basr %r14,%r1 # call function 708 stap __SF_EMPTY(%r15) # store cpu address 709 lh %r3,__SF_EMPTY(%r15) 7102: sigp %r4,%r3,SIGP_STOP # sigp stop to current cpu 711 brc 2,2b 7123: j 3b 713 714 .section .kprobes.text, "ax" 715 716#ifdef CONFIG_CHECK_STACK 717/* 718 * The synchronous or the asynchronous stack overflowed. We are dead. 719 * No need to properly save the registers, we are going to panic anyway. 720 * Setup a pt_regs so that show_trace can provide a good call trace. 721 */ 722stack_overflow: 723 l %r15,__LC_PANIC_STACK # change to panic stack 724 la %r11,STACK_FRAME_OVERHEAD(%r15) 725 stm %r0,%r7,__PT_R0(%r11) 726 stm %r8,%r9,__PT_PSW(%r11) 727 mvc __PT_R8(32,%r11),0(%r14) 728 l %r1,BASED(1f) 729 xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15) 730 lr %r2,%r11 # pass pointer to pt_regs 731 br %r1 # branch to kernel_stack_overflow 7321: .long kernel_stack_overflow 733#endif 734 735cleanup_table: 736 .long system_call + 0x80000000 737 .long sysc_do_svc + 0x80000000 738 .long sysc_tif + 0x80000000 739 .long sysc_restore + 0x80000000 740 .long sysc_done + 0x80000000 741 .long io_tif + 0x80000000 742 .long io_restore + 0x80000000 743 .long io_done + 0x80000000 744 .long psw_idle + 0x80000000 745 .long psw_idle_end + 0x80000000 746 747cleanup_critical: 748 cl %r9,BASED(cleanup_table) # system_call 749 jl 0f 750 cl %r9,BASED(cleanup_table+4) # sysc_do_svc 751 jl cleanup_system_call 752 cl %r9,BASED(cleanup_table+8) # sysc_tif 753 jl 0f 754 cl %r9,BASED(cleanup_table+12) # sysc_restore 755 jl cleanup_sysc_tif 756 cl %r9,BASED(cleanup_table+16) # sysc_done 757 jl cleanup_sysc_restore 758 cl %r9,BASED(cleanup_table+20) # io_tif 759 jl 0f 760 cl %r9,BASED(cleanup_table+24) # io_restore 761 jl cleanup_io_tif 762 cl %r9,BASED(cleanup_table+28) # io_done 763 jl cleanup_io_restore 764 cl %r9,BASED(cleanup_table+32) # psw_idle 765 jl 0f 766 cl %r9,BASED(cleanup_table+36) # psw_idle_end 767 jl cleanup_idle 7680: br %r14 769 770cleanup_system_call: 771 # check if stpt has been executed 772 cl %r9,BASED(cleanup_system_call_insn) 773 jh 0f 774 mvc __LC_SYNC_ENTER_TIMER(8),__LC_ASYNC_ENTER_TIMER 775 chi %r11,__LC_SAVE_AREA_ASYNC 776 je 0f 777 mvc __LC_SYNC_ENTER_TIMER(8),__LC_MCCK_ENTER_TIMER 7780: # check if stm has been executed 779 cl %r9,BASED(cleanup_system_call_insn+4) 780 jh 0f 781 mvc __LC_SAVE_AREA_SYNC(32),0(%r11) 7820: # set up saved registers r12, and r13 783 st %r12,16(%r11) # r12 thread-info pointer 784 st %r13,20(%r11) # r13 literal-pool pointer 785 # check if the user time calculation has been done 786 cl %r9,BASED(cleanup_system_call_insn+8) 787 jh 0f 788 l %r10,__LC_EXIT_TIMER 789 l %r15,__LC_EXIT_TIMER+4 790 SUB64 %r10,%r15,__LC_SYNC_ENTER_TIMER 791 ADD64 %r10,%r15,__LC_USER_TIMER 792 st %r10,__LC_USER_TIMER 793 st %r15,__LC_USER_TIMER+4 7940: # check if the system time calculation has been done 795 cl %r9,BASED(cleanup_system_call_insn+12) 796 jh 0f 797 l %r10,__LC_LAST_UPDATE_TIMER 798 l %r15,__LC_LAST_UPDATE_TIMER+4 799 SUB64 %r10,%r15,__LC_EXIT_TIMER 800 ADD64 %r10,%r15,__LC_SYSTEM_TIMER 801 st %r10,__LC_SYSTEM_TIMER 802 st %r15,__LC_SYSTEM_TIMER+4 8030: # update accounting time stamp 804 mvc __LC_LAST_UPDATE_TIMER(8),__LC_SYNC_ENTER_TIMER 805 # set up saved register 11 806 l %r15,__LC_KERNEL_STACK 807 la %r9,STACK_FRAME_OVERHEAD(%r15) 808 st %r9,12(%r11) # r11 pt_regs pointer 809 # fill pt_regs 810 mvc __PT_R8(32,%r9),__LC_SAVE_AREA_SYNC 811 stm %r0,%r7,__PT_R0(%r9) 812 mvc __PT_PSW(8,%r9),__LC_SVC_OLD_PSW 813 mvc __PT_INT_CODE(4,%r9),__LC_SVC_ILC 814 # setup saved register 15 815 st %r15,28(%r11) # r15 stack pointer 816 # set new psw address and exit 817 l %r9,BASED(cleanup_table+4) # sysc_do_svc + 0x80000000 818 br %r14 819cleanup_system_call_insn: 820 .long system_call + 0x80000000 821 .long sysc_stm + 0x80000000 822 .long sysc_vtime + 0x80000000 + 36 823 .long sysc_vtime + 0x80000000 + 76 824 825cleanup_sysc_tif: 826 l %r9,BASED(cleanup_table+8) # sysc_tif + 0x80000000 827 br %r14 828 829cleanup_sysc_restore: 830 cl %r9,BASED(cleanup_sysc_restore_insn) 831 jhe 0f 832 l %r9,12(%r11) # get saved pointer to pt_regs 833 mvc __LC_RETURN_PSW(8),__PT_PSW(%r9) 834 mvc 0(32,%r11),__PT_R8(%r9) 835 lm %r0,%r7,__PT_R0(%r9) 8360: lm %r8,%r9,__LC_RETURN_PSW 837 br %r14 838cleanup_sysc_restore_insn: 839 .long sysc_done - 4 + 0x80000000 840 841cleanup_io_tif: 842 l %r9,BASED(cleanup_table+20) # io_tif + 0x80000000 843 br %r14 844 845cleanup_io_restore: 846 cl %r9,BASED(cleanup_io_restore_insn) 847 jhe 0f 848 l %r9,12(%r11) # get saved r11 pointer to pt_regs 849 mvc __LC_RETURN_PSW(8),__PT_PSW(%r9) 850 mvc 0(32,%r11),__PT_R8(%r9) 851 lm %r0,%r7,__PT_R0(%r9) 8520: lm %r8,%r9,__LC_RETURN_PSW 853 br %r14 854cleanup_io_restore_insn: 855 .long io_done - 4 + 0x80000000 856 857cleanup_idle: 858 # copy interrupt clock & cpu timer 859 mvc __CLOCK_IDLE_EXIT(8,%r2),__LC_INT_CLOCK 860 mvc __TIMER_IDLE_EXIT(8,%r2),__LC_ASYNC_ENTER_TIMER 861 chi %r11,__LC_SAVE_AREA_ASYNC 862 je 0f 863 mvc __CLOCK_IDLE_EXIT(8,%r2),__LC_MCCK_CLOCK 864 mvc __TIMER_IDLE_EXIT(8,%r2),__LC_MCCK_ENTER_TIMER 8650: # check if stck has been executed 866 cl %r9,BASED(cleanup_idle_insn) 867 jhe 1f 868 mvc __CLOCK_IDLE_ENTER(8,%r2),__CLOCK_IDLE_EXIT(%r2) 869 mvc __TIMER_IDLE_ENTER(8,%r2),__TIMER_IDLE_EXIT(%r3) 8701: # account system time going idle 871 lm %r9,%r10,__LC_STEAL_TIMER 872 ADD64 %r9,%r10,__CLOCK_IDLE_ENTER(%r2) 873 SUB64 %r9,%r10,__LC_LAST_UPDATE_CLOCK 874 stm %r9,%r10,__LC_STEAL_TIMER 875 mvc __LC_LAST_UPDATE_CLOCK(8),__CLOCK_IDLE_EXIT(%r2) 876 lm %r9,%r10,__LC_SYSTEM_TIMER 877 ADD64 %r9,%r10,__LC_LAST_UPDATE_TIMER 878 SUB64 %r9,%r10,__TIMER_IDLE_ENTER(%r2) 879 stm %r9,%r10,__LC_SYSTEM_TIMER 880 mvc __LC_LAST_UPDATE_TIMER(8),__TIMER_IDLE_EXIT(%r2) 881 # prepare return psw 882 n %r8,BASED(cleanup_idle_wait) # clear wait state bit 883 l %r9,24(%r11) # return from psw_idle 884 br %r14 885cleanup_idle_insn: 886 .long psw_idle_lpsw + 0x80000000 887cleanup_idle_wait: 888 .long 0xfffdffff 889 890/* 891 * Integer constants 892 */ 893 .align 4 894.Lnr_syscalls: 895 .long NR_syscalls 896.Lvtimer_max: 897 .quad 0x7fffffffffffffff 898 899/* 900 * Symbol constants 901 */ 902.Ldo_machine_check: .long s390_do_machine_check 903.Lhandle_mcck: .long s390_handle_mcck 904.Ldo_IRQ: .long do_IRQ 905.Ldo_extint: .long do_extint 906.Ldo_signal: .long do_signal 907.Ldo_notify_resume: .long do_notify_resume 908.Ldo_per_trap: .long do_per_trap 909.Ljump_table: .long pgm_check_table 910.Lschedule: .long schedule 911#ifdef CONFIG_PREEMPT 912.Lpreempt_irq: .long preempt_schedule_irq 913#endif 914.Ltrace_enter: .long do_syscall_trace_enter 915.Ltrace_exit: .long do_syscall_trace_exit 916.Lschedule_tail: .long schedule_tail 917.Lsysc_per: .long sysc_per + 0x80000000 918#ifdef CONFIG_TRACE_IRQFLAGS 919.Lhardirqs_on: .long trace_hardirqs_on_caller 920.Lhardirqs_off: .long trace_hardirqs_off_caller 921#endif 922#ifdef CONFIG_LOCKDEP 923.Llockdep_sys_exit: .long lockdep_sys_exit 924#endif 925.Lcritical_start: .long __critical_start + 0x80000000 926.Lcritical_length: .long __critical_end - __critical_start 927 928 .section .rodata, "a" 929#define SYSCALL(esa,esame,emu) .long esa 930 .globl sys_call_table 931sys_call_table: 932#include "syscalls.S" 933#undef SYSCALL 934