1/* 2 * PowerPC version 3 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org) 4 * Rewritten by Cort Dougan (cort@fsmlabs.com) for PReP 5 * Copyright (C) 1996 Cort Dougan <cort@fsmlabs.com> 6 * Adapted for Power Macintosh by Paul Mackerras. 7 * Low-level exception handlers and MMU support 8 * rewritten by Paul Mackerras. 9 * Copyright (C) 1996 Paul Mackerras. 10 * MPC8xx modifications Copyright (C) 1997 Dan Malek (dmalek@jlc.net). 11 * 12 * This file contains the system call entry code, context switch 13 * code, and exception/interrupt return code for PowerPC. 14 * 15 * This program is free software; you can redistribute it and/or 16 * modify it under the terms of the GNU General Public License 17 * as published by the Free Software Foundation; either version 18 * 2 of the License, or (at your option) any later version. 19 * 20 */ 21 22#include <linux/errno.h> 23#include <linux/sys.h> 24#include <linux/threads.h> 25#include <asm/reg.h> 26#include <asm/page.h> 27#include <asm/mmu.h> 28#include <asm/cputable.h> 29#include <asm/thread_info.h> 30#include <asm/ppc_asm.h> 31#include <asm/asm-offsets.h> 32#include <asm/unistd.h> 33#include <asm/ftrace.h> 34#include <asm/ptrace.h> 35 36#undef SHOW_SYSCALLS 37#undef SHOW_SYSCALLS_TASK 38 39/* 40 * MSR_KERNEL is > 0x10000 on 4xx/Book-E since it include MSR_CE. 41 */ 42#if MSR_KERNEL >= 0x10000 43#define LOAD_MSR_KERNEL(r, x) lis r,(x)@h; ori r,r,(x)@l 44#else 45#define LOAD_MSR_KERNEL(r, x) li r,(x) 46#endif 47 48#ifdef CONFIG_BOOKE 49 .globl mcheck_transfer_to_handler 50mcheck_transfer_to_handler: 51 mfspr r0,SPRN_DSRR0 52 stw r0,_DSRR0(r11) 53 mfspr r0,SPRN_DSRR1 54 stw r0,_DSRR1(r11) 55 /* fall through */ 56 57 .globl debug_transfer_to_handler 58debug_transfer_to_handler: 59 mfspr r0,SPRN_CSRR0 60 stw r0,_CSRR0(r11) 61 mfspr r0,SPRN_CSRR1 62 stw r0,_CSRR1(r11) 63 /* fall through */ 64 65 .globl crit_transfer_to_handler 66crit_transfer_to_handler: 67#ifdef CONFIG_PPC_BOOK3E_MMU 68 mfspr r0,SPRN_MAS0 69 stw r0,MAS0(r11) 70 mfspr r0,SPRN_MAS1 71 stw r0,MAS1(r11) 72 mfspr r0,SPRN_MAS2 73 stw r0,MAS2(r11) 74 mfspr r0,SPRN_MAS3 75 stw r0,MAS3(r11) 76 mfspr r0,SPRN_MAS6 77 stw r0,MAS6(r11) 78#ifdef CONFIG_PHYS_64BIT 79 mfspr r0,SPRN_MAS7 80 stw r0,MAS7(r11) 81#endif /* CONFIG_PHYS_64BIT */ 82#endif /* CONFIG_PPC_BOOK3E_MMU */ 83#ifdef CONFIG_44x 84 mfspr r0,SPRN_MMUCR 85 stw r0,MMUCR(r11) 86#endif 87 mfspr r0,SPRN_SRR0 88 stw r0,_SRR0(r11) 89 mfspr r0,SPRN_SRR1 90 stw r0,_SRR1(r11) 91 92 /* set the stack limit to the current stack 93 * and set the limit to protect the thread_info 94 * struct 95 */ 96 mfspr r8,SPRN_SPRG_THREAD 97 lwz r0,KSP_LIMIT(r8) 98 stw r0,SAVED_KSP_LIMIT(r11) 99 rlwimi r0,r1,0,0,(31-THREAD_SHIFT) 100 stw r0,KSP_LIMIT(r8) 101 /* fall through */ 102#endif 103 104#ifdef CONFIG_40x 105 .globl crit_transfer_to_handler 106crit_transfer_to_handler: 107 lwz r0,crit_r10@l(0) 108 stw r0,GPR10(r11) 109 lwz r0,crit_r11@l(0) 110 stw r0,GPR11(r11) 111 mfspr r0,SPRN_SRR0 112 stw r0,crit_srr0@l(0) 113 mfspr r0,SPRN_SRR1 114 stw r0,crit_srr1@l(0) 115 116 /* set the stack limit to the current stack 117 * and set the limit to protect the thread_info 118 * struct 119 */ 120 mfspr r8,SPRN_SPRG_THREAD 121 lwz r0,KSP_LIMIT(r8) 122 stw r0,saved_ksp_limit@l(0) 123 rlwimi r0,r1,0,0,(31-THREAD_SHIFT) 124 stw r0,KSP_LIMIT(r8) 125 /* fall through */ 126#endif 127 128/* 129 * This code finishes saving the registers to the exception frame 130 * and jumps to the appropriate handler for the exception, turning 131 * on address translation. 132 * Note that we rely on the caller having set cr0.eq iff the exception 133 * occurred in kernel mode (i.e. MSR:PR = 0). 134 */ 135 .globl transfer_to_handler_full 136transfer_to_handler_full: 137 SAVE_NVGPRS(r11) 138 /* fall through */ 139 140 .globl transfer_to_handler 141transfer_to_handler: 142 stw r2,GPR2(r11) 143 stw r12,_NIP(r11) 144 stw r9,_MSR(r11) 145 andi. r2,r9,MSR_PR 146 mfctr r12 147 mfspr r2,SPRN_XER 148 stw r12,_CTR(r11) 149 stw r2,_XER(r11) 150 mfspr r12,SPRN_SPRG_THREAD 151 addi r2,r12,-THREAD 152 tovirt(r2,r2) /* set r2 to current */ 153 beq 2f /* if from user, fix up THREAD.regs */ 154 addi r11,r1,STACK_FRAME_OVERHEAD 155 stw r11,PT_REGS(r12) 156#if defined(CONFIG_40x) || defined(CONFIG_BOOKE) 157 /* Check to see if the dbcr0 register is set up to debug. Use the 158 internal debug mode bit to do this. */ 159 lwz r12,THREAD_DBCR0(r12) 160 andis. r12,r12,DBCR0_IDM@h 161 beq+ 3f 162 /* From user and task is ptraced - load up global dbcr0 */ 163 li r12,-1 /* clear all pending debug events */ 164 mtspr SPRN_DBSR,r12 165 lis r11,global_dbcr0@ha 166 tophys(r11,r11) 167 addi r11,r11,global_dbcr0@l 168#ifdef CONFIG_SMP 169 CURRENT_THREAD_INFO(r9, r1) 170 lwz r9,TI_CPU(r9) 171 slwi r9,r9,3 172 add r11,r11,r9 173#endif 174 lwz r12,0(r11) 175 mtspr SPRN_DBCR0,r12 176 lwz r12,4(r11) 177 addi r12,r12,-1 178 stw r12,4(r11) 179#endif 180 b 3f 181 1822: /* if from kernel, check interrupted DOZE/NAP mode and 183 * check for stack overflow 184 */ 185 lwz r9,KSP_LIMIT(r12) 186 cmplw r1,r9 /* if r1 <= ksp_limit */ 187 ble- stack_ovf /* then the kernel stack overflowed */ 1885: 189#if defined(CONFIG_6xx) || defined(CONFIG_E500) 190 CURRENT_THREAD_INFO(r9, r1) 191 tophys(r9,r9) /* check local flags */ 192 lwz r12,TI_LOCAL_FLAGS(r9) 193 mtcrf 0x01,r12 194 bt- 31-TLF_NAPPING,4f 195 bt- 31-TLF_SLEEPING,7f 196#endif /* CONFIG_6xx || CONFIG_E500 */ 197 .globl transfer_to_handler_cont 198transfer_to_handler_cont: 1993: 200 mflr r9 201 lwz r11,0(r9) /* virtual address of handler */ 202 lwz r9,4(r9) /* where to go when done */ 203#ifdef CONFIG_TRACE_IRQFLAGS 204 lis r12,reenable_mmu@h 205 ori r12,r12,reenable_mmu@l 206 mtspr SPRN_SRR0,r12 207 mtspr SPRN_SRR1,r10 208 SYNC 209 RFI 210reenable_mmu: /* re-enable mmu so we can */ 211 mfmsr r10 212 lwz r12,_MSR(r1) 213 xor r10,r10,r12 214 andi. r10,r10,MSR_EE /* Did EE change? */ 215 beq 1f 216 217 /* 218 * The trace_hardirqs_off will use CALLER_ADDR0 and CALLER_ADDR1. 219 * If from user mode there is only one stack frame on the stack, and 220 * accessing CALLER_ADDR1 will cause oops. So we need create a dummy 221 * stack frame to make trace_hardirqs_off happy. 222 * 223 * This is handy because we also need to save a bunch of GPRs, 224 * r3 can be different from GPR3(r1) at this point, r9 and r11 225 * contains the old MSR and handler address respectively, 226 * r4 & r5 can contain page fault arguments that need to be passed 227 * along as well. r12, CCR, CTR, XER etc... are left clobbered as 228 * they aren't useful past this point (aren't syscall arguments), 229 * the rest is restored from the exception frame. 230 */ 231 stwu r1,-32(r1) 232 stw r9,8(r1) 233 stw r11,12(r1) 234 stw r3,16(r1) 235 stw r4,20(r1) 236 stw r5,24(r1) 237 bl trace_hardirqs_off 238 lwz r5,24(r1) 239 lwz r4,20(r1) 240 lwz r3,16(r1) 241 lwz r11,12(r1) 242 lwz r9,8(r1) 243 addi r1,r1,32 244 lwz r0,GPR0(r1) 245 lwz r6,GPR6(r1) 246 lwz r7,GPR7(r1) 247 lwz r8,GPR8(r1) 2481: mtctr r11 249 mtlr r9 250 bctr /* jump to handler */ 251#else /* CONFIG_TRACE_IRQFLAGS */ 252 mtspr SPRN_SRR0,r11 253 mtspr SPRN_SRR1,r10 254 mtlr r9 255 SYNC 256 RFI /* jump to handler, enable MMU */ 257#endif /* CONFIG_TRACE_IRQFLAGS */ 258 259#if defined (CONFIG_6xx) || defined(CONFIG_E500) 2604: rlwinm r12,r12,0,~_TLF_NAPPING 261 stw r12,TI_LOCAL_FLAGS(r9) 262 b power_save_ppc32_restore 263 2647: rlwinm r12,r12,0,~_TLF_SLEEPING 265 stw r12,TI_LOCAL_FLAGS(r9) 266 lwz r9,_MSR(r11) /* if sleeping, clear MSR.EE */ 267 rlwinm r9,r9,0,~MSR_EE 268 lwz r12,_LINK(r11) /* and return to address in LR */ 269 b fast_exception_return 270#endif 271 272/* 273 * On kernel stack overflow, load up an initial stack pointer 274 * and call StackOverflow(regs), which should not return. 275 */ 276stack_ovf: 277 /* sometimes we use a statically-allocated stack, which is OK. */ 278 lis r12,_end@h 279 ori r12,r12,_end@l 280 cmplw r1,r12 281 ble 5b /* r1 <= &_end is OK */ 282 SAVE_NVGPRS(r11) 283 addi r3,r1,STACK_FRAME_OVERHEAD 284 lis r1,init_thread_union@ha 285 addi r1,r1,init_thread_union@l 286 addi r1,r1,THREAD_SIZE-STACK_FRAME_OVERHEAD 287 lis r9,StackOverflow@ha 288 addi r9,r9,StackOverflow@l 289 LOAD_MSR_KERNEL(r10,MSR_KERNEL) 290 FIX_SRR1(r10,r12) 291 mtspr SPRN_SRR0,r9 292 mtspr SPRN_SRR1,r10 293 SYNC 294 RFI 295 296/* 297 * Handle a system call. 298 */ 299 .stabs "arch/powerpc/kernel/",N_SO,0,0,0f 300 .stabs "entry_32.S",N_SO,0,0,0f 3010: 302 303_GLOBAL(DoSyscall) 304 stw r3,ORIG_GPR3(r1) 305 li r12,0 306 stw r12,RESULT(r1) 307 lwz r11,_CCR(r1) /* Clear SO bit in CR */ 308 rlwinm r11,r11,0,4,2 309 stw r11,_CCR(r1) 310#ifdef SHOW_SYSCALLS 311 bl do_show_syscall 312#endif /* SHOW_SYSCALLS */ 313#ifdef CONFIG_TRACE_IRQFLAGS 314 /* Return from syscalls can (and generally will) hard enable 315 * interrupts. You aren't supposed to call a syscall with 316 * interrupts disabled in the first place. However, to ensure 317 * that we get it right vs. lockdep if it happens, we force 318 * that hard enable here with appropriate tracing if we see 319 * that we have been called with interrupts off 320 */ 321 mfmsr r11 322 andi. r12,r11,MSR_EE 323 bne+ 1f 324 /* We came in with interrupts disabled, we enable them now */ 325 bl trace_hardirqs_on 326 mfmsr r11 327 lwz r0,GPR0(r1) 328 lwz r3,GPR3(r1) 329 lwz r4,GPR4(r1) 330 ori r11,r11,MSR_EE 331 lwz r5,GPR5(r1) 332 lwz r6,GPR6(r1) 333 lwz r7,GPR7(r1) 334 lwz r8,GPR8(r1) 335 mtmsr r11 3361: 337#endif /* CONFIG_TRACE_IRQFLAGS */ 338 CURRENT_THREAD_INFO(r10, r1) 339 lwz r11,TI_FLAGS(r10) 340 andi. r11,r11,_TIF_SYSCALL_T_OR_A 341 bne- syscall_dotrace 342syscall_dotrace_cont: 343 cmplwi 0,r0,NR_syscalls 344 lis r10,sys_call_table@h 345 ori r10,r10,sys_call_table@l 346 slwi r0,r0,2 347 bge- 66f 348 lwzx r10,r10,r0 /* Fetch system call handler [ptr] */ 349 mtlr r10 350 addi r9,r1,STACK_FRAME_OVERHEAD 351 PPC440EP_ERR42 352 blrl /* Call handler */ 353 .globl ret_from_syscall 354ret_from_syscall: 355#ifdef SHOW_SYSCALLS 356 bl do_show_syscall_exit 357#endif 358 mr r6,r3 359 CURRENT_THREAD_INFO(r12, r1) 360 /* disable interrupts so current_thread_info()->flags can't change */ 361 LOAD_MSR_KERNEL(r10,MSR_KERNEL) /* doesn't include MSR_EE */ 362 /* Note: We don't bother telling lockdep about it */ 363 SYNC 364 MTMSRD(r10) 365 lwz r9,TI_FLAGS(r12) 366 li r8,-_LAST_ERRNO 367 andi. r0,r9,(_TIF_SYSCALL_T_OR_A|_TIF_SINGLESTEP|_TIF_USER_WORK_MASK|_TIF_PERSYSCALL_MASK) 368 bne- syscall_exit_work 369 cmplw 0,r3,r8 370 blt+ syscall_exit_cont 371 lwz r11,_CCR(r1) /* Load CR */ 372 neg r3,r3 373 oris r11,r11,0x1000 /* Set SO bit in CR */ 374 stw r11,_CCR(r1) 375syscall_exit_cont: 376 lwz r8,_MSR(r1) 377#ifdef CONFIG_TRACE_IRQFLAGS 378 /* If we are going to return from the syscall with interrupts 379 * off, we trace that here. It shouldn't happen though but we 380 * want to catch the bugger if it does right ? 381 */ 382 andi. r10,r8,MSR_EE 383 bne+ 1f 384 stw r3,GPR3(r1) 385 bl trace_hardirqs_off 386 lwz r3,GPR3(r1) 3871: 388#endif /* CONFIG_TRACE_IRQFLAGS */ 389#if defined(CONFIG_4xx) || defined(CONFIG_BOOKE) 390 /* If the process has its own DBCR0 value, load it up. The internal 391 debug mode bit tells us that dbcr0 should be loaded. */ 392 lwz r0,THREAD+THREAD_DBCR0(r2) 393 andis. r10,r0,DBCR0_IDM@h 394 bnel- load_dbcr0 395#endif 396#ifdef CONFIG_44x 397BEGIN_MMU_FTR_SECTION 398 lis r4,icache_44x_need_flush@ha 399 lwz r5,icache_44x_need_flush@l(r4) 400 cmplwi cr0,r5,0 401 bne- 2f 4021: 403END_MMU_FTR_SECTION_IFCLR(MMU_FTR_TYPE_47x) 404#endif /* CONFIG_44x */ 405BEGIN_FTR_SECTION 406 lwarx r7,0,r1 407END_FTR_SECTION_IFSET(CPU_FTR_NEED_PAIRED_STWCX) 408 stwcx. r0,0,r1 /* to clear the reservation */ 409 lwz r4,_LINK(r1) 410 lwz r5,_CCR(r1) 411 mtlr r4 412 mtcr r5 413 lwz r7,_NIP(r1) 414 FIX_SRR1(r8, r0) 415 lwz r2,GPR2(r1) 416 lwz r1,GPR1(r1) 417 mtspr SPRN_SRR0,r7 418 mtspr SPRN_SRR1,r8 419 SYNC 420 RFI 421#ifdef CONFIG_44x 4222: li r7,0 423 iccci r0,r0 424 stw r7,icache_44x_need_flush@l(r4) 425 b 1b 426#endif /* CONFIG_44x */ 427 42866: li r3,-ENOSYS 429 b ret_from_syscall 430 431 .globl ret_from_fork 432ret_from_fork: 433 REST_NVGPRS(r1) 434 bl schedule_tail 435 li r3,0 436 b ret_from_syscall 437 438 .globl ret_from_kernel_thread 439ret_from_kernel_thread: 440 REST_NVGPRS(r1) 441 bl schedule_tail 442 mtlr r14 443 mr r3,r15 444 PPC440EP_ERR42 445 blrl 446 li r3,0 447 b ret_from_syscall 448 449/* Traced system call support */ 450syscall_dotrace: 451 SAVE_NVGPRS(r1) 452 li r0,0xc00 453 stw r0,_TRAP(r1) 454 addi r3,r1,STACK_FRAME_OVERHEAD 455 bl do_syscall_trace_enter 456 /* 457 * Restore argument registers possibly just changed. 458 * We use the return value of do_syscall_trace_enter 459 * for call number to look up in the table (r0). 460 */ 461 mr r0,r3 462 lwz r3,GPR3(r1) 463 lwz r4,GPR4(r1) 464 lwz r5,GPR5(r1) 465 lwz r6,GPR6(r1) 466 lwz r7,GPR7(r1) 467 lwz r8,GPR8(r1) 468 REST_NVGPRS(r1) 469 b syscall_dotrace_cont 470 471syscall_exit_work: 472 andi. r0,r9,_TIF_RESTOREALL 473 beq+ 0f 474 REST_NVGPRS(r1) 475 b 2f 4760: cmplw 0,r3,r8 477 blt+ 1f 478 andi. r0,r9,_TIF_NOERROR 479 bne- 1f 480 lwz r11,_CCR(r1) /* Load CR */ 481 neg r3,r3 482 oris r11,r11,0x1000 /* Set SO bit in CR */ 483 stw r11,_CCR(r1) 484 4851: stw r6,RESULT(r1) /* Save result */ 486 stw r3,GPR3(r1) /* Update return value */ 4872: andi. r0,r9,(_TIF_PERSYSCALL_MASK) 488 beq 4f 489 490 /* Clear per-syscall TIF flags if any are set. */ 491 492 li r11,_TIF_PERSYSCALL_MASK 493 addi r12,r12,TI_FLAGS 4943: lwarx r8,0,r12 495 andc r8,r8,r11 496#ifdef CONFIG_IBM405_ERR77 497 dcbt 0,r12 498#endif 499 stwcx. r8,0,r12 500 bne- 3b 501 subi r12,r12,TI_FLAGS 502 5034: /* Anything which requires enabling interrupts? */ 504 andi. r0,r9,(_TIF_SYSCALL_T_OR_A|_TIF_SINGLESTEP) 505 beq ret_from_except 506 507 /* Re-enable interrupts. There is no need to trace that with 508 * lockdep as we are supposed to have IRQs on at this point 509 */ 510 ori r10,r10,MSR_EE 511 SYNC 512 MTMSRD(r10) 513 514 /* Save NVGPRS if they're not saved already */ 515 lwz r4,_TRAP(r1) 516 andi. r4,r4,1 517 beq 5f 518 SAVE_NVGPRS(r1) 519 li r4,0xc00 520 stw r4,_TRAP(r1) 5215: 522 addi r3,r1,STACK_FRAME_OVERHEAD 523 bl do_syscall_trace_leave 524 b ret_from_except_full 525 526#ifdef SHOW_SYSCALLS 527do_show_syscall: 528#ifdef SHOW_SYSCALLS_TASK 529 lis r11,show_syscalls_task@ha 530 lwz r11,show_syscalls_task@l(r11) 531 cmp 0,r2,r11 532 bnelr 533#endif 534 stw r31,GPR31(r1) 535 mflr r31 536 lis r3,7f@ha 537 addi r3,r3,7f@l 538 lwz r4,GPR0(r1) 539 lwz r5,GPR3(r1) 540 lwz r6,GPR4(r1) 541 lwz r7,GPR5(r1) 542 lwz r8,GPR6(r1) 543 lwz r9,GPR7(r1) 544 bl printk 545 lis r3,77f@ha 546 addi r3,r3,77f@l 547 lwz r4,GPR8(r1) 548 mr r5,r2 549 bl printk 550 lwz r0,GPR0(r1) 551 lwz r3,GPR3(r1) 552 lwz r4,GPR4(r1) 553 lwz r5,GPR5(r1) 554 lwz r6,GPR6(r1) 555 lwz r7,GPR7(r1) 556 lwz r8,GPR8(r1) 557 mtlr r31 558 lwz r31,GPR31(r1) 559 blr 560 561do_show_syscall_exit: 562#ifdef SHOW_SYSCALLS_TASK 563 lis r11,show_syscalls_task@ha 564 lwz r11,show_syscalls_task@l(r11) 565 cmp 0,r2,r11 566 bnelr 567#endif 568 stw r31,GPR31(r1) 569 mflr r31 570 stw r3,RESULT(r1) /* Save result */ 571 mr r4,r3 572 lis r3,79f@ha 573 addi r3,r3,79f@l 574 bl printk 575 lwz r3,RESULT(r1) 576 mtlr r31 577 lwz r31,GPR31(r1) 578 blr 579 5807: .string "syscall %d(%x, %x, %x, %x, %x, " 58177: .string "%x), current=%p\n" 58279: .string " -> %x\n" 583 .align 2,0 584 585#ifdef SHOW_SYSCALLS_TASK 586 .data 587 .globl show_syscalls_task 588show_syscalls_task: 589 .long -1 590 .text 591#endif 592#endif /* SHOW_SYSCALLS */ 593 594/* 595 * The fork/clone functions need to copy the full register set into 596 * the child process. Therefore we need to save all the nonvolatile 597 * registers (r13 - r31) before calling the C code. 598 */ 599 .globl ppc_fork 600ppc_fork: 601 SAVE_NVGPRS(r1) 602 lwz r0,_TRAP(r1) 603 rlwinm r0,r0,0,0,30 /* clear LSB to indicate full */ 604 stw r0,_TRAP(r1) /* register set saved */ 605 b sys_fork 606 607 .globl ppc_vfork 608ppc_vfork: 609 SAVE_NVGPRS(r1) 610 lwz r0,_TRAP(r1) 611 rlwinm r0,r0,0,0,30 /* clear LSB to indicate full */ 612 stw r0,_TRAP(r1) /* register set saved */ 613 b sys_vfork 614 615 .globl ppc_clone 616ppc_clone: 617 SAVE_NVGPRS(r1) 618 lwz r0,_TRAP(r1) 619 rlwinm r0,r0,0,0,30 /* clear LSB to indicate full */ 620 stw r0,_TRAP(r1) /* register set saved */ 621 b sys_clone 622 623 .globl ppc_swapcontext 624ppc_swapcontext: 625 SAVE_NVGPRS(r1) 626 lwz r0,_TRAP(r1) 627 rlwinm r0,r0,0,0,30 /* clear LSB to indicate full */ 628 stw r0,_TRAP(r1) /* register set saved */ 629 b sys_swapcontext 630 631/* 632 * Top-level page fault handling. 633 * This is in assembler because if do_page_fault tells us that 634 * it is a bad kernel page fault, we want to save the non-volatile 635 * registers before calling bad_page_fault. 636 */ 637 .globl handle_page_fault 638handle_page_fault: 639 stw r4,_DAR(r1) 640 addi r3,r1,STACK_FRAME_OVERHEAD 641 bl do_page_fault 642 cmpwi r3,0 643 beq+ ret_from_except 644 SAVE_NVGPRS(r1) 645 lwz r0,_TRAP(r1) 646 clrrwi r0,r0,1 647 stw r0,_TRAP(r1) 648 mr r5,r3 649 addi r3,r1,STACK_FRAME_OVERHEAD 650 lwz r4,_DAR(r1) 651 bl bad_page_fault 652 b ret_from_except_full 653 654/* 655 * This routine switches between two different tasks. The process 656 * state of one is saved on its kernel stack. Then the state 657 * of the other is restored from its kernel stack. The memory 658 * management hardware is updated to the second process's state. 659 * Finally, we can return to the second process. 660 * On entry, r3 points to the THREAD for the current task, r4 661 * points to the THREAD for the new task. 662 * 663 * This routine is always called with interrupts disabled. 664 * 665 * Note: there are two ways to get to the "going out" portion 666 * of this code; either by coming in via the entry (_switch) 667 * or via "fork" which must set up an environment equivalent 668 * to the "_switch" path. If you change this , you'll have to 669 * change the fork code also. 670 * 671 * The code which creates the new task context is in 'copy_thread' 672 * in arch/ppc/kernel/process.c 673 */ 674_GLOBAL(_switch) 675 stwu r1,-INT_FRAME_SIZE(r1) 676 mflr r0 677 stw r0,INT_FRAME_SIZE+4(r1) 678 /* r3-r12 are caller saved -- Cort */ 679 SAVE_NVGPRS(r1) 680 stw r0,_NIP(r1) /* Return to switch caller */ 681 mfmsr r11 682 li r0,MSR_FP /* Disable floating-point */ 683#ifdef CONFIG_ALTIVEC 684BEGIN_FTR_SECTION 685 oris r0,r0,MSR_VEC@h /* Disable altivec */ 686 mfspr r12,SPRN_VRSAVE /* save vrsave register value */ 687 stw r12,THREAD+THREAD_VRSAVE(r2) 688END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 689#endif /* CONFIG_ALTIVEC */ 690#ifdef CONFIG_SPE 691BEGIN_FTR_SECTION 692 oris r0,r0,MSR_SPE@h /* Disable SPE */ 693 mfspr r12,SPRN_SPEFSCR /* save spefscr register value */ 694 stw r12,THREAD+THREAD_SPEFSCR(r2) 695END_FTR_SECTION_IFSET(CPU_FTR_SPE) 696#endif /* CONFIG_SPE */ 697 and. r0,r0,r11 /* FP or altivec or SPE enabled? */ 698 beq+ 1f 699 andc r11,r11,r0 700 MTMSRD(r11) 701 isync 7021: stw r11,_MSR(r1) 703 mfcr r10 704 stw r10,_CCR(r1) 705 stw r1,KSP(r3) /* Set old stack pointer */ 706 707#ifdef CONFIG_SMP 708 /* We need a sync somewhere here to make sure that if the 709 * previous task gets rescheduled on another CPU, it sees all 710 * stores it has performed on this one. 711 */ 712 sync 713#endif /* CONFIG_SMP */ 714 715 tophys(r0,r4) 716 CLR_TOP32(r0) 717 mtspr SPRN_SPRG_THREAD,r0 /* Update current THREAD phys addr */ 718 lwz r1,KSP(r4) /* Load new stack pointer */ 719 720 /* save the old current 'last' for return value */ 721 mr r3,r2 722 addi r2,r4,-THREAD /* Update current */ 723 724#ifdef CONFIG_ALTIVEC 725BEGIN_FTR_SECTION 726 lwz r0,THREAD+THREAD_VRSAVE(r2) 727 mtspr SPRN_VRSAVE,r0 /* if G4, restore VRSAVE reg */ 728END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 729#endif /* CONFIG_ALTIVEC */ 730#ifdef CONFIG_SPE 731BEGIN_FTR_SECTION 732 lwz r0,THREAD+THREAD_SPEFSCR(r2) 733 mtspr SPRN_SPEFSCR,r0 /* restore SPEFSCR reg */ 734END_FTR_SECTION_IFSET(CPU_FTR_SPE) 735#endif /* CONFIG_SPE */ 736 737 lwz r0,_CCR(r1) 738 mtcrf 0xFF,r0 739 /* r3-r12 are destroyed -- Cort */ 740 REST_NVGPRS(r1) 741 742 lwz r4,_NIP(r1) /* Return to _switch caller in new task */ 743 mtlr r4 744 addi r1,r1,INT_FRAME_SIZE 745 blr 746 747 .globl fast_exception_return 748fast_exception_return: 749#if !(defined(CONFIG_4xx) || defined(CONFIG_BOOKE)) 750 andi. r10,r9,MSR_RI /* check for recoverable interrupt */ 751 beq 1f /* if not, we've got problems */ 752#endif 753 7542: REST_4GPRS(3, r11) 755 lwz r10,_CCR(r11) 756 REST_GPR(1, r11) 757 mtcr r10 758 lwz r10,_LINK(r11) 759 mtlr r10 760 REST_GPR(10, r11) 761 mtspr SPRN_SRR1,r9 762 mtspr SPRN_SRR0,r12 763 REST_GPR(9, r11) 764 REST_GPR(12, r11) 765 lwz r11,GPR11(r11) 766 SYNC 767 RFI 768 769#if !(defined(CONFIG_4xx) || defined(CONFIG_BOOKE)) 770/* check if the exception happened in a restartable section */ 7711: lis r3,exc_exit_restart_end@ha 772 addi r3,r3,exc_exit_restart_end@l 773 cmplw r12,r3 774 bge 3f 775 lis r4,exc_exit_restart@ha 776 addi r4,r4,exc_exit_restart@l 777 cmplw r12,r4 778 blt 3f 779 lis r3,fee_restarts@ha 780 tophys(r3,r3) 781 lwz r5,fee_restarts@l(r3) 782 addi r5,r5,1 783 stw r5,fee_restarts@l(r3) 784 mr r12,r4 /* restart at exc_exit_restart */ 785 b 2b 786 787 .section .bss 788 .align 2 789fee_restarts: 790 .space 4 791 .previous 792 793/* aargh, a nonrecoverable interrupt, panic */ 794/* aargh, we don't know which trap this is */ 795/* but the 601 doesn't implement the RI bit, so assume it's OK */ 7963: 797BEGIN_FTR_SECTION 798 b 2b 799END_FTR_SECTION_IFSET(CPU_FTR_601) 800 li r10,-1 801 stw r10,_TRAP(r11) 802 addi r3,r1,STACK_FRAME_OVERHEAD 803 lis r10,MSR_KERNEL@h 804 ori r10,r10,MSR_KERNEL@l 805 bl transfer_to_handler_full 806 .long nonrecoverable_exception 807 .long ret_from_except 808#endif 809 810 .globl ret_from_except_full 811ret_from_except_full: 812 REST_NVGPRS(r1) 813 /* fall through */ 814 815 .globl ret_from_except 816ret_from_except: 817 /* Hard-disable interrupts so that current_thread_info()->flags 818 * can't change between when we test it and when we return 819 * from the interrupt. */ 820 /* Note: We don't bother telling lockdep about it */ 821 LOAD_MSR_KERNEL(r10,MSR_KERNEL) 822 SYNC /* Some chip revs have problems here... */ 823 MTMSRD(r10) /* disable interrupts */ 824 825 lwz r3,_MSR(r1) /* Returning to user mode? */ 826 andi. r0,r3,MSR_PR 827 beq resume_kernel 828 829user_exc_return: /* r10 contains MSR_KERNEL here */ 830 /* Check current_thread_info()->flags */ 831 CURRENT_THREAD_INFO(r9, r1) 832 lwz r9,TI_FLAGS(r9) 833 andi. r0,r9,_TIF_USER_WORK_MASK 834 bne do_work 835 836restore_user: 837#if defined(CONFIG_4xx) || defined(CONFIG_BOOKE) 838 /* Check whether this process has its own DBCR0 value. The internal 839 debug mode bit tells us that dbcr0 should be loaded. */ 840 lwz r0,THREAD+THREAD_DBCR0(r2) 841 andis. r10,r0,DBCR0_IDM@h 842 bnel- load_dbcr0 843#endif 844 845 b restore 846 847/* N.B. the only way to get here is from the beq following ret_from_except. */ 848resume_kernel: 849 /* check current_thread_info, _TIF_EMULATE_STACK_STORE */ 850 CURRENT_THREAD_INFO(r9, r1) 851 lwz r8,TI_FLAGS(r9) 852 andis. r0,r8,_TIF_EMULATE_STACK_STORE@h 853 beq+ 1f 854 855 addi r8,r1,INT_FRAME_SIZE /* Get the kprobed function entry */ 856 857 lwz r3,GPR1(r1) 858 subi r3,r3,INT_FRAME_SIZE /* dst: Allocate a trampoline exception frame */ 859 mr r4,r1 /* src: current exception frame */ 860 mr r1,r3 /* Reroute the trampoline frame to r1 */ 861 862 /* Copy from the original to the trampoline. */ 863 li r5,INT_FRAME_SIZE/4 /* size: INT_FRAME_SIZE */ 864 li r6,0 /* start offset: 0 */ 865 mtctr r5 8662: lwzx r0,r6,r4 867 stwx r0,r6,r3 868 addi r6,r6,4 869 bdnz 2b 870 871 /* Do real store operation to complete stwu */ 872 lwz r5,GPR1(r1) 873 stw r8,0(r5) 874 875 /* Clear _TIF_EMULATE_STACK_STORE flag */ 876 lis r11,_TIF_EMULATE_STACK_STORE@h 877 addi r5,r9,TI_FLAGS 8780: lwarx r8,0,r5 879 andc r8,r8,r11 880#ifdef CONFIG_IBM405_ERR77 881 dcbt 0,r5 882#endif 883 stwcx. r8,0,r5 884 bne- 0b 8851: 886 887#ifdef CONFIG_PREEMPT 888 /* check current_thread_info->preempt_count */ 889 lwz r0,TI_PREEMPT(r9) 890 cmpwi 0,r0,0 /* if non-zero, just restore regs and return */ 891 bne restore 892 andi. r8,r8,_TIF_NEED_RESCHED 893 beq+ restore 894 lwz r3,_MSR(r1) 895 andi. r0,r3,MSR_EE /* interrupts off? */ 896 beq restore /* don't schedule if so */ 897#ifdef CONFIG_TRACE_IRQFLAGS 898 /* Lockdep thinks irqs are enabled, we need to call 899 * preempt_schedule_irq with IRQs off, so we inform lockdep 900 * now that we -did- turn them off already 901 */ 902 bl trace_hardirqs_off 903#endif 9041: bl preempt_schedule_irq 905 CURRENT_THREAD_INFO(r9, r1) 906 lwz r3,TI_FLAGS(r9) 907 andi. r0,r3,_TIF_NEED_RESCHED 908 bne- 1b 909#ifdef CONFIG_TRACE_IRQFLAGS 910 /* And now, to properly rebalance the above, we tell lockdep they 911 * are being turned back on, which will happen when we return 912 */ 913 bl trace_hardirqs_on 914#endif 915#endif /* CONFIG_PREEMPT */ 916 917 /* interrupts are hard-disabled at this point */ 918restore: 919#ifdef CONFIG_44x 920BEGIN_MMU_FTR_SECTION 921 b 1f 922END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_47x) 923 lis r4,icache_44x_need_flush@ha 924 lwz r5,icache_44x_need_flush@l(r4) 925 cmplwi cr0,r5,0 926 beq+ 1f 927 li r6,0 928 iccci r0,r0 929 stw r6,icache_44x_need_flush@l(r4) 9301: 931#endif /* CONFIG_44x */ 932 933 lwz r9,_MSR(r1) 934#ifdef CONFIG_TRACE_IRQFLAGS 935 /* Lockdep doesn't know about the fact that IRQs are temporarily turned 936 * off in this assembly code while peeking at TI_FLAGS() and such. However 937 * we need to inform it if the exception turned interrupts off, and we 938 * are about to trun them back on. 939 * 940 * The problem here sadly is that we don't know whether the exceptions was 941 * one that turned interrupts off or not. So we always tell lockdep about 942 * turning them on here when we go back to wherever we came from with EE 943 * on, even if that may meen some redudant calls being tracked. Maybe later 944 * we could encode what the exception did somewhere or test the exception 945 * type in the pt_regs but that sounds overkill 946 */ 947 andi. r10,r9,MSR_EE 948 beq 1f 949 /* 950 * Since the ftrace irqsoff latency trace checks CALLER_ADDR1, 951 * which is the stack frame here, we need to force a stack frame 952 * in case we came from user space. 953 */ 954 stwu r1,-32(r1) 955 mflr r0 956 stw r0,4(r1) 957 stwu r1,-32(r1) 958 bl trace_hardirqs_on 959 lwz r1,0(r1) 960 lwz r1,0(r1) 961 lwz r9,_MSR(r1) 9621: 963#endif /* CONFIG_TRACE_IRQFLAGS */ 964 965 lwz r0,GPR0(r1) 966 lwz r2,GPR2(r1) 967 REST_4GPRS(3, r1) 968 REST_2GPRS(7, r1) 969 970 lwz r10,_XER(r1) 971 lwz r11,_CTR(r1) 972 mtspr SPRN_XER,r10 973 mtctr r11 974 975 PPC405_ERR77(0,r1) 976BEGIN_FTR_SECTION 977 lwarx r11,0,r1 978END_FTR_SECTION_IFSET(CPU_FTR_NEED_PAIRED_STWCX) 979 stwcx. r0,0,r1 /* to clear the reservation */ 980 981#if !(defined(CONFIG_4xx) || defined(CONFIG_BOOKE)) 982 andi. r10,r9,MSR_RI /* check if this exception occurred */ 983 beql nonrecoverable /* at a bad place (MSR:RI = 0) */ 984 985 lwz r10,_CCR(r1) 986 lwz r11,_LINK(r1) 987 mtcrf 0xFF,r10 988 mtlr r11 989 990 /* 991 * Once we put values in SRR0 and SRR1, we are in a state 992 * where exceptions are not recoverable, since taking an 993 * exception will trash SRR0 and SRR1. Therefore we clear the 994 * MSR:RI bit to indicate this. If we do take an exception, 995 * we can't return to the point of the exception but we 996 * can restart the exception exit path at the label 997 * exc_exit_restart below. -- paulus 998 */ 999 LOAD_MSR_KERNEL(r10,MSR_KERNEL & ~MSR_RI) 1000 SYNC 1001 MTMSRD(r10) /* clear the RI bit */ 1002 .globl exc_exit_restart 1003exc_exit_restart: 1004 lwz r12,_NIP(r1) 1005 FIX_SRR1(r9,r10) 1006 mtspr SPRN_SRR0,r12 1007 mtspr SPRN_SRR1,r9 1008 REST_4GPRS(9, r1) 1009 lwz r1,GPR1(r1) 1010 .globl exc_exit_restart_end 1011exc_exit_restart_end: 1012 SYNC 1013 RFI 1014 1015#else /* !(CONFIG_4xx || CONFIG_BOOKE) */ 1016 /* 1017 * This is a bit different on 4xx/Book-E because it doesn't have 1018 * the RI bit in the MSR. 1019 * The TLB miss handler checks if we have interrupted 1020 * the exception exit path and restarts it if so 1021 * (well maybe one day it will... :). 1022 */ 1023 lwz r11,_LINK(r1) 1024 mtlr r11 1025 lwz r10,_CCR(r1) 1026 mtcrf 0xff,r10 1027 REST_2GPRS(9, r1) 1028 .globl exc_exit_restart 1029exc_exit_restart: 1030 lwz r11,_NIP(r1) 1031 lwz r12,_MSR(r1) 1032exc_exit_start: 1033 mtspr SPRN_SRR0,r11 1034 mtspr SPRN_SRR1,r12 1035 REST_2GPRS(11, r1) 1036 lwz r1,GPR1(r1) 1037 .globl exc_exit_restart_end 1038exc_exit_restart_end: 1039 PPC405_ERR77_SYNC 1040 rfi 1041 b . /* prevent prefetch past rfi */ 1042 1043/* 1044 * Returning from a critical interrupt in user mode doesn't need 1045 * to be any different from a normal exception. For a critical 1046 * interrupt in the kernel, we just return (without checking for 1047 * preemption) since the interrupt may have happened at some crucial 1048 * place (e.g. inside the TLB miss handler), and because we will be 1049 * running with r1 pointing into critical_stack, not the current 1050 * process's kernel stack (and therefore current_thread_info() will 1051 * give the wrong answer). 1052 * We have to restore various SPRs that may have been in use at the 1053 * time of the critical interrupt. 1054 * 1055 */ 1056#ifdef CONFIG_40x 1057#define PPC_40x_TURN_OFF_MSR_DR \ 1058 /* avoid any possible TLB misses here by turning off MSR.DR, we \ 1059 * assume the instructions here are mapped by a pinned TLB entry */ \ 1060 li r10,MSR_IR; \ 1061 mtmsr r10; \ 1062 isync; \ 1063 tophys(r1, r1); 1064#else 1065#define PPC_40x_TURN_OFF_MSR_DR 1066#endif 1067 1068#define RET_FROM_EXC_LEVEL(exc_lvl_srr0, exc_lvl_srr1, exc_lvl_rfi) \ 1069 REST_NVGPRS(r1); \ 1070 lwz r3,_MSR(r1); \ 1071 andi. r3,r3,MSR_PR; \ 1072 LOAD_MSR_KERNEL(r10,MSR_KERNEL); \ 1073 bne user_exc_return; \ 1074 lwz r0,GPR0(r1); \ 1075 lwz r2,GPR2(r1); \ 1076 REST_4GPRS(3, r1); \ 1077 REST_2GPRS(7, r1); \ 1078 lwz r10,_XER(r1); \ 1079 lwz r11,_CTR(r1); \ 1080 mtspr SPRN_XER,r10; \ 1081 mtctr r11; \ 1082 PPC405_ERR77(0,r1); \ 1083 stwcx. r0,0,r1; /* to clear the reservation */ \ 1084 lwz r11,_LINK(r1); \ 1085 mtlr r11; \ 1086 lwz r10,_CCR(r1); \ 1087 mtcrf 0xff,r10; \ 1088 PPC_40x_TURN_OFF_MSR_DR; \ 1089 lwz r9,_DEAR(r1); \ 1090 lwz r10,_ESR(r1); \ 1091 mtspr SPRN_DEAR,r9; \ 1092 mtspr SPRN_ESR,r10; \ 1093 lwz r11,_NIP(r1); \ 1094 lwz r12,_MSR(r1); \ 1095 mtspr exc_lvl_srr0,r11; \ 1096 mtspr exc_lvl_srr1,r12; \ 1097 lwz r9,GPR9(r1); \ 1098 lwz r12,GPR12(r1); \ 1099 lwz r10,GPR10(r1); \ 1100 lwz r11,GPR11(r1); \ 1101 lwz r1,GPR1(r1); \ 1102 PPC405_ERR77_SYNC; \ 1103 exc_lvl_rfi; \ 1104 b .; /* prevent prefetch past exc_lvl_rfi */ 1105 1106#define RESTORE_xSRR(exc_lvl_srr0, exc_lvl_srr1) \ 1107 lwz r9,_##exc_lvl_srr0(r1); \ 1108 lwz r10,_##exc_lvl_srr1(r1); \ 1109 mtspr SPRN_##exc_lvl_srr0,r9; \ 1110 mtspr SPRN_##exc_lvl_srr1,r10; 1111 1112#if defined(CONFIG_PPC_BOOK3E_MMU) 1113#ifdef CONFIG_PHYS_64BIT 1114#define RESTORE_MAS7 \ 1115 lwz r11,MAS7(r1); \ 1116 mtspr SPRN_MAS7,r11; 1117#else 1118#define RESTORE_MAS7 1119#endif /* CONFIG_PHYS_64BIT */ 1120#define RESTORE_MMU_REGS \ 1121 lwz r9,MAS0(r1); \ 1122 lwz r10,MAS1(r1); \ 1123 lwz r11,MAS2(r1); \ 1124 mtspr SPRN_MAS0,r9; \ 1125 lwz r9,MAS3(r1); \ 1126 mtspr SPRN_MAS1,r10; \ 1127 lwz r10,MAS6(r1); \ 1128 mtspr SPRN_MAS2,r11; \ 1129 mtspr SPRN_MAS3,r9; \ 1130 mtspr SPRN_MAS6,r10; \ 1131 RESTORE_MAS7; 1132#elif defined(CONFIG_44x) 1133#define RESTORE_MMU_REGS \ 1134 lwz r9,MMUCR(r1); \ 1135 mtspr SPRN_MMUCR,r9; 1136#else 1137#define RESTORE_MMU_REGS 1138#endif 1139 1140#ifdef CONFIG_40x 1141 .globl ret_from_crit_exc 1142ret_from_crit_exc: 1143 mfspr r9,SPRN_SPRG_THREAD 1144 lis r10,saved_ksp_limit@ha; 1145 lwz r10,saved_ksp_limit@l(r10); 1146 tovirt(r9,r9); 1147 stw r10,KSP_LIMIT(r9) 1148 lis r9,crit_srr0@ha; 1149 lwz r9,crit_srr0@l(r9); 1150 lis r10,crit_srr1@ha; 1151 lwz r10,crit_srr1@l(r10); 1152 mtspr SPRN_SRR0,r9; 1153 mtspr SPRN_SRR1,r10; 1154 RET_FROM_EXC_LEVEL(SPRN_CSRR0, SPRN_CSRR1, PPC_RFCI) 1155#endif /* CONFIG_40x */ 1156 1157#ifdef CONFIG_BOOKE 1158 .globl ret_from_crit_exc 1159ret_from_crit_exc: 1160 mfspr r9,SPRN_SPRG_THREAD 1161 lwz r10,SAVED_KSP_LIMIT(r1) 1162 stw r10,KSP_LIMIT(r9) 1163 RESTORE_xSRR(SRR0,SRR1); 1164 RESTORE_MMU_REGS; 1165 RET_FROM_EXC_LEVEL(SPRN_CSRR0, SPRN_CSRR1, PPC_RFCI) 1166 1167 .globl ret_from_debug_exc 1168ret_from_debug_exc: 1169 mfspr r9,SPRN_SPRG_THREAD 1170 lwz r10,SAVED_KSP_LIMIT(r1) 1171 stw r10,KSP_LIMIT(r9) 1172 lwz r9,THREAD_INFO-THREAD(r9) 1173 CURRENT_THREAD_INFO(r10, r1) 1174 lwz r10,TI_PREEMPT(r10) 1175 stw r10,TI_PREEMPT(r9) 1176 RESTORE_xSRR(SRR0,SRR1); 1177 RESTORE_xSRR(CSRR0,CSRR1); 1178 RESTORE_MMU_REGS; 1179 RET_FROM_EXC_LEVEL(SPRN_DSRR0, SPRN_DSRR1, PPC_RFDI) 1180 1181 .globl ret_from_mcheck_exc 1182ret_from_mcheck_exc: 1183 mfspr r9,SPRN_SPRG_THREAD 1184 lwz r10,SAVED_KSP_LIMIT(r1) 1185 stw r10,KSP_LIMIT(r9) 1186 RESTORE_xSRR(SRR0,SRR1); 1187 RESTORE_xSRR(CSRR0,CSRR1); 1188 RESTORE_xSRR(DSRR0,DSRR1); 1189 RESTORE_MMU_REGS; 1190 RET_FROM_EXC_LEVEL(SPRN_MCSRR0, SPRN_MCSRR1, PPC_RFMCI) 1191#endif /* CONFIG_BOOKE */ 1192 1193/* 1194 * Load the DBCR0 value for a task that is being ptraced, 1195 * having first saved away the global DBCR0. Note that r0 1196 * has the dbcr0 value to set upon entry to this. 1197 */ 1198load_dbcr0: 1199 mfmsr r10 /* first disable debug exceptions */ 1200 rlwinm r10,r10,0,~MSR_DE 1201 mtmsr r10 1202 isync 1203 mfspr r10,SPRN_DBCR0 1204 lis r11,global_dbcr0@ha 1205 addi r11,r11,global_dbcr0@l 1206#ifdef CONFIG_SMP 1207 CURRENT_THREAD_INFO(r9, r1) 1208 lwz r9,TI_CPU(r9) 1209 slwi r9,r9,3 1210 add r11,r11,r9 1211#endif 1212 stw r10,0(r11) 1213 mtspr SPRN_DBCR0,r0 1214 lwz r10,4(r11) 1215 addi r10,r10,1 1216 stw r10,4(r11) 1217 li r11,-1 1218 mtspr SPRN_DBSR,r11 /* clear all pending debug events */ 1219 blr 1220 1221 .section .bss 1222 .align 4 1223global_dbcr0: 1224 .space 8*NR_CPUS 1225 .previous 1226#endif /* !(CONFIG_4xx || CONFIG_BOOKE) */ 1227 1228do_work: /* r10 contains MSR_KERNEL here */ 1229 andi. r0,r9,_TIF_NEED_RESCHED 1230 beq do_user_signal 1231 1232do_resched: /* r10 contains MSR_KERNEL here */ 1233 /* Note: We don't need to inform lockdep that we are enabling 1234 * interrupts here. As far as it knows, they are already enabled 1235 */ 1236 ori r10,r10,MSR_EE 1237 SYNC 1238 MTMSRD(r10) /* hard-enable interrupts */ 1239 bl schedule 1240recheck: 1241 /* Note: And we don't tell it we are disabling them again 1242 * neither. Those disable/enable cycles used to peek at 1243 * TI_FLAGS aren't advertised. 1244 */ 1245 LOAD_MSR_KERNEL(r10,MSR_KERNEL) 1246 SYNC 1247 MTMSRD(r10) /* disable interrupts */ 1248 CURRENT_THREAD_INFO(r9, r1) 1249 lwz r9,TI_FLAGS(r9) 1250 andi. r0,r9,_TIF_NEED_RESCHED 1251 bne- do_resched 1252 andi. r0,r9,_TIF_USER_WORK_MASK 1253 beq restore_user 1254do_user_signal: /* r10 contains MSR_KERNEL here */ 1255 ori r10,r10,MSR_EE 1256 SYNC 1257 MTMSRD(r10) /* hard-enable interrupts */ 1258 /* save r13-r31 in the exception frame, if not already done */ 1259 lwz r3,_TRAP(r1) 1260 andi. r0,r3,1 1261 beq 2f 1262 SAVE_NVGPRS(r1) 1263 rlwinm r3,r3,0,0,30 1264 stw r3,_TRAP(r1) 12652: addi r3,r1,STACK_FRAME_OVERHEAD 1266 mr r4,r9 1267 bl do_notify_resume 1268 REST_NVGPRS(r1) 1269 b recheck 1270 1271/* 1272 * We come here when we are at the end of handling an exception 1273 * that occurred at a place where taking an exception will lose 1274 * state information, such as the contents of SRR0 and SRR1. 1275 */ 1276nonrecoverable: 1277 lis r10,exc_exit_restart_end@ha 1278 addi r10,r10,exc_exit_restart_end@l 1279 cmplw r12,r10 1280 bge 3f 1281 lis r11,exc_exit_restart@ha 1282 addi r11,r11,exc_exit_restart@l 1283 cmplw r12,r11 1284 blt 3f 1285 lis r10,ee_restarts@ha 1286 lwz r12,ee_restarts@l(r10) 1287 addi r12,r12,1 1288 stw r12,ee_restarts@l(r10) 1289 mr r12,r11 /* restart at exc_exit_restart */ 1290 blr 12913: /* OK, we can't recover, kill this process */ 1292 /* but the 601 doesn't implement the RI bit, so assume it's OK */ 1293BEGIN_FTR_SECTION 1294 blr 1295END_FTR_SECTION_IFSET(CPU_FTR_601) 1296 lwz r3,_TRAP(r1) 1297 andi. r0,r3,1 1298 beq 4f 1299 SAVE_NVGPRS(r1) 1300 rlwinm r3,r3,0,0,30 1301 stw r3,_TRAP(r1) 13024: addi r3,r1,STACK_FRAME_OVERHEAD 1303 bl nonrecoverable_exception 1304 /* shouldn't return */ 1305 b 4b 1306 1307 .section .bss 1308 .align 2 1309ee_restarts: 1310 .space 4 1311 .previous 1312 1313/* 1314 * PROM code for specific machines follows. Put it 1315 * here so it's easy to add arch-specific sections later. 1316 * -- Cort 1317 */ 1318#ifdef CONFIG_PPC_RTAS 1319/* 1320 * On CHRP, the Run-Time Abstraction Services (RTAS) have to be 1321 * called with the MMU off. 1322 */ 1323_GLOBAL(enter_rtas) 1324 stwu r1,-INT_FRAME_SIZE(r1) 1325 mflr r0 1326 stw r0,INT_FRAME_SIZE+4(r1) 1327 LOAD_REG_ADDR(r4, rtas) 1328 lis r6,1f@ha /* physical return address for rtas */ 1329 addi r6,r6,1f@l 1330 tophys(r6,r6) 1331 tophys(r7,r1) 1332 lwz r8,RTASENTRY(r4) 1333 lwz r4,RTASBASE(r4) 1334 mfmsr r9 1335 stw r9,8(r1) 1336 LOAD_MSR_KERNEL(r0,MSR_KERNEL) 1337 SYNC /* disable interrupts so SRR0/1 */ 1338 MTMSRD(r0) /* don't get trashed */ 1339 li r9,MSR_KERNEL & ~(MSR_IR|MSR_DR) 1340 mtlr r6 1341 mtspr SPRN_SPRG_RTAS,r7 1342 mtspr SPRN_SRR0,r8 1343 mtspr SPRN_SRR1,r9 1344 RFI 13451: tophys(r9,r1) 1346 lwz r8,INT_FRAME_SIZE+4(r9) /* get return address */ 1347 lwz r9,8(r9) /* original msr value */ 1348 FIX_SRR1(r9,r0) 1349 addi r1,r1,INT_FRAME_SIZE 1350 li r0,0 1351 mtspr SPRN_SPRG_RTAS,r0 1352 mtspr SPRN_SRR0,r8 1353 mtspr SPRN_SRR1,r9 1354 RFI /* return to caller */ 1355 1356 .globl machine_check_in_rtas 1357machine_check_in_rtas: 1358 twi 31,0,0 1359 /* XXX load up BATs and panic */ 1360 1361#endif /* CONFIG_PPC_RTAS */ 1362 1363#ifdef CONFIG_FUNCTION_TRACER 1364#ifdef CONFIG_DYNAMIC_FTRACE 1365_GLOBAL(mcount) 1366_GLOBAL(_mcount) 1367 /* 1368 * It is required that _mcount on PPC32 must preserve the 1369 * link register. But we have r0 to play with. We use r0 1370 * to push the return address back to the caller of mcount 1371 * into the ctr register, restore the link register and 1372 * then jump back using the ctr register. 1373 */ 1374 mflr r0 1375 mtctr r0 1376 lwz r0, 4(r1) 1377 mtlr r0 1378 bctr 1379 1380_GLOBAL(ftrace_caller) 1381 MCOUNT_SAVE_FRAME 1382 /* r3 ends up with link register */ 1383 subi r3, r3, MCOUNT_INSN_SIZE 1384.globl ftrace_call 1385ftrace_call: 1386 bl ftrace_stub 1387 nop 1388#ifdef CONFIG_FUNCTION_GRAPH_TRACER 1389.globl ftrace_graph_call 1390ftrace_graph_call: 1391 b ftrace_graph_stub 1392_GLOBAL(ftrace_graph_stub) 1393#endif 1394 MCOUNT_RESTORE_FRAME 1395 /* old link register ends up in ctr reg */ 1396 bctr 1397#else 1398_GLOBAL(mcount) 1399_GLOBAL(_mcount) 1400 1401 MCOUNT_SAVE_FRAME 1402 1403 subi r3, r3, MCOUNT_INSN_SIZE 1404 LOAD_REG_ADDR(r5, ftrace_trace_function) 1405 lwz r5,0(r5) 1406 1407 mtctr r5 1408 bctrl 1409 nop 1410 1411#ifdef CONFIG_FUNCTION_GRAPH_TRACER 1412 b ftrace_graph_caller 1413#endif 1414 MCOUNT_RESTORE_FRAME 1415 bctr 1416#endif 1417 1418_GLOBAL(ftrace_stub) 1419 blr 1420 1421#ifdef CONFIG_FUNCTION_GRAPH_TRACER 1422_GLOBAL(ftrace_graph_caller) 1423 /* load r4 with local address */ 1424 lwz r4, 44(r1) 1425 subi r4, r4, MCOUNT_INSN_SIZE 1426 1427 /* get the parent address */ 1428 addi r3, r1, 52 1429 1430 bl prepare_ftrace_return 1431 nop 1432 1433 MCOUNT_RESTORE_FRAME 1434 /* old link register ends up in ctr reg */ 1435 bctr 1436 1437_GLOBAL(return_to_handler) 1438 /* need to save return values */ 1439 stwu r1, -32(r1) 1440 stw r3, 20(r1) 1441 stw r4, 16(r1) 1442 stw r31, 12(r1) 1443 mr r31, r1 1444 1445 bl ftrace_return_to_handler 1446 nop 1447 1448 /* return value has real return address */ 1449 mtlr r3 1450 1451 lwz r3, 20(r1) 1452 lwz r4, 16(r1) 1453 lwz r31,12(r1) 1454 lwz r1, 0(r1) 1455 1456 /* Jump back to real return address */ 1457 blr 1458#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 1459 1460#endif /* CONFIG_MCOUNT */ 1461