1/* SPDX-License-Identifier: GPL-2.0-or-later */ 2/* 3 * PowerPC version 4 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org) 5 * Rewritten by Cort Dougan (cort@fsmlabs.com) for PReP 6 * Copyright (C) 1996 Cort Dougan <cort@fsmlabs.com> 7 * Adapted for Power Macintosh by Paul Mackerras. 8 * Low-level exception handlers and MMU support 9 * rewritten by Paul Mackerras. 10 * Copyright (C) 1996 Paul Mackerras. 11 * MPC8xx modifications Copyright (C) 1997 Dan Malek (dmalek@jlc.net). 12 * 13 * This file contains the system call entry code, context switch 14 * code, and exception/interrupt return code for PowerPC. 15 */ 16 17#include <linux/errno.h> 18#include <linux/err.h> 19#include <linux/sys.h> 20#include <linux/threads.h> 21#include <asm/reg.h> 22#include <asm/page.h> 23#include <asm/mmu.h> 24#include <asm/cputable.h> 25#include <asm/thread_info.h> 26#include <asm/ppc_asm.h> 27#include <asm/asm-offsets.h> 28#include <asm/unistd.h> 29#include <asm/ptrace.h> 30#include <asm/export.h> 31#include <asm/asm-405.h> 32#include <asm/feature-fixups.h> 33#include <asm/barrier.h> 34#include <asm/kup.h> 35#include <asm/bug.h> 36 37#include "head_32.h" 38 39/* 40 * Align to 4k in order to ensure that all functions modyfing srr0/srr1 41 * fit into one page in order to not encounter a TLB miss between the 42 * modification of srr0/srr1 and the associated rfi. 43 */ 44 .align 12 45 46#ifdef CONFIG_BOOKE 47 .globl mcheck_transfer_to_handler 48mcheck_transfer_to_handler: 49 mfspr r0,SPRN_DSRR0 50 stw r0,_DSRR0(r11) 51 mfspr r0,SPRN_DSRR1 52 stw r0,_DSRR1(r11) 53 /* fall through */ 54 55 .globl debug_transfer_to_handler 56debug_transfer_to_handler: 57 mfspr r0,SPRN_CSRR0 58 stw r0,_CSRR0(r11) 59 mfspr r0,SPRN_CSRR1 60 stw r0,_CSRR1(r11) 61 /* fall through */ 62 63 .globl crit_transfer_to_handler 64crit_transfer_to_handler: 65#ifdef CONFIG_PPC_BOOK3E_MMU 66 mfspr r0,SPRN_MAS0 67 stw r0,MAS0(r11) 68 mfspr r0,SPRN_MAS1 69 stw r0,MAS1(r11) 70 mfspr r0,SPRN_MAS2 71 stw r0,MAS2(r11) 72 mfspr r0,SPRN_MAS3 73 stw r0,MAS3(r11) 74 mfspr r0,SPRN_MAS6 75 stw r0,MAS6(r11) 76#ifdef CONFIG_PHYS_64BIT 77 mfspr r0,SPRN_MAS7 78 stw r0,MAS7(r11) 79#endif /* CONFIG_PHYS_64BIT */ 80#endif /* CONFIG_PPC_BOOK3E_MMU */ 81#ifdef CONFIG_44x 82 mfspr r0,SPRN_MMUCR 83 stw r0,MMUCR(r11) 84#endif 85 mfspr r0,SPRN_SRR0 86 stw r0,_SRR0(r11) 87 mfspr r0,SPRN_SRR1 88 stw r0,_SRR1(r11) 89 90 /* set the stack limit to the current stack */ 91 mfspr r8,SPRN_SPRG_THREAD 92 lwz r0,KSP_LIMIT(r8) 93 stw r0,SAVED_KSP_LIMIT(r11) 94 rlwinm r0,r1,0,0,(31 - THREAD_SHIFT) 95 stw r0,KSP_LIMIT(r8) 96 /* fall through */ 97#endif 98 99#ifdef CONFIG_40x 100 .globl crit_transfer_to_handler 101crit_transfer_to_handler: 102 lwz r0,crit_r10@l(0) 103 stw r0,GPR10(r11) 104 lwz r0,crit_r11@l(0) 105 stw r0,GPR11(r11) 106 mfspr r0,SPRN_SRR0 107 stw r0,crit_srr0@l(0) 108 mfspr r0,SPRN_SRR1 109 stw r0,crit_srr1@l(0) 110 111 /* set the stack limit to the current stack */ 112 mfspr r8,SPRN_SPRG_THREAD 113 lwz r0,KSP_LIMIT(r8) 114 stw r0,saved_ksp_limit@l(0) 115 rlwinm r0,r1,0,0,(31 - THREAD_SHIFT) 116 stw r0,KSP_LIMIT(r8) 117 /* fall through */ 118#endif 119 120/* 121 * This code finishes saving the registers to the exception frame 122 * and jumps to the appropriate handler for the exception, turning 123 * on address translation. 124 * Note that we rely on the caller having set cr0.eq iff the exception 125 * occurred in kernel mode (i.e. MSR:PR = 0). 126 */ 127 .globl transfer_to_handler_full 128transfer_to_handler_full: 129 SAVE_NVGPRS(r11) 130 /* fall through */ 131 132 .globl transfer_to_handler 133transfer_to_handler: 134 stw r2,GPR2(r11) 135 stw r12,_NIP(r11) 136 stw r9,_MSR(r11) 137 andi. r2,r9,MSR_PR 138 mfctr r12 139 mfspr r2,SPRN_XER 140 stw r12,_CTR(r11) 141 stw r2,_XER(r11) 142 mfspr r12,SPRN_SPRG_THREAD 143 beq 2f /* if from user, fix up THREAD.regs */ 144 addi r2, r12, -THREAD 145 addi r11,r1,STACK_FRAME_OVERHEAD 146 stw r11,PT_REGS(r12) 147#if defined(CONFIG_40x) || defined(CONFIG_BOOKE) 148 /* Check to see if the dbcr0 register is set up to debug. Use the 149 internal debug mode bit to do this. */ 150 lwz r12,THREAD_DBCR0(r12) 151 andis. r12,r12,DBCR0_IDM@h 152#endif 153 ACCOUNT_CPU_USER_ENTRY(r2, r11, r12) 154#ifdef CONFIG_PPC_BOOK3S_32 155 kuep_lock r11, r12 156#endif 157#if defined(CONFIG_40x) || defined(CONFIG_BOOKE) 158 beq+ 3f 159 /* From user and task is ptraced - load up global dbcr0 */ 160 li r12,-1 /* clear all pending debug events */ 161 mtspr SPRN_DBSR,r12 162 lis r11,global_dbcr0@ha 163 tophys(r11,r11) 164 addi r11,r11,global_dbcr0@l 165#ifdef CONFIG_SMP 166 lwz r9,TASK_CPU(r2) 167 slwi r9,r9,3 168 add r11,r11,r9 169#endif 170 lwz r12,0(r11) 171 mtspr SPRN_DBCR0,r12 172 lwz r12,4(r11) 173 addi r12,r12,-1 174 stw r12,4(r11) 175#endif 176 177 b 3f 178 1792: /* if from kernel, check interrupted DOZE/NAP mode and 180 * check for stack overflow 181 */ 182 kuap_save_and_lock r11, r12, r9, r2, r0 183 addi r2, r12, -THREAD 184 lwz r9,KSP_LIMIT(r12) 185 cmplw r1,r9 /* if r1 <= ksp_limit */ 186 ble- stack_ovf /* then the kernel stack overflowed */ 1875: 188#if defined(CONFIG_PPC_BOOK3S_32) || defined(CONFIG_E500) 189 lwz r12,TI_LOCAL_FLAGS(r2) 190 mtcrf 0x01,r12 191 bt- 31-TLF_NAPPING,4f 192 bt- 31-TLF_SLEEPING,7f 193#endif /* CONFIG_PPC_BOOK3S_32 || CONFIG_E500 */ 194 .globl transfer_to_handler_cont 195transfer_to_handler_cont: 1963: 197 mflr r9 198 tovirt(r2, r2) /* set r2 to current */ 199 lwz r11,0(r9) /* virtual address of handler */ 200 lwz r9,4(r9) /* where to go when done */ 201#if defined(CONFIG_PPC_8xx) && defined(CONFIG_PERF_EVENTS) 202 mtspr SPRN_NRI, r0 203#endif 204#ifdef CONFIG_TRACE_IRQFLAGS 205 /* 206 * When tracing IRQ state (lockdep) we enable the MMU before we call 207 * the IRQ tracing functions as they might access vmalloc space or 208 * perform IOs for console output. 209 * 210 * To speed up the syscall path where interrupts stay on, let's check 211 * first if we are changing the MSR value at all. 212 */ 213 tophys(r12, r1) 214 lwz r12,_MSR(r12) 215 andi. r12,r12,MSR_EE 216 bne 1f 217 218 /* MSR isn't changing, just transition directly */ 219#endif 220 mtspr SPRN_SRR0,r11 221 mtspr SPRN_SRR1,r10 222 mtlr r9 223 SYNC 224 RFI /* jump to handler, enable MMU */ 225 226#ifdef CONFIG_TRACE_IRQFLAGS 2271: /* MSR is changing, re-enable MMU so we can notify lockdep. We need to 228 * keep interrupts disabled at this point otherwise we might risk 229 * taking an interrupt before we tell lockdep they are enabled. 230 */ 231 lis r12,reenable_mmu@h 232 ori r12,r12,reenable_mmu@l 233 LOAD_MSR_KERNEL(r0, MSR_KERNEL) 234 mtspr SPRN_SRR0,r12 235 mtspr SPRN_SRR1,r0 236 SYNC 237 RFI 238 239reenable_mmu: 240 /* 241 * We save a bunch of GPRs, 242 * r3 can be different from GPR3(r1) at this point, r9 and r11 243 * contains the old MSR and handler address respectively, 244 * r4 & r5 can contain page fault arguments that need to be passed 245 * along as well. r12, CCR, CTR, XER etc... are left clobbered as 246 * they aren't useful past this point (aren't syscall arguments), 247 * the rest is restored from the exception frame. 248 */ 249 250 stwu r1,-32(r1) 251 stw r9,8(r1) 252 stw r11,12(r1) 253 stw r3,16(r1) 254 stw r4,20(r1) 255 stw r5,24(r1) 256 257 /* If we are disabling interrupts (normal case), simply log it with 258 * lockdep 259 */ 2601: bl trace_hardirqs_off 2612: lwz r5,24(r1) 262 lwz r4,20(r1) 263 lwz r3,16(r1) 264 lwz r11,12(r1) 265 lwz r9,8(r1) 266 addi r1,r1,32 267 lwz r0,GPR0(r1) 268 lwz r6,GPR6(r1) 269 lwz r7,GPR7(r1) 270 lwz r8,GPR8(r1) 271 mtctr r11 272 mtlr r9 273 bctr /* jump to handler */ 274#endif /* CONFIG_TRACE_IRQFLAGS */ 275 276#if defined (CONFIG_PPC_BOOK3S_32) || defined(CONFIG_E500) 2774: rlwinm r12,r12,0,~_TLF_NAPPING 278 stw r12,TI_LOCAL_FLAGS(r2) 279 b power_save_ppc32_restore 280 2817: rlwinm r12,r12,0,~_TLF_SLEEPING 282 stw r12,TI_LOCAL_FLAGS(r2) 283 lwz r9,_MSR(r11) /* if sleeping, clear MSR.EE */ 284 rlwinm r9,r9,0,~MSR_EE 285 lwz r12,_LINK(r11) /* and return to address in LR */ 286 kuap_restore r11, r2, r3, r4, r5 287 b fast_exception_return 288#endif 289 290/* 291 * On kernel stack overflow, load up an initial stack pointer 292 * and call StackOverflow(regs), which should not return. 293 */ 294stack_ovf: 295 /* sometimes we use a statically-allocated stack, which is OK. */ 296 lis r12,_end@h 297 ori r12,r12,_end@l 298 cmplw r1,r12 299 ble 5b /* r1 <= &_end is OK */ 300 SAVE_NVGPRS(r11) 301 addi r3,r1,STACK_FRAME_OVERHEAD 302 lis r1,init_thread_union@ha 303 addi r1,r1,init_thread_union@l 304 addi r1,r1,THREAD_SIZE-STACK_FRAME_OVERHEAD 305 lis r9,StackOverflow@ha 306 addi r9,r9,StackOverflow@l 307 LOAD_MSR_KERNEL(r10,MSR_KERNEL) 308#if defined(CONFIG_PPC_8xx) && defined(CONFIG_PERF_EVENTS) 309 mtspr SPRN_NRI, r0 310#endif 311 mtspr SPRN_SRR0,r9 312 mtspr SPRN_SRR1,r10 313 SYNC 314 RFI 315 316#ifdef CONFIG_TRACE_IRQFLAGS 317trace_syscall_entry_irq_off: 318 /* 319 * Syscall shouldn't happen while interrupts are disabled, 320 * so let's do a warning here. 321 */ 3220: trap 323 EMIT_BUG_ENTRY 0b,__FILE__,__LINE__, BUGFLAG_WARNING 324 bl trace_hardirqs_on 325 326 /* Now enable for real */ 327 LOAD_MSR_KERNEL(r10, MSR_KERNEL | MSR_EE) 328 mtmsr r10 329 330 REST_GPR(0, r1) 331 REST_4GPRS(3, r1) 332 REST_2GPRS(7, r1) 333 b DoSyscall 334#endif /* CONFIG_TRACE_IRQFLAGS */ 335 336 .globl transfer_to_syscall 337transfer_to_syscall: 338#ifdef CONFIG_TRACE_IRQFLAGS 339 andi. r12,r9,MSR_EE 340 beq- trace_syscall_entry_irq_off 341#endif /* CONFIG_TRACE_IRQFLAGS */ 342 343/* 344 * Handle a system call. 345 */ 346 .stabs "arch/powerpc/kernel/",N_SO,0,0,0f 347 .stabs "entry_32.S",N_SO,0,0,0f 3480: 349 350_GLOBAL(DoSyscall) 351 stw r3,ORIG_GPR3(r1) 352 li r12,0 353 stw r12,RESULT(r1) 354#ifdef CONFIG_TRACE_IRQFLAGS 355 /* Make sure interrupts are enabled */ 356 mfmsr r11 357 andi. r12,r11,MSR_EE 358 /* We came in with interrupts disabled, we WARN and mark them enabled 359 * for lockdep now */ 3600: tweqi r12, 0 361 EMIT_BUG_ENTRY 0b,__FILE__,__LINE__, BUGFLAG_WARNING 362#endif /* CONFIG_TRACE_IRQFLAGS */ 363 lwz r11,TI_FLAGS(r2) 364 andi. r11,r11,_TIF_SYSCALL_DOTRACE 365 bne- syscall_dotrace 366syscall_dotrace_cont: 367 cmplwi 0,r0,NR_syscalls 368 lis r10,sys_call_table@h 369 ori r10,r10,sys_call_table@l 370 slwi r0,r0,2 371 bge- 66f 372 373 barrier_nospec_asm 374 /* 375 * Prevent the load of the handler below (based on the user-passed 376 * system call number) being speculatively executed until the test 377 * against NR_syscalls and branch to .66f above has 378 * committed. 379 */ 380 381 lwzx r10,r10,r0 /* Fetch system call handler [ptr] */ 382 mtlr r10 383 addi r9,r1,STACK_FRAME_OVERHEAD 384 PPC440EP_ERR42 385 blrl /* Call handler */ 386 .globl ret_from_syscall 387ret_from_syscall: 388#ifdef CONFIG_DEBUG_RSEQ 389 /* Check whether the syscall is issued inside a restartable sequence */ 390 stw r3,GPR3(r1) 391 addi r3,r1,STACK_FRAME_OVERHEAD 392 bl rseq_syscall 393 lwz r3,GPR3(r1) 394#endif 395 mr r6,r3 396 /* disable interrupts so current_thread_info()->flags can't change */ 397 LOAD_MSR_KERNEL(r10,MSR_KERNEL) /* doesn't include MSR_EE */ 398 /* Note: We don't bother telling lockdep about it */ 399 SYNC 400 MTMSRD(r10) 401 lwz r9,TI_FLAGS(r2) 402 li r8,-MAX_ERRNO 403 andi. r0,r9,(_TIF_SYSCALL_DOTRACE|_TIF_SINGLESTEP|_TIF_USER_WORK_MASK|_TIF_PERSYSCALL_MASK) 404 bne- syscall_exit_work 405 cmplw 0,r3,r8 406 blt+ syscall_exit_cont 407 lwz r11,_CCR(r1) /* Load CR */ 408 neg r3,r3 409 oris r11,r11,0x1000 /* Set SO bit in CR */ 410 stw r11,_CCR(r1) 411syscall_exit_cont: 412 lwz r8,_MSR(r1) 413#ifdef CONFIG_TRACE_IRQFLAGS 414 /* If we are going to return from the syscall with interrupts 415 * off, we trace that here. It shouldn't normally happen. 416 */ 417 andi. r10,r8,MSR_EE 418 bne+ 1f 419 stw r3,GPR3(r1) 420 bl trace_hardirqs_off 421 lwz r3,GPR3(r1) 4221: 423#endif /* CONFIG_TRACE_IRQFLAGS */ 424#if defined(CONFIG_4xx) || defined(CONFIG_BOOKE) 425 /* If the process has its own DBCR0 value, load it up. The internal 426 debug mode bit tells us that dbcr0 should be loaded. */ 427 lwz r0,THREAD+THREAD_DBCR0(r2) 428 andis. r10,r0,DBCR0_IDM@h 429 bnel- load_dbcr0 430#endif 431#ifdef CONFIG_44x 432BEGIN_MMU_FTR_SECTION 433 lis r4,icache_44x_need_flush@ha 434 lwz r5,icache_44x_need_flush@l(r4) 435 cmplwi cr0,r5,0 436 bne- 2f 4371: 438END_MMU_FTR_SECTION_IFCLR(MMU_FTR_TYPE_47x) 439#endif /* CONFIG_44x */ 440BEGIN_FTR_SECTION 441 lwarx r7,0,r1 442END_FTR_SECTION_IFSET(CPU_FTR_NEED_PAIRED_STWCX) 443 stwcx. r0,0,r1 /* to clear the reservation */ 444 ACCOUNT_CPU_USER_EXIT(r2, r5, r7) 445#ifdef CONFIG_PPC_BOOK3S_32 446 kuep_unlock r5, r7 447#endif 448 kuap_check r2, r4 449 lwz r4,_LINK(r1) 450 lwz r5,_CCR(r1) 451 mtlr r4 452 mtcr r5 453 lwz r7,_NIP(r1) 454 lwz r2,GPR2(r1) 455 lwz r1,GPR1(r1) 456#if defined(CONFIG_PPC_8xx) && defined(CONFIG_PERF_EVENTS) 457 mtspr SPRN_NRI, r0 458#endif 459 mtspr SPRN_SRR0,r7 460 mtspr SPRN_SRR1,r8 461 SYNC 462 RFI 463#ifdef CONFIG_44x 4642: li r7,0 465 iccci r0,r0 466 stw r7,icache_44x_need_flush@l(r4) 467 b 1b 468#endif /* CONFIG_44x */ 469 47066: li r3,-ENOSYS 471 b ret_from_syscall 472 473 .globl ret_from_fork 474ret_from_fork: 475 REST_NVGPRS(r1) 476 bl schedule_tail 477 li r3,0 478 b ret_from_syscall 479 480 .globl ret_from_kernel_thread 481ret_from_kernel_thread: 482 REST_NVGPRS(r1) 483 bl schedule_tail 484 mtlr r14 485 mr r3,r15 486 PPC440EP_ERR42 487 blrl 488 li r3,0 489 b ret_from_syscall 490 491/* Traced system call support */ 492syscall_dotrace: 493 SAVE_NVGPRS(r1) 494 li r0,0xc00 495 stw r0,_TRAP(r1) 496 addi r3,r1,STACK_FRAME_OVERHEAD 497 bl do_syscall_trace_enter 498 /* 499 * Restore argument registers possibly just changed. 500 * We use the return value of do_syscall_trace_enter 501 * for call number to look up in the table (r0). 502 */ 503 mr r0,r3 504 lwz r3,GPR3(r1) 505 lwz r4,GPR4(r1) 506 lwz r5,GPR5(r1) 507 lwz r6,GPR6(r1) 508 lwz r7,GPR7(r1) 509 lwz r8,GPR8(r1) 510 REST_NVGPRS(r1) 511 512 cmplwi r0,NR_syscalls 513 /* Return code is already in r3 thanks to do_syscall_trace_enter() */ 514 bge- ret_from_syscall 515 b syscall_dotrace_cont 516 517syscall_exit_work: 518 andi. r0,r9,_TIF_RESTOREALL 519 beq+ 0f 520 REST_NVGPRS(r1) 521 b 2f 5220: cmplw 0,r3,r8 523 blt+ 1f 524 andi. r0,r9,_TIF_NOERROR 525 bne- 1f 526 lwz r11,_CCR(r1) /* Load CR */ 527 neg r3,r3 528 oris r11,r11,0x1000 /* Set SO bit in CR */ 529 stw r11,_CCR(r1) 530 5311: stw r6,RESULT(r1) /* Save result */ 532 stw r3,GPR3(r1) /* Update return value */ 5332: andi. r0,r9,(_TIF_PERSYSCALL_MASK) 534 beq 4f 535 536 /* Clear per-syscall TIF flags if any are set. */ 537 538 li r11,_TIF_PERSYSCALL_MASK 539 addi r12,r2,TI_FLAGS 5403: lwarx r8,0,r12 541 andc r8,r8,r11 542#ifdef CONFIG_IBM405_ERR77 543 dcbt 0,r12 544#endif 545 stwcx. r8,0,r12 546 bne- 3b 547 5484: /* Anything which requires enabling interrupts? */ 549 andi. r0,r9,(_TIF_SYSCALL_DOTRACE|_TIF_SINGLESTEP) 550 beq ret_from_except 551 552 /* Re-enable interrupts. There is no need to trace that with 553 * lockdep as we are supposed to have IRQs on at this point 554 */ 555 ori r10,r10,MSR_EE 556 SYNC 557 MTMSRD(r10) 558 559 /* Save NVGPRS if they're not saved already */ 560 lwz r4,_TRAP(r1) 561 andi. r4,r4,1 562 beq 5f 563 SAVE_NVGPRS(r1) 564 li r4,0xc00 565 stw r4,_TRAP(r1) 5665: 567 addi r3,r1,STACK_FRAME_OVERHEAD 568 bl do_syscall_trace_leave 569 b ret_from_except_full 570 571/* 572 * The fork/clone functions need to copy the full register set into 573 * the child process. Therefore we need to save all the nonvolatile 574 * registers (r13 - r31) before calling the C code. 575 */ 576 .globl ppc_fork 577ppc_fork: 578 SAVE_NVGPRS(r1) 579 lwz r0,_TRAP(r1) 580 rlwinm r0,r0,0,0,30 /* clear LSB to indicate full */ 581 stw r0,_TRAP(r1) /* register set saved */ 582 b sys_fork 583 584 .globl ppc_vfork 585ppc_vfork: 586 SAVE_NVGPRS(r1) 587 lwz r0,_TRAP(r1) 588 rlwinm r0,r0,0,0,30 /* clear LSB to indicate full */ 589 stw r0,_TRAP(r1) /* register set saved */ 590 b sys_vfork 591 592 .globl ppc_clone 593ppc_clone: 594 SAVE_NVGPRS(r1) 595 lwz r0,_TRAP(r1) 596 rlwinm r0,r0,0,0,30 /* clear LSB to indicate full */ 597 stw r0,_TRAP(r1) /* register set saved */ 598 b sys_clone 599 600 .globl ppc_clone3 601ppc_clone3: 602 SAVE_NVGPRS(r1) 603 lwz r0,_TRAP(r1) 604 rlwinm r0,r0,0,0,30 /* clear LSB to indicate full */ 605 stw r0,_TRAP(r1) /* register set saved */ 606 b sys_clone3 607 608 .globl ppc_swapcontext 609ppc_swapcontext: 610 SAVE_NVGPRS(r1) 611 lwz r0,_TRAP(r1) 612 rlwinm r0,r0,0,0,30 /* clear LSB to indicate full */ 613 stw r0,_TRAP(r1) /* register set saved */ 614 b sys_swapcontext 615 616/* 617 * Top-level page fault handling. 618 * This is in assembler because if do_page_fault tells us that 619 * it is a bad kernel page fault, we want to save the non-volatile 620 * registers before calling bad_page_fault. 621 */ 622 .globl handle_page_fault 623handle_page_fault: 624 stw r4,_DAR(r1) 625 addi r3,r1,STACK_FRAME_OVERHEAD 626#ifdef CONFIG_PPC_BOOK3S_32 627 andis. r0,r5,DSISR_DABRMATCH@h 628 bne- handle_dabr_fault 629#endif 630 bl do_page_fault 631 cmpwi r3,0 632 beq+ ret_from_except 633 SAVE_NVGPRS(r1) 634 lwz r0,_TRAP(r1) 635 clrrwi r0,r0,1 636 stw r0,_TRAP(r1) 637 mr r5,r3 638 addi r3,r1,STACK_FRAME_OVERHEAD 639 lwz r4,_DAR(r1) 640 bl bad_page_fault 641 b ret_from_except_full 642 643#ifdef CONFIG_PPC_BOOK3S_32 644 /* We have a data breakpoint exception - handle it */ 645handle_dabr_fault: 646 SAVE_NVGPRS(r1) 647 lwz r0,_TRAP(r1) 648 clrrwi r0,r0,1 649 stw r0,_TRAP(r1) 650 bl do_break 651 b ret_from_except_full 652#endif 653 654/* 655 * This routine switches between two different tasks. The process 656 * state of one is saved on its kernel stack. Then the state 657 * of the other is restored from its kernel stack. The memory 658 * management hardware is updated to the second process's state. 659 * Finally, we can return to the second process. 660 * On entry, r3 points to the THREAD for the current task, r4 661 * points to the THREAD for the new task. 662 * 663 * This routine is always called with interrupts disabled. 664 * 665 * Note: there are two ways to get to the "going out" portion 666 * of this code; either by coming in via the entry (_switch) 667 * or via "fork" which must set up an environment equivalent 668 * to the "_switch" path. If you change this , you'll have to 669 * change the fork code also. 670 * 671 * The code which creates the new task context is in 'copy_thread' 672 * in arch/ppc/kernel/process.c 673 */ 674_GLOBAL(_switch) 675 stwu r1,-INT_FRAME_SIZE(r1) 676 mflr r0 677 stw r0,INT_FRAME_SIZE+4(r1) 678 /* r3-r12 are caller saved -- Cort */ 679 SAVE_NVGPRS(r1) 680 stw r0,_NIP(r1) /* Return to switch caller */ 681 mfmsr r11 682 li r0,MSR_FP /* Disable floating-point */ 683#ifdef CONFIG_ALTIVEC 684BEGIN_FTR_SECTION 685 oris r0,r0,MSR_VEC@h /* Disable altivec */ 686 mfspr r12,SPRN_VRSAVE /* save vrsave register value */ 687 stw r12,THREAD+THREAD_VRSAVE(r2) 688END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 689#endif /* CONFIG_ALTIVEC */ 690#ifdef CONFIG_SPE 691BEGIN_FTR_SECTION 692 oris r0,r0,MSR_SPE@h /* Disable SPE */ 693 mfspr r12,SPRN_SPEFSCR /* save spefscr register value */ 694 stw r12,THREAD+THREAD_SPEFSCR(r2) 695END_FTR_SECTION_IFSET(CPU_FTR_SPE) 696#endif /* CONFIG_SPE */ 697 and. r0,r0,r11 /* FP or altivec or SPE enabled? */ 698 beq+ 1f 699 andc r11,r11,r0 700 MTMSRD(r11) 701 isync 7021: stw r11,_MSR(r1) 703 mfcr r10 704 stw r10,_CCR(r1) 705 stw r1,KSP(r3) /* Set old stack pointer */ 706 707 kuap_check r2, r4 708#ifdef CONFIG_SMP 709 /* We need a sync somewhere here to make sure that if the 710 * previous task gets rescheduled on another CPU, it sees all 711 * stores it has performed on this one. 712 */ 713 sync 714#endif /* CONFIG_SMP */ 715 716 tophys(r0,r4) 717 mtspr SPRN_SPRG_THREAD,r0 /* Update current THREAD phys addr */ 718 lwz r1,KSP(r4) /* Load new stack pointer */ 719 720 /* save the old current 'last' for return value */ 721 mr r3,r2 722 addi r2,r4,-THREAD /* Update current */ 723 724#ifdef CONFIG_ALTIVEC 725BEGIN_FTR_SECTION 726 lwz r0,THREAD+THREAD_VRSAVE(r2) 727 mtspr SPRN_VRSAVE,r0 /* if G4, restore VRSAVE reg */ 728END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 729#endif /* CONFIG_ALTIVEC */ 730#ifdef CONFIG_SPE 731BEGIN_FTR_SECTION 732 lwz r0,THREAD+THREAD_SPEFSCR(r2) 733 mtspr SPRN_SPEFSCR,r0 /* restore SPEFSCR reg */ 734END_FTR_SECTION_IFSET(CPU_FTR_SPE) 735#endif /* CONFIG_SPE */ 736 737 lwz r0,_CCR(r1) 738 mtcrf 0xFF,r0 739 /* r3-r12 are destroyed -- Cort */ 740 REST_NVGPRS(r1) 741 742 lwz r4,_NIP(r1) /* Return to _switch caller in new task */ 743 mtlr r4 744 addi r1,r1,INT_FRAME_SIZE 745 blr 746 747 .globl fast_exception_return 748fast_exception_return: 749#if !(defined(CONFIG_4xx) || defined(CONFIG_BOOKE)) 750 andi. r10,r9,MSR_RI /* check for recoverable interrupt */ 751 beq 1f /* if not, we've got problems */ 752#endif 753 7542: REST_4GPRS(3, r11) 755 lwz r10,_CCR(r11) 756 REST_GPR(1, r11) 757 mtcr r10 758 lwz r10,_LINK(r11) 759 mtlr r10 760 /* Clear the exception_marker on the stack to avoid confusing stacktrace */ 761 li r10, 0 762 stw r10, 8(r11) 763 REST_GPR(10, r11) 764#if defined(CONFIG_PPC_8xx) && defined(CONFIG_PERF_EVENTS) 765 mtspr SPRN_NRI, r0 766#endif 767 mtspr SPRN_SRR1,r9 768 mtspr SPRN_SRR0,r12 769 REST_GPR(9, r11) 770 REST_GPR(12, r11) 771 lwz r11,GPR11(r11) 772 SYNC 773 RFI 774 775#if !(defined(CONFIG_4xx) || defined(CONFIG_BOOKE)) 776/* check if the exception happened in a restartable section */ 7771: lis r3,exc_exit_restart_end@ha 778 addi r3,r3,exc_exit_restart_end@l 779 cmplw r12,r3 780 bge 3f 781 lis r4,exc_exit_restart@ha 782 addi r4,r4,exc_exit_restart@l 783 cmplw r12,r4 784 blt 3f 785 lis r3,fee_restarts@ha 786 tophys(r3,r3) 787 lwz r5,fee_restarts@l(r3) 788 addi r5,r5,1 789 stw r5,fee_restarts@l(r3) 790 mr r12,r4 /* restart at exc_exit_restart */ 791 b 2b 792 793 .section .bss 794 .align 2 795fee_restarts: 796 .space 4 797 .previous 798 799/* aargh, a nonrecoverable interrupt, panic */ 800/* aargh, we don't know which trap this is */ 801/* but the 601 doesn't implement the RI bit, so assume it's OK */ 8023: 803BEGIN_FTR_SECTION 804 b 2b 805END_FTR_SECTION_IFSET(CPU_FTR_601) 806 li r10,-1 807 stw r10,_TRAP(r11) 808 addi r3,r1,STACK_FRAME_OVERHEAD 809 lis r10,MSR_KERNEL@h 810 ori r10,r10,MSR_KERNEL@l 811 bl transfer_to_handler_full 812 .long unrecoverable_exception 813 .long ret_from_except 814#endif 815 816 .globl ret_from_except_full 817ret_from_except_full: 818 REST_NVGPRS(r1) 819 /* fall through */ 820 821 .globl ret_from_except 822ret_from_except: 823 /* Hard-disable interrupts so that current_thread_info()->flags 824 * can't change between when we test it and when we return 825 * from the interrupt. */ 826 /* Note: We don't bother telling lockdep about it */ 827 LOAD_MSR_KERNEL(r10,MSR_KERNEL) 828 SYNC /* Some chip revs have problems here... */ 829 MTMSRD(r10) /* disable interrupts */ 830 831 lwz r3,_MSR(r1) /* Returning to user mode? */ 832 andi. r0,r3,MSR_PR 833 beq resume_kernel 834 835user_exc_return: /* r10 contains MSR_KERNEL here */ 836 /* Check current_thread_info()->flags */ 837 lwz r9,TI_FLAGS(r2) 838 andi. r0,r9,_TIF_USER_WORK_MASK 839 bne do_work 840 841restore_user: 842#if defined(CONFIG_4xx) || defined(CONFIG_BOOKE) 843 /* Check whether this process has its own DBCR0 value. The internal 844 debug mode bit tells us that dbcr0 should be loaded. */ 845 lwz r0,THREAD+THREAD_DBCR0(r2) 846 andis. r10,r0,DBCR0_IDM@h 847 bnel- load_dbcr0 848#endif 849 ACCOUNT_CPU_USER_EXIT(r2, r10, r11) 850#ifdef CONFIG_PPC_BOOK3S_32 851 kuep_unlock r10, r11 852#endif 853 854 b restore 855 856/* N.B. the only way to get here is from the beq following ret_from_except. */ 857resume_kernel: 858 /* check current_thread_info, _TIF_EMULATE_STACK_STORE */ 859 lwz r8,TI_FLAGS(r2) 860 andis. r0,r8,_TIF_EMULATE_STACK_STORE@h 861 beq+ 1f 862 863 addi r8,r1,INT_FRAME_SIZE /* Get the kprobed function entry */ 864 865 lwz r3,GPR1(r1) 866 subi r3,r3,INT_FRAME_SIZE /* dst: Allocate a trampoline exception frame */ 867 mr r4,r1 /* src: current exception frame */ 868 mr r1,r3 /* Reroute the trampoline frame to r1 */ 869 870 /* Copy from the original to the trampoline. */ 871 li r5,INT_FRAME_SIZE/4 /* size: INT_FRAME_SIZE */ 872 li r6,0 /* start offset: 0 */ 873 mtctr r5 8742: lwzx r0,r6,r4 875 stwx r0,r6,r3 876 addi r6,r6,4 877 bdnz 2b 878 879 /* Do real store operation to complete stwu */ 880 lwz r5,GPR1(r1) 881 stw r8,0(r5) 882 883 /* Clear _TIF_EMULATE_STACK_STORE flag */ 884 lis r11,_TIF_EMULATE_STACK_STORE@h 885 addi r5,r2,TI_FLAGS 8860: lwarx r8,0,r5 887 andc r8,r8,r11 888#ifdef CONFIG_IBM405_ERR77 889 dcbt 0,r5 890#endif 891 stwcx. r8,0,r5 892 bne- 0b 8931: 894 895#ifdef CONFIG_PREEMPT 896 /* check current_thread_info->preempt_count */ 897 lwz r0,TI_PREEMPT(r2) 898 cmpwi 0,r0,0 /* if non-zero, just restore regs and return */ 899 bne restore_kuap 900 andi. r8,r8,_TIF_NEED_RESCHED 901 beq+ restore_kuap 902 lwz r3,_MSR(r1) 903 andi. r0,r3,MSR_EE /* interrupts off? */ 904 beq restore_kuap /* don't schedule if so */ 905#ifdef CONFIG_TRACE_IRQFLAGS 906 /* Lockdep thinks irqs are enabled, we need to call 907 * preempt_schedule_irq with IRQs off, so we inform lockdep 908 * now that we -did- turn them off already 909 */ 910 bl trace_hardirqs_off 911#endif 912 bl preempt_schedule_irq 913#ifdef CONFIG_TRACE_IRQFLAGS 914 /* And now, to properly rebalance the above, we tell lockdep they 915 * are being turned back on, which will happen when we return 916 */ 917 bl trace_hardirqs_on 918#endif 919#endif /* CONFIG_PREEMPT */ 920restore_kuap: 921 kuap_restore r1, r2, r9, r10, r0 922 923 /* interrupts are hard-disabled at this point */ 924restore: 925#ifdef CONFIG_44x 926BEGIN_MMU_FTR_SECTION 927 b 1f 928END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_47x) 929 lis r4,icache_44x_need_flush@ha 930 lwz r5,icache_44x_need_flush@l(r4) 931 cmplwi cr0,r5,0 932 beq+ 1f 933 li r6,0 934 iccci r0,r0 935 stw r6,icache_44x_need_flush@l(r4) 9361: 937#endif /* CONFIG_44x */ 938 939 lwz r9,_MSR(r1) 940#ifdef CONFIG_TRACE_IRQFLAGS 941 /* Lockdep doesn't know about the fact that IRQs are temporarily turned 942 * off in this assembly code while peeking at TI_FLAGS() and such. However 943 * we need to inform it if the exception turned interrupts off, and we 944 * are about to trun them back on. 945 */ 946 andi. r10,r9,MSR_EE 947 beq 1f 948 stwu r1,-32(r1) 949 mflr r0 950 stw r0,4(r1) 951 bl trace_hardirqs_on 952 addi r1, r1, 32 953 lwz r9,_MSR(r1) 9541: 955#endif /* CONFIG_TRACE_IRQFLAGS */ 956 957 lwz r0,GPR0(r1) 958 lwz r2,GPR2(r1) 959 REST_4GPRS(3, r1) 960 REST_2GPRS(7, r1) 961 962 lwz r10,_XER(r1) 963 lwz r11,_CTR(r1) 964 mtspr SPRN_XER,r10 965 mtctr r11 966 967 PPC405_ERR77(0,r1) 968BEGIN_FTR_SECTION 969 lwarx r11,0,r1 970END_FTR_SECTION_IFSET(CPU_FTR_NEED_PAIRED_STWCX) 971 stwcx. r0,0,r1 /* to clear the reservation */ 972 973#if !(defined(CONFIG_4xx) || defined(CONFIG_BOOKE)) 974 andi. r10,r9,MSR_RI /* check if this exception occurred */ 975 beql nonrecoverable /* at a bad place (MSR:RI = 0) */ 976 977 lwz r10,_CCR(r1) 978 lwz r11,_LINK(r1) 979 mtcrf 0xFF,r10 980 mtlr r11 981 982 /* Clear the exception_marker on the stack to avoid confusing stacktrace */ 983 li r10, 0 984 stw r10, 8(r1) 985 /* 986 * Once we put values in SRR0 and SRR1, we are in a state 987 * where exceptions are not recoverable, since taking an 988 * exception will trash SRR0 and SRR1. Therefore we clear the 989 * MSR:RI bit to indicate this. If we do take an exception, 990 * we can't return to the point of the exception but we 991 * can restart the exception exit path at the label 992 * exc_exit_restart below. -- paulus 993 */ 994 LOAD_MSR_KERNEL(r10,MSR_KERNEL & ~MSR_RI) 995 SYNC 996 MTMSRD(r10) /* clear the RI bit */ 997 .globl exc_exit_restart 998exc_exit_restart: 999 lwz r12,_NIP(r1) 1000 mtspr SPRN_SRR0,r12 1001 mtspr SPRN_SRR1,r9 1002 REST_4GPRS(9, r1) 1003 lwz r1,GPR1(r1) 1004 .globl exc_exit_restart_end 1005exc_exit_restart_end: 1006 SYNC 1007 RFI 1008 1009#else /* !(CONFIG_4xx || CONFIG_BOOKE) */ 1010 /* 1011 * This is a bit different on 4xx/Book-E because it doesn't have 1012 * the RI bit in the MSR. 1013 * The TLB miss handler checks if we have interrupted 1014 * the exception exit path and restarts it if so 1015 * (well maybe one day it will... :). 1016 */ 1017 lwz r11,_LINK(r1) 1018 mtlr r11 1019 lwz r10,_CCR(r1) 1020 mtcrf 0xff,r10 1021 /* Clear the exception_marker on the stack to avoid confusing stacktrace */ 1022 li r10, 0 1023 stw r10, 8(r1) 1024 REST_2GPRS(9, r1) 1025 .globl exc_exit_restart 1026exc_exit_restart: 1027 lwz r11,_NIP(r1) 1028 lwz r12,_MSR(r1) 1029exc_exit_start: 1030 mtspr SPRN_SRR0,r11 1031 mtspr SPRN_SRR1,r12 1032 REST_2GPRS(11, r1) 1033 lwz r1,GPR1(r1) 1034 .globl exc_exit_restart_end 1035exc_exit_restart_end: 1036 PPC405_ERR77_SYNC 1037 rfi 1038 b . /* prevent prefetch past rfi */ 1039 1040/* 1041 * Returning from a critical interrupt in user mode doesn't need 1042 * to be any different from a normal exception. For a critical 1043 * interrupt in the kernel, we just return (without checking for 1044 * preemption) since the interrupt may have happened at some crucial 1045 * place (e.g. inside the TLB miss handler), and because we will be 1046 * running with r1 pointing into critical_stack, not the current 1047 * process's kernel stack (and therefore current_thread_info() will 1048 * give the wrong answer). 1049 * We have to restore various SPRs that may have been in use at the 1050 * time of the critical interrupt. 1051 * 1052 */ 1053#ifdef CONFIG_40x 1054#define PPC_40x_TURN_OFF_MSR_DR \ 1055 /* avoid any possible TLB misses here by turning off MSR.DR, we \ 1056 * assume the instructions here are mapped by a pinned TLB entry */ \ 1057 li r10,MSR_IR; \ 1058 mtmsr r10; \ 1059 isync; \ 1060 tophys(r1, r1); 1061#else 1062#define PPC_40x_TURN_OFF_MSR_DR 1063#endif 1064 1065#define RET_FROM_EXC_LEVEL(exc_lvl_srr0, exc_lvl_srr1, exc_lvl_rfi) \ 1066 REST_NVGPRS(r1); \ 1067 lwz r3,_MSR(r1); \ 1068 andi. r3,r3,MSR_PR; \ 1069 LOAD_MSR_KERNEL(r10,MSR_KERNEL); \ 1070 bne user_exc_return; \ 1071 lwz r0,GPR0(r1); \ 1072 lwz r2,GPR2(r1); \ 1073 REST_4GPRS(3, r1); \ 1074 REST_2GPRS(7, r1); \ 1075 lwz r10,_XER(r1); \ 1076 lwz r11,_CTR(r1); \ 1077 mtspr SPRN_XER,r10; \ 1078 mtctr r11; \ 1079 PPC405_ERR77(0,r1); \ 1080 stwcx. r0,0,r1; /* to clear the reservation */ \ 1081 lwz r11,_LINK(r1); \ 1082 mtlr r11; \ 1083 lwz r10,_CCR(r1); \ 1084 mtcrf 0xff,r10; \ 1085 PPC_40x_TURN_OFF_MSR_DR; \ 1086 lwz r9,_DEAR(r1); \ 1087 lwz r10,_ESR(r1); \ 1088 mtspr SPRN_DEAR,r9; \ 1089 mtspr SPRN_ESR,r10; \ 1090 lwz r11,_NIP(r1); \ 1091 lwz r12,_MSR(r1); \ 1092 mtspr exc_lvl_srr0,r11; \ 1093 mtspr exc_lvl_srr1,r12; \ 1094 lwz r9,GPR9(r1); \ 1095 lwz r12,GPR12(r1); \ 1096 lwz r10,GPR10(r1); \ 1097 lwz r11,GPR11(r1); \ 1098 lwz r1,GPR1(r1); \ 1099 PPC405_ERR77_SYNC; \ 1100 exc_lvl_rfi; \ 1101 b .; /* prevent prefetch past exc_lvl_rfi */ 1102 1103#define RESTORE_xSRR(exc_lvl_srr0, exc_lvl_srr1) \ 1104 lwz r9,_##exc_lvl_srr0(r1); \ 1105 lwz r10,_##exc_lvl_srr1(r1); \ 1106 mtspr SPRN_##exc_lvl_srr0,r9; \ 1107 mtspr SPRN_##exc_lvl_srr1,r10; 1108 1109#if defined(CONFIG_PPC_BOOK3E_MMU) 1110#ifdef CONFIG_PHYS_64BIT 1111#define RESTORE_MAS7 \ 1112 lwz r11,MAS7(r1); \ 1113 mtspr SPRN_MAS7,r11; 1114#else 1115#define RESTORE_MAS7 1116#endif /* CONFIG_PHYS_64BIT */ 1117#define RESTORE_MMU_REGS \ 1118 lwz r9,MAS0(r1); \ 1119 lwz r10,MAS1(r1); \ 1120 lwz r11,MAS2(r1); \ 1121 mtspr SPRN_MAS0,r9; \ 1122 lwz r9,MAS3(r1); \ 1123 mtspr SPRN_MAS1,r10; \ 1124 lwz r10,MAS6(r1); \ 1125 mtspr SPRN_MAS2,r11; \ 1126 mtspr SPRN_MAS3,r9; \ 1127 mtspr SPRN_MAS6,r10; \ 1128 RESTORE_MAS7; 1129#elif defined(CONFIG_44x) 1130#define RESTORE_MMU_REGS \ 1131 lwz r9,MMUCR(r1); \ 1132 mtspr SPRN_MMUCR,r9; 1133#else 1134#define RESTORE_MMU_REGS 1135#endif 1136 1137#ifdef CONFIG_40x 1138 .globl ret_from_crit_exc 1139ret_from_crit_exc: 1140 mfspr r9,SPRN_SPRG_THREAD 1141 lis r10,saved_ksp_limit@ha; 1142 lwz r10,saved_ksp_limit@l(r10); 1143 tovirt(r9,r9); 1144 stw r10,KSP_LIMIT(r9) 1145 lis r9,crit_srr0@ha; 1146 lwz r9,crit_srr0@l(r9); 1147 lis r10,crit_srr1@ha; 1148 lwz r10,crit_srr1@l(r10); 1149 mtspr SPRN_SRR0,r9; 1150 mtspr SPRN_SRR1,r10; 1151 RET_FROM_EXC_LEVEL(SPRN_CSRR0, SPRN_CSRR1, PPC_RFCI) 1152#endif /* CONFIG_40x */ 1153 1154#ifdef CONFIG_BOOKE 1155 .globl ret_from_crit_exc 1156ret_from_crit_exc: 1157 mfspr r9,SPRN_SPRG_THREAD 1158 lwz r10,SAVED_KSP_LIMIT(r1) 1159 stw r10,KSP_LIMIT(r9) 1160 RESTORE_xSRR(SRR0,SRR1); 1161 RESTORE_MMU_REGS; 1162 RET_FROM_EXC_LEVEL(SPRN_CSRR0, SPRN_CSRR1, PPC_RFCI) 1163 1164 .globl ret_from_debug_exc 1165ret_from_debug_exc: 1166 mfspr r9,SPRN_SPRG_THREAD 1167 lwz r10,SAVED_KSP_LIMIT(r1) 1168 stw r10,KSP_LIMIT(r9) 1169 RESTORE_xSRR(SRR0,SRR1); 1170 RESTORE_xSRR(CSRR0,CSRR1); 1171 RESTORE_MMU_REGS; 1172 RET_FROM_EXC_LEVEL(SPRN_DSRR0, SPRN_DSRR1, PPC_RFDI) 1173 1174 .globl ret_from_mcheck_exc 1175ret_from_mcheck_exc: 1176 mfspr r9,SPRN_SPRG_THREAD 1177 lwz r10,SAVED_KSP_LIMIT(r1) 1178 stw r10,KSP_LIMIT(r9) 1179 RESTORE_xSRR(SRR0,SRR1); 1180 RESTORE_xSRR(CSRR0,CSRR1); 1181 RESTORE_xSRR(DSRR0,DSRR1); 1182 RESTORE_MMU_REGS; 1183 RET_FROM_EXC_LEVEL(SPRN_MCSRR0, SPRN_MCSRR1, PPC_RFMCI) 1184#endif /* CONFIG_BOOKE */ 1185 1186/* 1187 * Load the DBCR0 value for a task that is being ptraced, 1188 * having first saved away the global DBCR0. Note that r0 1189 * has the dbcr0 value to set upon entry to this. 1190 */ 1191load_dbcr0: 1192 mfmsr r10 /* first disable debug exceptions */ 1193 rlwinm r10,r10,0,~MSR_DE 1194 mtmsr r10 1195 isync 1196 mfspr r10,SPRN_DBCR0 1197 lis r11,global_dbcr0@ha 1198 addi r11,r11,global_dbcr0@l 1199#ifdef CONFIG_SMP 1200 lwz r9,TASK_CPU(r2) 1201 slwi r9,r9,3 1202 add r11,r11,r9 1203#endif 1204 stw r10,0(r11) 1205 mtspr SPRN_DBCR0,r0 1206 lwz r10,4(r11) 1207 addi r10,r10,1 1208 stw r10,4(r11) 1209 li r11,-1 1210 mtspr SPRN_DBSR,r11 /* clear all pending debug events */ 1211 blr 1212 1213 .section .bss 1214 .align 4 1215 .global global_dbcr0 1216global_dbcr0: 1217 .space 8*NR_CPUS 1218 .previous 1219#endif /* !(CONFIG_4xx || CONFIG_BOOKE) */ 1220 1221do_work: /* r10 contains MSR_KERNEL here */ 1222 andi. r0,r9,_TIF_NEED_RESCHED 1223 beq do_user_signal 1224 1225do_resched: /* r10 contains MSR_KERNEL here */ 1226#ifdef CONFIG_TRACE_IRQFLAGS 1227 bl trace_hardirqs_on 1228 mfmsr r10 1229#endif 1230 ori r10,r10,MSR_EE 1231 SYNC 1232 MTMSRD(r10) /* hard-enable interrupts */ 1233 bl schedule 1234recheck: 1235 /* Note: And we don't tell it we are disabling them again 1236 * neither. Those disable/enable cycles used to peek at 1237 * TI_FLAGS aren't advertised. 1238 */ 1239 LOAD_MSR_KERNEL(r10,MSR_KERNEL) 1240 SYNC 1241 MTMSRD(r10) /* disable interrupts */ 1242 lwz r9,TI_FLAGS(r2) 1243 andi. r0,r9,_TIF_NEED_RESCHED 1244 bne- do_resched 1245 andi. r0,r9,_TIF_USER_WORK_MASK 1246 beq restore_user 1247do_user_signal: /* r10 contains MSR_KERNEL here */ 1248 ori r10,r10,MSR_EE 1249 SYNC 1250 MTMSRD(r10) /* hard-enable interrupts */ 1251 /* save r13-r31 in the exception frame, if not already done */ 1252 lwz r3,_TRAP(r1) 1253 andi. r0,r3,1 1254 beq 2f 1255 SAVE_NVGPRS(r1) 1256 rlwinm r3,r3,0,0,30 1257 stw r3,_TRAP(r1) 12582: addi r3,r1,STACK_FRAME_OVERHEAD 1259 mr r4,r9 1260 bl do_notify_resume 1261 REST_NVGPRS(r1) 1262 b recheck 1263 1264/* 1265 * We come here when we are at the end of handling an exception 1266 * that occurred at a place where taking an exception will lose 1267 * state information, such as the contents of SRR0 and SRR1. 1268 */ 1269nonrecoverable: 1270 lis r10,exc_exit_restart_end@ha 1271 addi r10,r10,exc_exit_restart_end@l 1272 cmplw r12,r10 1273 bge 3f 1274 lis r11,exc_exit_restart@ha 1275 addi r11,r11,exc_exit_restart@l 1276 cmplw r12,r11 1277 blt 3f 1278 lis r10,ee_restarts@ha 1279 lwz r12,ee_restarts@l(r10) 1280 addi r12,r12,1 1281 stw r12,ee_restarts@l(r10) 1282 mr r12,r11 /* restart at exc_exit_restart */ 1283 blr 12843: /* OK, we can't recover, kill this process */ 1285 /* but the 601 doesn't implement the RI bit, so assume it's OK */ 1286BEGIN_FTR_SECTION 1287 blr 1288END_FTR_SECTION_IFSET(CPU_FTR_601) 1289 lwz r3,_TRAP(r1) 1290 andi. r0,r3,1 1291 beq 5f 1292 SAVE_NVGPRS(r1) 1293 rlwinm r3,r3,0,0,30 1294 stw r3,_TRAP(r1) 12955: mfspr r2,SPRN_SPRG_THREAD 1296 addi r2,r2,-THREAD 1297 tovirt(r2,r2) /* set back r2 to current */ 12984: addi r3,r1,STACK_FRAME_OVERHEAD 1299 bl unrecoverable_exception 1300 /* shouldn't return */ 1301 b 4b 1302 1303 .section .bss 1304 .align 2 1305ee_restarts: 1306 .space 4 1307 .previous 1308 1309/* 1310 * PROM code for specific machines follows. Put it 1311 * here so it's easy to add arch-specific sections later. 1312 * -- Cort 1313 */ 1314#ifdef CONFIG_PPC_RTAS 1315/* 1316 * On CHRP, the Run-Time Abstraction Services (RTAS) have to be 1317 * called with the MMU off. 1318 */ 1319_GLOBAL(enter_rtas) 1320 stwu r1,-INT_FRAME_SIZE(r1) 1321 mflr r0 1322 stw r0,INT_FRAME_SIZE+4(r1) 1323 LOAD_REG_ADDR(r4, rtas) 1324 lis r6,1f@ha /* physical return address for rtas */ 1325 addi r6,r6,1f@l 1326 tophys(r6,r6) 1327 tophys(r7,r1) 1328 lwz r8,RTASENTRY(r4) 1329 lwz r4,RTASBASE(r4) 1330 mfmsr r9 1331 stw r9,8(r1) 1332 LOAD_MSR_KERNEL(r0,MSR_KERNEL) 1333 SYNC /* disable interrupts so SRR0/1 */ 1334 MTMSRD(r0) /* don't get trashed */ 1335 li r9,MSR_KERNEL & ~(MSR_IR|MSR_DR) 1336 mtlr r6 1337 stw r7, THREAD + RTAS_SP(r2) 1338 mtspr SPRN_SRR0,r8 1339 mtspr SPRN_SRR1,r9 1340 RFI 13411: tophys(r9,r1) 1342 lwz r8,INT_FRAME_SIZE+4(r9) /* get return address */ 1343 lwz r9,8(r9) /* original msr value */ 1344 addi r1,r1,INT_FRAME_SIZE 1345 li r0,0 1346 tophys(r7, r2) 1347 stw r0, THREAD + RTAS_SP(r7) 1348 mtspr SPRN_SRR0,r8 1349 mtspr SPRN_SRR1,r9 1350 RFI /* return to caller */ 1351 1352 .globl machine_check_in_rtas 1353machine_check_in_rtas: 1354 twi 31,0,0 1355 /* XXX load up BATs and panic */ 1356 1357#endif /* CONFIG_PPC_RTAS */ 1358