1 /* 2 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org) 3 * Copyright 2007-2010 Freescale Semiconductor, Inc. 4 * 5 * This program is free software; you can redistribute it and/or 6 * modify it under the terms of the GNU General Public License 7 * as published by the Free Software Foundation; either version 8 * 2 of the License, or (at your option) any later version. 9 * 10 * Modified by Cort Dougan (cort@cs.nmt.edu) 11 * and Paul Mackerras (paulus@samba.org) 12 */ 13 14 /* 15 * This file handles the architecture-dependent parts of hardware exceptions 16 */ 17 18 #include <linux/errno.h> 19 #include <linux/sched.h> 20 #include <linux/sched/debug.h> 21 #include <linux/kernel.h> 22 #include <linux/mm.h> 23 #include <linux/stddef.h> 24 #include <linux/unistd.h> 25 #include <linux/ptrace.h> 26 #include <linux/user.h> 27 #include <linux/interrupt.h> 28 #include <linux/init.h> 29 #include <linux/extable.h> 30 #include <linux/module.h> /* print_modules */ 31 #include <linux/prctl.h> 32 #include <linux/delay.h> 33 #include <linux/kprobes.h> 34 #include <linux/kexec.h> 35 #include <linux/backlight.h> 36 #include <linux/bug.h> 37 #include <linux/kdebug.h> 38 #include <linux/ratelimit.h> 39 #include <linux/context_tracking.h> 40 41 #include <asm/emulated_ops.h> 42 #include <asm/pgtable.h> 43 #include <linux/uaccess.h> 44 #include <asm/debugfs.h> 45 #include <asm/io.h> 46 #include <asm/machdep.h> 47 #include <asm/rtas.h> 48 #include <asm/pmc.h> 49 #include <asm/reg.h> 50 #ifdef CONFIG_PMAC_BACKLIGHT 51 #include <asm/backlight.h> 52 #endif 53 #ifdef CONFIG_PPC64 54 #include <asm/firmware.h> 55 #include <asm/processor.h> 56 #include <asm/tm.h> 57 #endif 58 #include <asm/kexec.h> 59 #include <asm/ppc-opcode.h> 60 #include <asm/rio.h> 61 #include <asm/fadump.h> 62 #include <asm/switch_to.h> 63 #include <asm/tm.h> 64 #include <asm/debug.h> 65 #include <asm/asm-prototypes.h> 66 #include <asm/hmi.h> 67 #include <sysdev/fsl_pci.h> 68 #include <asm/kprobes.h> 69 70 #if defined(CONFIG_DEBUGGER) || defined(CONFIG_KEXEC_CORE) 71 int (*__debugger)(struct pt_regs *regs) __read_mostly; 72 int (*__debugger_ipi)(struct pt_regs *regs) __read_mostly; 73 int (*__debugger_bpt)(struct pt_regs *regs) __read_mostly; 74 int (*__debugger_sstep)(struct pt_regs *regs) __read_mostly; 75 int (*__debugger_iabr_match)(struct pt_regs *regs) __read_mostly; 76 int (*__debugger_break_match)(struct pt_regs *regs) __read_mostly; 77 int (*__debugger_fault_handler)(struct pt_regs *regs) __read_mostly; 78 79 EXPORT_SYMBOL(__debugger); 80 EXPORT_SYMBOL(__debugger_ipi); 81 EXPORT_SYMBOL(__debugger_bpt); 82 EXPORT_SYMBOL(__debugger_sstep); 83 EXPORT_SYMBOL(__debugger_iabr_match); 84 EXPORT_SYMBOL(__debugger_break_match); 85 EXPORT_SYMBOL(__debugger_fault_handler); 86 #endif 87 88 /* Transactional Memory trap debug */ 89 #ifdef TM_DEBUG_SW 90 #define TM_DEBUG(x...) printk(KERN_INFO x) 91 #else 92 #define TM_DEBUG(x...) do { } while(0) 93 #endif 94 95 /* 96 * Trap & Exception support 97 */ 98 99 #ifdef CONFIG_PMAC_BACKLIGHT 100 static void pmac_backlight_unblank(void) 101 { 102 mutex_lock(&pmac_backlight_mutex); 103 if (pmac_backlight) { 104 struct backlight_properties *props; 105 106 props = &pmac_backlight->props; 107 props->brightness = props->max_brightness; 108 props->power = FB_BLANK_UNBLANK; 109 backlight_update_status(pmac_backlight); 110 } 111 mutex_unlock(&pmac_backlight_mutex); 112 } 113 #else 114 static inline void pmac_backlight_unblank(void) { } 115 #endif 116 117 /* 118 * If oops/die is expected to crash the machine, return true here. 119 * 120 * This should not be expected to be 100% accurate, there may be 121 * notifiers registered or other unexpected conditions that may bring 122 * down the kernel. Or if the current process in the kernel is holding 123 * locks or has other critical state, the kernel may become effectively 124 * unusable anyway. 125 */ 126 bool die_will_crash(void) 127 { 128 if (should_fadump_crash()) 129 return true; 130 if (kexec_should_crash(current)) 131 return true; 132 if (in_interrupt() || panic_on_oops || 133 !current->pid || is_global_init(current)) 134 return true; 135 136 return false; 137 } 138 139 static arch_spinlock_t die_lock = __ARCH_SPIN_LOCK_UNLOCKED; 140 static int die_owner = -1; 141 static unsigned int die_nest_count; 142 static int die_counter; 143 144 static unsigned long oops_begin(struct pt_regs *regs) 145 { 146 int cpu; 147 unsigned long flags; 148 149 oops_enter(); 150 151 /* racy, but better than risking deadlock. */ 152 raw_local_irq_save(flags); 153 cpu = smp_processor_id(); 154 if (!arch_spin_trylock(&die_lock)) { 155 if (cpu == die_owner) 156 /* nested oops. should stop eventually */; 157 else 158 arch_spin_lock(&die_lock); 159 } 160 die_nest_count++; 161 die_owner = cpu; 162 console_verbose(); 163 bust_spinlocks(1); 164 if (machine_is(powermac)) 165 pmac_backlight_unblank(); 166 return flags; 167 } 168 NOKPROBE_SYMBOL(oops_begin); 169 170 static void oops_end(unsigned long flags, struct pt_regs *regs, 171 int signr) 172 { 173 bust_spinlocks(0); 174 add_taint(TAINT_DIE, LOCKDEP_NOW_UNRELIABLE); 175 die_nest_count--; 176 oops_exit(); 177 printk("\n"); 178 if (!die_nest_count) { 179 /* Nest count reaches zero, release the lock. */ 180 die_owner = -1; 181 arch_spin_unlock(&die_lock); 182 } 183 raw_local_irq_restore(flags); 184 185 crash_fadump(regs, "die oops"); 186 187 if (kexec_should_crash(current)) 188 crash_kexec(regs); 189 190 if (!signr) 191 return; 192 193 /* 194 * While our oops output is serialised by a spinlock, output 195 * from panic() called below can race and corrupt it. If we 196 * know we are going to panic, delay for 1 second so we have a 197 * chance to get clean backtraces from all CPUs that are oopsing. 198 */ 199 if (in_interrupt() || panic_on_oops || !current->pid || 200 is_global_init(current)) { 201 mdelay(MSEC_PER_SEC); 202 } 203 204 if (in_interrupt()) 205 panic("Fatal exception in interrupt"); 206 if (panic_on_oops) 207 panic("Fatal exception"); 208 do_exit(signr); 209 } 210 NOKPROBE_SYMBOL(oops_end); 211 212 static int __die(const char *str, struct pt_regs *regs, long err) 213 { 214 printk("Oops: %s, sig: %ld [#%d]\n", str, err, ++die_counter); 215 216 if (IS_ENABLED(CONFIG_CPU_LITTLE_ENDIAN)) 217 printk("LE "); 218 else 219 printk("BE "); 220 221 if (IS_ENABLED(CONFIG_PREEMPT)) 222 pr_cont("PREEMPT "); 223 224 if (IS_ENABLED(CONFIG_SMP)) 225 pr_cont("SMP NR_CPUS=%d ", NR_CPUS); 226 227 if (debug_pagealloc_enabled()) 228 pr_cont("DEBUG_PAGEALLOC "); 229 230 if (IS_ENABLED(CONFIG_NUMA)) 231 pr_cont("NUMA "); 232 233 pr_cont("%s\n", ppc_md.name ? ppc_md.name : ""); 234 235 if (notify_die(DIE_OOPS, str, regs, err, 255, SIGSEGV) == NOTIFY_STOP) 236 return 1; 237 238 print_modules(); 239 show_regs(regs); 240 241 return 0; 242 } 243 NOKPROBE_SYMBOL(__die); 244 245 void die(const char *str, struct pt_regs *regs, long err) 246 { 247 unsigned long flags; 248 249 if (debugger(regs)) 250 return; 251 252 flags = oops_begin(regs); 253 if (__die(str, regs, err)) 254 err = 0; 255 oops_end(flags, regs, err); 256 } 257 NOKPROBE_SYMBOL(die); 258 259 void user_single_step_siginfo(struct task_struct *tsk, 260 struct pt_regs *regs, siginfo_t *info) 261 { 262 memset(info, 0, sizeof(*info)); 263 info->si_signo = SIGTRAP; 264 info->si_code = TRAP_TRACE; 265 info->si_addr = (void __user *)regs->nip; 266 } 267 268 void _exception(int signr, struct pt_regs *regs, int code, unsigned long addr) 269 { 270 siginfo_t info; 271 const char fmt32[] = KERN_INFO "%s[%d]: unhandled signal %d " \ 272 "at %08lx nip %08lx lr %08lx code %x\n"; 273 const char fmt64[] = KERN_INFO "%s[%d]: unhandled signal %d " \ 274 "at %016lx nip %016lx lr %016lx code %x\n"; 275 276 if (!user_mode(regs)) { 277 die("Exception in kernel mode", regs, signr); 278 return; 279 } 280 281 if (show_unhandled_signals && unhandled_signal(current, signr)) { 282 printk_ratelimited(regs->msr & MSR_64BIT ? fmt64 : fmt32, 283 current->comm, current->pid, signr, 284 addr, regs->nip, regs->link, code); 285 } 286 287 if (arch_irqs_disabled() && !arch_irq_disabled_regs(regs)) 288 local_irq_enable(); 289 290 current->thread.trap_nr = code; 291 memset(&info, 0, sizeof(info)); 292 info.si_signo = signr; 293 info.si_code = code; 294 info.si_addr = (void __user *) addr; 295 force_sig_info(signr, &info, current); 296 } 297 298 void system_reset_exception(struct pt_regs *regs) 299 { 300 /* 301 * Avoid crashes in case of nested NMI exceptions. Recoverability 302 * is determined by RI and in_nmi 303 */ 304 bool nested = in_nmi(); 305 if (!nested) 306 nmi_enter(); 307 308 __this_cpu_inc(irq_stat.sreset_irqs); 309 310 /* See if any machine dependent calls */ 311 if (ppc_md.system_reset_exception) { 312 if (ppc_md.system_reset_exception(regs)) 313 goto out; 314 } 315 316 if (debugger(regs)) 317 goto out; 318 319 /* 320 * A system reset is a request to dump, so we always send 321 * it through the crashdump code (if fadump or kdump are 322 * registered). 323 */ 324 crash_fadump(regs, "System Reset"); 325 326 crash_kexec(regs); 327 328 /* 329 * We aren't the primary crash CPU. We need to send it 330 * to a holding pattern to avoid it ending up in the panic 331 * code. 332 */ 333 crash_kexec_secondary(regs); 334 335 /* 336 * No debugger or crash dump registered, print logs then 337 * panic. 338 */ 339 __die("System Reset", regs, SIGABRT); 340 341 mdelay(2*MSEC_PER_SEC); /* Wait a little while for others to print */ 342 add_taint(TAINT_DIE, LOCKDEP_NOW_UNRELIABLE); 343 nmi_panic(regs, "System Reset"); 344 345 out: 346 #ifdef CONFIG_PPC_BOOK3S_64 347 BUG_ON(get_paca()->in_nmi == 0); 348 if (get_paca()->in_nmi > 1) 349 nmi_panic(regs, "Unrecoverable nested System Reset"); 350 #endif 351 /* Must die if the interrupt is not recoverable */ 352 if (!(regs->msr & MSR_RI)) 353 nmi_panic(regs, "Unrecoverable System Reset"); 354 355 if (!nested) 356 nmi_exit(); 357 358 /* What should we do here? We could issue a shutdown or hard reset. */ 359 } 360 361 /* 362 * I/O accesses can cause machine checks on powermacs. 363 * Check if the NIP corresponds to the address of a sync 364 * instruction for which there is an entry in the exception 365 * table. 366 * Note that the 601 only takes a machine check on TEA 367 * (transfer error ack) signal assertion, and does not 368 * set any of the top 16 bits of SRR1. 369 * -- paulus. 370 */ 371 static inline int check_io_access(struct pt_regs *regs) 372 { 373 #ifdef CONFIG_PPC32 374 unsigned long msr = regs->msr; 375 const struct exception_table_entry *entry; 376 unsigned int *nip = (unsigned int *)regs->nip; 377 378 if (((msr & 0xffff0000) == 0 || (msr & (0x80000 | 0x40000))) 379 && (entry = search_exception_tables(regs->nip)) != NULL) { 380 /* 381 * Check that it's a sync instruction, or somewhere 382 * in the twi; isync; nop sequence that inb/inw/inl uses. 383 * As the address is in the exception table 384 * we should be able to read the instr there. 385 * For the debug message, we look at the preceding 386 * load or store. 387 */ 388 if (*nip == PPC_INST_NOP) 389 nip -= 2; 390 else if (*nip == PPC_INST_ISYNC) 391 --nip; 392 if (*nip == PPC_INST_SYNC || (*nip >> 26) == OP_TRAP) { 393 unsigned int rb; 394 395 --nip; 396 rb = (*nip >> 11) & 0x1f; 397 printk(KERN_DEBUG "%s bad port %lx at %p\n", 398 (*nip & 0x100)? "OUT to": "IN from", 399 regs->gpr[rb] - _IO_BASE, nip); 400 regs->msr |= MSR_RI; 401 regs->nip = extable_fixup(entry); 402 return 1; 403 } 404 } 405 #endif /* CONFIG_PPC32 */ 406 return 0; 407 } 408 409 #ifdef CONFIG_PPC_ADV_DEBUG_REGS 410 /* On 4xx, the reason for the machine check or program exception 411 is in the ESR. */ 412 #define get_reason(regs) ((regs)->dsisr) 413 #define REASON_FP ESR_FP 414 #define REASON_ILLEGAL (ESR_PIL | ESR_PUO) 415 #define REASON_PRIVILEGED ESR_PPR 416 #define REASON_TRAP ESR_PTR 417 418 /* single-step stuff */ 419 #define single_stepping(regs) (current->thread.debug.dbcr0 & DBCR0_IC) 420 #define clear_single_step(regs) (current->thread.debug.dbcr0 &= ~DBCR0_IC) 421 422 #else 423 /* On non-4xx, the reason for the machine check or program 424 exception is in the MSR. */ 425 #define get_reason(regs) ((regs)->msr) 426 #define REASON_TM SRR1_PROGTM 427 #define REASON_FP SRR1_PROGFPE 428 #define REASON_ILLEGAL SRR1_PROGILL 429 #define REASON_PRIVILEGED SRR1_PROGPRIV 430 #define REASON_TRAP SRR1_PROGTRAP 431 432 #define single_stepping(regs) ((regs)->msr & MSR_SE) 433 #define clear_single_step(regs) ((regs)->msr &= ~MSR_SE) 434 #endif 435 436 #if defined(CONFIG_E500) 437 int machine_check_e500mc(struct pt_regs *regs) 438 { 439 unsigned long mcsr = mfspr(SPRN_MCSR); 440 unsigned long reason = mcsr; 441 int recoverable = 1; 442 443 if (reason & MCSR_LD) { 444 recoverable = fsl_rio_mcheck_exception(regs); 445 if (recoverable == 1) 446 goto silent_out; 447 } 448 449 printk("Machine check in kernel mode.\n"); 450 printk("Caused by (from MCSR=%lx): ", reason); 451 452 if (reason & MCSR_MCP) 453 printk("Machine Check Signal\n"); 454 455 if (reason & MCSR_ICPERR) { 456 printk("Instruction Cache Parity Error\n"); 457 458 /* 459 * This is recoverable by invalidating the i-cache. 460 */ 461 mtspr(SPRN_L1CSR1, mfspr(SPRN_L1CSR1) | L1CSR1_ICFI); 462 while (mfspr(SPRN_L1CSR1) & L1CSR1_ICFI) 463 ; 464 465 /* 466 * This will generally be accompanied by an instruction 467 * fetch error report -- only treat MCSR_IF as fatal 468 * if it wasn't due to an L1 parity error. 469 */ 470 reason &= ~MCSR_IF; 471 } 472 473 if (reason & MCSR_DCPERR_MC) { 474 printk("Data Cache Parity Error\n"); 475 476 /* 477 * In write shadow mode we auto-recover from the error, but it 478 * may still get logged and cause a machine check. We should 479 * only treat the non-write shadow case as non-recoverable. 480 */ 481 if (!(mfspr(SPRN_L1CSR2) & L1CSR2_DCWS)) 482 recoverable = 0; 483 } 484 485 if (reason & MCSR_L2MMU_MHIT) { 486 printk("Hit on multiple TLB entries\n"); 487 recoverable = 0; 488 } 489 490 if (reason & MCSR_NMI) 491 printk("Non-maskable interrupt\n"); 492 493 if (reason & MCSR_IF) { 494 printk("Instruction Fetch Error Report\n"); 495 recoverable = 0; 496 } 497 498 if (reason & MCSR_LD) { 499 printk("Load Error Report\n"); 500 recoverable = 0; 501 } 502 503 if (reason & MCSR_ST) { 504 printk("Store Error Report\n"); 505 recoverable = 0; 506 } 507 508 if (reason & MCSR_LDG) { 509 printk("Guarded Load Error Report\n"); 510 recoverable = 0; 511 } 512 513 if (reason & MCSR_TLBSYNC) 514 printk("Simultaneous tlbsync operations\n"); 515 516 if (reason & MCSR_BSL2_ERR) { 517 printk("Level 2 Cache Error\n"); 518 recoverable = 0; 519 } 520 521 if (reason & MCSR_MAV) { 522 u64 addr; 523 524 addr = mfspr(SPRN_MCAR); 525 addr |= (u64)mfspr(SPRN_MCARU) << 32; 526 527 printk("Machine Check %s Address: %#llx\n", 528 reason & MCSR_MEA ? "Effective" : "Physical", addr); 529 } 530 531 silent_out: 532 mtspr(SPRN_MCSR, mcsr); 533 return mfspr(SPRN_MCSR) == 0 && recoverable; 534 } 535 536 int machine_check_e500(struct pt_regs *regs) 537 { 538 unsigned long reason = mfspr(SPRN_MCSR); 539 540 if (reason & MCSR_BUS_RBERR) { 541 if (fsl_rio_mcheck_exception(regs)) 542 return 1; 543 if (fsl_pci_mcheck_exception(regs)) 544 return 1; 545 } 546 547 printk("Machine check in kernel mode.\n"); 548 printk("Caused by (from MCSR=%lx): ", reason); 549 550 if (reason & MCSR_MCP) 551 printk("Machine Check Signal\n"); 552 if (reason & MCSR_ICPERR) 553 printk("Instruction Cache Parity Error\n"); 554 if (reason & MCSR_DCP_PERR) 555 printk("Data Cache Push Parity Error\n"); 556 if (reason & MCSR_DCPERR) 557 printk("Data Cache Parity Error\n"); 558 if (reason & MCSR_BUS_IAERR) 559 printk("Bus - Instruction Address Error\n"); 560 if (reason & MCSR_BUS_RAERR) 561 printk("Bus - Read Address Error\n"); 562 if (reason & MCSR_BUS_WAERR) 563 printk("Bus - Write Address Error\n"); 564 if (reason & MCSR_BUS_IBERR) 565 printk("Bus - Instruction Data Error\n"); 566 if (reason & MCSR_BUS_RBERR) 567 printk("Bus - Read Data Bus Error\n"); 568 if (reason & MCSR_BUS_WBERR) 569 printk("Bus - Write Data Bus Error\n"); 570 if (reason & MCSR_BUS_IPERR) 571 printk("Bus - Instruction Parity Error\n"); 572 if (reason & MCSR_BUS_RPERR) 573 printk("Bus - Read Parity Error\n"); 574 575 return 0; 576 } 577 578 int machine_check_generic(struct pt_regs *regs) 579 { 580 return 0; 581 } 582 #elif defined(CONFIG_E200) 583 int machine_check_e200(struct pt_regs *regs) 584 { 585 unsigned long reason = mfspr(SPRN_MCSR); 586 587 printk("Machine check in kernel mode.\n"); 588 printk("Caused by (from MCSR=%lx): ", reason); 589 590 if (reason & MCSR_MCP) 591 printk("Machine Check Signal\n"); 592 if (reason & MCSR_CP_PERR) 593 printk("Cache Push Parity Error\n"); 594 if (reason & MCSR_CPERR) 595 printk("Cache Parity Error\n"); 596 if (reason & MCSR_EXCP_ERR) 597 printk("ISI, ITLB, or Bus Error on first instruction fetch for an exception handler\n"); 598 if (reason & MCSR_BUS_IRERR) 599 printk("Bus - Read Bus Error on instruction fetch\n"); 600 if (reason & MCSR_BUS_DRERR) 601 printk("Bus - Read Bus Error on data load\n"); 602 if (reason & MCSR_BUS_WRERR) 603 printk("Bus - Write Bus Error on buffered store or cache line push\n"); 604 605 return 0; 606 } 607 #elif defined(CONFIG_PPC32) 608 int machine_check_generic(struct pt_regs *regs) 609 { 610 unsigned long reason = regs->msr; 611 612 printk("Machine check in kernel mode.\n"); 613 printk("Caused by (from SRR1=%lx): ", reason); 614 switch (reason & 0x601F0000) { 615 case 0x80000: 616 printk("Machine check signal\n"); 617 break; 618 case 0: /* for 601 */ 619 case 0x40000: 620 case 0x140000: /* 7450 MSS error and TEA */ 621 printk("Transfer error ack signal\n"); 622 break; 623 case 0x20000: 624 printk("Data parity error signal\n"); 625 break; 626 case 0x10000: 627 printk("Address parity error signal\n"); 628 break; 629 case 0x20000000: 630 printk("L1 Data Cache error\n"); 631 break; 632 case 0x40000000: 633 printk("L1 Instruction Cache error\n"); 634 break; 635 case 0x00100000: 636 printk("L2 data cache parity error\n"); 637 break; 638 default: 639 printk("Unknown values in msr\n"); 640 } 641 return 0; 642 } 643 #endif /* everything else */ 644 645 void machine_check_exception(struct pt_regs *regs) 646 { 647 int recover = 0; 648 bool nested = in_nmi(); 649 if (!nested) 650 nmi_enter(); 651 652 /* 64s accounts the mce in machine_check_early when in HVMODE */ 653 if (!IS_ENABLED(CONFIG_PPC_BOOK3S_64) || !cpu_has_feature(CPU_FTR_HVMODE)) 654 __this_cpu_inc(irq_stat.mce_exceptions); 655 656 add_taint(TAINT_MACHINE_CHECK, LOCKDEP_NOW_UNRELIABLE); 657 658 /* See if any machine dependent calls. In theory, we would want 659 * to call the CPU first, and call the ppc_md. one if the CPU 660 * one returns a positive number. However there is existing code 661 * that assumes the board gets a first chance, so let's keep it 662 * that way for now and fix things later. --BenH. 663 */ 664 if (ppc_md.machine_check_exception) 665 recover = ppc_md.machine_check_exception(regs); 666 else if (cur_cpu_spec->machine_check) 667 recover = cur_cpu_spec->machine_check(regs); 668 669 if (recover > 0) 670 goto bail; 671 672 if (debugger_fault_handler(regs)) 673 goto bail; 674 675 if (check_io_access(regs)) 676 goto bail; 677 678 die("Machine check", regs, SIGBUS); 679 680 /* Must die if the interrupt is not recoverable */ 681 if (!(regs->msr & MSR_RI)) 682 nmi_panic(regs, "Unrecoverable Machine check"); 683 684 bail: 685 if (!nested) 686 nmi_exit(); 687 } 688 689 void SMIException(struct pt_regs *regs) 690 { 691 die("System Management Interrupt", regs, SIGABRT); 692 } 693 694 void handle_hmi_exception(struct pt_regs *regs) 695 { 696 struct pt_regs *old_regs; 697 698 old_regs = set_irq_regs(regs); 699 irq_enter(); 700 701 if (ppc_md.handle_hmi_exception) 702 ppc_md.handle_hmi_exception(regs); 703 704 irq_exit(); 705 set_irq_regs(old_regs); 706 } 707 708 void unknown_exception(struct pt_regs *regs) 709 { 710 enum ctx_state prev_state = exception_enter(); 711 712 printk("Bad trap at PC: %lx, SR: %lx, vector=%lx\n", 713 regs->nip, regs->msr, regs->trap); 714 715 _exception(SIGTRAP, regs, 0, 0); 716 717 exception_exit(prev_state); 718 } 719 720 void instruction_breakpoint_exception(struct pt_regs *regs) 721 { 722 enum ctx_state prev_state = exception_enter(); 723 724 if (notify_die(DIE_IABR_MATCH, "iabr_match", regs, 5, 725 5, SIGTRAP) == NOTIFY_STOP) 726 goto bail; 727 if (debugger_iabr_match(regs)) 728 goto bail; 729 _exception(SIGTRAP, regs, TRAP_BRKPT, regs->nip); 730 731 bail: 732 exception_exit(prev_state); 733 } 734 735 void RunModeException(struct pt_regs *regs) 736 { 737 _exception(SIGTRAP, regs, 0, 0); 738 } 739 740 void single_step_exception(struct pt_regs *regs) 741 { 742 enum ctx_state prev_state = exception_enter(); 743 744 clear_single_step(regs); 745 746 if (kprobe_post_handler(regs)) 747 return; 748 749 if (notify_die(DIE_SSTEP, "single_step", regs, 5, 750 5, SIGTRAP) == NOTIFY_STOP) 751 goto bail; 752 if (debugger_sstep(regs)) 753 goto bail; 754 755 _exception(SIGTRAP, regs, TRAP_TRACE, regs->nip); 756 757 bail: 758 exception_exit(prev_state); 759 } 760 NOKPROBE_SYMBOL(single_step_exception); 761 762 /* 763 * After we have successfully emulated an instruction, we have to 764 * check if the instruction was being single-stepped, and if so, 765 * pretend we got a single-step exception. This was pointed out 766 * by Kumar Gala. -- paulus 767 */ 768 static void emulate_single_step(struct pt_regs *regs) 769 { 770 if (single_stepping(regs)) 771 single_step_exception(regs); 772 } 773 774 static inline int __parse_fpscr(unsigned long fpscr) 775 { 776 int ret = 0; 777 778 /* Invalid operation */ 779 if ((fpscr & FPSCR_VE) && (fpscr & FPSCR_VX)) 780 ret = FPE_FLTINV; 781 782 /* Overflow */ 783 else if ((fpscr & FPSCR_OE) && (fpscr & FPSCR_OX)) 784 ret = FPE_FLTOVF; 785 786 /* Underflow */ 787 else if ((fpscr & FPSCR_UE) && (fpscr & FPSCR_UX)) 788 ret = FPE_FLTUND; 789 790 /* Divide by zero */ 791 else if ((fpscr & FPSCR_ZE) && (fpscr & FPSCR_ZX)) 792 ret = FPE_FLTDIV; 793 794 /* Inexact result */ 795 else if ((fpscr & FPSCR_XE) && (fpscr & FPSCR_XX)) 796 ret = FPE_FLTRES; 797 798 return ret; 799 } 800 801 static void parse_fpe(struct pt_regs *regs) 802 { 803 int code = 0; 804 805 flush_fp_to_thread(current); 806 807 code = __parse_fpscr(current->thread.fp_state.fpscr); 808 809 _exception(SIGFPE, regs, code, regs->nip); 810 } 811 812 /* 813 * Illegal instruction emulation support. Originally written to 814 * provide the PVR to user applications using the mfspr rd, PVR. 815 * Return non-zero if we can't emulate, or -EFAULT if the associated 816 * memory access caused an access fault. Return zero on success. 817 * 818 * There are a couple of ways to do this, either "decode" the instruction 819 * or directly match lots of bits. In this case, matching lots of 820 * bits is faster and easier. 821 * 822 */ 823 static int emulate_string_inst(struct pt_regs *regs, u32 instword) 824 { 825 u8 rT = (instword >> 21) & 0x1f; 826 u8 rA = (instword >> 16) & 0x1f; 827 u8 NB_RB = (instword >> 11) & 0x1f; 828 u32 num_bytes; 829 unsigned long EA; 830 int pos = 0; 831 832 /* Early out if we are an invalid form of lswx */ 833 if ((instword & PPC_INST_STRING_MASK) == PPC_INST_LSWX) 834 if ((rT == rA) || (rT == NB_RB)) 835 return -EINVAL; 836 837 EA = (rA == 0) ? 0 : regs->gpr[rA]; 838 839 switch (instword & PPC_INST_STRING_MASK) { 840 case PPC_INST_LSWX: 841 case PPC_INST_STSWX: 842 EA += NB_RB; 843 num_bytes = regs->xer & 0x7f; 844 break; 845 case PPC_INST_LSWI: 846 case PPC_INST_STSWI: 847 num_bytes = (NB_RB == 0) ? 32 : NB_RB; 848 break; 849 default: 850 return -EINVAL; 851 } 852 853 while (num_bytes != 0) 854 { 855 u8 val; 856 u32 shift = 8 * (3 - (pos & 0x3)); 857 858 /* if process is 32-bit, clear upper 32 bits of EA */ 859 if ((regs->msr & MSR_64BIT) == 0) 860 EA &= 0xFFFFFFFF; 861 862 switch ((instword & PPC_INST_STRING_MASK)) { 863 case PPC_INST_LSWX: 864 case PPC_INST_LSWI: 865 if (get_user(val, (u8 __user *)EA)) 866 return -EFAULT; 867 /* first time updating this reg, 868 * zero it out */ 869 if (pos == 0) 870 regs->gpr[rT] = 0; 871 regs->gpr[rT] |= val << shift; 872 break; 873 case PPC_INST_STSWI: 874 case PPC_INST_STSWX: 875 val = regs->gpr[rT] >> shift; 876 if (put_user(val, (u8 __user *)EA)) 877 return -EFAULT; 878 break; 879 } 880 /* move EA to next address */ 881 EA += 1; 882 num_bytes--; 883 884 /* manage our position within the register */ 885 if (++pos == 4) { 886 pos = 0; 887 if (++rT == 32) 888 rT = 0; 889 } 890 } 891 892 return 0; 893 } 894 895 static int emulate_popcntb_inst(struct pt_regs *regs, u32 instword) 896 { 897 u32 ra,rs; 898 unsigned long tmp; 899 900 ra = (instword >> 16) & 0x1f; 901 rs = (instword >> 21) & 0x1f; 902 903 tmp = regs->gpr[rs]; 904 tmp = tmp - ((tmp >> 1) & 0x5555555555555555ULL); 905 tmp = (tmp & 0x3333333333333333ULL) + ((tmp >> 2) & 0x3333333333333333ULL); 906 tmp = (tmp + (tmp >> 4)) & 0x0f0f0f0f0f0f0f0fULL; 907 regs->gpr[ra] = tmp; 908 909 return 0; 910 } 911 912 static int emulate_isel(struct pt_regs *regs, u32 instword) 913 { 914 u8 rT = (instword >> 21) & 0x1f; 915 u8 rA = (instword >> 16) & 0x1f; 916 u8 rB = (instword >> 11) & 0x1f; 917 u8 BC = (instword >> 6) & 0x1f; 918 u8 bit; 919 unsigned long tmp; 920 921 tmp = (rA == 0) ? 0 : regs->gpr[rA]; 922 bit = (regs->ccr >> (31 - BC)) & 0x1; 923 924 regs->gpr[rT] = bit ? tmp : regs->gpr[rB]; 925 926 return 0; 927 } 928 929 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM 930 static inline bool tm_abort_check(struct pt_regs *regs, int cause) 931 { 932 /* If we're emulating a load/store in an active transaction, we cannot 933 * emulate it as the kernel operates in transaction suspended context. 934 * We need to abort the transaction. This creates a persistent TM 935 * abort so tell the user what caused it with a new code. 936 */ 937 if (MSR_TM_TRANSACTIONAL(regs->msr)) { 938 tm_enable(); 939 tm_abort(cause); 940 return true; 941 } 942 return false; 943 } 944 #else 945 static inline bool tm_abort_check(struct pt_regs *regs, int reason) 946 { 947 return false; 948 } 949 #endif 950 951 static int emulate_instruction(struct pt_regs *regs) 952 { 953 u32 instword; 954 u32 rd; 955 956 if (!user_mode(regs)) 957 return -EINVAL; 958 CHECK_FULL_REGS(regs); 959 960 if (get_user(instword, (u32 __user *)(regs->nip))) 961 return -EFAULT; 962 963 /* Emulate the mfspr rD, PVR. */ 964 if ((instword & PPC_INST_MFSPR_PVR_MASK) == PPC_INST_MFSPR_PVR) { 965 PPC_WARN_EMULATED(mfpvr, regs); 966 rd = (instword >> 21) & 0x1f; 967 regs->gpr[rd] = mfspr(SPRN_PVR); 968 return 0; 969 } 970 971 /* Emulating the dcba insn is just a no-op. */ 972 if ((instword & PPC_INST_DCBA_MASK) == PPC_INST_DCBA) { 973 PPC_WARN_EMULATED(dcba, regs); 974 return 0; 975 } 976 977 /* Emulate the mcrxr insn. */ 978 if ((instword & PPC_INST_MCRXR_MASK) == PPC_INST_MCRXR) { 979 int shift = (instword >> 21) & 0x1c; 980 unsigned long msk = 0xf0000000UL >> shift; 981 982 PPC_WARN_EMULATED(mcrxr, regs); 983 regs->ccr = (regs->ccr & ~msk) | ((regs->xer >> shift) & msk); 984 regs->xer &= ~0xf0000000UL; 985 return 0; 986 } 987 988 /* Emulate load/store string insn. */ 989 if ((instword & PPC_INST_STRING_GEN_MASK) == PPC_INST_STRING) { 990 if (tm_abort_check(regs, 991 TM_CAUSE_EMULATE | TM_CAUSE_PERSISTENT)) 992 return -EINVAL; 993 PPC_WARN_EMULATED(string, regs); 994 return emulate_string_inst(regs, instword); 995 } 996 997 /* Emulate the popcntb (Population Count Bytes) instruction. */ 998 if ((instword & PPC_INST_POPCNTB_MASK) == PPC_INST_POPCNTB) { 999 PPC_WARN_EMULATED(popcntb, regs); 1000 return emulate_popcntb_inst(regs, instword); 1001 } 1002 1003 /* Emulate isel (Integer Select) instruction */ 1004 if ((instword & PPC_INST_ISEL_MASK) == PPC_INST_ISEL) { 1005 PPC_WARN_EMULATED(isel, regs); 1006 return emulate_isel(regs, instword); 1007 } 1008 1009 /* Emulate sync instruction variants */ 1010 if ((instword & PPC_INST_SYNC_MASK) == PPC_INST_SYNC) { 1011 PPC_WARN_EMULATED(sync, regs); 1012 asm volatile("sync"); 1013 return 0; 1014 } 1015 1016 #ifdef CONFIG_PPC64 1017 /* Emulate the mfspr rD, DSCR. */ 1018 if ((((instword & PPC_INST_MFSPR_DSCR_USER_MASK) == 1019 PPC_INST_MFSPR_DSCR_USER) || 1020 ((instword & PPC_INST_MFSPR_DSCR_MASK) == 1021 PPC_INST_MFSPR_DSCR)) && 1022 cpu_has_feature(CPU_FTR_DSCR)) { 1023 PPC_WARN_EMULATED(mfdscr, regs); 1024 rd = (instword >> 21) & 0x1f; 1025 regs->gpr[rd] = mfspr(SPRN_DSCR); 1026 return 0; 1027 } 1028 /* Emulate the mtspr DSCR, rD. */ 1029 if ((((instword & PPC_INST_MTSPR_DSCR_USER_MASK) == 1030 PPC_INST_MTSPR_DSCR_USER) || 1031 ((instword & PPC_INST_MTSPR_DSCR_MASK) == 1032 PPC_INST_MTSPR_DSCR)) && 1033 cpu_has_feature(CPU_FTR_DSCR)) { 1034 PPC_WARN_EMULATED(mtdscr, regs); 1035 rd = (instword >> 21) & 0x1f; 1036 current->thread.dscr = regs->gpr[rd]; 1037 current->thread.dscr_inherit = 1; 1038 mtspr(SPRN_DSCR, current->thread.dscr); 1039 return 0; 1040 } 1041 #endif 1042 1043 return -EINVAL; 1044 } 1045 1046 int is_valid_bugaddr(unsigned long addr) 1047 { 1048 return is_kernel_addr(addr); 1049 } 1050 1051 #ifdef CONFIG_MATH_EMULATION 1052 static int emulate_math(struct pt_regs *regs) 1053 { 1054 int ret; 1055 extern int do_mathemu(struct pt_regs *regs); 1056 1057 ret = do_mathemu(regs); 1058 if (ret >= 0) 1059 PPC_WARN_EMULATED(math, regs); 1060 1061 switch (ret) { 1062 case 0: 1063 emulate_single_step(regs); 1064 return 0; 1065 case 1: { 1066 int code = 0; 1067 code = __parse_fpscr(current->thread.fp_state.fpscr); 1068 _exception(SIGFPE, regs, code, regs->nip); 1069 return 0; 1070 } 1071 case -EFAULT: 1072 _exception(SIGSEGV, regs, SEGV_MAPERR, regs->nip); 1073 return 0; 1074 } 1075 1076 return -1; 1077 } 1078 #else 1079 static inline int emulate_math(struct pt_regs *regs) { return -1; } 1080 #endif 1081 1082 void program_check_exception(struct pt_regs *regs) 1083 { 1084 enum ctx_state prev_state = exception_enter(); 1085 unsigned int reason = get_reason(regs); 1086 1087 /* We can now get here via a FP Unavailable exception if the core 1088 * has no FPU, in that case the reason flags will be 0 */ 1089 1090 if (reason & REASON_FP) { 1091 /* IEEE FP exception */ 1092 parse_fpe(regs); 1093 goto bail; 1094 } 1095 if (reason & REASON_TRAP) { 1096 unsigned long bugaddr; 1097 /* Debugger is first in line to stop recursive faults in 1098 * rcu_lock, notify_die, or atomic_notifier_call_chain */ 1099 if (debugger_bpt(regs)) 1100 goto bail; 1101 1102 if (kprobe_handler(regs)) 1103 goto bail; 1104 1105 /* trap exception */ 1106 if (notify_die(DIE_BPT, "breakpoint", regs, 5, 5, SIGTRAP) 1107 == NOTIFY_STOP) 1108 goto bail; 1109 1110 bugaddr = regs->nip; 1111 /* 1112 * Fixup bugaddr for BUG_ON() in real mode 1113 */ 1114 if (!is_kernel_addr(bugaddr) && !(regs->msr & MSR_IR)) 1115 bugaddr += PAGE_OFFSET; 1116 1117 if (!(regs->msr & MSR_PR) && /* not user-mode */ 1118 report_bug(bugaddr, regs) == BUG_TRAP_TYPE_WARN) { 1119 regs->nip += 4; 1120 goto bail; 1121 } 1122 _exception(SIGTRAP, regs, TRAP_BRKPT, regs->nip); 1123 goto bail; 1124 } 1125 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM 1126 if (reason & REASON_TM) { 1127 /* This is a TM "Bad Thing Exception" program check. 1128 * This occurs when: 1129 * - An rfid/hrfid/mtmsrd attempts to cause an illegal 1130 * transition in TM states. 1131 * - A trechkpt is attempted when transactional. 1132 * - A treclaim is attempted when non transactional. 1133 * - A tend is illegally attempted. 1134 * - writing a TM SPR when transactional. 1135 */ 1136 if (!user_mode(regs) && 1137 report_bug(regs->nip, regs) == BUG_TRAP_TYPE_WARN) { 1138 regs->nip += 4; 1139 goto bail; 1140 } 1141 /* If usermode caused this, it's done something illegal and 1142 * gets a SIGILL slap on the wrist. We call it an illegal 1143 * operand to distinguish from the instruction just being bad 1144 * (e.g. executing a 'tend' on a CPU without TM!); it's an 1145 * illegal /placement/ of a valid instruction. 1146 */ 1147 if (user_mode(regs)) { 1148 _exception(SIGILL, regs, ILL_ILLOPN, regs->nip); 1149 goto bail; 1150 } else { 1151 printk(KERN_EMERG "Unexpected TM Bad Thing exception " 1152 "at %lx (msr 0x%x)\n", regs->nip, reason); 1153 die("Unrecoverable exception", regs, SIGABRT); 1154 } 1155 } 1156 #endif 1157 1158 /* 1159 * If we took the program check in the kernel skip down to sending a 1160 * SIGILL. The subsequent cases all relate to emulating instructions 1161 * which we should only do for userspace. We also do not want to enable 1162 * interrupts for kernel faults because that might lead to further 1163 * faults, and loose the context of the original exception. 1164 */ 1165 if (!user_mode(regs)) 1166 goto sigill; 1167 1168 /* We restore the interrupt state now */ 1169 if (!arch_irq_disabled_regs(regs)) 1170 local_irq_enable(); 1171 1172 /* (reason & REASON_ILLEGAL) would be the obvious thing here, 1173 * but there seems to be a hardware bug on the 405GP (RevD) 1174 * that means ESR is sometimes set incorrectly - either to 1175 * ESR_DST (!?) or 0. In the process of chasing this with the 1176 * hardware people - not sure if it can happen on any illegal 1177 * instruction or only on FP instructions, whether there is a 1178 * pattern to occurrences etc. -dgibson 31/Mar/2003 1179 */ 1180 if (!emulate_math(regs)) 1181 goto bail; 1182 1183 /* Try to emulate it if we should. */ 1184 if (reason & (REASON_ILLEGAL | REASON_PRIVILEGED)) { 1185 switch (emulate_instruction(regs)) { 1186 case 0: 1187 regs->nip += 4; 1188 emulate_single_step(regs); 1189 goto bail; 1190 case -EFAULT: 1191 _exception(SIGSEGV, regs, SEGV_MAPERR, regs->nip); 1192 goto bail; 1193 } 1194 } 1195 1196 sigill: 1197 if (reason & REASON_PRIVILEGED) 1198 _exception(SIGILL, regs, ILL_PRVOPC, regs->nip); 1199 else 1200 _exception(SIGILL, regs, ILL_ILLOPC, regs->nip); 1201 1202 bail: 1203 exception_exit(prev_state); 1204 } 1205 NOKPROBE_SYMBOL(program_check_exception); 1206 1207 /* 1208 * This occurs when running in hypervisor mode on POWER6 or later 1209 * and an illegal instruction is encountered. 1210 */ 1211 void emulation_assist_interrupt(struct pt_regs *regs) 1212 { 1213 regs->msr |= REASON_ILLEGAL; 1214 program_check_exception(regs); 1215 } 1216 NOKPROBE_SYMBOL(emulation_assist_interrupt); 1217 1218 void alignment_exception(struct pt_regs *regs) 1219 { 1220 enum ctx_state prev_state = exception_enter(); 1221 int sig, code, fixed = 0; 1222 1223 /* We restore the interrupt state now */ 1224 if (!arch_irq_disabled_regs(regs)) 1225 local_irq_enable(); 1226 1227 if (tm_abort_check(regs, TM_CAUSE_ALIGNMENT | TM_CAUSE_PERSISTENT)) 1228 goto bail; 1229 1230 /* we don't implement logging of alignment exceptions */ 1231 if (!(current->thread.align_ctl & PR_UNALIGN_SIGBUS)) 1232 fixed = fix_alignment(regs); 1233 1234 if (fixed == 1) { 1235 regs->nip += 4; /* skip over emulated instruction */ 1236 emulate_single_step(regs); 1237 goto bail; 1238 } 1239 1240 /* Operand address was bad */ 1241 if (fixed == -EFAULT) { 1242 sig = SIGSEGV; 1243 code = SEGV_ACCERR; 1244 } else { 1245 sig = SIGBUS; 1246 code = BUS_ADRALN; 1247 } 1248 if (user_mode(regs)) 1249 _exception(sig, regs, code, regs->dar); 1250 else 1251 bad_page_fault(regs, regs->dar, sig); 1252 1253 bail: 1254 exception_exit(prev_state); 1255 } 1256 1257 void slb_miss_bad_addr(struct pt_regs *regs) 1258 { 1259 enum ctx_state prev_state = exception_enter(); 1260 1261 if (user_mode(regs)) 1262 _exception(SIGSEGV, regs, SEGV_BNDERR, regs->dar); 1263 else 1264 bad_page_fault(regs, regs->dar, SIGSEGV); 1265 1266 exception_exit(prev_state); 1267 } 1268 1269 void StackOverflow(struct pt_regs *regs) 1270 { 1271 printk(KERN_CRIT "Kernel stack overflow in process %p, r1=%lx\n", 1272 current, regs->gpr[1]); 1273 debugger(regs); 1274 show_regs(regs); 1275 panic("kernel stack overflow"); 1276 } 1277 1278 void nonrecoverable_exception(struct pt_regs *regs) 1279 { 1280 printk(KERN_ERR "Non-recoverable exception at PC=%lx MSR=%lx\n", 1281 regs->nip, regs->msr); 1282 debugger(regs); 1283 die("nonrecoverable exception", regs, SIGKILL); 1284 } 1285 1286 void kernel_fp_unavailable_exception(struct pt_regs *regs) 1287 { 1288 enum ctx_state prev_state = exception_enter(); 1289 1290 printk(KERN_EMERG "Unrecoverable FP Unavailable Exception " 1291 "%lx at %lx\n", regs->trap, regs->nip); 1292 die("Unrecoverable FP Unavailable Exception", regs, SIGABRT); 1293 1294 exception_exit(prev_state); 1295 } 1296 1297 void altivec_unavailable_exception(struct pt_regs *regs) 1298 { 1299 enum ctx_state prev_state = exception_enter(); 1300 1301 if (user_mode(regs)) { 1302 /* A user program has executed an altivec instruction, 1303 but this kernel doesn't support altivec. */ 1304 _exception(SIGILL, regs, ILL_ILLOPC, regs->nip); 1305 goto bail; 1306 } 1307 1308 printk(KERN_EMERG "Unrecoverable VMX/Altivec Unavailable Exception " 1309 "%lx at %lx\n", regs->trap, regs->nip); 1310 die("Unrecoverable VMX/Altivec Unavailable Exception", regs, SIGABRT); 1311 1312 bail: 1313 exception_exit(prev_state); 1314 } 1315 1316 void vsx_unavailable_exception(struct pt_regs *regs) 1317 { 1318 if (user_mode(regs)) { 1319 /* A user program has executed an vsx instruction, 1320 but this kernel doesn't support vsx. */ 1321 _exception(SIGILL, regs, ILL_ILLOPC, regs->nip); 1322 return; 1323 } 1324 1325 printk(KERN_EMERG "Unrecoverable VSX Unavailable Exception " 1326 "%lx at %lx\n", regs->trap, regs->nip); 1327 die("Unrecoverable VSX Unavailable Exception", regs, SIGABRT); 1328 } 1329 1330 #ifdef CONFIG_PPC64 1331 static void tm_unavailable(struct pt_regs *regs) 1332 { 1333 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM 1334 if (user_mode(regs)) { 1335 current->thread.load_tm++; 1336 regs->msr |= MSR_TM; 1337 tm_enable(); 1338 tm_restore_sprs(¤t->thread); 1339 return; 1340 } 1341 #endif 1342 pr_emerg("Unrecoverable TM Unavailable Exception " 1343 "%lx at %lx\n", regs->trap, regs->nip); 1344 die("Unrecoverable TM Unavailable Exception", regs, SIGABRT); 1345 } 1346 1347 void facility_unavailable_exception(struct pt_regs *regs) 1348 { 1349 static char *facility_strings[] = { 1350 [FSCR_FP_LG] = "FPU", 1351 [FSCR_VECVSX_LG] = "VMX/VSX", 1352 [FSCR_DSCR_LG] = "DSCR", 1353 [FSCR_PM_LG] = "PMU SPRs", 1354 [FSCR_BHRB_LG] = "BHRB", 1355 [FSCR_TM_LG] = "TM", 1356 [FSCR_EBB_LG] = "EBB", 1357 [FSCR_TAR_LG] = "TAR", 1358 [FSCR_MSGP_LG] = "MSGP", 1359 [FSCR_SCV_LG] = "SCV", 1360 }; 1361 char *facility = "unknown"; 1362 u64 value; 1363 u32 instword, rd; 1364 u8 status; 1365 bool hv; 1366 1367 hv = (regs->trap == 0xf80); 1368 if (hv) 1369 value = mfspr(SPRN_HFSCR); 1370 else 1371 value = mfspr(SPRN_FSCR); 1372 1373 status = value >> 56; 1374 if (status == FSCR_DSCR_LG) { 1375 /* 1376 * User is accessing the DSCR register using the problem 1377 * state only SPR number (0x03) either through a mfspr or 1378 * a mtspr instruction. If it is a write attempt through 1379 * a mtspr, then we set the inherit bit. This also allows 1380 * the user to write or read the register directly in the 1381 * future by setting via the FSCR DSCR bit. But in case it 1382 * is a read DSCR attempt through a mfspr instruction, we 1383 * just emulate the instruction instead. This code path will 1384 * always emulate all the mfspr instructions till the user 1385 * has attempted at least one mtspr instruction. This way it 1386 * preserves the same behaviour when the user is accessing 1387 * the DSCR through privilege level only SPR number (0x11) 1388 * which is emulated through illegal instruction exception. 1389 * We always leave HFSCR DSCR set. 1390 */ 1391 if (get_user(instword, (u32 __user *)(regs->nip))) { 1392 pr_err("Failed to fetch the user instruction\n"); 1393 return; 1394 } 1395 1396 /* Write into DSCR (mtspr 0x03, RS) */ 1397 if ((instword & PPC_INST_MTSPR_DSCR_USER_MASK) 1398 == PPC_INST_MTSPR_DSCR_USER) { 1399 rd = (instword >> 21) & 0x1f; 1400 current->thread.dscr = regs->gpr[rd]; 1401 current->thread.dscr_inherit = 1; 1402 current->thread.fscr |= FSCR_DSCR; 1403 mtspr(SPRN_FSCR, current->thread.fscr); 1404 } 1405 1406 /* Read from DSCR (mfspr RT, 0x03) */ 1407 if ((instword & PPC_INST_MFSPR_DSCR_USER_MASK) 1408 == PPC_INST_MFSPR_DSCR_USER) { 1409 if (emulate_instruction(regs)) { 1410 pr_err("DSCR based mfspr emulation failed\n"); 1411 return; 1412 } 1413 regs->nip += 4; 1414 emulate_single_step(regs); 1415 } 1416 return; 1417 } 1418 1419 if (status == FSCR_TM_LG) { 1420 /* 1421 * If we're here then the hardware is TM aware because it 1422 * generated an exception with FSRM_TM set. 1423 * 1424 * If cpu_has_feature(CPU_FTR_TM) is false, then either firmware 1425 * told us not to do TM, or the kernel is not built with TM 1426 * support. 1427 * 1428 * If both of those things are true, then userspace can spam the 1429 * console by triggering the printk() below just by continually 1430 * doing tbegin (or any TM instruction). So in that case just 1431 * send the process a SIGILL immediately. 1432 */ 1433 if (!cpu_has_feature(CPU_FTR_TM)) 1434 goto out; 1435 1436 tm_unavailable(regs); 1437 return; 1438 } 1439 1440 if ((hv || status >= 2) && 1441 (status < ARRAY_SIZE(facility_strings)) && 1442 facility_strings[status]) 1443 facility = facility_strings[status]; 1444 1445 /* We restore the interrupt state now */ 1446 if (!arch_irq_disabled_regs(regs)) 1447 local_irq_enable(); 1448 1449 pr_err_ratelimited("%sFacility '%s' unavailable (%d), exception at 0x%lx, MSR=%lx\n", 1450 hv ? "Hypervisor " : "", facility, status, regs->nip, regs->msr); 1451 1452 out: 1453 if (user_mode(regs)) { 1454 _exception(SIGILL, regs, ILL_ILLOPC, regs->nip); 1455 return; 1456 } 1457 1458 die("Unexpected facility unavailable exception", regs, SIGABRT); 1459 } 1460 #endif 1461 1462 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM 1463 1464 void fp_unavailable_tm(struct pt_regs *regs) 1465 { 1466 /* Note: This does not handle any kind of FP laziness. */ 1467 1468 TM_DEBUG("FP Unavailable trap whilst transactional at 0x%lx, MSR=%lx\n", 1469 regs->nip, regs->msr); 1470 1471 /* We can only have got here if the task started using FP after 1472 * beginning the transaction. So, the transactional regs are just a 1473 * copy of the checkpointed ones. But, we still need to recheckpoint 1474 * as we're enabling FP for the process; it will return, abort the 1475 * transaction, and probably retry but now with FP enabled. So the 1476 * checkpointed FP registers need to be loaded. 1477 */ 1478 tm_reclaim_current(TM_CAUSE_FAC_UNAV); 1479 /* Reclaim didn't save out any FPRs to transact_fprs. */ 1480 1481 /* Enable FP for the task: */ 1482 regs->msr |= (MSR_FP | current->thread.fpexc_mode); 1483 1484 /* This loads and recheckpoints the FP registers from 1485 * thread.fpr[]. They will remain in registers after the 1486 * checkpoint so we don't need to reload them after. 1487 * If VMX is in use, the VRs now hold checkpointed values, 1488 * so we don't want to load the VRs from the thread_struct. 1489 */ 1490 tm_recheckpoint(¤t->thread, MSR_FP); 1491 1492 /* If VMX is in use, get the transactional values back */ 1493 if (regs->msr & MSR_VEC) { 1494 msr_check_and_set(MSR_VEC); 1495 load_vr_state(¤t->thread.vr_state); 1496 /* At this point all the VSX state is loaded, so enable it */ 1497 regs->msr |= MSR_VSX; 1498 } 1499 } 1500 1501 void altivec_unavailable_tm(struct pt_regs *regs) 1502 { 1503 /* See the comments in fp_unavailable_tm(). This function operates 1504 * the same way. 1505 */ 1506 1507 TM_DEBUG("Vector Unavailable trap whilst transactional at 0x%lx," 1508 "MSR=%lx\n", 1509 regs->nip, regs->msr); 1510 tm_reclaim_current(TM_CAUSE_FAC_UNAV); 1511 regs->msr |= MSR_VEC; 1512 tm_recheckpoint(¤t->thread, MSR_VEC); 1513 current->thread.used_vr = 1; 1514 1515 if (regs->msr & MSR_FP) { 1516 msr_check_and_set(MSR_FP); 1517 load_fp_state(¤t->thread.fp_state); 1518 regs->msr |= MSR_VSX; 1519 } 1520 } 1521 1522 void vsx_unavailable_tm(struct pt_regs *regs) 1523 { 1524 unsigned long orig_msr = regs->msr; 1525 1526 /* See the comments in fp_unavailable_tm(). This works similarly, 1527 * though we're loading both FP and VEC registers in here. 1528 * 1529 * If FP isn't in use, load FP regs. If VEC isn't in use, load VEC 1530 * regs. Either way, set MSR_VSX. 1531 */ 1532 1533 TM_DEBUG("VSX Unavailable trap whilst transactional at 0x%lx," 1534 "MSR=%lx\n", 1535 regs->nip, regs->msr); 1536 1537 current->thread.used_vsr = 1; 1538 1539 /* If FP and VMX are already loaded, we have all the state we need */ 1540 if ((orig_msr & (MSR_FP | MSR_VEC)) == (MSR_FP | MSR_VEC)) { 1541 regs->msr |= MSR_VSX; 1542 return; 1543 } 1544 1545 /* This reclaims FP and/or VR regs if they're already enabled */ 1546 tm_reclaim_current(TM_CAUSE_FAC_UNAV); 1547 1548 regs->msr |= MSR_VEC | MSR_FP | current->thread.fpexc_mode | 1549 MSR_VSX; 1550 1551 /* This loads & recheckpoints FP and VRs; but we have 1552 * to be sure not to overwrite previously-valid state. 1553 */ 1554 tm_recheckpoint(¤t->thread, regs->msr & ~orig_msr); 1555 1556 msr_check_and_set(orig_msr & (MSR_FP | MSR_VEC)); 1557 1558 if (orig_msr & MSR_FP) 1559 load_fp_state(¤t->thread.fp_state); 1560 if (orig_msr & MSR_VEC) 1561 load_vr_state(¤t->thread.vr_state); 1562 } 1563 #endif /* CONFIG_PPC_TRANSACTIONAL_MEM */ 1564 1565 void performance_monitor_exception(struct pt_regs *regs) 1566 { 1567 __this_cpu_inc(irq_stat.pmu_irqs); 1568 1569 perf_irq(regs); 1570 } 1571 1572 #ifdef CONFIG_PPC_ADV_DEBUG_REGS 1573 static void handle_debug(struct pt_regs *regs, unsigned long debug_status) 1574 { 1575 int changed = 0; 1576 /* 1577 * Determine the cause of the debug event, clear the 1578 * event flags and send a trap to the handler. Torez 1579 */ 1580 if (debug_status & (DBSR_DAC1R | DBSR_DAC1W)) { 1581 dbcr_dac(current) &= ~(DBCR_DAC1R | DBCR_DAC1W); 1582 #ifdef CONFIG_PPC_ADV_DEBUG_DAC_RANGE 1583 current->thread.debug.dbcr2 &= ~DBCR2_DAC12MODE; 1584 #endif 1585 do_send_trap(regs, mfspr(SPRN_DAC1), debug_status, TRAP_HWBKPT, 1586 5); 1587 changed |= 0x01; 1588 } else if (debug_status & (DBSR_DAC2R | DBSR_DAC2W)) { 1589 dbcr_dac(current) &= ~(DBCR_DAC2R | DBCR_DAC2W); 1590 do_send_trap(regs, mfspr(SPRN_DAC2), debug_status, TRAP_HWBKPT, 1591 6); 1592 changed |= 0x01; 1593 } else if (debug_status & DBSR_IAC1) { 1594 current->thread.debug.dbcr0 &= ~DBCR0_IAC1; 1595 dbcr_iac_range(current) &= ~DBCR_IAC12MODE; 1596 do_send_trap(regs, mfspr(SPRN_IAC1), debug_status, TRAP_HWBKPT, 1597 1); 1598 changed |= 0x01; 1599 } else if (debug_status & DBSR_IAC2) { 1600 current->thread.debug.dbcr0 &= ~DBCR0_IAC2; 1601 do_send_trap(regs, mfspr(SPRN_IAC2), debug_status, TRAP_HWBKPT, 1602 2); 1603 changed |= 0x01; 1604 } else if (debug_status & DBSR_IAC3) { 1605 current->thread.debug.dbcr0 &= ~DBCR0_IAC3; 1606 dbcr_iac_range(current) &= ~DBCR_IAC34MODE; 1607 do_send_trap(regs, mfspr(SPRN_IAC3), debug_status, TRAP_HWBKPT, 1608 3); 1609 changed |= 0x01; 1610 } else if (debug_status & DBSR_IAC4) { 1611 current->thread.debug.dbcr0 &= ~DBCR0_IAC4; 1612 do_send_trap(regs, mfspr(SPRN_IAC4), debug_status, TRAP_HWBKPT, 1613 4); 1614 changed |= 0x01; 1615 } 1616 /* 1617 * At the point this routine was called, the MSR(DE) was turned off. 1618 * Check all other debug flags and see if that bit needs to be turned 1619 * back on or not. 1620 */ 1621 if (DBCR_ACTIVE_EVENTS(current->thread.debug.dbcr0, 1622 current->thread.debug.dbcr1)) 1623 regs->msr |= MSR_DE; 1624 else 1625 /* Make sure the IDM flag is off */ 1626 current->thread.debug.dbcr0 &= ~DBCR0_IDM; 1627 1628 if (changed & 0x01) 1629 mtspr(SPRN_DBCR0, current->thread.debug.dbcr0); 1630 } 1631 1632 void DebugException(struct pt_regs *regs, unsigned long debug_status) 1633 { 1634 current->thread.debug.dbsr = debug_status; 1635 1636 /* Hack alert: On BookE, Branch Taken stops on the branch itself, while 1637 * on server, it stops on the target of the branch. In order to simulate 1638 * the server behaviour, we thus restart right away with a single step 1639 * instead of stopping here when hitting a BT 1640 */ 1641 if (debug_status & DBSR_BT) { 1642 regs->msr &= ~MSR_DE; 1643 1644 /* Disable BT */ 1645 mtspr(SPRN_DBCR0, mfspr(SPRN_DBCR0) & ~DBCR0_BT); 1646 /* Clear the BT event */ 1647 mtspr(SPRN_DBSR, DBSR_BT); 1648 1649 /* Do the single step trick only when coming from userspace */ 1650 if (user_mode(regs)) { 1651 current->thread.debug.dbcr0 &= ~DBCR0_BT; 1652 current->thread.debug.dbcr0 |= DBCR0_IDM | DBCR0_IC; 1653 regs->msr |= MSR_DE; 1654 return; 1655 } 1656 1657 if (kprobe_post_handler(regs)) 1658 return; 1659 1660 if (notify_die(DIE_SSTEP, "block_step", regs, 5, 1661 5, SIGTRAP) == NOTIFY_STOP) { 1662 return; 1663 } 1664 if (debugger_sstep(regs)) 1665 return; 1666 } else if (debug_status & DBSR_IC) { /* Instruction complete */ 1667 regs->msr &= ~MSR_DE; 1668 1669 /* Disable instruction completion */ 1670 mtspr(SPRN_DBCR0, mfspr(SPRN_DBCR0) & ~DBCR0_IC); 1671 /* Clear the instruction completion event */ 1672 mtspr(SPRN_DBSR, DBSR_IC); 1673 1674 if (kprobe_post_handler(regs)) 1675 return; 1676 1677 if (notify_die(DIE_SSTEP, "single_step", regs, 5, 1678 5, SIGTRAP) == NOTIFY_STOP) { 1679 return; 1680 } 1681 1682 if (debugger_sstep(regs)) 1683 return; 1684 1685 if (user_mode(regs)) { 1686 current->thread.debug.dbcr0 &= ~DBCR0_IC; 1687 if (DBCR_ACTIVE_EVENTS(current->thread.debug.dbcr0, 1688 current->thread.debug.dbcr1)) 1689 regs->msr |= MSR_DE; 1690 else 1691 /* Make sure the IDM bit is off */ 1692 current->thread.debug.dbcr0 &= ~DBCR0_IDM; 1693 } 1694 1695 _exception(SIGTRAP, regs, TRAP_TRACE, regs->nip); 1696 } else 1697 handle_debug(regs, debug_status); 1698 } 1699 NOKPROBE_SYMBOL(DebugException); 1700 #endif /* CONFIG_PPC_ADV_DEBUG_REGS */ 1701 1702 #if !defined(CONFIG_TAU_INT) 1703 void TAUException(struct pt_regs *regs) 1704 { 1705 printk("TAU trap at PC: %lx, MSR: %lx, vector=%lx %s\n", 1706 regs->nip, regs->msr, regs->trap, print_tainted()); 1707 } 1708 #endif /* CONFIG_INT_TAU */ 1709 1710 #ifdef CONFIG_ALTIVEC 1711 void altivec_assist_exception(struct pt_regs *regs) 1712 { 1713 int err; 1714 1715 if (!user_mode(regs)) { 1716 printk(KERN_EMERG "VMX/Altivec assist exception in kernel mode" 1717 " at %lx\n", regs->nip); 1718 die("Kernel VMX/Altivec assist exception", regs, SIGILL); 1719 } 1720 1721 flush_altivec_to_thread(current); 1722 1723 PPC_WARN_EMULATED(altivec, regs); 1724 err = emulate_altivec(regs); 1725 if (err == 0) { 1726 regs->nip += 4; /* skip emulated instruction */ 1727 emulate_single_step(regs); 1728 return; 1729 } 1730 1731 if (err == -EFAULT) { 1732 /* got an error reading the instruction */ 1733 _exception(SIGSEGV, regs, SEGV_ACCERR, regs->nip); 1734 } else { 1735 /* didn't recognize the instruction */ 1736 /* XXX quick hack for now: set the non-Java bit in the VSCR */ 1737 printk_ratelimited(KERN_ERR "Unrecognized altivec instruction " 1738 "in %s at %lx\n", current->comm, regs->nip); 1739 current->thread.vr_state.vscr.u[3] |= 0x10000; 1740 } 1741 } 1742 #endif /* CONFIG_ALTIVEC */ 1743 1744 #ifdef CONFIG_FSL_BOOKE 1745 void CacheLockingException(struct pt_regs *regs, unsigned long address, 1746 unsigned long error_code) 1747 { 1748 /* We treat cache locking instructions from the user 1749 * as priv ops, in the future we could try to do 1750 * something smarter 1751 */ 1752 if (error_code & (ESR_DLK|ESR_ILK)) 1753 _exception(SIGILL, regs, ILL_PRVOPC, regs->nip); 1754 return; 1755 } 1756 #endif /* CONFIG_FSL_BOOKE */ 1757 1758 #ifdef CONFIG_SPE 1759 void SPEFloatingPointException(struct pt_regs *regs) 1760 { 1761 extern int do_spe_mathemu(struct pt_regs *regs); 1762 unsigned long spefscr; 1763 int fpexc_mode; 1764 int code = 0; 1765 int err; 1766 1767 flush_spe_to_thread(current); 1768 1769 spefscr = current->thread.spefscr; 1770 fpexc_mode = current->thread.fpexc_mode; 1771 1772 if ((spefscr & SPEFSCR_FOVF) && (fpexc_mode & PR_FP_EXC_OVF)) { 1773 code = FPE_FLTOVF; 1774 } 1775 else if ((spefscr & SPEFSCR_FUNF) && (fpexc_mode & PR_FP_EXC_UND)) { 1776 code = FPE_FLTUND; 1777 } 1778 else if ((spefscr & SPEFSCR_FDBZ) && (fpexc_mode & PR_FP_EXC_DIV)) 1779 code = FPE_FLTDIV; 1780 else if ((spefscr & SPEFSCR_FINV) && (fpexc_mode & PR_FP_EXC_INV)) { 1781 code = FPE_FLTINV; 1782 } 1783 else if ((spefscr & (SPEFSCR_FG | SPEFSCR_FX)) && (fpexc_mode & PR_FP_EXC_RES)) 1784 code = FPE_FLTRES; 1785 1786 err = do_spe_mathemu(regs); 1787 if (err == 0) { 1788 regs->nip += 4; /* skip emulated instruction */ 1789 emulate_single_step(regs); 1790 return; 1791 } 1792 1793 if (err == -EFAULT) { 1794 /* got an error reading the instruction */ 1795 _exception(SIGSEGV, regs, SEGV_ACCERR, regs->nip); 1796 } else if (err == -EINVAL) { 1797 /* didn't recognize the instruction */ 1798 printk(KERN_ERR "unrecognized spe instruction " 1799 "in %s at %lx\n", current->comm, regs->nip); 1800 } else { 1801 _exception(SIGFPE, regs, code, regs->nip); 1802 } 1803 1804 return; 1805 } 1806 1807 void SPEFloatingPointRoundException(struct pt_regs *regs) 1808 { 1809 extern int speround_handler(struct pt_regs *regs); 1810 int err; 1811 1812 preempt_disable(); 1813 if (regs->msr & MSR_SPE) 1814 giveup_spe(current); 1815 preempt_enable(); 1816 1817 regs->nip -= 4; 1818 err = speround_handler(regs); 1819 if (err == 0) { 1820 regs->nip += 4; /* skip emulated instruction */ 1821 emulate_single_step(regs); 1822 return; 1823 } 1824 1825 if (err == -EFAULT) { 1826 /* got an error reading the instruction */ 1827 _exception(SIGSEGV, regs, SEGV_ACCERR, regs->nip); 1828 } else if (err == -EINVAL) { 1829 /* didn't recognize the instruction */ 1830 printk(KERN_ERR "unrecognized spe instruction " 1831 "in %s at %lx\n", current->comm, regs->nip); 1832 } else { 1833 _exception(SIGFPE, regs, 0, regs->nip); 1834 return; 1835 } 1836 } 1837 #endif 1838 1839 /* 1840 * We enter here if we get an unrecoverable exception, that is, one 1841 * that happened at a point where the RI (recoverable interrupt) bit 1842 * in the MSR is 0. This indicates that SRR0/1 are live, and that 1843 * we therefore lost state by taking this exception. 1844 */ 1845 void unrecoverable_exception(struct pt_regs *regs) 1846 { 1847 printk(KERN_EMERG "Unrecoverable exception %lx at %lx\n", 1848 regs->trap, regs->nip); 1849 die("Unrecoverable exception", regs, SIGABRT); 1850 } 1851 NOKPROBE_SYMBOL(unrecoverable_exception); 1852 1853 #if defined(CONFIG_BOOKE_WDT) || defined(CONFIG_40x) 1854 /* 1855 * Default handler for a Watchdog exception, 1856 * spins until a reboot occurs 1857 */ 1858 void __attribute__ ((weak)) WatchdogHandler(struct pt_regs *regs) 1859 { 1860 /* Generic WatchdogHandler, implement your own */ 1861 mtspr(SPRN_TCR, mfspr(SPRN_TCR)&(~TCR_WIE)); 1862 return; 1863 } 1864 1865 void WatchdogException(struct pt_regs *regs) 1866 { 1867 printk (KERN_EMERG "PowerPC Book-E Watchdog Exception\n"); 1868 WatchdogHandler(regs); 1869 } 1870 #endif 1871 1872 /* 1873 * We enter here if we discover during exception entry that we are 1874 * running in supervisor mode with a userspace value in the stack pointer. 1875 */ 1876 void kernel_bad_stack(struct pt_regs *regs) 1877 { 1878 printk(KERN_EMERG "Bad kernel stack pointer %lx at %lx\n", 1879 regs->gpr[1], regs->nip); 1880 die("Bad kernel stack pointer", regs, SIGABRT); 1881 } 1882 NOKPROBE_SYMBOL(kernel_bad_stack); 1883 1884 void __init trap_init(void) 1885 { 1886 } 1887 1888 1889 #ifdef CONFIG_PPC_EMULATED_STATS 1890 1891 #define WARN_EMULATED_SETUP(type) .type = { .name = #type } 1892 1893 struct ppc_emulated ppc_emulated = { 1894 #ifdef CONFIG_ALTIVEC 1895 WARN_EMULATED_SETUP(altivec), 1896 #endif 1897 WARN_EMULATED_SETUP(dcba), 1898 WARN_EMULATED_SETUP(dcbz), 1899 WARN_EMULATED_SETUP(fp_pair), 1900 WARN_EMULATED_SETUP(isel), 1901 WARN_EMULATED_SETUP(mcrxr), 1902 WARN_EMULATED_SETUP(mfpvr), 1903 WARN_EMULATED_SETUP(multiple), 1904 WARN_EMULATED_SETUP(popcntb), 1905 WARN_EMULATED_SETUP(spe), 1906 WARN_EMULATED_SETUP(string), 1907 WARN_EMULATED_SETUP(sync), 1908 WARN_EMULATED_SETUP(unaligned), 1909 #ifdef CONFIG_MATH_EMULATION 1910 WARN_EMULATED_SETUP(math), 1911 #endif 1912 #ifdef CONFIG_VSX 1913 WARN_EMULATED_SETUP(vsx), 1914 #endif 1915 #ifdef CONFIG_PPC64 1916 WARN_EMULATED_SETUP(mfdscr), 1917 WARN_EMULATED_SETUP(mtdscr), 1918 WARN_EMULATED_SETUP(lq_stq), 1919 #endif 1920 }; 1921 1922 u32 ppc_warn_emulated; 1923 1924 void ppc_warn_emulated_print(const char *type) 1925 { 1926 pr_warn_ratelimited("%s used emulated %s instruction\n", current->comm, 1927 type); 1928 } 1929 1930 static int __init ppc_warn_emulated_init(void) 1931 { 1932 struct dentry *dir, *d; 1933 unsigned int i; 1934 struct ppc_emulated_entry *entries = (void *)&ppc_emulated; 1935 1936 if (!powerpc_debugfs_root) 1937 return -ENODEV; 1938 1939 dir = debugfs_create_dir("emulated_instructions", 1940 powerpc_debugfs_root); 1941 if (!dir) 1942 return -ENOMEM; 1943 1944 d = debugfs_create_u32("do_warn", S_IRUGO | S_IWUSR, dir, 1945 &ppc_warn_emulated); 1946 if (!d) 1947 goto fail; 1948 1949 for (i = 0; i < sizeof(ppc_emulated)/sizeof(*entries); i++) { 1950 d = debugfs_create_u32(entries[i].name, S_IRUGO | S_IWUSR, dir, 1951 (u32 *)&entries[i].val.counter); 1952 if (!d) 1953 goto fail; 1954 } 1955 1956 return 0; 1957 1958 fail: 1959 debugfs_remove_recursive(dir); 1960 return -ENOMEM; 1961 } 1962 1963 device_initcall(ppc_warn_emulated_init); 1964 1965 #endif /* CONFIG_PPC_EMULATED_STATS */ 1966