1 /* arch/sparc64/kernel/traps.c 2 * 3 * Copyright (C) 1995,1997,2008,2009,2012 David S. Miller (davem@davemloft.net) 4 * Copyright (C) 1997,1999,2000 Jakub Jelinek (jakub@redhat.com) 5 */ 6 7 /* 8 * I like traps on v9, :)))) 9 */ 10 11 #include <linux/extable.h> 12 #include <linux/sched/mm.h> 13 #include <linux/sched/debug.h> 14 #include <linux/linkage.h> 15 #include <linux/kernel.h> 16 #include <linux/signal.h> 17 #include <linux/smp.h> 18 #include <linux/mm.h> 19 #include <linux/init.h> 20 #include <linux/kdebug.h> 21 #include <linux/ftrace.h> 22 #include <linux/reboot.h> 23 #include <linux/gfp.h> 24 #include <linux/context_tracking.h> 25 26 #include <asm/smp.h> 27 #include <asm/delay.h> 28 #include <asm/ptrace.h> 29 #include <asm/oplib.h> 30 #include <asm/page.h> 31 #include <asm/pgtable.h> 32 #include <asm/unistd.h> 33 #include <linux/uaccess.h> 34 #include <asm/fpumacro.h> 35 #include <asm/lsu.h> 36 #include <asm/dcu.h> 37 #include <asm/estate.h> 38 #include <asm/chafsr.h> 39 #include <asm/sfafsr.h> 40 #include <asm/psrcompat.h> 41 #include <asm/processor.h> 42 #include <asm/timer.h> 43 #include <asm/head.h> 44 #include <asm/prom.h> 45 #include <asm/memctrl.h> 46 #include <asm/cacheflush.h> 47 #include <asm/setup.h> 48 49 #include "entry.h" 50 #include "kernel.h" 51 #include "kstack.h" 52 53 /* When an irrecoverable trap occurs at tl > 0, the trap entry 54 * code logs the trap state registers at every level in the trap 55 * stack. It is found at (pt_regs + sizeof(pt_regs)) and the layout 56 * is as follows: 57 */ 58 struct tl1_traplog { 59 struct { 60 unsigned long tstate; 61 unsigned long tpc; 62 unsigned long tnpc; 63 unsigned long tt; 64 } trapstack[4]; 65 unsigned long tl; 66 }; 67 68 static void dump_tl1_traplog(struct tl1_traplog *p) 69 { 70 int i, limit; 71 72 printk(KERN_EMERG "TRAPLOG: Error at trap level 0x%lx, " 73 "dumping track stack.\n", p->tl); 74 75 limit = (tlb_type == hypervisor) ? 2 : 4; 76 for (i = 0; i < limit; i++) { 77 printk(KERN_EMERG 78 "TRAPLOG: Trap level %d TSTATE[%016lx] TPC[%016lx] " 79 "TNPC[%016lx] TT[%lx]\n", 80 i + 1, 81 p->trapstack[i].tstate, p->trapstack[i].tpc, 82 p->trapstack[i].tnpc, p->trapstack[i].tt); 83 printk("TRAPLOG: TPC<%pS>\n", (void *) p->trapstack[i].tpc); 84 } 85 } 86 87 void bad_trap(struct pt_regs *regs, long lvl) 88 { 89 char buffer[36]; 90 siginfo_t info; 91 92 if (notify_die(DIE_TRAP, "bad trap", regs, 93 0, lvl, SIGTRAP) == NOTIFY_STOP) 94 return; 95 96 if (lvl < 0x100) { 97 sprintf(buffer, "Bad hw trap %lx at tl0\n", lvl); 98 die_if_kernel(buffer, regs); 99 } 100 101 lvl -= 0x100; 102 if (regs->tstate & TSTATE_PRIV) { 103 sprintf(buffer, "Kernel bad sw trap %lx", lvl); 104 die_if_kernel(buffer, regs); 105 } 106 if (test_thread_flag(TIF_32BIT)) { 107 regs->tpc &= 0xffffffff; 108 regs->tnpc &= 0xffffffff; 109 } 110 info.si_signo = SIGILL; 111 info.si_errno = 0; 112 info.si_code = ILL_ILLTRP; 113 info.si_addr = (void __user *)regs->tpc; 114 info.si_trapno = lvl; 115 force_sig_info(SIGILL, &info, current); 116 } 117 118 void bad_trap_tl1(struct pt_regs *regs, long lvl) 119 { 120 char buffer[36]; 121 122 if (notify_die(DIE_TRAP_TL1, "bad trap tl1", regs, 123 0, lvl, SIGTRAP) == NOTIFY_STOP) 124 return; 125 126 dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); 127 128 sprintf (buffer, "Bad trap %lx at tl>0", lvl); 129 die_if_kernel (buffer, regs); 130 } 131 132 #ifdef CONFIG_DEBUG_BUGVERBOSE 133 void do_BUG(const char *file, int line) 134 { 135 bust_spinlocks(1); 136 printk("kernel BUG at %s:%d!\n", file, line); 137 } 138 EXPORT_SYMBOL(do_BUG); 139 #endif 140 141 static DEFINE_SPINLOCK(dimm_handler_lock); 142 static dimm_printer_t dimm_handler; 143 144 static int sprintf_dimm(int synd_code, unsigned long paddr, char *buf, int buflen) 145 { 146 unsigned long flags; 147 int ret = -ENODEV; 148 149 spin_lock_irqsave(&dimm_handler_lock, flags); 150 if (dimm_handler) { 151 ret = dimm_handler(synd_code, paddr, buf, buflen); 152 } else if (tlb_type == spitfire) { 153 if (prom_getunumber(synd_code, paddr, buf, buflen) == -1) 154 ret = -EINVAL; 155 else 156 ret = 0; 157 } else 158 ret = -ENODEV; 159 spin_unlock_irqrestore(&dimm_handler_lock, flags); 160 161 return ret; 162 } 163 164 int register_dimm_printer(dimm_printer_t func) 165 { 166 unsigned long flags; 167 int ret = 0; 168 169 spin_lock_irqsave(&dimm_handler_lock, flags); 170 if (!dimm_handler) 171 dimm_handler = func; 172 else 173 ret = -EEXIST; 174 spin_unlock_irqrestore(&dimm_handler_lock, flags); 175 176 return ret; 177 } 178 EXPORT_SYMBOL_GPL(register_dimm_printer); 179 180 void unregister_dimm_printer(dimm_printer_t func) 181 { 182 unsigned long flags; 183 184 spin_lock_irqsave(&dimm_handler_lock, flags); 185 if (dimm_handler == func) 186 dimm_handler = NULL; 187 spin_unlock_irqrestore(&dimm_handler_lock, flags); 188 } 189 EXPORT_SYMBOL_GPL(unregister_dimm_printer); 190 191 void spitfire_insn_access_exception(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar) 192 { 193 enum ctx_state prev_state = exception_enter(); 194 siginfo_t info; 195 196 if (notify_die(DIE_TRAP, "instruction access exception", regs, 197 0, 0x8, SIGTRAP) == NOTIFY_STOP) 198 goto out; 199 200 if (regs->tstate & TSTATE_PRIV) { 201 printk("spitfire_insn_access_exception: SFSR[%016lx] " 202 "SFAR[%016lx], going.\n", sfsr, sfar); 203 die_if_kernel("Iax", regs); 204 } 205 if (test_thread_flag(TIF_32BIT)) { 206 regs->tpc &= 0xffffffff; 207 regs->tnpc &= 0xffffffff; 208 } 209 info.si_signo = SIGSEGV; 210 info.si_errno = 0; 211 info.si_code = SEGV_MAPERR; 212 info.si_addr = (void __user *)regs->tpc; 213 info.si_trapno = 0; 214 force_sig_info(SIGSEGV, &info, current); 215 out: 216 exception_exit(prev_state); 217 } 218 219 void spitfire_insn_access_exception_tl1(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar) 220 { 221 if (notify_die(DIE_TRAP_TL1, "instruction access exception tl1", regs, 222 0, 0x8, SIGTRAP) == NOTIFY_STOP) 223 return; 224 225 dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); 226 spitfire_insn_access_exception(regs, sfsr, sfar); 227 } 228 229 void sun4v_insn_access_exception(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx) 230 { 231 unsigned short type = (type_ctx >> 16); 232 unsigned short ctx = (type_ctx & 0xffff); 233 siginfo_t info; 234 235 if (notify_die(DIE_TRAP, "instruction access exception", regs, 236 0, 0x8, SIGTRAP) == NOTIFY_STOP) 237 return; 238 239 if (regs->tstate & TSTATE_PRIV) { 240 printk("sun4v_insn_access_exception: ADDR[%016lx] " 241 "CTX[%04x] TYPE[%04x], going.\n", 242 addr, ctx, type); 243 die_if_kernel("Iax", regs); 244 } 245 246 if (test_thread_flag(TIF_32BIT)) { 247 regs->tpc &= 0xffffffff; 248 regs->tnpc &= 0xffffffff; 249 } 250 info.si_signo = SIGSEGV; 251 info.si_errno = 0; 252 info.si_code = SEGV_MAPERR; 253 info.si_addr = (void __user *) addr; 254 info.si_trapno = 0; 255 force_sig_info(SIGSEGV, &info, current); 256 } 257 258 void sun4v_insn_access_exception_tl1(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx) 259 { 260 if (notify_die(DIE_TRAP_TL1, "instruction access exception tl1", regs, 261 0, 0x8, SIGTRAP) == NOTIFY_STOP) 262 return; 263 264 dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); 265 sun4v_insn_access_exception(regs, addr, type_ctx); 266 } 267 268 bool is_no_fault_exception(struct pt_regs *regs) 269 { 270 unsigned char asi; 271 u32 insn; 272 273 if (get_user(insn, (u32 __user *)regs->tpc) == -EFAULT) 274 return false; 275 276 /* 277 * Must do a little instruction decoding here in order to 278 * decide on a course of action. The bits of interest are: 279 * insn[31:30] = op, where 3 indicates the load/store group 280 * insn[24:19] = op3, which identifies individual opcodes 281 * insn[13] indicates an immediate offset 282 * op3[4]=1 identifies alternate space instructions 283 * op3[5:4]=3 identifies floating point instructions 284 * op3[2]=1 identifies stores 285 * See "Opcode Maps" in the appendix of any Sparc V9 286 * architecture spec for full details. 287 */ 288 if ((insn & 0xc0800000) == 0xc0800000) { /* op=3, op3[4]=1 */ 289 if (insn & 0x2000) /* immediate offset */ 290 asi = (regs->tstate >> 24); /* saved %asi */ 291 else 292 asi = (insn >> 5); /* immediate asi */ 293 if ((asi & 0xf2) == ASI_PNF) { 294 if (insn & 0x1000000) { /* op3[5:4]=3 */ 295 handle_ldf_stq(insn, regs); 296 return true; 297 } else if (insn & 0x200000) { /* op3[2], stores */ 298 return false; 299 } 300 handle_ld_nf(insn, regs); 301 return true; 302 } 303 } 304 return false; 305 } 306 307 void spitfire_data_access_exception(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar) 308 { 309 enum ctx_state prev_state = exception_enter(); 310 siginfo_t info; 311 312 if (notify_die(DIE_TRAP, "data access exception", regs, 313 0, 0x30, SIGTRAP) == NOTIFY_STOP) 314 goto out; 315 316 if (regs->tstate & TSTATE_PRIV) { 317 /* Test if this comes from uaccess places. */ 318 const struct exception_table_entry *entry; 319 320 entry = search_exception_tables(regs->tpc); 321 if (entry) { 322 /* Ouch, somebody is trying VM hole tricks on us... */ 323 #ifdef DEBUG_EXCEPTIONS 324 printk("Exception: PC<%016lx> faddr<UNKNOWN>\n", regs->tpc); 325 printk("EX_TABLE: insn<%016lx> fixup<%016lx>\n", 326 regs->tpc, entry->fixup); 327 #endif 328 regs->tpc = entry->fixup; 329 regs->tnpc = regs->tpc + 4; 330 goto out; 331 } 332 /* Shit... */ 333 printk("spitfire_data_access_exception: SFSR[%016lx] " 334 "SFAR[%016lx], going.\n", sfsr, sfar); 335 die_if_kernel("Dax", regs); 336 } 337 338 if (is_no_fault_exception(regs)) 339 return; 340 341 info.si_signo = SIGSEGV; 342 info.si_errno = 0; 343 info.si_code = SEGV_MAPERR; 344 info.si_addr = (void __user *)sfar; 345 info.si_trapno = 0; 346 force_sig_info(SIGSEGV, &info, current); 347 out: 348 exception_exit(prev_state); 349 } 350 351 void spitfire_data_access_exception_tl1(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar) 352 { 353 if (notify_die(DIE_TRAP_TL1, "data access exception tl1", regs, 354 0, 0x30, SIGTRAP) == NOTIFY_STOP) 355 return; 356 357 dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); 358 spitfire_data_access_exception(regs, sfsr, sfar); 359 } 360 361 void sun4v_data_access_exception(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx) 362 { 363 unsigned short type = (type_ctx >> 16); 364 unsigned short ctx = (type_ctx & 0xffff); 365 siginfo_t info; 366 367 if (notify_die(DIE_TRAP, "data access exception", regs, 368 0, 0x8, SIGTRAP) == NOTIFY_STOP) 369 return; 370 371 if (regs->tstate & TSTATE_PRIV) { 372 /* Test if this comes from uaccess places. */ 373 const struct exception_table_entry *entry; 374 375 entry = search_exception_tables(regs->tpc); 376 if (entry) { 377 /* Ouch, somebody is trying VM hole tricks on us... */ 378 #ifdef DEBUG_EXCEPTIONS 379 printk("Exception: PC<%016lx> faddr<UNKNOWN>\n", regs->tpc); 380 printk("EX_TABLE: insn<%016lx> fixup<%016lx>\n", 381 regs->tpc, entry->fixup); 382 #endif 383 regs->tpc = entry->fixup; 384 regs->tnpc = regs->tpc + 4; 385 return; 386 } 387 printk("sun4v_data_access_exception: ADDR[%016lx] " 388 "CTX[%04x] TYPE[%04x], going.\n", 389 addr, ctx, type); 390 die_if_kernel("Dax", regs); 391 } 392 393 if (test_thread_flag(TIF_32BIT)) { 394 regs->tpc &= 0xffffffff; 395 regs->tnpc &= 0xffffffff; 396 } 397 if (is_no_fault_exception(regs)) 398 return; 399 400 info.si_signo = SIGSEGV; 401 info.si_errno = 0; 402 info.si_code = SEGV_MAPERR; 403 info.si_addr = (void __user *) addr; 404 info.si_trapno = 0; 405 force_sig_info(SIGSEGV, &info, current); 406 } 407 408 void sun4v_data_access_exception_tl1(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx) 409 { 410 if (notify_die(DIE_TRAP_TL1, "data access exception tl1", regs, 411 0, 0x8, SIGTRAP) == NOTIFY_STOP) 412 return; 413 414 dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); 415 sun4v_data_access_exception(regs, addr, type_ctx); 416 } 417 418 #ifdef CONFIG_PCI 419 #include "pci_impl.h" 420 #endif 421 422 /* When access exceptions happen, we must do this. */ 423 static void spitfire_clean_and_reenable_l1_caches(void) 424 { 425 unsigned long va; 426 427 if (tlb_type != spitfire) 428 BUG(); 429 430 /* Clean 'em. */ 431 for (va = 0; va < (PAGE_SIZE << 1); va += 32) { 432 spitfire_put_icache_tag(va, 0x0); 433 spitfire_put_dcache_tag(va, 0x0); 434 } 435 436 /* Re-enable in LSU. */ 437 __asm__ __volatile__("flush %%g6\n\t" 438 "membar #Sync\n\t" 439 "stxa %0, [%%g0] %1\n\t" 440 "membar #Sync" 441 : /* no outputs */ 442 : "r" (LSU_CONTROL_IC | LSU_CONTROL_DC | 443 LSU_CONTROL_IM | LSU_CONTROL_DM), 444 "i" (ASI_LSU_CONTROL) 445 : "memory"); 446 } 447 448 static void spitfire_enable_estate_errors(void) 449 { 450 __asm__ __volatile__("stxa %0, [%%g0] %1\n\t" 451 "membar #Sync" 452 : /* no outputs */ 453 : "r" (ESTATE_ERR_ALL), 454 "i" (ASI_ESTATE_ERROR_EN)); 455 } 456 457 static char ecc_syndrome_table[] = { 458 0x4c, 0x40, 0x41, 0x48, 0x42, 0x48, 0x48, 0x49, 459 0x43, 0x48, 0x48, 0x49, 0x48, 0x49, 0x49, 0x4a, 460 0x44, 0x48, 0x48, 0x20, 0x48, 0x39, 0x4b, 0x48, 461 0x48, 0x25, 0x31, 0x48, 0x28, 0x48, 0x48, 0x2c, 462 0x45, 0x48, 0x48, 0x21, 0x48, 0x3d, 0x04, 0x48, 463 0x48, 0x4b, 0x35, 0x48, 0x2d, 0x48, 0x48, 0x29, 464 0x48, 0x00, 0x01, 0x48, 0x0a, 0x48, 0x48, 0x4b, 465 0x0f, 0x48, 0x48, 0x4b, 0x48, 0x49, 0x49, 0x48, 466 0x46, 0x48, 0x48, 0x2a, 0x48, 0x3b, 0x27, 0x48, 467 0x48, 0x4b, 0x33, 0x48, 0x22, 0x48, 0x48, 0x2e, 468 0x48, 0x19, 0x1d, 0x48, 0x1b, 0x4a, 0x48, 0x4b, 469 0x1f, 0x48, 0x4a, 0x4b, 0x48, 0x4b, 0x4b, 0x48, 470 0x48, 0x4b, 0x24, 0x48, 0x07, 0x48, 0x48, 0x36, 471 0x4b, 0x48, 0x48, 0x3e, 0x48, 0x30, 0x38, 0x48, 472 0x49, 0x48, 0x48, 0x4b, 0x48, 0x4b, 0x16, 0x48, 473 0x48, 0x12, 0x4b, 0x48, 0x49, 0x48, 0x48, 0x4b, 474 0x47, 0x48, 0x48, 0x2f, 0x48, 0x3f, 0x4b, 0x48, 475 0x48, 0x06, 0x37, 0x48, 0x23, 0x48, 0x48, 0x2b, 476 0x48, 0x05, 0x4b, 0x48, 0x4b, 0x48, 0x48, 0x32, 477 0x26, 0x48, 0x48, 0x3a, 0x48, 0x34, 0x3c, 0x48, 478 0x48, 0x11, 0x15, 0x48, 0x13, 0x4a, 0x48, 0x4b, 479 0x17, 0x48, 0x4a, 0x4b, 0x48, 0x4b, 0x4b, 0x48, 480 0x49, 0x48, 0x48, 0x4b, 0x48, 0x4b, 0x1e, 0x48, 481 0x48, 0x1a, 0x4b, 0x48, 0x49, 0x48, 0x48, 0x4b, 482 0x48, 0x08, 0x0d, 0x48, 0x02, 0x48, 0x48, 0x49, 483 0x03, 0x48, 0x48, 0x49, 0x48, 0x4b, 0x4b, 0x48, 484 0x49, 0x48, 0x48, 0x49, 0x48, 0x4b, 0x10, 0x48, 485 0x48, 0x14, 0x4b, 0x48, 0x4b, 0x48, 0x48, 0x4b, 486 0x49, 0x48, 0x48, 0x49, 0x48, 0x4b, 0x18, 0x48, 487 0x48, 0x1c, 0x4b, 0x48, 0x4b, 0x48, 0x48, 0x4b, 488 0x4a, 0x0c, 0x09, 0x48, 0x0e, 0x48, 0x48, 0x4b, 489 0x0b, 0x48, 0x48, 0x4b, 0x48, 0x4b, 0x4b, 0x4a 490 }; 491 492 static char *syndrome_unknown = "<Unknown>"; 493 494 static void spitfire_log_udb_syndrome(unsigned long afar, unsigned long udbh, unsigned long udbl, unsigned long bit) 495 { 496 unsigned short scode; 497 char memmod_str[64], *p; 498 499 if (udbl & bit) { 500 scode = ecc_syndrome_table[udbl & 0xff]; 501 if (sprintf_dimm(scode, afar, memmod_str, sizeof(memmod_str)) < 0) 502 p = syndrome_unknown; 503 else 504 p = memmod_str; 505 printk(KERN_WARNING "CPU[%d]: UDBL Syndrome[%x] " 506 "Memory Module \"%s\"\n", 507 smp_processor_id(), scode, p); 508 } 509 510 if (udbh & bit) { 511 scode = ecc_syndrome_table[udbh & 0xff]; 512 if (sprintf_dimm(scode, afar, memmod_str, sizeof(memmod_str)) < 0) 513 p = syndrome_unknown; 514 else 515 p = memmod_str; 516 printk(KERN_WARNING "CPU[%d]: UDBH Syndrome[%x] " 517 "Memory Module \"%s\"\n", 518 smp_processor_id(), scode, p); 519 } 520 521 } 522 523 static void spitfire_cee_log(unsigned long afsr, unsigned long afar, unsigned long udbh, unsigned long udbl, int tl1, struct pt_regs *regs) 524 { 525 526 printk(KERN_WARNING "CPU[%d]: Correctable ECC Error " 527 "AFSR[%lx] AFAR[%016lx] UDBL[%lx] UDBH[%lx] TL>1[%d]\n", 528 smp_processor_id(), afsr, afar, udbl, udbh, tl1); 529 530 spitfire_log_udb_syndrome(afar, udbh, udbl, UDBE_CE); 531 532 /* We always log it, even if someone is listening for this 533 * trap. 534 */ 535 notify_die(DIE_TRAP, "Correctable ECC Error", regs, 536 0, TRAP_TYPE_CEE, SIGTRAP); 537 538 /* The Correctable ECC Error trap does not disable I/D caches. So 539 * we only have to restore the ESTATE Error Enable register. 540 */ 541 spitfire_enable_estate_errors(); 542 } 543 544 static void spitfire_ue_log(unsigned long afsr, unsigned long afar, unsigned long udbh, unsigned long udbl, unsigned long tt, int tl1, struct pt_regs *regs) 545 { 546 siginfo_t info; 547 548 printk(KERN_WARNING "CPU[%d]: Uncorrectable Error AFSR[%lx] " 549 "AFAR[%lx] UDBL[%lx] UDBH[%ld] TT[%lx] TL>1[%d]\n", 550 smp_processor_id(), afsr, afar, udbl, udbh, tt, tl1); 551 552 /* XXX add more human friendly logging of the error status 553 * XXX as is implemented for cheetah 554 */ 555 556 spitfire_log_udb_syndrome(afar, udbh, udbl, UDBE_UE); 557 558 /* We always log it, even if someone is listening for this 559 * trap. 560 */ 561 notify_die(DIE_TRAP, "Uncorrectable Error", regs, 562 0, tt, SIGTRAP); 563 564 if (regs->tstate & TSTATE_PRIV) { 565 if (tl1) 566 dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); 567 die_if_kernel("UE", regs); 568 } 569 570 /* XXX need more intelligent processing here, such as is implemented 571 * XXX for cheetah errors, in fact if the E-cache still holds the 572 * XXX line with bad parity this will loop 573 */ 574 575 spitfire_clean_and_reenable_l1_caches(); 576 spitfire_enable_estate_errors(); 577 578 if (test_thread_flag(TIF_32BIT)) { 579 regs->tpc &= 0xffffffff; 580 regs->tnpc &= 0xffffffff; 581 } 582 info.si_signo = SIGBUS; 583 info.si_errno = 0; 584 info.si_code = BUS_OBJERR; 585 info.si_addr = (void *)0; 586 info.si_trapno = 0; 587 force_sig_info(SIGBUS, &info, current); 588 } 589 590 void spitfire_access_error(struct pt_regs *regs, unsigned long status_encoded, unsigned long afar) 591 { 592 unsigned long afsr, tt, udbh, udbl; 593 int tl1; 594 595 afsr = (status_encoded & SFSTAT_AFSR_MASK) >> SFSTAT_AFSR_SHIFT; 596 tt = (status_encoded & SFSTAT_TRAP_TYPE) >> SFSTAT_TRAP_TYPE_SHIFT; 597 tl1 = (status_encoded & SFSTAT_TL_GT_ONE) ? 1 : 0; 598 udbl = (status_encoded & SFSTAT_UDBL_MASK) >> SFSTAT_UDBL_SHIFT; 599 udbh = (status_encoded & SFSTAT_UDBH_MASK) >> SFSTAT_UDBH_SHIFT; 600 601 #ifdef CONFIG_PCI 602 if (tt == TRAP_TYPE_DAE && 603 pci_poke_in_progress && pci_poke_cpu == smp_processor_id()) { 604 spitfire_clean_and_reenable_l1_caches(); 605 spitfire_enable_estate_errors(); 606 607 pci_poke_faulted = 1; 608 regs->tnpc = regs->tpc + 4; 609 return; 610 } 611 #endif 612 613 if (afsr & SFAFSR_UE) 614 spitfire_ue_log(afsr, afar, udbh, udbl, tt, tl1, regs); 615 616 if (tt == TRAP_TYPE_CEE) { 617 /* Handle the case where we took a CEE trap, but ACK'd 618 * only the UE state in the UDB error registers. 619 */ 620 if (afsr & SFAFSR_UE) { 621 if (udbh & UDBE_CE) { 622 __asm__ __volatile__( 623 "stxa %0, [%1] %2\n\t" 624 "membar #Sync" 625 : /* no outputs */ 626 : "r" (udbh & UDBE_CE), 627 "r" (0x0), "i" (ASI_UDB_ERROR_W)); 628 } 629 if (udbl & UDBE_CE) { 630 __asm__ __volatile__( 631 "stxa %0, [%1] %2\n\t" 632 "membar #Sync" 633 : /* no outputs */ 634 : "r" (udbl & UDBE_CE), 635 "r" (0x18), "i" (ASI_UDB_ERROR_W)); 636 } 637 } 638 639 spitfire_cee_log(afsr, afar, udbh, udbl, tl1, regs); 640 } 641 } 642 643 int cheetah_pcache_forced_on; 644 645 void cheetah_enable_pcache(void) 646 { 647 unsigned long dcr; 648 649 printk("CHEETAH: Enabling P-Cache on cpu %d.\n", 650 smp_processor_id()); 651 652 __asm__ __volatile__("ldxa [%%g0] %1, %0" 653 : "=r" (dcr) 654 : "i" (ASI_DCU_CONTROL_REG)); 655 dcr |= (DCU_PE | DCU_HPE | DCU_SPE | DCU_SL); 656 __asm__ __volatile__("stxa %0, [%%g0] %1\n\t" 657 "membar #Sync" 658 : /* no outputs */ 659 : "r" (dcr), "i" (ASI_DCU_CONTROL_REG)); 660 } 661 662 /* Cheetah error trap handling. */ 663 static unsigned long ecache_flush_physbase; 664 static unsigned long ecache_flush_linesize; 665 static unsigned long ecache_flush_size; 666 667 /* This table is ordered in priority of errors and matches the 668 * AFAR overwrite policy as well. 669 */ 670 671 struct afsr_error_table { 672 unsigned long mask; 673 const char *name; 674 }; 675 676 static const char CHAFSR_PERR_msg[] = 677 "System interface protocol error"; 678 static const char CHAFSR_IERR_msg[] = 679 "Internal processor error"; 680 static const char CHAFSR_ISAP_msg[] = 681 "System request parity error on incoming address"; 682 static const char CHAFSR_UCU_msg[] = 683 "Uncorrectable E-cache ECC error for ifetch/data"; 684 static const char CHAFSR_UCC_msg[] = 685 "SW Correctable E-cache ECC error for ifetch/data"; 686 static const char CHAFSR_UE_msg[] = 687 "Uncorrectable system bus data ECC error for read"; 688 static const char CHAFSR_EDU_msg[] = 689 "Uncorrectable E-cache ECC error for stmerge/blkld"; 690 static const char CHAFSR_EMU_msg[] = 691 "Uncorrectable system bus MTAG error"; 692 static const char CHAFSR_WDU_msg[] = 693 "Uncorrectable E-cache ECC error for writeback"; 694 static const char CHAFSR_CPU_msg[] = 695 "Uncorrectable ECC error for copyout"; 696 static const char CHAFSR_CE_msg[] = 697 "HW corrected system bus data ECC error for read"; 698 static const char CHAFSR_EDC_msg[] = 699 "HW corrected E-cache ECC error for stmerge/blkld"; 700 static const char CHAFSR_EMC_msg[] = 701 "HW corrected system bus MTAG ECC error"; 702 static const char CHAFSR_WDC_msg[] = 703 "HW corrected E-cache ECC error for writeback"; 704 static const char CHAFSR_CPC_msg[] = 705 "HW corrected ECC error for copyout"; 706 static const char CHAFSR_TO_msg[] = 707 "Unmapped error from system bus"; 708 static const char CHAFSR_BERR_msg[] = 709 "Bus error response from system bus"; 710 static const char CHAFSR_IVC_msg[] = 711 "HW corrected system bus data ECC error for ivec read"; 712 static const char CHAFSR_IVU_msg[] = 713 "Uncorrectable system bus data ECC error for ivec read"; 714 static struct afsr_error_table __cheetah_error_table[] = { 715 { CHAFSR_PERR, CHAFSR_PERR_msg }, 716 { CHAFSR_IERR, CHAFSR_IERR_msg }, 717 { CHAFSR_ISAP, CHAFSR_ISAP_msg }, 718 { CHAFSR_UCU, CHAFSR_UCU_msg }, 719 { CHAFSR_UCC, CHAFSR_UCC_msg }, 720 { CHAFSR_UE, CHAFSR_UE_msg }, 721 { CHAFSR_EDU, CHAFSR_EDU_msg }, 722 { CHAFSR_EMU, CHAFSR_EMU_msg }, 723 { CHAFSR_WDU, CHAFSR_WDU_msg }, 724 { CHAFSR_CPU, CHAFSR_CPU_msg }, 725 { CHAFSR_CE, CHAFSR_CE_msg }, 726 { CHAFSR_EDC, CHAFSR_EDC_msg }, 727 { CHAFSR_EMC, CHAFSR_EMC_msg }, 728 { CHAFSR_WDC, CHAFSR_WDC_msg }, 729 { CHAFSR_CPC, CHAFSR_CPC_msg }, 730 { CHAFSR_TO, CHAFSR_TO_msg }, 731 { CHAFSR_BERR, CHAFSR_BERR_msg }, 732 /* These two do not update the AFAR. */ 733 { CHAFSR_IVC, CHAFSR_IVC_msg }, 734 { CHAFSR_IVU, CHAFSR_IVU_msg }, 735 { 0, NULL }, 736 }; 737 static const char CHPAFSR_DTO_msg[] = 738 "System bus unmapped error for prefetch/storequeue-read"; 739 static const char CHPAFSR_DBERR_msg[] = 740 "System bus error for prefetch/storequeue-read"; 741 static const char CHPAFSR_THCE_msg[] = 742 "Hardware corrected E-cache Tag ECC error"; 743 static const char CHPAFSR_TSCE_msg[] = 744 "SW handled correctable E-cache Tag ECC error"; 745 static const char CHPAFSR_TUE_msg[] = 746 "Uncorrectable E-cache Tag ECC error"; 747 static const char CHPAFSR_DUE_msg[] = 748 "System bus uncorrectable data ECC error due to prefetch/store-fill"; 749 static struct afsr_error_table __cheetah_plus_error_table[] = { 750 { CHAFSR_PERR, CHAFSR_PERR_msg }, 751 { CHAFSR_IERR, CHAFSR_IERR_msg }, 752 { CHAFSR_ISAP, CHAFSR_ISAP_msg }, 753 { CHAFSR_UCU, CHAFSR_UCU_msg }, 754 { CHAFSR_UCC, CHAFSR_UCC_msg }, 755 { CHAFSR_UE, CHAFSR_UE_msg }, 756 { CHAFSR_EDU, CHAFSR_EDU_msg }, 757 { CHAFSR_EMU, CHAFSR_EMU_msg }, 758 { CHAFSR_WDU, CHAFSR_WDU_msg }, 759 { CHAFSR_CPU, CHAFSR_CPU_msg }, 760 { CHAFSR_CE, CHAFSR_CE_msg }, 761 { CHAFSR_EDC, CHAFSR_EDC_msg }, 762 { CHAFSR_EMC, CHAFSR_EMC_msg }, 763 { CHAFSR_WDC, CHAFSR_WDC_msg }, 764 { CHAFSR_CPC, CHAFSR_CPC_msg }, 765 { CHAFSR_TO, CHAFSR_TO_msg }, 766 { CHAFSR_BERR, CHAFSR_BERR_msg }, 767 { CHPAFSR_DTO, CHPAFSR_DTO_msg }, 768 { CHPAFSR_DBERR, CHPAFSR_DBERR_msg }, 769 { CHPAFSR_THCE, CHPAFSR_THCE_msg }, 770 { CHPAFSR_TSCE, CHPAFSR_TSCE_msg }, 771 { CHPAFSR_TUE, CHPAFSR_TUE_msg }, 772 { CHPAFSR_DUE, CHPAFSR_DUE_msg }, 773 /* These two do not update the AFAR. */ 774 { CHAFSR_IVC, CHAFSR_IVC_msg }, 775 { CHAFSR_IVU, CHAFSR_IVU_msg }, 776 { 0, NULL }, 777 }; 778 static const char JPAFSR_JETO_msg[] = 779 "System interface protocol error, hw timeout caused"; 780 static const char JPAFSR_SCE_msg[] = 781 "Parity error on system snoop results"; 782 static const char JPAFSR_JEIC_msg[] = 783 "System interface protocol error, illegal command detected"; 784 static const char JPAFSR_JEIT_msg[] = 785 "System interface protocol error, illegal ADTYPE detected"; 786 static const char JPAFSR_OM_msg[] = 787 "Out of range memory error has occurred"; 788 static const char JPAFSR_ETP_msg[] = 789 "Parity error on L2 cache tag SRAM"; 790 static const char JPAFSR_UMS_msg[] = 791 "Error due to unsupported store"; 792 static const char JPAFSR_RUE_msg[] = 793 "Uncorrectable ECC error from remote cache/memory"; 794 static const char JPAFSR_RCE_msg[] = 795 "Correctable ECC error from remote cache/memory"; 796 static const char JPAFSR_BP_msg[] = 797 "JBUS parity error on returned read data"; 798 static const char JPAFSR_WBP_msg[] = 799 "JBUS parity error on data for writeback or block store"; 800 static const char JPAFSR_FRC_msg[] = 801 "Foreign read to DRAM incurring correctable ECC error"; 802 static const char JPAFSR_FRU_msg[] = 803 "Foreign read to DRAM incurring uncorrectable ECC error"; 804 static struct afsr_error_table __jalapeno_error_table[] = { 805 { JPAFSR_JETO, JPAFSR_JETO_msg }, 806 { JPAFSR_SCE, JPAFSR_SCE_msg }, 807 { JPAFSR_JEIC, JPAFSR_JEIC_msg }, 808 { JPAFSR_JEIT, JPAFSR_JEIT_msg }, 809 { CHAFSR_PERR, CHAFSR_PERR_msg }, 810 { CHAFSR_IERR, CHAFSR_IERR_msg }, 811 { CHAFSR_ISAP, CHAFSR_ISAP_msg }, 812 { CHAFSR_UCU, CHAFSR_UCU_msg }, 813 { CHAFSR_UCC, CHAFSR_UCC_msg }, 814 { CHAFSR_UE, CHAFSR_UE_msg }, 815 { CHAFSR_EDU, CHAFSR_EDU_msg }, 816 { JPAFSR_OM, JPAFSR_OM_msg }, 817 { CHAFSR_WDU, CHAFSR_WDU_msg }, 818 { CHAFSR_CPU, CHAFSR_CPU_msg }, 819 { CHAFSR_CE, CHAFSR_CE_msg }, 820 { CHAFSR_EDC, CHAFSR_EDC_msg }, 821 { JPAFSR_ETP, JPAFSR_ETP_msg }, 822 { CHAFSR_WDC, CHAFSR_WDC_msg }, 823 { CHAFSR_CPC, CHAFSR_CPC_msg }, 824 { CHAFSR_TO, CHAFSR_TO_msg }, 825 { CHAFSR_BERR, CHAFSR_BERR_msg }, 826 { JPAFSR_UMS, JPAFSR_UMS_msg }, 827 { JPAFSR_RUE, JPAFSR_RUE_msg }, 828 { JPAFSR_RCE, JPAFSR_RCE_msg }, 829 { JPAFSR_BP, JPAFSR_BP_msg }, 830 { JPAFSR_WBP, JPAFSR_WBP_msg }, 831 { JPAFSR_FRC, JPAFSR_FRC_msg }, 832 { JPAFSR_FRU, JPAFSR_FRU_msg }, 833 /* These two do not update the AFAR. */ 834 { CHAFSR_IVU, CHAFSR_IVU_msg }, 835 { 0, NULL }, 836 }; 837 static struct afsr_error_table *cheetah_error_table; 838 static unsigned long cheetah_afsr_errors; 839 840 struct cheetah_err_info *cheetah_error_log; 841 842 static inline struct cheetah_err_info *cheetah_get_error_log(unsigned long afsr) 843 { 844 struct cheetah_err_info *p; 845 int cpu = smp_processor_id(); 846 847 if (!cheetah_error_log) 848 return NULL; 849 850 p = cheetah_error_log + (cpu * 2); 851 if ((afsr & CHAFSR_TL1) != 0UL) 852 p++; 853 854 return p; 855 } 856 857 extern unsigned int tl0_icpe[], tl1_icpe[]; 858 extern unsigned int tl0_dcpe[], tl1_dcpe[]; 859 extern unsigned int tl0_fecc[], tl1_fecc[]; 860 extern unsigned int tl0_cee[], tl1_cee[]; 861 extern unsigned int tl0_iae[], tl1_iae[]; 862 extern unsigned int tl0_dae[], tl1_dae[]; 863 extern unsigned int cheetah_plus_icpe_trap_vector[], cheetah_plus_icpe_trap_vector_tl1[]; 864 extern unsigned int cheetah_plus_dcpe_trap_vector[], cheetah_plus_dcpe_trap_vector_tl1[]; 865 extern unsigned int cheetah_fecc_trap_vector[], cheetah_fecc_trap_vector_tl1[]; 866 extern unsigned int cheetah_cee_trap_vector[], cheetah_cee_trap_vector_tl1[]; 867 extern unsigned int cheetah_deferred_trap_vector[], cheetah_deferred_trap_vector_tl1[]; 868 869 void __init cheetah_ecache_flush_init(void) 870 { 871 unsigned long largest_size, smallest_linesize, order, ver; 872 int i, sz; 873 874 /* Scan all cpu device tree nodes, note two values: 875 * 1) largest E-cache size 876 * 2) smallest E-cache line size 877 */ 878 largest_size = 0UL; 879 smallest_linesize = ~0UL; 880 881 for (i = 0; i < NR_CPUS; i++) { 882 unsigned long val; 883 884 val = cpu_data(i).ecache_size; 885 if (!val) 886 continue; 887 888 if (val > largest_size) 889 largest_size = val; 890 891 val = cpu_data(i).ecache_line_size; 892 if (val < smallest_linesize) 893 smallest_linesize = val; 894 895 } 896 897 if (largest_size == 0UL || smallest_linesize == ~0UL) { 898 prom_printf("cheetah_ecache_flush_init: Cannot probe cpu E-cache " 899 "parameters.\n"); 900 prom_halt(); 901 } 902 903 ecache_flush_size = (2 * largest_size); 904 ecache_flush_linesize = smallest_linesize; 905 906 ecache_flush_physbase = find_ecache_flush_span(ecache_flush_size); 907 908 if (ecache_flush_physbase == ~0UL) { 909 prom_printf("cheetah_ecache_flush_init: Cannot find %ld byte " 910 "contiguous physical memory.\n", 911 ecache_flush_size); 912 prom_halt(); 913 } 914 915 /* Now allocate error trap reporting scoreboard. */ 916 sz = NR_CPUS * (2 * sizeof(struct cheetah_err_info)); 917 for (order = 0; order < MAX_ORDER; order++) { 918 if ((PAGE_SIZE << order) >= sz) 919 break; 920 } 921 cheetah_error_log = (struct cheetah_err_info *) 922 __get_free_pages(GFP_KERNEL, order); 923 if (!cheetah_error_log) { 924 prom_printf("cheetah_ecache_flush_init: Failed to allocate " 925 "error logging scoreboard (%d bytes).\n", sz); 926 prom_halt(); 927 } 928 memset(cheetah_error_log, 0, PAGE_SIZE << order); 929 930 /* Mark all AFSRs as invalid so that the trap handler will 931 * log new new information there. 932 */ 933 for (i = 0; i < 2 * NR_CPUS; i++) 934 cheetah_error_log[i].afsr = CHAFSR_INVALID; 935 936 __asm__ ("rdpr %%ver, %0" : "=r" (ver)); 937 if ((ver >> 32) == __JALAPENO_ID || 938 (ver >> 32) == __SERRANO_ID) { 939 cheetah_error_table = &__jalapeno_error_table[0]; 940 cheetah_afsr_errors = JPAFSR_ERRORS; 941 } else if ((ver >> 32) == 0x003e0015) { 942 cheetah_error_table = &__cheetah_plus_error_table[0]; 943 cheetah_afsr_errors = CHPAFSR_ERRORS; 944 } else { 945 cheetah_error_table = &__cheetah_error_table[0]; 946 cheetah_afsr_errors = CHAFSR_ERRORS; 947 } 948 949 /* Now patch trap tables. */ 950 memcpy(tl0_fecc, cheetah_fecc_trap_vector, (8 * 4)); 951 memcpy(tl1_fecc, cheetah_fecc_trap_vector_tl1, (8 * 4)); 952 memcpy(tl0_cee, cheetah_cee_trap_vector, (8 * 4)); 953 memcpy(tl1_cee, cheetah_cee_trap_vector_tl1, (8 * 4)); 954 memcpy(tl0_iae, cheetah_deferred_trap_vector, (8 * 4)); 955 memcpy(tl1_iae, cheetah_deferred_trap_vector_tl1, (8 * 4)); 956 memcpy(tl0_dae, cheetah_deferred_trap_vector, (8 * 4)); 957 memcpy(tl1_dae, cheetah_deferred_trap_vector_tl1, (8 * 4)); 958 if (tlb_type == cheetah_plus) { 959 memcpy(tl0_dcpe, cheetah_plus_dcpe_trap_vector, (8 * 4)); 960 memcpy(tl1_dcpe, cheetah_plus_dcpe_trap_vector_tl1, (8 * 4)); 961 memcpy(tl0_icpe, cheetah_plus_icpe_trap_vector, (8 * 4)); 962 memcpy(tl1_icpe, cheetah_plus_icpe_trap_vector_tl1, (8 * 4)); 963 } 964 flushi(PAGE_OFFSET); 965 } 966 967 static void cheetah_flush_ecache(void) 968 { 969 unsigned long flush_base = ecache_flush_physbase; 970 unsigned long flush_linesize = ecache_flush_linesize; 971 unsigned long flush_size = ecache_flush_size; 972 973 __asm__ __volatile__("1: subcc %0, %4, %0\n\t" 974 " bne,pt %%xcc, 1b\n\t" 975 " ldxa [%2 + %0] %3, %%g0\n\t" 976 : "=&r" (flush_size) 977 : "0" (flush_size), "r" (flush_base), 978 "i" (ASI_PHYS_USE_EC), "r" (flush_linesize)); 979 } 980 981 static void cheetah_flush_ecache_line(unsigned long physaddr) 982 { 983 unsigned long alias; 984 985 physaddr &= ~(8UL - 1UL); 986 physaddr = (ecache_flush_physbase + 987 (physaddr & ((ecache_flush_size>>1UL) - 1UL))); 988 alias = physaddr + (ecache_flush_size >> 1UL); 989 __asm__ __volatile__("ldxa [%0] %2, %%g0\n\t" 990 "ldxa [%1] %2, %%g0\n\t" 991 "membar #Sync" 992 : /* no outputs */ 993 : "r" (physaddr), "r" (alias), 994 "i" (ASI_PHYS_USE_EC)); 995 } 996 997 /* Unfortunately, the diagnostic access to the I-cache tags we need to 998 * use to clear the thing interferes with I-cache coherency transactions. 999 * 1000 * So we must only flush the I-cache when it is disabled. 1001 */ 1002 static void __cheetah_flush_icache(void) 1003 { 1004 unsigned int icache_size, icache_line_size; 1005 unsigned long addr; 1006 1007 icache_size = local_cpu_data().icache_size; 1008 icache_line_size = local_cpu_data().icache_line_size; 1009 1010 /* Clear the valid bits in all the tags. */ 1011 for (addr = 0; addr < icache_size; addr += icache_line_size) { 1012 __asm__ __volatile__("stxa %%g0, [%0] %1\n\t" 1013 "membar #Sync" 1014 : /* no outputs */ 1015 : "r" (addr | (2 << 3)), 1016 "i" (ASI_IC_TAG)); 1017 } 1018 } 1019 1020 static void cheetah_flush_icache(void) 1021 { 1022 unsigned long dcu_save; 1023 1024 /* Save current DCU, disable I-cache. */ 1025 __asm__ __volatile__("ldxa [%%g0] %1, %0\n\t" 1026 "or %0, %2, %%g1\n\t" 1027 "stxa %%g1, [%%g0] %1\n\t" 1028 "membar #Sync" 1029 : "=r" (dcu_save) 1030 : "i" (ASI_DCU_CONTROL_REG), "i" (DCU_IC) 1031 : "g1"); 1032 1033 __cheetah_flush_icache(); 1034 1035 /* Restore DCU register */ 1036 __asm__ __volatile__("stxa %0, [%%g0] %1\n\t" 1037 "membar #Sync" 1038 : /* no outputs */ 1039 : "r" (dcu_save), "i" (ASI_DCU_CONTROL_REG)); 1040 } 1041 1042 static void cheetah_flush_dcache(void) 1043 { 1044 unsigned int dcache_size, dcache_line_size; 1045 unsigned long addr; 1046 1047 dcache_size = local_cpu_data().dcache_size; 1048 dcache_line_size = local_cpu_data().dcache_line_size; 1049 1050 for (addr = 0; addr < dcache_size; addr += dcache_line_size) { 1051 __asm__ __volatile__("stxa %%g0, [%0] %1\n\t" 1052 "membar #Sync" 1053 : /* no outputs */ 1054 : "r" (addr), "i" (ASI_DCACHE_TAG)); 1055 } 1056 } 1057 1058 /* In order to make the even parity correct we must do two things. 1059 * First, we clear DC_data_parity and set DC_utag to an appropriate value. 1060 * Next, we clear out all 32-bytes of data for that line. Data of 1061 * all-zero + tag parity value of zero == correct parity. 1062 */ 1063 static void cheetah_plus_zap_dcache_parity(void) 1064 { 1065 unsigned int dcache_size, dcache_line_size; 1066 unsigned long addr; 1067 1068 dcache_size = local_cpu_data().dcache_size; 1069 dcache_line_size = local_cpu_data().dcache_line_size; 1070 1071 for (addr = 0; addr < dcache_size; addr += dcache_line_size) { 1072 unsigned long tag = (addr >> 14); 1073 unsigned long line; 1074 1075 __asm__ __volatile__("membar #Sync\n\t" 1076 "stxa %0, [%1] %2\n\t" 1077 "membar #Sync" 1078 : /* no outputs */ 1079 : "r" (tag), "r" (addr), 1080 "i" (ASI_DCACHE_UTAG)); 1081 for (line = addr; line < addr + dcache_line_size; line += 8) 1082 __asm__ __volatile__("membar #Sync\n\t" 1083 "stxa %%g0, [%0] %1\n\t" 1084 "membar #Sync" 1085 : /* no outputs */ 1086 : "r" (line), 1087 "i" (ASI_DCACHE_DATA)); 1088 } 1089 } 1090 1091 /* Conversion tables used to frob Cheetah AFSR syndrome values into 1092 * something palatable to the memory controller driver get_unumber 1093 * routine. 1094 */ 1095 #define MT0 137 1096 #define MT1 138 1097 #define MT2 139 1098 #define NONE 254 1099 #define MTC0 140 1100 #define MTC1 141 1101 #define MTC2 142 1102 #define MTC3 143 1103 #define C0 128 1104 #define C1 129 1105 #define C2 130 1106 #define C3 131 1107 #define C4 132 1108 #define C5 133 1109 #define C6 134 1110 #define C7 135 1111 #define C8 136 1112 #define M2 144 1113 #define M3 145 1114 #define M4 146 1115 #define M 147 1116 static unsigned char cheetah_ecc_syntab[] = { 1117 /*00*/NONE, C0, C1, M2, C2, M2, M3, 47, C3, M2, M2, 53, M2, 41, 29, M, 1118 /*01*/C4, M, M, 50, M2, 38, 25, M2, M2, 33, 24, M2, 11, M, M2, 16, 1119 /*02*/C5, M, M, 46, M2, 37, 19, M2, M, 31, 32, M, 7, M2, M2, 10, 1120 /*03*/M2, 40, 13, M2, 59, M, M2, 66, M, M2, M2, 0, M2, 67, 71, M, 1121 /*04*/C6, M, M, 43, M, 36, 18, M, M2, 49, 15, M, 63, M2, M2, 6, 1122 /*05*/M2, 44, 28, M2, M, M2, M2, 52, 68, M2, M2, 62, M2, M3, M3, M4, 1123 /*06*/M2, 26, 106, M2, 64, M, M2, 2, 120, M, M2, M3, M, M3, M3, M4, 1124 /*07*/116, M2, M2, M3, M2, M3, M, M4, M2, 58, 54, M2, M, M4, M4, M3, 1125 /*08*/C7, M2, M, 42, M, 35, 17, M2, M, 45, 14, M2, 21, M2, M2, 5, 1126 /*09*/M, 27, M, M, 99, M, M, 3, 114, M2, M2, 20, M2, M3, M3, M, 1127 /*0a*/M2, 23, 113, M2, 112, M2, M, 51, 95, M, M2, M3, M2, M3, M3, M2, 1128 /*0b*/103, M, M2, M3, M2, M3, M3, M4, M2, 48, M, M, 73, M2, M, M3, 1129 /*0c*/M2, 22, 110, M2, 109, M2, M, 9, 108, M2, M, M3, M2, M3, M3, M, 1130 /*0d*/102, M2, M, M, M2, M3, M3, M, M2, M3, M3, M2, M, M4, M, M3, 1131 /*0e*/98, M, M2, M3, M2, M, M3, M4, M2, M3, M3, M4, M3, M, M, M, 1132 /*0f*/M2, M3, M3, M, M3, M, M, M, 56, M4, M, M3, M4, M, M, M, 1133 /*10*/C8, M, M2, 39, M, 34, 105, M2, M, 30, 104, M, 101, M, M, 4, 1134 /*11*/M, M, 100, M, 83, M, M2, 12, 87, M, M, 57, M2, M, M3, M, 1135 /*12*/M2, 97, 82, M2, 78, M2, M2, 1, 96, M, M, M, M, M, M3, M2, 1136 /*13*/94, M, M2, M3, M2, M, M3, M, M2, M, 79, M, 69, M, M4, M, 1137 /*14*/M2, 93, 92, M, 91, M, M2, 8, 90, M2, M2, M, M, M, M, M4, 1138 /*15*/89, M, M, M3, M2, M3, M3, M, M, M, M3, M2, M3, M2, M, M3, 1139 /*16*/86, M, M2, M3, M2, M, M3, M, M2, M, M3, M, M3, M, M, M3, 1140 /*17*/M, M, M3, M2, M3, M2, M4, M, 60, M, M2, M3, M4, M, M, M2, 1141 /*18*/M2, 88, 85, M2, 84, M, M2, 55, 81, M2, M2, M3, M2, M3, M3, M4, 1142 /*19*/77, M, M, M, M2, M3, M, M, M2, M3, M3, M4, M3, M2, M, M, 1143 /*1a*/74, M, M2, M3, M, M, M3, M, M, M, M3, M, M3, M, M4, M3, 1144 /*1b*/M2, 70, 107, M4, 65, M2, M2, M, 127, M, M, M, M2, M3, M3, M, 1145 /*1c*/80, M2, M2, 72, M, 119, 118, M, M2, 126, 76, M, 125, M, M4, M3, 1146 /*1d*/M2, 115, 124, M, 75, M, M, M3, 61, M, M4, M, M4, M, M, M, 1147 /*1e*/M, 123, 122, M4, 121, M4, M, M3, 117, M2, M2, M3, M4, M3, M, M, 1148 /*1f*/111, M, M, M, M4, M3, M3, M, M, M, M3, M, M3, M2, M, M 1149 }; 1150 static unsigned char cheetah_mtag_syntab[] = { 1151 NONE, MTC0, 1152 MTC1, NONE, 1153 MTC2, NONE, 1154 NONE, MT0, 1155 MTC3, NONE, 1156 NONE, MT1, 1157 NONE, MT2, 1158 NONE, NONE 1159 }; 1160 1161 /* Return the highest priority error conditon mentioned. */ 1162 static inline unsigned long cheetah_get_hipri(unsigned long afsr) 1163 { 1164 unsigned long tmp = 0; 1165 int i; 1166 1167 for (i = 0; cheetah_error_table[i].mask; i++) { 1168 if ((tmp = (afsr & cheetah_error_table[i].mask)) != 0UL) 1169 return tmp; 1170 } 1171 return tmp; 1172 } 1173 1174 static const char *cheetah_get_string(unsigned long bit) 1175 { 1176 int i; 1177 1178 for (i = 0; cheetah_error_table[i].mask; i++) { 1179 if ((bit & cheetah_error_table[i].mask) != 0UL) 1180 return cheetah_error_table[i].name; 1181 } 1182 return "???"; 1183 } 1184 1185 static void cheetah_log_errors(struct pt_regs *regs, struct cheetah_err_info *info, 1186 unsigned long afsr, unsigned long afar, int recoverable) 1187 { 1188 unsigned long hipri; 1189 char unum[256]; 1190 1191 printk("%s" "ERROR(%d): Cheetah error trap taken afsr[%016lx] afar[%016lx] TL1(%d)\n", 1192 (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(), 1193 afsr, afar, 1194 (afsr & CHAFSR_TL1) ? 1 : 0); 1195 printk("%s" "ERROR(%d): TPC[%lx] TNPC[%lx] O7[%lx] TSTATE[%lx]\n", 1196 (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(), 1197 regs->tpc, regs->tnpc, regs->u_regs[UREG_I7], regs->tstate); 1198 printk("%s" "ERROR(%d): ", 1199 (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id()); 1200 printk("TPC<%pS>\n", (void *) regs->tpc); 1201 printk("%s" "ERROR(%d): M_SYND(%lx), E_SYND(%lx)%s%s\n", 1202 (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(), 1203 (afsr & CHAFSR_M_SYNDROME) >> CHAFSR_M_SYNDROME_SHIFT, 1204 (afsr & CHAFSR_E_SYNDROME) >> CHAFSR_E_SYNDROME_SHIFT, 1205 (afsr & CHAFSR_ME) ? ", Multiple Errors" : "", 1206 (afsr & CHAFSR_PRIV) ? ", Privileged" : ""); 1207 hipri = cheetah_get_hipri(afsr); 1208 printk("%s" "ERROR(%d): Highest priority error (%016lx) \"%s\"\n", 1209 (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(), 1210 hipri, cheetah_get_string(hipri)); 1211 1212 /* Try to get unumber if relevant. */ 1213 #define ESYND_ERRORS (CHAFSR_IVC | CHAFSR_IVU | \ 1214 CHAFSR_CPC | CHAFSR_CPU | \ 1215 CHAFSR_UE | CHAFSR_CE | \ 1216 CHAFSR_EDC | CHAFSR_EDU | \ 1217 CHAFSR_UCC | CHAFSR_UCU | \ 1218 CHAFSR_WDU | CHAFSR_WDC) 1219 #define MSYND_ERRORS (CHAFSR_EMC | CHAFSR_EMU) 1220 if (afsr & ESYND_ERRORS) { 1221 int syndrome; 1222 int ret; 1223 1224 syndrome = (afsr & CHAFSR_E_SYNDROME) >> CHAFSR_E_SYNDROME_SHIFT; 1225 syndrome = cheetah_ecc_syntab[syndrome]; 1226 ret = sprintf_dimm(syndrome, afar, unum, sizeof(unum)); 1227 if (ret != -1) 1228 printk("%s" "ERROR(%d): AFAR E-syndrome [%s]\n", 1229 (recoverable ? KERN_WARNING : KERN_CRIT), 1230 smp_processor_id(), unum); 1231 } else if (afsr & MSYND_ERRORS) { 1232 int syndrome; 1233 int ret; 1234 1235 syndrome = (afsr & CHAFSR_M_SYNDROME) >> CHAFSR_M_SYNDROME_SHIFT; 1236 syndrome = cheetah_mtag_syntab[syndrome]; 1237 ret = sprintf_dimm(syndrome, afar, unum, sizeof(unum)); 1238 if (ret != -1) 1239 printk("%s" "ERROR(%d): AFAR M-syndrome [%s]\n", 1240 (recoverable ? KERN_WARNING : KERN_CRIT), 1241 smp_processor_id(), unum); 1242 } 1243 1244 /* Now dump the cache snapshots. */ 1245 printk("%s" "ERROR(%d): D-cache idx[%x] tag[%016llx] utag[%016llx] stag[%016llx]\n", 1246 (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(), 1247 (int) info->dcache_index, 1248 info->dcache_tag, 1249 info->dcache_utag, 1250 info->dcache_stag); 1251 printk("%s" "ERROR(%d): D-cache data0[%016llx] data1[%016llx] data2[%016llx] data3[%016llx]\n", 1252 (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(), 1253 info->dcache_data[0], 1254 info->dcache_data[1], 1255 info->dcache_data[2], 1256 info->dcache_data[3]); 1257 printk("%s" "ERROR(%d): I-cache idx[%x] tag[%016llx] utag[%016llx] stag[%016llx] " 1258 "u[%016llx] l[%016llx]\n", 1259 (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(), 1260 (int) info->icache_index, 1261 info->icache_tag, 1262 info->icache_utag, 1263 info->icache_stag, 1264 info->icache_upper, 1265 info->icache_lower); 1266 printk("%s" "ERROR(%d): I-cache INSN0[%016llx] INSN1[%016llx] INSN2[%016llx] INSN3[%016llx]\n", 1267 (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(), 1268 info->icache_data[0], 1269 info->icache_data[1], 1270 info->icache_data[2], 1271 info->icache_data[3]); 1272 printk("%s" "ERROR(%d): I-cache INSN4[%016llx] INSN5[%016llx] INSN6[%016llx] INSN7[%016llx]\n", 1273 (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(), 1274 info->icache_data[4], 1275 info->icache_data[5], 1276 info->icache_data[6], 1277 info->icache_data[7]); 1278 printk("%s" "ERROR(%d): E-cache idx[%x] tag[%016llx]\n", 1279 (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(), 1280 (int) info->ecache_index, info->ecache_tag); 1281 printk("%s" "ERROR(%d): E-cache data0[%016llx] data1[%016llx] data2[%016llx] data3[%016llx]\n", 1282 (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(), 1283 info->ecache_data[0], 1284 info->ecache_data[1], 1285 info->ecache_data[2], 1286 info->ecache_data[3]); 1287 1288 afsr = (afsr & ~hipri) & cheetah_afsr_errors; 1289 while (afsr != 0UL) { 1290 unsigned long bit = cheetah_get_hipri(afsr); 1291 1292 printk("%s" "ERROR: Multiple-error (%016lx) \"%s\"\n", 1293 (recoverable ? KERN_WARNING : KERN_CRIT), 1294 bit, cheetah_get_string(bit)); 1295 1296 afsr &= ~bit; 1297 } 1298 1299 if (!recoverable) 1300 printk(KERN_CRIT "ERROR: This condition is not recoverable.\n"); 1301 } 1302 1303 static int cheetah_recheck_errors(struct cheetah_err_info *logp) 1304 { 1305 unsigned long afsr, afar; 1306 int ret = 0; 1307 1308 __asm__ __volatile__("ldxa [%%g0] %1, %0\n\t" 1309 : "=r" (afsr) 1310 : "i" (ASI_AFSR)); 1311 if ((afsr & cheetah_afsr_errors) != 0) { 1312 if (logp != NULL) { 1313 __asm__ __volatile__("ldxa [%%g0] %1, %0\n\t" 1314 : "=r" (afar) 1315 : "i" (ASI_AFAR)); 1316 logp->afsr = afsr; 1317 logp->afar = afar; 1318 } 1319 ret = 1; 1320 } 1321 __asm__ __volatile__("stxa %0, [%%g0] %1\n\t" 1322 "membar #Sync\n\t" 1323 : : "r" (afsr), "i" (ASI_AFSR)); 1324 1325 return ret; 1326 } 1327 1328 void cheetah_fecc_handler(struct pt_regs *regs, unsigned long afsr, unsigned long afar) 1329 { 1330 struct cheetah_err_info local_snapshot, *p; 1331 int recoverable; 1332 1333 /* Flush E-cache */ 1334 cheetah_flush_ecache(); 1335 1336 p = cheetah_get_error_log(afsr); 1337 if (!p) { 1338 prom_printf("ERROR: Early Fast-ECC error afsr[%016lx] afar[%016lx]\n", 1339 afsr, afar); 1340 prom_printf("ERROR: CPU(%d) TPC[%016lx] TNPC[%016lx] TSTATE[%016lx]\n", 1341 smp_processor_id(), regs->tpc, regs->tnpc, regs->tstate); 1342 prom_halt(); 1343 } 1344 1345 /* Grab snapshot of logged error. */ 1346 memcpy(&local_snapshot, p, sizeof(local_snapshot)); 1347 1348 /* If the current trap snapshot does not match what the 1349 * trap handler passed along into our args, big trouble. 1350 * In such a case, mark the local copy as invalid. 1351 * 1352 * Else, it matches and we mark the afsr in the non-local 1353 * copy as invalid so we may log new error traps there. 1354 */ 1355 if (p->afsr != afsr || p->afar != afar) 1356 local_snapshot.afsr = CHAFSR_INVALID; 1357 else 1358 p->afsr = CHAFSR_INVALID; 1359 1360 cheetah_flush_icache(); 1361 cheetah_flush_dcache(); 1362 1363 /* Re-enable I-cache/D-cache */ 1364 __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t" 1365 "or %%g1, %1, %%g1\n\t" 1366 "stxa %%g1, [%%g0] %0\n\t" 1367 "membar #Sync" 1368 : /* no outputs */ 1369 : "i" (ASI_DCU_CONTROL_REG), 1370 "i" (DCU_DC | DCU_IC) 1371 : "g1"); 1372 1373 /* Re-enable error reporting */ 1374 __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t" 1375 "or %%g1, %1, %%g1\n\t" 1376 "stxa %%g1, [%%g0] %0\n\t" 1377 "membar #Sync" 1378 : /* no outputs */ 1379 : "i" (ASI_ESTATE_ERROR_EN), 1380 "i" (ESTATE_ERROR_NCEEN | ESTATE_ERROR_CEEN) 1381 : "g1"); 1382 1383 /* Decide if we can continue after handling this trap and 1384 * logging the error. 1385 */ 1386 recoverable = 1; 1387 if (afsr & (CHAFSR_PERR | CHAFSR_IERR | CHAFSR_ISAP)) 1388 recoverable = 0; 1389 1390 /* Re-check AFSR/AFAR. What we are looking for here is whether a new 1391 * error was logged while we had error reporting traps disabled. 1392 */ 1393 if (cheetah_recheck_errors(&local_snapshot)) { 1394 unsigned long new_afsr = local_snapshot.afsr; 1395 1396 /* If we got a new asynchronous error, die... */ 1397 if (new_afsr & (CHAFSR_EMU | CHAFSR_EDU | 1398 CHAFSR_WDU | CHAFSR_CPU | 1399 CHAFSR_IVU | CHAFSR_UE | 1400 CHAFSR_BERR | CHAFSR_TO)) 1401 recoverable = 0; 1402 } 1403 1404 /* Log errors. */ 1405 cheetah_log_errors(regs, &local_snapshot, afsr, afar, recoverable); 1406 1407 if (!recoverable) 1408 panic("Irrecoverable Fast-ECC error trap.\n"); 1409 1410 /* Flush E-cache to kick the error trap handlers out. */ 1411 cheetah_flush_ecache(); 1412 } 1413 1414 /* Try to fix a correctable error by pushing the line out from 1415 * the E-cache. Recheck error reporting registers to see if the 1416 * problem is intermittent. 1417 */ 1418 static int cheetah_fix_ce(unsigned long physaddr) 1419 { 1420 unsigned long orig_estate; 1421 unsigned long alias1, alias2; 1422 int ret; 1423 1424 /* Make sure correctable error traps are disabled. */ 1425 __asm__ __volatile__("ldxa [%%g0] %2, %0\n\t" 1426 "andn %0, %1, %%g1\n\t" 1427 "stxa %%g1, [%%g0] %2\n\t" 1428 "membar #Sync" 1429 : "=&r" (orig_estate) 1430 : "i" (ESTATE_ERROR_CEEN), 1431 "i" (ASI_ESTATE_ERROR_EN) 1432 : "g1"); 1433 1434 /* We calculate alias addresses that will force the 1435 * cache line in question out of the E-cache. Then 1436 * we bring it back in with an atomic instruction so 1437 * that we get it in some modified/exclusive state, 1438 * then we displace it again to try and get proper ECC 1439 * pushed back into the system. 1440 */ 1441 physaddr &= ~(8UL - 1UL); 1442 alias1 = (ecache_flush_physbase + 1443 (physaddr & ((ecache_flush_size >> 1) - 1))); 1444 alias2 = alias1 + (ecache_flush_size >> 1); 1445 __asm__ __volatile__("ldxa [%0] %3, %%g0\n\t" 1446 "ldxa [%1] %3, %%g0\n\t" 1447 "casxa [%2] %3, %%g0, %%g0\n\t" 1448 "ldxa [%0] %3, %%g0\n\t" 1449 "ldxa [%1] %3, %%g0\n\t" 1450 "membar #Sync" 1451 : /* no outputs */ 1452 : "r" (alias1), "r" (alias2), 1453 "r" (physaddr), "i" (ASI_PHYS_USE_EC)); 1454 1455 /* Did that trigger another error? */ 1456 if (cheetah_recheck_errors(NULL)) { 1457 /* Try one more time. */ 1458 __asm__ __volatile__("ldxa [%0] %1, %%g0\n\t" 1459 "membar #Sync" 1460 : : "r" (physaddr), "i" (ASI_PHYS_USE_EC)); 1461 if (cheetah_recheck_errors(NULL)) 1462 ret = 2; 1463 else 1464 ret = 1; 1465 } else { 1466 /* No new error, intermittent problem. */ 1467 ret = 0; 1468 } 1469 1470 /* Restore error enables. */ 1471 __asm__ __volatile__("stxa %0, [%%g0] %1\n\t" 1472 "membar #Sync" 1473 : : "r" (orig_estate), "i" (ASI_ESTATE_ERROR_EN)); 1474 1475 return ret; 1476 } 1477 1478 /* Return non-zero if PADDR is a valid physical memory address. */ 1479 static int cheetah_check_main_memory(unsigned long paddr) 1480 { 1481 unsigned long vaddr = PAGE_OFFSET + paddr; 1482 1483 if (vaddr > (unsigned long) high_memory) 1484 return 0; 1485 1486 return kern_addr_valid(vaddr); 1487 } 1488 1489 void cheetah_cee_handler(struct pt_regs *regs, unsigned long afsr, unsigned long afar) 1490 { 1491 struct cheetah_err_info local_snapshot, *p; 1492 int recoverable, is_memory; 1493 1494 p = cheetah_get_error_log(afsr); 1495 if (!p) { 1496 prom_printf("ERROR: Early CEE error afsr[%016lx] afar[%016lx]\n", 1497 afsr, afar); 1498 prom_printf("ERROR: CPU(%d) TPC[%016lx] TNPC[%016lx] TSTATE[%016lx]\n", 1499 smp_processor_id(), regs->tpc, regs->tnpc, regs->tstate); 1500 prom_halt(); 1501 } 1502 1503 /* Grab snapshot of logged error. */ 1504 memcpy(&local_snapshot, p, sizeof(local_snapshot)); 1505 1506 /* If the current trap snapshot does not match what the 1507 * trap handler passed along into our args, big trouble. 1508 * In such a case, mark the local copy as invalid. 1509 * 1510 * Else, it matches and we mark the afsr in the non-local 1511 * copy as invalid so we may log new error traps there. 1512 */ 1513 if (p->afsr != afsr || p->afar != afar) 1514 local_snapshot.afsr = CHAFSR_INVALID; 1515 else 1516 p->afsr = CHAFSR_INVALID; 1517 1518 is_memory = cheetah_check_main_memory(afar); 1519 1520 if (is_memory && (afsr & CHAFSR_CE) != 0UL) { 1521 /* XXX Might want to log the results of this operation 1522 * XXX somewhere... -DaveM 1523 */ 1524 cheetah_fix_ce(afar); 1525 } 1526 1527 { 1528 int flush_all, flush_line; 1529 1530 flush_all = flush_line = 0; 1531 if ((afsr & CHAFSR_EDC) != 0UL) { 1532 if ((afsr & cheetah_afsr_errors) == CHAFSR_EDC) 1533 flush_line = 1; 1534 else 1535 flush_all = 1; 1536 } else if ((afsr & CHAFSR_CPC) != 0UL) { 1537 if ((afsr & cheetah_afsr_errors) == CHAFSR_CPC) 1538 flush_line = 1; 1539 else 1540 flush_all = 1; 1541 } 1542 1543 /* Trap handler only disabled I-cache, flush it. */ 1544 cheetah_flush_icache(); 1545 1546 /* Re-enable I-cache */ 1547 __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t" 1548 "or %%g1, %1, %%g1\n\t" 1549 "stxa %%g1, [%%g0] %0\n\t" 1550 "membar #Sync" 1551 : /* no outputs */ 1552 : "i" (ASI_DCU_CONTROL_REG), 1553 "i" (DCU_IC) 1554 : "g1"); 1555 1556 if (flush_all) 1557 cheetah_flush_ecache(); 1558 else if (flush_line) 1559 cheetah_flush_ecache_line(afar); 1560 } 1561 1562 /* Re-enable error reporting */ 1563 __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t" 1564 "or %%g1, %1, %%g1\n\t" 1565 "stxa %%g1, [%%g0] %0\n\t" 1566 "membar #Sync" 1567 : /* no outputs */ 1568 : "i" (ASI_ESTATE_ERROR_EN), 1569 "i" (ESTATE_ERROR_CEEN) 1570 : "g1"); 1571 1572 /* Decide if we can continue after handling this trap and 1573 * logging the error. 1574 */ 1575 recoverable = 1; 1576 if (afsr & (CHAFSR_PERR | CHAFSR_IERR | CHAFSR_ISAP)) 1577 recoverable = 0; 1578 1579 /* Re-check AFSR/AFAR */ 1580 (void) cheetah_recheck_errors(&local_snapshot); 1581 1582 /* Log errors. */ 1583 cheetah_log_errors(regs, &local_snapshot, afsr, afar, recoverable); 1584 1585 if (!recoverable) 1586 panic("Irrecoverable Correctable-ECC error trap.\n"); 1587 } 1588 1589 void cheetah_deferred_handler(struct pt_regs *regs, unsigned long afsr, unsigned long afar) 1590 { 1591 struct cheetah_err_info local_snapshot, *p; 1592 int recoverable, is_memory; 1593 1594 #ifdef CONFIG_PCI 1595 /* Check for the special PCI poke sequence. */ 1596 if (pci_poke_in_progress && pci_poke_cpu == smp_processor_id()) { 1597 cheetah_flush_icache(); 1598 cheetah_flush_dcache(); 1599 1600 /* Re-enable I-cache/D-cache */ 1601 __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t" 1602 "or %%g1, %1, %%g1\n\t" 1603 "stxa %%g1, [%%g0] %0\n\t" 1604 "membar #Sync" 1605 : /* no outputs */ 1606 : "i" (ASI_DCU_CONTROL_REG), 1607 "i" (DCU_DC | DCU_IC) 1608 : "g1"); 1609 1610 /* Re-enable error reporting */ 1611 __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t" 1612 "or %%g1, %1, %%g1\n\t" 1613 "stxa %%g1, [%%g0] %0\n\t" 1614 "membar #Sync" 1615 : /* no outputs */ 1616 : "i" (ASI_ESTATE_ERROR_EN), 1617 "i" (ESTATE_ERROR_NCEEN | ESTATE_ERROR_CEEN) 1618 : "g1"); 1619 1620 (void) cheetah_recheck_errors(NULL); 1621 1622 pci_poke_faulted = 1; 1623 regs->tpc += 4; 1624 regs->tnpc = regs->tpc + 4; 1625 return; 1626 } 1627 #endif 1628 1629 p = cheetah_get_error_log(afsr); 1630 if (!p) { 1631 prom_printf("ERROR: Early deferred error afsr[%016lx] afar[%016lx]\n", 1632 afsr, afar); 1633 prom_printf("ERROR: CPU(%d) TPC[%016lx] TNPC[%016lx] TSTATE[%016lx]\n", 1634 smp_processor_id(), regs->tpc, regs->tnpc, regs->tstate); 1635 prom_halt(); 1636 } 1637 1638 /* Grab snapshot of logged error. */ 1639 memcpy(&local_snapshot, p, sizeof(local_snapshot)); 1640 1641 /* If the current trap snapshot does not match what the 1642 * trap handler passed along into our args, big trouble. 1643 * In such a case, mark the local copy as invalid. 1644 * 1645 * Else, it matches and we mark the afsr in the non-local 1646 * copy as invalid so we may log new error traps there. 1647 */ 1648 if (p->afsr != afsr || p->afar != afar) 1649 local_snapshot.afsr = CHAFSR_INVALID; 1650 else 1651 p->afsr = CHAFSR_INVALID; 1652 1653 is_memory = cheetah_check_main_memory(afar); 1654 1655 { 1656 int flush_all, flush_line; 1657 1658 flush_all = flush_line = 0; 1659 if ((afsr & CHAFSR_EDU) != 0UL) { 1660 if ((afsr & cheetah_afsr_errors) == CHAFSR_EDU) 1661 flush_line = 1; 1662 else 1663 flush_all = 1; 1664 } else if ((afsr & CHAFSR_BERR) != 0UL) { 1665 if ((afsr & cheetah_afsr_errors) == CHAFSR_BERR) 1666 flush_line = 1; 1667 else 1668 flush_all = 1; 1669 } 1670 1671 cheetah_flush_icache(); 1672 cheetah_flush_dcache(); 1673 1674 /* Re-enable I/D caches */ 1675 __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t" 1676 "or %%g1, %1, %%g1\n\t" 1677 "stxa %%g1, [%%g0] %0\n\t" 1678 "membar #Sync" 1679 : /* no outputs */ 1680 : "i" (ASI_DCU_CONTROL_REG), 1681 "i" (DCU_IC | DCU_DC) 1682 : "g1"); 1683 1684 if (flush_all) 1685 cheetah_flush_ecache(); 1686 else if (flush_line) 1687 cheetah_flush_ecache_line(afar); 1688 } 1689 1690 /* Re-enable error reporting */ 1691 __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t" 1692 "or %%g1, %1, %%g1\n\t" 1693 "stxa %%g1, [%%g0] %0\n\t" 1694 "membar #Sync" 1695 : /* no outputs */ 1696 : "i" (ASI_ESTATE_ERROR_EN), 1697 "i" (ESTATE_ERROR_NCEEN | ESTATE_ERROR_CEEN) 1698 : "g1"); 1699 1700 /* Decide if we can continue after handling this trap and 1701 * logging the error. 1702 */ 1703 recoverable = 1; 1704 if (afsr & (CHAFSR_PERR | CHAFSR_IERR | CHAFSR_ISAP)) 1705 recoverable = 0; 1706 1707 /* Re-check AFSR/AFAR. What we are looking for here is whether a new 1708 * error was logged while we had error reporting traps disabled. 1709 */ 1710 if (cheetah_recheck_errors(&local_snapshot)) { 1711 unsigned long new_afsr = local_snapshot.afsr; 1712 1713 /* If we got a new asynchronous error, die... */ 1714 if (new_afsr & (CHAFSR_EMU | CHAFSR_EDU | 1715 CHAFSR_WDU | CHAFSR_CPU | 1716 CHAFSR_IVU | CHAFSR_UE | 1717 CHAFSR_BERR | CHAFSR_TO)) 1718 recoverable = 0; 1719 } 1720 1721 /* Log errors. */ 1722 cheetah_log_errors(regs, &local_snapshot, afsr, afar, recoverable); 1723 1724 /* "Recoverable" here means we try to yank the page from ever 1725 * being newly used again. This depends upon a few things: 1726 * 1) Must be main memory, and AFAR must be valid. 1727 * 2) If we trapped from user, OK. 1728 * 3) Else, if we trapped from kernel we must find exception 1729 * table entry (ie. we have to have been accessing user 1730 * space). 1731 * 1732 * If AFAR is not in main memory, or we trapped from kernel 1733 * and cannot find an exception table entry, it is unacceptable 1734 * to try and continue. 1735 */ 1736 if (recoverable && is_memory) { 1737 if ((regs->tstate & TSTATE_PRIV) == 0UL) { 1738 /* OK, usermode access. */ 1739 recoverable = 1; 1740 } else { 1741 const struct exception_table_entry *entry; 1742 1743 entry = search_exception_tables(regs->tpc); 1744 if (entry) { 1745 /* OK, kernel access to userspace. */ 1746 recoverable = 1; 1747 1748 } else { 1749 /* BAD, privileged state is corrupted. */ 1750 recoverable = 0; 1751 } 1752 1753 if (recoverable) { 1754 if (pfn_valid(afar >> PAGE_SHIFT)) 1755 get_page(pfn_to_page(afar >> PAGE_SHIFT)); 1756 else 1757 recoverable = 0; 1758 1759 /* Only perform fixup if we still have a 1760 * recoverable condition. 1761 */ 1762 if (recoverable) { 1763 regs->tpc = entry->fixup; 1764 regs->tnpc = regs->tpc + 4; 1765 } 1766 } 1767 } 1768 } else { 1769 recoverable = 0; 1770 } 1771 1772 if (!recoverable) 1773 panic("Irrecoverable deferred error trap.\n"); 1774 } 1775 1776 /* Handle a D/I cache parity error trap. TYPE is encoded as: 1777 * 1778 * Bit0: 0=dcache,1=icache 1779 * Bit1: 0=recoverable,1=unrecoverable 1780 * 1781 * The hardware has disabled both the I-cache and D-cache in 1782 * the %dcr register. 1783 */ 1784 void cheetah_plus_parity_error(int type, struct pt_regs *regs) 1785 { 1786 if (type & 0x1) 1787 __cheetah_flush_icache(); 1788 else 1789 cheetah_plus_zap_dcache_parity(); 1790 cheetah_flush_dcache(); 1791 1792 /* Re-enable I-cache/D-cache */ 1793 __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t" 1794 "or %%g1, %1, %%g1\n\t" 1795 "stxa %%g1, [%%g0] %0\n\t" 1796 "membar #Sync" 1797 : /* no outputs */ 1798 : "i" (ASI_DCU_CONTROL_REG), 1799 "i" (DCU_DC | DCU_IC) 1800 : "g1"); 1801 1802 if (type & 0x2) { 1803 printk(KERN_EMERG "CPU[%d]: Cheetah+ %c-cache parity error at TPC[%016lx]\n", 1804 smp_processor_id(), 1805 (type & 0x1) ? 'I' : 'D', 1806 regs->tpc); 1807 printk(KERN_EMERG "TPC<%pS>\n", (void *) regs->tpc); 1808 panic("Irrecoverable Cheetah+ parity error."); 1809 } 1810 1811 printk(KERN_WARNING "CPU[%d]: Cheetah+ %c-cache parity error at TPC[%016lx]\n", 1812 smp_processor_id(), 1813 (type & 0x1) ? 'I' : 'D', 1814 regs->tpc); 1815 printk(KERN_WARNING "TPC<%pS>\n", (void *) regs->tpc); 1816 } 1817 1818 struct sun4v_error_entry { 1819 /* Unique error handle */ 1820 /*0x00*/u64 err_handle; 1821 1822 /* %stick value at the time of the error */ 1823 /*0x08*/u64 err_stick; 1824 1825 /*0x10*/u8 reserved_1[3]; 1826 1827 /* Error type */ 1828 /*0x13*/u8 err_type; 1829 #define SUN4V_ERR_TYPE_UNDEFINED 0 1830 #define SUN4V_ERR_TYPE_UNCORRECTED_RES 1 1831 #define SUN4V_ERR_TYPE_PRECISE_NONRES 2 1832 #define SUN4V_ERR_TYPE_DEFERRED_NONRES 3 1833 #define SUN4V_ERR_TYPE_SHUTDOWN_RQST 4 1834 #define SUN4V_ERR_TYPE_DUMP_CORE 5 1835 #define SUN4V_ERR_TYPE_SP_STATE_CHANGE 6 1836 #define SUN4V_ERR_TYPE_NUM 7 1837 1838 /* Error attributes */ 1839 /*0x14*/u32 err_attrs; 1840 #define SUN4V_ERR_ATTRS_PROCESSOR 0x00000001 1841 #define SUN4V_ERR_ATTRS_MEMORY 0x00000002 1842 #define SUN4V_ERR_ATTRS_PIO 0x00000004 1843 #define SUN4V_ERR_ATTRS_INT_REGISTERS 0x00000008 1844 #define SUN4V_ERR_ATTRS_FPU_REGISTERS 0x00000010 1845 #define SUN4V_ERR_ATTRS_SHUTDOWN_RQST 0x00000020 1846 #define SUN4V_ERR_ATTRS_ASR 0x00000040 1847 #define SUN4V_ERR_ATTRS_ASI 0x00000080 1848 #define SUN4V_ERR_ATTRS_PRIV_REG 0x00000100 1849 #define SUN4V_ERR_ATTRS_SPSTATE_MSK 0x00000600 1850 #define SUN4V_ERR_ATTRS_SPSTATE_SHFT 9 1851 #define SUN4V_ERR_ATTRS_MODE_MSK 0x03000000 1852 #define SUN4V_ERR_ATTRS_MODE_SHFT 24 1853 #define SUN4V_ERR_ATTRS_RES_QUEUE_FULL 0x80000000 1854 1855 #define SUN4V_ERR_SPSTATE_FAULTED 0 1856 #define SUN4V_ERR_SPSTATE_AVAILABLE 1 1857 #define SUN4V_ERR_SPSTATE_NOT_PRESENT 2 1858 1859 #define SUN4V_ERR_MODE_USER 1 1860 #define SUN4V_ERR_MODE_PRIV 2 1861 1862 /* Real address of the memory region or PIO transaction */ 1863 /*0x18*/u64 err_raddr; 1864 1865 /* Size of the operation triggering the error, in bytes */ 1866 /*0x20*/u32 err_size; 1867 1868 /* ID of the CPU */ 1869 /*0x24*/u16 err_cpu; 1870 1871 /* Grace periof for shutdown, in seconds */ 1872 /*0x26*/u16 err_secs; 1873 1874 /* Value of the %asi register */ 1875 /*0x28*/u8 err_asi; 1876 1877 /*0x29*/u8 reserved_2; 1878 1879 /* Value of the ASR register number */ 1880 /*0x2a*/u16 err_asr; 1881 #define SUN4V_ERR_ASR_VALID 0x8000 1882 1883 /*0x2c*/u32 reserved_3; 1884 /*0x30*/u64 reserved_4; 1885 /*0x38*/u64 reserved_5; 1886 }; 1887 1888 static atomic_t sun4v_resum_oflow_cnt = ATOMIC_INIT(0); 1889 static atomic_t sun4v_nonresum_oflow_cnt = ATOMIC_INIT(0); 1890 1891 static const char *sun4v_err_type_to_str(u8 type) 1892 { 1893 static const char *types[SUN4V_ERR_TYPE_NUM] = { 1894 "undefined", 1895 "uncorrected resumable", 1896 "precise nonresumable", 1897 "deferred nonresumable", 1898 "shutdown request", 1899 "dump core", 1900 "SP state change", 1901 }; 1902 1903 if (type < SUN4V_ERR_TYPE_NUM) 1904 return types[type]; 1905 1906 return "unknown"; 1907 } 1908 1909 static void sun4v_emit_err_attr_strings(u32 attrs) 1910 { 1911 static const char *attr_names[] = { 1912 "processor", 1913 "memory", 1914 "PIO", 1915 "int-registers", 1916 "fpu-registers", 1917 "shutdown-request", 1918 "ASR", 1919 "ASI", 1920 "priv-reg", 1921 }; 1922 static const char *sp_states[] = { 1923 "sp-faulted", 1924 "sp-available", 1925 "sp-not-present", 1926 "sp-state-reserved", 1927 }; 1928 static const char *modes[] = { 1929 "mode-reserved0", 1930 "user", 1931 "priv", 1932 "mode-reserved1", 1933 }; 1934 u32 sp_state, mode; 1935 int i; 1936 1937 for (i = 0; i < ARRAY_SIZE(attr_names); i++) { 1938 if (attrs & (1U << i)) { 1939 const char *s = attr_names[i]; 1940 1941 pr_cont("%s ", s); 1942 } 1943 } 1944 1945 sp_state = ((attrs & SUN4V_ERR_ATTRS_SPSTATE_MSK) >> 1946 SUN4V_ERR_ATTRS_SPSTATE_SHFT); 1947 pr_cont("%s ", sp_states[sp_state]); 1948 1949 mode = ((attrs & SUN4V_ERR_ATTRS_MODE_MSK) >> 1950 SUN4V_ERR_ATTRS_MODE_SHFT); 1951 pr_cont("%s ", modes[mode]); 1952 1953 if (attrs & SUN4V_ERR_ATTRS_RES_QUEUE_FULL) 1954 pr_cont("res-queue-full "); 1955 } 1956 1957 /* When the report contains a real-address of "-1" it means that the 1958 * hardware did not provide the address. So we compute the effective 1959 * address of the load or store instruction at regs->tpc and report 1960 * that. Usually when this happens it's a PIO and in such a case we 1961 * are using physical addresses with bypass ASIs anyways, so what we 1962 * report here is exactly what we want. 1963 */ 1964 static void sun4v_report_real_raddr(const char *pfx, struct pt_regs *regs) 1965 { 1966 unsigned int insn; 1967 u64 addr; 1968 1969 if (!(regs->tstate & TSTATE_PRIV)) 1970 return; 1971 1972 insn = *(unsigned int *) regs->tpc; 1973 1974 addr = compute_effective_address(regs, insn, 0); 1975 1976 printk("%s: insn effective address [0x%016llx]\n", 1977 pfx, addr); 1978 } 1979 1980 static void sun4v_log_error(struct pt_regs *regs, struct sun4v_error_entry *ent, 1981 int cpu, const char *pfx, atomic_t *ocnt) 1982 { 1983 u64 *raw_ptr = (u64 *) ent; 1984 u32 attrs; 1985 int cnt; 1986 1987 printk("%s: Reporting on cpu %d\n", pfx, cpu); 1988 printk("%s: TPC [0x%016lx] <%pS>\n", 1989 pfx, regs->tpc, (void *) regs->tpc); 1990 1991 printk("%s: RAW [%016llx:%016llx:%016llx:%016llx\n", 1992 pfx, raw_ptr[0], raw_ptr[1], raw_ptr[2], raw_ptr[3]); 1993 printk("%s: %016llx:%016llx:%016llx:%016llx]\n", 1994 pfx, raw_ptr[4], raw_ptr[5], raw_ptr[6], raw_ptr[7]); 1995 1996 printk("%s: handle [0x%016llx] stick [0x%016llx]\n", 1997 pfx, ent->err_handle, ent->err_stick); 1998 1999 printk("%s: type [%s]\n", pfx, sun4v_err_type_to_str(ent->err_type)); 2000 2001 attrs = ent->err_attrs; 2002 printk("%s: attrs [0x%08x] < ", pfx, attrs); 2003 sun4v_emit_err_attr_strings(attrs); 2004 pr_cont(">\n"); 2005 2006 /* Various fields in the error report are only valid if 2007 * certain attribute bits are set. 2008 */ 2009 if (attrs & (SUN4V_ERR_ATTRS_MEMORY | 2010 SUN4V_ERR_ATTRS_PIO | 2011 SUN4V_ERR_ATTRS_ASI)) { 2012 printk("%s: raddr [0x%016llx]\n", pfx, ent->err_raddr); 2013 2014 if (ent->err_raddr == ~(u64)0) 2015 sun4v_report_real_raddr(pfx, regs); 2016 } 2017 2018 if (attrs & (SUN4V_ERR_ATTRS_MEMORY | SUN4V_ERR_ATTRS_ASI)) 2019 printk("%s: size [0x%x]\n", pfx, ent->err_size); 2020 2021 if (attrs & (SUN4V_ERR_ATTRS_PROCESSOR | 2022 SUN4V_ERR_ATTRS_INT_REGISTERS | 2023 SUN4V_ERR_ATTRS_FPU_REGISTERS | 2024 SUN4V_ERR_ATTRS_PRIV_REG)) 2025 printk("%s: cpu[%u]\n", pfx, ent->err_cpu); 2026 2027 if (attrs & SUN4V_ERR_ATTRS_ASI) 2028 printk("%s: asi [0x%02x]\n", pfx, ent->err_asi); 2029 2030 if ((attrs & (SUN4V_ERR_ATTRS_INT_REGISTERS | 2031 SUN4V_ERR_ATTRS_FPU_REGISTERS | 2032 SUN4V_ERR_ATTRS_PRIV_REG)) && 2033 (ent->err_asr & SUN4V_ERR_ASR_VALID) != 0) 2034 printk("%s: reg [0x%04x]\n", 2035 pfx, ent->err_asr & ~SUN4V_ERR_ASR_VALID); 2036 2037 show_regs(regs); 2038 2039 if ((cnt = atomic_read(ocnt)) != 0) { 2040 atomic_set(ocnt, 0); 2041 wmb(); 2042 printk("%s: Queue overflowed %d times.\n", 2043 pfx, cnt); 2044 } 2045 } 2046 2047 /* We run with %pil set to PIL_NORMAL_MAX and PSTATE_IE enabled in %pstate. 2048 * Log the event and clear the first word of the entry. 2049 */ 2050 void sun4v_resum_error(struct pt_regs *regs, unsigned long offset) 2051 { 2052 enum ctx_state prev_state = exception_enter(); 2053 struct sun4v_error_entry *ent, local_copy; 2054 struct trap_per_cpu *tb; 2055 unsigned long paddr; 2056 int cpu; 2057 2058 cpu = get_cpu(); 2059 2060 tb = &trap_block[cpu]; 2061 paddr = tb->resum_kernel_buf_pa + offset; 2062 ent = __va(paddr); 2063 2064 memcpy(&local_copy, ent, sizeof(struct sun4v_error_entry)); 2065 2066 /* We have a local copy now, so release the entry. */ 2067 ent->err_handle = 0; 2068 wmb(); 2069 2070 put_cpu(); 2071 2072 if (local_copy.err_type == SUN4V_ERR_TYPE_SHUTDOWN_RQST) { 2073 /* We should really take the seconds field of 2074 * the error report and use it for the shutdown 2075 * invocation, but for now do the same thing we 2076 * do for a DS shutdown request. 2077 */ 2078 pr_info("Shutdown request, %u seconds...\n", 2079 local_copy.err_secs); 2080 orderly_poweroff(true); 2081 goto out; 2082 } 2083 2084 sun4v_log_error(regs, &local_copy, cpu, 2085 KERN_ERR "RESUMABLE ERROR", 2086 &sun4v_resum_oflow_cnt); 2087 out: 2088 exception_exit(prev_state); 2089 } 2090 2091 /* If we try to printk() we'll probably make matters worse, by trying 2092 * to retake locks this cpu already holds or causing more errors. So 2093 * just bump a counter, and we'll report these counter bumps above. 2094 */ 2095 void sun4v_resum_overflow(struct pt_regs *regs) 2096 { 2097 atomic_inc(&sun4v_resum_oflow_cnt); 2098 } 2099 2100 /* Given a set of registers, get the virtual addressi that was being accessed 2101 * by the faulting instructions at tpc. 2102 */ 2103 static unsigned long sun4v_get_vaddr(struct pt_regs *regs) 2104 { 2105 unsigned int insn; 2106 2107 if (!copy_from_user(&insn, (void __user *)regs->tpc, 4)) { 2108 return compute_effective_address(regs, insn, 2109 (insn >> 25) & 0x1f); 2110 } 2111 return 0; 2112 } 2113 2114 /* Attempt to handle non-resumable errors generated from userspace. 2115 * Returns true if the signal was handled, false otherwise. 2116 */ 2117 bool sun4v_nonresum_error_user_handled(struct pt_regs *regs, 2118 struct sun4v_error_entry *ent) { 2119 2120 unsigned int attrs = ent->err_attrs; 2121 2122 if (attrs & SUN4V_ERR_ATTRS_MEMORY) { 2123 unsigned long addr = ent->err_raddr; 2124 siginfo_t info; 2125 2126 if (addr == ~(u64)0) { 2127 /* This seems highly unlikely to ever occur */ 2128 pr_emerg("SUN4V NON-RECOVERABLE ERROR: Memory error detected in unknown location!\n"); 2129 } else { 2130 unsigned long page_cnt = DIV_ROUND_UP(ent->err_size, 2131 PAGE_SIZE); 2132 2133 /* Break the unfortunate news. */ 2134 pr_emerg("SUN4V NON-RECOVERABLE ERROR: Memory failed at %016lX\n", 2135 addr); 2136 pr_emerg("SUN4V NON-RECOVERABLE ERROR: Claiming %lu ages.\n", 2137 page_cnt); 2138 2139 while (page_cnt-- > 0) { 2140 if (pfn_valid(addr >> PAGE_SHIFT)) 2141 get_page(pfn_to_page(addr >> PAGE_SHIFT)); 2142 addr += PAGE_SIZE; 2143 } 2144 } 2145 info.si_signo = SIGKILL; 2146 info.si_errno = 0; 2147 info.si_trapno = 0; 2148 force_sig_info(info.si_signo, &info, current); 2149 2150 return true; 2151 } 2152 if (attrs & SUN4V_ERR_ATTRS_PIO) { 2153 siginfo_t info; 2154 2155 info.si_signo = SIGBUS; 2156 info.si_code = BUS_ADRERR; 2157 info.si_addr = (void __user *)sun4v_get_vaddr(regs); 2158 force_sig_info(info.si_signo, &info, current); 2159 2160 return true; 2161 } 2162 2163 /* Default to doing nothing */ 2164 return false; 2165 } 2166 2167 /* We run with %pil set to PIL_NORMAL_MAX and PSTATE_IE enabled in %pstate. 2168 * Log the event, clear the first word of the entry, and die. 2169 */ 2170 void sun4v_nonresum_error(struct pt_regs *regs, unsigned long offset) 2171 { 2172 struct sun4v_error_entry *ent, local_copy; 2173 struct trap_per_cpu *tb; 2174 unsigned long paddr; 2175 int cpu; 2176 2177 cpu = get_cpu(); 2178 2179 tb = &trap_block[cpu]; 2180 paddr = tb->nonresum_kernel_buf_pa + offset; 2181 ent = __va(paddr); 2182 2183 memcpy(&local_copy, ent, sizeof(struct sun4v_error_entry)); 2184 2185 /* We have a local copy now, so release the entry. */ 2186 ent->err_handle = 0; 2187 wmb(); 2188 2189 put_cpu(); 2190 2191 if (!(regs->tstate & TSTATE_PRIV) && 2192 sun4v_nonresum_error_user_handled(regs, &local_copy)) { 2193 /* DON'T PANIC: This userspace error was handled. */ 2194 return; 2195 } 2196 2197 #ifdef CONFIG_PCI 2198 /* Check for the special PCI poke sequence. */ 2199 if (pci_poke_in_progress && pci_poke_cpu == cpu) { 2200 pci_poke_faulted = 1; 2201 regs->tpc += 4; 2202 regs->tnpc = regs->tpc + 4; 2203 return; 2204 } 2205 #endif 2206 2207 sun4v_log_error(regs, &local_copy, cpu, 2208 KERN_EMERG "NON-RESUMABLE ERROR", 2209 &sun4v_nonresum_oflow_cnt); 2210 2211 panic("Non-resumable error."); 2212 } 2213 2214 /* If we try to printk() we'll probably make matters worse, by trying 2215 * to retake locks this cpu already holds or causing more errors. So 2216 * just bump a counter, and we'll report these counter bumps above. 2217 */ 2218 void sun4v_nonresum_overflow(struct pt_regs *regs) 2219 { 2220 /* XXX Actually even this can make not that much sense. Perhaps 2221 * XXX we should just pull the plug and panic directly from here? 2222 */ 2223 atomic_inc(&sun4v_nonresum_oflow_cnt); 2224 } 2225 2226 static void sun4v_tlb_error(struct pt_regs *regs) 2227 { 2228 die_if_kernel("TLB/TSB error", regs); 2229 } 2230 2231 unsigned long sun4v_err_itlb_vaddr; 2232 unsigned long sun4v_err_itlb_ctx; 2233 unsigned long sun4v_err_itlb_pte; 2234 unsigned long sun4v_err_itlb_error; 2235 2236 void sun4v_itlb_error_report(struct pt_regs *regs, int tl) 2237 { 2238 dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); 2239 2240 printk(KERN_EMERG "SUN4V-ITLB: Error at TPC[%lx], tl %d\n", 2241 regs->tpc, tl); 2242 printk(KERN_EMERG "SUN4V-ITLB: TPC<%pS>\n", (void *) regs->tpc); 2243 printk(KERN_EMERG "SUN4V-ITLB: O7[%lx]\n", regs->u_regs[UREG_I7]); 2244 printk(KERN_EMERG "SUN4V-ITLB: O7<%pS>\n", 2245 (void *) regs->u_regs[UREG_I7]); 2246 printk(KERN_EMERG "SUN4V-ITLB: vaddr[%lx] ctx[%lx] " 2247 "pte[%lx] error[%lx]\n", 2248 sun4v_err_itlb_vaddr, sun4v_err_itlb_ctx, 2249 sun4v_err_itlb_pte, sun4v_err_itlb_error); 2250 2251 sun4v_tlb_error(regs); 2252 } 2253 2254 unsigned long sun4v_err_dtlb_vaddr; 2255 unsigned long sun4v_err_dtlb_ctx; 2256 unsigned long sun4v_err_dtlb_pte; 2257 unsigned long sun4v_err_dtlb_error; 2258 2259 void sun4v_dtlb_error_report(struct pt_regs *regs, int tl) 2260 { 2261 dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); 2262 2263 printk(KERN_EMERG "SUN4V-DTLB: Error at TPC[%lx], tl %d\n", 2264 regs->tpc, tl); 2265 printk(KERN_EMERG "SUN4V-DTLB: TPC<%pS>\n", (void *) regs->tpc); 2266 printk(KERN_EMERG "SUN4V-DTLB: O7[%lx]\n", regs->u_regs[UREG_I7]); 2267 printk(KERN_EMERG "SUN4V-DTLB: O7<%pS>\n", 2268 (void *) regs->u_regs[UREG_I7]); 2269 printk(KERN_EMERG "SUN4V-DTLB: vaddr[%lx] ctx[%lx] " 2270 "pte[%lx] error[%lx]\n", 2271 sun4v_err_dtlb_vaddr, sun4v_err_dtlb_ctx, 2272 sun4v_err_dtlb_pte, sun4v_err_dtlb_error); 2273 2274 sun4v_tlb_error(regs); 2275 } 2276 2277 void hypervisor_tlbop_error(unsigned long err, unsigned long op) 2278 { 2279 printk(KERN_CRIT "SUN4V: TLB hv call error %lu for op %lu\n", 2280 err, op); 2281 } 2282 2283 void hypervisor_tlbop_error_xcall(unsigned long err, unsigned long op) 2284 { 2285 printk(KERN_CRIT "SUN4V: XCALL TLB hv call error %lu for op %lu\n", 2286 err, op); 2287 } 2288 2289 static void do_fpe_common(struct pt_regs *regs) 2290 { 2291 if (regs->tstate & TSTATE_PRIV) { 2292 regs->tpc = regs->tnpc; 2293 regs->tnpc += 4; 2294 } else { 2295 unsigned long fsr = current_thread_info()->xfsr[0]; 2296 siginfo_t info; 2297 2298 if (test_thread_flag(TIF_32BIT)) { 2299 regs->tpc &= 0xffffffff; 2300 regs->tnpc &= 0xffffffff; 2301 } 2302 info.si_signo = SIGFPE; 2303 info.si_errno = 0; 2304 info.si_addr = (void __user *)regs->tpc; 2305 info.si_trapno = 0; 2306 info.si_code = FPE_FIXME; 2307 if ((fsr & 0x1c000) == (1 << 14)) { 2308 if (fsr & 0x10) 2309 info.si_code = FPE_FLTINV; 2310 else if (fsr & 0x08) 2311 info.si_code = FPE_FLTOVF; 2312 else if (fsr & 0x04) 2313 info.si_code = FPE_FLTUND; 2314 else if (fsr & 0x02) 2315 info.si_code = FPE_FLTDIV; 2316 else if (fsr & 0x01) 2317 info.si_code = FPE_FLTRES; 2318 } 2319 force_sig_info(SIGFPE, &info, current); 2320 } 2321 } 2322 2323 void do_fpieee(struct pt_regs *regs) 2324 { 2325 enum ctx_state prev_state = exception_enter(); 2326 2327 if (notify_die(DIE_TRAP, "fpu exception ieee", regs, 2328 0, 0x24, SIGFPE) == NOTIFY_STOP) 2329 goto out; 2330 2331 do_fpe_common(regs); 2332 out: 2333 exception_exit(prev_state); 2334 } 2335 2336 void do_fpother(struct pt_regs *regs) 2337 { 2338 enum ctx_state prev_state = exception_enter(); 2339 struct fpustate *f = FPUSTATE; 2340 int ret = 0; 2341 2342 if (notify_die(DIE_TRAP, "fpu exception other", regs, 2343 0, 0x25, SIGFPE) == NOTIFY_STOP) 2344 goto out; 2345 2346 switch ((current_thread_info()->xfsr[0] & 0x1c000)) { 2347 case (2 << 14): /* unfinished_FPop */ 2348 case (3 << 14): /* unimplemented_FPop */ 2349 ret = do_mathemu(regs, f, false); 2350 break; 2351 } 2352 if (ret) 2353 goto out; 2354 do_fpe_common(regs); 2355 out: 2356 exception_exit(prev_state); 2357 } 2358 2359 void do_tof(struct pt_regs *regs) 2360 { 2361 enum ctx_state prev_state = exception_enter(); 2362 siginfo_t info; 2363 2364 if (notify_die(DIE_TRAP, "tagged arithmetic overflow", regs, 2365 0, 0x26, SIGEMT) == NOTIFY_STOP) 2366 goto out; 2367 2368 if (regs->tstate & TSTATE_PRIV) 2369 die_if_kernel("Penguin overflow trap from kernel mode", regs); 2370 if (test_thread_flag(TIF_32BIT)) { 2371 regs->tpc &= 0xffffffff; 2372 regs->tnpc &= 0xffffffff; 2373 } 2374 info.si_signo = SIGEMT; 2375 info.si_errno = 0; 2376 info.si_code = EMT_TAGOVF; 2377 info.si_addr = (void __user *)regs->tpc; 2378 info.si_trapno = 0; 2379 force_sig_info(SIGEMT, &info, current); 2380 out: 2381 exception_exit(prev_state); 2382 } 2383 2384 void do_div0(struct pt_regs *regs) 2385 { 2386 enum ctx_state prev_state = exception_enter(); 2387 siginfo_t info; 2388 2389 if (notify_die(DIE_TRAP, "integer division by zero", regs, 2390 0, 0x28, SIGFPE) == NOTIFY_STOP) 2391 goto out; 2392 2393 if (regs->tstate & TSTATE_PRIV) 2394 die_if_kernel("TL0: Kernel divide by zero.", regs); 2395 if (test_thread_flag(TIF_32BIT)) { 2396 regs->tpc &= 0xffffffff; 2397 regs->tnpc &= 0xffffffff; 2398 } 2399 info.si_signo = SIGFPE; 2400 info.si_errno = 0; 2401 info.si_code = FPE_INTDIV; 2402 info.si_addr = (void __user *)regs->tpc; 2403 info.si_trapno = 0; 2404 force_sig_info(SIGFPE, &info, current); 2405 out: 2406 exception_exit(prev_state); 2407 } 2408 2409 static void instruction_dump(unsigned int *pc) 2410 { 2411 int i; 2412 2413 if ((((unsigned long) pc) & 3)) 2414 return; 2415 2416 printk("Instruction DUMP:"); 2417 for (i = -3; i < 6; i++) 2418 printk("%c%08x%c",i?' ':'<',pc[i],i?' ':'>'); 2419 printk("\n"); 2420 } 2421 2422 static void user_instruction_dump(unsigned int __user *pc) 2423 { 2424 int i; 2425 unsigned int buf[9]; 2426 2427 if ((((unsigned long) pc) & 3)) 2428 return; 2429 2430 if (copy_from_user(buf, pc - 3, sizeof(buf))) 2431 return; 2432 2433 printk("Instruction DUMP:"); 2434 for (i = 0; i < 9; i++) 2435 printk("%c%08x%c",i==3?' ':'<',buf[i],i==3?' ':'>'); 2436 printk("\n"); 2437 } 2438 2439 void show_stack(struct task_struct *tsk, unsigned long *_ksp) 2440 { 2441 unsigned long fp, ksp; 2442 struct thread_info *tp; 2443 int count = 0; 2444 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 2445 int graph = 0; 2446 #endif 2447 2448 ksp = (unsigned long) _ksp; 2449 if (!tsk) 2450 tsk = current; 2451 tp = task_thread_info(tsk); 2452 if (ksp == 0UL) { 2453 if (tsk == current) 2454 asm("mov %%fp, %0" : "=r" (ksp)); 2455 else 2456 ksp = tp->ksp; 2457 } 2458 if (tp == current_thread_info()) 2459 flushw_all(); 2460 2461 fp = ksp + STACK_BIAS; 2462 2463 printk("Call Trace:\n"); 2464 do { 2465 struct sparc_stackf *sf; 2466 struct pt_regs *regs; 2467 unsigned long pc; 2468 2469 if (!kstack_valid(tp, fp)) 2470 break; 2471 sf = (struct sparc_stackf *) fp; 2472 regs = (struct pt_regs *) (sf + 1); 2473 2474 if (kstack_is_trap_frame(tp, regs)) { 2475 if (!(regs->tstate & TSTATE_PRIV)) 2476 break; 2477 pc = regs->tpc; 2478 fp = regs->u_regs[UREG_I6] + STACK_BIAS; 2479 } else { 2480 pc = sf->callers_pc; 2481 fp = (unsigned long)sf->fp + STACK_BIAS; 2482 } 2483 2484 printk(" [%016lx] %pS\n", pc, (void *) pc); 2485 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 2486 if ((pc + 8UL) == (unsigned long) &return_to_handler) { 2487 int index = tsk->curr_ret_stack; 2488 if (tsk->ret_stack && index >= graph) { 2489 pc = tsk->ret_stack[index - graph].ret; 2490 printk(" [%016lx] %pS\n", pc, (void *) pc); 2491 graph++; 2492 } 2493 } 2494 #endif 2495 } while (++count < 16); 2496 } 2497 2498 static inline struct reg_window *kernel_stack_up(struct reg_window *rw) 2499 { 2500 unsigned long fp = rw->ins[6]; 2501 2502 if (!fp) 2503 return NULL; 2504 2505 return (struct reg_window *) (fp + STACK_BIAS); 2506 } 2507 2508 void __noreturn die_if_kernel(char *str, struct pt_regs *regs) 2509 { 2510 static int die_counter; 2511 int count = 0; 2512 2513 /* Amuse the user. */ 2514 printk( 2515 " \\|/ ____ \\|/\n" 2516 " \"@'/ .. \\`@\"\n" 2517 " /_| \\__/ |_\\\n" 2518 " \\__U_/\n"); 2519 2520 printk("%s(%d): %s [#%d]\n", current->comm, task_pid_nr(current), str, ++die_counter); 2521 notify_die(DIE_OOPS, str, regs, 0, 255, SIGSEGV); 2522 __asm__ __volatile__("flushw"); 2523 show_regs(regs); 2524 add_taint(TAINT_DIE, LOCKDEP_NOW_UNRELIABLE); 2525 if (regs->tstate & TSTATE_PRIV) { 2526 struct thread_info *tp = current_thread_info(); 2527 struct reg_window *rw = (struct reg_window *) 2528 (regs->u_regs[UREG_FP] + STACK_BIAS); 2529 2530 /* Stop the back trace when we hit userland or we 2531 * find some badly aligned kernel stack. 2532 */ 2533 while (rw && 2534 count++ < 30 && 2535 kstack_valid(tp, (unsigned long) rw)) { 2536 printk("Caller[%016lx]: %pS\n", rw->ins[7], 2537 (void *) rw->ins[7]); 2538 2539 rw = kernel_stack_up(rw); 2540 } 2541 instruction_dump ((unsigned int *) regs->tpc); 2542 } else { 2543 if (test_thread_flag(TIF_32BIT)) { 2544 regs->tpc &= 0xffffffff; 2545 regs->tnpc &= 0xffffffff; 2546 } 2547 user_instruction_dump ((unsigned int __user *) regs->tpc); 2548 } 2549 if (panic_on_oops) 2550 panic("Fatal exception"); 2551 if (regs->tstate & TSTATE_PRIV) 2552 do_exit(SIGKILL); 2553 do_exit(SIGSEGV); 2554 } 2555 EXPORT_SYMBOL(die_if_kernel); 2556 2557 #define VIS_OPCODE_MASK ((0x3 << 30) | (0x3f << 19)) 2558 #define VIS_OPCODE_VAL ((0x2 << 30) | (0x36 << 19)) 2559 2560 void do_illegal_instruction(struct pt_regs *regs) 2561 { 2562 enum ctx_state prev_state = exception_enter(); 2563 unsigned long pc = regs->tpc; 2564 unsigned long tstate = regs->tstate; 2565 u32 insn; 2566 siginfo_t info; 2567 2568 if (notify_die(DIE_TRAP, "illegal instruction", regs, 2569 0, 0x10, SIGILL) == NOTIFY_STOP) 2570 goto out; 2571 2572 if (tstate & TSTATE_PRIV) 2573 die_if_kernel("Kernel illegal instruction", regs); 2574 if (test_thread_flag(TIF_32BIT)) 2575 pc = (u32)pc; 2576 if (get_user(insn, (u32 __user *) pc) != -EFAULT) { 2577 if ((insn & 0xc1ffc000) == 0x81700000) /* POPC */ { 2578 if (handle_popc(insn, regs)) 2579 goto out; 2580 } else if ((insn & 0xc1580000) == 0xc1100000) /* LDQ/STQ */ { 2581 if (handle_ldf_stq(insn, regs)) 2582 goto out; 2583 } else if (tlb_type == hypervisor) { 2584 if ((insn & VIS_OPCODE_MASK) == VIS_OPCODE_VAL) { 2585 if (!vis_emul(regs, insn)) 2586 goto out; 2587 } else { 2588 struct fpustate *f = FPUSTATE; 2589 2590 /* On UltraSPARC T2 and later, FPU insns which 2591 * are not implemented in HW signal an illegal 2592 * instruction trap and do not set the FP Trap 2593 * Trap in the %fsr to unimplemented_FPop. 2594 */ 2595 if (do_mathemu(regs, f, true)) 2596 goto out; 2597 } 2598 } 2599 } 2600 info.si_signo = SIGILL; 2601 info.si_errno = 0; 2602 info.si_code = ILL_ILLOPC; 2603 info.si_addr = (void __user *)pc; 2604 info.si_trapno = 0; 2605 force_sig_info(SIGILL, &info, current); 2606 out: 2607 exception_exit(prev_state); 2608 } 2609 2610 void mem_address_unaligned(struct pt_regs *regs, unsigned long sfar, unsigned long sfsr) 2611 { 2612 enum ctx_state prev_state = exception_enter(); 2613 siginfo_t info; 2614 2615 if (notify_die(DIE_TRAP, "memory address unaligned", regs, 2616 0, 0x34, SIGSEGV) == NOTIFY_STOP) 2617 goto out; 2618 2619 if (regs->tstate & TSTATE_PRIV) { 2620 kernel_unaligned_trap(regs, *((unsigned int *)regs->tpc)); 2621 goto out; 2622 } 2623 if (is_no_fault_exception(regs)) 2624 return; 2625 2626 info.si_signo = SIGBUS; 2627 info.si_errno = 0; 2628 info.si_code = BUS_ADRALN; 2629 info.si_addr = (void __user *)sfar; 2630 info.si_trapno = 0; 2631 force_sig_info(SIGBUS, &info, current); 2632 out: 2633 exception_exit(prev_state); 2634 } 2635 2636 void sun4v_do_mna(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx) 2637 { 2638 siginfo_t info; 2639 2640 if (notify_die(DIE_TRAP, "memory address unaligned", regs, 2641 0, 0x34, SIGSEGV) == NOTIFY_STOP) 2642 return; 2643 2644 if (regs->tstate & TSTATE_PRIV) { 2645 kernel_unaligned_trap(regs, *((unsigned int *)regs->tpc)); 2646 return; 2647 } 2648 if (is_no_fault_exception(regs)) 2649 return; 2650 2651 info.si_signo = SIGBUS; 2652 info.si_errno = 0; 2653 info.si_code = BUS_ADRALN; 2654 info.si_addr = (void __user *) addr; 2655 info.si_trapno = 0; 2656 force_sig_info(SIGBUS, &info, current); 2657 } 2658 2659 void do_privop(struct pt_regs *regs) 2660 { 2661 enum ctx_state prev_state = exception_enter(); 2662 siginfo_t info; 2663 2664 if (notify_die(DIE_TRAP, "privileged operation", regs, 2665 0, 0x11, SIGILL) == NOTIFY_STOP) 2666 goto out; 2667 2668 if (test_thread_flag(TIF_32BIT)) { 2669 regs->tpc &= 0xffffffff; 2670 regs->tnpc &= 0xffffffff; 2671 } 2672 info.si_signo = SIGILL; 2673 info.si_errno = 0; 2674 info.si_code = ILL_PRVOPC; 2675 info.si_addr = (void __user *)regs->tpc; 2676 info.si_trapno = 0; 2677 force_sig_info(SIGILL, &info, current); 2678 out: 2679 exception_exit(prev_state); 2680 } 2681 2682 void do_privact(struct pt_regs *regs) 2683 { 2684 do_privop(regs); 2685 } 2686 2687 /* Trap level 1 stuff or other traps we should never see... */ 2688 void do_cee(struct pt_regs *regs) 2689 { 2690 exception_enter(); 2691 die_if_kernel("TL0: Cache Error Exception", regs); 2692 } 2693 2694 void do_div0_tl1(struct pt_regs *regs) 2695 { 2696 exception_enter(); 2697 dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); 2698 die_if_kernel("TL1: DIV0 Exception", regs); 2699 } 2700 2701 void do_fpieee_tl1(struct pt_regs *regs) 2702 { 2703 exception_enter(); 2704 dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); 2705 die_if_kernel("TL1: FPU IEEE Exception", regs); 2706 } 2707 2708 void do_fpother_tl1(struct pt_regs *regs) 2709 { 2710 exception_enter(); 2711 dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); 2712 die_if_kernel("TL1: FPU Other Exception", regs); 2713 } 2714 2715 void do_ill_tl1(struct pt_regs *regs) 2716 { 2717 exception_enter(); 2718 dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); 2719 die_if_kernel("TL1: Illegal Instruction Exception", regs); 2720 } 2721 2722 void do_irq_tl1(struct pt_regs *regs) 2723 { 2724 exception_enter(); 2725 dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); 2726 die_if_kernel("TL1: IRQ Exception", regs); 2727 } 2728 2729 void do_lddfmna_tl1(struct pt_regs *regs) 2730 { 2731 exception_enter(); 2732 dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); 2733 die_if_kernel("TL1: LDDF Exception", regs); 2734 } 2735 2736 void do_stdfmna_tl1(struct pt_regs *regs) 2737 { 2738 exception_enter(); 2739 dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); 2740 die_if_kernel("TL1: STDF Exception", regs); 2741 } 2742 2743 void do_paw(struct pt_regs *regs) 2744 { 2745 exception_enter(); 2746 die_if_kernel("TL0: Phys Watchpoint Exception", regs); 2747 } 2748 2749 void do_paw_tl1(struct pt_regs *regs) 2750 { 2751 exception_enter(); 2752 dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); 2753 die_if_kernel("TL1: Phys Watchpoint Exception", regs); 2754 } 2755 2756 void do_vaw(struct pt_regs *regs) 2757 { 2758 exception_enter(); 2759 die_if_kernel("TL0: Virt Watchpoint Exception", regs); 2760 } 2761 2762 void do_vaw_tl1(struct pt_regs *regs) 2763 { 2764 exception_enter(); 2765 dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); 2766 die_if_kernel("TL1: Virt Watchpoint Exception", regs); 2767 } 2768 2769 void do_tof_tl1(struct pt_regs *regs) 2770 { 2771 exception_enter(); 2772 dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); 2773 die_if_kernel("TL1: Tag Overflow Exception", regs); 2774 } 2775 2776 void do_getpsr(struct pt_regs *regs) 2777 { 2778 regs->u_regs[UREG_I0] = tstate_to_psr(regs->tstate); 2779 regs->tpc = regs->tnpc; 2780 regs->tnpc += 4; 2781 if (test_thread_flag(TIF_32BIT)) { 2782 regs->tpc &= 0xffffffff; 2783 regs->tnpc &= 0xffffffff; 2784 } 2785 } 2786 2787 u64 cpu_mondo_counter[NR_CPUS] = {0}; 2788 struct trap_per_cpu trap_block[NR_CPUS]; 2789 EXPORT_SYMBOL(trap_block); 2790 2791 /* This can get invoked before sched_init() so play it super safe 2792 * and use hard_smp_processor_id(). 2793 */ 2794 void notrace init_cur_cpu_trap(struct thread_info *t) 2795 { 2796 int cpu = hard_smp_processor_id(); 2797 struct trap_per_cpu *p = &trap_block[cpu]; 2798 2799 p->thread = t; 2800 p->pgd_paddr = 0; 2801 } 2802 2803 extern void thread_info_offsets_are_bolixed_dave(void); 2804 extern void trap_per_cpu_offsets_are_bolixed_dave(void); 2805 extern void tsb_config_offsets_are_bolixed_dave(void); 2806 2807 /* Only invoked on boot processor. */ 2808 void __init trap_init(void) 2809 { 2810 /* Compile time sanity check. */ 2811 BUILD_BUG_ON(TI_TASK != offsetof(struct thread_info, task) || 2812 TI_FLAGS != offsetof(struct thread_info, flags) || 2813 TI_CPU != offsetof(struct thread_info, cpu) || 2814 TI_FPSAVED != offsetof(struct thread_info, fpsaved) || 2815 TI_KSP != offsetof(struct thread_info, ksp) || 2816 TI_FAULT_ADDR != offsetof(struct thread_info, 2817 fault_address) || 2818 TI_KREGS != offsetof(struct thread_info, kregs) || 2819 TI_UTRAPS != offsetof(struct thread_info, utraps) || 2820 TI_REG_WINDOW != offsetof(struct thread_info, 2821 reg_window) || 2822 TI_RWIN_SPTRS != offsetof(struct thread_info, 2823 rwbuf_stkptrs) || 2824 TI_GSR != offsetof(struct thread_info, gsr) || 2825 TI_XFSR != offsetof(struct thread_info, xfsr) || 2826 TI_PRE_COUNT != offsetof(struct thread_info, 2827 preempt_count) || 2828 TI_NEW_CHILD != offsetof(struct thread_info, new_child) || 2829 TI_CURRENT_DS != offsetof(struct thread_info, 2830 current_ds) || 2831 TI_KUNA_REGS != offsetof(struct thread_info, 2832 kern_una_regs) || 2833 TI_KUNA_INSN != offsetof(struct thread_info, 2834 kern_una_insn) || 2835 TI_FPREGS != offsetof(struct thread_info, fpregs) || 2836 (TI_FPREGS & (64 - 1))); 2837 2838 BUILD_BUG_ON(TRAP_PER_CPU_THREAD != offsetof(struct trap_per_cpu, 2839 thread) || 2840 (TRAP_PER_CPU_PGD_PADDR != 2841 offsetof(struct trap_per_cpu, pgd_paddr)) || 2842 (TRAP_PER_CPU_CPU_MONDO_PA != 2843 offsetof(struct trap_per_cpu, cpu_mondo_pa)) || 2844 (TRAP_PER_CPU_DEV_MONDO_PA != 2845 offsetof(struct trap_per_cpu, dev_mondo_pa)) || 2846 (TRAP_PER_CPU_RESUM_MONDO_PA != 2847 offsetof(struct trap_per_cpu, resum_mondo_pa)) || 2848 (TRAP_PER_CPU_RESUM_KBUF_PA != 2849 offsetof(struct trap_per_cpu, resum_kernel_buf_pa)) || 2850 (TRAP_PER_CPU_NONRESUM_MONDO_PA != 2851 offsetof(struct trap_per_cpu, nonresum_mondo_pa)) || 2852 (TRAP_PER_CPU_NONRESUM_KBUF_PA != 2853 offsetof(struct trap_per_cpu, nonresum_kernel_buf_pa)) || 2854 (TRAP_PER_CPU_FAULT_INFO != 2855 offsetof(struct trap_per_cpu, fault_info)) || 2856 (TRAP_PER_CPU_CPU_MONDO_BLOCK_PA != 2857 offsetof(struct trap_per_cpu, cpu_mondo_block_pa)) || 2858 (TRAP_PER_CPU_CPU_LIST_PA != 2859 offsetof(struct trap_per_cpu, cpu_list_pa)) || 2860 (TRAP_PER_CPU_TSB_HUGE != 2861 offsetof(struct trap_per_cpu, tsb_huge)) || 2862 (TRAP_PER_CPU_TSB_HUGE_TEMP != 2863 offsetof(struct trap_per_cpu, tsb_huge_temp)) || 2864 (TRAP_PER_CPU_IRQ_WORKLIST_PA != 2865 offsetof(struct trap_per_cpu, irq_worklist_pa)) || 2866 (TRAP_PER_CPU_CPU_MONDO_QMASK != 2867 offsetof(struct trap_per_cpu, cpu_mondo_qmask)) || 2868 (TRAP_PER_CPU_DEV_MONDO_QMASK != 2869 offsetof(struct trap_per_cpu, dev_mondo_qmask)) || 2870 (TRAP_PER_CPU_RESUM_QMASK != 2871 offsetof(struct trap_per_cpu, resum_qmask)) || 2872 (TRAP_PER_CPU_NONRESUM_QMASK != 2873 offsetof(struct trap_per_cpu, nonresum_qmask)) || 2874 (TRAP_PER_CPU_PER_CPU_BASE != 2875 offsetof(struct trap_per_cpu, __per_cpu_base))); 2876 2877 BUILD_BUG_ON((TSB_CONFIG_TSB != 2878 offsetof(struct tsb_config, tsb)) || 2879 (TSB_CONFIG_RSS_LIMIT != 2880 offsetof(struct tsb_config, tsb_rss_limit)) || 2881 (TSB_CONFIG_NENTRIES != 2882 offsetof(struct tsb_config, tsb_nentries)) || 2883 (TSB_CONFIG_REG_VAL != 2884 offsetof(struct tsb_config, tsb_reg_val)) || 2885 (TSB_CONFIG_MAP_VADDR != 2886 offsetof(struct tsb_config, tsb_map_vaddr)) || 2887 (TSB_CONFIG_MAP_PTE != 2888 offsetof(struct tsb_config, tsb_map_pte))); 2889 2890 /* Attach to the address space of init_task. On SMP we 2891 * do this in smp.c:smp_callin for other cpus. 2892 */ 2893 mmgrab(&init_mm); 2894 current->active_mm = &init_mm; 2895 } 2896