1 /* 2 * unaligned.c: Unaligned load/store trap handling with special 3 * cases for the kernel to do them more quickly. 4 * 5 * Copyright (C) 1996,2008 David S. Miller (davem@davemloft.net) 6 * Copyright (C) 1996,1997 Jakub Jelinek (jj@sunsite.mff.cuni.cz) 7 */ 8 9 10 #include <linux/jiffies.h> 11 #include <linux/kernel.h> 12 #include <linux/sched.h> 13 #include <linux/mm.h> 14 #include <linux/module.h> 15 #include <asm/asi.h> 16 #include <asm/ptrace.h> 17 #include <asm/pstate.h> 18 #include <asm/processor.h> 19 #include <asm/system.h> 20 #include <asm/uaccess.h> 21 #include <linux/smp.h> 22 #include <linux/bitops.h> 23 #include <linux/perf_event.h> 24 #include <asm/fpumacro.h> 25 26 enum direction { 27 load, /* ld, ldd, ldh, ldsh */ 28 store, /* st, std, sth, stsh */ 29 both, /* Swap, ldstub, cas, ... */ 30 fpld, 31 fpst, 32 invalid, 33 }; 34 35 static inline enum direction decode_direction(unsigned int insn) 36 { 37 unsigned long tmp = (insn >> 21) & 1; 38 39 if (!tmp) 40 return load; 41 else { 42 switch ((insn>>19)&0xf) { 43 case 15: /* swap* */ 44 return both; 45 default: 46 return store; 47 } 48 } 49 } 50 51 /* 16 = double-word, 8 = extra-word, 4 = word, 2 = half-word */ 52 static inline int decode_access_size(unsigned int insn) 53 { 54 unsigned int tmp; 55 56 tmp = ((insn >> 19) & 0xf); 57 if (tmp == 11 || tmp == 14) /* ldx/stx */ 58 return 8; 59 tmp &= 3; 60 if (!tmp) 61 return 4; 62 else if (tmp == 3) 63 return 16; /* ldd/std - Although it is actually 8 */ 64 else if (tmp == 2) 65 return 2; 66 else { 67 printk("Impossible unaligned trap. insn=%08x\n", insn); 68 die_if_kernel("Byte sized unaligned access?!?!", current_thread_info()->kregs); 69 70 /* GCC should never warn that control reaches the end 71 * of this function without returning a value because 72 * die_if_kernel() is marked with attribute 'noreturn'. 73 * Alas, some versions do... 74 */ 75 76 return 0; 77 } 78 } 79 80 static inline int decode_asi(unsigned int insn, struct pt_regs *regs) 81 { 82 if (insn & 0x800000) { 83 if (insn & 0x2000) 84 return (unsigned char)(regs->tstate >> 24); /* %asi */ 85 else 86 return (unsigned char)(insn >> 5); /* imm_asi */ 87 } else 88 return ASI_P; 89 } 90 91 /* 0x400000 = signed, 0 = unsigned */ 92 static inline int decode_signedness(unsigned int insn) 93 { 94 return (insn & 0x400000); 95 } 96 97 static inline void maybe_flush_windows(unsigned int rs1, unsigned int rs2, 98 unsigned int rd, int from_kernel) 99 { 100 if (rs2 >= 16 || rs1 >= 16 || rd >= 16) { 101 if (from_kernel != 0) 102 __asm__ __volatile__("flushw"); 103 else 104 flushw_user(); 105 } 106 } 107 108 static inline long sign_extend_imm13(long imm) 109 { 110 return imm << 51 >> 51; 111 } 112 113 static unsigned long fetch_reg(unsigned int reg, struct pt_regs *regs) 114 { 115 unsigned long value; 116 117 if (reg < 16) 118 return (!reg ? 0 : regs->u_regs[reg]); 119 if (regs->tstate & TSTATE_PRIV) { 120 struct reg_window *win; 121 win = (struct reg_window *)(regs->u_regs[UREG_FP] + STACK_BIAS); 122 value = win->locals[reg - 16]; 123 } else if (test_thread_flag(TIF_32BIT)) { 124 struct reg_window32 __user *win32; 125 win32 = (struct reg_window32 __user *)((unsigned long)((u32)regs->u_regs[UREG_FP])); 126 get_user(value, &win32->locals[reg - 16]); 127 } else { 128 struct reg_window __user *win; 129 win = (struct reg_window __user *)(regs->u_regs[UREG_FP] + STACK_BIAS); 130 get_user(value, &win->locals[reg - 16]); 131 } 132 return value; 133 } 134 135 static unsigned long *fetch_reg_addr(unsigned int reg, struct pt_regs *regs) 136 { 137 if (reg < 16) 138 return ®s->u_regs[reg]; 139 if (regs->tstate & TSTATE_PRIV) { 140 struct reg_window *win; 141 win = (struct reg_window *)(regs->u_regs[UREG_FP] + STACK_BIAS); 142 return &win->locals[reg - 16]; 143 } else if (test_thread_flag(TIF_32BIT)) { 144 struct reg_window32 *win32; 145 win32 = (struct reg_window32 *)((unsigned long)((u32)regs->u_regs[UREG_FP])); 146 return (unsigned long *)&win32->locals[reg - 16]; 147 } else { 148 struct reg_window *win; 149 win = (struct reg_window *)(regs->u_regs[UREG_FP] + STACK_BIAS); 150 return &win->locals[reg - 16]; 151 } 152 } 153 154 unsigned long compute_effective_address(struct pt_regs *regs, 155 unsigned int insn, unsigned int rd) 156 { 157 unsigned int rs1 = (insn >> 14) & 0x1f; 158 unsigned int rs2 = insn & 0x1f; 159 int from_kernel = (regs->tstate & TSTATE_PRIV) != 0; 160 161 if (insn & 0x2000) { 162 maybe_flush_windows(rs1, 0, rd, from_kernel); 163 return (fetch_reg(rs1, regs) + sign_extend_imm13(insn)); 164 } else { 165 maybe_flush_windows(rs1, rs2, rd, from_kernel); 166 return (fetch_reg(rs1, regs) + fetch_reg(rs2, regs)); 167 } 168 } 169 170 /* This is just to make gcc think die_if_kernel does return... */ 171 static void __used unaligned_panic(char *str, struct pt_regs *regs) 172 { 173 die_if_kernel(str, regs); 174 } 175 176 extern int do_int_load(unsigned long *dest_reg, int size, 177 unsigned long *saddr, int is_signed, int asi); 178 179 extern int __do_int_store(unsigned long *dst_addr, int size, 180 unsigned long src_val, int asi); 181 182 static inline int do_int_store(int reg_num, int size, unsigned long *dst_addr, 183 struct pt_regs *regs, int asi, int orig_asi) 184 { 185 unsigned long zero = 0; 186 unsigned long *src_val_p = &zero; 187 unsigned long src_val; 188 189 if (size == 16) { 190 size = 8; 191 zero = (((long)(reg_num ? 192 (unsigned)fetch_reg(reg_num, regs) : 0)) << 32) | 193 (unsigned)fetch_reg(reg_num + 1, regs); 194 } else if (reg_num) { 195 src_val_p = fetch_reg_addr(reg_num, regs); 196 } 197 src_val = *src_val_p; 198 if (unlikely(asi != orig_asi)) { 199 switch (size) { 200 case 2: 201 src_val = swab16(src_val); 202 break; 203 case 4: 204 src_val = swab32(src_val); 205 break; 206 case 8: 207 src_val = swab64(src_val); 208 break; 209 case 16: 210 default: 211 BUG(); 212 break; 213 }; 214 } 215 return __do_int_store(dst_addr, size, src_val, asi); 216 } 217 218 static inline void advance(struct pt_regs *regs) 219 { 220 regs->tpc = regs->tnpc; 221 regs->tnpc += 4; 222 if (test_thread_flag(TIF_32BIT)) { 223 regs->tpc &= 0xffffffff; 224 regs->tnpc &= 0xffffffff; 225 } 226 } 227 228 static inline int floating_point_load_or_store_p(unsigned int insn) 229 { 230 return (insn >> 24) & 1; 231 } 232 233 static inline int ok_for_kernel(unsigned int insn) 234 { 235 return !floating_point_load_or_store_p(insn); 236 } 237 238 static void kernel_mna_trap_fault(int fixup_tstate_asi) 239 { 240 struct pt_regs *regs = current_thread_info()->kern_una_regs; 241 unsigned int insn = current_thread_info()->kern_una_insn; 242 const struct exception_table_entry *entry; 243 244 entry = search_exception_tables(regs->tpc); 245 if (!entry) { 246 unsigned long address; 247 248 address = compute_effective_address(regs, insn, 249 ((insn >> 25) & 0x1f)); 250 if (address < PAGE_SIZE) { 251 printk(KERN_ALERT "Unable to handle kernel NULL " 252 "pointer dereference in mna handler"); 253 } else 254 printk(KERN_ALERT "Unable to handle kernel paging " 255 "request in mna handler"); 256 printk(KERN_ALERT " at virtual address %016lx\n",address); 257 printk(KERN_ALERT "current->{active_,}mm->context = %016lx\n", 258 (current->mm ? CTX_HWBITS(current->mm->context) : 259 CTX_HWBITS(current->active_mm->context))); 260 printk(KERN_ALERT "current->{active_,}mm->pgd = %016lx\n", 261 (current->mm ? (unsigned long) current->mm->pgd : 262 (unsigned long) current->active_mm->pgd)); 263 die_if_kernel("Oops", regs); 264 /* Not reached */ 265 } 266 regs->tpc = entry->fixup; 267 regs->tnpc = regs->tpc + 4; 268 269 if (fixup_tstate_asi) { 270 regs->tstate &= ~TSTATE_ASI; 271 regs->tstate |= (ASI_AIUS << 24UL); 272 } 273 } 274 275 static void log_unaligned(struct pt_regs *regs) 276 { 277 static unsigned long count, last_time; 278 279 if (time_after(jiffies, last_time + 5 * HZ)) 280 count = 0; 281 if (count < 5) { 282 last_time = jiffies; 283 count++; 284 printk("Kernel unaligned access at TPC[%lx] %pS\n", 285 regs->tpc, (void *) regs->tpc); 286 } 287 } 288 289 asmlinkage void kernel_unaligned_trap(struct pt_regs *regs, unsigned int insn) 290 { 291 enum direction dir = decode_direction(insn); 292 int size = decode_access_size(insn); 293 int orig_asi, asi; 294 295 current_thread_info()->kern_una_regs = regs; 296 current_thread_info()->kern_una_insn = insn; 297 298 orig_asi = asi = decode_asi(insn, regs); 299 300 /* If this is a {get,put}_user() on an unaligned userspace pointer, 301 * just signal a fault and do not log the event. 302 */ 303 if (asi == ASI_AIUS) { 304 kernel_mna_trap_fault(0); 305 return; 306 } 307 308 log_unaligned(regs); 309 310 if (!ok_for_kernel(insn) || dir == both) { 311 printk("Unsupported unaligned load/store trap for kernel " 312 "at <%016lx>.\n", regs->tpc); 313 unaligned_panic("Kernel does fpu/atomic " 314 "unaligned load/store.", regs); 315 316 kernel_mna_trap_fault(0); 317 } else { 318 unsigned long addr, *reg_addr; 319 int err; 320 321 addr = compute_effective_address(regs, insn, 322 ((insn >> 25) & 0x1f)); 323 perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, 0, regs, addr); 324 switch (asi) { 325 case ASI_NL: 326 case ASI_AIUPL: 327 case ASI_AIUSL: 328 case ASI_PL: 329 case ASI_SL: 330 case ASI_PNFL: 331 case ASI_SNFL: 332 asi &= ~0x08; 333 break; 334 }; 335 switch (dir) { 336 case load: 337 reg_addr = fetch_reg_addr(((insn>>25)&0x1f), regs); 338 err = do_int_load(reg_addr, size, 339 (unsigned long *) addr, 340 decode_signedness(insn), asi); 341 if (likely(!err) && unlikely(asi != orig_asi)) { 342 unsigned long val_in = *reg_addr; 343 switch (size) { 344 case 2: 345 val_in = swab16(val_in); 346 break; 347 case 4: 348 val_in = swab32(val_in); 349 break; 350 case 8: 351 val_in = swab64(val_in); 352 break; 353 case 16: 354 default: 355 BUG(); 356 break; 357 }; 358 *reg_addr = val_in; 359 } 360 break; 361 362 case store: 363 err = do_int_store(((insn>>25)&0x1f), size, 364 (unsigned long *) addr, regs, 365 asi, orig_asi); 366 break; 367 368 default: 369 panic("Impossible kernel unaligned trap."); 370 /* Not reached... */ 371 } 372 if (unlikely(err)) 373 kernel_mna_trap_fault(1); 374 else 375 advance(regs); 376 } 377 } 378 379 static char popc_helper[] = { 380 0, 1, 1, 2, 1, 2, 2, 3, 381 1, 2, 2, 3, 2, 3, 3, 4, 382 }; 383 384 int handle_popc(u32 insn, struct pt_regs *regs) 385 { 386 u64 value; 387 int ret, i, rd = ((insn >> 25) & 0x1f); 388 int from_kernel = (regs->tstate & TSTATE_PRIV) != 0; 389 390 perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, 0, regs, 0); 391 if (insn & 0x2000) { 392 maybe_flush_windows(0, 0, rd, from_kernel); 393 value = sign_extend_imm13(insn); 394 } else { 395 maybe_flush_windows(0, insn & 0x1f, rd, from_kernel); 396 value = fetch_reg(insn & 0x1f, regs); 397 } 398 for (ret = 0, i = 0; i < 16; i++) { 399 ret += popc_helper[value & 0xf]; 400 value >>= 4; 401 } 402 if (rd < 16) { 403 if (rd) 404 regs->u_regs[rd] = ret; 405 } else { 406 if (test_thread_flag(TIF_32BIT)) { 407 struct reg_window32 __user *win32; 408 win32 = (struct reg_window32 __user *)((unsigned long)((u32)regs->u_regs[UREG_FP])); 409 put_user(ret, &win32->locals[rd - 16]); 410 } else { 411 struct reg_window __user *win; 412 win = (struct reg_window __user *)(regs->u_regs[UREG_FP] + STACK_BIAS); 413 put_user(ret, &win->locals[rd - 16]); 414 } 415 } 416 advance(regs); 417 return 1; 418 } 419 420 extern void do_fpother(struct pt_regs *regs); 421 extern void do_privact(struct pt_regs *regs); 422 extern void spitfire_data_access_exception(struct pt_regs *regs, 423 unsigned long sfsr, 424 unsigned long sfar); 425 extern void sun4v_data_access_exception(struct pt_regs *regs, 426 unsigned long addr, 427 unsigned long type_ctx); 428 429 int handle_ldf_stq(u32 insn, struct pt_regs *regs) 430 { 431 unsigned long addr = compute_effective_address(regs, insn, 0); 432 int freg = ((insn >> 25) & 0x1e) | ((insn >> 20) & 0x20); 433 struct fpustate *f = FPUSTATE; 434 int asi = decode_asi(insn, regs); 435 int flag = (freg < 32) ? FPRS_DL : FPRS_DU; 436 437 perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, 0, regs, 0); 438 439 save_and_clear_fpu(); 440 current_thread_info()->xfsr[0] &= ~0x1c000; 441 if (freg & 3) { 442 current_thread_info()->xfsr[0] |= (6 << 14) /* invalid_fp_register */; 443 do_fpother(regs); 444 return 0; 445 } 446 if (insn & 0x200000) { 447 /* STQ */ 448 u64 first = 0, second = 0; 449 450 if (current_thread_info()->fpsaved[0] & flag) { 451 first = *(u64 *)&f->regs[freg]; 452 second = *(u64 *)&f->regs[freg+2]; 453 } 454 if (asi < 0x80) { 455 do_privact(regs); 456 return 1; 457 } 458 switch (asi) { 459 case ASI_P: 460 case ASI_S: break; 461 case ASI_PL: 462 case ASI_SL: 463 { 464 /* Need to convert endians */ 465 u64 tmp = __swab64p(&first); 466 467 first = __swab64p(&second); 468 second = tmp; 469 break; 470 } 471 default: 472 if (tlb_type == hypervisor) 473 sun4v_data_access_exception(regs, addr, 0); 474 else 475 spitfire_data_access_exception(regs, 0, addr); 476 return 1; 477 } 478 if (put_user (first >> 32, (u32 __user *)addr) || 479 __put_user ((u32)first, (u32 __user *)(addr + 4)) || 480 __put_user (second >> 32, (u32 __user *)(addr + 8)) || 481 __put_user ((u32)second, (u32 __user *)(addr + 12))) { 482 if (tlb_type == hypervisor) 483 sun4v_data_access_exception(regs, addr, 0); 484 else 485 spitfire_data_access_exception(regs, 0, addr); 486 return 1; 487 } 488 } else { 489 /* LDF, LDDF, LDQF */ 490 u32 data[4] __attribute__ ((aligned(8))); 491 int size, i; 492 int err; 493 494 if (asi < 0x80) { 495 do_privact(regs); 496 return 1; 497 } else if (asi > ASI_SNFL) { 498 if (tlb_type == hypervisor) 499 sun4v_data_access_exception(regs, addr, 0); 500 else 501 spitfire_data_access_exception(regs, 0, addr); 502 return 1; 503 } 504 switch (insn & 0x180000) { 505 case 0x000000: size = 1; break; 506 case 0x100000: size = 4; break; 507 default: size = 2; break; 508 } 509 for (i = 0; i < size; i++) 510 data[i] = 0; 511 512 err = get_user (data[0], (u32 __user *) addr); 513 if (!err) { 514 for (i = 1; i < size; i++) 515 err |= __get_user (data[i], (u32 __user *)(addr + 4*i)); 516 } 517 if (err && !(asi & 0x2 /* NF */)) { 518 if (tlb_type == hypervisor) 519 sun4v_data_access_exception(regs, addr, 0); 520 else 521 spitfire_data_access_exception(regs, 0, addr); 522 return 1; 523 } 524 if (asi & 0x8) /* Little */ { 525 u64 tmp; 526 527 switch (size) { 528 case 1: data[0] = le32_to_cpup(data + 0); break; 529 default:*(u64 *)(data + 0) = le64_to_cpup((u64 *)(data + 0)); 530 break; 531 case 4: tmp = le64_to_cpup((u64 *)(data + 0)); 532 *(u64 *)(data + 0) = le64_to_cpup((u64 *)(data + 2)); 533 *(u64 *)(data + 2) = tmp; 534 break; 535 } 536 } 537 if (!(current_thread_info()->fpsaved[0] & FPRS_FEF)) { 538 current_thread_info()->fpsaved[0] = FPRS_FEF; 539 current_thread_info()->gsr[0] = 0; 540 } 541 if (!(current_thread_info()->fpsaved[0] & flag)) { 542 if (freg < 32) 543 memset(f->regs, 0, 32*sizeof(u32)); 544 else 545 memset(f->regs+32, 0, 32*sizeof(u32)); 546 } 547 memcpy(f->regs + freg, data, size * 4); 548 current_thread_info()->fpsaved[0] |= flag; 549 } 550 advance(regs); 551 return 1; 552 } 553 554 void handle_ld_nf(u32 insn, struct pt_regs *regs) 555 { 556 int rd = ((insn >> 25) & 0x1f); 557 int from_kernel = (regs->tstate & TSTATE_PRIV) != 0; 558 unsigned long *reg; 559 560 perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, 0, regs, 0); 561 562 maybe_flush_windows(0, 0, rd, from_kernel); 563 reg = fetch_reg_addr(rd, regs); 564 if (from_kernel || rd < 16) { 565 reg[0] = 0; 566 if ((insn & 0x780000) == 0x180000) 567 reg[1] = 0; 568 } else if (test_thread_flag(TIF_32BIT)) { 569 put_user(0, (int __user *) reg); 570 if ((insn & 0x780000) == 0x180000) 571 put_user(0, ((int __user *) reg) + 1); 572 } else { 573 put_user(0, (unsigned long __user *) reg); 574 if ((insn & 0x780000) == 0x180000) 575 put_user(0, (unsigned long __user *) reg + 1); 576 } 577 advance(regs); 578 } 579 580 void handle_lddfmna(struct pt_regs *regs, unsigned long sfar, unsigned long sfsr) 581 { 582 unsigned long pc = regs->tpc; 583 unsigned long tstate = regs->tstate; 584 u32 insn; 585 u64 value; 586 u8 freg; 587 int flag; 588 struct fpustate *f = FPUSTATE; 589 590 if (tstate & TSTATE_PRIV) 591 die_if_kernel("lddfmna from kernel", regs); 592 perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, 0, regs, sfar); 593 if (test_thread_flag(TIF_32BIT)) 594 pc = (u32)pc; 595 if (get_user(insn, (u32 __user *) pc) != -EFAULT) { 596 int asi = decode_asi(insn, regs); 597 u32 first, second; 598 int err; 599 600 if ((asi > ASI_SNFL) || 601 (asi < ASI_P)) 602 goto daex; 603 first = second = 0; 604 err = get_user(first, (u32 __user *)sfar); 605 if (!err) 606 err = get_user(second, (u32 __user *)(sfar + 4)); 607 if (err) { 608 if (!(asi & 0x2)) 609 goto daex; 610 first = second = 0; 611 } 612 save_and_clear_fpu(); 613 freg = ((insn >> 25) & 0x1e) | ((insn >> 20) & 0x20); 614 value = (((u64)first) << 32) | second; 615 if (asi & 0x8) /* Little */ 616 value = __swab64p(&value); 617 flag = (freg < 32) ? FPRS_DL : FPRS_DU; 618 if (!(current_thread_info()->fpsaved[0] & FPRS_FEF)) { 619 current_thread_info()->fpsaved[0] = FPRS_FEF; 620 current_thread_info()->gsr[0] = 0; 621 } 622 if (!(current_thread_info()->fpsaved[0] & flag)) { 623 if (freg < 32) 624 memset(f->regs, 0, 32*sizeof(u32)); 625 else 626 memset(f->regs+32, 0, 32*sizeof(u32)); 627 } 628 *(u64 *)(f->regs + freg) = value; 629 current_thread_info()->fpsaved[0] |= flag; 630 } else { 631 daex: 632 if (tlb_type == hypervisor) 633 sun4v_data_access_exception(regs, sfar, sfsr); 634 else 635 spitfire_data_access_exception(regs, sfsr, sfar); 636 return; 637 } 638 advance(regs); 639 return; 640 } 641 642 void handle_stdfmna(struct pt_regs *regs, unsigned long sfar, unsigned long sfsr) 643 { 644 unsigned long pc = regs->tpc; 645 unsigned long tstate = regs->tstate; 646 u32 insn; 647 u64 value; 648 u8 freg; 649 int flag; 650 struct fpustate *f = FPUSTATE; 651 652 if (tstate & TSTATE_PRIV) 653 die_if_kernel("stdfmna from kernel", regs); 654 perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, 0, regs, sfar); 655 if (test_thread_flag(TIF_32BIT)) 656 pc = (u32)pc; 657 if (get_user(insn, (u32 __user *) pc) != -EFAULT) { 658 int asi = decode_asi(insn, regs); 659 freg = ((insn >> 25) & 0x1e) | ((insn >> 20) & 0x20); 660 value = 0; 661 flag = (freg < 32) ? FPRS_DL : FPRS_DU; 662 if ((asi > ASI_SNFL) || 663 (asi < ASI_P)) 664 goto daex; 665 save_and_clear_fpu(); 666 if (current_thread_info()->fpsaved[0] & flag) 667 value = *(u64 *)&f->regs[freg]; 668 switch (asi) { 669 case ASI_P: 670 case ASI_S: break; 671 case ASI_PL: 672 case ASI_SL: 673 value = __swab64p(&value); break; 674 default: goto daex; 675 } 676 if (put_user (value >> 32, (u32 __user *) sfar) || 677 __put_user ((u32)value, (u32 __user *)(sfar + 4))) 678 goto daex; 679 } else { 680 daex: 681 if (tlb_type == hypervisor) 682 sun4v_data_access_exception(regs, sfar, sfsr); 683 else 684 spitfire_data_access_exception(regs, sfsr, sfar); 685 return; 686 } 687 advance(regs); 688 return; 689 } 690