1 /* 2 * Handle unaligned accesses by emulation. 3 * 4 * This file is subject to the terms and conditions of the GNU General Public 5 * License. See the file "COPYING" in the main directory of this archive 6 * for more details. 7 * 8 * Copyright (C) 1996, 1998, 1999, 2002 by Ralf Baechle 9 * Copyright (C) 1999 Silicon Graphics, Inc. 10 * Copyright (C) 2014 Imagination Technologies Ltd. 11 * 12 * This file contains exception handler for address error exception with the 13 * special capability to execute faulting instructions in software. The 14 * handler does not try to handle the case when the program counter points 15 * to an address not aligned to a word boundary. 16 * 17 * Putting data to unaligned addresses is a bad practice even on Intel where 18 * only the performance is affected. Much worse is that such code is non- 19 * portable. Due to several programs that die on MIPS due to alignment 20 * problems I decided to implement this handler anyway though I originally 21 * didn't intend to do this at all for user code. 22 * 23 * For now I enable fixing of address errors by default to make life easier. 24 * I however intend to disable this somewhen in the future when the alignment 25 * problems with user programs have been fixed. For programmers this is the 26 * right way to go. 27 * 28 * Fixing address errors is a per process option. The option is inherited 29 * across fork(2) and execve(2) calls. If you really want to use the 30 * option in your user programs - I discourage the use of the software 31 * emulation strongly - use the following code in your userland stuff: 32 * 33 * #include <sys/sysmips.h> 34 * 35 * ... 36 * sysmips(MIPS_FIXADE, x); 37 * ... 38 * 39 * The argument x is 0 for disabling software emulation, enabled otherwise. 40 * 41 * Below a little program to play around with this feature. 42 * 43 * #include <stdio.h> 44 * #include <sys/sysmips.h> 45 * 46 * struct foo { 47 * unsigned char bar[8]; 48 * }; 49 * 50 * main(int argc, char *argv[]) 51 * { 52 * struct foo x = {0, 1, 2, 3, 4, 5, 6, 7}; 53 * unsigned int *p = (unsigned int *) (x.bar + 3); 54 * int i; 55 * 56 * if (argc > 1) 57 * sysmips(MIPS_FIXADE, atoi(argv[1])); 58 * 59 * printf("*p = %08lx\n", *p); 60 * 61 * *p = 0xdeadface; 62 * 63 * for(i = 0; i <= 7; i++) 64 * printf("%02x ", x.bar[i]); 65 * printf("\n"); 66 * } 67 * 68 * Coprocessor loads are not supported; I think this case is unimportant 69 * in the practice. 70 * 71 * TODO: Handle ndc (attempted store to doubleword in uncached memory) 72 * exception for the R6000. 73 * A store crossing a page boundary might be executed only partially. 74 * Undo the partial store in this case. 75 */ 76 #include <linux/context_tracking.h> 77 #include <linux/mm.h> 78 #include <linux/signal.h> 79 #include <linux/smp.h> 80 #include <linux/sched.h> 81 #include <linux/debugfs.h> 82 #include <linux/perf_event.h> 83 84 #include <asm/asm.h> 85 #include <asm/branch.h> 86 #include <asm/byteorder.h> 87 #include <asm/cop2.h> 88 #include <asm/debug.h> 89 #include <asm/fpu.h> 90 #include <asm/fpu_emulator.h> 91 #include <asm/inst.h> 92 #include <asm/unaligned-emul.h> 93 #include <asm/mmu_context.h> 94 #include <linux/uaccess.h> 95 96 enum { 97 UNALIGNED_ACTION_QUIET, 98 UNALIGNED_ACTION_SIGNAL, 99 UNALIGNED_ACTION_SHOW, 100 }; 101 #ifdef CONFIG_DEBUG_FS 102 static u32 unaligned_instructions; 103 static u32 unaligned_action; 104 #else 105 #define unaligned_action UNALIGNED_ACTION_QUIET 106 #endif 107 extern void show_registers(struct pt_regs *regs); 108 109 static void emulate_load_store_insn(struct pt_regs *regs, 110 void __user *addr, unsigned int __user *pc) 111 { 112 unsigned long origpc, orig31, value; 113 union mips_instruction insn; 114 unsigned int res; 115 #ifdef CONFIG_EVA 116 mm_segment_t seg; 117 #endif 118 origpc = (unsigned long)pc; 119 orig31 = regs->regs[31]; 120 121 perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, 0); 122 123 /* 124 * This load never faults. 125 */ 126 __get_user(insn.word, pc); 127 128 switch (insn.i_format.opcode) { 129 /* 130 * These are instructions that a compiler doesn't generate. We 131 * can assume therefore that the code is MIPS-aware and 132 * really buggy. Emulating these instructions would break the 133 * semantics anyway. 134 */ 135 case ll_op: 136 case lld_op: 137 case sc_op: 138 case scd_op: 139 140 /* 141 * For these instructions the only way to create an address 142 * error is an attempted access to kernel/supervisor address 143 * space. 144 */ 145 case ldl_op: 146 case ldr_op: 147 case lwl_op: 148 case lwr_op: 149 case sdl_op: 150 case sdr_op: 151 case swl_op: 152 case swr_op: 153 case lb_op: 154 case lbu_op: 155 case sb_op: 156 goto sigbus; 157 158 /* 159 * The remaining opcodes are the ones that are really of 160 * interest. 161 */ 162 case spec3_op: 163 if (insn.dsp_format.func == lx_op) { 164 switch (insn.dsp_format.op) { 165 case lwx_op: 166 if (!access_ok(addr, 4)) 167 goto sigbus; 168 LoadW(addr, value, res); 169 if (res) 170 goto fault; 171 compute_return_epc(regs); 172 regs->regs[insn.dsp_format.rd] = value; 173 break; 174 case lhx_op: 175 if (!access_ok(addr, 2)) 176 goto sigbus; 177 LoadHW(addr, value, res); 178 if (res) 179 goto fault; 180 compute_return_epc(regs); 181 regs->regs[insn.dsp_format.rd] = value; 182 break; 183 default: 184 goto sigill; 185 } 186 } 187 #ifdef CONFIG_EVA 188 else { 189 /* 190 * we can land here only from kernel accessing user 191 * memory, so we need to "switch" the address limit to 192 * user space, so that address check can work properly. 193 */ 194 seg = force_uaccess_begin(); 195 switch (insn.spec3_format.func) { 196 case lhe_op: 197 if (!access_ok(addr, 2)) { 198 force_uaccess_end(seg); 199 goto sigbus; 200 } 201 LoadHWE(addr, value, res); 202 if (res) { 203 force_uaccess_end(seg); 204 goto fault; 205 } 206 compute_return_epc(regs); 207 regs->regs[insn.spec3_format.rt] = value; 208 break; 209 case lwe_op: 210 if (!access_ok(addr, 4)) { 211 force_uaccess_end(seg); 212 goto sigbus; 213 } 214 LoadWE(addr, value, res); 215 if (res) { 216 force_uaccess_end(seg); 217 goto fault; 218 } 219 compute_return_epc(regs); 220 regs->regs[insn.spec3_format.rt] = value; 221 break; 222 case lhue_op: 223 if (!access_ok(addr, 2)) { 224 force_uaccess_end(seg); 225 goto sigbus; 226 } 227 LoadHWUE(addr, value, res); 228 if (res) { 229 force_uaccess_end(seg); 230 goto fault; 231 } 232 compute_return_epc(regs); 233 regs->regs[insn.spec3_format.rt] = value; 234 break; 235 case she_op: 236 if (!access_ok(addr, 2)) { 237 force_uaccess_end(seg); 238 goto sigbus; 239 } 240 compute_return_epc(regs); 241 value = regs->regs[insn.spec3_format.rt]; 242 StoreHWE(addr, value, res); 243 if (res) { 244 force_uaccess_end(seg); 245 goto fault; 246 } 247 break; 248 case swe_op: 249 if (!access_ok(addr, 4)) { 250 force_uaccess_end(seg); 251 goto sigbus; 252 } 253 compute_return_epc(regs); 254 value = regs->regs[insn.spec3_format.rt]; 255 StoreWE(addr, value, res); 256 if (res) { 257 force_uaccess_end(seg); 258 goto fault; 259 } 260 break; 261 default: 262 force_uaccess_end(seg); 263 goto sigill; 264 } 265 force_uaccess_end(seg); 266 } 267 #endif 268 break; 269 case lh_op: 270 if (!access_ok(addr, 2)) 271 goto sigbus; 272 273 if (IS_ENABLED(CONFIG_EVA)) { 274 if (uaccess_kernel()) 275 LoadHW(addr, value, res); 276 else 277 LoadHWE(addr, value, res); 278 } else { 279 LoadHW(addr, value, res); 280 } 281 282 if (res) 283 goto fault; 284 compute_return_epc(regs); 285 regs->regs[insn.i_format.rt] = value; 286 break; 287 288 case lw_op: 289 if (!access_ok(addr, 4)) 290 goto sigbus; 291 292 if (IS_ENABLED(CONFIG_EVA)) { 293 if (uaccess_kernel()) 294 LoadW(addr, value, res); 295 else 296 LoadWE(addr, value, res); 297 } else { 298 LoadW(addr, value, res); 299 } 300 301 if (res) 302 goto fault; 303 compute_return_epc(regs); 304 regs->regs[insn.i_format.rt] = value; 305 break; 306 307 case lhu_op: 308 if (!access_ok(addr, 2)) 309 goto sigbus; 310 311 if (IS_ENABLED(CONFIG_EVA)) { 312 if (uaccess_kernel()) 313 LoadHWU(addr, value, res); 314 else 315 LoadHWUE(addr, value, res); 316 } else { 317 LoadHWU(addr, value, res); 318 } 319 320 if (res) 321 goto fault; 322 compute_return_epc(regs); 323 regs->regs[insn.i_format.rt] = value; 324 break; 325 326 case lwu_op: 327 #ifdef CONFIG_64BIT 328 /* 329 * A 32-bit kernel might be running on a 64-bit processor. But 330 * if we're on a 32-bit processor and an i-cache incoherency 331 * or race makes us see a 64-bit instruction here the sdl/sdr 332 * would blow up, so for now we don't handle unaligned 64-bit 333 * instructions on 32-bit kernels. 334 */ 335 if (!access_ok(addr, 4)) 336 goto sigbus; 337 338 LoadWU(addr, value, res); 339 if (res) 340 goto fault; 341 compute_return_epc(regs); 342 regs->regs[insn.i_format.rt] = value; 343 break; 344 #endif /* CONFIG_64BIT */ 345 346 /* Cannot handle 64-bit instructions in 32-bit kernel */ 347 goto sigill; 348 349 case ld_op: 350 #ifdef CONFIG_64BIT 351 /* 352 * A 32-bit kernel might be running on a 64-bit processor. But 353 * if we're on a 32-bit processor and an i-cache incoherency 354 * or race makes us see a 64-bit instruction here the sdl/sdr 355 * would blow up, so for now we don't handle unaligned 64-bit 356 * instructions on 32-bit kernels. 357 */ 358 if (!access_ok(addr, 8)) 359 goto sigbus; 360 361 LoadDW(addr, value, res); 362 if (res) 363 goto fault; 364 compute_return_epc(regs); 365 regs->regs[insn.i_format.rt] = value; 366 break; 367 #endif /* CONFIG_64BIT */ 368 369 /* Cannot handle 64-bit instructions in 32-bit kernel */ 370 goto sigill; 371 372 case sh_op: 373 if (!access_ok(addr, 2)) 374 goto sigbus; 375 376 compute_return_epc(regs); 377 value = regs->regs[insn.i_format.rt]; 378 379 if (IS_ENABLED(CONFIG_EVA)) { 380 if (uaccess_kernel()) 381 StoreHW(addr, value, res); 382 else 383 StoreHWE(addr, value, res); 384 } else { 385 StoreHW(addr, value, res); 386 } 387 388 if (res) 389 goto fault; 390 break; 391 392 case sw_op: 393 if (!access_ok(addr, 4)) 394 goto sigbus; 395 396 compute_return_epc(regs); 397 value = regs->regs[insn.i_format.rt]; 398 399 if (IS_ENABLED(CONFIG_EVA)) { 400 if (uaccess_kernel()) 401 StoreW(addr, value, res); 402 else 403 StoreWE(addr, value, res); 404 } else { 405 StoreW(addr, value, res); 406 } 407 408 if (res) 409 goto fault; 410 break; 411 412 case sd_op: 413 #ifdef CONFIG_64BIT 414 /* 415 * A 32-bit kernel might be running on a 64-bit processor. But 416 * if we're on a 32-bit processor and an i-cache incoherency 417 * or race makes us see a 64-bit instruction here the sdl/sdr 418 * would blow up, so for now we don't handle unaligned 64-bit 419 * instructions on 32-bit kernels. 420 */ 421 if (!access_ok(addr, 8)) 422 goto sigbus; 423 424 compute_return_epc(regs); 425 value = regs->regs[insn.i_format.rt]; 426 StoreDW(addr, value, res); 427 if (res) 428 goto fault; 429 break; 430 #endif /* CONFIG_64BIT */ 431 432 /* Cannot handle 64-bit instructions in 32-bit kernel */ 433 goto sigill; 434 435 #ifdef CONFIG_MIPS_FP_SUPPORT 436 437 case lwc1_op: 438 case ldc1_op: 439 case swc1_op: 440 case sdc1_op: 441 case cop1x_op: { 442 void __user *fault_addr = NULL; 443 444 die_if_kernel("Unaligned FP access in kernel code", regs); 445 BUG_ON(!used_math()); 446 447 res = fpu_emulator_cop1Handler(regs, ¤t->thread.fpu, 1, 448 &fault_addr); 449 own_fpu(1); /* Restore FPU state. */ 450 451 /* Signal if something went wrong. */ 452 process_fpemu_return(res, fault_addr, 0); 453 454 if (res == 0) 455 break; 456 return; 457 } 458 #endif /* CONFIG_MIPS_FP_SUPPORT */ 459 460 #ifdef CONFIG_CPU_HAS_MSA 461 462 case msa_op: { 463 unsigned int wd, preempted; 464 enum msa_2b_fmt df; 465 union fpureg *fpr; 466 467 if (!cpu_has_msa) 468 goto sigill; 469 470 /* 471 * If we've reached this point then userland should have taken 472 * the MSA disabled exception & initialised vector context at 473 * some point in the past. 474 */ 475 BUG_ON(!thread_msa_context_live()); 476 477 df = insn.msa_mi10_format.df; 478 wd = insn.msa_mi10_format.wd; 479 fpr = ¤t->thread.fpu.fpr[wd]; 480 481 switch (insn.msa_mi10_format.func) { 482 case msa_ld_op: 483 if (!access_ok(addr, sizeof(*fpr))) 484 goto sigbus; 485 486 do { 487 /* 488 * If we have live MSA context keep track of 489 * whether we get preempted in order to avoid 490 * the register context we load being clobbered 491 * by the live context as it's saved during 492 * preemption. If we don't have live context 493 * then it can't be saved to clobber the value 494 * we load. 495 */ 496 preempted = test_thread_flag(TIF_USEDMSA); 497 498 res = __copy_from_user_inatomic(fpr, addr, 499 sizeof(*fpr)); 500 if (res) 501 goto fault; 502 503 /* 504 * Update the hardware register if it is in use 505 * by the task in this quantum, in order to 506 * avoid having to save & restore the whole 507 * vector context. 508 */ 509 preempt_disable(); 510 if (test_thread_flag(TIF_USEDMSA)) { 511 write_msa_wr(wd, fpr, df); 512 preempted = 0; 513 } 514 preempt_enable(); 515 } while (preempted); 516 break; 517 518 case msa_st_op: 519 if (!access_ok(addr, sizeof(*fpr))) 520 goto sigbus; 521 522 /* 523 * Update from the hardware register if it is in use by 524 * the task in this quantum, in order to avoid having to 525 * save & restore the whole vector context. 526 */ 527 preempt_disable(); 528 if (test_thread_flag(TIF_USEDMSA)) 529 read_msa_wr(wd, fpr, df); 530 preempt_enable(); 531 532 res = __copy_to_user_inatomic(addr, fpr, sizeof(*fpr)); 533 if (res) 534 goto fault; 535 break; 536 537 default: 538 goto sigbus; 539 } 540 541 compute_return_epc(regs); 542 break; 543 } 544 #endif /* CONFIG_CPU_HAS_MSA */ 545 546 #ifndef CONFIG_CPU_MIPSR6 547 /* 548 * COP2 is available to implementor for application specific use. 549 * It's up to applications to register a notifier chain and do 550 * whatever they have to do, including possible sending of signals. 551 * 552 * This instruction has been reallocated in Release 6 553 */ 554 case lwc2_op: 555 cu2_notifier_call_chain(CU2_LWC2_OP, regs); 556 break; 557 558 case ldc2_op: 559 cu2_notifier_call_chain(CU2_LDC2_OP, regs); 560 break; 561 562 case swc2_op: 563 cu2_notifier_call_chain(CU2_SWC2_OP, regs); 564 break; 565 566 case sdc2_op: 567 cu2_notifier_call_chain(CU2_SDC2_OP, regs); 568 break; 569 #endif 570 default: 571 /* 572 * Pheeee... We encountered an yet unknown instruction or 573 * cache coherence problem. Die sucker, die ... 574 */ 575 goto sigill; 576 } 577 578 #ifdef CONFIG_DEBUG_FS 579 unaligned_instructions++; 580 #endif 581 582 return; 583 584 fault: 585 /* roll back jump/branch */ 586 regs->cp0_epc = origpc; 587 regs->regs[31] = orig31; 588 /* Did we have an exception handler installed? */ 589 if (fixup_exception(regs)) 590 return; 591 592 die_if_kernel("Unhandled kernel unaligned access", regs); 593 force_sig(SIGSEGV); 594 595 return; 596 597 sigbus: 598 die_if_kernel("Unhandled kernel unaligned access", regs); 599 force_sig(SIGBUS); 600 601 return; 602 603 sigill: 604 die_if_kernel 605 ("Unhandled kernel unaligned access or invalid instruction", regs); 606 force_sig(SIGILL); 607 } 608 609 /* Recode table from 16-bit register notation to 32-bit GPR. */ 610 const int reg16to32[] = { 16, 17, 2, 3, 4, 5, 6, 7 }; 611 612 /* Recode table from 16-bit STORE register notation to 32-bit GPR. */ 613 static const int reg16to32st[] = { 0, 17, 2, 3, 4, 5, 6, 7 }; 614 615 static void emulate_load_store_microMIPS(struct pt_regs *regs, 616 void __user *addr) 617 { 618 unsigned long value; 619 unsigned int res; 620 int i; 621 unsigned int reg = 0, rvar; 622 unsigned long orig31; 623 u16 __user *pc16; 624 u16 halfword; 625 unsigned int word; 626 unsigned long origpc, contpc; 627 union mips_instruction insn; 628 struct mm_decoded_insn mminsn; 629 630 origpc = regs->cp0_epc; 631 orig31 = regs->regs[31]; 632 633 mminsn.micro_mips_mode = 1; 634 635 /* 636 * This load never faults. 637 */ 638 pc16 = (unsigned short __user *)msk_isa16_mode(regs->cp0_epc); 639 __get_user(halfword, pc16); 640 pc16++; 641 contpc = regs->cp0_epc + 2; 642 word = ((unsigned int)halfword << 16); 643 mminsn.pc_inc = 2; 644 645 if (!mm_insn_16bit(halfword)) { 646 __get_user(halfword, pc16); 647 pc16++; 648 contpc = regs->cp0_epc + 4; 649 mminsn.pc_inc = 4; 650 word |= halfword; 651 } 652 mminsn.insn = word; 653 654 if (get_user(halfword, pc16)) 655 goto fault; 656 mminsn.next_pc_inc = 2; 657 word = ((unsigned int)halfword << 16); 658 659 if (!mm_insn_16bit(halfword)) { 660 pc16++; 661 if (get_user(halfword, pc16)) 662 goto fault; 663 mminsn.next_pc_inc = 4; 664 word |= halfword; 665 } 666 mminsn.next_insn = word; 667 668 insn = (union mips_instruction)(mminsn.insn); 669 if (mm_isBranchInstr(regs, mminsn, &contpc)) 670 insn = (union mips_instruction)(mminsn.next_insn); 671 672 /* Parse instruction to find what to do */ 673 674 switch (insn.mm_i_format.opcode) { 675 676 case mm_pool32a_op: 677 switch (insn.mm_x_format.func) { 678 case mm_lwxs_op: 679 reg = insn.mm_x_format.rd; 680 goto loadW; 681 } 682 683 goto sigbus; 684 685 case mm_pool32b_op: 686 switch (insn.mm_m_format.func) { 687 case mm_lwp_func: 688 reg = insn.mm_m_format.rd; 689 if (reg == 31) 690 goto sigbus; 691 692 if (!access_ok(addr, 8)) 693 goto sigbus; 694 695 LoadW(addr, value, res); 696 if (res) 697 goto fault; 698 regs->regs[reg] = value; 699 addr += 4; 700 LoadW(addr, value, res); 701 if (res) 702 goto fault; 703 regs->regs[reg + 1] = value; 704 goto success; 705 706 case mm_swp_func: 707 reg = insn.mm_m_format.rd; 708 if (reg == 31) 709 goto sigbus; 710 711 if (!access_ok(addr, 8)) 712 goto sigbus; 713 714 value = regs->regs[reg]; 715 StoreW(addr, value, res); 716 if (res) 717 goto fault; 718 addr += 4; 719 value = regs->regs[reg + 1]; 720 StoreW(addr, value, res); 721 if (res) 722 goto fault; 723 goto success; 724 725 case mm_ldp_func: 726 #ifdef CONFIG_64BIT 727 reg = insn.mm_m_format.rd; 728 if (reg == 31) 729 goto sigbus; 730 731 if (!access_ok(addr, 16)) 732 goto sigbus; 733 734 LoadDW(addr, value, res); 735 if (res) 736 goto fault; 737 regs->regs[reg] = value; 738 addr += 8; 739 LoadDW(addr, value, res); 740 if (res) 741 goto fault; 742 regs->regs[reg + 1] = value; 743 goto success; 744 #endif /* CONFIG_64BIT */ 745 746 goto sigill; 747 748 case mm_sdp_func: 749 #ifdef CONFIG_64BIT 750 reg = insn.mm_m_format.rd; 751 if (reg == 31) 752 goto sigbus; 753 754 if (!access_ok(addr, 16)) 755 goto sigbus; 756 757 value = regs->regs[reg]; 758 StoreDW(addr, value, res); 759 if (res) 760 goto fault; 761 addr += 8; 762 value = regs->regs[reg + 1]; 763 StoreDW(addr, value, res); 764 if (res) 765 goto fault; 766 goto success; 767 #endif /* CONFIG_64BIT */ 768 769 goto sigill; 770 771 case mm_lwm32_func: 772 reg = insn.mm_m_format.rd; 773 rvar = reg & 0xf; 774 if ((rvar > 9) || !reg) 775 goto sigill; 776 if (reg & 0x10) { 777 if (!access_ok(addr, 4 * (rvar + 1))) 778 goto sigbus; 779 } else { 780 if (!access_ok(addr, 4 * rvar)) 781 goto sigbus; 782 } 783 if (rvar == 9) 784 rvar = 8; 785 for (i = 16; rvar; rvar--, i++) { 786 LoadW(addr, value, res); 787 if (res) 788 goto fault; 789 addr += 4; 790 regs->regs[i] = value; 791 } 792 if ((reg & 0xf) == 9) { 793 LoadW(addr, value, res); 794 if (res) 795 goto fault; 796 addr += 4; 797 regs->regs[30] = value; 798 } 799 if (reg & 0x10) { 800 LoadW(addr, value, res); 801 if (res) 802 goto fault; 803 regs->regs[31] = value; 804 } 805 goto success; 806 807 case mm_swm32_func: 808 reg = insn.mm_m_format.rd; 809 rvar = reg & 0xf; 810 if ((rvar > 9) || !reg) 811 goto sigill; 812 if (reg & 0x10) { 813 if (!access_ok(addr, 4 * (rvar + 1))) 814 goto sigbus; 815 } else { 816 if (!access_ok(addr, 4 * rvar)) 817 goto sigbus; 818 } 819 if (rvar == 9) 820 rvar = 8; 821 for (i = 16; rvar; rvar--, i++) { 822 value = regs->regs[i]; 823 StoreW(addr, value, res); 824 if (res) 825 goto fault; 826 addr += 4; 827 } 828 if ((reg & 0xf) == 9) { 829 value = regs->regs[30]; 830 StoreW(addr, value, res); 831 if (res) 832 goto fault; 833 addr += 4; 834 } 835 if (reg & 0x10) { 836 value = regs->regs[31]; 837 StoreW(addr, value, res); 838 if (res) 839 goto fault; 840 } 841 goto success; 842 843 case mm_ldm_func: 844 #ifdef CONFIG_64BIT 845 reg = insn.mm_m_format.rd; 846 rvar = reg & 0xf; 847 if ((rvar > 9) || !reg) 848 goto sigill; 849 if (reg & 0x10) { 850 if (!access_ok(addr, 8 * (rvar + 1))) 851 goto sigbus; 852 } else { 853 if (!access_ok(addr, 8 * rvar)) 854 goto sigbus; 855 } 856 if (rvar == 9) 857 rvar = 8; 858 859 for (i = 16; rvar; rvar--, i++) { 860 LoadDW(addr, value, res); 861 if (res) 862 goto fault; 863 addr += 4; 864 regs->regs[i] = value; 865 } 866 if ((reg & 0xf) == 9) { 867 LoadDW(addr, value, res); 868 if (res) 869 goto fault; 870 addr += 8; 871 regs->regs[30] = value; 872 } 873 if (reg & 0x10) { 874 LoadDW(addr, value, res); 875 if (res) 876 goto fault; 877 regs->regs[31] = value; 878 } 879 goto success; 880 #endif /* CONFIG_64BIT */ 881 882 goto sigill; 883 884 case mm_sdm_func: 885 #ifdef CONFIG_64BIT 886 reg = insn.mm_m_format.rd; 887 rvar = reg & 0xf; 888 if ((rvar > 9) || !reg) 889 goto sigill; 890 if (reg & 0x10) { 891 if (!access_ok(addr, 8 * (rvar + 1))) 892 goto sigbus; 893 } else { 894 if (!access_ok(addr, 8 * rvar)) 895 goto sigbus; 896 } 897 if (rvar == 9) 898 rvar = 8; 899 900 for (i = 16; rvar; rvar--, i++) { 901 value = regs->regs[i]; 902 StoreDW(addr, value, res); 903 if (res) 904 goto fault; 905 addr += 8; 906 } 907 if ((reg & 0xf) == 9) { 908 value = regs->regs[30]; 909 StoreDW(addr, value, res); 910 if (res) 911 goto fault; 912 addr += 8; 913 } 914 if (reg & 0x10) { 915 value = regs->regs[31]; 916 StoreDW(addr, value, res); 917 if (res) 918 goto fault; 919 } 920 goto success; 921 #endif /* CONFIG_64BIT */ 922 923 goto sigill; 924 925 /* LWC2, SWC2, LDC2, SDC2 are not serviced */ 926 } 927 928 goto sigbus; 929 930 case mm_pool32c_op: 931 switch (insn.mm_m_format.func) { 932 case mm_lwu_func: 933 reg = insn.mm_m_format.rd; 934 goto loadWU; 935 } 936 937 /* LL,SC,LLD,SCD are not serviced */ 938 goto sigbus; 939 940 #ifdef CONFIG_MIPS_FP_SUPPORT 941 case mm_pool32f_op: 942 switch (insn.mm_x_format.func) { 943 case mm_lwxc1_func: 944 case mm_swxc1_func: 945 case mm_ldxc1_func: 946 case mm_sdxc1_func: 947 goto fpu_emul; 948 } 949 950 goto sigbus; 951 952 case mm_ldc132_op: 953 case mm_sdc132_op: 954 case mm_lwc132_op: 955 case mm_swc132_op: { 956 void __user *fault_addr = NULL; 957 958 fpu_emul: 959 /* roll back jump/branch */ 960 regs->cp0_epc = origpc; 961 regs->regs[31] = orig31; 962 963 die_if_kernel("Unaligned FP access in kernel code", regs); 964 BUG_ON(!used_math()); 965 BUG_ON(!is_fpu_owner()); 966 967 res = fpu_emulator_cop1Handler(regs, ¤t->thread.fpu, 1, 968 &fault_addr); 969 own_fpu(1); /* restore FPU state */ 970 971 /* If something went wrong, signal */ 972 process_fpemu_return(res, fault_addr, 0); 973 974 if (res == 0) 975 goto success; 976 return; 977 } 978 #endif /* CONFIG_MIPS_FP_SUPPORT */ 979 980 case mm_lh32_op: 981 reg = insn.mm_i_format.rt; 982 goto loadHW; 983 984 case mm_lhu32_op: 985 reg = insn.mm_i_format.rt; 986 goto loadHWU; 987 988 case mm_lw32_op: 989 reg = insn.mm_i_format.rt; 990 goto loadW; 991 992 case mm_sh32_op: 993 reg = insn.mm_i_format.rt; 994 goto storeHW; 995 996 case mm_sw32_op: 997 reg = insn.mm_i_format.rt; 998 goto storeW; 999 1000 case mm_ld32_op: 1001 reg = insn.mm_i_format.rt; 1002 goto loadDW; 1003 1004 case mm_sd32_op: 1005 reg = insn.mm_i_format.rt; 1006 goto storeDW; 1007 1008 case mm_pool16c_op: 1009 switch (insn.mm16_m_format.func) { 1010 case mm_lwm16_op: 1011 reg = insn.mm16_m_format.rlist; 1012 rvar = reg + 1; 1013 if (!access_ok(addr, 4 * rvar)) 1014 goto sigbus; 1015 1016 for (i = 16; rvar; rvar--, i++) { 1017 LoadW(addr, value, res); 1018 if (res) 1019 goto fault; 1020 addr += 4; 1021 regs->regs[i] = value; 1022 } 1023 LoadW(addr, value, res); 1024 if (res) 1025 goto fault; 1026 regs->regs[31] = value; 1027 1028 goto success; 1029 1030 case mm_swm16_op: 1031 reg = insn.mm16_m_format.rlist; 1032 rvar = reg + 1; 1033 if (!access_ok(addr, 4 * rvar)) 1034 goto sigbus; 1035 1036 for (i = 16; rvar; rvar--, i++) { 1037 value = regs->regs[i]; 1038 StoreW(addr, value, res); 1039 if (res) 1040 goto fault; 1041 addr += 4; 1042 } 1043 value = regs->regs[31]; 1044 StoreW(addr, value, res); 1045 if (res) 1046 goto fault; 1047 1048 goto success; 1049 1050 } 1051 1052 goto sigbus; 1053 1054 case mm_lhu16_op: 1055 reg = reg16to32[insn.mm16_rb_format.rt]; 1056 goto loadHWU; 1057 1058 case mm_lw16_op: 1059 reg = reg16to32[insn.mm16_rb_format.rt]; 1060 goto loadW; 1061 1062 case mm_sh16_op: 1063 reg = reg16to32st[insn.mm16_rb_format.rt]; 1064 goto storeHW; 1065 1066 case mm_sw16_op: 1067 reg = reg16to32st[insn.mm16_rb_format.rt]; 1068 goto storeW; 1069 1070 case mm_lwsp16_op: 1071 reg = insn.mm16_r5_format.rt; 1072 goto loadW; 1073 1074 case mm_swsp16_op: 1075 reg = insn.mm16_r5_format.rt; 1076 goto storeW; 1077 1078 case mm_lwgp16_op: 1079 reg = reg16to32[insn.mm16_r3_format.rt]; 1080 goto loadW; 1081 1082 default: 1083 goto sigill; 1084 } 1085 1086 loadHW: 1087 if (!access_ok(addr, 2)) 1088 goto sigbus; 1089 1090 LoadHW(addr, value, res); 1091 if (res) 1092 goto fault; 1093 regs->regs[reg] = value; 1094 goto success; 1095 1096 loadHWU: 1097 if (!access_ok(addr, 2)) 1098 goto sigbus; 1099 1100 LoadHWU(addr, value, res); 1101 if (res) 1102 goto fault; 1103 regs->regs[reg] = value; 1104 goto success; 1105 1106 loadW: 1107 if (!access_ok(addr, 4)) 1108 goto sigbus; 1109 1110 LoadW(addr, value, res); 1111 if (res) 1112 goto fault; 1113 regs->regs[reg] = value; 1114 goto success; 1115 1116 loadWU: 1117 #ifdef CONFIG_64BIT 1118 /* 1119 * A 32-bit kernel might be running on a 64-bit processor. But 1120 * if we're on a 32-bit processor and an i-cache incoherency 1121 * or race makes us see a 64-bit instruction here the sdl/sdr 1122 * would blow up, so for now we don't handle unaligned 64-bit 1123 * instructions on 32-bit kernels. 1124 */ 1125 if (!access_ok(addr, 4)) 1126 goto sigbus; 1127 1128 LoadWU(addr, value, res); 1129 if (res) 1130 goto fault; 1131 regs->regs[reg] = value; 1132 goto success; 1133 #endif /* CONFIG_64BIT */ 1134 1135 /* Cannot handle 64-bit instructions in 32-bit kernel */ 1136 goto sigill; 1137 1138 loadDW: 1139 #ifdef CONFIG_64BIT 1140 /* 1141 * A 32-bit kernel might be running on a 64-bit processor. But 1142 * if we're on a 32-bit processor and an i-cache incoherency 1143 * or race makes us see a 64-bit instruction here the sdl/sdr 1144 * would blow up, so for now we don't handle unaligned 64-bit 1145 * instructions on 32-bit kernels. 1146 */ 1147 if (!access_ok(addr, 8)) 1148 goto sigbus; 1149 1150 LoadDW(addr, value, res); 1151 if (res) 1152 goto fault; 1153 regs->regs[reg] = value; 1154 goto success; 1155 #endif /* CONFIG_64BIT */ 1156 1157 /* Cannot handle 64-bit instructions in 32-bit kernel */ 1158 goto sigill; 1159 1160 storeHW: 1161 if (!access_ok(addr, 2)) 1162 goto sigbus; 1163 1164 value = regs->regs[reg]; 1165 StoreHW(addr, value, res); 1166 if (res) 1167 goto fault; 1168 goto success; 1169 1170 storeW: 1171 if (!access_ok(addr, 4)) 1172 goto sigbus; 1173 1174 value = regs->regs[reg]; 1175 StoreW(addr, value, res); 1176 if (res) 1177 goto fault; 1178 goto success; 1179 1180 storeDW: 1181 #ifdef CONFIG_64BIT 1182 /* 1183 * A 32-bit kernel might be running on a 64-bit processor. But 1184 * if we're on a 32-bit processor and an i-cache incoherency 1185 * or race makes us see a 64-bit instruction here the sdl/sdr 1186 * would blow up, so for now we don't handle unaligned 64-bit 1187 * instructions on 32-bit kernels. 1188 */ 1189 if (!access_ok(addr, 8)) 1190 goto sigbus; 1191 1192 value = regs->regs[reg]; 1193 StoreDW(addr, value, res); 1194 if (res) 1195 goto fault; 1196 goto success; 1197 #endif /* CONFIG_64BIT */ 1198 1199 /* Cannot handle 64-bit instructions in 32-bit kernel */ 1200 goto sigill; 1201 1202 success: 1203 regs->cp0_epc = contpc; /* advance or branch */ 1204 1205 #ifdef CONFIG_DEBUG_FS 1206 unaligned_instructions++; 1207 #endif 1208 return; 1209 1210 fault: 1211 /* roll back jump/branch */ 1212 regs->cp0_epc = origpc; 1213 regs->regs[31] = orig31; 1214 /* Did we have an exception handler installed? */ 1215 if (fixup_exception(regs)) 1216 return; 1217 1218 die_if_kernel("Unhandled kernel unaligned access", regs); 1219 force_sig(SIGSEGV); 1220 1221 return; 1222 1223 sigbus: 1224 die_if_kernel("Unhandled kernel unaligned access", regs); 1225 force_sig(SIGBUS); 1226 1227 return; 1228 1229 sigill: 1230 die_if_kernel 1231 ("Unhandled kernel unaligned access or invalid instruction", regs); 1232 force_sig(SIGILL); 1233 } 1234 1235 static void emulate_load_store_MIPS16e(struct pt_regs *regs, void __user * addr) 1236 { 1237 unsigned long value; 1238 unsigned int res; 1239 int reg; 1240 unsigned long orig31; 1241 u16 __user *pc16; 1242 unsigned long origpc; 1243 union mips16e_instruction mips16inst, oldinst; 1244 unsigned int opcode; 1245 int extended = 0; 1246 1247 origpc = regs->cp0_epc; 1248 orig31 = regs->regs[31]; 1249 pc16 = (unsigned short __user *)msk_isa16_mode(origpc); 1250 /* 1251 * This load never faults. 1252 */ 1253 __get_user(mips16inst.full, pc16); 1254 oldinst = mips16inst; 1255 1256 /* skip EXTEND instruction */ 1257 if (mips16inst.ri.opcode == MIPS16e_extend_op) { 1258 extended = 1; 1259 pc16++; 1260 __get_user(mips16inst.full, pc16); 1261 } else if (delay_slot(regs)) { 1262 /* skip jump instructions */ 1263 /* JAL/JALX are 32 bits but have OPCODE in first short int */ 1264 if (mips16inst.ri.opcode == MIPS16e_jal_op) 1265 pc16++; 1266 pc16++; 1267 if (get_user(mips16inst.full, pc16)) 1268 goto sigbus; 1269 } 1270 1271 opcode = mips16inst.ri.opcode; 1272 switch (opcode) { 1273 case MIPS16e_i64_op: /* I64 or RI64 instruction */ 1274 switch (mips16inst.i64.func) { /* I64/RI64 func field check */ 1275 case MIPS16e_ldpc_func: 1276 case MIPS16e_ldsp_func: 1277 reg = reg16to32[mips16inst.ri64.ry]; 1278 goto loadDW; 1279 1280 case MIPS16e_sdsp_func: 1281 reg = reg16to32[mips16inst.ri64.ry]; 1282 goto writeDW; 1283 1284 case MIPS16e_sdrasp_func: 1285 reg = 29; /* GPRSP */ 1286 goto writeDW; 1287 } 1288 1289 goto sigbus; 1290 1291 case MIPS16e_swsp_op: 1292 reg = reg16to32[mips16inst.ri.rx]; 1293 if (extended && cpu_has_mips16e2) 1294 switch (mips16inst.ri.imm >> 5) { 1295 case 0: /* SWSP */ 1296 case 1: /* SWGP */ 1297 break; 1298 case 2: /* SHGP */ 1299 opcode = MIPS16e_sh_op; 1300 break; 1301 default: 1302 goto sigbus; 1303 } 1304 break; 1305 1306 case MIPS16e_lwpc_op: 1307 reg = reg16to32[mips16inst.ri.rx]; 1308 break; 1309 1310 case MIPS16e_lwsp_op: 1311 reg = reg16to32[mips16inst.ri.rx]; 1312 if (extended && cpu_has_mips16e2) 1313 switch (mips16inst.ri.imm >> 5) { 1314 case 0: /* LWSP */ 1315 case 1: /* LWGP */ 1316 break; 1317 case 2: /* LHGP */ 1318 opcode = MIPS16e_lh_op; 1319 break; 1320 case 4: /* LHUGP */ 1321 opcode = MIPS16e_lhu_op; 1322 break; 1323 default: 1324 goto sigbus; 1325 } 1326 break; 1327 1328 case MIPS16e_i8_op: 1329 if (mips16inst.i8.func != MIPS16e_swrasp_func) 1330 goto sigbus; 1331 reg = 29; /* GPRSP */ 1332 break; 1333 1334 default: 1335 reg = reg16to32[mips16inst.rri.ry]; 1336 break; 1337 } 1338 1339 switch (opcode) { 1340 1341 case MIPS16e_lb_op: 1342 case MIPS16e_lbu_op: 1343 case MIPS16e_sb_op: 1344 goto sigbus; 1345 1346 case MIPS16e_lh_op: 1347 if (!access_ok(addr, 2)) 1348 goto sigbus; 1349 1350 LoadHW(addr, value, res); 1351 if (res) 1352 goto fault; 1353 MIPS16e_compute_return_epc(regs, &oldinst); 1354 regs->regs[reg] = value; 1355 break; 1356 1357 case MIPS16e_lhu_op: 1358 if (!access_ok(addr, 2)) 1359 goto sigbus; 1360 1361 LoadHWU(addr, value, res); 1362 if (res) 1363 goto fault; 1364 MIPS16e_compute_return_epc(regs, &oldinst); 1365 regs->regs[reg] = value; 1366 break; 1367 1368 case MIPS16e_lw_op: 1369 case MIPS16e_lwpc_op: 1370 case MIPS16e_lwsp_op: 1371 if (!access_ok(addr, 4)) 1372 goto sigbus; 1373 1374 LoadW(addr, value, res); 1375 if (res) 1376 goto fault; 1377 MIPS16e_compute_return_epc(regs, &oldinst); 1378 regs->regs[reg] = value; 1379 break; 1380 1381 case MIPS16e_lwu_op: 1382 #ifdef CONFIG_64BIT 1383 /* 1384 * A 32-bit kernel might be running on a 64-bit processor. But 1385 * if we're on a 32-bit processor and an i-cache incoherency 1386 * or race makes us see a 64-bit instruction here the sdl/sdr 1387 * would blow up, so for now we don't handle unaligned 64-bit 1388 * instructions on 32-bit kernels. 1389 */ 1390 if (!access_ok(addr, 4)) 1391 goto sigbus; 1392 1393 LoadWU(addr, value, res); 1394 if (res) 1395 goto fault; 1396 MIPS16e_compute_return_epc(regs, &oldinst); 1397 regs->regs[reg] = value; 1398 break; 1399 #endif /* CONFIG_64BIT */ 1400 1401 /* Cannot handle 64-bit instructions in 32-bit kernel */ 1402 goto sigill; 1403 1404 case MIPS16e_ld_op: 1405 loadDW: 1406 #ifdef CONFIG_64BIT 1407 /* 1408 * A 32-bit kernel might be running on a 64-bit processor. But 1409 * if we're on a 32-bit processor and an i-cache incoherency 1410 * or race makes us see a 64-bit instruction here the sdl/sdr 1411 * would blow up, so for now we don't handle unaligned 64-bit 1412 * instructions on 32-bit kernels. 1413 */ 1414 if (!access_ok(addr, 8)) 1415 goto sigbus; 1416 1417 LoadDW(addr, value, res); 1418 if (res) 1419 goto fault; 1420 MIPS16e_compute_return_epc(regs, &oldinst); 1421 regs->regs[reg] = value; 1422 break; 1423 #endif /* CONFIG_64BIT */ 1424 1425 /* Cannot handle 64-bit instructions in 32-bit kernel */ 1426 goto sigill; 1427 1428 case MIPS16e_sh_op: 1429 if (!access_ok(addr, 2)) 1430 goto sigbus; 1431 1432 MIPS16e_compute_return_epc(regs, &oldinst); 1433 value = regs->regs[reg]; 1434 StoreHW(addr, value, res); 1435 if (res) 1436 goto fault; 1437 break; 1438 1439 case MIPS16e_sw_op: 1440 case MIPS16e_swsp_op: 1441 case MIPS16e_i8_op: /* actually - MIPS16e_swrasp_func */ 1442 if (!access_ok(addr, 4)) 1443 goto sigbus; 1444 1445 MIPS16e_compute_return_epc(regs, &oldinst); 1446 value = regs->regs[reg]; 1447 StoreW(addr, value, res); 1448 if (res) 1449 goto fault; 1450 break; 1451 1452 case MIPS16e_sd_op: 1453 writeDW: 1454 #ifdef CONFIG_64BIT 1455 /* 1456 * A 32-bit kernel might be running on a 64-bit processor. But 1457 * if we're on a 32-bit processor and an i-cache incoherency 1458 * or race makes us see a 64-bit instruction here the sdl/sdr 1459 * would blow up, so for now we don't handle unaligned 64-bit 1460 * instructions on 32-bit kernels. 1461 */ 1462 if (!access_ok(addr, 8)) 1463 goto sigbus; 1464 1465 MIPS16e_compute_return_epc(regs, &oldinst); 1466 value = regs->regs[reg]; 1467 StoreDW(addr, value, res); 1468 if (res) 1469 goto fault; 1470 break; 1471 #endif /* CONFIG_64BIT */ 1472 1473 /* Cannot handle 64-bit instructions in 32-bit kernel */ 1474 goto sigill; 1475 1476 default: 1477 /* 1478 * Pheeee... We encountered an yet unknown instruction or 1479 * cache coherence problem. Die sucker, die ... 1480 */ 1481 goto sigill; 1482 } 1483 1484 #ifdef CONFIG_DEBUG_FS 1485 unaligned_instructions++; 1486 #endif 1487 1488 return; 1489 1490 fault: 1491 /* roll back jump/branch */ 1492 regs->cp0_epc = origpc; 1493 regs->regs[31] = orig31; 1494 /* Did we have an exception handler installed? */ 1495 if (fixup_exception(regs)) 1496 return; 1497 1498 die_if_kernel("Unhandled kernel unaligned access", regs); 1499 force_sig(SIGSEGV); 1500 1501 return; 1502 1503 sigbus: 1504 die_if_kernel("Unhandled kernel unaligned access", regs); 1505 force_sig(SIGBUS); 1506 1507 return; 1508 1509 sigill: 1510 die_if_kernel 1511 ("Unhandled kernel unaligned access or invalid instruction", regs); 1512 force_sig(SIGILL); 1513 } 1514 1515 asmlinkage void do_ade(struct pt_regs *regs) 1516 { 1517 enum ctx_state prev_state; 1518 unsigned int __user *pc; 1519 mm_segment_t seg; 1520 1521 prev_state = exception_enter(); 1522 perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1523 1, regs, regs->cp0_badvaddr); 1524 /* 1525 * Did we catch a fault trying to load an instruction? 1526 */ 1527 if (regs->cp0_badvaddr == regs->cp0_epc) 1528 goto sigbus; 1529 1530 if (user_mode(regs) && !test_thread_flag(TIF_FIXADE)) 1531 goto sigbus; 1532 if (unaligned_action == UNALIGNED_ACTION_SIGNAL) 1533 goto sigbus; 1534 1535 /* 1536 * Do branch emulation only if we didn't forward the exception. 1537 * This is all so but ugly ... 1538 */ 1539 1540 /* 1541 * Are we running in microMIPS mode? 1542 */ 1543 if (get_isa16_mode(regs->cp0_epc)) { 1544 /* 1545 * Did we catch a fault trying to load an instruction in 1546 * 16-bit mode? 1547 */ 1548 if (regs->cp0_badvaddr == msk_isa16_mode(regs->cp0_epc)) 1549 goto sigbus; 1550 if (unaligned_action == UNALIGNED_ACTION_SHOW) 1551 show_registers(regs); 1552 1553 if (cpu_has_mmips) { 1554 seg = get_fs(); 1555 if (!user_mode(regs)) 1556 set_fs(KERNEL_DS); 1557 emulate_load_store_microMIPS(regs, 1558 (void __user *)regs->cp0_badvaddr); 1559 set_fs(seg); 1560 1561 return; 1562 } 1563 1564 if (cpu_has_mips16) { 1565 seg = get_fs(); 1566 if (!user_mode(regs)) 1567 set_fs(KERNEL_DS); 1568 emulate_load_store_MIPS16e(regs, 1569 (void __user *)regs->cp0_badvaddr); 1570 set_fs(seg); 1571 1572 return; 1573 } 1574 1575 goto sigbus; 1576 } 1577 1578 if (unaligned_action == UNALIGNED_ACTION_SHOW) 1579 show_registers(regs); 1580 pc = (unsigned int __user *)exception_epc(regs); 1581 1582 seg = get_fs(); 1583 if (!user_mode(regs)) 1584 set_fs(KERNEL_DS); 1585 emulate_load_store_insn(regs, (void __user *)regs->cp0_badvaddr, pc); 1586 set_fs(seg); 1587 1588 return; 1589 1590 sigbus: 1591 die_if_kernel("Kernel unaligned instruction access", regs); 1592 force_sig(SIGBUS); 1593 1594 /* 1595 * XXX On return from the signal handler we should advance the epc 1596 */ 1597 exception_exit(prev_state); 1598 } 1599 1600 #ifdef CONFIG_DEBUG_FS 1601 static int __init debugfs_unaligned(void) 1602 { 1603 debugfs_create_u32("unaligned_instructions", S_IRUGO, mips_debugfs_dir, 1604 &unaligned_instructions); 1605 debugfs_create_u32("unaligned_action", S_IRUGO | S_IWUSR, 1606 mips_debugfs_dir, &unaligned_action); 1607 return 0; 1608 } 1609 arch_initcall(debugfs_unaligned); 1610 #endif 1611