1 /* 2 * Handle unaligned accesses by emulation. 3 * 4 * This file is subject to the terms and conditions of the GNU General Public 5 * License. See the file "COPYING" in the main directory of this archive 6 * for more details. 7 * 8 * Copyright (C) 1996, 1998, 1999, 2002 by Ralf Baechle 9 * Copyright (C) 1999 Silicon Graphics, Inc. 10 * Copyright (C) 2014 Imagination Technologies Ltd. 11 * 12 * This file contains exception handler for address error exception with the 13 * special capability to execute faulting instructions in software. The 14 * handler does not try to handle the case when the program counter points 15 * to an address not aligned to a word boundary. 16 * 17 * Putting data to unaligned addresses is a bad practice even on Intel where 18 * only the performance is affected. Much worse is that such code is non- 19 * portable. Due to several programs that die on MIPS due to alignment 20 * problems I decided to implement this handler anyway though I originally 21 * didn't intend to do this at all for user code. 22 * 23 * For now I enable fixing of address errors by default to make life easier. 24 * I however intend to disable this somewhen in the future when the alignment 25 * problems with user programs have been fixed. For programmers this is the 26 * right way to go. 27 * 28 * Fixing address errors is a per process option. The option is inherited 29 * across fork(2) and execve(2) calls. If you really want to use the 30 * option in your user programs - I discourage the use of the software 31 * emulation strongly - use the following code in your userland stuff: 32 * 33 * #include <sys/sysmips.h> 34 * 35 * ... 36 * sysmips(MIPS_FIXADE, x); 37 * ... 38 * 39 * The argument x is 0 for disabling software emulation, enabled otherwise. 40 * 41 * Below a little program to play around with this feature. 42 * 43 * #include <stdio.h> 44 * #include <sys/sysmips.h> 45 * 46 * struct foo { 47 * unsigned char bar[8]; 48 * }; 49 * 50 * main(int argc, char *argv[]) 51 * { 52 * struct foo x = {0, 1, 2, 3, 4, 5, 6, 7}; 53 * unsigned int *p = (unsigned int *) (x.bar + 3); 54 * int i; 55 * 56 * if (argc > 1) 57 * sysmips(MIPS_FIXADE, atoi(argv[1])); 58 * 59 * printf("*p = %08lx\n", *p); 60 * 61 * *p = 0xdeadface; 62 * 63 * for(i = 0; i <= 7; i++) 64 * printf("%02x ", x.bar[i]); 65 * printf("\n"); 66 * } 67 * 68 * Coprocessor loads are not supported; I think this case is unimportant 69 * in the practice. 70 * 71 * TODO: Handle ndc (attempted store to doubleword in uncached memory) 72 * exception for the R6000. 73 * A store crossing a page boundary might be executed only partially. 74 * Undo the partial store in this case. 75 */ 76 #include <linux/context_tracking.h> 77 #include <linux/mm.h> 78 #include <linux/signal.h> 79 #include <linux/smp.h> 80 #include <linux/sched.h> 81 #include <linux/debugfs.h> 82 #include <linux/perf_event.h> 83 84 #include <asm/asm.h> 85 #include <asm/branch.h> 86 #include <asm/byteorder.h> 87 #include <asm/cop2.h> 88 #include <asm/debug.h> 89 #include <asm/fpu.h> 90 #include <asm/fpu_emulator.h> 91 #include <asm/inst.h> 92 #include <linux/uaccess.h> 93 94 #define STR(x) __STR(x) 95 #define __STR(x) #x 96 97 enum { 98 UNALIGNED_ACTION_QUIET, 99 UNALIGNED_ACTION_SIGNAL, 100 UNALIGNED_ACTION_SHOW, 101 }; 102 #ifdef CONFIG_DEBUG_FS 103 static u32 unaligned_instructions; 104 static u32 unaligned_action; 105 #else 106 #define unaligned_action UNALIGNED_ACTION_QUIET 107 #endif 108 extern void show_registers(struct pt_regs *regs); 109 110 #ifdef __BIG_ENDIAN 111 #define _LoadHW(addr, value, res, type) \ 112 do { \ 113 __asm__ __volatile__ (".set\tnoat\n" \ 114 "1:\t"type##_lb("%0", "0(%2)")"\n" \ 115 "2:\t"type##_lbu("$1", "1(%2)")"\n\t"\ 116 "sll\t%0, 0x8\n\t" \ 117 "or\t%0, $1\n\t" \ 118 "li\t%1, 0\n" \ 119 "3:\t.set\tat\n\t" \ 120 ".insn\n\t" \ 121 ".section\t.fixup,\"ax\"\n\t" \ 122 "4:\tli\t%1, %3\n\t" \ 123 "j\t3b\n\t" \ 124 ".previous\n\t" \ 125 ".section\t__ex_table,\"a\"\n\t" \ 126 STR(PTR)"\t1b, 4b\n\t" \ 127 STR(PTR)"\t2b, 4b\n\t" \ 128 ".previous" \ 129 : "=&r" (value), "=r" (res) \ 130 : "r" (addr), "i" (-EFAULT)); \ 131 } while(0) 132 133 #ifdef CONFIG_CPU_HAS_LOAD_STORE_LR 134 #define _LoadW(addr, value, res, type) \ 135 do { \ 136 __asm__ __volatile__ ( \ 137 "1:\t"type##_lwl("%0", "(%2)")"\n" \ 138 "2:\t"type##_lwr("%0", "3(%2)")"\n\t"\ 139 "li\t%1, 0\n" \ 140 "3:\n\t" \ 141 ".insn\n\t" \ 142 ".section\t.fixup,\"ax\"\n\t" \ 143 "4:\tli\t%1, %3\n\t" \ 144 "j\t3b\n\t" \ 145 ".previous\n\t" \ 146 ".section\t__ex_table,\"a\"\n\t" \ 147 STR(PTR)"\t1b, 4b\n\t" \ 148 STR(PTR)"\t2b, 4b\n\t" \ 149 ".previous" \ 150 : "=&r" (value), "=r" (res) \ 151 : "r" (addr), "i" (-EFAULT)); \ 152 } while(0) 153 154 #else /* !CONFIG_CPU_HAS_LOAD_STORE_LR */ 155 /* For CPUs without lwl instruction */ 156 #define _LoadW(addr, value, res, type) \ 157 do { \ 158 __asm__ __volatile__ ( \ 159 ".set\tpush\n" \ 160 ".set\tnoat\n\t" \ 161 "1:"type##_lb("%0", "0(%2)")"\n\t" \ 162 "2:"type##_lbu("$1", "1(%2)")"\n\t" \ 163 "sll\t%0, 0x8\n\t" \ 164 "or\t%0, $1\n\t" \ 165 "3:"type##_lbu("$1", "2(%2)")"\n\t" \ 166 "sll\t%0, 0x8\n\t" \ 167 "or\t%0, $1\n\t" \ 168 "4:"type##_lbu("$1", "3(%2)")"\n\t" \ 169 "sll\t%0, 0x8\n\t" \ 170 "or\t%0, $1\n\t" \ 171 "li\t%1, 0\n" \ 172 ".set\tpop\n" \ 173 "10:\n\t" \ 174 ".insn\n\t" \ 175 ".section\t.fixup,\"ax\"\n\t" \ 176 "11:\tli\t%1, %3\n\t" \ 177 "j\t10b\n\t" \ 178 ".previous\n\t" \ 179 ".section\t__ex_table,\"a\"\n\t" \ 180 STR(PTR)"\t1b, 11b\n\t" \ 181 STR(PTR)"\t2b, 11b\n\t" \ 182 STR(PTR)"\t3b, 11b\n\t" \ 183 STR(PTR)"\t4b, 11b\n\t" \ 184 ".previous" \ 185 : "=&r" (value), "=r" (res) \ 186 : "r" (addr), "i" (-EFAULT)); \ 187 } while(0) 188 189 #endif /* !CONFIG_CPU_HAS_LOAD_STORE_LR */ 190 191 #define _LoadHWU(addr, value, res, type) \ 192 do { \ 193 __asm__ __volatile__ ( \ 194 ".set\tnoat\n" \ 195 "1:\t"type##_lbu("%0", "0(%2)")"\n" \ 196 "2:\t"type##_lbu("$1", "1(%2)")"\n\t"\ 197 "sll\t%0, 0x8\n\t" \ 198 "or\t%0, $1\n\t" \ 199 "li\t%1, 0\n" \ 200 "3:\n\t" \ 201 ".insn\n\t" \ 202 ".set\tat\n\t" \ 203 ".section\t.fixup,\"ax\"\n\t" \ 204 "4:\tli\t%1, %3\n\t" \ 205 "j\t3b\n\t" \ 206 ".previous\n\t" \ 207 ".section\t__ex_table,\"a\"\n\t" \ 208 STR(PTR)"\t1b, 4b\n\t" \ 209 STR(PTR)"\t2b, 4b\n\t" \ 210 ".previous" \ 211 : "=&r" (value), "=r" (res) \ 212 : "r" (addr), "i" (-EFAULT)); \ 213 } while(0) 214 215 #ifdef CONFIG_CPU_HAS_LOAD_STORE_LR 216 #define _LoadWU(addr, value, res, type) \ 217 do { \ 218 __asm__ __volatile__ ( \ 219 "1:\t"type##_lwl("%0", "(%2)")"\n" \ 220 "2:\t"type##_lwr("%0", "3(%2)")"\n\t"\ 221 "dsll\t%0, %0, 32\n\t" \ 222 "dsrl\t%0, %0, 32\n\t" \ 223 "li\t%1, 0\n" \ 224 "3:\n\t" \ 225 ".insn\n\t" \ 226 "\t.section\t.fixup,\"ax\"\n\t" \ 227 "4:\tli\t%1, %3\n\t" \ 228 "j\t3b\n\t" \ 229 ".previous\n\t" \ 230 ".section\t__ex_table,\"a\"\n\t" \ 231 STR(PTR)"\t1b, 4b\n\t" \ 232 STR(PTR)"\t2b, 4b\n\t" \ 233 ".previous" \ 234 : "=&r" (value), "=r" (res) \ 235 : "r" (addr), "i" (-EFAULT)); \ 236 } while(0) 237 238 #define _LoadDW(addr, value, res) \ 239 do { \ 240 __asm__ __volatile__ ( \ 241 "1:\tldl\t%0, (%2)\n" \ 242 "2:\tldr\t%0, 7(%2)\n\t" \ 243 "li\t%1, 0\n" \ 244 "3:\n\t" \ 245 ".insn\n\t" \ 246 "\t.section\t.fixup,\"ax\"\n\t" \ 247 "4:\tli\t%1, %3\n\t" \ 248 "j\t3b\n\t" \ 249 ".previous\n\t" \ 250 ".section\t__ex_table,\"a\"\n\t" \ 251 STR(PTR)"\t1b, 4b\n\t" \ 252 STR(PTR)"\t2b, 4b\n\t" \ 253 ".previous" \ 254 : "=&r" (value), "=r" (res) \ 255 : "r" (addr), "i" (-EFAULT)); \ 256 } while(0) 257 258 #else /* !CONFIG_CPU_HAS_LOAD_STORE_LR */ 259 /* For CPUs without lwl and ldl instructions */ 260 #define _LoadWU(addr, value, res, type) \ 261 do { \ 262 __asm__ __volatile__ ( \ 263 ".set\tpush\n\t" \ 264 ".set\tnoat\n\t" \ 265 "1:"type##_lbu("%0", "0(%2)")"\n\t" \ 266 "2:"type##_lbu("$1", "1(%2)")"\n\t" \ 267 "sll\t%0, 0x8\n\t" \ 268 "or\t%0, $1\n\t" \ 269 "3:"type##_lbu("$1", "2(%2)")"\n\t" \ 270 "sll\t%0, 0x8\n\t" \ 271 "or\t%0, $1\n\t" \ 272 "4:"type##_lbu("$1", "3(%2)")"\n\t" \ 273 "sll\t%0, 0x8\n\t" \ 274 "or\t%0, $1\n\t" \ 275 "li\t%1, 0\n" \ 276 ".set\tpop\n" \ 277 "10:\n\t" \ 278 ".insn\n\t" \ 279 ".section\t.fixup,\"ax\"\n\t" \ 280 "11:\tli\t%1, %3\n\t" \ 281 "j\t10b\n\t" \ 282 ".previous\n\t" \ 283 ".section\t__ex_table,\"a\"\n\t" \ 284 STR(PTR)"\t1b, 11b\n\t" \ 285 STR(PTR)"\t2b, 11b\n\t" \ 286 STR(PTR)"\t3b, 11b\n\t" \ 287 STR(PTR)"\t4b, 11b\n\t" \ 288 ".previous" \ 289 : "=&r" (value), "=r" (res) \ 290 : "r" (addr), "i" (-EFAULT)); \ 291 } while(0) 292 293 #define _LoadDW(addr, value, res) \ 294 do { \ 295 __asm__ __volatile__ ( \ 296 ".set\tpush\n\t" \ 297 ".set\tnoat\n\t" \ 298 "1:lb\t%0, 0(%2)\n\t" \ 299 "2:lbu\t $1, 1(%2)\n\t" \ 300 "dsll\t%0, 0x8\n\t" \ 301 "or\t%0, $1\n\t" \ 302 "3:lbu\t$1, 2(%2)\n\t" \ 303 "dsll\t%0, 0x8\n\t" \ 304 "or\t%0, $1\n\t" \ 305 "4:lbu\t$1, 3(%2)\n\t" \ 306 "dsll\t%0, 0x8\n\t" \ 307 "or\t%0, $1\n\t" \ 308 "5:lbu\t$1, 4(%2)\n\t" \ 309 "dsll\t%0, 0x8\n\t" \ 310 "or\t%0, $1\n\t" \ 311 "6:lbu\t$1, 5(%2)\n\t" \ 312 "dsll\t%0, 0x8\n\t" \ 313 "or\t%0, $1\n\t" \ 314 "7:lbu\t$1, 6(%2)\n\t" \ 315 "dsll\t%0, 0x8\n\t" \ 316 "or\t%0, $1\n\t" \ 317 "8:lbu\t$1, 7(%2)\n\t" \ 318 "dsll\t%0, 0x8\n\t" \ 319 "or\t%0, $1\n\t" \ 320 "li\t%1, 0\n" \ 321 ".set\tpop\n\t" \ 322 "10:\n\t" \ 323 ".insn\n\t" \ 324 ".section\t.fixup,\"ax\"\n\t" \ 325 "11:\tli\t%1, %3\n\t" \ 326 "j\t10b\n\t" \ 327 ".previous\n\t" \ 328 ".section\t__ex_table,\"a\"\n\t" \ 329 STR(PTR)"\t1b, 11b\n\t" \ 330 STR(PTR)"\t2b, 11b\n\t" \ 331 STR(PTR)"\t3b, 11b\n\t" \ 332 STR(PTR)"\t4b, 11b\n\t" \ 333 STR(PTR)"\t5b, 11b\n\t" \ 334 STR(PTR)"\t6b, 11b\n\t" \ 335 STR(PTR)"\t7b, 11b\n\t" \ 336 STR(PTR)"\t8b, 11b\n\t" \ 337 ".previous" \ 338 : "=&r" (value), "=r" (res) \ 339 : "r" (addr), "i" (-EFAULT)); \ 340 } while(0) 341 342 #endif /* !CONFIG_CPU_HAS_LOAD_STORE_LR */ 343 344 345 #define _StoreHW(addr, value, res, type) \ 346 do { \ 347 __asm__ __volatile__ ( \ 348 ".set\tnoat\n" \ 349 "1:\t"type##_sb("%1", "1(%2)")"\n" \ 350 "srl\t$1, %1, 0x8\n" \ 351 "2:\t"type##_sb("$1", "0(%2)")"\n" \ 352 ".set\tat\n\t" \ 353 "li\t%0, 0\n" \ 354 "3:\n\t" \ 355 ".insn\n\t" \ 356 ".section\t.fixup,\"ax\"\n\t" \ 357 "4:\tli\t%0, %3\n\t" \ 358 "j\t3b\n\t" \ 359 ".previous\n\t" \ 360 ".section\t__ex_table,\"a\"\n\t" \ 361 STR(PTR)"\t1b, 4b\n\t" \ 362 STR(PTR)"\t2b, 4b\n\t" \ 363 ".previous" \ 364 : "=r" (res) \ 365 : "r" (value), "r" (addr), "i" (-EFAULT));\ 366 } while(0) 367 368 #ifdef CONFIG_CPU_HAS_LOAD_STORE_LR 369 #define _StoreW(addr, value, res, type) \ 370 do { \ 371 __asm__ __volatile__ ( \ 372 "1:\t"type##_swl("%1", "(%2)")"\n" \ 373 "2:\t"type##_swr("%1", "3(%2)")"\n\t"\ 374 "li\t%0, 0\n" \ 375 "3:\n\t" \ 376 ".insn\n\t" \ 377 ".section\t.fixup,\"ax\"\n\t" \ 378 "4:\tli\t%0, %3\n\t" \ 379 "j\t3b\n\t" \ 380 ".previous\n\t" \ 381 ".section\t__ex_table,\"a\"\n\t" \ 382 STR(PTR)"\t1b, 4b\n\t" \ 383 STR(PTR)"\t2b, 4b\n\t" \ 384 ".previous" \ 385 : "=r" (res) \ 386 : "r" (value), "r" (addr), "i" (-EFAULT)); \ 387 } while(0) 388 389 #define _StoreDW(addr, value, res) \ 390 do { \ 391 __asm__ __volatile__ ( \ 392 "1:\tsdl\t%1,(%2)\n" \ 393 "2:\tsdr\t%1, 7(%2)\n\t" \ 394 "li\t%0, 0\n" \ 395 "3:\n\t" \ 396 ".insn\n\t" \ 397 ".section\t.fixup,\"ax\"\n\t" \ 398 "4:\tli\t%0, %3\n\t" \ 399 "j\t3b\n\t" \ 400 ".previous\n\t" \ 401 ".section\t__ex_table,\"a\"\n\t" \ 402 STR(PTR)"\t1b, 4b\n\t" \ 403 STR(PTR)"\t2b, 4b\n\t" \ 404 ".previous" \ 405 : "=r" (res) \ 406 : "r" (value), "r" (addr), "i" (-EFAULT)); \ 407 } while(0) 408 409 #else /* !CONFIG_CPU_HAS_LOAD_STORE_LR */ 410 #define _StoreW(addr, value, res, type) \ 411 do { \ 412 __asm__ __volatile__ ( \ 413 ".set\tpush\n\t" \ 414 ".set\tnoat\n\t" \ 415 "1:"type##_sb("%1", "3(%2)")"\n\t" \ 416 "srl\t$1, %1, 0x8\n\t" \ 417 "2:"type##_sb("$1", "2(%2)")"\n\t" \ 418 "srl\t$1, $1, 0x8\n\t" \ 419 "3:"type##_sb("$1", "1(%2)")"\n\t" \ 420 "srl\t$1, $1, 0x8\n\t" \ 421 "4:"type##_sb("$1", "0(%2)")"\n\t" \ 422 ".set\tpop\n\t" \ 423 "li\t%0, 0\n" \ 424 "10:\n\t" \ 425 ".insn\n\t" \ 426 ".section\t.fixup,\"ax\"\n\t" \ 427 "11:\tli\t%0, %3\n\t" \ 428 "j\t10b\n\t" \ 429 ".previous\n\t" \ 430 ".section\t__ex_table,\"a\"\n\t" \ 431 STR(PTR)"\t1b, 11b\n\t" \ 432 STR(PTR)"\t2b, 11b\n\t" \ 433 STR(PTR)"\t3b, 11b\n\t" \ 434 STR(PTR)"\t4b, 11b\n\t" \ 435 ".previous" \ 436 : "=&r" (res) \ 437 : "r" (value), "r" (addr), "i" (-EFAULT) \ 438 : "memory"); \ 439 } while(0) 440 441 #define _StoreDW(addr, value, res) \ 442 do { \ 443 __asm__ __volatile__ ( \ 444 ".set\tpush\n\t" \ 445 ".set\tnoat\n\t" \ 446 "1:sb\t%1, 7(%2)\n\t" \ 447 "dsrl\t$1, %1, 0x8\n\t" \ 448 "2:sb\t$1, 6(%2)\n\t" \ 449 "dsrl\t$1, $1, 0x8\n\t" \ 450 "3:sb\t$1, 5(%2)\n\t" \ 451 "dsrl\t$1, $1, 0x8\n\t" \ 452 "4:sb\t$1, 4(%2)\n\t" \ 453 "dsrl\t$1, $1, 0x8\n\t" \ 454 "5:sb\t$1, 3(%2)\n\t" \ 455 "dsrl\t$1, $1, 0x8\n\t" \ 456 "6:sb\t$1, 2(%2)\n\t" \ 457 "dsrl\t$1, $1, 0x8\n\t" \ 458 "7:sb\t$1, 1(%2)\n\t" \ 459 "dsrl\t$1, $1, 0x8\n\t" \ 460 "8:sb\t$1, 0(%2)\n\t" \ 461 "dsrl\t$1, $1, 0x8\n\t" \ 462 ".set\tpop\n\t" \ 463 "li\t%0, 0\n" \ 464 "10:\n\t" \ 465 ".insn\n\t" \ 466 ".section\t.fixup,\"ax\"\n\t" \ 467 "11:\tli\t%0, %3\n\t" \ 468 "j\t10b\n\t" \ 469 ".previous\n\t" \ 470 ".section\t__ex_table,\"a\"\n\t" \ 471 STR(PTR)"\t1b, 11b\n\t" \ 472 STR(PTR)"\t2b, 11b\n\t" \ 473 STR(PTR)"\t3b, 11b\n\t" \ 474 STR(PTR)"\t4b, 11b\n\t" \ 475 STR(PTR)"\t5b, 11b\n\t" \ 476 STR(PTR)"\t6b, 11b\n\t" \ 477 STR(PTR)"\t7b, 11b\n\t" \ 478 STR(PTR)"\t8b, 11b\n\t" \ 479 ".previous" \ 480 : "=&r" (res) \ 481 : "r" (value), "r" (addr), "i" (-EFAULT) \ 482 : "memory"); \ 483 } while(0) 484 485 #endif /* !CONFIG_CPU_HAS_LOAD_STORE_LR */ 486 487 #else /* __BIG_ENDIAN */ 488 489 #define _LoadHW(addr, value, res, type) \ 490 do { \ 491 __asm__ __volatile__ (".set\tnoat\n" \ 492 "1:\t"type##_lb("%0", "1(%2)")"\n" \ 493 "2:\t"type##_lbu("$1", "0(%2)")"\n\t"\ 494 "sll\t%0, 0x8\n\t" \ 495 "or\t%0, $1\n\t" \ 496 "li\t%1, 0\n" \ 497 "3:\t.set\tat\n\t" \ 498 ".insn\n\t" \ 499 ".section\t.fixup,\"ax\"\n\t" \ 500 "4:\tli\t%1, %3\n\t" \ 501 "j\t3b\n\t" \ 502 ".previous\n\t" \ 503 ".section\t__ex_table,\"a\"\n\t" \ 504 STR(PTR)"\t1b, 4b\n\t" \ 505 STR(PTR)"\t2b, 4b\n\t" \ 506 ".previous" \ 507 : "=&r" (value), "=r" (res) \ 508 : "r" (addr), "i" (-EFAULT)); \ 509 } while(0) 510 511 #ifdef CONFIG_CPU_HAS_LOAD_STORE_LR 512 #define _LoadW(addr, value, res, type) \ 513 do { \ 514 __asm__ __volatile__ ( \ 515 "1:\t"type##_lwl("%0", "3(%2)")"\n" \ 516 "2:\t"type##_lwr("%0", "(%2)")"\n\t"\ 517 "li\t%1, 0\n" \ 518 "3:\n\t" \ 519 ".insn\n\t" \ 520 ".section\t.fixup,\"ax\"\n\t" \ 521 "4:\tli\t%1, %3\n\t" \ 522 "j\t3b\n\t" \ 523 ".previous\n\t" \ 524 ".section\t__ex_table,\"a\"\n\t" \ 525 STR(PTR)"\t1b, 4b\n\t" \ 526 STR(PTR)"\t2b, 4b\n\t" \ 527 ".previous" \ 528 : "=&r" (value), "=r" (res) \ 529 : "r" (addr), "i" (-EFAULT)); \ 530 } while(0) 531 532 #else /* !CONFIG_CPU_HAS_LOAD_STORE_LR */ 533 /* For CPUs without lwl instruction */ 534 #define _LoadW(addr, value, res, type) \ 535 do { \ 536 __asm__ __volatile__ ( \ 537 ".set\tpush\n" \ 538 ".set\tnoat\n\t" \ 539 "1:"type##_lb("%0", "3(%2)")"\n\t" \ 540 "2:"type##_lbu("$1", "2(%2)")"\n\t" \ 541 "sll\t%0, 0x8\n\t" \ 542 "or\t%0, $1\n\t" \ 543 "3:"type##_lbu("$1", "1(%2)")"\n\t" \ 544 "sll\t%0, 0x8\n\t" \ 545 "or\t%0, $1\n\t" \ 546 "4:"type##_lbu("$1", "0(%2)")"\n\t" \ 547 "sll\t%0, 0x8\n\t" \ 548 "or\t%0, $1\n\t" \ 549 "li\t%1, 0\n" \ 550 ".set\tpop\n" \ 551 "10:\n\t" \ 552 ".insn\n\t" \ 553 ".section\t.fixup,\"ax\"\n\t" \ 554 "11:\tli\t%1, %3\n\t" \ 555 "j\t10b\n\t" \ 556 ".previous\n\t" \ 557 ".section\t__ex_table,\"a\"\n\t" \ 558 STR(PTR)"\t1b, 11b\n\t" \ 559 STR(PTR)"\t2b, 11b\n\t" \ 560 STR(PTR)"\t3b, 11b\n\t" \ 561 STR(PTR)"\t4b, 11b\n\t" \ 562 ".previous" \ 563 : "=&r" (value), "=r" (res) \ 564 : "r" (addr), "i" (-EFAULT)); \ 565 } while(0) 566 567 #endif /* !CONFIG_CPU_HAS_LOAD_STORE_LR */ 568 569 570 #define _LoadHWU(addr, value, res, type) \ 571 do { \ 572 __asm__ __volatile__ ( \ 573 ".set\tnoat\n" \ 574 "1:\t"type##_lbu("%0", "1(%2)")"\n" \ 575 "2:\t"type##_lbu("$1", "0(%2)")"\n\t"\ 576 "sll\t%0, 0x8\n\t" \ 577 "or\t%0, $1\n\t" \ 578 "li\t%1, 0\n" \ 579 "3:\n\t" \ 580 ".insn\n\t" \ 581 ".set\tat\n\t" \ 582 ".section\t.fixup,\"ax\"\n\t" \ 583 "4:\tli\t%1, %3\n\t" \ 584 "j\t3b\n\t" \ 585 ".previous\n\t" \ 586 ".section\t__ex_table,\"a\"\n\t" \ 587 STR(PTR)"\t1b, 4b\n\t" \ 588 STR(PTR)"\t2b, 4b\n\t" \ 589 ".previous" \ 590 : "=&r" (value), "=r" (res) \ 591 : "r" (addr), "i" (-EFAULT)); \ 592 } while(0) 593 594 #ifdef CONFIG_CPU_HAS_LOAD_STORE_LR 595 #define _LoadWU(addr, value, res, type) \ 596 do { \ 597 __asm__ __volatile__ ( \ 598 "1:\t"type##_lwl("%0", "3(%2)")"\n" \ 599 "2:\t"type##_lwr("%0", "(%2)")"\n\t"\ 600 "dsll\t%0, %0, 32\n\t" \ 601 "dsrl\t%0, %0, 32\n\t" \ 602 "li\t%1, 0\n" \ 603 "3:\n\t" \ 604 ".insn\n\t" \ 605 "\t.section\t.fixup,\"ax\"\n\t" \ 606 "4:\tli\t%1, %3\n\t" \ 607 "j\t3b\n\t" \ 608 ".previous\n\t" \ 609 ".section\t__ex_table,\"a\"\n\t" \ 610 STR(PTR)"\t1b, 4b\n\t" \ 611 STR(PTR)"\t2b, 4b\n\t" \ 612 ".previous" \ 613 : "=&r" (value), "=r" (res) \ 614 : "r" (addr), "i" (-EFAULT)); \ 615 } while(0) 616 617 #define _LoadDW(addr, value, res) \ 618 do { \ 619 __asm__ __volatile__ ( \ 620 "1:\tldl\t%0, 7(%2)\n" \ 621 "2:\tldr\t%0, (%2)\n\t" \ 622 "li\t%1, 0\n" \ 623 "3:\n\t" \ 624 ".insn\n\t" \ 625 "\t.section\t.fixup,\"ax\"\n\t" \ 626 "4:\tli\t%1, %3\n\t" \ 627 "j\t3b\n\t" \ 628 ".previous\n\t" \ 629 ".section\t__ex_table,\"a\"\n\t" \ 630 STR(PTR)"\t1b, 4b\n\t" \ 631 STR(PTR)"\t2b, 4b\n\t" \ 632 ".previous" \ 633 : "=&r" (value), "=r" (res) \ 634 : "r" (addr), "i" (-EFAULT)); \ 635 } while(0) 636 637 #else /* !CONFIG_CPU_HAS_LOAD_STORE_LR */ 638 /* For CPUs without lwl and ldl instructions */ 639 #define _LoadWU(addr, value, res, type) \ 640 do { \ 641 __asm__ __volatile__ ( \ 642 ".set\tpush\n\t" \ 643 ".set\tnoat\n\t" \ 644 "1:"type##_lbu("%0", "3(%2)")"\n\t" \ 645 "2:"type##_lbu("$1", "2(%2)")"\n\t" \ 646 "sll\t%0, 0x8\n\t" \ 647 "or\t%0, $1\n\t" \ 648 "3:"type##_lbu("$1", "1(%2)")"\n\t" \ 649 "sll\t%0, 0x8\n\t" \ 650 "or\t%0, $1\n\t" \ 651 "4:"type##_lbu("$1", "0(%2)")"\n\t" \ 652 "sll\t%0, 0x8\n\t" \ 653 "or\t%0, $1\n\t" \ 654 "li\t%1, 0\n" \ 655 ".set\tpop\n" \ 656 "10:\n\t" \ 657 ".insn\n\t" \ 658 ".section\t.fixup,\"ax\"\n\t" \ 659 "11:\tli\t%1, %3\n\t" \ 660 "j\t10b\n\t" \ 661 ".previous\n\t" \ 662 ".section\t__ex_table,\"a\"\n\t" \ 663 STR(PTR)"\t1b, 11b\n\t" \ 664 STR(PTR)"\t2b, 11b\n\t" \ 665 STR(PTR)"\t3b, 11b\n\t" \ 666 STR(PTR)"\t4b, 11b\n\t" \ 667 ".previous" \ 668 : "=&r" (value), "=r" (res) \ 669 : "r" (addr), "i" (-EFAULT)); \ 670 } while(0) 671 672 #define _LoadDW(addr, value, res) \ 673 do { \ 674 __asm__ __volatile__ ( \ 675 ".set\tpush\n\t" \ 676 ".set\tnoat\n\t" \ 677 "1:lb\t%0, 7(%2)\n\t" \ 678 "2:lbu\t$1, 6(%2)\n\t" \ 679 "dsll\t%0, 0x8\n\t" \ 680 "or\t%0, $1\n\t" \ 681 "3:lbu\t$1, 5(%2)\n\t" \ 682 "dsll\t%0, 0x8\n\t" \ 683 "or\t%0, $1\n\t" \ 684 "4:lbu\t$1, 4(%2)\n\t" \ 685 "dsll\t%0, 0x8\n\t" \ 686 "or\t%0, $1\n\t" \ 687 "5:lbu\t$1, 3(%2)\n\t" \ 688 "dsll\t%0, 0x8\n\t" \ 689 "or\t%0, $1\n\t" \ 690 "6:lbu\t$1, 2(%2)\n\t" \ 691 "dsll\t%0, 0x8\n\t" \ 692 "or\t%0, $1\n\t" \ 693 "7:lbu\t$1, 1(%2)\n\t" \ 694 "dsll\t%0, 0x8\n\t" \ 695 "or\t%0, $1\n\t" \ 696 "8:lbu\t$1, 0(%2)\n\t" \ 697 "dsll\t%0, 0x8\n\t" \ 698 "or\t%0, $1\n\t" \ 699 "li\t%1, 0\n" \ 700 ".set\tpop\n\t" \ 701 "10:\n\t" \ 702 ".insn\n\t" \ 703 ".section\t.fixup,\"ax\"\n\t" \ 704 "11:\tli\t%1, %3\n\t" \ 705 "j\t10b\n\t" \ 706 ".previous\n\t" \ 707 ".section\t__ex_table,\"a\"\n\t" \ 708 STR(PTR)"\t1b, 11b\n\t" \ 709 STR(PTR)"\t2b, 11b\n\t" \ 710 STR(PTR)"\t3b, 11b\n\t" \ 711 STR(PTR)"\t4b, 11b\n\t" \ 712 STR(PTR)"\t5b, 11b\n\t" \ 713 STR(PTR)"\t6b, 11b\n\t" \ 714 STR(PTR)"\t7b, 11b\n\t" \ 715 STR(PTR)"\t8b, 11b\n\t" \ 716 ".previous" \ 717 : "=&r" (value), "=r" (res) \ 718 : "r" (addr), "i" (-EFAULT)); \ 719 } while(0) 720 #endif /* !CONFIG_CPU_HAS_LOAD_STORE_LR */ 721 722 #define _StoreHW(addr, value, res, type) \ 723 do { \ 724 __asm__ __volatile__ ( \ 725 ".set\tnoat\n" \ 726 "1:\t"type##_sb("%1", "0(%2)")"\n" \ 727 "srl\t$1,%1, 0x8\n" \ 728 "2:\t"type##_sb("$1", "1(%2)")"\n" \ 729 ".set\tat\n\t" \ 730 "li\t%0, 0\n" \ 731 "3:\n\t" \ 732 ".insn\n\t" \ 733 ".section\t.fixup,\"ax\"\n\t" \ 734 "4:\tli\t%0, %3\n\t" \ 735 "j\t3b\n\t" \ 736 ".previous\n\t" \ 737 ".section\t__ex_table,\"a\"\n\t" \ 738 STR(PTR)"\t1b, 4b\n\t" \ 739 STR(PTR)"\t2b, 4b\n\t" \ 740 ".previous" \ 741 : "=r" (res) \ 742 : "r" (value), "r" (addr), "i" (-EFAULT));\ 743 } while(0) 744 745 #ifdef CONFIG_CPU_HAS_LOAD_STORE_LR 746 #define _StoreW(addr, value, res, type) \ 747 do { \ 748 __asm__ __volatile__ ( \ 749 "1:\t"type##_swl("%1", "3(%2)")"\n" \ 750 "2:\t"type##_swr("%1", "(%2)")"\n\t"\ 751 "li\t%0, 0\n" \ 752 "3:\n\t" \ 753 ".insn\n\t" \ 754 ".section\t.fixup,\"ax\"\n\t" \ 755 "4:\tli\t%0, %3\n\t" \ 756 "j\t3b\n\t" \ 757 ".previous\n\t" \ 758 ".section\t__ex_table,\"a\"\n\t" \ 759 STR(PTR)"\t1b, 4b\n\t" \ 760 STR(PTR)"\t2b, 4b\n\t" \ 761 ".previous" \ 762 : "=r" (res) \ 763 : "r" (value), "r" (addr), "i" (-EFAULT)); \ 764 } while(0) 765 766 #define _StoreDW(addr, value, res) \ 767 do { \ 768 __asm__ __volatile__ ( \ 769 "1:\tsdl\t%1, 7(%2)\n" \ 770 "2:\tsdr\t%1, (%2)\n\t" \ 771 "li\t%0, 0\n" \ 772 "3:\n\t" \ 773 ".insn\n\t" \ 774 ".section\t.fixup,\"ax\"\n\t" \ 775 "4:\tli\t%0, %3\n\t" \ 776 "j\t3b\n\t" \ 777 ".previous\n\t" \ 778 ".section\t__ex_table,\"a\"\n\t" \ 779 STR(PTR)"\t1b, 4b\n\t" \ 780 STR(PTR)"\t2b, 4b\n\t" \ 781 ".previous" \ 782 : "=r" (res) \ 783 : "r" (value), "r" (addr), "i" (-EFAULT)); \ 784 } while(0) 785 786 #else /* !CONFIG_CPU_HAS_LOAD_STORE_LR */ 787 /* For CPUs without swl and sdl instructions */ 788 #define _StoreW(addr, value, res, type) \ 789 do { \ 790 __asm__ __volatile__ ( \ 791 ".set\tpush\n\t" \ 792 ".set\tnoat\n\t" \ 793 "1:"type##_sb("%1", "0(%2)")"\n\t" \ 794 "srl\t$1, %1, 0x8\n\t" \ 795 "2:"type##_sb("$1", "1(%2)")"\n\t" \ 796 "srl\t$1, $1, 0x8\n\t" \ 797 "3:"type##_sb("$1", "2(%2)")"\n\t" \ 798 "srl\t$1, $1, 0x8\n\t" \ 799 "4:"type##_sb("$1", "3(%2)")"\n\t" \ 800 ".set\tpop\n\t" \ 801 "li\t%0, 0\n" \ 802 "10:\n\t" \ 803 ".insn\n\t" \ 804 ".section\t.fixup,\"ax\"\n\t" \ 805 "11:\tli\t%0, %3\n\t" \ 806 "j\t10b\n\t" \ 807 ".previous\n\t" \ 808 ".section\t__ex_table,\"a\"\n\t" \ 809 STR(PTR)"\t1b, 11b\n\t" \ 810 STR(PTR)"\t2b, 11b\n\t" \ 811 STR(PTR)"\t3b, 11b\n\t" \ 812 STR(PTR)"\t4b, 11b\n\t" \ 813 ".previous" \ 814 : "=&r" (res) \ 815 : "r" (value), "r" (addr), "i" (-EFAULT) \ 816 : "memory"); \ 817 } while(0) 818 819 #define _StoreDW(addr, value, res) \ 820 do { \ 821 __asm__ __volatile__ ( \ 822 ".set\tpush\n\t" \ 823 ".set\tnoat\n\t" \ 824 "1:sb\t%1, 0(%2)\n\t" \ 825 "dsrl\t$1, %1, 0x8\n\t" \ 826 "2:sb\t$1, 1(%2)\n\t" \ 827 "dsrl\t$1, $1, 0x8\n\t" \ 828 "3:sb\t$1, 2(%2)\n\t" \ 829 "dsrl\t$1, $1, 0x8\n\t" \ 830 "4:sb\t$1, 3(%2)\n\t" \ 831 "dsrl\t$1, $1, 0x8\n\t" \ 832 "5:sb\t$1, 4(%2)\n\t" \ 833 "dsrl\t$1, $1, 0x8\n\t" \ 834 "6:sb\t$1, 5(%2)\n\t" \ 835 "dsrl\t$1, $1, 0x8\n\t" \ 836 "7:sb\t$1, 6(%2)\n\t" \ 837 "dsrl\t$1, $1, 0x8\n\t" \ 838 "8:sb\t$1, 7(%2)\n\t" \ 839 "dsrl\t$1, $1, 0x8\n\t" \ 840 ".set\tpop\n\t" \ 841 "li\t%0, 0\n" \ 842 "10:\n\t" \ 843 ".insn\n\t" \ 844 ".section\t.fixup,\"ax\"\n\t" \ 845 "11:\tli\t%0, %3\n\t" \ 846 "j\t10b\n\t" \ 847 ".previous\n\t" \ 848 ".section\t__ex_table,\"a\"\n\t" \ 849 STR(PTR)"\t1b, 11b\n\t" \ 850 STR(PTR)"\t2b, 11b\n\t" \ 851 STR(PTR)"\t3b, 11b\n\t" \ 852 STR(PTR)"\t4b, 11b\n\t" \ 853 STR(PTR)"\t5b, 11b\n\t" \ 854 STR(PTR)"\t6b, 11b\n\t" \ 855 STR(PTR)"\t7b, 11b\n\t" \ 856 STR(PTR)"\t8b, 11b\n\t" \ 857 ".previous" \ 858 : "=&r" (res) \ 859 : "r" (value), "r" (addr), "i" (-EFAULT) \ 860 : "memory"); \ 861 } while(0) 862 863 #endif /* !CONFIG_CPU_HAS_LOAD_STORE_LR */ 864 #endif 865 866 #define LoadHWU(addr, value, res) _LoadHWU(addr, value, res, kernel) 867 #define LoadHWUE(addr, value, res) _LoadHWU(addr, value, res, user) 868 #define LoadWU(addr, value, res) _LoadWU(addr, value, res, kernel) 869 #define LoadWUE(addr, value, res) _LoadWU(addr, value, res, user) 870 #define LoadHW(addr, value, res) _LoadHW(addr, value, res, kernel) 871 #define LoadHWE(addr, value, res) _LoadHW(addr, value, res, user) 872 #define LoadW(addr, value, res) _LoadW(addr, value, res, kernel) 873 #define LoadWE(addr, value, res) _LoadW(addr, value, res, user) 874 #define LoadDW(addr, value, res) _LoadDW(addr, value, res) 875 876 #define StoreHW(addr, value, res) _StoreHW(addr, value, res, kernel) 877 #define StoreHWE(addr, value, res) _StoreHW(addr, value, res, user) 878 #define StoreW(addr, value, res) _StoreW(addr, value, res, kernel) 879 #define StoreWE(addr, value, res) _StoreW(addr, value, res, user) 880 #define StoreDW(addr, value, res) _StoreDW(addr, value, res) 881 882 static void emulate_load_store_insn(struct pt_regs *regs, 883 void __user *addr, unsigned int __user *pc) 884 { 885 union mips_instruction insn; 886 unsigned long value; 887 unsigned int res, preempted; 888 unsigned long origpc; 889 unsigned long orig31; 890 void __user *fault_addr = NULL; 891 #ifdef CONFIG_EVA 892 mm_segment_t seg; 893 #endif 894 union fpureg *fpr; 895 enum msa_2b_fmt df; 896 unsigned int wd; 897 origpc = (unsigned long)pc; 898 orig31 = regs->regs[31]; 899 900 perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, 0); 901 902 /* 903 * This load never faults. 904 */ 905 __get_user(insn.word, pc); 906 907 switch (insn.i_format.opcode) { 908 /* 909 * These are instructions that a compiler doesn't generate. We 910 * can assume therefore that the code is MIPS-aware and 911 * really buggy. Emulating these instructions would break the 912 * semantics anyway. 913 */ 914 case ll_op: 915 case lld_op: 916 case sc_op: 917 case scd_op: 918 919 /* 920 * For these instructions the only way to create an address 921 * error is an attempted access to kernel/supervisor address 922 * space. 923 */ 924 case ldl_op: 925 case ldr_op: 926 case lwl_op: 927 case lwr_op: 928 case sdl_op: 929 case sdr_op: 930 case swl_op: 931 case swr_op: 932 case lb_op: 933 case lbu_op: 934 case sb_op: 935 goto sigbus; 936 937 /* 938 * The remaining opcodes are the ones that are really of 939 * interest. 940 */ 941 case spec3_op: 942 if (insn.dsp_format.func == lx_op) { 943 switch (insn.dsp_format.op) { 944 case lwx_op: 945 if (!access_ok(VERIFY_READ, addr, 4)) 946 goto sigbus; 947 LoadW(addr, value, res); 948 if (res) 949 goto fault; 950 compute_return_epc(regs); 951 regs->regs[insn.dsp_format.rd] = value; 952 break; 953 case lhx_op: 954 if (!access_ok(VERIFY_READ, addr, 2)) 955 goto sigbus; 956 LoadHW(addr, value, res); 957 if (res) 958 goto fault; 959 compute_return_epc(regs); 960 regs->regs[insn.dsp_format.rd] = value; 961 break; 962 default: 963 goto sigill; 964 } 965 } 966 #ifdef CONFIG_EVA 967 else { 968 /* 969 * we can land here only from kernel accessing user 970 * memory, so we need to "switch" the address limit to 971 * user space, so that address check can work properly. 972 */ 973 seg = get_fs(); 974 set_fs(USER_DS); 975 switch (insn.spec3_format.func) { 976 case lhe_op: 977 if (!access_ok(VERIFY_READ, addr, 2)) { 978 set_fs(seg); 979 goto sigbus; 980 } 981 LoadHWE(addr, value, res); 982 if (res) { 983 set_fs(seg); 984 goto fault; 985 } 986 compute_return_epc(regs); 987 regs->regs[insn.spec3_format.rt] = value; 988 break; 989 case lwe_op: 990 if (!access_ok(VERIFY_READ, addr, 4)) { 991 set_fs(seg); 992 goto sigbus; 993 } 994 LoadWE(addr, value, res); 995 if (res) { 996 set_fs(seg); 997 goto fault; 998 } 999 compute_return_epc(regs); 1000 regs->regs[insn.spec3_format.rt] = value; 1001 break; 1002 case lhue_op: 1003 if (!access_ok(VERIFY_READ, addr, 2)) { 1004 set_fs(seg); 1005 goto sigbus; 1006 } 1007 LoadHWUE(addr, value, res); 1008 if (res) { 1009 set_fs(seg); 1010 goto fault; 1011 } 1012 compute_return_epc(regs); 1013 regs->regs[insn.spec3_format.rt] = value; 1014 break; 1015 case she_op: 1016 if (!access_ok(VERIFY_WRITE, addr, 2)) { 1017 set_fs(seg); 1018 goto sigbus; 1019 } 1020 compute_return_epc(regs); 1021 value = regs->regs[insn.spec3_format.rt]; 1022 StoreHWE(addr, value, res); 1023 if (res) { 1024 set_fs(seg); 1025 goto fault; 1026 } 1027 break; 1028 case swe_op: 1029 if (!access_ok(VERIFY_WRITE, addr, 4)) { 1030 set_fs(seg); 1031 goto sigbus; 1032 } 1033 compute_return_epc(regs); 1034 value = regs->regs[insn.spec3_format.rt]; 1035 StoreWE(addr, value, res); 1036 if (res) { 1037 set_fs(seg); 1038 goto fault; 1039 } 1040 break; 1041 default: 1042 set_fs(seg); 1043 goto sigill; 1044 } 1045 set_fs(seg); 1046 } 1047 #endif 1048 break; 1049 case lh_op: 1050 if (!access_ok(VERIFY_READ, addr, 2)) 1051 goto sigbus; 1052 1053 if (IS_ENABLED(CONFIG_EVA)) { 1054 if (uaccess_kernel()) 1055 LoadHW(addr, value, res); 1056 else 1057 LoadHWE(addr, value, res); 1058 } else { 1059 LoadHW(addr, value, res); 1060 } 1061 1062 if (res) 1063 goto fault; 1064 compute_return_epc(regs); 1065 regs->regs[insn.i_format.rt] = value; 1066 break; 1067 1068 case lw_op: 1069 if (!access_ok(VERIFY_READ, addr, 4)) 1070 goto sigbus; 1071 1072 if (IS_ENABLED(CONFIG_EVA)) { 1073 if (uaccess_kernel()) 1074 LoadW(addr, value, res); 1075 else 1076 LoadWE(addr, value, res); 1077 } else { 1078 LoadW(addr, value, res); 1079 } 1080 1081 if (res) 1082 goto fault; 1083 compute_return_epc(regs); 1084 regs->regs[insn.i_format.rt] = value; 1085 break; 1086 1087 case lhu_op: 1088 if (!access_ok(VERIFY_READ, addr, 2)) 1089 goto sigbus; 1090 1091 if (IS_ENABLED(CONFIG_EVA)) { 1092 if (uaccess_kernel()) 1093 LoadHWU(addr, value, res); 1094 else 1095 LoadHWUE(addr, value, res); 1096 } else { 1097 LoadHWU(addr, value, res); 1098 } 1099 1100 if (res) 1101 goto fault; 1102 compute_return_epc(regs); 1103 regs->regs[insn.i_format.rt] = value; 1104 break; 1105 1106 case lwu_op: 1107 #ifdef CONFIG_64BIT 1108 /* 1109 * A 32-bit kernel might be running on a 64-bit processor. But 1110 * if we're on a 32-bit processor and an i-cache incoherency 1111 * or race makes us see a 64-bit instruction here the sdl/sdr 1112 * would blow up, so for now we don't handle unaligned 64-bit 1113 * instructions on 32-bit kernels. 1114 */ 1115 if (!access_ok(VERIFY_READ, addr, 4)) 1116 goto sigbus; 1117 1118 LoadWU(addr, value, res); 1119 if (res) 1120 goto fault; 1121 compute_return_epc(regs); 1122 regs->regs[insn.i_format.rt] = value; 1123 break; 1124 #endif /* CONFIG_64BIT */ 1125 1126 /* Cannot handle 64-bit instructions in 32-bit kernel */ 1127 goto sigill; 1128 1129 case ld_op: 1130 #ifdef CONFIG_64BIT 1131 /* 1132 * A 32-bit kernel might be running on a 64-bit processor. But 1133 * if we're on a 32-bit processor and an i-cache incoherency 1134 * or race makes us see a 64-bit instruction here the sdl/sdr 1135 * would blow up, so for now we don't handle unaligned 64-bit 1136 * instructions on 32-bit kernels. 1137 */ 1138 if (!access_ok(VERIFY_READ, addr, 8)) 1139 goto sigbus; 1140 1141 LoadDW(addr, value, res); 1142 if (res) 1143 goto fault; 1144 compute_return_epc(regs); 1145 regs->regs[insn.i_format.rt] = value; 1146 break; 1147 #endif /* CONFIG_64BIT */ 1148 1149 /* Cannot handle 64-bit instructions in 32-bit kernel */ 1150 goto sigill; 1151 1152 case sh_op: 1153 if (!access_ok(VERIFY_WRITE, addr, 2)) 1154 goto sigbus; 1155 1156 compute_return_epc(regs); 1157 value = regs->regs[insn.i_format.rt]; 1158 1159 if (IS_ENABLED(CONFIG_EVA)) { 1160 if (uaccess_kernel()) 1161 StoreHW(addr, value, res); 1162 else 1163 StoreHWE(addr, value, res); 1164 } else { 1165 StoreHW(addr, value, res); 1166 } 1167 1168 if (res) 1169 goto fault; 1170 break; 1171 1172 case sw_op: 1173 if (!access_ok(VERIFY_WRITE, addr, 4)) 1174 goto sigbus; 1175 1176 compute_return_epc(regs); 1177 value = regs->regs[insn.i_format.rt]; 1178 1179 if (IS_ENABLED(CONFIG_EVA)) { 1180 if (uaccess_kernel()) 1181 StoreW(addr, value, res); 1182 else 1183 StoreWE(addr, value, res); 1184 } else { 1185 StoreW(addr, value, res); 1186 } 1187 1188 if (res) 1189 goto fault; 1190 break; 1191 1192 case sd_op: 1193 #ifdef CONFIG_64BIT 1194 /* 1195 * A 32-bit kernel might be running on a 64-bit processor. But 1196 * if we're on a 32-bit processor and an i-cache incoherency 1197 * or race makes us see a 64-bit instruction here the sdl/sdr 1198 * would blow up, so for now we don't handle unaligned 64-bit 1199 * instructions on 32-bit kernels. 1200 */ 1201 if (!access_ok(VERIFY_WRITE, addr, 8)) 1202 goto sigbus; 1203 1204 compute_return_epc(regs); 1205 value = regs->regs[insn.i_format.rt]; 1206 StoreDW(addr, value, res); 1207 if (res) 1208 goto fault; 1209 break; 1210 #endif /* CONFIG_64BIT */ 1211 1212 /* Cannot handle 64-bit instructions in 32-bit kernel */ 1213 goto sigill; 1214 1215 case lwc1_op: 1216 case ldc1_op: 1217 case swc1_op: 1218 case sdc1_op: 1219 case cop1x_op: 1220 die_if_kernel("Unaligned FP access in kernel code", regs); 1221 BUG_ON(!used_math()); 1222 1223 lose_fpu(1); /* Save FPU state for the emulator. */ 1224 res = fpu_emulator_cop1Handler(regs, ¤t->thread.fpu, 1, 1225 &fault_addr); 1226 own_fpu(1); /* Restore FPU state. */ 1227 1228 /* Signal if something went wrong. */ 1229 process_fpemu_return(res, fault_addr, 0); 1230 1231 if (res == 0) 1232 break; 1233 return; 1234 1235 case msa_op: 1236 if (!cpu_has_msa) 1237 goto sigill; 1238 1239 /* 1240 * If we've reached this point then userland should have taken 1241 * the MSA disabled exception & initialised vector context at 1242 * some point in the past. 1243 */ 1244 BUG_ON(!thread_msa_context_live()); 1245 1246 df = insn.msa_mi10_format.df; 1247 wd = insn.msa_mi10_format.wd; 1248 fpr = ¤t->thread.fpu.fpr[wd]; 1249 1250 switch (insn.msa_mi10_format.func) { 1251 case msa_ld_op: 1252 if (!access_ok(VERIFY_READ, addr, sizeof(*fpr))) 1253 goto sigbus; 1254 1255 do { 1256 /* 1257 * If we have live MSA context keep track of 1258 * whether we get preempted in order to avoid 1259 * the register context we load being clobbered 1260 * by the live context as it's saved during 1261 * preemption. If we don't have live context 1262 * then it can't be saved to clobber the value 1263 * we load. 1264 */ 1265 preempted = test_thread_flag(TIF_USEDMSA); 1266 1267 res = __copy_from_user_inatomic(fpr, addr, 1268 sizeof(*fpr)); 1269 if (res) 1270 goto fault; 1271 1272 /* 1273 * Update the hardware register if it is in use 1274 * by the task in this quantum, in order to 1275 * avoid having to save & restore the whole 1276 * vector context. 1277 */ 1278 preempt_disable(); 1279 if (test_thread_flag(TIF_USEDMSA)) { 1280 write_msa_wr(wd, fpr, df); 1281 preempted = 0; 1282 } 1283 preempt_enable(); 1284 } while (preempted); 1285 break; 1286 1287 case msa_st_op: 1288 if (!access_ok(VERIFY_WRITE, addr, sizeof(*fpr))) 1289 goto sigbus; 1290 1291 /* 1292 * Update from the hardware register if it is in use by 1293 * the task in this quantum, in order to avoid having to 1294 * save & restore the whole vector context. 1295 */ 1296 preempt_disable(); 1297 if (test_thread_flag(TIF_USEDMSA)) 1298 read_msa_wr(wd, fpr, df); 1299 preempt_enable(); 1300 1301 res = __copy_to_user_inatomic(addr, fpr, sizeof(*fpr)); 1302 if (res) 1303 goto fault; 1304 break; 1305 1306 default: 1307 goto sigbus; 1308 } 1309 1310 compute_return_epc(regs); 1311 break; 1312 1313 #ifndef CONFIG_CPU_MIPSR6 1314 /* 1315 * COP2 is available to implementor for application specific use. 1316 * It's up to applications to register a notifier chain and do 1317 * whatever they have to do, including possible sending of signals. 1318 * 1319 * This instruction has been reallocated in Release 6 1320 */ 1321 case lwc2_op: 1322 cu2_notifier_call_chain(CU2_LWC2_OP, regs); 1323 break; 1324 1325 case ldc2_op: 1326 cu2_notifier_call_chain(CU2_LDC2_OP, regs); 1327 break; 1328 1329 case swc2_op: 1330 cu2_notifier_call_chain(CU2_SWC2_OP, regs); 1331 break; 1332 1333 case sdc2_op: 1334 cu2_notifier_call_chain(CU2_SDC2_OP, regs); 1335 break; 1336 #endif 1337 default: 1338 /* 1339 * Pheeee... We encountered an yet unknown instruction or 1340 * cache coherence problem. Die sucker, die ... 1341 */ 1342 goto sigill; 1343 } 1344 1345 #ifdef CONFIG_DEBUG_FS 1346 unaligned_instructions++; 1347 #endif 1348 1349 return; 1350 1351 fault: 1352 /* roll back jump/branch */ 1353 regs->cp0_epc = origpc; 1354 regs->regs[31] = orig31; 1355 /* Did we have an exception handler installed? */ 1356 if (fixup_exception(regs)) 1357 return; 1358 1359 die_if_kernel("Unhandled kernel unaligned access", regs); 1360 force_sig(SIGSEGV, current); 1361 1362 return; 1363 1364 sigbus: 1365 die_if_kernel("Unhandled kernel unaligned access", regs); 1366 force_sig(SIGBUS, current); 1367 1368 return; 1369 1370 sigill: 1371 die_if_kernel 1372 ("Unhandled kernel unaligned access or invalid instruction", regs); 1373 force_sig(SIGILL, current); 1374 } 1375 1376 /* Recode table from 16-bit register notation to 32-bit GPR. */ 1377 const int reg16to32[] = { 16, 17, 2, 3, 4, 5, 6, 7 }; 1378 1379 /* Recode table from 16-bit STORE register notation to 32-bit GPR. */ 1380 static const int reg16to32st[] = { 0, 17, 2, 3, 4, 5, 6, 7 }; 1381 1382 static void emulate_load_store_microMIPS(struct pt_regs *regs, 1383 void __user *addr) 1384 { 1385 unsigned long value; 1386 unsigned int res; 1387 int i; 1388 unsigned int reg = 0, rvar; 1389 unsigned long orig31; 1390 u16 __user *pc16; 1391 u16 halfword; 1392 unsigned int word; 1393 unsigned long origpc, contpc; 1394 union mips_instruction insn; 1395 struct mm_decoded_insn mminsn; 1396 void __user *fault_addr = NULL; 1397 1398 origpc = regs->cp0_epc; 1399 orig31 = regs->regs[31]; 1400 1401 mminsn.micro_mips_mode = 1; 1402 1403 /* 1404 * This load never faults. 1405 */ 1406 pc16 = (unsigned short __user *)msk_isa16_mode(regs->cp0_epc); 1407 __get_user(halfword, pc16); 1408 pc16++; 1409 contpc = regs->cp0_epc + 2; 1410 word = ((unsigned int)halfword << 16); 1411 mminsn.pc_inc = 2; 1412 1413 if (!mm_insn_16bit(halfword)) { 1414 __get_user(halfword, pc16); 1415 pc16++; 1416 contpc = regs->cp0_epc + 4; 1417 mminsn.pc_inc = 4; 1418 word |= halfword; 1419 } 1420 mminsn.insn = word; 1421 1422 if (get_user(halfword, pc16)) 1423 goto fault; 1424 mminsn.next_pc_inc = 2; 1425 word = ((unsigned int)halfword << 16); 1426 1427 if (!mm_insn_16bit(halfword)) { 1428 pc16++; 1429 if (get_user(halfword, pc16)) 1430 goto fault; 1431 mminsn.next_pc_inc = 4; 1432 word |= halfword; 1433 } 1434 mminsn.next_insn = word; 1435 1436 insn = (union mips_instruction)(mminsn.insn); 1437 if (mm_isBranchInstr(regs, mminsn, &contpc)) 1438 insn = (union mips_instruction)(mminsn.next_insn); 1439 1440 /* Parse instruction to find what to do */ 1441 1442 switch (insn.mm_i_format.opcode) { 1443 1444 case mm_pool32a_op: 1445 switch (insn.mm_x_format.func) { 1446 case mm_lwxs_op: 1447 reg = insn.mm_x_format.rd; 1448 goto loadW; 1449 } 1450 1451 goto sigbus; 1452 1453 case mm_pool32b_op: 1454 switch (insn.mm_m_format.func) { 1455 case mm_lwp_func: 1456 reg = insn.mm_m_format.rd; 1457 if (reg == 31) 1458 goto sigbus; 1459 1460 if (!access_ok(VERIFY_READ, addr, 8)) 1461 goto sigbus; 1462 1463 LoadW(addr, value, res); 1464 if (res) 1465 goto fault; 1466 regs->regs[reg] = value; 1467 addr += 4; 1468 LoadW(addr, value, res); 1469 if (res) 1470 goto fault; 1471 regs->regs[reg + 1] = value; 1472 goto success; 1473 1474 case mm_swp_func: 1475 reg = insn.mm_m_format.rd; 1476 if (reg == 31) 1477 goto sigbus; 1478 1479 if (!access_ok(VERIFY_WRITE, addr, 8)) 1480 goto sigbus; 1481 1482 value = regs->regs[reg]; 1483 StoreW(addr, value, res); 1484 if (res) 1485 goto fault; 1486 addr += 4; 1487 value = regs->regs[reg + 1]; 1488 StoreW(addr, value, res); 1489 if (res) 1490 goto fault; 1491 goto success; 1492 1493 case mm_ldp_func: 1494 #ifdef CONFIG_64BIT 1495 reg = insn.mm_m_format.rd; 1496 if (reg == 31) 1497 goto sigbus; 1498 1499 if (!access_ok(VERIFY_READ, addr, 16)) 1500 goto sigbus; 1501 1502 LoadDW(addr, value, res); 1503 if (res) 1504 goto fault; 1505 regs->regs[reg] = value; 1506 addr += 8; 1507 LoadDW(addr, value, res); 1508 if (res) 1509 goto fault; 1510 regs->regs[reg + 1] = value; 1511 goto success; 1512 #endif /* CONFIG_64BIT */ 1513 1514 goto sigill; 1515 1516 case mm_sdp_func: 1517 #ifdef CONFIG_64BIT 1518 reg = insn.mm_m_format.rd; 1519 if (reg == 31) 1520 goto sigbus; 1521 1522 if (!access_ok(VERIFY_WRITE, addr, 16)) 1523 goto sigbus; 1524 1525 value = regs->regs[reg]; 1526 StoreDW(addr, value, res); 1527 if (res) 1528 goto fault; 1529 addr += 8; 1530 value = regs->regs[reg + 1]; 1531 StoreDW(addr, value, res); 1532 if (res) 1533 goto fault; 1534 goto success; 1535 #endif /* CONFIG_64BIT */ 1536 1537 goto sigill; 1538 1539 case mm_lwm32_func: 1540 reg = insn.mm_m_format.rd; 1541 rvar = reg & 0xf; 1542 if ((rvar > 9) || !reg) 1543 goto sigill; 1544 if (reg & 0x10) { 1545 if (!access_ok 1546 (VERIFY_READ, addr, 4 * (rvar + 1))) 1547 goto sigbus; 1548 } else { 1549 if (!access_ok(VERIFY_READ, addr, 4 * rvar)) 1550 goto sigbus; 1551 } 1552 if (rvar == 9) 1553 rvar = 8; 1554 for (i = 16; rvar; rvar--, i++) { 1555 LoadW(addr, value, res); 1556 if (res) 1557 goto fault; 1558 addr += 4; 1559 regs->regs[i] = value; 1560 } 1561 if ((reg & 0xf) == 9) { 1562 LoadW(addr, value, res); 1563 if (res) 1564 goto fault; 1565 addr += 4; 1566 regs->regs[30] = value; 1567 } 1568 if (reg & 0x10) { 1569 LoadW(addr, value, res); 1570 if (res) 1571 goto fault; 1572 regs->regs[31] = value; 1573 } 1574 goto success; 1575 1576 case mm_swm32_func: 1577 reg = insn.mm_m_format.rd; 1578 rvar = reg & 0xf; 1579 if ((rvar > 9) || !reg) 1580 goto sigill; 1581 if (reg & 0x10) { 1582 if (!access_ok 1583 (VERIFY_WRITE, addr, 4 * (rvar + 1))) 1584 goto sigbus; 1585 } else { 1586 if (!access_ok(VERIFY_WRITE, addr, 4 * rvar)) 1587 goto sigbus; 1588 } 1589 if (rvar == 9) 1590 rvar = 8; 1591 for (i = 16; rvar; rvar--, i++) { 1592 value = regs->regs[i]; 1593 StoreW(addr, value, res); 1594 if (res) 1595 goto fault; 1596 addr += 4; 1597 } 1598 if ((reg & 0xf) == 9) { 1599 value = regs->regs[30]; 1600 StoreW(addr, value, res); 1601 if (res) 1602 goto fault; 1603 addr += 4; 1604 } 1605 if (reg & 0x10) { 1606 value = regs->regs[31]; 1607 StoreW(addr, value, res); 1608 if (res) 1609 goto fault; 1610 } 1611 goto success; 1612 1613 case mm_ldm_func: 1614 #ifdef CONFIG_64BIT 1615 reg = insn.mm_m_format.rd; 1616 rvar = reg & 0xf; 1617 if ((rvar > 9) || !reg) 1618 goto sigill; 1619 if (reg & 0x10) { 1620 if (!access_ok 1621 (VERIFY_READ, addr, 8 * (rvar + 1))) 1622 goto sigbus; 1623 } else { 1624 if (!access_ok(VERIFY_READ, addr, 8 * rvar)) 1625 goto sigbus; 1626 } 1627 if (rvar == 9) 1628 rvar = 8; 1629 1630 for (i = 16; rvar; rvar--, i++) { 1631 LoadDW(addr, value, res); 1632 if (res) 1633 goto fault; 1634 addr += 4; 1635 regs->regs[i] = value; 1636 } 1637 if ((reg & 0xf) == 9) { 1638 LoadDW(addr, value, res); 1639 if (res) 1640 goto fault; 1641 addr += 8; 1642 regs->regs[30] = value; 1643 } 1644 if (reg & 0x10) { 1645 LoadDW(addr, value, res); 1646 if (res) 1647 goto fault; 1648 regs->regs[31] = value; 1649 } 1650 goto success; 1651 #endif /* CONFIG_64BIT */ 1652 1653 goto sigill; 1654 1655 case mm_sdm_func: 1656 #ifdef CONFIG_64BIT 1657 reg = insn.mm_m_format.rd; 1658 rvar = reg & 0xf; 1659 if ((rvar > 9) || !reg) 1660 goto sigill; 1661 if (reg & 0x10) { 1662 if (!access_ok 1663 (VERIFY_WRITE, addr, 8 * (rvar + 1))) 1664 goto sigbus; 1665 } else { 1666 if (!access_ok(VERIFY_WRITE, addr, 8 * rvar)) 1667 goto sigbus; 1668 } 1669 if (rvar == 9) 1670 rvar = 8; 1671 1672 for (i = 16; rvar; rvar--, i++) { 1673 value = regs->regs[i]; 1674 StoreDW(addr, value, res); 1675 if (res) 1676 goto fault; 1677 addr += 8; 1678 } 1679 if ((reg & 0xf) == 9) { 1680 value = regs->regs[30]; 1681 StoreDW(addr, value, res); 1682 if (res) 1683 goto fault; 1684 addr += 8; 1685 } 1686 if (reg & 0x10) { 1687 value = regs->regs[31]; 1688 StoreDW(addr, value, res); 1689 if (res) 1690 goto fault; 1691 } 1692 goto success; 1693 #endif /* CONFIG_64BIT */ 1694 1695 goto sigill; 1696 1697 /* LWC2, SWC2, LDC2, SDC2 are not serviced */ 1698 } 1699 1700 goto sigbus; 1701 1702 case mm_pool32c_op: 1703 switch (insn.mm_m_format.func) { 1704 case mm_lwu_func: 1705 reg = insn.mm_m_format.rd; 1706 goto loadWU; 1707 } 1708 1709 /* LL,SC,LLD,SCD are not serviced */ 1710 goto sigbus; 1711 1712 case mm_pool32f_op: 1713 switch (insn.mm_x_format.func) { 1714 case mm_lwxc1_func: 1715 case mm_swxc1_func: 1716 case mm_ldxc1_func: 1717 case mm_sdxc1_func: 1718 goto fpu_emul; 1719 } 1720 1721 goto sigbus; 1722 1723 case mm_ldc132_op: 1724 case mm_sdc132_op: 1725 case mm_lwc132_op: 1726 case mm_swc132_op: 1727 fpu_emul: 1728 /* roll back jump/branch */ 1729 regs->cp0_epc = origpc; 1730 regs->regs[31] = orig31; 1731 1732 die_if_kernel("Unaligned FP access in kernel code", regs); 1733 BUG_ON(!used_math()); 1734 BUG_ON(!is_fpu_owner()); 1735 1736 lose_fpu(1); /* save the FPU state for the emulator */ 1737 res = fpu_emulator_cop1Handler(regs, ¤t->thread.fpu, 1, 1738 &fault_addr); 1739 own_fpu(1); /* restore FPU state */ 1740 1741 /* If something went wrong, signal */ 1742 process_fpemu_return(res, fault_addr, 0); 1743 1744 if (res == 0) 1745 goto success; 1746 return; 1747 1748 case mm_lh32_op: 1749 reg = insn.mm_i_format.rt; 1750 goto loadHW; 1751 1752 case mm_lhu32_op: 1753 reg = insn.mm_i_format.rt; 1754 goto loadHWU; 1755 1756 case mm_lw32_op: 1757 reg = insn.mm_i_format.rt; 1758 goto loadW; 1759 1760 case mm_sh32_op: 1761 reg = insn.mm_i_format.rt; 1762 goto storeHW; 1763 1764 case mm_sw32_op: 1765 reg = insn.mm_i_format.rt; 1766 goto storeW; 1767 1768 case mm_ld32_op: 1769 reg = insn.mm_i_format.rt; 1770 goto loadDW; 1771 1772 case mm_sd32_op: 1773 reg = insn.mm_i_format.rt; 1774 goto storeDW; 1775 1776 case mm_pool16c_op: 1777 switch (insn.mm16_m_format.func) { 1778 case mm_lwm16_op: 1779 reg = insn.mm16_m_format.rlist; 1780 rvar = reg + 1; 1781 if (!access_ok(VERIFY_READ, addr, 4 * rvar)) 1782 goto sigbus; 1783 1784 for (i = 16; rvar; rvar--, i++) { 1785 LoadW(addr, value, res); 1786 if (res) 1787 goto fault; 1788 addr += 4; 1789 regs->regs[i] = value; 1790 } 1791 LoadW(addr, value, res); 1792 if (res) 1793 goto fault; 1794 regs->regs[31] = value; 1795 1796 goto success; 1797 1798 case mm_swm16_op: 1799 reg = insn.mm16_m_format.rlist; 1800 rvar = reg + 1; 1801 if (!access_ok(VERIFY_WRITE, addr, 4 * rvar)) 1802 goto sigbus; 1803 1804 for (i = 16; rvar; rvar--, i++) { 1805 value = regs->regs[i]; 1806 StoreW(addr, value, res); 1807 if (res) 1808 goto fault; 1809 addr += 4; 1810 } 1811 value = regs->regs[31]; 1812 StoreW(addr, value, res); 1813 if (res) 1814 goto fault; 1815 1816 goto success; 1817 1818 } 1819 1820 goto sigbus; 1821 1822 case mm_lhu16_op: 1823 reg = reg16to32[insn.mm16_rb_format.rt]; 1824 goto loadHWU; 1825 1826 case mm_lw16_op: 1827 reg = reg16to32[insn.mm16_rb_format.rt]; 1828 goto loadW; 1829 1830 case mm_sh16_op: 1831 reg = reg16to32st[insn.mm16_rb_format.rt]; 1832 goto storeHW; 1833 1834 case mm_sw16_op: 1835 reg = reg16to32st[insn.mm16_rb_format.rt]; 1836 goto storeW; 1837 1838 case mm_lwsp16_op: 1839 reg = insn.mm16_r5_format.rt; 1840 goto loadW; 1841 1842 case mm_swsp16_op: 1843 reg = insn.mm16_r5_format.rt; 1844 goto storeW; 1845 1846 case mm_lwgp16_op: 1847 reg = reg16to32[insn.mm16_r3_format.rt]; 1848 goto loadW; 1849 1850 default: 1851 goto sigill; 1852 } 1853 1854 loadHW: 1855 if (!access_ok(VERIFY_READ, addr, 2)) 1856 goto sigbus; 1857 1858 LoadHW(addr, value, res); 1859 if (res) 1860 goto fault; 1861 regs->regs[reg] = value; 1862 goto success; 1863 1864 loadHWU: 1865 if (!access_ok(VERIFY_READ, addr, 2)) 1866 goto sigbus; 1867 1868 LoadHWU(addr, value, res); 1869 if (res) 1870 goto fault; 1871 regs->regs[reg] = value; 1872 goto success; 1873 1874 loadW: 1875 if (!access_ok(VERIFY_READ, addr, 4)) 1876 goto sigbus; 1877 1878 LoadW(addr, value, res); 1879 if (res) 1880 goto fault; 1881 regs->regs[reg] = value; 1882 goto success; 1883 1884 loadWU: 1885 #ifdef CONFIG_64BIT 1886 /* 1887 * A 32-bit kernel might be running on a 64-bit processor. But 1888 * if we're on a 32-bit processor and an i-cache incoherency 1889 * or race makes us see a 64-bit instruction here the sdl/sdr 1890 * would blow up, so for now we don't handle unaligned 64-bit 1891 * instructions on 32-bit kernels. 1892 */ 1893 if (!access_ok(VERIFY_READ, addr, 4)) 1894 goto sigbus; 1895 1896 LoadWU(addr, value, res); 1897 if (res) 1898 goto fault; 1899 regs->regs[reg] = value; 1900 goto success; 1901 #endif /* CONFIG_64BIT */ 1902 1903 /* Cannot handle 64-bit instructions in 32-bit kernel */ 1904 goto sigill; 1905 1906 loadDW: 1907 #ifdef CONFIG_64BIT 1908 /* 1909 * A 32-bit kernel might be running on a 64-bit processor. But 1910 * if we're on a 32-bit processor and an i-cache incoherency 1911 * or race makes us see a 64-bit instruction here the sdl/sdr 1912 * would blow up, so for now we don't handle unaligned 64-bit 1913 * instructions on 32-bit kernels. 1914 */ 1915 if (!access_ok(VERIFY_READ, addr, 8)) 1916 goto sigbus; 1917 1918 LoadDW(addr, value, res); 1919 if (res) 1920 goto fault; 1921 regs->regs[reg] = value; 1922 goto success; 1923 #endif /* CONFIG_64BIT */ 1924 1925 /* Cannot handle 64-bit instructions in 32-bit kernel */ 1926 goto sigill; 1927 1928 storeHW: 1929 if (!access_ok(VERIFY_WRITE, addr, 2)) 1930 goto sigbus; 1931 1932 value = regs->regs[reg]; 1933 StoreHW(addr, value, res); 1934 if (res) 1935 goto fault; 1936 goto success; 1937 1938 storeW: 1939 if (!access_ok(VERIFY_WRITE, addr, 4)) 1940 goto sigbus; 1941 1942 value = regs->regs[reg]; 1943 StoreW(addr, value, res); 1944 if (res) 1945 goto fault; 1946 goto success; 1947 1948 storeDW: 1949 #ifdef CONFIG_64BIT 1950 /* 1951 * A 32-bit kernel might be running on a 64-bit processor. But 1952 * if we're on a 32-bit processor and an i-cache incoherency 1953 * or race makes us see a 64-bit instruction here the sdl/sdr 1954 * would blow up, so for now we don't handle unaligned 64-bit 1955 * instructions on 32-bit kernels. 1956 */ 1957 if (!access_ok(VERIFY_WRITE, addr, 8)) 1958 goto sigbus; 1959 1960 value = regs->regs[reg]; 1961 StoreDW(addr, value, res); 1962 if (res) 1963 goto fault; 1964 goto success; 1965 #endif /* CONFIG_64BIT */ 1966 1967 /* Cannot handle 64-bit instructions in 32-bit kernel */ 1968 goto sigill; 1969 1970 success: 1971 regs->cp0_epc = contpc; /* advance or branch */ 1972 1973 #ifdef CONFIG_DEBUG_FS 1974 unaligned_instructions++; 1975 #endif 1976 return; 1977 1978 fault: 1979 /* roll back jump/branch */ 1980 regs->cp0_epc = origpc; 1981 regs->regs[31] = orig31; 1982 /* Did we have an exception handler installed? */ 1983 if (fixup_exception(regs)) 1984 return; 1985 1986 die_if_kernel("Unhandled kernel unaligned access", regs); 1987 force_sig(SIGSEGV, current); 1988 1989 return; 1990 1991 sigbus: 1992 die_if_kernel("Unhandled kernel unaligned access", regs); 1993 force_sig(SIGBUS, current); 1994 1995 return; 1996 1997 sigill: 1998 die_if_kernel 1999 ("Unhandled kernel unaligned access or invalid instruction", regs); 2000 force_sig(SIGILL, current); 2001 } 2002 2003 static void emulate_load_store_MIPS16e(struct pt_regs *regs, void __user * addr) 2004 { 2005 unsigned long value; 2006 unsigned int res; 2007 int reg; 2008 unsigned long orig31; 2009 u16 __user *pc16; 2010 unsigned long origpc; 2011 union mips16e_instruction mips16inst, oldinst; 2012 unsigned int opcode; 2013 int extended = 0; 2014 2015 origpc = regs->cp0_epc; 2016 orig31 = regs->regs[31]; 2017 pc16 = (unsigned short __user *)msk_isa16_mode(origpc); 2018 /* 2019 * This load never faults. 2020 */ 2021 __get_user(mips16inst.full, pc16); 2022 oldinst = mips16inst; 2023 2024 /* skip EXTEND instruction */ 2025 if (mips16inst.ri.opcode == MIPS16e_extend_op) { 2026 extended = 1; 2027 pc16++; 2028 __get_user(mips16inst.full, pc16); 2029 } else if (delay_slot(regs)) { 2030 /* skip jump instructions */ 2031 /* JAL/JALX are 32 bits but have OPCODE in first short int */ 2032 if (mips16inst.ri.opcode == MIPS16e_jal_op) 2033 pc16++; 2034 pc16++; 2035 if (get_user(mips16inst.full, pc16)) 2036 goto sigbus; 2037 } 2038 2039 opcode = mips16inst.ri.opcode; 2040 switch (opcode) { 2041 case MIPS16e_i64_op: /* I64 or RI64 instruction */ 2042 switch (mips16inst.i64.func) { /* I64/RI64 func field check */ 2043 case MIPS16e_ldpc_func: 2044 case MIPS16e_ldsp_func: 2045 reg = reg16to32[mips16inst.ri64.ry]; 2046 goto loadDW; 2047 2048 case MIPS16e_sdsp_func: 2049 reg = reg16to32[mips16inst.ri64.ry]; 2050 goto writeDW; 2051 2052 case MIPS16e_sdrasp_func: 2053 reg = 29; /* GPRSP */ 2054 goto writeDW; 2055 } 2056 2057 goto sigbus; 2058 2059 case MIPS16e_swsp_op: 2060 reg = reg16to32[mips16inst.ri.rx]; 2061 if (extended && cpu_has_mips16e2) 2062 switch (mips16inst.ri.imm >> 5) { 2063 case 0: /* SWSP */ 2064 case 1: /* SWGP */ 2065 break; 2066 case 2: /* SHGP */ 2067 opcode = MIPS16e_sh_op; 2068 break; 2069 default: 2070 goto sigbus; 2071 } 2072 break; 2073 2074 case MIPS16e_lwpc_op: 2075 reg = reg16to32[mips16inst.ri.rx]; 2076 break; 2077 2078 case MIPS16e_lwsp_op: 2079 reg = reg16to32[mips16inst.ri.rx]; 2080 if (extended && cpu_has_mips16e2) 2081 switch (mips16inst.ri.imm >> 5) { 2082 case 0: /* LWSP */ 2083 case 1: /* LWGP */ 2084 break; 2085 case 2: /* LHGP */ 2086 opcode = MIPS16e_lh_op; 2087 break; 2088 case 4: /* LHUGP */ 2089 opcode = MIPS16e_lhu_op; 2090 break; 2091 default: 2092 goto sigbus; 2093 } 2094 break; 2095 2096 case MIPS16e_i8_op: 2097 if (mips16inst.i8.func != MIPS16e_swrasp_func) 2098 goto sigbus; 2099 reg = 29; /* GPRSP */ 2100 break; 2101 2102 default: 2103 reg = reg16to32[mips16inst.rri.ry]; 2104 break; 2105 } 2106 2107 switch (opcode) { 2108 2109 case MIPS16e_lb_op: 2110 case MIPS16e_lbu_op: 2111 case MIPS16e_sb_op: 2112 goto sigbus; 2113 2114 case MIPS16e_lh_op: 2115 if (!access_ok(VERIFY_READ, addr, 2)) 2116 goto sigbus; 2117 2118 LoadHW(addr, value, res); 2119 if (res) 2120 goto fault; 2121 MIPS16e_compute_return_epc(regs, &oldinst); 2122 regs->regs[reg] = value; 2123 break; 2124 2125 case MIPS16e_lhu_op: 2126 if (!access_ok(VERIFY_READ, addr, 2)) 2127 goto sigbus; 2128 2129 LoadHWU(addr, value, res); 2130 if (res) 2131 goto fault; 2132 MIPS16e_compute_return_epc(regs, &oldinst); 2133 regs->regs[reg] = value; 2134 break; 2135 2136 case MIPS16e_lw_op: 2137 case MIPS16e_lwpc_op: 2138 case MIPS16e_lwsp_op: 2139 if (!access_ok(VERIFY_READ, addr, 4)) 2140 goto sigbus; 2141 2142 LoadW(addr, value, res); 2143 if (res) 2144 goto fault; 2145 MIPS16e_compute_return_epc(regs, &oldinst); 2146 regs->regs[reg] = value; 2147 break; 2148 2149 case MIPS16e_lwu_op: 2150 #ifdef CONFIG_64BIT 2151 /* 2152 * A 32-bit kernel might be running on a 64-bit processor. But 2153 * if we're on a 32-bit processor and an i-cache incoherency 2154 * or race makes us see a 64-bit instruction here the sdl/sdr 2155 * would blow up, so for now we don't handle unaligned 64-bit 2156 * instructions on 32-bit kernels. 2157 */ 2158 if (!access_ok(VERIFY_READ, addr, 4)) 2159 goto sigbus; 2160 2161 LoadWU(addr, value, res); 2162 if (res) 2163 goto fault; 2164 MIPS16e_compute_return_epc(regs, &oldinst); 2165 regs->regs[reg] = value; 2166 break; 2167 #endif /* CONFIG_64BIT */ 2168 2169 /* Cannot handle 64-bit instructions in 32-bit kernel */ 2170 goto sigill; 2171 2172 case MIPS16e_ld_op: 2173 loadDW: 2174 #ifdef CONFIG_64BIT 2175 /* 2176 * A 32-bit kernel might be running on a 64-bit processor. But 2177 * if we're on a 32-bit processor and an i-cache incoherency 2178 * or race makes us see a 64-bit instruction here the sdl/sdr 2179 * would blow up, so for now we don't handle unaligned 64-bit 2180 * instructions on 32-bit kernels. 2181 */ 2182 if (!access_ok(VERIFY_READ, addr, 8)) 2183 goto sigbus; 2184 2185 LoadDW(addr, value, res); 2186 if (res) 2187 goto fault; 2188 MIPS16e_compute_return_epc(regs, &oldinst); 2189 regs->regs[reg] = value; 2190 break; 2191 #endif /* CONFIG_64BIT */ 2192 2193 /* Cannot handle 64-bit instructions in 32-bit kernel */ 2194 goto sigill; 2195 2196 case MIPS16e_sh_op: 2197 if (!access_ok(VERIFY_WRITE, addr, 2)) 2198 goto sigbus; 2199 2200 MIPS16e_compute_return_epc(regs, &oldinst); 2201 value = regs->regs[reg]; 2202 StoreHW(addr, value, res); 2203 if (res) 2204 goto fault; 2205 break; 2206 2207 case MIPS16e_sw_op: 2208 case MIPS16e_swsp_op: 2209 case MIPS16e_i8_op: /* actually - MIPS16e_swrasp_func */ 2210 if (!access_ok(VERIFY_WRITE, addr, 4)) 2211 goto sigbus; 2212 2213 MIPS16e_compute_return_epc(regs, &oldinst); 2214 value = regs->regs[reg]; 2215 StoreW(addr, value, res); 2216 if (res) 2217 goto fault; 2218 break; 2219 2220 case MIPS16e_sd_op: 2221 writeDW: 2222 #ifdef CONFIG_64BIT 2223 /* 2224 * A 32-bit kernel might be running on a 64-bit processor. But 2225 * if we're on a 32-bit processor and an i-cache incoherency 2226 * or race makes us see a 64-bit instruction here the sdl/sdr 2227 * would blow up, so for now we don't handle unaligned 64-bit 2228 * instructions on 32-bit kernels. 2229 */ 2230 if (!access_ok(VERIFY_WRITE, addr, 8)) 2231 goto sigbus; 2232 2233 MIPS16e_compute_return_epc(regs, &oldinst); 2234 value = regs->regs[reg]; 2235 StoreDW(addr, value, res); 2236 if (res) 2237 goto fault; 2238 break; 2239 #endif /* CONFIG_64BIT */ 2240 2241 /* Cannot handle 64-bit instructions in 32-bit kernel */ 2242 goto sigill; 2243 2244 default: 2245 /* 2246 * Pheeee... We encountered an yet unknown instruction or 2247 * cache coherence problem. Die sucker, die ... 2248 */ 2249 goto sigill; 2250 } 2251 2252 #ifdef CONFIG_DEBUG_FS 2253 unaligned_instructions++; 2254 #endif 2255 2256 return; 2257 2258 fault: 2259 /* roll back jump/branch */ 2260 regs->cp0_epc = origpc; 2261 regs->regs[31] = orig31; 2262 /* Did we have an exception handler installed? */ 2263 if (fixup_exception(regs)) 2264 return; 2265 2266 die_if_kernel("Unhandled kernel unaligned access", regs); 2267 force_sig(SIGSEGV, current); 2268 2269 return; 2270 2271 sigbus: 2272 die_if_kernel("Unhandled kernel unaligned access", regs); 2273 force_sig(SIGBUS, current); 2274 2275 return; 2276 2277 sigill: 2278 die_if_kernel 2279 ("Unhandled kernel unaligned access or invalid instruction", regs); 2280 force_sig(SIGILL, current); 2281 } 2282 2283 asmlinkage void do_ade(struct pt_regs *regs) 2284 { 2285 enum ctx_state prev_state; 2286 unsigned int __user *pc; 2287 mm_segment_t seg; 2288 2289 prev_state = exception_enter(); 2290 perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 2291 1, regs, regs->cp0_badvaddr); 2292 /* 2293 * Did we catch a fault trying to load an instruction? 2294 */ 2295 if (regs->cp0_badvaddr == regs->cp0_epc) 2296 goto sigbus; 2297 2298 if (user_mode(regs) && !test_thread_flag(TIF_FIXADE)) 2299 goto sigbus; 2300 if (unaligned_action == UNALIGNED_ACTION_SIGNAL) 2301 goto sigbus; 2302 2303 /* 2304 * Do branch emulation only if we didn't forward the exception. 2305 * This is all so but ugly ... 2306 */ 2307 2308 /* 2309 * Are we running in microMIPS mode? 2310 */ 2311 if (get_isa16_mode(regs->cp0_epc)) { 2312 /* 2313 * Did we catch a fault trying to load an instruction in 2314 * 16-bit mode? 2315 */ 2316 if (regs->cp0_badvaddr == msk_isa16_mode(regs->cp0_epc)) 2317 goto sigbus; 2318 if (unaligned_action == UNALIGNED_ACTION_SHOW) 2319 show_registers(regs); 2320 2321 if (cpu_has_mmips) { 2322 seg = get_fs(); 2323 if (!user_mode(regs)) 2324 set_fs(KERNEL_DS); 2325 emulate_load_store_microMIPS(regs, 2326 (void __user *)regs->cp0_badvaddr); 2327 set_fs(seg); 2328 2329 return; 2330 } 2331 2332 if (cpu_has_mips16) { 2333 seg = get_fs(); 2334 if (!user_mode(regs)) 2335 set_fs(KERNEL_DS); 2336 emulate_load_store_MIPS16e(regs, 2337 (void __user *)regs->cp0_badvaddr); 2338 set_fs(seg); 2339 2340 return; 2341 } 2342 2343 goto sigbus; 2344 } 2345 2346 if (unaligned_action == UNALIGNED_ACTION_SHOW) 2347 show_registers(regs); 2348 pc = (unsigned int __user *)exception_epc(regs); 2349 2350 seg = get_fs(); 2351 if (!user_mode(regs)) 2352 set_fs(KERNEL_DS); 2353 emulate_load_store_insn(regs, (void __user *)regs->cp0_badvaddr, pc); 2354 set_fs(seg); 2355 2356 return; 2357 2358 sigbus: 2359 die_if_kernel("Kernel unaligned instruction access", regs); 2360 force_sig(SIGBUS, current); 2361 2362 /* 2363 * XXX On return from the signal handler we should advance the epc 2364 */ 2365 exception_exit(prev_state); 2366 } 2367 2368 #ifdef CONFIG_DEBUG_FS 2369 static int __init debugfs_unaligned(void) 2370 { 2371 struct dentry *d; 2372 2373 if (!mips_debugfs_dir) 2374 return -ENODEV; 2375 d = debugfs_create_u32("unaligned_instructions", S_IRUGO, 2376 mips_debugfs_dir, &unaligned_instructions); 2377 if (!d) 2378 return -ENOMEM; 2379 d = debugfs_create_u32("unaligned_action", S_IRUGO | S_IWUSR, 2380 mips_debugfs_dir, &unaligned_action); 2381 if (!d) 2382 return -ENOMEM; 2383 return 0; 2384 } 2385 arch_initcall(debugfs_unaligned); 2386 #endif 2387