1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Architecture-specific unaligned trap handling. 4 * 5 * Copyright (C) 1999-2002, 2004 Hewlett-Packard Co 6 * Stephane Eranian <eranian@hpl.hp.com> 7 * David Mosberger-Tang <davidm@hpl.hp.com> 8 * 9 * 2002/12/09 Fix rotating register handling (off-by-1 error, missing fr-rotation). Fix 10 * get_rse_reg() to not leak kernel bits to user-level (reading an out-of-frame 11 * stacked register returns an undefined value; it does NOT trigger a 12 * "rsvd register fault"). 13 * 2001/10/11 Fix unaligned access to rotating registers in s/w pipelined loops. 14 * 2001/08/13 Correct size of extended floats (float_fsz) from 16 to 10 bytes. 15 * 2001/01/17 Add support emulation of unaligned kernel accesses. 16 */ 17 #include <linux/jiffies.h> 18 #include <linux/kernel.h> 19 #include <linux/sched/signal.h> 20 #include <linux/tty.h> 21 #include <linux/extable.h> 22 #include <linux/ratelimit.h> 23 #include <linux/uaccess.h> 24 25 #include <asm/intrinsics.h> 26 #include <asm/processor.h> 27 #include <asm/rse.h> 28 #include <asm/exception.h> 29 #include <asm/unaligned.h> 30 31 extern int die_if_kernel(char *str, struct pt_regs *regs, long err); 32 33 #undef DEBUG_UNALIGNED_TRAP 34 35 #ifdef DEBUG_UNALIGNED_TRAP 36 # define DPRINT(a...) do { printk("%s %u: ", __func__, __LINE__); printk (a); } while (0) 37 # define DDUMP(str,vp,len) dump(str, vp, len) 38 39 static void 40 dump (const char *str, void *vp, size_t len) 41 { 42 unsigned char *cp = vp; 43 int i; 44 45 printk("%s", str); 46 for (i = 0; i < len; ++i) 47 printk (" %02x", *cp++); 48 printk("\n"); 49 } 50 #else 51 # define DPRINT(a...) 52 # define DDUMP(str,vp,len) 53 #endif 54 55 #define IA64_FIRST_STACKED_GR 32 56 #define IA64_FIRST_ROTATING_FR 32 57 #define SIGN_EXT9 0xffffffffffffff00ul 58 59 /* 60 * sysctl settable hook which tells the kernel whether to honor the 61 * IA64_THREAD_UAC_NOPRINT prctl. Because this is user settable, we want 62 * to allow the super user to enable/disable this for security reasons 63 * (i.e. don't allow attacker to fill up logs with unaligned accesses). 64 */ 65 int no_unaligned_warning; 66 int unaligned_dump_stack; 67 68 /* 69 * For M-unit: 70 * 71 * opcode | m | x6 | 72 * --------|------|---------| 73 * [40-37] | [36] | [35:30] | 74 * --------|------|---------| 75 * 4 | 1 | 6 | = 11 bits 76 * -------------------------- 77 * However bits [31:30] are not directly useful to distinguish between 78 * load/store so we can use [35:32] instead, which gives the following 79 * mask ([40:32]) using 9 bits. The 'e' comes from the fact that we defer 80 * checking the m-bit until later in the load/store emulation. 81 */ 82 #define IA64_OPCODE_MASK 0x1ef 83 #define IA64_OPCODE_SHIFT 32 84 85 /* 86 * Table C-28 Integer Load/Store 87 * 88 * We ignore [35:32]= 0x6, 0x7, 0xE, 0xF 89 * 90 * ld8.fill, st8.fill MUST be aligned because the RNATs are based on 91 * the address (bits [8:3]), so we must failed. 92 */ 93 #define LD_OP 0x080 94 #define LDS_OP 0x081 95 #define LDA_OP 0x082 96 #define LDSA_OP 0x083 97 #define LDBIAS_OP 0x084 98 #define LDACQ_OP 0x085 99 /* 0x086, 0x087 are not relevant */ 100 #define LDCCLR_OP 0x088 101 #define LDCNC_OP 0x089 102 #define LDCCLRACQ_OP 0x08a 103 #define ST_OP 0x08c 104 #define STREL_OP 0x08d 105 /* 0x08e,0x8f are not relevant */ 106 107 /* 108 * Table C-29 Integer Load +Reg 109 * 110 * we use the ld->m (bit [36:36]) field to determine whether or not we have 111 * a load/store of this form. 112 */ 113 114 /* 115 * Table C-30 Integer Load/Store +Imm 116 * 117 * We ignore [35:32]= 0x6, 0x7, 0xE, 0xF 118 * 119 * ld8.fill, st8.fill must be aligned because the Nat register are based on 120 * the address, so we must fail and the program must be fixed. 121 */ 122 #define LD_IMM_OP 0x0a0 123 #define LDS_IMM_OP 0x0a1 124 #define LDA_IMM_OP 0x0a2 125 #define LDSA_IMM_OP 0x0a3 126 #define LDBIAS_IMM_OP 0x0a4 127 #define LDACQ_IMM_OP 0x0a5 128 /* 0x0a6, 0xa7 are not relevant */ 129 #define LDCCLR_IMM_OP 0x0a8 130 #define LDCNC_IMM_OP 0x0a9 131 #define LDCCLRACQ_IMM_OP 0x0aa 132 #define ST_IMM_OP 0x0ac 133 #define STREL_IMM_OP 0x0ad 134 /* 0x0ae,0xaf are not relevant */ 135 136 /* 137 * Table C-32 Floating-point Load/Store 138 */ 139 #define LDF_OP 0x0c0 140 #define LDFS_OP 0x0c1 141 #define LDFA_OP 0x0c2 142 #define LDFSA_OP 0x0c3 143 /* 0x0c6 is irrelevant */ 144 #define LDFCCLR_OP 0x0c8 145 #define LDFCNC_OP 0x0c9 146 /* 0x0cb is irrelevant */ 147 #define STF_OP 0x0cc 148 149 /* 150 * Table C-33 Floating-point Load +Reg 151 * 152 * we use the ld->m (bit [36:36]) field to determine whether or not we have 153 * a load/store of this form. 154 */ 155 156 /* 157 * Table C-34 Floating-point Load/Store +Imm 158 */ 159 #define LDF_IMM_OP 0x0e0 160 #define LDFS_IMM_OP 0x0e1 161 #define LDFA_IMM_OP 0x0e2 162 #define LDFSA_IMM_OP 0x0e3 163 /* 0x0e6 is irrelevant */ 164 #define LDFCCLR_IMM_OP 0x0e8 165 #define LDFCNC_IMM_OP 0x0e9 166 #define STF_IMM_OP 0x0ec 167 168 typedef struct { 169 unsigned long qp:6; /* [0:5] */ 170 unsigned long r1:7; /* [6:12] */ 171 unsigned long imm:7; /* [13:19] */ 172 unsigned long r3:7; /* [20:26] */ 173 unsigned long x:1; /* [27:27] */ 174 unsigned long hint:2; /* [28:29] */ 175 unsigned long x6_sz:2; /* [30:31] */ 176 unsigned long x6_op:4; /* [32:35], x6 = x6_sz|x6_op */ 177 unsigned long m:1; /* [36:36] */ 178 unsigned long op:4; /* [37:40] */ 179 unsigned long pad:23; /* [41:63] */ 180 } load_store_t; 181 182 183 typedef enum { 184 UPD_IMMEDIATE, /* ldXZ r1=[r3],imm(9) */ 185 UPD_REG /* ldXZ r1=[r3],r2 */ 186 } update_t; 187 188 /* 189 * We use tables to keep track of the offsets of registers in the saved state. 190 * This way we save having big switch/case statements. 191 * 192 * We use bit 0 to indicate switch_stack or pt_regs. 193 * The offset is simply shifted by 1 bit. 194 * A 2-byte value should be enough to hold any kind of offset 195 * 196 * In case the calling convention changes (and thus pt_regs/switch_stack) 197 * simply use RSW instead of RPT or vice-versa. 198 */ 199 200 #define RPO(x) ((size_t) &((struct pt_regs *)0)->x) 201 #define RSO(x) ((size_t) &((struct switch_stack *)0)->x) 202 203 #define RPT(x) (RPO(x) << 1) 204 #define RSW(x) (1| RSO(x)<<1) 205 206 #define GR_OFFS(x) (gr_info[x]>>1) 207 #define GR_IN_SW(x) (gr_info[x] & 0x1) 208 209 #define FR_OFFS(x) (fr_info[x]>>1) 210 #define FR_IN_SW(x) (fr_info[x] & 0x1) 211 212 static u16 gr_info[32]={ 213 0, /* r0 is read-only : WE SHOULD NEVER GET THIS */ 214 215 RPT(r1), RPT(r2), RPT(r3), 216 217 RSW(r4), RSW(r5), RSW(r6), RSW(r7), 218 219 RPT(r8), RPT(r9), RPT(r10), RPT(r11), 220 RPT(r12), RPT(r13), RPT(r14), RPT(r15), 221 222 RPT(r16), RPT(r17), RPT(r18), RPT(r19), 223 RPT(r20), RPT(r21), RPT(r22), RPT(r23), 224 RPT(r24), RPT(r25), RPT(r26), RPT(r27), 225 RPT(r28), RPT(r29), RPT(r30), RPT(r31) 226 }; 227 228 static u16 fr_info[32]={ 229 0, /* constant : WE SHOULD NEVER GET THIS */ 230 0, /* constant : WE SHOULD NEVER GET THIS */ 231 232 RSW(f2), RSW(f3), RSW(f4), RSW(f5), 233 234 RPT(f6), RPT(f7), RPT(f8), RPT(f9), 235 RPT(f10), RPT(f11), 236 237 RSW(f12), RSW(f13), RSW(f14), 238 RSW(f15), RSW(f16), RSW(f17), RSW(f18), RSW(f19), 239 RSW(f20), RSW(f21), RSW(f22), RSW(f23), RSW(f24), 240 RSW(f25), RSW(f26), RSW(f27), RSW(f28), RSW(f29), 241 RSW(f30), RSW(f31) 242 }; 243 244 /* Invalidate ALAT entry for integer register REGNO. */ 245 static void 246 invala_gr (int regno) 247 { 248 # define F(reg) case reg: ia64_invala_gr(reg); break 249 250 switch (regno) { 251 F( 0); F( 1); F( 2); F( 3); F( 4); F( 5); F( 6); F( 7); 252 F( 8); F( 9); F( 10); F( 11); F( 12); F( 13); F( 14); F( 15); 253 F( 16); F( 17); F( 18); F( 19); F( 20); F( 21); F( 22); F( 23); 254 F( 24); F( 25); F( 26); F( 27); F( 28); F( 29); F( 30); F( 31); 255 F( 32); F( 33); F( 34); F( 35); F( 36); F( 37); F( 38); F( 39); 256 F( 40); F( 41); F( 42); F( 43); F( 44); F( 45); F( 46); F( 47); 257 F( 48); F( 49); F( 50); F( 51); F( 52); F( 53); F( 54); F( 55); 258 F( 56); F( 57); F( 58); F( 59); F( 60); F( 61); F( 62); F( 63); 259 F( 64); F( 65); F( 66); F( 67); F( 68); F( 69); F( 70); F( 71); 260 F( 72); F( 73); F( 74); F( 75); F( 76); F( 77); F( 78); F( 79); 261 F( 80); F( 81); F( 82); F( 83); F( 84); F( 85); F( 86); F( 87); 262 F( 88); F( 89); F( 90); F( 91); F( 92); F( 93); F( 94); F( 95); 263 F( 96); F( 97); F( 98); F( 99); F(100); F(101); F(102); F(103); 264 F(104); F(105); F(106); F(107); F(108); F(109); F(110); F(111); 265 F(112); F(113); F(114); F(115); F(116); F(117); F(118); F(119); 266 F(120); F(121); F(122); F(123); F(124); F(125); F(126); F(127); 267 } 268 # undef F 269 } 270 271 /* Invalidate ALAT entry for floating-point register REGNO. */ 272 static void 273 invala_fr (int regno) 274 { 275 # define F(reg) case reg: ia64_invala_fr(reg); break 276 277 switch (regno) { 278 F( 0); F( 1); F( 2); F( 3); F( 4); F( 5); F( 6); F( 7); 279 F( 8); F( 9); F( 10); F( 11); F( 12); F( 13); F( 14); F( 15); 280 F( 16); F( 17); F( 18); F( 19); F( 20); F( 21); F( 22); F( 23); 281 F( 24); F( 25); F( 26); F( 27); F( 28); F( 29); F( 30); F( 31); 282 F( 32); F( 33); F( 34); F( 35); F( 36); F( 37); F( 38); F( 39); 283 F( 40); F( 41); F( 42); F( 43); F( 44); F( 45); F( 46); F( 47); 284 F( 48); F( 49); F( 50); F( 51); F( 52); F( 53); F( 54); F( 55); 285 F( 56); F( 57); F( 58); F( 59); F( 60); F( 61); F( 62); F( 63); 286 F( 64); F( 65); F( 66); F( 67); F( 68); F( 69); F( 70); F( 71); 287 F( 72); F( 73); F( 74); F( 75); F( 76); F( 77); F( 78); F( 79); 288 F( 80); F( 81); F( 82); F( 83); F( 84); F( 85); F( 86); F( 87); 289 F( 88); F( 89); F( 90); F( 91); F( 92); F( 93); F( 94); F( 95); 290 F( 96); F( 97); F( 98); F( 99); F(100); F(101); F(102); F(103); 291 F(104); F(105); F(106); F(107); F(108); F(109); F(110); F(111); 292 F(112); F(113); F(114); F(115); F(116); F(117); F(118); F(119); 293 F(120); F(121); F(122); F(123); F(124); F(125); F(126); F(127); 294 } 295 # undef F 296 } 297 298 static inline unsigned long 299 rotate_reg (unsigned long sor, unsigned long rrb, unsigned long reg) 300 { 301 reg += rrb; 302 if (reg >= sor) 303 reg -= sor; 304 return reg; 305 } 306 307 static void 308 set_rse_reg (struct pt_regs *regs, unsigned long r1, unsigned long val, int nat) 309 { 310 struct switch_stack *sw = (struct switch_stack *) regs - 1; 311 unsigned long *bsp, *bspstore, *addr, *rnat_addr, *ubs_end; 312 unsigned long *kbs = (void *) current + IA64_RBS_OFFSET; 313 unsigned long rnats, nat_mask; 314 unsigned long on_kbs; 315 long sof = (regs->cr_ifs) & 0x7f; 316 long sor = 8 * ((regs->cr_ifs >> 14) & 0xf); 317 long rrb_gr = (regs->cr_ifs >> 18) & 0x7f; 318 long ridx = r1 - 32; 319 320 if (ridx >= sof) { 321 /* this should never happen, as the "rsvd register fault" has higher priority */ 322 DPRINT("ignoring write to r%lu; only %lu registers are allocated!\n", r1, sof); 323 return; 324 } 325 326 if (ridx < sor) 327 ridx = rotate_reg(sor, rrb_gr, ridx); 328 329 DPRINT("r%lu, sw.bspstore=%lx pt.bspstore=%lx sof=%ld sol=%ld ridx=%ld\n", 330 r1, sw->ar_bspstore, regs->ar_bspstore, sof, (regs->cr_ifs >> 7) & 0x7f, ridx); 331 332 on_kbs = ia64_rse_num_regs(kbs, (unsigned long *) sw->ar_bspstore); 333 addr = ia64_rse_skip_regs((unsigned long *) sw->ar_bspstore, -sof + ridx); 334 if (addr >= kbs) { 335 /* the register is on the kernel backing store: easy... */ 336 rnat_addr = ia64_rse_rnat_addr(addr); 337 if ((unsigned long) rnat_addr >= sw->ar_bspstore) 338 rnat_addr = &sw->ar_rnat; 339 nat_mask = 1UL << ia64_rse_slot_num(addr); 340 341 *addr = val; 342 if (nat) 343 *rnat_addr |= nat_mask; 344 else 345 *rnat_addr &= ~nat_mask; 346 return; 347 } 348 349 if (!user_stack(current, regs)) { 350 DPRINT("ignoring kernel write to r%lu; register isn't on the kernel RBS!", r1); 351 return; 352 } 353 354 bspstore = (unsigned long *)regs->ar_bspstore; 355 ubs_end = ia64_rse_skip_regs(bspstore, on_kbs); 356 bsp = ia64_rse_skip_regs(ubs_end, -sof); 357 addr = ia64_rse_skip_regs(bsp, ridx); 358 359 DPRINT("ubs_end=%p bsp=%p addr=%p\n", (void *) ubs_end, (void *) bsp, (void *) addr); 360 361 ia64_poke(current, sw, (unsigned long) ubs_end, (unsigned long) addr, val); 362 363 rnat_addr = ia64_rse_rnat_addr(addr); 364 365 ia64_peek(current, sw, (unsigned long) ubs_end, (unsigned long) rnat_addr, &rnats); 366 DPRINT("rnat @%p = 0x%lx nat=%d old nat=%ld\n", 367 (void *) rnat_addr, rnats, nat, (rnats >> ia64_rse_slot_num(addr)) & 1); 368 369 nat_mask = 1UL << ia64_rse_slot_num(addr); 370 if (nat) 371 rnats |= nat_mask; 372 else 373 rnats &= ~nat_mask; 374 ia64_poke(current, sw, (unsigned long) ubs_end, (unsigned long) rnat_addr, rnats); 375 376 DPRINT("rnat changed to @%p = 0x%lx\n", (void *) rnat_addr, rnats); 377 } 378 379 380 static void 381 get_rse_reg (struct pt_regs *regs, unsigned long r1, unsigned long *val, int *nat) 382 { 383 struct switch_stack *sw = (struct switch_stack *) regs - 1; 384 unsigned long *bsp, *addr, *rnat_addr, *ubs_end, *bspstore; 385 unsigned long *kbs = (void *) current + IA64_RBS_OFFSET; 386 unsigned long rnats, nat_mask; 387 unsigned long on_kbs; 388 long sof = (regs->cr_ifs) & 0x7f; 389 long sor = 8 * ((regs->cr_ifs >> 14) & 0xf); 390 long rrb_gr = (regs->cr_ifs >> 18) & 0x7f; 391 long ridx = r1 - 32; 392 393 if (ridx >= sof) { 394 /* read of out-of-frame register returns an undefined value; 0 in our case. */ 395 DPRINT("ignoring read from r%lu; only %lu registers are allocated!\n", r1, sof); 396 goto fail; 397 } 398 399 if (ridx < sor) 400 ridx = rotate_reg(sor, rrb_gr, ridx); 401 402 DPRINT("r%lu, sw.bspstore=%lx pt.bspstore=%lx sof=%ld sol=%ld ridx=%ld\n", 403 r1, sw->ar_bspstore, regs->ar_bspstore, sof, (regs->cr_ifs >> 7) & 0x7f, ridx); 404 405 on_kbs = ia64_rse_num_regs(kbs, (unsigned long *) sw->ar_bspstore); 406 addr = ia64_rse_skip_regs((unsigned long *) sw->ar_bspstore, -sof + ridx); 407 if (addr >= kbs) { 408 /* the register is on the kernel backing store: easy... */ 409 *val = *addr; 410 if (nat) { 411 rnat_addr = ia64_rse_rnat_addr(addr); 412 if ((unsigned long) rnat_addr >= sw->ar_bspstore) 413 rnat_addr = &sw->ar_rnat; 414 nat_mask = 1UL << ia64_rse_slot_num(addr); 415 *nat = (*rnat_addr & nat_mask) != 0; 416 } 417 return; 418 } 419 420 if (!user_stack(current, regs)) { 421 DPRINT("ignoring kernel read of r%lu; register isn't on the RBS!", r1); 422 goto fail; 423 } 424 425 bspstore = (unsigned long *)regs->ar_bspstore; 426 ubs_end = ia64_rse_skip_regs(bspstore, on_kbs); 427 bsp = ia64_rse_skip_regs(ubs_end, -sof); 428 addr = ia64_rse_skip_regs(bsp, ridx); 429 430 DPRINT("ubs_end=%p bsp=%p addr=%p\n", (void *) ubs_end, (void *) bsp, (void *) addr); 431 432 ia64_peek(current, sw, (unsigned long) ubs_end, (unsigned long) addr, val); 433 434 if (nat) { 435 rnat_addr = ia64_rse_rnat_addr(addr); 436 nat_mask = 1UL << ia64_rse_slot_num(addr); 437 438 DPRINT("rnat @%p = 0x%lx\n", (void *) rnat_addr, rnats); 439 440 ia64_peek(current, sw, (unsigned long) ubs_end, (unsigned long) rnat_addr, &rnats); 441 *nat = (rnats & nat_mask) != 0; 442 } 443 return; 444 445 fail: 446 *val = 0; 447 if (nat) 448 *nat = 0; 449 return; 450 } 451 452 453 static void 454 setreg (unsigned long regnum, unsigned long val, int nat, struct pt_regs *regs) 455 { 456 struct switch_stack *sw = (struct switch_stack *) regs - 1; 457 unsigned long addr; 458 unsigned long bitmask; 459 unsigned long *unat; 460 461 /* 462 * First takes care of stacked registers 463 */ 464 if (regnum >= IA64_FIRST_STACKED_GR) { 465 set_rse_reg(regs, regnum, val, nat); 466 return; 467 } 468 469 /* 470 * Using r0 as a target raises a General Exception fault which has higher priority 471 * than the Unaligned Reference fault. 472 */ 473 474 /* 475 * Now look at registers in [0-31] range and init correct UNAT 476 */ 477 if (GR_IN_SW(regnum)) { 478 addr = (unsigned long)sw; 479 unat = &sw->ar_unat; 480 } else { 481 addr = (unsigned long)regs; 482 unat = &sw->caller_unat; 483 } 484 DPRINT("tmp_base=%lx switch_stack=%s offset=%d\n", 485 addr, unat==&sw->ar_unat ? "yes":"no", GR_OFFS(regnum)); 486 /* 487 * add offset from base of struct 488 * and do it ! 489 */ 490 addr += GR_OFFS(regnum); 491 492 *(unsigned long *)addr = val; 493 494 /* 495 * We need to clear the corresponding UNAT bit to fully emulate the load 496 * UNAT bit_pos = GR[r3]{8:3} form EAS-2.4 497 */ 498 bitmask = 1UL << (addr >> 3 & 0x3f); 499 DPRINT("*0x%lx=0x%lx NaT=%d prev_unat @%p=%lx\n", addr, val, nat, (void *) unat, *unat); 500 if (nat) { 501 *unat |= bitmask; 502 } else { 503 *unat &= ~bitmask; 504 } 505 DPRINT("*0x%lx=0x%lx NaT=%d new unat: %p=%lx\n", addr, val, nat, (void *) unat,*unat); 506 } 507 508 /* 509 * Return the (rotated) index for floating point register REGNUM (REGNUM must be in the 510 * range from 32-127, result is in the range from 0-95. 511 */ 512 static inline unsigned long 513 fph_index (struct pt_regs *regs, long regnum) 514 { 515 unsigned long rrb_fr = (regs->cr_ifs >> 25) & 0x7f; 516 return rotate_reg(96, rrb_fr, (regnum - IA64_FIRST_ROTATING_FR)); 517 } 518 519 static void 520 setfpreg (unsigned long regnum, struct ia64_fpreg *fpval, struct pt_regs *regs) 521 { 522 struct switch_stack *sw = (struct switch_stack *)regs - 1; 523 unsigned long addr; 524 525 /* 526 * From EAS-2.5: FPDisableFault has higher priority than Unaligned 527 * Fault. Thus, when we get here, we know the partition is enabled. 528 * To update f32-f127, there are three choices: 529 * 530 * (1) save f32-f127 to thread.fph and update the values there 531 * (2) use a gigantic switch statement to directly access the registers 532 * (3) generate code on the fly to update the desired register 533 * 534 * For now, we are using approach (1). 535 */ 536 if (regnum >= IA64_FIRST_ROTATING_FR) { 537 ia64_sync_fph(current); 538 current->thread.fph[fph_index(regs, regnum)] = *fpval; 539 } else { 540 /* 541 * pt_regs or switch_stack ? 542 */ 543 if (FR_IN_SW(regnum)) { 544 addr = (unsigned long)sw; 545 } else { 546 addr = (unsigned long)regs; 547 } 548 549 DPRINT("tmp_base=%lx offset=%d\n", addr, FR_OFFS(regnum)); 550 551 addr += FR_OFFS(regnum); 552 *(struct ia64_fpreg *)addr = *fpval; 553 554 /* 555 * mark the low partition as being used now 556 * 557 * It is highly unlikely that this bit is not already set, but 558 * let's do it for safety. 559 */ 560 regs->cr_ipsr |= IA64_PSR_MFL; 561 } 562 } 563 564 /* 565 * Those 2 inline functions generate the spilled versions of the constant floating point 566 * registers which can be used with stfX 567 */ 568 static inline void 569 float_spill_f0 (struct ia64_fpreg *final) 570 { 571 ia64_stf_spill(final, 0); 572 } 573 574 static inline void 575 float_spill_f1 (struct ia64_fpreg *final) 576 { 577 ia64_stf_spill(final, 1); 578 } 579 580 static void 581 getfpreg (unsigned long regnum, struct ia64_fpreg *fpval, struct pt_regs *regs) 582 { 583 struct switch_stack *sw = (struct switch_stack *) regs - 1; 584 unsigned long addr; 585 586 /* 587 * From EAS-2.5: FPDisableFault has higher priority than 588 * Unaligned Fault. Thus, when we get here, we know the partition is 589 * enabled. 590 * 591 * When regnum > 31, the register is still live and we need to force a save 592 * to current->thread.fph to get access to it. See discussion in setfpreg() 593 * for reasons and other ways of doing this. 594 */ 595 if (regnum >= IA64_FIRST_ROTATING_FR) { 596 ia64_flush_fph(current); 597 *fpval = current->thread.fph[fph_index(regs, regnum)]; 598 } else { 599 /* 600 * f0 = 0.0, f1= 1.0. Those registers are constant and are thus 601 * not saved, we must generate their spilled form on the fly 602 */ 603 switch(regnum) { 604 case 0: 605 float_spill_f0(fpval); 606 break; 607 case 1: 608 float_spill_f1(fpval); 609 break; 610 default: 611 /* 612 * pt_regs or switch_stack ? 613 */ 614 addr = FR_IN_SW(regnum) ? (unsigned long)sw 615 : (unsigned long)regs; 616 617 DPRINT("is_sw=%d tmp_base=%lx offset=0x%x\n", 618 FR_IN_SW(regnum), addr, FR_OFFS(regnum)); 619 620 addr += FR_OFFS(regnum); 621 *fpval = *(struct ia64_fpreg *)addr; 622 } 623 } 624 } 625 626 627 static void 628 getreg (unsigned long regnum, unsigned long *val, int *nat, struct pt_regs *regs) 629 { 630 struct switch_stack *sw = (struct switch_stack *) regs - 1; 631 unsigned long addr, *unat; 632 633 if (regnum >= IA64_FIRST_STACKED_GR) { 634 get_rse_reg(regs, regnum, val, nat); 635 return; 636 } 637 638 /* 639 * take care of r0 (read-only always evaluate to 0) 640 */ 641 if (regnum == 0) { 642 *val = 0; 643 if (nat) 644 *nat = 0; 645 return; 646 } 647 648 /* 649 * Now look at registers in [0-31] range and init correct UNAT 650 */ 651 if (GR_IN_SW(regnum)) { 652 addr = (unsigned long)sw; 653 unat = &sw->ar_unat; 654 } else { 655 addr = (unsigned long)regs; 656 unat = &sw->caller_unat; 657 } 658 659 DPRINT("addr_base=%lx offset=0x%x\n", addr, GR_OFFS(regnum)); 660 661 addr += GR_OFFS(regnum); 662 663 *val = *(unsigned long *)addr; 664 665 /* 666 * do it only when requested 667 */ 668 if (nat) 669 *nat = (*unat >> (addr >> 3 & 0x3f)) & 0x1UL; 670 } 671 672 static void 673 emulate_load_updates (update_t type, load_store_t ld, struct pt_regs *regs, unsigned long ifa) 674 { 675 /* 676 * IMPORTANT: 677 * Given the way we handle unaligned speculative loads, we should 678 * not get to this point in the code but we keep this sanity check, 679 * just in case. 680 */ 681 if (ld.x6_op == 1 || ld.x6_op == 3) { 682 printk(KERN_ERR "%s: register update on speculative load, error\n", __func__); 683 if (die_if_kernel("unaligned reference on speculative load with register update\n", 684 regs, 30)) 685 return; 686 } 687 688 689 /* 690 * at this point, we know that the base register to update is valid i.e., 691 * it's not r0 692 */ 693 if (type == UPD_IMMEDIATE) { 694 unsigned long imm; 695 696 /* 697 * Load +Imm: ldXZ r1=[r3],imm(9) 698 * 699 * 700 * form imm9: [13:19] contain the first 7 bits 701 */ 702 imm = ld.x << 7 | ld.imm; 703 704 /* 705 * sign extend (1+8bits) if m set 706 */ 707 if (ld.m) imm |= SIGN_EXT9; 708 709 /* 710 * ifa == r3 and we know that the NaT bit on r3 was clear so 711 * we can directly use ifa. 712 */ 713 ifa += imm; 714 715 setreg(ld.r3, ifa, 0, regs); 716 717 DPRINT("ld.x=%d ld.m=%d imm=%ld r3=0x%lx\n", ld.x, ld.m, imm, ifa); 718 719 } else if (ld.m) { 720 unsigned long r2; 721 int nat_r2; 722 723 /* 724 * Load +Reg Opcode: ldXZ r1=[r3],r2 725 * 726 * Note: that we update r3 even in the case of ldfX.a 727 * (where the load does not happen) 728 * 729 * The way the load algorithm works, we know that r3 does not 730 * have its NaT bit set (would have gotten NaT consumption 731 * before getting the unaligned fault). So we can use ifa 732 * which equals r3 at this point. 733 * 734 * IMPORTANT: 735 * The above statement holds ONLY because we know that we 736 * never reach this code when trying to do a ldX.s. 737 * If we ever make it to here on an ldfX.s then 738 */ 739 getreg(ld.imm, &r2, &nat_r2, regs); 740 741 ifa += r2; 742 743 /* 744 * propagate Nat r2 -> r3 745 */ 746 setreg(ld.r3, ifa, nat_r2, regs); 747 748 DPRINT("imm=%d r2=%ld r3=0x%lx nat_r2=%d\n",ld.imm, r2, ifa, nat_r2); 749 } 750 } 751 752 753 static int 754 emulate_load_int (unsigned long ifa, load_store_t ld, struct pt_regs *regs) 755 { 756 unsigned int len = 1 << ld.x6_sz; 757 unsigned long val = 0; 758 759 /* 760 * r0, as target, doesn't need to be checked because Illegal Instruction 761 * faults have higher priority than unaligned faults. 762 * 763 * r0 cannot be found as the base as it would never generate an 764 * unaligned reference. 765 */ 766 767 /* 768 * ldX.a we will emulate load and also invalidate the ALAT entry. 769 * See comment below for explanation on how we handle ldX.a 770 */ 771 772 if (len != 2 && len != 4 && len != 8) { 773 DPRINT("unknown size: x6=%d\n", ld.x6_sz); 774 return -1; 775 } 776 /* this assumes little-endian byte-order: */ 777 if (copy_from_user(&val, (void __user *) ifa, len)) 778 return -1; 779 setreg(ld.r1, val, 0, regs); 780 781 /* 782 * check for updates on any kind of loads 783 */ 784 if (ld.op == 0x5 || ld.m) 785 emulate_load_updates(ld.op == 0x5 ? UPD_IMMEDIATE: UPD_REG, ld, regs, ifa); 786 787 /* 788 * handling of various loads (based on EAS2.4): 789 * 790 * ldX.acq (ordered load): 791 * - acquire semantics would have been used, so force fence instead. 792 * 793 * ldX.c.clr (check load and clear): 794 * - if we get to this handler, it's because the entry was not in the ALAT. 795 * Therefore the operation reverts to a normal load 796 * 797 * ldX.c.nc (check load no clear): 798 * - same as previous one 799 * 800 * ldX.c.clr.acq (ordered check load and clear): 801 * - same as above for c.clr part. The load needs to have acquire semantics. So 802 * we use the fence semantics which is stronger and thus ensures correctness. 803 * 804 * ldX.a (advanced load): 805 * - suppose ldX.a r1=[r3]. If we get to the unaligned trap it's because the 806 * address doesn't match requested size alignment. This means that we would 807 * possibly need more than one load to get the result. 808 * 809 * The load part can be handled just like a normal load, however the difficult 810 * part is to get the right thing into the ALAT. The critical piece of information 811 * in the base address of the load & size. To do that, a ld.a must be executed, 812 * clearly any address can be pushed into the table by using ld1.a r1=[r3]. Now 813 * if we use the same target register, we will be okay for the check.a instruction. 814 * If we look at the store, basically a stX [r3]=r1 checks the ALAT for any entry 815 * which would overlap within [r3,r3+X] (the size of the load was store in the 816 * ALAT). If such an entry is found the entry is invalidated. But this is not good 817 * enough, take the following example: 818 * r3=3 819 * ld4.a r1=[r3] 820 * 821 * Could be emulated by doing: 822 * ld1.a r1=[r3],1 823 * store to temporary; 824 * ld1.a r1=[r3],1 825 * store & shift to temporary; 826 * ld1.a r1=[r3],1 827 * store & shift to temporary; 828 * ld1.a r1=[r3] 829 * store & shift to temporary; 830 * r1=temporary 831 * 832 * So in this case, you would get the right value is r1 but the wrong info in 833 * the ALAT. Notice that you could do it in reverse to finish with address 3 834 * but you would still get the size wrong. To get the size right, one needs to 835 * execute exactly the same kind of load. You could do it from a aligned 836 * temporary location, but you would get the address wrong. 837 * 838 * So no matter what, it is not possible to emulate an advanced load 839 * correctly. But is that really critical ? 840 * 841 * We will always convert ld.a into a normal load with ALAT invalidated. This 842 * will enable compiler to do optimization where certain code path after ld.a 843 * is not required to have ld.c/chk.a, e.g., code path with no intervening stores. 844 * 845 * If there is a store after the advanced load, one must either do a ld.c.* or 846 * chk.a.* to reuse the value stored in the ALAT. Both can "fail" (meaning no 847 * entry found in ALAT), and that's perfectly ok because: 848 * 849 * - ld.c.*, if the entry is not present a normal load is executed 850 * - chk.a.*, if the entry is not present, execution jumps to recovery code 851 * 852 * In either case, the load can be potentially retried in another form. 853 * 854 * ALAT must be invalidated for the register (so that chk.a or ld.c don't pick 855 * up a stale entry later). The register base update MUST also be performed. 856 */ 857 858 /* 859 * when the load has the .acq completer then 860 * use ordering fence. 861 */ 862 if (ld.x6_op == 0x5 || ld.x6_op == 0xa) 863 mb(); 864 865 /* 866 * invalidate ALAT entry in case of advanced load 867 */ 868 if (ld.x6_op == 0x2) 869 invala_gr(ld.r1); 870 871 return 0; 872 } 873 874 static int 875 emulate_store_int (unsigned long ifa, load_store_t ld, struct pt_regs *regs) 876 { 877 unsigned long r2; 878 unsigned int len = 1 << ld.x6_sz; 879 880 /* 881 * if we get to this handler, Nat bits on both r3 and r2 have already 882 * been checked. so we don't need to do it 883 * 884 * extract the value to be stored 885 */ 886 getreg(ld.imm, &r2, NULL, regs); 887 888 /* 889 * we rely on the macros in unaligned.h for now i.e., 890 * we let the compiler figure out how to read memory gracefully. 891 * 892 * We need this switch/case because the way the inline function 893 * works. The code is optimized by the compiler and looks like 894 * a single switch/case. 895 */ 896 DPRINT("st%d [%lx]=%lx\n", len, ifa, r2); 897 898 if (len != 2 && len != 4 && len != 8) { 899 DPRINT("unknown size: x6=%d\n", ld.x6_sz); 900 return -1; 901 } 902 903 /* this assumes little-endian byte-order: */ 904 if (copy_to_user((void __user *) ifa, &r2, len)) 905 return -1; 906 907 /* 908 * stX [r3]=r2,imm(9) 909 * 910 * NOTE: 911 * ld.r3 can never be r0, because r0 would not generate an 912 * unaligned access. 913 */ 914 if (ld.op == 0x5) { 915 unsigned long imm; 916 917 /* 918 * form imm9: [12:6] contain first 7bits 919 */ 920 imm = ld.x << 7 | ld.r1; 921 /* 922 * sign extend (8bits) if m set 923 */ 924 if (ld.m) imm |= SIGN_EXT9; 925 /* 926 * ifa == r3 (NaT is necessarily cleared) 927 */ 928 ifa += imm; 929 930 DPRINT("imm=%lx r3=%lx\n", imm, ifa); 931 932 setreg(ld.r3, ifa, 0, regs); 933 } 934 /* 935 * we don't have alat_invalidate_multiple() so we need 936 * to do the complete flush :-<< 937 */ 938 ia64_invala(); 939 940 /* 941 * stX.rel: use fence instead of release 942 */ 943 if (ld.x6_op == 0xd) 944 mb(); 945 946 return 0; 947 } 948 949 /* 950 * floating point operations sizes in bytes 951 */ 952 static const unsigned char float_fsz[4]={ 953 10, /* extended precision (e) */ 954 8, /* integer (8) */ 955 4, /* single precision (s) */ 956 8 /* double precision (d) */ 957 }; 958 959 static inline void 960 mem2float_extended (struct ia64_fpreg *init, struct ia64_fpreg *final) 961 { 962 ia64_ldfe(6, init); 963 ia64_stop(); 964 ia64_stf_spill(final, 6); 965 } 966 967 static inline void 968 mem2float_integer (struct ia64_fpreg *init, struct ia64_fpreg *final) 969 { 970 ia64_ldf8(6, init); 971 ia64_stop(); 972 ia64_stf_spill(final, 6); 973 } 974 975 static inline void 976 mem2float_single (struct ia64_fpreg *init, struct ia64_fpreg *final) 977 { 978 ia64_ldfs(6, init); 979 ia64_stop(); 980 ia64_stf_spill(final, 6); 981 } 982 983 static inline void 984 mem2float_double (struct ia64_fpreg *init, struct ia64_fpreg *final) 985 { 986 ia64_ldfd(6, init); 987 ia64_stop(); 988 ia64_stf_spill(final, 6); 989 } 990 991 static inline void 992 float2mem_extended (struct ia64_fpreg *init, struct ia64_fpreg *final) 993 { 994 ia64_ldf_fill(6, init); 995 ia64_stop(); 996 ia64_stfe(final, 6); 997 } 998 999 static inline void 1000 float2mem_integer (struct ia64_fpreg *init, struct ia64_fpreg *final) 1001 { 1002 ia64_ldf_fill(6, init); 1003 ia64_stop(); 1004 ia64_stf8(final, 6); 1005 } 1006 1007 static inline void 1008 float2mem_single (struct ia64_fpreg *init, struct ia64_fpreg *final) 1009 { 1010 ia64_ldf_fill(6, init); 1011 ia64_stop(); 1012 ia64_stfs(final, 6); 1013 } 1014 1015 static inline void 1016 float2mem_double (struct ia64_fpreg *init, struct ia64_fpreg *final) 1017 { 1018 ia64_ldf_fill(6, init); 1019 ia64_stop(); 1020 ia64_stfd(final, 6); 1021 } 1022 1023 static int 1024 emulate_load_floatpair (unsigned long ifa, load_store_t ld, struct pt_regs *regs) 1025 { 1026 struct ia64_fpreg fpr_init[2]; 1027 struct ia64_fpreg fpr_final[2]; 1028 unsigned long len = float_fsz[ld.x6_sz]; 1029 1030 /* 1031 * fr0 & fr1 don't need to be checked because Illegal Instruction faults have 1032 * higher priority than unaligned faults. 1033 * 1034 * r0 cannot be found as the base as it would never generate an unaligned 1035 * reference. 1036 */ 1037 1038 /* 1039 * make sure we get clean buffers 1040 */ 1041 memset(&fpr_init, 0, sizeof(fpr_init)); 1042 memset(&fpr_final, 0, sizeof(fpr_final)); 1043 1044 /* 1045 * ldfpX.a: we don't try to emulate anything but we must 1046 * invalidate the ALAT entry and execute updates, if any. 1047 */ 1048 if (ld.x6_op != 0x2) { 1049 /* 1050 * This assumes little-endian byte-order. Note that there is no "ldfpe" 1051 * instruction: 1052 */ 1053 if (copy_from_user(&fpr_init[0], (void __user *) ifa, len) 1054 || copy_from_user(&fpr_init[1], (void __user *) (ifa + len), len)) 1055 return -1; 1056 1057 DPRINT("ld.r1=%d ld.imm=%d x6_sz=%d\n", ld.r1, ld.imm, ld.x6_sz); 1058 DDUMP("frp_init =", &fpr_init, 2*len); 1059 /* 1060 * XXX fixme 1061 * Could optimize inlines by using ldfpX & 2 spills 1062 */ 1063 switch( ld.x6_sz ) { 1064 case 0: 1065 mem2float_extended(&fpr_init[0], &fpr_final[0]); 1066 mem2float_extended(&fpr_init[1], &fpr_final[1]); 1067 break; 1068 case 1: 1069 mem2float_integer(&fpr_init[0], &fpr_final[0]); 1070 mem2float_integer(&fpr_init[1], &fpr_final[1]); 1071 break; 1072 case 2: 1073 mem2float_single(&fpr_init[0], &fpr_final[0]); 1074 mem2float_single(&fpr_init[1], &fpr_final[1]); 1075 break; 1076 case 3: 1077 mem2float_double(&fpr_init[0], &fpr_final[0]); 1078 mem2float_double(&fpr_init[1], &fpr_final[1]); 1079 break; 1080 } 1081 DDUMP("fpr_final =", &fpr_final, 2*len); 1082 /* 1083 * XXX fixme 1084 * 1085 * A possible optimization would be to drop fpr_final and directly 1086 * use the storage from the saved context i.e., the actual final 1087 * destination (pt_regs, switch_stack or thread structure). 1088 */ 1089 setfpreg(ld.r1, &fpr_final[0], regs); 1090 setfpreg(ld.imm, &fpr_final[1], regs); 1091 } 1092 1093 /* 1094 * Check for updates: only immediate updates are available for this 1095 * instruction. 1096 */ 1097 if (ld.m) { 1098 /* 1099 * the immediate is implicit given the ldsz of the operation: 1100 * single: 8 (2x4) and for all others it's 16 (2x8) 1101 */ 1102 ifa += len<<1; 1103 1104 /* 1105 * IMPORTANT: 1106 * the fact that we force the NaT of r3 to zero is ONLY valid 1107 * as long as we don't come here with a ldfpX.s. 1108 * For this reason we keep this sanity check 1109 */ 1110 if (ld.x6_op == 1 || ld.x6_op == 3) 1111 printk(KERN_ERR "%s: register update on speculative load pair, error\n", 1112 __func__); 1113 1114 setreg(ld.r3, ifa, 0, regs); 1115 } 1116 1117 /* 1118 * Invalidate ALAT entries, if any, for both registers. 1119 */ 1120 if (ld.x6_op == 0x2) { 1121 invala_fr(ld.r1); 1122 invala_fr(ld.imm); 1123 } 1124 return 0; 1125 } 1126 1127 1128 static int 1129 emulate_load_float (unsigned long ifa, load_store_t ld, struct pt_regs *regs) 1130 { 1131 struct ia64_fpreg fpr_init; 1132 struct ia64_fpreg fpr_final; 1133 unsigned long len = float_fsz[ld.x6_sz]; 1134 1135 /* 1136 * fr0 & fr1 don't need to be checked because Illegal Instruction 1137 * faults have higher priority than unaligned faults. 1138 * 1139 * r0 cannot be found as the base as it would never generate an 1140 * unaligned reference. 1141 */ 1142 1143 /* 1144 * make sure we get clean buffers 1145 */ 1146 memset(&fpr_init,0, sizeof(fpr_init)); 1147 memset(&fpr_final,0, sizeof(fpr_final)); 1148 1149 /* 1150 * ldfX.a we don't try to emulate anything but we must 1151 * invalidate the ALAT entry. 1152 * See comments in ldX for descriptions on how the various loads are handled. 1153 */ 1154 if (ld.x6_op != 0x2) { 1155 if (copy_from_user(&fpr_init, (void __user *) ifa, len)) 1156 return -1; 1157 1158 DPRINT("ld.r1=%d x6_sz=%d\n", ld.r1, ld.x6_sz); 1159 DDUMP("fpr_init =", &fpr_init, len); 1160 /* 1161 * we only do something for x6_op={0,8,9} 1162 */ 1163 switch( ld.x6_sz ) { 1164 case 0: 1165 mem2float_extended(&fpr_init, &fpr_final); 1166 break; 1167 case 1: 1168 mem2float_integer(&fpr_init, &fpr_final); 1169 break; 1170 case 2: 1171 mem2float_single(&fpr_init, &fpr_final); 1172 break; 1173 case 3: 1174 mem2float_double(&fpr_init, &fpr_final); 1175 break; 1176 } 1177 DDUMP("fpr_final =", &fpr_final, len); 1178 /* 1179 * XXX fixme 1180 * 1181 * A possible optimization would be to drop fpr_final and directly 1182 * use the storage from the saved context i.e., the actual final 1183 * destination (pt_regs, switch_stack or thread structure). 1184 */ 1185 setfpreg(ld.r1, &fpr_final, regs); 1186 } 1187 1188 /* 1189 * check for updates on any loads 1190 */ 1191 if (ld.op == 0x7 || ld.m) 1192 emulate_load_updates(ld.op == 0x7 ? UPD_IMMEDIATE: UPD_REG, ld, regs, ifa); 1193 1194 /* 1195 * invalidate ALAT entry in case of advanced floating point loads 1196 */ 1197 if (ld.x6_op == 0x2) 1198 invala_fr(ld.r1); 1199 1200 return 0; 1201 } 1202 1203 1204 static int 1205 emulate_store_float (unsigned long ifa, load_store_t ld, struct pt_regs *regs) 1206 { 1207 struct ia64_fpreg fpr_init; 1208 struct ia64_fpreg fpr_final; 1209 unsigned long len = float_fsz[ld.x6_sz]; 1210 1211 /* 1212 * make sure we get clean buffers 1213 */ 1214 memset(&fpr_init,0, sizeof(fpr_init)); 1215 memset(&fpr_final,0, sizeof(fpr_final)); 1216 1217 /* 1218 * if we get to this handler, Nat bits on both r3 and r2 have already 1219 * been checked. so we don't need to do it 1220 * 1221 * extract the value to be stored 1222 */ 1223 getfpreg(ld.imm, &fpr_init, regs); 1224 /* 1225 * during this step, we extract the spilled registers from the saved 1226 * context i.e., we refill. Then we store (no spill) to temporary 1227 * aligned location 1228 */ 1229 switch( ld.x6_sz ) { 1230 case 0: 1231 float2mem_extended(&fpr_init, &fpr_final); 1232 break; 1233 case 1: 1234 float2mem_integer(&fpr_init, &fpr_final); 1235 break; 1236 case 2: 1237 float2mem_single(&fpr_init, &fpr_final); 1238 break; 1239 case 3: 1240 float2mem_double(&fpr_init, &fpr_final); 1241 break; 1242 } 1243 DPRINT("ld.r1=%d x6_sz=%d\n", ld.r1, ld.x6_sz); 1244 DDUMP("fpr_init =", &fpr_init, len); 1245 DDUMP("fpr_final =", &fpr_final, len); 1246 1247 if (copy_to_user((void __user *) ifa, &fpr_final, len)) 1248 return -1; 1249 1250 /* 1251 * stfX [r3]=r2,imm(9) 1252 * 1253 * NOTE: 1254 * ld.r3 can never be r0, because r0 would not generate an 1255 * unaligned access. 1256 */ 1257 if (ld.op == 0x7) { 1258 unsigned long imm; 1259 1260 /* 1261 * form imm9: [12:6] contain first 7bits 1262 */ 1263 imm = ld.x << 7 | ld.r1; 1264 /* 1265 * sign extend (8bits) if m set 1266 */ 1267 if (ld.m) 1268 imm |= SIGN_EXT9; 1269 /* 1270 * ifa == r3 (NaT is necessarily cleared) 1271 */ 1272 ifa += imm; 1273 1274 DPRINT("imm=%lx r3=%lx\n", imm, ifa); 1275 1276 setreg(ld.r3, ifa, 0, regs); 1277 } 1278 /* 1279 * we don't have alat_invalidate_multiple() so we need 1280 * to do the complete flush :-<< 1281 */ 1282 ia64_invala(); 1283 1284 return 0; 1285 } 1286 1287 /* 1288 * Make sure we log the unaligned access, so that user/sysadmin can notice it and 1289 * eventually fix the program. However, we don't want to do that for every access so we 1290 * pace it with jiffies. 1291 */ 1292 static DEFINE_RATELIMIT_STATE(logging_rate_limit, 5 * HZ, 5); 1293 1294 void 1295 ia64_handle_unaligned (unsigned long ifa, struct pt_regs *regs) 1296 { 1297 struct ia64_psr *ipsr = ia64_psr(regs); 1298 mm_segment_t old_fs = get_fs(); 1299 unsigned long bundle[2]; 1300 unsigned long opcode; 1301 const struct exception_table_entry *eh = NULL; 1302 union { 1303 unsigned long l; 1304 load_store_t insn; 1305 } u; 1306 int ret = -1; 1307 1308 if (ia64_psr(regs)->be) { 1309 /* we don't support big-endian accesses */ 1310 if (die_if_kernel("big-endian unaligned accesses are not supported", regs, 0)) 1311 return; 1312 goto force_sigbus; 1313 } 1314 1315 /* 1316 * Treat kernel accesses for which there is an exception handler entry the same as 1317 * user-level unaligned accesses. Otherwise, a clever program could trick this 1318 * handler into reading an arbitrary kernel addresses... 1319 */ 1320 if (!user_mode(regs)) 1321 eh = search_exception_tables(regs->cr_iip + ia64_psr(regs)->ri); 1322 if (user_mode(regs) || eh) { 1323 if ((current->thread.flags & IA64_THREAD_UAC_SIGBUS) != 0) 1324 goto force_sigbus; 1325 1326 if (!no_unaligned_warning && 1327 !(current->thread.flags & IA64_THREAD_UAC_NOPRINT) && 1328 __ratelimit(&logging_rate_limit)) 1329 { 1330 char buf[200]; /* comm[] is at most 16 bytes... */ 1331 size_t len; 1332 1333 len = sprintf(buf, "%s(%d): unaligned access to 0x%016lx, " 1334 "ip=0x%016lx\n\r", current->comm, 1335 task_pid_nr(current), 1336 ifa, regs->cr_iip + ipsr->ri); 1337 /* 1338 * Don't call tty_write_message() if we're in the kernel; we might 1339 * be holding locks... 1340 */ 1341 if (user_mode(regs)) { 1342 struct tty_struct *tty = get_current_tty(); 1343 tty_write_message(tty, buf); 1344 tty_kref_put(tty); 1345 } 1346 buf[len-1] = '\0'; /* drop '\r' */ 1347 /* watch for command names containing %s */ 1348 printk(KERN_WARNING "%s", buf); 1349 } else { 1350 if (no_unaligned_warning) { 1351 printk_once(KERN_WARNING "%s(%d) encountered an " 1352 "unaligned exception which required\n" 1353 "kernel assistance, which degrades " 1354 "the performance of the application.\n" 1355 "Unaligned exception warnings have " 1356 "been disabled by the system " 1357 "administrator\n" 1358 "echo 0 > /proc/sys/kernel/ignore-" 1359 "unaligned-usertrap to re-enable\n", 1360 current->comm, task_pid_nr(current)); 1361 } 1362 } 1363 } else { 1364 if (__ratelimit(&logging_rate_limit)) { 1365 printk(KERN_WARNING "kernel unaligned access to 0x%016lx, ip=0x%016lx\n", 1366 ifa, regs->cr_iip + ipsr->ri); 1367 if (unaligned_dump_stack) 1368 dump_stack(); 1369 } 1370 set_fs(KERNEL_DS); 1371 } 1372 1373 DPRINT("iip=%lx ifa=%lx isr=%lx (ei=%d, sp=%d)\n", 1374 regs->cr_iip, ifa, regs->cr_ipsr, ipsr->ri, ipsr->it); 1375 1376 if (__copy_from_user(bundle, (void __user *) regs->cr_iip, 16)) 1377 goto failure; 1378 1379 /* 1380 * extract the instruction from the bundle given the slot number 1381 */ 1382 switch (ipsr->ri) { 1383 default: 1384 case 0: u.l = (bundle[0] >> 5); break; 1385 case 1: u.l = (bundle[0] >> 46) | (bundle[1] << 18); break; 1386 case 2: u.l = (bundle[1] >> 23); break; 1387 } 1388 opcode = (u.l >> IA64_OPCODE_SHIFT) & IA64_OPCODE_MASK; 1389 1390 DPRINT("opcode=%lx ld.qp=%d ld.r1=%d ld.imm=%d ld.r3=%d ld.x=%d ld.hint=%d " 1391 "ld.x6=0x%x ld.m=%d ld.op=%d\n", opcode, u.insn.qp, u.insn.r1, u.insn.imm, 1392 u.insn.r3, u.insn.x, u.insn.hint, u.insn.x6_sz, u.insn.m, u.insn.op); 1393 1394 /* 1395 * IMPORTANT: 1396 * Notice that the switch statement DOES not cover all possible instructions 1397 * that DO generate unaligned references. This is made on purpose because for some 1398 * instructions it DOES NOT make sense to try and emulate the access. Sometimes it 1399 * is WRONG to try and emulate. Here is a list of instruction we don't emulate i.e., 1400 * the program will get a signal and die: 1401 * 1402 * load/store: 1403 * - ldX.spill 1404 * - stX.spill 1405 * Reason: RNATs are based on addresses 1406 * - ld16 1407 * - st16 1408 * Reason: ld16 and st16 are supposed to occur in a single 1409 * memory op 1410 * 1411 * synchronization: 1412 * - cmpxchg 1413 * - fetchadd 1414 * - xchg 1415 * Reason: ATOMIC operations cannot be emulated properly using multiple 1416 * instructions. 1417 * 1418 * speculative loads: 1419 * - ldX.sZ 1420 * Reason: side effects, code must be ready to deal with failure so simpler 1421 * to let the load fail. 1422 * --------------------------------------------------------------------------------- 1423 * XXX fixme 1424 * 1425 * I would like to get rid of this switch case and do something 1426 * more elegant. 1427 */ 1428 switch (opcode) { 1429 case LDS_OP: 1430 case LDSA_OP: 1431 if (u.insn.x) 1432 /* oops, really a semaphore op (cmpxchg, etc) */ 1433 goto failure; 1434 fallthrough; 1435 case LDS_IMM_OP: 1436 case LDSA_IMM_OP: 1437 case LDFS_OP: 1438 case LDFSA_OP: 1439 case LDFS_IMM_OP: 1440 /* 1441 * The instruction will be retried with deferred exceptions turned on, and 1442 * we should get Nat bit installed 1443 * 1444 * IMPORTANT: When PSR_ED is set, the register & immediate update forms 1445 * are actually executed even though the operation failed. So we don't 1446 * need to take care of this. 1447 */ 1448 DPRINT("forcing PSR_ED\n"); 1449 regs->cr_ipsr |= IA64_PSR_ED; 1450 goto done; 1451 1452 case LD_OP: 1453 case LDA_OP: 1454 case LDBIAS_OP: 1455 case LDACQ_OP: 1456 case LDCCLR_OP: 1457 case LDCNC_OP: 1458 case LDCCLRACQ_OP: 1459 if (u.insn.x) 1460 /* oops, really a semaphore op (cmpxchg, etc) */ 1461 goto failure; 1462 fallthrough; 1463 case LD_IMM_OP: 1464 case LDA_IMM_OP: 1465 case LDBIAS_IMM_OP: 1466 case LDACQ_IMM_OP: 1467 case LDCCLR_IMM_OP: 1468 case LDCNC_IMM_OP: 1469 case LDCCLRACQ_IMM_OP: 1470 ret = emulate_load_int(ifa, u.insn, regs); 1471 break; 1472 1473 case ST_OP: 1474 case STREL_OP: 1475 if (u.insn.x) 1476 /* oops, really a semaphore op (cmpxchg, etc) */ 1477 goto failure; 1478 fallthrough; 1479 case ST_IMM_OP: 1480 case STREL_IMM_OP: 1481 ret = emulate_store_int(ifa, u.insn, regs); 1482 break; 1483 1484 case LDF_OP: 1485 case LDFA_OP: 1486 case LDFCCLR_OP: 1487 case LDFCNC_OP: 1488 if (u.insn.x) 1489 ret = emulate_load_floatpair(ifa, u.insn, regs); 1490 else 1491 ret = emulate_load_float(ifa, u.insn, regs); 1492 break; 1493 1494 case LDF_IMM_OP: 1495 case LDFA_IMM_OP: 1496 case LDFCCLR_IMM_OP: 1497 case LDFCNC_IMM_OP: 1498 ret = emulate_load_float(ifa, u.insn, regs); 1499 break; 1500 1501 case STF_OP: 1502 case STF_IMM_OP: 1503 ret = emulate_store_float(ifa, u.insn, regs); 1504 break; 1505 1506 default: 1507 goto failure; 1508 } 1509 DPRINT("ret=%d\n", ret); 1510 if (ret) 1511 goto failure; 1512 1513 if (ipsr->ri == 2) 1514 /* 1515 * given today's architecture this case is not likely to happen because a 1516 * memory access instruction (M) can never be in the last slot of a 1517 * bundle. But let's keep it for now. 1518 */ 1519 regs->cr_iip += 16; 1520 ipsr->ri = (ipsr->ri + 1) & 0x3; 1521 1522 DPRINT("ipsr->ri=%d iip=%lx\n", ipsr->ri, regs->cr_iip); 1523 done: 1524 set_fs(old_fs); /* restore original address limit */ 1525 return; 1526 1527 failure: 1528 /* something went wrong... */ 1529 if (!user_mode(regs)) { 1530 if (eh) { 1531 ia64_handle_exception(regs, eh); 1532 goto done; 1533 } 1534 if (die_if_kernel("error during unaligned kernel access\n", regs, ret)) 1535 return; 1536 /* NOT_REACHED */ 1537 } 1538 force_sigbus: 1539 force_sig_fault(SIGBUS, BUS_ADRALN, (void __user *) ifa, 1540 0, 0, 0); 1541 goto done; 1542 } 1543