1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Architecture-specific unaligned trap handling. 4 * 5 * Copyright (C) 1999-2002, 2004 Hewlett-Packard Co 6 * Stephane Eranian <eranian@hpl.hp.com> 7 * David Mosberger-Tang <davidm@hpl.hp.com> 8 * 9 * 2002/12/09 Fix rotating register handling (off-by-1 error, missing fr-rotation). Fix 10 * get_rse_reg() to not leak kernel bits to user-level (reading an out-of-frame 11 * stacked register returns an undefined value; it does NOT trigger a 12 * "rsvd register fault"). 13 * 2001/10/11 Fix unaligned access to rotating registers in s/w pipelined loops. 14 * 2001/08/13 Correct size of extended floats (float_fsz) from 16 to 10 bytes. 15 * 2001/01/17 Add support emulation of unaligned kernel accesses. 16 */ 17 #include <linux/jiffies.h> 18 #include <linux/kernel.h> 19 #include <linux/sched/signal.h> 20 #include <linux/tty.h> 21 #include <linux/extable.h> 22 #include <linux/ratelimit.h> 23 #include <linux/uaccess.h> 24 25 #include <asm/intrinsics.h> 26 #include <asm/processor.h> 27 #include <asm/rse.h> 28 #include <asm/exception.h> 29 #include <asm/unaligned.h> 30 31 extern int die_if_kernel(char *str, struct pt_regs *regs, long err); 32 33 #undef DEBUG_UNALIGNED_TRAP 34 35 #ifdef DEBUG_UNALIGNED_TRAP 36 # define DPRINT(a...) do { printk("%s %u: ", __func__, __LINE__); printk (a); } while (0) 37 # define DDUMP(str,vp,len) dump(str, vp, len) 38 39 static void 40 dump (const char *str, void *vp, size_t len) 41 { 42 unsigned char *cp = vp; 43 int i; 44 45 printk("%s", str); 46 for (i = 0; i < len; ++i) 47 printk (" %02x", *cp++); 48 printk("\n"); 49 } 50 #else 51 # define DPRINT(a...) 52 # define DDUMP(str,vp,len) 53 #endif 54 55 #define IA64_FIRST_STACKED_GR 32 56 #define IA64_FIRST_ROTATING_FR 32 57 #define SIGN_EXT9 0xffffffffffffff00ul 58 59 /* 60 * sysctl settable hook which tells the kernel whether to honor the 61 * IA64_THREAD_UAC_NOPRINT prctl. Because this is user settable, we want 62 * to allow the super user to enable/disable this for security reasons 63 * (i.e. don't allow attacker to fill up logs with unaligned accesses). 64 */ 65 int no_unaligned_warning; 66 int unaligned_dump_stack; 67 68 /* 69 * For M-unit: 70 * 71 * opcode | m | x6 | 72 * --------|------|---------| 73 * [40-37] | [36] | [35:30] | 74 * --------|------|---------| 75 * 4 | 1 | 6 | = 11 bits 76 * -------------------------- 77 * However bits [31:30] are not directly useful to distinguish between 78 * load/store so we can use [35:32] instead, which gives the following 79 * mask ([40:32]) using 9 bits. The 'e' comes from the fact that we defer 80 * checking the m-bit until later in the load/store emulation. 81 */ 82 #define IA64_OPCODE_MASK 0x1ef 83 #define IA64_OPCODE_SHIFT 32 84 85 /* 86 * Table C-28 Integer Load/Store 87 * 88 * We ignore [35:32]= 0x6, 0x7, 0xE, 0xF 89 * 90 * ld8.fill, st8.fill MUST be aligned because the RNATs are based on 91 * the address (bits [8:3]), so we must failed. 92 */ 93 #define LD_OP 0x080 94 #define LDS_OP 0x081 95 #define LDA_OP 0x082 96 #define LDSA_OP 0x083 97 #define LDBIAS_OP 0x084 98 #define LDACQ_OP 0x085 99 /* 0x086, 0x087 are not relevant */ 100 #define LDCCLR_OP 0x088 101 #define LDCNC_OP 0x089 102 #define LDCCLRACQ_OP 0x08a 103 #define ST_OP 0x08c 104 #define STREL_OP 0x08d 105 /* 0x08e,0x8f are not relevant */ 106 107 /* 108 * Table C-29 Integer Load +Reg 109 * 110 * we use the ld->m (bit [36:36]) field to determine whether or not we have 111 * a load/store of this form. 112 */ 113 114 /* 115 * Table C-30 Integer Load/Store +Imm 116 * 117 * We ignore [35:32]= 0x6, 0x7, 0xE, 0xF 118 * 119 * ld8.fill, st8.fill must be aligned because the Nat register are based on 120 * the address, so we must fail and the program must be fixed. 121 */ 122 #define LD_IMM_OP 0x0a0 123 #define LDS_IMM_OP 0x0a1 124 #define LDA_IMM_OP 0x0a2 125 #define LDSA_IMM_OP 0x0a3 126 #define LDBIAS_IMM_OP 0x0a4 127 #define LDACQ_IMM_OP 0x0a5 128 /* 0x0a6, 0xa7 are not relevant */ 129 #define LDCCLR_IMM_OP 0x0a8 130 #define LDCNC_IMM_OP 0x0a9 131 #define LDCCLRACQ_IMM_OP 0x0aa 132 #define ST_IMM_OP 0x0ac 133 #define STREL_IMM_OP 0x0ad 134 /* 0x0ae,0xaf are not relevant */ 135 136 /* 137 * Table C-32 Floating-point Load/Store 138 */ 139 #define LDF_OP 0x0c0 140 #define LDFS_OP 0x0c1 141 #define LDFA_OP 0x0c2 142 #define LDFSA_OP 0x0c3 143 /* 0x0c6 is irrelevant */ 144 #define LDFCCLR_OP 0x0c8 145 #define LDFCNC_OP 0x0c9 146 /* 0x0cb is irrelevant */ 147 #define STF_OP 0x0cc 148 149 /* 150 * Table C-33 Floating-point Load +Reg 151 * 152 * we use the ld->m (bit [36:36]) field to determine whether or not we have 153 * a load/store of this form. 154 */ 155 156 /* 157 * Table C-34 Floating-point Load/Store +Imm 158 */ 159 #define LDF_IMM_OP 0x0e0 160 #define LDFS_IMM_OP 0x0e1 161 #define LDFA_IMM_OP 0x0e2 162 #define LDFSA_IMM_OP 0x0e3 163 /* 0x0e6 is irrelevant */ 164 #define LDFCCLR_IMM_OP 0x0e8 165 #define LDFCNC_IMM_OP 0x0e9 166 #define STF_IMM_OP 0x0ec 167 168 typedef struct { 169 unsigned long qp:6; /* [0:5] */ 170 unsigned long r1:7; /* [6:12] */ 171 unsigned long imm:7; /* [13:19] */ 172 unsigned long r3:7; /* [20:26] */ 173 unsigned long x:1; /* [27:27] */ 174 unsigned long hint:2; /* [28:29] */ 175 unsigned long x6_sz:2; /* [30:31] */ 176 unsigned long x6_op:4; /* [32:35], x6 = x6_sz|x6_op */ 177 unsigned long m:1; /* [36:36] */ 178 unsigned long op:4; /* [37:40] */ 179 unsigned long pad:23; /* [41:63] */ 180 } load_store_t; 181 182 183 typedef enum { 184 UPD_IMMEDIATE, /* ldXZ r1=[r3],imm(9) */ 185 UPD_REG /* ldXZ r1=[r3],r2 */ 186 } update_t; 187 188 /* 189 * We use tables to keep track of the offsets of registers in the saved state. 190 * This way we save having big switch/case statements. 191 * 192 * We use bit 0 to indicate switch_stack or pt_regs. 193 * The offset is simply shifted by 1 bit. 194 * A 2-byte value should be enough to hold any kind of offset 195 * 196 * In case the calling convention changes (and thus pt_regs/switch_stack) 197 * simply use RSW instead of RPT or vice-versa. 198 */ 199 200 #define RPO(x) ((size_t) &((struct pt_regs *)0)->x) 201 #define RSO(x) ((size_t) &((struct switch_stack *)0)->x) 202 203 #define RPT(x) (RPO(x) << 1) 204 #define RSW(x) (1| RSO(x)<<1) 205 206 #define GR_OFFS(x) (gr_info[x]>>1) 207 #define GR_IN_SW(x) (gr_info[x] & 0x1) 208 209 #define FR_OFFS(x) (fr_info[x]>>1) 210 #define FR_IN_SW(x) (fr_info[x] & 0x1) 211 212 static u16 gr_info[32]={ 213 0, /* r0 is read-only : WE SHOULD NEVER GET THIS */ 214 215 RPT(r1), RPT(r2), RPT(r3), 216 217 RSW(r4), RSW(r5), RSW(r6), RSW(r7), 218 219 RPT(r8), RPT(r9), RPT(r10), RPT(r11), 220 RPT(r12), RPT(r13), RPT(r14), RPT(r15), 221 222 RPT(r16), RPT(r17), RPT(r18), RPT(r19), 223 RPT(r20), RPT(r21), RPT(r22), RPT(r23), 224 RPT(r24), RPT(r25), RPT(r26), RPT(r27), 225 RPT(r28), RPT(r29), RPT(r30), RPT(r31) 226 }; 227 228 static u16 fr_info[32]={ 229 0, /* constant : WE SHOULD NEVER GET THIS */ 230 0, /* constant : WE SHOULD NEVER GET THIS */ 231 232 RSW(f2), RSW(f3), RSW(f4), RSW(f5), 233 234 RPT(f6), RPT(f7), RPT(f8), RPT(f9), 235 RPT(f10), RPT(f11), 236 237 RSW(f12), RSW(f13), RSW(f14), 238 RSW(f15), RSW(f16), RSW(f17), RSW(f18), RSW(f19), 239 RSW(f20), RSW(f21), RSW(f22), RSW(f23), RSW(f24), 240 RSW(f25), RSW(f26), RSW(f27), RSW(f28), RSW(f29), 241 RSW(f30), RSW(f31) 242 }; 243 244 /* Invalidate ALAT entry for integer register REGNO. */ 245 static void 246 invala_gr (int regno) 247 { 248 # define F(reg) case reg: ia64_invala_gr(reg); break 249 250 switch (regno) { 251 F( 0); F( 1); F( 2); F( 3); F( 4); F( 5); F( 6); F( 7); 252 F( 8); F( 9); F( 10); F( 11); F( 12); F( 13); F( 14); F( 15); 253 F( 16); F( 17); F( 18); F( 19); F( 20); F( 21); F( 22); F( 23); 254 F( 24); F( 25); F( 26); F( 27); F( 28); F( 29); F( 30); F( 31); 255 F( 32); F( 33); F( 34); F( 35); F( 36); F( 37); F( 38); F( 39); 256 F( 40); F( 41); F( 42); F( 43); F( 44); F( 45); F( 46); F( 47); 257 F( 48); F( 49); F( 50); F( 51); F( 52); F( 53); F( 54); F( 55); 258 F( 56); F( 57); F( 58); F( 59); F( 60); F( 61); F( 62); F( 63); 259 F( 64); F( 65); F( 66); F( 67); F( 68); F( 69); F( 70); F( 71); 260 F( 72); F( 73); F( 74); F( 75); F( 76); F( 77); F( 78); F( 79); 261 F( 80); F( 81); F( 82); F( 83); F( 84); F( 85); F( 86); F( 87); 262 F( 88); F( 89); F( 90); F( 91); F( 92); F( 93); F( 94); F( 95); 263 F( 96); F( 97); F( 98); F( 99); F(100); F(101); F(102); F(103); 264 F(104); F(105); F(106); F(107); F(108); F(109); F(110); F(111); 265 F(112); F(113); F(114); F(115); F(116); F(117); F(118); F(119); 266 F(120); F(121); F(122); F(123); F(124); F(125); F(126); F(127); 267 } 268 # undef F 269 } 270 271 /* Invalidate ALAT entry for floating-point register REGNO. */ 272 static void 273 invala_fr (int regno) 274 { 275 # define F(reg) case reg: ia64_invala_fr(reg); break 276 277 switch (regno) { 278 F( 0); F( 1); F( 2); F( 3); F( 4); F( 5); F( 6); F( 7); 279 F( 8); F( 9); F( 10); F( 11); F( 12); F( 13); F( 14); F( 15); 280 F( 16); F( 17); F( 18); F( 19); F( 20); F( 21); F( 22); F( 23); 281 F( 24); F( 25); F( 26); F( 27); F( 28); F( 29); F( 30); F( 31); 282 F( 32); F( 33); F( 34); F( 35); F( 36); F( 37); F( 38); F( 39); 283 F( 40); F( 41); F( 42); F( 43); F( 44); F( 45); F( 46); F( 47); 284 F( 48); F( 49); F( 50); F( 51); F( 52); F( 53); F( 54); F( 55); 285 F( 56); F( 57); F( 58); F( 59); F( 60); F( 61); F( 62); F( 63); 286 F( 64); F( 65); F( 66); F( 67); F( 68); F( 69); F( 70); F( 71); 287 F( 72); F( 73); F( 74); F( 75); F( 76); F( 77); F( 78); F( 79); 288 F( 80); F( 81); F( 82); F( 83); F( 84); F( 85); F( 86); F( 87); 289 F( 88); F( 89); F( 90); F( 91); F( 92); F( 93); F( 94); F( 95); 290 F( 96); F( 97); F( 98); F( 99); F(100); F(101); F(102); F(103); 291 F(104); F(105); F(106); F(107); F(108); F(109); F(110); F(111); 292 F(112); F(113); F(114); F(115); F(116); F(117); F(118); F(119); 293 F(120); F(121); F(122); F(123); F(124); F(125); F(126); F(127); 294 } 295 # undef F 296 } 297 298 static inline unsigned long 299 rotate_reg (unsigned long sor, unsigned long rrb, unsigned long reg) 300 { 301 reg += rrb; 302 if (reg >= sor) 303 reg -= sor; 304 return reg; 305 } 306 307 static void 308 set_rse_reg (struct pt_regs *regs, unsigned long r1, unsigned long val, int nat) 309 { 310 struct switch_stack *sw = (struct switch_stack *) regs - 1; 311 unsigned long *bsp, *bspstore, *addr, *rnat_addr, *ubs_end; 312 unsigned long *kbs = (void *) current + IA64_RBS_OFFSET; 313 unsigned long rnats, nat_mask; 314 unsigned long on_kbs; 315 long sof = (regs->cr_ifs) & 0x7f; 316 long sor = 8 * ((regs->cr_ifs >> 14) & 0xf); 317 long rrb_gr = (regs->cr_ifs >> 18) & 0x7f; 318 long ridx = r1 - 32; 319 320 if (ridx >= sof) { 321 /* this should never happen, as the "rsvd register fault" has higher priority */ 322 DPRINT("ignoring write to r%lu; only %lu registers are allocated!\n", r1, sof); 323 return; 324 } 325 326 if (ridx < sor) 327 ridx = rotate_reg(sor, rrb_gr, ridx); 328 329 DPRINT("r%lu, sw.bspstore=%lx pt.bspstore=%lx sof=%ld sol=%ld ridx=%ld\n", 330 r1, sw->ar_bspstore, regs->ar_bspstore, sof, (regs->cr_ifs >> 7) & 0x7f, ridx); 331 332 on_kbs = ia64_rse_num_regs(kbs, (unsigned long *) sw->ar_bspstore); 333 addr = ia64_rse_skip_regs((unsigned long *) sw->ar_bspstore, -sof + ridx); 334 if (addr >= kbs) { 335 /* the register is on the kernel backing store: easy... */ 336 rnat_addr = ia64_rse_rnat_addr(addr); 337 if ((unsigned long) rnat_addr >= sw->ar_bspstore) 338 rnat_addr = &sw->ar_rnat; 339 nat_mask = 1UL << ia64_rse_slot_num(addr); 340 341 *addr = val; 342 if (nat) 343 *rnat_addr |= nat_mask; 344 else 345 *rnat_addr &= ~nat_mask; 346 return; 347 } 348 349 if (!user_stack(current, regs)) { 350 DPRINT("ignoring kernel write to r%lu; register isn't on the kernel RBS!", r1); 351 return; 352 } 353 354 bspstore = (unsigned long *)regs->ar_bspstore; 355 ubs_end = ia64_rse_skip_regs(bspstore, on_kbs); 356 bsp = ia64_rse_skip_regs(ubs_end, -sof); 357 addr = ia64_rse_skip_regs(bsp, ridx); 358 359 DPRINT("ubs_end=%p bsp=%p addr=%p\n", (void *) ubs_end, (void *) bsp, (void *) addr); 360 361 ia64_poke(current, sw, (unsigned long) ubs_end, (unsigned long) addr, val); 362 363 rnat_addr = ia64_rse_rnat_addr(addr); 364 365 ia64_peek(current, sw, (unsigned long) ubs_end, (unsigned long) rnat_addr, &rnats); 366 DPRINT("rnat @%p = 0x%lx nat=%d old nat=%ld\n", 367 (void *) rnat_addr, rnats, nat, (rnats >> ia64_rse_slot_num(addr)) & 1); 368 369 nat_mask = 1UL << ia64_rse_slot_num(addr); 370 if (nat) 371 rnats |= nat_mask; 372 else 373 rnats &= ~nat_mask; 374 ia64_poke(current, sw, (unsigned long) ubs_end, (unsigned long) rnat_addr, rnats); 375 376 DPRINT("rnat changed to @%p = 0x%lx\n", (void *) rnat_addr, rnats); 377 } 378 379 380 static void 381 get_rse_reg (struct pt_regs *regs, unsigned long r1, unsigned long *val, int *nat) 382 { 383 struct switch_stack *sw = (struct switch_stack *) regs - 1; 384 unsigned long *bsp, *addr, *rnat_addr, *ubs_end, *bspstore; 385 unsigned long *kbs = (void *) current + IA64_RBS_OFFSET; 386 unsigned long rnats, nat_mask; 387 unsigned long on_kbs; 388 long sof = (regs->cr_ifs) & 0x7f; 389 long sor = 8 * ((regs->cr_ifs >> 14) & 0xf); 390 long rrb_gr = (regs->cr_ifs >> 18) & 0x7f; 391 long ridx = r1 - 32; 392 393 if (ridx >= sof) { 394 /* read of out-of-frame register returns an undefined value; 0 in our case. */ 395 DPRINT("ignoring read from r%lu; only %lu registers are allocated!\n", r1, sof); 396 goto fail; 397 } 398 399 if (ridx < sor) 400 ridx = rotate_reg(sor, rrb_gr, ridx); 401 402 DPRINT("r%lu, sw.bspstore=%lx pt.bspstore=%lx sof=%ld sol=%ld ridx=%ld\n", 403 r1, sw->ar_bspstore, regs->ar_bspstore, sof, (regs->cr_ifs >> 7) & 0x7f, ridx); 404 405 on_kbs = ia64_rse_num_regs(kbs, (unsigned long *) sw->ar_bspstore); 406 addr = ia64_rse_skip_regs((unsigned long *) sw->ar_bspstore, -sof + ridx); 407 if (addr >= kbs) { 408 /* the register is on the kernel backing store: easy... */ 409 *val = *addr; 410 if (nat) { 411 rnat_addr = ia64_rse_rnat_addr(addr); 412 if ((unsigned long) rnat_addr >= sw->ar_bspstore) 413 rnat_addr = &sw->ar_rnat; 414 nat_mask = 1UL << ia64_rse_slot_num(addr); 415 *nat = (*rnat_addr & nat_mask) != 0; 416 } 417 return; 418 } 419 420 if (!user_stack(current, regs)) { 421 DPRINT("ignoring kernel read of r%lu; register isn't on the RBS!", r1); 422 goto fail; 423 } 424 425 bspstore = (unsigned long *)regs->ar_bspstore; 426 ubs_end = ia64_rse_skip_regs(bspstore, on_kbs); 427 bsp = ia64_rse_skip_regs(ubs_end, -sof); 428 addr = ia64_rse_skip_regs(bsp, ridx); 429 430 DPRINT("ubs_end=%p bsp=%p addr=%p\n", (void *) ubs_end, (void *) bsp, (void *) addr); 431 432 ia64_peek(current, sw, (unsigned long) ubs_end, (unsigned long) addr, val); 433 434 if (nat) { 435 rnat_addr = ia64_rse_rnat_addr(addr); 436 nat_mask = 1UL << ia64_rse_slot_num(addr); 437 438 DPRINT("rnat @%p = 0x%lx\n", (void *) rnat_addr, rnats); 439 440 ia64_peek(current, sw, (unsigned long) ubs_end, (unsigned long) rnat_addr, &rnats); 441 *nat = (rnats & nat_mask) != 0; 442 } 443 return; 444 445 fail: 446 *val = 0; 447 if (nat) 448 *nat = 0; 449 return; 450 } 451 452 453 static void 454 setreg (unsigned long regnum, unsigned long val, int nat, struct pt_regs *regs) 455 { 456 struct switch_stack *sw = (struct switch_stack *) regs - 1; 457 unsigned long addr; 458 unsigned long bitmask; 459 unsigned long *unat; 460 461 /* 462 * First takes care of stacked registers 463 */ 464 if (regnum >= IA64_FIRST_STACKED_GR) { 465 set_rse_reg(regs, regnum, val, nat); 466 return; 467 } 468 469 /* 470 * Using r0 as a target raises a General Exception fault which has higher priority 471 * than the Unaligned Reference fault. 472 */ 473 474 /* 475 * Now look at registers in [0-31] range and init correct UNAT 476 */ 477 if (GR_IN_SW(regnum)) { 478 addr = (unsigned long)sw; 479 unat = &sw->ar_unat; 480 } else { 481 addr = (unsigned long)regs; 482 unat = &sw->caller_unat; 483 } 484 DPRINT("tmp_base=%lx switch_stack=%s offset=%d\n", 485 addr, unat==&sw->ar_unat ? "yes":"no", GR_OFFS(regnum)); 486 /* 487 * add offset from base of struct 488 * and do it ! 489 */ 490 addr += GR_OFFS(regnum); 491 492 *(unsigned long *)addr = val; 493 494 /* 495 * We need to clear the corresponding UNAT bit to fully emulate the load 496 * UNAT bit_pos = GR[r3]{8:3} form EAS-2.4 497 */ 498 bitmask = 1UL << (addr >> 3 & 0x3f); 499 DPRINT("*0x%lx=0x%lx NaT=%d prev_unat @%p=%lx\n", addr, val, nat, (void *) unat, *unat); 500 if (nat) { 501 *unat |= bitmask; 502 } else { 503 *unat &= ~bitmask; 504 } 505 DPRINT("*0x%lx=0x%lx NaT=%d new unat: %p=%lx\n", addr, val, nat, (void *) unat,*unat); 506 } 507 508 /* 509 * Return the (rotated) index for floating point register REGNUM (REGNUM must be in the 510 * range from 32-127, result is in the range from 0-95. 511 */ 512 static inline unsigned long 513 fph_index (struct pt_regs *regs, long regnum) 514 { 515 unsigned long rrb_fr = (regs->cr_ifs >> 25) & 0x7f; 516 return rotate_reg(96, rrb_fr, (regnum - IA64_FIRST_ROTATING_FR)); 517 } 518 519 static void 520 setfpreg (unsigned long regnum, struct ia64_fpreg *fpval, struct pt_regs *regs) 521 { 522 struct switch_stack *sw = (struct switch_stack *)regs - 1; 523 unsigned long addr; 524 525 /* 526 * From EAS-2.5: FPDisableFault has higher priority than Unaligned 527 * Fault. Thus, when we get here, we know the partition is enabled. 528 * To update f32-f127, there are three choices: 529 * 530 * (1) save f32-f127 to thread.fph and update the values there 531 * (2) use a gigantic switch statement to directly access the registers 532 * (3) generate code on the fly to update the desired register 533 * 534 * For now, we are using approach (1). 535 */ 536 if (regnum >= IA64_FIRST_ROTATING_FR) { 537 ia64_sync_fph(current); 538 current->thread.fph[fph_index(regs, regnum)] = *fpval; 539 } else { 540 /* 541 * pt_regs or switch_stack ? 542 */ 543 if (FR_IN_SW(regnum)) { 544 addr = (unsigned long)sw; 545 } else { 546 addr = (unsigned long)regs; 547 } 548 549 DPRINT("tmp_base=%lx offset=%d\n", addr, FR_OFFS(regnum)); 550 551 addr += FR_OFFS(regnum); 552 *(struct ia64_fpreg *)addr = *fpval; 553 554 /* 555 * mark the low partition as being used now 556 * 557 * It is highly unlikely that this bit is not already set, but 558 * let's do it for safety. 559 */ 560 regs->cr_ipsr |= IA64_PSR_MFL; 561 } 562 } 563 564 /* 565 * Those 2 inline functions generate the spilled versions of the constant floating point 566 * registers which can be used with stfX 567 */ 568 static inline void 569 float_spill_f0 (struct ia64_fpreg *final) 570 { 571 ia64_stf_spill(final, 0); 572 } 573 574 static inline void 575 float_spill_f1 (struct ia64_fpreg *final) 576 { 577 ia64_stf_spill(final, 1); 578 } 579 580 static void 581 getfpreg (unsigned long regnum, struct ia64_fpreg *fpval, struct pt_regs *regs) 582 { 583 struct switch_stack *sw = (struct switch_stack *) regs - 1; 584 unsigned long addr; 585 586 /* 587 * From EAS-2.5: FPDisableFault has higher priority than 588 * Unaligned Fault. Thus, when we get here, we know the partition is 589 * enabled. 590 * 591 * When regnum > 31, the register is still live and we need to force a save 592 * to current->thread.fph to get access to it. See discussion in setfpreg() 593 * for reasons and other ways of doing this. 594 */ 595 if (regnum >= IA64_FIRST_ROTATING_FR) { 596 ia64_flush_fph(current); 597 *fpval = current->thread.fph[fph_index(regs, regnum)]; 598 } else { 599 /* 600 * f0 = 0.0, f1= 1.0. Those registers are constant and are thus 601 * not saved, we must generate their spilled form on the fly 602 */ 603 switch(regnum) { 604 case 0: 605 float_spill_f0(fpval); 606 break; 607 case 1: 608 float_spill_f1(fpval); 609 break; 610 default: 611 /* 612 * pt_regs or switch_stack ? 613 */ 614 addr = FR_IN_SW(regnum) ? (unsigned long)sw 615 : (unsigned long)regs; 616 617 DPRINT("is_sw=%d tmp_base=%lx offset=0x%x\n", 618 FR_IN_SW(regnum), addr, FR_OFFS(regnum)); 619 620 addr += FR_OFFS(regnum); 621 *fpval = *(struct ia64_fpreg *)addr; 622 } 623 } 624 } 625 626 627 static void 628 getreg (unsigned long regnum, unsigned long *val, int *nat, struct pt_regs *regs) 629 { 630 struct switch_stack *sw = (struct switch_stack *) regs - 1; 631 unsigned long addr, *unat; 632 633 if (regnum >= IA64_FIRST_STACKED_GR) { 634 get_rse_reg(regs, regnum, val, nat); 635 return; 636 } 637 638 /* 639 * take care of r0 (read-only always evaluate to 0) 640 */ 641 if (regnum == 0) { 642 *val = 0; 643 if (nat) 644 *nat = 0; 645 return; 646 } 647 648 /* 649 * Now look at registers in [0-31] range and init correct UNAT 650 */ 651 if (GR_IN_SW(regnum)) { 652 addr = (unsigned long)sw; 653 unat = &sw->ar_unat; 654 } else { 655 addr = (unsigned long)regs; 656 unat = &sw->caller_unat; 657 } 658 659 DPRINT("addr_base=%lx offset=0x%x\n", addr, GR_OFFS(regnum)); 660 661 addr += GR_OFFS(regnum); 662 663 *val = *(unsigned long *)addr; 664 665 /* 666 * do it only when requested 667 */ 668 if (nat) 669 *nat = (*unat >> (addr >> 3 & 0x3f)) & 0x1UL; 670 } 671 672 static void 673 emulate_load_updates (update_t type, load_store_t ld, struct pt_regs *regs, unsigned long ifa) 674 { 675 /* 676 * IMPORTANT: 677 * Given the way we handle unaligned speculative loads, we should 678 * not get to this point in the code but we keep this sanity check, 679 * just in case. 680 */ 681 if (ld.x6_op == 1 || ld.x6_op == 3) { 682 printk(KERN_ERR "%s: register update on speculative load, error\n", __func__); 683 if (die_if_kernel("unaligned reference on speculative load with register update\n", 684 regs, 30)) 685 return; 686 } 687 688 689 /* 690 * at this point, we know that the base register to update is valid i.e., 691 * it's not r0 692 */ 693 if (type == UPD_IMMEDIATE) { 694 unsigned long imm; 695 696 /* 697 * Load +Imm: ldXZ r1=[r3],imm(9) 698 * 699 * 700 * form imm9: [13:19] contain the first 7 bits 701 */ 702 imm = ld.x << 7 | ld.imm; 703 704 /* 705 * sign extend (1+8bits) if m set 706 */ 707 if (ld.m) imm |= SIGN_EXT9; 708 709 /* 710 * ifa == r3 and we know that the NaT bit on r3 was clear so 711 * we can directly use ifa. 712 */ 713 ifa += imm; 714 715 setreg(ld.r3, ifa, 0, regs); 716 717 DPRINT("ld.x=%d ld.m=%d imm=%ld r3=0x%lx\n", ld.x, ld.m, imm, ifa); 718 719 } else if (ld.m) { 720 unsigned long r2; 721 int nat_r2; 722 723 /* 724 * Load +Reg Opcode: ldXZ r1=[r3],r2 725 * 726 * Note: that we update r3 even in the case of ldfX.a 727 * (where the load does not happen) 728 * 729 * The way the load algorithm works, we know that r3 does not 730 * have its NaT bit set (would have gotten NaT consumption 731 * before getting the unaligned fault). So we can use ifa 732 * which equals r3 at this point. 733 * 734 * IMPORTANT: 735 * The above statement holds ONLY because we know that we 736 * never reach this code when trying to do a ldX.s. 737 * If we ever make it to here on an ldfX.s then 738 */ 739 getreg(ld.imm, &r2, &nat_r2, regs); 740 741 ifa += r2; 742 743 /* 744 * propagate Nat r2 -> r3 745 */ 746 setreg(ld.r3, ifa, nat_r2, regs); 747 748 DPRINT("imm=%d r2=%ld r3=0x%lx nat_r2=%d\n",ld.imm, r2, ifa, nat_r2); 749 } 750 } 751 752 static int emulate_store(unsigned long ifa, void *val, int len, bool kernel_mode) 753 { 754 if (kernel_mode) 755 return copy_to_kernel_nofault((void *)ifa, val, len); 756 757 return copy_to_user((void __user *)ifa, val, len); 758 } 759 760 static int emulate_load(void *val, unsigned long ifa, int len, bool kernel_mode) 761 { 762 if (kernel_mode) 763 return copy_from_kernel_nofault(val, (void *)ifa, len); 764 765 return copy_from_user(val, (void __user *)ifa, len); 766 } 767 768 static int 769 emulate_load_int (unsigned long ifa, load_store_t ld, struct pt_regs *regs, 770 bool kernel_mode) 771 { 772 unsigned int len = 1 << ld.x6_sz; 773 unsigned long val = 0; 774 775 /* 776 * r0, as target, doesn't need to be checked because Illegal Instruction 777 * faults have higher priority than unaligned faults. 778 * 779 * r0 cannot be found as the base as it would never generate an 780 * unaligned reference. 781 */ 782 783 /* 784 * ldX.a we will emulate load and also invalidate the ALAT entry. 785 * See comment below for explanation on how we handle ldX.a 786 */ 787 788 if (len != 2 && len != 4 && len != 8) { 789 DPRINT("unknown size: x6=%d\n", ld.x6_sz); 790 return -1; 791 } 792 /* this assumes little-endian byte-order: */ 793 if (emulate_load(&val, ifa, len, kernel_mode)) 794 return -1; 795 setreg(ld.r1, val, 0, regs); 796 797 /* 798 * check for updates on any kind of loads 799 */ 800 if (ld.op == 0x5 || ld.m) 801 emulate_load_updates(ld.op == 0x5 ? UPD_IMMEDIATE: UPD_REG, ld, regs, ifa); 802 803 /* 804 * handling of various loads (based on EAS2.4): 805 * 806 * ldX.acq (ordered load): 807 * - acquire semantics would have been used, so force fence instead. 808 * 809 * ldX.c.clr (check load and clear): 810 * - if we get to this handler, it's because the entry was not in the ALAT. 811 * Therefore the operation reverts to a normal load 812 * 813 * ldX.c.nc (check load no clear): 814 * - same as previous one 815 * 816 * ldX.c.clr.acq (ordered check load and clear): 817 * - same as above for c.clr part. The load needs to have acquire semantics. So 818 * we use the fence semantics which is stronger and thus ensures correctness. 819 * 820 * ldX.a (advanced load): 821 * - suppose ldX.a r1=[r3]. If we get to the unaligned trap it's because the 822 * address doesn't match requested size alignment. This means that we would 823 * possibly need more than one load to get the result. 824 * 825 * The load part can be handled just like a normal load, however the difficult 826 * part is to get the right thing into the ALAT. The critical piece of information 827 * in the base address of the load & size. To do that, a ld.a must be executed, 828 * clearly any address can be pushed into the table by using ld1.a r1=[r3]. Now 829 * if we use the same target register, we will be okay for the check.a instruction. 830 * If we look at the store, basically a stX [r3]=r1 checks the ALAT for any entry 831 * which would overlap within [r3,r3+X] (the size of the load was store in the 832 * ALAT). If such an entry is found the entry is invalidated. But this is not good 833 * enough, take the following example: 834 * r3=3 835 * ld4.a r1=[r3] 836 * 837 * Could be emulated by doing: 838 * ld1.a r1=[r3],1 839 * store to temporary; 840 * ld1.a r1=[r3],1 841 * store & shift to temporary; 842 * ld1.a r1=[r3],1 843 * store & shift to temporary; 844 * ld1.a r1=[r3] 845 * store & shift to temporary; 846 * r1=temporary 847 * 848 * So in this case, you would get the right value is r1 but the wrong info in 849 * the ALAT. Notice that you could do it in reverse to finish with address 3 850 * but you would still get the size wrong. To get the size right, one needs to 851 * execute exactly the same kind of load. You could do it from a aligned 852 * temporary location, but you would get the address wrong. 853 * 854 * So no matter what, it is not possible to emulate an advanced load 855 * correctly. But is that really critical ? 856 * 857 * We will always convert ld.a into a normal load with ALAT invalidated. This 858 * will enable compiler to do optimization where certain code path after ld.a 859 * is not required to have ld.c/chk.a, e.g., code path with no intervening stores. 860 * 861 * If there is a store after the advanced load, one must either do a ld.c.* or 862 * chk.a.* to reuse the value stored in the ALAT. Both can "fail" (meaning no 863 * entry found in ALAT), and that's perfectly ok because: 864 * 865 * - ld.c.*, if the entry is not present a normal load is executed 866 * - chk.a.*, if the entry is not present, execution jumps to recovery code 867 * 868 * In either case, the load can be potentially retried in another form. 869 * 870 * ALAT must be invalidated for the register (so that chk.a or ld.c don't pick 871 * up a stale entry later). The register base update MUST also be performed. 872 */ 873 874 /* 875 * when the load has the .acq completer then 876 * use ordering fence. 877 */ 878 if (ld.x6_op == 0x5 || ld.x6_op == 0xa) 879 mb(); 880 881 /* 882 * invalidate ALAT entry in case of advanced load 883 */ 884 if (ld.x6_op == 0x2) 885 invala_gr(ld.r1); 886 887 return 0; 888 } 889 890 static int 891 emulate_store_int (unsigned long ifa, load_store_t ld, struct pt_regs *regs, 892 bool kernel_mode) 893 { 894 unsigned long r2; 895 unsigned int len = 1 << ld.x6_sz; 896 897 /* 898 * if we get to this handler, Nat bits on both r3 and r2 have already 899 * been checked. so we don't need to do it 900 * 901 * extract the value to be stored 902 */ 903 getreg(ld.imm, &r2, NULL, regs); 904 905 /* 906 * we rely on the macros in unaligned.h for now i.e., 907 * we let the compiler figure out how to read memory gracefully. 908 * 909 * We need this switch/case because the way the inline function 910 * works. The code is optimized by the compiler and looks like 911 * a single switch/case. 912 */ 913 DPRINT("st%d [%lx]=%lx\n", len, ifa, r2); 914 915 if (len != 2 && len != 4 && len != 8) { 916 DPRINT("unknown size: x6=%d\n", ld.x6_sz); 917 return -1; 918 } 919 920 /* this assumes little-endian byte-order: */ 921 if (emulate_store(ifa, &r2, len, kernel_mode)) 922 return -1; 923 924 /* 925 * stX [r3]=r2,imm(9) 926 * 927 * NOTE: 928 * ld.r3 can never be r0, because r0 would not generate an 929 * unaligned access. 930 */ 931 if (ld.op == 0x5) { 932 unsigned long imm; 933 934 /* 935 * form imm9: [12:6] contain first 7bits 936 */ 937 imm = ld.x << 7 | ld.r1; 938 /* 939 * sign extend (8bits) if m set 940 */ 941 if (ld.m) imm |= SIGN_EXT9; 942 /* 943 * ifa == r3 (NaT is necessarily cleared) 944 */ 945 ifa += imm; 946 947 DPRINT("imm=%lx r3=%lx\n", imm, ifa); 948 949 setreg(ld.r3, ifa, 0, regs); 950 } 951 /* 952 * we don't have alat_invalidate_multiple() so we need 953 * to do the complete flush :-<< 954 */ 955 ia64_invala(); 956 957 /* 958 * stX.rel: use fence instead of release 959 */ 960 if (ld.x6_op == 0xd) 961 mb(); 962 963 return 0; 964 } 965 966 /* 967 * floating point operations sizes in bytes 968 */ 969 static const unsigned char float_fsz[4]={ 970 10, /* extended precision (e) */ 971 8, /* integer (8) */ 972 4, /* single precision (s) */ 973 8 /* double precision (d) */ 974 }; 975 976 static inline void 977 mem2float_extended (struct ia64_fpreg *init, struct ia64_fpreg *final) 978 { 979 ia64_ldfe(6, init); 980 ia64_stop(); 981 ia64_stf_spill(final, 6); 982 } 983 984 static inline void 985 mem2float_integer (struct ia64_fpreg *init, struct ia64_fpreg *final) 986 { 987 ia64_ldf8(6, init); 988 ia64_stop(); 989 ia64_stf_spill(final, 6); 990 } 991 992 static inline void 993 mem2float_single (struct ia64_fpreg *init, struct ia64_fpreg *final) 994 { 995 ia64_ldfs(6, init); 996 ia64_stop(); 997 ia64_stf_spill(final, 6); 998 } 999 1000 static inline void 1001 mem2float_double (struct ia64_fpreg *init, struct ia64_fpreg *final) 1002 { 1003 ia64_ldfd(6, init); 1004 ia64_stop(); 1005 ia64_stf_spill(final, 6); 1006 } 1007 1008 static inline void 1009 float2mem_extended (struct ia64_fpreg *init, struct ia64_fpreg *final) 1010 { 1011 ia64_ldf_fill(6, init); 1012 ia64_stop(); 1013 ia64_stfe(final, 6); 1014 } 1015 1016 static inline void 1017 float2mem_integer (struct ia64_fpreg *init, struct ia64_fpreg *final) 1018 { 1019 ia64_ldf_fill(6, init); 1020 ia64_stop(); 1021 ia64_stf8(final, 6); 1022 } 1023 1024 static inline void 1025 float2mem_single (struct ia64_fpreg *init, struct ia64_fpreg *final) 1026 { 1027 ia64_ldf_fill(6, init); 1028 ia64_stop(); 1029 ia64_stfs(final, 6); 1030 } 1031 1032 static inline void 1033 float2mem_double (struct ia64_fpreg *init, struct ia64_fpreg *final) 1034 { 1035 ia64_ldf_fill(6, init); 1036 ia64_stop(); 1037 ia64_stfd(final, 6); 1038 } 1039 1040 static int 1041 emulate_load_floatpair (unsigned long ifa, load_store_t ld, struct pt_regs *regs, bool kernel_mode) 1042 { 1043 struct ia64_fpreg fpr_init[2]; 1044 struct ia64_fpreg fpr_final[2]; 1045 unsigned long len = float_fsz[ld.x6_sz]; 1046 1047 /* 1048 * fr0 & fr1 don't need to be checked because Illegal Instruction faults have 1049 * higher priority than unaligned faults. 1050 * 1051 * r0 cannot be found as the base as it would never generate an unaligned 1052 * reference. 1053 */ 1054 1055 /* 1056 * make sure we get clean buffers 1057 */ 1058 memset(&fpr_init, 0, sizeof(fpr_init)); 1059 memset(&fpr_final, 0, sizeof(fpr_final)); 1060 1061 /* 1062 * ldfpX.a: we don't try to emulate anything but we must 1063 * invalidate the ALAT entry and execute updates, if any. 1064 */ 1065 if (ld.x6_op != 0x2) { 1066 /* 1067 * This assumes little-endian byte-order. Note that there is no "ldfpe" 1068 * instruction: 1069 */ 1070 if (emulate_load(&fpr_init[0], ifa, len, kernel_mode) 1071 || emulate_load(&fpr_init[1], (ifa + len), len, kernel_mode)) 1072 return -1; 1073 1074 DPRINT("ld.r1=%d ld.imm=%d x6_sz=%d\n", ld.r1, ld.imm, ld.x6_sz); 1075 DDUMP("frp_init =", &fpr_init, 2*len); 1076 /* 1077 * XXX fixme 1078 * Could optimize inlines by using ldfpX & 2 spills 1079 */ 1080 switch( ld.x6_sz ) { 1081 case 0: 1082 mem2float_extended(&fpr_init[0], &fpr_final[0]); 1083 mem2float_extended(&fpr_init[1], &fpr_final[1]); 1084 break; 1085 case 1: 1086 mem2float_integer(&fpr_init[0], &fpr_final[0]); 1087 mem2float_integer(&fpr_init[1], &fpr_final[1]); 1088 break; 1089 case 2: 1090 mem2float_single(&fpr_init[0], &fpr_final[0]); 1091 mem2float_single(&fpr_init[1], &fpr_final[1]); 1092 break; 1093 case 3: 1094 mem2float_double(&fpr_init[0], &fpr_final[0]); 1095 mem2float_double(&fpr_init[1], &fpr_final[1]); 1096 break; 1097 } 1098 DDUMP("fpr_final =", &fpr_final, 2*len); 1099 /* 1100 * XXX fixme 1101 * 1102 * A possible optimization would be to drop fpr_final and directly 1103 * use the storage from the saved context i.e., the actual final 1104 * destination (pt_regs, switch_stack or thread structure). 1105 */ 1106 setfpreg(ld.r1, &fpr_final[0], regs); 1107 setfpreg(ld.imm, &fpr_final[1], regs); 1108 } 1109 1110 /* 1111 * Check for updates: only immediate updates are available for this 1112 * instruction. 1113 */ 1114 if (ld.m) { 1115 /* 1116 * the immediate is implicit given the ldsz of the operation: 1117 * single: 8 (2x4) and for all others it's 16 (2x8) 1118 */ 1119 ifa += len<<1; 1120 1121 /* 1122 * IMPORTANT: 1123 * the fact that we force the NaT of r3 to zero is ONLY valid 1124 * as long as we don't come here with a ldfpX.s. 1125 * For this reason we keep this sanity check 1126 */ 1127 if (ld.x6_op == 1 || ld.x6_op == 3) 1128 printk(KERN_ERR "%s: register update on speculative load pair, error\n", 1129 __func__); 1130 1131 setreg(ld.r3, ifa, 0, regs); 1132 } 1133 1134 /* 1135 * Invalidate ALAT entries, if any, for both registers. 1136 */ 1137 if (ld.x6_op == 0x2) { 1138 invala_fr(ld.r1); 1139 invala_fr(ld.imm); 1140 } 1141 return 0; 1142 } 1143 1144 1145 static int 1146 emulate_load_float (unsigned long ifa, load_store_t ld, struct pt_regs *regs, 1147 bool kernel_mode) 1148 { 1149 struct ia64_fpreg fpr_init; 1150 struct ia64_fpreg fpr_final; 1151 unsigned long len = float_fsz[ld.x6_sz]; 1152 1153 /* 1154 * fr0 & fr1 don't need to be checked because Illegal Instruction 1155 * faults have higher priority than unaligned faults. 1156 * 1157 * r0 cannot be found as the base as it would never generate an 1158 * unaligned reference. 1159 */ 1160 1161 /* 1162 * make sure we get clean buffers 1163 */ 1164 memset(&fpr_init,0, sizeof(fpr_init)); 1165 memset(&fpr_final,0, sizeof(fpr_final)); 1166 1167 /* 1168 * ldfX.a we don't try to emulate anything but we must 1169 * invalidate the ALAT entry. 1170 * See comments in ldX for descriptions on how the various loads are handled. 1171 */ 1172 if (ld.x6_op != 0x2) { 1173 if (emulate_load(&fpr_init, ifa, len, kernel_mode)) 1174 return -1; 1175 1176 DPRINT("ld.r1=%d x6_sz=%d\n", ld.r1, ld.x6_sz); 1177 DDUMP("fpr_init =", &fpr_init, len); 1178 /* 1179 * we only do something for x6_op={0,8,9} 1180 */ 1181 switch( ld.x6_sz ) { 1182 case 0: 1183 mem2float_extended(&fpr_init, &fpr_final); 1184 break; 1185 case 1: 1186 mem2float_integer(&fpr_init, &fpr_final); 1187 break; 1188 case 2: 1189 mem2float_single(&fpr_init, &fpr_final); 1190 break; 1191 case 3: 1192 mem2float_double(&fpr_init, &fpr_final); 1193 break; 1194 } 1195 DDUMP("fpr_final =", &fpr_final, len); 1196 /* 1197 * XXX fixme 1198 * 1199 * A possible optimization would be to drop fpr_final and directly 1200 * use the storage from the saved context i.e., the actual final 1201 * destination (pt_regs, switch_stack or thread structure). 1202 */ 1203 setfpreg(ld.r1, &fpr_final, regs); 1204 } 1205 1206 /* 1207 * check for updates on any loads 1208 */ 1209 if (ld.op == 0x7 || ld.m) 1210 emulate_load_updates(ld.op == 0x7 ? UPD_IMMEDIATE: UPD_REG, ld, regs, ifa); 1211 1212 /* 1213 * invalidate ALAT entry in case of advanced floating point loads 1214 */ 1215 if (ld.x6_op == 0x2) 1216 invala_fr(ld.r1); 1217 1218 return 0; 1219 } 1220 1221 1222 static int 1223 emulate_store_float (unsigned long ifa, load_store_t ld, struct pt_regs *regs, 1224 bool kernel_mode) 1225 { 1226 struct ia64_fpreg fpr_init; 1227 struct ia64_fpreg fpr_final; 1228 unsigned long len = float_fsz[ld.x6_sz]; 1229 1230 /* 1231 * make sure we get clean buffers 1232 */ 1233 memset(&fpr_init,0, sizeof(fpr_init)); 1234 memset(&fpr_final,0, sizeof(fpr_final)); 1235 1236 /* 1237 * if we get to this handler, Nat bits on both r3 and r2 have already 1238 * been checked. so we don't need to do it 1239 * 1240 * extract the value to be stored 1241 */ 1242 getfpreg(ld.imm, &fpr_init, regs); 1243 /* 1244 * during this step, we extract the spilled registers from the saved 1245 * context i.e., we refill. Then we store (no spill) to temporary 1246 * aligned location 1247 */ 1248 switch( ld.x6_sz ) { 1249 case 0: 1250 float2mem_extended(&fpr_init, &fpr_final); 1251 break; 1252 case 1: 1253 float2mem_integer(&fpr_init, &fpr_final); 1254 break; 1255 case 2: 1256 float2mem_single(&fpr_init, &fpr_final); 1257 break; 1258 case 3: 1259 float2mem_double(&fpr_init, &fpr_final); 1260 break; 1261 } 1262 DPRINT("ld.r1=%d x6_sz=%d\n", ld.r1, ld.x6_sz); 1263 DDUMP("fpr_init =", &fpr_init, len); 1264 DDUMP("fpr_final =", &fpr_final, len); 1265 1266 if (emulate_store(ifa, &fpr_final, len, kernel_mode)) 1267 return -1; 1268 1269 /* 1270 * stfX [r3]=r2,imm(9) 1271 * 1272 * NOTE: 1273 * ld.r3 can never be r0, because r0 would not generate an 1274 * unaligned access. 1275 */ 1276 if (ld.op == 0x7) { 1277 unsigned long imm; 1278 1279 /* 1280 * form imm9: [12:6] contain first 7bits 1281 */ 1282 imm = ld.x << 7 | ld.r1; 1283 /* 1284 * sign extend (8bits) if m set 1285 */ 1286 if (ld.m) 1287 imm |= SIGN_EXT9; 1288 /* 1289 * ifa == r3 (NaT is necessarily cleared) 1290 */ 1291 ifa += imm; 1292 1293 DPRINT("imm=%lx r3=%lx\n", imm, ifa); 1294 1295 setreg(ld.r3, ifa, 0, regs); 1296 } 1297 /* 1298 * we don't have alat_invalidate_multiple() so we need 1299 * to do the complete flush :-<< 1300 */ 1301 ia64_invala(); 1302 1303 return 0; 1304 } 1305 1306 /* 1307 * Make sure we log the unaligned access, so that user/sysadmin can notice it and 1308 * eventually fix the program. However, we don't want to do that for every access so we 1309 * pace it with jiffies. 1310 */ 1311 static DEFINE_RATELIMIT_STATE(logging_rate_limit, 5 * HZ, 5); 1312 1313 void 1314 ia64_handle_unaligned (unsigned long ifa, struct pt_regs *regs) 1315 { 1316 struct ia64_psr *ipsr = ia64_psr(regs); 1317 unsigned long bundle[2]; 1318 unsigned long opcode; 1319 const struct exception_table_entry *eh = NULL; 1320 union { 1321 unsigned long l; 1322 load_store_t insn; 1323 } u; 1324 int ret = -1; 1325 bool kernel_mode = false; 1326 1327 if (ia64_psr(regs)->be) { 1328 /* we don't support big-endian accesses */ 1329 if (die_if_kernel("big-endian unaligned accesses are not supported", regs, 0)) 1330 return; 1331 goto force_sigbus; 1332 } 1333 1334 /* 1335 * Treat kernel accesses for which there is an exception handler entry the same as 1336 * user-level unaligned accesses. Otherwise, a clever program could trick this 1337 * handler into reading an arbitrary kernel addresses... 1338 */ 1339 if (!user_mode(regs)) 1340 eh = search_exception_tables(regs->cr_iip + ia64_psr(regs)->ri); 1341 if (user_mode(regs) || eh) { 1342 if ((current->thread.flags & IA64_THREAD_UAC_SIGBUS) != 0) 1343 goto force_sigbus; 1344 1345 if (!no_unaligned_warning && 1346 !(current->thread.flags & IA64_THREAD_UAC_NOPRINT) && 1347 __ratelimit(&logging_rate_limit)) 1348 { 1349 char buf[200]; /* comm[] is at most 16 bytes... */ 1350 size_t len; 1351 1352 len = sprintf(buf, "%s(%d): unaligned access to 0x%016lx, " 1353 "ip=0x%016lx\n\r", current->comm, 1354 task_pid_nr(current), 1355 ifa, regs->cr_iip + ipsr->ri); 1356 /* 1357 * Don't call tty_write_message() if we're in the kernel; we might 1358 * be holding locks... 1359 */ 1360 if (user_mode(regs)) { 1361 struct tty_struct *tty = get_current_tty(); 1362 tty_write_message(tty, buf); 1363 tty_kref_put(tty); 1364 } 1365 buf[len-1] = '\0'; /* drop '\r' */ 1366 /* watch for command names containing %s */ 1367 printk(KERN_WARNING "%s", buf); 1368 } else { 1369 if (no_unaligned_warning) { 1370 printk_once(KERN_WARNING "%s(%d) encountered an " 1371 "unaligned exception which required\n" 1372 "kernel assistance, which degrades " 1373 "the performance of the application.\n" 1374 "Unaligned exception warnings have " 1375 "been disabled by the system " 1376 "administrator\n" 1377 "echo 0 > /proc/sys/kernel/ignore-" 1378 "unaligned-usertrap to re-enable\n", 1379 current->comm, task_pid_nr(current)); 1380 } 1381 } 1382 } else { 1383 if (__ratelimit(&logging_rate_limit)) { 1384 printk(KERN_WARNING "kernel unaligned access to 0x%016lx, ip=0x%016lx\n", 1385 ifa, regs->cr_iip + ipsr->ri); 1386 if (unaligned_dump_stack) 1387 dump_stack(); 1388 } 1389 kernel_mode = true; 1390 } 1391 1392 DPRINT("iip=%lx ifa=%lx isr=%lx (ei=%d, sp=%d)\n", 1393 regs->cr_iip, ifa, regs->cr_ipsr, ipsr->ri, ipsr->it); 1394 1395 if (emulate_load(bundle, regs->cr_iip, 16, kernel_mode)) 1396 goto failure; 1397 1398 /* 1399 * extract the instruction from the bundle given the slot number 1400 */ 1401 switch (ipsr->ri) { 1402 default: 1403 case 0: u.l = (bundle[0] >> 5); break; 1404 case 1: u.l = (bundle[0] >> 46) | (bundle[1] << 18); break; 1405 case 2: u.l = (bundle[1] >> 23); break; 1406 } 1407 opcode = (u.l >> IA64_OPCODE_SHIFT) & IA64_OPCODE_MASK; 1408 1409 DPRINT("opcode=%lx ld.qp=%d ld.r1=%d ld.imm=%d ld.r3=%d ld.x=%d ld.hint=%d " 1410 "ld.x6=0x%x ld.m=%d ld.op=%d\n", opcode, u.insn.qp, u.insn.r1, u.insn.imm, 1411 u.insn.r3, u.insn.x, u.insn.hint, u.insn.x6_sz, u.insn.m, u.insn.op); 1412 1413 /* 1414 * IMPORTANT: 1415 * Notice that the switch statement DOES not cover all possible instructions 1416 * that DO generate unaligned references. This is made on purpose because for some 1417 * instructions it DOES NOT make sense to try and emulate the access. Sometimes it 1418 * is WRONG to try and emulate. Here is a list of instruction we don't emulate i.e., 1419 * the program will get a signal and die: 1420 * 1421 * load/store: 1422 * - ldX.spill 1423 * - stX.spill 1424 * Reason: RNATs are based on addresses 1425 * - ld16 1426 * - st16 1427 * Reason: ld16 and st16 are supposed to occur in a single 1428 * memory op 1429 * 1430 * synchronization: 1431 * - cmpxchg 1432 * - fetchadd 1433 * - xchg 1434 * Reason: ATOMIC operations cannot be emulated properly using multiple 1435 * instructions. 1436 * 1437 * speculative loads: 1438 * - ldX.sZ 1439 * Reason: side effects, code must be ready to deal with failure so simpler 1440 * to let the load fail. 1441 * --------------------------------------------------------------------------------- 1442 * XXX fixme 1443 * 1444 * I would like to get rid of this switch case and do something 1445 * more elegant. 1446 */ 1447 switch (opcode) { 1448 case LDS_OP: 1449 case LDSA_OP: 1450 if (u.insn.x) 1451 /* oops, really a semaphore op (cmpxchg, etc) */ 1452 goto failure; 1453 fallthrough; 1454 case LDS_IMM_OP: 1455 case LDSA_IMM_OP: 1456 case LDFS_OP: 1457 case LDFSA_OP: 1458 case LDFS_IMM_OP: 1459 /* 1460 * The instruction will be retried with deferred exceptions turned on, and 1461 * we should get Nat bit installed 1462 * 1463 * IMPORTANT: When PSR_ED is set, the register & immediate update forms 1464 * are actually executed even though the operation failed. So we don't 1465 * need to take care of this. 1466 */ 1467 DPRINT("forcing PSR_ED\n"); 1468 regs->cr_ipsr |= IA64_PSR_ED; 1469 goto done; 1470 1471 case LD_OP: 1472 case LDA_OP: 1473 case LDBIAS_OP: 1474 case LDACQ_OP: 1475 case LDCCLR_OP: 1476 case LDCNC_OP: 1477 case LDCCLRACQ_OP: 1478 if (u.insn.x) 1479 /* oops, really a semaphore op (cmpxchg, etc) */ 1480 goto failure; 1481 fallthrough; 1482 case LD_IMM_OP: 1483 case LDA_IMM_OP: 1484 case LDBIAS_IMM_OP: 1485 case LDACQ_IMM_OP: 1486 case LDCCLR_IMM_OP: 1487 case LDCNC_IMM_OP: 1488 case LDCCLRACQ_IMM_OP: 1489 ret = emulate_load_int(ifa, u.insn, regs, kernel_mode); 1490 break; 1491 1492 case ST_OP: 1493 case STREL_OP: 1494 if (u.insn.x) 1495 /* oops, really a semaphore op (cmpxchg, etc) */ 1496 goto failure; 1497 fallthrough; 1498 case ST_IMM_OP: 1499 case STREL_IMM_OP: 1500 ret = emulate_store_int(ifa, u.insn, regs, kernel_mode); 1501 break; 1502 1503 case LDF_OP: 1504 case LDFA_OP: 1505 case LDFCCLR_OP: 1506 case LDFCNC_OP: 1507 if (u.insn.x) 1508 ret = emulate_load_floatpair(ifa, u.insn, regs, kernel_mode); 1509 else 1510 ret = emulate_load_float(ifa, u.insn, regs, kernel_mode); 1511 break; 1512 1513 case LDF_IMM_OP: 1514 case LDFA_IMM_OP: 1515 case LDFCCLR_IMM_OP: 1516 case LDFCNC_IMM_OP: 1517 ret = emulate_load_float(ifa, u.insn, regs, kernel_mode); 1518 break; 1519 1520 case STF_OP: 1521 case STF_IMM_OP: 1522 ret = emulate_store_float(ifa, u.insn, regs, kernel_mode); 1523 break; 1524 1525 default: 1526 goto failure; 1527 } 1528 DPRINT("ret=%d\n", ret); 1529 if (ret) 1530 goto failure; 1531 1532 if (ipsr->ri == 2) 1533 /* 1534 * given today's architecture this case is not likely to happen because a 1535 * memory access instruction (M) can never be in the last slot of a 1536 * bundle. But let's keep it for now. 1537 */ 1538 regs->cr_iip += 16; 1539 ipsr->ri = (ipsr->ri + 1) & 0x3; 1540 1541 DPRINT("ipsr->ri=%d iip=%lx\n", ipsr->ri, regs->cr_iip); 1542 done: 1543 return; 1544 1545 failure: 1546 /* something went wrong... */ 1547 if (!user_mode(regs)) { 1548 if (eh) { 1549 ia64_handle_exception(regs, eh); 1550 goto done; 1551 } 1552 if (die_if_kernel("error during unaligned kernel access\n", regs, ret)) 1553 return; 1554 /* NOT_REACHED */ 1555 } 1556 force_sigbus: 1557 force_sig_fault(SIGBUS, BUS_ADRALN, (void __user *) ifa, 1558 0, 0, 0); 1559 goto done; 1560 } 1561