1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * arch/sparc64/math-emu/math.c 4 * 5 * Copyright (C) 1997,1999 Jakub Jelinek (jj@ultra.linux.cz) 6 * Copyright (C) 1999 David S. Miller (davem@redhat.com) 7 * 8 * Emulation routines originate from soft-fp package, which is part 9 * of glibc and has appropriate copyrights in it. 10 */ 11 12 #include <linux/types.h> 13 #include <linux/sched.h> 14 #include <linux/errno.h> 15 #include <linux/perf_event.h> 16 17 #include <asm/fpumacro.h> 18 #include <asm/ptrace.h> 19 #include <linux/uaccess.h> 20 #include <asm/cacheflush.h> 21 22 #include "sfp-util_64.h" 23 #include <math-emu/soft-fp.h> 24 #include <math-emu/single.h> 25 #include <math-emu/double.h> 26 #include <math-emu/quad.h> 27 28 /* QUAD - ftt == 3 */ 29 #define FMOVQ 0x003 30 #define FNEGQ 0x007 31 #define FABSQ 0x00b 32 #define FSQRTQ 0x02b 33 #define FADDQ 0x043 34 #define FSUBQ 0x047 35 #define FMULQ 0x04b 36 #define FDIVQ 0x04f 37 #define FDMULQ 0x06e 38 #define FQTOX 0x083 39 #define FXTOQ 0x08c 40 #define FQTOS 0x0c7 41 #define FQTOD 0x0cb 42 #define FITOQ 0x0cc 43 #define FSTOQ 0x0cd 44 #define FDTOQ 0x0ce 45 #define FQTOI 0x0d3 46 /* SUBNORMAL - ftt == 2 */ 47 #define FSQRTS 0x029 48 #define FSQRTD 0x02a 49 #define FADDS 0x041 50 #define FADDD 0x042 51 #define FSUBS 0x045 52 #define FSUBD 0x046 53 #define FMULS 0x049 54 #define FMULD 0x04a 55 #define FDIVS 0x04d 56 #define FDIVD 0x04e 57 #define FSMULD 0x069 58 #define FSTOX 0x081 59 #define FDTOX 0x082 60 #define FDTOS 0x0c6 61 #define FSTOD 0x0c9 62 #define FSTOI 0x0d1 63 #define FDTOI 0x0d2 64 #define FXTOS 0x084 /* Only Ultra-III generates this. */ 65 #define FXTOD 0x088 /* Only Ultra-III generates this. */ 66 #if 0 /* Optimized inline in sparc64/kernel/entry.S */ 67 #define FITOS 0x0c4 /* Only Ultra-III generates this. */ 68 #endif 69 #define FITOD 0x0c8 /* Only Ultra-III generates this. */ 70 /* FPOP2 */ 71 #define FCMPQ 0x053 72 #define FCMPEQ 0x057 73 #define FMOVQ0 0x003 74 #define FMOVQ1 0x043 75 #define FMOVQ2 0x083 76 #define FMOVQ3 0x0c3 77 #define FMOVQI 0x103 78 #define FMOVQX 0x183 79 #define FMOVQZ 0x027 80 #define FMOVQLE 0x047 81 #define FMOVQLZ 0x067 82 #define FMOVQNZ 0x0a7 83 #define FMOVQGZ 0x0c7 84 #define FMOVQGE 0x0e7 85 86 #define FSR_TEM_SHIFT 23UL 87 #define FSR_TEM_MASK (0x1fUL << FSR_TEM_SHIFT) 88 #define FSR_AEXC_SHIFT 5UL 89 #define FSR_AEXC_MASK (0x1fUL << FSR_AEXC_SHIFT) 90 #define FSR_CEXC_SHIFT 0UL 91 #define FSR_CEXC_MASK (0x1fUL << FSR_CEXC_SHIFT) 92 93 /* All routines returning an exception to raise should detect 94 * such exceptions _before_ rounding to be consistent with 95 * the behavior of the hardware in the implemented cases 96 * (and thus with the recommendations in the V9 architecture 97 * manual). 98 * 99 * We return 0 if a SIGFPE should be sent, 1 otherwise. 100 */ 101 static inline int record_exception(struct pt_regs *regs, int eflag) 102 { 103 u64 fsr = current_thread_info()->xfsr[0]; 104 int would_trap; 105 106 /* Determine if this exception would have generated a trap. */ 107 would_trap = (fsr & ((long)eflag << FSR_TEM_SHIFT)) != 0UL; 108 109 /* If trapping, we only want to signal one bit. */ 110 if(would_trap != 0) { 111 eflag &= ((fsr & FSR_TEM_MASK) >> FSR_TEM_SHIFT); 112 if((eflag & (eflag - 1)) != 0) { 113 if(eflag & FP_EX_INVALID) 114 eflag = FP_EX_INVALID; 115 else if(eflag & FP_EX_OVERFLOW) 116 eflag = FP_EX_OVERFLOW; 117 else if(eflag & FP_EX_UNDERFLOW) 118 eflag = FP_EX_UNDERFLOW; 119 else if(eflag & FP_EX_DIVZERO) 120 eflag = FP_EX_DIVZERO; 121 else if(eflag & FP_EX_INEXACT) 122 eflag = FP_EX_INEXACT; 123 } 124 } 125 126 /* Set CEXC, here is the rule: 127 * 128 * In general all FPU ops will set one and only one 129 * bit in the CEXC field, this is always the case 130 * when the IEEE exception trap is enabled in TEM. 131 */ 132 fsr &= ~(FSR_CEXC_MASK); 133 fsr |= ((long)eflag << FSR_CEXC_SHIFT); 134 135 /* Set the AEXC field, rule is: 136 * 137 * If a trap would not be generated, the 138 * CEXC just generated is OR'd into the 139 * existing value of AEXC. 140 */ 141 if(would_trap == 0) 142 fsr |= ((long)eflag << FSR_AEXC_SHIFT); 143 144 /* If trapping, indicate fault trap type IEEE. */ 145 if(would_trap != 0) 146 fsr |= (1UL << 14); 147 148 current_thread_info()->xfsr[0] = fsr; 149 150 /* If we will not trap, advance the program counter over 151 * the instruction being handled. 152 */ 153 if(would_trap == 0) { 154 regs->tpc = regs->tnpc; 155 regs->tnpc += 4; 156 } 157 158 return (would_trap ? 0 : 1); 159 } 160 161 typedef union { 162 u32 s; 163 u64 d; 164 u64 q[2]; 165 } *argp; 166 167 int do_mathemu(struct pt_regs *regs, struct fpustate *f, bool illegal_insn_trap) 168 { 169 unsigned long pc = regs->tpc; 170 unsigned long tstate = regs->tstate; 171 u32 insn = 0; 172 int type = 0; 173 /* ftt tells which ftt it may happen in, r is rd, b is rs2 and a is rs1. The *u arg tells 174 whether the argument should be packed/unpacked (0 - do not unpack/pack, 1 - unpack/pack) 175 non-u args tells the size of the argument (0 - no argument, 1 - single, 2 - double, 3 - quad */ 176 #define TYPE(ftt, r, ru, b, bu, a, au) type = (au << 2) | (a << 0) | (bu << 5) | (b << 3) | (ru << 8) | (r << 6) | (ftt << 9) 177 int freg; 178 static u64 zero[2] = { 0L, 0L }; 179 int flags; 180 FP_DECL_EX; 181 FP_DECL_S(SA); FP_DECL_S(SB); FP_DECL_S(SR); 182 FP_DECL_D(DA); FP_DECL_D(DB); FP_DECL_D(DR); 183 FP_DECL_Q(QA); FP_DECL_Q(QB); FP_DECL_Q(QR); 184 int IR; 185 long XR, xfsr; 186 187 if (tstate & TSTATE_PRIV) 188 die_if_kernel("unfinished/unimplemented FPop from kernel", regs); 189 perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, 0); 190 if (test_thread_flag(TIF_32BIT)) 191 pc = (u32)pc; 192 if (get_user(insn, (u32 __user *) pc) != -EFAULT) { 193 if ((insn & 0xc1f80000) == 0x81a00000) /* FPOP1 */ { 194 switch ((insn >> 5) & 0x1ff) { 195 /* QUAD - ftt == 3 */ 196 case FMOVQ: 197 case FNEGQ: 198 case FABSQ: TYPE(3,3,0,3,0,0,0); break; 199 case FSQRTQ: TYPE(3,3,1,3,1,0,0); break; 200 case FADDQ: 201 case FSUBQ: 202 case FMULQ: 203 case FDIVQ: TYPE(3,3,1,3,1,3,1); break; 204 case FDMULQ: TYPE(3,3,1,2,1,2,1); break; 205 case FQTOX: TYPE(3,2,0,3,1,0,0); break; 206 case FXTOQ: TYPE(3,3,1,2,0,0,0); break; 207 case FQTOS: TYPE(3,1,1,3,1,0,0); break; 208 case FQTOD: TYPE(3,2,1,3,1,0,0); break; 209 case FITOQ: TYPE(3,3,1,1,0,0,0); break; 210 case FSTOQ: TYPE(3,3,1,1,1,0,0); break; 211 case FDTOQ: TYPE(3,3,1,2,1,0,0); break; 212 case FQTOI: TYPE(3,1,0,3,1,0,0); break; 213 214 /* We can get either unimplemented or unfinished 215 * for these cases. Pre-Niagara systems generate 216 * unfinished fpop for SUBNORMAL cases, and Niagara 217 * always gives unimplemented fpop for fsqrt{s,d}. 218 */ 219 case FSQRTS: { 220 unsigned long x = current_thread_info()->xfsr[0]; 221 222 x = (x >> 14) & 0x7; 223 TYPE(x,1,1,1,1,0,0); 224 break; 225 } 226 227 case FSQRTD: { 228 unsigned long x = current_thread_info()->xfsr[0]; 229 230 x = (x >> 14) & 0x7; 231 TYPE(x,2,1,2,1,0,0); 232 break; 233 } 234 235 /* SUBNORMAL - ftt == 2 */ 236 case FADDD: 237 case FSUBD: 238 case FMULD: 239 case FDIVD: TYPE(2,2,1,2,1,2,1); break; 240 case FADDS: 241 case FSUBS: 242 case FMULS: 243 case FDIVS: TYPE(2,1,1,1,1,1,1); break; 244 case FSMULD: TYPE(2,2,1,1,1,1,1); break; 245 case FSTOX: TYPE(2,2,0,1,1,0,0); break; 246 case FDTOX: TYPE(2,2,0,2,1,0,0); break; 247 case FDTOS: TYPE(2,1,1,2,1,0,0); break; 248 case FSTOD: TYPE(2,2,1,1,1,0,0); break; 249 case FSTOI: TYPE(2,1,0,1,1,0,0); break; 250 case FDTOI: TYPE(2,1,0,2,1,0,0); break; 251 252 /* Only Ultra-III generates these */ 253 case FXTOS: TYPE(2,1,1,2,0,0,0); break; 254 case FXTOD: TYPE(2,2,1,2,0,0,0); break; 255 #if 0 /* Optimized inline in sparc64/kernel/entry.S */ 256 case FITOS: TYPE(2,1,1,1,0,0,0); break; 257 #endif 258 case FITOD: TYPE(2,2,1,1,0,0,0); break; 259 } 260 } 261 else if ((insn & 0xc1f80000) == 0x81a80000) /* FPOP2 */ { 262 IR = 2; 263 switch ((insn >> 5) & 0x1ff) { 264 case FCMPQ: TYPE(3,0,0,3,1,3,1); break; 265 case FCMPEQ: TYPE(3,0,0,3,1,3,1); break; 266 /* Now the conditional fmovq support */ 267 case FMOVQ0: 268 case FMOVQ1: 269 case FMOVQ2: 270 case FMOVQ3: 271 /* fmovq %fccX, %fY, %fZ */ 272 if (!((insn >> 11) & 3)) 273 XR = current_thread_info()->xfsr[0] >> 10; 274 else 275 XR = current_thread_info()->xfsr[0] >> (30 + ((insn >> 10) & 0x6)); 276 XR &= 3; 277 IR = 0; 278 switch ((insn >> 14) & 0x7) { 279 /* case 0: IR = 0; break; */ /* Never */ 280 case 1: if (XR) IR = 1; break; /* Not Equal */ 281 case 2: if (XR == 1 || XR == 2) IR = 1; break; /* Less or Greater */ 282 case 3: if (XR & 1) IR = 1; break; /* Unordered or Less */ 283 case 4: if (XR == 1) IR = 1; break; /* Less */ 284 case 5: if (XR & 2) IR = 1; break; /* Unordered or Greater */ 285 case 6: if (XR == 2) IR = 1; break; /* Greater */ 286 case 7: if (XR == 3) IR = 1; break; /* Unordered */ 287 } 288 if ((insn >> 14) & 8) 289 IR ^= 1; 290 break; 291 case FMOVQI: 292 case FMOVQX: 293 /* fmovq %[ix]cc, %fY, %fZ */ 294 XR = regs->tstate >> 32; 295 if ((insn >> 5) & 0x80) 296 XR >>= 4; 297 XR &= 0xf; 298 IR = 0; 299 freg = ((XR >> 2) ^ XR) & 2; 300 switch ((insn >> 14) & 0x7) { 301 /* case 0: IR = 0; break; */ /* Never */ 302 case 1: if (XR & 4) IR = 1; break; /* Equal */ 303 case 2: if ((XR & 4) || freg) IR = 1; break; /* Less or Equal */ 304 case 3: if (freg) IR = 1; break; /* Less */ 305 case 4: if (XR & 5) IR = 1; break; /* Less or Equal Unsigned */ 306 case 5: if (XR & 1) IR = 1; break; /* Carry Set */ 307 case 6: if (XR & 8) IR = 1; break; /* Negative */ 308 case 7: if (XR & 2) IR = 1; break; /* Overflow Set */ 309 } 310 if ((insn >> 14) & 8) 311 IR ^= 1; 312 break; 313 case FMOVQZ: 314 case FMOVQLE: 315 case FMOVQLZ: 316 case FMOVQNZ: 317 case FMOVQGZ: 318 case FMOVQGE: 319 freg = (insn >> 14) & 0x1f; 320 if (!freg) 321 XR = 0; 322 else if (freg < 16) 323 XR = regs->u_regs[freg]; 324 else if (!test_thread_64bit_stack(regs->u_regs[UREG_FP])) { 325 struct reg_window32 __user *win32; 326 flushw_user (); 327 win32 = (struct reg_window32 __user *)((unsigned long)((u32)regs->u_regs[UREG_FP])); 328 get_user(XR, &win32->locals[freg - 16]); 329 } else { 330 struct reg_window __user *win; 331 flushw_user (); 332 win = (struct reg_window __user *)(regs->u_regs[UREG_FP] + STACK_BIAS); 333 get_user(XR, &win->locals[freg - 16]); 334 } 335 IR = 0; 336 switch ((insn >> 10) & 3) { 337 case 1: if (!XR) IR = 1; break; /* Register Zero */ 338 case 2: if (XR <= 0) IR = 1; break; /* Register Less Than or Equal to Zero */ 339 case 3: if (XR < 0) IR = 1; break; /* Register Less Than Zero */ 340 } 341 if ((insn >> 10) & 4) 342 IR ^= 1; 343 break; 344 } 345 if (IR == 0) { 346 /* The fmov test was false. Do a nop instead */ 347 current_thread_info()->xfsr[0] &= ~(FSR_CEXC_MASK); 348 regs->tpc = regs->tnpc; 349 regs->tnpc += 4; 350 return 1; 351 } else if (IR == 1) { 352 /* Change the instruction into plain fmovq */ 353 insn = (insn & 0x3e00001f) | 0x81a00060; 354 TYPE(3,3,0,3,0,0,0); 355 } 356 } 357 } 358 if (type) { 359 argp rs1 = NULL, rs2 = NULL, rd = NULL; 360 361 /* Starting with UltraSPARC-T2, the cpu does not set the FP Trap 362 * Type field in the %fsr to unimplemented_FPop. Nor does it 363 * use the fp_exception_other trap. Instead it signals an 364 * illegal instruction and leaves the FP trap type field of 365 * the %fsr unchanged. 366 */ 367 if (!illegal_insn_trap) { 368 int ftt = (current_thread_info()->xfsr[0] >> 14) & 0x7; 369 if (ftt != (type >> 9)) 370 goto err; 371 } 372 current_thread_info()->xfsr[0] &= ~0x1c000; 373 freg = ((insn >> 14) & 0x1f); 374 switch (type & 0x3) { 375 case 3: if (freg & 2) { 376 current_thread_info()->xfsr[0] |= (6 << 14) /* invalid_fp_register */; 377 goto err; 378 } 379 case 2: freg = ((freg & 1) << 5) | (freg & 0x1e); 380 case 1: rs1 = (argp)&f->regs[freg]; 381 flags = (freg < 32) ? FPRS_DL : FPRS_DU; 382 if (!(current_thread_info()->fpsaved[0] & flags)) 383 rs1 = (argp)&zero; 384 break; 385 } 386 switch (type & 0x7) { 387 case 7: FP_UNPACK_QP (QA, rs1); break; 388 case 6: FP_UNPACK_DP (DA, rs1); break; 389 case 5: FP_UNPACK_SP (SA, rs1); break; 390 } 391 freg = (insn & 0x1f); 392 switch ((type >> 3) & 0x3) { 393 case 3: if (freg & 2) { 394 current_thread_info()->xfsr[0] |= (6 << 14) /* invalid_fp_register */; 395 goto err; 396 } 397 case 2: freg = ((freg & 1) << 5) | (freg & 0x1e); 398 case 1: rs2 = (argp)&f->regs[freg]; 399 flags = (freg < 32) ? FPRS_DL : FPRS_DU; 400 if (!(current_thread_info()->fpsaved[0] & flags)) 401 rs2 = (argp)&zero; 402 break; 403 } 404 switch ((type >> 3) & 0x7) { 405 case 7: FP_UNPACK_QP (QB, rs2); break; 406 case 6: FP_UNPACK_DP (DB, rs2); break; 407 case 5: FP_UNPACK_SP (SB, rs2); break; 408 } 409 freg = ((insn >> 25) & 0x1f); 410 switch ((type >> 6) & 0x3) { 411 case 3: if (freg & 2) { 412 current_thread_info()->xfsr[0] |= (6 << 14) /* invalid_fp_register */; 413 goto err; 414 } 415 case 2: freg = ((freg & 1) << 5) | (freg & 0x1e); 416 case 1: rd = (argp)&f->regs[freg]; 417 flags = (freg < 32) ? FPRS_DL : FPRS_DU; 418 if (!(current_thread_info()->fpsaved[0] & FPRS_FEF)) { 419 current_thread_info()->fpsaved[0] = FPRS_FEF; 420 current_thread_info()->gsr[0] = 0; 421 } 422 if (!(current_thread_info()->fpsaved[0] & flags)) { 423 if (freg < 32) 424 memset(f->regs, 0, 32*sizeof(u32)); 425 else 426 memset(f->regs+32, 0, 32*sizeof(u32)); 427 } 428 current_thread_info()->fpsaved[0] |= flags; 429 break; 430 } 431 switch ((insn >> 5) & 0x1ff) { 432 /* + */ 433 case FADDS: FP_ADD_S (SR, SA, SB); break; 434 case FADDD: FP_ADD_D (DR, DA, DB); break; 435 case FADDQ: FP_ADD_Q (QR, QA, QB); break; 436 /* - */ 437 case FSUBS: FP_SUB_S (SR, SA, SB); break; 438 case FSUBD: FP_SUB_D (DR, DA, DB); break; 439 case FSUBQ: FP_SUB_Q (QR, QA, QB); break; 440 /* * */ 441 case FMULS: FP_MUL_S (SR, SA, SB); break; 442 case FSMULD: FP_CONV (D, S, 1, 1, DA, SA); 443 FP_CONV (D, S, 1, 1, DB, SB); 444 case FMULD: FP_MUL_D (DR, DA, DB); break; 445 case FDMULQ: FP_CONV (Q, D, 2, 1, QA, DA); 446 FP_CONV (Q, D, 2, 1, QB, DB); 447 case FMULQ: FP_MUL_Q (QR, QA, QB); break; 448 /* / */ 449 case FDIVS: FP_DIV_S (SR, SA, SB); break; 450 case FDIVD: FP_DIV_D (DR, DA, DB); break; 451 case FDIVQ: FP_DIV_Q (QR, QA, QB); break; 452 /* sqrt */ 453 case FSQRTS: FP_SQRT_S (SR, SB); break; 454 case FSQRTD: FP_SQRT_D (DR, DB); break; 455 case FSQRTQ: FP_SQRT_Q (QR, QB); break; 456 /* mov */ 457 case FMOVQ: rd->q[0] = rs2->q[0]; rd->q[1] = rs2->q[1]; break; 458 case FABSQ: rd->q[0] = rs2->q[0] & 0x7fffffffffffffffUL; rd->q[1] = rs2->q[1]; break; 459 case FNEGQ: rd->q[0] = rs2->q[0] ^ 0x8000000000000000UL; rd->q[1] = rs2->q[1]; break; 460 /* float to int */ 461 case FSTOI: FP_TO_INT_S (IR, SB, 32, 1); break; 462 case FDTOI: FP_TO_INT_D (IR, DB, 32, 1); break; 463 case FQTOI: FP_TO_INT_Q (IR, QB, 32, 1); break; 464 case FSTOX: FP_TO_INT_S (XR, SB, 64, 1); break; 465 case FDTOX: FP_TO_INT_D (XR, DB, 64, 1); break; 466 case FQTOX: FP_TO_INT_Q (XR, QB, 64, 1); break; 467 /* int to float */ 468 case FITOQ: IR = rs2->s; FP_FROM_INT_Q (QR, IR, 32, int); break; 469 case FXTOQ: XR = rs2->d; FP_FROM_INT_Q (QR, XR, 64, long); break; 470 /* Only Ultra-III generates these */ 471 case FXTOS: XR = rs2->d; FP_FROM_INT_S (SR, XR, 64, long); break; 472 case FXTOD: XR = rs2->d; FP_FROM_INT_D (DR, XR, 64, long); break; 473 #if 0 /* Optimized inline in sparc64/kernel/entry.S */ 474 case FITOS: IR = rs2->s; FP_FROM_INT_S (SR, IR, 32, int); break; 475 #endif 476 case FITOD: IR = rs2->s; FP_FROM_INT_D (DR, IR, 32, int); break; 477 /* float to float */ 478 case FSTOD: FP_CONV (D, S, 1, 1, DR, SB); break; 479 case FSTOQ: FP_CONV (Q, S, 2, 1, QR, SB); break; 480 case FDTOQ: FP_CONV (Q, D, 2, 1, QR, DB); break; 481 case FDTOS: FP_CONV (S, D, 1, 1, SR, DB); break; 482 case FQTOS: FP_CONV (S, Q, 1, 2, SR, QB); break; 483 case FQTOD: FP_CONV (D, Q, 1, 2, DR, QB); break; 484 /* comparison */ 485 case FCMPQ: 486 case FCMPEQ: 487 FP_CMP_Q(XR, QB, QA, 3); 488 if (XR == 3 && 489 (((insn >> 5) & 0x1ff) == FCMPEQ || 490 FP_ISSIGNAN_Q(QA) || 491 FP_ISSIGNAN_Q(QB))) 492 FP_SET_EXCEPTION (FP_EX_INVALID); 493 } 494 if (!FP_INHIBIT_RESULTS) { 495 switch ((type >> 6) & 0x7) { 496 case 0: xfsr = current_thread_info()->xfsr[0]; 497 if (XR == -1) XR = 2; 498 switch (freg & 3) { 499 /* fcc0, 1, 2, 3 */ 500 case 0: xfsr &= ~0xc00; xfsr |= (XR << 10); break; 501 case 1: xfsr &= ~0x300000000UL; xfsr |= (XR << 32); break; 502 case 2: xfsr &= ~0xc00000000UL; xfsr |= (XR << 34); break; 503 case 3: xfsr &= ~0x3000000000UL; xfsr |= (XR << 36); break; 504 } 505 current_thread_info()->xfsr[0] = xfsr; 506 break; 507 case 1: rd->s = IR; break; 508 case 2: rd->d = XR; break; 509 case 5: FP_PACK_SP (rd, SR); break; 510 case 6: FP_PACK_DP (rd, DR); break; 511 case 7: FP_PACK_QP (rd, QR); break; 512 } 513 } 514 515 if(_fex != 0) 516 return record_exception(regs, _fex); 517 518 /* Success and no exceptions detected. */ 519 current_thread_info()->xfsr[0] &= ~(FSR_CEXC_MASK); 520 regs->tpc = regs->tnpc; 521 regs->tnpc += 4; 522 return 1; 523 } 524 err: return 0; 525 } 526