1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0 20c306bcfSIngo Molnar /* 30c306bcfSIngo Molnar * FPU register's regset abstraction, for ptrace, core dumps, etc. 40c306bcfSIngo Molnar */ 543be46e8SThomas Gleixner #include <linux/sched/task_stack.h> 643be46e8SThomas Gleixner #include <linux/vmalloc.h> 743be46e8SThomas Gleixner 80c306bcfSIngo Molnar #include <asm/fpu/internal.h> 90c306bcfSIngo Molnar #include <asm/fpu/signal.h> 100c306bcfSIngo Molnar #include <asm/fpu/regset.h> 1191c3dba7SYu-cheng Yu #include <asm/fpu/xstate.h> 120c306bcfSIngo Molnar 130c306bcfSIngo Molnar /* 140c306bcfSIngo Molnar * The xstateregs_active() routine is the same as the regset_fpregs_active() routine, 150c306bcfSIngo Molnar * as the "regset->n" for the xstate regset will be updated based on the feature 166a6256f9SAdam Buchbinder * capabilities supported by the xsave. 170c306bcfSIngo Molnar */ 180c306bcfSIngo Molnar int regset_fpregs_active(struct task_struct *target, const struct user_regset *regset) 190c306bcfSIngo Molnar { 202722146eSSebastian Andrzej Siewior return regset->n; 210c306bcfSIngo Molnar } 220c306bcfSIngo Molnar 230c306bcfSIngo Molnar int regset_xregset_fpregs_active(struct task_struct *target, const struct user_regset *regset) 240c306bcfSIngo Molnar { 252722146eSSebastian Andrzej Siewior if (boot_cpu_has(X86_FEATURE_FXSR)) 2601f8fd73SBorislav Petkov return regset->n; 2701f8fd73SBorislav Petkov else 2801f8fd73SBorislav Petkov return 0; 290c306bcfSIngo Molnar } 300c306bcfSIngo Molnar 310c306bcfSIngo Molnar int xfpregs_get(struct task_struct *target, const struct user_regset *regset, 320557d64dSAl Viro struct membuf to) 330c306bcfSIngo Molnar { 340c306bcfSIngo Molnar struct fpu *fpu = &target->thread.fpu; 350c306bcfSIngo Molnar 3601f8fd73SBorislav Petkov if (!boot_cpu_has(X86_FEATURE_FXSR)) 370c306bcfSIngo Molnar return -ENODEV; 380c306bcfSIngo Molnar 39369a036dSIngo Molnar fpu__prepare_read(fpu); 400c306bcfSIngo Molnar fpstate_sanitize_xstate(fpu); 410c306bcfSIngo Molnar 420557d64dSAl Viro return membuf_write(&to, &fpu->state.fxsave, sizeof(struct fxregs_state)); 430c306bcfSIngo Molnar } 440c306bcfSIngo Molnar 450c306bcfSIngo Molnar int xfpregs_set(struct task_struct *target, const struct user_regset *regset, 460c306bcfSIngo Molnar unsigned int pos, unsigned int count, 470c306bcfSIngo Molnar const void *kbuf, const void __user *ubuf) 480c306bcfSIngo Molnar { 490c306bcfSIngo Molnar struct fpu *fpu = &target->thread.fpu; 506164331dSAndy Lutomirski struct user32_fxsr_struct newstate; 510c306bcfSIngo Molnar int ret; 520c306bcfSIngo Molnar 536164331dSAndy Lutomirski BUILD_BUG_ON(sizeof(newstate) != sizeof(struct fxregs_state)); 546164331dSAndy Lutomirski 556164331dSAndy Lutomirski if (!cpu_feature_enabled(X86_FEATURE_FXSR)) 560c306bcfSIngo Molnar return -ENODEV; 570c306bcfSIngo Molnar 586164331dSAndy Lutomirski /* No funny business with partial or oversized writes is permitted. */ 596164331dSAndy Lutomirski if (pos != 0 || count != sizeof(newstate)) 606164331dSAndy Lutomirski return -EINVAL; 616164331dSAndy Lutomirski 626164331dSAndy Lutomirski ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &newstate, 0, -1); 636164331dSAndy Lutomirski if (ret) 646164331dSAndy Lutomirski return ret; 656164331dSAndy Lutomirski 66*145e9e0dSAndy Lutomirski /* Do not allow an invalid MXCSR value. */ 67*145e9e0dSAndy Lutomirski if (newstate.mxcsr & ~mxcsr_feature_mask) 68*145e9e0dSAndy Lutomirski return -EINVAL; 696164331dSAndy Lutomirski 70369a036dSIngo Molnar fpu__prepare_write(fpu); 710c306bcfSIngo Molnar 726164331dSAndy Lutomirski /* Copy the state */ 736164331dSAndy Lutomirski memcpy(&fpu->state.fxsave, &newstate, sizeof(newstate)); 740c306bcfSIngo Molnar 756164331dSAndy Lutomirski /* Clear xmm8..15 */ 766164331dSAndy Lutomirski BUILD_BUG_ON(sizeof(fpu->state.fxsave.xmm_space) != 16 * 16); 776164331dSAndy Lutomirski memset(&fpu->state.fxsave.xmm_space[8], 0, 8 * 16); 780c306bcfSIngo Molnar 796164331dSAndy Lutomirski /* Mark FP and SSE as in use when XSAVE is enabled */ 806164331dSAndy Lutomirski if (use_xsave()) 81d91cab78SDave Hansen fpu->state.xsave.header.xfeatures |= XFEATURE_MASK_FPSSE; 820c306bcfSIngo Molnar 836164331dSAndy Lutomirski return 0; 840c306bcfSIngo Molnar } 850c306bcfSIngo Molnar 860c306bcfSIngo Molnar int xstateregs_get(struct task_struct *target, const struct user_regset *regset, 870557d64dSAl Viro struct membuf to) 880c306bcfSIngo Molnar { 890c306bcfSIngo Molnar struct fpu *fpu = &target->thread.fpu; 900c306bcfSIngo Molnar 913a335112SDave Hansen if (!cpu_feature_enabled(X86_FEATURE_XSAVE)) 920c306bcfSIngo Molnar return -ENODEV; 930c306bcfSIngo Molnar 94369a036dSIngo Molnar fpu__prepare_read(fpu); 9591c3dba7SYu-cheng Yu 963a335112SDave Hansen copy_xstate_to_kernel(to, &fpu->state.xsave); 970557d64dSAl Viro return 0; 980c306bcfSIngo Molnar } 990c306bcfSIngo Molnar 1000c306bcfSIngo Molnar int xstateregs_set(struct task_struct *target, const struct user_regset *regset, 1010c306bcfSIngo Molnar unsigned int pos, unsigned int count, 1020c306bcfSIngo Molnar const void *kbuf, const void __user *ubuf) 1030c306bcfSIngo Molnar { 1040c306bcfSIngo Molnar struct fpu *fpu = &target->thread.fpu; 10543be46e8SThomas Gleixner struct xregs_state *tmpbuf = NULL; 1060c306bcfSIngo Molnar int ret; 1070c306bcfSIngo Molnar 10843be46e8SThomas Gleixner if (!cpu_feature_enabled(X86_FEATURE_XSAVE)) 1090c306bcfSIngo Molnar return -ENODEV; 1100c306bcfSIngo Molnar 11191c3dba7SYu-cheng Yu /* 11291c3dba7SYu-cheng Yu * A whole standard-format XSAVE buffer is needed: 11391c3dba7SYu-cheng Yu */ 11407d6688bSThomas Gleixner if (pos != 0 || count != fpu_user_xstate_size) 11591c3dba7SYu-cheng Yu return -EFAULT; 1160c306bcfSIngo Molnar 11743be46e8SThomas Gleixner if (!kbuf) { 11843be46e8SThomas Gleixner tmpbuf = vmalloc(count); 11943be46e8SThomas Gleixner if (!tmpbuf) 12043be46e8SThomas Gleixner return -ENOMEM; 1210c306bcfSIngo Molnar 12243be46e8SThomas Gleixner if (copy_from_user(tmpbuf, ubuf, count)) { 12343be46e8SThomas Gleixner ret = -EFAULT; 12443be46e8SThomas Gleixner goto out; 12543be46e8SThomas Gleixner } 12679fecc2bSIngo Molnar } 12791c3dba7SYu-cheng Yu 12843be46e8SThomas Gleixner fpu__prepare_write(fpu); 12943be46e8SThomas Gleixner ret = copy_kernel_to_xstate(&fpu->state.xsave, kbuf ?: tmpbuf); 130cf9df81bSEric Biggers 13143be46e8SThomas Gleixner out: 13243be46e8SThomas Gleixner vfree(tmpbuf); 1330c306bcfSIngo Molnar return ret; 1340c306bcfSIngo Molnar } 1350c306bcfSIngo Molnar 1360c306bcfSIngo Molnar #if defined CONFIG_X86_32 || defined CONFIG_IA32_EMULATION 1370c306bcfSIngo Molnar 1380c306bcfSIngo Molnar /* 1390c306bcfSIngo Molnar * FPU tag word conversions. 1400c306bcfSIngo Molnar */ 1410c306bcfSIngo Molnar 1420c306bcfSIngo Molnar static inline unsigned short twd_i387_to_fxsr(unsigned short twd) 1430c306bcfSIngo Molnar { 1440c306bcfSIngo Molnar unsigned int tmp; /* to avoid 16 bit prefixes in the code */ 1450c306bcfSIngo Molnar 1460c306bcfSIngo Molnar /* Transform each pair of bits into 01 (valid) or 00 (empty) */ 1470c306bcfSIngo Molnar tmp = ~twd; 1480c306bcfSIngo Molnar tmp = (tmp | (tmp>>1)) & 0x5555; /* 0V0V0V0V0V0V0V0V */ 1490c306bcfSIngo Molnar /* and move the valid bits to the lower byte. */ 1500c306bcfSIngo Molnar tmp = (tmp | (tmp >> 1)) & 0x3333; /* 00VV00VV00VV00VV */ 1510c306bcfSIngo Molnar tmp = (tmp | (tmp >> 2)) & 0x0f0f; /* 0000VVVV0000VVVV */ 1520c306bcfSIngo Molnar tmp = (tmp | (tmp >> 4)) & 0x00ff; /* 00000000VVVVVVVV */ 1530c306bcfSIngo Molnar 1540c306bcfSIngo Molnar return tmp; 1550c306bcfSIngo Molnar } 1560c306bcfSIngo Molnar 1570c306bcfSIngo Molnar #define FPREG_ADDR(f, n) ((void *)&(f)->st_space + (n) * 16) 1580c306bcfSIngo Molnar #define FP_EXP_TAG_VALID 0 1590c306bcfSIngo Molnar #define FP_EXP_TAG_ZERO 1 1600c306bcfSIngo Molnar #define FP_EXP_TAG_SPECIAL 2 1610c306bcfSIngo Molnar #define FP_EXP_TAG_EMPTY 3 1620c306bcfSIngo Molnar 163c47ada30SIngo Molnar static inline u32 twd_fxsr_to_i387(struct fxregs_state *fxsave) 1640c306bcfSIngo Molnar { 1650c306bcfSIngo Molnar struct _fpxreg *st; 1660c306bcfSIngo Molnar u32 tos = (fxsave->swd >> 11) & 7; 1670c306bcfSIngo Molnar u32 twd = (unsigned long) fxsave->twd; 1680c306bcfSIngo Molnar u32 tag; 1690c306bcfSIngo Molnar u32 ret = 0xffff0000u; 1700c306bcfSIngo Molnar int i; 1710c306bcfSIngo Molnar 1720c306bcfSIngo Molnar for (i = 0; i < 8; i++, twd >>= 1) { 1730c306bcfSIngo Molnar if (twd & 0x1) { 1740c306bcfSIngo Molnar st = FPREG_ADDR(fxsave, (i - tos) & 7); 1750c306bcfSIngo Molnar 1760c306bcfSIngo Molnar switch (st->exponent & 0x7fff) { 1770c306bcfSIngo Molnar case 0x7fff: 1780c306bcfSIngo Molnar tag = FP_EXP_TAG_SPECIAL; 1790c306bcfSIngo Molnar break; 1800c306bcfSIngo Molnar case 0x0000: 1810c306bcfSIngo Molnar if (!st->significand[0] && 1820c306bcfSIngo Molnar !st->significand[1] && 1830c306bcfSIngo Molnar !st->significand[2] && 1840c306bcfSIngo Molnar !st->significand[3]) 1850c306bcfSIngo Molnar tag = FP_EXP_TAG_ZERO; 1860c306bcfSIngo Molnar else 1870c306bcfSIngo Molnar tag = FP_EXP_TAG_SPECIAL; 1880c306bcfSIngo Molnar break; 1890c306bcfSIngo Molnar default: 1900c306bcfSIngo Molnar if (st->significand[3] & 0x8000) 1910c306bcfSIngo Molnar tag = FP_EXP_TAG_VALID; 1920c306bcfSIngo Molnar else 1930c306bcfSIngo Molnar tag = FP_EXP_TAG_SPECIAL; 1940c306bcfSIngo Molnar break; 1950c306bcfSIngo Molnar } 1960c306bcfSIngo Molnar } else { 1970c306bcfSIngo Molnar tag = FP_EXP_TAG_EMPTY; 1980c306bcfSIngo Molnar } 1990c306bcfSIngo Molnar ret |= tag << (2 * i); 2000c306bcfSIngo Molnar } 2010c306bcfSIngo Molnar return ret; 2020c306bcfSIngo Molnar } 2030c306bcfSIngo Molnar 2040c306bcfSIngo Molnar /* 2050c306bcfSIngo Molnar * FXSR floating point environment conversions. 2060c306bcfSIngo Molnar */ 2070c306bcfSIngo Molnar 2080c306bcfSIngo Molnar void 2090c306bcfSIngo Molnar convert_from_fxsr(struct user_i387_ia32_struct *env, struct task_struct *tsk) 2100c306bcfSIngo Molnar { 211c47ada30SIngo Molnar struct fxregs_state *fxsave = &tsk->thread.fpu.state.fxsave; 2120c306bcfSIngo Molnar struct _fpreg *to = (struct _fpreg *) &env->st_space[0]; 2130c306bcfSIngo Molnar struct _fpxreg *from = (struct _fpxreg *) &fxsave->st_space[0]; 2140c306bcfSIngo Molnar int i; 2150c306bcfSIngo Molnar 2160c306bcfSIngo Molnar env->cwd = fxsave->cwd | 0xffff0000u; 2170c306bcfSIngo Molnar env->swd = fxsave->swd | 0xffff0000u; 2180c306bcfSIngo Molnar env->twd = twd_fxsr_to_i387(fxsave); 2190c306bcfSIngo Molnar 2200c306bcfSIngo Molnar #ifdef CONFIG_X86_64 2210c306bcfSIngo Molnar env->fip = fxsave->rip; 2220c306bcfSIngo Molnar env->foo = fxsave->rdp; 2230c306bcfSIngo Molnar /* 2240c306bcfSIngo Molnar * should be actually ds/cs at fpu exception time, but 2250c306bcfSIngo Molnar * that information is not available in 64bit mode. 2260c306bcfSIngo Molnar */ 2270c306bcfSIngo Molnar env->fcs = task_pt_regs(tsk)->cs; 2280c306bcfSIngo Molnar if (tsk == current) { 2290c306bcfSIngo Molnar savesegment(ds, env->fos); 2300c306bcfSIngo Molnar } else { 2310c306bcfSIngo Molnar env->fos = tsk->thread.ds; 2320c306bcfSIngo Molnar } 2330c306bcfSIngo Molnar env->fos |= 0xffff0000; 2340c306bcfSIngo Molnar #else 2350c306bcfSIngo Molnar env->fip = fxsave->fip; 2360c306bcfSIngo Molnar env->fcs = (u16) fxsave->fcs | ((u32) fxsave->fop << 16); 2370c306bcfSIngo Molnar env->foo = fxsave->foo; 2380c306bcfSIngo Molnar env->fos = fxsave->fos; 2390c306bcfSIngo Molnar #endif 2400c306bcfSIngo Molnar 2410c306bcfSIngo Molnar for (i = 0; i < 8; ++i) 2420c306bcfSIngo Molnar memcpy(&to[i], &from[i], sizeof(to[0])); 2430c306bcfSIngo Molnar } 2440c306bcfSIngo Molnar 24539ea9bafSSebastian Andrzej Siewior void convert_to_fxsr(struct fxregs_state *fxsave, 2460c306bcfSIngo Molnar const struct user_i387_ia32_struct *env) 2470c306bcfSIngo Molnar 2480c306bcfSIngo Molnar { 2490c306bcfSIngo Molnar struct _fpreg *from = (struct _fpreg *) &env->st_space[0]; 2500c306bcfSIngo Molnar struct _fpxreg *to = (struct _fpxreg *) &fxsave->st_space[0]; 2510c306bcfSIngo Molnar int i; 2520c306bcfSIngo Molnar 2530c306bcfSIngo Molnar fxsave->cwd = env->cwd; 2540c306bcfSIngo Molnar fxsave->swd = env->swd; 2550c306bcfSIngo Molnar fxsave->twd = twd_i387_to_fxsr(env->twd); 2560c306bcfSIngo Molnar fxsave->fop = (u16) ((u32) env->fcs >> 16); 2570c306bcfSIngo Molnar #ifdef CONFIG_X86_64 2580c306bcfSIngo Molnar fxsave->rip = env->fip; 2590c306bcfSIngo Molnar fxsave->rdp = env->foo; 2600c306bcfSIngo Molnar /* cs and ds ignored */ 2610c306bcfSIngo Molnar #else 2620c306bcfSIngo Molnar fxsave->fip = env->fip; 2630c306bcfSIngo Molnar fxsave->fcs = (env->fcs & 0xffff); 2640c306bcfSIngo Molnar fxsave->foo = env->foo; 2650c306bcfSIngo Molnar fxsave->fos = env->fos; 2660c306bcfSIngo Molnar #endif 2670c306bcfSIngo Molnar 2680c306bcfSIngo Molnar for (i = 0; i < 8; ++i) 2690c306bcfSIngo Molnar memcpy(&to[i], &from[i], sizeof(from[0])); 2700c306bcfSIngo Molnar } 2710c306bcfSIngo Molnar 2720c306bcfSIngo Molnar int fpregs_get(struct task_struct *target, const struct user_regset *regset, 2730557d64dSAl Viro struct membuf to) 2740c306bcfSIngo Molnar { 2750c306bcfSIngo Molnar struct fpu *fpu = &target->thread.fpu; 2760c306bcfSIngo Molnar struct user_i387_ia32_struct env; 2770c306bcfSIngo Molnar 278369a036dSIngo Molnar fpu__prepare_read(fpu); 2790c306bcfSIngo Molnar 28078df526cSBorislav Petkov if (!boot_cpu_has(X86_FEATURE_FPU)) 2810557d64dSAl Viro return fpregs_soft_get(target, regset, to); 2820c306bcfSIngo Molnar 2830557d64dSAl Viro if (!boot_cpu_has(X86_FEATURE_FXSR)) { 2840557d64dSAl Viro return membuf_write(&to, &fpu->state.fsave, 2850557d64dSAl Viro sizeof(struct fregs_state)); 2860557d64dSAl Viro } 2870c306bcfSIngo Molnar 2880c306bcfSIngo Molnar fpstate_sanitize_xstate(fpu); 2890c306bcfSIngo Molnar 2900557d64dSAl Viro if (to.left == sizeof(env)) { 2910557d64dSAl Viro convert_from_fxsr(to.p, target); 2920c306bcfSIngo Molnar return 0; 2930c306bcfSIngo Molnar } 2940c306bcfSIngo Molnar 2950c306bcfSIngo Molnar convert_from_fxsr(&env, target); 2960557d64dSAl Viro return membuf_write(&to, &env, sizeof(env)); 2970c306bcfSIngo Molnar } 2980c306bcfSIngo Molnar 2990c306bcfSIngo Molnar int fpregs_set(struct task_struct *target, const struct user_regset *regset, 3000c306bcfSIngo Molnar unsigned int pos, unsigned int count, 3010c306bcfSIngo Molnar const void *kbuf, const void __user *ubuf) 3020c306bcfSIngo Molnar { 3030c306bcfSIngo Molnar struct fpu *fpu = &target->thread.fpu; 3040c306bcfSIngo Molnar struct user_i387_ia32_struct env; 3050c306bcfSIngo Molnar int ret; 3060c306bcfSIngo Molnar 307369a036dSIngo Molnar fpu__prepare_write(fpu); 3080c306bcfSIngo Molnar fpstate_sanitize_xstate(fpu); 3090c306bcfSIngo Molnar 31078df526cSBorislav Petkov if (!boot_cpu_has(X86_FEATURE_FPU)) 3110c306bcfSIngo Molnar return fpregs_soft_set(target, regset, pos, count, kbuf, ubuf); 3120c306bcfSIngo Molnar 31301f8fd73SBorislav Petkov if (!boot_cpu_has(X86_FEATURE_FXSR)) 3140c306bcfSIngo Molnar return user_regset_copyin(&pos, &count, &kbuf, &ubuf, 3150c306bcfSIngo Molnar &fpu->state.fsave, 0, 3160c306bcfSIngo Molnar -1); 3170c306bcfSIngo Molnar 3180c306bcfSIngo Molnar if (pos > 0 || count < sizeof(env)) 3190c306bcfSIngo Molnar convert_from_fxsr(&env, target); 3200c306bcfSIngo Molnar 3210c306bcfSIngo Molnar ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &env, 0, -1); 3220c306bcfSIngo Molnar if (!ret) 32339ea9bafSSebastian Andrzej Siewior convert_to_fxsr(&target->thread.fpu.state.fxsave, &env); 3240c306bcfSIngo Molnar 3250c306bcfSIngo Molnar /* 3260c306bcfSIngo Molnar * update the header bit in the xsave header, indicating the 3270c306bcfSIngo Molnar * presence of FP. 3280c306bcfSIngo Molnar */ 329d366bf7eSBorislav Petkov if (boot_cpu_has(X86_FEATURE_XSAVE)) 330d91cab78SDave Hansen fpu->state.xsave.header.xfeatures |= XFEATURE_MASK_FP; 3310c306bcfSIngo Molnar return ret; 3320c306bcfSIngo Molnar } 3330c306bcfSIngo Molnar 3340c306bcfSIngo Molnar #endif /* CONFIG_X86_32 || CONFIG_IA32_EMULATION */ 335