1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0 20c306bcfSIngo Molnar /* 30c306bcfSIngo Molnar * FPU register's regset abstraction, for ptrace, core dumps, etc. 40c306bcfSIngo Molnar */ 543be46e8SThomas Gleixner #include <linux/sched/task_stack.h> 643be46e8SThomas Gleixner #include <linux/vmalloc.h> 743be46e8SThomas Gleixner 80c306bcfSIngo Molnar #include <asm/fpu/internal.h> 90c306bcfSIngo Molnar #include <asm/fpu/signal.h> 100c306bcfSIngo Molnar #include <asm/fpu/regset.h> 1191c3dba7SYu-cheng Yu #include <asm/fpu/xstate.h> 120c306bcfSIngo Molnar 130c306bcfSIngo Molnar /* 140c306bcfSIngo Molnar * The xstateregs_active() routine is the same as the regset_fpregs_active() routine, 150c306bcfSIngo Molnar * as the "regset->n" for the xstate regset will be updated based on the feature 166a6256f9SAdam Buchbinder * capabilities supported by the xsave. 170c306bcfSIngo Molnar */ 180c306bcfSIngo Molnar int regset_fpregs_active(struct task_struct *target, const struct user_regset *regset) 190c306bcfSIngo Molnar { 202722146eSSebastian Andrzej Siewior return regset->n; 210c306bcfSIngo Molnar } 220c306bcfSIngo Molnar 230c306bcfSIngo Molnar int regset_xregset_fpregs_active(struct task_struct *target, const struct user_regset *regset) 240c306bcfSIngo Molnar { 252722146eSSebastian Andrzej Siewior if (boot_cpu_has(X86_FEATURE_FXSR)) 2601f8fd73SBorislav Petkov return regset->n; 2701f8fd73SBorislav Petkov else 2801f8fd73SBorislav Petkov return 0; 290c306bcfSIngo Molnar } 300c306bcfSIngo Molnar 310c306bcfSIngo Molnar int xfpregs_get(struct task_struct *target, const struct user_regset *regset, 320557d64dSAl Viro struct membuf to) 330c306bcfSIngo Molnar { 340c306bcfSIngo Molnar struct fpu *fpu = &target->thread.fpu; 350c306bcfSIngo Molnar 36adc997b3SThomas Gleixner if (!cpu_feature_enabled(X86_FEATURE_FXSR)) 370c306bcfSIngo Molnar return -ENODEV; 380c306bcfSIngo Molnar 39369a036dSIngo Molnar fpu__prepare_read(fpu); 400c306bcfSIngo Molnar 41adc997b3SThomas Gleixner if (!use_xsave()) { 42adc997b3SThomas Gleixner return membuf_write(&to, &fpu->state.fxsave, 43adc997b3SThomas Gleixner sizeof(fpu->state.fxsave)); 44adc997b3SThomas Gleixner } 45adc997b3SThomas Gleixner 46adc997b3SThomas Gleixner copy_xstate_to_uabi_buf(to, &fpu->state.xsave, XSTATE_COPY_FX); 47adc997b3SThomas Gleixner return 0; 480c306bcfSIngo Molnar } 490c306bcfSIngo Molnar 500c306bcfSIngo Molnar int xfpregs_set(struct task_struct *target, const struct user_regset *regset, 510c306bcfSIngo Molnar unsigned int pos, unsigned int count, 520c306bcfSIngo Molnar const void *kbuf, const void __user *ubuf) 530c306bcfSIngo Molnar { 540c306bcfSIngo Molnar struct fpu *fpu = &target->thread.fpu; 556164331dSAndy Lutomirski struct user32_fxsr_struct newstate; 560c306bcfSIngo Molnar int ret; 570c306bcfSIngo Molnar 586164331dSAndy Lutomirski BUILD_BUG_ON(sizeof(newstate) != sizeof(struct fxregs_state)); 596164331dSAndy Lutomirski 606164331dSAndy Lutomirski if (!cpu_feature_enabled(X86_FEATURE_FXSR)) 610c306bcfSIngo Molnar return -ENODEV; 620c306bcfSIngo Molnar 636164331dSAndy Lutomirski /* No funny business with partial or oversized writes is permitted. */ 646164331dSAndy Lutomirski if (pos != 0 || count != sizeof(newstate)) 656164331dSAndy Lutomirski return -EINVAL; 666164331dSAndy Lutomirski 676164331dSAndy Lutomirski ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &newstate, 0, -1); 686164331dSAndy Lutomirski if (ret) 696164331dSAndy Lutomirski return ret; 706164331dSAndy Lutomirski 71145e9e0dSAndy Lutomirski /* Do not allow an invalid MXCSR value. */ 72145e9e0dSAndy Lutomirski if (newstate.mxcsr & ~mxcsr_feature_mask) 73145e9e0dSAndy Lutomirski return -EINVAL; 746164331dSAndy Lutomirski 75369a036dSIngo Molnar fpu__prepare_write(fpu); 760c306bcfSIngo Molnar 776164331dSAndy Lutomirski /* Copy the state */ 786164331dSAndy Lutomirski memcpy(&fpu->state.fxsave, &newstate, sizeof(newstate)); 790c306bcfSIngo Molnar 806164331dSAndy Lutomirski /* Clear xmm8..15 */ 816164331dSAndy Lutomirski BUILD_BUG_ON(sizeof(fpu->state.fxsave.xmm_space) != 16 * 16); 826164331dSAndy Lutomirski memset(&fpu->state.fxsave.xmm_space[8], 0, 8 * 16); 830c306bcfSIngo Molnar 846164331dSAndy Lutomirski /* Mark FP and SSE as in use when XSAVE is enabled */ 856164331dSAndy Lutomirski if (use_xsave()) 86d91cab78SDave Hansen fpu->state.xsave.header.xfeatures |= XFEATURE_MASK_FPSSE; 870c306bcfSIngo Molnar 886164331dSAndy Lutomirski return 0; 890c306bcfSIngo Molnar } 900c306bcfSIngo Molnar 910c306bcfSIngo Molnar int xstateregs_get(struct task_struct *target, const struct user_regset *regset, 920557d64dSAl Viro struct membuf to) 930c306bcfSIngo Molnar { 940c306bcfSIngo Molnar struct fpu *fpu = &target->thread.fpu; 950c306bcfSIngo Molnar 963a335112SDave Hansen if (!cpu_feature_enabled(X86_FEATURE_XSAVE)) 970c306bcfSIngo Molnar return -ENODEV; 980c306bcfSIngo Molnar 99369a036dSIngo Molnar fpu__prepare_read(fpu); 10091c3dba7SYu-cheng Yu 101eb6f5172SThomas Gleixner copy_xstate_to_uabi_buf(to, &fpu->state.xsave, XSTATE_COPY_XSAVE); 1020557d64dSAl Viro return 0; 1030c306bcfSIngo Molnar } 1040c306bcfSIngo Molnar 1050c306bcfSIngo Molnar int xstateregs_set(struct task_struct *target, const struct user_regset *regset, 1060c306bcfSIngo Molnar unsigned int pos, unsigned int count, 1070c306bcfSIngo Molnar const void *kbuf, const void __user *ubuf) 1080c306bcfSIngo Molnar { 1090c306bcfSIngo Molnar struct fpu *fpu = &target->thread.fpu; 11043be46e8SThomas Gleixner struct xregs_state *tmpbuf = NULL; 1110c306bcfSIngo Molnar int ret; 1120c306bcfSIngo Molnar 11343be46e8SThomas Gleixner if (!cpu_feature_enabled(X86_FEATURE_XSAVE)) 1140c306bcfSIngo Molnar return -ENODEV; 1150c306bcfSIngo Molnar 11691c3dba7SYu-cheng Yu /* 11791c3dba7SYu-cheng Yu * A whole standard-format XSAVE buffer is needed: 11891c3dba7SYu-cheng Yu */ 11907d6688bSThomas Gleixner if (pos != 0 || count != fpu_user_xstate_size) 12091c3dba7SYu-cheng Yu return -EFAULT; 1210c306bcfSIngo Molnar 12243be46e8SThomas Gleixner if (!kbuf) { 12343be46e8SThomas Gleixner tmpbuf = vmalloc(count); 12443be46e8SThomas Gleixner if (!tmpbuf) 12543be46e8SThomas Gleixner return -ENOMEM; 1260c306bcfSIngo Molnar 12743be46e8SThomas Gleixner if (copy_from_user(tmpbuf, ubuf, count)) { 12843be46e8SThomas Gleixner ret = -EFAULT; 12943be46e8SThomas Gleixner goto out; 13043be46e8SThomas Gleixner } 13179fecc2bSIngo Molnar } 13291c3dba7SYu-cheng Yu 13343be46e8SThomas Gleixner fpu__prepare_write(fpu); 13443be46e8SThomas Gleixner ret = copy_kernel_to_xstate(&fpu->state.xsave, kbuf ?: tmpbuf); 135cf9df81bSEric Biggers 13643be46e8SThomas Gleixner out: 13743be46e8SThomas Gleixner vfree(tmpbuf); 1380c306bcfSIngo Molnar return ret; 1390c306bcfSIngo Molnar } 1400c306bcfSIngo Molnar 1410c306bcfSIngo Molnar #if defined CONFIG_X86_32 || defined CONFIG_IA32_EMULATION 1420c306bcfSIngo Molnar 1430c306bcfSIngo Molnar /* 1440c306bcfSIngo Molnar * FPU tag word conversions. 1450c306bcfSIngo Molnar */ 1460c306bcfSIngo Molnar 1470c306bcfSIngo Molnar static inline unsigned short twd_i387_to_fxsr(unsigned short twd) 1480c306bcfSIngo Molnar { 1490c306bcfSIngo Molnar unsigned int tmp; /* to avoid 16 bit prefixes in the code */ 1500c306bcfSIngo Molnar 1510c306bcfSIngo Molnar /* Transform each pair of bits into 01 (valid) or 00 (empty) */ 1520c306bcfSIngo Molnar tmp = ~twd; 1530c306bcfSIngo Molnar tmp = (tmp | (tmp>>1)) & 0x5555; /* 0V0V0V0V0V0V0V0V */ 1540c306bcfSIngo Molnar /* and move the valid bits to the lower byte. */ 1550c306bcfSIngo Molnar tmp = (tmp | (tmp >> 1)) & 0x3333; /* 00VV00VV00VV00VV */ 1560c306bcfSIngo Molnar tmp = (tmp | (tmp >> 2)) & 0x0f0f; /* 0000VVVV0000VVVV */ 1570c306bcfSIngo Molnar tmp = (tmp | (tmp >> 4)) & 0x00ff; /* 00000000VVVVVVVV */ 1580c306bcfSIngo Molnar 1590c306bcfSIngo Molnar return tmp; 1600c306bcfSIngo Molnar } 1610c306bcfSIngo Molnar 1620c306bcfSIngo Molnar #define FPREG_ADDR(f, n) ((void *)&(f)->st_space + (n) * 16) 1630c306bcfSIngo Molnar #define FP_EXP_TAG_VALID 0 1640c306bcfSIngo Molnar #define FP_EXP_TAG_ZERO 1 1650c306bcfSIngo Molnar #define FP_EXP_TAG_SPECIAL 2 1660c306bcfSIngo Molnar #define FP_EXP_TAG_EMPTY 3 1670c306bcfSIngo Molnar 168c47ada30SIngo Molnar static inline u32 twd_fxsr_to_i387(struct fxregs_state *fxsave) 1690c306bcfSIngo Molnar { 1700c306bcfSIngo Molnar struct _fpxreg *st; 1710c306bcfSIngo Molnar u32 tos = (fxsave->swd >> 11) & 7; 1720c306bcfSIngo Molnar u32 twd = (unsigned long) fxsave->twd; 1730c306bcfSIngo Molnar u32 tag; 1740c306bcfSIngo Molnar u32 ret = 0xffff0000u; 1750c306bcfSIngo Molnar int i; 1760c306bcfSIngo Molnar 1770c306bcfSIngo Molnar for (i = 0; i < 8; i++, twd >>= 1) { 1780c306bcfSIngo Molnar if (twd & 0x1) { 1790c306bcfSIngo Molnar st = FPREG_ADDR(fxsave, (i - tos) & 7); 1800c306bcfSIngo Molnar 1810c306bcfSIngo Molnar switch (st->exponent & 0x7fff) { 1820c306bcfSIngo Molnar case 0x7fff: 1830c306bcfSIngo Molnar tag = FP_EXP_TAG_SPECIAL; 1840c306bcfSIngo Molnar break; 1850c306bcfSIngo Molnar case 0x0000: 1860c306bcfSIngo Molnar if (!st->significand[0] && 1870c306bcfSIngo Molnar !st->significand[1] && 1880c306bcfSIngo Molnar !st->significand[2] && 1890c306bcfSIngo Molnar !st->significand[3]) 1900c306bcfSIngo Molnar tag = FP_EXP_TAG_ZERO; 1910c306bcfSIngo Molnar else 1920c306bcfSIngo Molnar tag = FP_EXP_TAG_SPECIAL; 1930c306bcfSIngo Molnar break; 1940c306bcfSIngo Molnar default: 1950c306bcfSIngo Molnar if (st->significand[3] & 0x8000) 1960c306bcfSIngo Molnar tag = FP_EXP_TAG_VALID; 1970c306bcfSIngo Molnar else 1980c306bcfSIngo Molnar tag = FP_EXP_TAG_SPECIAL; 1990c306bcfSIngo Molnar break; 2000c306bcfSIngo Molnar } 2010c306bcfSIngo Molnar } else { 2020c306bcfSIngo Molnar tag = FP_EXP_TAG_EMPTY; 2030c306bcfSIngo Molnar } 2040c306bcfSIngo Molnar ret |= tag << (2 * i); 2050c306bcfSIngo Molnar } 2060c306bcfSIngo Molnar return ret; 2070c306bcfSIngo Molnar } 2080c306bcfSIngo Molnar 2090c306bcfSIngo Molnar /* 2100c306bcfSIngo Molnar * FXSR floating point environment conversions. 2110c306bcfSIngo Molnar */ 2120c306bcfSIngo Molnar 213*3f7f7563SThomas Gleixner static void __convert_from_fxsr(struct user_i387_ia32_struct *env, 214*3f7f7563SThomas Gleixner struct task_struct *tsk, 215*3f7f7563SThomas Gleixner struct fxregs_state *fxsave) 2160c306bcfSIngo Molnar { 2170c306bcfSIngo Molnar struct _fpreg *to = (struct _fpreg *) &env->st_space[0]; 2180c306bcfSIngo Molnar struct _fpxreg *from = (struct _fpxreg *) &fxsave->st_space[0]; 2190c306bcfSIngo Molnar int i; 2200c306bcfSIngo Molnar 2210c306bcfSIngo Molnar env->cwd = fxsave->cwd | 0xffff0000u; 2220c306bcfSIngo Molnar env->swd = fxsave->swd | 0xffff0000u; 2230c306bcfSIngo Molnar env->twd = twd_fxsr_to_i387(fxsave); 2240c306bcfSIngo Molnar 2250c306bcfSIngo Molnar #ifdef CONFIG_X86_64 2260c306bcfSIngo Molnar env->fip = fxsave->rip; 2270c306bcfSIngo Molnar env->foo = fxsave->rdp; 2280c306bcfSIngo Molnar /* 2290c306bcfSIngo Molnar * should be actually ds/cs at fpu exception time, but 2300c306bcfSIngo Molnar * that information is not available in 64bit mode. 2310c306bcfSIngo Molnar */ 2320c306bcfSIngo Molnar env->fcs = task_pt_regs(tsk)->cs; 2330c306bcfSIngo Molnar if (tsk == current) { 2340c306bcfSIngo Molnar savesegment(ds, env->fos); 2350c306bcfSIngo Molnar } else { 2360c306bcfSIngo Molnar env->fos = tsk->thread.ds; 2370c306bcfSIngo Molnar } 2380c306bcfSIngo Molnar env->fos |= 0xffff0000; 2390c306bcfSIngo Molnar #else 2400c306bcfSIngo Molnar env->fip = fxsave->fip; 2410c306bcfSIngo Molnar env->fcs = (u16) fxsave->fcs | ((u32) fxsave->fop << 16); 2420c306bcfSIngo Molnar env->foo = fxsave->foo; 2430c306bcfSIngo Molnar env->fos = fxsave->fos; 2440c306bcfSIngo Molnar #endif 2450c306bcfSIngo Molnar 2460c306bcfSIngo Molnar for (i = 0; i < 8; ++i) 2470c306bcfSIngo Molnar memcpy(&to[i], &from[i], sizeof(to[0])); 2480c306bcfSIngo Molnar } 2490c306bcfSIngo Molnar 250*3f7f7563SThomas Gleixner void 251*3f7f7563SThomas Gleixner convert_from_fxsr(struct user_i387_ia32_struct *env, struct task_struct *tsk) 252*3f7f7563SThomas Gleixner { 253*3f7f7563SThomas Gleixner __convert_from_fxsr(env, tsk, &tsk->thread.fpu.state.fxsave); 254*3f7f7563SThomas Gleixner } 255*3f7f7563SThomas Gleixner 25639ea9bafSSebastian Andrzej Siewior void convert_to_fxsr(struct fxregs_state *fxsave, 2570c306bcfSIngo Molnar const struct user_i387_ia32_struct *env) 2580c306bcfSIngo Molnar 2590c306bcfSIngo Molnar { 2600c306bcfSIngo Molnar struct _fpreg *from = (struct _fpreg *) &env->st_space[0]; 2610c306bcfSIngo Molnar struct _fpxreg *to = (struct _fpxreg *) &fxsave->st_space[0]; 2620c306bcfSIngo Molnar int i; 2630c306bcfSIngo Molnar 2640c306bcfSIngo Molnar fxsave->cwd = env->cwd; 2650c306bcfSIngo Molnar fxsave->swd = env->swd; 2660c306bcfSIngo Molnar fxsave->twd = twd_i387_to_fxsr(env->twd); 2670c306bcfSIngo Molnar fxsave->fop = (u16) ((u32) env->fcs >> 16); 2680c306bcfSIngo Molnar #ifdef CONFIG_X86_64 2690c306bcfSIngo Molnar fxsave->rip = env->fip; 2700c306bcfSIngo Molnar fxsave->rdp = env->foo; 2710c306bcfSIngo Molnar /* cs and ds ignored */ 2720c306bcfSIngo Molnar #else 2730c306bcfSIngo Molnar fxsave->fip = env->fip; 2740c306bcfSIngo Molnar fxsave->fcs = (env->fcs & 0xffff); 2750c306bcfSIngo Molnar fxsave->foo = env->foo; 2760c306bcfSIngo Molnar fxsave->fos = env->fos; 2770c306bcfSIngo Molnar #endif 2780c306bcfSIngo Molnar 2790c306bcfSIngo Molnar for (i = 0; i < 8; ++i) 2800c306bcfSIngo Molnar memcpy(&to[i], &from[i], sizeof(from[0])); 2810c306bcfSIngo Molnar } 2820c306bcfSIngo Molnar 2830c306bcfSIngo Molnar int fpregs_get(struct task_struct *target, const struct user_regset *regset, 2840557d64dSAl Viro struct membuf to) 2850c306bcfSIngo Molnar { 2860c306bcfSIngo Molnar struct fpu *fpu = &target->thread.fpu; 2870c306bcfSIngo Molnar struct user_i387_ia32_struct env; 288*3f7f7563SThomas Gleixner struct fxregs_state fxsave, *fx; 2890c306bcfSIngo Molnar 290369a036dSIngo Molnar fpu__prepare_read(fpu); 2910c306bcfSIngo Molnar 292*3f7f7563SThomas Gleixner if (!cpu_feature_enabled(X86_FEATURE_FPU)) 2930557d64dSAl Viro return fpregs_soft_get(target, regset, to); 2940c306bcfSIngo Molnar 295*3f7f7563SThomas Gleixner if (!cpu_feature_enabled(X86_FEATURE_FXSR)) { 2960557d64dSAl Viro return membuf_write(&to, &fpu->state.fsave, 2970557d64dSAl Viro sizeof(struct fregs_state)); 2980557d64dSAl Viro } 2990c306bcfSIngo Molnar 300*3f7f7563SThomas Gleixner if (use_xsave()) { 301*3f7f7563SThomas Gleixner struct membuf mb = { .p = &fxsave, .left = sizeof(fxsave) }; 3020c306bcfSIngo Molnar 303*3f7f7563SThomas Gleixner /* Handle init state optimized xstate correctly */ 304*3f7f7563SThomas Gleixner copy_xstate_to_uabi_buf(mb, &fpu->state.xsave, XSTATE_COPY_FP); 305*3f7f7563SThomas Gleixner fx = &fxsave; 306*3f7f7563SThomas Gleixner } else { 307*3f7f7563SThomas Gleixner fx = &fpu->state.fxsave; 3080c306bcfSIngo Molnar } 3090c306bcfSIngo Molnar 310*3f7f7563SThomas Gleixner __convert_from_fxsr(&env, target, fx); 3110557d64dSAl Viro return membuf_write(&to, &env, sizeof(env)); 3120c306bcfSIngo Molnar } 3130c306bcfSIngo Molnar 3140c306bcfSIngo Molnar int fpregs_set(struct task_struct *target, const struct user_regset *regset, 3150c306bcfSIngo Molnar unsigned int pos, unsigned int count, 3160c306bcfSIngo Molnar const void *kbuf, const void __user *ubuf) 3170c306bcfSIngo Molnar { 3180c306bcfSIngo Molnar struct fpu *fpu = &target->thread.fpu; 3190c306bcfSIngo Molnar struct user_i387_ia32_struct env; 3200c306bcfSIngo Molnar int ret; 3210c306bcfSIngo Molnar 322da53f60bSAndy Lutomirski /* No funny business with partial or oversized writes is permitted. */ 323da53f60bSAndy Lutomirski if (pos != 0 || count != sizeof(struct user_i387_ia32_struct)) 324da53f60bSAndy Lutomirski return -EINVAL; 3250c306bcfSIngo Molnar 326da53f60bSAndy Lutomirski if (!cpu_feature_enabled(X86_FEATURE_FPU)) 3270c306bcfSIngo Molnar return fpregs_soft_set(target, regset, pos, count, kbuf, ubuf); 3280c306bcfSIngo Molnar 3290c306bcfSIngo Molnar ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &env, 0, -1); 330da53f60bSAndy Lutomirski if (ret) 331da53f60bSAndy Lutomirski return ret; 332da53f60bSAndy Lutomirski 333da53f60bSAndy Lutomirski fpu__prepare_write(fpu); 334da53f60bSAndy Lutomirski 335da53f60bSAndy Lutomirski if (cpu_feature_enabled(X86_FEATURE_FXSR)) 336da53f60bSAndy Lutomirski convert_to_fxsr(&fpu->state.fxsave, &env); 337da53f60bSAndy Lutomirski else 338da53f60bSAndy Lutomirski memcpy(&fpu->state.fsave, &env, sizeof(env)); 3390c306bcfSIngo Molnar 3400c306bcfSIngo Molnar /* 341da53f60bSAndy Lutomirski * Update the header bit in the xsave header, indicating the 3420c306bcfSIngo Molnar * presence of FP. 3430c306bcfSIngo Molnar */ 344da53f60bSAndy Lutomirski if (cpu_feature_enabled(X86_FEATURE_XSAVE)) 345d91cab78SDave Hansen fpu->state.xsave.header.xfeatures |= XFEATURE_MASK_FP; 346da53f60bSAndy Lutomirski 347da53f60bSAndy Lutomirski return 0; 3480c306bcfSIngo Molnar } 3490c306bcfSIngo Molnar 3500c306bcfSIngo Molnar #endif /* CONFIG_X86_32 || CONFIG_IA32_EMULATION */ 351