1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * FPU register's regset abstraction, for ptrace, core dumps, etc. 4 */ 5 #include <linux/sched/task_stack.h> 6 #include <linux/vmalloc.h> 7 8 #include <asm/fpu/internal.h> 9 #include <asm/fpu/signal.h> 10 #include <asm/fpu/regset.h> 11 #include <asm/fpu/xstate.h> 12 13 /* 14 * The xstateregs_active() routine is the same as the regset_fpregs_active() routine, 15 * as the "regset->n" for the xstate regset will be updated based on the feature 16 * capabilities supported by the xsave. 17 */ 18 int regset_fpregs_active(struct task_struct *target, const struct user_regset *regset) 19 { 20 return regset->n; 21 } 22 23 int regset_xregset_fpregs_active(struct task_struct *target, const struct user_regset *regset) 24 { 25 if (boot_cpu_has(X86_FEATURE_FXSR)) 26 return regset->n; 27 else 28 return 0; 29 } 30 31 /* 32 * The regset get() functions are invoked from: 33 * 34 * - coredump to dump the current task's fpstate. If the current task 35 * owns the FPU then the memory state has to be synchronized and the 36 * FPU register state preserved. Otherwise fpstate is already in sync. 37 * 38 * - ptrace to dump fpstate of a stopped task, in which case the registers 39 * have already been saved to fpstate on context switch. 40 */ 41 static void sync_fpstate(struct fpu *fpu) 42 { 43 if (fpu == ¤t->thread.fpu) 44 fpu__save(fpu); 45 } 46 47 /* 48 * Invalidate cached FPU registers before modifying the stopped target 49 * task's fpstate. 50 * 51 * This forces the target task on resume to restore the FPU registers from 52 * modified fpstate. Otherwise the task might skip the restore and operate 53 * with the cached FPU registers which discards the modifications. 54 */ 55 static void fpu_force_restore(struct fpu *fpu) 56 { 57 /* 58 * Only stopped child tasks can be used to modify the FPU 59 * state in the fpstate buffer: 60 */ 61 WARN_ON_FPU(fpu == ¤t->thread.fpu); 62 63 __fpu_invalidate_fpregs_state(fpu); 64 } 65 66 int xfpregs_get(struct task_struct *target, const struct user_regset *regset, 67 struct membuf to) 68 { 69 struct fpu *fpu = &target->thread.fpu; 70 71 if (!cpu_feature_enabled(X86_FEATURE_FXSR)) 72 return -ENODEV; 73 74 sync_fpstate(fpu); 75 76 if (!use_xsave()) { 77 return membuf_write(&to, &fpu->state.fxsave, 78 sizeof(fpu->state.fxsave)); 79 } 80 81 copy_xstate_to_uabi_buf(to, &fpu->state.xsave, XSTATE_COPY_FX); 82 return 0; 83 } 84 85 int xfpregs_set(struct task_struct *target, const struct user_regset *regset, 86 unsigned int pos, unsigned int count, 87 const void *kbuf, const void __user *ubuf) 88 { 89 struct fpu *fpu = &target->thread.fpu; 90 struct user32_fxsr_struct newstate; 91 int ret; 92 93 BUILD_BUG_ON(sizeof(newstate) != sizeof(struct fxregs_state)); 94 95 if (!cpu_feature_enabled(X86_FEATURE_FXSR)) 96 return -ENODEV; 97 98 /* No funny business with partial or oversized writes is permitted. */ 99 if (pos != 0 || count != sizeof(newstate)) 100 return -EINVAL; 101 102 ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &newstate, 0, -1); 103 if (ret) 104 return ret; 105 106 /* Do not allow an invalid MXCSR value. */ 107 if (newstate.mxcsr & ~mxcsr_feature_mask) 108 return -EINVAL; 109 110 fpu_force_restore(fpu); 111 112 /* Copy the state */ 113 memcpy(&fpu->state.fxsave, &newstate, sizeof(newstate)); 114 115 /* Clear xmm8..15 */ 116 BUILD_BUG_ON(sizeof(fpu->state.fxsave.xmm_space) != 16 * 16); 117 memset(&fpu->state.fxsave.xmm_space[8], 0, 8 * 16); 118 119 /* Mark FP and SSE as in use when XSAVE is enabled */ 120 if (use_xsave()) 121 fpu->state.xsave.header.xfeatures |= XFEATURE_MASK_FPSSE; 122 123 return 0; 124 } 125 126 int xstateregs_get(struct task_struct *target, const struct user_regset *regset, 127 struct membuf to) 128 { 129 struct fpu *fpu = &target->thread.fpu; 130 131 if (!cpu_feature_enabled(X86_FEATURE_XSAVE)) 132 return -ENODEV; 133 134 sync_fpstate(fpu); 135 136 copy_xstate_to_uabi_buf(to, &fpu->state.xsave, XSTATE_COPY_XSAVE); 137 return 0; 138 } 139 140 int xstateregs_set(struct task_struct *target, const struct user_regset *regset, 141 unsigned int pos, unsigned int count, 142 const void *kbuf, const void __user *ubuf) 143 { 144 struct fpu *fpu = &target->thread.fpu; 145 struct xregs_state *tmpbuf = NULL; 146 int ret; 147 148 if (!cpu_feature_enabled(X86_FEATURE_XSAVE)) 149 return -ENODEV; 150 151 /* 152 * A whole standard-format XSAVE buffer is needed: 153 */ 154 if (pos != 0 || count != fpu_user_xstate_size) 155 return -EFAULT; 156 157 if (!kbuf) { 158 tmpbuf = vmalloc(count); 159 if (!tmpbuf) 160 return -ENOMEM; 161 162 if (copy_from_user(tmpbuf, ubuf, count)) { 163 ret = -EFAULT; 164 goto out; 165 } 166 } 167 168 fpu_force_restore(fpu); 169 ret = copy_kernel_to_xstate(&fpu->state.xsave, kbuf ?: tmpbuf); 170 171 out: 172 vfree(tmpbuf); 173 return ret; 174 } 175 176 #if defined CONFIG_X86_32 || defined CONFIG_IA32_EMULATION 177 178 /* 179 * FPU tag word conversions. 180 */ 181 182 static inline unsigned short twd_i387_to_fxsr(unsigned short twd) 183 { 184 unsigned int tmp; /* to avoid 16 bit prefixes in the code */ 185 186 /* Transform each pair of bits into 01 (valid) or 00 (empty) */ 187 tmp = ~twd; 188 tmp = (tmp | (tmp>>1)) & 0x5555; /* 0V0V0V0V0V0V0V0V */ 189 /* and move the valid bits to the lower byte. */ 190 tmp = (tmp | (tmp >> 1)) & 0x3333; /* 00VV00VV00VV00VV */ 191 tmp = (tmp | (tmp >> 2)) & 0x0f0f; /* 0000VVVV0000VVVV */ 192 tmp = (tmp | (tmp >> 4)) & 0x00ff; /* 00000000VVVVVVVV */ 193 194 return tmp; 195 } 196 197 #define FPREG_ADDR(f, n) ((void *)&(f)->st_space + (n) * 16) 198 #define FP_EXP_TAG_VALID 0 199 #define FP_EXP_TAG_ZERO 1 200 #define FP_EXP_TAG_SPECIAL 2 201 #define FP_EXP_TAG_EMPTY 3 202 203 static inline u32 twd_fxsr_to_i387(struct fxregs_state *fxsave) 204 { 205 struct _fpxreg *st; 206 u32 tos = (fxsave->swd >> 11) & 7; 207 u32 twd = (unsigned long) fxsave->twd; 208 u32 tag; 209 u32 ret = 0xffff0000u; 210 int i; 211 212 for (i = 0; i < 8; i++, twd >>= 1) { 213 if (twd & 0x1) { 214 st = FPREG_ADDR(fxsave, (i - tos) & 7); 215 216 switch (st->exponent & 0x7fff) { 217 case 0x7fff: 218 tag = FP_EXP_TAG_SPECIAL; 219 break; 220 case 0x0000: 221 if (!st->significand[0] && 222 !st->significand[1] && 223 !st->significand[2] && 224 !st->significand[3]) 225 tag = FP_EXP_TAG_ZERO; 226 else 227 tag = FP_EXP_TAG_SPECIAL; 228 break; 229 default: 230 if (st->significand[3] & 0x8000) 231 tag = FP_EXP_TAG_VALID; 232 else 233 tag = FP_EXP_TAG_SPECIAL; 234 break; 235 } 236 } else { 237 tag = FP_EXP_TAG_EMPTY; 238 } 239 ret |= tag << (2 * i); 240 } 241 return ret; 242 } 243 244 /* 245 * FXSR floating point environment conversions. 246 */ 247 248 static void __convert_from_fxsr(struct user_i387_ia32_struct *env, 249 struct task_struct *tsk, 250 struct fxregs_state *fxsave) 251 { 252 struct _fpreg *to = (struct _fpreg *) &env->st_space[0]; 253 struct _fpxreg *from = (struct _fpxreg *) &fxsave->st_space[0]; 254 int i; 255 256 env->cwd = fxsave->cwd | 0xffff0000u; 257 env->swd = fxsave->swd | 0xffff0000u; 258 env->twd = twd_fxsr_to_i387(fxsave); 259 260 #ifdef CONFIG_X86_64 261 env->fip = fxsave->rip; 262 env->foo = fxsave->rdp; 263 /* 264 * should be actually ds/cs at fpu exception time, but 265 * that information is not available in 64bit mode. 266 */ 267 env->fcs = task_pt_regs(tsk)->cs; 268 if (tsk == current) { 269 savesegment(ds, env->fos); 270 } else { 271 env->fos = tsk->thread.ds; 272 } 273 env->fos |= 0xffff0000; 274 #else 275 env->fip = fxsave->fip; 276 env->fcs = (u16) fxsave->fcs | ((u32) fxsave->fop << 16); 277 env->foo = fxsave->foo; 278 env->fos = fxsave->fos; 279 #endif 280 281 for (i = 0; i < 8; ++i) 282 memcpy(&to[i], &from[i], sizeof(to[0])); 283 } 284 285 void 286 convert_from_fxsr(struct user_i387_ia32_struct *env, struct task_struct *tsk) 287 { 288 __convert_from_fxsr(env, tsk, &tsk->thread.fpu.state.fxsave); 289 } 290 291 void convert_to_fxsr(struct fxregs_state *fxsave, 292 const struct user_i387_ia32_struct *env) 293 294 { 295 struct _fpreg *from = (struct _fpreg *) &env->st_space[0]; 296 struct _fpxreg *to = (struct _fpxreg *) &fxsave->st_space[0]; 297 int i; 298 299 fxsave->cwd = env->cwd; 300 fxsave->swd = env->swd; 301 fxsave->twd = twd_i387_to_fxsr(env->twd); 302 fxsave->fop = (u16) ((u32) env->fcs >> 16); 303 #ifdef CONFIG_X86_64 304 fxsave->rip = env->fip; 305 fxsave->rdp = env->foo; 306 /* cs and ds ignored */ 307 #else 308 fxsave->fip = env->fip; 309 fxsave->fcs = (env->fcs & 0xffff); 310 fxsave->foo = env->foo; 311 fxsave->fos = env->fos; 312 #endif 313 314 for (i = 0; i < 8; ++i) 315 memcpy(&to[i], &from[i], sizeof(from[0])); 316 } 317 318 int fpregs_get(struct task_struct *target, const struct user_regset *regset, 319 struct membuf to) 320 { 321 struct fpu *fpu = &target->thread.fpu; 322 struct user_i387_ia32_struct env; 323 struct fxregs_state fxsave, *fx; 324 325 sync_fpstate(fpu); 326 327 if (!cpu_feature_enabled(X86_FEATURE_FPU)) 328 return fpregs_soft_get(target, regset, to); 329 330 if (!cpu_feature_enabled(X86_FEATURE_FXSR)) { 331 return membuf_write(&to, &fpu->state.fsave, 332 sizeof(struct fregs_state)); 333 } 334 335 if (use_xsave()) { 336 struct membuf mb = { .p = &fxsave, .left = sizeof(fxsave) }; 337 338 /* Handle init state optimized xstate correctly */ 339 copy_xstate_to_uabi_buf(mb, &fpu->state.xsave, XSTATE_COPY_FP); 340 fx = &fxsave; 341 } else { 342 fx = &fpu->state.fxsave; 343 } 344 345 __convert_from_fxsr(&env, target, fx); 346 return membuf_write(&to, &env, sizeof(env)); 347 } 348 349 int fpregs_set(struct task_struct *target, const struct user_regset *regset, 350 unsigned int pos, unsigned int count, 351 const void *kbuf, const void __user *ubuf) 352 { 353 struct fpu *fpu = &target->thread.fpu; 354 struct user_i387_ia32_struct env; 355 int ret; 356 357 /* No funny business with partial or oversized writes is permitted. */ 358 if (pos != 0 || count != sizeof(struct user_i387_ia32_struct)) 359 return -EINVAL; 360 361 if (!cpu_feature_enabled(X86_FEATURE_FPU)) 362 return fpregs_soft_set(target, regset, pos, count, kbuf, ubuf); 363 364 ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &env, 0, -1); 365 if (ret) 366 return ret; 367 368 fpu_force_restore(fpu); 369 370 if (cpu_feature_enabled(X86_FEATURE_FXSR)) 371 convert_to_fxsr(&fpu->state.fxsave, &env); 372 else 373 memcpy(&fpu->state.fsave, &env, sizeof(env)); 374 375 /* 376 * Update the header bit in the xsave header, indicating the 377 * presence of FP. 378 */ 379 if (cpu_feature_enabled(X86_FEATURE_XSAVE)) 380 fpu->state.xsave.header.xfeatures |= XFEATURE_MASK_FP; 381 382 return 0; 383 } 384 385 #endif /* CONFIG_X86_32 || CONFIG_IA32_EMULATION */ 386