1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 1994 Linus Torvalds 4 * 5 * Pentium III FXSR, SSE support 6 * General FPU state handling cleanups 7 * Gareth Hughes <gareth@valinux.com>, May 2000 8 */ 9 #include <asm/fpu/internal.h> 10 #include <asm/fpu/regset.h> 11 #include <asm/fpu/signal.h> 12 #include <asm/fpu/types.h> 13 #include <asm/traps.h> 14 #include <asm/irq_regs.h> 15 16 #include <linux/hardirq.h> 17 #include <linux/pkeys.h> 18 19 #define CREATE_TRACE_POINTS 20 #include <asm/trace/fpu.h> 21 22 /* 23 * Represents the initial FPU state. It's mostly (but not completely) zeroes, 24 * depending on the FPU hardware format: 25 */ 26 union fpregs_state init_fpstate __read_mostly; 27 28 /* 29 * Track whether the kernel is using the FPU state 30 * currently. 31 * 32 * This flag is used: 33 * 34 * - by IRQ context code to potentially use the FPU 35 * if it's unused. 36 * 37 * - to debug kernel_fpu_begin()/end() correctness 38 */ 39 static DEFINE_PER_CPU(bool, in_kernel_fpu); 40 41 /* 42 * Track which context is using the FPU on the CPU: 43 */ 44 DEFINE_PER_CPU(struct fpu *, fpu_fpregs_owner_ctx); 45 46 static bool kernel_fpu_disabled(void) 47 { 48 return this_cpu_read(in_kernel_fpu); 49 } 50 51 static bool interrupted_kernel_fpu_idle(void) 52 { 53 return !kernel_fpu_disabled(); 54 } 55 56 /* 57 * Were we in user mode (or vm86 mode) when we were 58 * interrupted? 59 * 60 * Doing kernel_fpu_begin/end() is ok if we are running 61 * in an interrupt context from user mode - we'll just 62 * save the FPU state as required. 63 */ 64 static bool interrupted_user_mode(void) 65 { 66 struct pt_regs *regs = get_irq_regs(); 67 return regs && user_mode(regs); 68 } 69 70 /* 71 * Can we use the FPU in kernel mode with the 72 * whole "kernel_fpu_begin/end()" sequence? 73 * 74 * It's always ok in process context (ie "not interrupt") 75 * but it is sometimes ok even from an irq. 76 */ 77 bool irq_fpu_usable(void) 78 { 79 return !in_interrupt() || 80 interrupted_user_mode() || 81 interrupted_kernel_fpu_idle(); 82 } 83 EXPORT_SYMBOL(irq_fpu_usable); 84 85 void kernel_fpu_begin(void) 86 { 87 preempt_disable(); 88 89 WARN_ON_FPU(!irq_fpu_usable()); 90 WARN_ON_FPU(this_cpu_read(in_kernel_fpu)); 91 92 this_cpu_write(in_kernel_fpu, true); 93 94 if (!(current->flags & PF_KTHREAD) && 95 !test_thread_flag(TIF_NEED_FPU_LOAD)) { 96 set_thread_flag(TIF_NEED_FPU_LOAD); 97 /* 98 * Ignore return value -- we don't care if reg state 99 * is clobbered. 100 */ 101 copy_fpregs_to_fpstate(¤t->thread.fpu); 102 } 103 __cpu_invalidate_fpregs_state(); 104 105 if (boot_cpu_has(X86_FEATURE_XMM)) 106 ldmxcsr(MXCSR_DEFAULT); 107 108 if (boot_cpu_has(X86_FEATURE_FPU)) 109 asm volatile ("fninit"); 110 } 111 EXPORT_SYMBOL_GPL(kernel_fpu_begin); 112 113 void kernel_fpu_end(void) 114 { 115 WARN_ON_FPU(!this_cpu_read(in_kernel_fpu)); 116 117 this_cpu_write(in_kernel_fpu, false); 118 preempt_enable(); 119 } 120 EXPORT_SYMBOL_GPL(kernel_fpu_end); 121 122 /* 123 * Save the FPU state (mark it for reload if necessary): 124 * 125 * This only ever gets called for the current task. 126 */ 127 void fpu__save(struct fpu *fpu) 128 { 129 WARN_ON_FPU(fpu != ¤t->thread.fpu); 130 131 fpregs_lock(); 132 trace_x86_fpu_before_save(fpu); 133 134 if (!test_thread_flag(TIF_NEED_FPU_LOAD)) { 135 if (!copy_fpregs_to_fpstate(fpu)) { 136 copy_kernel_to_fpregs(&fpu->state); 137 } 138 } 139 140 trace_x86_fpu_after_save(fpu); 141 fpregs_unlock(); 142 } 143 144 /* 145 * Legacy x87 fpstate state init: 146 */ 147 static inline void fpstate_init_fstate(struct fregs_state *fp) 148 { 149 fp->cwd = 0xffff037fu; 150 fp->swd = 0xffff0000u; 151 fp->twd = 0xffffffffu; 152 fp->fos = 0xffff0000u; 153 } 154 155 void fpstate_init(union fpregs_state *state) 156 { 157 if (!static_cpu_has(X86_FEATURE_FPU)) { 158 fpstate_init_soft(&state->soft); 159 return; 160 } 161 162 memset(state, 0, fpu_kernel_xstate_size); 163 164 if (static_cpu_has(X86_FEATURE_XSAVES)) 165 fpstate_init_xstate(&state->xsave); 166 if (static_cpu_has(X86_FEATURE_FXSR)) 167 fpstate_init_fxstate(&state->fxsave); 168 else 169 fpstate_init_fstate(&state->fsave); 170 } 171 EXPORT_SYMBOL_GPL(fpstate_init); 172 173 int fpu__copy(struct task_struct *dst, struct task_struct *src) 174 { 175 struct fpu *dst_fpu = &dst->thread.fpu; 176 struct fpu *src_fpu = &src->thread.fpu; 177 178 dst_fpu->last_cpu = -1; 179 180 if (!static_cpu_has(X86_FEATURE_FPU)) 181 return 0; 182 183 WARN_ON_FPU(src_fpu != ¤t->thread.fpu); 184 185 /* 186 * Don't let 'init optimized' areas of the XSAVE area 187 * leak into the child task: 188 */ 189 memset(&dst_fpu->state.xsave, 0, fpu_kernel_xstate_size); 190 191 /* 192 * If the FPU registers are not current just memcpy() the state. 193 * Otherwise save current FPU registers directly into the child's FPU 194 * context, without any memory-to-memory copying. 195 * 196 * ( The function 'fails' in the FNSAVE case, which destroys 197 * register contents so we have to load them back. ) 198 */ 199 fpregs_lock(); 200 if (test_thread_flag(TIF_NEED_FPU_LOAD)) 201 memcpy(&dst_fpu->state, &src_fpu->state, fpu_kernel_xstate_size); 202 203 else if (!copy_fpregs_to_fpstate(dst_fpu)) 204 copy_kernel_to_fpregs(&dst_fpu->state); 205 206 fpregs_unlock(); 207 208 set_tsk_thread_flag(dst, TIF_NEED_FPU_LOAD); 209 210 trace_x86_fpu_copy_src(src_fpu); 211 trace_x86_fpu_copy_dst(dst_fpu); 212 213 return 0; 214 } 215 216 /* 217 * Activate the current task's in-memory FPU context, 218 * if it has not been used before: 219 */ 220 static void fpu__initialize(struct fpu *fpu) 221 { 222 WARN_ON_FPU(fpu != ¤t->thread.fpu); 223 224 set_thread_flag(TIF_NEED_FPU_LOAD); 225 fpstate_init(&fpu->state); 226 trace_x86_fpu_init_state(fpu); 227 } 228 229 /* 230 * This function must be called before we read a task's fpstate. 231 * 232 * There's two cases where this gets called: 233 * 234 * - for the current task (when coredumping), in which case we have 235 * to save the latest FPU registers into the fpstate, 236 * 237 * - or it's called for stopped tasks (ptrace), in which case the 238 * registers were already saved by the context-switch code when 239 * the task scheduled out. 240 * 241 * If the task has used the FPU before then save it. 242 */ 243 void fpu__prepare_read(struct fpu *fpu) 244 { 245 if (fpu == ¤t->thread.fpu) 246 fpu__save(fpu); 247 } 248 249 /* 250 * This function must be called before we write a task's fpstate. 251 * 252 * Invalidate any cached FPU registers. 253 * 254 * After this function call, after registers in the fpstate are 255 * modified and the child task has woken up, the child task will 256 * restore the modified FPU state from the modified context. If we 257 * didn't clear its cached status here then the cached in-registers 258 * state pending on its former CPU could be restored, corrupting 259 * the modifications. 260 */ 261 void fpu__prepare_write(struct fpu *fpu) 262 { 263 /* 264 * Only stopped child tasks can be used to modify the FPU 265 * state in the fpstate buffer: 266 */ 267 WARN_ON_FPU(fpu == ¤t->thread.fpu); 268 269 /* Invalidate any cached state: */ 270 __fpu_invalidate_fpregs_state(fpu); 271 } 272 273 /* 274 * Drops current FPU state: deactivates the fpregs and 275 * the fpstate. NOTE: it still leaves previous contents 276 * in the fpregs in the eager-FPU case. 277 * 278 * This function can be used in cases where we know that 279 * a state-restore is coming: either an explicit one, 280 * or a reschedule. 281 */ 282 void fpu__drop(struct fpu *fpu) 283 { 284 preempt_disable(); 285 286 if (fpu == ¤t->thread.fpu) { 287 /* Ignore delayed exceptions from user space */ 288 asm volatile("1: fwait\n" 289 "2:\n" 290 _ASM_EXTABLE(1b, 2b)); 291 fpregs_deactivate(fpu); 292 } 293 294 trace_x86_fpu_dropped(fpu); 295 296 preempt_enable(); 297 } 298 299 /* 300 * Clear FPU registers by setting them up from the init fpstate. 301 * Caller must do fpregs_[un]lock() around it. 302 */ 303 static inline void copy_init_fpstate_to_fpregs(u64 features_mask) 304 { 305 if (use_xsave()) 306 copy_kernel_to_xregs(&init_fpstate.xsave, features_mask); 307 else if (static_cpu_has(X86_FEATURE_FXSR)) 308 copy_kernel_to_fxregs(&init_fpstate.fxsave); 309 else 310 copy_kernel_to_fregs(&init_fpstate.fsave); 311 312 if (boot_cpu_has(X86_FEATURE_OSPKE)) 313 copy_init_pkru_to_fpregs(); 314 } 315 316 /* 317 * Clear the FPU state back to init state. 318 * 319 * Called by sys_execve(), by the signal handler code and by various 320 * error paths. 321 */ 322 static void fpu__clear(struct fpu *fpu, bool user_only) 323 { 324 WARN_ON_FPU(fpu != ¤t->thread.fpu); 325 326 if (!static_cpu_has(X86_FEATURE_FPU)) { 327 fpu__drop(fpu); 328 fpu__initialize(fpu); 329 return; 330 } 331 332 fpregs_lock(); 333 334 if (user_only) { 335 if (!fpregs_state_valid(fpu, smp_processor_id()) && 336 xfeatures_mask_supervisor()) 337 copy_kernel_to_xregs(&fpu->state.xsave, 338 xfeatures_mask_supervisor()); 339 copy_init_fpstate_to_fpregs(xfeatures_mask_user()); 340 } else { 341 copy_init_fpstate_to_fpregs(xfeatures_mask_all); 342 } 343 344 fpregs_mark_activate(); 345 fpregs_unlock(); 346 } 347 348 void fpu__clear_user_states(struct fpu *fpu) 349 { 350 fpu__clear(fpu, true); 351 } 352 353 void fpu__clear_all(struct fpu *fpu) 354 { 355 fpu__clear(fpu, false); 356 } 357 358 /* 359 * Load FPU context before returning to userspace. 360 */ 361 void switch_fpu_return(void) 362 { 363 if (!static_cpu_has(X86_FEATURE_FPU)) 364 return; 365 366 __fpregs_load_activate(); 367 } 368 EXPORT_SYMBOL_GPL(switch_fpu_return); 369 370 #ifdef CONFIG_X86_DEBUG_FPU 371 /* 372 * If current FPU state according to its tracking (loaded FPU context on this 373 * CPU) is not valid then we must have TIF_NEED_FPU_LOAD set so the context is 374 * loaded on return to userland. 375 */ 376 void fpregs_assert_state_consistent(void) 377 { 378 struct fpu *fpu = ¤t->thread.fpu; 379 380 if (test_thread_flag(TIF_NEED_FPU_LOAD)) 381 return; 382 383 WARN_ON_FPU(!fpregs_state_valid(fpu, smp_processor_id())); 384 } 385 EXPORT_SYMBOL_GPL(fpregs_assert_state_consistent); 386 #endif 387 388 void fpregs_mark_activate(void) 389 { 390 struct fpu *fpu = ¤t->thread.fpu; 391 392 fpregs_activate(fpu); 393 fpu->last_cpu = smp_processor_id(); 394 clear_thread_flag(TIF_NEED_FPU_LOAD); 395 } 396 EXPORT_SYMBOL_GPL(fpregs_mark_activate); 397 398 /* 399 * x87 math exception handling: 400 */ 401 402 int fpu__exception_code(struct fpu *fpu, int trap_nr) 403 { 404 int err; 405 406 if (trap_nr == X86_TRAP_MF) { 407 unsigned short cwd, swd; 408 /* 409 * (~cwd & swd) will mask out exceptions that are not set to unmasked 410 * status. 0x3f is the exception bits in these regs, 0x200 is the 411 * C1 reg you need in case of a stack fault, 0x040 is the stack 412 * fault bit. We should only be taking one exception at a time, 413 * so if this combination doesn't produce any single exception, 414 * then we have a bad program that isn't synchronizing its FPU usage 415 * and it will suffer the consequences since we won't be able to 416 * fully reproduce the context of the exception. 417 */ 418 if (boot_cpu_has(X86_FEATURE_FXSR)) { 419 cwd = fpu->state.fxsave.cwd; 420 swd = fpu->state.fxsave.swd; 421 } else { 422 cwd = (unsigned short)fpu->state.fsave.cwd; 423 swd = (unsigned short)fpu->state.fsave.swd; 424 } 425 426 err = swd & ~cwd; 427 } else { 428 /* 429 * The SIMD FPU exceptions are handled a little differently, as there 430 * is only a single status/control register. Thus, to determine which 431 * unmasked exception was caught we must mask the exception mask bits 432 * at 0x1f80, and then use these to mask the exception bits at 0x3f. 433 */ 434 unsigned short mxcsr = MXCSR_DEFAULT; 435 436 if (boot_cpu_has(X86_FEATURE_XMM)) 437 mxcsr = fpu->state.fxsave.mxcsr; 438 439 err = ~(mxcsr >> 7) & mxcsr; 440 } 441 442 if (err & 0x001) { /* Invalid op */ 443 /* 444 * swd & 0x240 == 0x040: Stack Underflow 445 * swd & 0x240 == 0x240: Stack Overflow 446 * User must clear the SF bit (0x40) if set 447 */ 448 return FPE_FLTINV; 449 } else if (err & 0x004) { /* Divide by Zero */ 450 return FPE_FLTDIV; 451 } else if (err & 0x008) { /* Overflow */ 452 return FPE_FLTOVF; 453 } else if (err & 0x012) { /* Denormal, Underflow */ 454 return FPE_FLTUND; 455 } else if (err & 0x020) { /* Precision */ 456 return FPE_FLTRES; 457 } 458 459 /* 460 * If we're using IRQ 13, or supposedly even some trap 461 * X86_TRAP_MF implementations, it's possible 462 * we get a spurious trap, which is not an error. 463 */ 464 return 0; 465 } 466