1 /* 2 * Copyright (C) 1994 Linus Torvalds 3 * 4 * Pentium III FXSR, SSE support 5 * General FPU state handling cleanups 6 * Gareth Hughes <gareth@valinux.com>, May 2000 7 */ 8 #include <asm/fpu/internal.h> 9 #include <asm/fpu/regset.h> 10 #include <asm/fpu/signal.h> 11 #include <asm/fpu/types.h> 12 #include <asm/traps.h> 13 #include <asm/irq_regs.h> 14 15 #include <linux/hardirq.h> 16 #include <linux/pkeys.h> 17 18 #define CREATE_TRACE_POINTS 19 #include <asm/trace/fpu.h> 20 21 /* 22 * Represents the initial FPU state. It's mostly (but not completely) zeroes, 23 * depending on the FPU hardware format: 24 */ 25 union fpregs_state init_fpstate __read_mostly; 26 27 /* 28 * Track whether the kernel is using the FPU state 29 * currently. 30 * 31 * This flag is used: 32 * 33 * - by IRQ context code to potentially use the FPU 34 * if it's unused. 35 * 36 * - to debug kernel_fpu_begin()/end() correctness 37 */ 38 static DEFINE_PER_CPU(bool, in_kernel_fpu); 39 40 /* 41 * Track which context is using the FPU on the CPU: 42 */ 43 DEFINE_PER_CPU(struct fpu *, fpu_fpregs_owner_ctx); 44 45 static void kernel_fpu_disable(void) 46 { 47 WARN_ON_FPU(this_cpu_read(in_kernel_fpu)); 48 this_cpu_write(in_kernel_fpu, true); 49 } 50 51 static void kernel_fpu_enable(void) 52 { 53 WARN_ON_FPU(!this_cpu_read(in_kernel_fpu)); 54 this_cpu_write(in_kernel_fpu, false); 55 } 56 57 static bool kernel_fpu_disabled(void) 58 { 59 return this_cpu_read(in_kernel_fpu); 60 } 61 62 static bool interrupted_kernel_fpu_idle(void) 63 { 64 return !kernel_fpu_disabled(); 65 } 66 67 /* 68 * Were we in user mode (or vm86 mode) when we were 69 * interrupted? 70 * 71 * Doing kernel_fpu_begin/end() is ok if we are running 72 * in an interrupt context from user mode - we'll just 73 * save the FPU state as required. 74 */ 75 static bool interrupted_user_mode(void) 76 { 77 struct pt_regs *regs = get_irq_regs(); 78 return regs && user_mode(regs); 79 } 80 81 /* 82 * Can we use the FPU in kernel mode with the 83 * whole "kernel_fpu_begin/end()" sequence? 84 * 85 * It's always ok in process context (ie "not interrupt") 86 * but it is sometimes ok even from an irq. 87 */ 88 bool irq_fpu_usable(void) 89 { 90 return !in_interrupt() || 91 interrupted_user_mode() || 92 interrupted_kernel_fpu_idle(); 93 } 94 EXPORT_SYMBOL(irq_fpu_usable); 95 96 void __kernel_fpu_begin(void) 97 { 98 struct fpu *fpu = ¤t->thread.fpu; 99 100 WARN_ON_FPU(!irq_fpu_usable()); 101 102 kernel_fpu_disable(); 103 104 if (fpu->initialized) { 105 /* 106 * Ignore return value -- we don't care if reg state 107 * is clobbered. 108 */ 109 copy_fpregs_to_fpstate(fpu); 110 } else { 111 __cpu_invalidate_fpregs_state(); 112 } 113 } 114 EXPORT_SYMBOL(__kernel_fpu_begin); 115 116 void __kernel_fpu_end(void) 117 { 118 struct fpu *fpu = ¤t->thread.fpu; 119 120 if (fpu->initialized) 121 copy_kernel_to_fpregs(&fpu->state); 122 123 kernel_fpu_enable(); 124 } 125 EXPORT_SYMBOL(__kernel_fpu_end); 126 127 void kernel_fpu_begin(void) 128 { 129 preempt_disable(); 130 __kernel_fpu_begin(); 131 } 132 EXPORT_SYMBOL_GPL(kernel_fpu_begin); 133 134 void kernel_fpu_end(void) 135 { 136 __kernel_fpu_end(); 137 preempt_enable(); 138 } 139 EXPORT_SYMBOL_GPL(kernel_fpu_end); 140 141 /* 142 * Save the FPU state (mark it for reload if necessary): 143 * 144 * This only ever gets called for the current task. 145 */ 146 void fpu__save(struct fpu *fpu) 147 { 148 WARN_ON_FPU(fpu != ¤t->thread.fpu); 149 150 preempt_disable(); 151 trace_x86_fpu_before_save(fpu); 152 if (fpu->initialized) { 153 if (!copy_fpregs_to_fpstate(fpu)) { 154 copy_kernel_to_fpregs(&fpu->state); 155 } 156 } 157 trace_x86_fpu_after_save(fpu); 158 preempt_enable(); 159 } 160 EXPORT_SYMBOL_GPL(fpu__save); 161 162 /* 163 * Legacy x87 fpstate state init: 164 */ 165 static inline void fpstate_init_fstate(struct fregs_state *fp) 166 { 167 fp->cwd = 0xffff037fu; 168 fp->swd = 0xffff0000u; 169 fp->twd = 0xffffffffu; 170 fp->fos = 0xffff0000u; 171 } 172 173 void fpstate_init(union fpregs_state *state) 174 { 175 if (!static_cpu_has(X86_FEATURE_FPU)) { 176 fpstate_init_soft(&state->soft); 177 return; 178 } 179 180 memset(state, 0, fpu_kernel_xstate_size); 181 182 if (static_cpu_has(X86_FEATURE_XSAVES)) 183 fpstate_init_xstate(&state->xsave); 184 if (static_cpu_has(X86_FEATURE_FXSR)) 185 fpstate_init_fxstate(&state->fxsave); 186 else 187 fpstate_init_fstate(&state->fsave); 188 } 189 EXPORT_SYMBOL_GPL(fpstate_init); 190 191 int fpu__copy(struct fpu *dst_fpu, struct fpu *src_fpu) 192 { 193 dst_fpu->last_cpu = -1; 194 195 if (!src_fpu->initialized || !static_cpu_has(X86_FEATURE_FPU)) 196 return 0; 197 198 WARN_ON_FPU(src_fpu != ¤t->thread.fpu); 199 200 /* 201 * Don't let 'init optimized' areas of the XSAVE area 202 * leak into the child task: 203 */ 204 memset(&dst_fpu->state.xsave, 0, fpu_kernel_xstate_size); 205 206 /* 207 * Save current FPU registers directly into the child 208 * FPU context, without any memory-to-memory copying. 209 * 210 * ( The function 'fails' in the FNSAVE case, which destroys 211 * register contents so we have to copy them back. ) 212 */ 213 if (!copy_fpregs_to_fpstate(dst_fpu)) { 214 memcpy(&src_fpu->state, &dst_fpu->state, fpu_kernel_xstate_size); 215 copy_kernel_to_fpregs(&src_fpu->state); 216 } 217 218 trace_x86_fpu_copy_src(src_fpu); 219 trace_x86_fpu_copy_dst(dst_fpu); 220 221 return 0; 222 } 223 224 /* 225 * Activate the current task's in-memory FPU context, 226 * if it has not been used before: 227 */ 228 void fpu__initialize(struct fpu *fpu) 229 { 230 WARN_ON_FPU(fpu != ¤t->thread.fpu); 231 232 if (!fpu->initialized) { 233 fpstate_init(&fpu->state); 234 trace_x86_fpu_init_state(fpu); 235 236 trace_x86_fpu_activate_state(fpu); 237 /* Safe to do for the current task: */ 238 fpu->initialized = 1; 239 } 240 } 241 EXPORT_SYMBOL_GPL(fpu__initialize); 242 243 /* 244 * This function must be called before we read a task's fpstate. 245 * 246 * There's two cases where this gets called: 247 * 248 * - for the current task (when coredumping), in which case we have 249 * to save the latest FPU registers into the fpstate, 250 * 251 * - or it's called for stopped tasks (ptrace), in which case the 252 * registers were already saved by the context-switch code when 253 * the task scheduled out - we only have to initialize the registers 254 * if they've never been initialized. 255 * 256 * If the task has used the FPU before then save it. 257 */ 258 void fpu__prepare_read(struct fpu *fpu) 259 { 260 if (fpu == ¤t->thread.fpu) { 261 fpu__save(fpu); 262 } else { 263 if (!fpu->initialized) { 264 fpstate_init(&fpu->state); 265 trace_x86_fpu_init_state(fpu); 266 267 trace_x86_fpu_activate_state(fpu); 268 /* Safe to do for current and for stopped child tasks: */ 269 fpu->initialized = 1; 270 } 271 } 272 } 273 274 /* 275 * This function must be called before we write a task's fpstate. 276 * 277 * If the task has used the FPU before then invalidate any cached FPU registers. 278 * If the task has not used the FPU before then initialize its fpstate. 279 * 280 * After this function call, after registers in the fpstate are 281 * modified and the child task has woken up, the child task will 282 * restore the modified FPU state from the modified context. If we 283 * didn't clear its cached status here then the cached in-registers 284 * state pending on its former CPU could be restored, corrupting 285 * the modifications. 286 */ 287 void fpu__prepare_write(struct fpu *fpu) 288 { 289 /* 290 * Only stopped child tasks can be used to modify the FPU 291 * state in the fpstate buffer: 292 */ 293 WARN_ON_FPU(fpu == ¤t->thread.fpu); 294 295 if (fpu->initialized) { 296 /* Invalidate any cached state: */ 297 __fpu_invalidate_fpregs_state(fpu); 298 } else { 299 fpstate_init(&fpu->state); 300 trace_x86_fpu_init_state(fpu); 301 302 trace_x86_fpu_activate_state(fpu); 303 /* Safe to do for stopped child tasks: */ 304 fpu->initialized = 1; 305 } 306 } 307 308 /* 309 * 'fpu__restore()' is called to copy FPU registers from 310 * the FPU fpstate to the live hw registers and to activate 311 * access to the hardware registers, so that FPU instructions 312 * can be used afterwards. 313 * 314 * Must be called with kernel preemption disabled (for example 315 * with local interrupts disabled, as it is in the case of 316 * do_device_not_available()). 317 */ 318 void fpu__restore(struct fpu *fpu) 319 { 320 fpu__initialize(fpu); 321 322 /* Avoid __kernel_fpu_begin() right after fpregs_activate() */ 323 kernel_fpu_disable(); 324 trace_x86_fpu_before_restore(fpu); 325 fpregs_activate(fpu); 326 copy_kernel_to_fpregs(&fpu->state); 327 trace_x86_fpu_after_restore(fpu); 328 kernel_fpu_enable(); 329 } 330 EXPORT_SYMBOL_GPL(fpu__restore); 331 332 /* 333 * Drops current FPU state: deactivates the fpregs and 334 * the fpstate. NOTE: it still leaves previous contents 335 * in the fpregs in the eager-FPU case. 336 * 337 * This function can be used in cases where we know that 338 * a state-restore is coming: either an explicit one, 339 * or a reschedule. 340 */ 341 void fpu__drop(struct fpu *fpu) 342 { 343 preempt_disable(); 344 345 if (fpu == ¤t->thread.fpu) { 346 if (fpu->initialized) { 347 /* Ignore delayed exceptions from user space */ 348 asm volatile("1: fwait\n" 349 "2:\n" 350 _ASM_EXTABLE(1b, 2b)); 351 fpregs_deactivate(fpu); 352 } 353 } 354 355 fpu->initialized = 0; 356 357 trace_x86_fpu_dropped(fpu); 358 359 preempt_enable(); 360 } 361 362 /* 363 * Clear FPU registers by setting them up from 364 * the init fpstate: 365 */ 366 static inline void copy_init_fpstate_to_fpregs(void) 367 { 368 if (use_xsave()) 369 copy_kernel_to_xregs(&init_fpstate.xsave, -1); 370 else if (static_cpu_has(X86_FEATURE_FXSR)) 371 copy_kernel_to_fxregs(&init_fpstate.fxsave); 372 else 373 copy_kernel_to_fregs(&init_fpstate.fsave); 374 375 if (boot_cpu_has(X86_FEATURE_OSPKE)) 376 copy_init_pkru_to_fpregs(); 377 } 378 379 /* 380 * Clear the FPU state back to init state. 381 * 382 * Called by sys_execve(), by the signal handler code and by various 383 * error paths. 384 */ 385 void fpu__clear(struct fpu *fpu) 386 { 387 WARN_ON_FPU(fpu != ¤t->thread.fpu); /* Almost certainly an anomaly */ 388 389 fpu__drop(fpu); 390 391 /* 392 * Make sure fpstate is cleared and initialized. 393 */ 394 if (static_cpu_has(X86_FEATURE_FPU)) { 395 preempt_disable(); 396 fpu__initialize(fpu); 397 user_fpu_begin(); 398 copy_init_fpstate_to_fpregs(); 399 preempt_enable(); 400 } 401 } 402 403 /* 404 * x87 math exception handling: 405 */ 406 407 int fpu__exception_code(struct fpu *fpu, int trap_nr) 408 { 409 int err; 410 411 if (trap_nr == X86_TRAP_MF) { 412 unsigned short cwd, swd; 413 /* 414 * (~cwd & swd) will mask out exceptions that are not set to unmasked 415 * status. 0x3f is the exception bits in these regs, 0x200 is the 416 * C1 reg you need in case of a stack fault, 0x040 is the stack 417 * fault bit. We should only be taking one exception at a time, 418 * so if this combination doesn't produce any single exception, 419 * then we have a bad program that isn't synchronizing its FPU usage 420 * and it will suffer the consequences since we won't be able to 421 * fully reproduce the context of the exception. 422 */ 423 if (boot_cpu_has(X86_FEATURE_FXSR)) { 424 cwd = fpu->state.fxsave.cwd; 425 swd = fpu->state.fxsave.swd; 426 } else { 427 cwd = (unsigned short)fpu->state.fsave.cwd; 428 swd = (unsigned short)fpu->state.fsave.swd; 429 } 430 431 err = swd & ~cwd; 432 } else { 433 /* 434 * The SIMD FPU exceptions are handled a little differently, as there 435 * is only a single status/control register. Thus, to determine which 436 * unmasked exception was caught we must mask the exception mask bits 437 * at 0x1f80, and then use these to mask the exception bits at 0x3f. 438 */ 439 unsigned short mxcsr = MXCSR_DEFAULT; 440 441 if (boot_cpu_has(X86_FEATURE_XMM)) 442 mxcsr = fpu->state.fxsave.mxcsr; 443 444 err = ~(mxcsr >> 7) & mxcsr; 445 } 446 447 if (err & 0x001) { /* Invalid op */ 448 /* 449 * swd & 0x240 == 0x040: Stack Underflow 450 * swd & 0x240 == 0x240: Stack Overflow 451 * User must clear the SF bit (0x40) if set 452 */ 453 return FPE_FLTINV; 454 } else if (err & 0x004) { /* Divide by Zero */ 455 return FPE_FLTDIV; 456 } else if (err & 0x008) { /* Overflow */ 457 return FPE_FLTOVF; 458 } else if (err & 0x012) { /* Denormal, Underflow */ 459 return FPE_FLTUND; 460 } else if (err & 0x020) { /* Precision */ 461 return FPE_FLTRES; 462 } 463 464 /* 465 * If we're using IRQ 13, or supposedly even some trap 466 * X86_TRAP_MF implementations, it's possible 467 * we get a spurious trap, which is not an error. 468 */ 469 return 0; 470 } 471