1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * FPU data structures: 4 */ 5 #ifndef _ASM_X86_FPU_H 6 #define _ASM_X86_FPU_H 7 8 /* 9 * The legacy x87 FPU state format, as saved by FSAVE and 10 * restored by the FRSTOR instructions: 11 */ 12 struct fregs_state { 13 u32 cwd; /* FPU Control Word */ 14 u32 swd; /* FPU Status Word */ 15 u32 twd; /* FPU Tag Word */ 16 u32 fip; /* FPU IP Offset */ 17 u32 fcs; /* FPU IP Selector */ 18 u32 foo; /* FPU Operand Pointer Offset */ 19 u32 fos; /* FPU Operand Pointer Selector */ 20 21 /* 8*10 bytes for each FP-reg = 80 bytes: */ 22 u32 st_space[20]; 23 24 /* Software status information [not touched by FSAVE]: */ 25 u32 status; 26 }; 27 28 /* 29 * The legacy fx SSE/MMX FPU state format, as saved by FXSAVE and 30 * restored by the FXRSTOR instructions. It's similar to the FSAVE 31 * format, but differs in some areas, plus has extensions at 32 * the end for the XMM registers. 33 */ 34 struct fxregs_state { 35 u16 cwd; /* Control Word */ 36 u16 swd; /* Status Word */ 37 u16 twd; /* Tag Word */ 38 u16 fop; /* Last Instruction Opcode */ 39 union { 40 struct { 41 u64 rip; /* Instruction Pointer */ 42 u64 rdp; /* Data Pointer */ 43 }; 44 struct { 45 u32 fip; /* FPU IP Offset */ 46 u32 fcs; /* FPU IP Selector */ 47 u32 foo; /* FPU Operand Offset */ 48 u32 fos; /* FPU Operand Selector */ 49 }; 50 }; 51 u32 mxcsr; /* MXCSR Register State */ 52 u32 mxcsr_mask; /* MXCSR Mask */ 53 54 /* 8*16 bytes for each FP-reg = 128 bytes: */ 55 u32 st_space[32]; 56 57 /* 16*16 bytes for each XMM-reg = 256 bytes: */ 58 u32 xmm_space[64]; 59 60 u32 padding[12]; 61 62 union { 63 u32 padding1[12]; 64 u32 sw_reserved[12]; 65 }; 66 67 } __attribute__((aligned(16))); 68 69 /* Default value for fxregs_state.mxcsr: */ 70 #define MXCSR_DEFAULT 0x1f80 71 72 /* Copy both mxcsr & mxcsr_flags with a single u64 memcpy: */ 73 #define MXCSR_AND_FLAGS_SIZE sizeof(u64) 74 75 /* 76 * Software based FPU emulation state. This is arbitrary really, 77 * it matches the x87 format to make it easier to understand: 78 */ 79 struct swregs_state { 80 u32 cwd; 81 u32 swd; 82 u32 twd; 83 u32 fip; 84 u32 fcs; 85 u32 foo; 86 u32 fos; 87 /* 8*10 bytes for each FP-reg = 80 bytes: */ 88 u32 st_space[20]; 89 u8 ftop; 90 u8 changed; 91 u8 lookahead; 92 u8 no_update; 93 u8 rm; 94 u8 alimit; 95 struct math_emu_info *info; 96 u32 entry_eip; 97 }; 98 99 /* 100 * List of XSAVE features Linux knows about: 101 */ 102 enum xfeature { 103 XFEATURE_FP, 104 XFEATURE_SSE, 105 /* 106 * Values above here are "legacy states". 107 * Those below are "extended states". 108 */ 109 XFEATURE_YMM, 110 XFEATURE_BNDREGS, 111 XFEATURE_BNDCSR, 112 XFEATURE_OPMASK, 113 XFEATURE_ZMM_Hi256, 114 XFEATURE_Hi16_ZMM, 115 XFEATURE_PT_UNIMPLEMENTED_SO_FAR, 116 XFEATURE_PKRU, 117 XFEATURE_PASID, 118 XFEATURE_CET_USER, 119 XFEATURE_CET_KERNEL_UNUSED, 120 XFEATURE_RSRVD_COMP_13, 121 XFEATURE_RSRVD_COMP_14, 122 XFEATURE_LBR, 123 XFEATURE_RSRVD_COMP_16, 124 XFEATURE_XTILE_CFG, 125 XFEATURE_XTILE_DATA, 126 127 XFEATURE_MAX, 128 }; 129 130 #define XFEATURE_MASK_FP (1 << XFEATURE_FP) 131 #define XFEATURE_MASK_SSE (1 << XFEATURE_SSE) 132 #define XFEATURE_MASK_YMM (1 << XFEATURE_YMM) 133 #define XFEATURE_MASK_BNDREGS (1 << XFEATURE_BNDREGS) 134 #define XFEATURE_MASK_BNDCSR (1 << XFEATURE_BNDCSR) 135 #define XFEATURE_MASK_OPMASK (1 << XFEATURE_OPMASK) 136 #define XFEATURE_MASK_ZMM_Hi256 (1 << XFEATURE_ZMM_Hi256) 137 #define XFEATURE_MASK_Hi16_ZMM (1 << XFEATURE_Hi16_ZMM) 138 #define XFEATURE_MASK_PT (1 << XFEATURE_PT_UNIMPLEMENTED_SO_FAR) 139 #define XFEATURE_MASK_PKRU (1 << XFEATURE_PKRU) 140 #define XFEATURE_MASK_PASID (1 << XFEATURE_PASID) 141 #define XFEATURE_MASK_CET_USER (1 << XFEATURE_CET_USER) 142 #define XFEATURE_MASK_CET_KERNEL (1 << XFEATURE_CET_KERNEL_UNUSED) 143 #define XFEATURE_MASK_LBR (1 << XFEATURE_LBR) 144 #define XFEATURE_MASK_XTILE_CFG (1 << XFEATURE_XTILE_CFG) 145 #define XFEATURE_MASK_XTILE_DATA (1 << XFEATURE_XTILE_DATA) 146 147 #define XFEATURE_MASK_FPSSE (XFEATURE_MASK_FP | XFEATURE_MASK_SSE) 148 #define XFEATURE_MASK_AVX512 (XFEATURE_MASK_OPMASK \ 149 | XFEATURE_MASK_ZMM_Hi256 \ 150 | XFEATURE_MASK_Hi16_ZMM) 151 152 #ifdef CONFIG_X86_64 153 # define XFEATURE_MASK_XTILE (XFEATURE_MASK_XTILE_DATA \ 154 | XFEATURE_MASK_XTILE_CFG) 155 #else 156 # define XFEATURE_MASK_XTILE (0) 157 #endif 158 159 #define FIRST_EXTENDED_XFEATURE XFEATURE_YMM 160 161 struct reg_128_bit { 162 u8 regbytes[128/8]; 163 }; 164 struct reg_256_bit { 165 u8 regbytes[256/8]; 166 }; 167 struct reg_512_bit { 168 u8 regbytes[512/8]; 169 }; 170 struct reg_1024_byte { 171 u8 regbytes[1024]; 172 }; 173 174 /* 175 * State component 2: 176 * 177 * There are 16x 256-bit AVX registers named YMM0-YMM15. 178 * The low 128 bits are aliased to the 16 SSE registers (XMM0-XMM15) 179 * and are stored in 'struct fxregs_state::xmm_space[]' in the 180 * "legacy" area. 181 * 182 * The high 128 bits are stored here. 183 */ 184 struct ymmh_struct { 185 struct reg_128_bit hi_ymm[16]; 186 } __packed; 187 188 /* Intel MPX support: */ 189 190 struct mpx_bndreg { 191 u64 lower_bound; 192 u64 upper_bound; 193 } __packed; 194 /* 195 * State component 3 is used for the 4 128-bit bounds registers 196 */ 197 struct mpx_bndreg_state { 198 struct mpx_bndreg bndreg[4]; 199 } __packed; 200 201 /* 202 * State component 4 is used for the 64-bit user-mode MPX 203 * configuration register BNDCFGU and the 64-bit MPX status 204 * register BNDSTATUS. We call the pair "BNDCSR". 205 */ 206 struct mpx_bndcsr { 207 u64 bndcfgu; 208 u64 bndstatus; 209 } __packed; 210 211 /* 212 * The BNDCSR state is padded out to be 64-bytes in size. 213 */ 214 struct mpx_bndcsr_state { 215 union { 216 struct mpx_bndcsr bndcsr; 217 u8 pad_to_64_bytes[64]; 218 }; 219 } __packed; 220 221 /* AVX-512 Components: */ 222 223 /* 224 * State component 5 is used for the 8 64-bit opmask registers 225 * k0-k7 (opmask state). 226 */ 227 struct avx_512_opmask_state { 228 u64 opmask_reg[8]; 229 } __packed; 230 231 /* 232 * State component 6 is used for the upper 256 bits of the 233 * registers ZMM0-ZMM15. These 16 256-bit values are denoted 234 * ZMM0_H-ZMM15_H (ZMM_Hi256 state). 235 */ 236 struct avx_512_zmm_uppers_state { 237 struct reg_256_bit zmm_upper[16]; 238 } __packed; 239 240 /* 241 * State component 7 is used for the 16 512-bit registers 242 * ZMM16-ZMM31 (Hi16_ZMM state). 243 */ 244 struct avx_512_hi16_state { 245 struct reg_512_bit hi16_zmm[16]; 246 } __packed; 247 248 /* 249 * State component 9: 32-bit PKRU register. The state is 250 * 8 bytes long but only 4 bytes is used currently. 251 */ 252 struct pkru_state { 253 u32 pkru; 254 u32 pad; 255 } __packed; 256 257 /* 258 * State component 11 is Control-flow Enforcement user states 259 */ 260 struct cet_user_state { 261 /* user control-flow settings */ 262 u64 user_cet; 263 /* user shadow stack pointer */ 264 u64 user_ssp; 265 }; 266 267 /* 268 * State component 15: Architectural LBR configuration state. 269 * The size of Arch LBR state depends on the number of LBRs (lbr_depth). 270 */ 271 272 struct lbr_entry { 273 u64 from; 274 u64 to; 275 u64 info; 276 }; 277 278 struct arch_lbr_state { 279 u64 lbr_ctl; 280 u64 lbr_depth; 281 u64 ler_from; 282 u64 ler_to; 283 u64 ler_info; 284 struct lbr_entry entries[]; 285 }; 286 287 /* 288 * State component 17: 64-byte tile configuration register. 289 */ 290 struct xtile_cfg { 291 u64 tcfg[8]; 292 } __packed; 293 294 /* 295 * State component 18: 1KB tile data register. 296 * Each register represents 16 64-byte rows of the matrix 297 * data. But the number of registers depends on the actual 298 * implementation. 299 */ 300 struct xtile_data { 301 struct reg_1024_byte tmm; 302 } __packed; 303 304 /* 305 * State component 10 is supervisor state used for context-switching the 306 * PASID state. 307 */ 308 struct ia32_pasid_state { 309 u64 pasid; 310 } __packed; 311 312 struct xstate_header { 313 u64 xfeatures; 314 u64 xcomp_bv; 315 u64 reserved[6]; 316 } __attribute__((packed)); 317 318 /* 319 * xstate_header.xcomp_bv[63] indicates that the extended_state_area 320 * is in compacted format. 321 */ 322 #define XCOMP_BV_COMPACTED_FORMAT ((u64)1 << 63) 323 324 /* 325 * This is our most modern FPU state format, as saved by the XSAVE 326 * and restored by the XRSTOR instructions. 327 * 328 * It consists of a legacy fxregs portion, an xstate header and 329 * subsequent areas as defined by the xstate header. Not all CPUs 330 * support all the extensions, so the size of the extended area 331 * can vary quite a bit between CPUs. 332 */ 333 struct xregs_state { 334 struct fxregs_state i387; 335 struct xstate_header header; 336 u8 extended_state_area[]; 337 } __attribute__ ((packed, aligned (64))); 338 339 /* 340 * This is a union of all the possible FPU state formats 341 * put together, so that we can pick the right one runtime. 342 * 343 * The size of the structure is determined by the largest 344 * member - which is the xsave area. The padding is there 345 * to ensure that statically-allocated task_structs (just 346 * the init_task today) have enough space. 347 */ 348 union fpregs_state { 349 struct fregs_state fsave; 350 struct fxregs_state fxsave; 351 struct swregs_state soft; 352 struct xregs_state xsave; 353 u8 __padding[PAGE_SIZE]; 354 }; 355 356 struct fpstate { 357 /* @kernel_size: The size of the kernel register image */ 358 unsigned int size; 359 360 /* @user_size: The size in non-compacted UABI format */ 361 unsigned int user_size; 362 363 /* @xfeatures: xfeatures for which the storage is sized */ 364 u64 xfeatures; 365 366 /* @user_xfeatures: xfeatures valid in UABI buffers */ 367 u64 user_xfeatures; 368 369 /* @xfd: xfeatures disabled to trap userspace use. */ 370 u64 xfd; 371 372 /* @is_valloc: Indicator for dynamically allocated state */ 373 unsigned int is_valloc : 1; 374 375 /* @is_guest: Indicator for guest state (KVM) */ 376 unsigned int is_guest : 1; 377 378 /* 379 * @is_confidential: Indicator for KVM confidential mode. 380 * The FPU registers are restored by the 381 * vmentry firmware from encrypted guest 382 * memory. On vmexit the FPU registers are 383 * saved by firmware to encrypted guest memory 384 * and the registers are scrubbed before 385 * returning to the host. So there is no 386 * content which is worth saving and restoring. 387 * The fpstate has to be there so that 388 * preemption and softirq FPU usage works 389 * without special casing. 390 */ 391 unsigned int is_confidential : 1; 392 393 /* @in_use: State is in use */ 394 unsigned int in_use : 1; 395 396 /* @regs: The register state union for all supported formats */ 397 union fpregs_state regs; 398 399 /* @regs is dynamically sized! Don't add anything after @regs! */ 400 } __aligned(64); 401 402 #define FPU_GUEST_PERM_LOCKED BIT_ULL(63) 403 404 struct fpu_state_perm { 405 /* 406 * @__state_perm: 407 * 408 * This bitmap indicates the permission for state components, which 409 * are available to a thread group. The permission prctl() sets the 410 * enabled state bits in thread_group_leader()->thread.fpu. 411 * 412 * All run time operations use the per thread information in the 413 * currently active fpu.fpstate which contains the xfeature masks 414 * and sizes for kernel and user space. 415 * 416 * This master permission field is only to be used when 417 * task.fpu.fpstate based checks fail to validate whether the task 418 * is allowed to expand it's xfeatures set which requires to 419 * allocate a larger sized fpstate buffer. 420 * 421 * Do not access this field directly. Use the provided helper 422 * function. Unlocked access is possible for quick checks. 423 */ 424 u64 __state_perm; 425 426 /* 427 * @__state_size: 428 * 429 * The size required for @__state_perm. Only valid to access 430 * with sighand locked. 431 */ 432 unsigned int __state_size; 433 434 /* 435 * @__user_state_size: 436 * 437 * The size required for @__state_perm user part. Only valid to 438 * access with sighand locked. 439 */ 440 unsigned int __user_state_size; 441 }; 442 443 /* 444 * Highest level per task FPU state data structure that 445 * contains the FPU register state plus various FPU 446 * state fields: 447 */ 448 struct fpu { 449 /* 450 * @last_cpu: 451 * 452 * Records the last CPU on which this context was loaded into 453 * FPU registers. (In the lazy-restore case we might be 454 * able to reuse FPU registers across multiple context switches 455 * this way, if no intermediate task used the FPU.) 456 * 457 * A value of -1 is used to indicate that the FPU state in context 458 * memory is newer than the FPU state in registers, and that the 459 * FPU state should be reloaded next time the task is run. 460 */ 461 unsigned int last_cpu; 462 463 /* 464 * @avx512_timestamp: 465 * 466 * Records the timestamp of AVX512 use during last context switch. 467 */ 468 unsigned long avx512_timestamp; 469 470 /* 471 * @fpstate: 472 * 473 * Pointer to the active struct fpstate. Initialized to 474 * point at @__fpstate below. 475 */ 476 struct fpstate *fpstate; 477 478 /* 479 * @__task_fpstate: 480 * 481 * Pointer to an inactive struct fpstate. Initialized to NULL. Is 482 * used only for KVM support to swap out the regular task fpstate. 483 */ 484 struct fpstate *__task_fpstate; 485 486 /* 487 * @perm: 488 * 489 * Permission related information 490 */ 491 struct fpu_state_perm perm; 492 493 /* 494 * @guest_perm: 495 * 496 * Permission related information for guest pseudo FPUs 497 */ 498 struct fpu_state_perm guest_perm; 499 500 /* 501 * @__fpstate: 502 * 503 * Initial in-memory storage for FPU registers which are saved in 504 * context switch and when the kernel uses the FPU. The registers 505 * are restored from this storage on return to user space if they 506 * are not longer containing the tasks FPU register state. 507 */ 508 struct fpstate __fpstate; 509 /* 510 * WARNING: '__fpstate' is dynamically-sized. Do not put 511 * anything after it here. 512 */ 513 }; 514 515 /* 516 * Guest pseudo FPU container 517 */ 518 struct fpu_guest { 519 /* 520 * @xfeatures: xfeature bitmap of features which are 521 * currently enabled for the guest vCPU. 522 */ 523 u64 xfeatures; 524 525 /* 526 * @perm: xfeature bitmap of features which are 527 * permitted to be enabled for the guest 528 * vCPU. 529 */ 530 u64 perm; 531 532 /* 533 * @xfd_err: Save the guest value. 534 */ 535 u64 xfd_err; 536 537 /* 538 * @uabi_size: Size required for save/restore 539 */ 540 unsigned int uabi_size; 541 542 /* 543 * @fpstate: Pointer to the allocated guest fpstate 544 */ 545 struct fpstate *fpstate; 546 }; 547 548 /* 549 * FPU state configuration data. Initialized at boot time. Read only after init. 550 */ 551 struct fpu_state_config { 552 /* 553 * @max_size: 554 * 555 * The maximum size of the register state buffer. Includes all 556 * supported features except independent managed features. 557 */ 558 unsigned int max_size; 559 560 /* 561 * @default_size: 562 * 563 * The default size of the register state buffer. Includes all 564 * supported features except independent managed features and 565 * features which have to be requested by user space before usage. 566 */ 567 unsigned int default_size; 568 569 /* 570 * @max_features: 571 * 572 * The maximum supported features bitmap. Does not include 573 * independent managed features. 574 */ 575 u64 max_features; 576 577 /* 578 * @default_features: 579 * 580 * The default supported features bitmap. Does not include 581 * independent managed features and features which have to 582 * be requested by user space before usage. 583 */ 584 u64 default_features; 585 /* 586 * @legacy_features: 587 * 588 * Features which can be reported back to user space 589 * even without XSAVE support, i.e. legacy features FP + SSE 590 */ 591 u64 legacy_features; 592 /* 593 * @independent_features: 594 * 595 * Features that are supported by XSAVES, but not managed as part of 596 * the FPU core, such as LBR 597 */ 598 u64 independent_features; 599 }; 600 601 /* FPU state configuration information */ 602 extern struct fpu_state_config fpu_kernel_cfg, fpu_user_cfg; 603 604 #endif /* _ASM_X86_FPU_H */ 605