1457c8996SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 262784854SIngo Molnar /* 362784854SIngo Molnar * xsave/xrstor support. 462784854SIngo Molnar * 562784854SIngo Molnar * Author: Suresh Siddha <suresh.b.siddha@intel.com> 662784854SIngo Molnar */ 762784854SIngo Molnar #include <linux/compat.h> 862784854SIngo Molnar #include <linux/cpu.h> 9e8c24d3aSDave Hansen #include <linux/mman.h> 1084594296SDave Hansen #include <linux/pkeys.h> 110c608dadSAubrey Li #include <linux/seq_file.h> 120c608dadSAubrey Li #include <linux/proc_fs.h> 1359a36d16SIngo Molnar 1462784854SIngo Molnar #include <asm/fpu/api.h> 1562784854SIngo Molnar #include <asm/fpu/internal.h> 16fcbc99c4SIngo Molnar #include <asm/fpu/signal.h> 1759a36d16SIngo Molnar #include <asm/fpu/regset.h> 1891c3dba7SYu-cheng Yu #include <asm/fpu/xstate.h> 19b992c660SIngo Molnar 2062784854SIngo Molnar #include <asm/tlbflush.h> 21ccb18db2SAndi Kleen #include <asm/cpufeature.h> 2262784854SIngo Molnar 231f96b1efSDave Hansen /* 241f96b1efSDave Hansen * Although we spell it out in here, the Processor Trace 251f96b1efSDave Hansen * xfeature is completely unused. We use other mechanisms 261f96b1efSDave Hansen * to save/restore PT state in Linux. 271f96b1efSDave Hansen */ 285b073430SIngo Molnar static const char *xfeature_names[] = 295b073430SIngo Molnar { 305b073430SIngo Molnar "x87 floating point registers" , 315b073430SIngo Molnar "SSE registers" , 325b073430SIngo Molnar "AVX registers" , 335b073430SIngo Molnar "MPX bounds registers" , 345b073430SIngo Molnar "MPX CSR" , 355b073430SIngo Molnar "AVX-512 opmask" , 365b073430SIngo Molnar "AVX-512 Hi256" , 375b073430SIngo Molnar "AVX-512 ZMM_Hi256" , 381f96b1efSDave Hansen "Processor Trace (unused)" , 39c8df4009SDave Hansen "Protection Keys User registers", 40b454feb9SYu-cheng Yu "PASID state", 415b073430SIngo Molnar "unknown xstate feature" , 425b073430SIngo Molnar }; 435b073430SIngo Molnar 44ccb18db2SAndi Kleen static short xsave_cpuid_features[] __initdata = { 45ccb18db2SAndi Kleen X86_FEATURE_FPU, 46ccb18db2SAndi Kleen X86_FEATURE_XMM, 47ccb18db2SAndi Kleen X86_FEATURE_AVX, 48ccb18db2SAndi Kleen X86_FEATURE_MPX, 49ccb18db2SAndi Kleen X86_FEATURE_MPX, 50ccb18db2SAndi Kleen X86_FEATURE_AVX512F, 51ccb18db2SAndi Kleen X86_FEATURE_AVX512F, 52ccb18db2SAndi Kleen X86_FEATURE_AVX512F, 53ccb18db2SAndi Kleen X86_FEATURE_INTEL_PT, 54ccb18db2SAndi Kleen X86_FEATURE_PKU, 55b454feb9SYu-cheng Yu X86_FEATURE_ENQCMD, 56ccb18db2SAndi Kleen }; 57ccb18db2SAndi Kleen 5862784854SIngo Molnar /* 59524bb73bSYu-cheng Yu * This represents the full set of bits that should ever be set in a kernel 60524bb73bSYu-cheng Yu * XSAVE buffer, both supervisor and user xstates. 6162784854SIngo Molnar */ 62524bb73bSYu-cheng Yu u64 xfeatures_mask_all __read_mostly; 6362784854SIngo Molnar 64dad8c4feSDave Hansen static unsigned int xstate_offsets[XFEATURE_MAX] = { [ 0 ... XFEATURE_MAX - 1] = -1}; 65dad8c4feSDave Hansen static unsigned int xstate_sizes[XFEATURE_MAX] = { [ 0 ... XFEATURE_MAX - 1] = -1}; 66c0855051SCyrill Gorcunov static unsigned int xstate_comp_offsets[XFEATURE_MAX] = { [ 0 ... XFEATURE_MAX - 1] = -1}; 67eeedf153SYu-cheng Yu static unsigned int xstate_supervisor_only_offsets[XFEATURE_MAX] = { [ 0 ... XFEATURE_MAX - 1] = -1}; 6862784854SIngo Molnar 6962784854SIngo Molnar /* 70a1141e0bSFenghua Yu * The XSAVE area of kernel can be in standard or compacted format; 71a1141e0bSFenghua Yu * it is always in standard format for user mode. This is the user 72a1141e0bSFenghua Yu * mode standard format size used for signal and ptrace frames. 73a1141e0bSFenghua Yu */ 74a1141e0bSFenghua Yu unsigned int fpu_user_xstate_size; 75a1141e0bSFenghua Yu 76a1141e0bSFenghua Yu /* 775b073430SIngo Molnar * Return whether the system supports a given xfeature. 785b073430SIngo Molnar * 795b073430SIngo Molnar * Also return the name of the (most advanced) feature that the caller requested: 805b073430SIngo Molnar */ 815b073430SIngo Molnar int cpu_has_xfeatures(u64 xfeatures_needed, const char **feature_name) 825b073430SIngo Molnar { 83524bb73bSYu-cheng Yu u64 xfeatures_missing = xfeatures_needed & ~xfeatures_mask_all; 845b073430SIngo Molnar 855b073430SIngo Molnar if (unlikely(feature_name)) { 865b073430SIngo Molnar long xfeature_idx, max_idx; 875b073430SIngo Molnar u64 xfeatures_print; 885b073430SIngo Molnar /* 895b073430SIngo Molnar * So we use FLS here to be able to print the most advanced 905b073430SIngo Molnar * feature that was requested but is missing. So if a driver 91d91cab78SDave Hansen * asks about "XFEATURE_MASK_SSE | XFEATURE_MASK_YMM" we'll print the 925b073430SIngo Molnar * missing AVX feature - this is the most informative message 935b073430SIngo Molnar * to users: 945b073430SIngo Molnar */ 955b073430SIngo Molnar if (xfeatures_missing) 965b073430SIngo Molnar xfeatures_print = xfeatures_missing; 975b073430SIngo Molnar else 985b073430SIngo Molnar xfeatures_print = xfeatures_needed; 995b073430SIngo Molnar 1005b073430SIngo Molnar xfeature_idx = fls64(xfeatures_print)-1; 1015b073430SIngo Molnar max_idx = ARRAY_SIZE(xfeature_names)-1; 1025b073430SIngo Molnar xfeature_idx = min(xfeature_idx, max_idx); 1035b073430SIngo Molnar 1045b073430SIngo Molnar *feature_name = xfeature_names[xfeature_idx]; 1055b073430SIngo Molnar } 1065b073430SIngo Molnar 1075b073430SIngo Molnar if (xfeatures_missing) 1085b073430SIngo Molnar return 0; 1095b073430SIngo Molnar 1105b073430SIngo Molnar return 1; 1115b073430SIngo Molnar } 1125b073430SIngo Molnar EXPORT_SYMBOL_GPL(cpu_has_xfeatures); 1135b073430SIngo Molnar 114158e2ee6SYu-cheng Yu static bool xfeature_is_supervisor(int xfeature_nr) 1151499ce2dSYu-cheng Yu { 1161499ce2dSYu-cheng Yu /* 1178c9e6073SYu-cheng Yu * Extended State Enumeration Sub-leaves (EAX = 0DH, ECX = n, n > 1) 1188c9e6073SYu-cheng Yu * returns ECX[0] set to (1) for a supervisor state, and cleared (0) 1198c9e6073SYu-cheng Yu * for a user state. 1201499ce2dSYu-cheng Yu */ 1211499ce2dSYu-cheng Yu u32 eax, ebx, ecx, edx; 1221499ce2dSYu-cheng Yu 1231499ce2dSYu-cheng Yu cpuid_count(XSTATE_CPUID, xfeature_nr, &eax, &ebx, &ecx, &edx); 124158e2ee6SYu-cheng Yu return ecx & 1; 1251499ce2dSYu-cheng Yu } 1261499ce2dSYu-cheng Yu 1275b073430SIngo Molnar /* 128aeb997b9SIngo Molnar * When executing XSAVEOPT (or other optimized XSAVE instructions), if 129aeb997b9SIngo Molnar * a processor implementation detects that an FPU state component is still 130aeb997b9SIngo Molnar * (or is again) in its initialized state, it may clear the corresponding 131aeb997b9SIngo Molnar * bit in the header.xfeatures field, and can skip the writeout of registers 132aeb997b9SIngo Molnar * to the corresponding memory layout. 13362784854SIngo Molnar * 13462784854SIngo Molnar * This means that when the bit is zero, the state component might still contain 13562784854SIngo Molnar * some previous - non-initialized register state. 13662784854SIngo Molnar * 13762784854SIngo Molnar * Before writing xstate information to user-space we sanitize those components, 13862784854SIngo Molnar * to always ensure that the memory layout of a feature will be in the init state 13962784854SIngo Molnar * if the corresponding header bit is zero. This is to ensure that user-space doesn't 14062784854SIngo Molnar * see some stale state in the memory layout during signal handling, debugging etc. 14162784854SIngo Molnar */ 14236e49e7fSIngo Molnar void fpstate_sanitize_xstate(struct fpu *fpu) 14362784854SIngo Molnar { 144c47ada30SIngo Molnar struct fxregs_state *fx = &fpu->state.fxsave; 14562784854SIngo Molnar int feature_bit; 14662784854SIngo Molnar u64 xfeatures; 14762784854SIngo Molnar 1481ac91a76SIngo Molnar if (!use_xsaveopt()) 14962784854SIngo Molnar return; 15062784854SIngo Molnar 15136e49e7fSIngo Molnar xfeatures = fpu->state.xsave.header.xfeatures; 15262784854SIngo Molnar 15362784854SIngo Molnar /* 15462784854SIngo Molnar * None of the feature bits are in init state. So nothing else 15562784854SIngo Molnar * to do for us, as the memory layout is up to date. 15662784854SIngo Molnar */ 157524bb73bSYu-cheng Yu if ((xfeatures & xfeatures_mask_all) == xfeatures_mask_all) 15862784854SIngo Molnar return; 15962784854SIngo Molnar 16062784854SIngo Molnar /* 16162784854SIngo Molnar * FP is in init state 16262784854SIngo Molnar */ 163d91cab78SDave Hansen if (!(xfeatures & XFEATURE_MASK_FP)) { 16462784854SIngo Molnar fx->cwd = 0x37f; 16562784854SIngo Molnar fx->swd = 0; 16662784854SIngo Molnar fx->twd = 0; 16762784854SIngo Molnar fx->fop = 0; 16862784854SIngo Molnar fx->rip = 0; 16962784854SIngo Molnar fx->rdp = 0; 17062784854SIngo Molnar memset(&fx->st_space[0], 0, 128); 17162784854SIngo Molnar } 17262784854SIngo Molnar 17362784854SIngo Molnar /* 17462784854SIngo Molnar * SSE is in init state 17562784854SIngo Molnar */ 176d91cab78SDave Hansen if (!(xfeatures & XFEATURE_MASK_SSE)) 17762784854SIngo Molnar memset(&fx->xmm_space[0], 0, 256); 17862784854SIngo Molnar 17962784854SIngo Molnar /* 18062784854SIngo Molnar * First two features are FPU and SSE, which above we handled 18162784854SIngo Molnar * in a special way already: 18262784854SIngo Molnar */ 18362784854SIngo Molnar feature_bit = 0x2; 184524bb73bSYu-cheng Yu xfeatures = (xfeatures_mask_user() & ~xfeatures) >> 2; 18562784854SIngo Molnar 18662784854SIngo Molnar /* 18762784854SIngo Molnar * Update all the remaining memory layouts according to their 18862784854SIngo Molnar * standard xstate layout, if their header bit is in the init 18962784854SIngo Molnar * state: 19062784854SIngo Molnar */ 19162784854SIngo Molnar while (xfeatures) { 19262784854SIngo Molnar if (xfeatures & 0x1) { 193a1141e0bSFenghua Yu int offset = xstate_comp_offsets[feature_bit]; 19462784854SIngo Molnar int size = xstate_sizes[feature_bit]; 19562784854SIngo Molnar 19662784854SIngo Molnar memcpy((void *)fx + offset, 1976f575023SIngo Molnar (void *)&init_fpstate.xsave + offset, 19862784854SIngo Molnar size); 19962784854SIngo Molnar } 20062784854SIngo Molnar 20162784854SIngo Molnar xfeatures >>= 1; 20262784854SIngo Molnar feature_bit++; 20362784854SIngo Molnar } 20462784854SIngo Molnar } 20562784854SIngo Molnar 20662784854SIngo Molnar /* 20762784854SIngo Molnar * Enable the extended processor state save/restore feature. 20862784854SIngo Molnar * Called once per CPU onlining. 20962784854SIngo Molnar */ 21062784854SIngo Molnar void fpu__init_cpu_xstate(void) 21162784854SIngo Molnar { 212524bb73bSYu-cheng Yu u64 unsup_bits; 213524bb73bSYu-cheng Yu 214524bb73bSYu-cheng Yu if (!boot_cpu_has(X86_FEATURE_XSAVE) || !xfeatures_mask_all) 21562784854SIngo Molnar return; 216b8be15d5SYu-cheng Yu /* 2178ab22804SFenghua Yu * Unsupported supervisor xstates should not be found in 2188ab22804SFenghua Yu * the xfeatures mask. 219b8be15d5SYu-cheng Yu */ 220524bb73bSYu-cheng Yu unsup_bits = xfeatures_mask_all & XFEATURE_MASK_SUPERVISOR_UNSUPPORTED; 221524bb73bSYu-cheng Yu WARN_ONCE(unsup_bits, "x86/fpu: Found unsupported supervisor xstates: 0x%llx\n", 222524bb73bSYu-cheng Yu unsup_bits); 223b8be15d5SYu-cheng Yu 224524bb73bSYu-cheng Yu xfeatures_mask_all &= ~XFEATURE_MASK_SUPERVISOR_UNSUPPORTED; 22562784854SIngo Molnar 22662784854SIngo Molnar cr4_set_bits(X86_CR4_OSXSAVE); 22762784854SIngo Molnar 22862784854SIngo Molnar /* 229524bb73bSYu-cheng Yu * XCR_XFEATURE_ENABLED_MASK (aka. XCR0) sets user features 230524bb73bSYu-cheng Yu * managed by XSAVE{C, OPT, S} and XRSTOR{S}. Only XSAVE user 231524bb73bSYu-cheng Yu * states can be set here. 232e6e888f9SDave Hansen */ 233524bb73bSYu-cheng Yu xsetbv(XCR_XFEATURE_ENABLED_MASK, xfeatures_mask_user()); 23462784854SIngo Molnar 23562784854SIngo Molnar /* 23671581eefSYu-cheng Yu * MSR_IA32_XSS sets supervisor states managed by XSAVES. 237e6e888f9SDave Hansen */ 238f0dccc9dSKan Liang if (boot_cpu_has(X86_FEATURE_XSAVES)) { 239f0dccc9dSKan Liang wrmsrl(MSR_IA32_XSS, xfeatures_mask_supervisor() | 240f0dccc9dSKan Liang xfeatures_mask_dynamic()); 241f0dccc9dSKan Liang } 24271581eefSYu-cheng Yu } 24371581eefSYu-cheng Yu 244524bb73bSYu-cheng Yu static bool xfeature_enabled(enum xfeature xfeature) 245e6e888f9SDave Hansen { 246524bb73bSYu-cheng Yu return xfeatures_mask_all & BIT_ULL(xfeature); 247e6e888f9SDave Hansen } 248e6e888f9SDave Hansen 249e6e888f9SDave Hansen /* 25039f1acd2SIngo Molnar * Record the offsets and sizes of various xstates contained 25139f1acd2SIngo Molnar * in the XSAVE state memory layout. 25262784854SIngo Molnar */ 25362784854SIngo Molnar static void __init setup_xstate_features(void) 25462784854SIngo Molnar { 255ee9ae257SDave Hansen u32 eax, ebx, ecx, edx, i; 256e6e888f9SDave Hansen /* start at the beginnning of the "extended state" */ 257e6e888f9SDave Hansen unsigned int last_good_offset = offsetof(struct xregs_state, 258e6e888f9SDave Hansen extended_state_area); 259ac73b27aSYu-cheng Yu /* 260ac73b27aSYu-cheng Yu * The FP xstates and SSE xstates are legacy states. They are always 261ac73b27aSYu-cheng Yu * in the fixed offsets in the xsave area in either compacted form 262ac73b27aSYu-cheng Yu * or standard form. 263ac73b27aSYu-cheng Yu */ 264446e693cSCyrill Gorcunov xstate_offsets[XFEATURE_FP] = 0; 265446e693cSCyrill Gorcunov xstate_sizes[XFEATURE_FP] = offsetof(struct fxregs_state, 266446e693cSCyrill Gorcunov xmm_space); 267446e693cSCyrill Gorcunov 268446e693cSCyrill Gorcunov xstate_offsets[XFEATURE_SSE] = xstate_sizes[XFEATURE_FP]; 269c593642cSPankaj Bharadiya xstate_sizes[XFEATURE_SSE] = sizeof_field(struct fxregs_state, 270446e693cSCyrill Gorcunov xmm_space); 27162784854SIngo Molnar 272ee9ae257SDave Hansen for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) { 273e6e888f9SDave Hansen if (!xfeature_enabled(i)) 274e6e888f9SDave Hansen continue; 27562784854SIngo Molnar 276e6e888f9SDave Hansen cpuid_count(XSTATE_CPUID, i, &eax, &ebx, &ecx, &edx); 2771499ce2dSYu-cheng Yu 278c12e13dcSYu-cheng Yu xstate_sizes[i] = eax; 279c12e13dcSYu-cheng Yu 2801499ce2dSYu-cheng Yu /* 281c12e13dcSYu-cheng Yu * If an xfeature is supervisor state, the offset in EBX is 282c12e13dcSYu-cheng Yu * invalid, leave it to -1. 2831499ce2dSYu-cheng Yu */ 284c12e13dcSYu-cheng Yu if (xfeature_is_supervisor(i)) 285c12e13dcSYu-cheng Yu continue; 286c12e13dcSYu-cheng Yu 287ee9ae257SDave Hansen xstate_offsets[i] = ebx; 2881499ce2dSYu-cheng Yu 289e6e888f9SDave Hansen /* 290c12e13dcSYu-cheng Yu * In our xstate size checks, we assume that the highest-numbered 291c12e13dcSYu-cheng Yu * xstate feature has the highest offset in the buffer. Ensure 292c12e13dcSYu-cheng Yu * it does. 293e6e888f9SDave Hansen */ 294e6e888f9SDave Hansen WARN_ONCE(last_good_offset > xstate_offsets[i], 295e6e888f9SDave Hansen "x86/fpu: misordered xstate at %d\n", last_good_offset); 296c12e13dcSYu-cheng Yu 297e6e888f9SDave Hansen last_good_offset = xstate_offsets[i]; 29839f1acd2SIngo Molnar } 29962784854SIngo Molnar } 30062784854SIngo Molnar 30132231879SIngo Molnar static void __init print_xstate_feature(u64 xstate_mask) 30262784854SIngo Molnar { 30333588b52SIngo Molnar const char *feature_name; 30462784854SIngo Molnar 30533588b52SIngo Molnar if (cpu_has_xfeatures(xstate_mask, &feature_name)) 306c8df4009SDave Hansen pr_info("x86/fpu: Supporting XSAVE feature 0x%03Lx: '%s'\n", xstate_mask, feature_name); 30762784854SIngo Molnar } 30862784854SIngo Molnar 30962784854SIngo Molnar /* 31062784854SIngo Molnar * Print out all the supported xstate features: 31162784854SIngo Molnar */ 31232231879SIngo Molnar static void __init print_xstate_features(void) 31362784854SIngo Molnar { 314d91cab78SDave Hansen print_xstate_feature(XFEATURE_MASK_FP); 315d91cab78SDave Hansen print_xstate_feature(XFEATURE_MASK_SSE); 316d91cab78SDave Hansen print_xstate_feature(XFEATURE_MASK_YMM); 317d91cab78SDave Hansen print_xstate_feature(XFEATURE_MASK_BNDREGS); 318d91cab78SDave Hansen print_xstate_feature(XFEATURE_MASK_BNDCSR); 319d91cab78SDave Hansen print_xstate_feature(XFEATURE_MASK_OPMASK); 320d91cab78SDave Hansen print_xstate_feature(XFEATURE_MASK_ZMM_Hi256); 321d91cab78SDave Hansen print_xstate_feature(XFEATURE_MASK_Hi16_ZMM); 322c8df4009SDave Hansen print_xstate_feature(XFEATURE_MASK_PKRU); 323b454feb9SYu-cheng Yu print_xstate_feature(XFEATURE_MASK_PASID); 32462784854SIngo Molnar } 32562784854SIngo Molnar 32662784854SIngo Molnar /* 32703482e08SYu-cheng Yu * This check is important because it is easy to get XSTATE_* 32803482e08SYu-cheng Yu * confused with XSTATE_BIT_*. 32903482e08SYu-cheng Yu */ 33003482e08SYu-cheng Yu #define CHECK_XFEATURE(nr) do { \ 33103482e08SYu-cheng Yu WARN_ON(nr < FIRST_EXTENDED_XFEATURE); \ 33203482e08SYu-cheng Yu WARN_ON(nr >= XFEATURE_MAX); \ 33303482e08SYu-cheng Yu } while (0) 33403482e08SYu-cheng Yu 33503482e08SYu-cheng Yu /* 33603482e08SYu-cheng Yu * We could cache this like xstate_size[], but we only use 33703482e08SYu-cheng Yu * it here, so it would be a waste of space. 33803482e08SYu-cheng Yu */ 33903482e08SYu-cheng Yu static int xfeature_is_aligned(int xfeature_nr) 34003482e08SYu-cheng Yu { 34103482e08SYu-cheng Yu u32 eax, ebx, ecx, edx; 34203482e08SYu-cheng Yu 34303482e08SYu-cheng Yu CHECK_XFEATURE(xfeature_nr); 344e70b1008SYu-cheng Yu 345e70b1008SYu-cheng Yu if (!xfeature_enabled(xfeature_nr)) { 346e70b1008SYu-cheng Yu WARN_ONCE(1, "Checking alignment of disabled xfeature %d\n", 347e70b1008SYu-cheng Yu xfeature_nr); 348e70b1008SYu-cheng Yu return 0; 349e70b1008SYu-cheng Yu } 350e70b1008SYu-cheng Yu 35103482e08SYu-cheng Yu cpuid_count(XSTATE_CPUID, xfeature_nr, &eax, &ebx, &ecx, &edx); 35203482e08SYu-cheng Yu /* 35303482e08SYu-cheng Yu * The value returned by ECX[1] indicates the alignment 35403482e08SYu-cheng Yu * of state component 'i' when the compacted format 35503482e08SYu-cheng Yu * of the extended region of an XSAVE area is used: 35603482e08SYu-cheng Yu */ 35703482e08SYu-cheng Yu return !!(ecx & 2); 35803482e08SYu-cheng Yu } 35903482e08SYu-cheng Yu 36003482e08SYu-cheng Yu /* 36162784854SIngo Molnar * This function sets up offsets and sizes of all extended states in 36262784854SIngo Molnar * xsave area. This supports both standard format and compacted format 36349a91d61SYu-cheng Yu * of the xsave area. 36462784854SIngo Molnar */ 36549a91d61SYu-cheng Yu static void __init setup_xstate_comp_offsets(void) 36662784854SIngo Molnar { 36749a91d61SYu-cheng Yu unsigned int next_offset; 36862784854SIngo Molnar int i; 36962784854SIngo Molnar 37062784854SIngo Molnar /* 37162784854SIngo Molnar * The FP xstates and SSE xstates are legacy states. They are always 37262784854SIngo Molnar * in the fixed offsets in the xsave area in either compacted form 37362784854SIngo Molnar * or standard form. 37462784854SIngo Molnar */ 375446e693cSCyrill Gorcunov xstate_comp_offsets[XFEATURE_FP] = 0; 376446e693cSCyrill Gorcunov xstate_comp_offsets[XFEATURE_SSE] = offsetof(struct fxregs_state, 377446e693cSCyrill Gorcunov xmm_space); 37862784854SIngo Molnar 379782511b0SBorislav Petkov if (!boot_cpu_has(X86_FEATURE_XSAVES)) { 380ee9ae257SDave Hansen for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) { 38149a91d61SYu-cheng Yu if (xfeature_enabled(i)) 38262784854SIngo Molnar xstate_comp_offsets[i] = xstate_offsets[i]; 38362784854SIngo Molnar } 38462784854SIngo Molnar return; 38562784854SIngo Molnar } 38662784854SIngo Molnar 38749a91d61SYu-cheng Yu next_offset = FXSAVE_SIZE + XSAVE_HDR_SIZE; 38862784854SIngo Molnar 389ee9ae257SDave Hansen for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) { 39049a91d61SYu-cheng Yu if (!xfeature_enabled(i)) 39149a91d61SYu-cheng Yu continue; 39262784854SIngo Molnar 39303482e08SYu-cheng Yu if (xfeature_is_aligned(i)) 39449a91d61SYu-cheng Yu next_offset = ALIGN(next_offset, 64); 39549a91d61SYu-cheng Yu 39649a91d61SYu-cheng Yu xstate_comp_offsets[i] = next_offset; 39749a91d61SYu-cheng Yu next_offset += xstate_sizes[i]; 39862784854SIngo Molnar } 39962784854SIngo Molnar } 40062784854SIngo Molnar 40162784854SIngo Molnar /* 402eeedf153SYu-cheng Yu * Setup offsets of a supervisor-state-only XSAVES buffer: 403eeedf153SYu-cheng Yu * 404eeedf153SYu-cheng Yu * The offsets stored in xstate_comp_offsets[] only work for one specific 405eeedf153SYu-cheng Yu * value of the Requested Feature BitMap (RFBM). In cases where a different 406eeedf153SYu-cheng Yu * RFBM value is used, a different set of offsets is required. This set of 407eeedf153SYu-cheng Yu * offsets is for when RFBM=xfeatures_mask_supervisor(). 408eeedf153SYu-cheng Yu */ 409eeedf153SYu-cheng Yu static void __init setup_supervisor_only_offsets(void) 410eeedf153SYu-cheng Yu { 411eeedf153SYu-cheng Yu unsigned int next_offset; 412eeedf153SYu-cheng Yu int i; 413eeedf153SYu-cheng Yu 414eeedf153SYu-cheng Yu next_offset = FXSAVE_SIZE + XSAVE_HDR_SIZE; 415eeedf153SYu-cheng Yu 416eeedf153SYu-cheng Yu for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) { 417eeedf153SYu-cheng Yu if (!xfeature_enabled(i) || !xfeature_is_supervisor(i)) 418eeedf153SYu-cheng Yu continue; 419eeedf153SYu-cheng Yu 420eeedf153SYu-cheng Yu if (xfeature_is_aligned(i)) 421eeedf153SYu-cheng Yu next_offset = ALIGN(next_offset, 64); 422eeedf153SYu-cheng Yu 423eeedf153SYu-cheng Yu xstate_supervisor_only_offsets[i] = next_offset; 424eeedf153SYu-cheng Yu next_offset += xstate_sizes[i]; 425eeedf153SYu-cheng Yu } 426eeedf153SYu-cheng Yu } 427eeedf153SYu-cheng Yu 428eeedf153SYu-cheng Yu /* 429996952e0SYu-cheng Yu * Print out xstate component offsets and sizes 430996952e0SYu-cheng Yu */ 431996952e0SYu-cheng Yu static void __init print_xstate_offset_size(void) 432996952e0SYu-cheng Yu { 433996952e0SYu-cheng Yu int i; 434996952e0SYu-cheng Yu 435996952e0SYu-cheng Yu for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) { 436996952e0SYu-cheng Yu if (!xfeature_enabled(i)) 437996952e0SYu-cheng Yu continue; 438996952e0SYu-cheng Yu pr_info("x86/fpu: xstate_offset[%d]: %4d, xstate_sizes[%d]: %4d\n", 439996952e0SYu-cheng Yu i, xstate_comp_offsets[i], i, xstate_sizes[i]); 440996952e0SYu-cheng Yu } 441996952e0SYu-cheng Yu } 442996952e0SYu-cheng Yu 443996952e0SYu-cheng Yu /* 44462784854SIngo Molnar * setup the xstate image representing the init state 44562784854SIngo Molnar */ 44632231879SIngo Molnar static void __init setup_init_fpu_buf(void) 44762784854SIngo Molnar { 448e49a449bSRasmus Villemoes static int on_boot_cpu __initdata = 1; 449e97131a8SIngo Molnar 450e97131a8SIngo Molnar WARN_ON_FPU(!on_boot_cpu); 451e97131a8SIngo Molnar on_boot_cpu = 0; 452e97131a8SIngo Molnar 453d366bf7eSBorislav Petkov if (!boot_cpu_has(X86_FEATURE_XSAVE)) 45462784854SIngo Molnar return; 45562784854SIngo Molnar 45662784854SIngo Molnar setup_xstate_features(); 45762784854SIngo Molnar print_xstate_features(); 45862784854SIngo Molnar 4597d937060SFenghua Yu if (boot_cpu_has(X86_FEATURE_XSAVES)) 4608c9e6073SYu-cheng Yu init_fpstate.xsave.header.xcomp_bv = XCOMP_BV_COMPACTED_FORMAT | 461524bb73bSYu-cheng Yu xfeatures_mask_all; 46262784854SIngo Molnar 46362784854SIngo Molnar /* 4647d937060SFenghua Yu * Init all the features state with header.xfeatures being 0x0 46562784854SIngo Molnar */ 466d65fcd60SIngo Molnar copy_kernel_to_xregs_booting(&init_fpstate.xsave); 46762784854SIngo Molnar 46862784854SIngo Molnar /* 46962784854SIngo Molnar * Dump the init state again. This is to identify the init state 47062784854SIngo Molnar * of any feature which is not represented by all zero's. 47162784854SIngo Molnar */ 472c6813144SIngo Molnar copy_xregs_to_kernel_booting(&init_fpstate.xsave); 47362784854SIngo Molnar } 47462784854SIngo Molnar 47565ac2e9bSDave Hansen static int xfeature_uncompacted_offset(int xfeature_nr) 47665ac2e9bSDave Hansen { 47765ac2e9bSDave Hansen u32 eax, ebx, ecx, edx; 47865ac2e9bSDave Hansen 4791499ce2dSYu-cheng Yu /* 4801499ce2dSYu-cheng Yu * Only XSAVES supports supervisor states and it uses compacted 4811499ce2dSYu-cheng Yu * format. Checking a supervisor state's uncompacted offset is 4821499ce2dSYu-cheng Yu * an error. 4831499ce2dSYu-cheng Yu */ 4848ab22804SFenghua Yu if (XFEATURE_MASK_SUPERVISOR_ALL & BIT_ULL(xfeature_nr)) { 4851499ce2dSYu-cheng Yu WARN_ONCE(1, "No fixed offset for xstate %d\n", xfeature_nr); 4861499ce2dSYu-cheng Yu return -1; 4871499ce2dSYu-cheng Yu } 4881499ce2dSYu-cheng Yu 48965ac2e9bSDave Hansen CHECK_XFEATURE(xfeature_nr); 49065ac2e9bSDave Hansen cpuid_count(XSTATE_CPUID, xfeature_nr, &eax, &ebx, &ecx, &edx); 49165ac2e9bSDave Hansen return ebx; 49265ac2e9bSDave Hansen } 49365ac2e9bSDave Hansen 494ce711ea3SKan Liang int xfeature_size(int xfeature_nr) 49565ac2e9bSDave Hansen { 49665ac2e9bSDave Hansen u32 eax, ebx, ecx, edx; 49765ac2e9bSDave Hansen 49865ac2e9bSDave Hansen CHECK_XFEATURE(xfeature_nr); 49965ac2e9bSDave Hansen cpuid_count(XSTATE_CPUID, xfeature_nr, &eax, &ebx, &ecx, &edx); 50065ac2e9bSDave Hansen return eax; 50165ac2e9bSDave Hansen } 50265ac2e9bSDave Hansen 50365ac2e9bSDave Hansen /* 50465ac2e9bSDave Hansen * 'XSAVES' implies two different things: 50565ac2e9bSDave Hansen * 1. saving of supervisor/system state 50665ac2e9bSDave Hansen * 2. using the compacted format 50765ac2e9bSDave Hansen * 50865ac2e9bSDave Hansen * Use this function when dealing with the compacted format so 50965ac2e9bSDave Hansen * that it is obvious which aspect of 'XSAVES' is being handled 51065ac2e9bSDave Hansen * by the calling code. 51165ac2e9bSDave Hansen */ 51299aa22d0SYu-cheng Yu int using_compacted_format(void) 51365ac2e9bSDave Hansen { 514782511b0SBorislav Petkov return boot_cpu_has(X86_FEATURE_XSAVES); 51565ac2e9bSDave Hansen } 51665ac2e9bSDave Hansen 517e63e5d5cSEric Biggers /* Validate an xstate header supplied by userspace (ptrace or sigreturn) */ 5185274e6c1SFenghua Yu int validate_user_xstate_header(const struct xstate_header *hdr) 519e63e5d5cSEric Biggers { 520e63e5d5cSEric Biggers /* No unknown or supervisor features may be set */ 521524bb73bSYu-cheng Yu if (hdr->xfeatures & ~xfeatures_mask_user()) 522e63e5d5cSEric Biggers return -EINVAL; 523e63e5d5cSEric Biggers 524e63e5d5cSEric Biggers /* Userspace must use the uncompacted format */ 525e63e5d5cSEric Biggers if (hdr->xcomp_bv) 526e63e5d5cSEric Biggers return -EINVAL; 527e63e5d5cSEric Biggers 528e63e5d5cSEric Biggers /* 529e63e5d5cSEric Biggers * If 'reserved' is shrunken to add a new field, make sure to validate 530e63e5d5cSEric Biggers * that new field here! 531e63e5d5cSEric Biggers */ 532e63e5d5cSEric Biggers BUILD_BUG_ON(sizeof(hdr->reserved) != 48); 533e63e5d5cSEric Biggers 534e63e5d5cSEric Biggers /* No reserved bits may be set */ 535e63e5d5cSEric Biggers if (memchr_inv(hdr->reserved, 0, sizeof(hdr->reserved))) 536e63e5d5cSEric Biggers return -EINVAL; 537e63e5d5cSEric Biggers 538e63e5d5cSEric Biggers return 0; 539e63e5d5cSEric Biggers } 540e63e5d5cSEric Biggers 54165ac2e9bSDave Hansen static void __xstate_dump_leaves(void) 54265ac2e9bSDave Hansen { 54365ac2e9bSDave Hansen int i; 54465ac2e9bSDave Hansen u32 eax, ebx, ecx, edx; 54565ac2e9bSDave Hansen static int should_dump = 1; 54665ac2e9bSDave Hansen 54765ac2e9bSDave Hansen if (!should_dump) 54865ac2e9bSDave Hansen return; 54965ac2e9bSDave Hansen should_dump = 0; 55065ac2e9bSDave Hansen /* 55165ac2e9bSDave Hansen * Dump out a few leaves past the ones that we support 55265ac2e9bSDave Hansen * just in case there are some goodies up there 55365ac2e9bSDave Hansen */ 55465ac2e9bSDave Hansen for (i = 0; i < XFEATURE_MAX + 10; i++) { 55565ac2e9bSDave Hansen cpuid_count(XSTATE_CPUID, i, &eax, &ebx, &ecx, &edx); 55665ac2e9bSDave Hansen pr_warn("CPUID[%02x, %02x]: eax=%08x ebx=%08x ecx=%08x edx=%08x\n", 55765ac2e9bSDave Hansen XSTATE_CPUID, i, eax, ebx, ecx, edx); 55865ac2e9bSDave Hansen } 55965ac2e9bSDave Hansen } 56065ac2e9bSDave Hansen 56165ac2e9bSDave Hansen #define XSTATE_WARN_ON(x) do { \ 56265ac2e9bSDave Hansen if (WARN_ONCE(x, "XSAVE consistency problem, dumping leaves")) { \ 56365ac2e9bSDave Hansen __xstate_dump_leaves(); \ 56465ac2e9bSDave Hansen } \ 56565ac2e9bSDave Hansen } while (0) 56665ac2e9bSDave Hansen 567ef78f2a4SDave Hansen #define XCHECK_SZ(sz, nr, nr_macro, __struct) do { \ 568ef78f2a4SDave Hansen if ((nr == nr_macro) && \ 569ef78f2a4SDave Hansen WARN_ONCE(sz != sizeof(__struct), \ 570ef78f2a4SDave Hansen "%s: struct is %zu bytes, cpu state %d bytes\n", \ 571ef78f2a4SDave Hansen __stringify(nr_macro), sizeof(__struct), sz)) { \ 572ef78f2a4SDave Hansen __xstate_dump_leaves(); \ 573ef78f2a4SDave Hansen } \ 574ef78f2a4SDave Hansen } while (0) 575ef78f2a4SDave Hansen 576ef78f2a4SDave Hansen /* 577ef78f2a4SDave Hansen * We have a C struct for each 'xstate'. We need to ensure 578ef78f2a4SDave Hansen * that our software representation matches what the CPU 579ef78f2a4SDave Hansen * tells us about the state's size. 580ef78f2a4SDave Hansen */ 581ef78f2a4SDave Hansen static void check_xstate_against_struct(int nr) 582ef78f2a4SDave Hansen { 583ef78f2a4SDave Hansen /* 584ef78f2a4SDave Hansen * Ask the CPU for the size of the state. 585ef78f2a4SDave Hansen */ 586ef78f2a4SDave Hansen int sz = xfeature_size(nr); 587ef78f2a4SDave Hansen /* 588ef78f2a4SDave Hansen * Match each CPU state with the corresponding software 589ef78f2a4SDave Hansen * structure. 590ef78f2a4SDave Hansen */ 591ef78f2a4SDave Hansen XCHECK_SZ(sz, nr, XFEATURE_YMM, struct ymmh_struct); 592ef78f2a4SDave Hansen XCHECK_SZ(sz, nr, XFEATURE_BNDREGS, struct mpx_bndreg_state); 593ef78f2a4SDave Hansen XCHECK_SZ(sz, nr, XFEATURE_BNDCSR, struct mpx_bndcsr_state); 594ef78f2a4SDave Hansen XCHECK_SZ(sz, nr, XFEATURE_OPMASK, struct avx_512_opmask_state); 595ef78f2a4SDave Hansen XCHECK_SZ(sz, nr, XFEATURE_ZMM_Hi256, struct avx_512_zmm_uppers_state); 596ef78f2a4SDave Hansen XCHECK_SZ(sz, nr, XFEATURE_Hi16_ZMM, struct avx_512_hi16_state); 597c8df4009SDave Hansen XCHECK_SZ(sz, nr, XFEATURE_PKRU, struct pkru_state); 598b454feb9SYu-cheng Yu XCHECK_SZ(sz, nr, XFEATURE_PASID, struct ia32_pasid_state); 599ef78f2a4SDave Hansen 600ef78f2a4SDave Hansen /* 601ef78f2a4SDave Hansen * Make *SURE* to add any feature numbers in below if 602ef78f2a4SDave Hansen * there are "holes" in the xsave state component 603ef78f2a4SDave Hansen * numbers. 604ef78f2a4SDave Hansen */ 605ef78f2a4SDave Hansen if ((nr < XFEATURE_YMM) || 6061f96b1efSDave Hansen (nr >= XFEATURE_MAX) || 607f0dccc9dSKan Liang (nr == XFEATURE_PT_UNIMPLEMENTED_SO_FAR) || 608b454feb9SYu-cheng Yu ((nr >= XFEATURE_RSRVD_COMP_11) && (nr <= XFEATURE_LBR))) { 609ef78f2a4SDave Hansen WARN_ONCE(1, "no structure for xstate: %d\n", nr); 610ef78f2a4SDave Hansen XSTATE_WARN_ON(1); 611ef78f2a4SDave Hansen } 612ef78f2a4SDave Hansen } 613ef78f2a4SDave Hansen 61465ac2e9bSDave Hansen /* 61565ac2e9bSDave Hansen * This essentially double-checks what the cpu told us about 61665ac2e9bSDave Hansen * how large the XSAVE buffer needs to be. We are recalculating 61765ac2e9bSDave Hansen * it to be safe. 61876d10256SKan Liang * 61976d10256SKan Liang * Dynamic XSAVE features allocate their own buffers and are not 62076d10256SKan Liang * covered by these checks. Only the size of the buffer for task->fpu 62176d10256SKan Liang * is checked here. 62265ac2e9bSDave Hansen */ 62365ac2e9bSDave Hansen static void do_extra_xstate_size_checks(void) 62465ac2e9bSDave Hansen { 62565ac2e9bSDave Hansen int paranoid_xstate_size = FXSAVE_SIZE + XSAVE_HDR_SIZE; 62665ac2e9bSDave Hansen int i; 62765ac2e9bSDave Hansen 62865ac2e9bSDave Hansen for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) { 62965ac2e9bSDave Hansen if (!xfeature_enabled(i)) 63065ac2e9bSDave Hansen continue; 631ef78f2a4SDave Hansen 632ef78f2a4SDave Hansen check_xstate_against_struct(i); 63365ac2e9bSDave Hansen /* 63465ac2e9bSDave Hansen * Supervisor state components can be managed only by 63565ac2e9bSDave Hansen * XSAVES, which is compacted-format only. 63665ac2e9bSDave Hansen */ 63765ac2e9bSDave Hansen if (!using_compacted_format()) 63865ac2e9bSDave Hansen XSTATE_WARN_ON(xfeature_is_supervisor(i)); 63965ac2e9bSDave Hansen 64065ac2e9bSDave Hansen /* Align from the end of the previous feature */ 64165ac2e9bSDave Hansen if (xfeature_is_aligned(i)) 64265ac2e9bSDave Hansen paranoid_xstate_size = ALIGN(paranoid_xstate_size, 64); 64365ac2e9bSDave Hansen /* 64465ac2e9bSDave Hansen * The offset of a given state in the non-compacted 64565ac2e9bSDave Hansen * format is given to us in a CPUID leaf. We check 64665ac2e9bSDave Hansen * them for being ordered (increasing offsets) in 64765ac2e9bSDave Hansen * setup_xstate_features(). 64865ac2e9bSDave Hansen */ 64965ac2e9bSDave Hansen if (!using_compacted_format()) 65065ac2e9bSDave Hansen paranoid_xstate_size = xfeature_uncompacted_offset(i); 65165ac2e9bSDave Hansen /* 65265ac2e9bSDave Hansen * The compacted-format offset always depends on where 65365ac2e9bSDave Hansen * the previous state ended. 65465ac2e9bSDave Hansen */ 65565ac2e9bSDave Hansen paranoid_xstate_size += xfeature_size(i); 65665ac2e9bSDave Hansen } 657bf15a8cfSFenghua Yu XSTATE_WARN_ON(paranoid_xstate_size != fpu_kernel_xstate_size); 65865ac2e9bSDave Hansen } 65965ac2e9bSDave Hansen 660a1141e0bSFenghua Yu 66162784854SIngo Molnar /* 662524bb73bSYu-cheng Yu * Get total size of enabled xstates in XCR0 | IA32_XSS. 66365ac2e9bSDave Hansen * 66465ac2e9bSDave Hansen * Note the SDM's wording here. "sub-function 0" only enumerates 66565ac2e9bSDave Hansen * the size of the *user* states. If we use it to size a buffer 66665ac2e9bSDave Hansen * that we use 'XSAVES' on, we could potentially overflow the 66765ac2e9bSDave Hansen * buffer because 'XSAVES' saves system states too. 66862784854SIngo Molnar */ 669a1141e0bSFenghua Yu static unsigned int __init get_xsaves_size(void) 67062784854SIngo Molnar { 67162784854SIngo Molnar unsigned int eax, ebx, ecx, edx; 67265ac2e9bSDave Hansen /* 67365ac2e9bSDave Hansen * - CPUID function 0DH, sub-function 1: 67465ac2e9bSDave Hansen * EBX enumerates the size (in bytes) required by 67565ac2e9bSDave Hansen * the XSAVES instruction for an XSAVE area 67665ac2e9bSDave Hansen * containing all the state components 67765ac2e9bSDave Hansen * corresponding to bits currently set in 67865ac2e9bSDave Hansen * XCR0 | IA32_XSS. 67965ac2e9bSDave Hansen */ 68065ac2e9bSDave Hansen cpuid_count(XSTATE_CPUID, 1, &eax, &ebx, &ecx, &edx); 681a1141e0bSFenghua Yu return ebx; 68262784854SIngo Molnar } 683a1141e0bSFenghua Yu 68476d10256SKan Liang /* 68576d10256SKan Liang * Get the total size of the enabled xstates without the dynamic supervisor 68676d10256SKan Liang * features. 68776d10256SKan Liang */ 68876d10256SKan Liang static unsigned int __init get_xsaves_size_no_dynamic(void) 68976d10256SKan Liang { 69076d10256SKan Liang u64 mask = xfeatures_mask_dynamic(); 69176d10256SKan Liang unsigned int size; 69276d10256SKan Liang 69376d10256SKan Liang if (!mask) 69476d10256SKan Liang return get_xsaves_size(); 69576d10256SKan Liang 69676d10256SKan Liang /* Disable dynamic features. */ 69776d10256SKan Liang wrmsrl(MSR_IA32_XSS, xfeatures_mask_supervisor()); 69876d10256SKan Liang 69976d10256SKan Liang /* 70076d10256SKan Liang * Ask the hardware what size is required of the buffer. 70176d10256SKan Liang * This is the size required for the task->fpu buffer. 70276d10256SKan Liang */ 70376d10256SKan Liang size = get_xsaves_size(); 70476d10256SKan Liang 70576d10256SKan Liang /* Re-enable dynamic features so XSAVES will work on them again. */ 70676d10256SKan Liang wrmsrl(MSR_IA32_XSS, xfeatures_mask_supervisor() | mask); 70776d10256SKan Liang 70876d10256SKan Liang return size; 70976d10256SKan Liang } 71076d10256SKan Liang 711a1141e0bSFenghua Yu static unsigned int __init get_xsave_size(void) 712a1141e0bSFenghua Yu { 713a1141e0bSFenghua Yu unsigned int eax, ebx, ecx, edx; 714a1141e0bSFenghua Yu /* 715a1141e0bSFenghua Yu * - CPUID function 0DH, sub-function 0: 716a1141e0bSFenghua Yu * EBX enumerates the size (in bytes) required by 717a1141e0bSFenghua Yu * the XSAVE instruction for an XSAVE area 718a1141e0bSFenghua Yu * containing all the *user* state components 719a1141e0bSFenghua Yu * corresponding to bits currently set in XCR0. 720a1141e0bSFenghua Yu */ 721a1141e0bSFenghua Yu cpuid_count(XSTATE_CPUID, 0, &eax, &ebx, &ecx, &edx); 722a1141e0bSFenghua Yu return ebx; 7234109ca06SDave Hansen } 7244109ca06SDave Hansen 7254109ca06SDave Hansen /* 7264109ca06SDave Hansen * Will the runtime-enumerated 'xstate_size' fit in the init 7274109ca06SDave Hansen * task's statically-allocated buffer? 7284109ca06SDave Hansen */ 7294109ca06SDave Hansen static bool is_supported_xstate_size(unsigned int test_xstate_size) 7304109ca06SDave Hansen { 7314109ca06SDave Hansen if (test_xstate_size <= sizeof(union fpregs_state)) 7324109ca06SDave Hansen return true; 7334109ca06SDave Hansen 7344109ca06SDave Hansen pr_warn("x86/fpu: xstate buffer too small (%zu < %d), disabling xsave\n", 7354109ca06SDave Hansen sizeof(union fpregs_state), test_xstate_size); 7364109ca06SDave Hansen return false; 7374109ca06SDave Hansen } 7384109ca06SDave Hansen 739653a561bSSergey Senozhatsky static int __init init_xstate_size(void) 7404109ca06SDave Hansen { 7414109ca06SDave Hansen /* Recompute the context size for enabled features: */ 742a1141e0bSFenghua Yu unsigned int possible_xstate_size; 743a1141e0bSFenghua Yu unsigned int xsave_size; 744a1141e0bSFenghua Yu 745a1141e0bSFenghua Yu xsave_size = get_xsave_size(); 746a1141e0bSFenghua Yu 747a1141e0bSFenghua Yu if (boot_cpu_has(X86_FEATURE_XSAVES)) 74876d10256SKan Liang possible_xstate_size = get_xsaves_size_no_dynamic(); 749a1141e0bSFenghua Yu else 750a1141e0bSFenghua Yu possible_xstate_size = xsave_size; 7514109ca06SDave Hansen 7524109ca06SDave Hansen /* Ensure we have the space to store all enabled: */ 7534109ca06SDave Hansen if (!is_supported_xstate_size(possible_xstate_size)) 7544109ca06SDave Hansen return -EINVAL; 7554109ca06SDave Hansen 7564109ca06SDave Hansen /* 7574109ca06SDave Hansen * The size is OK, we are definitely going to use xsave, 7584109ca06SDave Hansen * make it known to the world that we need more space. 7594109ca06SDave Hansen */ 760bf15a8cfSFenghua Yu fpu_kernel_xstate_size = possible_xstate_size; 76165ac2e9bSDave Hansen do_extra_xstate_size_checks(); 762a1141e0bSFenghua Yu 763a1141e0bSFenghua Yu /* 764a1141e0bSFenghua Yu * User space is always in standard format. 765a1141e0bSFenghua Yu */ 766a1141e0bSFenghua Yu fpu_user_xstate_size = xsave_size; 7674109ca06SDave Hansen return 0; 7684109ca06SDave Hansen } 7694109ca06SDave Hansen 770d91cab78SDave Hansen /* 771d91cab78SDave Hansen * We enabled the XSAVE hardware, but something went wrong and 772d91cab78SDave Hansen * we can not use it. Disable it. 773d91cab78SDave Hansen */ 774d91cab78SDave Hansen static void fpu__init_disable_system_xstate(void) 7754109ca06SDave Hansen { 776524bb73bSYu-cheng Yu xfeatures_mask_all = 0; 7774109ca06SDave Hansen cr4_clear_bits(X86_CR4_OSXSAVE); 7787891bc0aSSebastian Andrzej Siewior setup_clear_cpu_cap(X86_FEATURE_XSAVE); 77962784854SIngo Molnar } 78062784854SIngo Molnar 78162784854SIngo Molnar /* 78262784854SIngo Molnar * Enable and initialize the xsave feature. 78362784854SIngo Molnar * Called once per system bootup. 78462784854SIngo Molnar */ 78532231879SIngo Molnar void __init fpu__init_system_xstate(void) 78662784854SIngo Molnar { 78762784854SIngo Molnar unsigned int eax, ebx, ecx, edx; 788e49a449bSRasmus Villemoes static int on_boot_cpu __initdata = 1; 7894109ca06SDave Hansen int err; 790ccb18db2SAndi Kleen int i; 791e97131a8SIngo Molnar 792e97131a8SIngo Molnar WARN_ON_FPU(!on_boot_cpu); 793e97131a8SIngo Molnar on_boot_cpu = 0; 79462784854SIngo Molnar 7959170fb40SAndy Lutomirski if (!boot_cpu_has(X86_FEATURE_FPU)) { 7969170fb40SAndy Lutomirski pr_info("x86/fpu: No FPU detected\n"); 7979170fb40SAndy Lutomirski return; 7989170fb40SAndy Lutomirski } 7999170fb40SAndy Lutomirski 800d366bf7eSBorislav Petkov if (!boot_cpu_has(X86_FEATURE_XSAVE)) { 8019170fb40SAndy Lutomirski pr_info("x86/fpu: x87 FPU will use %s\n", 8029170fb40SAndy Lutomirski boot_cpu_has(X86_FEATURE_FXSR) ? "FXSAVE" : "FSAVE"); 80362784854SIngo Molnar return; 80462784854SIngo Molnar } 80562784854SIngo Molnar 80662784854SIngo Molnar if (boot_cpu_data.cpuid_level < XSTATE_CPUID) { 807e97131a8SIngo Molnar WARN_ON_FPU(1); 80862784854SIngo Molnar return; 80962784854SIngo Molnar } 81062784854SIngo Molnar 811524bb73bSYu-cheng Yu /* 812524bb73bSYu-cheng Yu * Find user xstates supported by the processor. 813524bb73bSYu-cheng Yu */ 81462784854SIngo Molnar cpuid_count(XSTATE_CPUID, 0, &eax, &ebx, &ecx, &edx); 815524bb73bSYu-cheng Yu xfeatures_mask_all = eax + ((u64)edx << 32); 81662784854SIngo Molnar 81771581eefSYu-cheng Yu /* 81871581eefSYu-cheng Yu * Find supervisor xstates supported by the processor. 81971581eefSYu-cheng Yu */ 82071581eefSYu-cheng Yu cpuid_count(XSTATE_CPUID, 1, &eax, &ebx, &ecx, &edx); 82171581eefSYu-cheng Yu xfeatures_mask_all |= ecx + ((u64)edx << 32); 82271581eefSYu-cheng Yu 823524bb73bSYu-cheng Yu if ((xfeatures_mask_user() & XFEATURE_MASK_FPSSE) != XFEATURE_MASK_FPSSE) { 824ec3ed4a2SDave Hansen /* 825ec3ed4a2SDave Hansen * This indicates that something really unexpected happened 826ec3ed4a2SDave Hansen * with the enumeration. Disable XSAVE and try to continue 827ec3ed4a2SDave Hansen * booting without it. This is too early to BUG(). 828ec3ed4a2SDave Hansen */ 829524bb73bSYu-cheng Yu pr_err("x86/fpu: FP/SSE not present amongst the CPU's xstate features: 0x%llx.\n", 830524bb73bSYu-cheng Yu xfeatures_mask_all); 831ec3ed4a2SDave Hansen goto out_disable; 83262784854SIngo Molnar } 83362784854SIngo Molnar 834ccb18db2SAndi Kleen /* 835ccb18db2SAndi Kleen * Clear XSAVE features that are disabled in the normal CPUID. 836ccb18db2SAndi Kleen */ 837ccb18db2SAndi Kleen for (i = 0; i < ARRAY_SIZE(xsave_cpuid_features); i++) { 838ccb18db2SAndi Kleen if (!boot_cpu_has(xsave_cpuid_features[i])) 839524bb73bSYu-cheng Yu xfeatures_mask_all &= ~BIT_ULL(i); 840ccb18db2SAndi Kleen } 841ccb18db2SAndi Kleen 842524bb73bSYu-cheng Yu xfeatures_mask_all &= fpu__get_supported_xfeatures_mask(); 84362784854SIngo Molnar 84462784854SIngo Molnar /* Enable xstate instructions to be able to continue with initialization: */ 84562784854SIngo Molnar fpu__init_cpu_xstate(); 8464109ca06SDave Hansen err = init_xstate_size(); 847ec3ed4a2SDave Hansen if (err) 848ec3ed4a2SDave Hansen goto out_disable; 84962784854SIngo Molnar 85091c3dba7SYu-cheng Yu /* 85191c3dba7SYu-cheng Yu * Update info used for ptrace frames; use standard-format size and no 85291c3dba7SYu-cheng Yu * supervisor xstates: 85391c3dba7SYu-cheng Yu */ 854524bb73bSYu-cheng Yu update_regset_xstate_info(fpu_user_xstate_size, xfeatures_mask_user()); 85591c3dba7SYu-cheng Yu 856b992c660SIngo Molnar fpu__init_prepare_fx_sw_frame(); 85762784854SIngo Molnar setup_init_fpu_buf(); 85849a91d61SYu-cheng Yu setup_xstate_comp_offsets(); 859eeedf153SYu-cheng Yu setup_supervisor_only_offsets(); 860996952e0SYu-cheng Yu print_xstate_offset_size(); 86162784854SIngo Molnar 862b0815359SDave Hansen pr_info("x86/fpu: Enabled xstate features 0x%llx, context size is %d bytes, using '%s' format.\n", 863524bb73bSYu-cheng Yu xfeatures_mask_all, 864bf15a8cfSFenghua Yu fpu_kernel_xstate_size, 865782511b0SBorislav Petkov boot_cpu_has(X86_FEATURE_XSAVES) ? "compacted" : "standard"); 866ec3ed4a2SDave Hansen return; 867ec3ed4a2SDave Hansen 868ec3ed4a2SDave Hansen out_disable: 869ec3ed4a2SDave Hansen /* something went wrong, try to boot without any XSAVE support */ 870ec3ed4a2SDave Hansen fpu__init_disable_system_xstate(); 87162784854SIngo Molnar } 87262784854SIngo Molnar 87362784854SIngo Molnar /* 87462784854SIngo Molnar * Restore minimal FPU state after suspend: 87562784854SIngo Molnar */ 87662784854SIngo Molnar void fpu__resume_cpu(void) 87762784854SIngo Molnar { 87862784854SIngo Molnar /* 87962784854SIngo Molnar * Restore XCR0 on xsave capable CPUs: 88062784854SIngo Molnar */ 881d366bf7eSBorislav Petkov if (boot_cpu_has(X86_FEATURE_XSAVE)) 882524bb73bSYu-cheng Yu xsetbv(XCR_XFEATURE_ENABLED_MASK, xfeatures_mask_user()); 88371581eefSYu-cheng Yu 88471581eefSYu-cheng Yu /* 88571581eefSYu-cheng Yu * Restore IA32_XSS. The same CPUID bit enumerates support 88671581eefSYu-cheng Yu * of XSAVES and MSR_IA32_XSS. 88771581eefSYu-cheng Yu */ 888f0dccc9dSKan Liang if (boot_cpu_has(X86_FEATURE_XSAVES)) { 889f0dccc9dSKan Liang wrmsrl(MSR_IA32_XSS, xfeatures_mask_supervisor() | 890f0dccc9dSKan Liang xfeatures_mask_dynamic()); 891f0dccc9dSKan Liang } 89262784854SIngo Molnar } 89362784854SIngo Molnar 89462784854SIngo Molnar /* 89507baeb04SSebastian Andrzej Siewior * Given an xstate feature nr, calculate where in the xsave 896b8b9b6baSDave Hansen * buffer the state is. Callers should ensure that the buffer 897b8b9b6baSDave Hansen * is valid. 898b8b9b6baSDave Hansen */ 89907baeb04SSebastian Andrzej Siewior static void *__raw_xsave_addr(struct xregs_state *xsave, int xfeature_nr) 900b8b9b6baSDave Hansen { 90107baeb04SSebastian Andrzej Siewior if (!xfeature_enabled(xfeature_nr)) { 9025060b915SYu-cheng Yu WARN_ON_FPU(1); 9035060b915SYu-cheng Yu return NULL; 9045060b915SYu-cheng Yu } 9055060b915SYu-cheng Yu 90607baeb04SSebastian Andrzej Siewior return (void *)xsave + xstate_comp_offsets[xfeature_nr]; 907b8b9b6baSDave Hansen } 908b8b9b6baSDave Hansen /* 90962784854SIngo Molnar * Given the xsave area and a state inside, this function returns the 91062784854SIngo Molnar * address of the state. 91162784854SIngo Molnar * 91262784854SIngo Molnar * This is the API that is called to get xstate address in either 91362784854SIngo Molnar * standard format or compacted format of xsave area. 91462784854SIngo Molnar * 9150c4109beSDave Hansen * Note that if there is no data for the field in the xsave buffer 9160c4109beSDave Hansen * this will return NULL. 9170c4109beSDave Hansen * 91862784854SIngo Molnar * Inputs: 9190c4109beSDave Hansen * xstate: the thread's storage area for all FPU data 920abd16d68SSebastian Andrzej Siewior * xfeature_nr: state which is defined in xsave.h (e.g. XFEATURE_FP, 921abd16d68SSebastian Andrzej Siewior * XFEATURE_SSE, etc...) 92262784854SIngo Molnar * Output: 9230c4109beSDave Hansen * address of the state in the xsave area, or NULL if the 9240c4109beSDave Hansen * field is not present in the xsave buffer. 92562784854SIngo Molnar */ 926abd16d68SSebastian Andrzej Siewior void *get_xsave_addr(struct xregs_state *xsave, int xfeature_nr) 92762784854SIngo Molnar { 9280c4109beSDave Hansen /* 9290c4109beSDave Hansen * Do we even *have* xsave state? 9300c4109beSDave Hansen */ 9310c4109beSDave Hansen if (!boot_cpu_has(X86_FEATURE_XSAVE)) 93262784854SIngo Molnar return NULL; 93362784854SIngo Molnar 9340c4109beSDave Hansen /* 9350c4109beSDave Hansen * We should not ever be requesting features that we 936524bb73bSYu-cheng Yu * have not enabled. 9370c4109beSDave Hansen */ 938524bb73bSYu-cheng Yu WARN_ONCE(!(xfeatures_mask_all & BIT_ULL(xfeature_nr)), 9390c4109beSDave Hansen "get of unsupported state"); 9400c4109beSDave Hansen /* 9410c4109beSDave Hansen * This assumes the last 'xsave*' instruction to 942abd16d68SSebastian Andrzej Siewior * have requested that 'xfeature_nr' be saved. 9430c4109beSDave Hansen * If it did not, we might be seeing and old value 9440c4109beSDave Hansen * of the field in the buffer. 9450c4109beSDave Hansen * 9460c4109beSDave Hansen * This can happen because the last 'xsave' did not 9470c4109beSDave Hansen * request that this feature be saved (unlikely) 9480c4109beSDave Hansen * or because the "init optimization" caused it 9490c4109beSDave Hansen * to not be saved. 9500c4109beSDave Hansen */ 951abd16d68SSebastian Andrzej Siewior if (!(xsave->header.xfeatures & BIT_ULL(xfeature_nr))) 9520c4109beSDave Hansen return NULL; 9530c4109beSDave Hansen 95407baeb04SSebastian Andrzej Siewior return __raw_xsave_addr(xsave, xfeature_nr); 95562784854SIngo Molnar } 95662784854SIngo Molnar EXPORT_SYMBOL_GPL(get_xsave_addr); 95704cd027bSDave Hansen 95804cd027bSDave Hansen /* 95904cd027bSDave Hansen * This wraps up the common operations that need to occur when retrieving 96004cd027bSDave Hansen * data from xsave state. It first ensures that the current task was 96104cd027bSDave Hansen * using the FPU and retrieves the data in to a buffer. It then calculates 96204cd027bSDave Hansen * the offset of the requested field in the buffer. 96304cd027bSDave Hansen * 96404cd027bSDave Hansen * This function is safe to call whether the FPU is in use or not. 96504cd027bSDave Hansen * 96604cd027bSDave Hansen * Note that this only works on the current task. 96704cd027bSDave Hansen * 96804cd027bSDave Hansen * Inputs: 969abd16d68SSebastian Andrzej Siewior * @xfeature_nr: state which is defined in xsave.h (e.g. XFEATURE_FP, 970abd16d68SSebastian Andrzej Siewior * XFEATURE_SSE, etc...) 97104cd027bSDave Hansen * Output: 97204cd027bSDave Hansen * address of the state in the xsave area or NULL if the state 97304cd027bSDave Hansen * is not present or is in its 'init state'. 97404cd027bSDave Hansen */ 975abd16d68SSebastian Andrzej Siewior const void *get_xsave_field_ptr(int xfeature_nr) 97604cd027bSDave Hansen { 97704cd027bSDave Hansen struct fpu *fpu = ¤t->thread.fpu; 97804cd027bSDave Hansen 97904cd027bSDave Hansen /* 98004cd027bSDave Hansen * fpu__save() takes the CPU's xstate registers 98104cd027bSDave Hansen * and saves them off to the 'fpu memory buffer. 98204cd027bSDave Hansen */ 98304cd027bSDave Hansen fpu__save(fpu); 98404cd027bSDave Hansen 985abd16d68SSebastian Andrzej Siewior return get_xsave_addr(&fpu->state.xsave, xfeature_nr); 98604cd027bSDave Hansen } 987b8b9b6baSDave Hansen 988e8c24d3aSDave Hansen #ifdef CONFIG_ARCH_HAS_PKEYS 989e8c24d3aSDave Hansen 99084594296SDave Hansen /* 991b79daf85SDave Hansen * This will go out and modify PKRU register to set the access 992b79daf85SDave Hansen * rights for @pkey to @init_val. 99384594296SDave Hansen */ 99484594296SDave Hansen int arch_set_user_pkey_access(struct task_struct *tsk, int pkey, 99584594296SDave Hansen unsigned long init_val) 99684594296SDave Hansen { 997b79daf85SDave Hansen u32 old_pkru; 99884594296SDave Hansen int pkey_shift = (pkey * PKRU_BITS_PER_PKEY); 99984594296SDave Hansen u32 new_pkru_bits = 0; 100084594296SDave Hansen 100184594296SDave Hansen /* 100284594296SDave Hansen * This check implies XSAVE support. OSPKE only gets 100384594296SDave Hansen * set if we enable XSAVE and we enable PKU in XCR0. 100484594296SDave Hansen */ 100584594296SDave Hansen if (!boot_cpu_has(X86_FEATURE_OSPKE)) 100684594296SDave Hansen return -EINVAL; 100784594296SDave Hansen 100816171bffSDave Hansen /* 100916171bffSDave Hansen * This code should only be called with valid 'pkey' 101016171bffSDave Hansen * values originating from in-kernel users. Complain 101116171bffSDave Hansen * if a bad value is observed. 101216171bffSDave Hansen */ 101316171bffSDave Hansen WARN_ON_ONCE(pkey >= arch_max_pkey()); 101416171bffSDave Hansen 101591c3dba7SYu-cheng Yu /* Set the bits we need in PKRU: */ 101684594296SDave Hansen if (init_val & PKEY_DISABLE_ACCESS) 101784594296SDave Hansen new_pkru_bits |= PKRU_AD_BIT; 101884594296SDave Hansen if (init_val & PKEY_DISABLE_WRITE) 101984594296SDave Hansen new_pkru_bits |= PKRU_WD_BIT; 102084594296SDave Hansen 102191c3dba7SYu-cheng Yu /* Shift the bits in to the correct place in PKRU for pkey: */ 102284594296SDave Hansen new_pkru_bits <<= pkey_shift; 102384594296SDave Hansen 1024b79daf85SDave Hansen /* Get old PKRU and mask off any old bits in place: */ 1025b79daf85SDave Hansen old_pkru = read_pkru(); 1026b79daf85SDave Hansen old_pkru &= ~((PKRU_AD_BIT|PKRU_WD_BIT) << pkey_shift); 102784594296SDave Hansen 1028b79daf85SDave Hansen /* Write old part along with new part: */ 1029b79daf85SDave Hansen write_pkru(old_pkru | new_pkru_bits); 103091c3dba7SYu-cheng Yu 103191c3dba7SYu-cheng Yu return 0; 103291c3dba7SYu-cheng Yu } 1033e8c24d3aSDave Hansen #endif /* ! CONFIG_ARCH_HAS_PKEYS */ 103491c3dba7SYu-cheng Yu 103591c3dba7SYu-cheng Yu /* 10360852b374SRik van Riel * Weird legacy quirk: SSE and YMM states store information in the 10370852b374SRik van Riel * MXCSR and MXCSR_FLAGS fields of the FP area. That means if the FP 10380852b374SRik van Riel * area is marked as unused in the xfeatures header, we need to copy 10390852b374SRik van Riel * MXCSR and MXCSR_FLAGS if either SSE or YMM are in use. 10400852b374SRik van Riel */ 10410852b374SRik van Riel static inline bool xfeatures_mxcsr_quirk(u64 xfeatures) 10420852b374SRik van Riel { 10430852b374SRik van Riel if (!(xfeatures & (XFEATURE_MASK_SSE|XFEATURE_MASK_YMM))) 10444f8cef59Skbuild test robot return false; 10450852b374SRik van Riel 10460852b374SRik van Riel if (xfeatures & XFEATURE_MASK_FP) 10474f8cef59Skbuild test robot return false; 10480852b374SRik van Riel 10494f8cef59Skbuild test robot return true; 10500852b374SRik van Riel } 10510852b374SRik van Riel 10520557d64dSAl Viro static void fill_gap(struct membuf *to, unsigned *last, unsigned offset) 1053f0d4f30aSIngo Molnar { 10540557d64dSAl Viro if (*last >= offset) 10550557d64dSAl Viro return; 10560557d64dSAl Viro membuf_write(to, (void *)&init_fpstate.xsave + *last, offset - *last); 10570557d64dSAl Viro *last = offset; 10589e463654SAl Viro } 10599e463654SAl Viro 10600557d64dSAl Viro static void copy_part(struct membuf *to, unsigned *last, unsigned offset, 10610557d64dSAl Viro unsigned size, void *from) 10629e463654SAl Viro { 10630557d64dSAl Viro fill_gap(to, last, offset); 10640557d64dSAl Viro membuf_write(to, from, size); 10650557d64dSAl Viro *last = offset + size; 1066f0d4f30aSIngo Molnar } 1067f0d4f30aSIngo Molnar 1068f0d4f30aSIngo Molnar /* 1069f0d4f30aSIngo Molnar * Convert from kernel XSAVES compacted format to standard format and copy 1070f0d4f30aSIngo Molnar * to a kernel-space ptrace buffer. 1071f0d4f30aSIngo Molnar * 1072f0d4f30aSIngo Molnar * It supports partial copy but pos always starts from zero. This is called 1073f0d4f30aSIngo Molnar * from xstateregs_get() and there we check the CPU has XSAVES. 1074f0d4f30aSIngo Molnar */ 10750557d64dSAl Viro void copy_xstate_to_kernel(struct membuf to, struct xregs_state *xsave) 1076f0d4f30aSIngo Molnar { 1077f0d4f30aSIngo Molnar struct xstate_header header; 10789e463654SAl Viro const unsigned off_mxcsr = offsetof(struct fxregs_state, mxcsr); 10790557d64dSAl Viro unsigned size = to.left; 10800557d64dSAl Viro unsigned last = 0; 10818c0817f4SIngo Molnar int i; 1082f0d4f30aSIngo Molnar 1083f0d4f30aSIngo Molnar /* 1084f0d4f30aSIngo Molnar * The destination is a ptrace buffer; we put in only user xstates: 1085f0d4f30aSIngo Molnar */ 1086f0d4f30aSIngo Molnar memset(&header, 0, sizeof(header)); 1087f0d4f30aSIngo Molnar header.xfeatures = xsave->header.xfeatures; 1088524bb73bSYu-cheng Yu header.xfeatures &= xfeatures_mask_user(); 1089f0d4f30aSIngo Molnar 10909e463654SAl Viro if (header.xfeatures & XFEATURE_MASK_FP) 10910557d64dSAl Viro copy_part(&to, &last, 0, off_mxcsr, &xsave->i387); 10929e463654SAl Viro if (header.xfeatures & (XFEATURE_MASK_SSE | XFEATURE_MASK_YMM)) 10930557d64dSAl Viro copy_part(&to, &last, off_mxcsr, 10940557d64dSAl Viro MXCSR_AND_FLAGS_SIZE, &xsave->i387.mxcsr); 10959e463654SAl Viro if (header.xfeatures & XFEATURE_MASK_FP) 10960557d64dSAl Viro copy_part(&to, &last, offsetof(struct fxregs_state, st_space), 10970557d64dSAl Viro 128, &xsave->i387.st_space); 10989e463654SAl Viro if (header.xfeatures & XFEATURE_MASK_SSE) 10990557d64dSAl Viro copy_part(&to, &last, xstate_offsets[XFEATURE_SSE], 11000557d64dSAl Viro 256, &xsave->i387.xmm_space); 11019e463654SAl Viro /* 11029e463654SAl Viro * Fill xsave->i387.sw_reserved value for ptrace frame: 11039e463654SAl Viro */ 11040557d64dSAl Viro copy_part(&to, &last, offsetof(struct fxregs_state, sw_reserved), 11050557d64dSAl Viro 48, xstate_fx_sw_bytes); 1106f0d4f30aSIngo Molnar /* 1107f0d4f30aSIngo Molnar * Copy xregs_state->header: 1108f0d4f30aSIngo Molnar */ 11090557d64dSAl Viro copy_part(&to, &last, offsetof(struct xregs_state, header), 11100557d64dSAl Viro sizeof(header), &header); 1111f0d4f30aSIngo Molnar 11129e463654SAl Viro for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) { 1113f0d4f30aSIngo Molnar /* 1114f0d4f30aSIngo Molnar * Copy only in-use xstates: 1115f0d4f30aSIngo Molnar */ 1116f0d4f30aSIngo Molnar if ((header.xfeatures >> i) & 1) { 111707baeb04SSebastian Andrzej Siewior void *src = __raw_xsave_addr(xsave, i); 1118f0d4f30aSIngo Molnar 11190557d64dSAl Viro copy_part(&to, &last, xstate_offsets[i], 11200557d64dSAl Viro xstate_sizes[i], src); 1121f0d4f30aSIngo Molnar } 1122f0d4f30aSIngo Molnar 1123f0d4f30aSIngo Molnar } 11240557d64dSAl Viro fill_gap(&to, &last, size); 112591c3dba7SYu-cheng Yu } 112691c3dba7SYu-cheng Yu 112791c3dba7SYu-cheng Yu /* 112879fecc2bSIngo Molnar * Convert from a ptrace standard-format kernel buffer to kernel XSAVES format 1129af95774bSEric Biggers * and copy to the target thread. This is called from xstateregs_set(). 113079fecc2bSIngo Molnar */ 11316d7f7da5SIngo Molnar int copy_kernel_to_xstate(struct xregs_state *xsave, const void *kbuf) 113279fecc2bSIngo Molnar { 113379fecc2bSIngo Molnar unsigned int offset, size; 113479fecc2bSIngo Molnar int i; 113580d8ae86SEric Biggers struct xstate_header hdr; 113679fecc2bSIngo Molnar 113779fecc2bSIngo Molnar offset = offsetof(struct xregs_state, header); 113880d8ae86SEric Biggers size = sizeof(hdr); 113979fecc2bSIngo Molnar 114080d8ae86SEric Biggers memcpy(&hdr, kbuf + offset, size); 114179fecc2bSIngo Molnar 11425274e6c1SFenghua Yu if (validate_user_xstate_header(&hdr)) 114379fecc2bSIngo Molnar return -EINVAL; 114479fecc2bSIngo Molnar 114579fecc2bSIngo Molnar for (i = 0; i < XFEATURE_MAX; i++) { 114679fecc2bSIngo Molnar u64 mask = ((u64)1 << i); 114779fecc2bSIngo Molnar 1148b89eda48SEric Biggers if (hdr.xfeatures & mask) { 114907baeb04SSebastian Andrzej Siewior void *dst = __raw_xsave_addr(xsave, i); 115079fecc2bSIngo Molnar 115179fecc2bSIngo Molnar offset = xstate_offsets[i]; 115279fecc2bSIngo Molnar size = xstate_sizes[i]; 115379fecc2bSIngo Molnar 115479fecc2bSIngo Molnar memcpy(dst, kbuf + offset, size); 115579fecc2bSIngo Molnar } 115679fecc2bSIngo Molnar } 115779fecc2bSIngo Molnar 1158b89eda48SEric Biggers if (xfeatures_mxcsr_quirk(hdr.xfeatures)) { 11590852b374SRik van Riel offset = offsetof(struct fxregs_state, mxcsr); 11600852b374SRik van Riel size = MXCSR_AND_FLAGS_SIZE; 11610852b374SRik van Riel memcpy(&xsave->i387.mxcsr, kbuf + offset, size); 11620852b374SRik van Riel } 11630852b374SRik van Riel 116479fecc2bSIngo Molnar /* 116579fecc2bSIngo Molnar * The state that came in from userspace was user-state only. 116679fecc2bSIngo Molnar * Mask all the user states out of 'xfeatures': 116779fecc2bSIngo Molnar */ 11688ab22804SFenghua Yu xsave->header.xfeatures &= XFEATURE_MASK_SUPERVISOR_ALL; 116979fecc2bSIngo Molnar 117079fecc2bSIngo Molnar /* 117179fecc2bSIngo Molnar * Add back in the features that came in from userspace: 117279fecc2bSIngo Molnar */ 1173b89eda48SEric Biggers xsave->header.xfeatures |= hdr.xfeatures; 117479fecc2bSIngo Molnar 117579fecc2bSIngo Molnar return 0; 117679fecc2bSIngo Molnar } 117779fecc2bSIngo Molnar 117879fecc2bSIngo Molnar /* 117998c0fad9SEric Biggers * Convert from a ptrace or sigreturn standard-format user-space buffer to 118098c0fad9SEric Biggers * kernel XSAVES format and copy to the target thread. This is called from 118198c0fad9SEric Biggers * xstateregs_set(), as well as potentially from the sigreturn() and 118298c0fad9SEric Biggers * rt_sigreturn() system calls. 118391c3dba7SYu-cheng Yu */ 11846d7f7da5SIngo Molnar int copy_user_to_xstate(struct xregs_state *xsave, const void __user *ubuf) 118591c3dba7SYu-cheng Yu { 118691c3dba7SYu-cheng Yu unsigned int offset, size; 118791c3dba7SYu-cheng Yu int i; 1188af2c4322SEric Biggers struct xstate_header hdr; 118991c3dba7SYu-cheng Yu 119091c3dba7SYu-cheng Yu offset = offsetof(struct xregs_state, header); 1191af2c4322SEric Biggers size = sizeof(hdr); 119291c3dba7SYu-cheng Yu 1193af2c4322SEric Biggers if (__copy_from_user(&hdr, ubuf + offset, size)) 119491c3dba7SYu-cheng Yu return -EFAULT; 119591c3dba7SYu-cheng Yu 11965274e6c1SFenghua Yu if (validate_user_xstate_header(&hdr)) 119791c3dba7SYu-cheng Yu return -EINVAL; 119891c3dba7SYu-cheng Yu 119991c3dba7SYu-cheng Yu for (i = 0; i < XFEATURE_MAX; i++) { 120091c3dba7SYu-cheng Yu u64 mask = ((u64)1 << i); 120191c3dba7SYu-cheng Yu 12023d703477SEric Biggers if (hdr.xfeatures & mask) { 120307baeb04SSebastian Andrzej Siewior void *dst = __raw_xsave_addr(xsave, i); 120491c3dba7SYu-cheng Yu 120591c3dba7SYu-cheng Yu offset = xstate_offsets[i]; 120691c3dba7SYu-cheng Yu size = xstate_sizes[i]; 120791c3dba7SYu-cheng Yu 120891c3dba7SYu-cheng Yu if (__copy_from_user(dst, ubuf + offset, size)) 120991c3dba7SYu-cheng Yu return -EFAULT; 121091c3dba7SYu-cheng Yu } 121191c3dba7SYu-cheng Yu } 121291c3dba7SYu-cheng Yu 12133d703477SEric Biggers if (xfeatures_mxcsr_quirk(hdr.xfeatures)) { 12140852b374SRik van Riel offset = offsetof(struct fxregs_state, mxcsr); 12150852b374SRik van Riel size = MXCSR_AND_FLAGS_SIZE; 12160852b374SRik van Riel if (__copy_from_user(&xsave->i387.mxcsr, ubuf + offset, size)) 12170852b374SRik van Riel return -EFAULT; 12180852b374SRik van Riel } 12190852b374SRik van Riel 122091c3dba7SYu-cheng Yu /* 122191c3dba7SYu-cheng Yu * The state that came in from userspace was user-state only. 122291c3dba7SYu-cheng Yu * Mask all the user states out of 'xfeatures': 122391c3dba7SYu-cheng Yu */ 12248ab22804SFenghua Yu xsave->header.xfeatures &= XFEATURE_MASK_SUPERVISOR_ALL; 122591c3dba7SYu-cheng Yu 122691c3dba7SYu-cheng Yu /* 122791c3dba7SYu-cheng Yu * Add back in the features that came in from userspace: 122891c3dba7SYu-cheng Yu */ 12293d703477SEric Biggers xsave->header.xfeatures |= hdr.xfeatures; 123084594296SDave Hansen 123184594296SDave Hansen return 0; 123284594296SDave Hansen } 12330c608dadSAubrey Li 1234eeedf153SYu-cheng Yu /* 1235eeedf153SYu-cheng Yu * Save only supervisor states to the kernel buffer. This blows away all 1236eeedf153SYu-cheng Yu * old states, and is intended to be used only in __fpu__restore_sig(), where 1237eeedf153SYu-cheng Yu * user states are restored from the user buffer. 1238eeedf153SYu-cheng Yu */ 1239eeedf153SYu-cheng Yu void copy_supervisor_to_kernel(struct xregs_state *xstate) 1240eeedf153SYu-cheng Yu { 1241eeedf153SYu-cheng Yu struct xstate_header *header; 1242eeedf153SYu-cheng Yu u64 max_bit, min_bit; 1243eeedf153SYu-cheng Yu u32 lmask, hmask; 1244eeedf153SYu-cheng Yu int err, i; 1245eeedf153SYu-cheng Yu 1246eeedf153SYu-cheng Yu if (WARN_ON(!boot_cpu_has(X86_FEATURE_XSAVES))) 1247eeedf153SYu-cheng Yu return; 1248eeedf153SYu-cheng Yu 1249eeedf153SYu-cheng Yu if (!xfeatures_mask_supervisor()) 1250eeedf153SYu-cheng Yu return; 1251eeedf153SYu-cheng Yu 1252eeedf153SYu-cheng Yu max_bit = __fls(xfeatures_mask_supervisor()); 1253eeedf153SYu-cheng Yu min_bit = __ffs(xfeatures_mask_supervisor()); 1254eeedf153SYu-cheng Yu 1255eeedf153SYu-cheng Yu lmask = xfeatures_mask_supervisor(); 1256eeedf153SYu-cheng Yu hmask = xfeatures_mask_supervisor() >> 32; 1257eeedf153SYu-cheng Yu XSTATE_OP(XSAVES, xstate, lmask, hmask, err); 1258eeedf153SYu-cheng Yu 1259eeedf153SYu-cheng Yu /* We should never fault when copying to a kernel buffer: */ 1260eeedf153SYu-cheng Yu if (WARN_ON_FPU(err)) 1261eeedf153SYu-cheng Yu return; 1262eeedf153SYu-cheng Yu 1263eeedf153SYu-cheng Yu /* 1264eeedf153SYu-cheng Yu * At this point, the buffer has only supervisor states and must be 1265eeedf153SYu-cheng Yu * converted back to normal kernel format. 1266eeedf153SYu-cheng Yu */ 1267eeedf153SYu-cheng Yu header = &xstate->header; 1268eeedf153SYu-cheng Yu header->xcomp_bv |= xfeatures_mask_all; 1269eeedf153SYu-cheng Yu 1270eeedf153SYu-cheng Yu /* 1271eeedf153SYu-cheng Yu * This only moves states up in the buffer. Start with 1272eeedf153SYu-cheng Yu * the last state and move backwards so that states are 1273eeedf153SYu-cheng Yu * not overwritten until after they are moved. Note: 1274eeedf153SYu-cheng Yu * memmove() allows overlapping src/dst buffers. 1275eeedf153SYu-cheng Yu */ 1276eeedf153SYu-cheng Yu for (i = max_bit; i >= min_bit; i--) { 1277eeedf153SYu-cheng Yu u8 *xbuf = (u8 *)xstate; 1278eeedf153SYu-cheng Yu 1279eeedf153SYu-cheng Yu if (!((header->xfeatures >> i) & 1)) 1280eeedf153SYu-cheng Yu continue; 1281eeedf153SYu-cheng Yu 1282eeedf153SYu-cheng Yu /* Move xfeature 'i' into its normal location */ 1283eeedf153SYu-cheng Yu memmove(xbuf + xstate_comp_offsets[i], 1284eeedf153SYu-cheng Yu xbuf + xstate_supervisor_only_offsets[i], 1285eeedf153SYu-cheng Yu xstate_sizes[i]); 1286eeedf153SYu-cheng Yu } 1287eeedf153SYu-cheng Yu } 1288eeedf153SYu-cheng Yu 128950f408d9SKan Liang /** 129050f408d9SKan Liang * copy_dynamic_supervisor_to_kernel() - Save dynamic supervisor states to 129150f408d9SKan Liang * an xsave area 129250f408d9SKan Liang * @xstate: A pointer to an xsave area 129350f408d9SKan Liang * @mask: Represent the dynamic supervisor features saved into the xsave area 129450f408d9SKan Liang * 129550f408d9SKan Liang * Only the dynamic supervisor states sets in the mask are saved into the xsave 129650f408d9SKan Liang * area (See the comment in XFEATURE_MASK_DYNAMIC for the details of dynamic 129750f408d9SKan Liang * supervisor feature). Besides the dynamic supervisor states, the legacy 129850f408d9SKan Liang * region and XSAVE header are also saved into the xsave area. The supervisor 129950f408d9SKan Liang * features in the XFEATURE_MASK_SUPERVISOR_SUPPORTED and 130050f408d9SKan Liang * XFEATURE_MASK_SUPERVISOR_UNSUPPORTED are not saved. 130150f408d9SKan Liang * 130250f408d9SKan Liang * The xsave area must be 64-bytes aligned. 130350f408d9SKan Liang */ 130450f408d9SKan Liang void copy_dynamic_supervisor_to_kernel(struct xregs_state *xstate, u64 mask) 130550f408d9SKan Liang { 130650f408d9SKan Liang u64 dynamic_mask = xfeatures_mask_dynamic() & mask; 130750f408d9SKan Liang u32 lmask, hmask; 130850f408d9SKan Liang int err; 130950f408d9SKan Liang 131050f408d9SKan Liang if (WARN_ON_FPU(!boot_cpu_has(X86_FEATURE_XSAVES))) 131150f408d9SKan Liang return; 131250f408d9SKan Liang 131350f408d9SKan Liang if (WARN_ON_FPU(!dynamic_mask)) 131450f408d9SKan Liang return; 131550f408d9SKan Liang 131650f408d9SKan Liang lmask = dynamic_mask; 131750f408d9SKan Liang hmask = dynamic_mask >> 32; 131850f408d9SKan Liang 131950f408d9SKan Liang XSTATE_OP(XSAVES, xstate, lmask, hmask, err); 132050f408d9SKan Liang 132150f408d9SKan Liang /* Should never fault when copying to a kernel buffer */ 132250f408d9SKan Liang WARN_ON_FPU(err); 132350f408d9SKan Liang } 132450f408d9SKan Liang 132550f408d9SKan Liang /** 132650f408d9SKan Liang * copy_kernel_to_dynamic_supervisor() - Restore dynamic supervisor states from 132750f408d9SKan Liang * an xsave area 132850f408d9SKan Liang * @xstate: A pointer to an xsave area 132950f408d9SKan Liang * @mask: Represent the dynamic supervisor features restored from the xsave area 133050f408d9SKan Liang * 133150f408d9SKan Liang * Only the dynamic supervisor states sets in the mask are restored from the 133250f408d9SKan Liang * xsave area (See the comment in XFEATURE_MASK_DYNAMIC for the details of 133350f408d9SKan Liang * dynamic supervisor feature). Besides the dynamic supervisor states, the 133450f408d9SKan Liang * legacy region and XSAVE header are also restored from the xsave area. The 133550f408d9SKan Liang * supervisor features in the XFEATURE_MASK_SUPERVISOR_SUPPORTED and 133650f408d9SKan Liang * XFEATURE_MASK_SUPERVISOR_UNSUPPORTED are not restored. 133750f408d9SKan Liang * 133850f408d9SKan Liang * The xsave area must be 64-bytes aligned. 133950f408d9SKan Liang */ 134050f408d9SKan Liang void copy_kernel_to_dynamic_supervisor(struct xregs_state *xstate, u64 mask) 134150f408d9SKan Liang { 134250f408d9SKan Liang u64 dynamic_mask = xfeatures_mask_dynamic() & mask; 134350f408d9SKan Liang u32 lmask, hmask; 134450f408d9SKan Liang int err; 134550f408d9SKan Liang 134650f408d9SKan Liang if (WARN_ON_FPU(!boot_cpu_has(X86_FEATURE_XSAVES))) 134750f408d9SKan Liang return; 134850f408d9SKan Liang 134950f408d9SKan Liang if (WARN_ON_FPU(!dynamic_mask)) 135050f408d9SKan Liang return; 135150f408d9SKan Liang 135250f408d9SKan Liang lmask = dynamic_mask; 135350f408d9SKan Liang hmask = dynamic_mask >> 32; 135450f408d9SKan Liang 135550f408d9SKan Liang XSTATE_OP(XRSTORS, xstate, lmask, hmask, err); 135650f408d9SKan Liang 135750f408d9SKan Liang /* Should never fault when copying from a kernel buffer */ 135850f408d9SKan Liang WARN_ON_FPU(err); 135950f408d9SKan Liang } 136050f408d9SKan Liang 13610c608dadSAubrey Li #ifdef CONFIG_PROC_PID_ARCH_STATUS 13620c608dadSAubrey Li /* 13630c608dadSAubrey Li * Report the amount of time elapsed in millisecond since last AVX512 13640c608dadSAubrey Li * use in the task. 13650c608dadSAubrey Li */ 13660c608dadSAubrey Li static void avx512_status(struct seq_file *m, struct task_struct *task) 13670c608dadSAubrey Li { 13680c608dadSAubrey Li unsigned long timestamp = READ_ONCE(task->thread.fpu.avx512_timestamp); 13690c608dadSAubrey Li long delta; 13700c608dadSAubrey Li 13710c608dadSAubrey Li if (!timestamp) { 13720c608dadSAubrey Li /* 13730c608dadSAubrey Li * Report -1 if no AVX512 usage 13740c608dadSAubrey Li */ 13750c608dadSAubrey Li delta = -1; 13760c608dadSAubrey Li } else { 13770c608dadSAubrey Li delta = (long)(jiffies - timestamp); 13780c608dadSAubrey Li /* 13790c608dadSAubrey Li * Cap to LONG_MAX if time difference > LONG_MAX 13800c608dadSAubrey Li */ 13810c608dadSAubrey Li if (delta < 0) 13820c608dadSAubrey Li delta = LONG_MAX; 13830c608dadSAubrey Li delta = jiffies_to_msecs(delta); 13840c608dadSAubrey Li } 13850c608dadSAubrey Li 13860c608dadSAubrey Li seq_put_decimal_ll(m, "AVX512_elapsed_ms:\t", delta); 13870c608dadSAubrey Li seq_putc(m, '\n'); 13880c608dadSAubrey Li } 13890c608dadSAubrey Li 13900c608dadSAubrey Li /* 13910c608dadSAubrey Li * Report architecture specific information 13920c608dadSAubrey Li */ 13930c608dadSAubrey Li int proc_pid_arch_status(struct seq_file *m, struct pid_namespace *ns, 13940c608dadSAubrey Li struct pid *pid, struct task_struct *task) 13950c608dadSAubrey Li { 13960c608dadSAubrey Li /* 13970c608dadSAubrey Li * Report AVX512 state if the processor and build option supported. 13980c608dadSAubrey Li */ 13990c608dadSAubrey Li if (cpu_feature_enabled(X86_FEATURE_AVX512F)) 14000c608dadSAubrey Li avx512_status(m, task); 14010c608dadSAubrey Li 14020c608dadSAubrey Li return 0; 14030c608dadSAubrey Li } 14040c608dadSAubrey Li #endif /* CONFIG_PROC_PID_ARCH_STATUS */ 1405