1457c8996SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 262784854SIngo Molnar /* 362784854SIngo Molnar * xsave/xrstor support. 462784854SIngo Molnar * 562784854SIngo Molnar * Author: Suresh Siddha <suresh.b.siddha@intel.com> 662784854SIngo Molnar */ 762784854SIngo Molnar #include <linux/compat.h> 862784854SIngo Molnar #include <linux/cpu.h> 9e8c24d3aSDave Hansen #include <linux/mman.h> 1084594296SDave Hansen #include <linux/pkeys.h> 110c608dadSAubrey Li #include <linux/seq_file.h> 120c608dadSAubrey Li #include <linux/proc_fs.h> 1359a36d16SIngo Molnar 1462784854SIngo Molnar #include <asm/fpu/api.h> 1562784854SIngo Molnar #include <asm/fpu/internal.h> 16fcbc99c4SIngo Molnar #include <asm/fpu/signal.h> 1759a36d16SIngo Molnar #include <asm/fpu/regset.h> 1891c3dba7SYu-cheng Yu #include <asm/fpu/xstate.h> 19b992c660SIngo Molnar 2062784854SIngo Molnar #include <asm/tlbflush.h> 21ccb18db2SAndi Kleen #include <asm/cpufeature.h> 2262784854SIngo Molnar 231f96b1efSDave Hansen /* 241f96b1efSDave Hansen * Although we spell it out in here, the Processor Trace 251f96b1efSDave Hansen * xfeature is completely unused. We use other mechanisms 261f96b1efSDave Hansen * to save/restore PT state in Linux. 271f96b1efSDave Hansen */ 285b073430SIngo Molnar static const char *xfeature_names[] = 295b073430SIngo Molnar { 305b073430SIngo Molnar "x87 floating point registers" , 315b073430SIngo Molnar "SSE registers" , 325b073430SIngo Molnar "AVX registers" , 335b073430SIngo Molnar "MPX bounds registers" , 345b073430SIngo Molnar "MPX CSR" , 355b073430SIngo Molnar "AVX-512 opmask" , 365b073430SIngo Molnar "AVX-512 Hi256" , 375b073430SIngo Molnar "AVX-512 ZMM_Hi256" , 381f96b1efSDave Hansen "Processor Trace (unused)" , 39c8df4009SDave Hansen "Protection Keys User registers", 405b073430SIngo Molnar "unknown xstate feature" , 415b073430SIngo Molnar }; 425b073430SIngo Molnar 43ccb18db2SAndi Kleen static short xsave_cpuid_features[] __initdata = { 44ccb18db2SAndi Kleen X86_FEATURE_FPU, 45ccb18db2SAndi Kleen X86_FEATURE_XMM, 46ccb18db2SAndi Kleen X86_FEATURE_AVX, 47ccb18db2SAndi Kleen X86_FEATURE_MPX, 48ccb18db2SAndi Kleen X86_FEATURE_MPX, 49ccb18db2SAndi Kleen X86_FEATURE_AVX512F, 50ccb18db2SAndi Kleen X86_FEATURE_AVX512F, 51ccb18db2SAndi Kleen X86_FEATURE_AVX512F, 52ccb18db2SAndi Kleen X86_FEATURE_INTEL_PT, 53ccb18db2SAndi Kleen X86_FEATURE_PKU, 54ccb18db2SAndi Kleen }; 55ccb18db2SAndi Kleen 5662784854SIngo Molnar /* 57524bb73bSYu-cheng Yu * This represents the full set of bits that should ever be set in a kernel 58524bb73bSYu-cheng Yu * XSAVE buffer, both supervisor and user xstates. 5962784854SIngo Molnar */ 60524bb73bSYu-cheng Yu u64 xfeatures_mask_all __read_mostly; 6162784854SIngo Molnar 62dad8c4feSDave Hansen static unsigned int xstate_offsets[XFEATURE_MAX] = { [ 0 ... XFEATURE_MAX - 1] = -1}; 63dad8c4feSDave Hansen static unsigned int xstate_sizes[XFEATURE_MAX] = { [ 0 ... XFEATURE_MAX - 1] = -1}; 64c0855051SCyrill Gorcunov static unsigned int xstate_comp_offsets[XFEATURE_MAX] = { [ 0 ... XFEATURE_MAX - 1] = -1}; 65eeedf153SYu-cheng Yu static unsigned int xstate_supervisor_only_offsets[XFEATURE_MAX] = { [ 0 ... XFEATURE_MAX - 1] = -1}; 6662784854SIngo Molnar 6762784854SIngo Molnar /* 68a1141e0bSFenghua Yu * The XSAVE area of kernel can be in standard or compacted format; 69a1141e0bSFenghua Yu * it is always in standard format for user mode. This is the user 70a1141e0bSFenghua Yu * mode standard format size used for signal and ptrace frames. 71a1141e0bSFenghua Yu */ 72a1141e0bSFenghua Yu unsigned int fpu_user_xstate_size; 73a1141e0bSFenghua Yu 74a1141e0bSFenghua Yu /* 755b073430SIngo Molnar * Return whether the system supports a given xfeature. 765b073430SIngo Molnar * 775b073430SIngo Molnar * Also return the name of the (most advanced) feature that the caller requested: 785b073430SIngo Molnar */ 795b073430SIngo Molnar int cpu_has_xfeatures(u64 xfeatures_needed, const char **feature_name) 805b073430SIngo Molnar { 81524bb73bSYu-cheng Yu u64 xfeatures_missing = xfeatures_needed & ~xfeatures_mask_all; 825b073430SIngo Molnar 835b073430SIngo Molnar if (unlikely(feature_name)) { 845b073430SIngo Molnar long xfeature_idx, max_idx; 855b073430SIngo Molnar u64 xfeatures_print; 865b073430SIngo Molnar /* 875b073430SIngo Molnar * So we use FLS here to be able to print the most advanced 885b073430SIngo Molnar * feature that was requested but is missing. So if a driver 89d91cab78SDave Hansen * asks about "XFEATURE_MASK_SSE | XFEATURE_MASK_YMM" we'll print the 905b073430SIngo Molnar * missing AVX feature - this is the most informative message 915b073430SIngo Molnar * to users: 925b073430SIngo Molnar */ 935b073430SIngo Molnar if (xfeatures_missing) 945b073430SIngo Molnar xfeatures_print = xfeatures_missing; 955b073430SIngo Molnar else 965b073430SIngo Molnar xfeatures_print = xfeatures_needed; 975b073430SIngo Molnar 985b073430SIngo Molnar xfeature_idx = fls64(xfeatures_print)-1; 995b073430SIngo Molnar max_idx = ARRAY_SIZE(xfeature_names)-1; 1005b073430SIngo Molnar xfeature_idx = min(xfeature_idx, max_idx); 1015b073430SIngo Molnar 1025b073430SIngo Molnar *feature_name = xfeature_names[xfeature_idx]; 1035b073430SIngo Molnar } 1045b073430SIngo Molnar 1055b073430SIngo Molnar if (xfeatures_missing) 1065b073430SIngo Molnar return 0; 1075b073430SIngo Molnar 1085b073430SIngo Molnar return 1; 1095b073430SIngo Molnar } 1105b073430SIngo Molnar EXPORT_SYMBOL_GPL(cpu_has_xfeatures); 1115b073430SIngo Molnar 112158e2ee6SYu-cheng Yu static bool xfeature_is_supervisor(int xfeature_nr) 1131499ce2dSYu-cheng Yu { 1141499ce2dSYu-cheng Yu /* 1158c9e6073SYu-cheng Yu * Extended State Enumeration Sub-leaves (EAX = 0DH, ECX = n, n > 1) 1168c9e6073SYu-cheng Yu * returns ECX[0] set to (1) for a supervisor state, and cleared (0) 1178c9e6073SYu-cheng Yu * for a user state. 1181499ce2dSYu-cheng Yu */ 1191499ce2dSYu-cheng Yu u32 eax, ebx, ecx, edx; 1201499ce2dSYu-cheng Yu 1211499ce2dSYu-cheng Yu cpuid_count(XSTATE_CPUID, xfeature_nr, &eax, &ebx, &ecx, &edx); 122158e2ee6SYu-cheng Yu return ecx & 1; 1231499ce2dSYu-cheng Yu } 1241499ce2dSYu-cheng Yu 1255b073430SIngo Molnar /* 126aeb997b9SIngo Molnar * When executing XSAVEOPT (or other optimized XSAVE instructions), if 127aeb997b9SIngo Molnar * a processor implementation detects that an FPU state component is still 128aeb997b9SIngo Molnar * (or is again) in its initialized state, it may clear the corresponding 129aeb997b9SIngo Molnar * bit in the header.xfeatures field, and can skip the writeout of registers 130aeb997b9SIngo Molnar * to the corresponding memory layout. 13162784854SIngo Molnar * 13262784854SIngo Molnar * This means that when the bit is zero, the state component might still contain 13362784854SIngo Molnar * some previous - non-initialized register state. 13462784854SIngo Molnar * 13562784854SIngo Molnar * Before writing xstate information to user-space we sanitize those components, 13662784854SIngo Molnar * to always ensure that the memory layout of a feature will be in the init state 13762784854SIngo Molnar * if the corresponding header bit is zero. This is to ensure that user-space doesn't 13862784854SIngo Molnar * see some stale state in the memory layout during signal handling, debugging etc. 13962784854SIngo Molnar */ 14036e49e7fSIngo Molnar void fpstate_sanitize_xstate(struct fpu *fpu) 14162784854SIngo Molnar { 142c47ada30SIngo Molnar struct fxregs_state *fx = &fpu->state.fxsave; 14362784854SIngo Molnar int feature_bit; 14462784854SIngo Molnar u64 xfeatures; 14562784854SIngo Molnar 1461ac91a76SIngo Molnar if (!use_xsaveopt()) 14762784854SIngo Molnar return; 14862784854SIngo Molnar 14936e49e7fSIngo Molnar xfeatures = fpu->state.xsave.header.xfeatures; 15062784854SIngo Molnar 15162784854SIngo Molnar /* 15262784854SIngo Molnar * None of the feature bits are in init state. So nothing else 15362784854SIngo Molnar * to do for us, as the memory layout is up to date. 15462784854SIngo Molnar */ 155524bb73bSYu-cheng Yu if ((xfeatures & xfeatures_mask_all) == xfeatures_mask_all) 15662784854SIngo Molnar return; 15762784854SIngo Molnar 15862784854SIngo Molnar /* 15962784854SIngo Molnar * FP is in init state 16062784854SIngo Molnar */ 161d91cab78SDave Hansen if (!(xfeatures & XFEATURE_MASK_FP)) { 16262784854SIngo Molnar fx->cwd = 0x37f; 16362784854SIngo Molnar fx->swd = 0; 16462784854SIngo Molnar fx->twd = 0; 16562784854SIngo Molnar fx->fop = 0; 16662784854SIngo Molnar fx->rip = 0; 16762784854SIngo Molnar fx->rdp = 0; 16862784854SIngo Molnar memset(&fx->st_space[0], 0, 128); 16962784854SIngo Molnar } 17062784854SIngo Molnar 17162784854SIngo Molnar /* 17262784854SIngo Molnar * SSE is in init state 17362784854SIngo Molnar */ 174d91cab78SDave Hansen if (!(xfeatures & XFEATURE_MASK_SSE)) 17562784854SIngo Molnar memset(&fx->xmm_space[0], 0, 256); 17662784854SIngo Molnar 17762784854SIngo Molnar /* 17862784854SIngo Molnar * First two features are FPU and SSE, which above we handled 17962784854SIngo Molnar * in a special way already: 18062784854SIngo Molnar */ 18162784854SIngo Molnar feature_bit = 0x2; 182524bb73bSYu-cheng Yu xfeatures = (xfeatures_mask_user() & ~xfeatures) >> 2; 18362784854SIngo Molnar 18462784854SIngo Molnar /* 18562784854SIngo Molnar * Update all the remaining memory layouts according to their 18662784854SIngo Molnar * standard xstate layout, if their header bit is in the init 18762784854SIngo Molnar * state: 18862784854SIngo Molnar */ 18962784854SIngo Molnar while (xfeatures) { 19062784854SIngo Molnar if (xfeatures & 0x1) { 191a1141e0bSFenghua Yu int offset = xstate_comp_offsets[feature_bit]; 19262784854SIngo Molnar int size = xstate_sizes[feature_bit]; 19362784854SIngo Molnar 19462784854SIngo Molnar memcpy((void *)fx + offset, 1956f575023SIngo Molnar (void *)&init_fpstate.xsave + offset, 19662784854SIngo Molnar size); 19762784854SIngo Molnar } 19862784854SIngo Molnar 19962784854SIngo Molnar xfeatures >>= 1; 20062784854SIngo Molnar feature_bit++; 20162784854SIngo Molnar } 20262784854SIngo Molnar } 20362784854SIngo Molnar 20462784854SIngo Molnar /* 20562784854SIngo Molnar * Enable the extended processor state save/restore feature. 20662784854SIngo Molnar * Called once per CPU onlining. 20762784854SIngo Molnar */ 20862784854SIngo Molnar void fpu__init_cpu_xstate(void) 20962784854SIngo Molnar { 210524bb73bSYu-cheng Yu u64 unsup_bits; 211524bb73bSYu-cheng Yu 212524bb73bSYu-cheng Yu if (!boot_cpu_has(X86_FEATURE_XSAVE) || !xfeatures_mask_all) 21362784854SIngo Molnar return; 214b8be15d5SYu-cheng Yu /* 2158ab22804SFenghua Yu * Unsupported supervisor xstates should not be found in 2168ab22804SFenghua Yu * the xfeatures mask. 217b8be15d5SYu-cheng Yu */ 218524bb73bSYu-cheng Yu unsup_bits = xfeatures_mask_all & XFEATURE_MASK_SUPERVISOR_UNSUPPORTED; 219524bb73bSYu-cheng Yu WARN_ONCE(unsup_bits, "x86/fpu: Found unsupported supervisor xstates: 0x%llx\n", 220524bb73bSYu-cheng Yu unsup_bits); 221b8be15d5SYu-cheng Yu 222524bb73bSYu-cheng Yu xfeatures_mask_all &= ~XFEATURE_MASK_SUPERVISOR_UNSUPPORTED; 22362784854SIngo Molnar 22462784854SIngo Molnar cr4_set_bits(X86_CR4_OSXSAVE); 22562784854SIngo Molnar 22662784854SIngo Molnar /* 227524bb73bSYu-cheng Yu * XCR_XFEATURE_ENABLED_MASK (aka. XCR0) sets user features 228524bb73bSYu-cheng Yu * managed by XSAVE{C, OPT, S} and XRSTOR{S}. Only XSAVE user 229524bb73bSYu-cheng Yu * states can be set here. 230e6e888f9SDave Hansen */ 231524bb73bSYu-cheng Yu xsetbv(XCR_XFEATURE_ENABLED_MASK, xfeatures_mask_user()); 23262784854SIngo Molnar 23362784854SIngo Molnar /* 23471581eefSYu-cheng Yu * MSR_IA32_XSS sets supervisor states managed by XSAVES. 235e6e888f9SDave Hansen */ 236f0dccc9dSKan Liang if (boot_cpu_has(X86_FEATURE_XSAVES)) { 237f0dccc9dSKan Liang wrmsrl(MSR_IA32_XSS, xfeatures_mask_supervisor() | 238f0dccc9dSKan Liang xfeatures_mask_dynamic()); 239f0dccc9dSKan Liang } 24071581eefSYu-cheng Yu } 24171581eefSYu-cheng Yu 242524bb73bSYu-cheng Yu static bool xfeature_enabled(enum xfeature xfeature) 243e6e888f9SDave Hansen { 244524bb73bSYu-cheng Yu return xfeatures_mask_all & BIT_ULL(xfeature); 245e6e888f9SDave Hansen } 246e6e888f9SDave Hansen 247e6e888f9SDave Hansen /* 24839f1acd2SIngo Molnar * Record the offsets and sizes of various xstates contained 24939f1acd2SIngo Molnar * in the XSAVE state memory layout. 25062784854SIngo Molnar */ 25162784854SIngo Molnar static void __init setup_xstate_features(void) 25262784854SIngo Molnar { 253ee9ae257SDave Hansen u32 eax, ebx, ecx, edx, i; 254e6e888f9SDave Hansen /* start at the beginnning of the "extended state" */ 255e6e888f9SDave Hansen unsigned int last_good_offset = offsetof(struct xregs_state, 256e6e888f9SDave Hansen extended_state_area); 257ac73b27aSYu-cheng Yu /* 258ac73b27aSYu-cheng Yu * The FP xstates and SSE xstates are legacy states. They are always 259ac73b27aSYu-cheng Yu * in the fixed offsets in the xsave area in either compacted form 260ac73b27aSYu-cheng Yu * or standard form. 261ac73b27aSYu-cheng Yu */ 262446e693cSCyrill Gorcunov xstate_offsets[XFEATURE_FP] = 0; 263446e693cSCyrill Gorcunov xstate_sizes[XFEATURE_FP] = offsetof(struct fxregs_state, 264446e693cSCyrill Gorcunov xmm_space); 265446e693cSCyrill Gorcunov 266446e693cSCyrill Gorcunov xstate_offsets[XFEATURE_SSE] = xstate_sizes[XFEATURE_FP]; 267c593642cSPankaj Bharadiya xstate_sizes[XFEATURE_SSE] = sizeof_field(struct fxregs_state, 268446e693cSCyrill Gorcunov xmm_space); 26962784854SIngo Molnar 270ee9ae257SDave Hansen for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) { 271e6e888f9SDave Hansen if (!xfeature_enabled(i)) 272e6e888f9SDave Hansen continue; 27362784854SIngo Molnar 274e6e888f9SDave Hansen cpuid_count(XSTATE_CPUID, i, &eax, &ebx, &ecx, &edx); 2751499ce2dSYu-cheng Yu 276c12e13dcSYu-cheng Yu xstate_sizes[i] = eax; 277c12e13dcSYu-cheng Yu 2781499ce2dSYu-cheng Yu /* 279c12e13dcSYu-cheng Yu * If an xfeature is supervisor state, the offset in EBX is 280c12e13dcSYu-cheng Yu * invalid, leave it to -1. 2811499ce2dSYu-cheng Yu */ 282c12e13dcSYu-cheng Yu if (xfeature_is_supervisor(i)) 283c12e13dcSYu-cheng Yu continue; 284c12e13dcSYu-cheng Yu 285ee9ae257SDave Hansen xstate_offsets[i] = ebx; 2861499ce2dSYu-cheng Yu 287e6e888f9SDave Hansen /* 288c12e13dcSYu-cheng Yu * In our xstate size checks, we assume that the highest-numbered 289c12e13dcSYu-cheng Yu * xstate feature has the highest offset in the buffer. Ensure 290c12e13dcSYu-cheng Yu * it does. 291e6e888f9SDave Hansen */ 292e6e888f9SDave Hansen WARN_ONCE(last_good_offset > xstate_offsets[i], 293e6e888f9SDave Hansen "x86/fpu: misordered xstate at %d\n", last_good_offset); 294c12e13dcSYu-cheng Yu 295e6e888f9SDave Hansen last_good_offset = xstate_offsets[i]; 29639f1acd2SIngo Molnar } 29762784854SIngo Molnar } 29862784854SIngo Molnar 29932231879SIngo Molnar static void __init print_xstate_feature(u64 xstate_mask) 30062784854SIngo Molnar { 30133588b52SIngo Molnar const char *feature_name; 30262784854SIngo Molnar 30333588b52SIngo Molnar if (cpu_has_xfeatures(xstate_mask, &feature_name)) 304c8df4009SDave Hansen pr_info("x86/fpu: Supporting XSAVE feature 0x%03Lx: '%s'\n", xstate_mask, feature_name); 30562784854SIngo Molnar } 30662784854SIngo Molnar 30762784854SIngo Molnar /* 30862784854SIngo Molnar * Print out all the supported xstate features: 30962784854SIngo Molnar */ 31032231879SIngo Molnar static void __init print_xstate_features(void) 31162784854SIngo Molnar { 312d91cab78SDave Hansen print_xstate_feature(XFEATURE_MASK_FP); 313d91cab78SDave Hansen print_xstate_feature(XFEATURE_MASK_SSE); 314d91cab78SDave Hansen print_xstate_feature(XFEATURE_MASK_YMM); 315d91cab78SDave Hansen print_xstate_feature(XFEATURE_MASK_BNDREGS); 316d91cab78SDave Hansen print_xstate_feature(XFEATURE_MASK_BNDCSR); 317d91cab78SDave Hansen print_xstate_feature(XFEATURE_MASK_OPMASK); 318d91cab78SDave Hansen print_xstate_feature(XFEATURE_MASK_ZMM_Hi256); 319d91cab78SDave Hansen print_xstate_feature(XFEATURE_MASK_Hi16_ZMM); 320c8df4009SDave Hansen print_xstate_feature(XFEATURE_MASK_PKRU); 32162784854SIngo Molnar } 32262784854SIngo Molnar 32362784854SIngo Molnar /* 32403482e08SYu-cheng Yu * This check is important because it is easy to get XSTATE_* 32503482e08SYu-cheng Yu * confused with XSTATE_BIT_*. 32603482e08SYu-cheng Yu */ 32703482e08SYu-cheng Yu #define CHECK_XFEATURE(nr) do { \ 32803482e08SYu-cheng Yu WARN_ON(nr < FIRST_EXTENDED_XFEATURE); \ 32903482e08SYu-cheng Yu WARN_ON(nr >= XFEATURE_MAX); \ 33003482e08SYu-cheng Yu } while (0) 33103482e08SYu-cheng Yu 33203482e08SYu-cheng Yu /* 33303482e08SYu-cheng Yu * We could cache this like xstate_size[], but we only use 33403482e08SYu-cheng Yu * it here, so it would be a waste of space. 33503482e08SYu-cheng Yu */ 33603482e08SYu-cheng Yu static int xfeature_is_aligned(int xfeature_nr) 33703482e08SYu-cheng Yu { 33803482e08SYu-cheng Yu u32 eax, ebx, ecx, edx; 33903482e08SYu-cheng Yu 34003482e08SYu-cheng Yu CHECK_XFEATURE(xfeature_nr); 341e70b1008SYu-cheng Yu 342e70b1008SYu-cheng Yu if (!xfeature_enabled(xfeature_nr)) { 343e70b1008SYu-cheng Yu WARN_ONCE(1, "Checking alignment of disabled xfeature %d\n", 344e70b1008SYu-cheng Yu xfeature_nr); 345e70b1008SYu-cheng Yu return 0; 346e70b1008SYu-cheng Yu } 347e70b1008SYu-cheng Yu 34803482e08SYu-cheng Yu cpuid_count(XSTATE_CPUID, xfeature_nr, &eax, &ebx, &ecx, &edx); 34903482e08SYu-cheng Yu /* 35003482e08SYu-cheng Yu * The value returned by ECX[1] indicates the alignment 35103482e08SYu-cheng Yu * of state component 'i' when the compacted format 35203482e08SYu-cheng Yu * of the extended region of an XSAVE area is used: 35303482e08SYu-cheng Yu */ 35403482e08SYu-cheng Yu return !!(ecx & 2); 35503482e08SYu-cheng Yu } 35603482e08SYu-cheng Yu 35703482e08SYu-cheng Yu /* 35862784854SIngo Molnar * This function sets up offsets and sizes of all extended states in 35962784854SIngo Molnar * xsave area. This supports both standard format and compacted format 36049a91d61SYu-cheng Yu * of the xsave area. 36162784854SIngo Molnar */ 36249a91d61SYu-cheng Yu static void __init setup_xstate_comp_offsets(void) 36362784854SIngo Molnar { 36449a91d61SYu-cheng Yu unsigned int next_offset; 36562784854SIngo Molnar int i; 36662784854SIngo Molnar 36762784854SIngo Molnar /* 36862784854SIngo Molnar * The FP xstates and SSE xstates are legacy states. They are always 36962784854SIngo Molnar * in the fixed offsets in the xsave area in either compacted form 37062784854SIngo Molnar * or standard form. 37162784854SIngo Molnar */ 372446e693cSCyrill Gorcunov xstate_comp_offsets[XFEATURE_FP] = 0; 373446e693cSCyrill Gorcunov xstate_comp_offsets[XFEATURE_SSE] = offsetof(struct fxregs_state, 374446e693cSCyrill Gorcunov xmm_space); 37562784854SIngo Molnar 376782511b0SBorislav Petkov if (!boot_cpu_has(X86_FEATURE_XSAVES)) { 377ee9ae257SDave Hansen for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) { 37849a91d61SYu-cheng Yu if (xfeature_enabled(i)) 37962784854SIngo Molnar xstate_comp_offsets[i] = xstate_offsets[i]; 38062784854SIngo Molnar } 38162784854SIngo Molnar return; 38262784854SIngo Molnar } 38362784854SIngo Molnar 38449a91d61SYu-cheng Yu next_offset = FXSAVE_SIZE + XSAVE_HDR_SIZE; 38562784854SIngo Molnar 386ee9ae257SDave Hansen for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) { 38749a91d61SYu-cheng Yu if (!xfeature_enabled(i)) 38849a91d61SYu-cheng Yu continue; 38962784854SIngo Molnar 39003482e08SYu-cheng Yu if (xfeature_is_aligned(i)) 39149a91d61SYu-cheng Yu next_offset = ALIGN(next_offset, 64); 39249a91d61SYu-cheng Yu 39349a91d61SYu-cheng Yu xstate_comp_offsets[i] = next_offset; 39449a91d61SYu-cheng Yu next_offset += xstate_sizes[i]; 39562784854SIngo Molnar } 39662784854SIngo Molnar } 39762784854SIngo Molnar 39862784854SIngo Molnar /* 399eeedf153SYu-cheng Yu * Setup offsets of a supervisor-state-only XSAVES buffer: 400eeedf153SYu-cheng Yu * 401eeedf153SYu-cheng Yu * The offsets stored in xstate_comp_offsets[] only work for one specific 402eeedf153SYu-cheng Yu * value of the Requested Feature BitMap (RFBM). In cases where a different 403eeedf153SYu-cheng Yu * RFBM value is used, a different set of offsets is required. This set of 404eeedf153SYu-cheng Yu * offsets is for when RFBM=xfeatures_mask_supervisor(). 405eeedf153SYu-cheng Yu */ 406eeedf153SYu-cheng Yu static void __init setup_supervisor_only_offsets(void) 407eeedf153SYu-cheng Yu { 408eeedf153SYu-cheng Yu unsigned int next_offset; 409eeedf153SYu-cheng Yu int i; 410eeedf153SYu-cheng Yu 411eeedf153SYu-cheng Yu next_offset = FXSAVE_SIZE + XSAVE_HDR_SIZE; 412eeedf153SYu-cheng Yu 413eeedf153SYu-cheng Yu for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) { 414eeedf153SYu-cheng Yu if (!xfeature_enabled(i) || !xfeature_is_supervisor(i)) 415eeedf153SYu-cheng Yu continue; 416eeedf153SYu-cheng Yu 417eeedf153SYu-cheng Yu if (xfeature_is_aligned(i)) 418eeedf153SYu-cheng Yu next_offset = ALIGN(next_offset, 64); 419eeedf153SYu-cheng Yu 420eeedf153SYu-cheng Yu xstate_supervisor_only_offsets[i] = next_offset; 421eeedf153SYu-cheng Yu next_offset += xstate_sizes[i]; 422eeedf153SYu-cheng Yu } 423eeedf153SYu-cheng Yu } 424eeedf153SYu-cheng Yu 425eeedf153SYu-cheng Yu /* 426996952e0SYu-cheng Yu * Print out xstate component offsets and sizes 427996952e0SYu-cheng Yu */ 428996952e0SYu-cheng Yu static void __init print_xstate_offset_size(void) 429996952e0SYu-cheng Yu { 430996952e0SYu-cheng Yu int i; 431996952e0SYu-cheng Yu 432996952e0SYu-cheng Yu for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) { 433996952e0SYu-cheng Yu if (!xfeature_enabled(i)) 434996952e0SYu-cheng Yu continue; 435996952e0SYu-cheng Yu pr_info("x86/fpu: xstate_offset[%d]: %4d, xstate_sizes[%d]: %4d\n", 436996952e0SYu-cheng Yu i, xstate_comp_offsets[i], i, xstate_sizes[i]); 437996952e0SYu-cheng Yu } 438996952e0SYu-cheng Yu } 439996952e0SYu-cheng Yu 440996952e0SYu-cheng Yu /* 44162784854SIngo Molnar * setup the xstate image representing the init state 44262784854SIngo Molnar */ 44332231879SIngo Molnar static void __init setup_init_fpu_buf(void) 44462784854SIngo Molnar { 445e49a449bSRasmus Villemoes static int on_boot_cpu __initdata = 1; 446e97131a8SIngo Molnar 447e97131a8SIngo Molnar WARN_ON_FPU(!on_boot_cpu); 448e97131a8SIngo Molnar on_boot_cpu = 0; 449e97131a8SIngo Molnar 450d366bf7eSBorislav Petkov if (!boot_cpu_has(X86_FEATURE_XSAVE)) 45162784854SIngo Molnar return; 45262784854SIngo Molnar 45362784854SIngo Molnar setup_xstate_features(); 45462784854SIngo Molnar print_xstate_features(); 45562784854SIngo Molnar 4567d937060SFenghua Yu if (boot_cpu_has(X86_FEATURE_XSAVES)) 4578c9e6073SYu-cheng Yu init_fpstate.xsave.header.xcomp_bv = XCOMP_BV_COMPACTED_FORMAT | 458524bb73bSYu-cheng Yu xfeatures_mask_all; 45962784854SIngo Molnar 46062784854SIngo Molnar /* 4617d937060SFenghua Yu * Init all the features state with header.xfeatures being 0x0 46262784854SIngo Molnar */ 463d65fcd60SIngo Molnar copy_kernel_to_xregs_booting(&init_fpstate.xsave); 46462784854SIngo Molnar 46562784854SIngo Molnar /* 46662784854SIngo Molnar * Dump the init state again. This is to identify the init state 46762784854SIngo Molnar * of any feature which is not represented by all zero's. 46862784854SIngo Molnar */ 469c6813144SIngo Molnar copy_xregs_to_kernel_booting(&init_fpstate.xsave); 47062784854SIngo Molnar } 47162784854SIngo Molnar 47265ac2e9bSDave Hansen static int xfeature_uncompacted_offset(int xfeature_nr) 47365ac2e9bSDave Hansen { 47465ac2e9bSDave Hansen u32 eax, ebx, ecx, edx; 47565ac2e9bSDave Hansen 4761499ce2dSYu-cheng Yu /* 4771499ce2dSYu-cheng Yu * Only XSAVES supports supervisor states and it uses compacted 4781499ce2dSYu-cheng Yu * format. Checking a supervisor state's uncompacted offset is 4791499ce2dSYu-cheng Yu * an error. 4801499ce2dSYu-cheng Yu */ 4818ab22804SFenghua Yu if (XFEATURE_MASK_SUPERVISOR_ALL & BIT_ULL(xfeature_nr)) { 4821499ce2dSYu-cheng Yu WARN_ONCE(1, "No fixed offset for xstate %d\n", xfeature_nr); 4831499ce2dSYu-cheng Yu return -1; 4841499ce2dSYu-cheng Yu } 4851499ce2dSYu-cheng Yu 48665ac2e9bSDave Hansen CHECK_XFEATURE(xfeature_nr); 48765ac2e9bSDave Hansen cpuid_count(XSTATE_CPUID, xfeature_nr, &eax, &ebx, &ecx, &edx); 48865ac2e9bSDave Hansen return ebx; 48965ac2e9bSDave Hansen } 49065ac2e9bSDave Hansen 49165ac2e9bSDave Hansen static int xfeature_size(int xfeature_nr) 49265ac2e9bSDave Hansen { 49365ac2e9bSDave Hansen u32 eax, ebx, ecx, edx; 49465ac2e9bSDave Hansen 49565ac2e9bSDave Hansen CHECK_XFEATURE(xfeature_nr); 49665ac2e9bSDave Hansen cpuid_count(XSTATE_CPUID, xfeature_nr, &eax, &ebx, &ecx, &edx); 49765ac2e9bSDave Hansen return eax; 49865ac2e9bSDave Hansen } 49965ac2e9bSDave Hansen 50065ac2e9bSDave Hansen /* 50165ac2e9bSDave Hansen * 'XSAVES' implies two different things: 50265ac2e9bSDave Hansen * 1. saving of supervisor/system state 50365ac2e9bSDave Hansen * 2. using the compacted format 50465ac2e9bSDave Hansen * 50565ac2e9bSDave Hansen * Use this function when dealing with the compacted format so 50665ac2e9bSDave Hansen * that it is obvious which aspect of 'XSAVES' is being handled 50765ac2e9bSDave Hansen * by the calling code. 50865ac2e9bSDave Hansen */ 50999aa22d0SYu-cheng Yu int using_compacted_format(void) 51065ac2e9bSDave Hansen { 511782511b0SBorislav Petkov return boot_cpu_has(X86_FEATURE_XSAVES); 51265ac2e9bSDave Hansen } 51365ac2e9bSDave Hansen 514e63e5d5cSEric Biggers /* Validate an xstate header supplied by userspace (ptrace or sigreturn) */ 5155274e6c1SFenghua Yu int validate_user_xstate_header(const struct xstate_header *hdr) 516e63e5d5cSEric Biggers { 517e63e5d5cSEric Biggers /* No unknown or supervisor features may be set */ 518524bb73bSYu-cheng Yu if (hdr->xfeatures & ~xfeatures_mask_user()) 519e63e5d5cSEric Biggers return -EINVAL; 520e63e5d5cSEric Biggers 521e63e5d5cSEric Biggers /* Userspace must use the uncompacted format */ 522e63e5d5cSEric Biggers if (hdr->xcomp_bv) 523e63e5d5cSEric Biggers return -EINVAL; 524e63e5d5cSEric Biggers 525e63e5d5cSEric Biggers /* 526e63e5d5cSEric Biggers * If 'reserved' is shrunken to add a new field, make sure to validate 527e63e5d5cSEric Biggers * that new field here! 528e63e5d5cSEric Biggers */ 529e63e5d5cSEric Biggers BUILD_BUG_ON(sizeof(hdr->reserved) != 48); 530e63e5d5cSEric Biggers 531e63e5d5cSEric Biggers /* No reserved bits may be set */ 532e63e5d5cSEric Biggers if (memchr_inv(hdr->reserved, 0, sizeof(hdr->reserved))) 533e63e5d5cSEric Biggers return -EINVAL; 534e63e5d5cSEric Biggers 535e63e5d5cSEric Biggers return 0; 536e63e5d5cSEric Biggers } 537e63e5d5cSEric Biggers 53865ac2e9bSDave Hansen static void __xstate_dump_leaves(void) 53965ac2e9bSDave Hansen { 54065ac2e9bSDave Hansen int i; 54165ac2e9bSDave Hansen u32 eax, ebx, ecx, edx; 54265ac2e9bSDave Hansen static int should_dump = 1; 54365ac2e9bSDave Hansen 54465ac2e9bSDave Hansen if (!should_dump) 54565ac2e9bSDave Hansen return; 54665ac2e9bSDave Hansen should_dump = 0; 54765ac2e9bSDave Hansen /* 54865ac2e9bSDave Hansen * Dump out a few leaves past the ones that we support 54965ac2e9bSDave Hansen * just in case there are some goodies up there 55065ac2e9bSDave Hansen */ 55165ac2e9bSDave Hansen for (i = 0; i < XFEATURE_MAX + 10; i++) { 55265ac2e9bSDave Hansen cpuid_count(XSTATE_CPUID, i, &eax, &ebx, &ecx, &edx); 55365ac2e9bSDave Hansen pr_warn("CPUID[%02x, %02x]: eax=%08x ebx=%08x ecx=%08x edx=%08x\n", 55465ac2e9bSDave Hansen XSTATE_CPUID, i, eax, ebx, ecx, edx); 55565ac2e9bSDave Hansen } 55665ac2e9bSDave Hansen } 55765ac2e9bSDave Hansen 55865ac2e9bSDave Hansen #define XSTATE_WARN_ON(x) do { \ 55965ac2e9bSDave Hansen if (WARN_ONCE(x, "XSAVE consistency problem, dumping leaves")) { \ 56065ac2e9bSDave Hansen __xstate_dump_leaves(); \ 56165ac2e9bSDave Hansen } \ 56265ac2e9bSDave Hansen } while (0) 56365ac2e9bSDave Hansen 564ef78f2a4SDave Hansen #define XCHECK_SZ(sz, nr, nr_macro, __struct) do { \ 565ef78f2a4SDave Hansen if ((nr == nr_macro) && \ 566ef78f2a4SDave Hansen WARN_ONCE(sz != sizeof(__struct), \ 567ef78f2a4SDave Hansen "%s: struct is %zu bytes, cpu state %d bytes\n", \ 568ef78f2a4SDave Hansen __stringify(nr_macro), sizeof(__struct), sz)) { \ 569ef78f2a4SDave Hansen __xstate_dump_leaves(); \ 570ef78f2a4SDave Hansen } \ 571ef78f2a4SDave Hansen } while (0) 572ef78f2a4SDave Hansen 573ef78f2a4SDave Hansen /* 574ef78f2a4SDave Hansen * We have a C struct for each 'xstate'. We need to ensure 575ef78f2a4SDave Hansen * that our software representation matches what the CPU 576ef78f2a4SDave Hansen * tells us about the state's size. 577ef78f2a4SDave Hansen */ 578ef78f2a4SDave Hansen static void check_xstate_against_struct(int nr) 579ef78f2a4SDave Hansen { 580ef78f2a4SDave Hansen /* 581ef78f2a4SDave Hansen * Ask the CPU for the size of the state. 582ef78f2a4SDave Hansen */ 583ef78f2a4SDave Hansen int sz = xfeature_size(nr); 584ef78f2a4SDave Hansen /* 585ef78f2a4SDave Hansen * Match each CPU state with the corresponding software 586ef78f2a4SDave Hansen * structure. 587ef78f2a4SDave Hansen */ 588ef78f2a4SDave Hansen XCHECK_SZ(sz, nr, XFEATURE_YMM, struct ymmh_struct); 589ef78f2a4SDave Hansen XCHECK_SZ(sz, nr, XFEATURE_BNDREGS, struct mpx_bndreg_state); 590ef78f2a4SDave Hansen XCHECK_SZ(sz, nr, XFEATURE_BNDCSR, struct mpx_bndcsr_state); 591ef78f2a4SDave Hansen XCHECK_SZ(sz, nr, XFEATURE_OPMASK, struct avx_512_opmask_state); 592ef78f2a4SDave Hansen XCHECK_SZ(sz, nr, XFEATURE_ZMM_Hi256, struct avx_512_zmm_uppers_state); 593ef78f2a4SDave Hansen XCHECK_SZ(sz, nr, XFEATURE_Hi16_ZMM, struct avx_512_hi16_state); 594c8df4009SDave Hansen XCHECK_SZ(sz, nr, XFEATURE_PKRU, struct pkru_state); 595ef78f2a4SDave Hansen 596ef78f2a4SDave Hansen /* 597ef78f2a4SDave Hansen * Make *SURE* to add any feature numbers in below if 598ef78f2a4SDave Hansen * there are "holes" in the xsave state component 599ef78f2a4SDave Hansen * numbers. 600ef78f2a4SDave Hansen */ 601ef78f2a4SDave Hansen if ((nr < XFEATURE_YMM) || 6021f96b1efSDave Hansen (nr >= XFEATURE_MAX) || 603f0dccc9dSKan Liang (nr == XFEATURE_PT_UNIMPLEMENTED_SO_FAR) || 604f0dccc9dSKan Liang ((nr >= XFEATURE_RSRVD_COMP_10) && (nr <= XFEATURE_LBR))) { 605ef78f2a4SDave Hansen WARN_ONCE(1, "no structure for xstate: %d\n", nr); 606ef78f2a4SDave Hansen XSTATE_WARN_ON(1); 607ef78f2a4SDave Hansen } 608ef78f2a4SDave Hansen } 609ef78f2a4SDave Hansen 61065ac2e9bSDave Hansen /* 61165ac2e9bSDave Hansen * This essentially double-checks what the cpu told us about 61265ac2e9bSDave Hansen * how large the XSAVE buffer needs to be. We are recalculating 61365ac2e9bSDave Hansen * it to be safe. 61465ac2e9bSDave Hansen */ 61565ac2e9bSDave Hansen static void do_extra_xstate_size_checks(void) 61665ac2e9bSDave Hansen { 61765ac2e9bSDave Hansen int paranoid_xstate_size = FXSAVE_SIZE + XSAVE_HDR_SIZE; 61865ac2e9bSDave Hansen int i; 61965ac2e9bSDave Hansen 62065ac2e9bSDave Hansen for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) { 62165ac2e9bSDave Hansen if (!xfeature_enabled(i)) 62265ac2e9bSDave Hansen continue; 623ef78f2a4SDave Hansen 624ef78f2a4SDave Hansen check_xstate_against_struct(i); 62565ac2e9bSDave Hansen /* 62665ac2e9bSDave Hansen * Supervisor state components can be managed only by 62765ac2e9bSDave Hansen * XSAVES, which is compacted-format only. 62865ac2e9bSDave Hansen */ 62965ac2e9bSDave Hansen if (!using_compacted_format()) 63065ac2e9bSDave Hansen XSTATE_WARN_ON(xfeature_is_supervisor(i)); 63165ac2e9bSDave Hansen 63265ac2e9bSDave Hansen /* Align from the end of the previous feature */ 63365ac2e9bSDave Hansen if (xfeature_is_aligned(i)) 63465ac2e9bSDave Hansen paranoid_xstate_size = ALIGN(paranoid_xstate_size, 64); 63565ac2e9bSDave Hansen /* 63665ac2e9bSDave Hansen * The offset of a given state in the non-compacted 63765ac2e9bSDave Hansen * format is given to us in a CPUID leaf. We check 63865ac2e9bSDave Hansen * them for being ordered (increasing offsets) in 63965ac2e9bSDave Hansen * setup_xstate_features(). 64065ac2e9bSDave Hansen */ 64165ac2e9bSDave Hansen if (!using_compacted_format()) 64265ac2e9bSDave Hansen paranoid_xstate_size = xfeature_uncompacted_offset(i); 64365ac2e9bSDave Hansen /* 64465ac2e9bSDave Hansen * The compacted-format offset always depends on where 64565ac2e9bSDave Hansen * the previous state ended. 64665ac2e9bSDave Hansen */ 64765ac2e9bSDave Hansen paranoid_xstate_size += xfeature_size(i); 64865ac2e9bSDave Hansen } 649bf15a8cfSFenghua Yu XSTATE_WARN_ON(paranoid_xstate_size != fpu_kernel_xstate_size); 65065ac2e9bSDave Hansen } 65165ac2e9bSDave Hansen 652a1141e0bSFenghua Yu 65362784854SIngo Molnar /* 654524bb73bSYu-cheng Yu * Get total size of enabled xstates in XCR0 | IA32_XSS. 65565ac2e9bSDave Hansen * 65665ac2e9bSDave Hansen * Note the SDM's wording here. "sub-function 0" only enumerates 65765ac2e9bSDave Hansen * the size of the *user* states. If we use it to size a buffer 65865ac2e9bSDave Hansen * that we use 'XSAVES' on, we could potentially overflow the 65965ac2e9bSDave Hansen * buffer because 'XSAVES' saves system states too. 66062784854SIngo Molnar */ 661a1141e0bSFenghua Yu static unsigned int __init get_xsaves_size(void) 66262784854SIngo Molnar { 66362784854SIngo Molnar unsigned int eax, ebx, ecx, edx; 66465ac2e9bSDave Hansen /* 66565ac2e9bSDave Hansen * - CPUID function 0DH, sub-function 1: 66665ac2e9bSDave Hansen * EBX enumerates the size (in bytes) required by 66765ac2e9bSDave Hansen * the XSAVES instruction for an XSAVE area 66865ac2e9bSDave Hansen * containing all the state components 66965ac2e9bSDave Hansen * corresponding to bits currently set in 67065ac2e9bSDave Hansen * XCR0 | IA32_XSS. 67165ac2e9bSDave Hansen */ 67265ac2e9bSDave Hansen cpuid_count(XSTATE_CPUID, 1, &eax, &ebx, &ecx, &edx); 673a1141e0bSFenghua Yu return ebx; 67462784854SIngo Molnar } 675a1141e0bSFenghua Yu 676a1141e0bSFenghua Yu static unsigned int __init get_xsave_size(void) 677a1141e0bSFenghua Yu { 678a1141e0bSFenghua Yu unsigned int eax, ebx, ecx, edx; 679a1141e0bSFenghua Yu /* 680a1141e0bSFenghua Yu * - CPUID function 0DH, sub-function 0: 681a1141e0bSFenghua Yu * EBX enumerates the size (in bytes) required by 682a1141e0bSFenghua Yu * the XSAVE instruction for an XSAVE area 683a1141e0bSFenghua Yu * containing all the *user* state components 684a1141e0bSFenghua Yu * corresponding to bits currently set in XCR0. 685a1141e0bSFenghua Yu */ 686a1141e0bSFenghua Yu cpuid_count(XSTATE_CPUID, 0, &eax, &ebx, &ecx, &edx); 687a1141e0bSFenghua Yu return ebx; 6884109ca06SDave Hansen } 6894109ca06SDave Hansen 6904109ca06SDave Hansen /* 6914109ca06SDave Hansen * Will the runtime-enumerated 'xstate_size' fit in the init 6924109ca06SDave Hansen * task's statically-allocated buffer? 6934109ca06SDave Hansen */ 6944109ca06SDave Hansen static bool is_supported_xstate_size(unsigned int test_xstate_size) 6954109ca06SDave Hansen { 6964109ca06SDave Hansen if (test_xstate_size <= sizeof(union fpregs_state)) 6974109ca06SDave Hansen return true; 6984109ca06SDave Hansen 6994109ca06SDave Hansen pr_warn("x86/fpu: xstate buffer too small (%zu < %d), disabling xsave\n", 7004109ca06SDave Hansen sizeof(union fpregs_state), test_xstate_size); 7014109ca06SDave Hansen return false; 7024109ca06SDave Hansen } 7034109ca06SDave Hansen 704653a561bSSergey Senozhatsky static int __init init_xstate_size(void) 7054109ca06SDave Hansen { 7064109ca06SDave Hansen /* Recompute the context size for enabled features: */ 707a1141e0bSFenghua Yu unsigned int possible_xstate_size; 708a1141e0bSFenghua Yu unsigned int xsave_size; 709a1141e0bSFenghua Yu 710a1141e0bSFenghua Yu xsave_size = get_xsave_size(); 711a1141e0bSFenghua Yu 712a1141e0bSFenghua Yu if (boot_cpu_has(X86_FEATURE_XSAVES)) 713a1141e0bSFenghua Yu possible_xstate_size = get_xsaves_size(); 714a1141e0bSFenghua Yu else 715a1141e0bSFenghua Yu possible_xstate_size = xsave_size; 7164109ca06SDave Hansen 7174109ca06SDave Hansen /* Ensure we have the space to store all enabled: */ 7184109ca06SDave Hansen if (!is_supported_xstate_size(possible_xstate_size)) 7194109ca06SDave Hansen return -EINVAL; 7204109ca06SDave Hansen 7214109ca06SDave Hansen /* 7224109ca06SDave Hansen * The size is OK, we are definitely going to use xsave, 7234109ca06SDave Hansen * make it known to the world that we need more space. 7244109ca06SDave Hansen */ 725bf15a8cfSFenghua Yu fpu_kernel_xstate_size = possible_xstate_size; 72665ac2e9bSDave Hansen do_extra_xstate_size_checks(); 727a1141e0bSFenghua Yu 728a1141e0bSFenghua Yu /* 729a1141e0bSFenghua Yu * User space is always in standard format. 730a1141e0bSFenghua Yu */ 731a1141e0bSFenghua Yu fpu_user_xstate_size = xsave_size; 7324109ca06SDave Hansen return 0; 7334109ca06SDave Hansen } 7344109ca06SDave Hansen 735d91cab78SDave Hansen /* 736d91cab78SDave Hansen * We enabled the XSAVE hardware, but something went wrong and 737d91cab78SDave Hansen * we can not use it. Disable it. 738d91cab78SDave Hansen */ 739d91cab78SDave Hansen static void fpu__init_disable_system_xstate(void) 7404109ca06SDave Hansen { 741524bb73bSYu-cheng Yu xfeatures_mask_all = 0; 7424109ca06SDave Hansen cr4_clear_bits(X86_CR4_OSXSAVE); 7437891bc0aSSebastian Andrzej Siewior setup_clear_cpu_cap(X86_FEATURE_XSAVE); 74462784854SIngo Molnar } 74562784854SIngo Molnar 74662784854SIngo Molnar /* 74762784854SIngo Molnar * Enable and initialize the xsave feature. 74862784854SIngo Molnar * Called once per system bootup. 74962784854SIngo Molnar */ 75032231879SIngo Molnar void __init fpu__init_system_xstate(void) 75162784854SIngo Molnar { 75262784854SIngo Molnar unsigned int eax, ebx, ecx, edx; 753e49a449bSRasmus Villemoes static int on_boot_cpu __initdata = 1; 7544109ca06SDave Hansen int err; 755ccb18db2SAndi Kleen int i; 756e97131a8SIngo Molnar 757e97131a8SIngo Molnar WARN_ON_FPU(!on_boot_cpu); 758e97131a8SIngo Molnar on_boot_cpu = 0; 75962784854SIngo Molnar 7609170fb40SAndy Lutomirski if (!boot_cpu_has(X86_FEATURE_FPU)) { 7619170fb40SAndy Lutomirski pr_info("x86/fpu: No FPU detected\n"); 7629170fb40SAndy Lutomirski return; 7639170fb40SAndy Lutomirski } 7649170fb40SAndy Lutomirski 765d366bf7eSBorislav Petkov if (!boot_cpu_has(X86_FEATURE_XSAVE)) { 7669170fb40SAndy Lutomirski pr_info("x86/fpu: x87 FPU will use %s\n", 7679170fb40SAndy Lutomirski boot_cpu_has(X86_FEATURE_FXSR) ? "FXSAVE" : "FSAVE"); 76862784854SIngo Molnar return; 76962784854SIngo Molnar } 77062784854SIngo Molnar 77162784854SIngo Molnar if (boot_cpu_data.cpuid_level < XSTATE_CPUID) { 772e97131a8SIngo Molnar WARN_ON_FPU(1); 77362784854SIngo Molnar return; 77462784854SIngo Molnar } 77562784854SIngo Molnar 776524bb73bSYu-cheng Yu /* 777524bb73bSYu-cheng Yu * Find user xstates supported by the processor. 778524bb73bSYu-cheng Yu */ 77962784854SIngo Molnar cpuid_count(XSTATE_CPUID, 0, &eax, &ebx, &ecx, &edx); 780524bb73bSYu-cheng Yu xfeatures_mask_all = eax + ((u64)edx << 32); 78162784854SIngo Molnar 78271581eefSYu-cheng Yu /* 78371581eefSYu-cheng Yu * Find supervisor xstates supported by the processor. 78471581eefSYu-cheng Yu */ 78571581eefSYu-cheng Yu cpuid_count(XSTATE_CPUID, 1, &eax, &ebx, &ecx, &edx); 78671581eefSYu-cheng Yu xfeatures_mask_all |= ecx + ((u64)edx << 32); 78771581eefSYu-cheng Yu 788524bb73bSYu-cheng Yu if ((xfeatures_mask_user() & XFEATURE_MASK_FPSSE) != XFEATURE_MASK_FPSSE) { 789ec3ed4a2SDave Hansen /* 790ec3ed4a2SDave Hansen * This indicates that something really unexpected happened 791ec3ed4a2SDave Hansen * with the enumeration. Disable XSAVE and try to continue 792ec3ed4a2SDave Hansen * booting without it. This is too early to BUG(). 793ec3ed4a2SDave Hansen */ 794524bb73bSYu-cheng Yu pr_err("x86/fpu: FP/SSE not present amongst the CPU's xstate features: 0x%llx.\n", 795524bb73bSYu-cheng Yu xfeatures_mask_all); 796ec3ed4a2SDave Hansen goto out_disable; 79762784854SIngo Molnar } 79862784854SIngo Molnar 799ccb18db2SAndi Kleen /* 800ccb18db2SAndi Kleen * Clear XSAVE features that are disabled in the normal CPUID. 801ccb18db2SAndi Kleen */ 802ccb18db2SAndi Kleen for (i = 0; i < ARRAY_SIZE(xsave_cpuid_features); i++) { 803ccb18db2SAndi Kleen if (!boot_cpu_has(xsave_cpuid_features[i])) 804524bb73bSYu-cheng Yu xfeatures_mask_all &= ~BIT_ULL(i); 805ccb18db2SAndi Kleen } 806ccb18db2SAndi Kleen 807524bb73bSYu-cheng Yu xfeatures_mask_all &= fpu__get_supported_xfeatures_mask(); 80862784854SIngo Molnar 80962784854SIngo Molnar /* Enable xstate instructions to be able to continue with initialization: */ 81062784854SIngo Molnar fpu__init_cpu_xstate(); 8114109ca06SDave Hansen err = init_xstate_size(); 812ec3ed4a2SDave Hansen if (err) 813ec3ed4a2SDave Hansen goto out_disable; 81462784854SIngo Molnar 81591c3dba7SYu-cheng Yu /* 81691c3dba7SYu-cheng Yu * Update info used for ptrace frames; use standard-format size and no 81791c3dba7SYu-cheng Yu * supervisor xstates: 81891c3dba7SYu-cheng Yu */ 819524bb73bSYu-cheng Yu update_regset_xstate_info(fpu_user_xstate_size, xfeatures_mask_user()); 82091c3dba7SYu-cheng Yu 821b992c660SIngo Molnar fpu__init_prepare_fx_sw_frame(); 82262784854SIngo Molnar setup_init_fpu_buf(); 82349a91d61SYu-cheng Yu setup_xstate_comp_offsets(); 824eeedf153SYu-cheng Yu setup_supervisor_only_offsets(); 825996952e0SYu-cheng Yu print_xstate_offset_size(); 82662784854SIngo Molnar 827b0815359SDave Hansen pr_info("x86/fpu: Enabled xstate features 0x%llx, context size is %d bytes, using '%s' format.\n", 828524bb73bSYu-cheng Yu xfeatures_mask_all, 829bf15a8cfSFenghua Yu fpu_kernel_xstate_size, 830782511b0SBorislav Petkov boot_cpu_has(X86_FEATURE_XSAVES) ? "compacted" : "standard"); 831ec3ed4a2SDave Hansen return; 832ec3ed4a2SDave Hansen 833ec3ed4a2SDave Hansen out_disable: 834ec3ed4a2SDave Hansen /* something went wrong, try to boot without any XSAVE support */ 835ec3ed4a2SDave Hansen fpu__init_disable_system_xstate(); 83662784854SIngo Molnar } 83762784854SIngo Molnar 83862784854SIngo Molnar /* 83962784854SIngo Molnar * Restore minimal FPU state after suspend: 84062784854SIngo Molnar */ 84162784854SIngo Molnar void fpu__resume_cpu(void) 84262784854SIngo Molnar { 84362784854SIngo Molnar /* 84462784854SIngo Molnar * Restore XCR0 on xsave capable CPUs: 84562784854SIngo Molnar */ 846d366bf7eSBorislav Petkov if (boot_cpu_has(X86_FEATURE_XSAVE)) 847524bb73bSYu-cheng Yu xsetbv(XCR_XFEATURE_ENABLED_MASK, xfeatures_mask_user()); 84871581eefSYu-cheng Yu 84971581eefSYu-cheng Yu /* 85071581eefSYu-cheng Yu * Restore IA32_XSS. The same CPUID bit enumerates support 85171581eefSYu-cheng Yu * of XSAVES and MSR_IA32_XSS. 85271581eefSYu-cheng Yu */ 853f0dccc9dSKan Liang if (boot_cpu_has(X86_FEATURE_XSAVES)) { 854f0dccc9dSKan Liang wrmsrl(MSR_IA32_XSS, xfeatures_mask_supervisor() | 855f0dccc9dSKan Liang xfeatures_mask_dynamic()); 856f0dccc9dSKan Liang } 85762784854SIngo Molnar } 85862784854SIngo Molnar 85962784854SIngo Molnar /* 86007baeb04SSebastian Andrzej Siewior * Given an xstate feature nr, calculate where in the xsave 861b8b9b6baSDave Hansen * buffer the state is. Callers should ensure that the buffer 862b8b9b6baSDave Hansen * is valid. 863b8b9b6baSDave Hansen */ 86407baeb04SSebastian Andrzej Siewior static void *__raw_xsave_addr(struct xregs_state *xsave, int xfeature_nr) 865b8b9b6baSDave Hansen { 86607baeb04SSebastian Andrzej Siewior if (!xfeature_enabled(xfeature_nr)) { 8675060b915SYu-cheng Yu WARN_ON_FPU(1); 8685060b915SYu-cheng Yu return NULL; 8695060b915SYu-cheng Yu } 8705060b915SYu-cheng Yu 87107baeb04SSebastian Andrzej Siewior return (void *)xsave + xstate_comp_offsets[xfeature_nr]; 872b8b9b6baSDave Hansen } 873b8b9b6baSDave Hansen /* 87462784854SIngo Molnar * Given the xsave area and a state inside, this function returns the 87562784854SIngo Molnar * address of the state. 87662784854SIngo Molnar * 87762784854SIngo Molnar * This is the API that is called to get xstate address in either 87862784854SIngo Molnar * standard format or compacted format of xsave area. 87962784854SIngo Molnar * 8800c4109beSDave Hansen * Note that if there is no data for the field in the xsave buffer 8810c4109beSDave Hansen * this will return NULL. 8820c4109beSDave Hansen * 88362784854SIngo Molnar * Inputs: 8840c4109beSDave Hansen * xstate: the thread's storage area for all FPU data 885abd16d68SSebastian Andrzej Siewior * xfeature_nr: state which is defined in xsave.h (e.g. XFEATURE_FP, 886abd16d68SSebastian Andrzej Siewior * XFEATURE_SSE, etc...) 88762784854SIngo Molnar * Output: 8880c4109beSDave Hansen * address of the state in the xsave area, or NULL if the 8890c4109beSDave Hansen * field is not present in the xsave buffer. 89062784854SIngo Molnar */ 891abd16d68SSebastian Andrzej Siewior void *get_xsave_addr(struct xregs_state *xsave, int xfeature_nr) 89262784854SIngo Molnar { 8930c4109beSDave Hansen /* 8940c4109beSDave Hansen * Do we even *have* xsave state? 8950c4109beSDave Hansen */ 8960c4109beSDave Hansen if (!boot_cpu_has(X86_FEATURE_XSAVE)) 89762784854SIngo Molnar return NULL; 89862784854SIngo Molnar 8990c4109beSDave Hansen /* 9000c4109beSDave Hansen * We should not ever be requesting features that we 901524bb73bSYu-cheng Yu * have not enabled. 9020c4109beSDave Hansen */ 903524bb73bSYu-cheng Yu WARN_ONCE(!(xfeatures_mask_all & BIT_ULL(xfeature_nr)), 9040c4109beSDave Hansen "get of unsupported state"); 9050c4109beSDave Hansen /* 9060c4109beSDave Hansen * This assumes the last 'xsave*' instruction to 907abd16d68SSebastian Andrzej Siewior * have requested that 'xfeature_nr' be saved. 9080c4109beSDave Hansen * If it did not, we might be seeing and old value 9090c4109beSDave Hansen * of the field in the buffer. 9100c4109beSDave Hansen * 9110c4109beSDave Hansen * This can happen because the last 'xsave' did not 9120c4109beSDave Hansen * request that this feature be saved (unlikely) 9130c4109beSDave Hansen * or because the "init optimization" caused it 9140c4109beSDave Hansen * to not be saved. 9150c4109beSDave Hansen */ 916abd16d68SSebastian Andrzej Siewior if (!(xsave->header.xfeatures & BIT_ULL(xfeature_nr))) 9170c4109beSDave Hansen return NULL; 9180c4109beSDave Hansen 91907baeb04SSebastian Andrzej Siewior return __raw_xsave_addr(xsave, xfeature_nr); 92062784854SIngo Molnar } 92162784854SIngo Molnar EXPORT_SYMBOL_GPL(get_xsave_addr); 92204cd027bSDave Hansen 92304cd027bSDave Hansen /* 92404cd027bSDave Hansen * This wraps up the common operations that need to occur when retrieving 92504cd027bSDave Hansen * data from xsave state. It first ensures that the current task was 92604cd027bSDave Hansen * using the FPU and retrieves the data in to a buffer. It then calculates 92704cd027bSDave Hansen * the offset of the requested field in the buffer. 92804cd027bSDave Hansen * 92904cd027bSDave Hansen * This function is safe to call whether the FPU is in use or not. 93004cd027bSDave Hansen * 93104cd027bSDave Hansen * Note that this only works on the current task. 93204cd027bSDave Hansen * 93304cd027bSDave Hansen * Inputs: 934abd16d68SSebastian Andrzej Siewior * @xfeature_nr: state which is defined in xsave.h (e.g. XFEATURE_FP, 935abd16d68SSebastian Andrzej Siewior * XFEATURE_SSE, etc...) 93604cd027bSDave Hansen * Output: 93704cd027bSDave Hansen * address of the state in the xsave area or NULL if the state 93804cd027bSDave Hansen * is not present or is in its 'init state'. 93904cd027bSDave Hansen */ 940abd16d68SSebastian Andrzej Siewior const void *get_xsave_field_ptr(int xfeature_nr) 94104cd027bSDave Hansen { 94204cd027bSDave Hansen struct fpu *fpu = ¤t->thread.fpu; 94304cd027bSDave Hansen 94404cd027bSDave Hansen /* 94504cd027bSDave Hansen * fpu__save() takes the CPU's xstate registers 94604cd027bSDave Hansen * and saves them off to the 'fpu memory buffer. 94704cd027bSDave Hansen */ 94804cd027bSDave Hansen fpu__save(fpu); 94904cd027bSDave Hansen 950abd16d68SSebastian Andrzej Siewior return get_xsave_addr(&fpu->state.xsave, xfeature_nr); 95104cd027bSDave Hansen } 952b8b9b6baSDave Hansen 953e8c24d3aSDave Hansen #ifdef CONFIG_ARCH_HAS_PKEYS 954e8c24d3aSDave Hansen 95584594296SDave Hansen /* 956b79daf85SDave Hansen * This will go out and modify PKRU register to set the access 957b79daf85SDave Hansen * rights for @pkey to @init_val. 95884594296SDave Hansen */ 95984594296SDave Hansen int arch_set_user_pkey_access(struct task_struct *tsk, int pkey, 96084594296SDave Hansen unsigned long init_val) 96184594296SDave Hansen { 962b79daf85SDave Hansen u32 old_pkru; 96384594296SDave Hansen int pkey_shift = (pkey * PKRU_BITS_PER_PKEY); 96484594296SDave Hansen u32 new_pkru_bits = 0; 96584594296SDave Hansen 96684594296SDave Hansen /* 96784594296SDave Hansen * This check implies XSAVE support. OSPKE only gets 96884594296SDave Hansen * set if we enable XSAVE and we enable PKU in XCR0. 96984594296SDave Hansen */ 97084594296SDave Hansen if (!boot_cpu_has(X86_FEATURE_OSPKE)) 97184594296SDave Hansen return -EINVAL; 97284594296SDave Hansen 97316171bffSDave Hansen /* 97416171bffSDave Hansen * This code should only be called with valid 'pkey' 97516171bffSDave Hansen * values originating from in-kernel users. Complain 97616171bffSDave Hansen * if a bad value is observed. 97716171bffSDave Hansen */ 97816171bffSDave Hansen WARN_ON_ONCE(pkey >= arch_max_pkey()); 97916171bffSDave Hansen 98091c3dba7SYu-cheng Yu /* Set the bits we need in PKRU: */ 98184594296SDave Hansen if (init_val & PKEY_DISABLE_ACCESS) 98284594296SDave Hansen new_pkru_bits |= PKRU_AD_BIT; 98384594296SDave Hansen if (init_val & PKEY_DISABLE_WRITE) 98484594296SDave Hansen new_pkru_bits |= PKRU_WD_BIT; 98584594296SDave Hansen 98691c3dba7SYu-cheng Yu /* Shift the bits in to the correct place in PKRU for pkey: */ 98784594296SDave Hansen new_pkru_bits <<= pkey_shift; 98884594296SDave Hansen 989b79daf85SDave Hansen /* Get old PKRU and mask off any old bits in place: */ 990b79daf85SDave Hansen old_pkru = read_pkru(); 991b79daf85SDave Hansen old_pkru &= ~((PKRU_AD_BIT|PKRU_WD_BIT) << pkey_shift); 99284594296SDave Hansen 993b79daf85SDave Hansen /* Write old part along with new part: */ 994b79daf85SDave Hansen write_pkru(old_pkru | new_pkru_bits); 99591c3dba7SYu-cheng Yu 99691c3dba7SYu-cheng Yu return 0; 99791c3dba7SYu-cheng Yu } 998e8c24d3aSDave Hansen #endif /* ! CONFIG_ARCH_HAS_PKEYS */ 99991c3dba7SYu-cheng Yu 100091c3dba7SYu-cheng Yu /* 10010852b374SRik van Riel * Weird legacy quirk: SSE and YMM states store information in the 10020852b374SRik van Riel * MXCSR and MXCSR_FLAGS fields of the FP area. That means if the FP 10030852b374SRik van Riel * area is marked as unused in the xfeatures header, we need to copy 10040852b374SRik van Riel * MXCSR and MXCSR_FLAGS if either SSE or YMM are in use. 10050852b374SRik van Riel */ 10060852b374SRik van Riel static inline bool xfeatures_mxcsr_quirk(u64 xfeatures) 10070852b374SRik van Riel { 10080852b374SRik van Riel if (!(xfeatures & (XFEATURE_MASK_SSE|XFEATURE_MASK_YMM))) 10094f8cef59Skbuild test robot return false; 10100852b374SRik van Riel 10110852b374SRik van Riel if (xfeatures & XFEATURE_MASK_FP) 10124f8cef59Skbuild test robot return false; 10130852b374SRik van Riel 10144f8cef59Skbuild test robot return true; 10150852b374SRik van Riel } 10160852b374SRik van Riel 10179e463654SAl Viro static void fill_gap(unsigned to, void **kbuf, unsigned *pos, unsigned *count) 1018f0d4f30aSIngo Molnar { 10199e463654SAl Viro if (*pos < to) { 10209e463654SAl Viro unsigned size = to - *pos; 1021f0d4f30aSIngo Molnar 10229e463654SAl Viro if (size > *count) 10239e463654SAl Viro size = *count; 10249e463654SAl Viro memcpy(*kbuf, (void *)&init_fpstate.xsave + *pos, size); 10259e463654SAl Viro *kbuf += size; 10269e463654SAl Viro *pos += size; 10279e463654SAl Viro *count -= size; 10289e463654SAl Viro } 10299e463654SAl Viro } 10309e463654SAl Viro 10319e463654SAl Viro static void copy_part(unsigned offset, unsigned size, void *from, 10329e463654SAl Viro void **kbuf, unsigned *pos, unsigned *count) 10339e463654SAl Viro { 10349e463654SAl Viro fill_gap(offset, kbuf, pos, count); 10359e463654SAl Viro if (size > *count) 10369e463654SAl Viro size = *count; 10379e463654SAl Viro if (size) { 10389e463654SAl Viro memcpy(*kbuf, from, size); 10399e463654SAl Viro *kbuf += size; 10409e463654SAl Viro *pos += size; 10419e463654SAl Viro *count -= size; 1042f0d4f30aSIngo Molnar } 1043f0d4f30aSIngo Molnar } 1044f0d4f30aSIngo Molnar 1045f0d4f30aSIngo Molnar /* 1046f0d4f30aSIngo Molnar * Convert from kernel XSAVES compacted format to standard format and copy 1047f0d4f30aSIngo Molnar * to a kernel-space ptrace buffer. 1048f0d4f30aSIngo Molnar * 1049f0d4f30aSIngo Molnar * It supports partial copy but pos always starts from zero. This is called 1050f0d4f30aSIngo Molnar * from xstateregs_get() and there we check the CPU has XSAVES. 1051f0d4f30aSIngo Molnar */ 105256583c9aSIngo Molnar int copy_xstate_to_kernel(void *kbuf, struct xregs_state *xsave, unsigned int offset_start, unsigned int size_total) 1053f0d4f30aSIngo Molnar { 1054f0d4f30aSIngo Molnar struct xstate_header header; 10559e463654SAl Viro const unsigned off_mxcsr = offsetof(struct fxregs_state, mxcsr); 10569e463654SAl Viro unsigned count = size_total; 10578c0817f4SIngo Molnar int i; 1058f0d4f30aSIngo Molnar 1059f0d4f30aSIngo Molnar /* 1060f0d4f30aSIngo Molnar * Currently copy_regset_to_user() starts from pos 0: 1061f0d4f30aSIngo Molnar */ 106256583c9aSIngo Molnar if (unlikely(offset_start != 0)) 1063f0d4f30aSIngo Molnar return -EFAULT; 1064f0d4f30aSIngo Molnar 1065f0d4f30aSIngo Molnar /* 1066f0d4f30aSIngo Molnar * The destination is a ptrace buffer; we put in only user xstates: 1067f0d4f30aSIngo Molnar */ 1068f0d4f30aSIngo Molnar memset(&header, 0, sizeof(header)); 1069f0d4f30aSIngo Molnar header.xfeatures = xsave->header.xfeatures; 1070524bb73bSYu-cheng Yu header.xfeatures &= xfeatures_mask_user(); 1071f0d4f30aSIngo Molnar 10729e463654SAl Viro if (header.xfeatures & XFEATURE_MASK_FP) 10739e463654SAl Viro copy_part(0, off_mxcsr, 10749e463654SAl Viro &xsave->i387, &kbuf, &offset_start, &count); 10759e463654SAl Viro if (header.xfeatures & (XFEATURE_MASK_SSE | XFEATURE_MASK_YMM)) 10769e463654SAl Viro copy_part(off_mxcsr, MXCSR_AND_FLAGS_SIZE, 10779e463654SAl Viro &xsave->i387.mxcsr, &kbuf, &offset_start, &count); 10789e463654SAl Viro if (header.xfeatures & XFEATURE_MASK_FP) 10799e463654SAl Viro copy_part(offsetof(struct fxregs_state, st_space), 128, 10809e463654SAl Viro &xsave->i387.st_space, &kbuf, &offset_start, &count); 10819e463654SAl Viro if (header.xfeatures & XFEATURE_MASK_SSE) 10829e463654SAl Viro copy_part(xstate_offsets[XFEATURE_MASK_SSE], 256, 10839e463654SAl Viro &xsave->i387.xmm_space, &kbuf, &offset_start, &count); 10849e463654SAl Viro /* 10859e463654SAl Viro * Fill xsave->i387.sw_reserved value for ptrace frame: 10869e463654SAl Viro */ 10879e463654SAl Viro copy_part(offsetof(struct fxregs_state, sw_reserved), 48, 10889e463654SAl Viro xstate_fx_sw_bytes, &kbuf, &offset_start, &count); 1089f0d4f30aSIngo Molnar /* 1090f0d4f30aSIngo Molnar * Copy xregs_state->header: 1091f0d4f30aSIngo Molnar */ 10929e463654SAl Viro copy_part(offsetof(struct xregs_state, header), sizeof(header), 10939e463654SAl Viro &header, &kbuf, &offset_start, &count); 1094f0d4f30aSIngo Molnar 10959e463654SAl Viro for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) { 1096f0d4f30aSIngo Molnar /* 1097f0d4f30aSIngo Molnar * Copy only in-use xstates: 1098f0d4f30aSIngo Molnar */ 1099f0d4f30aSIngo Molnar if ((header.xfeatures >> i) & 1) { 110007baeb04SSebastian Andrzej Siewior void *src = __raw_xsave_addr(xsave, i); 1101f0d4f30aSIngo Molnar 11029e463654SAl Viro copy_part(xstate_offsets[i], xstate_sizes[i], 11039e463654SAl Viro src, &kbuf, &offset_start, &count); 1104f0d4f30aSIngo Molnar } 1105f0d4f30aSIngo Molnar 1106f0d4f30aSIngo Molnar } 11079e463654SAl Viro fill_gap(size_total, &kbuf, &offset_start, &count); 1108f0d4f30aSIngo Molnar 1109f0d4f30aSIngo Molnar return 0; 1110f0d4f30aSIngo Molnar } 1111f0d4f30aSIngo Molnar 1112f0d4f30aSIngo Molnar static inline int 11136ff15f8dSIngo Molnar __copy_xstate_to_user(void __user *ubuf, const void *data, unsigned int offset, unsigned int size, unsigned int size_total) 111491c3dba7SYu-cheng Yu { 111556583c9aSIngo Molnar if (!size) 111691c3dba7SYu-cheng Yu return 0; 111791c3dba7SYu-cheng Yu 11186ff15f8dSIngo Molnar if (offset < size_total) { 11196ff15f8dSIngo Molnar unsigned int copy = min(size, size_total - offset); 112091c3dba7SYu-cheng Yu 112156583c9aSIngo Molnar if (__copy_to_user(ubuf + offset, data, copy)) 112291c3dba7SYu-cheng Yu return -EFAULT; 112391c3dba7SYu-cheng Yu } 112491c3dba7SYu-cheng Yu return 0; 112591c3dba7SYu-cheng Yu } 112691c3dba7SYu-cheng Yu 112791c3dba7SYu-cheng Yu /* 112891c3dba7SYu-cheng Yu * Convert from kernel XSAVES compacted format to standard format and copy 1129becb2bb7SIngo Molnar * to a user-space buffer. It supports partial copy but pos always starts from 113091c3dba7SYu-cheng Yu * zero. This is called from xstateregs_get() and there we check the CPU 113191c3dba7SYu-cheng Yu * has XSAVES. 113291c3dba7SYu-cheng Yu */ 113356583c9aSIngo Molnar int copy_xstate_to_user(void __user *ubuf, struct xregs_state *xsave, unsigned int offset_start, unsigned int size_total) 113491c3dba7SYu-cheng Yu { 113591c3dba7SYu-cheng Yu unsigned int offset, size; 113691c3dba7SYu-cheng Yu int ret, i; 113791c3dba7SYu-cheng Yu struct xstate_header header; 113891c3dba7SYu-cheng Yu 113991c3dba7SYu-cheng Yu /* 114091c3dba7SYu-cheng Yu * Currently copy_regset_to_user() starts from pos 0: 114191c3dba7SYu-cheng Yu */ 114256583c9aSIngo Molnar if (unlikely(offset_start != 0)) 114391c3dba7SYu-cheng Yu return -EFAULT; 114491c3dba7SYu-cheng Yu 114591c3dba7SYu-cheng Yu /* 114691c3dba7SYu-cheng Yu * The destination is a ptrace buffer; we put in only user xstates: 114791c3dba7SYu-cheng Yu */ 114891c3dba7SYu-cheng Yu memset(&header, 0, sizeof(header)); 114991c3dba7SYu-cheng Yu header.xfeatures = xsave->header.xfeatures; 1150524bb73bSYu-cheng Yu header.xfeatures &= xfeatures_mask_user(); 115191c3dba7SYu-cheng Yu 115291c3dba7SYu-cheng Yu /* 115391c3dba7SYu-cheng Yu * Copy xregs_state->header: 115491c3dba7SYu-cheng Yu */ 115591c3dba7SYu-cheng Yu offset = offsetof(struct xregs_state, header); 115691c3dba7SYu-cheng Yu size = sizeof(header); 115791c3dba7SYu-cheng Yu 115856583c9aSIngo Molnar ret = __copy_xstate_to_user(ubuf, &header, offset, size, size_total); 115991c3dba7SYu-cheng Yu if (ret) 116091c3dba7SYu-cheng Yu return ret; 116191c3dba7SYu-cheng Yu 116291c3dba7SYu-cheng Yu for (i = 0; i < XFEATURE_MAX; i++) { 116391c3dba7SYu-cheng Yu /* 116491c3dba7SYu-cheng Yu * Copy only in-use xstates: 116591c3dba7SYu-cheng Yu */ 116691c3dba7SYu-cheng Yu if ((header.xfeatures >> i) & 1) { 116707baeb04SSebastian Andrzej Siewior void *src = __raw_xsave_addr(xsave, i); 116891c3dba7SYu-cheng Yu 116991c3dba7SYu-cheng Yu offset = xstate_offsets[i]; 117091c3dba7SYu-cheng Yu size = xstate_sizes[i]; 117191c3dba7SYu-cheng Yu 11726ff15f8dSIngo Molnar /* The next component has to fit fully into the output buffer: */ 11736ff15f8dSIngo Molnar if (offset + size > size_total) 11746ff15f8dSIngo Molnar break; 11756ff15f8dSIngo Molnar 117656583c9aSIngo Molnar ret = __copy_xstate_to_user(ubuf, src, offset, size, size_total); 117791c3dba7SYu-cheng Yu if (ret) 117891c3dba7SYu-cheng Yu return ret; 117991c3dba7SYu-cheng Yu } 118091c3dba7SYu-cheng Yu 118191c3dba7SYu-cheng Yu } 118291c3dba7SYu-cheng Yu 11830852b374SRik van Riel if (xfeatures_mxcsr_quirk(header.xfeatures)) { 11840852b374SRik van Riel offset = offsetof(struct fxregs_state, mxcsr); 11850852b374SRik van Riel size = MXCSR_AND_FLAGS_SIZE; 11860852b374SRik van Riel __copy_xstate_to_user(ubuf, &xsave->i387.mxcsr, offset, size, size_total); 11870852b374SRik van Riel } 11880852b374SRik van Riel 118991c3dba7SYu-cheng Yu /* 119091c3dba7SYu-cheng Yu * Fill xsave->i387.sw_reserved value for ptrace frame: 119191c3dba7SYu-cheng Yu */ 119291c3dba7SYu-cheng Yu offset = offsetof(struct fxregs_state, sw_reserved); 119391c3dba7SYu-cheng Yu size = sizeof(xstate_fx_sw_bytes); 119491c3dba7SYu-cheng Yu 119556583c9aSIngo Molnar ret = __copy_xstate_to_user(ubuf, xstate_fx_sw_bytes, offset, size, size_total); 119691c3dba7SYu-cheng Yu if (ret) 119791c3dba7SYu-cheng Yu return ret; 119891c3dba7SYu-cheng Yu 119991c3dba7SYu-cheng Yu return 0; 120091c3dba7SYu-cheng Yu } 120191c3dba7SYu-cheng Yu 120291c3dba7SYu-cheng Yu /* 120379fecc2bSIngo Molnar * Convert from a ptrace standard-format kernel buffer to kernel XSAVES format 1204af95774bSEric Biggers * and copy to the target thread. This is called from xstateregs_set(). 120579fecc2bSIngo Molnar */ 12066d7f7da5SIngo Molnar int copy_kernel_to_xstate(struct xregs_state *xsave, const void *kbuf) 120779fecc2bSIngo Molnar { 120879fecc2bSIngo Molnar unsigned int offset, size; 120979fecc2bSIngo Molnar int i; 121080d8ae86SEric Biggers struct xstate_header hdr; 121179fecc2bSIngo Molnar 121279fecc2bSIngo Molnar offset = offsetof(struct xregs_state, header); 121380d8ae86SEric Biggers size = sizeof(hdr); 121479fecc2bSIngo Molnar 121580d8ae86SEric Biggers memcpy(&hdr, kbuf + offset, size); 121679fecc2bSIngo Molnar 12175274e6c1SFenghua Yu if (validate_user_xstate_header(&hdr)) 121879fecc2bSIngo Molnar return -EINVAL; 121979fecc2bSIngo Molnar 122079fecc2bSIngo Molnar for (i = 0; i < XFEATURE_MAX; i++) { 122179fecc2bSIngo Molnar u64 mask = ((u64)1 << i); 122279fecc2bSIngo Molnar 1223b89eda48SEric Biggers if (hdr.xfeatures & mask) { 122407baeb04SSebastian Andrzej Siewior void *dst = __raw_xsave_addr(xsave, i); 122579fecc2bSIngo Molnar 122679fecc2bSIngo Molnar offset = xstate_offsets[i]; 122779fecc2bSIngo Molnar size = xstate_sizes[i]; 122879fecc2bSIngo Molnar 122979fecc2bSIngo Molnar memcpy(dst, kbuf + offset, size); 123079fecc2bSIngo Molnar } 123179fecc2bSIngo Molnar } 123279fecc2bSIngo Molnar 1233b89eda48SEric Biggers if (xfeatures_mxcsr_quirk(hdr.xfeatures)) { 12340852b374SRik van Riel offset = offsetof(struct fxregs_state, mxcsr); 12350852b374SRik van Riel size = MXCSR_AND_FLAGS_SIZE; 12360852b374SRik van Riel memcpy(&xsave->i387.mxcsr, kbuf + offset, size); 12370852b374SRik van Riel } 12380852b374SRik van Riel 123979fecc2bSIngo Molnar /* 124079fecc2bSIngo Molnar * The state that came in from userspace was user-state only. 124179fecc2bSIngo Molnar * Mask all the user states out of 'xfeatures': 124279fecc2bSIngo Molnar */ 12438ab22804SFenghua Yu xsave->header.xfeatures &= XFEATURE_MASK_SUPERVISOR_ALL; 124479fecc2bSIngo Molnar 124579fecc2bSIngo Molnar /* 124679fecc2bSIngo Molnar * Add back in the features that came in from userspace: 124779fecc2bSIngo Molnar */ 1248b89eda48SEric Biggers xsave->header.xfeatures |= hdr.xfeatures; 124979fecc2bSIngo Molnar 125079fecc2bSIngo Molnar return 0; 125179fecc2bSIngo Molnar } 125279fecc2bSIngo Molnar 125379fecc2bSIngo Molnar /* 125498c0fad9SEric Biggers * Convert from a ptrace or sigreturn standard-format user-space buffer to 125598c0fad9SEric Biggers * kernel XSAVES format and copy to the target thread. This is called from 125698c0fad9SEric Biggers * xstateregs_set(), as well as potentially from the sigreturn() and 125798c0fad9SEric Biggers * rt_sigreturn() system calls. 125891c3dba7SYu-cheng Yu */ 12596d7f7da5SIngo Molnar int copy_user_to_xstate(struct xregs_state *xsave, const void __user *ubuf) 126091c3dba7SYu-cheng Yu { 126191c3dba7SYu-cheng Yu unsigned int offset, size; 126291c3dba7SYu-cheng Yu int i; 1263af2c4322SEric Biggers struct xstate_header hdr; 126491c3dba7SYu-cheng Yu 126591c3dba7SYu-cheng Yu offset = offsetof(struct xregs_state, header); 1266af2c4322SEric Biggers size = sizeof(hdr); 126791c3dba7SYu-cheng Yu 1268af2c4322SEric Biggers if (__copy_from_user(&hdr, ubuf + offset, size)) 126991c3dba7SYu-cheng Yu return -EFAULT; 127091c3dba7SYu-cheng Yu 12715274e6c1SFenghua Yu if (validate_user_xstate_header(&hdr)) 127291c3dba7SYu-cheng Yu return -EINVAL; 127391c3dba7SYu-cheng Yu 127491c3dba7SYu-cheng Yu for (i = 0; i < XFEATURE_MAX; i++) { 127591c3dba7SYu-cheng Yu u64 mask = ((u64)1 << i); 127691c3dba7SYu-cheng Yu 12773d703477SEric Biggers if (hdr.xfeatures & mask) { 127807baeb04SSebastian Andrzej Siewior void *dst = __raw_xsave_addr(xsave, i); 127991c3dba7SYu-cheng Yu 128091c3dba7SYu-cheng Yu offset = xstate_offsets[i]; 128191c3dba7SYu-cheng Yu size = xstate_sizes[i]; 128291c3dba7SYu-cheng Yu 128391c3dba7SYu-cheng Yu if (__copy_from_user(dst, ubuf + offset, size)) 128491c3dba7SYu-cheng Yu return -EFAULT; 128591c3dba7SYu-cheng Yu } 128691c3dba7SYu-cheng Yu } 128791c3dba7SYu-cheng Yu 12883d703477SEric Biggers if (xfeatures_mxcsr_quirk(hdr.xfeatures)) { 12890852b374SRik van Riel offset = offsetof(struct fxregs_state, mxcsr); 12900852b374SRik van Riel size = MXCSR_AND_FLAGS_SIZE; 12910852b374SRik van Riel if (__copy_from_user(&xsave->i387.mxcsr, ubuf + offset, size)) 12920852b374SRik van Riel return -EFAULT; 12930852b374SRik van Riel } 12940852b374SRik van Riel 129591c3dba7SYu-cheng Yu /* 129691c3dba7SYu-cheng Yu * The state that came in from userspace was user-state only. 129791c3dba7SYu-cheng Yu * Mask all the user states out of 'xfeatures': 129891c3dba7SYu-cheng Yu */ 12998ab22804SFenghua Yu xsave->header.xfeatures &= XFEATURE_MASK_SUPERVISOR_ALL; 130091c3dba7SYu-cheng Yu 130191c3dba7SYu-cheng Yu /* 130291c3dba7SYu-cheng Yu * Add back in the features that came in from userspace: 130391c3dba7SYu-cheng Yu */ 13043d703477SEric Biggers xsave->header.xfeatures |= hdr.xfeatures; 130584594296SDave Hansen 130684594296SDave Hansen return 0; 130784594296SDave Hansen } 13080c608dadSAubrey Li 1309eeedf153SYu-cheng Yu /* 1310eeedf153SYu-cheng Yu * Save only supervisor states to the kernel buffer. This blows away all 1311eeedf153SYu-cheng Yu * old states, and is intended to be used only in __fpu__restore_sig(), where 1312eeedf153SYu-cheng Yu * user states are restored from the user buffer. 1313eeedf153SYu-cheng Yu */ 1314eeedf153SYu-cheng Yu void copy_supervisor_to_kernel(struct xregs_state *xstate) 1315eeedf153SYu-cheng Yu { 1316eeedf153SYu-cheng Yu struct xstate_header *header; 1317eeedf153SYu-cheng Yu u64 max_bit, min_bit; 1318eeedf153SYu-cheng Yu u32 lmask, hmask; 1319eeedf153SYu-cheng Yu int err, i; 1320eeedf153SYu-cheng Yu 1321eeedf153SYu-cheng Yu if (WARN_ON(!boot_cpu_has(X86_FEATURE_XSAVES))) 1322eeedf153SYu-cheng Yu return; 1323eeedf153SYu-cheng Yu 1324eeedf153SYu-cheng Yu if (!xfeatures_mask_supervisor()) 1325eeedf153SYu-cheng Yu return; 1326eeedf153SYu-cheng Yu 1327eeedf153SYu-cheng Yu max_bit = __fls(xfeatures_mask_supervisor()); 1328eeedf153SYu-cheng Yu min_bit = __ffs(xfeatures_mask_supervisor()); 1329eeedf153SYu-cheng Yu 1330eeedf153SYu-cheng Yu lmask = xfeatures_mask_supervisor(); 1331eeedf153SYu-cheng Yu hmask = xfeatures_mask_supervisor() >> 32; 1332eeedf153SYu-cheng Yu XSTATE_OP(XSAVES, xstate, lmask, hmask, err); 1333eeedf153SYu-cheng Yu 1334eeedf153SYu-cheng Yu /* We should never fault when copying to a kernel buffer: */ 1335eeedf153SYu-cheng Yu if (WARN_ON_FPU(err)) 1336eeedf153SYu-cheng Yu return; 1337eeedf153SYu-cheng Yu 1338eeedf153SYu-cheng Yu /* 1339eeedf153SYu-cheng Yu * At this point, the buffer has only supervisor states and must be 1340eeedf153SYu-cheng Yu * converted back to normal kernel format. 1341eeedf153SYu-cheng Yu */ 1342eeedf153SYu-cheng Yu header = &xstate->header; 1343eeedf153SYu-cheng Yu header->xcomp_bv |= xfeatures_mask_all; 1344eeedf153SYu-cheng Yu 1345eeedf153SYu-cheng Yu /* 1346eeedf153SYu-cheng Yu * This only moves states up in the buffer. Start with 1347eeedf153SYu-cheng Yu * the last state and move backwards so that states are 1348eeedf153SYu-cheng Yu * not overwritten until after they are moved. Note: 1349eeedf153SYu-cheng Yu * memmove() allows overlapping src/dst buffers. 1350eeedf153SYu-cheng Yu */ 1351eeedf153SYu-cheng Yu for (i = max_bit; i >= min_bit; i--) { 1352eeedf153SYu-cheng Yu u8 *xbuf = (u8 *)xstate; 1353eeedf153SYu-cheng Yu 1354eeedf153SYu-cheng Yu if (!((header->xfeatures >> i) & 1)) 1355eeedf153SYu-cheng Yu continue; 1356eeedf153SYu-cheng Yu 1357eeedf153SYu-cheng Yu /* Move xfeature 'i' into its normal location */ 1358eeedf153SYu-cheng Yu memmove(xbuf + xstate_comp_offsets[i], 1359eeedf153SYu-cheng Yu xbuf + xstate_supervisor_only_offsets[i], 1360eeedf153SYu-cheng Yu xstate_sizes[i]); 1361eeedf153SYu-cheng Yu } 1362eeedf153SYu-cheng Yu } 1363eeedf153SYu-cheng Yu 13640c608dadSAubrey Li #ifdef CONFIG_PROC_PID_ARCH_STATUS 13650c608dadSAubrey Li /* 13660c608dadSAubrey Li * Report the amount of time elapsed in millisecond since last AVX512 13670c608dadSAubrey Li * use in the task. 13680c608dadSAubrey Li */ 13690c608dadSAubrey Li static void avx512_status(struct seq_file *m, struct task_struct *task) 13700c608dadSAubrey Li { 13710c608dadSAubrey Li unsigned long timestamp = READ_ONCE(task->thread.fpu.avx512_timestamp); 13720c608dadSAubrey Li long delta; 13730c608dadSAubrey Li 13740c608dadSAubrey Li if (!timestamp) { 13750c608dadSAubrey Li /* 13760c608dadSAubrey Li * Report -1 if no AVX512 usage 13770c608dadSAubrey Li */ 13780c608dadSAubrey Li delta = -1; 13790c608dadSAubrey Li } else { 13800c608dadSAubrey Li delta = (long)(jiffies - timestamp); 13810c608dadSAubrey Li /* 13820c608dadSAubrey Li * Cap to LONG_MAX if time difference > LONG_MAX 13830c608dadSAubrey Li */ 13840c608dadSAubrey Li if (delta < 0) 13850c608dadSAubrey Li delta = LONG_MAX; 13860c608dadSAubrey Li delta = jiffies_to_msecs(delta); 13870c608dadSAubrey Li } 13880c608dadSAubrey Li 13890c608dadSAubrey Li seq_put_decimal_ll(m, "AVX512_elapsed_ms:\t", delta); 13900c608dadSAubrey Li seq_putc(m, '\n'); 13910c608dadSAubrey Li } 13920c608dadSAubrey Li 13930c608dadSAubrey Li /* 13940c608dadSAubrey Li * Report architecture specific information 13950c608dadSAubrey Li */ 13960c608dadSAubrey Li int proc_pid_arch_status(struct seq_file *m, struct pid_namespace *ns, 13970c608dadSAubrey Li struct pid *pid, struct task_struct *task) 13980c608dadSAubrey Li { 13990c608dadSAubrey Li /* 14000c608dadSAubrey Li * Report AVX512 state if the processor and build option supported. 14010c608dadSAubrey Li */ 14020c608dadSAubrey Li if (cpu_feature_enabled(X86_FEATURE_AVX512F)) 14030c608dadSAubrey Li avx512_status(m, task); 14040c608dadSAubrey Li 14050c608dadSAubrey Li return 0; 14060c608dadSAubrey Li } 14070c608dadSAubrey Li #endif /* CONFIG_PROC_PID_ARCH_STATUS */ 1408