1457c8996SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 262784854SIngo Molnar /* 362784854SIngo Molnar * xsave/xrstor support. 462784854SIngo Molnar * 562784854SIngo Molnar * Author: Suresh Siddha <suresh.b.siddha@intel.com> 662784854SIngo Molnar */ 7ffd3e504SThomas Gleixner #include <linux/bitops.h> 862784854SIngo Molnar #include <linux/compat.h> 962784854SIngo Molnar #include <linux/cpu.h> 10e8c24d3aSDave Hansen #include <linux/mman.h> 1184594296SDave Hansen #include <linux/pkeys.h> 120c608dadSAubrey Li #include <linux/seq_file.h> 130c608dadSAubrey Li #include <linux/proc_fs.h> 1459a36d16SIngo Molnar 1562784854SIngo Molnar #include <asm/fpu/api.h> 1659a36d16SIngo Molnar #include <asm/fpu/regset.h> 17d9d005f3SThomas Gleixner #include <asm/fpu/signal.h> 18d9d005f3SThomas Gleixner #include <asm/fpu/xcr.h> 19b992c660SIngo Molnar 2062784854SIngo Molnar #include <asm/tlbflush.h> 21126fe040SThomas Gleixner 2296034455SThomas Gleixner #include "internal.h" 2334002571SThomas Gleixner #include "legacy.h" 24126fe040SThomas Gleixner #include "xstate.h" 2562784854SIngo Molnar 26ffd3e504SThomas Gleixner #define for_each_extended_xfeature(bit, mask) \ 27ffd3e504SThomas Gleixner (bit) = FIRST_EXTENDED_XFEATURE; \ 28ffd3e504SThomas Gleixner for_each_set_bit_from(bit, (unsigned long *)&(mask), 8 * sizeof(mask)) 29ffd3e504SThomas Gleixner 301f96b1efSDave Hansen /* 311f96b1efSDave Hansen * Although we spell it out in here, the Processor Trace 321f96b1efSDave Hansen * xfeature is completely unused. We use other mechanisms 331f96b1efSDave Hansen * to save/restore PT state in Linux. 341f96b1efSDave Hansen */ 355b073430SIngo Molnar static const char *xfeature_names[] = 365b073430SIngo Molnar { 375b073430SIngo Molnar "x87 floating point registers" , 385b073430SIngo Molnar "SSE registers" , 395b073430SIngo Molnar "AVX registers" , 405b073430SIngo Molnar "MPX bounds registers" , 415b073430SIngo Molnar "MPX CSR" , 425b073430SIngo Molnar "AVX-512 opmask" , 435b073430SIngo Molnar "AVX-512 Hi256" , 445b073430SIngo Molnar "AVX-512 ZMM_Hi256" , 451f96b1efSDave Hansen "Processor Trace (unused)" , 46c8df4009SDave Hansen "Protection Keys User registers", 47b454feb9SYu-cheng Yu "PASID state", 485b073430SIngo Molnar "unknown xstate feature" , 495b073430SIngo Molnar }; 505b073430SIngo Molnar 51ccb18db2SAndi Kleen static short xsave_cpuid_features[] __initdata = { 52ccb18db2SAndi Kleen X86_FEATURE_FPU, 53ccb18db2SAndi Kleen X86_FEATURE_XMM, 54ccb18db2SAndi Kleen X86_FEATURE_AVX, 55ccb18db2SAndi Kleen X86_FEATURE_MPX, 56ccb18db2SAndi Kleen X86_FEATURE_MPX, 57ccb18db2SAndi Kleen X86_FEATURE_AVX512F, 58ccb18db2SAndi Kleen X86_FEATURE_AVX512F, 59ccb18db2SAndi Kleen X86_FEATURE_AVX512F, 60ccb18db2SAndi Kleen X86_FEATURE_INTEL_PT, 61ccb18db2SAndi Kleen X86_FEATURE_PKU, 62b454feb9SYu-cheng Yu X86_FEATURE_ENQCMD, 63ccb18db2SAndi Kleen }; 64ccb18db2SAndi Kleen 6562784854SIngo Molnar /* 66524bb73bSYu-cheng Yu * This represents the full set of bits that should ever be set in a kernel 67524bb73bSYu-cheng Yu * XSAVE buffer, both supervisor and user xstates. 6862784854SIngo Molnar */ 694e8e4313SThomas Gleixner u64 xfeatures_mask_all __ro_after_init; 7062784854SIngo Molnar 71ce578f16SThomas Gleixner static unsigned int xstate_offsets[XFEATURE_MAX] __ro_after_init = 72ce578f16SThomas Gleixner { [ 0 ... XFEATURE_MAX - 1] = -1}; 73ce578f16SThomas Gleixner static unsigned int xstate_sizes[XFEATURE_MAX] __ro_after_init = 74ce578f16SThomas Gleixner { [ 0 ... XFEATURE_MAX - 1] = -1}; 75ce578f16SThomas Gleixner static unsigned int xstate_comp_offsets[XFEATURE_MAX] __ro_after_init = 76ce578f16SThomas Gleixner { [ 0 ... XFEATURE_MAX - 1] = -1}; 77ce578f16SThomas Gleixner static unsigned int xstate_supervisor_only_offsets[XFEATURE_MAX] __ro_after_init = 78ce578f16SThomas Gleixner { [ 0 ... XFEATURE_MAX - 1] = -1}; 7962784854SIngo Molnar 8062784854SIngo Molnar /* 81a1141e0bSFenghua Yu * The XSAVE area of kernel can be in standard or compacted format; 82a1141e0bSFenghua Yu * it is always in standard format for user mode. This is the user 83a1141e0bSFenghua Yu * mode standard format size used for signal and ptrace frames. 84a1141e0bSFenghua Yu */ 85ce578f16SThomas Gleixner unsigned int fpu_user_xstate_size __ro_after_init; 86a1141e0bSFenghua Yu 87a1141e0bSFenghua Yu /* 885b073430SIngo Molnar * Return whether the system supports a given xfeature. 895b073430SIngo Molnar * 905b073430SIngo Molnar * Also return the name of the (most advanced) feature that the caller requested: 915b073430SIngo Molnar */ 925b073430SIngo Molnar int cpu_has_xfeatures(u64 xfeatures_needed, const char **feature_name) 935b073430SIngo Molnar { 94524bb73bSYu-cheng Yu u64 xfeatures_missing = xfeatures_needed & ~xfeatures_mask_all; 955b073430SIngo Molnar 965b073430SIngo Molnar if (unlikely(feature_name)) { 975b073430SIngo Molnar long xfeature_idx, max_idx; 985b073430SIngo Molnar u64 xfeatures_print; 995b073430SIngo Molnar /* 1005b073430SIngo Molnar * So we use FLS here to be able to print the most advanced 1015b073430SIngo Molnar * feature that was requested but is missing. So if a driver 102d91cab78SDave Hansen * asks about "XFEATURE_MASK_SSE | XFEATURE_MASK_YMM" we'll print the 1035b073430SIngo Molnar * missing AVX feature - this is the most informative message 1045b073430SIngo Molnar * to users: 1055b073430SIngo Molnar */ 1065b073430SIngo Molnar if (xfeatures_missing) 1075b073430SIngo Molnar xfeatures_print = xfeatures_missing; 1085b073430SIngo Molnar else 1095b073430SIngo Molnar xfeatures_print = xfeatures_needed; 1105b073430SIngo Molnar 1115b073430SIngo Molnar xfeature_idx = fls64(xfeatures_print)-1; 1125b073430SIngo Molnar max_idx = ARRAY_SIZE(xfeature_names)-1; 1135b073430SIngo Molnar xfeature_idx = min(xfeature_idx, max_idx); 1145b073430SIngo Molnar 1155b073430SIngo Molnar *feature_name = xfeature_names[xfeature_idx]; 1165b073430SIngo Molnar } 1175b073430SIngo Molnar 1185b073430SIngo Molnar if (xfeatures_missing) 1195b073430SIngo Molnar return 0; 1205b073430SIngo Molnar 1215b073430SIngo Molnar return 1; 1225b073430SIngo Molnar } 1235b073430SIngo Molnar EXPORT_SYMBOL_GPL(cpu_has_xfeatures); 1245b073430SIngo Molnar 125158e2ee6SYu-cheng Yu static bool xfeature_is_supervisor(int xfeature_nr) 1261499ce2dSYu-cheng Yu { 1271499ce2dSYu-cheng Yu /* 1288c9e6073SYu-cheng Yu * Extended State Enumeration Sub-leaves (EAX = 0DH, ECX = n, n > 1) 1298c9e6073SYu-cheng Yu * returns ECX[0] set to (1) for a supervisor state, and cleared (0) 1308c9e6073SYu-cheng Yu * for a user state. 1311499ce2dSYu-cheng Yu */ 1321499ce2dSYu-cheng Yu u32 eax, ebx, ecx, edx; 1331499ce2dSYu-cheng Yu 1341499ce2dSYu-cheng Yu cpuid_count(XSTATE_CPUID, xfeature_nr, &eax, &ebx, &ecx, &edx); 135158e2ee6SYu-cheng Yu return ecx & 1; 1361499ce2dSYu-cheng Yu } 1371499ce2dSYu-cheng Yu 1385b073430SIngo Molnar /* 13962784854SIngo Molnar * Enable the extended processor state save/restore feature. 14062784854SIngo Molnar * Called once per CPU onlining. 14162784854SIngo Molnar */ 14262784854SIngo Molnar void fpu__init_cpu_xstate(void) 14362784854SIngo Molnar { 144524bb73bSYu-cheng Yu if (!boot_cpu_has(X86_FEATURE_XSAVE) || !xfeatures_mask_all) 14562784854SIngo Molnar return; 14662784854SIngo Molnar 14762784854SIngo Molnar cr4_set_bits(X86_CR4_OSXSAVE); 14862784854SIngo Molnar 14962784854SIngo Molnar /* 150524bb73bSYu-cheng Yu * XCR_XFEATURE_ENABLED_MASK (aka. XCR0) sets user features 151524bb73bSYu-cheng Yu * managed by XSAVE{C, OPT, S} and XRSTOR{S}. Only XSAVE user 152524bb73bSYu-cheng Yu * states can be set here. 153e6e888f9SDave Hansen */ 15465e95210SThomas Gleixner xsetbv(XCR_XFEATURE_ENABLED_MASK, xfeatures_mask_uabi()); 15562784854SIngo Molnar 15662784854SIngo Molnar /* 15771581eefSYu-cheng Yu * MSR_IA32_XSS sets supervisor states managed by XSAVES. 158e6e888f9SDave Hansen */ 159f0dccc9dSKan Liang if (boot_cpu_has(X86_FEATURE_XSAVES)) { 160f0dccc9dSKan Liang wrmsrl(MSR_IA32_XSS, xfeatures_mask_supervisor() | 16101707b66SAndy Lutomirski xfeatures_mask_independent()); 162f0dccc9dSKan Liang } 16371581eefSYu-cheng Yu } 16471581eefSYu-cheng Yu 165524bb73bSYu-cheng Yu static bool xfeature_enabled(enum xfeature xfeature) 166e6e888f9SDave Hansen { 167524bb73bSYu-cheng Yu return xfeatures_mask_all & BIT_ULL(xfeature); 168e6e888f9SDave Hansen } 169e6e888f9SDave Hansen 170e6e888f9SDave Hansen /* 17139f1acd2SIngo Molnar * Record the offsets and sizes of various xstates contained 17239f1acd2SIngo Molnar * in the XSAVE state memory layout. 17362784854SIngo Molnar */ 17462784854SIngo Molnar static void __init setup_xstate_features(void) 17562784854SIngo Molnar { 176ee9ae257SDave Hansen u32 eax, ebx, ecx, edx, i; 177d9f6e12fSIngo Molnar /* start at the beginning of the "extended state" */ 178e6e888f9SDave Hansen unsigned int last_good_offset = offsetof(struct xregs_state, 179e6e888f9SDave Hansen extended_state_area); 180ac73b27aSYu-cheng Yu /* 181ac73b27aSYu-cheng Yu * The FP xstates and SSE xstates are legacy states. They are always 182ac73b27aSYu-cheng Yu * in the fixed offsets in the xsave area in either compacted form 183ac73b27aSYu-cheng Yu * or standard form. 184ac73b27aSYu-cheng Yu */ 185446e693cSCyrill Gorcunov xstate_offsets[XFEATURE_FP] = 0; 186446e693cSCyrill Gorcunov xstate_sizes[XFEATURE_FP] = offsetof(struct fxregs_state, 187446e693cSCyrill Gorcunov xmm_space); 188446e693cSCyrill Gorcunov 189446e693cSCyrill Gorcunov xstate_offsets[XFEATURE_SSE] = xstate_sizes[XFEATURE_FP]; 190c593642cSPankaj Bharadiya xstate_sizes[XFEATURE_SSE] = sizeof_field(struct fxregs_state, 191446e693cSCyrill Gorcunov xmm_space); 19262784854SIngo Molnar 193ffd3e504SThomas Gleixner for_each_extended_xfeature(i, xfeatures_mask_all) { 194e6e888f9SDave Hansen cpuid_count(XSTATE_CPUID, i, &eax, &ebx, &ecx, &edx); 1951499ce2dSYu-cheng Yu 196c12e13dcSYu-cheng Yu xstate_sizes[i] = eax; 197c12e13dcSYu-cheng Yu 1981499ce2dSYu-cheng Yu /* 199c12e13dcSYu-cheng Yu * If an xfeature is supervisor state, the offset in EBX is 200c12e13dcSYu-cheng Yu * invalid, leave it to -1. 2011499ce2dSYu-cheng Yu */ 202c12e13dcSYu-cheng Yu if (xfeature_is_supervisor(i)) 203c12e13dcSYu-cheng Yu continue; 204c12e13dcSYu-cheng Yu 205ee9ae257SDave Hansen xstate_offsets[i] = ebx; 2061499ce2dSYu-cheng Yu 207e6e888f9SDave Hansen /* 208c12e13dcSYu-cheng Yu * In our xstate size checks, we assume that the highest-numbered 209c12e13dcSYu-cheng Yu * xstate feature has the highest offset in the buffer. Ensure 210c12e13dcSYu-cheng Yu * it does. 211e6e888f9SDave Hansen */ 212e6e888f9SDave Hansen WARN_ONCE(last_good_offset > xstate_offsets[i], 213e6e888f9SDave Hansen "x86/fpu: misordered xstate at %d\n", last_good_offset); 214c12e13dcSYu-cheng Yu 215e6e888f9SDave Hansen last_good_offset = xstate_offsets[i]; 21639f1acd2SIngo Molnar } 21762784854SIngo Molnar } 21862784854SIngo Molnar 21932231879SIngo Molnar static void __init print_xstate_feature(u64 xstate_mask) 22062784854SIngo Molnar { 22133588b52SIngo Molnar const char *feature_name; 22262784854SIngo Molnar 22333588b52SIngo Molnar if (cpu_has_xfeatures(xstate_mask, &feature_name)) 224c8df4009SDave Hansen pr_info("x86/fpu: Supporting XSAVE feature 0x%03Lx: '%s'\n", xstate_mask, feature_name); 22562784854SIngo Molnar } 22662784854SIngo Molnar 22762784854SIngo Molnar /* 22862784854SIngo Molnar * Print out all the supported xstate features: 22962784854SIngo Molnar */ 23032231879SIngo Molnar static void __init print_xstate_features(void) 23162784854SIngo Molnar { 232d91cab78SDave Hansen print_xstate_feature(XFEATURE_MASK_FP); 233d91cab78SDave Hansen print_xstate_feature(XFEATURE_MASK_SSE); 234d91cab78SDave Hansen print_xstate_feature(XFEATURE_MASK_YMM); 235d91cab78SDave Hansen print_xstate_feature(XFEATURE_MASK_BNDREGS); 236d91cab78SDave Hansen print_xstate_feature(XFEATURE_MASK_BNDCSR); 237d91cab78SDave Hansen print_xstate_feature(XFEATURE_MASK_OPMASK); 238d91cab78SDave Hansen print_xstate_feature(XFEATURE_MASK_ZMM_Hi256); 239d91cab78SDave Hansen print_xstate_feature(XFEATURE_MASK_Hi16_ZMM); 240c8df4009SDave Hansen print_xstate_feature(XFEATURE_MASK_PKRU); 241b454feb9SYu-cheng Yu print_xstate_feature(XFEATURE_MASK_PASID); 24262784854SIngo Molnar } 24362784854SIngo Molnar 24462784854SIngo Molnar /* 24503482e08SYu-cheng Yu * This check is important because it is easy to get XSTATE_* 24603482e08SYu-cheng Yu * confused with XSTATE_BIT_*. 24703482e08SYu-cheng Yu */ 24803482e08SYu-cheng Yu #define CHECK_XFEATURE(nr) do { \ 24903482e08SYu-cheng Yu WARN_ON(nr < FIRST_EXTENDED_XFEATURE); \ 25003482e08SYu-cheng Yu WARN_ON(nr >= XFEATURE_MAX); \ 25103482e08SYu-cheng Yu } while (0) 25203482e08SYu-cheng Yu 25303482e08SYu-cheng Yu /* 25403482e08SYu-cheng Yu * We could cache this like xstate_size[], but we only use 25503482e08SYu-cheng Yu * it here, so it would be a waste of space. 25603482e08SYu-cheng Yu */ 25703482e08SYu-cheng Yu static int xfeature_is_aligned(int xfeature_nr) 25803482e08SYu-cheng Yu { 25903482e08SYu-cheng Yu u32 eax, ebx, ecx, edx; 26003482e08SYu-cheng Yu 26103482e08SYu-cheng Yu CHECK_XFEATURE(xfeature_nr); 262e70b1008SYu-cheng Yu 263e70b1008SYu-cheng Yu if (!xfeature_enabled(xfeature_nr)) { 264e70b1008SYu-cheng Yu WARN_ONCE(1, "Checking alignment of disabled xfeature %d\n", 265e70b1008SYu-cheng Yu xfeature_nr); 266e70b1008SYu-cheng Yu return 0; 267e70b1008SYu-cheng Yu } 268e70b1008SYu-cheng Yu 26903482e08SYu-cheng Yu cpuid_count(XSTATE_CPUID, xfeature_nr, &eax, &ebx, &ecx, &edx); 27003482e08SYu-cheng Yu /* 27103482e08SYu-cheng Yu * The value returned by ECX[1] indicates the alignment 27203482e08SYu-cheng Yu * of state component 'i' when the compacted format 27303482e08SYu-cheng Yu * of the extended region of an XSAVE area is used: 27403482e08SYu-cheng Yu */ 27503482e08SYu-cheng Yu return !!(ecx & 2); 27603482e08SYu-cheng Yu } 27703482e08SYu-cheng Yu 27803482e08SYu-cheng Yu /* 27962784854SIngo Molnar * This function sets up offsets and sizes of all extended states in 28062784854SIngo Molnar * xsave area. This supports both standard format and compacted format 28149a91d61SYu-cheng Yu * of the xsave area. 28262784854SIngo Molnar */ 28349a91d61SYu-cheng Yu static void __init setup_xstate_comp_offsets(void) 28462784854SIngo Molnar { 28549a91d61SYu-cheng Yu unsigned int next_offset; 28662784854SIngo Molnar int i; 28762784854SIngo Molnar 28862784854SIngo Molnar /* 28962784854SIngo Molnar * The FP xstates and SSE xstates are legacy states. They are always 29062784854SIngo Molnar * in the fixed offsets in the xsave area in either compacted form 29162784854SIngo Molnar * or standard form. 29262784854SIngo Molnar */ 293446e693cSCyrill Gorcunov xstate_comp_offsets[XFEATURE_FP] = 0; 294446e693cSCyrill Gorcunov xstate_comp_offsets[XFEATURE_SSE] = offsetof(struct fxregs_state, 295446e693cSCyrill Gorcunov xmm_space); 29662784854SIngo Molnar 297ffd3e504SThomas Gleixner if (!cpu_feature_enabled(X86_FEATURE_XSAVES)) { 298ffd3e504SThomas Gleixner for_each_extended_xfeature(i, xfeatures_mask_all) 29962784854SIngo Molnar xstate_comp_offsets[i] = xstate_offsets[i]; 30062784854SIngo Molnar return; 30162784854SIngo Molnar } 30262784854SIngo Molnar 30349a91d61SYu-cheng Yu next_offset = FXSAVE_SIZE + XSAVE_HDR_SIZE; 30462784854SIngo Molnar 305ffd3e504SThomas Gleixner for_each_extended_xfeature(i, xfeatures_mask_all) { 30603482e08SYu-cheng Yu if (xfeature_is_aligned(i)) 30749a91d61SYu-cheng Yu next_offset = ALIGN(next_offset, 64); 30849a91d61SYu-cheng Yu 30949a91d61SYu-cheng Yu xstate_comp_offsets[i] = next_offset; 31049a91d61SYu-cheng Yu next_offset += xstate_sizes[i]; 31162784854SIngo Molnar } 31262784854SIngo Molnar } 31362784854SIngo Molnar 31462784854SIngo Molnar /* 315eeedf153SYu-cheng Yu * Setup offsets of a supervisor-state-only XSAVES buffer: 316eeedf153SYu-cheng Yu * 317eeedf153SYu-cheng Yu * The offsets stored in xstate_comp_offsets[] only work for one specific 318eeedf153SYu-cheng Yu * value of the Requested Feature BitMap (RFBM). In cases where a different 319eeedf153SYu-cheng Yu * RFBM value is used, a different set of offsets is required. This set of 320eeedf153SYu-cheng Yu * offsets is for when RFBM=xfeatures_mask_supervisor(). 321eeedf153SYu-cheng Yu */ 322eeedf153SYu-cheng Yu static void __init setup_supervisor_only_offsets(void) 323eeedf153SYu-cheng Yu { 324eeedf153SYu-cheng Yu unsigned int next_offset; 325eeedf153SYu-cheng Yu int i; 326eeedf153SYu-cheng Yu 327eeedf153SYu-cheng Yu next_offset = FXSAVE_SIZE + XSAVE_HDR_SIZE; 328eeedf153SYu-cheng Yu 329ffd3e504SThomas Gleixner for_each_extended_xfeature(i, xfeatures_mask_all) { 330ffd3e504SThomas Gleixner if (!xfeature_is_supervisor(i)) 331eeedf153SYu-cheng Yu continue; 332eeedf153SYu-cheng Yu 333eeedf153SYu-cheng Yu if (xfeature_is_aligned(i)) 334eeedf153SYu-cheng Yu next_offset = ALIGN(next_offset, 64); 335eeedf153SYu-cheng Yu 336eeedf153SYu-cheng Yu xstate_supervisor_only_offsets[i] = next_offset; 337eeedf153SYu-cheng Yu next_offset += xstate_sizes[i]; 338eeedf153SYu-cheng Yu } 339eeedf153SYu-cheng Yu } 340eeedf153SYu-cheng Yu 341eeedf153SYu-cheng Yu /* 342996952e0SYu-cheng Yu * Print out xstate component offsets and sizes 343996952e0SYu-cheng Yu */ 344996952e0SYu-cheng Yu static void __init print_xstate_offset_size(void) 345996952e0SYu-cheng Yu { 346996952e0SYu-cheng Yu int i; 347996952e0SYu-cheng Yu 348ffd3e504SThomas Gleixner for_each_extended_xfeature(i, xfeatures_mask_all) { 349996952e0SYu-cheng Yu pr_info("x86/fpu: xstate_offset[%d]: %4d, xstate_sizes[%d]: %4d\n", 350996952e0SYu-cheng Yu i, xstate_comp_offsets[i], i, xstate_sizes[i]); 351996952e0SYu-cheng Yu } 352996952e0SYu-cheng Yu } 353996952e0SYu-cheng Yu 354996952e0SYu-cheng Yu /* 355b579d0c3SThomas Gleixner * This function is called only during boot time when x86 caps are not set 356b579d0c3SThomas Gleixner * up and alternative can not be used yet. 357b579d0c3SThomas Gleixner */ 358b579d0c3SThomas Gleixner static __init void os_xrstor_booting(struct xregs_state *xstate) 359b579d0c3SThomas Gleixner { 360b579d0c3SThomas Gleixner u64 mask = xfeatures_mask_fpstate(); 361b579d0c3SThomas Gleixner u32 lmask = mask; 362b579d0c3SThomas Gleixner u32 hmask = mask >> 32; 363b579d0c3SThomas Gleixner int err; 364b579d0c3SThomas Gleixner 365b579d0c3SThomas Gleixner if (cpu_feature_enabled(X86_FEATURE_XSAVES)) 366b579d0c3SThomas Gleixner XSTATE_OP(XRSTORS, xstate, lmask, hmask, err); 367b579d0c3SThomas Gleixner else 368b579d0c3SThomas Gleixner XSTATE_OP(XRSTOR, xstate, lmask, hmask, err); 369b579d0c3SThomas Gleixner 370b579d0c3SThomas Gleixner /* 371b579d0c3SThomas Gleixner * We should never fault when copying from a kernel buffer, and the FPU 372b579d0c3SThomas Gleixner * state we set at boot time should be valid. 373b579d0c3SThomas Gleixner */ 374b579d0c3SThomas Gleixner WARN_ON_FPU(err); 375b579d0c3SThomas Gleixner } 376b579d0c3SThomas Gleixner 377b579d0c3SThomas Gleixner /* 378f9dfb5e3SThomas Gleixner * All supported features have either init state all zeros or are 379f9dfb5e3SThomas Gleixner * handled in setup_init_fpu() individually. This is an explicit 380f9dfb5e3SThomas Gleixner * feature list and does not use XFEATURE_MASK*SUPPORTED to catch 381f9dfb5e3SThomas Gleixner * newly added supported features at build time and make people 382f9dfb5e3SThomas Gleixner * actually look at the init state for the new feature. 383f9dfb5e3SThomas Gleixner */ 384f9dfb5e3SThomas Gleixner #define XFEATURES_INIT_FPSTATE_HANDLED \ 385f9dfb5e3SThomas Gleixner (XFEATURE_MASK_FP | \ 386f9dfb5e3SThomas Gleixner XFEATURE_MASK_SSE | \ 387f9dfb5e3SThomas Gleixner XFEATURE_MASK_YMM | \ 388f9dfb5e3SThomas Gleixner XFEATURE_MASK_OPMASK | \ 389f9dfb5e3SThomas Gleixner XFEATURE_MASK_ZMM_Hi256 | \ 390f9dfb5e3SThomas Gleixner XFEATURE_MASK_Hi16_ZMM | \ 391f9dfb5e3SThomas Gleixner XFEATURE_MASK_PKRU | \ 392f9dfb5e3SThomas Gleixner XFEATURE_MASK_BNDREGS | \ 393f9dfb5e3SThomas Gleixner XFEATURE_MASK_BNDCSR | \ 394f9dfb5e3SThomas Gleixner XFEATURE_MASK_PASID) 395f9dfb5e3SThomas Gleixner 396f9dfb5e3SThomas Gleixner /* 39762784854SIngo Molnar * setup the xstate image representing the init state 39862784854SIngo Molnar */ 39932231879SIngo Molnar static void __init setup_init_fpu_buf(void) 40062784854SIngo Molnar { 401f9dfb5e3SThomas Gleixner BUILD_BUG_ON((XFEATURE_MASK_USER_SUPPORTED | 402f9dfb5e3SThomas Gleixner XFEATURE_MASK_SUPERVISOR_SUPPORTED) != 403f9dfb5e3SThomas Gleixner XFEATURES_INIT_FPSTATE_HANDLED); 404f9dfb5e3SThomas Gleixner 405d366bf7eSBorislav Petkov if (!boot_cpu_has(X86_FEATURE_XSAVE)) 40662784854SIngo Molnar return; 40762784854SIngo Molnar 40862784854SIngo Molnar setup_xstate_features(); 40962784854SIngo Molnar print_xstate_features(); 41062784854SIngo Molnar 411f83ac56aSThomas Gleixner xstate_init_xcomp_bv(&init_fpstate.regs.xsave, xfeatures_mask_all); 41262784854SIngo Molnar 41362784854SIngo Molnar /* 4147d937060SFenghua Yu * Init all the features state with header.xfeatures being 0x0 41562784854SIngo Molnar */ 416f83ac56aSThomas Gleixner os_xrstor_booting(&init_fpstate.regs.xsave); 41762784854SIngo Molnar 41862784854SIngo Molnar /* 419f9dfb5e3SThomas Gleixner * All components are now in init state. Read the state back so 420f9dfb5e3SThomas Gleixner * that init_fpstate contains all non-zero init state. This only 421f9dfb5e3SThomas Gleixner * works with XSAVE, but not with XSAVEOPT and XSAVES because 422f9dfb5e3SThomas Gleixner * those use the init optimization which skips writing data for 423f9dfb5e3SThomas Gleixner * components in init state. 424f9dfb5e3SThomas Gleixner * 425f9dfb5e3SThomas Gleixner * XSAVE could be used, but that would require to reshuffle the 426f9dfb5e3SThomas Gleixner * data when XSAVES is available because XSAVES uses xstate 427f9dfb5e3SThomas Gleixner * compaction. But doing so is a pointless exercise because most 428f9dfb5e3SThomas Gleixner * components have an all zeros init state except for the legacy 429f9dfb5e3SThomas Gleixner * ones (FP and SSE). Those can be saved with FXSAVE into the 430f9dfb5e3SThomas Gleixner * legacy area. Adding new features requires to ensure that init 431f9dfb5e3SThomas Gleixner * state is all zeroes or if not to add the necessary handling 432f9dfb5e3SThomas Gleixner * here. 43362784854SIngo Molnar */ 434f83ac56aSThomas Gleixner fxsave(&init_fpstate.regs.fxsave); 43562784854SIngo Molnar } 43662784854SIngo Molnar 43765ac2e9bSDave Hansen static int xfeature_uncompacted_offset(int xfeature_nr) 43865ac2e9bSDave Hansen { 43965ac2e9bSDave Hansen u32 eax, ebx, ecx, edx; 44065ac2e9bSDave Hansen 4411499ce2dSYu-cheng Yu /* 4421499ce2dSYu-cheng Yu * Only XSAVES supports supervisor states and it uses compacted 4431499ce2dSYu-cheng Yu * format. Checking a supervisor state's uncompacted offset is 4441499ce2dSYu-cheng Yu * an error. 4451499ce2dSYu-cheng Yu */ 4468ab22804SFenghua Yu if (XFEATURE_MASK_SUPERVISOR_ALL & BIT_ULL(xfeature_nr)) { 4471499ce2dSYu-cheng Yu WARN_ONCE(1, "No fixed offset for xstate %d\n", xfeature_nr); 4481499ce2dSYu-cheng Yu return -1; 4491499ce2dSYu-cheng Yu } 4501499ce2dSYu-cheng Yu 45165ac2e9bSDave Hansen CHECK_XFEATURE(xfeature_nr); 45265ac2e9bSDave Hansen cpuid_count(XSTATE_CPUID, xfeature_nr, &eax, &ebx, &ecx, &edx); 45365ac2e9bSDave Hansen return ebx; 45465ac2e9bSDave Hansen } 45565ac2e9bSDave Hansen 456ce711ea3SKan Liang int xfeature_size(int xfeature_nr) 45765ac2e9bSDave Hansen { 45865ac2e9bSDave Hansen u32 eax, ebx, ecx, edx; 45965ac2e9bSDave Hansen 46065ac2e9bSDave Hansen CHECK_XFEATURE(xfeature_nr); 46165ac2e9bSDave Hansen cpuid_count(XSTATE_CPUID, xfeature_nr, &eax, &ebx, &ecx, &edx); 46265ac2e9bSDave Hansen return eax; 46365ac2e9bSDave Hansen } 46465ac2e9bSDave Hansen 465e63e5d5cSEric Biggers /* Validate an xstate header supplied by userspace (ptrace or sigreturn) */ 46643be46e8SThomas Gleixner static int validate_user_xstate_header(const struct xstate_header *hdr) 467e63e5d5cSEric Biggers { 468e63e5d5cSEric Biggers /* No unknown or supervisor features may be set */ 46965e95210SThomas Gleixner if (hdr->xfeatures & ~xfeatures_mask_uabi()) 470e63e5d5cSEric Biggers return -EINVAL; 471e63e5d5cSEric Biggers 472e63e5d5cSEric Biggers /* Userspace must use the uncompacted format */ 473e63e5d5cSEric Biggers if (hdr->xcomp_bv) 474e63e5d5cSEric Biggers return -EINVAL; 475e63e5d5cSEric Biggers 476e63e5d5cSEric Biggers /* 477e63e5d5cSEric Biggers * If 'reserved' is shrunken to add a new field, make sure to validate 478e63e5d5cSEric Biggers * that new field here! 479e63e5d5cSEric Biggers */ 480e63e5d5cSEric Biggers BUILD_BUG_ON(sizeof(hdr->reserved) != 48); 481e63e5d5cSEric Biggers 482e63e5d5cSEric Biggers /* No reserved bits may be set */ 483e63e5d5cSEric Biggers if (memchr_inv(hdr->reserved, 0, sizeof(hdr->reserved))) 484e63e5d5cSEric Biggers return -EINVAL; 485e63e5d5cSEric Biggers 486e63e5d5cSEric Biggers return 0; 487e63e5d5cSEric Biggers } 488e63e5d5cSEric Biggers 48963cf05a1SThomas Gleixner static void __init __xstate_dump_leaves(void) 49065ac2e9bSDave Hansen { 49165ac2e9bSDave Hansen int i; 49265ac2e9bSDave Hansen u32 eax, ebx, ecx, edx; 49365ac2e9bSDave Hansen static int should_dump = 1; 49465ac2e9bSDave Hansen 49565ac2e9bSDave Hansen if (!should_dump) 49665ac2e9bSDave Hansen return; 49765ac2e9bSDave Hansen should_dump = 0; 49865ac2e9bSDave Hansen /* 49965ac2e9bSDave Hansen * Dump out a few leaves past the ones that we support 50065ac2e9bSDave Hansen * just in case there are some goodies up there 50165ac2e9bSDave Hansen */ 50265ac2e9bSDave Hansen for (i = 0; i < XFEATURE_MAX + 10; i++) { 50365ac2e9bSDave Hansen cpuid_count(XSTATE_CPUID, i, &eax, &ebx, &ecx, &edx); 50465ac2e9bSDave Hansen pr_warn("CPUID[%02x, %02x]: eax=%08x ebx=%08x ecx=%08x edx=%08x\n", 50565ac2e9bSDave Hansen XSTATE_CPUID, i, eax, ebx, ecx, edx); 50665ac2e9bSDave Hansen } 50765ac2e9bSDave Hansen } 50865ac2e9bSDave Hansen 50965ac2e9bSDave Hansen #define XSTATE_WARN_ON(x) do { \ 51065ac2e9bSDave Hansen if (WARN_ONCE(x, "XSAVE consistency problem, dumping leaves")) { \ 51165ac2e9bSDave Hansen __xstate_dump_leaves(); \ 51265ac2e9bSDave Hansen } \ 51365ac2e9bSDave Hansen } while (0) 51465ac2e9bSDave Hansen 515ef78f2a4SDave Hansen #define XCHECK_SZ(sz, nr, nr_macro, __struct) do { \ 516ef78f2a4SDave Hansen if ((nr == nr_macro) && \ 517ef78f2a4SDave Hansen WARN_ONCE(sz != sizeof(__struct), \ 518ef78f2a4SDave Hansen "%s: struct is %zu bytes, cpu state %d bytes\n", \ 519ef78f2a4SDave Hansen __stringify(nr_macro), sizeof(__struct), sz)) { \ 520ef78f2a4SDave Hansen __xstate_dump_leaves(); \ 521ef78f2a4SDave Hansen } \ 522ef78f2a4SDave Hansen } while (0) 523ef78f2a4SDave Hansen 524ef78f2a4SDave Hansen /* 525ef78f2a4SDave Hansen * We have a C struct for each 'xstate'. We need to ensure 526ef78f2a4SDave Hansen * that our software representation matches what the CPU 527ef78f2a4SDave Hansen * tells us about the state's size. 528ef78f2a4SDave Hansen */ 52963cf05a1SThomas Gleixner static void __init check_xstate_against_struct(int nr) 530ef78f2a4SDave Hansen { 531ef78f2a4SDave Hansen /* 532ef78f2a4SDave Hansen * Ask the CPU for the size of the state. 533ef78f2a4SDave Hansen */ 534ef78f2a4SDave Hansen int sz = xfeature_size(nr); 535ef78f2a4SDave Hansen /* 536ef78f2a4SDave Hansen * Match each CPU state with the corresponding software 537ef78f2a4SDave Hansen * structure. 538ef78f2a4SDave Hansen */ 539ef78f2a4SDave Hansen XCHECK_SZ(sz, nr, XFEATURE_YMM, struct ymmh_struct); 540ef78f2a4SDave Hansen XCHECK_SZ(sz, nr, XFEATURE_BNDREGS, struct mpx_bndreg_state); 541ef78f2a4SDave Hansen XCHECK_SZ(sz, nr, XFEATURE_BNDCSR, struct mpx_bndcsr_state); 542ef78f2a4SDave Hansen XCHECK_SZ(sz, nr, XFEATURE_OPMASK, struct avx_512_opmask_state); 543ef78f2a4SDave Hansen XCHECK_SZ(sz, nr, XFEATURE_ZMM_Hi256, struct avx_512_zmm_uppers_state); 544ef78f2a4SDave Hansen XCHECK_SZ(sz, nr, XFEATURE_Hi16_ZMM, struct avx_512_hi16_state); 545c8df4009SDave Hansen XCHECK_SZ(sz, nr, XFEATURE_PKRU, struct pkru_state); 546b454feb9SYu-cheng Yu XCHECK_SZ(sz, nr, XFEATURE_PASID, struct ia32_pasid_state); 547ef78f2a4SDave Hansen 548ef78f2a4SDave Hansen /* 549ef78f2a4SDave Hansen * Make *SURE* to add any feature numbers in below if 550ef78f2a4SDave Hansen * there are "holes" in the xsave state component 551ef78f2a4SDave Hansen * numbers. 552ef78f2a4SDave Hansen */ 553ef78f2a4SDave Hansen if ((nr < XFEATURE_YMM) || 5541f96b1efSDave Hansen (nr >= XFEATURE_MAX) || 555f0dccc9dSKan Liang (nr == XFEATURE_PT_UNIMPLEMENTED_SO_FAR) || 556b454feb9SYu-cheng Yu ((nr >= XFEATURE_RSRVD_COMP_11) && (nr <= XFEATURE_LBR))) { 557ef78f2a4SDave Hansen WARN_ONCE(1, "no structure for xstate: %d\n", nr); 558ef78f2a4SDave Hansen XSTATE_WARN_ON(1); 559ef78f2a4SDave Hansen } 560ef78f2a4SDave Hansen } 561ef78f2a4SDave Hansen 56265ac2e9bSDave Hansen /* 56365ac2e9bSDave Hansen * This essentially double-checks what the cpu told us about 56465ac2e9bSDave Hansen * how large the XSAVE buffer needs to be. We are recalculating 56565ac2e9bSDave Hansen * it to be safe. 56676d10256SKan Liang * 56701707b66SAndy Lutomirski * Independent XSAVE features allocate their own buffers and are not 56876d10256SKan Liang * covered by these checks. Only the size of the buffer for task->fpu 56976d10256SKan Liang * is checked here. 57065ac2e9bSDave Hansen */ 57163cf05a1SThomas Gleixner static void __init do_extra_xstate_size_checks(void) 57265ac2e9bSDave Hansen { 57365ac2e9bSDave Hansen int paranoid_xstate_size = FXSAVE_SIZE + XSAVE_HDR_SIZE; 57465ac2e9bSDave Hansen int i; 57565ac2e9bSDave Hansen 576ffd3e504SThomas Gleixner for_each_extended_xfeature(i, xfeatures_mask_all) { 577ef78f2a4SDave Hansen check_xstate_against_struct(i); 57865ac2e9bSDave Hansen /* 57965ac2e9bSDave Hansen * Supervisor state components can be managed only by 58002b93c0bSThomas Gleixner * XSAVES. 58165ac2e9bSDave Hansen */ 58202b93c0bSThomas Gleixner if (!cpu_feature_enabled(X86_FEATURE_XSAVES)) 58365ac2e9bSDave Hansen XSTATE_WARN_ON(xfeature_is_supervisor(i)); 58465ac2e9bSDave Hansen 58565ac2e9bSDave Hansen /* Align from the end of the previous feature */ 58665ac2e9bSDave Hansen if (xfeature_is_aligned(i)) 58765ac2e9bSDave Hansen paranoid_xstate_size = ALIGN(paranoid_xstate_size, 64); 58865ac2e9bSDave Hansen /* 58965ac2e9bSDave Hansen * The offset of a given state in the non-compacted 59065ac2e9bSDave Hansen * format is given to us in a CPUID leaf. We check 59165ac2e9bSDave Hansen * them for being ordered (increasing offsets) in 59202b93c0bSThomas Gleixner * setup_xstate_features(). XSAVES uses compacted format. 59365ac2e9bSDave Hansen */ 59402b93c0bSThomas Gleixner if (!cpu_feature_enabled(X86_FEATURE_XSAVES)) 59565ac2e9bSDave Hansen paranoid_xstate_size = xfeature_uncompacted_offset(i); 59665ac2e9bSDave Hansen /* 59765ac2e9bSDave Hansen * The compacted-format offset always depends on where 59865ac2e9bSDave Hansen * the previous state ended. 59965ac2e9bSDave Hansen */ 60065ac2e9bSDave Hansen paranoid_xstate_size += xfeature_size(i); 60165ac2e9bSDave Hansen } 602bf15a8cfSFenghua Yu XSTATE_WARN_ON(paranoid_xstate_size != fpu_kernel_xstate_size); 60365ac2e9bSDave Hansen } 60465ac2e9bSDave Hansen 60562784854SIngo Molnar /* 606524bb73bSYu-cheng Yu * Get total size of enabled xstates in XCR0 | IA32_XSS. 60765ac2e9bSDave Hansen * 60865ac2e9bSDave Hansen * Note the SDM's wording here. "sub-function 0" only enumerates 60965ac2e9bSDave Hansen * the size of the *user* states. If we use it to size a buffer 61065ac2e9bSDave Hansen * that we use 'XSAVES' on, we could potentially overflow the 61165ac2e9bSDave Hansen * buffer because 'XSAVES' saves system states too. 61262784854SIngo Molnar */ 613a1141e0bSFenghua Yu static unsigned int __init get_xsaves_size(void) 61462784854SIngo Molnar { 61562784854SIngo Molnar unsigned int eax, ebx, ecx, edx; 61665ac2e9bSDave Hansen /* 61765ac2e9bSDave Hansen * - CPUID function 0DH, sub-function 1: 61865ac2e9bSDave Hansen * EBX enumerates the size (in bytes) required by 61965ac2e9bSDave Hansen * the XSAVES instruction for an XSAVE area 62065ac2e9bSDave Hansen * containing all the state components 62165ac2e9bSDave Hansen * corresponding to bits currently set in 62265ac2e9bSDave Hansen * XCR0 | IA32_XSS. 62365ac2e9bSDave Hansen */ 62465ac2e9bSDave Hansen cpuid_count(XSTATE_CPUID, 1, &eax, &ebx, &ecx, &edx); 625a1141e0bSFenghua Yu return ebx; 62662784854SIngo Molnar } 627a1141e0bSFenghua Yu 62876d10256SKan Liang /* 62901707b66SAndy Lutomirski * Get the total size of the enabled xstates without the independent supervisor 63076d10256SKan Liang * features. 63176d10256SKan Liang */ 63201707b66SAndy Lutomirski static unsigned int __init get_xsaves_size_no_independent(void) 63376d10256SKan Liang { 63401707b66SAndy Lutomirski u64 mask = xfeatures_mask_independent(); 63576d10256SKan Liang unsigned int size; 63676d10256SKan Liang 63776d10256SKan Liang if (!mask) 63876d10256SKan Liang return get_xsaves_size(); 63976d10256SKan Liang 64001707b66SAndy Lutomirski /* Disable independent features. */ 64176d10256SKan Liang wrmsrl(MSR_IA32_XSS, xfeatures_mask_supervisor()); 64276d10256SKan Liang 64376d10256SKan Liang /* 64476d10256SKan Liang * Ask the hardware what size is required of the buffer. 64576d10256SKan Liang * This is the size required for the task->fpu buffer. 64676d10256SKan Liang */ 64776d10256SKan Liang size = get_xsaves_size(); 64876d10256SKan Liang 64901707b66SAndy Lutomirski /* Re-enable independent features so XSAVES will work on them again. */ 65076d10256SKan Liang wrmsrl(MSR_IA32_XSS, xfeatures_mask_supervisor() | mask); 65176d10256SKan Liang 65276d10256SKan Liang return size; 65376d10256SKan Liang } 65476d10256SKan Liang 655a1141e0bSFenghua Yu static unsigned int __init get_xsave_size(void) 656a1141e0bSFenghua Yu { 657a1141e0bSFenghua Yu unsigned int eax, ebx, ecx, edx; 658a1141e0bSFenghua Yu /* 659a1141e0bSFenghua Yu * - CPUID function 0DH, sub-function 0: 660a1141e0bSFenghua Yu * EBX enumerates the size (in bytes) required by 661a1141e0bSFenghua Yu * the XSAVE instruction for an XSAVE area 662a1141e0bSFenghua Yu * containing all the *user* state components 663a1141e0bSFenghua Yu * corresponding to bits currently set in XCR0. 664a1141e0bSFenghua Yu */ 665a1141e0bSFenghua Yu cpuid_count(XSTATE_CPUID, 0, &eax, &ebx, &ecx, &edx); 666a1141e0bSFenghua Yu return ebx; 6674109ca06SDave Hansen } 6684109ca06SDave Hansen 6694109ca06SDave Hansen /* 6704109ca06SDave Hansen * Will the runtime-enumerated 'xstate_size' fit in the init 6714109ca06SDave Hansen * task's statically-allocated buffer? 6724109ca06SDave Hansen */ 67363cf05a1SThomas Gleixner static bool __init is_supported_xstate_size(unsigned int test_xstate_size) 6744109ca06SDave Hansen { 675f83ac56aSThomas Gleixner if (test_xstate_size <= sizeof(init_fpstate.regs)) 6764109ca06SDave Hansen return true; 6774109ca06SDave Hansen 6784109ca06SDave Hansen pr_warn("x86/fpu: xstate buffer too small (%zu < %d), disabling xsave\n", 679f83ac56aSThomas Gleixner sizeof(init_fpstate.regs), test_xstate_size); 6804109ca06SDave Hansen return false; 6814109ca06SDave Hansen } 6824109ca06SDave Hansen 683653a561bSSergey Senozhatsky static int __init init_xstate_size(void) 6844109ca06SDave Hansen { 6854109ca06SDave Hansen /* Recompute the context size for enabled features: */ 686a1141e0bSFenghua Yu unsigned int possible_xstate_size; 687a1141e0bSFenghua Yu unsigned int xsave_size; 688a1141e0bSFenghua Yu 689a1141e0bSFenghua Yu xsave_size = get_xsave_size(); 690a1141e0bSFenghua Yu 691a1141e0bSFenghua Yu if (boot_cpu_has(X86_FEATURE_XSAVES)) 69201707b66SAndy Lutomirski possible_xstate_size = get_xsaves_size_no_independent(); 693a1141e0bSFenghua Yu else 694a1141e0bSFenghua Yu possible_xstate_size = xsave_size; 6954109ca06SDave Hansen 6964109ca06SDave Hansen /* Ensure we have the space to store all enabled: */ 6974109ca06SDave Hansen if (!is_supported_xstate_size(possible_xstate_size)) 6984109ca06SDave Hansen return -EINVAL; 6994109ca06SDave Hansen 7004109ca06SDave Hansen /* 7014109ca06SDave Hansen * The size is OK, we are definitely going to use xsave, 7024109ca06SDave Hansen * make it known to the world that we need more space. 7034109ca06SDave Hansen */ 704bf15a8cfSFenghua Yu fpu_kernel_xstate_size = possible_xstate_size; 70565ac2e9bSDave Hansen do_extra_xstate_size_checks(); 706a1141e0bSFenghua Yu 707a1141e0bSFenghua Yu /* 708a1141e0bSFenghua Yu * User space is always in standard format. 709a1141e0bSFenghua Yu */ 710a1141e0bSFenghua Yu fpu_user_xstate_size = xsave_size; 7114109ca06SDave Hansen return 0; 7124109ca06SDave Hansen } 7134109ca06SDave Hansen 714d91cab78SDave Hansen /* 715d91cab78SDave Hansen * We enabled the XSAVE hardware, but something went wrong and 716d91cab78SDave Hansen * we can not use it. Disable it. 717d91cab78SDave Hansen */ 71863cf05a1SThomas Gleixner static void __init fpu__init_disable_system_xstate(void) 7194109ca06SDave Hansen { 720524bb73bSYu-cheng Yu xfeatures_mask_all = 0; 7214109ca06SDave Hansen cr4_clear_bits(X86_CR4_OSXSAVE); 7227891bc0aSSebastian Andrzej Siewior setup_clear_cpu_cap(X86_FEATURE_XSAVE); 723248452ceSThomas Gleixner fpstate_reset(¤t->thread.fpu); 72462784854SIngo Molnar } 72562784854SIngo Molnar 72662784854SIngo Molnar /* 72762784854SIngo Molnar * Enable and initialize the xsave feature. 72862784854SIngo Molnar * Called once per system bootup. 72962784854SIngo Molnar */ 73032231879SIngo Molnar void __init fpu__init_system_xstate(void) 73162784854SIngo Molnar { 73262784854SIngo Molnar unsigned int eax, ebx, ecx, edx; 7334e8e4313SThomas Gleixner u64 xfeatures; 7344109ca06SDave Hansen int err; 735ccb18db2SAndi Kleen int i; 736e97131a8SIngo Molnar 7379170fb40SAndy Lutomirski if (!boot_cpu_has(X86_FEATURE_FPU)) { 7389170fb40SAndy Lutomirski pr_info("x86/fpu: No FPU detected\n"); 7399170fb40SAndy Lutomirski return; 7409170fb40SAndy Lutomirski } 7419170fb40SAndy Lutomirski 742d366bf7eSBorislav Petkov if (!boot_cpu_has(X86_FEATURE_XSAVE)) { 7439170fb40SAndy Lutomirski pr_info("x86/fpu: x87 FPU will use %s\n", 7449170fb40SAndy Lutomirski boot_cpu_has(X86_FEATURE_FXSR) ? "FXSAVE" : "FSAVE"); 74562784854SIngo Molnar return; 74662784854SIngo Molnar } 74762784854SIngo Molnar 74862784854SIngo Molnar if (boot_cpu_data.cpuid_level < XSTATE_CPUID) { 749e97131a8SIngo Molnar WARN_ON_FPU(1); 75062784854SIngo Molnar return; 75162784854SIngo Molnar } 75262784854SIngo Molnar 753524bb73bSYu-cheng Yu /* 754524bb73bSYu-cheng Yu * Find user xstates supported by the processor. 755524bb73bSYu-cheng Yu */ 75662784854SIngo Molnar cpuid_count(XSTATE_CPUID, 0, &eax, &ebx, &ecx, &edx); 757524bb73bSYu-cheng Yu xfeatures_mask_all = eax + ((u64)edx << 32); 75862784854SIngo Molnar 75971581eefSYu-cheng Yu /* 76071581eefSYu-cheng Yu * Find supervisor xstates supported by the processor. 76171581eefSYu-cheng Yu */ 76271581eefSYu-cheng Yu cpuid_count(XSTATE_CPUID, 1, &eax, &ebx, &ecx, &edx); 76371581eefSYu-cheng Yu xfeatures_mask_all |= ecx + ((u64)edx << 32); 76471581eefSYu-cheng Yu 76565e95210SThomas Gleixner if ((xfeatures_mask_uabi() & XFEATURE_MASK_FPSSE) != XFEATURE_MASK_FPSSE) { 766ec3ed4a2SDave Hansen /* 767ec3ed4a2SDave Hansen * This indicates that something really unexpected happened 768ec3ed4a2SDave Hansen * with the enumeration. Disable XSAVE and try to continue 769ec3ed4a2SDave Hansen * booting without it. This is too early to BUG(). 770ec3ed4a2SDave Hansen */ 771524bb73bSYu-cheng Yu pr_err("x86/fpu: FP/SSE not present amongst the CPU's xstate features: 0x%llx.\n", 772524bb73bSYu-cheng Yu xfeatures_mask_all); 773ec3ed4a2SDave Hansen goto out_disable; 77462784854SIngo Molnar } 77562784854SIngo Molnar 776ccb18db2SAndi Kleen /* 777ccb18db2SAndi Kleen * Clear XSAVE features that are disabled in the normal CPUID. 778ccb18db2SAndi Kleen */ 779ccb18db2SAndi Kleen for (i = 0; i < ARRAY_SIZE(xsave_cpuid_features); i++) { 780ccb18db2SAndi Kleen if (!boot_cpu_has(xsave_cpuid_features[i])) 781524bb73bSYu-cheng Yu xfeatures_mask_all &= ~BIT_ULL(i); 782ccb18db2SAndi Kleen } 783ccb18db2SAndi Kleen 784ce38f038SThomas Gleixner xfeatures_mask_all &= XFEATURE_MASK_USER_SUPPORTED | 785ce38f038SThomas Gleixner XFEATURE_MASK_SUPERVISOR_SUPPORTED; 786ce38f038SThomas Gleixner 7874e8e4313SThomas Gleixner /* Store it for paranoia check at the end */ 7884e8e4313SThomas Gleixner xfeatures = xfeatures_mask_all; 78962784854SIngo Molnar 79062784854SIngo Molnar /* Enable xstate instructions to be able to continue with initialization: */ 79162784854SIngo Molnar fpu__init_cpu_xstate(); 7924109ca06SDave Hansen err = init_xstate_size(); 793ec3ed4a2SDave Hansen if (err) 794ec3ed4a2SDave Hansen goto out_disable; 79562784854SIngo Molnar 796248452ceSThomas Gleixner fpstate_reset(¤t->thread.fpu); 797248452ceSThomas Gleixner 79891c3dba7SYu-cheng Yu /* 79991c3dba7SYu-cheng Yu * Update info used for ptrace frames; use standard-format size and no 80091c3dba7SYu-cheng Yu * supervisor xstates: 80191c3dba7SYu-cheng Yu */ 80265e95210SThomas Gleixner update_regset_xstate_info(fpu_user_xstate_size, xfeatures_mask_uabi()); 80391c3dba7SYu-cheng Yu 804b992c660SIngo Molnar fpu__init_prepare_fx_sw_frame(); 80562784854SIngo Molnar setup_init_fpu_buf(); 80649a91d61SYu-cheng Yu setup_xstate_comp_offsets(); 807eeedf153SYu-cheng Yu setup_supervisor_only_offsets(); 80862784854SIngo Molnar 8094e8e4313SThomas Gleixner /* 8104e8e4313SThomas Gleixner * Paranoia check whether something in the setup modified the 8114e8e4313SThomas Gleixner * xfeatures mask. 8124e8e4313SThomas Gleixner */ 8134e8e4313SThomas Gleixner if (xfeatures != xfeatures_mask_all) { 8144e8e4313SThomas Gleixner pr_err("x86/fpu: xfeatures modified from 0x%016llx to 0x%016llx during init, disabling XSAVE\n", 8154e8e4313SThomas Gleixner xfeatures, xfeatures_mask_all); 8164e8e4313SThomas Gleixner goto out_disable; 8174e8e4313SThomas Gleixner } 8184e8e4313SThomas Gleixner 8194e8e4313SThomas Gleixner print_xstate_offset_size(); 820b0815359SDave Hansen pr_info("x86/fpu: Enabled xstate features 0x%llx, context size is %d bytes, using '%s' format.\n", 821524bb73bSYu-cheng Yu xfeatures_mask_all, 822bf15a8cfSFenghua Yu fpu_kernel_xstate_size, 823782511b0SBorislav Petkov boot_cpu_has(X86_FEATURE_XSAVES) ? "compacted" : "standard"); 824ec3ed4a2SDave Hansen return; 825ec3ed4a2SDave Hansen 826ec3ed4a2SDave Hansen out_disable: 827ec3ed4a2SDave Hansen /* something went wrong, try to boot without any XSAVE support */ 828ec3ed4a2SDave Hansen fpu__init_disable_system_xstate(); 82962784854SIngo Molnar } 83062784854SIngo Molnar 83162784854SIngo Molnar /* 83262784854SIngo Molnar * Restore minimal FPU state after suspend: 83362784854SIngo Molnar */ 83462784854SIngo Molnar void fpu__resume_cpu(void) 83562784854SIngo Molnar { 83662784854SIngo Molnar /* 83762784854SIngo Molnar * Restore XCR0 on xsave capable CPUs: 83862784854SIngo Molnar */ 83965e95210SThomas Gleixner if (cpu_feature_enabled(X86_FEATURE_XSAVE)) 84065e95210SThomas Gleixner xsetbv(XCR_XFEATURE_ENABLED_MASK, xfeatures_mask_uabi()); 84171581eefSYu-cheng Yu 84271581eefSYu-cheng Yu /* 84371581eefSYu-cheng Yu * Restore IA32_XSS. The same CPUID bit enumerates support 84471581eefSYu-cheng Yu * of XSAVES and MSR_IA32_XSS. 84571581eefSYu-cheng Yu */ 84665e95210SThomas Gleixner if (cpu_feature_enabled(X86_FEATURE_XSAVES)) { 847f0dccc9dSKan Liang wrmsrl(MSR_IA32_XSS, xfeatures_mask_supervisor() | 84801707b66SAndy Lutomirski xfeatures_mask_independent()); 849f0dccc9dSKan Liang } 85062784854SIngo Molnar } 85162784854SIngo Molnar 85262784854SIngo Molnar /* 85307baeb04SSebastian Andrzej Siewior * Given an xstate feature nr, calculate where in the xsave 854b8b9b6baSDave Hansen * buffer the state is. Callers should ensure that the buffer 855b8b9b6baSDave Hansen * is valid. 856b8b9b6baSDave Hansen */ 85707baeb04SSebastian Andrzej Siewior static void *__raw_xsave_addr(struct xregs_state *xsave, int xfeature_nr) 858b8b9b6baSDave Hansen { 85907baeb04SSebastian Andrzej Siewior if (!xfeature_enabled(xfeature_nr)) { 8605060b915SYu-cheng Yu WARN_ON_FPU(1); 8615060b915SYu-cheng Yu return NULL; 8625060b915SYu-cheng Yu } 8635060b915SYu-cheng Yu 86407baeb04SSebastian Andrzej Siewior return (void *)xsave + xstate_comp_offsets[xfeature_nr]; 865b8b9b6baSDave Hansen } 866b8b9b6baSDave Hansen /* 86762784854SIngo Molnar * Given the xsave area and a state inside, this function returns the 86862784854SIngo Molnar * address of the state. 86962784854SIngo Molnar * 87062784854SIngo Molnar * This is the API that is called to get xstate address in either 87162784854SIngo Molnar * standard format or compacted format of xsave area. 87262784854SIngo Molnar * 8730c4109beSDave Hansen * Note that if there is no data for the field in the xsave buffer 8740c4109beSDave Hansen * this will return NULL. 8750c4109beSDave Hansen * 87662784854SIngo Molnar * Inputs: 8770c4109beSDave Hansen * xstate: the thread's storage area for all FPU data 878abd16d68SSebastian Andrzej Siewior * xfeature_nr: state which is defined in xsave.h (e.g. XFEATURE_FP, 879abd16d68SSebastian Andrzej Siewior * XFEATURE_SSE, etc...) 88062784854SIngo Molnar * Output: 8810c4109beSDave Hansen * address of the state in the xsave area, or NULL if the 8820c4109beSDave Hansen * field is not present in the xsave buffer. 88362784854SIngo Molnar */ 884abd16d68SSebastian Andrzej Siewior void *get_xsave_addr(struct xregs_state *xsave, int xfeature_nr) 88562784854SIngo Molnar { 8860c4109beSDave Hansen /* 8870c4109beSDave Hansen * Do we even *have* xsave state? 8880c4109beSDave Hansen */ 8890c4109beSDave Hansen if (!boot_cpu_has(X86_FEATURE_XSAVE)) 89062784854SIngo Molnar return NULL; 89162784854SIngo Molnar 8920c4109beSDave Hansen /* 8930c4109beSDave Hansen * We should not ever be requesting features that we 894524bb73bSYu-cheng Yu * have not enabled. 8950c4109beSDave Hansen */ 896524bb73bSYu-cheng Yu WARN_ONCE(!(xfeatures_mask_all & BIT_ULL(xfeature_nr)), 8970c4109beSDave Hansen "get of unsupported state"); 8980c4109beSDave Hansen /* 8990c4109beSDave Hansen * This assumes the last 'xsave*' instruction to 900abd16d68SSebastian Andrzej Siewior * have requested that 'xfeature_nr' be saved. 9010c4109beSDave Hansen * If it did not, we might be seeing and old value 9020c4109beSDave Hansen * of the field in the buffer. 9030c4109beSDave Hansen * 9040c4109beSDave Hansen * This can happen because the last 'xsave' did not 9050c4109beSDave Hansen * request that this feature be saved (unlikely) 9060c4109beSDave Hansen * or because the "init optimization" caused it 9070c4109beSDave Hansen * to not be saved. 9080c4109beSDave Hansen */ 909abd16d68SSebastian Andrzej Siewior if (!(xsave->header.xfeatures & BIT_ULL(xfeature_nr))) 9100c4109beSDave Hansen return NULL; 9110c4109beSDave Hansen 91207baeb04SSebastian Andrzej Siewior return __raw_xsave_addr(xsave, xfeature_nr); 91362784854SIngo Molnar } 91404cd027bSDave Hansen 915e8c24d3aSDave Hansen #ifdef CONFIG_ARCH_HAS_PKEYS 916e8c24d3aSDave Hansen 91784594296SDave Hansen /* 918b79daf85SDave Hansen * This will go out and modify PKRU register to set the access 919b79daf85SDave Hansen * rights for @pkey to @init_val. 92084594296SDave Hansen */ 92184594296SDave Hansen int arch_set_user_pkey_access(struct task_struct *tsk, int pkey, 92284594296SDave Hansen unsigned long init_val) 92384594296SDave Hansen { 9249fe8a6f5SThomas Gleixner u32 old_pkru, new_pkru_bits = 0; 9259fe8a6f5SThomas Gleixner int pkey_shift; 92684594296SDave Hansen 92784594296SDave Hansen /* 92884594296SDave Hansen * This check implies XSAVE support. OSPKE only gets 92984594296SDave Hansen * set if we enable XSAVE and we enable PKU in XCR0. 93084594296SDave Hansen */ 9318a1dc55aSThomas Gleixner if (!cpu_feature_enabled(X86_FEATURE_OSPKE)) 93284594296SDave Hansen return -EINVAL; 93384594296SDave Hansen 93416171bffSDave Hansen /* 93516171bffSDave Hansen * This code should only be called with valid 'pkey' 93616171bffSDave Hansen * values originating from in-kernel users. Complain 93716171bffSDave Hansen * if a bad value is observed. 93816171bffSDave Hansen */ 9399fe8a6f5SThomas Gleixner if (WARN_ON_ONCE(pkey >= arch_max_pkey())) 9409fe8a6f5SThomas Gleixner return -EINVAL; 94116171bffSDave Hansen 94291c3dba7SYu-cheng Yu /* Set the bits we need in PKRU: */ 94384594296SDave Hansen if (init_val & PKEY_DISABLE_ACCESS) 94484594296SDave Hansen new_pkru_bits |= PKRU_AD_BIT; 94584594296SDave Hansen if (init_val & PKEY_DISABLE_WRITE) 94684594296SDave Hansen new_pkru_bits |= PKRU_WD_BIT; 94784594296SDave Hansen 94891c3dba7SYu-cheng Yu /* Shift the bits in to the correct place in PKRU for pkey: */ 9499fe8a6f5SThomas Gleixner pkey_shift = pkey * PKRU_BITS_PER_PKEY; 95084594296SDave Hansen new_pkru_bits <<= pkey_shift; 95184594296SDave Hansen 952b79daf85SDave Hansen /* Get old PKRU and mask off any old bits in place: */ 953b79daf85SDave Hansen old_pkru = read_pkru(); 954b79daf85SDave Hansen old_pkru &= ~((PKRU_AD_BIT|PKRU_WD_BIT) << pkey_shift); 95584594296SDave Hansen 956b79daf85SDave Hansen /* Write old part along with new part: */ 957b79daf85SDave Hansen write_pkru(old_pkru | new_pkru_bits); 95891c3dba7SYu-cheng Yu 95991c3dba7SYu-cheng Yu return 0; 96091c3dba7SYu-cheng Yu } 961e8c24d3aSDave Hansen #endif /* ! CONFIG_ARCH_HAS_PKEYS */ 96291c3dba7SYu-cheng Yu 96396258950SThomas Gleixner static void copy_feature(bool from_xstate, struct membuf *to, void *xstate, 96496258950SThomas Gleixner void *init_xstate, unsigned int size) 965f0d4f30aSIngo Molnar { 96696258950SThomas Gleixner membuf_write(to, from_xstate ? xstate : init_xstate, size); 967f0d4f30aSIngo Molnar } 968f0d4f30aSIngo Molnar 969eb6f5172SThomas Gleixner /** 970ca834defSThomas Gleixner * __copy_xstate_to_uabi_buf - Copy kernel saved xstate to a UABI buffer 971eb6f5172SThomas Gleixner * @to: membuf descriptor 972*3ac8d757SThomas Gleixner * @fpstate: The fpstate buffer from which to copy 973ca834defSThomas Gleixner * @pkru_val: The PKRU value to store in the PKRU component 974eb6f5172SThomas Gleixner * @copy_mode: The requested copy mode 975f0d4f30aSIngo Molnar * 976eb6f5172SThomas Gleixner * Converts from kernel XSAVE or XSAVES compacted format to UABI conforming 977eb6f5172SThomas Gleixner * format, i.e. from the kernel internal hardware dependent storage format 978eb6f5172SThomas Gleixner * to the requested @mode. UABI XSTATE is always uncompacted! 979eb6f5172SThomas Gleixner * 980eb6f5172SThomas Gleixner * It supports partial copy but @to.pos always starts from zero. 981f0d4f30aSIngo Molnar */ 982*3ac8d757SThomas Gleixner void __copy_xstate_to_uabi_buf(struct membuf to, struct fpstate *fpstate, 983ca834defSThomas Gleixner u32 pkru_val, enum xstate_copy_mode copy_mode) 984f0d4f30aSIngo Molnar { 98596258950SThomas Gleixner const unsigned int off_mxcsr = offsetof(struct fxregs_state, mxcsr); 986f83ac56aSThomas Gleixner struct xregs_state *xinit = &init_fpstate.regs.xsave; 987*3ac8d757SThomas Gleixner struct xregs_state *xsave = &fpstate->regs.xsave; 988f0d4f30aSIngo Molnar struct xstate_header header; 98996258950SThomas Gleixner unsigned int zerofrom; 990ffd3e504SThomas Gleixner u64 mask; 9918c0817f4SIngo Molnar int i; 992f0d4f30aSIngo Molnar 99393c2cdc9SThomas Gleixner memset(&header, 0, sizeof(header)); 994f0d4f30aSIngo Molnar header.xfeatures = xsave->header.xfeatures; 995eb6f5172SThomas Gleixner 996eb6f5172SThomas Gleixner /* Mask out the feature bits depending on copy mode */ 997eb6f5172SThomas Gleixner switch (copy_mode) { 998eb6f5172SThomas Gleixner case XSTATE_COPY_FP: 999eb6f5172SThomas Gleixner header.xfeatures &= XFEATURE_MASK_FP; 1000eb6f5172SThomas Gleixner break; 1001eb6f5172SThomas Gleixner 1002eb6f5172SThomas Gleixner case XSTATE_COPY_FX: 1003eb6f5172SThomas Gleixner header.xfeatures &= XFEATURE_MASK_FP | XFEATURE_MASK_SSE; 1004eb6f5172SThomas Gleixner break; 1005eb6f5172SThomas Gleixner 1006eb6f5172SThomas Gleixner case XSTATE_COPY_XSAVE: 1007*3ac8d757SThomas Gleixner header.xfeatures &= fpstate->user_xfeatures; 1008eb6f5172SThomas Gleixner break; 1009eb6f5172SThomas Gleixner } 1010f0d4f30aSIngo Molnar 101196258950SThomas Gleixner /* Copy FP state up to MXCSR */ 101296258950SThomas Gleixner copy_feature(header.xfeatures & XFEATURE_MASK_FP, &to, &xsave->i387, 101396258950SThomas Gleixner &xinit->i387, off_mxcsr); 101496258950SThomas Gleixner 101596258950SThomas Gleixner /* Copy MXCSR when SSE or YMM are set in the feature mask */ 101696258950SThomas Gleixner copy_feature(header.xfeatures & (XFEATURE_MASK_SSE | XFEATURE_MASK_YMM), 101796258950SThomas Gleixner &to, &xsave->i387.mxcsr, &xinit->i387.mxcsr, 101896258950SThomas Gleixner MXCSR_AND_FLAGS_SIZE); 101996258950SThomas Gleixner 102096258950SThomas Gleixner /* Copy the remaining FP state */ 102196258950SThomas Gleixner copy_feature(header.xfeatures & XFEATURE_MASK_FP, 102296258950SThomas Gleixner &to, &xsave->i387.st_space, &xinit->i387.st_space, 102396258950SThomas Gleixner sizeof(xsave->i387.st_space)); 102496258950SThomas Gleixner 102596258950SThomas Gleixner /* Copy the SSE state - shared with YMM, but independently managed */ 102696258950SThomas Gleixner copy_feature(header.xfeatures & XFEATURE_MASK_SSE, 102796258950SThomas Gleixner &to, &xsave->i387.xmm_space, &xinit->i387.xmm_space, 102896258950SThomas Gleixner sizeof(xsave->i387.xmm_space)); 102996258950SThomas Gleixner 1030eb6f5172SThomas Gleixner if (copy_mode != XSTATE_COPY_XSAVE) 1031eb6f5172SThomas Gleixner goto out; 1032eb6f5172SThomas Gleixner 103396258950SThomas Gleixner /* Zero the padding area */ 103496258950SThomas Gleixner membuf_zero(&to, sizeof(xsave->i387.padding)); 103596258950SThomas Gleixner 103696258950SThomas Gleixner /* Copy xsave->i387.sw_reserved */ 103796258950SThomas Gleixner membuf_write(&to, xstate_fx_sw_bytes, sizeof(xsave->i387.sw_reserved)); 103896258950SThomas Gleixner 103996258950SThomas Gleixner /* Copy the user space relevant state of @xsave->header */ 104096258950SThomas Gleixner membuf_write(&to, &header, sizeof(header)); 104196258950SThomas Gleixner 104296258950SThomas Gleixner zerofrom = offsetof(struct xregs_state, extended_state_area); 1043f0d4f30aSIngo Molnar 1044f0d4f30aSIngo Molnar /* 1045ffd3e504SThomas Gleixner * The ptrace buffer is in non-compacted XSAVE format. In 1046ffd3e504SThomas Gleixner * non-compacted format disabled features still occupy state space, 1047ffd3e504SThomas Gleixner * but there is no state to copy from in the compacted 1048ffd3e504SThomas Gleixner * init_fpstate. The gap tracking will zero these states. 1049f0d4f30aSIngo Molnar */ 1050*3ac8d757SThomas Gleixner mask = fpstate->user_xfeatures; 1051f0d4f30aSIngo Molnar 1052ffd3e504SThomas Gleixner for_each_extended_xfeature(i, mask) { 105396258950SThomas Gleixner /* 105496258950SThomas Gleixner * If there was a feature or alignment gap, zero the space 105596258950SThomas Gleixner * in the destination buffer. 105696258950SThomas Gleixner */ 105796258950SThomas Gleixner if (zerofrom < xstate_offsets[i]) 105896258950SThomas Gleixner membuf_zero(&to, xstate_offsets[i] - zerofrom); 105996258950SThomas Gleixner 1060e84ba47eSDave Hansen if (i == XFEATURE_PKRU) { 1061e84ba47eSDave Hansen struct pkru_state pkru = {0}; 1062e84ba47eSDave Hansen /* 1063e84ba47eSDave Hansen * PKRU is not necessarily up to date in the 1064ca834defSThomas Gleixner * XSAVE buffer. Use the provided value. 1065e84ba47eSDave Hansen */ 1066ca834defSThomas Gleixner pkru.pkru = pkru_val; 1067e84ba47eSDave Hansen membuf_write(&to, &pkru, sizeof(pkru)); 1068e84ba47eSDave Hansen } else { 106996258950SThomas Gleixner copy_feature(header.xfeatures & BIT_ULL(i), &to, 107096258950SThomas Gleixner __raw_xsave_addr(xsave, i), 107196258950SThomas Gleixner __raw_xsave_addr(xinit, i), 107296258950SThomas Gleixner xstate_sizes[i]); 1073e84ba47eSDave Hansen } 107496258950SThomas Gleixner /* 107596258950SThomas Gleixner * Keep track of the last copied state in the non-compacted 107696258950SThomas Gleixner * target buffer for gap zeroing. 107796258950SThomas Gleixner */ 107896258950SThomas Gleixner zerofrom = xstate_offsets[i] + xstate_sizes[i]; 1079f0d4f30aSIngo Molnar } 1080f0d4f30aSIngo Molnar 1081eb6f5172SThomas Gleixner out: 108296258950SThomas Gleixner if (to.left) 108396258950SThomas Gleixner membuf_zero(&to, to.left); 108491c3dba7SYu-cheng Yu } 108591c3dba7SYu-cheng Yu 1086ca834defSThomas Gleixner /** 1087ca834defSThomas Gleixner * copy_xstate_to_uabi_buf - Copy kernel saved xstate to a UABI buffer 1088ca834defSThomas Gleixner * @to: membuf descriptor 1089ca834defSThomas Gleixner * @tsk: The task from which to copy the saved xstate 1090ca834defSThomas Gleixner * @copy_mode: The requested copy mode 1091ca834defSThomas Gleixner * 1092ca834defSThomas Gleixner * Converts from kernel XSAVE or XSAVES compacted format to UABI conforming 1093ca834defSThomas Gleixner * format, i.e. from the kernel internal hardware dependent storage format 1094ca834defSThomas Gleixner * to the requested @mode. UABI XSTATE is always uncompacted! 1095ca834defSThomas Gleixner * 1096ca834defSThomas Gleixner * It supports partial copy but @to.pos always starts from zero. 1097ca834defSThomas Gleixner */ 1098ca834defSThomas Gleixner void copy_xstate_to_uabi_buf(struct membuf to, struct task_struct *tsk, 1099ca834defSThomas Gleixner enum xstate_copy_mode copy_mode) 1100ca834defSThomas Gleixner { 1101*3ac8d757SThomas Gleixner __copy_xstate_to_uabi_buf(to, tsk->thread.fpu.fpstate, 1102ca834defSThomas Gleixner tsk->thread.pkru, copy_mode); 1103ca834defSThomas Gleixner } 1104ca834defSThomas Gleixner 1105522e9274SThomas Gleixner static int copy_from_buffer(void *dst, unsigned int offset, unsigned int size, 1106522e9274SThomas Gleixner const void *kbuf, const void __user *ubuf) 1107947f4947SThomas Gleixner { 1108522e9274SThomas Gleixner if (kbuf) { 1109522e9274SThomas Gleixner memcpy(dst, kbuf + offset, size); 1110522e9274SThomas Gleixner } else { 1111522e9274SThomas Gleixner if (copy_from_user(dst, ubuf + offset, size)) 1112522e9274SThomas Gleixner return -EFAULT; 1113947f4947SThomas Gleixner } 1114522e9274SThomas Gleixner return 0; 1115947f4947SThomas Gleixner } 1116947f4947SThomas Gleixner 1117522e9274SThomas Gleixner 1118522e9274SThomas Gleixner static int copy_uabi_to_xstate(struct xregs_state *xsave, const void *kbuf, 1119522e9274SThomas Gleixner const void __user *ubuf) 112079fecc2bSIngo Molnar { 112179fecc2bSIngo Molnar unsigned int offset, size; 112280d8ae86SEric Biggers struct xstate_header hdr; 1123522e9274SThomas Gleixner u64 mask; 1124522e9274SThomas Gleixner int i; 112579fecc2bSIngo Molnar 112679fecc2bSIngo Molnar offset = offsetof(struct xregs_state, header); 1127522e9274SThomas Gleixner if (copy_from_buffer(&hdr, offset, sizeof(hdr), kbuf, ubuf)) 1128522e9274SThomas Gleixner return -EFAULT; 112979fecc2bSIngo Molnar 11305274e6c1SFenghua Yu if (validate_user_xstate_header(&hdr)) 113179fecc2bSIngo Molnar return -EINVAL; 113279fecc2bSIngo Molnar 1133522e9274SThomas Gleixner /* Validate MXCSR when any of the related features is in use */ 1134522e9274SThomas Gleixner mask = XFEATURE_MASK_FP | XFEATURE_MASK_SSE | XFEATURE_MASK_YMM; 1135522e9274SThomas Gleixner if (hdr.xfeatures & mask) { 1136522e9274SThomas Gleixner u32 mxcsr[2]; 1137522e9274SThomas Gleixner 1138522e9274SThomas Gleixner offset = offsetof(struct fxregs_state, mxcsr); 1139522e9274SThomas Gleixner if (copy_from_buffer(mxcsr, offset, sizeof(mxcsr), kbuf, ubuf)) 1140522e9274SThomas Gleixner return -EFAULT; 1141522e9274SThomas Gleixner 1142522e9274SThomas Gleixner /* Reserved bits in MXCSR must be zero. */ 1143522e9274SThomas Gleixner if (mxcsr[0] & ~mxcsr_feature_mask) 1144947f4947SThomas Gleixner return -EINVAL; 1145947f4947SThomas Gleixner 1146522e9274SThomas Gleixner /* SSE and YMM require MXCSR even when FP is not in use. */ 1147522e9274SThomas Gleixner if (!(hdr.xfeatures & XFEATURE_MASK_FP)) { 1148522e9274SThomas Gleixner xsave->i387.mxcsr = mxcsr[0]; 1149522e9274SThomas Gleixner xsave->i387.mxcsr_mask = mxcsr[1]; 1150522e9274SThomas Gleixner } 1151522e9274SThomas Gleixner } 1152522e9274SThomas Gleixner 115379fecc2bSIngo Molnar for (i = 0; i < XFEATURE_MAX; i++) { 115479fecc2bSIngo Molnar u64 mask = ((u64)1 << i); 115579fecc2bSIngo Molnar 1156b89eda48SEric Biggers if (hdr.xfeatures & mask) { 115707baeb04SSebastian Andrzej Siewior void *dst = __raw_xsave_addr(xsave, i); 115879fecc2bSIngo Molnar 115979fecc2bSIngo Molnar offset = xstate_offsets[i]; 116079fecc2bSIngo Molnar size = xstate_sizes[i]; 116179fecc2bSIngo Molnar 1162522e9274SThomas Gleixner if (copy_from_buffer(dst, offset, size, kbuf, ubuf)) 1163522e9274SThomas Gleixner return -EFAULT; 116479fecc2bSIngo Molnar } 116579fecc2bSIngo Molnar } 116679fecc2bSIngo Molnar 116779fecc2bSIngo Molnar /* 116879fecc2bSIngo Molnar * The state that came in from userspace was user-state only. 116979fecc2bSIngo Molnar * Mask all the user states out of 'xfeatures': 117079fecc2bSIngo Molnar */ 11718ab22804SFenghua Yu xsave->header.xfeatures &= XFEATURE_MASK_SUPERVISOR_ALL; 117279fecc2bSIngo Molnar 117379fecc2bSIngo Molnar /* 117479fecc2bSIngo Molnar * Add back in the features that came in from userspace: 117579fecc2bSIngo Molnar */ 1176b89eda48SEric Biggers xsave->header.xfeatures |= hdr.xfeatures; 117779fecc2bSIngo Molnar 117879fecc2bSIngo Molnar return 0; 117979fecc2bSIngo Molnar } 118079fecc2bSIngo Molnar 118179fecc2bSIngo Molnar /* 1182522e9274SThomas Gleixner * Convert from a ptrace standard-format kernel buffer to kernel XSAVE[S] 1183ea4d6938SThomas Gleixner * format and copy to the target thread. Used by ptrace and KVM. 1184522e9274SThomas Gleixner */ 1185522e9274SThomas Gleixner int copy_uabi_from_kernel_to_xstate(struct xregs_state *xsave, const void *kbuf) 1186522e9274SThomas Gleixner { 1187522e9274SThomas Gleixner return copy_uabi_to_xstate(xsave, kbuf, NULL); 1188522e9274SThomas Gleixner } 1189522e9274SThomas Gleixner 1190522e9274SThomas Gleixner /* 119143be46e8SThomas Gleixner * Convert from a sigreturn standard-format user-space buffer to kernel 119243be46e8SThomas Gleixner * XSAVE[S] format and copy to the target thread. This is called from the 119343be46e8SThomas Gleixner * sigreturn() and rt_sigreturn() system calls. 119491c3dba7SYu-cheng Yu */ 11951cc34413SThomas Gleixner int copy_sigframe_from_user_to_xstate(struct xregs_state *xsave, 11961cc34413SThomas Gleixner const void __user *ubuf) 119791c3dba7SYu-cheng Yu { 1198522e9274SThomas Gleixner return copy_uabi_to_xstate(xsave, NULL, ubuf); 119984594296SDave Hansen } 12000c608dadSAubrey Li 1201f5daf836SThomas Gleixner static bool validate_independent_components(u64 mask) 120250f408d9SKan Liang { 1203a75c5289SThomas Gleixner u64 xchk; 120450f408d9SKan Liang 1205a75c5289SThomas Gleixner if (WARN_ON_FPU(!cpu_feature_enabled(X86_FEATURE_XSAVES))) 1206a75c5289SThomas Gleixner return false; 1207f5daf836SThomas Gleixner 1208a75c5289SThomas Gleixner xchk = ~xfeatures_mask_independent(); 120950f408d9SKan Liang 1210a75c5289SThomas Gleixner if (WARN_ON_ONCE(!mask || mask & xchk)) 1211a75c5289SThomas Gleixner return false; 121250f408d9SKan Liang 1213a75c5289SThomas Gleixner return true; 121450f408d9SKan Liang } 121550f408d9SKan Liang 121650f408d9SKan Liang /** 1217a75c5289SThomas Gleixner * xsaves - Save selected components to a kernel xstate buffer 1218a75c5289SThomas Gleixner * @xstate: Pointer to the buffer 1219a75c5289SThomas Gleixner * @mask: Feature mask to select the components to save 122050f408d9SKan Liang * 1221a75c5289SThomas Gleixner * The @xstate buffer must be 64 byte aligned and correctly initialized as 1222a75c5289SThomas Gleixner * XSAVES does not write the full xstate header. Before first use the 1223a75c5289SThomas Gleixner * buffer should be zeroed otherwise a consecutive XRSTORS from that buffer 1224a75c5289SThomas Gleixner * can #GP. 122550f408d9SKan Liang * 1226f5daf836SThomas Gleixner * The feature mask must be a subset of the independent features. 122750f408d9SKan Liang */ 1228a75c5289SThomas Gleixner void xsaves(struct xregs_state *xstate, u64 mask) 122950f408d9SKan Liang { 123050f408d9SKan Liang int err; 123150f408d9SKan Liang 1232f5daf836SThomas Gleixner if (!validate_independent_components(mask)) 123350f408d9SKan Liang return; 123450f408d9SKan Liang 1235a75c5289SThomas Gleixner XSTATE_OP(XSAVES, xstate, (u32)mask, (u32)(mask >> 32), err); 1236a75c5289SThomas Gleixner WARN_ON_ONCE(err); 1237a75c5289SThomas Gleixner } 1238a75c5289SThomas Gleixner 1239a75c5289SThomas Gleixner /** 1240a75c5289SThomas Gleixner * xrstors - Restore selected components from a kernel xstate buffer 1241a75c5289SThomas Gleixner * @xstate: Pointer to the buffer 1242a75c5289SThomas Gleixner * @mask: Feature mask to select the components to restore 1243a75c5289SThomas Gleixner * 1244a75c5289SThomas Gleixner * The @xstate buffer must be 64 byte aligned and correctly initialized 1245a75c5289SThomas Gleixner * otherwise XRSTORS from that buffer can #GP. 1246a75c5289SThomas Gleixner * 1247a75c5289SThomas Gleixner * Proper usage is to restore the state which was saved with 1248a75c5289SThomas Gleixner * xsaves() into @xstate. 1249a75c5289SThomas Gleixner * 1250f5daf836SThomas Gleixner * The feature mask must be a subset of the independent features. 1251a75c5289SThomas Gleixner */ 1252a75c5289SThomas Gleixner void xrstors(struct xregs_state *xstate, u64 mask) 1253a75c5289SThomas Gleixner { 1254a75c5289SThomas Gleixner int err; 1255a75c5289SThomas Gleixner 1256f5daf836SThomas Gleixner if (!validate_independent_components(mask)) 125750f408d9SKan Liang return; 125850f408d9SKan Liang 1259a75c5289SThomas Gleixner XSTATE_OP(XRSTORS, xstate, (u32)mask, (u32)(mask >> 32), err); 1260a75c5289SThomas Gleixner WARN_ON_ONCE(err); 126150f408d9SKan Liang } 126250f408d9SKan Liang 1263087df48cSThomas Gleixner #if IS_ENABLED(CONFIG_KVM) 1264087df48cSThomas Gleixner void fpstate_clear_xstate_component(struct fpstate *fps, unsigned int xfeature) 1265087df48cSThomas Gleixner { 1266087df48cSThomas Gleixner void *addr = get_xsave_addr(&fps->regs.xsave, xfeature); 1267087df48cSThomas Gleixner 1268087df48cSThomas Gleixner if (addr) 1269087df48cSThomas Gleixner memset(addr, 0, xstate_sizes[xfeature]); 1270087df48cSThomas Gleixner } 1271087df48cSThomas Gleixner EXPORT_SYMBOL_GPL(fpstate_clear_xstate_component); 1272087df48cSThomas Gleixner #endif 1273087df48cSThomas Gleixner 12740c608dadSAubrey Li #ifdef CONFIG_PROC_PID_ARCH_STATUS 12750c608dadSAubrey Li /* 12760c608dadSAubrey Li * Report the amount of time elapsed in millisecond since last AVX512 12770c608dadSAubrey Li * use in the task. 12780c608dadSAubrey Li */ 12790c608dadSAubrey Li static void avx512_status(struct seq_file *m, struct task_struct *task) 12800c608dadSAubrey Li { 12810c608dadSAubrey Li unsigned long timestamp = READ_ONCE(task->thread.fpu.avx512_timestamp); 12820c608dadSAubrey Li long delta; 12830c608dadSAubrey Li 12840c608dadSAubrey Li if (!timestamp) { 12850c608dadSAubrey Li /* 12860c608dadSAubrey Li * Report -1 if no AVX512 usage 12870c608dadSAubrey Li */ 12880c608dadSAubrey Li delta = -1; 12890c608dadSAubrey Li } else { 12900c608dadSAubrey Li delta = (long)(jiffies - timestamp); 12910c608dadSAubrey Li /* 12920c608dadSAubrey Li * Cap to LONG_MAX if time difference > LONG_MAX 12930c608dadSAubrey Li */ 12940c608dadSAubrey Li if (delta < 0) 12950c608dadSAubrey Li delta = LONG_MAX; 12960c608dadSAubrey Li delta = jiffies_to_msecs(delta); 12970c608dadSAubrey Li } 12980c608dadSAubrey Li 12990c608dadSAubrey Li seq_put_decimal_ll(m, "AVX512_elapsed_ms:\t", delta); 13000c608dadSAubrey Li seq_putc(m, '\n'); 13010c608dadSAubrey Li } 13020c608dadSAubrey Li 13030c608dadSAubrey Li /* 13040c608dadSAubrey Li * Report architecture specific information 13050c608dadSAubrey Li */ 13060c608dadSAubrey Li int proc_pid_arch_status(struct seq_file *m, struct pid_namespace *ns, 13070c608dadSAubrey Li struct pid *pid, struct task_struct *task) 13080c608dadSAubrey Li { 13090c608dadSAubrey Li /* 13100c608dadSAubrey Li * Report AVX512 state if the processor and build option supported. 13110c608dadSAubrey Li */ 13120c608dadSAubrey Li if (cpu_feature_enabled(X86_FEATURE_AVX512F)) 13130c608dadSAubrey Li avx512_status(m, task); 13140c608dadSAubrey Li 13150c608dadSAubrey Li return 0; 13160c608dadSAubrey Li } 13170c608dadSAubrey Li #endif /* CONFIG_PROC_PID_ARCH_STATUS */ 1318