1457c8996SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 262784854SIngo Molnar /* 362784854SIngo Molnar * xsave/xrstor support. 462784854SIngo Molnar * 562784854SIngo Molnar * Author: Suresh Siddha <suresh.b.siddha@intel.com> 662784854SIngo Molnar */ 7ffd3e504SThomas Gleixner #include <linux/bitops.h> 862784854SIngo Molnar #include <linux/compat.h> 962784854SIngo Molnar #include <linux/cpu.h> 10e8c24d3aSDave Hansen #include <linux/mman.h> 1184594296SDave Hansen #include <linux/pkeys.h> 120c608dadSAubrey Li #include <linux/seq_file.h> 130c608dadSAubrey Li #include <linux/proc_fs.h> 1459a36d16SIngo Molnar 1562784854SIngo Molnar #include <asm/fpu/api.h> 1662784854SIngo Molnar #include <asm/fpu/internal.h> 17fcbc99c4SIngo Molnar #include <asm/fpu/signal.h> 1859a36d16SIngo Molnar #include <asm/fpu/regset.h> 19b992c660SIngo Molnar 2062784854SIngo Molnar #include <asm/tlbflush.h> 21126fe040SThomas Gleixner 22*96034455SThomas Gleixner #include "internal.h" 23126fe040SThomas Gleixner #include "xstate.h" 2462784854SIngo Molnar 25ffd3e504SThomas Gleixner #define for_each_extended_xfeature(bit, mask) \ 26ffd3e504SThomas Gleixner (bit) = FIRST_EXTENDED_XFEATURE; \ 27ffd3e504SThomas Gleixner for_each_set_bit_from(bit, (unsigned long *)&(mask), 8 * sizeof(mask)) 28ffd3e504SThomas Gleixner 291f96b1efSDave Hansen /* 301f96b1efSDave Hansen * Although we spell it out in here, the Processor Trace 311f96b1efSDave Hansen * xfeature is completely unused. We use other mechanisms 321f96b1efSDave Hansen * to save/restore PT state in Linux. 331f96b1efSDave Hansen */ 345b073430SIngo Molnar static const char *xfeature_names[] = 355b073430SIngo Molnar { 365b073430SIngo Molnar "x87 floating point registers" , 375b073430SIngo Molnar "SSE registers" , 385b073430SIngo Molnar "AVX registers" , 395b073430SIngo Molnar "MPX bounds registers" , 405b073430SIngo Molnar "MPX CSR" , 415b073430SIngo Molnar "AVX-512 opmask" , 425b073430SIngo Molnar "AVX-512 Hi256" , 435b073430SIngo Molnar "AVX-512 ZMM_Hi256" , 441f96b1efSDave Hansen "Processor Trace (unused)" , 45c8df4009SDave Hansen "Protection Keys User registers", 46b454feb9SYu-cheng Yu "PASID state", 475b073430SIngo Molnar "unknown xstate feature" , 485b073430SIngo Molnar }; 495b073430SIngo Molnar 50ccb18db2SAndi Kleen static short xsave_cpuid_features[] __initdata = { 51ccb18db2SAndi Kleen X86_FEATURE_FPU, 52ccb18db2SAndi Kleen X86_FEATURE_XMM, 53ccb18db2SAndi Kleen X86_FEATURE_AVX, 54ccb18db2SAndi Kleen X86_FEATURE_MPX, 55ccb18db2SAndi Kleen X86_FEATURE_MPX, 56ccb18db2SAndi Kleen X86_FEATURE_AVX512F, 57ccb18db2SAndi Kleen X86_FEATURE_AVX512F, 58ccb18db2SAndi Kleen X86_FEATURE_AVX512F, 59ccb18db2SAndi Kleen X86_FEATURE_INTEL_PT, 60ccb18db2SAndi Kleen X86_FEATURE_PKU, 61b454feb9SYu-cheng Yu X86_FEATURE_ENQCMD, 62ccb18db2SAndi Kleen }; 63ccb18db2SAndi Kleen 6462784854SIngo Molnar /* 65524bb73bSYu-cheng Yu * This represents the full set of bits that should ever be set in a kernel 66524bb73bSYu-cheng Yu * XSAVE buffer, both supervisor and user xstates. 6762784854SIngo Molnar */ 684e8e4313SThomas Gleixner u64 xfeatures_mask_all __ro_after_init; 6962784854SIngo Molnar 70ce578f16SThomas Gleixner static unsigned int xstate_offsets[XFEATURE_MAX] __ro_after_init = 71ce578f16SThomas Gleixner { [ 0 ... XFEATURE_MAX - 1] = -1}; 72ce578f16SThomas Gleixner static unsigned int xstate_sizes[XFEATURE_MAX] __ro_after_init = 73ce578f16SThomas Gleixner { [ 0 ... XFEATURE_MAX - 1] = -1}; 74ce578f16SThomas Gleixner static unsigned int xstate_comp_offsets[XFEATURE_MAX] __ro_after_init = 75ce578f16SThomas Gleixner { [ 0 ... XFEATURE_MAX - 1] = -1}; 76ce578f16SThomas Gleixner static unsigned int xstate_supervisor_only_offsets[XFEATURE_MAX] __ro_after_init = 77ce578f16SThomas Gleixner { [ 0 ... XFEATURE_MAX - 1] = -1}; 7862784854SIngo Molnar 7962784854SIngo Molnar /* 80a1141e0bSFenghua Yu * The XSAVE area of kernel can be in standard or compacted format; 81a1141e0bSFenghua Yu * it is always in standard format for user mode. This is the user 82a1141e0bSFenghua Yu * mode standard format size used for signal and ptrace frames. 83a1141e0bSFenghua Yu */ 84ce578f16SThomas Gleixner unsigned int fpu_user_xstate_size __ro_after_init; 85a1141e0bSFenghua Yu 86a1141e0bSFenghua Yu /* 875b073430SIngo Molnar * Return whether the system supports a given xfeature. 885b073430SIngo Molnar * 895b073430SIngo Molnar * Also return the name of the (most advanced) feature that the caller requested: 905b073430SIngo Molnar */ 915b073430SIngo Molnar int cpu_has_xfeatures(u64 xfeatures_needed, const char **feature_name) 925b073430SIngo Molnar { 93524bb73bSYu-cheng Yu u64 xfeatures_missing = xfeatures_needed & ~xfeatures_mask_all; 945b073430SIngo Molnar 955b073430SIngo Molnar if (unlikely(feature_name)) { 965b073430SIngo Molnar long xfeature_idx, max_idx; 975b073430SIngo Molnar u64 xfeatures_print; 985b073430SIngo Molnar /* 995b073430SIngo Molnar * So we use FLS here to be able to print the most advanced 1005b073430SIngo Molnar * feature that was requested but is missing. So if a driver 101d91cab78SDave Hansen * asks about "XFEATURE_MASK_SSE | XFEATURE_MASK_YMM" we'll print the 1025b073430SIngo Molnar * missing AVX feature - this is the most informative message 1035b073430SIngo Molnar * to users: 1045b073430SIngo Molnar */ 1055b073430SIngo Molnar if (xfeatures_missing) 1065b073430SIngo Molnar xfeatures_print = xfeatures_missing; 1075b073430SIngo Molnar else 1085b073430SIngo Molnar xfeatures_print = xfeatures_needed; 1095b073430SIngo Molnar 1105b073430SIngo Molnar xfeature_idx = fls64(xfeatures_print)-1; 1115b073430SIngo Molnar max_idx = ARRAY_SIZE(xfeature_names)-1; 1125b073430SIngo Molnar xfeature_idx = min(xfeature_idx, max_idx); 1135b073430SIngo Molnar 1145b073430SIngo Molnar *feature_name = xfeature_names[xfeature_idx]; 1155b073430SIngo Molnar } 1165b073430SIngo Molnar 1175b073430SIngo Molnar if (xfeatures_missing) 1185b073430SIngo Molnar return 0; 1195b073430SIngo Molnar 1205b073430SIngo Molnar return 1; 1215b073430SIngo Molnar } 1225b073430SIngo Molnar EXPORT_SYMBOL_GPL(cpu_has_xfeatures); 1235b073430SIngo Molnar 124158e2ee6SYu-cheng Yu static bool xfeature_is_supervisor(int xfeature_nr) 1251499ce2dSYu-cheng Yu { 1261499ce2dSYu-cheng Yu /* 1278c9e6073SYu-cheng Yu * Extended State Enumeration Sub-leaves (EAX = 0DH, ECX = n, n > 1) 1288c9e6073SYu-cheng Yu * returns ECX[0] set to (1) for a supervisor state, and cleared (0) 1298c9e6073SYu-cheng Yu * for a user state. 1301499ce2dSYu-cheng Yu */ 1311499ce2dSYu-cheng Yu u32 eax, ebx, ecx, edx; 1321499ce2dSYu-cheng Yu 1331499ce2dSYu-cheng Yu cpuid_count(XSTATE_CPUID, xfeature_nr, &eax, &ebx, &ecx, &edx); 134158e2ee6SYu-cheng Yu return ecx & 1; 1351499ce2dSYu-cheng Yu } 1361499ce2dSYu-cheng Yu 1375b073430SIngo Molnar /* 13862784854SIngo Molnar * Enable the extended processor state save/restore feature. 13962784854SIngo Molnar * Called once per CPU onlining. 14062784854SIngo Molnar */ 14162784854SIngo Molnar void fpu__init_cpu_xstate(void) 14262784854SIngo Molnar { 143524bb73bSYu-cheng Yu if (!boot_cpu_has(X86_FEATURE_XSAVE) || !xfeatures_mask_all) 14462784854SIngo Molnar return; 14562784854SIngo Molnar 14662784854SIngo Molnar cr4_set_bits(X86_CR4_OSXSAVE); 14762784854SIngo Molnar 14862784854SIngo Molnar /* 149524bb73bSYu-cheng Yu * XCR_XFEATURE_ENABLED_MASK (aka. XCR0) sets user features 150524bb73bSYu-cheng Yu * managed by XSAVE{C, OPT, S} and XRSTOR{S}. Only XSAVE user 151524bb73bSYu-cheng Yu * states can be set here. 152e6e888f9SDave Hansen */ 15365e95210SThomas Gleixner xsetbv(XCR_XFEATURE_ENABLED_MASK, xfeatures_mask_uabi()); 15462784854SIngo Molnar 15562784854SIngo Molnar /* 15671581eefSYu-cheng Yu * MSR_IA32_XSS sets supervisor states managed by XSAVES. 157e6e888f9SDave Hansen */ 158f0dccc9dSKan Liang if (boot_cpu_has(X86_FEATURE_XSAVES)) { 159f0dccc9dSKan Liang wrmsrl(MSR_IA32_XSS, xfeatures_mask_supervisor() | 16001707b66SAndy Lutomirski xfeatures_mask_independent()); 161f0dccc9dSKan Liang } 16271581eefSYu-cheng Yu } 16371581eefSYu-cheng Yu 164524bb73bSYu-cheng Yu static bool xfeature_enabled(enum xfeature xfeature) 165e6e888f9SDave Hansen { 166524bb73bSYu-cheng Yu return xfeatures_mask_all & BIT_ULL(xfeature); 167e6e888f9SDave Hansen } 168e6e888f9SDave Hansen 169e6e888f9SDave Hansen /* 17039f1acd2SIngo Molnar * Record the offsets and sizes of various xstates contained 17139f1acd2SIngo Molnar * in the XSAVE state memory layout. 17262784854SIngo Molnar */ 17362784854SIngo Molnar static void __init setup_xstate_features(void) 17462784854SIngo Molnar { 175ee9ae257SDave Hansen u32 eax, ebx, ecx, edx, i; 176d9f6e12fSIngo Molnar /* start at the beginning of the "extended state" */ 177e6e888f9SDave Hansen unsigned int last_good_offset = offsetof(struct xregs_state, 178e6e888f9SDave Hansen extended_state_area); 179ac73b27aSYu-cheng Yu /* 180ac73b27aSYu-cheng Yu * The FP xstates and SSE xstates are legacy states. They are always 181ac73b27aSYu-cheng Yu * in the fixed offsets in the xsave area in either compacted form 182ac73b27aSYu-cheng Yu * or standard form. 183ac73b27aSYu-cheng Yu */ 184446e693cSCyrill Gorcunov xstate_offsets[XFEATURE_FP] = 0; 185446e693cSCyrill Gorcunov xstate_sizes[XFEATURE_FP] = offsetof(struct fxregs_state, 186446e693cSCyrill Gorcunov xmm_space); 187446e693cSCyrill Gorcunov 188446e693cSCyrill Gorcunov xstate_offsets[XFEATURE_SSE] = xstate_sizes[XFEATURE_FP]; 189c593642cSPankaj Bharadiya xstate_sizes[XFEATURE_SSE] = sizeof_field(struct fxregs_state, 190446e693cSCyrill Gorcunov xmm_space); 19162784854SIngo Molnar 192ffd3e504SThomas Gleixner for_each_extended_xfeature(i, xfeatures_mask_all) { 193e6e888f9SDave Hansen cpuid_count(XSTATE_CPUID, i, &eax, &ebx, &ecx, &edx); 1941499ce2dSYu-cheng Yu 195c12e13dcSYu-cheng Yu xstate_sizes[i] = eax; 196c12e13dcSYu-cheng Yu 1971499ce2dSYu-cheng Yu /* 198c12e13dcSYu-cheng Yu * If an xfeature is supervisor state, the offset in EBX is 199c12e13dcSYu-cheng Yu * invalid, leave it to -1. 2001499ce2dSYu-cheng Yu */ 201c12e13dcSYu-cheng Yu if (xfeature_is_supervisor(i)) 202c12e13dcSYu-cheng Yu continue; 203c12e13dcSYu-cheng Yu 204ee9ae257SDave Hansen xstate_offsets[i] = ebx; 2051499ce2dSYu-cheng Yu 206e6e888f9SDave Hansen /* 207c12e13dcSYu-cheng Yu * In our xstate size checks, we assume that the highest-numbered 208c12e13dcSYu-cheng Yu * xstate feature has the highest offset in the buffer. Ensure 209c12e13dcSYu-cheng Yu * it does. 210e6e888f9SDave Hansen */ 211e6e888f9SDave Hansen WARN_ONCE(last_good_offset > xstate_offsets[i], 212e6e888f9SDave Hansen "x86/fpu: misordered xstate at %d\n", last_good_offset); 213c12e13dcSYu-cheng Yu 214e6e888f9SDave Hansen last_good_offset = xstate_offsets[i]; 21539f1acd2SIngo Molnar } 21662784854SIngo Molnar } 21762784854SIngo Molnar 21832231879SIngo Molnar static void __init print_xstate_feature(u64 xstate_mask) 21962784854SIngo Molnar { 22033588b52SIngo Molnar const char *feature_name; 22162784854SIngo Molnar 22233588b52SIngo Molnar if (cpu_has_xfeatures(xstate_mask, &feature_name)) 223c8df4009SDave Hansen pr_info("x86/fpu: Supporting XSAVE feature 0x%03Lx: '%s'\n", xstate_mask, feature_name); 22462784854SIngo Molnar } 22562784854SIngo Molnar 22662784854SIngo Molnar /* 22762784854SIngo Molnar * Print out all the supported xstate features: 22862784854SIngo Molnar */ 22932231879SIngo Molnar static void __init print_xstate_features(void) 23062784854SIngo Molnar { 231d91cab78SDave Hansen print_xstate_feature(XFEATURE_MASK_FP); 232d91cab78SDave Hansen print_xstate_feature(XFEATURE_MASK_SSE); 233d91cab78SDave Hansen print_xstate_feature(XFEATURE_MASK_YMM); 234d91cab78SDave Hansen print_xstate_feature(XFEATURE_MASK_BNDREGS); 235d91cab78SDave Hansen print_xstate_feature(XFEATURE_MASK_BNDCSR); 236d91cab78SDave Hansen print_xstate_feature(XFEATURE_MASK_OPMASK); 237d91cab78SDave Hansen print_xstate_feature(XFEATURE_MASK_ZMM_Hi256); 238d91cab78SDave Hansen print_xstate_feature(XFEATURE_MASK_Hi16_ZMM); 239c8df4009SDave Hansen print_xstate_feature(XFEATURE_MASK_PKRU); 240b454feb9SYu-cheng Yu print_xstate_feature(XFEATURE_MASK_PASID); 24162784854SIngo Molnar } 24262784854SIngo Molnar 24362784854SIngo Molnar /* 24403482e08SYu-cheng Yu * This check is important because it is easy to get XSTATE_* 24503482e08SYu-cheng Yu * confused with XSTATE_BIT_*. 24603482e08SYu-cheng Yu */ 24703482e08SYu-cheng Yu #define CHECK_XFEATURE(nr) do { \ 24803482e08SYu-cheng Yu WARN_ON(nr < FIRST_EXTENDED_XFEATURE); \ 24903482e08SYu-cheng Yu WARN_ON(nr >= XFEATURE_MAX); \ 25003482e08SYu-cheng Yu } while (0) 25103482e08SYu-cheng Yu 25203482e08SYu-cheng Yu /* 25303482e08SYu-cheng Yu * We could cache this like xstate_size[], but we only use 25403482e08SYu-cheng Yu * it here, so it would be a waste of space. 25503482e08SYu-cheng Yu */ 25603482e08SYu-cheng Yu static int xfeature_is_aligned(int xfeature_nr) 25703482e08SYu-cheng Yu { 25803482e08SYu-cheng Yu u32 eax, ebx, ecx, edx; 25903482e08SYu-cheng Yu 26003482e08SYu-cheng Yu CHECK_XFEATURE(xfeature_nr); 261e70b1008SYu-cheng Yu 262e70b1008SYu-cheng Yu if (!xfeature_enabled(xfeature_nr)) { 263e70b1008SYu-cheng Yu WARN_ONCE(1, "Checking alignment of disabled xfeature %d\n", 264e70b1008SYu-cheng Yu xfeature_nr); 265e70b1008SYu-cheng Yu return 0; 266e70b1008SYu-cheng Yu } 267e70b1008SYu-cheng Yu 26803482e08SYu-cheng Yu cpuid_count(XSTATE_CPUID, xfeature_nr, &eax, &ebx, &ecx, &edx); 26903482e08SYu-cheng Yu /* 27003482e08SYu-cheng Yu * The value returned by ECX[1] indicates the alignment 27103482e08SYu-cheng Yu * of state component 'i' when the compacted format 27203482e08SYu-cheng Yu * of the extended region of an XSAVE area is used: 27303482e08SYu-cheng Yu */ 27403482e08SYu-cheng Yu return !!(ecx & 2); 27503482e08SYu-cheng Yu } 27603482e08SYu-cheng Yu 27703482e08SYu-cheng Yu /* 27862784854SIngo Molnar * This function sets up offsets and sizes of all extended states in 27962784854SIngo Molnar * xsave area. This supports both standard format and compacted format 28049a91d61SYu-cheng Yu * of the xsave area. 28162784854SIngo Molnar */ 28249a91d61SYu-cheng Yu static void __init setup_xstate_comp_offsets(void) 28362784854SIngo Molnar { 28449a91d61SYu-cheng Yu unsigned int next_offset; 28562784854SIngo Molnar int i; 28662784854SIngo Molnar 28762784854SIngo Molnar /* 28862784854SIngo Molnar * The FP xstates and SSE xstates are legacy states. They are always 28962784854SIngo Molnar * in the fixed offsets in the xsave area in either compacted form 29062784854SIngo Molnar * or standard form. 29162784854SIngo Molnar */ 292446e693cSCyrill Gorcunov xstate_comp_offsets[XFEATURE_FP] = 0; 293446e693cSCyrill Gorcunov xstate_comp_offsets[XFEATURE_SSE] = offsetof(struct fxregs_state, 294446e693cSCyrill Gorcunov xmm_space); 29562784854SIngo Molnar 296ffd3e504SThomas Gleixner if (!cpu_feature_enabled(X86_FEATURE_XSAVES)) { 297ffd3e504SThomas Gleixner for_each_extended_xfeature(i, xfeatures_mask_all) 29862784854SIngo Molnar xstate_comp_offsets[i] = xstate_offsets[i]; 29962784854SIngo Molnar return; 30062784854SIngo Molnar } 30162784854SIngo Molnar 30249a91d61SYu-cheng Yu next_offset = FXSAVE_SIZE + XSAVE_HDR_SIZE; 30362784854SIngo Molnar 304ffd3e504SThomas Gleixner for_each_extended_xfeature(i, xfeatures_mask_all) { 30503482e08SYu-cheng Yu if (xfeature_is_aligned(i)) 30649a91d61SYu-cheng Yu next_offset = ALIGN(next_offset, 64); 30749a91d61SYu-cheng Yu 30849a91d61SYu-cheng Yu xstate_comp_offsets[i] = next_offset; 30949a91d61SYu-cheng Yu next_offset += xstate_sizes[i]; 31062784854SIngo Molnar } 31162784854SIngo Molnar } 31262784854SIngo Molnar 31362784854SIngo Molnar /* 314eeedf153SYu-cheng Yu * Setup offsets of a supervisor-state-only XSAVES buffer: 315eeedf153SYu-cheng Yu * 316eeedf153SYu-cheng Yu * The offsets stored in xstate_comp_offsets[] only work for one specific 317eeedf153SYu-cheng Yu * value of the Requested Feature BitMap (RFBM). In cases where a different 318eeedf153SYu-cheng Yu * RFBM value is used, a different set of offsets is required. This set of 319eeedf153SYu-cheng Yu * offsets is for when RFBM=xfeatures_mask_supervisor(). 320eeedf153SYu-cheng Yu */ 321eeedf153SYu-cheng Yu static void __init setup_supervisor_only_offsets(void) 322eeedf153SYu-cheng Yu { 323eeedf153SYu-cheng Yu unsigned int next_offset; 324eeedf153SYu-cheng Yu int i; 325eeedf153SYu-cheng Yu 326eeedf153SYu-cheng Yu next_offset = FXSAVE_SIZE + XSAVE_HDR_SIZE; 327eeedf153SYu-cheng Yu 328ffd3e504SThomas Gleixner for_each_extended_xfeature(i, xfeatures_mask_all) { 329ffd3e504SThomas Gleixner if (!xfeature_is_supervisor(i)) 330eeedf153SYu-cheng Yu continue; 331eeedf153SYu-cheng Yu 332eeedf153SYu-cheng Yu if (xfeature_is_aligned(i)) 333eeedf153SYu-cheng Yu next_offset = ALIGN(next_offset, 64); 334eeedf153SYu-cheng Yu 335eeedf153SYu-cheng Yu xstate_supervisor_only_offsets[i] = next_offset; 336eeedf153SYu-cheng Yu next_offset += xstate_sizes[i]; 337eeedf153SYu-cheng Yu } 338eeedf153SYu-cheng Yu } 339eeedf153SYu-cheng Yu 340eeedf153SYu-cheng Yu /* 341996952e0SYu-cheng Yu * Print out xstate component offsets and sizes 342996952e0SYu-cheng Yu */ 343996952e0SYu-cheng Yu static void __init print_xstate_offset_size(void) 344996952e0SYu-cheng Yu { 345996952e0SYu-cheng Yu int i; 346996952e0SYu-cheng Yu 347ffd3e504SThomas Gleixner for_each_extended_xfeature(i, xfeatures_mask_all) { 348996952e0SYu-cheng Yu pr_info("x86/fpu: xstate_offset[%d]: %4d, xstate_sizes[%d]: %4d\n", 349996952e0SYu-cheng Yu i, xstate_comp_offsets[i], i, xstate_sizes[i]); 350996952e0SYu-cheng Yu } 351996952e0SYu-cheng Yu } 352996952e0SYu-cheng Yu 353996952e0SYu-cheng Yu /* 354f9dfb5e3SThomas Gleixner * All supported features have either init state all zeros or are 355f9dfb5e3SThomas Gleixner * handled in setup_init_fpu() individually. This is an explicit 356f9dfb5e3SThomas Gleixner * feature list and does not use XFEATURE_MASK*SUPPORTED to catch 357f9dfb5e3SThomas Gleixner * newly added supported features at build time and make people 358f9dfb5e3SThomas Gleixner * actually look at the init state for the new feature. 359f9dfb5e3SThomas Gleixner */ 360f9dfb5e3SThomas Gleixner #define XFEATURES_INIT_FPSTATE_HANDLED \ 361f9dfb5e3SThomas Gleixner (XFEATURE_MASK_FP | \ 362f9dfb5e3SThomas Gleixner XFEATURE_MASK_SSE | \ 363f9dfb5e3SThomas Gleixner XFEATURE_MASK_YMM | \ 364f9dfb5e3SThomas Gleixner XFEATURE_MASK_OPMASK | \ 365f9dfb5e3SThomas Gleixner XFEATURE_MASK_ZMM_Hi256 | \ 366f9dfb5e3SThomas Gleixner XFEATURE_MASK_Hi16_ZMM | \ 367f9dfb5e3SThomas Gleixner XFEATURE_MASK_PKRU | \ 368f9dfb5e3SThomas Gleixner XFEATURE_MASK_BNDREGS | \ 369f9dfb5e3SThomas Gleixner XFEATURE_MASK_BNDCSR | \ 370f9dfb5e3SThomas Gleixner XFEATURE_MASK_PASID) 371f9dfb5e3SThomas Gleixner 372f9dfb5e3SThomas Gleixner /* 37362784854SIngo Molnar * setup the xstate image representing the init state 37462784854SIngo Molnar */ 37532231879SIngo Molnar static void __init setup_init_fpu_buf(void) 37662784854SIngo Molnar { 377f9dfb5e3SThomas Gleixner BUILD_BUG_ON((XFEATURE_MASK_USER_SUPPORTED | 378f9dfb5e3SThomas Gleixner XFEATURE_MASK_SUPERVISOR_SUPPORTED) != 379f9dfb5e3SThomas Gleixner XFEATURES_INIT_FPSTATE_HANDLED); 380f9dfb5e3SThomas Gleixner 381d366bf7eSBorislav Petkov if (!boot_cpu_has(X86_FEATURE_XSAVE)) 38262784854SIngo Molnar return; 38362784854SIngo Molnar 38462784854SIngo Molnar setup_xstate_features(); 38562784854SIngo Molnar print_xstate_features(); 38662784854SIngo Molnar 387126fe040SThomas Gleixner xstate_init_xcomp_bv(&init_fpstate.xsave, xfeatures_mask_all); 38862784854SIngo Molnar 38962784854SIngo Molnar /* 3907d937060SFenghua Yu * Init all the features state with header.xfeatures being 0x0 39162784854SIngo Molnar */ 392b16313f7SThomas Gleixner os_xrstor_booting(&init_fpstate.xsave); 39362784854SIngo Molnar 39462784854SIngo Molnar /* 395f9dfb5e3SThomas Gleixner * All components are now in init state. Read the state back so 396f9dfb5e3SThomas Gleixner * that init_fpstate contains all non-zero init state. This only 397f9dfb5e3SThomas Gleixner * works with XSAVE, but not with XSAVEOPT and XSAVES because 398f9dfb5e3SThomas Gleixner * those use the init optimization which skips writing data for 399f9dfb5e3SThomas Gleixner * components in init state. 400f9dfb5e3SThomas Gleixner * 401f9dfb5e3SThomas Gleixner * XSAVE could be used, but that would require to reshuffle the 402f9dfb5e3SThomas Gleixner * data when XSAVES is available because XSAVES uses xstate 403f9dfb5e3SThomas Gleixner * compaction. But doing so is a pointless exercise because most 404f9dfb5e3SThomas Gleixner * components have an all zeros init state except for the legacy 405f9dfb5e3SThomas Gleixner * ones (FP and SSE). Those can be saved with FXSAVE into the 406f9dfb5e3SThomas Gleixner * legacy area. Adding new features requires to ensure that init 407f9dfb5e3SThomas Gleixner * state is all zeroes or if not to add the necessary handling 408f9dfb5e3SThomas Gleixner * here. 40962784854SIngo Molnar */ 410f9dfb5e3SThomas Gleixner fxsave(&init_fpstate.fxsave); 41162784854SIngo Molnar } 41262784854SIngo Molnar 41365ac2e9bSDave Hansen static int xfeature_uncompacted_offset(int xfeature_nr) 41465ac2e9bSDave Hansen { 41565ac2e9bSDave Hansen u32 eax, ebx, ecx, edx; 41665ac2e9bSDave Hansen 4171499ce2dSYu-cheng Yu /* 4181499ce2dSYu-cheng Yu * Only XSAVES supports supervisor states and it uses compacted 4191499ce2dSYu-cheng Yu * format. Checking a supervisor state's uncompacted offset is 4201499ce2dSYu-cheng Yu * an error. 4211499ce2dSYu-cheng Yu */ 4228ab22804SFenghua Yu if (XFEATURE_MASK_SUPERVISOR_ALL & BIT_ULL(xfeature_nr)) { 4231499ce2dSYu-cheng Yu WARN_ONCE(1, "No fixed offset for xstate %d\n", xfeature_nr); 4241499ce2dSYu-cheng Yu return -1; 4251499ce2dSYu-cheng Yu } 4261499ce2dSYu-cheng Yu 42765ac2e9bSDave Hansen CHECK_XFEATURE(xfeature_nr); 42865ac2e9bSDave Hansen cpuid_count(XSTATE_CPUID, xfeature_nr, &eax, &ebx, &ecx, &edx); 42965ac2e9bSDave Hansen return ebx; 43065ac2e9bSDave Hansen } 43165ac2e9bSDave Hansen 432ce711ea3SKan Liang int xfeature_size(int xfeature_nr) 43365ac2e9bSDave Hansen { 43465ac2e9bSDave Hansen u32 eax, ebx, ecx, edx; 43565ac2e9bSDave Hansen 43665ac2e9bSDave Hansen CHECK_XFEATURE(xfeature_nr); 43765ac2e9bSDave Hansen cpuid_count(XSTATE_CPUID, xfeature_nr, &eax, &ebx, &ecx, &edx); 43865ac2e9bSDave Hansen return eax; 43965ac2e9bSDave Hansen } 44065ac2e9bSDave Hansen 441e63e5d5cSEric Biggers /* Validate an xstate header supplied by userspace (ptrace or sigreturn) */ 44243be46e8SThomas Gleixner static int validate_user_xstate_header(const struct xstate_header *hdr) 443e63e5d5cSEric Biggers { 444e63e5d5cSEric Biggers /* No unknown or supervisor features may be set */ 44565e95210SThomas Gleixner if (hdr->xfeatures & ~xfeatures_mask_uabi()) 446e63e5d5cSEric Biggers return -EINVAL; 447e63e5d5cSEric Biggers 448e63e5d5cSEric Biggers /* Userspace must use the uncompacted format */ 449e63e5d5cSEric Biggers if (hdr->xcomp_bv) 450e63e5d5cSEric Biggers return -EINVAL; 451e63e5d5cSEric Biggers 452e63e5d5cSEric Biggers /* 453e63e5d5cSEric Biggers * If 'reserved' is shrunken to add a new field, make sure to validate 454e63e5d5cSEric Biggers * that new field here! 455e63e5d5cSEric Biggers */ 456e63e5d5cSEric Biggers BUILD_BUG_ON(sizeof(hdr->reserved) != 48); 457e63e5d5cSEric Biggers 458e63e5d5cSEric Biggers /* No reserved bits may be set */ 459e63e5d5cSEric Biggers if (memchr_inv(hdr->reserved, 0, sizeof(hdr->reserved))) 460e63e5d5cSEric Biggers return -EINVAL; 461e63e5d5cSEric Biggers 462e63e5d5cSEric Biggers return 0; 463e63e5d5cSEric Biggers } 464e63e5d5cSEric Biggers 46563cf05a1SThomas Gleixner static void __init __xstate_dump_leaves(void) 46665ac2e9bSDave Hansen { 46765ac2e9bSDave Hansen int i; 46865ac2e9bSDave Hansen u32 eax, ebx, ecx, edx; 46965ac2e9bSDave Hansen static int should_dump = 1; 47065ac2e9bSDave Hansen 47165ac2e9bSDave Hansen if (!should_dump) 47265ac2e9bSDave Hansen return; 47365ac2e9bSDave Hansen should_dump = 0; 47465ac2e9bSDave Hansen /* 47565ac2e9bSDave Hansen * Dump out a few leaves past the ones that we support 47665ac2e9bSDave Hansen * just in case there are some goodies up there 47765ac2e9bSDave Hansen */ 47865ac2e9bSDave Hansen for (i = 0; i < XFEATURE_MAX + 10; i++) { 47965ac2e9bSDave Hansen cpuid_count(XSTATE_CPUID, i, &eax, &ebx, &ecx, &edx); 48065ac2e9bSDave Hansen pr_warn("CPUID[%02x, %02x]: eax=%08x ebx=%08x ecx=%08x edx=%08x\n", 48165ac2e9bSDave Hansen XSTATE_CPUID, i, eax, ebx, ecx, edx); 48265ac2e9bSDave Hansen } 48365ac2e9bSDave Hansen } 48465ac2e9bSDave Hansen 48565ac2e9bSDave Hansen #define XSTATE_WARN_ON(x) do { \ 48665ac2e9bSDave Hansen if (WARN_ONCE(x, "XSAVE consistency problem, dumping leaves")) { \ 48765ac2e9bSDave Hansen __xstate_dump_leaves(); \ 48865ac2e9bSDave Hansen } \ 48965ac2e9bSDave Hansen } while (0) 49065ac2e9bSDave Hansen 491ef78f2a4SDave Hansen #define XCHECK_SZ(sz, nr, nr_macro, __struct) do { \ 492ef78f2a4SDave Hansen if ((nr == nr_macro) && \ 493ef78f2a4SDave Hansen WARN_ONCE(sz != sizeof(__struct), \ 494ef78f2a4SDave Hansen "%s: struct is %zu bytes, cpu state %d bytes\n", \ 495ef78f2a4SDave Hansen __stringify(nr_macro), sizeof(__struct), sz)) { \ 496ef78f2a4SDave Hansen __xstate_dump_leaves(); \ 497ef78f2a4SDave Hansen } \ 498ef78f2a4SDave Hansen } while (0) 499ef78f2a4SDave Hansen 500ef78f2a4SDave Hansen /* 501ef78f2a4SDave Hansen * We have a C struct for each 'xstate'. We need to ensure 502ef78f2a4SDave Hansen * that our software representation matches what the CPU 503ef78f2a4SDave Hansen * tells us about the state's size. 504ef78f2a4SDave Hansen */ 50563cf05a1SThomas Gleixner static void __init check_xstate_against_struct(int nr) 506ef78f2a4SDave Hansen { 507ef78f2a4SDave Hansen /* 508ef78f2a4SDave Hansen * Ask the CPU for the size of the state. 509ef78f2a4SDave Hansen */ 510ef78f2a4SDave Hansen int sz = xfeature_size(nr); 511ef78f2a4SDave Hansen /* 512ef78f2a4SDave Hansen * Match each CPU state with the corresponding software 513ef78f2a4SDave Hansen * structure. 514ef78f2a4SDave Hansen */ 515ef78f2a4SDave Hansen XCHECK_SZ(sz, nr, XFEATURE_YMM, struct ymmh_struct); 516ef78f2a4SDave Hansen XCHECK_SZ(sz, nr, XFEATURE_BNDREGS, struct mpx_bndreg_state); 517ef78f2a4SDave Hansen XCHECK_SZ(sz, nr, XFEATURE_BNDCSR, struct mpx_bndcsr_state); 518ef78f2a4SDave Hansen XCHECK_SZ(sz, nr, XFEATURE_OPMASK, struct avx_512_opmask_state); 519ef78f2a4SDave Hansen XCHECK_SZ(sz, nr, XFEATURE_ZMM_Hi256, struct avx_512_zmm_uppers_state); 520ef78f2a4SDave Hansen XCHECK_SZ(sz, nr, XFEATURE_Hi16_ZMM, struct avx_512_hi16_state); 521c8df4009SDave Hansen XCHECK_SZ(sz, nr, XFEATURE_PKRU, struct pkru_state); 522b454feb9SYu-cheng Yu XCHECK_SZ(sz, nr, XFEATURE_PASID, struct ia32_pasid_state); 523ef78f2a4SDave Hansen 524ef78f2a4SDave Hansen /* 525ef78f2a4SDave Hansen * Make *SURE* to add any feature numbers in below if 526ef78f2a4SDave Hansen * there are "holes" in the xsave state component 527ef78f2a4SDave Hansen * numbers. 528ef78f2a4SDave Hansen */ 529ef78f2a4SDave Hansen if ((nr < XFEATURE_YMM) || 5301f96b1efSDave Hansen (nr >= XFEATURE_MAX) || 531f0dccc9dSKan Liang (nr == XFEATURE_PT_UNIMPLEMENTED_SO_FAR) || 532b454feb9SYu-cheng Yu ((nr >= XFEATURE_RSRVD_COMP_11) && (nr <= XFEATURE_LBR))) { 533ef78f2a4SDave Hansen WARN_ONCE(1, "no structure for xstate: %d\n", nr); 534ef78f2a4SDave Hansen XSTATE_WARN_ON(1); 535ef78f2a4SDave Hansen } 536ef78f2a4SDave Hansen } 537ef78f2a4SDave Hansen 53865ac2e9bSDave Hansen /* 53965ac2e9bSDave Hansen * This essentially double-checks what the cpu told us about 54065ac2e9bSDave Hansen * how large the XSAVE buffer needs to be. We are recalculating 54165ac2e9bSDave Hansen * it to be safe. 54276d10256SKan Liang * 54301707b66SAndy Lutomirski * Independent XSAVE features allocate their own buffers and are not 54476d10256SKan Liang * covered by these checks. Only the size of the buffer for task->fpu 54576d10256SKan Liang * is checked here. 54665ac2e9bSDave Hansen */ 54763cf05a1SThomas Gleixner static void __init do_extra_xstate_size_checks(void) 54865ac2e9bSDave Hansen { 54965ac2e9bSDave Hansen int paranoid_xstate_size = FXSAVE_SIZE + XSAVE_HDR_SIZE; 55065ac2e9bSDave Hansen int i; 55165ac2e9bSDave Hansen 552ffd3e504SThomas Gleixner for_each_extended_xfeature(i, xfeatures_mask_all) { 553ef78f2a4SDave Hansen check_xstate_against_struct(i); 55465ac2e9bSDave Hansen /* 55565ac2e9bSDave Hansen * Supervisor state components can be managed only by 55602b93c0bSThomas Gleixner * XSAVES. 55765ac2e9bSDave Hansen */ 55802b93c0bSThomas Gleixner if (!cpu_feature_enabled(X86_FEATURE_XSAVES)) 55965ac2e9bSDave Hansen XSTATE_WARN_ON(xfeature_is_supervisor(i)); 56065ac2e9bSDave Hansen 56165ac2e9bSDave Hansen /* Align from the end of the previous feature */ 56265ac2e9bSDave Hansen if (xfeature_is_aligned(i)) 56365ac2e9bSDave Hansen paranoid_xstate_size = ALIGN(paranoid_xstate_size, 64); 56465ac2e9bSDave Hansen /* 56565ac2e9bSDave Hansen * The offset of a given state in the non-compacted 56665ac2e9bSDave Hansen * format is given to us in a CPUID leaf. We check 56765ac2e9bSDave Hansen * them for being ordered (increasing offsets) in 56802b93c0bSThomas Gleixner * setup_xstate_features(). XSAVES uses compacted format. 56965ac2e9bSDave Hansen */ 57002b93c0bSThomas Gleixner if (!cpu_feature_enabled(X86_FEATURE_XSAVES)) 57165ac2e9bSDave Hansen paranoid_xstate_size = xfeature_uncompacted_offset(i); 57265ac2e9bSDave Hansen /* 57365ac2e9bSDave Hansen * The compacted-format offset always depends on where 57465ac2e9bSDave Hansen * the previous state ended. 57565ac2e9bSDave Hansen */ 57665ac2e9bSDave Hansen paranoid_xstate_size += xfeature_size(i); 57765ac2e9bSDave Hansen } 578bf15a8cfSFenghua Yu XSTATE_WARN_ON(paranoid_xstate_size != fpu_kernel_xstate_size); 57965ac2e9bSDave Hansen } 58065ac2e9bSDave Hansen 58162784854SIngo Molnar /* 582524bb73bSYu-cheng Yu * Get total size of enabled xstates in XCR0 | IA32_XSS. 58365ac2e9bSDave Hansen * 58465ac2e9bSDave Hansen * Note the SDM's wording here. "sub-function 0" only enumerates 58565ac2e9bSDave Hansen * the size of the *user* states. If we use it to size a buffer 58665ac2e9bSDave Hansen * that we use 'XSAVES' on, we could potentially overflow the 58765ac2e9bSDave Hansen * buffer because 'XSAVES' saves system states too. 58862784854SIngo Molnar */ 589a1141e0bSFenghua Yu static unsigned int __init get_xsaves_size(void) 59062784854SIngo Molnar { 59162784854SIngo Molnar unsigned int eax, ebx, ecx, edx; 59265ac2e9bSDave Hansen /* 59365ac2e9bSDave Hansen * - CPUID function 0DH, sub-function 1: 59465ac2e9bSDave Hansen * EBX enumerates the size (in bytes) required by 59565ac2e9bSDave Hansen * the XSAVES instruction for an XSAVE area 59665ac2e9bSDave Hansen * containing all the state components 59765ac2e9bSDave Hansen * corresponding to bits currently set in 59865ac2e9bSDave Hansen * XCR0 | IA32_XSS. 59965ac2e9bSDave Hansen */ 60065ac2e9bSDave Hansen cpuid_count(XSTATE_CPUID, 1, &eax, &ebx, &ecx, &edx); 601a1141e0bSFenghua Yu return ebx; 60262784854SIngo Molnar } 603a1141e0bSFenghua Yu 60476d10256SKan Liang /* 60501707b66SAndy Lutomirski * Get the total size of the enabled xstates without the independent supervisor 60676d10256SKan Liang * features. 60776d10256SKan Liang */ 60801707b66SAndy Lutomirski static unsigned int __init get_xsaves_size_no_independent(void) 60976d10256SKan Liang { 61001707b66SAndy Lutomirski u64 mask = xfeatures_mask_independent(); 61176d10256SKan Liang unsigned int size; 61276d10256SKan Liang 61376d10256SKan Liang if (!mask) 61476d10256SKan Liang return get_xsaves_size(); 61576d10256SKan Liang 61601707b66SAndy Lutomirski /* Disable independent features. */ 61776d10256SKan Liang wrmsrl(MSR_IA32_XSS, xfeatures_mask_supervisor()); 61876d10256SKan Liang 61976d10256SKan Liang /* 62076d10256SKan Liang * Ask the hardware what size is required of the buffer. 62176d10256SKan Liang * This is the size required for the task->fpu buffer. 62276d10256SKan Liang */ 62376d10256SKan Liang size = get_xsaves_size(); 62476d10256SKan Liang 62501707b66SAndy Lutomirski /* Re-enable independent features so XSAVES will work on them again. */ 62676d10256SKan Liang wrmsrl(MSR_IA32_XSS, xfeatures_mask_supervisor() | mask); 62776d10256SKan Liang 62876d10256SKan Liang return size; 62976d10256SKan Liang } 63076d10256SKan Liang 631a1141e0bSFenghua Yu static unsigned int __init get_xsave_size(void) 632a1141e0bSFenghua Yu { 633a1141e0bSFenghua Yu unsigned int eax, ebx, ecx, edx; 634a1141e0bSFenghua Yu /* 635a1141e0bSFenghua Yu * - CPUID function 0DH, sub-function 0: 636a1141e0bSFenghua Yu * EBX enumerates the size (in bytes) required by 637a1141e0bSFenghua Yu * the XSAVE instruction for an XSAVE area 638a1141e0bSFenghua Yu * containing all the *user* state components 639a1141e0bSFenghua Yu * corresponding to bits currently set in XCR0. 640a1141e0bSFenghua Yu */ 641a1141e0bSFenghua Yu cpuid_count(XSTATE_CPUID, 0, &eax, &ebx, &ecx, &edx); 642a1141e0bSFenghua Yu return ebx; 6434109ca06SDave Hansen } 6444109ca06SDave Hansen 6454109ca06SDave Hansen /* 6464109ca06SDave Hansen * Will the runtime-enumerated 'xstate_size' fit in the init 6474109ca06SDave Hansen * task's statically-allocated buffer? 6484109ca06SDave Hansen */ 64963cf05a1SThomas Gleixner static bool __init is_supported_xstate_size(unsigned int test_xstate_size) 6504109ca06SDave Hansen { 6514109ca06SDave Hansen if (test_xstate_size <= sizeof(union fpregs_state)) 6524109ca06SDave Hansen return true; 6534109ca06SDave Hansen 6544109ca06SDave Hansen pr_warn("x86/fpu: xstate buffer too small (%zu < %d), disabling xsave\n", 6554109ca06SDave Hansen sizeof(union fpregs_state), test_xstate_size); 6564109ca06SDave Hansen return false; 6574109ca06SDave Hansen } 6584109ca06SDave Hansen 659653a561bSSergey Senozhatsky static int __init init_xstate_size(void) 6604109ca06SDave Hansen { 6614109ca06SDave Hansen /* Recompute the context size for enabled features: */ 662a1141e0bSFenghua Yu unsigned int possible_xstate_size; 663a1141e0bSFenghua Yu unsigned int xsave_size; 664a1141e0bSFenghua Yu 665a1141e0bSFenghua Yu xsave_size = get_xsave_size(); 666a1141e0bSFenghua Yu 667a1141e0bSFenghua Yu if (boot_cpu_has(X86_FEATURE_XSAVES)) 66801707b66SAndy Lutomirski possible_xstate_size = get_xsaves_size_no_independent(); 669a1141e0bSFenghua Yu else 670a1141e0bSFenghua Yu possible_xstate_size = xsave_size; 6714109ca06SDave Hansen 6724109ca06SDave Hansen /* Ensure we have the space to store all enabled: */ 6734109ca06SDave Hansen if (!is_supported_xstate_size(possible_xstate_size)) 6744109ca06SDave Hansen return -EINVAL; 6754109ca06SDave Hansen 6764109ca06SDave Hansen /* 6774109ca06SDave Hansen * The size is OK, we are definitely going to use xsave, 6784109ca06SDave Hansen * make it known to the world that we need more space. 6794109ca06SDave Hansen */ 680bf15a8cfSFenghua Yu fpu_kernel_xstate_size = possible_xstate_size; 68165ac2e9bSDave Hansen do_extra_xstate_size_checks(); 682a1141e0bSFenghua Yu 683a1141e0bSFenghua Yu /* 684a1141e0bSFenghua Yu * User space is always in standard format. 685a1141e0bSFenghua Yu */ 686a1141e0bSFenghua Yu fpu_user_xstate_size = xsave_size; 6874109ca06SDave Hansen return 0; 6884109ca06SDave Hansen } 6894109ca06SDave Hansen 690d91cab78SDave Hansen /* 691d91cab78SDave Hansen * We enabled the XSAVE hardware, but something went wrong and 692d91cab78SDave Hansen * we can not use it. Disable it. 693d91cab78SDave Hansen */ 69463cf05a1SThomas Gleixner static void __init fpu__init_disable_system_xstate(void) 6954109ca06SDave Hansen { 696524bb73bSYu-cheng Yu xfeatures_mask_all = 0; 6974109ca06SDave Hansen cr4_clear_bits(X86_CR4_OSXSAVE); 6987891bc0aSSebastian Andrzej Siewior setup_clear_cpu_cap(X86_FEATURE_XSAVE); 69962784854SIngo Molnar } 70062784854SIngo Molnar 70162784854SIngo Molnar /* 70262784854SIngo Molnar * Enable and initialize the xsave feature. 70362784854SIngo Molnar * Called once per system bootup. 70462784854SIngo Molnar */ 70532231879SIngo Molnar void __init fpu__init_system_xstate(void) 70662784854SIngo Molnar { 70762784854SIngo Molnar unsigned int eax, ebx, ecx, edx; 7084e8e4313SThomas Gleixner u64 xfeatures; 7094109ca06SDave Hansen int err; 710ccb18db2SAndi Kleen int i; 711e97131a8SIngo Molnar 7129170fb40SAndy Lutomirski if (!boot_cpu_has(X86_FEATURE_FPU)) { 7139170fb40SAndy Lutomirski pr_info("x86/fpu: No FPU detected\n"); 7149170fb40SAndy Lutomirski return; 7159170fb40SAndy Lutomirski } 7169170fb40SAndy Lutomirski 717d366bf7eSBorislav Petkov if (!boot_cpu_has(X86_FEATURE_XSAVE)) { 7189170fb40SAndy Lutomirski pr_info("x86/fpu: x87 FPU will use %s\n", 7199170fb40SAndy Lutomirski boot_cpu_has(X86_FEATURE_FXSR) ? "FXSAVE" : "FSAVE"); 72062784854SIngo Molnar return; 72162784854SIngo Molnar } 72262784854SIngo Molnar 72362784854SIngo Molnar if (boot_cpu_data.cpuid_level < XSTATE_CPUID) { 724e97131a8SIngo Molnar WARN_ON_FPU(1); 72562784854SIngo Molnar return; 72662784854SIngo Molnar } 72762784854SIngo Molnar 728524bb73bSYu-cheng Yu /* 729524bb73bSYu-cheng Yu * Find user xstates supported by the processor. 730524bb73bSYu-cheng Yu */ 73162784854SIngo Molnar cpuid_count(XSTATE_CPUID, 0, &eax, &ebx, &ecx, &edx); 732524bb73bSYu-cheng Yu xfeatures_mask_all = eax + ((u64)edx << 32); 73362784854SIngo Molnar 73471581eefSYu-cheng Yu /* 73571581eefSYu-cheng Yu * Find supervisor xstates supported by the processor. 73671581eefSYu-cheng Yu */ 73771581eefSYu-cheng Yu cpuid_count(XSTATE_CPUID, 1, &eax, &ebx, &ecx, &edx); 73871581eefSYu-cheng Yu xfeatures_mask_all |= ecx + ((u64)edx << 32); 73971581eefSYu-cheng Yu 74065e95210SThomas Gleixner if ((xfeatures_mask_uabi() & XFEATURE_MASK_FPSSE) != XFEATURE_MASK_FPSSE) { 741ec3ed4a2SDave Hansen /* 742ec3ed4a2SDave Hansen * This indicates that something really unexpected happened 743ec3ed4a2SDave Hansen * with the enumeration. Disable XSAVE and try to continue 744ec3ed4a2SDave Hansen * booting without it. This is too early to BUG(). 745ec3ed4a2SDave Hansen */ 746524bb73bSYu-cheng Yu pr_err("x86/fpu: FP/SSE not present amongst the CPU's xstate features: 0x%llx.\n", 747524bb73bSYu-cheng Yu xfeatures_mask_all); 748ec3ed4a2SDave Hansen goto out_disable; 74962784854SIngo Molnar } 75062784854SIngo Molnar 751ccb18db2SAndi Kleen /* 752ccb18db2SAndi Kleen * Clear XSAVE features that are disabled in the normal CPUID. 753ccb18db2SAndi Kleen */ 754ccb18db2SAndi Kleen for (i = 0; i < ARRAY_SIZE(xsave_cpuid_features); i++) { 755ccb18db2SAndi Kleen if (!boot_cpu_has(xsave_cpuid_features[i])) 756524bb73bSYu-cheng Yu xfeatures_mask_all &= ~BIT_ULL(i); 757ccb18db2SAndi Kleen } 758ccb18db2SAndi Kleen 759ce38f038SThomas Gleixner xfeatures_mask_all &= XFEATURE_MASK_USER_SUPPORTED | 760ce38f038SThomas Gleixner XFEATURE_MASK_SUPERVISOR_SUPPORTED; 761ce38f038SThomas Gleixner 7624e8e4313SThomas Gleixner /* Store it for paranoia check at the end */ 7634e8e4313SThomas Gleixner xfeatures = xfeatures_mask_all; 76462784854SIngo Molnar 76562784854SIngo Molnar /* Enable xstate instructions to be able to continue with initialization: */ 76662784854SIngo Molnar fpu__init_cpu_xstate(); 7674109ca06SDave Hansen err = init_xstate_size(); 768ec3ed4a2SDave Hansen if (err) 769ec3ed4a2SDave Hansen goto out_disable; 77062784854SIngo Molnar 77191c3dba7SYu-cheng Yu /* 77291c3dba7SYu-cheng Yu * Update info used for ptrace frames; use standard-format size and no 77391c3dba7SYu-cheng Yu * supervisor xstates: 77491c3dba7SYu-cheng Yu */ 77565e95210SThomas Gleixner update_regset_xstate_info(fpu_user_xstate_size, xfeatures_mask_uabi()); 77691c3dba7SYu-cheng Yu 777b992c660SIngo Molnar fpu__init_prepare_fx_sw_frame(); 77862784854SIngo Molnar setup_init_fpu_buf(); 77949a91d61SYu-cheng Yu setup_xstate_comp_offsets(); 780eeedf153SYu-cheng Yu setup_supervisor_only_offsets(); 78162784854SIngo Molnar 7824e8e4313SThomas Gleixner /* 7834e8e4313SThomas Gleixner * Paranoia check whether something in the setup modified the 7844e8e4313SThomas Gleixner * xfeatures mask. 7854e8e4313SThomas Gleixner */ 7864e8e4313SThomas Gleixner if (xfeatures != xfeatures_mask_all) { 7874e8e4313SThomas Gleixner pr_err("x86/fpu: xfeatures modified from 0x%016llx to 0x%016llx during init, disabling XSAVE\n", 7884e8e4313SThomas Gleixner xfeatures, xfeatures_mask_all); 7894e8e4313SThomas Gleixner goto out_disable; 7904e8e4313SThomas Gleixner } 7914e8e4313SThomas Gleixner 7924e8e4313SThomas Gleixner print_xstate_offset_size(); 793b0815359SDave Hansen pr_info("x86/fpu: Enabled xstate features 0x%llx, context size is %d bytes, using '%s' format.\n", 794524bb73bSYu-cheng Yu xfeatures_mask_all, 795bf15a8cfSFenghua Yu fpu_kernel_xstate_size, 796782511b0SBorislav Petkov boot_cpu_has(X86_FEATURE_XSAVES) ? "compacted" : "standard"); 797ec3ed4a2SDave Hansen return; 798ec3ed4a2SDave Hansen 799ec3ed4a2SDave Hansen out_disable: 800ec3ed4a2SDave Hansen /* something went wrong, try to boot without any XSAVE support */ 801ec3ed4a2SDave Hansen fpu__init_disable_system_xstate(); 80262784854SIngo Molnar } 80362784854SIngo Molnar 80462784854SIngo Molnar /* 80562784854SIngo Molnar * Restore minimal FPU state after suspend: 80662784854SIngo Molnar */ 80762784854SIngo Molnar void fpu__resume_cpu(void) 80862784854SIngo Molnar { 80962784854SIngo Molnar /* 81062784854SIngo Molnar * Restore XCR0 on xsave capable CPUs: 81162784854SIngo Molnar */ 81265e95210SThomas Gleixner if (cpu_feature_enabled(X86_FEATURE_XSAVE)) 81365e95210SThomas Gleixner xsetbv(XCR_XFEATURE_ENABLED_MASK, xfeatures_mask_uabi()); 81471581eefSYu-cheng Yu 81571581eefSYu-cheng Yu /* 81671581eefSYu-cheng Yu * Restore IA32_XSS. The same CPUID bit enumerates support 81771581eefSYu-cheng Yu * of XSAVES and MSR_IA32_XSS. 81871581eefSYu-cheng Yu */ 81965e95210SThomas Gleixner if (cpu_feature_enabled(X86_FEATURE_XSAVES)) { 820f0dccc9dSKan Liang wrmsrl(MSR_IA32_XSS, xfeatures_mask_supervisor() | 82101707b66SAndy Lutomirski xfeatures_mask_independent()); 822f0dccc9dSKan Liang } 82362784854SIngo Molnar } 82462784854SIngo Molnar 82562784854SIngo Molnar /* 82607baeb04SSebastian Andrzej Siewior * Given an xstate feature nr, calculate where in the xsave 827b8b9b6baSDave Hansen * buffer the state is. Callers should ensure that the buffer 828b8b9b6baSDave Hansen * is valid. 829b8b9b6baSDave Hansen */ 83007baeb04SSebastian Andrzej Siewior static void *__raw_xsave_addr(struct xregs_state *xsave, int xfeature_nr) 831b8b9b6baSDave Hansen { 83207baeb04SSebastian Andrzej Siewior if (!xfeature_enabled(xfeature_nr)) { 8335060b915SYu-cheng Yu WARN_ON_FPU(1); 8345060b915SYu-cheng Yu return NULL; 8355060b915SYu-cheng Yu } 8365060b915SYu-cheng Yu 83707baeb04SSebastian Andrzej Siewior return (void *)xsave + xstate_comp_offsets[xfeature_nr]; 838b8b9b6baSDave Hansen } 839b8b9b6baSDave Hansen /* 84062784854SIngo Molnar * Given the xsave area and a state inside, this function returns the 84162784854SIngo Molnar * address of the state. 84262784854SIngo Molnar * 84362784854SIngo Molnar * This is the API that is called to get xstate address in either 84462784854SIngo Molnar * standard format or compacted format of xsave area. 84562784854SIngo Molnar * 8460c4109beSDave Hansen * Note that if there is no data for the field in the xsave buffer 8470c4109beSDave Hansen * this will return NULL. 8480c4109beSDave Hansen * 84962784854SIngo Molnar * Inputs: 8500c4109beSDave Hansen * xstate: the thread's storage area for all FPU data 851abd16d68SSebastian Andrzej Siewior * xfeature_nr: state which is defined in xsave.h (e.g. XFEATURE_FP, 852abd16d68SSebastian Andrzej Siewior * XFEATURE_SSE, etc...) 85362784854SIngo Molnar * Output: 8540c4109beSDave Hansen * address of the state in the xsave area, or NULL if the 8550c4109beSDave Hansen * field is not present in the xsave buffer. 85662784854SIngo Molnar */ 857abd16d68SSebastian Andrzej Siewior void *get_xsave_addr(struct xregs_state *xsave, int xfeature_nr) 85862784854SIngo Molnar { 8590c4109beSDave Hansen /* 8600c4109beSDave Hansen * Do we even *have* xsave state? 8610c4109beSDave Hansen */ 8620c4109beSDave Hansen if (!boot_cpu_has(X86_FEATURE_XSAVE)) 86362784854SIngo Molnar return NULL; 86462784854SIngo Molnar 8650c4109beSDave Hansen /* 8660c4109beSDave Hansen * We should not ever be requesting features that we 867524bb73bSYu-cheng Yu * have not enabled. 8680c4109beSDave Hansen */ 869524bb73bSYu-cheng Yu WARN_ONCE(!(xfeatures_mask_all & BIT_ULL(xfeature_nr)), 8700c4109beSDave Hansen "get of unsupported state"); 8710c4109beSDave Hansen /* 8720c4109beSDave Hansen * This assumes the last 'xsave*' instruction to 873abd16d68SSebastian Andrzej Siewior * have requested that 'xfeature_nr' be saved. 8740c4109beSDave Hansen * If it did not, we might be seeing and old value 8750c4109beSDave Hansen * of the field in the buffer. 8760c4109beSDave Hansen * 8770c4109beSDave Hansen * This can happen because the last 'xsave' did not 8780c4109beSDave Hansen * request that this feature be saved (unlikely) 8790c4109beSDave Hansen * or because the "init optimization" caused it 8800c4109beSDave Hansen * to not be saved. 8810c4109beSDave Hansen */ 882abd16d68SSebastian Andrzej Siewior if (!(xsave->header.xfeatures & BIT_ULL(xfeature_nr))) 8830c4109beSDave Hansen return NULL; 8840c4109beSDave Hansen 88507baeb04SSebastian Andrzej Siewior return __raw_xsave_addr(xsave, xfeature_nr); 88662784854SIngo Molnar } 88762784854SIngo Molnar EXPORT_SYMBOL_GPL(get_xsave_addr); 88804cd027bSDave Hansen 889e8c24d3aSDave Hansen #ifdef CONFIG_ARCH_HAS_PKEYS 890e8c24d3aSDave Hansen 89184594296SDave Hansen /* 892b79daf85SDave Hansen * This will go out and modify PKRU register to set the access 893b79daf85SDave Hansen * rights for @pkey to @init_val. 89484594296SDave Hansen */ 89584594296SDave Hansen int arch_set_user_pkey_access(struct task_struct *tsk, int pkey, 89684594296SDave Hansen unsigned long init_val) 89784594296SDave Hansen { 8989fe8a6f5SThomas Gleixner u32 old_pkru, new_pkru_bits = 0; 8999fe8a6f5SThomas Gleixner int pkey_shift; 90084594296SDave Hansen 90184594296SDave Hansen /* 90284594296SDave Hansen * This check implies XSAVE support. OSPKE only gets 90384594296SDave Hansen * set if we enable XSAVE and we enable PKU in XCR0. 90484594296SDave Hansen */ 9058a1dc55aSThomas Gleixner if (!cpu_feature_enabled(X86_FEATURE_OSPKE)) 90684594296SDave Hansen return -EINVAL; 90784594296SDave Hansen 90816171bffSDave Hansen /* 90916171bffSDave Hansen * This code should only be called with valid 'pkey' 91016171bffSDave Hansen * values originating from in-kernel users. Complain 91116171bffSDave Hansen * if a bad value is observed. 91216171bffSDave Hansen */ 9139fe8a6f5SThomas Gleixner if (WARN_ON_ONCE(pkey >= arch_max_pkey())) 9149fe8a6f5SThomas Gleixner return -EINVAL; 91516171bffSDave Hansen 91691c3dba7SYu-cheng Yu /* Set the bits we need in PKRU: */ 91784594296SDave Hansen if (init_val & PKEY_DISABLE_ACCESS) 91884594296SDave Hansen new_pkru_bits |= PKRU_AD_BIT; 91984594296SDave Hansen if (init_val & PKEY_DISABLE_WRITE) 92084594296SDave Hansen new_pkru_bits |= PKRU_WD_BIT; 92184594296SDave Hansen 92291c3dba7SYu-cheng Yu /* Shift the bits in to the correct place in PKRU for pkey: */ 9239fe8a6f5SThomas Gleixner pkey_shift = pkey * PKRU_BITS_PER_PKEY; 92484594296SDave Hansen new_pkru_bits <<= pkey_shift; 92584594296SDave Hansen 926b79daf85SDave Hansen /* Get old PKRU and mask off any old bits in place: */ 927b79daf85SDave Hansen old_pkru = read_pkru(); 928b79daf85SDave Hansen old_pkru &= ~((PKRU_AD_BIT|PKRU_WD_BIT) << pkey_shift); 92984594296SDave Hansen 930b79daf85SDave Hansen /* Write old part along with new part: */ 931b79daf85SDave Hansen write_pkru(old_pkru | new_pkru_bits); 93291c3dba7SYu-cheng Yu 93391c3dba7SYu-cheng Yu return 0; 93491c3dba7SYu-cheng Yu } 935e8c24d3aSDave Hansen #endif /* ! CONFIG_ARCH_HAS_PKEYS */ 93691c3dba7SYu-cheng Yu 93796258950SThomas Gleixner static void copy_feature(bool from_xstate, struct membuf *to, void *xstate, 93896258950SThomas Gleixner void *init_xstate, unsigned int size) 939f0d4f30aSIngo Molnar { 94096258950SThomas Gleixner membuf_write(to, from_xstate ? xstate : init_xstate, size); 941f0d4f30aSIngo Molnar } 942f0d4f30aSIngo Molnar 943eb6f5172SThomas Gleixner /** 944ca834defSThomas Gleixner * __copy_xstate_to_uabi_buf - Copy kernel saved xstate to a UABI buffer 945eb6f5172SThomas Gleixner * @to: membuf descriptor 946ca834defSThomas Gleixner * @xsave: The xsave from which to copy 947ca834defSThomas Gleixner * @pkru_val: The PKRU value to store in the PKRU component 948eb6f5172SThomas Gleixner * @copy_mode: The requested copy mode 949f0d4f30aSIngo Molnar * 950eb6f5172SThomas Gleixner * Converts from kernel XSAVE or XSAVES compacted format to UABI conforming 951eb6f5172SThomas Gleixner * format, i.e. from the kernel internal hardware dependent storage format 952eb6f5172SThomas Gleixner * to the requested @mode. UABI XSTATE is always uncompacted! 953eb6f5172SThomas Gleixner * 954eb6f5172SThomas Gleixner * It supports partial copy but @to.pos always starts from zero. 955f0d4f30aSIngo Molnar */ 956ca834defSThomas Gleixner void __copy_xstate_to_uabi_buf(struct membuf to, struct xregs_state *xsave, 957ca834defSThomas Gleixner u32 pkru_val, enum xstate_copy_mode copy_mode) 958f0d4f30aSIngo Molnar { 95996258950SThomas Gleixner const unsigned int off_mxcsr = offsetof(struct fxregs_state, mxcsr); 96096258950SThomas Gleixner struct xregs_state *xinit = &init_fpstate.xsave; 961f0d4f30aSIngo Molnar struct xstate_header header; 96296258950SThomas Gleixner unsigned int zerofrom; 963ffd3e504SThomas Gleixner u64 mask; 9648c0817f4SIngo Molnar int i; 965f0d4f30aSIngo Molnar 96693c2cdc9SThomas Gleixner memset(&header, 0, sizeof(header)); 967f0d4f30aSIngo Molnar header.xfeatures = xsave->header.xfeatures; 968eb6f5172SThomas Gleixner 969eb6f5172SThomas Gleixner /* Mask out the feature bits depending on copy mode */ 970eb6f5172SThomas Gleixner switch (copy_mode) { 971eb6f5172SThomas Gleixner case XSTATE_COPY_FP: 972eb6f5172SThomas Gleixner header.xfeatures &= XFEATURE_MASK_FP; 973eb6f5172SThomas Gleixner break; 974eb6f5172SThomas Gleixner 975eb6f5172SThomas Gleixner case XSTATE_COPY_FX: 976eb6f5172SThomas Gleixner header.xfeatures &= XFEATURE_MASK_FP | XFEATURE_MASK_SSE; 977eb6f5172SThomas Gleixner break; 978eb6f5172SThomas Gleixner 979eb6f5172SThomas Gleixner case XSTATE_COPY_XSAVE: 98065e95210SThomas Gleixner header.xfeatures &= xfeatures_mask_uabi(); 981eb6f5172SThomas Gleixner break; 982eb6f5172SThomas Gleixner } 983f0d4f30aSIngo Molnar 98496258950SThomas Gleixner /* Copy FP state up to MXCSR */ 98596258950SThomas Gleixner copy_feature(header.xfeatures & XFEATURE_MASK_FP, &to, &xsave->i387, 98696258950SThomas Gleixner &xinit->i387, off_mxcsr); 98796258950SThomas Gleixner 98896258950SThomas Gleixner /* Copy MXCSR when SSE or YMM are set in the feature mask */ 98996258950SThomas Gleixner copy_feature(header.xfeatures & (XFEATURE_MASK_SSE | XFEATURE_MASK_YMM), 99096258950SThomas Gleixner &to, &xsave->i387.mxcsr, &xinit->i387.mxcsr, 99196258950SThomas Gleixner MXCSR_AND_FLAGS_SIZE); 99296258950SThomas Gleixner 99396258950SThomas Gleixner /* Copy the remaining FP state */ 99496258950SThomas Gleixner copy_feature(header.xfeatures & XFEATURE_MASK_FP, 99596258950SThomas Gleixner &to, &xsave->i387.st_space, &xinit->i387.st_space, 99696258950SThomas Gleixner sizeof(xsave->i387.st_space)); 99796258950SThomas Gleixner 99896258950SThomas Gleixner /* Copy the SSE state - shared with YMM, but independently managed */ 99996258950SThomas Gleixner copy_feature(header.xfeatures & XFEATURE_MASK_SSE, 100096258950SThomas Gleixner &to, &xsave->i387.xmm_space, &xinit->i387.xmm_space, 100196258950SThomas Gleixner sizeof(xsave->i387.xmm_space)); 100296258950SThomas Gleixner 1003eb6f5172SThomas Gleixner if (copy_mode != XSTATE_COPY_XSAVE) 1004eb6f5172SThomas Gleixner goto out; 1005eb6f5172SThomas Gleixner 100696258950SThomas Gleixner /* Zero the padding area */ 100796258950SThomas Gleixner membuf_zero(&to, sizeof(xsave->i387.padding)); 100896258950SThomas Gleixner 100996258950SThomas Gleixner /* Copy xsave->i387.sw_reserved */ 101096258950SThomas Gleixner membuf_write(&to, xstate_fx_sw_bytes, sizeof(xsave->i387.sw_reserved)); 101196258950SThomas Gleixner 101296258950SThomas Gleixner /* Copy the user space relevant state of @xsave->header */ 101396258950SThomas Gleixner membuf_write(&to, &header, sizeof(header)); 101496258950SThomas Gleixner 101596258950SThomas Gleixner zerofrom = offsetof(struct xregs_state, extended_state_area); 1016f0d4f30aSIngo Molnar 1017f0d4f30aSIngo Molnar /* 1018ffd3e504SThomas Gleixner * The ptrace buffer is in non-compacted XSAVE format. In 1019ffd3e504SThomas Gleixner * non-compacted format disabled features still occupy state space, 1020ffd3e504SThomas Gleixner * but there is no state to copy from in the compacted 1021ffd3e504SThomas Gleixner * init_fpstate. The gap tracking will zero these states. 1022f0d4f30aSIngo Molnar */ 1023ffd3e504SThomas Gleixner mask = xfeatures_mask_uabi(); 1024f0d4f30aSIngo Molnar 1025ffd3e504SThomas Gleixner for_each_extended_xfeature(i, mask) { 102696258950SThomas Gleixner /* 102796258950SThomas Gleixner * If there was a feature or alignment gap, zero the space 102896258950SThomas Gleixner * in the destination buffer. 102996258950SThomas Gleixner */ 103096258950SThomas Gleixner if (zerofrom < xstate_offsets[i]) 103196258950SThomas Gleixner membuf_zero(&to, xstate_offsets[i] - zerofrom); 103296258950SThomas Gleixner 1033e84ba47eSDave Hansen if (i == XFEATURE_PKRU) { 1034e84ba47eSDave Hansen struct pkru_state pkru = {0}; 1035e84ba47eSDave Hansen /* 1036e84ba47eSDave Hansen * PKRU is not necessarily up to date in the 1037ca834defSThomas Gleixner * XSAVE buffer. Use the provided value. 1038e84ba47eSDave Hansen */ 1039ca834defSThomas Gleixner pkru.pkru = pkru_val; 1040e84ba47eSDave Hansen membuf_write(&to, &pkru, sizeof(pkru)); 1041e84ba47eSDave Hansen } else { 104296258950SThomas Gleixner copy_feature(header.xfeatures & BIT_ULL(i), &to, 104396258950SThomas Gleixner __raw_xsave_addr(xsave, i), 104496258950SThomas Gleixner __raw_xsave_addr(xinit, i), 104596258950SThomas Gleixner xstate_sizes[i]); 1046e84ba47eSDave Hansen } 104796258950SThomas Gleixner /* 104896258950SThomas Gleixner * Keep track of the last copied state in the non-compacted 104996258950SThomas Gleixner * target buffer for gap zeroing. 105096258950SThomas Gleixner */ 105196258950SThomas Gleixner zerofrom = xstate_offsets[i] + xstate_sizes[i]; 1052f0d4f30aSIngo Molnar } 1053f0d4f30aSIngo Molnar 1054eb6f5172SThomas Gleixner out: 105596258950SThomas Gleixner if (to.left) 105696258950SThomas Gleixner membuf_zero(&to, to.left); 105791c3dba7SYu-cheng Yu } 105891c3dba7SYu-cheng Yu 1059ca834defSThomas Gleixner /** 1060ca834defSThomas Gleixner * copy_xstate_to_uabi_buf - Copy kernel saved xstate to a UABI buffer 1061ca834defSThomas Gleixner * @to: membuf descriptor 1062ca834defSThomas Gleixner * @tsk: The task from which to copy the saved xstate 1063ca834defSThomas Gleixner * @copy_mode: The requested copy mode 1064ca834defSThomas Gleixner * 1065ca834defSThomas Gleixner * Converts from kernel XSAVE or XSAVES compacted format to UABI conforming 1066ca834defSThomas Gleixner * format, i.e. from the kernel internal hardware dependent storage format 1067ca834defSThomas Gleixner * to the requested @mode. UABI XSTATE is always uncompacted! 1068ca834defSThomas Gleixner * 1069ca834defSThomas Gleixner * It supports partial copy but @to.pos always starts from zero. 1070ca834defSThomas Gleixner */ 1071ca834defSThomas Gleixner void copy_xstate_to_uabi_buf(struct membuf to, struct task_struct *tsk, 1072ca834defSThomas Gleixner enum xstate_copy_mode copy_mode) 1073ca834defSThomas Gleixner { 1074ca834defSThomas Gleixner __copy_xstate_to_uabi_buf(to, &tsk->thread.fpu.state.xsave, 1075ca834defSThomas Gleixner tsk->thread.pkru, copy_mode); 1076ca834defSThomas Gleixner } 1077ca834defSThomas Gleixner 1078522e9274SThomas Gleixner static int copy_from_buffer(void *dst, unsigned int offset, unsigned int size, 1079522e9274SThomas Gleixner const void *kbuf, const void __user *ubuf) 1080947f4947SThomas Gleixner { 1081522e9274SThomas Gleixner if (kbuf) { 1082522e9274SThomas Gleixner memcpy(dst, kbuf + offset, size); 1083522e9274SThomas Gleixner } else { 1084522e9274SThomas Gleixner if (copy_from_user(dst, ubuf + offset, size)) 1085522e9274SThomas Gleixner return -EFAULT; 1086947f4947SThomas Gleixner } 1087522e9274SThomas Gleixner return 0; 1088947f4947SThomas Gleixner } 1089947f4947SThomas Gleixner 1090522e9274SThomas Gleixner 1091522e9274SThomas Gleixner static int copy_uabi_to_xstate(struct xregs_state *xsave, const void *kbuf, 1092522e9274SThomas Gleixner const void __user *ubuf) 109379fecc2bSIngo Molnar { 109479fecc2bSIngo Molnar unsigned int offset, size; 109580d8ae86SEric Biggers struct xstate_header hdr; 1096522e9274SThomas Gleixner u64 mask; 1097522e9274SThomas Gleixner int i; 109879fecc2bSIngo Molnar 109979fecc2bSIngo Molnar offset = offsetof(struct xregs_state, header); 1100522e9274SThomas Gleixner if (copy_from_buffer(&hdr, offset, sizeof(hdr), kbuf, ubuf)) 1101522e9274SThomas Gleixner return -EFAULT; 110279fecc2bSIngo Molnar 11035274e6c1SFenghua Yu if (validate_user_xstate_header(&hdr)) 110479fecc2bSIngo Molnar return -EINVAL; 110579fecc2bSIngo Molnar 1106522e9274SThomas Gleixner /* Validate MXCSR when any of the related features is in use */ 1107522e9274SThomas Gleixner mask = XFEATURE_MASK_FP | XFEATURE_MASK_SSE | XFEATURE_MASK_YMM; 1108522e9274SThomas Gleixner if (hdr.xfeatures & mask) { 1109522e9274SThomas Gleixner u32 mxcsr[2]; 1110522e9274SThomas Gleixner 1111522e9274SThomas Gleixner offset = offsetof(struct fxregs_state, mxcsr); 1112522e9274SThomas Gleixner if (copy_from_buffer(mxcsr, offset, sizeof(mxcsr), kbuf, ubuf)) 1113522e9274SThomas Gleixner return -EFAULT; 1114522e9274SThomas Gleixner 1115522e9274SThomas Gleixner /* Reserved bits in MXCSR must be zero. */ 1116522e9274SThomas Gleixner if (mxcsr[0] & ~mxcsr_feature_mask) 1117947f4947SThomas Gleixner return -EINVAL; 1118947f4947SThomas Gleixner 1119522e9274SThomas Gleixner /* SSE and YMM require MXCSR even when FP is not in use. */ 1120522e9274SThomas Gleixner if (!(hdr.xfeatures & XFEATURE_MASK_FP)) { 1121522e9274SThomas Gleixner xsave->i387.mxcsr = mxcsr[0]; 1122522e9274SThomas Gleixner xsave->i387.mxcsr_mask = mxcsr[1]; 1123522e9274SThomas Gleixner } 1124522e9274SThomas Gleixner } 1125522e9274SThomas Gleixner 112679fecc2bSIngo Molnar for (i = 0; i < XFEATURE_MAX; i++) { 112779fecc2bSIngo Molnar u64 mask = ((u64)1 << i); 112879fecc2bSIngo Molnar 1129b89eda48SEric Biggers if (hdr.xfeatures & mask) { 113007baeb04SSebastian Andrzej Siewior void *dst = __raw_xsave_addr(xsave, i); 113179fecc2bSIngo Molnar 113279fecc2bSIngo Molnar offset = xstate_offsets[i]; 113379fecc2bSIngo Molnar size = xstate_sizes[i]; 113479fecc2bSIngo Molnar 1135522e9274SThomas Gleixner if (copy_from_buffer(dst, offset, size, kbuf, ubuf)) 1136522e9274SThomas Gleixner return -EFAULT; 113779fecc2bSIngo Molnar } 113879fecc2bSIngo Molnar } 113979fecc2bSIngo Molnar 114079fecc2bSIngo Molnar /* 114179fecc2bSIngo Molnar * The state that came in from userspace was user-state only. 114279fecc2bSIngo Molnar * Mask all the user states out of 'xfeatures': 114379fecc2bSIngo Molnar */ 11448ab22804SFenghua Yu xsave->header.xfeatures &= XFEATURE_MASK_SUPERVISOR_ALL; 114579fecc2bSIngo Molnar 114679fecc2bSIngo Molnar /* 114779fecc2bSIngo Molnar * Add back in the features that came in from userspace: 114879fecc2bSIngo Molnar */ 1149b89eda48SEric Biggers xsave->header.xfeatures |= hdr.xfeatures; 115079fecc2bSIngo Molnar 115179fecc2bSIngo Molnar return 0; 115279fecc2bSIngo Molnar } 115379fecc2bSIngo Molnar 115479fecc2bSIngo Molnar /* 1155522e9274SThomas Gleixner * Convert from a ptrace standard-format kernel buffer to kernel XSAVE[S] 1156ea4d6938SThomas Gleixner * format and copy to the target thread. Used by ptrace and KVM. 1157522e9274SThomas Gleixner */ 1158522e9274SThomas Gleixner int copy_uabi_from_kernel_to_xstate(struct xregs_state *xsave, const void *kbuf) 1159522e9274SThomas Gleixner { 1160522e9274SThomas Gleixner return copy_uabi_to_xstate(xsave, kbuf, NULL); 1161522e9274SThomas Gleixner } 1162522e9274SThomas Gleixner 1163522e9274SThomas Gleixner /* 116443be46e8SThomas Gleixner * Convert from a sigreturn standard-format user-space buffer to kernel 116543be46e8SThomas Gleixner * XSAVE[S] format and copy to the target thread. This is called from the 116643be46e8SThomas Gleixner * sigreturn() and rt_sigreturn() system calls. 116791c3dba7SYu-cheng Yu */ 11681cc34413SThomas Gleixner int copy_sigframe_from_user_to_xstate(struct xregs_state *xsave, 11691cc34413SThomas Gleixner const void __user *ubuf) 117091c3dba7SYu-cheng Yu { 1171522e9274SThomas Gleixner return copy_uabi_to_xstate(xsave, NULL, ubuf); 117284594296SDave Hansen } 11730c608dadSAubrey Li 1174f5daf836SThomas Gleixner static bool validate_independent_components(u64 mask) 117550f408d9SKan Liang { 1176a75c5289SThomas Gleixner u64 xchk; 117750f408d9SKan Liang 1178a75c5289SThomas Gleixner if (WARN_ON_FPU(!cpu_feature_enabled(X86_FEATURE_XSAVES))) 1179a75c5289SThomas Gleixner return false; 1180f5daf836SThomas Gleixner 1181a75c5289SThomas Gleixner xchk = ~xfeatures_mask_independent(); 118250f408d9SKan Liang 1183a75c5289SThomas Gleixner if (WARN_ON_ONCE(!mask || mask & xchk)) 1184a75c5289SThomas Gleixner return false; 118550f408d9SKan Liang 1186a75c5289SThomas Gleixner return true; 118750f408d9SKan Liang } 118850f408d9SKan Liang 118950f408d9SKan Liang /** 1190a75c5289SThomas Gleixner * xsaves - Save selected components to a kernel xstate buffer 1191a75c5289SThomas Gleixner * @xstate: Pointer to the buffer 1192a75c5289SThomas Gleixner * @mask: Feature mask to select the components to save 119350f408d9SKan Liang * 1194a75c5289SThomas Gleixner * The @xstate buffer must be 64 byte aligned and correctly initialized as 1195a75c5289SThomas Gleixner * XSAVES does not write the full xstate header. Before first use the 1196a75c5289SThomas Gleixner * buffer should be zeroed otherwise a consecutive XRSTORS from that buffer 1197a75c5289SThomas Gleixner * can #GP. 119850f408d9SKan Liang * 1199f5daf836SThomas Gleixner * The feature mask must be a subset of the independent features. 120050f408d9SKan Liang */ 1201a75c5289SThomas Gleixner void xsaves(struct xregs_state *xstate, u64 mask) 120250f408d9SKan Liang { 120350f408d9SKan Liang int err; 120450f408d9SKan Liang 1205f5daf836SThomas Gleixner if (!validate_independent_components(mask)) 120650f408d9SKan Liang return; 120750f408d9SKan Liang 1208a75c5289SThomas Gleixner XSTATE_OP(XSAVES, xstate, (u32)mask, (u32)(mask >> 32), err); 1209a75c5289SThomas Gleixner WARN_ON_ONCE(err); 1210a75c5289SThomas Gleixner } 1211a75c5289SThomas Gleixner 1212a75c5289SThomas Gleixner /** 1213a75c5289SThomas Gleixner * xrstors - Restore selected components from a kernel xstate buffer 1214a75c5289SThomas Gleixner * @xstate: Pointer to the buffer 1215a75c5289SThomas Gleixner * @mask: Feature mask to select the components to restore 1216a75c5289SThomas Gleixner * 1217a75c5289SThomas Gleixner * The @xstate buffer must be 64 byte aligned and correctly initialized 1218a75c5289SThomas Gleixner * otherwise XRSTORS from that buffer can #GP. 1219a75c5289SThomas Gleixner * 1220a75c5289SThomas Gleixner * Proper usage is to restore the state which was saved with 1221a75c5289SThomas Gleixner * xsaves() into @xstate. 1222a75c5289SThomas Gleixner * 1223f5daf836SThomas Gleixner * The feature mask must be a subset of the independent features. 1224a75c5289SThomas Gleixner */ 1225a75c5289SThomas Gleixner void xrstors(struct xregs_state *xstate, u64 mask) 1226a75c5289SThomas Gleixner { 1227a75c5289SThomas Gleixner int err; 1228a75c5289SThomas Gleixner 1229f5daf836SThomas Gleixner if (!validate_independent_components(mask)) 123050f408d9SKan Liang return; 123150f408d9SKan Liang 1232a75c5289SThomas Gleixner XSTATE_OP(XRSTORS, xstate, (u32)mask, (u32)(mask >> 32), err); 1233a75c5289SThomas Gleixner WARN_ON_ONCE(err); 123450f408d9SKan Liang } 123550f408d9SKan Liang 12360c608dadSAubrey Li #ifdef CONFIG_PROC_PID_ARCH_STATUS 12370c608dadSAubrey Li /* 12380c608dadSAubrey Li * Report the amount of time elapsed in millisecond since last AVX512 12390c608dadSAubrey Li * use in the task. 12400c608dadSAubrey Li */ 12410c608dadSAubrey Li static void avx512_status(struct seq_file *m, struct task_struct *task) 12420c608dadSAubrey Li { 12430c608dadSAubrey Li unsigned long timestamp = READ_ONCE(task->thread.fpu.avx512_timestamp); 12440c608dadSAubrey Li long delta; 12450c608dadSAubrey Li 12460c608dadSAubrey Li if (!timestamp) { 12470c608dadSAubrey Li /* 12480c608dadSAubrey Li * Report -1 if no AVX512 usage 12490c608dadSAubrey Li */ 12500c608dadSAubrey Li delta = -1; 12510c608dadSAubrey Li } else { 12520c608dadSAubrey Li delta = (long)(jiffies - timestamp); 12530c608dadSAubrey Li /* 12540c608dadSAubrey Li * Cap to LONG_MAX if time difference > LONG_MAX 12550c608dadSAubrey Li */ 12560c608dadSAubrey Li if (delta < 0) 12570c608dadSAubrey Li delta = LONG_MAX; 12580c608dadSAubrey Li delta = jiffies_to_msecs(delta); 12590c608dadSAubrey Li } 12600c608dadSAubrey Li 12610c608dadSAubrey Li seq_put_decimal_ll(m, "AVX512_elapsed_ms:\t", delta); 12620c608dadSAubrey Li seq_putc(m, '\n'); 12630c608dadSAubrey Li } 12640c608dadSAubrey Li 12650c608dadSAubrey Li /* 12660c608dadSAubrey Li * Report architecture specific information 12670c608dadSAubrey Li */ 12680c608dadSAubrey Li int proc_pid_arch_status(struct seq_file *m, struct pid_namespace *ns, 12690c608dadSAubrey Li struct pid *pid, struct task_struct *task) 12700c608dadSAubrey Li { 12710c608dadSAubrey Li /* 12720c608dadSAubrey Li * Report AVX512 state if the processor and build option supported. 12730c608dadSAubrey Li */ 12740c608dadSAubrey Li if (cpu_feature_enabled(X86_FEATURE_AVX512F)) 12750c608dadSAubrey Li avx512_status(m, task); 12760c608dadSAubrey Li 12770c608dadSAubrey Li return 0; 12780c608dadSAubrey Li } 12790c608dadSAubrey Li #endif /* CONFIG_PROC_PID_ARCH_STATUS */ 1280