1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * x86 FPU boot time init code: 4 */ 5 #include <asm/fpu/internal.h> 6 #include <asm/tlbflush.h> 7 #include <asm/setup.h> 8 #include <asm/cmdline.h> 9 10 #include <linux/sched.h> 11 #include <linux/sched/task.h> 12 #include <linux/init.h> 13 14 /* 15 * Initialize the registers found in all CPUs, CR0 and CR4: 16 */ 17 static void fpu__init_cpu_generic(void) 18 { 19 unsigned long cr0; 20 unsigned long cr4_mask = 0; 21 22 if (boot_cpu_has(X86_FEATURE_FXSR)) 23 cr4_mask |= X86_CR4_OSFXSR; 24 if (boot_cpu_has(X86_FEATURE_XMM)) 25 cr4_mask |= X86_CR4_OSXMMEXCPT; 26 if (cr4_mask) 27 cr4_set_bits(cr4_mask); 28 29 cr0 = read_cr0(); 30 cr0 &= ~(X86_CR0_TS|X86_CR0_EM); /* clear TS and EM */ 31 if (!boot_cpu_has(X86_FEATURE_FPU)) 32 cr0 |= X86_CR0_EM; 33 write_cr0(cr0); 34 35 /* Flush out any pending x87 state: */ 36 #ifdef CONFIG_MATH_EMULATION 37 if (!boot_cpu_has(X86_FEATURE_FPU)) 38 fpstate_init_soft(¤t->thread.fpu.state.soft); 39 else 40 #endif 41 asm volatile ("fninit"); 42 } 43 44 /* 45 * Enable all supported FPU features. Called when a CPU is brought online: 46 */ 47 void fpu__init_cpu(void) 48 { 49 fpu__init_cpu_generic(); 50 fpu__init_cpu_xstate(); 51 } 52 53 static bool fpu__probe_without_cpuid(void) 54 { 55 unsigned long cr0; 56 u16 fsw, fcw; 57 58 fsw = fcw = 0xffff; 59 60 cr0 = read_cr0(); 61 cr0 &= ~(X86_CR0_TS | X86_CR0_EM); 62 write_cr0(cr0); 63 64 asm volatile("fninit ; fnstsw %0 ; fnstcw %1" : "+m" (fsw), "+m" (fcw)); 65 66 pr_info("x86/fpu: Probing for FPU: FSW=0x%04hx FCW=0x%04hx\n", fsw, fcw); 67 68 return fsw == 0 && (fcw & 0x103f) == 0x003f; 69 } 70 71 static void fpu__init_system_early_generic(struct cpuinfo_x86 *c) 72 { 73 if (!boot_cpu_has(X86_FEATURE_CPUID) && 74 !test_bit(X86_FEATURE_FPU, (unsigned long *)cpu_caps_cleared)) { 75 if (fpu__probe_without_cpuid()) 76 setup_force_cpu_cap(X86_FEATURE_FPU); 77 else 78 setup_clear_cpu_cap(X86_FEATURE_FPU); 79 } 80 81 #ifndef CONFIG_MATH_EMULATION 82 if (!test_cpu_cap(&boot_cpu_data, X86_FEATURE_FPU)) { 83 pr_emerg("x86/fpu: Giving up, no FPU found and no math emulation present\n"); 84 for (;;) 85 asm volatile("hlt"); 86 } 87 #endif 88 } 89 90 /* 91 * Boot time FPU feature detection code: 92 */ 93 unsigned int mxcsr_feature_mask __read_mostly = 0xffffffffu; 94 EXPORT_SYMBOL_GPL(mxcsr_feature_mask); 95 96 static void __init fpu__init_system_mxcsr(void) 97 { 98 unsigned int mask = 0; 99 100 if (boot_cpu_has(X86_FEATURE_FXSR)) { 101 /* Static because GCC does not get 16-byte stack alignment right: */ 102 static struct fxregs_state fxregs __initdata; 103 104 asm volatile("fxsave %0" : "+m" (fxregs)); 105 106 mask = fxregs.mxcsr_mask; 107 108 /* 109 * If zero then use the default features mask, 110 * which has all features set, except the 111 * denormals-are-zero feature bit: 112 */ 113 if (mask == 0) 114 mask = 0x0000ffbf; 115 } 116 mxcsr_feature_mask &= mask; 117 } 118 119 /* 120 * Once per bootup FPU initialization sequences that will run on most x86 CPUs: 121 */ 122 static void __init fpu__init_system_generic(void) 123 { 124 /* 125 * Set up the legacy init FPU context. (xstate init might overwrite this 126 * with a more modern format, if the CPU supports it.) 127 */ 128 fpstate_init(&init_fpstate); 129 130 fpu__init_system_mxcsr(); 131 } 132 133 /* 134 * Size of the FPU context state. All tasks in the system use the 135 * same context size, regardless of what portion they use. 136 * This is inherent to the XSAVE architecture which puts all state 137 * components into a single, continuous memory block: 138 */ 139 unsigned int fpu_kernel_xstate_size; 140 EXPORT_SYMBOL_GPL(fpu_kernel_xstate_size); 141 142 /* Get alignment of the TYPE. */ 143 #define TYPE_ALIGN(TYPE) offsetof(struct { char x; TYPE test; }, test) 144 145 /* 146 * Enforce that 'MEMBER' is the last field of 'TYPE'. 147 * 148 * Align the computed size with alignment of the TYPE, 149 * because that's how C aligns structs. 150 */ 151 #define CHECK_MEMBER_AT_END_OF(TYPE, MEMBER) \ 152 BUILD_BUG_ON(sizeof(TYPE) != ALIGN(offsetofend(TYPE, MEMBER), \ 153 TYPE_ALIGN(TYPE))) 154 155 /* 156 * We append the 'struct fpu' to the task_struct: 157 */ 158 static void __init fpu__init_task_struct_size(void) 159 { 160 int task_size = sizeof(struct task_struct); 161 162 /* 163 * Subtract off the static size of the register state. 164 * It potentially has a bunch of padding. 165 */ 166 task_size -= sizeof(((struct task_struct *)0)->thread.fpu.state); 167 168 /* 169 * Add back the dynamically-calculated register state 170 * size. 171 */ 172 task_size += fpu_kernel_xstate_size; 173 174 /* 175 * We dynamically size 'struct fpu', so we require that 176 * it be at the end of 'thread_struct' and that 177 * 'thread_struct' be at the end of 'task_struct'. If 178 * you hit a compile error here, check the structure to 179 * see if something got added to the end. 180 */ 181 CHECK_MEMBER_AT_END_OF(struct fpu, state); 182 CHECK_MEMBER_AT_END_OF(struct thread_struct, fpu); 183 CHECK_MEMBER_AT_END_OF(struct task_struct, thread); 184 185 arch_task_struct_size = task_size; 186 } 187 188 /* 189 * Set up the user and kernel xstate sizes based on the legacy FPU context size. 190 * 191 * We set this up first, and later it will be overwritten by 192 * fpu__init_system_xstate() if the CPU knows about xstates. 193 */ 194 static void __init fpu__init_system_xstate_size_legacy(void) 195 { 196 static int on_boot_cpu __initdata = 1; 197 198 WARN_ON_FPU(!on_boot_cpu); 199 on_boot_cpu = 0; 200 201 /* 202 * Note that xstate sizes might be overwritten later during 203 * fpu__init_system_xstate(). 204 */ 205 206 if (!boot_cpu_has(X86_FEATURE_FPU)) { 207 fpu_kernel_xstate_size = sizeof(struct swregs_state); 208 } else { 209 if (boot_cpu_has(X86_FEATURE_FXSR)) 210 fpu_kernel_xstate_size = 211 sizeof(struct fxregs_state); 212 else 213 fpu_kernel_xstate_size = 214 sizeof(struct fregs_state); 215 } 216 217 fpu_user_xstate_size = fpu_kernel_xstate_size; 218 } 219 220 /* 221 * Find supported xfeatures based on cpu features and command-line input. 222 * This must be called after fpu__init_parse_early_param() is called and 223 * xfeatures_mask is enumerated. 224 */ 225 u64 __init fpu__get_supported_xfeatures_mask(void) 226 { 227 return XFEATURE_MASK_USER_SUPPORTED | 228 XFEATURE_MASK_SUPERVISOR_SUPPORTED; 229 } 230 231 /* Legacy code to initialize eager fpu mode. */ 232 static void __init fpu__init_system_ctx_switch(void) 233 { 234 static bool on_boot_cpu __initdata = 1; 235 236 WARN_ON_FPU(!on_boot_cpu); 237 on_boot_cpu = 0; 238 } 239 240 /* 241 * We parse fpu parameters early because fpu__init_system() is executed 242 * before parse_early_param(). 243 */ 244 static void __init fpu__init_parse_early_param(void) 245 { 246 char arg[32]; 247 char *argptr = arg; 248 int bit; 249 250 #ifdef CONFIG_X86_32 251 if (cmdline_find_option_bool(boot_command_line, "no387")) 252 #ifdef CONFIG_MATH_EMULATION 253 setup_clear_cpu_cap(X86_FEATURE_FPU); 254 #else 255 pr_err("Option 'no387' required CONFIG_MATH_EMULATION enabled.\n"); 256 #endif 257 258 if (cmdline_find_option_bool(boot_command_line, "nofxsr")) 259 setup_clear_cpu_cap(X86_FEATURE_FXSR); 260 #endif 261 262 if (cmdline_find_option_bool(boot_command_line, "noxsave")) 263 setup_clear_cpu_cap(X86_FEATURE_XSAVE); 264 265 if (cmdline_find_option_bool(boot_command_line, "noxsaveopt")) 266 setup_clear_cpu_cap(X86_FEATURE_XSAVEOPT); 267 268 if (cmdline_find_option_bool(boot_command_line, "noxsaves")) 269 setup_clear_cpu_cap(X86_FEATURE_XSAVES); 270 271 if (cmdline_find_option(boot_command_line, "clearcpuid", arg, 272 sizeof(arg)) && 273 get_option(&argptr, &bit) && 274 bit >= 0 && 275 bit < NCAPINTS * 32) 276 setup_clear_cpu_cap(bit); 277 } 278 279 /* 280 * Called on the boot CPU once per system bootup, to set up the initial 281 * FPU state that is later cloned into all processes: 282 */ 283 void __init fpu__init_system(struct cpuinfo_x86 *c) 284 { 285 fpu__init_parse_early_param(); 286 fpu__init_system_early_generic(c); 287 288 /* 289 * The FPU has to be operational for some of the 290 * later FPU init activities: 291 */ 292 fpu__init_cpu(); 293 294 fpu__init_system_generic(); 295 fpu__init_system_xstate_size_legacy(); 296 fpu__init_system_xstate(); 297 fpu__init_task_struct_size(); 298 299 fpu__init_system_ctx_switch(); 300 } 301