1 /* SPDX-License-Identifier: GPL-2.0 */ 2 3 #ifndef _ASM_X86_NOSPEC_BRANCH_H_ 4 #define _ASM_X86_NOSPEC_BRANCH_H_ 5 6 #include <linux/static_key.h> 7 #include <linux/objtool.h> 8 #include <linux/linkage.h> 9 10 #include <asm/alternative.h> 11 #include <asm/cpufeatures.h> 12 #include <asm/msr-index.h> 13 #include <asm/unwind_hints.h> 14 #include <asm/percpu.h> 15 16 #define RETPOLINE_THUNK_SIZE 32 17 18 /* 19 * Fill the CPU return stack buffer. 20 * 21 * Each entry in the RSB, if used for a speculative 'ret', contains an 22 * infinite 'pause; lfence; jmp' loop to capture speculative execution. 23 * 24 * This is required in various cases for retpoline and IBRS-based 25 * mitigations for the Spectre variant 2 vulnerability. Sometimes to 26 * eliminate potentially bogus entries from the RSB, and sometimes 27 * purely to ensure that it doesn't get empty, which on some CPUs would 28 * allow predictions from other (unwanted!) sources to be used. 29 * 30 * We define a CPP macro such that it can be used from both .S files and 31 * inline assembly. It's possible to do a .macro and then include that 32 * from C via asm(".include <asm/nospec-branch.h>") but let's not go there. 33 */ 34 35 #define RSB_CLEAR_LOOPS 32 /* To forcibly overwrite all entries */ 36 37 /* 38 * Google experimented with loop-unrolling and this turned out to be 39 * the optimal version - two calls, each with their own speculation 40 * trap should their return address end up getting used, in a loop. 41 */ 42 #define __FILL_RETURN_BUFFER(reg, nr, sp) \ 43 mov $(nr/2), reg; \ 44 771: \ 45 ANNOTATE_INTRA_FUNCTION_CALL; \ 46 call 772f; \ 47 773: /* speculation trap */ \ 48 UNWIND_HINT_EMPTY; \ 49 pause; \ 50 lfence; \ 51 jmp 773b; \ 52 772: \ 53 ANNOTATE_INTRA_FUNCTION_CALL; \ 54 call 774f; \ 55 775: /* speculation trap */ \ 56 UNWIND_HINT_EMPTY; \ 57 pause; \ 58 lfence; \ 59 jmp 775b; \ 60 774: \ 61 add $(BITS_PER_LONG/8) * 2, sp; \ 62 dec reg; \ 63 jnz 771b; 64 65 #ifdef __ASSEMBLY__ 66 67 /* 68 * This should be used immediately before an indirect jump/call. It tells 69 * objtool the subsequent indirect jump/call is vouched safe for retpoline 70 * builds. 71 */ 72 .macro ANNOTATE_RETPOLINE_SAFE 73 .Lannotate_\@: 74 .pushsection .discard.retpoline_safe 75 _ASM_PTR .Lannotate_\@ 76 .popsection 77 .endm 78 79 /* 80 * (ab)use RETPOLINE_SAFE on RET to annotate away 'bare' RET instructions 81 * vs RETBleed validation. 82 */ 83 #define ANNOTATE_UNRET_SAFE ANNOTATE_RETPOLINE_SAFE 84 85 /* 86 * Abuse ANNOTATE_RETPOLINE_SAFE on a NOP to indicate UNRET_END, should 87 * eventually turn into it's own annotation. 88 */ 89 .macro ANNOTATE_UNRET_END 90 #ifdef CONFIG_DEBUG_ENTRY 91 ANNOTATE_RETPOLINE_SAFE 92 nop 93 #endif 94 .endm 95 96 /* 97 * JMP_NOSPEC and CALL_NOSPEC macros can be used instead of a simple 98 * indirect jmp/call which may be susceptible to the Spectre variant 2 99 * attack. 100 */ 101 .macro JMP_NOSPEC reg:req 102 #ifdef CONFIG_RETPOLINE 103 ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; jmp *%\reg), \ 104 __stringify(jmp __x86_indirect_thunk_\reg), X86_FEATURE_RETPOLINE, \ 105 __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; jmp *%\reg), X86_FEATURE_RETPOLINE_LFENCE 106 #else 107 jmp *%\reg 108 #endif 109 .endm 110 111 .macro CALL_NOSPEC reg:req 112 #ifdef CONFIG_RETPOLINE 113 ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; call *%\reg), \ 114 __stringify(call __x86_indirect_thunk_\reg), X86_FEATURE_RETPOLINE, \ 115 __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; call *%\reg), X86_FEATURE_RETPOLINE_LFENCE 116 #else 117 call *%\reg 118 #endif 119 .endm 120 121 /* 122 * A simpler FILL_RETURN_BUFFER macro. Don't make people use the CPP 123 * monstrosity above, manually. 124 */ 125 .macro FILL_RETURN_BUFFER reg:req nr:req ftr:req 126 ALTERNATIVE "jmp .Lskip_rsb_\@", "", \ftr 127 __FILL_RETURN_BUFFER(\reg,\nr,%_ASM_SP) 128 .Lskip_rsb_\@: 129 .endm 130 131 #ifdef CONFIG_CPU_UNRET_ENTRY 132 #define CALL_ZEN_UNTRAIN_RET "call zen_untrain_ret" 133 #else 134 #define CALL_ZEN_UNTRAIN_RET "" 135 #endif 136 137 /* 138 * Mitigate RETBleed for AMD/Hygon Zen uarch. Requires KERNEL CR3 because the 139 * return thunk isn't mapped into the userspace tables (then again, AMD 140 * typically has NO_MELTDOWN). 141 * 142 * While zen_untrain_ret() doesn't clobber anything but requires stack, 143 * entry_ibpb() will clobber AX, CX, DX. 144 * 145 * As such, this must be placed after every *SWITCH_TO_KERNEL_CR3 at a point 146 * where we have a stack but before any RET instruction. 147 */ 148 .macro UNTRAIN_RET 149 #if defined(CONFIG_CPU_UNRET_ENTRY) || defined(CONFIG_CPU_IBPB_ENTRY) 150 ANNOTATE_UNRET_END 151 ALTERNATIVE_2 "", \ 152 CALL_ZEN_UNTRAIN_RET, X86_FEATURE_UNRET, \ 153 "call entry_ibpb", X86_FEATURE_ENTRY_IBPB 154 #endif 155 .endm 156 157 #else /* __ASSEMBLY__ */ 158 159 #define ANNOTATE_RETPOLINE_SAFE \ 160 "999:\n\t" \ 161 ".pushsection .discard.retpoline_safe\n\t" \ 162 _ASM_PTR " 999b\n\t" \ 163 ".popsection\n\t" 164 165 typedef u8 retpoline_thunk_t[RETPOLINE_THUNK_SIZE]; 166 extern retpoline_thunk_t __x86_indirect_thunk_array[]; 167 168 extern void __x86_return_thunk(void); 169 extern void zen_untrain_ret(void); 170 extern void entry_ibpb(void); 171 172 #ifdef CONFIG_RETPOLINE 173 174 #define GEN(reg) \ 175 extern retpoline_thunk_t __x86_indirect_thunk_ ## reg; 176 #include <asm/GEN-for-each-reg.h> 177 #undef GEN 178 179 #ifdef CONFIG_X86_64 180 181 /* 182 * Inline asm uses the %V modifier which is only in newer GCC 183 * which is ensured when CONFIG_RETPOLINE is defined. 184 */ 185 # define CALL_NOSPEC \ 186 ALTERNATIVE_2( \ 187 ANNOTATE_RETPOLINE_SAFE \ 188 "call *%[thunk_target]\n", \ 189 "call __x86_indirect_thunk_%V[thunk_target]\n", \ 190 X86_FEATURE_RETPOLINE, \ 191 "lfence;\n" \ 192 ANNOTATE_RETPOLINE_SAFE \ 193 "call *%[thunk_target]\n", \ 194 X86_FEATURE_RETPOLINE_LFENCE) 195 196 # define THUNK_TARGET(addr) [thunk_target] "r" (addr) 197 198 #else /* CONFIG_X86_32 */ 199 /* 200 * For i386 we use the original ret-equivalent retpoline, because 201 * otherwise we'll run out of registers. We don't care about CET 202 * here, anyway. 203 */ 204 # define CALL_NOSPEC \ 205 ALTERNATIVE_2( \ 206 ANNOTATE_RETPOLINE_SAFE \ 207 "call *%[thunk_target]\n", \ 208 " jmp 904f;\n" \ 209 " .align 16\n" \ 210 "901: call 903f;\n" \ 211 "902: pause;\n" \ 212 " lfence;\n" \ 213 " jmp 902b;\n" \ 214 " .align 16\n" \ 215 "903: lea 4(%%esp), %%esp;\n" \ 216 " pushl %[thunk_target];\n" \ 217 " ret;\n" \ 218 " .align 16\n" \ 219 "904: call 901b;\n", \ 220 X86_FEATURE_RETPOLINE, \ 221 "lfence;\n" \ 222 ANNOTATE_RETPOLINE_SAFE \ 223 "call *%[thunk_target]\n", \ 224 X86_FEATURE_RETPOLINE_LFENCE) 225 226 # define THUNK_TARGET(addr) [thunk_target] "rm" (addr) 227 #endif 228 #else /* No retpoline for C / inline asm */ 229 # define CALL_NOSPEC "call *%[thunk_target]\n" 230 # define THUNK_TARGET(addr) [thunk_target] "rm" (addr) 231 #endif 232 233 /* The Spectre V2 mitigation variants */ 234 enum spectre_v2_mitigation { 235 SPECTRE_V2_NONE, 236 SPECTRE_V2_RETPOLINE, 237 SPECTRE_V2_LFENCE, 238 SPECTRE_V2_EIBRS, 239 SPECTRE_V2_EIBRS_RETPOLINE, 240 SPECTRE_V2_EIBRS_LFENCE, 241 SPECTRE_V2_IBRS, 242 }; 243 244 /* The indirect branch speculation control variants */ 245 enum spectre_v2_user_mitigation { 246 SPECTRE_V2_USER_NONE, 247 SPECTRE_V2_USER_STRICT, 248 SPECTRE_V2_USER_STRICT_PREFERRED, 249 SPECTRE_V2_USER_PRCTL, 250 SPECTRE_V2_USER_SECCOMP, 251 }; 252 253 /* The Speculative Store Bypass disable variants */ 254 enum ssb_mitigation { 255 SPEC_STORE_BYPASS_NONE, 256 SPEC_STORE_BYPASS_DISABLE, 257 SPEC_STORE_BYPASS_PRCTL, 258 SPEC_STORE_BYPASS_SECCOMP, 259 }; 260 261 extern char __indirect_thunk_start[]; 262 extern char __indirect_thunk_end[]; 263 264 static __always_inline 265 void alternative_msr_write(unsigned int msr, u64 val, unsigned int feature) 266 { 267 asm volatile(ALTERNATIVE("", "wrmsr", %c[feature]) 268 : : "c" (msr), 269 "a" ((u32)val), 270 "d" ((u32)(val >> 32)), 271 [feature] "i" (feature) 272 : "memory"); 273 } 274 275 static inline void indirect_branch_prediction_barrier(void) 276 { 277 u64 val = PRED_CMD_IBPB; 278 279 alternative_msr_write(MSR_IA32_PRED_CMD, val, X86_FEATURE_USE_IBPB); 280 } 281 282 /* The Intel SPEC CTRL MSR base value cache */ 283 extern u64 x86_spec_ctrl_base; 284 DECLARE_PER_CPU(u64, x86_spec_ctrl_current); 285 extern void write_spec_ctrl_current(u64 val, bool force); 286 extern u64 spec_ctrl_current(void); 287 288 /* 289 * With retpoline, we must use IBRS to restrict branch prediction 290 * before calling into firmware. 291 * 292 * (Implemented as CPP macros due to header hell.) 293 */ 294 #define firmware_restrict_branch_speculation_start() \ 295 do { \ 296 preempt_disable(); \ 297 alternative_msr_write(MSR_IA32_SPEC_CTRL, \ 298 spec_ctrl_current() | SPEC_CTRL_IBRS, \ 299 X86_FEATURE_USE_IBRS_FW); \ 300 alternative_msr_write(MSR_IA32_PRED_CMD, PRED_CMD_IBPB, \ 301 X86_FEATURE_USE_IBPB_FW); \ 302 } while (0) 303 304 #define firmware_restrict_branch_speculation_end() \ 305 do { \ 306 alternative_msr_write(MSR_IA32_SPEC_CTRL, \ 307 spec_ctrl_current(), \ 308 X86_FEATURE_USE_IBRS_FW); \ 309 preempt_enable(); \ 310 } while (0) 311 312 DECLARE_STATIC_KEY_FALSE(switch_to_cond_stibp); 313 DECLARE_STATIC_KEY_FALSE(switch_mm_cond_ibpb); 314 DECLARE_STATIC_KEY_FALSE(switch_mm_always_ibpb); 315 316 DECLARE_STATIC_KEY_FALSE(mds_user_clear); 317 DECLARE_STATIC_KEY_FALSE(mds_idle_clear); 318 319 DECLARE_STATIC_KEY_FALSE(switch_mm_cond_l1d_flush); 320 321 DECLARE_STATIC_KEY_FALSE(mmio_stale_data_clear); 322 323 #include <asm/segment.h> 324 325 /** 326 * mds_clear_cpu_buffers - Mitigation for MDS and TAA vulnerability 327 * 328 * This uses the otherwise unused and obsolete VERW instruction in 329 * combination with microcode which triggers a CPU buffer flush when the 330 * instruction is executed. 331 */ 332 static __always_inline void mds_clear_cpu_buffers(void) 333 { 334 static const u16 ds = __KERNEL_DS; 335 336 /* 337 * Has to be the memory-operand variant because only that 338 * guarantees the CPU buffer flush functionality according to 339 * documentation. The register-operand variant does not. 340 * Works with any segment selector, but a valid writable 341 * data segment is the fastest variant. 342 * 343 * "cc" clobber is required because VERW modifies ZF. 344 */ 345 asm volatile("verw %[ds]" : : [ds] "m" (ds) : "cc"); 346 } 347 348 /** 349 * mds_user_clear_cpu_buffers - Mitigation for MDS and TAA vulnerability 350 * 351 * Clear CPU buffers if the corresponding static key is enabled 352 */ 353 static __always_inline void mds_user_clear_cpu_buffers(void) 354 { 355 if (static_branch_likely(&mds_user_clear)) 356 mds_clear_cpu_buffers(); 357 } 358 359 /** 360 * mds_idle_clear_cpu_buffers - Mitigation for MDS vulnerability 361 * 362 * Clear CPU buffers if the corresponding static key is enabled 363 */ 364 static inline void mds_idle_clear_cpu_buffers(void) 365 { 366 if (static_branch_likely(&mds_idle_clear)) 367 mds_clear_cpu_buffers(); 368 } 369 370 #endif /* __ASSEMBLY__ */ 371 372 #endif /* _ASM_X86_NOSPEC_BRANCH_H_ */ 373