1 /* SPDX-License-Identifier: GPL-2.0 */ 2 3 #ifndef _ASM_X86_NOSPEC_BRANCH_H_ 4 #define _ASM_X86_NOSPEC_BRANCH_H_ 5 6 #include <linux/static_key.h> 7 8 #include <asm/alternative.h> 9 #include <asm/alternative-asm.h> 10 #include <asm/cpufeatures.h> 11 #include <asm/msr-index.h> 12 13 /* 14 * This should be used immediately before a retpoline alternative. It tells 15 * objtool where the retpolines are so that it can make sense of the control 16 * flow by just reading the original instruction(s) and ignoring the 17 * alternatives. 18 */ 19 #define ANNOTATE_NOSPEC_ALTERNATIVE \ 20 ANNOTATE_IGNORE_ALTERNATIVE 21 22 /* 23 * Fill the CPU return stack buffer. 24 * 25 * Each entry in the RSB, if used for a speculative 'ret', contains an 26 * infinite 'pause; lfence; jmp' loop to capture speculative execution. 27 * 28 * This is required in various cases for retpoline and IBRS-based 29 * mitigations for the Spectre variant 2 vulnerability. Sometimes to 30 * eliminate potentially bogus entries from the RSB, and sometimes 31 * purely to ensure that it doesn't get empty, which on some CPUs would 32 * allow predictions from other (unwanted!) sources to be used. 33 * 34 * We define a CPP macro such that it can be used from both .S files and 35 * inline assembly. It's possible to do a .macro and then include that 36 * from C via asm(".include <asm/nospec-branch.h>") but let's not go there. 37 */ 38 39 #define RSB_CLEAR_LOOPS 32 /* To forcibly overwrite all entries */ 40 41 /* 42 * Google experimented with loop-unrolling and this turned out to be 43 * the optimal version — two calls, each with their own speculation 44 * trap should their return address end up getting used, in a loop. 45 */ 46 #define __FILL_RETURN_BUFFER(reg, nr, sp) \ 47 mov $(nr/2), reg; \ 48 771: \ 49 call 772f; \ 50 773: /* speculation trap */ \ 51 pause; \ 52 lfence; \ 53 jmp 773b; \ 54 772: \ 55 call 774f; \ 56 775: /* speculation trap */ \ 57 pause; \ 58 lfence; \ 59 jmp 775b; \ 60 774: \ 61 dec reg; \ 62 jnz 771b; \ 63 add $(BITS_PER_LONG/8) * nr, sp; 64 65 #ifdef __ASSEMBLY__ 66 67 /* 68 * This should be used immediately before an indirect jump/call. It tells 69 * objtool the subsequent indirect jump/call is vouched safe for retpoline 70 * builds. 71 */ 72 .macro ANNOTATE_RETPOLINE_SAFE 73 .Lannotate_\@: 74 .pushsection .discard.retpoline_safe 75 _ASM_PTR .Lannotate_\@ 76 .popsection 77 .endm 78 79 /* 80 * These are the bare retpoline primitives for indirect jmp and call. 81 * Do not use these directly; they only exist to make the ALTERNATIVE 82 * invocation below less ugly. 83 */ 84 .macro RETPOLINE_JMP reg:req 85 call .Ldo_rop_\@ 86 .Lspec_trap_\@: 87 pause 88 lfence 89 jmp .Lspec_trap_\@ 90 .Ldo_rop_\@: 91 mov \reg, (%_ASM_SP) 92 ret 93 .endm 94 95 /* 96 * This is a wrapper around RETPOLINE_JMP so the called function in reg 97 * returns to the instruction after the macro. 98 */ 99 .macro RETPOLINE_CALL reg:req 100 jmp .Ldo_call_\@ 101 .Ldo_retpoline_jmp_\@: 102 RETPOLINE_JMP \reg 103 .Ldo_call_\@: 104 call .Ldo_retpoline_jmp_\@ 105 .endm 106 107 /* 108 * JMP_NOSPEC and CALL_NOSPEC macros can be used instead of a simple 109 * indirect jmp/call which may be susceptible to the Spectre variant 2 110 * attack. 111 */ 112 .macro JMP_NOSPEC reg:req 113 #ifdef CONFIG_RETPOLINE 114 ANNOTATE_NOSPEC_ALTERNATIVE 115 ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; jmp *\reg), \ 116 __stringify(RETPOLINE_JMP \reg), X86_FEATURE_RETPOLINE, \ 117 __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; jmp *\reg), X86_FEATURE_RETPOLINE_AMD 118 #else 119 jmp *\reg 120 #endif 121 .endm 122 123 .macro CALL_NOSPEC reg:req 124 #ifdef CONFIG_RETPOLINE 125 ANNOTATE_NOSPEC_ALTERNATIVE 126 ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; call *\reg), \ 127 __stringify(RETPOLINE_CALL \reg), X86_FEATURE_RETPOLINE,\ 128 __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; call *\reg), X86_FEATURE_RETPOLINE_AMD 129 #else 130 call *\reg 131 #endif 132 .endm 133 134 /* 135 * A simpler FILL_RETURN_BUFFER macro. Don't make people use the CPP 136 * monstrosity above, manually. 137 */ 138 .macro FILL_RETURN_BUFFER reg:req nr:req ftr:req 139 #ifdef CONFIG_RETPOLINE 140 ANNOTATE_NOSPEC_ALTERNATIVE 141 ALTERNATIVE "jmp .Lskip_rsb_\@", \ 142 __stringify(__FILL_RETURN_BUFFER(\reg,\nr,%_ASM_SP)) \ 143 \ftr 144 .Lskip_rsb_\@: 145 #endif 146 .endm 147 148 #else /* __ASSEMBLY__ */ 149 150 #define ANNOTATE_RETPOLINE_SAFE \ 151 "999:\n\t" \ 152 ".pushsection .discard.retpoline_safe\n\t" \ 153 _ASM_PTR " 999b\n\t" \ 154 ".popsection\n\t" 155 156 #ifdef CONFIG_RETPOLINE 157 #ifdef CONFIG_X86_64 158 159 /* 160 * Inline asm uses the %V modifier which is only in newer GCC 161 * which is ensured when CONFIG_RETPOLINE is defined. 162 */ 163 # define CALL_NOSPEC \ 164 ANNOTATE_NOSPEC_ALTERNATIVE \ 165 ALTERNATIVE_2( \ 166 ANNOTATE_RETPOLINE_SAFE \ 167 "call *%[thunk_target]\n", \ 168 "call __x86_indirect_thunk_%V[thunk_target]\n", \ 169 X86_FEATURE_RETPOLINE, \ 170 "lfence;\n" \ 171 ANNOTATE_RETPOLINE_SAFE \ 172 "call *%[thunk_target]\n", \ 173 X86_FEATURE_RETPOLINE_AMD) 174 # define THUNK_TARGET(addr) [thunk_target] "r" (addr) 175 176 #else /* CONFIG_X86_32 */ 177 /* 178 * For i386 we use the original ret-equivalent retpoline, because 179 * otherwise we'll run out of registers. We don't care about CET 180 * here, anyway. 181 */ 182 # define CALL_NOSPEC \ 183 ANNOTATE_NOSPEC_ALTERNATIVE \ 184 ALTERNATIVE_2( \ 185 ANNOTATE_RETPOLINE_SAFE \ 186 "call *%[thunk_target]\n", \ 187 " jmp 904f;\n" \ 188 " .align 16\n" \ 189 "901: call 903f;\n" \ 190 "902: pause;\n" \ 191 " lfence;\n" \ 192 " jmp 902b;\n" \ 193 " .align 16\n" \ 194 "903: lea 4(%%esp), %%esp;\n" \ 195 " pushl %[thunk_target];\n" \ 196 " ret;\n" \ 197 " .align 16\n" \ 198 "904: call 901b;\n", \ 199 X86_FEATURE_RETPOLINE, \ 200 "lfence;\n" \ 201 ANNOTATE_RETPOLINE_SAFE \ 202 "call *%[thunk_target]\n", \ 203 X86_FEATURE_RETPOLINE_AMD) 204 205 # define THUNK_TARGET(addr) [thunk_target] "rm" (addr) 206 #endif 207 #else /* No retpoline for C / inline asm */ 208 # define CALL_NOSPEC "call *%[thunk_target]\n" 209 # define THUNK_TARGET(addr) [thunk_target] "rm" (addr) 210 #endif 211 212 /* The Spectre V2 mitigation variants */ 213 enum spectre_v2_mitigation { 214 SPECTRE_V2_NONE, 215 SPECTRE_V2_RETPOLINE_GENERIC, 216 SPECTRE_V2_RETPOLINE_AMD, 217 SPECTRE_V2_IBRS_ENHANCED, 218 }; 219 220 /* The indirect branch speculation control variants */ 221 enum spectre_v2_user_mitigation { 222 SPECTRE_V2_USER_NONE, 223 SPECTRE_V2_USER_STRICT, 224 SPECTRE_V2_USER_STRICT_PREFERRED, 225 SPECTRE_V2_USER_PRCTL, 226 SPECTRE_V2_USER_SECCOMP, 227 }; 228 229 /* The Speculative Store Bypass disable variants */ 230 enum ssb_mitigation { 231 SPEC_STORE_BYPASS_NONE, 232 SPEC_STORE_BYPASS_DISABLE, 233 SPEC_STORE_BYPASS_PRCTL, 234 SPEC_STORE_BYPASS_SECCOMP, 235 }; 236 237 extern char __indirect_thunk_start[]; 238 extern char __indirect_thunk_end[]; 239 240 /* 241 * On VMEXIT we must ensure that no RSB predictions learned in the guest 242 * can be followed in the host, by overwriting the RSB completely. Both 243 * retpoline and IBRS mitigations for Spectre v2 need this; only on future 244 * CPUs with IBRS_ALL *might* it be avoided. 245 */ 246 static inline void vmexit_fill_RSB(void) 247 { 248 #ifdef CONFIG_RETPOLINE 249 unsigned long loops; 250 251 asm volatile (ANNOTATE_NOSPEC_ALTERNATIVE 252 ALTERNATIVE("jmp 910f", 253 __stringify(__FILL_RETURN_BUFFER(%0, RSB_CLEAR_LOOPS, %1)), 254 X86_FEATURE_RETPOLINE) 255 "910:" 256 : "=r" (loops), ASM_CALL_CONSTRAINT 257 : : "memory" ); 258 #endif 259 } 260 261 static __always_inline 262 void alternative_msr_write(unsigned int msr, u64 val, unsigned int feature) 263 { 264 asm volatile(ALTERNATIVE("", "wrmsr", %c[feature]) 265 : : "c" (msr), 266 "a" ((u32)val), 267 "d" ((u32)(val >> 32)), 268 [feature] "i" (feature) 269 : "memory"); 270 } 271 272 static inline void indirect_branch_prediction_barrier(void) 273 { 274 u64 val = PRED_CMD_IBPB; 275 276 alternative_msr_write(MSR_IA32_PRED_CMD, val, X86_FEATURE_USE_IBPB); 277 } 278 279 /* The Intel SPEC CTRL MSR base value cache */ 280 extern u64 x86_spec_ctrl_base; 281 282 /* 283 * With retpoline, we must use IBRS to restrict branch prediction 284 * before calling into firmware. 285 * 286 * (Implemented as CPP macros due to header hell.) 287 */ 288 #define firmware_restrict_branch_speculation_start() \ 289 do { \ 290 u64 val = x86_spec_ctrl_base | SPEC_CTRL_IBRS; \ 291 \ 292 preempt_disable(); \ 293 alternative_msr_write(MSR_IA32_SPEC_CTRL, val, \ 294 X86_FEATURE_USE_IBRS_FW); \ 295 } while (0) 296 297 #define firmware_restrict_branch_speculation_end() \ 298 do { \ 299 u64 val = x86_spec_ctrl_base; \ 300 \ 301 alternative_msr_write(MSR_IA32_SPEC_CTRL, val, \ 302 X86_FEATURE_USE_IBRS_FW); \ 303 preempt_enable(); \ 304 } while (0) 305 306 DECLARE_STATIC_KEY_FALSE(switch_to_cond_stibp); 307 DECLARE_STATIC_KEY_FALSE(switch_mm_cond_ibpb); 308 DECLARE_STATIC_KEY_FALSE(switch_mm_always_ibpb); 309 310 DECLARE_STATIC_KEY_FALSE(mds_user_clear); 311 DECLARE_STATIC_KEY_FALSE(mds_idle_clear); 312 313 #include <asm/segment.h> 314 315 /** 316 * mds_clear_cpu_buffers - Mitigation for MDS and TAA vulnerability 317 * 318 * This uses the otherwise unused and obsolete VERW instruction in 319 * combination with microcode which triggers a CPU buffer flush when the 320 * instruction is executed. 321 */ 322 static inline void mds_clear_cpu_buffers(void) 323 { 324 static const u16 ds = __KERNEL_DS; 325 326 /* 327 * Has to be the memory-operand variant because only that 328 * guarantees the CPU buffer flush functionality according to 329 * documentation. The register-operand variant does not. 330 * Works with any segment selector, but a valid writable 331 * data segment is the fastest variant. 332 * 333 * "cc" clobber is required because VERW modifies ZF. 334 */ 335 asm volatile("verw %[ds]" : : [ds] "m" (ds) : "cc"); 336 } 337 338 /** 339 * mds_user_clear_cpu_buffers - Mitigation for MDS and TAA vulnerability 340 * 341 * Clear CPU buffers if the corresponding static key is enabled 342 */ 343 static inline void mds_user_clear_cpu_buffers(void) 344 { 345 if (static_branch_likely(&mds_user_clear)) 346 mds_clear_cpu_buffers(); 347 } 348 349 /** 350 * mds_idle_clear_cpu_buffers - Mitigation for MDS vulnerability 351 * 352 * Clear CPU buffers if the corresponding static key is enabled 353 */ 354 static inline void mds_idle_clear_cpu_buffers(void) 355 { 356 if (static_branch_likely(&mds_idle_clear)) 357 mds_clear_cpu_buffers(); 358 } 359 360 #endif /* __ASSEMBLY__ */ 361 362 /* 363 * Below is used in the eBPF JIT compiler and emits the byte sequence 364 * for the following assembly: 365 * 366 * With retpolines configured: 367 * 368 * callq do_rop 369 * spec_trap: 370 * pause 371 * lfence 372 * jmp spec_trap 373 * do_rop: 374 * mov %rax,(%rsp) for x86_64 375 * mov %edx,(%esp) for x86_32 376 * retq 377 * 378 * Without retpolines configured: 379 * 380 * jmp *%rax for x86_64 381 * jmp *%edx for x86_32 382 */ 383 #ifdef CONFIG_RETPOLINE 384 # ifdef CONFIG_X86_64 385 # define RETPOLINE_RAX_BPF_JIT_SIZE 17 386 # define RETPOLINE_RAX_BPF_JIT() \ 387 do { \ 388 EMIT1_off32(0xE8, 7); /* callq do_rop */ \ 389 /* spec_trap: */ \ 390 EMIT2(0xF3, 0x90); /* pause */ \ 391 EMIT3(0x0F, 0xAE, 0xE8); /* lfence */ \ 392 EMIT2(0xEB, 0xF9); /* jmp spec_trap */ \ 393 /* do_rop: */ \ 394 EMIT4(0x48, 0x89, 0x04, 0x24); /* mov %rax,(%rsp) */ \ 395 EMIT1(0xC3); /* retq */ \ 396 } while (0) 397 # else /* !CONFIG_X86_64 */ 398 # define RETPOLINE_EDX_BPF_JIT() \ 399 do { \ 400 EMIT1_off32(0xE8, 7); /* call do_rop */ \ 401 /* spec_trap: */ \ 402 EMIT2(0xF3, 0x90); /* pause */ \ 403 EMIT3(0x0F, 0xAE, 0xE8); /* lfence */ \ 404 EMIT2(0xEB, 0xF9); /* jmp spec_trap */ \ 405 /* do_rop: */ \ 406 EMIT3(0x89, 0x14, 0x24); /* mov %edx,(%esp) */ \ 407 EMIT1(0xC3); /* ret */ \ 408 } while (0) 409 # endif 410 #else /* !CONFIG_RETPOLINE */ 411 # ifdef CONFIG_X86_64 412 # define RETPOLINE_RAX_BPF_JIT_SIZE 2 413 # define RETPOLINE_RAX_BPF_JIT() \ 414 EMIT2(0xFF, 0xE0); /* jmp *%rax */ 415 # else /* !CONFIG_X86_64 */ 416 # define RETPOLINE_EDX_BPF_JIT() \ 417 EMIT2(0xFF, 0xE2) /* jmp *%edx */ 418 # endif 419 #endif 420 421 #endif /* _ASM_X86_NOSPEC_BRANCH_H_ */ 422