1 /* SPDX-License-Identifier: GPL-2.0 */ 2 3 #ifndef _ASM_X86_NOSPEC_BRANCH_H_ 4 #define _ASM_X86_NOSPEC_BRANCH_H_ 5 6 #include <linux/static_key.h> 7 8 #include <asm/alternative.h> 9 #include <asm/alternative-asm.h> 10 #include <asm/cpufeatures.h> 11 #include <asm/msr-index.h> 12 13 /* 14 * This should be used immediately before a retpoline alternative. It tells 15 * objtool where the retpolines are so that it can make sense of the control 16 * flow by just reading the original instruction(s) and ignoring the 17 * alternatives. 18 */ 19 #define ANNOTATE_NOSPEC_ALTERNATIVE \ 20 ANNOTATE_IGNORE_ALTERNATIVE 21 22 /* 23 * Fill the CPU return stack buffer. 24 * 25 * Each entry in the RSB, if used for a speculative 'ret', contains an 26 * infinite 'pause; lfence; jmp' loop to capture speculative execution. 27 * 28 * This is required in various cases for retpoline and IBRS-based 29 * mitigations for the Spectre variant 2 vulnerability. Sometimes to 30 * eliminate potentially bogus entries from the RSB, and sometimes 31 * purely to ensure that it doesn't get empty, which on some CPUs would 32 * allow predictions from other (unwanted!) sources to be used. 33 * 34 * We define a CPP macro such that it can be used from both .S files and 35 * inline assembly. It's possible to do a .macro and then include that 36 * from C via asm(".include <asm/nospec-branch.h>") but let's not go there. 37 */ 38 39 #define RSB_CLEAR_LOOPS 32 /* To forcibly overwrite all entries */ 40 41 /* 42 * Google experimented with loop-unrolling and this turned out to be 43 * the optimal version — two calls, each with their own speculation 44 * trap should their return address end up getting used, in a loop. 45 */ 46 #define __FILL_RETURN_BUFFER(reg, nr, sp) \ 47 mov $(nr/2), reg; \ 48 771: \ 49 call 772f; \ 50 773: /* speculation trap */ \ 51 pause; \ 52 lfence; \ 53 jmp 773b; \ 54 772: \ 55 call 774f; \ 56 775: /* speculation trap */ \ 57 pause; \ 58 lfence; \ 59 jmp 775b; \ 60 774: \ 61 dec reg; \ 62 jnz 771b; \ 63 add $(BITS_PER_LONG/8) * nr, sp; 64 65 #ifdef __ASSEMBLY__ 66 67 /* 68 * This should be used immediately before an indirect jump/call. It tells 69 * objtool the subsequent indirect jump/call is vouched safe for retpoline 70 * builds. 71 */ 72 .macro ANNOTATE_RETPOLINE_SAFE 73 .Lannotate_\@: 74 .pushsection .discard.retpoline_safe 75 _ASM_PTR .Lannotate_\@ 76 .popsection 77 .endm 78 79 /* 80 * These are the bare retpoline primitives for indirect jmp and call. 81 * Do not use these directly; they only exist to make the ALTERNATIVE 82 * invocation below less ugly. 83 */ 84 .macro RETPOLINE_JMP reg:req 85 call .Ldo_rop_\@ 86 .Lspec_trap_\@: 87 pause 88 lfence 89 jmp .Lspec_trap_\@ 90 .Ldo_rop_\@: 91 mov \reg, (%_ASM_SP) 92 ret 93 .endm 94 95 /* 96 * This is a wrapper around RETPOLINE_JMP so the called function in reg 97 * returns to the instruction after the macro. 98 */ 99 .macro RETPOLINE_CALL reg:req 100 jmp .Ldo_call_\@ 101 .Ldo_retpoline_jmp_\@: 102 RETPOLINE_JMP \reg 103 .Ldo_call_\@: 104 call .Ldo_retpoline_jmp_\@ 105 .endm 106 107 /* 108 * JMP_NOSPEC and CALL_NOSPEC macros can be used instead of a simple 109 * indirect jmp/call which may be susceptible to the Spectre variant 2 110 * attack. 111 */ 112 .macro JMP_NOSPEC reg:req 113 #ifdef CONFIG_RETPOLINE 114 ANNOTATE_NOSPEC_ALTERNATIVE 115 ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; jmp *\reg), \ 116 __stringify(RETPOLINE_JMP \reg), X86_FEATURE_RETPOLINE, \ 117 __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; jmp *\reg), X86_FEATURE_RETPOLINE_AMD 118 #else 119 jmp *\reg 120 #endif 121 .endm 122 123 .macro CALL_NOSPEC reg:req 124 #ifdef CONFIG_RETPOLINE 125 ANNOTATE_NOSPEC_ALTERNATIVE 126 ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; call *\reg), \ 127 __stringify(RETPOLINE_CALL \reg), X86_FEATURE_RETPOLINE,\ 128 __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; call *\reg), X86_FEATURE_RETPOLINE_AMD 129 #else 130 call *\reg 131 #endif 132 .endm 133 134 /* 135 * A simpler FILL_RETURN_BUFFER macro. Don't make people use the CPP 136 * monstrosity above, manually. 137 */ 138 .macro FILL_RETURN_BUFFER reg:req nr:req ftr:req 139 #ifdef CONFIG_RETPOLINE 140 ANNOTATE_NOSPEC_ALTERNATIVE 141 ALTERNATIVE "jmp .Lskip_rsb_\@", \ 142 __stringify(__FILL_RETURN_BUFFER(\reg,\nr,%_ASM_SP)) \ 143 \ftr 144 .Lskip_rsb_\@: 145 #endif 146 .endm 147 148 #else /* __ASSEMBLY__ */ 149 150 #define ANNOTATE_RETPOLINE_SAFE \ 151 "999:\n\t" \ 152 ".pushsection .discard.retpoline_safe\n\t" \ 153 _ASM_PTR " 999b\n\t" \ 154 ".popsection\n\t" 155 156 #ifdef CONFIG_RETPOLINE 157 #ifdef CONFIG_X86_64 158 159 /* 160 * Inline asm uses the %V modifier which is only in newer GCC 161 * which is ensured when CONFIG_RETPOLINE is defined. 162 */ 163 # define CALL_NOSPEC \ 164 ANNOTATE_NOSPEC_ALTERNATIVE \ 165 ALTERNATIVE_2( \ 166 ANNOTATE_RETPOLINE_SAFE \ 167 "call *%[thunk_target]\n", \ 168 "call __x86_indirect_thunk_%V[thunk_target]\n", \ 169 X86_FEATURE_RETPOLINE, \ 170 "lfence;\n" \ 171 ANNOTATE_RETPOLINE_SAFE \ 172 "call *%[thunk_target]\n", \ 173 X86_FEATURE_RETPOLINE_AMD) 174 # define THUNK_TARGET(addr) [thunk_target] "r" (addr) 175 176 #else /* CONFIG_X86_32 */ 177 /* 178 * For i386 we use the original ret-equivalent retpoline, because 179 * otherwise we'll run out of registers. We don't care about CET 180 * here, anyway. 181 */ 182 # define CALL_NOSPEC \ 183 ANNOTATE_NOSPEC_ALTERNATIVE \ 184 ALTERNATIVE_2( \ 185 ANNOTATE_RETPOLINE_SAFE \ 186 "call *%[thunk_target]\n", \ 187 " jmp 904f;\n" \ 188 " .align 16\n" \ 189 "901: call 903f;\n" \ 190 "902: pause;\n" \ 191 " lfence;\n" \ 192 " jmp 902b;\n" \ 193 " .align 16\n" \ 194 "903: lea 4(%%esp), %%esp;\n" \ 195 " pushl %[thunk_target];\n" \ 196 " ret;\n" \ 197 " .align 16\n" \ 198 "904: call 901b;\n", \ 199 X86_FEATURE_RETPOLINE, \ 200 "lfence;\n" \ 201 ANNOTATE_RETPOLINE_SAFE \ 202 "call *%[thunk_target]\n", \ 203 X86_FEATURE_RETPOLINE_AMD) 204 205 # define THUNK_TARGET(addr) [thunk_target] "rm" (addr) 206 #endif 207 #else /* No retpoline for C / inline asm */ 208 # define CALL_NOSPEC "call *%[thunk_target]\n" 209 # define THUNK_TARGET(addr) [thunk_target] "rm" (addr) 210 #endif 211 212 /* The Spectre V2 mitigation variants */ 213 enum spectre_v2_mitigation { 214 SPECTRE_V2_NONE, 215 SPECTRE_V2_RETPOLINE_GENERIC, 216 SPECTRE_V2_RETPOLINE_AMD, 217 SPECTRE_V2_IBRS_ENHANCED, 218 }; 219 220 /* The indirect branch speculation control variants */ 221 enum spectre_v2_user_mitigation { 222 SPECTRE_V2_USER_NONE, 223 SPECTRE_V2_USER_STRICT, 224 SPECTRE_V2_USER_STRICT_PREFERRED, 225 SPECTRE_V2_USER_PRCTL, 226 SPECTRE_V2_USER_SECCOMP, 227 }; 228 229 /* The Speculative Store Bypass disable variants */ 230 enum ssb_mitigation { 231 SPEC_STORE_BYPASS_NONE, 232 SPEC_STORE_BYPASS_DISABLE, 233 SPEC_STORE_BYPASS_PRCTL, 234 SPEC_STORE_BYPASS_SECCOMP, 235 }; 236 237 extern char __indirect_thunk_start[]; 238 extern char __indirect_thunk_end[]; 239 240 static __always_inline 241 void alternative_msr_write(unsigned int msr, u64 val, unsigned int feature) 242 { 243 asm volatile(ALTERNATIVE("", "wrmsr", %c[feature]) 244 : : "c" (msr), 245 "a" ((u32)val), 246 "d" ((u32)(val >> 32)), 247 [feature] "i" (feature) 248 : "memory"); 249 } 250 251 static inline void indirect_branch_prediction_barrier(void) 252 { 253 u64 val = PRED_CMD_IBPB; 254 255 alternative_msr_write(MSR_IA32_PRED_CMD, val, X86_FEATURE_USE_IBPB); 256 } 257 258 /* The Intel SPEC CTRL MSR base value cache */ 259 extern u64 x86_spec_ctrl_base; 260 261 /* 262 * With retpoline, we must use IBRS to restrict branch prediction 263 * before calling into firmware. 264 * 265 * (Implemented as CPP macros due to header hell.) 266 */ 267 #define firmware_restrict_branch_speculation_start() \ 268 do { \ 269 u64 val = x86_spec_ctrl_base | SPEC_CTRL_IBRS; \ 270 \ 271 preempt_disable(); \ 272 alternative_msr_write(MSR_IA32_SPEC_CTRL, val, \ 273 X86_FEATURE_USE_IBRS_FW); \ 274 } while (0) 275 276 #define firmware_restrict_branch_speculation_end() \ 277 do { \ 278 u64 val = x86_spec_ctrl_base; \ 279 \ 280 alternative_msr_write(MSR_IA32_SPEC_CTRL, val, \ 281 X86_FEATURE_USE_IBRS_FW); \ 282 preempt_enable(); \ 283 } while (0) 284 285 DECLARE_STATIC_KEY_FALSE(switch_to_cond_stibp); 286 DECLARE_STATIC_KEY_FALSE(switch_mm_cond_ibpb); 287 DECLARE_STATIC_KEY_FALSE(switch_mm_always_ibpb); 288 289 DECLARE_STATIC_KEY_FALSE(mds_user_clear); 290 DECLARE_STATIC_KEY_FALSE(mds_idle_clear); 291 292 #include <asm/segment.h> 293 294 /** 295 * mds_clear_cpu_buffers - Mitigation for MDS and TAA vulnerability 296 * 297 * This uses the otherwise unused and obsolete VERW instruction in 298 * combination with microcode which triggers a CPU buffer flush when the 299 * instruction is executed. 300 */ 301 static inline void mds_clear_cpu_buffers(void) 302 { 303 static const u16 ds = __KERNEL_DS; 304 305 /* 306 * Has to be the memory-operand variant because only that 307 * guarantees the CPU buffer flush functionality according to 308 * documentation. The register-operand variant does not. 309 * Works with any segment selector, but a valid writable 310 * data segment is the fastest variant. 311 * 312 * "cc" clobber is required because VERW modifies ZF. 313 */ 314 asm volatile("verw %[ds]" : : [ds] "m" (ds) : "cc"); 315 } 316 317 /** 318 * mds_user_clear_cpu_buffers - Mitigation for MDS and TAA vulnerability 319 * 320 * Clear CPU buffers if the corresponding static key is enabled 321 */ 322 static inline void mds_user_clear_cpu_buffers(void) 323 { 324 if (static_branch_likely(&mds_user_clear)) 325 mds_clear_cpu_buffers(); 326 } 327 328 /** 329 * mds_idle_clear_cpu_buffers - Mitigation for MDS vulnerability 330 * 331 * Clear CPU buffers if the corresponding static key is enabled 332 */ 333 static inline void mds_idle_clear_cpu_buffers(void) 334 { 335 if (static_branch_likely(&mds_idle_clear)) 336 mds_clear_cpu_buffers(); 337 } 338 339 #endif /* __ASSEMBLY__ */ 340 341 /* 342 * Below is used in the eBPF JIT compiler and emits the byte sequence 343 * for the following assembly: 344 * 345 * With retpolines configured: 346 * 347 * callq do_rop 348 * spec_trap: 349 * pause 350 * lfence 351 * jmp spec_trap 352 * do_rop: 353 * mov %rax,(%rsp) for x86_64 354 * mov %edx,(%esp) for x86_32 355 * retq 356 * 357 * Without retpolines configured: 358 * 359 * jmp *%rax for x86_64 360 * jmp *%edx for x86_32 361 */ 362 #ifdef CONFIG_RETPOLINE 363 # ifdef CONFIG_X86_64 364 # define RETPOLINE_RAX_BPF_JIT_SIZE 17 365 # define RETPOLINE_RAX_BPF_JIT() \ 366 do { \ 367 EMIT1_off32(0xE8, 7); /* callq do_rop */ \ 368 /* spec_trap: */ \ 369 EMIT2(0xF3, 0x90); /* pause */ \ 370 EMIT3(0x0F, 0xAE, 0xE8); /* lfence */ \ 371 EMIT2(0xEB, 0xF9); /* jmp spec_trap */ \ 372 /* do_rop: */ \ 373 EMIT4(0x48, 0x89, 0x04, 0x24); /* mov %rax,(%rsp) */ \ 374 EMIT1(0xC3); /* retq */ \ 375 } while (0) 376 # else /* !CONFIG_X86_64 */ 377 # define RETPOLINE_EDX_BPF_JIT() \ 378 do { \ 379 EMIT1_off32(0xE8, 7); /* call do_rop */ \ 380 /* spec_trap: */ \ 381 EMIT2(0xF3, 0x90); /* pause */ \ 382 EMIT3(0x0F, 0xAE, 0xE8); /* lfence */ \ 383 EMIT2(0xEB, 0xF9); /* jmp spec_trap */ \ 384 /* do_rop: */ \ 385 EMIT3(0x89, 0x14, 0x24); /* mov %edx,(%esp) */ \ 386 EMIT1(0xC3); /* ret */ \ 387 } while (0) 388 # endif 389 #else /* !CONFIG_RETPOLINE */ 390 # ifdef CONFIG_X86_64 391 # define RETPOLINE_RAX_BPF_JIT_SIZE 2 392 # define RETPOLINE_RAX_BPF_JIT() \ 393 EMIT2(0xFF, 0xE0); /* jmp *%rax */ 394 # else /* !CONFIG_X86_64 */ 395 # define RETPOLINE_EDX_BPF_JIT() \ 396 EMIT2(0xFF, 0xE2) /* jmp *%edx */ 397 # endif 398 #endif 399 400 #endif /* _ASM_X86_NOSPEC_BRANCH_H_ */ 401