1 /* SPDX-License-Identifier: GPL-2.0 */ 2 3 #ifndef _ASM_X86_NOSPEC_BRANCH_H_ 4 #define _ASM_X86_NOSPEC_BRANCH_H_ 5 6 #include <asm/alternative.h> 7 #include <asm/alternative-asm.h> 8 #include <asm/cpufeatures.h> 9 10 #ifdef __ASSEMBLY__ 11 12 /* 13 * This should be used immediately before a retpoline alternative. It tells 14 * objtool where the retpolines are so that it can make sense of the control 15 * flow by just reading the original instruction(s) and ignoring the 16 * alternatives. 17 */ 18 .macro ANNOTATE_NOSPEC_ALTERNATIVE 19 .Lannotate_\@: 20 .pushsection .discard.nospec 21 .long .Lannotate_\@ - . 22 .popsection 23 .endm 24 25 /* 26 * These are the bare retpoline primitives for indirect jmp and call. 27 * Do not use these directly; they only exist to make the ALTERNATIVE 28 * invocation below less ugly. 29 */ 30 .macro RETPOLINE_JMP reg:req 31 call .Ldo_rop_\@ 32 .Lspec_trap_\@: 33 pause 34 lfence 35 jmp .Lspec_trap_\@ 36 .Ldo_rop_\@: 37 mov \reg, (%_ASM_SP) 38 ret 39 .endm 40 41 /* 42 * This is a wrapper around RETPOLINE_JMP so the called function in reg 43 * returns to the instruction after the macro. 44 */ 45 .macro RETPOLINE_CALL reg:req 46 jmp .Ldo_call_\@ 47 .Ldo_retpoline_jmp_\@: 48 RETPOLINE_JMP \reg 49 .Ldo_call_\@: 50 call .Ldo_retpoline_jmp_\@ 51 .endm 52 53 /* 54 * JMP_NOSPEC and CALL_NOSPEC macros can be used instead of a simple 55 * indirect jmp/call which may be susceptible to the Spectre variant 2 56 * attack. 57 */ 58 .macro JMP_NOSPEC reg:req 59 #ifdef CONFIG_RETPOLINE 60 ANNOTATE_NOSPEC_ALTERNATIVE 61 ALTERNATIVE_2 __stringify(jmp *\reg), \ 62 __stringify(RETPOLINE_JMP \reg), X86_FEATURE_RETPOLINE, \ 63 __stringify(lfence; jmp *\reg), X86_FEATURE_RETPOLINE_AMD 64 #else 65 jmp *\reg 66 #endif 67 .endm 68 69 .macro CALL_NOSPEC reg:req 70 #ifdef CONFIG_RETPOLINE 71 ANNOTATE_NOSPEC_ALTERNATIVE 72 ALTERNATIVE_2 __stringify(call *\reg), \ 73 __stringify(RETPOLINE_CALL \reg), X86_FEATURE_RETPOLINE,\ 74 __stringify(lfence; call *\reg), X86_FEATURE_RETPOLINE_AMD 75 #else 76 call *\reg 77 #endif 78 .endm 79 80 /* This clobbers the BX register */ 81 .macro FILL_RETURN_BUFFER nr:req ftr:req 82 #ifdef CONFIG_RETPOLINE 83 ALTERNATIVE "", "call __clear_rsb", \ftr 84 #endif 85 .endm 86 87 #else /* __ASSEMBLY__ */ 88 89 #define ANNOTATE_NOSPEC_ALTERNATIVE \ 90 "999:\n\t" \ 91 ".pushsection .discard.nospec\n\t" \ 92 ".long 999b - .\n\t" \ 93 ".popsection\n\t" 94 95 #if defined(CONFIG_X86_64) && defined(RETPOLINE) 96 97 /* 98 * Since the inline asm uses the %V modifier which is only in newer GCC, 99 * the 64-bit one is dependent on RETPOLINE not CONFIG_RETPOLINE. 100 */ 101 # define CALL_NOSPEC \ 102 ANNOTATE_NOSPEC_ALTERNATIVE \ 103 ALTERNATIVE( \ 104 "call *%[thunk_target]\n", \ 105 "call __x86_indirect_thunk_%V[thunk_target]\n", \ 106 X86_FEATURE_RETPOLINE) 107 # define THUNK_TARGET(addr) [thunk_target] "r" (addr) 108 109 #elif defined(CONFIG_X86_32) && defined(CONFIG_RETPOLINE) 110 /* 111 * For i386 we use the original ret-equivalent retpoline, because 112 * otherwise we'll run out of registers. We don't care about CET 113 * here, anyway. 114 */ 115 # define CALL_NOSPEC ALTERNATIVE("call *%[thunk_target]\n", \ 116 " jmp 904f;\n" \ 117 " .align 16\n" \ 118 "901: call 903f;\n" \ 119 "902: pause;\n" \ 120 " lfence;\n" \ 121 " jmp 902b;\n" \ 122 " .align 16\n" \ 123 "903: addl $4, %%esp;\n" \ 124 " pushl %[thunk_target];\n" \ 125 " ret;\n" \ 126 " .align 16\n" \ 127 "904: call 901b;\n", \ 128 X86_FEATURE_RETPOLINE) 129 130 # define THUNK_TARGET(addr) [thunk_target] "rm" (addr) 131 #else /* No retpoline for C / inline asm */ 132 # define CALL_NOSPEC "call *%[thunk_target]\n" 133 # define THUNK_TARGET(addr) [thunk_target] "rm" (addr) 134 #endif 135 136 /* The Spectre V2 mitigation variants */ 137 enum spectre_v2_mitigation { 138 SPECTRE_V2_NONE, 139 SPECTRE_V2_RETPOLINE_MINIMAL, 140 SPECTRE_V2_RETPOLINE_MINIMAL_AMD, 141 SPECTRE_V2_RETPOLINE_GENERIC, 142 SPECTRE_V2_RETPOLINE_AMD, 143 SPECTRE_V2_IBRS, 144 }; 145 146 extern char __indirect_thunk_start[]; 147 extern char __indirect_thunk_end[]; 148 149 /* 150 * On VMEXIT we must ensure that no RSB predictions learned in the guest 151 * can be followed in the host, by overwriting the RSB completely. Both 152 * retpoline and IBRS mitigations for Spectre v2 need this; only on future 153 * CPUs with IBRS_ALL *might* it be avoided. 154 */ 155 static inline void vmexit_fill_RSB(void) 156 { 157 #ifdef CONFIG_RETPOLINE 158 alternative_input("", 159 "call __fill_rsb", 160 X86_FEATURE_RETPOLINE, 161 ASM_NO_INPUT_CLOBBER(_ASM_BX, "memory")); 162 #endif 163 } 164 165 static inline void indirect_branch_prediction_barrier(void) 166 { 167 alternative_input("", 168 "call __ibp_barrier", 169 X86_FEATURE_USE_IBPB, 170 ASM_NO_INPUT_CLOBBER("eax", "ecx", "edx", "memory")); 171 } 172 173 #endif /* __ASSEMBLY__ */ 174 #endif /* _ASM_X86_NOSPEC_BRANCH_H_ */ 175