1/* SPDX-License-Identifier: GPL-2.0 */ 2 3#include <linux/stringify.h> 4#include <linux/linkage.h> 5#include <asm/dwarf2.h> 6#include <asm/cpufeatures.h> 7#include <asm/alternative.h> 8#include <asm/asm-offsets.h> 9#include <asm/export.h> 10#include <asm/nospec-branch.h> 11#include <asm/unwind_hints.h> 12#include <asm/percpu.h> 13#include <asm/frame.h> 14 15 .section .text.__x86.indirect_thunk 16 17 18.macro POLINE reg 19 ANNOTATE_INTRA_FUNCTION_CALL 20 call .Ldo_rop_\@ 21 int3 22.Ldo_rop_\@: 23 mov %\reg, (%_ASM_SP) 24 UNWIND_HINT_FUNC 25.endm 26 27.macro RETPOLINE reg 28 POLINE \reg 29 RET 30.endm 31 32.macro THUNK reg 33 34 .align RETPOLINE_THUNK_SIZE 35SYM_INNER_LABEL(__x86_indirect_thunk_\reg, SYM_L_GLOBAL) 36 UNWIND_HINT_EMPTY 37 ANNOTATE_NOENDBR 38 39 ALTERNATIVE_2 __stringify(RETPOLINE \reg), \ 40 __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; jmp *%\reg; int3), X86_FEATURE_RETPOLINE_LFENCE, \ 41 __stringify(ANNOTATE_RETPOLINE_SAFE; jmp *%\reg), ALT_NOT(X86_FEATURE_RETPOLINE) 42 43.endm 44 45/* 46 * Despite being an assembler file we can't just use .irp here 47 * because __KSYM_DEPS__ only uses the C preprocessor and would 48 * only see one instance of "__x86_indirect_thunk_\reg" rather 49 * than one per register with the correct names. So we do it 50 * the simple and nasty way... 51 * 52 * Worse, you can only have a single EXPORT_SYMBOL per line, 53 * and CPP can't insert newlines, so we have to repeat everything 54 * at least twice. 55 */ 56 57#define __EXPORT_THUNK(sym) _ASM_NOKPROBE(sym); EXPORT_SYMBOL(sym) 58 59 .align RETPOLINE_THUNK_SIZE 60SYM_CODE_START(__x86_indirect_thunk_array) 61 62#define GEN(reg) THUNK reg 63#include <asm/GEN-for-each-reg.h> 64#undef GEN 65 66 .align RETPOLINE_THUNK_SIZE 67SYM_CODE_END(__x86_indirect_thunk_array) 68 69#define GEN(reg) __EXPORT_THUNK(__x86_indirect_thunk_ ## reg) 70#include <asm/GEN-for-each-reg.h> 71#undef GEN 72 73#ifdef CONFIG_CALL_DEPTH_TRACKING 74.macro CALL_THUNK reg 75 .align RETPOLINE_THUNK_SIZE 76 77SYM_INNER_LABEL(__x86_indirect_call_thunk_\reg, SYM_L_GLOBAL) 78 UNWIND_HINT_EMPTY 79 ANNOTATE_NOENDBR 80 81 CALL_DEPTH_ACCOUNT 82 POLINE \reg 83 ANNOTATE_UNRET_SAFE 84 ret 85 int3 86.endm 87 88 .align RETPOLINE_THUNK_SIZE 89SYM_CODE_START(__x86_indirect_call_thunk_array) 90 91#define GEN(reg) CALL_THUNK reg 92#include <asm/GEN-for-each-reg.h> 93#undef GEN 94 95 .align RETPOLINE_THUNK_SIZE 96SYM_CODE_END(__x86_indirect_call_thunk_array) 97 98#define GEN(reg) __EXPORT_THUNK(__x86_indirect_call_thunk_ ## reg) 99#include <asm/GEN-for-each-reg.h> 100#undef GEN 101 102.macro JUMP_THUNK reg 103 .align RETPOLINE_THUNK_SIZE 104 105SYM_INNER_LABEL(__x86_indirect_jump_thunk_\reg, SYM_L_GLOBAL) 106 UNWIND_HINT_EMPTY 107 ANNOTATE_NOENDBR 108 POLINE \reg 109 ANNOTATE_UNRET_SAFE 110 ret 111 int3 112.endm 113 114 .align RETPOLINE_THUNK_SIZE 115SYM_CODE_START(__x86_indirect_jump_thunk_array) 116 117#define GEN(reg) JUMP_THUNK reg 118#include <asm/GEN-for-each-reg.h> 119#undef GEN 120 121 .align RETPOLINE_THUNK_SIZE 122SYM_CODE_END(__x86_indirect_jump_thunk_array) 123 124#define GEN(reg) __EXPORT_THUNK(__x86_indirect_jump_thunk_ ## reg) 125#include <asm/GEN-for-each-reg.h> 126#undef GEN 127#endif 128/* 129 * This function name is magical and is used by -mfunction-return=thunk-extern 130 * for the compiler to generate JMPs to it. 131 */ 132#ifdef CONFIG_RETHUNK 133 134 .section .text.__x86.return_thunk 135 136/* 137 * Safety details here pertain to the AMD Zen{1,2} microarchitecture: 138 * 1) The RET at __x86_return_thunk must be on a 64 byte boundary, for 139 * alignment within the BTB. 140 * 2) The instruction at zen_untrain_ret must contain, and not 141 * end with, the 0xc3 byte of the RET. 142 * 3) STIBP must be enabled, or SMT disabled, to prevent the sibling thread 143 * from re-poisioning the BTB prediction. 144 */ 145 .align 64 146 .skip 63, 0xcc 147SYM_FUNC_START_NOALIGN(zen_untrain_ret); 148 149 /* 150 * As executed from zen_untrain_ret, this is: 151 * 152 * TEST $0xcc, %bl 153 * LFENCE 154 * JMP __x86_return_thunk 155 * 156 * Executing the TEST instruction has a side effect of evicting any BTB 157 * prediction (potentially attacker controlled) attached to the RET, as 158 * __x86_return_thunk + 1 isn't an instruction boundary at the moment. 159 */ 160 .byte 0xf6 161 162 /* 163 * As executed from __x86_return_thunk, this is a plain RET. 164 * 165 * As part of the TEST above, RET is the ModRM byte, and INT3 the imm8. 166 * 167 * We subsequently jump backwards and architecturally execute the RET. 168 * This creates a correct BTB prediction (type=ret), but in the 169 * meantime we suffer Straight Line Speculation (because the type was 170 * no branch) which is halted by the INT3. 171 * 172 * With SMT enabled and STIBP active, a sibling thread cannot poison 173 * RET's prediction to a type of its choice, but can evict the 174 * prediction due to competitive sharing. If the prediction is 175 * evicted, __x86_return_thunk will suffer Straight Line Speculation 176 * which will be contained safely by the INT3. 177 */ 178SYM_INNER_LABEL(__x86_return_thunk, SYM_L_GLOBAL) 179 ret 180 int3 181SYM_CODE_END(__x86_return_thunk) 182 183 /* 184 * Ensure the TEST decoding / BTB invalidation is complete. 185 */ 186 lfence 187 188 /* 189 * Jump back and execute the RET in the middle of the TEST instruction. 190 * INT3 is for SLS protection. 191 */ 192 jmp __x86_return_thunk 193 int3 194SYM_FUNC_END(zen_untrain_ret) 195__EXPORT_THUNK(zen_untrain_ret) 196 197EXPORT_SYMBOL(__x86_return_thunk) 198 199#endif /* CONFIG_RETHUNK */ 200 201#ifdef CONFIG_CALL_DEPTH_TRACKING 202 203 .align 64 204SYM_FUNC_START(__x86_return_skl) 205 ANNOTATE_NOENDBR 206 /* 207 * Keep the hotpath in a 16byte I-fetch for the non-debug 208 * case. 209 */ 210 CALL_THUNKS_DEBUG_INC_RETS 211 shlq $5, PER_CPU_VAR(pcpu_hot + X86_call_depth) 212 jz 1f 213 ANNOTATE_UNRET_SAFE 214 ret 215 int3 2161: 217 CALL_THUNKS_DEBUG_INC_STUFFS 218 .rept 16 219 ANNOTATE_INTRA_FUNCTION_CALL 220 call 2f 221 int3 2222: 223 .endr 224 add $(8*16), %rsp 225 226 CREDIT_CALL_DEPTH 227 228 ANNOTATE_UNRET_SAFE 229 ret 230 int3 231SYM_FUNC_END(__x86_return_skl) 232 233#endif /* CONFIG_CALL_DEPTH_TRACKING */ 234