1/* SPDX-License-Identifier: GPL-2.0 */ 2 3#include <linux/stringify.h> 4#include <linux/linkage.h> 5#include <asm/dwarf2.h> 6#include <asm/cpufeatures.h> 7#include <asm/alternative.h> 8#include <asm/export.h> 9#include <asm/nospec-branch.h> 10#include <asm/unwind_hints.h> 11#include <asm/frame.h> 12 13 .section .text.__x86.indirect_thunk 14 15.macro RETPOLINE reg 16 ANNOTATE_INTRA_FUNCTION_CALL 17 call .Ldo_rop_\@ 18.Lspec_trap_\@: 19 UNWIND_HINT_EMPTY 20 pause 21 lfence 22 jmp .Lspec_trap_\@ 23.Ldo_rop_\@: 24 mov %\reg, (%_ASM_SP) 25 UNWIND_HINT_FUNC 26 RET 27.endm 28 29.macro THUNK reg 30 31 .align RETPOLINE_THUNK_SIZE 32SYM_INNER_LABEL(__x86_indirect_thunk_\reg, SYM_L_GLOBAL) 33 UNWIND_HINT_EMPTY 34 ANNOTATE_NOENDBR 35 36 ALTERNATIVE_2 __stringify(RETPOLINE \reg), \ 37 __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; jmp *%\reg; int3), X86_FEATURE_RETPOLINE_LFENCE, \ 38 __stringify(ANNOTATE_RETPOLINE_SAFE; jmp *%\reg), ALT_NOT(X86_FEATURE_RETPOLINE) 39 40.endm 41 42/* 43 * Despite being an assembler file we can't just use .irp here 44 * because __KSYM_DEPS__ only uses the C preprocessor and would 45 * only see one instance of "__x86_indirect_thunk_\reg" rather 46 * than one per register with the correct names. So we do it 47 * the simple and nasty way... 48 * 49 * Worse, you can only have a single EXPORT_SYMBOL per line, 50 * and CPP can't insert newlines, so we have to repeat everything 51 * at least twice. 52 */ 53 54#define __EXPORT_THUNK(sym) _ASM_NOKPROBE(sym); EXPORT_SYMBOL(sym) 55#define EXPORT_THUNK(reg) __EXPORT_THUNK(__x86_indirect_thunk_ ## reg) 56 57 .align RETPOLINE_THUNK_SIZE 58SYM_CODE_START(__x86_indirect_thunk_array) 59 60#define GEN(reg) THUNK reg 61#include <asm/GEN-for-each-reg.h> 62#undef GEN 63 64 .align RETPOLINE_THUNK_SIZE 65SYM_CODE_END(__x86_indirect_thunk_array) 66 67#define GEN(reg) EXPORT_THUNK(reg) 68#include <asm/GEN-for-each-reg.h> 69#undef GEN 70 71/* 72 * This function name is magical and is used by -mfunction-return=thunk-extern 73 * for the compiler to generate JMPs to it. 74 */ 75#ifdef CONFIG_RETHUNK 76 77 .section .text.__x86.return_thunk 78 79/* 80 * Safety details here pertain to the AMD Zen{1,2} microarchitecture: 81 * 1) The RET at __x86_return_thunk must be on a 64 byte boundary, for 82 * alignment within the BTB. 83 * 2) The instruction at zen_untrain_ret must contain, and not 84 * end with, the 0xc3 byte of the RET. 85 * 3) STIBP must be enabled, or SMT disabled, to prevent the sibling thread 86 * from re-poisioning the BTB prediction. 87 */ 88 .align 64 89 .skip 63, 0xcc 90SYM_FUNC_START_NOALIGN(zen_untrain_ret); 91 92 /* 93 * As executed from zen_untrain_ret, this is: 94 * 95 * TEST $0xcc, %bl 96 * LFENCE 97 * JMP __x86_return_thunk 98 * 99 * Executing the TEST instruction has a side effect of evicting any BTB 100 * prediction (potentially attacker controlled) attached to the RET, as 101 * __x86_return_thunk + 1 isn't an instruction boundary at the moment. 102 */ 103 .byte 0xf6 104 105 /* 106 * As executed from __x86_return_thunk, this is a plain RET. 107 * 108 * As part of the TEST above, RET is the ModRM byte, and INT3 the imm8. 109 * 110 * We subsequently jump backwards and architecturally execute the RET. 111 * This creates a correct BTB prediction (type=ret), but in the 112 * meantime we suffer Straight Line Speculation (because the type was 113 * no branch) which is halted by the INT3. 114 * 115 * With SMT enabled and STIBP active, a sibling thread cannot poison 116 * RET's prediction to a type of its choice, but can evict the 117 * prediction due to competitive sharing. If the prediction is 118 * evicted, __x86_return_thunk will suffer Straight Line Speculation 119 * which will be contained safely by the INT3. 120 */ 121SYM_INNER_LABEL(__x86_return_thunk, SYM_L_GLOBAL) 122 ret 123 int3 124SYM_CODE_END(__x86_return_thunk) 125 126 /* 127 * Ensure the TEST decoding / BTB invalidation is complete. 128 */ 129 lfence 130 131 /* 132 * Jump back and execute the RET in the middle of the TEST instruction. 133 * INT3 is for SLS protection. 134 */ 135 jmp __x86_return_thunk 136 int3 137SYM_FUNC_END(zen_untrain_ret) 138__EXPORT_THUNK(zen_untrain_ret) 139 140EXPORT_SYMBOL(__x86_return_thunk) 141 142#endif /* CONFIG_RETHUNK */ 143