1/* SPDX-License-Identifier: GPL-2.0 */ 2 3#include <linux/stringify.h> 4#include <linux/linkage.h> 5#include <asm/dwarf2.h> 6#include <asm/cpufeatures.h> 7#include <asm/alternative.h> 8#include <asm/export.h> 9#include <asm/nospec-branch.h> 10#include <asm/unwind_hints.h> 11#include <asm/frame.h> 12 13 .section .text.__x86.indirect_thunk 14 15.macro RETPOLINE reg 16 ANNOTATE_INTRA_FUNCTION_CALL 17 call .Ldo_rop_\@ 18.Lspec_trap_\@: 19 UNWIND_HINT_EMPTY 20 pause 21 lfence 22 jmp .Lspec_trap_\@ 23.Ldo_rop_\@: 24 mov %\reg, (%_ASM_SP) 25 UNWIND_HINT_FUNC 26 ret 27.endm 28 29.macro THUNK reg 30 31 .align 32 32 33SYM_FUNC_START(__x86_indirect_thunk_\reg) 34 35 ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; jmp *%\reg), \ 36 __stringify(RETPOLINE \reg), X86_FEATURE_RETPOLINE, \ 37 __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; jmp *%\reg), X86_FEATURE_RETPOLINE_AMD 38 39SYM_FUNC_END(__x86_indirect_thunk_\reg) 40 41.endm 42 43/* 44 * This generates .altinstr_replacement symbols for use by objtool. They, 45 * however, must not actually live in .altinstr_replacement since that will be 46 * discarded after init, but module alternatives will also reference these 47 * symbols. 48 * 49 * Their names matches the "__x86_indirect_" prefix to mark them as retpolines. 50 */ 51.macro ALT_THUNK reg 52 53 .align 1 54 55SYM_FUNC_START_NOALIGN(__x86_indirect_alt_call_\reg) 56 ANNOTATE_RETPOLINE_SAFE 571: call *%\reg 582: .skip 5-(2b-1b), 0x90 59SYM_FUNC_END(__x86_indirect_alt_call_\reg) 60 61STACK_FRAME_NON_STANDARD(__x86_indirect_alt_call_\reg) 62 63SYM_FUNC_START_NOALIGN(__x86_indirect_alt_jmp_\reg) 64 ANNOTATE_RETPOLINE_SAFE 651: jmp *%\reg 662: .skip 5-(2b-1b), 0x90 67SYM_FUNC_END(__x86_indirect_alt_jmp_\reg) 68 69STACK_FRAME_NON_STANDARD(__x86_indirect_alt_jmp_\reg) 70 71.endm 72 73/* 74 * Despite being an assembler file we can't just use .irp here 75 * because __KSYM_DEPS__ only uses the C preprocessor and would 76 * only see one instance of "__x86_indirect_thunk_\reg" rather 77 * than one per register with the correct names. So we do it 78 * the simple and nasty way... 79 * 80 * Worse, you can only have a single EXPORT_SYMBOL per line, 81 * and CPP can't insert newlines, so we have to repeat everything 82 * at least twice. 83 */ 84 85#define __EXPORT_THUNK(sym) _ASM_NOKPROBE(sym); EXPORT_SYMBOL(sym) 86#define EXPORT_THUNK(reg) __EXPORT_THUNK(__x86_indirect_thunk_ ## reg) 87 88#undef GEN 89#define GEN(reg) THUNK reg 90#include <asm/GEN-for-each-reg.h> 91 92#undef GEN 93#define GEN(reg) EXPORT_THUNK(reg) 94#include <asm/GEN-for-each-reg.h> 95 96#undef GEN 97#define GEN(reg) ALT_THUNK reg 98#include <asm/GEN-for-each-reg.h> 99 100#undef GEN 101#define GEN(reg) __EXPORT_THUNK(__x86_indirect_alt_call_ ## reg) 102#include <asm/GEN-for-each-reg.h> 103 104#undef GEN 105#define GEN(reg) __EXPORT_THUNK(__x86_indirect_alt_jmp_ ## reg) 106#include <asm/GEN-for-each-reg.h> 107