1*e81dc127SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 2*e81dc127SThomas Gleixner 3*e81dc127SThomas Gleixner #define pr_fmt(fmt) "callthunks: " fmt 4*e81dc127SThomas Gleixner 5*e81dc127SThomas Gleixner #include <linux/kallsyms.h> 6*e81dc127SThomas Gleixner #include <linux/memory.h> 7*e81dc127SThomas Gleixner #include <linux/moduleloader.h> 8*e81dc127SThomas Gleixner 9*e81dc127SThomas Gleixner #include <asm/alternative.h> 10*e81dc127SThomas Gleixner #include <asm/cpu.h> 11*e81dc127SThomas Gleixner #include <asm/ftrace.h> 12*e81dc127SThomas Gleixner #include <asm/insn.h> 13*e81dc127SThomas Gleixner #include <asm/kexec.h> 14*e81dc127SThomas Gleixner #include <asm/nospec-branch.h> 15*e81dc127SThomas Gleixner #include <asm/paravirt.h> 16*e81dc127SThomas Gleixner #include <asm/sections.h> 17*e81dc127SThomas Gleixner #include <asm/switch_to.h> 18*e81dc127SThomas Gleixner #include <asm/sync_core.h> 19*e81dc127SThomas Gleixner #include <asm/text-patching.h> 20*e81dc127SThomas Gleixner #include <asm/xen/hypercall.h> 21*e81dc127SThomas Gleixner 22*e81dc127SThomas Gleixner static int __initdata_or_module debug_callthunks; 23*e81dc127SThomas Gleixner 24*e81dc127SThomas Gleixner #define prdbg(fmt, args...) \ 25*e81dc127SThomas Gleixner do { \ 26*e81dc127SThomas Gleixner if (debug_callthunks) \ 27*e81dc127SThomas Gleixner printk(KERN_DEBUG pr_fmt(fmt), ##args); \ 28*e81dc127SThomas Gleixner } while(0) 29*e81dc127SThomas Gleixner 30*e81dc127SThomas Gleixner static int __init debug_thunks(char *str) 31*e81dc127SThomas Gleixner { 32*e81dc127SThomas Gleixner debug_callthunks = 1; 33*e81dc127SThomas Gleixner return 1; 34*e81dc127SThomas Gleixner } 35*e81dc127SThomas Gleixner __setup("debug-callthunks", debug_thunks); 36*e81dc127SThomas Gleixner 37*e81dc127SThomas Gleixner extern s32 __call_sites[], __call_sites_end[]; 38*e81dc127SThomas Gleixner 39*e81dc127SThomas Gleixner struct thunk_desc { 40*e81dc127SThomas Gleixner void *template; 41*e81dc127SThomas Gleixner unsigned int template_size; 42*e81dc127SThomas Gleixner }; 43*e81dc127SThomas Gleixner 44*e81dc127SThomas Gleixner struct core_text { 45*e81dc127SThomas Gleixner unsigned long base; 46*e81dc127SThomas Gleixner unsigned long end; 47*e81dc127SThomas Gleixner const char *name; 48*e81dc127SThomas Gleixner }; 49*e81dc127SThomas Gleixner 50*e81dc127SThomas Gleixner static bool thunks_initialized __ro_after_init; 51*e81dc127SThomas Gleixner 52*e81dc127SThomas Gleixner static const struct core_text builtin_coretext = { 53*e81dc127SThomas Gleixner .base = (unsigned long)_text, 54*e81dc127SThomas Gleixner .end = (unsigned long)_etext, 55*e81dc127SThomas Gleixner .name = "builtin", 56*e81dc127SThomas Gleixner }; 57*e81dc127SThomas Gleixner 58*e81dc127SThomas Gleixner static struct thunk_desc callthunk_desc __ro_after_init; 59*e81dc127SThomas Gleixner 60*e81dc127SThomas Gleixner extern void error_entry(void); 61*e81dc127SThomas Gleixner extern void xen_error_entry(void); 62*e81dc127SThomas Gleixner extern void paranoid_entry(void); 63*e81dc127SThomas Gleixner 64*e81dc127SThomas Gleixner static inline bool within_coretext(const struct core_text *ct, void *addr) 65*e81dc127SThomas Gleixner { 66*e81dc127SThomas Gleixner unsigned long p = (unsigned long)addr; 67*e81dc127SThomas Gleixner 68*e81dc127SThomas Gleixner return ct->base <= p && p < ct->end; 69*e81dc127SThomas Gleixner } 70*e81dc127SThomas Gleixner 71*e81dc127SThomas Gleixner static inline bool within_module_coretext(void *addr) 72*e81dc127SThomas Gleixner { 73*e81dc127SThomas Gleixner bool ret = false; 74*e81dc127SThomas Gleixner 75*e81dc127SThomas Gleixner #ifdef CONFIG_MODULES 76*e81dc127SThomas Gleixner struct module *mod; 77*e81dc127SThomas Gleixner 78*e81dc127SThomas Gleixner preempt_disable(); 79*e81dc127SThomas Gleixner mod = __module_address((unsigned long)addr); 80*e81dc127SThomas Gleixner if (mod && within_module_core((unsigned long)addr, mod)) 81*e81dc127SThomas Gleixner ret = true; 82*e81dc127SThomas Gleixner preempt_enable(); 83*e81dc127SThomas Gleixner #endif 84*e81dc127SThomas Gleixner return ret; 85*e81dc127SThomas Gleixner } 86*e81dc127SThomas Gleixner 87*e81dc127SThomas Gleixner static bool is_coretext(const struct core_text *ct, void *addr) 88*e81dc127SThomas Gleixner { 89*e81dc127SThomas Gleixner if (ct && within_coretext(ct, addr)) 90*e81dc127SThomas Gleixner return true; 91*e81dc127SThomas Gleixner if (within_coretext(&builtin_coretext, addr)) 92*e81dc127SThomas Gleixner return true; 93*e81dc127SThomas Gleixner return within_module_coretext(addr); 94*e81dc127SThomas Gleixner } 95*e81dc127SThomas Gleixner 96*e81dc127SThomas Gleixner static __init_or_module bool skip_addr(void *dest) 97*e81dc127SThomas Gleixner { 98*e81dc127SThomas Gleixner if (dest == error_entry) 99*e81dc127SThomas Gleixner return true; 100*e81dc127SThomas Gleixner if (dest == paranoid_entry) 101*e81dc127SThomas Gleixner return true; 102*e81dc127SThomas Gleixner if (dest == xen_error_entry) 103*e81dc127SThomas Gleixner return true; 104*e81dc127SThomas Gleixner /* Does FILL_RSB... */ 105*e81dc127SThomas Gleixner if (dest == __switch_to_asm) 106*e81dc127SThomas Gleixner return true; 107*e81dc127SThomas Gleixner /* Accounts directly */ 108*e81dc127SThomas Gleixner if (dest == ret_from_fork) 109*e81dc127SThomas Gleixner return true; 110*e81dc127SThomas Gleixner #ifdef CONFIG_HOTPLUG_CPU 111*e81dc127SThomas Gleixner if (dest == start_cpu0) 112*e81dc127SThomas Gleixner return true; 113*e81dc127SThomas Gleixner #endif 114*e81dc127SThomas Gleixner #ifdef CONFIG_FUNCTION_TRACER 115*e81dc127SThomas Gleixner if (dest == __fentry__) 116*e81dc127SThomas Gleixner return true; 117*e81dc127SThomas Gleixner #endif 118*e81dc127SThomas Gleixner #ifdef CONFIG_KEXEC_CORE 119*e81dc127SThomas Gleixner if (dest >= (void *)relocate_kernel && 120*e81dc127SThomas Gleixner dest < (void*)relocate_kernel + KEXEC_CONTROL_CODE_MAX_SIZE) 121*e81dc127SThomas Gleixner return true; 122*e81dc127SThomas Gleixner #endif 123*e81dc127SThomas Gleixner #ifdef CONFIG_XEN 124*e81dc127SThomas Gleixner if (dest >= (void *)hypercall_page && 125*e81dc127SThomas Gleixner dest < (void*)hypercall_page + PAGE_SIZE) 126*e81dc127SThomas Gleixner return true; 127*e81dc127SThomas Gleixner #endif 128*e81dc127SThomas Gleixner return false; 129*e81dc127SThomas Gleixner } 130*e81dc127SThomas Gleixner 131*e81dc127SThomas Gleixner static __init_or_module void *call_get_dest(void *addr) 132*e81dc127SThomas Gleixner { 133*e81dc127SThomas Gleixner struct insn insn; 134*e81dc127SThomas Gleixner void *dest; 135*e81dc127SThomas Gleixner int ret; 136*e81dc127SThomas Gleixner 137*e81dc127SThomas Gleixner ret = insn_decode_kernel(&insn, addr); 138*e81dc127SThomas Gleixner if (ret) 139*e81dc127SThomas Gleixner return ERR_PTR(ret); 140*e81dc127SThomas Gleixner 141*e81dc127SThomas Gleixner /* Patched out call? */ 142*e81dc127SThomas Gleixner if (insn.opcode.bytes[0] != CALL_INSN_OPCODE) 143*e81dc127SThomas Gleixner return NULL; 144*e81dc127SThomas Gleixner 145*e81dc127SThomas Gleixner dest = addr + insn.length + insn.immediate.value; 146*e81dc127SThomas Gleixner if (skip_addr(dest)) 147*e81dc127SThomas Gleixner return NULL; 148*e81dc127SThomas Gleixner return dest; 149*e81dc127SThomas Gleixner } 150*e81dc127SThomas Gleixner 151*e81dc127SThomas Gleixner static const u8 nops[] = { 152*e81dc127SThomas Gleixner 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 153*e81dc127SThomas Gleixner 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 154*e81dc127SThomas Gleixner 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 155*e81dc127SThomas Gleixner 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 156*e81dc127SThomas Gleixner }; 157*e81dc127SThomas Gleixner 158*e81dc127SThomas Gleixner static __init_or_module void *patch_dest(void *dest, bool direct) 159*e81dc127SThomas Gleixner { 160*e81dc127SThomas Gleixner unsigned int tsize = callthunk_desc.template_size; 161*e81dc127SThomas Gleixner u8 *pad = dest - tsize; 162*e81dc127SThomas Gleixner 163*e81dc127SThomas Gleixner /* Already patched? */ 164*e81dc127SThomas Gleixner if (!bcmp(pad, callthunk_desc.template, tsize)) 165*e81dc127SThomas Gleixner return pad; 166*e81dc127SThomas Gleixner 167*e81dc127SThomas Gleixner /* Ensure there are nops */ 168*e81dc127SThomas Gleixner if (bcmp(pad, nops, tsize)) { 169*e81dc127SThomas Gleixner pr_warn_once("Invalid padding area for %pS\n", dest); 170*e81dc127SThomas Gleixner return NULL; 171*e81dc127SThomas Gleixner } 172*e81dc127SThomas Gleixner 173*e81dc127SThomas Gleixner if (direct) 174*e81dc127SThomas Gleixner memcpy(pad, callthunk_desc.template, tsize); 175*e81dc127SThomas Gleixner else 176*e81dc127SThomas Gleixner text_poke_copy_locked(pad, callthunk_desc.template, tsize, true); 177*e81dc127SThomas Gleixner return pad; 178*e81dc127SThomas Gleixner } 179*e81dc127SThomas Gleixner 180*e81dc127SThomas Gleixner static __init_or_module void patch_call(void *addr, const struct core_text *ct) 181*e81dc127SThomas Gleixner { 182*e81dc127SThomas Gleixner void *pad, *dest; 183*e81dc127SThomas Gleixner u8 bytes[8]; 184*e81dc127SThomas Gleixner 185*e81dc127SThomas Gleixner if (!within_coretext(ct, addr)) 186*e81dc127SThomas Gleixner return; 187*e81dc127SThomas Gleixner 188*e81dc127SThomas Gleixner dest = call_get_dest(addr); 189*e81dc127SThomas Gleixner if (!dest || WARN_ON_ONCE(IS_ERR(dest))) 190*e81dc127SThomas Gleixner return; 191*e81dc127SThomas Gleixner 192*e81dc127SThomas Gleixner if (!is_coretext(ct, dest)) 193*e81dc127SThomas Gleixner return; 194*e81dc127SThomas Gleixner 195*e81dc127SThomas Gleixner pad = patch_dest(dest, within_coretext(ct, dest)); 196*e81dc127SThomas Gleixner if (!pad) 197*e81dc127SThomas Gleixner return; 198*e81dc127SThomas Gleixner 199*e81dc127SThomas Gleixner prdbg("Patch call at: %pS %px to %pS %px -> %px \n", addr, addr, 200*e81dc127SThomas Gleixner dest, dest, pad); 201*e81dc127SThomas Gleixner __text_gen_insn(bytes, CALL_INSN_OPCODE, addr, pad, CALL_INSN_SIZE); 202*e81dc127SThomas Gleixner text_poke_early(addr, bytes, CALL_INSN_SIZE); 203*e81dc127SThomas Gleixner } 204*e81dc127SThomas Gleixner 205*e81dc127SThomas Gleixner static __init_or_module void 206*e81dc127SThomas Gleixner patch_call_sites(s32 *start, s32 *end, const struct core_text *ct) 207*e81dc127SThomas Gleixner { 208*e81dc127SThomas Gleixner s32 *s; 209*e81dc127SThomas Gleixner 210*e81dc127SThomas Gleixner for (s = start; s < end; s++) 211*e81dc127SThomas Gleixner patch_call((void *)s + *s, ct); 212*e81dc127SThomas Gleixner } 213*e81dc127SThomas Gleixner 214*e81dc127SThomas Gleixner static __init_or_module void 215*e81dc127SThomas Gleixner patch_paravirt_call_sites(struct paravirt_patch_site *start, 216*e81dc127SThomas Gleixner struct paravirt_patch_site *end, 217*e81dc127SThomas Gleixner const struct core_text *ct) 218*e81dc127SThomas Gleixner { 219*e81dc127SThomas Gleixner struct paravirt_patch_site *p; 220*e81dc127SThomas Gleixner 221*e81dc127SThomas Gleixner for (p = start; p < end; p++) 222*e81dc127SThomas Gleixner patch_call(p->instr, ct); 223*e81dc127SThomas Gleixner } 224*e81dc127SThomas Gleixner 225*e81dc127SThomas Gleixner static __init_or_module void 226*e81dc127SThomas Gleixner callthunks_setup(struct callthunk_sites *cs, const struct core_text *ct) 227*e81dc127SThomas Gleixner { 228*e81dc127SThomas Gleixner prdbg("Patching call sites %s\n", ct->name); 229*e81dc127SThomas Gleixner patch_call_sites(cs->call_start, cs->call_end, ct); 230*e81dc127SThomas Gleixner patch_paravirt_call_sites(cs->pv_start, cs->pv_end, ct); 231*e81dc127SThomas Gleixner prdbg("Patching call sites done%s\n", ct->name); 232*e81dc127SThomas Gleixner } 233*e81dc127SThomas Gleixner 234*e81dc127SThomas Gleixner void __init callthunks_patch_builtin_calls(void) 235*e81dc127SThomas Gleixner { 236*e81dc127SThomas Gleixner struct callthunk_sites cs = { 237*e81dc127SThomas Gleixner .call_start = __call_sites, 238*e81dc127SThomas Gleixner .call_end = __call_sites_end, 239*e81dc127SThomas Gleixner .pv_start = __parainstructions, 240*e81dc127SThomas Gleixner .pv_end = __parainstructions_end 241*e81dc127SThomas Gleixner }; 242*e81dc127SThomas Gleixner 243*e81dc127SThomas Gleixner if (!cpu_feature_enabled(X86_FEATURE_CALL_DEPTH)) 244*e81dc127SThomas Gleixner return; 245*e81dc127SThomas Gleixner 246*e81dc127SThomas Gleixner pr_info("Setting up call depth tracking\n"); 247*e81dc127SThomas Gleixner mutex_lock(&text_mutex); 248*e81dc127SThomas Gleixner callthunks_setup(&cs, &builtin_coretext); 249*e81dc127SThomas Gleixner thunks_initialized = true; 250*e81dc127SThomas Gleixner mutex_unlock(&text_mutex); 251*e81dc127SThomas Gleixner } 252