1e81dc127SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 2e81dc127SThomas Gleixner 3e81dc127SThomas Gleixner #define pr_fmt(fmt) "callthunks: " fmt 4e81dc127SThomas Gleixner 5e81dc127SThomas Gleixner #include <linux/kallsyms.h> 6e81dc127SThomas Gleixner #include <linux/memory.h> 7e81dc127SThomas Gleixner #include <linux/moduleloader.h> 8e81dc127SThomas Gleixner 9e81dc127SThomas Gleixner #include <asm/alternative.h> 10*bbaceb18SThomas Gleixner #include <asm/asm-offsets.h> 11e81dc127SThomas Gleixner #include <asm/cpu.h> 12e81dc127SThomas Gleixner #include <asm/ftrace.h> 13e81dc127SThomas Gleixner #include <asm/insn.h> 14e81dc127SThomas Gleixner #include <asm/kexec.h> 15e81dc127SThomas Gleixner #include <asm/nospec-branch.h> 16e81dc127SThomas Gleixner #include <asm/paravirt.h> 17e81dc127SThomas Gleixner #include <asm/sections.h> 18e81dc127SThomas Gleixner #include <asm/switch_to.h> 19e81dc127SThomas Gleixner #include <asm/sync_core.h> 20e81dc127SThomas Gleixner #include <asm/text-patching.h> 21e81dc127SThomas Gleixner #include <asm/xen/hypercall.h> 22e81dc127SThomas Gleixner 23e81dc127SThomas Gleixner static int __initdata_or_module debug_callthunks; 24e81dc127SThomas Gleixner 25e81dc127SThomas Gleixner #define prdbg(fmt, args...) \ 26e81dc127SThomas Gleixner do { \ 27e81dc127SThomas Gleixner if (debug_callthunks) \ 28e81dc127SThomas Gleixner printk(KERN_DEBUG pr_fmt(fmt), ##args); \ 29e81dc127SThomas Gleixner } while(0) 30e81dc127SThomas Gleixner 31e81dc127SThomas Gleixner static int __init debug_thunks(char *str) 32e81dc127SThomas Gleixner { 33e81dc127SThomas Gleixner debug_callthunks = 1; 34e81dc127SThomas Gleixner return 1; 35e81dc127SThomas Gleixner } 36e81dc127SThomas Gleixner __setup("debug-callthunks", debug_thunks); 37e81dc127SThomas Gleixner 38e81dc127SThomas Gleixner extern s32 __call_sites[], __call_sites_end[]; 39e81dc127SThomas Gleixner 40e81dc127SThomas Gleixner struct thunk_desc { 41e81dc127SThomas Gleixner void *template; 42e81dc127SThomas Gleixner unsigned int template_size; 43e81dc127SThomas Gleixner }; 44e81dc127SThomas Gleixner 45e81dc127SThomas Gleixner struct core_text { 46e81dc127SThomas Gleixner unsigned long base; 47e81dc127SThomas Gleixner unsigned long end; 48e81dc127SThomas Gleixner const char *name; 49e81dc127SThomas Gleixner }; 50e81dc127SThomas Gleixner 51e81dc127SThomas Gleixner static bool thunks_initialized __ro_after_init; 52e81dc127SThomas Gleixner 53e81dc127SThomas Gleixner static const struct core_text builtin_coretext = { 54e81dc127SThomas Gleixner .base = (unsigned long)_text, 55e81dc127SThomas Gleixner .end = (unsigned long)_etext, 56e81dc127SThomas Gleixner .name = "builtin", 57e81dc127SThomas Gleixner }; 58e81dc127SThomas Gleixner 59*bbaceb18SThomas Gleixner asm ( 60*bbaceb18SThomas Gleixner ".pushsection .rodata \n" 61*bbaceb18SThomas Gleixner ".global skl_call_thunk_template \n" 62*bbaceb18SThomas Gleixner "skl_call_thunk_template: \n" 63*bbaceb18SThomas Gleixner __stringify(INCREMENT_CALL_DEPTH)" \n" 64*bbaceb18SThomas Gleixner ".global skl_call_thunk_tail \n" 65*bbaceb18SThomas Gleixner "skl_call_thunk_tail: \n" 66*bbaceb18SThomas Gleixner ".popsection \n" 67*bbaceb18SThomas Gleixner ); 68*bbaceb18SThomas Gleixner 69*bbaceb18SThomas Gleixner extern u8 skl_call_thunk_template[]; 70*bbaceb18SThomas Gleixner extern u8 skl_call_thunk_tail[]; 71*bbaceb18SThomas Gleixner 72*bbaceb18SThomas Gleixner #define SKL_TMPL_SIZE \ 73*bbaceb18SThomas Gleixner ((unsigned int)(skl_call_thunk_tail - skl_call_thunk_template)) 74e81dc127SThomas Gleixner 75e81dc127SThomas Gleixner extern void error_entry(void); 76e81dc127SThomas Gleixner extern void xen_error_entry(void); 77e81dc127SThomas Gleixner extern void paranoid_entry(void); 78e81dc127SThomas Gleixner 79e81dc127SThomas Gleixner static inline bool within_coretext(const struct core_text *ct, void *addr) 80e81dc127SThomas Gleixner { 81e81dc127SThomas Gleixner unsigned long p = (unsigned long)addr; 82e81dc127SThomas Gleixner 83e81dc127SThomas Gleixner return ct->base <= p && p < ct->end; 84e81dc127SThomas Gleixner } 85e81dc127SThomas Gleixner 86e81dc127SThomas Gleixner static inline bool within_module_coretext(void *addr) 87e81dc127SThomas Gleixner { 88e81dc127SThomas Gleixner bool ret = false; 89e81dc127SThomas Gleixner 90e81dc127SThomas Gleixner #ifdef CONFIG_MODULES 91e81dc127SThomas Gleixner struct module *mod; 92e81dc127SThomas Gleixner 93e81dc127SThomas Gleixner preempt_disable(); 94e81dc127SThomas Gleixner mod = __module_address((unsigned long)addr); 95e81dc127SThomas Gleixner if (mod && within_module_core((unsigned long)addr, mod)) 96e81dc127SThomas Gleixner ret = true; 97e81dc127SThomas Gleixner preempt_enable(); 98e81dc127SThomas Gleixner #endif 99e81dc127SThomas Gleixner return ret; 100e81dc127SThomas Gleixner } 101e81dc127SThomas Gleixner 102e81dc127SThomas Gleixner static bool is_coretext(const struct core_text *ct, void *addr) 103e81dc127SThomas Gleixner { 104e81dc127SThomas Gleixner if (ct && within_coretext(ct, addr)) 105e81dc127SThomas Gleixner return true; 106e81dc127SThomas Gleixner if (within_coretext(&builtin_coretext, addr)) 107e81dc127SThomas Gleixner return true; 108e81dc127SThomas Gleixner return within_module_coretext(addr); 109e81dc127SThomas Gleixner } 110e81dc127SThomas Gleixner 111e81dc127SThomas Gleixner static __init_or_module bool skip_addr(void *dest) 112e81dc127SThomas Gleixner { 113e81dc127SThomas Gleixner if (dest == error_entry) 114e81dc127SThomas Gleixner return true; 115e81dc127SThomas Gleixner if (dest == paranoid_entry) 116e81dc127SThomas Gleixner return true; 117e81dc127SThomas Gleixner if (dest == xen_error_entry) 118e81dc127SThomas Gleixner return true; 119e81dc127SThomas Gleixner /* Does FILL_RSB... */ 120e81dc127SThomas Gleixner if (dest == __switch_to_asm) 121e81dc127SThomas Gleixner return true; 122e81dc127SThomas Gleixner /* Accounts directly */ 123e81dc127SThomas Gleixner if (dest == ret_from_fork) 124e81dc127SThomas Gleixner return true; 125e81dc127SThomas Gleixner #ifdef CONFIG_HOTPLUG_CPU 126e81dc127SThomas Gleixner if (dest == start_cpu0) 127e81dc127SThomas Gleixner return true; 128e81dc127SThomas Gleixner #endif 129e81dc127SThomas Gleixner #ifdef CONFIG_FUNCTION_TRACER 130e81dc127SThomas Gleixner if (dest == __fentry__) 131e81dc127SThomas Gleixner return true; 132e81dc127SThomas Gleixner #endif 133e81dc127SThomas Gleixner #ifdef CONFIG_KEXEC_CORE 134e81dc127SThomas Gleixner if (dest >= (void *)relocate_kernel && 135e81dc127SThomas Gleixner dest < (void*)relocate_kernel + KEXEC_CONTROL_CODE_MAX_SIZE) 136e81dc127SThomas Gleixner return true; 137e81dc127SThomas Gleixner #endif 138e81dc127SThomas Gleixner #ifdef CONFIG_XEN 139e81dc127SThomas Gleixner if (dest >= (void *)hypercall_page && 140e81dc127SThomas Gleixner dest < (void*)hypercall_page + PAGE_SIZE) 141e81dc127SThomas Gleixner return true; 142e81dc127SThomas Gleixner #endif 143e81dc127SThomas Gleixner return false; 144e81dc127SThomas Gleixner } 145e81dc127SThomas Gleixner 146e81dc127SThomas Gleixner static __init_or_module void *call_get_dest(void *addr) 147e81dc127SThomas Gleixner { 148e81dc127SThomas Gleixner struct insn insn; 149e81dc127SThomas Gleixner void *dest; 150e81dc127SThomas Gleixner int ret; 151e81dc127SThomas Gleixner 152e81dc127SThomas Gleixner ret = insn_decode_kernel(&insn, addr); 153e81dc127SThomas Gleixner if (ret) 154e81dc127SThomas Gleixner return ERR_PTR(ret); 155e81dc127SThomas Gleixner 156e81dc127SThomas Gleixner /* Patched out call? */ 157e81dc127SThomas Gleixner if (insn.opcode.bytes[0] != CALL_INSN_OPCODE) 158e81dc127SThomas Gleixner return NULL; 159e81dc127SThomas Gleixner 160e81dc127SThomas Gleixner dest = addr + insn.length + insn.immediate.value; 161e81dc127SThomas Gleixner if (skip_addr(dest)) 162e81dc127SThomas Gleixner return NULL; 163e81dc127SThomas Gleixner return dest; 164e81dc127SThomas Gleixner } 165e81dc127SThomas Gleixner 166e81dc127SThomas Gleixner static const u8 nops[] = { 167e81dc127SThomas Gleixner 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 168e81dc127SThomas Gleixner 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 169e81dc127SThomas Gleixner 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 170e81dc127SThomas Gleixner 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 171e81dc127SThomas Gleixner }; 172e81dc127SThomas Gleixner 173e81dc127SThomas Gleixner static __init_or_module void *patch_dest(void *dest, bool direct) 174e81dc127SThomas Gleixner { 175*bbaceb18SThomas Gleixner unsigned int tsize = SKL_TMPL_SIZE; 176e81dc127SThomas Gleixner u8 *pad = dest - tsize; 177e81dc127SThomas Gleixner 178e81dc127SThomas Gleixner /* Already patched? */ 179*bbaceb18SThomas Gleixner if (!bcmp(pad, skl_call_thunk_template, tsize)) 180e81dc127SThomas Gleixner return pad; 181e81dc127SThomas Gleixner 182e81dc127SThomas Gleixner /* Ensure there are nops */ 183e81dc127SThomas Gleixner if (bcmp(pad, nops, tsize)) { 184e81dc127SThomas Gleixner pr_warn_once("Invalid padding area for %pS\n", dest); 185e81dc127SThomas Gleixner return NULL; 186e81dc127SThomas Gleixner } 187e81dc127SThomas Gleixner 188e81dc127SThomas Gleixner if (direct) 189*bbaceb18SThomas Gleixner memcpy(pad, skl_call_thunk_template, tsize); 190e81dc127SThomas Gleixner else 191*bbaceb18SThomas Gleixner text_poke_copy_locked(pad, skl_call_thunk_template, tsize, true); 192e81dc127SThomas Gleixner return pad; 193e81dc127SThomas Gleixner } 194e81dc127SThomas Gleixner 195e81dc127SThomas Gleixner static __init_or_module void patch_call(void *addr, const struct core_text *ct) 196e81dc127SThomas Gleixner { 197e81dc127SThomas Gleixner void *pad, *dest; 198e81dc127SThomas Gleixner u8 bytes[8]; 199e81dc127SThomas Gleixner 200e81dc127SThomas Gleixner if (!within_coretext(ct, addr)) 201e81dc127SThomas Gleixner return; 202e81dc127SThomas Gleixner 203e81dc127SThomas Gleixner dest = call_get_dest(addr); 204e81dc127SThomas Gleixner if (!dest || WARN_ON_ONCE(IS_ERR(dest))) 205e81dc127SThomas Gleixner return; 206e81dc127SThomas Gleixner 207e81dc127SThomas Gleixner if (!is_coretext(ct, dest)) 208e81dc127SThomas Gleixner return; 209e81dc127SThomas Gleixner 210e81dc127SThomas Gleixner pad = patch_dest(dest, within_coretext(ct, dest)); 211e81dc127SThomas Gleixner if (!pad) 212e81dc127SThomas Gleixner return; 213e81dc127SThomas Gleixner 214e81dc127SThomas Gleixner prdbg("Patch call at: %pS %px to %pS %px -> %px \n", addr, addr, 215e81dc127SThomas Gleixner dest, dest, pad); 216e81dc127SThomas Gleixner __text_gen_insn(bytes, CALL_INSN_OPCODE, addr, pad, CALL_INSN_SIZE); 217e81dc127SThomas Gleixner text_poke_early(addr, bytes, CALL_INSN_SIZE); 218e81dc127SThomas Gleixner } 219e81dc127SThomas Gleixner 220e81dc127SThomas Gleixner static __init_or_module void 221e81dc127SThomas Gleixner patch_call_sites(s32 *start, s32 *end, const struct core_text *ct) 222e81dc127SThomas Gleixner { 223e81dc127SThomas Gleixner s32 *s; 224e81dc127SThomas Gleixner 225e81dc127SThomas Gleixner for (s = start; s < end; s++) 226e81dc127SThomas Gleixner patch_call((void *)s + *s, ct); 227e81dc127SThomas Gleixner } 228e81dc127SThomas Gleixner 229e81dc127SThomas Gleixner static __init_or_module void 230e81dc127SThomas Gleixner patch_paravirt_call_sites(struct paravirt_patch_site *start, 231e81dc127SThomas Gleixner struct paravirt_patch_site *end, 232e81dc127SThomas Gleixner const struct core_text *ct) 233e81dc127SThomas Gleixner { 234e81dc127SThomas Gleixner struct paravirt_patch_site *p; 235e81dc127SThomas Gleixner 236e81dc127SThomas Gleixner for (p = start; p < end; p++) 237e81dc127SThomas Gleixner patch_call(p->instr, ct); 238e81dc127SThomas Gleixner } 239e81dc127SThomas Gleixner 240e81dc127SThomas Gleixner static __init_or_module void 241e81dc127SThomas Gleixner callthunks_setup(struct callthunk_sites *cs, const struct core_text *ct) 242e81dc127SThomas Gleixner { 243e81dc127SThomas Gleixner prdbg("Patching call sites %s\n", ct->name); 244e81dc127SThomas Gleixner patch_call_sites(cs->call_start, cs->call_end, ct); 245e81dc127SThomas Gleixner patch_paravirt_call_sites(cs->pv_start, cs->pv_end, ct); 246e81dc127SThomas Gleixner prdbg("Patching call sites done%s\n", ct->name); 247e81dc127SThomas Gleixner } 248e81dc127SThomas Gleixner 249e81dc127SThomas Gleixner void __init callthunks_patch_builtin_calls(void) 250e81dc127SThomas Gleixner { 251e81dc127SThomas Gleixner struct callthunk_sites cs = { 252e81dc127SThomas Gleixner .call_start = __call_sites, 253e81dc127SThomas Gleixner .call_end = __call_sites_end, 254e81dc127SThomas Gleixner .pv_start = __parainstructions, 255e81dc127SThomas Gleixner .pv_end = __parainstructions_end 256e81dc127SThomas Gleixner }; 257e81dc127SThomas Gleixner 258e81dc127SThomas Gleixner if (!cpu_feature_enabled(X86_FEATURE_CALL_DEPTH)) 259e81dc127SThomas Gleixner return; 260e81dc127SThomas Gleixner 261e81dc127SThomas Gleixner pr_info("Setting up call depth tracking\n"); 262e81dc127SThomas Gleixner mutex_lock(&text_mutex); 263e81dc127SThomas Gleixner callthunks_setup(&cs, &builtin_coretext); 264e81dc127SThomas Gleixner thunks_initialized = true; 265e81dc127SThomas Gleixner mutex_unlock(&text_mutex); 266e81dc127SThomas Gleixner } 267eaf44c81SThomas Gleixner 268eaf44c81SThomas Gleixner #ifdef CONFIG_MODULES 269eaf44c81SThomas Gleixner void noinline callthunks_patch_module_calls(struct callthunk_sites *cs, 270eaf44c81SThomas Gleixner struct module *mod) 271eaf44c81SThomas Gleixner { 272eaf44c81SThomas Gleixner struct core_text ct = { 273eaf44c81SThomas Gleixner .base = (unsigned long)mod->core_layout.base, 274eaf44c81SThomas Gleixner .end = (unsigned long)mod->core_layout.base + mod->core_layout.size, 275eaf44c81SThomas Gleixner .name = mod->name, 276eaf44c81SThomas Gleixner }; 277eaf44c81SThomas Gleixner 278eaf44c81SThomas Gleixner if (!thunks_initialized) 279eaf44c81SThomas Gleixner return; 280eaf44c81SThomas Gleixner 281eaf44c81SThomas Gleixner mutex_lock(&text_mutex); 282eaf44c81SThomas Gleixner callthunks_setup(cs, &ct); 283eaf44c81SThomas Gleixner mutex_unlock(&text_mutex); 284eaf44c81SThomas Gleixner } 285eaf44c81SThomas Gleixner #endif /* CONFIG_MODULES */ 286