1e81dc127SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 2e81dc127SThomas Gleixner 3e81dc127SThomas Gleixner #define pr_fmt(fmt) "callthunks: " fmt 4e81dc127SThomas Gleixner 5e81dc127SThomas Gleixner #include <linux/kallsyms.h> 6e81dc127SThomas Gleixner #include <linux/memory.h> 7e81dc127SThomas Gleixner #include <linux/moduleloader.h> 8e81dc127SThomas Gleixner 9e81dc127SThomas Gleixner #include <asm/alternative.h> 10e81dc127SThomas Gleixner #include <asm/cpu.h> 11e81dc127SThomas Gleixner #include <asm/ftrace.h> 12e81dc127SThomas Gleixner #include <asm/insn.h> 13e81dc127SThomas Gleixner #include <asm/kexec.h> 14e81dc127SThomas Gleixner #include <asm/nospec-branch.h> 15e81dc127SThomas Gleixner #include <asm/paravirt.h> 16e81dc127SThomas Gleixner #include <asm/sections.h> 17e81dc127SThomas Gleixner #include <asm/switch_to.h> 18e81dc127SThomas Gleixner #include <asm/sync_core.h> 19e81dc127SThomas Gleixner #include <asm/text-patching.h> 20e81dc127SThomas Gleixner #include <asm/xen/hypercall.h> 21e81dc127SThomas Gleixner 22e81dc127SThomas Gleixner static int __initdata_or_module debug_callthunks; 23e81dc127SThomas Gleixner 24e81dc127SThomas Gleixner #define prdbg(fmt, args...) \ 25e81dc127SThomas Gleixner do { \ 26e81dc127SThomas Gleixner if (debug_callthunks) \ 27e81dc127SThomas Gleixner printk(KERN_DEBUG pr_fmt(fmt), ##args); \ 28e81dc127SThomas Gleixner } while(0) 29e81dc127SThomas Gleixner 30e81dc127SThomas Gleixner static int __init debug_thunks(char *str) 31e81dc127SThomas Gleixner { 32e81dc127SThomas Gleixner debug_callthunks = 1; 33e81dc127SThomas Gleixner return 1; 34e81dc127SThomas Gleixner } 35e81dc127SThomas Gleixner __setup("debug-callthunks", debug_thunks); 36e81dc127SThomas Gleixner 37e81dc127SThomas Gleixner extern s32 __call_sites[], __call_sites_end[]; 38e81dc127SThomas Gleixner 39e81dc127SThomas Gleixner struct thunk_desc { 40e81dc127SThomas Gleixner void *template; 41e81dc127SThomas Gleixner unsigned int template_size; 42e81dc127SThomas Gleixner }; 43e81dc127SThomas Gleixner 44e81dc127SThomas Gleixner struct core_text { 45e81dc127SThomas Gleixner unsigned long base; 46e81dc127SThomas Gleixner unsigned long end; 47e81dc127SThomas Gleixner const char *name; 48e81dc127SThomas Gleixner }; 49e81dc127SThomas Gleixner 50e81dc127SThomas Gleixner static bool thunks_initialized __ro_after_init; 51e81dc127SThomas Gleixner 52e81dc127SThomas Gleixner static const struct core_text builtin_coretext = { 53e81dc127SThomas Gleixner .base = (unsigned long)_text, 54e81dc127SThomas Gleixner .end = (unsigned long)_etext, 55e81dc127SThomas Gleixner .name = "builtin", 56e81dc127SThomas Gleixner }; 57e81dc127SThomas Gleixner 58e81dc127SThomas Gleixner static struct thunk_desc callthunk_desc __ro_after_init; 59e81dc127SThomas Gleixner 60e81dc127SThomas Gleixner extern void error_entry(void); 61e81dc127SThomas Gleixner extern void xen_error_entry(void); 62e81dc127SThomas Gleixner extern void paranoid_entry(void); 63e81dc127SThomas Gleixner 64e81dc127SThomas Gleixner static inline bool within_coretext(const struct core_text *ct, void *addr) 65e81dc127SThomas Gleixner { 66e81dc127SThomas Gleixner unsigned long p = (unsigned long)addr; 67e81dc127SThomas Gleixner 68e81dc127SThomas Gleixner return ct->base <= p && p < ct->end; 69e81dc127SThomas Gleixner } 70e81dc127SThomas Gleixner 71e81dc127SThomas Gleixner static inline bool within_module_coretext(void *addr) 72e81dc127SThomas Gleixner { 73e81dc127SThomas Gleixner bool ret = false; 74e81dc127SThomas Gleixner 75e81dc127SThomas Gleixner #ifdef CONFIG_MODULES 76e81dc127SThomas Gleixner struct module *mod; 77e81dc127SThomas Gleixner 78e81dc127SThomas Gleixner preempt_disable(); 79e81dc127SThomas Gleixner mod = __module_address((unsigned long)addr); 80e81dc127SThomas Gleixner if (mod && within_module_core((unsigned long)addr, mod)) 81e81dc127SThomas Gleixner ret = true; 82e81dc127SThomas Gleixner preempt_enable(); 83e81dc127SThomas Gleixner #endif 84e81dc127SThomas Gleixner return ret; 85e81dc127SThomas Gleixner } 86e81dc127SThomas Gleixner 87e81dc127SThomas Gleixner static bool is_coretext(const struct core_text *ct, void *addr) 88e81dc127SThomas Gleixner { 89e81dc127SThomas Gleixner if (ct && within_coretext(ct, addr)) 90e81dc127SThomas Gleixner return true; 91e81dc127SThomas Gleixner if (within_coretext(&builtin_coretext, addr)) 92e81dc127SThomas Gleixner return true; 93e81dc127SThomas Gleixner return within_module_coretext(addr); 94e81dc127SThomas Gleixner } 95e81dc127SThomas Gleixner 96e81dc127SThomas Gleixner static __init_or_module bool skip_addr(void *dest) 97e81dc127SThomas Gleixner { 98e81dc127SThomas Gleixner if (dest == error_entry) 99e81dc127SThomas Gleixner return true; 100e81dc127SThomas Gleixner if (dest == paranoid_entry) 101e81dc127SThomas Gleixner return true; 102e81dc127SThomas Gleixner if (dest == xen_error_entry) 103e81dc127SThomas Gleixner return true; 104e81dc127SThomas Gleixner /* Does FILL_RSB... */ 105e81dc127SThomas Gleixner if (dest == __switch_to_asm) 106e81dc127SThomas Gleixner return true; 107e81dc127SThomas Gleixner /* Accounts directly */ 108e81dc127SThomas Gleixner if (dest == ret_from_fork) 109e81dc127SThomas Gleixner return true; 110e81dc127SThomas Gleixner #ifdef CONFIG_HOTPLUG_CPU 111e81dc127SThomas Gleixner if (dest == start_cpu0) 112e81dc127SThomas Gleixner return true; 113e81dc127SThomas Gleixner #endif 114e81dc127SThomas Gleixner #ifdef CONFIG_FUNCTION_TRACER 115e81dc127SThomas Gleixner if (dest == __fentry__) 116e81dc127SThomas Gleixner return true; 117e81dc127SThomas Gleixner #endif 118e81dc127SThomas Gleixner #ifdef CONFIG_KEXEC_CORE 119e81dc127SThomas Gleixner if (dest >= (void *)relocate_kernel && 120e81dc127SThomas Gleixner dest < (void*)relocate_kernel + KEXEC_CONTROL_CODE_MAX_SIZE) 121e81dc127SThomas Gleixner return true; 122e81dc127SThomas Gleixner #endif 123e81dc127SThomas Gleixner #ifdef CONFIG_XEN 124e81dc127SThomas Gleixner if (dest >= (void *)hypercall_page && 125e81dc127SThomas Gleixner dest < (void*)hypercall_page + PAGE_SIZE) 126e81dc127SThomas Gleixner return true; 127e81dc127SThomas Gleixner #endif 128e81dc127SThomas Gleixner return false; 129e81dc127SThomas Gleixner } 130e81dc127SThomas Gleixner 131e81dc127SThomas Gleixner static __init_or_module void *call_get_dest(void *addr) 132e81dc127SThomas Gleixner { 133e81dc127SThomas Gleixner struct insn insn; 134e81dc127SThomas Gleixner void *dest; 135e81dc127SThomas Gleixner int ret; 136e81dc127SThomas Gleixner 137e81dc127SThomas Gleixner ret = insn_decode_kernel(&insn, addr); 138e81dc127SThomas Gleixner if (ret) 139e81dc127SThomas Gleixner return ERR_PTR(ret); 140e81dc127SThomas Gleixner 141e81dc127SThomas Gleixner /* Patched out call? */ 142e81dc127SThomas Gleixner if (insn.opcode.bytes[0] != CALL_INSN_OPCODE) 143e81dc127SThomas Gleixner return NULL; 144e81dc127SThomas Gleixner 145e81dc127SThomas Gleixner dest = addr + insn.length + insn.immediate.value; 146e81dc127SThomas Gleixner if (skip_addr(dest)) 147e81dc127SThomas Gleixner return NULL; 148e81dc127SThomas Gleixner return dest; 149e81dc127SThomas Gleixner } 150e81dc127SThomas Gleixner 151e81dc127SThomas Gleixner static const u8 nops[] = { 152e81dc127SThomas Gleixner 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 153e81dc127SThomas Gleixner 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 154e81dc127SThomas Gleixner 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 155e81dc127SThomas Gleixner 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 156e81dc127SThomas Gleixner }; 157e81dc127SThomas Gleixner 158e81dc127SThomas Gleixner static __init_or_module void *patch_dest(void *dest, bool direct) 159e81dc127SThomas Gleixner { 160e81dc127SThomas Gleixner unsigned int tsize = callthunk_desc.template_size; 161e81dc127SThomas Gleixner u8 *pad = dest - tsize; 162e81dc127SThomas Gleixner 163e81dc127SThomas Gleixner /* Already patched? */ 164e81dc127SThomas Gleixner if (!bcmp(pad, callthunk_desc.template, tsize)) 165e81dc127SThomas Gleixner return pad; 166e81dc127SThomas Gleixner 167e81dc127SThomas Gleixner /* Ensure there are nops */ 168e81dc127SThomas Gleixner if (bcmp(pad, nops, tsize)) { 169e81dc127SThomas Gleixner pr_warn_once("Invalid padding area for %pS\n", dest); 170e81dc127SThomas Gleixner return NULL; 171e81dc127SThomas Gleixner } 172e81dc127SThomas Gleixner 173e81dc127SThomas Gleixner if (direct) 174e81dc127SThomas Gleixner memcpy(pad, callthunk_desc.template, tsize); 175e81dc127SThomas Gleixner else 176e81dc127SThomas Gleixner text_poke_copy_locked(pad, callthunk_desc.template, tsize, true); 177e81dc127SThomas Gleixner return pad; 178e81dc127SThomas Gleixner } 179e81dc127SThomas Gleixner 180e81dc127SThomas Gleixner static __init_or_module void patch_call(void *addr, const struct core_text *ct) 181e81dc127SThomas Gleixner { 182e81dc127SThomas Gleixner void *pad, *dest; 183e81dc127SThomas Gleixner u8 bytes[8]; 184e81dc127SThomas Gleixner 185e81dc127SThomas Gleixner if (!within_coretext(ct, addr)) 186e81dc127SThomas Gleixner return; 187e81dc127SThomas Gleixner 188e81dc127SThomas Gleixner dest = call_get_dest(addr); 189e81dc127SThomas Gleixner if (!dest || WARN_ON_ONCE(IS_ERR(dest))) 190e81dc127SThomas Gleixner return; 191e81dc127SThomas Gleixner 192e81dc127SThomas Gleixner if (!is_coretext(ct, dest)) 193e81dc127SThomas Gleixner return; 194e81dc127SThomas Gleixner 195e81dc127SThomas Gleixner pad = patch_dest(dest, within_coretext(ct, dest)); 196e81dc127SThomas Gleixner if (!pad) 197e81dc127SThomas Gleixner return; 198e81dc127SThomas Gleixner 199e81dc127SThomas Gleixner prdbg("Patch call at: %pS %px to %pS %px -> %px \n", addr, addr, 200e81dc127SThomas Gleixner dest, dest, pad); 201e81dc127SThomas Gleixner __text_gen_insn(bytes, CALL_INSN_OPCODE, addr, pad, CALL_INSN_SIZE); 202e81dc127SThomas Gleixner text_poke_early(addr, bytes, CALL_INSN_SIZE); 203e81dc127SThomas Gleixner } 204e81dc127SThomas Gleixner 205e81dc127SThomas Gleixner static __init_or_module void 206e81dc127SThomas Gleixner patch_call_sites(s32 *start, s32 *end, const struct core_text *ct) 207e81dc127SThomas Gleixner { 208e81dc127SThomas Gleixner s32 *s; 209e81dc127SThomas Gleixner 210e81dc127SThomas Gleixner for (s = start; s < end; s++) 211e81dc127SThomas Gleixner patch_call((void *)s + *s, ct); 212e81dc127SThomas Gleixner } 213e81dc127SThomas Gleixner 214e81dc127SThomas Gleixner static __init_or_module void 215e81dc127SThomas Gleixner patch_paravirt_call_sites(struct paravirt_patch_site *start, 216e81dc127SThomas Gleixner struct paravirt_patch_site *end, 217e81dc127SThomas Gleixner const struct core_text *ct) 218e81dc127SThomas Gleixner { 219e81dc127SThomas Gleixner struct paravirt_patch_site *p; 220e81dc127SThomas Gleixner 221e81dc127SThomas Gleixner for (p = start; p < end; p++) 222e81dc127SThomas Gleixner patch_call(p->instr, ct); 223e81dc127SThomas Gleixner } 224e81dc127SThomas Gleixner 225e81dc127SThomas Gleixner static __init_or_module void 226e81dc127SThomas Gleixner callthunks_setup(struct callthunk_sites *cs, const struct core_text *ct) 227e81dc127SThomas Gleixner { 228e81dc127SThomas Gleixner prdbg("Patching call sites %s\n", ct->name); 229e81dc127SThomas Gleixner patch_call_sites(cs->call_start, cs->call_end, ct); 230e81dc127SThomas Gleixner patch_paravirt_call_sites(cs->pv_start, cs->pv_end, ct); 231e81dc127SThomas Gleixner prdbg("Patching call sites done%s\n", ct->name); 232e81dc127SThomas Gleixner } 233e81dc127SThomas Gleixner 234e81dc127SThomas Gleixner void __init callthunks_patch_builtin_calls(void) 235e81dc127SThomas Gleixner { 236e81dc127SThomas Gleixner struct callthunk_sites cs = { 237e81dc127SThomas Gleixner .call_start = __call_sites, 238e81dc127SThomas Gleixner .call_end = __call_sites_end, 239e81dc127SThomas Gleixner .pv_start = __parainstructions, 240e81dc127SThomas Gleixner .pv_end = __parainstructions_end 241e81dc127SThomas Gleixner }; 242e81dc127SThomas Gleixner 243e81dc127SThomas Gleixner if (!cpu_feature_enabled(X86_FEATURE_CALL_DEPTH)) 244e81dc127SThomas Gleixner return; 245e81dc127SThomas Gleixner 246e81dc127SThomas Gleixner pr_info("Setting up call depth tracking\n"); 247e81dc127SThomas Gleixner mutex_lock(&text_mutex); 248e81dc127SThomas Gleixner callthunks_setup(&cs, &builtin_coretext); 249e81dc127SThomas Gleixner thunks_initialized = true; 250e81dc127SThomas Gleixner mutex_unlock(&text_mutex); 251e81dc127SThomas Gleixner } 252*eaf44c81SThomas Gleixner 253*eaf44c81SThomas Gleixner #ifdef CONFIG_MODULES 254*eaf44c81SThomas Gleixner void noinline callthunks_patch_module_calls(struct callthunk_sites *cs, 255*eaf44c81SThomas Gleixner struct module *mod) 256*eaf44c81SThomas Gleixner { 257*eaf44c81SThomas Gleixner struct core_text ct = { 258*eaf44c81SThomas Gleixner .base = (unsigned long)mod->core_layout.base, 259*eaf44c81SThomas Gleixner .end = (unsigned long)mod->core_layout.base + mod->core_layout.size, 260*eaf44c81SThomas Gleixner .name = mod->name, 261*eaf44c81SThomas Gleixner }; 262*eaf44c81SThomas Gleixner 263*eaf44c81SThomas Gleixner if (!thunks_initialized) 264*eaf44c81SThomas Gleixner return; 265*eaf44c81SThomas Gleixner 266*eaf44c81SThomas Gleixner mutex_lock(&text_mutex); 267*eaf44c81SThomas Gleixner callthunks_setup(cs, &ct); 268*eaf44c81SThomas Gleixner mutex_unlock(&text_mutex); 269*eaf44c81SThomas Gleixner } 270*eaf44c81SThomas Gleixner #endif /* CONFIG_MODULES */ 271