1e81dc127SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 2e81dc127SThomas Gleixner 3e81dc127SThomas Gleixner #define pr_fmt(fmt) "callthunks: " fmt 4e81dc127SThomas Gleixner 5*f5c1bb2aSThomas Gleixner #include <linux/debugfs.h> 6e81dc127SThomas Gleixner #include <linux/kallsyms.h> 7e81dc127SThomas Gleixner #include <linux/memory.h> 8e81dc127SThomas Gleixner #include <linux/moduleloader.h> 9e81dc127SThomas Gleixner 10e81dc127SThomas Gleixner #include <asm/alternative.h> 11bbaceb18SThomas Gleixner #include <asm/asm-offsets.h> 12e81dc127SThomas Gleixner #include <asm/cpu.h> 13e81dc127SThomas Gleixner #include <asm/ftrace.h> 14e81dc127SThomas Gleixner #include <asm/insn.h> 15e81dc127SThomas Gleixner #include <asm/kexec.h> 16e81dc127SThomas Gleixner #include <asm/nospec-branch.h> 17e81dc127SThomas Gleixner #include <asm/paravirt.h> 18e81dc127SThomas Gleixner #include <asm/sections.h> 19e81dc127SThomas Gleixner #include <asm/switch_to.h> 20e81dc127SThomas Gleixner #include <asm/sync_core.h> 21e81dc127SThomas Gleixner #include <asm/text-patching.h> 22e81dc127SThomas Gleixner #include <asm/xen/hypercall.h> 23e81dc127SThomas Gleixner 24e81dc127SThomas Gleixner static int __initdata_or_module debug_callthunks; 25e81dc127SThomas Gleixner 26e81dc127SThomas Gleixner #define prdbg(fmt, args...) \ 27e81dc127SThomas Gleixner do { \ 28e81dc127SThomas Gleixner if (debug_callthunks) \ 29e81dc127SThomas Gleixner printk(KERN_DEBUG pr_fmt(fmt), ##args); \ 30e81dc127SThomas Gleixner } while(0) 31e81dc127SThomas Gleixner 32e81dc127SThomas Gleixner static int __init debug_thunks(char *str) 33e81dc127SThomas Gleixner { 34e81dc127SThomas Gleixner debug_callthunks = 1; 35e81dc127SThomas Gleixner return 1; 36e81dc127SThomas Gleixner } 37e81dc127SThomas Gleixner __setup("debug-callthunks", debug_thunks); 38e81dc127SThomas Gleixner 39*f5c1bb2aSThomas Gleixner #ifdef CONFIG_CALL_THUNKS_DEBUG 40*f5c1bb2aSThomas Gleixner DEFINE_PER_CPU(u64, __x86_call_count); 41*f5c1bb2aSThomas Gleixner DEFINE_PER_CPU(u64, __x86_ret_count); 42*f5c1bb2aSThomas Gleixner DEFINE_PER_CPU(u64, __x86_stuffs_count); 43*f5c1bb2aSThomas Gleixner DEFINE_PER_CPU(u64, __x86_ctxsw_count); 44*f5c1bb2aSThomas Gleixner EXPORT_SYMBOL_GPL(__x86_ctxsw_count); 45*f5c1bb2aSThomas Gleixner EXPORT_SYMBOL_GPL(__x86_call_count); 46*f5c1bb2aSThomas Gleixner #endif 47*f5c1bb2aSThomas Gleixner 48e81dc127SThomas Gleixner extern s32 __call_sites[], __call_sites_end[]; 49e81dc127SThomas Gleixner 50e81dc127SThomas Gleixner struct thunk_desc { 51e81dc127SThomas Gleixner void *template; 52e81dc127SThomas Gleixner unsigned int template_size; 53e81dc127SThomas Gleixner }; 54e81dc127SThomas Gleixner 55e81dc127SThomas Gleixner struct core_text { 56e81dc127SThomas Gleixner unsigned long base; 57e81dc127SThomas Gleixner unsigned long end; 58e81dc127SThomas Gleixner const char *name; 59e81dc127SThomas Gleixner }; 60e81dc127SThomas Gleixner 61e81dc127SThomas Gleixner static bool thunks_initialized __ro_after_init; 62e81dc127SThomas Gleixner 63e81dc127SThomas Gleixner static const struct core_text builtin_coretext = { 64e81dc127SThomas Gleixner .base = (unsigned long)_text, 65e81dc127SThomas Gleixner .end = (unsigned long)_etext, 66e81dc127SThomas Gleixner .name = "builtin", 67e81dc127SThomas Gleixner }; 68e81dc127SThomas Gleixner 69bbaceb18SThomas Gleixner asm ( 70bbaceb18SThomas Gleixner ".pushsection .rodata \n" 71bbaceb18SThomas Gleixner ".global skl_call_thunk_template \n" 72bbaceb18SThomas Gleixner "skl_call_thunk_template: \n" 73bbaceb18SThomas Gleixner __stringify(INCREMENT_CALL_DEPTH)" \n" 74bbaceb18SThomas Gleixner ".global skl_call_thunk_tail \n" 75bbaceb18SThomas Gleixner "skl_call_thunk_tail: \n" 76bbaceb18SThomas Gleixner ".popsection \n" 77bbaceb18SThomas Gleixner ); 78bbaceb18SThomas Gleixner 79bbaceb18SThomas Gleixner extern u8 skl_call_thunk_template[]; 80bbaceb18SThomas Gleixner extern u8 skl_call_thunk_tail[]; 81bbaceb18SThomas Gleixner 82bbaceb18SThomas Gleixner #define SKL_TMPL_SIZE \ 83bbaceb18SThomas Gleixner ((unsigned int)(skl_call_thunk_tail - skl_call_thunk_template)) 84e81dc127SThomas Gleixner 85e81dc127SThomas Gleixner extern void error_entry(void); 86e81dc127SThomas Gleixner extern void xen_error_entry(void); 87e81dc127SThomas Gleixner extern void paranoid_entry(void); 88e81dc127SThomas Gleixner 89e81dc127SThomas Gleixner static inline bool within_coretext(const struct core_text *ct, void *addr) 90e81dc127SThomas Gleixner { 91e81dc127SThomas Gleixner unsigned long p = (unsigned long)addr; 92e81dc127SThomas Gleixner 93e81dc127SThomas Gleixner return ct->base <= p && p < ct->end; 94e81dc127SThomas Gleixner } 95e81dc127SThomas Gleixner 96e81dc127SThomas Gleixner static inline bool within_module_coretext(void *addr) 97e81dc127SThomas Gleixner { 98e81dc127SThomas Gleixner bool ret = false; 99e81dc127SThomas Gleixner 100e81dc127SThomas Gleixner #ifdef CONFIG_MODULES 101e81dc127SThomas Gleixner struct module *mod; 102e81dc127SThomas Gleixner 103e81dc127SThomas Gleixner preempt_disable(); 104e81dc127SThomas Gleixner mod = __module_address((unsigned long)addr); 105e81dc127SThomas Gleixner if (mod && within_module_core((unsigned long)addr, mod)) 106e81dc127SThomas Gleixner ret = true; 107e81dc127SThomas Gleixner preempt_enable(); 108e81dc127SThomas Gleixner #endif 109e81dc127SThomas Gleixner return ret; 110e81dc127SThomas Gleixner } 111e81dc127SThomas Gleixner 112e81dc127SThomas Gleixner static bool is_coretext(const struct core_text *ct, void *addr) 113e81dc127SThomas Gleixner { 114e81dc127SThomas Gleixner if (ct && within_coretext(ct, addr)) 115e81dc127SThomas Gleixner return true; 116e81dc127SThomas Gleixner if (within_coretext(&builtin_coretext, addr)) 117e81dc127SThomas Gleixner return true; 118e81dc127SThomas Gleixner return within_module_coretext(addr); 119e81dc127SThomas Gleixner } 120e81dc127SThomas Gleixner 121e81dc127SThomas Gleixner static __init_or_module bool skip_addr(void *dest) 122e81dc127SThomas Gleixner { 123e81dc127SThomas Gleixner if (dest == error_entry) 124e81dc127SThomas Gleixner return true; 125e81dc127SThomas Gleixner if (dest == paranoid_entry) 126e81dc127SThomas Gleixner return true; 127e81dc127SThomas Gleixner if (dest == xen_error_entry) 128e81dc127SThomas Gleixner return true; 129e81dc127SThomas Gleixner /* Does FILL_RSB... */ 130e81dc127SThomas Gleixner if (dest == __switch_to_asm) 131e81dc127SThomas Gleixner return true; 132e81dc127SThomas Gleixner /* Accounts directly */ 133e81dc127SThomas Gleixner if (dest == ret_from_fork) 134e81dc127SThomas Gleixner return true; 135e81dc127SThomas Gleixner #ifdef CONFIG_HOTPLUG_CPU 136e81dc127SThomas Gleixner if (dest == start_cpu0) 137e81dc127SThomas Gleixner return true; 138e81dc127SThomas Gleixner #endif 139e81dc127SThomas Gleixner #ifdef CONFIG_FUNCTION_TRACER 140e81dc127SThomas Gleixner if (dest == __fentry__) 141e81dc127SThomas Gleixner return true; 142e81dc127SThomas Gleixner #endif 143e81dc127SThomas Gleixner #ifdef CONFIG_KEXEC_CORE 144e81dc127SThomas Gleixner if (dest >= (void *)relocate_kernel && 145e81dc127SThomas Gleixner dest < (void*)relocate_kernel + KEXEC_CONTROL_CODE_MAX_SIZE) 146e81dc127SThomas Gleixner return true; 147e81dc127SThomas Gleixner #endif 148e81dc127SThomas Gleixner #ifdef CONFIG_XEN 149e81dc127SThomas Gleixner if (dest >= (void *)hypercall_page && 150e81dc127SThomas Gleixner dest < (void*)hypercall_page + PAGE_SIZE) 151e81dc127SThomas Gleixner return true; 152e81dc127SThomas Gleixner #endif 153e81dc127SThomas Gleixner return false; 154e81dc127SThomas Gleixner } 155e81dc127SThomas Gleixner 156e81dc127SThomas Gleixner static __init_or_module void *call_get_dest(void *addr) 157e81dc127SThomas Gleixner { 158e81dc127SThomas Gleixner struct insn insn; 159e81dc127SThomas Gleixner void *dest; 160e81dc127SThomas Gleixner int ret; 161e81dc127SThomas Gleixner 162e81dc127SThomas Gleixner ret = insn_decode_kernel(&insn, addr); 163e81dc127SThomas Gleixner if (ret) 164e81dc127SThomas Gleixner return ERR_PTR(ret); 165e81dc127SThomas Gleixner 166e81dc127SThomas Gleixner /* Patched out call? */ 167e81dc127SThomas Gleixner if (insn.opcode.bytes[0] != CALL_INSN_OPCODE) 168e81dc127SThomas Gleixner return NULL; 169e81dc127SThomas Gleixner 170e81dc127SThomas Gleixner dest = addr + insn.length + insn.immediate.value; 171e81dc127SThomas Gleixner if (skip_addr(dest)) 172e81dc127SThomas Gleixner return NULL; 173e81dc127SThomas Gleixner return dest; 174e81dc127SThomas Gleixner } 175e81dc127SThomas Gleixner 176e81dc127SThomas Gleixner static const u8 nops[] = { 177e81dc127SThomas Gleixner 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 178e81dc127SThomas Gleixner 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 179e81dc127SThomas Gleixner 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 180e81dc127SThomas Gleixner 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 181e81dc127SThomas Gleixner }; 182e81dc127SThomas Gleixner 183e81dc127SThomas Gleixner static __init_or_module void *patch_dest(void *dest, bool direct) 184e81dc127SThomas Gleixner { 185bbaceb18SThomas Gleixner unsigned int tsize = SKL_TMPL_SIZE; 186e81dc127SThomas Gleixner u8 *pad = dest - tsize; 187e81dc127SThomas Gleixner 188e81dc127SThomas Gleixner /* Already patched? */ 189bbaceb18SThomas Gleixner if (!bcmp(pad, skl_call_thunk_template, tsize)) 190e81dc127SThomas Gleixner return pad; 191e81dc127SThomas Gleixner 192e81dc127SThomas Gleixner /* Ensure there are nops */ 193e81dc127SThomas Gleixner if (bcmp(pad, nops, tsize)) { 194e81dc127SThomas Gleixner pr_warn_once("Invalid padding area for %pS\n", dest); 195e81dc127SThomas Gleixner return NULL; 196e81dc127SThomas Gleixner } 197e81dc127SThomas Gleixner 198e81dc127SThomas Gleixner if (direct) 199bbaceb18SThomas Gleixner memcpy(pad, skl_call_thunk_template, tsize); 200e81dc127SThomas Gleixner else 201bbaceb18SThomas Gleixner text_poke_copy_locked(pad, skl_call_thunk_template, tsize, true); 202e81dc127SThomas Gleixner return pad; 203e81dc127SThomas Gleixner } 204e81dc127SThomas Gleixner 205e81dc127SThomas Gleixner static __init_or_module void patch_call(void *addr, const struct core_text *ct) 206e81dc127SThomas Gleixner { 207e81dc127SThomas Gleixner void *pad, *dest; 208e81dc127SThomas Gleixner u8 bytes[8]; 209e81dc127SThomas Gleixner 210e81dc127SThomas Gleixner if (!within_coretext(ct, addr)) 211e81dc127SThomas Gleixner return; 212e81dc127SThomas Gleixner 213e81dc127SThomas Gleixner dest = call_get_dest(addr); 214e81dc127SThomas Gleixner if (!dest || WARN_ON_ONCE(IS_ERR(dest))) 215e81dc127SThomas Gleixner return; 216e81dc127SThomas Gleixner 217e81dc127SThomas Gleixner if (!is_coretext(ct, dest)) 218e81dc127SThomas Gleixner return; 219e81dc127SThomas Gleixner 220e81dc127SThomas Gleixner pad = patch_dest(dest, within_coretext(ct, dest)); 221e81dc127SThomas Gleixner if (!pad) 222e81dc127SThomas Gleixner return; 223e81dc127SThomas Gleixner 224e81dc127SThomas Gleixner prdbg("Patch call at: %pS %px to %pS %px -> %px \n", addr, addr, 225e81dc127SThomas Gleixner dest, dest, pad); 226e81dc127SThomas Gleixner __text_gen_insn(bytes, CALL_INSN_OPCODE, addr, pad, CALL_INSN_SIZE); 227e81dc127SThomas Gleixner text_poke_early(addr, bytes, CALL_INSN_SIZE); 228e81dc127SThomas Gleixner } 229e81dc127SThomas Gleixner 230e81dc127SThomas Gleixner static __init_or_module void 231e81dc127SThomas Gleixner patch_call_sites(s32 *start, s32 *end, const struct core_text *ct) 232e81dc127SThomas Gleixner { 233e81dc127SThomas Gleixner s32 *s; 234e81dc127SThomas Gleixner 235e81dc127SThomas Gleixner for (s = start; s < end; s++) 236e81dc127SThomas Gleixner patch_call((void *)s + *s, ct); 237e81dc127SThomas Gleixner } 238e81dc127SThomas Gleixner 239e81dc127SThomas Gleixner static __init_or_module void 240e81dc127SThomas Gleixner patch_paravirt_call_sites(struct paravirt_patch_site *start, 241e81dc127SThomas Gleixner struct paravirt_patch_site *end, 242e81dc127SThomas Gleixner const struct core_text *ct) 243e81dc127SThomas Gleixner { 244e81dc127SThomas Gleixner struct paravirt_patch_site *p; 245e81dc127SThomas Gleixner 246e81dc127SThomas Gleixner for (p = start; p < end; p++) 247e81dc127SThomas Gleixner patch_call(p->instr, ct); 248e81dc127SThomas Gleixner } 249e81dc127SThomas Gleixner 250e81dc127SThomas Gleixner static __init_or_module void 251e81dc127SThomas Gleixner callthunks_setup(struct callthunk_sites *cs, const struct core_text *ct) 252e81dc127SThomas Gleixner { 253e81dc127SThomas Gleixner prdbg("Patching call sites %s\n", ct->name); 254e81dc127SThomas Gleixner patch_call_sites(cs->call_start, cs->call_end, ct); 255e81dc127SThomas Gleixner patch_paravirt_call_sites(cs->pv_start, cs->pv_end, ct); 256e81dc127SThomas Gleixner prdbg("Patching call sites done%s\n", ct->name); 257e81dc127SThomas Gleixner } 258e81dc127SThomas Gleixner 259e81dc127SThomas Gleixner void __init callthunks_patch_builtin_calls(void) 260e81dc127SThomas Gleixner { 261e81dc127SThomas Gleixner struct callthunk_sites cs = { 262e81dc127SThomas Gleixner .call_start = __call_sites, 263e81dc127SThomas Gleixner .call_end = __call_sites_end, 264e81dc127SThomas Gleixner .pv_start = __parainstructions, 265e81dc127SThomas Gleixner .pv_end = __parainstructions_end 266e81dc127SThomas Gleixner }; 267e81dc127SThomas Gleixner 268e81dc127SThomas Gleixner if (!cpu_feature_enabled(X86_FEATURE_CALL_DEPTH)) 269e81dc127SThomas Gleixner return; 270e81dc127SThomas Gleixner 271e81dc127SThomas Gleixner pr_info("Setting up call depth tracking\n"); 272e81dc127SThomas Gleixner mutex_lock(&text_mutex); 273e81dc127SThomas Gleixner callthunks_setup(&cs, &builtin_coretext); 274e81dc127SThomas Gleixner thunks_initialized = true; 275e81dc127SThomas Gleixner mutex_unlock(&text_mutex); 276e81dc127SThomas Gleixner } 277eaf44c81SThomas Gleixner 278eaf44c81SThomas Gleixner #ifdef CONFIG_MODULES 279eaf44c81SThomas Gleixner void noinline callthunks_patch_module_calls(struct callthunk_sites *cs, 280eaf44c81SThomas Gleixner struct module *mod) 281eaf44c81SThomas Gleixner { 282eaf44c81SThomas Gleixner struct core_text ct = { 283eaf44c81SThomas Gleixner .base = (unsigned long)mod->core_layout.base, 284eaf44c81SThomas Gleixner .end = (unsigned long)mod->core_layout.base + mod->core_layout.size, 285eaf44c81SThomas Gleixner .name = mod->name, 286eaf44c81SThomas Gleixner }; 287eaf44c81SThomas Gleixner 288eaf44c81SThomas Gleixner if (!thunks_initialized) 289eaf44c81SThomas Gleixner return; 290eaf44c81SThomas Gleixner 291eaf44c81SThomas Gleixner mutex_lock(&text_mutex); 292eaf44c81SThomas Gleixner callthunks_setup(cs, &ct); 293eaf44c81SThomas Gleixner mutex_unlock(&text_mutex); 294eaf44c81SThomas Gleixner } 295eaf44c81SThomas Gleixner #endif /* CONFIG_MODULES */ 296*f5c1bb2aSThomas Gleixner 297*f5c1bb2aSThomas Gleixner #if defined(CONFIG_CALL_THUNKS_DEBUG) && defined(CONFIG_DEBUG_FS) 298*f5c1bb2aSThomas Gleixner static int callthunks_debug_show(struct seq_file *m, void *p) 299*f5c1bb2aSThomas Gleixner { 300*f5c1bb2aSThomas Gleixner unsigned long cpu = (unsigned long)m->private; 301*f5c1bb2aSThomas Gleixner 302*f5c1bb2aSThomas Gleixner seq_printf(m, "C: %16llu R: %16llu S: %16llu X: %16llu\n,", 303*f5c1bb2aSThomas Gleixner per_cpu(__x86_call_count, cpu), 304*f5c1bb2aSThomas Gleixner per_cpu(__x86_ret_count, cpu), 305*f5c1bb2aSThomas Gleixner per_cpu(__x86_stuffs_count, cpu), 306*f5c1bb2aSThomas Gleixner per_cpu(__x86_ctxsw_count, cpu)); 307*f5c1bb2aSThomas Gleixner return 0; 308*f5c1bb2aSThomas Gleixner } 309*f5c1bb2aSThomas Gleixner 310*f5c1bb2aSThomas Gleixner static int callthunks_debug_open(struct inode *inode, struct file *file) 311*f5c1bb2aSThomas Gleixner { 312*f5c1bb2aSThomas Gleixner return single_open(file, callthunks_debug_show, inode->i_private); 313*f5c1bb2aSThomas Gleixner } 314*f5c1bb2aSThomas Gleixner 315*f5c1bb2aSThomas Gleixner static const struct file_operations dfs_ops = { 316*f5c1bb2aSThomas Gleixner .open = callthunks_debug_open, 317*f5c1bb2aSThomas Gleixner .read = seq_read, 318*f5c1bb2aSThomas Gleixner .llseek = seq_lseek, 319*f5c1bb2aSThomas Gleixner .release = single_release, 320*f5c1bb2aSThomas Gleixner }; 321*f5c1bb2aSThomas Gleixner 322*f5c1bb2aSThomas Gleixner static int __init callthunks_debugfs_init(void) 323*f5c1bb2aSThomas Gleixner { 324*f5c1bb2aSThomas Gleixner struct dentry *dir; 325*f5c1bb2aSThomas Gleixner unsigned long cpu; 326*f5c1bb2aSThomas Gleixner 327*f5c1bb2aSThomas Gleixner dir = debugfs_create_dir("callthunks", NULL); 328*f5c1bb2aSThomas Gleixner for_each_possible_cpu(cpu) { 329*f5c1bb2aSThomas Gleixner void *arg = (void *)cpu; 330*f5c1bb2aSThomas Gleixner char name [10]; 331*f5c1bb2aSThomas Gleixner 332*f5c1bb2aSThomas Gleixner sprintf(name, "cpu%lu", cpu); 333*f5c1bb2aSThomas Gleixner debugfs_create_file(name, 0644, dir, arg, &dfs_ops); 334*f5c1bb2aSThomas Gleixner } 335*f5c1bb2aSThomas Gleixner return 0; 336*f5c1bb2aSThomas Gleixner } 337*f5c1bb2aSThomas Gleixner __initcall(callthunks_debugfs_init); 338*f5c1bb2aSThomas Gleixner #endif 339