1 // SPDX-License-Identifier: GPL-2.0-only 2 #include <linux/extable.h> 3 #include <linux/uaccess.h> 4 #include <linux/sched/debug.h> 5 #include <xen/xen.h> 6 7 #include <asm/fpu/internal.h> 8 #include <asm/traps.h> 9 #include <asm/kdebug.h> 10 11 typedef bool (*ex_handler_t)(const struct exception_table_entry *, 12 struct pt_regs *, int, unsigned long, 13 unsigned long); 14 15 static inline unsigned long 16 ex_fixup_addr(const struct exception_table_entry *x) 17 { 18 return (unsigned long)&x->fixup + x->fixup; 19 } 20 static inline ex_handler_t 21 ex_fixup_handler(const struct exception_table_entry *x) 22 { 23 return (ex_handler_t)((unsigned long)&x->handler + x->handler); 24 } 25 26 __visible bool ex_handler_default(const struct exception_table_entry *fixup, 27 struct pt_regs *regs, int trapnr, 28 unsigned long error_code, 29 unsigned long fault_addr) 30 { 31 regs->ip = ex_fixup_addr(fixup); 32 return true; 33 } 34 EXPORT_SYMBOL(ex_handler_default); 35 36 __visible bool ex_handler_fault(const struct exception_table_entry *fixup, 37 struct pt_regs *regs, int trapnr, 38 unsigned long error_code, 39 unsigned long fault_addr) 40 { 41 regs->ip = ex_fixup_addr(fixup); 42 regs->ax = trapnr; 43 return true; 44 } 45 EXPORT_SYMBOL_GPL(ex_handler_fault); 46 47 /* 48 * Handler for when we fail to restore a task's FPU state. We should never get 49 * here because the FPU state of a task using the FPU (task->thread.fpu.state) 50 * should always be valid. However, past bugs have allowed userspace to set 51 * reserved bits in the XSAVE area using PTRACE_SETREGSET or sys_rt_sigreturn(). 52 * These caused XRSTOR to fail when switching to the task, leaking the FPU 53 * registers of the task previously executing on the CPU. Mitigate this class 54 * of vulnerability by restoring from the initial state (essentially, zeroing 55 * out all the FPU registers) if we can't restore from the task's FPU state. 56 */ 57 __visible bool ex_handler_fprestore(const struct exception_table_entry *fixup, 58 struct pt_regs *regs, int trapnr, 59 unsigned long error_code, 60 unsigned long fault_addr) 61 { 62 regs->ip = ex_fixup_addr(fixup); 63 64 WARN_ONCE(1, "Bad FPU state detected at %pB, reinitializing FPU registers.", 65 (void *)instruction_pointer(regs)); 66 67 __copy_kernel_to_fpregs(&init_fpstate, -1); 68 return true; 69 } 70 EXPORT_SYMBOL_GPL(ex_handler_fprestore); 71 72 __visible bool ex_handler_uaccess(const struct exception_table_entry *fixup, 73 struct pt_regs *regs, int trapnr, 74 unsigned long error_code, 75 unsigned long fault_addr) 76 { 77 WARN_ONCE(trapnr == X86_TRAP_GP, "General protection fault in user access. Non-canonical address?"); 78 regs->ip = ex_fixup_addr(fixup); 79 return true; 80 } 81 EXPORT_SYMBOL(ex_handler_uaccess); 82 83 __visible bool ex_handler_rdmsr_unsafe(const struct exception_table_entry *fixup, 84 struct pt_regs *regs, int trapnr, 85 unsigned long error_code, 86 unsigned long fault_addr) 87 { 88 if (pr_warn_once("unchecked MSR access error: RDMSR from 0x%x at rIP: 0x%lx (%pS)\n", 89 (unsigned int)regs->cx, regs->ip, (void *)regs->ip)) 90 show_stack_regs(regs); 91 92 /* Pretend that the read succeeded and returned 0. */ 93 regs->ip = ex_fixup_addr(fixup); 94 regs->ax = 0; 95 regs->dx = 0; 96 return true; 97 } 98 EXPORT_SYMBOL(ex_handler_rdmsr_unsafe); 99 100 __visible bool ex_handler_wrmsr_unsafe(const struct exception_table_entry *fixup, 101 struct pt_regs *regs, int trapnr, 102 unsigned long error_code, 103 unsigned long fault_addr) 104 { 105 if (pr_warn_once("unchecked MSR access error: WRMSR to 0x%x (tried to write 0x%08x%08x) at rIP: 0x%lx (%pS)\n", 106 (unsigned int)regs->cx, (unsigned int)regs->dx, 107 (unsigned int)regs->ax, regs->ip, (void *)regs->ip)) 108 show_stack_regs(regs); 109 110 /* Pretend that the write succeeded. */ 111 regs->ip = ex_fixup_addr(fixup); 112 return true; 113 } 114 EXPORT_SYMBOL(ex_handler_wrmsr_unsafe); 115 116 __visible bool ex_handler_clear_fs(const struct exception_table_entry *fixup, 117 struct pt_regs *regs, int trapnr, 118 unsigned long error_code, 119 unsigned long fault_addr) 120 { 121 if (static_cpu_has(X86_BUG_NULL_SEG)) 122 asm volatile ("mov %0, %%fs" : : "rm" (__USER_DS)); 123 asm volatile ("mov %0, %%fs" : : "rm" (0)); 124 return ex_handler_default(fixup, regs, trapnr, error_code, fault_addr); 125 } 126 EXPORT_SYMBOL(ex_handler_clear_fs); 127 128 __visible bool ex_has_fault_handler(unsigned long ip) 129 { 130 const struct exception_table_entry *e; 131 ex_handler_t handler; 132 133 e = search_exception_tables(ip); 134 if (!e) 135 return false; 136 handler = ex_fixup_handler(e); 137 138 return handler == ex_handler_fault; 139 } 140 141 int fixup_exception(struct pt_regs *regs, int trapnr, unsigned long error_code, 142 unsigned long fault_addr) 143 { 144 const struct exception_table_entry *e; 145 ex_handler_t handler; 146 147 #ifdef CONFIG_PNPBIOS 148 if (unlikely(SEGMENT_IS_PNP_CODE(regs->cs))) { 149 extern u32 pnp_bios_fault_eip, pnp_bios_fault_esp; 150 extern u32 pnp_bios_is_utter_crap; 151 pnp_bios_is_utter_crap = 1; 152 printk(KERN_CRIT "PNPBIOS fault.. attempting recovery.\n"); 153 __asm__ volatile( 154 "movl %0, %%esp\n\t" 155 "jmp *%1\n\t" 156 : : "g" (pnp_bios_fault_esp), "g" (pnp_bios_fault_eip)); 157 panic("do_trap: can't hit this"); 158 } 159 #endif 160 161 e = search_exception_tables(regs->ip); 162 if (!e) 163 return 0; 164 165 handler = ex_fixup_handler(e); 166 return handler(e, regs, trapnr, error_code, fault_addr); 167 } 168 169 extern unsigned int early_recursion_flag; 170 171 /* Restricted version used during very early boot */ 172 void __init early_fixup_exception(struct pt_regs *regs, int trapnr) 173 { 174 /* Ignore early NMIs. */ 175 if (trapnr == X86_TRAP_NMI) 176 return; 177 178 if (early_recursion_flag > 2) 179 goto halt_loop; 180 181 /* 182 * Old CPUs leave the high bits of CS on the stack 183 * undefined. I'm not sure which CPUs do this, but at least 184 * the 486 DX works this way. 185 * Xen pv domains are not using the default __KERNEL_CS. 186 */ 187 if (!xen_pv_domain() && regs->cs != __KERNEL_CS) 188 goto fail; 189 190 /* 191 * The full exception fixup machinery is available as soon as 192 * the early IDT is loaded. This means that it is the 193 * responsibility of extable users to either function correctly 194 * when handlers are invoked early or to simply avoid causing 195 * exceptions before they're ready to handle them. 196 * 197 * This is better than filtering which handlers can be used, 198 * because refusing to call a handler here is guaranteed to 199 * result in a hard-to-debug panic. 200 * 201 * Keep in mind that not all vectors actually get here. Early 202 * page faults, for example, are special. 203 */ 204 if (fixup_exception(regs, trapnr, regs->orig_ax, 0)) 205 return; 206 207 if (fixup_bug(regs, trapnr)) 208 return; 209 210 fail: 211 early_printk("PANIC: early exception 0x%02x IP %lx:%lx error %lx cr2 0x%lx\n", 212 (unsigned)trapnr, (unsigned long)regs->cs, regs->ip, 213 regs->orig_ax, read_cr2()); 214 215 show_regs(regs); 216 217 halt_loop: 218 while (true) 219 halt(); 220 } 221