1 // SPDX-License-Identifier: GPL-2.0-only 2 #include <linux/extable.h> 3 #include <linux/uaccess.h> 4 #include <linux/sched/debug.h> 5 #include <xen/xen.h> 6 7 #include <asm/fpu/internal.h> 8 #include <asm/traps.h> 9 #include <asm/kdebug.h> 10 11 typedef bool (*ex_handler_t)(const struct exception_table_entry *, 12 struct pt_regs *, int, unsigned long, 13 unsigned long); 14 15 static inline unsigned long 16 ex_fixup_addr(const struct exception_table_entry *x) 17 { 18 return (unsigned long)&x->fixup + x->fixup; 19 } 20 static inline ex_handler_t 21 ex_fixup_handler(const struct exception_table_entry *x) 22 { 23 return (ex_handler_t)((unsigned long)&x->handler + x->handler); 24 } 25 26 __visible bool ex_handler_default(const struct exception_table_entry *fixup, 27 struct pt_regs *regs, int trapnr, 28 unsigned long error_code, 29 unsigned long fault_addr) 30 { 31 regs->ip = ex_fixup_addr(fixup); 32 return true; 33 } 34 EXPORT_SYMBOL(ex_handler_default); 35 36 __visible bool ex_handler_fault(const struct exception_table_entry *fixup, 37 struct pt_regs *regs, int trapnr, 38 unsigned long error_code, 39 unsigned long fault_addr) 40 { 41 regs->ip = ex_fixup_addr(fixup); 42 regs->ax = trapnr; 43 return true; 44 } 45 EXPORT_SYMBOL_GPL(ex_handler_fault); 46 47 /* 48 * Handler for when we fail to restore a task's FPU state. We should never get 49 * here because the FPU state of a task using the FPU (task->thread.fpu.state) 50 * should always be valid. However, past bugs have allowed userspace to set 51 * reserved bits in the XSAVE area using PTRACE_SETREGSET or sys_rt_sigreturn(). 52 * These caused XRSTOR to fail when switching to the task, leaking the FPU 53 * registers of the task previously executing on the CPU. Mitigate this class 54 * of vulnerability by restoring from the initial state (essentially, zeroing 55 * out all the FPU registers) if we can't restore from the task's FPU state. 56 */ 57 __visible bool ex_handler_fprestore(const struct exception_table_entry *fixup, 58 struct pt_regs *regs, int trapnr, 59 unsigned long error_code, 60 unsigned long fault_addr) 61 { 62 regs->ip = ex_fixup_addr(fixup); 63 64 WARN_ONCE(1, "Bad FPU state detected at %pB, reinitializing FPU registers.", 65 (void *)instruction_pointer(regs)); 66 67 __copy_kernel_to_fpregs(&init_fpstate, -1); 68 return true; 69 } 70 EXPORT_SYMBOL_GPL(ex_handler_fprestore); 71 72 __visible bool ex_handler_uaccess(const struct exception_table_entry *fixup, 73 struct pt_regs *regs, int trapnr, 74 unsigned long error_code, 75 unsigned long fault_addr) 76 { 77 WARN_ONCE(trapnr == X86_TRAP_GP, "General protection fault in user access. Non-canonical address?"); 78 regs->ip = ex_fixup_addr(fixup); 79 return true; 80 } 81 EXPORT_SYMBOL(ex_handler_uaccess); 82 83 __visible bool ex_handler_ext(const struct exception_table_entry *fixup, 84 struct pt_regs *regs, int trapnr, 85 unsigned long error_code, 86 unsigned long fault_addr) 87 { 88 /* Special hack for uaccess_err */ 89 current->thread.uaccess_err = 1; 90 regs->ip = ex_fixup_addr(fixup); 91 return true; 92 } 93 EXPORT_SYMBOL(ex_handler_ext); 94 95 __visible bool ex_handler_rdmsr_unsafe(const struct exception_table_entry *fixup, 96 struct pt_regs *regs, int trapnr, 97 unsigned long error_code, 98 unsigned long fault_addr) 99 { 100 if (pr_warn_once("unchecked MSR access error: RDMSR from 0x%x at rIP: 0x%lx (%pS)\n", 101 (unsigned int)regs->cx, regs->ip, (void *)regs->ip)) 102 show_stack_regs(regs); 103 104 /* Pretend that the read succeeded and returned 0. */ 105 regs->ip = ex_fixup_addr(fixup); 106 regs->ax = 0; 107 regs->dx = 0; 108 return true; 109 } 110 EXPORT_SYMBOL(ex_handler_rdmsr_unsafe); 111 112 __visible bool ex_handler_wrmsr_unsafe(const struct exception_table_entry *fixup, 113 struct pt_regs *regs, int trapnr, 114 unsigned long error_code, 115 unsigned long fault_addr) 116 { 117 if (pr_warn_once("unchecked MSR access error: WRMSR to 0x%x (tried to write 0x%08x%08x) at rIP: 0x%lx (%pS)\n", 118 (unsigned int)regs->cx, (unsigned int)regs->dx, 119 (unsigned int)regs->ax, regs->ip, (void *)regs->ip)) 120 show_stack_regs(regs); 121 122 /* Pretend that the write succeeded. */ 123 regs->ip = ex_fixup_addr(fixup); 124 return true; 125 } 126 EXPORT_SYMBOL(ex_handler_wrmsr_unsafe); 127 128 __visible bool ex_handler_clear_fs(const struct exception_table_entry *fixup, 129 struct pt_regs *regs, int trapnr, 130 unsigned long error_code, 131 unsigned long fault_addr) 132 { 133 if (static_cpu_has(X86_BUG_NULL_SEG)) 134 asm volatile ("mov %0, %%fs" : : "rm" (__USER_DS)); 135 asm volatile ("mov %0, %%fs" : : "rm" (0)); 136 return ex_handler_default(fixup, regs, trapnr, error_code, fault_addr); 137 } 138 EXPORT_SYMBOL(ex_handler_clear_fs); 139 140 __visible bool ex_has_fault_handler(unsigned long ip) 141 { 142 const struct exception_table_entry *e; 143 ex_handler_t handler; 144 145 e = search_exception_tables(ip); 146 if (!e) 147 return false; 148 handler = ex_fixup_handler(e); 149 150 return handler == ex_handler_fault; 151 } 152 153 int fixup_exception(struct pt_regs *regs, int trapnr, unsigned long error_code, 154 unsigned long fault_addr) 155 { 156 const struct exception_table_entry *e; 157 ex_handler_t handler; 158 159 #ifdef CONFIG_PNPBIOS 160 if (unlikely(SEGMENT_IS_PNP_CODE(regs->cs))) { 161 extern u32 pnp_bios_fault_eip, pnp_bios_fault_esp; 162 extern u32 pnp_bios_is_utter_crap; 163 pnp_bios_is_utter_crap = 1; 164 printk(KERN_CRIT "PNPBIOS fault.. attempting recovery.\n"); 165 __asm__ volatile( 166 "movl %0, %%esp\n\t" 167 "jmp *%1\n\t" 168 : : "g" (pnp_bios_fault_esp), "g" (pnp_bios_fault_eip)); 169 panic("do_trap: can't hit this"); 170 } 171 #endif 172 173 e = search_exception_tables(regs->ip); 174 if (!e) 175 return 0; 176 177 handler = ex_fixup_handler(e); 178 return handler(e, regs, trapnr, error_code, fault_addr); 179 } 180 181 extern unsigned int early_recursion_flag; 182 183 /* Restricted version used during very early boot */ 184 void __init early_fixup_exception(struct pt_regs *regs, int trapnr) 185 { 186 /* Ignore early NMIs. */ 187 if (trapnr == X86_TRAP_NMI) 188 return; 189 190 if (early_recursion_flag > 2) 191 goto halt_loop; 192 193 /* 194 * Old CPUs leave the high bits of CS on the stack 195 * undefined. I'm not sure which CPUs do this, but at least 196 * the 486 DX works this way. 197 * Xen pv domains are not using the default __KERNEL_CS. 198 */ 199 if (!xen_pv_domain() && regs->cs != __KERNEL_CS) 200 goto fail; 201 202 /* 203 * The full exception fixup machinery is available as soon as 204 * the early IDT is loaded. This means that it is the 205 * responsibility of extable users to either function correctly 206 * when handlers are invoked early or to simply avoid causing 207 * exceptions before they're ready to handle them. 208 * 209 * This is better than filtering which handlers can be used, 210 * because refusing to call a handler here is guaranteed to 211 * result in a hard-to-debug panic. 212 * 213 * Keep in mind that not all vectors actually get here. Early 214 * page faults, for example, are special. 215 */ 216 if (fixup_exception(regs, trapnr, regs->orig_ax, 0)) 217 return; 218 219 if (fixup_bug(regs, trapnr)) 220 return; 221 222 fail: 223 early_printk("PANIC: early exception 0x%02x IP %lx:%lx error %lx cr2 0x%lx\n", 224 (unsigned)trapnr, (unsigned long)regs->cs, regs->ip, 225 regs->orig_ax, read_cr2()); 226 227 show_regs(regs); 228 229 halt_loop: 230 while (true) 231 halt(); 232 } 233