1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Exception handling code 4 * 5 * Copyright (C) 2019 ARM Ltd. 6 */ 7 8 #include <linux/context_tracking.h> 9 #include <linux/ptrace.h> 10 #include <linux/thread_info.h> 11 12 #include <asm/cpufeature.h> 13 #include <asm/daifflags.h> 14 #include <asm/esr.h> 15 #include <asm/exception.h> 16 #include <asm/kprobes.h> 17 #include <asm/mmu.h> 18 #include <asm/sysreg.h> 19 20 static void notrace el1_abort(struct pt_regs *regs, unsigned long esr) 21 { 22 unsigned long far = read_sysreg(far_el1); 23 24 local_daif_inherit(regs); 25 far = untagged_addr(far); 26 do_mem_abort(far, esr, regs); 27 } 28 NOKPROBE_SYMBOL(el1_abort); 29 30 static void notrace el1_pc(struct pt_regs *regs, unsigned long esr) 31 { 32 unsigned long far = read_sysreg(far_el1); 33 34 local_daif_inherit(regs); 35 do_sp_pc_abort(far, esr, regs); 36 } 37 NOKPROBE_SYMBOL(el1_pc); 38 39 static void notrace el1_undef(struct pt_regs *regs) 40 { 41 local_daif_inherit(regs); 42 do_undefinstr(regs); 43 } 44 NOKPROBE_SYMBOL(el1_undef); 45 46 static void notrace el1_inv(struct pt_regs *regs, unsigned long esr) 47 { 48 local_daif_inherit(regs); 49 bad_mode(regs, 0, esr); 50 } 51 NOKPROBE_SYMBOL(el1_inv); 52 53 static void notrace el1_dbg(struct pt_regs *regs, unsigned long esr) 54 { 55 unsigned long far = read_sysreg(far_el1); 56 57 /* 58 * The CPU masked interrupts, and we are leaving them masked during 59 * do_debug_exception(). Update PMR as if we had called 60 * local_mask_daif(). 61 */ 62 if (system_uses_irq_prio_masking()) 63 gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET); 64 65 do_debug_exception(far, esr, regs); 66 } 67 NOKPROBE_SYMBOL(el1_dbg); 68 69 asmlinkage void notrace el1_sync_handler(struct pt_regs *regs) 70 { 71 unsigned long esr = read_sysreg(esr_el1); 72 73 switch (ESR_ELx_EC(esr)) { 74 case ESR_ELx_EC_DABT_CUR: 75 case ESR_ELx_EC_IABT_CUR: 76 el1_abort(regs, esr); 77 break; 78 /* 79 * We don't handle ESR_ELx_EC_SP_ALIGN, since we will have hit a 80 * recursive exception when trying to push the initial pt_regs. 81 */ 82 case ESR_ELx_EC_PC_ALIGN: 83 el1_pc(regs, esr); 84 break; 85 case ESR_ELx_EC_SYS64: 86 case ESR_ELx_EC_UNKNOWN: 87 el1_undef(regs); 88 break; 89 case ESR_ELx_EC_BREAKPT_CUR: 90 case ESR_ELx_EC_SOFTSTP_CUR: 91 case ESR_ELx_EC_WATCHPT_CUR: 92 case ESR_ELx_EC_BRK64: 93 el1_dbg(regs, esr); 94 break; 95 default: 96 el1_inv(regs, esr); 97 }; 98 } 99 NOKPROBE_SYMBOL(el1_sync_handler); 100 101 static void notrace el0_da(struct pt_regs *regs, unsigned long esr) 102 { 103 unsigned long far = read_sysreg(far_el1); 104 105 user_exit_irqoff(); 106 local_daif_restore(DAIF_PROCCTX); 107 far = untagged_addr(far); 108 do_mem_abort(far, esr, regs); 109 } 110 NOKPROBE_SYMBOL(el0_da); 111 112 static void notrace el0_ia(struct pt_regs *regs, unsigned long esr) 113 { 114 unsigned long far = read_sysreg(far_el1); 115 116 /* 117 * We've taken an instruction abort from userspace and not yet 118 * re-enabled IRQs. If the address is a kernel address, apply 119 * BP hardening prior to enabling IRQs and pre-emption. 120 */ 121 if (!is_ttbr0_addr(far)) 122 arm64_apply_bp_hardening(); 123 124 user_exit_irqoff(); 125 local_daif_restore(DAIF_PROCCTX); 126 do_mem_abort(far, esr, regs); 127 } 128 NOKPROBE_SYMBOL(el0_ia); 129 130 static void notrace el0_fpsimd_acc(struct pt_regs *regs, unsigned long esr) 131 { 132 user_exit_irqoff(); 133 local_daif_restore(DAIF_PROCCTX); 134 do_fpsimd_acc(esr, regs); 135 } 136 NOKPROBE_SYMBOL(el0_fpsimd_acc); 137 138 static void notrace el0_sve_acc(struct pt_regs *regs, unsigned long esr) 139 { 140 user_exit_irqoff(); 141 local_daif_restore(DAIF_PROCCTX); 142 do_sve_acc(esr, regs); 143 } 144 NOKPROBE_SYMBOL(el0_sve_acc); 145 146 static void notrace el0_fpsimd_exc(struct pt_regs *regs, unsigned long esr) 147 { 148 user_exit_irqoff(); 149 local_daif_restore(DAIF_PROCCTX); 150 do_fpsimd_exc(esr, regs); 151 } 152 NOKPROBE_SYMBOL(el0_fpsimd_exc); 153 154 static void notrace el0_sys(struct pt_regs *regs, unsigned long esr) 155 { 156 user_exit_irqoff(); 157 local_daif_restore(DAIF_PROCCTX); 158 do_sysinstr(esr, regs); 159 } 160 NOKPROBE_SYMBOL(el0_sys); 161 162 static void notrace el0_pc(struct pt_regs *regs, unsigned long esr) 163 { 164 unsigned long far = read_sysreg(far_el1); 165 166 if (!is_ttbr0_addr(instruction_pointer(regs))) 167 arm64_apply_bp_hardening(); 168 169 user_exit_irqoff(); 170 local_daif_restore(DAIF_PROCCTX); 171 do_sp_pc_abort(far, esr, regs); 172 } 173 NOKPROBE_SYMBOL(el0_pc); 174 175 static void notrace el0_sp(struct pt_regs *regs, unsigned long esr) 176 { 177 user_exit_irqoff(); 178 local_daif_restore(DAIF_PROCCTX); 179 do_sp_pc_abort(regs->sp, esr, regs); 180 } 181 NOKPROBE_SYMBOL(el0_sp); 182 183 static void notrace el0_undef(struct pt_regs *regs) 184 { 185 user_exit_irqoff(); 186 local_daif_restore(DAIF_PROCCTX); 187 do_undefinstr(regs); 188 } 189 NOKPROBE_SYMBOL(el0_undef); 190 191 static void notrace el0_inv(struct pt_regs *regs, unsigned long esr) 192 { 193 user_exit_irqoff(); 194 local_daif_restore(DAIF_PROCCTX); 195 bad_el0_sync(regs, 0, esr); 196 } 197 NOKPROBE_SYMBOL(el0_inv); 198 199 static void notrace el0_dbg(struct pt_regs *regs, unsigned long esr) 200 { 201 /* Only watchpoints write FAR_EL1, otherwise its UNKNOWN */ 202 unsigned long far = read_sysreg(far_el1); 203 204 if (system_uses_irq_prio_masking()) 205 gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET); 206 207 user_exit_irqoff(); 208 do_debug_exception(far, esr, regs); 209 local_daif_restore(DAIF_PROCCTX_NOIRQ); 210 } 211 NOKPROBE_SYMBOL(el0_dbg); 212 213 static void notrace el0_svc(struct pt_regs *regs) 214 { 215 if (system_uses_irq_prio_masking()) 216 gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET); 217 218 do_el0_svc(regs); 219 } 220 NOKPROBE_SYMBOL(el0_svc); 221 222 asmlinkage void notrace el0_sync_handler(struct pt_regs *regs) 223 { 224 unsigned long esr = read_sysreg(esr_el1); 225 226 switch (ESR_ELx_EC(esr)) { 227 case ESR_ELx_EC_SVC64: 228 el0_svc(regs); 229 break; 230 case ESR_ELx_EC_DABT_LOW: 231 el0_da(regs, esr); 232 break; 233 case ESR_ELx_EC_IABT_LOW: 234 el0_ia(regs, esr); 235 break; 236 case ESR_ELx_EC_FP_ASIMD: 237 el0_fpsimd_acc(regs, esr); 238 break; 239 case ESR_ELx_EC_SVE: 240 el0_sve_acc(regs, esr); 241 break; 242 case ESR_ELx_EC_FP_EXC64: 243 el0_fpsimd_exc(regs, esr); 244 break; 245 case ESR_ELx_EC_SYS64: 246 case ESR_ELx_EC_WFx: 247 el0_sys(regs, esr); 248 break; 249 case ESR_ELx_EC_SP_ALIGN: 250 el0_sp(regs, esr); 251 break; 252 case ESR_ELx_EC_PC_ALIGN: 253 el0_pc(regs, esr); 254 break; 255 case ESR_ELx_EC_UNKNOWN: 256 el0_undef(regs); 257 break; 258 case ESR_ELx_EC_BREAKPT_LOW: 259 case ESR_ELx_EC_SOFTSTP_LOW: 260 case ESR_ELx_EC_WATCHPT_LOW: 261 case ESR_ELx_EC_BRK64: 262 el0_dbg(regs, esr); 263 break; 264 default: 265 el0_inv(regs, esr); 266 } 267 } 268 NOKPROBE_SYMBOL(el0_sync_handler); 269 270 #ifdef CONFIG_COMPAT 271 static void notrace el0_cp15(struct pt_regs *regs, unsigned long esr) 272 { 273 user_exit_irqoff(); 274 local_daif_restore(DAIF_PROCCTX); 275 do_cp15instr(esr, regs); 276 } 277 NOKPROBE_SYMBOL(el0_cp15); 278 279 static void notrace el0_svc_compat(struct pt_regs *regs) 280 { 281 if (system_uses_irq_prio_masking()) 282 gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET); 283 284 do_el0_svc_compat(regs); 285 } 286 NOKPROBE_SYMBOL(el0_svc_compat); 287 288 asmlinkage void notrace el0_sync_compat_handler(struct pt_regs *regs) 289 { 290 unsigned long esr = read_sysreg(esr_el1); 291 292 switch (ESR_ELx_EC(esr)) { 293 case ESR_ELx_EC_SVC32: 294 el0_svc_compat(regs); 295 break; 296 case ESR_ELx_EC_DABT_LOW: 297 el0_da(regs, esr); 298 break; 299 case ESR_ELx_EC_IABT_LOW: 300 el0_ia(regs, esr); 301 break; 302 case ESR_ELx_EC_FP_ASIMD: 303 el0_fpsimd_acc(regs, esr); 304 break; 305 case ESR_ELx_EC_FP_EXC32: 306 el0_fpsimd_exc(regs, esr); 307 break; 308 case ESR_ELx_EC_PC_ALIGN: 309 el0_pc(regs, esr); 310 break; 311 case ESR_ELx_EC_UNKNOWN: 312 case ESR_ELx_EC_CP14_MR: 313 case ESR_ELx_EC_CP14_LS: 314 case ESR_ELx_EC_CP14_64: 315 el0_undef(regs); 316 break; 317 case ESR_ELx_EC_CP15_32: 318 case ESR_ELx_EC_CP15_64: 319 el0_cp15(regs, esr); 320 break; 321 case ESR_ELx_EC_BREAKPT_LOW: 322 case ESR_ELx_EC_SOFTSTP_LOW: 323 case ESR_ELx_EC_WATCHPT_LOW: 324 case ESR_ELx_EC_BKPT32: 325 el0_dbg(regs, esr); 326 break; 327 default: 328 el0_inv(regs, esr); 329 } 330 } 331 NOKPROBE_SYMBOL(el0_sync_compat_handler); 332 #endif /* CONFIG_COMPAT */ 333