1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Exception handling code 4 * 5 * Copyright (C) 2019 ARM Ltd. 6 */ 7 8 #include <linux/context_tracking.h> 9 #include <linux/ptrace.h> 10 #include <linux/thread_info.h> 11 12 #include <asm/cpufeature.h> 13 #include <asm/daifflags.h> 14 #include <asm/esr.h> 15 #include <asm/exception.h> 16 #include <asm/kprobes.h> 17 #include <asm/mmu.h> 18 #include <asm/sysreg.h> 19 20 static void notrace el1_abort(struct pt_regs *regs, unsigned long esr) 21 { 22 unsigned long far = read_sysreg(far_el1); 23 24 local_daif_inherit(regs); 25 far = untagged_addr(far); 26 do_mem_abort(far, esr, regs); 27 } 28 NOKPROBE_SYMBOL(el1_abort); 29 30 static void notrace el1_pc(struct pt_regs *regs, unsigned long esr) 31 { 32 unsigned long far = read_sysreg(far_el1); 33 34 local_daif_inherit(regs); 35 do_sp_pc_abort(far, esr, regs); 36 } 37 NOKPROBE_SYMBOL(el1_pc); 38 39 static void notrace el1_undef(struct pt_regs *regs) 40 { 41 local_daif_inherit(regs); 42 do_undefinstr(regs); 43 } 44 NOKPROBE_SYMBOL(el1_undef); 45 46 static void notrace el1_inv(struct pt_regs *regs, unsigned long esr) 47 { 48 local_daif_inherit(regs); 49 bad_mode(regs, 0, esr); 50 } 51 NOKPROBE_SYMBOL(el1_inv); 52 53 static void notrace el1_dbg(struct pt_regs *regs, unsigned long esr) 54 { 55 unsigned long far = read_sysreg(far_el1); 56 57 /* 58 * The CPU masked interrupts, and we are leaving them masked during 59 * do_debug_exception(). Update PMR as if we had called 60 * local_daif_mask(). 61 */ 62 if (system_uses_irq_prio_masking()) 63 gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET); 64 65 do_debug_exception(far, esr, regs); 66 } 67 NOKPROBE_SYMBOL(el1_dbg); 68 69 asmlinkage void notrace el1_sync_handler(struct pt_regs *regs) 70 { 71 unsigned long esr = read_sysreg(esr_el1); 72 73 switch (ESR_ELx_EC(esr)) { 74 case ESR_ELx_EC_DABT_CUR: 75 case ESR_ELx_EC_IABT_CUR: 76 el1_abort(regs, esr); 77 break; 78 /* 79 * We don't handle ESR_ELx_EC_SP_ALIGN, since we will have hit a 80 * recursive exception when trying to push the initial pt_regs. 81 */ 82 case ESR_ELx_EC_PC_ALIGN: 83 el1_pc(regs, esr); 84 break; 85 case ESR_ELx_EC_SYS64: 86 case ESR_ELx_EC_UNKNOWN: 87 el1_undef(regs); 88 break; 89 case ESR_ELx_EC_BREAKPT_CUR: 90 case ESR_ELx_EC_SOFTSTP_CUR: 91 case ESR_ELx_EC_WATCHPT_CUR: 92 case ESR_ELx_EC_BRK64: 93 el1_dbg(regs, esr); 94 break; 95 default: 96 el1_inv(regs, esr); 97 } 98 } 99 NOKPROBE_SYMBOL(el1_sync_handler); 100 101 static void notrace el0_da(struct pt_regs *regs, unsigned long esr) 102 { 103 unsigned long far = read_sysreg(far_el1); 104 105 user_exit_irqoff(); 106 local_daif_restore(DAIF_PROCCTX); 107 far = untagged_addr(far); 108 do_mem_abort(far, esr, regs); 109 } 110 NOKPROBE_SYMBOL(el0_da); 111 112 static void notrace el0_ia(struct pt_regs *regs, unsigned long esr) 113 { 114 unsigned long far = read_sysreg(far_el1); 115 116 /* 117 * We've taken an instruction abort from userspace and not yet 118 * re-enabled IRQs. If the address is a kernel address, apply 119 * BP hardening prior to enabling IRQs and pre-emption. 120 */ 121 if (!is_ttbr0_addr(far)) 122 arm64_apply_bp_hardening(); 123 124 user_exit_irqoff(); 125 local_daif_restore(DAIF_PROCCTX); 126 do_mem_abort(far, esr, regs); 127 } 128 NOKPROBE_SYMBOL(el0_ia); 129 130 static void notrace el0_fpsimd_acc(struct pt_regs *regs, unsigned long esr) 131 { 132 user_exit_irqoff(); 133 local_daif_restore(DAIF_PROCCTX); 134 do_fpsimd_acc(esr, regs); 135 } 136 NOKPROBE_SYMBOL(el0_fpsimd_acc); 137 138 static void notrace el0_sve_acc(struct pt_regs *regs, unsigned long esr) 139 { 140 user_exit_irqoff(); 141 local_daif_restore(DAIF_PROCCTX); 142 do_sve_acc(esr, regs); 143 } 144 NOKPROBE_SYMBOL(el0_sve_acc); 145 146 static void notrace el0_fpsimd_exc(struct pt_regs *regs, unsigned long esr) 147 { 148 user_exit_irqoff(); 149 local_daif_restore(DAIF_PROCCTX); 150 do_fpsimd_exc(esr, regs); 151 } 152 NOKPROBE_SYMBOL(el0_fpsimd_exc); 153 154 static void notrace el0_sys(struct pt_regs *regs, unsigned long esr) 155 { 156 user_exit_irqoff(); 157 local_daif_restore(DAIF_PROCCTX); 158 do_sysinstr(esr, regs); 159 } 160 NOKPROBE_SYMBOL(el0_sys); 161 162 static void notrace el0_pc(struct pt_regs *regs, unsigned long esr) 163 { 164 unsigned long far = read_sysreg(far_el1); 165 166 if (!is_ttbr0_addr(instruction_pointer(regs))) 167 arm64_apply_bp_hardening(); 168 169 user_exit_irqoff(); 170 local_daif_restore(DAIF_PROCCTX); 171 do_sp_pc_abort(far, esr, regs); 172 } 173 NOKPROBE_SYMBOL(el0_pc); 174 175 static void notrace el0_sp(struct pt_regs *regs, unsigned long esr) 176 { 177 user_exit_irqoff(); 178 local_daif_restore(DAIF_PROCCTX); 179 do_sp_pc_abort(regs->sp, esr, regs); 180 } 181 NOKPROBE_SYMBOL(el0_sp); 182 183 static void notrace el0_undef(struct pt_regs *regs) 184 { 185 user_exit_irqoff(); 186 local_daif_restore(DAIF_PROCCTX); 187 do_undefinstr(regs); 188 } 189 NOKPROBE_SYMBOL(el0_undef); 190 191 static void notrace el0_bti(struct pt_regs *regs) 192 { 193 user_exit_irqoff(); 194 local_daif_restore(DAIF_PROCCTX); 195 do_bti(regs); 196 } 197 NOKPROBE_SYMBOL(el0_bti); 198 199 static void notrace el0_inv(struct pt_regs *regs, unsigned long esr) 200 { 201 user_exit_irqoff(); 202 local_daif_restore(DAIF_PROCCTX); 203 bad_el0_sync(regs, 0, esr); 204 } 205 NOKPROBE_SYMBOL(el0_inv); 206 207 static void notrace el0_dbg(struct pt_regs *regs, unsigned long esr) 208 { 209 /* Only watchpoints write FAR_EL1, otherwise its UNKNOWN */ 210 unsigned long far = read_sysreg(far_el1); 211 212 if (system_uses_irq_prio_masking()) 213 gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET); 214 215 user_exit_irqoff(); 216 do_debug_exception(far, esr, regs); 217 local_daif_restore(DAIF_PROCCTX_NOIRQ); 218 } 219 NOKPROBE_SYMBOL(el0_dbg); 220 221 static void notrace el0_svc(struct pt_regs *regs) 222 { 223 if (system_uses_irq_prio_masking()) 224 gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET); 225 226 do_el0_svc(regs); 227 } 228 NOKPROBE_SYMBOL(el0_svc); 229 230 asmlinkage void notrace el0_sync_handler(struct pt_regs *regs) 231 { 232 unsigned long esr = read_sysreg(esr_el1); 233 234 switch (ESR_ELx_EC(esr)) { 235 case ESR_ELx_EC_SVC64: 236 el0_svc(regs); 237 break; 238 case ESR_ELx_EC_DABT_LOW: 239 el0_da(regs, esr); 240 break; 241 case ESR_ELx_EC_IABT_LOW: 242 el0_ia(regs, esr); 243 break; 244 case ESR_ELx_EC_FP_ASIMD: 245 el0_fpsimd_acc(regs, esr); 246 break; 247 case ESR_ELx_EC_SVE: 248 el0_sve_acc(regs, esr); 249 break; 250 case ESR_ELx_EC_FP_EXC64: 251 el0_fpsimd_exc(regs, esr); 252 break; 253 case ESR_ELx_EC_SYS64: 254 case ESR_ELx_EC_WFx: 255 el0_sys(regs, esr); 256 break; 257 case ESR_ELx_EC_SP_ALIGN: 258 el0_sp(regs, esr); 259 break; 260 case ESR_ELx_EC_PC_ALIGN: 261 el0_pc(regs, esr); 262 break; 263 case ESR_ELx_EC_UNKNOWN: 264 el0_undef(regs); 265 break; 266 case ESR_ELx_EC_BTI: 267 el0_bti(regs); 268 break; 269 case ESR_ELx_EC_BREAKPT_LOW: 270 case ESR_ELx_EC_SOFTSTP_LOW: 271 case ESR_ELx_EC_WATCHPT_LOW: 272 case ESR_ELx_EC_BRK64: 273 el0_dbg(regs, esr); 274 break; 275 default: 276 el0_inv(regs, esr); 277 } 278 } 279 NOKPROBE_SYMBOL(el0_sync_handler); 280 281 #ifdef CONFIG_COMPAT 282 static void notrace el0_cp15(struct pt_regs *regs, unsigned long esr) 283 { 284 user_exit_irqoff(); 285 local_daif_restore(DAIF_PROCCTX); 286 do_cp15instr(esr, regs); 287 } 288 NOKPROBE_SYMBOL(el0_cp15); 289 290 static void notrace el0_svc_compat(struct pt_regs *regs) 291 { 292 if (system_uses_irq_prio_masking()) 293 gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET); 294 295 do_el0_svc_compat(regs); 296 } 297 NOKPROBE_SYMBOL(el0_svc_compat); 298 299 asmlinkage void notrace el0_sync_compat_handler(struct pt_regs *regs) 300 { 301 unsigned long esr = read_sysreg(esr_el1); 302 303 switch (ESR_ELx_EC(esr)) { 304 case ESR_ELx_EC_SVC32: 305 el0_svc_compat(regs); 306 break; 307 case ESR_ELx_EC_DABT_LOW: 308 el0_da(regs, esr); 309 break; 310 case ESR_ELx_EC_IABT_LOW: 311 el0_ia(regs, esr); 312 break; 313 case ESR_ELx_EC_FP_ASIMD: 314 el0_fpsimd_acc(regs, esr); 315 break; 316 case ESR_ELx_EC_FP_EXC32: 317 el0_fpsimd_exc(regs, esr); 318 break; 319 case ESR_ELx_EC_PC_ALIGN: 320 el0_pc(regs, esr); 321 break; 322 case ESR_ELx_EC_UNKNOWN: 323 case ESR_ELx_EC_CP14_MR: 324 case ESR_ELx_EC_CP14_LS: 325 case ESR_ELx_EC_CP14_64: 326 el0_undef(regs); 327 break; 328 case ESR_ELx_EC_CP15_32: 329 case ESR_ELx_EC_CP15_64: 330 el0_cp15(regs, esr); 331 break; 332 case ESR_ELx_EC_BREAKPT_LOW: 333 case ESR_ELx_EC_SOFTSTP_LOW: 334 case ESR_ELx_EC_WATCHPT_LOW: 335 case ESR_ELx_EC_BKPT32: 336 el0_dbg(regs, esr); 337 break; 338 default: 339 el0_inv(regs, esr); 340 } 341 } 342 NOKPROBE_SYMBOL(el0_sync_compat_handler); 343 #endif /* CONFIG_COMPAT */ 344