1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Exception handling code 4 * 5 * Copyright (C) 2019 ARM Ltd. 6 */ 7 8 #include <linux/context_tracking.h> 9 #include <linux/ptrace.h> 10 #include <linux/thread_info.h> 11 12 #include <asm/cpufeature.h> 13 #include <asm/daifflags.h> 14 #include <asm/esr.h> 15 #include <asm/exception.h> 16 #include <asm/kprobes.h> 17 #include <asm/mmu.h> 18 #include <asm/sysreg.h> 19 20 static void notrace el1_abort(struct pt_regs *regs, unsigned long esr) 21 { 22 unsigned long far = read_sysreg(far_el1); 23 24 local_daif_inherit(regs); 25 far = untagged_addr(far); 26 do_mem_abort(far, esr, regs); 27 } 28 NOKPROBE_SYMBOL(el1_abort); 29 30 static void notrace el1_pc(struct pt_regs *regs, unsigned long esr) 31 { 32 unsigned long far = read_sysreg(far_el1); 33 34 local_daif_inherit(regs); 35 do_sp_pc_abort(far, esr, regs); 36 } 37 NOKPROBE_SYMBOL(el1_pc); 38 39 static void notrace el1_undef(struct pt_regs *regs) 40 { 41 local_daif_inherit(regs); 42 do_undefinstr(regs); 43 } 44 NOKPROBE_SYMBOL(el1_undef); 45 46 static void notrace el1_inv(struct pt_regs *regs, unsigned long esr) 47 { 48 local_daif_inherit(regs); 49 bad_mode(regs, 0, esr); 50 } 51 NOKPROBE_SYMBOL(el1_inv); 52 53 static void notrace el1_dbg(struct pt_regs *regs, unsigned long esr) 54 { 55 unsigned long far = read_sysreg(far_el1); 56 57 /* 58 * The CPU masked interrupts, and we are leaving them masked during 59 * do_debug_exception(). Update PMR as if we had called 60 * local_daif_mask(). 61 */ 62 if (system_uses_irq_prio_masking()) 63 gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET); 64 65 do_debug_exception(far, esr, regs); 66 } 67 NOKPROBE_SYMBOL(el1_dbg); 68 69 static void notrace el1_fpac(struct pt_regs *regs, unsigned long esr) 70 { 71 local_daif_inherit(regs); 72 do_ptrauth_fault(regs, esr); 73 } 74 NOKPROBE_SYMBOL(el1_fpac); 75 76 asmlinkage void notrace el1_sync_handler(struct pt_regs *regs) 77 { 78 unsigned long esr = read_sysreg(esr_el1); 79 80 switch (ESR_ELx_EC(esr)) { 81 case ESR_ELx_EC_DABT_CUR: 82 case ESR_ELx_EC_IABT_CUR: 83 el1_abort(regs, esr); 84 break; 85 /* 86 * We don't handle ESR_ELx_EC_SP_ALIGN, since we will have hit a 87 * recursive exception when trying to push the initial pt_regs. 88 */ 89 case ESR_ELx_EC_PC_ALIGN: 90 el1_pc(regs, esr); 91 break; 92 case ESR_ELx_EC_SYS64: 93 case ESR_ELx_EC_UNKNOWN: 94 el1_undef(regs); 95 break; 96 case ESR_ELx_EC_BREAKPT_CUR: 97 case ESR_ELx_EC_SOFTSTP_CUR: 98 case ESR_ELx_EC_WATCHPT_CUR: 99 case ESR_ELx_EC_BRK64: 100 el1_dbg(regs, esr); 101 break; 102 case ESR_ELx_EC_FPAC: 103 el1_fpac(regs, esr); 104 break; 105 default: 106 el1_inv(regs, esr); 107 } 108 } 109 NOKPROBE_SYMBOL(el1_sync_handler); 110 111 static void notrace el0_da(struct pt_regs *regs, unsigned long esr) 112 { 113 unsigned long far = read_sysreg(far_el1); 114 115 user_exit_irqoff(); 116 local_daif_restore(DAIF_PROCCTX); 117 far = untagged_addr(far); 118 do_mem_abort(far, esr, regs); 119 } 120 NOKPROBE_SYMBOL(el0_da); 121 122 static void notrace el0_ia(struct pt_regs *regs, unsigned long esr) 123 { 124 unsigned long far = read_sysreg(far_el1); 125 126 /* 127 * We've taken an instruction abort from userspace and not yet 128 * re-enabled IRQs. If the address is a kernel address, apply 129 * BP hardening prior to enabling IRQs and pre-emption. 130 */ 131 if (!is_ttbr0_addr(far)) 132 arm64_apply_bp_hardening(); 133 134 user_exit_irqoff(); 135 local_daif_restore(DAIF_PROCCTX); 136 do_mem_abort(far, esr, regs); 137 } 138 NOKPROBE_SYMBOL(el0_ia); 139 140 static void notrace el0_fpsimd_acc(struct pt_regs *regs, unsigned long esr) 141 { 142 user_exit_irqoff(); 143 local_daif_restore(DAIF_PROCCTX); 144 do_fpsimd_acc(esr, regs); 145 } 146 NOKPROBE_SYMBOL(el0_fpsimd_acc); 147 148 static void notrace el0_sve_acc(struct pt_regs *regs, unsigned long esr) 149 { 150 user_exit_irqoff(); 151 local_daif_restore(DAIF_PROCCTX); 152 do_sve_acc(esr, regs); 153 } 154 NOKPROBE_SYMBOL(el0_sve_acc); 155 156 static void notrace el0_fpsimd_exc(struct pt_regs *regs, unsigned long esr) 157 { 158 user_exit_irqoff(); 159 local_daif_restore(DAIF_PROCCTX); 160 do_fpsimd_exc(esr, regs); 161 } 162 NOKPROBE_SYMBOL(el0_fpsimd_exc); 163 164 static void notrace el0_sys(struct pt_regs *regs, unsigned long esr) 165 { 166 user_exit_irqoff(); 167 local_daif_restore(DAIF_PROCCTX); 168 do_sysinstr(esr, regs); 169 } 170 NOKPROBE_SYMBOL(el0_sys); 171 172 static void notrace el0_pc(struct pt_regs *regs, unsigned long esr) 173 { 174 unsigned long far = read_sysreg(far_el1); 175 176 if (!is_ttbr0_addr(instruction_pointer(regs))) 177 arm64_apply_bp_hardening(); 178 179 user_exit_irqoff(); 180 local_daif_restore(DAIF_PROCCTX); 181 do_sp_pc_abort(far, esr, regs); 182 } 183 NOKPROBE_SYMBOL(el0_pc); 184 185 static void notrace el0_sp(struct pt_regs *regs, unsigned long esr) 186 { 187 user_exit_irqoff(); 188 local_daif_restore(DAIF_PROCCTX); 189 do_sp_pc_abort(regs->sp, esr, regs); 190 } 191 NOKPROBE_SYMBOL(el0_sp); 192 193 static void notrace el0_undef(struct pt_regs *regs) 194 { 195 user_exit_irqoff(); 196 local_daif_restore(DAIF_PROCCTX); 197 do_undefinstr(regs); 198 } 199 NOKPROBE_SYMBOL(el0_undef); 200 201 static void notrace el0_bti(struct pt_regs *regs) 202 { 203 user_exit_irqoff(); 204 local_daif_restore(DAIF_PROCCTX); 205 do_bti(regs); 206 } 207 NOKPROBE_SYMBOL(el0_bti); 208 209 static void notrace el0_inv(struct pt_regs *regs, unsigned long esr) 210 { 211 user_exit_irqoff(); 212 local_daif_restore(DAIF_PROCCTX); 213 bad_el0_sync(regs, 0, esr); 214 } 215 NOKPROBE_SYMBOL(el0_inv); 216 217 static void notrace el0_dbg(struct pt_regs *regs, unsigned long esr) 218 { 219 /* Only watchpoints write FAR_EL1, otherwise its UNKNOWN */ 220 unsigned long far = read_sysreg(far_el1); 221 222 if (system_uses_irq_prio_masking()) 223 gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET); 224 225 user_exit_irqoff(); 226 do_debug_exception(far, esr, regs); 227 local_daif_restore(DAIF_PROCCTX_NOIRQ); 228 } 229 NOKPROBE_SYMBOL(el0_dbg); 230 231 static void notrace el0_svc(struct pt_regs *regs) 232 { 233 if (system_uses_irq_prio_masking()) 234 gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET); 235 236 do_el0_svc(regs); 237 } 238 NOKPROBE_SYMBOL(el0_svc); 239 240 static void notrace el0_fpac(struct pt_regs *regs, unsigned long esr) 241 { 242 user_exit_irqoff(); 243 local_daif_restore(DAIF_PROCCTX); 244 do_ptrauth_fault(regs, esr); 245 } 246 NOKPROBE_SYMBOL(el0_fpac); 247 248 asmlinkage void notrace el0_sync_handler(struct pt_regs *regs) 249 { 250 unsigned long esr = read_sysreg(esr_el1); 251 252 switch (ESR_ELx_EC(esr)) { 253 case ESR_ELx_EC_SVC64: 254 el0_svc(regs); 255 break; 256 case ESR_ELx_EC_DABT_LOW: 257 el0_da(regs, esr); 258 break; 259 case ESR_ELx_EC_IABT_LOW: 260 el0_ia(regs, esr); 261 break; 262 case ESR_ELx_EC_FP_ASIMD: 263 el0_fpsimd_acc(regs, esr); 264 break; 265 case ESR_ELx_EC_SVE: 266 el0_sve_acc(regs, esr); 267 break; 268 case ESR_ELx_EC_FP_EXC64: 269 el0_fpsimd_exc(regs, esr); 270 break; 271 case ESR_ELx_EC_SYS64: 272 case ESR_ELx_EC_WFx: 273 el0_sys(regs, esr); 274 break; 275 case ESR_ELx_EC_SP_ALIGN: 276 el0_sp(regs, esr); 277 break; 278 case ESR_ELx_EC_PC_ALIGN: 279 el0_pc(regs, esr); 280 break; 281 case ESR_ELx_EC_UNKNOWN: 282 el0_undef(regs); 283 break; 284 case ESR_ELx_EC_BTI: 285 el0_bti(regs); 286 break; 287 case ESR_ELx_EC_BREAKPT_LOW: 288 case ESR_ELx_EC_SOFTSTP_LOW: 289 case ESR_ELx_EC_WATCHPT_LOW: 290 case ESR_ELx_EC_BRK64: 291 el0_dbg(regs, esr); 292 break; 293 case ESR_ELx_EC_FPAC: 294 el0_fpac(regs, esr); 295 break; 296 default: 297 el0_inv(regs, esr); 298 } 299 } 300 NOKPROBE_SYMBOL(el0_sync_handler); 301 302 #ifdef CONFIG_COMPAT 303 static void notrace el0_cp15(struct pt_regs *regs, unsigned long esr) 304 { 305 user_exit_irqoff(); 306 local_daif_restore(DAIF_PROCCTX); 307 do_cp15instr(esr, regs); 308 } 309 NOKPROBE_SYMBOL(el0_cp15); 310 311 static void notrace el0_svc_compat(struct pt_regs *regs) 312 { 313 if (system_uses_irq_prio_masking()) 314 gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET); 315 316 do_el0_svc_compat(regs); 317 } 318 NOKPROBE_SYMBOL(el0_svc_compat); 319 320 asmlinkage void notrace el0_sync_compat_handler(struct pt_regs *regs) 321 { 322 unsigned long esr = read_sysreg(esr_el1); 323 324 switch (ESR_ELx_EC(esr)) { 325 case ESR_ELx_EC_SVC32: 326 el0_svc_compat(regs); 327 break; 328 case ESR_ELx_EC_DABT_LOW: 329 el0_da(regs, esr); 330 break; 331 case ESR_ELx_EC_IABT_LOW: 332 el0_ia(regs, esr); 333 break; 334 case ESR_ELx_EC_FP_ASIMD: 335 el0_fpsimd_acc(regs, esr); 336 break; 337 case ESR_ELx_EC_FP_EXC32: 338 el0_fpsimd_exc(regs, esr); 339 break; 340 case ESR_ELx_EC_PC_ALIGN: 341 el0_pc(regs, esr); 342 break; 343 case ESR_ELx_EC_UNKNOWN: 344 case ESR_ELx_EC_CP14_MR: 345 case ESR_ELx_EC_CP14_LS: 346 case ESR_ELx_EC_CP14_64: 347 el0_undef(regs); 348 break; 349 case ESR_ELx_EC_CP15_32: 350 case ESR_ELx_EC_CP15_64: 351 el0_cp15(regs, esr); 352 break; 353 case ESR_ELx_EC_BREAKPT_LOW: 354 case ESR_ELx_EC_SOFTSTP_LOW: 355 case ESR_ELx_EC_WATCHPT_LOW: 356 case ESR_ELx_EC_BKPT32: 357 el0_dbg(regs, esr); 358 break; 359 default: 360 el0_inv(regs, esr); 361 } 362 } 363 NOKPROBE_SYMBOL(el0_sync_compat_handler); 364 #endif /* CONFIG_COMPAT */ 365