1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Exception handling code 4 * 5 * Copyright (C) 2019 ARM Ltd. 6 */ 7 8 #include <linux/context_tracking.h> 9 #include <linux/ptrace.h> 10 #include <linux/thread_info.h> 11 12 #include <asm/cpufeature.h> 13 #include <asm/daifflags.h> 14 #include <asm/esr.h> 15 #include <asm/exception.h> 16 #include <asm/kprobes.h> 17 #include <asm/mmu.h> 18 #include <asm/sysreg.h> 19 20 /* 21 * This is intended to match the logic in irqentry_enter(), handling the kernel 22 * mode transitions only. 23 */ 24 static void noinstr enter_from_kernel_mode(struct pt_regs *regs) 25 { 26 regs->exit_rcu = false; 27 28 if (!IS_ENABLED(CONFIG_TINY_RCU) && is_idle_task(current)) { 29 lockdep_hardirqs_off(CALLER_ADDR0); 30 rcu_irq_enter(); 31 trace_hardirqs_off_finish(); 32 33 regs->exit_rcu = true; 34 return; 35 } 36 37 lockdep_hardirqs_off(CALLER_ADDR0); 38 rcu_irq_enter_check_tick(); 39 trace_hardirqs_off_finish(); 40 } 41 42 /* 43 * This is intended to match the logic in irqentry_exit(), handling the kernel 44 * mode transitions only, and with preemption handled elsewhere. 45 */ 46 static void noinstr exit_to_kernel_mode(struct pt_regs *regs) 47 { 48 lockdep_assert_irqs_disabled(); 49 50 if (interrupts_enabled(regs)) { 51 if (regs->exit_rcu) { 52 trace_hardirqs_on_prepare(); 53 lockdep_hardirqs_on_prepare(CALLER_ADDR0); 54 rcu_irq_exit(); 55 lockdep_hardirqs_on(CALLER_ADDR0); 56 return; 57 } 58 59 trace_hardirqs_on(); 60 } else { 61 if (regs->exit_rcu) 62 rcu_irq_exit(); 63 } 64 } 65 66 void noinstr arm64_enter_nmi(struct pt_regs *regs) 67 { 68 regs->lockdep_hardirqs = lockdep_hardirqs_enabled(); 69 70 __nmi_enter(); 71 lockdep_hardirqs_off(CALLER_ADDR0); 72 lockdep_hardirq_enter(); 73 rcu_nmi_enter(); 74 75 trace_hardirqs_off_finish(); 76 ftrace_nmi_enter(); 77 } 78 79 void noinstr arm64_exit_nmi(struct pt_regs *regs) 80 { 81 bool restore = regs->lockdep_hardirqs; 82 83 ftrace_nmi_exit(); 84 if (restore) { 85 trace_hardirqs_on_prepare(); 86 lockdep_hardirqs_on_prepare(CALLER_ADDR0); 87 } 88 89 rcu_nmi_exit(); 90 lockdep_hardirq_exit(); 91 if (restore) 92 lockdep_hardirqs_on(CALLER_ADDR0); 93 __nmi_exit(); 94 } 95 96 asmlinkage void noinstr enter_el1_irq_or_nmi(struct pt_regs *regs) 97 { 98 if (IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && !interrupts_enabled(regs)) 99 arm64_enter_nmi(regs); 100 else 101 enter_from_kernel_mode(regs); 102 } 103 104 asmlinkage void noinstr exit_el1_irq_or_nmi(struct pt_regs *regs) 105 { 106 if (IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && !interrupts_enabled(regs)) 107 arm64_exit_nmi(regs); 108 else 109 exit_to_kernel_mode(regs); 110 } 111 112 static void noinstr el1_abort(struct pt_regs *regs, unsigned long esr) 113 { 114 unsigned long far = read_sysreg(far_el1); 115 116 enter_from_kernel_mode(regs); 117 local_daif_inherit(regs); 118 far = untagged_addr(far); 119 do_mem_abort(far, esr, regs); 120 local_daif_mask(); 121 exit_to_kernel_mode(regs); 122 } 123 124 static void noinstr el1_pc(struct pt_regs *regs, unsigned long esr) 125 { 126 unsigned long far = read_sysreg(far_el1); 127 128 enter_from_kernel_mode(regs); 129 local_daif_inherit(regs); 130 do_sp_pc_abort(far, esr, regs); 131 local_daif_mask(); 132 exit_to_kernel_mode(regs); 133 } 134 135 static void noinstr el1_undef(struct pt_regs *regs) 136 { 137 enter_from_kernel_mode(regs); 138 local_daif_inherit(regs); 139 do_undefinstr(regs); 140 local_daif_mask(); 141 exit_to_kernel_mode(regs); 142 } 143 144 static void noinstr el1_inv(struct pt_regs *regs, unsigned long esr) 145 { 146 enter_from_kernel_mode(regs); 147 local_daif_inherit(regs); 148 bad_mode(regs, 0, esr); 149 local_daif_mask(); 150 exit_to_kernel_mode(regs); 151 } 152 153 static void noinstr arm64_enter_el1_dbg(struct pt_regs *regs) 154 { 155 regs->lockdep_hardirqs = lockdep_hardirqs_enabled(); 156 157 lockdep_hardirqs_off(CALLER_ADDR0); 158 rcu_nmi_enter(); 159 160 trace_hardirqs_off_finish(); 161 } 162 163 static void noinstr arm64_exit_el1_dbg(struct pt_regs *regs) 164 { 165 bool restore = regs->lockdep_hardirqs; 166 167 if (restore) { 168 trace_hardirqs_on_prepare(); 169 lockdep_hardirqs_on_prepare(CALLER_ADDR0); 170 } 171 172 rcu_nmi_exit(); 173 if (restore) 174 lockdep_hardirqs_on(CALLER_ADDR0); 175 } 176 177 static void noinstr el1_dbg(struct pt_regs *regs, unsigned long esr) 178 { 179 unsigned long far = read_sysreg(far_el1); 180 181 /* 182 * The CPU masked interrupts, and we are leaving them masked during 183 * do_debug_exception(). Update PMR as if we had called 184 * local_daif_mask(). 185 */ 186 if (system_uses_irq_prio_masking()) 187 gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET); 188 189 arm64_enter_el1_dbg(regs); 190 do_debug_exception(far, esr, regs); 191 arm64_exit_el1_dbg(regs); 192 } 193 194 static void noinstr el1_fpac(struct pt_regs *regs, unsigned long esr) 195 { 196 enter_from_kernel_mode(regs); 197 local_daif_inherit(regs); 198 do_ptrauth_fault(regs, esr); 199 local_daif_mask(); 200 exit_to_kernel_mode(regs); 201 } 202 203 asmlinkage void noinstr el1_sync_handler(struct pt_regs *regs) 204 { 205 unsigned long esr = read_sysreg(esr_el1); 206 207 switch (ESR_ELx_EC(esr)) { 208 case ESR_ELx_EC_DABT_CUR: 209 case ESR_ELx_EC_IABT_CUR: 210 el1_abort(regs, esr); 211 break; 212 /* 213 * We don't handle ESR_ELx_EC_SP_ALIGN, since we will have hit a 214 * recursive exception when trying to push the initial pt_regs. 215 */ 216 case ESR_ELx_EC_PC_ALIGN: 217 el1_pc(regs, esr); 218 break; 219 case ESR_ELx_EC_SYS64: 220 case ESR_ELx_EC_UNKNOWN: 221 el1_undef(regs); 222 break; 223 case ESR_ELx_EC_BREAKPT_CUR: 224 case ESR_ELx_EC_SOFTSTP_CUR: 225 case ESR_ELx_EC_WATCHPT_CUR: 226 case ESR_ELx_EC_BRK64: 227 el1_dbg(regs, esr); 228 break; 229 case ESR_ELx_EC_FPAC: 230 el1_fpac(regs, esr); 231 break; 232 default: 233 el1_inv(regs, esr); 234 } 235 } 236 237 asmlinkage void noinstr enter_from_user_mode(void) 238 { 239 lockdep_hardirqs_off(CALLER_ADDR0); 240 CT_WARN_ON(ct_state() != CONTEXT_USER); 241 user_exit_irqoff(); 242 trace_hardirqs_off_finish(); 243 } 244 245 asmlinkage void noinstr exit_to_user_mode(void) 246 { 247 trace_hardirqs_on_prepare(); 248 lockdep_hardirqs_on_prepare(CALLER_ADDR0); 249 user_enter_irqoff(); 250 lockdep_hardirqs_on(CALLER_ADDR0); 251 } 252 253 static void noinstr el0_da(struct pt_regs *regs, unsigned long esr) 254 { 255 unsigned long far = read_sysreg(far_el1); 256 257 enter_from_user_mode(); 258 local_daif_restore(DAIF_PROCCTX); 259 far = untagged_addr(far); 260 do_mem_abort(far, esr, regs); 261 } 262 263 static void noinstr el0_ia(struct pt_regs *regs, unsigned long esr) 264 { 265 unsigned long far = read_sysreg(far_el1); 266 267 /* 268 * We've taken an instruction abort from userspace and not yet 269 * re-enabled IRQs. If the address is a kernel address, apply 270 * BP hardening prior to enabling IRQs and pre-emption. 271 */ 272 if (!is_ttbr0_addr(far)) 273 arm64_apply_bp_hardening(); 274 275 enter_from_user_mode(); 276 local_daif_restore(DAIF_PROCCTX); 277 do_mem_abort(far, esr, regs); 278 } 279 280 static void noinstr el0_fpsimd_acc(struct pt_regs *regs, unsigned long esr) 281 { 282 enter_from_user_mode(); 283 local_daif_restore(DAIF_PROCCTX); 284 do_fpsimd_acc(esr, regs); 285 } 286 287 static void noinstr el0_sve_acc(struct pt_regs *regs, unsigned long esr) 288 { 289 enter_from_user_mode(); 290 local_daif_restore(DAIF_PROCCTX); 291 do_sve_acc(esr, regs); 292 } 293 294 static void noinstr el0_fpsimd_exc(struct pt_regs *regs, unsigned long esr) 295 { 296 enter_from_user_mode(); 297 local_daif_restore(DAIF_PROCCTX); 298 do_fpsimd_exc(esr, regs); 299 } 300 301 static void noinstr el0_sys(struct pt_regs *regs, unsigned long esr) 302 { 303 enter_from_user_mode(); 304 local_daif_restore(DAIF_PROCCTX); 305 do_sysinstr(esr, regs); 306 } 307 308 static void noinstr el0_pc(struct pt_regs *regs, unsigned long esr) 309 { 310 unsigned long far = read_sysreg(far_el1); 311 312 if (!is_ttbr0_addr(instruction_pointer(regs))) 313 arm64_apply_bp_hardening(); 314 315 enter_from_user_mode(); 316 local_daif_restore(DAIF_PROCCTX); 317 do_sp_pc_abort(far, esr, regs); 318 } 319 320 static void noinstr el0_sp(struct pt_regs *regs, unsigned long esr) 321 { 322 enter_from_user_mode(); 323 local_daif_restore(DAIF_PROCCTX); 324 do_sp_pc_abort(regs->sp, esr, regs); 325 } 326 327 static void noinstr el0_undef(struct pt_regs *regs) 328 { 329 enter_from_user_mode(); 330 local_daif_restore(DAIF_PROCCTX); 331 do_undefinstr(regs); 332 } 333 334 static void noinstr el0_bti(struct pt_regs *regs) 335 { 336 enter_from_user_mode(); 337 local_daif_restore(DAIF_PROCCTX); 338 do_bti(regs); 339 } 340 341 static void noinstr el0_inv(struct pt_regs *regs, unsigned long esr) 342 { 343 enter_from_user_mode(); 344 local_daif_restore(DAIF_PROCCTX); 345 bad_el0_sync(regs, 0, esr); 346 } 347 348 static void noinstr el0_dbg(struct pt_regs *regs, unsigned long esr) 349 { 350 /* Only watchpoints write FAR_EL1, otherwise its UNKNOWN */ 351 unsigned long far = read_sysreg(far_el1); 352 353 if (system_uses_irq_prio_masking()) 354 gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET); 355 356 enter_from_user_mode(); 357 do_debug_exception(far, esr, regs); 358 local_daif_restore(DAIF_PROCCTX_NOIRQ); 359 } 360 361 static void noinstr el0_svc(struct pt_regs *regs) 362 { 363 if (system_uses_irq_prio_masking()) 364 gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET); 365 366 enter_from_user_mode(); 367 do_el0_svc(regs); 368 } 369 370 static void noinstr el0_fpac(struct pt_regs *regs, unsigned long esr) 371 { 372 enter_from_user_mode(); 373 local_daif_restore(DAIF_PROCCTX); 374 do_ptrauth_fault(regs, esr); 375 } 376 377 asmlinkage void noinstr el0_sync_handler(struct pt_regs *regs) 378 { 379 unsigned long esr = read_sysreg(esr_el1); 380 381 switch (ESR_ELx_EC(esr)) { 382 case ESR_ELx_EC_SVC64: 383 el0_svc(regs); 384 break; 385 case ESR_ELx_EC_DABT_LOW: 386 el0_da(regs, esr); 387 break; 388 case ESR_ELx_EC_IABT_LOW: 389 el0_ia(regs, esr); 390 break; 391 case ESR_ELx_EC_FP_ASIMD: 392 el0_fpsimd_acc(regs, esr); 393 break; 394 case ESR_ELx_EC_SVE: 395 el0_sve_acc(regs, esr); 396 break; 397 case ESR_ELx_EC_FP_EXC64: 398 el0_fpsimd_exc(regs, esr); 399 break; 400 case ESR_ELx_EC_SYS64: 401 case ESR_ELx_EC_WFx: 402 el0_sys(regs, esr); 403 break; 404 case ESR_ELx_EC_SP_ALIGN: 405 el0_sp(regs, esr); 406 break; 407 case ESR_ELx_EC_PC_ALIGN: 408 el0_pc(regs, esr); 409 break; 410 case ESR_ELx_EC_UNKNOWN: 411 el0_undef(regs); 412 break; 413 case ESR_ELx_EC_BTI: 414 el0_bti(regs); 415 break; 416 case ESR_ELx_EC_BREAKPT_LOW: 417 case ESR_ELx_EC_SOFTSTP_LOW: 418 case ESR_ELx_EC_WATCHPT_LOW: 419 case ESR_ELx_EC_BRK64: 420 el0_dbg(regs, esr); 421 break; 422 case ESR_ELx_EC_FPAC: 423 el0_fpac(regs, esr); 424 break; 425 default: 426 el0_inv(regs, esr); 427 } 428 } 429 430 #ifdef CONFIG_COMPAT 431 static void noinstr el0_cp15(struct pt_regs *regs, unsigned long esr) 432 { 433 enter_from_user_mode(); 434 local_daif_restore(DAIF_PROCCTX); 435 do_cp15instr(esr, regs); 436 } 437 438 static void noinstr el0_svc_compat(struct pt_regs *regs) 439 { 440 if (system_uses_irq_prio_masking()) 441 gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET); 442 443 enter_from_user_mode(); 444 do_el0_svc_compat(regs); 445 } 446 447 asmlinkage void noinstr el0_sync_compat_handler(struct pt_regs *regs) 448 { 449 unsigned long esr = read_sysreg(esr_el1); 450 451 switch (ESR_ELx_EC(esr)) { 452 case ESR_ELx_EC_SVC32: 453 el0_svc_compat(regs); 454 break; 455 case ESR_ELx_EC_DABT_LOW: 456 el0_da(regs, esr); 457 break; 458 case ESR_ELx_EC_IABT_LOW: 459 el0_ia(regs, esr); 460 break; 461 case ESR_ELx_EC_FP_ASIMD: 462 el0_fpsimd_acc(regs, esr); 463 break; 464 case ESR_ELx_EC_FP_EXC32: 465 el0_fpsimd_exc(regs, esr); 466 break; 467 case ESR_ELx_EC_PC_ALIGN: 468 el0_pc(regs, esr); 469 break; 470 case ESR_ELx_EC_UNKNOWN: 471 case ESR_ELx_EC_CP14_MR: 472 case ESR_ELx_EC_CP14_LS: 473 case ESR_ELx_EC_CP14_64: 474 el0_undef(regs); 475 break; 476 case ESR_ELx_EC_CP15_32: 477 case ESR_ELx_EC_CP15_64: 478 el0_cp15(regs, esr); 479 break; 480 case ESR_ELx_EC_BREAKPT_LOW: 481 case ESR_ELx_EC_SOFTSTP_LOW: 482 case ESR_ELx_EC_WATCHPT_LOW: 483 case ESR_ELx_EC_BKPT32: 484 el0_dbg(regs, esr); 485 break; 486 default: 487 el0_inv(regs, esr); 488 } 489 } 490 #endif /* CONFIG_COMPAT */ 491