1 // SPDX-License-Identifier: GPL-2.0-or-later 2 3 #include <linux/context_tracking.h> 4 #include <linux/err.h> 5 #include <linux/compat.h> 6 #include <linux/sched/debug.h> /* for show_regs */ 7 8 #include <asm/kup.h> 9 #include <asm/cputime.h> 10 #include <asm/hw_irq.h> 11 #include <asm/interrupt.h> 12 #include <asm/kprobes.h> 13 #include <asm/paca.h> 14 #include <asm/ptrace.h> 15 #include <asm/reg.h> 16 #include <asm/signal.h> 17 #include <asm/switch_to.h> 18 #include <asm/syscall.h> 19 #include <asm/time.h> 20 #include <asm/tm.h> 21 #include <asm/unistd.h> 22 23 #if defined(CONFIG_PPC_ADV_DEBUG_REGS) && defined(CONFIG_PPC32) 24 unsigned long global_dbcr0[NR_CPUS]; 25 #endif 26 27 #ifdef CONFIG_PPC_BOOK3S_64 28 DEFINE_STATIC_KEY_FALSE(interrupt_exit_not_reentrant); 29 static inline bool exit_must_hard_disable(void) 30 { 31 return static_branch_unlikely(&interrupt_exit_not_reentrant); 32 } 33 #else 34 static inline bool exit_must_hard_disable(void) 35 { 36 return true; 37 } 38 #endif 39 40 /* 41 * local irqs must be disabled. Returns false if the caller must re-enable 42 * them, check for new work, and try again. 43 * 44 * This should be called with local irqs disabled, but if they were previously 45 * enabled when the interrupt handler returns (indicating a process-context / 46 * synchronous interrupt) then irqs_enabled should be true. 47 * 48 * restartable is true then EE/RI can be left on because interrupts are handled 49 * with a restart sequence. 50 */ 51 static notrace __always_inline bool prep_irq_for_enabled_exit(bool restartable) 52 { 53 /* This must be done with RI=1 because tracing may touch vmaps */ 54 trace_hardirqs_on(); 55 56 if (exit_must_hard_disable() || !restartable) 57 __hard_EE_RI_disable(); 58 59 #ifdef CONFIG_PPC64 60 /* This pattern matches prep_irq_for_idle */ 61 if (unlikely(lazy_irq_pending_nocheck())) { 62 if (exit_must_hard_disable() || !restartable) { 63 local_paca->irq_happened |= PACA_IRQ_HARD_DIS; 64 __hard_RI_enable(); 65 } 66 trace_hardirqs_off(); 67 68 return false; 69 } 70 #endif 71 return true; 72 } 73 74 static notrace void booke_load_dbcr0(void) 75 { 76 #ifdef CONFIG_PPC_ADV_DEBUG_REGS 77 unsigned long dbcr0 = current->thread.debug.dbcr0; 78 79 if (likely(!(dbcr0 & DBCR0_IDM))) 80 return; 81 82 /* 83 * Check to see if the dbcr0 register is set up to debug. 84 * Use the internal debug mode bit to do this. 85 */ 86 mtmsr(mfmsr() & ~MSR_DE); 87 if (IS_ENABLED(CONFIG_PPC32)) { 88 isync(); 89 global_dbcr0[smp_processor_id()] = mfspr(SPRN_DBCR0); 90 } 91 mtspr(SPRN_DBCR0, dbcr0); 92 mtspr(SPRN_DBSR, -1); 93 #endif 94 } 95 96 static void check_return_regs_valid(struct pt_regs *regs) 97 { 98 #ifdef CONFIG_PPC_BOOK3S_64 99 unsigned long trap, srr0, srr1; 100 static bool warned; 101 u8 *validp; 102 char *h; 103 104 if (trap_is_scv(regs)) 105 return; 106 107 trap = TRAP(regs); 108 // EE in HV mode sets HSRRs like 0xea0 109 if (cpu_has_feature(CPU_FTR_HVMODE) && trap == INTERRUPT_EXTERNAL) 110 trap = 0xea0; 111 112 switch (trap) { 113 case 0x980: 114 case INTERRUPT_H_DATA_STORAGE: 115 case 0xe20: 116 case 0xe40: 117 case INTERRUPT_HMI: 118 case 0xe80: 119 case 0xea0: 120 case INTERRUPT_H_FAC_UNAVAIL: 121 case 0x1200: 122 case 0x1500: 123 case 0x1600: 124 case 0x1800: 125 validp = &local_paca->hsrr_valid; 126 if (!*validp) 127 return; 128 129 srr0 = mfspr(SPRN_HSRR0); 130 srr1 = mfspr(SPRN_HSRR1); 131 h = "H"; 132 133 break; 134 default: 135 validp = &local_paca->srr_valid; 136 if (!*validp) 137 return; 138 139 srr0 = mfspr(SPRN_SRR0); 140 srr1 = mfspr(SPRN_SRR1); 141 h = ""; 142 break; 143 } 144 145 if (srr0 == regs->nip && srr1 == regs->msr) 146 return; 147 148 /* 149 * A NMI / soft-NMI interrupt may have come in after we found 150 * srr_valid and before the SRRs are loaded. The interrupt then 151 * comes in and clobbers SRRs and clears srr_valid. Then we load 152 * the SRRs here and test them above and find they don't match. 153 * 154 * Test validity again after that, to catch such false positives. 155 * 156 * This test in general will have some window for false negatives 157 * and may not catch and fix all such cases if an NMI comes in 158 * later and clobbers SRRs without clearing srr_valid, but hopefully 159 * such things will get caught most of the time, statistically 160 * enough to be able to get a warning out. 161 */ 162 barrier(); 163 164 if (!*validp) 165 return; 166 167 if (!warned) { 168 warned = true; 169 printk("%sSRR0 was: %lx should be: %lx\n", h, srr0, regs->nip); 170 printk("%sSRR1 was: %lx should be: %lx\n", h, srr1, regs->msr); 171 show_regs(regs); 172 } 173 174 *validp = 0; /* fixup */ 175 #endif 176 } 177 178 static notrace unsigned long 179 interrupt_exit_user_prepare_main(unsigned long ret, struct pt_regs *regs) 180 { 181 unsigned long ti_flags; 182 183 again: 184 ti_flags = read_thread_flags(); 185 while (unlikely(ti_flags & (_TIF_USER_WORK_MASK & ~_TIF_RESTORE_TM))) { 186 local_irq_enable(); 187 if (ti_flags & _TIF_NEED_RESCHED) { 188 schedule(); 189 } else { 190 /* 191 * SIGPENDING must restore signal handler function 192 * argument GPRs, and some non-volatiles (e.g., r1). 193 * Restore all for now. This could be made lighter. 194 */ 195 if (ti_flags & _TIF_SIGPENDING) 196 ret |= _TIF_RESTOREALL; 197 do_notify_resume(regs, ti_flags); 198 } 199 local_irq_disable(); 200 ti_flags = read_thread_flags(); 201 } 202 203 if (IS_ENABLED(CONFIG_PPC_BOOK3S_64) && IS_ENABLED(CONFIG_PPC_FPU)) { 204 if (IS_ENABLED(CONFIG_PPC_TRANSACTIONAL_MEM) && 205 unlikely((ti_flags & _TIF_RESTORE_TM))) { 206 restore_tm_state(regs); 207 } else { 208 unsigned long mathflags = MSR_FP; 209 210 if (cpu_has_feature(CPU_FTR_VSX)) 211 mathflags |= MSR_VEC | MSR_VSX; 212 else if (cpu_has_feature(CPU_FTR_ALTIVEC)) 213 mathflags |= MSR_VEC; 214 215 /* 216 * If userspace MSR has all available FP bits set, 217 * then they are live and no need to restore. If not, 218 * it means the regs were given up and restore_math 219 * may decide to restore them (to avoid taking an FP 220 * fault). 221 */ 222 if ((regs->msr & mathflags) != mathflags) 223 restore_math(regs); 224 } 225 } 226 227 check_return_regs_valid(regs); 228 229 user_enter_irqoff(); 230 if (!prep_irq_for_enabled_exit(true)) { 231 user_exit_irqoff(); 232 local_irq_enable(); 233 local_irq_disable(); 234 goto again; 235 } 236 237 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM 238 local_paca->tm_scratch = regs->msr; 239 #endif 240 241 booke_load_dbcr0(); 242 243 account_cpu_user_exit(); 244 245 /* Restore user access locks last */ 246 kuap_user_restore(regs); 247 248 return ret; 249 } 250 251 /* 252 * This should be called after a syscall returns, with r3 the return value 253 * from the syscall. If this function returns non-zero, the system call 254 * exit assembly should additionally load all GPR registers and CTR and XER 255 * from the interrupt frame. 256 * 257 * The function graph tracer can not trace the return side of this function, 258 * because RI=0 and soft mask state is "unreconciled", so it is marked notrace. 259 */ 260 notrace unsigned long syscall_exit_prepare(unsigned long r3, 261 struct pt_regs *regs, 262 long scv) 263 { 264 unsigned long ti_flags; 265 unsigned long ret = 0; 266 bool is_not_scv = !IS_ENABLED(CONFIG_PPC_BOOK3S_64) || !scv; 267 268 CT_WARN_ON(ct_state() == CONTEXT_USER); 269 270 kuap_assert_locked(); 271 272 regs->result = r3; 273 274 /* Check whether the syscall is issued inside a restartable sequence */ 275 rseq_syscall(regs); 276 277 ti_flags = read_thread_flags(); 278 279 if (unlikely(r3 >= (unsigned long)-MAX_ERRNO) && is_not_scv) { 280 if (likely(!(ti_flags & (_TIF_NOERROR | _TIF_RESTOREALL)))) { 281 r3 = -r3; 282 regs->ccr |= 0x10000000; /* Set SO bit in CR */ 283 } 284 } 285 286 if (unlikely(ti_flags & _TIF_PERSYSCALL_MASK)) { 287 if (ti_flags & _TIF_RESTOREALL) 288 ret = _TIF_RESTOREALL; 289 else 290 regs->gpr[3] = r3; 291 clear_bits(_TIF_PERSYSCALL_MASK, ¤t_thread_info()->flags); 292 } else { 293 regs->gpr[3] = r3; 294 } 295 296 if (unlikely(ti_flags & _TIF_SYSCALL_DOTRACE)) { 297 do_syscall_trace_leave(regs); 298 ret |= _TIF_RESTOREALL; 299 } 300 301 local_irq_disable(); 302 ret = interrupt_exit_user_prepare_main(ret, regs); 303 304 #ifdef CONFIG_PPC64 305 regs->exit_result = ret; 306 #endif 307 308 return ret; 309 } 310 311 #ifdef CONFIG_PPC64 312 notrace unsigned long syscall_exit_restart(unsigned long r3, struct pt_regs *regs) 313 { 314 /* 315 * This is called when detecting a soft-pending interrupt as well as 316 * an alternate-return interrupt. So we can't just have the alternate 317 * return path clear SRR1[MSR] and set PACA_IRQ_HARD_DIS (unless 318 * the soft-pending case were to fix things up as well). RI might be 319 * disabled, in which case it gets re-enabled by __hard_irq_disable(). 320 */ 321 __hard_irq_disable(); 322 local_paca->irq_happened |= PACA_IRQ_HARD_DIS; 323 324 #ifdef CONFIG_PPC_BOOK3S_64 325 set_kuap(AMR_KUAP_BLOCKED); 326 #endif 327 328 trace_hardirqs_off(); 329 user_exit_irqoff(); 330 account_cpu_user_entry(); 331 332 BUG_ON(!user_mode(regs)); 333 334 regs->exit_result = interrupt_exit_user_prepare_main(regs->exit_result, regs); 335 336 return regs->exit_result; 337 } 338 #endif 339 340 notrace unsigned long interrupt_exit_user_prepare(struct pt_regs *regs) 341 { 342 unsigned long ret; 343 344 BUG_ON(regs_is_unrecoverable(regs)); 345 BUG_ON(arch_irq_disabled_regs(regs)); 346 CT_WARN_ON(ct_state() == CONTEXT_USER); 347 348 /* 349 * We don't need to restore AMR on the way back to userspace for KUAP. 350 * AMR can only have been unlocked if we interrupted the kernel. 351 */ 352 kuap_assert_locked(); 353 354 local_irq_disable(); 355 356 ret = interrupt_exit_user_prepare_main(0, regs); 357 358 #ifdef CONFIG_PPC64 359 regs->exit_result = ret; 360 #endif 361 362 return ret; 363 } 364 365 void preempt_schedule_irq(void); 366 367 notrace unsigned long interrupt_exit_kernel_prepare(struct pt_regs *regs) 368 { 369 unsigned long flags; 370 unsigned long ret = 0; 371 unsigned long kuap; 372 bool stack_store = read_thread_flags() & _TIF_EMULATE_STACK_STORE; 373 374 if (regs_is_unrecoverable(regs)) 375 unrecoverable_exception(regs); 376 /* 377 * CT_WARN_ON comes here via program_check_exception, 378 * so avoid recursion. 379 */ 380 if (TRAP(regs) != INTERRUPT_PROGRAM) 381 CT_WARN_ON(ct_state() == CONTEXT_USER); 382 383 kuap = kuap_get_and_assert_locked(); 384 385 local_irq_save(flags); 386 387 if (!arch_irq_disabled_regs(regs)) { 388 /* Returning to a kernel context with local irqs enabled. */ 389 WARN_ON_ONCE(!(regs->msr & MSR_EE)); 390 again: 391 if (IS_ENABLED(CONFIG_PREEMPT)) { 392 /* Return to preemptible kernel context */ 393 if (unlikely(read_thread_flags() & _TIF_NEED_RESCHED)) { 394 if (preempt_count() == 0) 395 preempt_schedule_irq(); 396 } 397 } 398 399 check_return_regs_valid(regs); 400 401 /* 402 * Stack store exit can't be restarted because the interrupt 403 * stack frame might have been clobbered. 404 */ 405 if (!prep_irq_for_enabled_exit(unlikely(stack_store))) { 406 /* 407 * Replay pending soft-masked interrupts now. Don't 408 * just local_irq_enabe(); local_irq_disable(); because 409 * if we are returning from an asynchronous interrupt 410 * here, another one might hit after irqs are enabled, 411 * and it would exit via this same path allowing 412 * another to fire, and so on unbounded. 413 */ 414 hard_irq_disable(); 415 replay_soft_interrupts(); 416 /* Took an interrupt, may have more exit work to do. */ 417 goto again; 418 } 419 #ifdef CONFIG_PPC64 420 /* 421 * An interrupt may clear MSR[EE] and set this concurrently, 422 * but it will be marked pending and the exit will be retried. 423 * This leaves a racy window where MSR[EE]=0 and HARD_DIS is 424 * clear, until interrupt_exit_kernel_restart() calls 425 * hard_irq_disable(), which will set HARD_DIS again. 426 */ 427 local_paca->irq_happened &= ~PACA_IRQ_HARD_DIS; 428 429 } else { 430 check_return_regs_valid(regs); 431 432 if (unlikely(stack_store)) 433 __hard_EE_RI_disable(); 434 #endif /* CONFIG_PPC64 */ 435 } 436 437 if (unlikely(stack_store)) { 438 clear_bits(_TIF_EMULATE_STACK_STORE, ¤t_thread_info()->flags); 439 ret = 1; 440 } 441 442 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM 443 local_paca->tm_scratch = regs->msr; 444 #endif 445 446 /* 447 * 64s does not want to mfspr(SPRN_AMR) here, because this comes after 448 * mtmsr, which would cause Read-After-Write stalls. Hence, take the 449 * AMR value from the check above. 450 */ 451 kuap_kernel_restore(regs, kuap); 452 453 return ret; 454 } 455 456 #ifdef CONFIG_PPC64 457 notrace unsigned long interrupt_exit_user_restart(struct pt_regs *regs) 458 { 459 __hard_irq_disable(); 460 local_paca->irq_happened |= PACA_IRQ_HARD_DIS; 461 462 #ifdef CONFIG_PPC_BOOK3S_64 463 set_kuap(AMR_KUAP_BLOCKED); 464 #endif 465 466 trace_hardirqs_off(); 467 user_exit_irqoff(); 468 account_cpu_user_entry(); 469 470 BUG_ON(!user_mode(regs)); 471 472 regs->exit_result |= interrupt_exit_user_prepare(regs); 473 474 return regs->exit_result; 475 } 476 477 /* 478 * No real need to return a value here because the stack store case does not 479 * get restarted. 480 */ 481 notrace unsigned long interrupt_exit_kernel_restart(struct pt_regs *regs) 482 { 483 __hard_irq_disable(); 484 local_paca->irq_happened |= PACA_IRQ_HARD_DIS; 485 486 #ifdef CONFIG_PPC_BOOK3S_64 487 set_kuap(AMR_KUAP_BLOCKED); 488 #endif 489 490 if (regs->softe == IRQS_ENABLED) 491 trace_hardirqs_off(); 492 493 BUG_ON(user_mode(regs)); 494 495 return interrupt_exit_kernel_prepare(regs); 496 } 497 #endif 498