1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 #ifndef _ASM_POWERPC_INTERRUPT_H 3 #define _ASM_POWERPC_INTERRUPT_H 4 5 #include <linux/context_tracking.h> 6 #include <linux/hardirq.h> 7 #include <asm/cputime.h> 8 #include <asm/ftrace.h> 9 #include <asm/kprobes.h> 10 #include <asm/runlatch.h> 11 12 struct interrupt_state { 13 #ifdef CONFIG_PPC_BOOK3E_64 14 enum ctx_state ctx_state; 15 #endif 16 }; 17 18 static inline void booke_restore_dbcr0(void) 19 { 20 #ifdef CONFIG_PPC_ADV_DEBUG_REGS 21 unsigned long dbcr0 = current->thread.debug.dbcr0; 22 23 if (IS_ENABLED(CONFIG_PPC32) && unlikely(dbcr0 & DBCR0_IDM)) { 24 mtspr(SPRN_DBSR, -1); 25 mtspr(SPRN_DBCR0, global_dbcr0[smp_processor_id()]); 26 } 27 #endif 28 } 29 30 static inline void interrupt_enter_prepare(struct pt_regs *regs, struct interrupt_state *state) 31 { 32 /* 33 * Book3E reconciles irq soft mask in asm 34 */ 35 #ifdef CONFIG_PPC_BOOK3S_64 36 if (irq_soft_mask_set_return(IRQS_ALL_DISABLED) == IRQS_ENABLED) 37 trace_hardirqs_off(); 38 local_paca->irq_happened |= PACA_IRQ_HARD_DIS; 39 40 if (user_mode(regs)) { 41 CT_WARN_ON(ct_state() != CONTEXT_USER); 42 user_exit_irqoff(); 43 44 account_cpu_user_entry(); 45 account_stolen_time(); 46 } else { 47 /* 48 * CT_WARN_ON comes here via program_check_exception, 49 * so avoid recursion. 50 */ 51 if (TRAP(regs) != 0x700) 52 CT_WARN_ON(ct_state() != CONTEXT_KERNEL); 53 } 54 #endif 55 56 #ifdef CONFIG_PPC_BOOK3E_64 57 state->ctx_state = exception_enter(); 58 if (user_mode(regs)) 59 account_cpu_user_entry(); 60 #endif 61 } 62 63 /* 64 * Care should be taken to note that interrupt_exit_prepare and 65 * interrupt_async_exit_prepare do not necessarily return immediately to 66 * regs context (e.g., if regs is usermode, we don't necessarily return to 67 * user mode). Other interrupts might be taken between here and return, 68 * context switch / preemption may occur in the exit path after this, or a 69 * signal may be delivered, etc. 70 * 71 * The real interrupt exit code is platform specific, e.g., 72 * interrupt_exit_user_prepare / interrupt_exit_kernel_prepare for 64s. 73 * 74 * However interrupt_nmi_exit_prepare does return directly to regs, because 75 * NMIs do not do "exit work" or replay soft-masked interrupts. 76 */ 77 static inline void interrupt_exit_prepare(struct pt_regs *regs, struct interrupt_state *state) 78 { 79 #ifdef CONFIG_PPC_BOOK3E_64 80 exception_exit(state->ctx_state); 81 #endif 82 83 /* 84 * Book3S exits to user via interrupt_exit_user_prepare(), which does 85 * context tracking, which is a cleaner way to handle PREEMPT=y 86 * and avoid context entry/exit in e.g., preempt_schedule_irq()), 87 * which is likely to be where the core code wants to end up. 88 * 89 * The above comment explains why we can't do the 90 * 91 * if (user_mode(regs)) 92 * user_exit_irqoff(); 93 * 94 * sequence here. 95 */ 96 } 97 98 static inline void interrupt_async_enter_prepare(struct pt_regs *regs, struct interrupt_state *state) 99 { 100 #ifdef CONFIG_PPC_BOOK3S_64 101 if (cpu_has_feature(CPU_FTR_CTRL) && 102 !test_thread_local_flags(_TLF_RUNLATCH)) 103 __ppc64_runlatch_on(); 104 #endif 105 106 interrupt_enter_prepare(regs, state); 107 irq_enter(); 108 } 109 110 static inline void interrupt_async_exit_prepare(struct pt_regs *regs, struct interrupt_state *state) 111 { 112 irq_exit(); 113 interrupt_exit_prepare(regs, state); 114 } 115 116 struct interrupt_nmi_state { 117 #ifdef CONFIG_PPC64 118 #ifdef CONFIG_PPC_BOOK3S_64 119 u8 irq_soft_mask; 120 u8 irq_happened; 121 #endif 122 u8 ftrace_enabled; 123 #endif 124 }; 125 126 static inline void interrupt_nmi_enter_prepare(struct pt_regs *regs, struct interrupt_nmi_state *state) 127 { 128 #ifdef CONFIG_PPC64 129 #ifdef CONFIG_PPC_BOOK3S_64 130 state->irq_soft_mask = local_paca->irq_soft_mask; 131 state->irq_happened = local_paca->irq_happened; 132 133 /* 134 * Set IRQS_ALL_DISABLED unconditionally so irqs_disabled() does 135 * the right thing, and set IRQ_HARD_DIS. We do not want to reconcile 136 * because that goes through irq tracing which we don't want in NMI. 137 */ 138 local_paca->irq_soft_mask = IRQS_ALL_DISABLED; 139 local_paca->irq_happened |= PACA_IRQ_HARD_DIS; 140 141 /* Don't do any per-CPU operations until interrupt state is fixed */ 142 #endif 143 /* Allow DEC and PMI to be traced when they are soft-NMI */ 144 if (TRAP(regs) != 0x900 && TRAP(regs) != 0xf00 && TRAP(regs) != 0x260) { 145 state->ftrace_enabled = this_cpu_get_ftrace_enabled(); 146 this_cpu_set_ftrace_enabled(0); 147 } 148 #endif 149 150 /* 151 * Do not use nmi_enter() for pseries hash guest taking a real-mode 152 * NMI because not everything it touches is within the RMA limit. 153 */ 154 if (!IS_ENABLED(CONFIG_PPC_BOOK3S_64) || 155 !firmware_has_feature(FW_FEATURE_LPAR) || 156 radix_enabled() || (mfmsr() & MSR_DR)) 157 nmi_enter(); 158 } 159 160 static inline void interrupt_nmi_exit_prepare(struct pt_regs *regs, struct interrupt_nmi_state *state) 161 { 162 if (!IS_ENABLED(CONFIG_PPC_BOOK3S_64) || 163 !firmware_has_feature(FW_FEATURE_LPAR) || 164 radix_enabled() || (mfmsr() & MSR_DR)) 165 nmi_exit(); 166 167 #ifdef CONFIG_PPC64 168 if (TRAP(regs) != 0x900 && TRAP(regs) != 0xf00 && TRAP(regs) != 0x260) 169 this_cpu_set_ftrace_enabled(state->ftrace_enabled); 170 171 #ifdef CONFIG_PPC_BOOK3S_64 172 /* Check we didn't change the pending interrupt mask. */ 173 WARN_ON_ONCE((state->irq_happened | PACA_IRQ_HARD_DIS) != local_paca->irq_happened); 174 local_paca->irq_happened = state->irq_happened; 175 local_paca->irq_soft_mask = state->irq_soft_mask; 176 #endif 177 #endif 178 } 179 180 /* 181 * Don't use noinstr here like x86, but rather add NOKPROBE_SYMBOL to each 182 * function definition. The reason for this is the noinstr section is placed 183 * after the main text section, i.e., very far away from the interrupt entry 184 * asm. That creates problems with fitting linker stubs when building large 185 * kernels. 186 */ 187 #define interrupt_handler __visible noinline notrace __no_kcsan __no_sanitize_address 188 189 /** 190 * DECLARE_INTERRUPT_HANDLER_RAW - Declare raw interrupt handler function 191 * @func: Function name of the entry point 192 * @returns: Returns a value back to asm caller 193 */ 194 #define DECLARE_INTERRUPT_HANDLER_RAW(func) \ 195 __visible long func(struct pt_regs *regs) 196 197 /** 198 * DEFINE_INTERRUPT_HANDLER_RAW - Define raw interrupt handler function 199 * @func: Function name of the entry point 200 * @returns: Returns a value back to asm caller 201 * 202 * @func is called from ASM entry code. 203 * 204 * This is a plain function which does no tracing, reconciling, etc. 205 * The macro is written so it acts as function definition. Append the 206 * body with a pair of curly brackets. 207 * 208 * raw interrupt handlers must not enable or disable interrupts, or 209 * schedule, tracing and instrumentation (ftrace, lockdep, etc) would 210 * not be advisable either, although may be possible in a pinch, the 211 * trace will look odd at least. 212 * 213 * A raw handler may call one of the other interrupt handler functions 214 * to be converted into that interrupt context without these restrictions. 215 * 216 * On PPC64, _RAW handlers may return with fast_interrupt_return. 217 * 218 * Specific handlers may have additional restrictions. 219 */ 220 #define DEFINE_INTERRUPT_HANDLER_RAW(func) \ 221 static __always_inline long ____##func(struct pt_regs *regs); \ 222 \ 223 interrupt_handler long func(struct pt_regs *regs) \ 224 { \ 225 long ret; \ 226 \ 227 ret = ____##func (regs); \ 228 \ 229 return ret; \ 230 } \ 231 NOKPROBE_SYMBOL(func); \ 232 \ 233 static __always_inline long ____##func(struct pt_regs *regs) 234 235 /** 236 * DECLARE_INTERRUPT_HANDLER - Declare synchronous interrupt handler function 237 * @func: Function name of the entry point 238 */ 239 #define DECLARE_INTERRUPT_HANDLER(func) \ 240 __visible void func(struct pt_regs *regs) 241 242 /** 243 * DEFINE_INTERRUPT_HANDLER - Define synchronous interrupt handler function 244 * @func: Function name of the entry point 245 * 246 * @func is called from ASM entry code. 247 * 248 * The macro is written so it acts as function definition. Append the 249 * body with a pair of curly brackets. 250 */ 251 #define DEFINE_INTERRUPT_HANDLER(func) \ 252 static __always_inline void ____##func(struct pt_regs *regs); \ 253 \ 254 interrupt_handler void func(struct pt_regs *regs) \ 255 { \ 256 struct interrupt_state state; \ 257 \ 258 interrupt_enter_prepare(regs, &state); \ 259 \ 260 ____##func (regs); \ 261 \ 262 interrupt_exit_prepare(regs, &state); \ 263 } \ 264 NOKPROBE_SYMBOL(func); \ 265 \ 266 static __always_inline void ____##func(struct pt_regs *regs) 267 268 /** 269 * DECLARE_INTERRUPT_HANDLER_RET - Declare synchronous interrupt handler function 270 * @func: Function name of the entry point 271 * @returns: Returns a value back to asm caller 272 */ 273 #define DECLARE_INTERRUPT_HANDLER_RET(func) \ 274 __visible long func(struct pt_regs *regs) 275 276 /** 277 * DEFINE_INTERRUPT_HANDLER_RET - Define synchronous interrupt handler function 278 * @func: Function name of the entry point 279 * @returns: Returns a value back to asm caller 280 * 281 * @func is called from ASM entry code. 282 * 283 * The macro is written so it acts as function definition. Append the 284 * body with a pair of curly brackets. 285 */ 286 #define DEFINE_INTERRUPT_HANDLER_RET(func) \ 287 static __always_inline long ____##func(struct pt_regs *regs); \ 288 \ 289 interrupt_handler long func(struct pt_regs *regs) \ 290 { \ 291 struct interrupt_state state; \ 292 long ret; \ 293 \ 294 interrupt_enter_prepare(regs, &state); \ 295 \ 296 ret = ____##func (regs); \ 297 \ 298 interrupt_exit_prepare(regs, &state); \ 299 \ 300 return ret; \ 301 } \ 302 NOKPROBE_SYMBOL(func); \ 303 \ 304 static __always_inline long ____##func(struct pt_regs *regs) 305 306 /** 307 * DECLARE_INTERRUPT_HANDLER_ASYNC - Declare asynchronous interrupt handler function 308 * @func: Function name of the entry point 309 */ 310 #define DECLARE_INTERRUPT_HANDLER_ASYNC(func) \ 311 __visible void func(struct pt_regs *regs) 312 313 /** 314 * DEFINE_INTERRUPT_HANDLER_ASYNC - Define asynchronous interrupt handler function 315 * @func: Function name of the entry point 316 * 317 * @func is called from ASM entry code. 318 * 319 * The macro is written so it acts as function definition. Append the 320 * body with a pair of curly brackets. 321 */ 322 #define DEFINE_INTERRUPT_HANDLER_ASYNC(func) \ 323 static __always_inline void ____##func(struct pt_regs *regs); \ 324 \ 325 interrupt_handler void func(struct pt_regs *regs) \ 326 { \ 327 struct interrupt_state state; \ 328 \ 329 interrupt_async_enter_prepare(regs, &state); \ 330 \ 331 ____##func (regs); \ 332 \ 333 interrupt_async_exit_prepare(regs, &state); \ 334 } \ 335 NOKPROBE_SYMBOL(func); \ 336 \ 337 static __always_inline void ____##func(struct pt_regs *regs) 338 339 /** 340 * DECLARE_INTERRUPT_HANDLER_NMI - Declare NMI interrupt handler function 341 * @func: Function name of the entry point 342 * @returns: Returns a value back to asm caller 343 */ 344 #define DECLARE_INTERRUPT_HANDLER_NMI(func) \ 345 __visible long func(struct pt_regs *regs) 346 347 /** 348 * DEFINE_INTERRUPT_HANDLER_NMI - Define NMI interrupt handler function 349 * @func: Function name of the entry point 350 * @returns: Returns a value back to asm caller 351 * 352 * @func is called from ASM entry code. 353 * 354 * The macro is written so it acts as function definition. Append the 355 * body with a pair of curly brackets. 356 */ 357 #define DEFINE_INTERRUPT_HANDLER_NMI(func) \ 358 static __always_inline long ____##func(struct pt_regs *regs); \ 359 \ 360 interrupt_handler long func(struct pt_regs *regs) \ 361 { \ 362 struct interrupt_nmi_state state; \ 363 long ret; \ 364 \ 365 interrupt_nmi_enter_prepare(regs, &state); \ 366 \ 367 ret = ____##func (regs); \ 368 \ 369 interrupt_nmi_exit_prepare(regs, &state); \ 370 \ 371 return ret; \ 372 } \ 373 NOKPROBE_SYMBOL(func); \ 374 \ 375 static __always_inline long ____##func(struct pt_regs *regs) 376 377 378 /* Interrupt handlers */ 379 /* kernel/traps.c */ 380 DECLARE_INTERRUPT_HANDLER_NMI(system_reset_exception); 381 #ifdef CONFIG_PPC_BOOK3S_64 382 DECLARE_INTERRUPT_HANDLER_ASYNC(machine_check_exception); 383 #else 384 DECLARE_INTERRUPT_HANDLER_NMI(machine_check_exception); 385 #endif 386 DECLARE_INTERRUPT_HANDLER(SMIException); 387 DECLARE_INTERRUPT_HANDLER(handle_hmi_exception); 388 DECLARE_INTERRUPT_HANDLER(unknown_exception); 389 DECLARE_INTERRUPT_HANDLER_ASYNC(unknown_async_exception); 390 DECLARE_INTERRUPT_HANDLER(instruction_breakpoint_exception); 391 DECLARE_INTERRUPT_HANDLER(RunModeException); 392 DECLARE_INTERRUPT_HANDLER(single_step_exception); 393 DECLARE_INTERRUPT_HANDLER(program_check_exception); 394 DECLARE_INTERRUPT_HANDLER(emulation_assist_interrupt); 395 DECLARE_INTERRUPT_HANDLER(alignment_exception); 396 DECLARE_INTERRUPT_HANDLER(StackOverflow); 397 DECLARE_INTERRUPT_HANDLER(stack_overflow_exception); 398 DECLARE_INTERRUPT_HANDLER(kernel_fp_unavailable_exception); 399 DECLARE_INTERRUPT_HANDLER(altivec_unavailable_exception); 400 DECLARE_INTERRUPT_HANDLER(vsx_unavailable_exception); 401 DECLARE_INTERRUPT_HANDLER(facility_unavailable_exception); 402 DECLARE_INTERRUPT_HANDLER(fp_unavailable_tm); 403 DECLARE_INTERRUPT_HANDLER(altivec_unavailable_tm); 404 DECLARE_INTERRUPT_HANDLER(vsx_unavailable_tm); 405 DECLARE_INTERRUPT_HANDLER_NMI(performance_monitor_exception_nmi); 406 DECLARE_INTERRUPT_HANDLER_ASYNC(performance_monitor_exception_async); 407 DECLARE_INTERRUPT_HANDLER_RAW(performance_monitor_exception); 408 DECLARE_INTERRUPT_HANDLER(DebugException); 409 DECLARE_INTERRUPT_HANDLER(altivec_assist_exception); 410 DECLARE_INTERRUPT_HANDLER(CacheLockingException); 411 DECLARE_INTERRUPT_HANDLER(SPEFloatingPointException); 412 DECLARE_INTERRUPT_HANDLER(SPEFloatingPointRoundException); 413 DECLARE_INTERRUPT_HANDLER(WatchdogException); 414 DECLARE_INTERRUPT_HANDLER(kernel_bad_stack); 415 416 /* slb.c */ 417 DECLARE_INTERRUPT_HANDLER_RAW(do_slb_fault); 418 DECLARE_INTERRUPT_HANDLER(do_bad_slb_fault); 419 420 /* hash_utils.c */ 421 DECLARE_INTERRUPT_HANDLER_RAW(do_hash_fault); 422 423 /* fault.c */ 424 DECLARE_INTERRUPT_HANDLER_RET(do_page_fault); 425 DECLARE_INTERRUPT_HANDLER(do_bad_page_fault_segv); 426 427 /* process.c */ 428 DECLARE_INTERRUPT_HANDLER(do_break); 429 430 /* time.c */ 431 DECLARE_INTERRUPT_HANDLER_ASYNC(timer_interrupt); 432 433 /* mce.c */ 434 DECLARE_INTERRUPT_HANDLER_NMI(machine_check_early); 435 DECLARE_INTERRUPT_HANDLER_NMI(hmi_exception_realmode); 436 437 DECLARE_INTERRUPT_HANDLER_ASYNC(TAUException); 438 439 void unrecoverable_exception(struct pt_regs *regs); 440 441 void replay_system_reset(void); 442 void replay_soft_interrupts(void); 443 444 static inline void interrupt_cond_local_irq_enable(struct pt_regs *regs) 445 { 446 if (!arch_irq_disabled_regs(regs)) 447 local_irq_enable(); 448 } 449 450 #endif /* _ASM_POWERPC_INTERRUPT_H */ 451