1142781e1SThomas Gleixner // SPDX-License-Identifier: GPL-2.0 2142781e1SThomas Gleixner 3142781e1SThomas Gleixner #include <linux/context_tracking.h> 4142781e1SThomas Gleixner #include <linux/entry-common.h> 5a9f3a74aSThomas Gleixner #include <linux/livepatch.h> 6a9f3a74aSThomas Gleixner #include <linux/audit.h> 7142781e1SThomas Gleixner 8142781e1SThomas Gleixner #define CREATE_TRACE_POINTS 9142781e1SThomas Gleixner #include <trace/events/syscalls.h> 10142781e1SThomas Gleixner 11142781e1SThomas Gleixner /** 12142781e1SThomas Gleixner * enter_from_user_mode - Establish state when coming from user mode 13142781e1SThomas Gleixner * 14142781e1SThomas Gleixner * Syscall/interrupt entry disables interrupts, but user mode is traced as 15142781e1SThomas Gleixner * interrupts enabled. Also with NO_HZ_FULL RCU might be idle. 16142781e1SThomas Gleixner * 17142781e1SThomas Gleixner * 1) Tell lockdep that interrupts are disabled 18142781e1SThomas Gleixner * 2) Invoke context tracking if enabled to reactivate RCU 19142781e1SThomas Gleixner * 3) Trace interrupts off state 20142781e1SThomas Gleixner */ 21142781e1SThomas Gleixner static __always_inline void enter_from_user_mode(struct pt_regs *regs) 22142781e1SThomas Gleixner { 23142781e1SThomas Gleixner arch_check_user_regs(regs); 24142781e1SThomas Gleixner lockdep_hardirqs_off(CALLER_ADDR0); 25142781e1SThomas Gleixner 26142781e1SThomas Gleixner CT_WARN_ON(ct_state() != CONTEXT_USER); 27142781e1SThomas Gleixner user_exit_irqoff(); 28142781e1SThomas Gleixner 29142781e1SThomas Gleixner instrumentation_begin(); 30142781e1SThomas Gleixner trace_hardirqs_off_finish(); 31142781e1SThomas Gleixner instrumentation_end(); 32142781e1SThomas Gleixner } 33142781e1SThomas Gleixner 34142781e1SThomas Gleixner static inline void syscall_enter_audit(struct pt_regs *regs, long syscall) 35142781e1SThomas Gleixner { 36142781e1SThomas Gleixner if (unlikely(audit_context())) { 37142781e1SThomas Gleixner unsigned long args[6]; 38142781e1SThomas Gleixner 39142781e1SThomas Gleixner syscall_get_arguments(current, regs, args); 40142781e1SThomas Gleixner audit_syscall_entry(syscall, args[0], args[1], args[2], args[3]); 41142781e1SThomas Gleixner } 42142781e1SThomas Gleixner } 43142781e1SThomas Gleixner 44142781e1SThomas Gleixner static long syscall_trace_enter(struct pt_regs *regs, long syscall, 45142781e1SThomas Gleixner unsigned long ti_work) 46142781e1SThomas Gleixner { 47142781e1SThomas Gleixner long ret = 0; 48142781e1SThomas Gleixner 49142781e1SThomas Gleixner /* Handle ptrace */ 50142781e1SThomas Gleixner if (ti_work & (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_EMU)) { 51142781e1SThomas Gleixner ret = arch_syscall_enter_tracehook(regs); 52142781e1SThomas Gleixner if (ret || (ti_work & _TIF_SYSCALL_EMU)) 53142781e1SThomas Gleixner return -1L; 54142781e1SThomas Gleixner } 55142781e1SThomas Gleixner 56142781e1SThomas Gleixner /* Do seccomp after ptrace, to catch any tracer changes. */ 57142781e1SThomas Gleixner if (ti_work & _TIF_SECCOMP) { 58142781e1SThomas Gleixner ret = __secure_computing(NULL); 59142781e1SThomas Gleixner if (ret == -1L) 60142781e1SThomas Gleixner return ret; 61142781e1SThomas Gleixner } 62142781e1SThomas Gleixner 63142781e1SThomas Gleixner if (unlikely(ti_work & _TIF_SYSCALL_TRACEPOINT)) 64142781e1SThomas Gleixner trace_sys_enter(regs, syscall); 65142781e1SThomas Gleixner 66142781e1SThomas Gleixner syscall_enter_audit(regs, syscall); 67142781e1SThomas Gleixner 68d88d59b6SThomas Gleixner /* The above might have changed the syscall number */ 69d88d59b6SThomas Gleixner return ret ? : syscall_get_nr(current, regs); 70142781e1SThomas Gleixner } 71142781e1SThomas Gleixner 72142781e1SThomas Gleixner noinstr long syscall_enter_from_user_mode(struct pt_regs *regs, long syscall) 73142781e1SThomas Gleixner { 74142781e1SThomas Gleixner unsigned long ti_work; 75142781e1SThomas Gleixner 76142781e1SThomas Gleixner enter_from_user_mode(regs); 77142781e1SThomas Gleixner instrumentation_begin(); 78142781e1SThomas Gleixner 79142781e1SThomas Gleixner local_irq_enable(); 80142781e1SThomas Gleixner ti_work = READ_ONCE(current_thread_info()->flags); 81142781e1SThomas Gleixner if (ti_work & SYSCALL_ENTER_WORK) 82142781e1SThomas Gleixner syscall = syscall_trace_enter(regs, syscall, ti_work); 83142781e1SThomas Gleixner instrumentation_end(); 84142781e1SThomas Gleixner 85142781e1SThomas Gleixner return syscall; 86142781e1SThomas Gleixner } 87142781e1SThomas Gleixner 88a9f3a74aSThomas Gleixner /** 89a9f3a74aSThomas Gleixner * exit_to_user_mode - Fixup state when exiting to user mode 90a9f3a74aSThomas Gleixner * 91a9f3a74aSThomas Gleixner * Syscall/interupt exit enables interrupts, but the kernel state is 92a9f3a74aSThomas Gleixner * interrupts disabled when this is invoked. Also tell RCU about it. 93a9f3a74aSThomas Gleixner * 94a9f3a74aSThomas Gleixner * 1) Trace interrupts on state 95a9f3a74aSThomas Gleixner * 2) Invoke context tracking if enabled to adjust RCU state 96a9f3a74aSThomas Gleixner * 3) Invoke architecture specific last minute exit code, e.g. speculation 97a9f3a74aSThomas Gleixner * mitigations, etc. 98a9f3a74aSThomas Gleixner * 4) Tell lockdep that interrupts are enabled 99a9f3a74aSThomas Gleixner */ 100a9f3a74aSThomas Gleixner static __always_inline void exit_to_user_mode(void) 101a9f3a74aSThomas Gleixner { 102a9f3a74aSThomas Gleixner instrumentation_begin(); 103a9f3a74aSThomas Gleixner trace_hardirqs_on_prepare(); 104a9f3a74aSThomas Gleixner lockdep_hardirqs_on_prepare(CALLER_ADDR0); 105a9f3a74aSThomas Gleixner instrumentation_end(); 106a9f3a74aSThomas Gleixner 107a9f3a74aSThomas Gleixner user_enter_irqoff(); 108a9f3a74aSThomas Gleixner arch_exit_to_user_mode(); 109a9f3a74aSThomas Gleixner lockdep_hardirqs_on(CALLER_ADDR0); 110a9f3a74aSThomas Gleixner } 111a9f3a74aSThomas Gleixner 112a9f3a74aSThomas Gleixner /* Workaround to allow gradual conversion of architecture code */ 113a9f3a74aSThomas Gleixner void __weak arch_do_signal(struct pt_regs *regs) { } 114a9f3a74aSThomas Gleixner 115a9f3a74aSThomas Gleixner static unsigned long exit_to_user_mode_loop(struct pt_regs *regs, 116a9f3a74aSThomas Gleixner unsigned long ti_work) 117a9f3a74aSThomas Gleixner { 118a9f3a74aSThomas Gleixner /* 119a9f3a74aSThomas Gleixner * Before returning to user space ensure that all pending work 120a9f3a74aSThomas Gleixner * items have been completed. 121a9f3a74aSThomas Gleixner */ 122a9f3a74aSThomas Gleixner while (ti_work & EXIT_TO_USER_MODE_WORK) { 123a9f3a74aSThomas Gleixner 124a9f3a74aSThomas Gleixner local_irq_enable_exit_to_user(ti_work); 125a9f3a74aSThomas Gleixner 126a9f3a74aSThomas Gleixner if (ti_work & _TIF_NEED_RESCHED) 127a9f3a74aSThomas Gleixner schedule(); 128a9f3a74aSThomas Gleixner 129a9f3a74aSThomas Gleixner if (ti_work & _TIF_UPROBE) 130a9f3a74aSThomas Gleixner uprobe_notify_resume(regs); 131a9f3a74aSThomas Gleixner 132a9f3a74aSThomas Gleixner if (ti_work & _TIF_PATCH_PENDING) 133a9f3a74aSThomas Gleixner klp_update_patch_state(current); 134a9f3a74aSThomas Gleixner 135a9f3a74aSThomas Gleixner if (ti_work & _TIF_SIGPENDING) 136a9f3a74aSThomas Gleixner arch_do_signal(regs); 137a9f3a74aSThomas Gleixner 138a9f3a74aSThomas Gleixner if (ti_work & _TIF_NOTIFY_RESUME) { 139a9f3a74aSThomas Gleixner clear_thread_flag(TIF_NOTIFY_RESUME); 140a9f3a74aSThomas Gleixner tracehook_notify_resume(regs); 141a9f3a74aSThomas Gleixner rseq_handle_notify_resume(NULL, regs); 142a9f3a74aSThomas Gleixner } 143a9f3a74aSThomas Gleixner 144a9f3a74aSThomas Gleixner /* Architecture specific TIF work */ 145a9f3a74aSThomas Gleixner arch_exit_to_user_mode_work(regs, ti_work); 146a9f3a74aSThomas Gleixner 147a9f3a74aSThomas Gleixner /* 148a9f3a74aSThomas Gleixner * Disable interrupts and reevaluate the work flags as they 149a9f3a74aSThomas Gleixner * might have changed while interrupts and preemption was 150a9f3a74aSThomas Gleixner * enabled above. 151a9f3a74aSThomas Gleixner */ 152a9f3a74aSThomas Gleixner local_irq_disable_exit_to_user(); 153a9f3a74aSThomas Gleixner ti_work = READ_ONCE(current_thread_info()->flags); 154a9f3a74aSThomas Gleixner } 155a9f3a74aSThomas Gleixner 156a9f3a74aSThomas Gleixner /* Return the latest work state for arch_exit_to_user_mode() */ 157a9f3a74aSThomas Gleixner return ti_work; 158a9f3a74aSThomas Gleixner } 159a9f3a74aSThomas Gleixner 160a9f3a74aSThomas Gleixner static void exit_to_user_mode_prepare(struct pt_regs *regs) 161a9f3a74aSThomas Gleixner { 162a9f3a74aSThomas Gleixner unsigned long ti_work = READ_ONCE(current_thread_info()->flags); 163a9f3a74aSThomas Gleixner 164a9f3a74aSThomas Gleixner lockdep_assert_irqs_disabled(); 165a9f3a74aSThomas Gleixner 166a9f3a74aSThomas Gleixner if (unlikely(ti_work & EXIT_TO_USER_MODE_WORK)) 167a9f3a74aSThomas Gleixner ti_work = exit_to_user_mode_loop(regs, ti_work); 168a9f3a74aSThomas Gleixner 169a9f3a74aSThomas Gleixner arch_exit_to_user_mode_prepare(regs, ti_work); 170a9f3a74aSThomas Gleixner 171a9f3a74aSThomas Gleixner /* Ensure that the address limit is intact and no locks are held */ 172a9f3a74aSThomas Gleixner addr_limit_user_check(); 173a9f3a74aSThomas Gleixner lockdep_assert_irqs_disabled(); 174a9f3a74aSThomas Gleixner lockdep_sys_exit(); 175a9f3a74aSThomas Gleixner } 176a9f3a74aSThomas Gleixner 177a9f3a74aSThomas Gleixner #ifndef _TIF_SINGLESTEP 178a9f3a74aSThomas Gleixner static inline bool report_single_step(unsigned long ti_work) 179a9f3a74aSThomas Gleixner { 180a9f3a74aSThomas Gleixner return false; 181a9f3a74aSThomas Gleixner } 182a9f3a74aSThomas Gleixner #else 183a9f3a74aSThomas Gleixner /* 184a9f3a74aSThomas Gleixner * If TIF_SYSCALL_EMU is set, then the only reason to report is when 185a9f3a74aSThomas Gleixner * TIF_SINGLESTEP is set (i.e. PTRACE_SYSEMU_SINGLESTEP). This syscall 186900ffe39SKees Cook * instruction has been already reported in syscall_enter_from_user_mode(). 187a9f3a74aSThomas Gleixner */ 188a9f3a74aSThomas Gleixner #define SYSEMU_STEP (_TIF_SINGLESTEP | _TIF_SYSCALL_EMU) 189a9f3a74aSThomas Gleixner 190a9f3a74aSThomas Gleixner static inline bool report_single_step(unsigned long ti_work) 191a9f3a74aSThomas Gleixner { 192a9f3a74aSThomas Gleixner return (ti_work & SYSEMU_STEP) == _TIF_SINGLESTEP; 193a9f3a74aSThomas Gleixner } 194a9f3a74aSThomas Gleixner #endif 195a9f3a74aSThomas Gleixner 196a9f3a74aSThomas Gleixner static void syscall_exit_work(struct pt_regs *regs, unsigned long ti_work) 197a9f3a74aSThomas Gleixner { 198a9f3a74aSThomas Gleixner bool step; 199a9f3a74aSThomas Gleixner 200a9f3a74aSThomas Gleixner audit_syscall_exit(regs); 201a9f3a74aSThomas Gleixner 202a9f3a74aSThomas Gleixner if (ti_work & _TIF_SYSCALL_TRACEPOINT) 203a9f3a74aSThomas Gleixner trace_sys_exit(regs, syscall_get_return_value(current, regs)); 204a9f3a74aSThomas Gleixner 205a9f3a74aSThomas Gleixner step = report_single_step(ti_work); 206a9f3a74aSThomas Gleixner if (step || ti_work & _TIF_SYSCALL_TRACE) 207a9f3a74aSThomas Gleixner arch_syscall_exit_tracehook(regs, step); 208a9f3a74aSThomas Gleixner } 209a9f3a74aSThomas Gleixner 210a9f3a74aSThomas Gleixner /* 211a9f3a74aSThomas Gleixner * Syscall specific exit to user mode preparation. Runs with interrupts 212a9f3a74aSThomas Gleixner * enabled. 213a9f3a74aSThomas Gleixner */ 214a9f3a74aSThomas Gleixner static void syscall_exit_to_user_mode_prepare(struct pt_regs *regs) 215a9f3a74aSThomas Gleixner { 216a9f3a74aSThomas Gleixner u32 cached_flags = READ_ONCE(current_thread_info()->flags); 217a9f3a74aSThomas Gleixner unsigned long nr = syscall_get_nr(current, regs); 218a9f3a74aSThomas Gleixner 219a9f3a74aSThomas Gleixner CT_WARN_ON(ct_state() != CONTEXT_KERNEL); 220a9f3a74aSThomas Gleixner 221a9f3a74aSThomas Gleixner if (IS_ENABLED(CONFIG_PROVE_LOCKING)) { 222a9f3a74aSThomas Gleixner if (WARN(irqs_disabled(), "syscall %lu left IRQs disabled", nr)) 223a9f3a74aSThomas Gleixner local_irq_enable(); 224a9f3a74aSThomas Gleixner } 225a9f3a74aSThomas Gleixner 226a9f3a74aSThomas Gleixner rseq_syscall(regs); 227a9f3a74aSThomas Gleixner 228a9f3a74aSThomas Gleixner /* 229a9f3a74aSThomas Gleixner * Do one-time syscall specific work. If these work items are 230a9f3a74aSThomas Gleixner * enabled, we want to run them exactly once per syscall exit with 231a9f3a74aSThomas Gleixner * interrupts enabled. 232a9f3a74aSThomas Gleixner */ 233a9f3a74aSThomas Gleixner if (unlikely(cached_flags & SYSCALL_EXIT_WORK)) 234a9f3a74aSThomas Gleixner syscall_exit_work(regs, cached_flags); 235a9f3a74aSThomas Gleixner } 236a9f3a74aSThomas Gleixner 237a9f3a74aSThomas Gleixner __visible noinstr void syscall_exit_to_user_mode(struct pt_regs *regs) 238a9f3a74aSThomas Gleixner { 239a9f3a74aSThomas Gleixner instrumentation_begin(); 240a9f3a74aSThomas Gleixner syscall_exit_to_user_mode_prepare(regs); 241a9f3a74aSThomas Gleixner local_irq_disable_exit_to_user(); 242a9f3a74aSThomas Gleixner exit_to_user_mode_prepare(regs); 243a9f3a74aSThomas Gleixner instrumentation_end(); 244a9f3a74aSThomas Gleixner exit_to_user_mode(); 245a9f3a74aSThomas Gleixner } 246a9f3a74aSThomas Gleixner 247142781e1SThomas Gleixner noinstr void irqentry_enter_from_user_mode(struct pt_regs *regs) 248142781e1SThomas Gleixner { 249142781e1SThomas Gleixner enter_from_user_mode(regs); 250142781e1SThomas Gleixner } 251a9f3a74aSThomas Gleixner 252a9f3a74aSThomas Gleixner noinstr void irqentry_exit_to_user_mode(struct pt_regs *regs) 253a9f3a74aSThomas Gleixner { 254a9f3a74aSThomas Gleixner instrumentation_begin(); 255a9f3a74aSThomas Gleixner exit_to_user_mode_prepare(regs); 256a9f3a74aSThomas Gleixner instrumentation_end(); 257a9f3a74aSThomas Gleixner exit_to_user_mode(); 258a9f3a74aSThomas Gleixner } 259a5497babSThomas Gleixner 260aadfc2f9SIngo Molnar noinstr irqentry_state_t irqentry_enter(struct pt_regs *regs) 261a5497babSThomas Gleixner { 262a5497babSThomas Gleixner irqentry_state_t ret = { 263a5497babSThomas Gleixner .exit_rcu = false, 264a5497babSThomas Gleixner }; 265a5497babSThomas Gleixner 266a5497babSThomas Gleixner if (user_mode(regs)) { 267a5497babSThomas Gleixner irqentry_enter_from_user_mode(regs); 268a5497babSThomas Gleixner return ret; 269a5497babSThomas Gleixner } 270a5497babSThomas Gleixner 271a5497babSThomas Gleixner /* 272a5497babSThomas Gleixner * If this entry hit the idle task invoke rcu_irq_enter() whether 273a5497babSThomas Gleixner * RCU is watching or not. 274a5497babSThomas Gleixner * 275a5497babSThomas Gleixner * Interupts can nest when the first interrupt invokes softirq 276a5497babSThomas Gleixner * processing on return which enables interrupts. 277a5497babSThomas Gleixner * 278a5497babSThomas Gleixner * Scheduler ticks in the idle task can mark quiescent state and 279a5497babSThomas Gleixner * terminate a grace period, if and only if the timer interrupt is 280a5497babSThomas Gleixner * not nested into another interrupt. 281a5497babSThomas Gleixner * 282a5497babSThomas Gleixner * Checking for __rcu_is_watching() here would prevent the nesting 283a5497babSThomas Gleixner * interrupt to invoke rcu_irq_enter(). If that nested interrupt is 284a5497babSThomas Gleixner * the tick then rcu_flavor_sched_clock_irq() would wrongfully 285a5497babSThomas Gleixner * assume that it is the first interupt and eventually claim 286a5497babSThomas Gleixner * quiescient state and end grace periods prematurely. 287a5497babSThomas Gleixner * 288a5497babSThomas Gleixner * Unconditionally invoke rcu_irq_enter() so RCU state stays 289a5497babSThomas Gleixner * consistent. 290a5497babSThomas Gleixner * 291a5497babSThomas Gleixner * TINY_RCU does not support EQS, so let the compiler eliminate 292a5497babSThomas Gleixner * this part when enabled. 293a5497babSThomas Gleixner */ 294a5497babSThomas Gleixner if (!IS_ENABLED(CONFIG_TINY_RCU) && is_idle_task(current)) { 295a5497babSThomas Gleixner /* 296a5497babSThomas Gleixner * If RCU is not watching then the same careful 297a5497babSThomas Gleixner * sequence vs. lockdep and tracing is required 298a5497babSThomas Gleixner * as in irq_enter_from_user_mode(). 299a5497babSThomas Gleixner */ 300a5497babSThomas Gleixner lockdep_hardirqs_off(CALLER_ADDR0); 301a5497babSThomas Gleixner rcu_irq_enter(); 302a5497babSThomas Gleixner instrumentation_begin(); 303a5497babSThomas Gleixner trace_hardirqs_off_finish(); 304a5497babSThomas Gleixner instrumentation_end(); 305a5497babSThomas Gleixner 306a5497babSThomas Gleixner ret.exit_rcu = true; 307a5497babSThomas Gleixner return ret; 308a5497babSThomas Gleixner } 309a5497babSThomas Gleixner 310a5497babSThomas Gleixner /* 311a5497babSThomas Gleixner * If RCU is watching then RCU only wants to check whether it needs 312a5497babSThomas Gleixner * to restart the tick in NOHZ mode. rcu_irq_enter_check_tick() 313a5497babSThomas Gleixner * already contains a warning when RCU is not watching, so no point 314a5497babSThomas Gleixner * in having another one here. 315a5497babSThomas Gleixner */ 316a5497babSThomas Gleixner instrumentation_begin(); 317a5497babSThomas Gleixner rcu_irq_enter_check_tick(); 318a5497babSThomas Gleixner /* Use the combo lockdep/tracing function */ 319a5497babSThomas Gleixner trace_hardirqs_off(); 320a5497babSThomas Gleixner instrumentation_end(); 321a5497babSThomas Gleixner 322a5497babSThomas Gleixner return ret; 323a5497babSThomas Gleixner } 324a5497babSThomas Gleixner 325a5497babSThomas Gleixner void irqentry_exit_cond_resched(void) 326a5497babSThomas Gleixner { 327a5497babSThomas Gleixner if (!preempt_count()) { 328a5497babSThomas Gleixner /* Sanity check RCU and thread stack */ 329a5497babSThomas Gleixner rcu_irq_exit_check_preempt(); 330a5497babSThomas Gleixner if (IS_ENABLED(CONFIG_DEBUG_ENTRY)) 331a5497babSThomas Gleixner WARN_ON_ONCE(!on_thread_stack()); 332a5497babSThomas Gleixner if (need_resched()) 333a5497babSThomas Gleixner preempt_schedule_irq(); 334a5497babSThomas Gleixner } 335a5497babSThomas Gleixner } 336a5497babSThomas Gleixner 337aadfc2f9SIngo Molnar noinstr void irqentry_exit(struct pt_regs *regs, irqentry_state_t state) 338a5497babSThomas Gleixner { 339a5497babSThomas Gleixner lockdep_assert_irqs_disabled(); 340a5497babSThomas Gleixner 341a5497babSThomas Gleixner /* Check whether this returns to user mode */ 342a5497babSThomas Gleixner if (user_mode(regs)) { 343a5497babSThomas Gleixner irqentry_exit_to_user_mode(regs); 344a5497babSThomas Gleixner } else if (!regs_irqs_disabled(regs)) { 345a5497babSThomas Gleixner /* 346a5497babSThomas Gleixner * If RCU was not watching on entry this needs to be done 347a5497babSThomas Gleixner * carefully and needs the same ordering of lockdep/tracing 348a5497babSThomas Gleixner * and RCU as the return to user mode path. 349a5497babSThomas Gleixner */ 350a5497babSThomas Gleixner if (state.exit_rcu) { 351a5497babSThomas Gleixner instrumentation_begin(); 352a5497babSThomas Gleixner /* Tell the tracer that IRET will enable interrupts */ 353a5497babSThomas Gleixner trace_hardirqs_on_prepare(); 354a5497babSThomas Gleixner lockdep_hardirqs_on_prepare(CALLER_ADDR0); 355a5497babSThomas Gleixner instrumentation_end(); 356a5497babSThomas Gleixner rcu_irq_exit(); 357a5497babSThomas Gleixner lockdep_hardirqs_on(CALLER_ADDR0); 358a5497babSThomas Gleixner return; 359a5497babSThomas Gleixner } 360a5497babSThomas Gleixner 361a5497babSThomas Gleixner instrumentation_begin(); 362a5497babSThomas Gleixner if (IS_ENABLED(CONFIG_PREEMPTION)) 363a5497babSThomas Gleixner irqentry_exit_cond_resched(); 364a5497babSThomas Gleixner /* Covers both tracing and lockdep */ 365a5497babSThomas Gleixner trace_hardirqs_on(); 366a5497babSThomas Gleixner instrumentation_end(); 367a5497babSThomas Gleixner } else { 368a5497babSThomas Gleixner /* 369a5497babSThomas Gleixner * IRQ flags state is correct already. Just tell RCU if it 370a5497babSThomas Gleixner * was not watching on entry. 371a5497babSThomas Gleixner */ 372a5497babSThomas Gleixner if (state.exit_rcu) 373a5497babSThomas Gleixner rcu_irq_exit(); 374a5497babSThomas Gleixner } 375a5497babSThomas Gleixner } 376