1457c8996SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 2e360adbeSPeter Zijlstra /* 390eec103SPeter Zijlstra * Copyright (C) 2010 Red Hat, Inc., Peter Zijlstra 4e360adbeSPeter Zijlstra * 5e360adbeSPeter Zijlstra * Provides a framework for enqueueing and running callbacks from hardirq 6e360adbeSPeter Zijlstra * context. The enqueueing is NMI-safe. 7e360adbeSPeter Zijlstra */ 8e360adbeSPeter Zijlstra 983e3fa6fSPaul Gortmaker #include <linux/bug.h> 10e360adbeSPeter Zijlstra #include <linux/kernel.h> 119984de1aSPaul Gortmaker #include <linux/export.h> 12e360adbeSPeter Zijlstra #include <linux/irq_work.h> 13967d1f90SPaul Gortmaker #include <linux/percpu.h> 14e360adbeSPeter Zijlstra #include <linux/hardirq.h> 15ef1f0982SChris Metcalf #include <linux/irqflags.h> 16bc6679aeSFrederic Weisbecker #include <linux/sched.h> 17bc6679aeSFrederic Weisbecker #include <linux/tick.h> 18c0e980a4SSteven Rostedt #include <linux/cpu.h> 19c0e980a4SSteven Rostedt #include <linux/notifier.h> 2047885016SFrederic Weisbecker #include <linux/smp.h> 21b4c6f86eSSebastian Andrzej Siewior #include <linux/smpboot.h> 22967d1f90SPaul Gortmaker #include <asm/processor.h> 23e2b5bcf9SZqiang #include <linux/kasan.h> 24e360adbeSPeter Zijlstra 25*4468161aSValentin Schneider #include <trace/events/ipi.h> 26*4468161aSValentin Schneider 27b93e0b8fSFrederic Weisbecker static DEFINE_PER_CPU(struct llist_head, raised_list); 28b93e0b8fSFrederic Weisbecker static DEFINE_PER_CPU(struct llist_head, lazy_list); 29b4c6f86eSSebastian Andrzej Siewior static DEFINE_PER_CPU(struct task_struct *, irq_workd); 30b4c6f86eSSebastian Andrzej Siewior 31b4c6f86eSSebastian Andrzej Siewior static void wake_irq_workd(void) 32b4c6f86eSSebastian Andrzej Siewior { 33b4c6f86eSSebastian Andrzej Siewior struct task_struct *tsk = __this_cpu_read(irq_workd); 34b4c6f86eSSebastian Andrzej Siewior 35b4c6f86eSSebastian Andrzej Siewior if (!llist_empty(this_cpu_ptr(&lazy_list)) && tsk) 36b4c6f86eSSebastian Andrzej Siewior wake_up_process(tsk); 37b4c6f86eSSebastian Andrzej Siewior } 38b4c6f86eSSebastian Andrzej Siewior 39b4c6f86eSSebastian Andrzej Siewior #ifdef CONFIG_SMP 40b4c6f86eSSebastian Andrzej Siewior static void irq_work_wake(struct irq_work *entry) 41b4c6f86eSSebastian Andrzej Siewior { 42b4c6f86eSSebastian Andrzej Siewior wake_irq_workd(); 43b4c6f86eSSebastian Andrzej Siewior } 44b4c6f86eSSebastian Andrzej Siewior 45b4c6f86eSSebastian Andrzej Siewior static DEFINE_PER_CPU(struct irq_work, irq_work_wakeup) = 46b4c6f86eSSebastian Andrzej Siewior IRQ_WORK_INIT_HARD(irq_work_wake); 47b4c6f86eSSebastian Andrzej Siewior #endif 48b4c6f86eSSebastian Andrzej Siewior 49b4c6f86eSSebastian Andrzej Siewior static int irq_workd_should_run(unsigned int cpu) 50b4c6f86eSSebastian Andrzej Siewior { 51b4c6f86eSSebastian Andrzej Siewior return !llist_empty(this_cpu_ptr(&lazy_list)); 52b4c6f86eSSebastian Andrzej Siewior } 53e360adbeSPeter Zijlstra 54e360adbeSPeter Zijlstra /* 55e360adbeSPeter Zijlstra * Claim the entry so that no one else will poke at it. 56e360adbeSPeter Zijlstra */ 5738aaf809SHuang Ying static bool irq_work_claim(struct irq_work *work) 58e360adbeSPeter Zijlstra { 5925269871SFrederic Weisbecker int oflags; 60e360adbeSPeter Zijlstra 617a9f50a0SPeter Zijlstra oflags = atomic_fetch_or(IRQ_WORK_CLAIMED | CSD_TYPE_IRQ_WORK, &work->node.a_flags); 62e0bbe2d8SFrederic Weisbecker /* 6325269871SFrederic Weisbecker * If the work is already pending, no need to raise the IPI. 642914b0baSPeter Zijlstra * The pairing smp_mb() in irq_work_single() makes sure 6525269871SFrederic Weisbecker * everything we did before is visible. 66e0bbe2d8SFrederic Weisbecker */ 67e0bbe2d8SFrederic Weisbecker if (oflags & IRQ_WORK_PENDING) 68e0bbe2d8SFrederic Weisbecker return false; 69e360adbeSPeter Zijlstra return true; 70e360adbeSPeter Zijlstra } 71e360adbeSPeter Zijlstra 72e360adbeSPeter Zijlstra void __weak arch_irq_work_raise(void) 73e360adbeSPeter Zijlstra { 74e360adbeSPeter Zijlstra /* 75e360adbeSPeter Zijlstra * Lame architectures will get the timer tick callback 76e360adbeSPeter Zijlstra */ 77e360adbeSPeter Zijlstra } 78e360adbeSPeter Zijlstra 79*4468161aSValentin Schneider static __always_inline void irq_work_raise(struct irq_work *work) 80*4468161aSValentin Schneider { 81*4468161aSValentin Schneider if (trace_ipi_send_cpumask_enabled() && arch_irq_work_has_interrupt()) 82*4468161aSValentin Schneider trace_ipi_send_cpumask(cpumask_of(smp_processor_id()), 83*4468161aSValentin Schneider _RET_IP_, 84*4468161aSValentin Schneider work->func); 85*4468161aSValentin Schneider 86*4468161aSValentin Schneider arch_irq_work_raise(); 87*4468161aSValentin Schneider } 88*4468161aSValentin Schneider 89471ba0e6SNicholas Piggin /* Enqueue on current CPU, work must already be claimed and preempt disabled */ 90471ba0e6SNicholas Piggin static void __irq_work_queue_local(struct irq_work *work) 9147885016SFrederic Weisbecker { 92b4c6f86eSSebastian Andrzej Siewior struct llist_head *list; 93b4c6f86eSSebastian Andrzej Siewior bool rt_lazy_work = false; 94b4c6f86eSSebastian Andrzej Siewior bool lazy_work = false; 95b4c6f86eSSebastian Andrzej Siewior int work_flags; 96b4c6f86eSSebastian Andrzej Siewior 97b4c6f86eSSebastian Andrzej Siewior work_flags = atomic_read(&work->node.a_flags); 98b4c6f86eSSebastian Andrzej Siewior if (work_flags & IRQ_WORK_LAZY) 99b4c6f86eSSebastian Andrzej Siewior lazy_work = true; 100b4c6f86eSSebastian Andrzej Siewior else if (IS_ENABLED(CONFIG_PREEMPT_RT) && 101b4c6f86eSSebastian Andrzej Siewior !(work_flags & IRQ_WORK_HARD_IRQ)) 102b4c6f86eSSebastian Andrzej Siewior rt_lazy_work = true; 103b4c6f86eSSebastian Andrzej Siewior 104b4c6f86eSSebastian Andrzej Siewior if (lazy_work || rt_lazy_work) 105b4c6f86eSSebastian Andrzej Siewior list = this_cpu_ptr(&lazy_list); 106b4c6f86eSSebastian Andrzej Siewior else 107b4c6f86eSSebastian Andrzej Siewior list = this_cpu_ptr(&raised_list); 108b4c6f86eSSebastian Andrzej Siewior 109b4c6f86eSSebastian Andrzej Siewior if (!llist_add(&work->node.llist, list)) 110b4c6f86eSSebastian Andrzej Siewior return; 111b4c6f86eSSebastian Andrzej Siewior 112471ba0e6SNicholas Piggin /* If the work is "lazy", handle it from next tick if any */ 113b4c6f86eSSebastian Andrzej Siewior if (!lazy_work || tick_nohz_tick_stopped()) 114*4468161aSValentin Schneider irq_work_raise(work); 11547885016SFrederic Weisbecker } 11647885016SFrederic Weisbecker 11747885016SFrederic Weisbecker /* Enqueue the irq work @work on the current CPU */ 118cd578abbSPeter Zijlstra bool irq_work_queue(struct irq_work *work) 119e360adbeSPeter Zijlstra { 120c02cf5f8Sanish kumar /* Only queue if not already pending */ 121c02cf5f8Sanish kumar if (!irq_work_claim(work)) 122cd578abbSPeter Zijlstra return false; 123c02cf5f8Sanish kumar 124c02cf5f8Sanish kumar /* Queue the entry and raise the IPI if needed. */ 12520b87691SChristoph Lameter preempt_disable(); 126471ba0e6SNicholas Piggin __irq_work_queue_local(work); 12720b87691SChristoph Lameter preempt_enable(); 128cd578abbSPeter Zijlstra 129cd578abbSPeter Zijlstra return true; 130e360adbeSPeter Zijlstra } 131e360adbeSPeter Zijlstra EXPORT_SYMBOL_GPL(irq_work_queue); 132e360adbeSPeter Zijlstra 133471ba0e6SNicholas Piggin /* 134471ba0e6SNicholas Piggin * Enqueue the irq_work @work on @cpu unless it's already pending 135471ba0e6SNicholas Piggin * somewhere. 136471ba0e6SNicholas Piggin * 137471ba0e6SNicholas Piggin * Can be re-enqueued while the callback is still in progress. 138471ba0e6SNicholas Piggin */ 139471ba0e6SNicholas Piggin bool irq_work_queue_on(struct irq_work *work, int cpu) 140471ba0e6SNicholas Piggin { 141471ba0e6SNicholas Piggin #ifndef CONFIG_SMP 142471ba0e6SNicholas Piggin return irq_work_queue(work); 143471ba0e6SNicholas Piggin 144471ba0e6SNicholas Piggin #else /* CONFIG_SMP: */ 145471ba0e6SNicholas Piggin /* All work should have been flushed before going offline */ 146471ba0e6SNicholas Piggin WARN_ON_ONCE(cpu_is_offline(cpu)); 147471ba0e6SNicholas Piggin 148471ba0e6SNicholas Piggin /* Only queue if not already pending */ 149471ba0e6SNicholas Piggin if (!irq_work_claim(work)) 150471ba0e6SNicholas Piggin return false; 151471ba0e6SNicholas Piggin 15225934fcfSZqiang kasan_record_aux_stack_noalloc(work); 153e2b5bcf9SZqiang 154471ba0e6SNicholas Piggin preempt_disable(); 155471ba0e6SNicholas Piggin if (cpu != smp_processor_id()) { 156471ba0e6SNicholas Piggin /* Arch remote IPI send/receive backend aren't NMI safe */ 157471ba0e6SNicholas Piggin WARN_ON_ONCE(in_nmi()); 158b4c6f86eSSebastian Andrzej Siewior 159b4c6f86eSSebastian Andrzej Siewior /* 160b4c6f86eSSebastian Andrzej Siewior * On PREEMPT_RT the items which are not marked as 161b4c6f86eSSebastian Andrzej Siewior * IRQ_WORK_HARD_IRQ are added to the lazy list and a HARD work 162b4c6f86eSSebastian Andrzej Siewior * item is used on the remote CPU to wake the thread. 163b4c6f86eSSebastian Andrzej Siewior */ 164b4c6f86eSSebastian Andrzej Siewior if (IS_ENABLED(CONFIG_PREEMPT_RT) && 165b4c6f86eSSebastian Andrzej Siewior !(atomic_read(&work->node.a_flags) & IRQ_WORK_HARD_IRQ)) { 166b4c6f86eSSebastian Andrzej Siewior 167b4c6f86eSSebastian Andrzej Siewior if (!llist_add(&work->node.llist, &per_cpu(lazy_list, cpu))) 168b4c6f86eSSebastian Andrzej Siewior goto out; 169b4c6f86eSSebastian Andrzej Siewior 170b4c6f86eSSebastian Andrzej Siewior work = &per_cpu(irq_work_wakeup, cpu); 171b4c6f86eSSebastian Andrzej Siewior if (!irq_work_claim(work)) 172b4c6f86eSSebastian Andrzej Siewior goto out; 173b4c6f86eSSebastian Andrzej Siewior } 174b4c6f86eSSebastian Andrzej Siewior 1757a9f50a0SPeter Zijlstra __smp_call_single_queue(cpu, &work->node.llist); 176471ba0e6SNicholas Piggin } else { 177471ba0e6SNicholas Piggin __irq_work_queue_local(work); 178471ba0e6SNicholas Piggin } 179b4c6f86eSSebastian Andrzej Siewior out: 180471ba0e6SNicholas Piggin preempt_enable(); 181471ba0e6SNicholas Piggin 182471ba0e6SNicholas Piggin return true; 183471ba0e6SNicholas Piggin #endif /* CONFIG_SMP */ 184471ba0e6SNicholas Piggin } 185471ba0e6SNicholas Piggin 18600b42959SFrederic Weisbecker bool irq_work_needs_cpu(void) 187e360adbeSPeter Zijlstra { 188b93e0b8fSFrederic Weisbecker struct llist_head *raised, *lazy; 18900b42959SFrederic Weisbecker 19022127e93SChristoph Lameter raised = this_cpu_ptr(&raised_list); 19122127e93SChristoph Lameter lazy = this_cpu_ptr(&lazy_list); 19276a33061SFrederic Weisbecker 19376a33061SFrederic Weisbecker if (llist_empty(raised) || arch_irq_work_has_interrupt()) 19476a33061SFrederic Weisbecker if (llist_empty(lazy)) 19500b42959SFrederic Weisbecker return false; 19600b42959SFrederic Weisbecker 1978aa2acceSSteven Rostedt /* All work should have been flushed before going offline */ 1988aa2acceSSteven Rostedt WARN_ON_ONCE(cpu_is_offline(smp_processor_id())); 1998aa2acceSSteven Rostedt 20000b42959SFrederic Weisbecker return true; 20100b42959SFrederic Weisbecker } 20200b42959SFrederic Weisbecker 2034b44a21dSPeter Zijlstra void irq_work_single(void *arg) 204e360adbeSPeter Zijlstra { 2054b44a21dSPeter Zijlstra struct irq_work *work = arg; 206feb4a513SFrederic Weisbecker int flags; 2074b44a21dSPeter Zijlstra 208e360adbeSPeter Zijlstra /* 2092914b0baSPeter Zijlstra * Clear the PENDING bit, after this point the @work can be re-used. 2102914b0baSPeter Zijlstra * The PENDING bit acts as a lock, and we own it, so we can clear it 2112914b0baSPeter Zijlstra * without atomic ops. 212e360adbeSPeter Zijlstra */ 2132914b0baSPeter Zijlstra flags = atomic_read(&work->node.a_flags); 214e9838bd5SFrederic Weisbecker flags &= ~IRQ_WORK_PENDING; 2152914b0baSPeter Zijlstra atomic_set(&work->node.a_flags, flags); 2162914b0baSPeter Zijlstra 2172914b0baSPeter Zijlstra /* 2182914b0baSPeter Zijlstra * See irq_work_claim(). 2192914b0baSPeter Zijlstra */ 2202914b0baSPeter Zijlstra smp_mb(); 2212914b0baSPeter Zijlstra 2222914b0baSPeter Zijlstra lockdep_irq_work_enter(flags); 2232914b0baSPeter Zijlstra work->func(work); 2242914b0baSPeter Zijlstra lockdep_irq_work_exit(flags); 2252914b0baSPeter Zijlstra 2262914b0baSPeter Zijlstra /* 2272914b0baSPeter Zijlstra * Clear the BUSY bit, if set, and return to the free state if no-one 2282914b0baSPeter Zijlstra * else claimed it meanwhile. 2292914b0baSPeter Zijlstra */ 2307a9f50a0SPeter Zijlstra (void)atomic_cmpxchg(&work->node.a_flags, flags, flags & ~IRQ_WORK_BUSY); 23181097968SSebastian Andrzej Siewior 23209089db7SSebastian Andrzej Siewior if ((IS_ENABLED(CONFIG_PREEMPT_RT) && !irq_work_is_hard(work)) || 23309089db7SSebastian Andrzej Siewior !arch_irq_work_has_interrupt()) 23481097968SSebastian Andrzej Siewior rcuwait_wake_up(&work->irqwait); 235e360adbeSPeter Zijlstra } 2364b44a21dSPeter Zijlstra 2374b44a21dSPeter Zijlstra static void irq_work_run_list(struct llist_head *list) 2384b44a21dSPeter Zijlstra { 2394b44a21dSPeter Zijlstra struct irq_work *work, *tmp; 2404b44a21dSPeter Zijlstra struct llist_node *llnode; 2414b44a21dSPeter Zijlstra 242b4c6f86eSSebastian Andrzej Siewior /* 243b4c6f86eSSebastian Andrzej Siewior * On PREEMPT_RT IRQ-work which is not marked as HARD will be processed 244b4c6f86eSSebastian Andrzej Siewior * in a per-CPU thread in preemptible context. Only the items which are 245b4c6f86eSSebastian Andrzej Siewior * marked as IRQ_WORK_HARD_IRQ will be processed in hardirq context. 246b4c6f86eSSebastian Andrzej Siewior */ 247b4c6f86eSSebastian Andrzej Siewior BUG_ON(!irqs_disabled() && !IS_ENABLED(CONFIG_PREEMPT_RT)); 2484b44a21dSPeter Zijlstra 2494b44a21dSPeter Zijlstra if (llist_empty(list)) 2504b44a21dSPeter Zijlstra return; 2514b44a21dSPeter Zijlstra 2524b44a21dSPeter Zijlstra llnode = llist_del_all(list); 2537a9f50a0SPeter Zijlstra llist_for_each_entry_safe(work, tmp, llnode, node.llist) 2544b44a21dSPeter Zijlstra irq_work_single(work); 255e360adbeSPeter Zijlstra } 256c0e980a4SSteven Rostedt 257c0e980a4SSteven Rostedt /* 258a77353e5SPeter Zijlstra * hotplug calls this through: 259a77353e5SPeter Zijlstra * hotplug_cfd() -> flush_smp_call_function_queue() 260c0e980a4SSteven Rostedt */ 261c0e980a4SSteven Rostedt void irq_work_run(void) 262c0e980a4SSteven Rostedt { 26322127e93SChristoph Lameter irq_work_run_list(this_cpu_ptr(&raised_list)); 264b4c6f86eSSebastian Andrzej Siewior if (!IS_ENABLED(CONFIG_PREEMPT_RT)) 26522127e93SChristoph Lameter irq_work_run_list(this_cpu_ptr(&lazy_list)); 266b4c6f86eSSebastian Andrzej Siewior else 267b4c6f86eSSebastian Andrzej Siewior wake_irq_workd(); 268c0e980a4SSteven Rostedt } 269e360adbeSPeter Zijlstra EXPORT_SYMBOL_GPL(irq_work_run); 270e360adbeSPeter Zijlstra 27176a33061SFrederic Weisbecker void irq_work_tick(void) 27276a33061SFrederic Weisbecker { 27356e4dea8SChristoph Lameter struct llist_head *raised = this_cpu_ptr(&raised_list); 27476a33061SFrederic Weisbecker 27576a33061SFrederic Weisbecker if (!llist_empty(raised) && !arch_irq_work_has_interrupt()) 27676a33061SFrederic Weisbecker irq_work_run_list(raised); 277b4c6f86eSSebastian Andrzej Siewior 278b4c6f86eSSebastian Andrzej Siewior if (!IS_ENABLED(CONFIG_PREEMPT_RT)) 27956e4dea8SChristoph Lameter irq_work_run_list(this_cpu_ptr(&lazy_list)); 280b4c6f86eSSebastian Andrzej Siewior else 281b4c6f86eSSebastian Andrzej Siewior wake_irq_workd(); 28276a33061SFrederic Weisbecker } 28376a33061SFrederic Weisbecker 284e360adbeSPeter Zijlstra /* 285e360adbeSPeter Zijlstra * Synchronize against the irq_work @entry, ensures the entry is not 286e360adbeSPeter Zijlstra * currently in use. 287e360adbeSPeter Zijlstra */ 28838aaf809SHuang Ying void irq_work_sync(struct irq_work *work) 289e360adbeSPeter Zijlstra { 2903c7169a3SFrederic Weisbecker lockdep_assert_irqs_enabled(); 29181097968SSebastian Andrzej Siewior might_sleep(); 29281097968SSebastian Andrzej Siewior 29309089db7SSebastian Andrzej Siewior if ((IS_ENABLED(CONFIG_PREEMPT_RT) && !irq_work_is_hard(work)) || 29409089db7SSebastian Andrzej Siewior !arch_irq_work_has_interrupt()) { 29581097968SSebastian Andrzej Siewior rcuwait_wait_event(&work->irqwait, !irq_work_is_busy(work), 29681097968SSebastian Andrzej Siewior TASK_UNINTERRUPTIBLE); 29781097968SSebastian Andrzej Siewior return; 29881097968SSebastian Andrzej Siewior } 299e360adbeSPeter Zijlstra 3007a9f50a0SPeter Zijlstra while (irq_work_is_busy(work)) 301e360adbeSPeter Zijlstra cpu_relax(); 302e360adbeSPeter Zijlstra } 303e360adbeSPeter Zijlstra EXPORT_SYMBOL_GPL(irq_work_sync); 304b4c6f86eSSebastian Andrzej Siewior 305b4c6f86eSSebastian Andrzej Siewior static void run_irq_workd(unsigned int cpu) 306b4c6f86eSSebastian Andrzej Siewior { 307b4c6f86eSSebastian Andrzej Siewior irq_work_run_list(this_cpu_ptr(&lazy_list)); 308b4c6f86eSSebastian Andrzej Siewior } 309b4c6f86eSSebastian Andrzej Siewior 310b4c6f86eSSebastian Andrzej Siewior static void irq_workd_setup(unsigned int cpu) 311b4c6f86eSSebastian Andrzej Siewior { 312b4c6f86eSSebastian Andrzej Siewior sched_set_fifo_low(current); 313b4c6f86eSSebastian Andrzej Siewior } 314b4c6f86eSSebastian Andrzej Siewior 315b4c6f86eSSebastian Andrzej Siewior static struct smp_hotplug_thread irqwork_threads = { 316b4c6f86eSSebastian Andrzej Siewior .store = &irq_workd, 317b4c6f86eSSebastian Andrzej Siewior .setup = irq_workd_setup, 318b4c6f86eSSebastian Andrzej Siewior .thread_should_run = irq_workd_should_run, 319b4c6f86eSSebastian Andrzej Siewior .thread_fn = run_irq_workd, 320b4c6f86eSSebastian Andrzej Siewior .thread_comm = "irq_work/%u", 321b4c6f86eSSebastian Andrzej Siewior }; 322b4c6f86eSSebastian Andrzej Siewior 323b4c6f86eSSebastian Andrzej Siewior static __init int irq_work_init_threads(void) 324b4c6f86eSSebastian Andrzej Siewior { 325b4c6f86eSSebastian Andrzej Siewior if (IS_ENABLED(CONFIG_PREEMPT_RT)) 326b4c6f86eSSebastian Andrzej Siewior BUG_ON(smpboot_register_percpu_thread(&irqwork_threads)); 327b4c6f86eSSebastian Andrzej Siewior return 0; 328b4c6f86eSSebastian Andrzej Siewior } 329b4c6f86eSSebastian Andrzej Siewior early_initcall(irq_work_init_threads); 330