1457c8996SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 2e360adbeSPeter Zijlstra /* 390eec103SPeter Zijlstra * Copyright (C) 2010 Red Hat, Inc., Peter Zijlstra 4e360adbeSPeter Zijlstra * 5e360adbeSPeter Zijlstra * Provides a framework for enqueueing and running callbacks from hardirq 6e360adbeSPeter Zijlstra * context. The enqueueing is NMI-safe. 7e360adbeSPeter Zijlstra */ 8e360adbeSPeter Zijlstra 983e3fa6fSPaul Gortmaker #include <linux/bug.h> 10e360adbeSPeter Zijlstra #include <linux/kernel.h> 119984de1aSPaul Gortmaker #include <linux/export.h> 12e360adbeSPeter Zijlstra #include <linux/irq_work.h> 13967d1f90SPaul Gortmaker #include <linux/percpu.h> 14e360adbeSPeter Zijlstra #include <linux/hardirq.h> 15ef1f0982SChris Metcalf #include <linux/irqflags.h> 16bc6679aeSFrederic Weisbecker #include <linux/sched.h> 17bc6679aeSFrederic Weisbecker #include <linux/tick.h> 18c0e980a4SSteven Rostedt #include <linux/cpu.h> 19c0e980a4SSteven Rostedt #include <linux/notifier.h> 2047885016SFrederic Weisbecker #include <linux/smp.h> 21*b4c6f86eSSebastian Andrzej Siewior #include <linux/smpboot.h> 22967d1f90SPaul Gortmaker #include <asm/processor.h> 23e2b5bcf9SZqiang #include <linux/kasan.h> 24e360adbeSPeter Zijlstra 25b93e0b8fSFrederic Weisbecker static DEFINE_PER_CPU(struct llist_head, raised_list); 26b93e0b8fSFrederic Weisbecker static DEFINE_PER_CPU(struct llist_head, lazy_list); 27*b4c6f86eSSebastian Andrzej Siewior static DEFINE_PER_CPU(struct task_struct *, irq_workd); 28*b4c6f86eSSebastian Andrzej Siewior 29*b4c6f86eSSebastian Andrzej Siewior static void wake_irq_workd(void) 30*b4c6f86eSSebastian Andrzej Siewior { 31*b4c6f86eSSebastian Andrzej Siewior struct task_struct *tsk = __this_cpu_read(irq_workd); 32*b4c6f86eSSebastian Andrzej Siewior 33*b4c6f86eSSebastian Andrzej Siewior if (!llist_empty(this_cpu_ptr(&lazy_list)) && tsk) 34*b4c6f86eSSebastian Andrzej Siewior wake_up_process(tsk); 35*b4c6f86eSSebastian Andrzej Siewior } 36*b4c6f86eSSebastian Andrzej Siewior 37*b4c6f86eSSebastian Andrzej Siewior #ifdef CONFIG_SMP 38*b4c6f86eSSebastian Andrzej Siewior static void irq_work_wake(struct irq_work *entry) 39*b4c6f86eSSebastian Andrzej Siewior { 40*b4c6f86eSSebastian Andrzej Siewior wake_irq_workd(); 41*b4c6f86eSSebastian Andrzej Siewior } 42*b4c6f86eSSebastian Andrzej Siewior 43*b4c6f86eSSebastian Andrzej Siewior static DEFINE_PER_CPU(struct irq_work, irq_work_wakeup) = 44*b4c6f86eSSebastian Andrzej Siewior IRQ_WORK_INIT_HARD(irq_work_wake); 45*b4c6f86eSSebastian Andrzej Siewior #endif 46*b4c6f86eSSebastian Andrzej Siewior 47*b4c6f86eSSebastian Andrzej Siewior static int irq_workd_should_run(unsigned int cpu) 48*b4c6f86eSSebastian Andrzej Siewior { 49*b4c6f86eSSebastian Andrzej Siewior return !llist_empty(this_cpu_ptr(&lazy_list)); 50*b4c6f86eSSebastian Andrzej Siewior } 51e360adbeSPeter Zijlstra 52e360adbeSPeter Zijlstra /* 53e360adbeSPeter Zijlstra * Claim the entry so that no one else will poke at it. 54e360adbeSPeter Zijlstra */ 5538aaf809SHuang Ying static bool irq_work_claim(struct irq_work *work) 56e360adbeSPeter Zijlstra { 5725269871SFrederic Weisbecker int oflags; 58e360adbeSPeter Zijlstra 597a9f50a0SPeter Zijlstra oflags = atomic_fetch_or(IRQ_WORK_CLAIMED | CSD_TYPE_IRQ_WORK, &work->node.a_flags); 60e0bbe2d8SFrederic Weisbecker /* 6125269871SFrederic Weisbecker * If the work is already pending, no need to raise the IPI. 622914b0baSPeter Zijlstra * The pairing smp_mb() in irq_work_single() makes sure 6325269871SFrederic Weisbecker * everything we did before is visible. 64e0bbe2d8SFrederic Weisbecker */ 65e0bbe2d8SFrederic Weisbecker if (oflags & IRQ_WORK_PENDING) 66e0bbe2d8SFrederic Weisbecker return false; 67e360adbeSPeter Zijlstra return true; 68e360adbeSPeter Zijlstra } 69e360adbeSPeter Zijlstra 70e360adbeSPeter Zijlstra void __weak arch_irq_work_raise(void) 71e360adbeSPeter Zijlstra { 72e360adbeSPeter Zijlstra /* 73e360adbeSPeter Zijlstra * Lame architectures will get the timer tick callback 74e360adbeSPeter Zijlstra */ 75e360adbeSPeter Zijlstra } 76e360adbeSPeter Zijlstra 77471ba0e6SNicholas Piggin /* Enqueue on current CPU, work must already be claimed and preempt disabled */ 78471ba0e6SNicholas Piggin static void __irq_work_queue_local(struct irq_work *work) 7947885016SFrederic Weisbecker { 80*b4c6f86eSSebastian Andrzej Siewior struct llist_head *list; 81*b4c6f86eSSebastian Andrzej Siewior bool rt_lazy_work = false; 82*b4c6f86eSSebastian Andrzej Siewior bool lazy_work = false; 83*b4c6f86eSSebastian Andrzej Siewior int work_flags; 84*b4c6f86eSSebastian Andrzej Siewior 85*b4c6f86eSSebastian Andrzej Siewior work_flags = atomic_read(&work->node.a_flags); 86*b4c6f86eSSebastian Andrzej Siewior if (work_flags & IRQ_WORK_LAZY) 87*b4c6f86eSSebastian Andrzej Siewior lazy_work = true; 88*b4c6f86eSSebastian Andrzej Siewior else if (IS_ENABLED(CONFIG_PREEMPT_RT) && 89*b4c6f86eSSebastian Andrzej Siewior !(work_flags & IRQ_WORK_HARD_IRQ)) 90*b4c6f86eSSebastian Andrzej Siewior rt_lazy_work = true; 91*b4c6f86eSSebastian Andrzej Siewior 92*b4c6f86eSSebastian Andrzej Siewior if (lazy_work || rt_lazy_work) 93*b4c6f86eSSebastian Andrzej Siewior list = this_cpu_ptr(&lazy_list); 94*b4c6f86eSSebastian Andrzej Siewior else 95*b4c6f86eSSebastian Andrzej Siewior list = this_cpu_ptr(&raised_list); 96*b4c6f86eSSebastian Andrzej Siewior 97*b4c6f86eSSebastian Andrzej Siewior if (!llist_add(&work->node.llist, list)) 98*b4c6f86eSSebastian Andrzej Siewior return; 99*b4c6f86eSSebastian Andrzej Siewior 100471ba0e6SNicholas Piggin /* If the work is "lazy", handle it from next tick if any */ 101*b4c6f86eSSebastian Andrzej Siewior if (!lazy_work || tick_nohz_tick_stopped()) 102471ba0e6SNicholas Piggin arch_irq_work_raise(); 10347885016SFrederic Weisbecker } 10447885016SFrederic Weisbecker 10547885016SFrederic Weisbecker /* Enqueue the irq work @work on the current CPU */ 106cd578abbSPeter Zijlstra bool irq_work_queue(struct irq_work *work) 107e360adbeSPeter Zijlstra { 108c02cf5f8Sanish kumar /* Only queue if not already pending */ 109c02cf5f8Sanish kumar if (!irq_work_claim(work)) 110cd578abbSPeter Zijlstra return false; 111c02cf5f8Sanish kumar 112c02cf5f8Sanish kumar /* Queue the entry and raise the IPI if needed. */ 11320b87691SChristoph Lameter preempt_disable(); 114471ba0e6SNicholas Piggin __irq_work_queue_local(work); 11520b87691SChristoph Lameter preempt_enable(); 116cd578abbSPeter Zijlstra 117cd578abbSPeter Zijlstra return true; 118e360adbeSPeter Zijlstra } 119e360adbeSPeter Zijlstra EXPORT_SYMBOL_GPL(irq_work_queue); 120e360adbeSPeter Zijlstra 121471ba0e6SNicholas Piggin /* 122471ba0e6SNicholas Piggin * Enqueue the irq_work @work on @cpu unless it's already pending 123471ba0e6SNicholas Piggin * somewhere. 124471ba0e6SNicholas Piggin * 125471ba0e6SNicholas Piggin * Can be re-enqueued while the callback is still in progress. 126471ba0e6SNicholas Piggin */ 127471ba0e6SNicholas Piggin bool irq_work_queue_on(struct irq_work *work, int cpu) 128471ba0e6SNicholas Piggin { 129471ba0e6SNicholas Piggin #ifndef CONFIG_SMP 130471ba0e6SNicholas Piggin return irq_work_queue(work); 131471ba0e6SNicholas Piggin 132471ba0e6SNicholas Piggin #else /* CONFIG_SMP: */ 133471ba0e6SNicholas Piggin /* All work should have been flushed before going offline */ 134471ba0e6SNicholas Piggin WARN_ON_ONCE(cpu_is_offline(cpu)); 135471ba0e6SNicholas Piggin 136471ba0e6SNicholas Piggin /* Only queue if not already pending */ 137471ba0e6SNicholas Piggin if (!irq_work_claim(work)) 138471ba0e6SNicholas Piggin return false; 139471ba0e6SNicholas Piggin 140e2b5bcf9SZqiang kasan_record_aux_stack(work); 141e2b5bcf9SZqiang 142471ba0e6SNicholas Piggin preempt_disable(); 143471ba0e6SNicholas Piggin if (cpu != smp_processor_id()) { 144471ba0e6SNicholas Piggin /* Arch remote IPI send/receive backend aren't NMI safe */ 145471ba0e6SNicholas Piggin WARN_ON_ONCE(in_nmi()); 146*b4c6f86eSSebastian Andrzej Siewior 147*b4c6f86eSSebastian Andrzej Siewior /* 148*b4c6f86eSSebastian Andrzej Siewior * On PREEMPT_RT the items which are not marked as 149*b4c6f86eSSebastian Andrzej Siewior * IRQ_WORK_HARD_IRQ are added to the lazy list and a HARD work 150*b4c6f86eSSebastian Andrzej Siewior * item is used on the remote CPU to wake the thread. 151*b4c6f86eSSebastian Andrzej Siewior */ 152*b4c6f86eSSebastian Andrzej Siewior if (IS_ENABLED(CONFIG_PREEMPT_RT) && 153*b4c6f86eSSebastian Andrzej Siewior !(atomic_read(&work->node.a_flags) & IRQ_WORK_HARD_IRQ)) { 154*b4c6f86eSSebastian Andrzej Siewior 155*b4c6f86eSSebastian Andrzej Siewior if (!llist_add(&work->node.llist, &per_cpu(lazy_list, cpu))) 156*b4c6f86eSSebastian Andrzej Siewior goto out; 157*b4c6f86eSSebastian Andrzej Siewior 158*b4c6f86eSSebastian Andrzej Siewior work = &per_cpu(irq_work_wakeup, cpu); 159*b4c6f86eSSebastian Andrzej Siewior if (!irq_work_claim(work)) 160*b4c6f86eSSebastian Andrzej Siewior goto out; 161*b4c6f86eSSebastian Andrzej Siewior } 162*b4c6f86eSSebastian Andrzej Siewior 1637a9f50a0SPeter Zijlstra __smp_call_single_queue(cpu, &work->node.llist); 164471ba0e6SNicholas Piggin } else { 165471ba0e6SNicholas Piggin __irq_work_queue_local(work); 166471ba0e6SNicholas Piggin } 167*b4c6f86eSSebastian Andrzej Siewior out: 168471ba0e6SNicholas Piggin preempt_enable(); 169471ba0e6SNicholas Piggin 170471ba0e6SNicholas Piggin return true; 171471ba0e6SNicholas Piggin #endif /* CONFIG_SMP */ 172471ba0e6SNicholas Piggin } 173471ba0e6SNicholas Piggin 17400b42959SFrederic Weisbecker bool irq_work_needs_cpu(void) 175e360adbeSPeter Zijlstra { 176b93e0b8fSFrederic Weisbecker struct llist_head *raised, *lazy; 17700b42959SFrederic Weisbecker 17822127e93SChristoph Lameter raised = this_cpu_ptr(&raised_list); 17922127e93SChristoph Lameter lazy = this_cpu_ptr(&lazy_list); 18076a33061SFrederic Weisbecker 18176a33061SFrederic Weisbecker if (llist_empty(raised) || arch_irq_work_has_interrupt()) 18276a33061SFrederic Weisbecker if (llist_empty(lazy)) 18300b42959SFrederic Weisbecker return false; 18400b42959SFrederic Weisbecker 1858aa2acceSSteven Rostedt /* All work should have been flushed before going offline */ 1868aa2acceSSteven Rostedt WARN_ON_ONCE(cpu_is_offline(smp_processor_id())); 1878aa2acceSSteven Rostedt 18800b42959SFrederic Weisbecker return true; 18900b42959SFrederic Weisbecker } 19000b42959SFrederic Weisbecker 1914b44a21dSPeter Zijlstra void irq_work_single(void *arg) 192e360adbeSPeter Zijlstra { 1934b44a21dSPeter Zijlstra struct irq_work *work = arg; 194feb4a513SFrederic Weisbecker int flags; 1954b44a21dSPeter Zijlstra 196e360adbeSPeter Zijlstra /* 1972914b0baSPeter Zijlstra * Clear the PENDING bit, after this point the @work can be re-used. 1982914b0baSPeter Zijlstra * The PENDING bit acts as a lock, and we own it, so we can clear it 1992914b0baSPeter Zijlstra * without atomic ops. 200e360adbeSPeter Zijlstra */ 2012914b0baSPeter Zijlstra flags = atomic_read(&work->node.a_flags); 202e9838bd5SFrederic Weisbecker flags &= ~IRQ_WORK_PENDING; 2032914b0baSPeter Zijlstra atomic_set(&work->node.a_flags, flags); 2042914b0baSPeter Zijlstra 2052914b0baSPeter Zijlstra /* 2062914b0baSPeter Zijlstra * See irq_work_claim(). 2072914b0baSPeter Zijlstra */ 2082914b0baSPeter Zijlstra smp_mb(); 2092914b0baSPeter Zijlstra 2102914b0baSPeter Zijlstra lockdep_irq_work_enter(flags); 2112914b0baSPeter Zijlstra work->func(work); 2122914b0baSPeter Zijlstra lockdep_irq_work_exit(flags); 2132914b0baSPeter Zijlstra 2142914b0baSPeter Zijlstra /* 2152914b0baSPeter Zijlstra * Clear the BUSY bit, if set, and return to the free state if no-one 2162914b0baSPeter Zijlstra * else claimed it meanwhile. 2172914b0baSPeter Zijlstra */ 2187a9f50a0SPeter Zijlstra (void)atomic_cmpxchg(&work->node.a_flags, flags, flags & ~IRQ_WORK_BUSY); 21981097968SSebastian Andrzej Siewior 22081097968SSebastian Andrzej Siewior if (!arch_irq_work_has_interrupt()) 22181097968SSebastian Andrzej Siewior rcuwait_wake_up(&work->irqwait); 222e360adbeSPeter Zijlstra } 2234b44a21dSPeter Zijlstra 2244b44a21dSPeter Zijlstra static void irq_work_run_list(struct llist_head *list) 2254b44a21dSPeter Zijlstra { 2264b44a21dSPeter Zijlstra struct irq_work *work, *tmp; 2274b44a21dSPeter Zijlstra struct llist_node *llnode; 2284b44a21dSPeter Zijlstra 229*b4c6f86eSSebastian Andrzej Siewior /* 230*b4c6f86eSSebastian Andrzej Siewior * On PREEMPT_RT IRQ-work which is not marked as HARD will be processed 231*b4c6f86eSSebastian Andrzej Siewior * in a per-CPU thread in preemptible context. Only the items which are 232*b4c6f86eSSebastian Andrzej Siewior * marked as IRQ_WORK_HARD_IRQ will be processed in hardirq context. 233*b4c6f86eSSebastian Andrzej Siewior */ 234*b4c6f86eSSebastian Andrzej Siewior BUG_ON(!irqs_disabled() && !IS_ENABLED(CONFIG_PREEMPT_RT)); 2354b44a21dSPeter Zijlstra 2364b44a21dSPeter Zijlstra if (llist_empty(list)) 2374b44a21dSPeter Zijlstra return; 2384b44a21dSPeter Zijlstra 2394b44a21dSPeter Zijlstra llnode = llist_del_all(list); 2407a9f50a0SPeter Zijlstra llist_for_each_entry_safe(work, tmp, llnode, node.llist) 2414b44a21dSPeter Zijlstra irq_work_single(work); 242e360adbeSPeter Zijlstra } 243c0e980a4SSteven Rostedt 244c0e980a4SSteven Rostedt /* 245a77353e5SPeter Zijlstra * hotplug calls this through: 246a77353e5SPeter Zijlstra * hotplug_cfd() -> flush_smp_call_function_queue() 247c0e980a4SSteven Rostedt */ 248c0e980a4SSteven Rostedt void irq_work_run(void) 249c0e980a4SSteven Rostedt { 25022127e93SChristoph Lameter irq_work_run_list(this_cpu_ptr(&raised_list)); 251*b4c6f86eSSebastian Andrzej Siewior if (!IS_ENABLED(CONFIG_PREEMPT_RT)) 25222127e93SChristoph Lameter irq_work_run_list(this_cpu_ptr(&lazy_list)); 253*b4c6f86eSSebastian Andrzej Siewior else 254*b4c6f86eSSebastian Andrzej Siewior wake_irq_workd(); 255c0e980a4SSteven Rostedt } 256e360adbeSPeter Zijlstra EXPORT_SYMBOL_GPL(irq_work_run); 257e360adbeSPeter Zijlstra 25876a33061SFrederic Weisbecker void irq_work_tick(void) 25976a33061SFrederic Weisbecker { 26056e4dea8SChristoph Lameter struct llist_head *raised = this_cpu_ptr(&raised_list); 26176a33061SFrederic Weisbecker 26276a33061SFrederic Weisbecker if (!llist_empty(raised) && !arch_irq_work_has_interrupt()) 26376a33061SFrederic Weisbecker irq_work_run_list(raised); 264*b4c6f86eSSebastian Andrzej Siewior 265*b4c6f86eSSebastian Andrzej Siewior if (!IS_ENABLED(CONFIG_PREEMPT_RT)) 26656e4dea8SChristoph Lameter irq_work_run_list(this_cpu_ptr(&lazy_list)); 267*b4c6f86eSSebastian Andrzej Siewior else 268*b4c6f86eSSebastian Andrzej Siewior wake_irq_workd(); 26976a33061SFrederic Weisbecker } 27076a33061SFrederic Weisbecker 271e360adbeSPeter Zijlstra /* 272e360adbeSPeter Zijlstra * Synchronize against the irq_work @entry, ensures the entry is not 273e360adbeSPeter Zijlstra * currently in use. 274e360adbeSPeter Zijlstra */ 27538aaf809SHuang Ying void irq_work_sync(struct irq_work *work) 276e360adbeSPeter Zijlstra { 2773c7169a3SFrederic Weisbecker lockdep_assert_irqs_enabled(); 27881097968SSebastian Andrzej Siewior might_sleep(); 27981097968SSebastian Andrzej Siewior 28081097968SSebastian Andrzej Siewior if (!arch_irq_work_has_interrupt()) { 28181097968SSebastian Andrzej Siewior rcuwait_wait_event(&work->irqwait, !irq_work_is_busy(work), 28281097968SSebastian Andrzej Siewior TASK_UNINTERRUPTIBLE); 28381097968SSebastian Andrzej Siewior return; 28481097968SSebastian Andrzej Siewior } 285e360adbeSPeter Zijlstra 2867a9f50a0SPeter Zijlstra while (irq_work_is_busy(work)) 287e360adbeSPeter Zijlstra cpu_relax(); 288e360adbeSPeter Zijlstra } 289e360adbeSPeter Zijlstra EXPORT_SYMBOL_GPL(irq_work_sync); 290*b4c6f86eSSebastian Andrzej Siewior 291*b4c6f86eSSebastian Andrzej Siewior static void run_irq_workd(unsigned int cpu) 292*b4c6f86eSSebastian Andrzej Siewior { 293*b4c6f86eSSebastian Andrzej Siewior irq_work_run_list(this_cpu_ptr(&lazy_list)); 294*b4c6f86eSSebastian Andrzej Siewior } 295*b4c6f86eSSebastian Andrzej Siewior 296*b4c6f86eSSebastian Andrzej Siewior static void irq_workd_setup(unsigned int cpu) 297*b4c6f86eSSebastian Andrzej Siewior { 298*b4c6f86eSSebastian Andrzej Siewior sched_set_fifo_low(current); 299*b4c6f86eSSebastian Andrzej Siewior } 300*b4c6f86eSSebastian Andrzej Siewior 301*b4c6f86eSSebastian Andrzej Siewior static struct smp_hotplug_thread irqwork_threads = { 302*b4c6f86eSSebastian Andrzej Siewior .store = &irq_workd, 303*b4c6f86eSSebastian Andrzej Siewior .setup = irq_workd_setup, 304*b4c6f86eSSebastian Andrzej Siewior .thread_should_run = irq_workd_should_run, 305*b4c6f86eSSebastian Andrzej Siewior .thread_fn = run_irq_workd, 306*b4c6f86eSSebastian Andrzej Siewior .thread_comm = "irq_work/%u", 307*b4c6f86eSSebastian Andrzej Siewior }; 308*b4c6f86eSSebastian Andrzej Siewior 309*b4c6f86eSSebastian Andrzej Siewior static __init int irq_work_init_threads(void) 310*b4c6f86eSSebastian Andrzej Siewior { 311*b4c6f86eSSebastian Andrzej Siewior if (IS_ENABLED(CONFIG_PREEMPT_RT)) 312*b4c6f86eSSebastian Andrzej Siewior BUG_ON(smpboot_register_percpu_thread(&irqwork_threads)); 313*b4c6f86eSSebastian Andrzej Siewior return 0; 314*b4c6f86eSSebastian Andrzej Siewior } 315*b4c6f86eSSebastian Andrzej Siewior early_initcall(irq_work_init_threads); 316