11da177e4SLinus Torvalds /* 21da177e4SLinus Torvalds * linux/kernel/workqueue.c 31da177e4SLinus Torvalds * 41da177e4SLinus Torvalds * Generic mechanism for defining kernel helper threads for running 51da177e4SLinus Torvalds * arbitrary tasks in process context. 61da177e4SLinus Torvalds * 71da177e4SLinus Torvalds * Started by Ingo Molnar, Copyright (C) 2002 81da177e4SLinus Torvalds * 91da177e4SLinus Torvalds * Derived from the taskqueue/keventd code by: 101da177e4SLinus Torvalds * 111da177e4SLinus Torvalds * David Woodhouse <dwmw2@infradead.org> 121da177e4SLinus Torvalds * Andrew Morton <andrewm@uow.edu.au> 131da177e4SLinus Torvalds * Kai Petzke <wpp@marie.physik.tu-berlin.de> 141da177e4SLinus Torvalds * Theodore Ts'o <tytso@mit.edu> 1589ada679SChristoph Lameter * 16cde53535SChristoph Lameter * Made to use alloc_percpu by Christoph Lameter. 171da177e4SLinus Torvalds */ 181da177e4SLinus Torvalds 191da177e4SLinus Torvalds #include <linux/module.h> 201da177e4SLinus Torvalds #include <linux/kernel.h> 211da177e4SLinus Torvalds #include <linux/sched.h> 221da177e4SLinus Torvalds #include <linux/init.h> 231da177e4SLinus Torvalds #include <linux/signal.h> 241da177e4SLinus Torvalds #include <linux/completion.h> 251da177e4SLinus Torvalds #include <linux/workqueue.h> 261da177e4SLinus Torvalds #include <linux/slab.h> 271da177e4SLinus Torvalds #include <linux/cpu.h> 281da177e4SLinus Torvalds #include <linux/notifier.h> 291da177e4SLinus Torvalds #include <linux/kthread.h> 301fa44ecaSJames Bottomley #include <linux/hardirq.h> 3146934023SChristoph Lameter #include <linux/mempolicy.h> 32341a5958SRafael J. Wysocki #include <linux/freezer.h> 33d5abe669SPeter Zijlstra #include <linux/kallsyms.h> 34d5abe669SPeter Zijlstra #include <linux/debug_locks.h> 354e6045f1SJohannes Berg #include <linux/lockdep.h> 361da177e4SLinus Torvalds 371da177e4SLinus Torvalds /* 38f756d5e2SNathan Lynch * The per-CPU workqueue (if single thread, we always use the first 39f756d5e2SNathan Lynch * possible cpu). 401da177e4SLinus Torvalds */ 411da177e4SLinus Torvalds struct cpu_workqueue_struct { 421da177e4SLinus Torvalds 431da177e4SLinus Torvalds spinlock_t lock; 441da177e4SLinus Torvalds 451da177e4SLinus Torvalds struct list_head worklist; 461da177e4SLinus Torvalds wait_queue_head_t more_work; 473af24433SOleg Nesterov struct work_struct *current_work; 481da177e4SLinus Torvalds 491da177e4SLinus Torvalds struct workqueue_struct *wq; 5036c8b586SIngo Molnar struct task_struct *thread; 511da177e4SLinus Torvalds 521da177e4SLinus Torvalds int run_depth; /* Detect run_workqueue() recursion depth */ 531da177e4SLinus Torvalds } ____cacheline_aligned; 541da177e4SLinus Torvalds 551da177e4SLinus Torvalds /* 561da177e4SLinus Torvalds * The externally visible workqueue abstraction is an array of 571da177e4SLinus Torvalds * per-CPU workqueues: 581da177e4SLinus Torvalds */ 591da177e4SLinus Torvalds struct workqueue_struct { 6089ada679SChristoph Lameter struct cpu_workqueue_struct *cpu_wq; 61cce1a165SOleg Nesterov struct list_head list; 621da177e4SLinus Torvalds const char *name; 63cce1a165SOleg Nesterov int singlethread; 64319c2a98SOleg Nesterov int freezeable; /* Freeze threads during suspend */ 654e6045f1SJohannes Berg #ifdef CONFIG_LOCKDEP 664e6045f1SJohannes Berg struct lockdep_map lockdep_map; 674e6045f1SJohannes Berg #endif 681da177e4SLinus Torvalds }; 691da177e4SLinus Torvalds 7095402b38SGautham R Shenoy /* Serializes the accesses to the list of workqueues. */ 7195402b38SGautham R Shenoy static DEFINE_SPINLOCK(workqueue_lock); 721da177e4SLinus Torvalds static LIST_HEAD(workqueues); 731da177e4SLinus Torvalds 743af24433SOleg Nesterov static int singlethread_cpu __read_mostly; 75b1f4ec17SOleg Nesterov static cpumask_t cpu_singlethread_map __read_mostly; 7614441960SOleg Nesterov /* 7714441960SOleg Nesterov * _cpu_down() first removes CPU from cpu_online_map, then CPU_DEAD 7814441960SOleg Nesterov * flushes cwq->worklist. This means that flush_workqueue/wait_on_work 7914441960SOleg Nesterov * which comes in between can't use for_each_online_cpu(). We could 8014441960SOleg Nesterov * use cpu_possible_map, the cpumask below is more a documentation 8114441960SOleg Nesterov * than optimization. 8214441960SOleg Nesterov */ 833af24433SOleg Nesterov static cpumask_t cpu_populated_map __read_mostly; 84f756d5e2SNathan Lynch 851da177e4SLinus Torvalds /* If it's single threaded, it isn't in the list of workqueues. */ 861da177e4SLinus Torvalds static inline int is_single_threaded(struct workqueue_struct *wq) 871da177e4SLinus Torvalds { 88cce1a165SOleg Nesterov return wq->singlethread; 891da177e4SLinus Torvalds } 901da177e4SLinus Torvalds 91b1f4ec17SOleg Nesterov static const cpumask_t *wq_cpu_map(struct workqueue_struct *wq) 92b1f4ec17SOleg Nesterov { 93b1f4ec17SOleg Nesterov return is_single_threaded(wq) 94b1f4ec17SOleg Nesterov ? &cpu_singlethread_map : &cpu_populated_map; 95b1f4ec17SOleg Nesterov } 96b1f4ec17SOleg Nesterov 97a848e3b6SOleg Nesterov static 98a848e3b6SOleg Nesterov struct cpu_workqueue_struct *wq_per_cpu(struct workqueue_struct *wq, int cpu) 99a848e3b6SOleg Nesterov { 100a848e3b6SOleg Nesterov if (unlikely(is_single_threaded(wq))) 101a848e3b6SOleg Nesterov cpu = singlethread_cpu; 102a848e3b6SOleg Nesterov return per_cpu_ptr(wq->cpu_wq, cpu); 103a848e3b6SOleg Nesterov } 104a848e3b6SOleg Nesterov 1054594bf15SDavid Howells /* 1064594bf15SDavid Howells * Set the workqueue on which a work item is to be run 1074594bf15SDavid Howells * - Must *only* be called if the pending flag is set 1084594bf15SDavid Howells */ 109ed7c0feeSOleg Nesterov static inline void set_wq_data(struct work_struct *work, 110ed7c0feeSOleg Nesterov struct cpu_workqueue_struct *cwq) 111365970a1SDavid Howells { 1124594bf15SDavid Howells unsigned long new; 113365970a1SDavid Howells 1144594bf15SDavid Howells BUG_ON(!work_pending(work)); 1154594bf15SDavid Howells 116ed7c0feeSOleg Nesterov new = (unsigned long) cwq | (1UL << WORK_STRUCT_PENDING); 117a08727baSLinus Torvalds new |= WORK_STRUCT_FLAG_MASK & *work_data_bits(work); 118a08727baSLinus Torvalds atomic_long_set(&work->data, new); 119365970a1SDavid Howells } 120365970a1SDavid Howells 121ed7c0feeSOleg Nesterov static inline 122ed7c0feeSOleg Nesterov struct cpu_workqueue_struct *get_wq_data(struct work_struct *work) 123365970a1SDavid Howells { 124a08727baSLinus Torvalds return (void *) (atomic_long_read(&work->data) & WORK_STRUCT_WQ_DATA_MASK); 125365970a1SDavid Howells } 126365970a1SDavid Howells 127b89deed3SOleg Nesterov static void insert_work(struct cpu_workqueue_struct *cwq, 1281a4d9b0aSOleg Nesterov struct work_struct *work, struct list_head *head) 129b89deed3SOleg Nesterov { 130b89deed3SOleg Nesterov set_wq_data(work, cwq); 1316e84d644SOleg Nesterov /* 1326e84d644SOleg Nesterov * Ensure that we get the right work->data if we see the 1336e84d644SOleg Nesterov * result of list_add() below, see try_to_grab_pending(). 1346e84d644SOleg Nesterov */ 1356e84d644SOleg Nesterov smp_wmb(); 1361a4d9b0aSOleg Nesterov list_add_tail(&work->entry, head); 137b89deed3SOleg Nesterov wake_up(&cwq->more_work); 138b89deed3SOleg Nesterov } 139b89deed3SOleg Nesterov 1401da177e4SLinus Torvalds static void __queue_work(struct cpu_workqueue_struct *cwq, 1411da177e4SLinus Torvalds struct work_struct *work) 1421da177e4SLinus Torvalds { 1431da177e4SLinus Torvalds unsigned long flags; 1441da177e4SLinus Torvalds 1451da177e4SLinus Torvalds spin_lock_irqsave(&cwq->lock, flags); 1461a4d9b0aSOleg Nesterov insert_work(cwq, work, &cwq->worklist); 1471da177e4SLinus Torvalds spin_unlock_irqrestore(&cwq->lock, flags); 1481da177e4SLinus Torvalds } 1491da177e4SLinus Torvalds 1500fcb78c2SRolf Eike Beer /** 1510fcb78c2SRolf Eike Beer * queue_work - queue work on a workqueue 1520fcb78c2SRolf Eike Beer * @wq: workqueue to use 1530fcb78c2SRolf Eike Beer * @work: work to queue 1540fcb78c2SRolf Eike Beer * 155057647fcSAlan Stern * Returns 0 if @work was already on a queue, non-zero otherwise. 1561da177e4SLinus Torvalds * 15700dfcaf7SOleg Nesterov * We queue the work to the CPU on which it was submitted, but if the CPU dies 15800dfcaf7SOleg Nesterov * it can be processed by another CPU. 1591da177e4SLinus Torvalds */ 1607ad5b3a5SHarvey Harrison int queue_work(struct workqueue_struct *wq, struct work_struct *work) 1611da177e4SLinus Torvalds { 162a848e3b6SOleg Nesterov int ret = 0; 1631da177e4SLinus Torvalds 164a08727baSLinus Torvalds if (!test_and_set_bit(WORK_STRUCT_PENDING, work_data_bits(work))) { 1651da177e4SLinus Torvalds BUG_ON(!list_empty(&work->entry)); 166a848e3b6SOleg Nesterov __queue_work(wq_per_cpu(wq, get_cpu()), work); 167a848e3b6SOleg Nesterov put_cpu(); 1681da177e4SLinus Torvalds ret = 1; 1691da177e4SLinus Torvalds } 1701da177e4SLinus Torvalds return ret; 1711da177e4SLinus Torvalds } 172ae90dd5dSDave Jones EXPORT_SYMBOL_GPL(queue_work); 1731da177e4SLinus Torvalds 174c1a220e7SZhang Rui /** 175c1a220e7SZhang Rui * queue_work_on - queue work on specific cpu 176c1a220e7SZhang Rui * @cpu: CPU number to execute work on 177c1a220e7SZhang Rui * @wq: workqueue to use 178c1a220e7SZhang Rui * @work: work to queue 179c1a220e7SZhang Rui * 180c1a220e7SZhang Rui * Returns 0 if @work was already on a queue, non-zero otherwise. 181c1a220e7SZhang Rui * 182c1a220e7SZhang Rui * We queue the work to a specific CPU, the caller must ensure it 183c1a220e7SZhang Rui * can't go away. 184c1a220e7SZhang Rui */ 185c1a220e7SZhang Rui int 186c1a220e7SZhang Rui queue_work_on(int cpu, struct workqueue_struct *wq, struct work_struct *work) 187c1a220e7SZhang Rui { 188c1a220e7SZhang Rui int ret = 0; 189c1a220e7SZhang Rui 190c1a220e7SZhang Rui if (!test_and_set_bit(WORK_STRUCT_PENDING, work_data_bits(work))) { 191c1a220e7SZhang Rui BUG_ON(!list_empty(&work->entry)); 192c1a220e7SZhang Rui __queue_work(wq_per_cpu(wq, cpu), work); 193c1a220e7SZhang Rui ret = 1; 194c1a220e7SZhang Rui } 195c1a220e7SZhang Rui return ret; 196c1a220e7SZhang Rui } 197c1a220e7SZhang Rui EXPORT_SYMBOL_GPL(queue_work_on); 198c1a220e7SZhang Rui 1996d141c3fSLi Zefan static void delayed_work_timer_fn(unsigned long __data) 2001da177e4SLinus Torvalds { 20152bad64dSDavid Howells struct delayed_work *dwork = (struct delayed_work *)__data; 202ed7c0feeSOleg Nesterov struct cpu_workqueue_struct *cwq = get_wq_data(&dwork->work); 203ed7c0feeSOleg Nesterov struct workqueue_struct *wq = cwq->wq; 2041da177e4SLinus Torvalds 205a848e3b6SOleg Nesterov __queue_work(wq_per_cpu(wq, smp_processor_id()), &dwork->work); 2061da177e4SLinus Torvalds } 2071da177e4SLinus Torvalds 2080fcb78c2SRolf Eike Beer /** 2090fcb78c2SRolf Eike Beer * queue_delayed_work - queue work on a workqueue after delay 2100fcb78c2SRolf Eike Beer * @wq: workqueue to use 211af9997e4SRandy Dunlap * @dwork: delayable work to queue 2120fcb78c2SRolf Eike Beer * @delay: number of jiffies to wait before queueing 2130fcb78c2SRolf Eike Beer * 214057647fcSAlan Stern * Returns 0 if @work was already on a queue, non-zero otherwise. 2150fcb78c2SRolf Eike Beer */ 2167ad5b3a5SHarvey Harrison int queue_delayed_work(struct workqueue_struct *wq, 21752bad64dSDavid Howells struct delayed_work *dwork, unsigned long delay) 2181da177e4SLinus Torvalds { 21952bad64dSDavid Howells if (delay == 0) 22063bc0362SOleg Nesterov return queue_work(wq, &dwork->work); 2211da177e4SLinus Torvalds 22263bc0362SOleg Nesterov return queue_delayed_work_on(-1, wq, dwork, delay); 2231da177e4SLinus Torvalds } 224ae90dd5dSDave Jones EXPORT_SYMBOL_GPL(queue_delayed_work); 2251da177e4SLinus Torvalds 2260fcb78c2SRolf Eike Beer /** 2270fcb78c2SRolf Eike Beer * queue_delayed_work_on - queue work on specific CPU after delay 2280fcb78c2SRolf Eike Beer * @cpu: CPU number to execute work on 2290fcb78c2SRolf Eike Beer * @wq: workqueue to use 230af9997e4SRandy Dunlap * @dwork: work to queue 2310fcb78c2SRolf Eike Beer * @delay: number of jiffies to wait before queueing 2320fcb78c2SRolf Eike Beer * 233057647fcSAlan Stern * Returns 0 if @work was already on a queue, non-zero otherwise. 2340fcb78c2SRolf Eike Beer */ 2357a6bc1cdSVenkatesh Pallipadi int queue_delayed_work_on(int cpu, struct workqueue_struct *wq, 23652bad64dSDavid Howells struct delayed_work *dwork, unsigned long delay) 2377a6bc1cdSVenkatesh Pallipadi { 2387a6bc1cdSVenkatesh Pallipadi int ret = 0; 23952bad64dSDavid Howells struct timer_list *timer = &dwork->timer; 24052bad64dSDavid Howells struct work_struct *work = &dwork->work; 2417a6bc1cdSVenkatesh Pallipadi 242a08727baSLinus Torvalds if (!test_and_set_bit(WORK_STRUCT_PENDING, work_data_bits(work))) { 2437a6bc1cdSVenkatesh Pallipadi BUG_ON(timer_pending(timer)); 2447a6bc1cdSVenkatesh Pallipadi BUG_ON(!list_empty(&work->entry)); 2457a6bc1cdSVenkatesh Pallipadi 2468a3e77ccSAndrew Liu timer_stats_timer_set_start_info(&dwork->timer); 2478a3e77ccSAndrew Liu 248ed7c0feeSOleg Nesterov /* This stores cwq for the moment, for the timer_fn */ 249a848e3b6SOleg Nesterov set_wq_data(work, wq_per_cpu(wq, raw_smp_processor_id())); 2507a6bc1cdSVenkatesh Pallipadi timer->expires = jiffies + delay; 25152bad64dSDavid Howells timer->data = (unsigned long)dwork; 2527a6bc1cdSVenkatesh Pallipadi timer->function = delayed_work_timer_fn; 25363bc0362SOleg Nesterov 25463bc0362SOleg Nesterov if (unlikely(cpu >= 0)) 2557a6bc1cdSVenkatesh Pallipadi add_timer_on(timer, cpu); 25663bc0362SOleg Nesterov else 25763bc0362SOleg Nesterov add_timer(timer); 2587a6bc1cdSVenkatesh Pallipadi ret = 1; 2597a6bc1cdSVenkatesh Pallipadi } 2607a6bc1cdSVenkatesh Pallipadi return ret; 2617a6bc1cdSVenkatesh Pallipadi } 262ae90dd5dSDave Jones EXPORT_SYMBOL_GPL(queue_delayed_work_on); 2631da177e4SLinus Torvalds 264858119e1SArjan van de Ven static void run_workqueue(struct cpu_workqueue_struct *cwq) 2651da177e4SLinus Torvalds { 266f293ea92SOleg Nesterov spin_lock_irq(&cwq->lock); 2671da177e4SLinus Torvalds cwq->run_depth++; 2681da177e4SLinus Torvalds if (cwq->run_depth > 3) { 2691da177e4SLinus Torvalds /* morton gets to eat his hat */ 2701da177e4SLinus Torvalds printk("%s: recursion depth exceeded: %d\n", 271af1f16d0SHarvey Harrison __func__, cwq->run_depth); 2721da177e4SLinus Torvalds dump_stack(); 2731da177e4SLinus Torvalds } 2741da177e4SLinus Torvalds while (!list_empty(&cwq->worklist)) { 2751da177e4SLinus Torvalds struct work_struct *work = list_entry(cwq->worklist.next, 2761da177e4SLinus Torvalds struct work_struct, entry); 2776bb49e59SDavid Howells work_func_t f = work->func; 2784e6045f1SJohannes Berg #ifdef CONFIG_LOCKDEP 2794e6045f1SJohannes Berg /* 2804e6045f1SJohannes Berg * It is permissible to free the struct work_struct 2814e6045f1SJohannes Berg * from inside the function that is called from it, 2824e6045f1SJohannes Berg * this we need to take into account for lockdep too. 2834e6045f1SJohannes Berg * To avoid bogus "held lock freed" warnings as well 2844e6045f1SJohannes Berg * as problems when looking into work->lockdep_map, 2854e6045f1SJohannes Berg * make a copy and use that here. 2864e6045f1SJohannes Berg */ 2874e6045f1SJohannes Berg struct lockdep_map lockdep_map = work->lockdep_map; 2884e6045f1SJohannes Berg #endif 2891da177e4SLinus Torvalds 290b89deed3SOleg Nesterov cwq->current_work = work; 2911da177e4SLinus Torvalds list_del_init(cwq->worklist.next); 292f293ea92SOleg Nesterov spin_unlock_irq(&cwq->lock); 2931da177e4SLinus Torvalds 294365970a1SDavid Howells BUG_ON(get_wq_data(work) != cwq); 29523b2e599SOleg Nesterov work_clear_pending(work); 2964e6045f1SJohannes Berg lock_acquire(&cwq->wq->lockdep_map, 0, 0, 0, 2, _THIS_IP_); 2974e6045f1SJohannes Berg lock_acquire(&lockdep_map, 0, 0, 0, 2, _THIS_IP_); 29865f27f38SDavid Howells f(work); 2994e6045f1SJohannes Berg lock_release(&lockdep_map, 1, _THIS_IP_); 3004e6045f1SJohannes Berg lock_release(&cwq->wq->lockdep_map, 1, _THIS_IP_); 3011da177e4SLinus Torvalds 302d5abe669SPeter Zijlstra if (unlikely(in_atomic() || lockdep_depth(current) > 0)) { 303d5abe669SPeter Zijlstra printk(KERN_ERR "BUG: workqueue leaked lock or atomic: " 304d5abe669SPeter Zijlstra "%s/0x%08x/%d\n", 305d5abe669SPeter Zijlstra current->comm, preempt_count(), 306ba25f9dcSPavel Emelyanov task_pid_nr(current)); 307d5abe669SPeter Zijlstra printk(KERN_ERR " last function: "); 308d5abe669SPeter Zijlstra print_symbol("%s\n", (unsigned long)f); 309d5abe669SPeter Zijlstra debug_show_held_locks(current); 310d5abe669SPeter Zijlstra dump_stack(); 311d5abe669SPeter Zijlstra } 312d5abe669SPeter Zijlstra 313f293ea92SOleg Nesterov spin_lock_irq(&cwq->lock); 314b89deed3SOleg Nesterov cwq->current_work = NULL; 3151da177e4SLinus Torvalds } 3161da177e4SLinus Torvalds cwq->run_depth--; 317f293ea92SOleg Nesterov spin_unlock_irq(&cwq->lock); 3181da177e4SLinus Torvalds } 3191da177e4SLinus Torvalds 3201da177e4SLinus Torvalds static int worker_thread(void *__cwq) 3211da177e4SLinus Torvalds { 3221da177e4SLinus Torvalds struct cpu_workqueue_struct *cwq = __cwq; 3233af24433SOleg Nesterov DEFINE_WAIT(wait); 3241da177e4SLinus Torvalds 32583144186SRafael J. Wysocki if (cwq->wq->freezeable) 32683144186SRafael J. Wysocki set_freezable(); 3271da177e4SLinus Torvalds 3281da177e4SLinus Torvalds set_user_nice(current, -5); 3291da177e4SLinus Torvalds 3303af24433SOleg Nesterov for (;;) { 3313af24433SOleg Nesterov prepare_to_wait(&cwq->more_work, &wait, TASK_INTERRUPTIBLE); 33214441960SOleg Nesterov if (!freezing(current) && 33314441960SOleg Nesterov !kthread_should_stop() && 33414441960SOleg Nesterov list_empty(&cwq->worklist)) 3351da177e4SLinus Torvalds schedule(); 3363af24433SOleg Nesterov finish_wait(&cwq->more_work, &wait); 3371da177e4SLinus Torvalds 33885f4186aSOleg Nesterov try_to_freeze(); 33985f4186aSOleg Nesterov 34014441960SOleg Nesterov if (kthread_should_stop()) 3413af24433SOleg Nesterov break; 3423af24433SOleg Nesterov 3431da177e4SLinus Torvalds run_workqueue(cwq); 3441da177e4SLinus Torvalds } 3453af24433SOleg Nesterov 3461da177e4SLinus Torvalds return 0; 3471da177e4SLinus Torvalds } 3481da177e4SLinus Torvalds 349fc2e4d70SOleg Nesterov struct wq_barrier { 350fc2e4d70SOleg Nesterov struct work_struct work; 351fc2e4d70SOleg Nesterov struct completion done; 352fc2e4d70SOleg Nesterov }; 353fc2e4d70SOleg Nesterov 354fc2e4d70SOleg Nesterov static void wq_barrier_func(struct work_struct *work) 355fc2e4d70SOleg Nesterov { 356fc2e4d70SOleg Nesterov struct wq_barrier *barr = container_of(work, struct wq_barrier, work); 357fc2e4d70SOleg Nesterov complete(&barr->done); 358fc2e4d70SOleg Nesterov } 359fc2e4d70SOleg Nesterov 36083c22520SOleg Nesterov static void insert_wq_barrier(struct cpu_workqueue_struct *cwq, 3611a4d9b0aSOleg Nesterov struct wq_barrier *barr, struct list_head *head) 362fc2e4d70SOleg Nesterov { 363fc2e4d70SOleg Nesterov INIT_WORK(&barr->work, wq_barrier_func); 364fc2e4d70SOleg Nesterov __set_bit(WORK_STRUCT_PENDING, work_data_bits(&barr->work)); 365fc2e4d70SOleg Nesterov 366fc2e4d70SOleg Nesterov init_completion(&barr->done); 36783c22520SOleg Nesterov 3681a4d9b0aSOleg Nesterov insert_work(cwq, &barr->work, head); 369fc2e4d70SOleg Nesterov } 370fc2e4d70SOleg Nesterov 37114441960SOleg Nesterov static int flush_cpu_workqueue(struct cpu_workqueue_struct *cwq) 3721da177e4SLinus Torvalds { 37314441960SOleg Nesterov int active; 37414441960SOleg Nesterov 3751da177e4SLinus Torvalds if (cwq->thread == current) { 3761da177e4SLinus Torvalds /* 3771da177e4SLinus Torvalds * Probably keventd trying to flush its own queue. So simply run 3781da177e4SLinus Torvalds * it by hand rather than deadlocking. 3791da177e4SLinus Torvalds */ 3801da177e4SLinus Torvalds run_workqueue(cwq); 38114441960SOleg Nesterov active = 1; 3821da177e4SLinus Torvalds } else { 383fc2e4d70SOleg Nesterov struct wq_barrier barr; 3841da177e4SLinus Torvalds 38514441960SOleg Nesterov active = 0; 38683c22520SOleg Nesterov spin_lock_irq(&cwq->lock); 38783c22520SOleg Nesterov if (!list_empty(&cwq->worklist) || cwq->current_work != NULL) { 3881a4d9b0aSOleg Nesterov insert_wq_barrier(cwq, &barr, &cwq->worklist); 38983c22520SOleg Nesterov active = 1; 39083c22520SOleg Nesterov } 39183c22520SOleg Nesterov spin_unlock_irq(&cwq->lock); 3921da177e4SLinus Torvalds 393d721304dSOleg Nesterov if (active) 394fc2e4d70SOleg Nesterov wait_for_completion(&barr.done); 3951da177e4SLinus Torvalds } 39614441960SOleg Nesterov 39714441960SOleg Nesterov return active; 39883c22520SOleg Nesterov } 3991da177e4SLinus Torvalds 4000fcb78c2SRolf Eike Beer /** 4011da177e4SLinus Torvalds * flush_workqueue - ensure that any scheduled work has run to completion. 4020fcb78c2SRolf Eike Beer * @wq: workqueue to flush 4031da177e4SLinus Torvalds * 4041da177e4SLinus Torvalds * Forces execution of the workqueue and blocks until its completion. 4051da177e4SLinus Torvalds * This is typically used in driver shutdown handlers. 4061da177e4SLinus Torvalds * 407fc2e4d70SOleg Nesterov * We sleep until all works which were queued on entry have been handled, 408fc2e4d70SOleg Nesterov * but we are not livelocked by new incoming ones. 4091da177e4SLinus Torvalds * 4101da177e4SLinus Torvalds * This function used to run the workqueues itself. Now we just wait for the 4111da177e4SLinus Torvalds * helper threads to do it. 4121da177e4SLinus Torvalds */ 4137ad5b3a5SHarvey Harrison void flush_workqueue(struct workqueue_struct *wq) 4141da177e4SLinus Torvalds { 415b1f4ec17SOleg Nesterov const cpumask_t *cpu_map = wq_cpu_map(wq); 416cce1a165SOleg Nesterov int cpu; 417b1f4ec17SOleg Nesterov 418f293ea92SOleg Nesterov might_sleep(); 4194e6045f1SJohannes Berg lock_acquire(&wq->lockdep_map, 0, 0, 0, 2, _THIS_IP_); 4204e6045f1SJohannes Berg lock_release(&wq->lockdep_map, 1, _THIS_IP_); 421363ab6f1SMike Travis for_each_cpu_mask_nr(cpu, *cpu_map) 42289ada679SChristoph Lameter flush_cpu_workqueue(per_cpu_ptr(wq->cpu_wq, cpu)); 4231da177e4SLinus Torvalds } 424ae90dd5dSDave Jones EXPORT_SYMBOL_GPL(flush_workqueue); 4251da177e4SLinus Torvalds 426db700897SOleg Nesterov /** 427db700897SOleg Nesterov * flush_work - block until a work_struct's callback has terminated 428db700897SOleg Nesterov * @work: the work which is to be flushed 429db700897SOleg Nesterov * 430db700897SOleg Nesterov * It is expected that, prior to calling flush_work(), the caller has 431db700897SOleg Nesterov * arranged for the work to not be requeued, otherwise it doesn't make 432db700897SOleg Nesterov * sense to use this function. 433db700897SOleg Nesterov */ 434db700897SOleg Nesterov int flush_work(struct work_struct *work) 435db700897SOleg Nesterov { 436db700897SOleg Nesterov struct cpu_workqueue_struct *cwq; 437db700897SOleg Nesterov struct list_head *prev; 438db700897SOleg Nesterov struct wq_barrier barr; 439db700897SOleg Nesterov 440db700897SOleg Nesterov might_sleep(); 441db700897SOleg Nesterov cwq = get_wq_data(work); 442db700897SOleg Nesterov if (!cwq) 443db700897SOleg Nesterov return 0; 444db700897SOleg Nesterov 445db700897SOleg Nesterov prev = NULL; 446db700897SOleg Nesterov spin_lock_irq(&cwq->lock); 447db700897SOleg Nesterov if (!list_empty(&work->entry)) { 448db700897SOleg Nesterov /* 449db700897SOleg Nesterov * See the comment near try_to_grab_pending()->smp_rmb(). 450db700897SOleg Nesterov * If it was re-queued under us we are not going to wait. 451db700897SOleg Nesterov */ 452db700897SOleg Nesterov smp_rmb(); 453db700897SOleg Nesterov if (unlikely(cwq != get_wq_data(work))) 454db700897SOleg Nesterov goto out; 455db700897SOleg Nesterov prev = &work->entry; 456db700897SOleg Nesterov } else { 457db700897SOleg Nesterov if (cwq->current_work != work) 458db700897SOleg Nesterov goto out; 459db700897SOleg Nesterov prev = &cwq->worklist; 460db700897SOleg Nesterov } 461db700897SOleg Nesterov insert_wq_barrier(cwq, &barr, prev->next); 462db700897SOleg Nesterov out: 463db700897SOleg Nesterov spin_unlock_irq(&cwq->lock); 464db700897SOleg Nesterov if (!prev) 465db700897SOleg Nesterov return 0; 466db700897SOleg Nesterov 467db700897SOleg Nesterov wait_for_completion(&barr.done); 468db700897SOleg Nesterov return 1; 469db700897SOleg Nesterov } 470db700897SOleg Nesterov EXPORT_SYMBOL_GPL(flush_work); 471db700897SOleg Nesterov 4726e84d644SOleg Nesterov /* 4731f1f642eSOleg Nesterov * Upon a successful return (>= 0), the caller "owns" WORK_STRUCT_PENDING bit, 4746e84d644SOleg Nesterov * so this work can't be re-armed in any way. 4756e84d644SOleg Nesterov */ 4766e84d644SOleg Nesterov static int try_to_grab_pending(struct work_struct *work) 4776e84d644SOleg Nesterov { 4786e84d644SOleg Nesterov struct cpu_workqueue_struct *cwq; 4791f1f642eSOleg Nesterov int ret = -1; 4806e84d644SOleg Nesterov 4816e84d644SOleg Nesterov if (!test_and_set_bit(WORK_STRUCT_PENDING, work_data_bits(work))) 4821f1f642eSOleg Nesterov return 0; 4836e84d644SOleg Nesterov 4846e84d644SOleg Nesterov /* 4856e84d644SOleg Nesterov * The queueing is in progress, or it is already queued. Try to 4866e84d644SOleg Nesterov * steal it from ->worklist without clearing WORK_STRUCT_PENDING. 4876e84d644SOleg Nesterov */ 4886e84d644SOleg Nesterov 4896e84d644SOleg Nesterov cwq = get_wq_data(work); 4906e84d644SOleg Nesterov if (!cwq) 4916e84d644SOleg Nesterov return ret; 4926e84d644SOleg Nesterov 4936e84d644SOleg Nesterov spin_lock_irq(&cwq->lock); 4946e84d644SOleg Nesterov if (!list_empty(&work->entry)) { 4956e84d644SOleg Nesterov /* 4966e84d644SOleg Nesterov * This work is queued, but perhaps we locked the wrong cwq. 4976e84d644SOleg Nesterov * In that case we must see the new value after rmb(), see 4986e84d644SOleg Nesterov * insert_work()->wmb(). 4996e84d644SOleg Nesterov */ 5006e84d644SOleg Nesterov smp_rmb(); 5016e84d644SOleg Nesterov if (cwq == get_wq_data(work)) { 5026e84d644SOleg Nesterov list_del_init(&work->entry); 5036e84d644SOleg Nesterov ret = 1; 5046e84d644SOleg Nesterov } 5056e84d644SOleg Nesterov } 5066e84d644SOleg Nesterov spin_unlock_irq(&cwq->lock); 5076e84d644SOleg Nesterov 5086e84d644SOleg Nesterov return ret; 5096e84d644SOleg Nesterov } 5106e84d644SOleg Nesterov 5116e84d644SOleg Nesterov static void wait_on_cpu_work(struct cpu_workqueue_struct *cwq, 512b89deed3SOleg Nesterov struct work_struct *work) 513b89deed3SOleg Nesterov { 514b89deed3SOleg Nesterov struct wq_barrier barr; 515b89deed3SOleg Nesterov int running = 0; 516b89deed3SOleg Nesterov 517b89deed3SOleg Nesterov spin_lock_irq(&cwq->lock); 518b89deed3SOleg Nesterov if (unlikely(cwq->current_work == work)) { 5191a4d9b0aSOleg Nesterov insert_wq_barrier(cwq, &barr, cwq->worklist.next); 520b89deed3SOleg Nesterov running = 1; 521b89deed3SOleg Nesterov } 522b89deed3SOleg Nesterov spin_unlock_irq(&cwq->lock); 523b89deed3SOleg Nesterov 5243af24433SOleg Nesterov if (unlikely(running)) 525b89deed3SOleg Nesterov wait_for_completion(&barr.done); 526b89deed3SOleg Nesterov } 527b89deed3SOleg Nesterov 5286e84d644SOleg Nesterov static void wait_on_work(struct work_struct *work) 529b89deed3SOleg Nesterov { 530b89deed3SOleg Nesterov struct cpu_workqueue_struct *cwq; 53128e53bddSOleg Nesterov struct workqueue_struct *wq; 53228e53bddSOleg Nesterov const cpumask_t *cpu_map; 533b1f4ec17SOleg Nesterov int cpu; 534b89deed3SOleg Nesterov 535f293ea92SOleg Nesterov might_sleep(); 536f293ea92SOleg Nesterov 5374e6045f1SJohannes Berg lock_acquire(&work->lockdep_map, 0, 0, 0, 2, _THIS_IP_); 5384e6045f1SJohannes Berg lock_release(&work->lockdep_map, 1, _THIS_IP_); 5394e6045f1SJohannes Berg 540b89deed3SOleg Nesterov cwq = get_wq_data(work); 541b89deed3SOleg Nesterov if (!cwq) 5423af24433SOleg Nesterov return; 543b89deed3SOleg Nesterov 54428e53bddSOleg Nesterov wq = cwq->wq; 54528e53bddSOleg Nesterov cpu_map = wq_cpu_map(wq); 54628e53bddSOleg Nesterov 547363ab6f1SMike Travis for_each_cpu_mask_nr(cpu, *cpu_map) 5486e84d644SOleg Nesterov wait_on_cpu_work(per_cpu_ptr(wq->cpu_wq, cpu), work); 5496e84d644SOleg Nesterov } 5506e84d644SOleg Nesterov 5511f1f642eSOleg Nesterov static int __cancel_work_timer(struct work_struct *work, 5521f1f642eSOleg Nesterov struct timer_list* timer) 5531f1f642eSOleg Nesterov { 5541f1f642eSOleg Nesterov int ret; 5551f1f642eSOleg Nesterov 5561f1f642eSOleg Nesterov do { 5571f1f642eSOleg Nesterov ret = (timer && likely(del_timer(timer))); 5581f1f642eSOleg Nesterov if (!ret) 5591f1f642eSOleg Nesterov ret = try_to_grab_pending(work); 5601f1f642eSOleg Nesterov wait_on_work(work); 5611f1f642eSOleg Nesterov } while (unlikely(ret < 0)); 5621f1f642eSOleg Nesterov 5631f1f642eSOleg Nesterov work_clear_pending(work); 5641f1f642eSOleg Nesterov return ret; 5651f1f642eSOleg Nesterov } 5661f1f642eSOleg Nesterov 5676e84d644SOleg Nesterov /** 5686e84d644SOleg Nesterov * cancel_work_sync - block until a work_struct's callback has terminated 5696e84d644SOleg Nesterov * @work: the work which is to be flushed 5706e84d644SOleg Nesterov * 5711f1f642eSOleg Nesterov * Returns true if @work was pending. 5721f1f642eSOleg Nesterov * 5736e84d644SOleg Nesterov * cancel_work_sync() will cancel the work if it is queued. If the work's 5746e84d644SOleg Nesterov * callback appears to be running, cancel_work_sync() will block until it 5756e84d644SOleg Nesterov * has completed. 5766e84d644SOleg Nesterov * 5776e84d644SOleg Nesterov * It is possible to use this function if the work re-queues itself. It can 5786e84d644SOleg Nesterov * cancel the work even if it migrates to another workqueue, however in that 5796e84d644SOleg Nesterov * case it only guarantees that work->func() has completed on the last queued 5806e84d644SOleg Nesterov * workqueue. 5816e84d644SOleg Nesterov * 5826e84d644SOleg Nesterov * cancel_work_sync(&delayed_work->work) should be used only if ->timer is not 5836e84d644SOleg Nesterov * pending, otherwise it goes into a busy-wait loop until the timer expires. 5846e84d644SOleg Nesterov * 5856e84d644SOleg Nesterov * The caller must ensure that workqueue_struct on which this work was last 5866e84d644SOleg Nesterov * queued can't be destroyed before this function returns. 5876e84d644SOleg Nesterov */ 5881f1f642eSOleg Nesterov int cancel_work_sync(struct work_struct *work) 5896e84d644SOleg Nesterov { 5901f1f642eSOleg Nesterov return __cancel_work_timer(work, NULL); 591b89deed3SOleg Nesterov } 59228e53bddSOleg Nesterov EXPORT_SYMBOL_GPL(cancel_work_sync); 593b89deed3SOleg Nesterov 5946e84d644SOleg Nesterov /** 595f5a421a4SOleg Nesterov * cancel_delayed_work_sync - reliably kill off a delayed work. 5966e84d644SOleg Nesterov * @dwork: the delayed work struct 5976e84d644SOleg Nesterov * 5981f1f642eSOleg Nesterov * Returns true if @dwork was pending. 5991f1f642eSOleg Nesterov * 6006e84d644SOleg Nesterov * It is possible to use this function if @dwork rearms itself via queue_work() 6016e84d644SOleg Nesterov * or queue_delayed_work(). See also the comment for cancel_work_sync(). 6026e84d644SOleg Nesterov */ 6031f1f642eSOleg Nesterov int cancel_delayed_work_sync(struct delayed_work *dwork) 6046e84d644SOleg Nesterov { 6051f1f642eSOleg Nesterov return __cancel_work_timer(&dwork->work, &dwork->timer); 6066e84d644SOleg Nesterov } 607f5a421a4SOleg Nesterov EXPORT_SYMBOL(cancel_delayed_work_sync); 6081da177e4SLinus Torvalds 6096e84d644SOleg Nesterov static struct workqueue_struct *keventd_wq __read_mostly; 6101da177e4SLinus Torvalds 6110fcb78c2SRolf Eike Beer /** 6120fcb78c2SRolf Eike Beer * schedule_work - put work task in global workqueue 6130fcb78c2SRolf Eike Beer * @work: job to be done 6140fcb78c2SRolf Eike Beer * 6150fcb78c2SRolf Eike Beer * This puts a job in the kernel-global workqueue. 6160fcb78c2SRolf Eike Beer */ 6177ad5b3a5SHarvey Harrison int schedule_work(struct work_struct *work) 6181da177e4SLinus Torvalds { 6191da177e4SLinus Torvalds return queue_work(keventd_wq, work); 6201da177e4SLinus Torvalds } 621ae90dd5dSDave Jones EXPORT_SYMBOL(schedule_work); 6221da177e4SLinus Torvalds 623c1a220e7SZhang Rui /* 624c1a220e7SZhang Rui * schedule_work_on - put work task on a specific cpu 625c1a220e7SZhang Rui * @cpu: cpu to put the work task on 626c1a220e7SZhang Rui * @work: job to be done 627c1a220e7SZhang Rui * 628c1a220e7SZhang Rui * This puts a job on a specific cpu 629c1a220e7SZhang Rui */ 630c1a220e7SZhang Rui int schedule_work_on(int cpu, struct work_struct *work) 631c1a220e7SZhang Rui { 632c1a220e7SZhang Rui return queue_work_on(cpu, keventd_wq, work); 633c1a220e7SZhang Rui } 634c1a220e7SZhang Rui EXPORT_SYMBOL(schedule_work_on); 635c1a220e7SZhang Rui 6360fcb78c2SRolf Eike Beer /** 6370fcb78c2SRolf Eike Beer * schedule_delayed_work - put work task in global workqueue after delay 63852bad64dSDavid Howells * @dwork: job to be done 63952bad64dSDavid Howells * @delay: number of jiffies to wait or 0 for immediate execution 6400fcb78c2SRolf Eike Beer * 6410fcb78c2SRolf Eike Beer * After waiting for a given time this puts a job in the kernel-global 6420fcb78c2SRolf Eike Beer * workqueue. 6430fcb78c2SRolf Eike Beer */ 6447ad5b3a5SHarvey Harrison int schedule_delayed_work(struct delayed_work *dwork, 64582f67cd9SIngo Molnar unsigned long delay) 6461da177e4SLinus Torvalds { 64752bad64dSDavid Howells return queue_delayed_work(keventd_wq, dwork, delay); 6481da177e4SLinus Torvalds } 649ae90dd5dSDave Jones EXPORT_SYMBOL(schedule_delayed_work); 6501da177e4SLinus Torvalds 6510fcb78c2SRolf Eike Beer /** 6520fcb78c2SRolf Eike Beer * schedule_delayed_work_on - queue work in global workqueue on CPU after delay 6530fcb78c2SRolf Eike Beer * @cpu: cpu to use 65452bad64dSDavid Howells * @dwork: job to be done 6550fcb78c2SRolf Eike Beer * @delay: number of jiffies to wait 6560fcb78c2SRolf Eike Beer * 6570fcb78c2SRolf Eike Beer * After waiting for a given time this puts a job in the kernel-global 6580fcb78c2SRolf Eike Beer * workqueue on the specified CPU. 6590fcb78c2SRolf Eike Beer */ 6601da177e4SLinus Torvalds int schedule_delayed_work_on(int cpu, 66152bad64dSDavid Howells struct delayed_work *dwork, unsigned long delay) 6621da177e4SLinus Torvalds { 66352bad64dSDavid Howells return queue_delayed_work_on(cpu, keventd_wq, dwork, delay); 6641da177e4SLinus Torvalds } 665ae90dd5dSDave Jones EXPORT_SYMBOL(schedule_delayed_work_on); 6661da177e4SLinus Torvalds 667b6136773SAndrew Morton /** 668b6136773SAndrew Morton * schedule_on_each_cpu - call a function on each online CPU from keventd 669b6136773SAndrew Morton * @func: the function to call 670b6136773SAndrew Morton * 671b6136773SAndrew Morton * Returns zero on success. 672b6136773SAndrew Morton * Returns -ve errno on failure. 673b6136773SAndrew Morton * 674b6136773SAndrew Morton * schedule_on_each_cpu() is very slow. 675b6136773SAndrew Morton */ 67665f27f38SDavid Howells int schedule_on_each_cpu(work_func_t func) 67715316ba8SChristoph Lameter { 67815316ba8SChristoph Lameter int cpu; 679b6136773SAndrew Morton struct work_struct *works; 68015316ba8SChristoph Lameter 681b6136773SAndrew Morton works = alloc_percpu(struct work_struct); 682b6136773SAndrew Morton if (!works) 68315316ba8SChristoph Lameter return -ENOMEM; 684b6136773SAndrew Morton 68595402b38SGautham R Shenoy get_online_cpus(); 68615316ba8SChristoph Lameter for_each_online_cpu(cpu) { 6879bfb1839SIngo Molnar struct work_struct *work = per_cpu_ptr(works, cpu); 6889bfb1839SIngo Molnar 6899bfb1839SIngo Molnar INIT_WORK(work, func); 6909bfb1839SIngo Molnar set_bit(WORK_STRUCT_PENDING, work_data_bits(work)); 6919bfb1839SIngo Molnar __queue_work(per_cpu_ptr(keventd_wq->cpu_wq, cpu), work); 69215316ba8SChristoph Lameter } 69315316ba8SChristoph Lameter flush_workqueue(keventd_wq); 69495402b38SGautham R Shenoy put_online_cpus(); 695b6136773SAndrew Morton free_percpu(works); 69615316ba8SChristoph Lameter return 0; 69715316ba8SChristoph Lameter } 69815316ba8SChristoph Lameter 6991da177e4SLinus Torvalds void flush_scheduled_work(void) 7001da177e4SLinus Torvalds { 7011da177e4SLinus Torvalds flush_workqueue(keventd_wq); 7021da177e4SLinus Torvalds } 703ae90dd5dSDave Jones EXPORT_SYMBOL(flush_scheduled_work); 7041da177e4SLinus Torvalds 7051da177e4SLinus Torvalds /** 7061fa44ecaSJames Bottomley * execute_in_process_context - reliably execute the routine with user context 7071fa44ecaSJames Bottomley * @fn: the function to execute 7081fa44ecaSJames Bottomley * @ew: guaranteed storage for the execute work structure (must 7091fa44ecaSJames Bottomley * be available when the work executes) 7101fa44ecaSJames Bottomley * 7111fa44ecaSJames Bottomley * Executes the function immediately if process context is available, 7121fa44ecaSJames Bottomley * otherwise schedules the function for delayed execution. 7131fa44ecaSJames Bottomley * 7141fa44ecaSJames Bottomley * Returns: 0 - function was executed 7151fa44ecaSJames Bottomley * 1 - function was scheduled for execution 7161fa44ecaSJames Bottomley */ 71765f27f38SDavid Howells int execute_in_process_context(work_func_t fn, struct execute_work *ew) 7181fa44ecaSJames Bottomley { 7191fa44ecaSJames Bottomley if (!in_interrupt()) { 72065f27f38SDavid Howells fn(&ew->work); 7211fa44ecaSJames Bottomley return 0; 7221fa44ecaSJames Bottomley } 7231fa44ecaSJames Bottomley 72465f27f38SDavid Howells INIT_WORK(&ew->work, fn); 7251fa44ecaSJames Bottomley schedule_work(&ew->work); 7261fa44ecaSJames Bottomley 7271fa44ecaSJames Bottomley return 1; 7281fa44ecaSJames Bottomley } 7291fa44ecaSJames Bottomley EXPORT_SYMBOL_GPL(execute_in_process_context); 7301fa44ecaSJames Bottomley 7311da177e4SLinus Torvalds int keventd_up(void) 7321da177e4SLinus Torvalds { 7331da177e4SLinus Torvalds return keventd_wq != NULL; 7341da177e4SLinus Torvalds } 7351da177e4SLinus Torvalds 7361da177e4SLinus Torvalds int current_is_keventd(void) 7371da177e4SLinus Torvalds { 7381da177e4SLinus Torvalds struct cpu_workqueue_struct *cwq; 739d243769dSHugh Dickins int cpu = raw_smp_processor_id(); /* preempt-safe: keventd is per-cpu */ 7401da177e4SLinus Torvalds int ret = 0; 7411da177e4SLinus Torvalds 7421da177e4SLinus Torvalds BUG_ON(!keventd_wq); 7431da177e4SLinus Torvalds 74489ada679SChristoph Lameter cwq = per_cpu_ptr(keventd_wq->cpu_wq, cpu); 7451da177e4SLinus Torvalds if (current == cwq->thread) 7461da177e4SLinus Torvalds ret = 1; 7471da177e4SLinus Torvalds 7481da177e4SLinus Torvalds return ret; 7491da177e4SLinus Torvalds 7501da177e4SLinus Torvalds } 7511da177e4SLinus Torvalds 7523af24433SOleg Nesterov static struct cpu_workqueue_struct * 7533af24433SOleg Nesterov init_cpu_workqueue(struct workqueue_struct *wq, int cpu) 7541da177e4SLinus Torvalds { 75589ada679SChristoph Lameter struct cpu_workqueue_struct *cwq = per_cpu_ptr(wq->cpu_wq, cpu); 7563af24433SOleg Nesterov 7573af24433SOleg Nesterov cwq->wq = wq; 7583af24433SOleg Nesterov spin_lock_init(&cwq->lock); 7593af24433SOleg Nesterov INIT_LIST_HEAD(&cwq->worklist); 7603af24433SOleg Nesterov init_waitqueue_head(&cwq->more_work); 7613af24433SOleg Nesterov 7623af24433SOleg Nesterov return cwq; 7633af24433SOleg Nesterov } 7643af24433SOleg Nesterov 7653af24433SOleg Nesterov static int create_workqueue_thread(struct cpu_workqueue_struct *cwq, int cpu) 7663af24433SOleg Nesterov { 7673af24433SOleg Nesterov struct workqueue_struct *wq = cwq->wq; 7683af24433SOleg Nesterov const char *fmt = is_single_threaded(wq) ? "%s" : "%s/%d"; 7693af24433SOleg Nesterov struct task_struct *p; 7703af24433SOleg Nesterov 7713af24433SOleg Nesterov p = kthread_create(worker_thread, cwq, fmt, wq->name, cpu); 7723af24433SOleg Nesterov /* 7733af24433SOleg Nesterov * Nobody can add the work_struct to this cwq, 7743af24433SOleg Nesterov * if (caller is __create_workqueue) 7753af24433SOleg Nesterov * nobody should see this wq 7763af24433SOleg Nesterov * else // caller is CPU_UP_PREPARE 7773af24433SOleg Nesterov * cpu is not on cpu_online_map 7783af24433SOleg Nesterov * so we can abort safely. 7793af24433SOleg Nesterov */ 7803af24433SOleg Nesterov if (IS_ERR(p)) 7813af24433SOleg Nesterov return PTR_ERR(p); 7823af24433SOleg Nesterov 7833af24433SOleg Nesterov cwq->thread = p; 7843af24433SOleg Nesterov 7853af24433SOleg Nesterov return 0; 7863af24433SOleg Nesterov } 7873af24433SOleg Nesterov 78806ba38a9SOleg Nesterov static void start_workqueue_thread(struct cpu_workqueue_struct *cwq, int cpu) 78906ba38a9SOleg Nesterov { 79006ba38a9SOleg Nesterov struct task_struct *p = cwq->thread; 79106ba38a9SOleg Nesterov 79206ba38a9SOleg Nesterov if (p != NULL) { 79306ba38a9SOleg Nesterov if (cpu >= 0) 79406ba38a9SOleg Nesterov kthread_bind(p, cpu); 79506ba38a9SOleg Nesterov wake_up_process(p); 79606ba38a9SOleg Nesterov } 79706ba38a9SOleg Nesterov } 79806ba38a9SOleg Nesterov 7994e6045f1SJohannes Berg struct workqueue_struct *__create_workqueue_key(const char *name, 8004e6045f1SJohannes Berg int singlethread, 8014e6045f1SJohannes Berg int freezeable, 802eb13ba87SJohannes Berg struct lock_class_key *key, 803eb13ba87SJohannes Berg const char *lock_name) 8043af24433SOleg Nesterov { 8053af24433SOleg Nesterov struct workqueue_struct *wq; 8063af24433SOleg Nesterov struct cpu_workqueue_struct *cwq; 8073af24433SOleg Nesterov int err = 0, cpu; 8083af24433SOleg Nesterov 8093af24433SOleg Nesterov wq = kzalloc(sizeof(*wq), GFP_KERNEL); 8103af24433SOleg Nesterov if (!wq) 8113af24433SOleg Nesterov return NULL; 8123af24433SOleg Nesterov 8133af24433SOleg Nesterov wq->cpu_wq = alloc_percpu(struct cpu_workqueue_struct); 8143af24433SOleg Nesterov if (!wq->cpu_wq) { 8153af24433SOleg Nesterov kfree(wq); 8163af24433SOleg Nesterov return NULL; 8173af24433SOleg Nesterov } 8183af24433SOleg Nesterov 8193af24433SOleg Nesterov wq->name = name; 820eb13ba87SJohannes Berg lockdep_init_map(&wq->lockdep_map, lock_name, key, 0); 821cce1a165SOleg Nesterov wq->singlethread = singlethread; 8223af24433SOleg Nesterov wq->freezeable = freezeable; 823cce1a165SOleg Nesterov INIT_LIST_HEAD(&wq->list); 8243af24433SOleg Nesterov 8253af24433SOleg Nesterov if (singlethread) { 8263af24433SOleg Nesterov cwq = init_cpu_workqueue(wq, singlethread_cpu); 8273af24433SOleg Nesterov err = create_workqueue_thread(cwq, singlethread_cpu); 82806ba38a9SOleg Nesterov start_workqueue_thread(cwq, -1); 8293af24433SOleg Nesterov } else { 83095402b38SGautham R Shenoy get_online_cpus(); 83195402b38SGautham R Shenoy spin_lock(&workqueue_lock); 8323af24433SOleg Nesterov list_add(&wq->list, &workqueues); 83395402b38SGautham R Shenoy spin_unlock(&workqueue_lock); 8343af24433SOleg Nesterov 8353af24433SOleg Nesterov for_each_possible_cpu(cpu) { 8363af24433SOleg Nesterov cwq = init_cpu_workqueue(wq, cpu); 8373af24433SOleg Nesterov if (err || !cpu_online(cpu)) 8383af24433SOleg Nesterov continue; 8393af24433SOleg Nesterov err = create_workqueue_thread(cwq, cpu); 84006ba38a9SOleg Nesterov start_workqueue_thread(cwq, cpu); 8413af24433SOleg Nesterov } 84295402b38SGautham R Shenoy put_online_cpus(); 8433af24433SOleg Nesterov } 8443af24433SOleg Nesterov 8453af24433SOleg Nesterov if (err) { 8463af24433SOleg Nesterov destroy_workqueue(wq); 8473af24433SOleg Nesterov wq = NULL; 8483af24433SOleg Nesterov } 8493af24433SOleg Nesterov return wq; 8503af24433SOleg Nesterov } 8514e6045f1SJohannes Berg EXPORT_SYMBOL_GPL(__create_workqueue_key); 8523af24433SOleg Nesterov 8531e35eaa2SOleg Nesterov static void cleanup_workqueue_thread(struct cpu_workqueue_struct *cwq) 8543af24433SOleg Nesterov { 8553af24433SOleg Nesterov /* 85614441960SOleg Nesterov * Our caller is either destroy_workqueue() or CPU_DEAD, 85795402b38SGautham R Shenoy * get_online_cpus() protects cwq->thread. 8583af24433SOleg Nesterov */ 85914441960SOleg Nesterov if (cwq->thread == NULL) 86014441960SOleg Nesterov return; 86114441960SOleg Nesterov 8624e6045f1SJohannes Berg lock_acquire(&cwq->wq->lockdep_map, 0, 0, 0, 2, _THIS_IP_); 8634e6045f1SJohannes Berg lock_release(&cwq->wq->lockdep_map, 1, _THIS_IP_); 8644e6045f1SJohannes Berg 86513c22168SOleg Nesterov flush_cpu_workqueue(cwq); 86614441960SOleg Nesterov /* 86713c22168SOleg Nesterov * If the caller is CPU_DEAD and cwq->worklist was not empty, 86813c22168SOleg Nesterov * a concurrent flush_workqueue() can insert a barrier after us. 86913c22168SOleg Nesterov * However, in that case run_workqueue() won't return and check 87013c22168SOleg Nesterov * kthread_should_stop() until it flushes all work_struct's. 87114441960SOleg Nesterov * When ->worklist becomes empty it is safe to exit because no 87214441960SOleg Nesterov * more work_structs can be queued on this cwq: flush_workqueue 87314441960SOleg Nesterov * checks list_empty(), and a "normal" queue_work() can't use 87414441960SOleg Nesterov * a dead CPU. 87514441960SOleg Nesterov */ 87614441960SOleg Nesterov kthread_stop(cwq->thread); 87714441960SOleg Nesterov cwq->thread = NULL; 8781da177e4SLinus Torvalds } 8791da177e4SLinus Torvalds 8803af24433SOleg Nesterov /** 8813af24433SOleg Nesterov * destroy_workqueue - safely terminate a workqueue 8823af24433SOleg Nesterov * @wq: target workqueue 8833af24433SOleg Nesterov * 8843af24433SOleg Nesterov * Safely destroy a workqueue. All work currently pending will be done first. 8853af24433SOleg Nesterov */ 8863af24433SOleg Nesterov void destroy_workqueue(struct workqueue_struct *wq) 8873af24433SOleg Nesterov { 888b1f4ec17SOleg Nesterov const cpumask_t *cpu_map = wq_cpu_map(wq); 8893af24433SOleg Nesterov int cpu; 8903af24433SOleg Nesterov 89195402b38SGautham R Shenoy get_online_cpus(); 89295402b38SGautham R Shenoy spin_lock(&workqueue_lock); 8933af24433SOleg Nesterov list_del(&wq->list); 89495402b38SGautham R Shenoy spin_unlock(&workqueue_lock); 8953af24433SOleg Nesterov 896363ab6f1SMike Travis for_each_cpu_mask_nr(cpu, *cpu_map) 8971e35eaa2SOleg Nesterov cleanup_workqueue_thread(per_cpu_ptr(wq->cpu_wq, cpu)); 89800dfcaf7SOleg Nesterov put_online_cpus(); 8993af24433SOleg Nesterov 9003af24433SOleg Nesterov free_percpu(wq->cpu_wq); 9013af24433SOleg Nesterov kfree(wq); 9023af24433SOleg Nesterov } 9033af24433SOleg Nesterov EXPORT_SYMBOL_GPL(destroy_workqueue); 9043af24433SOleg Nesterov 9059c7b216dSChandra Seetharaman static int __devinit workqueue_cpu_callback(struct notifier_block *nfb, 9061da177e4SLinus Torvalds unsigned long action, 9071da177e4SLinus Torvalds void *hcpu) 9081da177e4SLinus Torvalds { 9093af24433SOleg Nesterov unsigned int cpu = (unsigned long)hcpu; 9103af24433SOleg Nesterov struct cpu_workqueue_struct *cwq; 9111da177e4SLinus Torvalds struct workqueue_struct *wq; 9121da177e4SLinus Torvalds 9138bb78442SRafael J. Wysocki action &= ~CPU_TASKS_FROZEN; 9148bb78442SRafael J. Wysocki 9151da177e4SLinus Torvalds switch (action) { 9163af24433SOleg Nesterov case CPU_UP_PREPARE: 9173af24433SOleg Nesterov cpu_set(cpu, cpu_populated_map); 9183af24433SOleg Nesterov } 9193af24433SOleg Nesterov 9201da177e4SLinus Torvalds list_for_each_entry(wq, &workqueues, list) { 9213af24433SOleg Nesterov cwq = per_cpu_ptr(wq->cpu_wq, cpu); 9223af24433SOleg Nesterov 9233af24433SOleg Nesterov switch (action) { 9243af24433SOleg Nesterov case CPU_UP_PREPARE: 9253af24433SOleg Nesterov if (!create_workqueue_thread(cwq, cpu)) 9261da177e4SLinus Torvalds break; 92795402b38SGautham R Shenoy printk(KERN_ERR "workqueue [%s] for %i failed\n", 92895402b38SGautham R Shenoy wq->name, cpu); 9293af24433SOleg Nesterov return NOTIFY_BAD; 9301da177e4SLinus Torvalds 9311da177e4SLinus Torvalds case CPU_ONLINE: 93206ba38a9SOleg Nesterov start_workqueue_thread(cwq, cpu); 9331da177e4SLinus Torvalds break; 9341da177e4SLinus Torvalds 9351da177e4SLinus Torvalds case CPU_UP_CANCELED: 93606ba38a9SOleg Nesterov start_workqueue_thread(cwq, -1); 9371da177e4SLinus Torvalds case CPU_DEAD: 9381e35eaa2SOleg Nesterov cleanup_workqueue_thread(cwq); 9391da177e4SLinus Torvalds break; 9401da177e4SLinus Torvalds } 9413af24433SOleg Nesterov } 9421da177e4SLinus Torvalds 94300dfcaf7SOleg Nesterov switch (action) { 94400dfcaf7SOleg Nesterov case CPU_UP_CANCELED: 94500dfcaf7SOleg Nesterov case CPU_DEAD: 94600dfcaf7SOleg Nesterov cpu_clear(cpu, cpu_populated_map); 94700dfcaf7SOleg Nesterov } 94800dfcaf7SOleg Nesterov 9491da177e4SLinus Torvalds return NOTIFY_OK; 9501da177e4SLinus Torvalds } 9511da177e4SLinus Torvalds 952c12920d1SOleg Nesterov void __init init_workqueues(void) 9531da177e4SLinus Torvalds { 9543af24433SOleg Nesterov cpu_populated_map = cpu_online_map; 955f756d5e2SNathan Lynch singlethread_cpu = first_cpu(cpu_possible_map); 956b1f4ec17SOleg Nesterov cpu_singlethread_map = cpumask_of_cpu(singlethread_cpu); 9571da177e4SLinus Torvalds hotcpu_notifier(workqueue_cpu_callback, 0); 9581da177e4SLinus Torvalds keventd_wq = create_workqueue("events"); 9591da177e4SLinus Torvalds BUG_ON(!keventd_wq); 9601da177e4SLinus Torvalds } 961