11da177e4SLinus Torvalds /* 21da177e4SLinus Torvalds * linux/kernel/workqueue.c 31da177e4SLinus Torvalds * 41da177e4SLinus Torvalds * Generic mechanism for defining kernel helper threads for running 51da177e4SLinus Torvalds * arbitrary tasks in process context. 61da177e4SLinus Torvalds * 71da177e4SLinus Torvalds * Started by Ingo Molnar, Copyright (C) 2002 81da177e4SLinus Torvalds * 91da177e4SLinus Torvalds * Derived from the taskqueue/keventd code by: 101da177e4SLinus Torvalds * 111da177e4SLinus Torvalds * David Woodhouse <dwmw2@infradead.org> 12e1f8e874SFrancois Cami * Andrew Morton 131da177e4SLinus Torvalds * Kai Petzke <wpp@marie.physik.tu-berlin.de> 141da177e4SLinus Torvalds * Theodore Ts'o <tytso@mit.edu> 1589ada679SChristoph Lameter * 16cde53535SChristoph Lameter * Made to use alloc_percpu by Christoph Lameter. 171da177e4SLinus Torvalds */ 181da177e4SLinus Torvalds 191da177e4SLinus Torvalds #include <linux/module.h> 201da177e4SLinus Torvalds #include <linux/kernel.h> 211da177e4SLinus Torvalds #include <linux/sched.h> 221da177e4SLinus Torvalds #include <linux/init.h> 231da177e4SLinus Torvalds #include <linux/signal.h> 241da177e4SLinus Torvalds #include <linux/completion.h> 251da177e4SLinus Torvalds #include <linux/workqueue.h> 261da177e4SLinus Torvalds #include <linux/slab.h> 271da177e4SLinus Torvalds #include <linux/cpu.h> 281da177e4SLinus Torvalds #include <linux/notifier.h> 291da177e4SLinus Torvalds #include <linux/kthread.h> 301fa44ecaSJames Bottomley #include <linux/hardirq.h> 3146934023SChristoph Lameter #include <linux/mempolicy.h> 32341a5958SRafael J. Wysocki #include <linux/freezer.h> 33d5abe669SPeter Zijlstra #include <linux/kallsyms.h> 34d5abe669SPeter Zijlstra #include <linux/debug_locks.h> 354e6045f1SJohannes Berg #include <linux/lockdep.h> 36c34056a3STejun Heo #include <linux/idr.h> 371da177e4SLinus Torvalds 381da177e4SLinus Torvalds /* 394690c4abSTejun Heo * Structure fields follow one of the following exclusion rules. 404690c4abSTejun Heo * 414690c4abSTejun Heo * I: Set during initialization and read-only afterwards. 424690c4abSTejun Heo * 434690c4abSTejun Heo * L: cwq->lock protected. Access with cwq->lock held. 444690c4abSTejun Heo * 4573f53c4aSTejun Heo * F: wq->flush_mutex protected. 4673f53c4aSTejun Heo * 474690c4abSTejun Heo * W: workqueue_lock protected. 484690c4abSTejun Heo */ 494690c4abSTejun Heo 50c34056a3STejun Heo struct cpu_workqueue_struct; 51c34056a3STejun Heo 52c34056a3STejun Heo struct worker { 53c34056a3STejun Heo struct work_struct *current_work; /* L: work being processed */ 54affee4b2STejun Heo struct list_head scheduled; /* L: scheduled works */ 55c34056a3STejun Heo struct task_struct *task; /* I: worker task */ 56c34056a3STejun Heo struct cpu_workqueue_struct *cwq; /* I: the associated cwq */ 57c34056a3STejun Heo int id; /* I: worker id */ 58c34056a3STejun Heo }; 59c34056a3STejun Heo 604690c4abSTejun Heo /* 61f756d5e2SNathan Lynch * The per-CPU workqueue (if single thread, we always use the first 620f900049STejun Heo * possible cpu). The lower WORK_STRUCT_FLAG_BITS of 630f900049STejun Heo * work_struct->data are used for flags and thus cwqs need to be 640f900049STejun Heo * aligned at two's power of the number of flag bits. 651da177e4SLinus Torvalds */ 661da177e4SLinus Torvalds struct cpu_workqueue_struct { 671da177e4SLinus Torvalds 681da177e4SLinus Torvalds spinlock_t lock; 691da177e4SLinus Torvalds 701da177e4SLinus Torvalds struct list_head worklist; 711da177e4SLinus Torvalds wait_queue_head_t more_work; 721537663fSTejun Heo unsigned int cpu; 73c34056a3STejun Heo struct worker *worker; 741da177e4SLinus Torvalds 754690c4abSTejun Heo struct workqueue_struct *wq; /* I: the owning workqueue */ 7673f53c4aSTejun Heo int work_color; /* L: current color */ 7773f53c4aSTejun Heo int flush_color; /* L: flushing color */ 7873f53c4aSTejun Heo int nr_in_flight[WORK_NR_COLORS]; 7973f53c4aSTejun Heo /* L: nr of in_flight works */ 801e19ffc6STejun Heo int nr_active; /* L: nr of active works */ 81a0a1a5fdSTejun Heo int max_active; /* L: max active works */ 821e19ffc6STejun Heo struct list_head delayed_works; /* L: delayed works */ 830f900049STejun Heo }; 841da177e4SLinus Torvalds 851da177e4SLinus Torvalds /* 8673f53c4aSTejun Heo * Structure used to wait for workqueue flush. 8773f53c4aSTejun Heo */ 8873f53c4aSTejun Heo struct wq_flusher { 8973f53c4aSTejun Heo struct list_head list; /* F: list of flushers */ 9073f53c4aSTejun Heo int flush_color; /* F: flush color waiting for */ 9173f53c4aSTejun Heo struct completion done; /* flush completion */ 9273f53c4aSTejun Heo }; 9373f53c4aSTejun Heo 9473f53c4aSTejun Heo /* 951da177e4SLinus Torvalds * The externally visible workqueue abstraction is an array of 961da177e4SLinus Torvalds * per-CPU workqueues: 971da177e4SLinus Torvalds */ 981da177e4SLinus Torvalds struct workqueue_struct { 9997e37d7bSTejun Heo unsigned int flags; /* I: WQ_* flags */ 1004690c4abSTejun Heo struct cpu_workqueue_struct *cpu_wq; /* I: cwq's */ 1014690c4abSTejun Heo struct list_head list; /* W: list of all workqueues */ 10273f53c4aSTejun Heo 10373f53c4aSTejun Heo struct mutex flush_mutex; /* protects wq flushing */ 10473f53c4aSTejun Heo int work_color; /* F: current work color */ 10573f53c4aSTejun Heo int flush_color; /* F: current flush color */ 10673f53c4aSTejun Heo atomic_t nr_cwqs_to_flush; /* flush in progress */ 10773f53c4aSTejun Heo struct wq_flusher *first_flusher; /* F: first flusher */ 10873f53c4aSTejun Heo struct list_head flusher_queue; /* F: flush waiters */ 10973f53c4aSTejun Heo struct list_head flusher_overflow; /* F: flush overflow list */ 11073f53c4aSTejun Heo 111a0a1a5fdSTejun Heo int saved_max_active; /* I: saved cwq max_active */ 1124690c4abSTejun Heo const char *name; /* I: workqueue name */ 1134e6045f1SJohannes Berg #ifdef CONFIG_LOCKDEP 1144e6045f1SJohannes Berg struct lockdep_map lockdep_map; 1154e6045f1SJohannes Berg #endif 1161da177e4SLinus Torvalds }; 1171da177e4SLinus Torvalds 118dc186ad7SThomas Gleixner #ifdef CONFIG_DEBUG_OBJECTS_WORK 119dc186ad7SThomas Gleixner 120dc186ad7SThomas Gleixner static struct debug_obj_descr work_debug_descr; 121dc186ad7SThomas Gleixner 122dc186ad7SThomas Gleixner /* 123dc186ad7SThomas Gleixner * fixup_init is called when: 124dc186ad7SThomas Gleixner * - an active object is initialized 125dc186ad7SThomas Gleixner */ 126dc186ad7SThomas Gleixner static int work_fixup_init(void *addr, enum debug_obj_state state) 127dc186ad7SThomas Gleixner { 128dc186ad7SThomas Gleixner struct work_struct *work = addr; 129dc186ad7SThomas Gleixner 130dc186ad7SThomas Gleixner switch (state) { 131dc186ad7SThomas Gleixner case ODEBUG_STATE_ACTIVE: 132dc186ad7SThomas Gleixner cancel_work_sync(work); 133dc186ad7SThomas Gleixner debug_object_init(work, &work_debug_descr); 134dc186ad7SThomas Gleixner return 1; 135dc186ad7SThomas Gleixner default: 136dc186ad7SThomas Gleixner return 0; 137dc186ad7SThomas Gleixner } 138dc186ad7SThomas Gleixner } 139dc186ad7SThomas Gleixner 140dc186ad7SThomas Gleixner /* 141dc186ad7SThomas Gleixner * fixup_activate is called when: 142dc186ad7SThomas Gleixner * - an active object is activated 143dc186ad7SThomas Gleixner * - an unknown object is activated (might be a statically initialized object) 144dc186ad7SThomas Gleixner */ 145dc186ad7SThomas Gleixner static int work_fixup_activate(void *addr, enum debug_obj_state state) 146dc186ad7SThomas Gleixner { 147dc186ad7SThomas Gleixner struct work_struct *work = addr; 148dc186ad7SThomas Gleixner 149dc186ad7SThomas Gleixner switch (state) { 150dc186ad7SThomas Gleixner 151dc186ad7SThomas Gleixner case ODEBUG_STATE_NOTAVAILABLE: 152dc186ad7SThomas Gleixner /* 153dc186ad7SThomas Gleixner * This is not really a fixup. The work struct was 154dc186ad7SThomas Gleixner * statically initialized. We just make sure that it 155dc186ad7SThomas Gleixner * is tracked in the object tracker. 156dc186ad7SThomas Gleixner */ 15722df02bbSTejun Heo if (test_bit(WORK_STRUCT_STATIC_BIT, work_data_bits(work))) { 158dc186ad7SThomas Gleixner debug_object_init(work, &work_debug_descr); 159dc186ad7SThomas Gleixner debug_object_activate(work, &work_debug_descr); 160dc186ad7SThomas Gleixner return 0; 161dc186ad7SThomas Gleixner } 162dc186ad7SThomas Gleixner WARN_ON_ONCE(1); 163dc186ad7SThomas Gleixner return 0; 164dc186ad7SThomas Gleixner 165dc186ad7SThomas Gleixner case ODEBUG_STATE_ACTIVE: 166dc186ad7SThomas Gleixner WARN_ON(1); 167dc186ad7SThomas Gleixner 168dc186ad7SThomas Gleixner default: 169dc186ad7SThomas Gleixner return 0; 170dc186ad7SThomas Gleixner } 171dc186ad7SThomas Gleixner } 172dc186ad7SThomas Gleixner 173dc186ad7SThomas Gleixner /* 174dc186ad7SThomas Gleixner * fixup_free is called when: 175dc186ad7SThomas Gleixner * - an active object is freed 176dc186ad7SThomas Gleixner */ 177dc186ad7SThomas Gleixner static int work_fixup_free(void *addr, enum debug_obj_state state) 178dc186ad7SThomas Gleixner { 179dc186ad7SThomas Gleixner struct work_struct *work = addr; 180dc186ad7SThomas Gleixner 181dc186ad7SThomas Gleixner switch (state) { 182dc186ad7SThomas Gleixner case ODEBUG_STATE_ACTIVE: 183dc186ad7SThomas Gleixner cancel_work_sync(work); 184dc186ad7SThomas Gleixner debug_object_free(work, &work_debug_descr); 185dc186ad7SThomas Gleixner return 1; 186dc186ad7SThomas Gleixner default: 187dc186ad7SThomas Gleixner return 0; 188dc186ad7SThomas Gleixner } 189dc186ad7SThomas Gleixner } 190dc186ad7SThomas Gleixner 191dc186ad7SThomas Gleixner static struct debug_obj_descr work_debug_descr = { 192dc186ad7SThomas Gleixner .name = "work_struct", 193dc186ad7SThomas Gleixner .fixup_init = work_fixup_init, 194dc186ad7SThomas Gleixner .fixup_activate = work_fixup_activate, 195dc186ad7SThomas Gleixner .fixup_free = work_fixup_free, 196dc186ad7SThomas Gleixner }; 197dc186ad7SThomas Gleixner 198dc186ad7SThomas Gleixner static inline void debug_work_activate(struct work_struct *work) 199dc186ad7SThomas Gleixner { 200dc186ad7SThomas Gleixner debug_object_activate(work, &work_debug_descr); 201dc186ad7SThomas Gleixner } 202dc186ad7SThomas Gleixner 203dc186ad7SThomas Gleixner static inline void debug_work_deactivate(struct work_struct *work) 204dc186ad7SThomas Gleixner { 205dc186ad7SThomas Gleixner debug_object_deactivate(work, &work_debug_descr); 206dc186ad7SThomas Gleixner } 207dc186ad7SThomas Gleixner 208dc186ad7SThomas Gleixner void __init_work(struct work_struct *work, int onstack) 209dc186ad7SThomas Gleixner { 210dc186ad7SThomas Gleixner if (onstack) 211dc186ad7SThomas Gleixner debug_object_init_on_stack(work, &work_debug_descr); 212dc186ad7SThomas Gleixner else 213dc186ad7SThomas Gleixner debug_object_init(work, &work_debug_descr); 214dc186ad7SThomas Gleixner } 215dc186ad7SThomas Gleixner EXPORT_SYMBOL_GPL(__init_work); 216dc186ad7SThomas Gleixner 217dc186ad7SThomas Gleixner void destroy_work_on_stack(struct work_struct *work) 218dc186ad7SThomas Gleixner { 219dc186ad7SThomas Gleixner debug_object_free(work, &work_debug_descr); 220dc186ad7SThomas Gleixner } 221dc186ad7SThomas Gleixner EXPORT_SYMBOL_GPL(destroy_work_on_stack); 222dc186ad7SThomas Gleixner 223dc186ad7SThomas Gleixner #else 224dc186ad7SThomas Gleixner static inline void debug_work_activate(struct work_struct *work) { } 225dc186ad7SThomas Gleixner static inline void debug_work_deactivate(struct work_struct *work) { } 226dc186ad7SThomas Gleixner #endif 227dc186ad7SThomas Gleixner 22895402b38SGautham R Shenoy /* Serializes the accesses to the list of workqueues. */ 22995402b38SGautham R Shenoy static DEFINE_SPINLOCK(workqueue_lock); 2301da177e4SLinus Torvalds static LIST_HEAD(workqueues); 231c34056a3STejun Heo static DEFINE_PER_CPU(struct ida, worker_ida); 232a0a1a5fdSTejun Heo static bool workqueue_freezing; /* W: have wqs started freezing? */ 233c34056a3STejun Heo 234c34056a3STejun Heo static int worker_thread(void *__worker); 2351da177e4SLinus Torvalds 2363af24433SOleg Nesterov static int singlethread_cpu __read_mostly; 237b1f4ec17SOleg Nesterov 2384690c4abSTejun Heo static struct cpu_workqueue_struct *get_cwq(unsigned int cpu, 2394690c4abSTejun Heo struct workqueue_struct *wq) 240a848e3b6SOleg Nesterov { 241a848e3b6SOleg Nesterov return per_cpu_ptr(wq->cpu_wq, cpu); 242a848e3b6SOleg Nesterov } 243a848e3b6SOleg Nesterov 2441537663fSTejun Heo static struct cpu_workqueue_struct *target_cwq(unsigned int cpu, 2451537663fSTejun Heo struct workqueue_struct *wq) 2461537663fSTejun Heo { 2471537663fSTejun Heo if (unlikely(wq->flags & WQ_SINGLE_THREAD)) 2481537663fSTejun Heo cpu = singlethread_cpu; 2491537663fSTejun Heo return get_cwq(cpu, wq); 2501537663fSTejun Heo } 2511537663fSTejun Heo 25273f53c4aSTejun Heo static unsigned int work_color_to_flags(int color) 25373f53c4aSTejun Heo { 25473f53c4aSTejun Heo return color << WORK_STRUCT_COLOR_SHIFT; 25573f53c4aSTejun Heo } 25673f53c4aSTejun Heo 25773f53c4aSTejun Heo static int get_work_color(struct work_struct *work) 25873f53c4aSTejun Heo { 25973f53c4aSTejun Heo return (*work_data_bits(work) >> WORK_STRUCT_COLOR_SHIFT) & 26073f53c4aSTejun Heo ((1 << WORK_STRUCT_COLOR_BITS) - 1); 26173f53c4aSTejun Heo } 26273f53c4aSTejun Heo 26373f53c4aSTejun Heo static int work_next_color(int color) 26473f53c4aSTejun Heo { 26573f53c4aSTejun Heo return (color + 1) % WORK_NR_COLORS; 26673f53c4aSTejun Heo } 26773f53c4aSTejun Heo 2684594bf15SDavid Howells /* 2694594bf15SDavid Howells * Set the workqueue on which a work item is to be run 2704594bf15SDavid Howells * - Must *only* be called if the pending flag is set 2714594bf15SDavid Howells */ 272ed7c0feeSOleg Nesterov static inline void set_wq_data(struct work_struct *work, 2734690c4abSTejun Heo struct cpu_workqueue_struct *cwq, 2744690c4abSTejun Heo unsigned long extra_flags) 275365970a1SDavid Howells { 2764594bf15SDavid Howells BUG_ON(!work_pending(work)); 2774594bf15SDavid Howells 2784690c4abSTejun Heo atomic_long_set(&work->data, (unsigned long)cwq | work_static(work) | 27922df02bbSTejun Heo WORK_STRUCT_PENDING | extra_flags); 280365970a1SDavid Howells } 281365970a1SDavid Howells 2824d707b9fSOleg Nesterov /* 2834d707b9fSOleg Nesterov * Clear WORK_STRUCT_PENDING and the workqueue on which it was queued. 2844d707b9fSOleg Nesterov */ 2854d707b9fSOleg Nesterov static inline void clear_wq_data(struct work_struct *work) 2864d707b9fSOleg Nesterov { 2874690c4abSTejun Heo atomic_long_set(&work->data, work_static(work)); 2884d707b9fSOleg Nesterov } 2894d707b9fSOleg Nesterov 29064166699STejun Heo static inline struct cpu_workqueue_struct *get_wq_data(struct work_struct *work) 291365970a1SDavid Howells { 29264166699STejun Heo return (void *)(atomic_long_read(&work->data) & 29364166699STejun Heo WORK_STRUCT_WQ_DATA_MASK); 294365970a1SDavid Howells } 295365970a1SDavid Howells 2964690c4abSTejun Heo /** 2974690c4abSTejun Heo * insert_work - insert a work into cwq 2984690c4abSTejun Heo * @cwq: cwq @work belongs to 2994690c4abSTejun Heo * @work: work to insert 3004690c4abSTejun Heo * @head: insertion point 3014690c4abSTejun Heo * @extra_flags: extra WORK_STRUCT_* flags to set 3024690c4abSTejun Heo * 3034690c4abSTejun Heo * Insert @work into @cwq after @head. 3044690c4abSTejun Heo * 3054690c4abSTejun Heo * CONTEXT: 3064690c4abSTejun Heo * spin_lock_irq(cwq->lock). 3074690c4abSTejun Heo */ 308b89deed3SOleg Nesterov static void insert_work(struct cpu_workqueue_struct *cwq, 3094690c4abSTejun Heo struct work_struct *work, struct list_head *head, 3104690c4abSTejun Heo unsigned int extra_flags) 311b89deed3SOleg Nesterov { 3124690c4abSTejun Heo /* we own @work, set data and link */ 3134690c4abSTejun Heo set_wq_data(work, cwq, extra_flags); 3144690c4abSTejun Heo 3156e84d644SOleg Nesterov /* 3166e84d644SOleg Nesterov * Ensure that we get the right work->data if we see the 3176e84d644SOleg Nesterov * result of list_add() below, see try_to_grab_pending(). 3186e84d644SOleg Nesterov */ 3196e84d644SOleg Nesterov smp_wmb(); 3204690c4abSTejun Heo 3211a4d9b0aSOleg Nesterov list_add_tail(&work->entry, head); 322b89deed3SOleg Nesterov wake_up(&cwq->more_work); 323b89deed3SOleg Nesterov } 324b89deed3SOleg Nesterov 3254690c4abSTejun Heo static void __queue_work(unsigned int cpu, struct workqueue_struct *wq, 3261da177e4SLinus Torvalds struct work_struct *work) 3271da177e4SLinus Torvalds { 3281537663fSTejun Heo struct cpu_workqueue_struct *cwq = target_cwq(cpu, wq); 3291e19ffc6STejun Heo struct list_head *worklist; 3301da177e4SLinus Torvalds unsigned long flags; 3311da177e4SLinus Torvalds 332dc186ad7SThomas Gleixner debug_work_activate(work); 3331e19ffc6STejun Heo 3341da177e4SLinus Torvalds spin_lock_irqsave(&cwq->lock, flags); 3354690c4abSTejun Heo BUG_ON(!list_empty(&work->entry)); 3361e19ffc6STejun Heo 33773f53c4aSTejun Heo cwq->nr_in_flight[cwq->work_color]++; 3381e19ffc6STejun Heo 3391e19ffc6STejun Heo if (likely(cwq->nr_active < cwq->max_active)) { 3401e19ffc6STejun Heo cwq->nr_active++; 3411e19ffc6STejun Heo worklist = &cwq->worklist; 3421e19ffc6STejun Heo } else 3431e19ffc6STejun Heo worklist = &cwq->delayed_works; 3441e19ffc6STejun Heo 3451e19ffc6STejun Heo insert_work(cwq, work, worklist, work_color_to_flags(cwq->work_color)); 3461e19ffc6STejun Heo 3471da177e4SLinus Torvalds spin_unlock_irqrestore(&cwq->lock, flags); 3481da177e4SLinus Torvalds } 3491da177e4SLinus Torvalds 3500fcb78c2SRolf Eike Beer /** 3510fcb78c2SRolf Eike Beer * queue_work - queue work on a workqueue 3520fcb78c2SRolf Eike Beer * @wq: workqueue to use 3530fcb78c2SRolf Eike Beer * @work: work to queue 3540fcb78c2SRolf Eike Beer * 355057647fcSAlan Stern * Returns 0 if @work was already on a queue, non-zero otherwise. 3561da177e4SLinus Torvalds * 35700dfcaf7SOleg Nesterov * We queue the work to the CPU on which it was submitted, but if the CPU dies 35800dfcaf7SOleg Nesterov * it can be processed by another CPU. 3591da177e4SLinus Torvalds */ 3607ad5b3a5SHarvey Harrison int queue_work(struct workqueue_struct *wq, struct work_struct *work) 3611da177e4SLinus Torvalds { 362ef1ca236SOleg Nesterov int ret; 3631da177e4SLinus Torvalds 364ef1ca236SOleg Nesterov ret = queue_work_on(get_cpu(), wq, work); 365a848e3b6SOleg Nesterov put_cpu(); 366ef1ca236SOleg Nesterov 3671da177e4SLinus Torvalds return ret; 3681da177e4SLinus Torvalds } 369ae90dd5dSDave Jones EXPORT_SYMBOL_GPL(queue_work); 3701da177e4SLinus Torvalds 371c1a220e7SZhang Rui /** 372c1a220e7SZhang Rui * queue_work_on - queue work on specific cpu 373c1a220e7SZhang Rui * @cpu: CPU number to execute work on 374c1a220e7SZhang Rui * @wq: workqueue to use 375c1a220e7SZhang Rui * @work: work to queue 376c1a220e7SZhang Rui * 377c1a220e7SZhang Rui * Returns 0 if @work was already on a queue, non-zero otherwise. 378c1a220e7SZhang Rui * 379c1a220e7SZhang Rui * We queue the work to a specific CPU, the caller must ensure it 380c1a220e7SZhang Rui * can't go away. 381c1a220e7SZhang Rui */ 382c1a220e7SZhang Rui int 383c1a220e7SZhang Rui queue_work_on(int cpu, struct workqueue_struct *wq, struct work_struct *work) 384c1a220e7SZhang Rui { 385c1a220e7SZhang Rui int ret = 0; 386c1a220e7SZhang Rui 38722df02bbSTejun Heo if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) { 3884690c4abSTejun Heo __queue_work(cpu, wq, work); 389c1a220e7SZhang Rui ret = 1; 390c1a220e7SZhang Rui } 391c1a220e7SZhang Rui return ret; 392c1a220e7SZhang Rui } 393c1a220e7SZhang Rui EXPORT_SYMBOL_GPL(queue_work_on); 394c1a220e7SZhang Rui 3956d141c3fSLi Zefan static void delayed_work_timer_fn(unsigned long __data) 3961da177e4SLinus Torvalds { 39752bad64dSDavid Howells struct delayed_work *dwork = (struct delayed_work *)__data; 398ed7c0feeSOleg Nesterov struct cpu_workqueue_struct *cwq = get_wq_data(&dwork->work); 3991da177e4SLinus Torvalds 4004690c4abSTejun Heo __queue_work(smp_processor_id(), cwq->wq, &dwork->work); 4011da177e4SLinus Torvalds } 4021da177e4SLinus Torvalds 4030fcb78c2SRolf Eike Beer /** 4040fcb78c2SRolf Eike Beer * queue_delayed_work - queue work on a workqueue after delay 4050fcb78c2SRolf Eike Beer * @wq: workqueue to use 406af9997e4SRandy Dunlap * @dwork: delayable work to queue 4070fcb78c2SRolf Eike Beer * @delay: number of jiffies to wait before queueing 4080fcb78c2SRolf Eike Beer * 409057647fcSAlan Stern * Returns 0 if @work was already on a queue, non-zero otherwise. 4100fcb78c2SRolf Eike Beer */ 4117ad5b3a5SHarvey Harrison int queue_delayed_work(struct workqueue_struct *wq, 41252bad64dSDavid Howells struct delayed_work *dwork, unsigned long delay) 4131da177e4SLinus Torvalds { 41452bad64dSDavid Howells if (delay == 0) 41563bc0362SOleg Nesterov return queue_work(wq, &dwork->work); 4161da177e4SLinus Torvalds 41763bc0362SOleg Nesterov return queue_delayed_work_on(-1, wq, dwork, delay); 4181da177e4SLinus Torvalds } 419ae90dd5dSDave Jones EXPORT_SYMBOL_GPL(queue_delayed_work); 4201da177e4SLinus Torvalds 4210fcb78c2SRolf Eike Beer /** 4220fcb78c2SRolf Eike Beer * queue_delayed_work_on - queue work on specific CPU after delay 4230fcb78c2SRolf Eike Beer * @cpu: CPU number to execute work on 4240fcb78c2SRolf Eike Beer * @wq: workqueue to use 425af9997e4SRandy Dunlap * @dwork: work to queue 4260fcb78c2SRolf Eike Beer * @delay: number of jiffies to wait before queueing 4270fcb78c2SRolf Eike Beer * 428057647fcSAlan Stern * Returns 0 if @work was already on a queue, non-zero otherwise. 4290fcb78c2SRolf Eike Beer */ 4307a6bc1cdSVenkatesh Pallipadi int queue_delayed_work_on(int cpu, struct workqueue_struct *wq, 43152bad64dSDavid Howells struct delayed_work *dwork, unsigned long delay) 4327a6bc1cdSVenkatesh Pallipadi { 4337a6bc1cdSVenkatesh Pallipadi int ret = 0; 43452bad64dSDavid Howells struct timer_list *timer = &dwork->timer; 43552bad64dSDavid Howells struct work_struct *work = &dwork->work; 4367a6bc1cdSVenkatesh Pallipadi 43722df02bbSTejun Heo if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) { 4387a6bc1cdSVenkatesh Pallipadi BUG_ON(timer_pending(timer)); 4397a6bc1cdSVenkatesh Pallipadi BUG_ON(!list_empty(&work->entry)); 4407a6bc1cdSVenkatesh Pallipadi 4418a3e77ccSAndrew Liu timer_stats_timer_set_start_info(&dwork->timer); 4428a3e77ccSAndrew Liu 443ed7c0feeSOleg Nesterov /* This stores cwq for the moment, for the timer_fn */ 4441537663fSTejun Heo set_wq_data(work, target_cwq(raw_smp_processor_id(), wq), 0); 4457a6bc1cdSVenkatesh Pallipadi timer->expires = jiffies + delay; 44652bad64dSDavid Howells timer->data = (unsigned long)dwork; 4477a6bc1cdSVenkatesh Pallipadi timer->function = delayed_work_timer_fn; 44863bc0362SOleg Nesterov 44963bc0362SOleg Nesterov if (unlikely(cpu >= 0)) 4507a6bc1cdSVenkatesh Pallipadi add_timer_on(timer, cpu); 45163bc0362SOleg Nesterov else 45263bc0362SOleg Nesterov add_timer(timer); 4537a6bc1cdSVenkatesh Pallipadi ret = 1; 4547a6bc1cdSVenkatesh Pallipadi } 4557a6bc1cdSVenkatesh Pallipadi return ret; 4567a6bc1cdSVenkatesh Pallipadi } 457ae90dd5dSDave Jones EXPORT_SYMBOL_GPL(queue_delayed_work_on); 4581da177e4SLinus Torvalds 459c34056a3STejun Heo static struct worker *alloc_worker(void) 460c34056a3STejun Heo { 461c34056a3STejun Heo struct worker *worker; 462c34056a3STejun Heo 463c34056a3STejun Heo worker = kzalloc(sizeof(*worker), GFP_KERNEL); 464affee4b2STejun Heo if (worker) 465affee4b2STejun Heo INIT_LIST_HEAD(&worker->scheduled); 466c34056a3STejun Heo return worker; 467c34056a3STejun Heo } 468c34056a3STejun Heo 469c34056a3STejun Heo /** 470c34056a3STejun Heo * create_worker - create a new workqueue worker 471c34056a3STejun Heo * @cwq: cwq the new worker will belong to 472c34056a3STejun Heo * @bind: whether to set affinity to @cpu or not 473c34056a3STejun Heo * 474c34056a3STejun Heo * Create a new worker which is bound to @cwq. The returned worker 475c34056a3STejun Heo * can be started by calling start_worker() or destroyed using 476c34056a3STejun Heo * destroy_worker(). 477c34056a3STejun Heo * 478c34056a3STejun Heo * CONTEXT: 479c34056a3STejun Heo * Might sleep. Does GFP_KERNEL allocations. 480c34056a3STejun Heo * 481c34056a3STejun Heo * RETURNS: 482c34056a3STejun Heo * Pointer to the newly created worker. 483c34056a3STejun Heo */ 484c34056a3STejun Heo static struct worker *create_worker(struct cpu_workqueue_struct *cwq, bool bind) 485c34056a3STejun Heo { 486c34056a3STejun Heo int id = -1; 487c34056a3STejun Heo struct worker *worker = NULL; 488c34056a3STejun Heo 489c34056a3STejun Heo spin_lock(&workqueue_lock); 490c34056a3STejun Heo while (ida_get_new(&per_cpu(worker_ida, cwq->cpu), &id)) { 491c34056a3STejun Heo spin_unlock(&workqueue_lock); 492c34056a3STejun Heo if (!ida_pre_get(&per_cpu(worker_ida, cwq->cpu), GFP_KERNEL)) 493c34056a3STejun Heo goto fail; 494c34056a3STejun Heo spin_lock(&workqueue_lock); 495c34056a3STejun Heo } 496c34056a3STejun Heo spin_unlock(&workqueue_lock); 497c34056a3STejun Heo 498c34056a3STejun Heo worker = alloc_worker(); 499c34056a3STejun Heo if (!worker) 500c34056a3STejun Heo goto fail; 501c34056a3STejun Heo 502c34056a3STejun Heo worker->cwq = cwq; 503c34056a3STejun Heo worker->id = id; 504c34056a3STejun Heo 505c34056a3STejun Heo worker->task = kthread_create(worker_thread, worker, "kworker/%u:%d", 506c34056a3STejun Heo cwq->cpu, id); 507c34056a3STejun Heo if (IS_ERR(worker->task)) 508c34056a3STejun Heo goto fail; 509c34056a3STejun Heo 510c34056a3STejun Heo if (bind) 511c34056a3STejun Heo kthread_bind(worker->task, cwq->cpu); 512c34056a3STejun Heo 513c34056a3STejun Heo return worker; 514c34056a3STejun Heo fail: 515c34056a3STejun Heo if (id >= 0) { 516c34056a3STejun Heo spin_lock(&workqueue_lock); 517c34056a3STejun Heo ida_remove(&per_cpu(worker_ida, cwq->cpu), id); 518c34056a3STejun Heo spin_unlock(&workqueue_lock); 519c34056a3STejun Heo } 520c34056a3STejun Heo kfree(worker); 521c34056a3STejun Heo return NULL; 522c34056a3STejun Heo } 523c34056a3STejun Heo 524c34056a3STejun Heo /** 525c34056a3STejun Heo * start_worker - start a newly created worker 526c34056a3STejun Heo * @worker: worker to start 527c34056a3STejun Heo * 528c34056a3STejun Heo * Start @worker. 529c34056a3STejun Heo * 530c34056a3STejun Heo * CONTEXT: 531c34056a3STejun Heo * spin_lock_irq(cwq->lock). 532c34056a3STejun Heo */ 533c34056a3STejun Heo static void start_worker(struct worker *worker) 534c34056a3STejun Heo { 535c34056a3STejun Heo wake_up_process(worker->task); 536c34056a3STejun Heo } 537c34056a3STejun Heo 538c34056a3STejun Heo /** 539c34056a3STejun Heo * destroy_worker - destroy a workqueue worker 540c34056a3STejun Heo * @worker: worker to be destroyed 541c34056a3STejun Heo * 542c34056a3STejun Heo * Destroy @worker. 543c34056a3STejun Heo */ 544c34056a3STejun Heo static void destroy_worker(struct worker *worker) 545c34056a3STejun Heo { 546c34056a3STejun Heo int cpu = worker->cwq->cpu; 547c34056a3STejun Heo int id = worker->id; 548c34056a3STejun Heo 549c34056a3STejun Heo /* sanity check frenzy */ 550c34056a3STejun Heo BUG_ON(worker->current_work); 551affee4b2STejun Heo BUG_ON(!list_empty(&worker->scheduled)); 552c34056a3STejun Heo 553c34056a3STejun Heo kthread_stop(worker->task); 554c34056a3STejun Heo kfree(worker); 555c34056a3STejun Heo 556c34056a3STejun Heo spin_lock(&workqueue_lock); 557c34056a3STejun Heo ida_remove(&per_cpu(worker_ida, cpu), id); 558c34056a3STejun Heo spin_unlock(&workqueue_lock); 559c34056a3STejun Heo } 560c34056a3STejun Heo 561a62428c0STejun Heo /** 562affee4b2STejun Heo * move_linked_works - move linked works to a list 563affee4b2STejun Heo * @work: start of series of works to be scheduled 564affee4b2STejun Heo * @head: target list to append @work to 565affee4b2STejun Heo * @nextp: out paramter for nested worklist walking 566affee4b2STejun Heo * 567affee4b2STejun Heo * Schedule linked works starting from @work to @head. Work series to 568affee4b2STejun Heo * be scheduled starts at @work and includes any consecutive work with 569affee4b2STejun Heo * WORK_STRUCT_LINKED set in its predecessor. 570affee4b2STejun Heo * 571affee4b2STejun Heo * If @nextp is not NULL, it's updated to point to the next work of 572affee4b2STejun Heo * the last scheduled work. This allows move_linked_works() to be 573affee4b2STejun Heo * nested inside outer list_for_each_entry_safe(). 574affee4b2STejun Heo * 575affee4b2STejun Heo * CONTEXT: 576affee4b2STejun Heo * spin_lock_irq(cwq->lock). 577affee4b2STejun Heo */ 578affee4b2STejun Heo static void move_linked_works(struct work_struct *work, struct list_head *head, 579affee4b2STejun Heo struct work_struct **nextp) 580affee4b2STejun Heo { 581affee4b2STejun Heo struct work_struct *n; 582affee4b2STejun Heo 583affee4b2STejun Heo /* 584affee4b2STejun Heo * Linked worklist will always end before the end of the list, 585affee4b2STejun Heo * use NULL for list head. 586affee4b2STejun Heo */ 587affee4b2STejun Heo list_for_each_entry_safe_from(work, n, NULL, entry) { 588affee4b2STejun Heo list_move_tail(&work->entry, head); 589affee4b2STejun Heo if (!(*work_data_bits(work) & WORK_STRUCT_LINKED)) 590affee4b2STejun Heo break; 591affee4b2STejun Heo } 592affee4b2STejun Heo 593affee4b2STejun Heo /* 594affee4b2STejun Heo * If we're already inside safe list traversal and have moved 595affee4b2STejun Heo * multiple works to the scheduled queue, the next position 596affee4b2STejun Heo * needs to be updated. 597affee4b2STejun Heo */ 598affee4b2STejun Heo if (nextp) 599affee4b2STejun Heo *nextp = n; 600affee4b2STejun Heo } 601affee4b2STejun Heo 6021e19ffc6STejun Heo static void cwq_activate_first_delayed(struct cpu_workqueue_struct *cwq) 6031e19ffc6STejun Heo { 6041e19ffc6STejun Heo struct work_struct *work = list_first_entry(&cwq->delayed_works, 6051e19ffc6STejun Heo struct work_struct, entry); 6061e19ffc6STejun Heo 6071e19ffc6STejun Heo move_linked_works(work, &cwq->worklist, NULL); 6081e19ffc6STejun Heo cwq->nr_active++; 6091e19ffc6STejun Heo } 6101e19ffc6STejun Heo 611affee4b2STejun Heo /** 61273f53c4aSTejun Heo * cwq_dec_nr_in_flight - decrement cwq's nr_in_flight 61373f53c4aSTejun Heo * @cwq: cwq of interest 61473f53c4aSTejun Heo * @color: color of work which left the queue 61573f53c4aSTejun Heo * 61673f53c4aSTejun Heo * A work either has completed or is removed from pending queue, 61773f53c4aSTejun Heo * decrement nr_in_flight of its cwq and handle workqueue flushing. 61873f53c4aSTejun Heo * 61973f53c4aSTejun Heo * CONTEXT: 62073f53c4aSTejun Heo * spin_lock_irq(cwq->lock). 62173f53c4aSTejun Heo */ 62273f53c4aSTejun Heo static void cwq_dec_nr_in_flight(struct cpu_workqueue_struct *cwq, int color) 62373f53c4aSTejun Heo { 62473f53c4aSTejun Heo /* ignore uncolored works */ 62573f53c4aSTejun Heo if (color == WORK_NO_COLOR) 62673f53c4aSTejun Heo return; 62773f53c4aSTejun Heo 62873f53c4aSTejun Heo cwq->nr_in_flight[color]--; 6291e19ffc6STejun Heo cwq->nr_active--; 6301e19ffc6STejun Heo 6311e19ffc6STejun Heo /* one down, submit a delayed one */ 6321e19ffc6STejun Heo if (!list_empty(&cwq->delayed_works) && 6331e19ffc6STejun Heo cwq->nr_active < cwq->max_active) 6341e19ffc6STejun Heo cwq_activate_first_delayed(cwq); 63573f53c4aSTejun Heo 63673f53c4aSTejun Heo /* is flush in progress and are we at the flushing tip? */ 63773f53c4aSTejun Heo if (likely(cwq->flush_color != color)) 63873f53c4aSTejun Heo return; 63973f53c4aSTejun Heo 64073f53c4aSTejun Heo /* are there still in-flight works? */ 64173f53c4aSTejun Heo if (cwq->nr_in_flight[color]) 64273f53c4aSTejun Heo return; 64373f53c4aSTejun Heo 64473f53c4aSTejun Heo /* this cwq is done, clear flush_color */ 64573f53c4aSTejun Heo cwq->flush_color = -1; 64673f53c4aSTejun Heo 64773f53c4aSTejun Heo /* 64873f53c4aSTejun Heo * If this was the last cwq, wake up the first flusher. It 64973f53c4aSTejun Heo * will handle the rest. 65073f53c4aSTejun Heo */ 65173f53c4aSTejun Heo if (atomic_dec_and_test(&cwq->wq->nr_cwqs_to_flush)) 65273f53c4aSTejun Heo complete(&cwq->wq->first_flusher->done); 65373f53c4aSTejun Heo } 65473f53c4aSTejun Heo 65573f53c4aSTejun Heo /** 656a62428c0STejun Heo * process_one_work - process single work 657c34056a3STejun Heo * @worker: self 658a62428c0STejun Heo * @work: work to process 659a62428c0STejun Heo * 660a62428c0STejun Heo * Process @work. This function contains all the logics necessary to 661a62428c0STejun Heo * process a single work including synchronization against and 662a62428c0STejun Heo * interaction with other workers on the same cpu, queueing and 663a62428c0STejun Heo * flushing. As long as context requirement is met, any worker can 664a62428c0STejun Heo * call this function to process a work. 665a62428c0STejun Heo * 666a62428c0STejun Heo * CONTEXT: 667a62428c0STejun Heo * spin_lock_irq(cwq->lock) which is released and regrabbed. 668a62428c0STejun Heo */ 669c34056a3STejun Heo static void process_one_work(struct worker *worker, struct work_struct *work) 6701da177e4SLinus Torvalds { 671c34056a3STejun Heo struct cpu_workqueue_struct *cwq = worker->cwq; 6726bb49e59SDavid Howells work_func_t f = work->func; 67373f53c4aSTejun Heo int work_color; 6744e6045f1SJohannes Berg #ifdef CONFIG_LOCKDEP 6754e6045f1SJohannes Berg /* 676a62428c0STejun Heo * It is permissible to free the struct work_struct from 677a62428c0STejun Heo * inside the function that is called from it, this we need to 678a62428c0STejun Heo * take into account for lockdep too. To avoid bogus "held 679a62428c0STejun Heo * lock freed" warnings as well as problems when looking into 680a62428c0STejun Heo * work->lockdep_map, make a copy and use that here. 6814e6045f1SJohannes Berg */ 6824e6045f1SJohannes Berg struct lockdep_map lockdep_map = work->lockdep_map; 6834e6045f1SJohannes Berg #endif 684a62428c0STejun Heo /* claim and process */ 685dc186ad7SThomas Gleixner debug_work_deactivate(work); 686c34056a3STejun Heo worker->current_work = work; 68773f53c4aSTejun Heo work_color = get_work_color(work); 688a62428c0STejun Heo list_del_init(&work->entry); 689a62428c0STejun Heo 690f293ea92SOleg Nesterov spin_unlock_irq(&cwq->lock); 6911da177e4SLinus Torvalds 692365970a1SDavid Howells BUG_ON(get_wq_data(work) != cwq); 69323b2e599SOleg Nesterov work_clear_pending(work); 6943295f0efSIngo Molnar lock_map_acquire(&cwq->wq->lockdep_map); 6953295f0efSIngo Molnar lock_map_acquire(&lockdep_map); 69665f27f38SDavid Howells f(work); 6973295f0efSIngo Molnar lock_map_release(&lockdep_map); 6983295f0efSIngo Molnar lock_map_release(&cwq->wq->lockdep_map); 6991da177e4SLinus Torvalds 700d5abe669SPeter Zijlstra if (unlikely(in_atomic() || lockdep_depth(current) > 0)) { 701d5abe669SPeter Zijlstra printk(KERN_ERR "BUG: workqueue leaked lock or atomic: " 702d5abe669SPeter Zijlstra "%s/0x%08x/%d\n", 703a62428c0STejun Heo current->comm, preempt_count(), task_pid_nr(current)); 704d5abe669SPeter Zijlstra printk(KERN_ERR " last function: "); 705d5abe669SPeter Zijlstra print_symbol("%s\n", (unsigned long)f); 706d5abe669SPeter Zijlstra debug_show_held_locks(current); 707d5abe669SPeter Zijlstra dump_stack(); 708d5abe669SPeter Zijlstra } 709d5abe669SPeter Zijlstra 710f293ea92SOleg Nesterov spin_lock_irq(&cwq->lock); 711a62428c0STejun Heo 712a62428c0STejun Heo /* we're done with it, release */ 713c34056a3STejun Heo worker->current_work = NULL; 71473f53c4aSTejun Heo cwq_dec_nr_in_flight(cwq, work_color); 7151da177e4SLinus Torvalds } 716a62428c0STejun Heo 717affee4b2STejun Heo /** 718affee4b2STejun Heo * process_scheduled_works - process scheduled works 719affee4b2STejun Heo * @worker: self 720affee4b2STejun Heo * 721affee4b2STejun Heo * Process all scheduled works. Please note that the scheduled list 722affee4b2STejun Heo * may change while processing a work, so this function repeatedly 723affee4b2STejun Heo * fetches a work from the top and executes it. 724affee4b2STejun Heo * 725affee4b2STejun Heo * CONTEXT: 726affee4b2STejun Heo * spin_lock_irq(cwq->lock) which may be released and regrabbed 727affee4b2STejun Heo * multiple times. 728affee4b2STejun Heo */ 729affee4b2STejun Heo static void process_scheduled_works(struct worker *worker) 730a62428c0STejun Heo { 731affee4b2STejun Heo while (!list_empty(&worker->scheduled)) { 732affee4b2STejun Heo struct work_struct *work = list_first_entry(&worker->scheduled, 733a62428c0STejun Heo struct work_struct, entry); 734c34056a3STejun Heo process_one_work(worker, work); 735a62428c0STejun Heo } 7361da177e4SLinus Torvalds } 7371da177e4SLinus Torvalds 7384690c4abSTejun Heo /** 7394690c4abSTejun Heo * worker_thread - the worker thread function 740c34056a3STejun Heo * @__worker: self 7414690c4abSTejun Heo * 7424690c4abSTejun Heo * The cwq worker thread function. 7434690c4abSTejun Heo */ 744c34056a3STejun Heo static int worker_thread(void *__worker) 7451da177e4SLinus Torvalds { 746c34056a3STejun Heo struct worker *worker = __worker; 747c34056a3STejun Heo struct cpu_workqueue_struct *cwq = worker->cwq; 7483af24433SOleg Nesterov DEFINE_WAIT(wait); 7491da177e4SLinus Torvalds 7503af24433SOleg Nesterov for (;;) { 7513af24433SOleg Nesterov prepare_to_wait(&cwq->more_work, &wait, TASK_INTERRUPTIBLE); 752a0a1a5fdSTejun Heo if (!kthread_should_stop() && 75314441960SOleg Nesterov list_empty(&cwq->worklist)) 7541da177e4SLinus Torvalds schedule(); 7553af24433SOleg Nesterov finish_wait(&cwq->more_work, &wait); 7561da177e4SLinus Torvalds 75714441960SOleg Nesterov if (kthread_should_stop()) 7583af24433SOleg Nesterov break; 7593af24433SOleg Nesterov 760c34056a3STejun Heo if (unlikely(!cpumask_equal(&worker->task->cpus_allowed, 7611537663fSTejun Heo get_cpu_mask(cwq->cpu)))) 762c34056a3STejun Heo set_cpus_allowed_ptr(worker->task, 7631537663fSTejun Heo get_cpu_mask(cwq->cpu)); 764affee4b2STejun Heo 765affee4b2STejun Heo spin_lock_irq(&cwq->lock); 766affee4b2STejun Heo 767affee4b2STejun Heo while (!list_empty(&cwq->worklist)) { 768affee4b2STejun Heo struct work_struct *work = 769affee4b2STejun Heo list_first_entry(&cwq->worklist, 770affee4b2STejun Heo struct work_struct, entry); 771affee4b2STejun Heo 772affee4b2STejun Heo if (likely(!(*work_data_bits(work) & 773affee4b2STejun Heo WORK_STRUCT_LINKED))) { 774affee4b2STejun Heo /* optimization path, not strictly necessary */ 775affee4b2STejun Heo process_one_work(worker, work); 776affee4b2STejun Heo if (unlikely(!list_empty(&worker->scheduled))) 777affee4b2STejun Heo process_scheduled_works(worker); 778affee4b2STejun Heo } else { 779affee4b2STejun Heo move_linked_works(work, &worker->scheduled, 780affee4b2STejun Heo NULL); 781affee4b2STejun Heo process_scheduled_works(worker); 782affee4b2STejun Heo } 783affee4b2STejun Heo } 784affee4b2STejun Heo 785affee4b2STejun Heo spin_unlock_irq(&cwq->lock); 7861da177e4SLinus Torvalds } 7873af24433SOleg Nesterov 7881da177e4SLinus Torvalds return 0; 7891da177e4SLinus Torvalds } 7901da177e4SLinus Torvalds 791fc2e4d70SOleg Nesterov struct wq_barrier { 792fc2e4d70SOleg Nesterov struct work_struct work; 793fc2e4d70SOleg Nesterov struct completion done; 794fc2e4d70SOleg Nesterov }; 795fc2e4d70SOleg Nesterov 796fc2e4d70SOleg Nesterov static void wq_barrier_func(struct work_struct *work) 797fc2e4d70SOleg Nesterov { 798fc2e4d70SOleg Nesterov struct wq_barrier *barr = container_of(work, struct wq_barrier, work); 799fc2e4d70SOleg Nesterov complete(&barr->done); 800fc2e4d70SOleg Nesterov } 801fc2e4d70SOleg Nesterov 8024690c4abSTejun Heo /** 8034690c4abSTejun Heo * insert_wq_barrier - insert a barrier work 8044690c4abSTejun Heo * @cwq: cwq to insert barrier into 8054690c4abSTejun Heo * @barr: wq_barrier to insert 806affee4b2STejun Heo * @target: target work to attach @barr to 807affee4b2STejun Heo * @worker: worker currently executing @target, NULL if @target is not executing 8084690c4abSTejun Heo * 809affee4b2STejun Heo * @barr is linked to @target such that @barr is completed only after 810affee4b2STejun Heo * @target finishes execution. Please note that the ordering 811affee4b2STejun Heo * guarantee is observed only with respect to @target and on the local 812affee4b2STejun Heo * cpu. 813affee4b2STejun Heo * 814affee4b2STejun Heo * Currently, a queued barrier can't be canceled. This is because 815affee4b2STejun Heo * try_to_grab_pending() can't determine whether the work to be 816affee4b2STejun Heo * grabbed is at the head of the queue and thus can't clear LINKED 817affee4b2STejun Heo * flag of the previous work while there must be a valid next work 818affee4b2STejun Heo * after a work with LINKED flag set. 819affee4b2STejun Heo * 820affee4b2STejun Heo * Note that when @worker is non-NULL, @target may be modified 821affee4b2STejun Heo * underneath us, so we can't reliably determine cwq from @target. 8224690c4abSTejun Heo * 8234690c4abSTejun Heo * CONTEXT: 8244690c4abSTejun Heo * spin_lock_irq(cwq->lock). 8254690c4abSTejun Heo */ 82683c22520SOleg Nesterov static void insert_wq_barrier(struct cpu_workqueue_struct *cwq, 827affee4b2STejun Heo struct wq_barrier *barr, 828affee4b2STejun Heo struct work_struct *target, struct worker *worker) 829fc2e4d70SOleg Nesterov { 830affee4b2STejun Heo struct list_head *head; 831affee4b2STejun Heo unsigned int linked = 0; 832affee4b2STejun Heo 833dc186ad7SThomas Gleixner /* 834dc186ad7SThomas Gleixner * debugobject calls are safe here even with cwq->lock locked 835dc186ad7SThomas Gleixner * as we know for sure that this will not trigger any of the 836dc186ad7SThomas Gleixner * checks and call back into the fixup functions where we 837dc186ad7SThomas Gleixner * might deadlock. 838dc186ad7SThomas Gleixner */ 839dc186ad7SThomas Gleixner INIT_WORK_ON_STACK(&barr->work, wq_barrier_func); 84022df02bbSTejun Heo __set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(&barr->work)); 841fc2e4d70SOleg Nesterov init_completion(&barr->done); 84283c22520SOleg Nesterov 843affee4b2STejun Heo /* 844affee4b2STejun Heo * If @target is currently being executed, schedule the 845affee4b2STejun Heo * barrier to the worker; otherwise, put it after @target. 846affee4b2STejun Heo */ 847affee4b2STejun Heo if (worker) 848affee4b2STejun Heo head = worker->scheduled.next; 849affee4b2STejun Heo else { 850affee4b2STejun Heo unsigned long *bits = work_data_bits(target); 851affee4b2STejun Heo 852affee4b2STejun Heo head = target->entry.next; 853affee4b2STejun Heo /* there can already be other linked works, inherit and set */ 854affee4b2STejun Heo linked = *bits & WORK_STRUCT_LINKED; 855affee4b2STejun Heo __set_bit(WORK_STRUCT_LINKED_BIT, bits); 856affee4b2STejun Heo } 857affee4b2STejun Heo 858dc186ad7SThomas Gleixner debug_work_activate(&barr->work); 859affee4b2STejun Heo insert_work(cwq, &barr->work, head, 860affee4b2STejun Heo work_color_to_flags(WORK_NO_COLOR) | linked); 861fc2e4d70SOleg Nesterov } 862fc2e4d70SOleg Nesterov 86373f53c4aSTejun Heo /** 86473f53c4aSTejun Heo * flush_workqueue_prep_cwqs - prepare cwqs for workqueue flushing 86573f53c4aSTejun Heo * @wq: workqueue being flushed 86673f53c4aSTejun Heo * @flush_color: new flush color, < 0 for no-op 86773f53c4aSTejun Heo * @work_color: new work color, < 0 for no-op 86873f53c4aSTejun Heo * 86973f53c4aSTejun Heo * Prepare cwqs for workqueue flushing. 87073f53c4aSTejun Heo * 87173f53c4aSTejun Heo * If @flush_color is non-negative, flush_color on all cwqs should be 87273f53c4aSTejun Heo * -1. If no cwq has in-flight commands at the specified color, all 87373f53c4aSTejun Heo * cwq->flush_color's stay at -1 and %false is returned. If any cwq 87473f53c4aSTejun Heo * has in flight commands, its cwq->flush_color is set to 87573f53c4aSTejun Heo * @flush_color, @wq->nr_cwqs_to_flush is updated accordingly, cwq 87673f53c4aSTejun Heo * wakeup logic is armed and %true is returned. 87773f53c4aSTejun Heo * 87873f53c4aSTejun Heo * The caller should have initialized @wq->first_flusher prior to 87973f53c4aSTejun Heo * calling this function with non-negative @flush_color. If 88073f53c4aSTejun Heo * @flush_color is negative, no flush color update is done and %false 88173f53c4aSTejun Heo * is returned. 88273f53c4aSTejun Heo * 88373f53c4aSTejun Heo * If @work_color is non-negative, all cwqs should have the same 88473f53c4aSTejun Heo * work_color which is previous to @work_color and all will be 88573f53c4aSTejun Heo * advanced to @work_color. 88673f53c4aSTejun Heo * 88773f53c4aSTejun Heo * CONTEXT: 88873f53c4aSTejun Heo * mutex_lock(wq->flush_mutex). 88973f53c4aSTejun Heo * 89073f53c4aSTejun Heo * RETURNS: 89173f53c4aSTejun Heo * %true if @flush_color >= 0 and there's something to flush. %false 89273f53c4aSTejun Heo * otherwise. 89373f53c4aSTejun Heo */ 89473f53c4aSTejun Heo static bool flush_workqueue_prep_cwqs(struct workqueue_struct *wq, 89573f53c4aSTejun Heo int flush_color, int work_color) 8961da177e4SLinus Torvalds { 89773f53c4aSTejun Heo bool wait = false; 89873f53c4aSTejun Heo unsigned int cpu; 8991da177e4SLinus Torvalds 90073f53c4aSTejun Heo if (flush_color >= 0) { 90173f53c4aSTejun Heo BUG_ON(atomic_read(&wq->nr_cwqs_to_flush)); 90273f53c4aSTejun Heo atomic_set(&wq->nr_cwqs_to_flush, 1); 90373f53c4aSTejun Heo } 90473f53c4aSTejun Heo 90573f53c4aSTejun Heo for_each_possible_cpu(cpu) { 90673f53c4aSTejun Heo struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq); 9072355b70fSLai Jiangshan 90883c22520SOleg Nesterov spin_lock_irq(&cwq->lock); 90973f53c4aSTejun Heo 91073f53c4aSTejun Heo if (flush_color >= 0) { 91173f53c4aSTejun Heo BUG_ON(cwq->flush_color != -1); 91273f53c4aSTejun Heo 91373f53c4aSTejun Heo if (cwq->nr_in_flight[flush_color]) { 91473f53c4aSTejun Heo cwq->flush_color = flush_color; 91573f53c4aSTejun Heo atomic_inc(&wq->nr_cwqs_to_flush); 91673f53c4aSTejun Heo wait = true; 91783c22520SOleg Nesterov } 91873f53c4aSTejun Heo } 91973f53c4aSTejun Heo 92073f53c4aSTejun Heo if (work_color >= 0) { 92173f53c4aSTejun Heo BUG_ON(work_color != work_next_color(cwq->work_color)); 92273f53c4aSTejun Heo cwq->work_color = work_color; 92373f53c4aSTejun Heo } 92473f53c4aSTejun Heo 92583c22520SOleg Nesterov spin_unlock_irq(&cwq->lock); 926dc186ad7SThomas Gleixner } 92714441960SOleg Nesterov 92873f53c4aSTejun Heo if (flush_color >= 0 && atomic_dec_and_test(&wq->nr_cwqs_to_flush)) 92973f53c4aSTejun Heo complete(&wq->first_flusher->done); 93073f53c4aSTejun Heo 93173f53c4aSTejun Heo return wait; 93283c22520SOleg Nesterov } 9331da177e4SLinus Torvalds 9340fcb78c2SRolf Eike Beer /** 9351da177e4SLinus Torvalds * flush_workqueue - ensure that any scheduled work has run to completion. 9360fcb78c2SRolf Eike Beer * @wq: workqueue to flush 9371da177e4SLinus Torvalds * 9381da177e4SLinus Torvalds * Forces execution of the workqueue and blocks until its completion. 9391da177e4SLinus Torvalds * This is typically used in driver shutdown handlers. 9401da177e4SLinus Torvalds * 941fc2e4d70SOleg Nesterov * We sleep until all works which were queued on entry have been handled, 942fc2e4d70SOleg Nesterov * but we are not livelocked by new incoming ones. 9431da177e4SLinus Torvalds */ 9447ad5b3a5SHarvey Harrison void flush_workqueue(struct workqueue_struct *wq) 9451da177e4SLinus Torvalds { 94673f53c4aSTejun Heo struct wq_flusher this_flusher = { 94773f53c4aSTejun Heo .list = LIST_HEAD_INIT(this_flusher.list), 94873f53c4aSTejun Heo .flush_color = -1, 94973f53c4aSTejun Heo .done = COMPLETION_INITIALIZER_ONSTACK(this_flusher.done), 95073f53c4aSTejun Heo }; 95173f53c4aSTejun Heo int next_color; 952b1f4ec17SOleg Nesterov 9533295f0efSIngo Molnar lock_map_acquire(&wq->lockdep_map); 9543295f0efSIngo Molnar lock_map_release(&wq->lockdep_map); 95573f53c4aSTejun Heo 95673f53c4aSTejun Heo mutex_lock(&wq->flush_mutex); 95773f53c4aSTejun Heo 95873f53c4aSTejun Heo /* 95973f53c4aSTejun Heo * Start-to-wait phase 96073f53c4aSTejun Heo */ 96173f53c4aSTejun Heo next_color = work_next_color(wq->work_color); 96273f53c4aSTejun Heo 96373f53c4aSTejun Heo if (next_color != wq->flush_color) { 96473f53c4aSTejun Heo /* 96573f53c4aSTejun Heo * Color space is not full. The current work_color 96673f53c4aSTejun Heo * becomes our flush_color and work_color is advanced 96773f53c4aSTejun Heo * by one. 96873f53c4aSTejun Heo */ 96973f53c4aSTejun Heo BUG_ON(!list_empty(&wq->flusher_overflow)); 97073f53c4aSTejun Heo this_flusher.flush_color = wq->work_color; 97173f53c4aSTejun Heo wq->work_color = next_color; 97273f53c4aSTejun Heo 97373f53c4aSTejun Heo if (!wq->first_flusher) { 97473f53c4aSTejun Heo /* no flush in progress, become the first flusher */ 97573f53c4aSTejun Heo BUG_ON(wq->flush_color != this_flusher.flush_color); 97673f53c4aSTejun Heo 97773f53c4aSTejun Heo wq->first_flusher = &this_flusher; 97873f53c4aSTejun Heo 97973f53c4aSTejun Heo if (!flush_workqueue_prep_cwqs(wq, wq->flush_color, 98073f53c4aSTejun Heo wq->work_color)) { 98173f53c4aSTejun Heo /* nothing to flush, done */ 98273f53c4aSTejun Heo wq->flush_color = next_color; 98373f53c4aSTejun Heo wq->first_flusher = NULL; 98473f53c4aSTejun Heo goto out_unlock; 98573f53c4aSTejun Heo } 98673f53c4aSTejun Heo } else { 98773f53c4aSTejun Heo /* wait in queue */ 98873f53c4aSTejun Heo BUG_ON(wq->flush_color == this_flusher.flush_color); 98973f53c4aSTejun Heo list_add_tail(&this_flusher.list, &wq->flusher_queue); 99073f53c4aSTejun Heo flush_workqueue_prep_cwqs(wq, -1, wq->work_color); 99173f53c4aSTejun Heo } 99273f53c4aSTejun Heo } else { 99373f53c4aSTejun Heo /* 99473f53c4aSTejun Heo * Oops, color space is full, wait on overflow queue. 99573f53c4aSTejun Heo * The next flush completion will assign us 99673f53c4aSTejun Heo * flush_color and transfer to flusher_queue. 99773f53c4aSTejun Heo */ 99873f53c4aSTejun Heo list_add_tail(&this_flusher.list, &wq->flusher_overflow); 99973f53c4aSTejun Heo } 100073f53c4aSTejun Heo 100173f53c4aSTejun Heo mutex_unlock(&wq->flush_mutex); 100273f53c4aSTejun Heo 100373f53c4aSTejun Heo wait_for_completion(&this_flusher.done); 100473f53c4aSTejun Heo 100573f53c4aSTejun Heo /* 100673f53c4aSTejun Heo * Wake-up-and-cascade phase 100773f53c4aSTejun Heo * 100873f53c4aSTejun Heo * First flushers are responsible for cascading flushes and 100973f53c4aSTejun Heo * handling overflow. Non-first flushers can simply return. 101073f53c4aSTejun Heo */ 101173f53c4aSTejun Heo if (wq->first_flusher != &this_flusher) 101273f53c4aSTejun Heo return; 101373f53c4aSTejun Heo 101473f53c4aSTejun Heo mutex_lock(&wq->flush_mutex); 101573f53c4aSTejun Heo 101673f53c4aSTejun Heo wq->first_flusher = NULL; 101773f53c4aSTejun Heo 101873f53c4aSTejun Heo BUG_ON(!list_empty(&this_flusher.list)); 101973f53c4aSTejun Heo BUG_ON(wq->flush_color != this_flusher.flush_color); 102073f53c4aSTejun Heo 102173f53c4aSTejun Heo while (true) { 102273f53c4aSTejun Heo struct wq_flusher *next, *tmp; 102373f53c4aSTejun Heo 102473f53c4aSTejun Heo /* complete all the flushers sharing the current flush color */ 102573f53c4aSTejun Heo list_for_each_entry_safe(next, tmp, &wq->flusher_queue, list) { 102673f53c4aSTejun Heo if (next->flush_color != wq->flush_color) 102773f53c4aSTejun Heo break; 102873f53c4aSTejun Heo list_del_init(&next->list); 102973f53c4aSTejun Heo complete(&next->done); 103073f53c4aSTejun Heo } 103173f53c4aSTejun Heo 103273f53c4aSTejun Heo BUG_ON(!list_empty(&wq->flusher_overflow) && 103373f53c4aSTejun Heo wq->flush_color != work_next_color(wq->work_color)); 103473f53c4aSTejun Heo 103573f53c4aSTejun Heo /* this flush_color is finished, advance by one */ 103673f53c4aSTejun Heo wq->flush_color = work_next_color(wq->flush_color); 103773f53c4aSTejun Heo 103873f53c4aSTejun Heo /* one color has been freed, handle overflow queue */ 103973f53c4aSTejun Heo if (!list_empty(&wq->flusher_overflow)) { 104073f53c4aSTejun Heo /* 104173f53c4aSTejun Heo * Assign the same color to all overflowed 104273f53c4aSTejun Heo * flushers, advance work_color and append to 104373f53c4aSTejun Heo * flusher_queue. This is the start-to-wait 104473f53c4aSTejun Heo * phase for these overflowed flushers. 104573f53c4aSTejun Heo */ 104673f53c4aSTejun Heo list_for_each_entry(tmp, &wq->flusher_overflow, list) 104773f53c4aSTejun Heo tmp->flush_color = wq->work_color; 104873f53c4aSTejun Heo 104973f53c4aSTejun Heo wq->work_color = work_next_color(wq->work_color); 105073f53c4aSTejun Heo 105173f53c4aSTejun Heo list_splice_tail_init(&wq->flusher_overflow, 105273f53c4aSTejun Heo &wq->flusher_queue); 105373f53c4aSTejun Heo flush_workqueue_prep_cwqs(wq, -1, wq->work_color); 105473f53c4aSTejun Heo } 105573f53c4aSTejun Heo 105673f53c4aSTejun Heo if (list_empty(&wq->flusher_queue)) { 105773f53c4aSTejun Heo BUG_ON(wq->flush_color != wq->work_color); 105873f53c4aSTejun Heo break; 105973f53c4aSTejun Heo } 106073f53c4aSTejun Heo 106173f53c4aSTejun Heo /* 106273f53c4aSTejun Heo * Need to flush more colors. Make the next flusher 106373f53c4aSTejun Heo * the new first flusher and arm cwqs. 106473f53c4aSTejun Heo */ 106573f53c4aSTejun Heo BUG_ON(wq->flush_color == wq->work_color); 106673f53c4aSTejun Heo BUG_ON(wq->flush_color != next->flush_color); 106773f53c4aSTejun Heo 106873f53c4aSTejun Heo list_del_init(&next->list); 106973f53c4aSTejun Heo wq->first_flusher = next; 107073f53c4aSTejun Heo 107173f53c4aSTejun Heo if (flush_workqueue_prep_cwqs(wq, wq->flush_color, -1)) 107273f53c4aSTejun Heo break; 107373f53c4aSTejun Heo 107473f53c4aSTejun Heo /* 107573f53c4aSTejun Heo * Meh... this color is already done, clear first 107673f53c4aSTejun Heo * flusher and repeat cascading. 107773f53c4aSTejun Heo */ 107873f53c4aSTejun Heo wq->first_flusher = NULL; 107973f53c4aSTejun Heo } 108073f53c4aSTejun Heo 108173f53c4aSTejun Heo out_unlock: 108273f53c4aSTejun Heo mutex_unlock(&wq->flush_mutex); 10831da177e4SLinus Torvalds } 1084ae90dd5dSDave Jones EXPORT_SYMBOL_GPL(flush_workqueue); 10851da177e4SLinus Torvalds 1086db700897SOleg Nesterov /** 1087db700897SOleg Nesterov * flush_work - block until a work_struct's callback has terminated 1088db700897SOleg Nesterov * @work: the work which is to be flushed 1089db700897SOleg Nesterov * 1090a67da70dSOleg Nesterov * Returns false if @work has already terminated. 1091a67da70dSOleg Nesterov * 1092db700897SOleg Nesterov * It is expected that, prior to calling flush_work(), the caller has 1093db700897SOleg Nesterov * arranged for the work to not be requeued, otherwise it doesn't make 1094db700897SOleg Nesterov * sense to use this function. 1095db700897SOleg Nesterov */ 1096db700897SOleg Nesterov int flush_work(struct work_struct *work) 1097db700897SOleg Nesterov { 1098affee4b2STejun Heo struct worker *worker = NULL; 1099db700897SOleg Nesterov struct cpu_workqueue_struct *cwq; 1100db700897SOleg Nesterov struct wq_barrier barr; 1101db700897SOleg Nesterov 1102db700897SOleg Nesterov might_sleep(); 1103db700897SOleg Nesterov cwq = get_wq_data(work); 1104db700897SOleg Nesterov if (!cwq) 1105db700897SOleg Nesterov return 0; 1106db700897SOleg Nesterov 11073295f0efSIngo Molnar lock_map_acquire(&cwq->wq->lockdep_map); 11083295f0efSIngo Molnar lock_map_release(&cwq->wq->lockdep_map); 1109a67da70dSOleg Nesterov 1110db700897SOleg Nesterov spin_lock_irq(&cwq->lock); 1111db700897SOleg Nesterov if (!list_empty(&work->entry)) { 1112db700897SOleg Nesterov /* 1113db700897SOleg Nesterov * See the comment near try_to_grab_pending()->smp_rmb(). 1114db700897SOleg Nesterov * If it was re-queued under us we are not going to wait. 1115db700897SOleg Nesterov */ 1116db700897SOleg Nesterov smp_rmb(); 1117db700897SOleg Nesterov if (unlikely(cwq != get_wq_data(work))) 11184690c4abSTejun Heo goto already_gone; 1119db700897SOleg Nesterov } else { 1120affee4b2STejun Heo if (cwq->worker && cwq->worker->current_work == work) 1121affee4b2STejun Heo worker = cwq->worker; 1122affee4b2STejun Heo if (!worker) 11234690c4abSTejun Heo goto already_gone; 1124db700897SOleg Nesterov } 1125db700897SOleg Nesterov 1126affee4b2STejun Heo insert_wq_barrier(cwq, &barr, work, worker); 11274690c4abSTejun Heo spin_unlock_irq(&cwq->lock); 1128db700897SOleg Nesterov wait_for_completion(&barr.done); 1129dc186ad7SThomas Gleixner destroy_work_on_stack(&barr.work); 1130db700897SOleg Nesterov return 1; 11314690c4abSTejun Heo already_gone: 11324690c4abSTejun Heo spin_unlock_irq(&cwq->lock); 11334690c4abSTejun Heo return 0; 1134db700897SOleg Nesterov } 1135db700897SOleg Nesterov EXPORT_SYMBOL_GPL(flush_work); 1136db700897SOleg Nesterov 11376e84d644SOleg Nesterov /* 11381f1f642eSOleg Nesterov * Upon a successful return (>= 0), the caller "owns" WORK_STRUCT_PENDING bit, 11396e84d644SOleg Nesterov * so this work can't be re-armed in any way. 11406e84d644SOleg Nesterov */ 11416e84d644SOleg Nesterov static int try_to_grab_pending(struct work_struct *work) 11426e84d644SOleg Nesterov { 11436e84d644SOleg Nesterov struct cpu_workqueue_struct *cwq; 11441f1f642eSOleg Nesterov int ret = -1; 11456e84d644SOleg Nesterov 114622df02bbSTejun Heo if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) 11471f1f642eSOleg Nesterov return 0; 11486e84d644SOleg Nesterov 11496e84d644SOleg Nesterov /* 11506e84d644SOleg Nesterov * The queueing is in progress, or it is already queued. Try to 11516e84d644SOleg Nesterov * steal it from ->worklist without clearing WORK_STRUCT_PENDING. 11526e84d644SOleg Nesterov */ 11536e84d644SOleg Nesterov 11546e84d644SOleg Nesterov cwq = get_wq_data(work); 11556e84d644SOleg Nesterov if (!cwq) 11566e84d644SOleg Nesterov return ret; 11576e84d644SOleg Nesterov 11586e84d644SOleg Nesterov spin_lock_irq(&cwq->lock); 11596e84d644SOleg Nesterov if (!list_empty(&work->entry)) { 11606e84d644SOleg Nesterov /* 11616e84d644SOleg Nesterov * This work is queued, but perhaps we locked the wrong cwq. 11626e84d644SOleg Nesterov * In that case we must see the new value after rmb(), see 11636e84d644SOleg Nesterov * insert_work()->wmb(). 11646e84d644SOleg Nesterov */ 11656e84d644SOleg Nesterov smp_rmb(); 11666e84d644SOleg Nesterov if (cwq == get_wq_data(work)) { 1167dc186ad7SThomas Gleixner debug_work_deactivate(work); 11686e84d644SOleg Nesterov list_del_init(&work->entry); 116973f53c4aSTejun Heo cwq_dec_nr_in_flight(cwq, get_work_color(work)); 11706e84d644SOleg Nesterov ret = 1; 11716e84d644SOleg Nesterov } 11726e84d644SOleg Nesterov } 11736e84d644SOleg Nesterov spin_unlock_irq(&cwq->lock); 11746e84d644SOleg Nesterov 11756e84d644SOleg Nesterov return ret; 11766e84d644SOleg Nesterov } 11776e84d644SOleg Nesterov 11786e84d644SOleg Nesterov static void wait_on_cpu_work(struct cpu_workqueue_struct *cwq, 1179b89deed3SOleg Nesterov struct work_struct *work) 1180b89deed3SOleg Nesterov { 1181b89deed3SOleg Nesterov struct wq_barrier barr; 1182affee4b2STejun Heo struct worker *worker; 1183b89deed3SOleg Nesterov 1184b89deed3SOleg Nesterov spin_lock_irq(&cwq->lock); 1185affee4b2STejun Heo 1186affee4b2STejun Heo worker = NULL; 1187c34056a3STejun Heo if (unlikely(cwq->worker && cwq->worker->current_work == work)) { 1188affee4b2STejun Heo worker = cwq->worker; 1189affee4b2STejun Heo insert_wq_barrier(cwq, &barr, work, worker); 1190b89deed3SOleg Nesterov } 1191affee4b2STejun Heo 1192b89deed3SOleg Nesterov spin_unlock_irq(&cwq->lock); 1193b89deed3SOleg Nesterov 1194affee4b2STejun Heo if (unlikely(worker)) { 1195b89deed3SOleg Nesterov wait_for_completion(&barr.done); 1196dc186ad7SThomas Gleixner destroy_work_on_stack(&barr.work); 1197dc186ad7SThomas Gleixner } 1198b89deed3SOleg Nesterov } 1199b89deed3SOleg Nesterov 12006e84d644SOleg Nesterov static void wait_on_work(struct work_struct *work) 1201b89deed3SOleg Nesterov { 1202b89deed3SOleg Nesterov struct cpu_workqueue_struct *cwq; 120328e53bddSOleg Nesterov struct workqueue_struct *wq; 1204b1f4ec17SOleg Nesterov int cpu; 1205b89deed3SOleg Nesterov 1206f293ea92SOleg Nesterov might_sleep(); 1207f293ea92SOleg Nesterov 12083295f0efSIngo Molnar lock_map_acquire(&work->lockdep_map); 12093295f0efSIngo Molnar lock_map_release(&work->lockdep_map); 12104e6045f1SJohannes Berg 1211b89deed3SOleg Nesterov cwq = get_wq_data(work); 1212b89deed3SOleg Nesterov if (!cwq) 12133af24433SOleg Nesterov return; 1214b89deed3SOleg Nesterov 121528e53bddSOleg Nesterov wq = cwq->wq; 121628e53bddSOleg Nesterov 12171537663fSTejun Heo for_each_possible_cpu(cpu) 12184690c4abSTejun Heo wait_on_cpu_work(get_cwq(cpu, wq), work); 12196e84d644SOleg Nesterov } 12206e84d644SOleg Nesterov 12211f1f642eSOleg Nesterov static int __cancel_work_timer(struct work_struct *work, 12221f1f642eSOleg Nesterov struct timer_list* timer) 12231f1f642eSOleg Nesterov { 12241f1f642eSOleg Nesterov int ret; 12251f1f642eSOleg Nesterov 12261f1f642eSOleg Nesterov do { 12271f1f642eSOleg Nesterov ret = (timer && likely(del_timer(timer))); 12281f1f642eSOleg Nesterov if (!ret) 12291f1f642eSOleg Nesterov ret = try_to_grab_pending(work); 12301f1f642eSOleg Nesterov wait_on_work(work); 12311f1f642eSOleg Nesterov } while (unlikely(ret < 0)); 12321f1f642eSOleg Nesterov 12334d707b9fSOleg Nesterov clear_wq_data(work); 12341f1f642eSOleg Nesterov return ret; 12351f1f642eSOleg Nesterov } 12361f1f642eSOleg Nesterov 12376e84d644SOleg Nesterov /** 12386e84d644SOleg Nesterov * cancel_work_sync - block until a work_struct's callback has terminated 12396e84d644SOleg Nesterov * @work: the work which is to be flushed 12406e84d644SOleg Nesterov * 12411f1f642eSOleg Nesterov * Returns true if @work was pending. 12421f1f642eSOleg Nesterov * 12436e84d644SOleg Nesterov * cancel_work_sync() will cancel the work if it is queued. If the work's 12446e84d644SOleg Nesterov * callback appears to be running, cancel_work_sync() will block until it 12456e84d644SOleg Nesterov * has completed. 12466e84d644SOleg Nesterov * 12476e84d644SOleg Nesterov * It is possible to use this function if the work re-queues itself. It can 12486e84d644SOleg Nesterov * cancel the work even if it migrates to another workqueue, however in that 12496e84d644SOleg Nesterov * case it only guarantees that work->func() has completed on the last queued 12506e84d644SOleg Nesterov * workqueue. 12516e84d644SOleg Nesterov * 12526e84d644SOleg Nesterov * cancel_work_sync(&delayed_work->work) should be used only if ->timer is not 12536e84d644SOleg Nesterov * pending, otherwise it goes into a busy-wait loop until the timer expires. 12546e84d644SOleg Nesterov * 12556e84d644SOleg Nesterov * The caller must ensure that workqueue_struct on which this work was last 12566e84d644SOleg Nesterov * queued can't be destroyed before this function returns. 12576e84d644SOleg Nesterov */ 12581f1f642eSOleg Nesterov int cancel_work_sync(struct work_struct *work) 12596e84d644SOleg Nesterov { 12601f1f642eSOleg Nesterov return __cancel_work_timer(work, NULL); 1261b89deed3SOleg Nesterov } 126228e53bddSOleg Nesterov EXPORT_SYMBOL_GPL(cancel_work_sync); 1263b89deed3SOleg Nesterov 12646e84d644SOleg Nesterov /** 1265f5a421a4SOleg Nesterov * cancel_delayed_work_sync - reliably kill off a delayed work. 12666e84d644SOleg Nesterov * @dwork: the delayed work struct 12676e84d644SOleg Nesterov * 12681f1f642eSOleg Nesterov * Returns true if @dwork was pending. 12691f1f642eSOleg Nesterov * 12706e84d644SOleg Nesterov * It is possible to use this function if @dwork rearms itself via queue_work() 12716e84d644SOleg Nesterov * or queue_delayed_work(). See also the comment for cancel_work_sync(). 12726e84d644SOleg Nesterov */ 12731f1f642eSOleg Nesterov int cancel_delayed_work_sync(struct delayed_work *dwork) 12746e84d644SOleg Nesterov { 12751f1f642eSOleg Nesterov return __cancel_work_timer(&dwork->work, &dwork->timer); 12766e84d644SOleg Nesterov } 1277f5a421a4SOleg Nesterov EXPORT_SYMBOL(cancel_delayed_work_sync); 12781da177e4SLinus Torvalds 12796e84d644SOleg Nesterov static struct workqueue_struct *keventd_wq __read_mostly; 12801da177e4SLinus Torvalds 12810fcb78c2SRolf Eike Beer /** 12820fcb78c2SRolf Eike Beer * schedule_work - put work task in global workqueue 12830fcb78c2SRolf Eike Beer * @work: job to be done 12840fcb78c2SRolf Eike Beer * 12855b0f437dSBart Van Assche * Returns zero if @work was already on the kernel-global workqueue and 12865b0f437dSBart Van Assche * non-zero otherwise. 12875b0f437dSBart Van Assche * 12885b0f437dSBart Van Assche * This puts a job in the kernel-global workqueue if it was not already 12895b0f437dSBart Van Assche * queued and leaves it in the same position on the kernel-global 12905b0f437dSBart Van Assche * workqueue otherwise. 12910fcb78c2SRolf Eike Beer */ 12927ad5b3a5SHarvey Harrison int schedule_work(struct work_struct *work) 12931da177e4SLinus Torvalds { 12941da177e4SLinus Torvalds return queue_work(keventd_wq, work); 12951da177e4SLinus Torvalds } 1296ae90dd5dSDave Jones EXPORT_SYMBOL(schedule_work); 12971da177e4SLinus Torvalds 1298c1a220e7SZhang Rui /* 1299c1a220e7SZhang Rui * schedule_work_on - put work task on a specific cpu 1300c1a220e7SZhang Rui * @cpu: cpu to put the work task on 1301c1a220e7SZhang Rui * @work: job to be done 1302c1a220e7SZhang Rui * 1303c1a220e7SZhang Rui * This puts a job on a specific cpu 1304c1a220e7SZhang Rui */ 1305c1a220e7SZhang Rui int schedule_work_on(int cpu, struct work_struct *work) 1306c1a220e7SZhang Rui { 1307c1a220e7SZhang Rui return queue_work_on(cpu, keventd_wq, work); 1308c1a220e7SZhang Rui } 1309c1a220e7SZhang Rui EXPORT_SYMBOL(schedule_work_on); 1310c1a220e7SZhang Rui 13110fcb78c2SRolf Eike Beer /** 13120fcb78c2SRolf Eike Beer * schedule_delayed_work - put work task in global workqueue after delay 131352bad64dSDavid Howells * @dwork: job to be done 131452bad64dSDavid Howells * @delay: number of jiffies to wait or 0 for immediate execution 13150fcb78c2SRolf Eike Beer * 13160fcb78c2SRolf Eike Beer * After waiting for a given time this puts a job in the kernel-global 13170fcb78c2SRolf Eike Beer * workqueue. 13180fcb78c2SRolf Eike Beer */ 13197ad5b3a5SHarvey Harrison int schedule_delayed_work(struct delayed_work *dwork, 132082f67cd9SIngo Molnar unsigned long delay) 13211da177e4SLinus Torvalds { 132252bad64dSDavid Howells return queue_delayed_work(keventd_wq, dwork, delay); 13231da177e4SLinus Torvalds } 1324ae90dd5dSDave Jones EXPORT_SYMBOL(schedule_delayed_work); 13251da177e4SLinus Torvalds 13260fcb78c2SRolf Eike Beer /** 13278c53e463SLinus Torvalds * flush_delayed_work - block until a dwork_struct's callback has terminated 13288c53e463SLinus Torvalds * @dwork: the delayed work which is to be flushed 13298c53e463SLinus Torvalds * 13308c53e463SLinus Torvalds * Any timeout is cancelled, and any pending work is run immediately. 13318c53e463SLinus Torvalds */ 13328c53e463SLinus Torvalds void flush_delayed_work(struct delayed_work *dwork) 13338c53e463SLinus Torvalds { 13348c53e463SLinus Torvalds if (del_timer_sync(&dwork->timer)) { 13354690c4abSTejun Heo __queue_work(get_cpu(), get_wq_data(&dwork->work)->wq, 13364690c4abSTejun Heo &dwork->work); 13378c53e463SLinus Torvalds put_cpu(); 13388c53e463SLinus Torvalds } 13398c53e463SLinus Torvalds flush_work(&dwork->work); 13408c53e463SLinus Torvalds } 13418c53e463SLinus Torvalds EXPORT_SYMBOL(flush_delayed_work); 13428c53e463SLinus Torvalds 13438c53e463SLinus Torvalds /** 13440fcb78c2SRolf Eike Beer * schedule_delayed_work_on - queue work in global workqueue on CPU after delay 13450fcb78c2SRolf Eike Beer * @cpu: cpu to use 134652bad64dSDavid Howells * @dwork: job to be done 13470fcb78c2SRolf Eike Beer * @delay: number of jiffies to wait 13480fcb78c2SRolf Eike Beer * 13490fcb78c2SRolf Eike Beer * After waiting for a given time this puts a job in the kernel-global 13500fcb78c2SRolf Eike Beer * workqueue on the specified CPU. 13510fcb78c2SRolf Eike Beer */ 13521da177e4SLinus Torvalds int schedule_delayed_work_on(int cpu, 135352bad64dSDavid Howells struct delayed_work *dwork, unsigned long delay) 13541da177e4SLinus Torvalds { 135552bad64dSDavid Howells return queue_delayed_work_on(cpu, keventd_wq, dwork, delay); 13561da177e4SLinus Torvalds } 1357ae90dd5dSDave Jones EXPORT_SYMBOL(schedule_delayed_work_on); 13581da177e4SLinus Torvalds 1359b6136773SAndrew Morton /** 1360b6136773SAndrew Morton * schedule_on_each_cpu - call a function on each online CPU from keventd 1361b6136773SAndrew Morton * @func: the function to call 1362b6136773SAndrew Morton * 1363b6136773SAndrew Morton * Returns zero on success. 1364b6136773SAndrew Morton * Returns -ve errno on failure. 1365b6136773SAndrew Morton * 1366b6136773SAndrew Morton * schedule_on_each_cpu() is very slow. 1367b6136773SAndrew Morton */ 136865f27f38SDavid Howells int schedule_on_each_cpu(work_func_t func) 136915316ba8SChristoph Lameter { 137015316ba8SChristoph Lameter int cpu; 137165a64464SAndi Kleen int orig = -1; 1372b6136773SAndrew Morton struct work_struct *works; 137315316ba8SChristoph Lameter 1374b6136773SAndrew Morton works = alloc_percpu(struct work_struct); 1375b6136773SAndrew Morton if (!works) 137615316ba8SChristoph Lameter return -ENOMEM; 1377b6136773SAndrew Morton 137895402b38SGautham R Shenoy get_online_cpus(); 137993981800STejun Heo 138093981800STejun Heo /* 138193981800STejun Heo * When running in keventd don't schedule a work item on 138293981800STejun Heo * itself. Can just call directly because the work queue is 138393981800STejun Heo * already bound. This also is faster. 138493981800STejun Heo */ 138593981800STejun Heo if (current_is_keventd()) 138693981800STejun Heo orig = raw_smp_processor_id(); 138793981800STejun Heo 138815316ba8SChristoph Lameter for_each_online_cpu(cpu) { 13899bfb1839SIngo Molnar struct work_struct *work = per_cpu_ptr(works, cpu); 13909bfb1839SIngo Molnar 13919bfb1839SIngo Molnar INIT_WORK(work, func); 139293981800STejun Heo if (cpu != orig) 13938de6d308SOleg Nesterov schedule_work_on(cpu, work); 139415316ba8SChristoph Lameter } 139593981800STejun Heo if (orig >= 0) 139693981800STejun Heo func(per_cpu_ptr(works, orig)); 139793981800STejun Heo 139893981800STejun Heo for_each_online_cpu(cpu) 13998616a89aSOleg Nesterov flush_work(per_cpu_ptr(works, cpu)); 140093981800STejun Heo 140195402b38SGautham R Shenoy put_online_cpus(); 1402b6136773SAndrew Morton free_percpu(works); 140315316ba8SChristoph Lameter return 0; 140415316ba8SChristoph Lameter } 140515316ba8SChristoph Lameter 1406eef6a7d5SAlan Stern /** 1407eef6a7d5SAlan Stern * flush_scheduled_work - ensure that any scheduled work has run to completion. 1408eef6a7d5SAlan Stern * 1409eef6a7d5SAlan Stern * Forces execution of the kernel-global workqueue and blocks until its 1410eef6a7d5SAlan Stern * completion. 1411eef6a7d5SAlan Stern * 1412eef6a7d5SAlan Stern * Think twice before calling this function! It's very easy to get into 1413eef6a7d5SAlan Stern * trouble if you don't take great care. Either of the following situations 1414eef6a7d5SAlan Stern * will lead to deadlock: 1415eef6a7d5SAlan Stern * 1416eef6a7d5SAlan Stern * One of the work items currently on the workqueue needs to acquire 1417eef6a7d5SAlan Stern * a lock held by your code or its caller. 1418eef6a7d5SAlan Stern * 1419eef6a7d5SAlan Stern * Your code is running in the context of a work routine. 1420eef6a7d5SAlan Stern * 1421eef6a7d5SAlan Stern * They will be detected by lockdep when they occur, but the first might not 1422eef6a7d5SAlan Stern * occur very often. It depends on what work items are on the workqueue and 1423eef6a7d5SAlan Stern * what locks they need, which you have no control over. 1424eef6a7d5SAlan Stern * 1425eef6a7d5SAlan Stern * In most situations flushing the entire workqueue is overkill; you merely 1426eef6a7d5SAlan Stern * need to know that a particular work item isn't queued and isn't running. 1427eef6a7d5SAlan Stern * In such cases you should use cancel_delayed_work_sync() or 1428eef6a7d5SAlan Stern * cancel_work_sync() instead. 1429eef6a7d5SAlan Stern */ 14301da177e4SLinus Torvalds void flush_scheduled_work(void) 14311da177e4SLinus Torvalds { 14321da177e4SLinus Torvalds flush_workqueue(keventd_wq); 14331da177e4SLinus Torvalds } 1434ae90dd5dSDave Jones EXPORT_SYMBOL(flush_scheduled_work); 14351da177e4SLinus Torvalds 14361da177e4SLinus Torvalds /** 14371fa44ecaSJames Bottomley * execute_in_process_context - reliably execute the routine with user context 14381fa44ecaSJames Bottomley * @fn: the function to execute 14391fa44ecaSJames Bottomley * @ew: guaranteed storage for the execute work structure (must 14401fa44ecaSJames Bottomley * be available when the work executes) 14411fa44ecaSJames Bottomley * 14421fa44ecaSJames Bottomley * Executes the function immediately if process context is available, 14431fa44ecaSJames Bottomley * otherwise schedules the function for delayed execution. 14441fa44ecaSJames Bottomley * 14451fa44ecaSJames Bottomley * Returns: 0 - function was executed 14461fa44ecaSJames Bottomley * 1 - function was scheduled for execution 14471fa44ecaSJames Bottomley */ 144865f27f38SDavid Howells int execute_in_process_context(work_func_t fn, struct execute_work *ew) 14491fa44ecaSJames Bottomley { 14501fa44ecaSJames Bottomley if (!in_interrupt()) { 145165f27f38SDavid Howells fn(&ew->work); 14521fa44ecaSJames Bottomley return 0; 14531fa44ecaSJames Bottomley } 14541fa44ecaSJames Bottomley 145565f27f38SDavid Howells INIT_WORK(&ew->work, fn); 14561fa44ecaSJames Bottomley schedule_work(&ew->work); 14571fa44ecaSJames Bottomley 14581fa44ecaSJames Bottomley return 1; 14591fa44ecaSJames Bottomley } 14601fa44ecaSJames Bottomley EXPORT_SYMBOL_GPL(execute_in_process_context); 14611fa44ecaSJames Bottomley 14621da177e4SLinus Torvalds int keventd_up(void) 14631da177e4SLinus Torvalds { 14641da177e4SLinus Torvalds return keventd_wq != NULL; 14651da177e4SLinus Torvalds } 14661da177e4SLinus Torvalds 14671da177e4SLinus Torvalds int current_is_keventd(void) 14681da177e4SLinus Torvalds { 14691da177e4SLinus Torvalds struct cpu_workqueue_struct *cwq; 1470d243769dSHugh Dickins int cpu = raw_smp_processor_id(); /* preempt-safe: keventd is per-cpu */ 14711da177e4SLinus Torvalds int ret = 0; 14721da177e4SLinus Torvalds 14731da177e4SLinus Torvalds BUG_ON(!keventd_wq); 14741da177e4SLinus Torvalds 14751537663fSTejun Heo cwq = get_cwq(cpu, keventd_wq); 1476c34056a3STejun Heo if (current == cwq->worker->task) 14771da177e4SLinus Torvalds ret = 1; 14781da177e4SLinus Torvalds 14791da177e4SLinus Torvalds return ret; 14801da177e4SLinus Torvalds 14811da177e4SLinus Torvalds } 14821da177e4SLinus Torvalds 14830f900049STejun Heo static struct cpu_workqueue_struct *alloc_cwqs(void) 14840f900049STejun Heo { 14850f900049STejun Heo /* 14860f900049STejun Heo * cwqs are forced aligned according to WORK_STRUCT_FLAG_BITS. 14870f900049STejun Heo * Make sure that the alignment isn't lower than that of 14880f900049STejun Heo * unsigned long long. 14890f900049STejun Heo */ 14900f900049STejun Heo const size_t size = sizeof(struct cpu_workqueue_struct); 14910f900049STejun Heo const size_t align = max_t(size_t, 1 << WORK_STRUCT_FLAG_BITS, 14920f900049STejun Heo __alignof__(unsigned long long)); 14930f900049STejun Heo struct cpu_workqueue_struct *cwqs; 14940f900049STejun Heo #ifndef CONFIG_SMP 14950f900049STejun Heo void *ptr; 14960f900049STejun Heo 14970f900049STejun Heo /* 14980f900049STejun Heo * On UP, percpu allocator doesn't honor alignment parameter 14990f900049STejun Heo * and simply uses arch-dependent default. Allocate enough 15000f900049STejun Heo * room to align cwq and put an extra pointer at the end 15010f900049STejun Heo * pointing back to the originally allocated pointer which 15020f900049STejun Heo * will be used for free. 15030f900049STejun Heo * 15040f900049STejun Heo * FIXME: This really belongs to UP percpu code. Update UP 15050f900049STejun Heo * percpu code to honor alignment and remove this ugliness. 15060f900049STejun Heo */ 15070f900049STejun Heo ptr = __alloc_percpu(size + align + sizeof(void *), 1); 15080f900049STejun Heo cwqs = PTR_ALIGN(ptr, align); 15090f900049STejun Heo *(void **)per_cpu_ptr(cwqs + 1, 0) = ptr; 15100f900049STejun Heo #else 15110f900049STejun Heo /* On SMP, percpu allocator can do it itself */ 15120f900049STejun Heo cwqs = __alloc_percpu(size, align); 15130f900049STejun Heo #endif 15140f900049STejun Heo /* just in case, make sure it's actually aligned */ 15150f900049STejun Heo BUG_ON(!IS_ALIGNED((unsigned long)cwqs, align)); 15160f900049STejun Heo return cwqs; 15170f900049STejun Heo } 15180f900049STejun Heo 15190f900049STejun Heo static void free_cwqs(struct cpu_workqueue_struct *cwqs) 15200f900049STejun Heo { 15210f900049STejun Heo #ifndef CONFIG_SMP 15220f900049STejun Heo /* on UP, the pointer to free is stored right after the cwq */ 15230f900049STejun Heo if (cwqs) 15240f900049STejun Heo free_percpu(*(void **)per_cpu_ptr(cwqs + 1, 0)); 15250f900049STejun Heo #else 15260f900049STejun Heo free_percpu(cwqs); 15270f900049STejun Heo #endif 15280f900049STejun Heo } 15290f900049STejun Heo 15304e6045f1SJohannes Berg struct workqueue_struct *__create_workqueue_key(const char *name, 153197e37d7bSTejun Heo unsigned int flags, 15321e19ffc6STejun Heo int max_active, 1533eb13ba87SJohannes Berg struct lock_class_key *key, 1534eb13ba87SJohannes Berg const char *lock_name) 15353af24433SOleg Nesterov { 15361537663fSTejun Heo bool singlethread = flags & WQ_SINGLE_THREAD; 15373af24433SOleg Nesterov struct workqueue_struct *wq; 1538c34056a3STejun Heo bool failed = false; 1539c34056a3STejun Heo unsigned int cpu; 15403af24433SOleg Nesterov 15411e19ffc6STejun Heo max_active = clamp_val(max_active, 1, INT_MAX); 15421e19ffc6STejun Heo 15433af24433SOleg Nesterov wq = kzalloc(sizeof(*wq), GFP_KERNEL); 15443af24433SOleg Nesterov if (!wq) 15454690c4abSTejun Heo goto err; 15463af24433SOleg Nesterov 15470f900049STejun Heo wq->cpu_wq = alloc_cwqs(); 15484690c4abSTejun Heo if (!wq->cpu_wq) 15494690c4abSTejun Heo goto err; 15503af24433SOleg Nesterov 155197e37d7bSTejun Heo wq->flags = flags; 1552a0a1a5fdSTejun Heo wq->saved_max_active = max_active; 155373f53c4aSTejun Heo mutex_init(&wq->flush_mutex); 155473f53c4aSTejun Heo atomic_set(&wq->nr_cwqs_to_flush, 0); 155573f53c4aSTejun Heo INIT_LIST_HEAD(&wq->flusher_queue); 155673f53c4aSTejun Heo INIT_LIST_HEAD(&wq->flusher_overflow); 15573af24433SOleg Nesterov wq->name = name; 1558eb13ba87SJohannes Berg lockdep_init_map(&wq->lockdep_map, lock_name, key, 0); 1559cce1a165SOleg Nesterov INIT_LIST_HEAD(&wq->list); 15603af24433SOleg Nesterov 15613da1c84cSOleg Nesterov cpu_maps_update_begin(); 15626af8bf3dSOleg Nesterov /* 15636af8bf3dSOleg Nesterov * We must initialize cwqs for each possible cpu even if we 15646af8bf3dSOleg Nesterov * are going to call destroy_workqueue() finally. Otherwise 15656af8bf3dSOleg Nesterov * cpu_up() can hit the uninitialized cwq once we drop the 15666af8bf3dSOleg Nesterov * lock. 15676af8bf3dSOleg Nesterov */ 15683af24433SOleg Nesterov for_each_possible_cpu(cpu) { 15691537663fSTejun Heo struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq); 15701537663fSTejun Heo 15710f900049STejun Heo BUG_ON((unsigned long)cwq & WORK_STRUCT_FLAG_MASK); 15721537663fSTejun Heo cwq->cpu = cpu; 1573c34056a3STejun Heo cwq->wq = wq; 157473f53c4aSTejun Heo cwq->flush_color = -1; 15751e19ffc6STejun Heo cwq->max_active = max_active; 15761537663fSTejun Heo spin_lock_init(&cwq->lock); 15771537663fSTejun Heo INIT_LIST_HEAD(&cwq->worklist); 15781e19ffc6STejun Heo INIT_LIST_HEAD(&cwq->delayed_works); 15791537663fSTejun Heo init_waitqueue_head(&cwq->more_work); 15801537663fSTejun Heo 1581c34056a3STejun Heo if (failed) 15823af24433SOleg Nesterov continue; 1583c34056a3STejun Heo cwq->worker = create_worker(cwq, 1584c34056a3STejun Heo cpu_online(cpu) && !singlethread); 1585c34056a3STejun Heo if (cwq->worker) 1586c34056a3STejun Heo start_worker(cwq->worker); 15871537663fSTejun Heo else 1588c34056a3STejun Heo failed = true; 15893af24433SOleg Nesterov } 15901537663fSTejun Heo 1591a0a1a5fdSTejun Heo /* 1592a0a1a5fdSTejun Heo * workqueue_lock protects global freeze state and workqueues 1593a0a1a5fdSTejun Heo * list. Grab it, set max_active accordingly and add the new 1594a0a1a5fdSTejun Heo * workqueue to workqueues list. 1595a0a1a5fdSTejun Heo */ 15961537663fSTejun Heo spin_lock(&workqueue_lock); 1597a0a1a5fdSTejun Heo 1598a0a1a5fdSTejun Heo if (workqueue_freezing && wq->flags & WQ_FREEZEABLE) 1599a0a1a5fdSTejun Heo for_each_possible_cpu(cpu) 1600a0a1a5fdSTejun Heo get_cwq(cpu, wq)->max_active = 0; 1601a0a1a5fdSTejun Heo 16021537663fSTejun Heo list_add(&wq->list, &workqueues); 1603a0a1a5fdSTejun Heo 16041537663fSTejun Heo spin_unlock(&workqueue_lock); 16051537663fSTejun Heo 16063da1c84cSOleg Nesterov cpu_maps_update_done(); 16073af24433SOleg Nesterov 1608c34056a3STejun Heo if (failed) { 16093af24433SOleg Nesterov destroy_workqueue(wq); 16103af24433SOleg Nesterov wq = NULL; 16113af24433SOleg Nesterov } 16123af24433SOleg Nesterov return wq; 16134690c4abSTejun Heo err: 16144690c4abSTejun Heo if (wq) { 16150f900049STejun Heo free_cwqs(wq->cpu_wq); 16164690c4abSTejun Heo kfree(wq); 16174690c4abSTejun Heo } 16184690c4abSTejun Heo return NULL; 16193af24433SOleg Nesterov } 16204e6045f1SJohannes Berg EXPORT_SYMBOL_GPL(__create_workqueue_key); 16213af24433SOleg Nesterov 16223af24433SOleg Nesterov /** 16233af24433SOleg Nesterov * destroy_workqueue - safely terminate a workqueue 16243af24433SOleg Nesterov * @wq: target workqueue 16253af24433SOleg Nesterov * 16263af24433SOleg Nesterov * Safely destroy a workqueue. All work currently pending will be done first. 16273af24433SOleg Nesterov */ 16283af24433SOleg Nesterov void destroy_workqueue(struct workqueue_struct *wq) 16293af24433SOleg Nesterov { 16303af24433SOleg Nesterov int cpu; 16313af24433SOleg Nesterov 1632a0a1a5fdSTejun Heo flush_workqueue(wq); 1633a0a1a5fdSTejun Heo 1634a0a1a5fdSTejun Heo /* 1635a0a1a5fdSTejun Heo * wq list is used to freeze wq, remove from list after 1636a0a1a5fdSTejun Heo * flushing is complete in case freeze races us. 1637a0a1a5fdSTejun Heo */ 16383da1c84cSOleg Nesterov cpu_maps_update_begin(); 163995402b38SGautham R Shenoy spin_lock(&workqueue_lock); 16403af24433SOleg Nesterov list_del(&wq->list); 164195402b38SGautham R Shenoy spin_unlock(&workqueue_lock); 16423da1c84cSOleg Nesterov cpu_maps_update_done(); 16433af24433SOleg Nesterov 164473f53c4aSTejun Heo for_each_possible_cpu(cpu) { 164573f53c4aSTejun Heo struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq); 164673f53c4aSTejun Heo int i; 164773f53c4aSTejun Heo 1648c34056a3STejun Heo if (cwq->worker) { 1649c34056a3STejun Heo destroy_worker(cwq->worker); 1650c34056a3STejun Heo cwq->worker = NULL; 165173f53c4aSTejun Heo } 165273f53c4aSTejun Heo 165373f53c4aSTejun Heo for (i = 0; i < WORK_NR_COLORS; i++) 165473f53c4aSTejun Heo BUG_ON(cwq->nr_in_flight[i]); 16551e19ffc6STejun Heo BUG_ON(cwq->nr_active); 16561e19ffc6STejun Heo BUG_ON(!list_empty(&cwq->delayed_works)); 165773f53c4aSTejun Heo } 16581537663fSTejun Heo 16590f900049STejun Heo free_cwqs(wq->cpu_wq); 16603af24433SOleg Nesterov kfree(wq); 16613af24433SOleg Nesterov } 16623af24433SOleg Nesterov EXPORT_SYMBOL_GPL(destroy_workqueue); 16633af24433SOleg Nesterov 16649c7b216dSChandra Seetharaman static int __devinit workqueue_cpu_callback(struct notifier_block *nfb, 16651da177e4SLinus Torvalds unsigned long action, 16661da177e4SLinus Torvalds void *hcpu) 16671da177e4SLinus Torvalds { 16683af24433SOleg Nesterov unsigned int cpu = (unsigned long)hcpu; 16693af24433SOleg Nesterov struct cpu_workqueue_struct *cwq; 16701da177e4SLinus Torvalds struct workqueue_struct *wq; 16711da177e4SLinus Torvalds 16728bb78442SRafael J. Wysocki action &= ~CPU_TASKS_FROZEN; 16738bb78442SRafael J. Wysocki 16741da177e4SLinus Torvalds list_for_each_entry(wq, &workqueues, list) { 16751537663fSTejun Heo if (wq->flags & WQ_SINGLE_THREAD) 16761537663fSTejun Heo continue; 16771537663fSTejun Heo 16781537663fSTejun Heo cwq = get_cwq(cpu, wq); 16793af24433SOleg Nesterov 16803af24433SOleg Nesterov switch (action) { 16813da1c84cSOleg Nesterov case CPU_POST_DEAD: 168273f53c4aSTejun Heo flush_workqueue(wq); 16831da177e4SLinus Torvalds break; 16841da177e4SLinus Torvalds } 16853af24433SOleg Nesterov } 16861da177e4SLinus Torvalds 16871537663fSTejun Heo return notifier_from_errno(0); 16881da177e4SLinus Torvalds } 16891da177e4SLinus Torvalds 16902d3854a3SRusty Russell #ifdef CONFIG_SMP 16918ccad40dSRusty Russell 16922d3854a3SRusty Russell struct work_for_cpu { 16936b44003eSAndrew Morton struct completion completion; 16942d3854a3SRusty Russell long (*fn)(void *); 16952d3854a3SRusty Russell void *arg; 16962d3854a3SRusty Russell long ret; 16972d3854a3SRusty Russell }; 16982d3854a3SRusty Russell 16996b44003eSAndrew Morton static int do_work_for_cpu(void *_wfc) 17002d3854a3SRusty Russell { 17016b44003eSAndrew Morton struct work_for_cpu *wfc = _wfc; 17022d3854a3SRusty Russell wfc->ret = wfc->fn(wfc->arg); 17036b44003eSAndrew Morton complete(&wfc->completion); 17046b44003eSAndrew Morton return 0; 17052d3854a3SRusty Russell } 17062d3854a3SRusty Russell 17072d3854a3SRusty Russell /** 17082d3854a3SRusty Russell * work_on_cpu - run a function in user context on a particular cpu 17092d3854a3SRusty Russell * @cpu: the cpu to run on 17102d3854a3SRusty Russell * @fn: the function to run 17112d3854a3SRusty Russell * @arg: the function arg 17122d3854a3SRusty Russell * 171331ad9081SRusty Russell * This will return the value @fn returns. 171431ad9081SRusty Russell * It is up to the caller to ensure that the cpu doesn't go offline. 17156b44003eSAndrew Morton * The caller must not hold any locks which would prevent @fn from completing. 17162d3854a3SRusty Russell */ 17172d3854a3SRusty Russell long work_on_cpu(unsigned int cpu, long (*fn)(void *), void *arg) 17182d3854a3SRusty Russell { 17196b44003eSAndrew Morton struct task_struct *sub_thread; 17206b44003eSAndrew Morton struct work_for_cpu wfc = { 17216b44003eSAndrew Morton .completion = COMPLETION_INITIALIZER_ONSTACK(wfc.completion), 17226b44003eSAndrew Morton .fn = fn, 17236b44003eSAndrew Morton .arg = arg, 17246b44003eSAndrew Morton }; 17252d3854a3SRusty Russell 17266b44003eSAndrew Morton sub_thread = kthread_create(do_work_for_cpu, &wfc, "work_for_cpu"); 17276b44003eSAndrew Morton if (IS_ERR(sub_thread)) 17286b44003eSAndrew Morton return PTR_ERR(sub_thread); 17296b44003eSAndrew Morton kthread_bind(sub_thread, cpu); 17306b44003eSAndrew Morton wake_up_process(sub_thread); 17316b44003eSAndrew Morton wait_for_completion(&wfc.completion); 17322d3854a3SRusty Russell return wfc.ret; 17332d3854a3SRusty Russell } 17342d3854a3SRusty Russell EXPORT_SYMBOL_GPL(work_on_cpu); 17352d3854a3SRusty Russell #endif /* CONFIG_SMP */ 17362d3854a3SRusty Russell 1737a0a1a5fdSTejun Heo #ifdef CONFIG_FREEZER 1738a0a1a5fdSTejun Heo 1739a0a1a5fdSTejun Heo /** 1740a0a1a5fdSTejun Heo * freeze_workqueues_begin - begin freezing workqueues 1741a0a1a5fdSTejun Heo * 1742a0a1a5fdSTejun Heo * Start freezing workqueues. After this function returns, all 1743a0a1a5fdSTejun Heo * freezeable workqueues will queue new works to their frozen_works 1744a0a1a5fdSTejun Heo * list instead of the cwq ones. 1745a0a1a5fdSTejun Heo * 1746a0a1a5fdSTejun Heo * CONTEXT: 1747a0a1a5fdSTejun Heo * Grabs and releases workqueue_lock and cwq->lock's. 1748a0a1a5fdSTejun Heo */ 1749a0a1a5fdSTejun Heo void freeze_workqueues_begin(void) 1750a0a1a5fdSTejun Heo { 1751a0a1a5fdSTejun Heo struct workqueue_struct *wq; 1752a0a1a5fdSTejun Heo unsigned int cpu; 1753a0a1a5fdSTejun Heo 1754a0a1a5fdSTejun Heo spin_lock(&workqueue_lock); 1755a0a1a5fdSTejun Heo 1756a0a1a5fdSTejun Heo BUG_ON(workqueue_freezing); 1757a0a1a5fdSTejun Heo workqueue_freezing = true; 1758a0a1a5fdSTejun Heo 1759a0a1a5fdSTejun Heo for_each_possible_cpu(cpu) { 1760a0a1a5fdSTejun Heo list_for_each_entry(wq, &workqueues, list) { 1761a0a1a5fdSTejun Heo struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq); 1762a0a1a5fdSTejun Heo 1763a0a1a5fdSTejun Heo spin_lock_irq(&cwq->lock); 1764a0a1a5fdSTejun Heo 1765a0a1a5fdSTejun Heo if (wq->flags & WQ_FREEZEABLE) 1766a0a1a5fdSTejun Heo cwq->max_active = 0; 1767a0a1a5fdSTejun Heo 1768a0a1a5fdSTejun Heo spin_unlock_irq(&cwq->lock); 1769a0a1a5fdSTejun Heo } 1770a0a1a5fdSTejun Heo } 1771a0a1a5fdSTejun Heo 1772a0a1a5fdSTejun Heo spin_unlock(&workqueue_lock); 1773a0a1a5fdSTejun Heo } 1774a0a1a5fdSTejun Heo 1775a0a1a5fdSTejun Heo /** 1776a0a1a5fdSTejun Heo * freeze_workqueues_busy - are freezeable workqueues still busy? 1777a0a1a5fdSTejun Heo * 1778a0a1a5fdSTejun Heo * Check whether freezing is complete. This function must be called 1779a0a1a5fdSTejun Heo * between freeze_workqueues_begin() and thaw_workqueues(). 1780a0a1a5fdSTejun Heo * 1781a0a1a5fdSTejun Heo * CONTEXT: 1782a0a1a5fdSTejun Heo * Grabs and releases workqueue_lock. 1783a0a1a5fdSTejun Heo * 1784a0a1a5fdSTejun Heo * RETURNS: 1785a0a1a5fdSTejun Heo * %true if some freezeable workqueues are still busy. %false if 1786a0a1a5fdSTejun Heo * freezing is complete. 1787a0a1a5fdSTejun Heo */ 1788a0a1a5fdSTejun Heo bool freeze_workqueues_busy(void) 1789a0a1a5fdSTejun Heo { 1790a0a1a5fdSTejun Heo struct workqueue_struct *wq; 1791a0a1a5fdSTejun Heo unsigned int cpu; 1792a0a1a5fdSTejun Heo bool busy = false; 1793a0a1a5fdSTejun Heo 1794a0a1a5fdSTejun Heo spin_lock(&workqueue_lock); 1795a0a1a5fdSTejun Heo 1796a0a1a5fdSTejun Heo BUG_ON(!workqueue_freezing); 1797a0a1a5fdSTejun Heo 1798a0a1a5fdSTejun Heo for_each_possible_cpu(cpu) { 1799a0a1a5fdSTejun Heo /* 1800a0a1a5fdSTejun Heo * nr_active is monotonically decreasing. It's safe 1801a0a1a5fdSTejun Heo * to peek without lock. 1802a0a1a5fdSTejun Heo */ 1803a0a1a5fdSTejun Heo list_for_each_entry(wq, &workqueues, list) { 1804a0a1a5fdSTejun Heo struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq); 1805a0a1a5fdSTejun Heo 1806a0a1a5fdSTejun Heo if (!(wq->flags & WQ_FREEZEABLE)) 1807a0a1a5fdSTejun Heo continue; 1808a0a1a5fdSTejun Heo 1809a0a1a5fdSTejun Heo BUG_ON(cwq->nr_active < 0); 1810a0a1a5fdSTejun Heo if (cwq->nr_active) { 1811a0a1a5fdSTejun Heo busy = true; 1812a0a1a5fdSTejun Heo goto out_unlock; 1813a0a1a5fdSTejun Heo } 1814a0a1a5fdSTejun Heo } 1815a0a1a5fdSTejun Heo } 1816a0a1a5fdSTejun Heo out_unlock: 1817a0a1a5fdSTejun Heo spin_unlock(&workqueue_lock); 1818a0a1a5fdSTejun Heo return busy; 1819a0a1a5fdSTejun Heo } 1820a0a1a5fdSTejun Heo 1821a0a1a5fdSTejun Heo /** 1822a0a1a5fdSTejun Heo * thaw_workqueues - thaw workqueues 1823a0a1a5fdSTejun Heo * 1824a0a1a5fdSTejun Heo * Thaw workqueues. Normal queueing is restored and all collected 1825a0a1a5fdSTejun Heo * frozen works are transferred to their respective cwq worklists. 1826a0a1a5fdSTejun Heo * 1827a0a1a5fdSTejun Heo * CONTEXT: 1828a0a1a5fdSTejun Heo * Grabs and releases workqueue_lock and cwq->lock's. 1829a0a1a5fdSTejun Heo */ 1830a0a1a5fdSTejun Heo void thaw_workqueues(void) 1831a0a1a5fdSTejun Heo { 1832a0a1a5fdSTejun Heo struct workqueue_struct *wq; 1833a0a1a5fdSTejun Heo unsigned int cpu; 1834a0a1a5fdSTejun Heo 1835a0a1a5fdSTejun Heo spin_lock(&workqueue_lock); 1836a0a1a5fdSTejun Heo 1837a0a1a5fdSTejun Heo if (!workqueue_freezing) 1838a0a1a5fdSTejun Heo goto out_unlock; 1839a0a1a5fdSTejun Heo 1840a0a1a5fdSTejun Heo for_each_possible_cpu(cpu) { 1841a0a1a5fdSTejun Heo list_for_each_entry(wq, &workqueues, list) { 1842a0a1a5fdSTejun Heo struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq); 1843a0a1a5fdSTejun Heo 1844a0a1a5fdSTejun Heo if (!(wq->flags & WQ_FREEZEABLE)) 1845a0a1a5fdSTejun Heo continue; 1846a0a1a5fdSTejun Heo 1847a0a1a5fdSTejun Heo spin_lock_irq(&cwq->lock); 1848a0a1a5fdSTejun Heo 1849a0a1a5fdSTejun Heo /* restore max_active and repopulate worklist */ 1850a0a1a5fdSTejun Heo cwq->max_active = wq->saved_max_active; 1851a0a1a5fdSTejun Heo 1852a0a1a5fdSTejun Heo while (!list_empty(&cwq->delayed_works) && 1853a0a1a5fdSTejun Heo cwq->nr_active < cwq->max_active) 1854a0a1a5fdSTejun Heo cwq_activate_first_delayed(cwq); 1855a0a1a5fdSTejun Heo 1856a0a1a5fdSTejun Heo wake_up(&cwq->more_work); 1857a0a1a5fdSTejun Heo 1858a0a1a5fdSTejun Heo spin_unlock_irq(&cwq->lock); 1859a0a1a5fdSTejun Heo } 1860a0a1a5fdSTejun Heo } 1861a0a1a5fdSTejun Heo 1862a0a1a5fdSTejun Heo workqueue_freezing = false; 1863a0a1a5fdSTejun Heo out_unlock: 1864a0a1a5fdSTejun Heo spin_unlock(&workqueue_lock); 1865a0a1a5fdSTejun Heo } 1866a0a1a5fdSTejun Heo #endif /* CONFIG_FREEZER */ 1867a0a1a5fdSTejun Heo 1868c12920d1SOleg Nesterov void __init init_workqueues(void) 18691da177e4SLinus Torvalds { 1870c34056a3STejun Heo unsigned int cpu; 1871c34056a3STejun Heo 1872c34056a3STejun Heo for_each_possible_cpu(cpu) 1873c34056a3STejun Heo ida_init(&per_cpu(worker_ida, cpu)); 1874c34056a3STejun Heo 1875e7577c50SRusty Russell singlethread_cpu = cpumask_first(cpu_possible_mask); 18761da177e4SLinus Torvalds hotcpu_notifier(workqueue_cpu_callback, 0); 18771da177e4SLinus Torvalds keventd_wq = create_workqueue("events"); 18781da177e4SLinus Torvalds BUG_ON(!keventd_wq); 18791da177e4SLinus Torvalds } 1880