1 /* 2 * kernel/workqueue.c - generic async execution with shared worker pool 3 * 4 * Copyright (C) 2002 Ingo Molnar 5 * 6 * Derived from the taskqueue/keventd code by: 7 * David Woodhouse <dwmw2@infradead.org> 8 * Andrew Morton 9 * Kai Petzke <wpp@marie.physik.tu-berlin.de> 10 * Theodore Ts'o <tytso@mit.edu> 11 * 12 * Made to use alloc_percpu by Christoph Lameter. 13 * 14 * Copyright (C) 2010 SUSE Linux Products GmbH 15 * Copyright (C) 2010 Tejun Heo <tj@kernel.org> 16 * 17 * This is the generic async execution mechanism. Work items as are 18 * executed in process context. The worker pool is shared and 19 * automatically managed. There are two worker pools for each CPU (one for 20 * normal work items and the other for high priority ones) and some extra 21 * pools for workqueues which are not bound to any specific CPU - the 22 * number of these backing pools is dynamic. 23 * 24 * Please read Documentation/workqueue.txt for details. 25 */ 26 27 #include <linux/export.h> 28 #include <linux/kernel.h> 29 #include <linux/sched.h> 30 #include <linux/init.h> 31 #include <linux/signal.h> 32 #include <linux/completion.h> 33 #include <linux/workqueue.h> 34 #include <linux/slab.h> 35 #include <linux/cpu.h> 36 #include <linux/notifier.h> 37 #include <linux/kthread.h> 38 #include <linux/hardirq.h> 39 #include <linux/mempolicy.h> 40 #include <linux/freezer.h> 41 #include <linux/kallsyms.h> 42 #include <linux/debug_locks.h> 43 #include <linux/lockdep.h> 44 #include <linux/idr.h> 45 #include <linux/jhash.h> 46 #include <linux/hashtable.h> 47 #include <linux/rculist.h> 48 #include <linux/nodemask.h> 49 #include <linux/moduleparam.h> 50 #include <linux/uaccess.h> 51 52 #include "workqueue_internal.h" 53 54 enum { 55 /* 56 * worker_pool flags 57 * 58 * A bound pool is either associated or disassociated with its CPU. 59 * While associated (!DISASSOCIATED), all workers are bound to the 60 * CPU and none has %WORKER_UNBOUND set and concurrency management 61 * is in effect. 62 * 63 * While DISASSOCIATED, the cpu may be offline and all workers have 64 * %WORKER_UNBOUND set and concurrency management disabled, and may 65 * be executing on any CPU. The pool behaves as an unbound one. 66 * 67 * Note that DISASSOCIATED should be flipped only while holding 68 * attach_mutex to avoid changing binding state while 69 * worker_attach_to_pool() is in progress. 70 */ 71 POOL_DISASSOCIATED = 1 << 2, /* cpu can't serve workers */ 72 73 /* worker flags */ 74 WORKER_DIE = 1 << 1, /* die die die */ 75 WORKER_IDLE = 1 << 2, /* is idle */ 76 WORKER_PREP = 1 << 3, /* preparing to run works */ 77 WORKER_CPU_INTENSIVE = 1 << 6, /* cpu intensive */ 78 WORKER_UNBOUND = 1 << 7, /* worker is unbound */ 79 WORKER_REBOUND = 1 << 8, /* worker was rebound */ 80 81 WORKER_NOT_RUNNING = WORKER_PREP | WORKER_CPU_INTENSIVE | 82 WORKER_UNBOUND | WORKER_REBOUND, 83 84 NR_STD_WORKER_POOLS = 2, /* # standard pools per cpu */ 85 86 UNBOUND_POOL_HASH_ORDER = 6, /* hashed by pool->attrs */ 87 BUSY_WORKER_HASH_ORDER = 6, /* 64 pointers */ 88 89 MAX_IDLE_WORKERS_RATIO = 4, /* 1/4 of busy can be idle */ 90 IDLE_WORKER_TIMEOUT = 300 * HZ, /* keep idle ones for 5 mins */ 91 92 MAYDAY_INITIAL_TIMEOUT = HZ / 100 >= 2 ? HZ / 100 : 2, 93 /* call for help after 10ms 94 (min two ticks) */ 95 MAYDAY_INTERVAL = HZ / 10, /* and then every 100ms */ 96 CREATE_COOLDOWN = HZ, /* time to breath after fail */ 97 98 /* 99 * Rescue workers are used only on emergencies and shared by 100 * all cpus. Give MIN_NICE. 101 */ 102 RESCUER_NICE_LEVEL = MIN_NICE, 103 HIGHPRI_NICE_LEVEL = MIN_NICE, 104 105 WQ_NAME_LEN = 24, 106 }; 107 108 /* 109 * Structure fields follow one of the following exclusion rules. 110 * 111 * I: Modifiable by initialization/destruction paths and read-only for 112 * everyone else. 113 * 114 * P: Preemption protected. Disabling preemption is enough and should 115 * only be modified and accessed from the local cpu. 116 * 117 * L: pool->lock protected. Access with pool->lock held. 118 * 119 * X: During normal operation, modification requires pool->lock and should 120 * be done only from local cpu. Either disabling preemption on local 121 * cpu or grabbing pool->lock is enough for read access. If 122 * POOL_DISASSOCIATED is set, it's identical to L. 123 * 124 * A: pool->attach_mutex protected. 125 * 126 * PL: wq_pool_mutex protected. 127 * 128 * PR: wq_pool_mutex protected for writes. Sched-RCU protected for reads. 129 * 130 * PW: wq_pool_mutex and wq->mutex protected for writes. Either for reads. 131 * 132 * PWR: wq_pool_mutex and wq->mutex protected for writes. Either or 133 * sched-RCU for reads. 134 * 135 * WQ: wq->mutex protected. 136 * 137 * WR: wq->mutex protected for writes. Sched-RCU protected for reads. 138 * 139 * MD: wq_mayday_lock protected. 140 */ 141 142 /* struct worker is defined in workqueue_internal.h */ 143 144 struct worker_pool { 145 spinlock_t lock; /* the pool lock */ 146 int cpu; /* I: the associated cpu */ 147 int node; /* I: the associated node ID */ 148 int id; /* I: pool ID */ 149 unsigned int flags; /* X: flags */ 150 151 unsigned long watchdog_ts; /* L: watchdog timestamp */ 152 153 struct list_head worklist; /* L: list of pending works */ 154 int nr_workers; /* L: total number of workers */ 155 156 /* nr_idle includes the ones off idle_list for rebinding */ 157 int nr_idle; /* L: currently idle ones */ 158 159 struct list_head idle_list; /* X: list of idle workers */ 160 struct timer_list idle_timer; /* L: worker idle timeout */ 161 struct timer_list mayday_timer; /* L: SOS timer for workers */ 162 163 /* a workers is either on busy_hash or idle_list, or the manager */ 164 DECLARE_HASHTABLE(busy_hash, BUSY_WORKER_HASH_ORDER); 165 /* L: hash of busy workers */ 166 167 /* see manage_workers() for details on the two manager mutexes */ 168 struct mutex manager_arb; /* manager arbitration */ 169 struct worker *manager; /* L: purely informational */ 170 struct mutex attach_mutex; /* attach/detach exclusion */ 171 struct list_head workers; /* A: attached workers */ 172 struct completion *detach_completion; /* all workers detached */ 173 174 struct ida worker_ida; /* worker IDs for task name */ 175 176 struct workqueue_attrs *attrs; /* I: worker attributes */ 177 struct hlist_node hash_node; /* PL: unbound_pool_hash node */ 178 int refcnt; /* PL: refcnt for unbound pools */ 179 180 /* 181 * The current concurrency level. As it's likely to be accessed 182 * from other CPUs during try_to_wake_up(), put it in a separate 183 * cacheline. 184 */ 185 atomic_t nr_running ____cacheline_aligned_in_smp; 186 187 /* 188 * Destruction of pool is sched-RCU protected to allow dereferences 189 * from get_work_pool(). 190 */ 191 struct rcu_head rcu; 192 } ____cacheline_aligned_in_smp; 193 194 /* 195 * The per-pool workqueue. While queued, the lower WORK_STRUCT_FLAG_BITS 196 * of work_struct->data are used for flags and the remaining high bits 197 * point to the pwq; thus, pwqs need to be aligned at two's power of the 198 * number of flag bits. 199 */ 200 struct pool_workqueue { 201 struct worker_pool *pool; /* I: the associated pool */ 202 struct workqueue_struct *wq; /* I: the owning workqueue */ 203 int work_color; /* L: current color */ 204 int flush_color; /* L: flushing color */ 205 int refcnt; /* L: reference count */ 206 int nr_in_flight[WORK_NR_COLORS]; 207 /* L: nr of in_flight works */ 208 int nr_active; /* L: nr of active works */ 209 int max_active; /* L: max active works */ 210 struct list_head delayed_works; /* L: delayed works */ 211 struct list_head pwqs_node; /* WR: node on wq->pwqs */ 212 struct list_head mayday_node; /* MD: node on wq->maydays */ 213 214 /* 215 * Release of unbound pwq is punted to system_wq. See put_pwq() 216 * and pwq_unbound_release_workfn() for details. pool_workqueue 217 * itself is also sched-RCU protected so that the first pwq can be 218 * determined without grabbing wq->mutex. 219 */ 220 struct work_struct unbound_release_work; 221 struct rcu_head rcu; 222 } __aligned(1 << WORK_STRUCT_FLAG_BITS); 223 224 /* 225 * Structure used to wait for workqueue flush. 226 */ 227 struct wq_flusher { 228 struct list_head list; /* WQ: list of flushers */ 229 int flush_color; /* WQ: flush color waiting for */ 230 struct completion done; /* flush completion */ 231 }; 232 233 struct wq_device; 234 235 /* 236 * The externally visible workqueue. It relays the issued work items to 237 * the appropriate worker_pool through its pool_workqueues. 238 */ 239 struct workqueue_struct { 240 struct list_head pwqs; /* WR: all pwqs of this wq */ 241 struct list_head list; /* PR: list of all workqueues */ 242 243 struct mutex mutex; /* protects this wq */ 244 int work_color; /* WQ: current work color */ 245 int flush_color; /* WQ: current flush color */ 246 atomic_t nr_pwqs_to_flush; /* flush in progress */ 247 struct wq_flusher *first_flusher; /* WQ: first flusher */ 248 struct list_head flusher_queue; /* WQ: flush waiters */ 249 struct list_head flusher_overflow; /* WQ: flush overflow list */ 250 251 struct list_head maydays; /* MD: pwqs requesting rescue */ 252 struct worker *rescuer; /* I: rescue worker */ 253 254 int nr_drainers; /* WQ: drain in progress */ 255 int saved_max_active; /* WQ: saved pwq max_active */ 256 257 struct workqueue_attrs *unbound_attrs; /* PW: only for unbound wqs */ 258 struct pool_workqueue *dfl_pwq; /* PW: only for unbound wqs */ 259 260 #ifdef CONFIG_SYSFS 261 struct wq_device *wq_dev; /* I: for sysfs interface */ 262 #endif 263 #ifdef CONFIG_LOCKDEP 264 struct lockdep_map lockdep_map; 265 #endif 266 char name[WQ_NAME_LEN]; /* I: workqueue name */ 267 268 /* 269 * Destruction of workqueue_struct is sched-RCU protected to allow 270 * walking the workqueues list without grabbing wq_pool_mutex. 271 * This is used to dump all workqueues from sysrq. 272 */ 273 struct rcu_head rcu; 274 275 /* hot fields used during command issue, aligned to cacheline */ 276 unsigned int flags ____cacheline_aligned; /* WQ: WQ_* flags */ 277 struct pool_workqueue __percpu *cpu_pwqs; /* I: per-cpu pwqs */ 278 struct pool_workqueue __rcu *numa_pwq_tbl[]; /* PWR: unbound pwqs indexed by node */ 279 }; 280 281 static struct kmem_cache *pwq_cache; 282 283 static cpumask_var_t *wq_numa_possible_cpumask; 284 /* possible CPUs of each node */ 285 286 static bool wq_disable_numa; 287 module_param_named(disable_numa, wq_disable_numa, bool, 0444); 288 289 /* see the comment above the definition of WQ_POWER_EFFICIENT */ 290 static bool wq_power_efficient = IS_ENABLED(CONFIG_WQ_POWER_EFFICIENT_DEFAULT); 291 module_param_named(power_efficient, wq_power_efficient, bool, 0444); 292 293 static bool wq_online; /* can kworkers be created yet? */ 294 295 static bool wq_numa_enabled; /* unbound NUMA affinity enabled */ 296 297 /* buf for wq_update_unbound_numa_attrs(), protected by CPU hotplug exclusion */ 298 static struct workqueue_attrs *wq_update_unbound_numa_attrs_buf; 299 300 static DEFINE_MUTEX(wq_pool_mutex); /* protects pools and workqueues list */ 301 static DEFINE_SPINLOCK(wq_mayday_lock); /* protects wq->maydays list */ 302 303 static LIST_HEAD(workqueues); /* PR: list of all workqueues */ 304 static bool workqueue_freezing; /* PL: have wqs started freezing? */ 305 306 /* PL: allowable cpus for unbound wqs and work items */ 307 static cpumask_var_t wq_unbound_cpumask; 308 309 /* CPU where unbound work was last round robin scheduled from this CPU */ 310 static DEFINE_PER_CPU(int, wq_rr_cpu_last); 311 312 /* 313 * Local execution of unbound work items is no longer guaranteed. The 314 * following always forces round-robin CPU selection on unbound work items 315 * to uncover usages which depend on it. 316 */ 317 #ifdef CONFIG_DEBUG_WQ_FORCE_RR_CPU 318 static bool wq_debug_force_rr_cpu = true; 319 #else 320 static bool wq_debug_force_rr_cpu = false; 321 #endif 322 module_param_named(debug_force_rr_cpu, wq_debug_force_rr_cpu, bool, 0644); 323 324 /* the per-cpu worker pools */ 325 static DEFINE_PER_CPU_SHARED_ALIGNED(struct worker_pool [NR_STD_WORKER_POOLS], cpu_worker_pools); 326 327 static DEFINE_IDR(worker_pool_idr); /* PR: idr of all pools */ 328 329 /* PL: hash of all unbound pools keyed by pool->attrs */ 330 static DEFINE_HASHTABLE(unbound_pool_hash, UNBOUND_POOL_HASH_ORDER); 331 332 /* I: attributes used when instantiating standard unbound pools on demand */ 333 static struct workqueue_attrs *unbound_std_wq_attrs[NR_STD_WORKER_POOLS]; 334 335 /* I: attributes used when instantiating ordered pools on demand */ 336 static struct workqueue_attrs *ordered_wq_attrs[NR_STD_WORKER_POOLS]; 337 338 struct workqueue_struct *system_wq __read_mostly; 339 EXPORT_SYMBOL(system_wq); 340 struct workqueue_struct *system_highpri_wq __read_mostly; 341 EXPORT_SYMBOL_GPL(system_highpri_wq); 342 struct workqueue_struct *system_long_wq __read_mostly; 343 EXPORT_SYMBOL_GPL(system_long_wq); 344 struct workqueue_struct *system_unbound_wq __read_mostly; 345 EXPORT_SYMBOL_GPL(system_unbound_wq); 346 struct workqueue_struct *system_freezable_wq __read_mostly; 347 EXPORT_SYMBOL_GPL(system_freezable_wq); 348 struct workqueue_struct *system_power_efficient_wq __read_mostly; 349 EXPORT_SYMBOL_GPL(system_power_efficient_wq); 350 struct workqueue_struct *system_freezable_power_efficient_wq __read_mostly; 351 EXPORT_SYMBOL_GPL(system_freezable_power_efficient_wq); 352 353 static int worker_thread(void *__worker); 354 static void workqueue_sysfs_unregister(struct workqueue_struct *wq); 355 356 #define CREATE_TRACE_POINTS 357 #include <trace/events/workqueue.h> 358 359 #define assert_rcu_or_pool_mutex() \ 360 RCU_LOCKDEP_WARN(!rcu_read_lock_sched_held() && \ 361 !lockdep_is_held(&wq_pool_mutex), \ 362 "sched RCU or wq_pool_mutex should be held") 363 364 #define assert_rcu_or_wq_mutex(wq) \ 365 RCU_LOCKDEP_WARN(!rcu_read_lock_sched_held() && \ 366 !lockdep_is_held(&wq->mutex), \ 367 "sched RCU or wq->mutex should be held") 368 369 #define assert_rcu_or_wq_mutex_or_pool_mutex(wq) \ 370 RCU_LOCKDEP_WARN(!rcu_read_lock_sched_held() && \ 371 !lockdep_is_held(&wq->mutex) && \ 372 !lockdep_is_held(&wq_pool_mutex), \ 373 "sched RCU, wq->mutex or wq_pool_mutex should be held") 374 375 #define for_each_cpu_worker_pool(pool, cpu) \ 376 for ((pool) = &per_cpu(cpu_worker_pools, cpu)[0]; \ 377 (pool) < &per_cpu(cpu_worker_pools, cpu)[NR_STD_WORKER_POOLS]; \ 378 (pool)++) 379 380 /** 381 * for_each_pool - iterate through all worker_pools in the system 382 * @pool: iteration cursor 383 * @pi: integer used for iteration 384 * 385 * This must be called either with wq_pool_mutex held or sched RCU read 386 * locked. If the pool needs to be used beyond the locking in effect, the 387 * caller is responsible for guaranteeing that the pool stays online. 388 * 389 * The if/else clause exists only for the lockdep assertion and can be 390 * ignored. 391 */ 392 #define for_each_pool(pool, pi) \ 393 idr_for_each_entry(&worker_pool_idr, pool, pi) \ 394 if (({ assert_rcu_or_pool_mutex(); false; })) { } \ 395 else 396 397 /** 398 * for_each_pool_worker - iterate through all workers of a worker_pool 399 * @worker: iteration cursor 400 * @pool: worker_pool to iterate workers of 401 * 402 * This must be called with @pool->attach_mutex. 403 * 404 * The if/else clause exists only for the lockdep assertion and can be 405 * ignored. 406 */ 407 #define for_each_pool_worker(worker, pool) \ 408 list_for_each_entry((worker), &(pool)->workers, node) \ 409 if (({ lockdep_assert_held(&pool->attach_mutex); false; })) { } \ 410 else 411 412 /** 413 * for_each_pwq - iterate through all pool_workqueues of the specified workqueue 414 * @pwq: iteration cursor 415 * @wq: the target workqueue 416 * 417 * This must be called either with wq->mutex held or sched RCU read locked. 418 * If the pwq needs to be used beyond the locking in effect, the caller is 419 * responsible for guaranteeing that the pwq stays online. 420 * 421 * The if/else clause exists only for the lockdep assertion and can be 422 * ignored. 423 */ 424 #define for_each_pwq(pwq, wq) \ 425 list_for_each_entry_rcu((pwq), &(wq)->pwqs, pwqs_node) \ 426 if (({ assert_rcu_or_wq_mutex(wq); false; })) { } \ 427 else 428 429 #ifdef CONFIG_DEBUG_OBJECTS_WORK 430 431 static struct debug_obj_descr work_debug_descr; 432 433 static void *work_debug_hint(void *addr) 434 { 435 return ((struct work_struct *) addr)->func; 436 } 437 438 static bool work_is_static_object(void *addr) 439 { 440 struct work_struct *work = addr; 441 442 return test_bit(WORK_STRUCT_STATIC_BIT, work_data_bits(work)); 443 } 444 445 /* 446 * fixup_init is called when: 447 * - an active object is initialized 448 */ 449 static bool work_fixup_init(void *addr, enum debug_obj_state state) 450 { 451 struct work_struct *work = addr; 452 453 switch (state) { 454 case ODEBUG_STATE_ACTIVE: 455 cancel_work_sync(work); 456 debug_object_init(work, &work_debug_descr); 457 return true; 458 default: 459 return false; 460 } 461 } 462 463 /* 464 * fixup_free is called when: 465 * - an active object is freed 466 */ 467 static bool work_fixup_free(void *addr, enum debug_obj_state state) 468 { 469 struct work_struct *work = addr; 470 471 switch (state) { 472 case ODEBUG_STATE_ACTIVE: 473 cancel_work_sync(work); 474 debug_object_free(work, &work_debug_descr); 475 return true; 476 default: 477 return false; 478 } 479 } 480 481 static struct debug_obj_descr work_debug_descr = { 482 .name = "work_struct", 483 .debug_hint = work_debug_hint, 484 .is_static_object = work_is_static_object, 485 .fixup_init = work_fixup_init, 486 .fixup_free = work_fixup_free, 487 }; 488 489 static inline void debug_work_activate(struct work_struct *work) 490 { 491 debug_object_activate(work, &work_debug_descr); 492 } 493 494 static inline void debug_work_deactivate(struct work_struct *work) 495 { 496 debug_object_deactivate(work, &work_debug_descr); 497 } 498 499 void __init_work(struct work_struct *work, int onstack) 500 { 501 if (onstack) 502 debug_object_init_on_stack(work, &work_debug_descr); 503 else 504 debug_object_init(work, &work_debug_descr); 505 } 506 EXPORT_SYMBOL_GPL(__init_work); 507 508 void destroy_work_on_stack(struct work_struct *work) 509 { 510 debug_object_free(work, &work_debug_descr); 511 } 512 EXPORT_SYMBOL_GPL(destroy_work_on_stack); 513 514 void destroy_delayed_work_on_stack(struct delayed_work *work) 515 { 516 destroy_timer_on_stack(&work->timer); 517 debug_object_free(&work->work, &work_debug_descr); 518 } 519 EXPORT_SYMBOL_GPL(destroy_delayed_work_on_stack); 520 521 #else 522 static inline void debug_work_activate(struct work_struct *work) { } 523 static inline void debug_work_deactivate(struct work_struct *work) { } 524 #endif 525 526 /** 527 * worker_pool_assign_id - allocate ID and assing it to @pool 528 * @pool: the pool pointer of interest 529 * 530 * Returns 0 if ID in [0, WORK_OFFQ_POOL_NONE) is allocated and assigned 531 * successfully, -errno on failure. 532 */ 533 static int worker_pool_assign_id(struct worker_pool *pool) 534 { 535 int ret; 536 537 lockdep_assert_held(&wq_pool_mutex); 538 539 ret = idr_alloc(&worker_pool_idr, pool, 0, WORK_OFFQ_POOL_NONE, 540 GFP_KERNEL); 541 if (ret >= 0) { 542 pool->id = ret; 543 return 0; 544 } 545 return ret; 546 } 547 548 /** 549 * unbound_pwq_by_node - return the unbound pool_workqueue for the given node 550 * @wq: the target workqueue 551 * @node: the node ID 552 * 553 * This must be called with any of wq_pool_mutex, wq->mutex or sched RCU 554 * read locked. 555 * If the pwq needs to be used beyond the locking in effect, the caller is 556 * responsible for guaranteeing that the pwq stays online. 557 * 558 * Return: The unbound pool_workqueue for @node. 559 */ 560 static struct pool_workqueue *unbound_pwq_by_node(struct workqueue_struct *wq, 561 int node) 562 { 563 assert_rcu_or_wq_mutex_or_pool_mutex(wq); 564 565 /* 566 * XXX: @node can be NUMA_NO_NODE if CPU goes offline while a 567 * delayed item is pending. The plan is to keep CPU -> NODE 568 * mapping valid and stable across CPU on/offlines. Once that 569 * happens, this workaround can be removed. 570 */ 571 if (unlikely(node == NUMA_NO_NODE)) 572 return wq->dfl_pwq; 573 574 return rcu_dereference_raw(wq->numa_pwq_tbl[node]); 575 } 576 577 static unsigned int work_color_to_flags(int color) 578 { 579 return color << WORK_STRUCT_COLOR_SHIFT; 580 } 581 582 static int get_work_color(struct work_struct *work) 583 { 584 return (*work_data_bits(work) >> WORK_STRUCT_COLOR_SHIFT) & 585 ((1 << WORK_STRUCT_COLOR_BITS) - 1); 586 } 587 588 static int work_next_color(int color) 589 { 590 return (color + 1) % WORK_NR_COLORS; 591 } 592 593 /* 594 * While queued, %WORK_STRUCT_PWQ is set and non flag bits of a work's data 595 * contain the pointer to the queued pwq. Once execution starts, the flag 596 * is cleared and the high bits contain OFFQ flags and pool ID. 597 * 598 * set_work_pwq(), set_work_pool_and_clear_pending(), mark_work_canceling() 599 * and clear_work_data() can be used to set the pwq, pool or clear 600 * work->data. These functions should only be called while the work is 601 * owned - ie. while the PENDING bit is set. 602 * 603 * get_work_pool() and get_work_pwq() can be used to obtain the pool or pwq 604 * corresponding to a work. Pool is available once the work has been 605 * queued anywhere after initialization until it is sync canceled. pwq is 606 * available only while the work item is queued. 607 * 608 * %WORK_OFFQ_CANCELING is used to mark a work item which is being 609 * canceled. While being canceled, a work item may have its PENDING set 610 * but stay off timer and worklist for arbitrarily long and nobody should 611 * try to steal the PENDING bit. 612 */ 613 static inline void set_work_data(struct work_struct *work, unsigned long data, 614 unsigned long flags) 615 { 616 WARN_ON_ONCE(!work_pending(work)); 617 atomic_long_set(&work->data, data | flags | work_static(work)); 618 } 619 620 static void set_work_pwq(struct work_struct *work, struct pool_workqueue *pwq, 621 unsigned long extra_flags) 622 { 623 set_work_data(work, (unsigned long)pwq, 624 WORK_STRUCT_PENDING | WORK_STRUCT_PWQ | extra_flags); 625 } 626 627 static void set_work_pool_and_keep_pending(struct work_struct *work, 628 int pool_id) 629 { 630 set_work_data(work, (unsigned long)pool_id << WORK_OFFQ_POOL_SHIFT, 631 WORK_STRUCT_PENDING); 632 } 633 634 static void set_work_pool_and_clear_pending(struct work_struct *work, 635 int pool_id) 636 { 637 /* 638 * The following wmb is paired with the implied mb in 639 * test_and_set_bit(PENDING) and ensures all updates to @work made 640 * here are visible to and precede any updates by the next PENDING 641 * owner. 642 */ 643 smp_wmb(); 644 set_work_data(work, (unsigned long)pool_id << WORK_OFFQ_POOL_SHIFT, 0); 645 /* 646 * The following mb guarantees that previous clear of a PENDING bit 647 * will not be reordered with any speculative LOADS or STORES from 648 * work->current_func, which is executed afterwards. This possible 649 * reordering can lead to a missed execution on attempt to qeueue 650 * the same @work. E.g. consider this case: 651 * 652 * CPU#0 CPU#1 653 * ---------------------------- -------------------------------- 654 * 655 * 1 STORE event_indicated 656 * 2 queue_work_on() { 657 * 3 test_and_set_bit(PENDING) 658 * 4 } set_..._and_clear_pending() { 659 * 5 set_work_data() # clear bit 660 * 6 smp_mb() 661 * 7 work->current_func() { 662 * 8 LOAD event_indicated 663 * } 664 * 665 * Without an explicit full barrier speculative LOAD on line 8 can 666 * be executed before CPU#0 does STORE on line 1. If that happens, 667 * CPU#0 observes the PENDING bit is still set and new execution of 668 * a @work is not queued in a hope, that CPU#1 will eventually 669 * finish the queued @work. Meanwhile CPU#1 does not see 670 * event_indicated is set, because speculative LOAD was executed 671 * before actual STORE. 672 */ 673 smp_mb(); 674 } 675 676 static void clear_work_data(struct work_struct *work) 677 { 678 smp_wmb(); /* see set_work_pool_and_clear_pending() */ 679 set_work_data(work, WORK_STRUCT_NO_POOL, 0); 680 } 681 682 static struct pool_workqueue *get_work_pwq(struct work_struct *work) 683 { 684 unsigned long data = atomic_long_read(&work->data); 685 686 if (data & WORK_STRUCT_PWQ) 687 return (void *)(data & WORK_STRUCT_WQ_DATA_MASK); 688 else 689 return NULL; 690 } 691 692 /** 693 * get_work_pool - return the worker_pool a given work was associated with 694 * @work: the work item of interest 695 * 696 * Pools are created and destroyed under wq_pool_mutex, and allows read 697 * access under sched-RCU read lock. As such, this function should be 698 * called under wq_pool_mutex or with preemption disabled. 699 * 700 * All fields of the returned pool are accessible as long as the above 701 * mentioned locking is in effect. If the returned pool needs to be used 702 * beyond the critical section, the caller is responsible for ensuring the 703 * returned pool is and stays online. 704 * 705 * Return: The worker_pool @work was last associated with. %NULL if none. 706 */ 707 static struct worker_pool *get_work_pool(struct work_struct *work) 708 { 709 unsigned long data = atomic_long_read(&work->data); 710 int pool_id; 711 712 assert_rcu_or_pool_mutex(); 713 714 if (data & WORK_STRUCT_PWQ) 715 return ((struct pool_workqueue *) 716 (data & WORK_STRUCT_WQ_DATA_MASK))->pool; 717 718 pool_id = data >> WORK_OFFQ_POOL_SHIFT; 719 if (pool_id == WORK_OFFQ_POOL_NONE) 720 return NULL; 721 722 return idr_find(&worker_pool_idr, pool_id); 723 } 724 725 /** 726 * get_work_pool_id - return the worker pool ID a given work is associated with 727 * @work: the work item of interest 728 * 729 * Return: The worker_pool ID @work was last associated with. 730 * %WORK_OFFQ_POOL_NONE if none. 731 */ 732 static int get_work_pool_id(struct work_struct *work) 733 { 734 unsigned long data = atomic_long_read(&work->data); 735 736 if (data & WORK_STRUCT_PWQ) 737 return ((struct pool_workqueue *) 738 (data & WORK_STRUCT_WQ_DATA_MASK))->pool->id; 739 740 return data >> WORK_OFFQ_POOL_SHIFT; 741 } 742 743 static void mark_work_canceling(struct work_struct *work) 744 { 745 unsigned long pool_id = get_work_pool_id(work); 746 747 pool_id <<= WORK_OFFQ_POOL_SHIFT; 748 set_work_data(work, pool_id | WORK_OFFQ_CANCELING, WORK_STRUCT_PENDING); 749 } 750 751 static bool work_is_canceling(struct work_struct *work) 752 { 753 unsigned long data = atomic_long_read(&work->data); 754 755 return !(data & WORK_STRUCT_PWQ) && (data & WORK_OFFQ_CANCELING); 756 } 757 758 /* 759 * Policy functions. These define the policies on how the global worker 760 * pools are managed. Unless noted otherwise, these functions assume that 761 * they're being called with pool->lock held. 762 */ 763 764 static bool __need_more_worker(struct worker_pool *pool) 765 { 766 return !atomic_read(&pool->nr_running); 767 } 768 769 /* 770 * Need to wake up a worker? Called from anything but currently 771 * running workers. 772 * 773 * Note that, because unbound workers never contribute to nr_running, this 774 * function will always return %true for unbound pools as long as the 775 * worklist isn't empty. 776 */ 777 static bool need_more_worker(struct worker_pool *pool) 778 { 779 return !list_empty(&pool->worklist) && __need_more_worker(pool); 780 } 781 782 /* Can I start working? Called from busy but !running workers. */ 783 static bool may_start_working(struct worker_pool *pool) 784 { 785 return pool->nr_idle; 786 } 787 788 /* Do I need to keep working? Called from currently running workers. */ 789 static bool keep_working(struct worker_pool *pool) 790 { 791 return !list_empty(&pool->worklist) && 792 atomic_read(&pool->nr_running) <= 1; 793 } 794 795 /* Do we need a new worker? Called from manager. */ 796 static bool need_to_create_worker(struct worker_pool *pool) 797 { 798 return need_more_worker(pool) && !may_start_working(pool); 799 } 800 801 /* Do we have too many workers and should some go away? */ 802 static bool too_many_workers(struct worker_pool *pool) 803 { 804 bool managing = mutex_is_locked(&pool->manager_arb); 805 int nr_idle = pool->nr_idle + managing; /* manager is considered idle */ 806 int nr_busy = pool->nr_workers - nr_idle; 807 808 return nr_idle > 2 && (nr_idle - 2) * MAX_IDLE_WORKERS_RATIO >= nr_busy; 809 } 810 811 /* 812 * Wake up functions. 813 */ 814 815 /* Return the first idle worker. Safe with preemption disabled */ 816 static struct worker *first_idle_worker(struct worker_pool *pool) 817 { 818 if (unlikely(list_empty(&pool->idle_list))) 819 return NULL; 820 821 return list_first_entry(&pool->idle_list, struct worker, entry); 822 } 823 824 /** 825 * wake_up_worker - wake up an idle worker 826 * @pool: worker pool to wake worker from 827 * 828 * Wake up the first idle worker of @pool. 829 * 830 * CONTEXT: 831 * spin_lock_irq(pool->lock). 832 */ 833 static void wake_up_worker(struct worker_pool *pool) 834 { 835 struct worker *worker = first_idle_worker(pool); 836 837 if (likely(worker)) 838 wake_up_process(worker->task); 839 } 840 841 /** 842 * wq_worker_waking_up - a worker is waking up 843 * @task: task waking up 844 * @cpu: CPU @task is waking up to 845 * 846 * This function is called during try_to_wake_up() when a worker is 847 * being awoken. 848 * 849 * CONTEXT: 850 * spin_lock_irq(rq->lock) 851 */ 852 void wq_worker_waking_up(struct task_struct *task, int cpu) 853 { 854 struct worker *worker = kthread_data(task); 855 856 if (!(worker->flags & WORKER_NOT_RUNNING)) { 857 WARN_ON_ONCE(worker->pool->cpu != cpu); 858 atomic_inc(&worker->pool->nr_running); 859 } 860 } 861 862 /** 863 * wq_worker_sleeping - a worker is going to sleep 864 * @task: task going to sleep 865 * 866 * This function is called during schedule() when a busy worker is 867 * going to sleep. Worker on the same cpu can be woken up by 868 * returning pointer to its task. 869 * 870 * CONTEXT: 871 * spin_lock_irq(rq->lock) 872 * 873 * Return: 874 * Worker task on @cpu to wake up, %NULL if none. 875 */ 876 struct task_struct *wq_worker_sleeping(struct task_struct *task) 877 { 878 struct worker *worker = kthread_data(task), *to_wakeup = NULL; 879 struct worker_pool *pool; 880 881 /* 882 * Rescuers, which may not have all the fields set up like normal 883 * workers, also reach here, let's not access anything before 884 * checking NOT_RUNNING. 885 */ 886 if (worker->flags & WORKER_NOT_RUNNING) 887 return NULL; 888 889 pool = worker->pool; 890 891 /* this can only happen on the local cpu */ 892 if (WARN_ON_ONCE(pool->cpu != raw_smp_processor_id())) 893 return NULL; 894 895 /* 896 * The counterpart of the following dec_and_test, implied mb, 897 * worklist not empty test sequence is in insert_work(). 898 * Please read comment there. 899 * 900 * NOT_RUNNING is clear. This means that we're bound to and 901 * running on the local cpu w/ rq lock held and preemption 902 * disabled, which in turn means that none else could be 903 * manipulating idle_list, so dereferencing idle_list without pool 904 * lock is safe. 905 */ 906 if (atomic_dec_and_test(&pool->nr_running) && 907 !list_empty(&pool->worklist)) 908 to_wakeup = first_idle_worker(pool); 909 return to_wakeup ? to_wakeup->task : NULL; 910 } 911 912 /** 913 * worker_set_flags - set worker flags and adjust nr_running accordingly 914 * @worker: self 915 * @flags: flags to set 916 * 917 * Set @flags in @worker->flags and adjust nr_running accordingly. 918 * 919 * CONTEXT: 920 * spin_lock_irq(pool->lock) 921 */ 922 static inline void worker_set_flags(struct worker *worker, unsigned int flags) 923 { 924 struct worker_pool *pool = worker->pool; 925 926 WARN_ON_ONCE(worker->task != current); 927 928 /* If transitioning into NOT_RUNNING, adjust nr_running. */ 929 if ((flags & WORKER_NOT_RUNNING) && 930 !(worker->flags & WORKER_NOT_RUNNING)) { 931 atomic_dec(&pool->nr_running); 932 } 933 934 worker->flags |= flags; 935 } 936 937 /** 938 * worker_clr_flags - clear worker flags and adjust nr_running accordingly 939 * @worker: self 940 * @flags: flags to clear 941 * 942 * Clear @flags in @worker->flags and adjust nr_running accordingly. 943 * 944 * CONTEXT: 945 * spin_lock_irq(pool->lock) 946 */ 947 static inline void worker_clr_flags(struct worker *worker, unsigned int flags) 948 { 949 struct worker_pool *pool = worker->pool; 950 unsigned int oflags = worker->flags; 951 952 WARN_ON_ONCE(worker->task != current); 953 954 worker->flags &= ~flags; 955 956 /* 957 * If transitioning out of NOT_RUNNING, increment nr_running. Note 958 * that the nested NOT_RUNNING is not a noop. NOT_RUNNING is mask 959 * of multiple flags, not a single flag. 960 */ 961 if ((flags & WORKER_NOT_RUNNING) && (oflags & WORKER_NOT_RUNNING)) 962 if (!(worker->flags & WORKER_NOT_RUNNING)) 963 atomic_inc(&pool->nr_running); 964 } 965 966 /** 967 * find_worker_executing_work - find worker which is executing a work 968 * @pool: pool of interest 969 * @work: work to find worker for 970 * 971 * Find a worker which is executing @work on @pool by searching 972 * @pool->busy_hash which is keyed by the address of @work. For a worker 973 * to match, its current execution should match the address of @work and 974 * its work function. This is to avoid unwanted dependency between 975 * unrelated work executions through a work item being recycled while still 976 * being executed. 977 * 978 * This is a bit tricky. A work item may be freed once its execution 979 * starts and nothing prevents the freed area from being recycled for 980 * another work item. If the same work item address ends up being reused 981 * before the original execution finishes, workqueue will identify the 982 * recycled work item as currently executing and make it wait until the 983 * current execution finishes, introducing an unwanted dependency. 984 * 985 * This function checks the work item address and work function to avoid 986 * false positives. Note that this isn't complete as one may construct a 987 * work function which can introduce dependency onto itself through a 988 * recycled work item. Well, if somebody wants to shoot oneself in the 989 * foot that badly, there's only so much we can do, and if such deadlock 990 * actually occurs, it should be easy to locate the culprit work function. 991 * 992 * CONTEXT: 993 * spin_lock_irq(pool->lock). 994 * 995 * Return: 996 * Pointer to worker which is executing @work if found, %NULL 997 * otherwise. 998 */ 999 static struct worker *find_worker_executing_work(struct worker_pool *pool, 1000 struct work_struct *work) 1001 { 1002 struct worker *worker; 1003 1004 hash_for_each_possible(pool->busy_hash, worker, hentry, 1005 (unsigned long)work) 1006 if (worker->current_work == work && 1007 worker->current_func == work->func) 1008 return worker; 1009 1010 return NULL; 1011 } 1012 1013 /** 1014 * move_linked_works - move linked works to a list 1015 * @work: start of series of works to be scheduled 1016 * @head: target list to append @work to 1017 * @nextp: out parameter for nested worklist walking 1018 * 1019 * Schedule linked works starting from @work to @head. Work series to 1020 * be scheduled starts at @work and includes any consecutive work with 1021 * WORK_STRUCT_LINKED set in its predecessor. 1022 * 1023 * If @nextp is not NULL, it's updated to point to the next work of 1024 * the last scheduled work. This allows move_linked_works() to be 1025 * nested inside outer list_for_each_entry_safe(). 1026 * 1027 * CONTEXT: 1028 * spin_lock_irq(pool->lock). 1029 */ 1030 static void move_linked_works(struct work_struct *work, struct list_head *head, 1031 struct work_struct **nextp) 1032 { 1033 struct work_struct *n; 1034 1035 /* 1036 * Linked worklist will always end before the end of the list, 1037 * use NULL for list head. 1038 */ 1039 list_for_each_entry_safe_from(work, n, NULL, entry) { 1040 list_move_tail(&work->entry, head); 1041 if (!(*work_data_bits(work) & WORK_STRUCT_LINKED)) 1042 break; 1043 } 1044 1045 /* 1046 * If we're already inside safe list traversal and have moved 1047 * multiple works to the scheduled queue, the next position 1048 * needs to be updated. 1049 */ 1050 if (nextp) 1051 *nextp = n; 1052 } 1053 1054 /** 1055 * get_pwq - get an extra reference on the specified pool_workqueue 1056 * @pwq: pool_workqueue to get 1057 * 1058 * Obtain an extra reference on @pwq. The caller should guarantee that 1059 * @pwq has positive refcnt and be holding the matching pool->lock. 1060 */ 1061 static void get_pwq(struct pool_workqueue *pwq) 1062 { 1063 lockdep_assert_held(&pwq->pool->lock); 1064 WARN_ON_ONCE(pwq->refcnt <= 0); 1065 pwq->refcnt++; 1066 } 1067 1068 /** 1069 * put_pwq - put a pool_workqueue reference 1070 * @pwq: pool_workqueue to put 1071 * 1072 * Drop a reference of @pwq. If its refcnt reaches zero, schedule its 1073 * destruction. The caller should be holding the matching pool->lock. 1074 */ 1075 static void put_pwq(struct pool_workqueue *pwq) 1076 { 1077 lockdep_assert_held(&pwq->pool->lock); 1078 if (likely(--pwq->refcnt)) 1079 return; 1080 if (WARN_ON_ONCE(!(pwq->wq->flags & WQ_UNBOUND))) 1081 return; 1082 /* 1083 * @pwq can't be released under pool->lock, bounce to 1084 * pwq_unbound_release_workfn(). This never recurses on the same 1085 * pool->lock as this path is taken only for unbound workqueues and 1086 * the release work item is scheduled on a per-cpu workqueue. To 1087 * avoid lockdep warning, unbound pool->locks are given lockdep 1088 * subclass of 1 in get_unbound_pool(). 1089 */ 1090 schedule_work(&pwq->unbound_release_work); 1091 } 1092 1093 /** 1094 * put_pwq_unlocked - put_pwq() with surrounding pool lock/unlock 1095 * @pwq: pool_workqueue to put (can be %NULL) 1096 * 1097 * put_pwq() with locking. This function also allows %NULL @pwq. 1098 */ 1099 static void put_pwq_unlocked(struct pool_workqueue *pwq) 1100 { 1101 if (pwq) { 1102 /* 1103 * As both pwqs and pools are sched-RCU protected, the 1104 * following lock operations are safe. 1105 */ 1106 spin_lock_irq(&pwq->pool->lock); 1107 put_pwq(pwq); 1108 spin_unlock_irq(&pwq->pool->lock); 1109 } 1110 } 1111 1112 static void pwq_activate_delayed_work(struct work_struct *work) 1113 { 1114 struct pool_workqueue *pwq = get_work_pwq(work); 1115 1116 trace_workqueue_activate_work(work); 1117 if (list_empty(&pwq->pool->worklist)) 1118 pwq->pool->watchdog_ts = jiffies; 1119 move_linked_works(work, &pwq->pool->worklist, NULL); 1120 __clear_bit(WORK_STRUCT_DELAYED_BIT, work_data_bits(work)); 1121 pwq->nr_active++; 1122 } 1123 1124 static void pwq_activate_first_delayed(struct pool_workqueue *pwq) 1125 { 1126 struct work_struct *work = list_first_entry(&pwq->delayed_works, 1127 struct work_struct, entry); 1128 1129 pwq_activate_delayed_work(work); 1130 } 1131 1132 /** 1133 * pwq_dec_nr_in_flight - decrement pwq's nr_in_flight 1134 * @pwq: pwq of interest 1135 * @color: color of work which left the queue 1136 * 1137 * A work either has completed or is removed from pending queue, 1138 * decrement nr_in_flight of its pwq and handle workqueue flushing. 1139 * 1140 * CONTEXT: 1141 * spin_lock_irq(pool->lock). 1142 */ 1143 static void pwq_dec_nr_in_flight(struct pool_workqueue *pwq, int color) 1144 { 1145 /* uncolored work items don't participate in flushing or nr_active */ 1146 if (color == WORK_NO_COLOR) 1147 goto out_put; 1148 1149 pwq->nr_in_flight[color]--; 1150 1151 pwq->nr_active--; 1152 if (!list_empty(&pwq->delayed_works)) { 1153 /* one down, submit a delayed one */ 1154 if (pwq->nr_active < pwq->max_active) 1155 pwq_activate_first_delayed(pwq); 1156 } 1157 1158 /* is flush in progress and are we at the flushing tip? */ 1159 if (likely(pwq->flush_color != color)) 1160 goto out_put; 1161 1162 /* are there still in-flight works? */ 1163 if (pwq->nr_in_flight[color]) 1164 goto out_put; 1165 1166 /* this pwq is done, clear flush_color */ 1167 pwq->flush_color = -1; 1168 1169 /* 1170 * If this was the last pwq, wake up the first flusher. It 1171 * will handle the rest. 1172 */ 1173 if (atomic_dec_and_test(&pwq->wq->nr_pwqs_to_flush)) 1174 complete(&pwq->wq->first_flusher->done); 1175 out_put: 1176 put_pwq(pwq); 1177 } 1178 1179 /** 1180 * try_to_grab_pending - steal work item from worklist and disable irq 1181 * @work: work item to steal 1182 * @is_dwork: @work is a delayed_work 1183 * @flags: place to store irq state 1184 * 1185 * Try to grab PENDING bit of @work. This function can handle @work in any 1186 * stable state - idle, on timer or on worklist. 1187 * 1188 * Return: 1189 * 1 if @work was pending and we successfully stole PENDING 1190 * 0 if @work was idle and we claimed PENDING 1191 * -EAGAIN if PENDING couldn't be grabbed at the moment, safe to busy-retry 1192 * -ENOENT if someone else is canceling @work, this state may persist 1193 * for arbitrarily long 1194 * 1195 * Note: 1196 * On >= 0 return, the caller owns @work's PENDING bit. To avoid getting 1197 * interrupted while holding PENDING and @work off queue, irq must be 1198 * disabled on entry. This, combined with delayed_work->timer being 1199 * irqsafe, ensures that we return -EAGAIN for finite short period of time. 1200 * 1201 * On successful return, >= 0, irq is disabled and the caller is 1202 * responsible for releasing it using local_irq_restore(*@flags). 1203 * 1204 * This function is safe to call from any context including IRQ handler. 1205 */ 1206 static int try_to_grab_pending(struct work_struct *work, bool is_dwork, 1207 unsigned long *flags) 1208 { 1209 struct worker_pool *pool; 1210 struct pool_workqueue *pwq; 1211 1212 local_irq_save(*flags); 1213 1214 /* try to steal the timer if it exists */ 1215 if (is_dwork) { 1216 struct delayed_work *dwork = to_delayed_work(work); 1217 1218 /* 1219 * dwork->timer is irqsafe. If del_timer() fails, it's 1220 * guaranteed that the timer is not queued anywhere and not 1221 * running on the local CPU. 1222 */ 1223 if (likely(del_timer(&dwork->timer))) 1224 return 1; 1225 } 1226 1227 /* try to claim PENDING the normal way */ 1228 if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) 1229 return 0; 1230 1231 /* 1232 * The queueing is in progress, or it is already queued. Try to 1233 * steal it from ->worklist without clearing WORK_STRUCT_PENDING. 1234 */ 1235 pool = get_work_pool(work); 1236 if (!pool) 1237 goto fail; 1238 1239 spin_lock(&pool->lock); 1240 /* 1241 * work->data is guaranteed to point to pwq only while the work 1242 * item is queued on pwq->wq, and both updating work->data to point 1243 * to pwq on queueing and to pool on dequeueing are done under 1244 * pwq->pool->lock. This in turn guarantees that, if work->data 1245 * points to pwq which is associated with a locked pool, the work 1246 * item is currently queued on that pool. 1247 */ 1248 pwq = get_work_pwq(work); 1249 if (pwq && pwq->pool == pool) { 1250 debug_work_deactivate(work); 1251 1252 /* 1253 * A delayed work item cannot be grabbed directly because 1254 * it might have linked NO_COLOR work items which, if left 1255 * on the delayed_list, will confuse pwq->nr_active 1256 * management later on and cause stall. Make sure the work 1257 * item is activated before grabbing. 1258 */ 1259 if (*work_data_bits(work) & WORK_STRUCT_DELAYED) 1260 pwq_activate_delayed_work(work); 1261 1262 list_del_init(&work->entry); 1263 pwq_dec_nr_in_flight(pwq, get_work_color(work)); 1264 1265 /* work->data points to pwq iff queued, point to pool */ 1266 set_work_pool_and_keep_pending(work, pool->id); 1267 1268 spin_unlock(&pool->lock); 1269 return 1; 1270 } 1271 spin_unlock(&pool->lock); 1272 fail: 1273 local_irq_restore(*flags); 1274 if (work_is_canceling(work)) 1275 return -ENOENT; 1276 cpu_relax(); 1277 return -EAGAIN; 1278 } 1279 1280 /** 1281 * insert_work - insert a work into a pool 1282 * @pwq: pwq @work belongs to 1283 * @work: work to insert 1284 * @head: insertion point 1285 * @extra_flags: extra WORK_STRUCT_* flags to set 1286 * 1287 * Insert @work which belongs to @pwq after @head. @extra_flags is or'd to 1288 * work_struct flags. 1289 * 1290 * CONTEXT: 1291 * spin_lock_irq(pool->lock). 1292 */ 1293 static void insert_work(struct pool_workqueue *pwq, struct work_struct *work, 1294 struct list_head *head, unsigned int extra_flags) 1295 { 1296 struct worker_pool *pool = pwq->pool; 1297 1298 /* we own @work, set data and link */ 1299 set_work_pwq(work, pwq, extra_flags); 1300 list_add_tail(&work->entry, head); 1301 get_pwq(pwq); 1302 1303 /* 1304 * Ensure either wq_worker_sleeping() sees the above 1305 * list_add_tail() or we see zero nr_running to avoid workers lying 1306 * around lazily while there are works to be processed. 1307 */ 1308 smp_mb(); 1309 1310 if (__need_more_worker(pool)) 1311 wake_up_worker(pool); 1312 } 1313 1314 /* 1315 * Test whether @work is being queued from another work executing on the 1316 * same workqueue. 1317 */ 1318 static bool is_chained_work(struct workqueue_struct *wq) 1319 { 1320 struct worker *worker; 1321 1322 worker = current_wq_worker(); 1323 /* 1324 * Return %true iff I'm a worker execuing a work item on @wq. If 1325 * I'm @worker, it's safe to dereference it without locking. 1326 */ 1327 return worker && worker->current_pwq->wq == wq; 1328 } 1329 1330 /* 1331 * When queueing an unbound work item to a wq, prefer local CPU if allowed 1332 * by wq_unbound_cpumask. Otherwise, round robin among the allowed ones to 1333 * avoid perturbing sensitive tasks. 1334 */ 1335 static int wq_select_unbound_cpu(int cpu) 1336 { 1337 static bool printed_dbg_warning; 1338 int new_cpu; 1339 1340 if (likely(!wq_debug_force_rr_cpu)) { 1341 if (cpumask_test_cpu(cpu, wq_unbound_cpumask)) 1342 return cpu; 1343 } else if (!printed_dbg_warning) { 1344 pr_warn("workqueue: round-robin CPU selection forced, expect performance impact\n"); 1345 printed_dbg_warning = true; 1346 } 1347 1348 if (cpumask_empty(wq_unbound_cpumask)) 1349 return cpu; 1350 1351 new_cpu = __this_cpu_read(wq_rr_cpu_last); 1352 new_cpu = cpumask_next_and(new_cpu, wq_unbound_cpumask, cpu_online_mask); 1353 if (unlikely(new_cpu >= nr_cpu_ids)) { 1354 new_cpu = cpumask_first_and(wq_unbound_cpumask, cpu_online_mask); 1355 if (unlikely(new_cpu >= nr_cpu_ids)) 1356 return cpu; 1357 } 1358 __this_cpu_write(wq_rr_cpu_last, new_cpu); 1359 1360 return new_cpu; 1361 } 1362 1363 static void __queue_work(int cpu, struct workqueue_struct *wq, 1364 struct work_struct *work) 1365 { 1366 struct pool_workqueue *pwq; 1367 struct worker_pool *last_pool; 1368 struct list_head *worklist; 1369 unsigned int work_flags; 1370 unsigned int req_cpu = cpu; 1371 1372 /* 1373 * While a work item is PENDING && off queue, a task trying to 1374 * steal the PENDING will busy-loop waiting for it to either get 1375 * queued or lose PENDING. Grabbing PENDING and queueing should 1376 * happen with IRQ disabled. 1377 */ 1378 WARN_ON_ONCE(!irqs_disabled()); 1379 1380 debug_work_activate(work); 1381 1382 /* if draining, only works from the same workqueue are allowed */ 1383 if (unlikely(wq->flags & __WQ_DRAINING) && 1384 WARN_ON_ONCE(!is_chained_work(wq))) 1385 return; 1386 retry: 1387 if (req_cpu == WORK_CPU_UNBOUND) 1388 cpu = wq_select_unbound_cpu(raw_smp_processor_id()); 1389 1390 /* pwq which will be used unless @work is executing elsewhere */ 1391 if (!(wq->flags & WQ_UNBOUND)) 1392 pwq = per_cpu_ptr(wq->cpu_pwqs, cpu); 1393 else 1394 pwq = unbound_pwq_by_node(wq, cpu_to_node(cpu)); 1395 1396 /* 1397 * If @work was previously on a different pool, it might still be 1398 * running there, in which case the work needs to be queued on that 1399 * pool to guarantee non-reentrancy. 1400 */ 1401 last_pool = get_work_pool(work); 1402 if (last_pool && last_pool != pwq->pool) { 1403 struct worker *worker; 1404 1405 spin_lock(&last_pool->lock); 1406 1407 worker = find_worker_executing_work(last_pool, work); 1408 1409 if (worker && worker->current_pwq->wq == wq) { 1410 pwq = worker->current_pwq; 1411 } else { 1412 /* meh... not running there, queue here */ 1413 spin_unlock(&last_pool->lock); 1414 spin_lock(&pwq->pool->lock); 1415 } 1416 } else { 1417 spin_lock(&pwq->pool->lock); 1418 } 1419 1420 /* 1421 * pwq is determined and locked. For unbound pools, we could have 1422 * raced with pwq release and it could already be dead. If its 1423 * refcnt is zero, repeat pwq selection. Note that pwqs never die 1424 * without another pwq replacing it in the numa_pwq_tbl or while 1425 * work items are executing on it, so the retrying is guaranteed to 1426 * make forward-progress. 1427 */ 1428 if (unlikely(!pwq->refcnt)) { 1429 if (wq->flags & WQ_UNBOUND) { 1430 spin_unlock(&pwq->pool->lock); 1431 cpu_relax(); 1432 goto retry; 1433 } 1434 /* oops */ 1435 WARN_ONCE(true, "workqueue: per-cpu pwq for %s on cpu%d has 0 refcnt", 1436 wq->name, cpu); 1437 } 1438 1439 /* pwq determined, queue */ 1440 trace_workqueue_queue_work(req_cpu, pwq, work); 1441 1442 if (WARN_ON(!list_empty(&work->entry))) { 1443 spin_unlock(&pwq->pool->lock); 1444 return; 1445 } 1446 1447 pwq->nr_in_flight[pwq->work_color]++; 1448 work_flags = work_color_to_flags(pwq->work_color); 1449 1450 if (likely(pwq->nr_active < pwq->max_active)) { 1451 trace_workqueue_activate_work(work); 1452 pwq->nr_active++; 1453 worklist = &pwq->pool->worklist; 1454 if (list_empty(worklist)) 1455 pwq->pool->watchdog_ts = jiffies; 1456 } else { 1457 work_flags |= WORK_STRUCT_DELAYED; 1458 worklist = &pwq->delayed_works; 1459 } 1460 1461 insert_work(pwq, work, worklist, work_flags); 1462 1463 spin_unlock(&pwq->pool->lock); 1464 } 1465 1466 /** 1467 * queue_work_on - queue work on specific cpu 1468 * @cpu: CPU number to execute work on 1469 * @wq: workqueue to use 1470 * @work: work to queue 1471 * 1472 * We queue the work to a specific CPU, the caller must ensure it 1473 * can't go away. 1474 * 1475 * Return: %false if @work was already on a queue, %true otherwise. 1476 */ 1477 bool queue_work_on(int cpu, struct workqueue_struct *wq, 1478 struct work_struct *work) 1479 { 1480 bool ret = false; 1481 unsigned long flags; 1482 1483 local_irq_save(flags); 1484 1485 if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) { 1486 __queue_work(cpu, wq, work); 1487 ret = true; 1488 } 1489 1490 local_irq_restore(flags); 1491 return ret; 1492 } 1493 EXPORT_SYMBOL(queue_work_on); 1494 1495 void delayed_work_timer_fn(unsigned long __data) 1496 { 1497 struct delayed_work *dwork = (struct delayed_work *)__data; 1498 1499 /* should have been called from irqsafe timer with irq already off */ 1500 __queue_work(dwork->cpu, dwork->wq, &dwork->work); 1501 } 1502 EXPORT_SYMBOL(delayed_work_timer_fn); 1503 1504 static void __queue_delayed_work(int cpu, struct workqueue_struct *wq, 1505 struct delayed_work *dwork, unsigned long delay) 1506 { 1507 struct timer_list *timer = &dwork->timer; 1508 struct work_struct *work = &dwork->work; 1509 1510 WARN_ON_ONCE(!wq); 1511 WARN_ON_ONCE(timer->function != delayed_work_timer_fn || 1512 timer->data != (unsigned long)dwork); 1513 WARN_ON_ONCE(timer_pending(timer)); 1514 WARN_ON_ONCE(!list_empty(&work->entry)); 1515 1516 /* 1517 * If @delay is 0, queue @dwork->work immediately. This is for 1518 * both optimization and correctness. The earliest @timer can 1519 * expire is on the closest next tick and delayed_work users depend 1520 * on that there's no such delay when @delay is 0. 1521 */ 1522 if (!delay) { 1523 __queue_work(cpu, wq, &dwork->work); 1524 return; 1525 } 1526 1527 dwork->wq = wq; 1528 dwork->cpu = cpu; 1529 timer->expires = jiffies + delay; 1530 1531 if (unlikely(cpu != WORK_CPU_UNBOUND)) 1532 add_timer_on(timer, cpu); 1533 else 1534 add_timer(timer); 1535 } 1536 1537 /** 1538 * queue_delayed_work_on - queue work on specific CPU after delay 1539 * @cpu: CPU number to execute work on 1540 * @wq: workqueue to use 1541 * @dwork: work to queue 1542 * @delay: number of jiffies to wait before queueing 1543 * 1544 * Return: %false if @work was already on a queue, %true otherwise. If 1545 * @delay is zero and @dwork is idle, it will be scheduled for immediate 1546 * execution. 1547 */ 1548 bool queue_delayed_work_on(int cpu, struct workqueue_struct *wq, 1549 struct delayed_work *dwork, unsigned long delay) 1550 { 1551 struct work_struct *work = &dwork->work; 1552 bool ret = false; 1553 unsigned long flags; 1554 1555 /* read the comment in __queue_work() */ 1556 local_irq_save(flags); 1557 1558 if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) { 1559 __queue_delayed_work(cpu, wq, dwork, delay); 1560 ret = true; 1561 } 1562 1563 local_irq_restore(flags); 1564 return ret; 1565 } 1566 EXPORT_SYMBOL(queue_delayed_work_on); 1567 1568 /** 1569 * mod_delayed_work_on - modify delay of or queue a delayed work on specific CPU 1570 * @cpu: CPU number to execute work on 1571 * @wq: workqueue to use 1572 * @dwork: work to queue 1573 * @delay: number of jiffies to wait before queueing 1574 * 1575 * If @dwork is idle, equivalent to queue_delayed_work_on(); otherwise, 1576 * modify @dwork's timer so that it expires after @delay. If @delay is 1577 * zero, @work is guaranteed to be scheduled immediately regardless of its 1578 * current state. 1579 * 1580 * Return: %false if @dwork was idle and queued, %true if @dwork was 1581 * pending and its timer was modified. 1582 * 1583 * This function is safe to call from any context including IRQ handler. 1584 * See try_to_grab_pending() for details. 1585 */ 1586 bool mod_delayed_work_on(int cpu, struct workqueue_struct *wq, 1587 struct delayed_work *dwork, unsigned long delay) 1588 { 1589 unsigned long flags; 1590 int ret; 1591 1592 do { 1593 ret = try_to_grab_pending(&dwork->work, true, &flags); 1594 } while (unlikely(ret == -EAGAIN)); 1595 1596 if (likely(ret >= 0)) { 1597 __queue_delayed_work(cpu, wq, dwork, delay); 1598 local_irq_restore(flags); 1599 } 1600 1601 /* -ENOENT from try_to_grab_pending() becomes %true */ 1602 return ret; 1603 } 1604 EXPORT_SYMBOL_GPL(mod_delayed_work_on); 1605 1606 /** 1607 * worker_enter_idle - enter idle state 1608 * @worker: worker which is entering idle state 1609 * 1610 * @worker is entering idle state. Update stats and idle timer if 1611 * necessary. 1612 * 1613 * LOCKING: 1614 * spin_lock_irq(pool->lock). 1615 */ 1616 static void worker_enter_idle(struct worker *worker) 1617 { 1618 struct worker_pool *pool = worker->pool; 1619 1620 if (WARN_ON_ONCE(worker->flags & WORKER_IDLE) || 1621 WARN_ON_ONCE(!list_empty(&worker->entry) && 1622 (worker->hentry.next || worker->hentry.pprev))) 1623 return; 1624 1625 /* can't use worker_set_flags(), also called from create_worker() */ 1626 worker->flags |= WORKER_IDLE; 1627 pool->nr_idle++; 1628 worker->last_active = jiffies; 1629 1630 /* idle_list is LIFO */ 1631 list_add(&worker->entry, &pool->idle_list); 1632 1633 if (too_many_workers(pool) && !timer_pending(&pool->idle_timer)) 1634 mod_timer(&pool->idle_timer, jiffies + IDLE_WORKER_TIMEOUT); 1635 1636 /* 1637 * Sanity check nr_running. Because wq_unbind_fn() releases 1638 * pool->lock between setting %WORKER_UNBOUND and zapping 1639 * nr_running, the warning may trigger spuriously. Check iff 1640 * unbind is not in progress. 1641 */ 1642 WARN_ON_ONCE(!(pool->flags & POOL_DISASSOCIATED) && 1643 pool->nr_workers == pool->nr_idle && 1644 atomic_read(&pool->nr_running)); 1645 } 1646 1647 /** 1648 * worker_leave_idle - leave idle state 1649 * @worker: worker which is leaving idle state 1650 * 1651 * @worker is leaving idle state. Update stats. 1652 * 1653 * LOCKING: 1654 * spin_lock_irq(pool->lock). 1655 */ 1656 static void worker_leave_idle(struct worker *worker) 1657 { 1658 struct worker_pool *pool = worker->pool; 1659 1660 if (WARN_ON_ONCE(!(worker->flags & WORKER_IDLE))) 1661 return; 1662 worker_clr_flags(worker, WORKER_IDLE); 1663 pool->nr_idle--; 1664 list_del_init(&worker->entry); 1665 } 1666 1667 static struct worker *alloc_worker(int node) 1668 { 1669 struct worker *worker; 1670 1671 worker = kzalloc_node(sizeof(*worker), GFP_KERNEL, node); 1672 if (worker) { 1673 INIT_LIST_HEAD(&worker->entry); 1674 INIT_LIST_HEAD(&worker->scheduled); 1675 INIT_LIST_HEAD(&worker->node); 1676 /* on creation a worker is in !idle && prep state */ 1677 worker->flags = WORKER_PREP; 1678 } 1679 return worker; 1680 } 1681 1682 /** 1683 * worker_attach_to_pool() - attach a worker to a pool 1684 * @worker: worker to be attached 1685 * @pool: the target pool 1686 * 1687 * Attach @worker to @pool. Once attached, the %WORKER_UNBOUND flag and 1688 * cpu-binding of @worker are kept coordinated with the pool across 1689 * cpu-[un]hotplugs. 1690 */ 1691 static void worker_attach_to_pool(struct worker *worker, 1692 struct worker_pool *pool) 1693 { 1694 mutex_lock(&pool->attach_mutex); 1695 1696 /* 1697 * set_cpus_allowed_ptr() will fail if the cpumask doesn't have any 1698 * online CPUs. It'll be re-applied when any of the CPUs come up. 1699 */ 1700 set_cpus_allowed_ptr(worker->task, pool->attrs->cpumask); 1701 1702 /* 1703 * The pool->attach_mutex ensures %POOL_DISASSOCIATED remains 1704 * stable across this function. See the comments above the 1705 * flag definition for details. 1706 */ 1707 if (pool->flags & POOL_DISASSOCIATED) 1708 worker->flags |= WORKER_UNBOUND; 1709 1710 list_add_tail(&worker->node, &pool->workers); 1711 1712 mutex_unlock(&pool->attach_mutex); 1713 } 1714 1715 /** 1716 * worker_detach_from_pool() - detach a worker from its pool 1717 * @worker: worker which is attached to its pool 1718 * @pool: the pool @worker is attached to 1719 * 1720 * Undo the attaching which had been done in worker_attach_to_pool(). The 1721 * caller worker shouldn't access to the pool after detached except it has 1722 * other reference to the pool. 1723 */ 1724 static void worker_detach_from_pool(struct worker *worker, 1725 struct worker_pool *pool) 1726 { 1727 struct completion *detach_completion = NULL; 1728 1729 mutex_lock(&pool->attach_mutex); 1730 list_del(&worker->node); 1731 if (list_empty(&pool->workers)) 1732 detach_completion = pool->detach_completion; 1733 mutex_unlock(&pool->attach_mutex); 1734 1735 /* clear leftover flags without pool->lock after it is detached */ 1736 worker->flags &= ~(WORKER_UNBOUND | WORKER_REBOUND); 1737 1738 if (detach_completion) 1739 complete(detach_completion); 1740 } 1741 1742 /** 1743 * create_worker - create a new workqueue worker 1744 * @pool: pool the new worker will belong to 1745 * 1746 * Create and start a new worker which is attached to @pool. 1747 * 1748 * CONTEXT: 1749 * Might sleep. Does GFP_KERNEL allocations. 1750 * 1751 * Return: 1752 * Pointer to the newly created worker. 1753 */ 1754 static struct worker *create_worker(struct worker_pool *pool) 1755 { 1756 struct worker *worker = NULL; 1757 int id = -1; 1758 char id_buf[16]; 1759 1760 /* ID is needed to determine kthread name */ 1761 id = ida_simple_get(&pool->worker_ida, 0, 0, GFP_KERNEL); 1762 if (id < 0) 1763 goto fail; 1764 1765 worker = alloc_worker(pool->node); 1766 if (!worker) 1767 goto fail; 1768 1769 worker->pool = pool; 1770 worker->id = id; 1771 1772 if (pool->cpu >= 0) 1773 snprintf(id_buf, sizeof(id_buf), "%d:%d%s", pool->cpu, id, 1774 pool->attrs->nice < 0 ? "H" : ""); 1775 else 1776 snprintf(id_buf, sizeof(id_buf), "u%d:%d", pool->id, id); 1777 1778 worker->task = kthread_create_on_node(worker_thread, worker, pool->node, 1779 "kworker/%s", id_buf); 1780 if (IS_ERR(worker->task)) 1781 goto fail; 1782 1783 set_user_nice(worker->task, pool->attrs->nice); 1784 kthread_bind_mask(worker->task, pool->attrs->cpumask); 1785 1786 /* successful, attach the worker to the pool */ 1787 worker_attach_to_pool(worker, pool); 1788 1789 /* start the newly created worker */ 1790 spin_lock_irq(&pool->lock); 1791 worker->pool->nr_workers++; 1792 worker_enter_idle(worker); 1793 wake_up_process(worker->task); 1794 spin_unlock_irq(&pool->lock); 1795 1796 return worker; 1797 1798 fail: 1799 if (id >= 0) 1800 ida_simple_remove(&pool->worker_ida, id); 1801 kfree(worker); 1802 return NULL; 1803 } 1804 1805 /** 1806 * destroy_worker - destroy a workqueue worker 1807 * @worker: worker to be destroyed 1808 * 1809 * Destroy @worker and adjust @pool stats accordingly. The worker should 1810 * be idle. 1811 * 1812 * CONTEXT: 1813 * spin_lock_irq(pool->lock). 1814 */ 1815 static void destroy_worker(struct worker *worker) 1816 { 1817 struct worker_pool *pool = worker->pool; 1818 1819 lockdep_assert_held(&pool->lock); 1820 1821 /* sanity check frenzy */ 1822 if (WARN_ON(worker->current_work) || 1823 WARN_ON(!list_empty(&worker->scheduled)) || 1824 WARN_ON(!(worker->flags & WORKER_IDLE))) 1825 return; 1826 1827 pool->nr_workers--; 1828 pool->nr_idle--; 1829 1830 list_del_init(&worker->entry); 1831 worker->flags |= WORKER_DIE; 1832 wake_up_process(worker->task); 1833 } 1834 1835 static void idle_worker_timeout(unsigned long __pool) 1836 { 1837 struct worker_pool *pool = (void *)__pool; 1838 1839 spin_lock_irq(&pool->lock); 1840 1841 while (too_many_workers(pool)) { 1842 struct worker *worker; 1843 unsigned long expires; 1844 1845 /* idle_list is kept in LIFO order, check the last one */ 1846 worker = list_entry(pool->idle_list.prev, struct worker, entry); 1847 expires = worker->last_active + IDLE_WORKER_TIMEOUT; 1848 1849 if (time_before(jiffies, expires)) { 1850 mod_timer(&pool->idle_timer, expires); 1851 break; 1852 } 1853 1854 destroy_worker(worker); 1855 } 1856 1857 spin_unlock_irq(&pool->lock); 1858 } 1859 1860 static void send_mayday(struct work_struct *work) 1861 { 1862 struct pool_workqueue *pwq = get_work_pwq(work); 1863 struct workqueue_struct *wq = pwq->wq; 1864 1865 lockdep_assert_held(&wq_mayday_lock); 1866 1867 if (!wq->rescuer) 1868 return; 1869 1870 /* mayday mayday mayday */ 1871 if (list_empty(&pwq->mayday_node)) { 1872 /* 1873 * If @pwq is for an unbound wq, its base ref may be put at 1874 * any time due to an attribute change. Pin @pwq until the 1875 * rescuer is done with it. 1876 */ 1877 get_pwq(pwq); 1878 list_add_tail(&pwq->mayday_node, &wq->maydays); 1879 wake_up_process(wq->rescuer->task); 1880 } 1881 } 1882 1883 static void pool_mayday_timeout(unsigned long __pool) 1884 { 1885 struct worker_pool *pool = (void *)__pool; 1886 struct work_struct *work; 1887 1888 spin_lock_irq(&pool->lock); 1889 spin_lock(&wq_mayday_lock); /* for wq->maydays */ 1890 1891 if (need_to_create_worker(pool)) { 1892 /* 1893 * We've been trying to create a new worker but 1894 * haven't been successful. We might be hitting an 1895 * allocation deadlock. Send distress signals to 1896 * rescuers. 1897 */ 1898 list_for_each_entry(work, &pool->worklist, entry) 1899 send_mayday(work); 1900 } 1901 1902 spin_unlock(&wq_mayday_lock); 1903 spin_unlock_irq(&pool->lock); 1904 1905 mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INTERVAL); 1906 } 1907 1908 /** 1909 * maybe_create_worker - create a new worker if necessary 1910 * @pool: pool to create a new worker for 1911 * 1912 * Create a new worker for @pool if necessary. @pool is guaranteed to 1913 * have at least one idle worker on return from this function. If 1914 * creating a new worker takes longer than MAYDAY_INTERVAL, mayday is 1915 * sent to all rescuers with works scheduled on @pool to resolve 1916 * possible allocation deadlock. 1917 * 1918 * On return, need_to_create_worker() is guaranteed to be %false and 1919 * may_start_working() %true. 1920 * 1921 * LOCKING: 1922 * spin_lock_irq(pool->lock) which may be released and regrabbed 1923 * multiple times. Does GFP_KERNEL allocations. Called only from 1924 * manager. 1925 */ 1926 static void maybe_create_worker(struct worker_pool *pool) 1927 __releases(&pool->lock) 1928 __acquires(&pool->lock) 1929 { 1930 restart: 1931 spin_unlock_irq(&pool->lock); 1932 1933 /* if we don't make progress in MAYDAY_INITIAL_TIMEOUT, call for help */ 1934 mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INITIAL_TIMEOUT); 1935 1936 while (true) { 1937 if (create_worker(pool) || !need_to_create_worker(pool)) 1938 break; 1939 1940 schedule_timeout_interruptible(CREATE_COOLDOWN); 1941 1942 if (!need_to_create_worker(pool)) 1943 break; 1944 } 1945 1946 del_timer_sync(&pool->mayday_timer); 1947 spin_lock_irq(&pool->lock); 1948 /* 1949 * This is necessary even after a new worker was just successfully 1950 * created as @pool->lock was dropped and the new worker might have 1951 * already become busy. 1952 */ 1953 if (need_to_create_worker(pool)) 1954 goto restart; 1955 } 1956 1957 /** 1958 * manage_workers - manage worker pool 1959 * @worker: self 1960 * 1961 * Assume the manager role and manage the worker pool @worker belongs 1962 * to. At any given time, there can be only zero or one manager per 1963 * pool. The exclusion is handled automatically by this function. 1964 * 1965 * The caller can safely start processing works on false return. On 1966 * true return, it's guaranteed that need_to_create_worker() is false 1967 * and may_start_working() is true. 1968 * 1969 * CONTEXT: 1970 * spin_lock_irq(pool->lock) which may be released and regrabbed 1971 * multiple times. Does GFP_KERNEL allocations. 1972 * 1973 * Return: 1974 * %false if the pool doesn't need management and the caller can safely 1975 * start processing works, %true if management function was performed and 1976 * the conditions that the caller verified before calling the function may 1977 * no longer be true. 1978 */ 1979 static bool manage_workers(struct worker *worker) 1980 { 1981 struct worker_pool *pool = worker->pool; 1982 1983 /* 1984 * Anyone who successfully grabs manager_arb wins the arbitration 1985 * and becomes the manager. mutex_trylock() on pool->manager_arb 1986 * failure while holding pool->lock reliably indicates that someone 1987 * else is managing the pool and the worker which failed trylock 1988 * can proceed to executing work items. This means that anyone 1989 * grabbing manager_arb is responsible for actually performing 1990 * manager duties. If manager_arb is grabbed and released without 1991 * actual management, the pool may stall indefinitely. 1992 */ 1993 if (!mutex_trylock(&pool->manager_arb)) 1994 return false; 1995 pool->manager = worker; 1996 1997 maybe_create_worker(pool); 1998 1999 pool->manager = NULL; 2000 mutex_unlock(&pool->manager_arb); 2001 return true; 2002 } 2003 2004 /** 2005 * process_one_work - process single work 2006 * @worker: self 2007 * @work: work to process 2008 * 2009 * Process @work. This function contains all the logics necessary to 2010 * process a single work including synchronization against and 2011 * interaction with other workers on the same cpu, queueing and 2012 * flushing. As long as context requirement is met, any worker can 2013 * call this function to process a work. 2014 * 2015 * CONTEXT: 2016 * spin_lock_irq(pool->lock) which is released and regrabbed. 2017 */ 2018 static void process_one_work(struct worker *worker, struct work_struct *work) 2019 __releases(&pool->lock) 2020 __acquires(&pool->lock) 2021 { 2022 struct pool_workqueue *pwq = get_work_pwq(work); 2023 struct worker_pool *pool = worker->pool; 2024 bool cpu_intensive = pwq->wq->flags & WQ_CPU_INTENSIVE; 2025 int work_color; 2026 struct worker *collision; 2027 #ifdef CONFIG_LOCKDEP 2028 /* 2029 * It is permissible to free the struct work_struct from 2030 * inside the function that is called from it, this we need to 2031 * take into account for lockdep too. To avoid bogus "held 2032 * lock freed" warnings as well as problems when looking into 2033 * work->lockdep_map, make a copy and use that here. 2034 */ 2035 struct lockdep_map lockdep_map; 2036 2037 lockdep_copy_map(&lockdep_map, &work->lockdep_map); 2038 #endif 2039 /* ensure we're on the correct CPU */ 2040 WARN_ON_ONCE(!(pool->flags & POOL_DISASSOCIATED) && 2041 raw_smp_processor_id() != pool->cpu); 2042 2043 /* 2044 * A single work shouldn't be executed concurrently by 2045 * multiple workers on a single cpu. Check whether anyone is 2046 * already processing the work. If so, defer the work to the 2047 * currently executing one. 2048 */ 2049 collision = find_worker_executing_work(pool, work); 2050 if (unlikely(collision)) { 2051 move_linked_works(work, &collision->scheduled, NULL); 2052 return; 2053 } 2054 2055 /* claim and dequeue */ 2056 debug_work_deactivate(work); 2057 hash_add(pool->busy_hash, &worker->hentry, (unsigned long)work); 2058 worker->current_work = work; 2059 worker->current_func = work->func; 2060 worker->current_pwq = pwq; 2061 work_color = get_work_color(work); 2062 2063 list_del_init(&work->entry); 2064 2065 /* 2066 * CPU intensive works don't participate in concurrency management. 2067 * They're the scheduler's responsibility. This takes @worker out 2068 * of concurrency management and the next code block will chain 2069 * execution of the pending work items. 2070 */ 2071 if (unlikely(cpu_intensive)) 2072 worker_set_flags(worker, WORKER_CPU_INTENSIVE); 2073 2074 /* 2075 * Wake up another worker if necessary. The condition is always 2076 * false for normal per-cpu workers since nr_running would always 2077 * be >= 1 at this point. This is used to chain execution of the 2078 * pending work items for WORKER_NOT_RUNNING workers such as the 2079 * UNBOUND and CPU_INTENSIVE ones. 2080 */ 2081 if (need_more_worker(pool)) 2082 wake_up_worker(pool); 2083 2084 /* 2085 * Record the last pool and clear PENDING which should be the last 2086 * update to @work. Also, do this inside @pool->lock so that 2087 * PENDING and queued state changes happen together while IRQ is 2088 * disabled. 2089 */ 2090 set_work_pool_and_clear_pending(work, pool->id); 2091 2092 spin_unlock_irq(&pool->lock); 2093 2094 lock_map_acquire_read(&pwq->wq->lockdep_map); 2095 lock_map_acquire(&lockdep_map); 2096 trace_workqueue_execute_start(work); 2097 worker->current_func(work); 2098 /* 2099 * While we must be careful to not use "work" after this, the trace 2100 * point will only record its address. 2101 */ 2102 trace_workqueue_execute_end(work); 2103 lock_map_release(&lockdep_map); 2104 lock_map_release(&pwq->wq->lockdep_map); 2105 2106 if (unlikely(in_atomic() || lockdep_depth(current) > 0)) { 2107 pr_err("BUG: workqueue leaked lock or atomic: %s/0x%08x/%d\n" 2108 " last function: %pf\n", 2109 current->comm, preempt_count(), task_pid_nr(current), 2110 worker->current_func); 2111 debug_show_held_locks(current); 2112 dump_stack(); 2113 } 2114 2115 /* 2116 * The following prevents a kworker from hogging CPU on !PREEMPT 2117 * kernels, where a requeueing work item waiting for something to 2118 * happen could deadlock with stop_machine as such work item could 2119 * indefinitely requeue itself while all other CPUs are trapped in 2120 * stop_machine. At the same time, report a quiescent RCU state so 2121 * the same condition doesn't freeze RCU. 2122 */ 2123 cond_resched_rcu_qs(); 2124 2125 spin_lock_irq(&pool->lock); 2126 2127 /* clear cpu intensive status */ 2128 if (unlikely(cpu_intensive)) 2129 worker_clr_flags(worker, WORKER_CPU_INTENSIVE); 2130 2131 /* we're done with it, release */ 2132 hash_del(&worker->hentry); 2133 worker->current_work = NULL; 2134 worker->current_func = NULL; 2135 worker->current_pwq = NULL; 2136 worker->desc_valid = false; 2137 pwq_dec_nr_in_flight(pwq, work_color); 2138 } 2139 2140 /** 2141 * process_scheduled_works - process scheduled works 2142 * @worker: self 2143 * 2144 * Process all scheduled works. Please note that the scheduled list 2145 * may change while processing a work, so this function repeatedly 2146 * fetches a work from the top and executes it. 2147 * 2148 * CONTEXT: 2149 * spin_lock_irq(pool->lock) which may be released and regrabbed 2150 * multiple times. 2151 */ 2152 static void process_scheduled_works(struct worker *worker) 2153 { 2154 while (!list_empty(&worker->scheduled)) { 2155 struct work_struct *work = list_first_entry(&worker->scheduled, 2156 struct work_struct, entry); 2157 process_one_work(worker, work); 2158 } 2159 } 2160 2161 /** 2162 * worker_thread - the worker thread function 2163 * @__worker: self 2164 * 2165 * The worker thread function. All workers belong to a worker_pool - 2166 * either a per-cpu one or dynamic unbound one. These workers process all 2167 * work items regardless of their specific target workqueue. The only 2168 * exception is work items which belong to workqueues with a rescuer which 2169 * will be explained in rescuer_thread(). 2170 * 2171 * Return: 0 2172 */ 2173 static int worker_thread(void *__worker) 2174 { 2175 struct worker *worker = __worker; 2176 struct worker_pool *pool = worker->pool; 2177 2178 /* tell the scheduler that this is a workqueue worker */ 2179 worker->task->flags |= PF_WQ_WORKER; 2180 woke_up: 2181 spin_lock_irq(&pool->lock); 2182 2183 /* am I supposed to die? */ 2184 if (unlikely(worker->flags & WORKER_DIE)) { 2185 spin_unlock_irq(&pool->lock); 2186 WARN_ON_ONCE(!list_empty(&worker->entry)); 2187 worker->task->flags &= ~PF_WQ_WORKER; 2188 2189 set_task_comm(worker->task, "kworker/dying"); 2190 ida_simple_remove(&pool->worker_ida, worker->id); 2191 worker_detach_from_pool(worker, pool); 2192 kfree(worker); 2193 return 0; 2194 } 2195 2196 worker_leave_idle(worker); 2197 recheck: 2198 /* no more worker necessary? */ 2199 if (!need_more_worker(pool)) 2200 goto sleep; 2201 2202 /* do we need to manage? */ 2203 if (unlikely(!may_start_working(pool)) && manage_workers(worker)) 2204 goto recheck; 2205 2206 /* 2207 * ->scheduled list can only be filled while a worker is 2208 * preparing to process a work or actually processing it. 2209 * Make sure nobody diddled with it while I was sleeping. 2210 */ 2211 WARN_ON_ONCE(!list_empty(&worker->scheduled)); 2212 2213 /* 2214 * Finish PREP stage. We're guaranteed to have at least one idle 2215 * worker or that someone else has already assumed the manager 2216 * role. This is where @worker starts participating in concurrency 2217 * management if applicable and concurrency management is restored 2218 * after being rebound. See rebind_workers() for details. 2219 */ 2220 worker_clr_flags(worker, WORKER_PREP | WORKER_REBOUND); 2221 2222 do { 2223 struct work_struct *work = 2224 list_first_entry(&pool->worklist, 2225 struct work_struct, entry); 2226 2227 pool->watchdog_ts = jiffies; 2228 2229 if (likely(!(*work_data_bits(work) & WORK_STRUCT_LINKED))) { 2230 /* optimization path, not strictly necessary */ 2231 process_one_work(worker, work); 2232 if (unlikely(!list_empty(&worker->scheduled))) 2233 process_scheduled_works(worker); 2234 } else { 2235 move_linked_works(work, &worker->scheduled, NULL); 2236 process_scheduled_works(worker); 2237 } 2238 } while (keep_working(pool)); 2239 2240 worker_set_flags(worker, WORKER_PREP); 2241 sleep: 2242 /* 2243 * pool->lock is held and there's no work to process and no need to 2244 * manage, sleep. Workers are woken up only while holding 2245 * pool->lock or from local cpu, so setting the current state 2246 * before releasing pool->lock is enough to prevent losing any 2247 * event. 2248 */ 2249 worker_enter_idle(worker); 2250 __set_current_state(TASK_INTERRUPTIBLE); 2251 spin_unlock_irq(&pool->lock); 2252 schedule(); 2253 goto woke_up; 2254 } 2255 2256 /** 2257 * rescuer_thread - the rescuer thread function 2258 * @__rescuer: self 2259 * 2260 * Workqueue rescuer thread function. There's one rescuer for each 2261 * workqueue which has WQ_MEM_RECLAIM set. 2262 * 2263 * Regular work processing on a pool may block trying to create a new 2264 * worker which uses GFP_KERNEL allocation which has slight chance of 2265 * developing into deadlock if some works currently on the same queue 2266 * need to be processed to satisfy the GFP_KERNEL allocation. This is 2267 * the problem rescuer solves. 2268 * 2269 * When such condition is possible, the pool summons rescuers of all 2270 * workqueues which have works queued on the pool and let them process 2271 * those works so that forward progress can be guaranteed. 2272 * 2273 * This should happen rarely. 2274 * 2275 * Return: 0 2276 */ 2277 static int rescuer_thread(void *__rescuer) 2278 { 2279 struct worker *rescuer = __rescuer; 2280 struct workqueue_struct *wq = rescuer->rescue_wq; 2281 struct list_head *scheduled = &rescuer->scheduled; 2282 bool should_stop; 2283 2284 set_user_nice(current, RESCUER_NICE_LEVEL); 2285 2286 /* 2287 * Mark rescuer as worker too. As WORKER_PREP is never cleared, it 2288 * doesn't participate in concurrency management. 2289 */ 2290 rescuer->task->flags |= PF_WQ_WORKER; 2291 repeat: 2292 set_current_state(TASK_INTERRUPTIBLE); 2293 2294 /* 2295 * By the time the rescuer is requested to stop, the workqueue 2296 * shouldn't have any work pending, but @wq->maydays may still have 2297 * pwq(s) queued. This can happen by non-rescuer workers consuming 2298 * all the work items before the rescuer got to them. Go through 2299 * @wq->maydays processing before acting on should_stop so that the 2300 * list is always empty on exit. 2301 */ 2302 should_stop = kthread_should_stop(); 2303 2304 /* see whether any pwq is asking for help */ 2305 spin_lock_irq(&wq_mayday_lock); 2306 2307 while (!list_empty(&wq->maydays)) { 2308 struct pool_workqueue *pwq = list_first_entry(&wq->maydays, 2309 struct pool_workqueue, mayday_node); 2310 struct worker_pool *pool = pwq->pool; 2311 struct work_struct *work, *n; 2312 bool first = true; 2313 2314 __set_current_state(TASK_RUNNING); 2315 list_del_init(&pwq->mayday_node); 2316 2317 spin_unlock_irq(&wq_mayday_lock); 2318 2319 worker_attach_to_pool(rescuer, pool); 2320 2321 spin_lock_irq(&pool->lock); 2322 rescuer->pool = pool; 2323 2324 /* 2325 * Slurp in all works issued via this workqueue and 2326 * process'em. 2327 */ 2328 WARN_ON_ONCE(!list_empty(scheduled)); 2329 list_for_each_entry_safe(work, n, &pool->worklist, entry) { 2330 if (get_work_pwq(work) == pwq) { 2331 if (first) 2332 pool->watchdog_ts = jiffies; 2333 move_linked_works(work, scheduled, &n); 2334 } 2335 first = false; 2336 } 2337 2338 if (!list_empty(scheduled)) { 2339 process_scheduled_works(rescuer); 2340 2341 /* 2342 * The above execution of rescued work items could 2343 * have created more to rescue through 2344 * pwq_activate_first_delayed() or chained 2345 * queueing. Let's put @pwq back on mayday list so 2346 * that such back-to-back work items, which may be 2347 * being used to relieve memory pressure, don't 2348 * incur MAYDAY_INTERVAL delay inbetween. 2349 */ 2350 if (need_to_create_worker(pool)) { 2351 spin_lock(&wq_mayday_lock); 2352 get_pwq(pwq); 2353 list_move_tail(&pwq->mayday_node, &wq->maydays); 2354 spin_unlock(&wq_mayday_lock); 2355 } 2356 } 2357 2358 /* 2359 * Put the reference grabbed by send_mayday(). @pool won't 2360 * go away while we're still attached to it. 2361 */ 2362 put_pwq(pwq); 2363 2364 /* 2365 * Leave this pool. If need_more_worker() is %true, notify a 2366 * regular worker; otherwise, we end up with 0 concurrency 2367 * and stalling the execution. 2368 */ 2369 if (need_more_worker(pool)) 2370 wake_up_worker(pool); 2371 2372 rescuer->pool = NULL; 2373 spin_unlock_irq(&pool->lock); 2374 2375 worker_detach_from_pool(rescuer, pool); 2376 2377 spin_lock_irq(&wq_mayday_lock); 2378 } 2379 2380 spin_unlock_irq(&wq_mayday_lock); 2381 2382 if (should_stop) { 2383 __set_current_state(TASK_RUNNING); 2384 rescuer->task->flags &= ~PF_WQ_WORKER; 2385 return 0; 2386 } 2387 2388 /* rescuers should never participate in concurrency management */ 2389 WARN_ON_ONCE(!(rescuer->flags & WORKER_NOT_RUNNING)); 2390 schedule(); 2391 goto repeat; 2392 } 2393 2394 /** 2395 * check_flush_dependency - check for flush dependency sanity 2396 * @target_wq: workqueue being flushed 2397 * @target_work: work item being flushed (NULL for workqueue flushes) 2398 * 2399 * %current is trying to flush the whole @target_wq or @target_work on it. 2400 * If @target_wq doesn't have %WQ_MEM_RECLAIM, verify that %current is not 2401 * reclaiming memory or running on a workqueue which doesn't have 2402 * %WQ_MEM_RECLAIM as that can break forward-progress guarantee leading to 2403 * a deadlock. 2404 */ 2405 static void check_flush_dependency(struct workqueue_struct *target_wq, 2406 struct work_struct *target_work) 2407 { 2408 work_func_t target_func = target_work ? target_work->func : NULL; 2409 struct worker *worker; 2410 2411 if (target_wq->flags & WQ_MEM_RECLAIM) 2412 return; 2413 2414 worker = current_wq_worker(); 2415 2416 WARN_ONCE(current->flags & PF_MEMALLOC, 2417 "workqueue: PF_MEMALLOC task %d(%s) is flushing !WQ_MEM_RECLAIM %s:%pf", 2418 current->pid, current->comm, target_wq->name, target_func); 2419 WARN_ONCE(worker && ((worker->current_pwq->wq->flags & 2420 (WQ_MEM_RECLAIM | __WQ_LEGACY)) == WQ_MEM_RECLAIM), 2421 "workqueue: WQ_MEM_RECLAIM %s:%pf is flushing !WQ_MEM_RECLAIM %s:%pf", 2422 worker->current_pwq->wq->name, worker->current_func, 2423 target_wq->name, target_func); 2424 } 2425 2426 struct wq_barrier { 2427 struct work_struct work; 2428 struct completion done; 2429 struct task_struct *task; /* purely informational */ 2430 }; 2431 2432 static void wq_barrier_func(struct work_struct *work) 2433 { 2434 struct wq_barrier *barr = container_of(work, struct wq_barrier, work); 2435 complete(&barr->done); 2436 } 2437 2438 /** 2439 * insert_wq_barrier - insert a barrier work 2440 * @pwq: pwq to insert barrier into 2441 * @barr: wq_barrier to insert 2442 * @target: target work to attach @barr to 2443 * @worker: worker currently executing @target, NULL if @target is not executing 2444 * 2445 * @barr is linked to @target such that @barr is completed only after 2446 * @target finishes execution. Please note that the ordering 2447 * guarantee is observed only with respect to @target and on the local 2448 * cpu. 2449 * 2450 * Currently, a queued barrier can't be canceled. This is because 2451 * try_to_grab_pending() can't determine whether the work to be 2452 * grabbed is at the head of the queue and thus can't clear LINKED 2453 * flag of the previous work while there must be a valid next work 2454 * after a work with LINKED flag set. 2455 * 2456 * Note that when @worker is non-NULL, @target may be modified 2457 * underneath us, so we can't reliably determine pwq from @target. 2458 * 2459 * CONTEXT: 2460 * spin_lock_irq(pool->lock). 2461 */ 2462 static void insert_wq_barrier(struct pool_workqueue *pwq, 2463 struct wq_barrier *barr, 2464 struct work_struct *target, struct worker *worker) 2465 { 2466 struct list_head *head; 2467 unsigned int linked = 0; 2468 2469 /* 2470 * debugobject calls are safe here even with pool->lock locked 2471 * as we know for sure that this will not trigger any of the 2472 * checks and call back into the fixup functions where we 2473 * might deadlock. 2474 */ 2475 INIT_WORK_ONSTACK(&barr->work, wq_barrier_func); 2476 __set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(&barr->work)); 2477 init_completion(&barr->done); 2478 barr->task = current; 2479 2480 /* 2481 * If @target is currently being executed, schedule the 2482 * barrier to the worker; otherwise, put it after @target. 2483 */ 2484 if (worker) 2485 head = worker->scheduled.next; 2486 else { 2487 unsigned long *bits = work_data_bits(target); 2488 2489 head = target->entry.next; 2490 /* there can already be other linked works, inherit and set */ 2491 linked = *bits & WORK_STRUCT_LINKED; 2492 __set_bit(WORK_STRUCT_LINKED_BIT, bits); 2493 } 2494 2495 debug_work_activate(&barr->work); 2496 insert_work(pwq, &barr->work, head, 2497 work_color_to_flags(WORK_NO_COLOR) | linked); 2498 } 2499 2500 /** 2501 * flush_workqueue_prep_pwqs - prepare pwqs for workqueue flushing 2502 * @wq: workqueue being flushed 2503 * @flush_color: new flush color, < 0 for no-op 2504 * @work_color: new work color, < 0 for no-op 2505 * 2506 * Prepare pwqs for workqueue flushing. 2507 * 2508 * If @flush_color is non-negative, flush_color on all pwqs should be 2509 * -1. If no pwq has in-flight commands at the specified color, all 2510 * pwq->flush_color's stay at -1 and %false is returned. If any pwq 2511 * has in flight commands, its pwq->flush_color is set to 2512 * @flush_color, @wq->nr_pwqs_to_flush is updated accordingly, pwq 2513 * wakeup logic is armed and %true is returned. 2514 * 2515 * The caller should have initialized @wq->first_flusher prior to 2516 * calling this function with non-negative @flush_color. If 2517 * @flush_color is negative, no flush color update is done and %false 2518 * is returned. 2519 * 2520 * If @work_color is non-negative, all pwqs should have the same 2521 * work_color which is previous to @work_color and all will be 2522 * advanced to @work_color. 2523 * 2524 * CONTEXT: 2525 * mutex_lock(wq->mutex). 2526 * 2527 * Return: 2528 * %true if @flush_color >= 0 and there's something to flush. %false 2529 * otherwise. 2530 */ 2531 static bool flush_workqueue_prep_pwqs(struct workqueue_struct *wq, 2532 int flush_color, int work_color) 2533 { 2534 bool wait = false; 2535 struct pool_workqueue *pwq; 2536 2537 if (flush_color >= 0) { 2538 WARN_ON_ONCE(atomic_read(&wq->nr_pwqs_to_flush)); 2539 atomic_set(&wq->nr_pwqs_to_flush, 1); 2540 } 2541 2542 for_each_pwq(pwq, wq) { 2543 struct worker_pool *pool = pwq->pool; 2544 2545 spin_lock_irq(&pool->lock); 2546 2547 if (flush_color >= 0) { 2548 WARN_ON_ONCE(pwq->flush_color != -1); 2549 2550 if (pwq->nr_in_flight[flush_color]) { 2551 pwq->flush_color = flush_color; 2552 atomic_inc(&wq->nr_pwqs_to_flush); 2553 wait = true; 2554 } 2555 } 2556 2557 if (work_color >= 0) { 2558 WARN_ON_ONCE(work_color != work_next_color(pwq->work_color)); 2559 pwq->work_color = work_color; 2560 } 2561 2562 spin_unlock_irq(&pool->lock); 2563 } 2564 2565 if (flush_color >= 0 && atomic_dec_and_test(&wq->nr_pwqs_to_flush)) 2566 complete(&wq->first_flusher->done); 2567 2568 return wait; 2569 } 2570 2571 /** 2572 * flush_workqueue - ensure that any scheduled work has run to completion. 2573 * @wq: workqueue to flush 2574 * 2575 * This function sleeps until all work items which were queued on entry 2576 * have finished execution, but it is not livelocked by new incoming ones. 2577 */ 2578 void flush_workqueue(struct workqueue_struct *wq) 2579 { 2580 struct wq_flusher this_flusher = { 2581 .list = LIST_HEAD_INIT(this_flusher.list), 2582 .flush_color = -1, 2583 .done = COMPLETION_INITIALIZER_ONSTACK(this_flusher.done), 2584 }; 2585 int next_color; 2586 2587 if (WARN_ON(!wq_online)) 2588 return; 2589 2590 lock_map_acquire(&wq->lockdep_map); 2591 lock_map_release(&wq->lockdep_map); 2592 2593 mutex_lock(&wq->mutex); 2594 2595 /* 2596 * Start-to-wait phase 2597 */ 2598 next_color = work_next_color(wq->work_color); 2599 2600 if (next_color != wq->flush_color) { 2601 /* 2602 * Color space is not full. The current work_color 2603 * becomes our flush_color and work_color is advanced 2604 * by one. 2605 */ 2606 WARN_ON_ONCE(!list_empty(&wq->flusher_overflow)); 2607 this_flusher.flush_color = wq->work_color; 2608 wq->work_color = next_color; 2609 2610 if (!wq->first_flusher) { 2611 /* no flush in progress, become the first flusher */ 2612 WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color); 2613 2614 wq->first_flusher = &this_flusher; 2615 2616 if (!flush_workqueue_prep_pwqs(wq, wq->flush_color, 2617 wq->work_color)) { 2618 /* nothing to flush, done */ 2619 wq->flush_color = next_color; 2620 wq->first_flusher = NULL; 2621 goto out_unlock; 2622 } 2623 } else { 2624 /* wait in queue */ 2625 WARN_ON_ONCE(wq->flush_color == this_flusher.flush_color); 2626 list_add_tail(&this_flusher.list, &wq->flusher_queue); 2627 flush_workqueue_prep_pwqs(wq, -1, wq->work_color); 2628 } 2629 } else { 2630 /* 2631 * Oops, color space is full, wait on overflow queue. 2632 * The next flush completion will assign us 2633 * flush_color and transfer to flusher_queue. 2634 */ 2635 list_add_tail(&this_flusher.list, &wq->flusher_overflow); 2636 } 2637 2638 check_flush_dependency(wq, NULL); 2639 2640 mutex_unlock(&wq->mutex); 2641 2642 wait_for_completion(&this_flusher.done); 2643 2644 /* 2645 * Wake-up-and-cascade phase 2646 * 2647 * First flushers are responsible for cascading flushes and 2648 * handling overflow. Non-first flushers can simply return. 2649 */ 2650 if (wq->first_flusher != &this_flusher) 2651 return; 2652 2653 mutex_lock(&wq->mutex); 2654 2655 /* we might have raced, check again with mutex held */ 2656 if (wq->first_flusher != &this_flusher) 2657 goto out_unlock; 2658 2659 wq->first_flusher = NULL; 2660 2661 WARN_ON_ONCE(!list_empty(&this_flusher.list)); 2662 WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color); 2663 2664 while (true) { 2665 struct wq_flusher *next, *tmp; 2666 2667 /* complete all the flushers sharing the current flush color */ 2668 list_for_each_entry_safe(next, tmp, &wq->flusher_queue, list) { 2669 if (next->flush_color != wq->flush_color) 2670 break; 2671 list_del_init(&next->list); 2672 complete(&next->done); 2673 } 2674 2675 WARN_ON_ONCE(!list_empty(&wq->flusher_overflow) && 2676 wq->flush_color != work_next_color(wq->work_color)); 2677 2678 /* this flush_color is finished, advance by one */ 2679 wq->flush_color = work_next_color(wq->flush_color); 2680 2681 /* one color has been freed, handle overflow queue */ 2682 if (!list_empty(&wq->flusher_overflow)) { 2683 /* 2684 * Assign the same color to all overflowed 2685 * flushers, advance work_color and append to 2686 * flusher_queue. This is the start-to-wait 2687 * phase for these overflowed flushers. 2688 */ 2689 list_for_each_entry(tmp, &wq->flusher_overflow, list) 2690 tmp->flush_color = wq->work_color; 2691 2692 wq->work_color = work_next_color(wq->work_color); 2693 2694 list_splice_tail_init(&wq->flusher_overflow, 2695 &wq->flusher_queue); 2696 flush_workqueue_prep_pwqs(wq, -1, wq->work_color); 2697 } 2698 2699 if (list_empty(&wq->flusher_queue)) { 2700 WARN_ON_ONCE(wq->flush_color != wq->work_color); 2701 break; 2702 } 2703 2704 /* 2705 * Need to flush more colors. Make the next flusher 2706 * the new first flusher and arm pwqs. 2707 */ 2708 WARN_ON_ONCE(wq->flush_color == wq->work_color); 2709 WARN_ON_ONCE(wq->flush_color != next->flush_color); 2710 2711 list_del_init(&next->list); 2712 wq->first_flusher = next; 2713 2714 if (flush_workqueue_prep_pwqs(wq, wq->flush_color, -1)) 2715 break; 2716 2717 /* 2718 * Meh... this color is already done, clear first 2719 * flusher and repeat cascading. 2720 */ 2721 wq->first_flusher = NULL; 2722 } 2723 2724 out_unlock: 2725 mutex_unlock(&wq->mutex); 2726 } 2727 EXPORT_SYMBOL(flush_workqueue); 2728 2729 /** 2730 * drain_workqueue - drain a workqueue 2731 * @wq: workqueue to drain 2732 * 2733 * Wait until the workqueue becomes empty. While draining is in progress, 2734 * only chain queueing is allowed. IOW, only currently pending or running 2735 * work items on @wq can queue further work items on it. @wq is flushed 2736 * repeatedly until it becomes empty. The number of flushing is determined 2737 * by the depth of chaining and should be relatively short. Whine if it 2738 * takes too long. 2739 */ 2740 void drain_workqueue(struct workqueue_struct *wq) 2741 { 2742 unsigned int flush_cnt = 0; 2743 struct pool_workqueue *pwq; 2744 2745 /* 2746 * __queue_work() needs to test whether there are drainers, is much 2747 * hotter than drain_workqueue() and already looks at @wq->flags. 2748 * Use __WQ_DRAINING so that queue doesn't have to check nr_drainers. 2749 */ 2750 mutex_lock(&wq->mutex); 2751 if (!wq->nr_drainers++) 2752 wq->flags |= __WQ_DRAINING; 2753 mutex_unlock(&wq->mutex); 2754 reflush: 2755 flush_workqueue(wq); 2756 2757 mutex_lock(&wq->mutex); 2758 2759 for_each_pwq(pwq, wq) { 2760 bool drained; 2761 2762 spin_lock_irq(&pwq->pool->lock); 2763 drained = !pwq->nr_active && list_empty(&pwq->delayed_works); 2764 spin_unlock_irq(&pwq->pool->lock); 2765 2766 if (drained) 2767 continue; 2768 2769 if (++flush_cnt == 10 || 2770 (flush_cnt % 100 == 0 && flush_cnt <= 1000)) 2771 pr_warn("workqueue %s: drain_workqueue() isn't complete after %u tries\n", 2772 wq->name, flush_cnt); 2773 2774 mutex_unlock(&wq->mutex); 2775 goto reflush; 2776 } 2777 2778 if (!--wq->nr_drainers) 2779 wq->flags &= ~__WQ_DRAINING; 2780 mutex_unlock(&wq->mutex); 2781 } 2782 EXPORT_SYMBOL_GPL(drain_workqueue); 2783 2784 static bool start_flush_work(struct work_struct *work, struct wq_barrier *barr) 2785 { 2786 struct worker *worker = NULL; 2787 struct worker_pool *pool; 2788 struct pool_workqueue *pwq; 2789 2790 might_sleep(); 2791 2792 local_irq_disable(); 2793 pool = get_work_pool(work); 2794 if (!pool) { 2795 local_irq_enable(); 2796 return false; 2797 } 2798 2799 spin_lock(&pool->lock); 2800 /* see the comment in try_to_grab_pending() with the same code */ 2801 pwq = get_work_pwq(work); 2802 if (pwq) { 2803 if (unlikely(pwq->pool != pool)) 2804 goto already_gone; 2805 } else { 2806 worker = find_worker_executing_work(pool, work); 2807 if (!worker) 2808 goto already_gone; 2809 pwq = worker->current_pwq; 2810 } 2811 2812 check_flush_dependency(pwq->wq, work); 2813 2814 insert_wq_barrier(pwq, barr, work, worker); 2815 spin_unlock_irq(&pool->lock); 2816 2817 /* 2818 * If @max_active is 1 or rescuer is in use, flushing another work 2819 * item on the same workqueue may lead to deadlock. Make sure the 2820 * flusher is not running on the same workqueue by verifying write 2821 * access. 2822 */ 2823 if (pwq->wq->saved_max_active == 1 || pwq->wq->rescuer) 2824 lock_map_acquire(&pwq->wq->lockdep_map); 2825 else 2826 lock_map_acquire_read(&pwq->wq->lockdep_map); 2827 lock_map_release(&pwq->wq->lockdep_map); 2828 2829 return true; 2830 already_gone: 2831 spin_unlock_irq(&pool->lock); 2832 return false; 2833 } 2834 2835 /** 2836 * flush_work - wait for a work to finish executing the last queueing instance 2837 * @work: the work to flush 2838 * 2839 * Wait until @work has finished execution. @work is guaranteed to be idle 2840 * on return if it hasn't been requeued since flush started. 2841 * 2842 * Return: 2843 * %true if flush_work() waited for the work to finish execution, 2844 * %false if it was already idle. 2845 */ 2846 bool flush_work(struct work_struct *work) 2847 { 2848 struct wq_barrier barr; 2849 2850 if (WARN_ON(!wq_online)) 2851 return false; 2852 2853 lock_map_acquire(&work->lockdep_map); 2854 lock_map_release(&work->lockdep_map); 2855 2856 if (start_flush_work(work, &barr)) { 2857 wait_for_completion(&barr.done); 2858 destroy_work_on_stack(&barr.work); 2859 return true; 2860 } else { 2861 return false; 2862 } 2863 } 2864 EXPORT_SYMBOL_GPL(flush_work); 2865 2866 struct cwt_wait { 2867 wait_queue_t wait; 2868 struct work_struct *work; 2869 }; 2870 2871 static int cwt_wakefn(wait_queue_t *wait, unsigned mode, int sync, void *key) 2872 { 2873 struct cwt_wait *cwait = container_of(wait, struct cwt_wait, wait); 2874 2875 if (cwait->work != key) 2876 return 0; 2877 return autoremove_wake_function(wait, mode, sync, key); 2878 } 2879 2880 static bool __cancel_work_timer(struct work_struct *work, bool is_dwork) 2881 { 2882 static DECLARE_WAIT_QUEUE_HEAD(cancel_waitq); 2883 unsigned long flags; 2884 int ret; 2885 2886 do { 2887 ret = try_to_grab_pending(work, is_dwork, &flags); 2888 /* 2889 * If someone else is already canceling, wait for it to 2890 * finish. flush_work() doesn't work for PREEMPT_NONE 2891 * because we may get scheduled between @work's completion 2892 * and the other canceling task resuming and clearing 2893 * CANCELING - flush_work() will return false immediately 2894 * as @work is no longer busy, try_to_grab_pending() will 2895 * return -ENOENT as @work is still being canceled and the 2896 * other canceling task won't be able to clear CANCELING as 2897 * we're hogging the CPU. 2898 * 2899 * Let's wait for completion using a waitqueue. As this 2900 * may lead to the thundering herd problem, use a custom 2901 * wake function which matches @work along with exclusive 2902 * wait and wakeup. 2903 */ 2904 if (unlikely(ret == -ENOENT)) { 2905 struct cwt_wait cwait; 2906 2907 init_wait(&cwait.wait); 2908 cwait.wait.func = cwt_wakefn; 2909 cwait.work = work; 2910 2911 prepare_to_wait_exclusive(&cancel_waitq, &cwait.wait, 2912 TASK_UNINTERRUPTIBLE); 2913 if (work_is_canceling(work)) 2914 schedule(); 2915 finish_wait(&cancel_waitq, &cwait.wait); 2916 } 2917 } while (unlikely(ret < 0)); 2918 2919 /* tell other tasks trying to grab @work to back off */ 2920 mark_work_canceling(work); 2921 local_irq_restore(flags); 2922 2923 /* 2924 * This allows canceling during early boot. We know that @work 2925 * isn't executing. 2926 */ 2927 if (wq_online) 2928 flush_work(work); 2929 2930 clear_work_data(work); 2931 2932 /* 2933 * Paired with prepare_to_wait() above so that either 2934 * waitqueue_active() is visible here or !work_is_canceling() is 2935 * visible there. 2936 */ 2937 smp_mb(); 2938 if (waitqueue_active(&cancel_waitq)) 2939 __wake_up(&cancel_waitq, TASK_NORMAL, 1, work); 2940 2941 return ret; 2942 } 2943 2944 /** 2945 * cancel_work_sync - cancel a work and wait for it to finish 2946 * @work: the work to cancel 2947 * 2948 * Cancel @work and wait for its execution to finish. This function 2949 * can be used even if the work re-queues itself or migrates to 2950 * another workqueue. On return from this function, @work is 2951 * guaranteed to be not pending or executing on any CPU. 2952 * 2953 * cancel_work_sync(&delayed_work->work) must not be used for 2954 * delayed_work's. Use cancel_delayed_work_sync() instead. 2955 * 2956 * The caller must ensure that the workqueue on which @work was last 2957 * queued can't be destroyed before this function returns. 2958 * 2959 * Return: 2960 * %true if @work was pending, %false otherwise. 2961 */ 2962 bool cancel_work_sync(struct work_struct *work) 2963 { 2964 return __cancel_work_timer(work, false); 2965 } 2966 EXPORT_SYMBOL_GPL(cancel_work_sync); 2967 2968 /** 2969 * flush_delayed_work - wait for a dwork to finish executing the last queueing 2970 * @dwork: the delayed work to flush 2971 * 2972 * Delayed timer is cancelled and the pending work is queued for 2973 * immediate execution. Like flush_work(), this function only 2974 * considers the last queueing instance of @dwork. 2975 * 2976 * Return: 2977 * %true if flush_work() waited for the work to finish execution, 2978 * %false if it was already idle. 2979 */ 2980 bool flush_delayed_work(struct delayed_work *dwork) 2981 { 2982 local_irq_disable(); 2983 if (del_timer_sync(&dwork->timer)) 2984 __queue_work(dwork->cpu, dwork->wq, &dwork->work); 2985 local_irq_enable(); 2986 return flush_work(&dwork->work); 2987 } 2988 EXPORT_SYMBOL(flush_delayed_work); 2989 2990 static bool __cancel_work(struct work_struct *work, bool is_dwork) 2991 { 2992 unsigned long flags; 2993 int ret; 2994 2995 do { 2996 ret = try_to_grab_pending(work, is_dwork, &flags); 2997 } while (unlikely(ret == -EAGAIN)); 2998 2999 if (unlikely(ret < 0)) 3000 return false; 3001 3002 set_work_pool_and_clear_pending(work, get_work_pool_id(work)); 3003 local_irq_restore(flags); 3004 return ret; 3005 } 3006 3007 /* 3008 * See cancel_delayed_work() 3009 */ 3010 bool cancel_work(struct work_struct *work) 3011 { 3012 return __cancel_work(work, false); 3013 } 3014 3015 /** 3016 * cancel_delayed_work - cancel a delayed work 3017 * @dwork: delayed_work to cancel 3018 * 3019 * Kill off a pending delayed_work. 3020 * 3021 * Return: %true if @dwork was pending and canceled; %false if it wasn't 3022 * pending. 3023 * 3024 * Note: 3025 * The work callback function may still be running on return, unless 3026 * it returns %true and the work doesn't re-arm itself. Explicitly flush or 3027 * use cancel_delayed_work_sync() to wait on it. 3028 * 3029 * This function is safe to call from any context including IRQ handler. 3030 */ 3031 bool cancel_delayed_work(struct delayed_work *dwork) 3032 { 3033 return __cancel_work(&dwork->work, true); 3034 } 3035 EXPORT_SYMBOL(cancel_delayed_work); 3036 3037 /** 3038 * cancel_delayed_work_sync - cancel a delayed work and wait for it to finish 3039 * @dwork: the delayed work cancel 3040 * 3041 * This is cancel_work_sync() for delayed works. 3042 * 3043 * Return: 3044 * %true if @dwork was pending, %false otherwise. 3045 */ 3046 bool cancel_delayed_work_sync(struct delayed_work *dwork) 3047 { 3048 return __cancel_work_timer(&dwork->work, true); 3049 } 3050 EXPORT_SYMBOL(cancel_delayed_work_sync); 3051 3052 /** 3053 * schedule_on_each_cpu - execute a function synchronously on each online CPU 3054 * @func: the function to call 3055 * 3056 * schedule_on_each_cpu() executes @func on each online CPU using the 3057 * system workqueue and blocks until all CPUs have completed. 3058 * schedule_on_each_cpu() is very slow. 3059 * 3060 * Return: 3061 * 0 on success, -errno on failure. 3062 */ 3063 int schedule_on_each_cpu(work_func_t func) 3064 { 3065 int cpu; 3066 struct work_struct __percpu *works; 3067 3068 works = alloc_percpu(struct work_struct); 3069 if (!works) 3070 return -ENOMEM; 3071 3072 get_online_cpus(); 3073 3074 for_each_online_cpu(cpu) { 3075 struct work_struct *work = per_cpu_ptr(works, cpu); 3076 3077 INIT_WORK(work, func); 3078 schedule_work_on(cpu, work); 3079 } 3080 3081 for_each_online_cpu(cpu) 3082 flush_work(per_cpu_ptr(works, cpu)); 3083 3084 put_online_cpus(); 3085 free_percpu(works); 3086 return 0; 3087 } 3088 3089 /** 3090 * execute_in_process_context - reliably execute the routine with user context 3091 * @fn: the function to execute 3092 * @ew: guaranteed storage for the execute work structure (must 3093 * be available when the work executes) 3094 * 3095 * Executes the function immediately if process context is available, 3096 * otherwise schedules the function for delayed execution. 3097 * 3098 * Return: 0 - function was executed 3099 * 1 - function was scheduled for execution 3100 */ 3101 int execute_in_process_context(work_func_t fn, struct execute_work *ew) 3102 { 3103 if (!in_interrupt()) { 3104 fn(&ew->work); 3105 return 0; 3106 } 3107 3108 INIT_WORK(&ew->work, fn); 3109 schedule_work(&ew->work); 3110 3111 return 1; 3112 } 3113 EXPORT_SYMBOL_GPL(execute_in_process_context); 3114 3115 /** 3116 * free_workqueue_attrs - free a workqueue_attrs 3117 * @attrs: workqueue_attrs to free 3118 * 3119 * Undo alloc_workqueue_attrs(). 3120 */ 3121 void free_workqueue_attrs(struct workqueue_attrs *attrs) 3122 { 3123 if (attrs) { 3124 free_cpumask_var(attrs->cpumask); 3125 kfree(attrs); 3126 } 3127 } 3128 3129 /** 3130 * alloc_workqueue_attrs - allocate a workqueue_attrs 3131 * @gfp_mask: allocation mask to use 3132 * 3133 * Allocate a new workqueue_attrs, initialize with default settings and 3134 * return it. 3135 * 3136 * Return: The allocated new workqueue_attr on success. %NULL on failure. 3137 */ 3138 struct workqueue_attrs *alloc_workqueue_attrs(gfp_t gfp_mask) 3139 { 3140 struct workqueue_attrs *attrs; 3141 3142 attrs = kzalloc(sizeof(*attrs), gfp_mask); 3143 if (!attrs) 3144 goto fail; 3145 if (!alloc_cpumask_var(&attrs->cpumask, gfp_mask)) 3146 goto fail; 3147 3148 cpumask_copy(attrs->cpumask, cpu_possible_mask); 3149 return attrs; 3150 fail: 3151 free_workqueue_attrs(attrs); 3152 return NULL; 3153 } 3154 3155 static void copy_workqueue_attrs(struct workqueue_attrs *to, 3156 const struct workqueue_attrs *from) 3157 { 3158 to->nice = from->nice; 3159 cpumask_copy(to->cpumask, from->cpumask); 3160 /* 3161 * Unlike hash and equality test, this function doesn't ignore 3162 * ->no_numa as it is used for both pool and wq attrs. Instead, 3163 * get_unbound_pool() explicitly clears ->no_numa after copying. 3164 */ 3165 to->no_numa = from->no_numa; 3166 } 3167 3168 /* hash value of the content of @attr */ 3169 static u32 wqattrs_hash(const struct workqueue_attrs *attrs) 3170 { 3171 u32 hash = 0; 3172 3173 hash = jhash_1word(attrs->nice, hash); 3174 hash = jhash(cpumask_bits(attrs->cpumask), 3175 BITS_TO_LONGS(nr_cpumask_bits) * sizeof(long), hash); 3176 return hash; 3177 } 3178 3179 /* content equality test */ 3180 static bool wqattrs_equal(const struct workqueue_attrs *a, 3181 const struct workqueue_attrs *b) 3182 { 3183 if (a->nice != b->nice) 3184 return false; 3185 if (!cpumask_equal(a->cpumask, b->cpumask)) 3186 return false; 3187 return true; 3188 } 3189 3190 /** 3191 * init_worker_pool - initialize a newly zalloc'd worker_pool 3192 * @pool: worker_pool to initialize 3193 * 3194 * Initialize a newly zalloc'd @pool. It also allocates @pool->attrs. 3195 * 3196 * Return: 0 on success, -errno on failure. Even on failure, all fields 3197 * inside @pool proper are initialized and put_unbound_pool() can be called 3198 * on @pool safely to release it. 3199 */ 3200 static int init_worker_pool(struct worker_pool *pool) 3201 { 3202 spin_lock_init(&pool->lock); 3203 pool->id = -1; 3204 pool->cpu = -1; 3205 pool->node = NUMA_NO_NODE; 3206 pool->flags |= POOL_DISASSOCIATED; 3207 pool->watchdog_ts = jiffies; 3208 INIT_LIST_HEAD(&pool->worklist); 3209 INIT_LIST_HEAD(&pool->idle_list); 3210 hash_init(pool->busy_hash); 3211 3212 init_timer_deferrable(&pool->idle_timer); 3213 pool->idle_timer.function = idle_worker_timeout; 3214 pool->idle_timer.data = (unsigned long)pool; 3215 3216 setup_timer(&pool->mayday_timer, pool_mayday_timeout, 3217 (unsigned long)pool); 3218 3219 mutex_init(&pool->manager_arb); 3220 mutex_init(&pool->attach_mutex); 3221 INIT_LIST_HEAD(&pool->workers); 3222 3223 ida_init(&pool->worker_ida); 3224 INIT_HLIST_NODE(&pool->hash_node); 3225 pool->refcnt = 1; 3226 3227 /* shouldn't fail above this point */ 3228 pool->attrs = alloc_workqueue_attrs(GFP_KERNEL); 3229 if (!pool->attrs) 3230 return -ENOMEM; 3231 return 0; 3232 } 3233 3234 static void rcu_free_wq(struct rcu_head *rcu) 3235 { 3236 struct workqueue_struct *wq = 3237 container_of(rcu, struct workqueue_struct, rcu); 3238 3239 if (!(wq->flags & WQ_UNBOUND)) 3240 free_percpu(wq->cpu_pwqs); 3241 else 3242 free_workqueue_attrs(wq->unbound_attrs); 3243 3244 kfree(wq->rescuer); 3245 kfree(wq); 3246 } 3247 3248 static void rcu_free_pool(struct rcu_head *rcu) 3249 { 3250 struct worker_pool *pool = container_of(rcu, struct worker_pool, rcu); 3251 3252 ida_destroy(&pool->worker_ida); 3253 free_workqueue_attrs(pool->attrs); 3254 kfree(pool); 3255 } 3256 3257 /** 3258 * put_unbound_pool - put a worker_pool 3259 * @pool: worker_pool to put 3260 * 3261 * Put @pool. If its refcnt reaches zero, it gets destroyed in sched-RCU 3262 * safe manner. get_unbound_pool() calls this function on its failure path 3263 * and this function should be able to release pools which went through, 3264 * successfully or not, init_worker_pool(). 3265 * 3266 * Should be called with wq_pool_mutex held. 3267 */ 3268 static void put_unbound_pool(struct worker_pool *pool) 3269 { 3270 DECLARE_COMPLETION_ONSTACK(detach_completion); 3271 struct worker *worker; 3272 3273 lockdep_assert_held(&wq_pool_mutex); 3274 3275 if (--pool->refcnt) 3276 return; 3277 3278 /* sanity checks */ 3279 if (WARN_ON(!(pool->cpu < 0)) || 3280 WARN_ON(!list_empty(&pool->worklist))) 3281 return; 3282 3283 /* release id and unhash */ 3284 if (pool->id >= 0) 3285 idr_remove(&worker_pool_idr, pool->id); 3286 hash_del(&pool->hash_node); 3287 3288 /* 3289 * Become the manager and destroy all workers. Grabbing 3290 * manager_arb prevents @pool's workers from blocking on 3291 * attach_mutex. 3292 */ 3293 mutex_lock(&pool->manager_arb); 3294 3295 spin_lock_irq(&pool->lock); 3296 while ((worker = first_idle_worker(pool))) 3297 destroy_worker(worker); 3298 WARN_ON(pool->nr_workers || pool->nr_idle); 3299 spin_unlock_irq(&pool->lock); 3300 3301 mutex_lock(&pool->attach_mutex); 3302 if (!list_empty(&pool->workers)) 3303 pool->detach_completion = &detach_completion; 3304 mutex_unlock(&pool->attach_mutex); 3305 3306 if (pool->detach_completion) 3307 wait_for_completion(pool->detach_completion); 3308 3309 mutex_unlock(&pool->manager_arb); 3310 3311 /* shut down the timers */ 3312 del_timer_sync(&pool->idle_timer); 3313 del_timer_sync(&pool->mayday_timer); 3314 3315 /* sched-RCU protected to allow dereferences from get_work_pool() */ 3316 call_rcu_sched(&pool->rcu, rcu_free_pool); 3317 } 3318 3319 /** 3320 * get_unbound_pool - get a worker_pool with the specified attributes 3321 * @attrs: the attributes of the worker_pool to get 3322 * 3323 * Obtain a worker_pool which has the same attributes as @attrs, bump the 3324 * reference count and return it. If there already is a matching 3325 * worker_pool, it will be used; otherwise, this function attempts to 3326 * create a new one. 3327 * 3328 * Should be called with wq_pool_mutex held. 3329 * 3330 * Return: On success, a worker_pool with the same attributes as @attrs. 3331 * On failure, %NULL. 3332 */ 3333 static struct worker_pool *get_unbound_pool(const struct workqueue_attrs *attrs) 3334 { 3335 u32 hash = wqattrs_hash(attrs); 3336 struct worker_pool *pool; 3337 int node; 3338 int target_node = NUMA_NO_NODE; 3339 3340 lockdep_assert_held(&wq_pool_mutex); 3341 3342 /* do we already have a matching pool? */ 3343 hash_for_each_possible(unbound_pool_hash, pool, hash_node, hash) { 3344 if (wqattrs_equal(pool->attrs, attrs)) { 3345 pool->refcnt++; 3346 return pool; 3347 } 3348 } 3349 3350 /* if cpumask is contained inside a NUMA node, we belong to that node */ 3351 if (wq_numa_enabled) { 3352 for_each_node(node) { 3353 if (cpumask_subset(attrs->cpumask, 3354 wq_numa_possible_cpumask[node])) { 3355 target_node = node; 3356 break; 3357 } 3358 } 3359 } 3360 3361 /* nope, create a new one */ 3362 pool = kzalloc_node(sizeof(*pool), GFP_KERNEL, target_node); 3363 if (!pool || init_worker_pool(pool) < 0) 3364 goto fail; 3365 3366 lockdep_set_subclass(&pool->lock, 1); /* see put_pwq() */ 3367 copy_workqueue_attrs(pool->attrs, attrs); 3368 pool->node = target_node; 3369 3370 /* 3371 * no_numa isn't a worker_pool attribute, always clear it. See 3372 * 'struct workqueue_attrs' comments for detail. 3373 */ 3374 pool->attrs->no_numa = false; 3375 3376 if (worker_pool_assign_id(pool) < 0) 3377 goto fail; 3378 3379 /* create and start the initial worker */ 3380 if (wq_online && !create_worker(pool)) 3381 goto fail; 3382 3383 /* install */ 3384 hash_add(unbound_pool_hash, &pool->hash_node, hash); 3385 3386 return pool; 3387 fail: 3388 if (pool) 3389 put_unbound_pool(pool); 3390 return NULL; 3391 } 3392 3393 static void rcu_free_pwq(struct rcu_head *rcu) 3394 { 3395 kmem_cache_free(pwq_cache, 3396 container_of(rcu, struct pool_workqueue, rcu)); 3397 } 3398 3399 /* 3400 * Scheduled on system_wq by put_pwq() when an unbound pwq hits zero refcnt 3401 * and needs to be destroyed. 3402 */ 3403 static void pwq_unbound_release_workfn(struct work_struct *work) 3404 { 3405 struct pool_workqueue *pwq = container_of(work, struct pool_workqueue, 3406 unbound_release_work); 3407 struct workqueue_struct *wq = pwq->wq; 3408 struct worker_pool *pool = pwq->pool; 3409 bool is_last; 3410 3411 if (WARN_ON_ONCE(!(wq->flags & WQ_UNBOUND))) 3412 return; 3413 3414 mutex_lock(&wq->mutex); 3415 list_del_rcu(&pwq->pwqs_node); 3416 is_last = list_empty(&wq->pwqs); 3417 mutex_unlock(&wq->mutex); 3418 3419 mutex_lock(&wq_pool_mutex); 3420 put_unbound_pool(pool); 3421 mutex_unlock(&wq_pool_mutex); 3422 3423 call_rcu_sched(&pwq->rcu, rcu_free_pwq); 3424 3425 /* 3426 * If we're the last pwq going away, @wq is already dead and no one 3427 * is gonna access it anymore. Schedule RCU free. 3428 */ 3429 if (is_last) 3430 call_rcu_sched(&wq->rcu, rcu_free_wq); 3431 } 3432 3433 /** 3434 * pwq_adjust_max_active - update a pwq's max_active to the current setting 3435 * @pwq: target pool_workqueue 3436 * 3437 * If @pwq isn't freezing, set @pwq->max_active to the associated 3438 * workqueue's saved_max_active and activate delayed work items 3439 * accordingly. If @pwq is freezing, clear @pwq->max_active to zero. 3440 */ 3441 static void pwq_adjust_max_active(struct pool_workqueue *pwq) 3442 { 3443 struct workqueue_struct *wq = pwq->wq; 3444 bool freezable = wq->flags & WQ_FREEZABLE; 3445 unsigned long flags; 3446 3447 /* for @wq->saved_max_active */ 3448 lockdep_assert_held(&wq->mutex); 3449 3450 /* fast exit for non-freezable wqs */ 3451 if (!freezable && pwq->max_active == wq->saved_max_active) 3452 return; 3453 3454 /* this function can be called during early boot w/ irq disabled */ 3455 spin_lock_irqsave(&pwq->pool->lock, flags); 3456 3457 /* 3458 * During [un]freezing, the caller is responsible for ensuring that 3459 * this function is called at least once after @workqueue_freezing 3460 * is updated and visible. 3461 */ 3462 if (!freezable || !workqueue_freezing) { 3463 pwq->max_active = wq->saved_max_active; 3464 3465 while (!list_empty(&pwq->delayed_works) && 3466 pwq->nr_active < pwq->max_active) 3467 pwq_activate_first_delayed(pwq); 3468 3469 /* 3470 * Need to kick a worker after thawed or an unbound wq's 3471 * max_active is bumped. It's a slow path. Do it always. 3472 */ 3473 wake_up_worker(pwq->pool); 3474 } else { 3475 pwq->max_active = 0; 3476 } 3477 3478 spin_unlock_irqrestore(&pwq->pool->lock, flags); 3479 } 3480 3481 /* initialize newly alloced @pwq which is associated with @wq and @pool */ 3482 static void init_pwq(struct pool_workqueue *pwq, struct workqueue_struct *wq, 3483 struct worker_pool *pool) 3484 { 3485 BUG_ON((unsigned long)pwq & WORK_STRUCT_FLAG_MASK); 3486 3487 memset(pwq, 0, sizeof(*pwq)); 3488 3489 pwq->pool = pool; 3490 pwq->wq = wq; 3491 pwq->flush_color = -1; 3492 pwq->refcnt = 1; 3493 INIT_LIST_HEAD(&pwq->delayed_works); 3494 INIT_LIST_HEAD(&pwq->pwqs_node); 3495 INIT_LIST_HEAD(&pwq->mayday_node); 3496 INIT_WORK(&pwq->unbound_release_work, pwq_unbound_release_workfn); 3497 } 3498 3499 /* sync @pwq with the current state of its associated wq and link it */ 3500 static void link_pwq(struct pool_workqueue *pwq) 3501 { 3502 struct workqueue_struct *wq = pwq->wq; 3503 3504 lockdep_assert_held(&wq->mutex); 3505 3506 /* may be called multiple times, ignore if already linked */ 3507 if (!list_empty(&pwq->pwqs_node)) 3508 return; 3509 3510 /* set the matching work_color */ 3511 pwq->work_color = wq->work_color; 3512 3513 /* sync max_active to the current setting */ 3514 pwq_adjust_max_active(pwq); 3515 3516 /* link in @pwq */ 3517 list_add_rcu(&pwq->pwqs_node, &wq->pwqs); 3518 } 3519 3520 /* obtain a pool matching @attr and create a pwq associating the pool and @wq */ 3521 static struct pool_workqueue *alloc_unbound_pwq(struct workqueue_struct *wq, 3522 const struct workqueue_attrs *attrs) 3523 { 3524 struct worker_pool *pool; 3525 struct pool_workqueue *pwq; 3526 3527 lockdep_assert_held(&wq_pool_mutex); 3528 3529 pool = get_unbound_pool(attrs); 3530 if (!pool) 3531 return NULL; 3532 3533 pwq = kmem_cache_alloc_node(pwq_cache, GFP_KERNEL, pool->node); 3534 if (!pwq) { 3535 put_unbound_pool(pool); 3536 return NULL; 3537 } 3538 3539 init_pwq(pwq, wq, pool); 3540 return pwq; 3541 } 3542 3543 /** 3544 * wq_calc_node_cpumask - calculate a wq_attrs' cpumask for the specified node 3545 * @attrs: the wq_attrs of the default pwq of the target workqueue 3546 * @node: the target NUMA node 3547 * @cpu_going_down: if >= 0, the CPU to consider as offline 3548 * @cpumask: outarg, the resulting cpumask 3549 * 3550 * Calculate the cpumask a workqueue with @attrs should use on @node. If 3551 * @cpu_going_down is >= 0, that cpu is considered offline during 3552 * calculation. The result is stored in @cpumask. 3553 * 3554 * If NUMA affinity is not enabled, @attrs->cpumask is always used. If 3555 * enabled and @node has online CPUs requested by @attrs, the returned 3556 * cpumask is the intersection of the possible CPUs of @node and 3557 * @attrs->cpumask. 3558 * 3559 * The caller is responsible for ensuring that the cpumask of @node stays 3560 * stable. 3561 * 3562 * Return: %true if the resulting @cpumask is different from @attrs->cpumask, 3563 * %false if equal. 3564 */ 3565 static bool wq_calc_node_cpumask(const struct workqueue_attrs *attrs, int node, 3566 int cpu_going_down, cpumask_t *cpumask) 3567 { 3568 if (!wq_numa_enabled || attrs->no_numa) 3569 goto use_dfl; 3570 3571 /* does @node have any online CPUs @attrs wants? */ 3572 cpumask_and(cpumask, cpumask_of_node(node), attrs->cpumask); 3573 if (cpu_going_down >= 0) 3574 cpumask_clear_cpu(cpu_going_down, cpumask); 3575 3576 if (cpumask_empty(cpumask)) 3577 goto use_dfl; 3578 3579 /* yeap, return possible CPUs in @node that @attrs wants */ 3580 cpumask_and(cpumask, attrs->cpumask, wq_numa_possible_cpumask[node]); 3581 return !cpumask_equal(cpumask, attrs->cpumask); 3582 3583 use_dfl: 3584 cpumask_copy(cpumask, attrs->cpumask); 3585 return false; 3586 } 3587 3588 /* install @pwq into @wq's numa_pwq_tbl[] for @node and return the old pwq */ 3589 static struct pool_workqueue *numa_pwq_tbl_install(struct workqueue_struct *wq, 3590 int node, 3591 struct pool_workqueue *pwq) 3592 { 3593 struct pool_workqueue *old_pwq; 3594 3595 lockdep_assert_held(&wq_pool_mutex); 3596 lockdep_assert_held(&wq->mutex); 3597 3598 /* link_pwq() can handle duplicate calls */ 3599 link_pwq(pwq); 3600 3601 old_pwq = rcu_access_pointer(wq->numa_pwq_tbl[node]); 3602 rcu_assign_pointer(wq->numa_pwq_tbl[node], pwq); 3603 return old_pwq; 3604 } 3605 3606 /* context to store the prepared attrs & pwqs before applying */ 3607 struct apply_wqattrs_ctx { 3608 struct workqueue_struct *wq; /* target workqueue */ 3609 struct workqueue_attrs *attrs; /* attrs to apply */ 3610 struct list_head list; /* queued for batching commit */ 3611 struct pool_workqueue *dfl_pwq; 3612 struct pool_workqueue *pwq_tbl[]; 3613 }; 3614 3615 /* free the resources after success or abort */ 3616 static void apply_wqattrs_cleanup(struct apply_wqattrs_ctx *ctx) 3617 { 3618 if (ctx) { 3619 int node; 3620 3621 for_each_node(node) 3622 put_pwq_unlocked(ctx->pwq_tbl[node]); 3623 put_pwq_unlocked(ctx->dfl_pwq); 3624 3625 free_workqueue_attrs(ctx->attrs); 3626 3627 kfree(ctx); 3628 } 3629 } 3630 3631 /* allocate the attrs and pwqs for later installation */ 3632 static struct apply_wqattrs_ctx * 3633 apply_wqattrs_prepare(struct workqueue_struct *wq, 3634 const struct workqueue_attrs *attrs) 3635 { 3636 struct apply_wqattrs_ctx *ctx; 3637 struct workqueue_attrs *new_attrs, *tmp_attrs; 3638 int node; 3639 3640 lockdep_assert_held(&wq_pool_mutex); 3641 3642 ctx = kzalloc(sizeof(*ctx) + nr_node_ids * sizeof(ctx->pwq_tbl[0]), 3643 GFP_KERNEL); 3644 3645 new_attrs = alloc_workqueue_attrs(GFP_KERNEL); 3646 tmp_attrs = alloc_workqueue_attrs(GFP_KERNEL); 3647 if (!ctx || !new_attrs || !tmp_attrs) 3648 goto out_free; 3649 3650 /* 3651 * Calculate the attrs of the default pwq. 3652 * If the user configured cpumask doesn't overlap with the 3653 * wq_unbound_cpumask, we fallback to the wq_unbound_cpumask. 3654 */ 3655 copy_workqueue_attrs(new_attrs, attrs); 3656 cpumask_and(new_attrs->cpumask, new_attrs->cpumask, wq_unbound_cpumask); 3657 if (unlikely(cpumask_empty(new_attrs->cpumask))) 3658 cpumask_copy(new_attrs->cpumask, wq_unbound_cpumask); 3659 3660 /* 3661 * We may create multiple pwqs with differing cpumasks. Make a 3662 * copy of @new_attrs which will be modified and used to obtain 3663 * pools. 3664 */ 3665 copy_workqueue_attrs(tmp_attrs, new_attrs); 3666 3667 /* 3668 * If something goes wrong during CPU up/down, we'll fall back to 3669 * the default pwq covering whole @attrs->cpumask. Always create 3670 * it even if we don't use it immediately. 3671 */ 3672 ctx->dfl_pwq = alloc_unbound_pwq(wq, new_attrs); 3673 if (!ctx->dfl_pwq) 3674 goto out_free; 3675 3676 for_each_node(node) { 3677 if (wq_calc_node_cpumask(new_attrs, node, -1, tmp_attrs->cpumask)) { 3678 ctx->pwq_tbl[node] = alloc_unbound_pwq(wq, tmp_attrs); 3679 if (!ctx->pwq_tbl[node]) 3680 goto out_free; 3681 } else { 3682 ctx->dfl_pwq->refcnt++; 3683 ctx->pwq_tbl[node] = ctx->dfl_pwq; 3684 } 3685 } 3686 3687 /* save the user configured attrs and sanitize it. */ 3688 copy_workqueue_attrs(new_attrs, attrs); 3689 cpumask_and(new_attrs->cpumask, new_attrs->cpumask, cpu_possible_mask); 3690 ctx->attrs = new_attrs; 3691 3692 ctx->wq = wq; 3693 free_workqueue_attrs(tmp_attrs); 3694 return ctx; 3695 3696 out_free: 3697 free_workqueue_attrs(tmp_attrs); 3698 free_workqueue_attrs(new_attrs); 3699 apply_wqattrs_cleanup(ctx); 3700 return NULL; 3701 } 3702 3703 /* set attrs and install prepared pwqs, @ctx points to old pwqs on return */ 3704 static void apply_wqattrs_commit(struct apply_wqattrs_ctx *ctx) 3705 { 3706 int node; 3707 3708 /* all pwqs have been created successfully, let's install'em */ 3709 mutex_lock(&ctx->wq->mutex); 3710 3711 copy_workqueue_attrs(ctx->wq->unbound_attrs, ctx->attrs); 3712 3713 /* save the previous pwq and install the new one */ 3714 for_each_node(node) 3715 ctx->pwq_tbl[node] = numa_pwq_tbl_install(ctx->wq, node, 3716 ctx->pwq_tbl[node]); 3717 3718 /* @dfl_pwq might not have been used, ensure it's linked */ 3719 link_pwq(ctx->dfl_pwq); 3720 swap(ctx->wq->dfl_pwq, ctx->dfl_pwq); 3721 3722 mutex_unlock(&ctx->wq->mutex); 3723 } 3724 3725 static void apply_wqattrs_lock(void) 3726 { 3727 /* CPUs should stay stable across pwq creations and installations */ 3728 get_online_cpus(); 3729 mutex_lock(&wq_pool_mutex); 3730 } 3731 3732 static void apply_wqattrs_unlock(void) 3733 { 3734 mutex_unlock(&wq_pool_mutex); 3735 put_online_cpus(); 3736 } 3737 3738 static int apply_workqueue_attrs_locked(struct workqueue_struct *wq, 3739 const struct workqueue_attrs *attrs) 3740 { 3741 struct apply_wqattrs_ctx *ctx; 3742 3743 /* only unbound workqueues can change attributes */ 3744 if (WARN_ON(!(wq->flags & WQ_UNBOUND))) 3745 return -EINVAL; 3746 3747 /* creating multiple pwqs breaks ordering guarantee */ 3748 if (WARN_ON((wq->flags & __WQ_ORDERED) && !list_empty(&wq->pwqs))) 3749 return -EINVAL; 3750 3751 ctx = apply_wqattrs_prepare(wq, attrs); 3752 if (!ctx) 3753 return -ENOMEM; 3754 3755 /* the ctx has been prepared successfully, let's commit it */ 3756 apply_wqattrs_commit(ctx); 3757 apply_wqattrs_cleanup(ctx); 3758 3759 return 0; 3760 } 3761 3762 /** 3763 * apply_workqueue_attrs - apply new workqueue_attrs to an unbound workqueue 3764 * @wq: the target workqueue 3765 * @attrs: the workqueue_attrs to apply, allocated with alloc_workqueue_attrs() 3766 * 3767 * Apply @attrs to an unbound workqueue @wq. Unless disabled, on NUMA 3768 * machines, this function maps a separate pwq to each NUMA node with 3769 * possibles CPUs in @attrs->cpumask so that work items are affine to the 3770 * NUMA node it was issued on. Older pwqs are released as in-flight work 3771 * items finish. Note that a work item which repeatedly requeues itself 3772 * back-to-back will stay on its current pwq. 3773 * 3774 * Performs GFP_KERNEL allocations. 3775 * 3776 * Return: 0 on success and -errno on failure. 3777 */ 3778 int apply_workqueue_attrs(struct workqueue_struct *wq, 3779 const struct workqueue_attrs *attrs) 3780 { 3781 int ret; 3782 3783 apply_wqattrs_lock(); 3784 ret = apply_workqueue_attrs_locked(wq, attrs); 3785 apply_wqattrs_unlock(); 3786 3787 return ret; 3788 } 3789 3790 /** 3791 * wq_update_unbound_numa - update NUMA affinity of a wq for CPU hot[un]plug 3792 * @wq: the target workqueue 3793 * @cpu: the CPU coming up or going down 3794 * @online: whether @cpu is coming up or going down 3795 * 3796 * This function is to be called from %CPU_DOWN_PREPARE, %CPU_ONLINE and 3797 * %CPU_DOWN_FAILED. @cpu is being hot[un]plugged, update NUMA affinity of 3798 * @wq accordingly. 3799 * 3800 * If NUMA affinity can't be adjusted due to memory allocation failure, it 3801 * falls back to @wq->dfl_pwq which may not be optimal but is always 3802 * correct. 3803 * 3804 * Note that when the last allowed CPU of a NUMA node goes offline for a 3805 * workqueue with a cpumask spanning multiple nodes, the workers which were 3806 * already executing the work items for the workqueue will lose their CPU 3807 * affinity and may execute on any CPU. This is similar to how per-cpu 3808 * workqueues behave on CPU_DOWN. If a workqueue user wants strict 3809 * affinity, it's the user's responsibility to flush the work item from 3810 * CPU_DOWN_PREPARE. 3811 */ 3812 static void wq_update_unbound_numa(struct workqueue_struct *wq, int cpu, 3813 bool online) 3814 { 3815 int node = cpu_to_node(cpu); 3816 int cpu_off = online ? -1 : cpu; 3817 struct pool_workqueue *old_pwq = NULL, *pwq; 3818 struct workqueue_attrs *target_attrs; 3819 cpumask_t *cpumask; 3820 3821 lockdep_assert_held(&wq_pool_mutex); 3822 3823 if (!wq_numa_enabled || !(wq->flags & WQ_UNBOUND) || 3824 wq->unbound_attrs->no_numa) 3825 return; 3826 3827 /* 3828 * We don't wanna alloc/free wq_attrs for each wq for each CPU. 3829 * Let's use a preallocated one. The following buf is protected by 3830 * CPU hotplug exclusion. 3831 */ 3832 target_attrs = wq_update_unbound_numa_attrs_buf; 3833 cpumask = target_attrs->cpumask; 3834 3835 copy_workqueue_attrs(target_attrs, wq->unbound_attrs); 3836 pwq = unbound_pwq_by_node(wq, node); 3837 3838 /* 3839 * Let's determine what needs to be done. If the target cpumask is 3840 * different from the default pwq's, we need to compare it to @pwq's 3841 * and create a new one if they don't match. If the target cpumask 3842 * equals the default pwq's, the default pwq should be used. 3843 */ 3844 if (wq_calc_node_cpumask(wq->dfl_pwq->pool->attrs, node, cpu_off, cpumask)) { 3845 if (cpumask_equal(cpumask, pwq->pool->attrs->cpumask)) 3846 return; 3847 } else { 3848 goto use_dfl_pwq; 3849 } 3850 3851 /* create a new pwq */ 3852 pwq = alloc_unbound_pwq(wq, target_attrs); 3853 if (!pwq) { 3854 pr_warn("workqueue: allocation failed while updating NUMA affinity of \"%s\"\n", 3855 wq->name); 3856 goto use_dfl_pwq; 3857 } 3858 3859 /* Install the new pwq. */ 3860 mutex_lock(&wq->mutex); 3861 old_pwq = numa_pwq_tbl_install(wq, node, pwq); 3862 goto out_unlock; 3863 3864 use_dfl_pwq: 3865 mutex_lock(&wq->mutex); 3866 spin_lock_irq(&wq->dfl_pwq->pool->lock); 3867 get_pwq(wq->dfl_pwq); 3868 spin_unlock_irq(&wq->dfl_pwq->pool->lock); 3869 old_pwq = numa_pwq_tbl_install(wq, node, wq->dfl_pwq); 3870 out_unlock: 3871 mutex_unlock(&wq->mutex); 3872 put_pwq_unlocked(old_pwq); 3873 } 3874 3875 static int alloc_and_link_pwqs(struct workqueue_struct *wq) 3876 { 3877 bool highpri = wq->flags & WQ_HIGHPRI; 3878 int cpu, ret; 3879 3880 if (!(wq->flags & WQ_UNBOUND)) { 3881 wq->cpu_pwqs = alloc_percpu(struct pool_workqueue); 3882 if (!wq->cpu_pwqs) 3883 return -ENOMEM; 3884 3885 for_each_possible_cpu(cpu) { 3886 struct pool_workqueue *pwq = 3887 per_cpu_ptr(wq->cpu_pwqs, cpu); 3888 struct worker_pool *cpu_pools = 3889 per_cpu(cpu_worker_pools, cpu); 3890 3891 init_pwq(pwq, wq, &cpu_pools[highpri]); 3892 3893 mutex_lock(&wq->mutex); 3894 link_pwq(pwq); 3895 mutex_unlock(&wq->mutex); 3896 } 3897 return 0; 3898 } else if (wq->flags & __WQ_ORDERED) { 3899 ret = apply_workqueue_attrs(wq, ordered_wq_attrs[highpri]); 3900 /* there should only be single pwq for ordering guarantee */ 3901 WARN(!ret && (wq->pwqs.next != &wq->dfl_pwq->pwqs_node || 3902 wq->pwqs.prev != &wq->dfl_pwq->pwqs_node), 3903 "ordering guarantee broken for workqueue %s\n", wq->name); 3904 return ret; 3905 } else { 3906 return apply_workqueue_attrs(wq, unbound_std_wq_attrs[highpri]); 3907 } 3908 } 3909 3910 static int wq_clamp_max_active(int max_active, unsigned int flags, 3911 const char *name) 3912 { 3913 int lim = flags & WQ_UNBOUND ? WQ_UNBOUND_MAX_ACTIVE : WQ_MAX_ACTIVE; 3914 3915 if (max_active < 1 || max_active > lim) 3916 pr_warn("workqueue: max_active %d requested for %s is out of range, clamping between %d and %d\n", 3917 max_active, name, 1, lim); 3918 3919 return clamp_val(max_active, 1, lim); 3920 } 3921 3922 struct workqueue_struct *__alloc_workqueue_key(const char *fmt, 3923 unsigned int flags, 3924 int max_active, 3925 struct lock_class_key *key, 3926 const char *lock_name, ...) 3927 { 3928 size_t tbl_size = 0; 3929 va_list args; 3930 struct workqueue_struct *wq; 3931 struct pool_workqueue *pwq; 3932 3933 /* see the comment above the definition of WQ_POWER_EFFICIENT */ 3934 if ((flags & WQ_POWER_EFFICIENT) && wq_power_efficient) 3935 flags |= WQ_UNBOUND; 3936 3937 /* allocate wq and format name */ 3938 if (flags & WQ_UNBOUND) 3939 tbl_size = nr_node_ids * sizeof(wq->numa_pwq_tbl[0]); 3940 3941 wq = kzalloc(sizeof(*wq) + tbl_size, GFP_KERNEL); 3942 if (!wq) 3943 return NULL; 3944 3945 if (flags & WQ_UNBOUND) { 3946 wq->unbound_attrs = alloc_workqueue_attrs(GFP_KERNEL); 3947 if (!wq->unbound_attrs) 3948 goto err_free_wq; 3949 } 3950 3951 va_start(args, lock_name); 3952 vsnprintf(wq->name, sizeof(wq->name), fmt, args); 3953 va_end(args); 3954 3955 max_active = max_active ?: WQ_DFL_ACTIVE; 3956 max_active = wq_clamp_max_active(max_active, flags, wq->name); 3957 3958 /* init wq */ 3959 wq->flags = flags; 3960 wq->saved_max_active = max_active; 3961 mutex_init(&wq->mutex); 3962 atomic_set(&wq->nr_pwqs_to_flush, 0); 3963 INIT_LIST_HEAD(&wq->pwqs); 3964 INIT_LIST_HEAD(&wq->flusher_queue); 3965 INIT_LIST_HEAD(&wq->flusher_overflow); 3966 INIT_LIST_HEAD(&wq->maydays); 3967 3968 lockdep_init_map(&wq->lockdep_map, lock_name, key, 0); 3969 INIT_LIST_HEAD(&wq->list); 3970 3971 if (alloc_and_link_pwqs(wq) < 0) 3972 goto err_free_wq; 3973 3974 /* 3975 * Workqueues which may be used during memory reclaim should 3976 * have a rescuer to guarantee forward progress. 3977 */ 3978 if (flags & WQ_MEM_RECLAIM) { 3979 struct worker *rescuer; 3980 3981 rescuer = alloc_worker(NUMA_NO_NODE); 3982 if (!rescuer) 3983 goto err_destroy; 3984 3985 rescuer->rescue_wq = wq; 3986 rescuer->task = kthread_create(rescuer_thread, rescuer, "%s", 3987 wq->name); 3988 if (IS_ERR(rescuer->task)) { 3989 kfree(rescuer); 3990 goto err_destroy; 3991 } 3992 3993 wq->rescuer = rescuer; 3994 kthread_bind_mask(rescuer->task, cpu_possible_mask); 3995 wake_up_process(rescuer->task); 3996 } 3997 3998 if ((wq->flags & WQ_SYSFS) && workqueue_sysfs_register(wq)) 3999 goto err_destroy; 4000 4001 /* 4002 * wq_pool_mutex protects global freeze state and workqueues list. 4003 * Grab it, adjust max_active and add the new @wq to workqueues 4004 * list. 4005 */ 4006 mutex_lock(&wq_pool_mutex); 4007 4008 mutex_lock(&wq->mutex); 4009 for_each_pwq(pwq, wq) 4010 pwq_adjust_max_active(pwq); 4011 mutex_unlock(&wq->mutex); 4012 4013 list_add_tail_rcu(&wq->list, &workqueues); 4014 4015 mutex_unlock(&wq_pool_mutex); 4016 4017 return wq; 4018 4019 err_free_wq: 4020 free_workqueue_attrs(wq->unbound_attrs); 4021 kfree(wq); 4022 return NULL; 4023 err_destroy: 4024 destroy_workqueue(wq); 4025 return NULL; 4026 } 4027 EXPORT_SYMBOL_GPL(__alloc_workqueue_key); 4028 4029 /** 4030 * destroy_workqueue - safely terminate a workqueue 4031 * @wq: target workqueue 4032 * 4033 * Safely destroy a workqueue. All work currently pending will be done first. 4034 */ 4035 void destroy_workqueue(struct workqueue_struct *wq) 4036 { 4037 struct pool_workqueue *pwq; 4038 int node; 4039 4040 /* drain it before proceeding with destruction */ 4041 drain_workqueue(wq); 4042 4043 /* sanity checks */ 4044 mutex_lock(&wq->mutex); 4045 for_each_pwq(pwq, wq) { 4046 int i; 4047 4048 for (i = 0; i < WORK_NR_COLORS; i++) { 4049 if (WARN_ON(pwq->nr_in_flight[i])) { 4050 mutex_unlock(&wq->mutex); 4051 show_workqueue_state(); 4052 return; 4053 } 4054 } 4055 4056 if (WARN_ON((pwq != wq->dfl_pwq) && (pwq->refcnt > 1)) || 4057 WARN_ON(pwq->nr_active) || 4058 WARN_ON(!list_empty(&pwq->delayed_works))) { 4059 mutex_unlock(&wq->mutex); 4060 show_workqueue_state(); 4061 return; 4062 } 4063 } 4064 mutex_unlock(&wq->mutex); 4065 4066 /* 4067 * wq list is used to freeze wq, remove from list after 4068 * flushing is complete in case freeze races us. 4069 */ 4070 mutex_lock(&wq_pool_mutex); 4071 list_del_rcu(&wq->list); 4072 mutex_unlock(&wq_pool_mutex); 4073 4074 workqueue_sysfs_unregister(wq); 4075 4076 if (wq->rescuer) 4077 kthread_stop(wq->rescuer->task); 4078 4079 if (!(wq->flags & WQ_UNBOUND)) { 4080 /* 4081 * The base ref is never dropped on per-cpu pwqs. Directly 4082 * schedule RCU free. 4083 */ 4084 call_rcu_sched(&wq->rcu, rcu_free_wq); 4085 } else { 4086 /* 4087 * We're the sole accessor of @wq at this point. Directly 4088 * access numa_pwq_tbl[] and dfl_pwq to put the base refs. 4089 * @wq will be freed when the last pwq is released. 4090 */ 4091 for_each_node(node) { 4092 pwq = rcu_access_pointer(wq->numa_pwq_tbl[node]); 4093 RCU_INIT_POINTER(wq->numa_pwq_tbl[node], NULL); 4094 put_pwq_unlocked(pwq); 4095 } 4096 4097 /* 4098 * Put dfl_pwq. @wq may be freed any time after dfl_pwq is 4099 * put. Don't access it afterwards. 4100 */ 4101 pwq = wq->dfl_pwq; 4102 wq->dfl_pwq = NULL; 4103 put_pwq_unlocked(pwq); 4104 } 4105 } 4106 EXPORT_SYMBOL_GPL(destroy_workqueue); 4107 4108 /** 4109 * workqueue_set_max_active - adjust max_active of a workqueue 4110 * @wq: target workqueue 4111 * @max_active: new max_active value. 4112 * 4113 * Set max_active of @wq to @max_active. 4114 * 4115 * CONTEXT: 4116 * Don't call from IRQ context. 4117 */ 4118 void workqueue_set_max_active(struct workqueue_struct *wq, int max_active) 4119 { 4120 struct pool_workqueue *pwq; 4121 4122 /* disallow meddling with max_active for ordered workqueues */ 4123 if (WARN_ON(wq->flags & __WQ_ORDERED)) 4124 return; 4125 4126 max_active = wq_clamp_max_active(max_active, wq->flags, wq->name); 4127 4128 mutex_lock(&wq->mutex); 4129 4130 wq->saved_max_active = max_active; 4131 4132 for_each_pwq(pwq, wq) 4133 pwq_adjust_max_active(pwq); 4134 4135 mutex_unlock(&wq->mutex); 4136 } 4137 EXPORT_SYMBOL_GPL(workqueue_set_max_active); 4138 4139 /** 4140 * current_is_workqueue_rescuer - is %current workqueue rescuer? 4141 * 4142 * Determine whether %current is a workqueue rescuer. Can be used from 4143 * work functions to determine whether it's being run off the rescuer task. 4144 * 4145 * Return: %true if %current is a workqueue rescuer. %false otherwise. 4146 */ 4147 bool current_is_workqueue_rescuer(void) 4148 { 4149 struct worker *worker = current_wq_worker(); 4150 4151 return worker && worker->rescue_wq; 4152 } 4153 4154 /** 4155 * workqueue_congested - test whether a workqueue is congested 4156 * @cpu: CPU in question 4157 * @wq: target workqueue 4158 * 4159 * Test whether @wq's cpu workqueue for @cpu is congested. There is 4160 * no synchronization around this function and the test result is 4161 * unreliable and only useful as advisory hints or for debugging. 4162 * 4163 * If @cpu is WORK_CPU_UNBOUND, the test is performed on the local CPU. 4164 * Note that both per-cpu and unbound workqueues may be associated with 4165 * multiple pool_workqueues which have separate congested states. A 4166 * workqueue being congested on one CPU doesn't mean the workqueue is also 4167 * contested on other CPUs / NUMA nodes. 4168 * 4169 * Return: 4170 * %true if congested, %false otherwise. 4171 */ 4172 bool workqueue_congested(int cpu, struct workqueue_struct *wq) 4173 { 4174 struct pool_workqueue *pwq; 4175 bool ret; 4176 4177 rcu_read_lock_sched(); 4178 4179 if (cpu == WORK_CPU_UNBOUND) 4180 cpu = smp_processor_id(); 4181 4182 if (!(wq->flags & WQ_UNBOUND)) 4183 pwq = per_cpu_ptr(wq->cpu_pwqs, cpu); 4184 else 4185 pwq = unbound_pwq_by_node(wq, cpu_to_node(cpu)); 4186 4187 ret = !list_empty(&pwq->delayed_works); 4188 rcu_read_unlock_sched(); 4189 4190 return ret; 4191 } 4192 EXPORT_SYMBOL_GPL(workqueue_congested); 4193 4194 /** 4195 * work_busy - test whether a work is currently pending or running 4196 * @work: the work to be tested 4197 * 4198 * Test whether @work is currently pending or running. There is no 4199 * synchronization around this function and the test result is 4200 * unreliable and only useful as advisory hints or for debugging. 4201 * 4202 * Return: 4203 * OR'd bitmask of WORK_BUSY_* bits. 4204 */ 4205 unsigned int work_busy(struct work_struct *work) 4206 { 4207 struct worker_pool *pool; 4208 unsigned long flags; 4209 unsigned int ret = 0; 4210 4211 if (work_pending(work)) 4212 ret |= WORK_BUSY_PENDING; 4213 4214 local_irq_save(flags); 4215 pool = get_work_pool(work); 4216 if (pool) { 4217 spin_lock(&pool->lock); 4218 if (find_worker_executing_work(pool, work)) 4219 ret |= WORK_BUSY_RUNNING; 4220 spin_unlock(&pool->lock); 4221 } 4222 local_irq_restore(flags); 4223 4224 return ret; 4225 } 4226 EXPORT_SYMBOL_GPL(work_busy); 4227 4228 /** 4229 * set_worker_desc - set description for the current work item 4230 * @fmt: printf-style format string 4231 * @...: arguments for the format string 4232 * 4233 * This function can be called by a running work function to describe what 4234 * the work item is about. If the worker task gets dumped, this 4235 * information will be printed out together to help debugging. The 4236 * description can be at most WORKER_DESC_LEN including the trailing '\0'. 4237 */ 4238 void set_worker_desc(const char *fmt, ...) 4239 { 4240 struct worker *worker = current_wq_worker(); 4241 va_list args; 4242 4243 if (worker) { 4244 va_start(args, fmt); 4245 vsnprintf(worker->desc, sizeof(worker->desc), fmt, args); 4246 va_end(args); 4247 worker->desc_valid = true; 4248 } 4249 } 4250 4251 /** 4252 * print_worker_info - print out worker information and description 4253 * @log_lvl: the log level to use when printing 4254 * @task: target task 4255 * 4256 * If @task is a worker and currently executing a work item, print out the 4257 * name of the workqueue being serviced and worker description set with 4258 * set_worker_desc() by the currently executing work item. 4259 * 4260 * This function can be safely called on any task as long as the 4261 * task_struct itself is accessible. While safe, this function isn't 4262 * synchronized and may print out mixups or garbages of limited length. 4263 */ 4264 void print_worker_info(const char *log_lvl, struct task_struct *task) 4265 { 4266 work_func_t *fn = NULL; 4267 char name[WQ_NAME_LEN] = { }; 4268 char desc[WORKER_DESC_LEN] = { }; 4269 struct pool_workqueue *pwq = NULL; 4270 struct workqueue_struct *wq = NULL; 4271 bool desc_valid = false; 4272 struct worker *worker; 4273 4274 if (!(task->flags & PF_WQ_WORKER)) 4275 return; 4276 4277 /* 4278 * This function is called without any synchronization and @task 4279 * could be in any state. Be careful with dereferences. 4280 */ 4281 worker = kthread_probe_data(task); 4282 4283 /* 4284 * Carefully copy the associated workqueue's workfn and name. Keep 4285 * the original last '\0' in case the original contains garbage. 4286 */ 4287 probe_kernel_read(&fn, &worker->current_func, sizeof(fn)); 4288 probe_kernel_read(&pwq, &worker->current_pwq, sizeof(pwq)); 4289 probe_kernel_read(&wq, &pwq->wq, sizeof(wq)); 4290 probe_kernel_read(name, wq->name, sizeof(name) - 1); 4291 4292 /* copy worker description */ 4293 probe_kernel_read(&desc_valid, &worker->desc_valid, sizeof(desc_valid)); 4294 if (desc_valid) 4295 probe_kernel_read(desc, worker->desc, sizeof(desc) - 1); 4296 4297 if (fn || name[0] || desc[0]) { 4298 printk("%sWorkqueue: %s %pf", log_lvl, name, fn); 4299 if (desc[0]) 4300 pr_cont(" (%s)", desc); 4301 pr_cont("\n"); 4302 } 4303 } 4304 4305 static void pr_cont_pool_info(struct worker_pool *pool) 4306 { 4307 pr_cont(" cpus=%*pbl", nr_cpumask_bits, pool->attrs->cpumask); 4308 if (pool->node != NUMA_NO_NODE) 4309 pr_cont(" node=%d", pool->node); 4310 pr_cont(" flags=0x%x nice=%d", pool->flags, pool->attrs->nice); 4311 } 4312 4313 static void pr_cont_work(bool comma, struct work_struct *work) 4314 { 4315 if (work->func == wq_barrier_func) { 4316 struct wq_barrier *barr; 4317 4318 barr = container_of(work, struct wq_barrier, work); 4319 4320 pr_cont("%s BAR(%d)", comma ? "," : "", 4321 task_pid_nr(barr->task)); 4322 } else { 4323 pr_cont("%s %pf", comma ? "," : "", work->func); 4324 } 4325 } 4326 4327 static void show_pwq(struct pool_workqueue *pwq) 4328 { 4329 struct worker_pool *pool = pwq->pool; 4330 struct work_struct *work; 4331 struct worker *worker; 4332 bool has_in_flight = false, has_pending = false; 4333 int bkt; 4334 4335 pr_info(" pwq %d:", pool->id); 4336 pr_cont_pool_info(pool); 4337 4338 pr_cont(" active=%d/%d%s\n", pwq->nr_active, pwq->max_active, 4339 !list_empty(&pwq->mayday_node) ? " MAYDAY" : ""); 4340 4341 hash_for_each(pool->busy_hash, bkt, worker, hentry) { 4342 if (worker->current_pwq == pwq) { 4343 has_in_flight = true; 4344 break; 4345 } 4346 } 4347 if (has_in_flight) { 4348 bool comma = false; 4349 4350 pr_info(" in-flight:"); 4351 hash_for_each(pool->busy_hash, bkt, worker, hentry) { 4352 if (worker->current_pwq != pwq) 4353 continue; 4354 4355 pr_cont("%s %d%s:%pf", comma ? "," : "", 4356 task_pid_nr(worker->task), 4357 worker == pwq->wq->rescuer ? "(RESCUER)" : "", 4358 worker->current_func); 4359 list_for_each_entry(work, &worker->scheduled, entry) 4360 pr_cont_work(false, work); 4361 comma = true; 4362 } 4363 pr_cont("\n"); 4364 } 4365 4366 list_for_each_entry(work, &pool->worklist, entry) { 4367 if (get_work_pwq(work) == pwq) { 4368 has_pending = true; 4369 break; 4370 } 4371 } 4372 if (has_pending) { 4373 bool comma = false; 4374 4375 pr_info(" pending:"); 4376 list_for_each_entry(work, &pool->worklist, entry) { 4377 if (get_work_pwq(work) != pwq) 4378 continue; 4379 4380 pr_cont_work(comma, work); 4381 comma = !(*work_data_bits(work) & WORK_STRUCT_LINKED); 4382 } 4383 pr_cont("\n"); 4384 } 4385 4386 if (!list_empty(&pwq->delayed_works)) { 4387 bool comma = false; 4388 4389 pr_info(" delayed:"); 4390 list_for_each_entry(work, &pwq->delayed_works, entry) { 4391 pr_cont_work(comma, work); 4392 comma = !(*work_data_bits(work) & WORK_STRUCT_LINKED); 4393 } 4394 pr_cont("\n"); 4395 } 4396 } 4397 4398 /** 4399 * show_workqueue_state - dump workqueue state 4400 * 4401 * Called from a sysrq handler or try_to_freeze_tasks() and prints out 4402 * all busy workqueues and pools. 4403 */ 4404 void show_workqueue_state(void) 4405 { 4406 struct workqueue_struct *wq; 4407 struct worker_pool *pool; 4408 unsigned long flags; 4409 int pi; 4410 4411 rcu_read_lock_sched(); 4412 4413 pr_info("Showing busy workqueues and worker pools:\n"); 4414 4415 list_for_each_entry_rcu(wq, &workqueues, list) { 4416 struct pool_workqueue *pwq; 4417 bool idle = true; 4418 4419 for_each_pwq(pwq, wq) { 4420 if (pwq->nr_active || !list_empty(&pwq->delayed_works)) { 4421 idle = false; 4422 break; 4423 } 4424 } 4425 if (idle) 4426 continue; 4427 4428 pr_info("workqueue %s: flags=0x%x\n", wq->name, wq->flags); 4429 4430 for_each_pwq(pwq, wq) { 4431 spin_lock_irqsave(&pwq->pool->lock, flags); 4432 if (pwq->nr_active || !list_empty(&pwq->delayed_works)) 4433 show_pwq(pwq); 4434 spin_unlock_irqrestore(&pwq->pool->lock, flags); 4435 } 4436 } 4437 4438 for_each_pool(pool, pi) { 4439 struct worker *worker; 4440 bool first = true; 4441 4442 spin_lock_irqsave(&pool->lock, flags); 4443 if (pool->nr_workers == pool->nr_idle) 4444 goto next_pool; 4445 4446 pr_info("pool %d:", pool->id); 4447 pr_cont_pool_info(pool); 4448 pr_cont(" hung=%us workers=%d", 4449 jiffies_to_msecs(jiffies - pool->watchdog_ts) / 1000, 4450 pool->nr_workers); 4451 if (pool->manager) 4452 pr_cont(" manager: %d", 4453 task_pid_nr(pool->manager->task)); 4454 list_for_each_entry(worker, &pool->idle_list, entry) { 4455 pr_cont(" %s%d", first ? "idle: " : "", 4456 task_pid_nr(worker->task)); 4457 first = false; 4458 } 4459 pr_cont("\n"); 4460 next_pool: 4461 spin_unlock_irqrestore(&pool->lock, flags); 4462 } 4463 4464 rcu_read_unlock_sched(); 4465 } 4466 4467 /* 4468 * CPU hotplug. 4469 * 4470 * There are two challenges in supporting CPU hotplug. Firstly, there 4471 * are a lot of assumptions on strong associations among work, pwq and 4472 * pool which make migrating pending and scheduled works very 4473 * difficult to implement without impacting hot paths. Secondly, 4474 * worker pools serve mix of short, long and very long running works making 4475 * blocked draining impractical. 4476 * 4477 * This is solved by allowing the pools to be disassociated from the CPU 4478 * running as an unbound one and allowing it to be reattached later if the 4479 * cpu comes back online. 4480 */ 4481 4482 static void wq_unbind_fn(struct work_struct *work) 4483 { 4484 int cpu = smp_processor_id(); 4485 struct worker_pool *pool; 4486 struct worker *worker; 4487 4488 for_each_cpu_worker_pool(pool, cpu) { 4489 mutex_lock(&pool->attach_mutex); 4490 spin_lock_irq(&pool->lock); 4491 4492 /* 4493 * We've blocked all attach/detach operations. Make all workers 4494 * unbound and set DISASSOCIATED. Before this, all workers 4495 * except for the ones which are still executing works from 4496 * before the last CPU down must be on the cpu. After 4497 * this, they may become diasporas. 4498 */ 4499 for_each_pool_worker(worker, pool) 4500 worker->flags |= WORKER_UNBOUND; 4501 4502 pool->flags |= POOL_DISASSOCIATED; 4503 4504 spin_unlock_irq(&pool->lock); 4505 mutex_unlock(&pool->attach_mutex); 4506 4507 /* 4508 * Call schedule() so that we cross rq->lock and thus can 4509 * guarantee sched callbacks see the %WORKER_UNBOUND flag. 4510 * This is necessary as scheduler callbacks may be invoked 4511 * from other cpus. 4512 */ 4513 schedule(); 4514 4515 /* 4516 * Sched callbacks are disabled now. Zap nr_running. 4517 * After this, nr_running stays zero and need_more_worker() 4518 * and keep_working() are always true as long as the 4519 * worklist is not empty. This pool now behaves as an 4520 * unbound (in terms of concurrency management) pool which 4521 * are served by workers tied to the pool. 4522 */ 4523 atomic_set(&pool->nr_running, 0); 4524 4525 /* 4526 * With concurrency management just turned off, a busy 4527 * worker blocking could lead to lengthy stalls. Kick off 4528 * unbound chain execution of currently pending work items. 4529 */ 4530 spin_lock_irq(&pool->lock); 4531 wake_up_worker(pool); 4532 spin_unlock_irq(&pool->lock); 4533 } 4534 } 4535 4536 /** 4537 * rebind_workers - rebind all workers of a pool to the associated CPU 4538 * @pool: pool of interest 4539 * 4540 * @pool->cpu is coming online. Rebind all workers to the CPU. 4541 */ 4542 static void rebind_workers(struct worker_pool *pool) 4543 { 4544 struct worker *worker; 4545 4546 lockdep_assert_held(&pool->attach_mutex); 4547 4548 /* 4549 * Restore CPU affinity of all workers. As all idle workers should 4550 * be on the run-queue of the associated CPU before any local 4551 * wake-ups for concurrency management happen, restore CPU affinity 4552 * of all workers first and then clear UNBOUND. As we're called 4553 * from CPU_ONLINE, the following shouldn't fail. 4554 */ 4555 for_each_pool_worker(worker, pool) 4556 WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, 4557 pool->attrs->cpumask) < 0); 4558 4559 spin_lock_irq(&pool->lock); 4560 4561 /* 4562 * XXX: CPU hotplug notifiers are weird and can call DOWN_FAILED 4563 * w/o preceding DOWN_PREPARE. Work around it. CPU hotplug is 4564 * being reworked and this can go away in time. 4565 */ 4566 if (!(pool->flags & POOL_DISASSOCIATED)) { 4567 spin_unlock_irq(&pool->lock); 4568 return; 4569 } 4570 4571 pool->flags &= ~POOL_DISASSOCIATED; 4572 4573 for_each_pool_worker(worker, pool) { 4574 unsigned int worker_flags = worker->flags; 4575 4576 /* 4577 * A bound idle worker should actually be on the runqueue 4578 * of the associated CPU for local wake-ups targeting it to 4579 * work. Kick all idle workers so that they migrate to the 4580 * associated CPU. Doing this in the same loop as 4581 * replacing UNBOUND with REBOUND is safe as no worker will 4582 * be bound before @pool->lock is released. 4583 */ 4584 if (worker_flags & WORKER_IDLE) 4585 wake_up_process(worker->task); 4586 4587 /* 4588 * We want to clear UNBOUND but can't directly call 4589 * worker_clr_flags() or adjust nr_running. Atomically 4590 * replace UNBOUND with another NOT_RUNNING flag REBOUND. 4591 * @worker will clear REBOUND using worker_clr_flags() when 4592 * it initiates the next execution cycle thus restoring 4593 * concurrency management. Note that when or whether 4594 * @worker clears REBOUND doesn't affect correctness. 4595 * 4596 * ACCESS_ONCE() is necessary because @worker->flags may be 4597 * tested without holding any lock in 4598 * wq_worker_waking_up(). Without it, NOT_RUNNING test may 4599 * fail incorrectly leading to premature concurrency 4600 * management operations. 4601 */ 4602 WARN_ON_ONCE(!(worker_flags & WORKER_UNBOUND)); 4603 worker_flags |= WORKER_REBOUND; 4604 worker_flags &= ~WORKER_UNBOUND; 4605 ACCESS_ONCE(worker->flags) = worker_flags; 4606 } 4607 4608 spin_unlock_irq(&pool->lock); 4609 } 4610 4611 /** 4612 * restore_unbound_workers_cpumask - restore cpumask of unbound workers 4613 * @pool: unbound pool of interest 4614 * @cpu: the CPU which is coming up 4615 * 4616 * An unbound pool may end up with a cpumask which doesn't have any online 4617 * CPUs. When a worker of such pool get scheduled, the scheduler resets 4618 * its cpus_allowed. If @cpu is in @pool's cpumask which didn't have any 4619 * online CPU before, cpus_allowed of all its workers should be restored. 4620 */ 4621 static void restore_unbound_workers_cpumask(struct worker_pool *pool, int cpu) 4622 { 4623 static cpumask_t cpumask; 4624 struct worker *worker; 4625 4626 lockdep_assert_held(&pool->attach_mutex); 4627 4628 /* is @cpu allowed for @pool? */ 4629 if (!cpumask_test_cpu(cpu, pool->attrs->cpumask)) 4630 return; 4631 4632 cpumask_and(&cpumask, pool->attrs->cpumask, cpu_online_mask); 4633 4634 /* as we're called from CPU_ONLINE, the following shouldn't fail */ 4635 for_each_pool_worker(worker, pool) 4636 WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, &cpumask) < 0); 4637 } 4638 4639 int workqueue_prepare_cpu(unsigned int cpu) 4640 { 4641 struct worker_pool *pool; 4642 4643 for_each_cpu_worker_pool(pool, cpu) { 4644 if (pool->nr_workers) 4645 continue; 4646 if (!create_worker(pool)) 4647 return -ENOMEM; 4648 } 4649 return 0; 4650 } 4651 4652 int workqueue_online_cpu(unsigned int cpu) 4653 { 4654 struct worker_pool *pool; 4655 struct workqueue_struct *wq; 4656 int pi; 4657 4658 mutex_lock(&wq_pool_mutex); 4659 4660 for_each_pool(pool, pi) { 4661 mutex_lock(&pool->attach_mutex); 4662 4663 if (pool->cpu == cpu) 4664 rebind_workers(pool); 4665 else if (pool->cpu < 0) 4666 restore_unbound_workers_cpumask(pool, cpu); 4667 4668 mutex_unlock(&pool->attach_mutex); 4669 } 4670 4671 /* update NUMA affinity of unbound workqueues */ 4672 list_for_each_entry(wq, &workqueues, list) 4673 wq_update_unbound_numa(wq, cpu, true); 4674 4675 mutex_unlock(&wq_pool_mutex); 4676 return 0; 4677 } 4678 4679 int workqueue_offline_cpu(unsigned int cpu) 4680 { 4681 struct work_struct unbind_work; 4682 struct workqueue_struct *wq; 4683 4684 /* unbinding per-cpu workers should happen on the local CPU */ 4685 INIT_WORK_ONSTACK(&unbind_work, wq_unbind_fn); 4686 queue_work_on(cpu, system_highpri_wq, &unbind_work); 4687 4688 /* update NUMA affinity of unbound workqueues */ 4689 mutex_lock(&wq_pool_mutex); 4690 list_for_each_entry(wq, &workqueues, list) 4691 wq_update_unbound_numa(wq, cpu, false); 4692 mutex_unlock(&wq_pool_mutex); 4693 4694 /* wait for per-cpu unbinding to finish */ 4695 flush_work(&unbind_work); 4696 destroy_work_on_stack(&unbind_work); 4697 return 0; 4698 } 4699 4700 #ifdef CONFIG_SMP 4701 4702 struct work_for_cpu { 4703 struct work_struct work; 4704 long (*fn)(void *); 4705 void *arg; 4706 long ret; 4707 }; 4708 4709 static void work_for_cpu_fn(struct work_struct *work) 4710 { 4711 struct work_for_cpu *wfc = container_of(work, struct work_for_cpu, work); 4712 4713 wfc->ret = wfc->fn(wfc->arg); 4714 } 4715 4716 /** 4717 * work_on_cpu - run a function in thread context on a particular cpu 4718 * @cpu: the cpu to run on 4719 * @fn: the function to run 4720 * @arg: the function arg 4721 * 4722 * It is up to the caller to ensure that the cpu doesn't go offline. 4723 * The caller must not hold any locks which would prevent @fn from completing. 4724 * 4725 * Return: The value @fn returns. 4726 */ 4727 long work_on_cpu(int cpu, long (*fn)(void *), void *arg) 4728 { 4729 struct work_for_cpu wfc = { .fn = fn, .arg = arg }; 4730 4731 INIT_WORK_ONSTACK(&wfc.work, work_for_cpu_fn); 4732 schedule_work_on(cpu, &wfc.work); 4733 flush_work(&wfc.work); 4734 destroy_work_on_stack(&wfc.work); 4735 return wfc.ret; 4736 } 4737 EXPORT_SYMBOL_GPL(work_on_cpu); 4738 #endif /* CONFIG_SMP */ 4739 4740 #ifdef CONFIG_FREEZER 4741 4742 /** 4743 * freeze_workqueues_begin - begin freezing workqueues 4744 * 4745 * Start freezing workqueues. After this function returns, all freezable 4746 * workqueues will queue new works to their delayed_works list instead of 4747 * pool->worklist. 4748 * 4749 * CONTEXT: 4750 * Grabs and releases wq_pool_mutex, wq->mutex and pool->lock's. 4751 */ 4752 void freeze_workqueues_begin(void) 4753 { 4754 struct workqueue_struct *wq; 4755 struct pool_workqueue *pwq; 4756 4757 mutex_lock(&wq_pool_mutex); 4758 4759 WARN_ON_ONCE(workqueue_freezing); 4760 workqueue_freezing = true; 4761 4762 list_for_each_entry(wq, &workqueues, list) { 4763 mutex_lock(&wq->mutex); 4764 for_each_pwq(pwq, wq) 4765 pwq_adjust_max_active(pwq); 4766 mutex_unlock(&wq->mutex); 4767 } 4768 4769 mutex_unlock(&wq_pool_mutex); 4770 } 4771 4772 /** 4773 * freeze_workqueues_busy - are freezable workqueues still busy? 4774 * 4775 * Check whether freezing is complete. This function must be called 4776 * between freeze_workqueues_begin() and thaw_workqueues(). 4777 * 4778 * CONTEXT: 4779 * Grabs and releases wq_pool_mutex. 4780 * 4781 * Return: 4782 * %true if some freezable workqueues are still busy. %false if freezing 4783 * is complete. 4784 */ 4785 bool freeze_workqueues_busy(void) 4786 { 4787 bool busy = false; 4788 struct workqueue_struct *wq; 4789 struct pool_workqueue *pwq; 4790 4791 mutex_lock(&wq_pool_mutex); 4792 4793 WARN_ON_ONCE(!workqueue_freezing); 4794 4795 list_for_each_entry(wq, &workqueues, list) { 4796 if (!(wq->flags & WQ_FREEZABLE)) 4797 continue; 4798 /* 4799 * nr_active is monotonically decreasing. It's safe 4800 * to peek without lock. 4801 */ 4802 rcu_read_lock_sched(); 4803 for_each_pwq(pwq, wq) { 4804 WARN_ON_ONCE(pwq->nr_active < 0); 4805 if (pwq->nr_active) { 4806 busy = true; 4807 rcu_read_unlock_sched(); 4808 goto out_unlock; 4809 } 4810 } 4811 rcu_read_unlock_sched(); 4812 } 4813 out_unlock: 4814 mutex_unlock(&wq_pool_mutex); 4815 return busy; 4816 } 4817 4818 /** 4819 * thaw_workqueues - thaw workqueues 4820 * 4821 * Thaw workqueues. Normal queueing is restored and all collected 4822 * frozen works are transferred to their respective pool worklists. 4823 * 4824 * CONTEXT: 4825 * Grabs and releases wq_pool_mutex, wq->mutex and pool->lock's. 4826 */ 4827 void thaw_workqueues(void) 4828 { 4829 struct workqueue_struct *wq; 4830 struct pool_workqueue *pwq; 4831 4832 mutex_lock(&wq_pool_mutex); 4833 4834 if (!workqueue_freezing) 4835 goto out_unlock; 4836 4837 workqueue_freezing = false; 4838 4839 /* restore max_active and repopulate worklist */ 4840 list_for_each_entry(wq, &workqueues, list) { 4841 mutex_lock(&wq->mutex); 4842 for_each_pwq(pwq, wq) 4843 pwq_adjust_max_active(pwq); 4844 mutex_unlock(&wq->mutex); 4845 } 4846 4847 out_unlock: 4848 mutex_unlock(&wq_pool_mutex); 4849 } 4850 #endif /* CONFIG_FREEZER */ 4851 4852 static int workqueue_apply_unbound_cpumask(void) 4853 { 4854 LIST_HEAD(ctxs); 4855 int ret = 0; 4856 struct workqueue_struct *wq; 4857 struct apply_wqattrs_ctx *ctx, *n; 4858 4859 lockdep_assert_held(&wq_pool_mutex); 4860 4861 list_for_each_entry(wq, &workqueues, list) { 4862 if (!(wq->flags & WQ_UNBOUND)) 4863 continue; 4864 /* creating multiple pwqs breaks ordering guarantee */ 4865 if (wq->flags & __WQ_ORDERED) 4866 continue; 4867 4868 ctx = apply_wqattrs_prepare(wq, wq->unbound_attrs); 4869 if (!ctx) { 4870 ret = -ENOMEM; 4871 break; 4872 } 4873 4874 list_add_tail(&ctx->list, &ctxs); 4875 } 4876 4877 list_for_each_entry_safe(ctx, n, &ctxs, list) { 4878 if (!ret) 4879 apply_wqattrs_commit(ctx); 4880 apply_wqattrs_cleanup(ctx); 4881 } 4882 4883 return ret; 4884 } 4885 4886 /** 4887 * workqueue_set_unbound_cpumask - Set the low-level unbound cpumask 4888 * @cpumask: the cpumask to set 4889 * 4890 * The low-level workqueues cpumask is a global cpumask that limits 4891 * the affinity of all unbound workqueues. This function check the @cpumask 4892 * and apply it to all unbound workqueues and updates all pwqs of them. 4893 * 4894 * Retun: 0 - Success 4895 * -EINVAL - Invalid @cpumask 4896 * -ENOMEM - Failed to allocate memory for attrs or pwqs. 4897 */ 4898 int workqueue_set_unbound_cpumask(cpumask_var_t cpumask) 4899 { 4900 int ret = -EINVAL; 4901 cpumask_var_t saved_cpumask; 4902 4903 if (!zalloc_cpumask_var(&saved_cpumask, GFP_KERNEL)) 4904 return -ENOMEM; 4905 4906 cpumask_and(cpumask, cpumask, cpu_possible_mask); 4907 if (!cpumask_empty(cpumask)) { 4908 apply_wqattrs_lock(); 4909 4910 /* save the old wq_unbound_cpumask. */ 4911 cpumask_copy(saved_cpumask, wq_unbound_cpumask); 4912 4913 /* update wq_unbound_cpumask at first and apply it to wqs. */ 4914 cpumask_copy(wq_unbound_cpumask, cpumask); 4915 ret = workqueue_apply_unbound_cpumask(); 4916 4917 /* restore the wq_unbound_cpumask when failed. */ 4918 if (ret < 0) 4919 cpumask_copy(wq_unbound_cpumask, saved_cpumask); 4920 4921 apply_wqattrs_unlock(); 4922 } 4923 4924 free_cpumask_var(saved_cpumask); 4925 return ret; 4926 } 4927 4928 #ifdef CONFIG_SYSFS 4929 /* 4930 * Workqueues with WQ_SYSFS flag set is visible to userland via 4931 * /sys/bus/workqueue/devices/WQ_NAME. All visible workqueues have the 4932 * following attributes. 4933 * 4934 * per_cpu RO bool : whether the workqueue is per-cpu or unbound 4935 * max_active RW int : maximum number of in-flight work items 4936 * 4937 * Unbound workqueues have the following extra attributes. 4938 * 4939 * id RO int : the associated pool ID 4940 * nice RW int : nice value of the workers 4941 * cpumask RW mask : bitmask of allowed CPUs for the workers 4942 */ 4943 struct wq_device { 4944 struct workqueue_struct *wq; 4945 struct device dev; 4946 }; 4947 4948 static struct workqueue_struct *dev_to_wq(struct device *dev) 4949 { 4950 struct wq_device *wq_dev = container_of(dev, struct wq_device, dev); 4951 4952 return wq_dev->wq; 4953 } 4954 4955 static ssize_t per_cpu_show(struct device *dev, struct device_attribute *attr, 4956 char *buf) 4957 { 4958 struct workqueue_struct *wq = dev_to_wq(dev); 4959 4960 return scnprintf(buf, PAGE_SIZE, "%d\n", (bool)!(wq->flags & WQ_UNBOUND)); 4961 } 4962 static DEVICE_ATTR_RO(per_cpu); 4963 4964 static ssize_t max_active_show(struct device *dev, 4965 struct device_attribute *attr, char *buf) 4966 { 4967 struct workqueue_struct *wq = dev_to_wq(dev); 4968 4969 return scnprintf(buf, PAGE_SIZE, "%d\n", wq->saved_max_active); 4970 } 4971 4972 static ssize_t max_active_store(struct device *dev, 4973 struct device_attribute *attr, const char *buf, 4974 size_t count) 4975 { 4976 struct workqueue_struct *wq = dev_to_wq(dev); 4977 int val; 4978 4979 if (sscanf(buf, "%d", &val) != 1 || val <= 0) 4980 return -EINVAL; 4981 4982 workqueue_set_max_active(wq, val); 4983 return count; 4984 } 4985 static DEVICE_ATTR_RW(max_active); 4986 4987 static struct attribute *wq_sysfs_attrs[] = { 4988 &dev_attr_per_cpu.attr, 4989 &dev_attr_max_active.attr, 4990 NULL, 4991 }; 4992 ATTRIBUTE_GROUPS(wq_sysfs); 4993 4994 static ssize_t wq_pool_ids_show(struct device *dev, 4995 struct device_attribute *attr, char *buf) 4996 { 4997 struct workqueue_struct *wq = dev_to_wq(dev); 4998 const char *delim = ""; 4999 int node, written = 0; 5000 5001 rcu_read_lock_sched(); 5002 for_each_node(node) { 5003 written += scnprintf(buf + written, PAGE_SIZE - written, 5004 "%s%d:%d", delim, node, 5005 unbound_pwq_by_node(wq, node)->pool->id); 5006 delim = " "; 5007 } 5008 written += scnprintf(buf + written, PAGE_SIZE - written, "\n"); 5009 rcu_read_unlock_sched(); 5010 5011 return written; 5012 } 5013 5014 static ssize_t wq_nice_show(struct device *dev, struct device_attribute *attr, 5015 char *buf) 5016 { 5017 struct workqueue_struct *wq = dev_to_wq(dev); 5018 int written; 5019 5020 mutex_lock(&wq->mutex); 5021 written = scnprintf(buf, PAGE_SIZE, "%d\n", wq->unbound_attrs->nice); 5022 mutex_unlock(&wq->mutex); 5023 5024 return written; 5025 } 5026 5027 /* prepare workqueue_attrs for sysfs store operations */ 5028 static struct workqueue_attrs *wq_sysfs_prep_attrs(struct workqueue_struct *wq) 5029 { 5030 struct workqueue_attrs *attrs; 5031 5032 lockdep_assert_held(&wq_pool_mutex); 5033 5034 attrs = alloc_workqueue_attrs(GFP_KERNEL); 5035 if (!attrs) 5036 return NULL; 5037 5038 copy_workqueue_attrs(attrs, wq->unbound_attrs); 5039 return attrs; 5040 } 5041 5042 static ssize_t wq_nice_store(struct device *dev, struct device_attribute *attr, 5043 const char *buf, size_t count) 5044 { 5045 struct workqueue_struct *wq = dev_to_wq(dev); 5046 struct workqueue_attrs *attrs; 5047 int ret = -ENOMEM; 5048 5049 apply_wqattrs_lock(); 5050 5051 attrs = wq_sysfs_prep_attrs(wq); 5052 if (!attrs) 5053 goto out_unlock; 5054 5055 if (sscanf(buf, "%d", &attrs->nice) == 1 && 5056 attrs->nice >= MIN_NICE && attrs->nice <= MAX_NICE) 5057 ret = apply_workqueue_attrs_locked(wq, attrs); 5058 else 5059 ret = -EINVAL; 5060 5061 out_unlock: 5062 apply_wqattrs_unlock(); 5063 free_workqueue_attrs(attrs); 5064 return ret ?: count; 5065 } 5066 5067 static ssize_t wq_cpumask_show(struct device *dev, 5068 struct device_attribute *attr, char *buf) 5069 { 5070 struct workqueue_struct *wq = dev_to_wq(dev); 5071 int written; 5072 5073 mutex_lock(&wq->mutex); 5074 written = scnprintf(buf, PAGE_SIZE, "%*pb\n", 5075 cpumask_pr_args(wq->unbound_attrs->cpumask)); 5076 mutex_unlock(&wq->mutex); 5077 return written; 5078 } 5079 5080 static ssize_t wq_cpumask_store(struct device *dev, 5081 struct device_attribute *attr, 5082 const char *buf, size_t count) 5083 { 5084 struct workqueue_struct *wq = dev_to_wq(dev); 5085 struct workqueue_attrs *attrs; 5086 int ret = -ENOMEM; 5087 5088 apply_wqattrs_lock(); 5089 5090 attrs = wq_sysfs_prep_attrs(wq); 5091 if (!attrs) 5092 goto out_unlock; 5093 5094 ret = cpumask_parse(buf, attrs->cpumask); 5095 if (!ret) 5096 ret = apply_workqueue_attrs_locked(wq, attrs); 5097 5098 out_unlock: 5099 apply_wqattrs_unlock(); 5100 free_workqueue_attrs(attrs); 5101 return ret ?: count; 5102 } 5103 5104 static ssize_t wq_numa_show(struct device *dev, struct device_attribute *attr, 5105 char *buf) 5106 { 5107 struct workqueue_struct *wq = dev_to_wq(dev); 5108 int written; 5109 5110 mutex_lock(&wq->mutex); 5111 written = scnprintf(buf, PAGE_SIZE, "%d\n", 5112 !wq->unbound_attrs->no_numa); 5113 mutex_unlock(&wq->mutex); 5114 5115 return written; 5116 } 5117 5118 static ssize_t wq_numa_store(struct device *dev, struct device_attribute *attr, 5119 const char *buf, size_t count) 5120 { 5121 struct workqueue_struct *wq = dev_to_wq(dev); 5122 struct workqueue_attrs *attrs; 5123 int v, ret = -ENOMEM; 5124 5125 apply_wqattrs_lock(); 5126 5127 attrs = wq_sysfs_prep_attrs(wq); 5128 if (!attrs) 5129 goto out_unlock; 5130 5131 ret = -EINVAL; 5132 if (sscanf(buf, "%d", &v) == 1) { 5133 attrs->no_numa = !v; 5134 ret = apply_workqueue_attrs_locked(wq, attrs); 5135 } 5136 5137 out_unlock: 5138 apply_wqattrs_unlock(); 5139 free_workqueue_attrs(attrs); 5140 return ret ?: count; 5141 } 5142 5143 static struct device_attribute wq_sysfs_unbound_attrs[] = { 5144 __ATTR(pool_ids, 0444, wq_pool_ids_show, NULL), 5145 __ATTR(nice, 0644, wq_nice_show, wq_nice_store), 5146 __ATTR(cpumask, 0644, wq_cpumask_show, wq_cpumask_store), 5147 __ATTR(numa, 0644, wq_numa_show, wq_numa_store), 5148 __ATTR_NULL, 5149 }; 5150 5151 static struct bus_type wq_subsys = { 5152 .name = "workqueue", 5153 .dev_groups = wq_sysfs_groups, 5154 }; 5155 5156 static ssize_t wq_unbound_cpumask_show(struct device *dev, 5157 struct device_attribute *attr, char *buf) 5158 { 5159 int written; 5160 5161 mutex_lock(&wq_pool_mutex); 5162 written = scnprintf(buf, PAGE_SIZE, "%*pb\n", 5163 cpumask_pr_args(wq_unbound_cpumask)); 5164 mutex_unlock(&wq_pool_mutex); 5165 5166 return written; 5167 } 5168 5169 static ssize_t wq_unbound_cpumask_store(struct device *dev, 5170 struct device_attribute *attr, const char *buf, size_t count) 5171 { 5172 cpumask_var_t cpumask; 5173 int ret; 5174 5175 if (!zalloc_cpumask_var(&cpumask, GFP_KERNEL)) 5176 return -ENOMEM; 5177 5178 ret = cpumask_parse(buf, cpumask); 5179 if (!ret) 5180 ret = workqueue_set_unbound_cpumask(cpumask); 5181 5182 free_cpumask_var(cpumask); 5183 return ret ? ret : count; 5184 } 5185 5186 static struct device_attribute wq_sysfs_cpumask_attr = 5187 __ATTR(cpumask, 0644, wq_unbound_cpumask_show, 5188 wq_unbound_cpumask_store); 5189 5190 static int __init wq_sysfs_init(void) 5191 { 5192 int err; 5193 5194 err = subsys_virtual_register(&wq_subsys, NULL); 5195 if (err) 5196 return err; 5197 5198 return device_create_file(wq_subsys.dev_root, &wq_sysfs_cpumask_attr); 5199 } 5200 core_initcall(wq_sysfs_init); 5201 5202 static void wq_device_release(struct device *dev) 5203 { 5204 struct wq_device *wq_dev = container_of(dev, struct wq_device, dev); 5205 5206 kfree(wq_dev); 5207 } 5208 5209 /** 5210 * workqueue_sysfs_register - make a workqueue visible in sysfs 5211 * @wq: the workqueue to register 5212 * 5213 * Expose @wq in sysfs under /sys/bus/workqueue/devices. 5214 * alloc_workqueue*() automatically calls this function if WQ_SYSFS is set 5215 * which is the preferred method. 5216 * 5217 * Workqueue user should use this function directly iff it wants to apply 5218 * workqueue_attrs before making the workqueue visible in sysfs; otherwise, 5219 * apply_workqueue_attrs() may race against userland updating the 5220 * attributes. 5221 * 5222 * Return: 0 on success, -errno on failure. 5223 */ 5224 int workqueue_sysfs_register(struct workqueue_struct *wq) 5225 { 5226 struct wq_device *wq_dev; 5227 int ret; 5228 5229 /* 5230 * Adjusting max_active or creating new pwqs by applying 5231 * attributes breaks ordering guarantee. Disallow exposing ordered 5232 * workqueues. 5233 */ 5234 if (WARN_ON(wq->flags & __WQ_ORDERED)) 5235 return -EINVAL; 5236 5237 wq->wq_dev = wq_dev = kzalloc(sizeof(*wq_dev), GFP_KERNEL); 5238 if (!wq_dev) 5239 return -ENOMEM; 5240 5241 wq_dev->wq = wq; 5242 wq_dev->dev.bus = &wq_subsys; 5243 wq_dev->dev.release = wq_device_release; 5244 dev_set_name(&wq_dev->dev, "%s", wq->name); 5245 5246 /* 5247 * unbound_attrs are created separately. Suppress uevent until 5248 * everything is ready. 5249 */ 5250 dev_set_uevent_suppress(&wq_dev->dev, true); 5251 5252 ret = device_register(&wq_dev->dev); 5253 if (ret) { 5254 kfree(wq_dev); 5255 wq->wq_dev = NULL; 5256 return ret; 5257 } 5258 5259 if (wq->flags & WQ_UNBOUND) { 5260 struct device_attribute *attr; 5261 5262 for (attr = wq_sysfs_unbound_attrs; attr->attr.name; attr++) { 5263 ret = device_create_file(&wq_dev->dev, attr); 5264 if (ret) { 5265 device_unregister(&wq_dev->dev); 5266 wq->wq_dev = NULL; 5267 return ret; 5268 } 5269 } 5270 } 5271 5272 dev_set_uevent_suppress(&wq_dev->dev, false); 5273 kobject_uevent(&wq_dev->dev.kobj, KOBJ_ADD); 5274 return 0; 5275 } 5276 5277 /** 5278 * workqueue_sysfs_unregister - undo workqueue_sysfs_register() 5279 * @wq: the workqueue to unregister 5280 * 5281 * If @wq is registered to sysfs by workqueue_sysfs_register(), unregister. 5282 */ 5283 static void workqueue_sysfs_unregister(struct workqueue_struct *wq) 5284 { 5285 struct wq_device *wq_dev = wq->wq_dev; 5286 5287 if (!wq->wq_dev) 5288 return; 5289 5290 wq->wq_dev = NULL; 5291 device_unregister(&wq_dev->dev); 5292 } 5293 #else /* CONFIG_SYSFS */ 5294 static void workqueue_sysfs_unregister(struct workqueue_struct *wq) { } 5295 #endif /* CONFIG_SYSFS */ 5296 5297 /* 5298 * Workqueue watchdog. 5299 * 5300 * Stall may be caused by various bugs - missing WQ_MEM_RECLAIM, illegal 5301 * flush dependency, a concurrency managed work item which stays RUNNING 5302 * indefinitely. Workqueue stalls can be very difficult to debug as the 5303 * usual warning mechanisms don't trigger and internal workqueue state is 5304 * largely opaque. 5305 * 5306 * Workqueue watchdog monitors all worker pools periodically and dumps 5307 * state if some pools failed to make forward progress for a while where 5308 * forward progress is defined as the first item on ->worklist changing. 5309 * 5310 * This mechanism is controlled through the kernel parameter 5311 * "workqueue.watchdog_thresh" which can be updated at runtime through the 5312 * corresponding sysfs parameter file. 5313 */ 5314 #ifdef CONFIG_WQ_WATCHDOG 5315 5316 static void wq_watchdog_timer_fn(unsigned long data); 5317 5318 static unsigned long wq_watchdog_thresh = 30; 5319 static struct timer_list wq_watchdog_timer = 5320 TIMER_DEFERRED_INITIALIZER(wq_watchdog_timer_fn, 0, 0); 5321 5322 static unsigned long wq_watchdog_touched = INITIAL_JIFFIES; 5323 static DEFINE_PER_CPU(unsigned long, wq_watchdog_touched_cpu) = INITIAL_JIFFIES; 5324 5325 static void wq_watchdog_reset_touched(void) 5326 { 5327 int cpu; 5328 5329 wq_watchdog_touched = jiffies; 5330 for_each_possible_cpu(cpu) 5331 per_cpu(wq_watchdog_touched_cpu, cpu) = jiffies; 5332 } 5333 5334 static void wq_watchdog_timer_fn(unsigned long data) 5335 { 5336 unsigned long thresh = READ_ONCE(wq_watchdog_thresh) * HZ; 5337 bool lockup_detected = false; 5338 struct worker_pool *pool; 5339 int pi; 5340 5341 if (!thresh) 5342 return; 5343 5344 rcu_read_lock(); 5345 5346 for_each_pool(pool, pi) { 5347 unsigned long pool_ts, touched, ts; 5348 5349 if (list_empty(&pool->worklist)) 5350 continue; 5351 5352 /* get the latest of pool and touched timestamps */ 5353 pool_ts = READ_ONCE(pool->watchdog_ts); 5354 touched = READ_ONCE(wq_watchdog_touched); 5355 5356 if (time_after(pool_ts, touched)) 5357 ts = pool_ts; 5358 else 5359 ts = touched; 5360 5361 if (pool->cpu >= 0) { 5362 unsigned long cpu_touched = 5363 READ_ONCE(per_cpu(wq_watchdog_touched_cpu, 5364 pool->cpu)); 5365 if (time_after(cpu_touched, ts)) 5366 ts = cpu_touched; 5367 } 5368 5369 /* did we stall? */ 5370 if (time_after(jiffies, ts + thresh)) { 5371 lockup_detected = true; 5372 pr_emerg("BUG: workqueue lockup - pool"); 5373 pr_cont_pool_info(pool); 5374 pr_cont(" stuck for %us!\n", 5375 jiffies_to_msecs(jiffies - pool_ts) / 1000); 5376 } 5377 } 5378 5379 rcu_read_unlock(); 5380 5381 if (lockup_detected) 5382 show_workqueue_state(); 5383 5384 wq_watchdog_reset_touched(); 5385 mod_timer(&wq_watchdog_timer, jiffies + thresh); 5386 } 5387 5388 void wq_watchdog_touch(int cpu) 5389 { 5390 if (cpu >= 0) 5391 per_cpu(wq_watchdog_touched_cpu, cpu) = jiffies; 5392 else 5393 wq_watchdog_touched = jiffies; 5394 } 5395 5396 static void wq_watchdog_set_thresh(unsigned long thresh) 5397 { 5398 wq_watchdog_thresh = 0; 5399 del_timer_sync(&wq_watchdog_timer); 5400 5401 if (thresh) { 5402 wq_watchdog_thresh = thresh; 5403 wq_watchdog_reset_touched(); 5404 mod_timer(&wq_watchdog_timer, jiffies + thresh * HZ); 5405 } 5406 } 5407 5408 static int wq_watchdog_param_set_thresh(const char *val, 5409 const struct kernel_param *kp) 5410 { 5411 unsigned long thresh; 5412 int ret; 5413 5414 ret = kstrtoul(val, 0, &thresh); 5415 if (ret) 5416 return ret; 5417 5418 if (system_wq) 5419 wq_watchdog_set_thresh(thresh); 5420 else 5421 wq_watchdog_thresh = thresh; 5422 5423 return 0; 5424 } 5425 5426 static const struct kernel_param_ops wq_watchdog_thresh_ops = { 5427 .set = wq_watchdog_param_set_thresh, 5428 .get = param_get_ulong, 5429 }; 5430 5431 module_param_cb(watchdog_thresh, &wq_watchdog_thresh_ops, &wq_watchdog_thresh, 5432 0644); 5433 5434 static void wq_watchdog_init(void) 5435 { 5436 wq_watchdog_set_thresh(wq_watchdog_thresh); 5437 } 5438 5439 #else /* CONFIG_WQ_WATCHDOG */ 5440 5441 static inline void wq_watchdog_init(void) { } 5442 5443 #endif /* CONFIG_WQ_WATCHDOG */ 5444 5445 static void __init wq_numa_init(void) 5446 { 5447 cpumask_var_t *tbl; 5448 int node, cpu; 5449 5450 if (num_possible_nodes() <= 1) 5451 return; 5452 5453 if (wq_disable_numa) { 5454 pr_info("workqueue: NUMA affinity support disabled\n"); 5455 return; 5456 } 5457 5458 wq_update_unbound_numa_attrs_buf = alloc_workqueue_attrs(GFP_KERNEL); 5459 BUG_ON(!wq_update_unbound_numa_attrs_buf); 5460 5461 /* 5462 * We want masks of possible CPUs of each node which isn't readily 5463 * available. Build one from cpu_to_node() which should have been 5464 * fully initialized by now. 5465 */ 5466 tbl = kzalloc(nr_node_ids * sizeof(tbl[0]), GFP_KERNEL); 5467 BUG_ON(!tbl); 5468 5469 for_each_node(node) 5470 BUG_ON(!zalloc_cpumask_var_node(&tbl[node], GFP_KERNEL, 5471 node_online(node) ? node : NUMA_NO_NODE)); 5472 5473 for_each_possible_cpu(cpu) { 5474 node = cpu_to_node(cpu); 5475 if (WARN_ON(node == NUMA_NO_NODE)) { 5476 pr_warn("workqueue: NUMA node mapping not available for cpu%d, disabling NUMA support\n", cpu); 5477 /* happens iff arch is bonkers, let's just proceed */ 5478 return; 5479 } 5480 cpumask_set_cpu(cpu, tbl[node]); 5481 } 5482 5483 wq_numa_possible_cpumask = tbl; 5484 wq_numa_enabled = true; 5485 } 5486 5487 /** 5488 * workqueue_init_early - early init for workqueue subsystem 5489 * 5490 * This is the first half of two-staged workqueue subsystem initialization 5491 * and invoked as soon as the bare basics - memory allocation, cpumasks and 5492 * idr are up. It sets up all the data structures and system workqueues 5493 * and allows early boot code to create workqueues and queue/cancel work 5494 * items. Actual work item execution starts only after kthreads can be 5495 * created and scheduled right before early initcalls. 5496 */ 5497 int __init workqueue_init_early(void) 5498 { 5499 int std_nice[NR_STD_WORKER_POOLS] = { 0, HIGHPRI_NICE_LEVEL }; 5500 int i, cpu; 5501 5502 WARN_ON(__alignof__(struct pool_workqueue) < __alignof__(long long)); 5503 5504 BUG_ON(!alloc_cpumask_var(&wq_unbound_cpumask, GFP_KERNEL)); 5505 cpumask_copy(wq_unbound_cpumask, cpu_possible_mask); 5506 5507 pwq_cache = KMEM_CACHE(pool_workqueue, SLAB_PANIC); 5508 5509 /* initialize CPU pools */ 5510 for_each_possible_cpu(cpu) { 5511 struct worker_pool *pool; 5512 5513 i = 0; 5514 for_each_cpu_worker_pool(pool, cpu) { 5515 BUG_ON(init_worker_pool(pool)); 5516 pool->cpu = cpu; 5517 cpumask_copy(pool->attrs->cpumask, cpumask_of(cpu)); 5518 pool->attrs->nice = std_nice[i++]; 5519 pool->node = cpu_to_node(cpu); 5520 5521 /* alloc pool ID */ 5522 mutex_lock(&wq_pool_mutex); 5523 BUG_ON(worker_pool_assign_id(pool)); 5524 mutex_unlock(&wq_pool_mutex); 5525 } 5526 } 5527 5528 /* create default unbound and ordered wq attrs */ 5529 for (i = 0; i < NR_STD_WORKER_POOLS; i++) { 5530 struct workqueue_attrs *attrs; 5531 5532 BUG_ON(!(attrs = alloc_workqueue_attrs(GFP_KERNEL))); 5533 attrs->nice = std_nice[i]; 5534 unbound_std_wq_attrs[i] = attrs; 5535 5536 /* 5537 * An ordered wq should have only one pwq as ordering is 5538 * guaranteed by max_active which is enforced by pwqs. 5539 * Turn off NUMA so that dfl_pwq is used for all nodes. 5540 */ 5541 BUG_ON(!(attrs = alloc_workqueue_attrs(GFP_KERNEL))); 5542 attrs->nice = std_nice[i]; 5543 attrs->no_numa = true; 5544 ordered_wq_attrs[i] = attrs; 5545 } 5546 5547 system_wq = alloc_workqueue("events", 0, 0); 5548 system_highpri_wq = alloc_workqueue("events_highpri", WQ_HIGHPRI, 0); 5549 system_long_wq = alloc_workqueue("events_long", 0, 0); 5550 system_unbound_wq = alloc_workqueue("events_unbound", WQ_UNBOUND, 5551 WQ_UNBOUND_MAX_ACTIVE); 5552 system_freezable_wq = alloc_workqueue("events_freezable", 5553 WQ_FREEZABLE, 0); 5554 system_power_efficient_wq = alloc_workqueue("events_power_efficient", 5555 WQ_POWER_EFFICIENT, 0); 5556 system_freezable_power_efficient_wq = alloc_workqueue("events_freezable_power_efficient", 5557 WQ_FREEZABLE | WQ_POWER_EFFICIENT, 5558 0); 5559 BUG_ON(!system_wq || !system_highpri_wq || !system_long_wq || 5560 !system_unbound_wq || !system_freezable_wq || 5561 !system_power_efficient_wq || 5562 !system_freezable_power_efficient_wq); 5563 5564 return 0; 5565 } 5566 5567 /** 5568 * workqueue_init - bring workqueue subsystem fully online 5569 * 5570 * This is the latter half of two-staged workqueue subsystem initialization 5571 * and invoked as soon as kthreads can be created and scheduled. 5572 * Workqueues have been created and work items queued on them, but there 5573 * are no kworkers executing the work items yet. Populate the worker pools 5574 * with the initial workers and enable future kworker creations. 5575 */ 5576 int __init workqueue_init(void) 5577 { 5578 struct workqueue_struct *wq; 5579 struct worker_pool *pool; 5580 int cpu, bkt; 5581 5582 /* 5583 * It'd be simpler to initialize NUMA in workqueue_init_early() but 5584 * CPU to node mapping may not be available that early on some 5585 * archs such as power and arm64. As per-cpu pools created 5586 * previously could be missing node hint and unbound pools NUMA 5587 * affinity, fix them up. 5588 */ 5589 wq_numa_init(); 5590 5591 mutex_lock(&wq_pool_mutex); 5592 5593 for_each_possible_cpu(cpu) { 5594 for_each_cpu_worker_pool(pool, cpu) { 5595 pool->node = cpu_to_node(cpu); 5596 } 5597 } 5598 5599 list_for_each_entry(wq, &workqueues, list) 5600 wq_update_unbound_numa(wq, smp_processor_id(), true); 5601 5602 mutex_unlock(&wq_pool_mutex); 5603 5604 /* create the initial workers */ 5605 for_each_online_cpu(cpu) { 5606 for_each_cpu_worker_pool(pool, cpu) { 5607 pool->flags &= ~POOL_DISASSOCIATED; 5608 BUG_ON(!create_worker(pool)); 5609 } 5610 } 5611 5612 hash_for_each(unbound_pool_hash, bkt, pool, hash_node) 5613 BUG_ON(!create_worker(pool)); 5614 5615 wq_online = true; 5616 wq_watchdog_init(); 5617 5618 return 0; 5619 } 5620