1 /* Kernel thread helper functions. 2 * Copyright (C) 2004 IBM Corporation, Rusty Russell. 3 * 4 * Creation is done via kthreadd, so that we get a clean environment 5 * even if we're invoked from userspace (think modprobe, hotplug cpu, 6 * etc.). 7 */ 8 #include <uapi/linux/sched/types.h> 9 #include <linux/sched.h> 10 #include <linux/sched/task.h> 11 #include <linux/kthread.h> 12 #include <linux/completion.h> 13 #include <linux/err.h> 14 #include <linux/cpuset.h> 15 #include <linux/unistd.h> 16 #include <linux/file.h> 17 #include <linux/export.h> 18 #include <linux/mutex.h> 19 #include <linux/slab.h> 20 #include <linux/freezer.h> 21 #include <linux/ptrace.h> 22 #include <linux/uaccess.h> 23 #include <trace/events/sched.h> 24 25 static DEFINE_SPINLOCK(kthread_create_lock); 26 static LIST_HEAD(kthread_create_list); 27 struct task_struct *kthreadd_task; 28 29 struct kthread_create_info 30 { 31 /* Information passed to kthread() from kthreadd. */ 32 int (*threadfn)(void *data); 33 void *data; 34 int node; 35 36 /* Result passed back to kthread_create() from kthreadd. */ 37 struct task_struct *result; 38 struct completion *done; 39 40 struct list_head list; 41 }; 42 43 struct kthread { 44 unsigned long flags; 45 unsigned int cpu; 46 void *data; 47 struct completion parked; 48 struct completion exited; 49 #ifdef CONFIG_BLK_CGROUP 50 struct cgroup_subsys_state *blkcg_css; 51 #endif 52 }; 53 54 enum KTHREAD_BITS { 55 KTHREAD_IS_PER_CPU = 0, 56 KTHREAD_SHOULD_STOP, 57 KTHREAD_SHOULD_PARK, 58 }; 59 60 static inline void set_kthread_struct(void *kthread) 61 { 62 /* 63 * We abuse ->set_child_tid to avoid the new member and because it 64 * can't be wrongly copied by copy_process(). We also rely on fact 65 * that the caller can't exec, so PF_KTHREAD can't be cleared. 66 */ 67 current->set_child_tid = (__force void __user *)kthread; 68 } 69 70 static inline struct kthread *to_kthread(struct task_struct *k) 71 { 72 WARN_ON(!(k->flags & PF_KTHREAD)); 73 return (__force void *)k->set_child_tid; 74 } 75 76 void free_kthread_struct(struct task_struct *k) 77 { 78 struct kthread *kthread; 79 80 /* 81 * Can be NULL if this kthread was created by kernel_thread() 82 * or if kmalloc() in kthread() failed. 83 */ 84 kthread = to_kthread(k); 85 #ifdef CONFIG_BLK_CGROUP 86 WARN_ON_ONCE(kthread && kthread->blkcg_css); 87 #endif 88 kfree(kthread); 89 } 90 91 /** 92 * kthread_should_stop - should this kthread return now? 93 * 94 * When someone calls kthread_stop() on your kthread, it will be woken 95 * and this will return true. You should then return, and your return 96 * value will be passed through to kthread_stop(). 97 */ 98 bool kthread_should_stop(void) 99 { 100 return test_bit(KTHREAD_SHOULD_STOP, &to_kthread(current)->flags); 101 } 102 EXPORT_SYMBOL(kthread_should_stop); 103 104 /** 105 * kthread_should_park - should this kthread park now? 106 * 107 * When someone calls kthread_park() on your kthread, it will be woken 108 * and this will return true. You should then do the necessary 109 * cleanup and call kthread_parkme() 110 * 111 * Similar to kthread_should_stop(), but this keeps the thread alive 112 * and in a park position. kthread_unpark() "restarts" the thread and 113 * calls the thread function again. 114 */ 115 bool kthread_should_park(void) 116 { 117 return test_bit(KTHREAD_SHOULD_PARK, &to_kthread(current)->flags); 118 } 119 EXPORT_SYMBOL_GPL(kthread_should_park); 120 121 /** 122 * kthread_freezable_should_stop - should this freezable kthread return now? 123 * @was_frozen: optional out parameter, indicates whether %current was frozen 124 * 125 * kthread_should_stop() for freezable kthreads, which will enter 126 * refrigerator if necessary. This function is safe from kthread_stop() / 127 * freezer deadlock and freezable kthreads should use this function instead 128 * of calling try_to_freeze() directly. 129 */ 130 bool kthread_freezable_should_stop(bool *was_frozen) 131 { 132 bool frozen = false; 133 134 might_sleep(); 135 136 if (unlikely(freezing(current))) 137 frozen = __refrigerator(true); 138 139 if (was_frozen) 140 *was_frozen = frozen; 141 142 return kthread_should_stop(); 143 } 144 EXPORT_SYMBOL_GPL(kthread_freezable_should_stop); 145 146 /** 147 * kthread_data - return data value specified on kthread creation 148 * @task: kthread task in question 149 * 150 * Return the data value specified when kthread @task was created. 151 * The caller is responsible for ensuring the validity of @task when 152 * calling this function. 153 */ 154 void *kthread_data(struct task_struct *task) 155 { 156 return to_kthread(task)->data; 157 } 158 159 /** 160 * kthread_probe_data - speculative version of kthread_data() 161 * @task: possible kthread task in question 162 * 163 * @task could be a kthread task. Return the data value specified when it 164 * was created if accessible. If @task isn't a kthread task or its data is 165 * inaccessible for any reason, %NULL is returned. This function requires 166 * that @task itself is safe to dereference. 167 */ 168 void *kthread_probe_data(struct task_struct *task) 169 { 170 struct kthread *kthread = to_kthread(task); 171 void *data = NULL; 172 173 probe_kernel_read(&data, &kthread->data, sizeof(data)); 174 return data; 175 } 176 177 static void __kthread_parkme(struct kthread *self) 178 { 179 for (;;) { 180 set_current_state(TASK_PARKED); 181 if (!test_bit(KTHREAD_SHOULD_PARK, &self->flags)) 182 break; 183 schedule(); 184 } 185 __set_current_state(TASK_RUNNING); 186 } 187 188 void kthread_parkme(void) 189 { 190 __kthread_parkme(to_kthread(current)); 191 } 192 EXPORT_SYMBOL_GPL(kthread_parkme); 193 194 void kthread_park_complete(struct task_struct *k) 195 { 196 complete_all(&to_kthread(k)->parked); 197 } 198 199 static int kthread(void *_create) 200 { 201 /* Copy data: it's on kthread's stack */ 202 struct kthread_create_info *create = _create; 203 int (*threadfn)(void *data) = create->threadfn; 204 void *data = create->data; 205 struct completion *done; 206 struct kthread *self; 207 int ret; 208 209 self = kzalloc(sizeof(*self), GFP_KERNEL); 210 set_kthread_struct(self); 211 212 /* If user was SIGKILLed, I release the structure. */ 213 done = xchg(&create->done, NULL); 214 if (!done) { 215 kfree(create); 216 do_exit(-EINTR); 217 } 218 219 if (!self) { 220 create->result = ERR_PTR(-ENOMEM); 221 complete(done); 222 do_exit(-ENOMEM); 223 } 224 225 self->data = data; 226 init_completion(&self->exited); 227 init_completion(&self->parked); 228 current->vfork_done = &self->exited; 229 230 /* OK, tell user we're spawned, wait for stop or wakeup */ 231 __set_current_state(TASK_UNINTERRUPTIBLE); 232 create->result = current; 233 complete(done); 234 schedule(); 235 236 ret = -EINTR; 237 if (!test_bit(KTHREAD_SHOULD_STOP, &self->flags)) { 238 cgroup_kthread_ready(); 239 __kthread_parkme(self); 240 ret = threadfn(data); 241 } 242 do_exit(ret); 243 } 244 245 /* called from do_fork() to get node information for about to be created task */ 246 int tsk_fork_get_node(struct task_struct *tsk) 247 { 248 #ifdef CONFIG_NUMA 249 if (tsk == kthreadd_task) 250 return tsk->pref_node_fork; 251 #endif 252 return NUMA_NO_NODE; 253 } 254 255 static void create_kthread(struct kthread_create_info *create) 256 { 257 int pid; 258 259 #ifdef CONFIG_NUMA 260 current->pref_node_fork = create->node; 261 #endif 262 /* We want our own signal handler (we take no signals by default). */ 263 pid = kernel_thread(kthread, create, CLONE_FS | CLONE_FILES | SIGCHLD); 264 if (pid < 0) { 265 /* If user was SIGKILLed, I release the structure. */ 266 struct completion *done = xchg(&create->done, NULL); 267 268 if (!done) { 269 kfree(create); 270 return; 271 } 272 create->result = ERR_PTR(pid); 273 complete(done); 274 } 275 } 276 277 static __printf(4, 0) 278 struct task_struct *__kthread_create_on_node(int (*threadfn)(void *data), 279 void *data, int node, 280 const char namefmt[], 281 va_list args) 282 { 283 DECLARE_COMPLETION_ONSTACK(done); 284 struct task_struct *task; 285 struct kthread_create_info *create = kmalloc(sizeof(*create), 286 GFP_KERNEL); 287 288 if (!create) 289 return ERR_PTR(-ENOMEM); 290 create->threadfn = threadfn; 291 create->data = data; 292 create->node = node; 293 create->done = &done; 294 295 spin_lock(&kthread_create_lock); 296 list_add_tail(&create->list, &kthread_create_list); 297 spin_unlock(&kthread_create_lock); 298 299 wake_up_process(kthreadd_task); 300 /* 301 * Wait for completion in killable state, for I might be chosen by 302 * the OOM killer while kthreadd is trying to allocate memory for 303 * new kernel thread. 304 */ 305 if (unlikely(wait_for_completion_killable(&done))) { 306 /* 307 * If I was SIGKILLed before kthreadd (or new kernel thread) 308 * calls complete(), leave the cleanup of this structure to 309 * that thread. 310 */ 311 if (xchg(&create->done, NULL)) 312 return ERR_PTR(-EINTR); 313 /* 314 * kthreadd (or new kernel thread) will call complete() 315 * shortly. 316 */ 317 wait_for_completion(&done); 318 } 319 task = create->result; 320 if (!IS_ERR(task)) { 321 static const struct sched_param param = { .sched_priority = 0 }; 322 323 vsnprintf(task->comm, sizeof(task->comm), namefmt, args); 324 /* 325 * root may have changed our (kthreadd's) priority or CPU mask. 326 * The kernel thread should not inherit these properties. 327 */ 328 sched_setscheduler_nocheck(task, SCHED_NORMAL, ¶m); 329 set_cpus_allowed_ptr(task, cpu_all_mask); 330 } 331 kfree(create); 332 return task; 333 } 334 335 /** 336 * kthread_create_on_node - create a kthread. 337 * @threadfn: the function to run until signal_pending(current). 338 * @data: data ptr for @threadfn. 339 * @node: task and thread structures for the thread are allocated on this node 340 * @namefmt: printf-style name for the thread. 341 * 342 * Description: This helper function creates and names a kernel 343 * thread. The thread will be stopped: use wake_up_process() to start 344 * it. See also kthread_run(). The new thread has SCHED_NORMAL policy and 345 * is affine to all CPUs. 346 * 347 * If thread is going to be bound on a particular cpu, give its node 348 * in @node, to get NUMA affinity for kthread stack, or else give NUMA_NO_NODE. 349 * When woken, the thread will run @threadfn() with @data as its 350 * argument. @threadfn() can either call do_exit() directly if it is a 351 * standalone thread for which no one will call kthread_stop(), or 352 * return when 'kthread_should_stop()' is true (which means 353 * kthread_stop() has been called). The return value should be zero 354 * or a negative error number; it will be passed to kthread_stop(). 355 * 356 * Returns a task_struct or ERR_PTR(-ENOMEM) or ERR_PTR(-EINTR). 357 */ 358 struct task_struct *kthread_create_on_node(int (*threadfn)(void *data), 359 void *data, int node, 360 const char namefmt[], 361 ...) 362 { 363 struct task_struct *task; 364 va_list args; 365 366 va_start(args, namefmt); 367 task = __kthread_create_on_node(threadfn, data, node, namefmt, args); 368 va_end(args); 369 370 return task; 371 } 372 EXPORT_SYMBOL(kthread_create_on_node); 373 374 static void __kthread_bind_mask(struct task_struct *p, const struct cpumask *mask, long state) 375 { 376 unsigned long flags; 377 378 if (!wait_task_inactive(p, state)) { 379 WARN_ON(1); 380 return; 381 } 382 383 /* It's safe because the task is inactive. */ 384 raw_spin_lock_irqsave(&p->pi_lock, flags); 385 do_set_cpus_allowed(p, mask); 386 p->flags |= PF_NO_SETAFFINITY; 387 raw_spin_unlock_irqrestore(&p->pi_lock, flags); 388 } 389 390 static void __kthread_bind(struct task_struct *p, unsigned int cpu, long state) 391 { 392 __kthread_bind_mask(p, cpumask_of(cpu), state); 393 } 394 395 void kthread_bind_mask(struct task_struct *p, const struct cpumask *mask) 396 { 397 __kthread_bind_mask(p, mask, TASK_UNINTERRUPTIBLE); 398 } 399 400 /** 401 * kthread_bind - bind a just-created kthread to a cpu. 402 * @p: thread created by kthread_create(). 403 * @cpu: cpu (might not be online, must be possible) for @k to run on. 404 * 405 * Description: This function is equivalent to set_cpus_allowed(), 406 * except that @cpu doesn't need to be online, and the thread must be 407 * stopped (i.e., just returned from kthread_create()). 408 */ 409 void kthread_bind(struct task_struct *p, unsigned int cpu) 410 { 411 __kthread_bind(p, cpu, TASK_UNINTERRUPTIBLE); 412 } 413 EXPORT_SYMBOL(kthread_bind); 414 415 /** 416 * kthread_create_on_cpu - Create a cpu bound kthread 417 * @threadfn: the function to run until signal_pending(current). 418 * @data: data ptr for @threadfn. 419 * @cpu: The cpu on which the thread should be bound, 420 * @namefmt: printf-style name for the thread. Format is restricted 421 * to "name.*%u". Code fills in cpu number. 422 * 423 * Description: This helper function creates and names a kernel thread 424 * The thread will be woken and put into park mode. 425 */ 426 struct task_struct *kthread_create_on_cpu(int (*threadfn)(void *data), 427 void *data, unsigned int cpu, 428 const char *namefmt) 429 { 430 struct task_struct *p; 431 432 p = kthread_create_on_node(threadfn, data, cpu_to_node(cpu), namefmt, 433 cpu); 434 if (IS_ERR(p)) 435 return p; 436 kthread_bind(p, cpu); 437 /* CPU hotplug need to bind once again when unparking the thread. */ 438 set_bit(KTHREAD_IS_PER_CPU, &to_kthread(p)->flags); 439 to_kthread(p)->cpu = cpu; 440 return p; 441 } 442 443 /** 444 * kthread_unpark - unpark a thread created by kthread_create(). 445 * @k: thread created by kthread_create(). 446 * 447 * Sets kthread_should_park() for @k to return false, wakes it, and 448 * waits for it to return. If the thread is marked percpu then its 449 * bound to the cpu again. 450 */ 451 void kthread_unpark(struct task_struct *k) 452 { 453 struct kthread *kthread = to_kthread(k); 454 455 /* 456 * Newly created kthread was parked when the CPU was offline. 457 * The binding was lost and we need to set it again. 458 */ 459 if (test_bit(KTHREAD_IS_PER_CPU, &kthread->flags)) 460 __kthread_bind(k, kthread->cpu, TASK_PARKED); 461 462 reinit_completion(&kthread->parked); 463 clear_bit(KTHREAD_SHOULD_PARK, &kthread->flags); 464 wake_up_state(k, TASK_PARKED); 465 } 466 EXPORT_SYMBOL_GPL(kthread_unpark); 467 468 /** 469 * kthread_park - park a thread created by kthread_create(). 470 * @k: thread created by kthread_create(). 471 * 472 * Sets kthread_should_park() for @k to return true, wakes it, and 473 * waits for it to return. This can also be called after kthread_create() 474 * instead of calling wake_up_process(): the thread will park without 475 * calling threadfn(). 476 * 477 * Returns 0 if the thread is parked, -ENOSYS if the thread exited. 478 * If called by the kthread itself just the park bit is set. 479 */ 480 int kthread_park(struct task_struct *k) 481 { 482 struct kthread *kthread = to_kthread(k); 483 484 if (WARN_ON(k->flags & PF_EXITING)) 485 return -ENOSYS; 486 487 set_bit(KTHREAD_SHOULD_PARK, &kthread->flags); 488 if (k != current) { 489 wake_up_process(k); 490 wait_for_completion(&kthread->parked); 491 } 492 493 return 0; 494 } 495 EXPORT_SYMBOL_GPL(kthread_park); 496 497 /** 498 * kthread_stop - stop a thread created by kthread_create(). 499 * @k: thread created by kthread_create(). 500 * 501 * Sets kthread_should_stop() for @k to return true, wakes it, and 502 * waits for it to exit. This can also be called after kthread_create() 503 * instead of calling wake_up_process(): the thread will exit without 504 * calling threadfn(). 505 * 506 * If threadfn() may call do_exit() itself, the caller must ensure 507 * task_struct can't go away. 508 * 509 * Returns the result of threadfn(), or %-EINTR if wake_up_process() 510 * was never called. 511 */ 512 int kthread_stop(struct task_struct *k) 513 { 514 struct kthread *kthread; 515 int ret; 516 517 trace_sched_kthread_stop(k); 518 519 get_task_struct(k); 520 kthread = to_kthread(k); 521 set_bit(KTHREAD_SHOULD_STOP, &kthread->flags); 522 kthread_unpark(k); 523 wake_up_process(k); 524 wait_for_completion(&kthread->exited); 525 ret = k->exit_code; 526 put_task_struct(k); 527 528 trace_sched_kthread_stop_ret(ret); 529 return ret; 530 } 531 EXPORT_SYMBOL(kthread_stop); 532 533 int kthreadd(void *unused) 534 { 535 struct task_struct *tsk = current; 536 537 /* Setup a clean context for our children to inherit. */ 538 set_task_comm(tsk, "kthreadd"); 539 ignore_signals(tsk); 540 set_cpus_allowed_ptr(tsk, cpu_all_mask); 541 set_mems_allowed(node_states[N_MEMORY]); 542 543 current->flags |= PF_NOFREEZE; 544 cgroup_init_kthreadd(); 545 546 for (;;) { 547 set_current_state(TASK_INTERRUPTIBLE); 548 if (list_empty(&kthread_create_list)) 549 schedule(); 550 __set_current_state(TASK_RUNNING); 551 552 spin_lock(&kthread_create_lock); 553 while (!list_empty(&kthread_create_list)) { 554 struct kthread_create_info *create; 555 556 create = list_entry(kthread_create_list.next, 557 struct kthread_create_info, list); 558 list_del_init(&create->list); 559 spin_unlock(&kthread_create_lock); 560 561 create_kthread(create); 562 563 spin_lock(&kthread_create_lock); 564 } 565 spin_unlock(&kthread_create_lock); 566 } 567 568 return 0; 569 } 570 571 void __kthread_init_worker(struct kthread_worker *worker, 572 const char *name, 573 struct lock_class_key *key) 574 { 575 memset(worker, 0, sizeof(struct kthread_worker)); 576 spin_lock_init(&worker->lock); 577 lockdep_set_class_and_name(&worker->lock, key, name); 578 INIT_LIST_HEAD(&worker->work_list); 579 INIT_LIST_HEAD(&worker->delayed_work_list); 580 } 581 EXPORT_SYMBOL_GPL(__kthread_init_worker); 582 583 /** 584 * kthread_worker_fn - kthread function to process kthread_worker 585 * @worker_ptr: pointer to initialized kthread_worker 586 * 587 * This function implements the main cycle of kthread worker. It processes 588 * work_list until it is stopped with kthread_stop(). It sleeps when the queue 589 * is empty. 590 * 591 * The works are not allowed to keep any locks, disable preemption or interrupts 592 * when they finish. There is defined a safe point for freezing when one work 593 * finishes and before a new one is started. 594 * 595 * Also the works must not be handled by more than one worker at the same time, 596 * see also kthread_queue_work(). 597 */ 598 int kthread_worker_fn(void *worker_ptr) 599 { 600 struct kthread_worker *worker = worker_ptr; 601 struct kthread_work *work; 602 603 /* 604 * FIXME: Update the check and remove the assignment when all kthread 605 * worker users are created using kthread_create_worker*() functions. 606 */ 607 WARN_ON(worker->task && worker->task != current); 608 worker->task = current; 609 610 if (worker->flags & KTW_FREEZABLE) 611 set_freezable(); 612 613 repeat: 614 set_current_state(TASK_INTERRUPTIBLE); /* mb paired w/ kthread_stop */ 615 616 if (kthread_should_stop()) { 617 __set_current_state(TASK_RUNNING); 618 spin_lock_irq(&worker->lock); 619 worker->task = NULL; 620 spin_unlock_irq(&worker->lock); 621 return 0; 622 } 623 624 work = NULL; 625 spin_lock_irq(&worker->lock); 626 if (!list_empty(&worker->work_list)) { 627 work = list_first_entry(&worker->work_list, 628 struct kthread_work, node); 629 list_del_init(&work->node); 630 } 631 worker->current_work = work; 632 spin_unlock_irq(&worker->lock); 633 634 if (work) { 635 __set_current_state(TASK_RUNNING); 636 work->func(work); 637 } else if (!freezing(current)) 638 schedule(); 639 640 try_to_freeze(); 641 cond_resched(); 642 goto repeat; 643 } 644 EXPORT_SYMBOL_GPL(kthread_worker_fn); 645 646 static __printf(3, 0) struct kthread_worker * 647 __kthread_create_worker(int cpu, unsigned int flags, 648 const char namefmt[], va_list args) 649 { 650 struct kthread_worker *worker; 651 struct task_struct *task; 652 int node = -1; 653 654 worker = kzalloc(sizeof(*worker), GFP_KERNEL); 655 if (!worker) 656 return ERR_PTR(-ENOMEM); 657 658 kthread_init_worker(worker); 659 660 if (cpu >= 0) 661 node = cpu_to_node(cpu); 662 663 task = __kthread_create_on_node(kthread_worker_fn, worker, 664 node, namefmt, args); 665 if (IS_ERR(task)) 666 goto fail_task; 667 668 if (cpu >= 0) 669 kthread_bind(task, cpu); 670 671 worker->flags = flags; 672 worker->task = task; 673 wake_up_process(task); 674 return worker; 675 676 fail_task: 677 kfree(worker); 678 return ERR_CAST(task); 679 } 680 681 /** 682 * kthread_create_worker - create a kthread worker 683 * @flags: flags modifying the default behavior of the worker 684 * @namefmt: printf-style name for the kthread worker (task). 685 * 686 * Returns a pointer to the allocated worker on success, ERR_PTR(-ENOMEM) 687 * when the needed structures could not get allocated, and ERR_PTR(-EINTR) 688 * when the worker was SIGKILLed. 689 */ 690 struct kthread_worker * 691 kthread_create_worker(unsigned int flags, const char namefmt[], ...) 692 { 693 struct kthread_worker *worker; 694 va_list args; 695 696 va_start(args, namefmt); 697 worker = __kthread_create_worker(-1, flags, namefmt, args); 698 va_end(args); 699 700 return worker; 701 } 702 EXPORT_SYMBOL(kthread_create_worker); 703 704 /** 705 * kthread_create_worker_on_cpu - create a kthread worker and bind it 706 * it to a given CPU and the associated NUMA node. 707 * @cpu: CPU number 708 * @flags: flags modifying the default behavior of the worker 709 * @namefmt: printf-style name for the kthread worker (task). 710 * 711 * Use a valid CPU number if you want to bind the kthread worker 712 * to the given CPU and the associated NUMA node. 713 * 714 * A good practice is to add the cpu number also into the worker name. 715 * For example, use kthread_create_worker_on_cpu(cpu, "helper/%d", cpu). 716 * 717 * Returns a pointer to the allocated worker on success, ERR_PTR(-ENOMEM) 718 * when the needed structures could not get allocated, and ERR_PTR(-EINTR) 719 * when the worker was SIGKILLed. 720 */ 721 struct kthread_worker * 722 kthread_create_worker_on_cpu(int cpu, unsigned int flags, 723 const char namefmt[], ...) 724 { 725 struct kthread_worker *worker; 726 va_list args; 727 728 va_start(args, namefmt); 729 worker = __kthread_create_worker(cpu, flags, namefmt, args); 730 va_end(args); 731 732 return worker; 733 } 734 EXPORT_SYMBOL(kthread_create_worker_on_cpu); 735 736 /* 737 * Returns true when the work could not be queued at the moment. 738 * It happens when it is already pending in a worker list 739 * or when it is being cancelled. 740 */ 741 static inline bool queuing_blocked(struct kthread_worker *worker, 742 struct kthread_work *work) 743 { 744 lockdep_assert_held(&worker->lock); 745 746 return !list_empty(&work->node) || work->canceling; 747 } 748 749 static void kthread_insert_work_sanity_check(struct kthread_worker *worker, 750 struct kthread_work *work) 751 { 752 lockdep_assert_held(&worker->lock); 753 WARN_ON_ONCE(!list_empty(&work->node)); 754 /* Do not use a work with >1 worker, see kthread_queue_work() */ 755 WARN_ON_ONCE(work->worker && work->worker != worker); 756 } 757 758 /* insert @work before @pos in @worker */ 759 static void kthread_insert_work(struct kthread_worker *worker, 760 struct kthread_work *work, 761 struct list_head *pos) 762 { 763 kthread_insert_work_sanity_check(worker, work); 764 765 list_add_tail(&work->node, pos); 766 work->worker = worker; 767 if (!worker->current_work && likely(worker->task)) 768 wake_up_process(worker->task); 769 } 770 771 /** 772 * kthread_queue_work - queue a kthread_work 773 * @worker: target kthread_worker 774 * @work: kthread_work to queue 775 * 776 * Queue @work to work processor @task for async execution. @task 777 * must have been created with kthread_worker_create(). Returns %true 778 * if @work was successfully queued, %false if it was already pending. 779 * 780 * Reinitialize the work if it needs to be used by another worker. 781 * For example, when the worker was stopped and started again. 782 */ 783 bool kthread_queue_work(struct kthread_worker *worker, 784 struct kthread_work *work) 785 { 786 bool ret = false; 787 unsigned long flags; 788 789 spin_lock_irqsave(&worker->lock, flags); 790 if (!queuing_blocked(worker, work)) { 791 kthread_insert_work(worker, work, &worker->work_list); 792 ret = true; 793 } 794 spin_unlock_irqrestore(&worker->lock, flags); 795 return ret; 796 } 797 EXPORT_SYMBOL_GPL(kthread_queue_work); 798 799 /** 800 * kthread_delayed_work_timer_fn - callback that queues the associated kthread 801 * delayed work when the timer expires. 802 * @t: pointer to the expired timer 803 * 804 * The format of the function is defined by struct timer_list. 805 * It should have been called from irqsafe timer with irq already off. 806 */ 807 void kthread_delayed_work_timer_fn(struct timer_list *t) 808 { 809 struct kthread_delayed_work *dwork = from_timer(dwork, t, timer); 810 struct kthread_work *work = &dwork->work; 811 struct kthread_worker *worker = work->worker; 812 813 /* 814 * This might happen when a pending work is reinitialized. 815 * It means that it is used a wrong way. 816 */ 817 if (WARN_ON_ONCE(!worker)) 818 return; 819 820 spin_lock(&worker->lock); 821 /* Work must not be used with >1 worker, see kthread_queue_work(). */ 822 WARN_ON_ONCE(work->worker != worker); 823 824 /* Move the work from worker->delayed_work_list. */ 825 WARN_ON_ONCE(list_empty(&work->node)); 826 list_del_init(&work->node); 827 kthread_insert_work(worker, work, &worker->work_list); 828 829 spin_unlock(&worker->lock); 830 } 831 EXPORT_SYMBOL(kthread_delayed_work_timer_fn); 832 833 void __kthread_queue_delayed_work(struct kthread_worker *worker, 834 struct kthread_delayed_work *dwork, 835 unsigned long delay) 836 { 837 struct timer_list *timer = &dwork->timer; 838 struct kthread_work *work = &dwork->work; 839 840 WARN_ON_ONCE(timer->function != kthread_delayed_work_timer_fn); 841 842 /* 843 * If @delay is 0, queue @dwork->work immediately. This is for 844 * both optimization and correctness. The earliest @timer can 845 * expire is on the closest next tick and delayed_work users depend 846 * on that there's no such delay when @delay is 0. 847 */ 848 if (!delay) { 849 kthread_insert_work(worker, work, &worker->work_list); 850 return; 851 } 852 853 /* Be paranoid and try to detect possible races already now. */ 854 kthread_insert_work_sanity_check(worker, work); 855 856 list_add(&work->node, &worker->delayed_work_list); 857 work->worker = worker; 858 timer->expires = jiffies + delay; 859 add_timer(timer); 860 } 861 862 /** 863 * kthread_queue_delayed_work - queue the associated kthread work 864 * after a delay. 865 * @worker: target kthread_worker 866 * @dwork: kthread_delayed_work to queue 867 * @delay: number of jiffies to wait before queuing 868 * 869 * If the work has not been pending it starts a timer that will queue 870 * the work after the given @delay. If @delay is zero, it queues the 871 * work immediately. 872 * 873 * Return: %false if the @work has already been pending. It means that 874 * either the timer was running or the work was queued. It returns %true 875 * otherwise. 876 */ 877 bool kthread_queue_delayed_work(struct kthread_worker *worker, 878 struct kthread_delayed_work *dwork, 879 unsigned long delay) 880 { 881 struct kthread_work *work = &dwork->work; 882 unsigned long flags; 883 bool ret = false; 884 885 spin_lock_irqsave(&worker->lock, flags); 886 887 if (!queuing_blocked(worker, work)) { 888 __kthread_queue_delayed_work(worker, dwork, delay); 889 ret = true; 890 } 891 892 spin_unlock_irqrestore(&worker->lock, flags); 893 return ret; 894 } 895 EXPORT_SYMBOL_GPL(kthread_queue_delayed_work); 896 897 struct kthread_flush_work { 898 struct kthread_work work; 899 struct completion done; 900 }; 901 902 static void kthread_flush_work_fn(struct kthread_work *work) 903 { 904 struct kthread_flush_work *fwork = 905 container_of(work, struct kthread_flush_work, work); 906 complete(&fwork->done); 907 } 908 909 /** 910 * kthread_flush_work - flush a kthread_work 911 * @work: work to flush 912 * 913 * If @work is queued or executing, wait for it to finish execution. 914 */ 915 void kthread_flush_work(struct kthread_work *work) 916 { 917 struct kthread_flush_work fwork = { 918 KTHREAD_WORK_INIT(fwork.work, kthread_flush_work_fn), 919 COMPLETION_INITIALIZER_ONSTACK(fwork.done), 920 }; 921 struct kthread_worker *worker; 922 bool noop = false; 923 924 worker = work->worker; 925 if (!worker) 926 return; 927 928 spin_lock_irq(&worker->lock); 929 /* Work must not be used with >1 worker, see kthread_queue_work(). */ 930 WARN_ON_ONCE(work->worker != worker); 931 932 if (!list_empty(&work->node)) 933 kthread_insert_work(worker, &fwork.work, work->node.next); 934 else if (worker->current_work == work) 935 kthread_insert_work(worker, &fwork.work, 936 worker->work_list.next); 937 else 938 noop = true; 939 940 spin_unlock_irq(&worker->lock); 941 942 if (!noop) 943 wait_for_completion(&fwork.done); 944 } 945 EXPORT_SYMBOL_GPL(kthread_flush_work); 946 947 /* 948 * This function removes the work from the worker queue. Also it makes sure 949 * that it won't get queued later via the delayed work's timer. 950 * 951 * The work might still be in use when this function finishes. See the 952 * current_work proceed by the worker. 953 * 954 * Return: %true if @work was pending and successfully canceled, 955 * %false if @work was not pending 956 */ 957 static bool __kthread_cancel_work(struct kthread_work *work, bool is_dwork, 958 unsigned long *flags) 959 { 960 /* Try to cancel the timer if exists. */ 961 if (is_dwork) { 962 struct kthread_delayed_work *dwork = 963 container_of(work, struct kthread_delayed_work, work); 964 struct kthread_worker *worker = work->worker; 965 966 /* 967 * del_timer_sync() must be called to make sure that the timer 968 * callback is not running. The lock must be temporary released 969 * to avoid a deadlock with the callback. In the meantime, 970 * any queuing is blocked by setting the canceling counter. 971 */ 972 work->canceling++; 973 spin_unlock_irqrestore(&worker->lock, *flags); 974 del_timer_sync(&dwork->timer); 975 spin_lock_irqsave(&worker->lock, *flags); 976 work->canceling--; 977 } 978 979 /* 980 * Try to remove the work from a worker list. It might either 981 * be from worker->work_list or from worker->delayed_work_list. 982 */ 983 if (!list_empty(&work->node)) { 984 list_del_init(&work->node); 985 return true; 986 } 987 988 return false; 989 } 990 991 /** 992 * kthread_mod_delayed_work - modify delay of or queue a kthread delayed work 993 * @worker: kthread worker to use 994 * @dwork: kthread delayed work to queue 995 * @delay: number of jiffies to wait before queuing 996 * 997 * If @dwork is idle, equivalent to kthread_queue_delayed_work(). Otherwise, 998 * modify @dwork's timer so that it expires after @delay. If @delay is zero, 999 * @work is guaranteed to be queued immediately. 1000 * 1001 * Return: %true if @dwork was pending and its timer was modified, 1002 * %false otherwise. 1003 * 1004 * A special case is when the work is being canceled in parallel. 1005 * It might be caused either by the real kthread_cancel_delayed_work_sync() 1006 * or yet another kthread_mod_delayed_work() call. We let the other command 1007 * win and return %false here. The caller is supposed to synchronize these 1008 * operations a reasonable way. 1009 * 1010 * This function is safe to call from any context including IRQ handler. 1011 * See __kthread_cancel_work() and kthread_delayed_work_timer_fn() 1012 * for details. 1013 */ 1014 bool kthread_mod_delayed_work(struct kthread_worker *worker, 1015 struct kthread_delayed_work *dwork, 1016 unsigned long delay) 1017 { 1018 struct kthread_work *work = &dwork->work; 1019 unsigned long flags; 1020 int ret = false; 1021 1022 spin_lock_irqsave(&worker->lock, flags); 1023 1024 /* Do not bother with canceling when never queued. */ 1025 if (!work->worker) 1026 goto fast_queue; 1027 1028 /* Work must not be used with >1 worker, see kthread_queue_work() */ 1029 WARN_ON_ONCE(work->worker != worker); 1030 1031 /* Do not fight with another command that is canceling this work. */ 1032 if (work->canceling) 1033 goto out; 1034 1035 ret = __kthread_cancel_work(work, true, &flags); 1036 fast_queue: 1037 __kthread_queue_delayed_work(worker, dwork, delay); 1038 out: 1039 spin_unlock_irqrestore(&worker->lock, flags); 1040 return ret; 1041 } 1042 EXPORT_SYMBOL_GPL(kthread_mod_delayed_work); 1043 1044 static bool __kthread_cancel_work_sync(struct kthread_work *work, bool is_dwork) 1045 { 1046 struct kthread_worker *worker = work->worker; 1047 unsigned long flags; 1048 int ret = false; 1049 1050 if (!worker) 1051 goto out; 1052 1053 spin_lock_irqsave(&worker->lock, flags); 1054 /* Work must not be used with >1 worker, see kthread_queue_work(). */ 1055 WARN_ON_ONCE(work->worker != worker); 1056 1057 ret = __kthread_cancel_work(work, is_dwork, &flags); 1058 1059 if (worker->current_work != work) 1060 goto out_fast; 1061 1062 /* 1063 * The work is in progress and we need to wait with the lock released. 1064 * In the meantime, block any queuing by setting the canceling counter. 1065 */ 1066 work->canceling++; 1067 spin_unlock_irqrestore(&worker->lock, flags); 1068 kthread_flush_work(work); 1069 spin_lock_irqsave(&worker->lock, flags); 1070 work->canceling--; 1071 1072 out_fast: 1073 spin_unlock_irqrestore(&worker->lock, flags); 1074 out: 1075 return ret; 1076 } 1077 1078 /** 1079 * kthread_cancel_work_sync - cancel a kthread work and wait for it to finish 1080 * @work: the kthread work to cancel 1081 * 1082 * Cancel @work and wait for its execution to finish. This function 1083 * can be used even if the work re-queues itself. On return from this 1084 * function, @work is guaranteed to be not pending or executing on any CPU. 1085 * 1086 * kthread_cancel_work_sync(&delayed_work->work) must not be used for 1087 * delayed_work's. Use kthread_cancel_delayed_work_sync() instead. 1088 * 1089 * The caller must ensure that the worker on which @work was last 1090 * queued can't be destroyed before this function returns. 1091 * 1092 * Return: %true if @work was pending, %false otherwise. 1093 */ 1094 bool kthread_cancel_work_sync(struct kthread_work *work) 1095 { 1096 return __kthread_cancel_work_sync(work, false); 1097 } 1098 EXPORT_SYMBOL_GPL(kthread_cancel_work_sync); 1099 1100 /** 1101 * kthread_cancel_delayed_work_sync - cancel a kthread delayed work and 1102 * wait for it to finish. 1103 * @dwork: the kthread delayed work to cancel 1104 * 1105 * This is kthread_cancel_work_sync() for delayed works. 1106 * 1107 * Return: %true if @dwork was pending, %false otherwise. 1108 */ 1109 bool kthread_cancel_delayed_work_sync(struct kthread_delayed_work *dwork) 1110 { 1111 return __kthread_cancel_work_sync(&dwork->work, true); 1112 } 1113 EXPORT_SYMBOL_GPL(kthread_cancel_delayed_work_sync); 1114 1115 /** 1116 * kthread_flush_worker - flush all current works on a kthread_worker 1117 * @worker: worker to flush 1118 * 1119 * Wait until all currently executing or pending works on @worker are 1120 * finished. 1121 */ 1122 void kthread_flush_worker(struct kthread_worker *worker) 1123 { 1124 struct kthread_flush_work fwork = { 1125 KTHREAD_WORK_INIT(fwork.work, kthread_flush_work_fn), 1126 COMPLETION_INITIALIZER_ONSTACK(fwork.done), 1127 }; 1128 1129 kthread_queue_work(worker, &fwork.work); 1130 wait_for_completion(&fwork.done); 1131 } 1132 EXPORT_SYMBOL_GPL(kthread_flush_worker); 1133 1134 /** 1135 * kthread_destroy_worker - destroy a kthread worker 1136 * @worker: worker to be destroyed 1137 * 1138 * Flush and destroy @worker. The simple flush is enough because the kthread 1139 * worker API is used only in trivial scenarios. There are no multi-step state 1140 * machines needed. 1141 */ 1142 void kthread_destroy_worker(struct kthread_worker *worker) 1143 { 1144 struct task_struct *task; 1145 1146 task = worker->task; 1147 if (WARN_ON(!task)) 1148 return; 1149 1150 kthread_flush_worker(worker); 1151 kthread_stop(task); 1152 WARN_ON(!list_empty(&worker->work_list)); 1153 kfree(worker); 1154 } 1155 EXPORT_SYMBOL(kthread_destroy_worker); 1156 1157 #ifdef CONFIG_BLK_CGROUP 1158 /** 1159 * kthread_associate_blkcg - associate blkcg to current kthread 1160 * @css: the cgroup info 1161 * 1162 * Current thread must be a kthread. The thread is running jobs on behalf of 1163 * other threads. In some cases, we expect the jobs attach cgroup info of 1164 * original threads instead of that of current thread. This function stores 1165 * original thread's cgroup info in current kthread context for later 1166 * retrieval. 1167 */ 1168 void kthread_associate_blkcg(struct cgroup_subsys_state *css) 1169 { 1170 struct kthread *kthread; 1171 1172 if (!(current->flags & PF_KTHREAD)) 1173 return; 1174 kthread = to_kthread(current); 1175 if (!kthread) 1176 return; 1177 1178 if (kthread->blkcg_css) { 1179 css_put(kthread->blkcg_css); 1180 kthread->blkcg_css = NULL; 1181 } 1182 if (css) { 1183 css_get(css); 1184 kthread->blkcg_css = css; 1185 } 1186 } 1187 EXPORT_SYMBOL(kthread_associate_blkcg); 1188 1189 /** 1190 * kthread_blkcg - get associated blkcg css of current kthread 1191 * 1192 * Current thread must be a kthread. 1193 */ 1194 struct cgroup_subsys_state *kthread_blkcg(void) 1195 { 1196 struct kthread *kthread; 1197 1198 if (current->flags & PF_KTHREAD) { 1199 kthread = to_kthread(current); 1200 if (kthread) 1201 return kthread->blkcg_css; 1202 } 1203 return NULL; 1204 } 1205 EXPORT_SYMBOL(kthread_blkcg); 1206 #endif 1207