1 /* Kernel thread helper functions. 2 * Copyright (C) 2004 IBM Corporation, Rusty Russell. 3 * 4 * Creation is done via kthreadd, so that we get a clean environment 5 * even if we're invoked from userspace (think modprobe, hotplug cpu, 6 * etc.). 7 */ 8 #include <uapi/linux/sched/types.h> 9 #include <linux/sched.h> 10 #include <linux/sched/task.h> 11 #include <linux/kthread.h> 12 #include <linux/completion.h> 13 #include <linux/err.h> 14 #include <linux/cgroup.h> 15 #include <linux/cpuset.h> 16 #include <linux/unistd.h> 17 #include <linux/file.h> 18 #include <linux/export.h> 19 #include <linux/mutex.h> 20 #include <linux/slab.h> 21 #include <linux/freezer.h> 22 #include <linux/ptrace.h> 23 #include <linux/uaccess.h> 24 #include <linux/numa.h> 25 #include <trace/events/sched.h> 26 27 static DEFINE_SPINLOCK(kthread_create_lock); 28 static LIST_HEAD(kthread_create_list); 29 struct task_struct *kthreadd_task; 30 31 struct kthread_create_info 32 { 33 /* Information passed to kthread() from kthreadd. */ 34 int (*threadfn)(void *data); 35 void *data; 36 int node; 37 38 /* Result passed back to kthread_create() from kthreadd. */ 39 struct task_struct *result; 40 struct completion *done; 41 42 struct list_head list; 43 }; 44 45 struct kthread { 46 unsigned long flags; 47 unsigned int cpu; 48 void *data; 49 struct completion parked; 50 struct completion exited; 51 #ifdef CONFIG_BLK_CGROUP 52 struct cgroup_subsys_state *blkcg_css; 53 #endif 54 }; 55 56 enum KTHREAD_BITS { 57 KTHREAD_IS_PER_CPU = 0, 58 KTHREAD_SHOULD_STOP, 59 KTHREAD_SHOULD_PARK, 60 }; 61 62 static inline void set_kthread_struct(void *kthread) 63 { 64 /* 65 * We abuse ->set_child_tid to avoid the new member and because it 66 * can't be wrongly copied by copy_process(). We also rely on fact 67 * that the caller can't exec, so PF_KTHREAD can't be cleared. 68 */ 69 current->set_child_tid = (__force void __user *)kthread; 70 } 71 72 static inline struct kthread *to_kthread(struct task_struct *k) 73 { 74 WARN_ON(!(k->flags & PF_KTHREAD)); 75 return (__force void *)k->set_child_tid; 76 } 77 78 void free_kthread_struct(struct task_struct *k) 79 { 80 struct kthread *kthread; 81 82 /* 83 * Can be NULL if this kthread was created by kernel_thread() 84 * or if kmalloc() in kthread() failed. 85 */ 86 kthread = to_kthread(k); 87 #ifdef CONFIG_BLK_CGROUP 88 WARN_ON_ONCE(kthread && kthread->blkcg_css); 89 #endif 90 kfree(kthread); 91 } 92 93 /** 94 * kthread_should_stop - should this kthread return now? 95 * 96 * When someone calls kthread_stop() on your kthread, it will be woken 97 * and this will return true. You should then return, and your return 98 * value will be passed through to kthread_stop(). 99 */ 100 bool kthread_should_stop(void) 101 { 102 return test_bit(KTHREAD_SHOULD_STOP, &to_kthread(current)->flags); 103 } 104 EXPORT_SYMBOL(kthread_should_stop); 105 106 bool __kthread_should_park(struct task_struct *k) 107 { 108 return test_bit(KTHREAD_SHOULD_PARK, &to_kthread(k)->flags); 109 } 110 EXPORT_SYMBOL_GPL(__kthread_should_park); 111 112 /** 113 * kthread_should_park - should this kthread park now? 114 * 115 * When someone calls kthread_park() on your kthread, it will be woken 116 * and this will return true. You should then do the necessary 117 * cleanup and call kthread_parkme() 118 * 119 * Similar to kthread_should_stop(), but this keeps the thread alive 120 * and in a park position. kthread_unpark() "restarts" the thread and 121 * calls the thread function again. 122 */ 123 bool kthread_should_park(void) 124 { 125 return __kthread_should_park(current); 126 } 127 EXPORT_SYMBOL_GPL(kthread_should_park); 128 129 /** 130 * kthread_freezable_should_stop - should this freezable kthread return now? 131 * @was_frozen: optional out parameter, indicates whether %current was frozen 132 * 133 * kthread_should_stop() for freezable kthreads, which will enter 134 * refrigerator if necessary. This function is safe from kthread_stop() / 135 * freezer deadlock and freezable kthreads should use this function instead 136 * of calling try_to_freeze() directly. 137 */ 138 bool kthread_freezable_should_stop(bool *was_frozen) 139 { 140 bool frozen = false; 141 142 might_sleep(); 143 144 if (unlikely(freezing(current))) 145 frozen = __refrigerator(true); 146 147 if (was_frozen) 148 *was_frozen = frozen; 149 150 return kthread_should_stop(); 151 } 152 EXPORT_SYMBOL_GPL(kthread_freezable_should_stop); 153 154 /** 155 * kthread_data - return data value specified on kthread creation 156 * @task: kthread task in question 157 * 158 * Return the data value specified when kthread @task was created. 159 * The caller is responsible for ensuring the validity of @task when 160 * calling this function. 161 */ 162 void *kthread_data(struct task_struct *task) 163 { 164 return to_kthread(task)->data; 165 } 166 167 /** 168 * kthread_probe_data - speculative version of kthread_data() 169 * @task: possible kthread task in question 170 * 171 * @task could be a kthread task. Return the data value specified when it 172 * was created if accessible. If @task isn't a kthread task or its data is 173 * inaccessible for any reason, %NULL is returned. This function requires 174 * that @task itself is safe to dereference. 175 */ 176 void *kthread_probe_data(struct task_struct *task) 177 { 178 struct kthread *kthread = to_kthread(task); 179 void *data = NULL; 180 181 probe_kernel_read(&data, &kthread->data, sizeof(data)); 182 return data; 183 } 184 185 static void __kthread_parkme(struct kthread *self) 186 { 187 for (;;) { 188 /* 189 * TASK_PARKED is a special state; we must serialize against 190 * possible pending wakeups to avoid store-store collisions on 191 * task->state. 192 * 193 * Such a collision might possibly result in the task state 194 * changin from TASK_PARKED and us failing the 195 * wait_task_inactive() in kthread_park(). 196 */ 197 set_special_state(TASK_PARKED); 198 if (!test_bit(KTHREAD_SHOULD_PARK, &self->flags)) 199 break; 200 201 complete(&self->parked); 202 schedule(); 203 } 204 __set_current_state(TASK_RUNNING); 205 } 206 207 void kthread_parkme(void) 208 { 209 __kthread_parkme(to_kthread(current)); 210 } 211 EXPORT_SYMBOL_GPL(kthread_parkme); 212 213 static int kthread(void *_create) 214 { 215 /* Copy data: it's on kthread's stack */ 216 struct kthread_create_info *create = _create; 217 int (*threadfn)(void *data) = create->threadfn; 218 void *data = create->data; 219 struct completion *done; 220 struct kthread *self; 221 int ret; 222 223 self = kzalloc(sizeof(*self), GFP_KERNEL); 224 set_kthread_struct(self); 225 226 /* If user was SIGKILLed, I release the structure. */ 227 done = xchg(&create->done, NULL); 228 if (!done) { 229 kfree(create); 230 do_exit(-EINTR); 231 } 232 233 if (!self) { 234 create->result = ERR_PTR(-ENOMEM); 235 complete(done); 236 do_exit(-ENOMEM); 237 } 238 239 self->data = data; 240 init_completion(&self->exited); 241 init_completion(&self->parked); 242 current->vfork_done = &self->exited; 243 244 /* OK, tell user we're spawned, wait for stop or wakeup */ 245 __set_current_state(TASK_UNINTERRUPTIBLE); 246 create->result = current; 247 complete(done); 248 schedule(); 249 250 ret = -EINTR; 251 if (!test_bit(KTHREAD_SHOULD_STOP, &self->flags)) { 252 cgroup_kthread_ready(); 253 __kthread_parkme(self); 254 ret = threadfn(data); 255 } 256 do_exit(ret); 257 } 258 259 /* called from do_fork() to get node information for about to be created task */ 260 int tsk_fork_get_node(struct task_struct *tsk) 261 { 262 #ifdef CONFIG_NUMA 263 if (tsk == kthreadd_task) 264 return tsk->pref_node_fork; 265 #endif 266 return NUMA_NO_NODE; 267 } 268 269 static void create_kthread(struct kthread_create_info *create) 270 { 271 int pid; 272 273 #ifdef CONFIG_NUMA 274 current->pref_node_fork = create->node; 275 #endif 276 /* We want our own signal handler (we take no signals by default). */ 277 pid = kernel_thread(kthread, create, CLONE_FS | CLONE_FILES | SIGCHLD); 278 if (pid < 0) { 279 /* If user was SIGKILLed, I release the structure. */ 280 struct completion *done = xchg(&create->done, NULL); 281 282 if (!done) { 283 kfree(create); 284 return; 285 } 286 create->result = ERR_PTR(pid); 287 complete(done); 288 } 289 } 290 291 static __printf(4, 0) 292 struct task_struct *__kthread_create_on_node(int (*threadfn)(void *data), 293 void *data, int node, 294 const char namefmt[], 295 va_list args) 296 { 297 DECLARE_COMPLETION_ONSTACK(done); 298 struct task_struct *task; 299 struct kthread_create_info *create = kmalloc(sizeof(*create), 300 GFP_KERNEL); 301 302 if (!create) 303 return ERR_PTR(-ENOMEM); 304 create->threadfn = threadfn; 305 create->data = data; 306 create->node = node; 307 create->done = &done; 308 309 spin_lock(&kthread_create_lock); 310 list_add_tail(&create->list, &kthread_create_list); 311 spin_unlock(&kthread_create_lock); 312 313 wake_up_process(kthreadd_task); 314 /* 315 * Wait for completion in killable state, for I might be chosen by 316 * the OOM killer while kthreadd is trying to allocate memory for 317 * new kernel thread. 318 */ 319 if (unlikely(wait_for_completion_killable(&done))) { 320 /* 321 * If I was SIGKILLed before kthreadd (or new kernel thread) 322 * calls complete(), leave the cleanup of this structure to 323 * that thread. 324 */ 325 if (xchg(&create->done, NULL)) 326 return ERR_PTR(-EINTR); 327 /* 328 * kthreadd (or new kernel thread) will call complete() 329 * shortly. 330 */ 331 wait_for_completion(&done); 332 } 333 task = create->result; 334 if (!IS_ERR(task)) { 335 static const struct sched_param param = { .sched_priority = 0 }; 336 char name[TASK_COMM_LEN]; 337 338 /* 339 * task is already visible to other tasks, so updating 340 * COMM must be protected. 341 */ 342 vsnprintf(name, sizeof(name), namefmt, args); 343 set_task_comm(task, name); 344 /* 345 * root may have changed our (kthreadd's) priority or CPU mask. 346 * The kernel thread should not inherit these properties. 347 */ 348 sched_setscheduler_nocheck(task, SCHED_NORMAL, ¶m); 349 set_cpus_allowed_ptr(task, cpu_all_mask); 350 } 351 kfree(create); 352 return task; 353 } 354 355 /** 356 * kthread_create_on_node - create a kthread. 357 * @threadfn: the function to run until signal_pending(current). 358 * @data: data ptr for @threadfn. 359 * @node: task and thread structures for the thread are allocated on this node 360 * @namefmt: printf-style name for the thread. 361 * 362 * Description: This helper function creates and names a kernel 363 * thread. The thread will be stopped: use wake_up_process() to start 364 * it. See also kthread_run(). The new thread has SCHED_NORMAL policy and 365 * is affine to all CPUs. 366 * 367 * If thread is going to be bound on a particular cpu, give its node 368 * in @node, to get NUMA affinity for kthread stack, or else give NUMA_NO_NODE. 369 * When woken, the thread will run @threadfn() with @data as its 370 * argument. @threadfn() can either call do_exit() directly if it is a 371 * standalone thread for which no one will call kthread_stop(), or 372 * return when 'kthread_should_stop()' is true (which means 373 * kthread_stop() has been called). The return value should be zero 374 * or a negative error number; it will be passed to kthread_stop(). 375 * 376 * Returns a task_struct or ERR_PTR(-ENOMEM) or ERR_PTR(-EINTR). 377 */ 378 struct task_struct *kthread_create_on_node(int (*threadfn)(void *data), 379 void *data, int node, 380 const char namefmt[], 381 ...) 382 { 383 struct task_struct *task; 384 va_list args; 385 386 va_start(args, namefmt); 387 task = __kthread_create_on_node(threadfn, data, node, namefmt, args); 388 va_end(args); 389 390 return task; 391 } 392 EXPORT_SYMBOL(kthread_create_on_node); 393 394 static void __kthread_bind_mask(struct task_struct *p, const struct cpumask *mask, long state) 395 { 396 unsigned long flags; 397 398 if (!wait_task_inactive(p, state)) { 399 WARN_ON(1); 400 return; 401 } 402 403 /* It's safe because the task is inactive. */ 404 raw_spin_lock_irqsave(&p->pi_lock, flags); 405 do_set_cpus_allowed(p, mask); 406 p->flags |= PF_NO_SETAFFINITY; 407 raw_spin_unlock_irqrestore(&p->pi_lock, flags); 408 } 409 410 static void __kthread_bind(struct task_struct *p, unsigned int cpu, long state) 411 { 412 __kthread_bind_mask(p, cpumask_of(cpu), state); 413 } 414 415 void kthread_bind_mask(struct task_struct *p, const struct cpumask *mask) 416 { 417 __kthread_bind_mask(p, mask, TASK_UNINTERRUPTIBLE); 418 } 419 420 /** 421 * kthread_bind - bind a just-created kthread to a cpu. 422 * @p: thread created by kthread_create(). 423 * @cpu: cpu (might not be online, must be possible) for @k to run on. 424 * 425 * Description: This function is equivalent to set_cpus_allowed(), 426 * except that @cpu doesn't need to be online, and the thread must be 427 * stopped (i.e., just returned from kthread_create()). 428 */ 429 void kthread_bind(struct task_struct *p, unsigned int cpu) 430 { 431 __kthread_bind(p, cpu, TASK_UNINTERRUPTIBLE); 432 } 433 EXPORT_SYMBOL(kthread_bind); 434 435 /** 436 * kthread_create_on_cpu - Create a cpu bound kthread 437 * @threadfn: the function to run until signal_pending(current). 438 * @data: data ptr for @threadfn. 439 * @cpu: The cpu on which the thread should be bound, 440 * @namefmt: printf-style name for the thread. Format is restricted 441 * to "name.*%u". Code fills in cpu number. 442 * 443 * Description: This helper function creates and names a kernel thread 444 * The thread will be woken and put into park mode. 445 */ 446 struct task_struct *kthread_create_on_cpu(int (*threadfn)(void *data), 447 void *data, unsigned int cpu, 448 const char *namefmt) 449 { 450 struct task_struct *p; 451 452 p = kthread_create_on_node(threadfn, data, cpu_to_node(cpu), namefmt, 453 cpu); 454 if (IS_ERR(p)) 455 return p; 456 kthread_bind(p, cpu); 457 /* CPU hotplug need to bind once again when unparking the thread. */ 458 set_bit(KTHREAD_IS_PER_CPU, &to_kthread(p)->flags); 459 to_kthread(p)->cpu = cpu; 460 return p; 461 } 462 463 /** 464 * kthread_unpark - unpark a thread created by kthread_create(). 465 * @k: thread created by kthread_create(). 466 * 467 * Sets kthread_should_park() for @k to return false, wakes it, and 468 * waits for it to return. If the thread is marked percpu then its 469 * bound to the cpu again. 470 */ 471 void kthread_unpark(struct task_struct *k) 472 { 473 struct kthread *kthread = to_kthread(k); 474 475 /* 476 * Newly created kthread was parked when the CPU was offline. 477 * The binding was lost and we need to set it again. 478 */ 479 if (test_bit(KTHREAD_IS_PER_CPU, &kthread->flags)) 480 __kthread_bind(k, kthread->cpu, TASK_PARKED); 481 482 clear_bit(KTHREAD_SHOULD_PARK, &kthread->flags); 483 /* 484 * __kthread_parkme() will either see !SHOULD_PARK or get the wakeup. 485 */ 486 wake_up_state(k, TASK_PARKED); 487 } 488 EXPORT_SYMBOL_GPL(kthread_unpark); 489 490 /** 491 * kthread_park - park a thread created by kthread_create(). 492 * @k: thread created by kthread_create(). 493 * 494 * Sets kthread_should_park() for @k to return true, wakes it, and 495 * waits for it to return. This can also be called after kthread_create() 496 * instead of calling wake_up_process(): the thread will park without 497 * calling threadfn(). 498 * 499 * Returns 0 if the thread is parked, -ENOSYS if the thread exited. 500 * If called by the kthread itself just the park bit is set. 501 */ 502 int kthread_park(struct task_struct *k) 503 { 504 struct kthread *kthread = to_kthread(k); 505 506 if (WARN_ON(k->flags & PF_EXITING)) 507 return -ENOSYS; 508 509 if (WARN_ON_ONCE(test_bit(KTHREAD_SHOULD_PARK, &kthread->flags))) 510 return -EBUSY; 511 512 set_bit(KTHREAD_SHOULD_PARK, &kthread->flags); 513 if (k != current) { 514 wake_up_process(k); 515 /* 516 * Wait for __kthread_parkme() to complete(), this means we 517 * _will_ have TASK_PARKED and are about to call schedule(). 518 */ 519 wait_for_completion(&kthread->parked); 520 /* 521 * Now wait for that schedule() to complete and the task to 522 * get scheduled out. 523 */ 524 WARN_ON_ONCE(!wait_task_inactive(k, TASK_PARKED)); 525 } 526 527 return 0; 528 } 529 EXPORT_SYMBOL_GPL(kthread_park); 530 531 /** 532 * kthread_stop - stop a thread created by kthread_create(). 533 * @k: thread created by kthread_create(). 534 * 535 * Sets kthread_should_stop() for @k to return true, wakes it, and 536 * waits for it to exit. This can also be called after kthread_create() 537 * instead of calling wake_up_process(): the thread will exit without 538 * calling threadfn(). 539 * 540 * If threadfn() may call do_exit() itself, the caller must ensure 541 * task_struct can't go away. 542 * 543 * Returns the result of threadfn(), or %-EINTR if wake_up_process() 544 * was never called. 545 */ 546 int kthread_stop(struct task_struct *k) 547 { 548 struct kthread *kthread; 549 int ret; 550 551 trace_sched_kthread_stop(k); 552 553 get_task_struct(k); 554 kthread = to_kthread(k); 555 set_bit(KTHREAD_SHOULD_STOP, &kthread->flags); 556 kthread_unpark(k); 557 wake_up_process(k); 558 wait_for_completion(&kthread->exited); 559 ret = k->exit_code; 560 put_task_struct(k); 561 562 trace_sched_kthread_stop_ret(ret); 563 return ret; 564 } 565 EXPORT_SYMBOL(kthread_stop); 566 567 int kthreadd(void *unused) 568 { 569 struct task_struct *tsk = current; 570 571 /* Setup a clean context for our children to inherit. */ 572 set_task_comm(tsk, "kthreadd"); 573 ignore_signals(tsk); 574 set_cpus_allowed_ptr(tsk, cpu_all_mask); 575 set_mems_allowed(node_states[N_MEMORY]); 576 577 current->flags |= PF_NOFREEZE; 578 cgroup_init_kthreadd(); 579 580 for (;;) { 581 set_current_state(TASK_INTERRUPTIBLE); 582 if (list_empty(&kthread_create_list)) 583 schedule(); 584 __set_current_state(TASK_RUNNING); 585 586 spin_lock(&kthread_create_lock); 587 while (!list_empty(&kthread_create_list)) { 588 struct kthread_create_info *create; 589 590 create = list_entry(kthread_create_list.next, 591 struct kthread_create_info, list); 592 list_del_init(&create->list); 593 spin_unlock(&kthread_create_lock); 594 595 create_kthread(create); 596 597 spin_lock(&kthread_create_lock); 598 } 599 spin_unlock(&kthread_create_lock); 600 } 601 602 return 0; 603 } 604 605 void __kthread_init_worker(struct kthread_worker *worker, 606 const char *name, 607 struct lock_class_key *key) 608 { 609 memset(worker, 0, sizeof(struct kthread_worker)); 610 raw_spin_lock_init(&worker->lock); 611 lockdep_set_class_and_name(&worker->lock, key, name); 612 INIT_LIST_HEAD(&worker->work_list); 613 INIT_LIST_HEAD(&worker->delayed_work_list); 614 } 615 EXPORT_SYMBOL_GPL(__kthread_init_worker); 616 617 /** 618 * kthread_worker_fn - kthread function to process kthread_worker 619 * @worker_ptr: pointer to initialized kthread_worker 620 * 621 * This function implements the main cycle of kthread worker. It processes 622 * work_list until it is stopped with kthread_stop(). It sleeps when the queue 623 * is empty. 624 * 625 * The works are not allowed to keep any locks, disable preemption or interrupts 626 * when they finish. There is defined a safe point for freezing when one work 627 * finishes and before a new one is started. 628 * 629 * Also the works must not be handled by more than one worker at the same time, 630 * see also kthread_queue_work(). 631 */ 632 int kthread_worker_fn(void *worker_ptr) 633 { 634 struct kthread_worker *worker = worker_ptr; 635 struct kthread_work *work; 636 637 /* 638 * FIXME: Update the check and remove the assignment when all kthread 639 * worker users are created using kthread_create_worker*() functions. 640 */ 641 WARN_ON(worker->task && worker->task != current); 642 worker->task = current; 643 644 if (worker->flags & KTW_FREEZABLE) 645 set_freezable(); 646 647 repeat: 648 set_current_state(TASK_INTERRUPTIBLE); /* mb paired w/ kthread_stop */ 649 650 if (kthread_should_stop()) { 651 __set_current_state(TASK_RUNNING); 652 raw_spin_lock_irq(&worker->lock); 653 worker->task = NULL; 654 raw_spin_unlock_irq(&worker->lock); 655 return 0; 656 } 657 658 work = NULL; 659 raw_spin_lock_irq(&worker->lock); 660 if (!list_empty(&worker->work_list)) { 661 work = list_first_entry(&worker->work_list, 662 struct kthread_work, node); 663 list_del_init(&work->node); 664 } 665 worker->current_work = work; 666 raw_spin_unlock_irq(&worker->lock); 667 668 if (work) { 669 __set_current_state(TASK_RUNNING); 670 work->func(work); 671 } else if (!freezing(current)) 672 schedule(); 673 674 try_to_freeze(); 675 cond_resched(); 676 goto repeat; 677 } 678 EXPORT_SYMBOL_GPL(kthread_worker_fn); 679 680 static __printf(3, 0) struct kthread_worker * 681 __kthread_create_worker(int cpu, unsigned int flags, 682 const char namefmt[], va_list args) 683 { 684 struct kthread_worker *worker; 685 struct task_struct *task; 686 int node = NUMA_NO_NODE; 687 688 worker = kzalloc(sizeof(*worker), GFP_KERNEL); 689 if (!worker) 690 return ERR_PTR(-ENOMEM); 691 692 kthread_init_worker(worker); 693 694 if (cpu >= 0) 695 node = cpu_to_node(cpu); 696 697 task = __kthread_create_on_node(kthread_worker_fn, worker, 698 node, namefmt, args); 699 if (IS_ERR(task)) 700 goto fail_task; 701 702 if (cpu >= 0) 703 kthread_bind(task, cpu); 704 705 worker->flags = flags; 706 worker->task = task; 707 wake_up_process(task); 708 return worker; 709 710 fail_task: 711 kfree(worker); 712 return ERR_CAST(task); 713 } 714 715 /** 716 * kthread_create_worker - create a kthread worker 717 * @flags: flags modifying the default behavior of the worker 718 * @namefmt: printf-style name for the kthread worker (task). 719 * 720 * Returns a pointer to the allocated worker on success, ERR_PTR(-ENOMEM) 721 * when the needed structures could not get allocated, and ERR_PTR(-EINTR) 722 * when the worker was SIGKILLed. 723 */ 724 struct kthread_worker * 725 kthread_create_worker(unsigned int flags, const char namefmt[], ...) 726 { 727 struct kthread_worker *worker; 728 va_list args; 729 730 va_start(args, namefmt); 731 worker = __kthread_create_worker(-1, flags, namefmt, args); 732 va_end(args); 733 734 return worker; 735 } 736 EXPORT_SYMBOL(kthread_create_worker); 737 738 /** 739 * kthread_create_worker_on_cpu - create a kthread worker and bind it 740 * it to a given CPU and the associated NUMA node. 741 * @cpu: CPU number 742 * @flags: flags modifying the default behavior of the worker 743 * @namefmt: printf-style name for the kthread worker (task). 744 * 745 * Use a valid CPU number if you want to bind the kthread worker 746 * to the given CPU and the associated NUMA node. 747 * 748 * A good practice is to add the cpu number also into the worker name. 749 * For example, use kthread_create_worker_on_cpu(cpu, "helper/%d", cpu). 750 * 751 * Returns a pointer to the allocated worker on success, ERR_PTR(-ENOMEM) 752 * when the needed structures could not get allocated, and ERR_PTR(-EINTR) 753 * when the worker was SIGKILLed. 754 */ 755 struct kthread_worker * 756 kthread_create_worker_on_cpu(int cpu, unsigned int flags, 757 const char namefmt[], ...) 758 { 759 struct kthread_worker *worker; 760 va_list args; 761 762 va_start(args, namefmt); 763 worker = __kthread_create_worker(cpu, flags, namefmt, args); 764 va_end(args); 765 766 return worker; 767 } 768 EXPORT_SYMBOL(kthread_create_worker_on_cpu); 769 770 /* 771 * Returns true when the work could not be queued at the moment. 772 * It happens when it is already pending in a worker list 773 * or when it is being cancelled. 774 */ 775 static inline bool queuing_blocked(struct kthread_worker *worker, 776 struct kthread_work *work) 777 { 778 lockdep_assert_held(&worker->lock); 779 780 return !list_empty(&work->node) || work->canceling; 781 } 782 783 static void kthread_insert_work_sanity_check(struct kthread_worker *worker, 784 struct kthread_work *work) 785 { 786 lockdep_assert_held(&worker->lock); 787 WARN_ON_ONCE(!list_empty(&work->node)); 788 /* Do not use a work with >1 worker, see kthread_queue_work() */ 789 WARN_ON_ONCE(work->worker && work->worker != worker); 790 } 791 792 /* insert @work before @pos in @worker */ 793 static void kthread_insert_work(struct kthread_worker *worker, 794 struct kthread_work *work, 795 struct list_head *pos) 796 { 797 kthread_insert_work_sanity_check(worker, work); 798 799 list_add_tail(&work->node, pos); 800 work->worker = worker; 801 if (!worker->current_work && likely(worker->task)) 802 wake_up_process(worker->task); 803 } 804 805 /** 806 * kthread_queue_work - queue a kthread_work 807 * @worker: target kthread_worker 808 * @work: kthread_work to queue 809 * 810 * Queue @work to work processor @task for async execution. @task 811 * must have been created with kthread_worker_create(). Returns %true 812 * if @work was successfully queued, %false if it was already pending. 813 * 814 * Reinitialize the work if it needs to be used by another worker. 815 * For example, when the worker was stopped and started again. 816 */ 817 bool kthread_queue_work(struct kthread_worker *worker, 818 struct kthread_work *work) 819 { 820 bool ret = false; 821 unsigned long flags; 822 823 raw_spin_lock_irqsave(&worker->lock, flags); 824 if (!queuing_blocked(worker, work)) { 825 kthread_insert_work(worker, work, &worker->work_list); 826 ret = true; 827 } 828 raw_spin_unlock_irqrestore(&worker->lock, flags); 829 return ret; 830 } 831 EXPORT_SYMBOL_GPL(kthread_queue_work); 832 833 /** 834 * kthread_delayed_work_timer_fn - callback that queues the associated kthread 835 * delayed work when the timer expires. 836 * @t: pointer to the expired timer 837 * 838 * The format of the function is defined by struct timer_list. 839 * It should have been called from irqsafe timer with irq already off. 840 */ 841 void kthread_delayed_work_timer_fn(struct timer_list *t) 842 { 843 struct kthread_delayed_work *dwork = from_timer(dwork, t, timer); 844 struct kthread_work *work = &dwork->work; 845 struct kthread_worker *worker = work->worker; 846 unsigned long flags; 847 848 /* 849 * This might happen when a pending work is reinitialized. 850 * It means that it is used a wrong way. 851 */ 852 if (WARN_ON_ONCE(!worker)) 853 return; 854 855 raw_spin_lock_irqsave(&worker->lock, flags); 856 /* Work must not be used with >1 worker, see kthread_queue_work(). */ 857 WARN_ON_ONCE(work->worker != worker); 858 859 /* Move the work from worker->delayed_work_list. */ 860 WARN_ON_ONCE(list_empty(&work->node)); 861 list_del_init(&work->node); 862 kthread_insert_work(worker, work, &worker->work_list); 863 864 raw_spin_unlock_irqrestore(&worker->lock, flags); 865 } 866 EXPORT_SYMBOL(kthread_delayed_work_timer_fn); 867 868 void __kthread_queue_delayed_work(struct kthread_worker *worker, 869 struct kthread_delayed_work *dwork, 870 unsigned long delay) 871 { 872 struct timer_list *timer = &dwork->timer; 873 struct kthread_work *work = &dwork->work; 874 875 WARN_ON_ONCE(timer->function != kthread_delayed_work_timer_fn); 876 877 /* 878 * If @delay is 0, queue @dwork->work immediately. This is for 879 * both optimization and correctness. The earliest @timer can 880 * expire is on the closest next tick and delayed_work users depend 881 * on that there's no such delay when @delay is 0. 882 */ 883 if (!delay) { 884 kthread_insert_work(worker, work, &worker->work_list); 885 return; 886 } 887 888 /* Be paranoid and try to detect possible races already now. */ 889 kthread_insert_work_sanity_check(worker, work); 890 891 list_add(&work->node, &worker->delayed_work_list); 892 work->worker = worker; 893 timer->expires = jiffies + delay; 894 add_timer(timer); 895 } 896 897 /** 898 * kthread_queue_delayed_work - queue the associated kthread work 899 * after a delay. 900 * @worker: target kthread_worker 901 * @dwork: kthread_delayed_work to queue 902 * @delay: number of jiffies to wait before queuing 903 * 904 * If the work has not been pending it starts a timer that will queue 905 * the work after the given @delay. If @delay is zero, it queues the 906 * work immediately. 907 * 908 * Return: %false if the @work has already been pending. It means that 909 * either the timer was running or the work was queued. It returns %true 910 * otherwise. 911 */ 912 bool kthread_queue_delayed_work(struct kthread_worker *worker, 913 struct kthread_delayed_work *dwork, 914 unsigned long delay) 915 { 916 struct kthread_work *work = &dwork->work; 917 unsigned long flags; 918 bool ret = false; 919 920 raw_spin_lock_irqsave(&worker->lock, flags); 921 922 if (!queuing_blocked(worker, work)) { 923 __kthread_queue_delayed_work(worker, dwork, delay); 924 ret = true; 925 } 926 927 raw_spin_unlock_irqrestore(&worker->lock, flags); 928 return ret; 929 } 930 EXPORT_SYMBOL_GPL(kthread_queue_delayed_work); 931 932 struct kthread_flush_work { 933 struct kthread_work work; 934 struct completion done; 935 }; 936 937 static void kthread_flush_work_fn(struct kthread_work *work) 938 { 939 struct kthread_flush_work *fwork = 940 container_of(work, struct kthread_flush_work, work); 941 complete(&fwork->done); 942 } 943 944 /** 945 * kthread_flush_work - flush a kthread_work 946 * @work: work to flush 947 * 948 * If @work is queued or executing, wait for it to finish execution. 949 */ 950 void kthread_flush_work(struct kthread_work *work) 951 { 952 struct kthread_flush_work fwork = { 953 KTHREAD_WORK_INIT(fwork.work, kthread_flush_work_fn), 954 COMPLETION_INITIALIZER_ONSTACK(fwork.done), 955 }; 956 struct kthread_worker *worker; 957 bool noop = false; 958 959 worker = work->worker; 960 if (!worker) 961 return; 962 963 raw_spin_lock_irq(&worker->lock); 964 /* Work must not be used with >1 worker, see kthread_queue_work(). */ 965 WARN_ON_ONCE(work->worker != worker); 966 967 if (!list_empty(&work->node)) 968 kthread_insert_work(worker, &fwork.work, work->node.next); 969 else if (worker->current_work == work) 970 kthread_insert_work(worker, &fwork.work, 971 worker->work_list.next); 972 else 973 noop = true; 974 975 raw_spin_unlock_irq(&worker->lock); 976 977 if (!noop) 978 wait_for_completion(&fwork.done); 979 } 980 EXPORT_SYMBOL_GPL(kthread_flush_work); 981 982 /* 983 * This function removes the work from the worker queue. Also it makes sure 984 * that it won't get queued later via the delayed work's timer. 985 * 986 * The work might still be in use when this function finishes. See the 987 * current_work proceed by the worker. 988 * 989 * Return: %true if @work was pending and successfully canceled, 990 * %false if @work was not pending 991 */ 992 static bool __kthread_cancel_work(struct kthread_work *work, bool is_dwork, 993 unsigned long *flags) 994 { 995 /* Try to cancel the timer if exists. */ 996 if (is_dwork) { 997 struct kthread_delayed_work *dwork = 998 container_of(work, struct kthread_delayed_work, work); 999 struct kthread_worker *worker = work->worker; 1000 1001 /* 1002 * del_timer_sync() must be called to make sure that the timer 1003 * callback is not running. The lock must be temporary released 1004 * to avoid a deadlock with the callback. In the meantime, 1005 * any queuing is blocked by setting the canceling counter. 1006 */ 1007 work->canceling++; 1008 raw_spin_unlock_irqrestore(&worker->lock, *flags); 1009 del_timer_sync(&dwork->timer); 1010 raw_spin_lock_irqsave(&worker->lock, *flags); 1011 work->canceling--; 1012 } 1013 1014 /* 1015 * Try to remove the work from a worker list. It might either 1016 * be from worker->work_list or from worker->delayed_work_list. 1017 */ 1018 if (!list_empty(&work->node)) { 1019 list_del_init(&work->node); 1020 return true; 1021 } 1022 1023 return false; 1024 } 1025 1026 /** 1027 * kthread_mod_delayed_work - modify delay of or queue a kthread delayed work 1028 * @worker: kthread worker to use 1029 * @dwork: kthread delayed work to queue 1030 * @delay: number of jiffies to wait before queuing 1031 * 1032 * If @dwork is idle, equivalent to kthread_queue_delayed_work(). Otherwise, 1033 * modify @dwork's timer so that it expires after @delay. If @delay is zero, 1034 * @work is guaranteed to be queued immediately. 1035 * 1036 * Return: %true if @dwork was pending and its timer was modified, 1037 * %false otherwise. 1038 * 1039 * A special case is when the work is being canceled in parallel. 1040 * It might be caused either by the real kthread_cancel_delayed_work_sync() 1041 * or yet another kthread_mod_delayed_work() call. We let the other command 1042 * win and return %false here. The caller is supposed to synchronize these 1043 * operations a reasonable way. 1044 * 1045 * This function is safe to call from any context including IRQ handler. 1046 * See __kthread_cancel_work() and kthread_delayed_work_timer_fn() 1047 * for details. 1048 */ 1049 bool kthread_mod_delayed_work(struct kthread_worker *worker, 1050 struct kthread_delayed_work *dwork, 1051 unsigned long delay) 1052 { 1053 struct kthread_work *work = &dwork->work; 1054 unsigned long flags; 1055 int ret = false; 1056 1057 raw_spin_lock_irqsave(&worker->lock, flags); 1058 1059 /* Do not bother with canceling when never queued. */ 1060 if (!work->worker) 1061 goto fast_queue; 1062 1063 /* Work must not be used with >1 worker, see kthread_queue_work() */ 1064 WARN_ON_ONCE(work->worker != worker); 1065 1066 /* Do not fight with another command that is canceling this work. */ 1067 if (work->canceling) 1068 goto out; 1069 1070 ret = __kthread_cancel_work(work, true, &flags); 1071 fast_queue: 1072 __kthread_queue_delayed_work(worker, dwork, delay); 1073 out: 1074 raw_spin_unlock_irqrestore(&worker->lock, flags); 1075 return ret; 1076 } 1077 EXPORT_SYMBOL_GPL(kthread_mod_delayed_work); 1078 1079 static bool __kthread_cancel_work_sync(struct kthread_work *work, bool is_dwork) 1080 { 1081 struct kthread_worker *worker = work->worker; 1082 unsigned long flags; 1083 int ret = false; 1084 1085 if (!worker) 1086 goto out; 1087 1088 raw_spin_lock_irqsave(&worker->lock, flags); 1089 /* Work must not be used with >1 worker, see kthread_queue_work(). */ 1090 WARN_ON_ONCE(work->worker != worker); 1091 1092 ret = __kthread_cancel_work(work, is_dwork, &flags); 1093 1094 if (worker->current_work != work) 1095 goto out_fast; 1096 1097 /* 1098 * The work is in progress and we need to wait with the lock released. 1099 * In the meantime, block any queuing by setting the canceling counter. 1100 */ 1101 work->canceling++; 1102 raw_spin_unlock_irqrestore(&worker->lock, flags); 1103 kthread_flush_work(work); 1104 raw_spin_lock_irqsave(&worker->lock, flags); 1105 work->canceling--; 1106 1107 out_fast: 1108 raw_spin_unlock_irqrestore(&worker->lock, flags); 1109 out: 1110 return ret; 1111 } 1112 1113 /** 1114 * kthread_cancel_work_sync - cancel a kthread work and wait for it to finish 1115 * @work: the kthread work to cancel 1116 * 1117 * Cancel @work and wait for its execution to finish. This function 1118 * can be used even if the work re-queues itself. On return from this 1119 * function, @work is guaranteed to be not pending or executing on any CPU. 1120 * 1121 * kthread_cancel_work_sync(&delayed_work->work) must not be used for 1122 * delayed_work's. Use kthread_cancel_delayed_work_sync() instead. 1123 * 1124 * The caller must ensure that the worker on which @work was last 1125 * queued can't be destroyed before this function returns. 1126 * 1127 * Return: %true if @work was pending, %false otherwise. 1128 */ 1129 bool kthread_cancel_work_sync(struct kthread_work *work) 1130 { 1131 return __kthread_cancel_work_sync(work, false); 1132 } 1133 EXPORT_SYMBOL_GPL(kthread_cancel_work_sync); 1134 1135 /** 1136 * kthread_cancel_delayed_work_sync - cancel a kthread delayed work and 1137 * wait for it to finish. 1138 * @dwork: the kthread delayed work to cancel 1139 * 1140 * This is kthread_cancel_work_sync() for delayed works. 1141 * 1142 * Return: %true if @dwork was pending, %false otherwise. 1143 */ 1144 bool kthread_cancel_delayed_work_sync(struct kthread_delayed_work *dwork) 1145 { 1146 return __kthread_cancel_work_sync(&dwork->work, true); 1147 } 1148 EXPORT_SYMBOL_GPL(kthread_cancel_delayed_work_sync); 1149 1150 /** 1151 * kthread_flush_worker - flush all current works on a kthread_worker 1152 * @worker: worker to flush 1153 * 1154 * Wait until all currently executing or pending works on @worker are 1155 * finished. 1156 */ 1157 void kthread_flush_worker(struct kthread_worker *worker) 1158 { 1159 struct kthread_flush_work fwork = { 1160 KTHREAD_WORK_INIT(fwork.work, kthread_flush_work_fn), 1161 COMPLETION_INITIALIZER_ONSTACK(fwork.done), 1162 }; 1163 1164 kthread_queue_work(worker, &fwork.work); 1165 wait_for_completion(&fwork.done); 1166 } 1167 EXPORT_SYMBOL_GPL(kthread_flush_worker); 1168 1169 /** 1170 * kthread_destroy_worker - destroy a kthread worker 1171 * @worker: worker to be destroyed 1172 * 1173 * Flush and destroy @worker. The simple flush is enough because the kthread 1174 * worker API is used only in trivial scenarios. There are no multi-step state 1175 * machines needed. 1176 */ 1177 void kthread_destroy_worker(struct kthread_worker *worker) 1178 { 1179 struct task_struct *task; 1180 1181 task = worker->task; 1182 if (WARN_ON(!task)) 1183 return; 1184 1185 kthread_flush_worker(worker); 1186 kthread_stop(task); 1187 WARN_ON(!list_empty(&worker->work_list)); 1188 kfree(worker); 1189 } 1190 EXPORT_SYMBOL(kthread_destroy_worker); 1191 1192 #ifdef CONFIG_BLK_CGROUP 1193 /** 1194 * kthread_associate_blkcg - associate blkcg to current kthread 1195 * @css: the cgroup info 1196 * 1197 * Current thread must be a kthread. The thread is running jobs on behalf of 1198 * other threads. In some cases, we expect the jobs attach cgroup info of 1199 * original threads instead of that of current thread. This function stores 1200 * original thread's cgroup info in current kthread context for later 1201 * retrieval. 1202 */ 1203 void kthread_associate_blkcg(struct cgroup_subsys_state *css) 1204 { 1205 struct kthread *kthread; 1206 1207 if (!(current->flags & PF_KTHREAD)) 1208 return; 1209 kthread = to_kthread(current); 1210 if (!kthread) 1211 return; 1212 1213 if (kthread->blkcg_css) { 1214 css_put(kthread->blkcg_css); 1215 kthread->blkcg_css = NULL; 1216 } 1217 if (css) { 1218 css_get(css); 1219 kthread->blkcg_css = css; 1220 } 1221 } 1222 EXPORT_SYMBOL(kthread_associate_blkcg); 1223 1224 /** 1225 * kthread_blkcg - get associated blkcg css of current kthread 1226 * 1227 * Current thread must be a kthread. 1228 */ 1229 struct cgroup_subsys_state *kthread_blkcg(void) 1230 { 1231 struct kthread *kthread; 1232 1233 if (current->flags & PF_KTHREAD) { 1234 kthread = to_kthread(current); 1235 if (kthread) 1236 return kthread->blkcg_css; 1237 } 1238 return NULL; 1239 } 1240 EXPORT_SYMBOL(kthread_blkcg); 1241 #endif 1242