1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Kernel thread helper functions. 3 * Copyright (C) 2004 IBM Corporation, Rusty Russell. 4 * Copyright (C) 2009 Red Hat, Inc. 5 * 6 * Creation is done via kthreadd, so that we get a clean environment 7 * even if we're invoked from userspace (think modprobe, hotplug cpu, 8 * etc.). 9 */ 10 #include <uapi/linux/sched/types.h> 11 #include <linux/mm.h> 12 #include <linux/mmu_context.h> 13 #include <linux/sched.h> 14 #include <linux/sched/mm.h> 15 #include <linux/sched/task.h> 16 #include <linux/kthread.h> 17 #include <linux/completion.h> 18 #include <linux/err.h> 19 #include <linux/cgroup.h> 20 #include <linux/cpuset.h> 21 #include <linux/unistd.h> 22 #include <linux/file.h> 23 #include <linux/export.h> 24 #include <linux/mutex.h> 25 #include <linux/slab.h> 26 #include <linux/freezer.h> 27 #include <linux/ptrace.h> 28 #include <linux/uaccess.h> 29 #include <linux/numa.h> 30 #include <linux/sched/isolation.h> 31 #include <trace/events/sched.h> 32 33 34 static DEFINE_SPINLOCK(kthread_create_lock); 35 static LIST_HEAD(kthread_create_list); 36 struct task_struct *kthreadd_task; 37 38 struct kthread_create_info 39 { 40 /* Information passed to kthread() from kthreadd. */ 41 int (*threadfn)(void *data); 42 void *data; 43 int node; 44 45 /* Result passed back to kthread_create() from kthreadd. */ 46 struct task_struct *result; 47 struct completion *done; 48 49 struct list_head list; 50 }; 51 52 struct kthread { 53 unsigned long flags; 54 unsigned int cpu; 55 int (*threadfn)(void *); 56 void *data; 57 mm_segment_t oldfs; 58 struct completion parked; 59 struct completion exited; 60 #ifdef CONFIG_BLK_CGROUP 61 struct cgroup_subsys_state *blkcg_css; 62 #endif 63 }; 64 65 enum KTHREAD_BITS { 66 KTHREAD_IS_PER_CPU = 0, 67 KTHREAD_SHOULD_STOP, 68 KTHREAD_SHOULD_PARK, 69 }; 70 71 static inline struct kthread *to_kthread(struct task_struct *k) 72 { 73 WARN_ON(!(k->flags & PF_KTHREAD)); 74 return (__force void *)k->set_child_tid; 75 } 76 77 /* 78 * Variant of to_kthread() that doesn't assume @p is a kthread. 79 * 80 * Per construction; when: 81 * 82 * (p->flags & PF_KTHREAD) && p->set_child_tid 83 * 84 * the task is both a kthread and struct kthread is persistent. However 85 * PF_KTHREAD on it's own is not, kernel_thread() can exec() (See umh.c and 86 * begin_new_exec()). 87 */ 88 static inline struct kthread *__to_kthread(struct task_struct *p) 89 { 90 void *kthread = (__force void *)p->set_child_tid; 91 if (kthread && !(p->flags & PF_KTHREAD)) 92 kthread = NULL; 93 return kthread; 94 } 95 96 void set_kthread_struct(struct task_struct *p) 97 { 98 struct kthread *kthread; 99 100 if (__to_kthread(p)) 101 return; 102 103 kthread = kzalloc(sizeof(*kthread), GFP_KERNEL); 104 /* 105 * We abuse ->set_child_tid to avoid the new member and because it 106 * can't be wrongly copied by copy_process(). We also rely on fact 107 * that the caller can't exec, so PF_KTHREAD can't be cleared. 108 */ 109 p->set_child_tid = (__force void __user *)kthread; 110 } 111 112 void free_kthread_struct(struct task_struct *k) 113 { 114 struct kthread *kthread; 115 116 /* 117 * Can be NULL if this kthread was created by kernel_thread() 118 * or if kmalloc() in kthread() failed. 119 */ 120 kthread = to_kthread(k); 121 #ifdef CONFIG_BLK_CGROUP 122 WARN_ON_ONCE(kthread && kthread->blkcg_css); 123 #endif 124 kfree(kthread); 125 } 126 127 /** 128 * kthread_should_stop - should this kthread return now? 129 * 130 * When someone calls kthread_stop() on your kthread, it will be woken 131 * and this will return true. You should then return, and your return 132 * value will be passed through to kthread_stop(). 133 */ 134 bool kthread_should_stop(void) 135 { 136 return test_bit(KTHREAD_SHOULD_STOP, &to_kthread(current)->flags); 137 } 138 EXPORT_SYMBOL(kthread_should_stop); 139 140 bool __kthread_should_park(struct task_struct *k) 141 { 142 return test_bit(KTHREAD_SHOULD_PARK, &to_kthread(k)->flags); 143 } 144 EXPORT_SYMBOL_GPL(__kthread_should_park); 145 146 /** 147 * kthread_should_park - should this kthread park now? 148 * 149 * When someone calls kthread_park() on your kthread, it will be woken 150 * and this will return true. You should then do the necessary 151 * cleanup and call kthread_parkme() 152 * 153 * Similar to kthread_should_stop(), but this keeps the thread alive 154 * and in a park position. kthread_unpark() "restarts" the thread and 155 * calls the thread function again. 156 */ 157 bool kthread_should_park(void) 158 { 159 return __kthread_should_park(current); 160 } 161 EXPORT_SYMBOL_GPL(kthread_should_park); 162 163 /** 164 * kthread_freezable_should_stop - should this freezable kthread return now? 165 * @was_frozen: optional out parameter, indicates whether %current was frozen 166 * 167 * kthread_should_stop() for freezable kthreads, which will enter 168 * refrigerator if necessary. This function is safe from kthread_stop() / 169 * freezer deadlock and freezable kthreads should use this function instead 170 * of calling try_to_freeze() directly. 171 */ 172 bool kthread_freezable_should_stop(bool *was_frozen) 173 { 174 bool frozen = false; 175 176 might_sleep(); 177 178 if (unlikely(freezing(current))) 179 frozen = __refrigerator(true); 180 181 if (was_frozen) 182 *was_frozen = frozen; 183 184 return kthread_should_stop(); 185 } 186 EXPORT_SYMBOL_GPL(kthread_freezable_should_stop); 187 188 /** 189 * kthread_func - return the function specified on kthread creation 190 * @task: kthread task in question 191 * 192 * Returns NULL if the task is not a kthread. 193 */ 194 void *kthread_func(struct task_struct *task) 195 { 196 struct kthread *kthread = __to_kthread(task); 197 if (kthread) 198 return kthread->threadfn; 199 return NULL; 200 } 201 EXPORT_SYMBOL_GPL(kthread_func); 202 203 /** 204 * kthread_data - return data value specified on kthread creation 205 * @task: kthread task in question 206 * 207 * Return the data value specified when kthread @task was created. 208 * The caller is responsible for ensuring the validity of @task when 209 * calling this function. 210 */ 211 void *kthread_data(struct task_struct *task) 212 { 213 return to_kthread(task)->data; 214 } 215 EXPORT_SYMBOL_GPL(kthread_data); 216 217 /** 218 * kthread_probe_data - speculative version of kthread_data() 219 * @task: possible kthread task in question 220 * 221 * @task could be a kthread task. Return the data value specified when it 222 * was created if accessible. If @task isn't a kthread task or its data is 223 * inaccessible for any reason, %NULL is returned. This function requires 224 * that @task itself is safe to dereference. 225 */ 226 void *kthread_probe_data(struct task_struct *task) 227 { 228 struct kthread *kthread = __to_kthread(task); 229 void *data = NULL; 230 231 if (kthread) 232 copy_from_kernel_nofault(&data, &kthread->data, sizeof(data)); 233 return data; 234 } 235 236 static void __kthread_parkme(struct kthread *self) 237 { 238 for (;;) { 239 /* 240 * TASK_PARKED is a special state; we must serialize against 241 * possible pending wakeups to avoid store-store collisions on 242 * task->state. 243 * 244 * Such a collision might possibly result in the task state 245 * changin from TASK_PARKED and us failing the 246 * wait_task_inactive() in kthread_park(). 247 */ 248 set_special_state(TASK_PARKED); 249 if (!test_bit(KTHREAD_SHOULD_PARK, &self->flags)) 250 break; 251 252 /* 253 * Thread is going to call schedule(), do not preempt it, 254 * or the caller of kthread_park() may spend more time in 255 * wait_task_inactive(). 256 */ 257 preempt_disable(); 258 complete(&self->parked); 259 schedule_preempt_disabled(); 260 preempt_enable(); 261 } 262 __set_current_state(TASK_RUNNING); 263 } 264 265 void kthread_parkme(void) 266 { 267 __kthread_parkme(to_kthread(current)); 268 } 269 EXPORT_SYMBOL_GPL(kthread_parkme); 270 271 /** 272 * kthread_exit - Cause the current kthread return @result to kthread_stop(). 273 * @result: The integer value to return to kthread_stop(). 274 * 275 * While kthread_exit can be called directly, it exists so that 276 * functions which do some additional work in non-modular code such as 277 * module_put_and_kthread_exit can be implemented. 278 * 279 * Does not return. 280 */ 281 void __noreturn kthread_exit(long result) 282 { 283 do_exit(result); 284 } 285 286 /** 287 * kthread_complete_and exit - Exit the current kthread. 288 * @comp: Completion to complete 289 * @code: The integer value to return to kthread_stop(). 290 * 291 * If present complete @comp and the reuturn code to kthread_stop(). 292 * 293 * A kernel thread whose module may be removed after the completion of 294 * @comp can use this function exit safely. 295 * 296 * Does not return. 297 */ 298 void __noreturn kthread_complete_and_exit(struct completion *comp, long code) 299 { 300 if (comp) 301 complete(comp); 302 303 kthread_exit(code); 304 } 305 EXPORT_SYMBOL(kthread_complete_and_exit); 306 307 static int kthread(void *_create) 308 { 309 static const struct sched_param param = { .sched_priority = 0 }; 310 /* Copy data: it's on kthread's stack */ 311 struct kthread_create_info *create = _create; 312 int (*threadfn)(void *data) = create->threadfn; 313 void *data = create->data; 314 struct completion *done; 315 struct kthread *self; 316 int ret; 317 318 set_kthread_struct(current); 319 self = to_kthread(current); 320 321 /* If user was SIGKILLed, I release the structure. */ 322 done = xchg(&create->done, NULL); 323 if (!done) { 324 kfree(create); 325 kthread_exit(-EINTR); 326 } 327 328 if (!self) { 329 create->result = ERR_PTR(-ENOMEM); 330 complete(done); 331 kthread_exit(-ENOMEM); 332 } 333 334 self->threadfn = threadfn; 335 self->data = data; 336 init_completion(&self->exited); 337 init_completion(&self->parked); 338 current->vfork_done = &self->exited; 339 340 /* 341 * The new thread inherited kthreadd's priority and CPU mask. Reset 342 * back to default in case they have been changed. 343 */ 344 sched_setscheduler_nocheck(current, SCHED_NORMAL, ¶m); 345 set_cpus_allowed_ptr(current, housekeeping_cpumask(HK_FLAG_KTHREAD)); 346 347 /* OK, tell user we're spawned, wait for stop or wakeup */ 348 __set_current_state(TASK_UNINTERRUPTIBLE); 349 create->result = current; 350 /* 351 * Thread is going to call schedule(), do not preempt it, 352 * or the creator may spend more time in wait_task_inactive(). 353 */ 354 preempt_disable(); 355 complete(done); 356 schedule_preempt_disabled(); 357 preempt_enable(); 358 359 ret = -EINTR; 360 if (!test_bit(KTHREAD_SHOULD_STOP, &self->flags)) { 361 cgroup_kthread_ready(); 362 __kthread_parkme(self); 363 ret = threadfn(data); 364 } 365 kthread_exit(ret); 366 } 367 368 /* called from kernel_clone() to get node information for about to be created task */ 369 int tsk_fork_get_node(struct task_struct *tsk) 370 { 371 #ifdef CONFIG_NUMA 372 if (tsk == kthreadd_task) 373 return tsk->pref_node_fork; 374 #endif 375 return NUMA_NO_NODE; 376 } 377 378 static void create_kthread(struct kthread_create_info *create) 379 { 380 int pid; 381 382 #ifdef CONFIG_NUMA 383 current->pref_node_fork = create->node; 384 #endif 385 /* We want our own signal handler (we take no signals by default). */ 386 pid = kernel_thread(kthread, create, CLONE_FS | CLONE_FILES | SIGCHLD); 387 if (pid < 0) { 388 /* If user was SIGKILLed, I release the structure. */ 389 struct completion *done = xchg(&create->done, NULL); 390 391 if (!done) { 392 kfree(create); 393 return; 394 } 395 create->result = ERR_PTR(pid); 396 complete(done); 397 } 398 } 399 400 static __printf(4, 0) 401 struct task_struct *__kthread_create_on_node(int (*threadfn)(void *data), 402 void *data, int node, 403 const char namefmt[], 404 va_list args) 405 { 406 DECLARE_COMPLETION_ONSTACK(done); 407 struct task_struct *task; 408 struct kthread_create_info *create = kmalloc(sizeof(*create), 409 GFP_KERNEL); 410 411 if (!create) 412 return ERR_PTR(-ENOMEM); 413 create->threadfn = threadfn; 414 create->data = data; 415 create->node = node; 416 create->done = &done; 417 418 spin_lock(&kthread_create_lock); 419 list_add_tail(&create->list, &kthread_create_list); 420 spin_unlock(&kthread_create_lock); 421 422 wake_up_process(kthreadd_task); 423 /* 424 * Wait for completion in killable state, for I might be chosen by 425 * the OOM killer while kthreadd is trying to allocate memory for 426 * new kernel thread. 427 */ 428 if (unlikely(wait_for_completion_killable(&done))) { 429 /* 430 * If I was SIGKILLed before kthreadd (or new kernel thread) 431 * calls complete(), leave the cleanup of this structure to 432 * that thread. 433 */ 434 if (xchg(&create->done, NULL)) 435 return ERR_PTR(-EINTR); 436 /* 437 * kthreadd (or new kernel thread) will call complete() 438 * shortly. 439 */ 440 wait_for_completion(&done); 441 } 442 task = create->result; 443 if (!IS_ERR(task)) { 444 char name[TASK_COMM_LEN]; 445 446 /* 447 * task is already visible to other tasks, so updating 448 * COMM must be protected. 449 */ 450 vsnprintf(name, sizeof(name), namefmt, args); 451 set_task_comm(task, name); 452 } 453 kfree(create); 454 return task; 455 } 456 457 /** 458 * kthread_create_on_node - create a kthread. 459 * @threadfn: the function to run until signal_pending(current). 460 * @data: data ptr for @threadfn. 461 * @node: task and thread structures for the thread are allocated on this node 462 * @namefmt: printf-style name for the thread. 463 * 464 * Description: This helper function creates and names a kernel 465 * thread. The thread will be stopped: use wake_up_process() to start 466 * it. See also kthread_run(). The new thread has SCHED_NORMAL policy and 467 * is affine to all CPUs. 468 * 469 * If thread is going to be bound on a particular cpu, give its node 470 * in @node, to get NUMA affinity for kthread stack, or else give NUMA_NO_NODE. 471 * When woken, the thread will run @threadfn() with @data as its 472 * argument. @threadfn() can either return directly if it is a 473 * standalone thread for which no one will call kthread_stop(), or 474 * return when 'kthread_should_stop()' is true (which means 475 * kthread_stop() has been called). The return value should be zero 476 * or a negative error number; it will be passed to kthread_stop(). 477 * 478 * Returns a task_struct or ERR_PTR(-ENOMEM) or ERR_PTR(-EINTR). 479 */ 480 struct task_struct *kthread_create_on_node(int (*threadfn)(void *data), 481 void *data, int node, 482 const char namefmt[], 483 ...) 484 { 485 struct task_struct *task; 486 va_list args; 487 488 va_start(args, namefmt); 489 task = __kthread_create_on_node(threadfn, data, node, namefmt, args); 490 va_end(args); 491 492 return task; 493 } 494 EXPORT_SYMBOL(kthread_create_on_node); 495 496 static void __kthread_bind_mask(struct task_struct *p, const struct cpumask *mask, unsigned int state) 497 { 498 unsigned long flags; 499 500 if (!wait_task_inactive(p, state)) { 501 WARN_ON(1); 502 return; 503 } 504 505 /* It's safe because the task is inactive. */ 506 raw_spin_lock_irqsave(&p->pi_lock, flags); 507 do_set_cpus_allowed(p, mask); 508 p->flags |= PF_NO_SETAFFINITY; 509 raw_spin_unlock_irqrestore(&p->pi_lock, flags); 510 } 511 512 static void __kthread_bind(struct task_struct *p, unsigned int cpu, unsigned int state) 513 { 514 __kthread_bind_mask(p, cpumask_of(cpu), state); 515 } 516 517 void kthread_bind_mask(struct task_struct *p, const struct cpumask *mask) 518 { 519 __kthread_bind_mask(p, mask, TASK_UNINTERRUPTIBLE); 520 } 521 522 /** 523 * kthread_bind - bind a just-created kthread to a cpu. 524 * @p: thread created by kthread_create(). 525 * @cpu: cpu (might not be online, must be possible) for @k to run on. 526 * 527 * Description: This function is equivalent to set_cpus_allowed(), 528 * except that @cpu doesn't need to be online, and the thread must be 529 * stopped (i.e., just returned from kthread_create()). 530 */ 531 void kthread_bind(struct task_struct *p, unsigned int cpu) 532 { 533 __kthread_bind(p, cpu, TASK_UNINTERRUPTIBLE); 534 } 535 EXPORT_SYMBOL(kthread_bind); 536 537 /** 538 * kthread_create_on_cpu - Create a cpu bound kthread 539 * @threadfn: the function to run until signal_pending(current). 540 * @data: data ptr for @threadfn. 541 * @cpu: The cpu on which the thread should be bound, 542 * @namefmt: printf-style name for the thread. Format is restricted 543 * to "name.*%u". Code fills in cpu number. 544 * 545 * Description: This helper function creates and names a kernel thread 546 */ 547 struct task_struct *kthread_create_on_cpu(int (*threadfn)(void *data), 548 void *data, unsigned int cpu, 549 const char *namefmt) 550 { 551 struct task_struct *p; 552 553 p = kthread_create_on_node(threadfn, data, cpu_to_node(cpu), namefmt, 554 cpu); 555 if (IS_ERR(p)) 556 return p; 557 kthread_bind(p, cpu); 558 /* CPU hotplug need to bind once again when unparking the thread. */ 559 to_kthread(p)->cpu = cpu; 560 return p; 561 } 562 563 void kthread_set_per_cpu(struct task_struct *k, int cpu) 564 { 565 struct kthread *kthread = to_kthread(k); 566 if (!kthread) 567 return; 568 569 WARN_ON_ONCE(!(k->flags & PF_NO_SETAFFINITY)); 570 571 if (cpu < 0) { 572 clear_bit(KTHREAD_IS_PER_CPU, &kthread->flags); 573 return; 574 } 575 576 kthread->cpu = cpu; 577 set_bit(KTHREAD_IS_PER_CPU, &kthread->flags); 578 } 579 580 bool kthread_is_per_cpu(struct task_struct *p) 581 { 582 struct kthread *kthread = __to_kthread(p); 583 if (!kthread) 584 return false; 585 586 return test_bit(KTHREAD_IS_PER_CPU, &kthread->flags); 587 } 588 589 /** 590 * kthread_unpark - unpark a thread created by kthread_create(). 591 * @k: thread created by kthread_create(). 592 * 593 * Sets kthread_should_park() for @k to return false, wakes it, and 594 * waits for it to return. If the thread is marked percpu then its 595 * bound to the cpu again. 596 */ 597 void kthread_unpark(struct task_struct *k) 598 { 599 struct kthread *kthread = to_kthread(k); 600 601 /* 602 * Newly created kthread was parked when the CPU was offline. 603 * The binding was lost and we need to set it again. 604 */ 605 if (test_bit(KTHREAD_IS_PER_CPU, &kthread->flags)) 606 __kthread_bind(k, kthread->cpu, TASK_PARKED); 607 608 clear_bit(KTHREAD_SHOULD_PARK, &kthread->flags); 609 /* 610 * __kthread_parkme() will either see !SHOULD_PARK or get the wakeup. 611 */ 612 wake_up_state(k, TASK_PARKED); 613 } 614 EXPORT_SYMBOL_GPL(kthread_unpark); 615 616 /** 617 * kthread_park - park a thread created by kthread_create(). 618 * @k: thread created by kthread_create(). 619 * 620 * Sets kthread_should_park() for @k to return true, wakes it, and 621 * waits for it to return. This can also be called after kthread_create() 622 * instead of calling wake_up_process(): the thread will park without 623 * calling threadfn(). 624 * 625 * Returns 0 if the thread is parked, -ENOSYS if the thread exited. 626 * If called by the kthread itself just the park bit is set. 627 */ 628 int kthread_park(struct task_struct *k) 629 { 630 struct kthread *kthread = to_kthread(k); 631 632 if (WARN_ON(k->flags & PF_EXITING)) 633 return -ENOSYS; 634 635 if (WARN_ON_ONCE(test_bit(KTHREAD_SHOULD_PARK, &kthread->flags))) 636 return -EBUSY; 637 638 set_bit(KTHREAD_SHOULD_PARK, &kthread->flags); 639 if (k != current) { 640 wake_up_process(k); 641 /* 642 * Wait for __kthread_parkme() to complete(), this means we 643 * _will_ have TASK_PARKED and are about to call schedule(). 644 */ 645 wait_for_completion(&kthread->parked); 646 /* 647 * Now wait for that schedule() to complete and the task to 648 * get scheduled out. 649 */ 650 WARN_ON_ONCE(!wait_task_inactive(k, TASK_PARKED)); 651 } 652 653 return 0; 654 } 655 EXPORT_SYMBOL_GPL(kthread_park); 656 657 /** 658 * kthread_stop - stop a thread created by kthread_create(). 659 * @k: thread created by kthread_create(). 660 * 661 * Sets kthread_should_stop() for @k to return true, wakes it, and 662 * waits for it to exit. This can also be called after kthread_create() 663 * instead of calling wake_up_process(): the thread will exit without 664 * calling threadfn(). 665 * 666 * If threadfn() may call kthread_exit() itself, the caller must ensure 667 * task_struct can't go away. 668 * 669 * Returns the result of threadfn(), or %-EINTR if wake_up_process() 670 * was never called. 671 */ 672 int kthread_stop(struct task_struct *k) 673 { 674 struct kthread *kthread; 675 int ret; 676 677 trace_sched_kthread_stop(k); 678 679 get_task_struct(k); 680 kthread = to_kthread(k); 681 set_bit(KTHREAD_SHOULD_STOP, &kthread->flags); 682 kthread_unpark(k); 683 wake_up_process(k); 684 wait_for_completion(&kthread->exited); 685 ret = k->exit_code; 686 put_task_struct(k); 687 688 trace_sched_kthread_stop_ret(ret); 689 return ret; 690 } 691 EXPORT_SYMBOL(kthread_stop); 692 693 int kthreadd(void *unused) 694 { 695 struct task_struct *tsk = current; 696 697 /* Setup a clean context for our children to inherit. */ 698 set_task_comm(tsk, "kthreadd"); 699 ignore_signals(tsk); 700 set_cpus_allowed_ptr(tsk, housekeeping_cpumask(HK_FLAG_KTHREAD)); 701 set_mems_allowed(node_states[N_MEMORY]); 702 703 current->flags |= PF_NOFREEZE; 704 cgroup_init_kthreadd(); 705 706 for (;;) { 707 set_current_state(TASK_INTERRUPTIBLE); 708 if (list_empty(&kthread_create_list)) 709 schedule(); 710 __set_current_state(TASK_RUNNING); 711 712 spin_lock(&kthread_create_lock); 713 while (!list_empty(&kthread_create_list)) { 714 struct kthread_create_info *create; 715 716 create = list_entry(kthread_create_list.next, 717 struct kthread_create_info, list); 718 list_del_init(&create->list); 719 spin_unlock(&kthread_create_lock); 720 721 create_kthread(create); 722 723 spin_lock(&kthread_create_lock); 724 } 725 spin_unlock(&kthread_create_lock); 726 } 727 728 return 0; 729 } 730 731 void __kthread_init_worker(struct kthread_worker *worker, 732 const char *name, 733 struct lock_class_key *key) 734 { 735 memset(worker, 0, sizeof(struct kthread_worker)); 736 raw_spin_lock_init(&worker->lock); 737 lockdep_set_class_and_name(&worker->lock, key, name); 738 INIT_LIST_HEAD(&worker->work_list); 739 INIT_LIST_HEAD(&worker->delayed_work_list); 740 } 741 EXPORT_SYMBOL_GPL(__kthread_init_worker); 742 743 /** 744 * kthread_worker_fn - kthread function to process kthread_worker 745 * @worker_ptr: pointer to initialized kthread_worker 746 * 747 * This function implements the main cycle of kthread worker. It processes 748 * work_list until it is stopped with kthread_stop(). It sleeps when the queue 749 * is empty. 750 * 751 * The works are not allowed to keep any locks, disable preemption or interrupts 752 * when they finish. There is defined a safe point for freezing when one work 753 * finishes and before a new one is started. 754 * 755 * Also the works must not be handled by more than one worker at the same time, 756 * see also kthread_queue_work(). 757 */ 758 int kthread_worker_fn(void *worker_ptr) 759 { 760 struct kthread_worker *worker = worker_ptr; 761 struct kthread_work *work; 762 763 /* 764 * FIXME: Update the check and remove the assignment when all kthread 765 * worker users are created using kthread_create_worker*() functions. 766 */ 767 WARN_ON(worker->task && worker->task != current); 768 worker->task = current; 769 770 if (worker->flags & KTW_FREEZABLE) 771 set_freezable(); 772 773 repeat: 774 set_current_state(TASK_INTERRUPTIBLE); /* mb paired w/ kthread_stop */ 775 776 if (kthread_should_stop()) { 777 __set_current_state(TASK_RUNNING); 778 raw_spin_lock_irq(&worker->lock); 779 worker->task = NULL; 780 raw_spin_unlock_irq(&worker->lock); 781 return 0; 782 } 783 784 work = NULL; 785 raw_spin_lock_irq(&worker->lock); 786 if (!list_empty(&worker->work_list)) { 787 work = list_first_entry(&worker->work_list, 788 struct kthread_work, node); 789 list_del_init(&work->node); 790 } 791 worker->current_work = work; 792 raw_spin_unlock_irq(&worker->lock); 793 794 if (work) { 795 kthread_work_func_t func = work->func; 796 __set_current_state(TASK_RUNNING); 797 trace_sched_kthread_work_execute_start(work); 798 work->func(work); 799 /* 800 * Avoid dereferencing work after this point. The trace 801 * event only cares about the address. 802 */ 803 trace_sched_kthread_work_execute_end(work, func); 804 } else if (!freezing(current)) 805 schedule(); 806 807 try_to_freeze(); 808 cond_resched(); 809 goto repeat; 810 } 811 EXPORT_SYMBOL_GPL(kthread_worker_fn); 812 813 static __printf(3, 0) struct kthread_worker * 814 __kthread_create_worker(int cpu, unsigned int flags, 815 const char namefmt[], va_list args) 816 { 817 struct kthread_worker *worker; 818 struct task_struct *task; 819 int node = NUMA_NO_NODE; 820 821 worker = kzalloc(sizeof(*worker), GFP_KERNEL); 822 if (!worker) 823 return ERR_PTR(-ENOMEM); 824 825 kthread_init_worker(worker); 826 827 if (cpu >= 0) 828 node = cpu_to_node(cpu); 829 830 task = __kthread_create_on_node(kthread_worker_fn, worker, 831 node, namefmt, args); 832 if (IS_ERR(task)) 833 goto fail_task; 834 835 if (cpu >= 0) 836 kthread_bind(task, cpu); 837 838 worker->flags = flags; 839 worker->task = task; 840 wake_up_process(task); 841 return worker; 842 843 fail_task: 844 kfree(worker); 845 return ERR_CAST(task); 846 } 847 848 /** 849 * kthread_create_worker - create a kthread worker 850 * @flags: flags modifying the default behavior of the worker 851 * @namefmt: printf-style name for the kthread worker (task). 852 * 853 * Returns a pointer to the allocated worker on success, ERR_PTR(-ENOMEM) 854 * when the needed structures could not get allocated, and ERR_PTR(-EINTR) 855 * when the worker was SIGKILLed. 856 */ 857 struct kthread_worker * 858 kthread_create_worker(unsigned int flags, const char namefmt[], ...) 859 { 860 struct kthread_worker *worker; 861 va_list args; 862 863 va_start(args, namefmt); 864 worker = __kthread_create_worker(-1, flags, namefmt, args); 865 va_end(args); 866 867 return worker; 868 } 869 EXPORT_SYMBOL(kthread_create_worker); 870 871 /** 872 * kthread_create_worker_on_cpu - create a kthread worker and bind it 873 * to a given CPU and the associated NUMA node. 874 * @cpu: CPU number 875 * @flags: flags modifying the default behavior of the worker 876 * @namefmt: printf-style name for the kthread worker (task). 877 * 878 * Use a valid CPU number if you want to bind the kthread worker 879 * to the given CPU and the associated NUMA node. 880 * 881 * A good practice is to add the cpu number also into the worker name. 882 * For example, use kthread_create_worker_on_cpu(cpu, "helper/%d", cpu). 883 * 884 * CPU hotplug: 885 * The kthread worker API is simple and generic. It just provides a way 886 * to create, use, and destroy workers. 887 * 888 * It is up to the API user how to handle CPU hotplug. They have to decide 889 * how to handle pending work items, prevent queuing new ones, and 890 * restore the functionality when the CPU goes off and on. There are a 891 * few catches: 892 * 893 * - CPU affinity gets lost when it is scheduled on an offline CPU. 894 * 895 * - The worker might not exist when the CPU was off when the user 896 * created the workers. 897 * 898 * Good practice is to implement two CPU hotplug callbacks and to 899 * destroy/create the worker when the CPU goes down/up. 900 * 901 * Return: 902 * The pointer to the allocated worker on success, ERR_PTR(-ENOMEM) 903 * when the needed structures could not get allocated, and ERR_PTR(-EINTR) 904 * when the worker was SIGKILLed. 905 */ 906 struct kthread_worker * 907 kthread_create_worker_on_cpu(int cpu, unsigned int flags, 908 const char namefmt[], ...) 909 { 910 struct kthread_worker *worker; 911 va_list args; 912 913 va_start(args, namefmt); 914 worker = __kthread_create_worker(cpu, flags, namefmt, args); 915 va_end(args); 916 917 return worker; 918 } 919 EXPORT_SYMBOL(kthread_create_worker_on_cpu); 920 921 /* 922 * Returns true when the work could not be queued at the moment. 923 * It happens when it is already pending in a worker list 924 * or when it is being cancelled. 925 */ 926 static inline bool queuing_blocked(struct kthread_worker *worker, 927 struct kthread_work *work) 928 { 929 lockdep_assert_held(&worker->lock); 930 931 return !list_empty(&work->node) || work->canceling; 932 } 933 934 static void kthread_insert_work_sanity_check(struct kthread_worker *worker, 935 struct kthread_work *work) 936 { 937 lockdep_assert_held(&worker->lock); 938 WARN_ON_ONCE(!list_empty(&work->node)); 939 /* Do not use a work with >1 worker, see kthread_queue_work() */ 940 WARN_ON_ONCE(work->worker && work->worker != worker); 941 } 942 943 /* insert @work before @pos in @worker */ 944 static void kthread_insert_work(struct kthread_worker *worker, 945 struct kthread_work *work, 946 struct list_head *pos) 947 { 948 kthread_insert_work_sanity_check(worker, work); 949 950 trace_sched_kthread_work_queue_work(worker, work); 951 952 list_add_tail(&work->node, pos); 953 work->worker = worker; 954 if (!worker->current_work && likely(worker->task)) 955 wake_up_process(worker->task); 956 } 957 958 /** 959 * kthread_queue_work - queue a kthread_work 960 * @worker: target kthread_worker 961 * @work: kthread_work to queue 962 * 963 * Queue @work to work processor @task for async execution. @task 964 * must have been created with kthread_worker_create(). Returns %true 965 * if @work was successfully queued, %false if it was already pending. 966 * 967 * Reinitialize the work if it needs to be used by another worker. 968 * For example, when the worker was stopped and started again. 969 */ 970 bool kthread_queue_work(struct kthread_worker *worker, 971 struct kthread_work *work) 972 { 973 bool ret = false; 974 unsigned long flags; 975 976 raw_spin_lock_irqsave(&worker->lock, flags); 977 if (!queuing_blocked(worker, work)) { 978 kthread_insert_work(worker, work, &worker->work_list); 979 ret = true; 980 } 981 raw_spin_unlock_irqrestore(&worker->lock, flags); 982 return ret; 983 } 984 EXPORT_SYMBOL_GPL(kthread_queue_work); 985 986 /** 987 * kthread_delayed_work_timer_fn - callback that queues the associated kthread 988 * delayed work when the timer expires. 989 * @t: pointer to the expired timer 990 * 991 * The format of the function is defined by struct timer_list. 992 * It should have been called from irqsafe timer with irq already off. 993 */ 994 void kthread_delayed_work_timer_fn(struct timer_list *t) 995 { 996 struct kthread_delayed_work *dwork = from_timer(dwork, t, timer); 997 struct kthread_work *work = &dwork->work; 998 struct kthread_worker *worker = work->worker; 999 unsigned long flags; 1000 1001 /* 1002 * This might happen when a pending work is reinitialized. 1003 * It means that it is used a wrong way. 1004 */ 1005 if (WARN_ON_ONCE(!worker)) 1006 return; 1007 1008 raw_spin_lock_irqsave(&worker->lock, flags); 1009 /* Work must not be used with >1 worker, see kthread_queue_work(). */ 1010 WARN_ON_ONCE(work->worker != worker); 1011 1012 /* Move the work from worker->delayed_work_list. */ 1013 WARN_ON_ONCE(list_empty(&work->node)); 1014 list_del_init(&work->node); 1015 if (!work->canceling) 1016 kthread_insert_work(worker, work, &worker->work_list); 1017 1018 raw_spin_unlock_irqrestore(&worker->lock, flags); 1019 } 1020 EXPORT_SYMBOL(kthread_delayed_work_timer_fn); 1021 1022 static void __kthread_queue_delayed_work(struct kthread_worker *worker, 1023 struct kthread_delayed_work *dwork, 1024 unsigned long delay) 1025 { 1026 struct timer_list *timer = &dwork->timer; 1027 struct kthread_work *work = &dwork->work; 1028 1029 WARN_ON_FUNCTION_MISMATCH(timer->function, 1030 kthread_delayed_work_timer_fn); 1031 1032 /* 1033 * If @delay is 0, queue @dwork->work immediately. This is for 1034 * both optimization and correctness. The earliest @timer can 1035 * expire is on the closest next tick and delayed_work users depend 1036 * on that there's no such delay when @delay is 0. 1037 */ 1038 if (!delay) { 1039 kthread_insert_work(worker, work, &worker->work_list); 1040 return; 1041 } 1042 1043 /* Be paranoid and try to detect possible races already now. */ 1044 kthread_insert_work_sanity_check(worker, work); 1045 1046 list_add(&work->node, &worker->delayed_work_list); 1047 work->worker = worker; 1048 timer->expires = jiffies + delay; 1049 add_timer(timer); 1050 } 1051 1052 /** 1053 * kthread_queue_delayed_work - queue the associated kthread work 1054 * after a delay. 1055 * @worker: target kthread_worker 1056 * @dwork: kthread_delayed_work to queue 1057 * @delay: number of jiffies to wait before queuing 1058 * 1059 * If the work has not been pending it starts a timer that will queue 1060 * the work after the given @delay. If @delay is zero, it queues the 1061 * work immediately. 1062 * 1063 * Return: %false if the @work has already been pending. It means that 1064 * either the timer was running or the work was queued. It returns %true 1065 * otherwise. 1066 */ 1067 bool kthread_queue_delayed_work(struct kthread_worker *worker, 1068 struct kthread_delayed_work *dwork, 1069 unsigned long delay) 1070 { 1071 struct kthread_work *work = &dwork->work; 1072 unsigned long flags; 1073 bool ret = false; 1074 1075 raw_spin_lock_irqsave(&worker->lock, flags); 1076 1077 if (!queuing_blocked(worker, work)) { 1078 __kthread_queue_delayed_work(worker, dwork, delay); 1079 ret = true; 1080 } 1081 1082 raw_spin_unlock_irqrestore(&worker->lock, flags); 1083 return ret; 1084 } 1085 EXPORT_SYMBOL_GPL(kthread_queue_delayed_work); 1086 1087 struct kthread_flush_work { 1088 struct kthread_work work; 1089 struct completion done; 1090 }; 1091 1092 static void kthread_flush_work_fn(struct kthread_work *work) 1093 { 1094 struct kthread_flush_work *fwork = 1095 container_of(work, struct kthread_flush_work, work); 1096 complete(&fwork->done); 1097 } 1098 1099 /** 1100 * kthread_flush_work - flush a kthread_work 1101 * @work: work to flush 1102 * 1103 * If @work is queued or executing, wait for it to finish execution. 1104 */ 1105 void kthread_flush_work(struct kthread_work *work) 1106 { 1107 struct kthread_flush_work fwork = { 1108 KTHREAD_WORK_INIT(fwork.work, kthread_flush_work_fn), 1109 COMPLETION_INITIALIZER_ONSTACK(fwork.done), 1110 }; 1111 struct kthread_worker *worker; 1112 bool noop = false; 1113 1114 worker = work->worker; 1115 if (!worker) 1116 return; 1117 1118 raw_spin_lock_irq(&worker->lock); 1119 /* Work must not be used with >1 worker, see kthread_queue_work(). */ 1120 WARN_ON_ONCE(work->worker != worker); 1121 1122 if (!list_empty(&work->node)) 1123 kthread_insert_work(worker, &fwork.work, work->node.next); 1124 else if (worker->current_work == work) 1125 kthread_insert_work(worker, &fwork.work, 1126 worker->work_list.next); 1127 else 1128 noop = true; 1129 1130 raw_spin_unlock_irq(&worker->lock); 1131 1132 if (!noop) 1133 wait_for_completion(&fwork.done); 1134 } 1135 EXPORT_SYMBOL_GPL(kthread_flush_work); 1136 1137 /* 1138 * Make sure that the timer is neither set nor running and could 1139 * not manipulate the work list_head any longer. 1140 * 1141 * The function is called under worker->lock. The lock is temporary 1142 * released but the timer can't be set again in the meantime. 1143 */ 1144 static void kthread_cancel_delayed_work_timer(struct kthread_work *work, 1145 unsigned long *flags) 1146 { 1147 struct kthread_delayed_work *dwork = 1148 container_of(work, struct kthread_delayed_work, work); 1149 struct kthread_worker *worker = work->worker; 1150 1151 /* 1152 * del_timer_sync() must be called to make sure that the timer 1153 * callback is not running. The lock must be temporary released 1154 * to avoid a deadlock with the callback. In the meantime, 1155 * any queuing is blocked by setting the canceling counter. 1156 */ 1157 work->canceling++; 1158 raw_spin_unlock_irqrestore(&worker->lock, *flags); 1159 del_timer_sync(&dwork->timer); 1160 raw_spin_lock_irqsave(&worker->lock, *flags); 1161 work->canceling--; 1162 } 1163 1164 /* 1165 * This function removes the work from the worker queue. 1166 * 1167 * It is called under worker->lock. The caller must make sure that 1168 * the timer used by delayed work is not running, e.g. by calling 1169 * kthread_cancel_delayed_work_timer(). 1170 * 1171 * The work might still be in use when this function finishes. See the 1172 * current_work proceed by the worker. 1173 * 1174 * Return: %true if @work was pending and successfully canceled, 1175 * %false if @work was not pending 1176 */ 1177 static bool __kthread_cancel_work(struct kthread_work *work) 1178 { 1179 /* 1180 * Try to remove the work from a worker list. It might either 1181 * be from worker->work_list or from worker->delayed_work_list. 1182 */ 1183 if (!list_empty(&work->node)) { 1184 list_del_init(&work->node); 1185 return true; 1186 } 1187 1188 return false; 1189 } 1190 1191 /** 1192 * kthread_mod_delayed_work - modify delay of or queue a kthread delayed work 1193 * @worker: kthread worker to use 1194 * @dwork: kthread delayed work to queue 1195 * @delay: number of jiffies to wait before queuing 1196 * 1197 * If @dwork is idle, equivalent to kthread_queue_delayed_work(). Otherwise, 1198 * modify @dwork's timer so that it expires after @delay. If @delay is zero, 1199 * @work is guaranteed to be queued immediately. 1200 * 1201 * Return: %false if @dwork was idle and queued, %true otherwise. 1202 * 1203 * A special case is when the work is being canceled in parallel. 1204 * It might be caused either by the real kthread_cancel_delayed_work_sync() 1205 * or yet another kthread_mod_delayed_work() call. We let the other command 1206 * win and return %true here. The return value can be used for reference 1207 * counting and the number of queued works stays the same. Anyway, the caller 1208 * is supposed to synchronize these operations a reasonable way. 1209 * 1210 * This function is safe to call from any context including IRQ handler. 1211 * See __kthread_cancel_work() and kthread_delayed_work_timer_fn() 1212 * for details. 1213 */ 1214 bool kthread_mod_delayed_work(struct kthread_worker *worker, 1215 struct kthread_delayed_work *dwork, 1216 unsigned long delay) 1217 { 1218 struct kthread_work *work = &dwork->work; 1219 unsigned long flags; 1220 int ret; 1221 1222 raw_spin_lock_irqsave(&worker->lock, flags); 1223 1224 /* Do not bother with canceling when never queued. */ 1225 if (!work->worker) { 1226 ret = false; 1227 goto fast_queue; 1228 } 1229 1230 /* Work must not be used with >1 worker, see kthread_queue_work() */ 1231 WARN_ON_ONCE(work->worker != worker); 1232 1233 /* 1234 * Temporary cancel the work but do not fight with another command 1235 * that is canceling the work as well. 1236 * 1237 * It is a bit tricky because of possible races with another 1238 * mod_delayed_work() and cancel_delayed_work() callers. 1239 * 1240 * The timer must be canceled first because worker->lock is released 1241 * when doing so. But the work can be removed from the queue (list) 1242 * only when it can be queued again so that the return value can 1243 * be used for reference counting. 1244 */ 1245 kthread_cancel_delayed_work_timer(work, &flags); 1246 if (work->canceling) { 1247 /* The number of works in the queue does not change. */ 1248 ret = true; 1249 goto out; 1250 } 1251 ret = __kthread_cancel_work(work); 1252 1253 fast_queue: 1254 __kthread_queue_delayed_work(worker, dwork, delay); 1255 out: 1256 raw_spin_unlock_irqrestore(&worker->lock, flags); 1257 return ret; 1258 } 1259 EXPORT_SYMBOL_GPL(kthread_mod_delayed_work); 1260 1261 static bool __kthread_cancel_work_sync(struct kthread_work *work, bool is_dwork) 1262 { 1263 struct kthread_worker *worker = work->worker; 1264 unsigned long flags; 1265 int ret = false; 1266 1267 if (!worker) 1268 goto out; 1269 1270 raw_spin_lock_irqsave(&worker->lock, flags); 1271 /* Work must not be used with >1 worker, see kthread_queue_work(). */ 1272 WARN_ON_ONCE(work->worker != worker); 1273 1274 if (is_dwork) 1275 kthread_cancel_delayed_work_timer(work, &flags); 1276 1277 ret = __kthread_cancel_work(work); 1278 1279 if (worker->current_work != work) 1280 goto out_fast; 1281 1282 /* 1283 * The work is in progress and we need to wait with the lock released. 1284 * In the meantime, block any queuing by setting the canceling counter. 1285 */ 1286 work->canceling++; 1287 raw_spin_unlock_irqrestore(&worker->lock, flags); 1288 kthread_flush_work(work); 1289 raw_spin_lock_irqsave(&worker->lock, flags); 1290 work->canceling--; 1291 1292 out_fast: 1293 raw_spin_unlock_irqrestore(&worker->lock, flags); 1294 out: 1295 return ret; 1296 } 1297 1298 /** 1299 * kthread_cancel_work_sync - cancel a kthread work and wait for it to finish 1300 * @work: the kthread work to cancel 1301 * 1302 * Cancel @work and wait for its execution to finish. This function 1303 * can be used even if the work re-queues itself. On return from this 1304 * function, @work is guaranteed to be not pending or executing on any CPU. 1305 * 1306 * kthread_cancel_work_sync(&delayed_work->work) must not be used for 1307 * delayed_work's. Use kthread_cancel_delayed_work_sync() instead. 1308 * 1309 * The caller must ensure that the worker on which @work was last 1310 * queued can't be destroyed before this function returns. 1311 * 1312 * Return: %true if @work was pending, %false otherwise. 1313 */ 1314 bool kthread_cancel_work_sync(struct kthread_work *work) 1315 { 1316 return __kthread_cancel_work_sync(work, false); 1317 } 1318 EXPORT_SYMBOL_GPL(kthread_cancel_work_sync); 1319 1320 /** 1321 * kthread_cancel_delayed_work_sync - cancel a kthread delayed work and 1322 * wait for it to finish. 1323 * @dwork: the kthread delayed work to cancel 1324 * 1325 * This is kthread_cancel_work_sync() for delayed works. 1326 * 1327 * Return: %true if @dwork was pending, %false otherwise. 1328 */ 1329 bool kthread_cancel_delayed_work_sync(struct kthread_delayed_work *dwork) 1330 { 1331 return __kthread_cancel_work_sync(&dwork->work, true); 1332 } 1333 EXPORT_SYMBOL_GPL(kthread_cancel_delayed_work_sync); 1334 1335 /** 1336 * kthread_flush_worker - flush all current works on a kthread_worker 1337 * @worker: worker to flush 1338 * 1339 * Wait until all currently executing or pending works on @worker are 1340 * finished. 1341 */ 1342 void kthread_flush_worker(struct kthread_worker *worker) 1343 { 1344 struct kthread_flush_work fwork = { 1345 KTHREAD_WORK_INIT(fwork.work, kthread_flush_work_fn), 1346 COMPLETION_INITIALIZER_ONSTACK(fwork.done), 1347 }; 1348 1349 kthread_queue_work(worker, &fwork.work); 1350 wait_for_completion(&fwork.done); 1351 } 1352 EXPORT_SYMBOL_GPL(kthread_flush_worker); 1353 1354 /** 1355 * kthread_destroy_worker - destroy a kthread worker 1356 * @worker: worker to be destroyed 1357 * 1358 * Flush and destroy @worker. The simple flush is enough because the kthread 1359 * worker API is used only in trivial scenarios. There are no multi-step state 1360 * machines needed. 1361 */ 1362 void kthread_destroy_worker(struct kthread_worker *worker) 1363 { 1364 struct task_struct *task; 1365 1366 task = worker->task; 1367 if (WARN_ON(!task)) 1368 return; 1369 1370 kthread_flush_worker(worker); 1371 kthread_stop(task); 1372 WARN_ON(!list_empty(&worker->work_list)); 1373 kfree(worker); 1374 } 1375 EXPORT_SYMBOL(kthread_destroy_worker); 1376 1377 /** 1378 * kthread_use_mm - make the calling kthread operate on an address space 1379 * @mm: address space to operate on 1380 */ 1381 void kthread_use_mm(struct mm_struct *mm) 1382 { 1383 struct mm_struct *active_mm; 1384 struct task_struct *tsk = current; 1385 1386 WARN_ON_ONCE(!(tsk->flags & PF_KTHREAD)); 1387 WARN_ON_ONCE(tsk->mm); 1388 1389 task_lock(tsk); 1390 /* Hold off tlb flush IPIs while switching mm's */ 1391 local_irq_disable(); 1392 active_mm = tsk->active_mm; 1393 if (active_mm != mm) { 1394 mmgrab(mm); 1395 tsk->active_mm = mm; 1396 } 1397 tsk->mm = mm; 1398 membarrier_update_current_mm(mm); 1399 switch_mm_irqs_off(active_mm, mm, tsk); 1400 local_irq_enable(); 1401 task_unlock(tsk); 1402 #ifdef finish_arch_post_lock_switch 1403 finish_arch_post_lock_switch(); 1404 #endif 1405 1406 /* 1407 * When a kthread starts operating on an address space, the loop 1408 * in membarrier_{private,global}_expedited() may not observe 1409 * that tsk->mm, and not issue an IPI. Membarrier requires a 1410 * memory barrier after storing to tsk->mm, before accessing 1411 * user-space memory. A full memory barrier for membarrier 1412 * {PRIVATE,GLOBAL}_EXPEDITED is implicitly provided by 1413 * mmdrop(), or explicitly with smp_mb(). 1414 */ 1415 if (active_mm != mm) 1416 mmdrop(active_mm); 1417 else 1418 smp_mb(); 1419 1420 to_kthread(tsk)->oldfs = force_uaccess_begin(); 1421 } 1422 EXPORT_SYMBOL_GPL(kthread_use_mm); 1423 1424 /** 1425 * kthread_unuse_mm - reverse the effect of kthread_use_mm() 1426 * @mm: address space to operate on 1427 */ 1428 void kthread_unuse_mm(struct mm_struct *mm) 1429 { 1430 struct task_struct *tsk = current; 1431 1432 WARN_ON_ONCE(!(tsk->flags & PF_KTHREAD)); 1433 WARN_ON_ONCE(!tsk->mm); 1434 1435 force_uaccess_end(to_kthread(tsk)->oldfs); 1436 1437 task_lock(tsk); 1438 /* 1439 * When a kthread stops operating on an address space, the loop 1440 * in membarrier_{private,global}_expedited() may not observe 1441 * that tsk->mm, and not issue an IPI. Membarrier requires a 1442 * memory barrier after accessing user-space memory, before 1443 * clearing tsk->mm. 1444 */ 1445 smp_mb__after_spinlock(); 1446 sync_mm_rss(mm); 1447 local_irq_disable(); 1448 tsk->mm = NULL; 1449 membarrier_update_current_mm(NULL); 1450 /* active_mm is still 'mm' */ 1451 enter_lazy_tlb(mm, tsk); 1452 local_irq_enable(); 1453 task_unlock(tsk); 1454 } 1455 EXPORT_SYMBOL_GPL(kthread_unuse_mm); 1456 1457 #ifdef CONFIG_BLK_CGROUP 1458 /** 1459 * kthread_associate_blkcg - associate blkcg to current kthread 1460 * @css: the cgroup info 1461 * 1462 * Current thread must be a kthread. The thread is running jobs on behalf of 1463 * other threads. In some cases, we expect the jobs attach cgroup info of 1464 * original threads instead of that of current thread. This function stores 1465 * original thread's cgroup info in current kthread context for later 1466 * retrieval. 1467 */ 1468 void kthread_associate_blkcg(struct cgroup_subsys_state *css) 1469 { 1470 struct kthread *kthread; 1471 1472 if (!(current->flags & PF_KTHREAD)) 1473 return; 1474 kthread = to_kthread(current); 1475 if (!kthread) 1476 return; 1477 1478 if (kthread->blkcg_css) { 1479 css_put(kthread->blkcg_css); 1480 kthread->blkcg_css = NULL; 1481 } 1482 if (css) { 1483 css_get(css); 1484 kthread->blkcg_css = css; 1485 } 1486 } 1487 EXPORT_SYMBOL(kthread_associate_blkcg); 1488 1489 /** 1490 * kthread_blkcg - get associated blkcg css of current kthread 1491 * 1492 * Current thread must be a kthread. 1493 */ 1494 struct cgroup_subsys_state *kthread_blkcg(void) 1495 { 1496 struct kthread *kthread; 1497 1498 if (current->flags & PF_KTHREAD) { 1499 kthread = to_kthread(current); 1500 if (kthread) 1501 return kthread->blkcg_css; 1502 } 1503 return NULL; 1504 } 1505 EXPORT_SYMBOL(kthread_blkcg); 1506 #endif 1507