1e73f8959SOleg Nesterov #include <linux/spinlock.h> 2e73f8959SOleg Nesterov #include <linux/task_work.h> 3e73f8959SOleg Nesterov #include <linux/tracehook.h> 4e73f8959SOleg Nesterov 59da33de6SOleg Nesterov static struct callback_head work_exited; /* all we need is ->next == NULL */ 69da33de6SOleg Nesterov 7892f6668SOleg Nesterov /** 8892f6668SOleg Nesterov * task_work_add - ask the @task to execute @work->func() 9892f6668SOleg Nesterov * @task: the task which should run the callback 10892f6668SOleg Nesterov * @work: the callback to run 11892f6668SOleg Nesterov * @notify: send the notification if true 12892f6668SOleg Nesterov * 13892f6668SOleg Nesterov * Queue @work for task_work_run() below and notify the @task if @notify. 14892f6668SOleg Nesterov * Fails if the @task is exiting/exited and thus it can't process this @work. 15892f6668SOleg Nesterov * Otherwise @work->func() will be called when the @task returns from kernel 16892f6668SOleg Nesterov * mode or exits. 17892f6668SOleg Nesterov * 18892f6668SOleg Nesterov * This is like the signal handler which runs in kernel mode, but it doesn't 19892f6668SOleg Nesterov * try to wake up the @task. 20892f6668SOleg Nesterov * 21c8219906SEric Dumazet * Note: there is no ordering guarantee on works queued here. 22c8219906SEric Dumazet * 23892f6668SOleg Nesterov * RETURNS: 24892f6668SOleg Nesterov * 0 if succeeds or -ESRCH. 25892f6668SOleg Nesterov */ 26e73f8959SOleg Nesterov int 27ac3d0da8SOleg Nesterov task_work_add(struct task_struct *task, struct callback_head *work, bool notify) 28e73f8959SOleg Nesterov { 29ac3d0da8SOleg Nesterov struct callback_head *head; 309da33de6SOleg Nesterov 31ac3d0da8SOleg Nesterov do { 3261e96496SOleg Nesterov head = READ_ONCE(task->task_works); 339da33de6SOleg Nesterov if (unlikely(head == &work_exited)) 349da33de6SOleg Nesterov return -ESRCH; 35ac3d0da8SOleg Nesterov work->next = head; 36ac3d0da8SOleg Nesterov } while (cmpxchg(&task->task_works, head, work) != head); 37e73f8959SOleg Nesterov 38ed3e694dSAl Viro if (notify) 39e73f8959SOleg Nesterov set_notify_resume(task); 40ed3e694dSAl Viro return 0; 41e73f8959SOleg Nesterov } 42e73f8959SOleg Nesterov 43892f6668SOleg Nesterov /** 44892f6668SOleg Nesterov * task_work_cancel - cancel a pending work added by task_work_add() 45892f6668SOleg Nesterov * @task: the task which should execute the work 46892f6668SOleg Nesterov * @func: identifies the work to remove 47892f6668SOleg Nesterov * 48892f6668SOleg Nesterov * Find the last queued pending work with ->func == @func and remove 49892f6668SOleg Nesterov * it from queue. 50892f6668SOleg Nesterov * 51892f6668SOleg Nesterov * RETURNS: 52892f6668SOleg Nesterov * The found work or NULL if not found. 53892f6668SOleg Nesterov */ 5467d12145SAl Viro struct callback_head * 55e73f8959SOleg Nesterov task_work_cancel(struct task_struct *task, task_work_func_t func) 56e73f8959SOleg Nesterov { 57ac3d0da8SOleg Nesterov struct callback_head **pprev = &task->task_works; 58205e550aSOleg Nesterov struct callback_head *work; 59e73f8959SOleg Nesterov unsigned long flags; 6061e96496SOleg Nesterov 6161e96496SOleg Nesterov if (likely(!task->task_works)) 6261e96496SOleg Nesterov return NULL; 63ac3d0da8SOleg Nesterov /* 64ac3d0da8SOleg Nesterov * If cmpxchg() fails we continue without updating pprev. 65ac3d0da8SOleg Nesterov * Either we raced with task_work_add() which added the 66ac3d0da8SOleg Nesterov * new entry before this work, we will find it again. Or 679da33de6SOleg Nesterov * we raced with task_work_run(), *pprev == NULL/exited. 68ac3d0da8SOleg Nesterov */ 69e73f8959SOleg Nesterov raw_spin_lock_irqsave(&task->pi_lock, flags); 7061e96496SOleg Nesterov while ((work = lockless_dereference(*pprev))) { 71ac3d0da8SOleg Nesterov if (work->func != func) 72ac3d0da8SOleg Nesterov pprev = &work->next; 73ac3d0da8SOleg Nesterov else if (cmpxchg(pprev, work, work->next) == work) 74158e1645SAl Viro break; 75158e1645SAl Viro } 76e73f8959SOleg Nesterov raw_spin_unlock_irqrestore(&task->pi_lock, flags); 77ac3d0da8SOleg Nesterov 78ac3d0da8SOleg Nesterov return work; 79e73f8959SOleg Nesterov } 80e73f8959SOleg Nesterov 81892f6668SOleg Nesterov /** 82892f6668SOleg Nesterov * task_work_run - execute the works added by task_work_add() 83892f6668SOleg Nesterov * 84892f6668SOleg Nesterov * Flush the pending works. Should be used by the core kernel code. 85892f6668SOleg Nesterov * Called before the task returns to the user-mode or stops, or when 86892f6668SOleg Nesterov * it exits. In the latter case task_work_add() can no longer add the 87892f6668SOleg Nesterov * new work after task_work_run() returns. 88892f6668SOleg Nesterov */ 89e73f8959SOleg Nesterov void task_work_run(void) 90e73f8959SOleg Nesterov { 91e73f8959SOleg Nesterov struct task_struct *task = current; 92ac3d0da8SOleg Nesterov struct callback_head *work, *head, *next; 93e73f8959SOleg Nesterov 94ac3d0da8SOleg Nesterov for (;;) { 959da33de6SOleg Nesterov /* 969da33de6SOleg Nesterov * work->func() can do task_work_add(), do not set 979da33de6SOleg Nesterov * work_exited unless the list is empty. 989da33de6SOleg Nesterov */ 999da33de6SOleg Nesterov do { 10061e96496SOleg Nesterov work = READ_ONCE(task->task_works); 1019da33de6SOleg Nesterov head = !work && (task->flags & PF_EXITING) ? 1029da33de6SOleg Nesterov &work_exited : NULL; 1039da33de6SOleg Nesterov } while (cmpxchg(&task->task_works, work, head) != work); 1049da33de6SOleg Nesterov 105ac3d0da8SOleg Nesterov if (!work) 106ac3d0da8SOleg Nesterov break; 107ac3d0da8SOleg Nesterov /* 108ac3d0da8SOleg Nesterov * Synchronize with task_work_cancel(). It can't remove 109ac3d0da8SOleg Nesterov * the first entry == work, cmpxchg(task_works) should 110ac3d0da8SOleg Nesterov * fail, but it can play with *work and other entries. 111ac3d0da8SOleg Nesterov */ 112ac3d0da8SOleg Nesterov raw_spin_unlock_wait(&task->pi_lock); 113e73f8959SOleg Nesterov 114ac3d0da8SOleg Nesterov do { 115ac3d0da8SOleg Nesterov next = work->next; 116ac3d0da8SOleg Nesterov work->func(work); 117ac3d0da8SOleg Nesterov work = next; 118f341861fSEric Dumazet cond_resched(); 119ac3d0da8SOleg Nesterov } while (work); 120e73f8959SOleg Nesterov } 121a2d4c71dSAl Viro } 122