1e73f8959SOleg Nesterov #include <linux/spinlock.h> 2e73f8959SOleg Nesterov #include <linux/task_work.h> 3e73f8959SOleg Nesterov #include <linux/tracehook.h> 4e73f8959SOleg Nesterov 59da33de6SOleg Nesterov static struct callback_head work_exited; /* all we need is ->next == NULL */ 69da33de6SOleg Nesterov 7892f6668SOleg Nesterov /** 8892f6668SOleg Nesterov * task_work_add - ask the @task to execute @work->func() 9892f6668SOleg Nesterov * @task: the task which should run the callback 10892f6668SOleg Nesterov * @work: the callback to run 11892f6668SOleg Nesterov * @notify: send the notification if true 12892f6668SOleg Nesterov * 13892f6668SOleg Nesterov * Queue @work for task_work_run() below and notify the @task if @notify. 14892f6668SOleg Nesterov * Fails if the @task is exiting/exited and thus it can't process this @work. 15892f6668SOleg Nesterov * Otherwise @work->func() will be called when the @task returns from kernel 16892f6668SOleg Nesterov * mode or exits. 17892f6668SOleg Nesterov * 18892f6668SOleg Nesterov * This is like the signal handler which runs in kernel mode, but it doesn't 19892f6668SOleg Nesterov * try to wake up the @task. 20892f6668SOleg Nesterov * 21892f6668SOleg Nesterov * RETURNS: 22892f6668SOleg Nesterov * 0 if succeeds or -ESRCH. 23892f6668SOleg Nesterov */ 24e73f8959SOleg Nesterov int 25ac3d0da8SOleg Nesterov task_work_add(struct task_struct *task, struct callback_head *work, bool notify) 26e73f8959SOleg Nesterov { 27ac3d0da8SOleg Nesterov struct callback_head *head; 289da33de6SOleg Nesterov 29ac3d0da8SOleg Nesterov do { 30ac3d0da8SOleg Nesterov head = ACCESS_ONCE(task->task_works); 319da33de6SOleg Nesterov if (unlikely(head == &work_exited)) 329da33de6SOleg Nesterov return -ESRCH; 33ac3d0da8SOleg Nesterov work->next = head; 34ac3d0da8SOleg Nesterov } while (cmpxchg(&task->task_works, head, work) != head); 35e73f8959SOleg Nesterov 36ed3e694dSAl Viro if (notify) 37e73f8959SOleg Nesterov set_notify_resume(task); 38ed3e694dSAl Viro return 0; 39e73f8959SOleg Nesterov } 40e73f8959SOleg Nesterov 41892f6668SOleg Nesterov /** 42892f6668SOleg Nesterov * task_work_cancel - cancel a pending work added by task_work_add() 43892f6668SOleg Nesterov * @task: the task which should execute the work 44892f6668SOleg Nesterov * @func: identifies the work to remove 45892f6668SOleg Nesterov * 46892f6668SOleg Nesterov * Find the last queued pending work with ->func == @func and remove 47892f6668SOleg Nesterov * it from queue. 48892f6668SOleg Nesterov * 49892f6668SOleg Nesterov * RETURNS: 50892f6668SOleg Nesterov * The found work or NULL if not found. 51892f6668SOleg Nesterov */ 5267d12145SAl Viro struct callback_head * 53e73f8959SOleg Nesterov task_work_cancel(struct task_struct *task, task_work_func_t func) 54e73f8959SOleg Nesterov { 55ac3d0da8SOleg Nesterov struct callback_head **pprev = &task->task_works; 56205e550aSOleg Nesterov struct callback_head *work; 57e73f8959SOleg Nesterov unsigned long flags; 58ac3d0da8SOleg Nesterov /* 59ac3d0da8SOleg Nesterov * If cmpxchg() fails we continue without updating pprev. 60ac3d0da8SOleg Nesterov * Either we raced with task_work_add() which added the 61ac3d0da8SOleg Nesterov * new entry before this work, we will find it again. Or 629da33de6SOleg Nesterov * we raced with task_work_run(), *pprev == NULL/exited. 63ac3d0da8SOleg Nesterov */ 64e73f8959SOleg Nesterov raw_spin_lock_irqsave(&task->pi_lock, flags); 65ac3d0da8SOleg Nesterov while ((work = ACCESS_ONCE(*pprev))) { 66205e550aSOleg Nesterov smp_read_barrier_depends(); 67ac3d0da8SOleg Nesterov if (work->func != func) 68ac3d0da8SOleg Nesterov pprev = &work->next; 69ac3d0da8SOleg Nesterov else if (cmpxchg(pprev, work, work->next) == work) 70158e1645SAl Viro break; 71158e1645SAl Viro } 72e73f8959SOleg Nesterov raw_spin_unlock_irqrestore(&task->pi_lock, flags); 73ac3d0da8SOleg Nesterov 74ac3d0da8SOleg Nesterov return work; 75e73f8959SOleg Nesterov } 76e73f8959SOleg Nesterov 77892f6668SOleg Nesterov /** 78892f6668SOleg Nesterov * task_work_run - execute the works added by task_work_add() 79892f6668SOleg Nesterov * 80892f6668SOleg Nesterov * Flush the pending works. Should be used by the core kernel code. 81892f6668SOleg Nesterov * Called before the task returns to the user-mode or stops, or when 82892f6668SOleg Nesterov * it exits. In the latter case task_work_add() can no longer add the 83892f6668SOleg Nesterov * new work after task_work_run() returns. 84892f6668SOleg Nesterov */ 85e73f8959SOleg Nesterov void task_work_run(void) 86e73f8959SOleg Nesterov { 87e73f8959SOleg Nesterov struct task_struct *task = current; 88ac3d0da8SOleg Nesterov struct callback_head *work, *head, *next; 89e73f8959SOleg Nesterov 90ac3d0da8SOleg Nesterov for (;;) { 919da33de6SOleg Nesterov /* 929da33de6SOleg Nesterov * work->func() can do task_work_add(), do not set 939da33de6SOleg Nesterov * work_exited unless the list is empty. 949da33de6SOleg Nesterov */ 959da33de6SOleg Nesterov do { 969da33de6SOleg Nesterov work = ACCESS_ONCE(task->task_works); 979da33de6SOleg Nesterov head = !work && (task->flags & PF_EXITING) ? 989da33de6SOleg Nesterov &work_exited : NULL; 999da33de6SOleg Nesterov } while (cmpxchg(&task->task_works, work, head) != work); 1009da33de6SOleg Nesterov 101ac3d0da8SOleg Nesterov if (!work) 102ac3d0da8SOleg Nesterov break; 103ac3d0da8SOleg Nesterov /* 104ac3d0da8SOleg Nesterov * Synchronize with task_work_cancel(). It can't remove 105ac3d0da8SOleg Nesterov * the first entry == work, cmpxchg(task_works) should 106ac3d0da8SOleg Nesterov * fail, but it can play with *work and other entries. 107ac3d0da8SOleg Nesterov */ 108ac3d0da8SOleg Nesterov raw_spin_unlock_wait(&task->pi_lock); 109ac3d0da8SOleg Nesterov smp_mb(); 110e73f8959SOleg Nesterov 111ac3d0da8SOleg Nesterov /* Reverse the list to run the works in fifo order */ 112ac3d0da8SOleg Nesterov head = NULL; 113ac3d0da8SOleg Nesterov do { 114ac3d0da8SOleg Nesterov next = work->next; 115ac3d0da8SOleg Nesterov work->next = head; 116ac3d0da8SOleg Nesterov head = work; 117ac3d0da8SOleg Nesterov work = next; 118ac3d0da8SOleg Nesterov } while (work); 119e73f8959SOleg Nesterov 120ac3d0da8SOleg Nesterov work = head; 121ac3d0da8SOleg Nesterov do { 122ac3d0da8SOleg Nesterov next = work->next; 123ac3d0da8SOleg Nesterov work->func(work); 124ac3d0da8SOleg Nesterov work = next; 125f341861fSEric Dumazet cond_resched(); 126ac3d0da8SOleg Nesterov } while (work); 127e73f8959SOleg Nesterov } 128a2d4c71dSAl Viro } 129