1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0 2e73f8959SOleg Nesterov #include <linux/spinlock.h> 3e73f8959SOleg Nesterov #include <linux/task_work.h> 4e73f8959SOleg Nesterov #include <linux/tracehook.h> 5e73f8959SOleg Nesterov 69da33de6SOleg Nesterov static struct callback_head work_exited; /* all we need is ->next == NULL */ 79da33de6SOleg Nesterov 8892f6668SOleg Nesterov /** 9892f6668SOleg Nesterov * task_work_add - ask the @task to execute @work->func() 10892f6668SOleg Nesterov * @task: the task which should run the callback 11892f6668SOleg Nesterov * @work: the callback to run 1291989c70SJens Axboe * @notify: how to notify the targeted task 13892f6668SOleg Nesterov * 1491989c70SJens Axboe * Queue @work for task_work_run() below and notify the @task if @notify 1591989c70SJens Axboe * is @TWA_RESUME or @TWA_SIGNAL. @TWA_SIGNAL works like signals, in that the 1691989c70SJens Axboe * it will interrupt the targeted task and run the task_work. @TWA_RESUME 1791989c70SJens Axboe * work is run only when the task exits the kernel and returns to user mode, 1891989c70SJens Axboe * or before entering guest mode. Fails if the @task is exiting/exited and thus 1991989c70SJens Axboe * it can't process this @work. Otherwise @work->func() will be called when the 2091989c70SJens Axboe * @task goes through one of the aforementioned transitions, or exits. 21892f6668SOleg Nesterov * 2291989c70SJens Axboe * If the targeted task is exiting, then an error is returned and the work item 2391989c70SJens Axboe * is not queued. It's up to the caller to arrange for an alternative mechanism 2491989c70SJens Axboe * in that case. 25892f6668SOleg Nesterov * 2691989c70SJens Axboe * Note: there is no ordering guarantee on works queued here. The task_work 2791989c70SJens Axboe * list is LIFO. 28c8219906SEric Dumazet * 29892f6668SOleg Nesterov * RETURNS: 30892f6668SOleg Nesterov * 0 if succeeds or -ESRCH. 31892f6668SOleg Nesterov */ 3291989c70SJens Axboe int task_work_add(struct task_struct *task, struct callback_head *work, 3391989c70SJens Axboe enum task_work_notify_mode notify) 34e73f8959SOleg Nesterov { 35ac3d0da8SOleg Nesterov struct callback_head *head; 369da33de6SOleg Nesterov 37*23f61f0fSWalter Wu /* record the work call stack in order to print it in KASAN reports */ 38*23f61f0fSWalter Wu kasan_record_aux_stack(work); 39*23f61f0fSWalter Wu 40ac3d0da8SOleg Nesterov do { 4161e96496SOleg Nesterov head = READ_ONCE(task->task_works); 429da33de6SOleg Nesterov if (unlikely(head == &work_exited)) 439da33de6SOleg Nesterov return -ESRCH; 44ac3d0da8SOleg Nesterov work->next = head; 45ac3d0da8SOleg Nesterov } while (cmpxchg(&task->task_works, head, work) != head); 46e73f8959SOleg Nesterov 47e91b4816SOleg Nesterov switch (notify) { 4891989c70SJens Axboe case TWA_NONE: 4991989c70SJens Axboe break; 50e91b4816SOleg Nesterov case TWA_RESUME: 51e73f8959SOleg Nesterov set_notify_resume(task); 52e91b4816SOleg Nesterov break; 53e91b4816SOleg Nesterov case TWA_SIGNAL: 5403941ccfSJens Axboe set_notify_signal(task); 55e91b4816SOleg Nesterov break; 5691989c70SJens Axboe default: 5791989c70SJens Axboe WARN_ON_ONCE(1); 5891989c70SJens Axboe break; 59e91b4816SOleg Nesterov } 60e91b4816SOleg Nesterov 61ed3e694dSAl Viro return 0; 62e73f8959SOleg Nesterov } 63e73f8959SOleg Nesterov 64892f6668SOleg Nesterov /** 65c7aab1a7SJens Axboe * task_work_cancel_match - cancel a pending work added by task_work_add() 66892f6668SOleg Nesterov * @task: the task which should execute the work 67c7aab1a7SJens Axboe * @match: match function to call 68892f6668SOleg Nesterov * 69892f6668SOleg Nesterov * RETURNS: 70892f6668SOleg Nesterov * The found work or NULL if not found. 71892f6668SOleg Nesterov */ 7267d12145SAl Viro struct callback_head * 73c7aab1a7SJens Axboe task_work_cancel_match(struct task_struct *task, 74c7aab1a7SJens Axboe bool (*match)(struct callback_head *, void *data), 75c7aab1a7SJens Axboe void *data) 76e73f8959SOleg Nesterov { 77ac3d0da8SOleg Nesterov struct callback_head **pprev = &task->task_works; 78205e550aSOleg Nesterov struct callback_head *work; 79e73f8959SOleg Nesterov unsigned long flags; 8061e96496SOleg Nesterov 8161e96496SOleg Nesterov if (likely(!task->task_works)) 8261e96496SOleg Nesterov return NULL; 83ac3d0da8SOleg Nesterov /* 84ac3d0da8SOleg Nesterov * If cmpxchg() fails we continue without updating pprev. 85ac3d0da8SOleg Nesterov * Either we raced with task_work_add() which added the 86ac3d0da8SOleg Nesterov * new entry before this work, we will find it again. Or 879da33de6SOleg Nesterov * we raced with task_work_run(), *pprev == NULL/exited. 88ac3d0da8SOleg Nesterov */ 89e73f8959SOleg Nesterov raw_spin_lock_irqsave(&task->pi_lock, flags); 90506458efSWill Deacon while ((work = READ_ONCE(*pprev))) { 91c7aab1a7SJens Axboe if (!match(work, data)) 92ac3d0da8SOleg Nesterov pprev = &work->next; 93ac3d0da8SOleg Nesterov else if (cmpxchg(pprev, work, work->next) == work) 94158e1645SAl Viro break; 95158e1645SAl Viro } 96e73f8959SOleg Nesterov raw_spin_unlock_irqrestore(&task->pi_lock, flags); 97ac3d0da8SOleg Nesterov 98ac3d0da8SOleg Nesterov return work; 99e73f8959SOleg Nesterov } 100e73f8959SOleg Nesterov 101c7aab1a7SJens Axboe static bool task_work_func_match(struct callback_head *cb, void *data) 102c7aab1a7SJens Axboe { 103c7aab1a7SJens Axboe return cb->func == data; 104c7aab1a7SJens Axboe } 105c7aab1a7SJens Axboe 106c7aab1a7SJens Axboe /** 107c7aab1a7SJens Axboe * task_work_cancel - cancel a pending work added by task_work_add() 108c7aab1a7SJens Axboe * @task: the task which should execute the work 109c7aab1a7SJens Axboe * @func: identifies the work to remove 110c7aab1a7SJens Axboe * 111c7aab1a7SJens Axboe * Find the last queued pending work with ->func == @func and remove 112c7aab1a7SJens Axboe * it from queue. 113c7aab1a7SJens Axboe * 114c7aab1a7SJens Axboe * RETURNS: 115c7aab1a7SJens Axboe * The found work or NULL if not found. 116c7aab1a7SJens Axboe */ 117c7aab1a7SJens Axboe struct callback_head * 118c7aab1a7SJens Axboe task_work_cancel(struct task_struct *task, task_work_func_t func) 119c7aab1a7SJens Axboe { 120c7aab1a7SJens Axboe return task_work_cancel_match(task, task_work_func_match, func); 121c7aab1a7SJens Axboe } 122c7aab1a7SJens Axboe 123892f6668SOleg Nesterov /** 124892f6668SOleg Nesterov * task_work_run - execute the works added by task_work_add() 125892f6668SOleg Nesterov * 126892f6668SOleg Nesterov * Flush the pending works. Should be used by the core kernel code. 127892f6668SOleg Nesterov * Called before the task returns to the user-mode or stops, or when 128892f6668SOleg Nesterov * it exits. In the latter case task_work_add() can no longer add the 129892f6668SOleg Nesterov * new work after task_work_run() returns. 130892f6668SOleg Nesterov */ 131e73f8959SOleg Nesterov void task_work_run(void) 132e73f8959SOleg Nesterov { 133e73f8959SOleg Nesterov struct task_struct *task = current; 134ac3d0da8SOleg Nesterov struct callback_head *work, *head, *next; 135e73f8959SOleg Nesterov 136ac3d0da8SOleg Nesterov for (;;) { 1379da33de6SOleg Nesterov /* 1389da33de6SOleg Nesterov * work->func() can do task_work_add(), do not set 1399da33de6SOleg Nesterov * work_exited unless the list is empty. 1409da33de6SOleg Nesterov */ 1419da33de6SOleg Nesterov do { 1426fb61492SOleg Nesterov head = NULL; 14361e96496SOleg Nesterov work = READ_ONCE(task->task_works); 1446fb61492SOleg Nesterov if (!work) { 1456fb61492SOleg Nesterov if (task->flags & PF_EXITING) 1466fb61492SOleg Nesterov head = &work_exited; 1476fb61492SOleg Nesterov else 1486fb61492SOleg Nesterov break; 1496fb61492SOleg Nesterov } 1509da33de6SOleg Nesterov } while (cmpxchg(&task->task_works, work, head) != work); 1519da33de6SOleg Nesterov 152ac3d0da8SOleg Nesterov if (!work) 153ac3d0da8SOleg Nesterov break; 1546fb61492SOleg Nesterov /* 1556fb61492SOleg Nesterov * Synchronize with task_work_cancel(). It can not remove 1566fb61492SOleg Nesterov * the first entry == work, cmpxchg(task_works) must fail. 1576fb61492SOleg Nesterov * But it can remove another entry from the ->next list. 1586fb61492SOleg Nesterov */ 1596fb61492SOleg Nesterov raw_spin_lock_irq(&task->pi_lock); 1606fb61492SOleg Nesterov raw_spin_unlock_irq(&task->pi_lock); 161e73f8959SOleg Nesterov 162ac3d0da8SOleg Nesterov do { 163ac3d0da8SOleg Nesterov next = work->next; 164ac3d0da8SOleg Nesterov work->func(work); 165ac3d0da8SOleg Nesterov work = next; 166f341861fSEric Dumazet cond_resched(); 167ac3d0da8SOleg Nesterov } while (work); 168e73f8959SOleg Nesterov } 169a2d4c71dSAl Viro } 170