1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0 2e73f8959SOleg Nesterov #include <linux/spinlock.h> 3e73f8959SOleg Nesterov #include <linux/task_work.h> 4e73f8959SOleg Nesterov #include <linux/tracehook.h> 5e73f8959SOleg Nesterov 69da33de6SOleg Nesterov static struct callback_head work_exited; /* all we need is ->next == NULL */ 79da33de6SOleg Nesterov 8892f6668SOleg Nesterov /** 9892f6668SOleg Nesterov * task_work_add - ask the @task to execute @work->func() 10892f6668SOleg Nesterov * @task: the task which should run the callback 11892f6668SOleg Nesterov * @work: the callback to run 1291989c70SJens Axboe * @notify: how to notify the targeted task 13892f6668SOleg Nesterov * 1491989c70SJens Axboe * Queue @work for task_work_run() below and notify the @task if @notify 1591989c70SJens Axboe * is @TWA_RESUME or @TWA_SIGNAL. @TWA_SIGNAL works like signals, in that the 1691989c70SJens Axboe * it will interrupt the targeted task and run the task_work. @TWA_RESUME 1791989c70SJens Axboe * work is run only when the task exits the kernel and returns to user mode, 1891989c70SJens Axboe * or before entering guest mode. Fails if the @task is exiting/exited and thus 1991989c70SJens Axboe * it can't process this @work. Otherwise @work->func() will be called when the 2091989c70SJens Axboe * @task goes through one of the aforementioned transitions, or exits. 21892f6668SOleg Nesterov * 2291989c70SJens Axboe * If the targeted task is exiting, then an error is returned and the work item 2391989c70SJens Axboe * is not queued. It's up to the caller to arrange for an alternative mechanism 2491989c70SJens Axboe * in that case. 25892f6668SOleg Nesterov * 2691989c70SJens Axboe * Note: there is no ordering guarantee on works queued here. The task_work 2791989c70SJens Axboe * list is LIFO. 28c8219906SEric Dumazet * 29892f6668SOleg Nesterov * RETURNS: 30892f6668SOleg Nesterov * 0 if succeeds or -ESRCH. 31892f6668SOleg Nesterov */ 3291989c70SJens Axboe int task_work_add(struct task_struct *task, struct callback_head *work, 3391989c70SJens Axboe enum task_work_notify_mode notify) 34e73f8959SOleg Nesterov { 35ac3d0da8SOleg Nesterov struct callback_head *head; 369da33de6SOleg Nesterov 37ac3d0da8SOleg Nesterov do { 3861e96496SOleg Nesterov head = READ_ONCE(task->task_works); 399da33de6SOleg Nesterov if (unlikely(head == &work_exited)) 409da33de6SOleg Nesterov return -ESRCH; 41ac3d0da8SOleg Nesterov work->next = head; 42ac3d0da8SOleg Nesterov } while (cmpxchg(&task->task_works, head, work) != head); 43e73f8959SOleg Nesterov 44e91b4816SOleg Nesterov switch (notify) { 4591989c70SJens Axboe case TWA_NONE: 4691989c70SJens Axboe break; 47e91b4816SOleg Nesterov case TWA_RESUME: 48e73f8959SOleg Nesterov set_notify_resume(task); 49e91b4816SOleg Nesterov break; 50e91b4816SOleg Nesterov case TWA_SIGNAL: 5103941ccfSJens Axboe set_notify_signal(task); 52e91b4816SOleg Nesterov break; 5391989c70SJens Axboe default: 5491989c70SJens Axboe WARN_ON_ONCE(1); 5591989c70SJens Axboe break; 56e91b4816SOleg Nesterov } 57e91b4816SOleg Nesterov 58ed3e694dSAl Viro return 0; 59e73f8959SOleg Nesterov } 60e73f8959SOleg Nesterov 61892f6668SOleg Nesterov /** 62*c7aab1a7SJens Axboe * task_work_cancel_match - cancel a pending work added by task_work_add() 63892f6668SOleg Nesterov * @task: the task which should execute the work 64*c7aab1a7SJens Axboe * @match: match function to call 65892f6668SOleg Nesterov * 66892f6668SOleg Nesterov * RETURNS: 67892f6668SOleg Nesterov * The found work or NULL if not found. 68892f6668SOleg Nesterov */ 6967d12145SAl Viro struct callback_head * 70*c7aab1a7SJens Axboe task_work_cancel_match(struct task_struct *task, 71*c7aab1a7SJens Axboe bool (*match)(struct callback_head *, void *data), 72*c7aab1a7SJens Axboe void *data) 73e73f8959SOleg Nesterov { 74ac3d0da8SOleg Nesterov struct callback_head **pprev = &task->task_works; 75205e550aSOleg Nesterov struct callback_head *work; 76e73f8959SOleg Nesterov unsigned long flags; 7761e96496SOleg Nesterov 7861e96496SOleg Nesterov if (likely(!task->task_works)) 7961e96496SOleg Nesterov return NULL; 80ac3d0da8SOleg Nesterov /* 81ac3d0da8SOleg Nesterov * If cmpxchg() fails we continue without updating pprev. 82ac3d0da8SOleg Nesterov * Either we raced with task_work_add() which added the 83ac3d0da8SOleg Nesterov * new entry before this work, we will find it again. Or 849da33de6SOleg Nesterov * we raced with task_work_run(), *pprev == NULL/exited. 85ac3d0da8SOleg Nesterov */ 86e73f8959SOleg Nesterov raw_spin_lock_irqsave(&task->pi_lock, flags); 87506458efSWill Deacon while ((work = READ_ONCE(*pprev))) { 88*c7aab1a7SJens Axboe if (!match(work, data)) 89ac3d0da8SOleg Nesterov pprev = &work->next; 90ac3d0da8SOleg Nesterov else if (cmpxchg(pprev, work, work->next) == work) 91158e1645SAl Viro break; 92158e1645SAl Viro } 93e73f8959SOleg Nesterov raw_spin_unlock_irqrestore(&task->pi_lock, flags); 94ac3d0da8SOleg Nesterov 95ac3d0da8SOleg Nesterov return work; 96e73f8959SOleg Nesterov } 97e73f8959SOleg Nesterov 98*c7aab1a7SJens Axboe static bool task_work_func_match(struct callback_head *cb, void *data) 99*c7aab1a7SJens Axboe { 100*c7aab1a7SJens Axboe return cb->func == data; 101*c7aab1a7SJens Axboe } 102*c7aab1a7SJens Axboe 103*c7aab1a7SJens Axboe /** 104*c7aab1a7SJens Axboe * task_work_cancel - cancel a pending work added by task_work_add() 105*c7aab1a7SJens Axboe * @task: the task which should execute the work 106*c7aab1a7SJens Axboe * @func: identifies the work to remove 107*c7aab1a7SJens Axboe * 108*c7aab1a7SJens Axboe * Find the last queued pending work with ->func == @func and remove 109*c7aab1a7SJens Axboe * it from queue. 110*c7aab1a7SJens Axboe * 111*c7aab1a7SJens Axboe * RETURNS: 112*c7aab1a7SJens Axboe * The found work or NULL if not found. 113*c7aab1a7SJens Axboe */ 114*c7aab1a7SJens Axboe struct callback_head * 115*c7aab1a7SJens Axboe task_work_cancel(struct task_struct *task, task_work_func_t func) 116*c7aab1a7SJens Axboe { 117*c7aab1a7SJens Axboe return task_work_cancel_match(task, task_work_func_match, func); 118*c7aab1a7SJens Axboe } 119*c7aab1a7SJens Axboe 120892f6668SOleg Nesterov /** 121892f6668SOleg Nesterov * task_work_run - execute the works added by task_work_add() 122892f6668SOleg Nesterov * 123892f6668SOleg Nesterov * Flush the pending works. Should be used by the core kernel code. 124892f6668SOleg Nesterov * Called before the task returns to the user-mode or stops, or when 125892f6668SOleg Nesterov * it exits. In the latter case task_work_add() can no longer add the 126892f6668SOleg Nesterov * new work after task_work_run() returns. 127892f6668SOleg Nesterov */ 128e73f8959SOleg Nesterov void task_work_run(void) 129e73f8959SOleg Nesterov { 130e73f8959SOleg Nesterov struct task_struct *task = current; 131ac3d0da8SOleg Nesterov struct callback_head *work, *head, *next; 132e73f8959SOleg Nesterov 133ac3d0da8SOleg Nesterov for (;;) { 1349da33de6SOleg Nesterov /* 1359da33de6SOleg Nesterov * work->func() can do task_work_add(), do not set 1369da33de6SOleg Nesterov * work_exited unless the list is empty. 1379da33de6SOleg Nesterov */ 1389da33de6SOleg Nesterov do { 1396fb61492SOleg Nesterov head = NULL; 14061e96496SOleg Nesterov work = READ_ONCE(task->task_works); 1416fb61492SOleg Nesterov if (!work) { 1426fb61492SOleg Nesterov if (task->flags & PF_EXITING) 1436fb61492SOleg Nesterov head = &work_exited; 1446fb61492SOleg Nesterov else 1456fb61492SOleg Nesterov break; 1466fb61492SOleg Nesterov } 1479da33de6SOleg Nesterov } while (cmpxchg(&task->task_works, work, head) != work); 1489da33de6SOleg Nesterov 149ac3d0da8SOleg Nesterov if (!work) 150ac3d0da8SOleg Nesterov break; 1516fb61492SOleg Nesterov /* 1526fb61492SOleg Nesterov * Synchronize with task_work_cancel(). It can not remove 1536fb61492SOleg Nesterov * the first entry == work, cmpxchg(task_works) must fail. 1546fb61492SOleg Nesterov * But it can remove another entry from the ->next list. 1556fb61492SOleg Nesterov */ 1566fb61492SOleg Nesterov raw_spin_lock_irq(&task->pi_lock); 1576fb61492SOleg Nesterov raw_spin_unlock_irq(&task->pi_lock); 158e73f8959SOleg Nesterov 159ac3d0da8SOleg Nesterov do { 160ac3d0da8SOleg Nesterov next = work->next; 161ac3d0da8SOleg Nesterov work->func(work); 162ac3d0da8SOleg Nesterov work = next; 163f341861fSEric Dumazet cond_resched(); 164ac3d0da8SOleg Nesterov } while (work); 165e73f8959SOleg Nesterov } 166a2d4c71dSAl Viro } 167