1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0
2e73f8959SOleg Nesterov #include <linux/spinlock.h>
3e73f8959SOleg Nesterov #include <linux/task_work.h>
403248addSEric W. Biederman #include <linux/resume_user_mode.h>
5e73f8959SOleg Nesterov
69da33de6SOleg Nesterov static struct callback_head work_exited; /* all we need is ->next == NULL */
79da33de6SOleg Nesterov
8892f6668SOleg Nesterov /**
9892f6668SOleg Nesterov * task_work_add - ask the @task to execute @work->func()
10892f6668SOleg Nesterov * @task: the task which should run the callback
11892f6668SOleg Nesterov * @work: the callback to run
1291989c70SJens Axboe * @notify: how to notify the targeted task
13892f6668SOleg Nesterov *
1491989c70SJens Axboe * Queue @work for task_work_run() below and notify the @task if @notify
15e788be95SJens Axboe * is @TWA_RESUME, @TWA_SIGNAL, or @TWA_SIGNAL_NO_IPI.
16e788be95SJens Axboe *
17e788be95SJens Axboe * @TWA_SIGNAL works like signals, in that the it will interrupt the targeted
18e788be95SJens Axboe * task and run the task_work, regardless of whether the task is currently
19e788be95SJens Axboe * running in the kernel or userspace.
20e788be95SJens Axboe * @TWA_SIGNAL_NO_IPI works like @TWA_SIGNAL, except it doesn't send a
21e788be95SJens Axboe * reschedule IPI to force the targeted task to reschedule and run task_work.
22e788be95SJens Axboe * This can be advantageous if there's no strict requirement that the
23e788be95SJens Axboe * task_work be run as soon as possible, just whenever the task enters the
24e788be95SJens Axboe * kernel anyway.
25e788be95SJens Axboe * @TWA_RESUME work is run only when the task exits the kernel and returns to
26e788be95SJens Axboe * user mode, or before entering guest mode.
27e788be95SJens Axboe *
28e788be95SJens Axboe * Fails if the @task is exiting/exited and thus it can't process this @work.
29e788be95SJens Axboe * Otherwise @work->func() will be called when the @task goes through one of
30e788be95SJens Axboe * the aforementioned transitions, or exits.
31892f6668SOleg Nesterov *
3291989c70SJens Axboe * If the targeted task is exiting, then an error is returned and the work item
3391989c70SJens Axboe * is not queued. It's up to the caller to arrange for an alternative mechanism
3491989c70SJens Axboe * in that case.
35892f6668SOleg Nesterov *
3691989c70SJens Axboe * Note: there is no ordering guarantee on works queued here. The task_work
3791989c70SJens Axboe * list is LIFO.
38c8219906SEric Dumazet *
39892f6668SOleg Nesterov * RETURNS:
40892f6668SOleg Nesterov * 0 if succeeds or -ESRCH.
41892f6668SOleg Nesterov */
task_work_add(struct task_struct * task,struct callback_head * work,enum task_work_notify_mode notify)4291989c70SJens Axboe int task_work_add(struct task_struct *task, struct callback_head *work,
4391989c70SJens Axboe enum task_work_notify_mode notify)
44e73f8959SOleg Nesterov {
45ac3d0da8SOleg Nesterov struct callback_head *head;
469da33de6SOleg Nesterov
4723f61f0fSWalter Wu /* record the work call stack in order to print it in KASAN reports */
4823f61f0fSWalter Wu kasan_record_aux_stack(work);
4923f61f0fSWalter Wu
5061e96496SOleg Nesterov head = READ_ONCE(task->task_works);
515fdfa161SUros Bizjak do {
529da33de6SOleg Nesterov if (unlikely(head == &work_exited))
539da33de6SOleg Nesterov return -ESRCH;
54ac3d0da8SOleg Nesterov work->next = head;
555fdfa161SUros Bizjak } while (!try_cmpxchg(&task->task_works, &head, work));
56e73f8959SOleg Nesterov
57e91b4816SOleg Nesterov switch (notify) {
5891989c70SJens Axboe case TWA_NONE:
5991989c70SJens Axboe break;
60e91b4816SOleg Nesterov case TWA_RESUME:
61e73f8959SOleg Nesterov set_notify_resume(task);
62e91b4816SOleg Nesterov break;
63e91b4816SOleg Nesterov case TWA_SIGNAL:
6403941ccfSJens Axboe set_notify_signal(task);
65e91b4816SOleg Nesterov break;
66e788be95SJens Axboe case TWA_SIGNAL_NO_IPI:
67e788be95SJens Axboe __set_notify_signal(task);
68e788be95SJens Axboe break;
6991989c70SJens Axboe default:
7091989c70SJens Axboe WARN_ON_ONCE(1);
7191989c70SJens Axboe break;
72e91b4816SOleg Nesterov }
73e91b4816SOleg Nesterov
74ed3e694dSAl Viro return 0;
75e73f8959SOleg Nesterov }
76e73f8959SOleg Nesterov
77892f6668SOleg Nesterov /**
78c7aab1a7SJens Axboe * task_work_cancel_match - cancel a pending work added by task_work_add()
79892f6668SOleg Nesterov * @task: the task which should execute the work
80c7aab1a7SJens Axboe * @match: match function to call
81*4653e5ddSJens Axboe * @data: data to be passed in to match function
82892f6668SOleg Nesterov *
83892f6668SOleg Nesterov * RETURNS:
84892f6668SOleg Nesterov * The found work or NULL if not found.
85892f6668SOleg Nesterov */
8667d12145SAl Viro struct callback_head *
task_work_cancel_match(struct task_struct * task,bool (* match)(struct callback_head *,void * data),void * data)87c7aab1a7SJens Axboe task_work_cancel_match(struct task_struct *task,
88c7aab1a7SJens Axboe bool (*match)(struct callback_head *, void *data),
89c7aab1a7SJens Axboe void *data)
90e73f8959SOleg Nesterov {
91ac3d0da8SOleg Nesterov struct callback_head **pprev = &task->task_works;
92205e550aSOleg Nesterov struct callback_head *work;
93e73f8959SOleg Nesterov unsigned long flags;
9461e96496SOleg Nesterov
957f62d40dSEric W. Biederman if (likely(!task_work_pending(task)))
9661e96496SOleg Nesterov return NULL;
97ac3d0da8SOleg Nesterov /*
98ac3d0da8SOleg Nesterov * If cmpxchg() fails we continue without updating pprev.
99ac3d0da8SOleg Nesterov * Either we raced with task_work_add() which added the
100ac3d0da8SOleg Nesterov * new entry before this work, we will find it again. Or
1019da33de6SOleg Nesterov * we raced with task_work_run(), *pprev == NULL/exited.
102ac3d0da8SOleg Nesterov */
103e73f8959SOleg Nesterov raw_spin_lock_irqsave(&task->pi_lock, flags);
1045fdfa161SUros Bizjak work = READ_ONCE(*pprev);
1055fdfa161SUros Bizjak while (work) {
1065fdfa161SUros Bizjak if (!match(work, data)) {
107ac3d0da8SOleg Nesterov pprev = &work->next;
1085fdfa161SUros Bizjak work = READ_ONCE(*pprev);
1095fdfa161SUros Bizjak } else if (try_cmpxchg(pprev, &work, work->next))
110158e1645SAl Viro break;
111158e1645SAl Viro }
112e73f8959SOleg Nesterov raw_spin_unlock_irqrestore(&task->pi_lock, flags);
113ac3d0da8SOleg Nesterov
114ac3d0da8SOleg Nesterov return work;
115e73f8959SOleg Nesterov }
116e73f8959SOleg Nesterov
task_work_func_match(struct callback_head * cb,void * data)117c7aab1a7SJens Axboe static bool task_work_func_match(struct callback_head *cb, void *data)
118c7aab1a7SJens Axboe {
119c7aab1a7SJens Axboe return cb->func == data;
120c7aab1a7SJens Axboe }
121c7aab1a7SJens Axboe
122c7aab1a7SJens Axboe /**
123c7aab1a7SJens Axboe * task_work_cancel - cancel a pending work added by task_work_add()
124c7aab1a7SJens Axboe * @task: the task which should execute the work
125c7aab1a7SJens Axboe * @func: identifies the work to remove
126c7aab1a7SJens Axboe *
127c7aab1a7SJens Axboe * Find the last queued pending work with ->func == @func and remove
128c7aab1a7SJens Axboe * it from queue.
129c7aab1a7SJens Axboe *
130c7aab1a7SJens Axboe * RETURNS:
131c7aab1a7SJens Axboe * The found work or NULL if not found.
132c7aab1a7SJens Axboe */
133c7aab1a7SJens Axboe struct callback_head *
task_work_cancel(struct task_struct * task,task_work_func_t func)134c7aab1a7SJens Axboe task_work_cancel(struct task_struct *task, task_work_func_t func)
135c7aab1a7SJens Axboe {
136c7aab1a7SJens Axboe return task_work_cancel_match(task, task_work_func_match, func);
137c7aab1a7SJens Axboe }
138c7aab1a7SJens Axboe
139892f6668SOleg Nesterov /**
140892f6668SOleg Nesterov * task_work_run - execute the works added by task_work_add()
141892f6668SOleg Nesterov *
142892f6668SOleg Nesterov * Flush the pending works. Should be used by the core kernel code.
143892f6668SOleg Nesterov * Called before the task returns to the user-mode or stops, or when
144892f6668SOleg Nesterov * it exits. In the latter case task_work_add() can no longer add the
145892f6668SOleg Nesterov * new work after task_work_run() returns.
146892f6668SOleg Nesterov */
task_work_run(void)147e73f8959SOleg Nesterov void task_work_run(void)
148e73f8959SOleg Nesterov {
149e73f8959SOleg Nesterov struct task_struct *task = current;
150ac3d0da8SOleg Nesterov struct callback_head *work, *head, *next;
151e73f8959SOleg Nesterov
152ac3d0da8SOleg Nesterov for (;;) {
1539da33de6SOleg Nesterov /*
1549da33de6SOleg Nesterov * work->func() can do task_work_add(), do not set
1559da33de6SOleg Nesterov * work_exited unless the list is empty.
1569da33de6SOleg Nesterov */
1575fdfa161SUros Bizjak work = READ_ONCE(task->task_works);
1589da33de6SOleg Nesterov do {
1596fb61492SOleg Nesterov head = NULL;
1606fb61492SOleg Nesterov if (!work) {
1616fb61492SOleg Nesterov if (task->flags & PF_EXITING)
1626fb61492SOleg Nesterov head = &work_exited;
1636fb61492SOleg Nesterov else
1646fb61492SOleg Nesterov break;
1656fb61492SOleg Nesterov }
1665fdfa161SUros Bizjak } while (!try_cmpxchg(&task->task_works, &work, head));
1679da33de6SOleg Nesterov
168ac3d0da8SOleg Nesterov if (!work)
169ac3d0da8SOleg Nesterov break;
1706fb61492SOleg Nesterov /*
1716fb61492SOleg Nesterov * Synchronize with task_work_cancel(). It can not remove
1726fb61492SOleg Nesterov * the first entry == work, cmpxchg(task_works) must fail.
1736fb61492SOleg Nesterov * But it can remove another entry from the ->next list.
1746fb61492SOleg Nesterov */
1756fb61492SOleg Nesterov raw_spin_lock_irq(&task->pi_lock);
1766fb61492SOleg Nesterov raw_spin_unlock_irq(&task->pi_lock);
177e73f8959SOleg Nesterov
178ac3d0da8SOleg Nesterov do {
179ac3d0da8SOleg Nesterov next = work->next;
180ac3d0da8SOleg Nesterov work->func(work);
181ac3d0da8SOleg Nesterov work = next;
182f341861fSEric Dumazet cond_resched();
183ac3d0da8SOleg Nesterov } while (work);
184e73f8959SOleg Nesterov }
185a2d4c71dSAl Viro }
186