1 #include <linux/spinlock.h> 2 #include <linux/task_work.h> 3 #include <linux/tracehook.h> 4 5 int 6 task_work_add(struct task_struct *task, struct callback_head *work, bool notify) 7 { 8 struct callback_head *head; 9 /* 10 * Not inserting the new work if the task has already passed 11 * exit_task_work() is the responisbility of callers. 12 */ 13 do { 14 head = ACCESS_ONCE(task->task_works); 15 work->next = head; 16 } while (cmpxchg(&task->task_works, head, work) != head); 17 18 if (notify) 19 set_notify_resume(task); 20 return 0; 21 } 22 23 struct callback_head * 24 task_work_cancel(struct task_struct *task, task_work_func_t func) 25 { 26 struct callback_head **pprev = &task->task_works; 27 struct callback_head *work = NULL; 28 unsigned long flags; 29 /* 30 * If cmpxchg() fails we continue without updating pprev. 31 * Either we raced with task_work_add() which added the 32 * new entry before this work, we will find it again. Or 33 * we raced with task_work_run(), *pprev == NULL. 34 */ 35 raw_spin_lock_irqsave(&task->pi_lock, flags); 36 while ((work = ACCESS_ONCE(*pprev))) { 37 read_barrier_depends(); 38 if (work->func != func) 39 pprev = &work->next; 40 else if (cmpxchg(pprev, work, work->next) == work) 41 break; 42 } 43 raw_spin_unlock_irqrestore(&task->pi_lock, flags); 44 45 return work; 46 } 47 48 void task_work_run(void) 49 { 50 struct task_struct *task = current; 51 struct callback_head *work, *head, *next; 52 53 for (;;) { 54 work = xchg(&task->task_works, NULL); 55 if (!work) 56 break; 57 /* 58 * Synchronize with task_work_cancel(). It can't remove 59 * the first entry == work, cmpxchg(task_works) should 60 * fail, but it can play with *work and other entries. 61 */ 62 raw_spin_unlock_wait(&task->pi_lock); 63 smp_mb(); 64 65 /* Reverse the list to run the works in fifo order */ 66 head = NULL; 67 do { 68 next = work->next; 69 work->next = head; 70 head = work; 71 work = next; 72 } while (work); 73 74 work = head; 75 do { 76 next = work->next; 77 work->func(work); 78 work = next; 79 cond_resched(); 80 } while (work); 81 } 82 } 83