xref: /openbmc/linux/kernel/sched/completion.c (revision d2c43ff1)
1 /*
2  * Generic wait-for-completion handler;
3  *
4  * It differs from semaphores in that their default case is the opposite,
5  * wait_for_completion default blocks whereas semaphore default non-block. The
6  * interface also makes it easy to 'complete' multiple waiting threads,
7  * something which isn't entirely natural for semaphores.
8  *
9  * But more importantly, the primitive documents the usage. Semaphores would
10  * typically be used for exclusion which gives rise to priority inversion.
11  * Waiting for completion is a typically sync point, but not an exclusion point.
12  */
13 
14 #include <linux/sched/signal.h>
15 #include <linux/sched/debug.h>
16 #include <linux/completion.h>
17 
18 /**
19  * complete: - signals a single thread waiting on this completion
20  * @x:  holds the state of this particular completion
21  *
22  * This will wake up a single thread waiting on this completion. Threads will be
23  * awakened in the same order in which they were queued.
24  *
25  * See also complete_all(), wait_for_completion() and related routines.
26  *
27  * It may be assumed that this function implies a write memory barrier before
28  * changing the task state if and only if any tasks are woken up.
29  */
30 void complete(struct completion *x)
31 {
32 	unsigned long flags;
33 
34 	spin_lock_irqsave(&x->wait.lock, flags);
35 	if (x->done != UINT_MAX)
36 		x->done++;
37 	__wake_up_locked(&x->wait, TASK_NORMAL, 1);
38 	spin_unlock_irqrestore(&x->wait.lock, flags);
39 }
40 EXPORT_SYMBOL(complete);
41 
42 /**
43  * complete_all: - signals all threads waiting on this completion
44  * @x:  holds the state of this particular completion
45  *
46  * This will wake up all threads waiting on this particular completion event.
47  *
48  * It may be assumed that this function implies a write memory barrier before
49  * changing the task state if and only if any tasks are woken up.
50  */
51 void complete_all(struct completion *x)
52 {
53 	unsigned long flags;
54 
55 	spin_lock_irqsave(&x->wait.lock, flags);
56 	x->done = UINT_MAX;
57 	__wake_up_locked(&x->wait, TASK_NORMAL, 0);
58 	spin_unlock_irqrestore(&x->wait.lock, flags);
59 }
60 EXPORT_SYMBOL(complete_all);
61 
62 static inline long __sched
63 do_wait_for_common(struct completion *x,
64 		   long (*action)(long), long timeout, int state)
65 {
66 	if (!x->done) {
67 		DECLARE_WAITQUEUE(wait, current);
68 
69 		__add_wait_queue_entry_tail_exclusive(&x->wait, &wait);
70 		do {
71 			if (signal_pending_state(state, current)) {
72 				timeout = -ERESTARTSYS;
73 				break;
74 			}
75 			__set_current_state(state);
76 			spin_unlock_irq(&x->wait.lock);
77 			timeout = action(timeout);
78 			spin_lock_irq(&x->wait.lock);
79 		} while (!x->done && timeout);
80 		__remove_wait_queue(&x->wait, &wait);
81 		if (!x->done)
82 			return timeout;
83 	}
84 	if (x->done != UINT_MAX)
85 		x->done--;
86 	return timeout ?: 1;
87 }
88 
89 static inline long __sched
90 __wait_for_common(struct completion *x,
91 		  long (*action)(long), long timeout, int state)
92 {
93 	might_sleep();
94 
95 	spin_lock_irq(&x->wait.lock);
96 	timeout = do_wait_for_common(x, action, timeout, state);
97 	spin_unlock_irq(&x->wait.lock);
98 	return timeout;
99 }
100 
101 static long __sched
102 wait_for_common(struct completion *x, long timeout, int state)
103 {
104 	return __wait_for_common(x, schedule_timeout, timeout, state);
105 }
106 
107 static long __sched
108 wait_for_common_io(struct completion *x, long timeout, int state)
109 {
110 	return __wait_for_common(x, io_schedule_timeout, timeout, state);
111 }
112 
113 /**
114  * wait_for_completion: - waits for completion of a task
115  * @x:  holds the state of this particular completion
116  *
117  * This waits to be signaled for completion of a specific task. It is NOT
118  * interruptible and there is no timeout.
119  *
120  * See also similar routines (i.e. wait_for_completion_timeout()) with timeout
121  * and interrupt capability. Also see complete().
122  */
123 void __sched wait_for_completion(struct completion *x)
124 {
125 	wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE);
126 }
127 EXPORT_SYMBOL(wait_for_completion);
128 
129 /**
130  * wait_for_completion_timeout: - waits for completion of a task (w/timeout)
131  * @x:  holds the state of this particular completion
132  * @timeout:  timeout value in jiffies
133  *
134  * This waits for either a completion of a specific task to be signaled or for a
135  * specified timeout to expire. The timeout is in jiffies. It is not
136  * interruptible.
137  *
138  * Return: 0 if timed out, and positive (at least 1, or number of jiffies left
139  * till timeout) if completed.
140  */
141 unsigned long __sched
142 wait_for_completion_timeout(struct completion *x, unsigned long timeout)
143 {
144 	return wait_for_common(x, timeout, TASK_UNINTERRUPTIBLE);
145 }
146 EXPORT_SYMBOL(wait_for_completion_timeout);
147 
148 /**
149  * wait_for_completion_io: - waits for completion of a task
150  * @x:  holds the state of this particular completion
151  *
152  * This waits to be signaled for completion of a specific task. It is NOT
153  * interruptible and there is no timeout. The caller is accounted as waiting
154  * for IO (which traditionally means blkio only).
155  */
156 void __sched wait_for_completion_io(struct completion *x)
157 {
158 	wait_for_common_io(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE);
159 }
160 EXPORT_SYMBOL(wait_for_completion_io);
161 
162 /**
163  * wait_for_completion_io_timeout: - waits for completion of a task (w/timeout)
164  * @x:  holds the state of this particular completion
165  * @timeout:  timeout value in jiffies
166  *
167  * This waits for either a completion of a specific task to be signaled or for a
168  * specified timeout to expire. The timeout is in jiffies. It is not
169  * interruptible. The caller is accounted as waiting for IO (which traditionally
170  * means blkio only).
171  *
172  * Return: 0 if timed out, and positive (at least 1, or number of jiffies left
173  * till timeout) if completed.
174  */
175 unsigned long __sched
176 wait_for_completion_io_timeout(struct completion *x, unsigned long timeout)
177 {
178 	return wait_for_common_io(x, timeout, TASK_UNINTERRUPTIBLE);
179 }
180 EXPORT_SYMBOL(wait_for_completion_io_timeout);
181 
182 /**
183  * wait_for_completion_interruptible: - waits for completion of a task (w/intr)
184  * @x:  holds the state of this particular completion
185  *
186  * This waits for completion of a specific task to be signaled. It is
187  * interruptible.
188  *
189  * Return: -ERESTARTSYS if interrupted, 0 if completed.
190  */
191 int __sched wait_for_completion_interruptible(struct completion *x)
192 {
193 	long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_INTERRUPTIBLE);
194 	if (t == -ERESTARTSYS)
195 		return t;
196 	return 0;
197 }
198 EXPORT_SYMBOL(wait_for_completion_interruptible);
199 
200 /**
201  * wait_for_completion_interruptible_timeout: - waits for completion (w/(to,intr))
202  * @x:  holds the state of this particular completion
203  * @timeout:  timeout value in jiffies
204  *
205  * This waits for either a completion of a specific task to be signaled or for a
206  * specified timeout to expire. It is interruptible. The timeout is in jiffies.
207  *
208  * Return: -ERESTARTSYS if interrupted, 0 if timed out, positive (at least 1,
209  * or number of jiffies left till timeout) if completed.
210  */
211 long __sched
212 wait_for_completion_interruptible_timeout(struct completion *x,
213 					  unsigned long timeout)
214 {
215 	return wait_for_common(x, timeout, TASK_INTERRUPTIBLE);
216 }
217 EXPORT_SYMBOL(wait_for_completion_interruptible_timeout);
218 
219 /**
220  * wait_for_completion_killable: - waits for completion of a task (killable)
221  * @x:  holds the state of this particular completion
222  *
223  * This waits to be signaled for completion of a specific task. It can be
224  * interrupted by a kill signal.
225  *
226  * Return: -ERESTARTSYS if interrupted, 0 if completed.
227  */
228 int __sched wait_for_completion_killable(struct completion *x)
229 {
230 	long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_KILLABLE);
231 	if (t == -ERESTARTSYS)
232 		return t;
233 	return 0;
234 }
235 EXPORT_SYMBOL(wait_for_completion_killable);
236 
237 /**
238  * wait_for_completion_killable_timeout: - waits for completion of a task (w/(to,killable))
239  * @x:  holds the state of this particular completion
240  * @timeout:  timeout value in jiffies
241  *
242  * This waits for either a completion of a specific task to be
243  * signaled or for a specified timeout to expire. It can be
244  * interrupted by a kill signal. The timeout is in jiffies.
245  *
246  * Return: -ERESTARTSYS if interrupted, 0 if timed out, positive (at least 1,
247  * or number of jiffies left till timeout) if completed.
248  */
249 long __sched
250 wait_for_completion_killable_timeout(struct completion *x,
251 				     unsigned long timeout)
252 {
253 	return wait_for_common(x, timeout, TASK_KILLABLE);
254 }
255 EXPORT_SYMBOL(wait_for_completion_killable_timeout);
256 
257 /**
258  *	try_wait_for_completion - try to decrement a completion without blocking
259  *	@x:	completion structure
260  *
261  *	Return: 0 if a decrement cannot be done without blocking
262  *		 1 if a decrement succeeded.
263  *
264  *	If a completion is being used as a counting completion,
265  *	attempt to decrement the counter without blocking. This
266  *	enables us to avoid waiting if the resource the completion
267  *	is protecting is not available.
268  */
269 bool try_wait_for_completion(struct completion *x)
270 {
271 	unsigned long flags;
272 	int ret = 1;
273 
274 	/*
275 	 * Since x->done will need to be locked only
276 	 * in the non-blocking case, we check x->done
277 	 * first without taking the lock so we can
278 	 * return early in the blocking case.
279 	 */
280 	if (!READ_ONCE(x->done))
281 		return 0;
282 
283 	spin_lock_irqsave(&x->wait.lock, flags);
284 	if (!x->done)
285 		ret = 0;
286 	else if (x->done != UINT_MAX)
287 		x->done--;
288 	spin_unlock_irqrestore(&x->wait.lock, flags);
289 	return ret;
290 }
291 EXPORT_SYMBOL(try_wait_for_completion);
292 
293 /**
294  *	completion_done - Test to see if a completion has any waiters
295  *	@x:	completion structure
296  *
297  *	Return: 0 if there are waiters (wait_for_completion() in progress)
298  *		 1 if there are no waiters.
299  *
300  */
301 bool completion_done(struct completion *x)
302 {
303 	if (!READ_ONCE(x->done))
304 		return false;
305 
306 	/*
307 	 * If ->done, we need to wait for complete() to release ->wait.lock
308 	 * otherwise we can end up freeing the completion before complete()
309 	 * is done referencing it.
310 	 *
311 	 * The RMB pairs with complete()'s RELEASE of ->wait.lock and orders
312 	 * the loads of ->done and ->wait.lock such that we cannot observe
313 	 * the lock before complete() acquires it while observing the ->done
314 	 * after it's acquired the lock.
315 	 */
316 	smp_rmb();
317 	spin_unlock_wait(&x->wait.lock);
318 	return true;
319 }
320 EXPORT_SYMBOL(completion_done);
321