1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Generic wait-for-completion handler; 4 * 5 * It differs from semaphores in that their default case is the opposite, 6 * wait_for_completion default blocks whereas semaphore default non-block. The 7 * interface also makes it easy to 'complete' multiple waiting threads, 8 * something which isn't entirely natural for semaphores. 9 * 10 * But more importantly, the primitive documents the usage. Semaphores would 11 * typically be used for exclusion which gives rise to priority inversion. 12 * Waiting for completion is a typically sync point, but not an exclusion point. 13 */ 14 15 #include <linux/sched/signal.h> 16 #include <linux/sched/debug.h> 17 #include <linux/completion.h> 18 19 /** 20 * complete: - signals a single thread waiting on this completion 21 * @x: holds the state of this particular completion 22 * 23 * This will wake up a single thread waiting on this completion. Threads will be 24 * awakened in the same order in which they were queued. 25 * 26 * See also complete_all(), wait_for_completion() and related routines. 27 * 28 * It may be assumed that this function implies a write memory barrier before 29 * changing the task state if and only if any tasks are woken up. 30 */ 31 void complete(struct completion *x) 32 { 33 unsigned long flags; 34 35 spin_lock_irqsave(&x->wait.lock, flags); 36 37 if (x->done != UINT_MAX) 38 x->done++; 39 __wake_up_locked(&x->wait, TASK_NORMAL, 1); 40 spin_unlock_irqrestore(&x->wait.lock, flags); 41 } 42 EXPORT_SYMBOL(complete); 43 44 /** 45 * complete_all: - signals all threads waiting on this completion 46 * @x: holds the state of this particular completion 47 * 48 * This will wake up all threads waiting on this particular completion event. 49 * 50 * It may be assumed that this function implies a write memory barrier before 51 * changing the task state if and only if any tasks are woken up. 52 * 53 * Since complete_all() sets the completion of @x permanently to done 54 * to allow multiple waiters to finish, a call to reinit_completion() 55 * must be used on @x if @x is to be used again. The code must make 56 * sure that all waiters have woken and finished before reinitializing 57 * @x. Also note that the function completion_done() can not be used 58 * to know if there are still waiters after complete_all() has been called. 59 */ 60 void complete_all(struct completion *x) 61 { 62 unsigned long flags; 63 64 spin_lock_irqsave(&x->wait.lock, flags); 65 x->done = UINT_MAX; 66 __wake_up_locked(&x->wait, TASK_NORMAL, 0); 67 spin_unlock_irqrestore(&x->wait.lock, flags); 68 } 69 EXPORT_SYMBOL(complete_all); 70 71 static inline long __sched 72 do_wait_for_common(struct completion *x, 73 long (*action)(long), long timeout, int state) 74 { 75 if (!x->done) { 76 DECLARE_WAITQUEUE(wait, current); 77 78 __add_wait_queue_entry_tail_exclusive(&x->wait, &wait); 79 do { 80 if (signal_pending_state(state, current)) { 81 timeout = -ERESTARTSYS; 82 break; 83 } 84 __set_current_state(state); 85 spin_unlock_irq(&x->wait.lock); 86 timeout = action(timeout); 87 spin_lock_irq(&x->wait.lock); 88 } while (!x->done && timeout); 89 __remove_wait_queue(&x->wait, &wait); 90 if (!x->done) 91 return timeout; 92 } 93 if (x->done != UINT_MAX) 94 x->done--; 95 return timeout ?: 1; 96 } 97 98 static inline long __sched 99 __wait_for_common(struct completion *x, 100 long (*action)(long), long timeout, int state) 101 { 102 might_sleep(); 103 104 complete_acquire(x); 105 106 spin_lock_irq(&x->wait.lock); 107 timeout = do_wait_for_common(x, action, timeout, state); 108 spin_unlock_irq(&x->wait.lock); 109 110 complete_release(x); 111 112 return timeout; 113 } 114 115 static long __sched 116 wait_for_common(struct completion *x, long timeout, int state) 117 { 118 return __wait_for_common(x, schedule_timeout, timeout, state); 119 } 120 121 static long __sched 122 wait_for_common_io(struct completion *x, long timeout, int state) 123 { 124 return __wait_for_common(x, io_schedule_timeout, timeout, state); 125 } 126 127 /** 128 * wait_for_completion: - waits for completion of a task 129 * @x: holds the state of this particular completion 130 * 131 * This waits to be signaled for completion of a specific task. It is NOT 132 * interruptible and there is no timeout. 133 * 134 * See also similar routines (i.e. wait_for_completion_timeout()) with timeout 135 * and interrupt capability. Also see complete(). 136 */ 137 void __sched wait_for_completion(struct completion *x) 138 { 139 wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE); 140 } 141 EXPORT_SYMBOL(wait_for_completion); 142 143 /** 144 * wait_for_completion_timeout: - waits for completion of a task (w/timeout) 145 * @x: holds the state of this particular completion 146 * @timeout: timeout value in jiffies 147 * 148 * This waits for either a completion of a specific task to be signaled or for a 149 * specified timeout to expire. The timeout is in jiffies. It is not 150 * interruptible. 151 * 152 * Return: 0 if timed out, and positive (at least 1, or number of jiffies left 153 * till timeout) if completed. 154 */ 155 unsigned long __sched 156 wait_for_completion_timeout(struct completion *x, unsigned long timeout) 157 { 158 return wait_for_common(x, timeout, TASK_UNINTERRUPTIBLE); 159 } 160 EXPORT_SYMBOL(wait_for_completion_timeout); 161 162 /** 163 * wait_for_completion_io: - waits for completion of a task 164 * @x: holds the state of this particular completion 165 * 166 * This waits to be signaled for completion of a specific task. It is NOT 167 * interruptible and there is no timeout. The caller is accounted as waiting 168 * for IO (which traditionally means blkio only). 169 */ 170 void __sched wait_for_completion_io(struct completion *x) 171 { 172 wait_for_common_io(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE); 173 } 174 EXPORT_SYMBOL(wait_for_completion_io); 175 176 /** 177 * wait_for_completion_io_timeout: - waits for completion of a task (w/timeout) 178 * @x: holds the state of this particular completion 179 * @timeout: timeout value in jiffies 180 * 181 * This waits for either a completion of a specific task to be signaled or for a 182 * specified timeout to expire. The timeout is in jiffies. It is not 183 * interruptible. The caller is accounted as waiting for IO (which traditionally 184 * means blkio only). 185 * 186 * Return: 0 if timed out, and positive (at least 1, or number of jiffies left 187 * till timeout) if completed. 188 */ 189 unsigned long __sched 190 wait_for_completion_io_timeout(struct completion *x, unsigned long timeout) 191 { 192 return wait_for_common_io(x, timeout, TASK_UNINTERRUPTIBLE); 193 } 194 EXPORT_SYMBOL(wait_for_completion_io_timeout); 195 196 /** 197 * wait_for_completion_interruptible: - waits for completion of a task (w/intr) 198 * @x: holds the state of this particular completion 199 * 200 * This waits for completion of a specific task to be signaled. It is 201 * interruptible. 202 * 203 * Return: -ERESTARTSYS if interrupted, 0 if completed. 204 */ 205 int __sched wait_for_completion_interruptible(struct completion *x) 206 { 207 long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_INTERRUPTIBLE); 208 if (t == -ERESTARTSYS) 209 return t; 210 return 0; 211 } 212 EXPORT_SYMBOL(wait_for_completion_interruptible); 213 214 /** 215 * wait_for_completion_interruptible_timeout: - waits for completion (w/(to,intr)) 216 * @x: holds the state of this particular completion 217 * @timeout: timeout value in jiffies 218 * 219 * This waits for either a completion of a specific task to be signaled or for a 220 * specified timeout to expire. It is interruptible. The timeout is in jiffies. 221 * 222 * Return: -ERESTARTSYS if interrupted, 0 if timed out, positive (at least 1, 223 * or number of jiffies left till timeout) if completed. 224 */ 225 long __sched 226 wait_for_completion_interruptible_timeout(struct completion *x, 227 unsigned long timeout) 228 { 229 return wait_for_common(x, timeout, TASK_INTERRUPTIBLE); 230 } 231 EXPORT_SYMBOL(wait_for_completion_interruptible_timeout); 232 233 /** 234 * wait_for_completion_killable: - waits for completion of a task (killable) 235 * @x: holds the state of this particular completion 236 * 237 * This waits to be signaled for completion of a specific task. It can be 238 * interrupted by a kill signal. 239 * 240 * Return: -ERESTARTSYS if interrupted, 0 if completed. 241 */ 242 int __sched wait_for_completion_killable(struct completion *x) 243 { 244 long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_KILLABLE); 245 if (t == -ERESTARTSYS) 246 return t; 247 return 0; 248 } 249 EXPORT_SYMBOL(wait_for_completion_killable); 250 251 /** 252 * wait_for_completion_killable_timeout: - waits for completion of a task (w/(to,killable)) 253 * @x: holds the state of this particular completion 254 * @timeout: timeout value in jiffies 255 * 256 * This waits for either a completion of a specific task to be 257 * signaled or for a specified timeout to expire. It can be 258 * interrupted by a kill signal. The timeout is in jiffies. 259 * 260 * Return: -ERESTARTSYS if interrupted, 0 if timed out, positive (at least 1, 261 * or number of jiffies left till timeout) if completed. 262 */ 263 long __sched 264 wait_for_completion_killable_timeout(struct completion *x, 265 unsigned long timeout) 266 { 267 return wait_for_common(x, timeout, TASK_KILLABLE); 268 } 269 EXPORT_SYMBOL(wait_for_completion_killable_timeout); 270 271 /** 272 * try_wait_for_completion - try to decrement a completion without blocking 273 * @x: completion structure 274 * 275 * Return: 0 if a decrement cannot be done without blocking 276 * 1 if a decrement succeeded. 277 * 278 * If a completion is being used as a counting completion, 279 * attempt to decrement the counter without blocking. This 280 * enables us to avoid waiting if the resource the completion 281 * is protecting is not available. 282 */ 283 bool try_wait_for_completion(struct completion *x) 284 { 285 unsigned long flags; 286 int ret = 1; 287 288 /* 289 * Since x->done will need to be locked only 290 * in the non-blocking case, we check x->done 291 * first without taking the lock so we can 292 * return early in the blocking case. 293 */ 294 if (!READ_ONCE(x->done)) 295 return 0; 296 297 spin_lock_irqsave(&x->wait.lock, flags); 298 if (!x->done) 299 ret = 0; 300 else if (x->done != UINT_MAX) 301 x->done--; 302 spin_unlock_irqrestore(&x->wait.lock, flags); 303 return ret; 304 } 305 EXPORT_SYMBOL(try_wait_for_completion); 306 307 /** 308 * completion_done - Test to see if a completion has any waiters 309 * @x: completion structure 310 * 311 * Return: 0 if there are waiters (wait_for_completion() in progress) 312 * 1 if there are no waiters. 313 * 314 * Note, this will always return true if complete_all() was called on @X. 315 */ 316 bool completion_done(struct completion *x) 317 { 318 unsigned long flags; 319 320 if (!READ_ONCE(x->done)) 321 return false; 322 323 /* 324 * If ->done, we need to wait for complete() to release ->wait.lock 325 * otherwise we can end up freeing the completion before complete() 326 * is done referencing it. 327 */ 328 spin_lock_irqsave(&x->wait.lock, flags); 329 spin_unlock_irqrestore(&x->wait.lock, flags); 330 return true; 331 } 332 EXPORT_SYMBOL(completion_done); 333