1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Generic wait-for-completion handler; 4 * 5 * It differs from semaphores in that their default case is the opposite, 6 * wait_for_completion default blocks whereas semaphore default non-block. The 7 * interface also makes it easy to 'complete' multiple waiting threads, 8 * something which isn't entirely natural for semaphores. 9 * 10 * But more importantly, the primitive documents the usage. Semaphores would 11 * typically be used for exclusion which gives rise to priority inversion. 12 * Waiting for completion is a typically sync point, but not an exclusion point. 13 */ 14 #include "sched.h" 15 16 /** 17 * complete: - signals a single thread waiting on this completion 18 * @x: holds the state of this particular completion 19 * 20 * This will wake up a single thread waiting on this completion. Threads will be 21 * awakened in the same order in which they were queued. 22 * 23 * See also complete_all(), wait_for_completion() and related routines. 24 * 25 * If this function wakes up a task, it executes a full memory barrier before 26 * accessing the task state. 27 */ 28 void complete(struct completion *x) 29 { 30 unsigned long flags; 31 32 spin_lock_irqsave(&x->wait.lock, flags); 33 34 if (x->done != UINT_MAX) 35 x->done++; 36 __wake_up_locked(&x->wait, TASK_NORMAL, 1); 37 spin_unlock_irqrestore(&x->wait.lock, flags); 38 } 39 EXPORT_SYMBOL(complete); 40 41 /** 42 * complete_all: - signals all threads waiting on this completion 43 * @x: holds the state of this particular completion 44 * 45 * This will wake up all threads waiting on this particular completion event. 46 * 47 * If this function wakes up a task, it executes a full memory barrier before 48 * accessing the task state. 49 * 50 * Since complete_all() sets the completion of @x permanently to done 51 * to allow multiple waiters to finish, a call to reinit_completion() 52 * must be used on @x if @x is to be used again. The code must make 53 * sure that all waiters have woken and finished before reinitializing 54 * @x. Also note that the function completion_done() can not be used 55 * to know if there are still waiters after complete_all() has been called. 56 */ 57 void complete_all(struct completion *x) 58 { 59 unsigned long flags; 60 61 spin_lock_irqsave(&x->wait.lock, flags); 62 x->done = UINT_MAX; 63 __wake_up_locked(&x->wait, TASK_NORMAL, 0); 64 spin_unlock_irqrestore(&x->wait.lock, flags); 65 } 66 EXPORT_SYMBOL(complete_all); 67 68 static inline long __sched 69 do_wait_for_common(struct completion *x, 70 long (*action)(long), long timeout, int state) 71 { 72 if (!x->done) { 73 DECLARE_WAITQUEUE(wait, current); 74 75 __add_wait_queue_entry_tail_exclusive(&x->wait, &wait); 76 do { 77 if (signal_pending_state(state, current)) { 78 timeout = -ERESTARTSYS; 79 break; 80 } 81 __set_current_state(state); 82 spin_unlock_irq(&x->wait.lock); 83 timeout = action(timeout); 84 spin_lock_irq(&x->wait.lock); 85 } while (!x->done && timeout); 86 __remove_wait_queue(&x->wait, &wait); 87 if (!x->done) 88 return timeout; 89 } 90 if (x->done != UINT_MAX) 91 x->done--; 92 return timeout ?: 1; 93 } 94 95 static inline long __sched 96 __wait_for_common(struct completion *x, 97 long (*action)(long), long timeout, int state) 98 { 99 might_sleep(); 100 101 complete_acquire(x); 102 103 spin_lock_irq(&x->wait.lock); 104 timeout = do_wait_for_common(x, action, timeout, state); 105 spin_unlock_irq(&x->wait.lock); 106 107 complete_release(x); 108 109 return timeout; 110 } 111 112 static long __sched 113 wait_for_common(struct completion *x, long timeout, int state) 114 { 115 return __wait_for_common(x, schedule_timeout, timeout, state); 116 } 117 118 static long __sched 119 wait_for_common_io(struct completion *x, long timeout, int state) 120 { 121 return __wait_for_common(x, io_schedule_timeout, timeout, state); 122 } 123 124 /** 125 * wait_for_completion: - waits for completion of a task 126 * @x: holds the state of this particular completion 127 * 128 * This waits to be signaled for completion of a specific task. It is NOT 129 * interruptible and there is no timeout. 130 * 131 * See also similar routines (i.e. wait_for_completion_timeout()) with timeout 132 * and interrupt capability. Also see complete(). 133 */ 134 void __sched wait_for_completion(struct completion *x) 135 { 136 wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE); 137 } 138 EXPORT_SYMBOL(wait_for_completion); 139 140 /** 141 * wait_for_completion_timeout: - waits for completion of a task (w/timeout) 142 * @x: holds the state of this particular completion 143 * @timeout: timeout value in jiffies 144 * 145 * This waits for either a completion of a specific task to be signaled or for a 146 * specified timeout to expire. The timeout is in jiffies. It is not 147 * interruptible. 148 * 149 * Return: 0 if timed out, and positive (at least 1, or number of jiffies left 150 * till timeout) if completed. 151 */ 152 unsigned long __sched 153 wait_for_completion_timeout(struct completion *x, unsigned long timeout) 154 { 155 return wait_for_common(x, timeout, TASK_UNINTERRUPTIBLE); 156 } 157 EXPORT_SYMBOL(wait_for_completion_timeout); 158 159 /** 160 * wait_for_completion_io: - waits for completion of a task 161 * @x: holds the state of this particular completion 162 * 163 * This waits to be signaled for completion of a specific task. It is NOT 164 * interruptible and there is no timeout. The caller is accounted as waiting 165 * for IO (which traditionally means blkio only). 166 */ 167 void __sched wait_for_completion_io(struct completion *x) 168 { 169 wait_for_common_io(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE); 170 } 171 EXPORT_SYMBOL(wait_for_completion_io); 172 173 /** 174 * wait_for_completion_io_timeout: - waits for completion of a task (w/timeout) 175 * @x: holds the state of this particular completion 176 * @timeout: timeout value in jiffies 177 * 178 * This waits for either a completion of a specific task to be signaled or for a 179 * specified timeout to expire. The timeout is in jiffies. It is not 180 * interruptible. The caller is accounted as waiting for IO (which traditionally 181 * means blkio only). 182 * 183 * Return: 0 if timed out, and positive (at least 1, or number of jiffies left 184 * till timeout) if completed. 185 */ 186 unsigned long __sched 187 wait_for_completion_io_timeout(struct completion *x, unsigned long timeout) 188 { 189 return wait_for_common_io(x, timeout, TASK_UNINTERRUPTIBLE); 190 } 191 EXPORT_SYMBOL(wait_for_completion_io_timeout); 192 193 /** 194 * wait_for_completion_interruptible: - waits for completion of a task (w/intr) 195 * @x: holds the state of this particular completion 196 * 197 * This waits for completion of a specific task to be signaled. It is 198 * interruptible. 199 * 200 * Return: -ERESTARTSYS if interrupted, 0 if completed. 201 */ 202 int __sched wait_for_completion_interruptible(struct completion *x) 203 { 204 long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_INTERRUPTIBLE); 205 if (t == -ERESTARTSYS) 206 return t; 207 return 0; 208 } 209 EXPORT_SYMBOL(wait_for_completion_interruptible); 210 211 /** 212 * wait_for_completion_interruptible_timeout: - waits for completion (w/(to,intr)) 213 * @x: holds the state of this particular completion 214 * @timeout: timeout value in jiffies 215 * 216 * This waits for either a completion of a specific task to be signaled or for a 217 * specified timeout to expire. It is interruptible. The timeout is in jiffies. 218 * 219 * Return: -ERESTARTSYS if interrupted, 0 if timed out, positive (at least 1, 220 * or number of jiffies left till timeout) if completed. 221 */ 222 long __sched 223 wait_for_completion_interruptible_timeout(struct completion *x, 224 unsigned long timeout) 225 { 226 return wait_for_common(x, timeout, TASK_INTERRUPTIBLE); 227 } 228 EXPORT_SYMBOL(wait_for_completion_interruptible_timeout); 229 230 /** 231 * wait_for_completion_killable: - waits for completion of a task (killable) 232 * @x: holds the state of this particular completion 233 * 234 * This waits to be signaled for completion of a specific task. It can be 235 * interrupted by a kill signal. 236 * 237 * Return: -ERESTARTSYS if interrupted, 0 if completed. 238 */ 239 int __sched wait_for_completion_killable(struct completion *x) 240 { 241 long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_KILLABLE); 242 if (t == -ERESTARTSYS) 243 return t; 244 return 0; 245 } 246 EXPORT_SYMBOL(wait_for_completion_killable); 247 248 /** 249 * wait_for_completion_killable_timeout: - waits for completion of a task (w/(to,killable)) 250 * @x: holds the state of this particular completion 251 * @timeout: timeout value in jiffies 252 * 253 * This waits for either a completion of a specific task to be 254 * signaled or for a specified timeout to expire. It can be 255 * interrupted by a kill signal. The timeout is in jiffies. 256 * 257 * Return: -ERESTARTSYS if interrupted, 0 if timed out, positive (at least 1, 258 * or number of jiffies left till timeout) if completed. 259 */ 260 long __sched 261 wait_for_completion_killable_timeout(struct completion *x, 262 unsigned long timeout) 263 { 264 return wait_for_common(x, timeout, TASK_KILLABLE); 265 } 266 EXPORT_SYMBOL(wait_for_completion_killable_timeout); 267 268 /** 269 * try_wait_for_completion - try to decrement a completion without blocking 270 * @x: completion structure 271 * 272 * Return: 0 if a decrement cannot be done without blocking 273 * 1 if a decrement succeeded. 274 * 275 * If a completion is being used as a counting completion, 276 * attempt to decrement the counter without blocking. This 277 * enables us to avoid waiting if the resource the completion 278 * is protecting is not available. 279 */ 280 bool try_wait_for_completion(struct completion *x) 281 { 282 unsigned long flags; 283 bool ret = true; 284 285 /* 286 * Since x->done will need to be locked only 287 * in the non-blocking case, we check x->done 288 * first without taking the lock so we can 289 * return early in the blocking case. 290 */ 291 if (!READ_ONCE(x->done)) 292 return false; 293 294 spin_lock_irqsave(&x->wait.lock, flags); 295 if (!x->done) 296 ret = false; 297 else if (x->done != UINT_MAX) 298 x->done--; 299 spin_unlock_irqrestore(&x->wait.lock, flags); 300 return ret; 301 } 302 EXPORT_SYMBOL(try_wait_for_completion); 303 304 /** 305 * completion_done - Test to see if a completion has any waiters 306 * @x: completion structure 307 * 308 * Return: 0 if there are waiters (wait_for_completion() in progress) 309 * 1 if there are no waiters. 310 * 311 * Note, this will always return true if complete_all() was called on @X. 312 */ 313 bool completion_done(struct completion *x) 314 { 315 unsigned long flags; 316 317 if (!READ_ONCE(x->done)) 318 return false; 319 320 /* 321 * If ->done, we need to wait for complete() to release ->wait.lock 322 * otherwise we can end up freeing the completion before complete() 323 * is done referencing it. 324 */ 325 spin_lock_irqsave(&x->wait.lock, flags); 326 spin_unlock_irqrestore(&x->wait.lock, flags); 327 return true; 328 } 329 EXPORT_SYMBOL(completion_done); 330