Loading...
1// SPDX-License-Identifier: GPL-2.0
2
3/*
4 * Generic wait-for-completion handler;
5 *
6 * It differs from semaphores in that their default case is the opposite,
7 * wait_for_completion default blocks whereas semaphore default non-block. The
8 * interface also makes it easy to 'complete' multiple waiting threads,
9 * something which isn't entirely natural for semaphores.
10 *
11 * But more importantly, the primitive documents the usage. Semaphores would
12 * typically be used for exclusion which gives rise to priority inversion.
13 * Waiting for completion is a typically sync point, but not an exclusion point.
14 */
15
16/**
17 * complete: - signals a single thread waiting on this completion
18 * @x: holds the state of this particular completion
19 *
20 * This will wake up a single thread waiting on this completion. Threads will be
21 * awakened in the same order in which they were queued.
22 *
23 * See also complete_all(), wait_for_completion() and related routines.
24 *
25 * If this function wakes up a task, it executes a full memory barrier before
26 * accessing the task state.
27 */
28void complete(struct completion *x)
29{
30 unsigned long flags;
31
32 raw_spin_lock_irqsave(&x->wait.lock, flags);
33
34 if (x->done != UINT_MAX)
35 x->done++;
36 swake_up_locked(&x->wait);
37 raw_spin_unlock_irqrestore(&x->wait.lock, flags);
38}
39EXPORT_SYMBOL(complete);
40
41/**
42 * complete_all: - signals all threads waiting on this completion
43 * @x: holds the state of this particular completion
44 *
45 * This will wake up all threads waiting on this particular completion event.
46 *
47 * If this function wakes up a task, it executes a full memory barrier before
48 * accessing the task state.
49 *
50 * Since complete_all() sets the completion of @x permanently to done
51 * to allow multiple waiters to finish, a call to reinit_completion()
52 * must be used on @x if @x is to be used again. The code must make
53 * sure that all waiters have woken and finished before reinitializing
54 * @x. Also note that the function completion_done() can not be used
55 * to know if there are still waiters after complete_all() has been called.
56 */
57void complete_all(struct completion *x)
58{
59 unsigned long flags;
60
61 lockdep_assert_RT_in_threaded_ctx();
62
63 raw_spin_lock_irqsave(&x->wait.lock, flags);
64 x->done = UINT_MAX;
65 swake_up_all_locked(&x->wait);
66 raw_spin_unlock_irqrestore(&x->wait.lock, flags);
67}
68EXPORT_SYMBOL(complete_all);
69
70static inline long __sched
71do_wait_for_common(struct completion *x,
72 long (*action)(long), long timeout, int state)
73{
74 if (!x->done) {
75 DECLARE_SWAITQUEUE(wait);
76
77 do {
78 if (signal_pending_state(state, current)) {
79 timeout = -ERESTARTSYS;
80 break;
81 }
82 __prepare_to_swait(&x->wait, &wait);
83 __set_current_state(state);
84 raw_spin_unlock_irq(&x->wait.lock);
85 timeout = action(timeout);
86 raw_spin_lock_irq(&x->wait.lock);
87 } while (!x->done && timeout);
88 __finish_swait(&x->wait, &wait);
89 if (!x->done)
90 return timeout;
91 }
92 if (x->done != UINT_MAX)
93 x->done--;
94 return timeout ?: 1;
95}
96
97static inline long __sched
98__wait_for_common(struct completion *x,
99 long (*action)(long), long timeout, int state)
100{
101 might_sleep();
102
103 complete_acquire(x);
104
105 raw_spin_lock_irq(&x->wait.lock);
106 timeout = do_wait_for_common(x, action, timeout, state);
107 raw_spin_unlock_irq(&x->wait.lock);
108
109 complete_release(x);
110
111 return timeout;
112}
113
114static long __sched
115wait_for_common(struct completion *x, long timeout, int state)
116{
117 return __wait_for_common(x, schedule_timeout, timeout, state);
118}
119
120static long __sched
121wait_for_common_io(struct completion *x, long timeout, int state)
122{
123 return __wait_for_common(x, io_schedule_timeout, timeout, state);
124}
125
126/**
127 * wait_for_completion: - waits for completion of a task
128 * @x: holds the state of this particular completion
129 *
130 * This waits to be signaled for completion of a specific task. It is NOT
131 * interruptible and there is no timeout.
132 *
133 * See also similar routines (i.e. wait_for_completion_timeout()) with timeout
134 * and interrupt capability. Also see complete().
135 */
136void __sched wait_for_completion(struct completion *x)
137{
138 wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE);
139}
140EXPORT_SYMBOL(wait_for_completion);
141
142/**
143 * wait_for_completion_timeout: - waits for completion of a task (w/timeout)
144 * @x: holds the state of this particular completion
145 * @timeout: timeout value in jiffies
146 *
147 * This waits for either a completion of a specific task to be signaled or for a
148 * specified timeout to expire. The timeout is in jiffies. It is not
149 * interruptible.
150 *
151 * Return: 0 if timed out, and positive (at least 1, or number of jiffies left
152 * till timeout) if completed.
153 */
154unsigned long __sched
155wait_for_completion_timeout(struct completion *x, unsigned long timeout)
156{
157 return wait_for_common(x, timeout, TASK_UNINTERRUPTIBLE);
158}
159EXPORT_SYMBOL(wait_for_completion_timeout);
160
161/**
162 * wait_for_completion_io: - waits for completion of a task
163 * @x: holds the state of this particular completion
164 *
165 * This waits to be signaled for completion of a specific task. It is NOT
166 * interruptible and there is no timeout. The caller is accounted as waiting
167 * for IO (which traditionally means blkio only).
168 */
169void __sched wait_for_completion_io(struct completion *x)
170{
171 wait_for_common_io(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE);
172}
173EXPORT_SYMBOL(wait_for_completion_io);
174
175/**
176 * wait_for_completion_io_timeout: - waits for completion of a task (w/timeout)
177 * @x: holds the state of this particular completion
178 * @timeout: timeout value in jiffies
179 *
180 * This waits for either a completion of a specific task to be signaled or for a
181 * specified timeout to expire. The timeout is in jiffies. It is not
182 * interruptible. The caller is accounted as waiting for IO (which traditionally
183 * means blkio only).
184 *
185 * Return: 0 if timed out, and positive (at least 1, or number of jiffies left
186 * till timeout) if completed.
187 */
188unsigned long __sched
189wait_for_completion_io_timeout(struct completion *x, unsigned long timeout)
190{
191 return wait_for_common_io(x, timeout, TASK_UNINTERRUPTIBLE);
192}
193EXPORT_SYMBOL(wait_for_completion_io_timeout);
194
195/**
196 * wait_for_completion_interruptible: - waits for completion of a task (w/intr)
197 * @x: holds the state of this particular completion
198 *
199 * This waits for completion of a specific task to be signaled. It is
200 * interruptible.
201 *
202 * Return: -ERESTARTSYS if interrupted, 0 if completed.
203 */
204int __sched wait_for_completion_interruptible(struct completion *x)
205{
206 long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_INTERRUPTIBLE);
207
208 if (t == -ERESTARTSYS)
209 return t;
210 return 0;
211}
212EXPORT_SYMBOL(wait_for_completion_interruptible);
213
214/**
215 * wait_for_completion_interruptible_timeout: - waits for completion (w/(to,intr))
216 * @x: holds the state of this particular completion
217 * @timeout: timeout value in jiffies
218 *
219 * This waits for either a completion of a specific task to be signaled or for a
220 * specified timeout to expire. It is interruptible. The timeout is in jiffies.
221 *
222 * Return: -ERESTARTSYS if interrupted, 0 if timed out, positive (at least 1,
223 * or number of jiffies left till timeout) if completed.
224 */
225long __sched
226wait_for_completion_interruptible_timeout(struct completion *x,
227 unsigned long timeout)
228{
229 return wait_for_common(x, timeout, TASK_INTERRUPTIBLE);
230}
231EXPORT_SYMBOL(wait_for_completion_interruptible_timeout);
232
233/**
234 * wait_for_completion_killable: - waits for completion of a task (killable)
235 * @x: holds the state of this particular completion
236 *
237 * This waits to be signaled for completion of a specific task. It can be
238 * interrupted by a kill signal.
239 *
240 * Return: -ERESTARTSYS if interrupted, 0 if completed.
241 */
242int __sched wait_for_completion_killable(struct completion *x)
243{
244 long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_KILLABLE);
245
246 if (t == -ERESTARTSYS)
247 return t;
248 return 0;
249}
250EXPORT_SYMBOL(wait_for_completion_killable);
251
252int __sched wait_for_completion_state(struct completion *x, unsigned int state)
253{
254 long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, state);
255
256 if (t == -ERESTARTSYS)
257 return t;
258 return 0;
259}
260EXPORT_SYMBOL(wait_for_completion_state);
261
262/**
263 * wait_for_completion_killable_timeout: - waits for completion of a task (w/(to,killable))
264 * @x: holds the state of this particular completion
265 * @timeout: timeout value in jiffies
266 *
267 * This waits for either a completion of a specific task to be
268 * signaled or for a specified timeout to expire. It can be
269 * interrupted by a kill signal. The timeout is in jiffies.
270 *
271 * Return: -ERESTARTSYS if interrupted, 0 if timed out, positive (at least 1,
272 * or number of jiffies left till timeout) if completed.
273 */
274long __sched
275wait_for_completion_killable_timeout(struct completion *x,
276 unsigned long timeout)
277{
278 return wait_for_common(x, timeout, TASK_KILLABLE);
279}
280EXPORT_SYMBOL(wait_for_completion_killable_timeout);
281
282/**
283 * try_wait_for_completion - try to decrement a completion without blocking
284 * @x: completion structure
285 *
286 * Return: 0 if a decrement cannot be done without blocking
287 * 1 if a decrement succeeded.
288 *
289 * If a completion is being used as a counting completion,
290 * attempt to decrement the counter without blocking. This
291 * enables us to avoid waiting if the resource the completion
292 * is protecting is not available.
293 */
294bool try_wait_for_completion(struct completion *x)
295{
296 unsigned long flags;
297 bool ret = true;
298
299 /*
300 * Since x->done will need to be locked only
301 * in the non-blocking case, we check x->done
302 * first without taking the lock so we can
303 * return early in the blocking case.
304 */
305 if (!READ_ONCE(x->done))
306 return false;
307
308 raw_spin_lock_irqsave(&x->wait.lock, flags);
309 if (!x->done)
310 ret = false;
311 else if (x->done != UINT_MAX)
312 x->done--;
313 raw_spin_unlock_irqrestore(&x->wait.lock, flags);
314 return ret;
315}
316EXPORT_SYMBOL(try_wait_for_completion);
317
318/**
319 * completion_done - Test to see if a completion has any waiters
320 * @x: completion structure
321 *
322 * Return: 0 if there are waiters (wait_for_completion() in progress)
323 * 1 if there are no waiters.
324 *
325 * Note, this will always return true if complete_all() was called on @X.
326 */
327bool completion_done(struct completion *x)
328{
329 unsigned long flags;
330
331 if (!READ_ONCE(x->done))
332 return false;
333
334 /*
335 * If ->done, we need to wait for complete() to release ->wait.lock
336 * otherwise we can end up freeing the completion before complete()
337 * is done referencing it.
338 */
339 raw_spin_lock_irqsave(&x->wait.lock, flags);
340 raw_spin_unlock_irqrestore(&x->wait.lock, flags);
341 return true;
342}
343EXPORT_SYMBOL(completion_done);
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Generic wait-for-completion handler;
4 *
5 * It differs from semaphores in that their default case is the opposite,
6 * wait_for_completion default blocks whereas semaphore default non-block. The
7 * interface also makes it easy to 'complete' multiple waiting threads,
8 * something which isn't entirely natural for semaphores.
9 *
10 * But more importantly, the primitive documents the usage. Semaphores would
11 * typically be used for exclusion which gives rise to priority inversion.
12 * Waiting for completion is a typically sync point, but not an exclusion point.
13 */
14#include "sched.h"
15
16/**
17 * complete: - signals a single thread waiting on this completion
18 * @x: holds the state of this particular completion
19 *
20 * This will wake up a single thread waiting on this completion. Threads will be
21 * awakened in the same order in which they were queued.
22 *
23 * See also complete_all(), wait_for_completion() and related routines.
24 *
25 * If this function wakes up a task, it executes a full memory barrier before
26 * accessing the task state.
27 */
28void complete(struct completion *x)
29{
30 unsigned long flags;
31
32 spin_lock_irqsave(&x->wait.lock, flags);
33
34 if (x->done != UINT_MAX)
35 x->done++;
36 __wake_up_locked(&x->wait, TASK_NORMAL, 1);
37 spin_unlock_irqrestore(&x->wait.lock, flags);
38}
39EXPORT_SYMBOL(complete);
40
41/**
42 * complete_all: - signals all threads waiting on this completion
43 * @x: holds the state of this particular completion
44 *
45 * This will wake up all threads waiting on this particular completion event.
46 *
47 * If this function wakes up a task, it executes a full memory barrier before
48 * accessing the task state.
49 *
50 * Since complete_all() sets the completion of @x permanently to done
51 * to allow multiple waiters to finish, a call to reinit_completion()
52 * must be used on @x if @x is to be used again. The code must make
53 * sure that all waiters have woken and finished before reinitializing
54 * @x. Also note that the function completion_done() can not be used
55 * to know if there are still waiters after complete_all() has been called.
56 */
57void complete_all(struct completion *x)
58{
59 unsigned long flags;
60
61 spin_lock_irqsave(&x->wait.lock, flags);
62 x->done = UINT_MAX;
63 __wake_up_locked(&x->wait, TASK_NORMAL, 0);
64 spin_unlock_irqrestore(&x->wait.lock, flags);
65}
66EXPORT_SYMBOL(complete_all);
67
68static inline long __sched
69do_wait_for_common(struct completion *x,
70 long (*action)(long), long timeout, int state)
71{
72 if (!x->done) {
73 DECLARE_WAITQUEUE(wait, current);
74
75 __add_wait_queue_entry_tail_exclusive(&x->wait, &wait);
76 do {
77 if (signal_pending_state(state, current)) {
78 timeout = -ERESTARTSYS;
79 break;
80 }
81 __set_current_state(state);
82 spin_unlock_irq(&x->wait.lock);
83 timeout = action(timeout);
84 spin_lock_irq(&x->wait.lock);
85 } while (!x->done && timeout);
86 __remove_wait_queue(&x->wait, &wait);
87 if (!x->done)
88 return timeout;
89 }
90 if (x->done != UINT_MAX)
91 x->done--;
92 return timeout ?: 1;
93}
94
95static inline long __sched
96__wait_for_common(struct completion *x,
97 long (*action)(long), long timeout, int state)
98{
99 might_sleep();
100
101 complete_acquire(x);
102
103 spin_lock_irq(&x->wait.lock);
104 timeout = do_wait_for_common(x, action, timeout, state);
105 spin_unlock_irq(&x->wait.lock);
106
107 complete_release(x);
108
109 return timeout;
110}
111
112static long __sched
113wait_for_common(struct completion *x, long timeout, int state)
114{
115 return __wait_for_common(x, schedule_timeout, timeout, state);
116}
117
118static long __sched
119wait_for_common_io(struct completion *x, long timeout, int state)
120{
121 return __wait_for_common(x, io_schedule_timeout, timeout, state);
122}
123
124/**
125 * wait_for_completion: - waits for completion of a task
126 * @x: holds the state of this particular completion
127 *
128 * This waits to be signaled for completion of a specific task. It is NOT
129 * interruptible and there is no timeout.
130 *
131 * See also similar routines (i.e. wait_for_completion_timeout()) with timeout
132 * and interrupt capability. Also see complete().
133 */
134void __sched wait_for_completion(struct completion *x)
135{
136 wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE);
137}
138EXPORT_SYMBOL(wait_for_completion);
139
140/**
141 * wait_for_completion_timeout: - waits for completion of a task (w/timeout)
142 * @x: holds the state of this particular completion
143 * @timeout: timeout value in jiffies
144 *
145 * This waits for either a completion of a specific task to be signaled or for a
146 * specified timeout to expire. The timeout is in jiffies. It is not
147 * interruptible.
148 *
149 * Return: 0 if timed out, and positive (at least 1, or number of jiffies left
150 * till timeout) if completed.
151 */
152unsigned long __sched
153wait_for_completion_timeout(struct completion *x, unsigned long timeout)
154{
155 return wait_for_common(x, timeout, TASK_UNINTERRUPTIBLE);
156}
157EXPORT_SYMBOL(wait_for_completion_timeout);
158
159/**
160 * wait_for_completion_io: - waits for completion of a task
161 * @x: holds the state of this particular completion
162 *
163 * This waits to be signaled for completion of a specific task. It is NOT
164 * interruptible and there is no timeout. The caller is accounted as waiting
165 * for IO (which traditionally means blkio only).
166 */
167void __sched wait_for_completion_io(struct completion *x)
168{
169 wait_for_common_io(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE);
170}
171EXPORT_SYMBOL(wait_for_completion_io);
172
173/**
174 * wait_for_completion_io_timeout: - waits for completion of a task (w/timeout)
175 * @x: holds the state of this particular completion
176 * @timeout: timeout value in jiffies
177 *
178 * This waits for either a completion of a specific task to be signaled or for a
179 * specified timeout to expire. The timeout is in jiffies. It is not
180 * interruptible. The caller is accounted as waiting for IO (which traditionally
181 * means blkio only).
182 *
183 * Return: 0 if timed out, and positive (at least 1, or number of jiffies left
184 * till timeout) if completed.
185 */
186unsigned long __sched
187wait_for_completion_io_timeout(struct completion *x, unsigned long timeout)
188{
189 return wait_for_common_io(x, timeout, TASK_UNINTERRUPTIBLE);
190}
191EXPORT_SYMBOL(wait_for_completion_io_timeout);
192
193/**
194 * wait_for_completion_interruptible: - waits for completion of a task (w/intr)
195 * @x: holds the state of this particular completion
196 *
197 * This waits for completion of a specific task to be signaled. It is
198 * interruptible.
199 *
200 * Return: -ERESTARTSYS if interrupted, 0 if completed.
201 */
202int __sched wait_for_completion_interruptible(struct completion *x)
203{
204 long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_INTERRUPTIBLE);
205 if (t == -ERESTARTSYS)
206 return t;
207 return 0;
208}
209EXPORT_SYMBOL(wait_for_completion_interruptible);
210
211/**
212 * wait_for_completion_interruptible_timeout: - waits for completion (w/(to,intr))
213 * @x: holds the state of this particular completion
214 * @timeout: timeout value in jiffies
215 *
216 * This waits for either a completion of a specific task to be signaled or for a
217 * specified timeout to expire. It is interruptible. The timeout is in jiffies.
218 *
219 * Return: -ERESTARTSYS if interrupted, 0 if timed out, positive (at least 1,
220 * or number of jiffies left till timeout) if completed.
221 */
222long __sched
223wait_for_completion_interruptible_timeout(struct completion *x,
224 unsigned long timeout)
225{
226 return wait_for_common(x, timeout, TASK_INTERRUPTIBLE);
227}
228EXPORT_SYMBOL(wait_for_completion_interruptible_timeout);
229
230/**
231 * wait_for_completion_killable: - waits for completion of a task (killable)
232 * @x: holds the state of this particular completion
233 *
234 * This waits to be signaled for completion of a specific task. It can be
235 * interrupted by a kill signal.
236 *
237 * Return: -ERESTARTSYS if interrupted, 0 if completed.
238 */
239int __sched wait_for_completion_killable(struct completion *x)
240{
241 long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_KILLABLE);
242 if (t == -ERESTARTSYS)
243 return t;
244 return 0;
245}
246EXPORT_SYMBOL(wait_for_completion_killable);
247
248/**
249 * wait_for_completion_killable_timeout: - waits for completion of a task (w/(to,killable))
250 * @x: holds the state of this particular completion
251 * @timeout: timeout value in jiffies
252 *
253 * This waits for either a completion of a specific task to be
254 * signaled or for a specified timeout to expire. It can be
255 * interrupted by a kill signal. The timeout is in jiffies.
256 *
257 * Return: -ERESTARTSYS if interrupted, 0 if timed out, positive (at least 1,
258 * or number of jiffies left till timeout) if completed.
259 */
260long __sched
261wait_for_completion_killable_timeout(struct completion *x,
262 unsigned long timeout)
263{
264 return wait_for_common(x, timeout, TASK_KILLABLE);
265}
266EXPORT_SYMBOL(wait_for_completion_killable_timeout);
267
268/**
269 * try_wait_for_completion - try to decrement a completion without blocking
270 * @x: completion structure
271 *
272 * Return: 0 if a decrement cannot be done without blocking
273 * 1 if a decrement succeeded.
274 *
275 * If a completion is being used as a counting completion,
276 * attempt to decrement the counter without blocking. This
277 * enables us to avoid waiting if the resource the completion
278 * is protecting is not available.
279 */
280bool try_wait_for_completion(struct completion *x)
281{
282 unsigned long flags;
283 bool ret = true;
284
285 /*
286 * Since x->done will need to be locked only
287 * in the non-blocking case, we check x->done
288 * first without taking the lock so we can
289 * return early in the blocking case.
290 */
291 if (!READ_ONCE(x->done))
292 return false;
293
294 spin_lock_irqsave(&x->wait.lock, flags);
295 if (!x->done)
296 ret = false;
297 else if (x->done != UINT_MAX)
298 x->done--;
299 spin_unlock_irqrestore(&x->wait.lock, flags);
300 return ret;
301}
302EXPORT_SYMBOL(try_wait_for_completion);
303
304/**
305 * completion_done - Test to see if a completion has any waiters
306 * @x: completion structure
307 *
308 * Return: 0 if there are waiters (wait_for_completion() in progress)
309 * 1 if there are no waiters.
310 *
311 * Note, this will always return true if complete_all() was called on @X.
312 */
313bool completion_done(struct completion *x)
314{
315 unsigned long flags;
316
317 if (!READ_ONCE(x->done))
318 return false;
319
320 /*
321 * If ->done, we need to wait for complete() to release ->wait.lock
322 * otherwise we can end up freeing the completion before complete()
323 * is done referencing it.
324 */
325 spin_lock_irqsave(&x->wait.lock, flags);
326 spin_unlock_irqrestore(&x->wait.lock, flags);
327 return true;
328}
329EXPORT_SYMBOL(completion_done);