Loading...
1/*
2 * Generic wait-for-completion handler;
3 *
4 * It differs from semaphores in that their default case is the opposite,
5 * wait_for_completion default blocks whereas semaphore default non-block. The
6 * interface also makes it easy to 'complete' multiple waiting threads,
7 * something which isn't entirely natural for semaphores.
8 *
9 * But more importantly, the primitive documents the usage. Semaphores would
10 * typically be used for exclusion which gives rise to priority inversion.
11 * Waiting for completion is a typically sync point, but not an exclusion point.
12 */
13
14#include <linux/sched.h>
15#include <linux/completion.h>
16
17/**
18 * complete: - signals a single thread waiting on this completion
19 * @x: holds the state of this particular completion
20 *
21 * This will wake up a single thread waiting on this completion. Threads will be
22 * awakened in the same order in which they were queued.
23 *
24 * See also complete_all(), wait_for_completion() and related routines.
25 *
26 * It may be assumed that this function implies a write memory barrier before
27 * changing the task state if and only if any tasks are woken up.
28 */
29void complete(struct completion *x)
30{
31 unsigned long flags;
32
33 spin_lock_irqsave(&x->wait.lock, flags);
34 x->done++;
35 __wake_up_locked(&x->wait, TASK_NORMAL, 1);
36 spin_unlock_irqrestore(&x->wait.lock, flags);
37}
38EXPORT_SYMBOL(complete);
39
40/**
41 * complete_all: - signals all threads waiting on this completion
42 * @x: holds the state of this particular completion
43 *
44 * This will wake up all threads waiting on this particular completion event.
45 *
46 * It may be assumed that this function implies a write memory barrier before
47 * changing the task state if and only if any tasks are woken up.
48 */
49void complete_all(struct completion *x)
50{
51 unsigned long flags;
52
53 spin_lock_irqsave(&x->wait.lock, flags);
54 x->done += UINT_MAX/2;
55 __wake_up_locked(&x->wait, TASK_NORMAL, 0);
56 spin_unlock_irqrestore(&x->wait.lock, flags);
57}
58EXPORT_SYMBOL(complete_all);
59
60static inline long __sched
61do_wait_for_common(struct completion *x,
62 long (*action)(long), long timeout, int state)
63{
64 if (!x->done) {
65 DECLARE_WAITQUEUE(wait, current);
66
67 __add_wait_queue_tail_exclusive(&x->wait, &wait);
68 do {
69 if (signal_pending_state(state, current)) {
70 timeout = -ERESTARTSYS;
71 break;
72 }
73 __set_current_state(state);
74 spin_unlock_irq(&x->wait.lock);
75 timeout = action(timeout);
76 spin_lock_irq(&x->wait.lock);
77 } while (!x->done && timeout);
78 __remove_wait_queue(&x->wait, &wait);
79 if (!x->done)
80 return timeout;
81 }
82 x->done--;
83 return timeout ?: 1;
84}
85
86static inline long __sched
87__wait_for_common(struct completion *x,
88 long (*action)(long), long timeout, int state)
89{
90 might_sleep();
91
92 spin_lock_irq(&x->wait.lock);
93 timeout = do_wait_for_common(x, action, timeout, state);
94 spin_unlock_irq(&x->wait.lock);
95 return timeout;
96}
97
98static long __sched
99wait_for_common(struct completion *x, long timeout, int state)
100{
101 return __wait_for_common(x, schedule_timeout, timeout, state);
102}
103
104static long __sched
105wait_for_common_io(struct completion *x, long timeout, int state)
106{
107 return __wait_for_common(x, io_schedule_timeout, timeout, state);
108}
109
110/**
111 * wait_for_completion: - waits for completion of a task
112 * @x: holds the state of this particular completion
113 *
114 * This waits to be signaled for completion of a specific task. It is NOT
115 * interruptible and there is no timeout.
116 *
117 * See also similar routines (i.e. wait_for_completion_timeout()) with timeout
118 * and interrupt capability. Also see complete().
119 */
120void __sched wait_for_completion(struct completion *x)
121{
122 wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE);
123}
124EXPORT_SYMBOL(wait_for_completion);
125
126/**
127 * wait_for_completion_timeout: - waits for completion of a task (w/timeout)
128 * @x: holds the state of this particular completion
129 * @timeout: timeout value in jiffies
130 *
131 * This waits for either a completion of a specific task to be signaled or for a
132 * specified timeout to expire. The timeout is in jiffies. It is not
133 * interruptible.
134 *
135 * Return: 0 if timed out, and positive (at least 1, or number of jiffies left
136 * till timeout) if completed.
137 */
138unsigned long __sched
139wait_for_completion_timeout(struct completion *x, unsigned long timeout)
140{
141 return wait_for_common(x, timeout, TASK_UNINTERRUPTIBLE);
142}
143EXPORT_SYMBOL(wait_for_completion_timeout);
144
145/**
146 * wait_for_completion_io: - waits for completion of a task
147 * @x: holds the state of this particular completion
148 *
149 * This waits to be signaled for completion of a specific task. It is NOT
150 * interruptible and there is no timeout. The caller is accounted as waiting
151 * for IO (which traditionally means blkio only).
152 */
153void __sched wait_for_completion_io(struct completion *x)
154{
155 wait_for_common_io(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE);
156}
157EXPORT_SYMBOL(wait_for_completion_io);
158
159/**
160 * wait_for_completion_io_timeout: - waits for completion of a task (w/timeout)
161 * @x: holds the state of this particular completion
162 * @timeout: timeout value in jiffies
163 *
164 * This waits for either a completion of a specific task to be signaled or for a
165 * specified timeout to expire. The timeout is in jiffies. It is not
166 * interruptible. The caller is accounted as waiting for IO (which traditionally
167 * means blkio only).
168 *
169 * Return: 0 if timed out, and positive (at least 1, or number of jiffies left
170 * till timeout) if completed.
171 */
172unsigned long __sched
173wait_for_completion_io_timeout(struct completion *x, unsigned long timeout)
174{
175 return wait_for_common_io(x, timeout, TASK_UNINTERRUPTIBLE);
176}
177EXPORT_SYMBOL(wait_for_completion_io_timeout);
178
179/**
180 * wait_for_completion_interruptible: - waits for completion of a task (w/intr)
181 * @x: holds the state of this particular completion
182 *
183 * This waits for completion of a specific task to be signaled. It is
184 * interruptible.
185 *
186 * Return: -ERESTARTSYS if interrupted, 0 if completed.
187 */
188int __sched wait_for_completion_interruptible(struct completion *x)
189{
190 long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_INTERRUPTIBLE);
191 if (t == -ERESTARTSYS)
192 return t;
193 return 0;
194}
195EXPORT_SYMBOL(wait_for_completion_interruptible);
196
197/**
198 * wait_for_completion_interruptible_timeout: - waits for completion (w/(to,intr))
199 * @x: holds the state of this particular completion
200 * @timeout: timeout value in jiffies
201 *
202 * This waits for either a completion of a specific task to be signaled or for a
203 * specified timeout to expire. It is interruptible. The timeout is in jiffies.
204 *
205 * Return: -ERESTARTSYS if interrupted, 0 if timed out, positive (at least 1,
206 * or number of jiffies left till timeout) if completed.
207 */
208long __sched
209wait_for_completion_interruptible_timeout(struct completion *x,
210 unsigned long timeout)
211{
212 return wait_for_common(x, timeout, TASK_INTERRUPTIBLE);
213}
214EXPORT_SYMBOL(wait_for_completion_interruptible_timeout);
215
216/**
217 * wait_for_completion_killable: - waits for completion of a task (killable)
218 * @x: holds the state of this particular completion
219 *
220 * This waits to be signaled for completion of a specific task. It can be
221 * interrupted by a kill signal.
222 *
223 * Return: -ERESTARTSYS if interrupted, 0 if completed.
224 */
225int __sched wait_for_completion_killable(struct completion *x)
226{
227 long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_KILLABLE);
228 if (t == -ERESTARTSYS)
229 return t;
230 return 0;
231}
232EXPORT_SYMBOL(wait_for_completion_killable);
233
234/**
235 * wait_for_completion_killable_timeout: - waits for completion of a task (w/(to,killable))
236 * @x: holds the state of this particular completion
237 * @timeout: timeout value in jiffies
238 *
239 * This waits for either a completion of a specific task to be
240 * signaled or for a specified timeout to expire. It can be
241 * interrupted by a kill signal. The timeout is in jiffies.
242 *
243 * Return: -ERESTARTSYS if interrupted, 0 if timed out, positive (at least 1,
244 * or number of jiffies left till timeout) if completed.
245 */
246long __sched
247wait_for_completion_killable_timeout(struct completion *x,
248 unsigned long timeout)
249{
250 return wait_for_common(x, timeout, TASK_KILLABLE);
251}
252EXPORT_SYMBOL(wait_for_completion_killable_timeout);
253
254/**
255 * try_wait_for_completion - try to decrement a completion without blocking
256 * @x: completion structure
257 *
258 * Return: 0 if a decrement cannot be done without blocking
259 * 1 if a decrement succeeded.
260 *
261 * If a completion is being used as a counting completion,
262 * attempt to decrement the counter without blocking. This
263 * enables us to avoid waiting if the resource the completion
264 * is protecting is not available.
265 */
266bool try_wait_for_completion(struct completion *x)
267{
268 unsigned long flags;
269 int ret = 1;
270
271 /*
272 * Since x->done will need to be locked only
273 * in the non-blocking case, we check x->done
274 * first without taking the lock so we can
275 * return early in the blocking case.
276 */
277 if (!READ_ONCE(x->done))
278 return 0;
279
280 spin_lock_irqsave(&x->wait.lock, flags);
281 if (!x->done)
282 ret = 0;
283 else
284 x->done--;
285 spin_unlock_irqrestore(&x->wait.lock, flags);
286 return ret;
287}
288EXPORT_SYMBOL(try_wait_for_completion);
289
290/**
291 * completion_done - Test to see if a completion has any waiters
292 * @x: completion structure
293 *
294 * Return: 0 if there are waiters (wait_for_completion() in progress)
295 * 1 if there are no waiters.
296 *
297 */
298bool completion_done(struct completion *x)
299{
300 if (!READ_ONCE(x->done))
301 return false;
302
303 /*
304 * If ->done, we need to wait for complete() to release ->wait.lock
305 * otherwise we can end up freeing the completion before complete()
306 * is done referencing it.
307 *
308 * The RMB pairs with complete()'s RELEASE of ->wait.lock and orders
309 * the loads of ->done and ->wait.lock such that we cannot observe
310 * the lock before complete() acquires it while observing the ->done
311 * after it's acquired the lock.
312 */
313 smp_rmb();
314 spin_unlock_wait(&x->wait.lock);
315 return true;
316}
317EXPORT_SYMBOL(completion_done);
1// SPDX-License-Identifier: GPL-2.0
2
3/*
4 * Generic wait-for-completion handler;
5 *
6 * It differs from semaphores in that their default case is the opposite,
7 * wait_for_completion default blocks whereas semaphore default non-block. The
8 * interface also makes it easy to 'complete' multiple waiting threads,
9 * something which isn't entirely natural for semaphores.
10 *
11 * But more importantly, the primitive documents the usage. Semaphores would
12 * typically be used for exclusion which gives rise to priority inversion.
13 * Waiting for completion is a typically sync point, but not an exclusion point.
14 */
15
16static void complete_with_flags(struct completion *x, int wake_flags)
17{
18 unsigned long flags;
19
20 raw_spin_lock_irqsave(&x->wait.lock, flags);
21
22 if (x->done != UINT_MAX)
23 x->done++;
24 swake_up_locked(&x->wait, wake_flags);
25 raw_spin_unlock_irqrestore(&x->wait.lock, flags);
26}
27
28void complete_on_current_cpu(struct completion *x)
29{
30 return complete_with_flags(x, WF_CURRENT_CPU);
31}
32
33/**
34 * complete: - signals a single thread waiting on this completion
35 * @x: holds the state of this particular completion
36 *
37 * This will wake up a single thread waiting on this completion. Threads will be
38 * awakened in the same order in which they were queued.
39 *
40 * See also complete_all(), wait_for_completion() and related routines.
41 *
42 * If this function wakes up a task, it executes a full memory barrier before
43 * accessing the task state.
44 */
45void complete(struct completion *x)
46{
47 complete_with_flags(x, 0);
48}
49EXPORT_SYMBOL(complete);
50
51/**
52 * complete_all: - signals all threads waiting on this completion
53 * @x: holds the state of this particular completion
54 *
55 * This will wake up all threads waiting on this particular completion event.
56 *
57 * If this function wakes up a task, it executes a full memory barrier before
58 * accessing the task state.
59 *
60 * Since complete_all() sets the completion of @x permanently to done
61 * to allow multiple waiters to finish, a call to reinit_completion()
62 * must be used on @x if @x is to be used again. The code must make
63 * sure that all waiters have woken and finished before reinitializing
64 * @x. Also note that the function completion_done() can not be used
65 * to know if there are still waiters after complete_all() has been called.
66 */
67void complete_all(struct completion *x)
68{
69 unsigned long flags;
70
71 lockdep_assert_RT_in_threaded_ctx();
72
73 raw_spin_lock_irqsave(&x->wait.lock, flags);
74 x->done = UINT_MAX;
75 swake_up_all_locked(&x->wait);
76 raw_spin_unlock_irqrestore(&x->wait.lock, flags);
77}
78EXPORT_SYMBOL(complete_all);
79
80static inline long __sched
81do_wait_for_common(struct completion *x,
82 long (*action)(long), long timeout, int state)
83{
84 if (!x->done) {
85 DECLARE_SWAITQUEUE(wait);
86
87 do {
88 if (signal_pending_state(state, current)) {
89 timeout = -ERESTARTSYS;
90 break;
91 }
92 __prepare_to_swait(&x->wait, &wait);
93 __set_current_state(state);
94 raw_spin_unlock_irq(&x->wait.lock);
95 timeout = action(timeout);
96 raw_spin_lock_irq(&x->wait.lock);
97 } while (!x->done && timeout);
98 __finish_swait(&x->wait, &wait);
99 if (!x->done)
100 return timeout;
101 }
102 if (x->done != UINT_MAX)
103 x->done--;
104 return timeout ?: 1;
105}
106
107static inline long __sched
108__wait_for_common(struct completion *x,
109 long (*action)(long), long timeout, int state)
110{
111 might_sleep();
112
113 complete_acquire(x);
114
115 raw_spin_lock_irq(&x->wait.lock);
116 timeout = do_wait_for_common(x, action, timeout, state);
117 raw_spin_unlock_irq(&x->wait.lock);
118
119 complete_release(x);
120
121 return timeout;
122}
123
124static long __sched
125wait_for_common(struct completion *x, long timeout, int state)
126{
127 return __wait_for_common(x, schedule_timeout, timeout, state);
128}
129
130static long __sched
131wait_for_common_io(struct completion *x, long timeout, int state)
132{
133 return __wait_for_common(x, io_schedule_timeout, timeout, state);
134}
135
136/**
137 * wait_for_completion: - waits for completion of a task
138 * @x: holds the state of this particular completion
139 *
140 * This waits to be signaled for completion of a specific task. It is NOT
141 * interruptible and there is no timeout.
142 *
143 * See also similar routines (i.e. wait_for_completion_timeout()) with timeout
144 * and interrupt capability. Also see complete().
145 */
146void __sched wait_for_completion(struct completion *x)
147{
148 wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE);
149}
150EXPORT_SYMBOL(wait_for_completion);
151
152/**
153 * wait_for_completion_timeout: - waits for completion of a task (w/timeout)
154 * @x: holds the state of this particular completion
155 * @timeout: timeout value in jiffies
156 *
157 * This waits for either a completion of a specific task to be signaled or for a
158 * specified timeout to expire. The timeout is in jiffies. It is not
159 * interruptible.
160 *
161 * Return: 0 if timed out, and positive (at least 1, or number of jiffies left
162 * till timeout) if completed.
163 */
164unsigned long __sched
165wait_for_completion_timeout(struct completion *x, unsigned long timeout)
166{
167 return wait_for_common(x, timeout, TASK_UNINTERRUPTIBLE);
168}
169EXPORT_SYMBOL(wait_for_completion_timeout);
170
171/**
172 * wait_for_completion_io: - waits for completion of a task
173 * @x: holds the state of this particular completion
174 *
175 * This waits to be signaled for completion of a specific task. It is NOT
176 * interruptible and there is no timeout. The caller is accounted as waiting
177 * for IO (which traditionally means blkio only).
178 */
179void __sched wait_for_completion_io(struct completion *x)
180{
181 wait_for_common_io(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE);
182}
183EXPORT_SYMBOL(wait_for_completion_io);
184
185/**
186 * wait_for_completion_io_timeout: - waits for completion of a task (w/timeout)
187 * @x: holds the state of this particular completion
188 * @timeout: timeout value in jiffies
189 *
190 * This waits for either a completion of a specific task to be signaled or for a
191 * specified timeout to expire. The timeout is in jiffies. It is not
192 * interruptible. The caller is accounted as waiting for IO (which traditionally
193 * means blkio only).
194 *
195 * Return: 0 if timed out, and positive (at least 1, or number of jiffies left
196 * till timeout) if completed.
197 */
198unsigned long __sched
199wait_for_completion_io_timeout(struct completion *x, unsigned long timeout)
200{
201 return wait_for_common_io(x, timeout, TASK_UNINTERRUPTIBLE);
202}
203EXPORT_SYMBOL(wait_for_completion_io_timeout);
204
205/**
206 * wait_for_completion_interruptible: - waits for completion of a task (w/intr)
207 * @x: holds the state of this particular completion
208 *
209 * This waits for completion of a specific task to be signaled. It is
210 * interruptible.
211 *
212 * Return: -ERESTARTSYS if interrupted, 0 if completed.
213 */
214int __sched wait_for_completion_interruptible(struct completion *x)
215{
216 long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_INTERRUPTIBLE);
217
218 if (t == -ERESTARTSYS)
219 return t;
220 return 0;
221}
222EXPORT_SYMBOL(wait_for_completion_interruptible);
223
224/**
225 * wait_for_completion_interruptible_timeout: - waits for completion (w/(to,intr))
226 * @x: holds the state of this particular completion
227 * @timeout: timeout value in jiffies
228 *
229 * This waits for either a completion of a specific task to be signaled or for a
230 * specified timeout to expire. It is interruptible. The timeout is in jiffies.
231 *
232 * Return: -ERESTARTSYS if interrupted, 0 if timed out, positive (at least 1,
233 * or number of jiffies left till timeout) if completed.
234 */
235long __sched
236wait_for_completion_interruptible_timeout(struct completion *x,
237 unsigned long timeout)
238{
239 return wait_for_common(x, timeout, TASK_INTERRUPTIBLE);
240}
241EXPORT_SYMBOL(wait_for_completion_interruptible_timeout);
242
243/**
244 * wait_for_completion_killable: - waits for completion of a task (killable)
245 * @x: holds the state of this particular completion
246 *
247 * This waits to be signaled for completion of a specific task. It can be
248 * interrupted by a kill signal.
249 *
250 * Return: -ERESTARTSYS if interrupted, 0 if completed.
251 */
252int __sched wait_for_completion_killable(struct completion *x)
253{
254 long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_KILLABLE);
255
256 if (t == -ERESTARTSYS)
257 return t;
258 return 0;
259}
260EXPORT_SYMBOL(wait_for_completion_killable);
261
262int __sched wait_for_completion_state(struct completion *x, unsigned int state)
263{
264 long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, state);
265
266 if (t == -ERESTARTSYS)
267 return t;
268 return 0;
269}
270EXPORT_SYMBOL(wait_for_completion_state);
271
272/**
273 * wait_for_completion_killable_timeout: - waits for completion of a task (w/(to,killable))
274 * @x: holds the state of this particular completion
275 * @timeout: timeout value in jiffies
276 *
277 * This waits for either a completion of a specific task to be
278 * signaled or for a specified timeout to expire. It can be
279 * interrupted by a kill signal. The timeout is in jiffies.
280 *
281 * Return: -ERESTARTSYS if interrupted, 0 if timed out, positive (at least 1,
282 * or number of jiffies left till timeout) if completed.
283 */
284long __sched
285wait_for_completion_killable_timeout(struct completion *x,
286 unsigned long timeout)
287{
288 return wait_for_common(x, timeout, TASK_KILLABLE);
289}
290EXPORT_SYMBOL(wait_for_completion_killable_timeout);
291
292/**
293 * try_wait_for_completion - try to decrement a completion without blocking
294 * @x: completion structure
295 *
296 * Return: 0 if a decrement cannot be done without blocking
297 * 1 if a decrement succeeded.
298 *
299 * If a completion is being used as a counting completion,
300 * attempt to decrement the counter without blocking. This
301 * enables us to avoid waiting if the resource the completion
302 * is protecting is not available.
303 */
304bool try_wait_for_completion(struct completion *x)
305{
306 unsigned long flags;
307 bool ret = true;
308
309 /*
310 * Since x->done will need to be locked only
311 * in the non-blocking case, we check x->done
312 * first without taking the lock so we can
313 * return early in the blocking case.
314 */
315 if (!READ_ONCE(x->done))
316 return false;
317
318 raw_spin_lock_irqsave(&x->wait.lock, flags);
319 if (!x->done)
320 ret = false;
321 else if (x->done != UINT_MAX)
322 x->done--;
323 raw_spin_unlock_irqrestore(&x->wait.lock, flags);
324 return ret;
325}
326EXPORT_SYMBOL(try_wait_for_completion);
327
328/**
329 * completion_done - Test to see if a completion has any waiters
330 * @x: completion structure
331 *
332 * Return: 0 if there are waiters (wait_for_completion() in progress)
333 * 1 if there are no waiters.
334 *
335 * Note, this will always return true if complete_all() was called on @X.
336 */
337bool completion_done(struct completion *x)
338{
339 unsigned long flags;
340
341 if (!READ_ONCE(x->done))
342 return false;
343
344 /*
345 * If ->done, we need to wait for complete() to release ->wait.lock
346 * otherwise we can end up freeing the completion before complete()
347 * is done referencing it.
348 */
349 raw_spin_lock_irqsave(&x->wait.lock, flags);
350 raw_spin_unlock_irqrestore(&x->wait.lock, flags);
351 return true;
352}
353EXPORT_SYMBOL(completion_done);