1// SPDX-License-Identifier: GPL-2.0
 
  2/*
  3 * Generic wait-for-completion handler;
  4 *
  5 * It differs from semaphores in that their default case is the opposite,
  6 * wait_for_completion default blocks whereas semaphore default non-block. The
  7 * interface also makes it easy to 'complete' multiple waiting threads,
  8 * something which isn't entirely natural for semaphores.
  9 *
 10 * But more importantly, the primitive documents the usage. Semaphores would
 11 * typically be used for exclusion which gives rise to priority inversion.
 12 * Waiting for completion is a typically sync point, but not an exclusion point.
 13 */
 14#include "sched.h"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 15
 16/**
 17 * complete: - signals a single thread waiting on this completion
 18 * @x:  holds the state of this particular completion
 19 *
 20 * This will wake up a single thread waiting on this completion. Threads will be
 21 * awakened in the same order in which they were queued.
 22 *
 23 * See also complete_all(), wait_for_completion() and related routines.
 24 *
 25 * It may be assumed that this function implies a write memory barrier before
 26 * changing the task state if and only if any tasks are woken up.
 27 */
 28void complete(struct completion *x)
 29{
 30	unsigned long flags;
 31
 32	spin_lock_irqsave(&x->wait.lock, flags);
 33
 34	if (x->done != UINT_MAX)
 35		x->done++;
 36	__wake_up_locked(&x->wait, TASK_NORMAL, 1);
 37	spin_unlock_irqrestore(&x->wait.lock, flags);
 38}
 39EXPORT_SYMBOL(complete);
 40
 41/**
 42 * complete_all: - signals all threads waiting on this completion
 43 * @x:  holds the state of this particular completion
 44 *
 45 * This will wake up all threads waiting on this particular completion event.
 46 *
 47 * It may be assumed that this function implies a write memory barrier before
 48 * changing the task state if and only if any tasks are woken up.
 49 *
 50 * Since complete_all() sets the completion of @x permanently to done
 51 * to allow multiple waiters to finish, a call to reinit_completion()
 52 * must be used on @x if @x is to be used again. The code must make
 53 * sure that all waiters have woken and finished before reinitializing
 54 * @x. Also note that the function completion_done() can not be used
 55 * to know if there are still waiters after complete_all() has been called.
 56 */
 57void complete_all(struct completion *x)
 58{
 59	unsigned long flags;
 60
 61	spin_lock_irqsave(&x->wait.lock, flags);
 
 
 62	x->done = UINT_MAX;
 63	__wake_up_locked(&x->wait, TASK_NORMAL, 0);
 64	spin_unlock_irqrestore(&x->wait.lock, flags);
 65}
 66EXPORT_SYMBOL(complete_all);
 67
 68static inline long __sched
 69do_wait_for_common(struct completion *x,
 70		   long (*action)(long), long timeout, int state)
 71{
 72	if (!x->done) {
 73		DECLARE_WAITQUEUE(wait, current);
 74
 75		__add_wait_queue_entry_tail_exclusive(&x->wait, &wait);
 76		do {
 77			if (signal_pending_state(state, current)) {
 78				timeout = -ERESTARTSYS;
 79				break;
 80			}
 
 81			__set_current_state(state);
 82			spin_unlock_irq(&x->wait.lock);
 83			timeout = action(timeout);
 84			spin_lock_irq(&x->wait.lock);
 85		} while (!x->done && timeout);
 86		__remove_wait_queue(&x->wait, &wait);
 87		if (!x->done)
 88			return timeout;
 89	}
 90	if (x->done != UINT_MAX)
 91		x->done--;
 92	return timeout ?: 1;
 93}
 94
 95static inline long __sched
 96__wait_for_common(struct completion *x,
 97		  long (*action)(long), long timeout, int state)
 98{
 99	might_sleep();
100
101	complete_acquire(x);
102
103	spin_lock_irq(&x->wait.lock);
104	timeout = do_wait_for_common(x, action, timeout, state);
105	spin_unlock_irq(&x->wait.lock);
106
107	complete_release(x);
108
109	return timeout;
110}
111
112static long __sched
113wait_for_common(struct completion *x, long timeout, int state)
114{
115	return __wait_for_common(x, schedule_timeout, timeout, state);
116}
117
118static long __sched
119wait_for_common_io(struct completion *x, long timeout, int state)
120{
121	return __wait_for_common(x, io_schedule_timeout, timeout, state);
122}
123
124/**
125 * wait_for_completion: - waits for completion of a task
126 * @x:  holds the state of this particular completion
127 *
128 * This waits to be signaled for completion of a specific task. It is NOT
129 * interruptible and there is no timeout.
130 *
131 * See also similar routines (i.e. wait_for_completion_timeout()) with timeout
132 * and interrupt capability. Also see complete().
133 */
134void __sched wait_for_completion(struct completion *x)
135{
136	wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE);
137}
138EXPORT_SYMBOL(wait_for_completion);
139
140/**
141 * wait_for_completion_timeout: - waits for completion of a task (w/timeout)
142 * @x:  holds the state of this particular completion
143 * @timeout:  timeout value in jiffies
144 *
145 * This waits for either a completion of a specific task to be signaled or for a
146 * specified timeout to expire. The timeout is in jiffies. It is not
147 * interruptible.
148 *
149 * Return: 0 if timed out, and positive (at least 1, or number of jiffies left
150 * till timeout) if completed.
151 */
152unsigned long __sched
153wait_for_completion_timeout(struct completion *x, unsigned long timeout)
154{
155	return wait_for_common(x, timeout, TASK_UNINTERRUPTIBLE);
156}
157EXPORT_SYMBOL(wait_for_completion_timeout);
158
159/**
160 * wait_for_completion_io: - waits for completion of a task
161 * @x:  holds the state of this particular completion
162 *
163 * This waits to be signaled for completion of a specific task. It is NOT
164 * interruptible and there is no timeout. The caller is accounted as waiting
165 * for IO (which traditionally means blkio only).
166 */
167void __sched wait_for_completion_io(struct completion *x)
168{
169	wait_for_common_io(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE);
170}
171EXPORT_SYMBOL(wait_for_completion_io);
172
173/**
174 * wait_for_completion_io_timeout: - waits for completion of a task (w/timeout)
175 * @x:  holds the state of this particular completion
176 * @timeout:  timeout value in jiffies
177 *
178 * This waits for either a completion of a specific task to be signaled or for a
179 * specified timeout to expire. The timeout is in jiffies. It is not
180 * interruptible. The caller is accounted as waiting for IO (which traditionally
181 * means blkio only).
182 *
183 * Return: 0 if timed out, and positive (at least 1, or number of jiffies left
184 * till timeout) if completed.
185 */
186unsigned long __sched
187wait_for_completion_io_timeout(struct completion *x, unsigned long timeout)
188{
189	return wait_for_common_io(x, timeout, TASK_UNINTERRUPTIBLE);
190}
191EXPORT_SYMBOL(wait_for_completion_io_timeout);
192
193/**
194 * wait_for_completion_interruptible: - waits for completion of a task (w/intr)
195 * @x:  holds the state of this particular completion
196 *
197 * This waits for completion of a specific task to be signaled. It is
198 * interruptible.
199 *
200 * Return: -ERESTARTSYS if interrupted, 0 if completed.
201 */
202int __sched wait_for_completion_interruptible(struct completion *x)
203{
204	long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_INTERRUPTIBLE);
 
205	if (t == -ERESTARTSYS)
206		return t;
207	return 0;
208}
209EXPORT_SYMBOL(wait_for_completion_interruptible);
210
211/**
212 * wait_for_completion_interruptible_timeout: - waits for completion (w/(to,intr))
213 * @x:  holds the state of this particular completion
214 * @timeout:  timeout value in jiffies
215 *
216 * This waits for either a completion of a specific task to be signaled or for a
217 * specified timeout to expire. It is interruptible. The timeout is in jiffies.
218 *
219 * Return: -ERESTARTSYS if interrupted, 0 if timed out, positive (at least 1,
220 * or number of jiffies left till timeout) if completed.
221 */
222long __sched
223wait_for_completion_interruptible_timeout(struct completion *x,
224					  unsigned long timeout)
225{
226	return wait_for_common(x, timeout, TASK_INTERRUPTIBLE);
227}
228EXPORT_SYMBOL(wait_for_completion_interruptible_timeout);
229
230/**
231 * wait_for_completion_killable: - waits for completion of a task (killable)
232 * @x:  holds the state of this particular completion
233 *
234 * This waits to be signaled for completion of a specific task. It can be
235 * interrupted by a kill signal.
236 *
237 * Return: -ERESTARTSYS if interrupted, 0 if completed.
238 */
239int __sched wait_for_completion_killable(struct completion *x)
240{
241	long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_KILLABLE);
 
242	if (t == -ERESTARTSYS)
243		return t;
244	return 0;
245}
246EXPORT_SYMBOL(wait_for_completion_killable);
247
 
 
 
 
 
 
 
 
 
 
248/**
249 * wait_for_completion_killable_timeout: - waits for completion of a task (w/(to,killable))
250 * @x:  holds the state of this particular completion
251 * @timeout:  timeout value in jiffies
252 *
253 * This waits for either a completion of a specific task to be
254 * signaled or for a specified timeout to expire. It can be
255 * interrupted by a kill signal. The timeout is in jiffies.
256 *
257 * Return: -ERESTARTSYS if interrupted, 0 if timed out, positive (at least 1,
258 * or number of jiffies left till timeout) if completed.
259 */
260long __sched
261wait_for_completion_killable_timeout(struct completion *x,
262				     unsigned long timeout)
263{
264	return wait_for_common(x, timeout, TASK_KILLABLE);
265}
266EXPORT_SYMBOL(wait_for_completion_killable_timeout);
267
268/**
269 *	try_wait_for_completion - try to decrement a completion without blocking
270 *	@x:	completion structure
271 *
272 *	Return: 0 if a decrement cannot be done without blocking
273 *		 1 if a decrement succeeded.
274 *
275 *	If a completion is being used as a counting completion,
276 *	attempt to decrement the counter without blocking. This
277 *	enables us to avoid waiting if the resource the completion
278 *	is protecting is not available.
279 */
280bool try_wait_for_completion(struct completion *x)
281{
282	unsigned long flags;
283	bool ret = true;
284
285	/*
286	 * Since x->done will need to be locked only
287	 * in the non-blocking case, we check x->done
288	 * first without taking the lock so we can
289	 * return early in the blocking case.
290	 */
291	if (!READ_ONCE(x->done))
292		return false;
293
294	spin_lock_irqsave(&x->wait.lock, flags);
295	if (!x->done)
296		ret = false;
297	else if (x->done != UINT_MAX)
298		x->done--;
299	spin_unlock_irqrestore(&x->wait.lock, flags);
300	return ret;
301}
302EXPORT_SYMBOL(try_wait_for_completion);
303
304/**
305 *	completion_done - Test to see if a completion has any waiters
306 *	@x:	completion structure
307 *
308 *	Return: 0 if there are waiters (wait_for_completion() in progress)
309 *		 1 if there are no waiters.
310 *
311 *	Note, this will always return true if complete_all() was called on @X.
312 */
313bool completion_done(struct completion *x)
314{
315	unsigned long flags;
316
317	if (!READ_ONCE(x->done))
318		return false;
319
320	/*
321	 * If ->done, we need to wait for complete() to release ->wait.lock
322	 * otherwise we can end up freeing the completion before complete()
323	 * is done referencing it.
324	 */
325	spin_lock_irqsave(&x->wait.lock, flags);
326	spin_unlock_irqrestore(&x->wait.lock, flags);
327	return true;
328}
329EXPORT_SYMBOL(completion_done);

  1// SPDX-License-Identifier: GPL-2.0
  2
  3/*
  4 * Generic wait-for-completion handler;
  5 *
  6 * It differs from semaphores in that their default case is the opposite,
  7 * wait_for_completion default blocks whereas semaphore default non-block. The
  8 * interface also makes it easy to 'complete' multiple waiting threads,
  9 * something which isn't entirely natural for semaphores.
 10 *
 11 * But more importantly, the primitive documents the usage. Semaphores would
 12 * typically be used for exclusion which gives rise to priority inversion.
 13 * Waiting for completion is a typically sync point, but not an exclusion point.
 14 */
 15
 16static void complete_with_flags(struct completion *x, int wake_flags)
 17{
 18	unsigned long flags;
 19
 20	raw_spin_lock_irqsave(&x->wait.lock, flags);
 21
 22	if (x->done != UINT_MAX)
 23		x->done++;
 24	swake_up_locked(&x->wait, wake_flags);
 25	raw_spin_unlock_irqrestore(&x->wait.lock, flags);
 26}
 27
 28void complete_on_current_cpu(struct completion *x)
 29{
 30	return complete_with_flags(x, WF_CURRENT_CPU);
 31}
 32
 33/**
 34 * complete: - signals a single thread waiting on this completion
 35 * @x:  holds the state of this particular completion
 36 *
 37 * This will wake up a single thread waiting on this completion. Threads will be
 38 * awakened in the same order in which they were queued.
 39 *
 40 * See also complete_all(), wait_for_completion() and related routines.
 41 *
 42 * If this function wakes up a task, it executes a full memory barrier before
 43 * accessing the task state.
 44 */
 45void complete(struct completion *x)
 46{
 47	complete_with_flags(x, 0);
 
 
 
 
 
 
 
 48}
 49EXPORT_SYMBOL(complete);
 50
 51/**
 52 * complete_all: - signals all threads waiting on this completion
 53 * @x:  holds the state of this particular completion
 54 *
 55 * This will wake up all threads waiting on this particular completion event.
 56 *
 57 * If this function wakes up a task, it executes a full memory barrier before
 58 * accessing the task state.
 59 *
 60 * Since complete_all() sets the completion of @x permanently to done
 61 * to allow multiple waiters to finish, a call to reinit_completion()
 62 * must be used on @x if @x is to be used again. The code must make
 63 * sure that all waiters have woken and finished before reinitializing
 64 * @x. Also note that the function completion_done() can not be used
 65 * to know if there are still waiters after complete_all() has been called.
 66 */
 67void complete_all(struct completion *x)
 68{
 69	unsigned long flags;
 70
 71	lockdep_assert_RT_in_threaded_ctx();
 72
 73	raw_spin_lock_irqsave(&x->wait.lock, flags);
 74	x->done = UINT_MAX;
 75	swake_up_all_locked(&x->wait);
 76	raw_spin_unlock_irqrestore(&x->wait.lock, flags);
 77}
 78EXPORT_SYMBOL(complete_all);
 79
 80static inline long __sched
 81do_wait_for_common(struct completion *x,
 82		   long (*action)(long), long timeout, int state)
 83{
 84	if (!x->done) {
 85		DECLARE_SWAITQUEUE(wait);
 86
 
 87		do {
 88			if (signal_pending_state(state, current)) {
 89				timeout = -ERESTARTSYS;
 90				break;
 91			}
 92			__prepare_to_swait(&x->wait, &wait);
 93			__set_current_state(state);
 94			raw_spin_unlock_irq(&x->wait.lock);
 95			timeout = action(timeout);
 96			raw_spin_lock_irq(&x->wait.lock);
 97		} while (!x->done && timeout);
 98		__finish_swait(&x->wait, &wait);
 99		if (!x->done)
100			return timeout;
101	}
102	if (x->done != UINT_MAX)
103		x->done--;
104	return timeout ?: 1;
105}
106
107static inline long __sched
108__wait_for_common(struct completion *x,
109		  long (*action)(long), long timeout, int state)
110{
111	might_sleep();
112
113	complete_acquire(x);
114
115	raw_spin_lock_irq(&x->wait.lock);
116	timeout = do_wait_for_common(x, action, timeout, state);
117	raw_spin_unlock_irq(&x->wait.lock);
118
119	complete_release(x);
120
121	return timeout;
122}
123
124static long __sched
125wait_for_common(struct completion *x, long timeout, int state)
126{
127	return __wait_for_common(x, schedule_timeout, timeout, state);
128}
129
130static long __sched
131wait_for_common_io(struct completion *x, long timeout, int state)
132{
133	return __wait_for_common(x, io_schedule_timeout, timeout, state);
134}
135
136/**
137 * wait_for_completion: - waits for completion of a task
138 * @x:  holds the state of this particular completion
139 *
140 * This waits to be signaled for completion of a specific task. It is NOT
141 * interruptible and there is no timeout.
142 *
143 * See also similar routines (i.e. wait_for_completion_timeout()) with timeout
144 * and interrupt capability. Also see complete().
145 */
146void __sched wait_for_completion(struct completion *x)
147{
148	wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE);
149}
150EXPORT_SYMBOL(wait_for_completion);
151
152/**
153 * wait_for_completion_timeout: - waits for completion of a task (w/timeout)
154 * @x:  holds the state of this particular completion
155 * @timeout:  timeout value in jiffies
156 *
157 * This waits for either a completion of a specific task to be signaled or for a
158 * specified timeout to expire. The timeout is in jiffies. It is not
159 * interruptible.
160 *
161 * Return: 0 if timed out, and positive (at least 1, or number of jiffies left
162 * till timeout) if completed.
163 */
164unsigned long __sched
165wait_for_completion_timeout(struct completion *x, unsigned long timeout)
166{
167	return wait_for_common(x, timeout, TASK_UNINTERRUPTIBLE);
168}
169EXPORT_SYMBOL(wait_for_completion_timeout);
170
171/**
172 * wait_for_completion_io: - waits for completion of a task
173 * @x:  holds the state of this particular completion
174 *
175 * This waits to be signaled for completion of a specific task. It is NOT
176 * interruptible and there is no timeout. The caller is accounted as waiting
177 * for IO (which traditionally means blkio only).
178 */
179void __sched wait_for_completion_io(struct completion *x)
180{
181	wait_for_common_io(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE);
182}
183EXPORT_SYMBOL(wait_for_completion_io);
184
185/**
186 * wait_for_completion_io_timeout: - waits for completion of a task (w/timeout)
187 * @x:  holds the state of this particular completion
188 * @timeout:  timeout value in jiffies
189 *
190 * This waits for either a completion of a specific task to be signaled or for a
191 * specified timeout to expire. The timeout is in jiffies. It is not
192 * interruptible. The caller is accounted as waiting for IO (which traditionally
193 * means blkio only).
194 *
195 * Return: 0 if timed out, and positive (at least 1, or number of jiffies left
196 * till timeout) if completed.
197 */
198unsigned long __sched
199wait_for_completion_io_timeout(struct completion *x, unsigned long timeout)
200{
201	return wait_for_common_io(x, timeout, TASK_UNINTERRUPTIBLE);
202}
203EXPORT_SYMBOL(wait_for_completion_io_timeout);
204
205/**
206 * wait_for_completion_interruptible: - waits for completion of a task (w/intr)
207 * @x:  holds the state of this particular completion
208 *
209 * This waits for completion of a specific task to be signaled. It is
210 * interruptible.
211 *
212 * Return: -ERESTARTSYS if interrupted, 0 if completed.
213 */
214int __sched wait_for_completion_interruptible(struct completion *x)
215{
216	long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_INTERRUPTIBLE);
217
218	if (t == -ERESTARTSYS)
219		return t;
220	return 0;
221}
222EXPORT_SYMBOL(wait_for_completion_interruptible);
223
224/**
225 * wait_for_completion_interruptible_timeout: - waits for completion (w/(to,intr))
226 * @x:  holds the state of this particular completion
227 * @timeout:  timeout value in jiffies
228 *
229 * This waits for either a completion of a specific task to be signaled or for a
230 * specified timeout to expire. It is interruptible. The timeout is in jiffies.
231 *
232 * Return: -ERESTARTSYS if interrupted, 0 if timed out, positive (at least 1,
233 * or number of jiffies left till timeout) if completed.
234 */
235long __sched
236wait_for_completion_interruptible_timeout(struct completion *x,
237					  unsigned long timeout)
238{
239	return wait_for_common(x, timeout, TASK_INTERRUPTIBLE);
240}
241EXPORT_SYMBOL(wait_for_completion_interruptible_timeout);
242
243/**
244 * wait_for_completion_killable: - waits for completion of a task (killable)
245 * @x:  holds the state of this particular completion
246 *
247 * This waits to be signaled for completion of a specific task. It can be
248 * interrupted by a kill signal.
249 *
250 * Return: -ERESTARTSYS if interrupted, 0 if completed.
251 */
252int __sched wait_for_completion_killable(struct completion *x)
253{
254	long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_KILLABLE);
255
256	if (t == -ERESTARTSYS)
257		return t;
258	return 0;
259}
260EXPORT_SYMBOL(wait_for_completion_killable);
261
262int __sched wait_for_completion_state(struct completion *x, unsigned int state)
263{
264	long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, state);
265
266	if (t == -ERESTARTSYS)
267		return t;
268	return 0;
269}
270EXPORT_SYMBOL(wait_for_completion_state);
271
272/**
273 * wait_for_completion_killable_timeout: - waits for completion of a task (w/(to,killable))
274 * @x:  holds the state of this particular completion
275 * @timeout:  timeout value in jiffies
276 *
277 * This waits for either a completion of a specific task to be
278 * signaled or for a specified timeout to expire. It can be
279 * interrupted by a kill signal. The timeout is in jiffies.
280 *
281 * Return: -ERESTARTSYS if interrupted, 0 if timed out, positive (at least 1,
282 * or number of jiffies left till timeout) if completed.
283 */
284long __sched
285wait_for_completion_killable_timeout(struct completion *x,
286				     unsigned long timeout)
287{
288	return wait_for_common(x, timeout, TASK_KILLABLE);
289}
290EXPORT_SYMBOL(wait_for_completion_killable_timeout);
291
292/**
293 *	try_wait_for_completion - try to decrement a completion without blocking
294 *	@x:	completion structure
295 *
296 *	Return: 0 if a decrement cannot be done without blocking
297 *		 1 if a decrement succeeded.
298 *
299 *	If a completion is being used as a counting completion,
300 *	attempt to decrement the counter without blocking. This
301 *	enables us to avoid waiting if the resource the completion
302 *	is protecting is not available.
303 */
304bool try_wait_for_completion(struct completion *x)
305{
306	unsigned long flags;
307	bool ret = true;
308
309	/*
310	 * Since x->done will need to be locked only
311	 * in the non-blocking case, we check x->done
312	 * first without taking the lock so we can
313	 * return early in the blocking case.
314	 */
315	if (!READ_ONCE(x->done))
316		return false;
317
318	raw_spin_lock_irqsave(&x->wait.lock, flags);
319	if (!x->done)
320		ret = false;
321	else if (x->done != UINT_MAX)
322		x->done--;
323	raw_spin_unlock_irqrestore(&x->wait.lock, flags);
324	return ret;
325}
326EXPORT_SYMBOL(try_wait_for_completion);
327
328/**
329 *	completion_done - Test to see if a completion has any waiters
330 *	@x:	completion structure
331 *
332 *	Return: 0 if there are waiters (wait_for_completion() in progress)
333 *		 1 if there are no waiters.
334 *
335 *	Note, this will always return true if complete_all() was called on @X.
336 */
337bool completion_done(struct completion *x)
338{
339	unsigned long flags;
340
341	if (!READ_ONCE(x->done))
342		return false;
343
344	/*
345	 * If ->done, we need to wait for complete() to release ->wait.lock
346	 * otherwise we can end up freeing the completion before complete()
347	 * is done referencing it.
348	 */
349	raw_spin_lock_irqsave(&x->wait.lock, flags);
350	raw_spin_unlock_irqrestore(&x->wait.lock, flags);
351	return true;
352}
353EXPORT_SYMBOL(completion_done);