1// SPDX-License-Identifier: GPL-2.0
  2
  3/*
  4 * Generic wait-for-completion handler;
  5 *
  6 * It differs from semaphores in that their default case is the opposite,
  7 * wait_for_completion default blocks whereas semaphore default non-block. The
  8 * interface also makes it easy to 'complete' multiple waiting threads,
  9 * something which isn't entirely natural for semaphores.
 10 *
 11 * But more importantly, the primitive documents the usage. Semaphores would
 12 * typically be used for exclusion which gives rise to priority inversion.
 13 * Waiting for completion is a typically sync point, but not an exclusion point.
 14 */
 15
 16/**
 17 * complete: - signals a single thread waiting on this completion
 18 * @x:  holds the state of this particular completion
 19 *
 20 * This will wake up a single thread waiting on this completion. Threads will be
 21 * awakened in the same order in which they were queued.
 22 *
 23 * See also complete_all(), wait_for_completion() and related routines.
 24 *
 25 * If this function wakes up a task, it executes a full memory barrier before
 26 * accessing the task state.
 27 */
 28void complete(struct completion *x)
 29{
 30	unsigned long flags;
 31
 32	raw_spin_lock_irqsave(&x->wait.lock, flags);
 33
 34	if (x->done != UINT_MAX)
 35		x->done++;
 36	swake_up_locked(&x->wait);
 37	raw_spin_unlock_irqrestore(&x->wait.lock, flags);
 38}
 39EXPORT_SYMBOL(complete);
 40
 41/**
 42 * complete_all: - signals all threads waiting on this completion
 43 * @x:  holds the state of this particular completion
 44 *
 45 * This will wake up all threads waiting on this particular completion event.
 46 *
 47 * If this function wakes up a task, it executes a full memory barrier before
 48 * accessing the task state.
 49 *
 50 * Since complete_all() sets the completion of @x permanently to done
 51 * to allow multiple waiters to finish, a call to reinit_completion()
 52 * must be used on @x if @x is to be used again. The code must make
 53 * sure that all waiters have woken and finished before reinitializing
 54 * @x. Also note that the function completion_done() can not be used
 55 * to know if there are still waiters after complete_all() has been called.
 56 */
 57void complete_all(struct completion *x)
 58{
 59	unsigned long flags;
 60
 61	lockdep_assert_RT_in_threaded_ctx();
 62
 63	raw_spin_lock_irqsave(&x->wait.lock, flags);
 64	x->done = UINT_MAX;
 65	swake_up_all_locked(&x->wait);
 66	raw_spin_unlock_irqrestore(&x->wait.lock, flags);
 67}
 68EXPORT_SYMBOL(complete_all);
 69
 70static inline long __sched
 71do_wait_for_common(struct completion *x,
 72		   long (*action)(long), long timeout, int state)
 73{
 74	if (!x->done) {
 75		DECLARE_SWAITQUEUE(wait);
 76
 77		do {
 78			if (signal_pending_state(state, current)) {
 79				timeout = -ERESTARTSYS;
 80				break;
 81			}
 82			__prepare_to_swait(&x->wait, &wait);
 83			__set_current_state(state);
 84			raw_spin_unlock_irq(&x->wait.lock);
 85			timeout = action(timeout);
 86			raw_spin_lock_irq(&x->wait.lock);
 87		} while (!x->done && timeout);
 88		__finish_swait(&x->wait, &wait);
 89		if (!x->done)
 90			return timeout;
 91	}
 92	if (x->done != UINT_MAX)
 93		x->done--;
 94	return timeout ?: 1;
 95}
 96
 97static inline long __sched
 98__wait_for_common(struct completion *x,
 99		  long (*action)(long), long timeout, int state)
100{
101	might_sleep();
102
103	complete_acquire(x);
104
105	raw_spin_lock_irq(&x->wait.lock);
106	timeout = do_wait_for_common(x, action, timeout, state);
107	raw_spin_unlock_irq(&x->wait.lock);
108
109	complete_release(x);
110
111	return timeout;
112}
113
114static long __sched
115wait_for_common(struct completion *x, long timeout, int state)
116{
117	return __wait_for_common(x, schedule_timeout, timeout, state);
118}
119
120static long __sched
121wait_for_common_io(struct completion *x, long timeout, int state)
122{
123	return __wait_for_common(x, io_schedule_timeout, timeout, state);
124}
125
126/**
127 * wait_for_completion: - waits for completion of a task
128 * @x:  holds the state of this particular completion
129 *
130 * This waits to be signaled for completion of a specific task. It is NOT
131 * interruptible and there is no timeout.
132 *
133 * See also similar routines (i.e. wait_for_completion_timeout()) with timeout
134 * and interrupt capability. Also see complete().
135 */
136void __sched wait_for_completion(struct completion *x)
137{
138	wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE);
139}
140EXPORT_SYMBOL(wait_for_completion);
141
142/**
143 * wait_for_completion_timeout: - waits for completion of a task (w/timeout)
144 * @x:  holds the state of this particular completion
145 * @timeout:  timeout value in jiffies
146 *
147 * This waits for either a completion of a specific task to be signaled or for a
148 * specified timeout to expire. The timeout is in jiffies. It is not
149 * interruptible.
150 *
151 * Return: 0 if timed out, and positive (at least 1, or number of jiffies left
152 * till timeout) if completed.
153 */
154unsigned long __sched
155wait_for_completion_timeout(struct completion *x, unsigned long timeout)
156{
157	return wait_for_common(x, timeout, TASK_UNINTERRUPTIBLE);
158}
159EXPORT_SYMBOL(wait_for_completion_timeout);
160
161/**
162 * wait_for_completion_io: - waits for completion of a task
163 * @x:  holds the state of this particular completion
164 *
165 * This waits to be signaled for completion of a specific task. It is NOT
166 * interruptible and there is no timeout. The caller is accounted as waiting
167 * for IO (which traditionally means blkio only).
168 */
169void __sched wait_for_completion_io(struct completion *x)
170{
171	wait_for_common_io(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE);
172}
173EXPORT_SYMBOL(wait_for_completion_io);
174
175/**
176 * wait_for_completion_io_timeout: - waits for completion of a task (w/timeout)
177 * @x:  holds the state of this particular completion
178 * @timeout:  timeout value in jiffies
179 *
180 * This waits for either a completion of a specific task to be signaled or for a
181 * specified timeout to expire. The timeout is in jiffies. It is not
182 * interruptible. The caller is accounted as waiting for IO (which traditionally
183 * means blkio only).
184 *
185 * Return: 0 if timed out, and positive (at least 1, or number of jiffies left
186 * till timeout) if completed.
187 */
188unsigned long __sched
189wait_for_completion_io_timeout(struct completion *x, unsigned long timeout)
190{
191	return wait_for_common_io(x, timeout, TASK_UNINTERRUPTIBLE);
192}
193EXPORT_SYMBOL(wait_for_completion_io_timeout);
194
195/**
196 * wait_for_completion_interruptible: - waits for completion of a task (w/intr)
197 * @x:  holds the state of this particular completion
198 *
199 * This waits for completion of a specific task to be signaled. It is
200 * interruptible.
201 *
202 * Return: -ERESTARTSYS if interrupted, 0 if completed.
203 */
204int __sched wait_for_completion_interruptible(struct completion *x)
205{
206	long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_INTERRUPTIBLE);
207
208	if (t == -ERESTARTSYS)
209		return t;
210	return 0;
211}
212EXPORT_SYMBOL(wait_for_completion_interruptible);
213
214/**
215 * wait_for_completion_interruptible_timeout: - waits for completion (w/(to,intr))
216 * @x:  holds the state of this particular completion
217 * @timeout:  timeout value in jiffies
218 *
219 * This waits for either a completion of a specific task to be signaled or for a
220 * specified timeout to expire. It is interruptible. The timeout is in jiffies.
221 *
222 * Return: -ERESTARTSYS if interrupted, 0 if timed out, positive (at least 1,
223 * or number of jiffies left till timeout) if completed.
224 */
225long __sched
226wait_for_completion_interruptible_timeout(struct completion *x,
227					  unsigned long timeout)
228{
229	return wait_for_common(x, timeout, TASK_INTERRUPTIBLE);
230}
231EXPORT_SYMBOL(wait_for_completion_interruptible_timeout);
232
233/**
234 * wait_for_completion_killable: - waits for completion of a task (killable)
235 * @x:  holds the state of this particular completion
236 *
237 * This waits to be signaled for completion of a specific task. It can be
238 * interrupted by a kill signal.
239 *
240 * Return: -ERESTARTSYS if interrupted, 0 if completed.
241 */
242int __sched wait_for_completion_killable(struct completion *x)
243{
244	long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_KILLABLE);
245
246	if (t == -ERESTARTSYS)
247		return t;
248	return 0;
249}
250EXPORT_SYMBOL(wait_for_completion_killable);
251
252int __sched wait_for_completion_state(struct completion *x, unsigned int state)
253{
254	long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, state);
255
256	if (t == -ERESTARTSYS)
257		return t;
258	return 0;
259}
260EXPORT_SYMBOL(wait_for_completion_state);
261
262/**
263 * wait_for_completion_killable_timeout: - waits for completion of a task (w/(to,killable))
264 * @x:  holds the state of this particular completion
265 * @timeout:  timeout value in jiffies
266 *
267 * This waits for either a completion of a specific task to be
268 * signaled or for a specified timeout to expire. It can be
269 * interrupted by a kill signal. The timeout is in jiffies.
270 *
271 * Return: -ERESTARTSYS if interrupted, 0 if timed out, positive (at least 1,
272 * or number of jiffies left till timeout) if completed.
273 */
274long __sched
275wait_for_completion_killable_timeout(struct completion *x,
276				     unsigned long timeout)
277{
278	return wait_for_common(x, timeout, TASK_KILLABLE);
279}
280EXPORT_SYMBOL(wait_for_completion_killable_timeout);
281
282/**
283 *	try_wait_for_completion - try to decrement a completion without blocking
284 *	@x:	completion structure
285 *
286 *	Return: 0 if a decrement cannot be done without blocking
287 *		 1 if a decrement succeeded.
288 *
289 *	If a completion is being used as a counting completion,
290 *	attempt to decrement the counter without blocking. This
291 *	enables us to avoid waiting if the resource the completion
292 *	is protecting is not available.
293 */
294bool try_wait_for_completion(struct completion *x)
295{
296	unsigned long flags;
297	bool ret = true;
298
299	/*
300	 * Since x->done will need to be locked only
301	 * in the non-blocking case, we check x->done
302	 * first without taking the lock so we can
303	 * return early in the blocking case.
304	 */
305	if (!READ_ONCE(x->done))
306		return false;
307
308	raw_spin_lock_irqsave(&x->wait.lock, flags);
309	if (!x->done)
310		ret = false;
311	else if (x->done != UINT_MAX)
312		x->done--;
313	raw_spin_unlock_irqrestore(&x->wait.lock, flags);
314	return ret;
315}
316EXPORT_SYMBOL(try_wait_for_completion);
317
318/**
319 *	completion_done - Test to see if a completion has any waiters
320 *	@x:	completion structure
321 *
322 *	Return: 0 if there are waiters (wait_for_completion() in progress)
323 *		 1 if there are no waiters.
324 *
325 *	Note, this will always return true if complete_all() was called on @X.
326 */
327bool completion_done(struct completion *x)
328{
329	unsigned long flags;
330
331	if (!READ_ONCE(x->done))
332		return false;
333
334	/*
335	 * If ->done, we need to wait for complete() to release ->wait.lock
336	 * otherwise we can end up freeing the completion before complete()
337	 * is done referencing it.
338	 */
339	raw_spin_lock_irqsave(&x->wait.lock, flags);
340	raw_spin_unlock_irqrestore(&x->wait.lock, flags);
341	return true;
342}
343EXPORT_SYMBOL(completion_done);