Loading...
1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef _LINUX_WAIT_H
3#define _LINUX_WAIT_H
4/*
5 * Linux wait queue related types and methods
6 */
7#include <linux/list.h>
8#include <linux/stddef.h>
9#include <linux/spinlock.h>
10
11#include <asm/current.h>
12#include <uapi/linux/wait.h>
13
14typedef struct wait_queue_entry wait_queue_entry_t;
15
16typedef int (*wait_queue_func_t)(struct wait_queue_entry *wq_entry, unsigned mode, int flags, void *key);
17int default_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, int flags, void *key);
18
19/* wait_queue_entry::flags */
20#define WQ_FLAG_EXCLUSIVE 0x01
21#define WQ_FLAG_WOKEN 0x02
22#define WQ_FLAG_BOOKMARK 0x04
23#define WQ_FLAG_CUSTOM 0x08
24#define WQ_FLAG_DONE 0x10
25
26/*
27 * A single wait-queue entry structure:
28 */
29struct wait_queue_entry {
30 unsigned int flags;
31 void *private;
32 wait_queue_func_t func;
33 struct list_head entry;
34};
35
36struct wait_queue_head {
37 spinlock_t lock;
38 struct list_head head;
39};
40typedef struct wait_queue_head wait_queue_head_t;
41
42struct task_struct;
43
44/*
45 * Macros for declaration and initialisaton of the datatypes
46 */
47
48#define __WAITQUEUE_INITIALIZER(name, tsk) { \
49 .private = tsk, \
50 .func = default_wake_function, \
51 .entry = { NULL, NULL } }
52
53#define DECLARE_WAITQUEUE(name, tsk) \
54 struct wait_queue_entry name = __WAITQUEUE_INITIALIZER(name, tsk)
55
56#define __WAIT_QUEUE_HEAD_INITIALIZER(name) { \
57 .lock = __SPIN_LOCK_UNLOCKED(name.lock), \
58 .head = { &(name).head, &(name).head } }
59
60#define DECLARE_WAIT_QUEUE_HEAD(name) \
61 struct wait_queue_head name = __WAIT_QUEUE_HEAD_INITIALIZER(name)
62
63extern void __init_waitqueue_head(struct wait_queue_head *wq_head, const char *name, struct lock_class_key *);
64
65#define init_waitqueue_head(wq_head) \
66 do { \
67 static struct lock_class_key __key; \
68 \
69 __init_waitqueue_head((wq_head), #wq_head, &__key); \
70 } while (0)
71
72#ifdef CONFIG_LOCKDEP
73# define __WAIT_QUEUE_HEAD_INIT_ONSTACK(name) \
74 ({ init_waitqueue_head(&name); name; })
75# define DECLARE_WAIT_QUEUE_HEAD_ONSTACK(name) \
76 struct wait_queue_head name = __WAIT_QUEUE_HEAD_INIT_ONSTACK(name)
77#else
78# define DECLARE_WAIT_QUEUE_HEAD_ONSTACK(name) DECLARE_WAIT_QUEUE_HEAD(name)
79#endif
80
81static inline void init_waitqueue_entry(struct wait_queue_entry *wq_entry, struct task_struct *p)
82{
83 wq_entry->flags = 0;
84 wq_entry->private = p;
85 wq_entry->func = default_wake_function;
86}
87
88static inline void
89init_waitqueue_func_entry(struct wait_queue_entry *wq_entry, wait_queue_func_t func)
90{
91 wq_entry->flags = 0;
92 wq_entry->private = NULL;
93 wq_entry->func = func;
94}
95
96/**
97 * waitqueue_active -- locklessly test for waiters on the queue
98 * @wq_head: the waitqueue to test for waiters
99 *
100 * returns true if the wait list is not empty
101 *
102 * NOTE: this function is lockless and requires care, incorrect usage _will_
103 * lead to sporadic and non-obvious failure.
104 *
105 * Use either while holding wait_queue_head::lock or when used for wakeups
106 * with an extra smp_mb() like::
107 *
108 * CPU0 - waker CPU1 - waiter
109 *
110 * for (;;) {
111 * @cond = true; prepare_to_wait(&wq_head, &wait, state);
112 * smp_mb(); // smp_mb() from set_current_state()
113 * if (waitqueue_active(wq_head)) if (@cond)
114 * wake_up(wq_head); break;
115 * schedule();
116 * }
117 * finish_wait(&wq_head, &wait);
118 *
119 * Because without the explicit smp_mb() it's possible for the
120 * waitqueue_active() load to get hoisted over the @cond store such that we'll
121 * observe an empty wait list while the waiter might not observe @cond.
122 *
123 * Also note that this 'optimization' trades a spin_lock() for an smp_mb(),
124 * which (when the lock is uncontended) are of roughly equal cost.
125 */
126static inline int waitqueue_active(struct wait_queue_head *wq_head)
127{
128 return !list_empty(&wq_head->head);
129}
130
131/**
132 * wq_has_single_sleeper - check if there is only one sleeper
133 * @wq_head: wait queue head
134 *
135 * Returns true of wq_head has only one sleeper on the list.
136 *
137 * Please refer to the comment for waitqueue_active.
138 */
139static inline bool wq_has_single_sleeper(struct wait_queue_head *wq_head)
140{
141 return list_is_singular(&wq_head->head);
142}
143
144/**
145 * wq_has_sleeper - check if there are any waiting processes
146 * @wq_head: wait queue head
147 *
148 * Returns true if wq_head has waiting processes
149 *
150 * Please refer to the comment for waitqueue_active.
151 */
152static inline bool wq_has_sleeper(struct wait_queue_head *wq_head)
153{
154 /*
155 * We need to be sure we are in sync with the
156 * add_wait_queue modifications to the wait queue.
157 *
158 * This memory barrier should be paired with one on the
159 * waiting side.
160 */
161 smp_mb();
162 return waitqueue_active(wq_head);
163}
164
165extern void add_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);
166extern void add_wait_queue_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);
167extern void remove_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);
168
169static inline void __add_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
170{
171 list_add(&wq_entry->entry, &wq_head->head);
172}
173
174/*
175 * Used for wake-one threads:
176 */
177static inline void
178__add_wait_queue_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
179{
180 wq_entry->flags |= WQ_FLAG_EXCLUSIVE;
181 __add_wait_queue(wq_head, wq_entry);
182}
183
184static inline void __add_wait_queue_entry_tail(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
185{
186 list_add_tail(&wq_entry->entry, &wq_head->head);
187}
188
189static inline void
190__add_wait_queue_entry_tail_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
191{
192 wq_entry->flags |= WQ_FLAG_EXCLUSIVE;
193 __add_wait_queue_entry_tail(wq_head, wq_entry);
194}
195
196static inline void
197__remove_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
198{
199 list_del(&wq_entry->entry);
200}
201
202void __wake_up(struct wait_queue_head *wq_head, unsigned int mode, int nr, void *key);
203void __wake_up_locked_key(struct wait_queue_head *wq_head, unsigned int mode, void *key);
204void __wake_up_locked_key_bookmark(struct wait_queue_head *wq_head,
205 unsigned int mode, void *key, wait_queue_entry_t *bookmark);
206void __wake_up_sync_key(struct wait_queue_head *wq_head, unsigned int mode, void *key);
207void __wake_up_locked_sync_key(struct wait_queue_head *wq_head, unsigned int mode, void *key);
208void __wake_up_locked(struct wait_queue_head *wq_head, unsigned int mode, int nr);
209void __wake_up_sync(struct wait_queue_head *wq_head, unsigned int mode);
210
211#define wake_up(x) __wake_up(x, TASK_NORMAL, 1, NULL)
212#define wake_up_nr(x, nr) __wake_up(x, TASK_NORMAL, nr, NULL)
213#define wake_up_all(x) __wake_up(x, TASK_NORMAL, 0, NULL)
214#define wake_up_locked(x) __wake_up_locked((x), TASK_NORMAL, 1)
215#define wake_up_all_locked(x) __wake_up_locked((x), TASK_NORMAL, 0)
216
217#define wake_up_interruptible(x) __wake_up(x, TASK_INTERRUPTIBLE, 1, NULL)
218#define wake_up_interruptible_nr(x, nr) __wake_up(x, TASK_INTERRUPTIBLE, nr, NULL)
219#define wake_up_interruptible_all(x) __wake_up(x, TASK_INTERRUPTIBLE, 0, NULL)
220#define wake_up_interruptible_sync(x) __wake_up_sync((x), TASK_INTERRUPTIBLE)
221
222/*
223 * Wakeup macros to be used to report events to the targets.
224 */
225#define poll_to_key(m) ((void *)(__force uintptr_t)(__poll_t)(m))
226#define key_to_poll(m) ((__force __poll_t)(uintptr_t)(void *)(m))
227#define wake_up_poll(x, m) \
228 __wake_up(x, TASK_NORMAL, 1, poll_to_key(m))
229#define wake_up_locked_poll(x, m) \
230 __wake_up_locked_key((x), TASK_NORMAL, poll_to_key(m))
231#define wake_up_interruptible_poll(x, m) \
232 __wake_up(x, TASK_INTERRUPTIBLE, 1, poll_to_key(m))
233#define wake_up_interruptible_sync_poll(x, m) \
234 __wake_up_sync_key((x), TASK_INTERRUPTIBLE, poll_to_key(m))
235#define wake_up_interruptible_sync_poll_locked(x, m) \
236 __wake_up_locked_sync_key((x), TASK_INTERRUPTIBLE, poll_to_key(m))
237
238#define ___wait_cond_timeout(condition) \
239({ \
240 bool __cond = (condition); \
241 if (__cond && !__ret) \
242 __ret = 1; \
243 __cond || !__ret; \
244})
245
246#define ___wait_is_interruptible(state) \
247 (!__builtin_constant_p(state) || \
248 state == TASK_INTERRUPTIBLE || state == TASK_KILLABLE) \
249
250extern void init_wait_entry(struct wait_queue_entry *wq_entry, int flags);
251
252/*
253 * The below macro ___wait_event() has an explicit shadow of the __ret
254 * variable when used from the wait_event_*() macros.
255 *
256 * This is so that both can use the ___wait_cond_timeout() construct
257 * to wrap the condition.
258 *
259 * The type inconsistency of the wait_event_*() __ret variable is also
260 * on purpose; we use long where we can return timeout values and int
261 * otherwise.
262 */
263
264#define ___wait_event(wq_head, condition, state, exclusive, ret, cmd) \
265({ \
266 __label__ __out; \
267 struct wait_queue_entry __wq_entry; \
268 long __ret = ret; /* explicit shadow */ \
269 \
270 init_wait_entry(&__wq_entry, exclusive ? WQ_FLAG_EXCLUSIVE : 0); \
271 for (;;) { \
272 long __int = prepare_to_wait_event(&wq_head, &__wq_entry, state);\
273 \
274 if (condition) \
275 break; \
276 \
277 if (___wait_is_interruptible(state) && __int) { \
278 __ret = __int; \
279 goto __out; \
280 } \
281 \
282 cmd; \
283 } \
284 finish_wait(&wq_head, &__wq_entry); \
285__out: __ret; \
286})
287
288#define __wait_event(wq_head, condition) \
289 (void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 0, 0, \
290 schedule())
291
292/**
293 * wait_event - sleep until a condition gets true
294 * @wq_head: the waitqueue to wait on
295 * @condition: a C expression for the event to wait for
296 *
297 * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
298 * @condition evaluates to true. The @condition is checked each time
299 * the waitqueue @wq_head is woken up.
300 *
301 * wake_up() has to be called after changing any variable that could
302 * change the result of the wait condition.
303 */
304#define wait_event(wq_head, condition) \
305do { \
306 might_sleep(); \
307 if (condition) \
308 break; \
309 __wait_event(wq_head, condition); \
310} while (0)
311
312#define __io_wait_event(wq_head, condition) \
313 (void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 0, 0, \
314 io_schedule())
315
316/*
317 * io_wait_event() -- like wait_event() but with io_schedule()
318 */
319#define io_wait_event(wq_head, condition) \
320do { \
321 might_sleep(); \
322 if (condition) \
323 break; \
324 __io_wait_event(wq_head, condition); \
325} while (0)
326
327#define __wait_event_freezable(wq_head, condition) \
328 ___wait_event(wq_head, condition, TASK_INTERRUPTIBLE, 0, 0, \
329 freezable_schedule())
330
331/**
332 * wait_event_freezable - sleep (or freeze) until a condition gets true
333 * @wq_head: the waitqueue to wait on
334 * @condition: a C expression for the event to wait for
335 *
336 * The process is put to sleep (TASK_INTERRUPTIBLE -- so as not to contribute
337 * to system load) until the @condition evaluates to true. The
338 * @condition is checked each time the waitqueue @wq_head is woken up.
339 *
340 * wake_up() has to be called after changing any variable that could
341 * change the result of the wait condition.
342 */
343#define wait_event_freezable(wq_head, condition) \
344({ \
345 int __ret = 0; \
346 might_sleep(); \
347 if (!(condition)) \
348 __ret = __wait_event_freezable(wq_head, condition); \
349 __ret; \
350})
351
352#define __wait_event_timeout(wq_head, condition, timeout) \
353 ___wait_event(wq_head, ___wait_cond_timeout(condition), \
354 TASK_UNINTERRUPTIBLE, 0, timeout, \
355 __ret = schedule_timeout(__ret))
356
357/**
358 * wait_event_timeout - sleep until a condition gets true or a timeout elapses
359 * @wq_head: the waitqueue to wait on
360 * @condition: a C expression for the event to wait for
361 * @timeout: timeout, in jiffies
362 *
363 * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
364 * @condition evaluates to true. The @condition is checked each time
365 * the waitqueue @wq_head is woken up.
366 *
367 * wake_up() has to be called after changing any variable that could
368 * change the result of the wait condition.
369 *
370 * Returns:
371 * 0 if the @condition evaluated to %false after the @timeout elapsed,
372 * 1 if the @condition evaluated to %true after the @timeout elapsed,
373 * or the remaining jiffies (at least 1) if the @condition evaluated
374 * to %true before the @timeout elapsed.
375 */
376#define wait_event_timeout(wq_head, condition, timeout) \
377({ \
378 long __ret = timeout; \
379 might_sleep(); \
380 if (!___wait_cond_timeout(condition)) \
381 __ret = __wait_event_timeout(wq_head, condition, timeout); \
382 __ret; \
383})
384
385#define __wait_event_freezable_timeout(wq_head, condition, timeout) \
386 ___wait_event(wq_head, ___wait_cond_timeout(condition), \
387 TASK_INTERRUPTIBLE, 0, timeout, \
388 __ret = freezable_schedule_timeout(__ret))
389
390/*
391 * like wait_event_timeout() -- except it uses TASK_INTERRUPTIBLE to avoid
392 * increasing load and is freezable.
393 */
394#define wait_event_freezable_timeout(wq_head, condition, timeout) \
395({ \
396 long __ret = timeout; \
397 might_sleep(); \
398 if (!___wait_cond_timeout(condition)) \
399 __ret = __wait_event_freezable_timeout(wq_head, condition, timeout); \
400 __ret; \
401})
402
403#define __wait_event_exclusive_cmd(wq_head, condition, cmd1, cmd2) \
404 (void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 1, 0, \
405 cmd1; schedule(); cmd2)
406/*
407 * Just like wait_event_cmd(), except it sets exclusive flag
408 */
409#define wait_event_exclusive_cmd(wq_head, condition, cmd1, cmd2) \
410do { \
411 if (condition) \
412 break; \
413 __wait_event_exclusive_cmd(wq_head, condition, cmd1, cmd2); \
414} while (0)
415
416#define __wait_event_cmd(wq_head, condition, cmd1, cmd2) \
417 (void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 0, 0, \
418 cmd1; schedule(); cmd2)
419
420/**
421 * wait_event_cmd - sleep until a condition gets true
422 * @wq_head: the waitqueue to wait on
423 * @condition: a C expression for the event to wait for
424 * @cmd1: the command will be executed before sleep
425 * @cmd2: the command will be executed after sleep
426 *
427 * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
428 * @condition evaluates to true. The @condition is checked each time
429 * the waitqueue @wq_head is woken up.
430 *
431 * wake_up() has to be called after changing any variable that could
432 * change the result of the wait condition.
433 */
434#define wait_event_cmd(wq_head, condition, cmd1, cmd2) \
435do { \
436 if (condition) \
437 break; \
438 __wait_event_cmd(wq_head, condition, cmd1, cmd2); \
439} while (0)
440
441#define __wait_event_interruptible(wq_head, condition) \
442 ___wait_event(wq_head, condition, TASK_INTERRUPTIBLE, 0, 0, \
443 schedule())
444
445/**
446 * wait_event_interruptible - sleep until a condition gets true
447 * @wq_head: the waitqueue to wait on
448 * @condition: a C expression for the event to wait for
449 *
450 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
451 * @condition evaluates to true or a signal is received.
452 * The @condition is checked each time the waitqueue @wq_head is woken up.
453 *
454 * wake_up() has to be called after changing any variable that could
455 * change the result of the wait condition.
456 *
457 * The function will return -ERESTARTSYS if it was interrupted by a
458 * signal and 0 if @condition evaluated to true.
459 */
460#define wait_event_interruptible(wq_head, condition) \
461({ \
462 int __ret = 0; \
463 might_sleep(); \
464 if (!(condition)) \
465 __ret = __wait_event_interruptible(wq_head, condition); \
466 __ret; \
467})
468
469#define __wait_event_interruptible_timeout(wq_head, condition, timeout) \
470 ___wait_event(wq_head, ___wait_cond_timeout(condition), \
471 TASK_INTERRUPTIBLE, 0, timeout, \
472 __ret = schedule_timeout(__ret))
473
474/**
475 * wait_event_interruptible_timeout - sleep until a condition gets true or a timeout elapses
476 * @wq_head: the waitqueue to wait on
477 * @condition: a C expression for the event to wait for
478 * @timeout: timeout, in jiffies
479 *
480 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
481 * @condition evaluates to true or a signal is received.
482 * The @condition is checked each time the waitqueue @wq_head is woken up.
483 *
484 * wake_up() has to be called after changing any variable that could
485 * change the result of the wait condition.
486 *
487 * Returns:
488 * 0 if the @condition evaluated to %false after the @timeout elapsed,
489 * 1 if the @condition evaluated to %true after the @timeout elapsed,
490 * the remaining jiffies (at least 1) if the @condition evaluated
491 * to %true before the @timeout elapsed, or -%ERESTARTSYS if it was
492 * interrupted by a signal.
493 */
494#define wait_event_interruptible_timeout(wq_head, condition, timeout) \
495({ \
496 long __ret = timeout; \
497 might_sleep(); \
498 if (!___wait_cond_timeout(condition)) \
499 __ret = __wait_event_interruptible_timeout(wq_head, \
500 condition, timeout); \
501 __ret; \
502})
503
504#define __wait_event_hrtimeout(wq_head, condition, timeout, state) \
505({ \
506 int __ret = 0; \
507 struct hrtimer_sleeper __t; \
508 \
509 hrtimer_init_sleeper_on_stack(&__t, CLOCK_MONOTONIC, \
510 HRTIMER_MODE_REL); \
511 if ((timeout) != KTIME_MAX) \
512 hrtimer_start_range_ns(&__t.timer, timeout, \
513 current->timer_slack_ns, \
514 HRTIMER_MODE_REL); \
515 \
516 __ret = ___wait_event(wq_head, condition, state, 0, 0, \
517 if (!__t.task) { \
518 __ret = -ETIME; \
519 break; \
520 } \
521 schedule()); \
522 \
523 hrtimer_cancel(&__t.timer); \
524 destroy_hrtimer_on_stack(&__t.timer); \
525 __ret; \
526})
527
528/**
529 * wait_event_hrtimeout - sleep until a condition gets true or a timeout elapses
530 * @wq_head: the waitqueue to wait on
531 * @condition: a C expression for the event to wait for
532 * @timeout: timeout, as a ktime_t
533 *
534 * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
535 * @condition evaluates to true or a signal is received.
536 * The @condition is checked each time the waitqueue @wq_head is woken up.
537 *
538 * wake_up() has to be called after changing any variable that could
539 * change the result of the wait condition.
540 *
541 * The function returns 0 if @condition became true, or -ETIME if the timeout
542 * elapsed.
543 */
544#define wait_event_hrtimeout(wq_head, condition, timeout) \
545({ \
546 int __ret = 0; \
547 might_sleep(); \
548 if (!(condition)) \
549 __ret = __wait_event_hrtimeout(wq_head, condition, timeout, \
550 TASK_UNINTERRUPTIBLE); \
551 __ret; \
552})
553
554/**
555 * wait_event_interruptible_hrtimeout - sleep until a condition gets true or a timeout elapses
556 * @wq: the waitqueue to wait on
557 * @condition: a C expression for the event to wait for
558 * @timeout: timeout, as a ktime_t
559 *
560 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
561 * @condition evaluates to true or a signal is received.
562 * The @condition is checked each time the waitqueue @wq is woken up.
563 *
564 * wake_up() has to be called after changing any variable that could
565 * change the result of the wait condition.
566 *
567 * The function returns 0 if @condition became true, -ERESTARTSYS if it was
568 * interrupted by a signal, or -ETIME if the timeout elapsed.
569 */
570#define wait_event_interruptible_hrtimeout(wq, condition, timeout) \
571({ \
572 long __ret = 0; \
573 might_sleep(); \
574 if (!(condition)) \
575 __ret = __wait_event_hrtimeout(wq, condition, timeout, \
576 TASK_INTERRUPTIBLE); \
577 __ret; \
578})
579
580#define __wait_event_interruptible_exclusive(wq, condition) \
581 ___wait_event(wq, condition, TASK_INTERRUPTIBLE, 1, 0, \
582 schedule())
583
584#define wait_event_interruptible_exclusive(wq, condition) \
585({ \
586 int __ret = 0; \
587 might_sleep(); \
588 if (!(condition)) \
589 __ret = __wait_event_interruptible_exclusive(wq, condition); \
590 __ret; \
591})
592
593#define __wait_event_killable_exclusive(wq, condition) \
594 ___wait_event(wq, condition, TASK_KILLABLE, 1, 0, \
595 schedule())
596
597#define wait_event_killable_exclusive(wq, condition) \
598({ \
599 int __ret = 0; \
600 might_sleep(); \
601 if (!(condition)) \
602 __ret = __wait_event_killable_exclusive(wq, condition); \
603 __ret; \
604})
605
606
607#define __wait_event_freezable_exclusive(wq, condition) \
608 ___wait_event(wq, condition, TASK_INTERRUPTIBLE, 1, 0, \
609 freezable_schedule())
610
611#define wait_event_freezable_exclusive(wq, condition) \
612({ \
613 int __ret = 0; \
614 might_sleep(); \
615 if (!(condition)) \
616 __ret = __wait_event_freezable_exclusive(wq, condition); \
617 __ret; \
618})
619
620/**
621 * wait_event_idle - wait for a condition without contributing to system load
622 * @wq_head: the waitqueue to wait on
623 * @condition: a C expression for the event to wait for
624 *
625 * The process is put to sleep (TASK_IDLE) until the
626 * @condition evaluates to true.
627 * The @condition is checked each time the waitqueue @wq_head is woken up.
628 *
629 * wake_up() has to be called after changing any variable that could
630 * change the result of the wait condition.
631 *
632 */
633#define wait_event_idle(wq_head, condition) \
634do { \
635 might_sleep(); \
636 if (!(condition)) \
637 ___wait_event(wq_head, condition, TASK_IDLE, 0, 0, schedule()); \
638} while (0)
639
640/**
641 * wait_event_idle_exclusive - wait for a condition with contributing to system load
642 * @wq_head: the waitqueue to wait on
643 * @condition: a C expression for the event to wait for
644 *
645 * The process is put to sleep (TASK_IDLE) until the
646 * @condition evaluates to true.
647 * The @condition is checked each time the waitqueue @wq_head is woken up.
648 *
649 * The process is put on the wait queue with an WQ_FLAG_EXCLUSIVE flag
650 * set thus if other processes wait on the same list, when this
651 * process is woken further processes are not considered.
652 *
653 * wake_up() has to be called after changing any variable that could
654 * change the result of the wait condition.
655 *
656 */
657#define wait_event_idle_exclusive(wq_head, condition) \
658do { \
659 might_sleep(); \
660 if (!(condition)) \
661 ___wait_event(wq_head, condition, TASK_IDLE, 1, 0, schedule()); \
662} while (0)
663
664#define __wait_event_idle_timeout(wq_head, condition, timeout) \
665 ___wait_event(wq_head, ___wait_cond_timeout(condition), \
666 TASK_IDLE, 0, timeout, \
667 __ret = schedule_timeout(__ret))
668
669/**
670 * wait_event_idle_timeout - sleep without load until a condition becomes true or a timeout elapses
671 * @wq_head: the waitqueue to wait on
672 * @condition: a C expression for the event to wait for
673 * @timeout: timeout, in jiffies
674 *
675 * The process is put to sleep (TASK_IDLE) until the
676 * @condition evaluates to true. The @condition is checked each time
677 * the waitqueue @wq_head is woken up.
678 *
679 * wake_up() has to be called after changing any variable that could
680 * change the result of the wait condition.
681 *
682 * Returns:
683 * 0 if the @condition evaluated to %false after the @timeout elapsed,
684 * 1 if the @condition evaluated to %true after the @timeout elapsed,
685 * or the remaining jiffies (at least 1) if the @condition evaluated
686 * to %true before the @timeout elapsed.
687 */
688#define wait_event_idle_timeout(wq_head, condition, timeout) \
689({ \
690 long __ret = timeout; \
691 might_sleep(); \
692 if (!___wait_cond_timeout(condition)) \
693 __ret = __wait_event_idle_timeout(wq_head, condition, timeout); \
694 __ret; \
695})
696
697#define __wait_event_idle_exclusive_timeout(wq_head, condition, timeout) \
698 ___wait_event(wq_head, ___wait_cond_timeout(condition), \
699 TASK_IDLE, 1, timeout, \
700 __ret = schedule_timeout(__ret))
701
702/**
703 * wait_event_idle_exclusive_timeout - sleep without load until a condition becomes true or a timeout elapses
704 * @wq_head: the waitqueue to wait on
705 * @condition: a C expression for the event to wait for
706 * @timeout: timeout, in jiffies
707 *
708 * The process is put to sleep (TASK_IDLE) until the
709 * @condition evaluates to true. The @condition is checked each time
710 * the waitqueue @wq_head is woken up.
711 *
712 * The process is put on the wait queue with an WQ_FLAG_EXCLUSIVE flag
713 * set thus if other processes wait on the same list, when this
714 * process is woken further processes are not considered.
715 *
716 * wake_up() has to be called after changing any variable that could
717 * change the result of the wait condition.
718 *
719 * Returns:
720 * 0 if the @condition evaluated to %false after the @timeout elapsed,
721 * 1 if the @condition evaluated to %true after the @timeout elapsed,
722 * or the remaining jiffies (at least 1) if the @condition evaluated
723 * to %true before the @timeout elapsed.
724 */
725#define wait_event_idle_exclusive_timeout(wq_head, condition, timeout) \
726({ \
727 long __ret = timeout; \
728 might_sleep(); \
729 if (!___wait_cond_timeout(condition)) \
730 __ret = __wait_event_idle_exclusive_timeout(wq_head, condition, timeout);\
731 __ret; \
732})
733
734extern int do_wait_intr(wait_queue_head_t *, wait_queue_entry_t *);
735extern int do_wait_intr_irq(wait_queue_head_t *, wait_queue_entry_t *);
736
737#define __wait_event_interruptible_locked(wq, condition, exclusive, fn) \
738({ \
739 int __ret; \
740 DEFINE_WAIT(__wait); \
741 if (exclusive) \
742 __wait.flags |= WQ_FLAG_EXCLUSIVE; \
743 do { \
744 __ret = fn(&(wq), &__wait); \
745 if (__ret) \
746 break; \
747 } while (!(condition)); \
748 __remove_wait_queue(&(wq), &__wait); \
749 __set_current_state(TASK_RUNNING); \
750 __ret; \
751})
752
753
754/**
755 * wait_event_interruptible_locked - sleep until a condition gets true
756 * @wq: the waitqueue to wait on
757 * @condition: a C expression for the event to wait for
758 *
759 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
760 * @condition evaluates to true or a signal is received.
761 * The @condition is checked each time the waitqueue @wq is woken up.
762 *
763 * It must be called with wq.lock being held. This spinlock is
764 * unlocked while sleeping but @condition testing is done while lock
765 * is held and when this macro exits the lock is held.
766 *
767 * The lock is locked/unlocked using spin_lock()/spin_unlock()
768 * functions which must match the way they are locked/unlocked outside
769 * of this macro.
770 *
771 * wake_up_locked() has to be called after changing any variable that could
772 * change the result of the wait condition.
773 *
774 * The function will return -ERESTARTSYS if it was interrupted by a
775 * signal and 0 if @condition evaluated to true.
776 */
777#define wait_event_interruptible_locked(wq, condition) \
778 ((condition) \
779 ? 0 : __wait_event_interruptible_locked(wq, condition, 0, do_wait_intr))
780
781/**
782 * wait_event_interruptible_locked_irq - sleep until a condition gets true
783 * @wq: the waitqueue to wait on
784 * @condition: a C expression for the event to wait for
785 *
786 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
787 * @condition evaluates to true or a signal is received.
788 * The @condition is checked each time the waitqueue @wq is woken up.
789 *
790 * It must be called with wq.lock being held. This spinlock is
791 * unlocked while sleeping but @condition testing is done while lock
792 * is held and when this macro exits the lock is held.
793 *
794 * The lock is locked/unlocked using spin_lock_irq()/spin_unlock_irq()
795 * functions which must match the way they are locked/unlocked outside
796 * of this macro.
797 *
798 * wake_up_locked() has to be called after changing any variable that could
799 * change the result of the wait condition.
800 *
801 * The function will return -ERESTARTSYS if it was interrupted by a
802 * signal and 0 if @condition evaluated to true.
803 */
804#define wait_event_interruptible_locked_irq(wq, condition) \
805 ((condition) \
806 ? 0 : __wait_event_interruptible_locked(wq, condition, 0, do_wait_intr_irq))
807
808/**
809 * wait_event_interruptible_exclusive_locked - sleep exclusively until a condition gets true
810 * @wq: the waitqueue to wait on
811 * @condition: a C expression for the event to wait for
812 *
813 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
814 * @condition evaluates to true or a signal is received.
815 * The @condition is checked each time the waitqueue @wq is woken up.
816 *
817 * It must be called with wq.lock being held. This spinlock is
818 * unlocked while sleeping but @condition testing is done while lock
819 * is held and when this macro exits the lock is held.
820 *
821 * The lock is locked/unlocked using spin_lock()/spin_unlock()
822 * functions which must match the way they are locked/unlocked outside
823 * of this macro.
824 *
825 * The process is put on the wait queue with an WQ_FLAG_EXCLUSIVE flag
826 * set thus when other process waits process on the list if this
827 * process is awaken further processes are not considered.
828 *
829 * wake_up_locked() has to be called after changing any variable that could
830 * change the result of the wait condition.
831 *
832 * The function will return -ERESTARTSYS if it was interrupted by a
833 * signal and 0 if @condition evaluated to true.
834 */
835#define wait_event_interruptible_exclusive_locked(wq, condition) \
836 ((condition) \
837 ? 0 : __wait_event_interruptible_locked(wq, condition, 1, do_wait_intr))
838
839/**
840 * wait_event_interruptible_exclusive_locked_irq - sleep until a condition gets true
841 * @wq: the waitqueue to wait on
842 * @condition: a C expression for the event to wait for
843 *
844 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
845 * @condition evaluates to true or a signal is received.
846 * The @condition is checked each time the waitqueue @wq is woken up.
847 *
848 * It must be called with wq.lock being held. This spinlock is
849 * unlocked while sleeping but @condition testing is done while lock
850 * is held and when this macro exits the lock is held.
851 *
852 * The lock is locked/unlocked using spin_lock_irq()/spin_unlock_irq()
853 * functions which must match the way they are locked/unlocked outside
854 * of this macro.
855 *
856 * The process is put on the wait queue with an WQ_FLAG_EXCLUSIVE flag
857 * set thus when other process waits process on the list if this
858 * process is awaken further processes are not considered.
859 *
860 * wake_up_locked() has to be called after changing any variable that could
861 * change the result of the wait condition.
862 *
863 * The function will return -ERESTARTSYS if it was interrupted by a
864 * signal and 0 if @condition evaluated to true.
865 */
866#define wait_event_interruptible_exclusive_locked_irq(wq, condition) \
867 ((condition) \
868 ? 0 : __wait_event_interruptible_locked(wq, condition, 1, do_wait_intr_irq))
869
870
871#define __wait_event_killable(wq, condition) \
872 ___wait_event(wq, condition, TASK_KILLABLE, 0, 0, schedule())
873
874/**
875 * wait_event_killable - sleep until a condition gets true
876 * @wq_head: the waitqueue to wait on
877 * @condition: a C expression for the event to wait for
878 *
879 * The process is put to sleep (TASK_KILLABLE) until the
880 * @condition evaluates to true or a signal is received.
881 * The @condition is checked each time the waitqueue @wq_head is woken up.
882 *
883 * wake_up() has to be called after changing any variable that could
884 * change the result of the wait condition.
885 *
886 * The function will return -ERESTARTSYS if it was interrupted by a
887 * signal and 0 if @condition evaluated to true.
888 */
889#define wait_event_killable(wq_head, condition) \
890({ \
891 int __ret = 0; \
892 might_sleep(); \
893 if (!(condition)) \
894 __ret = __wait_event_killable(wq_head, condition); \
895 __ret; \
896})
897
898#define __wait_event_killable_timeout(wq_head, condition, timeout) \
899 ___wait_event(wq_head, ___wait_cond_timeout(condition), \
900 TASK_KILLABLE, 0, timeout, \
901 __ret = schedule_timeout(__ret))
902
903/**
904 * wait_event_killable_timeout - sleep until a condition gets true or a timeout elapses
905 * @wq_head: the waitqueue to wait on
906 * @condition: a C expression for the event to wait for
907 * @timeout: timeout, in jiffies
908 *
909 * The process is put to sleep (TASK_KILLABLE) until the
910 * @condition evaluates to true or a kill signal is received.
911 * The @condition is checked each time the waitqueue @wq_head is woken up.
912 *
913 * wake_up() has to be called after changing any variable that could
914 * change the result of the wait condition.
915 *
916 * Returns:
917 * 0 if the @condition evaluated to %false after the @timeout elapsed,
918 * 1 if the @condition evaluated to %true after the @timeout elapsed,
919 * the remaining jiffies (at least 1) if the @condition evaluated
920 * to %true before the @timeout elapsed, or -%ERESTARTSYS if it was
921 * interrupted by a kill signal.
922 *
923 * Only kill signals interrupt this process.
924 */
925#define wait_event_killable_timeout(wq_head, condition, timeout) \
926({ \
927 long __ret = timeout; \
928 might_sleep(); \
929 if (!___wait_cond_timeout(condition)) \
930 __ret = __wait_event_killable_timeout(wq_head, \
931 condition, timeout); \
932 __ret; \
933})
934
935
936#define __wait_event_lock_irq(wq_head, condition, lock, cmd) \
937 (void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 0, 0, \
938 spin_unlock_irq(&lock); \
939 cmd; \
940 schedule(); \
941 spin_lock_irq(&lock))
942
943/**
944 * wait_event_lock_irq_cmd - sleep until a condition gets true. The
945 * condition is checked under the lock. This
946 * is expected to be called with the lock
947 * taken.
948 * @wq_head: the waitqueue to wait on
949 * @condition: a C expression for the event to wait for
950 * @lock: a locked spinlock_t, which will be released before cmd
951 * and schedule() and reacquired afterwards.
952 * @cmd: a command which is invoked outside the critical section before
953 * sleep
954 *
955 * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
956 * @condition evaluates to true. The @condition is checked each time
957 * the waitqueue @wq_head is woken up.
958 *
959 * wake_up() has to be called after changing any variable that could
960 * change the result of the wait condition.
961 *
962 * This is supposed to be called while holding the lock. The lock is
963 * dropped before invoking the cmd and going to sleep and is reacquired
964 * afterwards.
965 */
966#define wait_event_lock_irq_cmd(wq_head, condition, lock, cmd) \
967do { \
968 if (condition) \
969 break; \
970 __wait_event_lock_irq(wq_head, condition, lock, cmd); \
971} while (0)
972
973/**
974 * wait_event_lock_irq - sleep until a condition gets true. The
975 * condition is checked under the lock. This
976 * is expected to be called with the lock
977 * taken.
978 * @wq_head: the waitqueue to wait on
979 * @condition: a C expression for the event to wait for
980 * @lock: a locked spinlock_t, which will be released before schedule()
981 * and reacquired afterwards.
982 *
983 * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
984 * @condition evaluates to true. The @condition is checked each time
985 * the waitqueue @wq_head is woken up.
986 *
987 * wake_up() has to be called after changing any variable that could
988 * change the result of the wait condition.
989 *
990 * This is supposed to be called while holding the lock. The lock is
991 * dropped before going to sleep and is reacquired afterwards.
992 */
993#define wait_event_lock_irq(wq_head, condition, lock) \
994do { \
995 if (condition) \
996 break; \
997 __wait_event_lock_irq(wq_head, condition, lock, ); \
998} while (0)
999
1000
1001#define __wait_event_interruptible_lock_irq(wq_head, condition, lock, cmd) \
1002 ___wait_event(wq_head, condition, TASK_INTERRUPTIBLE, 0, 0, \
1003 spin_unlock_irq(&lock); \
1004 cmd; \
1005 schedule(); \
1006 spin_lock_irq(&lock))
1007
1008/**
1009 * wait_event_interruptible_lock_irq_cmd - sleep until a condition gets true.
1010 * The condition is checked under the lock. This is expected to
1011 * be called with the lock taken.
1012 * @wq_head: the waitqueue to wait on
1013 * @condition: a C expression for the event to wait for
1014 * @lock: a locked spinlock_t, which will be released before cmd and
1015 * schedule() and reacquired afterwards.
1016 * @cmd: a command which is invoked outside the critical section before
1017 * sleep
1018 *
1019 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
1020 * @condition evaluates to true or a signal is received. The @condition is
1021 * checked each time the waitqueue @wq_head is woken up.
1022 *
1023 * wake_up() has to be called after changing any variable that could
1024 * change the result of the wait condition.
1025 *
1026 * This is supposed to be called while holding the lock. The lock is
1027 * dropped before invoking the cmd and going to sleep and is reacquired
1028 * afterwards.
1029 *
1030 * The macro will return -ERESTARTSYS if it was interrupted by a signal
1031 * and 0 if @condition evaluated to true.
1032 */
1033#define wait_event_interruptible_lock_irq_cmd(wq_head, condition, lock, cmd) \
1034({ \
1035 int __ret = 0; \
1036 if (!(condition)) \
1037 __ret = __wait_event_interruptible_lock_irq(wq_head, \
1038 condition, lock, cmd); \
1039 __ret; \
1040})
1041
1042/**
1043 * wait_event_interruptible_lock_irq - sleep until a condition gets true.
1044 * The condition is checked under the lock. This is expected
1045 * to be called with the lock taken.
1046 * @wq_head: the waitqueue to wait on
1047 * @condition: a C expression for the event to wait for
1048 * @lock: a locked spinlock_t, which will be released before schedule()
1049 * and reacquired afterwards.
1050 *
1051 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
1052 * @condition evaluates to true or signal is received. The @condition is
1053 * checked each time the waitqueue @wq_head is woken up.
1054 *
1055 * wake_up() has to be called after changing any variable that could
1056 * change the result of the wait condition.
1057 *
1058 * This is supposed to be called while holding the lock. The lock is
1059 * dropped before going to sleep and is reacquired afterwards.
1060 *
1061 * The macro will return -ERESTARTSYS if it was interrupted by a signal
1062 * and 0 if @condition evaluated to true.
1063 */
1064#define wait_event_interruptible_lock_irq(wq_head, condition, lock) \
1065({ \
1066 int __ret = 0; \
1067 if (!(condition)) \
1068 __ret = __wait_event_interruptible_lock_irq(wq_head, \
1069 condition, lock,); \
1070 __ret; \
1071})
1072
1073#define __wait_event_lock_irq_timeout(wq_head, condition, lock, timeout, state) \
1074 ___wait_event(wq_head, ___wait_cond_timeout(condition), \
1075 state, 0, timeout, \
1076 spin_unlock_irq(&lock); \
1077 __ret = schedule_timeout(__ret); \
1078 spin_lock_irq(&lock));
1079
1080/**
1081 * wait_event_interruptible_lock_irq_timeout - sleep until a condition gets
1082 * true or a timeout elapses. The condition is checked under
1083 * the lock. This is expected to be called with the lock taken.
1084 * @wq_head: the waitqueue to wait on
1085 * @condition: a C expression for the event to wait for
1086 * @lock: a locked spinlock_t, which will be released before schedule()
1087 * and reacquired afterwards.
1088 * @timeout: timeout, in jiffies
1089 *
1090 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
1091 * @condition evaluates to true or signal is received. The @condition is
1092 * checked each time the waitqueue @wq_head is woken up.
1093 *
1094 * wake_up() has to be called after changing any variable that could
1095 * change the result of the wait condition.
1096 *
1097 * This is supposed to be called while holding the lock. The lock is
1098 * dropped before going to sleep and is reacquired afterwards.
1099 *
1100 * The function returns 0 if the @timeout elapsed, -ERESTARTSYS if it
1101 * was interrupted by a signal, and the remaining jiffies otherwise
1102 * if the condition evaluated to true before the timeout elapsed.
1103 */
1104#define wait_event_interruptible_lock_irq_timeout(wq_head, condition, lock, \
1105 timeout) \
1106({ \
1107 long __ret = timeout; \
1108 if (!___wait_cond_timeout(condition)) \
1109 __ret = __wait_event_lock_irq_timeout( \
1110 wq_head, condition, lock, timeout, \
1111 TASK_INTERRUPTIBLE); \
1112 __ret; \
1113})
1114
1115#define wait_event_lock_irq_timeout(wq_head, condition, lock, timeout) \
1116({ \
1117 long __ret = timeout; \
1118 if (!___wait_cond_timeout(condition)) \
1119 __ret = __wait_event_lock_irq_timeout( \
1120 wq_head, condition, lock, timeout, \
1121 TASK_UNINTERRUPTIBLE); \
1122 __ret; \
1123})
1124
1125/*
1126 * Waitqueues which are removed from the waitqueue_head at wakeup time
1127 */
1128void prepare_to_wait(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state);
1129void prepare_to_wait_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state);
1130long prepare_to_wait_event(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state);
1131void finish_wait(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);
1132long wait_woken(struct wait_queue_entry *wq_entry, unsigned mode, long timeout);
1133int woken_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *key);
1134int autoremove_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *key);
1135
1136#define DEFINE_WAIT_FUNC(name, function) \
1137 struct wait_queue_entry name = { \
1138 .private = current, \
1139 .func = function, \
1140 .entry = LIST_HEAD_INIT((name).entry), \
1141 }
1142
1143#define DEFINE_WAIT(name) DEFINE_WAIT_FUNC(name, autoremove_wake_function)
1144
1145#define init_wait(wait) \
1146 do { \
1147 (wait)->private = current; \
1148 (wait)->func = autoremove_wake_function; \
1149 INIT_LIST_HEAD(&(wait)->entry); \
1150 (wait)->flags = 0; \
1151 } while (0)
1152
1153bool try_invoke_on_locked_down_task(struct task_struct *p, bool (*func)(struct task_struct *t, void *arg), void *arg);
1154
1155#endif /* _LINUX_WAIT_H */
1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef _LINUX_WAIT_H
3#define _LINUX_WAIT_H
4/*
5 * Linux wait queue related types and methods
6 */
7#include <linux/list.h>
8#include <linux/stddef.h>
9#include <linux/spinlock.h>
10
11#include <asm/current.h>
12#include <uapi/linux/wait.h>
13
14typedef struct wait_queue_entry wait_queue_entry_t;
15
16typedef int (*wait_queue_func_t)(struct wait_queue_entry *wq_entry, unsigned mode, int flags, void *key);
17int default_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, int flags, void *key);
18
19/* wait_queue_entry::flags */
20#define WQ_FLAG_EXCLUSIVE 0x01
21#define WQ_FLAG_WOKEN 0x02
22#define WQ_FLAG_BOOKMARK 0x04
23#define WQ_FLAG_CUSTOM 0x08
24#define WQ_FLAG_DONE 0x10
25#define WQ_FLAG_PRIORITY 0x20
26
27/*
28 * A single wait-queue entry structure:
29 */
30struct wait_queue_entry {
31 unsigned int flags;
32 void *private;
33 wait_queue_func_t func;
34 struct list_head entry;
35};
36
37struct wait_queue_head {
38 spinlock_t lock;
39 struct list_head head;
40};
41typedef struct wait_queue_head wait_queue_head_t;
42
43struct task_struct;
44
45/*
46 * Macros for declaration and initialisaton of the datatypes
47 */
48
49#define __WAITQUEUE_INITIALIZER(name, tsk) { \
50 .private = tsk, \
51 .func = default_wake_function, \
52 .entry = { NULL, NULL } }
53
54#define DECLARE_WAITQUEUE(name, tsk) \
55 struct wait_queue_entry name = __WAITQUEUE_INITIALIZER(name, tsk)
56
57#define __WAIT_QUEUE_HEAD_INITIALIZER(name) { \
58 .lock = __SPIN_LOCK_UNLOCKED(name.lock), \
59 .head = LIST_HEAD_INIT(name.head) }
60
61#define DECLARE_WAIT_QUEUE_HEAD(name) \
62 struct wait_queue_head name = __WAIT_QUEUE_HEAD_INITIALIZER(name)
63
64extern void __init_waitqueue_head(struct wait_queue_head *wq_head, const char *name, struct lock_class_key *);
65
66#define init_waitqueue_head(wq_head) \
67 do { \
68 static struct lock_class_key __key; \
69 \
70 __init_waitqueue_head((wq_head), #wq_head, &__key); \
71 } while (0)
72
73#ifdef CONFIG_LOCKDEP
74# define __WAIT_QUEUE_HEAD_INIT_ONSTACK(name) \
75 ({ init_waitqueue_head(&name); name; })
76# define DECLARE_WAIT_QUEUE_HEAD_ONSTACK(name) \
77 struct wait_queue_head name = __WAIT_QUEUE_HEAD_INIT_ONSTACK(name)
78#else
79# define DECLARE_WAIT_QUEUE_HEAD_ONSTACK(name) DECLARE_WAIT_QUEUE_HEAD(name)
80#endif
81
82static inline void init_waitqueue_entry(struct wait_queue_entry *wq_entry, struct task_struct *p)
83{
84 wq_entry->flags = 0;
85 wq_entry->private = p;
86 wq_entry->func = default_wake_function;
87}
88
89static inline void
90init_waitqueue_func_entry(struct wait_queue_entry *wq_entry, wait_queue_func_t func)
91{
92 wq_entry->flags = 0;
93 wq_entry->private = NULL;
94 wq_entry->func = func;
95}
96
97/**
98 * waitqueue_active -- locklessly test for waiters on the queue
99 * @wq_head: the waitqueue to test for waiters
100 *
101 * returns true if the wait list is not empty
102 *
103 * NOTE: this function is lockless and requires care, incorrect usage _will_
104 * lead to sporadic and non-obvious failure.
105 *
106 * Use either while holding wait_queue_head::lock or when used for wakeups
107 * with an extra smp_mb() like::
108 *
109 * CPU0 - waker CPU1 - waiter
110 *
111 * for (;;) {
112 * @cond = true; prepare_to_wait(&wq_head, &wait, state);
113 * smp_mb(); // smp_mb() from set_current_state()
114 * if (waitqueue_active(wq_head)) if (@cond)
115 * wake_up(wq_head); break;
116 * schedule();
117 * }
118 * finish_wait(&wq_head, &wait);
119 *
120 * Because without the explicit smp_mb() it's possible for the
121 * waitqueue_active() load to get hoisted over the @cond store such that we'll
122 * observe an empty wait list while the waiter might not observe @cond.
123 *
124 * Also note that this 'optimization' trades a spin_lock() for an smp_mb(),
125 * which (when the lock is uncontended) are of roughly equal cost.
126 */
127static inline int waitqueue_active(struct wait_queue_head *wq_head)
128{
129 return !list_empty(&wq_head->head);
130}
131
132/**
133 * wq_has_single_sleeper - check if there is only one sleeper
134 * @wq_head: wait queue head
135 *
136 * Returns true of wq_head has only one sleeper on the list.
137 *
138 * Please refer to the comment for waitqueue_active.
139 */
140static inline bool wq_has_single_sleeper(struct wait_queue_head *wq_head)
141{
142 return list_is_singular(&wq_head->head);
143}
144
145/**
146 * wq_has_sleeper - check if there are any waiting processes
147 * @wq_head: wait queue head
148 *
149 * Returns true if wq_head has waiting processes
150 *
151 * Please refer to the comment for waitqueue_active.
152 */
153static inline bool wq_has_sleeper(struct wait_queue_head *wq_head)
154{
155 /*
156 * We need to be sure we are in sync with the
157 * add_wait_queue modifications to the wait queue.
158 *
159 * This memory barrier should be paired with one on the
160 * waiting side.
161 */
162 smp_mb();
163 return waitqueue_active(wq_head);
164}
165
166extern void add_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);
167extern void add_wait_queue_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);
168extern void add_wait_queue_priority(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);
169extern void remove_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);
170
171static inline void __add_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
172{
173 struct list_head *head = &wq_head->head;
174 struct wait_queue_entry *wq;
175
176 list_for_each_entry(wq, &wq_head->head, entry) {
177 if (!(wq->flags & WQ_FLAG_PRIORITY))
178 break;
179 head = &wq->entry;
180 }
181 list_add(&wq_entry->entry, head);
182}
183
184/*
185 * Used for wake-one threads:
186 */
187static inline void
188__add_wait_queue_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
189{
190 wq_entry->flags |= WQ_FLAG_EXCLUSIVE;
191 __add_wait_queue(wq_head, wq_entry);
192}
193
194static inline void __add_wait_queue_entry_tail(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
195{
196 list_add_tail(&wq_entry->entry, &wq_head->head);
197}
198
199static inline void
200__add_wait_queue_entry_tail_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
201{
202 wq_entry->flags |= WQ_FLAG_EXCLUSIVE;
203 __add_wait_queue_entry_tail(wq_head, wq_entry);
204}
205
206static inline void
207__remove_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
208{
209 list_del(&wq_entry->entry);
210}
211
212int __wake_up(struct wait_queue_head *wq_head, unsigned int mode, int nr, void *key);
213void __wake_up_locked_key(struct wait_queue_head *wq_head, unsigned int mode, void *key);
214void __wake_up_locked_key_bookmark(struct wait_queue_head *wq_head,
215 unsigned int mode, void *key, wait_queue_entry_t *bookmark);
216void __wake_up_sync_key(struct wait_queue_head *wq_head, unsigned int mode, void *key);
217void __wake_up_locked_sync_key(struct wait_queue_head *wq_head, unsigned int mode, void *key);
218void __wake_up_locked(struct wait_queue_head *wq_head, unsigned int mode, int nr);
219void __wake_up_sync(struct wait_queue_head *wq_head, unsigned int mode);
220void __wake_up_pollfree(struct wait_queue_head *wq_head);
221
222#define wake_up(x) __wake_up(x, TASK_NORMAL, 1, NULL)
223#define wake_up_nr(x, nr) __wake_up(x, TASK_NORMAL, nr, NULL)
224#define wake_up_all(x) __wake_up(x, TASK_NORMAL, 0, NULL)
225#define wake_up_locked(x) __wake_up_locked((x), TASK_NORMAL, 1)
226#define wake_up_all_locked(x) __wake_up_locked((x), TASK_NORMAL, 0)
227
228#define wake_up_interruptible(x) __wake_up(x, TASK_INTERRUPTIBLE, 1, NULL)
229#define wake_up_interruptible_nr(x, nr) __wake_up(x, TASK_INTERRUPTIBLE, nr, NULL)
230#define wake_up_interruptible_all(x) __wake_up(x, TASK_INTERRUPTIBLE, 0, NULL)
231#define wake_up_interruptible_sync(x) __wake_up_sync((x), TASK_INTERRUPTIBLE)
232
233/*
234 * Wakeup macros to be used to report events to the targets.
235 */
236#define poll_to_key(m) ((void *)(__force uintptr_t)(__poll_t)(m))
237#define key_to_poll(m) ((__force __poll_t)(uintptr_t)(void *)(m))
238#define wake_up_poll(x, m) \
239 __wake_up(x, TASK_NORMAL, 1, poll_to_key(m))
240#define wake_up_locked_poll(x, m) \
241 __wake_up_locked_key((x), TASK_NORMAL, poll_to_key(m))
242#define wake_up_interruptible_poll(x, m) \
243 __wake_up(x, TASK_INTERRUPTIBLE, 1, poll_to_key(m))
244#define wake_up_interruptible_sync_poll(x, m) \
245 __wake_up_sync_key((x), TASK_INTERRUPTIBLE, poll_to_key(m))
246#define wake_up_interruptible_sync_poll_locked(x, m) \
247 __wake_up_locked_sync_key((x), TASK_INTERRUPTIBLE, poll_to_key(m))
248
249/**
250 * wake_up_pollfree - signal that a polled waitqueue is going away
251 * @wq_head: the wait queue head
252 *
253 * In the very rare cases where a ->poll() implementation uses a waitqueue whose
254 * lifetime is tied to a task rather than to the 'struct file' being polled,
255 * this function must be called before the waitqueue is freed so that
256 * non-blocking polls (e.g. epoll) are notified that the queue is going away.
257 *
258 * The caller must also RCU-delay the freeing of the wait_queue_head, e.g. via
259 * an explicit synchronize_rcu() or call_rcu(), or via SLAB_TYPESAFE_BY_RCU.
260 */
261static inline void wake_up_pollfree(struct wait_queue_head *wq_head)
262{
263 /*
264 * For performance reasons, we don't always take the queue lock here.
265 * Therefore, we might race with someone removing the last entry from
266 * the queue, and proceed while they still hold the queue lock.
267 * However, rcu_read_lock() is required to be held in such cases, so we
268 * can safely proceed with an RCU-delayed free.
269 */
270 if (waitqueue_active(wq_head))
271 __wake_up_pollfree(wq_head);
272}
273
274#define ___wait_cond_timeout(condition) \
275({ \
276 bool __cond = (condition); \
277 if (__cond && !__ret) \
278 __ret = 1; \
279 __cond || !__ret; \
280})
281
282#define ___wait_is_interruptible(state) \
283 (!__builtin_constant_p(state) || \
284 (state & (TASK_INTERRUPTIBLE | TASK_WAKEKILL)))
285
286extern void init_wait_entry(struct wait_queue_entry *wq_entry, int flags);
287
288/*
289 * The below macro ___wait_event() has an explicit shadow of the __ret
290 * variable when used from the wait_event_*() macros.
291 *
292 * This is so that both can use the ___wait_cond_timeout() construct
293 * to wrap the condition.
294 *
295 * The type inconsistency of the wait_event_*() __ret variable is also
296 * on purpose; we use long where we can return timeout values and int
297 * otherwise.
298 */
299
300#define ___wait_event(wq_head, condition, state, exclusive, ret, cmd) \
301({ \
302 __label__ __out; \
303 struct wait_queue_entry __wq_entry; \
304 long __ret = ret; /* explicit shadow */ \
305 \
306 init_wait_entry(&__wq_entry, exclusive ? WQ_FLAG_EXCLUSIVE : 0); \
307 for (;;) { \
308 long __int = prepare_to_wait_event(&wq_head, &__wq_entry, state);\
309 \
310 if (condition) \
311 break; \
312 \
313 if (___wait_is_interruptible(state) && __int) { \
314 __ret = __int; \
315 goto __out; \
316 } \
317 \
318 cmd; \
319 } \
320 finish_wait(&wq_head, &__wq_entry); \
321__out: __ret; \
322})
323
324#define __wait_event(wq_head, condition) \
325 (void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 0, 0, \
326 schedule())
327
328/**
329 * wait_event - sleep until a condition gets true
330 * @wq_head: the waitqueue to wait on
331 * @condition: a C expression for the event to wait for
332 *
333 * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
334 * @condition evaluates to true. The @condition is checked each time
335 * the waitqueue @wq_head is woken up.
336 *
337 * wake_up() has to be called after changing any variable that could
338 * change the result of the wait condition.
339 */
340#define wait_event(wq_head, condition) \
341do { \
342 might_sleep(); \
343 if (condition) \
344 break; \
345 __wait_event(wq_head, condition); \
346} while (0)
347
348#define __io_wait_event(wq_head, condition) \
349 (void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 0, 0, \
350 io_schedule())
351
352/*
353 * io_wait_event() -- like wait_event() but with io_schedule()
354 */
355#define io_wait_event(wq_head, condition) \
356do { \
357 might_sleep(); \
358 if (condition) \
359 break; \
360 __io_wait_event(wq_head, condition); \
361} while (0)
362
363#define __wait_event_freezable(wq_head, condition) \
364 ___wait_event(wq_head, condition, (TASK_INTERRUPTIBLE|TASK_FREEZABLE), \
365 0, 0, schedule())
366
367/**
368 * wait_event_freezable - sleep (or freeze) until a condition gets true
369 * @wq_head: the waitqueue to wait on
370 * @condition: a C expression for the event to wait for
371 *
372 * The process is put to sleep (TASK_INTERRUPTIBLE -- so as not to contribute
373 * to system load) until the @condition evaluates to true. The
374 * @condition is checked each time the waitqueue @wq_head is woken up.
375 *
376 * wake_up() has to be called after changing any variable that could
377 * change the result of the wait condition.
378 */
379#define wait_event_freezable(wq_head, condition) \
380({ \
381 int __ret = 0; \
382 might_sleep(); \
383 if (!(condition)) \
384 __ret = __wait_event_freezable(wq_head, condition); \
385 __ret; \
386})
387
388#define __wait_event_timeout(wq_head, condition, timeout) \
389 ___wait_event(wq_head, ___wait_cond_timeout(condition), \
390 TASK_UNINTERRUPTIBLE, 0, timeout, \
391 __ret = schedule_timeout(__ret))
392
393/**
394 * wait_event_timeout - sleep until a condition gets true or a timeout elapses
395 * @wq_head: the waitqueue to wait on
396 * @condition: a C expression for the event to wait for
397 * @timeout: timeout, in jiffies
398 *
399 * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
400 * @condition evaluates to true. The @condition is checked each time
401 * the waitqueue @wq_head is woken up.
402 *
403 * wake_up() has to be called after changing any variable that could
404 * change the result of the wait condition.
405 *
406 * Returns:
407 * 0 if the @condition evaluated to %false after the @timeout elapsed,
408 * 1 if the @condition evaluated to %true after the @timeout elapsed,
409 * or the remaining jiffies (at least 1) if the @condition evaluated
410 * to %true before the @timeout elapsed.
411 */
412#define wait_event_timeout(wq_head, condition, timeout) \
413({ \
414 long __ret = timeout; \
415 might_sleep(); \
416 if (!___wait_cond_timeout(condition)) \
417 __ret = __wait_event_timeout(wq_head, condition, timeout); \
418 __ret; \
419})
420
421#define __wait_event_freezable_timeout(wq_head, condition, timeout) \
422 ___wait_event(wq_head, ___wait_cond_timeout(condition), \
423 (TASK_INTERRUPTIBLE|TASK_FREEZABLE), 0, timeout, \
424 __ret = schedule_timeout(__ret))
425
426/*
427 * like wait_event_timeout() -- except it uses TASK_INTERRUPTIBLE to avoid
428 * increasing load and is freezable.
429 */
430#define wait_event_freezable_timeout(wq_head, condition, timeout) \
431({ \
432 long __ret = timeout; \
433 might_sleep(); \
434 if (!___wait_cond_timeout(condition)) \
435 __ret = __wait_event_freezable_timeout(wq_head, condition, timeout); \
436 __ret; \
437})
438
439#define __wait_event_exclusive_cmd(wq_head, condition, cmd1, cmd2) \
440 (void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 1, 0, \
441 cmd1; schedule(); cmd2)
442/*
443 * Just like wait_event_cmd(), except it sets exclusive flag
444 */
445#define wait_event_exclusive_cmd(wq_head, condition, cmd1, cmd2) \
446do { \
447 if (condition) \
448 break; \
449 __wait_event_exclusive_cmd(wq_head, condition, cmd1, cmd2); \
450} while (0)
451
452#define __wait_event_cmd(wq_head, condition, cmd1, cmd2) \
453 (void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 0, 0, \
454 cmd1; schedule(); cmd2)
455
456/**
457 * wait_event_cmd - sleep until a condition gets true
458 * @wq_head: the waitqueue to wait on
459 * @condition: a C expression for the event to wait for
460 * @cmd1: the command will be executed before sleep
461 * @cmd2: the command will be executed after sleep
462 *
463 * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
464 * @condition evaluates to true. The @condition is checked each time
465 * the waitqueue @wq_head is woken up.
466 *
467 * wake_up() has to be called after changing any variable that could
468 * change the result of the wait condition.
469 */
470#define wait_event_cmd(wq_head, condition, cmd1, cmd2) \
471do { \
472 if (condition) \
473 break; \
474 __wait_event_cmd(wq_head, condition, cmd1, cmd2); \
475} while (0)
476
477#define __wait_event_interruptible(wq_head, condition) \
478 ___wait_event(wq_head, condition, TASK_INTERRUPTIBLE, 0, 0, \
479 schedule())
480
481/**
482 * wait_event_interruptible - sleep until a condition gets true
483 * @wq_head: the waitqueue to wait on
484 * @condition: a C expression for the event to wait for
485 *
486 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
487 * @condition evaluates to true or a signal is received.
488 * The @condition is checked each time the waitqueue @wq_head is woken up.
489 *
490 * wake_up() has to be called after changing any variable that could
491 * change the result of the wait condition.
492 *
493 * The function will return -ERESTARTSYS if it was interrupted by a
494 * signal and 0 if @condition evaluated to true.
495 */
496#define wait_event_interruptible(wq_head, condition) \
497({ \
498 int __ret = 0; \
499 might_sleep(); \
500 if (!(condition)) \
501 __ret = __wait_event_interruptible(wq_head, condition); \
502 __ret; \
503})
504
505#define __wait_event_interruptible_timeout(wq_head, condition, timeout) \
506 ___wait_event(wq_head, ___wait_cond_timeout(condition), \
507 TASK_INTERRUPTIBLE, 0, timeout, \
508 __ret = schedule_timeout(__ret))
509
510/**
511 * wait_event_interruptible_timeout - sleep until a condition gets true or a timeout elapses
512 * @wq_head: the waitqueue to wait on
513 * @condition: a C expression for the event to wait for
514 * @timeout: timeout, in jiffies
515 *
516 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
517 * @condition evaluates to true or a signal is received.
518 * The @condition is checked each time the waitqueue @wq_head is woken up.
519 *
520 * wake_up() has to be called after changing any variable that could
521 * change the result of the wait condition.
522 *
523 * Returns:
524 * 0 if the @condition evaluated to %false after the @timeout elapsed,
525 * 1 if the @condition evaluated to %true after the @timeout elapsed,
526 * the remaining jiffies (at least 1) if the @condition evaluated
527 * to %true before the @timeout elapsed, or -%ERESTARTSYS if it was
528 * interrupted by a signal.
529 */
530#define wait_event_interruptible_timeout(wq_head, condition, timeout) \
531({ \
532 long __ret = timeout; \
533 might_sleep(); \
534 if (!___wait_cond_timeout(condition)) \
535 __ret = __wait_event_interruptible_timeout(wq_head, \
536 condition, timeout); \
537 __ret; \
538})
539
540#define __wait_event_hrtimeout(wq_head, condition, timeout, state) \
541({ \
542 int __ret = 0; \
543 struct hrtimer_sleeper __t; \
544 \
545 hrtimer_init_sleeper_on_stack(&__t, CLOCK_MONOTONIC, \
546 HRTIMER_MODE_REL); \
547 if ((timeout) != KTIME_MAX) { \
548 hrtimer_set_expires_range_ns(&__t.timer, timeout, \
549 current->timer_slack_ns); \
550 hrtimer_sleeper_start_expires(&__t, HRTIMER_MODE_REL); \
551 } \
552 \
553 __ret = ___wait_event(wq_head, condition, state, 0, 0, \
554 if (!__t.task) { \
555 __ret = -ETIME; \
556 break; \
557 } \
558 schedule()); \
559 \
560 hrtimer_cancel(&__t.timer); \
561 destroy_hrtimer_on_stack(&__t.timer); \
562 __ret; \
563})
564
565/**
566 * wait_event_hrtimeout - sleep until a condition gets true or a timeout elapses
567 * @wq_head: the waitqueue to wait on
568 * @condition: a C expression for the event to wait for
569 * @timeout: timeout, as a ktime_t
570 *
571 * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
572 * @condition evaluates to true or a signal is received.
573 * The @condition is checked each time the waitqueue @wq_head is woken up.
574 *
575 * wake_up() has to be called after changing any variable that could
576 * change the result of the wait condition.
577 *
578 * The function returns 0 if @condition became true, or -ETIME if the timeout
579 * elapsed.
580 */
581#define wait_event_hrtimeout(wq_head, condition, timeout) \
582({ \
583 int __ret = 0; \
584 might_sleep(); \
585 if (!(condition)) \
586 __ret = __wait_event_hrtimeout(wq_head, condition, timeout, \
587 TASK_UNINTERRUPTIBLE); \
588 __ret; \
589})
590
591/**
592 * wait_event_interruptible_hrtimeout - sleep until a condition gets true or a timeout elapses
593 * @wq: the waitqueue to wait on
594 * @condition: a C expression for the event to wait for
595 * @timeout: timeout, as a ktime_t
596 *
597 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
598 * @condition evaluates to true or a signal is received.
599 * The @condition is checked each time the waitqueue @wq is woken up.
600 *
601 * wake_up() has to be called after changing any variable that could
602 * change the result of the wait condition.
603 *
604 * The function returns 0 if @condition became true, -ERESTARTSYS if it was
605 * interrupted by a signal, or -ETIME if the timeout elapsed.
606 */
607#define wait_event_interruptible_hrtimeout(wq, condition, timeout) \
608({ \
609 long __ret = 0; \
610 might_sleep(); \
611 if (!(condition)) \
612 __ret = __wait_event_hrtimeout(wq, condition, timeout, \
613 TASK_INTERRUPTIBLE); \
614 __ret; \
615})
616
617#define __wait_event_interruptible_exclusive(wq, condition) \
618 ___wait_event(wq, condition, TASK_INTERRUPTIBLE, 1, 0, \
619 schedule())
620
621#define wait_event_interruptible_exclusive(wq, condition) \
622({ \
623 int __ret = 0; \
624 might_sleep(); \
625 if (!(condition)) \
626 __ret = __wait_event_interruptible_exclusive(wq, condition); \
627 __ret; \
628})
629
630#define __wait_event_killable_exclusive(wq, condition) \
631 ___wait_event(wq, condition, TASK_KILLABLE, 1, 0, \
632 schedule())
633
634#define wait_event_killable_exclusive(wq, condition) \
635({ \
636 int __ret = 0; \
637 might_sleep(); \
638 if (!(condition)) \
639 __ret = __wait_event_killable_exclusive(wq, condition); \
640 __ret; \
641})
642
643
644#define __wait_event_freezable_exclusive(wq, condition) \
645 ___wait_event(wq, condition, (TASK_INTERRUPTIBLE|TASK_FREEZABLE), 1, 0,\
646 schedule())
647
648#define wait_event_freezable_exclusive(wq, condition) \
649({ \
650 int __ret = 0; \
651 might_sleep(); \
652 if (!(condition)) \
653 __ret = __wait_event_freezable_exclusive(wq, condition); \
654 __ret; \
655})
656
657/**
658 * wait_event_idle - wait for a condition without contributing to system load
659 * @wq_head: the waitqueue to wait on
660 * @condition: a C expression for the event to wait for
661 *
662 * The process is put to sleep (TASK_IDLE) until the
663 * @condition evaluates to true.
664 * The @condition is checked each time the waitqueue @wq_head is woken up.
665 *
666 * wake_up() has to be called after changing any variable that could
667 * change the result of the wait condition.
668 *
669 */
670#define wait_event_idle(wq_head, condition) \
671do { \
672 might_sleep(); \
673 if (!(condition)) \
674 ___wait_event(wq_head, condition, TASK_IDLE, 0, 0, schedule()); \
675} while (0)
676
677/**
678 * wait_event_idle_exclusive - wait for a condition with contributing to system load
679 * @wq_head: the waitqueue to wait on
680 * @condition: a C expression for the event to wait for
681 *
682 * The process is put to sleep (TASK_IDLE) until the
683 * @condition evaluates to true.
684 * The @condition is checked each time the waitqueue @wq_head is woken up.
685 *
686 * The process is put on the wait queue with an WQ_FLAG_EXCLUSIVE flag
687 * set thus if other processes wait on the same list, when this
688 * process is woken further processes are not considered.
689 *
690 * wake_up() has to be called after changing any variable that could
691 * change the result of the wait condition.
692 *
693 */
694#define wait_event_idle_exclusive(wq_head, condition) \
695do { \
696 might_sleep(); \
697 if (!(condition)) \
698 ___wait_event(wq_head, condition, TASK_IDLE, 1, 0, schedule()); \
699} while (0)
700
701#define __wait_event_idle_timeout(wq_head, condition, timeout) \
702 ___wait_event(wq_head, ___wait_cond_timeout(condition), \
703 TASK_IDLE, 0, timeout, \
704 __ret = schedule_timeout(__ret))
705
706/**
707 * wait_event_idle_timeout - sleep without load until a condition becomes true or a timeout elapses
708 * @wq_head: the waitqueue to wait on
709 * @condition: a C expression for the event to wait for
710 * @timeout: timeout, in jiffies
711 *
712 * The process is put to sleep (TASK_IDLE) until the
713 * @condition evaluates to true. The @condition is checked each time
714 * the waitqueue @wq_head is woken up.
715 *
716 * wake_up() has to be called after changing any variable that could
717 * change the result of the wait condition.
718 *
719 * Returns:
720 * 0 if the @condition evaluated to %false after the @timeout elapsed,
721 * 1 if the @condition evaluated to %true after the @timeout elapsed,
722 * or the remaining jiffies (at least 1) if the @condition evaluated
723 * to %true before the @timeout elapsed.
724 */
725#define wait_event_idle_timeout(wq_head, condition, timeout) \
726({ \
727 long __ret = timeout; \
728 might_sleep(); \
729 if (!___wait_cond_timeout(condition)) \
730 __ret = __wait_event_idle_timeout(wq_head, condition, timeout); \
731 __ret; \
732})
733
734#define __wait_event_idle_exclusive_timeout(wq_head, condition, timeout) \
735 ___wait_event(wq_head, ___wait_cond_timeout(condition), \
736 TASK_IDLE, 1, timeout, \
737 __ret = schedule_timeout(__ret))
738
739/**
740 * wait_event_idle_exclusive_timeout - sleep without load until a condition becomes true or a timeout elapses
741 * @wq_head: the waitqueue to wait on
742 * @condition: a C expression for the event to wait for
743 * @timeout: timeout, in jiffies
744 *
745 * The process is put to sleep (TASK_IDLE) until the
746 * @condition evaluates to true. The @condition is checked each time
747 * the waitqueue @wq_head is woken up.
748 *
749 * The process is put on the wait queue with an WQ_FLAG_EXCLUSIVE flag
750 * set thus if other processes wait on the same list, when this
751 * process is woken further processes are not considered.
752 *
753 * wake_up() has to be called after changing any variable that could
754 * change the result of the wait condition.
755 *
756 * Returns:
757 * 0 if the @condition evaluated to %false after the @timeout elapsed,
758 * 1 if the @condition evaluated to %true after the @timeout elapsed,
759 * or the remaining jiffies (at least 1) if the @condition evaluated
760 * to %true before the @timeout elapsed.
761 */
762#define wait_event_idle_exclusive_timeout(wq_head, condition, timeout) \
763({ \
764 long __ret = timeout; \
765 might_sleep(); \
766 if (!___wait_cond_timeout(condition)) \
767 __ret = __wait_event_idle_exclusive_timeout(wq_head, condition, timeout);\
768 __ret; \
769})
770
771extern int do_wait_intr(wait_queue_head_t *, wait_queue_entry_t *);
772extern int do_wait_intr_irq(wait_queue_head_t *, wait_queue_entry_t *);
773
774#define __wait_event_interruptible_locked(wq, condition, exclusive, fn) \
775({ \
776 int __ret; \
777 DEFINE_WAIT(__wait); \
778 if (exclusive) \
779 __wait.flags |= WQ_FLAG_EXCLUSIVE; \
780 do { \
781 __ret = fn(&(wq), &__wait); \
782 if (__ret) \
783 break; \
784 } while (!(condition)); \
785 __remove_wait_queue(&(wq), &__wait); \
786 __set_current_state(TASK_RUNNING); \
787 __ret; \
788})
789
790
791/**
792 * wait_event_interruptible_locked - sleep until a condition gets true
793 * @wq: the waitqueue to wait on
794 * @condition: a C expression for the event to wait for
795 *
796 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
797 * @condition evaluates to true or a signal is received.
798 * The @condition is checked each time the waitqueue @wq is woken up.
799 *
800 * It must be called with wq.lock being held. This spinlock is
801 * unlocked while sleeping but @condition testing is done while lock
802 * is held and when this macro exits the lock is held.
803 *
804 * The lock is locked/unlocked using spin_lock()/spin_unlock()
805 * functions which must match the way they are locked/unlocked outside
806 * of this macro.
807 *
808 * wake_up_locked() has to be called after changing any variable that could
809 * change the result of the wait condition.
810 *
811 * The function will return -ERESTARTSYS if it was interrupted by a
812 * signal and 0 if @condition evaluated to true.
813 */
814#define wait_event_interruptible_locked(wq, condition) \
815 ((condition) \
816 ? 0 : __wait_event_interruptible_locked(wq, condition, 0, do_wait_intr))
817
818/**
819 * wait_event_interruptible_locked_irq - sleep until a condition gets true
820 * @wq: the waitqueue to wait on
821 * @condition: a C expression for the event to wait for
822 *
823 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
824 * @condition evaluates to true or a signal is received.
825 * The @condition is checked each time the waitqueue @wq is woken up.
826 *
827 * It must be called with wq.lock being held. This spinlock is
828 * unlocked while sleeping but @condition testing is done while lock
829 * is held and when this macro exits the lock is held.
830 *
831 * The lock is locked/unlocked using spin_lock_irq()/spin_unlock_irq()
832 * functions which must match the way they are locked/unlocked outside
833 * of this macro.
834 *
835 * wake_up_locked() has to be called after changing any variable that could
836 * change the result of the wait condition.
837 *
838 * The function will return -ERESTARTSYS if it was interrupted by a
839 * signal and 0 if @condition evaluated to true.
840 */
841#define wait_event_interruptible_locked_irq(wq, condition) \
842 ((condition) \
843 ? 0 : __wait_event_interruptible_locked(wq, condition, 0, do_wait_intr_irq))
844
845/**
846 * wait_event_interruptible_exclusive_locked - sleep exclusively until a condition gets true
847 * @wq: the waitqueue to wait on
848 * @condition: a C expression for the event to wait for
849 *
850 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
851 * @condition evaluates to true or a signal is received.
852 * The @condition is checked each time the waitqueue @wq is woken up.
853 *
854 * It must be called with wq.lock being held. This spinlock is
855 * unlocked while sleeping but @condition testing is done while lock
856 * is held and when this macro exits the lock is held.
857 *
858 * The lock is locked/unlocked using spin_lock()/spin_unlock()
859 * functions which must match the way they are locked/unlocked outside
860 * of this macro.
861 *
862 * The process is put on the wait queue with an WQ_FLAG_EXCLUSIVE flag
863 * set thus when other process waits process on the list if this
864 * process is awaken further processes are not considered.
865 *
866 * wake_up_locked() has to be called after changing any variable that could
867 * change the result of the wait condition.
868 *
869 * The function will return -ERESTARTSYS if it was interrupted by a
870 * signal and 0 if @condition evaluated to true.
871 */
872#define wait_event_interruptible_exclusive_locked(wq, condition) \
873 ((condition) \
874 ? 0 : __wait_event_interruptible_locked(wq, condition, 1, do_wait_intr))
875
876/**
877 * wait_event_interruptible_exclusive_locked_irq - sleep until a condition gets true
878 * @wq: the waitqueue to wait on
879 * @condition: a C expression for the event to wait for
880 *
881 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
882 * @condition evaluates to true or a signal is received.
883 * The @condition is checked each time the waitqueue @wq is woken up.
884 *
885 * It must be called with wq.lock being held. This spinlock is
886 * unlocked while sleeping but @condition testing is done while lock
887 * is held and when this macro exits the lock is held.
888 *
889 * The lock is locked/unlocked using spin_lock_irq()/spin_unlock_irq()
890 * functions which must match the way they are locked/unlocked outside
891 * of this macro.
892 *
893 * The process is put on the wait queue with an WQ_FLAG_EXCLUSIVE flag
894 * set thus when other process waits process on the list if this
895 * process is awaken further processes are not considered.
896 *
897 * wake_up_locked() has to be called after changing any variable that could
898 * change the result of the wait condition.
899 *
900 * The function will return -ERESTARTSYS if it was interrupted by a
901 * signal and 0 if @condition evaluated to true.
902 */
903#define wait_event_interruptible_exclusive_locked_irq(wq, condition) \
904 ((condition) \
905 ? 0 : __wait_event_interruptible_locked(wq, condition, 1, do_wait_intr_irq))
906
907
908#define __wait_event_killable(wq, condition) \
909 ___wait_event(wq, condition, TASK_KILLABLE, 0, 0, schedule())
910
911/**
912 * wait_event_killable - sleep until a condition gets true
913 * @wq_head: the waitqueue to wait on
914 * @condition: a C expression for the event to wait for
915 *
916 * The process is put to sleep (TASK_KILLABLE) until the
917 * @condition evaluates to true or a signal is received.
918 * The @condition is checked each time the waitqueue @wq_head is woken up.
919 *
920 * wake_up() has to be called after changing any variable that could
921 * change the result of the wait condition.
922 *
923 * The function will return -ERESTARTSYS if it was interrupted by a
924 * signal and 0 if @condition evaluated to true.
925 */
926#define wait_event_killable(wq_head, condition) \
927({ \
928 int __ret = 0; \
929 might_sleep(); \
930 if (!(condition)) \
931 __ret = __wait_event_killable(wq_head, condition); \
932 __ret; \
933})
934
935#define __wait_event_state(wq, condition, state) \
936 ___wait_event(wq, condition, state, 0, 0, schedule())
937
938/**
939 * wait_event_state - sleep until a condition gets true
940 * @wq_head: the waitqueue to wait on
941 * @condition: a C expression for the event to wait for
942 * @state: state to sleep in
943 *
944 * The process is put to sleep (@state) until the @condition evaluates to true
945 * or a signal is received (when allowed by @state). The @condition is checked
946 * each time the waitqueue @wq_head is woken up.
947 *
948 * wake_up() has to be called after changing any variable that could
949 * change the result of the wait condition.
950 *
951 * The function will return -ERESTARTSYS if it was interrupted by a signal
952 * (when allowed by @state) and 0 if @condition evaluated to true.
953 */
954#define wait_event_state(wq_head, condition, state) \
955({ \
956 int __ret = 0; \
957 might_sleep(); \
958 if (!(condition)) \
959 __ret = __wait_event_state(wq_head, condition, state); \
960 __ret; \
961})
962
963#define __wait_event_killable_timeout(wq_head, condition, timeout) \
964 ___wait_event(wq_head, ___wait_cond_timeout(condition), \
965 TASK_KILLABLE, 0, timeout, \
966 __ret = schedule_timeout(__ret))
967
968/**
969 * wait_event_killable_timeout - sleep until a condition gets true or a timeout elapses
970 * @wq_head: the waitqueue to wait on
971 * @condition: a C expression for the event to wait for
972 * @timeout: timeout, in jiffies
973 *
974 * The process is put to sleep (TASK_KILLABLE) until the
975 * @condition evaluates to true or a kill signal is received.
976 * The @condition is checked each time the waitqueue @wq_head is woken up.
977 *
978 * wake_up() has to be called after changing any variable that could
979 * change the result of the wait condition.
980 *
981 * Returns:
982 * 0 if the @condition evaluated to %false after the @timeout elapsed,
983 * 1 if the @condition evaluated to %true after the @timeout elapsed,
984 * the remaining jiffies (at least 1) if the @condition evaluated
985 * to %true before the @timeout elapsed, or -%ERESTARTSYS if it was
986 * interrupted by a kill signal.
987 *
988 * Only kill signals interrupt this process.
989 */
990#define wait_event_killable_timeout(wq_head, condition, timeout) \
991({ \
992 long __ret = timeout; \
993 might_sleep(); \
994 if (!___wait_cond_timeout(condition)) \
995 __ret = __wait_event_killable_timeout(wq_head, \
996 condition, timeout); \
997 __ret; \
998})
999
1000
1001#define __wait_event_lock_irq(wq_head, condition, lock, cmd) \
1002 (void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 0, 0, \
1003 spin_unlock_irq(&lock); \
1004 cmd; \
1005 schedule(); \
1006 spin_lock_irq(&lock))
1007
1008/**
1009 * wait_event_lock_irq_cmd - sleep until a condition gets true. The
1010 * condition is checked under the lock. This
1011 * is expected to be called with the lock
1012 * taken.
1013 * @wq_head: the waitqueue to wait on
1014 * @condition: a C expression for the event to wait for
1015 * @lock: a locked spinlock_t, which will be released before cmd
1016 * and schedule() and reacquired afterwards.
1017 * @cmd: a command which is invoked outside the critical section before
1018 * sleep
1019 *
1020 * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
1021 * @condition evaluates to true. The @condition is checked each time
1022 * the waitqueue @wq_head is woken up.
1023 *
1024 * wake_up() has to be called after changing any variable that could
1025 * change the result of the wait condition.
1026 *
1027 * This is supposed to be called while holding the lock. The lock is
1028 * dropped before invoking the cmd and going to sleep and is reacquired
1029 * afterwards.
1030 */
1031#define wait_event_lock_irq_cmd(wq_head, condition, lock, cmd) \
1032do { \
1033 if (condition) \
1034 break; \
1035 __wait_event_lock_irq(wq_head, condition, lock, cmd); \
1036} while (0)
1037
1038/**
1039 * wait_event_lock_irq - sleep until a condition gets true. The
1040 * condition is checked under the lock. This
1041 * is expected to be called with the lock
1042 * taken.
1043 * @wq_head: the waitqueue to wait on
1044 * @condition: a C expression for the event to wait for
1045 * @lock: a locked spinlock_t, which will be released before schedule()
1046 * and reacquired afterwards.
1047 *
1048 * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
1049 * @condition evaluates to true. The @condition is checked each time
1050 * the waitqueue @wq_head is woken up.
1051 *
1052 * wake_up() has to be called after changing any variable that could
1053 * change the result of the wait condition.
1054 *
1055 * This is supposed to be called while holding the lock. The lock is
1056 * dropped before going to sleep and is reacquired afterwards.
1057 */
1058#define wait_event_lock_irq(wq_head, condition, lock) \
1059do { \
1060 if (condition) \
1061 break; \
1062 __wait_event_lock_irq(wq_head, condition, lock, ); \
1063} while (0)
1064
1065
1066#define __wait_event_interruptible_lock_irq(wq_head, condition, lock, cmd) \
1067 ___wait_event(wq_head, condition, TASK_INTERRUPTIBLE, 0, 0, \
1068 spin_unlock_irq(&lock); \
1069 cmd; \
1070 schedule(); \
1071 spin_lock_irq(&lock))
1072
1073/**
1074 * wait_event_interruptible_lock_irq_cmd - sleep until a condition gets true.
1075 * The condition is checked under the lock. This is expected to
1076 * be called with the lock taken.
1077 * @wq_head: the waitqueue to wait on
1078 * @condition: a C expression for the event to wait for
1079 * @lock: a locked spinlock_t, which will be released before cmd and
1080 * schedule() and reacquired afterwards.
1081 * @cmd: a command which is invoked outside the critical section before
1082 * sleep
1083 *
1084 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
1085 * @condition evaluates to true or a signal is received. The @condition is
1086 * checked each time the waitqueue @wq_head is woken up.
1087 *
1088 * wake_up() has to be called after changing any variable that could
1089 * change the result of the wait condition.
1090 *
1091 * This is supposed to be called while holding the lock. The lock is
1092 * dropped before invoking the cmd and going to sleep and is reacquired
1093 * afterwards.
1094 *
1095 * The macro will return -ERESTARTSYS if it was interrupted by a signal
1096 * and 0 if @condition evaluated to true.
1097 */
1098#define wait_event_interruptible_lock_irq_cmd(wq_head, condition, lock, cmd) \
1099({ \
1100 int __ret = 0; \
1101 if (!(condition)) \
1102 __ret = __wait_event_interruptible_lock_irq(wq_head, \
1103 condition, lock, cmd); \
1104 __ret; \
1105})
1106
1107/**
1108 * wait_event_interruptible_lock_irq - sleep until a condition gets true.
1109 * The condition is checked under the lock. This is expected
1110 * to be called with the lock taken.
1111 * @wq_head: the waitqueue to wait on
1112 * @condition: a C expression for the event to wait for
1113 * @lock: a locked spinlock_t, which will be released before schedule()
1114 * and reacquired afterwards.
1115 *
1116 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
1117 * @condition evaluates to true or signal is received. The @condition is
1118 * checked each time the waitqueue @wq_head is woken up.
1119 *
1120 * wake_up() has to be called after changing any variable that could
1121 * change the result of the wait condition.
1122 *
1123 * This is supposed to be called while holding the lock. The lock is
1124 * dropped before going to sleep and is reacquired afterwards.
1125 *
1126 * The macro will return -ERESTARTSYS if it was interrupted by a signal
1127 * and 0 if @condition evaluated to true.
1128 */
1129#define wait_event_interruptible_lock_irq(wq_head, condition, lock) \
1130({ \
1131 int __ret = 0; \
1132 if (!(condition)) \
1133 __ret = __wait_event_interruptible_lock_irq(wq_head, \
1134 condition, lock,); \
1135 __ret; \
1136})
1137
1138#define __wait_event_lock_irq_timeout(wq_head, condition, lock, timeout, state) \
1139 ___wait_event(wq_head, ___wait_cond_timeout(condition), \
1140 state, 0, timeout, \
1141 spin_unlock_irq(&lock); \
1142 __ret = schedule_timeout(__ret); \
1143 spin_lock_irq(&lock));
1144
1145/**
1146 * wait_event_interruptible_lock_irq_timeout - sleep until a condition gets
1147 * true or a timeout elapses. The condition is checked under
1148 * the lock. This is expected to be called with the lock taken.
1149 * @wq_head: the waitqueue to wait on
1150 * @condition: a C expression for the event to wait for
1151 * @lock: a locked spinlock_t, which will be released before schedule()
1152 * and reacquired afterwards.
1153 * @timeout: timeout, in jiffies
1154 *
1155 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
1156 * @condition evaluates to true or signal is received. The @condition is
1157 * checked each time the waitqueue @wq_head is woken up.
1158 *
1159 * wake_up() has to be called after changing any variable that could
1160 * change the result of the wait condition.
1161 *
1162 * This is supposed to be called while holding the lock. The lock is
1163 * dropped before going to sleep and is reacquired afterwards.
1164 *
1165 * The function returns 0 if the @timeout elapsed, -ERESTARTSYS if it
1166 * was interrupted by a signal, and the remaining jiffies otherwise
1167 * if the condition evaluated to true before the timeout elapsed.
1168 */
1169#define wait_event_interruptible_lock_irq_timeout(wq_head, condition, lock, \
1170 timeout) \
1171({ \
1172 long __ret = timeout; \
1173 if (!___wait_cond_timeout(condition)) \
1174 __ret = __wait_event_lock_irq_timeout( \
1175 wq_head, condition, lock, timeout, \
1176 TASK_INTERRUPTIBLE); \
1177 __ret; \
1178})
1179
1180#define wait_event_lock_irq_timeout(wq_head, condition, lock, timeout) \
1181({ \
1182 long __ret = timeout; \
1183 if (!___wait_cond_timeout(condition)) \
1184 __ret = __wait_event_lock_irq_timeout( \
1185 wq_head, condition, lock, timeout, \
1186 TASK_UNINTERRUPTIBLE); \
1187 __ret; \
1188})
1189
1190/*
1191 * Waitqueues which are removed from the waitqueue_head at wakeup time
1192 */
1193void prepare_to_wait(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state);
1194bool prepare_to_wait_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state);
1195long prepare_to_wait_event(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state);
1196void finish_wait(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);
1197long wait_woken(struct wait_queue_entry *wq_entry, unsigned mode, long timeout);
1198int woken_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *key);
1199int autoremove_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *key);
1200
1201#define DEFINE_WAIT_FUNC(name, function) \
1202 struct wait_queue_entry name = { \
1203 .private = current, \
1204 .func = function, \
1205 .entry = LIST_HEAD_INIT((name).entry), \
1206 }
1207
1208#define DEFINE_WAIT(name) DEFINE_WAIT_FUNC(name, autoremove_wake_function)
1209
1210#define init_wait(wait) \
1211 do { \
1212 (wait)->private = current; \
1213 (wait)->func = autoremove_wake_function; \
1214 INIT_LIST_HEAD(&(wait)->entry); \
1215 (wait)->flags = 0; \
1216 } while (0)
1217
1218typedef int (*task_call_f)(struct task_struct *p, void *arg);
1219extern int task_call_func(struct task_struct *p, task_call_f func, void *arg);
1220
1221#endif /* _LINUX_WAIT_H */