Loading...
1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef _LINUX_WAIT_H
3#define _LINUX_WAIT_H
4/*
5 * Linux wait queue related types and methods
6 */
7#include <linux/list.h>
8#include <linux/stddef.h>
9#include <linux/spinlock.h>
10
11#include <asm/current.h>
12#include <uapi/linux/wait.h>
13
14typedef struct wait_queue_entry wait_queue_entry_t;
15
16typedef int (*wait_queue_func_t)(struct wait_queue_entry *wq_entry, unsigned mode, int flags, void *key);
17int default_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, int flags, void *key);
18
19/* wait_queue_entry::flags */
20#define WQ_FLAG_EXCLUSIVE 0x01
21#define WQ_FLAG_WOKEN 0x02
22#define WQ_FLAG_BOOKMARK 0x04
23#define WQ_FLAG_CUSTOM 0x08
24#define WQ_FLAG_DONE 0x10
25
26/*
27 * A single wait-queue entry structure:
28 */
29struct wait_queue_entry {
30 unsigned int flags;
31 void *private;
32 wait_queue_func_t func;
33 struct list_head entry;
34};
35
36struct wait_queue_head {
37 spinlock_t lock;
38 struct list_head head;
39};
40typedef struct wait_queue_head wait_queue_head_t;
41
42struct task_struct;
43
44/*
45 * Macros for declaration and initialisaton of the datatypes
46 */
47
48#define __WAITQUEUE_INITIALIZER(name, tsk) { \
49 .private = tsk, \
50 .func = default_wake_function, \
51 .entry = { NULL, NULL } }
52
53#define DECLARE_WAITQUEUE(name, tsk) \
54 struct wait_queue_entry name = __WAITQUEUE_INITIALIZER(name, tsk)
55
56#define __WAIT_QUEUE_HEAD_INITIALIZER(name) { \
57 .lock = __SPIN_LOCK_UNLOCKED(name.lock), \
58 .head = { &(name).head, &(name).head } }
59
60#define DECLARE_WAIT_QUEUE_HEAD(name) \
61 struct wait_queue_head name = __WAIT_QUEUE_HEAD_INITIALIZER(name)
62
63extern void __init_waitqueue_head(struct wait_queue_head *wq_head, const char *name, struct lock_class_key *);
64
65#define init_waitqueue_head(wq_head) \
66 do { \
67 static struct lock_class_key __key; \
68 \
69 __init_waitqueue_head((wq_head), #wq_head, &__key); \
70 } while (0)
71
72#ifdef CONFIG_LOCKDEP
73# define __WAIT_QUEUE_HEAD_INIT_ONSTACK(name) \
74 ({ init_waitqueue_head(&name); name; })
75# define DECLARE_WAIT_QUEUE_HEAD_ONSTACK(name) \
76 struct wait_queue_head name = __WAIT_QUEUE_HEAD_INIT_ONSTACK(name)
77#else
78# define DECLARE_WAIT_QUEUE_HEAD_ONSTACK(name) DECLARE_WAIT_QUEUE_HEAD(name)
79#endif
80
81static inline void init_waitqueue_entry(struct wait_queue_entry *wq_entry, struct task_struct *p)
82{
83 wq_entry->flags = 0;
84 wq_entry->private = p;
85 wq_entry->func = default_wake_function;
86}
87
88static inline void
89init_waitqueue_func_entry(struct wait_queue_entry *wq_entry, wait_queue_func_t func)
90{
91 wq_entry->flags = 0;
92 wq_entry->private = NULL;
93 wq_entry->func = func;
94}
95
96/**
97 * waitqueue_active -- locklessly test for waiters on the queue
98 * @wq_head: the waitqueue to test for waiters
99 *
100 * returns true if the wait list is not empty
101 *
102 * NOTE: this function is lockless and requires care, incorrect usage _will_
103 * lead to sporadic and non-obvious failure.
104 *
105 * Use either while holding wait_queue_head::lock or when used for wakeups
106 * with an extra smp_mb() like::
107 *
108 * CPU0 - waker CPU1 - waiter
109 *
110 * for (;;) {
111 * @cond = true; prepare_to_wait(&wq_head, &wait, state);
112 * smp_mb(); // smp_mb() from set_current_state()
113 * if (waitqueue_active(wq_head)) if (@cond)
114 * wake_up(wq_head); break;
115 * schedule();
116 * }
117 * finish_wait(&wq_head, &wait);
118 *
119 * Because without the explicit smp_mb() it's possible for the
120 * waitqueue_active() load to get hoisted over the @cond store such that we'll
121 * observe an empty wait list while the waiter might not observe @cond.
122 *
123 * Also note that this 'optimization' trades a spin_lock() for an smp_mb(),
124 * which (when the lock is uncontended) are of roughly equal cost.
125 */
126static inline int waitqueue_active(struct wait_queue_head *wq_head)
127{
128 return !list_empty(&wq_head->head);
129}
130
131/**
132 * wq_has_single_sleeper - check if there is only one sleeper
133 * @wq_head: wait queue head
134 *
135 * Returns true of wq_head has only one sleeper on the list.
136 *
137 * Please refer to the comment for waitqueue_active.
138 */
139static inline bool wq_has_single_sleeper(struct wait_queue_head *wq_head)
140{
141 return list_is_singular(&wq_head->head);
142}
143
144/**
145 * wq_has_sleeper - check if there are any waiting processes
146 * @wq_head: wait queue head
147 *
148 * Returns true if wq_head has waiting processes
149 *
150 * Please refer to the comment for waitqueue_active.
151 */
152static inline bool wq_has_sleeper(struct wait_queue_head *wq_head)
153{
154 /*
155 * We need to be sure we are in sync with the
156 * add_wait_queue modifications to the wait queue.
157 *
158 * This memory barrier should be paired with one on the
159 * waiting side.
160 */
161 smp_mb();
162 return waitqueue_active(wq_head);
163}
164
165extern void add_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);
166extern void add_wait_queue_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);
167extern void remove_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);
168
169static inline void __add_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
170{
171 list_add(&wq_entry->entry, &wq_head->head);
172}
173
174/*
175 * Used for wake-one threads:
176 */
177static inline void
178__add_wait_queue_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
179{
180 wq_entry->flags |= WQ_FLAG_EXCLUSIVE;
181 __add_wait_queue(wq_head, wq_entry);
182}
183
184static inline void __add_wait_queue_entry_tail(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
185{
186 list_add_tail(&wq_entry->entry, &wq_head->head);
187}
188
189static inline void
190__add_wait_queue_entry_tail_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
191{
192 wq_entry->flags |= WQ_FLAG_EXCLUSIVE;
193 __add_wait_queue_entry_tail(wq_head, wq_entry);
194}
195
196static inline void
197__remove_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
198{
199 list_del(&wq_entry->entry);
200}
201
202void __wake_up(struct wait_queue_head *wq_head, unsigned int mode, int nr, void *key);
203void __wake_up_locked_key(struct wait_queue_head *wq_head, unsigned int mode, void *key);
204void __wake_up_locked_key_bookmark(struct wait_queue_head *wq_head,
205 unsigned int mode, void *key, wait_queue_entry_t *bookmark);
206void __wake_up_sync_key(struct wait_queue_head *wq_head, unsigned int mode, void *key);
207void __wake_up_locked_sync_key(struct wait_queue_head *wq_head, unsigned int mode, void *key);
208void __wake_up_locked(struct wait_queue_head *wq_head, unsigned int mode, int nr);
209void __wake_up_sync(struct wait_queue_head *wq_head, unsigned int mode);
210
211#define wake_up(x) __wake_up(x, TASK_NORMAL, 1, NULL)
212#define wake_up_nr(x, nr) __wake_up(x, TASK_NORMAL, nr, NULL)
213#define wake_up_all(x) __wake_up(x, TASK_NORMAL, 0, NULL)
214#define wake_up_locked(x) __wake_up_locked((x), TASK_NORMAL, 1)
215#define wake_up_all_locked(x) __wake_up_locked((x), TASK_NORMAL, 0)
216
217#define wake_up_interruptible(x) __wake_up(x, TASK_INTERRUPTIBLE, 1, NULL)
218#define wake_up_interruptible_nr(x, nr) __wake_up(x, TASK_INTERRUPTIBLE, nr, NULL)
219#define wake_up_interruptible_all(x) __wake_up(x, TASK_INTERRUPTIBLE, 0, NULL)
220#define wake_up_interruptible_sync(x) __wake_up_sync((x), TASK_INTERRUPTIBLE)
221
222/*
223 * Wakeup macros to be used to report events to the targets.
224 */
225#define poll_to_key(m) ((void *)(__force uintptr_t)(__poll_t)(m))
226#define key_to_poll(m) ((__force __poll_t)(uintptr_t)(void *)(m))
227#define wake_up_poll(x, m) \
228 __wake_up(x, TASK_NORMAL, 1, poll_to_key(m))
229#define wake_up_locked_poll(x, m) \
230 __wake_up_locked_key((x), TASK_NORMAL, poll_to_key(m))
231#define wake_up_interruptible_poll(x, m) \
232 __wake_up(x, TASK_INTERRUPTIBLE, 1, poll_to_key(m))
233#define wake_up_interruptible_sync_poll(x, m) \
234 __wake_up_sync_key((x), TASK_INTERRUPTIBLE, poll_to_key(m))
235#define wake_up_interruptible_sync_poll_locked(x, m) \
236 __wake_up_locked_sync_key((x), TASK_INTERRUPTIBLE, poll_to_key(m))
237
238#define ___wait_cond_timeout(condition) \
239({ \
240 bool __cond = (condition); \
241 if (__cond && !__ret) \
242 __ret = 1; \
243 __cond || !__ret; \
244})
245
246#define ___wait_is_interruptible(state) \
247 (!__builtin_constant_p(state) || \
248 state == TASK_INTERRUPTIBLE || state == TASK_KILLABLE) \
249
250extern void init_wait_entry(struct wait_queue_entry *wq_entry, int flags);
251
252/*
253 * The below macro ___wait_event() has an explicit shadow of the __ret
254 * variable when used from the wait_event_*() macros.
255 *
256 * This is so that both can use the ___wait_cond_timeout() construct
257 * to wrap the condition.
258 *
259 * The type inconsistency of the wait_event_*() __ret variable is also
260 * on purpose; we use long where we can return timeout values and int
261 * otherwise.
262 */
263
264#define ___wait_event(wq_head, condition, state, exclusive, ret, cmd) \
265({ \
266 __label__ __out; \
267 struct wait_queue_entry __wq_entry; \
268 long __ret = ret; /* explicit shadow */ \
269 \
270 init_wait_entry(&__wq_entry, exclusive ? WQ_FLAG_EXCLUSIVE : 0); \
271 for (;;) { \
272 long __int = prepare_to_wait_event(&wq_head, &__wq_entry, state);\
273 \
274 if (condition) \
275 break; \
276 \
277 if (___wait_is_interruptible(state) && __int) { \
278 __ret = __int; \
279 goto __out; \
280 } \
281 \
282 cmd; \
283 } \
284 finish_wait(&wq_head, &__wq_entry); \
285__out: __ret; \
286})
287
288#define __wait_event(wq_head, condition) \
289 (void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 0, 0, \
290 schedule())
291
292/**
293 * wait_event - sleep until a condition gets true
294 * @wq_head: the waitqueue to wait on
295 * @condition: a C expression for the event to wait for
296 *
297 * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
298 * @condition evaluates to true. The @condition is checked each time
299 * the waitqueue @wq_head is woken up.
300 *
301 * wake_up() has to be called after changing any variable that could
302 * change the result of the wait condition.
303 */
304#define wait_event(wq_head, condition) \
305do { \
306 might_sleep(); \
307 if (condition) \
308 break; \
309 __wait_event(wq_head, condition); \
310} while (0)
311
312#define __io_wait_event(wq_head, condition) \
313 (void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 0, 0, \
314 io_schedule())
315
316/*
317 * io_wait_event() -- like wait_event() but with io_schedule()
318 */
319#define io_wait_event(wq_head, condition) \
320do { \
321 might_sleep(); \
322 if (condition) \
323 break; \
324 __io_wait_event(wq_head, condition); \
325} while (0)
326
327#define __wait_event_freezable(wq_head, condition) \
328 ___wait_event(wq_head, condition, TASK_INTERRUPTIBLE, 0, 0, \
329 freezable_schedule())
330
331/**
332 * wait_event_freezable - sleep (or freeze) until a condition gets true
333 * @wq_head: the waitqueue to wait on
334 * @condition: a C expression for the event to wait for
335 *
336 * The process is put to sleep (TASK_INTERRUPTIBLE -- so as not to contribute
337 * to system load) until the @condition evaluates to true. The
338 * @condition is checked each time the waitqueue @wq_head is woken up.
339 *
340 * wake_up() has to be called after changing any variable that could
341 * change the result of the wait condition.
342 */
343#define wait_event_freezable(wq_head, condition) \
344({ \
345 int __ret = 0; \
346 might_sleep(); \
347 if (!(condition)) \
348 __ret = __wait_event_freezable(wq_head, condition); \
349 __ret; \
350})
351
352#define __wait_event_timeout(wq_head, condition, timeout) \
353 ___wait_event(wq_head, ___wait_cond_timeout(condition), \
354 TASK_UNINTERRUPTIBLE, 0, timeout, \
355 __ret = schedule_timeout(__ret))
356
357/**
358 * wait_event_timeout - sleep until a condition gets true or a timeout elapses
359 * @wq_head: the waitqueue to wait on
360 * @condition: a C expression for the event to wait for
361 * @timeout: timeout, in jiffies
362 *
363 * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
364 * @condition evaluates to true. The @condition is checked each time
365 * the waitqueue @wq_head is woken up.
366 *
367 * wake_up() has to be called after changing any variable that could
368 * change the result of the wait condition.
369 *
370 * Returns:
371 * 0 if the @condition evaluated to %false after the @timeout elapsed,
372 * 1 if the @condition evaluated to %true after the @timeout elapsed,
373 * or the remaining jiffies (at least 1) if the @condition evaluated
374 * to %true before the @timeout elapsed.
375 */
376#define wait_event_timeout(wq_head, condition, timeout) \
377({ \
378 long __ret = timeout; \
379 might_sleep(); \
380 if (!___wait_cond_timeout(condition)) \
381 __ret = __wait_event_timeout(wq_head, condition, timeout); \
382 __ret; \
383})
384
385#define __wait_event_freezable_timeout(wq_head, condition, timeout) \
386 ___wait_event(wq_head, ___wait_cond_timeout(condition), \
387 TASK_INTERRUPTIBLE, 0, timeout, \
388 __ret = freezable_schedule_timeout(__ret))
389
390/*
391 * like wait_event_timeout() -- except it uses TASK_INTERRUPTIBLE to avoid
392 * increasing load and is freezable.
393 */
394#define wait_event_freezable_timeout(wq_head, condition, timeout) \
395({ \
396 long __ret = timeout; \
397 might_sleep(); \
398 if (!___wait_cond_timeout(condition)) \
399 __ret = __wait_event_freezable_timeout(wq_head, condition, timeout); \
400 __ret; \
401})
402
403#define __wait_event_exclusive_cmd(wq_head, condition, cmd1, cmd2) \
404 (void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 1, 0, \
405 cmd1; schedule(); cmd2)
406/*
407 * Just like wait_event_cmd(), except it sets exclusive flag
408 */
409#define wait_event_exclusive_cmd(wq_head, condition, cmd1, cmd2) \
410do { \
411 if (condition) \
412 break; \
413 __wait_event_exclusive_cmd(wq_head, condition, cmd1, cmd2); \
414} while (0)
415
416#define __wait_event_cmd(wq_head, condition, cmd1, cmd2) \
417 (void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 0, 0, \
418 cmd1; schedule(); cmd2)
419
420/**
421 * wait_event_cmd - sleep until a condition gets true
422 * @wq_head: the waitqueue to wait on
423 * @condition: a C expression for the event to wait for
424 * @cmd1: the command will be executed before sleep
425 * @cmd2: the command will be executed after sleep
426 *
427 * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
428 * @condition evaluates to true. The @condition is checked each time
429 * the waitqueue @wq_head is woken up.
430 *
431 * wake_up() has to be called after changing any variable that could
432 * change the result of the wait condition.
433 */
434#define wait_event_cmd(wq_head, condition, cmd1, cmd2) \
435do { \
436 if (condition) \
437 break; \
438 __wait_event_cmd(wq_head, condition, cmd1, cmd2); \
439} while (0)
440
441#define __wait_event_interruptible(wq_head, condition) \
442 ___wait_event(wq_head, condition, TASK_INTERRUPTIBLE, 0, 0, \
443 schedule())
444
445/**
446 * wait_event_interruptible - sleep until a condition gets true
447 * @wq_head: the waitqueue to wait on
448 * @condition: a C expression for the event to wait for
449 *
450 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
451 * @condition evaluates to true or a signal is received.
452 * The @condition is checked each time the waitqueue @wq_head is woken up.
453 *
454 * wake_up() has to be called after changing any variable that could
455 * change the result of the wait condition.
456 *
457 * The function will return -ERESTARTSYS if it was interrupted by a
458 * signal and 0 if @condition evaluated to true.
459 */
460#define wait_event_interruptible(wq_head, condition) \
461({ \
462 int __ret = 0; \
463 might_sleep(); \
464 if (!(condition)) \
465 __ret = __wait_event_interruptible(wq_head, condition); \
466 __ret; \
467})
468
469#define __wait_event_interruptible_timeout(wq_head, condition, timeout) \
470 ___wait_event(wq_head, ___wait_cond_timeout(condition), \
471 TASK_INTERRUPTIBLE, 0, timeout, \
472 __ret = schedule_timeout(__ret))
473
474/**
475 * wait_event_interruptible_timeout - sleep until a condition gets true or a timeout elapses
476 * @wq_head: the waitqueue to wait on
477 * @condition: a C expression for the event to wait for
478 * @timeout: timeout, in jiffies
479 *
480 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
481 * @condition evaluates to true or a signal is received.
482 * The @condition is checked each time the waitqueue @wq_head is woken up.
483 *
484 * wake_up() has to be called after changing any variable that could
485 * change the result of the wait condition.
486 *
487 * Returns:
488 * 0 if the @condition evaluated to %false after the @timeout elapsed,
489 * 1 if the @condition evaluated to %true after the @timeout elapsed,
490 * the remaining jiffies (at least 1) if the @condition evaluated
491 * to %true before the @timeout elapsed, or -%ERESTARTSYS if it was
492 * interrupted by a signal.
493 */
494#define wait_event_interruptible_timeout(wq_head, condition, timeout) \
495({ \
496 long __ret = timeout; \
497 might_sleep(); \
498 if (!___wait_cond_timeout(condition)) \
499 __ret = __wait_event_interruptible_timeout(wq_head, \
500 condition, timeout); \
501 __ret; \
502})
503
504#define __wait_event_hrtimeout(wq_head, condition, timeout, state) \
505({ \
506 int __ret = 0; \
507 struct hrtimer_sleeper __t; \
508 \
509 hrtimer_init_sleeper_on_stack(&__t, CLOCK_MONOTONIC, \
510 HRTIMER_MODE_REL); \
511 if ((timeout) != KTIME_MAX) \
512 hrtimer_start_range_ns(&__t.timer, timeout, \
513 current->timer_slack_ns, \
514 HRTIMER_MODE_REL); \
515 \
516 __ret = ___wait_event(wq_head, condition, state, 0, 0, \
517 if (!__t.task) { \
518 __ret = -ETIME; \
519 break; \
520 } \
521 schedule()); \
522 \
523 hrtimer_cancel(&__t.timer); \
524 destroy_hrtimer_on_stack(&__t.timer); \
525 __ret; \
526})
527
528/**
529 * wait_event_hrtimeout - sleep until a condition gets true or a timeout elapses
530 * @wq_head: the waitqueue to wait on
531 * @condition: a C expression for the event to wait for
532 * @timeout: timeout, as a ktime_t
533 *
534 * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
535 * @condition evaluates to true or a signal is received.
536 * The @condition is checked each time the waitqueue @wq_head is woken up.
537 *
538 * wake_up() has to be called after changing any variable that could
539 * change the result of the wait condition.
540 *
541 * The function returns 0 if @condition became true, or -ETIME if the timeout
542 * elapsed.
543 */
544#define wait_event_hrtimeout(wq_head, condition, timeout) \
545({ \
546 int __ret = 0; \
547 might_sleep(); \
548 if (!(condition)) \
549 __ret = __wait_event_hrtimeout(wq_head, condition, timeout, \
550 TASK_UNINTERRUPTIBLE); \
551 __ret; \
552})
553
554/**
555 * wait_event_interruptible_hrtimeout - sleep until a condition gets true or a timeout elapses
556 * @wq: the waitqueue to wait on
557 * @condition: a C expression for the event to wait for
558 * @timeout: timeout, as a ktime_t
559 *
560 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
561 * @condition evaluates to true or a signal is received.
562 * The @condition is checked each time the waitqueue @wq is woken up.
563 *
564 * wake_up() has to be called after changing any variable that could
565 * change the result of the wait condition.
566 *
567 * The function returns 0 if @condition became true, -ERESTARTSYS if it was
568 * interrupted by a signal, or -ETIME if the timeout elapsed.
569 */
570#define wait_event_interruptible_hrtimeout(wq, condition, timeout) \
571({ \
572 long __ret = 0; \
573 might_sleep(); \
574 if (!(condition)) \
575 __ret = __wait_event_hrtimeout(wq, condition, timeout, \
576 TASK_INTERRUPTIBLE); \
577 __ret; \
578})
579
580#define __wait_event_interruptible_exclusive(wq, condition) \
581 ___wait_event(wq, condition, TASK_INTERRUPTIBLE, 1, 0, \
582 schedule())
583
584#define wait_event_interruptible_exclusive(wq, condition) \
585({ \
586 int __ret = 0; \
587 might_sleep(); \
588 if (!(condition)) \
589 __ret = __wait_event_interruptible_exclusive(wq, condition); \
590 __ret; \
591})
592
593#define __wait_event_killable_exclusive(wq, condition) \
594 ___wait_event(wq, condition, TASK_KILLABLE, 1, 0, \
595 schedule())
596
597#define wait_event_killable_exclusive(wq, condition) \
598({ \
599 int __ret = 0; \
600 might_sleep(); \
601 if (!(condition)) \
602 __ret = __wait_event_killable_exclusive(wq, condition); \
603 __ret; \
604})
605
606
607#define __wait_event_freezable_exclusive(wq, condition) \
608 ___wait_event(wq, condition, TASK_INTERRUPTIBLE, 1, 0, \
609 freezable_schedule())
610
611#define wait_event_freezable_exclusive(wq, condition) \
612({ \
613 int __ret = 0; \
614 might_sleep(); \
615 if (!(condition)) \
616 __ret = __wait_event_freezable_exclusive(wq, condition); \
617 __ret; \
618})
619
620/**
621 * wait_event_idle - wait for a condition without contributing to system load
622 * @wq_head: the waitqueue to wait on
623 * @condition: a C expression for the event to wait for
624 *
625 * The process is put to sleep (TASK_IDLE) until the
626 * @condition evaluates to true.
627 * The @condition is checked each time the waitqueue @wq_head is woken up.
628 *
629 * wake_up() has to be called after changing any variable that could
630 * change the result of the wait condition.
631 *
632 */
633#define wait_event_idle(wq_head, condition) \
634do { \
635 might_sleep(); \
636 if (!(condition)) \
637 ___wait_event(wq_head, condition, TASK_IDLE, 0, 0, schedule()); \
638} while (0)
639
640/**
641 * wait_event_idle_exclusive - wait for a condition with contributing to system load
642 * @wq_head: the waitqueue to wait on
643 * @condition: a C expression for the event to wait for
644 *
645 * The process is put to sleep (TASK_IDLE) until the
646 * @condition evaluates to true.
647 * The @condition is checked each time the waitqueue @wq_head is woken up.
648 *
649 * The process is put on the wait queue with an WQ_FLAG_EXCLUSIVE flag
650 * set thus if other processes wait on the same list, when this
651 * process is woken further processes are not considered.
652 *
653 * wake_up() has to be called after changing any variable that could
654 * change the result of the wait condition.
655 *
656 */
657#define wait_event_idle_exclusive(wq_head, condition) \
658do { \
659 might_sleep(); \
660 if (!(condition)) \
661 ___wait_event(wq_head, condition, TASK_IDLE, 1, 0, schedule()); \
662} while (0)
663
664#define __wait_event_idle_timeout(wq_head, condition, timeout) \
665 ___wait_event(wq_head, ___wait_cond_timeout(condition), \
666 TASK_IDLE, 0, timeout, \
667 __ret = schedule_timeout(__ret))
668
669/**
670 * wait_event_idle_timeout - sleep without load until a condition becomes true or a timeout elapses
671 * @wq_head: the waitqueue to wait on
672 * @condition: a C expression for the event to wait for
673 * @timeout: timeout, in jiffies
674 *
675 * The process is put to sleep (TASK_IDLE) until the
676 * @condition evaluates to true. The @condition is checked each time
677 * the waitqueue @wq_head is woken up.
678 *
679 * wake_up() has to be called after changing any variable that could
680 * change the result of the wait condition.
681 *
682 * Returns:
683 * 0 if the @condition evaluated to %false after the @timeout elapsed,
684 * 1 if the @condition evaluated to %true after the @timeout elapsed,
685 * or the remaining jiffies (at least 1) if the @condition evaluated
686 * to %true before the @timeout elapsed.
687 */
688#define wait_event_idle_timeout(wq_head, condition, timeout) \
689({ \
690 long __ret = timeout; \
691 might_sleep(); \
692 if (!___wait_cond_timeout(condition)) \
693 __ret = __wait_event_idle_timeout(wq_head, condition, timeout); \
694 __ret; \
695})
696
697#define __wait_event_idle_exclusive_timeout(wq_head, condition, timeout) \
698 ___wait_event(wq_head, ___wait_cond_timeout(condition), \
699 TASK_IDLE, 1, timeout, \
700 __ret = schedule_timeout(__ret))
701
702/**
703 * wait_event_idle_exclusive_timeout - sleep without load until a condition becomes true or a timeout elapses
704 * @wq_head: the waitqueue to wait on
705 * @condition: a C expression for the event to wait for
706 * @timeout: timeout, in jiffies
707 *
708 * The process is put to sleep (TASK_IDLE) until the
709 * @condition evaluates to true. The @condition is checked each time
710 * the waitqueue @wq_head is woken up.
711 *
712 * The process is put on the wait queue with an WQ_FLAG_EXCLUSIVE flag
713 * set thus if other processes wait on the same list, when this
714 * process is woken further processes are not considered.
715 *
716 * wake_up() has to be called after changing any variable that could
717 * change the result of the wait condition.
718 *
719 * Returns:
720 * 0 if the @condition evaluated to %false after the @timeout elapsed,
721 * 1 if the @condition evaluated to %true after the @timeout elapsed,
722 * or the remaining jiffies (at least 1) if the @condition evaluated
723 * to %true before the @timeout elapsed.
724 */
725#define wait_event_idle_exclusive_timeout(wq_head, condition, timeout) \
726({ \
727 long __ret = timeout; \
728 might_sleep(); \
729 if (!___wait_cond_timeout(condition)) \
730 __ret = __wait_event_idle_exclusive_timeout(wq_head, condition, timeout);\
731 __ret; \
732})
733
734extern int do_wait_intr(wait_queue_head_t *, wait_queue_entry_t *);
735extern int do_wait_intr_irq(wait_queue_head_t *, wait_queue_entry_t *);
736
737#define __wait_event_interruptible_locked(wq, condition, exclusive, fn) \
738({ \
739 int __ret; \
740 DEFINE_WAIT(__wait); \
741 if (exclusive) \
742 __wait.flags |= WQ_FLAG_EXCLUSIVE; \
743 do { \
744 __ret = fn(&(wq), &__wait); \
745 if (__ret) \
746 break; \
747 } while (!(condition)); \
748 __remove_wait_queue(&(wq), &__wait); \
749 __set_current_state(TASK_RUNNING); \
750 __ret; \
751})
752
753
754/**
755 * wait_event_interruptible_locked - sleep until a condition gets true
756 * @wq: the waitqueue to wait on
757 * @condition: a C expression for the event to wait for
758 *
759 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
760 * @condition evaluates to true or a signal is received.
761 * The @condition is checked each time the waitqueue @wq is woken up.
762 *
763 * It must be called with wq.lock being held. This spinlock is
764 * unlocked while sleeping but @condition testing is done while lock
765 * is held and when this macro exits the lock is held.
766 *
767 * The lock is locked/unlocked using spin_lock()/spin_unlock()
768 * functions which must match the way they are locked/unlocked outside
769 * of this macro.
770 *
771 * wake_up_locked() has to be called after changing any variable that could
772 * change the result of the wait condition.
773 *
774 * The function will return -ERESTARTSYS if it was interrupted by a
775 * signal and 0 if @condition evaluated to true.
776 */
777#define wait_event_interruptible_locked(wq, condition) \
778 ((condition) \
779 ? 0 : __wait_event_interruptible_locked(wq, condition, 0, do_wait_intr))
780
781/**
782 * wait_event_interruptible_locked_irq - sleep until a condition gets true
783 * @wq: the waitqueue to wait on
784 * @condition: a C expression for the event to wait for
785 *
786 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
787 * @condition evaluates to true or a signal is received.
788 * The @condition is checked each time the waitqueue @wq is woken up.
789 *
790 * It must be called with wq.lock being held. This spinlock is
791 * unlocked while sleeping but @condition testing is done while lock
792 * is held and when this macro exits the lock is held.
793 *
794 * The lock is locked/unlocked using spin_lock_irq()/spin_unlock_irq()
795 * functions which must match the way they are locked/unlocked outside
796 * of this macro.
797 *
798 * wake_up_locked() has to be called after changing any variable that could
799 * change the result of the wait condition.
800 *
801 * The function will return -ERESTARTSYS if it was interrupted by a
802 * signal and 0 if @condition evaluated to true.
803 */
804#define wait_event_interruptible_locked_irq(wq, condition) \
805 ((condition) \
806 ? 0 : __wait_event_interruptible_locked(wq, condition, 0, do_wait_intr_irq))
807
808/**
809 * wait_event_interruptible_exclusive_locked - sleep exclusively until a condition gets true
810 * @wq: the waitqueue to wait on
811 * @condition: a C expression for the event to wait for
812 *
813 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
814 * @condition evaluates to true or a signal is received.
815 * The @condition is checked each time the waitqueue @wq is woken up.
816 *
817 * It must be called with wq.lock being held. This spinlock is
818 * unlocked while sleeping but @condition testing is done while lock
819 * is held and when this macro exits the lock is held.
820 *
821 * The lock is locked/unlocked using spin_lock()/spin_unlock()
822 * functions which must match the way they are locked/unlocked outside
823 * of this macro.
824 *
825 * The process is put on the wait queue with an WQ_FLAG_EXCLUSIVE flag
826 * set thus when other process waits process on the list if this
827 * process is awaken further processes are not considered.
828 *
829 * wake_up_locked() has to be called after changing any variable that could
830 * change the result of the wait condition.
831 *
832 * The function will return -ERESTARTSYS if it was interrupted by a
833 * signal and 0 if @condition evaluated to true.
834 */
835#define wait_event_interruptible_exclusive_locked(wq, condition) \
836 ((condition) \
837 ? 0 : __wait_event_interruptible_locked(wq, condition, 1, do_wait_intr))
838
839/**
840 * wait_event_interruptible_exclusive_locked_irq - sleep until a condition gets true
841 * @wq: the waitqueue to wait on
842 * @condition: a C expression for the event to wait for
843 *
844 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
845 * @condition evaluates to true or a signal is received.
846 * The @condition is checked each time the waitqueue @wq is woken up.
847 *
848 * It must be called with wq.lock being held. This spinlock is
849 * unlocked while sleeping but @condition testing is done while lock
850 * is held and when this macro exits the lock is held.
851 *
852 * The lock is locked/unlocked using spin_lock_irq()/spin_unlock_irq()
853 * functions which must match the way they are locked/unlocked outside
854 * of this macro.
855 *
856 * The process is put on the wait queue with an WQ_FLAG_EXCLUSIVE flag
857 * set thus when other process waits process on the list if this
858 * process is awaken further processes are not considered.
859 *
860 * wake_up_locked() has to be called after changing any variable that could
861 * change the result of the wait condition.
862 *
863 * The function will return -ERESTARTSYS if it was interrupted by a
864 * signal and 0 if @condition evaluated to true.
865 */
866#define wait_event_interruptible_exclusive_locked_irq(wq, condition) \
867 ((condition) \
868 ? 0 : __wait_event_interruptible_locked(wq, condition, 1, do_wait_intr_irq))
869
870
871#define __wait_event_killable(wq, condition) \
872 ___wait_event(wq, condition, TASK_KILLABLE, 0, 0, schedule())
873
874/**
875 * wait_event_killable - sleep until a condition gets true
876 * @wq_head: the waitqueue to wait on
877 * @condition: a C expression for the event to wait for
878 *
879 * The process is put to sleep (TASK_KILLABLE) until the
880 * @condition evaluates to true or a signal is received.
881 * The @condition is checked each time the waitqueue @wq_head is woken up.
882 *
883 * wake_up() has to be called after changing any variable that could
884 * change the result of the wait condition.
885 *
886 * The function will return -ERESTARTSYS if it was interrupted by a
887 * signal and 0 if @condition evaluated to true.
888 */
889#define wait_event_killable(wq_head, condition) \
890({ \
891 int __ret = 0; \
892 might_sleep(); \
893 if (!(condition)) \
894 __ret = __wait_event_killable(wq_head, condition); \
895 __ret; \
896})
897
898#define __wait_event_killable_timeout(wq_head, condition, timeout) \
899 ___wait_event(wq_head, ___wait_cond_timeout(condition), \
900 TASK_KILLABLE, 0, timeout, \
901 __ret = schedule_timeout(__ret))
902
903/**
904 * wait_event_killable_timeout - sleep until a condition gets true or a timeout elapses
905 * @wq_head: the waitqueue to wait on
906 * @condition: a C expression for the event to wait for
907 * @timeout: timeout, in jiffies
908 *
909 * The process is put to sleep (TASK_KILLABLE) until the
910 * @condition evaluates to true or a kill signal is received.
911 * The @condition is checked each time the waitqueue @wq_head is woken up.
912 *
913 * wake_up() has to be called after changing any variable that could
914 * change the result of the wait condition.
915 *
916 * Returns:
917 * 0 if the @condition evaluated to %false after the @timeout elapsed,
918 * 1 if the @condition evaluated to %true after the @timeout elapsed,
919 * the remaining jiffies (at least 1) if the @condition evaluated
920 * to %true before the @timeout elapsed, or -%ERESTARTSYS if it was
921 * interrupted by a kill signal.
922 *
923 * Only kill signals interrupt this process.
924 */
925#define wait_event_killable_timeout(wq_head, condition, timeout) \
926({ \
927 long __ret = timeout; \
928 might_sleep(); \
929 if (!___wait_cond_timeout(condition)) \
930 __ret = __wait_event_killable_timeout(wq_head, \
931 condition, timeout); \
932 __ret; \
933})
934
935
936#define __wait_event_lock_irq(wq_head, condition, lock, cmd) \
937 (void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 0, 0, \
938 spin_unlock_irq(&lock); \
939 cmd; \
940 schedule(); \
941 spin_lock_irq(&lock))
942
943/**
944 * wait_event_lock_irq_cmd - sleep until a condition gets true. The
945 * condition is checked under the lock. This
946 * is expected to be called with the lock
947 * taken.
948 * @wq_head: the waitqueue to wait on
949 * @condition: a C expression for the event to wait for
950 * @lock: a locked spinlock_t, which will be released before cmd
951 * and schedule() and reacquired afterwards.
952 * @cmd: a command which is invoked outside the critical section before
953 * sleep
954 *
955 * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
956 * @condition evaluates to true. The @condition is checked each time
957 * the waitqueue @wq_head is woken up.
958 *
959 * wake_up() has to be called after changing any variable that could
960 * change the result of the wait condition.
961 *
962 * This is supposed to be called while holding the lock. The lock is
963 * dropped before invoking the cmd and going to sleep and is reacquired
964 * afterwards.
965 */
966#define wait_event_lock_irq_cmd(wq_head, condition, lock, cmd) \
967do { \
968 if (condition) \
969 break; \
970 __wait_event_lock_irq(wq_head, condition, lock, cmd); \
971} while (0)
972
973/**
974 * wait_event_lock_irq - sleep until a condition gets true. The
975 * condition is checked under the lock. This
976 * is expected to be called with the lock
977 * taken.
978 * @wq_head: the waitqueue to wait on
979 * @condition: a C expression for the event to wait for
980 * @lock: a locked spinlock_t, which will be released before schedule()
981 * and reacquired afterwards.
982 *
983 * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
984 * @condition evaluates to true. The @condition is checked each time
985 * the waitqueue @wq_head is woken up.
986 *
987 * wake_up() has to be called after changing any variable that could
988 * change the result of the wait condition.
989 *
990 * This is supposed to be called while holding the lock. The lock is
991 * dropped before going to sleep and is reacquired afterwards.
992 */
993#define wait_event_lock_irq(wq_head, condition, lock) \
994do { \
995 if (condition) \
996 break; \
997 __wait_event_lock_irq(wq_head, condition, lock, ); \
998} while (0)
999
1000
1001#define __wait_event_interruptible_lock_irq(wq_head, condition, lock, cmd) \
1002 ___wait_event(wq_head, condition, TASK_INTERRUPTIBLE, 0, 0, \
1003 spin_unlock_irq(&lock); \
1004 cmd; \
1005 schedule(); \
1006 spin_lock_irq(&lock))
1007
1008/**
1009 * wait_event_interruptible_lock_irq_cmd - sleep until a condition gets true.
1010 * The condition is checked under the lock. This is expected to
1011 * be called with the lock taken.
1012 * @wq_head: the waitqueue to wait on
1013 * @condition: a C expression for the event to wait for
1014 * @lock: a locked spinlock_t, which will be released before cmd and
1015 * schedule() and reacquired afterwards.
1016 * @cmd: a command which is invoked outside the critical section before
1017 * sleep
1018 *
1019 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
1020 * @condition evaluates to true or a signal is received. The @condition is
1021 * checked each time the waitqueue @wq_head is woken up.
1022 *
1023 * wake_up() has to be called after changing any variable that could
1024 * change the result of the wait condition.
1025 *
1026 * This is supposed to be called while holding the lock. The lock is
1027 * dropped before invoking the cmd and going to sleep and is reacquired
1028 * afterwards.
1029 *
1030 * The macro will return -ERESTARTSYS if it was interrupted by a signal
1031 * and 0 if @condition evaluated to true.
1032 */
1033#define wait_event_interruptible_lock_irq_cmd(wq_head, condition, lock, cmd) \
1034({ \
1035 int __ret = 0; \
1036 if (!(condition)) \
1037 __ret = __wait_event_interruptible_lock_irq(wq_head, \
1038 condition, lock, cmd); \
1039 __ret; \
1040})
1041
1042/**
1043 * wait_event_interruptible_lock_irq - sleep until a condition gets true.
1044 * The condition is checked under the lock. This is expected
1045 * to be called with the lock taken.
1046 * @wq_head: the waitqueue to wait on
1047 * @condition: a C expression for the event to wait for
1048 * @lock: a locked spinlock_t, which will be released before schedule()
1049 * and reacquired afterwards.
1050 *
1051 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
1052 * @condition evaluates to true or signal is received. The @condition is
1053 * checked each time the waitqueue @wq_head is woken up.
1054 *
1055 * wake_up() has to be called after changing any variable that could
1056 * change the result of the wait condition.
1057 *
1058 * This is supposed to be called while holding the lock. The lock is
1059 * dropped before going to sleep and is reacquired afterwards.
1060 *
1061 * The macro will return -ERESTARTSYS if it was interrupted by a signal
1062 * and 0 if @condition evaluated to true.
1063 */
1064#define wait_event_interruptible_lock_irq(wq_head, condition, lock) \
1065({ \
1066 int __ret = 0; \
1067 if (!(condition)) \
1068 __ret = __wait_event_interruptible_lock_irq(wq_head, \
1069 condition, lock,); \
1070 __ret; \
1071})
1072
1073#define __wait_event_lock_irq_timeout(wq_head, condition, lock, timeout, state) \
1074 ___wait_event(wq_head, ___wait_cond_timeout(condition), \
1075 state, 0, timeout, \
1076 spin_unlock_irq(&lock); \
1077 __ret = schedule_timeout(__ret); \
1078 spin_lock_irq(&lock));
1079
1080/**
1081 * wait_event_interruptible_lock_irq_timeout - sleep until a condition gets
1082 * true or a timeout elapses. The condition is checked under
1083 * the lock. This is expected to be called with the lock taken.
1084 * @wq_head: the waitqueue to wait on
1085 * @condition: a C expression for the event to wait for
1086 * @lock: a locked spinlock_t, which will be released before schedule()
1087 * and reacquired afterwards.
1088 * @timeout: timeout, in jiffies
1089 *
1090 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
1091 * @condition evaluates to true or signal is received. The @condition is
1092 * checked each time the waitqueue @wq_head is woken up.
1093 *
1094 * wake_up() has to be called after changing any variable that could
1095 * change the result of the wait condition.
1096 *
1097 * This is supposed to be called while holding the lock. The lock is
1098 * dropped before going to sleep and is reacquired afterwards.
1099 *
1100 * The function returns 0 if the @timeout elapsed, -ERESTARTSYS if it
1101 * was interrupted by a signal, and the remaining jiffies otherwise
1102 * if the condition evaluated to true before the timeout elapsed.
1103 */
1104#define wait_event_interruptible_lock_irq_timeout(wq_head, condition, lock, \
1105 timeout) \
1106({ \
1107 long __ret = timeout; \
1108 if (!___wait_cond_timeout(condition)) \
1109 __ret = __wait_event_lock_irq_timeout( \
1110 wq_head, condition, lock, timeout, \
1111 TASK_INTERRUPTIBLE); \
1112 __ret; \
1113})
1114
1115#define wait_event_lock_irq_timeout(wq_head, condition, lock, timeout) \
1116({ \
1117 long __ret = timeout; \
1118 if (!___wait_cond_timeout(condition)) \
1119 __ret = __wait_event_lock_irq_timeout( \
1120 wq_head, condition, lock, timeout, \
1121 TASK_UNINTERRUPTIBLE); \
1122 __ret; \
1123})
1124
1125/*
1126 * Waitqueues which are removed from the waitqueue_head at wakeup time
1127 */
1128void prepare_to_wait(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state);
1129void prepare_to_wait_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state);
1130long prepare_to_wait_event(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state);
1131void finish_wait(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);
1132long wait_woken(struct wait_queue_entry *wq_entry, unsigned mode, long timeout);
1133int woken_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *key);
1134int autoremove_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *key);
1135
1136#define DEFINE_WAIT_FUNC(name, function) \
1137 struct wait_queue_entry name = { \
1138 .private = current, \
1139 .func = function, \
1140 .entry = LIST_HEAD_INIT((name).entry), \
1141 }
1142
1143#define DEFINE_WAIT(name) DEFINE_WAIT_FUNC(name, autoremove_wake_function)
1144
1145#define init_wait(wait) \
1146 do { \
1147 (wait)->private = current; \
1148 (wait)->func = autoremove_wake_function; \
1149 INIT_LIST_HEAD(&(wait)->entry); \
1150 (wait)->flags = 0; \
1151 } while (0)
1152
1153bool try_invoke_on_locked_down_task(struct task_struct *p, bool (*func)(struct task_struct *t, void *arg), void *arg);
1154
1155#endif /* _LINUX_WAIT_H */
1#ifndef _LINUX_WAIT_H
2#define _LINUX_WAIT_H
3
4#define WNOHANG 0x00000001
5#define WUNTRACED 0x00000002
6#define WSTOPPED WUNTRACED
7#define WEXITED 0x00000004
8#define WCONTINUED 0x00000008
9#define WNOWAIT 0x01000000 /* Don't reap, just poll status. */
10
11#define __WNOTHREAD 0x20000000 /* Don't wait on children of other threads in this group */
12#define __WALL 0x40000000 /* Wait on all children, regardless of type */
13#define __WCLONE 0x80000000 /* Wait only on non-SIGCHLD children */
14
15/* First argument to waitid: */
16#define P_ALL 0
17#define P_PID 1
18#define P_PGID 2
19
20#ifdef __KERNEL__
21
22#include <linux/list.h>
23#include <linux/stddef.h>
24#include <linux/spinlock.h>
25#include <asm/current.h>
26
27typedef struct __wait_queue wait_queue_t;
28typedef int (*wait_queue_func_t)(wait_queue_t *wait, unsigned mode, int flags, void *key);
29int default_wake_function(wait_queue_t *wait, unsigned mode, int flags, void *key);
30
31struct __wait_queue {
32 unsigned int flags;
33#define WQ_FLAG_EXCLUSIVE 0x01
34 void *private;
35 wait_queue_func_t func;
36 struct list_head task_list;
37};
38
39struct wait_bit_key {
40 void *flags;
41 int bit_nr;
42};
43
44struct wait_bit_queue {
45 struct wait_bit_key key;
46 wait_queue_t wait;
47};
48
49struct __wait_queue_head {
50 spinlock_t lock;
51 struct list_head task_list;
52};
53typedef struct __wait_queue_head wait_queue_head_t;
54
55struct task_struct;
56
57/*
58 * Macros for declaration and initialisaton of the datatypes
59 */
60
61#define __WAITQUEUE_INITIALIZER(name, tsk) { \
62 .private = tsk, \
63 .func = default_wake_function, \
64 .task_list = { NULL, NULL } }
65
66#define DECLARE_WAITQUEUE(name, tsk) \
67 wait_queue_t name = __WAITQUEUE_INITIALIZER(name, tsk)
68
69#define __WAIT_QUEUE_HEAD_INITIALIZER(name) { \
70 .lock = __SPIN_LOCK_UNLOCKED(name.lock), \
71 .task_list = { &(name).task_list, &(name).task_list } }
72
73#define DECLARE_WAIT_QUEUE_HEAD(name) \
74 wait_queue_head_t name = __WAIT_QUEUE_HEAD_INITIALIZER(name)
75
76#define __WAIT_BIT_KEY_INITIALIZER(word, bit) \
77 { .flags = word, .bit_nr = bit, }
78
79extern void __init_waitqueue_head(wait_queue_head_t *q, const char *name, struct lock_class_key *);
80
81#define init_waitqueue_head(q) \
82 do { \
83 static struct lock_class_key __key; \
84 \
85 __init_waitqueue_head((q), #q, &__key); \
86 } while (0)
87
88#ifdef CONFIG_LOCKDEP
89# define __WAIT_QUEUE_HEAD_INIT_ONSTACK(name) \
90 ({ init_waitqueue_head(&name); name; })
91# define DECLARE_WAIT_QUEUE_HEAD_ONSTACK(name) \
92 wait_queue_head_t name = __WAIT_QUEUE_HEAD_INIT_ONSTACK(name)
93#else
94# define DECLARE_WAIT_QUEUE_HEAD_ONSTACK(name) DECLARE_WAIT_QUEUE_HEAD(name)
95#endif
96
97static inline void init_waitqueue_entry(wait_queue_t *q, struct task_struct *p)
98{
99 q->flags = 0;
100 q->private = p;
101 q->func = default_wake_function;
102}
103
104static inline void init_waitqueue_func_entry(wait_queue_t *q,
105 wait_queue_func_t func)
106{
107 q->flags = 0;
108 q->private = NULL;
109 q->func = func;
110}
111
112static inline int waitqueue_active(wait_queue_head_t *q)
113{
114 return !list_empty(&q->task_list);
115}
116
117extern void add_wait_queue(wait_queue_head_t *q, wait_queue_t *wait);
118extern void add_wait_queue_exclusive(wait_queue_head_t *q, wait_queue_t *wait);
119extern void remove_wait_queue(wait_queue_head_t *q, wait_queue_t *wait);
120
121static inline void __add_wait_queue(wait_queue_head_t *head, wait_queue_t *new)
122{
123 list_add(&new->task_list, &head->task_list);
124}
125
126/*
127 * Used for wake-one threads:
128 */
129static inline void __add_wait_queue_exclusive(wait_queue_head_t *q,
130 wait_queue_t *wait)
131{
132 wait->flags |= WQ_FLAG_EXCLUSIVE;
133 __add_wait_queue(q, wait);
134}
135
136static inline void __add_wait_queue_tail(wait_queue_head_t *head,
137 wait_queue_t *new)
138{
139 list_add_tail(&new->task_list, &head->task_list);
140}
141
142static inline void __add_wait_queue_tail_exclusive(wait_queue_head_t *q,
143 wait_queue_t *wait)
144{
145 wait->flags |= WQ_FLAG_EXCLUSIVE;
146 __add_wait_queue_tail(q, wait);
147}
148
149static inline void __remove_wait_queue(wait_queue_head_t *head,
150 wait_queue_t *old)
151{
152 list_del(&old->task_list);
153}
154
155void __wake_up(wait_queue_head_t *q, unsigned int mode, int nr, void *key);
156void __wake_up_locked_key(wait_queue_head_t *q, unsigned int mode, void *key);
157void __wake_up_sync_key(wait_queue_head_t *q, unsigned int mode, int nr,
158 void *key);
159void __wake_up_locked(wait_queue_head_t *q, unsigned int mode, int nr);
160void __wake_up_sync(wait_queue_head_t *q, unsigned int mode, int nr);
161void __wake_up_bit(wait_queue_head_t *, void *, int);
162int __wait_on_bit(wait_queue_head_t *, struct wait_bit_queue *, int (*)(void *), unsigned);
163int __wait_on_bit_lock(wait_queue_head_t *, struct wait_bit_queue *, int (*)(void *), unsigned);
164void wake_up_bit(void *, int);
165int out_of_line_wait_on_bit(void *, int, int (*)(void *), unsigned);
166int out_of_line_wait_on_bit_lock(void *, int, int (*)(void *), unsigned);
167wait_queue_head_t *bit_waitqueue(void *, int);
168
169#define wake_up(x) __wake_up(x, TASK_NORMAL, 1, NULL)
170#define wake_up_nr(x, nr) __wake_up(x, TASK_NORMAL, nr, NULL)
171#define wake_up_all(x) __wake_up(x, TASK_NORMAL, 0, NULL)
172#define wake_up_locked(x) __wake_up_locked((x), TASK_NORMAL, 1)
173#define wake_up_all_locked(x) __wake_up_locked((x), TASK_NORMAL, 0)
174
175#define wake_up_interruptible(x) __wake_up(x, TASK_INTERRUPTIBLE, 1, NULL)
176#define wake_up_interruptible_nr(x, nr) __wake_up(x, TASK_INTERRUPTIBLE, nr, NULL)
177#define wake_up_interruptible_all(x) __wake_up(x, TASK_INTERRUPTIBLE, 0, NULL)
178#define wake_up_interruptible_sync(x) __wake_up_sync((x), TASK_INTERRUPTIBLE, 1)
179
180/*
181 * Wakeup macros to be used to report events to the targets.
182 */
183#define wake_up_poll(x, m) \
184 __wake_up(x, TASK_NORMAL, 1, (void *) (m))
185#define wake_up_locked_poll(x, m) \
186 __wake_up_locked_key((x), TASK_NORMAL, (void *) (m))
187#define wake_up_interruptible_poll(x, m) \
188 __wake_up(x, TASK_INTERRUPTIBLE, 1, (void *) (m))
189#define wake_up_interruptible_sync_poll(x, m) \
190 __wake_up_sync_key((x), TASK_INTERRUPTIBLE, 1, (void *) (m))
191
192#define __wait_event(wq, condition) \
193do { \
194 DEFINE_WAIT(__wait); \
195 \
196 for (;;) { \
197 prepare_to_wait(&wq, &__wait, TASK_UNINTERRUPTIBLE); \
198 if (condition) \
199 break; \
200 schedule(); \
201 } \
202 finish_wait(&wq, &__wait); \
203} while (0)
204
205/**
206 * wait_event - sleep until a condition gets true
207 * @wq: the waitqueue to wait on
208 * @condition: a C expression for the event to wait for
209 *
210 * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
211 * @condition evaluates to true. The @condition is checked each time
212 * the waitqueue @wq is woken up.
213 *
214 * wake_up() has to be called after changing any variable that could
215 * change the result of the wait condition.
216 */
217#define wait_event(wq, condition) \
218do { \
219 if (condition) \
220 break; \
221 __wait_event(wq, condition); \
222} while (0)
223
224#define __wait_event_timeout(wq, condition, ret) \
225do { \
226 DEFINE_WAIT(__wait); \
227 \
228 for (;;) { \
229 prepare_to_wait(&wq, &__wait, TASK_UNINTERRUPTIBLE); \
230 if (condition) \
231 break; \
232 ret = schedule_timeout(ret); \
233 if (!ret) \
234 break; \
235 } \
236 finish_wait(&wq, &__wait); \
237} while (0)
238
239/**
240 * wait_event_timeout - sleep until a condition gets true or a timeout elapses
241 * @wq: the waitqueue to wait on
242 * @condition: a C expression for the event to wait for
243 * @timeout: timeout, in jiffies
244 *
245 * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
246 * @condition evaluates to true. The @condition is checked each time
247 * the waitqueue @wq is woken up.
248 *
249 * wake_up() has to be called after changing any variable that could
250 * change the result of the wait condition.
251 *
252 * The function returns 0 if the @timeout elapsed, and the remaining
253 * jiffies if the condition evaluated to true before the timeout elapsed.
254 */
255#define wait_event_timeout(wq, condition, timeout) \
256({ \
257 long __ret = timeout; \
258 if (!(condition)) \
259 __wait_event_timeout(wq, condition, __ret); \
260 __ret; \
261})
262
263#define __wait_event_interruptible(wq, condition, ret) \
264do { \
265 DEFINE_WAIT(__wait); \
266 \
267 for (;;) { \
268 prepare_to_wait(&wq, &__wait, TASK_INTERRUPTIBLE); \
269 if (condition) \
270 break; \
271 if (!signal_pending(current)) { \
272 schedule(); \
273 continue; \
274 } \
275 ret = -ERESTARTSYS; \
276 break; \
277 } \
278 finish_wait(&wq, &__wait); \
279} while (0)
280
281/**
282 * wait_event_interruptible - sleep until a condition gets true
283 * @wq: the waitqueue to wait on
284 * @condition: a C expression for the event to wait for
285 *
286 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
287 * @condition evaluates to true or a signal is received.
288 * The @condition is checked each time the waitqueue @wq is woken up.
289 *
290 * wake_up() has to be called after changing any variable that could
291 * change the result of the wait condition.
292 *
293 * The function will return -ERESTARTSYS if it was interrupted by a
294 * signal and 0 if @condition evaluated to true.
295 */
296#define wait_event_interruptible(wq, condition) \
297({ \
298 int __ret = 0; \
299 if (!(condition)) \
300 __wait_event_interruptible(wq, condition, __ret); \
301 __ret; \
302})
303
304#define __wait_event_interruptible_timeout(wq, condition, ret) \
305do { \
306 DEFINE_WAIT(__wait); \
307 \
308 for (;;) { \
309 prepare_to_wait(&wq, &__wait, TASK_INTERRUPTIBLE); \
310 if (condition) \
311 break; \
312 if (!signal_pending(current)) { \
313 ret = schedule_timeout(ret); \
314 if (!ret) \
315 break; \
316 continue; \
317 } \
318 ret = -ERESTARTSYS; \
319 break; \
320 } \
321 finish_wait(&wq, &__wait); \
322} while (0)
323
324/**
325 * wait_event_interruptible_timeout - sleep until a condition gets true or a timeout elapses
326 * @wq: the waitqueue to wait on
327 * @condition: a C expression for the event to wait for
328 * @timeout: timeout, in jiffies
329 *
330 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
331 * @condition evaluates to true or a signal is received.
332 * The @condition is checked each time the waitqueue @wq is woken up.
333 *
334 * wake_up() has to be called after changing any variable that could
335 * change the result of the wait condition.
336 *
337 * The function returns 0 if the @timeout elapsed, -ERESTARTSYS if it
338 * was interrupted by a signal, and the remaining jiffies otherwise
339 * if the condition evaluated to true before the timeout elapsed.
340 */
341#define wait_event_interruptible_timeout(wq, condition, timeout) \
342({ \
343 long __ret = timeout; \
344 if (!(condition)) \
345 __wait_event_interruptible_timeout(wq, condition, __ret); \
346 __ret; \
347})
348
349#define __wait_event_interruptible_exclusive(wq, condition, ret) \
350do { \
351 DEFINE_WAIT(__wait); \
352 \
353 for (;;) { \
354 prepare_to_wait_exclusive(&wq, &__wait, \
355 TASK_INTERRUPTIBLE); \
356 if (condition) { \
357 finish_wait(&wq, &__wait); \
358 break; \
359 } \
360 if (!signal_pending(current)) { \
361 schedule(); \
362 continue; \
363 } \
364 ret = -ERESTARTSYS; \
365 abort_exclusive_wait(&wq, &__wait, \
366 TASK_INTERRUPTIBLE, NULL); \
367 break; \
368 } \
369} while (0)
370
371#define wait_event_interruptible_exclusive(wq, condition) \
372({ \
373 int __ret = 0; \
374 if (!(condition)) \
375 __wait_event_interruptible_exclusive(wq, condition, __ret);\
376 __ret; \
377})
378
379
380#define __wait_event_interruptible_locked(wq, condition, exclusive, irq) \
381({ \
382 int __ret = 0; \
383 DEFINE_WAIT(__wait); \
384 if (exclusive) \
385 __wait.flags |= WQ_FLAG_EXCLUSIVE; \
386 do { \
387 if (likely(list_empty(&__wait.task_list))) \
388 __add_wait_queue_tail(&(wq), &__wait); \
389 set_current_state(TASK_INTERRUPTIBLE); \
390 if (signal_pending(current)) { \
391 __ret = -ERESTARTSYS; \
392 break; \
393 } \
394 if (irq) \
395 spin_unlock_irq(&(wq).lock); \
396 else \
397 spin_unlock(&(wq).lock); \
398 schedule(); \
399 if (irq) \
400 spin_lock_irq(&(wq).lock); \
401 else \
402 spin_lock(&(wq).lock); \
403 } while (!(condition)); \
404 __remove_wait_queue(&(wq), &__wait); \
405 __set_current_state(TASK_RUNNING); \
406 __ret; \
407})
408
409
410/**
411 * wait_event_interruptible_locked - sleep until a condition gets true
412 * @wq: the waitqueue to wait on
413 * @condition: a C expression for the event to wait for
414 *
415 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
416 * @condition evaluates to true or a signal is received.
417 * The @condition is checked each time the waitqueue @wq is woken up.
418 *
419 * It must be called with wq.lock being held. This spinlock is
420 * unlocked while sleeping but @condition testing is done while lock
421 * is held and when this macro exits the lock is held.
422 *
423 * The lock is locked/unlocked using spin_lock()/spin_unlock()
424 * functions which must match the way they are locked/unlocked outside
425 * of this macro.
426 *
427 * wake_up_locked() has to be called after changing any variable that could
428 * change the result of the wait condition.
429 *
430 * The function will return -ERESTARTSYS if it was interrupted by a
431 * signal and 0 if @condition evaluated to true.
432 */
433#define wait_event_interruptible_locked(wq, condition) \
434 ((condition) \
435 ? 0 : __wait_event_interruptible_locked(wq, condition, 0, 0))
436
437/**
438 * wait_event_interruptible_locked_irq - sleep until a condition gets true
439 * @wq: the waitqueue to wait on
440 * @condition: a C expression for the event to wait for
441 *
442 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
443 * @condition evaluates to true or a signal is received.
444 * The @condition is checked each time the waitqueue @wq is woken up.
445 *
446 * It must be called with wq.lock being held. This spinlock is
447 * unlocked while sleeping but @condition testing is done while lock
448 * is held and when this macro exits the lock is held.
449 *
450 * The lock is locked/unlocked using spin_lock_irq()/spin_unlock_irq()
451 * functions which must match the way they are locked/unlocked outside
452 * of this macro.
453 *
454 * wake_up_locked() has to be called after changing any variable that could
455 * change the result of the wait condition.
456 *
457 * The function will return -ERESTARTSYS if it was interrupted by a
458 * signal and 0 if @condition evaluated to true.
459 */
460#define wait_event_interruptible_locked_irq(wq, condition) \
461 ((condition) \
462 ? 0 : __wait_event_interruptible_locked(wq, condition, 0, 1))
463
464/**
465 * wait_event_interruptible_exclusive_locked - sleep exclusively until a condition gets true
466 * @wq: the waitqueue to wait on
467 * @condition: a C expression for the event to wait for
468 *
469 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
470 * @condition evaluates to true or a signal is received.
471 * The @condition is checked each time the waitqueue @wq is woken up.
472 *
473 * It must be called with wq.lock being held. This spinlock is
474 * unlocked while sleeping but @condition testing is done while lock
475 * is held and when this macro exits the lock is held.
476 *
477 * The lock is locked/unlocked using spin_lock()/spin_unlock()
478 * functions which must match the way they are locked/unlocked outside
479 * of this macro.
480 *
481 * The process is put on the wait queue with an WQ_FLAG_EXCLUSIVE flag
482 * set thus when other process waits process on the list if this
483 * process is awaken further processes are not considered.
484 *
485 * wake_up_locked() has to be called after changing any variable that could
486 * change the result of the wait condition.
487 *
488 * The function will return -ERESTARTSYS if it was interrupted by a
489 * signal and 0 if @condition evaluated to true.
490 */
491#define wait_event_interruptible_exclusive_locked(wq, condition) \
492 ((condition) \
493 ? 0 : __wait_event_interruptible_locked(wq, condition, 1, 0))
494
495/**
496 * wait_event_interruptible_exclusive_locked_irq - sleep until a condition gets true
497 * @wq: the waitqueue to wait on
498 * @condition: a C expression for the event to wait for
499 *
500 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
501 * @condition evaluates to true or a signal is received.
502 * The @condition is checked each time the waitqueue @wq is woken up.
503 *
504 * It must be called with wq.lock being held. This spinlock is
505 * unlocked while sleeping but @condition testing is done while lock
506 * is held and when this macro exits the lock is held.
507 *
508 * The lock is locked/unlocked using spin_lock_irq()/spin_unlock_irq()
509 * functions which must match the way they are locked/unlocked outside
510 * of this macro.
511 *
512 * The process is put on the wait queue with an WQ_FLAG_EXCLUSIVE flag
513 * set thus when other process waits process on the list if this
514 * process is awaken further processes are not considered.
515 *
516 * wake_up_locked() has to be called after changing any variable that could
517 * change the result of the wait condition.
518 *
519 * The function will return -ERESTARTSYS if it was interrupted by a
520 * signal and 0 if @condition evaluated to true.
521 */
522#define wait_event_interruptible_exclusive_locked_irq(wq, condition) \
523 ((condition) \
524 ? 0 : __wait_event_interruptible_locked(wq, condition, 1, 1))
525
526
527
528#define __wait_event_killable(wq, condition, ret) \
529do { \
530 DEFINE_WAIT(__wait); \
531 \
532 for (;;) { \
533 prepare_to_wait(&wq, &__wait, TASK_KILLABLE); \
534 if (condition) \
535 break; \
536 if (!fatal_signal_pending(current)) { \
537 schedule(); \
538 continue; \
539 } \
540 ret = -ERESTARTSYS; \
541 break; \
542 } \
543 finish_wait(&wq, &__wait); \
544} while (0)
545
546/**
547 * wait_event_killable - sleep until a condition gets true
548 * @wq: the waitqueue to wait on
549 * @condition: a C expression for the event to wait for
550 *
551 * The process is put to sleep (TASK_KILLABLE) until the
552 * @condition evaluates to true or a signal is received.
553 * The @condition is checked each time the waitqueue @wq is woken up.
554 *
555 * wake_up() has to be called after changing any variable that could
556 * change the result of the wait condition.
557 *
558 * The function will return -ERESTARTSYS if it was interrupted by a
559 * signal and 0 if @condition evaluated to true.
560 */
561#define wait_event_killable(wq, condition) \
562({ \
563 int __ret = 0; \
564 if (!(condition)) \
565 __wait_event_killable(wq, condition, __ret); \
566 __ret; \
567})
568
569/*
570 * These are the old interfaces to sleep waiting for an event.
571 * They are racy. DO NOT use them, use the wait_event* interfaces above.
572 * We plan to remove these interfaces.
573 */
574extern void sleep_on(wait_queue_head_t *q);
575extern long sleep_on_timeout(wait_queue_head_t *q,
576 signed long timeout);
577extern void interruptible_sleep_on(wait_queue_head_t *q);
578extern long interruptible_sleep_on_timeout(wait_queue_head_t *q,
579 signed long timeout);
580
581/*
582 * Waitqueues which are removed from the waitqueue_head at wakeup time
583 */
584void prepare_to_wait(wait_queue_head_t *q, wait_queue_t *wait, int state);
585void prepare_to_wait_exclusive(wait_queue_head_t *q, wait_queue_t *wait, int state);
586void finish_wait(wait_queue_head_t *q, wait_queue_t *wait);
587void abort_exclusive_wait(wait_queue_head_t *q, wait_queue_t *wait,
588 unsigned int mode, void *key);
589int autoremove_wake_function(wait_queue_t *wait, unsigned mode, int sync, void *key);
590int wake_bit_function(wait_queue_t *wait, unsigned mode, int sync, void *key);
591
592#define DEFINE_WAIT_FUNC(name, function) \
593 wait_queue_t name = { \
594 .private = current, \
595 .func = function, \
596 .task_list = LIST_HEAD_INIT((name).task_list), \
597 }
598
599#define DEFINE_WAIT(name) DEFINE_WAIT_FUNC(name, autoremove_wake_function)
600
601#define DEFINE_WAIT_BIT(name, word, bit) \
602 struct wait_bit_queue name = { \
603 .key = __WAIT_BIT_KEY_INITIALIZER(word, bit), \
604 .wait = { \
605 .private = current, \
606 .func = wake_bit_function, \
607 .task_list = \
608 LIST_HEAD_INIT((name).wait.task_list), \
609 }, \
610 }
611
612#define init_wait(wait) \
613 do { \
614 (wait)->private = current; \
615 (wait)->func = autoremove_wake_function; \
616 INIT_LIST_HEAD(&(wait)->task_list); \
617 (wait)->flags = 0; \
618 } while (0)
619
620/**
621 * wait_on_bit - wait for a bit to be cleared
622 * @word: the word being waited on, a kernel virtual address
623 * @bit: the bit of the word being waited on
624 * @action: the function used to sleep, which may take special actions
625 * @mode: the task state to sleep in
626 *
627 * There is a standard hashed waitqueue table for generic use. This
628 * is the part of the hashtable's accessor API that waits on a bit.
629 * For instance, if one were to have waiters on a bitflag, one would
630 * call wait_on_bit() in threads waiting for the bit to clear.
631 * One uses wait_on_bit() where one is waiting for the bit to clear,
632 * but has no intention of setting it.
633 */
634static inline int wait_on_bit(void *word, int bit,
635 int (*action)(void *), unsigned mode)
636{
637 if (!test_bit(bit, word))
638 return 0;
639 return out_of_line_wait_on_bit(word, bit, action, mode);
640}
641
642/**
643 * wait_on_bit_lock - wait for a bit to be cleared, when wanting to set it
644 * @word: the word being waited on, a kernel virtual address
645 * @bit: the bit of the word being waited on
646 * @action: the function used to sleep, which may take special actions
647 * @mode: the task state to sleep in
648 *
649 * There is a standard hashed waitqueue table for generic use. This
650 * is the part of the hashtable's accessor API that waits on a bit
651 * when one intends to set it, for instance, trying to lock bitflags.
652 * For instance, if one were to have waiters trying to set bitflag
653 * and waiting for it to clear before setting it, one would call
654 * wait_on_bit() in threads waiting to be able to set the bit.
655 * One uses wait_on_bit_lock() where one is waiting for the bit to
656 * clear with the intention of setting it, and when done, clearing it.
657 */
658static inline int wait_on_bit_lock(void *word, int bit,
659 int (*action)(void *), unsigned mode)
660{
661 if (!test_and_set_bit(bit, word))
662 return 0;
663 return out_of_line_wait_on_bit_lock(word, bit, action, mode);
664}
665
666#endif /* __KERNEL__ */
667
668#endif