include/linux/wait.h

   1 #ifndef _LINUX_WAIT_H
   2 #define _LINUX_WAIT_H
   3 /*
   4  * Linux wait queue related types and methods
   5  */
   6 #include <linux/list.h>
   7 #include <linux/stddef.h>
   8 #include <linux/spinlock.h>
   9
  10 #include <asm/current.h>
  11 #include <uapi/linux/wait.h>
  12
  13 typedef struct wait_queue_entry wait_queue_entry_t;
  14
  15 typedef int (*wait_queue_func_t)(struct wait_queue_entry *wq_entry, unsigned mode, int flags, void *key);
  16 int default_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, int flags, void *key);
  17
  18 /* wait_queue_entry::flags */
  19 #define WQ_FLAG_EXCLUSIVE       0x01
  20 #define WQ_FLAG_WOKEN           0x02
  21
  22 /*
  23  * A single wait-queue entry structure:
  24  */
  25 struct wait_queue_entry {
  26         unsigned int            flags;
  27         void                    *private;
  28         wait_queue_func_t       func;
  29         struct list_head        entry;
  30 };
  31
  32 struct wait_queue_head {
  33         spinlock_t              lock;
  34         struct list_head        head;
  35 };
  36 typedef struct wait_queue_head wait_queue_head_t;
  37
  38 struct task_struct;
  39
  40 /*
  41  * Macros for declaration and initialisaton of the datatypes
  42  */
  43
  44 #define __WAITQUEUE_INITIALIZER(name, tsk) {                                    \
  45         .private        = tsk,                                                  \
  46         .func           = default_wake_function,                                \
  47         .entry          = { NULL, NULL } }
  48
  49 #define DECLARE_WAITQUEUE(name, tsk)                                            \
  50         struct wait_queue_entry name = __WAITQUEUE_INITIALIZER(name, tsk)
  51
  52 #define __WAIT_QUEUE_HEAD_INITIALIZER(name) {                                   \
  53         .lock           = __SPIN_LOCK_UNLOCKED(name.lock),                      \
  54         .head           = { &(name).head, &(name).head } }
  55
  56 #define DECLARE_WAIT_QUEUE_HEAD(name) \
  57         struct wait_queue_head name = __WAIT_QUEUE_HEAD_INITIALIZER(name)
  58
  59 extern void __init_waitqueue_head(struct wait_queue_head *wq_head, const char *name, struct lock_class_key *);
  60
  61 #define init_waitqueue_head(wq_head)                                            \
  62         do {                                                                    \
  63                 static struct lock_class_key __key;                             \
  64                                                                                 \
  65                 __init_waitqueue_head((wq_head), #wq_head, &__key);             \
  66         } while (0)
  67
  68 #ifdef CONFIG_LOCKDEP
  69 # define __WAIT_QUEUE_HEAD_INIT_ONSTACK(name) \
  70         ({ init_waitqueue_head(&name); name; })
  71 # define DECLARE_WAIT_QUEUE_HEAD_ONSTACK(name) \
  72         struct wait_queue_head name = __WAIT_QUEUE_HEAD_INIT_ONSTACK(name)
  73 #else
  74 # define DECLARE_WAIT_QUEUE_HEAD_ONSTACK(name) DECLARE_WAIT_QUEUE_HEAD(name)
  75 #endif
  76
  77 static inline void init_waitqueue_entry(struct wait_queue_entry *wq_entry, struct task_struct *p)
  78 {
  79         wq_entry->flags         = 0;
  80         wq_entry->private       = p;
  81         wq_entry->func          = default_wake_function;
  82 }
  83
  84 static inline void
  85 init_waitqueue_func_entry(struct wait_queue_entry *wq_entry, wait_queue_func_t func)
  86 {
  87         wq_entry->flags         = 0;
  88         wq_entry->private       = NULL;
  89         wq_entry->func          = func;
  90 }
  91
  92 /**
  93  * waitqueue_active -- locklessly test for waiters on the queue
  94  * @wq_head: the waitqueue to test for waiters
  95  *
  96  * returns true if the wait list is not empty
  97  *
  98  * NOTE: this function is lockless and requires care, incorrect usage _will_
  99  * lead to sporadic and non-obvious failure.
 100  *
 101  * Use either while holding wait_queue_head::lock or when used for wakeups
 102  * with an extra smp_mb() like:
 103  *
 104  *      CPU0 - waker                    CPU1 - waiter
 105  *
 106  *                                      for (;;) {
 107  *      @cond = true;                     prepare_to_wait(&wq_head, &wait, state);
 108  *      smp_mb();                         // smp_mb() from set_current_state()
 109  *      if (waitqueue_active(wq_head))         if (@cond)
 110  *        wake_up(wq_head);                      break;
 111  *                                        schedule();
 112  *                                      }
 113  *                                      finish_wait(&wq_head, &wait);
 114  *
 115  * Because without the explicit smp_mb() it's possible for the
 116  * waitqueue_active() load to get hoisted over the @cond store such that we'll
 117  * observe an empty wait list while the waiter might not observe @cond.
 118  *
 119  * Also note that this 'optimization' trades a spin_lock() for an smp_mb(),
 120  * which (when the lock is uncontended) are of roughly equal cost.
 121  */
 122 static inline int waitqueue_active(struct wait_queue_head *wq_head)
 123 {
 124         return !list_empty(&wq_head->head);
 125 }
 126
 127 /**
 128  * wq_has_sleeper - check if there are any waiting processes
 129  * @wq_head: wait queue head
 130  *
 131  * Returns true if wq_head has waiting processes
 132  *
 133  * Please refer to the comment for waitqueue_active.
 134  */
 135 static inline bool wq_has_sleeper(struct wait_queue_head *wq_head)
 136 {
 137         /*
 138          * We need to be sure we are in sync with the
 139          * add_wait_queue modifications to the wait queue.
 140          *
 141          * This memory barrier should be paired with one on the
 142          * waiting side.
 143          */
 144         smp_mb();
 145         return waitqueue_active(wq_head);
 146 }
 147
 148 extern void add_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);
 149 extern void add_wait_queue_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);
 150 extern void remove_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);
 151
 152 static inline void __add_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
 153 {
 154         list_add(&wq_entry->entry, &wq_head->head);
 155 }
 156
 157 /*
 158  * Used for wake-one threads:
 159  */
 160 static inline void
 161 __add_wait_queue_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
 162 {
 163         wq_entry->flags |= WQ_FLAG_EXCLUSIVE;
 164         __add_wait_queue(wq_head, wq_entry);
 165 }
 166
 167 static inline void __add_wait_queue_entry_tail(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
 168 {
 169         list_add_tail(&wq_entry->entry, &wq_head->head);
 170 }
 171
 172 static inline void
 173 __add_wait_queue_entry_tail_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
 174 {
 175         wq_entry->flags |= WQ_FLAG_EXCLUSIVE;
 176         __add_wait_queue_entry_tail(wq_head, wq_entry);
 177 }
 178
 179 static inline void
 180 __remove_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
 181 {
 182         list_del(&wq_entry->entry);
 183 }
 184
 185 void __wake_up(struct wait_queue_head *wq_head, unsigned int mode, int nr, void *key);
 186 void __wake_up_locked_key(struct wait_queue_head *wq_head, unsigned int mode, void *key);
 187 void __wake_up_sync_key(struct wait_queue_head *wq_head, unsigned int mode, int nr, void *key);
 188 void __wake_up_locked(struct wait_queue_head *wq_head, unsigned int mode, int nr);
 189 void __wake_up_sync(struct wait_queue_head *wq_head, unsigned int mode, int nr);
 190
 191 #define wake_up(x)                      __wake_up(x, TASK_NORMAL, 1, NULL)
 192 #define wake_up_nr(x, nr)               __wake_up(x, TASK_NORMAL, nr, NULL)
 193 #define wake_up_all(x)                  __wake_up(x, TASK_NORMAL, 0, NULL)
 194 #define wake_up_locked(x)               __wake_up_locked((x), TASK_NORMAL, 1)
 195 #define wake_up_all_locked(x)           __wake_up_locked((x), TASK_NORMAL, 0)
 196
 197 #define wake_up_interruptible(x)        __wake_up(x, TASK_INTERRUPTIBLE, 1, NULL)
 198 #define wake_up_interruptible_nr(x, nr) __wake_up(x, TASK_INTERRUPTIBLE, nr, NULL)
 199 #define wake_up_interruptible_all(x)    __wake_up(x, TASK_INTERRUPTIBLE, 0, NULL)
 200 #define wake_up_interruptible_sync(x)   __wake_up_sync((x), TASK_INTERRUPTIBLE, 1)
 201
 202 /*
 203  * Wakeup macros to be used to report events to the targets.
 204  */
 205 #define wake_up_poll(x, m)                                                      \
 206         __wake_up(x, TASK_NORMAL, 1, (void *) (m))
 207 #define wake_up_locked_poll(x, m)                                               \
 208         __wake_up_locked_key((x), TASK_NORMAL, (void *) (m))
 209 #define wake_up_interruptible_poll(x, m)                                        \
 210         __wake_up(x, TASK_INTERRUPTIBLE, 1, (void *) (m))
 211 #define wake_up_interruptible_sync_poll(x, m)                                   \
 212         __wake_up_sync_key((x), TASK_INTERRUPTIBLE, 1, (void *) (m))
 213
 214 #define ___wait_cond_timeout(condition)                                         \
 215 ({                                                                              \
 216         bool __cond = (condition);                                              \
 217         if (__cond && !__ret)                                                   \
 218                 __ret = 1;                                                      \
 219         __cond || !__ret;                                                       \
 220 })
 221
 222 #define ___wait_is_interruptible(state)                                         \
 223         (!__builtin_constant_p(state) ||                                        \
 224                 state == TASK_INTERRUPTIBLE || state == TASK_KILLABLE)          \
 225
 226 extern void init_wait_entry(struct wait_queue_entry *wq_entry, int flags);
 227
 228 /*
 229  * The below macro ___wait_event() has an explicit shadow of the __ret
 230  * variable when used from the wait_event_*() macros.
 231  *
 232  * This is so that both can use the ___wait_cond_timeout() construct
 233  * to wrap the condition.
 234  *
 235  * The type inconsistency of the wait_event_*() __ret variable is also
 236  * on purpose; we use long where we can return timeout values and int
 237  * otherwise.
 238  */
 239
 240 #define ___wait_event(wq_head, condition, state, exclusive, ret, cmd)           \
 241 ({                                                                              \
 242         __label__ __out;                                                        \
 243         struct wait_queue_entry __wq_entry;                                     \
 244         long __ret = ret;       /* explicit shadow */                           \
 245                                                                                 \
 246         init_wait_entry(&__wq_entry, exclusive ? WQ_FLAG_EXCLUSIVE : 0);        \
 247         for (;;) {                                                              \
 248                 long __int = prepare_to_wait_event(&wq_head, &__wq_entry, state);\
 249                                                                                 \
 250                 if (condition)                                                  \
 251                         break;                                                  \
 252                                                                                 \
 253                 if (___wait_is_interruptible(state) && __int) {                 \
 254                         __ret = __int;                                          \
 255                         goto __out;                                             \
 256                 }                                                               \
 257                                                                                 \
 258                 cmd;                                                            \
 259         }                                                                       \
 260         finish_wait(&wq_head, &__wq_entry);                                     \
 261 __out:  __ret;                                                                  \
 262 })
 263
 264 #define __wait_event(wq_head, condition)                                        \
 265         (void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 0, 0,     \
 266                             schedule())
 267
 268 /**
 269  * wait_event - sleep until a condition gets true
 270  * @wq_head: the waitqueue to wait on
 271  * @condition: a C expression for the event to wait for
 272  *
 273  * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
 274  * @condition evaluates to true. The @condition is checked each time
 275  * the waitqueue @wq_head is woken up.
 276  *
 277  * wake_up() has to be called after changing any variable that could
 278  * change the result of the wait condition.
 279  */
 280 #define wait_event(wq_head, condition)                                          \
 281 do {                                                                            \
 282         might_sleep();                                                          \
 283         if (condition)                                                          \
 284                 break;                                                          \
 285         __wait_event(wq_head, condition);                                       \
 286 } while (0)
 287
 288 #define __io_wait_event(wq_head, condition)                                     \
 289         (void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 0, 0,     \
 290                             io_schedule())
 291
 292 /*
 293  * io_wait_event() -- like wait_event() but with io_schedule()
 294  */
 295 #define io_wait_event(wq_head, condition)                                       \
 296 do {                                                                            \
 297         might_sleep();                                                          \
 298         if (condition)                                                          \
 299                 break;                                                          \
 300         __io_wait_event(wq_head, condition);                                    \
 301 } while (0)
 302
 303 #define __wait_event_freezable(wq_head, condition)                              \
 304         ___wait_event(wq_head, condition, TASK_INTERRUPTIBLE, 0, 0,             \
 305                             schedule(); try_to_freeze())
 306
 307 /**
 308  * wait_event_freezable - sleep (or freeze) until a condition gets true
 309  * @wq_head: the waitqueue to wait on
 310  * @condition: a C expression for the event to wait for
 311  *
 312  * The process is put to sleep (TASK_INTERRUPTIBLE -- so as not to contribute
 313  * to system load) until the @condition evaluates to true. The
 314  * @condition is checked each time the waitqueue @wq_head is woken up.
 315  *
 316  * wake_up() has to be called after changing any variable that could
 317  * change the result of the wait condition.
 318  */
 319 #define wait_event_freezable(wq_head, condition)                                \
 320 ({                                                                              \
 321         int __ret = 0;                                                          \
 322         might_sleep();                                                          \
 323         if (!(condition))                                                       \
 324                 __ret = __wait_event_freezable(wq_head, condition);             \
 325         __ret;                                                                  \
 326 })
 327
 328 #define __wait_event_timeout(wq_head, condition, timeout)                       \
 329         ___wait_event(wq_head, ___wait_cond_timeout(condition),                 \
 330                       TASK_UNINTERRUPTIBLE, 0, timeout,                         \
 331                       __ret = schedule_timeout(__ret))
 332
 333 /**
 334  * wait_event_timeout - sleep until a condition gets true or a timeout elapses
 335  * @wq_head: the waitqueue to wait on
 336  * @condition: a C expression for the event to wait for
 337  * @timeout: timeout, in jiffies
 338  *
 339  * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
 340  * @condition evaluates to true. The @condition is checked each time
 341  * the waitqueue @wq_head is woken up.
 342  *
 343  * wake_up() has to be called after changing any variable that could
 344  * change the result of the wait condition.
 345  *
 346  * Returns:
 347  * 0 if the @condition evaluated to %false after the @timeout elapsed,
 348  * 1 if the @condition evaluated to %true after the @timeout elapsed,
 349  * or the remaining jiffies (at least 1) if the @condition evaluated
 350  * to %true before the @timeout elapsed.
 351  */
 352 #define wait_event_timeout(wq_head, condition, timeout)                         \
 353 ({                                                                              \
 354         long __ret = timeout;                                                   \
 355         might_sleep();                                                          \
 356         if (!___wait_cond_timeout(condition))                                   \
 357                 __ret = __wait_event_timeout(wq_head, condition, timeout);      \
 358         __ret;                                                                  \
 359 })
 360
 361 #define __wait_event_freezable_timeout(wq_head, condition, timeout)             \
 362         ___wait_event(wq_head, ___wait_cond_timeout(condition),                 \
 363                       TASK_INTERRUPTIBLE, 0, timeout,                           \
 364                       __ret = schedule_timeout(__ret); try_to_freeze())
 365
 366 /*
 367  * like wait_event_timeout() -- except it uses TASK_INTERRUPTIBLE to avoid
 368  * increasing load and is freezable.
 369  */
 370 #define wait_event_freezable_timeout(wq_head, condition, timeout)               \
 371 ({                                                                              \
 372         long __ret = timeout;                                                   \
 373         might_sleep();                                                          \
 374         if (!___wait_cond_timeout(condition))                                   \
 375                 __ret = __wait_event_freezable_timeout(wq_head, condition, timeout); \
 376         __ret;                                                                  \
 377 })
 378
 379 #define __wait_event_exclusive_cmd(wq_head, condition, cmd1, cmd2)              \
 380         (void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 1, 0,     \
 381                             cmd1; schedule(); cmd2)
 382 /*
 383  * Just like wait_event_cmd(), except it sets exclusive flag
 384  */
 385 #define wait_event_exclusive_cmd(wq_head, condition, cmd1, cmd2)                \
 386 do {                                                                            \
 387         if (condition)                                                          \
 388                 break;                                                          \
 389         __wait_event_exclusive_cmd(wq_head, condition, cmd1, cmd2);             \
 390 } while (0)
 391
 392 #define __wait_event_cmd(wq_head, condition, cmd1, cmd2)                        \
 393         (void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 0, 0,     \
 394                             cmd1; schedule(); cmd2)
 395
 396 /**
 397  * wait_event_cmd - sleep until a condition gets true
 398  * @wq_head: the waitqueue to wait on
 399  * @condition: a C expression for the event to wait for
 400  * @cmd1: the command will be executed before sleep
 401  * @cmd2: the command will be executed after sleep
 402  *
 403  * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
 404  * @condition evaluates to true. The @condition is checked each time
 405  * the waitqueue @wq_head is woken up.
 406  *
 407  * wake_up() has to be called after changing any variable that could
 408  * change the result of the wait condition.
 409  */
 410 #define wait_event_cmd(wq_head, condition, cmd1, cmd2)                          \
 411 do {                                                                            \
 412         if (condition)                                                          \
 413                 break;                                                          \
 414         __wait_event_cmd(wq_head, condition, cmd1, cmd2);                       \
 415 } while (0)
 416
 417 #define __wait_event_interruptible(wq_head, condition)                          \
 418         ___wait_event(wq_head, condition, TASK_INTERRUPTIBLE, 0, 0,             \
 419                       schedule())
 420
 421 /**
 422  * wait_event_interruptible - sleep until a condition gets true
 423  * @wq_head: the waitqueue to wait on
 424  * @condition: a C expression for the event to wait for
 425  *
 426  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 427  * @condition evaluates to true or a signal is received.
 428  * The @condition is checked each time the waitqueue @wq_head is woken up.
 429  *
 430  * wake_up() has to be called after changing any variable that could
 431  * change the result of the wait condition.
 432  *
 433  * The function will return -ERESTARTSYS if it was interrupted by a
 434  * signal and 0 if @condition evaluated to true.
 435  */
 436 #define wait_event_interruptible(wq_head, condition)                            \
 437 ({                                                                              \
 438         int __ret = 0;                                                          \
 439         might_sleep();                                                          \
 440         if (!(condition))                                                       \
 441                 __ret = __wait_event_interruptible(wq_head, condition);         \
 442         __ret;                                                                  \
 443 })
 444
 445 #define __wait_event_interruptible_timeout(wq_head, condition, timeout)         \
 446         ___wait_event(wq_head, ___wait_cond_timeout(condition),                 \
 447                       TASK_INTERRUPTIBLE, 0, timeout,                           \
 448                       __ret = schedule_timeout(__ret))
 449
 450 /**
 451  * wait_event_interruptible_timeout - sleep until a condition gets true or a timeout elapses
 452  * @wq_head: the waitqueue to wait on
 453  * @condition: a C expression for the event to wait for
 454  * @timeout: timeout, in jiffies
 455  *
 456  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 457  * @condition evaluates to true or a signal is received.
 458  * The @condition is checked each time the waitqueue @wq_head is woken up.
 459  *
 460  * wake_up() has to be called after changing any variable that could
 461  * change the result of the wait condition.
 462  *
 463  * Returns:
 464  * 0 if the @condition evaluated to %false after the @timeout elapsed,
 465  * 1 if the @condition evaluated to %true after the @timeout elapsed,
 466  * the remaining jiffies (at least 1) if the @condition evaluated
 467  * to %true before the @timeout elapsed, or -%ERESTARTSYS if it was
 468  * interrupted by a signal.
 469  */
 470 #define wait_event_interruptible_timeout(wq_head, condition, timeout)           \
 471 ({                                                                              \
 472         long __ret = timeout;                                                   \
 473         might_sleep();                                                          \
 474         if (!___wait_cond_timeout(condition))                                   \
 475                 __ret = __wait_event_interruptible_timeout(wq_head,             \
 476                                                 condition, timeout);            \
 477         __ret;                                                                  \
 478 })
 479
 480 #define __wait_event_hrtimeout(wq_head, condition, timeout, state)              \
 481 ({                                                                              \
 482         int __ret = 0;                                                          \
 483         struct hrtimer_sleeper __t;                                             \
 484                                                                                 \
 485         hrtimer_init_on_stack(&__t.timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);   \
 486         hrtimer_init_sleeper(&__t, current);                                    \
 487         if ((timeout) != KTIME_MAX)                                             \
 488                 hrtimer_start_range_ns(&__t.timer, timeout,                     \
 489                                        current->timer_slack_ns,                 \
 490                                        HRTIMER_MODE_REL);                       \
 491                                                                                 \
 492         __ret = ___wait_event(wq_head, condition, state, 0, 0,                  \
 493                 if (!__t.task) {                                                \
 494                         __ret = -ETIME;                                         \
 495                         break;                                                  \
 496                 }                                                               \
 497                 schedule());                                                    \
 498                                                                                 \
 499         hrtimer_cancel(&__t.timer);                                             \
 500         destroy_hrtimer_on_stack(&__t.timer);                                   \
 501         __ret;                                                                  \
 502 })
 503
 504 /**
 505  * wait_event_hrtimeout - sleep until a condition gets true or a timeout elapses
 506  * @wq_head: the waitqueue to wait on
 507  * @condition: a C expression for the event to wait for
 508  * @timeout: timeout, as a ktime_t
 509  *
 510  * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
 511  * @condition evaluates to true or a signal is received.
 512  * The @condition is checked each time the waitqueue @wq_head is woken up.
 513  *
 514  * wake_up() has to be called after changing any variable that could
 515  * change the result of the wait condition.
 516  *
 517  * The function returns 0 if @condition became true, or -ETIME if the timeout
 518  * elapsed.
 519  */
 520 #define wait_event_hrtimeout(wq_head, condition, timeout)                       \
 521 ({                                                                              \
 522         int __ret = 0;                                                          \
 523         might_sleep();                                                          \
 524         if (!(condition))                                                       \
 525                 __ret = __wait_event_hrtimeout(wq_head, condition, timeout,     \
 526                                                TASK_UNINTERRUPTIBLE);           \
 527         __ret;                                                                  \
 528 })
 529
 530 /**
 531  * wait_event_interruptible_hrtimeout - sleep until a condition gets true or a timeout elapses
 532  * @wq: the waitqueue to wait on
 533  * @condition: a C expression for the event to wait for
 534  * @timeout: timeout, as a ktime_t
 535  *
 536  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 537  * @condition evaluates to true or a signal is received.
 538  * The @condition is checked each time the waitqueue @wq is woken up.
 539  *
 540  * wake_up() has to be called after changing any variable that could
 541  * change the result of the wait condition.
 542  *
 543  * The function returns 0 if @condition became true, -ERESTARTSYS if it was
 544  * interrupted by a signal, or -ETIME if the timeout elapsed.
 545  */
 546 #define wait_event_interruptible_hrtimeout(wq, condition, timeout)              \
 547 ({                                                                              \
 548         long __ret = 0;                                                         \
 549         might_sleep();                                                          \
 550         if (!(condition))                                                       \
 551                 __ret = __wait_event_hrtimeout(wq, condition, timeout,          \
 552                                                TASK_INTERRUPTIBLE);             \
 553         __ret;                                                                  \
 554 })
 555
 556 #define __wait_event_interruptible_exclusive(wq, condition)                     \
 557         ___wait_event(wq, condition, TASK_INTERRUPTIBLE, 1, 0,                  \
 558                       schedule())
 559
 560 #define wait_event_interruptible_exclusive(wq, condition)                       \
 561 ({                                                                              \
 562         int __ret = 0;                                                          \
 563         might_sleep();                                                          \
 564         if (!(condition))                                                       \
 565                 __ret = __wait_event_interruptible_exclusive(wq, condition);    \
 566         __ret;                                                                  \
 567 })
 568
 569 #define __wait_event_killable_exclusive(wq, condition)                          \
 570         ___wait_event(wq, condition, TASK_KILLABLE, 1, 0,                       \
 571                       schedule())
 572
 573 #define wait_event_killable_exclusive(wq, condition)                            \
 574 ({                                                                              \
 575         int __ret = 0;                                                          \
 576         might_sleep();                                                          \
 577         if (!(condition))                                                       \
 578                 __ret = __wait_event_killable_exclusive(wq, condition);         \
 579         __ret;                                                                  \
 580 })
 581
 582
 583 #define __wait_event_freezable_exclusive(wq, condition)                         \
 584         ___wait_event(wq, condition, TASK_INTERRUPTIBLE, 1, 0,                  \
 585                         schedule(); try_to_freeze())
 586
 587 #define wait_event_freezable_exclusive(wq, condition)                           \
 588 ({                                                                              \
 589         int __ret = 0;                                                          \
 590         might_sleep();                                                          \
 591         if (!(condition))                                                       \
 592                 __ret = __wait_event_freezable_exclusive(wq, condition);        \
 593         __ret;                                                                  \
 594 })
 595
 596 extern int do_wait_intr(wait_queue_head_t *, wait_queue_entry_t *);
 597 extern int do_wait_intr_irq(wait_queue_head_t *, wait_queue_entry_t *);
 598
 599 #define __wait_event_interruptible_locked(wq, condition, exclusive, fn)         \
 600 ({                                                                              \
 601         int __ret;                                                              \
 602         DEFINE_WAIT(__wait);                                                    \
 603         if (exclusive)                                                          \
 604                 __wait.flags |= WQ_FLAG_EXCLUSIVE;                              \
 605         do {                                                                    \
 606                 __ret = fn(&(wq), &__wait);                                     \
 607                 if (__ret)                                                      \
 608                         break;                                                  \
 609         } while (!(condition));                                                 \
 610         __remove_wait_queue(&(wq), &__wait);                                    \
 611         __set_current_state(TASK_RUNNING);                                      \
 612         __ret;                                                                  \
 613 })
 614
 615
 616 /**
 617  * wait_event_interruptible_locked - sleep until a condition gets true
 618  * @wq: the waitqueue to wait on
 619  * @condition: a C expression for the event to wait for
 620  *
 621  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 622  * @condition evaluates to true or a signal is received.
 623  * The @condition is checked each time the waitqueue @wq is woken up.
 624  *
 625  * It must be called with wq.lock being held.  This spinlock is
 626  * unlocked while sleeping but @condition testing is done while lock
 627  * is held and when this macro exits the lock is held.
 628  *
 629  * The lock is locked/unlocked using spin_lock()/spin_unlock()
 630  * functions which must match the way they are locked/unlocked outside
 631  * of this macro.
 632  *
 633  * wake_up_locked() has to be called after changing any variable that could
 634  * change the result of the wait condition.
 635  *
 636  * The function will return -ERESTARTSYS if it was interrupted by a
 637  * signal and 0 if @condition evaluated to true.
 638  */
 639 #define wait_event_interruptible_locked(wq, condition)                          \
 640         ((condition)                                                            \
 641          ? 0 : __wait_event_interruptible_locked(wq, condition, 0, do_wait_intr))
 642
 643 /**
 644  * wait_event_interruptible_locked_irq - sleep until a condition gets true
 645  * @wq: the waitqueue to wait on
 646  * @condition: a C expression for the event to wait for
 647  *
 648  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 649  * @condition evaluates to true or a signal is received.
 650  * The @condition is checked each time the waitqueue @wq is woken up.
 651  *
 652  * It must be called with wq.lock being held.  This spinlock is
 653  * unlocked while sleeping but @condition testing is done while lock
 654  * is held and when this macro exits the lock is held.
 655  *
 656  * The lock is locked/unlocked using spin_lock_irq()/spin_unlock_irq()
 657  * functions which must match the way they are locked/unlocked outside
 658  * of this macro.
 659  *
 660  * wake_up_locked() has to be called after changing any variable that could
 661  * change the result of the wait condition.
 662  *
 663  * The function will return -ERESTARTSYS if it was interrupted by a
 664  * signal and 0 if @condition evaluated to true.
 665  */
 666 #define wait_event_interruptible_locked_irq(wq, condition)                      \
 667         ((condition)                                                            \
 668          ? 0 : __wait_event_interruptible_locked(wq, condition, 0, do_wait_intr_irq))
 669
 670 /**
 671  * wait_event_interruptible_exclusive_locked - sleep exclusively until a condition gets true
 672  * @wq: the waitqueue to wait on
 673  * @condition: a C expression for the event to wait for
 674  *
 675  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 676  * @condition evaluates to true or a signal is received.
 677  * The @condition is checked each time the waitqueue @wq is woken up.
 678  *
 679  * It must be called with wq.lock being held.  This spinlock is
 680  * unlocked while sleeping but @condition testing is done while lock
 681  * is held and when this macro exits the lock is held.
 682  *
 683  * The lock is locked/unlocked using spin_lock()/spin_unlock()
 684  * functions which must match the way they are locked/unlocked outside
 685  * of this macro.
 686  *
 687  * The process is put on the wait queue with an WQ_FLAG_EXCLUSIVE flag
 688  * set thus when other process waits process on the list if this
 689  * process is awaken further processes are not considered.
 690  *
 691  * wake_up_locked() has to be called after changing any variable that could
 692  * change the result of the wait condition.
 693  *
 694  * The function will return -ERESTARTSYS if it was interrupted by a
 695  * signal and 0 if @condition evaluated to true.
 696  */
 697 #define wait_event_interruptible_exclusive_locked(wq, condition)                \
 698         ((condition)                                                            \
 699          ? 0 : __wait_event_interruptible_locked(wq, condition, 1, do_wait_intr))
 700
 701 /**
 702  * wait_event_interruptible_exclusive_locked_irq - sleep until a condition gets true
 703  * @wq: the waitqueue to wait on
 704  * @condition: a C expression for the event to wait for
 705  *
 706  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 707  * @condition evaluates to true or a signal is received.
 708  * The @condition is checked each time the waitqueue @wq is woken up.
 709  *
 710  * It must be called with wq.lock being held.  This spinlock is
 711  * unlocked while sleeping but @condition testing is done while lock
 712  * is held and when this macro exits the lock is held.
 713  *
 714  * The lock is locked/unlocked using spin_lock_irq()/spin_unlock_irq()
 715  * functions which must match the way they are locked/unlocked outside
 716  * of this macro.
 717  *
 718  * The process is put on the wait queue with an WQ_FLAG_EXCLUSIVE flag
 719  * set thus when other process waits process on the list if this
 720  * process is awaken further processes are not considered.
 721  *
 722  * wake_up_locked() has to be called after changing any variable that could
 723  * change the result of the wait condition.
 724  *
 725  * The function will return -ERESTARTSYS if it was interrupted by a
 726  * signal and 0 if @condition evaluated to true.
 727  */
 728 #define wait_event_interruptible_exclusive_locked_irq(wq, condition)            \
 729         ((condition)                                                            \
 730          ? 0 : __wait_event_interruptible_locked(wq, condition, 1, do_wait_intr_irq))
 731
 732
 733 #define __wait_event_killable(wq, condition)                                    \
 734         ___wait_event(wq, condition, TASK_KILLABLE, 0, 0, schedule())
 735
 736 /**
 737  * wait_event_killable - sleep until a condition gets true
 738  * @wq_head: the waitqueue to wait on
 739  * @condition: a C expression for the event to wait for
 740  *
 741  * The process is put to sleep (TASK_KILLABLE) until the
 742  * @condition evaluates to true or a signal is received.
 743  * The @condition is checked each time the waitqueue @wq_head is woken up.
 744  *
 745  * wake_up() has to be called after changing any variable that could
 746  * change the result of the wait condition.
 747  *
 748  * The function will return -ERESTARTSYS if it was interrupted by a
 749  * signal and 0 if @condition evaluated to true.
 750  */
 751 #define wait_event_killable(wq_head, condition)                                 \
 752 ({                                                                              \
 753         int __ret = 0;                                                          \
 754         might_sleep();                                                          \
 755         if (!(condition))                                                       \
 756                 __ret = __wait_event_killable(wq_head, condition);              \
 757         __ret;                                                                  \
 758 })
 759
 760
 761 #define __wait_event_lock_irq(wq_head, condition, lock, cmd)                    \
 762         (void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 0, 0,     \
 763                             spin_unlock_irq(&lock);                             \
 764                             cmd;                                                \
 765                             schedule();                                         \
 766                             spin_lock_irq(&lock))
 767
 768 /**
 769  * wait_event_lock_irq_cmd - sleep until a condition gets true. The
 770  *                           condition is checked under the lock. This
 771  *                           is expected to be called with the lock
 772  *                           taken.
 773  * @wq_head: the waitqueue to wait on
 774  * @condition: a C expression for the event to wait for
 775  * @lock: a locked spinlock_t, which will be released before cmd
 776  *        and schedule() and reacquired afterwards.
 777  * @cmd: a command which is invoked outside the critical section before
 778  *       sleep
 779  *
 780  * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
 781  * @condition evaluates to true. The @condition is checked each time
 782  * the waitqueue @wq_head is woken up.
 783  *
 784  * wake_up() has to be called after changing any variable that could
 785  * change the result of the wait condition.
 786  *
 787  * This is supposed to be called while holding the lock. The lock is
 788  * dropped before invoking the cmd and going to sleep and is reacquired
 789  * afterwards.
 790  */
 791 #define wait_event_lock_irq_cmd(wq_head, condition, lock, cmd)                  \
 792 do {                                                                            \
 793         if (condition)                                                          \
 794                 break;                                                          \
 795         __wait_event_lock_irq(wq_head, condition, lock, cmd);                   \
 796 } while (0)
 797
 798 /**
 799  * wait_event_lock_irq - sleep until a condition gets true. The
 800  *                       condition is checked under the lock. This
 801  *                       is expected to be called with the lock
 802  *                       taken.
 803  * @wq_head: the waitqueue to wait on
 804  * @condition: a C expression for the event to wait for
 805  * @lock: a locked spinlock_t, which will be released before schedule()
 806  *        and reacquired afterwards.
 807  *
 808  * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
 809  * @condition evaluates to true. The @condition is checked each time
 810  * the waitqueue @wq_head is woken up.
 811  *
 812  * wake_up() has to be called after changing any variable that could
 813  * change the result of the wait condition.
 814  *
 815  * This is supposed to be called while holding the lock. The lock is
 816  * dropped before going to sleep and is reacquired afterwards.
 817  */
 818 #define wait_event_lock_irq(wq_head, condition, lock)                           \
 819 do {                                                                            \
 820         if (condition)                                                          \
 821                 break;                                                          \
 822         __wait_event_lock_irq(wq_head, condition, lock, );                      \
 823 } while (0)
 824
 825
 826 #define __wait_event_interruptible_lock_irq(wq_head, condition, lock, cmd)      \
 827         ___wait_event(wq_head, condition, TASK_INTERRUPTIBLE, 0, 0,             \
 828                       spin_unlock_irq(&lock);                                   \
 829                       cmd;                                                      \
 830                       schedule();                                               \
 831                       spin_lock_irq(&lock))
 832
 833 /**
 834  * wait_event_interruptible_lock_irq_cmd - sleep until a condition gets true.
 835  *              The condition is checked under the lock. This is expected to
 836  *              be called with the lock taken.
 837  * @wq_head: the waitqueue to wait on
 838  * @condition: a C expression for the event to wait for
 839  * @lock: a locked spinlock_t, which will be released before cmd and
 840  *        schedule() and reacquired afterwards.
 841  * @cmd: a command which is invoked outside the critical section before
 842  *       sleep
 843  *
 844  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 845  * @condition evaluates to true or a signal is received. The @condition is
 846  * checked each time the waitqueue @wq_head is woken up.
 847  *
 848  * wake_up() has to be called after changing any variable that could
 849  * change the result of the wait condition.
 850  *
 851  * This is supposed to be called while holding the lock. The lock is
 852  * dropped before invoking the cmd and going to sleep and is reacquired
 853  * afterwards.
 854  *
 855  * The macro will return -ERESTARTSYS if it was interrupted by a signal
 856  * and 0 if @condition evaluated to true.
 857  */
 858 #define wait_event_interruptible_lock_irq_cmd(wq_head, condition, lock, cmd)    \
 859 ({                                                                              \
 860         int __ret = 0;                                                          \
 861         if (!(condition))                                                       \
 862                 __ret = __wait_event_interruptible_lock_irq(wq_head,            \
 863                                                 condition, lock, cmd);          \
 864         __ret;                                                                  \
 865 })
 866
 867 /**
 868  * wait_event_interruptible_lock_irq - sleep until a condition gets true.
 869  *              The condition is checked under the lock. This is expected
 870  *              to be called with the lock taken.
 871  * @wq_head: the waitqueue to wait on
 872  * @condition: a C expression for the event to wait for
 873  * @lock: a locked spinlock_t, which will be released before schedule()
 874  *        and reacquired afterwards.
 875  *
 876  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 877  * @condition evaluates to true or signal is received. The @condition is
 878  * checked each time the waitqueue @wq_head is woken up.
 879  *
 880  * wake_up() has to be called after changing any variable that could
 881  * change the result of the wait condition.
 882  *
 883  * This is supposed to be called while holding the lock. The lock is
 884  * dropped before going to sleep and is reacquired afterwards.
 885  *
 886  * The macro will return -ERESTARTSYS if it was interrupted by a signal
 887  * and 0 if @condition evaluated to true.
 888  */
 889 #define wait_event_interruptible_lock_irq(wq_head, condition, lock)             \
 890 ({                                                                              \
 891         int __ret = 0;                                                          \
 892         if (!(condition))                                                       \
 893                 __ret = __wait_event_interruptible_lock_irq(wq_head,            \
 894                                                 condition, lock,);              \
 895         __ret;                                                                  \
 896 })
 897
 898 #define __wait_event_interruptible_lock_irq_timeout(wq_head, condition,         \
 899                                                     lock, timeout)              \
 900         ___wait_event(wq_head, ___wait_cond_timeout(condition),                 \
 901                       TASK_INTERRUPTIBLE, 0, timeout,                           \
 902                       spin_unlock_irq(&lock);                                   \
 903                       __ret = schedule_timeout(__ret);                          \
 904                       spin_lock_irq(&lock));
 905
 906 /**
 907  * wait_event_interruptible_lock_irq_timeout - sleep until a condition gets
 908  *              true or a timeout elapses. The condition is checked under
 909  *              the lock. This is expected to be called with the lock taken.
 910  * @wq_head: the waitqueue to wait on
 911  * @condition: a C expression for the event to wait for
 912  * @lock: a locked spinlock_t, which will be released before schedule()
 913  *        and reacquired afterwards.
 914  * @timeout: timeout, in jiffies
 915  *
 916  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 917  * @condition evaluates to true or signal is received. The @condition is
 918  * checked each time the waitqueue @wq_head is woken up.
 919  *
 920  * wake_up() has to be called after changing any variable that could
 921  * change the result of the wait condition.
 922  *
 923  * This is supposed to be called while holding the lock. The lock is
 924  * dropped before going to sleep and is reacquired afterwards.
 925  *
 926  * The function returns 0 if the @timeout elapsed, -ERESTARTSYS if it
 927  * was interrupted by a signal, and the remaining jiffies otherwise
 928  * if the condition evaluated to true before the timeout elapsed.
 929  */
 930 #define wait_event_interruptible_lock_irq_timeout(wq_head, condition, lock,     \
 931                                                   timeout)                      \
 932 ({                                                                              \
 933         long __ret = timeout;                                                   \
 934         if (!___wait_cond_timeout(condition))                                   \
 935                 __ret = __wait_event_interruptible_lock_irq_timeout(            \
 936                                         wq_head, condition, lock, timeout);     \
 937         __ret;                                                                  \
 938 })
 939
 940 /*
 941  * Waitqueues which are removed from the waitqueue_head at wakeup time
 942  */
 943 void prepare_to_wait(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state);
 944 void prepare_to_wait_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state);
 945 long prepare_to_wait_event(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state);
 946 void finish_wait(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);
 947 long wait_woken(struct wait_queue_entry *wq_entry, unsigned mode, long timeout);
 948 int woken_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *key);
 949 int autoremove_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *key);
 950
 951 #define DEFINE_WAIT_FUNC(name, function)                                        \
 952         struct wait_queue_entry name = {                                        \
 953                 .private        = current,                                      \
 954                 .func           = function,                                     \
 955                 .entry          = LIST_HEAD_INIT((name).entry),                 \
 956         }
 957
 958 #define DEFINE_WAIT(name) DEFINE_WAIT_FUNC(name, autoremove_wake_function)
 959
 960 #define init_wait(wait)                                                         \
 961         do {                                                                    \
 962                 (wait)->private = current;                                      \
 963                 (wait)->func = autoremove_wake_function;                        \
 964                 INIT_LIST_HEAD(&(wait)->entry);                                 \
 965                 (wait)->flags = 0;                                              \
 966         } while (0)
 967
 968 #endif /* _LINUX_WAIT_H */