futex: separate futex_wait_queue_me() logic from futex_wait()

Refactor futex_wait() in preparation for futex_wait_requeue_pi(). In
order to reuse a good chunk of the futex_wait() code for the upcoming
futex_wait_requeue_pi() function, this patch breaks out the
queue-to-wakeup section of futex_wait() into futex_wait_queue_me().

Signed-off-by: Darren Hart <dvhltc@us.ibm.com>
Reviewed-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

diff --git a/kernel/futex.c b/kernel/futex.c
index 6b50a024bca22e32b0a606fb4b7ea6daf1525967..ebb48d6d1a87e1bd2893a84c6f25f53984e7c08e 100644 (file)
--- a/kernel/futex.c
+++ b/kernel/futex.c
@@ -1115,24 +1115,87 @@ handle_fault:
 
 static long futex_wait_restart(struct restart_block *restart);
 
+/**
+ * futex_wait_queue_me() - queue_me() and wait for wakeup, timeout, or signal
+ * @hb:                the futex hash bucket, must be locked by the caller
+ * @q:         the futex_q to queue up on
+ * @timeout:   the prepared hrtimer_sleeper, or null for no timeout
+ * @wait:      the wait_queue to add to the futex_q after queueing in the hb
+ */
+static void futex_wait_queue_me(struct futex_hash_bucket *hb, struct futex_q *q,
+                               struct hrtimer_sleeper *timeout,
+                               wait_queue_t *wait)
+{
+       queue_me(q, hb);
+
+       /*
+        * There might have been scheduling since the queue_me(), as we
+        * cannot hold a spinlock across the get_user() in case it
+        * faults, and we cannot just set TASK_INTERRUPTIBLE state when
+        * queueing ourselves into the futex hash.  This code thus has to
+        * rely on the futex_wake() code removing us from hash when it
+        * wakes us up.
+        */
+
+       /* add_wait_queue is the barrier after __set_current_state. */
+       __set_current_state(TASK_INTERRUPTIBLE);
+
+       /*
+        * Add current as the futex_q waiter.  We don't remove ourselves from
+        * the wait_queue because we are the only user of it.
+        */
+       add_wait_queue(&q->waiter, wait);
+
+       /* Arm the timer */
+       if (timeout) {
+               hrtimer_start_expires(&timeout->timer, HRTIMER_MODE_ABS);
+               if (!hrtimer_active(&timeout->timer))
+                       timeout->task = NULL;
+       }
+
+       /*
+        * !plist_node_empty() is safe here without any lock.
+        * q.lock_ptr != 0 is not safe, because of ordering against wakeup.
+        */
+       if (likely(!plist_node_empty(&q->list))) {
+               /*
+                * If the timer has already expired, current will already be
+                * flagged for rescheduling. Only call schedule if there
+                * is no timeout, or if it has yet to expire.
+                */
+               if (!timeout || timeout->task)
+                       schedule();
+       }
+       __set_current_state(TASK_RUNNING);
+}
+
 static int futex_wait(u32 __user *uaddr, int fshared,
                      u32 val, ktime_t *abs_time, u32 bitset, int clockrt)
 {
-       struct task_struct *curr = current;
+       struct hrtimer_sleeper timeout, *to = NULL;
+       DECLARE_WAITQUEUE(wait, current);
        struct restart_block *restart;
-       DECLARE_WAITQUEUE(wait, curr);
        struct futex_hash_bucket *hb;
        struct futex_q q;
        u32 uval;
        int ret;
-       struct hrtimer_sleeper t;
-       int rem = 0;
 
        if (!bitset)
                return -EINVAL;
 
        q.pi_state = NULL;
        q.bitset = bitset;
+
+       if (abs_time) {
+               to = &timeout;
+
+               hrtimer_init_on_stack(&to->timer, clockrt ? CLOCK_REALTIME :
+                                     CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
+               hrtimer_init_sleeper(to, current);
+               hrtimer_set_expires_range_ns(&to->timer, *abs_time,
+                                            current->timer_slack_ns);
+       }
+
 retry:
        q.key = FUTEX_KEY_INIT;
        ret = get_futex_key(uaddr, fshared, &q.key);
@@ -1178,75 +1241,22 @@ retry_private:
                goto retry;
        }
        ret = -EWOULDBLOCK;
+
+       /* Only actually queue if *uaddr contained val.  */
        if (unlikely(uval != val)) {
                queue_unlock(&q, hb);
                goto out_put_key;
        }
 
-       /* Only actually queue if *uaddr contained val.  */
-       queue_me(&q, hb);
-
-       /*
-        * There might have been scheduling since the queue_me(), as we
-        * cannot hold a spinlock across the get_user() in case it
-        * faults, and we cannot just set TASK_INTERRUPTIBLE state when
-        * queueing ourselves into the futex hash.  This code thus has to
-        * rely on the futex_wake() code removing us from hash when it
-        * wakes us up.
-        */
-
-       /* add_wait_queue is the barrier after __set_current_state. */
-       __set_current_state(TASK_INTERRUPTIBLE);
-       add_wait_queue(&q.waiter, &wait);
-       /*
-        * !plist_node_empty() is safe here without any lock.
-        * q.lock_ptr != 0 is not safe, because of ordering against wakeup.
-        */
-       if (likely(!plist_node_empty(&q.list))) {
-               if (!abs_time)
-                       schedule();
-               else {
-                       hrtimer_init_on_stack(&t.timer,
-                                             clockrt ? CLOCK_REALTIME :
-                                             CLOCK_MONOTONIC,
-                                             HRTIMER_MODE_ABS);
-                       hrtimer_init_sleeper(&t, current);
-                       hrtimer_set_expires_range_ns(&t.timer, *abs_time,
-                                                    current->timer_slack_ns);
-
-                       hrtimer_start_expires(&t.timer, HRTIMER_MODE_ABS);
-                       if (!hrtimer_active(&t.timer))
-                               t.task = NULL;
-
-                       /*
-                        * the timer could have already expired, in which
-                        * case current would be flagged for rescheduling.
-                        * Don't bother calling schedule.
-                        */
-                       if (likely(t.task))
-                               schedule();
-
-                       hrtimer_cancel(&t.timer);
-
-                       /* Flag if a timeout occured */
-                       rem = (t.task == NULL);
-
-                       destroy_hrtimer_on_stack(&t.timer);
-               }
-       }
-       __set_current_state(TASK_RUNNING);
-
-       /*
-        * NOTE: we don't remove ourselves from the waitqueue because
-        * we are the only user of it.
-        */
+       /* queue_me and wait for wakeup, timeout, or a signal. */
+       futex_wait_queue_me(hb, &q, to, &wait);
 
        /* If we were woken (and unqueued), we succeeded, whatever. */
        ret = 0;
        if (!unqueue_me(&q))
                goto out_put_key;
        ret = -ETIMEDOUT;
-       if (rem)
+       if (to && !to->task)
                goto out_put_key;
 
        /*
@@ -1275,6 +1285,10 @@ retry_private:
 out_put_key:
        put_futex_key(fshared, &q.key);
 out:
+       if (to) {
+               hrtimer_cancel(&to->timer);
+               destroy_hrtimer_on_stack(&to->timer);
+       }
        return ret;
 }