Merge branch 'sched-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel...

[karo-tx-linux.git] / kernel / sched / sched.h

diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 7ef596837dac2d1126c790b06f8fd35e69409756..84d48790bb6d0607e5dd67dd449f0353c51b0136 100644 (file)
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -624,9 +624,10 @@ struct rq {
        unsigned long cpu_capacity;
        unsigned long cpu_capacity_orig;
 
+       struct callback_head *balance_callback;
+
        unsigned char idle_balance;
        /* For active balancing */
-       int post_schedule;
        int active_balance;
        int push_cpu;
        struct cpu_stop_work active_balance_work;
@@ -767,6 +768,21 @@ extern int migrate_swap(struct task_struct *, struct task_struct *);
 
 #ifdef CONFIG_SMP
 
+static inline void
+queue_balance_callback(struct rq *rq,
+                      struct callback_head *head,
+                      void (*func)(struct rq *rq))
+{
+       lockdep_assert_held(&rq->lock);
+
+       if (unlikely(head->next))
+               return;
+
+       head->func = (void (*)(struct callback_head *))func;
+       head->next = rq->balance_callback;
+       rq->balance_callback = head;
+}
+
 extern void sched_ttwu_pending(void);
 
 #define rcu_dereference_check_sched_domain(p) \
@@ -1192,7 +1208,6 @@ struct sched_class {
        int  (*select_task_rq)(struct task_struct *p, int task_cpu, int sd_flag, int flags);
        void (*migrate_task_rq)(struct task_struct *p, int next_cpu);
 
-       void (*post_schedule) (struct rq *this_rq);
        void (*task_waking) (struct task_struct *task);
        void (*task_woken) (struct rq *this_rq, struct task_struct *task);
 
@@ -1423,8 +1438,10 @@ static inline struct rq *__task_rq_lock(struct task_struct *p)
        for (;;) {
                rq = task_rq(p);
                raw_spin_lock(&rq->lock);
-               if (likely(rq == task_rq(p) && !task_on_rq_migrating(p)))
+               if (likely(rq == task_rq(p) && !task_on_rq_migrating(p))) {
+                       lockdep_pin_lock(&rq->lock);
                        return rq;
+               }
                raw_spin_unlock(&rq->lock);
 
                while (unlikely(task_on_rq_migrating(p)))
@@ -1461,8 +1478,10 @@ static inline struct rq *task_rq_lock(struct task_struct *p, unsigned long *flag
                 * If we observe the new cpu in task_rq_lock, the acquire will
                 * pair with the WMB to ensure we must then also see migrating.
                 */
-               if (likely(rq == task_rq(p) && !task_on_rq_migrating(p)))
+               if (likely(rq == task_rq(p) && !task_on_rq_migrating(p))) {
+                       lockdep_pin_lock(&rq->lock);
                        return rq;
+               }
                raw_spin_unlock(&rq->lock);
                raw_spin_unlock_irqrestore(&p->pi_lock, *flags);
 
@@ -1474,6 +1493,7 @@ static inline struct rq *task_rq_lock(struct task_struct *p, unsigned long *flag
 static inline void __task_rq_unlock(struct rq *rq)
        __releases(rq->lock)
 {
+       lockdep_unpin_lock(&rq->lock);
        raw_spin_unlock(&rq->lock);
 }
 
@@ -1482,6 +1502,7 @@ task_rq_unlock(struct rq *rq, struct task_struct *p, unsigned long *flags)
        __releases(rq->lock)
        __releases(p->pi_lock)
 {
+       lockdep_unpin_lock(&rq->lock);
        raw_spin_unlock(&rq->lock);
        raw_spin_unlock_irqrestore(&p->pi_lock, *flags);
 }