sched/idle: Move latency tracing stop/start calls deeper inside the idle loop

[karo-tx-linux.git] / drivers / cpuidle / cpuidle.c

diff --git a/drivers/cpuidle/cpuidle.c b/drivers/cpuidle/cpuidle.c
index 61c417b9e53f8795175b29d71dfd201f15be151c..a5d9f2e470ea1fc9f94536f230d60c0633933dda 100644 (file)
--- a/drivers/cpuidle/cpuidle.c
+++ b/drivers/cpuidle/cpuidle.c
@@ -65,7 +65,7 @@ int cpuidle_play_dead(void)
                return -ENODEV;
 
        /* Find lowest-power state that supports long-term idle */
-       for (i = drv->state_count - 1; i >= CPUIDLE_DRIVER_STATE_START; i--)
+       for (i = drv->state_count - 1; i >= 0; i--)
                if (drv->states[i].enter_dead)
                        return drv->states[i].enter_dead(dev, i);
 
@@ -73,16 +73,21 @@ int cpuidle_play_dead(void)
 }
 
 static int find_deepest_state(struct cpuidle_driver *drv,
-                             struct cpuidle_device *dev, bool freeze)
+                             struct cpuidle_device *dev,
+                             unsigned int max_latency,
+                             unsigned int forbidden_flags,
+                             bool freeze)
 {
        unsigned int latency_req = 0;
-       int i, ret = freeze ? -1 : CPUIDLE_DRIVER_STATE_START - 1;
+       int i, ret = -ENXIO;
 
-       for (i = CPUIDLE_DRIVER_STATE_START; i < drv->state_count; i++) {
+       for (i = 0; i < drv->state_count; i++) {
                struct cpuidle_state *s = &drv->states[i];
                struct cpuidle_state_usage *su = &dev->states_usage[i];
 
                if (s->disabled || su->disable || s->exit_latency <= latency_req
+                   || s->exit_latency > max_latency
+                   || (s->flags & forbidden_flags)
                    || (freeze && !s->enter_freeze))
                        continue;
 
@@ -92,6 +97,7 @@ static int find_deepest_state(struct cpuidle_driver *drv,
        return ret;
 }
 
+#ifdef CONFIG_SUSPEND
 /**
  * cpuidle_find_deepest_state - Find the deepest available idle state.
  * @drv: cpuidle driver for the given CPU.
@@ -100,7 +106,7 @@ static int find_deepest_state(struct cpuidle_driver *drv,
 int cpuidle_find_deepest_state(struct cpuidle_driver *drv,
                               struct cpuidle_device *dev)
 {
-       return find_deepest_state(drv, dev, false);
+       return find_deepest_state(drv, dev, UINT_MAX, 0, false);
 }
 
 static void enter_freeze_proper(struct cpuidle_driver *drv,
@@ -112,6 +118,7 @@ static void enter_freeze_proper(struct cpuidle_driver *drv,
         * cpuidle mechanism enables interrupts and doing that with timekeeping
         * suspended is generally unsafe.
         */
+       stop_critical_timings();
        drv->states[index].enter_freeze(dev, drv, index);
        WARN_ON(!irqs_disabled());
        /*
@@ -120,6 +127,7 @@ static void enter_freeze_proper(struct cpuidle_driver *drv,
         * critical sections, so tell RCU about that.
         */
        RCU_NONIDLE(tick_unfreeze());
+       start_critical_timings();
 }
 
 /**
@@ -139,18 +147,19 @@ int cpuidle_enter_freeze(struct cpuidle_driver *drv, struct cpuidle_device *dev)
         * that interrupts won't be enabled when it exits and allows the tick to
         * be frozen safely.
         */
-       index = find_deepest_state(drv, dev, true);
+       index = find_deepest_state(drv, dev, UINT_MAX, 0, true);
        if (index >= 0)
                enter_freeze_proper(drv, dev, index);
 
        return index;
 }
+#endif /* CONFIG_SUSPEND */
 
 /**
  * cpuidle_enter_state - enter the state and update stats
  * @dev: cpuidle device for this cpu
  * @drv: cpuidle driver for this cpu
- * @next_state: index into drv->states of the state to enter
+ * @index: index into the states table in @drv of the state to enter
  */
 int cpuidle_enter_state(struct cpuidle_device *dev, struct cpuidle_driver *drv,
                        int index)
@@ -167,17 +176,32 @@ int cpuidle_enter_state(struct cpuidle_device *dev, struct cpuidle_driver *drv,
         * local timer will be shut down.  If a local timer is used from another
         * CPU as a broadcast timer, this call may fail if it is not available.
         */
-       if (broadcast && tick_broadcast_enter())
-               return -EBUSY;
+       if (broadcast && tick_broadcast_enter()) {
+               index = find_deepest_state(drv, dev, target_state->exit_latency,
+                                          CPUIDLE_FLAG_TIMER_STOP, false);
+               if (index < 0) {
+                       default_idle_call();
+                       return -EBUSY;
+               }
+               target_state = &drv->states[index];
+       }
+
+       /* Take note of the planned idle state. */
+       sched_idle_set_state(target_state);
 
        trace_cpu_idle_rcuidle(index, dev->cpu);
        time_start = ktime_get();
 
+       stop_critical_timings();
        entered_state = target_state->enter(dev, drv, index);
+       start_critical_timings();
 
        time_end = ktime_get();
        trace_cpu_idle_rcuidle(PWR_EVENT_EXIT, dev->cpu);
 
+       /* The cpu is no longer idle or about to enter idle. */
+       sched_idle_set_state(NULL);
+
        if (broadcast) {
                if (WARN_ON_ONCE(!irqs_disabled()))
                        local_irq_disable();
@@ -249,7 +273,7 @@ int cpuidle_enter(struct cpuidle_driver *drv, struct cpuidle_device *dev,
  */
 void cpuidle_reflect(struct cpuidle_device *dev, int index)
 {
-       if (cpuidle_curr_governor->reflect)
+       if (cpuidle_curr_governor->reflect && index >= 0)
                cpuidle_curr_governor->reflect(dev, index);
 }