drivers/cpufreq/cpufreq.c

   1 /*
   2  *  linux/drivers/cpufreq/cpufreq.c
   3  *
   4  *  Copyright (C) 2001 Russell King
   5  *            (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
   6  *            (C) 2013 Viresh Kumar <viresh.kumar@linaro.org>
   7  *
   8  *  Oct 2005 - Ashok Raj <ashok.raj@intel.com>
   9  *      Added handling for CPU hotplug
  10  *  Feb 2006 - Jacob Shin <jacob.shin@amd.com>
  11  *      Fix handling for CPU hotplug -- affected CPUs
  12  *
  13  * This program is free software; you can redistribute it and/or modify
  14  * it under the terms of the GNU General Public License version 2 as
  15  * published by the Free Software Foundation.
  16  */
  17
  18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  19
  20 #include <linux/cpu.h>
  21 #include <linux/cpufreq.h>
  22 #include <linux/delay.h>
  23 #include <linux/device.h>
  24 #include <linux/init.h>
  25 #include <linux/kernel_stat.h>
  26 #include <linux/module.h>
  27 #include <linux/mutex.h>
  28 #include <linux/slab.h>
  29 #include <linux/suspend.h>
  30 #include <linux/syscore_ops.h>
  31 #include <linux/tick.h>
  32 #include <trace/events/power.h>
  33
  34 static LIST_HEAD(cpufreq_policy_list);
  35
  36 static inline bool policy_is_inactive(struct cpufreq_policy *policy)
  37 {
  38         return cpumask_empty(policy->cpus);
  39 }
  40
  41 static bool suitable_policy(struct cpufreq_policy *policy, bool active)
  42 {
  43         return active == !policy_is_inactive(policy);
  44 }
  45
  46 /* Finds Next Acive/Inactive policy */
  47 static struct cpufreq_policy *next_policy(struct cpufreq_policy *policy,
  48                                           bool active)
  49 {
  50         do {
  51                 policy = list_next_entry(policy, policy_list);
  52
  53                 /* No more policies in the list */
  54                 if (&policy->policy_list == &cpufreq_policy_list)
  55                         return NULL;
  56         } while (!suitable_policy(policy, active));
  57
  58         return policy;
  59 }
  60
  61 static struct cpufreq_policy *first_policy(bool active)
  62 {
  63         struct cpufreq_policy *policy;
  64
  65         /* No policies in the list */
  66         if (list_empty(&cpufreq_policy_list))
  67                 return NULL;
  68
  69         policy = list_first_entry(&cpufreq_policy_list, typeof(*policy),
  70                                   policy_list);
  71
  72         if (!suitable_policy(policy, active))
  73                 policy = next_policy(policy, active);
  74
  75         return policy;
  76 }
  77
  78 /* Macros to iterate over CPU policies */
  79 #define for_each_suitable_policy(__policy, __active)    \
  80         for (__policy = first_policy(__active);         \
  81              __policy;                                  \
  82              __policy = next_policy(__policy, __active))
  83
  84 #define for_each_active_policy(__policy)                \
  85         for_each_suitable_policy(__policy, true)
  86 #define for_each_inactive_policy(__policy)              \
  87         for_each_suitable_policy(__policy, false)
  88
  89 #define for_each_policy(__policy)                       \
  90         list_for_each_entry(__policy, &cpufreq_policy_list, policy_list)
  91
  92 /* Iterate over governors */
  93 static LIST_HEAD(cpufreq_governor_list);
  94 #define for_each_governor(__governor)                           \
  95         list_for_each_entry(__governor, &cpufreq_governor_list, governor_list)
  96
  97 /**
  98  * The "cpufreq driver" - the arch- or hardware-dependent low
  99  * level driver of CPUFreq support, and its spinlock. This lock
 100  * also protects the cpufreq_cpu_data array.
 101  */
 102 static struct cpufreq_driver *cpufreq_driver;
 103 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data);
 104 static DEFINE_RWLOCK(cpufreq_driver_lock);
 105 DEFINE_MUTEX(cpufreq_governor_lock);
 106
 107 /* Flag to suspend/resume CPUFreq governors */
 108 static bool cpufreq_suspended;
 109
 110 static inline bool has_target(void)
 111 {
 112         return cpufreq_driver->target_index || cpufreq_driver->target;
 113 }
 114
 115 /* internal prototypes */
 116 static int __cpufreq_governor(struct cpufreq_policy *policy,
 117                 unsigned int event);
 118 static unsigned int __cpufreq_get(struct cpufreq_policy *policy);
 119 static void handle_update(struct work_struct *work);
 120
 121 /**
 122  * Two notifier lists: the "policy" list is involved in the
 123  * validation process for a new CPU frequency policy; the
 124  * "transition" list for kernel code that needs to handle
 125  * changes to devices when the CPU clock speed changes.
 126  * The mutex locks both lists.
 127  */
 128 static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
 129 static struct srcu_notifier_head cpufreq_transition_notifier_list;
 130
 131 static bool init_cpufreq_transition_notifier_list_called;
 132 static int __init init_cpufreq_transition_notifier_list(void)
 133 {
 134         srcu_init_notifier_head(&cpufreq_transition_notifier_list);
 135         init_cpufreq_transition_notifier_list_called = true;
 136         return 0;
 137 }
 138 pure_initcall(init_cpufreq_transition_notifier_list);
 139
 140 static int off __read_mostly;
 141 static int cpufreq_disabled(void)
 142 {
 143         return off;
 144 }
 145 void disable_cpufreq(void)
 146 {
 147         off = 1;
 148 }
 149 static DEFINE_MUTEX(cpufreq_governor_mutex);
 150
 151 bool have_governor_per_policy(void)
 152 {
 153         return !!(cpufreq_driver->flags & CPUFREQ_HAVE_GOVERNOR_PER_POLICY);
 154 }
 155 EXPORT_SYMBOL_GPL(have_governor_per_policy);
 156
 157 struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy)
 158 {
 159         if (have_governor_per_policy())
 160                 return &policy->kobj;
 161         else
 162                 return cpufreq_global_kobject;
 163 }
 164 EXPORT_SYMBOL_GPL(get_governor_parent_kobj);
 165
 166 struct cpufreq_frequency_table *cpufreq_frequency_get_table(unsigned int cpu)
 167 {
 168         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
 169
 170         return policy && !policy_is_inactive(policy) ?
 171                 policy->freq_table : NULL;
 172 }
 173 EXPORT_SYMBOL_GPL(cpufreq_frequency_get_table);
 174
 175 static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall)
 176 {
 177         u64 idle_time;
 178         u64 cur_wall_time;
 179         u64 busy_time;
 180
 181         cur_wall_time = jiffies64_to_cputime64(get_jiffies_64());
 182
 183         busy_time = kcpustat_cpu(cpu).cpustat[CPUTIME_USER];
 184         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SYSTEM];
 185         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_IRQ];
 186         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SOFTIRQ];
 187         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_STEAL];
 188         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_NICE];
 189
 190         idle_time = cur_wall_time - busy_time;
 191         if (wall)
 192                 *wall = cputime_to_usecs(cur_wall_time);
 193
 194         return cputime_to_usecs(idle_time);
 195 }
 196
 197 u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy)
 198 {
 199         u64 idle_time = get_cpu_idle_time_us(cpu, io_busy ? wall : NULL);
 200
 201         if (idle_time == -1ULL)
 202                 return get_cpu_idle_time_jiffy(cpu, wall);
 203         else if (!io_busy)
 204                 idle_time += get_cpu_iowait_time_us(cpu, wall);
 205
 206         return idle_time;
 207 }
 208 EXPORT_SYMBOL_GPL(get_cpu_idle_time);
 209
 210 /*
 211  * This is a generic cpufreq init() routine which can be used by cpufreq
 212  * drivers of SMP systems. It will do following:
 213  * - validate & show freq table passed
 214  * - set policies transition latency
 215  * - policy->cpus with all possible CPUs
 216  */
 217 int cpufreq_generic_init(struct cpufreq_policy *policy,
 218                 struct cpufreq_frequency_table *table,
 219                 unsigned int transition_latency)
 220 {
 221         int ret;
 222
 223         ret = cpufreq_table_validate_and_show(policy, table);
 224         if (ret) {
 225                 pr_err("%s: invalid frequency table: %d\n", __func__, ret);
 226                 return ret;
 227         }
 228
 229         policy->cpuinfo.transition_latency = transition_latency;
 230
 231         /*
 232          * The driver only supports the SMP configuration where all processors
 233          * share the clock and voltage and clock.
 234          */
 235         cpumask_setall(policy->cpus);
 236
 237         return 0;
 238 }
 239 EXPORT_SYMBOL_GPL(cpufreq_generic_init);
 240
 241 struct cpufreq_policy *cpufreq_cpu_get_raw(unsigned int cpu)
 242 {
 243         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
 244
 245         return policy && cpumask_test_cpu(cpu, policy->cpus) ? policy : NULL;
 246 }
 247 EXPORT_SYMBOL_GPL(cpufreq_cpu_get_raw);
 248
 249 unsigned int cpufreq_generic_get(unsigned int cpu)
 250 {
 251         struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu);
 252
 253         if (!policy || IS_ERR(policy->clk)) {
 254                 pr_err("%s: No %s associated to cpu: %d\n",
 255                        __func__, policy ? "clk" : "policy", cpu);
 256                 return 0;
 257         }
 258
 259         return clk_get_rate(policy->clk) / 1000;
 260 }
 261 EXPORT_SYMBOL_GPL(cpufreq_generic_get);
 262
 263 /**
 264  * cpufreq_cpu_get: returns policy for a cpu and marks it busy.
 265  *
 266  * @cpu: cpu to find policy for.
 267  *
 268  * This returns policy for 'cpu', returns NULL if it doesn't exist.
 269  * It also increments the kobject reference count to mark it busy and so would
 270  * require a corresponding call to cpufreq_cpu_put() to decrement it back.
 271  * If corresponding call cpufreq_cpu_put() isn't made, the policy wouldn't be
 272  * freed as that depends on the kobj count.
 273  *
 274  * Return: A valid policy on success, otherwise NULL on failure.
 275  */
 276 struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
 277 {
 278         struct cpufreq_policy *policy = NULL;
 279         unsigned long flags;
 280
 281         if (WARN_ON(cpu >= nr_cpu_ids))
 282                 return NULL;
 283
 284         /* get the cpufreq driver */
 285         read_lock_irqsave(&cpufreq_driver_lock, flags);
 286
 287         if (cpufreq_driver) {
 288                 /* get the CPU */
 289                 policy = cpufreq_cpu_get_raw(cpu);
 290                 if (policy)
 291                         kobject_get(&policy->kobj);
 292         }
 293
 294         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
 295
 296         return policy;
 297 }
 298 EXPORT_SYMBOL_GPL(cpufreq_cpu_get);
 299
 300 /**
 301  * cpufreq_cpu_put: Decrements the usage count of a policy
 302  *
 303  * @policy: policy earlier returned by cpufreq_cpu_get().
 304  *
 305  * This decrements the kobject reference count incremented earlier by calling
 306  * cpufreq_cpu_get().
 307  */
 308 void cpufreq_cpu_put(struct cpufreq_policy *policy)
 309 {
 310         kobject_put(&policy->kobj);
 311 }
 312 EXPORT_SYMBOL_GPL(cpufreq_cpu_put);
 313
 314 /*********************************************************************
 315  *            EXTERNALLY AFFECTING FREQUENCY CHANGES                 *
 316  *********************************************************************/
 317
 318 /**
 319  * adjust_jiffies - adjust the system "loops_per_jiffy"
 320  *
 321  * This function alters the system "loops_per_jiffy" for the clock
 322  * speed change. Note that loops_per_jiffy cannot be updated on SMP
 323  * systems as each CPU might be scaled differently. So, use the arch
 324  * per-CPU loops_per_jiffy value wherever possible.
 325  */
 326 static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 327 {
 328 #ifndef CONFIG_SMP
 329         static unsigned long l_p_j_ref;
 330         static unsigned int l_p_j_ref_freq;
 331
 332         if (ci->flags & CPUFREQ_CONST_LOOPS)
 333                 return;
 334
 335         if (!l_p_j_ref_freq) {
 336                 l_p_j_ref = loops_per_jiffy;
 337                 l_p_j_ref_freq = ci->old;
 338                 pr_debug("saving %lu as reference value for loops_per_jiffy; freq is %u kHz\n",
 339                          l_p_j_ref, l_p_j_ref_freq);
 340         }
 341         if (val == CPUFREQ_POSTCHANGE && ci->old != ci->new) {
 342                 loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq,
 343                                                                 ci->new);
 344                 pr_debug("scaling loops_per_jiffy to %lu for frequency %u kHz\n",
 345                          loops_per_jiffy, ci->new);
 346         }
 347 #endif
 348 }
 349
 350 static void __cpufreq_notify_transition(struct cpufreq_policy *policy,
 351                 struct cpufreq_freqs *freqs, unsigned int state)
 352 {
 353         BUG_ON(irqs_disabled());
 354
 355         if (cpufreq_disabled())
 356                 return;
 357
 358         freqs->flags = cpufreq_driver->flags;
 359         pr_debug("notification %u of frequency transition to %u kHz\n",
 360                  state, freqs->new);
 361
 362         switch (state) {
 363
 364         case CPUFREQ_PRECHANGE:
 365                 /* detect if the driver reported a value as "old frequency"
 366                  * which is not equal to what the cpufreq core thinks is
 367                  * "old frequency".
 368                  */
 369                 if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
 370                         if ((policy) && (policy->cpu == freqs->cpu) &&
 371                             (policy->cur) && (policy->cur != freqs->old)) {
 372                                 pr_debug("Warning: CPU frequency is %u, cpufreq assumed %u kHz\n",
 373                                          freqs->old, policy->cur);
 374                                 freqs->old = policy->cur;
 375                         }
 376                 }
 377                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 378                                 CPUFREQ_PRECHANGE, freqs);
 379                 adjust_jiffies(CPUFREQ_PRECHANGE, freqs);
 380                 break;
 381
 382         case CPUFREQ_POSTCHANGE:
 383                 adjust_jiffies(CPUFREQ_POSTCHANGE, freqs);
 384                 pr_debug("FREQ: %lu - CPU: %lu\n",
 385                          (unsigned long)freqs->new, (unsigned long)freqs->cpu);
 386                 trace_cpu_frequency(freqs->new, freqs->cpu);
 387                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 388                                 CPUFREQ_POSTCHANGE, freqs);
 389                 if (likely(policy) && likely(policy->cpu == freqs->cpu))
 390                         policy->cur = freqs->new;
 391                 break;
 392         }
 393 }
 394
 395 /**
 396  * cpufreq_notify_transition - call notifier chain and adjust_jiffies
 397  * on frequency transition.
 398  *
 399  * This function calls the transition notifiers and the "adjust_jiffies"
 400  * function. It is called twice on all CPU frequency changes that have
 401  * external effects.
 402  */
 403 static void cpufreq_notify_transition(struct cpufreq_policy *policy,
 404                 struct cpufreq_freqs *freqs, unsigned int state)
 405 {
 406         for_each_cpu(freqs->cpu, policy->cpus)
 407                 __cpufreq_notify_transition(policy, freqs, state);
 408 }
 409
 410 /* Do post notifications when there are chances that transition has failed */
 411 static void cpufreq_notify_post_transition(struct cpufreq_policy *policy,
 412                 struct cpufreq_freqs *freqs, int transition_failed)
 413 {
 414         cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
 415         if (!transition_failed)
 416                 return;
 417
 418         swap(freqs->old, freqs->new);
 419         cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);
 420         cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
 421 }
 422
 423 void cpufreq_freq_transition_begin(struct cpufreq_policy *policy,
 424                 struct cpufreq_freqs *freqs)
 425 {
 426
 427         /*
 428          * Catch double invocations of _begin() which lead to self-deadlock.
 429          * ASYNC_NOTIFICATION drivers are left out because the cpufreq core
 430          * doesn't invoke _begin() on their behalf, and hence the chances of
 431          * double invocations are very low. Moreover, there are scenarios
 432          * where these checks can emit false-positive warnings in these
 433          * drivers; so we avoid that by skipping them altogether.
 434          */
 435         WARN_ON(!(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION)
 436                                 && current == policy->transition_task);
 437
 438 wait:
 439         wait_event(policy->transition_wait, !policy->transition_ongoing);
 440
 441         spin_lock(&policy->transition_lock);
 442
 443         if (unlikely(policy->transition_ongoing)) {
 444                 spin_unlock(&policy->transition_lock);
 445                 goto wait;
 446         }
 447
 448         policy->transition_ongoing = true;
 449         policy->transition_task = current;
 450
 451         spin_unlock(&policy->transition_lock);
 452
 453         cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);
 454 }
 455 EXPORT_SYMBOL_GPL(cpufreq_freq_transition_begin);
 456
 457 void cpufreq_freq_transition_end(struct cpufreq_policy *policy,
 458                 struct cpufreq_freqs *freqs, int transition_failed)
 459 {
 460         if (unlikely(WARN_ON(!policy->transition_ongoing)))
 461                 return;
 462
 463         cpufreq_notify_post_transition(policy, freqs, transition_failed);
 464
 465         policy->transition_ongoing = false;
 466         policy->transition_task = NULL;
 467
 468         wake_up(&policy->transition_wait);
 469 }
 470 EXPORT_SYMBOL_GPL(cpufreq_freq_transition_end);
 471
 472
 473 /*********************************************************************
 474  *                          SYSFS INTERFACE                          *
 475  *********************************************************************/
 476 static ssize_t show_boost(struct kobject *kobj,
 477                                  struct attribute *attr, char *buf)
 478 {
 479         return sprintf(buf, "%d\n", cpufreq_driver->boost_enabled);
 480 }
 481
 482 static ssize_t store_boost(struct kobject *kobj, struct attribute *attr,
 483                                   const char *buf, size_t count)
 484 {
 485         int ret, enable;
 486
 487         ret = sscanf(buf, "%d", &enable);
 488         if (ret != 1 || enable < 0 || enable > 1)
 489                 return -EINVAL;
 490
 491         if (cpufreq_boost_trigger_state(enable)) {
 492                 pr_err("%s: Cannot %s BOOST!\n",
 493                        __func__, enable ? "enable" : "disable");
 494                 return -EINVAL;
 495         }
 496
 497         pr_debug("%s: cpufreq BOOST %s\n",
 498                  __func__, enable ? "enabled" : "disabled");
 499
 500         return count;
 501 }
 502 define_one_global_rw(boost);
 503
 504 static struct cpufreq_governor *find_governor(const char *str_governor)
 505 {
 506         struct cpufreq_governor *t;
 507
 508         for_each_governor(t)
 509                 if (!strncasecmp(str_governor, t->name, CPUFREQ_NAME_LEN))
 510                         return t;
 511
 512         return NULL;
 513 }
 514
 515 /**
 516  * cpufreq_parse_governor - parse a governor string
 517  */
 518 static int cpufreq_parse_governor(char *str_governor, unsigned int *policy,
 519                                 struct cpufreq_governor **governor)
 520 {
 521         int err = -EINVAL;
 522
 523         if (cpufreq_driver->setpolicy) {
 524                 if (!strncasecmp(str_governor, "performance", CPUFREQ_NAME_LEN)) {
 525                         *policy = CPUFREQ_POLICY_PERFORMANCE;
 526                         err = 0;
 527                 } else if (!strncasecmp(str_governor, "powersave",
 528                                                 CPUFREQ_NAME_LEN)) {
 529                         *policy = CPUFREQ_POLICY_POWERSAVE;
 530                         err = 0;
 531                 }
 532         } else {
 533                 struct cpufreq_governor *t;
 534
 535                 mutex_lock(&cpufreq_governor_mutex);
 536
 537                 t = find_governor(str_governor);
 538
 539                 if (t == NULL) {
 540                         int ret;
 541
 542                         mutex_unlock(&cpufreq_governor_mutex);
 543                         ret = request_module("cpufreq_%s", str_governor);
 544                         mutex_lock(&cpufreq_governor_mutex);
 545
 546                         if (ret == 0)
 547                                 t = find_governor(str_governor);
 548                 }
 549
 550                 if (t != NULL) {
 551                         *governor = t;
 552                         err = 0;
 553                 }
 554
 555                 mutex_unlock(&cpufreq_governor_mutex);
 556         }
 557         return err;
 558 }
 559
 560 /**
 561  * cpufreq_per_cpu_attr_read() / show_##file_name() -
 562  * print out cpufreq information
 563  *
 564  * Write out information from cpufreq_driver->policy[cpu]; object must be
 565  * "unsigned int".
 566  */
 567
 568 #define show_one(file_name, object)                     \
 569 static ssize_t show_##file_name                         \
 570 (struct cpufreq_policy *policy, char *buf)              \
 571 {                                                       \
 572         return sprintf(buf, "%u\n", policy->object);    \
 573 }
 574
 575 show_one(cpuinfo_min_freq, cpuinfo.min_freq);
 576 show_one(cpuinfo_max_freq, cpuinfo.max_freq);
 577 show_one(cpuinfo_transition_latency, cpuinfo.transition_latency);
 578 show_one(scaling_min_freq, min);
 579 show_one(scaling_max_freq, max);
 580
 581 static ssize_t show_scaling_cur_freq(struct cpufreq_policy *policy, char *buf)
 582 {
 583         ssize_t ret;
 584
 585         if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get)
 586                 ret = sprintf(buf, "%u\n", cpufreq_driver->get(policy->cpu));
 587         else
 588                 ret = sprintf(buf, "%u\n", policy->cur);
 589         return ret;
 590 }
 591
 592 static int cpufreq_set_policy(struct cpufreq_policy *policy,
 593                                 struct cpufreq_policy *new_policy);
 594
 595 /**
 596  * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access
 597  */
 598 #define store_one(file_name, object)                    \
 599 static ssize_t store_##file_name                                        \
 600 (struct cpufreq_policy *policy, const char *buf, size_t count)          \
 601 {                                                                       \
 602         int ret, temp;                                                  \
 603         struct cpufreq_policy new_policy;                               \
 604                                                                         \
 605         memcpy(&new_policy, policy, sizeof(*policy));                   \
 606                                                                         \
 607         ret = sscanf(buf, "%u", &new_policy.object);                    \
 608         if (ret != 1)                                                   \
 609                 return -EINVAL;                                         \
 610                                                                         \
 611         temp = new_policy.object;                                       \
 612         ret = cpufreq_set_policy(policy, &new_policy);          \
 613         if (!ret)                                                       \
 614                 policy->user_policy.object = temp;                      \
 615                                                                         \
 616         return ret ? ret : count;                                       \
 617 }
 618
 619 store_one(scaling_min_freq, min);
 620 store_one(scaling_max_freq, max);
 621
 622 /**
 623  * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware
 624  */
 625 static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy,
 626                                         char *buf)
 627 {
 628         unsigned int cur_freq = __cpufreq_get(policy);
 629         if (!cur_freq)
 630                 return sprintf(buf, "<unknown>");
 631         return sprintf(buf, "%u\n", cur_freq);
 632 }
 633
 634 /**
 635  * show_scaling_governor - show the current policy for the specified CPU
 636  */
 637 static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf)
 638 {
 639         if (policy->policy == CPUFREQ_POLICY_POWERSAVE)
 640                 return sprintf(buf, "powersave\n");
 641         else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
 642                 return sprintf(buf, "performance\n");
 643         else if (policy->governor)
 644                 return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n",
 645                                 policy->governor->name);
 646         return -EINVAL;
 647 }
 648
 649 /**
 650  * store_scaling_governor - store policy for the specified CPU
 651  */
 652 static ssize_t store_scaling_governor(struct cpufreq_policy *policy,
 653                                         const char *buf, size_t count)
 654 {
 655         int ret;
 656         char    str_governor[16];
 657         struct cpufreq_policy new_policy;
 658
 659         memcpy(&new_policy, policy, sizeof(*policy));
 660
 661         ret = sscanf(buf, "%15s", str_governor);
 662         if (ret != 1)
 663                 return -EINVAL;
 664
 665         if (cpufreq_parse_governor(str_governor, &new_policy.policy,
 666                                                 &new_policy.governor))
 667                 return -EINVAL;
 668
 669         ret = cpufreq_set_policy(policy, &new_policy);
 670         return ret ? ret : count;
 671 }
 672
 673 /**
 674  * show_scaling_driver - show the cpufreq driver currently loaded
 675  */
 676 static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf)
 677 {
 678         return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n", cpufreq_driver->name);
 679 }
 680
 681 /**
 682  * show_scaling_available_governors - show the available CPUfreq governors
 683  */
 684 static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy,
 685                                                 char *buf)
 686 {
 687         ssize_t i = 0;
 688         struct cpufreq_governor *t;
 689
 690         if (!has_target()) {
 691                 i += sprintf(buf, "performance powersave");
 692                 goto out;
 693         }
 694
 695         for_each_governor(t) {
 696                 if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char))
 697                     - (CPUFREQ_NAME_LEN + 2)))
 698                         goto out;
 699                 i += scnprintf(&buf[i], CPUFREQ_NAME_PLEN, "%s ", t->name);
 700         }
 701 out:
 702         i += sprintf(&buf[i], "\n");
 703         return i;
 704 }
 705
 706 ssize_t cpufreq_show_cpus(const struct cpumask *mask, char *buf)
 707 {
 708         ssize_t i = 0;
 709         unsigned int cpu;
 710
 711         for_each_cpu(cpu, mask) {
 712                 if (i)
 713                         i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), " ");
 714                 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u", cpu);
 715                 if (i >= (PAGE_SIZE - 5))
 716                         break;
 717         }
 718         i += sprintf(&buf[i], "\n");
 719         return i;
 720 }
 721 EXPORT_SYMBOL_GPL(cpufreq_show_cpus);
 722
 723 /**
 724  * show_related_cpus - show the CPUs affected by each transition even if
 725  * hw coordination is in use
 726  */
 727 static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf)
 728 {
 729         return cpufreq_show_cpus(policy->related_cpus, buf);
 730 }
 731
 732 /**
 733  * show_affected_cpus - show the CPUs affected by each transition
 734  */
 735 static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf)
 736 {
 737         return cpufreq_show_cpus(policy->cpus, buf);
 738 }
 739
 740 static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,
 741                                         const char *buf, size_t count)
 742 {
 743         unsigned int freq = 0;
 744         unsigned int ret;
 745
 746         if (!policy->governor || !policy->governor->store_setspeed)
 747                 return -EINVAL;
 748
 749         ret = sscanf(buf, "%u", &freq);
 750         if (ret != 1)
 751                 return -EINVAL;
 752
 753         policy->governor->store_setspeed(policy, freq);
 754
 755         return count;
 756 }
 757
 758 static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)
 759 {
 760         if (!policy->governor || !policy->governor->show_setspeed)
 761                 return sprintf(buf, "<unsupported>\n");
 762
 763         return policy->governor->show_setspeed(policy, buf);
 764 }
 765
 766 /**
 767  * show_bios_limit - show the current cpufreq HW/BIOS limitation
 768  */
 769 static ssize_t show_bios_limit(struct cpufreq_policy *policy, char *buf)
 770 {
 771         unsigned int limit;
 772         int ret;
 773         if (cpufreq_driver->bios_limit) {
 774                 ret = cpufreq_driver->bios_limit(policy->cpu, &limit);
 775                 if (!ret)
 776                         return sprintf(buf, "%u\n", limit);
 777         }
 778         return sprintf(buf, "%u\n", policy->cpuinfo.max_freq);
 779 }
 780
 781 cpufreq_freq_attr_ro_perm(cpuinfo_cur_freq, 0400);
 782 cpufreq_freq_attr_ro(cpuinfo_min_freq);
 783 cpufreq_freq_attr_ro(cpuinfo_max_freq);
 784 cpufreq_freq_attr_ro(cpuinfo_transition_latency);
 785 cpufreq_freq_attr_ro(scaling_available_governors);
 786 cpufreq_freq_attr_ro(scaling_driver);
 787 cpufreq_freq_attr_ro(scaling_cur_freq);
 788 cpufreq_freq_attr_ro(bios_limit);
 789 cpufreq_freq_attr_ro(related_cpus);
 790 cpufreq_freq_attr_ro(affected_cpus);
 791 cpufreq_freq_attr_rw(scaling_min_freq);
 792 cpufreq_freq_attr_rw(scaling_max_freq);
 793 cpufreq_freq_attr_rw(scaling_governor);
 794 cpufreq_freq_attr_rw(scaling_setspeed);
 795
 796 static struct attribute *default_attrs[] = {
 797         &cpuinfo_min_freq.attr,
 798         &cpuinfo_max_freq.attr,
 799         &cpuinfo_transition_latency.attr,
 800         &scaling_min_freq.attr,
 801         &scaling_max_freq.attr,
 802         &affected_cpus.attr,
 803         &related_cpus.attr,
 804         &scaling_governor.attr,
 805         &scaling_driver.attr,
 806         &scaling_available_governors.attr,
 807         &scaling_setspeed.attr,
 808         NULL
 809 };
 810
 811 #define to_policy(k) container_of(k, struct cpufreq_policy, kobj)
 812 #define to_attr(a) container_of(a, struct freq_attr, attr)
 813
 814 static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf)
 815 {
 816         struct cpufreq_policy *policy = to_policy(kobj);
 817         struct freq_attr *fattr = to_attr(attr);
 818         ssize_t ret;
 819
 820         down_read(&policy->rwsem);
 821
 822         if (fattr->show)
 823                 ret = fattr->show(policy, buf);
 824         else
 825                 ret = -EIO;
 826
 827         up_read(&policy->rwsem);
 828
 829         return ret;
 830 }
 831
 832 static ssize_t store(struct kobject *kobj, struct attribute *attr,
 833                      const char *buf, size_t count)
 834 {
 835         struct cpufreq_policy *policy = to_policy(kobj);
 836         struct freq_attr *fattr = to_attr(attr);
 837         ssize_t ret = -EINVAL;
 838
 839         get_online_cpus();
 840
 841         if (!cpu_online(policy->cpu))
 842                 goto unlock;
 843
 844         down_write(&policy->rwsem);
 845
 846         if (fattr->store)
 847                 ret = fattr->store(policy, buf, count);
 848         else
 849                 ret = -EIO;
 850
 851         up_write(&policy->rwsem);
 852 unlock:
 853         put_online_cpus();
 854
 855         return ret;
 856 }
 857
 858 static void cpufreq_sysfs_release(struct kobject *kobj)
 859 {
 860         struct cpufreq_policy *policy = to_policy(kobj);
 861         pr_debug("last reference is dropped\n");
 862         complete(&policy->kobj_unregister);
 863 }
 864
 865 static const struct sysfs_ops sysfs_ops = {
 866         .show   = show,
 867         .store  = store,
 868 };
 869
 870 static struct kobj_type ktype_cpufreq = {
 871         .sysfs_ops      = &sysfs_ops,
 872         .default_attrs  = default_attrs,
 873         .release        = cpufreq_sysfs_release,
 874 };
 875
 876 static int add_cpu_dev_symlink(struct cpufreq_policy *policy, int cpu)
 877 {
 878         struct device *cpu_dev;
 879
 880         pr_debug("%s: Adding symlink for CPU: %u\n", __func__, cpu);
 881
 882         if (!policy)
 883                 return 0;
 884
 885         cpu_dev = get_cpu_device(cpu);
 886         if (WARN_ON(!cpu_dev))
 887                 return 0;
 888
 889         return sysfs_create_link(&cpu_dev->kobj, &policy->kobj, "cpufreq");
 890 }
 891
 892 static void remove_cpu_dev_symlink(struct cpufreq_policy *policy, int cpu)
 893 {
 894         struct device *cpu_dev;
 895
 896         pr_debug("%s: Removing symlink for CPU: %u\n", __func__, cpu);
 897
 898         cpu_dev = get_cpu_device(cpu);
 899         if (WARN_ON(!cpu_dev))
 900                 return;
 901
 902         sysfs_remove_link(&cpu_dev->kobj, "cpufreq");
 903 }
 904
 905 /* Add/remove symlinks for all related CPUs */
 906 static int cpufreq_add_dev_symlink(struct cpufreq_policy *policy)
 907 {
 908         unsigned int j;
 909         int ret = 0;
 910
 911         /* Some related CPUs might not be present (physically hotplugged) */
 912         for_each_cpu(j, policy->real_cpus) {
 913                 ret = add_cpu_dev_symlink(policy, j);
 914                 if (ret)
 915                         break;
 916         }
 917
 918         return ret;
 919 }
 920
 921 static void cpufreq_remove_dev_symlink(struct cpufreq_policy *policy)
 922 {
 923         unsigned int j;
 924
 925         /* Some related CPUs might not be present (physically hotplugged) */
 926         for_each_cpu(j, policy->real_cpus)
 927                 remove_cpu_dev_symlink(policy, j);
 928 }
 929
 930 static int cpufreq_add_dev_interface(struct cpufreq_policy *policy)
 931 {
 932         struct freq_attr **drv_attr;
 933         int ret = 0;
 934
 935         /* set up files for this cpu device */
 936         drv_attr = cpufreq_driver->attr;
 937         while (drv_attr && *drv_attr) {
 938                 ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr));
 939                 if (ret)
 940                         return ret;
 941                 drv_attr++;
 942         }
 943         if (cpufreq_driver->get) {
 944                 ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr);
 945                 if (ret)
 946                         return ret;
 947         }
 948
 949         ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr);
 950         if (ret)
 951                 return ret;
 952
 953         if (cpufreq_driver->bios_limit) {
 954                 ret = sysfs_create_file(&policy->kobj, &bios_limit.attr);
 955                 if (ret)
 956                         return ret;
 957         }
 958
 959         return cpufreq_add_dev_symlink(policy);
 960 }
 961
 962 static int cpufreq_init_policy(struct cpufreq_policy *policy)
 963 {
 964         struct cpufreq_governor *gov = NULL;
 965         struct cpufreq_policy new_policy;
 966
 967         memcpy(&new_policy, policy, sizeof(*policy));
 968
 969         /* Update governor of new_policy to the governor used before hotplug */
 970         gov = find_governor(policy->last_governor);
 971         if (gov)
 972                 pr_debug("Restoring governor %s for cpu %d\n",
 973                                 policy->governor->name, policy->cpu);
 974         else
 975                 gov = CPUFREQ_DEFAULT_GOVERNOR;
 976
 977         new_policy.governor = gov;
 978
 979         /* Use the default policy if its valid. */
 980         if (cpufreq_driver->setpolicy)
 981                 cpufreq_parse_governor(gov->name, &new_policy.policy, NULL);
 982
 983         /* set default policy */
 984         return cpufreq_set_policy(policy, &new_policy);
 985 }
 986
 987 static int cpufreq_add_policy_cpu(struct cpufreq_policy *policy, unsigned int cpu)
 988 {
 989         int ret = 0;
 990
 991         /* Has this CPU been taken care of already? */
 992         if (cpumask_test_cpu(cpu, policy->cpus))
 993                 return 0;
 994
 995         if (has_target()) {
 996                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
 997                 if (ret) {
 998                         pr_err("%s: Failed to stop governor\n", __func__);
 999                         return ret;
1000                 }
1001         }
1002
1003         down_write(&policy->rwsem);
1004         cpumask_set_cpu(cpu, policy->cpus);
1005         up_write(&policy->rwsem);
1006
1007         if (has_target()) {
1008                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_START);
1009                 if (!ret)
1010                         ret = __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
1011
1012                 if (ret) {
1013                         pr_err("%s: Failed to start governor\n", __func__);
1014                         return ret;
1015                 }
1016         }
1017
1018         return 0;
1019 }
1020
1021 static struct cpufreq_policy *cpufreq_policy_alloc(unsigned int cpu)
1022 {
1023         struct device *dev = get_cpu_device(cpu);
1024         struct cpufreq_policy *policy;
1025
1026         if (WARN_ON(!dev))
1027                 return NULL;
1028
1029         policy = kzalloc(sizeof(*policy), GFP_KERNEL);
1030         if (!policy)
1031                 return NULL;
1032
1033         if (!alloc_cpumask_var(&policy->cpus, GFP_KERNEL))
1034                 goto err_free_policy;
1035
1036         if (!zalloc_cpumask_var(&policy->related_cpus, GFP_KERNEL))
1037                 goto err_free_cpumask;
1038
1039         if (!zalloc_cpumask_var(&policy->real_cpus, GFP_KERNEL))
1040                 goto err_free_rcpumask;
1041
1042         kobject_init(&policy->kobj, &ktype_cpufreq);
1043         INIT_LIST_HEAD(&policy->policy_list);
1044         init_rwsem(&policy->rwsem);
1045         spin_lock_init(&policy->transition_lock);
1046         init_waitqueue_head(&policy->transition_wait);
1047         init_completion(&policy->kobj_unregister);
1048         INIT_WORK(&policy->update, handle_update);
1049
1050         policy->cpu = cpu;
1051         return policy;
1052
1053 err_free_rcpumask:
1054         free_cpumask_var(policy->related_cpus);
1055 err_free_cpumask:
1056         free_cpumask_var(policy->cpus);
1057 err_free_policy:
1058         kfree(policy);
1059
1060         return NULL;
1061 }
1062
1063 static void cpufreq_policy_put_kobj(struct cpufreq_policy *policy, bool notify)
1064 {
1065         struct kobject *kobj;
1066         struct completion *cmp;
1067
1068         if (notify)
1069                 blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1070                                              CPUFREQ_REMOVE_POLICY, policy);
1071
1072         down_write(&policy->rwsem);
1073         cpufreq_remove_dev_symlink(policy);
1074         kobj = &policy->kobj;
1075         cmp = &policy->kobj_unregister;
1076         up_write(&policy->rwsem);
1077         kobject_put(kobj);
1078
1079         /*
1080          * We need to make sure that the underlying kobj is
1081          * actually not referenced anymore by anybody before we
1082          * proceed with unloading.
1083          */
1084         pr_debug("waiting for dropping of refcount\n");
1085         wait_for_completion(cmp);
1086         pr_debug("wait complete\n");
1087 }
1088
1089 static void cpufreq_policy_free(struct cpufreq_policy *policy, bool notify)
1090 {
1091         unsigned long flags;
1092         int cpu;
1093
1094         /* Remove policy from list */
1095         write_lock_irqsave(&cpufreq_driver_lock, flags);
1096         list_del(&policy->policy_list);
1097
1098         for_each_cpu(cpu, policy->related_cpus)
1099                 per_cpu(cpufreq_cpu_data, cpu) = NULL;
1100         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1101
1102         cpufreq_policy_put_kobj(policy, notify);
1103         free_cpumask_var(policy->real_cpus);
1104         free_cpumask_var(policy->related_cpus);
1105         free_cpumask_var(policy->cpus);
1106         kfree(policy);
1107 }
1108
1109 static int cpufreq_online(unsigned int cpu)
1110 {
1111         struct cpufreq_policy *policy;
1112         bool new_policy;
1113         unsigned long flags;
1114         unsigned int j;
1115         int ret;
1116
1117         pr_debug("%s: bringing CPU%u online\n", __func__, cpu);
1118
1119         /* Check if this CPU already has a policy to manage it */
1120         policy = per_cpu(cpufreq_cpu_data, cpu);
1121         if (policy) {
1122                 WARN_ON(!cpumask_test_cpu(cpu, policy->related_cpus));
1123                 if (!policy_is_inactive(policy))
1124                         return cpufreq_add_policy_cpu(policy, cpu);
1125
1126                 /* This is the only online CPU for the policy.  Start over. */
1127                 new_policy = false;
1128                 down_write(&policy->rwsem);
1129                 policy->cpu = cpu;
1130                 policy->governor = NULL;
1131                 up_write(&policy->rwsem);
1132         } else {
1133                 new_policy = true;
1134                 policy = cpufreq_policy_alloc(cpu);
1135                 if (!policy)
1136                         return -ENOMEM;
1137         }
1138
1139         cpumask_copy(policy->cpus, cpumask_of(cpu));
1140
1141         /* call driver. From then on the cpufreq must be able
1142          * to accept all calls to ->verify and ->setpolicy for this CPU
1143          */
1144         ret = cpufreq_driver->init(policy);
1145         if (ret) {
1146                 pr_debug("initialization failed\n");
1147                 goto out_free_policy;
1148         }
1149
1150         down_write(&policy->rwsem);
1151
1152         if (new_policy) {
1153                 /* related_cpus should at least include policy->cpus. */
1154                 cpumask_copy(policy->related_cpus, policy->cpus);
1155                 /* Remember CPUs present at the policy creation time. */
1156                 cpumask_and(policy->real_cpus, policy->cpus, cpu_present_mask);
1157
1158                 /* Name and add the kobject */
1159                 ret = kobject_add(&policy->kobj, cpufreq_global_kobject,
1160                                   "policy%u",
1161                                   cpumask_first(policy->related_cpus));
1162                 if (ret) {
1163                         pr_err("%s: failed to add policy->kobj: %d\n", __func__,
1164                                ret);
1165                         goto out_exit_policy;
1166                 }
1167         }
1168
1169         /*
1170          * affected cpus must always be the one, which are online. We aren't
1171          * managing offline cpus here.
1172          */
1173         cpumask_and(policy->cpus, policy->cpus, cpu_online_mask);
1174
1175         if (new_policy) {
1176                 policy->user_policy.min = policy->min;
1177                 policy->user_policy.max = policy->max;
1178
1179                 write_lock_irqsave(&cpufreq_driver_lock, flags);
1180                 for_each_cpu(j, policy->related_cpus)
1181                         per_cpu(cpufreq_cpu_data, j) = policy;
1182                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1183         }
1184
1185         if (cpufreq_driver->get && !cpufreq_driver->setpolicy) {
1186                 policy->cur = cpufreq_driver->get(policy->cpu);
1187                 if (!policy->cur) {
1188                         pr_err("%s: ->get() failed\n", __func__);
1189                         goto out_exit_policy;
1190                 }
1191         }
1192
1193         /*
1194          * Sometimes boot loaders set CPU frequency to a value outside of
1195          * frequency table present with cpufreq core. In such cases CPU might be
1196          * unstable if it has to run on that frequency for long duration of time
1197          * and so its better to set it to a frequency which is specified in
1198          * freq-table. This also makes cpufreq stats inconsistent as
1199          * cpufreq-stats would fail to register because current frequency of CPU
1200          * isn't found in freq-table.
1201          *
1202          * Because we don't want this change to effect boot process badly, we go
1203          * for the next freq which is >= policy->cur ('cur' must be set by now,
1204          * otherwise we will end up setting freq to lowest of the table as 'cur'
1205          * is initialized to zero).
1206          *
1207          * We are passing target-freq as "policy->cur - 1" otherwise
1208          * __cpufreq_driver_target() would simply fail, as policy->cur will be
1209          * equal to target-freq.
1210          */
1211         if ((cpufreq_driver->flags & CPUFREQ_NEED_INITIAL_FREQ_CHECK)
1212             && has_target()) {
1213                 /* Are we running at unknown frequency ? */
1214                 ret = cpufreq_frequency_table_get_index(policy, policy->cur);
1215                 if (ret == -EINVAL) {
1216                         /* Warn user and fix it */
1217                         pr_warn("%s: CPU%d: Running at unlisted freq: %u KHz\n",
1218                                 __func__, policy->cpu, policy->cur);
1219                         ret = __cpufreq_driver_target(policy, policy->cur - 1,
1220                                 CPUFREQ_RELATION_L);
1221
1222                         /*
1223                          * Reaching here after boot in a few seconds may not
1224                          * mean that system will remain stable at "unknown"
1225                          * frequency for longer duration. Hence, a BUG_ON().
1226                          */
1227                         BUG_ON(ret);
1228                         pr_warn("%s: CPU%d: Unlisted initial frequency changed to: %u KHz\n",
1229                                 __func__, policy->cpu, policy->cur);
1230                 }
1231         }
1232
1233         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1234                                      CPUFREQ_START, policy);
1235
1236         if (new_policy) {
1237                 ret = cpufreq_add_dev_interface(policy);
1238                 if (ret)
1239                         goto out_exit_policy;
1240                 blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1241                                 CPUFREQ_CREATE_POLICY, policy);
1242
1243                 write_lock_irqsave(&cpufreq_driver_lock, flags);
1244                 list_add(&policy->policy_list, &cpufreq_policy_list);
1245                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1246         }
1247
1248         ret = cpufreq_init_policy(policy);
1249         if (ret) {
1250                 pr_err("%s: Failed to initialize policy for cpu: %d (%d)\n",
1251                        __func__, cpu, ret);
1252                 /* cpufreq_policy_free() will notify based on this */
1253                 new_policy = false;
1254                 goto out_exit_policy;
1255         }
1256
1257         up_write(&policy->rwsem);
1258
1259         kobject_uevent(&policy->kobj, KOBJ_ADD);
1260
1261         /* Callback for handling stuff after policy is ready */
1262         if (cpufreq_driver->ready)
1263                 cpufreq_driver->ready(policy);
1264
1265         pr_debug("initialization complete\n");
1266
1267         return 0;
1268
1269 out_exit_policy:
1270         up_write(&policy->rwsem);
1271
1272         if (cpufreq_driver->exit)
1273                 cpufreq_driver->exit(policy);
1274 out_free_policy:
1275         cpufreq_policy_free(policy, !new_policy);
1276         return ret;
1277 }
1278
1279 /**
1280  * cpufreq_add_dev - the cpufreq interface for a CPU device.
1281  * @dev: CPU device.
1282  * @sif: Subsystem interface structure pointer (not used)
1283  */
1284 static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif)
1285 {
1286         unsigned cpu = dev->id;
1287         int ret;
1288
1289         dev_dbg(dev, "%s: adding CPU%u\n", __func__, cpu);
1290
1291         if (cpu_online(cpu)) {
1292                 ret = cpufreq_online(cpu);
1293         } else {
1294                 /*
1295                  * A hotplug notifier will follow and we will handle it as CPU
1296                  * online then.  For now, just create the sysfs link, unless
1297                  * there is no policy or the link is already present.
1298                  */
1299                 struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1300
1301                 ret = policy && !cpumask_test_and_set_cpu(cpu, policy->real_cpus)
1302                         ? add_cpu_dev_symlink(policy, cpu) : 0;
1303         }
1304
1305         return ret;
1306 }
1307
1308 static void cpufreq_offline_prepare(unsigned int cpu)
1309 {
1310         struct cpufreq_policy *policy;
1311
1312         pr_debug("%s: unregistering CPU %u\n", __func__, cpu);
1313
1314         policy = cpufreq_cpu_get_raw(cpu);
1315         if (!policy) {
1316                 pr_debug("%s: No cpu_data found\n", __func__);
1317                 return;
1318         }
1319
1320         if (has_target()) {
1321                 int ret = __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
1322                 if (ret)
1323                         pr_err("%s: Failed to stop governor\n", __func__);
1324         }
1325
1326         down_write(&policy->rwsem);
1327         cpumask_clear_cpu(cpu, policy->cpus);
1328
1329         if (policy_is_inactive(policy)) {
1330                 if (has_target())
1331                         strncpy(policy->last_governor, policy->governor->name,
1332                                 CPUFREQ_NAME_LEN);
1333         } else if (cpu == policy->cpu) {
1334                 /* Nominate new CPU */
1335                 policy->cpu = cpumask_any(policy->cpus);
1336         }
1337         up_write(&policy->rwsem);
1338
1339         /* Start governor again for active policy */
1340         if (!policy_is_inactive(policy)) {
1341                 if (has_target()) {
1342                         int ret = __cpufreq_governor(policy, CPUFREQ_GOV_START);
1343                         if (!ret)
1344                                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
1345
1346                         if (ret)
1347                                 pr_err("%s: Failed to start governor\n", __func__);
1348                 }
1349         } else if (cpufreq_driver->stop_cpu) {
1350                 cpufreq_driver->stop_cpu(policy);
1351         }
1352 }
1353
1354 static void cpufreq_offline_finish(unsigned int cpu)
1355 {
1356         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1357
1358         if (!policy) {
1359                 pr_debug("%s: No cpu_data found\n", __func__);
1360                 return;
1361         }
1362
1363         /* Only proceed for inactive policies */
1364         if (!policy_is_inactive(policy))
1365                 return;
1366
1367         /* If cpu is last user of policy, free policy */
1368         if (has_target()) {
1369                 int ret = __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT);
1370                 if (ret)
1371                         pr_err("%s: Failed to exit governor\n", __func__);
1372         }
1373
1374         /*
1375          * Perform the ->exit() even during light-weight tear-down,
1376          * since this is a core component, and is essential for the
1377          * subsequent light-weight ->init() to succeed.
1378          */
1379         if (cpufreq_driver->exit) {
1380                 cpufreq_driver->exit(policy);
1381                 policy->freq_table = NULL;
1382         }
1383 }
1384
1385 /**
1386  * cpufreq_remove_dev - remove a CPU device
1387  *
1388  * Removes the cpufreq interface for a CPU device.
1389  */
1390 static void cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif)
1391 {
1392         unsigned int cpu = dev->id;
1393         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1394
1395         if (!policy)
1396                 return;
1397
1398         if (cpu_online(cpu)) {
1399                 cpufreq_offline_prepare(cpu);
1400                 cpufreq_offline_finish(cpu);
1401         }
1402
1403         cpumask_clear_cpu(cpu, policy->real_cpus);
1404
1405         if (cpumask_empty(policy->real_cpus)) {
1406                 cpufreq_policy_free(policy, true);
1407                 return;
1408         }
1409
1410         remove_cpu_dev_symlink(policy, cpu);
1411 }
1412
1413 static void handle_update(struct work_struct *work)
1414 {
1415         struct cpufreq_policy *policy =
1416                 container_of(work, struct cpufreq_policy, update);
1417         unsigned int cpu = policy->cpu;
1418         pr_debug("handle_update for cpu %u called\n", cpu);
1419         cpufreq_update_policy(cpu);
1420 }
1421
1422 /**
1423  *      cpufreq_out_of_sync - If actual and saved CPU frequency differs, we're
1424  *      in deep trouble.
1425  *      @policy: policy managing CPUs
1426  *      @new_freq: CPU frequency the CPU actually runs at
1427  *
1428  *      We adjust to current frequency first, and need to clean up later.
1429  *      So either call to cpufreq_update_policy() or schedule handle_update()).
1430  */
1431 static void cpufreq_out_of_sync(struct cpufreq_policy *policy,
1432                                 unsigned int new_freq)
1433 {
1434         struct cpufreq_freqs freqs;
1435
1436         pr_debug("Warning: CPU frequency out of sync: cpufreq and timing core thinks of %u, is %u kHz\n",
1437                  policy->cur, new_freq);
1438
1439         freqs.old = policy->cur;
1440         freqs.new = new_freq;
1441
1442         cpufreq_freq_transition_begin(policy, &freqs);
1443         cpufreq_freq_transition_end(policy, &freqs, 0);
1444 }
1445
1446 /**
1447  * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur
1448  * @cpu: CPU number
1449  *
1450  * This is the last known freq, without actually getting it from the driver.
1451  * Return value will be same as what is shown in scaling_cur_freq in sysfs.
1452  */
1453 unsigned int cpufreq_quick_get(unsigned int cpu)
1454 {
1455         struct cpufreq_policy *policy;
1456         unsigned int ret_freq = 0;
1457
1458         if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get)
1459                 return cpufreq_driver->get(cpu);
1460
1461         policy = cpufreq_cpu_get(cpu);
1462         if (policy) {
1463                 ret_freq = policy->cur;
1464                 cpufreq_cpu_put(policy);
1465         }
1466
1467         return ret_freq;
1468 }
1469 EXPORT_SYMBOL(cpufreq_quick_get);
1470
1471 /**
1472  * cpufreq_quick_get_max - get the max reported CPU frequency for this CPU
1473  * @cpu: CPU number
1474  *
1475  * Just return the max possible frequency for a given CPU.
1476  */
1477 unsigned int cpufreq_quick_get_max(unsigned int cpu)
1478 {
1479         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1480         unsigned int ret_freq = 0;
1481
1482         if (policy) {
1483                 ret_freq = policy->max;
1484                 cpufreq_cpu_put(policy);
1485         }
1486
1487         return ret_freq;
1488 }
1489 EXPORT_SYMBOL(cpufreq_quick_get_max);
1490
1491 static unsigned int __cpufreq_get(struct cpufreq_policy *policy)
1492 {
1493         unsigned int ret_freq = 0;
1494
1495         if (!cpufreq_driver->get)
1496                 return ret_freq;
1497
1498         ret_freq = cpufreq_driver->get(policy->cpu);
1499
1500         /* Updating inactive policies is invalid, so avoid doing that. */
1501         if (unlikely(policy_is_inactive(policy)))
1502                 return ret_freq;
1503
1504         if (ret_freq && policy->cur &&
1505                 !(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
1506                 /* verify no discrepancy between actual and
1507                                         saved value exists */
1508                 if (unlikely(ret_freq != policy->cur)) {
1509                         cpufreq_out_of_sync(policy, ret_freq);
1510                         schedule_work(&policy->update);
1511                 }
1512         }
1513
1514         return ret_freq;
1515 }
1516
1517 /**
1518  * cpufreq_get - get the current CPU frequency (in kHz)
1519  * @cpu: CPU number
1520  *
1521  * Get the CPU current (static) CPU frequency
1522  */
1523 unsigned int cpufreq_get(unsigned int cpu)
1524 {
1525         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1526         unsigned int ret_freq = 0;
1527
1528         if (policy) {
1529                 down_read(&policy->rwsem);
1530                 ret_freq = __cpufreq_get(policy);
1531                 up_read(&policy->rwsem);
1532
1533                 cpufreq_cpu_put(policy);
1534         }
1535
1536         return ret_freq;
1537 }
1538 EXPORT_SYMBOL(cpufreq_get);
1539
1540 static struct subsys_interface cpufreq_interface = {
1541         .name           = "cpufreq",
1542         .subsys         = &cpu_subsys,
1543         .add_dev        = cpufreq_add_dev,
1544         .remove_dev     = cpufreq_remove_dev,
1545 };
1546
1547 /*
1548  * In case platform wants some specific frequency to be configured
1549  * during suspend..
1550  */
1551 int cpufreq_generic_suspend(struct cpufreq_policy *policy)
1552 {
1553         int ret;
1554
1555         if (!policy->suspend_freq) {
1556                 pr_debug("%s: suspend_freq not defined\n", __func__);
1557                 return 0;
1558         }
1559
1560         pr_debug("%s: Setting suspend-freq: %u\n", __func__,
1561                         policy->suspend_freq);
1562
1563         ret = __cpufreq_driver_target(policy, policy->suspend_freq,
1564                         CPUFREQ_RELATION_H);
1565         if (ret)
1566                 pr_err("%s: unable to set suspend-freq: %u. err: %d\n",
1567                                 __func__, policy->suspend_freq, ret);
1568
1569         return ret;
1570 }
1571 EXPORT_SYMBOL(cpufreq_generic_suspend);
1572
1573 /**
1574  * cpufreq_suspend() - Suspend CPUFreq governors
1575  *
1576  * Called during system wide Suspend/Hibernate cycles for suspending governors
1577  * as some platforms can't change frequency after this point in suspend cycle.
1578  * Because some of the devices (like: i2c, regulators, etc) they use for
1579  * changing frequency are suspended quickly after this point.
1580  */
1581 void cpufreq_suspend(void)
1582 {
1583         struct cpufreq_policy *policy;
1584
1585         if (!cpufreq_driver)
1586                 return;
1587
1588         if (!has_target())
1589                 goto suspend;
1590
1591         pr_debug("%s: Suspending Governors\n", __func__);
1592
1593         for_each_active_policy(policy) {
1594                 if (__cpufreq_governor(policy, CPUFREQ_GOV_STOP))
1595                         pr_err("%s: Failed to stop governor for policy: %p\n",
1596                                 __func__, policy);
1597                 else if (cpufreq_driver->suspend
1598                     && cpufreq_driver->suspend(policy))
1599                         pr_err("%s: Failed to suspend driver: %p\n", __func__,
1600                                 policy);
1601         }
1602
1603 suspend:
1604         cpufreq_suspended = true;
1605 }
1606
1607 /**
1608  * cpufreq_resume() - Resume CPUFreq governors
1609  *
1610  * Called during system wide Suspend/Hibernate cycle for resuming governors that
1611  * are suspended with cpufreq_suspend().
1612  */
1613 void cpufreq_resume(void)
1614 {
1615         struct cpufreq_policy *policy;
1616
1617         if (!cpufreq_driver)
1618                 return;
1619
1620         cpufreq_suspended = false;
1621
1622         if (!has_target())
1623                 return;
1624
1625         pr_debug("%s: Resuming Governors\n", __func__);
1626
1627         for_each_active_policy(policy) {
1628                 if (cpufreq_driver->resume && cpufreq_driver->resume(policy))
1629                         pr_err("%s: Failed to resume driver: %p\n", __func__,
1630                                 policy);
1631                 else if (__cpufreq_governor(policy, CPUFREQ_GOV_START)
1632                     || __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS))
1633                         pr_err("%s: Failed to start governor for policy: %p\n",
1634                                 __func__, policy);
1635         }
1636
1637         /*
1638          * schedule call cpufreq_update_policy() for first-online CPU, as that
1639          * wouldn't be hotplugged-out on suspend. It will verify that the
1640          * current freq is in sync with what we believe it to be.
1641          */
1642         policy = cpufreq_cpu_get_raw(cpumask_first(cpu_online_mask));
1643         if (WARN_ON(!policy))
1644                 return;
1645
1646         schedule_work(&policy->update);
1647 }
1648
1649 /**
1650  *      cpufreq_get_current_driver - return current driver's name
1651  *
1652  *      Return the name string of the currently loaded cpufreq driver
1653  *      or NULL, if none.
1654  */
1655 const char *cpufreq_get_current_driver(void)
1656 {
1657         if (cpufreq_driver)
1658                 return cpufreq_driver->name;
1659
1660         return NULL;
1661 }
1662 EXPORT_SYMBOL_GPL(cpufreq_get_current_driver);
1663
1664 /**
1665  *      cpufreq_get_driver_data - return current driver data
1666  *
1667  *      Return the private data of the currently loaded cpufreq
1668  *      driver, or NULL if no cpufreq driver is loaded.
1669  */
1670 void *cpufreq_get_driver_data(void)
1671 {
1672         if (cpufreq_driver)
1673                 return cpufreq_driver->driver_data;
1674
1675         return NULL;
1676 }
1677 EXPORT_SYMBOL_GPL(cpufreq_get_driver_data);
1678
1679 /*********************************************************************
1680  *                     NOTIFIER LISTS INTERFACE                      *
1681  *********************************************************************/
1682
1683 /**
1684  *      cpufreq_register_notifier - register a driver with cpufreq
1685  *      @nb: notifier function to register
1686  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1687  *
1688  *      Add a driver to one of two lists: either a list of drivers that
1689  *      are notified about clock rate changes (once before and once after
1690  *      the transition), or a list of drivers that are notified about
1691  *      changes in cpufreq policy.
1692  *
1693  *      This function may sleep, and has the same return conditions as
1694  *      blocking_notifier_chain_register.
1695  */
1696 int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list)
1697 {
1698         int ret;
1699
1700         if (cpufreq_disabled())
1701                 return -EINVAL;
1702
1703         WARN_ON(!init_cpufreq_transition_notifier_list_called);
1704
1705         switch (list) {
1706         case CPUFREQ_TRANSITION_NOTIFIER:
1707                 ret = srcu_notifier_chain_register(
1708                                 &cpufreq_transition_notifier_list, nb);
1709                 break;
1710         case CPUFREQ_POLICY_NOTIFIER:
1711                 ret = blocking_notifier_chain_register(
1712                                 &cpufreq_policy_notifier_list, nb);
1713                 break;
1714         default:
1715                 ret = -EINVAL;
1716         }
1717
1718         return ret;
1719 }
1720 EXPORT_SYMBOL(cpufreq_register_notifier);
1721
1722 /**
1723  *      cpufreq_unregister_notifier - unregister a driver with cpufreq
1724  *      @nb: notifier block to be unregistered
1725  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1726  *
1727  *      Remove a driver from the CPU frequency notifier list.
1728  *
1729  *      This function may sleep, and has the same return conditions as
1730  *      blocking_notifier_chain_unregister.
1731  */
1732 int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list)
1733 {
1734         int ret;
1735
1736         if (cpufreq_disabled())
1737                 return -EINVAL;
1738
1739         switch (list) {
1740         case CPUFREQ_TRANSITION_NOTIFIER:
1741                 ret = srcu_notifier_chain_unregister(
1742                                 &cpufreq_transition_notifier_list, nb);
1743                 break;
1744         case CPUFREQ_POLICY_NOTIFIER:
1745                 ret = blocking_notifier_chain_unregister(
1746                                 &cpufreq_policy_notifier_list, nb);
1747                 break;
1748         default:
1749                 ret = -EINVAL;
1750         }
1751
1752         return ret;
1753 }
1754 EXPORT_SYMBOL(cpufreq_unregister_notifier);
1755
1756
1757 /*********************************************************************
1758  *                              GOVERNORS                            *
1759  *********************************************************************/
1760
1761 /* Must set freqs->new to intermediate frequency */
1762 static int __target_intermediate(struct cpufreq_policy *policy,
1763                                  struct cpufreq_freqs *freqs, int index)
1764 {
1765         int ret;
1766
1767         freqs->new = cpufreq_driver->get_intermediate(policy, index);
1768
1769         /* We don't need to switch to intermediate freq */
1770         if (!freqs->new)
1771                 return 0;
1772
1773         pr_debug("%s: cpu: %d, switching to intermediate freq: oldfreq: %u, intermediate freq: %u\n",
1774                  __func__, policy->cpu, freqs->old, freqs->new);
1775
1776         cpufreq_freq_transition_begin(policy, freqs);
1777         ret = cpufreq_driver->target_intermediate(policy, index);
1778         cpufreq_freq_transition_end(policy, freqs, ret);
1779
1780         if (ret)
1781                 pr_err("%s: Failed to change to intermediate frequency: %d\n",
1782                        __func__, ret);
1783
1784         return ret;
1785 }
1786
1787 static int __target_index(struct cpufreq_policy *policy,
1788                           struct cpufreq_frequency_table *freq_table, int index)
1789 {
1790         struct cpufreq_freqs freqs = {.old = policy->cur, .flags = 0};
1791         unsigned int intermediate_freq = 0;
1792         int retval = -EINVAL;
1793         bool notify;
1794
1795         notify = !(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION);
1796         if (notify) {
1797                 /* Handle switching to intermediate frequency */
1798                 if (cpufreq_driver->get_intermediate) {
1799                         retval = __target_intermediate(policy, &freqs, index);
1800                         if (retval)
1801                                 return retval;
1802
1803                         intermediate_freq = freqs.new;
1804                         /* Set old freq to intermediate */
1805                         if (intermediate_freq)
1806                                 freqs.old = freqs.new;
1807                 }
1808
1809                 freqs.new = freq_table[index].frequency;
1810                 pr_debug("%s: cpu: %d, oldfreq: %u, new freq: %u\n",
1811                          __func__, policy->cpu, freqs.old, freqs.new);
1812
1813                 cpufreq_freq_transition_begin(policy, &freqs);
1814         }
1815
1816         retval = cpufreq_driver->target_index(policy, index);
1817         if (retval)
1818                 pr_err("%s: Failed to change cpu frequency: %d\n", __func__,
1819                        retval);
1820
1821         if (notify) {
1822                 cpufreq_freq_transition_end(policy, &freqs, retval);
1823
1824                 /*
1825                  * Failed after setting to intermediate freq? Driver should have
1826                  * reverted back to initial frequency and so should we. Check
1827                  * here for intermediate_freq instead of get_intermediate, in
1828                  * case we haven't switched to intermediate freq at all.
1829                  */
1830                 if (unlikely(retval && intermediate_freq)) {
1831                         freqs.old = intermediate_freq;
1832                         freqs.new = policy->restore_freq;
1833                         cpufreq_freq_transition_begin(policy, &freqs);
1834                         cpufreq_freq_transition_end(policy, &freqs, 0);
1835                 }
1836         }
1837
1838         return retval;
1839 }
1840
1841 int __cpufreq_driver_target(struct cpufreq_policy *policy,
1842                             unsigned int target_freq,
1843                             unsigned int relation)
1844 {
1845         unsigned int old_target_freq = target_freq;
1846         int retval = -EINVAL;
1847
1848         if (cpufreq_disabled())
1849                 return -ENODEV;
1850
1851         /* Make sure that target_freq is within supported range */
1852         if (target_freq > policy->max)
1853                 target_freq = policy->max;
1854         if (target_freq < policy->min)
1855                 target_freq = policy->min;
1856
1857         pr_debug("target for CPU %u: %u kHz, relation %u, requested %u kHz\n",
1858                  policy->cpu, target_freq, relation, old_target_freq);
1859
1860         /*
1861          * This might look like a redundant call as we are checking it again
1862          * after finding index. But it is left intentionally for cases where
1863          * exactly same freq is called again and so we can save on few function
1864          * calls.
1865          */
1866         if (target_freq == policy->cur)
1867                 return 0;
1868
1869         /* Save last value to restore later on errors */
1870         policy->restore_freq = policy->cur;
1871
1872         if (cpufreq_driver->target)
1873                 retval = cpufreq_driver->target(policy, target_freq, relation);
1874         else if (cpufreq_driver->target_index) {
1875                 struct cpufreq_frequency_table *freq_table;
1876                 int index;
1877
1878                 freq_table = cpufreq_frequency_get_table(policy->cpu);
1879                 if (unlikely(!freq_table)) {
1880                         pr_err("%s: Unable to find freq_table\n", __func__);
1881                         goto out;
1882                 }
1883
1884                 retval = cpufreq_frequency_table_target(policy, freq_table,
1885                                 target_freq, relation, &index);
1886                 if (unlikely(retval)) {
1887                         pr_err("%s: Unable to find matching freq\n", __func__);
1888                         goto out;
1889                 }
1890
1891                 if (freq_table[index].frequency == policy->cur) {
1892                         retval = 0;
1893                         goto out;
1894                 }
1895
1896                 retval = __target_index(policy, freq_table, index);
1897         }
1898
1899 out:
1900         return retval;
1901 }
1902 EXPORT_SYMBOL_GPL(__cpufreq_driver_target);
1903
1904 int cpufreq_driver_target(struct cpufreq_policy *policy,
1905                           unsigned int target_freq,
1906                           unsigned int relation)
1907 {
1908         int ret = -EINVAL;
1909
1910         down_write(&policy->rwsem);
1911
1912         ret = __cpufreq_driver_target(policy, target_freq, relation);
1913
1914         up_write(&policy->rwsem);
1915
1916         return ret;
1917 }
1918 EXPORT_SYMBOL_GPL(cpufreq_driver_target);
1919
1920 static int __cpufreq_governor(struct cpufreq_policy *policy,
1921                                         unsigned int event)
1922 {
1923         int ret;
1924
1925         /* Only must be defined when default governor is known to have latency
1926            restrictions, like e.g. conservative or ondemand.
1927            That this is the case is already ensured in Kconfig
1928         */
1929 #ifdef CONFIG_CPU_FREQ_GOV_PERFORMANCE
1930         struct cpufreq_governor *gov = &cpufreq_gov_performance;
1931 #else
1932         struct cpufreq_governor *gov = NULL;
1933 #endif
1934
1935         /* Don't start any governor operations if we are entering suspend */
1936         if (cpufreq_suspended)
1937                 return 0;
1938         /*
1939          * Governor might not be initiated here if ACPI _PPC changed
1940          * notification happened, so check it.
1941          */
1942         if (!policy->governor)
1943                 return -EINVAL;
1944
1945         if (policy->governor->max_transition_latency &&
1946             policy->cpuinfo.transition_latency >
1947             policy->governor->max_transition_latency) {
1948                 if (!gov)
1949                         return -EINVAL;
1950                 else {
1951                         pr_warn("%s governor failed, too long transition latency of HW, fallback to %s governor\n",
1952                                 policy->governor->name, gov->name);
1953                         policy->governor = gov;
1954                 }
1955         }
1956
1957         if (event == CPUFREQ_GOV_POLICY_INIT)
1958                 if (!try_module_get(policy->governor->owner))
1959                         return -EINVAL;
1960
1961         pr_debug("%s: for CPU %u, event %u\n", __func__, policy->cpu, event);
1962
1963         mutex_lock(&cpufreq_governor_lock);
1964         if ((policy->governor_enabled && event == CPUFREQ_GOV_START)
1965             || (!policy->governor_enabled
1966             && (event == CPUFREQ_GOV_LIMITS || event == CPUFREQ_GOV_STOP))) {
1967                 mutex_unlock(&cpufreq_governor_lock);
1968                 return -EBUSY;
1969         }
1970
1971         if (event == CPUFREQ_GOV_STOP)
1972                 policy->governor_enabled = false;
1973         else if (event == CPUFREQ_GOV_START)
1974                 policy->governor_enabled = true;
1975
1976         mutex_unlock(&cpufreq_governor_lock);
1977
1978         ret = policy->governor->governor(policy, event);
1979
1980         if (!ret) {
1981                 if (event == CPUFREQ_GOV_POLICY_INIT)
1982                         policy->governor->initialized++;
1983                 else if (event == CPUFREQ_GOV_POLICY_EXIT)
1984                         policy->governor->initialized--;
1985         } else {
1986                 /* Restore original values */
1987                 mutex_lock(&cpufreq_governor_lock);
1988                 if (event == CPUFREQ_GOV_STOP)
1989                         policy->governor_enabled = true;
1990                 else if (event == CPUFREQ_GOV_START)
1991                         policy->governor_enabled = false;
1992                 mutex_unlock(&cpufreq_governor_lock);
1993         }
1994
1995         if (((event == CPUFREQ_GOV_POLICY_INIT) && ret) ||
1996                         ((event == CPUFREQ_GOV_POLICY_EXIT) && !ret))
1997                 module_put(policy->governor->owner);
1998
1999         return ret;
2000 }
2001
2002 int cpufreq_register_governor(struct cpufreq_governor *governor)
2003 {
2004         int err;
2005
2006         if (!governor)
2007                 return -EINVAL;
2008
2009         if (cpufreq_disabled())
2010                 return -ENODEV;
2011
2012         mutex_lock(&cpufreq_governor_mutex);
2013
2014         governor->initialized = 0;
2015         err = -EBUSY;
2016         if (!find_governor(governor->name)) {
2017                 err = 0;
2018                 list_add(&governor->governor_list, &cpufreq_governor_list);
2019         }
2020
2021         mutex_unlock(&cpufreq_governor_mutex);
2022         return err;
2023 }
2024 EXPORT_SYMBOL_GPL(cpufreq_register_governor);
2025
2026 void cpufreq_unregister_governor(struct cpufreq_governor *governor)
2027 {
2028         struct cpufreq_policy *policy;
2029         unsigned long flags;
2030
2031         if (!governor)
2032                 return;
2033
2034         if (cpufreq_disabled())
2035                 return;
2036
2037         /* clear last_governor for all inactive policies */
2038         read_lock_irqsave(&cpufreq_driver_lock, flags);
2039         for_each_inactive_policy(policy) {
2040                 if (!strcmp(policy->last_governor, governor->name)) {
2041                         policy->governor = NULL;
2042                         strcpy(policy->last_governor, "\0");
2043                 }
2044         }
2045         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
2046
2047         mutex_lock(&cpufreq_governor_mutex);
2048         list_del(&governor->governor_list);
2049         mutex_unlock(&cpufreq_governor_mutex);
2050         return;
2051 }
2052 EXPORT_SYMBOL_GPL(cpufreq_unregister_governor);
2053
2054
2055 /*********************************************************************
2056  *                          POLICY INTERFACE                         *
2057  *********************************************************************/
2058
2059 /**
2060  * cpufreq_get_policy - get the current cpufreq_policy
2061  * @policy: struct cpufreq_policy into which the current cpufreq_policy
2062  *      is written
2063  *
2064  * Reads the current cpufreq policy.
2065  */
2066 int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu)
2067 {
2068         struct cpufreq_policy *cpu_policy;
2069         if (!policy)
2070                 return -EINVAL;
2071
2072         cpu_policy = cpufreq_cpu_get(cpu);
2073         if (!cpu_policy)
2074                 return -EINVAL;
2075
2076         memcpy(policy, cpu_policy, sizeof(*policy));
2077
2078         cpufreq_cpu_put(cpu_policy);
2079         return 0;
2080 }
2081 EXPORT_SYMBOL(cpufreq_get_policy);
2082
2083 /*
2084  * policy : current policy.
2085  * new_policy: policy to be set.
2086  */
2087 static int cpufreq_set_policy(struct cpufreq_policy *policy,
2088                                 struct cpufreq_policy *new_policy)
2089 {
2090         struct cpufreq_governor *old_gov;
2091         int ret;
2092
2093         pr_debug("setting new policy for CPU %u: %u - %u kHz\n",
2094                  new_policy->cpu, new_policy->min, new_policy->max);
2095
2096         memcpy(&new_policy->cpuinfo, &policy->cpuinfo, sizeof(policy->cpuinfo));
2097
2098         /*
2099         * This check works well when we store new min/max freq attributes,
2100         * because new_policy is a copy of policy with one field updated.
2101         */
2102         if (new_policy->min > new_policy->max)
2103                 return -EINVAL;
2104
2105         /* verify the cpu speed can be set within this limit */
2106         ret = cpufreq_driver->verify(new_policy);
2107         if (ret)
2108                 return ret;
2109
2110         /* adjust if necessary - all reasons */
2111         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
2112                         CPUFREQ_ADJUST, new_policy);
2113
2114         /*
2115          * verify the cpu speed can be set within this limit, which might be
2116          * different to the first one
2117          */
2118         ret = cpufreq_driver->verify(new_policy);
2119         if (ret)
2120                 return ret;
2121
2122         /* notification of the new policy */
2123         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
2124                         CPUFREQ_NOTIFY, new_policy);
2125
2126         policy->min = new_policy->min;
2127         policy->max = new_policy->max;
2128
2129         pr_debug("new min and max freqs are %u - %u kHz\n",
2130                  policy->min, policy->max);
2131
2132         if (cpufreq_driver->setpolicy) {
2133                 policy->policy = new_policy->policy;
2134                 pr_debug("setting range\n");
2135                 return cpufreq_driver->setpolicy(new_policy);
2136         }
2137
2138         if (new_policy->governor == policy->governor)
2139                 goto out;
2140
2141         pr_debug("governor switch\n");
2142
2143         /* save old, working values */
2144         old_gov = policy->governor;
2145         /* end old governor */
2146         if (old_gov) {
2147                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
2148                 if (ret) {
2149                         /* This can happen due to race with other operations */
2150                         pr_debug("%s: Failed to Stop Governor: %s (%d)\n",
2151                                  __func__, old_gov->name, ret);
2152                         return ret;
2153                 }
2154
2155                 up_write(&policy->rwsem);
2156                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT);
2157                 down_write(&policy->rwsem);
2158
2159                 if (ret) {
2160                         pr_err("%s: Failed to Exit Governor: %s (%d)\n",
2161                                __func__, old_gov->name, ret);
2162                         return ret;
2163                 }
2164         }
2165
2166         /* start new governor */
2167         policy->governor = new_policy->governor;
2168         ret = __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_INIT);
2169         if (!ret) {
2170                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_START);
2171                 if (!ret)
2172                         goto out;
2173
2174                 up_write(&policy->rwsem);
2175                 __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT);
2176                 down_write(&policy->rwsem);
2177         }
2178
2179         /* new governor failed, so re-start old one */
2180         pr_debug("starting governor %s failed\n", policy->governor->name);
2181         if (old_gov) {
2182                 policy->governor = old_gov;
2183                 if (__cpufreq_governor(policy, CPUFREQ_GOV_POLICY_INIT))
2184                         policy->governor = NULL;
2185                 else
2186                         __cpufreq_governor(policy, CPUFREQ_GOV_START);
2187         }
2188
2189         return ret;
2190
2191  out:
2192         pr_debug("governor: change or update limits\n");
2193         return __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
2194 }
2195
2196 /**
2197  *      cpufreq_update_policy - re-evaluate an existing cpufreq policy
2198  *      @cpu: CPU which shall be re-evaluated
2199  *
2200  *      Useful for policy notifiers which have different necessities
2201  *      at different times.
2202  */
2203 int cpufreq_update_policy(unsigned int cpu)
2204 {
2205         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
2206         struct cpufreq_policy new_policy;
2207         int ret;
2208
2209         if (!policy)
2210                 return -ENODEV;
2211
2212         down_write(&policy->rwsem);
2213
2214         pr_debug("updating policy for CPU %u\n", cpu);
2215         memcpy(&new_policy, policy, sizeof(*policy));
2216         new_policy.min = policy->user_policy.min;
2217         new_policy.max = policy->user_policy.max;
2218
2219         /*
2220          * BIOS might change freq behind our back
2221          * -> ask driver for current freq and notify governors about a change
2222          */
2223         if (cpufreq_driver->get && !cpufreq_driver->setpolicy) {
2224                 new_policy.cur = cpufreq_driver->get(cpu);
2225                 if (WARN_ON(!new_policy.cur)) {
2226                         ret = -EIO;
2227                         goto unlock;
2228                 }
2229
2230                 if (!policy->cur) {
2231                         pr_debug("Driver did not initialize current freq\n");
2232                         policy->cur = new_policy.cur;
2233                 } else {
2234                         if (policy->cur != new_policy.cur && has_target())
2235                                 cpufreq_out_of_sync(policy, new_policy.cur);
2236                 }
2237         }
2238
2239         ret = cpufreq_set_policy(policy, &new_policy);
2240
2241 unlock:
2242         up_write(&policy->rwsem);
2243
2244         cpufreq_cpu_put(policy);
2245         return ret;
2246 }
2247 EXPORT_SYMBOL(cpufreq_update_policy);
2248
2249 static int cpufreq_cpu_callback(struct notifier_block *nfb,
2250                                         unsigned long action, void *hcpu)
2251 {
2252         unsigned int cpu = (unsigned long)hcpu;
2253
2254         switch (action & ~CPU_TASKS_FROZEN) {
2255         case CPU_ONLINE:
2256                 cpufreq_online(cpu);
2257                 break;
2258
2259         case CPU_DOWN_PREPARE:
2260                 cpufreq_offline_prepare(cpu);
2261                 break;
2262
2263         case CPU_POST_DEAD:
2264                 cpufreq_offline_finish(cpu);
2265                 break;
2266
2267         case CPU_DOWN_FAILED:
2268                 cpufreq_online(cpu);
2269                 break;
2270         }
2271         return NOTIFY_OK;
2272 }
2273
2274 static struct notifier_block __refdata cpufreq_cpu_notifier = {
2275         .notifier_call = cpufreq_cpu_callback,
2276 };
2277
2278 /*********************************************************************
2279  *               BOOST                                               *
2280  *********************************************************************/
2281 static int cpufreq_boost_set_sw(int state)
2282 {
2283         struct cpufreq_frequency_table *freq_table;
2284         struct cpufreq_policy *policy;
2285         int ret = -EINVAL;
2286
2287         for_each_active_policy(policy) {
2288                 freq_table = cpufreq_frequency_get_table(policy->cpu);
2289                 if (freq_table) {
2290                         ret = cpufreq_frequency_table_cpuinfo(policy,
2291                                                         freq_table);
2292                         if (ret) {
2293                                 pr_err("%s: Policy frequency update failed\n",
2294                                        __func__);
2295                                 break;
2296                         }
2297                         policy->user_policy.max = policy->max;
2298                         __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
2299                 }
2300         }
2301
2302         return ret;
2303 }
2304
2305 int cpufreq_boost_trigger_state(int state)
2306 {
2307         unsigned long flags;
2308         int ret = 0;
2309
2310         if (cpufreq_driver->boost_enabled == state)
2311                 return 0;
2312
2313         write_lock_irqsave(&cpufreq_driver_lock, flags);
2314         cpufreq_driver->boost_enabled = state;
2315         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2316
2317         ret = cpufreq_driver->set_boost(state);
2318         if (ret) {
2319                 write_lock_irqsave(&cpufreq_driver_lock, flags);
2320                 cpufreq_driver->boost_enabled = !state;
2321                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2322
2323                 pr_err("%s: Cannot %s BOOST\n",
2324                        __func__, state ? "enable" : "disable");
2325         }
2326
2327         return ret;
2328 }
2329
2330 int cpufreq_boost_supported(void)
2331 {
2332         if (likely(cpufreq_driver))
2333                 return cpufreq_driver->boost_supported;
2334
2335         return 0;
2336 }
2337 EXPORT_SYMBOL_GPL(cpufreq_boost_supported);
2338
2339 static int create_boost_sysfs_file(void)
2340 {
2341         int ret;
2342
2343         if (!cpufreq_boost_supported())
2344                 return 0;
2345
2346         /*
2347          * Check if driver provides function to enable boost -
2348          * if not, use cpufreq_boost_set_sw as default
2349          */
2350         if (!cpufreq_driver->set_boost)
2351                 cpufreq_driver->set_boost = cpufreq_boost_set_sw;
2352
2353         ret = sysfs_create_file(cpufreq_global_kobject, &boost.attr);
2354         if (ret)
2355                 pr_err("%s: cannot register global BOOST sysfs file\n",
2356                        __func__);
2357
2358         return ret;
2359 }
2360
2361 static void remove_boost_sysfs_file(void)
2362 {
2363         if (cpufreq_boost_supported())
2364                 sysfs_remove_file(cpufreq_global_kobject, &boost.attr);
2365 }
2366
2367 int cpufreq_enable_boost_support(void)
2368 {
2369         if (!cpufreq_driver)
2370                 return -EINVAL;
2371
2372         if (cpufreq_boost_supported())
2373                 return 0;
2374
2375         cpufreq_driver->boost_supported = true;
2376
2377         /* This will get removed on driver unregister */
2378         return create_boost_sysfs_file();
2379 }
2380 EXPORT_SYMBOL_GPL(cpufreq_enable_boost_support);
2381
2382 int cpufreq_boost_enabled(void)
2383 {
2384         return cpufreq_driver->boost_enabled;
2385 }
2386 EXPORT_SYMBOL_GPL(cpufreq_boost_enabled);
2387
2388 /*********************************************************************
2389  *               REGISTER / UNREGISTER CPUFREQ DRIVER                *
2390  *********************************************************************/
2391
2392 /**
2393  * cpufreq_register_driver - register a CPU Frequency driver
2394  * @driver_data: A struct cpufreq_driver containing the values#
2395  * submitted by the CPU Frequency driver.
2396  *
2397  * Registers a CPU Frequency driver to this core code. This code
2398  * returns zero on success, -EBUSY when another driver got here first
2399  * (and isn't unregistered in the meantime).
2400  *
2401  */
2402 int cpufreq_register_driver(struct cpufreq_driver *driver_data)
2403 {
2404         unsigned long flags;
2405         int ret;
2406
2407         if (cpufreq_disabled())
2408                 return -ENODEV;
2409
2410         if (!driver_data || !driver_data->verify || !driver_data->init ||
2411             !(driver_data->setpolicy || driver_data->target_index ||
2412                     driver_data->target) ||
2413              (driver_data->setpolicy && (driver_data->target_index ||
2414                     driver_data->target)) ||
2415              (!!driver_data->get_intermediate != !!driver_data->target_intermediate))
2416                 return -EINVAL;
2417
2418         pr_debug("trying to register driver %s\n", driver_data->name);
2419
2420         /* Protect against concurrent CPU online/offline. */
2421         get_online_cpus();
2422
2423         write_lock_irqsave(&cpufreq_driver_lock, flags);
2424         if (cpufreq_driver) {
2425                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2426                 ret = -EEXIST;
2427                 goto out;
2428         }
2429         cpufreq_driver = driver_data;
2430         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2431
2432         if (driver_data->setpolicy)
2433                 driver_data->flags |= CPUFREQ_CONST_LOOPS;
2434
2435         ret = create_boost_sysfs_file();
2436         if (ret)
2437                 goto err_null_driver;
2438
2439         ret = subsys_interface_register(&cpufreq_interface);
2440         if (ret)
2441                 goto err_boost_unreg;
2442
2443         if (!(cpufreq_driver->flags & CPUFREQ_STICKY) &&
2444             list_empty(&cpufreq_policy_list)) {
2445                 /* if all ->init() calls failed, unregister */
2446                 pr_debug("%s: No CPU initialized for driver %s\n", __func__,
2447                          driver_data->name);
2448                 goto err_if_unreg;
2449         }
2450
2451         register_hotcpu_notifier(&cpufreq_cpu_notifier);
2452         pr_debug("driver %s up and running\n", driver_data->name);
2453
2454 out:
2455         put_online_cpus();
2456         return ret;
2457
2458 err_if_unreg:
2459         subsys_interface_unregister(&cpufreq_interface);
2460 err_boost_unreg:
2461         remove_boost_sysfs_file();
2462 err_null_driver:
2463         write_lock_irqsave(&cpufreq_driver_lock, flags);
2464         cpufreq_driver = NULL;
2465         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2466         goto out;
2467 }
2468 EXPORT_SYMBOL_GPL(cpufreq_register_driver);
2469
2470 /**
2471  * cpufreq_unregister_driver - unregister the current CPUFreq driver
2472  *
2473  * Unregister the current CPUFreq driver. Only call this if you have
2474  * the right to do so, i.e. if you have succeeded in initialising before!
2475  * Returns zero if successful, and -EINVAL if the cpufreq_driver is
2476  * currently not initialised.
2477  */
2478 int cpufreq_unregister_driver(struct cpufreq_driver *driver)
2479 {
2480         unsigned long flags;
2481
2482         if (!cpufreq_driver || (driver != cpufreq_driver))
2483                 return -EINVAL;
2484
2485         pr_debug("unregistering driver %s\n", driver->name);
2486
2487         /* Protect against concurrent cpu hotplug */
2488         get_online_cpus();
2489         subsys_interface_unregister(&cpufreq_interface);
2490         remove_boost_sysfs_file();
2491         unregister_hotcpu_notifier(&cpufreq_cpu_notifier);
2492
2493         write_lock_irqsave(&cpufreq_driver_lock, flags);
2494
2495         cpufreq_driver = NULL;
2496
2497         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2498         put_online_cpus();
2499
2500         return 0;
2501 }
2502 EXPORT_SYMBOL_GPL(cpufreq_unregister_driver);
2503
2504 /*
2505  * Stop cpufreq at shutdown to make sure it isn't holding any locks
2506  * or mutexes when secondary CPUs are halted.
2507  */
2508 static struct syscore_ops cpufreq_syscore_ops = {
2509         .shutdown = cpufreq_suspend,
2510 };
2511
2512 struct kobject *cpufreq_global_kobject;
2513 EXPORT_SYMBOL(cpufreq_global_kobject);
2514
2515 static int __init cpufreq_core_init(void)
2516 {
2517         if (cpufreq_disabled())
2518                 return -ENODEV;
2519
2520         cpufreq_global_kobject = kobject_create_and_add("cpufreq", &cpu_subsys.dev_root->kobj);
2521         BUG_ON(!cpufreq_global_kobject);
2522
2523         register_syscore_ops(&cpufreq_syscore_ops);
2524
2525         return 0;
2526 }
2527 core_initcall(cpufreq_core_init);