drivers/cpufreq/powernv-cpufreq.c

   1 /*
   2  * POWERNV cpufreq driver for the IBM POWER processors
   3  *
   4  * (C) Copyright IBM 2014
   5  *
   6  * Author: Vaidyanathan Srinivasan <svaidy at linux.vnet.ibm.com>
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License as published by
  10  * the Free Software Foundation; either version 2, or (at your option)
  11  * any later version.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16  * GNU General Public License for more details.
  17  *
  18  */
  19
  20 #define pr_fmt(fmt)     "powernv-cpufreq: " fmt
  21
  22 #include <linux/kernel.h>
  23 #include <linux/sysfs.h>
  24 #include <linux/cpumask.h>
  25 #include <linux/module.h>
  26 #include <linux/cpufreq.h>
  27 #include <linux/smp.h>
  28 #include <linux/of.h>
  29 #include <linux/reboot.h>
  30 #include <linux/slab.h>
  31 #include <linux/cpu.h>
  32 #include <trace/events/power.h>
  33
  34 #include <asm/cputhreads.h>
  35 #include <asm/firmware.h>
  36 #include <asm/reg.h>
  37 #include <asm/smp.h> /* Required for cpu_sibling_mask() in UP configs */
  38 #include <asm/opal.h>
  39 #include <linux/timer.h>
  40
  41 #define POWERNV_MAX_PSTATES     256
  42 #define PMSR_PSAFE_ENABLE       (1UL << 30)
  43 #define PMSR_SPR_EM_DISABLE     (1UL << 31)
  44 #define PMSR_MAX(x)             ((x >> 32) & 0xFF)
  45
  46 #define MAX_RAMP_DOWN_TIME                              5120
  47 /*
  48  * On an idle system we want the global pstate to ramp-down from max value to
  49  * min over a span of ~5 secs. Also we want it to initially ramp-down slowly and
  50  * then ramp-down rapidly later on.
  51  *
  52  * This gives a percentage rampdown for time elapsed in milliseconds.
  53  * ramp_down_percentage = ((ms * ms) >> 18)
  54  *                      ~= 3.8 * (sec * sec)
  55  *
  56  * At 0 ms      ramp_down_percent = 0
  57  * At 5120 ms   ramp_down_percent = 100
  58  */
  59 #define ramp_down_percent(time)         ((time * time) >> 18)
  60
  61 /* Interval after which the timer is queued to bring down global pstate */
  62 #define GPSTATE_TIMER_INTERVAL                          2000
  63
  64 /**
  65  * struct global_pstate_info -  Per policy data structure to maintain history of
  66  *                              global pstates
  67  * @highest_lpstate:            The local pstate from which we are ramping down
  68  * @elapsed_time:               Time in ms spent in ramping down from
  69  *                              highest_lpstate
  70  * @last_sampled_time:          Time from boot in ms when global pstates were
  71  *                              last set
  72  * @last_lpstate,last_gpstate:  Last set values for local and global pstates
  73  * @timer:                      Is used for ramping down if cpu goes idle for
  74  *                              a long time with global pstate held high
  75  * @gpstate_lock:               A spinlock to maintain synchronization between
  76  *                              routines called by the timer handler and
  77  *                              governer's target_index calls
  78  */
  79 struct global_pstate_info {
  80         int highest_lpstate;
  81         unsigned int elapsed_time;
  82         unsigned int last_sampled_time;
  83         int last_lpstate;
  84         int last_gpstate;
  85         spinlock_t gpstate_lock;
  86         struct timer_list timer;
  87 };
  88
  89 static struct cpufreq_frequency_table powernv_freqs[POWERNV_MAX_PSTATES+1];
  90 static bool rebooting, throttled, occ_reset;
  91
  92 static const char * const throttle_reason[] = {
  93         "No throttling",
  94         "Power Cap",
  95         "Processor Over Temperature",
  96         "Power Supply Failure",
  97         "Over Current",
  98         "OCC Reset"
  99 };
 100
 101 enum throttle_reason_type {
 102         NO_THROTTLE = 0,
 103         POWERCAP,
 104         CPU_OVERTEMP,
 105         POWER_SUPPLY_FAILURE,
 106         OVERCURRENT,
 107         OCC_RESET_THROTTLE,
 108         OCC_MAX_REASON
 109 };
 110
 111 static struct chip {
 112         unsigned int id;
 113         bool throttled;
 114         bool restore;
 115         u8 throttle_reason;
 116         cpumask_t mask;
 117         struct work_struct throttle;
 118         int throttle_turbo;
 119         int throttle_sub_turbo;
 120         int reason[OCC_MAX_REASON];
 121 } *chips;
 122
 123 static int nr_chips;
 124 static DEFINE_PER_CPU(struct chip *, chip_info);
 125
 126 /*
 127  * Note: The set of pstates consists of contiguous integers, the
 128  * smallest of which is indicated by powernv_pstate_info.min, the
 129  * largest of which is indicated by powernv_pstate_info.max.
 130  *
 131  * The nominal pstate is the highest non-turbo pstate in this
 132  * platform. This is indicated by powernv_pstate_info.nominal.
 133  */
 134 static struct powernv_pstate_info {
 135         int min;
 136         int max;
 137         int nominal;
 138         int nr_pstates;
 139 } powernv_pstate_info;
 140
 141 static inline void reset_gpstates(struct cpufreq_policy *policy)
 142 {
 143         struct global_pstate_info *gpstates = policy->driver_data;
 144
 145         gpstates->highest_lpstate = 0;
 146         gpstates->elapsed_time = 0;
 147         gpstates->last_sampled_time = 0;
 148         gpstates->last_lpstate = 0;
 149         gpstates->last_gpstate = 0;
 150 }
 151
 152 /*
 153  * Initialize the freq table based on data obtained
 154  * from the firmware passed via device-tree
 155  */
 156 static int init_powernv_pstates(void)
 157 {
 158         struct device_node *power_mgt;
 159         int i, pstate_min, pstate_max, pstate_nominal, nr_pstates = 0;
 160         const __be32 *pstate_ids, *pstate_freqs;
 161         u32 len_ids, len_freqs;
 162
 163         power_mgt = of_find_node_by_path("/ibm,opal/power-mgt");
 164         if (!power_mgt) {
 165                 pr_warn("power-mgt node not found\n");
 166                 return -ENODEV;
 167         }
 168
 169         if (of_property_read_u32(power_mgt, "ibm,pstate-min", &pstate_min)) {
 170                 pr_warn("ibm,pstate-min node not found\n");
 171                 return -ENODEV;
 172         }
 173
 174         if (of_property_read_u32(power_mgt, "ibm,pstate-max", &pstate_max)) {
 175                 pr_warn("ibm,pstate-max node not found\n");
 176                 return -ENODEV;
 177         }
 178
 179         if (of_property_read_u32(power_mgt, "ibm,pstate-nominal",
 180                                  &pstate_nominal)) {
 181                 pr_warn("ibm,pstate-nominal not found\n");
 182                 return -ENODEV;
 183         }
 184         pr_info("cpufreq pstate min %d nominal %d max %d\n", pstate_min,
 185                 pstate_nominal, pstate_max);
 186
 187         pstate_ids = of_get_property(power_mgt, "ibm,pstate-ids", &len_ids);
 188         if (!pstate_ids) {
 189                 pr_warn("ibm,pstate-ids not found\n");
 190                 return -ENODEV;
 191         }
 192
 193         pstate_freqs = of_get_property(power_mgt, "ibm,pstate-frequencies-mhz",
 194                                       &len_freqs);
 195         if (!pstate_freqs) {
 196                 pr_warn("ibm,pstate-frequencies-mhz not found\n");
 197                 return -ENODEV;
 198         }
 199
 200         if (len_ids != len_freqs) {
 201                 pr_warn("Entries in ibm,pstate-ids and "
 202                         "ibm,pstate-frequencies-mhz does not match\n");
 203         }
 204
 205         nr_pstates = min(len_ids, len_freqs) / sizeof(u32);
 206         if (!nr_pstates) {
 207                 pr_warn("No PStates found\n");
 208                 return -ENODEV;
 209         }
 210
 211         pr_debug("NR PStates %d\n", nr_pstates);
 212         for (i = 0; i < nr_pstates; i++) {
 213                 u32 id = be32_to_cpu(pstate_ids[i]);
 214                 u32 freq = be32_to_cpu(pstate_freqs[i]);
 215
 216                 pr_debug("PState id %d freq %d MHz\n", id, freq);
 217                 powernv_freqs[i].frequency = freq * 1000; /* kHz */
 218                 powernv_freqs[i].driver_data = id;
 219         }
 220         /* End of list marker entry */
 221         powernv_freqs[i].frequency = CPUFREQ_TABLE_END;
 222
 223         powernv_pstate_info.min = pstate_min;
 224         powernv_pstate_info.max = pstate_max;
 225         powernv_pstate_info.nominal = pstate_nominal;
 226         powernv_pstate_info.nr_pstates = nr_pstates;
 227
 228         return 0;
 229 }
 230
 231 /* Returns the CPU frequency corresponding to the pstate_id. */
 232 static unsigned int pstate_id_to_freq(int pstate_id)
 233 {
 234         int i;
 235
 236         i = powernv_pstate_info.max - pstate_id;
 237         if (i >= powernv_pstate_info.nr_pstates || i < 0) {
 238                 pr_warn("PState id %d outside of PState table, "
 239                         "reporting nominal id %d instead\n",
 240                         pstate_id, powernv_pstate_info.nominal);
 241                 i = powernv_pstate_info.max - powernv_pstate_info.nominal;
 242         }
 243
 244         return powernv_freqs[i].frequency;
 245 }
 246
 247 /*
 248  * cpuinfo_nominal_freq_show - Show the nominal CPU frequency as indicated by
 249  * the firmware
 250  */
 251 static ssize_t cpuinfo_nominal_freq_show(struct cpufreq_policy *policy,
 252                                         char *buf)
 253 {
 254         return sprintf(buf, "%u\n",
 255                 pstate_id_to_freq(powernv_pstate_info.nominal));
 256 }
 257
 258 struct freq_attr cpufreq_freq_attr_cpuinfo_nominal_freq =
 259         __ATTR_RO(cpuinfo_nominal_freq);
 260
 261 static struct freq_attr *powernv_cpu_freq_attr[] = {
 262         &cpufreq_freq_attr_scaling_available_freqs,
 263         &cpufreq_freq_attr_cpuinfo_nominal_freq,
 264         NULL,
 265 };
 266
 267 #define throttle_attr(name, member)                                     \
 268 static ssize_t name##_show(struct cpufreq_policy *policy, char *buf)    \
 269 {                                                                       \
 270         struct chip *chip = per_cpu(chip_info, policy->cpu);            \
 271                                                                         \
 272         return sprintf(buf, "%u\n", chip->member);                      \
 273 }                                                                       \
 274                                                                         \
 275 static struct freq_attr throttle_attr_##name = __ATTR_RO(name)          \
 276
 277 throttle_attr(unthrottle, reason[NO_THROTTLE]);
 278 throttle_attr(powercap, reason[POWERCAP]);
 279 throttle_attr(overtemp, reason[CPU_OVERTEMP]);
 280 throttle_attr(supply_fault, reason[POWER_SUPPLY_FAILURE]);
 281 throttle_attr(overcurrent, reason[OVERCURRENT]);
 282 throttle_attr(occ_reset, reason[OCC_RESET_THROTTLE]);
 283 throttle_attr(turbo_stat, throttle_turbo);
 284 throttle_attr(sub_turbo_stat, throttle_sub_turbo);
 285
 286 static struct attribute *throttle_attrs[] = {
 287         &throttle_attr_unthrottle.attr,
 288         &throttle_attr_powercap.attr,
 289         &throttle_attr_overtemp.attr,
 290         &throttle_attr_supply_fault.attr,
 291         &throttle_attr_overcurrent.attr,
 292         &throttle_attr_occ_reset.attr,
 293         &throttle_attr_turbo_stat.attr,
 294         &throttle_attr_sub_turbo_stat.attr,
 295         NULL,
 296 };
 297
 298 static const struct attribute_group throttle_attr_grp = {
 299         .name   = "throttle_stats",
 300         .attrs  = throttle_attrs,
 301 };
 302
 303 /* Helper routines */
 304
 305 /* Access helpers to power mgt SPR */
 306
 307 static inline unsigned long get_pmspr(unsigned long sprn)
 308 {
 309         switch (sprn) {
 310         case SPRN_PMCR:
 311                 return mfspr(SPRN_PMCR);
 312
 313         case SPRN_PMICR:
 314                 return mfspr(SPRN_PMICR);
 315
 316         case SPRN_PMSR:
 317                 return mfspr(SPRN_PMSR);
 318         }
 319         BUG();
 320 }
 321
 322 static inline void set_pmspr(unsigned long sprn, unsigned long val)
 323 {
 324         switch (sprn) {
 325         case SPRN_PMCR:
 326                 mtspr(SPRN_PMCR, val);
 327                 return;
 328
 329         case SPRN_PMICR:
 330                 mtspr(SPRN_PMICR, val);
 331                 return;
 332         }
 333         BUG();
 334 }
 335
 336 /*
 337  * Use objects of this type to query/update
 338  * pstates on a remote CPU via smp_call_function.
 339  */
 340 struct powernv_smp_call_data {
 341         unsigned int freq;
 342         int pstate_id;
 343         int gpstate_id;
 344 };
 345
 346 /*
 347  * powernv_read_cpu_freq: Reads the current frequency on this CPU.
 348  *
 349  * Called via smp_call_function.
 350  *
 351  * Note: The caller of the smp_call_function should pass an argument of
 352  * the type 'struct powernv_smp_call_data *' along with this function.
 353  *
 354  * The current frequency on this CPU will be returned via
 355  * ((struct powernv_smp_call_data *)arg)->freq;
 356  */
 357 static void powernv_read_cpu_freq(void *arg)
 358 {
 359         unsigned long pmspr_val;
 360         s8 local_pstate_id;
 361         struct powernv_smp_call_data *freq_data = arg;
 362
 363         pmspr_val = get_pmspr(SPRN_PMSR);
 364
 365         /*
 366          * The local pstate id corresponds bits 48..55 in the PMSR.
 367          * Note: Watch out for the sign!
 368          */
 369         local_pstate_id = (pmspr_val >> 48) & 0xFF;
 370         freq_data->pstate_id = local_pstate_id;
 371         freq_data->freq = pstate_id_to_freq(freq_data->pstate_id);
 372
 373         pr_debug("cpu %d pmsr %016lX pstate_id %d frequency %d kHz\n",
 374                 raw_smp_processor_id(), pmspr_val, freq_data->pstate_id,
 375                 freq_data->freq);
 376 }
 377
 378 /*
 379  * powernv_cpufreq_get: Returns the CPU frequency as reported by the
 380  * firmware for CPU 'cpu'. This value is reported through the sysfs
 381  * file cpuinfo_cur_freq.
 382  */
 383 static unsigned int powernv_cpufreq_get(unsigned int cpu)
 384 {
 385         struct powernv_smp_call_data freq_data;
 386
 387         smp_call_function_any(cpu_sibling_mask(cpu), powernv_read_cpu_freq,
 388                         &freq_data, 1);
 389
 390         return freq_data.freq;
 391 }
 392
 393 /*
 394  * set_pstate: Sets the pstate on this CPU.
 395  *
 396  * This is called via an smp_call_function.
 397  *
 398  * The caller must ensure that freq_data is of the type
 399  * (struct powernv_smp_call_data *) and the pstate_id which needs to be set
 400  * on this CPU should be present in freq_data->pstate_id.
 401  */
 402 static void set_pstate(void *data)
 403 {
 404         unsigned long val;
 405         struct powernv_smp_call_data *freq_data = data;
 406         unsigned long pstate_ul = freq_data->pstate_id;
 407         unsigned long gpstate_ul = freq_data->gpstate_id;
 408
 409         val = get_pmspr(SPRN_PMCR);
 410         val = val & 0x0000FFFFFFFFFFFFULL;
 411
 412         pstate_ul = pstate_ul & 0xFF;
 413         gpstate_ul = gpstate_ul & 0xFF;
 414
 415         /* Set both global(bits 56..63) and local(bits 48..55) PStates */
 416         val = val | (gpstate_ul << 56) | (pstate_ul << 48);
 417
 418         pr_debug("Setting cpu %d pmcr to %016lX\n",
 419                         raw_smp_processor_id(), val);
 420         set_pmspr(SPRN_PMCR, val);
 421 }
 422
 423 /*
 424  * get_nominal_index: Returns the index corresponding to the nominal
 425  * pstate in the cpufreq table
 426  */
 427 static inline unsigned int get_nominal_index(void)
 428 {
 429         return powernv_pstate_info.max - powernv_pstate_info.nominal;
 430 }
 431
 432 static void powernv_cpufreq_throttle_check(void *data)
 433 {
 434         struct chip *chip;
 435         unsigned int cpu = smp_processor_id();
 436         unsigned long pmsr;
 437         int pmsr_pmax;
 438
 439         pmsr = get_pmspr(SPRN_PMSR);
 440         chip = this_cpu_read(chip_info);
 441
 442         /* Check for Pmax Capping */
 443         pmsr_pmax = (s8)PMSR_MAX(pmsr);
 444         if (pmsr_pmax != powernv_pstate_info.max) {
 445                 if (chip->throttled)
 446                         goto next;
 447                 chip->throttled = true;
 448                 if (pmsr_pmax < powernv_pstate_info.nominal) {
 449                         pr_warn_once("CPU %d on Chip %u has Pmax reduced below nominal frequency (%d < %d)\n",
 450                                      cpu, chip->id, pmsr_pmax,
 451                                      powernv_pstate_info.nominal);
 452                         chip->throttle_sub_turbo++;
 453                 } else {
 454                         chip->throttle_turbo++;
 455                 }
 456                 trace_powernv_throttle(chip->id,
 457                                       throttle_reason[chip->throttle_reason],
 458                                       pmsr_pmax);
 459         } else if (chip->throttled) {
 460                 chip->throttled = false;
 461                 trace_powernv_throttle(chip->id,
 462                                       throttle_reason[chip->throttle_reason],
 463                                       pmsr_pmax);
 464         }
 465
 466         /* Check if Psafe_mode_active is set in PMSR. */
 467 next:
 468         if (pmsr & PMSR_PSAFE_ENABLE) {
 469                 throttled = true;
 470                 pr_info("Pstate set to safe frequency\n");
 471         }
 472
 473         /* Check if SPR_EM_DISABLE is set in PMSR */
 474         if (pmsr & PMSR_SPR_EM_DISABLE) {
 475                 throttled = true;
 476                 pr_info("Frequency Control disabled from OS\n");
 477         }
 478
 479         if (throttled) {
 480                 pr_info("PMSR = %16lx\n", pmsr);
 481                 pr_warn("CPU Frequency could be throttled\n");
 482         }
 483 }
 484
 485 /**
 486  * calc_global_pstate - Calculate global pstate
 487  * @elapsed_time:       Elapsed time in milliseconds
 488  * @local_pstate:       New local pstate
 489  * @highest_lpstate:    pstate from which its ramping down
 490  *
 491  * Finds the appropriate global pstate based on the pstate from which its
 492  * ramping down and the time elapsed in ramping down. It follows a quadratic
 493  * equation which ensures that it reaches ramping down to pmin in 5sec.
 494  */
 495 static inline int calc_global_pstate(unsigned int elapsed_time,
 496                                      int highest_lpstate, int local_pstate)
 497 {
 498         int pstate_diff;
 499
 500         /*
 501          * Using ramp_down_percent we get the percentage of rampdown
 502          * that we are expecting to be dropping. Difference between
 503          * highest_lpstate and powernv_pstate_info.min will give a absolute
 504          * number of how many pstates we will drop eventually by the end of
 505          * 5 seconds, then just scale it get the number pstates to be dropped.
 506          */
 507         pstate_diff =  ((int)ramp_down_percent(elapsed_time) *
 508                         (highest_lpstate - powernv_pstate_info.min)) / 100;
 509
 510         /* Ensure that global pstate is >= to local pstate */
 511         if (highest_lpstate - pstate_diff < local_pstate)
 512                 return local_pstate;
 513         else
 514                 return highest_lpstate - pstate_diff;
 515 }
 516
 517 static inline void  queue_gpstate_timer(struct global_pstate_info *gpstates)
 518 {
 519         unsigned int timer_interval;
 520
 521         /*
 522          * Setting up timer to fire after GPSTATE_TIMER_INTERVAL ms, But
 523          * if it exceeds MAX_RAMP_DOWN_TIME ms for ramp down time.
 524          * Set timer such that it fires exactly at MAX_RAMP_DOWN_TIME
 525          * seconds of ramp down time.
 526          */
 527         if ((gpstates->elapsed_time + GPSTATE_TIMER_INTERVAL)
 528              > MAX_RAMP_DOWN_TIME)
 529                 timer_interval = MAX_RAMP_DOWN_TIME - gpstates->elapsed_time;
 530         else
 531                 timer_interval = GPSTATE_TIMER_INTERVAL;
 532
 533         mod_timer(&gpstates->timer, jiffies + msecs_to_jiffies(timer_interval));
 534 }
 535
 536 /**
 537  * gpstate_timer_handler
 538  *
 539  * @data: pointer to cpufreq_policy on which timer was queued
 540  *
 541  * This handler brings down the global pstate closer to the local pstate
 542  * according quadratic equation. Queues a new timer if it is still not equal
 543  * to local pstate
 544  */
 545 void gpstate_timer_handler(unsigned long data)
 546 {
 547         struct cpufreq_policy *policy = (struct cpufreq_policy *)data;
 548         struct global_pstate_info *gpstates = policy->driver_data;
 549         int gpstate_id;
 550         unsigned int time_diff = jiffies_to_msecs(jiffies)
 551                                         - gpstates->last_sampled_time;
 552         struct powernv_smp_call_data freq_data;
 553
 554         if (!spin_trylock(&gpstates->gpstate_lock))
 555                 return;
 556
 557         gpstates->last_sampled_time += time_diff;
 558         gpstates->elapsed_time += time_diff;
 559         freq_data.pstate_id = gpstates->last_lpstate;
 560
 561         if ((gpstates->last_gpstate == freq_data.pstate_id) ||
 562             (gpstates->elapsed_time > MAX_RAMP_DOWN_TIME)) {
 563                 gpstate_id = freq_data.pstate_id;
 564                 reset_gpstates(policy);
 565                 gpstates->highest_lpstate = freq_data.pstate_id;
 566         } else {
 567                 gpstate_id = calc_global_pstate(gpstates->elapsed_time,
 568                                                 gpstates->highest_lpstate,
 569                                                 freq_data.pstate_id);
 570         }
 571
 572         /*
 573          * If local pstate is equal to global pstate, rampdown is over
 574          * So timer is not required to be queued.
 575          */
 576         if (gpstate_id != freq_data.pstate_id)
 577                 queue_gpstate_timer(gpstates);
 578
 579         freq_data.gpstate_id = gpstate_id;
 580         gpstates->last_gpstate = freq_data.gpstate_id;
 581         gpstates->last_lpstate = freq_data.pstate_id;
 582
 583         spin_unlock(&gpstates->gpstate_lock);
 584
 585         /* Timer may get migrated to a different cpu on cpu hot unplug */
 586         smp_call_function_any(policy->cpus, set_pstate, &freq_data, 1);
 587 }
 588
 589 /*
 590  * powernv_cpufreq_target_index: Sets the frequency corresponding to
 591  * the cpufreq table entry indexed by new_index on the cpus in the
 592  * mask policy->cpus
 593  */
 594 static int powernv_cpufreq_target_index(struct cpufreq_policy *policy,
 595                                         unsigned int new_index)
 596 {
 597         struct powernv_smp_call_data freq_data;
 598         unsigned int cur_msec, gpstate_id;
 599         struct global_pstate_info *gpstates = policy->driver_data;
 600
 601         if (unlikely(rebooting) && new_index != get_nominal_index())
 602                 return 0;
 603
 604         if (!throttled)
 605                 powernv_cpufreq_throttle_check(NULL);
 606
 607         cur_msec = jiffies_to_msecs(get_jiffies_64());
 608
 609         spin_lock(&gpstates->gpstate_lock);
 610         freq_data.pstate_id = powernv_freqs[new_index].driver_data;
 611
 612         if (!gpstates->last_sampled_time) {
 613                 gpstate_id = freq_data.pstate_id;
 614                 gpstates->highest_lpstate = freq_data.pstate_id;
 615                 goto gpstates_done;
 616         }
 617
 618         if (gpstates->last_gpstate > freq_data.pstate_id) {
 619                 gpstates->elapsed_time += cur_msec -
 620                                                  gpstates->last_sampled_time;
 621
 622                 /*
 623                  * If its has been ramping down for more than MAX_RAMP_DOWN_TIME
 624                  * we should be resetting all global pstate related data. Set it
 625                  * equal to local pstate to start fresh.
 626                  */
 627                 if (gpstates->elapsed_time > MAX_RAMP_DOWN_TIME) {
 628                         reset_gpstates(policy);
 629                         gpstates->highest_lpstate = freq_data.pstate_id;
 630                         gpstate_id = freq_data.pstate_id;
 631                 } else {
 632                 /* Elaspsed_time is less than 5 seconds, continue to rampdown */
 633                         gpstate_id = calc_global_pstate(gpstates->elapsed_time,
 634                                                         gpstates->highest_lpstate,
 635                                                         freq_data.pstate_id);
 636                 }
 637         } else {
 638                 reset_gpstates(policy);
 639                 gpstates->highest_lpstate = freq_data.pstate_id;
 640                 gpstate_id = freq_data.pstate_id;
 641         }
 642
 643         /*
 644          * If local pstate is equal to global pstate, rampdown is over
 645          * So timer is not required to be queued.
 646          */
 647         if (gpstate_id != freq_data.pstate_id)
 648                 queue_gpstate_timer(gpstates);
 649         else
 650                 del_timer_sync(&gpstates->timer);
 651
 652 gpstates_done:
 653         freq_data.gpstate_id = gpstate_id;
 654         gpstates->last_sampled_time = cur_msec;
 655         gpstates->last_gpstate = freq_data.gpstate_id;
 656         gpstates->last_lpstate = freq_data.pstate_id;
 657
 658         spin_unlock(&gpstates->gpstate_lock);
 659
 660         /*
 661          * Use smp_call_function to send IPI and execute the
 662          * mtspr on target CPU.  We could do that without IPI
 663          * if current CPU is within policy->cpus (core)
 664          */
 665         smp_call_function_any(policy->cpus, set_pstate, &freq_data, 1);
 666         return 0;
 667 }
 668
 669 static int powernv_cpufreq_cpu_init(struct cpufreq_policy *policy)
 670 {
 671         int base, i, ret;
 672         struct kernfs_node *kn;
 673         struct global_pstate_info *gpstates;
 674
 675         base = cpu_first_thread_sibling(policy->cpu);
 676
 677         for (i = 0; i < threads_per_core; i++)
 678                 cpumask_set_cpu(base + i, policy->cpus);
 679
 680         kn = kernfs_find_and_get(policy->kobj.sd, throttle_attr_grp.name);
 681         if (!kn) {
 682                 int ret;
 683
 684                 ret = sysfs_create_group(&policy->kobj, &throttle_attr_grp);
 685                 if (ret) {
 686                         pr_info("Failed to create throttle stats directory for cpu %d\n",
 687                                 policy->cpu);
 688                         return ret;
 689                 }
 690         } else {
 691                 kernfs_put(kn);
 692         }
 693
 694         gpstates =  kzalloc(sizeof(*gpstates), GFP_KERNEL);
 695         if (!gpstates)
 696                 return -ENOMEM;
 697
 698         policy->driver_data = gpstates;
 699
 700         /* initialize timer */
 701         init_timer_pinned_deferrable(&gpstates->timer);
 702         gpstates->timer.data = (unsigned long)policy;
 703         gpstates->timer.function = gpstate_timer_handler;
 704         gpstates->timer.expires = jiffies +
 705                                 msecs_to_jiffies(GPSTATE_TIMER_INTERVAL);
 706         spin_lock_init(&gpstates->gpstate_lock);
 707         ret = cpufreq_table_validate_and_show(policy, powernv_freqs);
 708
 709         if (ret < 0)
 710                 kfree(policy->driver_data);
 711
 712         return ret;
 713 }
 714
 715 static int powernv_cpufreq_cpu_exit(struct cpufreq_policy *policy)
 716 {
 717         /* timer is deleted in cpufreq_cpu_stop() */
 718         kfree(policy->driver_data);
 719
 720         return 0;
 721 }
 722
 723 static int powernv_cpufreq_reboot_notifier(struct notifier_block *nb,
 724                                 unsigned long action, void *unused)
 725 {
 726         int cpu;
 727         struct cpufreq_policy cpu_policy;
 728
 729         rebooting = true;
 730         for_each_online_cpu(cpu) {
 731                 cpufreq_get_policy(&cpu_policy, cpu);
 732                 powernv_cpufreq_target_index(&cpu_policy, get_nominal_index());
 733         }
 734
 735         return NOTIFY_DONE;
 736 }
 737
 738 static struct notifier_block powernv_cpufreq_reboot_nb = {
 739         .notifier_call = powernv_cpufreq_reboot_notifier,
 740 };
 741
 742 void powernv_cpufreq_work_fn(struct work_struct *work)
 743 {
 744         struct chip *chip = container_of(work, struct chip, throttle);
 745         unsigned int cpu;
 746         cpumask_t mask;
 747
 748         get_online_cpus();
 749         cpumask_and(&mask, &chip->mask, cpu_online_mask);
 750         smp_call_function_any(&mask,
 751                               powernv_cpufreq_throttle_check, NULL, 0);
 752
 753         if (!chip->restore)
 754                 goto out;
 755
 756         chip->restore = false;
 757         for_each_cpu(cpu, &mask) {
 758                 int index;
 759                 struct cpufreq_policy policy;
 760
 761                 cpufreq_get_policy(&policy, cpu);
 762                 cpufreq_frequency_table_target(&policy, policy.freq_table,
 763                                                policy.cur,
 764                                                CPUFREQ_RELATION_C, &index);
 765                 powernv_cpufreq_target_index(&policy, index);
 766                 cpumask_andnot(&mask, &mask, policy.cpus);
 767         }
 768 out:
 769         put_online_cpus();
 770 }
 771
 772 static int powernv_cpufreq_occ_msg(struct notifier_block *nb,
 773                                    unsigned long msg_type, void *_msg)
 774 {
 775         struct opal_msg *msg = _msg;
 776         struct opal_occ_msg omsg;
 777         int i;
 778
 779         if (msg_type != OPAL_MSG_OCC)
 780                 return 0;
 781
 782         omsg.type = be64_to_cpu(msg->params[0]);
 783
 784         switch (omsg.type) {
 785         case OCC_RESET:
 786                 occ_reset = true;
 787                 pr_info("OCC (On Chip Controller - enforces hard thermal/power limits) Resetting\n");
 788                 /*
 789                  * powernv_cpufreq_throttle_check() is called in
 790                  * target() callback which can detect the throttle state
 791                  * for governors like ondemand.
 792                  * But static governors will not call target() often thus
 793                  * report throttling here.
 794                  */
 795                 if (!throttled) {
 796                         throttled = true;
 797                         pr_warn("CPU frequency is throttled for duration\n");
 798                 }
 799
 800                 break;
 801         case OCC_LOAD:
 802                 pr_info("OCC Loading, CPU frequency is throttled until OCC is started\n");
 803                 break;
 804         case OCC_THROTTLE:
 805                 omsg.chip = be64_to_cpu(msg->params[1]);
 806                 omsg.throttle_status = be64_to_cpu(msg->params[2]);
 807
 808                 if (occ_reset) {
 809                         occ_reset = false;
 810                         throttled = false;
 811                         pr_info("OCC Active, CPU frequency is no longer throttled\n");
 812
 813                         for (i = 0; i < nr_chips; i++) {
 814                                 chips[i].restore = true;
 815                                 schedule_work(&chips[i].throttle);
 816                         }
 817
 818                         return 0;
 819                 }
 820
 821                 for (i = 0; i < nr_chips; i++)
 822                         if (chips[i].id == omsg.chip)
 823                                 break;
 824
 825                 if (omsg.throttle_status >= 0 &&
 826                     omsg.throttle_status <= OCC_MAX_THROTTLE_STATUS) {
 827                         chips[i].throttle_reason = omsg.throttle_status;
 828                         chips[i].reason[omsg.throttle_status]++;
 829                 }
 830
 831                 if (!omsg.throttle_status)
 832                         chips[i].restore = true;
 833
 834                 schedule_work(&chips[i].throttle);
 835         }
 836         return 0;
 837 }
 838
 839 static struct notifier_block powernv_cpufreq_opal_nb = {
 840         .notifier_call  = powernv_cpufreq_occ_msg,
 841         .next           = NULL,
 842         .priority       = 0,
 843 };
 844
 845 static void powernv_cpufreq_stop_cpu(struct cpufreq_policy *policy)
 846 {
 847         struct powernv_smp_call_data freq_data;
 848         struct global_pstate_info *gpstates = policy->driver_data;
 849
 850         freq_data.pstate_id = powernv_pstate_info.min;
 851         freq_data.gpstate_id = powernv_pstate_info.min;
 852         smp_call_function_single(policy->cpu, set_pstate, &freq_data, 1);
 853         del_timer_sync(&gpstates->timer);
 854 }
 855
 856 static struct cpufreq_driver powernv_cpufreq_driver = {
 857         .name           = "powernv-cpufreq",
 858         .flags          = CPUFREQ_CONST_LOOPS,
 859         .init           = powernv_cpufreq_cpu_init,
 860         .exit           = powernv_cpufreq_cpu_exit,
 861         .verify         = cpufreq_generic_frequency_table_verify,
 862         .target_index   = powernv_cpufreq_target_index,
 863         .get            = powernv_cpufreq_get,
 864         .stop_cpu       = powernv_cpufreq_stop_cpu,
 865         .attr           = powernv_cpu_freq_attr,
 866 };
 867
 868 static int init_chip_info(void)
 869 {
 870         unsigned int chip[256];
 871         unsigned int cpu, i;
 872         unsigned int prev_chip_id = UINT_MAX;
 873
 874         for_each_possible_cpu(cpu) {
 875                 unsigned int id = cpu_to_chip_id(cpu);
 876
 877                 if (prev_chip_id != id) {
 878                         prev_chip_id = id;
 879                         chip[nr_chips++] = id;
 880                 }
 881         }
 882
 883         chips = kcalloc(nr_chips, sizeof(struct chip), GFP_KERNEL);
 884         if (!chips)
 885                 return -ENOMEM;
 886
 887         for (i = 0; i < nr_chips; i++) {
 888                 chips[i].id = chip[i];
 889                 cpumask_copy(&chips[i].mask, cpumask_of_node(chip[i]));
 890                 INIT_WORK(&chips[i].throttle, powernv_cpufreq_work_fn);
 891                 for_each_cpu(cpu, &chips[i].mask)
 892                         per_cpu(chip_info, cpu) =  &chips[i];
 893         }
 894
 895         return 0;
 896 }
 897
 898 static inline void clean_chip_info(void)
 899 {
 900         kfree(chips);
 901 }
 902
 903 static inline void unregister_all_notifiers(void)
 904 {
 905         opal_message_notifier_unregister(OPAL_MSG_OCC,
 906                                          &powernv_cpufreq_opal_nb);
 907         unregister_reboot_notifier(&powernv_cpufreq_reboot_nb);
 908 }
 909
 910 static int __init powernv_cpufreq_init(void)
 911 {
 912         int rc = 0;
 913
 914         /* Don't probe on pseries (guest) platforms */
 915         if (!firmware_has_feature(FW_FEATURE_OPAL))
 916                 return -ENODEV;
 917
 918         /* Discover pstates from device tree and init */
 919         rc = init_powernv_pstates();
 920         if (rc)
 921                 goto out;
 922
 923         /* Populate chip info */
 924         rc = init_chip_info();
 925         if (rc)
 926                 goto out;
 927
 928         register_reboot_notifier(&powernv_cpufreq_reboot_nb);
 929         opal_message_notifier_register(OPAL_MSG_OCC, &powernv_cpufreq_opal_nb);
 930
 931         rc = cpufreq_register_driver(&powernv_cpufreq_driver);
 932         if (!rc)
 933                 return 0;
 934
 935         pr_info("Failed to register the cpufreq driver (%d)\n", rc);
 936         unregister_all_notifiers();
 937         clean_chip_info();
 938 out:
 939         pr_info("Platform driver disabled. System does not support PState control\n");
 940         return rc;
 941 }
 942 module_init(powernv_cpufreq_init);
 943
 944 static void __exit powernv_cpufreq_exit(void)
 945 {
 946         cpufreq_unregister_driver(&powernv_cpufreq_driver);
 947         unregister_all_notifiers();
 948         clean_chip_info();
 949 }
 950 module_exit(powernv_cpufreq_exit);
 951
 952 MODULE_LICENSE("GPL");
 953 MODULE_AUTHOR("Vaidyanathan Srinivasan <svaidy at linux.vnet.ibm.com>");