drivers/cpufreq/acpi-cpufreq.c

   1 /*
   2  * acpi-cpufreq.c - ACPI Processor P-States Driver
   3  *
   4  *  Copyright (C) 2001, 2002 Andy Grover <andrew.grover@intel.com>
   5  *  Copyright (C) 2001, 2002 Paul Diefenbaugh <paul.s.diefenbaugh@intel.com>
   6  *  Copyright (C) 2002 - 2004 Dominik Brodowski <linux@brodo.de>
   7  *  Copyright (C) 2006       Denis Sadykov <denis.m.sadykov@intel.com>
   8  *
   9  * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  10  *
  11  *  This program is free software; you can redistribute it and/or modify
  12  *  it under the terms of the GNU General Public License as published by
  13  *  the Free Software Foundation; either version 2 of the License, or (at
  14  *  your option) any later version.
  15  *
  16  *  This program is distributed in the hope that it will be useful, but
  17  *  WITHOUT ANY WARRANTY; without even the implied warranty of
  18  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  19  *  General Public License for more details.
  20  *
  21  *  You should have received a copy of the GNU General Public License along
  22  *  with this program; if not, write to the Free Software Foundation, Inc.,
  23  *  59 Temple Place, Suite 330, Boston, MA 02111-1307 USA.
  24  *
  25  * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  26  */
  27
  28 #include <linux/kernel.h>
  29 #include <linux/module.h>
  30 #include <linux/init.h>
  31 #include <linux/smp.h>
  32 #include <linux/sched.h>
  33 #include <linux/cpufreq.h>
  34 #include <linux/compiler.h>
  35 #include <linux/dmi.h>
  36 #include <linux/slab.h>
  37
  38 #include <linux/acpi.h>
  39 #include <linux/io.h>
  40 #include <linux/delay.h>
  41 #include <linux/uaccess.h>
  42
  43 #include <acpi/processor.h>
  44
  45 #include <asm/msr.h>
  46 #include <asm/processor.h>
  47 #include <asm/cpufeature.h>
  48
  49 MODULE_AUTHOR("Paul Diefenbaugh, Dominik Brodowski");
  50 MODULE_DESCRIPTION("ACPI Processor P-States Driver");
  51 MODULE_LICENSE("GPL");
  52
  53 #define PFX "acpi-cpufreq: "
  54
  55 enum {
  56         UNDEFINED_CAPABLE = 0,
  57         SYSTEM_INTEL_MSR_CAPABLE,
  58         SYSTEM_AMD_MSR_CAPABLE,
  59         SYSTEM_IO_CAPABLE,
  60 };
  61
  62 #define INTEL_MSR_RANGE         (0xffff)
  63 #define AMD_MSR_RANGE           (0x7)
  64
  65 #define MSR_K7_HWCR_CPB_DIS     (1ULL << 25)
  66
  67 struct acpi_cpufreq_data {
  68         struct cpufreq_frequency_table *freq_table;
  69         unsigned int resume;
  70         unsigned int cpu_feature;
  71         unsigned int acpi_perf_cpu;
  72         cpumask_var_t freqdomain_cpus;
  73 };
  74
  75 /* acpi_perf_data is a pointer to percpu data. */
  76 static struct acpi_processor_performance __percpu *acpi_perf_data;
  77
  78 static inline struct acpi_processor_performance *to_perf_data(struct acpi_cpufreq_data *data)
  79 {
  80         return per_cpu_ptr(acpi_perf_data, data->acpi_perf_cpu);
  81 }
  82
  83 static struct cpufreq_driver acpi_cpufreq_driver;
  84
  85 static unsigned int acpi_pstate_strict;
  86 static struct msr __percpu *msrs;
  87
  88 static bool boost_state(unsigned int cpu)
  89 {
  90         u32 lo, hi;
  91         u64 msr;
  92
  93         switch (boot_cpu_data.x86_vendor) {
  94         case X86_VENDOR_INTEL:
  95                 rdmsr_on_cpu(cpu, MSR_IA32_MISC_ENABLE, &lo, &hi);
  96                 msr = lo | ((u64)hi << 32);
  97                 return !(msr & MSR_IA32_MISC_ENABLE_TURBO_DISABLE);
  98         case X86_VENDOR_AMD:
  99                 rdmsr_on_cpu(cpu, MSR_K7_HWCR, &lo, &hi);
 100                 msr = lo | ((u64)hi << 32);
 101                 return !(msr & MSR_K7_HWCR_CPB_DIS);
 102         }
 103         return false;
 104 }
 105
 106 static void boost_set_msrs(bool enable, const struct cpumask *cpumask)
 107 {
 108         u32 cpu;
 109         u32 msr_addr;
 110         u64 msr_mask;
 111
 112         switch (boot_cpu_data.x86_vendor) {
 113         case X86_VENDOR_INTEL:
 114                 msr_addr = MSR_IA32_MISC_ENABLE;
 115                 msr_mask = MSR_IA32_MISC_ENABLE_TURBO_DISABLE;
 116                 break;
 117         case X86_VENDOR_AMD:
 118                 msr_addr = MSR_K7_HWCR;
 119                 msr_mask = MSR_K7_HWCR_CPB_DIS;
 120                 break;
 121         default:
 122                 return;
 123         }
 124
 125         rdmsr_on_cpus(cpumask, msr_addr, msrs);
 126
 127         for_each_cpu(cpu, cpumask) {
 128                 struct msr *reg = per_cpu_ptr(msrs, cpu);
 129                 if (enable)
 130                         reg->q &= ~msr_mask;
 131                 else
 132                         reg->q |= msr_mask;
 133         }
 134
 135         wrmsr_on_cpus(cpumask, msr_addr, msrs);
 136 }
 137
 138 static int set_boost(int val)
 139 {
 140         get_online_cpus();
 141         boost_set_msrs(val, cpu_online_mask);
 142         put_online_cpus();
 143         pr_debug("Core Boosting %sabled.\n", val ? "en" : "dis");
 144
 145         return 0;
 146 }
 147
 148 static ssize_t show_freqdomain_cpus(struct cpufreq_policy *policy, char *buf)
 149 {
 150         struct acpi_cpufreq_data *data = policy->driver_data;
 151
 152         if (unlikely(!data))
 153                 return -ENODEV;
 154
 155         return cpufreq_show_cpus(data->freqdomain_cpus, buf);
 156 }
 157
 158 cpufreq_freq_attr_ro(freqdomain_cpus);
 159
 160 #ifdef CONFIG_X86_ACPI_CPUFREQ_CPB
 161 static ssize_t store_cpb(struct cpufreq_policy *policy, const char *buf,
 162                          size_t count)
 163 {
 164         int ret;
 165         unsigned int val = 0;
 166
 167         if (!acpi_cpufreq_driver.set_boost)
 168                 return -EINVAL;
 169
 170         ret = kstrtouint(buf, 10, &val);
 171         if (ret || val > 1)
 172                 return -EINVAL;
 173
 174         set_boost(val);
 175
 176         return count;
 177 }
 178
 179 static ssize_t show_cpb(struct cpufreq_policy *policy, char *buf)
 180 {
 181         return sprintf(buf, "%u\n", acpi_cpufreq_driver.boost_enabled);
 182 }
 183
 184 cpufreq_freq_attr_rw(cpb);
 185 #endif
 186
 187 static int check_est_cpu(unsigned int cpuid)
 188 {
 189         struct cpuinfo_x86 *cpu = &cpu_data(cpuid);
 190
 191         return cpu_has(cpu, X86_FEATURE_EST);
 192 }
 193
 194 static int check_amd_hwpstate_cpu(unsigned int cpuid)
 195 {
 196         struct cpuinfo_x86 *cpu = &cpu_data(cpuid);
 197
 198         return cpu_has(cpu, X86_FEATURE_HW_PSTATE);
 199 }
 200
 201 static unsigned extract_io(u32 value, struct acpi_cpufreq_data *data)
 202 {
 203         struct acpi_processor_performance *perf;
 204         int i;
 205
 206         perf = to_perf_data(data);
 207
 208         for (i = 0; i < perf->state_count; i++) {
 209                 if (value == perf->states[i].status)
 210                         return data->freq_table[i].frequency;
 211         }
 212         return 0;
 213 }
 214
 215 static unsigned extract_msr(u32 msr, struct acpi_cpufreq_data *data)
 216 {
 217         struct cpufreq_frequency_table *pos;
 218         struct acpi_processor_performance *perf;
 219
 220         if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD)
 221                 msr &= AMD_MSR_RANGE;
 222         else
 223                 msr &= INTEL_MSR_RANGE;
 224
 225         perf = to_perf_data(data);
 226
 227         cpufreq_for_each_entry(pos, data->freq_table)
 228                 if (msr == perf->states[pos->driver_data].status)
 229                         return pos->frequency;
 230         return data->freq_table[0].frequency;
 231 }
 232
 233 static unsigned extract_freq(u32 val, struct acpi_cpufreq_data *data)
 234 {
 235         switch (data->cpu_feature) {
 236         case SYSTEM_INTEL_MSR_CAPABLE:
 237         case SYSTEM_AMD_MSR_CAPABLE:
 238                 return extract_msr(val, data);
 239         case SYSTEM_IO_CAPABLE:
 240                 return extract_io(val, data);
 241         default:
 242                 return 0;
 243         }
 244 }
 245
 246 struct msr_addr {
 247         u32 reg;
 248 };
 249
 250 struct io_addr {
 251         u16 port;
 252         u8 bit_width;
 253 };
 254
 255 struct drv_cmd {
 256         unsigned int type;
 257         const struct cpumask *mask;
 258         union {
 259                 struct msr_addr msr;
 260                 struct io_addr io;
 261         } addr;
 262         u32 val;
 263 };
 264
 265 /* Called via smp_call_function_single(), on the target CPU */
 266 static void do_drv_read(void *_cmd)
 267 {
 268         struct drv_cmd *cmd = _cmd;
 269         u32 h;
 270
 271         switch (cmd->type) {
 272         case SYSTEM_INTEL_MSR_CAPABLE:
 273         case SYSTEM_AMD_MSR_CAPABLE:
 274                 rdmsr(cmd->addr.msr.reg, cmd->val, h);
 275                 break;
 276         case SYSTEM_IO_CAPABLE:
 277                 acpi_os_read_port((acpi_io_address)cmd->addr.io.port,
 278                                 &cmd->val,
 279                                 (u32)cmd->addr.io.bit_width);
 280                 break;
 281         default:
 282                 break;
 283         }
 284 }
 285
 286 /* Called via smp_call_function_many(), on the target CPUs */
 287 static void do_drv_write(void *_cmd)
 288 {
 289         struct drv_cmd *cmd = _cmd;
 290         u32 lo, hi;
 291
 292         switch (cmd->type) {
 293         case SYSTEM_INTEL_MSR_CAPABLE:
 294                 rdmsr(cmd->addr.msr.reg, lo, hi);
 295                 lo = (lo & ~INTEL_MSR_RANGE) | (cmd->val & INTEL_MSR_RANGE);
 296                 wrmsr(cmd->addr.msr.reg, lo, hi);
 297                 break;
 298         case SYSTEM_AMD_MSR_CAPABLE:
 299                 wrmsr(cmd->addr.msr.reg, cmd->val, 0);
 300                 break;
 301         case SYSTEM_IO_CAPABLE:
 302                 acpi_os_write_port((acpi_io_address)cmd->addr.io.port,
 303                                 cmd->val,
 304                                 (u32)cmd->addr.io.bit_width);
 305                 break;
 306         default:
 307                 break;
 308         }
 309 }
 310
 311 static void drv_read(struct drv_cmd *cmd)
 312 {
 313         int err;
 314         cmd->val = 0;
 315
 316         err = smp_call_function_any(cmd->mask, do_drv_read, cmd, 1);
 317         WARN_ON_ONCE(err);      /* smp_call_function_any() was buggy? */
 318 }
 319
 320 static void drv_write(struct drv_cmd *cmd)
 321 {
 322         int this_cpu;
 323
 324         this_cpu = get_cpu();
 325         if (cpumask_test_cpu(this_cpu, cmd->mask))
 326                 do_drv_write(cmd);
 327         smp_call_function_many(cmd->mask, do_drv_write, cmd, 1);
 328         put_cpu();
 329 }
 330
 331 static u32
 332 get_cur_val(const struct cpumask *mask, struct acpi_cpufreq_data *data)
 333 {
 334         struct acpi_processor_performance *perf;
 335         struct drv_cmd cmd;
 336
 337         if (unlikely(cpumask_empty(mask)))
 338                 return 0;
 339
 340         switch (data->cpu_feature) {
 341         case SYSTEM_INTEL_MSR_CAPABLE:
 342                 cmd.type = SYSTEM_INTEL_MSR_CAPABLE;
 343                 cmd.addr.msr.reg = MSR_IA32_PERF_CTL;
 344                 break;
 345         case SYSTEM_AMD_MSR_CAPABLE:
 346                 cmd.type = SYSTEM_AMD_MSR_CAPABLE;
 347                 cmd.addr.msr.reg = MSR_AMD_PERF_CTL;
 348                 break;
 349         case SYSTEM_IO_CAPABLE:
 350                 cmd.type = SYSTEM_IO_CAPABLE;
 351                 perf = to_perf_data(data);
 352                 cmd.addr.io.port = perf->control_register.address;
 353                 cmd.addr.io.bit_width = perf->control_register.bit_width;
 354                 break;
 355         default:
 356                 return 0;
 357         }
 358
 359         cmd.mask = mask;
 360         drv_read(&cmd);
 361
 362         pr_debug("get_cur_val = %u\n", cmd.val);
 363
 364         return cmd.val;
 365 }
 366
 367 static unsigned int get_cur_freq_on_cpu(unsigned int cpu)
 368 {
 369         struct acpi_cpufreq_data *data;
 370         struct cpufreq_policy *policy;
 371         unsigned int freq;
 372         unsigned int cached_freq;
 373
 374         pr_debug("get_cur_freq_on_cpu (%d)\n", cpu);
 375
 376         policy = cpufreq_cpu_get_raw(cpu);
 377         if (unlikely(!policy))
 378                 return 0;
 379
 380         data = policy->driver_data;
 381         if (unlikely(!data || !data->freq_table))
 382                 return 0;
 383
 384         cached_freq = data->freq_table[to_perf_data(data)->state].frequency;
 385         freq = extract_freq(get_cur_val(cpumask_of(cpu), data), data);
 386         if (freq != cached_freq) {
 387                 /*
 388                  * The dreaded BIOS frequency change behind our back.
 389                  * Force set the frequency on next target call.
 390                  */
 391                 data->resume = 1;
 392         }
 393
 394         pr_debug("cur freq = %u\n", freq);
 395
 396         return freq;
 397 }
 398
 399 static unsigned int check_freqs(const struct cpumask *mask, unsigned int freq,
 400                                 struct acpi_cpufreq_data *data)
 401 {
 402         unsigned int cur_freq;
 403         unsigned int i;
 404
 405         for (i = 0; i < 100; i++) {
 406                 cur_freq = extract_freq(get_cur_val(mask, data), data);
 407                 if (cur_freq == freq)
 408                         return 1;
 409                 udelay(10);
 410         }
 411         return 0;
 412 }
 413
 414 static int acpi_cpufreq_target(struct cpufreq_policy *policy,
 415                                unsigned int index)
 416 {
 417         struct acpi_cpufreq_data *data = policy->driver_data;
 418         struct acpi_processor_performance *perf;
 419         struct drv_cmd cmd;
 420         unsigned int next_perf_state = 0; /* Index into perf table */
 421         int result = 0;
 422
 423         if (unlikely(data == NULL || data->freq_table == NULL)) {
 424                 return -ENODEV;
 425         }
 426
 427         perf = to_perf_data(data);
 428         next_perf_state = data->freq_table[index].driver_data;
 429         if (perf->state == next_perf_state) {
 430                 if (unlikely(data->resume)) {
 431                         pr_debug("Called after resume, resetting to P%d\n",
 432                                 next_perf_state);
 433                         data->resume = 0;
 434                 } else {
 435                         pr_debug("Already at target state (P%d)\n",
 436                                 next_perf_state);
 437                         goto out;
 438                 }
 439         }
 440
 441         switch (data->cpu_feature) {
 442         case SYSTEM_INTEL_MSR_CAPABLE:
 443                 cmd.type = SYSTEM_INTEL_MSR_CAPABLE;
 444                 cmd.addr.msr.reg = MSR_IA32_PERF_CTL;
 445                 cmd.val = (u32) perf->states[next_perf_state].control;
 446                 break;
 447         case SYSTEM_AMD_MSR_CAPABLE:
 448                 cmd.type = SYSTEM_AMD_MSR_CAPABLE;
 449                 cmd.addr.msr.reg = MSR_AMD_PERF_CTL;
 450                 cmd.val = (u32) perf->states[next_perf_state].control;
 451                 break;
 452         case SYSTEM_IO_CAPABLE:
 453                 cmd.type = SYSTEM_IO_CAPABLE;
 454                 cmd.addr.io.port = perf->control_register.address;
 455                 cmd.addr.io.bit_width = perf->control_register.bit_width;
 456                 cmd.val = (u32) perf->states[next_perf_state].control;
 457                 break;
 458         default:
 459                 result = -ENODEV;
 460                 goto out;
 461         }
 462
 463         /* cpufreq holds the hotplug lock, so we are safe from here on */
 464         if (policy->shared_type != CPUFREQ_SHARED_TYPE_ANY)
 465                 cmd.mask = policy->cpus;
 466         else
 467                 cmd.mask = cpumask_of(policy->cpu);
 468
 469         drv_write(&cmd);
 470
 471         if (acpi_pstate_strict) {
 472                 if (!check_freqs(cmd.mask, data->freq_table[index].frequency,
 473                                         data)) {
 474                         pr_debug("acpi_cpufreq_target failed (%d)\n",
 475                                 policy->cpu);
 476                         result = -EAGAIN;
 477                 }
 478         }
 479
 480         if (!result)
 481                 perf->state = next_perf_state;
 482
 483 out:
 484         return result;
 485 }
 486
 487 static unsigned long
 488 acpi_cpufreq_guess_freq(struct acpi_cpufreq_data *data, unsigned int cpu)
 489 {
 490         struct acpi_processor_performance *perf;
 491
 492         perf = to_perf_data(data);
 493         if (cpu_khz) {
 494                 /* search the closest match to cpu_khz */
 495                 unsigned int i;
 496                 unsigned long freq;
 497                 unsigned long freqn = perf->states[0].core_frequency * 1000;
 498
 499                 for (i = 0; i < (perf->state_count-1); i++) {
 500                         freq = freqn;
 501                         freqn = perf->states[i+1].core_frequency * 1000;
 502                         if ((2 * cpu_khz) > (freqn + freq)) {
 503                                 perf->state = i;
 504                                 return freq;
 505                         }
 506                 }
 507                 perf->state = perf->state_count-1;
 508                 return freqn;
 509         } else {
 510                 /* assume CPU is at P0... */
 511                 perf->state = 0;
 512                 return perf->states[0].core_frequency * 1000;
 513         }
 514 }
 515
 516 static void free_acpi_perf_data(void)
 517 {
 518         unsigned int i;
 519
 520         /* Freeing a NULL pointer is OK, and alloc_percpu zeroes. */
 521         for_each_possible_cpu(i)
 522                 free_cpumask_var(per_cpu_ptr(acpi_perf_data, i)
 523                                  ->shared_cpu_map);
 524         free_percpu(acpi_perf_data);
 525 }
 526
 527 static int boost_notify(struct notifier_block *nb, unsigned long action,
 528                       void *hcpu)
 529 {
 530         unsigned cpu = (long)hcpu;
 531         const struct cpumask *cpumask;
 532
 533         cpumask = get_cpu_mask(cpu);
 534
 535         /*
 536          * Clear the boost-disable bit on the CPU_DOWN path so that
 537          * this cpu cannot block the remaining ones from boosting. On
 538          * the CPU_UP path we simply keep the boost-disable flag in
 539          * sync with the current global state.
 540          */
 541
 542         switch (action) {
 543         case CPU_UP_PREPARE:
 544         case CPU_UP_PREPARE_FROZEN:
 545                 boost_set_msrs(acpi_cpufreq_driver.boost_enabled, cpumask);
 546                 break;
 547
 548         case CPU_DOWN_PREPARE:
 549         case CPU_DOWN_PREPARE_FROZEN:
 550                 boost_set_msrs(1, cpumask);
 551                 break;
 552
 553         default:
 554                 break;
 555         }
 556
 557         return NOTIFY_OK;
 558 }
 559
 560
 561 static struct notifier_block boost_nb = {
 562         .notifier_call          = boost_notify,
 563 };
 564
 565 /*
 566  * acpi_cpufreq_early_init - initialize ACPI P-States library
 567  *
 568  * Initialize the ACPI P-States library (drivers/acpi/processor_perflib.c)
 569  * in order to determine correct frequency and voltage pairings. We can
 570  * do _PDC and _PSD and find out the processor dependency for the
 571  * actual init that will happen later...
 572  */
 573 static int __init acpi_cpufreq_early_init(void)
 574 {
 575         unsigned int i;
 576         pr_debug("acpi_cpufreq_early_init\n");
 577
 578         acpi_perf_data = alloc_percpu(struct acpi_processor_performance);
 579         if (!acpi_perf_data) {
 580                 pr_debug("Memory allocation error for acpi_perf_data.\n");
 581                 return -ENOMEM;
 582         }
 583         for_each_possible_cpu(i) {
 584                 if (!zalloc_cpumask_var_node(
 585                         &per_cpu_ptr(acpi_perf_data, i)->shared_cpu_map,
 586                         GFP_KERNEL, cpu_to_node(i))) {
 587
 588                         /* Freeing a NULL pointer is OK: alloc_percpu zeroes. */
 589                         free_acpi_perf_data();
 590                         return -ENOMEM;
 591                 }
 592         }
 593
 594         /* Do initialization in ACPI core */
 595         acpi_processor_preregister_performance(acpi_perf_data);
 596         return 0;
 597 }
 598
 599 #ifdef CONFIG_SMP
 600 /*
 601  * Some BIOSes do SW_ANY coordination internally, either set it up in hw
 602  * or do it in BIOS firmware and won't inform about it to OS. If not
 603  * detected, this has a side effect of making CPU run at a different speed
 604  * than OS intended it to run at. Detect it and handle it cleanly.
 605  */
 606 static int bios_with_sw_any_bug;
 607
 608 static int sw_any_bug_found(const struct dmi_system_id *d)
 609 {
 610         bios_with_sw_any_bug = 1;
 611         return 0;
 612 }
 613
 614 static const struct dmi_system_id sw_any_bug_dmi_table[] = {
 615         {
 616                 .callback = sw_any_bug_found,
 617                 .ident = "Supermicro Server X6DLP",
 618                 .matches = {
 619                         DMI_MATCH(DMI_SYS_VENDOR, "Supermicro"),
 620                         DMI_MATCH(DMI_BIOS_VERSION, "080010"),
 621                         DMI_MATCH(DMI_PRODUCT_NAME, "X6DLP"),
 622                 },
 623         },
 624         { }
 625 };
 626
 627 static int acpi_cpufreq_blacklist(struct cpuinfo_x86 *c)
 628 {
 629         /* Intel Xeon Processor 7100 Series Specification Update
 630          * http://www.intel.com/Assets/PDF/specupdate/314554.pdf
 631          * AL30: A Machine Check Exception (MCE) Occurring during an
 632          * Enhanced Intel SpeedStep Technology Ratio Change May Cause
 633          * Both Processor Cores to Lock Up. */
 634         if (c->x86_vendor == X86_VENDOR_INTEL) {
 635                 if ((c->x86 == 15) &&
 636                     (c->x86_model == 6) &&
 637                     (c->x86_mask == 8)) {
 638                         printk(KERN_INFO "acpi-cpufreq: Intel(R) "
 639                             "Xeon(R) 7100 Errata AL30, processors may "
 640                             "lock up on frequency changes: disabling "
 641                             "acpi-cpufreq.\n");
 642                         return -ENODEV;
 643                     }
 644                 }
 645         return 0;
 646 }
 647 #endif
 648
 649 static int acpi_cpufreq_cpu_init(struct cpufreq_policy *policy)
 650 {
 651         unsigned int i;
 652         unsigned int valid_states = 0;
 653         unsigned int cpu = policy->cpu;
 654         struct acpi_cpufreq_data *data;
 655         unsigned int result = 0;
 656         struct cpuinfo_x86 *c = &cpu_data(policy->cpu);
 657         struct acpi_processor_performance *perf;
 658 #ifdef CONFIG_SMP
 659         static int blacklisted;
 660 #endif
 661
 662         pr_debug("acpi_cpufreq_cpu_init\n");
 663
 664 #ifdef CONFIG_SMP
 665         if (blacklisted)
 666                 return blacklisted;
 667         blacklisted = acpi_cpufreq_blacklist(c);
 668         if (blacklisted)
 669                 return blacklisted;
 670 #endif
 671
 672         data = kzalloc(sizeof(*data), GFP_KERNEL);
 673         if (!data)
 674                 return -ENOMEM;
 675
 676         if (!zalloc_cpumask_var(&data->freqdomain_cpus, GFP_KERNEL)) {
 677                 result = -ENOMEM;
 678                 goto err_free;
 679         }
 680
 681         perf = per_cpu_ptr(acpi_perf_data, cpu);
 682         data->acpi_perf_cpu = cpu;
 683         policy->driver_data = data;
 684
 685         if (cpu_has(c, X86_FEATURE_CONSTANT_TSC))
 686                 acpi_cpufreq_driver.flags |= CPUFREQ_CONST_LOOPS;
 687
 688         result = acpi_processor_register_performance(perf, cpu);
 689         if (result)
 690                 goto err_free_mask;
 691
 692         policy->shared_type = perf->shared_type;
 693
 694         /*
 695          * Will let policy->cpus know about dependency only when software
 696          * coordination is required.
 697          */
 698         if (policy->shared_type == CPUFREQ_SHARED_TYPE_ALL ||
 699             policy->shared_type == CPUFREQ_SHARED_TYPE_ANY) {
 700                 cpumask_copy(policy->cpus, perf->shared_cpu_map);
 701         }
 702         cpumask_copy(data->freqdomain_cpus, perf->shared_cpu_map);
 703
 704 #ifdef CONFIG_SMP
 705         dmi_check_system(sw_any_bug_dmi_table);
 706         if (bios_with_sw_any_bug && !policy_is_shared(policy)) {
 707                 policy->shared_type = CPUFREQ_SHARED_TYPE_ALL;
 708                 cpumask_copy(policy->cpus, topology_core_cpumask(cpu));
 709         }
 710
 711         if (check_amd_hwpstate_cpu(cpu) && !acpi_pstate_strict) {
 712                 cpumask_clear(policy->cpus);
 713                 cpumask_set_cpu(cpu, policy->cpus);
 714                 cpumask_copy(data->freqdomain_cpus,
 715                              topology_sibling_cpumask(cpu));
 716                 policy->shared_type = CPUFREQ_SHARED_TYPE_HW;
 717                 pr_info_once(PFX "overriding BIOS provided _PSD data\n");
 718         }
 719 #endif
 720
 721         /* capability check */
 722         if (perf->state_count <= 1) {
 723                 pr_debug("No P-States\n");
 724                 result = -ENODEV;
 725                 goto err_unreg;
 726         }
 727
 728         if (perf->control_register.space_id != perf->status_register.space_id) {
 729                 result = -ENODEV;
 730                 goto err_unreg;
 731         }
 732
 733         switch (perf->control_register.space_id) {
 734         case ACPI_ADR_SPACE_SYSTEM_IO:
 735                 if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD &&
 736                     boot_cpu_data.x86 == 0xf) {
 737                         pr_debug("AMD K8 systems must use native drivers.\n");
 738                         result = -ENODEV;
 739                         goto err_unreg;
 740                 }
 741                 pr_debug("SYSTEM IO addr space\n");
 742                 data->cpu_feature = SYSTEM_IO_CAPABLE;
 743                 break;
 744         case ACPI_ADR_SPACE_FIXED_HARDWARE:
 745                 pr_debug("HARDWARE addr space\n");
 746                 if (check_est_cpu(cpu)) {
 747                         data->cpu_feature = SYSTEM_INTEL_MSR_CAPABLE;
 748                         break;
 749                 }
 750                 if (check_amd_hwpstate_cpu(cpu)) {
 751                         data->cpu_feature = SYSTEM_AMD_MSR_CAPABLE;
 752                         break;
 753                 }
 754                 result = -ENODEV;
 755                 goto err_unreg;
 756         default:
 757                 pr_debug("Unknown addr space %d\n",
 758                         (u32) (perf->control_register.space_id));
 759                 result = -ENODEV;
 760                 goto err_unreg;
 761         }
 762
 763         data->freq_table = kzalloc(sizeof(*data->freq_table) *
 764                     (perf->state_count+1), GFP_KERNEL);
 765         if (!data->freq_table) {
 766                 result = -ENOMEM;
 767                 goto err_unreg;
 768         }
 769
 770         /* detect transition latency */
 771         policy->cpuinfo.transition_latency = 0;
 772         for (i = 0; i < perf->state_count; i++) {
 773                 if ((perf->states[i].transition_latency * 1000) >
 774                     policy->cpuinfo.transition_latency)
 775                         policy->cpuinfo.transition_latency =
 776                             perf->states[i].transition_latency * 1000;
 777         }
 778
 779         /* Check for high latency (>20uS) from buggy BIOSes, like on T42 */
 780         if (perf->control_register.space_id == ACPI_ADR_SPACE_FIXED_HARDWARE &&
 781             policy->cpuinfo.transition_latency > 20 * 1000) {
 782                 policy->cpuinfo.transition_latency = 20 * 1000;
 783                 printk_once(KERN_INFO
 784                             "P-state transition latency capped at 20 uS\n");
 785         }
 786
 787         /* table init */
 788         for (i = 0; i < perf->state_count; i++) {
 789                 if (i > 0 && perf->states[i].core_frequency >=
 790                     data->freq_table[valid_states-1].frequency / 1000)
 791                         continue;
 792
 793                 data->freq_table[valid_states].driver_data = i;
 794                 data->freq_table[valid_states].frequency =
 795                     perf->states[i].core_frequency * 1000;
 796                 valid_states++;
 797         }
 798         data->freq_table[valid_states].frequency = CPUFREQ_TABLE_END;
 799         perf->state = 0;
 800
 801         result = cpufreq_table_validate_and_show(policy, data->freq_table);
 802         if (result)
 803                 goto err_freqfree;
 804
 805         if (perf->states[0].core_frequency * 1000 != policy->cpuinfo.max_freq)
 806                 printk(KERN_WARNING FW_WARN "P-state 0 is not max freq\n");
 807
 808         switch (perf->control_register.space_id) {
 809         case ACPI_ADR_SPACE_SYSTEM_IO:
 810                 /*
 811                  * The core will not set policy->cur, because
 812                  * cpufreq_driver->get is NULL, so we need to set it here.
 813                  * However, we have to guess it, because the current speed is
 814                  * unknown and not detectable via IO ports.
 815                  */
 816                 policy->cur = acpi_cpufreq_guess_freq(data, policy->cpu);
 817                 break;
 818         case ACPI_ADR_SPACE_FIXED_HARDWARE:
 819                 acpi_cpufreq_driver.get = get_cur_freq_on_cpu;
 820                 break;
 821         default:
 822                 break;
 823         }
 824
 825         /* notify BIOS that we exist */
 826         acpi_processor_notify_smm(THIS_MODULE);
 827
 828         pr_debug("CPU%u - ACPI performance management activated.\n", cpu);
 829         for (i = 0; i < perf->state_count; i++)
 830                 pr_debug("     %cP%d: %d MHz, %d mW, %d uS\n",
 831                         (i == perf->state ? '*' : ' '), i,
 832                         (u32) perf->states[i].core_frequency,
 833                         (u32) perf->states[i].power,
 834                         (u32) perf->states[i].transition_latency);
 835
 836         /*
 837          * the first call to ->target() should result in us actually
 838          * writing something to the appropriate registers.
 839          */
 840         data->resume = 1;
 841
 842         return result;
 843
 844 err_freqfree:
 845         kfree(data->freq_table);
 846 err_unreg:
 847         acpi_processor_unregister_performance(cpu);
 848 err_free_mask:
 849         free_cpumask_var(data->freqdomain_cpus);
 850 err_free:
 851         kfree(data);
 852         policy->driver_data = NULL;
 853
 854         return result;
 855 }
 856
 857 static int acpi_cpufreq_cpu_exit(struct cpufreq_policy *policy)
 858 {
 859         struct acpi_cpufreq_data *data = policy->driver_data;
 860
 861         pr_debug("acpi_cpufreq_cpu_exit\n");
 862
 863         if (data) {
 864                 policy->driver_data = NULL;
 865                 acpi_processor_unregister_performance(data->acpi_perf_cpu);
 866                 free_cpumask_var(data->freqdomain_cpus);
 867                 kfree(data->freq_table);
 868                 kfree(data);
 869         }
 870
 871         return 0;
 872 }
 873
 874 static int acpi_cpufreq_resume(struct cpufreq_policy *policy)
 875 {
 876         struct acpi_cpufreq_data *data = policy->driver_data;
 877
 878         pr_debug("acpi_cpufreq_resume\n");
 879
 880         data->resume = 1;
 881
 882         return 0;
 883 }
 884
 885 static struct freq_attr *acpi_cpufreq_attr[] = {
 886         &cpufreq_freq_attr_scaling_available_freqs,
 887         &freqdomain_cpus,
 888 #ifdef CONFIG_X86_ACPI_CPUFREQ_CPB
 889         &cpb,
 890 #endif
 891         NULL,
 892 };
 893
 894 static struct cpufreq_driver acpi_cpufreq_driver = {
 895         .verify         = cpufreq_generic_frequency_table_verify,
 896         .target_index   = acpi_cpufreq_target,
 897         .bios_limit     = acpi_processor_get_bios_limit,
 898         .init           = acpi_cpufreq_cpu_init,
 899         .exit           = acpi_cpufreq_cpu_exit,
 900         .resume         = acpi_cpufreq_resume,
 901         .name           = "acpi-cpufreq",
 902         .attr           = acpi_cpufreq_attr,
 903 };
 904
 905 static void __init acpi_cpufreq_boost_init(void)
 906 {
 907         if (boot_cpu_has(X86_FEATURE_CPB) || boot_cpu_has(X86_FEATURE_IDA)) {
 908                 msrs = msrs_alloc();
 909
 910                 if (!msrs)
 911                         return;
 912
 913                 acpi_cpufreq_driver.set_boost = set_boost;
 914                 acpi_cpufreq_driver.boost_enabled = boost_state(0);
 915
 916                 cpu_notifier_register_begin();
 917
 918                 /* Force all MSRs to the same value */
 919                 boost_set_msrs(acpi_cpufreq_driver.boost_enabled,
 920                                cpu_online_mask);
 921
 922                 __register_cpu_notifier(&boost_nb);
 923
 924                 cpu_notifier_register_done();
 925         }
 926 }
 927
 928 static void acpi_cpufreq_boost_exit(void)
 929 {
 930         if (msrs) {
 931                 unregister_cpu_notifier(&boost_nb);
 932
 933                 msrs_free(msrs);
 934                 msrs = NULL;
 935         }
 936 }
 937
 938 static int __init acpi_cpufreq_init(void)
 939 {
 940         int ret;
 941
 942         if (acpi_disabled)
 943                 return -ENODEV;
 944
 945         /* don't keep reloading if cpufreq_driver exists */
 946         if (cpufreq_get_current_driver())
 947                 return -EEXIST;
 948
 949         pr_debug("acpi_cpufreq_init\n");
 950
 951         ret = acpi_cpufreq_early_init();
 952         if (ret)
 953                 return ret;
 954
 955 #ifdef CONFIG_X86_ACPI_CPUFREQ_CPB
 956         /* this is a sysfs file with a strange name and an even stranger
 957          * semantic - per CPU instantiation, but system global effect.
 958          * Lets enable it only on AMD CPUs for compatibility reasons and
 959          * only if configured. This is considered legacy code, which
 960          * will probably be removed at some point in the future.
 961          */
 962         if (!check_amd_hwpstate_cpu(0)) {
 963                 struct freq_attr **attr;
 964
 965                 pr_debug("CPB unsupported, do not expose it\n");
 966
 967                 for (attr = acpi_cpufreq_attr; *attr; attr++)
 968                         if (*attr == &cpb) {
 969                                 *attr = NULL;
 970                                 break;
 971                         }
 972         }
 973 #endif
 974         acpi_cpufreq_boost_init();
 975
 976         ret = cpufreq_register_driver(&acpi_cpufreq_driver);
 977         if (ret) {
 978                 free_acpi_perf_data();
 979                 acpi_cpufreq_boost_exit();
 980         }
 981         return ret;
 982 }
 983
 984 static void __exit acpi_cpufreq_exit(void)
 985 {
 986         pr_debug("acpi_cpufreq_exit\n");
 987
 988         acpi_cpufreq_boost_exit();
 989
 990         cpufreq_unregister_driver(&acpi_cpufreq_driver);
 991
 992         free_acpi_perf_data();
 993 }
 994
 995 module_param(acpi_pstate_strict, uint, 0644);
 996 MODULE_PARM_DESC(acpi_pstate_strict,
 997         "value 0 or non-zero. non-zero -> strict ACPI checks are "
 998         "performed during frequency changes.");
 999
1000 late_initcall(acpi_cpufreq_init);
1001 module_exit(acpi_cpufreq_exit);
1002
1003 static const struct x86_cpu_id acpi_cpufreq_ids[] = {
1004         X86_FEATURE_MATCH(X86_FEATURE_ACPI),
1005         X86_FEATURE_MATCH(X86_FEATURE_HW_PSTATE),
1006         {}
1007 };
1008 MODULE_DEVICE_TABLE(x86cpu, acpi_cpufreq_ids);
1009
1010 static const struct acpi_device_id processor_device_ids[] = {
1011         {ACPI_PROCESSOR_OBJECT_HID, },
1012         {ACPI_PROCESSOR_DEVICE_HID, },
1013         {},
1014 };
1015 MODULE_DEVICE_TABLE(acpi, processor_device_ids);
1016
1017 MODULE_ALIAS("acpi");