1 /* 2 * acpi-cpufreq.c - ACPI Processor P-States Driver 3 * 4 * Copyright (C) 2001, 2002 Andy Grover <andrew.grover@intel.com> 5 * Copyright (C) 2001, 2002 Paul Diefenbaugh <paul.s.diefenbaugh@intel.com> 6 * Copyright (C) 2002 - 2004 Dominik Brodowski <linux@brodo.de> 7 * Copyright (C) 2006 Denis Sadykov <denis.m.sadykov@intel.com> 8 * 9 * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 10 * 11 * This program is free software; you can redistribute it and/or modify 12 * it under the terms of the GNU General Public License as published by 13 * the Free Software Foundation; either version 2 of the License, or (at 14 * your option) any later version. 15 * 16 * This program is distributed in the hope that it will be useful, but 17 * WITHOUT ANY WARRANTY; without even the implied warranty of 18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 19 * General Public License for more details. 20 * 21 * You should have received a copy of the GNU General Public License along 22 * with this program; if not, write to the Free Software Foundation, Inc., 23 * 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. 24 * 25 * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 26 */ 27 28 #include <linux/kernel.h> 29 #include <linux/module.h> 30 #include <linux/init.h> 31 #include <linux/smp.h> 32 #include <linux/sched.h> 33 #include <linux/cpufreq.h> 34 #include <linux/compiler.h> 35 #include <linux/dmi.h> 36 #include <linux/slab.h> 37 38 #include <linux/acpi.h> 39 #include <linux/io.h> 40 #include <linux/delay.h> 41 #include <linux/uaccess.h> 42 43 #include <acpi/processor.h> 44 45 #include <asm/msr.h> 46 #include <asm/processor.h> 47 #include <asm/cpufeature.h> 48 #include "mperf.h" 49 50 MODULE_AUTHOR("Paul Diefenbaugh, Dominik Brodowski"); 51 MODULE_DESCRIPTION("ACPI Processor P-States Driver"); 52 MODULE_LICENSE("GPL"); 53 54 #define PFX "acpi-cpufreq: " 55 56 enum { 57 UNDEFINED_CAPABLE = 0, 58 SYSTEM_INTEL_MSR_CAPABLE, 59 SYSTEM_AMD_MSR_CAPABLE, 60 SYSTEM_IO_CAPABLE, 61 }; 62 63 #define INTEL_MSR_RANGE (0xffff) 64 #define AMD_MSR_RANGE (0x7) 65 66 #define MSR_K7_HWCR_CPB_DIS (1ULL << 25) 67 68 struct acpi_cpufreq_data { 69 struct acpi_processor_performance *acpi_data; 70 struct cpufreq_frequency_table *freq_table; 71 unsigned int resume; 72 unsigned int cpu_feature; 73 cpumask_var_t freqdomain_cpus; 74 }; 75 76 static DEFINE_PER_CPU(struct acpi_cpufreq_data *, acfreq_data); 77 78 /* acpi_perf_data is a pointer to percpu data. */ 79 static struct acpi_processor_performance __percpu *acpi_perf_data; 80 81 static struct cpufreq_driver acpi_cpufreq_driver; 82 83 static unsigned int acpi_pstate_strict; 84 static bool boost_enabled, boost_supported; 85 static struct msr __percpu *msrs; 86 87 static bool boost_state(unsigned int cpu) 88 { 89 u32 lo, hi; 90 u64 msr; 91 92 switch (boot_cpu_data.x86_vendor) { 93 case X86_VENDOR_INTEL: 94 rdmsr_on_cpu(cpu, MSR_IA32_MISC_ENABLE, &lo, &hi); 95 msr = lo | ((u64)hi << 32); 96 return !(msr & MSR_IA32_MISC_ENABLE_TURBO_DISABLE); 97 case X86_VENDOR_AMD: 98 rdmsr_on_cpu(cpu, MSR_K7_HWCR, &lo, &hi); 99 msr = lo | ((u64)hi << 32); 100 return !(msr & MSR_K7_HWCR_CPB_DIS); 101 } 102 return false; 103 } 104 105 static void boost_set_msrs(bool enable, const struct cpumask *cpumask) 106 { 107 u32 cpu; 108 u32 msr_addr; 109 u64 msr_mask; 110 111 switch (boot_cpu_data.x86_vendor) { 112 case X86_VENDOR_INTEL: 113 msr_addr = MSR_IA32_MISC_ENABLE; 114 msr_mask = MSR_IA32_MISC_ENABLE_TURBO_DISABLE; 115 break; 116 case X86_VENDOR_AMD: 117 msr_addr = MSR_K7_HWCR; 118 msr_mask = MSR_K7_HWCR_CPB_DIS; 119 break; 120 default: 121 return; 122 } 123 124 rdmsr_on_cpus(cpumask, msr_addr, msrs); 125 126 for_each_cpu(cpu, cpumask) { 127 struct msr *reg = per_cpu_ptr(msrs, cpu); 128 if (enable) 129 reg->q &= ~msr_mask; 130 else 131 reg->q |= msr_mask; 132 } 133 134 wrmsr_on_cpus(cpumask, msr_addr, msrs); 135 } 136 137 static ssize_t _store_boost(const char *buf, size_t count) 138 { 139 int ret; 140 unsigned long val = 0; 141 142 if (!boost_supported) 143 return -EINVAL; 144 145 ret = kstrtoul(buf, 10, &val); 146 if (ret || (val > 1)) 147 return -EINVAL; 148 149 if ((val && boost_enabled) || (!val && !boost_enabled)) 150 return count; 151 152 get_online_cpus(); 153 154 boost_set_msrs(val, cpu_online_mask); 155 156 put_online_cpus(); 157 158 boost_enabled = val; 159 pr_debug("Core Boosting %sabled.\n", val ? "en" : "dis"); 160 161 return count; 162 } 163 164 static ssize_t store_global_boost(struct kobject *kobj, struct attribute *attr, 165 const char *buf, size_t count) 166 { 167 return _store_boost(buf, count); 168 } 169 170 static ssize_t show_global_boost(struct kobject *kobj, 171 struct attribute *attr, char *buf) 172 { 173 return sprintf(buf, "%u\n", boost_enabled); 174 } 175 176 static struct global_attr global_boost = __ATTR(boost, 0644, 177 show_global_boost, 178 store_global_boost); 179 180 static ssize_t show_freqdomain_cpus(struct cpufreq_policy *policy, char *buf) 181 { 182 struct acpi_cpufreq_data *data = per_cpu(acfreq_data, policy->cpu); 183 184 return cpufreq_show_cpus(data->freqdomain_cpus, buf); 185 } 186 187 cpufreq_freq_attr_ro(freqdomain_cpus); 188 189 #ifdef CONFIG_X86_ACPI_CPUFREQ_CPB 190 static ssize_t store_cpb(struct cpufreq_policy *policy, const char *buf, 191 size_t count) 192 { 193 return _store_boost(buf, count); 194 } 195 196 static ssize_t show_cpb(struct cpufreq_policy *policy, char *buf) 197 { 198 return sprintf(buf, "%u\n", boost_enabled); 199 } 200 201 static struct freq_attr cpb = __ATTR(cpb, 0644, show_cpb, store_cpb); 202 #endif 203 204 static int check_est_cpu(unsigned int cpuid) 205 { 206 struct cpuinfo_x86 *cpu = &cpu_data(cpuid); 207 208 return cpu_has(cpu, X86_FEATURE_EST); 209 } 210 211 static int check_amd_hwpstate_cpu(unsigned int cpuid) 212 { 213 struct cpuinfo_x86 *cpu = &cpu_data(cpuid); 214 215 return cpu_has(cpu, X86_FEATURE_HW_PSTATE); 216 } 217 218 static unsigned extract_io(u32 value, struct acpi_cpufreq_data *data) 219 { 220 struct acpi_processor_performance *perf; 221 int i; 222 223 perf = data->acpi_data; 224 225 for (i = 0; i < perf->state_count; i++) { 226 if (value == perf->states[i].status) 227 return data->freq_table[i].frequency; 228 } 229 return 0; 230 } 231 232 static unsigned extract_msr(u32 msr, struct acpi_cpufreq_data *data) 233 { 234 int i; 235 struct acpi_processor_performance *perf; 236 237 if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) 238 msr &= AMD_MSR_RANGE; 239 else 240 msr &= INTEL_MSR_RANGE; 241 242 perf = data->acpi_data; 243 244 for (i = 0; data->freq_table[i].frequency != CPUFREQ_TABLE_END; i++) { 245 if (msr == perf->states[data->freq_table[i].driver_data].status) 246 return data->freq_table[i].frequency; 247 } 248 return data->freq_table[0].frequency; 249 } 250 251 static unsigned extract_freq(u32 val, struct acpi_cpufreq_data *data) 252 { 253 switch (data->cpu_feature) { 254 case SYSTEM_INTEL_MSR_CAPABLE: 255 case SYSTEM_AMD_MSR_CAPABLE: 256 return extract_msr(val, data); 257 case SYSTEM_IO_CAPABLE: 258 return extract_io(val, data); 259 default: 260 return 0; 261 } 262 } 263 264 struct msr_addr { 265 u32 reg; 266 }; 267 268 struct io_addr { 269 u16 port; 270 u8 bit_width; 271 }; 272 273 struct drv_cmd { 274 unsigned int type; 275 const struct cpumask *mask; 276 union { 277 struct msr_addr msr; 278 struct io_addr io; 279 } addr; 280 u32 val; 281 }; 282 283 /* Called via smp_call_function_single(), on the target CPU */ 284 static void do_drv_read(void *_cmd) 285 { 286 struct drv_cmd *cmd = _cmd; 287 u32 h; 288 289 switch (cmd->type) { 290 case SYSTEM_INTEL_MSR_CAPABLE: 291 case SYSTEM_AMD_MSR_CAPABLE: 292 rdmsr(cmd->addr.msr.reg, cmd->val, h); 293 break; 294 case SYSTEM_IO_CAPABLE: 295 acpi_os_read_port((acpi_io_address)cmd->addr.io.port, 296 &cmd->val, 297 (u32)cmd->addr.io.bit_width); 298 break; 299 default: 300 break; 301 } 302 } 303 304 /* Called via smp_call_function_many(), on the target CPUs */ 305 static void do_drv_write(void *_cmd) 306 { 307 struct drv_cmd *cmd = _cmd; 308 u32 lo, hi; 309 310 switch (cmd->type) { 311 case SYSTEM_INTEL_MSR_CAPABLE: 312 rdmsr(cmd->addr.msr.reg, lo, hi); 313 lo = (lo & ~INTEL_MSR_RANGE) | (cmd->val & INTEL_MSR_RANGE); 314 wrmsr(cmd->addr.msr.reg, lo, hi); 315 break; 316 case SYSTEM_AMD_MSR_CAPABLE: 317 wrmsr(cmd->addr.msr.reg, cmd->val, 0); 318 break; 319 case SYSTEM_IO_CAPABLE: 320 acpi_os_write_port((acpi_io_address)cmd->addr.io.port, 321 cmd->val, 322 (u32)cmd->addr.io.bit_width); 323 break; 324 default: 325 break; 326 } 327 } 328 329 static void drv_read(struct drv_cmd *cmd) 330 { 331 int err; 332 cmd->val = 0; 333 334 err = smp_call_function_any(cmd->mask, do_drv_read, cmd, 1); 335 WARN_ON_ONCE(err); /* smp_call_function_any() was buggy? */ 336 } 337 338 static void drv_write(struct drv_cmd *cmd) 339 { 340 int this_cpu; 341 342 this_cpu = get_cpu(); 343 if (cpumask_test_cpu(this_cpu, cmd->mask)) 344 do_drv_write(cmd); 345 smp_call_function_many(cmd->mask, do_drv_write, cmd, 1); 346 put_cpu(); 347 } 348 349 static u32 get_cur_val(const struct cpumask *mask) 350 { 351 struct acpi_processor_performance *perf; 352 struct drv_cmd cmd; 353 354 if (unlikely(cpumask_empty(mask))) 355 return 0; 356 357 switch (per_cpu(acfreq_data, cpumask_first(mask))->cpu_feature) { 358 case SYSTEM_INTEL_MSR_CAPABLE: 359 cmd.type = SYSTEM_INTEL_MSR_CAPABLE; 360 cmd.addr.msr.reg = MSR_IA32_PERF_CTL; 361 break; 362 case SYSTEM_AMD_MSR_CAPABLE: 363 cmd.type = SYSTEM_AMD_MSR_CAPABLE; 364 cmd.addr.msr.reg = MSR_AMD_PERF_CTL; 365 break; 366 case SYSTEM_IO_CAPABLE: 367 cmd.type = SYSTEM_IO_CAPABLE; 368 perf = per_cpu(acfreq_data, cpumask_first(mask))->acpi_data; 369 cmd.addr.io.port = perf->control_register.address; 370 cmd.addr.io.bit_width = perf->control_register.bit_width; 371 break; 372 default: 373 return 0; 374 } 375 376 cmd.mask = mask; 377 drv_read(&cmd); 378 379 pr_debug("get_cur_val = %u\n", cmd.val); 380 381 return cmd.val; 382 } 383 384 static unsigned int get_cur_freq_on_cpu(unsigned int cpu) 385 { 386 struct acpi_cpufreq_data *data = per_cpu(acfreq_data, cpu); 387 unsigned int freq; 388 unsigned int cached_freq; 389 390 pr_debug("get_cur_freq_on_cpu (%d)\n", cpu); 391 392 if (unlikely(data == NULL || 393 data->acpi_data == NULL || data->freq_table == NULL)) { 394 return 0; 395 } 396 397 cached_freq = data->freq_table[data->acpi_data->state].frequency; 398 freq = extract_freq(get_cur_val(cpumask_of(cpu)), data); 399 if (freq != cached_freq) { 400 /* 401 * The dreaded BIOS frequency change behind our back. 402 * Force set the frequency on next target call. 403 */ 404 data->resume = 1; 405 } 406 407 pr_debug("cur freq = %u\n", freq); 408 409 return freq; 410 } 411 412 static unsigned int check_freqs(const struct cpumask *mask, unsigned int freq, 413 struct acpi_cpufreq_data *data) 414 { 415 unsigned int cur_freq; 416 unsigned int i; 417 418 for (i = 0; i < 100; i++) { 419 cur_freq = extract_freq(get_cur_val(mask), data); 420 if (cur_freq == freq) 421 return 1; 422 udelay(10); 423 } 424 return 0; 425 } 426 427 static int acpi_cpufreq_target(struct cpufreq_policy *policy, 428 unsigned int target_freq, unsigned int relation) 429 { 430 struct acpi_cpufreq_data *data = per_cpu(acfreq_data, policy->cpu); 431 struct acpi_processor_performance *perf; 432 struct cpufreq_freqs freqs; 433 struct drv_cmd cmd; 434 unsigned int next_state = 0; /* Index into freq_table */ 435 unsigned int next_perf_state = 0; /* Index into perf table */ 436 int result = 0; 437 438 pr_debug("acpi_cpufreq_target %d (%d)\n", target_freq, policy->cpu); 439 440 if (unlikely(data == NULL || 441 data->acpi_data == NULL || data->freq_table == NULL)) { 442 return -ENODEV; 443 } 444 445 perf = data->acpi_data; 446 result = cpufreq_frequency_table_target(policy, 447 data->freq_table, 448 target_freq, 449 relation, &next_state); 450 if (unlikely(result)) { 451 result = -ENODEV; 452 goto out; 453 } 454 455 next_perf_state = data->freq_table[next_state].driver_data; 456 if (perf->state == next_perf_state) { 457 if (unlikely(data->resume)) { 458 pr_debug("Called after resume, resetting to P%d\n", 459 next_perf_state); 460 data->resume = 0; 461 } else { 462 pr_debug("Already at target state (P%d)\n", 463 next_perf_state); 464 goto out; 465 } 466 } 467 468 switch (data->cpu_feature) { 469 case SYSTEM_INTEL_MSR_CAPABLE: 470 cmd.type = SYSTEM_INTEL_MSR_CAPABLE; 471 cmd.addr.msr.reg = MSR_IA32_PERF_CTL; 472 cmd.val = (u32) perf->states[next_perf_state].control; 473 break; 474 case SYSTEM_AMD_MSR_CAPABLE: 475 cmd.type = SYSTEM_AMD_MSR_CAPABLE; 476 cmd.addr.msr.reg = MSR_AMD_PERF_CTL; 477 cmd.val = (u32) perf->states[next_perf_state].control; 478 break; 479 case SYSTEM_IO_CAPABLE: 480 cmd.type = SYSTEM_IO_CAPABLE; 481 cmd.addr.io.port = perf->control_register.address; 482 cmd.addr.io.bit_width = perf->control_register.bit_width; 483 cmd.val = (u32) perf->states[next_perf_state].control; 484 break; 485 default: 486 result = -ENODEV; 487 goto out; 488 } 489 490 /* cpufreq holds the hotplug lock, so we are safe from here on */ 491 if (policy->shared_type != CPUFREQ_SHARED_TYPE_ANY) 492 cmd.mask = policy->cpus; 493 else 494 cmd.mask = cpumask_of(policy->cpu); 495 496 freqs.old = perf->states[perf->state].core_frequency * 1000; 497 freqs.new = data->freq_table[next_state].frequency; 498 cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE); 499 500 drv_write(&cmd); 501 502 if (acpi_pstate_strict) { 503 if (!check_freqs(cmd.mask, freqs.new, data)) { 504 pr_debug("acpi_cpufreq_target failed (%d)\n", 505 policy->cpu); 506 result = -EAGAIN; 507 freqs.new = freqs.old; 508 } 509 } 510 511 cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE); 512 513 if (!result) 514 perf->state = next_perf_state; 515 516 out: 517 return result; 518 } 519 520 static int acpi_cpufreq_verify(struct cpufreq_policy *policy) 521 { 522 struct acpi_cpufreq_data *data = per_cpu(acfreq_data, policy->cpu); 523 524 pr_debug("acpi_cpufreq_verify\n"); 525 526 return cpufreq_frequency_table_verify(policy, data->freq_table); 527 } 528 529 static unsigned long 530 acpi_cpufreq_guess_freq(struct acpi_cpufreq_data *data, unsigned int cpu) 531 { 532 struct acpi_processor_performance *perf = data->acpi_data; 533 534 if (cpu_khz) { 535 /* search the closest match to cpu_khz */ 536 unsigned int i; 537 unsigned long freq; 538 unsigned long freqn = perf->states[0].core_frequency * 1000; 539 540 for (i = 0; i < (perf->state_count-1); i++) { 541 freq = freqn; 542 freqn = perf->states[i+1].core_frequency * 1000; 543 if ((2 * cpu_khz) > (freqn + freq)) { 544 perf->state = i; 545 return freq; 546 } 547 } 548 perf->state = perf->state_count-1; 549 return freqn; 550 } else { 551 /* assume CPU is at P0... */ 552 perf->state = 0; 553 return perf->states[0].core_frequency * 1000; 554 } 555 } 556 557 static void free_acpi_perf_data(void) 558 { 559 unsigned int i; 560 561 /* Freeing a NULL pointer is OK, and alloc_percpu zeroes. */ 562 for_each_possible_cpu(i) 563 free_cpumask_var(per_cpu_ptr(acpi_perf_data, i) 564 ->shared_cpu_map); 565 free_percpu(acpi_perf_data); 566 } 567 568 static int boost_notify(struct notifier_block *nb, unsigned long action, 569 void *hcpu) 570 { 571 unsigned cpu = (long)hcpu; 572 const struct cpumask *cpumask; 573 574 cpumask = get_cpu_mask(cpu); 575 576 /* 577 * Clear the boost-disable bit on the CPU_DOWN path so that 578 * this cpu cannot block the remaining ones from boosting. On 579 * the CPU_UP path we simply keep the boost-disable flag in 580 * sync with the current global state. 581 */ 582 583 switch (action) { 584 case CPU_UP_PREPARE: 585 case CPU_UP_PREPARE_FROZEN: 586 boost_set_msrs(boost_enabled, cpumask); 587 break; 588 589 case CPU_DOWN_PREPARE: 590 case CPU_DOWN_PREPARE_FROZEN: 591 boost_set_msrs(1, cpumask); 592 break; 593 594 default: 595 break; 596 } 597 598 return NOTIFY_OK; 599 } 600 601 602 static struct notifier_block boost_nb = { 603 .notifier_call = boost_notify, 604 }; 605 606 /* 607 * acpi_cpufreq_early_init - initialize ACPI P-States library 608 * 609 * Initialize the ACPI P-States library (drivers/acpi/processor_perflib.c) 610 * in order to determine correct frequency and voltage pairings. We can 611 * do _PDC and _PSD and find out the processor dependency for the 612 * actual init that will happen later... 613 */ 614 static int __init acpi_cpufreq_early_init(void) 615 { 616 unsigned int i; 617 pr_debug("acpi_cpufreq_early_init\n"); 618 619 acpi_perf_data = alloc_percpu(struct acpi_processor_performance); 620 if (!acpi_perf_data) { 621 pr_debug("Memory allocation error for acpi_perf_data.\n"); 622 return -ENOMEM; 623 } 624 for_each_possible_cpu(i) { 625 if (!zalloc_cpumask_var_node( 626 &per_cpu_ptr(acpi_perf_data, i)->shared_cpu_map, 627 GFP_KERNEL, cpu_to_node(i))) { 628 629 /* Freeing a NULL pointer is OK: alloc_percpu zeroes. */ 630 free_acpi_perf_data(); 631 return -ENOMEM; 632 } 633 } 634 635 /* Do initialization in ACPI core */ 636 acpi_processor_preregister_performance(acpi_perf_data); 637 return 0; 638 } 639 640 #ifdef CONFIG_SMP 641 /* 642 * Some BIOSes do SW_ANY coordination internally, either set it up in hw 643 * or do it in BIOS firmware and won't inform about it to OS. If not 644 * detected, this has a side effect of making CPU run at a different speed 645 * than OS intended it to run at. Detect it and handle it cleanly. 646 */ 647 static int bios_with_sw_any_bug; 648 649 static int sw_any_bug_found(const struct dmi_system_id *d) 650 { 651 bios_with_sw_any_bug = 1; 652 return 0; 653 } 654 655 static const struct dmi_system_id sw_any_bug_dmi_table[] = { 656 { 657 .callback = sw_any_bug_found, 658 .ident = "Supermicro Server X6DLP", 659 .matches = { 660 DMI_MATCH(DMI_SYS_VENDOR, "Supermicro"), 661 DMI_MATCH(DMI_BIOS_VERSION, "080010"), 662 DMI_MATCH(DMI_PRODUCT_NAME, "X6DLP"), 663 }, 664 }, 665 { } 666 }; 667 668 static int acpi_cpufreq_blacklist(struct cpuinfo_x86 *c) 669 { 670 /* Intel Xeon Processor 7100 Series Specification Update 671 * http://www.intel.com/Assets/PDF/specupdate/314554.pdf 672 * AL30: A Machine Check Exception (MCE) Occurring during an 673 * Enhanced Intel SpeedStep Technology Ratio Change May Cause 674 * Both Processor Cores to Lock Up. */ 675 if (c->x86_vendor == X86_VENDOR_INTEL) { 676 if ((c->x86 == 15) && 677 (c->x86_model == 6) && 678 (c->x86_mask == 8)) { 679 printk(KERN_INFO "acpi-cpufreq: Intel(R) " 680 "Xeon(R) 7100 Errata AL30, processors may " 681 "lock up on frequency changes: disabling " 682 "acpi-cpufreq.\n"); 683 return -ENODEV; 684 } 685 } 686 return 0; 687 } 688 #endif 689 690 static int acpi_cpufreq_cpu_init(struct cpufreq_policy *policy) 691 { 692 unsigned int i; 693 unsigned int valid_states = 0; 694 unsigned int cpu = policy->cpu; 695 struct acpi_cpufreq_data *data; 696 unsigned int result = 0; 697 struct cpuinfo_x86 *c = &cpu_data(policy->cpu); 698 struct acpi_processor_performance *perf; 699 #ifdef CONFIG_SMP 700 static int blacklisted; 701 #endif 702 703 pr_debug("acpi_cpufreq_cpu_init\n"); 704 705 #ifdef CONFIG_SMP 706 if (blacklisted) 707 return blacklisted; 708 blacklisted = acpi_cpufreq_blacklist(c); 709 if (blacklisted) 710 return blacklisted; 711 #endif 712 713 data = kzalloc(sizeof(struct acpi_cpufreq_data), GFP_KERNEL); 714 if (!data) 715 return -ENOMEM; 716 717 if (!zalloc_cpumask_var(&data->freqdomain_cpus, GFP_KERNEL)) { 718 result = -ENOMEM; 719 goto err_free; 720 } 721 722 data->acpi_data = per_cpu_ptr(acpi_perf_data, cpu); 723 per_cpu(acfreq_data, cpu) = data; 724 725 if (cpu_has(c, X86_FEATURE_CONSTANT_TSC)) 726 acpi_cpufreq_driver.flags |= CPUFREQ_CONST_LOOPS; 727 728 result = acpi_processor_register_performance(data->acpi_data, cpu); 729 if (result) 730 goto err_free_mask; 731 732 perf = data->acpi_data; 733 policy->shared_type = perf->shared_type; 734 735 /* 736 * Will let policy->cpus know about dependency only when software 737 * coordination is required. 738 */ 739 if (policy->shared_type == CPUFREQ_SHARED_TYPE_ALL || 740 policy->shared_type == CPUFREQ_SHARED_TYPE_ANY) { 741 cpumask_copy(policy->cpus, perf->shared_cpu_map); 742 } 743 cpumask_copy(data->freqdomain_cpus, perf->shared_cpu_map); 744 745 #ifdef CONFIG_SMP 746 dmi_check_system(sw_any_bug_dmi_table); 747 if (bios_with_sw_any_bug && !policy_is_shared(policy)) { 748 policy->shared_type = CPUFREQ_SHARED_TYPE_ALL; 749 cpumask_copy(policy->cpus, cpu_core_mask(cpu)); 750 } 751 752 if (check_amd_hwpstate_cpu(cpu) && !acpi_pstate_strict) { 753 cpumask_clear(policy->cpus); 754 cpumask_set_cpu(cpu, policy->cpus); 755 cpumask_copy(data->freqdomain_cpus, cpu_sibling_mask(cpu)); 756 policy->shared_type = CPUFREQ_SHARED_TYPE_HW; 757 pr_info_once(PFX "overriding BIOS provided _PSD data\n"); 758 } 759 #endif 760 761 /* capability check */ 762 if (perf->state_count <= 1) { 763 pr_debug("No P-States\n"); 764 result = -ENODEV; 765 goto err_unreg; 766 } 767 768 if (perf->control_register.space_id != perf->status_register.space_id) { 769 result = -ENODEV; 770 goto err_unreg; 771 } 772 773 switch (perf->control_register.space_id) { 774 case ACPI_ADR_SPACE_SYSTEM_IO: 775 if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD && 776 boot_cpu_data.x86 == 0xf) { 777 pr_debug("AMD K8 systems must use native drivers.\n"); 778 result = -ENODEV; 779 goto err_unreg; 780 } 781 pr_debug("SYSTEM IO addr space\n"); 782 data->cpu_feature = SYSTEM_IO_CAPABLE; 783 break; 784 case ACPI_ADR_SPACE_FIXED_HARDWARE: 785 pr_debug("HARDWARE addr space\n"); 786 if (check_est_cpu(cpu)) { 787 data->cpu_feature = SYSTEM_INTEL_MSR_CAPABLE; 788 break; 789 } 790 if (check_amd_hwpstate_cpu(cpu)) { 791 data->cpu_feature = SYSTEM_AMD_MSR_CAPABLE; 792 break; 793 } 794 result = -ENODEV; 795 goto err_unreg; 796 default: 797 pr_debug("Unknown addr space %d\n", 798 (u32) (perf->control_register.space_id)); 799 result = -ENODEV; 800 goto err_unreg; 801 } 802 803 data->freq_table = kmalloc(sizeof(struct cpufreq_frequency_table) * 804 (perf->state_count+1), GFP_KERNEL); 805 if (!data->freq_table) { 806 result = -ENOMEM; 807 goto err_unreg; 808 } 809 810 /* detect transition latency */ 811 policy->cpuinfo.transition_latency = 0; 812 for (i = 0; i < perf->state_count; i++) { 813 if ((perf->states[i].transition_latency * 1000) > 814 policy->cpuinfo.transition_latency) 815 policy->cpuinfo.transition_latency = 816 perf->states[i].transition_latency * 1000; 817 } 818 819 /* Check for high latency (>20uS) from buggy BIOSes, like on T42 */ 820 if (perf->control_register.space_id == ACPI_ADR_SPACE_FIXED_HARDWARE && 821 policy->cpuinfo.transition_latency > 20 * 1000) { 822 policy->cpuinfo.transition_latency = 20 * 1000; 823 printk_once(KERN_INFO 824 "P-state transition latency capped at 20 uS\n"); 825 } 826 827 /* table init */ 828 for (i = 0; i < perf->state_count; i++) { 829 if (i > 0 && perf->states[i].core_frequency >= 830 data->freq_table[valid_states-1].frequency / 1000) 831 continue; 832 833 data->freq_table[valid_states].driver_data = i; 834 data->freq_table[valid_states].frequency = 835 perf->states[i].core_frequency * 1000; 836 valid_states++; 837 } 838 data->freq_table[valid_states].frequency = CPUFREQ_TABLE_END; 839 perf->state = 0; 840 841 result = cpufreq_frequency_table_cpuinfo(policy, data->freq_table); 842 if (result) 843 goto err_freqfree; 844 845 if (perf->states[0].core_frequency * 1000 != policy->cpuinfo.max_freq) 846 printk(KERN_WARNING FW_WARN "P-state 0 is not max freq\n"); 847 848 switch (perf->control_register.space_id) { 849 case ACPI_ADR_SPACE_SYSTEM_IO: 850 /* Current speed is unknown and not detectable by IO port */ 851 policy->cur = acpi_cpufreq_guess_freq(data, policy->cpu); 852 break; 853 case ACPI_ADR_SPACE_FIXED_HARDWARE: 854 acpi_cpufreq_driver.get = get_cur_freq_on_cpu; 855 policy->cur = get_cur_freq_on_cpu(cpu); 856 break; 857 default: 858 break; 859 } 860 861 /* notify BIOS that we exist */ 862 acpi_processor_notify_smm(THIS_MODULE); 863 864 /* Check for APERF/MPERF support in hardware */ 865 if (boot_cpu_has(X86_FEATURE_APERFMPERF)) 866 acpi_cpufreq_driver.getavg = cpufreq_get_measured_perf; 867 868 pr_debug("CPU%u - ACPI performance management activated.\n", cpu); 869 for (i = 0; i < perf->state_count; i++) 870 pr_debug(" %cP%d: %d MHz, %d mW, %d uS\n", 871 (i == perf->state ? '*' : ' '), i, 872 (u32) perf->states[i].core_frequency, 873 (u32) perf->states[i].power, 874 (u32) perf->states[i].transition_latency); 875 876 cpufreq_frequency_table_get_attr(data->freq_table, policy->cpu); 877 878 /* 879 * the first call to ->target() should result in us actually 880 * writing something to the appropriate registers. 881 */ 882 data->resume = 1; 883 884 return result; 885 886 err_freqfree: 887 kfree(data->freq_table); 888 err_unreg: 889 acpi_processor_unregister_performance(perf, cpu); 890 err_free_mask: 891 free_cpumask_var(data->freqdomain_cpus); 892 err_free: 893 kfree(data); 894 per_cpu(acfreq_data, cpu) = NULL; 895 896 return result; 897 } 898 899 static int acpi_cpufreq_cpu_exit(struct cpufreq_policy *policy) 900 { 901 struct acpi_cpufreq_data *data = per_cpu(acfreq_data, policy->cpu); 902 903 pr_debug("acpi_cpufreq_cpu_exit\n"); 904 905 if (data) { 906 cpufreq_frequency_table_put_attr(policy->cpu); 907 per_cpu(acfreq_data, policy->cpu) = NULL; 908 acpi_processor_unregister_performance(data->acpi_data, 909 policy->cpu); 910 free_cpumask_var(data->freqdomain_cpus); 911 kfree(data->freq_table); 912 kfree(data); 913 } 914 915 return 0; 916 } 917 918 static int acpi_cpufreq_resume(struct cpufreq_policy *policy) 919 { 920 struct acpi_cpufreq_data *data = per_cpu(acfreq_data, policy->cpu); 921 922 pr_debug("acpi_cpufreq_resume\n"); 923 924 data->resume = 1; 925 926 return 0; 927 } 928 929 static struct freq_attr *acpi_cpufreq_attr[] = { 930 &cpufreq_freq_attr_scaling_available_freqs, 931 &freqdomain_cpus, 932 NULL, /* this is a placeholder for cpb, do not remove */ 933 NULL, 934 }; 935 936 static struct cpufreq_driver acpi_cpufreq_driver = { 937 .verify = acpi_cpufreq_verify, 938 .target = acpi_cpufreq_target, 939 .bios_limit = acpi_processor_get_bios_limit, 940 .init = acpi_cpufreq_cpu_init, 941 .exit = acpi_cpufreq_cpu_exit, 942 .resume = acpi_cpufreq_resume, 943 .name = "acpi-cpufreq", 944 .owner = THIS_MODULE, 945 .attr = acpi_cpufreq_attr, 946 }; 947 948 static void __init acpi_cpufreq_boost_init(void) 949 { 950 if (boot_cpu_has(X86_FEATURE_CPB) || boot_cpu_has(X86_FEATURE_IDA)) { 951 msrs = msrs_alloc(); 952 953 if (!msrs) 954 return; 955 956 boost_supported = true; 957 boost_enabled = boost_state(0); 958 959 get_online_cpus(); 960 961 /* Force all MSRs to the same value */ 962 boost_set_msrs(boost_enabled, cpu_online_mask); 963 964 register_cpu_notifier(&boost_nb); 965 966 put_online_cpus(); 967 } else 968 global_boost.attr.mode = 0444; 969 970 /* We create the boost file in any case, though for systems without 971 * hardware support it will be read-only and hardwired to return 0. 972 */ 973 if (cpufreq_sysfs_create_file(&(global_boost.attr))) 974 pr_warn(PFX "could not register global boost sysfs file\n"); 975 else 976 pr_debug("registered global boost sysfs file\n"); 977 } 978 979 static void __exit acpi_cpufreq_boost_exit(void) 980 { 981 cpufreq_sysfs_remove_file(&(global_boost.attr)); 982 983 if (msrs) { 984 unregister_cpu_notifier(&boost_nb); 985 986 msrs_free(msrs); 987 msrs = NULL; 988 } 989 } 990 991 static int __init acpi_cpufreq_init(void) 992 { 993 int ret; 994 995 if (acpi_disabled) 996 return 0; 997 998 pr_debug("acpi_cpufreq_init\n"); 999 1000 ret = acpi_cpufreq_early_init(); 1001 if (ret) 1002 return ret; 1003 1004 #ifdef CONFIG_X86_ACPI_CPUFREQ_CPB 1005 /* this is a sysfs file with a strange name and an even stranger 1006 * semantic - per CPU instantiation, but system global effect. 1007 * Lets enable it only on AMD CPUs for compatibility reasons and 1008 * only if configured. This is considered legacy code, which 1009 * will probably be removed at some point in the future. 1010 */ 1011 if (check_amd_hwpstate_cpu(0)) { 1012 struct freq_attr **iter; 1013 1014 pr_debug("adding sysfs entry for cpb\n"); 1015 1016 for (iter = acpi_cpufreq_attr; *iter != NULL; iter++) 1017 ; 1018 1019 /* make sure there is a terminator behind it */ 1020 if (iter[1] == NULL) 1021 *iter = &cpb; 1022 } 1023 #endif 1024 1025 ret = cpufreq_register_driver(&acpi_cpufreq_driver); 1026 if (ret) 1027 free_acpi_perf_data(); 1028 else 1029 acpi_cpufreq_boost_init(); 1030 1031 return ret; 1032 } 1033 1034 static void __exit acpi_cpufreq_exit(void) 1035 { 1036 pr_debug("acpi_cpufreq_exit\n"); 1037 1038 acpi_cpufreq_boost_exit(); 1039 1040 cpufreq_unregister_driver(&acpi_cpufreq_driver); 1041 1042 free_acpi_perf_data(); 1043 } 1044 1045 module_param(acpi_pstate_strict, uint, 0644); 1046 MODULE_PARM_DESC(acpi_pstate_strict, 1047 "value 0 or non-zero. non-zero -> strict ACPI checks are " 1048 "performed during frequency changes."); 1049 1050 late_initcall(acpi_cpufreq_init); 1051 module_exit(acpi_cpufreq_exit); 1052 1053 static const struct x86_cpu_id acpi_cpufreq_ids[] = { 1054 X86_FEATURE_MATCH(X86_FEATURE_ACPI), 1055 X86_FEATURE_MATCH(X86_FEATURE_HW_PSTATE), 1056 {} 1057 }; 1058 MODULE_DEVICE_TABLE(x86cpu, acpi_cpufreq_ids); 1059 1060 MODULE_ALIAS("acpi"); 1061