1 /* 2 * intel_pstate.c: Native P state management for Intel processors 3 * 4 * (C) Copyright 2012 Intel Corporation 5 * Author: Dirk Brandewie <dirk.j.brandewie@intel.com> 6 * 7 * This program is free software; you can redistribute it and/or 8 * modify it under the terms of the GNU General Public License 9 * as published by the Free Software Foundation; version 2 10 * of the License. 11 */ 12 13 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 14 15 #include <linux/kernel.h> 16 #include <linux/kernel_stat.h> 17 #include <linux/module.h> 18 #include <linux/ktime.h> 19 #include <linux/hrtimer.h> 20 #include <linux/tick.h> 21 #include <linux/slab.h> 22 #include <linux/sched/cpufreq.h> 23 #include <linux/list.h> 24 #include <linux/cpu.h> 25 #include <linux/cpufreq.h> 26 #include <linux/sysfs.h> 27 #include <linux/types.h> 28 #include <linux/fs.h> 29 #include <linux/debugfs.h> 30 #include <linux/acpi.h> 31 #include <linux/vmalloc.h> 32 #include <trace/events/power.h> 33 34 #include <asm/div64.h> 35 #include <asm/msr.h> 36 #include <asm/cpu_device_id.h> 37 #include <asm/cpufeature.h> 38 #include <asm/intel-family.h> 39 40 #define INTEL_PSTATE_SAMPLING_INTERVAL (10 * NSEC_PER_MSEC) 41 42 #define INTEL_CPUFREQ_TRANSITION_LATENCY 20000 43 #define INTEL_CPUFREQ_TRANSITION_DELAY 500 44 45 #ifdef CONFIG_ACPI 46 #include <acpi/processor.h> 47 #include <acpi/cppc_acpi.h> 48 #endif 49 50 #define FRAC_BITS 8 51 #define int_tofp(X) ((int64_t)(X) << FRAC_BITS) 52 #define fp_toint(X) ((X) >> FRAC_BITS) 53 54 #define EXT_BITS 6 55 #define EXT_FRAC_BITS (EXT_BITS + FRAC_BITS) 56 #define fp_ext_toint(X) ((X) >> EXT_FRAC_BITS) 57 #define int_ext_tofp(X) ((int64_t)(X) << EXT_FRAC_BITS) 58 59 static inline int32_t mul_fp(int32_t x, int32_t y) 60 { 61 return ((int64_t)x * (int64_t)y) >> FRAC_BITS; 62 } 63 64 static inline int32_t div_fp(s64 x, s64 y) 65 { 66 return div64_s64((int64_t)x << FRAC_BITS, y); 67 } 68 69 static inline int ceiling_fp(int32_t x) 70 { 71 int mask, ret; 72 73 ret = fp_toint(x); 74 mask = (1 << FRAC_BITS) - 1; 75 if (x & mask) 76 ret += 1; 77 return ret; 78 } 79 80 static inline int32_t percent_fp(int percent) 81 { 82 return div_fp(percent, 100); 83 } 84 85 static inline u64 mul_ext_fp(u64 x, u64 y) 86 { 87 return (x * y) >> EXT_FRAC_BITS; 88 } 89 90 static inline u64 div_ext_fp(u64 x, u64 y) 91 { 92 return div64_u64(x << EXT_FRAC_BITS, y); 93 } 94 95 static inline int32_t percent_ext_fp(int percent) 96 { 97 return div_ext_fp(percent, 100); 98 } 99 100 /** 101 * struct sample - Store performance sample 102 * @core_avg_perf: Ratio of APERF/MPERF which is the actual average 103 * performance during last sample period 104 * @busy_scaled: Scaled busy value which is used to calculate next 105 * P state. This can be different than core_avg_perf 106 * to account for cpu idle period 107 * @aperf: Difference of actual performance frequency clock count 108 * read from APERF MSR between last and current sample 109 * @mperf: Difference of maximum performance frequency clock count 110 * read from MPERF MSR between last and current sample 111 * @tsc: Difference of time stamp counter between last and 112 * current sample 113 * @time: Current time from scheduler 114 * 115 * This structure is used in the cpudata structure to store performance sample 116 * data for choosing next P State. 117 */ 118 struct sample { 119 int32_t core_avg_perf; 120 int32_t busy_scaled; 121 u64 aperf; 122 u64 mperf; 123 u64 tsc; 124 u64 time; 125 }; 126 127 /** 128 * struct pstate_data - Store P state data 129 * @current_pstate: Current requested P state 130 * @min_pstate: Min P state possible for this platform 131 * @max_pstate: Max P state possible for this platform 132 * @max_pstate_physical:This is physical Max P state for a processor 133 * This can be higher than the max_pstate which can 134 * be limited by platform thermal design power limits 135 * @scaling: Scaling factor to convert frequency to cpufreq 136 * frequency units 137 * @turbo_pstate: Max Turbo P state possible for this platform 138 * @max_freq: @max_pstate frequency in cpufreq units 139 * @turbo_freq: @turbo_pstate frequency in cpufreq units 140 * 141 * Stores the per cpu model P state limits and current P state. 142 */ 143 struct pstate_data { 144 int current_pstate; 145 int min_pstate; 146 int max_pstate; 147 int max_pstate_physical; 148 int scaling; 149 int turbo_pstate; 150 unsigned int max_freq; 151 unsigned int turbo_freq; 152 }; 153 154 /** 155 * struct vid_data - Stores voltage information data 156 * @min: VID data for this platform corresponding to 157 * the lowest P state 158 * @max: VID data corresponding to the highest P State. 159 * @turbo: VID data for turbo P state 160 * @ratio: Ratio of (vid max - vid min) / 161 * (max P state - Min P State) 162 * 163 * Stores the voltage data for DVFS (Dynamic Voltage and Frequency Scaling) 164 * This data is used in Atom platforms, where in addition to target P state, 165 * the voltage data needs to be specified to select next P State. 166 */ 167 struct vid_data { 168 int min; 169 int max; 170 int turbo; 171 int32_t ratio; 172 }; 173 174 /** 175 * struct global_params - Global parameters, mostly tunable via sysfs. 176 * @no_turbo: Whether or not to use turbo P-states. 177 * @turbo_disabled: Whethet or not turbo P-states are available at all, 178 * based on the MSR_IA32_MISC_ENABLE value and whether or 179 * not the maximum reported turbo P-state is different from 180 * the maximum reported non-turbo one. 181 * @min_perf_pct: Minimum capacity limit in percent of the maximum turbo 182 * P-state capacity. 183 * @max_perf_pct: Maximum capacity limit in percent of the maximum turbo 184 * P-state capacity. 185 */ 186 struct global_params { 187 bool no_turbo; 188 bool turbo_disabled; 189 int max_perf_pct; 190 int min_perf_pct; 191 }; 192 193 /** 194 * struct cpudata - Per CPU instance data storage 195 * @cpu: CPU number for this instance data 196 * @policy: CPUFreq policy value 197 * @update_util: CPUFreq utility callback information 198 * @update_util_set: CPUFreq utility callback is set 199 * @iowait_boost: iowait-related boost fraction 200 * @last_update: Time of the last update. 201 * @pstate: Stores P state limits for this CPU 202 * @vid: Stores VID limits for this CPU 203 * @last_sample_time: Last Sample time 204 * @aperf_mperf_shift: Number of clock cycles after aperf, merf is incremented 205 * This shift is a multiplier to mperf delta to 206 * calculate CPU busy. 207 * @prev_aperf: Last APERF value read from APERF MSR 208 * @prev_mperf: Last MPERF value read from MPERF MSR 209 * @prev_tsc: Last timestamp counter (TSC) value 210 * @prev_cummulative_iowait: IO Wait time difference from last and 211 * current sample 212 * @sample: Storage for storing last Sample data 213 * @min_perf_ratio: Minimum capacity in terms of PERF or HWP ratios 214 * @max_perf_ratio: Maximum capacity in terms of PERF or HWP ratios 215 * @acpi_perf_data: Stores ACPI perf information read from _PSS 216 * @valid_pss_table: Set to true for valid ACPI _PSS entries found 217 * @epp_powersave: Last saved HWP energy performance preference 218 * (EPP) or energy performance bias (EPB), 219 * when policy switched to performance 220 * @epp_policy: Last saved policy used to set EPP/EPB 221 * @epp_default: Power on default HWP energy performance 222 * preference/bias 223 * @epp_saved: Saved EPP/EPB during system suspend or CPU offline 224 * operation 225 * 226 * This structure stores per CPU instance data for all CPUs. 227 */ 228 struct cpudata { 229 int cpu; 230 231 unsigned int policy; 232 struct update_util_data update_util; 233 bool update_util_set; 234 235 struct pstate_data pstate; 236 struct vid_data vid; 237 238 u64 last_update; 239 u64 last_sample_time; 240 u64 aperf_mperf_shift; 241 u64 prev_aperf; 242 u64 prev_mperf; 243 u64 prev_tsc; 244 u64 prev_cummulative_iowait; 245 struct sample sample; 246 int32_t min_perf_ratio; 247 int32_t max_perf_ratio; 248 #ifdef CONFIG_ACPI 249 struct acpi_processor_performance acpi_perf_data; 250 bool valid_pss_table; 251 #endif 252 unsigned int iowait_boost; 253 s16 epp_powersave; 254 s16 epp_policy; 255 s16 epp_default; 256 s16 epp_saved; 257 }; 258 259 static struct cpudata **all_cpu_data; 260 261 /** 262 * struct pstate_funcs - Per CPU model specific callbacks 263 * @get_max: Callback to get maximum non turbo effective P state 264 * @get_max_physical: Callback to get maximum non turbo physical P state 265 * @get_min: Callback to get minimum P state 266 * @get_turbo: Callback to get turbo P state 267 * @get_scaling: Callback to get frequency scaling factor 268 * @get_val: Callback to convert P state to actual MSR write value 269 * @get_vid: Callback to get VID data for Atom platforms 270 * 271 * Core and Atom CPU models have different way to get P State limits. This 272 * structure is used to store those callbacks. 273 */ 274 struct pstate_funcs { 275 int (*get_max)(void); 276 int (*get_max_physical)(void); 277 int (*get_min)(void); 278 int (*get_turbo)(void); 279 int (*get_scaling)(void); 280 int (*get_aperf_mperf_shift)(void); 281 u64 (*get_val)(struct cpudata*, int pstate); 282 void (*get_vid)(struct cpudata *); 283 }; 284 285 static struct pstate_funcs pstate_funcs __read_mostly; 286 287 static int hwp_active __read_mostly; 288 static bool per_cpu_limits __read_mostly; 289 290 static struct cpufreq_driver *intel_pstate_driver __read_mostly; 291 292 #ifdef CONFIG_ACPI 293 static bool acpi_ppc; 294 #endif 295 296 static struct global_params global; 297 298 static DEFINE_MUTEX(intel_pstate_driver_lock); 299 static DEFINE_MUTEX(intel_pstate_limits_lock); 300 301 #ifdef CONFIG_ACPI 302 303 static bool intel_pstate_get_ppc_enable_status(void) 304 { 305 if (acpi_gbl_FADT.preferred_profile == PM_ENTERPRISE_SERVER || 306 acpi_gbl_FADT.preferred_profile == PM_PERFORMANCE_SERVER) 307 return true; 308 309 return acpi_ppc; 310 } 311 312 #ifdef CONFIG_ACPI_CPPC_LIB 313 314 /* The work item is needed to avoid CPU hotplug locking issues */ 315 static void intel_pstste_sched_itmt_work_fn(struct work_struct *work) 316 { 317 sched_set_itmt_support(); 318 } 319 320 static DECLARE_WORK(sched_itmt_work, intel_pstste_sched_itmt_work_fn); 321 322 static void intel_pstate_set_itmt_prio(int cpu) 323 { 324 struct cppc_perf_caps cppc_perf; 325 static u32 max_highest_perf = 0, min_highest_perf = U32_MAX; 326 int ret; 327 328 ret = cppc_get_perf_caps(cpu, &cppc_perf); 329 if (ret) 330 return; 331 332 /* 333 * The priorities can be set regardless of whether or not 334 * sched_set_itmt_support(true) has been called and it is valid to 335 * update them at any time after it has been called. 336 */ 337 sched_set_itmt_core_prio(cppc_perf.highest_perf, cpu); 338 339 if (max_highest_perf <= min_highest_perf) { 340 if (cppc_perf.highest_perf > max_highest_perf) 341 max_highest_perf = cppc_perf.highest_perf; 342 343 if (cppc_perf.highest_perf < min_highest_perf) 344 min_highest_perf = cppc_perf.highest_perf; 345 346 if (max_highest_perf > min_highest_perf) { 347 /* 348 * This code can be run during CPU online under the 349 * CPU hotplug locks, so sched_set_itmt_support() 350 * cannot be called from here. Queue up a work item 351 * to invoke it. 352 */ 353 schedule_work(&sched_itmt_work); 354 } 355 } 356 } 357 #else 358 static void intel_pstate_set_itmt_prio(int cpu) 359 { 360 } 361 #endif 362 363 static void intel_pstate_init_acpi_perf_limits(struct cpufreq_policy *policy) 364 { 365 struct cpudata *cpu; 366 int ret; 367 int i; 368 369 if (hwp_active) { 370 intel_pstate_set_itmt_prio(policy->cpu); 371 return; 372 } 373 374 if (!intel_pstate_get_ppc_enable_status()) 375 return; 376 377 cpu = all_cpu_data[policy->cpu]; 378 379 ret = acpi_processor_register_performance(&cpu->acpi_perf_data, 380 policy->cpu); 381 if (ret) 382 return; 383 384 /* 385 * Check if the control value in _PSS is for PERF_CTL MSR, which should 386 * guarantee that the states returned by it map to the states in our 387 * list directly. 388 */ 389 if (cpu->acpi_perf_data.control_register.space_id != 390 ACPI_ADR_SPACE_FIXED_HARDWARE) 391 goto err; 392 393 /* 394 * If there is only one entry _PSS, simply ignore _PSS and continue as 395 * usual without taking _PSS into account 396 */ 397 if (cpu->acpi_perf_data.state_count < 2) 398 goto err; 399 400 pr_debug("CPU%u - ACPI _PSS perf data\n", policy->cpu); 401 for (i = 0; i < cpu->acpi_perf_data.state_count; i++) { 402 pr_debug(" %cP%d: %u MHz, %u mW, 0x%x\n", 403 (i == cpu->acpi_perf_data.state ? '*' : ' '), i, 404 (u32) cpu->acpi_perf_data.states[i].core_frequency, 405 (u32) cpu->acpi_perf_data.states[i].power, 406 (u32) cpu->acpi_perf_data.states[i].control); 407 } 408 409 /* 410 * The _PSS table doesn't contain whole turbo frequency range. 411 * This just contains +1 MHZ above the max non turbo frequency, 412 * with control value corresponding to max turbo ratio. But 413 * when cpufreq set policy is called, it will call with this 414 * max frequency, which will cause a reduced performance as 415 * this driver uses real max turbo frequency as the max 416 * frequency. So correct this frequency in _PSS table to 417 * correct max turbo frequency based on the turbo state. 418 * Also need to convert to MHz as _PSS freq is in MHz. 419 */ 420 if (!global.turbo_disabled) 421 cpu->acpi_perf_data.states[0].core_frequency = 422 policy->cpuinfo.max_freq / 1000; 423 cpu->valid_pss_table = true; 424 pr_debug("_PPC limits will be enforced\n"); 425 426 return; 427 428 err: 429 cpu->valid_pss_table = false; 430 acpi_processor_unregister_performance(policy->cpu); 431 } 432 433 static void intel_pstate_exit_perf_limits(struct cpufreq_policy *policy) 434 { 435 struct cpudata *cpu; 436 437 cpu = all_cpu_data[policy->cpu]; 438 if (!cpu->valid_pss_table) 439 return; 440 441 acpi_processor_unregister_performance(policy->cpu); 442 } 443 #else 444 static inline void intel_pstate_init_acpi_perf_limits(struct cpufreq_policy *policy) 445 { 446 } 447 448 static inline void intel_pstate_exit_perf_limits(struct cpufreq_policy *policy) 449 { 450 } 451 #endif 452 453 static inline void update_turbo_state(void) 454 { 455 u64 misc_en; 456 struct cpudata *cpu; 457 458 cpu = all_cpu_data[0]; 459 rdmsrl(MSR_IA32_MISC_ENABLE, misc_en); 460 global.turbo_disabled = 461 (misc_en & MSR_IA32_MISC_ENABLE_TURBO_DISABLE || 462 cpu->pstate.max_pstate == cpu->pstate.turbo_pstate); 463 } 464 465 static int min_perf_pct_min(void) 466 { 467 struct cpudata *cpu = all_cpu_data[0]; 468 int turbo_pstate = cpu->pstate.turbo_pstate; 469 470 return turbo_pstate ? 471 (cpu->pstate.min_pstate * 100 / turbo_pstate) : 0; 472 } 473 474 static s16 intel_pstate_get_epb(struct cpudata *cpu_data) 475 { 476 u64 epb; 477 int ret; 478 479 if (!static_cpu_has(X86_FEATURE_EPB)) 480 return -ENXIO; 481 482 ret = rdmsrl_on_cpu(cpu_data->cpu, MSR_IA32_ENERGY_PERF_BIAS, &epb); 483 if (ret) 484 return (s16)ret; 485 486 return (s16)(epb & 0x0f); 487 } 488 489 static s16 intel_pstate_get_epp(struct cpudata *cpu_data, u64 hwp_req_data) 490 { 491 s16 epp; 492 493 if (static_cpu_has(X86_FEATURE_HWP_EPP)) { 494 /* 495 * When hwp_req_data is 0, means that caller didn't read 496 * MSR_HWP_REQUEST, so need to read and get EPP. 497 */ 498 if (!hwp_req_data) { 499 epp = rdmsrl_on_cpu(cpu_data->cpu, MSR_HWP_REQUEST, 500 &hwp_req_data); 501 if (epp) 502 return epp; 503 } 504 epp = (hwp_req_data >> 24) & 0xff; 505 } else { 506 /* When there is no EPP present, HWP uses EPB settings */ 507 epp = intel_pstate_get_epb(cpu_data); 508 } 509 510 return epp; 511 } 512 513 static int intel_pstate_set_epb(int cpu, s16 pref) 514 { 515 u64 epb; 516 int ret; 517 518 if (!static_cpu_has(X86_FEATURE_EPB)) 519 return -ENXIO; 520 521 ret = rdmsrl_on_cpu(cpu, MSR_IA32_ENERGY_PERF_BIAS, &epb); 522 if (ret) 523 return ret; 524 525 epb = (epb & ~0x0f) | pref; 526 wrmsrl_on_cpu(cpu, MSR_IA32_ENERGY_PERF_BIAS, epb); 527 528 return 0; 529 } 530 531 /* 532 * EPP/EPB display strings corresponding to EPP index in the 533 * energy_perf_strings[] 534 * index String 535 *------------------------------------- 536 * 0 default 537 * 1 performance 538 * 2 balance_performance 539 * 3 balance_power 540 * 4 power 541 */ 542 static const char * const energy_perf_strings[] = { 543 "default", 544 "performance", 545 "balance_performance", 546 "balance_power", 547 "power", 548 NULL 549 }; 550 static const unsigned int epp_values[] = { 551 HWP_EPP_PERFORMANCE, 552 HWP_EPP_BALANCE_PERFORMANCE, 553 HWP_EPP_BALANCE_POWERSAVE, 554 HWP_EPP_POWERSAVE 555 }; 556 557 static int intel_pstate_get_energy_pref_index(struct cpudata *cpu_data) 558 { 559 s16 epp; 560 int index = -EINVAL; 561 562 epp = intel_pstate_get_epp(cpu_data, 0); 563 if (epp < 0) 564 return epp; 565 566 if (static_cpu_has(X86_FEATURE_HWP_EPP)) { 567 if (epp == HWP_EPP_PERFORMANCE) 568 return 1; 569 if (epp <= HWP_EPP_BALANCE_PERFORMANCE) 570 return 2; 571 if (epp <= HWP_EPP_BALANCE_POWERSAVE) 572 return 3; 573 else 574 return 4; 575 } else if (static_cpu_has(X86_FEATURE_EPB)) { 576 /* 577 * Range: 578 * 0x00-0x03 : Performance 579 * 0x04-0x07 : Balance performance 580 * 0x08-0x0B : Balance power 581 * 0x0C-0x0F : Power 582 * The EPB is a 4 bit value, but our ranges restrict the 583 * value which can be set. Here only using top two bits 584 * effectively. 585 */ 586 index = (epp >> 2) + 1; 587 } 588 589 return index; 590 } 591 592 static int intel_pstate_set_energy_pref_index(struct cpudata *cpu_data, 593 int pref_index) 594 { 595 int epp = -EINVAL; 596 int ret; 597 598 if (!pref_index) 599 epp = cpu_data->epp_default; 600 601 mutex_lock(&intel_pstate_limits_lock); 602 603 if (static_cpu_has(X86_FEATURE_HWP_EPP)) { 604 u64 value; 605 606 ret = rdmsrl_on_cpu(cpu_data->cpu, MSR_HWP_REQUEST, &value); 607 if (ret) 608 goto return_pref; 609 610 value &= ~GENMASK_ULL(31, 24); 611 612 if (epp == -EINVAL) 613 epp = epp_values[pref_index - 1]; 614 615 value |= (u64)epp << 24; 616 ret = wrmsrl_on_cpu(cpu_data->cpu, MSR_HWP_REQUEST, value); 617 } else { 618 if (epp == -EINVAL) 619 epp = (pref_index - 1) << 2; 620 ret = intel_pstate_set_epb(cpu_data->cpu, epp); 621 } 622 return_pref: 623 mutex_unlock(&intel_pstate_limits_lock); 624 625 return ret; 626 } 627 628 static ssize_t show_energy_performance_available_preferences( 629 struct cpufreq_policy *policy, char *buf) 630 { 631 int i = 0; 632 int ret = 0; 633 634 while (energy_perf_strings[i] != NULL) 635 ret += sprintf(&buf[ret], "%s ", energy_perf_strings[i++]); 636 637 ret += sprintf(&buf[ret], "\n"); 638 639 return ret; 640 } 641 642 cpufreq_freq_attr_ro(energy_performance_available_preferences); 643 644 static ssize_t store_energy_performance_preference( 645 struct cpufreq_policy *policy, const char *buf, size_t count) 646 { 647 struct cpudata *cpu_data = all_cpu_data[policy->cpu]; 648 char str_preference[21]; 649 int ret, i = 0; 650 651 ret = sscanf(buf, "%20s", str_preference); 652 if (ret != 1) 653 return -EINVAL; 654 655 while (energy_perf_strings[i] != NULL) { 656 if (!strcmp(str_preference, energy_perf_strings[i])) { 657 intel_pstate_set_energy_pref_index(cpu_data, i); 658 return count; 659 } 660 ++i; 661 } 662 663 return -EINVAL; 664 } 665 666 static ssize_t show_energy_performance_preference( 667 struct cpufreq_policy *policy, char *buf) 668 { 669 struct cpudata *cpu_data = all_cpu_data[policy->cpu]; 670 int preference; 671 672 preference = intel_pstate_get_energy_pref_index(cpu_data); 673 if (preference < 0) 674 return preference; 675 676 return sprintf(buf, "%s\n", energy_perf_strings[preference]); 677 } 678 679 cpufreq_freq_attr_rw(energy_performance_preference); 680 681 static struct freq_attr *hwp_cpufreq_attrs[] = { 682 &energy_performance_preference, 683 &energy_performance_available_preferences, 684 NULL, 685 }; 686 687 static void intel_pstate_get_hwp_max(unsigned int cpu, int *phy_max, 688 int *current_max) 689 { 690 u64 cap; 691 692 rdmsrl_on_cpu(cpu, MSR_HWP_CAPABILITIES, &cap); 693 if (global.no_turbo) 694 *current_max = HWP_GUARANTEED_PERF(cap); 695 else 696 *current_max = HWP_HIGHEST_PERF(cap); 697 698 *phy_max = HWP_HIGHEST_PERF(cap); 699 } 700 701 static void intel_pstate_hwp_set(unsigned int cpu) 702 { 703 struct cpudata *cpu_data = all_cpu_data[cpu]; 704 int max, min; 705 u64 value; 706 s16 epp; 707 708 max = cpu_data->max_perf_ratio; 709 min = cpu_data->min_perf_ratio; 710 711 if (cpu_data->policy == CPUFREQ_POLICY_PERFORMANCE) 712 min = max; 713 714 rdmsrl_on_cpu(cpu, MSR_HWP_REQUEST, &value); 715 716 value &= ~HWP_MIN_PERF(~0L); 717 value |= HWP_MIN_PERF(min); 718 719 value &= ~HWP_MAX_PERF(~0L); 720 value |= HWP_MAX_PERF(max); 721 722 if (cpu_data->epp_policy == cpu_data->policy) 723 goto skip_epp; 724 725 cpu_data->epp_policy = cpu_data->policy; 726 727 if (cpu_data->epp_saved >= 0) { 728 epp = cpu_data->epp_saved; 729 cpu_data->epp_saved = -EINVAL; 730 goto update_epp; 731 } 732 733 if (cpu_data->policy == CPUFREQ_POLICY_PERFORMANCE) { 734 epp = intel_pstate_get_epp(cpu_data, value); 735 cpu_data->epp_powersave = epp; 736 /* If EPP read was failed, then don't try to write */ 737 if (epp < 0) 738 goto skip_epp; 739 740 epp = 0; 741 } else { 742 /* skip setting EPP, when saved value is invalid */ 743 if (cpu_data->epp_powersave < 0) 744 goto skip_epp; 745 746 /* 747 * No need to restore EPP when it is not zero. This 748 * means: 749 * - Policy is not changed 750 * - user has manually changed 751 * - Error reading EPB 752 */ 753 epp = intel_pstate_get_epp(cpu_data, value); 754 if (epp) 755 goto skip_epp; 756 757 epp = cpu_data->epp_powersave; 758 } 759 update_epp: 760 if (static_cpu_has(X86_FEATURE_HWP_EPP)) { 761 value &= ~GENMASK_ULL(31, 24); 762 value |= (u64)epp << 24; 763 } else { 764 intel_pstate_set_epb(cpu, epp); 765 } 766 skip_epp: 767 wrmsrl_on_cpu(cpu, MSR_HWP_REQUEST, value); 768 } 769 770 static int intel_pstate_hwp_save_state(struct cpufreq_policy *policy) 771 { 772 struct cpudata *cpu_data = all_cpu_data[policy->cpu]; 773 774 if (!hwp_active) 775 return 0; 776 777 cpu_data->epp_saved = intel_pstate_get_epp(cpu_data, 0); 778 779 return 0; 780 } 781 782 static int intel_pstate_resume(struct cpufreq_policy *policy) 783 { 784 if (!hwp_active) 785 return 0; 786 787 mutex_lock(&intel_pstate_limits_lock); 788 789 all_cpu_data[policy->cpu]->epp_policy = 0; 790 intel_pstate_hwp_set(policy->cpu); 791 792 mutex_unlock(&intel_pstate_limits_lock); 793 794 return 0; 795 } 796 797 static void intel_pstate_update_policies(void) 798 { 799 int cpu; 800 801 for_each_possible_cpu(cpu) 802 cpufreq_update_policy(cpu); 803 } 804 805 /************************** sysfs begin ************************/ 806 #define show_one(file_name, object) \ 807 static ssize_t show_##file_name \ 808 (struct kobject *kobj, struct attribute *attr, char *buf) \ 809 { \ 810 return sprintf(buf, "%u\n", global.object); \ 811 } 812 813 static ssize_t intel_pstate_show_status(char *buf); 814 static int intel_pstate_update_status(const char *buf, size_t size); 815 816 static ssize_t show_status(struct kobject *kobj, 817 struct attribute *attr, char *buf) 818 { 819 ssize_t ret; 820 821 mutex_lock(&intel_pstate_driver_lock); 822 ret = intel_pstate_show_status(buf); 823 mutex_unlock(&intel_pstate_driver_lock); 824 825 return ret; 826 } 827 828 static ssize_t store_status(struct kobject *a, struct attribute *b, 829 const char *buf, size_t count) 830 { 831 char *p = memchr(buf, '\n', count); 832 int ret; 833 834 mutex_lock(&intel_pstate_driver_lock); 835 ret = intel_pstate_update_status(buf, p ? p - buf : count); 836 mutex_unlock(&intel_pstate_driver_lock); 837 838 return ret < 0 ? ret : count; 839 } 840 841 static ssize_t show_turbo_pct(struct kobject *kobj, 842 struct attribute *attr, char *buf) 843 { 844 struct cpudata *cpu; 845 int total, no_turbo, turbo_pct; 846 uint32_t turbo_fp; 847 848 mutex_lock(&intel_pstate_driver_lock); 849 850 if (!intel_pstate_driver) { 851 mutex_unlock(&intel_pstate_driver_lock); 852 return -EAGAIN; 853 } 854 855 cpu = all_cpu_data[0]; 856 857 total = cpu->pstate.turbo_pstate - cpu->pstate.min_pstate + 1; 858 no_turbo = cpu->pstate.max_pstate - cpu->pstate.min_pstate + 1; 859 turbo_fp = div_fp(no_turbo, total); 860 turbo_pct = 100 - fp_toint(mul_fp(turbo_fp, int_tofp(100))); 861 862 mutex_unlock(&intel_pstate_driver_lock); 863 864 return sprintf(buf, "%u\n", turbo_pct); 865 } 866 867 static ssize_t show_num_pstates(struct kobject *kobj, 868 struct attribute *attr, char *buf) 869 { 870 struct cpudata *cpu; 871 int total; 872 873 mutex_lock(&intel_pstate_driver_lock); 874 875 if (!intel_pstate_driver) { 876 mutex_unlock(&intel_pstate_driver_lock); 877 return -EAGAIN; 878 } 879 880 cpu = all_cpu_data[0]; 881 total = cpu->pstate.turbo_pstate - cpu->pstate.min_pstate + 1; 882 883 mutex_unlock(&intel_pstate_driver_lock); 884 885 return sprintf(buf, "%u\n", total); 886 } 887 888 static ssize_t show_no_turbo(struct kobject *kobj, 889 struct attribute *attr, char *buf) 890 { 891 ssize_t ret; 892 893 mutex_lock(&intel_pstate_driver_lock); 894 895 if (!intel_pstate_driver) { 896 mutex_unlock(&intel_pstate_driver_lock); 897 return -EAGAIN; 898 } 899 900 update_turbo_state(); 901 if (global.turbo_disabled) 902 ret = sprintf(buf, "%u\n", global.turbo_disabled); 903 else 904 ret = sprintf(buf, "%u\n", global.no_turbo); 905 906 mutex_unlock(&intel_pstate_driver_lock); 907 908 return ret; 909 } 910 911 static ssize_t store_no_turbo(struct kobject *a, struct attribute *b, 912 const char *buf, size_t count) 913 { 914 unsigned int input; 915 int ret; 916 917 ret = sscanf(buf, "%u", &input); 918 if (ret != 1) 919 return -EINVAL; 920 921 mutex_lock(&intel_pstate_driver_lock); 922 923 if (!intel_pstate_driver) { 924 mutex_unlock(&intel_pstate_driver_lock); 925 return -EAGAIN; 926 } 927 928 mutex_lock(&intel_pstate_limits_lock); 929 930 update_turbo_state(); 931 if (global.turbo_disabled) { 932 pr_warn("Turbo disabled by BIOS or unavailable on processor\n"); 933 mutex_unlock(&intel_pstate_limits_lock); 934 mutex_unlock(&intel_pstate_driver_lock); 935 return -EPERM; 936 } 937 938 global.no_turbo = clamp_t(int, input, 0, 1); 939 940 if (global.no_turbo) { 941 struct cpudata *cpu = all_cpu_data[0]; 942 int pct = cpu->pstate.max_pstate * 100 / cpu->pstate.turbo_pstate; 943 944 /* Squash the global minimum into the permitted range. */ 945 if (global.min_perf_pct > pct) 946 global.min_perf_pct = pct; 947 } 948 949 mutex_unlock(&intel_pstate_limits_lock); 950 951 intel_pstate_update_policies(); 952 953 mutex_unlock(&intel_pstate_driver_lock); 954 955 return count; 956 } 957 958 static ssize_t store_max_perf_pct(struct kobject *a, struct attribute *b, 959 const char *buf, size_t count) 960 { 961 unsigned int input; 962 int ret; 963 964 ret = sscanf(buf, "%u", &input); 965 if (ret != 1) 966 return -EINVAL; 967 968 mutex_lock(&intel_pstate_driver_lock); 969 970 if (!intel_pstate_driver) { 971 mutex_unlock(&intel_pstate_driver_lock); 972 return -EAGAIN; 973 } 974 975 mutex_lock(&intel_pstate_limits_lock); 976 977 global.max_perf_pct = clamp_t(int, input, global.min_perf_pct, 100); 978 979 mutex_unlock(&intel_pstate_limits_lock); 980 981 intel_pstate_update_policies(); 982 983 mutex_unlock(&intel_pstate_driver_lock); 984 985 return count; 986 } 987 988 static ssize_t store_min_perf_pct(struct kobject *a, struct attribute *b, 989 const char *buf, size_t count) 990 { 991 unsigned int input; 992 int ret; 993 994 ret = sscanf(buf, "%u", &input); 995 if (ret != 1) 996 return -EINVAL; 997 998 mutex_lock(&intel_pstate_driver_lock); 999 1000 if (!intel_pstate_driver) { 1001 mutex_unlock(&intel_pstate_driver_lock); 1002 return -EAGAIN; 1003 } 1004 1005 mutex_lock(&intel_pstate_limits_lock); 1006 1007 global.min_perf_pct = clamp_t(int, input, 1008 min_perf_pct_min(), global.max_perf_pct); 1009 1010 mutex_unlock(&intel_pstate_limits_lock); 1011 1012 intel_pstate_update_policies(); 1013 1014 mutex_unlock(&intel_pstate_driver_lock); 1015 1016 return count; 1017 } 1018 1019 show_one(max_perf_pct, max_perf_pct); 1020 show_one(min_perf_pct, min_perf_pct); 1021 1022 define_one_global_rw(status); 1023 define_one_global_rw(no_turbo); 1024 define_one_global_rw(max_perf_pct); 1025 define_one_global_rw(min_perf_pct); 1026 define_one_global_ro(turbo_pct); 1027 define_one_global_ro(num_pstates); 1028 1029 static struct attribute *intel_pstate_attributes[] = { 1030 &status.attr, 1031 &no_turbo.attr, 1032 &turbo_pct.attr, 1033 &num_pstates.attr, 1034 NULL 1035 }; 1036 1037 static const struct attribute_group intel_pstate_attr_group = { 1038 .attrs = intel_pstate_attributes, 1039 }; 1040 1041 static void __init intel_pstate_sysfs_expose_params(void) 1042 { 1043 struct kobject *intel_pstate_kobject; 1044 int rc; 1045 1046 intel_pstate_kobject = kobject_create_and_add("intel_pstate", 1047 &cpu_subsys.dev_root->kobj); 1048 if (WARN_ON(!intel_pstate_kobject)) 1049 return; 1050 1051 rc = sysfs_create_group(intel_pstate_kobject, &intel_pstate_attr_group); 1052 if (WARN_ON(rc)) 1053 return; 1054 1055 /* 1056 * If per cpu limits are enforced there are no global limits, so 1057 * return without creating max/min_perf_pct attributes 1058 */ 1059 if (per_cpu_limits) 1060 return; 1061 1062 rc = sysfs_create_file(intel_pstate_kobject, &max_perf_pct.attr); 1063 WARN_ON(rc); 1064 1065 rc = sysfs_create_file(intel_pstate_kobject, &min_perf_pct.attr); 1066 WARN_ON(rc); 1067 1068 } 1069 /************************** sysfs end ************************/ 1070 1071 static void intel_pstate_hwp_enable(struct cpudata *cpudata) 1072 { 1073 /* First disable HWP notification interrupt as we don't process them */ 1074 if (static_cpu_has(X86_FEATURE_HWP_NOTIFY)) 1075 wrmsrl_on_cpu(cpudata->cpu, MSR_HWP_INTERRUPT, 0x00); 1076 1077 wrmsrl_on_cpu(cpudata->cpu, MSR_PM_ENABLE, 0x1); 1078 cpudata->epp_policy = 0; 1079 if (cpudata->epp_default == -EINVAL) 1080 cpudata->epp_default = intel_pstate_get_epp(cpudata, 0); 1081 } 1082 1083 #define MSR_IA32_POWER_CTL_BIT_EE 19 1084 1085 /* Disable energy efficiency optimization */ 1086 static void intel_pstate_disable_ee(int cpu) 1087 { 1088 u64 power_ctl; 1089 int ret; 1090 1091 ret = rdmsrl_on_cpu(cpu, MSR_IA32_POWER_CTL, &power_ctl); 1092 if (ret) 1093 return; 1094 1095 if (!(power_ctl & BIT(MSR_IA32_POWER_CTL_BIT_EE))) { 1096 pr_info("Disabling energy efficiency optimization\n"); 1097 power_ctl |= BIT(MSR_IA32_POWER_CTL_BIT_EE); 1098 wrmsrl_on_cpu(cpu, MSR_IA32_POWER_CTL, power_ctl); 1099 } 1100 } 1101 1102 static int atom_get_min_pstate(void) 1103 { 1104 u64 value; 1105 1106 rdmsrl(MSR_ATOM_CORE_RATIOS, value); 1107 return (value >> 8) & 0x7F; 1108 } 1109 1110 static int atom_get_max_pstate(void) 1111 { 1112 u64 value; 1113 1114 rdmsrl(MSR_ATOM_CORE_RATIOS, value); 1115 return (value >> 16) & 0x7F; 1116 } 1117 1118 static int atom_get_turbo_pstate(void) 1119 { 1120 u64 value; 1121 1122 rdmsrl(MSR_ATOM_CORE_TURBO_RATIOS, value); 1123 return value & 0x7F; 1124 } 1125 1126 static u64 atom_get_val(struct cpudata *cpudata, int pstate) 1127 { 1128 u64 val; 1129 int32_t vid_fp; 1130 u32 vid; 1131 1132 val = (u64)pstate << 8; 1133 if (global.no_turbo && !global.turbo_disabled) 1134 val |= (u64)1 << 32; 1135 1136 vid_fp = cpudata->vid.min + mul_fp( 1137 int_tofp(pstate - cpudata->pstate.min_pstate), 1138 cpudata->vid.ratio); 1139 1140 vid_fp = clamp_t(int32_t, vid_fp, cpudata->vid.min, cpudata->vid.max); 1141 vid = ceiling_fp(vid_fp); 1142 1143 if (pstate > cpudata->pstate.max_pstate) 1144 vid = cpudata->vid.turbo; 1145 1146 return val | vid; 1147 } 1148 1149 static int silvermont_get_scaling(void) 1150 { 1151 u64 value; 1152 int i; 1153 /* Defined in Table 35-6 from SDM (Sept 2015) */ 1154 static int silvermont_freq_table[] = { 1155 83300, 100000, 133300, 116700, 80000}; 1156 1157 rdmsrl(MSR_FSB_FREQ, value); 1158 i = value & 0x7; 1159 WARN_ON(i > 4); 1160 1161 return silvermont_freq_table[i]; 1162 } 1163 1164 static int airmont_get_scaling(void) 1165 { 1166 u64 value; 1167 int i; 1168 /* Defined in Table 35-10 from SDM (Sept 2015) */ 1169 static int airmont_freq_table[] = { 1170 83300, 100000, 133300, 116700, 80000, 1171 93300, 90000, 88900, 87500}; 1172 1173 rdmsrl(MSR_FSB_FREQ, value); 1174 i = value & 0xF; 1175 WARN_ON(i > 8); 1176 1177 return airmont_freq_table[i]; 1178 } 1179 1180 static void atom_get_vid(struct cpudata *cpudata) 1181 { 1182 u64 value; 1183 1184 rdmsrl(MSR_ATOM_CORE_VIDS, value); 1185 cpudata->vid.min = int_tofp((value >> 8) & 0x7f); 1186 cpudata->vid.max = int_tofp((value >> 16) & 0x7f); 1187 cpudata->vid.ratio = div_fp( 1188 cpudata->vid.max - cpudata->vid.min, 1189 int_tofp(cpudata->pstate.max_pstate - 1190 cpudata->pstate.min_pstate)); 1191 1192 rdmsrl(MSR_ATOM_CORE_TURBO_VIDS, value); 1193 cpudata->vid.turbo = value & 0x7f; 1194 } 1195 1196 static int core_get_min_pstate(void) 1197 { 1198 u64 value; 1199 1200 rdmsrl(MSR_PLATFORM_INFO, value); 1201 return (value >> 40) & 0xFF; 1202 } 1203 1204 static int core_get_max_pstate_physical(void) 1205 { 1206 u64 value; 1207 1208 rdmsrl(MSR_PLATFORM_INFO, value); 1209 return (value >> 8) & 0xFF; 1210 } 1211 1212 static int core_get_tdp_ratio(u64 plat_info) 1213 { 1214 /* Check how many TDP levels present */ 1215 if (plat_info & 0x600000000) { 1216 u64 tdp_ctrl; 1217 u64 tdp_ratio; 1218 int tdp_msr; 1219 int err; 1220 1221 /* Get the TDP level (0, 1, 2) to get ratios */ 1222 err = rdmsrl_safe(MSR_CONFIG_TDP_CONTROL, &tdp_ctrl); 1223 if (err) 1224 return err; 1225 1226 /* TDP MSR are continuous starting at 0x648 */ 1227 tdp_msr = MSR_CONFIG_TDP_NOMINAL + (tdp_ctrl & 0x03); 1228 err = rdmsrl_safe(tdp_msr, &tdp_ratio); 1229 if (err) 1230 return err; 1231 1232 /* For level 1 and 2, bits[23:16] contain the ratio */ 1233 if (tdp_ctrl & 0x03) 1234 tdp_ratio >>= 16; 1235 1236 tdp_ratio &= 0xff; /* ratios are only 8 bits long */ 1237 pr_debug("tdp_ratio %x\n", (int)tdp_ratio); 1238 1239 return (int)tdp_ratio; 1240 } 1241 1242 return -ENXIO; 1243 } 1244 1245 static int core_get_max_pstate(void) 1246 { 1247 u64 tar; 1248 u64 plat_info; 1249 int max_pstate; 1250 int tdp_ratio; 1251 int err; 1252 1253 rdmsrl(MSR_PLATFORM_INFO, plat_info); 1254 max_pstate = (plat_info >> 8) & 0xFF; 1255 1256 tdp_ratio = core_get_tdp_ratio(plat_info); 1257 if (tdp_ratio <= 0) 1258 return max_pstate; 1259 1260 if (hwp_active) { 1261 /* Turbo activation ratio is not used on HWP platforms */ 1262 return tdp_ratio; 1263 } 1264 1265 err = rdmsrl_safe(MSR_TURBO_ACTIVATION_RATIO, &tar); 1266 if (!err) { 1267 int tar_levels; 1268 1269 /* Do some sanity checking for safety */ 1270 tar_levels = tar & 0xff; 1271 if (tdp_ratio - 1 == tar_levels) { 1272 max_pstate = tar_levels; 1273 pr_debug("max_pstate=TAC %x\n", max_pstate); 1274 } 1275 } 1276 1277 return max_pstate; 1278 } 1279 1280 static int core_get_turbo_pstate(void) 1281 { 1282 u64 value; 1283 int nont, ret; 1284 1285 rdmsrl(MSR_TURBO_RATIO_LIMIT, value); 1286 nont = core_get_max_pstate(); 1287 ret = (value) & 255; 1288 if (ret <= nont) 1289 ret = nont; 1290 return ret; 1291 } 1292 1293 static inline int core_get_scaling(void) 1294 { 1295 return 100000; 1296 } 1297 1298 static u64 core_get_val(struct cpudata *cpudata, int pstate) 1299 { 1300 u64 val; 1301 1302 val = (u64)pstate << 8; 1303 if (global.no_turbo && !global.turbo_disabled) 1304 val |= (u64)1 << 32; 1305 1306 return val; 1307 } 1308 1309 static int knl_get_aperf_mperf_shift(void) 1310 { 1311 return 10; 1312 } 1313 1314 static int knl_get_turbo_pstate(void) 1315 { 1316 u64 value; 1317 int nont, ret; 1318 1319 rdmsrl(MSR_TURBO_RATIO_LIMIT, value); 1320 nont = core_get_max_pstate(); 1321 ret = (((value) >> 8) & 0xFF); 1322 if (ret <= nont) 1323 ret = nont; 1324 return ret; 1325 } 1326 1327 static int intel_pstate_get_base_pstate(struct cpudata *cpu) 1328 { 1329 return global.no_turbo || global.turbo_disabled ? 1330 cpu->pstate.max_pstate : cpu->pstate.turbo_pstate; 1331 } 1332 1333 static void intel_pstate_set_pstate(struct cpudata *cpu, int pstate) 1334 { 1335 trace_cpu_frequency(pstate * cpu->pstate.scaling, cpu->cpu); 1336 cpu->pstate.current_pstate = pstate; 1337 /* 1338 * Generally, there is no guarantee that this code will always run on 1339 * the CPU being updated, so force the register update to run on the 1340 * right CPU. 1341 */ 1342 wrmsrl_on_cpu(cpu->cpu, MSR_IA32_PERF_CTL, 1343 pstate_funcs.get_val(cpu, pstate)); 1344 } 1345 1346 static void intel_pstate_set_min_pstate(struct cpudata *cpu) 1347 { 1348 intel_pstate_set_pstate(cpu, cpu->pstate.min_pstate); 1349 } 1350 1351 static void intel_pstate_max_within_limits(struct cpudata *cpu) 1352 { 1353 int pstate; 1354 1355 update_turbo_state(); 1356 pstate = intel_pstate_get_base_pstate(cpu); 1357 pstate = max(cpu->pstate.min_pstate, cpu->max_perf_ratio); 1358 intel_pstate_set_pstate(cpu, pstate); 1359 } 1360 1361 static void intel_pstate_get_cpu_pstates(struct cpudata *cpu) 1362 { 1363 cpu->pstate.min_pstate = pstate_funcs.get_min(); 1364 cpu->pstate.max_pstate = pstate_funcs.get_max(); 1365 cpu->pstate.max_pstate_physical = pstate_funcs.get_max_physical(); 1366 cpu->pstate.turbo_pstate = pstate_funcs.get_turbo(); 1367 cpu->pstate.scaling = pstate_funcs.get_scaling(); 1368 cpu->pstate.max_freq = cpu->pstate.max_pstate * cpu->pstate.scaling; 1369 cpu->pstate.turbo_freq = cpu->pstate.turbo_pstate * cpu->pstate.scaling; 1370 1371 if (pstate_funcs.get_aperf_mperf_shift) 1372 cpu->aperf_mperf_shift = pstate_funcs.get_aperf_mperf_shift(); 1373 1374 if (pstate_funcs.get_vid) 1375 pstate_funcs.get_vid(cpu); 1376 1377 intel_pstate_set_min_pstate(cpu); 1378 } 1379 1380 static inline void intel_pstate_calc_avg_perf(struct cpudata *cpu) 1381 { 1382 struct sample *sample = &cpu->sample; 1383 1384 sample->core_avg_perf = div_ext_fp(sample->aperf, sample->mperf); 1385 } 1386 1387 static inline bool intel_pstate_sample(struct cpudata *cpu, u64 time) 1388 { 1389 u64 aperf, mperf; 1390 unsigned long flags; 1391 u64 tsc; 1392 1393 local_irq_save(flags); 1394 rdmsrl(MSR_IA32_APERF, aperf); 1395 rdmsrl(MSR_IA32_MPERF, mperf); 1396 tsc = rdtsc(); 1397 if (cpu->prev_mperf == mperf || cpu->prev_tsc == tsc) { 1398 local_irq_restore(flags); 1399 return false; 1400 } 1401 local_irq_restore(flags); 1402 1403 cpu->last_sample_time = cpu->sample.time; 1404 cpu->sample.time = time; 1405 cpu->sample.aperf = aperf; 1406 cpu->sample.mperf = mperf; 1407 cpu->sample.tsc = tsc; 1408 cpu->sample.aperf -= cpu->prev_aperf; 1409 cpu->sample.mperf -= cpu->prev_mperf; 1410 cpu->sample.tsc -= cpu->prev_tsc; 1411 1412 cpu->prev_aperf = aperf; 1413 cpu->prev_mperf = mperf; 1414 cpu->prev_tsc = tsc; 1415 /* 1416 * First time this function is invoked in a given cycle, all of the 1417 * previous sample data fields are equal to zero or stale and they must 1418 * be populated with meaningful numbers for things to work, so assume 1419 * that sample.time will always be reset before setting the utilization 1420 * update hook and make the caller skip the sample then. 1421 */ 1422 if (cpu->last_sample_time) { 1423 intel_pstate_calc_avg_perf(cpu); 1424 return true; 1425 } 1426 return false; 1427 } 1428 1429 static inline int32_t get_avg_frequency(struct cpudata *cpu) 1430 { 1431 return mul_ext_fp(cpu->sample.core_avg_perf, 1432 cpu->pstate.max_pstate_physical * cpu->pstate.scaling); 1433 } 1434 1435 static inline int32_t get_avg_pstate(struct cpudata *cpu) 1436 { 1437 return mul_ext_fp(cpu->pstate.max_pstate_physical, 1438 cpu->sample.core_avg_perf); 1439 } 1440 1441 static inline int32_t get_target_pstate(struct cpudata *cpu) 1442 { 1443 struct sample *sample = &cpu->sample; 1444 int32_t busy_frac, boost; 1445 int target, avg_pstate; 1446 1447 busy_frac = div_fp(sample->mperf << cpu->aperf_mperf_shift, 1448 sample->tsc); 1449 1450 boost = cpu->iowait_boost; 1451 cpu->iowait_boost >>= 1; 1452 1453 if (busy_frac < boost) 1454 busy_frac = boost; 1455 1456 sample->busy_scaled = busy_frac * 100; 1457 1458 target = global.no_turbo || global.turbo_disabled ? 1459 cpu->pstate.max_pstate : cpu->pstate.turbo_pstate; 1460 target += target >> 2; 1461 target = mul_fp(target, busy_frac); 1462 if (target < cpu->pstate.min_pstate) 1463 target = cpu->pstate.min_pstate; 1464 1465 /* 1466 * If the average P-state during the previous cycle was higher than the 1467 * current target, add 50% of the difference to the target to reduce 1468 * possible performance oscillations and offset possible performance 1469 * loss related to moving the workload from one CPU to another within 1470 * a package/module. 1471 */ 1472 avg_pstate = get_avg_pstate(cpu); 1473 if (avg_pstate > target) 1474 target += (avg_pstate - target) >> 1; 1475 1476 return target; 1477 } 1478 1479 static int intel_pstate_prepare_request(struct cpudata *cpu, int pstate) 1480 { 1481 int max_pstate = intel_pstate_get_base_pstate(cpu); 1482 int min_pstate; 1483 1484 min_pstate = max(cpu->pstate.min_pstate, cpu->min_perf_ratio); 1485 max_pstate = max(min_pstate, cpu->max_perf_ratio); 1486 return clamp_t(int, pstate, min_pstate, max_pstate); 1487 } 1488 1489 static void intel_pstate_update_pstate(struct cpudata *cpu, int pstate) 1490 { 1491 if (pstate == cpu->pstate.current_pstate) 1492 return; 1493 1494 cpu->pstate.current_pstate = pstate; 1495 wrmsrl(MSR_IA32_PERF_CTL, pstate_funcs.get_val(cpu, pstate)); 1496 } 1497 1498 static void intel_pstate_adjust_pstate(struct cpudata *cpu) 1499 { 1500 int from = cpu->pstate.current_pstate; 1501 struct sample *sample; 1502 int target_pstate; 1503 1504 update_turbo_state(); 1505 1506 target_pstate = get_target_pstate(cpu); 1507 target_pstate = intel_pstate_prepare_request(cpu, target_pstate); 1508 trace_cpu_frequency(target_pstate * cpu->pstate.scaling, cpu->cpu); 1509 intel_pstate_update_pstate(cpu, target_pstate); 1510 1511 sample = &cpu->sample; 1512 trace_pstate_sample(mul_ext_fp(100, sample->core_avg_perf), 1513 fp_toint(sample->busy_scaled), 1514 from, 1515 cpu->pstate.current_pstate, 1516 sample->mperf, 1517 sample->aperf, 1518 sample->tsc, 1519 get_avg_frequency(cpu), 1520 fp_toint(cpu->iowait_boost * 100)); 1521 } 1522 1523 static void intel_pstate_update_util(struct update_util_data *data, u64 time, 1524 unsigned int flags) 1525 { 1526 struct cpudata *cpu = container_of(data, struct cpudata, update_util); 1527 u64 delta_ns; 1528 1529 if (flags & SCHED_CPUFREQ_IOWAIT) { 1530 cpu->iowait_boost = int_tofp(1); 1531 cpu->last_update = time; 1532 /* 1533 * The last time the busy was 100% so P-state was max anyway 1534 * so avoid overhead of computation. 1535 */ 1536 if (fp_toint(cpu->sample.busy_scaled) == 100) 1537 return; 1538 1539 goto set_pstate; 1540 } else if (cpu->iowait_boost) { 1541 /* Clear iowait_boost if the CPU may have been idle. */ 1542 delta_ns = time - cpu->last_update; 1543 if (delta_ns > TICK_NSEC) 1544 cpu->iowait_boost = 0; 1545 } 1546 cpu->last_update = time; 1547 delta_ns = time - cpu->sample.time; 1548 if ((s64)delta_ns < INTEL_PSTATE_SAMPLING_INTERVAL) 1549 return; 1550 1551 set_pstate: 1552 if (intel_pstate_sample(cpu, time)) 1553 intel_pstate_adjust_pstate(cpu); 1554 } 1555 1556 static struct pstate_funcs core_funcs = { 1557 .get_max = core_get_max_pstate, 1558 .get_max_physical = core_get_max_pstate_physical, 1559 .get_min = core_get_min_pstate, 1560 .get_turbo = core_get_turbo_pstate, 1561 .get_scaling = core_get_scaling, 1562 .get_val = core_get_val, 1563 }; 1564 1565 static const struct pstate_funcs silvermont_funcs = { 1566 .get_max = atom_get_max_pstate, 1567 .get_max_physical = atom_get_max_pstate, 1568 .get_min = atom_get_min_pstate, 1569 .get_turbo = atom_get_turbo_pstate, 1570 .get_val = atom_get_val, 1571 .get_scaling = silvermont_get_scaling, 1572 .get_vid = atom_get_vid, 1573 }; 1574 1575 static const struct pstate_funcs airmont_funcs = { 1576 .get_max = atom_get_max_pstate, 1577 .get_max_physical = atom_get_max_pstate, 1578 .get_min = atom_get_min_pstate, 1579 .get_turbo = atom_get_turbo_pstate, 1580 .get_val = atom_get_val, 1581 .get_scaling = airmont_get_scaling, 1582 .get_vid = atom_get_vid, 1583 }; 1584 1585 static const struct pstate_funcs knl_funcs = { 1586 .get_max = core_get_max_pstate, 1587 .get_max_physical = core_get_max_pstate_physical, 1588 .get_min = core_get_min_pstate, 1589 .get_turbo = knl_get_turbo_pstate, 1590 .get_aperf_mperf_shift = knl_get_aperf_mperf_shift, 1591 .get_scaling = core_get_scaling, 1592 .get_val = core_get_val, 1593 }; 1594 1595 static const struct pstate_funcs bxt_funcs = { 1596 .get_max = core_get_max_pstate, 1597 .get_max_physical = core_get_max_pstate_physical, 1598 .get_min = core_get_min_pstate, 1599 .get_turbo = core_get_turbo_pstate, 1600 .get_scaling = core_get_scaling, 1601 .get_val = core_get_val, 1602 }; 1603 1604 #define ICPU(model, policy) \ 1605 { X86_VENDOR_INTEL, 6, model, X86_FEATURE_APERFMPERF,\ 1606 (unsigned long)&policy } 1607 1608 static const struct x86_cpu_id intel_pstate_cpu_ids[] = { 1609 ICPU(INTEL_FAM6_SANDYBRIDGE, core_funcs), 1610 ICPU(INTEL_FAM6_SANDYBRIDGE_X, core_funcs), 1611 ICPU(INTEL_FAM6_ATOM_SILVERMONT1, silvermont_funcs), 1612 ICPU(INTEL_FAM6_IVYBRIDGE, core_funcs), 1613 ICPU(INTEL_FAM6_HASWELL_CORE, core_funcs), 1614 ICPU(INTEL_FAM6_BROADWELL_CORE, core_funcs), 1615 ICPU(INTEL_FAM6_IVYBRIDGE_X, core_funcs), 1616 ICPU(INTEL_FAM6_HASWELL_X, core_funcs), 1617 ICPU(INTEL_FAM6_HASWELL_ULT, core_funcs), 1618 ICPU(INTEL_FAM6_HASWELL_GT3E, core_funcs), 1619 ICPU(INTEL_FAM6_BROADWELL_GT3E, core_funcs), 1620 ICPU(INTEL_FAM6_ATOM_AIRMONT, airmont_funcs), 1621 ICPU(INTEL_FAM6_SKYLAKE_MOBILE, core_funcs), 1622 ICPU(INTEL_FAM6_BROADWELL_X, core_funcs), 1623 ICPU(INTEL_FAM6_SKYLAKE_DESKTOP, core_funcs), 1624 ICPU(INTEL_FAM6_BROADWELL_XEON_D, core_funcs), 1625 ICPU(INTEL_FAM6_XEON_PHI_KNL, knl_funcs), 1626 ICPU(INTEL_FAM6_XEON_PHI_KNM, knl_funcs), 1627 ICPU(INTEL_FAM6_ATOM_GOLDMONT, bxt_funcs), 1628 ICPU(INTEL_FAM6_ATOM_GEMINI_LAKE, bxt_funcs), 1629 {} 1630 }; 1631 MODULE_DEVICE_TABLE(x86cpu, intel_pstate_cpu_ids); 1632 1633 static const struct x86_cpu_id intel_pstate_cpu_oob_ids[] __initconst = { 1634 ICPU(INTEL_FAM6_BROADWELL_XEON_D, core_funcs), 1635 ICPU(INTEL_FAM6_BROADWELL_X, core_funcs), 1636 ICPU(INTEL_FAM6_SKYLAKE_X, core_funcs), 1637 {} 1638 }; 1639 1640 static const struct x86_cpu_id intel_pstate_cpu_ee_disable_ids[] = { 1641 ICPU(INTEL_FAM6_KABYLAKE_DESKTOP, core_funcs), 1642 {} 1643 }; 1644 1645 static int intel_pstate_init_cpu(unsigned int cpunum) 1646 { 1647 struct cpudata *cpu; 1648 1649 cpu = all_cpu_data[cpunum]; 1650 1651 if (!cpu) { 1652 cpu = kzalloc(sizeof(*cpu), GFP_KERNEL); 1653 if (!cpu) 1654 return -ENOMEM; 1655 1656 all_cpu_data[cpunum] = cpu; 1657 1658 cpu->epp_default = -EINVAL; 1659 cpu->epp_powersave = -EINVAL; 1660 cpu->epp_saved = -EINVAL; 1661 } 1662 1663 cpu = all_cpu_data[cpunum]; 1664 1665 cpu->cpu = cpunum; 1666 1667 if (hwp_active) { 1668 const struct x86_cpu_id *id; 1669 1670 id = x86_match_cpu(intel_pstate_cpu_ee_disable_ids); 1671 if (id) 1672 intel_pstate_disable_ee(cpunum); 1673 1674 intel_pstate_hwp_enable(cpu); 1675 } 1676 1677 intel_pstate_get_cpu_pstates(cpu); 1678 1679 pr_debug("controlling: cpu %d\n", cpunum); 1680 1681 return 0; 1682 } 1683 1684 static void intel_pstate_set_update_util_hook(unsigned int cpu_num) 1685 { 1686 struct cpudata *cpu = all_cpu_data[cpu_num]; 1687 1688 if (hwp_active) 1689 return; 1690 1691 if (cpu->update_util_set) 1692 return; 1693 1694 /* Prevent intel_pstate_update_util() from using stale data. */ 1695 cpu->sample.time = 0; 1696 cpufreq_add_update_util_hook(cpu_num, &cpu->update_util, 1697 intel_pstate_update_util); 1698 cpu->update_util_set = true; 1699 } 1700 1701 static void intel_pstate_clear_update_util_hook(unsigned int cpu) 1702 { 1703 struct cpudata *cpu_data = all_cpu_data[cpu]; 1704 1705 if (!cpu_data->update_util_set) 1706 return; 1707 1708 cpufreq_remove_update_util_hook(cpu); 1709 cpu_data->update_util_set = false; 1710 synchronize_sched(); 1711 } 1712 1713 static int intel_pstate_get_max_freq(struct cpudata *cpu) 1714 { 1715 return global.turbo_disabled || global.no_turbo ? 1716 cpu->pstate.max_freq : cpu->pstate.turbo_freq; 1717 } 1718 1719 static void intel_pstate_update_perf_limits(struct cpufreq_policy *policy, 1720 struct cpudata *cpu) 1721 { 1722 int max_freq = intel_pstate_get_max_freq(cpu); 1723 int32_t max_policy_perf, min_policy_perf; 1724 int max_state, turbo_max; 1725 1726 /* 1727 * HWP needs some special consideration, because on BDX the 1728 * HWP_REQUEST uses abstract value to represent performance 1729 * rather than pure ratios. 1730 */ 1731 if (hwp_active) { 1732 intel_pstate_get_hwp_max(cpu->cpu, &turbo_max, &max_state); 1733 } else { 1734 max_state = intel_pstate_get_base_pstate(cpu); 1735 turbo_max = cpu->pstate.turbo_pstate; 1736 } 1737 1738 max_policy_perf = max_state * policy->max / max_freq; 1739 if (policy->max == policy->min) { 1740 min_policy_perf = max_policy_perf; 1741 } else { 1742 min_policy_perf = max_state * policy->min / max_freq; 1743 min_policy_perf = clamp_t(int32_t, min_policy_perf, 1744 0, max_policy_perf); 1745 } 1746 1747 pr_debug("cpu:%d max_state %d min_policy_perf:%d max_policy_perf:%d\n", 1748 policy->cpu, max_state, 1749 min_policy_perf, max_policy_perf); 1750 1751 /* Normalize user input to [min_perf, max_perf] */ 1752 if (per_cpu_limits) { 1753 cpu->min_perf_ratio = min_policy_perf; 1754 cpu->max_perf_ratio = max_policy_perf; 1755 } else { 1756 int32_t global_min, global_max; 1757 1758 /* Global limits are in percent of the maximum turbo P-state. */ 1759 global_max = DIV_ROUND_UP(turbo_max * global.max_perf_pct, 100); 1760 global_min = DIV_ROUND_UP(turbo_max * global.min_perf_pct, 100); 1761 global_min = clamp_t(int32_t, global_min, 0, global_max); 1762 1763 pr_debug("cpu:%d global_min:%d global_max:%d\n", policy->cpu, 1764 global_min, global_max); 1765 1766 cpu->min_perf_ratio = max(min_policy_perf, global_min); 1767 cpu->min_perf_ratio = min(cpu->min_perf_ratio, max_policy_perf); 1768 cpu->max_perf_ratio = min(max_policy_perf, global_max); 1769 cpu->max_perf_ratio = max(min_policy_perf, cpu->max_perf_ratio); 1770 1771 /* Make sure min_perf <= max_perf */ 1772 cpu->min_perf_ratio = min(cpu->min_perf_ratio, 1773 cpu->max_perf_ratio); 1774 1775 } 1776 pr_debug("cpu:%d max_perf_ratio:%d min_perf_ratio:%d\n", policy->cpu, 1777 cpu->max_perf_ratio, 1778 cpu->min_perf_ratio); 1779 } 1780 1781 static int intel_pstate_set_policy(struct cpufreq_policy *policy) 1782 { 1783 struct cpudata *cpu; 1784 1785 if (!policy->cpuinfo.max_freq) 1786 return -ENODEV; 1787 1788 pr_debug("set_policy cpuinfo.max %u policy->max %u\n", 1789 policy->cpuinfo.max_freq, policy->max); 1790 1791 cpu = all_cpu_data[policy->cpu]; 1792 cpu->policy = policy->policy; 1793 1794 mutex_lock(&intel_pstate_limits_lock); 1795 1796 intel_pstate_update_perf_limits(policy, cpu); 1797 1798 if (cpu->policy == CPUFREQ_POLICY_PERFORMANCE) { 1799 /* 1800 * NOHZ_FULL CPUs need this as the governor callback may not 1801 * be invoked on them. 1802 */ 1803 intel_pstate_clear_update_util_hook(policy->cpu); 1804 intel_pstate_max_within_limits(cpu); 1805 } else { 1806 intel_pstate_set_update_util_hook(policy->cpu); 1807 } 1808 1809 if (hwp_active) 1810 intel_pstate_hwp_set(policy->cpu); 1811 1812 mutex_unlock(&intel_pstate_limits_lock); 1813 1814 return 0; 1815 } 1816 1817 static void intel_pstate_adjust_policy_max(struct cpufreq_policy *policy, 1818 struct cpudata *cpu) 1819 { 1820 if (cpu->pstate.max_pstate_physical > cpu->pstate.max_pstate && 1821 policy->max < policy->cpuinfo.max_freq && 1822 policy->max > cpu->pstate.max_freq) { 1823 pr_debug("policy->max > max non turbo frequency\n"); 1824 policy->max = policy->cpuinfo.max_freq; 1825 } 1826 } 1827 1828 static int intel_pstate_verify_policy(struct cpufreq_policy *policy) 1829 { 1830 struct cpudata *cpu = all_cpu_data[policy->cpu]; 1831 1832 update_turbo_state(); 1833 cpufreq_verify_within_limits(policy, policy->cpuinfo.min_freq, 1834 intel_pstate_get_max_freq(cpu)); 1835 1836 if (policy->policy != CPUFREQ_POLICY_POWERSAVE && 1837 policy->policy != CPUFREQ_POLICY_PERFORMANCE) 1838 return -EINVAL; 1839 1840 intel_pstate_adjust_policy_max(policy, cpu); 1841 1842 return 0; 1843 } 1844 1845 static void intel_cpufreq_stop_cpu(struct cpufreq_policy *policy) 1846 { 1847 intel_pstate_set_min_pstate(all_cpu_data[policy->cpu]); 1848 } 1849 1850 static void intel_pstate_stop_cpu(struct cpufreq_policy *policy) 1851 { 1852 pr_debug("CPU %d exiting\n", policy->cpu); 1853 1854 intel_pstate_clear_update_util_hook(policy->cpu); 1855 if (hwp_active) 1856 intel_pstate_hwp_save_state(policy); 1857 else 1858 intel_cpufreq_stop_cpu(policy); 1859 } 1860 1861 static int intel_pstate_cpu_exit(struct cpufreq_policy *policy) 1862 { 1863 intel_pstate_exit_perf_limits(policy); 1864 1865 policy->fast_switch_possible = false; 1866 1867 return 0; 1868 } 1869 1870 static int __intel_pstate_cpu_init(struct cpufreq_policy *policy) 1871 { 1872 struct cpudata *cpu; 1873 int rc; 1874 1875 rc = intel_pstate_init_cpu(policy->cpu); 1876 if (rc) 1877 return rc; 1878 1879 cpu = all_cpu_data[policy->cpu]; 1880 1881 cpu->max_perf_ratio = 0xFF; 1882 cpu->min_perf_ratio = 0; 1883 1884 policy->min = cpu->pstate.min_pstate * cpu->pstate.scaling; 1885 policy->max = cpu->pstate.turbo_pstate * cpu->pstate.scaling; 1886 1887 /* cpuinfo and default policy values */ 1888 policy->cpuinfo.min_freq = cpu->pstate.min_pstate * cpu->pstate.scaling; 1889 update_turbo_state(); 1890 policy->cpuinfo.max_freq = global.turbo_disabled ? 1891 cpu->pstate.max_pstate : cpu->pstate.turbo_pstate; 1892 policy->cpuinfo.max_freq *= cpu->pstate.scaling; 1893 1894 intel_pstate_init_acpi_perf_limits(policy); 1895 cpumask_set_cpu(policy->cpu, policy->cpus); 1896 1897 policy->fast_switch_possible = true; 1898 1899 return 0; 1900 } 1901 1902 static int intel_pstate_cpu_init(struct cpufreq_policy *policy) 1903 { 1904 int ret = __intel_pstate_cpu_init(policy); 1905 1906 if (ret) 1907 return ret; 1908 1909 policy->cpuinfo.transition_latency = CPUFREQ_ETERNAL; 1910 if (IS_ENABLED(CONFIG_CPU_FREQ_DEFAULT_GOV_PERFORMANCE)) 1911 policy->policy = CPUFREQ_POLICY_PERFORMANCE; 1912 else 1913 policy->policy = CPUFREQ_POLICY_POWERSAVE; 1914 1915 return 0; 1916 } 1917 1918 static struct cpufreq_driver intel_pstate = { 1919 .flags = CPUFREQ_CONST_LOOPS, 1920 .verify = intel_pstate_verify_policy, 1921 .setpolicy = intel_pstate_set_policy, 1922 .suspend = intel_pstate_hwp_save_state, 1923 .resume = intel_pstate_resume, 1924 .init = intel_pstate_cpu_init, 1925 .exit = intel_pstate_cpu_exit, 1926 .stop_cpu = intel_pstate_stop_cpu, 1927 .name = "intel_pstate", 1928 }; 1929 1930 static int intel_cpufreq_verify_policy(struct cpufreq_policy *policy) 1931 { 1932 struct cpudata *cpu = all_cpu_data[policy->cpu]; 1933 1934 update_turbo_state(); 1935 cpufreq_verify_within_limits(policy, policy->cpuinfo.min_freq, 1936 intel_pstate_get_max_freq(cpu)); 1937 1938 intel_pstate_adjust_policy_max(policy, cpu); 1939 1940 intel_pstate_update_perf_limits(policy, cpu); 1941 1942 return 0; 1943 } 1944 1945 static int intel_cpufreq_target(struct cpufreq_policy *policy, 1946 unsigned int target_freq, 1947 unsigned int relation) 1948 { 1949 struct cpudata *cpu = all_cpu_data[policy->cpu]; 1950 struct cpufreq_freqs freqs; 1951 int target_pstate; 1952 1953 update_turbo_state(); 1954 1955 freqs.old = policy->cur; 1956 freqs.new = target_freq; 1957 1958 cpufreq_freq_transition_begin(policy, &freqs); 1959 switch (relation) { 1960 case CPUFREQ_RELATION_L: 1961 target_pstate = DIV_ROUND_UP(freqs.new, cpu->pstate.scaling); 1962 break; 1963 case CPUFREQ_RELATION_H: 1964 target_pstate = freqs.new / cpu->pstate.scaling; 1965 break; 1966 default: 1967 target_pstate = DIV_ROUND_CLOSEST(freqs.new, cpu->pstate.scaling); 1968 break; 1969 } 1970 target_pstate = intel_pstate_prepare_request(cpu, target_pstate); 1971 if (target_pstate != cpu->pstate.current_pstate) { 1972 cpu->pstate.current_pstate = target_pstate; 1973 wrmsrl_on_cpu(policy->cpu, MSR_IA32_PERF_CTL, 1974 pstate_funcs.get_val(cpu, target_pstate)); 1975 } 1976 freqs.new = target_pstate * cpu->pstate.scaling; 1977 cpufreq_freq_transition_end(policy, &freqs, false); 1978 1979 return 0; 1980 } 1981 1982 static unsigned int intel_cpufreq_fast_switch(struct cpufreq_policy *policy, 1983 unsigned int target_freq) 1984 { 1985 struct cpudata *cpu = all_cpu_data[policy->cpu]; 1986 int target_pstate; 1987 1988 update_turbo_state(); 1989 1990 target_pstate = DIV_ROUND_UP(target_freq, cpu->pstate.scaling); 1991 target_pstate = intel_pstate_prepare_request(cpu, target_pstate); 1992 intel_pstate_update_pstate(cpu, target_pstate); 1993 return target_pstate * cpu->pstate.scaling; 1994 } 1995 1996 static int intel_cpufreq_cpu_init(struct cpufreq_policy *policy) 1997 { 1998 int ret = __intel_pstate_cpu_init(policy); 1999 2000 if (ret) 2001 return ret; 2002 2003 policy->cpuinfo.transition_latency = INTEL_CPUFREQ_TRANSITION_LATENCY; 2004 policy->transition_delay_us = INTEL_CPUFREQ_TRANSITION_DELAY; 2005 /* This reflects the intel_pstate_get_cpu_pstates() setting. */ 2006 policy->cur = policy->cpuinfo.min_freq; 2007 2008 return 0; 2009 } 2010 2011 static struct cpufreq_driver intel_cpufreq = { 2012 .flags = CPUFREQ_CONST_LOOPS, 2013 .verify = intel_cpufreq_verify_policy, 2014 .target = intel_cpufreq_target, 2015 .fast_switch = intel_cpufreq_fast_switch, 2016 .init = intel_cpufreq_cpu_init, 2017 .exit = intel_pstate_cpu_exit, 2018 .stop_cpu = intel_cpufreq_stop_cpu, 2019 .name = "intel_cpufreq", 2020 }; 2021 2022 static struct cpufreq_driver *default_driver = &intel_pstate; 2023 2024 static void intel_pstate_driver_cleanup(void) 2025 { 2026 unsigned int cpu; 2027 2028 get_online_cpus(); 2029 for_each_online_cpu(cpu) { 2030 if (all_cpu_data[cpu]) { 2031 if (intel_pstate_driver == &intel_pstate) 2032 intel_pstate_clear_update_util_hook(cpu); 2033 2034 kfree(all_cpu_data[cpu]); 2035 all_cpu_data[cpu] = NULL; 2036 } 2037 } 2038 put_online_cpus(); 2039 intel_pstate_driver = NULL; 2040 } 2041 2042 static int intel_pstate_register_driver(struct cpufreq_driver *driver) 2043 { 2044 int ret; 2045 2046 memset(&global, 0, sizeof(global)); 2047 global.max_perf_pct = 100; 2048 2049 intel_pstate_driver = driver; 2050 ret = cpufreq_register_driver(intel_pstate_driver); 2051 if (ret) { 2052 intel_pstate_driver_cleanup(); 2053 return ret; 2054 } 2055 2056 global.min_perf_pct = min_perf_pct_min(); 2057 2058 return 0; 2059 } 2060 2061 static int intel_pstate_unregister_driver(void) 2062 { 2063 if (hwp_active) 2064 return -EBUSY; 2065 2066 cpufreq_unregister_driver(intel_pstate_driver); 2067 intel_pstate_driver_cleanup(); 2068 2069 return 0; 2070 } 2071 2072 static ssize_t intel_pstate_show_status(char *buf) 2073 { 2074 if (!intel_pstate_driver) 2075 return sprintf(buf, "off\n"); 2076 2077 return sprintf(buf, "%s\n", intel_pstate_driver == &intel_pstate ? 2078 "active" : "passive"); 2079 } 2080 2081 static int intel_pstate_update_status(const char *buf, size_t size) 2082 { 2083 int ret; 2084 2085 if (size == 3 && !strncmp(buf, "off", size)) 2086 return intel_pstate_driver ? 2087 intel_pstate_unregister_driver() : -EINVAL; 2088 2089 if (size == 6 && !strncmp(buf, "active", size)) { 2090 if (intel_pstate_driver) { 2091 if (intel_pstate_driver == &intel_pstate) 2092 return 0; 2093 2094 ret = intel_pstate_unregister_driver(); 2095 if (ret) 2096 return ret; 2097 } 2098 2099 return intel_pstate_register_driver(&intel_pstate); 2100 } 2101 2102 if (size == 7 && !strncmp(buf, "passive", size)) { 2103 if (intel_pstate_driver) { 2104 if (intel_pstate_driver == &intel_cpufreq) 2105 return 0; 2106 2107 ret = intel_pstate_unregister_driver(); 2108 if (ret) 2109 return ret; 2110 } 2111 2112 return intel_pstate_register_driver(&intel_cpufreq); 2113 } 2114 2115 return -EINVAL; 2116 } 2117 2118 static int no_load __initdata; 2119 static int no_hwp __initdata; 2120 static int hwp_only __initdata; 2121 static unsigned int force_load __initdata; 2122 2123 static int __init intel_pstate_msrs_not_valid(void) 2124 { 2125 if (!pstate_funcs.get_max() || 2126 !pstate_funcs.get_min() || 2127 !pstate_funcs.get_turbo()) 2128 return -ENODEV; 2129 2130 return 0; 2131 } 2132 2133 static void __init copy_cpu_funcs(struct pstate_funcs *funcs) 2134 { 2135 pstate_funcs.get_max = funcs->get_max; 2136 pstate_funcs.get_max_physical = funcs->get_max_physical; 2137 pstate_funcs.get_min = funcs->get_min; 2138 pstate_funcs.get_turbo = funcs->get_turbo; 2139 pstate_funcs.get_scaling = funcs->get_scaling; 2140 pstate_funcs.get_val = funcs->get_val; 2141 pstate_funcs.get_vid = funcs->get_vid; 2142 pstate_funcs.get_aperf_mperf_shift = funcs->get_aperf_mperf_shift; 2143 } 2144 2145 #ifdef CONFIG_ACPI 2146 2147 static bool __init intel_pstate_no_acpi_pss(void) 2148 { 2149 int i; 2150 2151 for_each_possible_cpu(i) { 2152 acpi_status status; 2153 union acpi_object *pss; 2154 struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL }; 2155 struct acpi_processor *pr = per_cpu(processors, i); 2156 2157 if (!pr) 2158 continue; 2159 2160 status = acpi_evaluate_object(pr->handle, "_PSS", NULL, &buffer); 2161 if (ACPI_FAILURE(status)) 2162 continue; 2163 2164 pss = buffer.pointer; 2165 if (pss && pss->type == ACPI_TYPE_PACKAGE) { 2166 kfree(pss); 2167 return false; 2168 } 2169 2170 kfree(pss); 2171 } 2172 2173 return true; 2174 } 2175 2176 static bool __init intel_pstate_has_acpi_ppc(void) 2177 { 2178 int i; 2179 2180 for_each_possible_cpu(i) { 2181 struct acpi_processor *pr = per_cpu(processors, i); 2182 2183 if (!pr) 2184 continue; 2185 if (acpi_has_method(pr->handle, "_PPC")) 2186 return true; 2187 } 2188 return false; 2189 } 2190 2191 enum { 2192 PSS, 2193 PPC, 2194 }; 2195 2196 struct hw_vendor_info { 2197 u16 valid; 2198 char oem_id[ACPI_OEM_ID_SIZE]; 2199 char oem_table_id[ACPI_OEM_TABLE_ID_SIZE]; 2200 int oem_pwr_table; 2201 }; 2202 2203 /* Hardware vendor-specific info that has its own power management modes */ 2204 static struct hw_vendor_info vendor_info[] __initdata = { 2205 {1, "HP ", "ProLiant", PSS}, 2206 {1, "ORACLE", "X4-2 ", PPC}, 2207 {1, "ORACLE", "X4-2L ", PPC}, 2208 {1, "ORACLE", "X4-2B ", PPC}, 2209 {1, "ORACLE", "X3-2 ", PPC}, 2210 {1, "ORACLE", "X3-2L ", PPC}, 2211 {1, "ORACLE", "X3-2B ", PPC}, 2212 {1, "ORACLE", "X4470M2 ", PPC}, 2213 {1, "ORACLE", "X4270M3 ", PPC}, 2214 {1, "ORACLE", "X4270M2 ", PPC}, 2215 {1, "ORACLE", "X4170M2 ", PPC}, 2216 {1, "ORACLE", "X4170 M3", PPC}, 2217 {1, "ORACLE", "X4275 M3", PPC}, 2218 {1, "ORACLE", "X6-2 ", PPC}, 2219 {1, "ORACLE", "Sudbury ", PPC}, 2220 {0, "", ""}, 2221 }; 2222 2223 static bool __init intel_pstate_platform_pwr_mgmt_exists(void) 2224 { 2225 struct acpi_table_header hdr; 2226 struct hw_vendor_info *v_info; 2227 const struct x86_cpu_id *id; 2228 u64 misc_pwr; 2229 2230 id = x86_match_cpu(intel_pstate_cpu_oob_ids); 2231 if (id) { 2232 rdmsrl(MSR_MISC_PWR_MGMT, misc_pwr); 2233 if ( misc_pwr & (1 << 8)) 2234 return true; 2235 } 2236 2237 if (acpi_disabled || 2238 ACPI_FAILURE(acpi_get_table_header(ACPI_SIG_FADT, 0, &hdr))) 2239 return false; 2240 2241 for (v_info = vendor_info; v_info->valid; v_info++) { 2242 if (!strncmp(hdr.oem_id, v_info->oem_id, ACPI_OEM_ID_SIZE) && 2243 !strncmp(hdr.oem_table_id, v_info->oem_table_id, 2244 ACPI_OEM_TABLE_ID_SIZE)) 2245 switch (v_info->oem_pwr_table) { 2246 case PSS: 2247 return intel_pstate_no_acpi_pss(); 2248 case PPC: 2249 return intel_pstate_has_acpi_ppc() && 2250 (!force_load); 2251 } 2252 } 2253 2254 return false; 2255 } 2256 2257 static void intel_pstate_request_control_from_smm(void) 2258 { 2259 /* 2260 * It may be unsafe to request P-states control from SMM if _PPC support 2261 * has not been enabled. 2262 */ 2263 if (acpi_ppc) 2264 acpi_processor_pstate_control(); 2265 } 2266 #else /* CONFIG_ACPI not enabled */ 2267 static inline bool intel_pstate_platform_pwr_mgmt_exists(void) { return false; } 2268 static inline bool intel_pstate_has_acpi_ppc(void) { return false; } 2269 static inline void intel_pstate_request_control_from_smm(void) {} 2270 #endif /* CONFIG_ACPI */ 2271 2272 static const struct x86_cpu_id hwp_support_ids[] __initconst = { 2273 { X86_VENDOR_INTEL, 6, X86_MODEL_ANY, X86_FEATURE_HWP }, 2274 {} 2275 }; 2276 2277 static int __init intel_pstate_init(void) 2278 { 2279 int rc; 2280 2281 if (no_load) 2282 return -ENODEV; 2283 2284 if (x86_match_cpu(hwp_support_ids)) { 2285 copy_cpu_funcs(&core_funcs); 2286 if (!no_hwp) { 2287 hwp_active++; 2288 intel_pstate.attr = hwp_cpufreq_attrs; 2289 goto hwp_cpu_matched; 2290 } 2291 } else { 2292 const struct x86_cpu_id *id; 2293 2294 id = x86_match_cpu(intel_pstate_cpu_ids); 2295 if (!id) 2296 return -ENODEV; 2297 2298 copy_cpu_funcs((struct pstate_funcs *)id->driver_data); 2299 } 2300 2301 if (intel_pstate_msrs_not_valid()) 2302 return -ENODEV; 2303 2304 hwp_cpu_matched: 2305 /* 2306 * The Intel pstate driver will be ignored if the platform 2307 * firmware has its own power management modes. 2308 */ 2309 if (intel_pstate_platform_pwr_mgmt_exists()) 2310 return -ENODEV; 2311 2312 if (!hwp_active && hwp_only) 2313 return -ENOTSUPP; 2314 2315 pr_info("Intel P-state driver initializing\n"); 2316 2317 all_cpu_data = vzalloc(sizeof(void *) * num_possible_cpus()); 2318 if (!all_cpu_data) 2319 return -ENOMEM; 2320 2321 intel_pstate_request_control_from_smm(); 2322 2323 intel_pstate_sysfs_expose_params(); 2324 2325 mutex_lock(&intel_pstate_driver_lock); 2326 rc = intel_pstate_register_driver(default_driver); 2327 mutex_unlock(&intel_pstate_driver_lock); 2328 if (rc) 2329 return rc; 2330 2331 if (hwp_active) 2332 pr_info("HWP enabled\n"); 2333 2334 return 0; 2335 } 2336 device_initcall(intel_pstate_init); 2337 2338 static int __init intel_pstate_setup(char *str) 2339 { 2340 if (!str) 2341 return -EINVAL; 2342 2343 if (!strcmp(str, "disable")) { 2344 no_load = 1; 2345 } else if (!strcmp(str, "passive")) { 2346 pr_info("Passive mode enabled\n"); 2347 default_driver = &intel_cpufreq; 2348 no_hwp = 1; 2349 } 2350 if (!strcmp(str, "no_hwp")) { 2351 pr_info("HWP disabled\n"); 2352 no_hwp = 1; 2353 } 2354 if (!strcmp(str, "force")) 2355 force_load = 1; 2356 if (!strcmp(str, "hwp_only")) 2357 hwp_only = 1; 2358 if (!strcmp(str, "per_cpu_perf_limits")) 2359 per_cpu_limits = true; 2360 2361 #ifdef CONFIG_ACPI 2362 if (!strcmp(str, "support_acpi_ppc")) 2363 acpi_ppc = true; 2364 #endif 2365 2366 return 0; 2367 } 2368 early_param("intel_pstate", intel_pstate_setup); 2369 2370 MODULE_AUTHOR("Dirk Brandewie <dirk.j.brandewie@intel.com>"); 2371 MODULE_DESCRIPTION("'intel_pstate' - P state driver Intel Core processors"); 2372 MODULE_LICENSE("GPL"); 2373