1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * linux/drivers/thermal/cpufreq_cooling.c 4 * 5 * Copyright (C) 2012 Samsung Electronics Co., Ltd(http://www.samsung.com) 6 * 7 * Copyright (C) 2012-2018 Linaro Limited. 8 * 9 * Authors: Amit Daniel <amit.kachhap@linaro.org> 10 * Viresh Kumar <viresh.kumar@linaro.org> 11 * 12 */ 13 #include <linux/cpu.h> 14 #include <linux/cpufreq.h> 15 #include <linux/cpu_cooling.h> 16 #include <linux/device.h> 17 #include <linux/energy_model.h> 18 #include <linux/err.h> 19 #include <linux/export.h> 20 #include <linux/pm_opp.h> 21 #include <linux/pm_qos.h> 22 #include <linux/slab.h> 23 #include <linux/thermal.h> 24 25 #include <trace/events/thermal.h> 26 27 /* 28 * Cooling state <-> CPUFreq frequency 29 * 30 * Cooling states are translated to frequencies throughout this driver and this 31 * is the relation between them. 32 * 33 * Highest cooling state corresponds to lowest possible frequency. 34 * 35 * i.e. 36 * level 0 --> 1st Max Freq 37 * level 1 --> 2nd Max Freq 38 * ... 39 */ 40 41 /** 42 * struct time_in_idle - Idle time stats 43 * @time: previous reading of the absolute time that this cpu was idle 44 * @timestamp: wall time of the last invocation of get_cpu_idle_time_us() 45 */ 46 struct time_in_idle { 47 u64 time; 48 u64 timestamp; 49 }; 50 51 /** 52 * struct cpufreq_cooling_device - data for cooling device with cpufreq 53 * @last_load: load measured by the latest call to cpufreq_get_requested_power() 54 * @cpufreq_state: integer value representing the current state of cpufreq 55 * cooling devices. 56 * @max_level: maximum cooling level. One less than total number of valid 57 * cpufreq frequencies. 58 * @em: Reference on the Energy Model of the device 59 * @cdev: thermal_cooling_device pointer to keep track of the 60 * registered cooling device. 61 * @policy: cpufreq policy. 62 * @cooling_ops: cpufreq callbacks to thermal cooling device ops 63 * @idle_time: idle time stats 64 * @qos_req: PM QoS contraint to apply 65 * 66 * This structure is required for keeping information of each registered 67 * cpufreq_cooling_device. 68 */ 69 struct cpufreq_cooling_device { 70 u32 last_load; 71 unsigned int cpufreq_state; 72 unsigned int max_level; 73 struct em_perf_domain *em; 74 struct cpufreq_policy *policy; 75 struct thermal_cooling_device_ops cooling_ops; 76 #ifndef CONFIG_SMP 77 struct time_in_idle *idle_time; 78 #endif 79 struct freq_qos_request qos_req; 80 }; 81 82 #ifdef CONFIG_THERMAL_GOV_POWER_ALLOCATOR 83 /** 84 * get_level: Find the level for a particular frequency 85 * @cpufreq_cdev: cpufreq_cdev for which the property is required 86 * @freq: Frequency 87 * 88 * Return: level corresponding to the frequency. 89 */ 90 static unsigned long get_level(struct cpufreq_cooling_device *cpufreq_cdev, 91 unsigned int freq) 92 { 93 int i; 94 95 for (i = cpufreq_cdev->max_level - 1; i >= 0; i--) { 96 if (freq > cpufreq_cdev->em->table[i].frequency) 97 break; 98 } 99 100 return cpufreq_cdev->max_level - i - 1; 101 } 102 103 static u32 cpu_freq_to_power(struct cpufreq_cooling_device *cpufreq_cdev, 104 u32 freq) 105 { 106 int i; 107 108 for (i = cpufreq_cdev->max_level - 1; i >= 0; i--) { 109 if (freq > cpufreq_cdev->em->table[i].frequency) 110 break; 111 } 112 113 return cpufreq_cdev->em->table[i + 1].power; 114 } 115 116 static u32 cpu_power_to_freq(struct cpufreq_cooling_device *cpufreq_cdev, 117 u32 power) 118 { 119 int i; 120 121 for (i = cpufreq_cdev->max_level; i > 0; i--) { 122 if (power >= cpufreq_cdev->em->table[i].power) 123 break; 124 } 125 126 return cpufreq_cdev->em->table[i].frequency; 127 } 128 129 /** 130 * get_load() - get load for a cpu 131 * @cpufreq_cdev: struct cpufreq_cooling_device for the cpu 132 * @cpu: cpu number 133 * @cpu_idx: index of the cpu in time_in_idle array 134 * 135 * Return: The average load of cpu @cpu in percentage since this 136 * function was last called. 137 */ 138 #ifdef CONFIG_SMP 139 static u32 get_load(struct cpufreq_cooling_device *cpufreq_cdev, int cpu, 140 int cpu_idx) 141 { 142 unsigned long max = arch_scale_cpu_capacity(cpu); 143 unsigned long util; 144 145 util = sched_cpu_util(cpu, max); 146 return (util * 100) / max; 147 } 148 #else /* !CONFIG_SMP */ 149 static u32 get_load(struct cpufreq_cooling_device *cpufreq_cdev, int cpu, 150 int cpu_idx) 151 { 152 u32 load; 153 u64 now, now_idle, delta_time, delta_idle; 154 struct time_in_idle *idle_time = &cpufreq_cdev->idle_time[cpu_idx]; 155 156 now_idle = get_cpu_idle_time(cpu, &now, 0); 157 delta_idle = now_idle - idle_time->time; 158 delta_time = now - idle_time->timestamp; 159 160 if (delta_time <= delta_idle) 161 load = 0; 162 else 163 load = div64_u64(100 * (delta_time - delta_idle), delta_time); 164 165 idle_time->time = now_idle; 166 idle_time->timestamp = now; 167 168 return load; 169 } 170 #endif /* CONFIG_SMP */ 171 172 /** 173 * get_dynamic_power() - calculate the dynamic power 174 * @cpufreq_cdev: &cpufreq_cooling_device for this cdev 175 * @freq: current frequency 176 * 177 * Return: the dynamic power consumed by the cpus described by 178 * @cpufreq_cdev. 179 */ 180 static u32 get_dynamic_power(struct cpufreq_cooling_device *cpufreq_cdev, 181 unsigned long freq) 182 { 183 u32 raw_cpu_power; 184 185 raw_cpu_power = cpu_freq_to_power(cpufreq_cdev, freq); 186 return (raw_cpu_power * cpufreq_cdev->last_load) / 100; 187 } 188 189 /** 190 * cpufreq_get_requested_power() - get the current power 191 * @cdev: &thermal_cooling_device pointer 192 * @power: pointer in which to store the resulting power 193 * 194 * Calculate the current power consumption of the cpus in milliwatts 195 * and store it in @power. This function should actually calculate 196 * the requested power, but it's hard to get the frequency that 197 * cpufreq would have assigned if there were no thermal limits. 198 * Instead, we calculate the current power on the assumption that the 199 * immediate future will look like the immediate past. 200 * 201 * We use the current frequency and the average load since this 202 * function was last called. In reality, there could have been 203 * multiple opps since this function was last called and that affects 204 * the load calculation. While it's not perfectly accurate, this 205 * simplification is good enough and works. REVISIT this, as more 206 * complex code may be needed if experiments show that it's not 207 * accurate enough. 208 * 209 * Return: 0 on success, -E* if getting the static power failed. 210 */ 211 static int cpufreq_get_requested_power(struct thermal_cooling_device *cdev, 212 u32 *power) 213 { 214 unsigned long freq; 215 int i = 0, cpu; 216 u32 total_load = 0; 217 struct cpufreq_cooling_device *cpufreq_cdev = cdev->devdata; 218 struct cpufreq_policy *policy = cpufreq_cdev->policy; 219 220 freq = cpufreq_quick_get(policy->cpu); 221 222 for_each_cpu(cpu, policy->related_cpus) { 223 u32 load; 224 225 if (cpu_online(cpu)) 226 load = get_load(cpufreq_cdev, cpu, i); 227 else 228 load = 0; 229 230 total_load += load; 231 } 232 233 cpufreq_cdev->last_load = total_load; 234 235 *power = get_dynamic_power(cpufreq_cdev, freq); 236 237 trace_thermal_power_cpu_get_power_simple(policy->cpu, *power); 238 239 return 0; 240 } 241 242 /** 243 * cpufreq_state2power() - convert a cpu cdev state to power consumed 244 * @cdev: &thermal_cooling_device pointer 245 * @state: cooling device state to be converted 246 * @power: pointer in which to store the resulting power 247 * 248 * Convert cooling device state @state into power consumption in 249 * milliwatts assuming 100% load. Store the calculated power in 250 * @power. 251 * 252 * Return: 0 on success, -EINVAL if the cooling device state could not 253 * be converted into a frequency or other -E* if there was an error 254 * when calculating the static power. 255 */ 256 static int cpufreq_state2power(struct thermal_cooling_device *cdev, 257 unsigned long state, u32 *power) 258 { 259 unsigned int freq, num_cpus, idx; 260 struct cpufreq_cooling_device *cpufreq_cdev = cdev->devdata; 261 262 /* Request state should be less than max_level */ 263 if (state > cpufreq_cdev->max_level) 264 return -EINVAL; 265 266 num_cpus = cpumask_weight(cpufreq_cdev->policy->cpus); 267 268 idx = cpufreq_cdev->max_level - state; 269 freq = cpufreq_cdev->em->table[idx].frequency; 270 *power = cpu_freq_to_power(cpufreq_cdev, freq) * num_cpus; 271 272 return 0; 273 } 274 275 /** 276 * cpufreq_power2state() - convert power to a cooling device state 277 * @cdev: &thermal_cooling_device pointer 278 * @power: power in milliwatts to be converted 279 * @state: pointer in which to store the resulting state 280 * 281 * Calculate a cooling device state for the cpus described by @cdev 282 * that would allow them to consume at most @power mW and store it in 283 * @state. Note that this calculation depends on external factors 284 * such as the cpu load or the current static power. Calling this 285 * function with the same power as input can yield different cooling 286 * device states depending on those external factors. 287 * 288 * Return: 0 on success, -ENODEV if no cpus are online or -EINVAL if 289 * the calculated frequency could not be converted to a valid state. 290 * The latter should not happen unless the frequencies available to 291 * cpufreq have changed since the initialization of the cpu cooling 292 * device. 293 */ 294 static int cpufreq_power2state(struct thermal_cooling_device *cdev, 295 u32 power, unsigned long *state) 296 { 297 unsigned int target_freq; 298 u32 last_load, normalised_power; 299 struct cpufreq_cooling_device *cpufreq_cdev = cdev->devdata; 300 struct cpufreq_policy *policy = cpufreq_cdev->policy; 301 302 last_load = cpufreq_cdev->last_load ?: 1; 303 normalised_power = (power * 100) / last_load; 304 target_freq = cpu_power_to_freq(cpufreq_cdev, normalised_power); 305 306 *state = get_level(cpufreq_cdev, target_freq); 307 trace_thermal_power_cpu_limit(policy->related_cpus, target_freq, *state, 308 power); 309 return 0; 310 } 311 312 static inline bool em_is_sane(struct cpufreq_cooling_device *cpufreq_cdev, 313 struct em_perf_domain *em) { 314 struct cpufreq_policy *policy; 315 unsigned int nr_levels; 316 317 if (!em || em_is_artificial(em)) 318 return false; 319 320 policy = cpufreq_cdev->policy; 321 if (!cpumask_equal(policy->related_cpus, em_span_cpus(em))) { 322 pr_err("The span of pd %*pbl is misaligned with cpufreq policy %*pbl\n", 323 cpumask_pr_args(em_span_cpus(em)), 324 cpumask_pr_args(policy->related_cpus)); 325 return false; 326 } 327 328 nr_levels = cpufreq_cdev->max_level + 1; 329 if (em_pd_nr_perf_states(em) != nr_levels) { 330 pr_err("The number of performance states in pd %*pbl (%u) doesn't match the number of cooling levels (%u)\n", 331 cpumask_pr_args(em_span_cpus(em)), 332 em_pd_nr_perf_states(em), nr_levels); 333 return false; 334 } 335 336 return true; 337 } 338 #endif /* CONFIG_THERMAL_GOV_POWER_ALLOCATOR */ 339 340 #ifdef CONFIG_SMP 341 static inline int allocate_idle_time(struct cpufreq_cooling_device *cpufreq_cdev) 342 { 343 return 0; 344 } 345 346 static inline void free_idle_time(struct cpufreq_cooling_device *cpufreq_cdev) 347 { 348 } 349 #else 350 static int allocate_idle_time(struct cpufreq_cooling_device *cpufreq_cdev) 351 { 352 unsigned int num_cpus = cpumask_weight(cpufreq_cdev->policy->related_cpus); 353 354 cpufreq_cdev->idle_time = kcalloc(num_cpus, 355 sizeof(*cpufreq_cdev->idle_time), 356 GFP_KERNEL); 357 if (!cpufreq_cdev->idle_time) 358 return -ENOMEM; 359 360 return 0; 361 } 362 363 static void free_idle_time(struct cpufreq_cooling_device *cpufreq_cdev) 364 { 365 kfree(cpufreq_cdev->idle_time); 366 cpufreq_cdev->idle_time = NULL; 367 } 368 #endif /* CONFIG_SMP */ 369 370 static unsigned int get_state_freq(struct cpufreq_cooling_device *cpufreq_cdev, 371 unsigned long state) 372 { 373 struct cpufreq_policy *policy; 374 unsigned long idx; 375 376 #ifdef CONFIG_THERMAL_GOV_POWER_ALLOCATOR 377 /* Use the Energy Model table if available */ 378 if (cpufreq_cdev->em) { 379 idx = cpufreq_cdev->max_level - state; 380 return cpufreq_cdev->em->table[idx].frequency; 381 } 382 #endif 383 384 /* Otherwise, fallback on the CPUFreq table */ 385 policy = cpufreq_cdev->policy; 386 if (policy->freq_table_sorted == CPUFREQ_TABLE_SORTED_ASCENDING) 387 idx = cpufreq_cdev->max_level - state; 388 else 389 idx = state; 390 391 return policy->freq_table[idx].frequency; 392 } 393 394 /* cpufreq cooling device callback functions are defined below */ 395 396 /** 397 * cpufreq_get_max_state - callback function to get the max cooling state. 398 * @cdev: thermal cooling device pointer. 399 * @state: fill this variable with the max cooling state. 400 * 401 * Callback for the thermal cooling device to return the cpufreq 402 * max cooling state. 403 * 404 * Return: 0 on success, an error code otherwise. 405 */ 406 static int cpufreq_get_max_state(struct thermal_cooling_device *cdev, 407 unsigned long *state) 408 { 409 struct cpufreq_cooling_device *cpufreq_cdev = cdev->devdata; 410 411 *state = cpufreq_cdev->max_level; 412 return 0; 413 } 414 415 /** 416 * cpufreq_get_cur_state - callback function to get the current cooling state. 417 * @cdev: thermal cooling device pointer. 418 * @state: fill this variable with the current cooling state. 419 * 420 * Callback for the thermal cooling device to return the cpufreq 421 * current cooling state. 422 * 423 * Return: 0 on success, an error code otherwise. 424 */ 425 static int cpufreq_get_cur_state(struct thermal_cooling_device *cdev, 426 unsigned long *state) 427 { 428 struct cpufreq_cooling_device *cpufreq_cdev = cdev->devdata; 429 430 *state = cpufreq_cdev->cpufreq_state; 431 432 return 0; 433 } 434 435 /** 436 * cpufreq_set_cur_state - callback function to set the current cooling state. 437 * @cdev: thermal cooling device pointer. 438 * @state: set this variable to the current cooling state. 439 * 440 * Callback for the thermal cooling device to change the cpufreq 441 * current cooling state. 442 * 443 * Return: 0 on success, an error code otherwise. 444 */ 445 static int cpufreq_set_cur_state(struct thermal_cooling_device *cdev, 446 unsigned long state) 447 { 448 struct cpufreq_cooling_device *cpufreq_cdev = cdev->devdata; 449 struct cpumask *cpus; 450 unsigned int frequency; 451 int ret; 452 453 /* Request state should be less than max_level */ 454 if (state > cpufreq_cdev->max_level) 455 return -EINVAL; 456 457 /* Check if the old cooling action is same as new cooling action */ 458 if (cpufreq_cdev->cpufreq_state == state) 459 return 0; 460 461 frequency = get_state_freq(cpufreq_cdev, state); 462 463 ret = freq_qos_update_request(&cpufreq_cdev->qos_req, frequency); 464 if (ret >= 0) { 465 cpufreq_cdev->cpufreq_state = state; 466 cpus = cpufreq_cdev->policy->related_cpus; 467 arch_update_thermal_pressure(cpus, frequency); 468 ret = 0; 469 } 470 471 return ret; 472 } 473 474 /** 475 * __cpufreq_cooling_register - helper function to create cpufreq cooling device 476 * @np: a valid struct device_node to the cooling device device tree node 477 * @policy: cpufreq policy 478 * Normally this should be same as cpufreq policy->related_cpus. 479 * @em: Energy Model of the cpufreq policy 480 * 481 * This interface function registers the cpufreq cooling device with the name 482 * "thermal-cpufreq-%x". This api can support multiple instances of cpufreq 483 * cooling devices. It also gives the opportunity to link the cooling device 484 * with a device tree node, in order to bind it via the thermal DT code. 485 * 486 * Return: a valid struct thermal_cooling_device pointer on success, 487 * on failure, it returns a corresponding ERR_PTR(). 488 */ 489 static struct thermal_cooling_device * 490 __cpufreq_cooling_register(struct device_node *np, 491 struct cpufreq_policy *policy, 492 struct em_perf_domain *em) 493 { 494 struct thermal_cooling_device *cdev; 495 struct cpufreq_cooling_device *cpufreq_cdev; 496 unsigned int i; 497 struct device *dev; 498 int ret; 499 struct thermal_cooling_device_ops *cooling_ops; 500 char *name; 501 502 dev = get_cpu_device(policy->cpu); 503 if (unlikely(!dev)) { 504 pr_warn("No cpu device for cpu %d\n", policy->cpu); 505 return ERR_PTR(-ENODEV); 506 } 507 508 if (IS_ERR_OR_NULL(policy)) { 509 pr_err("%s: cpufreq policy isn't valid: %p\n", __func__, policy); 510 return ERR_PTR(-EINVAL); 511 } 512 513 i = cpufreq_table_count_valid_entries(policy); 514 if (!i) { 515 pr_debug("%s: CPUFreq table not found or has no valid entries\n", 516 __func__); 517 return ERR_PTR(-ENODEV); 518 } 519 520 cpufreq_cdev = kzalloc(sizeof(*cpufreq_cdev), GFP_KERNEL); 521 if (!cpufreq_cdev) 522 return ERR_PTR(-ENOMEM); 523 524 cpufreq_cdev->policy = policy; 525 526 ret = allocate_idle_time(cpufreq_cdev); 527 if (ret) { 528 cdev = ERR_PTR(ret); 529 goto free_cdev; 530 } 531 532 /* max_level is an index, not a counter */ 533 cpufreq_cdev->max_level = i - 1; 534 535 cooling_ops = &cpufreq_cdev->cooling_ops; 536 cooling_ops->get_max_state = cpufreq_get_max_state; 537 cooling_ops->get_cur_state = cpufreq_get_cur_state; 538 cooling_ops->set_cur_state = cpufreq_set_cur_state; 539 540 #ifdef CONFIG_THERMAL_GOV_POWER_ALLOCATOR 541 if (em_is_sane(cpufreq_cdev, em)) { 542 cpufreq_cdev->em = em; 543 cooling_ops->get_requested_power = cpufreq_get_requested_power; 544 cooling_ops->state2power = cpufreq_state2power; 545 cooling_ops->power2state = cpufreq_power2state; 546 } else 547 #endif 548 if (policy->freq_table_sorted == CPUFREQ_TABLE_UNSORTED) { 549 pr_err("%s: unsorted frequency tables are not supported\n", 550 __func__); 551 cdev = ERR_PTR(-EINVAL); 552 goto free_idle_time; 553 } 554 555 ret = freq_qos_add_request(&policy->constraints, 556 &cpufreq_cdev->qos_req, FREQ_QOS_MAX, 557 get_state_freq(cpufreq_cdev, 0)); 558 if (ret < 0) { 559 pr_err("%s: Failed to add freq constraint (%d)\n", __func__, 560 ret); 561 cdev = ERR_PTR(ret); 562 goto free_idle_time; 563 } 564 565 cdev = ERR_PTR(-ENOMEM); 566 name = kasprintf(GFP_KERNEL, "cpufreq-%s", dev_name(dev)); 567 if (!name) 568 goto remove_qos_req; 569 570 cdev = thermal_of_cooling_device_register(np, name, cpufreq_cdev, 571 cooling_ops); 572 kfree(name); 573 574 if (IS_ERR(cdev)) 575 goto remove_qos_req; 576 577 return cdev; 578 579 remove_qos_req: 580 freq_qos_remove_request(&cpufreq_cdev->qos_req); 581 free_idle_time: 582 free_idle_time(cpufreq_cdev); 583 free_cdev: 584 kfree(cpufreq_cdev); 585 return cdev; 586 } 587 588 /** 589 * cpufreq_cooling_register - function to create cpufreq cooling device. 590 * @policy: cpufreq policy 591 * 592 * This interface function registers the cpufreq cooling device with the name 593 * "thermal-cpufreq-%x". This api can support multiple instances of cpufreq 594 * cooling devices. 595 * 596 * Return: a valid struct thermal_cooling_device pointer on success, 597 * on failure, it returns a corresponding ERR_PTR(). 598 */ 599 struct thermal_cooling_device * 600 cpufreq_cooling_register(struct cpufreq_policy *policy) 601 { 602 return __cpufreq_cooling_register(NULL, policy, NULL); 603 } 604 EXPORT_SYMBOL_GPL(cpufreq_cooling_register); 605 606 /** 607 * of_cpufreq_cooling_register - function to create cpufreq cooling device. 608 * @policy: cpufreq policy 609 * 610 * This interface function registers the cpufreq cooling device with the name 611 * "thermal-cpufreq-%x". This api can support multiple instances of cpufreq 612 * cooling devices. Using this API, the cpufreq cooling device will be 613 * linked to the device tree node provided. 614 * 615 * Using this function, the cooling device will implement the power 616 * extensions by using a simple cpu power model. The cpus must have 617 * registered their OPPs using the OPP library. 618 * 619 * It also takes into account, if property present in policy CPU node, the 620 * static power consumed by the cpu. 621 * 622 * Return: a valid struct thermal_cooling_device pointer on success, 623 * and NULL on failure. 624 */ 625 struct thermal_cooling_device * 626 of_cpufreq_cooling_register(struct cpufreq_policy *policy) 627 { 628 struct device_node *np = of_get_cpu_node(policy->cpu, NULL); 629 struct thermal_cooling_device *cdev = NULL; 630 631 if (!np) { 632 pr_err("cpufreq_cooling: OF node not available for cpu%d\n", 633 policy->cpu); 634 return NULL; 635 } 636 637 if (of_find_property(np, "#cooling-cells", NULL)) { 638 struct em_perf_domain *em = em_cpu_get(policy->cpu); 639 640 cdev = __cpufreq_cooling_register(np, policy, em); 641 if (IS_ERR(cdev)) { 642 pr_err("cpufreq_cooling: cpu%d failed to register as cooling device: %ld\n", 643 policy->cpu, PTR_ERR(cdev)); 644 cdev = NULL; 645 } 646 } 647 648 of_node_put(np); 649 return cdev; 650 } 651 EXPORT_SYMBOL_GPL(of_cpufreq_cooling_register); 652 653 /** 654 * cpufreq_cooling_unregister - function to remove cpufreq cooling device. 655 * @cdev: thermal cooling device pointer. 656 * 657 * This interface function unregisters the "thermal-cpufreq-%x" cooling device. 658 */ 659 void cpufreq_cooling_unregister(struct thermal_cooling_device *cdev) 660 { 661 struct cpufreq_cooling_device *cpufreq_cdev; 662 663 if (!cdev) 664 return; 665 666 cpufreq_cdev = cdev->devdata; 667 668 thermal_cooling_device_unregister(cdev); 669 freq_qos_remove_request(&cpufreq_cdev->qos_req); 670 free_idle_time(cpufreq_cdev); 671 kfree(cpufreq_cdev); 672 } 673 EXPORT_SYMBOL_GPL(cpufreq_cooling_unregister); 674