1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * x86_pkg_temp_thermal driver
4 * Copyright (c) 2013, Intel Corporation.
5 */
6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
7
8 #include <linux/module.h>
9 #include <linux/init.h>
10 #include <linux/intel_tcc.h>
11 #include <linux/err.h>
12 #include <linux/param.h>
13 #include <linux/device.h>
14 #include <linux/platform_device.h>
15 #include <linux/cpu.h>
16 #include <linux/smp.h>
17 #include <linux/slab.h>
18 #include <linux/pm.h>
19 #include <linux/thermal.h>
20 #include <linux/debugfs.h>
21
22 #include <asm/cpu_device_id.h>
23
24 #include "thermal_interrupt.h"
25
26 /*
27 * Rate control delay: Idea is to introduce denounce effect
28 * This should be long enough to avoid reduce events, when
29 * threshold is set to a temperature, which is constantly
30 * violated, but at the short enough to take any action.
31 * The action can be remove threshold or change it to next
32 * interesting setting. Based on experiments, in around
33 * every 5 seconds under load will give us a significant
34 * temperature change.
35 */
36 #define PKG_TEMP_THERMAL_NOTIFY_DELAY 5000
37 static int notify_delay_ms = PKG_TEMP_THERMAL_NOTIFY_DELAY;
38 module_param(notify_delay_ms, int, 0644);
39 MODULE_PARM_DESC(notify_delay_ms,
40 "User space notification delay in milli seconds.");
41
42 /* Number of trip points in thermal zone. Currently it can't
43 * be more than 2. MSR can allow setting and getting notifications
44 * for only 2 thresholds. This define enforces this, if there
45 * is some wrong values returned by cpuid for number of thresholds.
46 */
47 #define MAX_NUMBER_OF_TRIPS 2
48
49 struct zone_device {
50 int cpu;
51 bool work_scheduled;
52 u32 msr_pkg_therm_low;
53 u32 msr_pkg_therm_high;
54 struct delayed_work work;
55 struct thermal_zone_device *tzone;
56 struct thermal_trip *trips;
57 struct cpumask cpumask;
58 };
59
60 static struct thermal_zone_params pkg_temp_tz_params = {
61 .no_hwmon = true,
62 };
63
64 /* Keep track of how many zone pointers we allocated in init() */
65 static int max_id __read_mostly;
66 /* Array of zone pointers */
67 static struct zone_device **zones;
68 /* Serializes interrupt notification, work and hotplug */
69 static DEFINE_RAW_SPINLOCK(pkg_temp_lock);
70 /* Protects zone operation in the work function against hotplug removal */
71 static DEFINE_MUTEX(thermal_zone_mutex);
72
73 /* The dynamically assigned cpu hotplug state for module_exit() */
74 static enum cpuhp_state pkg_thermal_hp_state __read_mostly;
75
76 /* Debug counters to show using debugfs */
77 static struct dentry *debugfs;
78 static unsigned int pkg_interrupt_cnt;
79 static unsigned int pkg_work_cnt;
80
pkg_temp_debugfs_init(void)81 static void pkg_temp_debugfs_init(void)
82 {
83 debugfs = debugfs_create_dir("pkg_temp_thermal", NULL);
84
85 debugfs_create_u32("pkg_thres_interrupt", S_IRUGO, debugfs,
86 &pkg_interrupt_cnt);
87 debugfs_create_u32("pkg_thres_work", S_IRUGO, debugfs,
88 &pkg_work_cnt);
89 }
90
91 /*
92 * Protection:
93 *
94 * - cpu hotplug: Read serialized by cpu hotplug lock
95 * Write must hold pkg_temp_lock
96 *
97 * - Other callsites: Must hold pkg_temp_lock
98 */
pkg_temp_thermal_get_dev(unsigned int cpu)99 static struct zone_device *pkg_temp_thermal_get_dev(unsigned int cpu)
100 {
101 int id = topology_logical_die_id(cpu);
102
103 if (id >= 0 && id < max_id)
104 return zones[id];
105 return NULL;
106 }
107
sys_get_curr_temp(struct thermal_zone_device * tzd,int * temp)108 static int sys_get_curr_temp(struct thermal_zone_device *tzd, int *temp)
109 {
110 struct zone_device *zonedev = thermal_zone_device_priv(tzd);
111 int val, ret;
112
113 ret = intel_tcc_get_temp(zonedev->cpu, &val, true);
114 if (ret < 0)
115 return ret;
116
117 *temp = val * 1000;
118 pr_debug("sys_get_curr_temp %d\n", *temp);
119 return 0;
120 }
121
122 static int
sys_set_trip_temp(struct thermal_zone_device * tzd,int trip,int temp)123 sys_set_trip_temp(struct thermal_zone_device *tzd, int trip, int temp)
124 {
125 struct zone_device *zonedev = thermal_zone_device_priv(tzd);
126 u32 l, h, mask, shift, intr;
127 int tj_max, val, ret;
128
129 tj_max = intel_tcc_get_tjmax(zonedev->cpu);
130 if (tj_max < 0)
131 return tj_max;
132 tj_max *= 1000;
133
134 val = (tj_max - temp)/1000;
135
136 if (trip >= MAX_NUMBER_OF_TRIPS || val < 0 || val > 0x7f)
137 return -EINVAL;
138
139 ret = rdmsr_on_cpu(zonedev->cpu, MSR_IA32_PACKAGE_THERM_INTERRUPT,
140 &l, &h);
141 if (ret < 0)
142 return ret;
143
144 if (trip) {
145 mask = THERM_MASK_THRESHOLD1;
146 shift = THERM_SHIFT_THRESHOLD1;
147 intr = THERM_INT_THRESHOLD1_ENABLE;
148 } else {
149 mask = THERM_MASK_THRESHOLD0;
150 shift = THERM_SHIFT_THRESHOLD0;
151 intr = THERM_INT_THRESHOLD0_ENABLE;
152 }
153 l &= ~mask;
154 /*
155 * When users space sets a trip temperature == 0, which is indication
156 * that, it is no longer interested in receiving notifications.
157 */
158 if (!temp) {
159 l &= ~intr;
160 } else {
161 l |= val << shift;
162 l |= intr;
163 }
164
165 return wrmsr_on_cpu(zonedev->cpu, MSR_IA32_PACKAGE_THERM_INTERRUPT,
166 l, h);
167 }
168
169 /* Thermal zone callback registry */
170 static struct thermal_zone_device_ops tzone_ops = {
171 .get_temp = sys_get_curr_temp,
172 .set_trip_temp = sys_set_trip_temp,
173 };
174
pkg_thermal_rate_control(void)175 static bool pkg_thermal_rate_control(void)
176 {
177 return true;
178 }
179
180 /* Enable threshold interrupt on local package/cpu */
enable_pkg_thres_interrupt(void)181 static inline void enable_pkg_thres_interrupt(void)
182 {
183 u8 thres_0, thres_1;
184 u32 l, h;
185
186 rdmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);
187 /* only enable/disable if it had valid threshold value */
188 thres_0 = (l & THERM_MASK_THRESHOLD0) >> THERM_SHIFT_THRESHOLD0;
189 thres_1 = (l & THERM_MASK_THRESHOLD1) >> THERM_SHIFT_THRESHOLD1;
190 if (thres_0)
191 l |= THERM_INT_THRESHOLD0_ENABLE;
192 if (thres_1)
193 l |= THERM_INT_THRESHOLD1_ENABLE;
194 wrmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);
195 }
196
197 /* Disable threshold interrupt on local package/cpu */
disable_pkg_thres_interrupt(void)198 static inline void disable_pkg_thres_interrupt(void)
199 {
200 u32 l, h;
201
202 rdmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);
203
204 l &= ~(THERM_INT_THRESHOLD0_ENABLE | THERM_INT_THRESHOLD1_ENABLE);
205 wrmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);
206 }
207
pkg_temp_thermal_threshold_work_fn(struct work_struct * work)208 static void pkg_temp_thermal_threshold_work_fn(struct work_struct *work)
209 {
210 struct thermal_zone_device *tzone = NULL;
211 int cpu = smp_processor_id();
212 struct zone_device *zonedev;
213
214 mutex_lock(&thermal_zone_mutex);
215 raw_spin_lock_irq(&pkg_temp_lock);
216 ++pkg_work_cnt;
217
218 zonedev = pkg_temp_thermal_get_dev(cpu);
219 if (!zonedev) {
220 raw_spin_unlock_irq(&pkg_temp_lock);
221 mutex_unlock(&thermal_zone_mutex);
222 return;
223 }
224 zonedev->work_scheduled = false;
225
226 thermal_clear_package_intr_status(PACKAGE_LEVEL, THERM_LOG_THRESHOLD0 | THERM_LOG_THRESHOLD1);
227 tzone = zonedev->tzone;
228
229 enable_pkg_thres_interrupt();
230 raw_spin_unlock_irq(&pkg_temp_lock);
231
232 /*
233 * If tzone is not NULL, then thermal_zone_mutex will prevent the
234 * concurrent removal in the cpu offline callback.
235 */
236 if (tzone)
237 thermal_zone_device_update(tzone, THERMAL_EVENT_UNSPECIFIED);
238
239 mutex_unlock(&thermal_zone_mutex);
240 }
241
pkg_thermal_schedule_work(int cpu,struct delayed_work * work)242 static void pkg_thermal_schedule_work(int cpu, struct delayed_work *work)
243 {
244 unsigned long ms = msecs_to_jiffies(notify_delay_ms);
245
246 schedule_delayed_work_on(cpu, work, ms);
247 }
248
pkg_thermal_notify(u64 msr_val)249 static int pkg_thermal_notify(u64 msr_val)
250 {
251 int cpu = smp_processor_id();
252 struct zone_device *zonedev;
253 unsigned long flags;
254
255 raw_spin_lock_irqsave(&pkg_temp_lock, flags);
256 ++pkg_interrupt_cnt;
257
258 disable_pkg_thres_interrupt();
259
260 /* Work is per package, so scheduling it once is enough. */
261 zonedev = pkg_temp_thermal_get_dev(cpu);
262 if (zonedev && !zonedev->work_scheduled) {
263 zonedev->work_scheduled = true;
264 pkg_thermal_schedule_work(zonedev->cpu, &zonedev->work);
265 }
266
267 raw_spin_unlock_irqrestore(&pkg_temp_lock, flags);
268 return 0;
269 }
270
pkg_temp_thermal_trips_init(int cpu,int tj_max,int num_trips)271 static struct thermal_trip *pkg_temp_thermal_trips_init(int cpu, int tj_max, int num_trips)
272 {
273 struct thermal_trip *trips;
274 unsigned long thres_reg_value;
275 u32 mask, shift, eax, edx;
276 int ret, i;
277
278 trips = kzalloc(sizeof(*trips) * num_trips, GFP_KERNEL);
279 if (!trips)
280 return ERR_PTR(-ENOMEM);
281
282 for (i = 0; i < num_trips; i++) {
283
284 if (i) {
285 mask = THERM_MASK_THRESHOLD1;
286 shift = THERM_SHIFT_THRESHOLD1;
287 } else {
288 mask = THERM_MASK_THRESHOLD0;
289 shift = THERM_SHIFT_THRESHOLD0;
290 }
291
292 ret = rdmsr_on_cpu(cpu, MSR_IA32_PACKAGE_THERM_INTERRUPT,
293 &eax, &edx);
294 if (ret < 0) {
295 kfree(trips);
296 return ERR_PTR(ret);
297 }
298
299 thres_reg_value = (eax & mask) >> shift;
300
301 trips[i].temperature = thres_reg_value ?
302 tj_max - thres_reg_value * 1000 : THERMAL_TEMP_INVALID;
303
304 trips[i].type = THERMAL_TRIP_PASSIVE;
305
306 pr_debug("%s: cpu=%d, trip=%d, temp=%d\n",
307 __func__, cpu, i, trips[i].temperature);
308 }
309
310 return trips;
311 }
312
pkg_temp_thermal_device_add(unsigned int cpu)313 static int pkg_temp_thermal_device_add(unsigned int cpu)
314 {
315 int id = topology_logical_die_id(cpu);
316 u32 eax, ebx, ecx, edx;
317 struct zone_device *zonedev;
318 int thres_count, err;
319 int tj_max;
320
321 if (id >= max_id)
322 return -ENOMEM;
323
324 cpuid(6, &eax, &ebx, &ecx, &edx);
325 thres_count = ebx & 0x07;
326 if (!thres_count)
327 return -ENODEV;
328
329 thres_count = clamp_val(thres_count, 0, MAX_NUMBER_OF_TRIPS);
330
331 tj_max = intel_tcc_get_tjmax(cpu);
332 if (tj_max < 0)
333 return tj_max;
334 tj_max *= 1000;
335
336 zonedev = kzalloc(sizeof(*zonedev), GFP_KERNEL);
337 if (!zonedev)
338 return -ENOMEM;
339
340 zonedev->trips = pkg_temp_thermal_trips_init(cpu, tj_max, thres_count);
341 if (IS_ERR(zonedev->trips)) {
342 err = PTR_ERR(zonedev->trips);
343 goto out_kfree_zonedev;
344 }
345
346 INIT_DELAYED_WORK(&zonedev->work, pkg_temp_thermal_threshold_work_fn);
347 zonedev->cpu = cpu;
348 zonedev->tzone = thermal_zone_device_register_with_trips("x86_pkg_temp",
349 zonedev->trips, thres_count,
350 (thres_count == MAX_NUMBER_OF_TRIPS) ? 0x03 : 0x01,
351 zonedev, &tzone_ops, &pkg_temp_tz_params, 0, 0);
352 if (IS_ERR(zonedev->tzone)) {
353 err = PTR_ERR(zonedev->tzone);
354 goto out_kfree_trips;
355 }
356 err = thermal_zone_device_enable(zonedev->tzone);
357 if (err)
358 goto out_unregister_tz;
359
360 /* Store MSR value for package thermal interrupt, to restore at exit */
361 rdmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, zonedev->msr_pkg_therm_low,
362 zonedev->msr_pkg_therm_high);
363
364 cpumask_set_cpu(cpu, &zonedev->cpumask);
365 raw_spin_lock_irq(&pkg_temp_lock);
366 zones[id] = zonedev;
367 raw_spin_unlock_irq(&pkg_temp_lock);
368
369 return 0;
370
371 out_unregister_tz:
372 thermal_zone_device_unregister(zonedev->tzone);
373 out_kfree_trips:
374 kfree(zonedev->trips);
375 out_kfree_zonedev:
376 kfree(zonedev);
377 return err;
378 }
379
pkg_thermal_cpu_offline(unsigned int cpu)380 static int pkg_thermal_cpu_offline(unsigned int cpu)
381 {
382 struct zone_device *zonedev = pkg_temp_thermal_get_dev(cpu);
383 bool lastcpu, was_target;
384 int target;
385
386 if (!zonedev)
387 return 0;
388
389 target = cpumask_any_but(&zonedev->cpumask, cpu);
390 cpumask_clear_cpu(cpu, &zonedev->cpumask);
391 lastcpu = target >= nr_cpu_ids;
392 /*
393 * Remove the sysfs files, if this is the last cpu in the package
394 * before doing further cleanups.
395 */
396 if (lastcpu) {
397 struct thermal_zone_device *tzone = zonedev->tzone;
398
399 /*
400 * We must protect against a work function calling
401 * thermal_zone_update, after/while unregister. We null out
402 * the pointer under the zone mutex, so the worker function
403 * won't try to call.
404 */
405 mutex_lock(&thermal_zone_mutex);
406 zonedev->tzone = NULL;
407 mutex_unlock(&thermal_zone_mutex);
408
409 thermal_zone_device_unregister(tzone);
410 }
411
412 /* Protect against work and interrupts */
413 raw_spin_lock_irq(&pkg_temp_lock);
414
415 /*
416 * Check whether this cpu was the current target and store the new
417 * one. When we drop the lock, then the interrupt notify function
418 * will see the new target.
419 */
420 was_target = zonedev->cpu == cpu;
421 zonedev->cpu = target;
422
423 /*
424 * If this is the last CPU in the package remove the package
425 * reference from the array and restore the interrupt MSR. When we
426 * drop the lock neither the interrupt notify function nor the
427 * worker will see the package anymore.
428 */
429 if (lastcpu) {
430 zones[topology_logical_die_id(cpu)] = NULL;
431 /* After this point nothing touches the MSR anymore. */
432 wrmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT,
433 zonedev->msr_pkg_therm_low, zonedev->msr_pkg_therm_high);
434 }
435
436 /*
437 * Check whether there is work scheduled and whether the work is
438 * targeted at the outgoing CPU.
439 */
440 if (zonedev->work_scheduled && was_target) {
441 /*
442 * To cancel the work we need to drop the lock, otherwise
443 * we might deadlock if the work needs to be flushed.
444 */
445 raw_spin_unlock_irq(&pkg_temp_lock);
446 cancel_delayed_work_sync(&zonedev->work);
447 raw_spin_lock_irq(&pkg_temp_lock);
448 /*
449 * If this is not the last cpu in the package and the work
450 * did not run after we dropped the lock above, then we
451 * need to reschedule the work, otherwise the interrupt
452 * stays disabled forever.
453 */
454 if (!lastcpu && zonedev->work_scheduled)
455 pkg_thermal_schedule_work(target, &zonedev->work);
456 }
457
458 raw_spin_unlock_irq(&pkg_temp_lock);
459
460 /* Final cleanup if this is the last cpu */
461 if (lastcpu) {
462 kfree(zonedev->trips);
463 kfree(zonedev);
464 }
465 return 0;
466 }
467
pkg_thermal_cpu_online(unsigned int cpu)468 static int pkg_thermal_cpu_online(unsigned int cpu)
469 {
470 struct zone_device *zonedev = pkg_temp_thermal_get_dev(cpu);
471 struct cpuinfo_x86 *c = &cpu_data(cpu);
472
473 /* Paranoia check */
474 if (!cpu_has(c, X86_FEATURE_DTHERM) || !cpu_has(c, X86_FEATURE_PTS))
475 return -ENODEV;
476
477 /* If the package exists, nothing to do */
478 if (zonedev) {
479 cpumask_set_cpu(cpu, &zonedev->cpumask);
480 return 0;
481 }
482 return pkg_temp_thermal_device_add(cpu);
483 }
484
485 static const struct x86_cpu_id __initconst pkg_temp_thermal_ids[] = {
486 X86_MATCH_VENDOR_FEATURE(INTEL, X86_FEATURE_PTS, NULL),
487 {}
488 };
489 MODULE_DEVICE_TABLE(x86cpu, pkg_temp_thermal_ids);
490
pkg_temp_thermal_init(void)491 static int __init pkg_temp_thermal_init(void)
492 {
493 int ret;
494
495 if (!x86_match_cpu(pkg_temp_thermal_ids))
496 return -ENODEV;
497
498 max_id = topology_max_packages() * topology_max_die_per_package();
499 zones = kcalloc(max_id, sizeof(struct zone_device *),
500 GFP_KERNEL);
501 if (!zones)
502 return -ENOMEM;
503
504 ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "thermal/x86_pkg:online",
505 pkg_thermal_cpu_online, pkg_thermal_cpu_offline);
506 if (ret < 0)
507 goto err;
508
509 /* Store the state for module exit */
510 pkg_thermal_hp_state = ret;
511
512 platform_thermal_package_notify = pkg_thermal_notify;
513 platform_thermal_package_rate_control = pkg_thermal_rate_control;
514
515 /* Don't care if it fails */
516 pkg_temp_debugfs_init();
517 return 0;
518
519 err:
520 kfree(zones);
521 return ret;
522 }
module_init(pkg_temp_thermal_init)523 module_init(pkg_temp_thermal_init)
524
525 static void __exit pkg_temp_thermal_exit(void)
526 {
527 platform_thermal_package_notify = NULL;
528 platform_thermal_package_rate_control = NULL;
529
530 cpuhp_remove_state(pkg_thermal_hp_state);
531 debugfs_remove_recursive(debugfs);
532 kfree(zones);
533 }
534 module_exit(pkg_temp_thermal_exit)
535
536 MODULE_IMPORT_NS(INTEL_TCC);
537 MODULE_DESCRIPTION("X86 PKG TEMP Thermal Driver");
538 MODULE_AUTHOR("Srinivas Pandruvada <srinivas.pandruvada@linux.intel.com>");
539 MODULE_LICENSE("GPL v2");
540