1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * x86_pkg_temp_thermal driver
4  * Copyright (c) 2013, Intel Corporation.
5  */
6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
7 
8 #include <linux/module.h>
9 #include <linux/init.h>
10 #include <linux/err.h>
11 #include <linux/param.h>
12 #include <linux/device.h>
13 #include <linux/platform_device.h>
14 #include <linux/cpu.h>
15 #include <linux/smp.h>
16 #include <linux/slab.h>
17 #include <linux/pm.h>
18 #include <linux/thermal.h>
19 #include <linux/debugfs.h>
20 #include <asm/cpu_device_id.h>
21 #include <asm/mce.h>
22 
23 /*
24 * Rate control delay: Idea is to introduce denounce effect
25 * This should be long enough to avoid reduce events, when
26 * threshold is set to a temperature, which is constantly
27 * violated, but at the short enough to take any action.
28 * The action can be remove threshold or change it to next
29 * interesting setting. Based on experiments, in around
30 * every 5 seconds under load will give us a significant
31 * temperature change.
32 */
33 #define PKG_TEMP_THERMAL_NOTIFY_DELAY	5000
34 static int notify_delay_ms = PKG_TEMP_THERMAL_NOTIFY_DELAY;
35 module_param(notify_delay_ms, int, 0644);
36 MODULE_PARM_DESC(notify_delay_ms,
37 	"User space notification delay in milli seconds.");
38 
39 /* Number of trip points in thermal zone. Currently it can't
40 * be more than 2. MSR can allow setting and getting notifications
41 * for only 2 thresholds. This define enforces this, if there
42 * is some wrong values returned by cpuid for number of thresholds.
43 */
44 #define MAX_NUMBER_OF_TRIPS	2
45 
46 struct zone_device {
47 	int				cpu;
48 	bool				work_scheduled;
49 	u32				tj_max;
50 	u32				msr_pkg_therm_low;
51 	u32				msr_pkg_therm_high;
52 	struct delayed_work		work;
53 	struct thermal_zone_device	*tzone;
54 	struct cpumask			cpumask;
55 };
56 
57 static struct thermal_zone_params pkg_temp_tz_params = {
58 	.no_hwmon	= true,
59 };
60 
61 /* Keep track of how many zone pointers we allocated in init() */
62 static int max_id __read_mostly;
63 /* Array of zone pointers */
64 static struct zone_device **zones;
65 /* Serializes interrupt notification, work and hotplug */
66 static DEFINE_RAW_SPINLOCK(pkg_temp_lock);
67 /* Protects zone operation in the work function against hotplug removal */
68 static DEFINE_MUTEX(thermal_zone_mutex);
69 
70 /* The dynamically assigned cpu hotplug state for module_exit() */
71 static enum cpuhp_state pkg_thermal_hp_state __read_mostly;
72 
73 /* Debug counters to show using debugfs */
74 static struct dentry *debugfs;
75 static unsigned int pkg_interrupt_cnt;
76 static unsigned int pkg_work_cnt;
77 
78 static void pkg_temp_debugfs_init(void)
79 {
80 	debugfs = debugfs_create_dir("pkg_temp_thermal", NULL);
81 
82 	debugfs_create_u32("pkg_thres_interrupt", S_IRUGO, debugfs,
83 			   &pkg_interrupt_cnt);
84 	debugfs_create_u32("pkg_thres_work", S_IRUGO, debugfs,
85 			   &pkg_work_cnt);
86 }
87 
88 /*
89  * Protection:
90  *
91  * - cpu hotplug: Read serialized by cpu hotplug lock
92  *		  Write must hold pkg_temp_lock
93  *
94  * - Other callsites: Must hold pkg_temp_lock
95  */
96 static struct zone_device *pkg_temp_thermal_get_dev(unsigned int cpu)
97 {
98 	int id = topology_logical_die_id(cpu);
99 
100 	if (id >= 0 && id < max_id)
101 		return zones[id];
102 	return NULL;
103 }
104 
105 /*
106 * tj-max is is interesting because threshold is set relative to this
107 * temperature.
108 */
109 static int get_tj_max(int cpu, u32 *tj_max)
110 {
111 	u32 eax, edx, val;
112 	int err;
113 
114 	err = rdmsr_safe_on_cpu(cpu, MSR_IA32_TEMPERATURE_TARGET, &eax, &edx);
115 	if (err)
116 		return err;
117 
118 	val = (eax >> 16) & 0xff;
119 	*tj_max = val * 1000;
120 
121 	return val ? 0 : -EINVAL;
122 }
123 
124 static int sys_get_curr_temp(struct thermal_zone_device *tzd, int *temp)
125 {
126 	struct zone_device *zonedev = tzd->devdata;
127 	u32 eax, edx;
128 
129 	rdmsr_on_cpu(zonedev->cpu, MSR_IA32_PACKAGE_THERM_STATUS,
130 			&eax, &edx);
131 	if (eax & 0x80000000) {
132 		*temp = zonedev->tj_max - ((eax >> 16) & 0x7f) * 1000;
133 		pr_debug("sys_get_curr_temp %d\n", *temp);
134 		return 0;
135 	}
136 	return -EINVAL;
137 }
138 
139 static int sys_get_trip_temp(struct thermal_zone_device *tzd,
140 			     int trip, int *temp)
141 {
142 	struct zone_device *zonedev = tzd->devdata;
143 	unsigned long thres_reg_value;
144 	u32 mask, shift, eax, edx;
145 	int ret;
146 
147 	if (trip >= MAX_NUMBER_OF_TRIPS)
148 		return -EINVAL;
149 
150 	if (trip) {
151 		mask = THERM_MASK_THRESHOLD1;
152 		shift = THERM_SHIFT_THRESHOLD1;
153 	} else {
154 		mask = THERM_MASK_THRESHOLD0;
155 		shift = THERM_SHIFT_THRESHOLD0;
156 	}
157 
158 	ret = rdmsr_on_cpu(zonedev->cpu, MSR_IA32_PACKAGE_THERM_INTERRUPT,
159 			   &eax, &edx);
160 	if (ret < 0)
161 		return ret;
162 
163 	thres_reg_value = (eax & mask) >> shift;
164 	if (thres_reg_value)
165 		*temp = zonedev->tj_max - thres_reg_value * 1000;
166 	else
167 		*temp = 0;
168 	pr_debug("sys_get_trip_temp %d\n", *temp);
169 
170 	return 0;
171 }
172 
173 static int
174 sys_set_trip_temp(struct thermal_zone_device *tzd, int trip, int temp)
175 {
176 	struct zone_device *zonedev = tzd->devdata;
177 	u32 l, h, mask, shift, intr;
178 	int ret;
179 
180 	if (trip >= MAX_NUMBER_OF_TRIPS || temp >= zonedev->tj_max)
181 		return -EINVAL;
182 
183 	ret = rdmsr_on_cpu(zonedev->cpu, MSR_IA32_PACKAGE_THERM_INTERRUPT,
184 			   &l, &h);
185 	if (ret < 0)
186 		return ret;
187 
188 	if (trip) {
189 		mask = THERM_MASK_THRESHOLD1;
190 		shift = THERM_SHIFT_THRESHOLD1;
191 		intr = THERM_INT_THRESHOLD1_ENABLE;
192 	} else {
193 		mask = THERM_MASK_THRESHOLD0;
194 		shift = THERM_SHIFT_THRESHOLD0;
195 		intr = THERM_INT_THRESHOLD0_ENABLE;
196 	}
197 	l &= ~mask;
198 	/*
199 	* When users space sets a trip temperature == 0, which is indication
200 	* that, it is no longer interested in receiving notifications.
201 	*/
202 	if (!temp) {
203 		l &= ~intr;
204 	} else {
205 		l |= (zonedev->tj_max - temp)/1000 << shift;
206 		l |= intr;
207 	}
208 
209 	return wrmsr_on_cpu(zonedev->cpu, MSR_IA32_PACKAGE_THERM_INTERRUPT,
210 			l, h);
211 }
212 
213 static int sys_get_trip_type(struct thermal_zone_device *thermal, int trip,
214 			     enum thermal_trip_type *type)
215 {
216 	*type = THERMAL_TRIP_PASSIVE;
217 	return 0;
218 }
219 
220 /* Thermal zone callback registry */
221 static struct thermal_zone_device_ops tzone_ops = {
222 	.get_temp = sys_get_curr_temp,
223 	.get_trip_temp = sys_get_trip_temp,
224 	.get_trip_type = sys_get_trip_type,
225 	.set_trip_temp = sys_set_trip_temp,
226 };
227 
228 static bool pkg_thermal_rate_control(void)
229 {
230 	return true;
231 }
232 
233 /* Enable threshold interrupt on local package/cpu */
234 static inline void enable_pkg_thres_interrupt(void)
235 {
236 	u8 thres_0, thres_1;
237 	u32 l, h;
238 
239 	rdmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);
240 	/* only enable/disable if it had valid threshold value */
241 	thres_0 = (l & THERM_MASK_THRESHOLD0) >> THERM_SHIFT_THRESHOLD0;
242 	thres_1 = (l & THERM_MASK_THRESHOLD1) >> THERM_SHIFT_THRESHOLD1;
243 	if (thres_0)
244 		l |= THERM_INT_THRESHOLD0_ENABLE;
245 	if (thres_1)
246 		l |= THERM_INT_THRESHOLD1_ENABLE;
247 	wrmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);
248 }
249 
250 /* Disable threshold interrupt on local package/cpu */
251 static inline void disable_pkg_thres_interrupt(void)
252 {
253 	u32 l, h;
254 
255 	rdmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);
256 
257 	l &= ~(THERM_INT_THRESHOLD0_ENABLE | THERM_INT_THRESHOLD1_ENABLE);
258 	wrmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);
259 }
260 
261 static void pkg_temp_thermal_threshold_work_fn(struct work_struct *work)
262 {
263 	struct thermal_zone_device *tzone = NULL;
264 	int cpu = smp_processor_id();
265 	struct zone_device *zonedev;
266 	u64 msr_val, wr_val;
267 
268 	mutex_lock(&thermal_zone_mutex);
269 	raw_spin_lock_irq(&pkg_temp_lock);
270 	++pkg_work_cnt;
271 
272 	zonedev = pkg_temp_thermal_get_dev(cpu);
273 	if (!zonedev) {
274 		raw_spin_unlock_irq(&pkg_temp_lock);
275 		mutex_unlock(&thermal_zone_mutex);
276 		return;
277 	}
278 	zonedev->work_scheduled = false;
279 
280 	rdmsrl(MSR_IA32_PACKAGE_THERM_STATUS, msr_val);
281 	wr_val = msr_val & ~(THERM_LOG_THRESHOLD0 | THERM_LOG_THRESHOLD1);
282 	if (wr_val != msr_val) {
283 		wrmsrl(MSR_IA32_PACKAGE_THERM_STATUS, wr_val);
284 		tzone = zonedev->tzone;
285 	}
286 
287 	enable_pkg_thres_interrupt();
288 	raw_spin_unlock_irq(&pkg_temp_lock);
289 
290 	/*
291 	 * If tzone is not NULL, then thermal_zone_mutex will prevent the
292 	 * concurrent removal in the cpu offline callback.
293 	 */
294 	if (tzone)
295 		thermal_zone_device_update(tzone, THERMAL_EVENT_UNSPECIFIED);
296 
297 	mutex_unlock(&thermal_zone_mutex);
298 }
299 
300 static void pkg_thermal_schedule_work(int cpu, struct delayed_work *work)
301 {
302 	unsigned long ms = msecs_to_jiffies(notify_delay_ms);
303 
304 	schedule_delayed_work_on(cpu, work, ms);
305 }
306 
307 static int pkg_thermal_notify(u64 msr_val)
308 {
309 	int cpu = smp_processor_id();
310 	struct zone_device *zonedev;
311 	unsigned long flags;
312 
313 	raw_spin_lock_irqsave(&pkg_temp_lock, flags);
314 	++pkg_interrupt_cnt;
315 
316 	disable_pkg_thres_interrupt();
317 
318 	/* Work is per package, so scheduling it once is enough. */
319 	zonedev = pkg_temp_thermal_get_dev(cpu);
320 	if (zonedev && !zonedev->work_scheduled) {
321 		zonedev->work_scheduled = true;
322 		pkg_thermal_schedule_work(zonedev->cpu, &zonedev->work);
323 	}
324 
325 	raw_spin_unlock_irqrestore(&pkg_temp_lock, flags);
326 	return 0;
327 }
328 
329 static int pkg_temp_thermal_device_add(unsigned int cpu)
330 {
331 	int id = topology_logical_die_id(cpu);
332 	u32 tj_max, eax, ebx, ecx, edx;
333 	struct zone_device *zonedev;
334 	int thres_count, err;
335 
336 	if (id >= max_id)
337 		return -ENOMEM;
338 
339 	cpuid(6, &eax, &ebx, &ecx, &edx);
340 	thres_count = ebx & 0x07;
341 	if (!thres_count)
342 		return -ENODEV;
343 
344 	thres_count = clamp_val(thres_count, 0, MAX_NUMBER_OF_TRIPS);
345 
346 	err = get_tj_max(cpu, &tj_max);
347 	if (err)
348 		return err;
349 
350 	zonedev = kzalloc(sizeof(*zonedev), GFP_KERNEL);
351 	if (!zonedev)
352 		return -ENOMEM;
353 
354 	INIT_DELAYED_WORK(&zonedev->work, pkg_temp_thermal_threshold_work_fn);
355 	zonedev->cpu = cpu;
356 	zonedev->tj_max = tj_max;
357 	zonedev->tzone = thermal_zone_device_register("x86_pkg_temp",
358 			thres_count,
359 			(thres_count == MAX_NUMBER_OF_TRIPS) ? 0x03 : 0x01,
360 			zonedev, &tzone_ops, &pkg_temp_tz_params, 0, 0);
361 	if (IS_ERR(zonedev->tzone)) {
362 		err = PTR_ERR(zonedev->tzone);
363 		kfree(zonedev);
364 		return err;
365 	}
366 	err = thermal_zone_device_enable(zonedev->tzone);
367 	if (err) {
368 		thermal_zone_device_unregister(zonedev->tzone);
369 		kfree(zonedev);
370 		return err;
371 	}
372 	/* Store MSR value for package thermal interrupt, to restore at exit */
373 	rdmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, zonedev->msr_pkg_therm_low,
374 	      zonedev->msr_pkg_therm_high);
375 
376 	cpumask_set_cpu(cpu, &zonedev->cpumask);
377 	raw_spin_lock_irq(&pkg_temp_lock);
378 	zones[id] = zonedev;
379 	raw_spin_unlock_irq(&pkg_temp_lock);
380 	return 0;
381 }
382 
383 static int pkg_thermal_cpu_offline(unsigned int cpu)
384 {
385 	struct zone_device *zonedev = pkg_temp_thermal_get_dev(cpu);
386 	bool lastcpu, was_target;
387 	int target;
388 
389 	if (!zonedev)
390 		return 0;
391 
392 	target = cpumask_any_but(&zonedev->cpumask, cpu);
393 	cpumask_clear_cpu(cpu, &zonedev->cpumask);
394 	lastcpu = target >= nr_cpu_ids;
395 	/*
396 	 * Remove the sysfs files, if this is the last cpu in the package
397 	 * before doing further cleanups.
398 	 */
399 	if (lastcpu) {
400 		struct thermal_zone_device *tzone = zonedev->tzone;
401 
402 		/*
403 		 * We must protect against a work function calling
404 		 * thermal_zone_update, after/while unregister. We null out
405 		 * the pointer under the zone mutex, so the worker function
406 		 * won't try to call.
407 		 */
408 		mutex_lock(&thermal_zone_mutex);
409 		zonedev->tzone = NULL;
410 		mutex_unlock(&thermal_zone_mutex);
411 
412 		thermal_zone_device_unregister(tzone);
413 	}
414 
415 	/* Protect against work and interrupts */
416 	raw_spin_lock_irq(&pkg_temp_lock);
417 
418 	/*
419 	 * Check whether this cpu was the current target and store the new
420 	 * one. When we drop the lock, then the interrupt notify function
421 	 * will see the new target.
422 	 */
423 	was_target = zonedev->cpu == cpu;
424 	zonedev->cpu = target;
425 
426 	/*
427 	 * If this is the last CPU in the package remove the package
428 	 * reference from the array and restore the interrupt MSR. When we
429 	 * drop the lock neither the interrupt notify function nor the
430 	 * worker will see the package anymore.
431 	 */
432 	if (lastcpu) {
433 		zones[topology_logical_die_id(cpu)] = NULL;
434 		/* After this point nothing touches the MSR anymore. */
435 		wrmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT,
436 		      zonedev->msr_pkg_therm_low, zonedev->msr_pkg_therm_high);
437 	}
438 
439 	/*
440 	 * Check whether there is work scheduled and whether the work is
441 	 * targeted at the outgoing CPU.
442 	 */
443 	if (zonedev->work_scheduled && was_target) {
444 		/*
445 		 * To cancel the work we need to drop the lock, otherwise
446 		 * we might deadlock if the work needs to be flushed.
447 		 */
448 		raw_spin_unlock_irq(&pkg_temp_lock);
449 		cancel_delayed_work_sync(&zonedev->work);
450 		raw_spin_lock_irq(&pkg_temp_lock);
451 		/*
452 		 * If this is not the last cpu in the package and the work
453 		 * did not run after we dropped the lock above, then we
454 		 * need to reschedule the work, otherwise the interrupt
455 		 * stays disabled forever.
456 		 */
457 		if (!lastcpu && zonedev->work_scheduled)
458 			pkg_thermal_schedule_work(target, &zonedev->work);
459 	}
460 
461 	raw_spin_unlock_irq(&pkg_temp_lock);
462 
463 	/* Final cleanup if this is the last cpu */
464 	if (lastcpu)
465 		kfree(zonedev);
466 	return 0;
467 }
468 
469 static int pkg_thermal_cpu_online(unsigned int cpu)
470 {
471 	struct zone_device *zonedev = pkg_temp_thermal_get_dev(cpu);
472 	struct cpuinfo_x86 *c = &cpu_data(cpu);
473 
474 	/* Paranoia check */
475 	if (!cpu_has(c, X86_FEATURE_DTHERM) || !cpu_has(c, X86_FEATURE_PTS))
476 		return -ENODEV;
477 
478 	/* If the package exists, nothing to do */
479 	if (zonedev) {
480 		cpumask_set_cpu(cpu, &zonedev->cpumask);
481 		return 0;
482 	}
483 	return pkg_temp_thermal_device_add(cpu);
484 }
485 
486 static const struct x86_cpu_id __initconst pkg_temp_thermal_ids[] = {
487 	X86_MATCH_VENDOR_FEATURE(INTEL, X86_FEATURE_PTS, NULL),
488 	{}
489 };
490 MODULE_DEVICE_TABLE(x86cpu, pkg_temp_thermal_ids);
491 
492 static int __init pkg_temp_thermal_init(void)
493 {
494 	int ret;
495 
496 	if (!x86_match_cpu(pkg_temp_thermal_ids))
497 		return -ENODEV;
498 
499 	max_id = topology_max_packages() * topology_max_die_per_package();
500 	zones = kcalloc(max_id, sizeof(struct zone_device *),
501 			   GFP_KERNEL);
502 	if (!zones)
503 		return -ENOMEM;
504 
505 	ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "thermal/x86_pkg:online",
506 				pkg_thermal_cpu_online,	pkg_thermal_cpu_offline);
507 	if (ret < 0)
508 		goto err;
509 
510 	/* Store the state for module exit */
511 	pkg_thermal_hp_state = ret;
512 
513 	platform_thermal_package_notify = pkg_thermal_notify;
514 	platform_thermal_package_rate_control = pkg_thermal_rate_control;
515 
516 	 /* Don't care if it fails */
517 	pkg_temp_debugfs_init();
518 	return 0;
519 
520 err:
521 	kfree(zones);
522 	return ret;
523 }
524 module_init(pkg_temp_thermal_init)
525 
526 static void __exit pkg_temp_thermal_exit(void)
527 {
528 	platform_thermal_package_notify = NULL;
529 	platform_thermal_package_rate_control = NULL;
530 
531 	cpuhp_remove_state(pkg_thermal_hp_state);
532 	debugfs_remove_recursive(debugfs);
533 	kfree(zones);
534 }
535 module_exit(pkg_temp_thermal_exit)
536 
537 MODULE_DESCRIPTION("X86 PKG TEMP Thermal Driver");
538 MODULE_AUTHOR("Srinivas Pandruvada <srinivas.pandruvada@linux.intel.com>");
539 MODULE_LICENSE("GPL v2");
540