1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * x86_pkg_temp_thermal driver
4  * Copyright (c) 2013, Intel Corporation.
5  */
6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
7 
8 #include <linux/module.h>
9 #include <linux/init.h>
10 #include <linux/intel_tcc.h>
11 #include <linux/err.h>
12 #include <linux/param.h>
13 #include <linux/device.h>
14 #include <linux/platform_device.h>
15 #include <linux/cpu.h>
16 #include <linux/smp.h>
17 #include <linux/slab.h>
18 #include <linux/pm.h>
19 #include <linux/thermal.h>
20 #include <linux/debugfs.h>
21 
22 #include <asm/cpu_device_id.h>
23 
24 #include "thermal_interrupt.h"
25 
26 /*
27 * Rate control delay: Idea is to introduce denounce effect
28 * This should be long enough to avoid reduce events, when
29 * threshold is set to a temperature, which is constantly
30 * violated, but at the short enough to take any action.
31 * The action can be remove threshold or change it to next
32 * interesting setting. Based on experiments, in around
33 * every 5 seconds under load will give us a significant
34 * temperature change.
35 */
36 #define PKG_TEMP_THERMAL_NOTIFY_DELAY	5000
37 static int notify_delay_ms = PKG_TEMP_THERMAL_NOTIFY_DELAY;
38 module_param(notify_delay_ms, int, 0644);
39 MODULE_PARM_DESC(notify_delay_ms,
40 	"User space notification delay in milli seconds.");
41 
42 /* Number of trip points in thermal zone. Currently it can't
43 * be more than 2. MSR can allow setting and getting notifications
44 * for only 2 thresholds. This define enforces this, if there
45 * is some wrong values returned by cpuid for number of thresholds.
46 */
47 #define MAX_NUMBER_OF_TRIPS	2
48 
49 struct zone_device {
50 	int				cpu;
51 	bool				work_scheduled;
52 	u32				msr_pkg_therm_low;
53 	u32				msr_pkg_therm_high;
54 	struct delayed_work		work;
55 	struct thermal_zone_device	*tzone;
56 	struct thermal_trip		*trips;
57 	struct cpumask			cpumask;
58 };
59 
60 static struct thermal_zone_params pkg_temp_tz_params = {
61 	.no_hwmon	= true,
62 };
63 
64 /* Keep track of how many zone pointers we allocated in init() */
65 static int max_id __read_mostly;
66 /* Array of zone pointers */
67 static struct zone_device **zones;
68 /* Serializes interrupt notification, work and hotplug */
69 static DEFINE_RAW_SPINLOCK(pkg_temp_lock);
70 /* Protects zone operation in the work function against hotplug removal */
71 static DEFINE_MUTEX(thermal_zone_mutex);
72 
73 /* The dynamically assigned cpu hotplug state for module_exit() */
74 static enum cpuhp_state pkg_thermal_hp_state __read_mostly;
75 
76 /* Debug counters to show using debugfs */
77 static struct dentry *debugfs;
78 static unsigned int pkg_interrupt_cnt;
79 static unsigned int pkg_work_cnt;
80 
81 static void pkg_temp_debugfs_init(void)
82 {
83 	debugfs = debugfs_create_dir("pkg_temp_thermal", NULL);
84 
85 	debugfs_create_u32("pkg_thres_interrupt", S_IRUGO, debugfs,
86 			   &pkg_interrupt_cnt);
87 	debugfs_create_u32("pkg_thres_work", S_IRUGO, debugfs,
88 			   &pkg_work_cnt);
89 }
90 
91 /*
92  * Protection:
93  *
94  * - cpu hotplug: Read serialized by cpu hotplug lock
95  *		  Write must hold pkg_temp_lock
96  *
97  * - Other callsites: Must hold pkg_temp_lock
98  */
99 static struct zone_device *pkg_temp_thermal_get_dev(unsigned int cpu)
100 {
101 	int id = topology_logical_die_id(cpu);
102 
103 	if (id >= 0 && id < max_id)
104 		return zones[id];
105 	return NULL;
106 }
107 
108 static int sys_get_curr_temp(struct thermal_zone_device *tzd, int *temp)
109 {
110 	struct zone_device *zonedev = tzd->devdata;
111 	int val;
112 
113 	val = intel_tcc_get_temp(zonedev->cpu, true);
114 	if (val < 0)
115 		return val;
116 
117 	*temp = val * 1000;
118 	pr_debug("sys_get_curr_temp %d\n", *temp);
119 	return 0;
120 }
121 
122 static int
123 sys_set_trip_temp(struct thermal_zone_device *tzd, int trip, int temp)
124 {
125 	struct zone_device *zonedev = tzd->devdata;
126 	u32 l, h, mask, shift, intr;
127 	int tj_max, ret;
128 
129 	tj_max = intel_tcc_get_tjmax(zonedev->cpu);
130 	if (tj_max < 0)
131 		return tj_max;
132 	tj_max *= 1000;
133 
134 	if (trip >= MAX_NUMBER_OF_TRIPS || temp >= tj_max)
135 		return -EINVAL;
136 
137 	ret = rdmsr_on_cpu(zonedev->cpu, MSR_IA32_PACKAGE_THERM_INTERRUPT,
138 			   &l, &h);
139 	if (ret < 0)
140 		return ret;
141 
142 	if (trip) {
143 		mask = THERM_MASK_THRESHOLD1;
144 		shift = THERM_SHIFT_THRESHOLD1;
145 		intr = THERM_INT_THRESHOLD1_ENABLE;
146 	} else {
147 		mask = THERM_MASK_THRESHOLD0;
148 		shift = THERM_SHIFT_THRESHOLD0;
149 		intr = THERM_INT_THRESHOLD0_ENABLE;
150 	}
151 	l &= ~mask;
152 	/*
153 	* When users space sets a trip temperature == 0, which is indication
154 	* that, it is no longer interested in receiving notifications.
155 	*/
156 	if (!temp) {
157 		l &= ~intr;
158 	} else {
159 		l |= (tj_max - temp)/1000 << shift;
160 		l |= intr;
161 	}
162 
163 	return wrmsr_on_cpu(zonedev->cpu, MSR_IA32_PACKAGE_THERM_INTERRUPT,
164 			l, h);
165 }
166 
167 /* Thermal zone callback registry */
168 static struct thermal_zone_device_ops tzone_ops = {
169 	.get_temp = sys_get_curr_temp,
170 	.set_trip_temp = sys_set_trip_temp,
171 };
172 
173 static bool pkg_thermal_rate_control(void)
174 {
175 	return true;
176 }
177 
178 /* Enable threshold interrupt on local package/cpu */
179 static inline void enable_pkg_thres_interrupt(void)
180 {
181 	u8 thres_0, thres_1;
182 	u32 l, h;
183 
184 	rdmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);
185 	/* only enable/disable if it had valid threshold value */
186 	thres_0 = (l & THERM_MASK_THRESHOLD0) >> THERM_SHIFT_THRESHOLD0;
187 	thres_1 = (l & THERM_MASK_THRESHOLD1) >> THERM_SHIFT_THRESHOLD1;
188 	if (thres_0)
189 		l |= THERM_INT_THRESHOLD0_ENABLE;
190 	if (thres_1)
191 		l |= THERM_INT_THRESHOLD1_ENABLE;
192 	wrmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);
193 }
194 
195 /* Disable threshold interrupt on local package/cpu */
196 static inline void disable_pkg_thres_interrupt(void)
197 {
198 	u32 l, h;
199 
200 	rdmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);
201 
202 	l &= ~(THERM_INT_THRESHOLD0_ENABLE | THERM_INT_THRESHOLD1_ENABLE);
203 	wrmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);
204 }
205 
206 static void pkg_temp_thermal_threshold_work_fn(struct work_struct *work)
207 {
208 	struct thermal_zone_device *tzone = NULL;
209 	int cpu = smp_processor_id();
210 	struct zone_device *zonedev;
211 
212 	mutex_lock(&thermal_zone_mutex);
213 	raw_spin_lock_irq(&pkg_temp_lock);
214 	++pkg_work_cnt;
215 
216 	zonedev = pkg_temp_thermal_get_dev(cpu);
217 	if (!zonedev) {
218 		raw_spin_unlock_irq(&pkg_temp_lock);
219 		mutex_unlock(&thermal_zone_mutex);
220 		return;
221 	}
222 	zonedev->work_scheduled = false;
223 
224 	thermal_clear_package_intr_status(PACKAGE_LEVEL, THERM_LOG_THRESHOLD0 | THERM_LOG_THRESHOLD1);
225 	tzone = zonedev->tzone;
226 
227 	enable_pkg_thres_interrupt();
228 	raw_spin_unlock_irq(&pkg_temp_lock);
229 
230 	/*
231 	 * If tzone is not NULL, then thermal_zone_mutex will prevent the
232 	 * concurrent removal in the cpu offline callback.
233 	 */
234 	if (tzone)
235 		thermal_zone_device_update(tzone, THERMAL_EVENT_UNSPECIFIED);
236 
237 	mutex_unlock(&thermal_zone_mutex);
238 }
239 
240 static void pkg_thermal_schedule_work(int cpu, struct delayed_work *work)
241 {
242 	unsigned long ms = msecs_to_jiffies(notify_delay_ms);
243 
244 	schedule_delayed_work_on(cpu, work, ms);
245 }
246 
247 static int pkg_thermal_notify(u64 msr_val)
248 {
249 	int cpu = smp_processor_id();
250 	struct zone_device *zonedev;
251 	unsigned long flags;
252 
253 	raw_spin_lock_irqsave(&pkg_temp_lock, flags);
254 	++pkg_interrupt_cnt;
255 
256 	disable_pkg_thres_interrupt();
257 
258 	/* Work is per package, so scheduling it once is enough. */
259 	zonedev = pkg_temp_thermal_get_dev(cpu);
260 	if (zonedev && !zonedev->work_scheduled) {
261 		zonedev->work_scheduled = true;
262 		pkg_thermal_schedule_work(zonedev->cpu, &zonedev->work);
263 	}
264 
265 	raw_spin_unlock_irqrestore(&pkg_temp_lock, flags);
266 	return 0;
267 }
268 
269 static struct thermal_trip *pkg_temp_thermal_trips_init(int cpu, int tj_max, int num_trips)
270 {
271 	struct thermal_trip *trips;
272 	unsigned long thres_reg_value;
273 	u32 mask, shift, eax, edx;
274 	int ret, i;
275 
276 	trips = kzalloc(sizeof(*trips) * num_trips, GFP_KERNEL);
277 	if (!trips)
278 		return ERR_PTR(-ENOMEM);
279 
280 	for (i = 0; i < num_trips; i++) {
281 
282 		if (i) {
283 			mask = THERM_MASK_THRESHOLD1;
284 			shift = THERM_SHIFT_THRESHOLD1;
285 		} else {
286 			mask = THERM_MASK_THRESHOLD0;
287 			shift = THERM_SHIFT_THRESHOLD0;
288 		}
289 
290 		ret = rdmsr_on_cpu(cpu, MSR_IA32_PACKAGE_THERM_INTERRUPT,
291 				   &eax, &edx);
292 		if (ret < 0) {
293 			kfree(trips);
294 			return ERR_PTR(ret);
295 		}
296 
297 		thres_reg_value = (eax & mask) >> shift;
298 
299 		trips[i].temperature = thres_reg_value ?
300 			tj_max - thres_reg_value * 1000 : THERMAL_TEMP_INVALID;
301 
302 		trips[i].type = THERMAL_TRIP_PASSIVE;
303 
304 		pr_debug("%s: cpu=%d, trip=%d, temp=%d\n",
305 			 __func__, cpu, i, trips[i].temperature);
306 	}
307 
308 	return trips;
309 }
310 
311 static int pkg_temp_thermal_device_add(unsigned int cpu)
312 {
313 	int id = topology_logical_die_id(cpu);
314 	u32 eax, ebx, ecx, edx;
315 	struct zone_device *zonedev;
316 	int thres_count, err;
317 	int tj_max;
318 
319 	if (id >= max_id)
320 		return -ENOMEM;
321 
322 	cpuid(6, &eax, &ebx, &ecx, &edx);
323 	thres_count = ebx & 0x07;
324 	if (!thres_count)
325 		return -ENODEV;
326 
327 	thres_count = clamp_val(thres_count, 0, MAX_NUMBER_OF_TRIPS);
328 
329 	tj_max = intel_tcc_get_tjmax(cpu);
330 	if (tj_max < 0)
331 		return tj_max;
332 
333 	zonedev = kzalloc(sizeof(*zonedev), GFP_KERNEL);
334 	if (!zonedev)
335 		return -ENOMEM;
336 
337 	zonedev->trips = pkg_temp_thermal_trips_init(cpu, tj_max, thres_count);
338 	if (IS_ERR(zonedev->trips)) {
339 		err = PTR_ERR(zonedev->trips);
340 		goto out_kfree_zonedev;
341 	}
342 
343 	INIT_DELAYED_WORK(&zonedev->work, pkg_temp_thermal_threshold_work_fn);
344 	zonedev->cpu = cpu;
345 	zonedev->tzone = thermal_zone_device_register_with_trips("x86_pkg_temp",
346 			zonedev->trips, thres_count,
347 			(thres_count == MAX_NUMBER_OF_TRIPS) ? 0x03 : 0x01,
348 			zonedev, &tzone_ops, &pkg_temp_tz_params, 0, 0);
349 	if (IS_ERR(zonedev->tzone)) {
350 		err = PTR_ERR(zonedev->tzone);
351 		goto out_kfree_trips;
352 	}
353 	err = thermal_zone_device_enable(zonedev->tzone);
354 	if (err)
355 		goto out_unregister_tz;
356 
357 	/* Store MSR value for package thermal interrupt, to restore at exit */
358 	rdmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, zonedev->msr_pkg_therm_low,
359 	      zonedev->msr_pkg_therm_high);
360 
361 	cpumask_set_cpu(cpu, &zonedev->cpumask);
362 	raw_spin_lock_irq(&pkg_temp_lock);
363 	zones[id] = zonedev;
364 	raw_spin_unlock_irq(&pkg_temp_lock);
365 
366 	return 0;
367 
368 out_unregister_tz:
369 	thermal_zone_device_unregister(zonedev->tzone);
370 out_kfree_trips:
371 	kfree(zonedev->trips);
372 out_kfree_zonedev:
373 	kfree(zonedev);
374 	return err;
375 }
376 
377 static int pkg_thermal_cpu_offline(unsigned int cpu)
378 {
379 	struct zone_device *zonedev = pkg_temp_thermal_get_dev(cpu);
380 	bool lastcpu, was_target;
381 	int target;
382 
383 	if (!zonedev)
384 		return 0;
385 
386 	target = cpumask_any_but(&zonedev->cpumask, cpu);
387 	cpumask_clear_cpu(cpu, &zonedev->cpumask);
388 	lastcpu = target >= nr_cpu_ids;
389 	/*
390 	 * Remove the sysfs files, if this is the last cpu in the package
391 	 * before doing further cleanups.
392 	 */
393 	if (lastcpu) {
394 		struct thermal_zone_device *tzone = zonedev->tzone;
395 
396 		/*
397 		 * We must protect against a work function calling
398 		 * thermal_zone_update, after/while unregister. We null out
399 		 * the pointer under the zone mutex, so the worker function
400 		 * won't try to call.
401 		 */
402 		mutex_lock(&thermal_zone_mutex);
403 		zonedev->tzone = NULL;
404 		mutex_unlock(&thermal_zone_mutex);
405 
406 		thermal_zone_device_unregister(tzone);
407 	}
408 
409 	/* Protect against work and interrupts */
410 	raw_spin_lock_irq(&pkg_temp_lock);
411 
412 	/*
413 	 * Check whether this cpu was the current target and store the new
414 	 * one. When we drop the lock, then the interrupt notify function
415 	 * will see the new target.
416 	 */
417 	was_target = zonedev->cpu == cpu;
418 	zonedev->cpu = target;
419 
420 	/*
421 	 * If this is the last CPU in the package remove the package
422 	 * reference from the array and restore the interrupt MSR. When we
423 	 * drop the lock neither the interrupt notify function nor the
424 	 * worker will see the package anymore.
425 	 */
426 	if (lastcpu) {
427 		zones[topology_logical_die_id(cpu)] = NULL;
428 		/* After this point nothing touches the MSR anymore. */
429 		wrmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT,
430 		      zonedev->msr_pkg_therm_low, zonedev->msr_pkg_therm_high);
431 	}
432 
433 	/*
434 	 * Check whether there is work scheduled and whether the work is
435 	 * targeted at the outgoing CPU.
436 	 */
437 	if (zonedev->work_scheduled && was_target) {
438 		/*
439 		 * To cancel the work we need to drop the lock, otherwise
440 		 * we might deadlock if the work needs to be flushed.
441 		 */
442 		raw_spin_unlock_irq(&pkg_temp_lock);
443 		cancel_delayed_work_sync(&zonedev->work);
444 		raw_spin_lock_irq(&pkg_temp_lock);
445 		/*
446 		 * If this is not the last cpu in the package and the work
447 		 * did not run after we dropped the lock above, then we
448 		 * need to reschedule the work, otherwise the interrupt
449 		 * stays disabled forever.
450 		 */
451 		if (!lastcpu && zonedev->work_scheduled)
452 			pkg_thermal_schedule_work(target, &zonedev->work);
453 	}
454 
455 	raw_spin_unlock_irq(&pkg_temp_lock);
456 
457 	/* Final cleanup if this is the last cpu */
458 	if (lastcpu) {
459 		kfree(zonedev->trips);
460 		kfree(zonedev);
461 	}
462 	return 0;
463 }
464 
465 static int pkg_thermal_cpu_online(unsigned int cpu)
466 {
467 	struct zone_device *zonedev = pkg_temp_thermal_get_dev(cpu);
468 	struct cpuinfo_x86 *c = &cpu_data(cpu);
469 
470 	/* Paranoia check */
471 	if (!cpu_has(c, X86_FEATURE_DTHERM) || !cpu_has(c, X86_FEATURE_PTS))
472 		return -ENODEV;
473 
474 	/* If the package exists, nothing to do */
475 	if (zonedev) {
476 		cpumask_set_cpu(cpu, &zonedev->cpumask);
477 		return 0;
478 	}
479 	return pkg_temp_thermal_device_add(cpu);
480 }
481 
482 static const struct x86_cpu_id __initconst pkg_temp_thermal_ids[] = {
483 	X86_MATCH_VENDOR_FEATURE(INTEL, X86_FEATURE_PTS, NULL),
484 	{}
485 };
486 MODULE_DEVICE_TABLE(x86cpu, pkg_temp_thermal_ids);
487 
488 static int __init pkg_temp_thermal_init(void)
489 {
490 	int ret;
491 
492 	if (!x86_match_cpu(pkg_temp_thermal_ids))
493 		return -ENODEV;
494 
495 	max_id = topology_max_packages() * topology_max_die_per_package();
496 	zones = kcalloc(max_id, sizeof(struct zone_device *),
497 			   GFP_KERNEL);
498 	if (!zones)
499 		return -ENOMEM;
500 
501 	ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "thermal/x86_pkg:online",
502 				pkg_thermal_cpu_online,	pkg_thermal_cpu_offline);
503 	if (ret < 0)
504 		goto err;
505 
506 	/* Store the state for module exit */
507 	pkg_thermal_hp_state = ret;
508 
509 	platform_thermal_package_notify = pkg_thermal_notify;
510 	platform_thermal_package_rate_control = pkg_thermal_rate_control;
511 
512 	 /* Don't care if it fails */
513 	pkg_temp_debugfs_init();
514 	return 0;
515 
516 err:
517 	kfree(zones);
518 	return ret;
519 }
520 module_init(pkg_temp_thermal_init)
521 
522 static void __exit pkg_temp_thermal_exit(void)
523 {
524 	platform_thermal_package_notify = NULL;
525 	platform_thermal_package_rate_control = NULL;
526 
527 	cpuhp_remove_state(pkg_thermal_hp_state);
528 	debugfs_remove_recursive(debugfs);
529 	kfree(zones);
530 }
531 module_exit(pkg_temp_thermal_exit)
532 
533 MODULE_IMPORT_NS(INTEL_TCC);
534 MODULE_DESCRIPTION("X86 PKG TEMP Thermal Driver");
535 MODULE_AUTHOR("Srinivas Pandruvada <srinivas.pandruvada@linux.intel.com>");
536 MODULE_LICENSE("GPL v2");
537