1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * x86_pkg_temp_thermal driver
4  * Copyright (c) 2013, Intel Corporation.
5  */
6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
7 
8 #include <linux/module.h>
9 #include <linux/init.h>
10 #include <linux/intel_tcc.h>
11 #include <linux/err.h>
12 #include <linux/param.h>
13 #include <linux/device.h>
14 #include <linux/platform_device.h>
15 #include <linux/cpu.h>
16 #include <linux/smp.h>
17 #include <linux/slab.h>
18 #include <linux/pm.h>
19 #include <linux/thermal.h>
20 #include <linux/debugfs.h>
21 
22 #include <asm/cpu_device_id.h>
23 
24 #include "thermal_interrupt.h"
25 
26 /*
27 * Rate control delay: Idea is to introduce denounce effect
28 * This should be long enough to avoid reduce events, when
29 * threshold is set to a temperature, which is constantly
30 * violated, but at the short enough to take any action.
31 * The action can be remove threshold or change it to next
32 * interesting setting. Based on experiments, in around
33 * every 5 seconds under load will give us a significant
34 * temperature change.
35 */
36 #define PKG_TEMP_THERMAL_NOTIFY_DELAY	5000
37 static int notify_delay_ms = PKG_TEMP_THERMAL_NOTIFY_DELAY;
38 module_param(notify_delay_ms, int, 0644);
39 MODULE_PARM_DESC(notify_delay_ms,
40 	"User space notification delay in milli seconds.");
41 
42 /* Number of trip points in thermal zone. Currently it can't
43 * be more than 2. MSR can allow setting and getting notifications
44 * for only 2 thresholds. This define enforces this, if there
45 * is some wrong values returned by cpuid for number of thresholds.
46 */
47 #define MAX_NUMBER_OF_TRIPS	2
48 
49 struct zone_device {
50 	int				cpu;
51 	bool				work_scheduled;
52 	u32				msr_pkg_therm_low;
53 	u32				msr_pkg_therm_high;
54 	struct delayed_work		work;
55 	struct thermal_zone_device	*tzone;
56 	struct thermal_trip		*trips;
57 	struct cpumask			cpumask;
58 };
59 
60 static struct thermal_zone_params pkg_temp_tz_params = {
61 	.no_hwmon	= true,
62 };
63 
64 /* Keep track of how many zone pointers we allocated in init() */
65 static int max_id __read_mostly;
66 /* Array of zone pointers */
67 static struct zone_device **zones;
68 /* Serializes interrupt notification, work and hotplug */
69 static DEFINE_RAW_SPINLOCK(pkg_temp_lock);
70 /* Protects zone operation in the work function against hotplug removal */
71 static DEFINE_MUTEX(thermal_zone_mutex);
72 
73 /* The dynamically assigned cpu hotplug state for module_exit() */
74 static enum cpuhp_state pkg_thermal_hp_state __read_mostly;
75 
76 /* Debug counters to show using debugfs */
77 static struct dentry *debugfs;
78 static unsigned int pkg_interrupt_cnt;
79 static unsigned int pkg_work_cnt;
80 
81 static void pkg_temp_debugfs_init(void)
82 {
83 	debugfs = debugfs_create_dir("pkg_temp_thermal", NULL);
84 
85 	debugfs_create_u32("pkg_thres_interrupt", S_IRUGO, debugfs,
86 			   &pkg_interrupt_cnt);
87 	debugfs_create_u32("pkg_thres_work", S_IRUGO, debugfs,
88 			   &pkg_work_cnt);
89 }
90 
91 /*
92  * Protection:
93  *
94  * - cpu hotplug: Read serialized by cpu hotplug lock
95  *		  Write must hold pkg_temp_lock
96  *
97  * - Other callsites: Must hold pkg_temp_lock
98  */
99 static struct zone_device *pkg_temp_thermal_get_dev(unsigned int cpu)
100 {
101 	int id = topology_logical_die_id(cpu);
102 
103 	if (id >= 0 && id < max_id)
104 		return zones[id];
105 	return NULL;
106 }
107 
108 static int sys_get_curr_temp(struct thermal_zone_device *tzd, int *temp)
109 {
110 	struct zone_device *zonedev = thermal_zone_device_priv(tzd);
111 	int val;
112 
113 	val = intel_tcc_get_temp(zonedev->cpu, true);
114 	if (val < 0)
115 		return val;
116 
117 	*temp = val * 1000;
118 	pr_debug("sys_get_curr_temp %d\n", *temp);
119 	return 0;
120 }
121 
122 static int
123 sys_set_trip_temp(struct thermal_zone_device *tzd, int trip, int temp)
124 {
125 	struct zone_device *zonedev = thermal_zone_device_priv(tzd);
126 	u32 l, h, mask, shift, intr;
127 	int tj_max, val, ret;
128 
129 	tj_max = intel_tcc_get_tjmax(zonedev->cpu);
130 	if (tj_max < 0)
131 		return tj_max;
132 	tj_max *= 1000;
133 
134 	val = (tj_max - temp)/1000;
135 
136 	if (trip >= MAX_NUMBER_OF_TRIPS || val < 0 || val > 0x7f)
137 		return -EINVAL;
138 
139 	ret = rdmsr_on_cpu(zonedev->cpu, MSR_IA32_PACKAGE_THERM_INTERRUPT,
140 			   &l, &h);
141 	if (ret < 0)
142 		return ret;
143 
144 	if (trip) {
145 		mask = THERM_MASK_THRESHOLD1;
146 		shift = THERM_SHIFT_THRESHOLD1;
147 		intr = THERM_INT_THRESHOLD1_ENABLE;
148 	} else {
149 		mask = THERM_MASK_THRESHOLD0;
150 		shift = THERM_SHIFT_THRESHOLD0;
151 		intr = THERM_INT_THRESHOLD0_ENABLE;
152 	}
153 	l &= ~mask;
154 	/*
155 	* When users space sets a trip temperature == 0, which is indication
156 	* that, it is no longer interested in receiving notifications.
157 	*/
158 	if (!temp) {
159 		l &= ~intr;
160 	} else {
161 		l |= val << shift;
162 		l |= intr;
163 	}
164 
165 	return wrmsr_on_cpu(zonedev->cpu, MSR_IA32_PACKAGE_THERM_INTERRUPT,
166 			l, h);
167 }
168 
169 /* Thermal zone callback registry */
170 static struct thermal_zone_device_ops tzone_ops = {
171 	.get_temp = sys_get_curr_temp,
172 	.set_trip_temp = sys_set_trip_temp,
173 };
174 
175 static bool pkg_thermal_rate_control(void)
176 {
177 	return true;
178 }
179 
180 /* Enable threshold interrupt on local package/cpu */
181 static inline void enable_pkg_thres_interrupt(void)
182 {
183 	u8 thres_0, thres_1;
184 	u32 l, h;
185 
186 	rdmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);
187 	/* only enable/disable if it had valid threshold value */
188 	thres_0 = (l & THERM_MASK_THRESHOLD0) >> THERM_SHIFT_THRESHOLD0;
189 	thres_1 = (l & THERM_MASK_THRESHOLD1) >> THERM_SHIFT_THRESHOLD1;
190 	if (thres_0)
191 		l |= THERM_INT_THRESHOLD0_ENABLE;
192 	if (thres_1)
193 		l |= THERM_INT_THRESHOLD1_ENABLE;
194 	wrmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);
195 }
196 
197 /* Disable threshold interrupt on local package/cpu */
198 static inline void disable_pkg_thres_interrupt(void)
199 {
200 	u32 l, h;
201 
202 	rdmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);
203 
204 	l &= ~(THERM_INT_THRESHOLD0_ENABLE | THERM_INT_THRESHOLD1_ENABLE);
205 	wrmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);
206 }
207 
208 static void pkg_temp_thermal_threshold_work_fn(struct work_struct *work)
209 {
210 	struct thermal_zone_device *tzone = NULL;
211 	int cpu = smp_processor_id();
212 	struct zone_device *zonedev;
213 
214 	mutex_lock(&thermal_zone_mutex);
215 	raw_spin_lock_irq(&pkg_temp_lock);
216 	++pkg_work_cnt;
217 
218 	zonedev = pkg_temp_thermal_get_dev(cpu);
219 	if (!zonedev) {
220 		raw_spin_unlock_irq(&pkg_temp_lock);
221 		mutex_unlock(&thermal_zone_mutex);
222 		return;
223 	}
224 	zonedev->work_scheduled = false;
225 
226 	thermal_clear_package_intr_status(PACKAGE_LEVEL, THERM_LOG_THRESHOLD0 | THERM_LOG_THRESHOLD1);
227 	tzone = zonedev->tzone;
228 
229 	enable_pkg_thres_interrupt();
230 	raw_spin_unlock_irq(&pkg_temp_lock);
231 
232 	/*
233 	 * If tzone is not NULL, then thermal_zone_mutex will prevent the
234 	 * concurrent removal in the cpu offline callback.
235 	 */
236 	if (tzone)
237 		thermal_zone_device_update(tzone, THERMAL_EVENT_UNSPECIFIED);
238 
239 	mutex_unlock(&thermal_zone_mutex);
240 }
241 
242 static void pkg_thermal_schedule_work(int cpu, struct delayed_work *work)
243 {
244 	unsigned long ms = msecs_to_jiffies(notify_delay_ms);
245 
246 	schedule_delayed_work_on(cpu, work, ms);
247 }
248 
249 static int pkg_thermal_notify(u64 msr_val)
250 {
251 	int cpu = smp_processor_id();
252 	struct zone_device *zonedev;
253 	unsigned long flags;
254 
255 	raw_spin_lock_irqsave(&pkg_temp_lock, flags);
256 	++pkg_interrupt_cnt;
257 
258 	disable_pkg_thres_interrupt();
259 
260 	/* Work is per package, so scheduling it once is enough. */
261 	zonedev = pkg_temp_thermal_get_dev(cpu);
262 	if (zonedev && !zonedev->work_scheduled) {
263 		zonedev->work_scheduled = true;
264 		pkg_thermal_schedule_work(zonedev->cpu, &zonedev->work);
265 	}
266 
267 	raw_spin_unlock_irqrestore(&pkg_temp_lock, flags);
268 	return 0;
269 }
270 
271 static struct thermal_trip *pkg_temp_thermal_trips_init(int cpu, int tj_max, int num_trips)
272 {
273 	struct thermal_trip *trips;
274 	unsigned long thres_reg_value;
275 	u32 mask, shift, eax, edx;
276 	int ret, i;
277 
278 	trips = kzalloc(sizeof(*trips) * num_trips, GFP_KERNEL);
279 	if (!trips)
280 		return ERR_PTR(-ENOMEM);
281 
282 	for (i = 0; i < num_trips; i++) {
283 
284 		if (i) {
285 			mask = THERM_MASK_THRESHOLD1;
286 			shift = THERM_SHIFT_THRESHOLD1;
287 		} else {
288 			mask = THERM_MASK_THRESHOLD0;
289 			shift = THERM_SHIFT_THRESHOLD0;
290 		}
291 
292 		ret = rdmsr_on_cpu(cpu, MSR_IA32_PACKAGE_THERM_INTERRUPT,
293 				   &eax, &edx);
294 		if (ret < 0) {
295 			kfree(trips);
296 			return ERR_PTR(ret);
297 		}
298 
299 		thres_reg_value = (eax & mask) >> shift;
300 
301 		trips[i].temperature = thres_reg_value ?
302 			tj_max - thres_reg_value * 1000 : THERMAL_TEMP_INVALID;
303 
304 		trips[i].type = THERMAL_TRIP_PASSIVE;
305 
306 		pr_debug("%s: cpu=%d, trip=%d, temp=%d\n",
307 			 __func__, cpu, i, trips[i].temperature);
308 	}
309 
310 	return trips;
311 }
312 
313 static int pkg_temp_thermal_device_add(unsigned int cpu)
314 {
315 	int id = topology_logical_die_id(cpu);
316 	u32 eax, ebx, ecx, edx;
317 	struct zone_device *zonedev;
318 	int thres_count, err;
319 	int tj_max;
320 
321 	if (id >= max_id)
322 		return -ENOMEM;
323 
324 	cpuid(6, &eax, &ebx, &ecx, &edx);
325 	thres_count = ebx & 0x07;
326 	if (!thres_count)
327 		return -ENODEV;
328 
329 	thres_count = clamp_val(thres_count, 0, MAX_NUMBER_OF_TRIPS);
330 
331 	tj_max = intel_tcc_get_tjmax(cpu);
332 	if (tj_max < 0)
333 		return tj_max;
334 
335 	zonedev = kzalloc(sizeof(*zonedev), GFP_KERNEL);
336 	if (!zonedev)
337 		return -ENOMEM;
338 
339 	zonedev->trips = pkg_temp_thermal_trips_init(cpu, tj_max, thres_count);
340 	if (IS_ERR(zonedev->trips)) {
341 		err = PTR_ERR(zonedev->trips);
342 		goto out_kfree_zonedev;
343 	}
344 
345 	INIT_DELAYED_WORK(&zonedev->work, pkg_temp_thermal_threshold_work_fn);
346 	zonedev->cpu = cpu;
347 	zonedev->tzone = thermal_zone_device_register_with_trips("x86_pkg_temp",
348 			zonedev->trips, thres_count,
349 			(thres_count == MAX_NUMBER_OF_TRIPS) ? 0x03 : 0x01,
350 			zonedev, &tzone_ops, &pkg_temp_tz_params, 0, 0);
351 	if (IS_ERR(zonedev->tzone)) {
352 		err = PTR_ERR(zonedev->tzone);
353 		goto out_kfree_trips;
354 	}
355 	err = thermal_zone_device_enable(zonedev->tzone);
356 	if (err)
357 		goto out_unregister_tz;
358 
359 	/* Store MSR value for package thermal interrupt, to restore at exit */
360 	rdmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, zonedev->msr_pkg_therm_low,
361 	      zonedev->msr_pkg_therm_high);
362 
363 	cpumask_set_cpu(cpu, &zonedev->cpumask);
364 	raw_spin_lock_irq(&pkg_temp_lock);
365 	zones[id] = zonedev;
366 	raw_spin_unlock_irq(&pkg_temp_lock);
367 
368 	return 0;
369 
370 out_unregister_tz:
371 	thermal_zone_device_unregister(zonedev->tzone);
372 out_kfree_trips:
373 	kfree(zonedev->trips);
374 out_kfree_zonedev:
375 	kfree(zonedev);
376 	return err;
377 }
378 
379 static int pkg_thermal_cpu_offline(unsigned int cpu)
380 {
381 	struct zone_device *zonedev = pkg_temp_thermal_get_dev(cpu);
382 	bool lastcpu, was_target;
383 	int target;
384 
385 	if (!zonedev)
386 		return 0;
387 
388 	target = cpumask_any_but(&zonedev->cpumask, cpu);
389 	cpumask_clear_cpu(cpu, &zonedev->cpumask);
390 	lastcpu = target >= nr_cpu_ids;
391 	/*
392 	 * Remove the sysfs files, if this is the last cpu in the package
393 	 * before doing further cleanups.
394 	 */
395 	if (lastcpu) {
396 		struct thermal_zone_device *tzone = zonedev->tzone;
397 
398 		/*
399 		 * We must protect against a work function calling
400 		 * thermal_zone_update, after/while unregister. We null out
401 		 * the pointer under the zone mutex, so the worker function
402 		 * won't try to call.
403 		 */
404 		mutex_lock(&thermal_zone_mutex);
405 		zonedev->tzone = NULL;
406 		mutex_unlock(&thermal_zone_mutex);
407 
408 		thermal_zone_device_unregister(tzone);
409 	}
410 
411 	/* Protect against work and interrupts */
412 	raw_spin_lock_irq(&pkg_temp_lock);
413 
414 	/*
415 	 * Check whether this cpu was the current target and store the new
416 	 * one. When we drop the lock, then the interrupt notify function
417 	 * will see the new target.
418 	 */
419 	was_target = zonedev->cpu == cpu;
420 	zonedev->cpu = target;
421 
422 	/*
423 	 * If this is the last CPU in the package remove the package
424 	 * reference from the array and restore the interrupt MSR. When we
425 	 * drop the lock neither the interrupt notify function nor the
426 	 * worker will see the package anymore.
427 	 */
428 	if (lastcpu) {
429 		zones[topology_logical_die_id(cpu)] = NULL;
430 		/* After this point nothing touches the MSR anymore. */
431 		wrmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT,
432 		      zonedev->msr_pkg_therm_low, zonedev->msr_pkg_therm_high);
433 	}
434 
435 	/*
436 	 * Check whether there is work scheduled and whether the work is
437 	 * targeted at the outgoing CPU.
438 	 */
439 	if (zonedev->work_scheduled && was_target) {
440 		/*
441 		 * To cancel the work we need to drop the lock, otherwise
442 		 * we might deadlock if the work needs to be flushed.
443 		 */
444 		raw_spin_unlock_irq(&pkg_temp_lock);
445 		cancel_delayed_work_sync(&zonedev->work);
446 		raw_spin_lock_irq(&pkg_temp_lock);
447 		/*
448 		 * If this is not the last cpu in the package and the work
449 		 * did not run after we dropped the lock above, then we
450 		 * need to reschedule the work, otherwise the interrupt
451 		 * stays disabled forever.
452 		 */
453 		if (!lastcpu && zonedev->work_scheduled)
454 			pkg_thermal_schedule_work(target, &zonedev->work);
455 	}
456 
457 	raw_spin_unlock_irq(&pkg_temp_lock);
458 
459 	/* Final cleanup if this is the last cpu */
460 	if (lastcpu) {
461 		kfree(zonedev->trips);
462 		kfree(zonedev);
463 	}
464 	return 0;
465 }
466 
467 static int pkg_thermal_cpu_online(unsigned int cpu)
468 {
469 	struct zone_device *zonedev = pkg_temp_thermal_get_dev(cpu);
470 	struct cpuinfo_x86 *c = &cpu_data(cpu);
471 
472 	/* Paranoia check */
473 	if (!cpu_has(c, X86_FEATURE_DTHERM) || !cpu_has(c, X86_FEATURE_PTS))
474 		return -ENODEV;
475 
476 	/* If the package exists, nothing to do */
477 	if (zonedev) {
478 		cpumask_set_cpu(cpu, &zonedev->cpumask);
479 		return 0;
480 	}
481 	return pkg_temp_thermal_device_add(cpu);
482 }
483 
484 static const struct x86_cpu_id __initconst pkg_temp_thermal_ids[] = {
485 	X86_MATCH_VENDOR_FEATURE(INTEL, X86_FEATURE_PTS, NULL),
486 	{}
487 };
488 MODULE_DEVICE_TABLE(x86cpu, pkg_temp_thermal_ids);
489 
490 static int __init pkg_temp_thermal_init(void)
491 {
492 	int ret;
493 
494 	if (!x86_match_cpu(pkg_temp_thermal_ids))
495 		return -ENODEV;
496 
497 	max_id = topology_max_packages() * topology_max_die_per_package();
498 	zones = kcalloc(max_id, sizeof(struct zone_device *),
499 			   GFP_KERNEL);
500 	if (!zones)
501 		return -ENOMEM;
502 
503 	ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "thermal/x86_pkg:online",
504 				pkg_thermal_cpu_online,	pkg_thermal_cpu_offline);
505 	if (ret < 0)
506 		goto err;
507 
508 	/* Store the state for module exit */
509 	pkg_thermal_hp_state = ret;
510 
511 	platform_thermal_package_notify = pkg_thermal_notify;
512 	platform_thermal_package_rate_control = pkg_thermal_rate_control;
513 
514 	 /* Don't care if it fails */
515 	pkg_temp_debugfs_init();
516 	return 0;
517 
518 err:
519 	kfree(zones);
520 	return ret;
521 }
522 module_init(pkg_temp_thermal_init)
523 
524 static void __exit pkg_temp_thermal_exit(void)
525 {
526 	platform_thermal_package_notify = NULL;
527 	platform_thermal_package_rate_control = NULL;
528 
529 	cpuhp_remove_state(pkg_thermal_hp_state);
530 	debugfs_remove_recursive(debugfs);
531 	kfree(zones);
532 }
533 module_exit(pkg_temp_thermal_exit)
534 
535 MODULE_IMPORT_NS(INTEL_TCC);
536 MODULE_DESCRIPTION("X86 PKG TEMP Thermal Driver");
537 MODULE_AUTHOR("Srinivas Pandruvada <srinivas.pandruvada@linux.intel.com>");
538 MODULE_LICENSE("GPL v2");
539