1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Hardware Feedback Interface Driver
4  *
5  * Copyright (c) 2021, Intel Corporation.
6  *
7  * Authors: Aubrey Li <aubrey.li@linux.intel.com>
8  *          Ricardo Neri <ricardo.neri-calderon@linux.intel.com>
9  *
10  *
11  * The Hardware Feedback Interface provides a performance and energy efficiency
12  * capability information for each CPU in the system. Depending on the processor
13  * model, hardware may periodically update these capabilities as a result of
14  * changes in the operating conditions (e.g., power limits or thermal
15  * constraints). On other processor models, there is a single HFI update
16  * at boot.
17  *
18  * This file provides functionality to process HFI updates and relay these
19  * updates to userspace.
20  */
21 
22 #define pr_fmt(fmt)  "intel-hfi: " fmt
23 
24 #include <linux/bitops.h>
25 #include <linux/cpufeature.h>
26 #include <linux/cpumask.h>
27 #include <linux/gfp.h>
28 #include <linux/io.h>
29 #include <linux/kernel.h>
30 #include <linux/math.h>
31 #include <linux/mutex.h>
32 #include <linux/percpu-defs.h>
33 #include <linux/printk.h>
34 #include <linux/processor.h>
35 #include <linux/slab.h>
36 #include <linux/spinlock.h>
37 #include <linux/string.h>
38 #include <linux/topology.h>
39 #include <linux/workqueue.h>
40 
41 #include <asm/msr.h>
42 
43 #include "../thermal_core.h"
44 #include "intel_hfi.h"
45 
46 #define THERM_STATUS_CLEAR_PKG_MASK (BIT(1) | BIT(3) | BIT(5) | BIT(7) | \
47 				     BIT(9) | BIT(11) | BIT(26))
48 
49 /* Hardware Feedback Interface MSR configuration bits */
50 #define HW_FEEDBACK_PTR_VALID_BIT		BIT(0)
51 #define HW_FEEDBACK_CONFIG_HFI_ENABLE_BIT	BIT(0)
52 
53 /* CPUID detection and enumeration definitions for HFI */
54 
55 #define CPUID_HFI_LEAF 6
56 
57 union hfi_capabilities {
58 	struct {
59 		u8	performance:1;
60 		u8	energy_efficiency:1;
61 		u8	__reserved:6;
62 	} split;
63 	u8 bits;
64 };
65 
66 union cpuid6_edx {
67 	struct {
68 		union hfi_capabilities	capabilities;
69 		u32			table_pages:4;
70 		u32			__reserved:4;
71 		s32			index:16;
72 	} split;
73 	u32 full;
74 };
75 
76 /**
77  * struct hfi_cpu_data - HFI capabilities per CPU
78  * @perf_cap:		Performance capability
79  * @ee_cap:		Energy efficiency capability
80  *
81  * Capabilities of a logical processor in the HFI table. These capabilities are
82  * unitless.
83  */
84 struct hfi_cpu_data {
85 	u8	perf_cap;
86 	u8	ee_cap;
87 } __packed;
88 
89 /**
90  * struct hfi_hdr - Header of the HFI table
91  * @perf_updated:	Hardware updated performance capabilities
92  * @ee_updated:		Hardware updated energy efficiency capabilities
93  *
94  * Properties of the data in an HFI table.
95  */
96 struct hfi_hdr {
97 	u8	perf_updated;
98 	u8	ee_updated;
99 } __packed;
100 
101 /**
102  * struct hfi_instance - Representation of an HFI instance (i.e., a table)
103  * @local_table:	Base of the local copy of the HFI table
104  * @timestamp:		Timestamp of the last update of the local table.
105  *			Located at the base of the local table.
106  * @hdr:		Base address of the header of the local table
107  * @data:		Base address of the data of the local table
108  * @cpus:		CPUs represented in this HFI table instance
109  * @hw_table:		Pointer to the HFI table of this instance
110  * @update_work:	Delayed work to process HFI updates
111  * @table_lock:		Lock to protect acceses to the table of this instance
112  * @event_lock:		Lock to process HFI interrupts
113  *
114  * A set of parameters to parse and navigate a specific HFI table.
115  */
116 struct hfi_instance {
117 	union {
118 		void			*local_table;
119 		u64			*timestamp;
120 	};
121 	void			*hdr;
122 	void			*data;
123 	cpumask_var_t		cpus;
124 	void			*hw_table;
125 	struct delayed_work	update_work;
126 	raw_spinlock_t		table_lock;
127 	raw_spinlock_t		event_lock;
128 };
129 
130 /**
131  * struct hfi_features - Supported HFI features
132  * @nr_table_pages:	Size of the HFI table in 4KB pages
133  * @cpu_stride:		Stride size to locate the capability data of a logical
134  *			processor within the table (i.e., row stride)
135  * @hdr_size:		Size of the table header
136  *
137  * Parameters and supported features that are common to all HFI instances
138  */
139 struct hfi_features {
140 	unsigned int	nr_table_pages;
141 	unsigned int	cpu_stride;
142 	unsigned int	hdr_size;
143 };
144 
145 /**
146  * struct hfi_cpu_info - Per-CPU attributes to consume HFI data
147  * @index:		Row of this CPU in its HFI table
148  * @hfi_instance:	Attributes of the HFI table to which this CPU belongs
149  *
150  * Parameters to link a logical processor to an HFI table and a row within it.
151  */
152 struct hfi_cpu_info {
153 	s16			index;
154 	struct hfi_instance	*hfi_instance;
155 };
156 
157 static DEFINE_PER_CPU(struct hfi_cpu_info, hfi_cpu_info) = { .index = -1 };
158 
159 static int max_hfi_instances;
160 static struct hfi_instance *hfi_instances;
161 
162 static struct hfi_features hfi_features;
163 static DEFINE_MUTEX(hfi_instance_lock);
164 
165 static struct workqueue_struct *hfi_updates_wq;
166 #define HFI_UPDATE_INTERVAL		HZ
167 #define HFI_MAX_THERM_NOTIFY_COUNT	16
168 
169 static void get_hfi_caps(struct hfi_instance *hfi_instance,
170 			 struct thermal_genl_cpu_caps *cpu_caps)
171 {
172 	int cpu, i = 0;
173 
174 	raw_spin_lock_irq(&hfi_instance->table_lock);
175 	for_each_cpu(cpu, hfi_instance->cpus) {
176 		struct hfi_cpu_data *caps;
177 		s16 index;
178 
179 		index = per_cpu(hfi_cpu_info, cpu).index;
180 		caps = hfi_instance->data + index * hfi_features.cpu_stride;
181 		cpu_caps[i].cpu = cpu;
182 
183 		/*
184 		 * Scale performance and energy efficiency to
185 		 * the [0, 1023] interval that thermal netlink uses.
186 		 */
187 		cpu_caps[i].performance = caps->perf_cap << 2;
188 		cpu_caps[i].efficiency = caps->ee_cap << 2;
189 
190 		++i;
191 	}
192 	raw_spin_unlock_irq(&hfi_instance->table_lock);
193 }
194 
195 /*
196  * Call update_capabilities() when there are changes in the HFI table.
197  */
198 static void update_capabilities(struct hfi_instance *hfi_instance)
199 {
200 	struct thermal_genl_cpu_caps *cpu_caps;
201 	int i = 0, cpu_count;
202 
203 	/* CPUs may come online/offline while processing an HFI update. */
204 	mutex_lock(&hfi_instance_lock);
205 
206 	cpu_count = cpumask_weight(hfi_instance->cpus);
207 
208 	/* No CPUs to report in this hfi_instance. */
209 	if (!cpu_count)
210 		goto out;
211 
212 	cpu_caps = kcalloc(cpu_count, sizeof(*cpu_caps), GFP_KERNEL);
213 	if (!cpu_caps)
214 		goto out;
215 
216 	get_hfi_caps(hfi_instance, cpu_caps);
217 
218 	if (cpu_count < HFI_MAX_THERM_NOTIFY_COUNT)
219 		goto last_cmd;
220 
221 	/* Process complete chunks of HFI_MAX_THERM_NOTIFY_COUNT capabilities. */
222 	for (i = 0;
223 	     (i + HFI_MAX_THERM_NOTIFY_COUNT) <= cpu_count;
224 	     i += HFI_MAX_THERM_NOTIFY_COUNT)
225 		thermal_genl_cpu_capability_event(HFI_MAX_THERM_NOTIFY_COUNT,
226 						  &cpu_caps[i]);
227 
228 	cpu_count = cpu_count - i;
229 
230 last_cmd:
231 	/* Process the remaining capabilities if any. */
232 	if (cpu_count)
233 		thermal_genl_cpu_capability_event(cpu_count, &cpu_caps[i]);
234 
235 	kfree(cpu_caps);
236 out:
237 	mutex_unlock(&hfi_instance_lock);
238 }
239 
240 static void hfi_update_work_fn(struct work_struct *work)
241 {
242 	struct hfi_instance *hfi_instance;
243 
244 	hfi_instance = container_of(to_delayed_work(work), struct hfi_instance,
245 				    update_work);
246 
247 	update_capabilities(hfi_instance);
248 }
249 
250 void intel_hfi_process_event(__u64 pkg_therm_status_msr_val)
251 {
252 	struct hfi_instance *hfi_instance;
253 	int cpu = smp_processor_id();
254 	struct hfi_cpu_info *info;
255 	u64 new_timestamp;
256 
257 	if (!pkg_therm_status_msr_val)
258 		return;
259 
260 	info = &per_cpu(hfi_cpu_info, cpu);
261 	if (!info)
262 		return;
263 
264 	/*
265 	 * A CPU is linked to its HFI instance before the thermal vector in the
266 	 * local APIC is unmasked. Hence, info->hfi_instance cannot be NULL
267 	 * when receiving an HFI event.
268 	 */
269 	hfi_instance = info->hfi_instance;
270 	if (unlikely(!hfi_instance)) {
271 		pr_debug("Received event on CPU %d but instance was null", cpu);
272 		return;
273 	}
274 
275 	/*
276 	 * On most systems, all CPUs in the package receive a package-level
277 	 * thermal interrupt when there is an HFI update. It is sufficient to
278 	 * let a single CPU to acknowledge the update and queue work to
279 	 * process it. The remaining CPUs can resume their work.
280 	 */
281 	if (!raw_spin_trylock(&hfi_instance->event_lock))
282 		return;
283 
284 	/* Skip duplicated updates. */
285 	new_timestamp = *(u64 *)hfi_instance->hw_table;
286 	if (*hfi_instance->timestamp == new_timestamp) {
287 		raw_spin_unlock(&hfi_instance->event_lock);
288 		return;
289 	}
290 
291 	raw_spin_lock(&hfi_instance->table_lock);
292 
293 	/*
294 	 * Copy the updated table into our local copy. This includes the new
295 	 * timestamp.
296 	 */
297 	memcpy(hfi_instance->local_table, hfi_instance->hw_table,
298 	       hfi_features.nr_table_pages << PAGE_SHIFT);
299 
300 	raw_spin_unlock(&hfi_instance->table_lock);
301 	raw_spin_unlock(&hfi_instance->event_lock);
302 
303 	/*
304 	 * Let hardware know that we are done reading the HFI table and it is
305 	 * free to update it again.
306 	 */
307 	pkg_therm_status_msr_val &= THERM_STATUS_CLEAR_PKG_MASK &
308 				    ~PACKAGE_THERM_STATUS_HFI_UPDATED;
309 	wrmsrl(MSR_IA32_PACKAGE_THERM_STATUS, pkg_therm_status_msr_val);
310 
311 	queue_delayed_work(hfi_updates_wq, &hfi_instance->update_work,
312 			   HFI_UPDATE_INTERVAL);
313 }
314 
315 static void init_hfi_cpu_index(struct hfi_cpu_info *info)
316 {
317 	union cpuid6_edx edx;
318 
319 	/* Do not re-read @cpu's index if it has already been initialized. */
320 	if (info->index > -1)
321 		return;
322 
323 	edx.full = cpuid_edx(CPUID_HFI_LEAF);
324 	info->index = edx.split.index;
325 }
326 
327 /*
328  * The format of the HFI table depends on the number of capabilities that the
329  * hardware supports. Keep a data structure to navigate the table.
330  */
331 static void init_hfi_instance(struct hfi_instance *hfi_instance)
332 {
333 	/* The HFI header is below the time-stamp. */
334 	hfi_instance->hdr = hfi_instance->local_table +
335 			    sizeof(*hfi_instance->timestamp);
336 
337 	/* The HFI data starts below the header. */
338 	hfi_instance->data = hfi_instance->hdr + hfi_features.hdr_size;
339 }
340 
341 /**
342  * intel_hfi_online() - Enable HFI on @cpu
343  * @cpu:	CPU in which the HFI will be enabled
344  *
345  * Enable the HFI to be used in @cpu. The HFI is enabled at the die/package
346  * level. The first CPU in the die/package to come online does the full HFI
347  * initialization. Subsequent CPUs will just link themselves to the HFI
348  * instance of their die/package.
349  *
350  * This function is called before enabling the thermal vector in the local APIC
351  * in order to ensure that @cpu has an associated HFI instance when it receives
352  * an HFI event.
353  */
354 void intel_hfi_online(unsigned int cpu)
355 {
356 	struct hfi_instance *hfi_instance;
357 	struct hfi_cpu_info *info;
358 	phys_addr_t hw_table_pa;
359 	u64 msr_val;
360 	u16 die_id;
361 
362 	/* Nothing to do if hfi_instances are missing. */
363 	if (!hfi_instances)
364 		return;
365 
366 	/*
367 	 * Link @cpu to the HFI instance of its package/die. It does not
368 	 * matter whether the instance has been initialized.
369 	 */
370 	info = &per_cpu(hfi_cpu_info, cpu);
371 	die_id = topology_logical_die_id(cpu);
372 	hfi_instance = info->hfi_instance;
373 	if (!hfi_instance) {
374 		if (die_id < 0 || die_id >= max_hfi_instances)
375 			return;
376 
377 		hfi_instance = &hfi_instances[die_id];
378 		info->hfi_instance = hfi_instance;
379 	}
380 
381 	init_hfi_cpu_index(info);
382 
383 	/*
384 	 * Now check if the HFI instance of the package/die of @cpu has been
385 	 * initialized (by checking its header). In such case, all we have to
386 	 * do is to add @cpu to this instance's cpumask.
387 	 */
388 	mutex_lock(&hfi_instance_lock);
389 	if (hfi_instance->hdr) {
390 		cpumask_set_cpu(cpu, hfi_instance->cpus);
391 		goto unlock;
392 	}
393 
394 	/*
395 	 * Hardware is programmed with the physical address of the first page
396 	 * frame of the table. Hence, the allocated memory must be page-aligned.
397 	 */
398 	hfi_instance->hw_table = alloc_pages_exact(hfi_features.nr_table_pages,
399 						   GFP_KERNEL | __GFP_ZERO);
400 	if (!hfi_instance->hw_table)
401 		goto unlock;
402 
403 	hw_table_pa = virt_to_phys(hfi_instance->hw_table);
404 
405 	/*
406 	 * Allocate memory to keep a local copy of the table that
407 	 * hardware generates.
408 	 */
409 	hfi_instance->local_table = kzalloc(hfi_features.nr_table_pages << PAGE_SHIFT,
410 					    GFP_KERNEL);
411 	if (!hfi_instance->local_table)
412 		goto free_hw_table;
413 
414 	/*
415 	 * Program the address of the feedback table of this die/package. On
416 	 * some processors, hardware remembers the old address of the HFI table
417 	 * even after having been reprogrammed and re-enabled. Thus, do not free
418 	 * the pages allocated for the table or reprogram the hardware with a
419 	 * new base address. Namely, program the hardware only once.
420 	 */
421 	msr_val = hw_table_pa | HW_FEEDBACK_PTR_VALID_BIT;
422 	wrmsrl(MSR_IA32_HW_FEEDBACK_PTR, msr_val);
423 
424 	init_hfi_instance(hfi_instance);
425 
426 	INIT_DELAYED_WORK(&hfi_instance->update_work, hfi_update_work_fn);
427 	raw_spin_lock_init(&hfi_instance->table_lock);
428 	raw_spin_lock_init(&hfi_instance->event_lock);
429 
430 	cpumask_set_cpu(cpu, hfi_instance->cpus);
431 
432 	/*
433 	 * Enable the hardware feedback interface and never disable it. See
434 	 * comment on programming the address of the table.
435 	 */
436 	rdmsrl(MSR_IA32_HW_FEEDBACK_CONFIG, msr_val);
437 	msr_val |= HW_FEEDBACK_CONFIG_HFI_ENABLE_BIT;
438 	wrmsrl(MSR_IA32_HW_FEEDBACK_CONFIG, msr_val);
439 
440 unlock:
441 	mutex_unlock(&hfi_instance_lock);
442 	return;
443 
444 free_hw_table:
445 	free_pages_exact(hfi_instance->hw_table, hfi_features.nr_table_pages);
446 	goto unlock;
447 }
448 
449 /**
450  * intel_hfi_offline() - Disable HFI on @cpu
451  * @cpu:	CPU in which the HFI will be disabled
452  *
453  * Remove @cpu from those covered by its HFI instance.
454  *
455  * On some processors, hardware remembers previous programming settings even
456  * after being reprogrammed. Thus, keep HFI enabled even if all CPUs in the
457  * die/package of @cpu are offline. See note in intel_hfi_online().
458  */
459 void intel_hfi_offline(unsigned int cpu)
460 {
461 	struct hfi_cpu_info *info = &per_cpu(hfi_cpu_info, cpu);
462 	struct hfi_instance *hfi_instance;
463 
464 	/*
465 	 * Check if @cpu as an associated, initialized (i.e., with a non-NULL
466 	 * header). Also, HFI instances are only initialized if X86_FEATURE_HFI
467 	 * is present.
468 	 */
469 	hfi_instance = info->hfi_instance;
470 	if (!hfi_instance)
471 		return;
472 
473 	if (!hfi_instance->hdr)
474 		return;
475 
476 	mutex_lock(&hfi_instance_lock);
477 	cpumask_clear_cpu(cpu, hfi_instance->cpus);
478 	mutex_unlock(&hfi_instance_lock);
479 }
480 
481 static __init int hfi_parse_features(void)
482 {
483 	unsigned int nr_capabilities;
484 	union cpuid6_edx edx;
485 
486 	if (!boot_cpu_has(X86_FEATURE_HFI))
487 		return -ENODEV;
488 
489 	/*
490 	 * If we are here we know that CPUID_HFI_LEAF exists. Parse the
491 	 * supported capabilities and the size of the HFI table.
492 	 */
493 	edx.full = cpuid_edx(CPUID_HFI_LEAF);
494 
495 	if (!edx.split.capabilities.split.performance) {
496 		pr_debug("Performance reporting not supported! Not using HFI\n");
497 		return -ENODEV;
498 	}
499 
500 	/*
501 	 * The number of supported capabilities determines the number of
502 	 * columns in the HFI table. Exclude the reserved bits.
503 	 */
504 	edx.split.capabilities.split.__reserved = 0;
505 	nr_capabilities = hweight8(edx.split.capabilities.bits);
506 
507 	/* The number of 4KB pages required by the table */
508 	hfi_features.nr_table_pages = edx.split.table_pages + 1;
509 
510 	/*
511 	 * The header contains change indications for each supported feature.
512 	 * The size of the table header is rounded up to be a multiple of 8
513 	 * bytes.
514 	 */
515 	hfi_features.hdr_size = DIV_ROUND_UP(nr_capabilities, 8) * 8;
516 
517 	/*
518 	 * Data of each logical processor is also rounded up to be a multiple
519 	 * of 8 bytes.
520 	 */
521 	hfi_features.cpu_stride = DIV_ROUND_UP(nr_capabilities, 8) * 8;
522 
523 	return 0;
524 }
525 
526 void __init intel_hfi_init(void)
527 {
528 	struct hfi_instance *hfi_instance;
529 	int i, j;
530 
531 	if (hfi_parse_features())
532 		return;
533 
534 	/* There is one HFI instance per die/package. */
535 	max_hfi_instances = topology_max_packages() *
536 			    topology_max_die_per_package();
537 
538 	/*
539 	 * This allocation may fail. CPU hotplug callbacks must check
540 	 * for a null pointer.
541 	 */
542 	hfi_instances = kcalloc(max_hfi_instances, sizeof(*hfi_instances),
543 				GFP_KERNEL);
544 	if (!hfi_instances)
545 		return;
546 
547 	for (i = 0; i < max_hfi_instances; i++) {
548 		hfi_instance = &hfi_instances[i];
549 		if (!zalloc_cpumask_var(&hfi_instance->cpus, GFP_KERNEL))
550 			goto err_nomem;
551 	}
552 
553 	hfi_updates_wq = create_singlethread_workqueue("hfi-updates");
554 	if (!hfi_updates_wq)
555 		goto err_nomem;
556 
557 	return;
558 
559 err_nomem:
560 	for (j = 0; j < i; ++j) {
561 		hfi_instance = &hfi_instances[j];
562 		free_cpumask_var(hfi_instance->cpus);
563 	}
564 
565 	kfree(hfi_instances);
566 	hfi_instances = NULL;
567 }
568