11cb19cabSRicardo Neri // SPDX-License-Identifier: GPL-2.0-only
21cb19cabSRicardo Neri /*
31cb19cabSRicardo Neri * Hardware Feedback Interface Driver
41cb19cabSRicardo Neri *
51cb19cabSRicardo Neri * Copyright (c) 2021, Intel Corporation.
61cb19cabSRicardo Neri *
71cb19cabSRicardo Neri * Authors: Aubrey Li <aubrey.li@linux.intel.com>
81cb19cabSRicardo Neri * Ricardo Neri <ricardo.neri-calderon@linux.intel.com>
91cb19cabSRicardo Neri *
101cb19cabSRicardo Neri *
111cb19cabSRicardo Neri * The Hardware Feedback Interface provides a performance and energy efficiency
121cb19cabSRicardo Neri * capability information for each CPU in the system. Depending on the processor
131cb19cabSRicardo Neri * model, hardware may periodically update these capabilities as a result of
141cb19cabSRicardo Neri * changes in the operating conditions (e.g., power limits or thermal
151cb19cabSRicardo Neri * constraints). On other processor models, there is a single HFI update
161cb19cabSRicardo Neri * at boot.
171cb19cabSRicardo Neri *
181cb19cabSRicardo Neri * This file provides functionality to process HFI updates and relay these
191cb19cabSRicardo Neri * updates to userspace.
201cb19cabSRicardo Neri */
211cb19cabSRicardo Neri
221cb19cabSRicardo Neri #define pr_fmt(fmt) "intel-hfi: " fmt
231cb19cabSRicardo Neri
241cb19cabSRicardo Neri #include <linux/bitops.h>
251cb19cabSRicardo Neri #include <linux/cpufeature.h>
262d74e631SRicardo Neri #include <linux/cpumask.h>
270caf5dd0SRicardo Neri #include <linux/delay.h>
282d74e631SRicardo Neri #include <linux/gfp.h>
292d74e631SRicardo Neri #include <linux/io.h>
30ab09b074SRicardo Neri #include <linux/kernel.h>
311cb19cabSRicardo Neri #include <linux/math.h>
322d74e631SRicardo Neri #include <linux/mutex.h>
332d74e631SRicardo Neri #include <linux/percpu-defs.h>
341cb19cabSRicardo Neri #include <linux/printk.h>
351cb19cabSRicardo Neri #include <linux/processor.h>
361cb19cabSRicardo Neri #include <linux/slab.h>
37ab09b074SRicardo Neri #include <linux/spinlock.h>
38*019ccc66SRicardo Neri #include <linux/suspend.h>
39ab09b074SRicardo Neri #include <linux/string.h>
40*019ccc66SRicardo Neri #include <linux/syscore_ops.h>
411cb19cabSRicardo Neri #include <linux/topology.h>
42ab09b074SRicardo Neri #include <linux/workqueue.h>
431cb19cabSRicardo Neri
442d74e631SRicardo Neri #include <asm/msr.h>
452d74e631SRicardo Neri
461cb19cabSRicardo Neri #include "intel_hfi.h"
47930d06bfSSrinivas Pandruvada #include "thermal_interrupt.h"
48ab09b074SRicardo Neri
499272d2d4SDaniel Lezcano #include "../thermal_netlink.h"
509272d2d4SDaniel Lezcano
512d74e631SRicardo Neri /* Hardware Feedback Interface MSR configuration bits */
522d74e631SRicardo Neri #define HW_FEEDBACK_PTR_VALID_BIT BIT(0)
53ab09b074SRicardo Neri #define HW_FEEDBACK_CONFIG_HFI_ENABLE_BIT BIT(0)
542d74e631SRicardo Neri
551cb19cabSRicardo Neri /* CPUID detection and enumeration definitions for HFI */
561cb19cabSRicardo Neri
571cb19cabSRicardo Neri #define CPUID_HFI_LEAF 6
581cb19cabSRicardo Neri
591cb19cabSRicardo Neri union hfi_capabilities {
601cb19cabSRicardo Neri struct {
611cb19cabSRicardo Neri u8 performance:1;
621cb19cabSRicardo Neri u8 energy_efficiency:1;
631cb19cabSRicardo Neri u8 __reserved:6;
641cb19cabSRicardo Neri } split;
651cb19cabSRicardo Neri u8 bits;
661cb19cabSRicardo Neri };
671cb19cabSRicardo Neri
681cb19cabSRicardo Neri union cpuid6_edx {
691cb19cabSRicardo Neri struct {
701cb19cabSRicardo Neri union hfi_capabilities capabilities;
711cb19cabSRicardo Neri u32 table_pages:4;
721cb19cabSRicardo Neri u32 __reserved:4;
731cb19cabSRicardo Neri s32 index:16;
741cb19cabSRicardo Neri } split;
751cb19cabSRicardo Neri u32 full;
761cb19cabSRicardo Neri };
771cb19cabSRicardo Neri
781cb19cabSRicardo Neri /**
791cb19cabSRicardo Neri * struct hfi_cpu_data - HFI capabilities per CPU
801cb19cabSRicardo Neri * @perf_cap: Performance capability
811cb19cabSRicardo Neri * @ee_cap: Energy efficiency capability
821cb19cabSRicardo Neri *
831cb19cabSRicardo Neri * Capabilities of a logical processor in the HFI table. These capabilities are
841cb19cabSRicardo Neri * unitless.
851cb19cabSRicardo Neri */
861cb19cabSRicardo Neri struct hfi_cpu_data {
871cb19cabSRicardo Neri u8 perf_cap;
881cb19cabSRicardo Neri u8 ee_cap;
891cb19cabSRicardo Neri } __packed;
901cb19cabSRicardo Neri
911cb19cabSRicardo Neri /**
921cb19cabSRicardo Neri * struct hfi_hdr - Header of the HFI table
931cb19cabSRicardo Neri * @perf_updated: Hardware updated performance capabilities
941cb19cabSRicardo Neri * @ee_updated: Hardware updated energy efficiency capabilities
951cb19cabSRicardo Neri *
961cb19cabSRicardo Neri * Properties of the data in an HFI table.
971cb19cabSRicardo Neri */
981cb19cabSRicardo Neri struct hfi_hdr {
991cb19cabSRicardo Neri u8 perf_updated;
1001cb19cabSRicardo Neri u8 ee_updated;
1011cb19cabSRicardo Neri } __packed;
1021cb19cabSRicardo Neri
1031cb19cabSRicardo Neri /**
1041cb19cabSRicardo Neri * struct hfi_instance - Representation of an HFI instance (i.e., a table)
1051cb19cabSRicardo Neri * @local_table: Base of the local copy of the HFI table
1061cb19cabSRicardo Neri * @timestamp: Timestamp of the last update of the local table.
1071cb19cabSRicardo Neri * Located at the base of the local table.
1081cb19cabSRicardo Neri * @hdr: Base address of the header of the local table
1091cb19cabSRicardo Neri * @data: Base address of the data of the local table
1102d74e631SRicardo Neri * @cpus: CPUs represented in this HFI table instance
1112d74e631SRicardo Neri * @hw_table: Pointer to the HFI table of this instance
112ab09b074SRicardo Neri * @update_work: Delayed work to process HFI updates
113ab09b074SRicardo Neri * @table_lock: Lock to protect acceses to the table of this instance
114ab09b074SRicardo Neri * @event_lock: Lock to process HFI interrupts
1151cb19cabSRicardo Neri *
1161cb19cabSRicardo Neri * A set of parameters to parse and navigate a specific HFI table.
1171cb19cabSRicardo Neri */
1181cb19cabSRicardo Neri struct hfi_instance {
1191cb19cabSRicardo Neri union {
1201cb19cabSRicardo Neri void *local_table;
1211cb19cabSRicardo Neri u64 *timestamp;
1221cb19cabSRicardo Neri };
1231cb19cabSRicardo Neri void *hdr;
1241cb19cabSRicardo Neri void *data;
1252d74e631SRicardo Neri cpumask_var_t cpus;
1262d74e631SRicardo Neri void *hw_table;
127ab09b074SRicardo Neri struct delayed_work update_work;
128ab09b074SRicardo Neri raw_spinlock_t table_lock;
129ab09b074SRicardo Neri raw_spinlock_t event_lock;
1301cb19cabSRicardo Neri };
1311cb19cabSRicardo Neri
1321cb19cabSRicardo Neri /**
1331cb19cabSRicardo Neri * struct hfi_features - Supported HFI features
1341cb19cabSRicardo Neri * @nr_table_pages: Size of the HFI table in 4KB pages
1351cb19cabSRicardo Neri * @cpu_stride: Stride size to locate the capability data of a logical
1361cb19cabSRicardo Neri * processor within the table (i.e., row stride)
1371cb19cabSRicardo Neri * @hdr_size: Size of the table header
1381cb19cabSRicardo Neri *
1391cb19cabSRicardo Neri * Parameters and supported features that are common to all HFI instances
1401cb19cabSRicardo Neri */
1411cb19cabSRicardo Neri struct hfi_features {
14254d9135cSRicardo Neri size_t nr_table_pages;
1431cb19cabSRicardo Neri unsigned int cpu_stride;
1441cb19cabSRicardo Neri unsigned int hdr_size;
1451cb19cabSRicardo Neri };
1461cb19cabSRicardo Neri
1472d74e631SRicardo Neri /**
1482d74e631SRicardo Neri * struct hfi_cpu_info - Per-CPU attributes to consume HFI data
1492d74e631SRicardo Neri * @index: Row of this CPU in its HFI table
1502d74e631SRicardo Neri * @hfi_instance: Attributes of the HFI table to which this CPU belongs
1512d74e631SRicardo Neri *
1522d74e631SRicardo Neri * Parameters to link a logical processor to an HFI table and a row within it.
1532d74e631SRicardo Neri */
1542d74e631SRicardo Neri struct hfi_cpu_info {
1552d74e631SRicardo Neri s16 index;
1562d74e631SRicardo Neri struct hfi_instance *hfi_instance;
1572d74e631SRicardo Neri };
1582d74e631SRicardo Neri
1592d74e631SRicardo Neri static DEFINE_PER_CPU(struct hfi_cpu_info, hfi_cpu_info) = { .index = -1 };
1602d74e631SRicardo Neri
1611cb19cabSRicardo Neri static int max_hfi_instances;
1621cb19cabSRicardo Neri static struct hfi_instance *hfi_instances;
1631cb19cabSRicardo Neri
1641cb19cabSRicardo Neri static struct hfi_features hfi_features;
1652d74e631SRicardo Neri static DEFINE_MUTEX(hfi_instance_lock);
1662d74e631SRicardo Neri
167ab09b074SRicardo Neri static struct workqueue_struct *hfi_updates_wq;
168ab09b074SRicardo Neri #define HFI_UPDATE_INTERVAL HZ
169bd30cdfdSSrinivas Pandruvada #define HFI_MAX_THERM_NOTIFY_COUNT 16
170bd30cdfdSSrinivas Pandruvada
get_hfi_caps(struct hfi_instance * hfi_instance,struct thermal_genl_cpu_caps * cpu_caps)171bd30cdfdSSrinivas Pandruvada static void get_hfi_caps(struct hfi_instance *hfi_instance,
172bd30cdfdSSrinivas Pandruvada struct thermal_genl_cpu_caps *cpu_caps)
173bd30cdfdSSrinivas Pandruvada {
174bd30cdfdSSrinivas Pandruvada int cpu, i = 0;
175bd30cdfdSSrinivas Pandruvada
176bd30cdfdSSrinivas Pandruvada raw_spin_lock_irq(&hfi_instance->table_lock);
177bd30cdfdSSrinivas Pandruvada for_each_cpu(cpu, hfi_instance->cpus) {
178bd30cdfdSSrinivas Pandruvada struct hfi_cpu_data *caps;
179bd30cdfdSSrinivas Pandruvada s16 index;
180bd30cdfdSSrinivas Pandruvada
181bd30cdfdSSrinivas Pandruvada index = per_cpu(hfi_cpu_info, cpu).index;
182bd30cdfdSSrinivas Pandruvada caps = hfi_instance->data + index * hfi_features.cpu_stride;
183bd30cdfdSSrinivas Pandruvada cpu_caps[i].cpu = cpu;
184bd30cdfdSSrinivas Pandruvada
185bd30cdfdSSrinivas Pandruvada /*
186bd30cdfdSSrinivas Pandruvada * Scale performance and energy efficiency to
187bd30cdfdSSrinivas Pandruvada * the [0, 1023] interval that thermal netlink uses.
188bd30cdfdSSrinivas Pandruvada */
189bd30cdfdSSrinivas Pandruvada cpu_caps[i].performance = caps->perf_cap << 2;
190bd30cdfdSSrinivas Pandruvada cpu_caps[i].efficiency = caps->ee_cap << 2;
191bd30cdfdSSrinivas Pandruvada
192bd30cdfdSSrinivas Pandruvada ++i;
193bd30cdfdSSrinivas Pandruvada }
194bd30cdfdSSrinivas Pandruvada raw_spin_unlock_irq(&hfi_instance->table_lock);
195bd30cdfdSSrinivas Pandruvada }
196bd30cdfdSSrinivas Pandruvada
197bd30cdfdSSrinivas Pandruvada /*
198bd30cdfdSSrinivas Pandruvada * Call update_capabilities() when there are changes in the HFI table.
199bd30cdfdSSrinivas Pandruvada */
update_capabilities(struct hfi_instance * hfi_instance)200bd30cdfdSSrinivas Pandruvada static void update_capabilities(struct hfi_instance *hfi_instance)
201bd30cdfdSSrinivas Pandruvada {
202bd30cdfdSSrinivas Pandruvada struct thermal_genl_cpu_caps *cpu_caps;
203bd30cdfdSSrinivas Pandruvada int i = 0, cpu_count;
204bd30cdfdSSrinivas Pandruvada
205bd30cdfdSSrinivas Pandruvada /* CPUs may come online/offline while processing an HFI update. */
206bd30cdfdSSrinivas Pandruvada mutex_lock(&hfi_instance_lock);
207bd30cdfdSSrinivas Pandruvada
208bd30cdfdSSrinivas Pandruvada cpu_count = cpumask_weight(hfi_instance->cpus);
209bd30cdfdSSrinivas Pandruvada
210bd30cdfdSSrinivas Pandruvada /* No CPUs to report in this hfi_instance. */
211bd30cdfdSSrinivas Pandruvada if (!cpu_count)
212bd30cdfdSSrinivas Pandruvada goto out;
213bd30cdfdSSrinivas Pandruvada
214bd30cdfdSSrinivas Pandruvada cpu_caps = kcalloc(cpu_count, sizeof(*cpu_caps), GFP_KERNEL);
215bd30cdfdSSrinivas Pandruvada if (!cpu_caps)
216bd30cdfdSSrinivas Pandruvada goto out;
217bd30cdfdSSrinivas Pandruvada
218bd30cdfdSSrinivas Pandruvada get_hfi_caps(hfi_instance, cpu_caps);
219bd30cdfdSSrinivas Pandruvada
220bd30cdfdSSrinivas Pandruvada if (cpu_count < HFI_MAX_THERM_NOTIFY_COUNT)
221bd30cdfdSSrinivas Pandruvada goto last_cmd;
222bd30cdfdSSrinivas Pandruvada
223bd30cdfdSSrinivas Pandruvada /* Process complete chunks of HFI_MAX_THERM_NOTIFY_COUNT capabilities. */
224bd30cdfdSSrinivas Pandruvada for (i = 0;
225bd30cdfdSSrinivas Pandruvada (i + HFI_MAX_THERM_NOTIFY_COUNT) <= cpu_count;
226bd30cdfdSSrinivas Pandruvada i += HFI_MAX_THERM_NOTIFY_COUNT)
227bd30cdfdSSrinivas Pandruvada thermal_genl_cpu_capability_event(HFI_MAX_THERM_NOTIFY_COUNT,
228bd30cdfdSSrinivas Pandruvada &cpu_caps[i]);
229bd30cdfdSSrinivas Pandruvada
230bd30cdfdSSrinivas Pandruvada cpu_count = cpu_count - i;
231bd30cdfdSSrinivas Pandruvada
232bd30cdfdSSrinivas Pandruvada last_cmd:
233bd30cdfdSSrinivas Pandruvada /* Process the remaining capabilities if any. */
234bd30cdfdSSrinivas Pandruvada if (cpu_count)
235bd30cdfdSSrinivas Pandruvada thermal_genl_cpu_capability_event(cpu_count, &cpu_caps[i]);
236bd30cdfdSSrinivas Pandruvada
237bd30cdfdSSrinivas Pandruvada kfree(cpu_caps);
238bd30cdfdSSrinivas Pandruvada out:
239bd30cdfdSSrinivas Pandruvada mutex_unlock(&hfi_instance_lock);
240bd30cdfdSSrinivas Pandruvada }
241ab09b074SRicardo Neri
hfi_update_work_fn(struct work_struct * work)242ab09b074SRicardo Neri static void hfi_update_work_fn(struct work_struct *work)
243ab09b074SRicardo Neri {
244ab09b074SRicardo Neri struct hfi_instance *hfi_instance;
245ab09b074SRicardo Neri
246ab09b074SRicardo Neri hfi_instance = container_of(to_delayed_work(work), struct hfi_instance,
247ab09b074SRicardo Neri update_work);
248ab09b074SRicardo Neri
249bd30cdfdSSrinivas Pandruvada update_capabilities(hfi_instance);
250ab09b074SRicardo Neri }
251ab09b074SRicardo Neri
intel_hfi_process_event(__u64 pkg_therm_status_msr_val)252ab09b074SRicardo Neri void intel_hfi_process_event(__u64 pkg_therm_status_msr_val)
253ab09b074SRicardo Neri {
254ab09b074SRicardo Neri struct hfi_instance *hfi_instance;
255ab09b074SRicardo Neri int cpu = smp_processor_id();
256ab09b074SRicardo Neri struct hfi_cpu_info *info;
257c0e3acdcSSrinivas Pandruvada u64 new_timestamp, msr, hfi;
258ab09b074SRicardo Neri
259ab09b074SRicardo Neri if (!pkg_therm_status_msr_val)
260ab09b074SRicardo Neri return;
261ab09b074SRicardo Neri
262ab09b074SRicardo Neri info = &per_cpu(hfi_cpu_info, cpu);
263ab09b074SRicardo Neri if (!info)
264ab09b074SRicardo Neri return;
265ab09b074SRicardo Neri
266ab09b074SRicardo Neri /*
267ab09b074SRicardo Neri * A CPU is linked to its HFI instance before the thermal vector in the
268ab09b074SRicardo Neri * local APIC is unmasked. Hence, info->hfi_instance cannot be NULL
269ab09b074SRicardo Neri * when receiving an HFI event.
270ab09b074SRicardo Neri */
271ab09b074SRicardo Neri hfi_instance = info->hfi_instance;
272ab09b074SRicardo Neri if (unlikely(!hfi_instance)) {
273ab09b074SRicardo Neri pr_debug("Received event on CPU %d but instance was null", cpu);
274ab09b074SRicardo Neri return;
275ab09b074SRicardo Neri }
276ab09b074SRicardo Neri
277ab09b074SRicardo Neri /*
278ab09b074SRicardo Neri * On most systems, all CPUs in the package receive a package-level
279ab09b074SRicardo Neri * thermal interrupt when there is an HFI update. It is sufficient to
280ab09b074SRicardo Neri * let a single CPU to acknowledge the update and queue work to
281ab09b074SRicardo Neri * process it. The remaining CPUs can resume their work.
282ab09b074SRicardo Neri */
283ab09b074SRicardo Neri if (!raw_spin_trylock(&hfi_instance->event_lock))
284ab09b074SRicardo Neri return;
285ab09b074SRicardo Neri
286c0e3acdcSSrinivas Pandruvada rdmsrl(MSR_IA32_PACKAGE_THERM_STATUS, msr);
287c0e3acdcSSrinivas Pandruvada hfi = msr & PACKAGE_THERM_STATUS_HFI_UPDATED;
288c0e3acdcSSrinivas Pandruvada if (!hfi) {
289c0e3acdcSSrinivas Pandruvada raw_spin_unlock(&hfi_instance->event_lock);
290c0e3acdcSSrinivas Pandruvada return;
291c0e3acdcSSrinivas Pandruvada }
292c0e3acdcSSrinivas Pandruvada
293c0e3acdcSSrinivas Pandruvada /*
294c0e3acdcSSrinivas Pandruvada * Ack duplicate update. Since there is an active HFI
295c0e3acdcSSrinivas Pandruvada * status from HW, it must be a new event, not a case
296c0e3acdcSSrinivas Pandruvada * where a lagging CPU entered the locked region.
297c0e3acdcSSrinivas Pandruvada */
298ab09b074SRicardo Neri new_timestamp = *(u64 *)hfi_instance->hw_table;
299ab09b074SRicardo Neri if (*hfi_instance->timestamp == new_timestamp) {
300c0e3acdcSSrinivas Pandruvada thermal_clear_package_intr_status(PACKAGE_LEVEL, PACKAGE_THERM_STATUS_HFI_UPDATED);
301ab09b074SRicardo Neri raw_spin_unlock(&hfi_instance->event_lock);
302ab09b074SRicardo Neri return;
303ab09b074SRicardo Neri }
304ab09b074SRicardo Neri
305ab09b074SRicardo Neri raw_spin_lock(&hfi_instance->table_lock);
306ab09b074SRicardo Neri
307ab09b074SRicardo Neri /*
308ab09b074SRicardo Neri * Copy the updated table into our local copy. This includes the new
309ab09b074SRicardo Neri * timestamp.
310ab09b074SRicardo Neri */
311ab09b074SRicardo Neri memcpy(hfi_instance->local_table, hfi_instance->hw_table,
312ab09b074SRicardo Neri hfi_features.nr_table_pages << PAGE_SHIFT);
313ab09b074SRicardo Neri
314ab09b074SRicardo Neri /*
315ab09b074SRicardo Neri * Let hardware know that we are done reading the HFI table and it is
316ab09b074SRicardo Neri * free to update it again.
317ab09b074SRicardo Neri */
318930d06bfSSrinivas Pandruvada thermal_clear_package_intr_status(PACKAGE_LEVEL, PACKAGE_THERM_STATUS_HFI_UPDATED);
319ab09b074SRicardo Neri
320c0e3acdcSSrinivas Pandruvada raw_spin_unlock(&hfi_instance->table_lock);
321c0e3acdcSSrinivas Pandruvada raw_spin_unlock(&hfi_instance->event_lock);
322c0e3acdcSSrinivas Pandruvada
323ab09b074SRicardo Neri queue_delayed_work(hfi_updates_wq, &hfi_instance->update_work,
324ab09b074SRicardo Neri HFI_UPDATE_INTERVAL);
325ab09b074SRicardo Neri }
326ab09b074SRicardo Neri
init_hfi_cpu_index(struct hfi_cpu_info * info)3272d74e631SRicardo Neri static void init_hfi_cpu_index(struct hfi_cpu_info *info)
3282d74e631SRicardo Neri {
3292d74e631SRicardo Neri union cpuid6_edx edx;
3302d74e631SRicardo Neri
3312d74e631SRicardo Neri /* Do not re-read @cpu's index if it has already been initialized. */
3322d74e631SRicardo Neri if (info->index > -1)
3332d74e631SRicardo Neri return;
3342d74e631SRicardo Neri
3352d74e631SRicardo Neri edx.full = cpuid_edx(CPUID_HFI_LEAF);
3362d74e631SRicardo Neri info->index = edx.split.index;
3372d74e631SRicardo Neri }
3382d74e631SRicardo Neri
3392d74e631SRicardo Neri /*
3402d74e631SRicardo Neri * The format of the HFI table depends on the number of capabilities that the
3412d74e631SRicardo Neri * hardware supports. Keep a data structure to navigate the table.
3422d74e631SRicardo Neri */
init_hfi_instance(struct hfi_instance * hfi_instance)3432d74e631SRicardo Neri static void init_hfi_instance(struct hfi_instance *hfi_instance)
3442d74e631SRicardo Neri {
3452d74e631SRicardo Neri /* The HFI header is below the time-stamp. */
3462d74e631SRicardo Neri hfi_instance->hdr = hfi_instance->local_table +
3472d74e631SRicardo Neri sizeof(*hfi_instance->timestamp);
3482d74e631SRicardo Neri
3492d74e631SRicardo Neri /* The HFI data starts below the header. */
3502d74e631SRicardo Neri hfi_instance->data = hfi_instance->hdr + hfi_features.hdr_size;
3512d74e631SRicardo Neri }
3522d74e631SRicardo Neri
353de791353SRicardo Neri /* Caller must hold hfi_instance_lock. */
hfi_enable(void)354de791353SRicardo Neri static void hfi_enable(void)
355de791353SRicardo Neri {
356de791353SRicardo Neri u64 msr_val;
357de791353SRicardo Neri
358de791353SRicardo Neri rdmsrl(MSR_IA32_HW_FEEDBACK_CONFIG, msr_val);
359de791353SRicardo Neri msr_val |= HW_FEEDBACK_CONFIG_HFI_ENABLE_BIT;
360de791353SRicardo Neri wrmsrl(MSR_IA32_HW_FEEDBACK_CONFIG, msr_val);
361de791353SRicardo Neri }
362de791353SRicardo Neri
hfi_set_hw_table(struct hfi_instance * hfi_instance)363de791353SRicardo Neri static void hfi_set_hw_table(struct hfi_instance *hfi_instance)
364de791353SRicardo Neri {
365de791353SRicardo Neri phys_addr_t hw_table_pa;
366de791353SRicardo Neri u64 msr_val;
367de791353SRicardo Neri
368de791353SRicardo Neri hw_table_pa = virt_to_phys(hfi_instance->hw_table);
369de791353SRicardo Neri msr_val = hw_table_pa | HW_FEEDBACK_PTR_VALID_BIT;
370de791353SRicardo Neri wrmsrl(MSR_IA32_HW_FEEDBACK_PTR, msr_val);
371de791353SRicardo Neri }
372de791353SRicardo Neri
3730caf5dd0SRicardo Neri /* Caller must hold hfi_instance_lock. */
hfi_disable(void)3740caf5dd0SRicardo Neri static void hfi_disable(void)
3750caf5dd0SRicardo Neri {
3760caf5dd0SRicardo Neri u64 msr_val;
3770caf5dd0SRicardo Neri int i;
3780caf5dd0SRicardo Neri
3790caf5dd0SRicardo Neri rdmsrl(MSR_IA32_HW_FEEDBACK_CONFIG, msr_val);
3800caf5dd0SRicardo Neri msr_val &= ~HW_FEEDBACK_CONFIG_HFI_ENABLE_BIT;
3810caf5dd0SRicardo Neri wrmsrl(MSR_IA32_HW_FEEDBACK_CONFIG, msr_val);
3820caf5dd0SRicardo Neri
3830caf5dd0SRicardo Neri /*
3840caf5dd0SRicardo Neri * Wait for hardware to acknowledge the disabling of HFI. Some
3850caf5dd0SRicardo Neri * processors may not do it. Wait for ~2ms. This is a reasonable
3860caf5dd0SRicardo Neri * time for hardware to complete any pending actions on the HFI
3870caf5dd0SRicardo Neri * memory.
3880caf5dd0SRicardo Neri */
3890caf5dd0SRicardo Neri for (i = 0; i < 2000; i++) {
3900caf5dd0SRicardo Neri rdmsrl(MSR_IA32_PACKAGE_THERM_STATUS, msr_val);
3910caf5dd0SRicardo Neri if (msr_val & PACKAGE_THERM_STATUS_HFI_UPDATED)
3920caf5dd0SRicardo Neri break;
3930caf5dd0SRicardo Neri
3940caf5dd0SRicardo Neri udelay(1);
3950caf5dd0SRicardo Neri cpu_relax();
3960caf5dd0SRicardo Neri }
3970caf5dd0SRicardo Neri }
3980caf5dd0SRicardo Neri
3992d74e631SRicardo Neri /**
4002d74e631SRicardo Neri * intel_hfi_online() - Enable HFI on @cpu
4012d74e631SRicardo Neri * @cpu: CPU in which the HFI will be enabled
4022d74e631SRicardo Neri *
4032d74e631SRicardo Neri * Enable the HFI to be used in @cpu. The HFI is enabled at the die/package
4042d74e631SRicardo Neri * level. The first CPU in the die/package to come online does the full HFI
4052d74e631SRicardo Neri * initialization. Subsequent CPUs will just link themselves to the HFI
4062d74e631SRicardo Neri * instance of their die/package.
4072d74e631SRicardo Neri *
4082d74e631SRicardo Neri * This function is called before enabling the thermal vector in the local APIC
4092d74e631SRicardo Neri * in order to ensure that @cpu has an associated HFI instance when it receives
4102d74e631SRicardo Neri * an HFI event.
4112d74e631SRicardo Neri */
intel_hfi_online(unsigned int cpu)4122d74e631SRicardo Neri void intel_hfi_online(unsigned int cpu)
4132d74e631SRicardo Neri {
4142d74e631SRicardo Neri struct hfi_instance *hfi_instance;
4152d74e631SRicardo Neri struct hfi_cpu_info *info;
4162d74e631SRicardo Neri u16 die_id;
4172d74e631SRicardo Neri
4182d74e631SRicardo Neri /* Nothing to do if hfi_instances are missing. */
4192d74e631SRicardo Neri if (!hfi_instances)
4202d74e631SRicardo Neri return;
4212d74e631SRicardo Neri
4222d74e631SRicardo Neri /*
4232d74e631SRicardo Neri * Link @cpu to the HFI instance of its package/die. It does not
4242d74e631SRicardo Neri * matter whether the instance has been initialized.
4252d74e631SRicardo Neri */
4262d74e631SRicardo Neri info = &per_cpu(hfi_cpu_info, cpu);
4272d74e631SRicardo Neri die_id = topology_logical_die_id(cpu);
4282d74e631SRicardo Neri hfi_instance = info->hfi_instance;
4292d74e631SRicardo Neri if (!hfi_instance) {
4303a3073b6SRicardo Neri if (die_id >= max_hfi_instances)
4312d74e631SRicardo Neri return;
4322d74e631SRicardo Neri
4332d74e631SRicardo Neri hfi_instance = &hfi_instances[die_id];
4342d74e631SRicardo Neri info->hfi_instance = hfi_instance;
4352d74e631SRicardo Neri }
4362d74e631SRicardo Neri
4372d74e631SRicardo Neri init_hfi_cpu_index(info);
4382d74e631SRicardo Neri
4392d74e631SRicardo Neri /*
4402d74e631SRicardo Neri * Now check if the HFI instance of the package/die of @cpu has been
4412d74e631SRicardo Neri * initialized (by checking its header). In such case, all we have to
4422d74e631SRicardo Neri * do is to add @cpu to this instance's cpumask.
4432d74e631SRicardo Neri */
4442d74e631SRicardo Neri mutex_lock(&hfi_instance_lock);
4452d74e631SRicardo Neri if (hfi_instance->hdr) {
4462d74e631SRicardo Neri cpumask_set_cpu(cpu, hfi_instance->cpus);
4472d74e631SRicardo Neri goto unlock;
4482d74e631SRicardo Neri }
4492d74e631SRicardo Neri
4502d74e631SRicardo Neri /*
4512d74e631SRicardo Neri * Hardware is programmed with the physical address of the first page
4522d74e631SRicardo Neri * frame of the table. Hence, the allocated memory must be page-aligned.
4530caf5dd0SRicardo Neri *
4540caf5dd0SRicardo Neri * Some processors do not forget the initial address of the HFI table
4550caf5dd0SRicardo Neri * even after having been reprogrammed. Keep using the same pages. Do
4560caf5dd0SRicardo Neri * not free them.
4572d74e631SRicardo Neri */
4582d74e631SRicardo Neri hfi_instance->hw_table = alloc_pages_exact(hfi_features.nr_table_pages,
4592d74e631SRicardo Neri GFP_KERNEL | __GFP_ZERO);
4602d74e631SRicardo Neri if (!hfi_instance->hw_table)
4612d74e631SRicardo Neri goto unlock;
4622d74e631SRicardo Neri
4632d74e631SRicardo Neri /*
4642d74e631SRicardo Neri * Allocate memory to keep a local copy of the table that
4652d74e631SRicardo Neri * hardware generates.
4662d74e631SRicardo Neri */
4672d74e631SRicardo Neri hfi_instance->local_table = kzalloc(hfi_features.nr_table_pages << PAGE_SHIFT,
4682d74e631SRicardo Neri GFP_KERNEL);
4692d74e631SRicardo Neri if (!hfi_instance->local_table)
4702d74e631SRicardo Neri goto free_hw_table;
4712d74e631SRicardo Neri
4722d74e631SRicardo Neri init_hfi_instance(hfi_instance);
4732d74e631SRicardo Neri
474ab09b074SRicardo Neri INIT_DELAYED_WORK(&hfi_instance->update_work, hfi_update_work_fn);
475ab09b074SRicardo Neri raw_spin_lock_init(&hfi_instance->table_lock);
476ab09b074SRicardo Neri raw_spin_lock_init(&hfi_instance->event_lock);
477ab09b074SRicardo Neri
4782d74e631SRicardo Neri cpumask_set_cpu(cpu, hfi_instance->cpus);
4792d74e631SRicardo Neri
480de791353SRicardo Neri hfi_set_hw_table(hfi_instance);
481de791353SRicardo Neri hfi_enable();
482ab09b074SRicardo Neri
4832d74e631SRicardo Neri unlock:
4842d74e631SRicardo Neri mutex_unlock(&hfi_instance_lock);
4852d74e631SRicardo Neri return;
4862d74e631SRicardo Neri
4872d74e631SRicardo Neri free_hw_table:
4882d74e631SRicardo Neri free_pages_exact(hfi_instance->hw_table, hfi_features.nr_table_pages);
4892d74e631SRicardo Neri goto unlock;
4902d74e631SRicardo Neri }
4912d74e631SRicardo Neri
4922d74e631SRicardo Neri /**
4932d74e631SRicardo Neri * intel_hfi_offline() - Disable HFI on @cpu
4942d74e631SRicardo Neri * @cpu: CPU in which the HFI will be disabled
4952d74e631SRicardo Neri *
4962d74e631SRicardo Neri * Remove @cpu from those covered by its HFI instance.
4972d74e631SRicardo Neri *
4982d74e631SRicardo Neri * On some processors, hardware remembers previous programming settings even
4992d74e631SRicardo Neri * after being reprogrammed. Thus, keep HFI enabled even if all CPUs in the
5002d74e631SRicardo Neri * die/package of @cpu are offline. See note in intel_hfi_online().
5012d74e631SRicardo Neri */
intel_hfi_offline(unsigned int cpu)5022d74e631SRicardo Neri void intel_hfi_offline(unsigned int cpu)
5032d74e631SRicardo Neri {
5042d74e631SRicardo Neri struct hfi_cpu_info *info = &per_cpu(hfi_cpu_info, cpu);
5052d74e631SRicardo Neri struct hfi_instance *hfi_instance;
5062d74e631SRicardo Neri
5072d74e631SRicardo Neri /*
5082d74e631SRicardo Neri * Check if @cpu as an associated, initialized (i.e., with a non-NULL
5092d74e631SRicardo Neri * header). Also, HFI instances are only initialized if X86_FEATURE_HFI
5102d74e631SRicardo Neri * is present.
5112d74e631SRicardo Neri */
5122d74e631SRicardo Neri hfi_instance = info->hfi_instance;
5132d74e631SRicardo Neri if (!hfi_instance)
5142d74e631SRicardo Neri return;
5152d74e631SRicardo Neri
5162d74e631SRicardo Neri if (!hfi_instance->hdr)
5172d74e631SRicardo Neri return;
5182d74e631SRicardo Neri
5192d74e631SRicardo Neri mutex_lock(&hfi_instance_lock);
5202d74e631SRicardo Neri cpumask_clear_cpu(cpu, hfi_instance->cpus);
5210caf5dd0SRicardo Neri
5220caf5dd0SRicardo Neri if (!cpumask_weight(hfi_instance->cpus))
5230caf5dd0SRicardo Neri hfi_disable();
5240caf5dd0SRicardo Neri
5252d74e631SRicardo Neri mutex_unlock(&hfi_instance_lock);
5262d74e631SRicardo Neri }
5271cb19cabSRicardo Neri
hfi_parse_features(void)5281cb19cabSRicardo Neri static __init int hfi_parse_features(void)
5291cb19cabSRicardo Neri {
5301cb19cabSRicardo Neri unsigned int nr_capabilities;
5311cb19cabSRicardo Neri union cpuid6_edx edx;
5321cb19cabSRicardo Neri
5331cb19cabSRicardo Neri if (!boot_cpu_has(X86_FEATURE_HFI))
5341cb19cabSRicardo Neri return -ENODEV;
5351cb19cabSRicardo Neri
5361cb19cabSRicardo Neri /*
5371cb19cabSRicardo Neri * If we are here we know that CPUID_HFI_LEAF exists. Parse the
5381cb19cabSRicardo Neri * supported capabilities and the size of the HFI table.
5391cb19cabSRicardo Neri */
5401cb19cabSRicardo Neri edx.full = cpuid_edx(CPUID_HFI_LEAF);
5411cb19cabSRicardo Neri
5421cb19cabSRicardo Neri if (!edx.split.capabilities.split.performance) {
5431cb19cabSRicardo Neri pr_debug("Performance reporting not supported! Not using HFI\n");
5441cb19cabSRicardo Neri return -ENODEV;
5451cb19cabSRicardo Neri }
5461cb19cabSRicardo Neri
5471cb19cabSRicardo Neri /*
5481cb19cabSRicardo Neri * The number of supported capabilities determines the number of
5491cb19cabSRicardo Neri * columns in the HFI table. Exclude the reserved bits.
5501cb19cabSRicardo Neri */
5511cb19cabSRicardo Neri edx.split.capabilities.split.__reserved = 0;
5521cb19cabSRicardo Neri nr_capabilities = hweight8(edx.split.capabilities.bits);
5531cb19cabSRicardo Neri
5541cb19cabSRicardo Neri /* The number of 4KB pages required by the table */
5551cb19cabSRicardo Neri hfi_features.nr_table_pages = edx.split.table_pages + 1;
5561cb19cabSRicardo Neri
5571cb19cabSRicardo Neri /*
5581cb19cabSRicardo Neri * The header contains change indications for each supported feature.
5591cb19cabSRicardo Neri * The size of the table header is rounded up to be a multiple of 8
5601cb19cabSRicardo Neri * bytes.
5611cb19cabSRicardo Neri */
5621cb19cabSRicardo Neri hfi_features.hdr_size = DIV_ROUND_UP(nr_capabilities, 8) * 8;
5631cb19cabSRicardo Neri
5641cb19cabSRicardo Neri /*
5651cb19cabSRicardo Neri * Data of each logical processor is also rounded up to be a multiple
5661cb19cabSRicardo Neri * of 8 bytes.
5671cb19cabSRicardo Neri */
5681cb19cabSRicardo Neri hfi_features.cpu_stride = DIV_ROUND_UP(nr_capabilities, 8) * 8;
5691cb19cabSRicardo Neri
5701cb19cabSRicardo Neri return 0;
5711cb19cabSRicardo Neri }
5721cb19cabSRicardo Neri
hfi_do_enable(void)573*019ccc66SRicardo Neri static void hfi_do_enable(void)
574*019ccc66SRicardo Neri {
575*019ccc66SRicardo Neri /* This code runs only on the boot CPU. */
576*019ccc66SRicardo Neri struct hfi_cpu_info *info = &per_cpu(hfi_cpu_info, 0);
577*019ccc66SRicardo Neri struct hfi_instance *hfi_instance = info->hfi_instance;
578*019ccc66SRicardo Neri
579*019ccc66SRicardo Neri /* No locking needed. There is no concurrency with CPU online. */
580*019ccc66SRicardo Neri hfi_set_hw_table(hfi_instance);
581*019ccc66SRicardo Neri hfi_enable();
582*019ccc66SRicardo Neri }
583*019ccc66SRicardo Neri
hfi_do_disable(void)584*019ccc66SRicardo Neri static int hfi_do_disable(void)
585*019ccc66SRicardo Neri {
586*019ccc66SRicardo Neri /* No locking needed. There is no concurrency with CPU offline. */
587*019ccc66SRicardo Neri hfi_disable();
588*019ccc66SRicardo Neri
589*019ccc66SRicardo Neri return 0;
590*019ccc66SRicardo Neri }
591*019ccc66SRicardo Neri
592*019ccc66SRicardo Neri static struct syscore_ops hfi_pm_ops = {
593*019ccc66SRicardo Neri .resume = hfi_do_enable,
594*019ccc66SRicardo Neri .suspend = hfi_do_disable,
595*019ccc66SRicardo Neri };
596*019ccc66SRicardo Neri
intel_hfi_init(void)5971cb19cabSRicardo Neri void __init intel_hfi_init(void)
5981cb19cabSRicardo Neri {
5992d74e631SRicardo Neri struct hfi_instance *hfi_instance;
6002d74e631SRicardo Neri int i, j;
6012d74e631SRicardo Neri
6021cb19cabSRicardo Neri if (hfi_parse_features())
6031cb19cabSRicardo Neri return;
6041cb19cabSRicardo Neri
6051cb19cabSRicardo Neri /* There is one HFI instance per die/package. */
6061cb19cabSRicardo Neri max_hfi_instances = topology_max_packages() *
6071cb19cabSRicardo Neri topology_max_die_per_package();
6081cb19cabSRicardo Neri
6091cb19cabSRicardo Neri /*
6101cb19cabSRicardo Neri * This allocation may fail. CPU hotplug callbacks must check
6111cb19cabSRicardo Neri * for a null pointer.
6121cb19cabSRicardo Neri */
6131cb19cabSRicardo Neri hfi_instances = kcalloc(max_hfi_instances, sizeof(*hfi_instances),
6141cb19cabSRicardo Neri GFP_KERNEL);
6152d74e631SRicardo Neri if (!hfi_instances)
6162d74e631SRicardo Neri return;
6172d74e631SRicardo Neri
6182d74e631SRicardo Neri for (i = 0; i < max_hfi_instances; i++) {
6192d74e631SRicardo Neri hfi_instance = &hfi_instances[i];
6202d74e631SRicardo Neri if (!zalloc_cpumask_var(&hfi_instance->cpus, GFP_KERNEL))
6212d74e631SRicardo Neri goto err_nomem;
6222d74e631SRicardo Neri }
6232d74e631SRicardo Neri
624ab09b074SRicardo Neri hfi_updates_wq = create_singlethread_workqueue("hfi-updates");
625ab09b074SRicardo Neri if (!hfi_updates_wq)
626ab09b074SRicardo Neri goto err_nomem;
627ab09b074SRicardo Neri
628*019ccc66SRicardo Neri register_syscore_ops(&hfi_pm_ops);
629*019ccc66SRicardo Neri
6302d74e631SRicardo Neri return;
6312d74e631SRicardo Neri
6322d74e631SRicardo Neri err_nomem:
6332d74e631SRicardo Neri for (j = 0; j < i; ++j) {
6342d74e631SRicardo Neri hfi_instance = &hfi_instances[j];
6352d74e631SRicardo Neri free_cpumask_var(hfi_instance->cpus);
6362d74e631SRicardo Neri }
6372d74e631SRicardo Neri
6382d74e631SRicardo Neri kfree(hfi_instances);
6392d74e631SRicardo Neri hfi_instances = NULL;
6401cb19cabSRicardo Neri }
641