11cb19cabSRicardo Neri // SPDX-License-Identifier: GPL-2.0-only 21cb19cabSRicardo Neri /* 31cb19cabSRicardo Neri * Hardware Feedback Interface Driver 41cb19cabSRicardo Neri * 51cb19cabSRicardo Neri * Copyright (c) 2021, Intel Corporation. 61cb19cabSRicardo Neri * 71cb19cabSRicardo Neri * Authors: Aubrey Li <aubrey.li@linux.intel.com> 81cb19cabSRicardo Neri * Ricardo Neri <ricardo.neri-calderon@linux.intel.com> 91cb19cabSRicardo Neri * 101cb19cabSRicardo Neri * 111cb19cabSRicardo Neri * The Hardware Feedback Interface provides a performance and energy efficiency 121cb19cabSRicardo Neri * capability information for each CPU in the system. Depending on the processor 131cb19cabSRicardo Neri * model, hardware may periodically update these capabilities as a result of 141cb19cabSRicardo Neri * changes in the operating conditions (e.g., power limits or thermal 151cb19cabSRicardo Neri * constraints). On other processor models, there is a single HFI update 161cb19cabSRicardo Neri * at boot. 171cb19cabSRicardo Neri * 181cb19cabSRicardo Neri * This file provides functionality to process HFI updates and relay these 191cb19cabSRicardo Neri * updates to userspace. 201cb19cabSRicardo Neri */ 211cb19cabSRicardo Neri 221cb19cabSRicardo Neri #define pr_fmt(fmt) "intel-hfi: " fmt 231cb19cabSRicardo Neri 241cb19cabSRicardo Neri #include <linux/bitops.h> 251cb19cabSRicardo Neri #include <linux/cpufeature.h> 262d74e631SRicardo Neri #include <linux/cpumask.h> 272d74e631SRicardo Neri #include <linux/gfp.h> 282d74e631SRicardo Neri #include <linux/io.h> 29ab09b074SRicardo Neri #include <linux/kernel.h> 301cb19cabSRicardo Neri #include <linux/math.h> 312d74e631SRicardo Neri #include <linux/mutex.h> 322d74e631SRicardo Neri #include <linux/percpu-defs.h> 331cb19cabSRicardo Neri #include <linux/printk.h> 341cb19cabSRicardo Neri #include <linux/processor.h> 351cb19cabSRicardo Neri #include <linux/slab.h> 36ab09b074SRicardo Neri #include <linux/spinlock.h> 37ab09b074SRicardo Neri #include <linux/string.h> 381cb19cabSRicardo Neri #include <linux/topology.h> 39ab09b074SRicardo Neri #include <linux/workqueue.h> 401cb19cabSRicardo Neri 412d74e631SRicardo Neri #include <asm/msr.h> 422d74e631SRicardo Neri 43bd30cdfdSSrinivas Pandruvada #include "../thermal_core.h" 441cb19cabSRicardo Neri #include "intel_hfi.h" 45930d06bfSSrinivas Pandruvada #include "thermal_interrupt.h" 46ab09b074SRicardo Neri 472d74e631SRicardo Neri /* Hardware Feedback Interface MSR configuration bits */ 482d74e631SRicardo Neri #define HW_FEEDBACK_PTR_VALID_BIT BIT(0) 49ab09b074SRicardo Neri #define HW_FEEDBACK_CONFIG_HFI_ENABLE_BIT BIT(0) 502d74e631SRicardo Neri 511cb19cabSRicardo Neri /* CPUID detection and enumeration definitions for HFI */ 521cb19cabSRicardo Neri 531cb19cabSRicardo Neri #define CPUID_HFI_LEAF 6 541cb19cabSRicardo Neri 551cb19cabSRicardo Neri union hfi_capabilities { 561cb19cabSRicardo Neri struct { 571cb19cabSRicardo Neri u8 performance:1; 581cb19cabSRicardo Neri u8 energy_efficiency:1; 591cb19cabSRicardo Neri u8 __reserved:6; 601cb19cabSRicardo Neri } split; 611cb19cabSRicardo Neri u8 bits; 621cb19cabSRicardo Neri }; 631cb19cabSRicardo Neri 641cb19cabSRicardo Neri union cpuid6_edx { 651cb19cabSRicardo Neri struct { 661cb19cabSRicardo Neri union hfi_capabilities capabilities; 671cb19cabSRicardo Neri u32 table_pages:4; 681cb19cabSRicardo Neri u32 __reserved:4; 691cb19cabSRicardo Neri s32 index:16; 701cb19cabSRicardo Neri } split; 711cb19cabSRicardo Neri u32 full; 721cb19cabSRicardo Neri }; 731cb19cabSRicardo Neri 741cb19cabSRicardo Neri /** 751cb19cabSRicardo Neri * struct hfi_cpu_data - HFI capabilities per CPU 761cb19cabSRicardo Neri * @perf_cap: Performance capability 771cb19cabSRicardo Neri * @ee_cap: Energy efficiency capability 781cb19cabSRicardo Neri * 791cb19cabSRicardo Neri * Capabilities of a logical processor in the HFI table. These capabilities are 801cb19cabSRicardo Neri * unitless. 811cb19cabSRicardo Neri */ 821cb19cabSRicardo Neri struct hfi_cpu_data { 831cb19cabSRicardo Neri u8 perf_cap; 841cb19cabSRicardo Neri u8 ee_cap; 851cb19cabSRicardo Neri } __packed; 861cb19cabSRicardo Neri 871cb19cabSRicardo Neri /** 881cb19cabSRicardo Neri * struct hfi_hdr - Header of the HFI table 891cb19cabSRicardo Neri * @perf_updated: Hardware updated performance capabilities 901cb19cabSRicardo Neri * @ee_updated: Hardware updated energy efficiency capabilities 911cb19cabSRicardo Neri * 921cb19cabSRicardo Neri * Properties of the data in an HFI table. 931cb19cabSRicardo Neri */ 941cb19cabSRicardo Neri struct hfi_hdr { 951cb19cabSRicardo Neri u8 perf_updated; 961cb19cabSRicardo Neri u8 ee_updated; 971cb19cabSRicardo Neri } __packed; 981cb19cabSRicardo Neri 991cb19cabSRicardo Neri /** 1001cb19cabSRicardo Neri * struct hfi_instance - Representation of an HFI instance (i.e., a table) 1011cb19cabSRicardo Neri * @local_table: Base of the local copy of the HFI table 1021cb19cabSRicardo Neri * @timestamp: Timestamp of the last update of the local table. 1031cb19cabSRicardo Neri * Located at the base of the local table. 1041cb19cabSRicardo Neri * @hdr: Base address of the header of the local table 1051cb19cabSRicardo Neri * @data: Base address of the data of the local table 1062d74e631SRicardo Neri * @cpus: CPUs represented in this HFI table instance 1072d74e631SRicardo Neri * @hw_table: Pointer to the HFI table of this instance 108ab09b074SRicardo Neri * @update_work: Delayed work to process HFI updates 109ab09b074SRicardo Neri * @table_lock: Lock to protect acceses to the table of this instance 110ab09b074SRicardo Neri * @event_lock: Lock to process HFI interrupts 1111cb19cabSRicardo Neri * 1121cb19cabSRicardo Neri * A set of parameters to parse and navigate a specific HFI table. 1131cb19cabSRicardo Neri */ 1141cb19cabSRicardo Neri struct hfi_instance { 1151cb19cabSRicardo Neri union { 1161cb19cabSRicardo Neri void *local_table; 1171cb19cabSRicardo Neri u64 *timestamp; 1181cb19cabSRicardo Neri }; 1191cb19cabSRicardo Neri void *hdr; 1201cb19cabSRicardo Neri void *data; 1212d74e631SRicardo Neri cpumask_var_t cpus; 1222d74e631SRicardo Neri void *hw_table; 123ab09b074SRicardo Neri struct delayed_work update_work; 124ab09b074SRicardo Neri raw_spinlock_t table_lock; 125ab09b074SRicardo Neri raw_spinlock_t event_lock; 1261cb19cabSRicardo Neri }; 1271cb19cabSRicardo Neri 1281cb19cabSRicardo Neri /** 1291cb19cabSRicardo Neri * struct hfi_features - Supported HFI features 1301cb19cabSRicardo Neri * @nr_table_pages: Size of the HFI table in 4KB pages 1311cb19cabSRicardo Neri * @cpu_stride: Stride size to locate the capability data of a logical 1321cb19cabSRicardo Neri * processor within the table (i.e., row stride) 1331cb19cabSRicardo Neri * @hdr_size: Size of the table header 1341cb19cabSRicardo Neri * 1351cb19cabSRicardo Neri * Parameters and supported features that are common to all HFI instances 1361cb19cabSRicardo Neri */ 1371cb19cabSRicardo Neri struct hfi_features { 13854d9135cSRicardo Neri size_t nr_table_pages; 1391cb19cabSRicardo Neri unsigned int cpu_stride; 1401cb19cabSRicardo Neri unsigned int hdr_size; 1411cb19cabSRicardo Neri }; 1421cb19cabSRicardo Neri 1432d74e631SRicardo Neri /** 1442d74e631SRicardo Neri * struct hfi_cpu_info - Per-CPU attributes to consume HFI data 1452d74e631SRicardo Neri * @index: Row of this CPU in its HFI table 1462d74e631SRicardo Neri * @hfi_instance: Attributes of the HFI table to which this CPU belongs 1472d74e631SRicardo Neri * 1482d74e631SRicardo Neri * Parameters to link a logical processor to an HFI table and a row within it. 1492d74e631SRicardo Neri */ 1502d74e631SRicardo Neri struct hfi_cpu_info { 1512d74e631SRicardo Neri s16 index; 1522d74e631SRicardo Neri struct hfi_instance *hfi_instance; 1532d74e631SRicardo Neri }; 1542d74e631SRicardo Neri 1552d74e631SRicardo Neri static DEFINE_PER_CPU(struct hfi_cpu_info, hfi_cpu_info) = { .index = -1 }; 1562d74e631SRicardo Neri 1571cb19cabSRicardo Neri static int max_hfi_instances; 1581cb19cabSRicardo Neri static struct hfi_instance *hfi_instances; 1591cb19cabSRicardo Neri 1601cb19cabSRicardo Neri static struct hfi_features hfi_features; 1612d74e631SRicardo Neri static DEFINE_MUTEX(hfi_instance_lock); 1622d74e631SRicardo Neri 163ab09b074SRicardo Neri static struct workqueue_struct *hfi_updates_wq; 164ab09b074SRicardo Neri #define HFI_UPDATE_INTERVAL HZ 165bd30cdfdSSrinivas Pandruvada #define HFI_MAX_THERM_NOTIFY_COUNT 16 166bd30cdfdSSrinivas Pandruvada 167bd30cdfdSSrinivas Pandruvada static void get_hfi_caps(struct hfi_instance *hfi_instance, 168bd30cdfdSSrinivas Pandruvada struct thermal_genl_cpu_caps *cpu_caps) 169bd30cdfdSSrinivas Pandruvada { 170bd30cdfdSSrinivas Pandruvada int cpu, i = 0; 171bd30cdfdSSrinivas Pandruvada 172bd30cdfdSSrinivas Pandruvada raw_spin_lock_irq(&hfi_instance->table_lock); 173bd30cdfdSSrinivas Pandruvada for_each_cpu(cpu, hfi_instance->cpus) { 174bd30cdfdSSrinivas Pandruvada struct hfi_cpu_data *caps; 175bd30cdfdSSrinivas Pandruvada s16 index; 176bd30cdfdSSrinivas Pandruvada 177bd30cdfdSSrinivas Pandruvada index = per_cpu(hfi_cpu_info, cpu).index; 178bd30cdfdSSrinivas Pandruvada caps = hfi_instance->data + index * hfi_features.cpu_stride; 179bd30cdfdSSrinivas Pandruvada cpu_caps[i].cpu = cpu; 180bd30cdfdSSrinivas Pandruvada 181bd30cdfdSSrinivas Pandruvada /* 182bd30cdfdSSrinivas Pandruvada * Scale performance and energy efficiency to 183bd30cdfdSSrinivas Pandruvada * the [0, 1023] interval that thermal netlink uses. 184bd30cdfdSSrinivas Pandruvada */ 185bd30cdfdSSrinivas Pandruvada cpu_caps[i].performance = caps->perf_cap << 2; 186bd30cdfdSSrinivas Pandruvada cpu_caps[i].efficiency = caps->ee_cap << 2; 187bd30cdfdSSrinivas Pandruvada 188bd30cdfdSSrinivas Pandruvada ++i; 189bd30cdfdSSrinivas Pandruvada } 190bd30cdfdSSrinivas Pandruvada raw_spin_unlock_irq(&hfi_instance->table_lock); 191bd30cdfdSSrinivas Pandruvada } 192bd30cdfdSSrinivas Pandruvada 193bd30cdfdSSrinivas Pandruvada /* 194bd30cdfdSSrinivas Pandruvada * Call update_capabilities() when there are changes in the HFI table. 195bd30cdfdSSrinivas Pandruvada */ 196bd30cdfdSSrinivas Pandruvada static void update_capabilities(struct hfi_instance *hfi_instance) 197bd30cdfdSSrinivas Pandruvada { 198bd30cdfdSSrinivas Pandruvada struct thermal_genl_cpu_caps *cpu_caps; 199bd30cdfdSSrinivas Pandruvada int i = 0, cpu_count; 200bd30cdfdSSrinivas Pandruvada 201bd30cdfdSSrinivas Pandruvada /* CPUs may come online/offline while processing an HFI update. */ 202bd30cdfdSSrinivas Pandruvada mutex_lock(&hfi_instance_lock); 203bd30cdfdSSrinivas Pandruvada 204bd30cdfdSSrinivas Pandruvada cpu_count = cpumask_weight(hfi_instance->cpus); 205bd30cdfdSSrinivas Pandruvada 206bd30cdfdSSrinivas Pandruvada /* No CPUs to report in this hfi_instance. */ 207bd30cdfdSSrinivas Pandruvada if (!cpu_count) 208bd30cdfdSSrinivas Pandruvada goto out; 209bd30cdfdSSrinivas Pandruvada 210bd30cdfdSSrinivas Pandruvada cpu_caps = kcalloc(cpu_count, sizeof(*cpu_caps), GFP_KERNEL); 211bd30cdfdSSrinivas Pandruvada if (!cpu_caps) 212bd30cdfdSSrinivas Pandruvada goto out; 213bd30cdfdSSrinivas Pandruvada 214bd30cdfdSSrinivas Pandruvada get_hfi_caps(hfi_instance, cpu_caps); 215bd30cdfdSSrinivas Pandruvada 216bd30cdfdSSrinivas Pandruvada if (cpu_count < HFI_MAX_THERM_NOTIFY_COUNT) 217bd30cdfdSSrinivas Pandruvada goto last_cmd; 218bd30cdfdSSrinivas Pandruvada 219bd30cdfdSSrinivas Pandruvada /* Process complete chunks of HFI_MAX_THERM_NOTIFY_COUNT capabilities. */ 220bd30cdfdSSrinivas Pandruvada for (i = 0; 221bd30cdfdSSrinivas Pandruvada (i + HFI_MAX_THERM_NOTIFY_COUNT) <= cpu_count; 222bd30cdfdSSrinivas Pandruvada i += HFI_MAX_THERM_NOTIFY_COUNT) 223bd30cdfdSSrinivas Pandruvada thermal_genl_cpu_capability_event(HFI_MAX_THERM_NOTIFY_COUNT, 224bd30cdfdSSrinivas Pandruvada &cpu_caps[i]); 225bd30cdfdSSrinivas Pandruvada 226bd30cdfdSSrinivas Pandruvada cpu_count = cpu_count - i; 227bd30cdfdSSrinivas Pandruvada 228bd30cdfdSSrinivas Pandruvada last_cmd: 229bd30cdfdSSrinivas Pandruvada /* Process the remaining capabilities if any. */ 230bd30cdfdSSrinivas Pandruvada if (cpu_count) 231bd30cdfdSSrinivas Pandruvada thermal_genl_cpu_capability_event(cpu_count, &cpu_caps[i]); 232bd30cdfdSSrinivas Pandruvada 233bd30cdfdSSrinivas Pandruvada kfree(cpu_caps); 234bd30cdfdSSrinivas Pandruvada out: 235bd30cdfdSSrinivas Pandruvada mutex_unlock(&hfi_instance_lock); 236bd30cdfdSSrinivas Pandruvada } 237ab09b074SRicardo Neri 238ab09b074SRicardo Neri static void hfi_update_work_fn(struct work_struct *work) 239ab09b074SRicardo Neri { 240ab09b074SRicardo Neri struct hfi_instance *hfi_instance; 241ab09b074SRicardo Neri 242ab09b074SRicardo Neri hfi_instance = container_of(to_delayed_work(work), struct hfi_instance, 243ab09b074SRicardo Neri update_work); 244ab09b074SRicardo Neri 245bd30cdfdSSrinivas Pandruvada update_capabilities(hfi_instance); 246ab09b074SRicardo Neri } 247ab09b074SRicardo Neri 248ab09b074SRicardo Neri void intel_hfi_process_event(__u64 pkg_therm_status_msr_val) 249ab09b074SRicardo Neri { 250ab09b074SRicardo Neri struct hfi_instance *hfi_instance; 251ab09b074SRicardo Neri int cpu = smp_processor_id(); 252ab09b074SRicardo Neri struct hfi_cpu_info *info; 253c0e3acdcSSrinivas Pandruvada u64 new_timestamp, msr, hfi; 254ab09b074SRicardo Neri 255ab09b074SRicardo Neri if (!pkg_therm_status_msr_val) 256ab09b074SRicardo Neri return; 257ab09b074SRicardo Neri 258ab09b074SRicardo Neri info = &per_cpu(hfi_cpu_info, cpu); 259ab09b074SRicardo Neri if (!info) 260ab09b074SRicardo Neri return; 261ab09b074SRicardo Neri 262ab09b074SRicardo Neri /* 263ab09b074SRicardo Neri * A CPU is linked to its HFI instance before the thermal vector in the 264ab09b074SRicardo Neri * local APIC is unmasked. Hence, info->hfi_instance cannot be NULL 265ab09b074SRicardo Neri * when receiving an HFI event. 266ab09b074SRicardo Neri */ 267ab09b074SRicardo Neri hfi_instance = info->hfi_instance; 268ab09b074SRicardo Neri if (unlikely(!hfi_instance)) { 269ab09b074SRicardo Neri pr_debug("Received event on CPU %d but instance was null", cpu); 270ab09b074SRicardo Neri return; 271ab09b074SRicardo Neri } 272ab09b074SRicardo Neri 273ab09b074SRicardo Neri /* 274ab09b074SRicardo Neri * On most systems, all CPUs in the package receive a package-level 275ab09b074SRicardo Neri * thermal interrupt when there is an HFI update. It is sufficient to 276ab09b074SRicardo Neri * let a single CPU to acknowledge the update and queue work to 277ab09b074SRicardo Neri * process it. The remaining CPUs can resume their work. 278ab09b074SRicardo Neri */ 279ab09b074SRicardo Neri if (!raw_spin_trylock(&hfi_instance->event_lock)) 280ab09b074SRicardo Neri return; 281ab09b074SRicardo Neri 282c0e3acdcSSrinivas Pandruvada rdmsrl(MSR_IA32_PACKAGE_THERM_STATUS, msr); 283c0e3acdcSSrinivas Pandruvada hfi = msr & PACKAGE_THERM_STATUS_HFI_UPDATED; 284c0e3acdcSSrinivas Pandruvada if (!hfi) { 285c0e3acdcSSrinivas Pandruvada raw_spin_unlock(&hfi_instance->event_lock); 286c0e3acdcSSrinivas Pandruvada return; 287c0e3acdcSSrinivas Pandruvada } 288c0e3acdcSSrinivas Pandruvada 289c0e3acdcSSrinivas Pandruvada /* 290c0e3acdcSSrinivas Pandruvada * Ack duplicate update. Since there is an active HFI 291c0e3acdcSSrinivas Pandruvada * status from HW, it must be a new event, not a case 292c0e3acdcSSrinivas Pandruvada * where a lagging CPU entered the locked region. 293c0e3acdcSSrinivas Pandruvada */ 294ab09b074SRicardo Neri new_timestamp = *(u64 *)hfi_instance->hw_table; 295ab09b074SRicardo Neri if (*hfi_instance->timestamp == new_timestamp) { 296c0e3acdcSSrinivas Pandruvada thermal_clear_package_intr_status(PACKAGE_LEVEL, PACKAGE_THERM_STATUS_HFI_UPDATED); 297ab09b074SRicardo Neri raw_spin_unlock(&hfi_instance->event_lock); 298ab09b074SRicardo Neri return; 299ab09b074SRicardo Neri } 300ab09b074SRicardo Neri 301ab09b074SRicardo Neri raw_spin_lock(&hfi_instance->table_lock); 302ab09b074SRicardo Neri 303ab09b074SRicardo Neri /* 304ab09b074SRicardo Neri * Copy the updated table into our local copy. This includes the new 305ab09b074SRicardo Neri * timestamp. 306ab09b074SRicardo Neri */ 307ab09b074SRicardo Neri memcpy(hfi_instance->local_table, hfi_instance->hw_table, 308ab09b074SRicardo Neri hfi_features.nr_table_pages << PAGE_SHIFT); 309ab09b074SRicardo Neri 310ab09b074SRicardo Neri /* 311ab09b074SRicardo Neri * Let hardware know that we are done reading the HFI table and it is 312ab09b074SRicardo Neri * free to update it again. 313ab09b074SRicardo Neri */ 314930d06bfSSrinivas Pandruvada thermal_clear_package_intr_status(PACKAGE_LEVEL, PACKAGE_THERM_STATUS_HFI_UPDATED); 315ab09b074SRicardo Neri 316c0e3acdcSSrinivas Pandruvada raw_spin_unlock(&hfi_instance->table_lock); 317c0e3acdcSSrinivas Pandruvada raw_spin_unlock(&hfi_instance->event_lock); 318c0e3acdcSSrinivas Pandruvada 319ab09b074SRicardo Neri queue_delayed_work(hfi_updates_wq, &hfi_instance->update_work, 320ab09b074SRicardo Neri HFI_UPDATE_INTERVAL); 321ab09b074SRicardo Neri } 322ab09b074SRicardo Neri 3232d74e631SRicardo Neri static void init_hfi_cpu_index(struct hfi_cpu_info *info) 3242d74e631SRicardo Neri { 3252d74e631SRicardo Neri union cpuid6_edx edx; 3262d74e631SRicardo Neri 3272d74e631SRicardo Neri /* Do not re-read @cpu's index if it has already been initialized. */ 3282d74e631SRicardo Neri if (info->index > -1) 3292d74e631SRicardo Neri return; 3302d74e631SRicardo Neri 3312d74e631SRicardo Neri edx.full = cpuid_edx(CPUID_HFI_LEAF); 3322d74e631SRicardo Neri info->index = edx.split.index; 3332d74e631SRicardo Neri } 3342d74e631SRicardo Neri 3352d74e631SRicardo Neri /* 3362d74e631SRicardo Neri * The format of the HFI table depends on the number of capabilities that the 3372d74e631SRicardo Neri * hardware supports. Keep a data structure to navigate the table. 3382d74e631SRicardo Neri */ 3392d74e631SRicardo Neri static void init_hfi_instance(struct hfi_instance *hfi_instance) 3402d74e631SRicardo Neri { 3412d74e631SRicardo Neri /* The HFI header is below the time-stamp. */ 3422d74e631SRicardo Neri hfi_instance->hdr = hfi_instance->local_table + 3432d74e631SRicardo Neri sizeof(*hfi_instance->timestamp); 3442d74e631SRicardo Neri 3452d74e631SRicardo Neri /* The HFI data starts below the header. */ 3462d74e631SRicardo Neri hfi_instance->data = hfi_instance->hdr + hfi_features.hdr_size; 3472d74e631SRicardo Neri } 3482d74e631SRicardo Neri 3492d74e631SRicardo Neri /** 3502d74e631SRicardo Neri * intel_hfi_online() - Enable HFI on @cpu 3512d74e631SRicardo Neri * @cpu: CPU in which the HFI will be enabled 3522d74e631SRicardo Neri * 3532d74e631SRicardo Neri * Enable the HFI to be used in @cpu. The HFI is enabled at the die/package 3542d74e631SRicardo Neri * level. The first CPU in the die/package to come online does the full HFI 3552d74e631SRicardo Neri * initialization. Subsequent CPUs will just link themselves to the HFI 3562d74e631SRicardo Neri * instance of their die/package. 3572d74e631SRicardo Neri * 3582d74e631SRicardo Neri * This function is called before enabling the thermal vector in the local APIC 3592d74e631SRicardo Neri * in order to ensure that @cpu has an associated HFI instance when it receives 3602d74e631SRicardo Neri * an HFI event. 3612d74e631SRicardo Neri */ 3622d74e631SRicardo Neri void intel_hfi_online(unsigned int cpu) 3632d74e631SRicardo Neri { 3642d74e631SRicardo Neri struct hfi_instance *hfi_instance; 3652d74e631SRicardo Neri struct hfi_cpu_info *info; 3662d74e631SRicardo Neri phys_addr_t hw_table_pa; 3672d74e631SRicardo Neri u64 msr_val; 3682d74e631SRicardo Neri u16 die_id; 3692d74e631SRicardo Neri 3702d74e631SRicardo Neri /* Nothing to do if hfi_instances are missing. */ 3712d74e631SRicardo Neri if (!hfi_instances) 3722d74e631SRicardo Neri return; 3732d74e631SRicardo Neri 3742d74e631SRicardo Neri /* 3752d74e631SRicardo Neri * Link @cpu to the HFI instance of its package/die. It does not 3762d74e631SRicardo Neri * matter whether the instance has been initialized. 3772d74e631SRicardo Neri */ 3782d74e631SRicardo Neri info = &per_cpu(hfi_cpu_info, cpu); 3792d74e631SRicardo Neri die_id = topology_logical_die_id(cpu); 3802d74e631SRicardo Neri hfi_instance = info->hfi_instance; 3812d74e631SRicardo Neri if (!hfi_instance) { 382*3a3073b6SRicardo Neri if (die_id >= max_hfi_instances) 3832d74e631SRicardo Neri return; 3842d74e631SRicardo Neri 3852d74e631SRicardo Neri hfi_instance = &hfi_instances[die_id]; 3862d74e631SRicardo Neri info->hfi_instance = hfi_instance; 3872d74e631SRicardo Neri } 3882d74e631SRicardo Neri 3892d74e631SRicardo Neri init_hfi_cpu_index(info); 3902d74e631SRicardo Neri 3912d74e631SRicardo Neri /* 3922d74e631SRicardo Neri * Now check if the HFI instance of the package/die of @cpu has been 3932d74e631SRicardo Neri * initialized (by checking its header). In such case, all we have to 3942d74e631SRicardo Neri * do is to add @cpu to this instance's cpumask. 3952d74e631SRicardo Neri */ 3962d74e631SRicardo Neri mutex_lock(&hfi_instance_lock); 3972d74e631SRicardo Neri if (hfi_instance->hdr) { 3982d74e631SRicardo Neri cpumask_set_cpu(cpu, hfi_instance->cpus); 3992d74e631SRicardo Neri goto unlock; 4002d74e631SRicardo Neri } 4012d74e631SRicardo Neri 4022d74e631SRicardo Neri /* 4032d74e631SRicardo Neri * Hardware is programmed with the physical address of the first page 4042d74e631SRicardo Neri * frame of the table. Hence, the allocated memory must be page-aligned. 4052d74e631SRicardo Neri */ 4062d74e631SRicardo Neri hfi_instance->hw_table = alloc_pages_exact(hfi_features.nr_table_pages, 4072d74e631SRicardo Neri GFP_KERNEL | __GFP_ZERO); 4082d74e631SRicardo Neri if (!hfi_instance->hw_table) 4092d74e631SRicardo Neri goto unlock; 4102d74e631SRicardo Neri 4112d74e631SRicardo Neri hw_table_pa = virt_to_phys(hfi_instance->hw_table); 4122d74e631SRicardo Neri 4132d74e631SRicardo Neri /* 4142d74e631SRicardo Neri * Allocate memory to keep a local copy of the table that 4152d74e631SRicardo Neri * hardware generates. 4162d74e631SRicardo Neri */ 4172d74e631SRicardo Neri hfi_instance->local_table = kzalloc(hfi_features.nr_table_pages << PAGE_SHIFT, 4182d74e631SRicardo Neri GFP_KERNEL); 4192d74e631SRicardo Neri if (!hfi_instance->local_table) 4202d74e631SRicardo Neri goto free_hw_table; 4212d74e631SRicardo Neri 4222d74e631SRicardo Neri /* 4232d74e631SRicardo Neri * Program the address of the feedback table of this die/package. On 4242d74e631SRicardo Neri * some processors, hardware remembers the old address of the HFI table 4252d74e631SRicardo Neri * even after having been reprogrammed and re-enabled. Thus, do not free 4262d74e631SRicardo Neri * the pages allocated for the table or reprogram the hardware with a 4272d74e631SRicardo Neri * new base address. Namely, program the hardware only once. 4282d74e631SRicardo Neri */ 4292d74e631SRicardo Neri msr_val = hw_table_pa | HW_FEEDBACK_PTR_VALID_BIT; 4302d74e631SRicardo Neri wrmsrl(MSR_IA32_HW_FEEDBACK_PTR, msr_val); 4312d74e631SRicardo Neri 4322d74e631SRicardo Neri init_hfi_instance(hfi_instance); 4332d74e631SRicardo Neri 434ab09b074SRicardo Neri INIT_DELAYED_WORK(&hfi_instance->update_work, hfi_update_work_fn); 435ab09b074SRicardo Neri raw_spin_lock_init(&hfi_instance->table_lock); 436ab09b074SRicardo Neri raw_spin_lock_init(&hfi_instance->event_lock); 437ab09b074SRicardo Neri 4382d74e631SRicardo Neri cpumask_set_cpu(cpu, hfi_instance->cpus); 4392d74e631SRicardo Neri 440ab09b074SRicardo Neri /* 441ab09b074SRicardo Neri * Enable the hardware feedback interface and never disable it. See 442ab09b074SRicardo Neri * comment on programming the address of the table. 443ab09b074SRicardo Neri */ 444ab09b074SRicardo Neri rdmsrl(MSR_IA32_HW_FEEDBACK_CONFIG, msr_val); 445ab09b074SRicardo Neri msr_val |= HW_FEEDBACK_CONFIG_HFI_ENABLE_BIT; 446ab09b074SRicardo Neri wrmsrl(MSR_IA32_HW_FEEDBACK_CONFIG, msr_val); 447ab09b074SRicardo Neri 4482d74e631SRicardo Neri unlock: 4492d74e631SRicardo Neri mutex_unlock(&hfi_instance_lock); 4502d74e631SRicardo Neri return; 4512d74e631SRicardo Neri 4522d74e631SRicardo Neri free_hw_table: 4532d74e631SRicardo Neri free_pages_exact(hfi_instance->hw_table, hfi_features.nr_table_pages); 4542d74e631SRicardo Neri goto unlock; 4552d74e631SRicardo Neri } 4562d74e631SRicardo Neri 4572d74e631SRicardo Neri /** 4582d74e631SRicardo Neri * intel_hfi_offline() - Disable HFI on @cpu 4592d74e631SRicardo Neri * @cpu: CPU in which the HFI will be disabled 4602d74e631SRicardo Neri * 4612d74e631SRicardo Neri * Remove @cpu from those covered by its HFI instance. 4622d74e631SRicardo Neri * 4632d74e631SRicardo Neri * On some processors, hardware remembers previous programming settings even 4642d74e631SRicardo Neri * after being reprogrammed. Thus, keep HFI enabled even if all CPUs in the 4652d74e631SRicardo Neri * die/package of @cpu are offline. See note in intel_hfi_online(). 4662d74e631SRicardo Neri */ 4672d74e631SRicardo Neri void intel_hfi_offline(unsigned int cpu) 4682d74e631SRicardo Neri { 4692d74e631SRicardo Neri struct hfi_cpu_info *info = &per_cpu(hfi_cpu_info, cpu); 4702d74e631SRicardo Neri struct hfi_instance *hfi_instance; 4712d74e631SRicardo Neri 4722d74e631SRicardo Neri /* 4732d74e631SRicardo Neri * Check if @cpu as an associated, initialized (i.e., with a non-NULL 4742d74e631SRicardo Neri * header). Also, HFI instances are only initialized if X86_FEATURE_HFI 4752d74e631SRicardo Neri * is present. 4762d74e631SRicardo Neri */ 4772d74e631SRicardo Neri hfi_instance = info->hfi_instance; 4782d74e631SRicardo Neri if (!hfi_instance) 4792d74e631SRicardo Neri return; 4802d74e631SRicardo Neri 4812d74e631SRicardo Neri if (!hfi_instance->hdr) 4822d74e631SRicardo Neri return; 4832d74e631SRicardo Neri 4842d74e631SRicardo Neri mutex_lock(&hfi_instance_lock); 4852d74e631SRicardo Neri cpumask_clear_cpu(cpu, hfi_instance->cpus); 4862d74e631SRicardo Neri mutex_unlock(&hfi_instance_lock); 4872d74e631SRicardo Neri } 4881cb19cabSRicardo Neri 4891cb19cabSRicardo Neri static __init int hfi_parse_features(void) 4901cb19cabSRicardo Neri { 4911cb19cabSRicardo Neri unsigned int nr_capabilities; 4921cb19cabSRicardo Neri union cpuid6_edx edx; 4931cb19cabSRicardo Neri 4941cb19cabSRicardo Neri if (!boot_cpu_has(X86_FEATURE_HFI)) 4951cb19cabSRicardo Neri return -ENODEV; 4961cb19cabSRicardo Neri 4971cb19cabSRicardo Neri /* 4981cb19cabSRicardo Neri * If we are here we know that CPUID_HFI_LEAF exists. Parse the 4991cb19cabSRicardo Neri * supported capabilities and the size of the HFI table. 5001cb19cabSRicardo Neri */ 5011cb19cabSRicardo Neri edx.full = cpuid_edx(CPUID_HFI_LEAF); 5021cb19cabSRicardo Neri 5031cb19cabSRicardo Neri if (!edx.split.capabilities.split.performance) { 5041cb19cabSRicardo Neri pr_debug("Performance reporting not supported! Not using HFI\n"); 5051cb19cabSRicardo Neri return -ENODEV; 5061cb19cabSRicardo Neri } 5071cb19cabSRicardo Neri 5081cb19cabSRicardo Neri /* 5091cb19cabSRicardo Neri * The number of supported capabilities determines the number of 5101cb19cabSRicardo Neri * columns in the HFI table. Exclude the reserved bits. 5111cb19cabSRicardo Neri */ 5121cb19cabSRicardo Neri edx.split.capabilities.split.__reserved = 0; 5131cb19cabSRicardo Neri nr_capabilities = hweight8(edx.split.capabilities.bits); 5141cb19cabSRicardo Neri 5151cb19cabSRicardo Neri /* The number of 4KB pages required by the table */ 5161cb19cabSRicardo Neri hfi_features.nr_table_pages = edx.split.table_pages + 1; 5171cb19cabSRicardo Neri 5181cb19cabSRicardo Neri /* 5191cb19cabSRicardo Neri * The header contains change indications for each supported feature. 5201cb19cabSRicardo Neri * The size of the table header is rounded up to be a multiple of 8 5211cb19cabSRicardo Neri * bytes. 5221cb19cabSRicardo Neri */ 5231cb19cabSRicardo Neri hfi_features.hdr_size = DIV_ROUND_UP(nr_capabilities, 8) * 8; 5241cb19cabSRicardo Neri 5251cb19cabSRicardo Neri /* 5261cb19cabSRicardo Neri * Data of each logical processor is also rounded up to be a multiple 5271cb19cabSRicardo Neri * of 8 bytes. 5281cb19cabSRicardo Neri */ 5291cb19cabSRicardo Neri hfi_features.cpu_stride = DIV_ROUND_UP(nr_capabilities, 8) * 8; 5301cb19cabSRicardo Neri 5311cb19cabSRicardo Neri return 0; 5321cb19cabSRicardo Neri } 5331cb19cabSRicardo Neri 5341cb19cabSRicardo Neri void __init intel_hfi_init(void) 5351cb19cabSRicardo Neri { 5362d74e631SRicardo Neri struct hfi_instance *hfi_instance; 5372d74e631SRicardo Neri int i, j; 5382d74e631SRicardo Neri 5391cb19cabSRicardo Neri if (hfi_parse_features()) 5401cb19cabSRicardo Neri return; 5411cb19cabSRicardo Neri 5421cb19cabSRicardo Neri /* There is one HFI instance per die/package. */ 5431cb19cabSRicardo Neri max_hfi_instances = topology_max_packages() * 5441cb19cabSRicardo Neri topology_max_die_per_package(); 5451cb19cabSRicardo Neri 5461cb19cabSRicardo Neri /* 5471cb19cabSRicardo Neri * This allocation may fail. CPU hotplug callbacks must check 5481cb19cabSRicardo Neri * for a null pointer. 5491cb19cabSRicardo Neri */ 5501cb19cabSRicardo Neri hfi_instances = kcalloc(max_hfi_instances, sizeof(*hfi_instances), 5511cb19cabSRicardo Neri GFP_KERNEL); 5522d74e631SRicardo Neri if (!hfi_instances) 5532d74e631SRicardo Neri return; 5542d74e631SRicardo Neri 5552d74e631SRicardo Neri for (i = 0; i < max_hfi_instances; i++) { 5562d74e631SRicardo Neri hfi_instance = &hfi_instances[i]; 5572d74e631SRicardo Neri if (!zalloc_cpumask_var(&hfi_instance->cpus, GFP_KERNEL)) 5582d74e631SRicardo Neri goto err_nomem; 5592d74e631SRicardo Neri } 5602d74e631SRicardo Neri 561ab09b074SRicardo Neri hfi_updates_wq = create_singlethread_workqueue("hfi-updates"); 562ab09b074SRicardo Neri if (!hfi_updates_wq) 563ab09b074SRicardo Neri goto err_nomem; 564ab09b074SRicardo Neri 5652d74e631SRicardo Neri return; 5662d74e631SRicardo Neri 5672d74e631SRicardo Neri err_nomem: 5682d74e631SRicardo Neri for (j = 0; j < i; ++j) { 5692d74e631SRicardo Neri hfi_instance = &hfi_instances[j]; 5702d74e631SRicardo Neri free_cpumask_var(hfi_instance->cpus); 5712d74e631SRicardo Neri } 5722d74e631SRicardo Neri 5732d74e631SRicardo Neri kfree(hfi_instances); 5742d74e631SRicardo Neri hfi_instances = NULL; 5751cb19cabSRicardo Neri } 576