11cb19cabSRicardo Neri // SPDX-License-Identifier: GPL-2.0-only 21cb19cabSRicardo Neri /* 31cb19cabSRicardo Neri * Hardware Feedback Interface Driver 41cb19cabSRicardo Neri * 51cb19cabSRicardo Neri * Copyright (c) 2021, Intel Corporation. 61cb19cabSRicardo Neri * 71cb19cabSRicardo Neri * Authors: Aubrey Li <aubrey.li@linux.intel.com> 81cb19cabSRicardo Neri * Ricardo Neri <ricardo.neri-calderon@linux.intel.com> 91cb19cabSRicardo Neri * 101cb19cabSRicardo Neri * 111cb19cabSRicardo Neri * The Hardware Feedback Interface provides a performance and energy efficiency 121cb19cabSRicardo Neri * capability information for each CPU in the system. Depending on the processor 131cb19cabSRicardo Neri * model, hardware may periodically update these capabilities as a result of 141cb19cabSRicardo Neri * changes in the operating conditions (e.g., power limits or thermal 151cb19cabSRicardo Neri * constraints). On other processor models, there is a single HFI update 161cb19cabSRicardo Neri * at boot. 171cb19cabSRicardo Neri * 181cb19cabSRicardo Neri * This file provides functionality to process HFI updates and relay these 191cb19cabSRicardo Neri * updates to userspace. 201cb19cabSRicardo Neri */ 211cb19cabSRicardo Neri 221cb19cabSRicardo Neri #define pr_fmt(fmt) "intel-hfi: " fmt 231cb19cabSRicardo Neri 241cb19cabSRicardo Neri #include <linux/bitops.h> 251cb19cabSRicardo Neri #include <linux/cpufeature.h> 262d74e631SRicardo Neri #include <linux/cpumask.h> 27*0caf5dd0SRicardo Neri #include <linux/delay.h> 282d74e631SRicardo Neri #include <linux/gfp.h> 292d74e631SRicardo Neri #include <linux/io.h> 30ab09b074SRicardo Neri #include <linux/kernel.h> 311cb19cabSRicardo Neri #include <linux/math.h> 322d74e631SRicardo Neri #include <linux/mutex.h> 332d74e631SRicardo Neri #include <linux/percpu-defs.h> 341cb19cabSRicardo Neri #include <linux/printk.h> 351cb19cabSRicardo Neri #include <linux/processor.h> 361cb19cabSRicardo Neri #include <linux/slab.h> 37ab09b074SRicardo Neri #include <linux/spinlock.h> 38ab09b074SRicardo Neri #include <linux/string.h> 391cb19cabSRicardo Neri #include <linux/topology.h> 40ab09b074SRicardo Neri #include <linux/workqueue.h> 411cb19cabSRicardo Neri 422d74e631SRicardo Neri #include <asm/msr.h> 432d74e631SRicardo Neri 441cb19cabSRicardo Neri #include "intel_hfi.h" 45930d06bfSSrinivas Pandruvada #include "thermal_interrupt.h" 46ab09b074SRicardo Neri 479272d2d4SDaniel Lezcano #include "../thermal_netlink.h" 489272d2d4SDaniel Lezcano 492d74e631SRicardo Neri /* Hardware Feedback Interface MSR configuration bits */ 502d74e631SRicardo Neri #define HW_FEEDBACK_PTR_VALID_BIT BIT(0) 51ab09b074SRicardo Neri #define HW_FEEDBACK_CONFIG_HFI_ENABLE_BIT BIT(0) 522d74e631SRicardo Neri 531cb19cabSRicardo Neri /* CPUID detection and enumeration definitions for HFI */ 541cb19cabSRicardo Neri 551cb19cabSRicardo Neri #define CPUID_HFI_LEAF 6 561cb19cabSRicardo Neri 571cb19cabSRicardo Neri union hfi_capabilities { 581cb19cabSRicardo Neri struct { 591cb19cabSRicardo Neri u8 performance:1; 601cb19cabSRicardo Neri u8 energy_efficiency:1; 611cb19cabSRicardo Neri u8 __reserved:6; 621cb19cabSRicardo Neri } split; 631cb19cabSRicardo Neri u8 bits; 641cb19cabSRicardo Neri }; 651cb19cabSRicardo Neri 661cb19cabSRicardo Neri union cpuid6_edx { 671cb19cabSRicardo Neri struct { 681cb19cabSRicardo Neri union hfi_capabilities capabilities; 691cb19cabSRicardo Neri u32 table_pages:4; 701cb19cabSRicardo Neri u32 __reserved:4; 711cb19cabSRicardo Neri s32 index:16; 721cb19cabSRicardo Neri } split; 731cb19cabSRicardo Neri u32 full; 741cb19cabSRicardo Neri }; 751cb19cabSRicardo Neri 761cb19cabSRicardo Neri /** 771cb19cabSRicardo Neri * struct hfi_cpu_data - HFI capabilities per CPU 781cb19cabSRicardo Neri * @perf_cap: Performance capability 791cb19cabSRicardo Neri * @ee_cap: Energy efficiency capability 801cb19cabSRicardo Neri * 811cb19cabSRicardo Neri * Capabilities of a logical processor in the HFI table. These capabilities are 821cb19cabSRicardo Neri * unitless. 831cb19cabSRicardo Neri */ 841cb19cabSRicardo Neri struct hfi_cpu_data { 851cb19cabSRicardo Neri u8 perf_cap; 861cb19cabSRicardo Neri u8 ee_cap; 871cb19cabSRicardo Neri } __packed; 881cb19cabSRicardo Neri 891cb19cabSRicardo Neri /** 901cb19cabSRicardo Neri * struct hfi_hdr - Header of the HFI table 911cb19cabSRicardo Neri * @perf_updated: Hardware updated performance capabilities 921cb19cabSRicardo Neri * @ee_updated: Hardware updated energy efficiency capabilities 931cb19cabSRicardo Neri * 941cb19cabSRicardo Neri * Properties of the data in an HFI table. 951cb19cabSRicardo Neri */ 961cb19cabSRicardo Neri struct hfi_hdr { 971cb19cabSRicardo Neri u8 perf_updated; 981cb19cabSRicardo Neri u8 ee_updated; 991cb19cabSRicardo Neri } __packed; 1001cb19cabSRicardo Neri 1011cb19cabSRicardo Neri /** 1021cb19cabSRicardo Neri * struct hfi_instance - Representation of an HFI instance (i.e., a table) 1031cb19cabSRicardo Neri * @local_table: Base of the local copy of the HFI table 1041cb19cabSRicardo Neri * @timestamp: Timestamp of the last update of the local table. 1051cb19cabSRicardo Neri * Located at the base of the local table. 1061cb19cabSRicardo Neri * @hdr: Base address of the header of the local table 1071cb19cabSRicardo Neri * @data: Base address of the data of the local table 1082d74e631SRicardo Neri * @cpus: CPUs represented in this HFI table instance 1092d74e631SRicardo Neri * @hw_table: Pointer to the HFI table of this instance 110ab09b074SRicardo Neri * @update_work: Delayed work to process HFI updates 111ab09b074SRicardo Neri * @table_lock: Lock to protect acceses to the table of this instance 112ab09b074SRicardo Neri * @event_lock: Lock to process HFI interrupts 1131cb19cabSRicardo Neri * 1141cb19cabSRicardo Neri * A set of parameters to parse and navigate a specific HFI table. 1151cb19cabSRicardo Neri */ 1161cb19cabSRicardo Neri struct hfi_instance { 1171cb19cabSRicardo Neri union { 1181cb19cabSRicardo Neri void *local_table; 1191cb19cabSRicardo Neri u64 *timestamp; 1201cb19cabSRicardo Neri }; 1211cb19cabSRicardo Neri void *hdr; 1221cb19cabSRicardo Neri void *data; 1232d74e631SRicardo Neri cpumask_var_t cpus; 1242d74e631SRicardo Neri void *hw_table; 125ab09b074SRicardo Neri struct delayed_work update_work; 126ab09b074SRicardo Neri raw_spinlock_t table_lock; 127ab09b074SRicardo Neri raw_spinlock_t event_lock; 1281cb19cabSRicardo Neri }; 1291cb19cabSRicardo Neri 1301cb19cabSRicardo Neri /** 1311cb19cabSRicardo Neri * struct hfi_features - Supported HFI features 1321cb19cabSRicardo Neri * @nr_table_pages: Size of the HFI table in 4KB pages 1331cb19cabSRicardo Neri * @cpu_stride: Stride size to locate the capability data of a logical 1341cb19cabSRicardo Neri * processor within the table (i.e., row stride) 1351cb19cabSRicardo Neri * @hdr_size: Size of the table header 1361cb19cabSRicardo Neri * 1371cb19cabSRicardo Neri * Parameters and supported features that are common to all HFI instances 1381cb19cabSRicardo Neri */ 1391cb19cabSRicardo Neri struct hfi_features { 14054d9135cSRicardo Neri size_t nr_table_pages; 1411cb19cabSRicardo Neri unsigned int cpu_stride; 1421cb19cabSRicardo Neri unsigned int hdr_size; 1431cb19cabSRicardo Neri }; 1441cb19cabSRicardo Neri 1452d74e631SRicardo Neri /** 1462d74e631SRicardo Neri * struct hfi_cpu_info - Per-CPU attributes to consume HFI data 1472d74e631SRicardo Neri * @index: Row of this CPU in its HFI table 1482d74e631SRicardo Neri * @hfi_instance: Attributes of the HFI table to which this CPU belongs 1492d74e631SRicardo Neri * 1502d74e631SRicardo Neri * Parameters to link a logical processor to an HFI table and a row within it. 1512d74e631SRicardo Neri */ 1522d74e631SRicardo Neri struct hfi_cpu_info { 1532d74e631SRicardo Neri s16 index; 1542d74e631SRicardo Neri struct hfi_instance *hfi_instance; 1552d74e631SRicardo Neri }; 1562d74e631SRicardo Neri 1572d74e631SRicardo Neri static DEFINE_PER_CPU(struct hfi_cpu_info, hfi_cpu_info) = { .index = -1 }; 1582d74e631SRicardo Neri 1591cb19cabSRicardo Neri static int max_hfi_instances; 1601cb19cabSRicardo Neri static struct hfi_instance *hfi_instances; 1611cb19cabSRicardo Neri 1621cb19cabSRicardo Neri static struct hfi_features hfi_features; 1632d74e631SRicardo Neri static DEFINE_MUTEX(hfi_instance_lock); 1642d74e631SRicardo Neri 165ab09b074SRicardo Neri static struct workqueue_struct *hfi_updates_wq; 166ab09b074SRicardo Neri #define HFI_UPDATE_INTERVAL HZ 167bd30cdfdSSrinivas Pandruvada #define HFI_MAX_THERM_NOTIFY_COUNT 16 168bd30cdfdSSrinivas Pandruvada 169bd30cdfdSSrinivas Pandruvada static void get_hfi_caps(struct hfi_instance *hfi_instance, 170bd30cdfdSSrinivas Pandruvada struct thermal_genl_cpu_caps *cpu_caps) 171bd30cdfdSSrinivas Pandruvada { 172bd30cdfdSSrinivas Pandruvada int cpu, i = 0; 173bd30cdfdSSrinivas Pandruvada 174bd30cdfdSSrinivas Pandruvada raw_spin_lock_irq(&hfi_instance->table_lock); 175bd30cdfdSSrinivas Pandruvada for_each_cpu(cpu, hfi_instance->cpus) { 176bd30cdfdSSrinivas Pandruvada struct hfi_cpu_data *caps; 177bd30cdfdSSrinivas Pandruvada s16 index; 178bd30cdfdSSrinivas Pandruvada 179bd30cdfdSSrinivas Pandruvada index = per_cpu(hfi_cpu_info, cpu).index; 180bd30cdfdSSrinivas Pandruvada caps = hfi_instance->data + index * hfi_features.cpu_stride; 181bd30cdfdSSrinivas Pandruvada cpu_caps[i].cpu = cpu; 182bd30cdfdSSrinivas Pandruvada 183bd30cdfdSSrinivas Pandruvada /* 184bd30cdfdSSrinivas Pandruvada * Scale performance and energy efficiency to 185bd30cdfdSSrinivas Pandruvada * the [0, 1023] interval that thermal netlink uses. 186bd30cdfdSSrinivas Pandruvada */ 187bd30cdfdSSrinivas Pandruvada cpu_caps[i].performance = caps->perf_cap << 2; 188bd30cdfdSSrinivas Pandruvada cpu_caps[i].efficiency = caps->ee_cap << 2; 189bd30cdfdSSrinivas Pandruvada 190bd30cdfdSSrinivas Pandruvada ++i; 191bd30cdfdSSrinivas Pandruvada } 192bd30cdfdSSrinivas Pandruvada raw_spin_unlock_irq(&hfi_instance->table_lock); 193bd30cdfdSSrinivas Pandruvada } 194bd30cdfdSSrinivas Pandruvada 195bd30cdfdSSrinivas Pandruvada /* 196bd30cdfdSSrinivas Pandruvada * Call update_capabilities() when there are changes in the HFI table. 197bd30cdfdSSrinivas Pandruvada */ 198bd30cdfdSSrinivas Pandruvada static void update_capabilities(struct hfi_instance *hfi_instance) 199bd30cdfdSSrinivas Pandruvada { 200bd30cdfdSSrinivas Pandruvada struct thermal_genl_cpu_caps *cpu_caps; 201bd30cdfdSSrinivas Pandruvada int i = 0, cpu_count; 202bd30cdfdSSrinivas Pandruvada 203bd30cdfdSSrinivas Pandruvada /* CPUs may come online/offline while processing an HFI update. */ 204bd30cdfdSSrinivas Pandruvada mutex_lock(&hfi_instance_lock); 205bd30cdfdSSrinivas Pandruvada 206bd30cdfdSSrinivas Pandruvada cpu_count = cpumask_weight(hfi_instance->cpus); 207bd30cdfdSSrinivas Pandruvada 208bd30cdfdSSrinivas Pandruvada /* No CPUs to report in this hfi_instance. */ 209bd30cdfdSSrinivas Pandruvada if (!cpu_count) 210bd30cdfdSSrinivas Pandruvada goto out; 211bd30cdfdSSrinivas Pandruvada 212bd30cdfdSSrinivas Pandruvada cpu_caps = kcalloc(cpu_count, sizeof(*cpu_caps), GFP_KERNEL); 213bd30cdfdSSrinivas Pandruvada if (!cpu_caps) 214bd30cdfdSSrinivas Pandruvada goto out; 215bd30cdfdSSrinivas Pandruvada 216bd30cdfdSSrinivas Pandruvada get_hfi_caps(hfi_instance, cpu_caps); 217bd30cdfdSSrinivas Pandruvada 218bd30cdfdSSrinivas Pandruvada if (cpu_count < HFI_MAX_THERM_NOTIFY_COUNT) 219bd30cdfdSSrinivas Pandruvada goto last_cmd; 220bd30cdfdSSrinivas Pandruvada 221bd30cdfdSSrinivas Pandruvada /* Process complete chunks of HFI_MAX_THERM_NOTIFY_COUNT capabilities. */ 222bd30cdfdSSrinivas Pandruvada for (i = 0; 223bd30cdfdSSrinivas Pandruvada (i + HFI_MAX_THERM_NOTIFY_COUNT) <= cpu_count; 224bd30cdfdSSrinivas Pandruvada i += HFI_MAX_THERM_NOTIFY_COUNT) 225bd30cdfdSSrinivas Pandruvada thermal_genl_cpu_capability_event(HFI_MAX_THERM_NOTIFY_COUNT, 226bd30cdfdSSrinivas Pandruvada &cpu_caps[i]); 227bd30cdfdSSrinivas Pandruvada 228bd30cdfdSSrinivas Pandruvada cpu_count = cpu_count - i; 229bd30cdfdSSrinivas Pandruvada 230bd30cdfdSSrinivas Pandruvada last_cmd: 231bd30cdfdSSrinivas Pandruvada /* Process the remaining capabilities if any. */ 232bd30cdfdSSrinivas Pandruvada if (cpu_count) 233bd30cdfdSSrinivas Pandruvada thermal_genl_cpu_capability_event(cpu_count, &cpu_caps[i]); 234bd30cdfdSSrinivas Pandruvada 235bd30cdfdSSrinivas Pandruvada kfree(cpu_caps); 236bd30cdfdSSrinivas Pandruvada out: 237bd30cdfdSSrinivas Pandruvada mutex_unlock(&hfi_instance_lock); 238bd30cdfdSSrinivas Pandruvada } 239ab09b074SRicardo Neri 240ab09b074SRicardo Neri static void hfi_update_work_fn(struct work_struct *work) 241ab09b074SRicardo Neri { 242ab09b074SRicardo Neri struct hfi_instance *hfi_instance; 243ab09b074SRicardo Neri 244ab09b074SRicardo Neri hfi_instance = container_of(to_delayed_work(work), struct hfi_instance, 245ab09b074SRicardo Neri update_work); 246ab09b074SRicardo Neri 247bd30cdfdSSrinivas Pandruvada update_capabilities(hfi_instance); 248ab09b074SRicardo Neri } 249ab09b074SRicardo Neri 250ab09b074SRicardo Neri void intel_hfi_process_event(__u64 pkg_therm_status_msr_val) 251ab09b074SRicardo Neri { 252ab09b074SRicardo Neri struct hfi_instance *hfi_instance; 253ab09b074SRicardo Neri int cpu = smp_processor_id(); 254ab09b074SRicardo Neri struct hfi_cpu_info *info; 255c0e3acdcSSrinivas Pandruvada u64 new_timestamp, msr, hfi; 256ab09b074SRicardo Neri 257ab09b074SRicardo Neri if (!pkg_therm_status_msr_val) 258ab09b074SRicardo Neri return; 259ab09b074SRicardo Neri 260ab09b074SRicardo Neri info = &per_cpu(hfi_cpu_info, cpu); 261ab09b074SRicardo Neri if (!info) 262ab09b074SRicardo Neri return; 263ab09b074SRicardo Neri 264ab09b074SRicardo Neri /* 265ab09b074SRicardo Neri * A CPU is linked to its HFI instance before the thermal vector in the 266ab09b074SRicardo Neri * local APIC is unmasked. Hence, info->hfi_instance cannot be NULL 267ab09b074SRicardo Neri * when receiving an HFI event. 268ab09b074SRicardo Neri */ 269ab09b074SRicardo Neri hfi_instance = info->hfi_instance; 270ab09b074SRicardo Neri if (unlikely(!hfi_instance)) { 271ab09b074SRicardo Neri pr_debug("Received event on CPU %d but instance was null", cpu); 272ab09b074SRicardo Neri return; 273ab09b074SRicardo Neri } 274ab09b074SRicardo Neri 275ab09b074SRicardo Neri /* 276ab09b074SRicardo Neri * On most systems, all CPUs in the package receive a package-level 277ab09b074SRicardo Neri * thermal interrupt when there is an HFI update. It is sufficient to 278ab09b074SRicardo Neri * let a single CPU to acknowledge the update and queue work to 279ab09b074SRicardo Neri * process it. The remaining CPUs can resume their work. 280ab09b074SRicardo Neri */ 281ab09b074SRicardo Neri if (!raw_spin_trylock(&hfi_instance->event_lock)) 282ab09b074SRicardo Neri return; 283ab09b074SRicardo Neri 284c0e3acdcSSrinivas Pandruvada rdmsrl(MSR_IA32_PACKAGE_THERM_STATUS, msr); 285c0e3acdcSSrinivas Pandruvada hfi = msr & PACKAGE_THERM_STATUS_HFI_UPDATED; 286c0e3acdcSSrinivas Pandruvada if (!hfi) { 287c0e3acdcSSrinivas Pandruvada raw_spin_unlock(&hfi_instance->event_lock); 288c0e3acdcSSrinivas Pandruvada return; 289c0e3acdcSSrinivas Pandruvada } 290c0e3acdcSSrinivas Pandruvada 291c0e3acdcSSrinivas Pandruvada /* 292c0e3acdcSSrinivas Pandruvada * Ack duplicate update. Since there is an active HFI 293c0e3acdcSSrinivas Pandruvada * status from HW, it must be a new event, not a case 294c0e3acdcSSrinivas Pandruvada * where a lagging CPU entered the locked region. 295c0e3acdcSSrinivas Pandruvada */ 296ab09b074SRicardo Neri new_timestamp = *(u64 *)hfi_instance->hw_table; 297ab09b074SRicardo Neri if (*hfi_instance->timestamp == new_timestamp) { 298c0e3acdcSSrinivas Pandruvada thermal_clear_package_intr_status(PACKAGE_LEVEL, PACKAGE_THERM_STATUS_HFI_UPDATED); 299ab09b074SRicardo Neri raw_spin_unlock(&hfi_instance->event_lock); 300ab09b074SRicardo Neri return; 301ab09b074SRicardo Neri } 302ab09b074SRicardo Neri 303ab09b074SRicardo Neri raw_spin_lock(&hfi_instance->table_lock); 304ab09b074SRicardo Neri 305ab09b074SRicardo Neri /* 306ab09b074SRicardo Neri * Copy the updated table into our local copy. This includes the new 307ab09b074SRicardo Neri * timestamp. 308ab09b074SRicardo Neri */ 309ab09b074SRicardo Neri memcpy(hfi_instance->local_table, hfi_instance->hw_table, 310ab09b074SRicardo Neri hfi_features.nr_table_pages << PAGE_SHIFT); 311ab09b074SRicardo Neri 312ab09b074SRicardo Neri /* 313ab09b074SRicardo Neri * Let hardware know that we are done reading the HFI table and it is 314ab09b074SRicardo Neri * free to update it again. 315ab09b074SRicardo Neri */ 316930d06bfSSrinivas Pandruvada thermal_clear_package_intr_status(PACKAGE_LEVEL, PACKAGE_THERM_STATUS_HFI_UPDATED); 317ab09b074SRicardo Neri 318c0e3acdcSSrinivas Pandruvada raw_spin_unlock(&hfi_instance->table_lock); 319c0e3acdcSSrinivas Pandruvada raw_spin_unlock(&hfi_instance->event_lock); 320c0e3acdcSSrinivas Pandruvada 321ab09b074SRicardo Neri queue_delayed_work(hfi_updates_wq, &hfi_instance->update_work, 322ab09b074SRicardo Neri HFI_UPDATE_INTERVAL); 323ab09b074SRicardo Neri } 324ab09b074SRicardo Neri 3252d74e631SRicardo Neri static void init_hfi_cpu_index(struct hfi_cpu_info *info) 3262d74e631SRicardo Neri { 3272d74e631SRicardo Neri union cpuid6_edx edx; 3282d74e631SRicardo Neri 3292d74e631SRicardo Neri /* Do not re-read @cpu's index if it has already been initialized. */ 3302d74e631SRicardo Neri if (info->index > -1) 3312d74e631SRicardo Neri return; 3322d74e631SRicardo Neri 3332d74e631SRicardo Neri edx.full = cpuid_edx(CPUID_HFI_LEAF); 3342d74e631SRicardo Neri info->index = edx.split.index; 3352d74e631SRicardo Neri } 3362d74e631SRicardo Neri 3372d74e631SRicardo Neri /* 3382d74e631SRicardo Neri * The format of the HFI table depends on the number of capabilities that the 3392d74e631SRicardo Neri * hardware supports. Keep a data structure to navigate the table. 3402d74e631SRicardo Neri */ 3412d74e631SRicardo Neri static void init_hfi_instance(struct hfi_instance *hfi_instance) 3422d74e631SRicardo Neri { 3432d74e631SRicardo Neri /* The HFI header is below the time-stamp. */ 3442d74e631SRicardo Neri hfi_instance->hdr = hfi_instance->local_table + 3452d74e631SRicardo Neri sizeof(*hfi_instance->timestamp); 3462d74e631SRicardo Neri 3472d74e631SRicardo Neri /* The HFI data starts below the header. */ 3482d74e631SRicardo Neri hfi_instance->data = hfi_instance->hdr + hfi_features.hdr_size; 3492d74e631SRicardo Neri } 3502d74e631SRicardo Neri 351de791353SRicardo Neri /* Caller must hold hfi_instance_lock. */ 352de791353SRicardo Neri static void hfi_enable(void) 353de791353SRicardo Neri { 354de791353SRicardo Neri u64 msr_val; 355de791353SRicardo Neri 356de791353SRicardo Neri rdmsrl(MSR_IA32_HW_FEEDBACK_CONFIG, msr_val); 357de791353SRicardo Neri msr_val |= HW_FEEDBACK_CONFIG_HFI_ENABLE_BIT; 358de791353SRicardo Neri wrmsrl(MSR_IA32_HW_FEEDBACK_CONFIG, msr_val); 359de791353SRicardo Neri } 360de791353SRicardo Neri 361de791353SRicardo Neri static void hfi_set_hw_table(struct hfi_instance *hfi_instance) 362de791353SRicardo Neri { 363de791353SRicardo Neri phys_addr_t hw_table_pa; 364de791353SRicardo Neri u64 msr_val; 365de791353SRicardo Neri 366de791353SRicardo Neri hw_table_pa = virt_to_phys(hfi_instance->hw_table); 367de791353SRicardo Neri msr_val = hw_table_pa | HW_FEEDBACK_PTR_VALID_BIT; 368de791353SRicardo Neri wrmsrl(MSR_IA32_HW_FEEDBACK_PTR, msr_val); 369de791353SRicardo Neri } 370de791353SRicardo Neri 371*0caf5dd0SRicardo Neri /* Caller must hold hfi_instance_lock. */ 372*0caf5dd0SRicardo Neri static void hfi_disable(void) 373*0caf5dd0SRicardo Neri { 374*0caf5dd0SRicardo Neri u64 msr_val; 375*0caf5dd0SRicardo Neri int i; 376*0caf5dd0SRicardo Neri 377*0caf5dd0SRicardo Neri rdmsrl(MSR_IA32_HW_FEEDBACK_CONFIG, msr_val); 378*0caf5dd0SRicardo Neri msr_val &= ~HW_FEEDBACK_CONFIG_HFI_ENABLE_BIT; 379*0caf5dd0SRicardo Neri wrmsrl(MSR_IA32_HW_FEEDBACK_CONFIG, msr_val); 380*0caf5dd0SRicardo Neri 381*0caf5dd0SRicardo Neri /* 382*0caf5dd0SRicardo Neri * Wait for hardware to acknowledge the disabling of HFI. Some 383*0caf5dd0SRicardo Neri * processors may not do it. Wait for ~2ms. This is a reasonable 384*0caf5dd0SRicardo Neri * time for hardware to complete any pending actions on the HFI 385*0caf5dd0SRicardo Neri * memory. 386*0caf5dd0SRicardo Neri */ 387*0caf5dd0SRicardo Neri for (i = 0; i < 2000; i++) { 388*0caf5dd0SRicardo Neri rdmsrl(MSR_IA32_PACKAGE_THERM_STATUS, msr_val); 389*0caf5dd0SRicardo Neri if (msr_val & PACKAGE_THERM_STATUS_HFI_UPDATED) 390*0caf5dd0SRicardo Neri break; 391*0caf5dd0SRicardo Neri 392*0caf5dd0SRicardo Neri udelay(1); 393*0caf5dd0SRicardo Neri cpu_relax(); 394*0caf5dd0SRicardo Neri } 395*0caf5dd0SRicardo Neri } 396*0caf5dd0SRicardo Neri 3972d74e631SRicardo Neri /** 3982d74e631SRicardo Neri * intel_hfi_online() - Enable HFI on @cpu 3992d74e631SRicardo Neri * @cpu: CPU in which the HFI will be enabled 4002d74e631SRicardo Neri * 4012d74e631SRicardo Neri * Enable the HFI to be used in @cpu. The HFI is enabled at the die/package 4022d74e631SRicardo Neri * level. The first CPU in the die/package to come online does the full HFI 4032d74e631SRicardo Neri * initialization. Subsequent CPUs will just link themselves to the HFI 4042d74e631SRicardo Neri * instance of their die/package. 4052d74e631SRicardo Neri * 4062d74e631SRicardo Neri * This function is called before enabling the thermal vector in the local APIC 4072d74e631SRicardo Neri * in order to ensure that @cpu has an associated HFI instance when it receives 4082d74e631SRicardo Neri * an HFI event. 4092d74e631SRicardo Neri */ 4102d74e631SRicardo Neri void intel_hfi_online(unsigned int cpu) 4112d74e631SRicardo Neri { 4122d74e631SRicardo Neri struct hfi_instance *hfi_instance; 4132d74e631SRicardo Neri struct hfi_cpu_info *info; 4142d74e631SRicardo Neri u16 die_id; 4152d74e631SRicardo Neri 4162d74e631SRicardo Neri /* Nothing to do if hfi_instances are missing. */ 4172d74e631SRicardo Neri if (!hfi_instances) 4182d74e631SRicardo Neri return; 4192d74e631SRicardo Neri 4202d74e631SRicardo Neri /* 4212d74e631SRicardo Neri * Link @cpu to the HFI instance of its package/die. It does not 4222d74e631SRicardo Neri * matter whether the instance has been initialized. 4232d74e631SRicardo Neri */ 4242d74e631SRicardo Neri info = &per_cpu(hfi_cpu_info, cpu); 4252d74e631SRicardo Neri die_id = topology_logical_die_id(cpu); 4262d74e631SRicardo Neri hfi_instance = info->hfi_instance; 4272d74e631SRicardo Neri if (!hfi_instance) { 4283a3073b6SRicardo Neri if (die_id >= max_hfi_instances) 4292d74e631SRicardo Neri return; 4302d74e631SRicardo Neri 4312d74e631SRicardo Neri hfi_instance = &hfi_instances[die_id]; 4322d74e631SRicardo Neri info->hfi_instance = hfi_instance; 4332d74e631SRicardo Neri } 4342d74e631SRicardo Neri 4352d74e631SRicardo Neri init_hfi_cpu_index(info); 4362d74e631SRicardo Neri 4372d74e631SRicardo Neri /* 4382d74e631SRicardo Neri * Now check if the HFI instance of the package/die of @cpu has been 4392d74e631SRicardo Neri * initialized (by checking its header). In such case, all we have to 4402d74e631SRicardo Neri * do is to add @cpu to this instance's cpumask. 4412d74e631SRicardo Neri */ 4422d74e631SRicardo Neri mutex_lock(&hfi_instance_lock); 4432d74e631SRicardo Neri if (hfi_instance->hdr) { 4442d74e631SRicardo Neri cpumask_set_cpu(cpu, hfi_instance->cpus); 4452d74e631SRicardo Neri goto unlock; 4462d74e631SRicardo Neri } 4472d74e631SRicardo Neri 4482d74e631SRicardo Neri /* 4492d74e631SRicardo Neri * Hardware is programmed with the physical address of the first page 4502d74e631SRicardo Neri * frame of the table. Hence, the allocated memory must be page-aligned. 451*0caf5dd0SRicardo Neri * 452*0caf5dd0SRicardo Neri * Some processors do not forget the initial address of the HFI table 453*0caf5dd0SRicardo Neri * even after having been reprogrammed. Keep using the same pages. Do 454*0caf5dd0SRicardo Neri * not free them. 4552d74e631SRicardo Neri */ 4562d74e631SRicardo Neri hfi_instance->hw_table = alloc_pages_exact(hfi_features.nr_table_pages, 4572d74e631SRicardo Neri GFP_KERNEL | __GFP_ZERO); 4582d74e631SRicardo Neri if (!hfi_instance->hw_table) 4592d74e631SRicardo Neri goto unlock; 4602d74e631SRicardo Neri 4612d74e631SRicardo Neri /* 4622d74e631SRicardo Neri * Allocate memory to keep a local copy of the table that 4632d74e631SRicardo Neri * hardware generates. 4642d74e631SRicardo Neri */ 4652d74e631SRicardo Neri hfi_instance->local_table = kzalloc(hfi_features.nr_table_pages << PAGE_SHIFT, 4662d74e631SRicardo Neri GFP_KERNEL); 4672d74e631SRicardo Neri if (!hfi_instance->local_table) 4682d74e631SRicardo Neri goto free_hw_table; 4692d74e631SRicardo Neri 4702d74e631SRicardo Neri init_hfi_instance(hfi_instance); 4712d74e631SRicardo Neri 472ab09b074SRicardo Neri INIT_DELAYED_WORK(&hfi_instance->update_work, hfi_update_work_fn); 473ab09b074SRicardo Neri raw_spin_lock_init(&hfi_instance->table_lock); 474ab09b074SRicardo Neri raw_spin_lock_init(&hfi_instance->event_lock); 475ab09b074SRicardo Neri 4762d74e631SRicardo Neri cpumask_set_cpu(cpu, hfi_instance->cpus); 4772d74e631SRicardo Neri 478de791353SRicardo Neri hfi_set_hw_table(hfi_instance); 479de791353SRicardo Neri hfi_enable(); 480ab09b074SRicardo Neri 4812d74e631SRicardo Neri unlock: 4822d74e631SRicardo Neri mutex_unlock(&hfi_instance_lock); 4832d74e631SRicardo Neri return; 4842d74e631SRicardo Neri 4852d74e631SRicardo Neri free_hw_table: 4862d74e631SRicardo Neri free_pages_exact(hfi_instance->hw_table, hfi_features.nr_table_pages); 4872d74e631SRicardo Neri goto unlock; 4882d74e631SRicardo Neri } 4892d74e631SRicardo Neri 4902d74e631SRicardo Neri /** 4912d74e631SRicardo Neri * intel_hfi_offline() - Disable HFI on @cpu 4922d74e631SRicardo Neri * @cpu: CPU in which the HFI will be disabled 4932d74e631SRicardo Neri * 4942d74e631SRicardo Neri * Remove @cpu from those covered by its HFI instance. 4952d74e631SRicardo Neri * 4962d74e631SRicardo Neri * On some processors, hardware remembers previous programming settings even 4972d74e631SRicardo Neri * after being reprogrammed. Thus, keep HFI enabled even if all CPUs in the 4982d74e631SRicardo Neri * die/package of @cpu are offline. See note in intel_hfi_online(). 4992d74e631SRicardo Neri */ 5002d74e631SRicardo Neri void intel_hfi_offline(unsigned int cpu) 5012d74e631SRicardo Neri { 5022d74e631SRicardo Neri struct hfi_cpu_info *info = &per_cpu(hfi_cpu_info, cpu); 5032d74e631SRicardo Neri struct hfi_instance *hfi_instance; 5042d74e631SRicardo Neri 5052d74e631SRicardo Neri /* 5062d74e631SRicardo Neri * Check if @cpu as an associated, initialized (i.e., with a non-NULL 5072d74e631SRicardo Neri * header). Also, HFI instances are only initialized if X86_FEATURE_HFI 5082d74e631SRicardo Neri * is present. 5092d74e631SRicardo Neri */ 5102d74e631SRicardo Neri hfi_instance = info->hfi_instance; 5112d74e631SRicardo Neri if (!hfi_instance) 5122d74e631SRicardo Neri return; 5132d74e631SRicardo Neri 5142d74e631SRicardo Neri if (!hfi_instance->hdr) 5152d74e631SRicardo Neri return; 5162d74e631SRicardo Neri 5172d74e631SRicardo Neri mutex_lock(&hfi_instance_lock); 5182d74e631SRicardo Neri cpumask_clear_cpu(cpu, hfi_instance->cpus); 519*0caf5dd0SRicardo Neri 520*0caf5dd0SRicardo Neri if (!cpumask_weight(hfi_instance->cpus)) 521*0caf5dd0SRicardo Neri hfi_disable(); 522*0caf5dd0SRicardo Neri 5232d74e631SRicardo Neri mutex_unlock(&hfi_instance_lock); 5242d74e631SRicardo Neri } 5251cb19cabSRicardo Neri 5261cb19cabSRicardo Neri static __init int hfi_parse_features(void) 5271cb19cabSRicardo Neri { 5281cb19cabSRicardo Neri unsigned int nr_capabilities; 5291cb19cabSRicardo Neri union cpuid6_edx edx; 5301cb19cabSRicardo Neri 5311cb19cabSRicardo Neri if (!boot_cpu_has(X86_FEATURE_HFI)) 5321cb19cabSRicardo Neri return -ENODEV; 5331cb19cabSRicardo Neri 5341cb19cabSRicardo Neri /* 5351cb19cabSRicardo Neri * If we are here we know that CPUID_HFI_LEAF exists. Parse the 5361cb19cabSRicardo Neri * supported capabilities and the size of the HFI table. 5371cb19cabSRicardo Neri */ 5381cb19cabSRicardo Neri edx.full = cpuid_edx(CPUID_HFI_LEAF); 5391cb19cabSRicardo Neri 5401cb19cabSRicardo Neri if (!edx.split.capabilities.split.performance) { 5411cb19cabSRicardo Neri pr_debug("Performance reporting not supported! Not using HFI\n"); 5421cb19cabSRicardo Neri return -ENODEV; 5431cb19cabSRicardo Neri } 5441cb19cabSRicardo Neri 5451cb19cabSRicardo Neri /* 5461cb19cabSRicardo Neri * The number of supported capabilities determines the number of 5471cb19cabSRicardo Neri * columns in the HFI table. Exclude the reserved bits. 5481cb19cabSRicardo Neri */ 5491cb19cabSRicardo Neri edx.split.capabilities.split.__reserved = 0; 5501cb19cabSRicardo Neri nr_capabilities = hweight8(edx.split.capabilities.bits); 5511cb19cabSRicardo Neri 5521cb19cabSRicardo Neri /* The number of 4KB pages required by the table */ 5531cb19cabSRicardo Neri hfi_features.nr_table_pages = edx.split.table_pages + 1; 5541cb19cabSRicardo Neri 5551cb19cabSRicardo Neri /* 5561cb19cabSRicardo Neri * The header contains change indications for each supported feature. 5571cb19cabSRicardo Neri * The size of the table header is rounded up to be a multiple of 8 5581cb19cabSRicardo Neri * bytes. 5591cb19cabSRicardo Neri */ 5601cb19cabSRicardo Neri hfi_features.hdr_size = DIV_ROUND_UP(nr_capabilities, 8) * 8; 5611cb19cabSRicardo Neri 5621cb19cabSRicardo Neri /* 5631cb19cabSRicardo Neri * Data of each logical processor is also rounded up to be a multiple 5641cb19cabSRicardo Neri * of 8 bytes. 5651cb19cabSRicardo Neri */ 5661cb19cabSRicardo Neri hfi_features.cpu_stride = DIV_ROUND_UP(nr_capabilities, 8) * 8; 5671cb19cabSRicardo Neri 5681cb19cabSRicardo Neri return 0; 5691cb19cabSRicardo Neri } 5701cb19cabSRicardo Neri 5711cb19cabSRicardo Neri void __init intel_hfi_init(void) 5721cb19cabSRicardo Neri { 5732d74e631SRicardo Neri struct hfi_instance *hfi_instance; 5742d74e631SRicardo Neri int i, j; 5752d74e631SRicardo Neri 5761cb19cabSRicardo Neri if (hfi_parse_features()) 5771cb19cabSRicardo Neri return; 5781cb19cabSRicardo Neri 5791cb19cabSRicardo Neri /* There is one HFI instance per die/package. */ 5801cb19cabSRicardo Neri max_hfi_instances = topology_max_packages() * 5811cb19cabSRicardo Neri topology_max_die_per_package(); 5821cb19cabSRicardo Neri 5831cb19cabSRicardo Neri /* 5841cb19cabSRicardo Neri * This allocation may fail. CPU hotplug callbacks must check 5851cb19cabSRicardo Neri * for a null pointer. 5861cb19cabSRicardo Neri */ 5871cb19cabSRicardo Neri hfi_instances = kcalloc(max_hfi_instances, sizeof(*hfi_instances), 5881cb19cabSRicardo Neri GFP_KERNEL); 5892d74e631SRicardo Neri if (!hfi_instances) 5902d74e631SRicardo Neri return; 5912d74e631SRicardo Neri 5922d74e631SRicardo Neri for (i = 0; i < max_hfi_instances; i++) { 5932d74e631SRicardo Neri hfi_instance = &hfi_instances[i]; 5942d74e631SRicardo Neri if (!zalloc_cpumask_var(&hfi_instance->cpus, GFP_KERNEL)) 5952d74e631SRicardo Neri goto err_nomem; 5962d74e631SRicardo Neri } 5972d74e631SRicardo Neri 598ab09b074SRicardo Neri hfi_updates_wq = create_singlethread_workqueue("hfi-updates"); 599ab09b074SRicardo Neri if (!hfi_updates_wq) 600ab09b074SRicardo Neri goto err_nomem; 601ab09b074SRicardo Neri 6022d74e631SRicardo Neri return; 6032d74e631SRicardo Neri 6042d74e631SRicardo Neri err_nomem: 6052d74e631SRicardo Neri for (j = 0; j < i; ++j) { 6062d74e631SRicardo Neri hfi_instance = &hfi_instances[j]; 6072d74e631SRicardo Neri free_cpumask_var(hfi_instance->cpus); 6082d74e631SRicardo Neri } 6092d74e631SRicardo Neri 6102d74e631SRicardo Neri kfree(hfi_instances); 6112d74e631SRicardo Neri hfi_instances = NULL; 6121cb19cabSRicardo Neri } 613