11cb19cabSRicardo Neri // SPDX-License-Identifier: GPL-2.0-only 21cb19cabSRicardo Neri /* 31cb19cabSRicardo Neri * Hardware Feedback Interface Driver 41cb19cabSRicardo Neri * 51cb19cabSRicardo Neri * Copyright (c) 2021, Intel Corporation. 61cb19cabSRicardo Neri * 71cb19cabSRicardo Neri * Authors: Aubrey Li <aubrey.li@linux.intel.com> 81cb19cabSRicardo Neri * Ricardo Neri <ricardo.neri-calderon@linux.intel.com> 91cb19cabSRicardo Neri * 101cb19cabSRicardo Neri * 111cb19cabSRicardo Neri * The Hardware Feedback Interface provides a performance and energy efficiency 121cb19cabSRicardo Neri * capability information for each CPU in the system. Depending on the processor 131cb19cabSRicardo Neri * model, hardware may periodically update these capabilities as a result of 141cb19cabSRicardo Neri * changes in the operating conditions (e.g., power limits or thermal 151cb19cabSRicardo Neri * constraints). On other processor models, there is a single HFI update 161cb19cabSRicardo Neri * at boot. 171cb19cabSRicardo Neri * 181cb19cabSRicardo Neri * This file provides functionality to process HFI updates and relay these 191cb19cabSRicardo Neri * updates to userspace. 201cb19cabSRicardo Neri */ 211cb19cabSRicardo Neri 221cb19cabSRicardo Neri #define pr_fmt(fmt) "intel-hfi: " fmt 231cb19cabSRicardo Neri 241cb19cabSRicardo Neri #include <linux/bitops.h> 251cb19cabSRicardo Neri #include <linux/cpufeature.h> 262d74e631SRicardo Neri #include <linux/cpumask.h> 272d74e631SRicardo Neri #include <linux/gfp.h> 282d74e631SRicardo Neri #include <linux/io.h> 29ab09b074SRicardo Neri #include <linux/kernel.h> 301cb19cabSRicardo Neri #include <linux/math.h> 312d74e631SRicardo Neri #include <linux/mutex.h> 322d74e631SRicardo Neri #include <linux/percpu-defs.h> 331cb19cabSRicardo Neri #include <linux/printk.h> 341cb19cabSRicardo Neri #include <linux/processor.h> 351cb19cabSRicardo Neri #include <linux/slab.h> 36ab09b074SRicardo Neri #include <linux/spinlock.h> 37ab09b074SRicardo Neri #include <linux/string.h> 381cb19cabSRicardo Neri #include <linux/topology.h> 39ab09b074SRicardo Neri #include <linux/workqueue.h> 401cb19cabSRicardo Neri 412d74e631SRicardo Neri #include <asm/msr.h> 422d74e631SRicardo Neri 431cb19cabSRicardo Neri #include "intel_hfi.h" 44930d06bfSSrinivas Pandruvada #include "thermal_interrupt.h" 45ab09b074SRicardo Neri 469272d2d4SDaniel Lezcano #include "../thermal_netlink.h" 479272d2d4SDaniel Lezcano 482d74e631SRicardo Neri /* Hardware Feedback Interface MSR configuration bits */ 492d74e631SRicardo Neri #define HW_FEEDBACK_PTR_VALID_BIT BIT(0) 50ab09b074SRicardo Neri #define HW_FEEDBACK_CONFIG_HFI_ENABLE_BIT BIT(0) 512d74e631SRicardo Neri 521cb19cabSRicardo Neri /* CPUID detection and enumeration definitions for HFI */ 531cb19cabSRicardo Neri 541cb19cabSRicardo Neri #define CPUID_HFI_LEAF 6 551cb19cabSRicardo Neri 561cb19cabSRicardo Neri union hfi_capabilities { 571cb19cabSRicardo Neri struct { 581cb19cabSRicardo Neri u8 performance:1; 591cb19cabSRicardo Neri u8 energy_efficiency:1; 601cb19cabSRicardo Neri u8 __reserved:6; 611cb19cabSRicardo Neri } split; 621cb19cabSRicardo Neri u8 bits; 631cb19cabSRicardo Neri }; 641cb19cabSRicardo Neri 651cb19cabSRicardo Neri union cpuid6_edx { 661cb19cabSRicardo Neri struct { 671cb19cabSRicardo Neri union hfi_capabilities capabilities; 681cb19cabSRicardo Neri u32 table_pages:4; 691cb19cabSRicardo Neri u32 __reserved:4; 701cb19cabSRicardo Neri s32 index:16; 711cb19cabSRicardo Neri } split; 721cb19cabSRicardo Neri u32 full; 731cb19cabSRicardo Neri }; 741cb19cabSRicardo Neri 751cb19cabSRicardo Neri /** 761cb19cabSRicardo Neri * struct hfi_cpu_data - HFI capabilities per CPU 771cb19cabSRicardo Neri * @perf_cap: Performance capability 781cb19cabSRicardo Neri * @ee_cap: Energy efficiency capability 791cb19cabSRicardo Neri * 801cb19cabSRicardo Neri * Capabilities of a logical processor in the HFI table. These capabilities are 811cb19cabSRicardo Neri * unitless. 821cb19cabSRicardo Neri */ 831cb19cabSRicardo Neri struct hfi_cpu_data { 841cb19cabSRicardo Neri u8 perf_cap; 851cb19cabSRicardo Neri u8 ee_cap; 861cb19cabSRicardo Neri } __packed; 871cb19cabSRicardo Neri 881cb19cabSRicardo Neri /** 891cb19cabSRicardo Neri * struct hfi_hdr - Header of the HFI table 901cb19cabSRicardo Neri * @perf_updated: Hardware updated performance capabilities 911cb19cabSRicardo Neri * @ee_updated: Hardware updated energy efficiency capabilities 921cb19cabSRicardo Neri * 931cb19cabSRicardo Neri * Properties of the data in an HFI table. 941cb19cabSRicardo Neri */ 951cb19cabSRicardo Neri struct hfi_hdr { 961cb19cabSRicardo Neri u8 perf_updated; 971cb19cabSRicardo Neri u8 ee_updated; 981cb19cabSRicardo Neri } __packed; 991cb19cabSRicardo Neri 1001cb19cabSRicardo Neri /** 1011cb19cabSRicardo Neri * struct hfi_instance - Representation of an HFI instance (i.e., a table) 1021cb19cabSRicardo Neri * @local_table: Base of the local copy of the HFI table 1031cb19cabSRicardo Neri * @timestamp: Timestamp of the last update of the local table. 1041cb19cabSRicardo Neri * Located at the base of the local table. 1051cb19cabSRicardo Neri * @hdr: Base address of the header of the local table 1061cb19cabSRicardo Neri * @data: Base address of the data of the local table 1072d74e631SRicardo Neri * @cpus: CPUs represented in this HFI table instance 1082d74e631SRicardo Neri * @hw_table: Pointer to the HFI table of this instance 109ab09b074SRicardo Neri * @update_work: Delayed work to process HFI updates 110ab09b074SRicardo Neri * @table_lock: Lock to protect acceses to the table of this instance 111ab09b074SRicardo Neri * @event_lock: Lock to process HFI interrupts 1121cb19cabSRicardo Neri * 1131cb19cabSRicardo Neri * A set of parameters to parse and navigate a specific HFI table. 1141cb19cabSRicardo Neri */ 1151cb19cabSRicardo Neri struct hfi_instance { 1161cb19cabSRicardo Neri union { 1171cb19cabSRicardo Neri void *local_table; 1181cb19cabSRicardo Neri u64 *timestamp; 1191cb19cabSRicardo Neri }; 1201cb19cabSRicardo Neri void *hdr; 1211cb19cabSRicardo Neri void *data; 1222d74e631SRicardo Neri cpumask_var_t cpus; 1232d74e631SRicardo Neri void *hw_table; 124ab09b074SRicardo Neri struct delayed_work update_work; 125ab09b074SRicardo Neri raw_spinlock_t table_lock; 126ab09b074SRicardo Neri raw_spinlock_t event_lock; 1271cb19cabSRicardo Neri }; 1281cb19cabSRicardo Neri 1291cb19cabSRicardo Neri /** 1301cb19cabSRicardo Neri * struct hfi_features - Supported HFI features 1311cb19cabSRicardo Neri * @nr_table_pages: Size of the HFI table in 4KB pages 1321cb19cabSRicardo Neri * @cpu_stride: Stride size to locate the capability data of a logical 1331cb19cabSRicardo Neri * processor within the table (i.e., row stride) 1341cb19cabSRicardo Neri * @hdr_size: Size of the table header 1351cb19cabSRicardo Neri * 1361cb19cabSRicardo Neri * Parameters and supported features that are common to all HFI instances 1371cb19cabSRicardo Neri */ 1381cb19cabSRicardo Neri struct hfi_features { 13954d9135cSRicardo Neri size_t nr_table_pages; 1401cb19cabSRicardo Neri unsigned int cpu_stride; 1411cb19cabSRicardo Neri unsigned int hdr_size; 1421cb19cabSRicardo Neri }; 1431cb19cabSRicardo Neri 1442d74e631SRicardo Neri /** 1452d74e631SRicardo Neri * struct hfi_cpu_info - Per-CPU attributes to consume HFI data 1462d74e631SRicardo Neri * @index: Row of this CPU in its HFI table 1472d74e631SRicardo Neri * @hfi_instance: Attributes of the HFI table to which this CPU belongs 1482d74e631SRicardo Neri * 1492d74e631SRicardo Neri * Parameters to link a logical processor to an HFI table and a row within it. 1502d74e631SRicardo Neri */ 1512d74e631SRicardo Neri struct hfi_cpu_info { 1522d74e631SRicardo Neri s16 index; 1532d74e631SRicardo Neri struct hfi_instance *hfi_instance; 1542d74e631SRicardo Neri }; 1552d74e631SRicardo Neri 1562d74e631SRicardo Neri static DEFINE_PER_CPU(struct hfi_cpu_info, hfi_cpu_info) = { .index = -1 }; 1572d74e631SRicardo Neri 1581cb19cabSRicardo Neri static int max_hfi_instances; 1591cb19cabSRicardo Neri static struct hfi_instance *hfi_instances; 1601cb19cabSRicardo Neri 1611cb19cabSRicardo Neri static struct hfi_features hfi_features; 1622d74e631SRicardo Neri static DEFINE_MUTEX(hfi_instance_lock); 1632d74e631SRicardo Neri 164ab09b074SRicardo Neri static struct workqueue_struct *hfi_updates_wq; 165ab09b074SRicardo Neri #define HFI_UPDATE_INTERVAL HZ 166bd30cdfdSSrinivas Pandruvada #define HFI_MAX_THERM_NOTIFY_COUNT 16 167bd30cdfdSSrinivas Pandruvada 168bd30cdfdSSrinivas Pandruvada static void get_hfi_caps(struct hfi_instance *hfi_instance, 169bd30cdfdSSrinivas Pandruvada struct thermal_genl_cpu_caps *cpu_caps) 170bd30cdfdSSrinivas Pandruvada { 171bd30cdfdSSrinivas Pandruvada int cpu, i = 0; 172bd30cdfdSSrinivas Pandruvada 173bd30cdfdSSrinivas Pandruvada raw_spin_lock_irq(&hfi_instance->table_lock); 174bd30cdfdSSrinivas Pandruvada for_each_cpu(cpu, hfi_instance->cpus) { 175bd30cdfdSSrinivas Pandruvada struct hfi_cpu_data *caps; 176bd30cdfdSSrinivas Pandruvada s16 index; 177bd30cdfdSSrinivas Pandruvada 178bd30cdfdSSrinivas Pandruvada index = per_cpu(hfi_cpu_info, cpu).index; 179bd30cdfdSSrinivas Pandruvada caps = hfi_instance->data + index * hfi_features.cpu_stride; 180bd30cdfdSSrinivas Pandruvada cpu_caps[i].cpu = cpu; 181bd30cdfdSSrinivas Pandruvada 182bd30cdfdSSrinivas Pandruvada /* 183bd30cdfdSSrinivas Pandruvada * Scale performance and energy efficiency to 184bd30cdfdSSrinivas Pandruvada * the [0, 1023] interval that thermal netlink uses. 185bd30cdfdSSrinivas Pandruvada */ 186bd30cdfdSSrinivas Pandruvada cpu_caps[i].performance = caps->perf_cap << 2; 187bd30cdfdSSrinivas Pandruvada cpu_caps[i].efficiency = caps->ee_cap << 2; 188bd30cdfdSSrinivas Pandruvada 189bd30cdfdSSrinivas Pandruvada ++i; 190bd30cdfdSSrinivas Pandruvada } 191bd30cdfdSSrinivas Pandruvada raw_spin_unlock_irq(&hfi_instance->table_lock); 192bd30cdfdSSrinivas Pandruvada } 193bd30cdfdSSrinivas Pandruvada 194bd30cdfdSSrinivas Pandruvada /* 195bd30cdfdSSrinivas Pandruvada * Call update_capabilities() when there are changes in the HFI table. 196bd30cdfdSSrinivas Pandruvada */ 197bd30cdfdSSrinivas Pandruvada static void update_capabilities(struct hfi_instance *hfi_instance) 198bd30cdfdSSrinivas Pandruvada { 199bd30cdfdSSrinivas Pandruvada struct thermal_genl_cpu_caps *cpu_caps; 200bd30cdfdSSrinivas Pandruvada int i = 0, cpu_count; 201bd30cdfdSSrinivas Pandruvada 202bd30cdfdSSrinivas Pandruvada /* CPUs may come online/offline while processing an HFI update. */ 203bd30cdfdSSrinivas Pandruvada mutex_lock(&hfi_instance_lock); 204bd30cdfdSSrinivas Pandruvada 205bd30cdfdSSrinivas Pandruvada cpu_count = cpumask_weight(hfi_instance->cpus); 206bd30cdfdSSrinivas Pandruvada 207bd30cdfdSSrinivas Pandruvada /* No CPUs to report in this hfi_instance. */ 208bd30cdfdSSrinivas Pandruvada if (!cpu_count) 209bd30cdfdSSrinivas Pandruvada goto out; 210bd30cdfdSSrinivas Pandruvada 211bd30cdfdSSrinivas Pandruvada cpu_caps = kcalloc(cpu_count, sizeof(*cpu_caps), GFP_KERNEL); 212bd30cdfdSSrinivas Pandruvada if (!cpu_caps) 213bd30cdfdSSrinivas Pandruvada goto out; 214bd30cdfdSSrinivas Pandruvada 215bd30cdfdSSrinivas Pandruvada get_hfi_caps(hfi_instance, cpu_caps); 216bd30cdfdSSrinivas Pandruvada 217bd30cdfdSSrinivas Pandruvada if (cpu_count < HFI_MAX_THERM_NOTIFY_COUNT) 218bd30cdfdSSrinivas Pandruvada goto last_cmd; 219bd30cdfdSSrinivas Pandruvada 220bd30cdfdSSrinivas Pandruvada /* Process complete chunks of HFI_MAX_THERM_NOTIFY_COUNT capabilities. */ 221bd30cdfdSSrinivas Pandruvada for (i = 0; 222bd30cdfdSSrinivas Pandruvada (i + HFI_MAX_THERM_NOTIFY_COUNT) <= cpu_count; 223bd30cdfdSSrinivas Pandruvada i += HFI_MAX_THERM_NOTIFY_COUNT) 224bd30cdfdSSrinivas Pandruvada thermal_genl_cpu_capability_event(HFI_MAX_THERM_NOTIFY_COUNT, 225bd30cdfdSSrinivas Pandruvada &cpu_caps[i]); 226bd30cdfdSSrinivas Pandruvada 227bd30cdfdSSrinivas Pandruvada cpu_count = cpu_count - i; 228bd30cdfdSSrinivas Pandruvada 229bd30cdfdSSrinivas Pandruvada last_cmd: 230bd30cdfdSSrinivas Pandruvada /* Process the remaining capabilities if any. */ 231bd30cdfdSSrinivas Pandruvada if (cpu_count) 232bd30cdfdSSrinivas Pandruvada thermal_genl_cpu_capability_event(cpu_count, &cpu_caps[i]); 233bd30cdfdSSrinivas Pandruvada 234bd30cdfdSSrinivas Pandruvada kfree(cpu_caps); 235bd30cdfdSSrinivas Pandruvada out: 236bd30cdfdSSrinivas Pandruvada mutex_unlock(&hfi_instance_lock); 237bd30cdfdSSrinivas Pandruvada } 238ab09b074SRicardo Neri 239ab09b074SRicardo Neri static void hfi_update_work_fn(struct work_struct *work) 240ab09b074SRicardo Neri { 241ab09b074SRicardo Neri struct hfi_instance *hfi_instance; 242ab09b074SRicardo Neri 243ab09b074SRicardo Neri hfi_instance = container_of(to_delayed_work(work), struct hfi_instance, 244ab09b074SRicardo Neri update_work); 245ab09b074SRicardo Neri 246bd30cdfdSSrinivas Pandruvada update_capabilities(hfi_instance); 247ab09b074SRicardo Neri } 248ab09b074SRicardo Neri 249ab09b074SRicardo Neri void intel_hfi_process_event(__u64 pkg_therm_status_msr_val) 250ab09b074SRicardo Neri { 251ab09b074SRicardo Neri struct hfi_instance *hfi_instance; 252ab09b074SRicardo Neri int cpu = smp_processor_id(); 253ab09b074SRicardo Neri struct hfi_cpu_info *info; 254c0e3acdcSSrinivas Pandruvada u64 new_timestamp, msr, hfi; 255ab09b074SRicardo Neri 256ab09b074SRicardo Neri if (!pkg_therm_status_msr_val) 257ab09b074SRicardo Neri return; 258ab09b074SRicardo Neri 259ab09b074SRicardo Neri info = &per_cpu(hfi_cpu_info, cpu); 260ab09b074SRicardo Neri if (!info) 261ab09b074SRicardo Neri return; 262ab09b074SRicardo Neri 263ab09b074SRicardo Neri /* 264ab09b074SRicardo Neri * A CPU is linked to its HFI instance before the thermal vector in the 265ab09b074SRicardo Neri * local APIC is unmasked. Hence, info->hfi_instance cannot be NULL 266ab09b074SRicardo Neri * when receiving an HFI event. 267ab09b074SRicardo Neri */ 268ab09b074SRicardo Neri hfi_instance = info->hfi_instance; 269ab09b074SRicardo Neri if (unlikely(!hfi_instance)) { 270ab09b074SRicardo Neri pr_debug("Received event on CPU %d but instance was null", cpu); 271ab09b074SRicardo Neri return; 272ab09b074SRicardo Neri } 273ab09b074SRicardo Neri 274ab09b074SRicardo Neri /* 275ab09b074SRicardo Neri * On most systems, all CPUs in the package receive a package-level 276ab09b074SRicardo Neri * thermal interrupt when there is an HFI update. It is sufficient to 277ab09b074SRicardo Neri * let a single CPU to acknowledge the update and queue work to 278ab09b074SRicardo Neri * process it. The remaining CPUs can resume their work. 279ab09b074SRicardo Neri */ 280ab09b074SRicardo Neri if (!raw_spin_trylock(&hfi_instance->event_lock)) 281ab09b074SRicardo Neri return; 282ab09b074SRicardo Neri 283c0e3acdcSSrinivas Pandruvada rdmsrl(MSR_IA32_PACKAGE_THERM_STATUS, msr); 284c0e3acdcSSrinivas Pandruvada hfi = msr & PACKAGE_THERM_STATUS_HFI_UPDATED; 285c0e3acdcSSrinivas Pandruvada if (!hfi) { 286c0e3acdcSSrinivas Pandruvada raw_spin_unlock(&hfi_instance->event_lock); 287c0e3acdcSSrinivas Pandruvada return; 288c0e3acdcSSrinivas Pandruvada } 289c0e3acdcSSrinivas Pandruvada 290c0e3acdcSSrinivas Pandruvada /* 291c0e3acdcSSrinivas Pandruvada * Ack duplicate update. Since there is an active HFI 292c0e3acdcSSrinivas Pandruvada * status from HW, it must be a new event, not a case 293c0e3acdcSSrinivas Pandruvada * where a lagging CPU entered the locked region. 294c0e3acdcSSrinivas Pandruvada */ 295ab09b074SRicardo Neri new_timestamp = *(u64 *)hfi_instance->hw_table; 296ab09b074SRicardo Neri if (*hfi_instance->timestamp == new_timestamp) { 297c0e3acdcSSrinivas Pandruvada thermal_clear_package_intr_status(PACKAGE_LEVEL, PACKAGE_THERM_STATUS_HFI_UPDATED); 298ab09b074SRicardo Neri raw_spin_unlock(&hfi_instance->event_lock); 299ab09b074SRicardo Neri return; 300ab09b074SRicardo Neri } 301ab09b074SRicardo Neri 302ab09b074SRicardo Neri raw_spin_lock(&hfi_instance->table_lock); 303ab09b074SRicardo Neri 304ab09b074SRicardo Neri /* 305ab09b074SRicardo Neri * Copy the updated table into our local copy. This includes the new 306ab09b074SRicardo Neri * timestamp. 307ab09b074SRicardo Neri */ 308ab09b074SRicardo Neri memcpy(hfi_instance->local_table, hfi_instance->hw_table, 309ab09b074SRicardo Neri hfi_features.nr_table_pages << PAGE_SHIFT); 310ab09b074SRicardo Neri 311ab09b074SRicardo Neri /* 312ab09b074SRicardo Neri * Let hardware know that we are done reading the HFI table and it is 313ab09b074SRicardo Neri * free to update it again. 314ab09b074SRicardo Neri */ 315930d06bfSSrinivas Pandruvada thermal_clear_package_intr_status(PACKAGE_LEVEL, PACKAGE_THERM_STATUS_HFI_UPDATED); 316ab09b074SRicardo Neri 317c0e3acdcSSrinivas Pandruvada raw_spin_unlock(&hfi_instance->table_lock); 318c0e3acdcSSrinivas Pandruvada raw_spin_unlock(&hfi_instance->event_lock); 319c0e3acdcSSrinivas Pandruvada 320ab09b074SRicardo Neri queue_delayed_work(hfi_updates_wq, &hfi_instance->update_work, 321ab09b074SRicardo Neri HFI_UPDATE_INTERVAL); 322ab09b074SRicardo Neri } 323ab09b074SRicardo Neri 3242d74e631SRicardo Neri static void init_hfi_cpu_index(struct hfi_cpu_info *info) 3252d74e631SRicardo Neri { 3262d74e631SRicardo Neri union cpuid6_edx edx; 3272d74e631SRicardo Neri 3282d74e631SRicardo Neri /* Do not re-read @cpu's index if it has already been initialized. */ 3292d74e631SRicardo Neri if (info->index > -1) 3302d74e631SRicardo Neri return; 3312d74e631SRicardo Neri 3322d74e631SRicardo Neri edx.full = cpuid_edx(CPUID_HFI_LEAF); 3332d74e631SRicardo Neri info->index = edx.split.index; 3342d74e631SRicardo Neri } 3352d74e631SRicardo Neri 3362d74e631SRicardo Neri /* 3372d74e631SRicardo Neri * The format of the HFI table depends on the number of capabilities that the 3382d74e631SRicardo Neri * hardware supports. Keep a data structure to navigate the table. 3392d74e631SRicardo Neri */ 3402d74e631SRicardo Neri static void init_hfi_instance(struct hfi_instance *hfi_instance) 3412d74e631SRicardo Neri { 3422d74e631SRicardo Neri /* The HFI header is below the time-stamp. */ 3432d74e631SRicardo Neri hfi_instance->hdr = hfi_instance->local_table + 3442d74e631SRicardo Neri sizeof(*hfi_instance->timestamp); 3452d74e631SRicardo Neri 3462d74e631SRicardo Neri /* The HFI data starts below the header. */ 3472d74e631SRicardo Neri hfi_instance->data = hfi_instance->hdr + hfi_features.hdr_size; 3482d74e631SRicardo Neri } 3492d74e631SRicardo Neri 350*de791353SRicardo Neri /* Caller must hold hfi_instance_lock. */ 351*de791353SRicardo Neri static void hfi_enable(void) 352*de791353SRicardo Neri { 353*de791353SRicardo Neri u64 msr_val; 354*de791353SRicardo Neri 355*de791353SRicardo Neri rdmsrl(MSR_IA32_HW_FEEDBACK_CONFIG, msr_val); 356*de791353SRicardo Neri msr_val |= HW_FEEDBACK_CONFIG_HFI_ENABLE_BIT; 357*de791353SRicardo Neri wrmsrl(MSR_IA32_HW_FEEDBACK_CONFIG, msr_val); 358*de791353SRicardo Neri } 359*de791353SRicardo Neri 360*de791353SRicardo Neri static void hfi_set_hw_table(struct hfi_instance *hfi_instance) 361*de791353SRicardo Neri { 362*de791353SRicardo Neri phys_addr_t hw_table_pa; 363*de791353SRicardo Neri u64 msr_val; 364*de791353SRicardo Neri 365*de791353SRicardo Neri hw_table_pa = virt_to_phys(hfi_instance->hw_table); 366*de791353SRicardo Neri msr_val = hw_table_pa | HW_FEEDBACK_PTR_VALID_BIT; 367*de791353SRicardo Neri wrmsrl(MSR_IA32_HW_FEEDBACK_PTR, msr_val); 368*de791353SRicardo Neri } 369*de791353SRicardo Neri 3702d74e631SRicardo Neri /** 3712d74e631SRicardo Neri * intel_hfi_online() - Enable HFI on @cpu 3722d74e631SRicardo Neri * @cpu: CPU in which the HFI will be enabled 3732d74e631SRicardo Neri * 3742d74e631SRicardo Neri * Enable the HFI to be used in @cpu. The HFI is enabled at the die/package 3752d74e631SRicardo Neri * level. The first CPU in the die/package to come online does the full HFI 3762d74e631SRicardo Neri * initialization. Subsequent CPUs will just link themselves to the HFI 3772d74e631SRicardo Neri * instance of their die/package. 3782d74e631SRicardo Neri * 3792d74e631SRicardo Neri * This function is called before enabling the thermal vector in the local APIC 3802d74e631SRicardo Neri * in order to ensure that @cpu has an associated HFI instance when it receives 3812d74e631SRicardo Neri * an HFI event. 3822d74e631SRicardo Neri */ 3832d74e631SRicardo Neri void intel_hfi_online(unsigned int cpu) 3842d74e631SRicardo Neri { 3852d74e631SRicardo Neri struct hfi_instance *hfi_instance; 3862d74e631SRicardo Neri struct hfi_cpu_info *info; 3872d74e631SRicardo Neri u16 die_id; 3882d74e631SRicardo Neri 3892d74e631SRicardo Neri /* Nothing to do if hfi_instances are missing. */ 3902d74e631SRicardo Neri if (!hfi_instances) 3912d74e631SRicardo Neri return; 3922d74e631SRicardo Neri 3932d74e631SRicardo Neri /* 3942d74e631SRicardo Neri * Link @cpu to the HFI instance of its package/die. It does not 3952d74e631SRicardo Neri * matter whether the instance has been initialized. 3962d74e631SRicardo Neri */ 3972d74e631SRicardo Neri info = &per_cpu(hfi_cpu_info, cpu); 3982d74e631SRicardo Neri die_id = topology_logical_die_id(cpu); 3992d74e631SRicardo Neri hfi_instance = info->hfi_instance; 4002d74e631SRicardo Neri if (!hfi_instance) { 4013a3073b6SRicardo Neri if (die_id >= max_hfi_instances) 4022d74e631SRicardo Neri return; 4032d74e631SRicardo Neri 4042d74e631SRicardo Neri hfi_instance = &hfi_instances[die_id]; 4052d74e631SRicardo Neri info->hfi_instance = hfi_instance; 4062d74e631SRicardo Neri } 4072d74e631SRicardo Neri 4082d74e631SRicardo Neri init_hfi_cpu_index(info); 4092d74e631SRicardo Neri 4102d74e631SRicardo Neri /* 4112d74e631SRicardo Neri * Now check if the HFI instance of the package/die of @cpu has been 4122d74e631SRicardo Neri * initialized (by checking its header). In such case, all we have to 4132d74e631SRicardo Neri * do is to add @cpu to this instance's cpumask. 4142d74e631SRicardo Neri */ 4152d74e631SRicardo Neri mutex_lock(&hfi_instance_lock); 4162d74e631SRicardo Neri if (hfi_instance->hdr) { 4172d74e631SRicardo Neri cpumask_set_cpu(cpu, hfi_instance->cpus); 4182d74e631SRicardo Neri goto unlock; 4192d74e631SRicardo Neri } 4202d74e631SRicardo Neri 4212d74e631SRicardo Neri /* 4222d74e631SRicardo Neri * Hardware is programmed with the physical address of the first page 4232d74e631SRicardo Neri * frame of the table. Hence, the allocated memory must be page-aligned. 4242d74e631SRicardo Neri */ 4252d74e631SRicardo Neri hfi_instance->hw_table = alloc_pages_exact(hfi_features.nr_table_pages, 4262d74e631SRicardo Neri GFP_KERNEL | __GFP_ZERO); 4272d74e631SRicardo Neri if (!hfi_instance->hw_table) 4282d74e631SRicardo Neri goto unlock; 4292d74e631SRicardo Neri 4302d74e631SRicardo Neri /* 4312d74e631SRicardo Neri * Allocate memory to keep a local copy of the table that 4322d74e631SRicardo Neri * hardware generates. 4332d74e631SRicardo Neri */ 4342d74e631SRicardo Neri hfi_instance->local_table = kzalloc(hfi_features.nr_table_pages << PAGE_SHIFT, 4352d74e631SRicardo Neri GFP_KERNEL); 4362d74e631SRicardo Neri if (!hfi_instance->local_table) 4372d74e631SRicardo Neri goto free_hw_table; 4382d74e631SRicardo Neri 4392d74e631SRicardo Neri init_hfi_instance(hfi_instance); 4402d74e631SRicardo Neri 441ab09b074SRicardo Neri INIT_DELAYED_WORK(&hfi_instance->update_work, hfi_update_work_fn); 442ab09b074SRicardo Neri raw_spin_lock_init(&hfi_instance->table_lock); 443ab09b074SRicardo Neri raw_spin_lock_init(&hfi_instance->event_lock); 444ab09b074SRicardo Neri 4452d74e631SRicardo Neri cpumask_set_cpu(cpu, hfi_instance->cpus); 4462d74e631SRicardo Neri 447*de791353SRicardo Neri hfi_set_hw_table(hfi_instance); 448*de791353SRicardo Neri hfi_enable(); 449ab09b074SRicardo Neri 4502d74e631SRicardo Neri unlock: 4512d74e631SRicardo Neri mutex_unlock(&hfi_instance_lock); 4522d74e631SRicardo Neri return; 4532d74e631SRicardo Neri 4542d74e631SRicardo Neri free_hw_table: 4552d74e631SRicardo Neri free_pages_exact(hfi_instance->hw_table, hfi_features.nr_table_pages); 4562d74e631SRicardo Neri goto unlock; 4572d74e631SRicardo Neri } 4582d74e631SRicardo Neri 4592d74e631SRicardo Neri /** 4602d74e631SRicardo Neri * intel_hfi_offline() - Disable HFI on @cpu 4612d74e631SRicardo Neri * @cpu: CPU in which the HFI will be disabled 4622d74e631SRicardo Neri * 4632d74e631SRicardo Neri * Remove @cpu from those covered by its HFI instance. 4642d74e631SRicardo Neri * 4652d74e631SRicardo Neri * On some processors, hardware remembers previous programming settings even 4662d74e631SRicardo Neri * after being reprogrammed. Thus, keep HFI enabled even if all CPUs in the 4672d74e631SRicardo Neri * die/package of @cpu are offline. See note in intel_hfi_online(). 4682d74e631SRicardo Neri */ 4692d74e631SRicardo Neri void intel_hfi_offline(unsigned int cpu) 4702d74e631SRicardo Neri { 4712d74e631SRicardo Neri struct hfi_cpu_info *info = &per_cpu(hfi_cpu_info, cpu); 4722d74e631SRicardo Neri struct hfi_instance *hfi_instance; 4732d74e631SRicardo Neri 4742d74e631SRicardo Neri /* 4752d74e631SRicardo Neri * Check if @cpu as an associated, initialized (i.e., with a non-NULL 4762d74e631SRicardo Neri * header). Also, HFI instances are only initialized if X86_FEATURE_HFI 4772d74e631SRicardo Neri * is present. 4782d74e631SRicardo Neri */ 4792d74e631SRicardo Neri hfi_instance = info->hfi_instance; 4802d74e631SRicardo Neri if (!hfi_instance) 4812d74e631SRicardo Neri return; 4822d74e631SRicardo Neri 4832d74e631SRicardo Neri if (!hfi_instance->hdr) 4842d74e631SRicardo Neri return; 4852d74e631SRicardo Neri 4862d74e631SRicardo Neri mutex_lock(&hfi_instance_lock); 4872d74e631SRicardo Neri cpumask_clear_cpu(cpu, hfi_instance->cpus); 4882d74e631SRicardo Neri mutex_unlock(&hfi_instance_lock); 4892d74e631SRicardo Neri } 4901cb19cabSRicardo Neri 4911cb19cabSRicardo Neri static __init int hfi_parse_features(void) 4921cb19cabSRicardo Neri { 4931cb19cabSRicardo Neri unsigned int nr_capabilities; 4941cb19cabSRicardo Neri union cpuid6_edx edx; 4951cb19cabSRicardo Neri 4961cb19cabSRicardo Neri if (!boot_cpu_has(X86_FEATURE_HFI)) 4971cb19cabSRicardo Neri return -ENODEV; 4981cb19cabSRicardo Neri 4991cb19cabSRicardo Neri /* 5001cb19cabSRicardo Neri * If we are here we know that CPUID_HFI_LEAF exists. Parse the 5011cb19cabSRicardo Neri * supported capabilities and the size of the HFI table. 5021cb19cabSRicardo Neri */ 5031cb19cabSRicardo Neri edx.full = cpuid_edx(CPUID_HFI_LEAF); 5041cb19cabSRicardo Neri 5051cb19cabSRicardo Neri if (!edx.split.capabilities.split.performance) { 5061cb19cabSRicardo Neri pr_debug("Performance reporting not supported! Not using HFI\n"); 5071cb19cabSRicardo Neri return -ENODEV; 5081cb19cabSRicardo Neri } 5091cb19cabSRicardo Neri 5101cb19cabSRicardo Neri /* 5111cb19cabSRicardo Neri * The number of supported capabilities determines the number of 5121cb19cabSRicardo Neri * columns in the HFI table. Exclude the reserved bits. 5131cb19cabSRicardo Neri */ 5141cb19cabSRicardo Neri edx.split.capabilities.split.__reserved = 0; 5151cb19cabSRicardo Neri nr_capabilities = hweight8(edx.split.capabilities.bits); 5161cb19cabSRicardo Neri 5171cb19cabSRicardo Neri /* The number of 4KB pages required by the table */ 5181cb19cabSRicardo Neri hfi_features.nr_table_pages = edx.split.table_pages + 1; 5191cb19cabSRicardo Neri 5201cb19cabSRicardo Neri /* 5211cb19cabSRicardo Neri * The header contains change indications for each supported feature. 5221cb19cabSRicardo Neri * The size of the table header is rounded up to be a multiple of 8 5231cb19cabSRicardo Neri * bytes. 5241cb19cabSRicardo Neri */ 5251cb19cabSRicardo Neri hfi_features.hdr_size = DIV_ROUND_UP(nr_capabilities, 8) * 8; 5261cb19cabSRicardo Neri 5271cb19cabSRicardo Neri /* 5281cb19cabSRicardo Neri * Data of each logical processor is also rounded up to be a multiple 5291cb19cabSRicardo Neri * of 8 bytes. 5301cb19cabSRicardo Neri */ 5311cb19cabSRicardo Neri hfi_features.cpu_stride = DIV_ROUND_UP(nr_capabilities, 8) * 8; 5321cb19cabSRicardo Neri 5331cb19cabSRicardo Neri return 0; 5341cb19cabSRicardo Neri } 5351cb19cabSRicardo Neri 5361cb19cabSRicardo Neri void __init intel_hfi_init(void) 5371cb19cabSRicardo Neri { 5382d74e631SRicardo Neri struct hfi_instance *hfi_instance; 5392d74e631SRicardo Neri int i, j; 5402d74e631SRicardo Neri 5411cb19cabSRicardo Neri if (hfi_parse_features()) 5421cb19cabSRicardo Neri return; 5431cb19cabSRicardo Neri 5441cb19cabSRicardo Neri /* There is one HFI instance per die/package. */ 5451cb19cabSRicardo Neri max_hfi_instances = topology_max_packages() * 5461cb19cabSRicardo Neri topology_max_die_per_package(); 5471cb19cabSRicardo Neri 5481cb19cabSRicardo Neri /* 5491cb19cabSRicardo Neri * This allocation may fail. CPU hotplug callbacks must check 5501cb19cabSRicardo Neri * for a null pointer. 5511cb19cabSRicardo Neri */ 5521cb19cabSRicardo Neri hfi_instances = kcalloc(max_hfi_instances, sizeof(*hfi_instances), 5531cb19cabSRicardo Neri GFP_KERNEL); 5542d74e631SRicardo Neri if (!hfi_instances) 5552d74e631SRicardo Neri return; 5562d74e631SRicardo Neri 5572d74e631SRicardo Neri for (i = 0; i < max_hfi_instances; i++) { 5582d74e631SRicardo Neri hfi_instance = &hfi_instances[i]; 5592d74e631SRicardo Neri if (!zalloc_cpumask_var(&hfi_instance->cpus, GFP_KERNEL)) 5602d74e631SRicardo Neri goto err_nomem; 5612d74e631SRicardo Neri } 5622d74e631SRicardo Neri 563ab09b074SRicardo Neri hfi_updates_wq = create_singlethread_workqueue("hfi-updates"); 564ab09b074SRicardo Neri if (!hfi_updates_wq) 565ab09b074SRicardo Neri goto err_nomem; 566ab09b074SRicardo Neri 5672d74e631SRicardo Neri return; 5682d74e631SRicardo Neri 5692d74e631SRicardo Neri err_nomem: 5702d74e631SRicardo Neri for (j = 0; j < i; ++j) { 5712d74e631SRicardo Neri hfi_instance = &hfi_instances[j]; 5722d74e631SRicardo Neri free_cpumask_var(hfi_instance->cpus); 5732d74e631SRicardo Neri } 5742d74e631SRicardo Neri 5752d74e631SRicardo Neri kfree(hfi_instances); 5762d74e631SRicardo Neri hfi_instances = NULL; 5771cb19cabSRicardo Neri } 578