1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Hardware Feedback Interface Driver 4 * 5 * Copyright (c) 2021, Intel Corporation. 6 * 7 * Authors: Aubrey Li <aubrey.li@linux.intel.com> 8 * Ricardo Neri <ricardo.neri-calderon@linux.intel.com> 9 * 10 * 11 * The Hardware Feedback Interface provides a performance and energy efficiency 12 * capability information for each CPU in the system. Depending on the processor 13 * model, hardware may periodically update these capabilities as a result of 14 * changes in the operating conditions (e.g., power limits or thermal 15 * constraints). On other processor models, there is a single HFI update 16 * at boot. 17 * 18 * This file provides functionality to process HFI updates and relay these 19 * updates to userspace. 20 */ 21 22 #define pr_fmt(fmt) "intel-hfi: " fmt 23 24 #include <linux/bitops.h> 25 #include <linux/cpufeature.h> 26 #include <linux/cpumask.h> 27 #include <linux/gfp.h> 28 #include <linux/io.h> 29 #include <linux/kernel.h> 30 #include <linux/math.h> 31 #include <linux/mutex.h> 32 #include <linux/percpu-defs.h> 33 #include <linux/printk.h> 34 #include <linux/processor.h> 35 #include <linux/slab.h> 36 #include <linux/spinlock.h> 37 #include <linux/string.h> 38 #include <linux/topology.h> 39 #include <linux/workqueue.h> 40 41 #include <asm/msr.h> 42 43 #include "../thermal_core.h" 44 #include "intel_hfi.h" 45 46 #define THERM_STATUS_CLEAR_PKG_MASK (BIT(1) | BIT(3) | BIT(5) | BIT(7) | \ 47 BIT(9) | BIT(11) | BIT(26)) 48 49 /* Hardware Feedback Interface MSR configuration bits */ 50 #define HW_FEEDBACK_PTR_VALID_BIT BIT(0) 51 #define HW_FEEDBACK_CONFIG_HFI_ENABLE_BIT BIT(0) 52 53 /* CPUID detection and enumeration definitions for HFI */ 54 55 #define CPUID_HFI_LEAF 6 56 57 union hfi_capabilities { 58 struct { 59 u8 performance:1; 60 u8 energy_efficiency:1; 61 u8 __reserved:6; 62 } split; 63 u8 bits; 64 }; 65 66 union cpuid6_edx { 67 struct { 68 union hfi_capabilities capabilities; 69 u32 table_pages:4; 70 u32 __reserved:4; 71 s32 index:16; 72 } split; 73 u32 full; 74 }; 75 76 /** 77 * struct hfi_cpu_data - HFI capabilities per CPU 78 * @perf_cap: Performance capability 79 * @ee_cap: Energy efficiency capability 80 * 81 * Capabilities of a logical processor in the HFI table. These capabilities are 82 * unitless. 83 */ 84 struct hfi_cpu_data { 85 u8 perf_cap; 86 u8 ee_cap; 87 } __packed; 88 89 /** 90 * struct hfi_hdr - Header of the HFI table 91 * @perf_updated: Hardware updated performance capabilities 92 * @ee_updated: Hardware updated energy efficiency capabilities 93 * 94 * Properties of the data in an HFI table. 95 */ 96 struct hfi_hdr { 97 u8 perf_updated; 98 u8 ee_updated; 99 } __packed; 100 101 /** 102 * struct hfi_instance - Representation of an HFI instance (i.e., a table) 103 * @local_table: Base of the local copy of the HFI table 104 * @timestamp: Timestamp of the last update of the local table. 105 * Located at the base of the local table. 106 * @hdr: Base address of the header of the local table 107 * @data: Base address of the data of the local table 108 * @cpus: CPUs represented in this HFI table instance 109 * @hw_table: Pointer to the HFI table of this instance 110 * @update_work: Delayed work to process HFI updates 111 * @table_lock: Lock to protect acceses to the table of this instance 112 * @event_lock: Lock to process HFI interrupts 113 * 114 * A set of parameters to parse and navigate a specific HFI table. 115 */ 116 struct hfi_instance { 117 union { 118 void *local_table; 119 u64 *timestamp; 120 }; 121 void *hdr; 122 void *data; 123 cpumask_var_t cpus; 124 void *hw_table; 125 struct delayed_work update_work; 126 raw_spinlock_t table_lock; 127 raw_spinlock_t event_lock; 128 }; 129 130 /** 131 * struct hfi_features - Supported HFI features 132 * @nr_table_pages: Size of the HFI table in 4KB pages 133 * @cpu_stride: Stride size to locate the capability data of a logical 134 * processor within the table (i.e., row stride) 135 * @hdr_size: Size of the table header 136 * 137 * Parameters and supported features that are common to all HFI instances 138 */ 139 struct hfi_features { 140 unsigned int nr_table_pages; 141 unsigned int cpu_stride; 142 unsigned int hdr_size; 143 }; 144 145 /** 146 * struct hfi_cpu_info - Per-CPU attributes to consume HFI data 147 * @index: Row of this CPU in its HFI table 148 * @hfi_instance: Attributes of the HFI table to which this CPU belongs 149 * 150 * Parameters to link a logical processor to an HFI table and a row within it. 151 */ 152 struct hfi_cpu_info { 153 s16 index; 154 struct hfi_instance *hfi_instance; 155 }; 156 157 static DEFINE_PER_CPU(struct hfi_cpu_info, hfi_cpu_info) = { .index = -1 }; 158 159 static int max_hfi_instances; 160 static struct hfi_instance *hfi_instances; 161 162 static struct hfi_features hfi_features; 163 static DEFINE_MUTEX(hfi_instance_lock); 164 165 static struct workqueue_struct *hfi_updates_wq; 166 #define HFI_UPDATE_INTERVAL HZ 167 #define HFI_MAX_THERM_NOTIFY_COUNT 16 168 169 static void get_hfi_caps(struct hfi_instance *hfi_instance, 170 struct thermal_genl_cpu_caps *cpu_caps) 171 { 172 int cpu, i = 0; 173 174 raw_spin_lock_irq(&hfi_instance->table_lock); 175 for_each_cpu(cpu, hfi_instance->cpus) { 176 struct hfi_cpu_data *caps; 177 s16 index; 178 179 index = per_cpu(hfi_cpu_info, cpu).index; 180 caps = hfi_instance->data + index * hfi_features.cpu_stride; 181 cpu_caps[i].cpu = cpu; 182 183 /* 184 * Scale performance and energy efficiency to 185 * the [0, 1023] interval that thermal netlink uses. 186 */ 187 cpu_caps[i].performance = caps->perf_cap << 2; 188 cpu_caps[i].efficiency = caps->ee_cap << 2; 189 190 ++i; 191 } 192 raw_spin_unlock_irq(&hfi_instance->table_lock); 193 } 194 195 /* 196 * Call update_capabilities() when there are changes in the HFI table. 197 */ 198 static void update_capabilities(struct hfi_instance *hfi_instance) 199 { 200 struct thermal_genl_cpu_caps *cpu_caps; 201 int i = 0, cpu_count; 202 203 /* CPUs may come online/offline while processing an HFI update. */ 204 mutex_lock(&hfi_instance_lock); 205 206 cpu_count = cpumask_weight(hfi_instance->cpus); 207 208 /* No CPUs to report in this hfi_instance. */ 209 if (!cpu_count) 210 goto out; 211 212 cpu_caps = kcalloc(cpu_count, sizeof(*cpu_caps), GFP_KERNEL); 213 if (!cpu_caps) 214 goto out; 215 216 get_hfi_caps(hfi_instance, cpu_caps); 217 218 if (cpu_count < HFI_MAX_THERM_NOTIFY_COUNT) 219 goto last_cmd; 220 221 /* Process complete chunks of HFI_MAX_THERM_NOTIFY_COUNT capabilities. */ 222 for (i = 0; 223 (i + HFI_MAX_THERM_NOTIFY_COUNT) <= cpu_count; 224 i += HFI_MAX_THERM_NOTIFY_COUNT) 225 thermal_genl_cpu_capability_event(HFI_MAX_THERM_NOTIFY_COUNT, 226 &cpu_caps[i]); 227 228 cpu_count = cpu_count - i; 229 230 last_cmd: 231 /* Process the remaining capabilities if any. */ 232 if (cpu_count) 233 thermal_genl_cpu_capability_event(cpu_count, &cpu_caps[i]); 234 235 kfree(cpu_caps); 236 out: 237 mutex_unlock(&hfi_instance_lock); 238 } 239 240 static void hfi_update_work_fn(struct work_struct *work) 241 { 242 struct hfi_instance *hfi_instance; 243 244 hfi_instance = container_of(to_delayed_work(work), struct hfi_instance, 245 update_work); 246 247 update_capabilities(hfi_instance); 248 } 249 250 void intel_hfi_process_event(__u64 pkg_therm_status_msr_val) 251 { 252 struct hfi_instance *hfi_instance; 253 int cpu = smp_processor_id(); 254 struct hfi_cpu_info *info; 255 u64 new_timestamp; 256 257 if (!pkg_therm_status_msr_val) 258 return; 259 260 info = &per_cpu(hfi_cpu_info, cpu); 261 if (!info) 262 return; 263 264 /* 265 * A CPU is linked to its HFI instance before the thermal vector in the 266 * local APIC is unmasked. Hence, info->hfi_instance cannot be NULL 267 * when receiving an HFI event. 268 */ 269 hfi_instance = info->hfi_instance; 270 if (unlikely(!hfi_instance)) { 271 pr_debug("Received event on CPU %d but instance was null", cpu); 272 return; 273 } 274 275 /* 276 * On most systems, all CPUs in the package receive a package-level 277 * thermal interrupt when there is an HFI update. It is sufficient to 278 * let a single CPU to acknowledge the update and queue work to 279 * process it. The remaining CPUs can resume their work. 280 */ 281 if (!raw_spin_trylock(&hfi_instance->event_lock)) 282 return; 283 284 /* Skip duplicated updates. */ 285 new_timestamp = *(u64 *)hfi_instance->hw_table; 286 if (*hfi_instance->timestamp == new_timestamp) { 287 raw_spin_unlock(&hfi_instance->event_lock); 288 return; 289 } 290 291 raw_spin_lock(&hfi_instance->table_lock); 292 293 /* 294 * Copy the updated table into our local copy. This includes the new 295 * timestamp. 296 */ 297 memcpy(hfi_instance->local_table, hfi_instance->hw_table, 298 hfi_features.nr_table_pages << PAGE_SHIFT); 299 300 raw_spin_unlock(&hfi_instance->table_lock); 301 raw_spin_unlock(&hfi_instance->event_lock); 302 303 /* 304 * Let hardware know that we are done reading the HFI table and it is 305 * free to update it again. 306 */ 307 pkg_therm_status_msr_val &= THERM_STATUS_CLEAR_PKG_MASK & 308 ~PACKAGE_THERM_STATUS_HFI_UPDATED; 309 wrmsrl(MSR_IA32_PACKAGE_THERM_STATUS, pkg_therm_status_msr_val); 310 311 queue_delayed_work(hfi_updates_wq, &hfi_instance->update_work, 312 HFI_UPDATE_INTERVAL); 313 } 314 315 static void init_hfi_cpu_index(struct hfi_cpu_info *info) 316 { 317 union cpuid6_edx edx; 318 319 /* Do not re-read @cpu's index if it has already been initialized. */ 320 if (info->index > -1) 321 return; 322 323 edx.full = cpuid_edx(CPUID_HFI_LEAF); 324 info->index = edx.split.index; 325 } 326 327 /* 328 * The format of the HFI table depends on the number of capabilities that the 329 * hardware supports. Keep a data structure to navigate the table. 330 */ 331 static void init_hfi_instance(struct hfi_instance *hfi_instance) 332 { 333 /* The HFI header is below the time-stamp. */ 334 hfi_instance->hdr = hfi_instance->local_table + 335 sizeof(*hfi_instance->timestamp); 336 337 /* The HFI data starts below the header. */ 338 hfi_instance->data = hfi_instance->hdr + hfi_features.hdr_size; 339 } 340 341 /** 342 * intel_hfi_online() - Enable HFI on @cpu 343 * @cpu: CPU in which the HFI will be enabled 344 * 345 * Enable the HFI to be used in @cpu. The HFI is enabled at the die/package 346 * level. The first CPU in the die/package to come online does the full HFI 347 * initialization. Subsequent CPUs will just link themselves to the HFI 348 * instance of their die/package. 349 * 350 * This function is called before enabling the thermal vector in the local APIC 351 * in order to ensure that @cpu has an associated HFI instance when it receives 352 * an HFI event. 353 */ 354 void intel_hfi_online(unsigned int cpu) 355 { 356 struct hfi_instance *hfi_instance; 357 struct hfi_cpu_info *info; 358 phys_addr_t hw_table_pa; 359 u64 msr_val; 360 u16 die_id; 361 362 /* Nothing to do if hfi_instances are missing. */ 363 if (!hfi_instances) 364 return; 365 366 /* 367 * Link @cpu to the HFI instance of its package/die. It does not 368 * matter whether the instance has been initialized. 369 */ 370 info = &per_cpu(hfi_cpu_info, cpu); 371 die_id = topology_logical_die_id(cpu); 372 hfi_instance = info->hfi_instance; 373 if (!hfi_instance) { 374 if (die_id < 0 || die_id >= max_hfi_instances) 375 return; 376 377 hfi_instance = &hfi_instances[die_id]; 378 info->hfi_instance = hfi_instance; 379 } 380 381 init_hfi_cpu_index(info); 382 383 /* 384 * Now check if the HFI instance of the package/die of @cpu has been 385 * initialized (by checking its header). In such case, all we have to 386 * do is to add @cpu to this instance's cpumask. 387 */ 388 mutex_lock(&hfi_instance_lock); 389 if (hfi_instance->hdr) { 390 cpumask_set_cpu(cpu, hfi_instance->cpus); 391 goto unlock; 392 } 393 394 /* 395 * Hardware is programmed with the physical address of the first page 396 * frame of the table. Hence, the allocated memory must be page-aligned. 397 */ 398 hfi_instance->hw_table = alloc_pages_exact(hfi_features.nr_table_pages, 399 GFP_KERNEL | __GFP_ZERO); 400 if (!hfi_instance->hw_table) 401 goto unlock; 402 403 hw_table_pa = virt_to_phys(hfi_instance->hw_table); 404 405 /* 406 * Allocate memory to keep a local copy of the table that 407 * hardware generates. 408 */ 409 hfi_instance->local_table = kzalloc(hfi_features.nr_table_pages << PAGE_SHIFT, 410 GFP_KERNEL); 411 if (!hfi_instance->local_table) 412 goto free_hw_table; 413 414 /* 415 * Program the address of the feedback table of this die/package. On 416 * some processors, hardware remembers the old address of the HFI table 417 * even after having been reprogrammed and re-enabled. Thus, do not free 418 * the pages allocated for the table or reprogram the hardware with a 419 * new base address. Namely, program the hardware only once. 420 */ 421 msr_val = hw_table_pa | HW_FEEDBACK_PTR_VALID_BIT; 422 wrmsrl(MSR_IA32_HW_FEEDBACK_PTR, msr_val); 423 424 init_hfi_instance(hfi_instance); 425 426 INIT_DELAYED_WORK(&hfi_instance->update_work, hfi_update_work_fn); 427 raw_spin_lock_init(&hfi_instance->table_lock); 428 raw_spin_lock_init(&hfi_instance->event_lock); 429 430 cpumask_set_cpu(cpu, hfi_instance->cpus); 431 432 /* 433 * Enable the hardware feedback interface and never disable it. See 434 * comment on programming the address of the table. 435 */ 436 rdmsrl(MSR_IA32_HW_FEEDBACK_CONFIG, msr_val); 437 msr_val |= HW_FEEDBACK_CONFIG_HFI_ENABLE_BIT; 438 wrmsrl(MSR_IA32_HW_FEEDBACK_CONFIG, msr_val); 439 440 unlock: 441 mutex_unlock(&hfi_instance_lock); 442 return; 443 444 free_hw_table: 445 free_pages_exact(hfi_instance->hw_table, hfi_features.nr_table_pages); 446 goto unlock; 447 } 448 449 /** 450 * intel_hfi_offline() - Disable HFI on @cpu 451 * @cpu: CPU in which the HFI will be disabled 452 * 453 * Remove @cpu from those covered by its HFI instance. 454 * 455 * On some processors, hardware remembers previous programming settings even 456 * after being reprogrammed. Thus, keep HFI enabled even if all CPUs in the 457 * die/package of @cpu are offline. See note in intel_hfi_online(). 458 */ 459 void intel_hfi_offline(unsigned int cpu) 460 { 461 struct hfi_cpu_info *info = &per_cpu(hfi_cpu_info, cpu); 462 struct hfi_instance *hfi_instance; 463 464 /* 465 * Check if @cpu as an associated, initialized (i.e., with a non-NULL 466 * header). Also, HFI instances are only initialized if X86_FEATURE_HFI 467 * is present. 468 */ 469 hfi_instance = info->hfi_instance; 470 if (!hfi_instance) 471 return; 472 473 if (!hfi_instance->hdr) 474 return; 475 476 mutex_lock(&hfi_instance_lock); 477 cpumask_clear_cpu(cpu, hfi_instance->cpus); 478 mutex_unlock(&hfi_instance_lock); 479 } 480 481 static __init int hfi_parse_features(void) 482 { 483 unsigned int nr_capabilities; 484 union cpuid6_edx edx; 485 486 if (!boot_cpu_has(X86_FEATURE_HFI)) 487 return -ENODEV; 488 489 /* 490 * If we are here we know that CPUID_HFI_LEAF exists. Parse the 491 * supported capabilities and the size of the HFI table. 492 */ 493 edx.full = cpuid_edx(CPUID_HFI_LEAF); 494 495 if (!edx.split.capabilities.split.performance) { 496 pr_debug("Performance reporting not supported! Not using HFI\n"); 497 return -ENODEV; 498 } 499 500 /* 501 * The number of supported capabilities determines the number of 502 * columns in the HFI table. Exclude the reserved bits. 503 */ 504 edx.split.capabilities.split.__reserved = 0; 505 nr_capabilities = hweight8(edx.split.capabilities.bits); 506 507 /* The number of 4KB pages required by the table */ 508 hfi_features.nr_table_pages = edx.split.table_pages + 1; 509 510 /* 511 * The header contains change indications for each supported feature. 512 * The size of the table header is rounded up to be a multiple of 8 513 * bytes. 514 */ 515 hfi_features.hdr_size = DIV_ROUND_UP(nr_capabilities, 8) * 8; 516 517 /* 518 * Data of each logical processor is also rounded up to be a multiple 519 * of 8 bytes. 520 */ 521 hfi_features.cpu_stride = DIV_ROUND_UP(nr_capabilities, 8) * 8; 522 523 return 0; 524 } 525 526 void __init intel_hfi_init(void) 527 { 528 struct hfi_instance *hfi_instance; 529 int i, j; 530 531 if (hfi_parse_features()) 532 return; 533 534 /* There is one HFI instance per die/package. */ 535 max_hfi_instances = topology_max_packages() * 536 topology_max_die_per_package(); 537 538 /* 539 * This allocation may fail. CPU hotplug callbacks must check 540 * for a null pointer. 541 */ 542 hfi_instances = kcalloc(max_hfi_instances, sizeof(*hfi_instances), 543 GFP_KERNEL); 544 if (!hfi_instances) 545 return; 546 547 for (i = 0; i < max_hfi_instances; i++) { 548 hfi_instance = &hfi_instances[i]; 549 if (!zalloc_cpumask_var(&hfi_instance->cpus, GFP_KERNEL)) 550 goto err_nomem; 551 } 552 553 hfi_updates_wq = create_singlethread_workqueue("hfi-updates"); 554 if (!hfi_updates_wq) 555 goto err_nomem; 556 557 return; 558 559 err_nomem: 560 for (j = 0; j < i; ++j) { 561 hfi_instance = &hfi_instances[j]; 562 free_cpumask_var(hfi_instance->cpus); 563 } 564 565 kfree(hfi_instances); 566 hfi_instances = NULL; 567 } 568