1 /* 2 * arch/arm64/kernel/topology.c 3 * 4 * Copyright (C) 2011,2013,2014 Linaro Limited. 5 * 6 * Based on the arm32 version written by Vincent Guittot in turn based on 7 * arch/sh/kernel/topology.c 8 * 9 * This file is subject to the terms and conditions of the GNU General Public 10 * License. See the file "COPYING" in the main directory of this archive 11 * for more details. 12 */ 13 14 #include <linux/acpi.h> 15 #include <linux/arch_topology.h> 16 #include <linux/cacheinfo.h> 17 #include <linux/cpufreq.h> 18 #include <linux/init.h> 19 #include <linux/percpu.h> 20 21 #include <asm/cpu.h> 22 #include <asm/cputype.h> 23 #include <asm/topology.h> 24 25 void store_cpu_topology(unsigned int cpuid) 26 { 27 struct cpu_topology *cpuid_topo = &cpu_topology[cpuid]; 28 u64 mpidr; 29 30 if (cpuid_topo->package_id != -1) 31 goto topology_populated; 32 33 mpidr = read_cpuid_mpidr(); 34 35 /* Uniprocessor systems can rely on default topology values */ 36 if (mpidr & MPIDR_UP_BITMASK) 37 return; 38 39 /* 40 * This would be the place to create cpu topology based on MPIDR. 41 * 42 * However, it cannot be trusted to depict the actual topology; some 43 * pieces of the architecture enforce an artificial cap on Aff0 values 44 * (e.g. GICv3's ICC_SGI1R_EL1 limits it to 15), leading to an 45 * artificial cycling of Aff1, Aff2 and Aff3 values. IOW, these end up 46 * having absolutely no relationship to the actual underlying system 47 * topology, and cannot be reasonably used as core / package ID. 48 * 49 * If the MT bit is set, Aff0 *could* be used to define a thread ID, but 50 * we still wouldn't be able to obtain a sane core ID. This means we 51 * need to entirely ignore MPIDR for any topology deduction. 52 */ 53 cpuid_topo->thread_id = -1; 54 cpuid_topo->core_id = cpuid; 55 cpuid_topo->package_id = cpu_to_node(cpuid); 56 57 pr_debug("CPU%u: cluster %d core %d thread %d mpidr %#016llx\n", 58 cpuid, cpuid_topo->package_id, cpuid_topo->core_id, 59 cpuid_topo->thread_id, mpidr); 60 61 topology_populated: 62 update_siblings_masks(cpuid); 63 } 64 65 #ifdef CONFIG_ACPI 66 static bool __init acpi_cpu_is_threaded(int cpu) 67 { 68 int is_threaded = acpi_pptt_cpu_is_thread(cpu); 69 70 /* 71 * if the PPTT doesn't have thread information, assume a homogeneous 72 * machine and return the current CPU's thread state. 73 */ 74 if (is_threaded < 0) 75 is_threaded = read_cpuid_mpidr() & MPIDR_MT_BITMASK; 76 77 return !!is_threaded; 78 } 79 80 /* 81 * Propagate the topology information of the processor_topology_node tree to the 82 * cpu_topology array. 83 */ 84 int __init parse_acpi_topology(void) 85 { 86 int cpu, topology_id; 87 88 if (acpi_disabled) 89 return 0; 90 91 for_each_possible_cpu(cpu) { 92 int i, cache_id; 93 94 topology_id = find_acpi_cpu_topology(cpu, 0); 95 if (topology_id < 0) 96 return topology_id; 97 98 if (acpi_cpu_is_threaded(cpu)) { 99 cpu_topology[cpu].thread_id = topology_id; 100 topology_id = find_acpi_cpu_topology(cpu, 1); 101 cpu_topology[cpu].core_id = topology_id; 102 } else { 103 cpu_topology[cpu].thread_id = -1; 104 cpu_topology[cpu].core_id = topology_id; 105 } 106 topology_id = find_acpi_cpu_topology_cluster(cpu); 107 cpu_topology[cpu].cluster_id = topology_id; 108 topology_id = find_acpi_cpu_topology_package(cpu); 109 cpu_topology[cpu].package_id = topology_id; 110 111 i = acpi_find_last_cache_level(cpu); 112 113 if (i > 0) { 114 /* 115 * this is the only part of cpu_topology that has 116 * a direct relationship with the cache topology 117 */ 118 cache_id = find_acpi_cpu_cache_topology(cpu, i); 119 if (cache_id > 0) 120 cpu_topology[cpu].llc_id = cache_id; 121 } 122 } 123 124 return 0; 125 } 126 #endif 127 128 #ifdef CONFIG_ARM64_AMU_EXTN 129 #define read_corecnt() read_sysreg_s(SYS_AMEVCNTR0_CORE_EL0) 130 #define read_constcnt() read_sysreg_s(SYS_AMEVCNTR0_CONST_EL0) 131 #else 132 #define read_corecnt() (0UL) 133 #define read_constcnt() (0UL) 134 #endif 135 136 #undef pr_fmt 137 #define pr_fmt(fmt) "AMU: " fmt 138 139 static DEFINE_PER_CPU_READ_MOSTLY(unsigned long, arch_max_freq_scale); 140 static DEFINE_PER_CPU(u64, arch_const_cycles_prev); 141 static DEFINE_PER_CPU(u64, arch_core_cycles_prev); 142 static cpumask_var_t amu_fie_cpus; 143 144 void update_freq_counters_refs(void) 145 { 146 this_cpu_write(arch_core_cycles_prev, read_corecnt()); 147 this_cpu_write(arch_const_cycles_prev, read_constcnt()); 148 } 149 150 static inline bool freq_counters_valid(int cpu) 151 { 152 if ((cpu >= nr_cpu_ids) || !cpumask_test_cpu(cpu, cpu_present_mask)) 153 return false; 154 155 if (!cpu_has_amu_feat(cpu)) { 156 pr_debug("CPU%d: counters are not supported.\n", cpu); 157 return false; 158 } 159 160 if (unlikely(!per_cpu(arch_const_cycles_prev, cpu) || 161 !per_cpu(arch_core_cycles_prev, cpu))) { 162 pr_debug("CPU%d: cycle counters are not enabled.\n", cpu); 163 return false; 164 } 165 166 return true; 167 } 168 169 static int freq_inv_set_max_ratio(int cpu, u64 max_rate, u64 ref_rate) 170 { 171 u64 ratio; 172 173 if (unlikely(!max_rate || !ref_rate)) { 174 pr_debug("CPU%d: invalid maximum or reference frequency.\n", 175 cpu); 176 return -EINVAL; 177 } 178 179 /* 180 * Pre-compute the fixed ratio between the frequency of the constant 181 * reference counter and the maximum frequency of the CPU. 182 * 183 * ref_rate 184 * arch_max_freq_scale = ---------- * SCHED_CAPACITY_SCALE² 185 * max_rate 186 * 187 * We use a factor of 2 * SCHED_CAPACITY_SHIFT -> SCHED_CAPACITY_SCALE² 188 * in order to ensure a good resolution for arch_max_freq_scale for 189 * very low reference frequencies (down to the KHz range which should 190 * be unlikely). 191 */ 192 ratio = ref_rate << (2 * SCHED_CAPACITY_SHIFT); 193 ratio = div64_u64(ratio, max_rate); 194 if (!ratio) { 195 WARN_ONCE(1, "Reference frequency too low.\n"); 196 return -EINVAL; 197 } 198 199 per_cpu(arch_max_freq_scale, cpu) = (unsigned long)ratio; 200 201 return 0; 202 } 203 204 static void amu_scale_freq_tick(void) 205 { 206 u64 prev_core_cnt, prev_const_cnt; 207 u64 core_cnt, const_cnt, scale; 208 209 prev_const_cnt = this_cpu_read(arch_const_cycles_prev); 210 prev_core_cnt = this_cpu_read(arch_core_cycles_prev); 211 212 update_freq_counters_refs(); 213 214 const_cnt = this_cpu_read(arch_const_cycles_prev); 215 core_cnt = this_cpu_read(arch_core_cycles_prev); 216 217 if (unlikely(core_cnt <= prev_core_cnt || 218 const_cnt <= prev_const_cnt)) 219 return; 220 221 /* 222 * /\core arch_max_freq_scale 223 * scale = ------- * -------------------- 224 * /\const SCHED_CAPACITY_SCALE 225 * 226 * See validate_cpu_freq_invariance_counters() for details on 227 * arch_max_freq_scale and the use of SCHED_CAPACITY_SHIFT. 228 */ 229 scale = core_cnt - prev_core_cnt; 230 scale *= this_cpu_read(arch_max_freq_scale); 231 scale = div64_u64(scale >> SCHED_CAPACITY_SHIFT, 232 const_cnt - prev_const_cnt); 233 234 scale = min_t(unsigned long, scale, SCHED_CAPACITY_SCALE); 235 this_cpu_write(arch_freq_scale, (unsigned long)scale); 236 } 237 238 static struct scale_freq_data amu_sfd = { 239 .source = SCALE_FREQ_SOURCE_ARCH, 240 .set_freq_scale = amu_scale_freq_tick, 241 }; 242 243 static void amu_fie_setup(const struct cpumask *cpus) 244 { 245 int cpu; 246 247 /* We are already set since the last insmod of cpufreq driver */ 248 if (unlikely(cpumask_subset(cpus, amu_fie_cpus))) 249 return; 250 251 for_each_cpu(cpu, cpus) { 252 if (!freq_counters_valid(cpu) || 253 freq_inv_set_max_ratio(cpu, 254 cpufreq_get_hw_max_freq(cpu) * 1000, 255 arch_timer_get_rate())) 256 return; 257 } 258 259 cpumask_or(amu_fie_cpus, amu_fie_cpus, cpus); 260 261 topology_set_scale_freq_source(&amu_sfd, amu_fie_cpus); 262 263 pr_debug("CPUs[%*pbl]: counters will be used for FIE.", 264 cpumask_pr_args(cpus)); 265 } 266 267 static int init_amu_fie_callback(struct notifier_block *nb, unsigned long val, 268 void *data) 269 { 270 struct cpufreq_policy *policy = data; 271 272 if (val == CPUFREQ_CREATE_POLICY) 273 amu_fie_setup(policy->related_cpus); 274 275 /* 276 * We don't need to handle CPUFREQ_REMOVE_POLICY event as the AMU 277 * counters don't have any dependency on cpufreq driver once we have 278 * initialized AMU support and enabled invariance. The AMU counters will 279 * keep on working just fine in the absence of the cpufreq driver, and 280 * for the CPUs for which there are no counters available, the last set 281 * value of arch_freq_scale will remain valid as that is the frequency 282 * those CPUs are running at. 283 */ 284 285 return 0; 286 } 287 288 static struct notifier_block init_amu_fie_notifier = { 289 .notifier_call = init_amu_fie_callback, 290 }; 291 292 static int __init init_amu_fie(void) 293 { 294 int ret; 295 296 if (!zalloc_cpumask_var(&amu_fie_cpus, GFP_KERNEL)) 297 return -ENOMEM; 298 299 ret = cpufreq_register_notifier(&init_amu_fie_notifier, 300 CPUFREQ_POLICY_NOTIFIER); 301 if (ret) 302 free_cpumask_var(amu_fie_cpus); 303 304 return ret; 305 } 306 core_initcall(init_amu_fie); 307 308 #ifdef CONFIG_ACPI_CPPC_LIB 309 #include <acpi/cppc_acpi.h> 310 311 static void cpu_read_corecnt(void *val) 312 { 313 *(u64 *)val = read_corecnt(); 314 } 315 316 static void cpu_read_constcnt(void *val) 317 { 318 *(u64 *)val = read_constcnt(); 319 } 320 321 static inline 322 int counters_read_on_cpu(int cpu, smp_call_func_t func, u64 *val) 323 { 324 /* 325 * Abort call on counterless CPU or when interrupts are 326 * disabled - can lead to deadlock in smp sync call. 327 */ 328 if (!cpu_has_amu_feat(cpu)) 329 return -EOPNOTSUPP; 330 331 if (WARN_ON_ONCE(irqs_disabled())) 332 return -EPERM; 333 334 smp_call_function_single(cpu, func, val, 1); 335 336 return 0; 337 } 338 339 /* 340 * Refer to drivers/acpi/cppc_acpi.c for the description of the functions 341 * below. 342 */ 343 bool cpc_ffh_supported(void) 344 { 345 return freq_counters_valid(get_cpu_with_amu_feat()); 346 } 347 348 int cpc_read_ffh(int cpu, struct cpc_reg *reg, u64 *val) 349 { 350 int ret = -EOPNOTSUPP; 351 352 switch ((u64)reg->address) { 353 case 0x0: 354 ret = counters_read_on_cpu(cpu, cpu_read_corecnt, val); 355 break; 356 case 0x1: 357 ret = counters_read_on_cpu(cpu, cpu_read_constcnt, val); 358 break; 359 } 360 361 if (!ret) { 362 *val &= GENMASK_ULL(reg->bit_offset + reg->bit_width - 1, 363 reg->bit_offset); 364 *val >>= reg->bit_offset; 365 } 366 367 return ret; 368 } 369 370 int cpc_write_ffh(int cpunum, struct cpc_reg *reg, u64 val) 371 { 372 return -EOPNOTSUPP; 373 } 374 #endif /* CONFIG_ACPI_CPPC_LIB */ 375