1 /* 2 * arch/arm64/kernel/topology.c 3 * 4 * Copyright (C) 2011,2013,2014 Linaro Limited. 5 * 6 * Based on the arm32 version written by Vincent Guittot in turn based on 7 * arch/sh/kernel/topology.c 8 * 9 * This file is subject to the terms and conditions of the GNU General Public 10 * License. See the file "COPYING" in the main directory of this archive 11 * for more details. 12 */ 13 14 #include <linux/acpi.h> 15 #include <linux/arch_topology.h> 16 #include <linux/cacheinfo.h> 17 #include <linux/cpufreq.h> 18 #include <linux/init.h> 19 #include <linux/percpu.h> 20 21 #include <asm/cpu.h> 22 #include <asm/cputype.h> 23 #include <asm/topology.h> 24 25 void store_cpu_topology(unsigned int cpuid) 26 { 27 struct cpu_topology *cpuid_topo = &cpu_topology[cpuid]; 28 u64 mpidr; 29 30 if (cpuid_topo->package_id != -1) 31 goto topology_populated; 32 33 mpidr = read_cpuid_mpidr(); 34 35 /* Uniprocessor systems can rely on default topology values */ 36 if (mpidr & MPIDR_UP_BITMASK) 37 return; 38 39 /* 40 * This would be the place to create cpu topology based on MPIDR. 41 * 42 * However, it cannot be trusted to depict the actual topology; some 43 * pieces of the architecture enforce an artificial cap on Aff0 values 44 * (e.g. GICv3's ICC_SGI1R_EL1 limits it to 15), leading to an 45 * artificial cycling of Aff1, Aff2 and Aff3 values. IOW, these end up 46 * having absolutely no relationship to the actual underlying system 47 * topology, and cannot be reasonably used as core / package ID. 48 * 49 * If the MT bit is set, Aff0 *could* be used to define a thread ID, but 50 * we still wouldn't be able to obtain a sane core ID. This means we 51 * need to entirely ignore MPIDR for any topology deduction. 52 */ 53 cpuid_topo->thread_id = -1; 54 cpuid_topo->core_id = cpuid; 55 cpuid_topo->package_id = cpu_to_node(cpuid); 56 57 pr_debug("CPU%u: cluster %d core %d thread %d mpidr %#016llx\n", 58 cpuid, cpuid_topo->package_id, cpuid_topo->core_id, 59 cpuid_topo->thread_id, mpidr); 60 61 topology_populated: 62 update_siblings_masks(cpuid); 63 } 64 65 #ifdef CONFIG_ACPI 66 static bool __init acpi_cpu_is_threaded(int cpu) 67 { 68 int is_threaded = acpi_pptt_cpu_is_thread(cpu); 69 70 /* 71 * if the PPTT doesn't have thread information, assume a homogeneous 72 * machine and return the current CPU's thread state. 73 */ 74 if (is_threaded < 0) 75 is_threaded = read_cpuid_mpidr() & MPIDR_MT_BITMASK; 76 77 return !!is_threaded; 78 } 79 80 /* 81 * Propagate the topology information of the processor_topology_node tree to the 82 * cpu_topology array. 83 */ 84 int __init parse_acpi_topology(void) 85 { 86 int cpu, topology_id; 87 88 if (acpi_disabled) 89 return 0; 90 91 for_each_possible_cpu(cpu) { 92 topology_id = find_acpi_cpu_topology(cpu, 0); 93 if (topology_id < 0) 94 return topology_id; 95 96 if (acpi_cpu_is_threaded(cpu)) { 97 cpu_topology[cpu].thread_id = topology_id; 98 topology_id = find_acpi_cpu_topology(cpu, 1); 99 cpu_topology[cpu].core_id = topology_id; 100 } else { 101 cpu_topology[cpu].thread_id = -1; 102 cpu_topology[cpu].core_id = topology_id; 103 } 104 topology_id = find_acpi_cpu_topology_cluster(cpu); 105 cpu_topology[cpu].cluster_id = topology_id; 106 topology_id = find_acpi_cpu_topology_package(cpu); 107 cpu_topology[cpu].package_id = topology_id; 108 } 109 110 return 0; 111 } 112 #endif 113 114 #ifdef CONFIG_ARM64_AMU_EXTN 115 #define read_corecnt() read_sysreg_s(SYS_AMEVCNTR0_CORE_EL0) 116 #define read_constcnt() read_sysreg_s(SYS_AMEVCNTR0_CONST_EL0) 117 #else 118 #define read_corecnt() (0UL) 119 #define read_constcnt() (0UL) 120 #endif 121 122 #undef pr_fmt 123 #define pr_fmt(fmt) "AMU: " fmt 124 125 static DEFINE_PER_CPU_READ_MOSTLY(unsigned long, arch_max_freq_scale); 126 static DEFINE_PER_CPU(u64, arch_const_cycles_prev); 127 static DEFINE_PER_CPU(u64, arch_core_cycles_prev); 128 static cpumask_var_t amu_fie_cpus; 129 130 void update_freq_counters_refs(void) 131 { 132 this_cpu_write(arch_core_cycles_prev, read_corecnt()); 133 this_cpu_write(arch_const_cycles_prev, read_constcnt()); 134 } 135 136 static inline bool freq_counters_valid(int cpu) 137 { 138 if ((cpu >= nr_cpu_ids) || !cpumask_test_cpu(cpu, cpu_present_mask)) 139 return false; 140 141 if (!cpu_has_amu_feat(cpu)) { 142 pr_debug("CPU%d: counters are not supported.\n", cpu); 143 return false; 144 } 145 146 if (unlikely(!per_cpu(arch_const_cycles_prev, cpu) || 147 !per_cpu(arch_core_cycles_prev, cpu))) { 148 pr_debug("CPU%d: cycle counters are not enabled.\n", cpu); 149 return false; 150 } 151 152 return true; 153 } 154 155 static int freq_inv_set_max_ratio(int cpu, u64 max_rate, u64 ref_rate) 156 { 157 u64 ratio; 158 159 if (unlikely(!max_rate || !ref_rate)) { 160 pr_debug("CPU%d: invalid maximum or reference frequency.\n", 161 cpu); 162 return -EINVAL; 163 } 164 165 /* 166 * Pre-compute the fixed ratio between the frequency of the constant 167 * reference counter and the maximum frequency of the CPU. 168 * 169 * ref_rate 170 * arch_max_freq_scale = ---------- * SCHED_CAPACITY_SCALE² 171 * max_rate 172 * 173 * We use a factor of 2 * SCHED_CAPACITY_SHIFT -> SCHED_CAPACITY_SCALE² 174 * in order to ensure a good resolution for arch_max_freq_scale for 175 * very low reference frequencies (down to the KHz range which should 176 * be unlikely). 177 */ 178 ratio = ref_rate << (2 * SCHED_CAPACITY_SHIFT); 179 ratio = div64_u64(ratio, max_rate); 180 if (!ratio) { 181 WARN_ONCE(1, "Reference frequency too low.\n"); 182 return -EINVAL; 183 } 184 185 per_cpu(arch_max_freq_scale, cpu) = (unsigned long)ratio; 186 187 return 0; 188 } 189 190 static void amu_scale_freq_tick(void) 191 { 192 u64 prev_core_cnt, prev_const_cnt; 193 u64 core_cnt, const_cnt, scale; 194 195 prev_const_cnt = this_cpu_read(arch_const_cycles_prev); 196 prev_core_cnt = this_cpu_read(arch_core_cycles_prev); 197 198 update_freq_counters_refs(); 199 200 const_cnt = this_cpu_read(arch_const_cycles_prev); 201 core_cnt = this_cpu_read(arch_core_cycles_prev); 202 203 if (unlikely(core_cnt <= prev_core_cnt || 204 const_cnt <= prev_const_cnt)) 205 return; 206 207 /* 208 * /\core arch_max_freq_scale 209 * scale = ------- * -------------------- 210 * /\const SCHED_CAPACITY_SCALE 211 * 212 * See validate_cpu_freq_invariance_counters() for details on 213 * arch_max_freq_scale and the use of SCHED_CAPACITY_SHIFT. 214 */ 215 scale = core_cnt - prev_core_cnt; 216 scale *= this_cpu_read(arch_max_freq_scale); 217 scale = div64_u64(scale >> SCHED_CAPACITY_SHIFT, 218 const_cnt - prev_const_cnt); 219 220 scale = min_t(unsigned long, scale, SCHED_CAPACITY_SCALE); 221 this_cpu_write(arch_freq_scale, (unsigned long)scale); 222 } 223 224 static struct scale_freq_data amu_sfd = { 225 .source = SCALE_FREQ_SOURCE_ARCH, 226 .set_freq_scale = amu_scale_freq_tick, 227 }; 228 229 static void amu_fie_setup(const struct cpumask *cpus) 230 { 231 int cpu; 232 233 /* We are already set since the last insmod of cpufreq driver */ 234 if (unlikely(cpumask_subset(cpus, amu_fie_cpus))) 235 return; 236 237 for_each_cpu(cpu, cpus) { 238 if (!freq_counters_valid(cpu) || 239 freq_inv_set_max_ratio(cpu, 240 cpufreq_get_hw_max_freq(cpu) * 1000, 241 arch_timer_get_rate())) 242 return; 243 } 244 245 cpumask_or(amu_fie_cpus, amu_fie_cpus, cpus); 246 247 topology_set_scale_freq_source(&amu_sfd, amu_fie_cpus); 248 249 pr_debug("CPUs[%*pbl]: counters will be used for FIE.", 250 cpumask_pr_args(cpus)); 251 } 252 253 static int init_amu_fie_callback(struct notifier_block *nb, unsigned long val, 254 void *data) 255 { 256 struct cpufreq_policy *policy = data; 257 258 if (val == CPUFREQ_CREATE_POLICY) 259 amu_fie_setup(policy->related_cpus); 260 261 /* 262 * We don't need to handle CPUFREQ_REMOVE_POLICY event as the AMU 263 * counters don't have any dependency on cpufreq driver once we have 264 * initialized AMU support and enabled invariance. The AMU counters will 265 * keep on working just fine in the absence of the cpufreq driver, and 266 * for the CPUs for which there are no counters available, the last set 267 * value of arch_freq_scale will remain valid as that is the frequency 268 * those CPUs are running at. 269 */ 270 271 return 0; 272 } 273 274 static struct notifier_block init_amu_fie_notifier = { 275 .notifier_call = init_amu_fie_callback, 276 }; 277 278 static int __init init_amu_fie(void) 279 { 280 int ret; 281 282 if (!zalloc_cpumask_var(&amu_fie_cpus, GFP_KERNEL)) 283 return -ENOMEM; 284 285 ret = cpufreq_register_notifier(&init_amu_fie_notifier, 286 CPUFREQ_POLICY_NOTIFIER); 287 if (ret) 288 free_cpumask_var(amu_fie_cpus); 289 290 return ret; 291 } 292 core_initcall(init_amu_fie); 293 294 #ifdef CONFIG_ACPI_CPPC_LIB 295 #include <acpi/cppc_acpi.h> 296 297 static void cpu_read_corecnt(void *val) 298 { 299 *(u64 *)val = read_corecnt(); 300 } 301 302 static void cpu_read_constcnt(void *val) 303 { 304 *(u64 *)val = read_constcnt(); 305 } 306 307 static inline 308 int counters_read_on_cpu(int cpu, smp_call_func_t func, u64 *val) 309 { 310 /* 311 * Abort call on counterless CPU or when interrupts are 312 * disabled - can lead to deadlock in smp sync call. 313 */ 314 if (!cpu_has_amu_feat(cpu)) 315 return -EOPNOTSUPP; 316 317 if (WARN_ON_ONCE(irqs_disabled())) 318 return -EPERM; 319 320 smp_call_function_single(cpu, func, val, 1); 321 322 return 0; 323 } 324 325 /* 326 * Refer to drivers/acpi/cppc_acpi.c for the description of the functions 327 * below. 328 */ 329 bool cpc_ffh_supported(void) 330 { 331 return freq_counters_valid(get_cpu_with_amu_feat()); 332 } 333 334 int cpc_read_ffh(int cpu, struct cpc_reg *reg, u64 *val) 335 { 336 int ret = -EOPNOTSUPP; 337 338 switch ((u64)reg->address) { 339 case 0x0: 340 ret = counters_read_on_cpu(cpu, cpu_read_corecnt, val); 341 break; 342 case 0x1: 343 ret = counters_read_on_cpu(cpu, cpu_read_constcnt, val); 344 break; 345 } 346 347 if (!ret) { 348 *val &= GENMASK_ULL(reg->bit_offset + reg->bit_width - 1, 349 reg->bit_offset); 350 *val >>= reg->bit_offset; 351 } 352 353 return ret; 354 } 355 356 int cpc_write_ffh(int cpunum, struct cpc_reg *reg, u64 val) 357 { 358 return -EOPNOTSUPP; 359 } 360 #endif /* CONFIG_ACPI_CPPC_LIB */ 361