1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Intel Uncore Frequency Setting
4 * Copyright (c) 2022, Intel Corporation.
5 * All rights reserved.
6 *
7 * Provide interface to set MSR 620 at a granularity of per die. On CPU online,
8 * one control CPU is identified per die to read/write limit. This control CPU
9 * is changed, if the CPU state is changed to offline. When the last CPU is
10 * offline in a die then remove the sysfs object for that die.
11 * The majority of actual code is related to sysfs create and read/write
12 * attributes.
13 *
14 * Author: Srinivas Pandruvada <srinivas.pandruvada@linux.intel.com>
15 */
16
17 #include <linux/cpu.h>
18 #include <linux/module.h>
19 #include <linux/slab.h>
20 #include <linux/suspend.h>
21 #include <asm/cpu_device_id.h>
22 #include <asm/intel-family.h>
23
24 #include "uncore-frequency-common.h"
25
26 /* Max instances for uncore data, one for each die */
27 static int uncore_max_entries __read_mostly;
28 /* Storage for uncore data for all instances */
29 static struct uncore_data *uncore_instances;
30 /* Stores the CPU mask of the target CPUs to use during uncore read/write */
31 static cpumask_t uncore_cpu_mask;
32 /* CPU online callback register instance */
33 static enum cpuhp_state uncore_hp_state __read_mostly;
34
35 #define MSR_UNCORE_RATIO_LIMIT 0x620
36 #define MSR_UNCORE_PERF_STATUS 0x621
37 #define UNCORE_FREQ_KHZ_MULTIPLIER 100000
38
uncore_read_control_freq(struct uncore_data * data,unsigned int * min,unsigned int * max)39 static int uncore_read_control_freq(struct uncore_data *data, unsigned int *min,
40 unsigned int *max)
41 {
42 u64 cap;
43 int ret;
44
45 if (data->control_cpu < 0)
46 return -ENXIO;
47
48 ret = rdmsrl_on_cpu(data->control_cpu, MSR_UNCORE_RATIO_LIMIT, &cap);
49 if (ret)
50 return ret;
51
52 *max = (cap & 0x7F) * UNCORE_FREQ_KHZ_MULTIPLIER;
53 *min = ((cap & GENMASK(14, 8)) >> 8) * UNCORE_FREQ_KHZ_MULTIPLIER;
54
55 return 0;
56 }
57
uncore_write_control_freq(struct uncore_data * data,unsigned int input,unsigned int min_max)58 static int uncore_write_control_freq(struct uncore_data *data, unsigned int input,
59 unsigned int min_max)
60 {
61 int ret;
62 u64 cap;
63
64 input /= UNCORE_FREQ_KHZ_MULTIPLIER;
65 if (!input || input > 0x7F)
66 return -EINVAL;
67
68 if (data->control_cpu < 0)
69 return -ENXIO;
70
71 ret = rdmsrl_on_cpu(data->control_cpu, MSR_UNCORE_RATIO_LIMIT, &cap);
72 if (ret)
73 return ret;
74
75 if (min_max) {
76 cap &= ~0x7F;
77 cap |= input;
78 } else {
79 cap &= ~GENMASK(14, 8);
80 cap |= (input << 8);
81 }
82
83 ret = wrmsrl_on_cpu(data->control_cpu, MSR_UNCORE_RATIO_LIMIT, cap);
84 if (ret)
85 return ret;
86
87 data->stored_uncore_data = cap;
88
89 return 0;
90 }
91
uncore_read_freq(struct uncore_data * data,unsigned int * freq)92 static int uncore_read_freq(struct uncore_data *data, unsigned int *freq)
93 {
94 u64 ratio;
95 int ret;
96
97 if (data->control_cpu < 0)
98 return -ENXIO;
99
100 ret = rdmsrl_on_cpu(data->control_cpu, MSR_UNCORE_PERF_STATUS, &ratio);
101 if (ret)
102 return ret;
103
104 *freq = (ratio & 0x7F) * UNCORE_FREQ_KHZ_MULTIPLIER;
105
106 return 0;
107 }
108
109 /* Caller provides protection */
uncore_get_instance(unsigned int cpu)110 static struct uncore_data *uncore_get_instance(unsigned int cpu)
111 {
112 int id = topology_logical_die_id(cpu);
113
114 if (id >= 0 && id < uncore_max_entries)
115 return &uncore_instances[id];
116
117 return NULL;
118 }
119
uncore_event_cpu_online(unsigned int cpu)120 static int uncore_event_cpu_online(unsigned int cpu)
121 {
122 struct uncore_data *data;
123 int target;
124
125 /* Check if there is an online cpu in the package for uncore MSR */
126 target = cpumask_any_and(&uncore_cpu_mask, topology_die_cpumask(cpu));
127 if (target < nr_cpu_ids)
128 return 0;
129
130 /* Use this CPU on this die as a control CPU */
131 cpumask_set_cpu(cpu, &uncore_cpu_mask);
132
133 data = uncore_get_instance(cpu);
134 if (!data)
135 return 0;
136
137 data->package_id = topology_physical_package_id(cpu);
138 data->die_id = topology_die_id(cpu);
139 data->domain_id = UNCORE_DOMAIN_ID_INVALID;
140
141 return uncore_freq_add_entry(data, cpu);
142 }
143
uncore_event_cpu_offline(unsigned int cpu)144 static int uncore_event_cpu_offline(unsigned int cpu)
145 {
146 struct uncore_data *data;
147 int target;
148
149 data = uncore_get_instance(cpu);
150 if (!data)
151 return 0;
152
153 /* Check if existing cpu is used for uncore MSRs */
154 if (!cpumask_test_and_clear_cpu(cpu, &uncore_cpu_mask))
155 return 0;
156
157 /* Find a new cpu to set uncore MSR */
158 target = cpumask_any_but(topology_die_cpumask(cpu), cpu);
159
160 if (target < nr_cpu_ids) {
161 cpumask_set_cpu(target, &uncore_cpu_mask);
162 uncore_freq_add_entry(data, target);
163 } else {
164 uncore_freq_remove_die_entry(data);
165 }
166
167 return 0;
168 }
169
uncore_pm_notify(struct notifier_block * nb,unsigned long mode,void * _unused)170 static int uncore_pm_notify(struct notifier_block *nb, unsigned long mode,
171 void *_unused)
172 {
173 int i;
174
175 switch (mode) {
176 case PM_POST_HIBERNATION:
177 case PM_POST_RESTORE:
178 case PM_POST_SUSPEND:
179 for (i = 0; i < uncore_max_entries; ++i) {
180 struct uncore_data *data = &uncore_instances[i];
181
182 if (!data || !data->valid || !data->stored_uncore_data)
183 return 0;
184
185 wrmsrl_on_cpu(data->control_cpu, MSR_UNCORE_RATIO_LIMIT,
186 data->stored_uncore_data);
187 }
188 break;
189 default:
190 break;
191 }
192 return 0;
193 }
194
195 static struct notifier_block uncore_pm_nb = {
196 .notifier_call = uncore_pm_notify,
197 };
198
199 static const struct x86_cpu_id intel_uncore_cpu_ids[] = {
200 X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_G, NULL),
201 X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_X, NULL),
202 X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_D, NULL),
203 X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE_X, NULL),
204 X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_X, NULL),
205 X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_D, NULL),
206 X86_MATCH_INTEL_FAM6_MODEL(SAPPHIRERAPIDS_X, NULL),
207 X86_MATCH_INTEL_FAM6_MODEL(EMERALDRAPIDS_X, NULL),
208 X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE, NULL),
209 X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE_L, NULL),
210 X86_MATCH_INTEL_FAM6_MODEL(RAPTORLAKE, NULL),
211 X86_MATCH_INTEL_FAM6_MODEL(RAPTORLAKE_P, NULL),
212 X86_MATCH_INTEL_FAM6_MODEL(RAPTORLAKE_S, NULL),
213 X86_MATCH_INTEL_FAM6_MODEL(METEORLAKE, NULL),
214 X86_MATCH_INTEL_FAM6_MODEL(METEORLAKE_L, NULL),
215 {}
216 };
217 MODULE_DEVICE_TABLE(x86cpu, intel_uncore_cpu_ids);
218
intel_uncore_init(void)219 static int __init intel_uncore_init(void)
220 {
221 const struct x86_cpu_id *id;
222 int ret;
223
224 if (cpu_feature_enabled(X86_FEATURE_HYPERVISOR))
225 return -ENODEV;
226
227 id = x86_match_cpu(intel_uncore_cpu_ids);
228 if (!id)
229 return -ENODEV;
230
231 uncore_max_entries = topology_max_packages() *
232 topology_max_die_per_package();
233 uncore_instances = kcalloc(uncore_max_entries,
234 sizeof(*uncore_instances), GFP_KERNEL);
235 if (!uncore_instances)
236 return -ENOMEM;
237
238 ret = uncore_freq_common_init(uncore_read_control_freq, uncore_write_control_freq,
239 uncore_read_freq);
240 if (ret)
241 goto err_free;
242
243 ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN,
244 "platform/x86/uncore-freq:online",
245 uncore_event_cpu_online,
246 uncore_event_cpu_offline);
247 if (ret < 0)
248 goto err_rem_kobj;
249
250 uncore_hp_state = ret;
251
252 ret = register_pm_notifier(&uncore_pm_nb);
253 if (ret)
254 goto err_rem_state;
255
256 return 0;
257
258 err_rem_state:
259 cpuhp_remove_state(uncore_hp_state);
260 err_rem_kobj:
261 uncore_freq_common_exit();
262 err_free:
263 kfree(uncore_instances);
264
265 return ret;
266 }
module_init(intel_uncore_init)267 module_init(intel_uncore_init)
268
269 static void __exit intel_uncore_exit(void)
270 {
271 int i;
272
273 unregister_pm_notifier(&uncore_pm_nb);
274 cpuhp_remove_state(uncore_hp_state);
275 for (i = 0; i < uncore_max_entries; ++i)
276 uncore_freq_remove_die_entry(&uncore_instances[i]);
277 uncore_freq_common_exit();
278 kfree(uncore_instances);
279 }
280 module_exit(intel_uncore_exit)
281
282 MODULE_IMPORT_NS(INTEL_UNCORE_FREQUENCY);
283 MODULE_LICENSE("GPL v2");
284 MODULE_DESCRIPTION("Intel Uncore Frequency Limits Driver");
285