1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * uncore-frquency-tpmi: Uncore frequency scaling using TPMI
4 *
5 * Copyright (c) 2023, Intel Corporation.
6 * All Rights Reserved.
7 *
8 * The hardware interface to read/write is basically substitution of
9 * MSR 0x620 and 0x621.
10 * There are specific MMIO offset and bits to get/set minimum and
11 * maximum uncore ratio, similar to MSRs.
12 * The scope of the uncore MSRs was package scope. But TPMI allows
13 * new gen CPUs to have multiple uncore controls at uncore-cluster
14 * level. Each package can have multiple power domains which further
15 * can have multiple clusters.
16 * Here number of power domains = number of resources in this aux
17 * device. There are offsets and bits to discover number of clusters
18 * and offset for each cluster level controls.
19 *
20 */
21
22 #include <linux/auxiliary_bus.h>
23 #include <linux/bitfield.h>
24 #include <linux/bits.h>
25 #include <linux/io.h>
26 #include <linux/module.h>
27 #include <linux/intel_tpmi.h>
28
29 #include "uncore-frequency-common.h"
30
31 #define UNCORE_HEADER_VERSION 1
32 #define UNCORE_HEADER_INDEX 0
33 #define UNCORE_FABRIC_CLUSTER_OFFSET 8
34
35 /* status + control + adv_ctl1 + adv_ctl2 */
36 #define UNCORE_FABRIC_CLUSTER_SIZE (4 * 8)
37
38 #define UNCORE_STATUS_INDEX 0
39 #define UNCORE_CONTROL_INDEX 8
40
41 #define UNCORE_FREQ_KHZ_MULTIPLIER 100000
42
43 struct tpmi_uncore_struct;
44
45 /* Information for each cluster */
46 struct tpmi_uncore_cluster_info {
47 bool root_domain;
48 u8 __iomem *cluster_base;
49 struct uncore_data uncore_data;
50 struct tpmi_uncore_struct *uncore_root;
51 };
52
53 /* Information for each power domain */
54 struct tpmi_uncore_power_domain_info {
55 u8 __iomem *uncore_base;
56 int ufs_header_ver;
57 int cluster_count;
58 struct tpmi_uncore_cluster_info *cluster_infos;
59 };
60
61 /* Information for all power domains in a package */
62 struct tpmi_uncore_struct {
63 int power_domain_count;
64 int max_ratio;
65 int min_ratio;
66 struct tpmi_uncore_power_domain_info *pd_info;
67 struct tpmi_uncore_cluster_info root_cluster;
68 };
69
70 #define UNCORE_GENMASK_MIN_RATIO GENMASK_ULL(21, 15)
71 #define UNCORE_GENMASK_MAX_RATIO GENMASK_ULL(14, 8)
72 #define UNCORE_GENMASK_CURRENT_RATIO GENMASK_ULL(6, 0)
73
74 /* Helper function to read MMIO offset for max/min control frequency */
read_control_freq(struct tpmi_uncore_cluster_info * cluster_info,unsigned int * min,unsigned int * max)75 static void read_control_freq(struct tpmi_uncore_cluster_info *cluster_info,
76 unsigned int *min, unsigned int *max)
77 {
78 u64 control;
79
80 control = readq(cluster_info->cluster_base + UNCORE_CONTROL_INDEX);
81 *max = FIELD_GET(UNCORE_GENMASK_MAX_RATIO, control) * UNCORE_FREQ_KHZ_MULTIPLIER;
82 *min = FIELD_GET(UNCORE_GENMASK_MIN_RATIO, control) * UNCORE_FREQ_KHZ_MULTIPLIER;
83 }
84
85 #define UNCORE_MAX_RATIO FIELD_MAX(UNCORE_GENMASK_MAX_RATIO)
86
87 /* Callback for sysfs read for max/min frequencies. Called under mutex locks */
uncore_read_control_freq(struct uncore_data * data,unsigned int * min,unsigned int * max)88 static int uncore_read_control_freq(struct uncore_data *data, unsigned int *min,
89 unsigned int *max)
90 {
91 struct tpmi_uncore_cluster_info *cluster_info;
92
93 cluster_info = container_of(data, struct tpmi_uncore_cluster_info, uncore_data);
94
95 if (cluster_info->root_domain) {
96 struct tpmi_uncore_struct *uncore_root = cluster_info->uncore_root;
97 int i, _min = 0, _max = 0;
98
99 *min = UNCORE_MAX_RATIO * UNCORE_FREQ_KHZ_MULTIPLIER;
100 *max = 0;
101
102 /*
103 * Get the max/min by looking at each cluster. Get the lowest
104 * min and highest max.
105 */
106 for (i = 0; i < uncore_root->power_domain_count; ++i) {
107 int j;
108
109 for (j = 0; j < uncore_root->pd_info[i].cluster_count; ++j) {
110 read_control_freq(&uncore_root->pd_info[i].cluster_infos[j],
111 &_min, &_max);
112 if (*min > _min)
113 *min = _min;
114 if (*max < _max)
115 *max = _max;
116 }
117 }
118 return 0;
119 }
120
121 read_control_freq(cluster_info, min, max);
122
123 return 0;
124 }
125
126 /* Helper function to write MMIO offset for max/min control frequency */
write_control_freq(struct tpmi_uncore_cluster_info * cluster_info,unsigned int input,unsigned int min_max)127 static void write_control_freq(struct tpmi_uncore_cluster_info *cluster_info, unsigned int input,
128 unsigned int min_max)
129 {
130 u64 control;
131
132 control = readq(cluster_info->cluster_base + UNCORE_CONTROL_INDEX);
133
134 if (min_max) {
135 control &= ~UNCORE_GENMASK_MAX_RATIO;
136 control |= FIELD_PREP(UNCORE_GENMASK_MAX_RATIO, input);
137 } else {
138 control &= ~UNCORE_GENMASK_MIN_RATIO;
139 control |= FIELD_PREP(UNCORE_GENMASK_MIN_RATIO, input);
140 }
141
142 writeq(control, (cluster_info->cluster_base + UNCORE_CONTROL_INDEX));
143 }
144
145 /* Callback for sysfs write for max/min frequencies. Called under mutex locks */
uncore_write_control_freq(struct uncore_data * data,unsigned int input,unsigned int min_max)146 static int uncore_write_control_freq(struct uncore_data *data, unsigned int input,
147 unsigned int min_max)
148 {
149 struct tpmi_uncore_cluster_info *cluster_info;
150 struct tpmi_uncore_struct *uncore_root;
151
152 input /= UNCORE_FREQ_KHZ_MULTIPLIER;
153 if (!input || input > UNCORE_MAX_RATIO)
154 return -EINVAL;
155
156 cluster_info = container_of(data, struct tpmi_uncore_cluster_info, uncore_data);
157 uncore_root = cluster_info->uncore_root;
158
159 /* Update each cluster in a package */
160 if (cluster_info->root_domain) {
161 struct tpmi_uncore_struct *uncore_root = cluster_info->uncore_root;
162 int i;
163
164 for (i = 0; i < uncore_root->power_domain_count; ++i) {
165 int j;
166
167 for (j = 0; j < uncore_root->pd_info[i].cluster_count; ++j)
168 write_control_freq(&uncore_root->pd_info[i].cluster_infos[j],
169 input, min_max);
170 }
171
172 if (min_max)
173 uncore_root->max_ratio = input;
174 else
175 uncore_root->min_ratio = input;
176
177 return 0;
178 }
179
180 if (min_max && uncore_root->max_ratio && uncore_root->max_ratio < input)
181 return -EINVAL;
182
183 if (!min_max && uncore_root->min_ratio && uncore_root->min_ratio > input)
184 return -EINVAL;
185
186 write_control_freq(cluster_info, input, min_max);
187
188 return 0;
189 }
190
191 /* Callback for sysfs read for the current uncore frequency. Called under mutex locks */
uncore_read_freq(struct uncore_data * data,unsigned int * freq)192 static int uncore_read_freq(struct uncore_data *data, unsigned int *freq)
193 {
194 struct tpmi_uncore_cluster_info *cluster_info;
195 u64 status;
196
197 cluster_info = container_of(data, struct tpmi_uncore_cluster_info, uncore_data);
198 if (cluster_info->root_domain)
199 return -ENODATA;
200
201 status = readq((u8 __iomem *)cluster_info->cluster_base + UNCORE_STATUS_INDEX);
202 *freq = FIELD_GET(UNCORE_GENMASK_CURRENT_RATIO, status) * UNCORE_FREQ_KHZ_MULTIPLIER;
203
204 return 0;
205 }
206
remove_cluster_entries(struct tpmi_uncore_struct * tpmi_uncore)207 static void remove_cluster_entries(struct tpmi_uncore_struct *tpmi_uncore)
208 {
209 int i;
210
211 for (i = 0; i < tpmi_uncore->power_domain_count; ++i) {
212 struct tpmi_uncore_power_domain_info *pd_info;
213 int j;
214
215 pd_info = &tpmi_uncore->pd_info[i];
216 if (!pd_info->uncore_base)
217 continue;
218
219 for (j = 0; j < pd_info->cluster_count; ++j) {
220 struct tpmi_uncore_cluster_info *cluster_info;
221
222 cluster_info = &pd_info->cluster_infos[j];
223 uncore_freq_remove_die_entry(&cluster_info->uncore_data);
224 }
225 }
226 }
227
228 #define UNCORE_VERSION_MASK GENMASK_ULL(7, 0)
229 #define UNCORE_LOCAL_FABRIC_CLUSTER_ID_MASK GENMASK_ULL(15, 8)
230 #define UNCORE_CLUSTER_OFF_MASK GENMASK_ULL(7, 0)
231 #define UNCORE_MAX_CLUSTER_PER_DOMAIN 8
232
uncore_probe(struct auxiliary_device * auxdev,const struct auxiliary_device_id * id)233 static int uncore_probe(struct auxiliary_device *auxdev, const struct auxiliary_device_id *id)
234 {
235 struct intel_tpmi_plat_info *plat_info;
236 struct tpmi_uncore_struct *tpmi_uncore;
237 bool uncore_sysfs_added = false;
238 int ret, i, pkg = 0;
239 int num_resources;
240
241 /* Get number of power domains, which is equal to number of resources */
242 num_resources = tpmi_get_resource_count(auxdev);
243 if (!num_resources)
244 return -EINVAL;
245
246 /* Register callbacks to uncore core */
247 ret = uncore_freq_common_init(uncore_read_control_freq, uncore_write_control_freq,
248 uncore_read_freq);
249 if (ret)
250 return ret;
251
252 /* Allocate uncore instance per package */
253 tpmi_uncore = devm_kzalloc(&auxdev->dev, sizeof(*tpmi_uncore), GFP_KERNEL);
254 if (!tpmi_uncore) {
255 ret = -ENOMEM;
256 goto err_rem_common;
257 }
258
259 /* Allocate memory for all power domains in a package */
260 tpmi_uncore->pd_info = devm_kcalloc(&auxdev->dev, num_resources,
261 sizeof(*tpmi_uncore->pd_info),
262 GFP_KERNEL);
263 if (!tpmi_uncore->pd_info) {
264 ret = -ENOMEM;
265 goto err_rem_common;
266 }
267
268 tpmi_uncore->power_domain_count = num_resources;
269
270 /* Get the package ID from the TPMI core */
271 plat_info = tpmi_get_platform_data(auxdev);
272 if (plat_info)
273 pkg = plat_info->package_id;
274 else
275 dev_info(&auxdev->dev, "Platform information is NULL\n");
276
277 for (i = 0; i < num_resources; ++i) {
278 struct tpmi_uncore_power_domain_info *pd_info;
279 struct resource *res;
280 u64 cluster_offset;
281 u8 cluster_mask;
282 int mask, j;
283 u64 header;
284
285 res = tpmi_get_resource_at_index(auxdev, i);
286 if (!res)
287 continue;
288
289 pd_info = &tpmi_uncore->pd_info[i];
290
291 pd_info->uncore_base = devm_ioremap_resource(&auxdev->dev, res);
292 if (IS_ERR(pd_info->uncore_base)) {
293 ret = PTR_ERR(pd_info->uncore_base);
294 /*
295 * Set to NULL so that clean up can still remove other
296 * entries already created if any by
297 * remove_cluster_entries()
298 */
299 pd_info->uncore_base = NULL;
300 goto remove_clusters;
301 }
302
303 /* Check for version and skip this resource if there is mismatch */
304 header = readq(pd_info->uncore_base);
305 pd_info->ufs_header_ver = header & UNCORE_VERSION_MASK;
306 if (pd_info->ufs_header_ver != UNCORE_HEADER_VERSION) {
307 dev_info(&auxdev->dev, "Uncore: Unsupported version:%d\n",
308 pd_info->ufs_header_ver);
309 continue;
310 }
311
312 /* Get Cluster ID Mask */
313 cluster_mask = FIELD_GET(UNCORE_LOCAL_FABRIC_CLUSTER_ID_MASK, header);
314 if (!cluster_mask) {
315 dev_info(&auxdev->dev, "Uncore: Invalid cluster mask:%x\n", cluster_mask);
316 continue;
317 }
318
319 /* Find out number of clusters in this resource */
320 pd_info->cluster_count = hweight8(cluster_mask);
321
322 pd_info->cluster_infos = devm_kcalloc(&auxdev->dev, pd_info->cluster_count,
323 sizeof(struct tpmi_uncore_cluster_info),
324 GFP_KERNEL);
325 if (!pd_info->cluster_infos) {
326 ret = -ENOMEM;
327 goto remove_clusters;
328 }
329 /*
330 * Each byte in the register point to status and control
331 * registers belonging to cluster id 0-8.
332 */
333 cluster_offset = readq(pd_info->uncore_base +
334 UNCORE_FABRIC_CLUSTER_OFFSET);
335
336 for (j = 0; j < pd_info->cluster_count; ++j) {
337 struct tpmi_uncore_cluster_info *cluster_info;
338
339 /* Get the offset for this cluster */
340 mask = (cluster_offset & UNCORE_CLUSTER_OFF_MASK);
341 /* Offset in QWORD, so change to bytes */
342 mask <<= 3;
343
344 cluster_info = &pd_info->cluster_infos[j];
345
346 cluster_info->cluster_base = pd_info->uncore_base + mask;
347
348 cluster_info->uncore_data.package_id = pkg;
349 /* There are no dies like Cascade Lake */
350 cluster_info->uncore_data.die_id = 0;
351 cluster_info->uncore_data.domain_id = i;
352 cluster_info->uncore_data.cluster_id = j;
353
354 cluster_info->uncore_root = tpmi_uncore;
355
356 ret = uncore_freq_add_entry(&cluster_info->uncore_data, 0);
357 if (ret) {
358 cluster_info->cluster_base = NULL;
359 goto remove_clusters;
360 }
361 /* Point to next cluster offset */
362 cluster_offset >>= UNCORE_MAX_CLUSTER_PER_DOMAIN;
363 uncore_sysfs_added = true;
364 }
365 }
366
367 if (!uncore_sysfs_added) {
368 ret = -ENODEV;
369 goto remove_clusters;
370 }
371
372 auxiliary_set_drvdata(auxdev, tpmi_uncore);
373
374 tpmi_uncore->root_cluster.root_domain = true;
375 tpmi_uncore->root_cluster.uncore_root = tpmi_uncore;
376
377 tpmi_uncore->root_cluster.uncore_data.package_id = pkg;
378 tpmi_uncore->root_cluster.uncore_data.domain_id = UNCORE_DOMAIN_ID_INVALID;
379 ret = uncore_freq_add_entry(&tpmi_uncore->root_cluster.uncore_data, 0);
380 if (ret)
381 goto remove_clusters;
382
383 return 0;
384
385 remove_clusters:
386 remove_cluster_entries(tpmi_uncore);
387 err_rem_common:
388 uncore_freq_common_exit();
389
390 return ret;
391 }
392
uncore_remove(struct auxiliary_device * auxdev)393 static void uncore_remove(struct auxiliary_device *auxdev)
394 {
395 struct tpmi_uncore_struct *tpmi_uncore = auxiliary_get_drvdata(auxdev);
396
397 uncore_freq_remove_die_entry(&tpmi_uncore->root_cluster.uncore_data);
398 remove_cluster_entries(tpmi_uncore);
399
400 uncore_freq_common_exit();
401 }
402
403 static const struct auxiliary_device_id intel_uncore_id_table[] = {
404 { .name = "intel_vsec.tpmi-uncore" },
405 {}
406 };
407 MODULE_DEVICE_TABLE(auxiliary, intel_uncore_id_table);
408
409 static struct auxiliary_driver intel_uncore_aux_driver = {
410 .id_table = intel_uncore_id_table,
411 .remove = uncore_remove,
412 .probe = uncore_probe,
413 };
414
415 module_auxiliary_driver(intel_uncore_aux_driver);
416
417 MODULE_IMPORT_NS(INTEL_TPMI);
418 MODULE_IMPORT_NS(INTEL_UNCORE_FREQUENCY);
419 MODULE_DESCRIPTION("Intel TPMI UFS Driver");
420 MODULE_LICENSE("GPL");
421