1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * uncore-frquency-tpmi: Uncore frequency scaling using TPMI
4  *
5  * Copyright (c) 2023, Intel Corporation.
6  * All Rights Reserved.
7  *
8  * The hardware interface to read/write is basically substitution of
9  * MSR 0x620 and 0x621.
10  * There are specific MMIO offset and bits to get/set minimum and
11  * maximum uncore ratio, similar to MSRs.
12  * The scope of the uncore MSRs was package scope. But TPMI allows
13  * new gen CPUs to have multiple uncore controls at uncore-cluster
14  * level. Each package can have multiple power domains which further
15  * can have multiple clusters.
16  * Here number of power domains = number of resources in this aux
17  * device. There are offsets and bits to discover number of clusters
18  * and offset for each cluster level controls.
19  *
20  */
21 
22 #include <linux/auxiliary_bus.h>
23 #include <linux/bitfield.h>
24 #include <linux/bits.h>
25 #include <linux/io.h>
26 #include <linux/module.h>
27 #include <linux/intel_tpmi.h>
28 
29 #include "uncore-frequency-common.h"
30 
31 #define	UNCORE_HEADER_VERSION		1
32 #define UNCORE_HEADER_INDEX		0
33 #define UNCORE_FABRIC_CLUSTER_OFFSET	8
34 
35 /* status + control + adv_ctl1 + adv_ctl2 */
36 #define UNCORE_FABRIC_CLUSTER_SIZE	(4 * 8)
37 
38 #define UNCORE_STATUS_INDEX		0
39 #define UNCORE_CONTROL_INDEX		8
40 
41 #define UNCORE_FREQ_KHZ_MULTIPLIER	100000
42 
43 struct tpmi_uncore_struct;
44 
45 /* Information for each cluster */
46 struct tpmi_uncore_cluster_info {
47 	bool root_domain;
48 	u8 __iomem *cluster_base;
49 	struct uncore_data uncore_data;
50 	struct tpmi_uncore_struct *uncore_root;
51 };
52 
53 /* Information for each power domain */
54 struct tpmi_uncore_power_domain_info {
55 	u8 __iomem *uncore_base;
56 	int ufs_header_ver;
57 	int cluster_count;
58 	struct tpmi_uncore_cluster_info *cluster_infos;
59 };
60 
61 /* Information for all power domains in a package */
62 struct tpmi_uncore_struct {
63 	int power_domain_count;
64 	int max_ratio;
65 	int min_ratio;
66 	struct tpmi_uncore_power_domain_info *pd_info;
67 	struct tpmi_uncore_cluster_info root_cluster;
68 };
69 
70 #define UNCORE_GENMASK_MIN_RATIO	GENMASK_ULL(21, 15)
71 #define UNCORE_GENMASK_MAX_RATIO	GENMASK_ULL(14, 8)
72 #define UNCORE_GENMASK_CURRENT_RATIO	GENMASK_ULL(6, 0)
73 
74 /* Helper function to read MMIO offset for max/min control frequency */
75 static void read_control_freq(struct tpmi_uncore_cluster_info *cluster_info,
76 			     unsigned int *min, unsigned int *max)
77 {
78 	u64 control;
79 
80 	control = readq(cluster_info->cluster_base + UNCORE_CONTROL_INDEX);
81 	*max = FIELD_GET(UNCORE_GENMASK_MAX_RATIO, control) * UNCORE_FREQ_KHZ_MULTIPLIER;
82 	*min = FIELD_GET(UNCORE_GENMASK_MIN_RATIO, control) * UNCORE_FREQ_KHZ_MULTIPLIER;
83 }
84 
85 #define UNCORE_MAX_RATIO	FIELD_MAX(UNCORE_GENMASK_MAX_RATIO)
86 
87 /* Callback for sysfs read for max/min frequencies. Called under mutex locks */
88 static int uncore_read_control_freq(struct uncore_data *data, unsigned int *min,
89 				    unsigned int *max)
90 {
91 	struct tpmi_uncore_cluster_info *cluster_info;
92 
93 	cluster_info = container_of(data, struct tpmi_uncore_cluster_info, uncore_data);
94 
95 	if (cluster_info->root_domain) {
96 		struct tpmi_uncore_struct *uncore_root = cluster_info->uncore_root;
97 		int i, _min = 0, _max = 0;
98 
99 		*min = UNCORE_MAX_RATIO * UNCORE_FREQ_KHZ_MULTIPLIER;
100 		*max = 0;
101 
102 		/*
103 		 * Get the max/min by looking at each cluster. Get the lowest
104 		 * min and highest max.
105 		 */
106 		for (i = 0; i < uncore_root->power_domain_count; ++i) {
107 			int j;
108 
109 			for (j = 0; j < uncore_root->pd_info[i].cluster_count; ++j) {
110 				read_control_freq(&uncore_root->pd_info[i].cluster_infos[j],
111 						  &_min, &_max);
112 				if (*min > _min)
113 					*min = _min;
114 				if (*max < _max)
115 					*max = _max;
116 			}
117 		}
118 		return 0;
119 	}
120 
121 	read_control_freq(cluster_info, min, max);
122 
123 	return 0;
124 }
125 
126 /* Helper function to write MMIO offset for max/min control frequency */
127 static void write_control_freq(struct tpmi_uncore_cluster_info *cluster_info, unsigned int input,
128 			      unsigned int min_max)
129 {
130 	u64 control;
131 
132 	control = readq(cluster_info->cluster_base + UNCORE_CONTROL_INDEX);
133 
134 	if (min_max) {
135 		control &= ~UNCORE_GENMASK_MAX_RATIO;
136 		control |= FIELD_PREP(UNCORE_GENMASK_MAX_RATIO, input);
137 	} else {
138 		control &= ~UNCORE_GENMASK_MIN_RATIO;
139 		control |= FIELD_PREP(UNCORE_GENMASK_MIN_RATIO, input);
140 	}
141 
142 	writeq(control, (cluster_info->cluster_base + UNCORE_CONTROL_INDEX));
143 }
144 
145 /* Callback for sysfs write for max/min frequencies. Called under mutex locks */
146 static int uncore_write_control_freq(struct uncore_data *data, unsigned int input,
147 				     unsigned int min_max)
148 {
149 	struct tpmi_uncore_cluster_info *cluster_info;
150 	struct tpmi_uncore_struct *uncore_root;
151 
152 	input /= UNCORE_FREQ_KHZ_MULTIPLIER;
153 	if (!input || input > UNCORE_MAX_RATIO)
154 		return -EINVAL;
155 
156 	cluster_info = container_of(data, struct tpmi_uncore_cluster_info, uncore_data);
157 	uncore_root = cluster_info->uncore_root;
158 
159 	/* Update each cluster in a package */
160 	if (cluster_info->root_domain) {
161 		struct tpmi_uncore_struct *uncore_root = cluster_info->uncore_root;
162 		int i;
163 
164 		for (i = 0; i < uncore_root->power_domain_count; ++i) {
165 			int j;
166 
167 			for (j = 0; j < uncore_root->pd_info[i].cluster_count; ++j)
168 				write_control_freq(&uncore_root->pd_info[i].cluster_infos[j],
169 						  input, min_max);
170 		}
171 
172 		if (min_max)
173 			uncore_root->max_ratio = input;
174 		else
175 			uncore_root->min_ratio = input;
176 
177 		return 0;
178 	}
179 
180 	if (min_max && uncore_root->max_ratio && uncore_root->max_ratio < input)
181 		return -EINVAL;
182 
183 	if (!min_max && uncore_root->min_ratio && uncore_root->min_ratio > input)
184 		return -EINVAL;
185 
186 	write_control_freq(cluster_info, input, min_max);
187 
188 	return 0;
189 }
190 
191 /* Callback for sysfs read for the current uncore frequency. Called under mutex locks */
192 static int uncore_read_freq(struct uncore_data *data, unsigned int *freq)
193 {
194 	struct tpmi_uncore_cluster_info *cluster_info;
195 	u64 status;
196 
197 	cluster_info = container_of(data, struct tpmi_uncore_cluster_info, uncore_data);
198 	if (cluster_info->root_domain)
199 		return -ENODATA;
200 
201 	status = readq((u8 __iomem *)cluster_info->cluster_base + UNCORE_STATUS_INDEX);
202 	*freq = FIELD_GET(UNCORE_GENMASK_CURRENT_RATIO, status) * UNCORE_FREQ_KHZ_MULTIPLIER;
203 
204 	return 0;
205 }
206 
207 static void remove_cluster_entries(struct tpmi_uncore_struct *tpmi_uncore)
208 {
209 	int i;
210 
211 	for (i = 0; i < tpmi_uncore->power_domain_count; ++i) {
212 		struct tpmi_uncore_power_domain_info *pd_info;
213 		int j;
214 
215 		pd_info = &tpmi_uncore->pd_info[i];
216 		if (!pd_info->uncore_base)
217 			continue;
218 
219 		for (j = 0; j < pd_info->cluster_count; ++j) {
220 			struct tpmi_uncore_cluster_info *cluster_info;
221 
222 			cluster_info = &pd_info->cluster_infos[j];
223 			uncore_freq_remove_die_entry(&cluster_info->uncore_data);
224 		}
225 	}
226 }
227 
228 #define UNCORE_VERSION_MASK			GENMASK_ULL(7, 0)
229 #define UNCORE_LOCAL_FABRIC_CLUSTER_ID_MASK	GENMASK_ULL(15, 8)
230 #define UNCORE_CLUSTER_OFF_MASK			GENMASK_ULL(7, 0)
231 #define UNCORE_MAX_CLUSTER_PER_DOMAIN		8
232 
233 static int uncore_probe(struct auxiliary_device *auxdev, const struct auxiliary_device_id *id)
234 {
235 	struct intel_tpmi_plat_info *plat_info;
236 	struct tpmi_uncore_struct *tpmi_uncore;
237 	int ret, i, pkg = 0;
238 	int num_resources;
239 
240 	/* Get number of power domains, which is equal to number of resources */
241 	num_resources = tpmi_get_resource_count(auxdev);
242 	if (!num_resources)
243 		return -EINVAL;
244 
245 	/* Register callbacks to uncore core */
246 	ret = uncore_freq_common_init(uncore_read_control_freq, uncore_write_control_freq,
247 				      uncore_read_freq);
248 	if (ret)
249 		return ret;
250 
251 	/* Allocate uncore instance per package */
252 	tpmi_uncore = devm_kzalloc(&auxdev->dev, sizeof(*tpmi_uncore), GFP_KERNEL);
253 	if (!tpmi_uncore) {
254 		ret = -ENOMEM;
255 		goto err_rem_common;
256 	}
257 
258 	/* Allocate memory for all power domains in a package */
259 	tpmi_uncore->pd_info = devm_kcalloc(&auxdev->dev, num_resources,
260 					    sizeof(*tpmi_uncore->pd_info),
261 					    GFP_KERNEL);
262 	if (!tpmi_uncore->pd_info) {
263 		ret = -ENOMEM;
264 		goto err_rem_common;
265 	}
266 
267 	tpmi_uncore->power_domain_count = num_resources;
268 
269 	/* Get the package ID from the TPMI core */
270 	plat_info = tpmi_get_platform_data(auxdev);
271 	if (plat_info)
272 		pkg = plat_info->package_id;
273 	else
274 		dev_info(&auxdev->dev, "Platform information is NULL\n");
275 
276 	for (i = 0; i < num_resources; ++i) {
277 		struct tpmi_uncore_power_domain_info *pd_info;
278 		struct resource *res;
279 		u64 cluster_offset;
280 		u8 cluster_mask;
281 		int mask, j;
282 		u64 header;
283 
284 		res = tpmi_get_resource_at_index(auxdev, i);
285 		if (!res)
286 			continue;
287 
288 		pd_info = &tpmi_uncore->pd_info[i];
289 
290 		pd_info->uncore_base = devm_ioremap_resource(&auxdev->dev, res);
291 		if (IS_ERR(pd_info->uncore_base)) {
292 			ret = PTR_ERR(pd_info->uncore_base);
293 			/*
294 			 * Set to NULL so that clean up can still remove other
295 			 * entries already created if any by
296 			 * remove_cluster_entries()
297 			 */
298 			pd_info->uncore_base = NULL;
299 			goto remove_clusters;
300 		}
301 
302 		/* Check for version and skip this resource if there is mismatch */
303 		header = readq(pd_info->uncore_base);
304 		pd_info->ufs_header_ver = header & UNCORE_VERSION_MASK;
305 		if (pd_info->ufs_header_ver != UNCORE_HEADER_VERSION) {
306 			dev_info(&auxdev->dev, "Uncore: Unsupported version:%d\n",
307 				pd_info->ufs_header_ver);
308 			continue;
309 		}
310 
311 		/* Get Cluster ID Mask */
312 		cluster_mask = FIELD_GET(UNCORE_LOCAL_FABRIC_CLUSTER_ID_MASK, header);
313 		if (!cluster_mask) {
314 			dev_info(&auxdev->dev, "Uncore: Invalid cluster mask:%x\n", cluster_mask);
315 			continue;
316 		}
317 
318 		/* Find out number of clusters in this resource */
319 		pd_info->cluster_count = hweight8(cluster_mask);
320 
321 		pd_info->cluster_infos = devm_kcalloc(&auxdev->dev, pd_info->cluster_count,
322 						      sizeof(struct tpmi_uncore_cluster_info),
323 						      GFP_KERNEL);
324 		if (!pd_info->cluster_infos) {
325 			ret = -ENOMEM;
326 			goto remove_clusters;
327 		}
328 		/*
329 		 * Each byte in the register point to status and control
330 		 * registers belonging to cluster id 0-8.
331 		 */
332 		cluster_offset = readq(pd_info->uncore_base +
333 					UNCORE_FABRIC_CLUSTER_OFFSET);
334 
335 		for (j = 0; j < pd_info->cluster_count; ++j) {
336 			struct tpmi_uncore_cluster_info *cluster_info;
337 
338 			/* Get the offset for this cluster */
339 			mask = (cluster_offset & UNCORE_CLUSTER_OFF_MASK);
340 			/* Offset in QWORD, so change to bytes */
341 			mask <<= 3;
342 
343 			cluster_info = &pd_info->cluster_infos[j];
344 
345 			cluster_info->cluster_base = pd_info->uncore_base + mask;
346 
347 			cluster_info->uncore_data.package_id = pkg;
348 			/* There are no dies like Cascade Lake */
349 			cluster_info->uncore_data.die_id = 0;
350 			cluster_info->uncore_data.domain_id = i;
351 			cluster_info->uncore_data.cluster_id = j;
352 
353 			cluster_info->uncore_root = tpmi_uncore;
354 
355 			ret = uncore_freq_add_entry(&cluster_info->uncore_data, 0);
356 			if (ret) {
357 				cluster_info->cluster_base = NULL;
358 				goto remove_clusters;
359 			}
360 			/* Point to next cluster offset */
361 			cluster_offset >>= UNCORE_MAX_CLUSTER_PER_DOMAIN;
362 		}
363 	}
364 
365 	auxiliary_set_drvdata(auxdev, tpmi_uncore);
366 
367 	tpmi_uncore->root_cluster.root_domain = true;
368 	tpmi_uncore->root_cluster.uncore_root = tpmi_uncore;
369 
370 	tpmi_uncore->root_cluster.uncore_data.package_id = pkg;
371 	tpmi_uncore->root_cluster.uncore_data.domain_id = UNCORE_DOMAIN_ID_INVALID;
372 	ret = uncore_freq_add_entry(&tpmi_uncore->root_cluster.uncore_data, 0);
373 	if (ret)
374 		goto remove_clusters;
375 
376 	return 0;
377 
378 remove_clusters:
379 	remove_cluster_entries(tpmi_uncore);
380 err_rem_common:
381 	uncore_freq_common_exit();
382 
383 	return ret;
384 }
385 
386 static void uncore_remove(struct auxiliary_device *auxdev)
387 {
388 	struct tpmi_uncore_struct *tpmi_uncore = auxiliary_get_drvdata(auxdev);
389 
390 	uncore_freq_remove_die_entry(&tpmi_uncore->root_cluster.uncore_data);
391 	remove_cluster_entries(tpmi_uncore);
392 
393 	uncore_freq_common_exit();
394 }
395 
396 static const struct auxiliary_device_id intel_uncore_id_table[] = {
397 	{ .name = "intel_vsec.tpmi-uncore" },
398 	{}
399 };
400 MODULE_DEVICE_TABLE(auxiliary, intel_uncore_id_table);
401 
402 static struct auxiliary_driver intel_uncore_aux_driver = {
403 	.id_table       = intel_uncore_id_table,
404 	.remove         = uncore_remove,
405 	.probe          = uncore_probe,
406 };
407 
408 module_auxiliary_driver(intel_uncore_aux_driver);
409 
410 MODULE_IMPORT_NS(INTEL_TPMI);
411 MODULE_IMPORT_NS(INTEL_UNCORE_FREQUENCY);
412 MODULE_DESCRIPTION("Intel TPMI UFS Driver");
413 MODULE_LICENSE("GPL");
414