1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * uncore-frquency-tpmi: Uncore frequency scaling using TPMI 4 * 5 * Copyright (c) 2023, Intel Corporation. 6 * All Rights Reserved. 7 * 8 * The hardware interface to read/write is basically substitution of 9 * MSR 0x620 and 0x621. 10 * There are specific MMIO offset and bits to get/set minimum and 11 * maximum uncore ratio, similar to MSRs. 12 * The scope of the uncore MSRs was package scope. But TPMI allows 13 * new gen CPUs to have multiple uncore controls at uncore-cluster 14 * level. Each package can have multiple power domains which further 15 * can have multiple clusters. 16 * Here number of power domains = number of resources in this aux 17 * device. There are offsets and bits to discover number of clusters 18 * and offset for each cluster level controls. 19 * 20 */ 21 22 #include <linux/auxiliary_bus.h> 23 #include <linux/bitfield.h> 24 #include <linux/bits.h> 25 #include <linux/io.h> 26 #include <linux/module.h> 27 #include <linux/intel_tpmi.h> 28 29 #include "uncore-frequency-common.h" 30 31 #define UNCORE_HEADER_VERSION 1 32 #define UNCORE_HEADER_INDEX 0 33 #define UNCORE_FABRIC_CLUSTER_OFFSET 8 34 35 /* status + control + adv_ctl1 + adv_ctl2 */ 36 #define UNCORE_FABRIC_CLUSTER_SIZE (4 * 8) 37 38 #define UNCORE_STATUS_INDEX 0 39 #define UNCORE_CONTROL_INDEX 8 40 41 #define UNCORE_FREQ_KHZ_MULTIPLIER 100000 42 43 struct tpmi_uncore_struct; 44 45 /* Information for each cluster */ 46 struct tpmi_uncore_cluster_info { 47 bool root_domain; 48 u8 __iomem *cluster_base; 49 struct uncore_data uncore_data; 50 struct tpmi_uncore_struct *uncore_root; 51 }; 52 53 /* Information for each power domain */ 54 struct tpmi_uncore_power_domain_info { 55 u8 __iomem *uncore_base; 56 int ufs_header_ver; 57 int cluster_count; 58 struct tpmi_uncore_cluster_info *cluster_infos; 59 }; 60 61 /* Information for all power domains in a package */ 62 struct tpmi_uncore_struct { 63 int power_domain_count; 64 int max_ratio; 65 int min_ratio; 66 struct tpmi_uncore_power_domain_info *pd_info; 67 struct tpmi_uncore_cluster_info root_cluster; 68 }; 69 70 #define UNCORE_GENMASK_MIN_RATIO GENMASK_ULL(21, 15) 71 #define UNCORE_GENMASK_MAX_RATIO GENMASK_ULL(14, 8) 72 #define UNCORE_GENMASK_CURRENT_RATIO GENMASK_ULL(6, 0) 73 74 /* Helper function to read MMIO offset for max/min control frequency */ 75 static void read_control_freq(struct tpmi_uncore_cluster_info *cluster_info, 76 unsigned int *min, unsigned int *max) 77 { 78 u64 control; 79 80 control = readq(cluster_info->cluster_base + UNCORE_CONTROL_INDEX); 81 *max = FIELD_GET(UNCORE_GENMASK_MAX_RATIO, control) * UNCORE_FREQ_KHZ_MULTIPLIER; 82 *min = FIELD_GET(UNCORE_GENMASK_MIN_RATIO, control) * UNCORE_FREQ_KHZ_MULTIPLIER; 83 } 84 85 #define UNCORE_MAX_RATIO FIELD_MAX(UNCORE_GENMASK_MAX_RATIO) 86 87 /* Callback for sysfs read for max/min frequencies. Called under mutex locks */ 88 static int uncore_read_control_freq(struct uncore_data *data, unsigned int *min, 89 unsigned int *max) 90 { 91 struct tpmi_uncore_cluster_info *cluster_info; 92 93 cluster_info = container_of(data, struct tpmi_uncore_cluster_info, uncore_data); 94 95 if (cluster_info->root_domain) { 96 struct tpmi_uncore_struct *uncore_root = cluster_info->uncore_root; 97 int i, _min = 0, _max = 0; 98 99 *min = UNCORE_MAX_RATIO * UNCORE_FREQ_KHZ_MULTIPLIER; 100 *max = 0; 101 102 /* 103 * Get the max/min by looking at each cluster. Get the lowest 104 * min and highest max. 105 */ 106 for (i = 0; i < uncore_root->power_domain_count; ++i) { 107 int j; 108 109 for (j = 0; j < uncore_root->pd_info[i].cluster_count; ++j) { 110 read_control_freq(&uncore_root->pd_info[i].cluster_infos[j], 111 &_min, &_max); 112 if (*min > _min) 113 *min = _min; 114 if (*max < _max) 115 *max = _max; 116 } 117 } 118 return 0; 119 } 120 121 read_control_freq(cluster_info, min, max); 122 123 return 0; 124 } 125 126 /* Helper function to write MMIO offset for max/min control frequency */ 127 static void write_control_freq(struct tpmi_uncore_cluster_info *cluster_info, unsigned int input, 128 unsigned int min_max) 129 { 130 u64 control; 131 132 control = readq(cluster_info->cluster_base + UNCORE_CONTROL_INDEX); 133 134 if (min_max) { 135 control &= ~UNCORE_GENMASK_MAX_RATIO; 136 control |= FIELD_PREP(UNCORE_GENMASK_MAX_RATIO, input); 137 } else { 138 control &= ~UNCORE_GENMASK_MIN_RATIO; 139 control |= FIELD_PREP(UNCORE_GENMASK_MIN_RATIO, input); 140 } 141 142 writeq(control, (cluster_info->cluster_base + UNCORE_CONTROL_INDEX)); 143 } 144 145 /* Callback for sysfs write for max/min frequencies. Called under mutex locks */ 146 static int uncore_write_control_freq(struct uncore_data *data, unsigned int input, 147 unsigned int min_max) 148 { 149 struct tpmi_uncore_cluster_info *cluster_info; 150 struct tpmi_uncore_struct *uncore_root; 151 152 input /= UNCORE_FREQ_KHZ_MULTIPLIER; 153 if (!input || input > UNCORE_MAX_RATIO) 154 return -EINVAL; 155 156 cluster_info = container_of(data, struct tpmi_uncore_cluster_info, uncore_data); 157 uncore_root = cluster_info->uncore_root; 158 159 /* Update each cluster in a package */ 160 if (cluster_info->root_domain) { 161 struct tpmi_uncore_struct *uncore_root = cluster_info->uncore_root; 162 int i; 163 164 for (i = 0; i < uncore_root->power_domain_count; ++i) { 165 int j; 166 167 for (j = 0; j < uncore_root->pd_info[i].cluster_count; ++j) 168 write_control_freq(&uncore_root->pd_info[i].cluster_infos[j], 169 input, min_max); 170 } 171 172 if (min_max) 173 uncore_root->max_ratio = input; 174 else 175 uncore_root->min_ratio = input; 176 177 return 0; 178 } 179 180 if (min_max && uncore_root->max_ratio && uncore_root->max_ratio < input) 181 return -EINVAL; 182 183 if (!min_max && uncore_root->min_ratio && uncore_root->min_ratio > input) 184 return -EINVAL; 185 186 write_control_freq(cluster_info, input, min_max); 187 188 return 0; 189 } 190 191 /* Callback for sysfs read for the current uncore frequency. Called under mutex locks */ 192 static int uncore_read_freq(struct uncore_data *data, unsigned int *freq) 193 { 194 struct tpmi_uncore_cluster_info *cluster_info; 195 u64 status; 196 197 cluster_info = container_of(data, struct tpmi_uncore_cluster_info, uncore_data); 198 if (cluster_info->root_domain) 199 return -ENODATA; 200 201 status = readq((u8 __iomem *)cluster_info->cluster_base + UNCORE_STATUS_INDEX); 202 *freq = FIELD_GET(UNCORE_GENMASK_CURRENT_RATIO, status) * UNCORE_FREQ_KHZ_MULTIPLIER; 203 204 return 0; 205 } 206 207 static void remove_cluster_entries(struct tpmi_uncore_struct *tpmi_uncore) 208 { 209 int i; 210 211 for (i = 0; i < tpmi_uncore->power_domain_count; ++i) { 212 struct tpmi_uncore_power_domain_info *pd_info; 213 int j; 214 215 pd_info = &tpmi_uncore->pd_info[i]; 216 if (!pd_info->uncore_base) 217 continue; 218 219 for (j = 0; j < pd_info->cluster_count; ++j) { 220 struct tpmi_uncore_cluster_info *cluster_info; 221 222 cluster_info = &pd_info->cluster_infos[j]; 223 uncore_freq_remove_die_entry(&cluster_info->uncore_data); 224 } 225 } 226 } 227 228 #define UNCORE_VERSION_MASK GENMASK_ULL(7, 0) 229 #define UNCORE_LOCAL_FABRIC_CLUSTER_ID_MASK GENMASK_ULL(15, 8) 230 #define UNCORE_CLUSTER_OFF_MASK GENMASK_ULL(7, 0) 231 #define UNCORE_MAX_CLUSTER_PER_DOMAIN 8 232 233 static int uncore_probe(struct auxiliary_device *auxdev, const struct auxiliary_device_id *id) 234 { 235 struct intel_tpmi_plat_info *plat_info; 236 struct tpmi_uncore_struct *tpmi_uncore; 237 bool uncore_sysfs_added = false; 238 int ret, i, pkg = 0; 239 int num_resources; 240 241 /* Get number of power domains, which is equal to number of resources */ 242 num_resources = tpmi_get_resource_count(auxdev); 243 if (!num_resources) 244 return -EINVAL; 245 246 /* Register callbacks to uncore core */ 247 ret = uncore_freq_common_init(uncore_read_control_freq, uncore_write_control_freq, 248 uncore_read_freq); 249 if (ret) 250 return ret; 251 252 /* Allocate uncore instance per package */ 253 tpmi_uncore = devm_kzalloc(&auxdev->dev, sizeof(*tpmi_uncore), GFP_KERNEL); 254 if (!tpmi_uncore) { 255 ret = -ENOMEM; 256 goto err_rem_common; 257 } 258 259 /* Allocate memory for all power domains in a package */ 260 tpmi_uncore->pd_info = devm_kcalloc(&auxdev->dev, num_resources, 261 sizeof(*tpmi_uncore->pd_info), 262 GFP_KERNEL); 263 if (!tpmi_uncore->pd_info) { 264 ret = -ENOMEM; 265 goto err_rem_common; 266 } 267 268 tpmi_uncore->power_domain_count = num_resources; 269 270 /* Get the package ID from the TPMI core */ 271 plat_info = tpmi_get_platform_data(auxdev); 272 if (plat_info) 273 pkg = plat_info->package_id; 274 else 275 dev_info(&auxdev->dev, "Platform information is NULL\n"); 276 277 for (i = 0; i < num_resources; ++i) { 278 struct tpmi_uncore_power_domain_info *pd_info; 279 struct resource *res; 280 u64 cluster_offset; 281 u8 cluster_mask; 282 int mask, j; 283 u64 header; 284 285 res = tpmi_get_resource_at_index(auxdev, i); 286 if (!res) 287 continue; 288 289 pd_info = &tpmi_uncore->pd_info[i]; 290 291 pd_info->uncore_base = devm_ioremap_resource(&auxdev->dev, res); 292 if (IS_ERR(pd_info->uncore_base)) { 293 ret = PTR_ERR(pd_info->uncore_base); 294 /* 295 * Set to NULL so that clean up can still remove other 296 * entries already created if any by 297 * remove_cluster_entries() 298 */ 299 pd_info->uncore_base = NULL; 300 goto remove_clusters; 301 } 302 303 /* Check for version and skip this resource if there is mismatch */ 304 header = readq(pd_info->uncore_base); 305 pd_info->ufs_header_ver = header & UNCORE_VERSION_MASK; 306 if (pd_info->ufs_header_ver != UNCORE_HEADER_VERSION) { 307 dev_info(&auxdev->dev, "Uncore: Unsupported version:%d\n", 308 pd_info->ufs_header_ver); 309 continue; 310 } 311 312 /* Get Cluster ID Mask */ 313 cluster_mask = FIELD_GET(UNCORE_LOCAL_FABRIC_CLUSTER_ID_MASK, header); 314 if (!cluster_mask) { 315 dev_info(&auxdev->dev, "Uncore: Invalid cluster mask:%x\n", cluster_mask); 316 continue; 317 } 318 319 /* Find out number of clusters in this resource */ 320 pd_info->cluster_count = hweight8(cluster_mask); 321 322 pd_info->cluster_infos = devm_kcalloc(&auxdev->dev, pd_info->cluster_count, 323 sizeof(struct tpmi_uncore_cluster_info), 324 GFP_KERNEL); 325 if (!pd_info->cluster_infos) { 326 ret = -ENOMEM; 327 goto remove_clusters; 328 } 329 /* 330 * Each byte in the register point to status and control 331 * registers belonging to cluster id 0-8. 332 */ 333 cluster_offset = readq(pd_info->uncore_base + 334 UNCORE_FABRIC_CLUSTER_OFFSET); 335 336 for (j = 0; j < pd_info->cluster_count; ++j) { 337 struct tpmi_uncore_cluster_info *cluster_info; 338 339 /* Get the offset for this cluster */ 340 mask = (cluster_offset & UNCORE_CLUSTER_OFF_MASK); 341 /* Offset in QWORD, so change to bytes */ 342 mask <<= 3; 343 344 cluster_info = &pd_info->cluster_infos[j]; 345 346 cluster_info->cluster_base = pd_info->uncore_base + mask; 347 348 cluster_info->uncore_data.package_id = pkg; 349 /* There are no dies like Cascade Lake */ 350 cluster_info->uncore_data.die_id = 0; 351 cluster_info->uncore_data.domain_id = i; 352 cluster_info->uncore_data.cluster_id = j; 353 354 cluster_info->uncore_root = tpmi_uncore; 355 356 ret = uncore_freq_add_entry(&cluster_info->uncore_data, 0); 357 if (ret) { 358 cluster_info->cluster_base = NULL; 359 goto remove_clusters; 360 } 361 /* Point to next cluster offset */ 362 cluster_offset >>= UNCORE_MAX_CLUSTER_PER_DOMAIN; 363 uncore_sysfs_added = true; 364 } 365 } 366 367 if (!uncore_sysfs_added) { 368 ret = -ENODEV; 369 goto remove_clusters; 370 } 371 372 auxiliary_set_drvdata(auxdev, tpmi_uncore); 373 374 tpmi_uncore->root_cluster.root_domain = true; 375 tpmi_uncore->root_cluster.uncore_root = tpmi_uncore; 376 377 tpmi_uncore->root_cluster.uncore_data.package_id = pkg; 378 tpmi_uncore->root_cluster.uncore_data.domain_id = UNCORE_DOMAIN_ID_INVALID; 379 ret = uncore_freq_add_entry(&tpmi_uncore->root_cluster.uncore_data, 0); 380 if (ret) 381 goto remove_clusters; 382 383 return 0; 384 385 remove_clusters: 386 remove_cluster_entries(tpmi_uncore); 387 err_rem_common: 388 uncore_freq_common_exit(); 389 390 return ret; 391 } 392 393 static void uncore_remove(struct auxiliary_device *auxdev) 394 { 395 struct tpmi_uncore_struct *tpmi_uncore = auxiliary_get_drvdata(auxdev); 396 397 uncore_freq_remove_die_entry(&tpmi_uncore->root_cluster.uncore_data); 398 remove_cluster_entries(tpmi_uncore); 399 400 uncore_freq_common_exit(); 401 } 402 403 static const struct auxiliary_device_id intel_uncore_id_table[] = { 404 { .name = "intel_vsec.tpmi-uncore" }, 405 {} 406 }; 407 MODULE_DEVICE_TABLE(auxiliary, intel_uncore_id_table); 408 409 static struct auxiliary_driver intel_uncore_aux_driver = { 410 .id_table = intel_uncore_id_table, 411 .remove = uncore_remove, 412 .probe = uncore_probe, 413 }; 414 415 module_auxiliary_driver(intel_uncore_aux_driver); 416 417 MODULE_IMPORT_NS(INTEL_TPMI); 418 MODULE_IMPORT_NS(INTEL_UNCORE_FREQUENCY); 419 MODULE_DESCRIPTION("Intel TPMI UFS Driver"); 420 MODULE_LICENSE("GPL"); 421