1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2014-2018, The Linux Foundation. All rights reserved. 4 * Copyright (C) 2021-2022 Linaro Ltd 5 * Author: Krzysztof Kozlowski <krzysztof.kozlowski@linaro.org>, based on 6 * previous work of Thara Gopinath and msm-4.9 downstream sources. 7 */ 8 9 #include <linux/err.h> 10 #include <linux/interconnect.h> 11 #include <linux/interrupt.h> 12 #include <linux/io.h> 13 #include <linux/kernel.h> 14 #include <linux/module.h> 15 #include <linux/of.h> 16 #include <linux/platform_device.h> 17 #include <linux/pm_opp.h> 18 #include <linux/regmap.h> 19 #include <linux/sizes.h> 20 21 /* 22 * The BWMON samples data throughput within 'sample_ms' time. With three 23 * configurable thresholds (Low, Medium and High) gives four windows (called 24 * zones) of current bandwidth: 25 * 26 * Zone 0: byte count < THRES_LO 27 * Zone 1: THRES_LO < byte count < THRES_MED 28 * Zone 2: THRES_MED < byte count < THRES_HIGH 29 * Zone 3: THRES_HIGH < byte count 30 * 31 * Zones 0 and 2 are not used by this driver. 32 */ 33 34 /* Internal sampling clock frequency */ 35 #define HW_TIMER_HZ 19200000 36 37 #define BWMON_V4_GLOBAL_IRQ_CLEAR 0x108 38 #define BWMON_V4_GLOBAL_IRQ_ENABLE 0x10c 39 /* 40 * All values here and further are matching regmap fields, so without absolute 41 * register offsets. 42 */ 43 #define BWMON_V4_GLOBAL_IRQ_ENABLE_ENABLE BIT(0) 44 45 /* 46 * Starting with SDM845, the BWMON4 register space has changed a bit: 47 * the global registers were jammed into the beginning of the monitor region. 48 * To keep the proper offsets, one would have to map <GLOBAL_BASE 0x200> and 49 * <GLOBAL_BASE+0x100 0x300>, which is straight up wrong. 50 * To facilitate for that, while allowing the older, arguably more proper 51 * implementations to work, offset the global registers by -0x100 to avoid 52 * having to map half of the global registers twice. 53 */ 54 #define BWMON_V4_845_OFFSET 0x100 55 #define BWMON_V4_GLOBAL_IRQ_CLEAR_845 (BWMON_V4_GLOBAL_IRQ_CLEAR - BWMON_V4_845_OFFSET) 56 #define BWMON_V4_GLOBAL_IRQ_ENABLE_845 (BWMON_V4_GLOBAL_IRQ_ENABLE - BWMON_V4_845_OFFSET) 57 58 #define BWMON_V4_IRQ_STATUS 0x100 59 #define BWMON_V4_IRQ_CLEAR 0x108 60 61 #define BWMON_V4_IRQ_ENABLE 0x10c 62 #define BWMON_IRQ_ENABLE_MASK (BIT(1) | BIT(3)) 63 #define BWMON_V5_IRQ_STATUS 0x000 64 #define BWMON_V5_IRQ_CLEAR 0x008 65 #define BWMON_V5_IRQ_ENABLE 0x00c 66 67 #define BWMON_V4_ENABLE 0x2a0 68 #define BWMON_V5_ENABLE 0x010 69 #define BWMON_ENABLE_ENABLE BIT(0) 70 71 #define BWMON_V4_CLEAR 0x2a4 72 #define BWMON_V5_CLEAR 0x014 73 #define BWMON_CLEAR_CLEAR BIT(0) 74 #define BWMON_CLEAR_CLEAR_ALL BIT(1) 75 76 #define BWMON_V4_SAMPLE_WINDOW 0x2a8 77 #define BWMON_V5_SAMPLE_WINDOW 0x020 78 79 #define BWMON_V4_THRESHOLD_HIGH 0x2ac 80 #define BWMON_V4_THRESHOLD_MED 0x2b0 81 #define BWMON_V4_THRESHOLD_LOW 0x2b4 82 #define BWMON_V5_THRESHOLD_HIGH 0x024 83 #define BWMON_V5_THRESHOLD_MED 0x028 84 #define BWMON_V5_THRESHOLD_LOW 0x02c 85 86 #define BWMON_V4_ZONE_ACTIONS 0x2b8 87 #define BWMON_V5_ZONE_ACTIONS 0x030 88 /* 89 * Actions to perform on some zone 'z' when current zone hits the threshold: 90 * Increment counter of zone 'z' 91 */ 92 #define BWMON_ZONE_ACTIONS_INCREMENT(z) (0x2 << ((z) * 2)) 93 /* Clear counter of zone 'z' */ 94 #define BWMON_ZONE_ACTIONS_CLEAR(z) (0x1 << ((z) * 2)) 95 96 /* Zone 0 threshold hit: Clear zone count */ 97 #define BWMON_ZONE_ACTIONS_ZONE0 (BWMON_ZONE_ACTIONS_CLEAR(0)) 98 99 /* Zone 1 threshold hit: Increment zone count & clear lower zones */ 100 #define BWMON_ZONE_ACTIONS_ZONE1 (BWMON_ZONE_ACTIONS_INCREMENT(1) | \ 101 BWMON_ZONE_ACTIONS_CLEAR(0)) 102 103 /* Zone 2 threshold hit: Increment zone count & clear lower zones */ 104 #define BWMON_ZONE_ACTIONS_ZONE2 (BWMON_ZONE_ACTIONS_INCREMENT(2) | \ 105 BWMON_ZONE_ACTIONS_CLEAR(1) | \ 106 BWMON_ZONE_ACTIONS_CLEAR(0)) 107 108 /* Zone 3 threshold hit: Increment zone count & clear lower zones */ 109 #define BWMON_ZONE_ACTIONS_ZONE3 (BWMON_ZONE_ACTIONS_INCREMENT(3) | \ 110 BWMON_ZONE_ACTIONS_CLEAR(2) | \ 111 BWMON_ZONE_ACTIONS_CLEAR(1) | \ 112 BWMON_ZONE_ACTIONS_CLEAR(0)) 113 114 /* 115 * There is no clear documentation/explanation of BWMON_V4_THRESHOLD_COUNT 116 * register. Based on observations, this is number of times one threshold has to 117 * be reached, to trigger interrupt in given zone. 118 * 119 * 0xff are maximum values meant to ignore the zones 0 and 2. 120 */ 121 #define BWMON_V4_THRESHOLD_COUNT 0x2bc 122 #define BWMON_V5_THRESHOLD_COUNT 0x034 123 #define BWMON_THRESHOLD_COUNT_ZONE0_DEFAULT 0xff 124 #define BWMON_THRESHOLD_COUNT_ZONE2_DEFAULT 0xff 125 126 #define BWMON_V4_ZONE_MAX(zone) (0x2e0 + 4 * (zone)) 127 #define BWMON_V5_ZONE_MAX(zone) (0x044 + 4 * (zone)) 128 129 /* Quirks for specific BWMON types */ 130 #define BWMON_HAS_GLOBAL_IRQ BIT(0) 131 #define BWMON_NEEDS_FORCE_CLEAR BIT(1) 132 133 enum bwmon_fields { 134 /* Global region fields, keep them at the top */ 135 F_GLOBAL_IRQ_CLEAR, 136 F_GLOBAL_IRQ_ENABLE, 137 F_NUM_GLOBAL_FIELDS, 138 139 /* Monitor region fields */ 140 F_IRQ_STATUS = F_NUM_GLOBAL_FIELDS, 141 F_IRQ_CLEAR, 142 F_IRQ_ENABLE, 143 F_ENABLE, 144 F_CLEAR, 145 F_SAMPLE_WINDOW, 146 F_THRESHOLD_HIGH, 147 F_THRESHOLD_MED, 148 F_THRESHOLD_LOW, 149 F_ZONE_ACTIONS_ZONE0, 150 F_ZONE_ACTIONS_ZONE1, 151 F_ZONE_ACTIONS_ZONE2, 152 F_ZONE_ACTIONS_ZONE3, 153 F_THRESHOLD_COUNT_ZONE0, 154 F_THRESHOLD_COUNT_ZONE1, 155 F_THRESHOLD_COUNT_ZONE2, 156 F_THRESHOLD_COUNT_ZONE3, 157 F_ZONE0_MAX, 158 F_ZONE1_MAX, 159 F_ZONE2_MAX, 160 F_ZONE3_MAX, 161 162 F_NUM_FIELDS 163 }; 164 165 struct icc_bwmon_data { 166 unsigned int sample_ms; 167 unsigned int count_unit_kb; /* kbytes */ 168 u8 zone1_thres_count; 169 u8 zone3_thres_count; 170 unsigned int quirks; 171 172 const struct regmap_config *regmap_cfg; 173 const struct reg_field *regmap_fields; 174 175 const struct regmap_config *global_regmap_cfg; 176 const struct reg_field *global_regmap_fields; 177 }; 178 179 struct icc_bwmon { 180 struct device *dev; 181 const struct icc_bwmon_data *data; 182 int irq; 183 184 struct regmap_field *regs[F_NUM_FIELDS]; 185 struct regmap_field *global_regs[F_NUM_GLOBAL_FIELDS]; 186 187 unsigned int max_bw_kbps; 188 unsigned int min_bw_kbps; 189 unsigned int target_kbps; 190 unsigned int current_kbps; 191 }; 192 193 /* BWMON v4 */ 194 static const struct reg_field msm8998_bwmon_reg_fields[] = { 195 [F_GLOBAL_IRQ_CLEAR] = {}, 196 [F_GLOBAL_IRQ_ENABLE] = {}, 197 [F_IRQ_STATUS] = REG_FIELD(BWMON_V4_IRQ_STATUS, 4, 7), 198 [F_IRQ_CLEAR] = REG_FIELD(BWMON_V4_IRQ_CLEAR, 4, 7), 199 [F_IRQ_ENABLE] = REG_FIELD(BWMON_V4_IRQ_ENABLE, 4, 7), 200 /* F_ENABLE covers entire register to disable other features */ 201 [F_ENABLE] = REG_FIELD(BWMON_V4_ENABLE, 0, 31), 202 [F_CLEAR] = REG_FIELD(BWMON_V4_CLEAR, 0, 1), 203 [F_SAMPLE_WINDOW] = REG_FIELD(BWMON_V4_SAMPLE_WINDOW, 0, 23), 204 [F_THRESHOLD_HIGH] = REG_FIELD(BWMON_V4_THRESHOLD_HIGH, 0, 11), 205 [F_THRESHOLD_MED] = REG_FIELD(BWMON_V4_THRESHOLD_MED, 0, 11), 206 [F_THRESHOLD_LOW] = REG_FIELD(BWMON_V4_THRESHOLD_LOW, 0, 11), 207 [F_ZONE_ACTIONS_ZONE0] = REG_FIELD(BWMON_V4_ZONE_ACTIONS, 0, 7), 208 [F_ZONE_ACTIONS_ZONE1] = REG_FIELD(BWMON_V4_ZONE_ACTIONS, 8, 15), 209 [F_ZONE_ACTIONS_ZONE2] = REG_FIELD(BWMON_V4_ZONE_ACTIONS, 16, 23), 210 [F_ZONE_ACTIONS_ZONE3] = REG_FIELD(BWMON_V4_ZONE_ACTIONS, 24, 31), 211 [F_THRESHOLD_COUNT_ZONE0] = REG_FIELD(BWMON_V4_THRESHOLD_COUNT, 0, 7), 212 [F_THRESHOLD_COUNT_ZONE1] = REG_FIELD(BWMON_V4_THRESHOLD_COUNT, 8, 15), 213 [F_THRESHOLD_COUNT_ZONE2] = REG_FIELD(BWMON_V4_THRESHOLD_COUNT, 16, 23), 214 [F_THRESHOLD_COUNT_ZONE3] = REG_FIELD(BWMON_V4_THRESHOLD_COUNT, 24, 31), 215 [F_ZONE0_MAX] = REG_FIELD(BWMON_V4_ZONE_MAX(0), 0, 11), 216 [F_ZONE1_MAX] = REG_FIELD(BWMON_V4_ZONE_MAX(1), 0, 11), 217 [F_ZONE2_MAX] = REG_FIELD(BWMON_V4_ZONE_MAX(2), 0, 11), 218 [F_ZONE3_MAX] = REG_FIELD(BWMON_V4_ZONE_MAX(3), 0, 11), 219 }; 220 221 static const struct regmap_range msm8998_bwmon_reg_noread_ranges[] = { 222 regmap_reg_range(BWMON_V4_IRQ_CLEAR, BWMON_V4_IRQ_CLEAR), 223 regmap_reg_range(BWMON_V4_CLEAR, BWMON_V4_CLEAR), 224 }; 225 226 static const struct regmap_access_table msm8998_bwmon_reg_read_table = { 227 .no_ranges = msm8998_bwmon_reg_noread_ranges, 228 .n_no_ranges = ARRAY_SIZE(msm8998_bwmon_reg_noread_ranges), 229 }; 230 231 static const struct regmap_range msm8998_bwmon_reg_volatile_ranges[] = { 232 regmap_reg_range(BWMON_V4_IRQ_STATUS, BWMON_V4_IRQ_STATUS), 233 regmap_reg_range(BWMON_V4_ZONE_MAX(0), BWMON_V4_ZONE_MAX(3)), 234 }; 235 236 static const struct regmap_access_table msm8998_bwmon_reg_volatile_table = { 237 .yes_ranges = msm8998_bwmon_reg_volatile_ranges, 238 .n_yes_ranges = ARRAY_SIZE(msm8998_bwmon_reg_volatile_ranges), 239 }; 240 241 static const struct reg_field msm8998_bwmon_global_reg_fields[] = { 242 [F_GLOBAL_IRQ_CLEAR] = REG_FIELD(BWMON_V4_GLOBAL_IRQ_CLEAR, 0, 0), 243 [F_GLOBAL_IRQ_ENABLE] = REG_FIELD(BWMON_V4_GLOBAL_IRQ_ENABLE, 0, 0), 244 }; 245 246 static const struct regmap_range msm8998_bwmon_global_reg_noread_ranges[] = { 247 regmap_reg_range(BWMON_V4_GLOBAL_IRQ_CLEAR, BWMON_V4_GLOBAL_IRQ_CLEAR), 248 }; 249 250 static const struct regmap_access_table msm8998_bwmon_global_reg_read_table = { 251 .no_ranges = msm8998_bwmon_global_reg_noread_ranges, 252 .n_no_ranges = ARRAY_SIZE(msm8998_bwmon_global_reg_noread_ranges), 253 }; 254 255 /* 256 * Fill the cache for non-readable registers only as rest does not really 257 * matter and can be read from the device. 258 */ 259 static const struct reg_default msm8998_bwmon_reg_defaults[] = { 260 { BWMON_V4_IRQ_CLEAR, 0x0 }, 261 { BWMON_V4_CLEAR, 0x0 }, 262 }; 263 264 static const struct reg_default msm8998_bwmon_global_reg_defaults[] = { 265 { BWMON_V4_GLOBAL_IRQ_CLEAR, 0x0 }, 266 }; 267 268 static const struct regmap_config msm8998_bwmon_regmap_cfg = { 269 .reg_bits = 32, 270 .reg_stride = 4, 271 .val_bits = 32, 272 /* 273 * No concurrent access expected - driver has one interrupt handler, 274 * regmap is not shared, no driver or user-space API. 275 */ 276 .disable_locking = true, 277 .rd_table = &msm8998_bwmon_reg_read_table, 278 .volatile_table = &msm8998_bwmon_reg_volatile_table, 279 .reg_defaults = msm8998_bwmon_reg_defaults, 280 .num_reg_defaults = ARRAY_SIZE(msm8998_bwmon_reg_defaults), 281 /* 282 * Cache is necessary for using regmap fields with non-readable 283 * registers. 284 */ 285 .cache_type = REGCACHE_RBTREE, 286 }; 287 288 static const struct regmap_config msm8998_bwmon_global_regmap_cfg = { 289 .reg_bits = 32, 290 .reg_stride = 4, 291 .val_bits = 32, 292 /* 293 * No concurrent access expected - driver has one interrupt handler, 294 * regmap is not shared, no driver or user-space API. 295 */ 296 .disable_locking = true, 297 .rd_table = &msm8998_bwmon_global_reg_read_table, 298 .reg_defaults = msm8998_bwmon_global_reg_defaults, 299 .num_reg_defaults = ARRAY_SIZE(msm8998_bwmon_global_reg_defaults), 300 /* 301 * Cache is necessary for using regmap fields with non-readable 302 * registers. 303 */ 304 .cache_type = REGCACHE_RBTREE, 305 }; 306 307 static const struct reg_field sdm845_cpu_bwmon_reg_fields[] = { 308 [F_GLOBAL_IRQ_CLEAR] = REG_FIELD(BWMON_V4_GLOBAL_IRQ_CLEAR_845, 0, 0), 309 [F_GLOBAL_IRQ_ENABLE] = REG_FIELD(BWMON_V4_GLOBAL_IRQ_ENABLE_845, 0, 0), 310 [F_IRQ_STATUS] = REG_FIELD(BWMON_V4_IRQ_STATUS, 4, 7), 311 [F_IRQ_CLEAR] = REG_FIELD(BWMON_V4_IRQ_CLEAR, 4, 7), 312 [F_IRQ_ENABLE] = REG_FIELD(BWMON_V4_IRQ_ENABLE, 4, 7), 313 /* F_ENABLE covers entire register to disable other features */ 314 [F_ENABLE] = REG_FIELD(BWMON_V4_ENABLE, 0, 31), 315 [F_CLEAR] = REG_FIELD(BWMON_V4_CLEAR, 0, 1), 316 [F_SAMPLE_WINDOW] = REG_FIELD(BWMON_V4_SAMPLE_WINDOW, 0, 23), 317 [F_THRESHOLD_HIGH] = REG_FIELD(BWMON_V4_THRESHOLD_HIGH, 0, 11), 318 [F_THRESHOLD_MED] = REG_FIELD(BWMON_V4_THRESHOLD_MED, 0, 11), 319 [F_THRESHOLD_LOW] = REG_FIELD(BWMON_V4_THRESHOLD_LOW, 0, 11), 320 [F_ZONE_ACTIONS_ZONE0] = REG_FIELD(BWMON_V4_ZONE_ACTIONS, 0, 7), 321 [F_ZONE_ACTIONS_ZONE1] = REG_FIELD(BWMON_V4_ZONE_ACTIONS, 8, 15), 322 [F_ZONE_ACTIONS_ZONE2] = REG_FIELD(BWMON_V4_ZONE_ACTIONS, 16, 23), 323 [F_ZONE_ACTIONS_ZONE3] = REG_FIELD(BWMON_V4_ZONE_ACTIONS, 24, 31), 324 [F_THRESHOLD_COUNT_ZONE0] = REG_FIELD(BWMON_V4_THRESHOLD_COUNT, 0, 7), 325 [F_THRESHOLD_COUNT_ZONE1] = REG_FIELD(BWMON_V4_THRESHOLD_COUNT, 8, 15), 326 [F_THRESHOLD_COUNT_ZONE2] = REG_FIELD(BWMON_V4_THRESHOLD_COUNT, 16, 23), 327 [F_THRESHOLD_COUNT_ZONE3] = REG_FIELD(BWMON_V4_THRESHOLD_COUNT, 24, 31), 328 [F_ZONE0_MAX] = REG_FIELD(BWMON_V4_ZONE_MAX(0), 0, 11), 329 [F_ZONE1_MAX] = REG_FIELD(BWMON_V4_ZONE_MAX(1), 0, 11), 330 [F_ZONE2_MAX] = REG_FIELD(BWMON_V4_ZONE_MAX(2), 0, 11), 331 [F_ZONE3_MAX] = REG_FIELD(BWMON_V4_ZONE_MAX(3), 0, 11), 332 }; 333 334 static const struct regmap_range sdm845_cpu_bwmon_reg_noread_ranges[] = { 335 regmap_reg_range(BWMON_V4_GLOBAL_IRQ_CLEAR_845, BWMON_V4_GLOBAL_IRQ_CLEAR_845), 336 regmap_reg_range(BWMON_V4_IRQ_CLEAR, BWMON_V4_IRQ_CLEAR), 337 regmap_reg_range(BWMON_V4_CLEAR, BWMON_V4_CLEAR), 338 }; 339 340 static const struct regmap_access_table sdm845_cpu_bwmon_reg_read_table = { 341 .no_ranges = sdm845_cpu_bwmon_reg_noread_ranges, 342 .n_no_ranges = ARRAY_SIZE(sdm845_cpu_bwmon_reg_noread_ranges), 343 }; 344 345 /* 346 * Fill the cache for non-readable registers only as rest does not really 347 * matter and can be read from the device. 348 */ 349 static const struct reg_default sdm845_cpu_bwmon_reg_defaults[] = { 350 { BWMON_V4_GLOBAL_IRQ_CLEAR_845, 0x0 }, 351 { BWMON_V4_IRQ_CLEAR, 0x0 }, 352 { BWMON_V4_CLEAR, 0x0 }, 353 }; 354 355 static const struct regmap_config sdm845_cpu_bwmon_regmap_cfg = { 356 .reg_bits = 32, 357 .reg_stride = 4, 358 .val_bits = 32, 359 /* 360 * No concurrent access expected - driver has one interrupt handler, 361 * regmap is not shared, no driver or user-space API. 362 */ 363 .disable_locking = true, 364 .rd_table = &sdm845_cpu_bwmon_reg_read_table, 365 .volatile_table = &msm8998_bwmon_reg_volatile_table, 366 .reg_defaults = sdm845_cpu_bwmon_reg_defaults, 367 .num_reg_defaults = ARRAY_SIZE(sdm845_cpu_bwmon_reg_defaults), 368 /* 369 * Cache is necessary for using regmap fields with non-readable 370 * registers. 371 */ 372 .cache_type = REGCACHE_RBTREE, 373 }; 374 375 /* BWMON v5 */ 376 static const struct reg_field sdm845_llcc_bwmon_reg_fields[] = { 377 [F_GLOBAL_IRQ_CLEAR] = {}, 378 [F_GLOBAL_IRQ_ENABLE] = {}, 379 [F_IRQ_STATUS] = REG_FIELD(BWMON_V5_IRQ_STATUS, 0, 3), 380 [F_IRQ_CLEAR] = REG_FIELD(BWMON_V5_IRQ_CLEAR, 0, 3), 381 [F_IRQ_ENABLE] = REG_FIELD(BWMON_V5_IRQ_ENABLE, 0, 3), 382 /* F_ENABLE covers entire register to disable other features */ 383 [F_ENABLE] = REG_FIELD(BWMON_V5_ENABLE, 0, 31), 384 [F_CLEAR] = REG_FIELD(BWMON_V5_CLEAR, 0, 1), 385 [F_SAMPLE_WINDOW] = REG_FIELD(BWMON_V5_SAMPLE_WINDOW, 0, 19), 386 [F_THRESHOLD_HIGH] = REG_FIELD(BWMON_V5_THRESHOLD_HIGH, 0, 11), 387 [F_THRESHOLD_MED] = REG_FIELD(BWMON_V5_THRESHOLD_MED, 0, 11), 388 [F_THRESHOLD_LOW] = REG_FIELD(BWMON_V5_THRESHOLD_LOW, 0, 11), 389 [F_ZONE_ACTIONS_ZONE0] = REG_FIELD(BWMON_V5_ZONE_ACTIONS, 0, 7), 390 [F_ZONE_ACTIONS_ZONE1] = REG_FIELD(BWMON_V5_ZONE_ACTIONS, 8, 15), 391 [F_ZONE_ACTIONS_ZONE2] = REG_FIELD(BWMON_V5_ZONE_ACTIONS, 16, 23), 392 [F_ZONE_ACTIONS_ZONE3] = REG_FIELD(BWMON_V5_ZONE_ACTIONS, 24, 31), 393 [F_THRESHOLD_COUNT_ZONE0] = REG_FIELD(BWMON_V5_THRESHOLD_COUNT, 0, 7), 394 [F_THRESHOLD_COUNT_ZONE1] = REG_FIELD(BWMON_V5_THRESHOLD_COUNT, 8, 15), 395 [F_THRESHOLD_COUNT_ZONE2] = REG_FIELD(BWMON_V5_THRESHOLD_COUNT, 16, 23), 396 [F_THRESHOLD_COUNT_ZONE3] = REG_FIELD(BWMON_V5_THRESHOLD_COUNT, 24, 31), 397 [F_ZONE0_MAX] = REG_FIELD(BWMON_V5_ZONE_MAX(0), 0, 11), 398 [F_ZONE1_MAX] = REG_FIELD(BWMON_V5_ZONE_MAX(1), 0, 11), 399 [F_ZONE2_MAX] = REG_FIELD(BWMON_V5_ZONE_MAX(2), 0, 11), 400 [F_ZONE3_MAX] = REG_FIELD(BWMON_V5_ZONE_MAX(3), 0, 11), 401 }; 402 403 static const struct regmap_range sdm845_llcc_bwmon_reg_noread_ranges[] = { 404 regmap_reg_range(BWMON_V5_IRQ_CLEAR, BWMON_V5_IRQ_CLEAR), 405 regmap_reg_range(BWMON_V5_CLEAR, BWMON_V5_CLEAR), 406 }; 407 408 static const struct regmap_access_table sdm845_llcc_bwmon_reg_read_table = { 409 .no_ranges = sdm845_llcc_bwmon_reg_noread_ranges, 410 .n_no_ranges = ARRAY_SIZE(sdm845_llcc_bwmon_reg_noread_ranges), 411 }; 412 413 static const struct regmap_range sdm845_llcc_bwmon_reg_volatile_ranges[] = { 414 regmap_reg_range(BWMON_V5_IRQ_STATUS, BWMON_V5_IRQ_STATUS), 415 regmap_reg_range(BWMON_V5_ZONE_MAX(0), BWMON_V5_ZONE_MAX(3)), 416 }; 417 418 static const struct regmap_access_table sdm845_llcc_bwmon_reg_volatile_table = { 419 .yes_ranges = sdm845_llcc_bwmon_reg_volatile_ranges, 420 .n_yes_ranges = ARRAY_SIZE(sdm845_llcc_bwmon_reg_volatile_ranges), 421 }; 422 423 /* 424 * Fill the cache for non-readable registers only as rest does not really 425 * matter and can be read from the device. 426 */ 427 static const struct reg_default sdm845_llcc_bwmon_reg_defaults[] = { 428 { BWMON_V5_IRQ_CLEAR, 0x0 }, 429 { BWMON_V5_CLEAR, 0x0 }, 430 }; 431 432 static const struct regmap_config sdm845_llcc_bwmon_regmap_cfg = { 433 .reg_bits = 32, 434 .reg_stride = 4, 435 .val_bits = 32, 436 /* 437 * No concurrent access expected - driver has one interrupt handler, 438 * regmap is not shared, no driver or user-space API. 439 */ 440 .disable_locking = true, 441 .rd_table = &sdm845_llcc_bwmon_reg_read_table, 442 .volatile_table = &sdm845_llcc_bwmon_reg_volatile_table, 443 .reg_defaults = sdm845_llcc_bwmon_reg_defaults, 444 .num_reg_defaults = ARRAY_SIZE(sdm845_llcc_bwmon_reg_defaults), 445 /* 446 * Cache is necessary for using regmap fields with non-readable 447 * registers. 448 */ 449 .cache_type = REGCACHE_RBTREE, 450 }; 451 452 static void bwmon_clear_counters(struct icc_bwmon *bwmon, bool clear_all) 453 { 454 unsigned int val = BWMON_CLEAR_CLEAR; 455 456 if (clear_all) 457 val |= BWMON_CLEAR_CLEAR_ALL; 458 /* 459 * Clear counters. The order and barriers are 460 * important. Quoting downstream Qualcomm msm-4.9 tree: 461 * 462 * The counter clear and IRQ clear bits are not in the same 4KB 463 * region. So, we need to make sure the counter clear is completed 464 * before we try to clear the IRQ or do any other counter operations. 465 */ 466 regmap_field_force_write(bwmon->regs[F_CLEAR], val); 467 if (bwmon->data->quirks & BWMON_NEEDS_FORCE_CLEAR) 468 regmap_field_force_write(bwmon->regs[F_CLEAR], 0); 469 } 470 471 static void bwmon_clear_irq(struct icc_bwmon *bwmon) 472 { 473 struct regmap_field *global_irq_clr; 474 475 if (bwmon->data->global_regmap_fields) 476 global_irq_clr = bwmon->global_regs[F_GLOBAL_IRQ_CLEAR]; 477 else 478 global_irq_clr = bwmon->regs[F_GLOBAL_IRQ_CLEAR]; 479 480 /* 481 * Clear zone and global interrupts. The order and barriers are 482 * important. Quoting downstream Qualcomm msm-4.9 tree: 483 * 484 * Synchronize the local interrupt clear in mon_irq_clear() 485 * with the global interrupt clear here. Otherwise, the CPU 486 * may reorder the two writes and clear the global interrupt 487 * before the local interrupt, causing the global interrupt 488 * to be retriggered by the local interrupt still being high. 489 * 490 * Similarly, because the global registers are in a different 491 * region than the local registers, we need to ensure any register 492 * writes to enable the monitor after this call are ordered with the 493 * clearing here so that local writes don't happen before the 494 * interrupt is cleared. 495 */ 496 regmap_field_force_write(bwmon->regs[F_IRQ_CLEAR], BWMON_IRQ_ENABLE_MASK); 497 if (bwmon->data->quirks & BWMON_NEEDS_FORCE_CLEAR) 498 regmap_field_force_write(bwmon->regs[F_IRQ_CLEAR], 0); 499 if (bwmon->data->quirks & BWMON_HAS_GLOBAL_IRQ) 500 regmap_field_force_write(global_irq_clr, 501 BWMON_V4_GLOBAL_IRQ_ENABLE_ENABLE); 502 } 503 504 static void bwmon_disable(struct icc_bwmon *bwmon) 505 { 506 struct regmap_field *global_irq_en; 507 508 if (bwmon->data->global_regmap_fields) 509 global_irq_en = bwmon->global_regs[F_GLOBAL_IRQ_ENABLE]; 510 else 511 global_irq_en = bwmon->regs[F_GLOBAL_IRQ_ENABLE]; 512 513 /* Disable interrupts. Strict ordering, see bwmon_clear_irq(). */ 514 if (bwmon->data->quirks & BWMON_HAS_GLOBAL_IRQ) 515 regmap_field_write(global_irq_en, 0x0); 516 regmap_field_write(bwmon->regs[F_IRQ_ENABLE], 0x0); 517 518 /* 519 * Disable bwmon. Must happen before bwmon_clear_irq() to avoid spurious 520 * IRQ. 521 */ 522 regmap_field_write(bwmon->regs[F_ENABLE], 0x0); 523 } 524 525 static void bwmon_enable(struct icc_bwmon *bwmon, unsigned int irq_enable) 526 { 527 struct regmap_field *global_irq_en; 528 529 if (bwmon->data->global_regmap_fields) 530 global_irq_en = bwmon->global_regs[F_GLOBAL_IRQ_ENABLE]; 531 else 532 global_irq_en = bwmon->regs[F_GLOBAL_IRQ_ENABLE]; 533 534 /* Enable interrupts */ 535 if (bwmon->data->quirks & BWMON_HAS_GLOBAL_IRQ) 536 regmap_field_write(global_irq_en, 537 BWMON_V4_GLOBAL_IRQ_ENABLE_ENABLE); 538 539 regmap_field_write(bwmon->regs[F_IRQ_ENABLE], irq_enable); 540 541 /* Enable bwmon */ 542 regmap_field_write(bwmon->regs[F_ENABLE], BWMON_ENABLE_ENABLE); 543 } 544 545 static unsigned int bwmon_kbps_to_count(struct icc_bwmon *bwmon, 546 unsigned int kbps) 547 { 548 return kbps / bwmon->data->count_unit_kb; 549 } 550 551 static void bwmon_set_threshold(struct icc_bwmon *bwmon, 552 struct regmap_field *reg, unsigned int kbps) 553 { 554 unsigned int thres; 555 556 thres = mult_frac(bwmon_kbps_to_count(bwmon, kbps), 557 bwmon->data->sample_ms, MSEC_PER_SEC); 558 regmap_field_write(reg, thres); 559 } 560 561 static void bwmon_start(struct icc_bwmon *bwmon) 562 { 563 const struct icc_bwmon_data *data = bwmon->data; 564 u32 bw_low = 0; 565 int window; 566 567 /* No need to check for errors, as this must have succeeded before. */ 568 dev_pm_opp_find_bw_ceil(bwmon->dev, &bw_low, 0); 569 570 bwmon_clear_counters(bwmon, true); 571 572 window = mult_frac(bwmon->data->sample_ms, HW_TIMER_HZ, MSEC_PER_SEC); 573 /* Maximum sampling window: 0xffffff for v4 and 0xfffff for v5 */ 574 regmap_field_write(bwmon->regs[F_SAMPLE_WINDOW], window); 575 576 bwmon_set_threshold(bwmon, bwmon->regs[F_THRESHOLD_HIGH], bw_low); 577 bwmon_set_threshold(bwmon, bwmon->regs[F_THRESHOLD_MED], bw_low); 578 bwmon_set_threshold(bwmon, bwmon->regs[F_THRESHOLD_LOW], 0); 579 580 regmap_field_write(bwmon->regs[F_THRESHOLD_COUNT_ZONE0], 581 BWMON_THRESHOLD_COUNT_ZONE0_DEFAULT); 582 regmap_field_write(bwmon->regs[F_THRESHOLD_COUNT_ZONE1], 583 data->zone1_thres_count); 584 regmap_field_write(bwmon->regs[F_THRESHOLD_COUNT_ZONE2], 585 BWMON_THRESHOLD_COUNT_ZONE2_DEFAULT); 586 regmap_field_write(bwmon->regs[F_THRESHOLD_COUNT_ZONE3], 587 data->zone3_thres_count); 588 589 regmap_field_write(bwmon->regs[F_ZONE_ACTIONS_ZONE0], 590 BWMON_ZONE_ACTIONS_ZONE0); 591 regmap_field_write(bwmon->regs[F_ZONE_ACTIONS_ZONE1], 592 BWMON_ZONE_ACTIONS_ZONE1); 593 regmap_field_write(bwmon->regs[F_ZONE_ACTIONS_ZONE2], 594 BWMON_ZONE_ACTIONS_ZONE2); 595 regmap_field_write(bwmon->regs[F_ZONE_ACTIONS_ZONE3], 596 BWMON_ZONE_ACTIONS_ZONE3); 597 598 bwmon_clear_irq(bwmon); 599 bwmon_enable(bwmon, BWMON_IRQ_ENABLE_MASK); 600 } 601 602 static irqreturn_t bwmon_intr(int irq, void *dev_id) 603 { 604 struct icc_bwmon *bwmon = dev_id; 605 unsigned int status, max; 606 int zone; 607 608 if (regmap_field_read(bwmon->regs[F_IRQ_STATUS], &status)) 609 return IRQ_NONE; 610 611 status &= BWMON_IRQ_ENABLE_MASK; 612 if (!status) { 613 /* 614 * Only zone 1 and zone 3 interrupts are enabled but zone 2 615 * threshold could be hit and trigger interrupt even if not 616 * enabled. 617 * Such spurious interrupt might come with valuable max count or 618 * not, so solution would be to always check all 619 * BWMON_ZONE_MAX() registers to find the highest value. 620 * Such case is currently ignored. 621 */ 622 return IRQ_NONE; 623 } 624 625 bwmon_disable(bwmon); 626 627 zone = get_bitmask_order(status) - 1; 628 /* 629 * Zone max bytes count register returns count units within sampling 630 * window. Downstream kernel for BWMONv4 (called BWMON type 2 in 631 * downstream) always increments the max bytes count by one. 632 */ 633 if (regmap_field_read(bwmon->regs[F_ZONE0_MAX + zone], &max)) 634 return IRQ_NONE; 635 636 max += 1; 637 max *= bwmon->data->count_unit_kb; 638 bwmon->target_kbps = mult_frac(max, MSEC_PER_SEC, bwmon->data->sample_ms); 639 640 return IRQ_WAKE_THREAD; 641 } 642 643 static irqreturn_t bwmon_intr_thread(int irq, void *dev_id) 644 { 645 struct icc_bwmon *bwmon = dev_id; 646 unsigned int irq_enable = 0; 647 struct dev_pm_opp *opp, *target_opp; 648 unsigned int bw_kbps, up_kbps, down_kbps; 649 650 bw_kbps = bwmon->target_kbps; 651 652 target_opp = dev_pm_opp_find_bw_ceil(bwmon->dev, &bw_kbps, 0); 653 if (IS_ERR(target_opp) && PTR_ERR(target_opp) == -ERANGE) 654 target_opp = dev_pm_opp_find_bw_floor(bwmon->dev, &bw_kbps, 0); 655 656 bwmon->target_kbps = bw_kbps; 657 658 bw_kbps--; 659 opp = dev_pm_opp_find_bw_floor(bwmon->dev, &bw_kbps, 0); 660 if (IS_ERR(opp) && PTR_ERR(opp) == -ERANGE) 661 down_kbps = bwmon->target_kbps; 662 else 663 down_kbps = bw_kbps; 664 665 up_kbps = bwmon->target_kbps + 1; 666 667 if (bwmon->target_kbps >= bwmon->max_bw_kbps) 668 irq_enable = BIT(1); 669 else if (bwmon->target_kbps <= bwmon->min_bw_kbps) 670 irq_enable = BIT(3); 671 else 672 irq_enable = BWMON_IRQ_ENABLE_MASK; 673 674 bwmon_set_threshold(bwmon, bwmon->regs[F_THRESHOLD_HIGH], 675 up_kbps); 676 bwmon_set_threshold(bwmon, bwmon->regs[F_THRESHOLD_MED], 677 down_kbps); 678 bwmon_clear_counters(bwmon, false); 679 bwmon_clear_irq(bwmon); 680 bwmon_enable(bwmon, irq_enable); 681 682 if (bwmon->target_kbps == bwmon->current_kbps) 683 goto out; 684 685 dev_pm_opp_set_opp(bwmon->dev, target_opp); 686 bwmon->current_kbps = bwmon->target_kbps; 687 688 out: 689 dev_pm_opp_put(target_opp); 690 if (!IS_ERR(opp)) 691 dev_pm_opp_put(opp); 692 693 return IRQ_HANDLED; 694 } 695 696 static int bwmon_init_regmap(struct platform_device *pdev, 697 struct icc_bwmon *bwmon) 698 { 699 struct device *dev = &pdev->dev; 700 void __iomem *base; 701 struct regmap *map; 702 int ret; 703 704 /* Map the monitor base */ 705 base = devm_platform_ioremap_resource(pdev, 0); 706 if (IS_ERR(base)) 707 return dev_err_probe(dev, PTR_ERR(base), 708 "failed to map bwmon registers\n"); 709 710 map = devm_regmap_init_mmio(dev, base, bwmon->data->regmap_cfg); 711 if (IS_ERR(map)) 712 return dev_err_probe(dev, PTR_ERR(map), 713 "failed to initialize regmap\n"); 714 715 BUILD_BUG_ON(ARRAY_SIZE(msm8998_bwmon_global_reg_fields) != F_NUM_GLOBAL_FIELDS); 716 BUILD_BUG_ON(ARRAY_SIZE(msm8998_bwmon_reg_fields) != F_NUM_FIELDS); 717 BUILD_BUG_ON(ARRAY_SIZE(sdm845_cpu_bwmon_reg_fields) != F_NUM_FIELDS); 718 BUILD_BUG_ON(ARRAY_SIZE(sdm845_llcc_bwmon_reg_fields) != F_NUM_FIELDS); 719 720 ret = devm_regmap_field_bulk_alloc(dev, map, bwmon->regs, 721 bwmon->data->regmap_fields, 722 F_NUM_FIELDS); 723 if (ret) 724 return ret; 725 726 if (bwmon->data->global_regmap_cfg) { 727 /* Map the global base, if separate */ 728 base = devm_platform_ioremap_resource(pdev, 1); 729 if (IS_ERR(base)) 730 return dev_err_probe(dev, PTR_ERR(base), 731 "failed to map bwmon global registers\n"); 732 733 map = devm_regmap_init_mmio(dev, base, bwmon->data->global_regmap_cfg); 734 if (IS_ERR(map)) 735 return dev_err_probe(dev, PTR_ERR(map), 736 "failed to initialize global regmap\n"); 737 738 ret = devm_regmap_field_bulk_alloc(dev, map, bwmon->global_regs, 739 bwmon->data->global_regmap_fields, 740 F_NUM_GLOBAL_FIELDS); 741 } 742 743 return ret; 744 } 745 746 static int bwmon_probe(struct platform_device *pdev) 747 { 748 struct device *dev = &pdev->dev; 749 struct dev_pm_opp *opp; 750 struct icc_bwmon *bwmon; 751 int ret; 752 753 bwmon = devm_kzalloc(dev, sizeof(*bwmon), GFP_KERNEL); 754 if (!bwmon) 755 return -ENOMEM; 756 757 bwmon->data = of_device_get_match_data(dev); 758 759 ret = bwmon_init_regmap(pdev, bwmon); 760 if (ret) 761 return ret; 762 763 bwmon->irq = platform_get_irq(pdev, 0); 764 if (bwmon->irq < 0) 765 return bwmon->irq; 766 767 ret = devm_pm_opp_of_add_table(dev); 768 if (ret) 769 return dev_err_probe(dev, ret, "failed to add OPP table\n"); 770 771 bwmon->max_bw_kbps = UINT_MAX; 772 opp = dev_pm_opp_find_bw_floor(dev, &bwmon->max_bw_kbps, 0); 773 if (IS_ERR(opp)) 774 return dev_err_probe(dev, PTR_ERR(opp), "failed to find max peak bandwidth\n"); 775 776 bwmon->min_bw_kbps = 0; 777 opp = dev_pm_opp_find_bw_ceil(dev, &bwmon->min_bw_kbps, 0); 778 if (IS_ERR(opp)) 779 return dev_err_probe(dev, PTR_ERR(opp), "failed to find min peak bandwidth\n"); 780 781 bwmon->dev = dev; 782 783 bwmon_disable(bwmon); 784 ret = devm_request_threaded_irq(dev, bwmon->irq, bwmon_intr, 785 bwmon_intr_thread, 786 IRQF_ONESHOT, dev_name(dev), bwmon); 787 if (ret) 788 return dev_err_probe(dev, ret, "failed to request IRQ\n"); 789 790 platform_set_drvdata(pdev, bwmon); 791 bwmon_start(bwmon); 792 793 return 0; 794 } 795 796 static int bwmon_remove(struct platform_device *pdev) 797 { 798 struct icc_bwmon *bwmon = platform_get_drvdata(pdev); 799 800 bwmon_disable(bwmon); 801 802 return 0; 803 } 804 805 static const struct icc_bwmon_data msm8998_bwmon_data = { 806 .sample_ms = 4, 807 .count_unit_kb = 1024, 808 .zone1_thres_count = 16, 809 .zone3_thres_count = 1, 810 .quirks = BWMON_HAS_GLOBAL_IRQ, 811 .regmap_fields = msm8998_bwmon_reg_fields, 812 .regmap_cfg = &msm8998_bwmon_regmap_cfg, 813 .global_regmap_fields = msm8998_bwmon_global_reg_fields, 814 .global_regmap_cfg = &msm8998_bwmon_global_regmap_cfg, 815 }; 816 817 static const struct icc_bwmon_data sdm845_cpu_bwmon_data = { 818 .sample_ms = 4, 819 .count_unit_kb = 64, 820 .zone1_thres_count = 16, 821 .zone3_thres_count = 1, 822 .quirks = BWMON_HAS_GLOBAL_IRQ, 823 .regmap_fields = sdm845_cpu_bwmon_reg_fields, 824 .regmap_cfg = &sdm845_cpu_bwmon_regmap_cfg, 825 }; 826 827 static const struct icc_bwmon_data sdm845_llcc_bwmon_data = { 828 .sample_ms = 4, 829 .count_unit_kb = 1024, 830 .zone1_thres_count = 16, 831 .zone3_thres_count = 1, 832 .regmap_fields = sdm845_llcc_bwmon_reg_fields, 833 .regmap_cfg = &sdm845_llcc_bwmon_regmap_cfg, 834 }; 835 836 static const struct icc_bwmon_data sc7280_llcc_bwmon_data = { 837 .sample_ms = 4, 838 .count_unit_kb = 64, 839 .zone1_thres_count = 16, 840 .zone3_thres_count = 1, 841 .quirks = BWMON_NEEDS_FORCE_CLEAR, 842 .regmap_fields = sdm845_llcc_bwmon_reg_fields, 843 .regmap_cfg = &sdm845_llcc_bwmon_regmap_cfg, 844 }; 845 846 static const struct of_device_id bwmon_of_match[] = { 847 /* BWMONv4, separate monitor and global register spaces */ 848 { .compatible = "qcom,msm8998-bwmon", .data = &msm8998_bwmon_data }, 849 /* BWMONv4, unified register space */ 850 { .compatible = "qcom,sdm845-bwmon", .data = &sdm845_cpu_bwmon_data }, 851 /* BWMONv5 */ 852 { .compatible = "qcom,sdm845-llcc-bwmon", .data = &sdm845_llcc_bwmon_data }, 853 { .compatible = "qcom,sc7280-llcc-bwmon", .data = &sc7280_llcc_bwmon_data }, 854 855 /* Compatibles kept for legacy reasons */ 856 { .compatible = "qcom,sc7280-cpu-bwmon", .data = &sdm845_cpu_bwmon_data }, 857 { .compatible = "qcom,sc8280xp-cpu-bwmon", .data = &sdm845_cpu_bwmon_data }, 858 { .compatible = "qcom,sm8550-cpu-bwmon", .data = &sdm845_cpu_bwmon_data }, 859 {} 860 }; 861 MODULE_DEVICE_TABLE(of, bwmon_of_match); 862 863 static struct platform_driver bwmon_driver = { 864 .probe = bwmon_probe, 865 .remove = bwmon_remove, 866 .driver = { 867 .name = "qcom-bwmon", 868 .of_match_table = bwmon_of_match, 869 }, 870 }; 871 module_platform_driver(bwmon_driver); 872 873 MODULE_AUTHOR("Krzysztof Kozlowski <krzysztof.kozlowski@linaro.org>"); 874 MODULE_DESCRIPTION("QCOM BWMON driver"); 875 MODULE_LICENSE("GPL"); 876