1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright 2019 NXP 4 */ 5 6 #include <linux/module.h> 7 #include <linux/device.h> 8 #include <linux/of_device.h> 9 #include <linux/platform_device.h> 10 #include <linux/devfreq.h> 11 #include <linux/pm_opp.h> 12 #include <linux/clk.h> 13 #include <linux/clk-provider.h> 14 #include <linux/arm-smccc.h> 15 16 #define IMX_SIP_DDR_DVFS 0xc2000004 17 18 /* Query available frequencies. */ 19 #define IMX_SIP_DDR_DVFS_GET_FREQ_COUNT 0x10 20 #define IMX_SIP_DDR_DVFS_GET_FREQ_INFO 0x11 21 22 /* 23 * This should be in a 1:1 mapping with devicetree OPPs but 24 * firmware provides additional info. 25 */ 26 struct imx8m_ddrc_freq { 27 unsigned long rate; 28 unsigned long smcarg; 29 int dram_core_parent_index; 30 int dram_alt_parent_index; 31 int dram_apb_parent_index; 32 }; 33 34 /* Hardware limitation */ 35 #define IMX8M_DDRC_MAX_FREQ_COUNT 4 36 37 /* 38 * i.MX8M DRAM Controller clocks have the following structure (abridged): 39 * 40 * +----------+ |\ +------+ 41 * | dram_pll |-------|M| dram_core | | 42 * +----------+ |U|---------->| D | 43 * /--|X| | D | 44 * dram_alt_root | |/ | R | 45 * | | C | 46 * +---------+ | | 47 * |FIX DIV/4| | | 48 * +---------+ | | 49 * composite: | | | 50 * +----------+ | | | 51 * | dram_alt |----/ | | 52 * +----------+ | | 53 * | dram_apb |-------------------->| | 54 * +----------+ +------+ 55 * 56 * The dram_pll is used for higher rates and dram_alt is used for lower rates. 57 * 58 * Frequency switching is implemented in TF-A (via SMC call) and can change the 59 * configuration of the clocks, including mux parents. The dram_alt and 60 * dram_apb clocks are "imx composite" and their parent can change too. 61 * 62 * We need to prepare/enable the new mux parents head of switching and update 63 * their information afterwards. 64 */ 65 struct imx8m_ddrc { 66 struct devfreq_dev_profile profile; 67 struct devfreq *devfreq; 68 69 /* For frequency switching: */ 70 struct clk *dram_core; 71 struct clk *dram_pll; 72 struct clk *dram_alt; 73 struct clk *dram_apb; 74 75 int freq_count; 76 struct imx8m_ddrc_freq freq_table[IMX8M_DDRC_MAX_FREQ_COUNT]; 77 }; 78 79 static struct imx8m_ddrc_freq *imx8m_ddrc_find_freq(struct imx8m_ddrc *priv, 80 unsigned long rate) 81 { 82 struct imx8m_ddrc_freq *freq; 83 int i; 84 85 /* 86 * Firmware reports values in MT/s, so we round-down from Hz 87 * Rounding is extra generous to ensure a match. 88 */ 89 rate = DIV_ROUND_CLOSEST(rate, 250000); 90 for (i = 0; i < priv->freq_count; ++i) { 91 freq = &priv->freq_table[i]; 92 if (freq->rate == rate || 93 freq->rate + 1 == rate || 94 freq->rate - 1 == rate) 95 return freq; 96 } 97 98 return NULL; 99 } 100 101 static void imx8m_ddrc_smc_set_freq(int target_freq) 102 { 103 struct arm_smccc_res res; 104 u32 online_cpus = 0; 105 int cpu; 106 107 local_irq_disable(); 108 109 for_each_online_cpu(cpu) 110 online_cpus |= (1 << (cpu * 8)); 111 112 /* change the ddr freqency */ 113 arm_smccc_smc(IMX_SIP_DDR_DVFS, target_freq, online_cpus, 114 0, 0, 0, 0, 0, &res); 115 116 local_irq_enable(); 117 } 118 119 static struct clk *clk_get_parent_by_index(struct clk *clk, int index) 120 { 121 struct clk_hw *hw; 122 123 hw = clk_hw_get_parent_by_index(__clk_get_hw(clk), index); 124 125 return hw ? hw->clk : NULL; 126 } 127 128 static int imx8m_ddrc_set_freq(struct device *dev, struct imx8m_ddrc_freq *freq) 129 { 130 struct imx8m_ddrc *priv = dev_get_drvdata(dev); 131 struct clk *new_dram_core_parent; 132 struct clk *new_dram_alt_parent; 133 struct clk *new_dram_apb_parent; 134 int ret; 135 136 /* 137 * Fetch new parents 138 * 139 * new_dram_alt_parent and new_dram_apb_parent are optional but 140 * new_dram_core_parent is not. 141 */ 142 new_dram_core_parent = clk_get_parent_by_index( 143 priv->dram_core, freq->dram_core_parent_index - 1); 144 if (!new_dram_core_parent) { 145 dev_err(dev, "failed to fetch new dram_core parent\n"); 146 return -EINVAL; 147 } 148 if (freq->dram_alt_parent_index) { 149 new_dram_alt_parent = clk_get_parent_by_index( 150 priv->dram_alt, 151 freq->dram_alt_parent_index - 1); 152 if (!new_dram_alt_parent) { 153 dev_err(dev, "failed to fetch new dram_alt parent\n"); 154 return -EINVAL; 155 } 156 } else 157 new_dram_alt_parent = NULL; 158 159 if (freq->dram_apb_parent_index) { 160 new_dram_apb_parent = clk_get_parent_by_index( 161 priv->dram_apb, 162 freq->dram_apb_parent_index - 1); 163 if (!new_dram_apb_parent) { 164 dev_err(dev, "failed to fetch new dram_apb parent\n"); 165 return -EINVAL; 166 } 167 } else 168 new_dram_apb_parent = NULL; 169 170 /* increase reference counts and ensure clks are ON before switch */ 171 ret = clk_prepare_enable(new_dram_core_parent); 172 if (ret) { 173 dev_err(dev, "failed to enable new dram_core parent: %d\n", 174 ret); 175 goto out; 176 } 177 ret = clk_prepare_enable(new_dram_alt_parent); 178 if (ret) { 179 dev_err(dev, "failed to enable new dram_alt parent: %d\n", 180 ret); 181 goto out_disable_core_parent; 182 } 183 ret = clk_prepare_enable(new_dram_apb_parent); 184 if (ret) { 185 dev_err(dev, "failed to enable new dram_apb parent: %d\n", 186 ret); 187 goto out_disable_alt_parent; 188 } 189 190 imx8m_ddrc_smc_set_freq(freq->smcarg); 191 192 /* update parents in clk tree after switch. */ 193 ret = clk_set_parent(priv->dram_core, new_dram_core_parent); 194 if (ret) 195 dev_warn(dev, "failed to set dram_core parent: %d\n", ret); 196 if (new_dram_alt_parent) { 197 ret = clk_set_parent(priv->dram_alt, new_dram_alt_parent); 198 if (ret) 199 dev_warn(dev, "failed to set dram_alt parent: %d\n", 200 ret); 201 } 202 if (new_dram_apb_parent) { 203 ret = clk_set_parent(priv->dram_apb, new_dram_apb_parent); 204 if (ret) 205 dev_warn(dev, "failed to set dram_apb parent: %d\n", 206 ret); 207 } 208 209 /* 210 * Explicitly refresh dram PLL rate. 211 * 212 * Even if it's marked with CLK_GET_RATE_NOCACHE the rate will not be 213 * automatically refreshed when clk_get_rate is called on children. 214 */ 215 clk_get_rate(priv->dram_pll); 216 217 /* 218 * clk_set_parent transfer the reference count from old parent. 219 * now we drop extra reference counts used during the switch 220 */ 221 clk_disable_unprepare(new_dram_apb_parent); 222 out_disable_alt_parent: 223 clk_disable_unprepare(new_dram_alt_parent); 224 out_disable_core_parent: 225 clk_disable_unprepare(new_dram_core_parent); 226 out: 227 return ret; 228 } 229 230 static int imx8m_ddrc_target(struct device *dev, unsigned long *freq, u32 flags) 231 { 232 struct imx8m_ddrc *priv = dev_get_drvdata(dev); 233 struct imx8m_ddrc_freq *freq_info; 234 struct dev_pm_opp *new_opp; 235 unsigned long old_freq, new_freq; 236 int ret; 237 238 new_opp = devfreq_recommended_opp(dev, freq, flags); 239 if (IS_ERR(new_opp)) { 240 ret = PTR_ERR(new_opp); 241 dev_err(dev, "failed to get recommended opp: %d\n", ret); 242 return ret; 243 } 244 dev_pm_opp_put(new_opp); 245 246 old_freq = clk_get_rate(priv->dram_core); 247 if (*freq == old_freq) 248 return 0; 249 250 freq_info = imx8m_ddrc_find_freq(priv, *freq); 251 if (!freq_info) 252 return -EINVAL; 253 254 /* 255 * Read back the clk rate to verify switch was correct and so that 256 * we can report it on all error paths. 257 */ 258 ret = imx8m_ddrc_set_freq(dev, freq_info); 259 260 new_freq = clk_get_rate(priv->dram_core); 261 if (ret) 262 dev_err(dev, "ddrc failed freq switch to %lu from %lu: error %d. now at %lu\n", 263 *freq, old_freq, ret, new_freq); 264 else if (*freq != new_freq) 265 dev_err(dev, "ddrc failed freq update to %lu from %lu, now at %lu\n", 266 *freq, old_freq, new_freq); 267 else 268 dev_dbg(dev, "ddrc freq set to %lu (was %lu)\n", 269 *freq, old_freq); 270 271 return ret; 272 } 273 274 static int imx8m_ddrc_get_cur_freq(struct device *dev, unsigned long *freq) 275 { 276 struct imx8m_ddrc *priv = dev_get_drvdata(dev); 277 278 *freq = clk_get_rate(priv->dram_core); 279 280 return 0; 281 } 282 283 static int imx8m_ddrc_get_dev_status(struct device *dev, 284 struct devfreq_dev_status *stat) 285 { 286 struct imx8m_ddrc *priv = dev_get_drvdata(dev); 287 288 stat->busy_time = 0; 289 stat->total_time = 0; 290 stat->current_frequency = clk_get_rate(priv->dram_core); 291 292 return 0; 293 } 294 295 static int imx8m_ddrc_init_freq_info(struct device *dev) 296 { 297 struct imx8m_ddrc *priv = dev_get_drvdata(dev); 298 struct arm_smccc_res res; 299 int index; 300 301 /* An error here means DDR DVFS API not supported by firmware */ 302 arm_smccc_smc(IMX_SIP_DDR_DVFS, IMX_SIP_DDR_DVFS_GET_FREQ_COUNT, 303 0, 0, 0, 0, 0, 0, &res); 304 priv->freq_count = res.a0; 305 if (priv->freq_count <= 0 || 306 priv->freq_count > IMX8M_DDRC_MAX_FREQ_COUNT) 307 return -ENODEV; 308 309 for (index = 0; index < priv->freq_count; ++index) { 310 struct imx8m_ddrc_freq *freq = &priv->freq_table[index]; 311 312 arm_smccc_smc(IMX_SIP_DDR_DVFS, IMX_SIP_DDR_DVFS_GET_FREQ_INFO, 313 index, 0, 0, 0, 0, 0, &res); 314 /* Result should be strictly positive */ 315 if ((long)res.a0 <= 0) 316 return -ENODEV; 317 318 freq->rate = res.a0; 319 freq->smcarg = index; 320 freq->dram_core_parent_index = res.a1; 321 freq->dram_alt_parent_index = res.a2; 322 freq->dram_apb_parent_index = res.a3; 323 324 /* dram_core has 2 options: dram_pll or dram_alt_root */ 325 if (freq->dram_core_parent_index != 1 && 326 freq->dram_core_parent_index != 2) 327 return -ENODEV; 328 /* dram_apb and dram_alt have exactly 8 possible parents */ 329 if (freq->dram_alt_parent_index > 8 || 330 freq->dram_apb_parent_index > 8) 331 return -ENODEV; 332 /* dram_core from alt requires explicit dram_alt parent */ 333 if (freq->dram_core_parent_index == 2 && 334 freq->dram_alt_parent_index == 0) 335 return -ENODEV; 336 } 337 338 return 0; 339 } 340 341 static int imx8m_ddrc_check_opps(struct device *dev) 342 { 343 struct imx8m_ddrc *priv = dev_get_drvdata(dev); 344 struct imx8m_ddrc_freq *freq_info; 345 struct dev_pm_opp *opp; 346 unsigned long freq; 347 int i, opp_count; 348 349 /* Enumerate DT OPPs and disable those not supported by firmware */ 350 opp_count = dev_pm_opp_get_opp_count(dev); 351 if (opp_count < 0) 352 return opp_count; 353 for (i = 0, freq = 0; i < opp_count; ++i, ++freq) { 354 opp = dev_pm_opp_find_freq_ceil(dev, &freq); 355 if (IS_ERR(opp)) { 356 dev_err(dev, "Failed enumerating OPPs: %ld\n", 357 PTR_ERR(opp)); 358 return PTR_ERR(opp); 359 } 360 dev_pm_opp_put(opp); 361 362 freq_info = imx8m_ddrc_find_freq(priv, freq); 363 if (!freq_info) { 364 dev_info(dev, "Disable unsupported OPP %luHz %luMT/s\n", 365 freq, DIV_ROUND_CLOSEST(freq, 250000)); 366 dev_pm_opp_disable(dev, freq); 367 } 368 } 369 370 return 0; 371 } 372 373 static void imx8m_ddrc_exit(struct device *dev) 374 { 375 dev_pm_opp_of_remove_table(dev); 376 } 377 378 static int imx8m_ddrc_probe(struct platform_device *pdev) 379 { 380 struct device *dev = &pdev->dev; 381 struct imx8m_ddrc *priv; 382 const char *gov = DEVFREQ_GOV_USERSPACE; 383 int ret; 384 385 priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); 386 if (!priv) 387 return -ENOMEM; 388 389 platform_set_drvdata(pdev, priv); 390 391 ret = imx8m_ddrc_init_freq_info(dev); 392 if (ret) { 393 dev_err(dev, "failed to init firmware freq info: %d\n", ret); 394 return ret; 395 } 396 397 priv->dram_core = devm_clk_get(dev, "core"); 398 if (IS_ERR(priv->dram_core)) { 399 ret = PTR_ERR(priv->dram_core); 400 dev_err(dev, "failed to fetch core clock: %d\n", ret); 401 return ret; 402 } 403 priv->dram_pll = devm_clk_get(dev, "pll"); 404 if (IS_ERR(priv->dram_pll)) { 405 ret = PTR_ERR(priv->dram_pll); 406 dev_err(dev, "failed to fetch pll clock: %d\n", ret); 407 return ret; 408 } 409 priv->dram_alt = devm_clk_get(dev, "alt"); 410 if (IS_ERR(priv->dram_alt)) { 411 ret = PTR_ERR(priv->dram_alt); 412 dev_err(dev, "failed to fetch alt clock: %d\n", ret); 413 return ret; 414 } 415 priv->dram_apb = devm_clk_get(dev, "apb"); 416 if (IS_ERR(priv->dram_apb)) { 417 ret = PTR_ERR(priv->dram_apb); 418 dev_err(dev, "failed to fetch apb clock: %d\n", ret); 419 return ret; 420 } 421 422 ret = dev_pm_opp_of_add_table(dev); 423 if (ret < 0) { 424 dev_err(dev, "failed to get OPP table\n"); 425 return ret; 426 } 427 428 ret = imx8m_ddrc_check_opps(dev); 429 if (ret < 0) 430 goto err; 431 432 priv->profile.polling_ms = 1000; 433 priv->profile.target = imx8m_ddrc_target; 434 priv->profile.get_dev_status = imx8m_ddrc_get_dev_status; 435 priv->profile.exit = imx8m_ddrc_exit; 436 priv->profile.get_cur_freq = imx8m_ddrc_get_cur_freq; 437 priv->profile.initial_freq = clk_get_rate(priv->dram_core); 438 439 priv->devfreq = devm_devfreq_add_device(dev, &priv->profile, 440 gov, NULL); 441 if (IS_ERR(priv->devfreq)) { 442 ret = PTR_ERR(priv->devfreq); 443 dev_err(dev, "failed to add devfreq device: %d\n", ret); 444 goto err; 445 } 446 447 return 0; 448 449 err: 450 dev_pm_opp_of_remove_table(dev); 451 return ret; 452 } 453 454 static const struct of_device_id imx8m_ddrc_of_match[] = { 455 { .compatible = "fsl,imx8m-ddrc", }, 456 { /* sentinel */ }, 457 }; 458 MODULE_DEVICE_TABLE(of, imx8m_ddrc_of_match); 459 460 static struct platform_driver imx8m_ddrc_platdrv = { 461 .probe = imx8m_ddrc_probe, 462 .driver = { 463 .name = "imx8m-ddrc-devfreq", 464 .of_match_table = of_match_ptr(imx8m_ddrc_of_match), 465 }, 466 }; 467 module_platform_driver(imx8m_ddrc_platdrv); 468 469 MODULE_DESCRIPTION("i.MX8M DDR Controller frequency driver"); 470 MODULE_AUTHOR("Leonard Crestez <leonard.crestez@nxp.com>"); 471 MODULE_LICENSE("GPL v2"); 472