1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2019 Samsung Electronics Co., Ltd. 4 * Author: Lukasz Luba <l.luba@partner.samsung.com> 5 */ 6 7 #include <linux/clk.h> 8 #include <linux/devfreq.h> 9 #include <linux/devfreq-event.h> 10 #include <linux/device.h> 11 #include <linux/interrupt.h> 12 #include <linux/io.h> 13 #include <linux/mfd/syscon.h> 14 #include <linux/module.h> 15 #include <linux/moduleparam.h> 16 #include <linux/of_device.h> 17 #include <linux/pm_opp.h> 18 #include <linux/platform_device.h> 19 #include <linux/regmap.h> 20 #include <linux/regulator/consumer.h> 21 #include <linux/slab.h> 22 #include "../jedec_ddr.h" 23 #include "../of_memory.h" 24 25 static int irqmode; 26 module_param(irqmode, int, 0644); 27 MODULE_PARM_DESC(irqmode, "Enable IRQ mode (0=off [default], 1=on)"); 28 29 #define EXYNOS5_DREXI_TIMINGAREF (0x0030) 30 #define EXYNOS5_DREXI_TIMINGROW0 (0x0034) 31 #define EXYNOS5_DREXI_TIMINGDATA0 (0x0038) 32 #define EXYNOS5_DREXI_TIMINGPOWER0 (0x003C) 33 #define EXYNOS5_DREXI_TIMINGROW1 (0x00E4) 34 #define EXYNOS5_DREXI_TIMINGDATA1 (0x00E8) 35 #define EXYNOS5_DREXI_TIMINGPOWER1 (0x00EC) 36 #define CDREX_PAUSE (0x2091c) 37 #define CDREX_LPDDR3PHY_CON3 (0x20a20) 38 #define CDREX_LPDDR3PHY_CLKM_SRC (0x20700) 39 #define EXYNOS5_TIMING_SET_SWI BIT(28) 40 #define USE_MX_MSPLL_TIMINGS (1) 41 #define USE_BPLL_TIMINGS (0) 42 #define EXYNOS5_AREF_NORMAL (0x2e) 43 44 #define DREX_PPCCLKCON (0x0130) 45 #define DREX_PEREV2CONFIG (0x013c) 46 #define DREX_PMNC_PPC (0xE000) 47 #define DREX_CNTENS_PPC (0xE010) 48 #define DREX_CNTENC_PPC (0xE020) 49 #define DREX_INTENS_PPC (0xE030) 50 #define DREX_INTENC_PPC (0xE040) 51 #define DREX_FLAG_PPC (0xE050) 52 #define DREX_PMCNT2_PPC (0xE130) 53 54 /* 55 * A value for register DREX_PMNC_PPC which should be written to reset 56 * the cycle counter CCNT (a reference wall clock). It sets zero to the 57 * CCNT counter. 58 */ 59 #define CC_RESET BIT(2) 60 61 /* 62 * A value for register DREX_PMNC_PPC which does the reset of all performance 63 * counters to zero. 64 */ 65 #define PPC_COUNTER_RESET BIT(1) 66 67 /* 68 * Enables all configured counters (including cycle counter). The value should 69 * be written to the register DREX_PMNC_PPC. 70 */ 71 #define PPC_ENABLE BIT(0) 72 73 /* A value for register DREX_PPCCLKCON which enables performance events clock. 74 * Must be written before first access to the performance counters register 75 * set, otherwise it could crash. 76 */ 77 #define PEREV_CLK_EN BIT(0) 78 79 /* 80 * Values which are used to enable counters, interrupts or configure flags of 81 * the performance counters. They configure counter 2 and cycle counter. 82 */ 83 #define PERF_CNT2 BIT(2) 84 #define PERF_CCNT BIT(31) 85 86 /* 87 * Performance event types which are used for setting the preferred event 88 * to track in the counters. 89 * There is a set of different types, the values are from range 0 to 0x6f. 90 * These settings should be written to the configuration register which manages 91 * the type of the event (register DREX_PEREV2CONFIG). 92 */ 93 #define READ_TRANSFER_CH0 (0x6d) 94 #define READ_TRANSFER_CH1 (0x6f) 95 96 #define PERF_COUNTER_START_VALUE 0xff000000 97 #define PERF_EVENT_UP_DOWN_THRESHOLD 900000000ULL 98 99 /** 100 * struct dmc_opp_table - Operating level desciption 101 * 102 * Covers frequency and voltage settings of the DMC operating mode. 103 */ 104 struct dmc_opp_table { 105 u32 freq_hz; 106 u32 volt_uv; 107 }; 108 109 /** 110 * struct exynos5_dmc - main structure describing DMC device 111 * 112 * The main structure for the Dynamic Memory Controller which covers clocks, 113 * memory regions, HW information, parameters and current operating mode. 114 */ 115 struct exynos5_dmc { 116 struct device *dev; 117 struct devfreq *df; 118 struct devfreq_simple_ondemand_data gov_data; 119 void __iomem *base_drexi0; 120 void __iomem *base_drexi1; 121 struct regmap *clk_regmap; 122 struct mutex lock; 123 unsigned long curr_rate; 124 unsigned long curr_volt; 125 unsigned long bypass_rate; 126 struct dmc_opp_table *opp; 127 struct dmc_opp_table opp_bypass; 128 int opp_count; 129 u32 timings_arr_size; 130 u32 *timing_row; 131 u32 *timing_data; 132 u32 *timing_power; 133 const struct lpddr3_timings *timings; 134 const struct lpddr3_min_tck *min_tck; 135 u32 bypass_timing_row; 136 u32 bypass_timing_data; 137 u32 bypass_timing_power; 138 struct regulator *vdd_mif; 139 struct clk *fout_spll; 140 struct clk *fout_bpll; 141 struct clk *mout_spll; 142 struct clk *mout_bpll; 143 struct clk *mout_mclk_cdrex; 144 struct clk *mout_mx_mspll_ccore; 145 struct clk *mx_mspll_ccore_phy; 146 struct clk *mout_mx_mspll_ccore_phy; 147 struct devfreq_event_dev **counter; 148 int num_counters; 149 u64 last_overflow_ts[2]; 150 unsigned long load; 151 unsigned long total; 152 bool in_irq_mode; 153 }; 154 155 #define TIMING_FIELD(t_name, t_bit_beg, t_bit_end) \ 156 { .name = t_name, .bit_beg = t_bit_beg, .bit_end = t_bit_end } 157 158 #define TIMING_VAL2REG(timing, t_val) \ 159 ({ \ 160 u32 __val; \ 161 __val = (t_val) << (timing)->bit_beg; \ 162 __val; \ 163 }) 164 165 struct timing_reg { 166 char *name; 167 int bit_beg; 168 int bit_end; 169 unsigned int val; 170 }; 171 172 static const struct timing_reg timing_row[] = { 173 TIMING_FIELD("tRFC", 24, 31), 174 TIMING_FIELD("tRRD", 20, 23), 175 TIMING_FIELD("tRP", 16, 19), 176 TIMING_FIELD("tRCD", 12, 15), 177 TIMING_FIELD("tRC", 6, 11), 178 TIMING_FIELD("tRAS", 0, 5), 179 }; 180 181 static const struct timing_reg timing_data[] = { 182 TIMING_FIELD("tWTR", 28, 31), 183 TIMING_FIELD("tWR", 24, 27), 184 TIMING_FIELD("tRTP", 20, 23), 185 TIMING_FIELD("tW2W-C2C", 14, 14), 186 TIMING_FIELD("tR2R-C2C", 12, 12), 187 TIMING_FIELD("WL", 8, 11), 188 TIMING_FIELD("tDQSCK", 4, 7), 189 TIMING_FIELD("RL", 0, 3), 190 }; 191 192 static const struct timing_reg timing_power[] = { 193 TIMING_FIELD("tFAW", 26, 31), 194 TIMING_FIELD("tXSR", 16, 25), 195 TIMING_FIELD("tXP", 8, 15), 196 TIMING_FIELD("tCKE", 4, 7), 197 TIMING_FIELD("tMRD", 0, 3), 198 }; 199 200 #define TIMING_COUNT (ARRAY_SIZE(timing_row) + ARRAY_SIZE(timing_data) + \ 201 ARRAY_SIZE(timing_power)) 202 203 static int exynos5_counters_set_event(struct exynos5_dmc *dmc) 204 { 205 int i, ret; 206 207 for (i = 0; i < dmc->num_counters; i++) { 208 if (!dmc->counter[i]) 209 continue; 210 ret = devfreq_event_set_event(dmc->counter[i]); 211 if (ret < 0) 212 return ret; 213 } 214 return 0; 215 } 216 217 static int exynos5_counters_enable_edev(struct exynos5_dmc *dmc) 218 { 219 int i, ret; 220 221 for (i = 0; i < dmc->num_counters; i++) { 222 if (!dmc->counter[i]) 223 continue; 224 ret = devfreq_event_enable_edev(dmc->counter[i]); 225 if (ret < 0) 226 return ret; 227 } 228 return 0; 229 } 230 231 static int exynos5_counters_disable_edev(struct exynos5_dmc *dmc) 232 { 233 int i, ret; 234 235 for (i = 0; i < dmc->num_counters; i++) { 236 if (!dmc->counter[i]) 237 continue; 238 ret = devfreq_event_disable_edev(dmc->counter[i]); 239 if (ret < 0) 240 return ret; 241 } 242 return 0; 243 } 244 245 /** 246 * find_target_freq_id() - Finds requested frequency in local DMC configuration 247 * @dmc: device for which the information is checked 248 * @target_rate: requested frequency in KHz 249 * 250 * Seeks in the local DMC driver structure for the requested frequency value 251 * and returns index or error value. 252 */ 253 static int find_target_freq_idx(struct exynos5_dmc *dmc, 254 unsigned long target_rate) 255 { 256 int i; 257 258 for (i = dmc->opp_count - 1; i >= 0; i--) 259 if (dmc->opp[i].freq_hz <= target_rate) 260 return i; 261 262 return -EINVAL; 263 } 264 265 /** 266 * exynos5_switch_timing_regs() - Changes bank register set for DRAM timings 267 * @dmc: device for which the new settings is going to be applied 268 * @set: boolean variable passing set value 269 * 270 * Changes the register set, which holds timing parameters. 271 * There is two register sets: 0 and 1. The register set 0 272 * is used in normal operation when the clock is provided from main PLL. 273 * The bank register set 1 is used when the main PLL frequency is going to be 274 * changed and the clock is taken from alternative, stable source. 275 * This function switches between these banks according to the 276 * currently used clock source. 277 */ 278 static int exynos5_switch_timing_regs(struct exynos5_dmc *dmc, bool set) 279 { 280 unsigned int reg; 281 int ret; 282 283 ret = regmap_read(dmc->clk_regmap, CDREX_LPDDR3PHY_CON3, ®); 284 if (ret) 285 return ret; 286 287 if (set) 288 reg |= EXYNOS5_TIMING_SET_SWI; 289 else 290 reg &= ~EXYNOS5_TIMING_SET_SWI; 291 292 regmap_write(dmc->clk_regmap, CDREX_LPDDR3PHY_CON3, reg); 293 294 return 0; 295 } 296 297 /** 298 * exynos5_init_freq_table() - Initialized PM OPP framework 299 * @dmc: DMC device for which the frequencies are used for OPP init 300 * @profile: devfreq device's profile 301 * 302 * Populate the devfreq device's OPP table based on current frequency, voltage. 303 */ 304 static int exynos5_init_freq_table(struct exynos5_dmc *dmc, 305 struct devfreq_dev_profile *profile) 306 { 307 int i, ret; 308 int idx; 309 unsigned long freq; 310 311 ret = dev_pm_opp_of_add_table(dmc->dev); 312 if (ret < 0) { 313 dev_err(dmc->dev, "Failed to get OPP table\n"); 314 return ret; 315 } 316 317 dmc->opp_count = dev_pm_opp_get_opp_count(dmc->dev); 318 319 dmc->opp = devm_kmalloc_array(dmc->dev, dmc->opp_count, 320 sizeof(struct dmc_opp_table), GFP_KERNEL); 321 if (!dmc->opp) 322 goto err_opp; 323 324 idx = dmc->opp_count - 1; 325 for (i = 0, freq = ULONG_MAX; i < dmc->opp_count; i++, freq--) { 326 struct dev_pm_opp *opp; 327 328 opp = dev_pm_opp_find_freq_floor(dmc->dev, &freq); 329 if (IS_ERR(opp)) 330 goto err_opp; 331 332 dmc->opp[idx - i].freq_hz = freq; 333 dmc->opp[idx - i].volt_uv = dev_pm_opp_get_voltage(opp); 334 335 dev_pm_opp_put(opp); 336 } 337 338 return 0; 339 340 err_opp: 341 dev_pm_opp_of_remove_table(dmc->dev); 342 343 return -EINVAL; 344 } 345 346 /** 347 * exynos5_set_bypass_dram_timings() - Low-level changes of the DRAM timings 348 * @dmc: device for which the new settings is going to be applied 349 * @param: DRAM parameters which passes timing data 350 * 351 * Low-level function for changing timings for DRAM memory clocking from 352 * 'bypass' clock source (fixed frequency @400MHz). 353 * It uses timing bank registers set 1. 354 */ 355 static void exynos5_set_bypass_dram_timings(struct exynos5_dmc *dmc) 356 { 357 writel(EXYNOS5_AREF_NORMAL, 358 dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGAREF); 359 360 writel(dmc->bypass_timing_row, 361 dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGROW1); 362 writel(dmc->bypass_timing_row, 363 dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGROW1); 364 writel(dmc->bypass_timing_data, 365 dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGDATA1); 366 writel(dmc->bypass_timing_data, 367 dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGDATA1); 368 writel(dmc->bypass_timing_power, 369 dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGPOWER1); 370 writel(dmc->bypass_timing_power, 371 dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGPOWER1); 372 } 373 374 /** 375 * exynos5_dram_change_timings() - Low-level changes of the DRAM final timings 376 * @dmc: device for which the new settings is going to be applied 377 * @target_rate: target frequency of the DMC 378 * 379 * Low-level function for changing timings for DRAM memory operating from main 380 * clock source (BPLL), which can have different frequencies. Thus, each 381 * frequency must have corresponding timings register values in order to keep 382 * the needed delays. 383 * It uses timing bank registers set 0. 384 */ 385 static int exynos5_dram_change_timings(struct exynos5_dmc *dmc, 386 unsigned long target_rate) 387 { 388 int idx; 389 390 for (idx = dmc->opp_count - 1; idx >= 0; idx--) 391 if (dmc->opp[idx].freq_hz <= target_rate) 392 break; 393 394 if (idx < 0) 395 return -EINVAL; 396 397 writel(EXYNOS5_AREF_NORMAL, 398 dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGAREF); 399 400 writel(dmc->timing_row[idx], 401 dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGROW0); 402 writel(dmc->timing_row[idx], 403 dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGROW0); 404 writel(dmc->timing_data[idx], 405 dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGDATA0); 406 writel(dmc->timing_data[idx], 407 dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGDATA0); 408 writel(dmc->timing_power[idx], 409 dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGPOWER0); 410 writel(dmc->timing_power[idx], 411 dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGPOWER0); 412 413 return 0; 414 } 415 416 /** 417 * exynos5_dmc_align_target_voltage() - Sets the final voltage for the DMC 418 * @dmc: device for which it is going to be set 419 * @target_volt: new voltage which is chosen to be final 420 * 421 * Function tries to align voltage to the safe level for 'normal' mode. 422 * It checks the need of higher voltage and changes the value. The target 423 * voltage might be lower that currently set and still the system will be 424 * stable. 425 */ 426 static int exynos5_dmc_align_target_voltage(struct exynos5_dmc *dmc, 427 unsigned long target_volt) 428 { 429 int ret = 0; 430 431 if (dmc->curr_volt <= target_volt) 432 return 0; 433 434 ret = regulator_set_voltage(dmc->vdd_mif, target_volt, 435 target_volt); 436 if (!ret) 437 dmc->curr_volt = target_volt; 438 439 return ret; 440 } 441 442 /** 443 * exynos5_dmc_align_bypass_voltage() - Sets the voltage for the DMC 444 * @dmc: device for which it is going to be set 445 * @target_volt: new voltage which is chosen to be final 446 * 447 * Function tries to align voltage to the safe level for the 'bypass' mode. 448 * It checks the need of higher voltage and changes the value. 449 * The target voltage must not be less than currently needed, because 450 * for current frequency the device might become unstable. 451 */ 452 static int exynos5_dmc_align_bypass_voltage(struct exynos5_dmc *dmc, 453 unsigned long target_volt) 454 { 455 int ret = 0; 456 unsigned long bypass_volt = dmc->opp_bypass.volt_uv; 457 458 target_volt = max(bypass_volt, target_volt); 459 460 if (dmc->curr_volt >= target_volt) 461 return 0; 462 463 ret = regulator_set_voltage(dmc->vdd_mif, target_volt, 464 target_volt); 465 if (!ret) 466 dmc->curr_volt = target_volt; 467 468 return ret; 469 } 470 471 /** 472 * exynos5_dmc_align_bypass_dram_timings() - Chooses and sets DRAM timings 473 * @dmc: device for which it is going to be set 474 * @target_rate: new frequency which is chosen to be final 475 * 476 * Function changes the DRAM timings for the temporary 'bypass' mode. 477 */ 478 static int exynos5_dmc_align_bypass_dram_timings(struct exynos5_dmc *dmc, 479 unsigned long target_rate) 480 { 481 int idx = find_target_freq_idx(dmc, target_rate); 482 483 if (idx < 0) 484 return -EINVAL; 485 486 exynos5_set_bypass_dram_timings(dmc); 487 488 return 0; 489 } 490 491 /** 492 * exynos5_dmc_switch_to_bypass_configuration() - Switching to temporary clock 493 * @dmc: DMC device for which the switching is going to happen 494 * @target_rate: new frequency which is going to be set as a final 495 * @target_volt: new voltage which is going to be set as a final 496 * 497 * Function configures DMC and clocks for operating in temporary 'bypass' mode. 498 * This mode is used only temporary but if required, changes voltage and timings 499 * for DRAM chips. It switches the main clock to stable clock source for the 500 * period of the main PLL reconfiguration. 501 */ 502 static int 503 exynos5_dmc_switch_to_bypass_configuration(struct exynos5_dmc *dmc, 504 unsigned long target_rate, 505 unsigned long target_volt) 506 { 507 int ret; 508 509 /* 510 * Having higher voltage for a particular frequency does not harm 511 * the chip. Use it for the temporary frequency change when one 512 * voltage manipulation might be avoided. 513 */ 514 ret = exynos5_dmc_align_bypass_voltage(dmc, target_volt); 515 if (ret) 516 return ret; 517 518 /* 519 * Longer delays for DRAM does not cause crash, the opposite does. 520 */ 521 ret = exynos5_dmc_align_bypass_dram_timings(dmc, target_rate); 522 if (ret) 523 return ret; 524 525 /* 526 * Delays are long enough, so use them for the new coming clock. 527 */ 528 ret = exynos5_switch_timing_regs(dmc, USE_MX_MSPLL_TIMINGS); 529 530 return ret; 531 } 532 533 /** 534 * exynos5_dmc_change_freq_and_volt() - Changes voltage and frequency of the DMC 535 * using safe procedure 536 * @dmc: device for which the frequency is going to be changed 537 * @target_rate: requested new frequency 538 * @target_volt: requested voltage which corresponds to the new frequency 539 * 540 * The DMC frequency change procedure requires a few steps. 541 * The main requirement is to change the clock source in the clk mux 542 * for the time of main clock PLL locking. The assumption is that the 543 * alternative clock source set as parent is stable. 544 * The second parent's clock frequency is fixed to 400MHz, it is named 'bypass' 545 * clock. This requires alignment in DRAM timing parameters for the new 546 * T-period. There is two bank sets for keeping DRAM 547 * timings: set 0 and set 1. The set 0 is used when main clock source is 548 * chosen. The 2nd set of regs is used for 'bypass' clock. Switching between 549 * the two bank sets is part of the process. 550 * The voltage must also be aligned to the minimum required level. There is 551 * this intermediate step with switching to 'bypass' parent clock source. 552 * if the old voltage is lower, it requires an increase of the voltage level. 553 * The complexity of the voltage manipulation is hidden in low level function. 554 * In this function there is last alignment of the voltage level at the end. 555 */ 556 static int 557 exynos5_dmc_change_freq_and_volt(struct exynos5_dmc *dmc, 558 unsigned long target_rate, 559 unsigned long target_volt) 560 { 561 int ret; 562 563 ret = exynos5_dmc_switch_to_bypass_configuration(dmc, target_rate, 564 target_volt); 565 if (ret) 566 return ret; 567 568 /* 569 * Voltage is set at least to a level needed for this frequency, 570 * so switching clock source is safe now. 571 */ 572 clk_prepare_enable(dmc->fout_spll); 573 clk_prepare_enable(dmc->mout_spll); 574 clk_prepare_enable(dmc->mout_mx_mspll_ccore); 575 576 ret = clk_set_parent(dmc->mout_mclk_cdrex, dmc->mout_mx_mspll_ccore); 577 if (ret) 578 goto disable_clocks; 579 580 /* 581 * We are safe to increase the timings for current bypass frequency. 582 * Thanks to this the settings will be ready for the upcoming clock 583 * source change. 584 */ 585 exynos5_dram_change_timings(dmc, target_rate); 586 587 clk_set_rate(dmc->fout_bpll, target_rate); 588 589 ret = exynos5_switch_timing_regs(dmc, USE_BPLL_TIMINGS); 590 if (ret) 591 goto disable_clocks; 592 593 ret = clk_set_parent(dmc->mout_mclk_cdrex, dmc->mout_bpll); 594 if (ret) 595 goto disable_clocks; 596 597 /* 598 * Make sure if the voltage is not from 'bypass' settings and align to 599 * the right level for power efficiency. 600 */ 601 ret = exynos5_dmc_align_target_voltage(dmc, target_volt); 602 603 disable_clocks: 604 clk_disable_unprepare(dmc->mout_mx_mspll_ccore); 605 clk_disable_unprepare(dmc->mout_spll); 606 clk_disable_unprepare(dmc->fout_spll); 607 608 return ret; 609 } 610 611 /** 612 * exynos5_dmc_get_volt_freq() - Gets the frequency and voltage from the OPP 613 * table. 614 * @dmc: device for which the frequency is going to be changed 615 * @freq: requested frequency in KHz 616 * @target_rate: returned frequency which is the same or lower than 617 * requested 618 * @target_volt: returned voltage which corresponds to the returned 619 * frequency 620 * 621 * Function gets requested frequency and checks OPP framework for needed 622 * frequency and voltage. It populates the values 'target_rate' and 623 * 'target_volt' or returns error value when OPP framework fails. 624 */ 625 static int exynos5_dmc_get_volt_freq(struct exynos5_dmc *dmc, 626 unsigned long *freq, 627 unsigned long *target_rate, 628 unsigned long *target_volt, u32 flags) 629 { 630 struct dev_pm_opp *opp; 631 632 opp = devfreq_recommended_opp(dmc->dev, freq, flags); 633 if (IS_ERR(opp)) 634 return PTR_ERR(opp); 635 636 *target_rate = dev_pm_opp_get_freq(opp); 637 *target_volt = dev_pm_opp_get_voltage(opp); 638 dev_pm_opp_put(opp); 639 640 return 0; 641 } 642 643 /** 644 * exynos5_dmc_target() - Function responsible for changing frequency of DMC 645 * @dev: device for which the frequency is going to be changed 646 * @freq: requested frequency in KHz 647 * @flags: flags provided for this frequency change request 648 * 649 * An entry function provided to the devfreq framework which provides frequency 650 * change of the DMC. The function gets the possible rate from OPP table based 651 * on requested frequency. It calls the next function responsible for the 652 * frequency and voltage change. In case of failure, does not set 'curr_rate' 653 * and returns error value to the framework. 654 */ 655 static int exynos5_dmc_target(struct device *dev, unsigned long *freq, 656 u32 flags) 657 { 658 struct exynos5_dmc *dmc = dev_get_drvdata(dev); 659 unsigned long target_rate = 0; 660 unsigned long target_volt = 0; 661 int ret; 662 663 ret = exynos5_dmc_get_volt_freq(dmc, freq, &target_rate, &target_volt, 664 flags); 665 666 if (ret) 667 return ret; 668 669 if (target_rate == dmc->curr_rate) 670 return 0; 671 672 mutex_lock(&dmc->lock); 673 674 ret = exynos5_dmc_change_freq_and_volt(dmc, target_rate, target_volt); 675 676 if (ret) { 677 mutex_unlock(&dmc->lock); 678 return ret; 679 } 680 681 dmc->curr_rate = target_rate; 682 683 mutex_unlock(&dmc->lock); 684 return 0; 685 } 686 687 /** 688 * exynos5_counters_get() - Gets the performance counters values. 689 * @dmc: device for which the counters are going to be checked 690 * @load_count: variable which is populated with counter value 691 * @total_count: variable which is used as 'wall clock' reference 692 * 693 * Function which provides performance counters values. It sums up counters for 694 * two DMC channels. The 'total_count' is used as a reference and max value. 695 * The ratio 'load_count/total_count' shows the busy percentage [0%, 100%]. 696 */ 697 static int exynos5_counters_get(struct exynos5_dmc *dmc, 698 unsigned long *load_count, 699 unsigned long *total_count) 700 { 701 unsigned long total = 0; 702 struct devfreq_event_data event; 703 int ret, i; 704 705 *load_count = 0; 706 707 /* Take into account only read+write counters, but stop all */ 708 for (i = 0; i < dmc->num_counters; i++) { 709 if (!dmc->counter[i]) 710 continue; 711 712 ret = devfreq_event_get_event(dmc->counter[i], &event); 713 if (ret < 0) 714 return ret; 715 716 *load_count += event.load_count; 717 718 if (total < event.total_count) 719 total = event.total_count; 720 } 721 722 *total_count = total; 723 724 return 0; 725 } 726 727 /** 728 * exynos5_dmc_start_perf_events() - Setup and start performance event counters 729 * @dmc: device for which the counters are going to be checked 730 * @beg_value: initial value for the counter 731 * 732 * Function which enables needed counters, interrupts and sets initial values 733 * then starts the counters. 734 */ 735 static void exynos5_dmc_start_perf_events(struct exynos5_dmc *dmc, 736 u32 beg_value) 737 { 738 /* Enable interrupts for counter 2 */ 739 writel(PERF_CNT2, dmc->base_drexi0 + DREX_INTENS_PPC); 740 writel(PERF_CNT2, dmc->base_drexi1 + DREX_INTENS_PPC); 741 742 /* Enable counter 2 and CCNT */ 743 writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi0 + DREX_CNTENS_PPC); 744 writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi1 + DREX_CNTENS_PPC); 745 746 /* Clear overflow flag for all counters */ 747 writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi0 + DREX_FLAG_PPC); 748 writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi1 + DREX_FLAG_PPC); 749 750 /* Reset all counters */ 751 writel(CC_RESET | PPC_COUNTER_RESET, dmc->base_drexi0 + DREX_PMNC_PPC); 752 writel(CC_RESET | PPC_COUNTER_RESET, dmc->base_drexi1 + DREX_PMNC_PPC); 753 754 /* 755 * Set start value for the counters, the number of samples that 756 * will be gathered is calculated as: 0xffffffff - beg_value 757 */ 758 writel(beg_value, dmc->base_drexi0 + DREX_PMCNT2_PPC); 759 writel(beg_value, dmc->base_drexi1 + DREX_PMCNT2_PPC); 760 761 /* Start all counters */ 762 writel(PPC_ENABLE, dmc->base_drexi0 + DREX_PMNC_PPC); 763 writel(PPC_ENABLE, dmc->base_drexi1 + DREX_PMNC_PPC); 764 } 765 766 /** 767 * exynos5_dmc_perf_events_calc() - Calculate utilization 768 * @dmc: device for which the counters are going to be checked 769 * @diff_ts: time between last interrupt and current one 770 * 771 * Function which calculates needed utilization for the devfreq governor. 772 * It prepares values for 'busy_time' and 'total_time' based on elapsed time 773 * between interrupts, which approximates utilization. 774 */ 775 static void exynos5_dmc_perf_events_calc(struct exynos5_dmc *dmc, u64 diff_ts) 776 { 777 /* 778 * This is a simple algorithm for managing traffic on DMC. 779 * When there is almost no load the counters overflow every 4s, 780 * no mater the DMC frequency. 781 * The high load might be approximated using linear function. 782 * Knowing that, simple calculation can provide 'busy_time' and 783 * 'total_time' to the devfreq governor which picks up target 784 * frequency. 785 * We want a fast ramp up and slow decay in frequency change function. 786 */ 787 if (diff_ts < PERF_EVENT_UP_DOWN_THRESHOLD) { 788 /* 789 * Set higher utilization for the simple_ondemand governor. 790 * The governor should increase the frequency of the DMC. 791 */ 792 dmc->load = 70; 793 dmc->total = 100; 794 } else { 795 /* 796 * Set low utilization for the simple_ondemand governor. 797 * The governor should decrease the frequency of the DMC. 798 */ 799 dmc->load = 35; 800 dmc->total = 100; 801 } 802 803 dev_dbg(dmc->dev, "diff_ts=%llu\n", diff_ts); 804 } 805 806 /** 807 * exynos5_dmc_perf_events_check() - Checks the status of the counters 808 * @dmc: device for which the counters are going to be checked 809 * 810 * Function which is called from threaded IRQ to check the counters state 811 * and to call approximation for the needed utilization. 812 */ 813 static void exynos5_dmc_perf_events_check(struct exynos5_dmc *dmc) 814 { 815 u32 val; 816 u64 diff_ts, ts; 817 818 ts = ktime_get_ns(); 819 820 /* Stop all counters */ 821 writel(0, dmc->base_drexi0 + DREX_PMNC_PPC); 822 writel(0, dmc->base_drexi1 + DREX_PMNC_PPC); 823 824 /* Check the source in interrupt flag registers (which channel) */ 825 val = readl(dmc->base_drexi0 + DREX_FLAG_PPC); 826 if (val) { 827 diff_ts = ts - dmc->last_overflow_ts[0]; 828 dmc->last_overflow_ts[0] = ts; 829 dev_dbg(dmc->dev, "drex0 0xE050 val= 0x%08x\n", val); 830 } else { 831 val = readl(dmc->base_drexi1 + DREX_FLAG_PPC); 832 diff_ts = ts - dmc->last_overflow_ts[1]; 833 dmc->last_overflow_ts[1] = ts; 834 dev_dbg(dmc->dev, "drex1 0xE050 val= 0x%08x\n", val); 835 } 836 837 exynos5_dmc_perf_events_calc(dmc, diff_ts); 838 839 exynos5_dmc_start_perf_events(dmc, PERF_COUNTER_START_VALUE); 840 } 841 842 /** 843 * exynos5_dmc_enable_perf_events() - Enable performance events 844 * @dmc: device for which the counters are going to be checked 845 * 846 * Function which is setup needed environment and enables counters. 847 */ 848 static void exynos5_dmc_enable_perf_events(struct exynos5_dmc *dmc) 849 { 850 u64 ts; 851 852 /* Enable Performance Event Clock */ 853 writel(PEREV_CLK_EN, dmc->base_drexi0 + DREX_PPCCLKCON); 854 writel(PEREV_CLK_EN, dmc->base_drexi1 + DREX_PPCCLKCON); 855 856 /* Select read transfers as performance event2 */ 857 writel(READ_TRANSFER_CH0, dmc->base_drexi0 + DREX_PEREV2CONFIG); 858 writel(READ_TRANSFER_CH1, dmc->base_drexi1 + DREX_PEREV2CONFIG); 859 860 ts = ktime_get_ns(); 861 dmc->last_overflow_ts[0] = ts; 862 dmc->last_overflow_ts[1] = ts; 863 864 /* Devfreq shouldn't be faster than initialization, play safe though. */ 865 dmc->load = 99; 866 dmc->total = 100; 867 } 868 869 /** 870 * exynos5_dmc_disable_perf_events() - Disable performance events 871 * @dmc: device for which the counters are going to be checked 872 * 873 * Function which stops, disables performance event counters and interrupts. 874 */ 875 static void exynos5_dmc_disable_perf_events(struct exynos5_dmc *dmc) 876 { 877 /* Stop all counters */ 878 writel(0, dmc->base_drexi0 + DREX_PMNC_PPC); 879 writel(0, dmc->base_drexi1 + DREX_PMNC_PPC); 880 881 /* Disable interrupts for counter 2 */ 882 writel(PERF_CNT2, dmc->base_drexi0 + DREX_INTENC_PPC); 883 writel(PERF_CNT2, dmc->base_drexi1 + DREX_INTENC_PPC); 884 885 /* Disable counter 2 and CCNT */ 886 writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi0 + DREX_CNTENC_PPC); 887 writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi1 + DREX_CNTENC_PPC); 888 889 /* Clear overflow flag for all counters */ 890 writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi0 + DREX_FLAG_PPC); 891 writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi1 + DREX_FLAG_PPC); 892 } 893 894 /** 895 * exynos5_dmc_get_status() - Read current DMC performance statistics. 896 * @dev: device for which the statistics are requested 897 * @stat: structure which has statistic fields 898 * 899 * Function reads the DMC performance counters and calculates 'busy_time' 900 * and 'total_time'. To protect from overflow, the values are shifted right 901 * by 10. After read out the counters are setup to count again. 902 */ 903 static int exynos5_dmc_get_status(struct device *dev, 904 struct devfreq_dev_status *stat) 905 { 906 struct exynos5_dmc *dmc = dev_get_drvdata(dev); 907 unsigned long load, total; 908 int ret; 909 910 if (dmc->in_irq_mode) { 911 stat->current_frequency = dmc->curr_rate; 912 stat->busy_time = dmc->load; 913 stat->total_time = dmc->total; 914 } else { 915 ret = exynos5_counters_get(dmc, &load, &total); 916 if (ret < 0) 917 return -EINVAL; 918 919 /* To protect from overflow, divide by 1024 */ 920 stat->busy_time = load >> 10; 921 stat->total_time = total >> 10; 922 923 ret = exynos5_counters_set_event(dmc); 924 if (ret < 0) { 925 dev_err(dev, "could not set event counter\n"); 926 return ret; 927 } 928 } 929 930 return 0; 931 } 932 933 /** 934 * exynos5_dmc_get_cur_freq() - Function returns current DMC frequency 935 * @dev: device for which the framework checks operating frequency 936 * @freq: returned frequency value 937 * 938 * It returns the currently used frequency of the DMC. The real operating 939 * frequency might be lower when the clock source value could not be divided 940 * to the requested value. 941 */ 942 static int exynos5_dmc_get_cur_freq(struct device *dev, unsigned long *freq) 943 { 944 struct exynos5_dmc *dmc = dev_get_drvdata(dev); 945 946 mutex_lock(&dmc->lock); 947 *freq = dmc->curr_rate; 948 mutex_unlock(&dmc->lock); 949 950 return 0; 951 } 952 953 /** 954 * exynos5_dmc_df_profile - Devfreq governor's profile structure 955 * 956 * It provides to the devfreq framework needed functions and polling period. 957 */ 958 static struct devfreq_dev_profile exynos5_dmc_df_profile = { 959 .timer = DEVFREQ_TIMER_DELAYED, 960 .target = exynos5_dmc_target, 961 .get_dev_status = exynos5_dmc_get_status, 962 .get_cur_freq = exynos5_dmc_get_cur_freq, 963 }; 964 965 /** 966 * exynos5_dmc_align_initial_frequency() - Align initial frequency value 967 * @dmc: device for which the frequency is going to be set 968 * @bootloader_init_freq: initial frequency set by the bootloader in KHz 969 * 970 * The initial bootloader frequency, which is present during boot, might be 971 * different that supported frequency values in the driver. It is possible 972 * due to different PLL settings or used PLL as a source. 973 * This function provides the 'initial_freq' for the devfreq framework 974 * statistics engine which supports only registered values. Thus, some alignment 975 * must be made. 976 */ 977 static unsigned long 978 exynos5_dmc_align_init_freq(struct exynos5_dmc *dmc, 979 unsigned long bootloader_init_freq) 980 { 981 unsigned long aligned_freq; 982 int idx; 983 984 idx = find_target_freq_idx(dmc, bootloader_init_freq); 985 if (idx >= 0) 986 aligned_freq = dmc->opp[idx].freq_hz; 987 else 988 aligned_freq = dmc->opp[dmc->opp_count - 1].freq_hz; 989 990 return aligned_freq; 991 } 992 993 /** 994 * create_timings_aligned() - Create register values and align with standard 995 * @dmc: device for which the frequency is going to be set 996 * @idx: speed bin in the OPP table 997 * @clk_period_ps: the period of the clock, known as tCK 998 * 999 * The function calculates timings and creates a register value ready for 1000 * a frequency transition. The register contains a few timings. They are 1001 * shifted by a known offset. The timing value is calculated based on memory 1002 * specyfication: minimal time required and minimal cycles required. 1003 */ 1004 static int create_timings_aligned(struct exynos5_dmc *dmc, u32 *reg_timing_row, 1005 u32 *reg_timing_data, u32 *reg_timing_power, 1006 u32 clk_period_ps) 1007 { 1008 u32 val; 1009 const struct timing_reg *reg; 1010 1011 if (clk_period_ps == 0) 1012 return -EINVAL; 1013 1014 *reg_timing_row = 0; 1015 *reg_timing_data = 0; 1016 *reg_timing_power = 0; 1017 1018 val = dmc->timings->tRFC / clk_period_ps; 1019 val += dmc->timings->tRFC % clk_period_ps ? 1 : 0; 1020 val = max(val, dmc->min_tck->tRFC); 1021 reg = &timing_row[0]; 1022 *reg_timing_row |= TIMING_VAL2REG(reg, val); 1023 1024 val = dmc->timings->tRRD / clk_period_ps; 1025 val += dmc->timings->tRRD % clk_period_ps ? 1 : 0; 1026 val = max(val, dmc->min_tck->tRRD); 1027 reg = &timing_row[1]; 1028 *reg_timing_row |= TIMING_VAL2REG(reg, val); 1029 1030 val = dmc->timings->tRPab / clk_period_ps; 1031 val += dmc->timings->tRPab % clk_period_ps ? 1 : 0; 1032 val = max(val, dmc->min_tck->tRPab); 1033 reg = &timing_row[2]; 1034 *reg_timing_row |= TIMING_VAL2REG(reg, val); 1035 1036 val = dmc->timings->tRCD / clk_period_ps; 1037 val += dmc->timings->tRCD % clk_period_ps ? 1 : 0; 1038 val = max(val, dmc->min_tck->tRCD); 1039 reg = &timing_row[3]; 1040 *reg_timing_row |= TIMING_VAL2REG(reg, val); 1041 1042 val = dmc->timings->tRC / clk_period_ps; 1043 val += dmc->timings->tRC % clk_period_ps ? 1 : 0; 1044 val = max(val, dmc->min_tck->tRC); 1045 reg = &timing_row[4]; 1046 *reg_timing_row |= TIMING_VAL2REG(reg, val); 1047 1048 val = dmc->timings->tRAS / clk_period_ps; 1049 val += dmc->timings->tRAS % clk_period_ps ? 1 : 0; 1050 val = max(val, dmc->min_tck->tRAS); 1051 reg = &timing_row[5]; 1052 *reg_timing_row |= TIMING_VAL2REG(reg, val); 1053 1054 /* data related timings */ 1055 val = dmc->timings->tWTR / clk_period_ps; 1056 val += dmc->timings->tWTR % clk_period_ps ? 1 : 0; 1057 val = max(val, dmc->min_tck->tWTR); 1058 reg = &timing_data[0]; 1059 *reg_timing_data |= TIMING_VAL2REG(reg, val); 1060 1061 val = dmc->timings->tWR / clk_period_ps; 1062 val += dmc->timings->tWR % clk_period_ps ? 1 : 0; 1063 val = max(val, dmc->min_tck->tWR); 1064 reg = &timing_data[1]; 1065 *reg_timing_data |= TIMING_VAL2REG(reg, val); 1066 1067 val = dmc->timings->tRTP / clk_period_ps; 1068 val += dmc->timings->tRTP % clk_period_ps ? 1 : 0; 1069 val = max(val, dmc->min_tck->tRTP); 1070 reg = &timing_data[2]; 1071 *reg_timing_data |= TIMING_VAL2REG(reg, val); 1072 1073 val = dmc->timings->tW2W_C2C / clk_period_ps; 1074 val += dmc->timings->tW2W_C2C % clk_period_ps ? 1 : 0; 1075 val = max(val, dmc->min_tck->tW2W_C2C); 1076 reg = &timing_data[3]; 1077 *reg_timing_data |= TIMING_VAL2REG(reg, val); 1078 1079 val = dmc->timings->tR2R_C2C / clk_period_ps; 1080 val += dmc->timings->tR2R_C2C % clk_period_ps ? 1 : 0; 1081 val = max(val, dmc->min_tck->tR2R_C2C); 1082 reg = &timing_data[4]; 1083 *reg_timing_data |= TIMING_VAL2REG(reg, val); 1084 1085 val = dmc->timings->tWL / clk_period_ps; 1086 val += dmc->timings->tWL % clk_period_ps ? 1 : 0; 1087 val = max(val, dmc->min_tck->tWL); 1088 reg = &timing_data[5]; 1089 *reg_timing_data |= TIMING_VAL2REG(reg, val); 1090 1091 val = dmc->timings->tDQSCK / clk_period_ps; 1092 val += dmc->timings->tDQSCK % clk_period_ps ? 1 : 0; 1093 val = max(val, dmc->min_tck->tDQSCK); 1094 reg = &timing_data[6]; 1095 *reg_timing_data |= TIMING_VAL2REG(reg, val); 1096 1097 val = dmc->timings->tRL / clk_period_ps; 1098 val += dmc->timings->tRL % clk_period_ps ? 1 : 0; 1099 val = max(val, dmc->min_tck->tRL); 1100 reg = &timing_data[7]; 1101 *reg_timing_data |= TIMING_VAL2REG(reg, val); 1102 1103 /* power related timings */ 1104 val = dmc->timings->tFAW / clk_period_ps; 1105 val += dmc->timings->tFAW % clk_period_ps ? 1 : 0; 1106 val = max(val, dmc->min_tck->tFAW); 1107 reg = &timing_power[0]; 1108 *reg_timing_power |= TIMING_VAL2REG(reg, val); 1109 1110 val = dmc->timings->tXSR / clk_period_ps; 1111 val += dmc->timings->tXSR % clk_period_ps ? 1 : 0; 1112 val = max(val, dmc->min_tck->tXSR); 1113 reg = &timing_power[1]; 1114 *reg_timing_power |= TIMING_VAL2REG(reg, val); 1115 1116 val = dmc->timings->tXP / clk_period_ps; 1117 val += dmc->timings->tXP % clk_period_ps ? 1 : 0; 1118 val = max(val, dmc->min_tck->tXP); 1119 reg = &timing_power[2]; 1120 *reg_timing_power |= TIMING_VAL2REG(reg, val); 1121 1122 val = dmc->timings->tCKE / clk_period_ps; 1123 val += dmc->timings->tCKE % clk_period_ps ? 1 : 0; 1124 val = max(val, dmc->min_tck->tCKE); 1125 reg = &timing_power[3]; 1126 *reg_timing_power |= TIMING_VAL2REG(reg, val); 1127 1128 val = dmc->timings->tMRD / clk_period_ps; 1129 val += dmc->timings->tMRD % clk_period_ps ? 1 : 0; 1130 val = max(val, dmc->min_tck->tMRD); 1131 reg = &timing_power[4]; 1132 *reg_timing_power |= TIMING_VAL2REG(reg, val); 1133 1134 return 0; 1135 } 1136 1137 /** 1138 * of_get_dram_timings() - helper function for parsing DT settings for DRAM 1139 * @dmc: device for which the frequency is going to be set 1140 * 1141 * The function parses DT entries with DRAM information. 1142 */ 1143 static int of_get_dram_timings(struct exynos5_dmc *dmc) 1144 { 1145 int ret = 0; 1146 int idx; 1147 struct device_node *np_ddr; 1148 u32 freq_mhz, clk_period_ps; 1149 1150 np_ddr = of_parse_phandle(dmc->dev->of_node, "device-handle", 0); 1151 if (!np_ddr) { 1152 dev_warn(dmc->dev, "could not find 'device-handle' in DT\n"); 1153 return -EINVAL; 1154 } 1155 1156 dmc->timing_row = devm_kmalloc_array(dmc->dev, TIMING_COUNT, 1157 sizeof(u32), GFP_KERNEL); 1158 if (!dmc->timing_row) 1159 return -ENOMEM; 1160 1161 dmc->timing_data = devm_kmalloc_array(dmc->dev, TIMING_COUNT, 1162 sizeof(u32), GFP_KERNEL); 1163 if (!dmc->timing_data) 1164 return -ENOMEM; 1165 1166 dmc->timing_power = devm_kmalloc_array(dmc->dev, TIMING_COUNT, 1167 sizeof(u32), GFP_KERNEL); 1168 if (!dmc->timing_power) 1169 return -ENOMEM; 1170 1171 dmc->timings = of_lpddr3_get_ddr_timings(np_ddr, dmc->dev, 1172 DDR_TYPE_LPDDR3, 1173 &dmc->timings_arr_size); 1174 if (!dmc->timings) { 1175 of_node_put(np_ddr); 1176 dev_warn(dmc->dev, "could not get timings from DT\n"); 1177 return -EINVAL; 1178 } 1179 1180 dmc->min_tck = of_lpddr3_get_min_tck(np_ddr, dmc->dev); 1181 if (!dmc->min_tck) { 1182 of_node_put(np_ddr); 1183 dev_warn(dmc->dev, "could not get tck from DT\n"); 1184 return -EINVAL; 1185 } 1186 1187 /* Sorted array of OPPs with frequency ascending */ 1188 for (idx = 0; idx < dmc->opp_count; idx++) { 1189 freq_mhz = dmc->opp[idx].freq_hz / 1000000; 1190 clk_period_ps = 1000000 / freq_mhz; 1191 1192 ret = create_timings_aligned(dmc, &dmc->timing_row[idx], 1193 &dmc->timing_data[idx], 1194 &dmc->timing_power[idx], 1195 clk_period_ps); 1196 } 1197 1198 of_node_put(np_ddr); 1199 1200 /* Take the highest frequency's timings as 'bypass' */ 1201 dmc->bypass_timing_row = dmc->timing_row[idx - 1]; 1202 dmc->bypass_timing_data = dmc->timing_data[idx - 1]; 1203 dmc->bypass_timing_power = dmc->timing_power[idx - 1]; 1204 1205 return ret; 1206 } 1207 1208 /** 1209 * exynos5_dmc_init_clks() - Initialize clocks needed for DMC operation. 1210 * @dmc: DMC structure containing needed fields 1211 * 1212 * Get the needed clocks defined in DT device, enable and set the right parents. 1213 * Read current frequency and initialize the initial rate for governor. 1214 */ 1215 static int exynos5_dmc_init_clks(struct exynos5_dmc *dmc) 1216 { 1217 int ret; 1218 unsigned long target_volt = 0; 1219 unsigned long target_rate = 0; 1220 unsigned int tmp; 1221 1222 dmc->fout_spll = devm_clk_get(dmc->dev, "fout_spll"); 1223 if (IS_ERR(dmc->fout_spll)) 1224 return PTR_ERR(dmc->fout_spll); 1225 1226 dmc->fout_bpll = devm_clk_get(dmc->dev, "fout_bpll"); 1227 if (IS_ERR(dmc->fout_bpll)) 1228 return PTR_ERR(dmc->fout_bpll); 1229 1230 dmc->mout_mclk_cdrex = devm_clk_get(dmc->dev, "mout_mclk_cdrex"); 1231 if (IS_ERR(dmc->mout_mclk_cdrex)) 1232 return PTR_ERR(dmc->mout_mclk_cdrex); 1233 1234 dmc->mout_bpll = devm_clk_get(dmc->dev, "mout_bpll"); 1235 if (IS_ERR(dmc->mout_bpll)) 1236 return PTR_ERR(dmc->mout_bpll); 1237 1238 dmc->mout_mx_mspll_ccore = devm_clk_get(dmc->dev, 1239 "mout_mx_mspll_ccore"); 1240 if (IS_ERR(dmc->mout_mx_mspll_ccore)) 1241 return PTR_ERR(dmc->mout_mx_mspll_ccore); 1242 1243 dmc->mout_spll = devm_clk_get(dmc->dev, "ff_dout_spll2"); 1244 if (IS_ERR(dmc->mout_spll)) { 1245 dmc->mout_spll = devm_clk_get(dmc->dev, "mout_sclk_spll"); 1246 if (IS_ERR(dmc->mout_spll)) 1247 return PTR_ERR(dmc->mout_spll); 1248 } 1249 1250 /* 1251 * Convert frequency to KHz values and set it for the governor. 1252 */ 1253 dmc->curr_rate = clk_get_rate(dmc->mout_mclk_cdrex); 1254 dmc->curr_rate = exynos5_dmc_align_init_freq(dmc, dmc->curr_rate); 1255 exynos5_dmc_df_profile.initial_freq = dmc->curr_rate; 1256 1257 ret = exynos5_dmc_get_volt_freq(dmc, &dmc->curr_rate, &target_rate, 1258 &target_volt, 0); 1259 if (ret) 1260 return ret; 1261 1262 dmc->curr_volt = target_volt; 1263 1264 clk_set_parent(dmc->mout_mx_mspll_ccore, dmc->mout_spll); 1265 1266 dmc->bypass_rate = clk_get_rate(dmc->mout_mx_mspll_ccore); 1267 1268 clk_prepare_enable(dmc->fout_bpll); 1269 clk_prepare_enable(dmc->mout_bpll); 1270 1271 /* 1272 * Some bootloaders do not set clock routes correctly. 1273 * Stop one path in clocks to PHY. 1274 */ 1275 regmap_read(dmc->clk_regmap, CDREX_LPDDR3PHY_CLKM_SRC, &tmp); 1276 tmp &= ~(BIT(1) | BIT(0)); 1277 regmap_write(dmc->clk_regmap, CDREX_LPDDR3PHY_CLKM_SRC, tmp); 1278 1279 return 0; 1280 } 1281 1282 /** 1283 * exynos5_performance_counters_init() - Initializes performance DMC's counters 1284 * @dmc: DMC for which it does the setup 1285 * 1286 * Initialization of performance counters in DMC for estimating usage. 1287 * The counter's values are used for calculation of a memory bandwidth and based 1288 * on that the governor changes the frequency. 1289 * The counters are not used when the governor is GOVERNOR_USERSPACE. 1290 */ 1291 static int exynos5_performance_counters_init(struct exynos5_dmc *dmc) 1292 { 1293 int counters_size; 1294 int ret, i; 1295 1296 dmc->num_counters = devfreq_event_get_edev_count(dmc->dev); 1297 if (dmc->num_counters < 0) { 1298 dev_err(dmc->dev, "could not get devfreq-event counters\n"); 1299 return dmc->num_counters; 1300 } 1301 1302 counters_size = sizeof(struct devfreq_event_dev) * dmc->num_counters; 1303 dmc->counter = devm_kzalloc(dmc->dev, counters_size, GFP_KERNEL); 1304 if (!dmc->counter) 1305 return -ENOMEM; 1306 1307 for (i = 0; i < dmc->num_counters; i++) { 1308 dmc->counter[i] = 1309 devfreq_event_get_edev_by_phandle(dmc->dev, i); 1310 if (IS_ERR_OR_NULL(dmc->counter[i])) 1311 return -EPROBE_DEFER; 1312 } 1313 1314 ret = exynos5_counters_enable_edev(dmc); 1315 if (ret < 0) { 1316 dev_err(dmc->dev, "could not enable event counter\n"); 1317 return ret; 1318 } 1319 1320 ret = exynos5_counters_set_event(dmc); 1321 if (ret < 0) { 1322 exynos5_counters_disable_edev(dmc); 1323 dev_err(dmc->dev, "could not set event counter\n"); 1324 return ret; 1325 } 1326 1327 return 0; 1328 } 1329 1330 /** 1331 * exynos5_dmc_set_pause_on_switching() - Controls a pause feature in DMC 1332 * @dmc: device which is used for changing this feature 1333 * @set: a boolean state passing enable/disable request 1334 * 1335 * There is a need of pausing DREX DMC when divider or MUX in clock tree 1336 * changes its configuration. In such situation access to the memory is blocked 1337 * in DMC automatically. This feature is used when clock frequency change 1338 * request appears and touches clock tree. 1339 */ 1340 static inline int exynos5_dmc_set_pause_on_switching(struct exynos5_dmc *dmc) 1341 { 1342 unsigned int val; 1343 int ret; 1344 1345 ret = regmap_read(dmc->clk_regmap, CDREX_PAUSE, &val); 1346 if (ret) 1347 return ret; 1348 1349 val |= 1UL; 1350 regmap_write(dmc->clk_regmap, CDREX_PAUSE, val); 1351 1352 return 0; 1353 } 1354 1355 static irqreturn_t dmc_irq_thread(int irq, void *priv) 1356 { 1357 int res; 1358 struct exynos5_dmc *dmc = priv; 1359 1360 mutex_lock(&dmc->df->lock); 1361 exynos5_dmc_perf_events_check(dmc); 1362 res = update_devfreq(dmc->df); 1363 mutex_unlock(&dmc->df->lock); 1364 1365 if (res) 1366 dev_warn(dmc->dev, "devfreq failed with %d\n", res); 1367 1368 return IRQ_HANDLED; 1369 } 1370 1371 /** 1372 * exynos5_dmc_probe() - Probe function for the DMC driver 1373 * @pdev: platform device for which the driver is going to be initialized 1374 * 1375 * Initialize basic components: clocks, regulators, performance counters, etc. 1376 * Read out product version and based on the information setup 1377 * internal structures for the controller (frequency and voltage) and for DRAM 1378 * memory parameters: timings for each operating frequency. 1379 * Register new devfreq device for controlling DVFS of the DMC. 1380 */ 1381 static int exynos5_dmc_probe(struct platform_device *pdev) 1382 { 1383 int ret = 0; 1384 struct device *dev = &pdev->dev; 1385 struct device_node *np = dev->of_node; 1386 struct exynos5_dmc *dmc; 1387 int irq[2]; 1388 1389 dmc = devm_kzalloc(dev, sizeof(*dmc), GFP_KERNEL); 1390 if (!dmc) 1391 return -ENOMEM; 1392 1393 mutex_init(&dmc->lock); 1394 1395 dmc->dev = dev; 1396 platform_set_drvdata(pdev, dmc); 1397 1398 dmc->base_drexi0 = devm_platform_ioremap_resource(pdev, 0); 1399 if (IS_ERR(dmc->base_drexi0)) 1400 return PTR_ERR(dmc->base_drexi0); 1401 1402 dmc->base_drexi1 = devm_platform_ioremap_resource(pdev, 1); 1403 if (IS_ERR(dmc->base_drexi1)) 1404 return PTR_ERR(dmc->base_drexi1); 1405 1406 dmc->clk_regmap = syscon_regmap_lookup_by_phandle(np, 1407 "samsung,syscon-clk"); 1408 if (IS_ERR(dmc->clk_regmap)) 1409 return PTR_ERR(dmc->clk_regmap); 1410 1411 ret = exynos5_init_freq_table(dmc, &exynos5_dmc_df_profile); 1412 if (ret) { 1413 dev_warn(dev, "couldn't initialize frequency settings\n"); 1414 return ret; 1415 } 1416 1417 dmc->vdd_mif = devm_regulator_get(dev, "vdd"); 1418 if (IS_ERR(dmc->vdd_mif)) { 1419 ret = PTR_ERR(dmc->vdd_mif); 1420 return ret; 1421 } 1422 1423 ret = exynos5_dmc_init_clks(dmc); 1424 if (ret) 1425 return ret; 1426 1427 ret = of_get_dram_timings(dmc); 1428 if (ret) { 1429 dev_warn(dev, "couldn't initialize timings settings\n"); 1430 goto remove_clocks; 1431 } 1432 1433 ret = exynos5_dmc_set_pause_on_switching(dmc); 1434 if (ret) { 1435 dev_warn(dev, "couldn't get access to PAUSE register\n"); 1436 goto remove_clocks; 1437 } 1438 1439 /* There is two modes in which the driver works: polling or IRQ */ 1440 irq[0] = platform_get_irq_byname(pdev, "drex_0"); 1441 irq[1] = platform_get_irq_byname(pdev, "drex_1"); 1442 if (irq[0] > 0 && irq[1] > 0 && irqmode) { 1443 ret = devm_request_threaded_irq(dev, irq[0], NULL, 1444 dmc_irq_thread, IRQF_ONESHOT, 1445 dev_name(dev), dmc); 1446 if (ret) { 1447 dev_err(dev, "couldn't grab IRQ\n"); 1448 goto remove_clocks; 1449 } 1450 1451 ret = devm_request_threaded_irq(dev, irq[1], NULL, 1452 dmc_irq_thread, IRQF_ONESHOT, 1453 dev_name(dev), dmc); 1454 if (ret) { 1455 dev_err(dev, "couldn't grab IRQ\n"); 1456 goto remove_clocks; 1457 } 1458 1459 /* 1460 * Setup default thresholds for the devfreq governor. 1461 * The values are chosen based on experiments. 1462 */ 1463 dmc->gov_data.upthreshold = 55; 1464 dmc->gov_data.downdifferential = 5; 1465 1466 exynos5_dmc_enable_perf_events(dmc); 1467 1468 dmc->in_irq_mode = 1; 1469 } else { 1470 ret = exynos5_performance_counters_init(dmc); 1471 if (ret) { 1472 dev_warn(dev, "couldn't probe performance counters\n"); 1473 goto remove_clocks; 1474 } 1475 1476 /* 1477 * Setup default thresholds for the devfreq governor. 1478 * The values are chosen based on experiments. 1479 */ 1480 dmc->gov_data.upthreshold = 10; 1481 dmc->gov_data.downdifferential = 5; 1482 1483 exynos5_dmc_df_profile.polling_ms = 100; 1484 } 1485 1486 dmc->df = devm_devfreq_add_device(dev, &exynos5_dmc_df_profile, 1487 DEVFREQ_GOV_SIMPLE_ONDEMAND, 1488 &dmc->gov_data); 1489 1490 if (IS_ERR(dmc->df)) { 1491 ret = PTR_ERR(dmc->df); 1492 goto err_devfreq_add; 1493 } 1494 1495 if (dmc->in_irq_mode) 1496 exynos5_dmc_start_perf_events(dmc, PERF_COUNTER_START_VALUE); 1497 1498 dev_info(dev, "DMC initialized, in irq mode: %d\n", dmc->in_irq_mode); 1499 1500 return 0; 1501 1502 err_devfreq_add: 1503 if (dmc->in_irq_mode) 1504 exynos5_dmc_disable_perf_events(dmc); 1505 else 1506 exynos5_counters_disable_edev(dmc); 1507 remove_clocks: 1508 clk_disable_unprepare(dmc->mout_bpll); 1509 clk_disable_unprepare(dmc->fout_bpll); 1510 1511 return ret; 1512 } 1513 1514 /** 1515 * exynos5_dmc_remove() - Remove function for the platform device 1516 * @pdev: platform device which is going to be removed 1517 * 1518 * The function relies on 'devm' framework function which automatically 1519 * clean the device's resources. It just calls explicitly disable function for 1520 * the performance counters. 1521 */ 1522 static int exynos5_dmc_remove(struct platform_device *pdev) 1523 { 1524 struct exynos5_dmc *dmc = dev_get_drvdata(&pdev->dev); 1525 1526 if (dmc->in_irq_mode) 1527 exynos5_dmc_disable_perf_events(dmc); 1528 else 1529 exynos5_counters_disable_edev(dmc); 1530 1531 clk_disable_unprepare(dmc->mout_bpll); 1532 clk_disable_unprepare(dmc->fout_bpll); 1533 1534 dev_pm_opp_remove_table(dmc->dev); 1535 1536 return 0; 1537 } 1538 1539 static const struct of_device_id exynos5_dmc_of_match[] = { 1540 { .compatible = "samsung,exynos5422-dmc", }, 1541 { }, 1542 }; 1543 MODULE_DEVICE_TABLE(of, exynos5_dmc_of_match); 1544 1545 static struct platform_driver exynos5_dmc_platdrv = { 1546 .probe = exynos5_dmc_probe, 1547 .remove = exynos5_dmc_remove, 1548 .driver = { 1549 .name = "exynos5-dmc", 1550 .of_match_table = exynos5_dmc_of_match, 1551 }, 1552 }; 1553 module_platform_driver(exynos5_dmc_platdrv); 1554 MODULE_DESCRIPTION("Driver for Exynos5422 Dynamic Memory Controller dynamic frequency and voltage change"); 1555 MODULE_LICENSE("GPL v2"); 1556 MODULE_AUTHOR("Lukasz Luba"); 1557