1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2014-2020, NVIDIA CORPORATION. All rights reserved. 4 */ 5 6 #include <linux/kernel.h> 7 #include <linux/io.h> 8 #include <linux/clk.h> 9 #include <linux/delay.h> 10 #include <linux/of.h> 11 12 #include <soc/tegra/mc.h> 13 14 #include "tegra210-emc.h" 15 #include "tegra210-mc.h" 16 17 /* 18 * Enable flags for specifying verbosity. 19 */ 20 #define INFO (1 << 0) 21 #define STEPS (1 << 1) 22 #define SUB_STEPS (1 << 2) 23 #define PRELOCK (1 << 3) 24 #define PRELOCK_STEPS (1 << 4) 25 #define ACTIVE_EN (1 << 5) 26 #define PRAMP_UP (1 << 6) 27 #define PRAMP_DN (1 << 7) 28 #define EMA_WRITES (1 << 10) 29 #define EMA_UPDATES (1 << 11) 30 #define PER_TRAIN (1 << 16) 31 #define CC_PRINT (1 << 17) 32 #define CCFIFO (1 << 29) 33 #define REGS (1 << 30) 34 #define REG_LISTS (1 << 31) 35 36 #define emc_dbg(emc, flags, ...) dev_dbg(emc->dev, __VA_ARGS__) 37 38 #define DVFS_CLOCK_CHANGE_VERSION 21021 39 #define EMC_PRELOCK_VERSION 2101 40 41 enum { 42 DVFS_SEQUENCE = 1, 43 WRITE_TRAINING_SEQUENCE = 2, 44 PERIODIC_TRAINING_SEQUENCE = 3, 45 DVFS_PT1 = 10, 46 DVFS_UPDATE = 11, 47 TRAINING_PT1 = 12, 48 TRAINING_UPDATE = 13, 49 PERIODIC_TRAINING_UPDATE = 14 50 }; 51 52 /* 53 * PTFV defines - basically just indexes into the per table PTFV array. 54 */ 55 #define PTFV_DQSOSC_MOVAVG_C0D0U0_INDEX 0 56 #define PTFV_DQSOSC_MOVAVG_C0D0U1_INDEX 1 57 #define PTFV_DQSOSC_MOVAVG_C0D1U0_INDEX 2 58 #define PTFV_DQSOSC_MOVAVG_C0D1U1_INDEX 3 59 #define PTFV_DQSOSC_MOVAVG_C1D0U0_INDEX 4 60 #define PTFV_DQSOSC_MOVAVG_C1D0U1_INDEX 5 61 #define PTFV_DQSOSC_MOVAVG_C1D1U0_INDEX 6 62 #define PTFV_DQSOSC_MOVAVG_C1D1U1_INDEX 7 63 #define PTFV_DVFS_SAMPLES_INDEX 9 64 #define PTFV_MOVAVG_WEIGHT_INDEX 10 65 #define PTFV_CONFIG_CTRL_INDEX 11 66 67 #define PTFV_CONFIG_CTRL_USE_PREVIOUS_EMA (1 << 0) 68 69 /* 70 * Do arithmetic in fixed point. 71 */ 72 #define MOVAVG_PRECISION_FACTOR 100 73 74 /* 75 * The division portion of the average operation. 76 */ 77 #define __AVERAGE_PTFV(dev) \ 78 ({ next->ptfv_list[PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX] = \ 79 next->ptfv_list[PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX] / \ 80 next->ptfv_list[PTFV_DVFS_SAMPLES_INDEX]; }) 81 82 /* 83 * Convert val to fixed point and add it to the temporary average. 84 */ 85 #define __INCREMENT_PTFV(dev, val) \ 86 ({ next->ptfv_list[PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX] += \ 87 ((val) * MOVAVG_PRECISION_FACTOR); }) 88 89 /* 90 * Convert a moving average back to integral form and return the value. 91 */ 92 #define __MOVAVG_AC(timing, dev) \ 93 ((timing)->ptfv_list[PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX] / \ 94 MOVAVG_PRECISION_FACTOR) 95 96 /* Weighted update. */ 97 #define __WEIGHTED_UPDATE_PTFV(dev, nval) \ 98 do { \ 99 int w = PTFV_MOVAVG_WEIGHT_INDEX; \ 100 int dqs = PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX; \ 101 \ 102 next->ptfv_list[dqs] = \ 103 ((nval * MOVAVG_PRECISION_FACTOR) + \ 104 (next->ptfv_list[dqs] * \ 105 next->ptfv_list[w])) / \ 106 (next->ptfv_list[w] + 1); \ 107 \ 108 emc_dbg(emc, EMA_UPDATES, "%s: (s=%lu) EMA: %u\n", \ 109 __stringify(dev), nval, next->ptfv_list[dqs]); \ 110 } while (0) 111 112 /* Access a particular average. */ 113 #define __MOVAVG(timing, dev) \ 114 ((timing)->ptfv_list[PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX]) 115 116 static u32 update_clock_tree_delay(struct tegra210_emc *emc, int type) 117 { 118 bool periodic_training_update = type == PERIODIC_TRAINING_UPDATE; 119 struct tegra210_emc_timing *last = emc->last; 120 struct tegra210_emc_timing *next = emc->next; 121 u32 last_timing_rate_mhz = last->rate / 1000; 122 u32 next_timing_rate_mhz = next->rate / 1000; 123 bool dvfs_update = type == DVFS_UPDATE; 124 s32 tdel = 0, tmdel = 0, adel = 0; 125 bool dvfs_pt1 = type == DVFS_PT1; 126 unsigned long cval = 0; 127 u32 temp[2][2], value; 128 unsigned int i; 129 130 /* 131 * Dev0 MSB. 132 */ 133 if (dvfs_pt1 || periodic_training_update) { 134 value = tegra210_emc_mrr_read(emc, 2, 19); 135 136 for (i = 0; i < emc->num_channels; i++) { 137 temp[i][0] = (value & 0x00ff) << 8; 138 temp[i][1] = (value & 0xff00) << 0; 139 value >>= 16; 140 } 141 142 /* 143 * Dev0 LSB. 144 */ 145 value = tegra210_emc_mrr_read(emc, 2, 18); 146 147 for (i = 0; i < emc->num_channels; i++) { 148 temp[i][0] |= (value & 0x00ff) >> 0; 149 temp[i][1] |= (value & 0xff00) >> 8; 150 value >>= 16; 151 } 152 } 153 154 if (dvfs_pt1 || periodic_training_update) { 155 cval = tegra210_emc_actual_osc_clocks(last->run_clocks); 156 cval *= 1000000; 157 cval /= last_timing_rate_mhz * 2 * temp[0][0]; 158 } 159 160 if (dvfs_pt1) 161 __INCREMENT_PTFV(C0D0U0, cval); 162 else if (dvfs_update) 163 __AVERAGE_PTFV(C0D0U0); 164 else if (periodic_training_update) 165 __WEIGHTED_UPDATE_PTFV(C0D0U0, cval); 166 167 if (dvfs_update || periodic_training_update) { 168 tdel = next->current_dram_clktree[C0D0U0] - 169 __MOVAVG_AC(next, C0D0U0); 170 tmdel = (tdel < 0) ? -1 * tdel : tdel; 171 adel = tmdel; 172 173 if (tmdel * 128 * next_timing_rate_mhz / 1000000 > 174 next->tree_margin) 175 next->current_dram_clktree[C0D0U0] = 176 __MOVAVG_AC(next, C0D0U0); 177 } 178 179 if (dvfs_pt1 || periodic_training_update) { 180 cval = tegra210_emc_actual_osc_clocks(last->run_clocks); 181 cval *= 1000000; 182 cval /= last_timing_rate_mhz * 2 * temp[0][1]; 183 } 184 185 if (dvfs_pt1) 186 __INCREMENT_PTFV(C0D0U1, cval); 187 else if (dvfs_update) 188 __AVERAGE_PTFV(C0D0U1); 189 else if (periodic_training_update) 190 __WEIGHTED_UPDATE_PTFV(C0D0U1, cval); 191 192 if (dvfs_update || periodic_training_update) { 193 tdel = next->current_dram_clktree[C0D0U1] - 194 __MOVAVG_AC(next, C0D0U1); 195 tmdel = (tdel < 0) ? -1 * tdel : tdel; 196 197 if (tmdel > adel) 198 adel = tmdel; 199 200 if (tmdel * 128 * next_timing_rate_mhz / 1000000 > 201 next->tree_margin) 202 next->current_dram_clktree[C0D0U1] = 203 __MOVAVG_AC(next, C0D0U1); 204 } 205 206 if (emc->num_channels > 1) { 207 if (dvfs_pt1 || periodic_training_update) { 208 cval = tegra210_emc_actual_osc_clocks(last->run_clocks); 209 cval *= 1000000; 210 cval /= last_timing_rate_mhz * 2 * temp[1][0]; 211 } 212 213 if (dvfs_pt1) 214 __INCREMENT_PTFV(C1D0U0, cval); 215 else if (dvfs_update) 216 __AVERAGE_PTFV(C1D0U0); 217 else if (periodic_training_update) 218 __WEIGHTED_UPDATE_PTFV(C1D0U0, cval); 219 220 if (dvfs_update || periodic_training_update) { 221 tdel = next->current_dram_clktree[C1D0U0] - 222 __MOVAVG_AC(next, C1D0U0); 223 tmdel = (tdel < 0) ? -1 * tdel : tdel; 224 225 if (tmdel > adel) 226 adel = tmdel; 227 228 if (tmdel * 128 * next_timing_rate_mhz / 1000000 > 229 next->tree_margin) 230 next->current_dram_clktree[C1D0U0] = 231 __MOVAVG_AC(next, C1D0U0); 232 } 233 234 if (dvfs_pt1 || periodic_training_update) { 235 cval = tegra210_emc_actual_osc_clocks(last->run_clocks); 236 cval *= 1000000; 237 cval /= last_timing_rate_mhz * 2 * temp[1][1]; 238 } 239 240 if (dvfs_pt1) 241 __INCREMENT_PTFV(C1D0U1, cval); 242 else if (dvfs_update) 243 __AVERAGE_PTFV(C1D0U1); 244 else if (periodic_training_update) 245 __WEIGHTED_UPDATE_PTFV(C1D0U1, cval); 246 247 if (dvfs_update || periodic_training_update) { 248 tdel = next->current_dram_clktree[C1D0U1] - 249 __MOVAVG_AC(next, C1D0U1); 250 tmdel = (tdel < 0) ? -1 * tdel : tdel; 251 252 if (tmdel > adel) 253 adel = tmdel; 254 255 if (tmdel * 128 * next_timing_rate_mhz / 1000000 > 256 next->tree_margin) 257 next->current_dram_clktree[C1D0U1] = 258 __MOVAVG_AC(next, C1D0U1); 259 } 260 } 261 262 if (emc->num_devices < 2) 263 goto done; 264 265 /* 266 * Dev1 MSB. 267 */ 268 if (dvfs_pt1 || periodic_training_update) { 269 value = tegra210_emc_mrr_read(emc, 1, 19); 270 271 for (i = 0; i < emc->num_channels; i++) { 272 temp[i][0] = (value & 0x00ff) << 8; 273 temp[i][1] = (value & 0xff00) << 0; 274 value >>= 16; 275 } 276 277 /* 278 * Dev1 LSB. 279 */ 280 value = tegra210_emc_mrr_read(emc, 2, 18); 281 282 for (i = 0; i < emc->num_channels; i++) { 283 temp[i][0] |= (value & 0x00ff) >> 0; 284 temp[i][1] |= (value & 0xff00) >> 8; 285 value >>= 16; 286 } 287 } 288 289 if (dvfs_pt1 || periodic_training_update) { 290 cval = tegra210_emc_actual_osc_clocks(last->run_clocks); 291 cval *= 1000000; 292 cval /= last_timing_rate_mhz * 2 * temp[0][0]; 293 } 294 295 if (dvfs_pt1) 296 __INCREMENT_PTFV(C0D1U0, cval); 297 else if (dvfs_update) 298 __AVERAGE_PTFV(C0D1U0); 299 else if (periodic_training_update) 300 __WEIGHTED_UPDATE_PTFV(C0D1U0, cval); 301 302 if (dvfs_update || periodic_training_update) { 303 tdel = next->current_dram_clktree[C0D1U0] - 304 __MOVAVG_AC(next, C0D1U0); 305 tmdel = (tdel < 0) ? -1 * tdel : tdel; 306 307 if (tmdel > adel) 308 adel = tmdel; 309 310 if (tmdel * 128 * next_timing_rate_mhz / 1000000 > 311 next->tree_margin) 312 next->current_dram_clktree[C0D1U0] = 313 __MOVAVG_AC(next, C0D1U0); 314 } 315 316 if (dvfs_pt1 || periodic_training_update) { 317 cval = tegra210_emc_actual_osc_clocks(last->run_clocks); 318 cval *= 1000000; 319 cval /= last_timing_rate_mhz * 2 * temp[0][1]; 320 } 321 322 if (dvfs_pt1) 323 __INCREMENT_PTFV(C0D1U1, cval); 324 else if (dvfs_update) 325 __AVERAGE_PTFV(C0D1U1); 326 else if (periodic_training_update) 327 __WEIGHTED_UPDATE_PTFV(C0D1U1, cval); 328 329 if (dvfs_update || periodic_training_update) { 330 tdel = next->current_dram_clktree[C0D1U1] - 331 __MOVAVG_AC(next, C0D1U1); 332 tmdel = (tdel < 0) ? -1 * tdel : tdel; 333 334 if (tmdel > adel) 335 adel = tmdel; 336 337 if (tmdel * 128 * next_timing_rate_mhz / 1000000 > 338 next->tree_margin) 339 next->current_dram_clktree[C0D1U1] = 340 __MOVAVG_AC(next, C0D1U1); 341 } 342 343 if (emc->num_channels > 1) { 344 if (dvfs_pt1 || periodic_training_update) { 345 cval = tegra210_emc_actual_osc_clocks(last->run_clocks); 346 cval *= 1000000; 347 cval /= last_timing_rate_mhz * 2 * temp[1][0]; 348 } 349 350 if (dvfs_pt1) 351 __INCREMENT_PTFV(C1D1U0, cval); 352 else if (dvfs_update) 353 __AVERAGE_PTFV(C1D1U0); 354 else if (periodic_training_update) 355 __WEIGHTED_UPDATE_PTFV(C1D1U0, cval); 356 357 if (dvfs_update || periodic_training_update) { 358 tdel = next->current_dram_clktree[C1D1U0] - 359 __MOVAVG_AC(next, C1D1U0); 360 tmdel = (tdel < 0) ? -1 * tdel : tdel; 361 362 if (tmdel > adel) 363 adel = tmdel; 364 365 if (tmdel * 128 * next_timing_rate_mhz / 1000000 > 366 next->tree_margin) 367 next->current_dram_clktree[C1D1U0] = 368 __MOVAVG_AC(next, C1D1U0); 369 } 370 371 if (dvfs_pt1 || periodic_training_update) { 372 cval = tegra210_emc_actual_osc_clocks(last->run_clocks); 373 cval *= 1000000; 374 cval /= last_timing_rate_mhz * 2 * temp[1][1]; 375 } 376 377 if (dvfs_pt1) 378 __INCREMENT_PTFV(C1D1U1, cval); 379 else if (dvfs_update) 380 __AVERAGE_PTFV(C1D1U1); 381 else if (periodic_training_update) 382 __WEIGHTED_UPDATE_PTFV(C1D1U1, cval); 383 384 if (dvfs_update || periodic_training_update) { 385 tdel = next->current_dram_clktree[C1D1U1] - 386 __MOVAVG_AC(next, C1D1U1); 387 tmdel = (tdel < 0) ? -1 * tdel : tdel; 388 389 if (tmdel > adel) 390 adel = tmdel; 391 392 if (tmdel * 128 * next_timing_rate_mhz / 1000000 > 393 next->tree_margin) 394 next->current_dram_clktree[C1D1U1] = 395 __MOVAVG_AC(next, C1D1U1); 396 } 397 } 398 399 done: 400 return adel; 401 } 402 403 static u32 periodic_compensation_handler(struct tegra210_emc *emc, u32 type, 404 struct tegra210_emc_timing *last, 405 struct tegra210_emc_timing *next) 406 { 407 #define __COPY_EMA(nt, lt, dev) \ 408 ({ __MOVAVG(nt, dev) = __MOVAVG(lt, dev) * \ 409 (nt)->ptfv_list[PTFV_DVFS_SAMPLES_INDEX]; }) 410 411 u32 i, adel = 0, samples = next->ptfv_list[PTFV_DVFS_SAMPLES_INDEX]; 412 u32 delay; 413 414 delay = tegra210_emc_actual_osc_clocks(last->run_clocks); 415 delay *= 1000; 416 delay = 2 + (delay / last->rate); 417 418 if (!next->periodic_training) 419 return 0; 420 421 if (type == DVFS_SEQUENCE) { 422 if (last->periodic_training && 423 (next->ptfv_list[PTFV_CONFIG_CTRL_INDEX] & 424 PTFV_CONFIG_CTRL_USE_PREVIOUS_EMA)) { 425 /* 426 * If the previous frequency was using periodic 427 * calibration then we can reuse the previous 428 * frequencies EMA data. 429 */ 430 __COPY_EMA(next, last, C0D0U0); 431 __COPY_EMA(next, last, C0D0U1); 432 __COPY_EMA(next, last, C1D0U0); 433 __COPY_EMA(next, last, C1D0U1); 434 __COPY_EMA(next, last, C0D1U0); 435 __COPY_EMA(next, last, C0D1U1); 436 __COPY_EMA(next, last, C1D1U0); 437 __COPY_EMA(next, last, C1D1U1); 438 } else { 439 /* Reset the EMA.*/ 440 __MOVAVG(next, C0D0U0) = 0; 441 __MOVAVG(next, C0D0U1) = 0; 442 __MOVAVG(next, C1D0U0) = 0; 443 __MOVAVG(next, C1D0U1) = 0; 444 __MOVAVG(next, C0D1U0) = 0; 445 __MOVAVG(next, C0D1U1) = 0; 446 __MOVAVG(next, C1D1U0) = 0; 447 __MOVAVG(next, C1D1U1) = 0; 448 449 for (i = 0; i < samples; i++) { 450 tegra210_emc_start_periodic_compensation(emc); 451 udelay(delay); 452 453 /* 454 * Generate next sample of data. 455 */ 456 adel = update_clock_tree_delay(emc, DVFS_PT1); 457 } 458 } 459 460 /* 461 * Seems like it should be part of the 462 * 'if (last_timing->periodic_training)' conditional 463 * since is already done for the else clause. 464 */ 465 adel = update_clock_tree_delay(emc, DVFS_UPDATE); 466 } 467 468 if (type == PERIODIC_TRAINING_SEQUENCE) { 469 tegra210_emc_start_periodic_compensation(emc); 470 udelay(delay); 471 472 adel = update_clock_tree_delay(emc, PERIODIC_TRAINING_UPDATE); 473 } 474 475 return adel; 476 } 477 478 static u32 tegra210_emc_r21021_periodic_compensation(struct tegra210_emc *emc) 479 { 480 u32 emc_cfg, emc_cfg_o, emc_cfg_update, del, value; 481 u32 list[] = { 482 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0, 483 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1, 484 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2, 485 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3, 486 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0, 487 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1, 488 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2, 489 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3, 490 EMC_DATA_BRLSHFT_0, 491 EMC_DATA_BRLSHFT_1 492 }; 493 struct tegra210_emc_timing *last = emc->last; 494 unsigned int items = ARRAY_SIZE(list), i; 495 unsigned long delay; 496 497 if (last->periodic_training) { 498 emc_dbg(emc, PER_TRAIN, "Periodic training starting\n"); 499 500 value = emc_readl(emc, EMC_DBG); 501 emc_cfg_o = emc_readl(emc, EMC_CFG); 502 emc_cfg = emc_cfg_o & ~(EMC_CFG_DYN_SELF_REF | 503 EMC_CFG_DRAM_ACPD | 504 EMC_CFG_DRAM_CLKSTOP_PD | 505 EMC_CFG_DRAM_CLKSTOP_PD); 506 507 508 /* 509 * 1. Power optimizations should be off. 510 */ 511 emc_writel(emc, emc_cfg, EMC_CFG); 512 513 /* Does emc_timing_update() for above changes. */ 514 tegra210_emc_dll_disable(emc); 515 516 for (i = 0; i < emc->num_channels; i++) 517 tegra210_emc_wait_for_update(emc, i, EMC_EMC_STATUS, 518 EMC_EMC_STATUS_DRAM_IN_POWERDOWN_MASK, 519 0); 520 521 for (i = 0; i < emc->num_channels; i++) 522 tegra210_emc_wait_for_update(emc, i, EMC_EMC_STATUS, 523 EMC_EMC_STATUS_DRAM_IN_SELF_REFRESH_MASK, 524 0); 525 526 emc_cfg_update = value = emc_readl(emc, EMC_CFG_UPDATE); 527 value &= ~EMC_CFG_UPDATE_UPDATE_DLL_IN_UPDATE_MASK; 528 value |= (2 << EMC_CFG_UPDATE_UPDATE_DLL_IN_UPDATE_SHIFT); 529 emc_writel(emc, value, EMC_CFG_UPDATE); 530 531 /* 532 * 2. osc kick off - this assumes training and dvfs have set 533 * correct MR23. 534 */ 535 tegra210_emc_start_periodic_compensation(emc); 536 537 /* 538 * 3. Let dram capture its clock tree delays. 539 */ 540 delay = tegra210_emc_actual_osc_clocks(last->run_clocks); 541 delay *= 1000; 542 delay /= last->rate + 1; 543 udelay(delay); 544 545 /* 546 * 4. Check delta wrt previous values (save value if margin 547 * exceeds what is set in table). 548 */ 549 del = periodic_compensation_handler(emc, 550 PERIODIC_TRAINING_SEQUENCE, 551 last, last); 552 553 /* 554 * 5. Apply compensation w.r.t. trained values (if clock tree 555 * has drifted more than the set margin). 556 */ 557 if (last->tree_margin < ((del * 128 * (last->rate / 1000)) / 1000000)) { 558 for (i = 0; i < items; i++) { 559 value = tegra210_emc_compensate(last, list[i]); 560 emc_dbg(emc, EMA_WRITES, "0x%08x <= 0x%08x\n", 561 list[i], value); 562 emc_writel(emc, value, list[i]); 563 } 564 } 565 566 emc_writel(emc, emc_cfg_o, EMC_CFG); 567 568 /* 569 * 6. Timing update actally applies the new trimmers. 570 */ 571 tegra210_emc_timing_update(emc); 572 573 /* 6.1. Restore the UPDATE_DLL_IN_UPDATE field. */ 574 emc_writel(emc, emc_cfg_update, EMC_CFG_UPDATE); 575 576 /* 6.2. Restore the DLL. */ 577 tegra210_emc_dll_enable(emc); 578 } 579 580 return 0; 581 } 582 583 /* 584 * Do the clock change sequence. 585 */ 586 static void tegra210_emc_r21021_set_clock(struct tegra210_emc *emc, u32 clksrc) 587 { 588 /* state variables */ 589 static bool fsp_for_next_freq; 590 /* constant configuration parameters */ 591 const bool save_restore_clkstop_pd = true; 592 const u32 zqcal_before_cc_cutoff = 2400; 593 const bool cya_allow_ref_cc = false; 594 const bool cya_issue_pc_ref = false; 595 const bool opt_cc_short_zcal = true; 596 const bool ref_b4_sref_en = false; 597 const u32 tZQCAL_lpddr4 = 1000000; 598 const bool opt_short_zcal = true; 599 const bool opt_do_sw_qrst = true; 600 const u32 opt_dvfs_mode = MAN_SR; 601 /* 602 * This is the timing table for the source frequency. It does _not_ 603 * necessarily correspond to the actual timing values in the EMC at the 604 * moment. If the boot BCT differs from the table then this can happen. 605 * However, we need it for accessing the dram_timings (which are not 606 * really registers) array for the current frequency. 607 */ 608 struct tegra210_emc_timing *fake, *last = emc->last, *next = emc->next; 609 u32 tRTM, RP_war, R2P_war, TRPab_war, deltaTWATM, W2P_war, tRPST; 610 u32 mr13_flip_fspwr, mr13_flip_fspop, ramp_up_wait, ramp_down_wait; 611 u32 zq_wait_long, zq_latch_dvfs_wait_time, tZQCAL_lpddr4_fc_adj; 612 u32 emc_auto_cal_config, auto_cal_en, emc_cfg, emc_sel_dpd_ctrl; 613 u32 tFC_lpddr4 = 1000 * next->dram_timings[T_FC_LPDDR4]; 614 u32 bg_reg_mode_change, enable_bglp_reg, enable_bg_reg; 615 bool opt_zcal_en_cc = false, is_lpddr3 = false; 616 bool compensate_trimmer_applicable = false; 617 u32 emc_dbg, emc_cfg_pipe_clk, emc_pin; 618 u32 src_clk_period, dst_clk_period; /* in picoseconds */ 619 bool shared_zq_resistor = false; 620 u32 value, dram_type; 621 u32 opt_dll_mode = 0; 622 unsigned long delay; 623 unsigned int i; 624 625 emc_dbg(emc, INFO, "Running clock change.\n"); 626 627 /* XXX fake == last */ 628 fake = tegra210_emc_find_timing(emc, last->rate * 1000UL); 629 fsp_for_next_freq = !fsp_for_next_freq; 630 631 value = emc_readl(emc, EMC_FBIO_CFG5) & EMC_FBIO_CFG5_DRAM_TYPE_MASK; 632 dram_type = value >> EMC_FBIO_CFG5_DRAM_TYPE_SHIFT; 633 634 if (last->burst_regs[EMC_ZCAL_WAIT_CNT_INDEX] & BIT(31)) 635 shared_zq_resistor = true; 636 637 if ((next->burst_regs[EMC_ZCAL_INTERVAL_INDEX] != 0 && 638 last->burst_regs[EMC_ZCAL_INTERVAL_INDEX] == 0) || 639 dram_type == DRAM_TYPE_LPDDR4) 640 opt_zcal_en_cc = true; 641 642 if (dram_type == DRAM_TYPE_DDR3) 643 opt_dll_mode = tegra210_emc_get_dll_state(next); 644 645 if ((next->burst_regs[EMC_FBIO_CFG5_INDEX] & BIT(25)) && 646 (dram_type == DRAM_TYPE_LPDDR2)) 647 is_lpddr3 = true; 648 649 emc_readl(emc, EMC_CFG); 650 emc_readl(emc, EMC_AUTO_CAL_CONFIG); 651 652 src_clk_period = 1000000000 / last->rate; 653 dst_clk_period = 1000000000 / next->rate; 654 655 if (dst_clk_period <= zqcal_before_cc_cutoff) 656 tZQCAL_lpddr4_fc_adj = tZQCAL_lpddr4 - tFC_lpddr4; 657 else 658 tZQCAL_lpddr4_fc_adj = tZQCAL_lpddr4; 659 660 tZQCAL_lpddr4_fc_adj /= dst_clk_period; 661 662 emc_dbg = emc_readl(emc, EMC_DBG); 663 emc_pin = emc_readl(emc, EMC_PIN); 664 emc_cfg_pipe_clk = emc_readl(emc, EMC_CFG_PIPE_CLK); 665 666 emc_cfg = next->burst_regs[EMC_CFG_INDEX]; 667 emc_cfg &= ~(EMC_CFG_DYN_SELF_REF | EMC_CFG_DRAM_ACPD | 668 EMC_CFG_DRAM_CLKSTOP_SR | EMC_CFG_DRAM_CLKSTOP_PD); 669 emc_sel_dpd_ctrl = next->emc_sel_dpd_ctrl; 670 emc_sel_dpd_ctrl &= ~(EMC_SEL_DPD_CTRL_CLK_SEL_DPD_EN | 671 EMC_SEL_DPD_CTRL_CA_SEL_DPD_EN | 672 EMC_SEL_DPD_CTRL_RESET_SEL_DPD_EN | 673 EMC_SEL_DPD_CTRL_ODT_SEL_DPD_EN | 674 EMC_SEL_DPD_CTRL_DATA_SEL_DPD_EN); 675 676 emc_dbg(emc, INFO, "Clock change version: %d\n", 677 DVFS_CLOCK_CHANGE_VERSION); 678 emc_dbg(emc, INFO, "DRAM type = %d\n", dram_type); 679 emc_dbg(emc, INFO, "DRAM dev #: %u\n", emc->num_devices); 680 emc_dbg(emc, INFO, "Next EMC clksrc: 0x%08x\n", clksrc); 681 emc_dbg(emc, INFO, "DLL clksrc: 0x%08x\n", next->dll_clk_src); 682 emc_dbg(emc, INFO, "last rate: %u, next rate %u\n", last->rate, 683 next->rate); 684 emc_dbg(emc, INFO, "last period: %u, next period: %u\n", 685 src_clk_period, dst_clk_period); 686 emc_dbg(emc, INFO, " shared_zq_resistor: %d\n", !!shared_zq_resistor); 687 emc_dbg(emc, INFO, " num_channels: %u\n", emc->num_channels); 688 emc_dbg(emc, INFO, " opt_dll_mode: %d\n", opt_dll_mode); 689 690 /* 691 * Step 1: 692 * Pre DVFS SW sequence. 693 */ 694 emc_dbg(emc, STEPS, "Step 1\n"); 695 emc_dbg(emc, STEPS, "Step 1.1: Disable DLL temporarily.\n"); 696 697 value = emc_readl(emc, EMC_CFG_DIG_DLL); 698 value &= ~EMC_CFG_DIG_DLL_CFG_DLL_EN; 699 emc_writel(emc, value, EMC_CFG_DIG_DLL); 700 701 tegra210_emc_timing_update(emc); 702 703 for (i = 0; i < emc->num_channels; i++) 704 tegra210_emc_wait_for_update(emc, i, EMC_CFG_DIG_DLL, 705 EMC_CFG_DIG_DLL_CFG_DLL_EN, 0); 706 707 emc_dbg(emc, STEPS, "Step 1.2: Disable AUTOCAL temporarily.\n"); 708 709 emc_auto_cal_config = next->emc_auto_cal_config; 710 auto_cal_en = emc_auto_cal_config & EMC_AUTO_CAL_CONFIG_AUTO_CAL_ENABLE; 711 emc_auto_cal_config &= ~EMC_AUTO_CAL_CONFIG_AUTO_CAL_START; 712 emc_auto_cal_config |= EMC_AUTO_CAL_CONFIG_AUTO_CAL_MEASURE_STALL; 713 emc_auto_cal_config |= EMC_AUTO_CAL_CONFIG_AUTO_CAL_UPDATE_STALL; 714 emc_auto_cal_config |= auto_cal_en; 715 emc_writel(emc, emc_auto_cal_config, EMC_AUTO_CAL_CONFIG); 716 emc_readl(emc, EMC_AUTO_CAL_CONFIG); /* Flush write. */ 717 718 emc_dbg(emc, STEPS, "Step 1.3: Disable other power features.\n"); 719 720 tegra210_emc_set_shadow_bypass(emc, ACTIVE); 721 emc_writel(emc, emc_cfg, EMC_CFG); 722 emc_writel(emc, emc_sel_dpd_ctrl, EMC_SEL_DPD_CTRL); 723 tegra210_emc_set_shadow_bypass(emc, ASSEMBLY); 724 725 if (next->periodic_training) { 726 tegra210_emc_reset_dram_clktree_values(next); 727 728 for (i = 0; i < emc->num_channels; i++) 729 tegra210_emc_wait_for_update(emc, i, EMC_EMC_STATUS, 730 EMC_EMC_STATUS_DRAM_IN_POWERDOWN_MASK, 731 0); 732 733 for (i = 0; i < emc->num_channels; i++) 734 tegra210_emc_wait_for_update(emc, i, EMC_EMC_STATUS, 735 EMC_EMC_STATUS_DRAM_IN_SELF_REFRESH_MASK, 736 0); 737 738 tegra210_emc_start_periodic_compensation(emc); 739 740 delay = 1000 * tegra210_emc_actual_osc_clocks(last->run_clocks); 741 udelay((delay / last->rate) + 2); 742 743 value = periodic_compensation_handler(emc, DVFS_SEQUENCE, fake, 744 next); 745 value = (value * 128 * next->rate / 1000) / 1000000; 746 747 if (next->periodic_training && value > next->tree_margin) 748 compensate_trimmer_applicable = true; 749 } 750 751 emc_writel(emc, EMC_INTSTATUS_CLKCHANGE_COMPLETE, EMC_INTSTATUS); 752 tegra210_emc_set_shadow_bypass(emc, ACTIVE); 753 emc_writel(emc, emc_cfg, EMC_CFG); 754 emc_writel(emc, emc_sel_dpd_ctrl, EMC_SEL_DPD_CTRL); 755 emc_writel(emc, emc_cfg_pipe_clk | EMC_CFG_PIPE_CLK_CLK_ALWAYS_ON, 756 EMC_CFG_PIPE_CLK); 757 emc_writel(emc, next->emc_fdpd_ctrl_cmd_no_ramp & 758 ~EMC_FDPD_CTRL_CMD_NO_RAMP_CMD_DPD_NO_RAMP_ENABLE, 759 EMC_FDPD_CTRL_CMD_NO_RAMP); 760 761 bg_reg_mode_change = 762 ((next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & 763 EMC_PMACRO_BG_BIAS_CTRL_0_BGLP_E_PWRD) ^ 764 (last->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & 765 EMC_PMACRO_BG_BIAS_CTRL_0_BGLP_E_PWRD)) || 766 ((next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & 767 EMC_PMACRO_BG_BIAS_CTRL_0_BG_E_PWRD) ^ 768 (last->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & 769 EMC_PMACRO_BG_BIAS_CTRL_0_BG_E_PWRD)); 770 enable_bglp_reg = 771 (next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & 772 EMC_PMACRO_BG_BIAS_CTRL_0_BGLP_E_PWRD) == 0; 773 enable_bg_reg = 774 (next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & 775 EMC_PMACRO_BG_BIAS_CTRL_0_BG_E_PWRD) == 0; 776 777 if (bg_reg_mode_change) { 778 if (enable_bg_reg) 779 emc_writel(emc, last->burst_regs 780 [EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & 781 ~EMC_PMACRO_BG_BIAS_CTRL_0_BG_E_PWRD, 782 EMC_PMACRO_BG_BIAS_CTRL_0); 783 784 if (enable_bglp_reg) 785 emc_writel(emc, last->burst_regs 786 [EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & 787 ~EMC_PMACRO_BG_BIAS_CTRL_0_BGLP_E_PWRD, 788 EMC_PMACRO_BG_BIAS_CTRL_0); 789 } 790 791 /* Check if we need to turn on VREF generator. */ 792 if ((((last->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX] & 793 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_E_IVREF) == 0) && 794 ((next->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX] & 795 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_E_IVREF) == 1)) || 796 (((last->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX] & 797 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQS_E_IVREF) == 0) && 798 ((next->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX] & 799 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQS_E_IVREF) != 0))) { 800 u32 pad_tx_ctrl = 801 next->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX]; 802 u32 last_pad_tx_ctrl = 803 last->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX]; 804 u32 next_dq_e_ivref, next_dqs_e_ivref; 805 806 next_dqs_e_ivref = pad_tx_ctrl & 807 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQS_E_IVREF; 808 next_dq_e_ivref = pad_tx_ctrl & 809 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_E_IVREF; 810 value = (last_pad_tx_ctrl & 811 ~EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_E_IVREF & 812 ~EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQS_E_IVREF) | 813 next_dq_e_ivref | next_dqs_e_ivref; 814 emc_writel(emc, value, EMC_PMACRO_DATA_PAD_TX_CTRL); 815 udelay(1); 816 } else if (bg_reg_mode_change) { 817 udelay(1); 818 } 819 820 tegra210_emc_set_shadow_bypass(emc, ASSEMBLY); 821 822 /* 823 * Step 2: 824 * Prelock the DLL. 825 */ 826 emc_dbg(emc, STEPS, "Step 2\n"); 827 828 if (next->burst_regs[EMC_CFG_DIG_DLL_INDEX] & 829 EMC_CFG_DIG_DLL_CFG_DLL_EN) { 830 emc_dbg(emc, INFO, "Prelock enabled for target frequency.\n"); 831 value = tegra210_emc_dll_prelock(emc, clksrc); 832 emc_dbg(emc, INFO, "DLL out: 0x%03x\n", value); 833 } else { 834 emc_dbg(emc, INFO, "Disabling DLL for target frequency.\n"); 835 tegra210_emc_dll_disable(emc); 836 } 837 838 /* 839 * Step 3: 840 * Prepare autocal for the clock change. 841 */ 842 emc_dbg(emc, STEPS, "Step 3\n"); 843 844 tegra210_emc_set_shadow_bypass(emc, ACTIVE); 845 emc_writel(emc, next->emc_auto_cal_config2, EMC_AUTO_CAL_CONFIG2); 846 emc_writel(emc, next->emc_auto_cal_config3, EMC_AUTO_CAL_CONFIG3); 847 emc_writel(emc, next->emc_auto_cal_config4, EMC_AUTO_CAL_CONFIG4); 848 emc_writel(emc, next->emc_auto_cal_config5, EMC_AUTO_CAL_CONFIG5); 849 emc_writel(emc, next->emc_auto_cal_config6, EMC_AUTO_CAL_CONFIG6); 850 emc_writel(emc, next->emc_auto_cal_config7, EMC_AUTO_CAL_CONFIG7); 851 emc_writel(emc, next->emc_auto_cal_config8, EMC_AUTO_CAL_CONFIG8); 852 tegra210_emc_set_shadow_bypass(emc, ASSEMBLY); 853 854 emc_auto_cal_config |= (EMC_AUTO_CAL_CONFIG_AUTO_CAL_COMPUTE_START | 855 auto_cal_en); 856 emc_writel(emc, emc_auto_cal_config, EMC_AUTO_CAL_CONFIG); 857 858 /* 859 * Step 4: 860 * Update EMC_CFG. (??) 861 */ 862 emc_dbg(emc, STEPS, "Step 4\n"); 863 864 if (src_clk_period > 50000 && dram_type == DRAM_TYPE_LPDDR4) 865 ccfifo_writel(emc, 1, EMC_SELF_REF, 0); 866 else 867 emc_writel(emc, next->emc_cfg_2, EMC_CFG_2); 868 869 /* 870 * Step 5: 871 * Prepare reference variables for ZQCAL regs. 872 */ 873 emc_dbg(emc, STEPS, "Step 5\n"); 874 875 if (dram_type == DRAM_TYPE_LPDDR4) 876 zq_wait_long = max((u32)1, div_o3(1000000, dst_clk_period)); 877 else if (dram_type == DRAM_TYPE_LPDDR2 || is_lpddr3) 878 zq_wait_long = max(next->min_mrs_wait, 879 div_o3(360000, dst_clk_period)) + 4; 880 else if (dram_type == DRAM_TYPE_DDR3) 881 zq_wait_long = max((u32)256, 882 div_o3(320000, dst_clk_period) + 2); 883 else 884 zq_wait_long = 0; 885 886 /* 887 * Step 6: 888 * Training code - removed. 889 */ 890 emc_dbg(emc, STEPS, "Step 6\n"); 891 892 /* 893 * Step 7: 894 * Program FSP reference registers and send MRWs to new FSPWR. 895 */ 896 emc_dbg(emc, STEPS, "Step 7\n"); 897 emc_dbg(emc, SUB_STEPS, "Step 7.1: Bug 200024907 - Patch RP R2P"); 898 899 /* WAR 200024907 */ 900 if (dram_type == DRAM_TYPE_LPDDR4) { 901 u32 nRTP = 16; 902 903 if (src_clk_period >= 1000000 / 1866) /* 535.91 ps */ 904 nRTP = 14; 905 906 if (src_clk_period >= 1000000 / 1600) /* 625.00 ps */ 907 nRTP = 12; 908 909 if (src_clk_period >= 1000000 / 1333) /* 750.19 ps */ 910 nRTP = 10; 911 912 if (src_clk_period >= 1000000 / 1066) /* 938.09 ps */ 913 nRTP = 8; 914 915 deltaTWATM = max_t(u32, div_o3(7500, src_clk_period), 8); 916 917 /* 918 * Originally there was a + .5 in the tRPST calculation. 919 * However since we can't do FP in the kernel and the tRTM 920 * computation was in a floating point ceiling function, adding 921 * one to tRTP should be ok. There is no other source of non 922 * integer values, so the result was always going to be 923 * something for the form: f_ceil(N + .5) = N + 1; 924 */ 925 tRPST = (last->emc_mrw & 0x80) >> 7; 926 tRTM = fake->dram_timings[RL] + div_o3(3600, src_clk_period) + 927 max_t(u32, div_o3(7500, src_clk_period), 8) + tRPST + 928 1 + nRTP; 929 930 emc_dbg(emc, INFO, "tRTM = %u, EMC_RP = %u\n", tRTM, 931 next->burst_regs[EMC_RP_INDEX]); 932 933 if (last->burst_regs[EMC_RP_INDEX] < tRTM) { 934 if (tRTM > (last->burst_regs[EMC_R2P_INDEX] + 935 last->burst_regs[EMC_RP_INDEX])) { 936 R2P_war = tRTM - last->burst_regs[EMC_RP_INDEX]; 937 RP_war = last->burst_regs[EMC_RP_INDEX]; 938 TRPab_war = last->burst_regs[EMC_TRPAB_INDEX]; 939 940 if (R2P_war > 63) { 941 RP_war = R2P_war + 942 last->burst_regs[EMC_RP_INDEX] - 63; 943 944 if (TRPab_war < RP_war) 945 TRPab_war = RP_war; 946 947 R2P_war = 63; 948 } 949 } else { 950 R2P_war = last->burst_regs[EMC_R2P_INDEX]; 951 RP_war = last->burst_regs[EMC_RP_INDEX]; 952 TRPab_war = last->burst_regs[EMC_TRPAB_INDEX]; 953 } 954 955 if (RP_war < deltaTWATM) { 956 W2P_war = last->burst_regs[EMC_W2P_INDEX] 957 + deltaTWATM - RP_war; 958 if (W2P_war > 63) { 959 RP_war = RP_war + W2P_war - 63; 960 if (TRPab_war < RP_war) 961 TRPab_war = RP_war; 962 W2P_war = 63; 963 } 964 } else { 965 W2P_war = last->burst_regs[ 966 EMC_W2P_INDEX]; 967 } 968 969 if ((last->burst_regs[EMC_W2P_INDEX] ^ W2P_war) || 970 (last->burst_regs[EMC_R2P_INDEX] ^ R2P_war) || 971 (last->burst_regs[EMC_RP_INDEX] ^ RP_war) || 972 (last->burst_regs[EMC_TRPAB_INDEX] ^ TRPab_war)) { 973 emc_writel(emc, RP_war, EMC_RP); 974 emc_writel(emc, R2P_war, EMC_R2P); 975 emc_writel(emc, W2P_war, EMC_W2P); 976 emc_writel(emc, TRPab_war, EMC_TRPAB); 977 } 978 979 tegra210_emc_timing_update(emc); 980 } else { 981 emc_dbg(emc, INFO, "Skipped WAR\n"); 982 } 983 } 984 985 if (!fsp_for_next_freq) { 986 mr13_flip_fspwr = (next->emc_mrw3 & 0xffffff3f) | 0x80; 987 mr13_flip_fspop = (next->emc_mrw3 & 0xffffff3f) | 0x00; 988 } else { 989 mr13_flip_fspwr = (next->emc_mrw3 & 0xffffff3f) | 0x40; 990 mr13_flip_fspop = (next->emc_mrw3 & 0xffffff3f) | 0xc0; 991 } 992 993 if (dram_type == DRAM_TYPE_LPDDR4) { 994 emc_writel(emc, mr13_flip_fspwr, EMC_MRW3); 995 emc_writel(emc, next->emc_mrw, EMC_MRW); 996 emc_writel(emc, next->emc_mrw2, EMC_MRW2); 997 } 998 999 /* 1000 * Step 8: 1001 * Program the shadow registers. 1002 */ 1003 emc_dbg(emc, STEPS, "Step 8\n"); 1004 emc_dbg(emc, SUB_STEPS, "Writing burst_regs\n"); 1005 1006 for (i = 0; i < next->num_burst; i++) { 1007 const u16 *offsets = emc->offsets->burst; 1008 u16 offset; 1009 1010 if (!offsets[i]) 1011 continue; 1012 1013 value = next->burst_regs[i]; 1014 offset = offsets[i]; 1015 1016 if (dram_type != DRAM_TYPE_LPDDR4 && 1017 (offset == EMC_MRW6 || offset == EMC_MRW7 || 1018 offset == EMC_MRW8 || offset == EMC_MRW9 || 1019 offset == EMC_MRW10 || offset == EMC_MRW11 || 1020 offset == EMC_MRW12 || offset == EMC_MRW13 || 1021 offset == EMC_MRW14 || offset == EMC_MRW15 || 1022 offset == EMC_TRAINING_CTRL)) 1023 continue; 1024 1025 /* Pain... And suffering. */ 1026 if (offset == EMC_CFG) { 1027 value &= ~EMC_CFG_DRAM_ACPD; 1028 value &= ~EMC_CFG_DYN_SELF_REF; 1029 1030 if (dram_type == DRAM_TYPE_LPDDR4) { 1031 value &= ~EMC_CFG_DRAM_CLKSTOP_SR; 1032 value &= ~EMC_CFG_DRAM_CLKSTOP_PD; 1033 } 1034 } else if (offset == EMC_MRS_WAIT_CNT && 1035 dram_type == DRAM_TYPE_LPDDR2 && 1036 opt_zcal_en_cc && !opt_cc_short_zcal && 1037 opt_short_zcal) { 1038 value = (value & ~(EMC_MRS_WAIT_CNT_SHORT_WAIT_MASK << 1039 EMC_MRS_WAIT_CNT_SHORT_WAIT_SHIFT)) | 1040 ((zq_wait_long & EMC_MRS_WAIT_CNT_SHORT_WAIT_MASK) << 1041 EMC_MRS_WAIT_CNT_SHORT_WAIT_SHIFT); 1042 } else if (offset == EMC_ZCAL_WAIT_CNT && 1043 dram_type == DRAM_TYPE_DDR3 && opt_zcal_en_cc && 1044 !opt_cc_short_zcal && opt_short_zcal) { 1045 value = (value & ~(EMC_ZCAL_WAIT_CNT_ZCAL_WAIT_CNT_MASK << 1046 EMC_ZCAL_WAIT_CNT_ZCAL_WAIT_CNT_SHIFT)) | 1047 ((zq_wait_long & EMC_ZCAL_WAIT_CNT_ZCAL_WAIT_CNT_MASK) << 1048 EMC_MRS_WAIT_CNT_SHORT_WAIT_SHIFT); 1049 } else if (offset == EMC_ZCAL_INTERVAL && opt_zcal_en_cc) { 1050 value = 0; /* EMC_ZCAL_INTERVAL reset value. */ 1051 } else if (offset == EMC_PMACRO_AUTOCAL_CFG_COMMON) { 1052 value |= EMC_PMACRO_AUTOCAL_CFG_COMMON_E_CAL_BYPASS_DVFS; 1053 } else if (offset == EMC_PMACRO_DATA_PAD_TX_CTRL) { 1054 value &= ~(EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSP_TX_E_DCC | 1055 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSN_TX_E_DCC | 1056 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_TX_E_DCC | 1057 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_CMD_TX_E_DCC); 1058 } else if (offset == EMC_PMACRO_CMD_PAD_TX_CTRL) { 1059 value |= EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_DRVFORCEON; 1060 value &= ~(EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSP_TX_E_DCC | 1061 EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSN_TX_E_DCC | 1062 EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_E_DCC | 1063 EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_CMD_TX_E_DCC); 1064 } else if (offset == EMC_PMACRO_BRICK_CTRL_RFU1) { 1065 value &= 0xf800f800; 1066 } else if (offset == EMC_PMACRO_COMMON_PAD_TX_CTRL) { 1067 value &= 0xfffffff0; 1068 } 1069 1070 emc_writel(emc, value, offset); 1071 } 1072 1073 /* SW addition: do EMC refresh adjustment here. */ 1074 tegra210_emc_adjust_timing(emc, next); 1075 1076 if (dram_type == DRAM_TYPE_LPDDR4) { 1077 value = (23 << EMC_MRW_MRW_MA_SHIFT) | 1078 (next->run_clocks & EMC_MRW_MRW_OP_MASK); 1079 emc_writel(emc, value, EMC_MRW); 1080 } 1081 1082 /* Per channel burst registers. */ 1083 emc_dbg(emc, SUB_STEPS, "Writing burst_regs_per_ch\n"); 1084 1085 for (i = 0; i < next->num_burst_per_ch; i++) { 1086 const struct tegra210_emc_per_channel_regs *burst = 1087 emc->offsets->burst_per_channel; 1088 1089 if (!burst[i].offset) 1090 continue; 1091 1092 if (dram_type != DRAM_TYPE_LPDDR4 && 1093 (burst[i].offset == EMC_MRW6 || 1094 burst[i].offset == EMC_MRW7 || 1095 burst[i].offset == EMC_MRW8 || 1096 burst[i].offset == EMC_MRW9 || 1097 burst[i].offset == EMC_MRW10 || 1098 burst[i].offset == EMC_MRW11 || 1099 burst[i].offset == EMC_MRW12 || 1100 burst[i].offset == EMC_MRW13 || 1101 burst[i].offset == EMC_MRW14 || 1102 burst[i].offset == EMC_MRW15)) 1103 continue; 1104 1105 /* Filter out second channel if not in DUAL_CHANNEL mode. */ 1106 if (emc->num_channels < 2 && burst[i].bank >= 1) 1107 continue; 1108 1109 emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i, 1110 next->burst_reg_per_ch[i], burst[i].offset); 1111 emc_channel_writel(emc, burst[i].bank, 1112 next->burst_reg_per_ch[i], 1113 burst[i].offset); 1114 } 1115 1116 /* Vref regs. */ 1117 emc_dbg(emc, SUB_STEPS, "Writing vref_regs\n"); 1118 1119 for (i = 0; i < next->vref_num; i++) { 1120 const struct tegra210_emc_per_channel_regs *vref = 1121 emc->offsets->vref_per_channel; 1122 1123 if (!vref[i].offset) 1124 continue; 1125 1126 if (emc->num_channels < 2 && vref[i].bank >= 1) 1127 continue; 1128 1129 emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i, 1130 next->vref_perch_regs[i], vref[i].offset); 1131 emc_channel_writel(emc, vref[i].bank, next->vref_perch_regs[i], 1132 vref[i].offset); 1133 } 1134 1135 /* Trimmers. */ 1136 emc_dbg(emc, SUB_STEPS, "Writing trim_regs\n"); 1137 1138 for (i = 0; i < next->num_trim; i++) { 1139 const u16 *offsets = emc->offsets->trim; 1140 1141 if (!offsets[i]) 1142 continue; 1143 1144 if (compensate_trimmer_applicable && 1145 (offsets[i] == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0 || 1146 offsets[i] == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1 || 1147 offsets[i] == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2 || 1148 offsets[i] == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3 || 1149 offsets[i] == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0 || 1150 offsets[i] == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1 || 1151 offsets[i] == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2 || 1152 offsets[i] == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3 || 1153 offsets[i] == EMC_DATA_BRLSHFT_0 || 1154 offsets[i] == EMC_DATA_BRLSHFT_1)) { 1155 value = tegra210_emc_compensate(next, offsets[i]); 1156 emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i, 1157 value, offsets[i]); 1158 emc_dbg(emc, EMA_WRITES, "0x%08x <= 0x%08x\n", 1159 (u32)(u64)offsets[i], value); 1160 emc_writel(emc, value, offsets[i]); 1161 } else { 1162 emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i, 1163 next->trim_regs[i], offsets[i]); 1164 emc_writel(emc, next->trim_regs[i], offsets[i]); 1165 } 1166 } 1167 1168 /* Per channel trimmers. */ 1169 emc_dbg(emc, SUB_STEPS, "Writing trim_regs_per_ch\n"); 1170 1171 for (i = 0; i < next->num_trim_per_ch; i++) { 1172 const struct tegra210_emc_per_channel_regs *trim = 1173 &emc->offsets->trim_per_channel[0]; 1174 unsigned int offset; 1175 1176 if (!trim[i].offset) 1177 continue; 1178 1179 if (emc->num_channels < 2 && trim[i].bank >= 1) 1180 continue; 1181 1182 offset = trim[i].offset; 1183 1184 if (compensate_trimmer_applicable && 1185 (offset == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0 || 1186 offset == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1 || 1187 offset == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2 || 1188 offset == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3 || 1189 offset == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0 || 1190 offset == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1 || 1191 offset == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2 || 1192 offset == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3 || 1193 offset == EMC_DATA_BRLSHFT_0 || 1194 offset == EMC_DATA_BRLSHFT_1)) { 1195 value = tegra210_emc_compensate(next, offset); 1196 emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i, 1197 value, offset); 1198 emc_dbg(emc, EMA_WRITES, "0x%08x <= 0x%08x\n", offset, 1199 value); 1200 emc_channel_writel(emc, trim[i].bank, value, offset); 1201 } else { 1202 emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i, 1203 next->trim_perch_regs[i], offset); 1204 emc_channel_writel(emc, trim[i].bank, 1205 next->trim_perch_regs[i], offset); 1206 } 1207 } 1208 1209 emc_dbg(emc, SUB_STEPS, "Writing burst_mc_regs\n"); 1210 1211 for (i = 0; i < next->num_mc_regs; i++) { 1212 const u16 *offsets = emc->offsets->burst_mc; 1213 u32 *values = next->burst_mc_regs; 1214 1215 emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i, 1216 values[i], offsets[i]); 1217 mc_writel(emc->mc, values[i], offsets[i]); 1218 } 1219 1220 /* Registers to be programmed on the faster clock. */ 1221 if (next->rate < last->rate) { 1222 const u16 *la = emc->offsets->la_scale; 1223 1224 emc_dbg(emc, SUB_STEPS, "Writing la_scale_regs\n"); 1225 1226 for (i = 0; i < next->num_up_down; i++) { 1227 emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i, 1228 next->la_scale_regs[i], la[i]); 1229 mc_writel(emc->mc, next->la_scale_regs[i], la[i]); 1230 } 1231 } 1232 1233 /* Flush all the burst register writes. */ 1234 mc_readl(emc->mc, MC_EMEM_ADR_CFG); 1235 1236 /* 1237 * Step 9: 1238 * LPDDR4 section A. 1239 */ 1240 emc_dbg(emc, STEPS, "Step 9\n"); 1241 1242 value = next->burst_regs[EMC_ZCAL_WAIT_CNT_INDEX]; 1243 value &= ~EMC_ZCAL_WAIT_CNT_ZCAL_WAIT_CNT_MASK; 1244 1245 if (dram_type == DRAM_TYPE_LPDDR4) { 1246 emc_writel(emc, 0, EMC_ZCAL_INTERVAL); 1247 emc_writel(emc, value, EMC_ZCAL_WAIT_CNT); 1248 1249 value = emc_dbg | (EMC_DBG_WRITE_MUX_ACTIVE | 1250 EMC_DBG_WRITE_ACTIVE_ONLY); 1251 1252 emc_writel(emc, value, EMC_DBG); 1253 emc_writel(emc, 0, EMC_ZCAL_INTERVAL); 1254 emc_writel(emc, emc_dbg, EMC_DBG); 1255 } 1256 1257 /* 1258 * Step 10: 1259 * LPDDR4 and DDR3 common section. 1260 */ 1261 emc_dbg(emc, STEPS, "Step 10\n"); 1262 1263 if (opt_dvfs_mode == MAN_SR || dram_type == DRAM_TYPE_LPDDR4) { 1264 if (dram_type == DRAM_TYPE_LPDDR4) 1265 ccfifo_writel(emc, 0x101, EMC_SELF_REF, 0); 1266 else 1267 ccfifo_writel(emc, 0x1, EMC_SELF_REF, 0); 1268 1269 if (dram_type == DRAM_TYPE_LPDDR4 && 1270 dst_clk_period <= zqcal_before_cc_cutoff) { 1271 ccfifo_writel(emc, mr13_flip_fspwr ^ 0x40, EMC_MRW3, 0); 1272 ccfifo_writel(emc, (next->burst_regs[EMC_MRW6_INDEX] & 1273 0xFFFF3F3F) | 1274 (last->burst_regs[EMC_MRW6_INDEX] & 1275 0x0000C0C0), EMC_MRW6, 0); 1276 ccfifo_writel(emc, (next->burst_regs[EMC_MRW14_INDEX] & 1277 0xFFFF0707) | 1278 (last->burst_regs[EMC_MRW14_INDEX] & 1279 0x00003838), EMC_MRW14, 0); 1280 1281 if (emc->num_devices > 1) { 1282 ccfifo_writel(emc, 1283 (next->burst_regs[EMC_MRW7_INDEX] & 1284 0xFFFF3F3F) | 1285 (last->burst_regs[EMC_MRW7_INDEX] & 1286 0x0000C0C0), EMC_MRW7, 0); 1287 ccfifo_writel(emc, 1288 (next->burst_regs[EMC_MRW15_INDEX] & 1289 0xFFFF0707) | 1290 (last->burst_regs[EMC_MRW15_INDEX] & 1291 0x00003838), EMC_MRW15, 0); 1292 } 1293 1294 if (opt_zcal_en_cc) { 1295 if (emc->num_devices < 2) 1296 ccfifo_writel(emc, 1297 2UL << EMC_ZQ_CAL_DEV_SEL_SHIFT 1298 | EMC_ZQ_CAL_ZQ_CAL_CMD, 1299 EMC_ZQ_CAL, 0); 1300 else if (shared_zq_resistor) 1301 ccfifo_writel(emc, 1302 2UL << EMC_ZQ_CAL_DEV_SEL_SHIFT 1303 | EMC_ZQ_CAL_ZQ_CAL_CMD, 1304 EMC_ZQ_CAL, 0); 1305 else 1306 ccfifo_writel(emc, 1307 EMC_ZQ_CAL_ZQ_CAL_CMD, 1308 EMC_ZQ_CAL, 0); 1309 } 1310 } 1311 } 1312 1313 if (dram_type == DRAM_TYPE_LPDDR4) { 1314 value = (1000 * fake->dram_timings[T_RP]) / src_clk_period; 1315 ccfifo_writel(emc, mr13_flip_fspop | 0x8, EMC_MRW3, value); 1316 ccfifo_writel(emc, 0, 0, tFC_lpddr4 / src_clk_period); 1317 } 1318 1319 if (dram_type == DRAM_TYPE_LPDDR4 || opt_dvfs_mode != MAN_SR) { 1320 delay = 30; 1321 1322 if (cya_allow_ref_cc) { 1323 delay += (1000 * fake->dram_timings[T_RP]) / 1324 src_clk_period; 1325 delay += 4000 * fake->dram_timings[T_RFC]; 1326 } 1327 1328 ccfifo_writel(emc, emc_pin & ~(EMC_PIN_PIN_CKE_PER_DEV | 1329 EMC_PIN_PIN_CKEB | 1330 EMC_PIN_PIN_CKE), 1331 EMC_PIN, delay); 1332 } 1333 1334 /* calculate reference delay multiplier */ 1335 value = 1; 1336 1337 if (ref_b4_sref_en) 1338 value++; 1339 1340 if (cya_allow_ref_cc) 1341 value++; 1342 1343 if (cya_issue_pc_ref) 1344 value++; 1345 1346 if (dram_type != DRAM_TYPE_LPDDR4) { 1347 delay = ((1000 * fake->dram_timings[T_RP] / src_clk_period) + 1348 (1000 * fake->dram_timings[T_RFC] / src_clk_period)); 1349 delay = value * delay + 20; 1350 } else { 1351 delay = 0; 1352 } 1353 1354 /* 1355 * Step 11: 1356 * Ramp down. 1357 */ 1358 emc_dbg(emc, STEPS, "Step 11\n"); 1359 1360 ccfifo_writel(emc, 0x0, EMC_CFG_SYNC, delay); 1361 1362 value = emc_dbg | EMC_DBG_WRITE_MUX_ACTIVE | EMC_DBG_WRITE_ACTIVE_ONLY; 1363 ccfifo_writel(emc, value, EMC_DBG, 0); 1364 1365 ramp_down_wait = tegra210_emc_dvfs_power_ramp_down(emc, src_clk_period, 1366 0); 1367 1368 /* 1369 * Step 12: 1370 * And finally - trigger the clock change. 1371 */ 1372 emc_dbg(emc, STEPS, "Step 12\n"); 1373 1374 ccfifo_writel(emc, 1, EMC_STALL_THEN_EXE_AFTER_CLKCHANGE, 0); 1375 value &= ~EMC_DBG_WRITE_ACTIVE_ONLY; 1376 ccfifo_writel(emc, value, EMC_DBG, 0); 1377 1378 /* 1379 * Step 13: 1380 * Ramp up. 1381 */ 1382 emc_dbg(emc, STEPS, "Step 13\n"); 1383 1384 ramp_up_wait = tegra210_emc_dvfs_power_ramp_up(emc, dst_clk_period, 0); 1385 ccfifo_writel(emc, emc_dbg, EMC_DBG, 0); 1386 1387 /* 1388 * Step 14: 1389 * Bringup CKE pins. 1390 */ 1391 emc_dbg(emc, STEPS, "Step 14\n"); 1392 1393 if (dram_type == DRAM_TYPE_LPDDR4) { 1394 value = emc_pin | EMC_PIN_PIN_CKE; 1395 1396 if (emc->num_devices <= 1) 1397 value &= ~(EMC_PIN_PIN_CKEB | EMC_PIN_PIN_CKE_PER_DEV); 1398 else 1399 value |= EMC_PIN_PIN_CKEB | EMC_PIN_PIN_CKE_PER_DEV; 1400 1401 ccfifo_writel(emc, value, EMC_PIN, 0); 1402 } 1403 1404 /* 1405 * Step 15: (two step 15s ??) 1406 * Calculate zqlatch wait time; has dependency on ramping times. 1407 */ 1408 emc_dbg(emc, STEPS, "Step 15\n"); 1409 1410 if (dst_clk_period <= zqcal_before_cc_cutoff) { 1411 s32 t = (s32)(ramp_up_wait + ramp_down_wait) / 1412 (s32)dst_clk_period; 1413 zq_latch_dvfs_wait_time = (s32)tZQCAL_lpddr4_fc_adj - t; 1414 } else { 1415 zq_latch_dvfs_wait_time = tZQCAL_lpddr4_fc_adj - 1416 div_o3(1000 * next->dram_timings[T_PDEX], 1417 dst_clk_period); 1418 } 1419 1420 emc_dbg(emc, INFO, "tZQCAL_lpddr4_fc_adj = %u\n", tZQCAL_lpddr4_fc_adj); 1421 emc_dbg(emc, INFO, "dst_clk_period = %u\n", 1422 dst_clk_period); 1423 emc_dbg(emc, INFO, "next->dram_timings[T_PDEX] = %u\n", 1424 next->dram_timings[T_PDEX]); 1425 emc_dbg(emc, INFO, "zq_latch_dvfs_wait_time = %d\n", 1426 max_t(s32, 0, zq_latch_dvfs_wait_time)); 1427 1428 if (dram_type == DRAM_TYPE_LPDDR4 && opt_zcal_en_cc) { 1429 delay = div_o3(1000 * next->dram_timings[T_PDEX], 1430 dst_clk_period); 1431 1432 if (emc->num_devices < 2) { 1433 if (dst_clk_period > zqcal_before_cc_cutoff) 1434 ccfifo_writel(emc, 1435 2UL << EMC_ZQ_CAL_DEV_SEL_SHIFT | 1436 EMC_ZQ_CAL_ZQ_CAL_CMD, EMC_ZQ_CAL, 1437 delay); 1438 1439 value = (mr13_flip_fspop & 0xfffffff7) | 0x0c000000; 1440 ccfifo_writel(emc, value, EMC_MRW3, delay); 1441 ccfifo_writel(emc, 0, EMC_SELF_REF, 0); 1442 ccfifo_writel(emc, 0, EMC_REF, 0); 1443 ccfifo_writel(emc, 2UL << EMC_ZQ_CAL_DEV_SEL_SHIFT | 1444 EMC_ZQ_CAL_ZQ_LATCH_CMD, 1445 EMC_ZQ_CAL, 1446 max_t(s32, 0, zq_latch_dvfs_wait_time)); 1447 } else if (shared_zq_resistor) { 1448 if (dst_clk_period > zqcal_before_cc_cutoff) 1449 ccfifo_writel(emc, 1450 2UL << EMC_ZQ_CAL_DEV_SEL_SHIFT | 1451 EMC_ZQ_CAL_ZQ_CAL_CMD, EMC_ZQ_CAL, 1452 delay); 1453 1454 ccfifo_writel(emc, 2UL << EMC_ZQ_CAL_DEV_SEL_SHIFT | 1455 EMC_ZQ_CAL_ZQ_LATCH_CMD, EMC_ZQ_CAL, 1456 max_t(s32, 0, zq_latch_dvfs_wait_time) + 1457 delay); 1458 ccfifo_writel(emc, 1UL << EMC_ZQ_CAL_DEV_SEL_SHIFT | 1459 EMC_ZQ_CAL_ZQ_LATCH_CMD, 1460 EMC_ZQ_CAL, 0); 1461 1462 value = (mr13_flip_fspop & 0xfffffff7) | 0x0c000000; 1463 ccfifo_writel(emc, value, EMC_MRW3, 0); 1464 ccfifo_writel(emc, 0, EMC_SELF_REF, 0); 1465 ccfifo_writel(emc, 0, EMC_REF, 0); 1466 1467 ccfifo_writel(emc, 1UL << EMC_ZQ_CAL_DEV_SEL_SHIFT | 1468 EMC_ZQ_CAL_ZQ_LATCH_CMD, EMC_ZQ_CAL, 1469 tZQCAL_lpddr4 / dst_clk_period); 1470 } else { 1471 if (dst_clk_period > zqcal_before_cc_cutoff) 1472 ccfifo_writel(emc, EMC_ZQ_CAL_ZQ_CAL_CMD, 1473 EMC_ZQ_CAL, delay); 1474 1475 value = (mr13_flip_fspop & 0xfffffff7) | 0x0c000000; 1476 ccfifo_writel(emc, value, EMC_MRW3, delay); 1477 ccfifo_writel(emc, 0, EMC_SELF_REF, 0); 1478 ccfifo_writel(emc, 0, EMC_REF, 0); 1479 1480 ccfifo_writel(emc, EMC_ZQ_CAL_ZQ_LATCH_CMD, EMC_ZQ_CAL, 1481 max_t(s32, 0, zq_latch_dvfs_wait_time)); 1482 } 1483 } 1484 1485 /* WAR: delay for zqlatch */ 1486 ccfifo_writel(emc, 0, 0, 10); 1487 1488 /* 1489 * Step 16: 1490 * LPDDR4 Conditional Training Kickoff. Removed. 1491 */ 1492 1493 /* 1494 * Step 17: 1495 * MANSR exit self refresh. 1496 */ 1497 emc_dbg(emc, STEPS, "Step 17\n"); 1498 1499 if (opt_dvfs_mode == MAN_SR && dram_type != DRAM_TYPE_LPDDR4) 1500 ccfifo_writel(emc, 0, EMC_SELF_REF, 0); 1501 1502 /* 1503 * Step 18: 1504 * Send MRWs to LPDDR3/DDR3. 1505 */ 1506 emc_dbg(emc, STEPS, "Step 18\n"); 1507 1508 if (dram_type == DRAM_TYPE_LPDDR2) { 1509 ccfifo_writel(emc, next->emc_mrw2, EMC_MRW2, 0); 1510 ccfifo_writel(emc, next->emc_mrw, EMC_MRW, 0); 1511 if (is_lpddr3) 1512 ccfifo_writel(emc, next->emc_mrw4, EMC_MRW4, 0); 1513 } else if (dram_type == DRAM_TYPE_DDR3) { 1514 if (opt_dll_mode) 1515 ccfifo_writel(emc, next->emc_emrs & 1516 ~EMC_EMRS_USE_EMRS_LONG_CNT, EMC_EMRS, 0); 1517 ccfifo_writel(emc, next->emc_emrs2 & 1518 ~EMC_EMRS2_USE_EMRS2_LONG_CNT, EMC_EMRS2, 0); 1519 ccfifo_writel(emc, next->emc_mrs | 1520 EMC_EMRS_USE_EMRS_LONG_CNT, EMC_MRS, 0); 1521 } 1522 1523 /* 1524 * Step 19: 1525 * ZQCAL for LPDDR3/DDR3 1526 */ 1527 emc_dbg(emc, STEPS, "Step 19\n"); 1528 1529 if (opt_zcal_en_cc) { 1530 if (dram_type == DRAM_TYPE_LPDDR2) { 1531 value = opt_cc_short_zcal ? 90000 : 360000; 1532 value = div_o3(value, dst_clk_period); 1533 value = value << 1534 EMC_MRS_WAIT_CNT2_MRS_EXT2_WAIT_CNT_SHIFT | 1535 value << 1536 EMC_MRS_WAIT_CNT2_MRS_EXT1_WAIT_CNT_SHIFT; 1537 ccfifo_writel(emc, value, EMC_MRS_WAIT_CNT2, 0); 1538 1539 value = opt_cc_short_zcal ? 0x56 : 0xab; 1540 ccfifo_writel(emc, 2 << EMC_MRW_MRW_DEV_SELECTN_SHIFT | 1541 EMC_MRW_USE_MRW_EXT_CNT | 1542 10 << EMC_MRW_MRW_MA_SHIFT | 1543 value << EMC_MRW_MRW_OP_SHIFT, 1544 EMC_MRW, 0); 1545 1546 if (emc->num_devices > 1) { 1547 value = 1 << EMC_MRW_MRW_DEV_SELECTN_SHIFT | 1548 EMC_MRW_USE_MRW_EXT_CNT | 1549 10 << EMC_MRW_MRW_MA_SHIFT | 1550 value << EMC_MRW_MRW_OP_SHIFT; 1551 ccfifo_writel(emc, value, EMC_MRW, 0); 1552 } 1553 } else if (dram_type == DRAM_TYPE_DDR3) { 1554 value = opt_cc_short_zcal ? 0 : EMC_ZQ_CAL_LONG; 1555 1556 ccfifo_writel(emc, value | 1557 2 << EMC_ZQ_CAL_DEV_SEL_SHIFT | 1558 EMC_ZQ_CAL_ZQ_CAL_CMD, EMC_ZQ_CAL, 1559 0); 1560 1561 if (emc->num_devices > 1) { 1562 value = value | 1 << EMC_ZQ_CAL_DEV_SEL_SHIFT | 1563 EMC_ZQ_CAL_ZQ_CAL_CMD; 1564 ccfifo_writel(emc, value, EMC_ZQ_CAL, 0); 1565 } 1566 } 1567 } 1568 1569 if (bg_reg_mode_change) { 1570 tegra210_emc_set_shadow_bypass(emc, ACTIVE); 1571 1572 if (ramp_up_wait <= 1250000) 1573 delay = (1250000 - ramp_up_wait) / dst_clk_period; 1574 else 1575 delay = 0; 1576 1577 ccfifo_writel(emc, 1578 next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX], 1579 EMC_PMACRO_BG_BIAS_CTRL_0, delay); 1580 tegra210_emc_set_shadow_bypass(emc, ASSEMBLY); 1581 } 1582 1583 /* 1584 * Step 20: 1585 * Issue ref and optional QRST. 1586 */ 1587 emc_dbg(emc, STEPS, "Step 20\n"); 1588 1589 if (dram_type != DRAM_TYPE_LPDDR4) 1590 ccfifo_writel(emc, 0, EMC_REF, 0); 1591 1592 if (opt_do_sw_qrst) { 1593 ccfifo_writel(emc, 1, EMC_ISSUE_QRST, 0); 1594 ccfifo_writel(emc, 0, EMC_ISSUE_QRST, 2); 1595 } 1596 1597 /* 1598 * Step 21: 1599 * Restore ZCAL and ZCAL interval. 1600 */ 1601 emc_dbg(emc, STEPS, "Step 21\n"); 1602 1603 if (save_restore_clkstop_pd || opt_zcal_en_cc) { 1604 ccfifo_writel(emc, emc_dbg | EMC_DBG_WRITE_MUX_ACTIVE, 1605 EMC_DBG, 0); 1606 if (opt_zcal_en_cc && dram_type != DRAM_TYPE_LPDDR4) 1607 ccfifo_writel(emc, next->burst_regs[EMC_ZCAL_INTERVAL_INDEX], 1608 EMC_ZCAL_INTERVAL, 0); 1609 1610 if (save_restore_clkstop_pd) 1611 ccfifo_writel(emc, next->burst_regs[EMC_CFG_INDEX] & 1612 ~EMC_CFG_DYN_SELF_REF, 1613 EMC_CFG, 0); 1614 ccfifo_writel(emc, emc_dbg, EMC_DBG, 0); 1615 } 1616 1617 /* 1618 * Step 22: 1619 * Restore EMC_CFG_PIPE_CLK. 1620 */ 1621 emc_dbg(emc, STEPS, "Step 22\n"); 1622 1623 ccfifo_writel(emc, emc_cfg_pipe_clk, EMC_CFG_PIPE_CLK, 0); 1624 1625 if (bg_reg_mode_change) { 1626 if (enable_bg_reg) 1627 emc_writel(emc, 1628 next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & 1629 ~EMC_PMACRO_BG_BIAS_CTRL_0_BGLP_E_PWRD, 1630 EMC_PMACRO_BG_BIAS_CTRL_0); 1631 else 1632 emc_writel(emc, 1633 next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & 1634 ~EMC_PMACRO_BG_BIAS_CTRL_0_BG_E_PWRD, 1635 EMC_PMACRO_BG_BIAS_CTRL_0); 1636 } 1637 1638 /* 1639 * Step 23: 1640 */ 1641 emc_dbg(emc, STEPS, "Step 23\n"); 1642 1643 value = emc_readl(emc, EMC_CFG_DIG_DLL); 1644 value |= EMC_CFG_DIG_DLL_CFG_DLL_STALL_ALL_TRAFFIC; 1645 value &= ~EMC_CFG_DIG_DLL_CFG_DLL_STALL_RW_UNTIL_LOCK; 1646 value &= ~EMC_CFG_DIG_DLL_CFG_DLL_STALL_ALL_UNTIL_LOCK; 1647 value &= ~EMC_CFG_DIG_DLL_CFG_DLL_EN; 1648 value = (value & ~EMC_CFG_DIG_DLL_CFG_DLL_MODE_MASK) | 1649 (2 << EMC_CFG_DIG_DLL_CFG_DLL_MODE_SHIFT); 1650 emc_writel(emc, value, EMC_CFG_DIG_DLL); 1651 1652 tegra210_emc_do_clock_change(emc, clksrc); 1653 1654 /* 1655 * Step 24: 1656 * Save training results. Removed. 1657 */ 1658 1659 /* 1660 * Step 25: 1661 * Program MC updown registers. 1662 */ 1663 emc_dbg(emc, STEPS, "Step 25\n"); 1664 1665 if (next->rate > last->rate) { 1666 for (i = 0; i < next->num_up_down; i++) 1667 mc_writel(emc->mc, next->la_scale_regs[i], 1668 emc->offsets->la_scale[i]); 1669 1670 tegra210_emc_timing_update(emc); 1671 } 1672 1673 /* 1674 * Step 26: 1675 * Restore ZCAL registers. 1676 */ 1677 emc_dbg(emc, STEPS, "Step 26\n"); 1678 1679 if (dram_type == DRAM_TYPE_LPDDR4) { 1680 tegra210_emc_set_shadow_bypass(emc, ACTIVE); 1681 emc_writel(emc, next->burst_regs[EMC_ZCAL_WAIT_CNT_INDEX], 1682 EMC_ZCAL_WAIT_CNT); 1683 emc_writel(emc, next->burst_regs[EMC_ZCAL_INTERVAL_INDEX], 1684 EMC_ZCAL_INTERVAL); 1685 tegra210_emc_set_shadow_bypass(emc, ASSEMBLY); 1686 } 1687 1688 if (dram_type != DRAM_TYPE_LPDDR4 && opt_zcal_en_cc && 1689 !opt_short_zcal && opt_cc_short_zcal) { 1690 udelay(2); 1691 1692 tegra210_emc_set_shadow_bypass(emc, ACTIVE); 1693 if (dram_type == DRAM_TYPE_LPDDR2) 1694 emc_writel(emc, next->burst_regs[EMC_MRS_WAIT_CNT_INDEX], 1695 EMC_MRS_WAIT_CNT); 1696 else if (dram_type == DRAM_TYPE_DDR3) 1697 emc_writel(emc, next->burst_regs[EMC_ZCAL_WAIT_CNT_INDEX], 1698 EMC_ZCAL_WAIT_CNT); 1699 tegra210_emc_set_shadow_bypass(emc, ASSEMBLY); 1700 } 1701 1702 /* 1703 * Step 27: 1704 * Restore EMC_CFG, FDPD registers. 1705 */ 1706 emc_dbg(emc, STEPS, "Step 27\n"); 1707 1708 tegra210_emc_set_shadow_bypass(emc, ACTIVE); 1709 emc_writel(emc, next->burst_regs[EMC_CFG_INDEX], EMC_CFG); 1710 tegra210_emc_set_shadow_bypass(emc, ASSEMBLY); 1711 emc_writel(emc, next->emc_fdpd_ctrl_cmd_no_ramp, 1712 EMC_FDPD_CTRL_CMD_NO_RAMP); 1713 emc_writel(emc, next->emc_sel_dpd_ctrl, EMC_SEL_DPD_CTRL); 1714 1715 /* 1716 * Step 28: 1717 * Training recover. Removed. 1718 */ 1719 emc_dbg(emc, STEPS, "Step 28\n"); 1720 1721 tegra210_emc_set_shadow_bypass(emc, ACTIVE); 1722 emc_writel(emc, 1723 next->burst_regs[EMC_PMACRO_AUTOCAL_CFG_COMMON_INDEX], 1724 EMC_PMACRO_AUTOCAL_CFG_COMMON); 1725 tegra210_emc_set_shadow_bypass(emc, ASSEMBLY); 1726 1727 /* 1728 * Step 29: 1729 * Power fix WAR. 1730 */ 1731 emc_dbg(emc, STEPS, "Step 29\n"); 1732 1733 emc_writel(emc, EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE0 | 1734 EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE1 | 1735 EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE2 | 1736 EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE3 | 1737 EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE4 | 1738 EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE5 | 1739 EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE6 | 1740 EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE7, 1741 EMC_PMACRO_CFG_PM_GLOBAL_0); 1742 emc_writel(emc, EMC_PMACRO_TRAINING_CTRL_0_CH0_TRAINING_E_WRPTR, 1743 EMC_PMACRO_TRAINING_CTRL_0); 1744 emc_writel(emc, EMC_PMACRO_TRAINING_CTRL_1_CH1_TRAINING_E_WRPTR, 1745 EMC_PMACRO_TRAINING_CTRL_1); 1746 emc_writel(emc, 0, EMC_PMACRO_CFG_PM_GLOBAL_0); 1747 1748 /* 1749 * Step 30: 1750 * Re-enable autocal. 1751 */ 1752 emc_dbg(emc, STEPS, "Step 30: Re-enable DLL and AUTOCAL\n"); 1753 1754 if (next->burst_regs[EMC_CFG_DIG_DLL_INDEX] & EMC_CFG_DIG_DLL_CFG_DLL_EN) { 1755 value = emc_readl(emc, EMC_CFG_DIG_DLL); 1756 value |= EMC_CFG_DIG_DLL_CFG_DLL_STALL_ALL_TRAFFIC; 1757 value |= EMC_CFG_DIG_DLL_CFG_DLL_EN; 1758 value &= ~EMC_CFG_DIG_DLL_CFG_DLL_STALL_RW_UNTIL_LOCK; 1759 value &= ~EMC_CFG_DIG_DLL_CFG_DLL_STALL_ALL_UNTIL_LOCK; 1760 value = (value & ~EMC_CFG_DIG_DLL_CFG_DLL_MODE_MASK) | 1761 (2 << EMC_CFG_DIG_DLL_CFG_DLL_MODE_SHIFT); 1762 emc_writel(emc, value, EMC_CFG_DIG_DLL); 1763 tegra210_emc_timing_update(emc); 1764 } 1765 1766 emc_writel(emc, next->emc_auto_cal_config, EMC_AUTO_CAL_CONFIG); 1767 1768 /* Done! Yay. */ 1769 } 1770 1771 const struct tegra210_emc_sequence tegra210_emc_r21021 = { 1772 .revision = 0x7, 1773 .set_clock = tegra210_emc_r21021_set_clock, 1774 .periodic_compensation = tegra210_emc_r21021_periodic_compensation, 1775 }; 1776