Lines Matching +full:emc +full:- +full:mode +full:- +full:2
1 // SPDX-License-Identifier: GPL-2.0
3 * Copyright (c) 2014-2020, NVIDIA CORPORATION. All rights reserved.
14 #include "tegra210-emc.h"
15 #include "tegra210-mc.h"
22 #define SUB_STEPS (1 << 2)
36 #define emc_dbg(emc, flags, ...) dev_dbg(emc->dev, __VA_ARGS__) argument
43 WRITE_TRAINING_SEQUENCE = 2,
53 * PTFV defines - basically just indexes into the per table PTFV array.
57 #define PTFV_DQSOSC_MOVAVG_C0D1U0_INDEX 2
78 ({ next->ptfv_list[PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX] = \
79 next->ptfv_list[PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX] / \
80 next->ptfv_list[PTFV_DVFS_SAMPLES_INDEX]; })
86 ({ next->ptfv_list[PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX] += \
93 ((timing)->ptfv_list[PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX] / \
102 next->ptfv_list[dqs] = \
104 (next->ptfv_list[dqs] * \
105 next->ptfv_list[w])) / \
106 (next->ptfv_list[w] + 1); \
108 emc_dbg(emc, EMA_UPDATES, "%s: (s=%lu) EMA: %u\n", \
109 __stringify(dev), nval, next->ptfv_list[dqs]); \
114 ((timing)->ptfv_list[PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX])
116 static u32 update_clock_tree_delay(struct tegra210_emc *emc, int type) in update_clock_tree_delay() argument
119 struct tegra210_emc_timing *last = emc->last; in update_clock_tree_delay()
120 struct tegra210_emc_timing *next = emc->next; in update_clock_tree_delay()
121 u32 last_timing_rate_mhz = last->rate / 1000; in update_clock_tree_delay()
122 u32 next_timing_rate_mhz = next->rate / 1000; in update_clock_tree_delay()
127 u32 temp[2][2], value; in update_clock_tree_delay()
134 value = tegra210_emc_mrr_read(emc, 2, 19); in update_clock_tree_delay()
136 for (i = 0; i < emc->num_channels; i++) { in update_clock_tree_delay()
145 value = tegra210_emc_mrr_read(emc, 2, 18); in update_clock_tree_delay()
147 for (i = 0; i < emc->num_channels; i++) { in update_clock_tree_delay()
155 cval = tegra210_emc_actual_osc_clocks(last->run_clocks); in update_clock_tree_delay()
157 cval /= last_timing_rate_mhz * 2 * temp[0][0]; in update_clock_tree_delay()
168 tdel = next->current_dram_clktree[C0D0U0] - in update_clock_tree_delay()
170 tmdel = (tdel < 0) ? -1 * tdel : tdel; in update_clock_tree_delay()
174 next->tree_margin) in update_clock_tree_delay()
175 next->current_dram_clktree[C0D0U0] = in update_clock_tree_delay()
180 cval = tegra210_emc_actual_osc_clocks(last->run_clocks); in update_clock_tree_delay()
182 cval /= last_timing_rate_mhz * 2 * temp[0][1]; in update_clock_tree_delay()
193 tdel = next->current_dram_clktree[C0D0U1] - in update_clock_tree_delay()
195 tmdel = (tdel < 0) ? -1 * tdel : tdel; in update_clock_tree_delay()
201 next->tree_margin) in update_clock_tree_delay()
202 next->current_dram_clktree[C0D0U1] = in update_clock_tree_delay()
206 if (emc->num_channels > 1) { in update_clock_tree_delay()
208 cval = tegra210_emc_actual_osc_clocks(last->run_clocks); in update_clock_tree_delay()
210 cval /= last_timing_rate_mhz * 2 * temp[1][0]; in update_clock_tree_delay()
221 tdel = next->current_dram_clktree[C1D0U0] - in update_clock_tree_delay()
223 tmdel = (tdel < 0) ? -1 * tdel : tdel; in update_clock_tree_delay()
229 next->tree_margin) in update_clock_tree_delay()
230 next->current_dram_clktree[C1D0U0] = in update_clock_tree_delay()
235 cval = tegra210_emc_actual_osc_clocks(last->run_clocks); in update_clock_tree_delay()
237 cval /= last_timing_rate_mhz * 2 * temp[1][1]; in update_clock_tree_delay()
248 tdel = next->current_dram_clktree[C1D0U1] - in update_clock_tree_delay()
250 tmdel = (tdel < 0) ? -1 * tdel : tdel; in update_clock_tree_delay()
256 next->tree_margin) in update_clock_tree_delay()
257 next->current_dram_clktree[C1D0U1] = in update_clock_tree_delay()
262 if (emc->num_devices < 2) in update_clock_tree_delay()
269 value = tegra210_emc_mrr_read(emc, 1, 19); in update_clock_tree_delay()
271 for (i = 0; i < emc->num_channels; i++) { in update_clock_tree_delay()
280 value = tegra210_emc_mrr_read(emc, 1, 18); in update_clock_tree_delay()
282 for (i = 0; i < emc->num_channels; i++) { in update_clock_tree_delay()
290 cval = tegra210_emc_actual_osc_clocks(last->run_clocks); in update_clock_tree_delay()
292 cval /= last_timing_rate_mhz * 2 * temp[0][0]; in update_clock_tree_delay()
303 tdel = next->current_dram_clktree[C0D1U0] - in update_clock_tree_delay()
305 tmdel = (tdel < 0) ? -1 * tdel : tdel; in update_clock_tree_delay()
311 next->tree_margin) in update_clock_tree_delay()
312 next->current_dram_clktree[C0D1U0] = in update_clock_tree_delay()
317 cval = tegra210_emc_actual_osc_clocks(last->run_clocks); in update_clock_tree_delay()
319 cval /= last_timing_rate_mhz * 2 * temp[0][1]; in update_clock_tree_delay()
330 tdel = next->current_dram_clktree[C0D1U1] - in update_clock_tree_delay()
332 tmdel = (tdel < 0) ? -1 * tdel : tdel; in update_clock_tree_delay()
338 next->tree_margin) in update_clock_tree_delay()
339 next->current_dram_clktree[C0D1U1] = in update_clock_tree_delay()
343 if (emc->num_channels > 1) { in update_clock_tree_delay()
345 cval = tegra210_emc_actual_osc_clocks(last->run_clocks); in update_clock_tree_delay()
347 cval /= last_timing_rate_mhz * 2 * temp[1][0]; in update_clock_tree_delay()
358 tdel = next->current_dram_clktree[C1D1U0] - in update_clock_tree_delay()
360 tmdel = (tdel < 0) ? -1 * tdel : tdel; in update_clock_tree_delay()
366 next->tree_margin) in update_clock_tree_delay()
367 next->current_dram_clktree[C1D1U0] = in update_clock_tree_delay()
372 cval = tegra210_emc_actual_osc_clocks(last->run_clocks); in update_clock_tree_delay()
374 cval /= last_timing_rate_mhz * 2 * temp[1][1]; in update_clock_tree_delay()
385 tdel = next->current_dram_clktree[C1D1U1] - in update_clock_tree_delay()
387 tmdel = (tdel < 0) ? -1 * tdel : tdel; in update_clock_tree_delay()
393 next->tree_margin) in update_clock_tree_delay()
394 next->current_dram_clktree[C1D1U1] = in update_clock_tree_delay()
403 static u32 periodic_compensation_handler(struct tegra210_emc *emc, u32 type, in periodic_compensation_handler() argument
409 (nt)->ptfv_list[PTFV_DVFS_SAMPLES_INDEX]; }) in periodic_compensation_handler()
411 u32 i, adel = 0, samples = next->ptfv_list[PTFV_DVFS_SAMPLES_INDEX]; in periodic_compensation_handler()
414 delay = tegra210_emc_actual_osc_clocks(last->run_clocks); in periodic_compensation_handler()
416 delay = 2 + (delay / last->rate); in periodic_compensation_handler()
418 if (!next->periodic_training) in periodic_compensation_handler()
422 if (last->periodic_training && in periodic_compensation_handler()
423 (next->ptfv_list[PTFV_CONFIG_CTRL_INDEX] & in periodic_compensation_handler()
450 tegra210_emc_start_periodic_compensation(emc); in periodic_compensation_handler()
456 adel = update_clock_tree_delay(emc, DVFS_PT1); in periodic_compensation_handler()
462 * 'if (last_timing->periodic_training)' conditional in periodic_compensation_handler()
465 adel = update_clock_tree_delay(emc, DVFS_UPDATE); in periodic_compensation_handler()
469 tegra210_emc_start_periodic_compensation(emc); in periodic_compensation_handler()
472 adel = update_clock_tree_delay(emc, PERIODIC_TRAINING_UPDATE); in periodic_compensation_handler()
478 static u32 tegra210_emc_r21021_periodic_compensation(struct tegra210_emc *emc) in tegra210_emc_r21021_periodic_compensation() argument
493 struct tegra210_emc_timing *last = emc->last; in tegra210_emc_r21021_periodic_compensation()
497 if (last->periodic_training) { in tegra210_emc_r21021_periodic_compensation()
498 emc_dbg(emc, PER_TRAIN, "Periodic training starting\n"); in tegra210_emc_r21021_periodic_compensation()
500 value = emc_readl(emc, EMC_DBG); in tegra210_emc_r21021_periodic_compensation()
501 emc_cfg_o = emc_readl(emc, EMC_CFG); in tegra210_emc_r21021_periodic_compensation()
510 emc_writel(emc, emc_cfg, EMC_CFG); in tegra210_emc_r21021_periodic_compensation()
513 tegra210_emc_dll_disable(emc); in tegra210_emc_r21021_periodic_compensation()
515 for (i = 0; i < emc->num_channels; i++) in tegra210_emc_r21021_periodic_compensation()
516 tegra210_emc_wait_for_update(emc, i, EMC_EMC_STATUS, in tegra210_emc_r21021_periodic_compensation()
520 for (i = 0; i < emc->num_channels; i++) in tegra210_emc_r21021_periodic_compensation()
521 tegra210_emc_wait_for_update(emc, i, EMC_EMC_STATUS, in tegra210_emc_r21021_periodic_compensation()
525 emc_cfg_update = value = emc_readl(emc, EMC_CFG_UPDATE); in tegra210_emc_r21021_periodic_compensation()
527 value |= (2 << EMC_CFG_UPDATE_UPDATE_DLL_IN_UPDATE_SHIFT); in tegra210_emc_r21021_periodic_compensation()
528 emc_writel(emc, value, EMC_CFG_UPDATE); in tegra210_emc_r21021_periodic_compensation()
531 * 2. osc kick off - this assumes training and dvfs have set in tegra210_emc_r21021_periodic_compensation()
534 tegra210_emc_start_periodic_compensation(emc); in tegra210_emc_r21021_periodic_compensation()
539 delay = tegra210_emc_actual_osc_clocks(last->run_clocks); in tegra210_emc_r21021_periodic_compensation()
541 delay /= last->rate + 1; in tegra210_emc_r21021_periodic_compensation()
548 del = periodic_compensation_handler(emc, in tegra210_emc_r21021_periodic_compensation()
556 if (last->tree_margin < ((del * 128 * (last->rate / 1000)) / 1000000)) { in tegra210_emc_r21021_periodic_compensation()
559 emc_dbg(emc, EMA_WRITES, "0x%08x <= 0x%08x\n", in tegra210_emc_r21021_periodic_compensation()
561 emc_writel(emc, value, list[i]); in tegra210_emc_r21021_periodic_compensation()
565 emc_writel(emc, emc_cfg_o, EMC_CFG); in tegra210_emc_r21021_periodic_compensation()
570 tegra210_emc_timing_update(emc); in tegra210_emc_r21021_periodic_compensation()
573 emc_writel(emc, emc_cfg_update, EMC_CFG_UPDATE); in tegra210_emc_r21021_periodic_compensation()
576 tegra210_emc_dll_enable(emc); in tegra210_emc_r21021_periodic_compensation()
585 static void tegra210_emc_r21021_set_clock(struct tegra210_emc *emc, u32 clksrc) in tegra210_emc_r21021_set_clock() argument
602 * necessarily correspond to the actual timing values in the EMC at the in tegra210_emc_r21021_set_clock()
607 struct tegra210_emc_timing *fake, *last = emc->last, *next = emc->next; in tegra210_emc_r21021_set_clock()
612 u32 tFC_lpddr4 = 1000 * next->dram_timings[T_FC_LPDDR4]; in tegra210_emc_r21021_set_clock()
624 emc_dbg(emc, INFO, "Running clock change.\n"); in tegra210_emc_r21021_set_clock()
627 fake = tegra210_emc_find_timing(emc, last->rate * 1000UL); in tegra210_emc_r21021_set_clock()
630 value = emc_readl(emc, EMC_FBIO_CFG5) & EMC_FBIO_CFG5_DRAM_TYPE_MASK; in tegra210_emc_r21021_set_clock()
633 if (last->burst_regs[EMC_ZCAL_WAIT_CNT_INDEX] & BIT(31)) in tegra210_emc_r21021_set_clock()
636 if ((next->burst_regs[EMC_ZCAL_INTERVAL_INDEX] != 0 && in tegra210_emc_r21021_set_clock()
637 last->burst_regs[EMC_ZCAL_INTERVAL_INDEX] == 0) || in tegra210_emc_r21021_set_clock()
644 if ((next->burst_regs[EMC_FBIO_CFG5_INDEX] & BIT(25)) && in tegra210_emc_r21021_set_clock()
648 emc_readl(emc, EMC_CFG); in tegra210_emc_r21021_set_clock()
649 emc_readl(emc, EMC_AUTO_CAL_CONFIG); in tegra210_emc_r21021_set_clock()
651 src_clk_period = 1000000000 / last->rate; in tegra210_emc_r21021_set_clock()
652 dst_clk_period = 1000000000 / next->rate; in tegra210_emc_r21021_set_clock()
655 tZQCAL_lpddr4_fc_adj = tZQCAL_lpddr4 - tFC_lpddr4; in tegra210_emc_r21021_set_clock()
661 emc_dbg = emc_readl(emc, EMC_DBG); in tegra210_emc_r21021_set_clock()
662 emc_pin = emc_readl(emc, EMC_PIN); in tegra210_emc_r21021_set_clock()
663 emc_cfg_pipe_clk = emc_readl(emc, EMC_CFG_PIPE_CLK); in tegra210_emc_r21021_set_clock()
665 emc_cfg = next->burst_regs[EMC_CFG_INDEX]; in tegra210_emc_r21021_set_clock()
668 emc_sel_dpd_ctrl = next->emc_sel_dpd_ctrl; in tegra210_emc_r21021_set_clock()
675 emc_dbg(emc, INFO, "Clock change version: %d\n", in tegra210_emc_r21021_set_clock()
677 emc_dbg(emc, INFO, "DRAM type = %d\n", dram_type); in tegra210_emc_r21021_set_clock()
678 emc_dbg(emc, INFO, "DRAM dev #: %u\n", emc->num_devices); in tegra210_emc_r21021_set_clock()
679 emc_dbg(emc, INFO, "Next EMC clksrc: 0x%08x\n", clksrc); in tegra210_emc_r21021_set_clock()
680 emc_dbg(emc, INFO, "DLL clksrc: 0x%08x\n", next->dll_clk_src); in tegra210_emc_r21021_set_clock()
681 emc_dbg(emc, INFO, "last rate: %u, next rate %u\n", last->rate, in tegra210_emc_r21021_set_clock()
682 next->rate); in tegra210_emc_r21021_set_clock()
683 emc_dbg(emc, INFO, "last period: %u, next period: %u\n", in tegra210_emc_r21021_set_clock()
685 emc_dbg(emc, INFO, " shared_zq_resistor: %d\n", !!shared_zq_resistor); in tegra210_emc_r21021_set_clock()
686 emc_dbg(emc, INFO, " num_channels: %u\n", emc->num_channels); in tegra210_emc_r21021_set_clock()
687 emc_dbg(emc, INFO, " opt_dll_mode: %d\n", opt_dll_mode); in tegra210_emc_r21021_set_clock()
693 emc_dbg(emc, STEPS, "Step 1\n"); in tegra210_emc_r21021_set_clock()
694 emc_dbg(emc, STEPS, "Step 1.1: Disable DLL temporarily.\n"); in tegra210_emc_r21021_set_clock()
696 value = emc_readl(emc, EMC_CFG_DIG_DLL); in tegra210_emc_r21021_set_clock()
698 emc_writel(emc, value, EMC_CFG_DIG_DLL); in tegra210_emc_r21021_set_clock()
700 tegra210_emc_timing_update(emc); in tegra210_emc_r21021_set_clock()
702 for (i = 0; i < emc->num_channels; i++) in tegra210_emc_r21021_set_clock()
703 tegra210_emc_wait_for_update(emc, i, EMC_CFG_DIG_DLL, in tegra210_emc_r21021_set_clock()
706 emc_dbg(emc, STEPS, "Step 1.2: Disable AUTOCAL temporarily.\n"); in tegra210_emc_r21021_set_clock()
708 emc_auto_cal_config = next->emc_auto_cal_config; in tegra210_emc_r21021_set_clock()
714 emc_writel(emc, emc_auto_cal_config, EMC_AUTO_CAL_CONFIG); in tegra210_emc_r21021_set_clock()
715 emc_readl(emc, EMC_AUTO_CAL_CONFIG); /* Flush write. */ in tegra210_emc_r21021_set_clock()
717 emc_dbg(emc, STEPS, "Step 1.3: Disable other power features.\n"); in tegra210_emc_r21021_set_clock()
719 tegra210_emc_set_shadow_bypass(emc, ACTIVE); in tegra210_emc_r21021_set_clock()
720 emc_writel(emc, emc_cfg, EMC_CFG); in tegra210_emc_r21021_set_clock()
721 emc_writel(emc, emc_sel_dpd_ctrl, EMC_SEL_DPD_CTRL); in tegra210_emc_r21021_set_clock()
722 tegra210_emc_set_shadow_bypass(emc, ASSEMBLY); in tegra210_emc_r21021_set_clock()
724 if (next->periodic_training) { in tegra210_emc_r21021_set_clock()
727 for (i = 0; i < emc->num_channels; i++) in tegra210_emc_r21021_set_clock()
728 tegra210_emc_wait_for_update(emc, i, EMC_EMC_STATUS, in tegra210_emc_r21021_set_clock()
732 for (i = 0; i < emc->num_channels; i++) in tegra210_emc_r21021_set_clock()
733 tegra210_emc_wait_for_update(emc, i, EMC_EMC_STATUS, in tegra210_emc_r21021_set_clock()
737 tegra210_emc_start_periodic_compensation(emc); in tegra210_emc_r21021_set_clock()
739 delay = 1000 * tegra210_emc_actual_osc_clocks(last->run_clocks); in tegra210_emc_r21021_set_clock()
740 udelay((delay / last->rate) + 2); in tegra210_emc_r21021_set_clock()
742 value = periodic_compensation_handler(emc, DVFS_SEQUENCE, fake, in tegra210_emc_r21021_set_clock()
744 value = (value * 128 * next->rate / 1000) / 1000000; in tegra210_emc_r21021_set_clock()
746 if (next->periodic_training && value > next->tree_margin) in tegra210_emc_r21021_set_clock()
750 emc_writel(emc, EMC_INTSTATUS_CLKCHANGE_COMPLETE, EMC_INTSTATUS); in tegra210_emc_r21021_set_clock()
751 tegra210_emc_set_shadow_bypass(emc, ACTIVE); in tegra210_emc_r21021_set_clock()
752 emc_writel(emc, emc_cfg, EMC_CFG); in tegra210_emc_r21021_set_clock()
753 emc_writel(emc, emc_sel_dpd_ctrl, EMC_SEL_DPD_CTRL); in tegra210_emc_r21021_set_clock()
754 emc_writel(emc, emc_cfg_pipe_clk | EMC_CFG_PIPE_CLK_CLK_ALWAYS_ON, in tegra210_emc_r21021_set_clock()
756 emc_writel(emc, next->emc_fdpd_ctrl_cmd_no_ramp & in tegra210_emc_r21021_set_clock()
761 ((next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & in tegra210_emc_r21021_set_clock()
763 (last->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & in tegra210_emc_r21021_set_clock()
765 ((next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & in tegra210_emc_r21021_set_clock()
767 (last->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & in tegra210_emc_r21021_set_clock()
770 (next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & in tegra210_emc_r21021_set_clock()
773 (next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & in tegra210_emc_r21021_set_clock()
778 emc_writel(emc, last->burst_regs in tegra210_emc_r21021_set_clock()
784 emc_writel(emc, last->burst_regs in tegra210_emc_r21021_set_clock()
791 if ((((last->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX] & in tegra210_emc_r21021_set_clock()
793 ((next->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX] & in tegra210_emc_r21021_set_clock()
795 (((last->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX] & in tegra210_emc_r21021_set_clock()
797 ((next->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX] & in tegra210_emc_r21021_set_clock()
800 next->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX]; in tegra210_emc_r21021_set_clock()
802 last->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX]; in tegra210_emc_r21021_set_clock()
813 emc_writel(emc, value, EMC_PMACRO_DATA_PAD_TX_CTRL); in tegra210_emc_r21021_set_clock()
819 tegra210_emc_set_shadow_bypass(emc, ASSEMBLY); in tegra210_emc_r21021_set_clock()
822 * Step 2: in tegra210_emc_r21021_set_clock()
825 emc_dbg(emc, STEPS, "Step 2\n"); in tegra210_emc_r21021_set_clock()
827 if (next->burst_regs[EMC_CFG_DIG_DLL_INDEX] & in tegra210_emc_r21021_set_clock()
829 emc_dbg(emc, INFO, "Prelock enabled for target frequency.\n"); in tegra210_emc_r21021_set_clock()
830 value = tegra210_emc_dll_prelock(emc, clksrc); in tegra210_emc_r21021_set_clock()
831 emc_dbg(emc, INFO, "DLL out: 0x%03x\n", value); in tegra210_emc_r21021_set_clock()
833 emc_dbg(emc, INFO, "Disabling DLL for target frequency.\n"); in tegra210_emc_r21021_set_clock()
834 tegra210_emc_dll_disable(emc); in tegra210_emc_r21021_set_clock()
841 emc_dbg(emc, STEPS, "Step 3\n"); in tegra210_emc_r21021_set_clock()
843 tegra210_emc_set_shadow_bypass(emc, ACTIVE); in tegra210_emc_r21021_set_clock()
844 emc_writel(emc, next->emc_auto_cal_config2, EMC_AUTO_CAL_CONFIG2); in tegra210_emc_r21021_set_clock()
845 emc_writel(emc, next->emc_auto_cal_config3, EMC_AUTO_CAL_CONFIG3); in tegra210_emc_r21021_set_clock()
846 emc_writel(emc, next->emc_auto_cal_config4, EMC_AUTO_CAL_CONFIG4); in tegra210_emc_r21021_set_clock()
847 emc_writel(emc, next->emc_auto_cal_config5, EMC_AUTO_CAL_CONFIG5); in tegra210_emc_r21021_set_clock()
848 emc_writel(emc, next->emc_auto_cal_config6, EMC_AUTO_CAL_CONFIG6); in tegra210_emc_r21021_set_clock()
849 emc_writel(emc, next->emc_auto_cal_config7, EMC_AUTO_CAL_CONFIG7); in tegra210_emc_r21021_set_clock()
850 emc_writel(emc, next->emc_auto_cal_config8, EMC_AUTO_CAL_CONFIG8); in tegra210_emc_r21021_set_clock()
851 tegra210_emc_set_shadow_bypass(emc, ASSEMBLY); in tegra210_emc_r21021_set_clock()
855 emc_writel(emc, emc_auto_cal_config, EMC_AUTO_CAL_CONFIG); in tegra210_emc_r21021_set_clock()
861 emc_dbg(emc, STEPS, "Step 4\n"); in tegra210_emc_r21021_set_clock()
864 ccfifo_writel(emc, 1, EMC_SELF_REF, 0); in tegra210_emc_r21021_set_clock()
866 emc_writel(emc, next->emc_cfg_2, EMC_CFG_2); in tegra210_emc_r21021_set_clock()
872 emc_dbg(emc, STEPS, "Step 5\n"); in tegra210_emc_r21021_set_clock()
877 zq_wait_long = max(next->min_mrs_wait, in tegra210_emc_r21021_set_clock()
881 div_o3(320000, dst_clk_period) + 2); in tegra210_emc_r21021_set_clock()
887 * Training code - removed. in tegra210_emc_r21021_set_clock()
889 emc_dbg(emc, STEPS, "Step 6\n"); in tegra210_emc_r21021_set_clock()
895 emc_dbg(emc, STEPS, "Step 7\n"); in tegra210_emc_r21021_set_clock()
896 emc_dbg(emc, SUB_STEPS, "Step 7.1: Bug 200024907 - Patch RP R2P"); in tegra210_emc_r21021_set_clock()
924 tRPST = (last->emc_mrw & 0x80) >> 7; in tegra210_emc_r21021_set_clock()
925 tRTM = fake->dram_timings[RL] + div_o3(3600, src_clk_period) + in tegra210_emc_r21021_set_clock()
929 emc_dbg(emc, INFO, "tRTM = %u, EMC_RP = %u\n", tRTM, in tegra210_emc_r21021_set_clock()
930 next->burst_regs[EMC_RP_INDEX]); in tegra210_emc_r21021_set_clock()
932 if (last->burst_regs[EMC_RP_INDEX] < tRTM) { in tegra210_emc_r21021_set_clock()
933 if (tRTM > (last->burst_regs[EMC_R2P_INDEX] + in tegra210_emc_r21021_set_clock()
934 last->burst_regs[EMC_RP_INDEX])) { in tegra210_emc_r21021_set_clock()
935 R2P_war = tRTM - last->burst_regs[EMC_RP_INDEX]; in tegra210_emc_r21021_set_clock()
936 RP_war = last->burst_regs[EMC_RP_INDEX]; in tegra210_emc_r21021_set_clock()
937 TRPab_war = last->burst_regs[EMC_TRPAB_INDEX]; in tegra210_emc_r21021_set_clock()
941 last->burst_regs[EMC_RP_INDEX] - 63; in tegra210_emc_r21021_set_clock()
949 R2P_war = last->burst_regs[EMC_R2P_INDEX]; in tegra210_emc_r21021_set_clock()
950 RP_war = last->burst_regs[EMC_RP_INDEX]; in tegra210_emc_r21021_set_clock()
951 TRPab_war = last->burst_regs[EMC_TRPAB_INDEX]; in tegra210_emc_r21021_set_clock()
955 W2P_war = last->burst_regs[EMC_W2P_INDEX] in tegra210_emc_r21021_set_clock()
956 + deltaTWATM - RP_war; in tegra210_emc_r21021_set_clock()
958 RP_war = RP_war + W2P_war - 63; in tegra210_emc_r21021_set_clock()
964 W2P_war = last->burst_regs[ in tegra210_emc_r21021_set_clock()
968 if ((last->burst_regs[EMC_W2P_INDEX] ^ W2P_war) || in tegra210_emc_r21021_set_clock()
969 (last->burst_regs[EMC_R2P_INDEX] ^ R2P_war) || in tegra210_emc_r21021_set_clock()
970 (last->burst_regs[EMC_RP_INDEX] ^ RP_war) || in tegra210_emc_r21021_set_clock()
971 (last->burst_regs[EMC_TRPAB_INDEX] ^ TRPab_war)) { in tegra210_emc_r21021_set_clock()
972 emc_writel(emc, RP_war, EMC_RP); in tegra210_emc_r21021_set_clock()
973 emc_writel(emc, R2P_war, EMC_R2P); in tegra210_emc_r21021_set_clock()
974 emc_writel(emc, W2P_war, EMC_W2P); in tegra210_emc_r21021_set_clock()
975 emc_writel(emc, TRPab_war, EMC_TRPAB); in tegra210_emc_r21021_set_clock()
978 tegra210_emc_timing_update(emc); in tegra210_emc_r21021_set_clock()
980 emc_dbg(emc, INFO, "Skipped WAR\n"); in tegra210_emc_r21021_set_clock()
985 mr13_flip_fspwr = (next->emc_mrw3 & 0xffffff3f) | 0x80; in tegra210_emc_r21021_set_clock()
986 mr13_flip_fspop = (next->emc_mrw3 & 0xffffff3f) | 0x00; in tegra210_emc_r21021_set_clock()
988 mr13_flip_fspwr = (next->emc_mrw3 & 0xffffff3f) | 0x40; in tegra210_emc_r21021_set_clock()
989 mr13_flip_fspop = (next->emc_mrw3 & 0xffffff3f) | 0xc0; in tegra210_emc_r21021_set_clock()
993 emc_writel(emc, mr13_flip_fspwr, EMC_MRW3); in tegra210_emc_r21021_set_clock()
994 emc_writel(emc, next->emc_mrw, EMC_MRW); in tegra210_emc_r21021_set_clock()
995 emc_writel(emc, next->emc_mrw2, EMC_MRW2); in tegra210_emc_r21021_set_clock()
1002 emc_dbg(emc, STEPS, "Step 8\n"); in tegra210_emc_r21021_set_clock()
1003 emc_dbg(emc, SUB_STEPS, "Writing burst_regs\n"); in tegra210_emc_r21021_set_clock()
1005 for (i = 0; i < next->num_burst; i++) { in tegra210_emc_r21021_set_clock()
1006 const u16 *offsets = emc->offsets->burst; in tegra210_emc_r21021_set_clock()
1012 value = next->burst_regs[i]; in tegra210_emc_r21021_set_clock()
1069 emc_writel(emc, value, offset); in tegra210_emc_r21021_set_clock()
1072 /* SW addition: do EMC refresh adjustment here. */ in tegra210_emc_r21021_set_clock()
1073 tegra210_emc_adjust_timing(emc, next); in tegra210_emc_r21021_set_clock()
1077 (next->run_clocks & EMC_MRW_MRW_OP_MASK); in tegra210_emc_r21021_set_clock()
1078 emc_writel(emc, value, EMC_MRW); in tegra210_emc_r21021_set_clock()
1082 emc_dbg(emc, SUB_STEPS, "Writing burst_regs_per_ch\n"); in tegra210_emc_r21021_set_clock()
1084 for (i = 0; i < next->num_burst_per_ch; i++) { in tegra210_emc_r21021_set_clock()
1086 emc->offsets->burst_per_channel; in tegra210_emc_r21021_set_clock()
1104 /* Filter out second channel if not in DUAL_CHANNEL mode. */ in tegra210_emc_r21021_set_clock()
1105 if (emc->num_channels < 2 && burst[i].bank >= 1) in tegra210_emc_r21021_set_clock()
1108 emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i, in tegra210_emc_r21021_set_clock()
1109 next->burst_reg_per_ch[i], burst[i].offset); in tegra210_emc_r21021_set_clock()
1110 emc_channel_writel(emc, burst[i].bank, in tegra210_emc_r21021_set_clock()
1111 next->burst_reg_per_ch[i], in tegra210_emc_r21021_set_clock()
1116 emc_dbg(emc, SUB_STEPS, "Writing vref_regs\n"); in tegra210_emc_r21021_set_clock()
1118 for (i = 0; i < next->vref_num; i++) { in tegra210_emc_r21021_set_clock()
1120 emc->offsets->vref_per_channel; in tegra210_emc_r21021_set_clock()
1125 if (emc->num_channels < 2 && vref[i].bank >= 1) in tegra210_emc_r21021_set_clock()
1128 emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i, in tegra210_emc_r21021_set_clock()
1129 next->vref_perch_regs[i], vref[i].offset); in tegra210_emc_r21021_set_clock()
1130 emc_channel_writel(emc, vref[i].bank, next->vref_perch_regs[i], in tegra210_emc_r21021_set_clock()
1135 emc_dbg(emc, SUB_STEPS, "Writing trim_regs\n"); in tegra210_emc_r21021_set_clock()
1137 for (i = 0; i < next->num_trim; i++) { in tegra210_emc_r21021_set_clock()
1138 const u16 *offsets = emc->offsets->trim; in tegra210_emc_r21021_set_clock()
1155 emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i, in tegra210_emc_r21021_set_clock()
1157 emc_dbg(emc, EMA_WRITES, "0x%08x <= 0x%08x\n", in tegra210_emc_r21021_set_clock()
1159 emc_writel(emc, value, offsets[i]); in tegra210_emc_r21021_set_clock()
1161 emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i, in tegra210_emc_r21021_set_clock()
1162 next->trim_regs[i], offsets[i]); in tegra210_emc_r21021_set_clock()
1163 emc_writel(emc, next->trim_regs[i], offsets[i]); in tegra210_emc_r21021_set_clock()
1168 emc_dbg(emc, SUB_STEPS, "Writing trim_regs_per_ch\n"); in tegra210_emc_r21021_set_clock()
1170 for (i = 0; i < next->num_trim_per_ch; i++) { in tegra210_emc_r21021_set_clock()
1172 &emc->offsets->trim_per_channel[0]; in tegra210_emc_r21021_set_clock()
1178 if (emc->num_channels < 2 && trim[i].bank >= 1) in tegra210_emc_r21021_set_clock()
1195 emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i, in tegra210_emc_r21021_set_clock()
1197 emc_dbg(emc, EMA_WRITES, "0x%08x <= 0x%08x\n", offset, in tegra210_emc_r21021_set_clock()
1199 emc_channel_writel(emc, trim[i].bank, value, offset); in tegra210_emc_r21021_set_clock()
1201 emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i, in tegra210_emc_r21021_set_clock()
1202 next->trim_perch_regs[i], offset); in tegra210_emc_r21021_set_clock()
1203 emc_channel_writel(emc, trim[i].bank, in tegra210_emc_r21021_set_clock()
1204 next->trim_perch_regs[i], offset); in tegra210_emc_r21021_set_clock()
1208 emc_dbg(emc, SUB_STEPS, "Writing burst_mc_regs\n"); in tegra210_emc_r21021_set_clock()
1210 for (i = 0; i < next->num_mc_regs; i++) { in tegra210_emc_r21021_set_clock()
1211 const u16 *offsets = emc->offsets->burst_mc; in tegra210_emc_r21021_set_clock()
1212 u32 *values = next->burst_mc_regs; in tegra210_emc_r21021_set_clock()
1214 emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i, in tegra210_emc_r21021_set_clock()
1216 mc_writel(emc->mc, values[i], offsets[i]); in tegra210_emc_r21021_set_clock()
1220 if (next->rate < last->rate) { in tegra210_emc_r21021_set_clock()
1221 const u16 *la = emc->offsets->la_scale; in tegra210_emc_r21021_set_clock()
1223 emc_dbg(emc, SUB_STEPS, "Writing la_scale_regs\n"); in tegra210_emc_r21021_set_clock()
1225 for (i = 0; i < next->num_up_down; i++) { in tegra210_emc_r21021_set_clock()
1226 emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i, in tegra210_emc_r21021_set_clock()
1227 next->la_scale_regs[i], la[i]); in tegra210_emc_r21021_set_clock()
1228 mc_writel(emc->mc, next->la_scale_regs[i], la[i]); in tegra210_emc_r21021_set_clock()
1233 mc_readl(emc->mc, MC_EMEM_ADR_CFG); in tegra210_emc_r21021_set_clock()
1239 emc_dbg(emc, STEPS, "Step 9\n"); in tegra210_emc_r21021_set_clock()
1241 value = next->burst_regs[EMC_ZCAL_WAIT_CNT_INDEX]; in tegra210_emc_r21021_set_clock()
1245 emc_writel(emc, 0, EMC_ZCAL_INTERVAL); in tegra210_emc_r21021_set_clock()
1246 emc_writel(emc, value, EMC_ZCAL_WAIT_CNT); in tegra210_emc_r21021_set_clock()
1251 emc_writel(emc, value, EMC_DBG); in tegra210_emc_r21021_set_clock()
1252 emc_writel(emc, 0, EMC_ZCAL_INTERVAL); in tegra210_emc_r21021_set_clock()
1253 emc_writel(emc, emc_dbg, EMC_DBG); in tegra210_emc_r21021_set_clock()
1260 emc_dbg(emc, STEPS, "Step 10\n"); in tegra210_emc_r21021_set_clock()
1264 ccfifo_writel(emc, 0x101, EMC_SELF_REF, 0); in tegra210_emc_r21021_set_clock()
1266 ccfifo_writel(emc, 0x1, EMC_SELF_REF, 0); in tegra210_emc_r21021_set_clock()
1270 ccfifo_writel(emc, mr13_flip_fspwr ^ 0x40, EMC_MRW3, 0); in tegra210_emc_r21021_set_clock()
1271 ccfifo_writel(emc, (next->burst_regs[EMC_MRW6_INDEX] & in tegra210_emc_r21021_set_clock()
1273 (last->burst_regs[EMC_MRW6_INDEX] & in tegra210_emc_r21021_set_clock()
1275 ccfifo_writel(emc, (next->burst_regs[EMC_MRW14_INDEX] & in tegra210_emc_r21021_set_clock()
1277 (last->burst_regs[EMC_MRW14_INDEX] & in tegra210_emc_r21021_set_clock()
1280 if (emc->num_devices > 1) { in tegra210_emc_r21021_set_clock()
1281 ccfifo_writel(emc, in tegra210_emc_r21021_set_clock()
1282 (next->burst_regs[EMC_MRW7_INDEX] & in tegra210_emc_r21021_set_clock()
1284 (last->burst_regs[EMC_MRW7_INDEX] & in tegra210_emc_r21021_set_clock()
1286 ccfifo_writel(emc, in tegra210_emc_r21021_set_clock()
1287 (next->burst_regs[EMC_MRW15_INDEX] & in tegra210_emc_r21021_set_clock()
1289 (last->burst_regs[EMC_MRW15_INDEX] & in tegra210_emc_r21021_set_clock()
1294 if (emc->num_devices < 2) in tegra210_emc_r21021_set_clock()
1295 ccfifo_writel(emc, in tegra210_emc_r21021_set_clock()
1296 2UL << EMC_ZQ_CAL_DEV_SEL_SHIFT in tegra210_emc_r21021_set_clock()
1300 ccfifo_writel(emc, in tegra210_emc_r21021_set_clock()
1301 2UL << EMC_ZQ_CAL_DEV_SEL_SHIFT in tegra210_emc_r21021_set_clock()
1305 ccfifo_writel(emc, in tegra210_emc_r21021_set_clock()
1313 value = (1000 * fake->dram_timings[T_RP]) / src_clk_period; in tegra210_emc_r21021_set_clock()
1314 ccfifo_writel(emc, mr13_flip_fspop | 0x8, EMC_MRW3, value); in tegra210_emc_r21021_set_clock()
1315 ccfifo_writel(emc, 0, 0, tFC_lpddr4 / src_clk_period); in tegra210_emc_r21021_set_clock()
1322 delay += (1000 * fake->dram_timings[T_RP]) / in tegra210_emc_r21021_set_clock()
1324 delay += 4000 * fake->dram_timings[T_RFC]; in tegra210_emc_r21021_set_clock()
1327 ccfifo_writel(emc, emc_pin & ~(EMC_PIN_PIN_CKE_PER_DEV | in tegra210_emc_r21021_set_clock()
1346 delay = ((1000 * fake->dram_timings[T_RP] / src_clk_period) + in tegra210_emc_r21021_set_clock()
1347 (1000 * fake->dram_timings[T_RFC] / src_clk_period)); in tegra210_emc_r21021_set_clock()
1357 emc_dbg(emc, STEPS, "Step 11\n"); in tegra210_emc_r21021_set_clock()
1359 ccfifo_writel(emc, 0x0, EMC_CFG_SYNC, delay); in tegra210_emc_r21021_set_clock()
1362 ccfifo_writel(emc, value, EMC_DBG, 0); in tegra210_emc_r21021_set_clock()
1364 ramp_down_wait = tegra210_emc_dvfs_power_ramp_down(emc, src_clk_period, in tegra210_emc_r21021_set_clock()
1369 * And finally - trigger the clock change. in tegra210_emc_r21021_set_clock()
1371 emc_dbg(emc, STEPS, "Step 12\n"); in tegra210_emc_r21021_set_clock()
1373 ccfifo_writel(emc, 1, EMC_STALL_THEN_EXE_AFTER_CLKCHANGE, 0); in tegra210_emc_r21021_set_clock()
1375 ccfifo_writel(emc, value, EMC_DBG, 0); in tegra210_emc_r21021_set_clock()
1381 emc_dbg(emc, STEPS, "Step 13\n"); in tegra210_emc_r21021_set_clock()
1383 ramp_up_wait = tegra210_emc_dvfs_power_ramp_up(emc, dst_clk_period, 0); in tegra210_emc_r21021_set_clock()
1384 ccfifo_writel(emc, emc_dbg, EMC_DBG, 0); in tegra210_emc_r21021_set_clock()
1390 emc_dbg(emc, STEPS, "Step 14\n"); in tegra210_emc_r21021_set_clock()
1395 if (emc->num_devices <= 1) in tegra210_emc_r21021_set_clock()
1400 ccfifo_writel(emc, value, EMC_PIN, 0); in tegra210_emc_r21021_set_clock()
1407 emc_dbg(emc, STEPS, "Step 15\n"); in tegra210_emc_r21021_set_clock()
1412 zq_latch_dvfs_wait_time = (s32)tZQCAL_lpddr4_fc_adj - t; in tegra210_emc_r21021_set_clock()
1414 zq_latch_dvfs_wait_time = tZQCAL_lpddr4_fc_adj - in tegra210_emc_r21021_set_clock()
1415 div_o3(1000 * next->dram_timings[T_PDEX], in tegra210_emc_r21021_set_clock()
1419 emc_dbg(emc, INFO, "tZQCAL_lpddr4_fc_adj = %u\n", tZQCAL_lpddr4_fc_adj); in tegra210_emc_r21021_set_clock()
1420 emc_dbg(emc, INFO, "dst_clk_period = %u\n", in tegra210_emc_r21021_set_clock()
1422 emc_dbg(emc, INFO, "next->dram_timings[T_PDEX] = %u\n", in tegra210_emc_r21021_set_clock()
1423 next->dram_timings[T_PDEX]); in tegra210_emc_r21021_set_clock()
1424 emc_dbg(emc, INFO, "zq_latch_dvfs_wait_time = %d\n", in tegra210_emc_r21021_set_clock()
1428 delay = div_o3(1000 * next->dram_timings[T_PDEX], in tegra210_emc_r21021_set_clock()
1431 if (emc->num_devices < 2) { in tegra210_emc_r21021_set_clock()
1433 ccfifo_writel(emc, in tegra210_emc_r21021_set_clock()
1434 2UL << EMC_ZQ_CAL_DEV_SEL_SHIFT | in tegra210_emc_r21021_set_clock()
1439 ccfifo_writel(emc, value, EMC_MRW3, delay); in tegra210_emc_r21021_set_clock()
1440 ccfifo_writel(emc, 0, EMC_SELF_REF, 0); in tegra210_emc_r21021_set_clock()
1441 ccfifo_writel(emc, 0, EMC_REF, 0); in tegra210_emc_r21021_set_clock()
1442 ccfifo_writel(emc, 2UL << EMC_ZQ_CAL_DEV_SEL_SHIFT | in tegra210_emc_r21021_set_clock()
1448 ccfifo_writel(emc, in tegra210_emc_r21021_set_clock()
1449 2UL << EMC_ZQ_CAL_DEV_SEL_SHIFT | in tegra210_emc_r21021_set_clock()
1453 ccfifo_writel(emc, 2UL << EMC_ZQ_CAL_DEV_SEL_SHIFT | in tegra210_emc_r21021_set_clock()
1457 ccfifo_writel(emc, 1UL << EMC_ZQ_CAL_DEV_SEL_SHIFT | in tegra210_emc_r21021_set_clock()
1462 ccfifo_writel(emc, value, EMC_MRW3, 0); in tegra210_emc_r21021_set_clock()
1463 ccfifo_writel(emc, 0, EMC_SELF_REF, 0); in tegra210_emc_r21021_set_clock()
1464 ccfifo_writel(emc, 0, EMC_REF, 0); in tegra210_emc_r21021_set_clock()
1466 ccfifo_writel(emc, 1UL << EMC_ZQ_CAL_DEV_SEL_SHIFT | in tegra210_emc_r21021_set_clock()
1471 ccfifo_writel(emc, EMC_ZQ_CAL_ZQ_CAL_CMD, in tegra210_emc_r21021_set_clock()
1475 ccfifo_writel(emc, value, EMC_MRW3, delay); in tegra210_emc_r21021_set_clock()
1476 ccfifo_writel(emc, 0, EMC_SELF_REF, 0); in tegra210_emc_r21021_set_clock()
1477 ccfifo_writel(emc, 0, EMC_REF, 0); in tegra210_emc_r21021_set_clock()
1479 ccfifo_writel(emc, EMC_ZQ_CAL_ZQ_LATCH_CMD, EMC_ZQ_CAL, in tegra210_emc_r21021_set_clock()
1485 ccfifo_writel(emc, 0, 0, 10); in tegra210_emc_r21021_set_clock()
1496 emc_dbg(emc, STEPS, "Step 17\n"); in tegra210_emc_r21021_set_clock()
1499 ccfifo_writel(emc, 0, EMC_SELF_REF, 0); in tegra210_emc_r21021_set_clock()
1505 emc_dbg(emc, STEPS, "Step 18\n"); in tegra210_emc_r21021_set_clock()
1508 ccfifo_writel(emc, next->emc_mrw2, EMC_MRW2, 0); in tegra210_emc_r21021_set_clock()
1509 ccfifo_writel(emc, next->emc_mrw, EMC_MRW, 0); in tegra210_emc_r21021_set_clock()
1511 ccfifo_writel(emc, next->emc_mrw4, EMC_MRW4, 0); in tegra210_emc_r21021_set_clock()
1514 ccfifo_writel(emc, next->emc_emrs & in tegra210_emc_r21021_set_clock()
1516 ccfifo_writel(emc, next->emc_emrs2 & in tegra210_emc_r21021_set_clock()
1518 ccfifo_writel(emc, next->emc_mrs | in tegra210_emc_r21021_set_clock()
1526 emc_dbg(emc, STEPS, "Step 19\n"); in tegra210_emc_r21021_set_clock()
1536 ccfifo_writel(emc, value, EMC_MRS_WAIT_CNT2, 0); in tegra210_emc_r21021_set_clock()
1539 ccfifo_writel(emc, 2 << EMC_MRW_MRW_DEV_SELECTN_SHIFT | in tegra210_emc_r21021_set_clock()
1545 if (emc->num_devices > 1) { in tegra210_emc_r21021_set_clock()
1550 ccfifo_writel(emc, value, EMC_MRW, 0); in tegra210_emc_r21021_set_clock()
1555 ccfifo_writel(emc, value | in tegra210_emc_r21021_set_clock()
1556 2 << EMC_ZQ_CAL_DEV_SEL_SHIFT | in tegra210_emc_r21021_set_clock()
1560 if (emc->num_devices > 1) { in tegra210_emc_r21021_set_clock()
1563 ccfifo_writel(emc, value, EMC_ZQ_CAL, 0); in tegra210_emc_r21021_set_clock()
1569 tegra210_emc_set_shadow_bypass(emc, ACTIVE); in tegra210_emc_r21021_set_clock()
1572 delay = (1250000 - ramp_up_wait) / dst_clk_period; in tegra210_emc_r21021_set_clock()
1576 ccfifo_writel(emc, in tegra210_emc_r21021_set_clock()
1577 next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX], in tegra210_emc_r21021_set_clock()
1579 tegra210_emc_set_shadow_bypass(emc, ASSEMBLY); in tegra210_emc_r21021_set_clock()
1586 emc_dbg(emc, STEPS, "Step 20\n"); in tegra210_emc_r21021_set_clock()
1589 ccfifo_writel(emc, 0, EMC_REF, 0); in tegra210_emc_r21021_set_clock()
1592 ccfifo_writel(emc, 1, EMC_ISSUE_QRST, 0); in tegra210_emc_r21021_set_clock()
1593 ccfifo_writel(emc, 0, EMC_ISSUE_QRST, 2); in tegra210_emc_r21021_set_clock()
1600 emc_dbg(emc, STEPS, "Step 21\n"); in tegra210_emc_r21021_set_clock()
1603 ccfifo_writel(emc, emc_dbg | EMC_DBG_WRITE_MUX_ACTIVE, in tegra210_emc_r21021_set_clock()
1606 ccfifo_writel(emc, next->burst_regs[EMC_ZCAL_INTERVAL_INDEX], in tegra210_emc_r21021_set_clock()
1610 ccfifo_writel(emc, next->burst_regs[EMC_CFG_INDEX] & in tegra210_emc_r21021_set_clock()
1613 ccfifo_writel(emc, emc_dbg, EMC_DBG, 0); in tegra210_emc_r21021_set_clock()
1620 emc_dbg(emc, STEPS, "Step 22\n"); in tegra210_emc_r21021_set_clock()
1622 ccfifo_writel(emc, emc_cfg_pipe_clk, EMC_CFG_PIPE_CLK, 0); in tegra210_emc_r21021_set_clock()
1626 emc_writel(emc, in tegra210_emc_r21021_set_clock()
1627 next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & in tegra210_emc_r21021_set_clock()
1631 emc_writel(emc, in tegra210_emc_r21021_set_clock()
1632 next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & in tegra210_emc_r21021_set_clock()
1640 emc_dbg(emc, STEPS, "Step 23\n"); in tegra210_emc_r21021_set_clock()
1642 value = emc_readl(emc, EMC_CFG_DIG_DLL); in tegra210_emc_r21021_set_clock()
1648 (2 << EMC_CFG_DIG_DLL_CFG_DLL_MODE_SHIFT); in tegra210_emc_r21021_set_clock()
1649 emc_writel(emc, value, EMC_CFG_DIG_DLL); in tegra210_emc_r21021_set_clock()
1651 tegra210_emc_do_clock_change(emc, clksrc); in tegra210_emc_r21021_set_clock()
1662 emc_dbg(emc, STEPS, "Step 25\n"); in tegra210_emc_r21021_set_clock()
1664 if (next->rate > last->rate) { in tegra210_emc_r21021_set_clock()
1665 for (i = 0; i < next->num_up_down; i++) in tegra210_emc_r21021_set_clock()
1666 mc_writel(emc->mc, next->la_scale_regs[i], in tegra210_emc_r21021_set_clock()
1667 emc->offsets->la_scale[i]); in tegra210_emc_r21021_set_clock()
1669 tegra210_emc_timing_update(emc); in tegra210_emc_r21021_set_clock()
1676 emc_dbg(emc, STEPS, "Step 26\n"); in tegra210_emc_r21021_set_clock()
1679 tegra210_emc_set_shadow_bypass(emc, ACTIVE); in tegra210_emc_r21021_set_clock()
1680 emc_writel(emc, next->burst_regs[EMC_ZCAL_WAIT_CNT_INDEX], in tegra210_emc_r21021_set_clock()
1682 emc_writel(emc, next->burst_regs[EMC_ZCAL_INTERVAL_INDEX], in tegra210_emc_r21021_set_clock()
1684 tegra210_emc_set_shadow_bypass(emc, ASSEMBLY); in tegra210_emc_r21021_set_clock()
1689 udelay(2); in tegra210_emc_r21021_set_clock()
1691 tegra210_emc_set_shadow_bypass(emc, ACTIVE); in tegra210_emc_r21021_set_clock()
1693 emc_writel(emc, next->burst_regs[EMC_MRS_WAIT_CNT_INDEX], in tegra210_emc_r21021_set_clock()
1696 emc_writel(emc, next->burst_regs[EMC_ZCAL_WAIT_CNT_INDEX], in tegra210_emc_r21021_set_clock()
1698 tegra210_emc_set_shadow_bypass(emc, ASSEMBLY); in tegra210_emc_r21021_set_clock()
1705 emc_dbg(emc, STEPS, "Step 27\n"); in tegra210_emc_r21021_set_clock()
1707 tegra210_emc_set_shadow_bypass(emc, ACTIVE); in tegra210_emc_r21021_set_clock()
1708 emc_writel(emc, next->burst_regs[EMC_CFG_INDEX], EMC_CFG); in tegra210_emc_r21021_set_clock()
1709 tegra210_emc_set_shadow_bypass(emc, ASSEMBLY); in tegra210_emc_r21021_set_clock()
1710 emc_writel(emc, next->emc_fdpd_ctrl_cmd_no_ramp, in tegra210_emc_r21021_set_clock()
1712 emc_writel(emc, next->emc_sel_dpd_ctrl, EMC_SEL_DPD_CTRL); in tegra210_emc_r21021_set_clock()
1718 emc_dbg(emc, STEPS, "Step 28\n"); in tegra210_emc_r21021_set_clock()
1720 tegra210_emc_set_shadow_bypass(emc, ACTIVE); in tegra210_emc_r21021_set_clock()
1721 emc_writel(emc, in tegra210_emc_r21021_set_clock()
1722 next->burst_regs[EMC_PMACRO_AUTOCAL_CFG_COMMON_INDEX], in tegra210_emc_r21021_set_clock()
1724 tegra210_emc_set_shadow_bypass(emc, ASSEMBLY); in tegra210_emc_r21021_set_clock()
1730 emc_dbg(emc, STEPS, "Step 29\n"); in tegra210_emc_r21021_set_clock()
1732 emc_writel(emc, EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE0 | in tegra210_emc_r21021_set_clock()
1741 emc_writel(emc, EMC_PMACRO_TRAINING_CTRL_0_CH0_TRAINING_E_WRPTR, in tegra210_emc_r21021_set_clock()
1743 emc_writel(emc, EMC_PMACRO_TRAINING_CTRL_1_CH1_TRAINING_E_WRPTR, in tegra210_emc_r21021_set_clock()
1745 emc_writel(emc, 0, EMC_PMACRO_CFG_PM_GLOBAL_0); in tegra210_emc_r21021_set_clock()
1749 * Re-enable autocal. in tegra210_emc_r21021_set_clock()
1751 emc_dbg(emc, STEPS, "Step 30: Re-enable DLL and AUTOCAL\n"); in tegra210_emc_r21021_set_clock()
1753 if (next->burst_regs[EMC_CFG_DIG_DLL_INDEX] & EMC_CFG_DIG_DLL_CFG_DLL_EN) { in tegra210_emc_r21021_set_clock()
1754 value = emc_readl(emc, EMC_CFG_DIG_DLL); in tegra210_emc_r21021_set_clock()
1760 (2 << EMC_CFG_DIG_DLL_CFG_DLL_MODE_SHIFT); in tegra210_emc_r21021_set_clock()
1761 emc_writel(emc, value, EMC_CFG_DIG_DLL); in tegra210_emc_r21021_set_clock()
1762 tegra210_emc_timing_update(emc); in tegra210_emc_r21021_set_clock()
1765 emc_writel(emc, next->emc_auto_cal_config, EMC_AUTO_CAL_CONFIG); in tegra210_emc_r21021_set_clock()