1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (c) 2014-2020, NVIDIA CORPORATION.  All rights reserved.
4  */
5 
6 #include <linux/kernel.h>
7 #include <linux/io.h>
8 #include <linux/clk.h>
9 #include <linux/delay.h>
10 #include <linux/of.h>
11 
12 #include <soc/tegra/mc.h>
13 
14 #include "tegra210-emc.h"
15 #include "tegra210-mc.h"
16 
17 /*
18  * Enable flags for specifying verbosity.
19  */
20 #define INFO            (1 << 0)
21 #define STEPS           (1 << 1)
22 #define SUB_STEPS       (1 << 2)
23 #define PRELOCK         (1 << 3)
24 #define PRELOCK_STEPS   (1 << 4)
25 #define ACTIVE_EN       (1 << 5)
26 #define PRAMP_UP        (1 << 6)
27 #define PRAMP_DN        (1 << 7)
28 #define EMA_WRITES      (1 << 10)
29 #define EMA_UPDATES     (1 << 11)
30 #define PER_TRAIN       (1 << 16)
31 #define CC_PRINT        (1 << 17)
32 #define CCFIFO          (1 << 29)
33 #define REGS            (1 << 30)
34 #define REG_LISTS       (1 << 31)
35 
36 #define emc_dbg(emc, flags, ...) dev_dbg(emc->dev, __VA_ARGS__)
37 
38 #define DVFS_CLOCK_CHANGE_VERSION	21021
39 #define EMC_PRELOCK_VERSION		2101
40 
41 enum {
42 	DVFS_SEQUENCE = 1,
43 	WRITE_TRAINING_SEQUENCE = 2,
44 	PERIODIC_TRAINING_SEQUENCE = 3,
45 	DVFS_PT1 = 10,
46 	DVFS_UPDATE = 11,
47 	TRAINING_PT1 = 12,
48 	TRAINING_UPDATE = 13,
49 	PERIODIC_TRAINING_UPDATE = 14
50 };
51 
52 /*
53  * PTFV defines - basically just indexes into the per table PTFV array.
54  */
55 #define PTFV_DQSOSC_MOVAVG_C0D0U0_INDEX		0
56 #define PTFV_DQSOSC_MOVAVG_C0D0U1_INDEX		1
57 #define PTFV_DQSOSC_MOVAVG_C0D1U0_INDEX		2
58 #define PTFV_DQSOSC_MOVAVG_C0D1U1_INDEX		3
59 #define PTFV_DQSOSC_MOVAVG_C1D0U0_INDEX		4
60 #define PTFV_DQSOSC_MOVAVG_C1D0U1_INDEX		5
61 #define PTFV_DQSOSC_MOVAVG_C1D1U0_INDEX		6
62 #define PTFV_DQSOSC_MOVAVG_C1D1U1_INDEX		7
63 #define PTFV_DVFS_SAMPLES_INDEX			9
64 #define PTFV_MOVAVG_WEIGHT_INDEX		10
65 #define PTFV_CONFIG_CTRL_INDEX			11
66 
67 #define PTFV_CONFIG_CTRL_USE_PREVIOUS_EMA	(1 << 0)
68 
69 /*
70  * Do arithmetic in fixed point.
71  */
72 #define MOVAVG_PRECISION_FACTOR		100
73 
74 /*
75  * The division portion of the average operation.
76  */
77 #define __AVERAGE_PTFV(dev)						\
78 	({ next->ptfv_list[PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX] =	\
79 	   next->ptfv_list[PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX] /	\
80 	   next->ptfv_list[PTFV_DVFS_SAMPLES_INDEX]; })
81 
82 /*
83  * Convert val to fixed point and add it to the temporary average.
84  */
85 #define __INCREMENT_PTFV(dev, val)					\
86 	({ next->ptfv_list[PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX] +=	\
87 	   ((val) * MOVAVG_PRECISION_FACTOR); })
88 
89 /*
90  * Convert a moving average back to integral form and return the value.
91  */
92 #define __MOVAVG_AC(timing, dev)					\
93 	((timing)->ptfv_list[PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX] /	\
94 	 MOVAVG_PRECISION_FACTOR)
95 
96 /* Weighted update. */
97 #define __WEIGHTED_UPDATE_PTFV(dev, nval)				\
98 	do {								\
99 		int w = PTFV_MOVAVG_WEIGHT_INDEX;			\
100 		int dqs = PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX;		\
101 									\
102 		next->ptfv_list[dqs] =					\
103 			((nval * MOVAVG_PRECISION_FACTOR) +		\
104 			 (next->ptfv_list[dqs] *			\
105 			  next->ptfv_list[w])) /			\
106 			(next->ptfv_list[w] + 1);			\
107 									\
108 		emc_dbg(emc, EMA_UPDATES, "%s: (s=%lu) EMA: %u\n",	\
109 			__stringify(dev), nval, next->ptfv_list[dqs]);	\
110 	} while (0)
111 
112 /* Access a particular average. */
113 #define __MOVAVG(timing, dev)                      \
114 	((timing)->ptfv_list[PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX])
115 
update_clock_tree_delay(struct tegra210_emc * emc,int type)116 static u32 update_clock_tree_delay(struct tegra210_emc *emc, int type)
117 {
118 	bool periodic_training_update = type == PERIODIC_TRAINING_UPDATE;
119 	struct tegra210_emc_timing *last = emc->last;
120 	struct tegra210_emc_timing *next = emc->next;
121 	u32 last_timing_rate_mhz = last->rate / 1000;
122 	u32 next_timing_rate_mhz = next->rate / 1000;
123 	bool dvfs_update = type == DVFS_UPDATE;
124 	s32 tdel = 0, tmdel = 0, adel = 0;
125 	bool dvfs_pt1 = type == DVFS_PT1;
126 	unsigned long cval = 0;
127 	u32 temp[2][2], value;
128 	unsigned int i;
129 
130 	/*
131 	 * Dev0 MSB.
132 	 */
133 	if (dvfs_pt1 || periodic_training_update) {
134 		value = tegra210_emc_mrr_read(emc, 2, 19);
135 
136 		for (i = 0; i < emc->num_channels; i++) {
137 			temp[i][0] = (value & 0x00ff) << 8;
138 			temp[i][1] = (value & 0xff00) << 0;
139 			value >>= 16;
140 		}
141 
142 		/*
143 		 * Dev0 LSB.
144 		 */
145 		value = tegra210_emc_mrr_read(emc, 2, 18);
146 
147 		for (i = 0; i < emc->num_channels; i++) {
148 			temp[i][0] |= (value & 0x00ff) >> 0;
149 			temp[i][1] |= (value & 0xff00) >> 8;
150 			value >>= 16;
151 		}
152 	}
153 
154 	if (dvfs_pt1 || periodic_training_update) {
155 		cval = tegra210_emc_actual_osc_clocks(last->run_clocks);
156 		cval *= 1000000;
157 		cval /= last_timing_rate_mhz * 2 * temp[0][0];
158 	}
159 
160 	if (dvfs_pt1)
161 		__INCREMENT_PTFV(C0D0U0, cval);
162 	else if (dvfs_update)
163 		__AVERAGE_PTFV(C0D0U0);
164 	else if (periodic_training_update)
165 		__WEIGHTED_UPDATE_PTFV(C0D0U0, cval);
166 
167 	if (dvfs_update || periodic_training_update) {
168 		tdel = next->current_dram_clktree[C0D0U0] -
169 				__MOVAVG_AC(next, C0D0U0);
170 		tmdel = (tdel < 0) ? -1 * tdel : tdel;
171 		adel = tmdel;
172 
173 		if (tmdel * 128 * next_timing_rate_mhz / 1000000 >
174 		    next->tree_margin)
175 			next->current_dram_clktree[C0D0U0] =
176 				__MOVAVG_AC(next, C0D0U0);
177 	}
178 
179 	if (dvfs_pt1 || periodic_training_update) {
180 		cval = tegra210_emc_actual_osc_clocks(last->run_clocks);
181 		cval *= 1000000;
182 		cval /= last_timing_rate_mhz * 2 * temp[0][1];
183 	}
184 
185 	if (dvfs_pt1)
186 		__INCREMENT_PTFV(C0D0U1, cval);
187 	else if (dvfs_update)
188 		__AVERAGE_PTFV(C0D0U1);
189 	else if (periodic_training_update)
190 		__WEIGHTED_UPDATE_PTFV(C0D0U1, cval);
191 
192 	if (dvfs_update || periodic_training_update) {
193 		tdel = next->current_dram_clktree[C0D0U1] -
194 				__MOVAVG_AC(next, C0D0U1);
195 		tmdel = (tdel < 0) ? -1 * tdel : tdel;
196 
197 		if (tmdel > adel)
198 			adel = tmdel;
199 
200 		if (tmdel * 128 * next_timing_rate_mhz / 1000000 >
201 		    next->tree_margin)
202 			next->current_dram_clktree[C0D0U1] =
203 				__MOVAVG_AC(next, C0D0U1);
204 	}
205 
206 	if (emc->num_channels > 1) {
207 		if (dvfs_pt1 || periodic_training_update) {
208 			cval = tegra210_emc_actual_osc_clocks(last->run_clocks);
209 			cval *= 1000000;
210 			cval /= last_timing_rate_mhz * 2 * temp[1][0];
211 		}
212 
213 		if (dvfs_pt1)
214 			__INCREMENT_PTFV(C1D0U0, cval);
215 		else if (dvfs_update)
216 			__AVERAGE_PTFV(C1D0U0);
217 		else if (periodic_training_update)
218 			__WEIGHTED_UPDATE_PTFV(C1D0U0, cval);
219 
220 		if (dvfs_update || periodic_training_update) {
221 			tdel = next->current_dram_clktree[C1D0U0] -
222 					__MOVAVG_AC(next, C1D0U0);
223 			tmdel = (tdel < 0) ? -1 * tdel : tdel;
224 
225 			if (tmdel > adel)
226 				adel = tmdel;
227 
228 			if (tmdel * 128 * next_timing_rate_mhz / 1000000 >
229 			    next->tree_margin)
230 				next->current_dram_clktree[C1D0U0] =
231 					__MOVAVG_AC(next, C1D0U0);
232 		}
233 
234 		if (dvfs_pt1 || periodic_training_update) {
235 			cval = tegra210_emc_actual_osc_clocks(last->run_clocks);
236 			cval *= 1000000;
237 			cval /= last_timing_rate_mhz * 2 * temp[1][1];
238 		}
239 
240 		if (dvfs_pt1)
241 			__INCREMENT_PTFV(C1D0U1, cval);
242 		else if (dvfs_update)
243 			__AVERAGE_PTFV(C1D0U1);
244 		else if (periodic_training_update)
245 			__WEIGHTED_UPDATE_PTFV(C1D0U1, cval);
246 
247 		if (dvfs_update || periodic_training_update) {
248 			tdel = next->current_dram_clktree[C1D0U1] -
249 					__MOVAVG_AC(next, C1D0U1);
250 			tmdel = (tdel < 0) ? -1 * tdel : tdel;
251 
252 			if (tmdel > adel)
253 				adel = tmdel;
254 
255 			if (tmdel * 128 * next_timing_rate_mhz / 1000000 >
256 			    next->tree_margin)
257 				next->current_dram_clktree[C1D0U1] =
258 					__MOVAVG_AC(next, C1D0U1);
259 		}
260 	}
261 
262 	if (emc->num_devices < 2)
263 		goto done;
264 
265 	/*
266 	 * Dev1 MSB.
267 	 */
268 	if (dvfs_pt1 || periodic_training_update) {
269 		value = tegra210_emc_mrr_read(emc, 1, 19);
270 
271 		for (i = 0; i < emc->num_channels; i++) {
272 			temp[i][0] = (value & 0x00ff) << 8;
273 			temp[i][1] = (value & 0xff00) << 0;
274 			value >>= 16;
275 		}
276 
277 		/*
278 		 * Dev1 LSB.
279 		 */
280 		value = tegra210_emc_mrr_read(emc, 1, 18);
281 
282 		for (i = 0; i < emc->num_channels; i++) {
283 			temp[i][0] |= (value & 0x00ff) >> 0;
284 			temp[i][1] |= (value & 0xff00) >> 8;
285 			value >>= 16;
286 		}
287 	}
288 
289 	if (dvfs_pt1 || periodic_training_update) {
290 		cval = tegra210_emc_actual_osc_clocks(last->run_clocks);
291 		cval *= 1000000;
292 		cval /= last_timing_rate_mhz * 2 * temp[0][0];
293 	}
294 
295 	if (dvfs_pt1)
296 		__INCREMENT_PTFV(C0D1U0, cval);
297 	else if (dvfs_update)
298 		__AVERAGE_PTFV(C0D1U0);
299 	else if (periodic_training_update)
300 		__WEIGHTED_UPDATE_PTFV(C0D1U0, cval);
301 
302 	if (dvfs_update || periodic_training_update) {
303 		tdel = next->current_dram_clktree[C0D1U0] -
304 				__MOVAVG_AC(next, C0D1U0);
305 		tmdel = (tdel < 0) ? -1 * tdel : tdel;
306 
307 		if (tmdel > adel)
308 			adel = tmdel;
309 
310 		if (tmdel * 128 * next_timing_rate_mhz / 1000000 >
311 		    next->tree_margin)
312 			next->current_dram_clktree[C0D1U0] =
313 				__MOVAVG_AC(next, C0D1U0);
314 	}
315 
316 	if (dvfs_pt1 || periodic_training_update) {
317 		cval = tegra210_emc_actual_osc_clocks(last->run_clocks);
318 		cval *= 1000000;
319 		cval /= last_timing_rate_mhz * 2 * temp[0][1];
320 	}
321 
322 	if (dvfs_pt1)
323 		__INCREMENT_PTFV(C0D1U1, cval);
324 	else if (dvfs_update)
325 		__AVERAGE_PTFV(C0D1U1);
326 	else if (periodic_training_update)
327 		__WEIGHTED_UPDATE_PTFV(C0D1U1, cval);
328 
329 	if (dvfs_update || periodic_training_update) {
330 		tdel = next->current_dram_clktree[C0D1U1] -
331 				__MOVAVG_AC(next, C0D1U1);
332 		tmdel = (tdel < 0) ? -1 * tdel : tdel;
333 
334 		if (tmdel > adel)
335 			adel = tmdel;
336 
337 		if (tmdel * 128 * next_timing_rate_mhz / 1000000 >
338 		    next->tree_margin)
339 			next->current_dram_clktree[C0D1U1] =
340 				__MOVAVG_AC(next, C0D1U1);
341 	}
342 
343 	if (emc->num_channels > 1) {
344 		if (dvfs_pt1 || periodic_training_update) {
345 			cval = tegra210_emc_actual_osc_clocks(last->run_clocks);
346 			cval *= 1000000;
347 			cval /= last_timing_rate_mhz * 2 * temp[1][0];
348 		}
349 
350 		if (dvfs_pt1)
351 			__INCREMENT_PTFV(C1D1U0, cval);
352 		else if (dvfs_update)
353 			__AVERAGE_PTFV(C1D1U0);
354 		else if (periodic_training_update)
355 			__WEIGHTED_UPDATE_PTFV(C1D1U0, cval);
356 
357 		if (dvfs_update || periodic_training_update) {
358 			tdel = next->current_dram_clktree[C1D1U0] -
359 					__MOVAVG_AC(next, C1D1U0);
360 			tmdel = (tdel < 0) ? -1 * tdel : tdel;
361 
362 			if (tmdel > adel)
363 				adel = tmdel;
364 
365 			if (tmdel * 128 * next_timing_rate_mhz / 1000000 >
366 			    next->tree_margin)
367 				next->current_dram_clktree[C1D1U0] =
368 					__MOVAVG_AC(next, C1D1U0);
369 		}
370 
371 		if (dvfs_pt1 || periodic_training_update) {
372 			cval = tegra210_emc_actual_osc_clocks(last->run_clocks);
373 			cval *= 1000000;
374 			cval /= last_timing_rate_mhz * 2 * temp[1][1];
375 		}
376 
377 		if (dvfs_pt1)
378 			__INCREMENT_PTFV(C1D1U1, cval);
379 		else if (dvfs_update)
380 			__AVERAGE_PTFV(C1D1U1);
381 		else if (periodic_training_update)
382 			__WEIGHTED_UPDATE_PTFV(C1D1U1, cval);
383 
384 		if (dvfs_update || periodic_training_update) {
385 			tdel = next->current_dram_clktree[C1D1U1] -
386 					__MOVAVG_AC(next, C1D1U1);
387 			tmdel = (tdel < 0) ? -1 * tdel : tdel;
388 
389 			if (tmdel > adel)
390 				adel = tmdel;
391 
392 			if (tmdel * 128 * next_timing_rate_mhz / 1000000 >
393 			    next->tree_margin)
394 				next->current_dram_clktree[C1D1U1] =
395 					__MOVAVG_AC(next, C1D1U1);
396 		}
397 	}
398 
399 done:
400 	return adel;
401 }
402 
periodic_compensation_handler(struct tegra210_emc * emc,u32 type,struct tegra210_emc_timing * last,struct tegra210_emc_timing * next)403 static u32 periodic_compensation_handler(struct tegra210_emc *emc, u32 type,
404 					 struct tegra210_emc_timing *last,
405 					 struct tegra210_emc_timing *next)
406 {
407 #define __COPY_EMA(nt, lt, dev)						\
408 	({ __MOVAVG(nt, dev) = __MOVAVG(lt, dev) *			\
409 	   (nt)->ptfv_list[PTFV_DVFS_SAMPLES_INDEX]; })
410 
411 	u32 i, adel = 0, samples = next->ptfv_list[PTFV_DVFS_SAMPLES_INDEX];
412 	u32 delay;
413 
414 	delay = tegra210_emc_actual_osc_clocks(last->run_clocks);
415 	delay *= 1000;
416 	delay = 2 + (delay / last->rate);
417 
418 	if (!next->periodic_training)
419 		return 0;
420 
421 	if (type == DVFS_SEQUENCE) {
422 		if (last->periodic_training &&
423 		    (next->ptfv_list[PTFV_CONFIG_CTRL_INDEX] &
424 		     PTFV_CONFIG_CTRL_USE_PREVIOUS_EMA)) {
425 			/*
426 			 * If the previous frequency was using periodic
427 			 * calibration then we can reuse the previous
428 			 * frequencies EMA data.
429 			 */
430 			__COPY_EMA(next, last, C0D0U0);
431 			__COPY_EMA(next, last, C0D0U1);
432 			__COPY_EMA(next, last, C1D0U0);
433 			__COPY_EMA(next, last, C1D0U1);
434 			__COPY_EMA(next, last, C0D1U0);
435 			__COPY_EMA(next, last, C0D1U1);
436 			__COPY_EMA(next, last, C1D1U0);
437 			__COPY_EMA(next, last, C1D1U1);
438 		} else {
439 			/* Reset the EMA.*/
440 			__MOVAVG(next, C0D0U0) = 0;
441 			__MOVAVG(next, C0D0U1) = 0;
442 			__MOVAVG(next, C1D0U0) = 0;
443 			__MOVAVG(next, C1D0U1) = 0;
444 			__MOVAVG(next, C0D1U0) = 0;
445 			__MOVAVG(next, C0D1U1) = 0;
446 			__MOVAVG(next, C1D1U0) = 0;
447 			__MOVAVG(next, C1D1U1) = 0;
448 
449 			for (i = 0; i < samples; i++) {
450 				tegra210_emc_start_periodic_compensation(emc);
451 				udelay(delay);
452 
453 				/*
454 				 * Generate next sample of data.
455 				 */
456 				adel = update_clock_tree_delay(emc, DVFS_PT1);
457 			}
458 		}
459 
460 		/*
461 		 * Seems like it should be part of the
462 		 * 'if (last_timing->periodic_training)' conditional
463 		 * since is already done for the else clause.
464 		 */
465 		adel = update_clock_tree_delay(emc, DVFS_UPDATE);
466 	}
467 
468 	if (type == PERIODIC_TRAINING_SEQUENCE) {
469 		tegra210_emc_start_periodic_compensation(emc);
470 		udelay(delay);
471 
472 		adel = update_clock_tree_delay(emc, PERIODIC_TRAINING_UPDATE);
473 	}
474 
475 	return adel;
476 }
477 
tegra210_emc_r21021_periodic_compensation(struct tegra210_emc * emc)478 static u32 tegra210_emc_r21021_periodic_compensation(struct tegra210_emc *emc)
479 {
480 	u32 emc_cfg, emc_cfg_o, emc_cfg_update, del, value;
481 	static const u32 list[] = {
482 		EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0,
483 		EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1,
484 		EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2,
485 		EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3,
486 		EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0,
487 		EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1,
488 		EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2,
489 		EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3,
490 		EMC_DATA_BRLSHFT_0,
491 		EMC_DATA_BRLSHFT_1
492 	};
493 	struct tegra210_emc_timing *last = emc->last;
494 	unsigned int items = ARRAY_SIZE(list), i;
495 	unsigned long delay;
496 
497 	if (last->periodic_training) {
498 		emc_dbg(emc, PER_TRAIN, "Periodic training starting\n");
499 
500 		value = emc_readl(emc, EMC_DBG);
501 		emc_cfg_o = emc_readl(emc, EMC_CFG);
502 		emc_cfg = emc_cfg_o & ~(EMC_CFG_DYN_SELF_REF |
503 					EMC_CFG_DRAM_ACPD |
504 					EMC_CFG_DRAM_CLKSTOP_PD);
505 
506 
507 		/*
508 		 * 1. Power optimizations should be off.
509 		 */
510 		emc_writel(emc, emc_cfg, EMC_CFG);
511 
512 		/* Does emc_timing_update() for above changes. */
513 		tegra210_emc_dll_disable(emc);
514 
515 		for (i = 0; i < emc->num_channels; i++)
516 			tegra210_emc_wait_for_update(emc, i, EMC_EMC_STATUS,
517 						     EMC_EMC_STATUS_DRAM_IN_POWERDOWN_MASK,
518 						     0);
519 
520 		for (i = 0; i < emc->num_channels; i++)
521 			tegra210_emc_wait_for_update(emc, i, EMC_EMC_STATUS,
522 						     EMC_EMC_STATUS_DRAM_IN_SELF_REFRESH_MASK,
523 						     0);
524 
525 		emc_cfg_update = value = emc_readl(emc, EMC_CFG_UPDATE);
526 		value &= ~EMC_CFG_UPDATE_UPDATE_DLL_IN_UPDATE_MASK;
527 		value |= (2 << EMC_CFG_UPDATE_UPDATE_DLL_IN_UPDATE_SHIFT);
528 		emc_writel(emc, value, EMC_CFG_UPDATE);
529 
530 		/*
531 		 * 2. osc kick off - this assumes training and dvfs have set
532 		 *    correct MR23.
533 		 */
534 		tegra210_emc_start_periodic_compensation(emc);
535 
536 		/*
537 		 * 3. Let dram capture its clock tree delays.
538 		 */
539 		delay = tegra210_emc_actual_osc_clocks(last->run_clocks);
540 		delay *= 1000;
541 		delay /= last->rate + 1;
542 		udelay(delay);
543 
544 		/*
545 		 * 4. Check delta wrt previous values (save value if margin
546 		 *    exceeds what is set in table).
547 		 */
548 		del = periodic_compensation_handler(emc,
549 						    PERIODIC_TRAINING_SEQUENCE,
550 						    last, last);
551 
552 		/*
553 		 * 5. Apply compensation w.r.t. trained values (if clock tree
554 		 *    has drifted more than the set margin).
555 		 */
556 		if (last->tree_margin < ((del * 128 * (last->rate / 1000)) / 1000000)) {
557 			for (i = 0; i < items; i++) {
558 				value = tegra210_emc_compensate(last, list[i]);
559 				emc_dbg(emc, EMA_WRITES, "0x%08x <= 0x%08x\n",
560 					list[i], value);
561 				emc_writel(emc, value, list[i]);
562 			}
563 		}
564 
565 		emc_writel(emc, emc_cfg_o, EMC_CFG);
566 
567 		/*
568 		 * 6. Timing update actally applies the new trimmers.
569 		 */
570 		tegra210_emc_timing_update(emc);
571 
572 		/* 6.1. Restore the UPDATE_DLL_IN_UPDATE field. */
573 		emc_writel(emc, emc_cfg_update, EMC_CFG_UPDATE);
574 
575 		/* 6.2. Restore the DLL. */
576 		tegra210_emc_dll_enable(emc);
577 	}
578 
579 	return 0;
580 }
581 
582 /*
583  * Do the clock change sequence.
584  */
tegra210_emc_r21021_set_clock(struct tegra210_emc * emc,u32 clksrc)585 static void tegra210_emc_r21021_set_clock(struct tegra210_emc *emc, u32 clksrc)
586 {
587 	/* state variables */
588 	static bool fsp_for_next_freq;
589 	/* constant configuration parameters */
590 	const bool save_restore_clkstop_pd = true;
591 	const u32 zqcal_before_cc_cutoff = 2400;
592 	const bool cya_allow_ref_cc = false;
593 	const bool cya_issue_pc_ref = false;
594 	const bool opt_cc_short_zcal = true;
595 	const bool ref_b4_sref_en = false;
596 	const u32 tZQCAL_lpddr4 = 1000000;
597 	const bool opt_short_zcal = true;
598 	const bool opt_do_sw_qrst = true;
599 	const u32 opt_dvfs_mode = MAN_SR;
600 	/*
601 	 * This is the timing table for the source frequency. It does _not_
602 	 * necessarily correspond to the actual timing values in the EMC at the
603 	 * moment. If the boot BCT differs from the table then this can happen.
604 	 * However, we need it for accessing the dram_timings (which are not
605 	 * really registers) array for the current frequency.
606 	 */
607 	struct tegra210_emc_timing *fake, *last = emc->last, *next = emc->next;
608 	u32 tRTM, RP_war, R2P_war, TRPab_war, deltaTWATM, W2P_war, tRPST;
609 	u32 mr13_flip_fspwr, mr13_flip_fspop, ramp_up_wait, ramp_down_wait;
610 	u32 zq_wait_long, zq_latch_dvfs_wait_time, tZQCAL_lpddr4_fc_adj;
611 	u32 emc_auto_cal_config, auto_cal_en, emc_cfg, emc_sel_dpd_ctrl;
612 	u32 tFC_lpddr4 = 1000 * next->dram_timings[T_FC_LPDDR4];
613 	u32 bg_reg_mode_change, enable_bglp_reg, enable_bg_reg;
614 	bool opt_zcal_en_cc = false, is_lpddr3 = false;
615 	bool compensate_trimmer_applicable = false;
616 	u32 emc_dbg, emc_cfg_pipe_clk, emc_pin;
617 	u32 src_clk_period, dst_clk_period; /* in picoseconds */
618 	bool shared_zq_resistor = false;
619 	u32 value, dram_type;
620 	u32 opt_dll_mode = 0;
621 	unsigned long delay;
622 	unsigned int i;
623 
624 	emc_dbg(emc, INFO, "Running clock change.\n");
625 
626 	/* XXX fake == last */
627 	fake = tegra210_emc_find_timing(emc, last->rate * 1000UL);
628 	fsp_for_next_freq = !fsp_for_next_freq;
629 
630 	value = emc_readl(emc, EMC_FBIO_CFG5) & EMC_FBIO_CFG5_DRAM_TYPE_MASK;
631 	dram_type = value >> EMC_FBIO_CFG5_DRAM_TYPE_SHIFT;
632 
633 	if (last->burst_regs[EMC_ZCAL_WAIT_CNT_INDEX] & BIT(31))
634 		shared_zq_resistor = true;
635 
636 	if ((next->burst_regs[EMC_ZCAL_INTERVAL_INDEX] != 0 &&
637 	     last->burst_regs[EMC_ZCAL_INTERVAL_INDEX] == 0) ||
638 	    dram_type == DRAM_TYPE_LPDDR4)
639 		opt_zcal_en_cc = true;
640 
641 	if (dram_type == DRAM_TYPE_DDR3)
642 		opt_dll_mode = tegra210_emc_get_dll_state(next);
643 
644 	if ((next->burst_regs[EMC_FBIO_CFG5_INDEX] & BIT(25)) &&
645 	    (dram_type == DRAM_TYPE_LPDDR2))
646 		is_lpddr3 = true;
647 
648 	emc_readl(emc, EMC_CFG);
649 	emc_readl(emc, EMC_AUTO_CAL_CONFIG);
650 
651 	src_clk_period = 1000000000 / last->rate;
652 	dst_clk_period = 1000000000 / next->rate;
653 
654 	if (dst_clk_period <= zqcal_before_cc_cutoff)
655 		tZQCAL_lpddr4_fc_adj = tZQCAL_lpddr4 - tFC_lpddr4;
656 	else
657 		tZQCAL_lpddr4_fc_adj = tZQCAL_lpddr4;
658 
659 	tZQCAL_lpddr4_fc_adj /= dst_clk_period;
660 
661 	emc_dbg = emc_readl(emc, EMC_DBG);
662 	emc_pin = emc_readl(emc, EMC_PIN);
663 	emc_cfg_pipe_clk = emc_readl(emc, EMC_CFG_PIPE_CLK);
664 
665 	emc_cfg = next->burst_regs[EMC_CFG_INDEX];
666 	emc_cfg &= ~(EMC_CFG_DYN_SELF_REF | EMC_CFG_DRAM_ACPD |
667 		     EMC_CFG_DRAM_CLKSTOP_SR | EMC_CFG_DRAM_CLKSTOP_PD);
668 	emc_sel_dpd_ctrl = next->emc_sel_dpd_ctrl;
669 	emc_sel_dpd_ctrl &= ~(EMC_SEL_DPD_CTRL_CLK_SEL_DPD_EN |
670 			      EMC_SEL_DPD_CTRL_CA_SEL_DPD_EN |
671 			      EMC_SEL_DPD_CTRL_RESET_SEL_DPD_EN |
672 			      EMC_SEL_DPD_CTRL_ODT_SEL_DPD_EN |
673 			      EMC_SEL_DPD_CTRL_DATA_SEL_DPD_EN);
674 
675 	emc_dbg(emc, INFO, "Clock change version: %d\n",
676 		DVFS_CLOCK_CHANGE_VERSION);
677 	emc_dbg(emc, INFO, "DRAM type = %d\n", dram_type);
678 	emc_dbg(emc, INFO, "DRAM dev #: %u\n", emc->num_devices);
679 	emc_dbg(emc, INFO, "Next EMC clksrc: 0x%08x\n", clksrc);
680 	emc_dbg(emc, INFO, "DLL clksrc:      0x%08x\n", next->dll_clk_src);
681 	emc_dbg(emc, INFO, "last rate: %u, next rate %u\n", last->rate,
682 		next->rate);
683 	emc_dbg(emc, INFO, "last period: %u, next period: %u\n",
684 		src_clk_period, dst_clk_period);
685 	emc_dbg(emc, INFO, "  shared_zq_resistor: %d\n", !!shared_zq_resistor);
686 	emc_dbg(emc, INFO, "  num_channels: %u\n", emc->num_channels);
687 	emc_dbg(emc, INFO, "  opt_dll_mode: %d\n", opt_dll_mode);
688 
689 	/*
690 	 * Step 1:
691 	 *   Pre DVFS SW sequence.
692 	 */
693 	emc_dbg(emc, STEPS, "Step 1\n");
694 	emc_dbg(emc, STEPS, "Step 1.1: Disable DLL temporarily.\n");
695 
696 	value = emc_readl(emc, EMC_CFG_DIG_DLL);
697 	value &= ~EMC_CFG_DIG_DLL_CFG_DLL_EN;
698 	emc_writel(emc, value, EMC_CFG_DIG_DLL);
699 
700 	tegra210_emc_timing_update(emc);
701 
702 	for (i = 0; i < emc->num_channels; i++)
703 		tegra210_emc_wait_for_update(emc, i, EMC_CFG_DIG_DLL,
704 					     EMC_CFG_DIG_DLL_CFG_DLL_EN, 0);
705 
706 	emc_dbg(emc, STEPS, "Step 1.2: Disable AUTOCAL temporarily.\n");
707 
708 	emc_auto_cal_config = next->emc_auto_cal_config;
709 	auto_cal_en = emc_auto_cal_config & EMC_AUTO_CAL_CONFIG_AUTO_CAL_ENABLE;
710 	emc_auto_cal_config &= ~EMC_AUTO_CAL_CONFIG_AUTO_CAL_START;
711 	emc_auto_cal_config |= EMC_AUTO_CAL_CONFIG_AUTO_CAL_MEASURE_STALL;
712 	emc_auto_cal_config |= EMC_AUTO_CAL_CONFIG_AUTO_CAL_UPDATE_STALL;
713 	emc_auto_cal_config |= auto_cal_en;
714 	emc_writel(emc, emc_auto_cal_config, EMC_AUTO_CAL_CONFIG);
715 	emc_readl(emc, EMC_AUTO_CAL_CONFIG); /* Flush write. */
716 
717 	emc_dbg(emc, STEPS, "Step 1.3: Disable other power features.\n");
718 
719 	tegra210_emc_set_shadow_bypass(emc, ACTIVE);
720 	emc_writel(emc, emc_cfg, EMC_CFG);
721 	emc_writel(emc, emc_sel_dpd_ctrl, EMC_SEL_DPD_CTRL);
722 	tegra210_emc_set_shadow_bypass(emc, ASSEMBLY);
723 
724 	if (next->periodic_training) {
725 		tegra210_emc_reset_dram_clktree_values(next);
726 
727 		for (i = 0; i < emc->num_channels; i++)
728 			tegra210_emc_wait_for_update(emc, i, EMC_EMC_STATUS,
729 						     EMC_EMC_STATUS_DRAM_IN_POWERDOWN_MASK,
730 						     0);
731 
732 		for (i = 0; i < emc->num_channels; i++)
733 			tegra210_emc_wait_for_update(emc, i, EMC_EMC_STATUS,
734 						     EMC_EMC_STATUS_DRAM_IN_SELF_REFRESH_MASK,
735 						     0);
736 
737 		tegra210_emc_start_periodic_compensation(emc);
738 
739 		delay = 1000 * tegra210_emc_actual_osc_clocks(last->run_clocks);
740 		udelay((delay / last->rate) + 2);
741 
742 		value = periodic_compensation_handler(emc, DVFS_SEQUENCE, fake,
743 						      next);
744 		value = (value * 128 * next->rate / 1000) / 1000000;
745 
746 		if (next->periodic_training && value > next->tree_margin)
747 			compensate_trimmer_applicable = true;
748 	}
749 
750 	emc_writel(emc, EMC_INTSTATUS_CLKCHANGE_COMPLETE, EMC_INTSTATUS);
751 	tegra210_emc_set_shadow_bypass(emc, ACTIVE);
752 	emc_writel(emc, emc_cfg, EMC_CFG);
753 	emc_writel(emc, emc_sel_dpd_ctrl, EMC_SEL_DPD_CTRL);
754 	emc_writel(emc, emc_cfg_pipe_clk | EMC_CFG_PIPE_CLK_CLK_ALWAYS_ON,
755 		   EMC_CFG_PIPE_CLK);
756 	emc_writel(emc, next->emc_fdpd_ctrl_cmd_no_ramp &
757 			~EMC_FDPD_CTRL_CMD_NO_RAMP_CMD_DPD_NO_RAMP_ENABLE,
758 		   EMC_FDPD_CTRL_CMD_NO_RAMP);
759 
760 	bg_reg_mode_change =
761 		((next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] &
762 		  EMC_PMACRO_BG_BIAS_CTRL_0_BGLP_E_PWRD) ^
763 		 (last->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] &
764 		  EMC_PMACRO_BG_BIAS_CTRL_0_BGLP_E_PWRD)) ||
765 		((next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] &
766 		  EMC_PMACRO_BG_BIAS_CTRL_0_BG_E_PWRD) ^
767 		 (last->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] &
768 		  EMC_PMACRO_BG_BIAS_CTRL_0_BG_E_PWRD));
769 	enable_bglp_reg =
770 		(next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] &
771 		 EMC_PMACRO_BG_BIAS_CTRL_0_BGLP_E_PWRD) == 0;
772 	enable_bg_reg =
773 		(next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] &
774 		 EMC_PMACRO_BG_BIAS_CTRL_0_BG_E_PWRD) == 0;
775 
776 	if (bg_reg_mode_change) {
777 		if (enable_bg_reg)
778 			emc_writel(emc, last->burst_regs
779 				   [EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] &
780 				   ~EMC_PMACRO_BG_BIAS_CTRL_0_BG_E_PWRD,
781 				   EMC_PMACRO_BG_BIAS_CTRL_0);
782 
783 		if (enable_bglp_reg)
784 			emc_writel(emc, last->burst_regs
785 				   [EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] &
786 				   ~EMC_PMACRO_BG_BIAS_CTRL_0_BGLP_E_PWRD,
787 				   EMC_PMACRO_BG_BIAS_CTRL_0);
788 	}
789 
790 	/* Check if we need to turn on VREF generator. */
791 	if ((((last->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX] &
792 	       EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_E_IVREF) == 0) &&
793 	     ((next->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX] &
794 	       EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_E_IVREF) == 1)) ||
795 	    (((last->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX] &
796 	       EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQS_E_IVREF) == 0) &&
797 	     ((next->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX] &
798 	       EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQS_E_IVREF) != 0))) {
799 		u32 pad_tx_ctrl =
800 		    next->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX];
801 		u32 last_pad_tx_ctrl =
802 		    last->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX];
803 		u32 next_dq_e_ivref, next_dqs_e_ivref;
804 
805 		next_dqs_e_ivref = pad_tx_ctrl &
806 				   EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQS_E_IVREF;
807 		next_dq_e_ivref = pad_tx_ctrl &
808 				  EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_E_IVREF;
809 		value = (last_pad_tx_ctrl &
810 				~EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_E_IVREF &
811 				~EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQS_E_IVREF) |
812 			next_dq_e_ivref | next_dqs_e_ivref;
813 		emc_writel(emc, value, EMC_PMACRO_DATA_PAD_TX_CTRL);
814 		udelay(1);
815 	} else if (bg_reg_mode_change) {
816 		udelay(1);
817 	}
818 
819 	tegra210_emc_set_shadow_bypass(emc, ASSEMBLY);
820 
821 	/*
822 	 * Step 2:
823 	 *   Prelock the DLL.
824 	 */
825 	emc_dbg(emc, STEPS, "Step 2\n");
826 
827 	if (next->burst_regs[EMC_CFG_DIG_DLL_INDEX] &
828 	    EMC_CFG_DIG_DLL_CFG_DLL_EN) {
829 		emc_dbg(emc, INFO, "Prelock enabled for target frequency.\n");
830 		value = tegra210_emc_dll_prelock(emc, clksrc);
831 		emc_dbg(emc, INFO, "DLL out: 0x%03x\n", value);
832 	} else {
833 		emc_dbg(emc, INFO, "Disabling DLL for target frequency.\n");
834 		tegra210_emc_dll_disable(emc);
835 	}
836 
837 	/*
838 	 * Step 3:
839 	 *   Prepare autocal for the clock change.
840 	 */
841 	emc_dbg(emc, STEPS, "Step 3\n");
842 
843 	tegra210_emc_set_shadow_bypass(emc, ACTIVE);
844 	emc_writel(emc, next->emc_auto_cal_config2, EMC_AUTO_CAL_CONFIG2);
845 	emc_writel(emc, next->emc_auto_cal_config3, EMC_AUTO_CAL_CONFIG3);
846 	emc_writel(emc, next->emc_auto_cal_config4, EMC_AUTO_CAL_CONFIG4);
847 	emc_writel(emc, next->emc_auto_cal_config5, EMC_AUTO_CAL_CONFIG5);
848 	emc_writel(emc, next->emc_auto_cal_config6, EMC_AUTO_CAL_CONFIG6);
849 	emc_writel(emc, next->emc_auto_cal_config7, EMC_AUTO_CAL_CONFIG7);
850 	emc_writel(emc, next->emc_auto_cal_config8, EMC_AUTO_CAL_CONFIG8);
851 	tegra210_emc_set_shadow_bypass(emc, ASSEMBLY);
852 
853 	emc_auto_cal_config |= (EMC_AUTO_CAL_CONFIG_AUTO_CAL_COMPUTE_START |
854 				auto_cal_en);
855 	emc_writel(emc, emc_auto_cal_config, EMC_AUTO_CAL_CONFIG);
856 
857 	/*
858 	 * Step 4:
859 	 *   Update EMC_CFG. (??)
860 	 */
861 	emc_dbg(emc, STEPS, "Step 4\n");
862 
863 	if (src_clk_period > 50000 && dram_type == DRAM_TYPE_LPDDR4)
864 		ccfifo_writel(emc, 1, EMC_SELF_REF, 0);
865 	else
866 		emc_writel(emc, next->emc_cfg_2, EMC_CFG_2);
867 
868 	/*
869 	 * Step 5:
870 	 *   Prepare reference variables for ZQCAL regs.
871 	 */
872 	emc_dbg(emc, STEPS, "Step 5\n");
873 
874 	if (dram_type == DRAM_TYPE_LPDDR4)
875 		zq_wait_long = max((u32)1, div_o3(1000000, dst_clk_period));
876 	else if (dram_type == DRAM_TYPE_LPDDR2 || is_lpddr3)
877 		zq_wait_long = max(next->min_mrs_wait,
878 				   div_o3(360000, dst_clk_period)) + 4;
879 	else if (dram_type == DRAM_TYPE_DDR3)
880 		zq_wait_long = max((u32)256,
881 				   div_o3(320000, dst_clk_period) + 2);
882 	else
883 		zq_wait_long = 0;
884 
885 	/*
886 	 * Step 6:
887 	 *   Training code - removed.
888 	 */
889 	emc_dbg(emc, STEPS, "Step 6\n");
890 
891 	/*
892 	 * Step 7:
893 	 *   Program FSP reference registers and send MRWs to new FSPWR.
894 	 */
895 	emc_dbg(emc, STEPS, "Step 7\n");
896 	emc_dbg(emc, SUB_STEPS, "Step 7.1: Bug 200024907 - Patch RP R2P");
897 
898 	/* WAR 200024907 */
899 	if (dram_type == DRAM_TYPE_LPDDR4) {
900 		u32 nRTP = 16;
901 
902 		if (src_clk_period >= 1000000 / 1866) /* 535.91 ps */
903 			nRTP = 14;
904 
905 		if (src_clk_period >= 1000000 / 1600) /* 625.00 ps */
906 			nRTP = 12;
907 
908 		if (src_clk_period >= 1000000 / 1333) /* 750.19 ps */
909 			nRTP = 10;
910 
911 		if (src_clk_period >= 1000000 / 1066) /* 938.09 ps */
912 			nRTP = 8;
913 
914 		deltaTWATM = max_t(u32, div_o3(7500, src_clk_period), 8);
915 
916 		/*
917 		 * Originally there was a + .5 in the tRPST calculation.
918 		 * However since we can't do FP in the kernel and the tRTM
919 		 * computation was in a floating point ceiling function, adding
920 		 * one to tRTP should be ok. There is no other source of non
921 		 * integer values, so the result was always going to be
922 		 * something for the form: f_ceil(N + .5) = N + 1;
923 		 */
924 		tRPST = (last->emc_mrw & 0x80) >> 7;
925 		tRTM = fake->dram_timings[RL] + div_o3(3600, src_clk_period) +
926 			max_t(u32, div_o3(7500, src_clk_period), 8) + tRPST +
927 			1 + nRTP;
928 
929 		emc_dbg(emc, INFO, "tRTM = %u, EMC_RP = %u\n", tRTM,
930 			next->burst_regs[EMC_RP_INDEX]);
931 
932 		if (last->burst_regs[EMC_RP_INDEX] < tRTM) {
933 			if (tRTM > (last->burst_regs[EMC_R2P_INDEX] +
934 				    last->burst_regs[EMC_RP_INDEX])) {
935 				R2P_war = tRTM - last->burst_regs[EMC_RP_INDEX];
936 				RP_war = last->burst_regs[EMC_RP_INDEX];
937 				TRPab_war = last->burst_regs[EMC_TRPAB_INDEX];
938 
939 				if (R2P_war > 63) {
940 					RP_war = R2P_war +
941 						 last->burst_regs[EMC_RP_INDEX] - 63;
942 
943 					if (TRPab_war < RP_war)
944 						TRPab_war = RP_war;
945 
946 					R2P_war = 63;
947 				}
948 			} else {
949 				R2P_war = last->burst_regs[EMC_R2P_INDEX];
950 				RP_war = last->burst_regs[EMC_RP_INDEX];
951 				TRPab_war = last->burst_regs[EMC_TRPAB_INDEX];
952 			}
953 
954 			if (RP_war < deltaTWATM) {
955 				W2P_war = last->burst_regs[EMC_W2P_INDEX]
956 					  + deltaTWATM - RP_war;
957 				if (W2P_war > 63) {
958 					RP_war = RP_war + W2P_war - 63;
959 					if (TRPab_war < RP_war)
960 						TRPab_war = RP_war;
961 					W2P_war = 63;
962 				}
963 			} else {
964 				W2P_war = last->burst_regs[
965 					  EMC_W2P_INDEX];
966 			}
967 
968 			if ((last->burst_regs[EMC_W2P_INDEX] ^ W2P_war) ||
969 			    (last->burst_regs[EMC_R2P_INDEX] ^ R2P_war) ||
970 			    (last->burst_regs[EMC_RP_INDEX] ^ RP_war) ||
971 			    (last->burst_regs[EMC_TRPAB_INDEX] ^ TRPab_war)) {
972 				emc_writel(emc, RP_war, EMC_RP);
973 				emc_writel(emc, R2P_war, EMC_R2P);
974 				emc_writel(emc, W2P_war, EMC_W2P);
975 				emc_writel(emc, TRPab_war, EMC_TRPAB);
976 			}
977 
978 			tegra210_emc_timing_update(emc);
979 		} else {
980 			emc_dbg(emc, INFO, "Skipped WAR\n");
981 		}
982 	}
983 
984 	if (!fsp_for_next_freq) {
985 		mr13_flip_fspwr = (next->emc_mrw3 & 0xffffff3f) | 0x80;
986 		mr13_flip_fspop = (next->emc_mrw3 & 0xffffff3f) | 0x00;
987 	} else {
988 		mr13_flip_fspwr = (next->emc_mrw3 & 0xffffff3f) | 0x40;
989 		mr13_flip_fspop = (next->emc_mrw3 & 0xffffff3f) | 0xc0;
990 	}
991 
992 	if (dram_type == DRAM_TYPE_LPDDR4) {
993 		emc_writel(emc, mr13_flip_fspwr, EMC_MRW3);
994 		emc_writel(emc, next->emc_mrw, EMC_MRW);
995 		emc_writel(emc, next->emc_mrw2, EMC_MRW2);
996 	}
997 
998 	/*
999 	 * Step 8:
1000 	 *   Program the shadow registers.
1001 	 */
1002 	emc_dbg(emc, STEPS, "Step 8\n");
1003 	emc_dbg(emc, SUB_STEPS, "Writing burst_regs\n");
1004 
1005 	for (i = 0; i < next->num_burst; i++) {
1006 		const u16 *offsets = emc->offsets->burst;
1007 		u16 offset;
1008 
1009 		if (!offsets[i])
1010 			continue;
1011 
1012 		value = next->burst_regs[i];
1013 		offset = offsets[i];
1014 
1015 		if (dram_type != DRAM_TYPE_LPDDR4 &&
1016 		    (offset == EMC_MRW6 || offset == EMC_MRW7 ||
1017 		     offset == EMC_MRW8 || offset == EMC_MRW9 ||
1018 		     offset == EMC_MRW10 || offset == EMC_MRW11 ||
1019 		     offset == EMC_MRW12 || offset == EMC_MRW13 ||
1020 		     offset == EMC_MRW14 || offset == EMC_MRW15 ||
1021 		     offset == EMC_TRAINING_CTRL))
1022 			continue;
1023 
1024 		/* Pain... And suffering. */
1025 		if (offset == EMC_CFG) {
1026 			value &= ~EMC_CFG_DRAM_ACPD;
1027 			value &= ~EMC_CFG_DYN_SELF_REF;
1028 
1029 			if (dram_type == DRAM_TYPE_LPDDR4) {
1030 				value &= ~EMC_CFG_DRAM_CLKSTOP_SR;
1031 				value &= ~EMC_CFG_DRAM_CLKSTOP_PD;
1032 			}
1033 		} else if (offset == EMC_MRS_WAIT_CNT &&
1034 			   dram_type == DRAM_TYPE_LPDDR2 &&
1035 			   opt_zcal_en_cc && !opt_cc_short_zcal &&
1036 			   opt_short_zcal) {
1037 			value = (value & ~(EMC_MRS_WAIT_CNT_SHORT_WAIT_MASK <<
1038 					   EMC_MRS_WAIT_CNT_SHORT_WAIT_SHIFT)) |
1039 				((zq_wait_long & EMC_MRS_WAIT_CNT_SHORT_WAIT_MASK) <<
1040 						 EMC_MRS_WAIT_CNT_SHORT_WAIT_SHIFT);
1041 		} else if (offset == EMC_ZCAL_WAIT_CNT &&
1042 			   dram_type == DRAM_TYPE_DDR3 && opt_zcal_en_cc &&
1043 			   !opt_cc_short_zcal && opt_short_zcal) {
1044 			value = (value & ~(EMC_ZCAL_WAIT_CNT_ZCAL_WAIT_CNT_MASK <<
1045 					   EMC_ZCAL_WAIT_CNT_ZCAL_WAIT_CNT_SHIFT)) |
1046 				((zq_wait_long & EMC_ZCAL_WAIT_CNT_ZCAL_WAIT_CNT_MASK) <<
1047 						 EMC_MRS_WAIT_CNT_SHORT_WAIT_SHIFT);
1048 		} else if (offset == EMC_ZCAL_INTERVAL && opt_zcal_en_cc) {
1049 			value = 0; /* EMC_ZCAL_INTERVAL reset value. */
1050 		} else if (offset == EMC_PMACRO_AUTOCAL_CFG_COMMON) {
1051 			value |= EMC_PMACRO_AUTOCAL_CFG_COMMON_E_CAL_BYPASS_DVFS;
1052 		} else if (offset == EMC_PMACRO_DATA_PAD_TX_CTRL) {
1053 			value &= ~(EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSP_TX_E_DCC |
1054 				   EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSN_TX_E_DCC |
1055 				   EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_TX_E_DCC |
1056 				   EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_CMD_TX_E_DCC);
1057 		} else if (offset == EMC_PMACRO_CMD_PAD_TX_CTRL) {
1058 			value |= EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_DRVFORCEON;
1059 			value &= ~(EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSP_TX_E_DCC |
1060 				   EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSN_TX_E_DCC |
1061 				   EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_E_DCC |
1062 				   EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_CMD_TX_E_DCC);
1063 		} else if (offset == EMC_PMACRO_BRICK_CTRL_RFU1) {
1064 			value &= 0xf800f800;
1065 		} else if (offset == EMC_PMACRO_COMMON_PAD_TX_CTRL) {
1066 			value &= 0xfffffff0;
1067 		}
1068 
1069 		emc_writel(emc, value, offset);
1070 	}
1071 
1072 	/* SW addition: do EMC refresh adjustment here. */
1073 	tegra210_emc_adjust_timing(emc, next);
1074 
1075 	if (dram_type == DRAM_TYPE_LPDDR4) {
1076 		value = (23 << EMC_MRW_MRW_MA_SHIFT) |
1077 			(next->run_clocks & EMC_MRW_MRW_OP_MASK);
1078 		emc_writel(emc, value, EMC_MRW);
1079 	}
1080 
1081 	/* Per channel burst registers. */
1082 	emc_dbg(emc, SUB_STEPS, "Writing burst_regs_per_ch\n");
1083 
1084 	for (i = 0; i < next->num_burst_per_ch; i++) {
1085 		const struct tegra210_emc_per_channel_regs *burst =
1086 				emc->offsets->burst_per_channel;
1087 
1088 		if (!burst[i].offset)
1089 			continue;
1090 
1091 		if (dram_type != DRAM_TYPE_LPDDR4 &&
1092 		    (burst[i].offset == EMC_MRW6 ||
1093 		     burst[i].offset == EMC_MRW7 ||
1094 		     burst[i].offset == EMC_MRW8 ||
1095 		     burst[i].offset == EMC_MRW9 ||
1096 		     burst[i].offset == EMC_MRW10 ||
1097 		     burst[i].offset == EMC_MRW11 ||
1098 		     burst[i].offset == EMC_MRW12 ||
1099 		     burst[i].offset == EMC_MRW13 ||
1100 		     burst[i].offset == EMC_MRW14 ||
1101 		     burst[i].offset == EMC_MRW15))
1102 			continue;
1103 
1104 		/* Filter out second channel if not in DUAL_CHANNEL mode. */
1105 		if (emc->num_channels < 2 && burst[i].bank >= 1)
1106 			continue;
1107 
1108 		emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i,
1109 			next->burst_reg_per_ch[i], burst[i].offset);
1110 		emc_channel_writel(emc, burst[i].bank,
1111 				   next->burst_reg_per_ch[i],
1112 				   burst[i].offset);
1113 	}
1114 
1115 	/* Vref regs. */
1116 	emc_dbg(emc, SUB_STEPS, "Writing vref_regs\n");
1117 
1118 	for (i = 0; i < next->vref_num; i++) {
1119 		const struct tegra210_emc_per_channel_regs *vref =
1120 					emc->offsets->vref_per_channel;
1121 
1122 		if (!vref[i].offset)
1123 			continue;
1124 
1125 		if (emc->num_channels < 2 && vref[i].bank >= 1)
1126 			continue;
1127 
1128 		emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i,
1129 			next->vref_perch_regs[i], vref[i].offset);
1130 		emc_channel_writel(emc, vref[i].bank, next->vref_perch_regs[i],
1131 				   vref[i].offset);
1132 	}
1133 
1134 	/* Trimmers. */
1135 	emc_dbg(emc, SUB_STEPS, "Writing trim_regs\n");
1136 
1137 	for (i = 0; i < next->num_trim; i++) {
1138 		const u16 *offsets = emc->offsets->trim;
1139 
1140 		if (!offsets[i])
1141 			continue;
1142 
1143 		if (compensate_trimmer_applicable &&
1144 		    (offsets[i] == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0 ||
1145 		     offsets[i] == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1 ||
1146 		     offsets[i] == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2 ||
1147 		     offsets[i] == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3 ||
1148 		     offsets[i] == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0 ||
1149 		     offsets[i] == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1 ||
1150 		     offsets[i] == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2 ||
1151 		     offsets[i] == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3 ||
1152 		     offsets[i] == EMC_DATA_BRLSHFT_0 ||
1153 		     offsets[i] == EMC_DATA_BRLSHFT_1)) {
1154 			value = tegra210_emc_compensate(next, offsets[i]);
1155 			emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i,
1156 				value, offsets[i]);
1157 			emc_dbg(emc, EMA_WRITES, "0x%08x <= 0x%08x\n",
1158 				(u32)(u64)offsets[i], value);
1159 			emc_writel(emc, value, offsets[i]);
1160 		} else {
1161 			emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i,
1162 				next->trim_regs[i], offsets[i]);
1163 			emc_writel(emc, next->trim_regs[i], offsets[i]);
1164 		}
1165 	}
1166 
1167 	/* Per channel trimmers. */
1168 	emc_dbg(emc, SUB_STEPS, "Writing trim_regs_per_ch\n");
1169 
1170 	for (i = 0; i < next->num_trim_per_ch; i++) {
1171 		const struct tegra210_emc_per_channel_regs *trim =
1172 				&emc->offsets->trim_per_channel[0];
1173 		unsigned int offset;
1174 
1175 		if (!trim[i].offset)
1176 			continue;
1177 
1178 		if (emc->num_channels < 2 && trim[i].bank >= 1)
1179 			continue;
1180 
1181 		offset = trim[i].offset;
1182 
1183 		if (compensate_trimmer_applicable &&
1184 		    (offset == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0 ||
1185 		     offset == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1 ||
1186 		     offset == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2 ||
1187 		     offset == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3 ||
1188 		     offset == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0 ||
1189 		     offset == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1 ||
1190 		     offset == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2 ||
1191 		     offset == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3 ||
1192 		     offset == EMC_DATA_BRLSHFT_0 ||
1193 		     offset == EMC_DATA_BRLSHFT_1)) {
1194 			value = tegra210_emc_compensate(next, offset);
1195 			emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i,
1196 				value, offset);
1197 			emc_dbg(emc, EMA_WRITES, "0x%08x <= 0x%08x\n", offset,
1198 				value);
1199 			emc_channel_writel(emc, trim[i].bank, value, offset);
1200 		} else {
1201 			emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i,
1202 				next->trim_perch_regs[i], offset);
1203 			emc_channel_writel(emc, trim[i].bank,
1204 					   next->trim_perch_regs[i], offset);
1205 		}
1206 	}
1207 
1208 	emc_dbg(emc, SUB_STEPS, "Writing burst_mc_regs\n");
1209 
1210 	for (i = 0; i < next->num_mc_regs; i++) {
1211 		const u16 *offsets = emc->offsets->burst_mc;
1212 		u32 *values = next->burst_mc_regs;
1213 
1214 		emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i,
1215 			values[i], offsets[i]);
1216 		mc_writel(emc->mc, values[i], offsets[i]);
1217 	}
1218 
1219 	/* Registers to be programmed on the faster clock. */
1220 	if (next->rate < last->rate) {
1221 		const u16 *la = emc->offsets->la_scale;
1222 
1223 		emc_dbg(emc, SUB_STEPS, "Writing la_scale_regs\n");
1224 
1225 		for (i = 0; i < next->num_up_down; i++) {
1226 			emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i,
1227 				next->la_scale_regs[i], la[i]);
1228 			mc_writel(emc->mc, next->la_scale_regs[i], la[i]);
1229 		}
1230 	}
1231 
1232 	/* Flush all the burst register writes. */
1233 	mc_readl(emc->mc, MC_EMEM_ADR_CFG);
1234 
1235 	/*
1236 	 * Step 9:
1237 	 *   LPDDR4 section A.
1238 	 */
1239 	emc_dbg(emc, STEPS, "Step 9\n");
1240 
1241 	value = next->burst_regs[EMC_ZCAL_WAIT_CNT_INDEX];
1242 	value &= ~EMC_ZCAL_WAIT_CNT_ZCAL_WAIT_CNT_MASK;
1243 
1244 	if (dram_type == DRAM_TYPE_LPDDR4) {
1245 		emc_writel(emc, 0, EMC_ZCAL_INTERVAL);
1246 		emc_writel(emc, value, EMC_ZCAL_WAIT_CNT);
1247 
1248 		value = emc_dbg | (EMC_DBG_WRITE_MUX_ACTIVE |
1249 				   EMC_DBG_WRITE_ACTIVE_ONLY);
1250 
1251 		emc_writel(emc, value, EMC_DBG);
1252 		emc_writel(emc, 0, EMC_ZCAL_INTERVAL);
1253 		emc_writel(emc, emc_dbg, EMC_DBG);
1254 	}
1255 
1256 	/*
1257 	 * Step 10:
1258 	 *   LPDDR4 and DDR3 common section.
1259 	 */
1260 	emc_dbg(emc, STEPS, "Step 10\n");
1261 
1262 	if (opt_dvfs_mode == MAN_SR || dram_type == DRAM_TYPE_LPDDR4) {
1263 		if (dram_type == DRAM_TYPE_LPDDR4)
1264 			ccfifo_writel(emc, 0x101, EMC_SELF_REF, 0);
1265 		else
1266 			ccfifo_writel(emc, 0x1, EMC_SELF_REF, 0);
1267 
1268 		if (dram_type == DRAM_TYPE_LPDDR4 &&
1269 		    dst_clk_period <= zqcal_before_cc_cutoff) {
1270 			ccfifo_writel(emc, mr13_flip_fspwr ^ 0x40, EMC_MRW3, 0);
1271 			ccfifo_writel(emc, (next->burst_regs[EMC_MRW6_INDEX] &
1272 						0xFFFF3F3F) |
1273 					   (last->burst_regs[EMC_MRW6_INDEX] &
1274 						0x0000C0C0), EMC_MRW6, 0);
1275 			ccfifo_writel(emc, (next->burst_regs[EMC_MRW14_INDEX] &
1276 						0xFFFF0707) |
1277 					   (last->burst_regs[EMC_MRW14_INDEX] &
1278 						0x00003838), EMC_MRW14, 0);
1279 
1280 			if (emc->num_devices > 1) {
1281 				ccfifo_writel(emc,
1282 				      (next->burst_regs[EMC_MRW7_INDEX] &
1283 				       0xFFFF3F3F) |
1284 				      (last->burst_regs[EMC_MRW7_INDEX] &
1285 				       0x0000C0C0), EMC_MRW7, 0);
1286 				ccfifo_writel(emc,
1287 				     (next->burst_regs[EMC_MRW15_INDEX] &
1288 				      0xFFFF0707) |
1289 				     (last->burst_regs[EMC_MRW15_INDEX] &
1290 				      0x00003838), EMC_MRW15, 0);
1291 			}
1292 
1293 			if (opt_zcal_en_cc) {
1294 				if (emc->num_devices < 2)
1295 					ccfifo_writel(emc,
1296 						2UL << EMC_ZQ_CAL_DEV_SEL_SHIFT
1297 						| EMC_ZQ_CAL_ZQ_CAL_CMD,
1298 						EMC_ZQ_CAL, 0);
1299 				else if (shared_zq_resistor)
1300 					ccfifo_writel(emc,
1301 						2UL << EMC_ZQ_CAL_DEV_SEL_SHIFT
1302 						| EMC_ZQ_CAL_ZQ_CAL_CMD,
1303 						EMC_ZQ_CAL, 0);
1304 				else
1305 					ccfifo_writel(emc,
1306 						      EMC_ZQ_CAL_ZQ_CAL_CMD,
1307 						      EMC_ZQ_CAL, 0);
1308 			}
1309 		}
1310 	}
1311 
1312 	if (dram_type == DRAM_TYPE_LPDDR4) {
1313 		value = (1000 * fake->dram_timings[T_RP]) / src_clk_period;
1314 		ccfifo_writel(emc, mr13_flip_fspop | 0x8, EMC_MRW3, value);
1315 		ccfifo_writel(emc, 0, 0, tFC_lpddr4 / src_clk_period);
1316 	}
1317 
1318 	if (dram_type == DRAM_TYPE_LPDDR4 || opt_dvfs_mode != MAN_SR) {
1319 		delay = 30;
1320 
1321 		if (cya_allow_ref_cc) {
1322 			delay += (1000 * fake->dram_timings[T_RP]) /
1323 					src_clk_period;
1324 			delay += 4000 * fake->dram_timings[T_RFC];
1325 		}
1326 
1327 		ccfifo_writel(emc, emc_pin & ~(EMC_PIN_PIN_CKE_PER_DEV |
1328 					       EMC_PIN_PIN_CKEB |
1329 					       EMC_PIN_PIN_CKE),
1330 			      EMC_PIN, delay);
1331 	}
1332 
1333 	/* calculate reference delay multiplier */
1334 	value = 1;
1335 
1336 	if (ref_b4_sref_en)
1337 		value++;
1338 
1339 	if (cya_allow_ref_cc)
1340 		value++;
1341 
1342 	if (cya_issue_pc_ref)
1343 		value++;
1344 
1345 	if (dram_type != DRAM_TYPE_LPDDR4) {
1346 		delay = ((1000 * fake->dram_timings[T_RP] / src_clk_period) +
1347 			 (1000 * fake->dram_timings[T_RFC] / src_clk_period));
1348 		delay = value * delay + 20;
1349 	} else {
1350 		delay = 0;
1351 	}
1352 
1353 	/*
1354 	 * Step 11:
1355 	 *   Ramp down.
1356 	 */
1357 	emc_dbg(emc, STEPS, "Step 11\n");
1358 
1359 	ccfifo_writel(emc, 0x0, EMC_CFG_SYNC, delay);
1360 
1361 	value = emc_dbg | EMC_DBG_WRITE_MUX_ACTIVE | EMC_DBG_WRITE_ACTIVE_ONLY;
1362 	ccfifo_writel(emc, value, EMC_DBG, 0);
1363 
1364 	ramp_down_wait = tegra210_emc_dvfs_power_ramp_down(emc, src_clk_period,
1365 							   0);
1366 
1367 	/*
1368 	 * Step 12:
1369 	 *   And finally - trigger the clock change.
1370 	 */
1371 	emc_dbg(emc, STEPS, "Step 12\n");
1372 
1373 	ccfifo_writel(emc, 1, EMC_STALL_THEN_EXE_AFTER_CLKCHANGE, 0);
1374 	value &= ~EMC_DBG_WRITE_ACTIVE_ONLY;
1375 	ccfifo_writel(emc, value, EMC_DBG, 0);
1376 
1377 	/*
1378 	 * Step 13:
1379 	 *   Ramp up.
1380 	 */
1381 	emc_dbg(emc, STEPS, "Step 13\n");
1382 
1383 	ramp_up_wait = tegra210_emc_dvfs_power_ramp_up(emc, dst_clk_period, 0);
1384 	ccfifo_writel(emc, emc_dbg, EMC_DBG, 0);
1385 
1386 	/*
1387 	 * Step 14:
1388 	 *   Bringup CKE pins.
1389 	 */
1390 	emc_dbg(emc, STEPS, "Step 14\n");
1391 
1392 	if (dram_type == DRAM_TYPE_LPDDR4) {
1393 		value = emc_pin | EMC_PIN_PIN_CKE;
1394 
1395 		if (emc->num_devices <= 1)
1396 			value &= ~(EMC_PIN_PIN_CKEB | EMC_PIN_PIN_CKE_PER_DEV);
1397 		else
1398 			value |= EMC_PIN_PIN_CKEB | EMC_PIN_PIN_CKE_PER_DEV;
1399 
1400 		ccfifo_writel(emc, value, EMC_PIN, 0);
1401 	}
1402 
1403 	/*
1404 	 * Step 15: (two step 15s ??)
1405 	 *   Calculate zqlatch wait time; has dependency on ramping times.
1406 	 */
1407 	emc_dbg(emc, STEPS, "Step 15\n");
1408 
1409 	if (dst_clk_period <= zqcal_before_cc_cutoff) {
1410 		s32 t = (s32)(ramp_up_wait + ramp_down_wait) /
1411 			(s32)dst_clk_period;
1412 		zq_latch_dvfs_wait_time = (s32)tZQCAL_lpddr4_fc_adj - t;
1413 	} else {
1414 		zq_latch_dvfs_wait_time = tZQCAL_lpddr4_fc_adj -
1415 			div_o3(1000 * next->dram_timings[T_PDEX],
1416 			       dst_clk_period);
1417 	}
1418 
1419 	emc_dbg(emc, INFO, "tZQCAL_lpddr4_fc_adj = %u\n", tZQCAL_lpddr4_fc_adj);
1420 	emc_dbg(emc, INFO, "dst_clk_period = %u\n",
1421 		dst_clk_period);
1422 	emc_dbg(emc, INFO, "next->dram_timings[T_PDEX] = %u\n",
1423 		next->dram_timings[T_PDEX]);
1424 	emc_dbg(emc, INFO, "zq_latch_dvfs_wait_time = %d\n",
1425 		max_t(s32, 0, zq_latch_dvfs_wait_time));
1426 
1427 	if (dram_type == DRAM_TYPE_LPDDR4 && opt_zcal_en_cc) {
1428 		delay = div_o3(1000 * next->dram_timings[T_PDEX],
1429 			       dst_clk_period);
1430 
1431 		if (emc->num_devices < 2) {
1432 			if (dst_clk_period > zqcal_before_cc_cutoff)
1433 				ccfifo_writel(emc,
1434 					      2UL << EMC_ZQ_CAL_DEV_SEL_SHIFT |
1435 					      EMC_ZQ_CAL_ZQ_CAL_CMD, EMC_ZQ_CAL,
1436 					      delay);
1437 
1438 			value = (mr13_flip_fspop & 0xfffffff7) | 0x0c000000;
1439 			ccfifo_writel(emc, value, EMC_MRW3, delay);
1440 			ccfifo_writel(emc, 0, EMC_SELF_REF, 0);
1441 			ccfifo_writel(emc, 0, EMC_REF, 0);
1442 			ccfifo_writel(emc, 2UL << EMC_ZQ_CAL_DEV_SEL_SHIFT |
1443 				      EMC_ZQ_CAL_ZQ_LATCH_CMD,
1444 				      EMC_ZQ_CAL,
1445 				      max_t(s32, 0, zq_latch_dvfs_wait_time));
1446 		} else if (shared_zq_resistor) {
1447 			if (dst_clk_period > zqcal_before_cc_cutoff)
1448 				ccfifo_writel(emc,
1449 					      2UL << EMC_ZQ_CAL_DEV_SEL_SHIFT |
1450 					      EMC_ZQ_CAL_ZQ_CAL_CMD, EMC_ZQ_CAL,
1451 					      delay);
1452 
1453 			ccfifo_writel(emc, 2UL << EMC_ZQ_CAL_DEV_SEL_SHIFT |
1454 				      EMC_ZQ_CAL_ZQ_LATCH_CMD, EMC_ZQ_CAL,
1455 				      max_t(s32, 0, zq_latch_dvfs_wait_time) +
1456 					delay);
1457 			ccfifo_writel(emc, 1UL << EMC_ZQ_CAL_DEV_SEL_SHIFT |
1458 				      EMC_ZQ_CAL_ZQ_LATCH_CMD,
1459 				      EMC_ZQ_CAL, 0);
1460 
1461 			value = (mr13_flip_fspop & 0xfffffff7) | 0x0c000000;
1462 			ccfifo_writel(emc, value, EMC_MRW3, 0);
1463 			ccfifo_writel(emc, 0, EMC_SELF_REF, 0);
1464 			ccfifo_writel(emc, 0, EMC_REF, 0);
1465 
1466 			ccfifo_writel(emc, 1UL << EMC_ZQ_CAL_DEV_SEL_SHIFT |
1467 				      EMC_ZQ_CAL_ZQ_LATCH_CMD, EMC_ZQ_CAL,
1468 				      tZQCAL_lpddr4 / dst_clk_period);
1469 		} else {
1470 			if (dst_clk_period > zqcal_before_cc_cutoff)
1471 				ccfifo_writel(emc, EMC_ZQ_CAL_ZQ_CAL_CMD,
1472 					      EMC_ZQ_CAL, delay);
1473 
1474 			value = (mr13_flip_fspop & 0xfffffff7) | 0x0c000000;
1475 			ccfifo_writel(emc, value, EMC_MRW3, delay);
1476 			ccfifo_writel(emc, 0, EMC_SELF_REF, 0);
1477 			ccfifo_writel(emc, 0, EMC_REF, 0);
1478 
1479 			ccfifo_writel(emc, EMC_ZQ_CAL_ZQ_LATCH_CMD, EMC_ZQ_CAL,
1480 				      max_t(s32, 0, zq_latch_dvfs_wait_time));
1481 		}
1482 	}
1483 
1484 	/* WAR: delay for zqlatch */
1485 	ccfifo_writel(emc, 0, 0, 10);
1486 
1487 	/*
1488 	 * Step 16:
1489 	 *   LPDDR4 Conditional Training Kickoff. Removed.
1490 	 */
1491 
1492 	/*
1493 	 * Step 17:
1494 	 *   MANSR exit self refresh.
1495 	 */
1496 	emc_dbg(emc, STEPS, "Step 17\n");
1497 
1498 	if (opt_dvfs_mode == MAN_SR && dram_type != DRAM_TYPE_LPDDR4)
1499 		ccfifo_writel(emc, 0, EMC_SELF_REF, 0);
1500 
1501 	/*
1502 	 * Step 18:
1503 	 *   Send MRWs to LPDDR3/DDR3.
1504 	 */
1505 	emc_dbg(emc, STEPS, "Step 18\n");
1506 
1507 	if (dram_type == DRAM_TYPE_LPDDR2) {
1508 		ccfifo_writel(emc, next->emc_mrw2, EMC_MRW2, 0);
1509 		ccfifo_writel(emc, next->emc_mrw,  EMC_MRW,  0);
1510 		if (is_lpddr3)
1511 			ccfifo_writel(emc, next->emc_mrw4, EMC_MRW4, 0);
1512 	} else if (dram_type == DRAM_TYPE_DDR3) {
1513 		if (opt_dll_mode)
1514 			ccfifo_writel(emc, next->emc_emrs &
1515 				      ~EMC_EMRS_USE_EMRS_LONG_CNT, EMC_EMRS, 0);
1516 		ccfifo_writel(emc, next->emc_emrs2 &
1517 			      ~EMC_EMRS2_USE_EMRS2_LONG_CNT, EMC_EMRS2, 0);
1518 		ccfifo_writel(emc, next->emc_mrs |
1519 			      EMC_EMRS_USE_EMRS_LONG_CNT, EMC_MRS, 0);
1520 	}
1521 
1522 	/*
1523 	 * Step 19:
1524 	 *   ZQCAL for LPDDR3/DDR3
1525 	 */
1526 	emc_dbg(emc, STEPS, "Step 19\n");
1527 
1528 	if (opt_zcal_en_cc) {
1529 		if (dram_type == DRAM_TYPE_LPDDR2) {
1530 			value = opt_cc_short_zcal ? 90000 : 360000;
1531 			value = div_o3(value, dst_clk_period);
1532 			value = value <<
1533 				EMC_MRS_WAIT_CNT2_MRS_EXT2_WAIT_CNT_SHIFT |
1534 				value <<
1535 				EMC_MRS_WAIT_CNT2_MRS_EXT1_WAIT_CNT_SHIFT;
1536 			ccfifo_writel(emc, value, EMC_MRS_WAIT_CNT2, 0);
1537 
1538 			value = opt_cc_short_zcal ? 0x56 : 0xab;
1539 			ccfifo_writel(emc, 2 << EMC_MRW_MRW_DEV_SELECTN_SHIFT |
1540 					   EMC_MRW_USE_MRW_EXT_CNT |
1541 					   10 << EMC_MRW_MRW_MA_SHIFT |
1542 					   value << EMC_MRW_MRW_OP_SHIFT,
1543 				      EMC_MRW, 0);
1544 
1545 			if (emc->num_devices > 1) {
1546 				value = 1 << EMC_MRW_MRW_DEV_SELECTN_SHIFT |
1547 					EMC_MRW_USE_MRW_EXT_CNT |
1548 					10 << EMC_MRW_MRW_MA_SHIFT |
1549 					value << EMC_MRW_MRW_OP_SHIFT;
1550 				ccfifo_writel(emc, value, EMC_MRW, 0);
1551 			}
1552 		} else if (dram_type == DRAM_TYPE_DDR3) {
1553 			value = opt_cc_short_zcal ? 0 : EMC_ZQ_CAL_LONG;
1554 
1555 			ccfifo_writel(emc, value |
1556 					   2 << EMC_ZQ_CAL_DEV_SEL_SHIFT |
1557 					   EMC_ZQ_CAL_ZQ_CAL_CMD, EMC_ZQ_CAL,
1558 					   0);
1559 
1560 			if (emc->num_devices > 1) {
1561 				value = value | 1 << EMC_ZQ_CAL_DEV_SEL_SHIFT |
1562 						EMC_ZQ_CAL_ZQ_CAL_CMD;
1563 				ccfifo_writel(emc, value, EMC_ZQ_CAL, 0);
1564 			}
1565 		}
1566 	}
1567 
1568 	if (bg_reg_mode_change) {
1569 		tegra210_emc_set_shadow_bypass(emc, ACTIVE);
1570 
1571 		if (ramp_up_wait <= 1250000)
1572 			delay = (1250000 - ramp_up_wait) / dst_clk_period;
1573 		else
1574 			delay = 0;
1575 
1576 		ccfifo_writel(emc,
1577 			      next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX],
1578 			      EMC_PMACRO_BG_BIAS_CTRL_0, delay);
1579 		tegra210_emc_set_shadow_bypass(emc, ASSEMBLY);
1580 	}
1581 
1582 	/*
1583 	 * Step 20:
1584 	 *   Issue ref and optional QRST.
1585 	 */
1586 	emc_dbg(emc, STEPS, "Step 20\n");
1587 
1588 	if (dram_type != DRAM_TYPE_LPDDR4)
1589 		ccfifo_writel(emc, 0, EMC_REF, 0);
1590 
1591 	if (opt_do_sw_qrst) {
1592 		ccfifo_writel(emc, 1, EMC_ISSUE_QRST, 0);
1593 		ccfifo_writel(emc, 0, EMC_ISSUE_QRST, 2);
1594 	}
1595 
1596 	/*
1597 	 * Step 21:
1598 	 *   Restore ZCAL and ZCAL interval.
1599 	 */
1600 	emc_dbg(emc, STEPS, "Step 21\n");
1601 
1602 	if (save_restore_clkstop_pd || opt_zcal_en_cc) {
1603 		ccfifo_writel(emc, emc_dbg | EMC_DBG_WRITE_MUX_ACTIVE,
1604 			      EMC_DBG, 0);
1605 		if (opt_zcal_en_cc && dram_type != DRAM_TYPE_LPDDR4)
1606 			ccfifo_writel(emc, next->burst_regs[EMC_ZCAL_INTERVAL_INDEX],
1607 				      EMC_ZCAL_INTERVAL, 0);
1608 
1609 		if (save_restore_clkstop_pd)
1610 			ccfifo_writel(emc, next->burst_regs[EMC_CFG_INDEX] &
1611 						~EMC_CFG_DYN_SELF_REF,
1612 				      EMC_CFG, 0);
1613 		ccfifo_writel(emc, emc_dbg, EMC_DBG, 0);
1614 	}
1615 
1616 	/*
1617 	 * Step 22:
1618 	 *   Restore EMC_CFG_PIPE_CLK.
1619 	 */
1620 	emc_dbg(emc, STEPS, "Step 22\n");
1621 
1622 	ccfifo_writel(emc, emc_cfg_pipe_clk, EMC_CFG_PIPE_CLK, 0);
1623 
1624 	if (bg_reg_mode_change) {
1625 		if (enable_bg_reg)
1626 			emc_writel(emc,
1627 				   next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] &
1628 					~EMC_PMACRO_BG_BIAS_CTRL_0_BGLP_E_PWRD,
1629 				   EMC_PMACRO_BG_BIAS_CTRL_0);
1630 		else
1631 			emc_writel(emc,
1632 				   next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] &
1633 					~EMC_PMACRO_BG_BIAS_CTRL_0_BG_E_PWRD,
1634 				   EMC_PMACRO_BG_BIAS_CTRL_0);
1635 	}
1636 
1637 	/*
1638 	 * Step 23:
1639 	 */
1640 	emc_dbg(emc, STEPS, "Step 23\n");
1641 
1642 	value = emc_readl(emc, EMC_CFG_DIG_DLL);
1643 	value |= EMC_CFG_DIG_DLL_CFG_DLL_STALL_ALL_TRAFFIC;
1644 	value &= ~EMC_CFG_DIG_DLL_CFG_DLL_STALL_RW_UNTIL_LOCK;
1645 	value &= ~EMC_CFG_DIG_DLL_CFG_DLL_STALL_ALL_UNTIL_LOCK;
1646 	value &= ~EMC_CFG_DIG_DLL_CFG_DLL_EN;
1647 	value = (value & ~EMC_CFG_DIG_DLL_CFG_DLL_MODE_MASK) |
1648 		(2 << EMC_CFG_DIG_DLL_CFG_DLL_MODE_SHIFT);
1649 	emc_writel(emc, value, EMC_CFG_DIG_DLL);
1650 
1651 	tegra210_emc_do_clock_change(emc, clksrc);
1652 
1653 	/*
1654 	 * Step 24:
1655 	 *   Save training results. Removed.
1656 	 */
1657 
1658 	/*
1659 	 * Step 25:
1660 	 *   Program MC updown registers.
1661 	 */
1662 	emc_dbg(emc, STEPS, "Step 25\n");
1663 
1664 	if (next->rate > last->rate) {
1665 		for (i = 0; i < next->num_up_down; i++)
1666 			mc_writel(emc->mc, next->la_scale_regs[i],
1667 				  emc->offsets->la_scale[i]);
1668 
1669 		tegra210_emc_timing_update(emc);
1670 	}
1671 
1672 	/*
1673 	 * Step 26:
1674 	 *   Restore ZCAL registers.
1675 	 */
1676 	emc_dbg(emc, STEPS, "Step 26\n");
1677 
1678 	if (dram_type == DRAM_TYPE_LPDDR4) {
1679 		tegra210_emc_set_shadow_bypass(emc, ACTIVE);
1680 		emc_writel(emc, next->burst_regs[EMC_ZCAL_WAIT_CNT_INDEX],
1681 			   EMC_ZCAL_WAIT_CNT);
1682 		emc_writel(emc, next->burst_regs[EMC_ZCAL_INTERVAL_INDEX],
1683 			   EMC_ZCAL_INTERVAL);
1684 		tegra210_emc_set_shadow_bypass(emc, ASSEMBLY);
1685 	}
1686 
1687 	if (dram_type != DRAM_TYPE_LPDDR4 && opt_zcal_en_cc &&
1688 	    !opt_short_zcal && opt_cc_short_zcal) {
1689 		udelay(2);
1690 
1691 		tegra210_emc_set_shadow_bypass(emc, ACTIVE);
1692 		if (dram_type == DRAM_TYPE_LPDDR2)
1693 			emc_writel(emc, next->burst_regs[EMC_MRS_WAIT_CNT_INDEX],
1694 				   EMC_MRS_WAIT_CNT);
1695 		else if (dram_type == DRAM_TYPE_DDR3)
1696 			emc_writel(emc, next->burst_regs[EMC_ZCAL_WAIT_CNT_INDEX],
1697 				   EMC_ZCAL_WAIT_CNT);
1698 		tegra210_emc_set_shadow_bypass(emc, ASSEMBLY);
1699 	}
1700 
1701 	/*
1702 	 * Step 27:
1703 	 *   Restore EMC_CFG, FDPD registers.
1704 	 */
1705 	emc_dbg(emc, STEPS, "Step 27\n");
1706 
1707 	tegra210_emc_set_shadow_bypass(emc, ACTIVE);
1708 	emc_writel(emc, next->burst_regs[EMC_CFG_INDEX], EMC_CFG);
1709 	tegra210_emc_set_shadow_bypass(emc, ASSEMBLY);
1710 	emc_writel(emc, next->emc_fdpd_ctrl_cmd_no_ramp,
1711 		   EMC_FDPD_CTRL_CMD_NO_RAMP);
1712 	emc_writel(emc, next->emc_sel_dpd_ctrl, EMC_SEL_DPD_CTRL);
1713 
1714 	/*
1715 	 * Step 28:
1716 	 *   Training recover. Removed.
1717 	 */
1718 	emc_dbg(emc, STEPS, "Step 28\n");
1719 
1720 	tegra210_emc_set_shadow_bypass(emc, ACTIVE);
1721 	emc_writel(emc,
1722 		   next->burst_regs[EMC_PMACRO_AUTOCAL_CFG_COMMON_INDEX],
1723 		   EMC_PMACRO_AUTOCAL_CFG_COMMON);
1724 	tegra210_emc_set_shadow_bypass(emc, ASSEMBLY);
1725 
1726 	/*
1727 	 * Step 29:
1728 	 *   Power fix WAR.
1729 	 */
1730 	emc_dbg(emc, STEPS, "Step 29\n");
1731 
1732 	emc_writel(emc, EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE0 |
1733 		   EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE1 |
1734 		   EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE2 |
1735 		   EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE3 |
1736 		   EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE4 |
1737 		   EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE5 |
1738 		   EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE6 |
1739 		   EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE7,
1740 		   EMC_PMACRO_CFG_PM_GLOBAL_0);
1741 	emc_writel(emc, EMC_PMACRO_TRAINING_CTRL_0_CH0_TRAINING_E_WRPTR,
1742 		   EMC_PMACRO_TRAINING_CTRL_0);
1743 	emc_writel(emc, EMC_PMACRO_TRAINING_CTRL_1_CH1_TRAINING_E_WRPTR,
1744 		   EMC_PMACRO_TRAINING_CTRL_1);
1745 	emc_writel(emc, 0, EMC_PMACRO_CFG_PM_GLOBAL_0);
1746 
1747 	/*
1748 	 * Step 30:
1749 	 *   Re-enable autocal.
1750 	 */
1751 	emc_dbg(emc, STEPS, "Step 30: Re-enable DLL and AUTOCAL\n");
1752 
1753 	if (next->burst_regs[EMC_CFG_DIG_DLL_INDEX] & EMC_CFG_DIG_DLL_CFG_DLL_EN) {
1754 		value = emc_readl(emc, EMC_CFG_DIG_DLL);
1755 		value |=  EMC_CFG_DIG_DLL_CFG_DLL_STALL_ALL_TRAFFIC;
1756 		value |=  EMC_CFG_DIG_DLL_CFG_DLL_EN;
1757 		value &= ~EMC_CFG_DIG_DLL_CFG_DLL_STALL_RW_UNTIL_LOCK;
1758 		value &= ~EMC_CFG_DIG_DLL_CFG_DLL_STALL_ALL_UNTIL_LOCK;
1759 		value = (value & ~EMC_CFG_DIG_DLL_CFG_DLL_MODE_MASK) |
1760 			(2 << EMC_CFG_DIG_DLL_CFG_DLL_MODE_SHIFT);
1761 		emc_writel(emc, value, EMC_CFG_DIG_DLL);
1762 		tegra210_emc_timing_update(emc);
1763 	}
1764 
1765 	emc_writel(emc, next->emc_auto_cal_config, EMC_AUTO_CAL_CONFIG);
1766 
1767 	/* Done! Yay. */
1768 }
1769 
1770 const struct tegra210_emc_sequence tegra210_emc_r21021 = {
1771 	.revision = 0x7,
1772 	.set_clock = tegra210_emc_r21021_set_clock,
1773 	.periodic_compensation = tegra210_emc_r21021_periodic_compensation,
1774 };
1775