1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (c) 2015-2020, NVIDIA CORPORATION.  All rights reserved.
4  */
5 
6 #include <linux/bitfield.h>
7 #include <linux/clk.h>
8 #include <linux/clk/tegra.h>
9 #include <linux/debugfs.h>
10 #include <linux/delay.h>
11 #include <linux/kernel.h>
12 #include <linux/module.h>
13 #include <linux/of_address.h>
14 #include <linux/of_platform.h>
15 #include <linux/of_reserved_mem.h>
16 #include <linux/slab.h>
17 #include <linux/thermal.h>
18 #include <soc/tegra/fuse.h>
19 #include <soc/tegra/mc.h>
20 
21 #include "tegra210-emc.h"
22 #include "tegra210-mc.h"
23 
24 /* CLK_RST_CONTROLLER_CLK_SOURCE_EMC */
25 #define EMC_CLK_EMC_2X_CLK_SRC_SHIFT			29
26 #define EMC_CLK_EMC_2X_CLK_SRC_MASK			\
27 	(0x7 << EMC_CLK_EMC_2X_CLK_SRC_SHIFT)
28 #define EMC_CLK_SOURCE_PLLM_LJ				0x4
29 #define EMC_CLK_SOURCE_PLLMB_LJ				0x5
30 #define EMC_CLK_FORCE_CC_TRIGGER			BIT(27)
31 #define EMC_CLK_MC_EMC_SAME_FREQ			BIT(16)
32 #define EMC_CLK_EMC_2X_CLK_DIVISOR_SHIFT		0
33 #define EMC_CLK_EMC_2X_CLK_DIVISOR_MASK			\
34 	(0xff << EMC_CLK_EMC_2X_CLK_DIVISOR_SHIFT)
35 
36 /* CLK_RST_CONTROLLER_CLK_SOURCE_EMC_DLL */
37 #define DLL_CLK_EMC_DLL_CLK_SRC_SHIFT			29
38 #define DLL_CLK_EMC_DLL_CLK_SRC_MASK			\
39 	(0x7 << DLL_CLK_EMC_DLL_CLK_SRC_SHIFT)
40 #define DLL_CLK_EMC_DLL_DDLL_CLK_SEL_SHIFT		10
41 #define DLL_CLK_EMC_DLL_DDLL_CLK_SEL_MASK		\
42 	(0x3 << DLL_CLK_EMC_DLL_DDLL_CLK_SEL_SHIFT)
43 #define PLLM_VCOA					0
44 #define PLLM_VCOB					1
45 #define EMC_DLL_SWITCH_OUT				2
46 #define DLL_CLK_EMC_DLL_CLK_DIVISOR_SHIFT		0
47 #define DLL_CLK_EMC_DLL_CLK_DIVISOR_MASK		\
48 	(0xff << DLL_CLK_EMC_DLL_CLK_DIVISOR_SHIFT)
49 
50 /* MC_EMEM_ARB_MISC0 */
51 #define MC_EMEM_ARB_MISC0_EMC_SAME_FREQ			BIT(27)
52 
53 /* EMC_DATA_BRLSHFT_X */
54 #define EMC0_EMC_DATA_BRLSHFT_0_INDEX	2
55 #define EMC1_EMC_DATA_BRLSHFT_0_INDEX	3
56 #define EMC0_EMC_DATA_BRLSHFT_1_INDEX	4
57 #define EMC1_EMC_DATA_BRLSHFT_1_INDEX	5
58 
59 #define TRIM_REG(chan, rank, reg, byte)					\
60 	(((EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## rank ## _ ## reg ##	\
61 	   _OB_DDLL_LONG_DQ_RANK ## rank ## _BYTE ## byte ## _MASK &	\
62 	   next->trim_regs[EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ##		\
63 				 rank ## _ ## reg ## _INDEX]) >>	\
64 	  EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## rank ## _ ## reg ##	\
65 	  _OB_DDLL_LONG_DQ_RANK ## rank ## _BYTE ## byte ## _SHIFT)	\
66 	 +								\
67 	 (((EMC_DATA_BRLSHFT_ ## rank ## _RANK ## rank ## _BYTE ##	\
68 	    byte ## _DATA_BRLSHFT_MASK &				\
69 	    next->trim_perch_regs[EMC ## chan ##			\
70 			      _EMC_DATA_BRLSHFT_ ## rank ## _INDEX]) >>	\
71 	   EMC_DATA_BRLSHFT_ ## rank ## _RANK ## rank ## _BYTE ##	\
72 	   byte ## _DATA_BRLSHFT_SHIFT) * 64))
73 
74 #define CALC_TEMP(rank, reg, byte1, byte2, n)				\
75 	(((new[n] << EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## rank ## _ ##	\
76 	   reg ## _OB_DDLL_LONG_DQ_RANK ## rank ## _BYTE ## byte1 ## _SHIFT) & \
77 	  EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## rank ## _ ## reg ##	\
78 	  _OB_DDLL_LONG_DQ_RANK ## rank ## _BYTE ## byte1 ## _MASK)	\
79 	 |								\
80 	 ((new[n + 1] << EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## rank ## _ ##\
81 	   reg ## _OB_DDLL_LONG_DQ_RANK ## rank ## _BYTE ## byte2 ## _SHIFT) & \
82 	  EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## rank ## _ ## reg ##	\
83 	  _OB_DDLL_LONG_DQ_RANK ## rank ## _BYTE ## byte2 ## _MASK))
84 
85 #define REFRESH_SPEEDUP(value, speedup) \
86 		(((value) & 0xffff0000) | ((value) & 0xffff) * (speedup))
87 
88 #define LPDDR2_MR4_SRR GENMASK(2, 0)
89 
90 static const struct tegra210_emc_sequence *tegra210_emc_sequences[] = {
91 	&tegra210_emc_r21021,
92 };
93 
94 static const struct tegra210_emc_table_register_offsets
95 tegra210_emc_table_register_offsets = {
96 	.burst = {
97 		EMC_RC,
98 		EMC_RFC,
99 		EMC_RFCPB,
100 		EMC_REFCTRL2,
101 		EMC_RFC_SLR,
102 		EMC_RAS,
103 		EMC_RP,
104 		EMC_R2W,
105 		EMC_W2R,
106 		EMC_R2P,
107 		EMC_W2P,
108 		EMC_R2R,
109 		EMC_TPPD,
110 		EMC_CCDMW,
111 		EMC_RD_RCD,
112 		EMC_WR_RCD,
113 		EMC_RRD,
114 		EMC_REXT,
115 		EMC_WEXT,
116 		EMC_WDV_CHK,
117 		EMC_WDV,
118 		EMC_WSV,
119 		EMC_WEV,
120 		EMC_WDV_MASK,
121 		EMC_WS_DURATION,
122 		EMC_WE_DURATION,
123 		EMC_QUSE,
124 		EMC_QUSE_WIDTH,
125 		EMC_IBDLY,
126 		EMC_OBDLY,
127 		EMC_EINPUT,
128 		EMC_MRW6,
129 		EMC_EINPUT_DURATION,
130 		EMC_PUTERM_EXTRA,
131 		EMC_PUTERM_WIDTH,
132 		EMC_QRST,
133 		EMC_QSAFE,
134 		EMC_RDV,
135 		EMC_RDV_MASK,
136 		EMC_RDV_EARLY,
137 		EMC_RDV_EARLY_MASK,
138 		EMC_REFRESH,
139 		EMC_BURST_REFRESH_NUM,
140 		EMC_PRE_REFRESH_REQ_CNT,
141 		EMC_PDEX2WR,
142 		EMC_PDEX2RD,
143 		EMC_PCHG2PDEN,
144 		EMC_ACT2PDEN,
145 		EMC_AR2PDEN,
146 		EMC_RW2PDEN,
147 		EMC_CKE2PDEN,
148 		EMC_PDEX2CKE,
149 		EMC_PDEX2MRR,
150 		EMC_TXSR,
151 		EMC_TXSRDLL,
152 		EMC_TCKE,
153 		EMC_TCKESR,
154 		EMC_TPD,
155 		EMC_TFAW,
156 		EMC_TRPAB,
157 		EMC_TCLKSTABLE,
158 		EMC_TCLKSTOP,
159 		EMC_MRW7,
160 		EMC_TREFBW,
161 		EMC_ODT_WRITE,
162 		EMC_FBIO_CFG5,
163 		EMC_FBIO_CFG7,
164 		EMC_CFG_DIG_DLL,
165 		EMC_CFG_DIG_DLL_PERIOD,
166 		EMC_PMACRO_IB_RXRT,
167 		EMC_CFG_PIPE_1,
168 		EMC_CFG_PIPE_2,
169 		EMC_PMACRO_QUSE_DDLL_RANK0_4,
170 		EMC_PMACRO_QUSE_DDLL_RANK0_5,
171 		EMC_PMACRO_QUSE_DDLL_RANK1_4,
172 		EMC_PMACRO_QUSE_DDLL_RANK1_5,
173 		EMC_MRW8,
174 		EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_4,
175 		EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_5,
176 		EMC_PMACRO_OB_DDLL_LONG_DQS_RANK0_0,
177 		EMC_PMACRO_OB_DDLL_LONG_DQS_RANK0_1,
178 		EMC_PMACRO_OB_DDLL_LONG_DQS_RANK0_2,
179 		EMC_PMACRO_OB_DDLL_LONG_DQS_RANK0_3,
180 		EMC_PMACRO_OB_DDLL_LONG_DQS_RANK0_4,
181 		EMC_PMACRO_OB_DDLL_LONG_DQS_RANK0_5,
182 		EMC_PMACRO_OB_DDLL_LONG_DQS_RANK1_0,
183 		EMC_PMACRO_OB_DDLL_LONG_DQS_RANK1_1,
184 		EMC_PMACRO_OB_DDLL_LONG_DQS_RANK1_2,
185 		EMC_PMACRO_OB_DDLL_LONG_DQS_RANK1_3,
186 		EMC_PMACRO_OB_DDLL_LONG_DQS_RANK1_4,
187 		EMC_PMACRO_OB_DDLL_LONG_DQS_RANK1_5,
188 		EMC_PMACRO_DDLL_LONG_CMD_0,
189 		EMC_PMACRO_DDLL_LONG_CMD_1,
190 		EMC_PMACRO_DDLL_LONG_CMD_2,
191 		EMC_PMACRO_DDLL_LONG_CMD_3,
192 		EMC_PMACRO_DDLL_LONG_CMD_4,
193 		EMC_PMACRO_DDLL_SHORT_CMD_0,
194 		EMC_PMACRO_DDLL_SHORT_CMD_1,
195 		EMC_PMACRO_DDLL_SHORT_CMD_2,
196 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE0_3,
197 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE1_3,
198 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE2_3,
199 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE3_3,
200 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE4_3,
201 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE5_3,
202 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE6_3,
203 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE7_3,
204 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD0_3,
205 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD1_3,
206 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD2_3,
207 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD3_3,
208 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE0_3,
209 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE1_3,
210 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE2_3,
211 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE3_3,
212 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE4_3,
213 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE5_3,
214 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE6_3,
215 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE7_3,
216 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD0_0,
217 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD0_1,
218 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD0_2,
219 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD0_3,
220 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD1_0,
221 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD1_1,
222 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD1_2,
223 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD1_3,
224 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD2_0,
225 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD2_1,
226 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD2_2,
227 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD2_3,
228 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD3_0,
229 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD3_1,
230 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD3_2,
231 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD3_3,
232 		EMC_TXDSRVTTGEN,
233 		EMC_FDPD_CTRL_DQ,
234 		EMC_FDPD_CTRL_CMD,
235 		EMC_FBIO_SPARE,
236 		EMC_ZCAL_INTERVAL,
237 		EMC_ZCAL_WAIT_CNT,
238 		EMC_MRS_WAIT_CNT,
239 		EMC_MRS_WAIT_CNT2,
240 		EMC_AUTO_CAL_CHANNEL,
241 		EMC_DLL_CFG_0,
242 		EMC_DLL_CFG_1,
243 		EMC_PMACRO_AUTOCAL_CFG_COMMON,
244 		EMC_PMACRO_ZCTRL,
245 		EMC_CFG,
246 		EMC_CFG_PIPE,
247 		EMC_DYN_SELF_REF_CONTROL,
248 		EMC_QPOP,
249 		EMC_DQS_BRLSHFT_0,
250 		EMC_DQS_BRLSHFT_1,
251 		EMC_CMD_BRLSHFT_2,
252 		EMC_CMD_BRLSHFT_3,
253 		EMC_PMACRO_PAD_CFG_CTRL,
254 		EMC_PMACRO_DATA_PAD_RX_CTRL,
255 		EMC_PMACRO_CMD_PAD_RX_CTRL,
256 		EMC_PMACRO_DATA_RX_TERM_MODE,
257 		EMC_PMACRO_CMD_RX_TERM_MODE,
258 		EMC_PMACRO_CMD_PAD_TX_CTRL,
259 		EMC_PMACRO_DATA_PAD_TX_CTRL,
260 		EMC_PMACRO_COMMON_PAD_TX_CTRL,
261 		EMC_PMACRO_VTTGEN_CTRL_0,
262 		EMC_PMACRO_VTTGEN_CTRL_1,
263 		EMC_PMACRO_VTTGEN_CTRL_2,
264 		EMC_PMACRO_BRICK_CTRL_RFU1,
265 		EMC_PMACRO_CMD_BRICK_CTRL_FDPD,
266 		EMC_PMACRO_BRICK_CTRL_RFU2,
267 		EMC_PMACRO_DATA_BRICK_CTRL_FDPD,
268 		EMC_PMACRO_BG_BIAS_CTRL_0,
269 		EMC_CFG_3,
270 		EMC_PMACRO_TX_PWRD_0,
271 		EMC_PMACRO_TX_PWRD_1,
272 		EMC_PMACRO_TX_PWRD_2,
273 		EMC_PMACRO_TX_PWRD_3,
274 		EMC_PMACRO_TX_PWRD_4,
275 		EMC_PMACRO_TX_PWRD_5,
276 		EMC_CONFIG_SAMPLE_DELAY,
277 		EMC_PMACRO_TX_SEL_CLK_SRC_0,
278 		EMC_PMACRO_TX_SEL_CLK_SRC_1,
279 		EMC_PMACRO_TX_SEL_CLK_SRC_2,
280 		EMC_PMACRO_TX_SEL_CLK_SRC_3,
281 		EMC_PMACRO_TX_SEL_CLK_SRC_4,
282 		EMC_PMACRO_TX_SEL_CLK_SRC_5,
283 		EMC_PMACRO_DDLL_BYPASS,
284 		EMC_PMACRO_DDLL_PWRD_0,
285 		EMC_PMACRO_DDLL_PWRD_1,
286 		EMC_PMACRO_DDLL_PWRD_2,
287 		EMC_PMACRO_CMD_CTRL_0,
288 		EMC_PMACRO_CMD_CTRL_1,
289 		EMC_PMACRO_CMD_CTRL_2,
290 		EMC_TR_TIMING_0,
291 		EMC_TR_DVFS,
292 		EMC_TR_CTRL_1,
293 		EMC_TR_RDV,
294 		EMC_TR_QPOP,
295 		EMC_TR_RDV_MASK,
296 		EMC_MRW14,
297 		EMC_TR_QSAFE,
298 		EMC_TR_QRST,
299 		EMC_TRAINING_CTRL,
300 		EMC_TRAINING_SETTLE,
301 		EMC_TRAINING_VREF_SETTLE,
302 		EMC_TRAINING_CA_FINE_CTRL,
303 		EMC_TRAINING_CA_CTRL_MISC,
304 		EMC_TRAINING_CA_CTRL_MISC1,
305 		EMC_TRAINING_CA_VREF_CTRL,
306 		EMC_TRAINING_QUSE_CORS_CTRL,
307 		EMC_TRAINING_QUSE_FINE_CTRL,
308 		EMC_TRAINING_QUSE_CTRL_MISC,
309 		EMC_TRAINING_QUSE_VREF_CTRL,
310 		EMC_TRAINING_READ_FINE_CTRL,
311 		EMC_TRAINING_READ_CTRL_MISC,
312 		EMC_TRAINING_READ_VREF_CTRL,
313 		EMC_TRAINING_WRITE_FINE_CTRL,
314 		EMC_TRAINING_WRITE_CTRL_MISC,
315 		EMC_TRAINING_WRITE_VREF_CTRL,
316 		EMC_TRAINING_MPC,
317 		EMC_MRW15,
318 	},
319 	.trim = {
320 		EMC_PMACRO_IB_DDLL_LONG_DQS_RANK0_0,
321 		EMC_PMACRO_IB_DDLL_LONG_DQS_RANK0_1,
322 		EMC_PMACRO_IB_DDLL_LONG_DQS_RANK0_2,
323 		EMC_PMACRO_IB_DDLL_LONG_DQS_RANK0_3,
324 		EMC_PMACRO_IB_DDLL_LONG_DQS_RANK1_0,
325 		EMC_PMACRO_IB_DDLL_LONG_DQS_RANK1_1,
326 		EMC_PMACRO_IB_DDLL_LONG_DQS_RANK1_2,
327 		EMC_PMACRO_IB_DDLL_LONG_DQS_RANK1_3,
328 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE0_0,
329 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE0_1,
330 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE0_2,
331 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE1_0,
332 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE1_1,
333 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE1_2,
334 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE2_0,
335 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE2_1,
336 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE2_2,
337 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE3_0,
338 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE3_1,
339 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE3_2,
340 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE4_0,
341 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE4_1,
342 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE4_2,
343 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE5_0,
344 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE5_1,
345 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE5_2,
346 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE6_0,
347 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE6_1,
348 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE6_2,
349 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE7_0,
350 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE7_1,
351 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE7_2,
352 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE0_0,
353 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE0_1,
354 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE0_2,
355 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE1_0,
356 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE1_1,
357 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE1_2,
358 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE2_0,
359 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE2_1,
360 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE2_2,
361 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE3_0,
362 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE3_1,
363 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE3_2,
364 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE4_0,
365 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE4_1,
366 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE4_2,
367 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE5_0,
368 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE5_1,
369 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE5_2,
370 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE6_0,
371 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE6_1,
372 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE6_2,
373 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE7_0,
374 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE7_1,
375 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE7_2,
376 		EMC_PMACRO_IB_VREF_DQS_0,
377 		EMC_PMACRO_IB_VREF_DQS_1,
378 		EMC_PMACRO_IB_VREF_DQ_0,
379 		EMC_PMACRO_IB_VREF_DQ_1,
380 		EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0,
381 		EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1,
382 		EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2,
383 		EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3,
384 		EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_4,
385 		EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_5,
386 		EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0,
387 		EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1,
388 		EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2,
389 		EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3,
390 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE0_0,
391 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE0_1,
392 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE0_2,
393 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE1_0,
394 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE1_1,
395 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE1_2,
396 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE2_0,
397 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE2_1,
398 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE2_2,
399 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE3_0,
400 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE3_1,
401 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE3_2,
402 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE4_0,
403 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE4_1,
404 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE4_2,
405 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE5_0,
406 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE5_1,
407 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE5_2,
408 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE6_0,
409 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE6_1,
410 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE6_2,
411 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE7_0,
412 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE7_1,
413 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE7_2,
414 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD0_0,
415 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD0_1,
416 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD0_2,
417 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD1_0,
418 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD1_1,
419 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD1_2,
420 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD2_0,
421 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD2_1,
422 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD2_2,
423 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD3_0,
424 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD3_1,
425 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD3_2,
426 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE0_0,
427 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE0_1,
428 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE0_2,
429 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE1_0,
430 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE1_1,
431 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE1_2,
432 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE2_0,
433 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE2_1,
434 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE2_2,
435 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE3_0,
436 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE3_1,
437 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE3_2,
438 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE4_0,
439 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE4_1,
440 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE4_2,
441 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE5_0,
442 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE5_1,
443 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE5_2,
444 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE6_0,
445 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE6_1,
446 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE6_2,
447 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE7_0,
448 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE7_1,
449 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE7_2,
450 		EMC_PMACRO_QUSE_DDLL_RANK0_0,
451 		EMC_PMACRO_QUSE_DDLL_RANK0_1,
452 		EMC_PMACRO_QUSE_DDLL_RANK0_2,
453 		EMC_PMACRO_QUSE_DDLL_RANK0_3,
454 		EMC_PMACRO_QUSE_DDLL_RANK1_0,
455 		EMC_PMACRO_QUSE_DDLL_RANK1_1,
456 		EMC_PMACRO_QUSE_DDLL_RANK1_2,
457 		EMC_PMACRO_QUSE_DDLL_RANK1_3
458 	},
459 	.burst_mc = {
460 		MC_EMEM_ARB_CFG,
461 		MC_EMEM_ARB_OUTSTANDING_REQ,
462 		MC_EMEM_ARB_REFPB_HP_CTRL,
463 		MC_EMEM_ARB_REFPB_BANK_CTRL,
464 		MC_EMEM_ARB_TIMING_RCD,
465 		MC_EMEM_ARB_TIMING_RP,
466 		MC_EMEM_ARB_TIMING_RC,
467 		MC_EMEM_ARB_TIMING_RAS,
468 		MC_EMEM_ARB_TIMING_FAW,
469 		MC_EMEM_ARB_TIMING_RRD,
470 		MC_EMEM_ARB_TIMING_RAP2PRE,
471 		MC_EMEM_ARB_TIMING_WAP2PRE,
472 		MC_EMEM_ARB_TIMING_R2R,
473 		MC_EMEM_ARB_TIMING_W2W,
474 		MC_EMEM_ARB_TIMING_R2W,
475 		MC_EMEM_ARB_TIMING_CCDMW,
476 		MC_EMEM_ARB_TIMING_W2R,
477 		MC_EMEM_ARB_TIMING_RFCPB,
478 		MC_EMEM_ARB_DA_TURNS,
479 		MC_EMEM_ARB_DA_COVERS,
480 		MC_EMEM_ARB_MISC0,
481 		MC_EMEM_ARB_MISC1,
482 		MC_EMEM_ARB_MISC2,
483 		MC_EMEM_ARB_RING1_THROTTLE,
484 		MC_EMEM_ARB_DHYST_CTRL,
485 		MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_0,
486 		MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_1,
487 		MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_2,
488 		MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_3,
489 		MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_4,
490 		MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_5,
491 		MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_6,
492 		MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_7,
493 	},
494 	.la_scale = {
495 		MC_MLL_MPCORER_PTSA_RATE,
496 		MC_FTOP_PTSA_RATE,
497 		MC_PTSA_GRANT_DECREMENT,
498 		MC_LATENCY_ALLOWANCE_XUSB_0,
499 		MC_LATENCY_ALLOWANCE_XUSB_1,
500 		MC_LATENCY_ALLOWANCE_TSEC_0,
501 		MC_LATENCY_ALLOWANCE_SDMMCA_0,
502 		MC_LATENCY_ALLOWANCE_SDMMCAA_0,
503 		MC_LATENCY_ALLOWANCE_SDMMC_0,
504 		MC_LATENCY_ALLOWANCE_SDMMCAB_0,
505 		MC_LATENCY_ALLOWANCE_PPCS_0,
506 		MC_LATENCY_ALLOWANCE_PPCS_1,
507 		MC_LATENCY_ALLOWANCE_MPCORE_0,
508 		MC_LATENCY_ALLOWANCE_HC_0,
509 		MC_LATENCY_ALLOWANCE_HC_1,
510 		MC_LATENCY_ALLOWANCE_AVPC_0,
511 		MC_LATENCY_ALLOWANCE_GPU_0,
512 		MC_LATENCY_ALLOWANCE_GPU2_0,
513 		MC_LATENCY_ALLOWANCE_NVENC_0,
514 		MC_LATENCY_ALLOWANCE_NVDEC_0,
515 		MC_LATENCY_ALLOWANCE_VIC_0,
516 		MC_LATENCY_ALLOWANCE_VI2_0,
517 		MC_LATENCY_ALLOWANCE_ISP2_0,
518 		MC_LATENCY_ALLOWANCE_ISP2_1,
519 	},
520 	.burst_per_channel = {
521 		{ .bank = 0, .offset = EMC_MRW10, },
522 		{ .bank = 1, .offset = EMC_MRW10, },
523 		{ .bank = 0, .offset = EMC_MRW11, },
524 		{ .bank = 1, .offset = EMC_MRW11, },
525 		{ .bank = 0, .offset = EMC_MRW12, },
526 		{ .bank = 1, .offset = EMC_MRW12, },
527 		{ .bank = 0, .offset = EMC_MRW13, },
528 		{ .bank = 1, .offset = EMC_MRW13, },
529 	},
530 	.trim_per_channel = {
531 		{ .bank = 0, .offset = EMC_CMD_BRLSHFT_0, },
532 		{ .bank = 1, .offset = EMC_CMD_BRLSHFT_1, },
533 		{ .bank = 0, .offset = EMC_DATA_BRLSHFT_0, },
534 		{ .bank = 1, .offset = EMC_DATA_BRLSHFT_0, },
535 		{ .bank = 0, .offset = EMC_DATA_BRLSHFT_1, },
536 		{ .bank = 1, .offset = EMC_DATA_BRLSHFT_1, },
537 		{ .bank = 0, .offset = EMC_QUSE_BRLSHFT_0, },
538 		{ .bank = 1, .offset = EMC_QUSE_BRLSHFT_1, },
539 		{ .bank = 0, .offset = EMC_QUSE_BRLSHFT_2, },
540 		{ .bank = 1, .offset = EMC_QUSE_BRLSHFT_3, },
541 	},
542 	.vref_per_channel = {
543 		{
544 			.bank = 0,
545 			.offset = EMC_TRAINING_OPT_DQS_IB_VREF_RANK0,
546 		}, {
547 			.bank = 1,
548 			.offset = EMC_TRAINING_OPT_DQS_IB_VREF_RANK0,
549 		}, {
550 			.bank = 0,
551 			.offset = EMC_TRAINING_OPT_DQS_IB_VREF_RANK1,
552 		}, {
553 			.bank = 1,
554 			.offset = EMC_TRAINING_OPT_DQS_IB_VREF_RANK1,
555 		},
556 	},
557 };
558 
559 static void tegra210_emc_train(struct timer_list *timer)
560 {
561 	struct tegra210_emc *emc = from_timer(emc, timer, training);
562 	unsigned long flags;
563 
564 	if (!emc->last)
565 		return;
566 
567 	spin_lock_irqsave(&emc->lock, flags);
568 
569 	if (emc->sequence->periodic_compensation)
570 		emc->sequence->periodic_compensation(emc);
571 
572 	spin_unlock_irqrestore(&emc->lock, flags);
573 
574 	mod_timer(&emc->training,
575 		  jiffies + msecs_to_jiffies(emc->training_interval));
576 }
577 
578 static void tegra210_emc_training_start(struct tegra210_emc *emc)
579 {
580 	mod_timer(&emc->training,
581 		  jiffies + msecs_to_jiffies(emc->training_interval));
582 }
583 
584 static void tegra210_emc_training_stop(struct tegra210_emc *emc)
585 {
586 	del_timer(&emc->training);
587 }
588 
589 static unsigned int tegra210_emc_get_temperature(struct tegra210_emc *emc)
590 {
591 	unsigned long flags;
592 	u32 value, max = 0;
593 	unsigned int i;
594 
595 	spin_lock_irqsave(&emc->lock, flags);
596 
597 	for (i = 0; i < emc->num_devices; i++) {
598 		value = tegra210_emc_mrr_read(emc, i, 4);
599 
600 		if (value & BIT(7))
601 			dev_dbg(emc->dev,
602 				"sensor reading changed for device %u: %08x\n",
603 				i, value);
604 
605 		value = FIELD_GET(LPDDR2_MR4_SRR, value);
606 		if (value > max)
607 			max = value;
608 	}
609 
610 	spin_unlock_irqrestore(&emc->lock, flags);
611 
612 	return max;
613 }
614 
615 static void tegra210_emc_poll_refresh(struct timer_list *timer)
616 {
617 	struct tegra210_emc *emc = from_timer(emc, timer, refresh_timer);
618 	unsigned int temperature;
619 
620 	if (!emc->debugfs.temperature)
621 		temperature = tegra210_emc_get_temperature(emc);
622 	else
623 		temperature = emc->debugfs.temperature;
624 
625 	if (temperature == emc->temperature)
626 		goto reset;
627 
628 	switch (temperature) {
629 	case 0 ... 3:
630 		/* temperature is fine, using regular refresh */
631 		dev_dbg(emc->dev, "switching to nominal refresh...\n");
632 		tegra210_emc_set_refresh(emc, TEGRA210_EMC_REFRESH_NOMINAL);
633 		break;
634 
635 	case 4:
636 		dev_dbg(emc->dev, "switching to 2x refresh...\n");
637 		tegra210_emc_set_refresh(emc, TEGRA210_EMC_REFRESH_2X);
638 		break;
639 
640 	case 5:
641 		dev_dbg(emc->dev, "switching to 4x refresh...\n");
642 		tegra210_emc_set_refresh(emc, TEGRA210_EMC_REFRESH_4X);
643 		break;
644 
645 	case 6 ... 7:
646 		dev_dbg(emc->dev, "switching to throttle refresh...\n");
647 		tegra210_emc_set_refresh(emc, TEGRA210_EMC_REFRESH_THROTTLE);
648 		break;
649 
650 	default:
651 		WARN(1, "invalid DRAM temperature state %u\n", temperature);
652 		return;
653 	}
654 
655 	emc->temperature = temperature;
656 
657 reset:
658 	if (atomic_read(&emc->refresh_poll) > 0) {
659 		unsigned int interval = emc->refresh_poll_interval;
660 		unsigned int timeout = msecs_to_jiffies(interval);
661 
662 		mod_timer(&emc->refresh_timer, jiffies + timeout);
663 	}
664 }
665 
666 static void tegra210_emc_poll_refresh_stop(struct tegra210_emc *emc)
667 {
668 	atomic_set(&emc->refresh_poll, 0);
669 	del_timer_sync(&emc->refresh_timer);
670 }
671 
672 static void tegra210_emc_poll_refresh_start(struct tegra210_emc *emc)
673 {
674 	atomic_set(&emc->refresh_poll, 1);
675 
676 	mod_timer(&emc->refresh_timer,
677 		  jiffies + msecs_to_jiffies(emc->refresh_poll_interval));
678 }
679 
680 static int tegra210_emc_cd_max_state(struct thermal_cooling_device *cd,
681 				     unsigned long *state)
682 {
683 	*state = 1;
684 
685 	return 0;
686 }
687 
688 static int tegra210_emc_cd_get_state(struct thermal_cooling_device *cd,
689 				     unsigned long *state)
690 {
691 	struct tegra210_emc *emc = cd->devdata;
692 
693 	*state = atomic_read(&emc->refresh_poll);
694 
695 	return 0;
696 }
697 
698 static int tegra210_emc_cd_set_state(struct thermal_cooling_device *cd,
699 				     unsigned long state)
700 {
701 	struct tegra210_emc *emc = cd->devdata;
702 
703 	if (state == atomic_read(&emc->refresh_poll))
704 		return 0;
705 
706 	if (state)
707 		tegra210_emc_poll_refresh_start(emc);
708 	else
709 		tegra210_emc_poll_refresh_stop(emc);
710 
711 	return 0;
712 }
713 
714 static const struct thermal_cooling_device_ops tegra210_emc_cd_ops = {
715 	.get_max_state = tegra210_emc_cd_max_state,
716 	.get_cur_state = tegra210_emc_cd_get_state,
717 	.set_cur_state = tegra210_emc_cd_set_state,
718 };
719 
720 static void tegra210_emc_set_clock(struct tegra210_emc *emc, u32 clksrc)
721 {
722 	emc->sequence->set_clock(emc, clksrc);
723 
724 	if (emc->next->periodic_training)
725 		tegra210_emc_training_start(emc);
726 	else
727 		tegra210_emc_training_stop(emc);
728 }
729 
730 static void tegra210_change_dll_src(struct tegra210_emc *emc,
731 				    u32 clksrc)
732 {
733 	u32 dll_setting = emc->next->dll_clk_src;
734 	u32 emc_clk_src;
735 	u32 emc_clk_div;
736 
737 	emc_clk_src = (clksrc & EMC_CLK_EMC_2X_CLK_SRC_MASK) >>
738 		       EMC_CLK_EMC_2X_CLK_SRC_SHIFT;
739 	emc_clk_div = (clksrc & EMC_CLK_EMC_2X_CLK_DIVISOR_MASK) >>
740 		       EMC_CLK_EMC_2X_CLK_DIVISOR_SHIFT;
741 
742 	dll_setting &= ~(DLL_CLK_EMC_DLL_CLK_SRC_MASK |
743 			 DLL_CLK_EMC_DLL_CLK_DIVISOR_MASK);
744 	dll_setting |= emc_clk_src << DLL_CLK_EMC_DLL_CLK_SRC_SHIFT;
745 	dll_setting |= emc_clk_div << DLL_CLK_EMC_DLL_CLK_DIVISOR_SHIFT;
746 
747 	dll_setting &= ~DLL_CLK_EMC_DLL_DDLL_CLK_SEL_MASK;
748 	if (emc_clk_src == EMC_CLK_SOURCE_PLLMB_LJ)
749 		dll_setting |= (PLLM_VCOB <<
750 				DLL_CLK_EMC_DLL_DDLL_CLK_SEL_SHIFT);
751 	else if (emc_clk_src == EMC_CLK_SOURCE_PLLM_LJ)
752 		dll_setting |= (PLLM_VCOA <<
753 				DLL_CLK_EMC_DLL_DDLL_CLK_SEL_SHIFT);
754 	else
755 		dll_setting |= (EMC_DLL_SWITCH_OUT <<
756 				DLL_CLK_EMC_DLL_DDLL_CLK_SEL_SHIFT);
757 
758 	tegra210_clk_emc_dll_update_setting(dll_setting);
759 
760 	if (emc->next->clk_out_enb_x_0_clk_enb_emc_dll)
761 		tegra210_clk_emc_dll_enable(true);
762 	else
763 		tegra210_clk_emc_dll_enable(false);
764 }
765 
766 int tegra210_emc_set_refresh(struct tegra210_emc *emc,
767 			     enum tegra210_emc_refresh refresh)
768 {
769 	struct tegra210_emc_timing *timings;
770 	unsigned long flags;
771 
772 	if ((emc->dram_type != DRAM_TYPE_LPDDR2 &&
773 	     emc->dram_type != DRAM_TYPE_LPDDR4) ||
774 	    !emc->last)
775 		return -ENODEV;
776 
777 	if (refresh > TEGRA210_EMC_REFRESH_THROTTLE)
778 		return -EINVAL;
779 
780 	if (refresh == emc->refresh)
781 		return 0;
782 
783 	spin_lock_irqsave(&emc->lock, flags);
784 
785 	if (refresh == TEGRA210_EMC_REFRESH_THROTTLE && emc->derated)
786 		timings = emc->derated;
787 	else
788 		timings = emc->nominal;
789 
790 	if (timings != emc->timings) {
791 		unsigned int index = emc->last - emc->timings;
792 		u32 clksrc;
793 
794 		clksrc = emc->provider.configs[index].value |
795 			 EMC_CLK_FORCE_CC_TRIGGER;
796 
797 		emc->next = &timings[index];
798 		emc->timings = timings;
799 
800 		tegra210_emc_set_clock(emc, clksrc);
801 	} else {
802 		tegra210_emc_adjust_timing(emc, emc->last);
803 		tegra210_emc_timing_update(emc);
804 
805 		if (refresh != TEGRA210_EMC_REFRESH_NOMINAL)
806 			emc_writel(emc, EMC_REF_REF_CMD, EMC_REF);
807 	}
808 
809 	spin_unlock_irqrestore(&emc->lock, flags);
810 
811 	return 0;
812 }
813 
814 u32 tegra210_emc_mrr_read(struct tegra210_emc *emc, unsigned int chip,
815 			  unsigned int address)
816 {
817 	u32 value, ret = 0;
818 	unsigned int i;
819 
820 	value = (chip & EMC_MRR_DEV_SEL_MASK) << EMC_MRR_DEV_SEL_SHIFT |
821 		(address & EMC_MRR_MA_MASK) << EMC_MRR_MA_SHIFT;
822 	emc_writel(emc, value, EMC_MRR);
823 
824 	for (i = 0; i < emc->num_channels; i++)
825 		WARN(tegra210_emc_wait_for_update(emc, i, EMC_EMC_STATUS,
826 						  EMC_EMC_STATUS_MRR_DIVLD, 1),
827 		     "Timed out waiting for MRR %u (ch=%u)\n", address, i);
828 
829 	for (i = 0; i < emc->num_channels; i++) {
830 		value = emc_channel_readl(emc, i, EMC_MRR);
831 		value &= EMC_MRR_DATA_MASK;
832 
833 		ret = (ret << 16) | value;
834 	}
835 
836 	return ret;
837 }
838 
839 void tegra210_emc_do_clock_change(struct tegra210_emc *emc, u32 clksrc)
840 {
841 	int err;
842 
843 	mc_readl(emc->mc, MC_EMEM_ADR_CFG);
844 	emc_readl(emc, EMC_INTSTATUS);
845 
846 	tegra210_clk_emc_update_setting(clksrc);
847 
848 	err = tegra210_emc_wait_for_update(emc, 0, EMC_INTSTATUS,
849 					   EMC_INTSTATUS_CLKCHANGE_COMPLETE,
850 					   true);
851 	if (err)
852 		dev_warn(emc->dev, "clock change completion error: %d\n", err);
853 }
854 
855 struct tegra210_emc_timing *tegra210_emc_find_timing(struct tegra210_emc *emc,
856 						     unsigned long rate)
857 {
858 	unsigned int i;
859 
860 	for (i = 0; i < emc->num_timings; i++)
861 		if (emc->timings[i].rate * 1000UL == rate)
862 			return &emc->timings[i];
863 
864 	return NULL;
865 }
866 
867 int tegra210_emc_wait_for_update(struct tegra210_emc *emc, unsigned int channel,
868 				 unsigned int offset, u32 bit_mask, bool state)
869 {
870 	unsigned int i;
871 	u32 value;
872 
873 	for (i = 0; i < EMC_STATUS_UPDATE_TIMEOUT; i++) {
874 		value = emc_channel_readl(emc, channel, offset);
875 		if (!!(value & bit_mask) == state)
876 			return 0;
877 
878 		udelay(1);
879 	}
880 
881 	return -ETIMEDOUT;
882 }
883 
884 void tegra210_emc_set_shadow_bypass(struct tegra210_emc *emc, int set)
885 {
886 	u32 emc_dbg = emc_readl(emc, EMC_DBG);
887 
888 	if (set)
889 		emc_writel(emc, emc_dbg | EMC_DBG_WRITE_MUX_ACTIVE, EMC_DBG);
890 	else
891 		emc_writel(emc, emc_dbg & ~EMC_DBG_WRITE_MUX_ACTIVE, EMC_DBG);
892 }
893 
894 u32 tegra210_emc_get_dll_state(struct tegra210_emc_timing *next)
895 {
896 	if (next->emc_emrs & 0x1)
897 		return 0;
898 
899 	return 1;
900 }
901 
902 void tegra210_emc_timing_update(struct tegra210_emc *emc)
903 {
904 	unsigned int i;
905 	int err = 0;
906 
907 	emc_writel(emc, 0x1, EMC_TIMING_CONTROL);
908 
909 	for (i = 0; i < emc->num_channels; i++) {
910 		err |= tegra210_emc_wait_for_update(emc, i, EMC_EMC_STATUS,
911 						    EMC_EMC_STATUS_TIMING_UPDATE_STALLED,
912 						    false);
913 	}
914 
915 	if (err)
916 		dev_warn(emc->dev, "timing update error: %d\n", err);
917 }
918 
919 unsigned long tegra210_emc_actual_osc_clocks(u32 in)
920 {
921 	if (in < 0x40)
922 		return in * 16;
923 	else if (in < 0x80)
924 		return 2048;
925 	else if (in < 0xc0)
926 		return 4096;
927 	else
928 		return 8192;
929 }
930 
931 void tegra210_emc_start_periodic_compensation(struct tegra210_emc *emc)
932 {
933 	u32 mpc_req = 0x4b;
934 
935 	emc_writel(emc, mpc_req, EMC_MPC);
936 	mpc_req = emc_readl(emc, EMC_MPC);
937 }
938 
939 u32 tegra210_emc_compensate(struct tegra210_emc_timing *next, u32 offset)
940 {
941 	u32 temp = 0, rate = next->rate / 1000;
942 	s32 delta[4], delta_taps[4];
943 	s32 new[] = {
944 		TRIM_REG(0, 0, 0, 0),
945 		TRIM_REG(0, 0, 0, 1),
946 		TRIM_REG(0, 0, 1, 2),
947 		TRIM_REG(0, 0, 1, 3),
948 
949 		TRIM_REG(1, 0, 2, 4),
950 		TRIM_REG(1, 0, 2, 5),
951 		TRIM_REG(1, 0, 3, 6),
952 		TRIM_REG(1, 0, 3, 7),
953 
954 		TRIM_REG(0, 1, 0, 0),
955 		TRIM_REG(0, 1, 0, 1),
956 		TRIM_REG(0, 1, 1, 2),
957 		TRIM_REG(0, 1, 1, 3),
958 
959 		TRIM_REG(1, 1, 2, 4),
960 		TRIM_REG(1, 1, 2, 5),
961 		TRIM_REG(1, 1, 3, 6),
962 		TRIM_REG(1, 1, 3, 7)
963 	};
964 	unsigned i;
965 
966 	switch (offset) {
967 	case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0:
968 	case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1:
969 	case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2:
970 	case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3:
971 	case EMC_DATA_BRLSHFT_0:
972 		delta[0] = 128 * (next->current_dram_clktree[C0D0U0] -
973 				  next->trained_dram_clktree[C0D0U0]);
974 		delta[1] = 128 * (next->current_dram_clktree[C0D0U1] -
975 				  next->trained_dram_clktree[C0D0U1]);
976 		delta[2] = 128 * (next->current_dram_clktree[C1D0U0] -
977 				  next->trained_dram_clktree[C1D0U0]);
978 		delta[3] = 128 * (next->current_dram_clktree[C1D0U1] -
979 				  next->trained_dram_clktree[C1D0U1]);
980 
981 		delta_taps[0] = (delta[0] * (s32)rate) / 1000000;
982 		delta_taps[1] = (delta[1] * (s32)rate) / 1000000;
983 		delta_taps[2] = (delta[2] * (s32)rate) / 1000000;
984 		delta_taps[3] = (delta[3] * (s32)rate) / 1000000;
985 
986 		for (i = 0; i < 4; i++) {
987 			if ((delta_taps[i] > next->tree_margin) ||
988 			    (delta_taps[i] < (-1 * next->tree_margin))) {
989 				new[i * 2] = new[i * 2] + delta_taps[i];
990 				new[i * 2 + 1] = new[i * 2 + 1] +
991 							delta_taps[i];
992 			}
993 		}
994 
995 		if (offset == EMC_DATA_BRLSHFT_0) {
996 			for (i = 0; i < 8; i++)
997 				new[i] = new[i] / 64;
998 		} else {
999 			for (i = 0; i < 8; i++)
1000 				new[i] = new[i] % 64;
1001 		}
1002 
1003 		break;
1004 
1005 	case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0:
1006 	case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1:
1007 	case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2:
1008 	case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3:
1009 	case EMC_DATA_BRLSHFT_1:
1010 		delta[0] = 128 * (next->current_dram_clktree[C0D1U0] -
1011 				  next->trained_dram_clktree[C0D1U0]);
1012 		delta[1] = 128 * (next->current_dram_clktree[C0D1U1] -
1013 				  next->trained_dram_clktree[C0D1U1]);
1014 		delta[2] = 128 * (next->current_dram_clktree[C1D1U0] -
1015 				  next->trained_dram_clktree[C1D1U0]);
1016 		delta[3] = 128 * (next->current_dram_clktree[C1D1U1] -
1017 				  next->trained_dram_clktree[C1D1U1]);
1018 
1019 		delta_taps[0] = (delta[0] * (s32)rate) / 1000000;
1020 		delta_taps[1] = (delta[1] * (s32)rate) / 1000000;
1021 		delta_taps[2] = (delta[2] * (s32)rate) / 1000000;
1022 		delta_taps[3] = (delta[3] * (s32)rate) / 1000000;
1023 
1024 		for (i = 0; i < 4; i++) {
1025 			if ((delta_taps[i] > next->tree_margin) ||
1026 			    (delta_taps[i] < (-1 * next->tree_margin))) {
1027 				new[8 + i * 2] = new[8 + i * 2] +
1028 							delta_taps[i];
1029 				new[8 + i * 2 + 1] = new[8 + i * 2 + 1] +
1030 							delta_taps[i];
1031 			}
1032 		}
1033 
1034 		if (offset == EMC_DATA_BRLSHFT_1) {
1035 			for (i = 0; i < 8; i++)
1036 				new[i + 8] = new[i + 8] / 64;
1037 		} else {
1038 			for (i = 0; i < 8; i++)
1039 				new[i + 8] = new[i + 8] % 64;
1040 		}
1041 
1042 		break;
1043 	}
1044 
1045 	switch (offset) {
1046 	case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0:
1047 		temp = CALC_TEMP(0, 0, 0, 1, 0);
1048 		break;
1049 
1050 	case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1:
1051 		temp = CALC_TEMP(0, 1, 2, 3, 2);
1052 		break;
1053 
1054 	case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2:
1055 		temp = CALC_TEMP(0, 2, 4, 5, 4);
1056 		break;
1057 
1058 	case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3:
1059 		temp = CALC_TEMP(0, 3, 6, 7, 6);
1060 		break;
1061 
1062 	case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0:
1063 		temp = CALC_TEMP(1, 0, 0, 1, 8);
1064 		break;
1065 
1066 	case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1:
1067 		temp = CALC_TEMP(1, 1, 2, 3, 10);
1068 		break;
1069 
1070 	case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2:
1071 		temp = CALC_TEMP(1, 2, 4, 5, 12);
1072 		break;
1073 
1074 	case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3:
1075 		temp = CALC_TEMP(1, 3, 6, 7, 14);
1076 		break;
1077 
1078 	case EMC_DATA_BRLSHFT_0:
1079 		temp = ((new[0] <<
1080 			 EMC_DATA_BRLSHFT_0_RANK0_BYTE0_DATA_BRLSHFT_SHIFT) &
1081 			 EMC_DATA_BRLSHFT_0_RANK0_BYTE0_DATA_BRLSHFT_MASK) |
1082 		       ((new[1] <<
1083 			 EMC_DATA_BRLSHFT_0_RANK0_BYTE1_DATA_BRLSHFT_SHIFT) &
1084 			 EMC_DATA_BRLSHFT_0_RANK0_BYTE1_DATA_BRLSHFT_MASK) |
1085 		       ((new[2] <<
1086 			 EMC_DATA_BRLSHFT_0_RANK0_BYTE2_DATA_BRLSHFT_SHIFT) &
1087 			 EMC_DATA_BRLSHFT_0_RANK0_BYTE2_DATA_BRLSHFT_MASK) |
1088 		       ((new[3] <<
1089 			 EMC_DATA_BRLSHFT_0_RANK0_BYTE3_DATA_BRLSHFT_SHIFT) &
1090 			 EMC_DATA_BRLSHFT_0_RANK0_BYTE3_DATA_BRLSHFT_MASK) |
1091 		       ((new[4] <<
1092 			 EMC_DATA_BRLSHFT_0_RANK0_BYTE4_DATA_BRLSHFT_SHIFT) &
1093 			 EMC_DATA_BRLSHFT_0_RANK0_BYTE4_DATA_BRLSHFT_MASK) |
1094 		       ((new[5] <<
1095 			 EMC_DATA_BRLSHFT_0_RANK0_BYTE5_DATA_BRLSHFT_SHIFT) &
1096 			 EMC_DATA_BRLSHFT_0_RANK0_BYTE5_DATA_BRLSHFT_MASK) |
1097 		       ((new[6] <<
1098 			 EMC_DATA_BRLSHFT_0_RANK0_BYTE6_DATA_BRLSHFT_SHIFT) &
1099 			 EMC_DATA_BRLSHFT_0_RANK0_BYTE6_DATA_BRLSHFT_MASK) |
1100 		       ((new[7] <<
1101 			 EMC_DATA_BRLSHFT_0_RANK0_BYTE7_DATA_BRLSHFT_SHIFT) &
1102 			 EMC_DATA_BRLSHFT_0_RANK0_BYTE7_DATA_BRLSHFT_MASK);
1103 		break;
1104 
1105 	case EMC_DATA_BRLSHFT_1:
1106 		temp = ((new[8] <<
1107 			 EMC_DATA_BRLSHFT_1_RANK1_BYTE0_DATA_BRLSHFT_SHIFT) &
1108 			 EMC_DATA_BRLSHFT_1_RANK1_BYTE0_DATA_BRLSHFT_MASK) |
1109 		       ((new[9] <<
1110 			 EMC_DATA_BRLSHFT_1_RANK1_BYTE1_DATA_BRLSHFT_SHIFT) &
1111 			 EMC_DATA_BRLSHFT_1_RANK1_BYTE1_DATA_BRLSHFT_MASK) |
1112 		       ((new[10] <<
1113 			 EMC_DATA_BRLSHFT_1_RANK1_BYTE2_DATA_BRLSHFT_SHIFT) &
1114 			 EMC_DATA_BRLSHFT_1_RANK1_BYTE2_DATA_BRLSHFT_MASK) |
1115 		       ((new[11] <<
1116 			 EMC_DATA_BRLSHFT_1_RANK1_BYTE3_DATA_BRLSHFT_SHIFT) &
1117 			 EMC_DATA_BRLSHFT_1_RANK1_BYTE3_DATA_BRLSHFT_MASK) |
1118 		       ((new[12] <<
1119 			 EMC_DATA_BRLSHFT_1_RANK1_BYTE4_DATA_BRLSHFT_SHIFT) &
1120 			 EMC_DATA_BRLSHFT_1_RANK1_BYTE4_DATA_BRLSHFT_MASK) |
1121 		       ((new[13] <<
1122 			 EMC_DATA_BRLSHFT_1_RANK1_BYTE5_DATA_BRLSHFT_SHIFT) &
1123 			 EMC_DATA_BRLSHFT_1_RANK1_BYTE5_DATA_BRLSHFT_MASK) |
1124 		       ((new[14] <<
1125 			 EMC_DATA_BRLSHFT_1_RANK1_BYTE6_DATA_BRLSHFT_SHIFT) &
1126 			 EMC_DATA_BRLSHFT_1_RANK1_BYTE6_DATA_BRLSHFT_MASK) |
1127 		       ((new[15] <<
1128 			 EMC_DATA_BRLSHFT_1_RANK1_BYTE7_DATA_BRLSHFT_SHIFT) &
1129 			 EMC_DATA_BRLSHFT_1_RANK1_BYTE7_DATA_BRLSHFT_MASK);
1130 		break;
1131 
1132 	default:
1133 		break;
1134 	}
1135 
1136 	return temp;
1137 }
1138 
1139 u32 tegra210_emc_dll_prelock(struct tegra210_emc *emc, u32 clksrc)
1140 {
1141 	unsigned int i;
1142 	u32 value;
1143 
1144 	value = emc_readl(emc, EMC_CFG_DIG_DLL);
1145 	value &= ~EMC_CFG_DIG_DLL_CFG_DLL_LOCK_LIMIT_MASK;
1146 	value |= (3 << EMC_CFG_DIG_DLL_CFG_DLL_LOCK_LIMIT_SHIFT);
1147 	value &= ~EMC_CFG_DIG_DLL_CFG_DLL_EN;
1148 	value &= ~EMC_CFG_DIG_DLL_CFG_DLL_MODE_MASK;
1149 	value |= (3 << EMC_CFG_DIG_DLL_CFG_DLL_MODE_SHIFT);
1150 	value |= EMC_CFG_DIG_DLL_CFG_DLL_STALL_ALL_TRAFFIC;
1151 	value &= ~EMC_CFG_DIG_DLL_CFG_DLL_STALL_RW_UNTIL_LOCK;
1152 	value &= ~EMC_CFG_DIG_DLL_CFG_DLL_STALL_ALL_UNTIL_LOCK;
1153 	emc_writel(emc, value, EMC_CFG_DIG_DLL);
1154 	emc_writel(emc, 1, EMC_TIMING_CONTROL);
1155 
1156 	for (i = 0; i < emc->num_channels; i++)
1157 		tegra210_emc_wait_for_update(emc, i, EMC_EMC_STATUS,
1158 					     EMC_EMC_STATUS_TIMING_UPDATE_STALLED,
1159 					     0);
1160 
1161 	for (i = 0; i < emc->num_channels; i++) {
1162 		while (true) {
1163 			value = emc_channel_readl(emc, i, EMC_CFG_DIG_DLL);
1164 			if ((value & EMC_CFG_DIG_DLL_CFG_DLL_EN) == 0)
1165 				break;
1166 		}
1167 	}
1168 
1169 	value = emc->next->burst_regs[EMC_DLL_CFG_0_INDEX];
1170 	emc_writel(emc, value, EMC_DLL_CFG_0);
1171 
1172 	value = emc_readl(emc, EMC_DLL_CFG_1);
1173 	value &= EMC_DLL_CFG_1_DDLLCAL_CTRL_START_TRIM_MASK;
1174 
1175 	if (emc->next->rate >= 400000 && emc->next->rate < 600000)
1176 		value |= 150;
1177 	else if (emc->next->rate >= 600000 && emc->next->rate < 800000)
1178 		value |= 100;
1179 	else if (emc->next->rate >= 800000 && emc->next->rate < 1000000)
1180 		value |= 70;
1181 	else if (emc->next->rate >= 1000000 && emc->next->rate < 1200000)
1182 		value |= 30;
1183 	else
1184 		value |= 20;
1185 
1186 	emc_writel(emc, value, EMC_DLL_CFG_1);
1187 
1188 	tegra210_change_dll_src(emc, clksrc);
1189 
1190 	value = emc_readl(emc, EMC_CFG_DIG_DLL);
1191 	value |= EMC_CFG_DIG_DLL_CFG_DLL_EN;
1192 	emc_writel(emc, value, EMC_CFG_DIG_DLL);
1193 
1194 	tegra210_emc_timing_update(emc);
1195 
1196 	for (i = 0; i < emc->num_channels; i++) {
1197 		while (true) {
1198 			value = emc_channel_readl(emc, 0, EMC_CFG_DIG_DLL);
1199 			if (value & EMC_CFG_DIG_DLL_CFG_DLL_EN)
1200 				break;
1201 		}
1202 	}
1203 
1204 	while (true) {
1205 		value = emc_readl(emc, EMC_DIG_DLL_STATUS);
1206 
1207 		if ((value & EMC_DIG_DLL_STATUS_DLL_PRIV_UPDATED) == 0)
1208 			continue;
1209 
1210 		if ((value & EMC_DIG_DLL_STATUS_DLL_LOCK) == 0)
1211 			continue;
1212 
1213 		break;
1214 	}
1215 
1216 	value = emc_readl(emc, EMC_DIG_DLL_STATUS);
1217 
1218 	return value & EMC_DIG_DLL_STATUS_DLL_OUT_MASK;
1219 }
1220 
1221 u32 tegra210_emc_dvfs_power_ramp_up(struct tegra210_emc *emc, u32 clk,
1222 				    bool flip_backward)
1223 {
1224 	u32 cmd_pad, dq_pad, rfu1, cfg5, common_tx, ramp_up_wait = 0;
1225 	const struct tegra210_emc_timing *timing;
1226 
1227 	if (flip_backward)
1228 		timing = emc->last;
1229 	else
1230 		timing = emc->next;
1231 
1232 	cmd_pad = timing->burst_regs[EMC_PMACRO_CMD_PAD_TX_CTRL_INDEX];
1233 	dq_pad = timing->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX];
1234 	rfu1 = timing->burst_regs[EMC_PMACRO_BRICK_CTRL_RFU1_INDEX];
1235 	cfg5 = timing->burst_regs[EMC_FBIO_CFG5_INDEX];
1236 	common_tx = timing->burst_regs[EMC_PMACRO_COMMON_PAD_TX_CTRL_INDEX];
1237 
1238 	cmd_pad |= EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_DRVFORCEON;
1239 
1240 	if (clk < 1000000 / DVFS_FGCG_MID_SPEED_THRESHOLD) {
1241 		ccfifo_writel(emc, common_tx & 0xa,
1242 			      EMC_PMACRO_COMMON_PAD_TX_CTRL, 0);
1243 		ccfifo_writel(emc, common_tx & 0xf,
1244 			      EMC_PMACRO_COMMON_PAD_TX_CTRL,
1245 			      (100000 / clk) + 1);
1246 		ramp_up_wait += 100000;
1247 	} else {
1248 		ccfifo_writel(emc, common_tx | 0x8,
1249 			      EMC_PMACRO_COMMON_PAD_TX_CTRL, 0);
1250 	}
1251 
1252 	if (clk < 1000000 / DVFS_FGCG_HIGH_SPEED_THRESHOLD) {
1253 		if (clk < 1000000 / IOBRICK_DCC_THRESHOLD) {
1254 			cmd_pad |=
1255 				EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSP_TX_E_DCC |
1256 				EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSN_TX_E_DCC;
1257 			cmd_pad &=
1258 				~(EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_E_DCC |
1259 				  EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_CMD_TX_E_DCC);
1260 			ccfifo_writel(emc, cmd_pad,
1261 				      EMC_PMACRO_CMD_PAD_TX_CTRL,
1262 				      (100000 / clk) + 1);
1263 			ramp_up_wait += 100000;
1264 
1265 			dq_pad |=
1266 				EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSP_TX_E_DCC |
1267 				EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSN_TX_E_DCC;
1268 			dq_pad &=
1269 			       ~(EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_TX_E_DCC |
1270 				 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_CMD_TX_E_DCC);
1271 			ccfifo_writel(emc, dq_pad,
1272 				      EMC_PMACRO_DATA_PAD_TX_CTRL, 0);
1273 			ccfifo_writel(emc, rfu1 & 0xfe40fe40,
1274 				      EMC_PMACRO_BRICK_CTRL_RFU1, 0);
1275 		} else {
1276 			ccfifo_writel(emc, rfu1 & 0xfe40fe40,
1277 				      EMC_PMACRO_BRICK_CTRL_RFU1,
1278 				      (100000 / clk) + 1);
1279 			ramp_up_wait += 100000;
1280 		}
1281 
1282 		ccfifo_writel(emc, rfu1 & 0xfeedfeed,
1283 			      EMC_PMACRO_BRICK_CTRL_RFU1, (100000 / clk) + 1);
1284 		ramp_up_wait += 100000;
1285 
1286 		if (clk < 1000000 / IOBRICK_DCC_THRESHOLD) {
1287 			cmd_pad |=
1288 				EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSP_TX_E_DCC |
1289 				EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSN_TX_E_DCC |
1290 				EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_E_DCC |
1291 				EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_CMD_TX_E_DCC;
1292 			ccfifo_writel(emc, cmd_pad,
1293 				      EMC_PMACRO_CMD_PAD_TX_CTRL,
1294 				      (100000 / clk) + 1);
1295 			ramp_up_wait += 100000;
1296 
1297 			dq_pad |=
1298 				EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSP_TX_E_DCC |
1299 				EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSN_TX_E_DCC |
1300 				EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_TX_E_DCC |
1301 				EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_CMD_TX_E_DCC;
1302 			ccfifo_writel(emc, dq_pad,
1303 				      EMC_PMACRO_DATA_PAD_TX_CTRL, 0);
1304 			ccfifo_writel(emc, rfu1,
1305 				      EMC_PMACRO_BRICK_CTRL_RFU1, 0);
1306 		} else {
1307 			ccfifo_writel(emc, rfu1,
1308 				      EMC_PMACRO_BRICK_CTRL_RFU1,
1309 				      (100000 / clk) + 1);
1310 			ramp_up_wait += 100000;
1311 		}
1312 
1313 		ccfifo_writel(emc, cfg5 & ~EMC_FBIO_CFG5_CMD_TX_DIS,
1314 			      EMC_FBIO_CFG5, (100000 / clk) + 10);
1315 		ramp_up_wait += 100000 + (10 * clk);
1316 	} else if (clk < 1000000 / DVFS_FGCG_MID_SPEED_THRESHOLD) {
1317 		ccfifo_writel(emc, rfu1 | 0x06000600,
1318 			      EMC_PMACRO_BRICK_CTRL_RFU1, (100000 / clk) + 1);
1319 		ccfifo_writel(emc, cfg5 & ~EMC_FBIO_CFG5_CMD_TX_DIS,
1320 			      EMC_FBIO_CFG5, (100000 / clk) + 10);
1321 		ramp_up_wait += 100000 + 10 * clk;
1322 	} else {
1323 		ccfifo_writel(emc, rfu1 | 0x00000600,
1324 			      EMC_PMACRO_BRICK_CTRL_RFU1, 0);
1325 		ccfifo_writel(emc, cfg5 & ~EMC_FBIO_CFG5_CMD_TX_DIS,
1326 			      EMC_FBIO_CFG5, 12);
1327 		ramp_up_wait += 12 * clk;
1328 	}
1329 
1330 	cmd_pad &= ~EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_DRVFORCEON;
1331 	ccfifo_writel(emc, cmd_pad, EMC_PMACRO_CMD_PAD_TX_CTRL, 5);
1332 
1333 	return ramp_up_wait;
1334 }
1335 
1336 u32 tegra210_emc_dvfs_power_ramp_down(struct tegra210_emc *emc, u32 clk,
1337 				      bool flip_backward)
1338 {
1339 	u32 ramp_down_wait = 0, cmd_pad, dq_pad, rfu1, cfg5, common_tx;
1340 	const struct tegra210_emc_timing *entry;
1341 	u32 seq_wait;
1342 
1343 	if (flip_backward)
1344 		entry = emc->next;
1345 	else
1346 		entry = emc->last;
1347 
1348 	cmd_pad = entry->burst_regs[EMC_PMACRO_CMD_PAD_TX_CTRL_INDEX];
1349 	dq_pad = entry->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX];
1350 	rfu1 = entry->burst_regs[EMC_PMACRO_BRICK_CTRL_RFU1_INDEX];
1351 	cfg5 = entry->burst_regs[EMC_FBIO_CFG5_INDEX];
1352 	common_tx = entry->burst_regs[EMC_PMACRO_COMMON_PAD_TX_CTRL_INDEX];
1353 
1354 	cmd_pad |= EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_DRVFORCEON;
1355 
1356 	ccfifo_writel(emc, cmd_pad, EMC_PMACRO_CMD_PAD_TX_CTRL, 0);
1357 	ccfifo_writel(emc, cfg5 | EMC_FBIO_CFG5_CMD_TX_DIS,
1358 		      EMC_FBIO_CFG5, 12);
1359 	ramp_down_wait = 12 * clk;
1360 
1361 	seq_wait = (100000 / clk) + 1;
1362 
1363 	if (clk < (1000000 / DVFS_FGCG_HIGH_SPEED_THRESHOLD)) {
1364 		if (clk < (1000000 / IOBRICK_DCC_THRESHOLD)) {
1365 			cmd_pad &=
1366 				~(EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_E_DCC |
1367 				  EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_CMD_TX_E_DCC);
1368 			cmd_pad |=
1369 				EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSP_TX_E_DCC |
1370 				EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSN_TX_E_DCC;
1371 			ccfifo_writel(emc, cmd_pad,
1372 				      EMC_PMACRO_CMD_PAD_TX_CTRL, seq_wait);
1373 			ramp_down_wait += 100000;
1374 
1375 			dq_pad &=
1376 			      ~(EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_TX_E_DCC |
1377 				EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_CMD_TX_E_DCC);
1378 			dq_pad |=
1379 				EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSP_TX_E_DCC |
1380 				EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSN_TX_E_DCC;
1381 			ccfifo_writel(emc, dq_pad,
1382 				      EMC_PMACRO_DATA_PAD_TX_CTRL, 0);
1383 			ccfifo_writel(emc, rfu1 & ~0x01120112,
1384 				      EMC_PMACRO_BRICK_CTRL_RFU1, 0);
1385 		} else {
1386 			ccfifo_writel(emc, rfu1 & ~0x01120112,
1387 				      EMC_PMACRO_BRICK_CTRL_RFU1, seq_wait);
1388 			ramp_down_wait += 100000;
1389 		}
1390 
1391 		ccfifo_writel(emc, rfu1 & ~0x01bf01bf,
1392 			      EMC_PMACRO_BRICK_CTRL_RFU1, seq_wait);
1393 		ramp_down_wait += 100000;
1394 
1395 		if (clk < (1000000 / IOBRICK_DCC_THRESHOLD)) {
1396 			cmd_pad &=
1397 				~(EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_E_DCC |
1398 				  EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_CMD_TX_E_DCC |
1399 				  EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSP_TX_E_DCC |
1400 				  EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSN_TX_E_DCC);
1401 			ccfifo_writel(emc, cmd_pad,
1402 				      EMC_PMACRO_CMD_PAD_TX_CTRL, seq_wait);
1403 			ramp_down_wait += 100000;
1404 
1405 			dq_pad &=
1406 			      ~(EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_TX_E_DCC |
1407 				EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_CMD_TX_E_DCC |
1408 				EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSP_TX_E_DCC |
1409 				EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSN_TX_E_DCC);
1410 			ccfifo_writel(emc, dq_pad,
1411 				      EMC_PMACRO_DATA_PAD_TX_CTRL, 0);
1412 			ccfifo_writel(emc, rfu1 & ~0x07ff07ff,
1413 				      EMC_PMACRO_BRICK_CTRL_RFU1, 0);
1414 		} else {
1415 			ccfifo_writel(emc, rfu1 & ~0x07ff07ff,
1416 				      EMC_PMACRO_BRICK_CTRL_RFU1, seq_wait);
1417 			ramp_down_wait += 100000;
1418 		}
1419 	} else {
1420 		ccfifo_writel(emc, rfu1 & ~0xffff07ff,
1421 			      EMC_PMACRO_BRICK_CTRL_RFU1, seq_wait + 19);
1422 		ramp_down_wait += 100000 + (20 * clk);
1423 	}
1424 
1425 	if (clk < (1000000 / DVFS_FGCG_MID_SPEED_THRESHOLD)) {
1426 		ramp_down_wait += 100000;
1427 		ccfifo_writel(emc, common_tx & ~0x5,
1428 			      EMC_PMACRO_COMMON_PAD_TX_CTRL, seq_wait);
1429 		ramp_down_wait += 100000;
1430 		ccfifo_writel(emc, common_tx & ~0xf,
1431 			      EMC_PMACRO_COMMON_PAD_TX_CTRL, seq_wait);
1432 		ramp_down_wait += 100000;
1433 		ccfifo_writel(emc, 0, 0, seq_wait);
1434 		ramp_down_wait += 100000;
1435 	} else {
1436 		ccfifo_writel(emc, common_tx & ~0xf,
1437 			      EMC_PMACRO_COMMON_PAD_TX_CTRL, seq_wait);
1438 	}
1439 
1440 	return ramp_down_wait;
1441 }
1442 
1443 void tegra210_emc_reset_dram_clktree_values(struct tegra210_emc_timing *timing)
1444 {
1445 	timing->current_dram_clktree[C0D0U0] =
1446 		timing->trained_dram_clktree[C0D0U0];
1447 	timing->current_dram_clktree[C0D0U1] =
1448 		timing->trained_dram_clktree[C0D0U1];
1449 	timing->current_dram_clktree[C1D0U0] =
1450 		timing->trained_dram_clktree[C1D0U0];
1451 	timing->current_dram_clktree[C1D0U1] =
1452 		timing->trained_dram_clktree[C1D0U1];
1453 	timing->current_dram_clktree[C1D1U0] =
1454 		timing->trained_dram_clktree[C1D1U0];
1455 	timing->current_dram_clktree[C1D1U1] =
1456 		timing->trained_dram_clktree[C1D1U1];
1457 }
1458 
1459 static void update_dll_control(struct tegra210_emc *emc, u32 value, bool state)
1460 {
1461 	unsigned int i;
1462 
1463 	emc_writel(emc, value, EMC_CFG_DIG_DLL);
1464 	tegra210_emc_timing_update(emc);
1465 
1466 	for (i = 0; i < emc->num_channels; i++)
1467 		tegra210_emc_wait_for_update(emc, i, EMC_CFG_DIG_DLL,
1468 					     EMC_CFG_DIG_DLL_CFG_DLL_EN,
1469 					     state);
1470 }
1471 
1472 void tegra210_emc_dll_disable(struct tegra210_emc *emc)
1473 {
1474 	u32 value;
1475 
1476 	value = emc_readl(emc, EMC_CFG_DIG_DLL);
1477 	value &= ~EMC_CFG_DIG_DLL_CFG_DLL_EN;
1478 
1479 	update_dll_control(emc, value, false);
1480 }
1481 
1482 void tegra210_emc_dll_enable(struct tegra210_emc *emc)
1483 {
1484 	u32 value;
1485 
1486 	value = emc_readl(emc, EMC_CFG_DIG_DLL);
1487 	value |= EMC_CFG_DIG_DLL_CFG_DLL_EN;
1488 
1489 	update_dll_control(emc, value, true);
1490 }
1491 
1492 void tegra210_emc_adjust_timing(struct tegra210_emc *emc,
1493 				struct tegra210_emc_timing *timing)
1494 {
1495 	u32 dsr_cntrl = timing->burst_regs[EMC_DYN_SELF_REF_CONTROL_INDEX];
1496 	u32 pre_ref = timing->burst_regs[EMC_PRE_REFRESH_REQ_CNT_INDEX];
1497 	u32 ref = timing->burst_regs[EMC_REFRESH_INDEX];
1498 
1499 	switch (emc->refresh) {
1500 	case TEGRA210_EMC_REFRESH_NOMINAL:
1501 	case TEGRA210_EMC_REFRESH_THROTTLE:
1502 		break;
1503 
1504 	case TEGRA210_EMC_REFRESH_2X:
1505 		ref = REFRESH_SPEEDUP(ref, 2);
1506 		pre_ref = REFRESH_SPEEDUP(pre_ref, 2);
1507 		dsr_cntrl = REFRESH_SPEEDUP(dsr_cntrl, 2);
1508 		break;
1509 
1510 	case TEGRA210_EMC_REFRESH_4X:
1511 		ref = REFRESH_SPEEDUP(ref, 4);
1512 		pre_ref = REFRESH_SPEEDUP(pre_ref, 4);
1513 		dsr_cntrl = REFRESH_SPEEDUP(dsr_cntrl, 4);
1514 		break;
1515 
1516 	default:
1517 		dev_warn(emc->dev, "failed to set refresh: %d\n", emc->refresh);
1518 		return;
1519 	}
1520 
1521 	emc_writel(emc, ref, emc->offsets->burst[EMC_REFRESH_INDEX]);
1522 	emc_writel(emc, pre_ref,
1523 		   emc->offsets->burst[EMC_PRE_REFRESH_REQ_CNT_INDEX]);
1524 	emc_writel(emc, dsr_cntrl,
1525 		   emc->offsets->burst[EMC_DYN_SELF_REF_CONTROL_INDEX]);
1526 }
1527 
1528 static int tegra210_emc_set_rate(struct device *dev,
1529 				 const struct tegra210_clk_emc_config *config)
1530 {
1531 	struct tegra210_emc *emc = dev_get_drvdata(dev);
1532 	struct tegra210_emc_timing *timing = NULL;
1533 	unsigned long rate = config->rate;
1534 	s64 last_change_delay;
1535 	unsigned long flags;
1536 	unsigned int i;
1537 
1538 	if (rate == emc->last->rate * 1000UL)
1539 		return 0;
1540 
1541 	for (i = 0; i < emc->num_timings; i++) {
1542 		if (emc->timings[i].rate * 1000UL == rate) {
1543 			timing = &emc->timings[i];
1544 			break;
1545 		}
1546 	}
1547 
1548 	if (!timing)
1549 		return -EINVAL;
1550 
1551 	if (rate > 204000000 && !timing->trained)
1552 		return -EINVAL;
1553 
1554 	emc->next = timing;
1555 	last_change_delay = ktime_us_delta(ktime_get(), emc->clkchange_time);
1556 
1557 	/* XXX use non-busy-looping sleep? */
1558 	if ((last_change_delay >= 0) &&
1559 	    (last_change_delay < emc->clkchange_delay))
1560 		udelay(emc->clkchange_delay - (int)last_change_delay);
1561 
1562 	spin_lock_irqsave(&emc->lock, flags);
1563 	tegra210_emc_set_clock(emc, config->value);
1564 	emc->clkchange_time = ktime_get();
1565 	emc->last = timing;
1566 	spin_unlock_irqrestore(&emc->lock, flags);
1567 
1568 	return 0;
1569 }
1570 
1571 /*
1572  * debugfs interface
1573  *
1574  * The memory controller driver exposes some files in debugfs that can be used
1575  * to control the EMC frequency. The top-level directory can be found here:
1576  *
1577  *   /sys/kernel/debug/emc
1578  *
1579  * It contains the following files:
1580  *
1581  *   - available_rates: This file contains a list of valid, space-separated
1582  *     EMC frequencies.
1583  *
1584  *   - min_rate: Writing a value to this file sets the given frequency as the
1585  *       floor of the permitted range. If this is higher than the currently
1586  *       configured EMC frequency, this will cause the frequency to be
1587  *       increased so that it stays within the valid range.
1588  *
1589  *   - max_rate: Similarily to the min_rate file, writing a value to this file
1590  *       sets the given frequency as the ceiling of the permitted range. If
1591  *       the value is lower than the currently configured EMC frequency, this
1592  *       will cause the frequency to be decreased so that it stays within the
1593  *       valid range.
1594  */
1595 
1596 static bool tegra210_emc_validate_rate(struct tegra210_emc *emc,
1597 				       unsigned long rate)
1598 {
1599 	unsigned int i;
1600 
1601 	for (i = 0; i < emc->num_timings; i++)
1602 		if (rate == emc->timings[i].rate * 1000UL)
1603 			return true;
1604 
1605 	return false;
1606 }
1607 
1608 static int tegra210_emc_debug_available_rates_show(struct seq_file *s,
1609 						   void *data)
1610 {
1611 	struct tegra210_emc *emc = s->private;
1612 	const char *prefix = "";
1613 	unsigned int i;
1614 
1615 	for (i = 0; i < emc->num_timings; i++) {
1616 		seq_printf(s, "%s%u", prefix, emc->timings[i].rate * 1000);
1617 		prefix = " ";
1618 	}
1619 
1620 	seq_puts(s, "\n");
1621 
1622 	return 0;
1623 }
1624 
1625 static int tegra210_emc_debug_available_rates_open(struct inode *inode,
1626 						   struct file *file)
1627 {
1628 	return single_open(file, tegra210_emc_debug_available_rates_show,
1629 			   inode->i_private);
1630 }
1631 
1632 static const struct file_operations tegra210_emc_debug_available_rates_fops = {
1633 	.open = tegra210_emc_debug_available_rates_open,
1634 	.read = seq_read,
1635 	.llseek = seq_lseek,
1636 	.release = single_release,
1637 };
1638 
1639 static int tegra210_emc_debug_min_rate_get(void *data, u64 *rate)
1640 {
1641 	struct tegra210_emc *emc = data;
1642 
1643 	*rate = emc->debugfs.min_rate;
1644 
1645 	return 0;
1646 }
1647 
1648 static int tegra210_emc_debug_min_rate_set(void *data, u64 rate)
1649 {
1650 	struct tegra210_emc *emc = data;
1651 	int err;
1652 
1653 	if (!tegra210_emc_validate_rate(emc, rate))
1654 		return -EINVAL;
1655 
1656 	err = clk_set_min_rate(emc->clk, rate);
1657 	if (err < 0)
1658 		return err;
1659 
1660 	emc->debugfs.min_rate = rate;
1661 
1662 	return 0;
1663 }
1664 
1665 DEFINE_DEBUGFS_ATTRIBUTE(tegra210_emc_debug_min_rate_fops,
1666 			tegra210_emc_debug_min_rate_get,
1667 			tegra210_emc_debug_min_rate_set, "%llu\n");
1668 
1669 static int tegra210_emc_debug_max_rate_get(void *data, u64 *rate)
1670 {
1671 	struct tegra210_emc *emc = data;
1672 
1673 	*rate = emc->debugfs.max_rate;
1674 
1675 	return 0;
1676 }
1677 
1678 static int tegra210_emc_debug_max_rate_set(void *data, u64 rate)
1679 {
1680 	struct tegra210_emc *emc = data;
1681 	int err;
1682 
1683 	if (!tegra210_emc_validate_rate(emc, rate))
1684 		return -EINVAL;
1685 
1686 	err = clk_set_max_rate(emc->clk, rate);
1687 	if (err < 0)
1688 		return err;
1689 
1690 	emc->debugfs.max_rate = rate;
1691 
1692 	return 0;
1693 }
1694 
1695 DEFINE_DEBUGFS_ATTRIBUTE(tegra210_emc_debug_max_rate_fops,
1696 			tegra210_emc_debug_max_rate_get,
1697 			tegra210_emc_debug_max_rate_set, "%llu\n");
1698 
1699 static int tegra210_emc_debug_temperature_get(void *data, u64 *temperature)
1700 {
1701 	struct tegra210_emc *emc = data;
1702 	unsigned int value;
1703 
1704 	if (!emc->debugfs.temperature)
1705 		value = tegra210_emc_get_temperature(emc);
1706 	else
1707 		value = emc->debugfs.temperature;
1708 
1709 	*temperature = value;
1710 
1711 	return 0;
1712 }
1713 
1714 static int tegra210_emc_debug_temperature_set(void *data, u64 temperature)
1715 {
1716 	struct tegra210_emc *emc = data;
1717 
1718 	if (temperature > 7)
1719 		return -EINVAL;
1720 
1721 	emc->debugfs.temperature = temperature;
1722 
1723 	return 0;
1724 }
1725 
1726 DEFINE_DEBUGFS_ATTRIBUTE(tegra210_emc_debug_temperature_fops,
1727 			tegra210_emc_debug_temperature_get,
1728 			tegra210_emc_debug_temperature_set, "%llu\n");
1729 
1730 static void tegra210_emc_debugfs_init(struct tegra210_emc *emc)
1731 {
1732 	struct device *dev = emc->dev;
1733 	unsigned int i;
1734 	int err;
1735 
1736 	emc->debugfs.min_rate = ULONG_MAX;
1737 	emc->debugfs.max_rate = 0;
1738 
1739 	for (i = 0; i < emc->num_timings; i++) {
1740 		if (emc->timings[i].rate * 1000UL < emc->debugfs.min_rate)
1741 			emc->debugfs.min_rate = emc->timings[i].rate * 1000UL;
1742 
1743 		if (emc->timings[i].rate * 1000UL > emc->debugfs.max_rate)
1744 			emc->debugfs.max_rate = emc->timings[i].rate * 1000UL;
1745 	}
1746 
1747 	if (!emc->num_timings) {
1748 		emc->debugfs.min_rate = clk_get_rate(emc->clk);
1749 		emc->debugfs.max_rate = emc->debugfs.min_rate;
1750 	}
1751 
1752 	err = clk_set_rate_range(emc->clk, emc->debugfs.min_rate,
1753 				 emc->debugfs.max_rate);
1754 	if (err < 0) {
1755 		dev_err(dev, "failed to set rate range [%lu-%lu] for %pC\n",
1756 			emc->debugfs.min_rate, emc->debugfs.max_rate,
1757 			emc->clk);
1758 		return;
1759 	}
1760 
1761 	emc->debugfs.root = debugfs_create_dir("emc", NULL);
1762 
1763 	debugfs_create_file("available_rates", 0444, emc->debugfs.root, emc,
1764 			    &tegra210_emc_debug_available_rates_fops);
1765 	debugfs_create_file("min_rate", 0644, emc->debugfs.root, emc,
1766 			    &tegra210_emc_debug_min_rate_fops);
1767 	debugfs_create_file("max_rate", 0644, emc->debugfs.root, emc,
1768 			    &tegra210_emc_debug_max_rate_fops);
1769 	debugfs_create_file("temperature", 0644, emc->debugfs.root, emc,
1770 			    &tegra210_emc_debug_temperature_fops);
1771 }
1772 
1773 static void tegra210_emc_detect(struct tegra210_emc *emc)
1774 {
1775 	u32 value;
1776 
1777 	/* probe the number of connected DRAM devices */
1778 	value = mc_readl(emc->mc, MC_EMEM_ADR_CFG);
1779 
1780 	if (value & MC_EMEM_ADR_CFG_EMEM_NUMDEV)
1781 		emc->num_devices = 2;
1782 	else
1783 		emc->num_devices = 1;
1784 
1785 	/* probe the type of DRAM */
1786 	value = emc_readl(emc, EMC_FBIO_CFG5);
1787 	emc->dram_type = value & 0x3;
1788 
1789 	/* probe the number of channels */
1790 	value = emc_readl(emc, EMC_FBIO_CFG7);
1791 
1792 	if ((value & EMC_FBIO_CFG7_CH1_ENABLE) &&
1793 	    (value & EMC_FBIO_CFG7_CH0_ENABLE))
1794 		emc->num_channels = 2;
1795 	else
1796 		emc->num_channels = 1;
1797 }
1798 
1799 static int tegra210_emc_validate_timings(struct tegra210_emc *emc,
1800 					 struct tegra210_emc_timing *timings,
1801 					 unsigned int num_timings)
1802 {
1803 	unsigned int i;
1804 
1805 	for (i = 0; i < num_timings; i++) {
1806 		u32 min_volt = timings[i].min_volt;
1807 		u32 rate = timings[i].rate;
1808 
1809 		if (!rate)
1810 			return -EINVAL;
1811 
1812 		if ((i > 0) && ((rate <= timings[i - 1].rate) ||
1813 		    (min_volt < timings[i - 1].min_volt)))
1814 			return -EINVAL;
1815 
1816 		if (timings[i].revision != timings[0].revision)
1817 			continue;
1818 	}
1819 
1820 	return 0;
1821 }
1822 
1823 static int tegra210_emc_probe(struct platform_device *pdev)
1824 {
1825 	struct thermal_cooling_device *cd;
1826 	unsigned long current_rate;
1827 	struct tegra210_emc *emc;
1828 	struct device_node *np;
1829 	unsigned int i;
1830 	int err;
1831 
1832 	emc = devm_kzalloc(&pdev->dev, sizeof(*emc), GFP_KERNEL);
1833 	if (!emc)
1834 		return -ENOMEM;
1835 
1836 	emc->clk = devm_clk_get(&pdev->dev, "emc");
1837 	if (IS_ERR(emc->clk))
1838 		return PTR_ERR(emc->clk);
1839 
1840 	platform_set_drvdata(pdev, emc);
1841 	spin_lock_init(&emc->lock);
1842 	emc->dev = &pdev->dev;
1843 
1844 	emc->mc = devm_tegra_memory_controller_get(&pdev->dev);
1845 	if (IS_ERR(emc->mc))
1846 		return PTR_ERR(emc->mc);
1847 
1848 	emc->regs = devm_platform_ioremap_resource(pdev, 0);
1849 	if (IS_ERR(emc->regs))
1850 		return PTR_ERR(emc->regs);
1851 
1852 	for (i = 0; i < 2; i++) {
1853 		emc->channel[i] = devm_platform_ioremap_resource(pdev, 1 + i);
1854 		if (IS_ERR(emc->channel[i]))
1855 			return PTR_ERR(emc->channel[i]);
1856 
1857 	}
1858 
1859 	tegra210_emc_detect(emc);
1860 	np = pdev->dev.of_node;
1861 
1862 	/* attach to the nominal and (optional) derated tables */
1863 	err = of_reserved_mem_device_init_by_name(emc->dev, np, "nominal");
1864 	if (err < 0) {
1865 		dev_err(emc->dev, "failed to get nominal EMC table: %d\n", err);
1866 		return err;
1867 	}
1868 
1869 	err = of_reserved_mem_device_init_by_name(emc->dev, np, "derated");
1870 	if (err < 0 && err != -ENODEV) {
1871 		dev_err(emc->dev, "failed to get derated EMC table: %d\n", err);
1872 		goto release;
1873 	}
1874 
1875 	/* validate the tables */
1876 	if (emc->nominal) {
1877 		err = tegra210_emc_validate_timings(emc, emc->nominal,
1878 						    emc->num_timings);
1879 		if (err < 0)
1880 			goto release;
1881 	}
1882 
1883 	if (emc->derated) {
1884 		err = tegra210_emc_validate_timings(emc, emc->derated,
1885 						    emc->num_timings);
1886 		if (err < 0)
1887 			goto release;
1888 	}
1889 
1890 	/* default to the nominal table */
1891 	emc->timings = emc->nominal;
1892 
1893 	/* pick the current timing based on the current EMC clock rate */
1894 	current_rate = clk_get_rate(emc->clk) / 1000;
1895 
1896 	for (i = 0; i < emc->num_timings; i++) {
1897 		if (emc->timings[i].rate == current_rate) {
1898 			emc->last = &emc->timings[i];
1899 			break;
1900 		}
1901 	}
1902 
1903 	if (i == emc->num_timings) {
1904 		dev_err(emc->dev, "no EMC table entry found for %lu kHz\n",
1905 			current_rate);
1906 		err = -ENOENT;
1907 		goto release;
1908 	}
1909 
1910 	/* pick a compatible clock change sequence for the EMC table */
1911 	for (i = 0; i < ARRAY_SIZE(tegra210_emc_sequences); i++) {
1912 		const struct tegra210_emc_sequence *sequence =
1913 				tegra210_emc_sequences[i];
1914 
1915 		if (emc->timings[0].revision == sequence->revision) {
1916 			emc->sequence = sequence;
1917 			break;
1918 		}
1919 	}
1920 
1921 	if (!emc->sequence) {
1922 		dev_err(&pdev->dev, "sequence %u not supported\n",
1923 			emc->timings[0].revision);
1924 		err = -ENOTSUPP;
1925 		goto release;
1926 	}
1927 
1928 	emc->offsets = &tegra210_emc_table_register_offsets;
1929 	emc->refresh = TEGRA210_EMC_REFRESH_NOMINAL;
1930 
1931 	emc->provider.owner = THIS_MODULE;
1932 	emc->provider.dev = &pdev->dev;
1933 	emc->provider.set_rate = tegra210_emc_set_rate;
1934 
1935 	emc->provider.configs = devm_kcalloc(&pdev->dev, emc->num_timings,
1936 					     sizeof(*emc->provider.configs),
1937 					     GFP_KERNEL);
1938 	if (!emc->provider.configs) {
1939 		err = -ENOMEM;
1940 		goto release;
1941 	}
1942 
1943 	emc->provider.num_configs = emc->num_timings;
1944 
1945 	for (i = 0; i < emc->provider.num_configs; i++) {
1946 		struct tegra210_emc_timing *timing = &emc->timings[i];
1947 		struct tegra210_clk_emc_config *config =
1948 				&emc->provider.configs[i];
1949 		u32 value;
1950 
1951 		config->rate = timing->rate * 1000UL;
1952 		config->value = timing->clk_src_emc;
1953 
1954 		value = timing->burst_mc_regs[MC_EMEM_ARB_MISC0_INDEX];
1955 
1956 		if ((value & MC_EMEM_ARB_MISC0_EMC_SAME_FREQ) == 0)
1957 			config->same_freq = false;
1958 		else
1959 			config->same_freq = true;
1960 	}
1961 
1962 	err = tegra210_clk_emc_attach(emc->clk, &emc->provider);
1963 	if (err < 0) {
1964 		dev_err(&pdev->dev, "failed to attach to EMC clock: %d\n", err);
1965 		goto release;
1966 	}
1967 
1968 	emc->clkchange_delay = 100;
1969 	emc->training_interval = 100;
1970 	dev_set_drvdata(emc->dev, emc);
1971 
1972 	timer_setup(&emc->refresh_timer, tegra210_emc_poll_refresh,
1973 		    TIMER_DEFERRABLE);
1974 	atomic_set(&emc->refresh_poll, 0);
1975 	emc->refresh_poll_interval = 1000;
1976 
1977 	timer_setup(&emc->training, tegra210_emc_train, 0);
1978 
1979 	tegra210_emc_debugfs_init(emc);
1980 
1981 	cd = devm_thermal_of_cooling_device_register(emc->dev, np, "emc", emc,
1982 						     &tegra210_emc_cd_ops);
1983 	if (IS_ERR(cd)) {
1984 		err = PTR_ERR(cd);
1985 		dev_err(emc->dev, "failed to register cooling device: %d\n",
1986 			err);
1987 		goto detach;
1988 	}
1989 
1990 	return 0;
1991 
1992 detach:
1993 	debugfs_remove_recursive(emc->debugfs.root);
1994 	tegra210_clk_emc_detach(emc->clk);
1995 release:
1996 	of_reserved_mem_device_release(emc->dev);
1997 
1998 	return err;
1999 }
2000 
2001 static int tegra210_emc_remove(struct platform_device *pdev)
2002 {
2003 	struct tegra210_emc *emc = platform_get_drvdata(pdev);
2004 
2005 	debugfs_remove_recursive(emc->debugfs.root);
2006 	tegra210_clk_emc_detach(emc->clk);
2007 	of_reserved_mem_device_release(emc->dev);
2008 
2009 	return 0;
2010 }
2011 
2012 static int __maybe_unused tegra210_emc_suspend(struct device *dev)
2013 {
2014 	struct tegra210_emc *emc = dev_get_drvdata(dev);
2015 	int err;
2016 
2017 	err = clk_rate_exclusive_get(emc->clk);
2018 	if (err < 0) {
2019 		dev_err(emc->dev, "failed to acquire clock: %d\n", err);
2020 		return err;
2021 	}
2022 
2023 	emc->resume_rate = clk_get_rate(emc->clk);
2024 
2025 	clk_set_rate(emc->clk, 204000000);
2026 	tegra210_clk_emc_detach(emc->clk);
2027 
2028 	dev_dbg(dev, "suspending at %lu Hz\n", clk_get_rate(emc->clk));
2029 
2030 	return 0;
2031 }
2032 
2033 static int __maybe_unused tegra210_emc_resume(struct device *dev)
2034 {
2035 	struct tegra210_emc *emc = dev_get_drvdata(dev);
2036 	int err;
2037 
2038 	err = tegra210_clk_emc_attach(emc->clk, &emc->provider);
2039 	if (err < 0) {
2040 		dev_err(dev, "failed to attach to EMC clock: %d\n", err);
2041 		return err;
2042 	}
2043 
2044 	clk_set_rate(emc->clk, emc->resume_rate);
2045 	clk_rate_exclusive_put(emc->clk);
2046 
2047 	dev_dbg(dev, "resuming at %lu Hz\n", clk_get_rate(emc->clk));
2048 
2049 	return 0;
2050 }
2051 
2052 static const struct dev_pm_ops tegra210_emc_pm_ops = {
2053 	SET_SYSTEM_SLEEP_PM_OPS(tegra210_emc_suspend, tegra210_emc_resume)
2054 };
2055 
2056 static const struct of_device_id tegra210_emc_of_match[] = {
2057 	{ .compatible = "nvidia,tegra210-emc", },
2058 	{ },
2059 };
2060 MODULE_DEVICE_TABLE(of, tegra210_emc_of_match);
2061 
2062 static struct platform_driver tegra210_emc_driver = {
2063 	.driver = {
2064 		.name = "tegra210-emc",
2065 		.of_match_table = tegra210_emc_of_match,
2066 		.pm = &tegra210_emc_pm_ops,
2067 	},
2068 	.probe = tegra210_emc_probe,
2069 	.remove = tegra210_emc_remove,
2070 };
2071 
2072 module_platform_driver(tegra210_emc_driver);
2073 
2074 MODULE_AUTHOR("Thierry Reding <treding@nvidia.com>");
2075 MODULE_AUTHOR("Joseph Lo <josephl@nvidia.com>");
2076 MODULE_DESCRIPTION("NVIDIA Tegra210 EMC driver");
2077 MODULE_LICENSE("GPL v2");
2078