1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (c) 2015-2020, NVIDIA CORPORATION.  All rights reserved.
4  */
5 
6 #include <linux/bitfield.h>
7 #include <linux/clk.h>
8 #include <linux/clk/tegra.h>
9 #include <linux/debugfs.h>
10 #include <linux/delay.h>
11 #include <linux/kernel.h>
12 #include <linux/mod_devicetable.h>
13 #include <linux/module.h>
14 #include <linux/of_reserved_mem.h>
15 #include <linux/platform_device.h>
16 #include <linux/slab.h>
17 #include <linux/thermal.h>
18 #include <soc/tegra/fuse.h>
19 #include <soc/tegra/mc.h>
20 
21 #include "tegra210-emc.h"
22 #include "tegra210-mc.h"
23 
24 /* CLK_RST_CONTROLLER_CLK_SOURCE_EMC */
25 #define EMC_CLK_EMC_2X_CLK_SRC_SHIFT			29
26 #define EMC_CLK_EMC_2X_CLK_SRC_MASK			\
27 	(0x7 << EMC_CLK_EMC_2X_CLK_SRC_SHIFT)
28 #define EMC_CLK_SOURCE_PLLM_LJ				0x4
29 #define EMC_CLK_SOURCE_PLLMB_LJ				0x5
30 #define EMC_CLK_FORCE_CC_TRIGGER			BIT(27)
31 #define EMC_CLK_MC_EMC_SAME_FREQ			BIT(16)
32 #define EMC_CLK_EMC_2X_CLK_DIVISOR_SHIFT		0
33 #define EMC_CLK_EMC_2X_CLK_DIVISOR_MASK			\
34 	(0xff << EMC_CLK_EMC_2X_CLK_DIVISOR_SHIFT)
35 
36 /* CLK_RST_CONTROLLER_CLK_SOURCE_EMC_DLL */
37 #define DLL_CLK_EMC_DLL_CLK_SRC_SHIFT			29
38 #define DLL_CLK_EMC_DLL_CLK_SRC_MASK			\
39 	(0x7 << DLL_CLK_EMC_DLL_CLK_SRC_SHIFT)
40 #define DLL_CLK_EMC_DLL_DDLL_CLK_SEL_SHIFT		10
41 #define DLL_CLK_EMC_DLL_DDLL_CLK_SEL_MASK		\
42 	(0x3 << DLL_CLK_EMC_DLL_DDLL_CLK_SEL_SHIFT)
43 #define PLLM_VCOA					0
44 #define PLLM_VCOB					1
45 #define EMC_DLL_SWITCH_OUT				2
46 #define DLL_CLK_EMC_DLL_CLK_DIVISOR_SHIFT		0
47 #define DLL_CLK_EMC_DLL_CLK_DIVISOR_MASK		\
48 	(0xff << DLL_CLK_EMC_DLL_CLK_DIVISOR_SHIFT)
49 
50 /* MC_EMEM_ARB_MISC0 */
51 #define MC_EMEM_ARB_MISC0_EMC_SAME_FREQ			BIT(27)
52 
53 /* EMC_DATA_BRLSHFT_X */
54 #define EMC0_EMC_DATA_BRLSHFT_0_INDEX	2
55 #define EMC1_EMC_DATA_BRLSHFT_0_INDEX	3
56 #define EMC0_EMC_DATA_BRLSHFT_1_INDEX	4
57 #define EMC1_EMC_DATA_BRLSHFT_1_INDEX	5
58 
59 #define TRIM_REG(chan, rank, reg, byte)					\
60 	(((EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## rank ## _ ## reg ##	\
61 	   _OB_DDLL_LONG_DQ_RANK ## rank ## _BYTE ## byte ## _MASK &	\
62 	   next->trim_regs[EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ##		\
63 				 rank ## _ ## reg ## _INDEX]) >>	\
64 	  EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## rank ## _ ## reg ##	\
65 	  _OB_DDLL_LONG_DQ_RANK ## rank ## _BYTE ## byte ## _SHIFT)	\
66 	 +								\
67 	 (((EMC_DATA_BRLSHFT_ ## rank ## _RANK ## rank ## _BYTE ##	\
68 	    byte ## _DATA_BRLSHFT_MASK &				\
69 	    next->trim_perch_regs[EMC ## chan ##			\
70 			      _EMC_DATA_BRLSHFT_ ## rank ## _INDEX]) >>	\
71 	   EMC_DATA_BRLSHFT_ ## rank ## _RANK ## rank ## _BYTE ##	\
72 	   byte ## _DATA_BRLSHFT_SHIFT) * 64))
73 
74 #define CALC_TEMP(rank, reg, byte1, byte2, n)				\
75 	(((new[n] << EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## rank ## _ ##	\
76 	   reg ## _OB_DDLL_LONG_DQ_RANK ## rank ## _BYTE ## byte1 ## _SHIFT) & \
77 	  EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## rank ## _ ## reg ##	\
78 	  _OB_DDLL_LONG_DQ_RANK ## rank ## _BYTE ## byte1 ## _MASK)	\
79 	 |								\
80 	 ((new[n + 1] << EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## rank ## _ ##\
81 	   reg ## _OB_DDLL_LONG_DQ_RANK ## rank ## _BYTE ## byte2 ## _SHIFT) & \
82 	  EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## rank ## _ ## reg ##	\
83 	  _OB_DDLL_LONG_DQ_RANK ## rank ## _BYTE ## byte2 ## _MASK))
84 
85 #define REFRESH_SPEEDUP(value, speedup) \
86 		(((value) & 0xffff0000) | ((value) & 0xffff) * (speedup))
87 
88 #define LPDDR2_MR4_SRR GENMASK(2, 0)
89 
90 static const struct tegra210_emc_sequence *tegra210_emc_sequences[] = {
91 	&tegra210_emc_r21021,
92 };
93 
94 static const struct tegra210_emc_table_register_offsets
95 tegra210_emc_table_register_offsets = {
96 	.burst = {
97 		EMC_RC,
98 		EMC_RFC,
99 		EMC_RFCPB,
100 		EMC_REFCTRL2,
101 		EMC_RFC_SLR,
102 		EMC_RAS,
103 		EMC_RP,
104 		EMC_R2W,
105 		EMC_W2R,
106 		EMC_R2P,
107 		EMC_W2P,
108 		EMC_R2R,
109 		EMC_TPPD,
110 		EMC_CCDMW,
111 		EMC_RD_RCD,
112 		EMC_WR_RCD,
113 		EMC_RRD,
114 		EMC_REXT,
115 		EMC_WEXT,
116 		EMC_WDV_CHK,
117 		EMC_WDV,
118 		EMC_WSV,
119 		EMC_WEV,
120 		EMC_WDV_MASK,
121 		EMC_WS_DURATION,
122 		EMC_WE_DURATION,
123 		EMC_QUSE,
124 		EMC_QUSE_WIDTH,
125 		EMC_IBDLY,
126 		EMC_OBDLY,
127 		EMC_EINPUT,
128 		EMC_MRW6,
129 		EMC_EINPUT_DURATION,
130 		EMC_PUTERM_EXTRA,
131 		EMC_PUTERM_WIDTH,
132 		EMC_QRST,
133 		EMC_QSAFE,
134 		EMC_RDV,
135 		EMC_RDV_MASK,
136 		EMC_RDV_EARLY,
137 		EMC_RDV_EARLY_MASK,
138 		EMC_REFRESH,
139 		EMC_BURST_REFRESH_NUM,
140 		EMC_PRE_REFRESH_REQ_CNT,
141 		EMC_PDEX2WR,
142 		EMC_PDEX2RD,
143 		EMC_PCHG2PDEN,
144 		EMC_ACT2PDEN,
145 		EMC_AR2PDEN,
146 		EMC_RW2PDEN,
147 		EMC_CKE2PDEN,
148 		EMC_PDEX2CKE,
149 		EMC_PDEX2MRR,
150 		EMC_TXSR,
151 		EMC_TXSRDLL,
152 		EMC_TCKE,
153 		EMC_TCKESR,
154 		EMC_TPD,
155 		EMC_TFAW,
156 		EMC_TRPAB,
157 		EMC_TCLKSTABLE,
158 		EMC_TCLKSTOP,
159 		EMC_MRW7,
160 		EMC_TREFBW,
161 		EMC_ODT_WRITE,
162 		EMC_FBIO_CFG5,
163 		EMC_FBIO_CFG7,
164 		EMC_CFG_DIG_DLL,
165 		EMC_CFG_DIG_DLL_PERIOD,
166 		EMC_PMACRO_IB_RXRT,
167 		EMC_CFG_PIPE_1,
168 		EMC_CFG_PIPE_2,
169 		EMC_PMACRO_QUSE_DDLL_RANK0_4,
170 		EMC_PMACRO_QUSE_DDLL_RANK0_5,
171 		EMC_PMACRO_QUSE_DDLL_RANK1_4,
172 		EMC_PMACRO_QUSE_DDLL_RANK1_5,
173 		EMC_MRW8,
174 		EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_4,
175 		EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_5,
176 		EMC_PMACRO_OB_DDLL_LONG_DQS_RANK0_0,
177 		EMC_PMACRO_OB_DDLL_LONG_DQS_RANK0_1,
178 		EMC_PMACRO_OB_DDLL_LONG_DQS_RANK0_2,
179 		EMC_PMACRO_OB_DDLL_LONG_DQS_RANK0_3,
180 		EMC_PMACRO_OB_DDLL_LONG_DQS_RANK0_4,
181 		EMC_PMACRO_OB_DDLL_LONG_DQS_RANK0_5,
182 		EMC_PMACRO_OB_DDLL_LONG_DQS_RANK1_0,
183 		EMC_PMACRO_OB_DDLL_LONG_DQS_RANK1_1,
184 		EMC_PMACRO_OB_DDLL_LONG_DQS_RANK1_2,
185 		EMC_PMACRO_OB_DDLL_LONG_DQS_RANK1_3,
186 		EMC_PMACRO_OB_DDLL_LONG_DQS_RANK1_4,
187 		EMC_PMACRO_OB_DDLL_LONG_DQS_RANK1_5,
188 		EMC_PMACRO_DDLL_LONG_CMD_0,
189 		EMC_PMACRO_DDLL_LONG_CMD_1,
190 		EMC_PMACRO_DDLL_LONG_CMD_2,
191 		EMC_PMACRO_DDLL_LONG_CMD_3,
192 		EMC_PMACRO_DDLL_LONG_CMD_4,
193 		EMC_PMACRO_DDLL_SHORT_CMD_0,
194 		EMC_PMACRO_DDLL_SHORT_CMD_1,
195 		EMC_PMACRO_DDLL_SHORT_CMD_2,
196 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE0_3,
197 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE1_3,
198 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE2_3,
199 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE3_3,
200 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE4_3,
201 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE5_3,
202 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE6_3,
203 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE7_3,
204 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD0_3,
205 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD1_3,
206 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD2_3,
207 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD3_3,
208 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE0_3,
209 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE1_3,
210 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE2_3,
211 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE3_3,
212 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE4_3,
213 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE5_3,
214 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE6_3,
215 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE7_3,
216 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD0_0,
217 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD0_1,
218 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD0_2,
219 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD0_3,
220 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD1_0,
221 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD1_1,
222 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD1_2,
223 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD1_3,
224 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD2_0,
225 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD2_1,
226 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD2_2,
227 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD2_3,
228 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD3_0,
229 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD3_1,
230 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD3_2,
231 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD3_3,
232 		EMC_TXDSRVTTGEN,
233 		EMC_FDPD_CTRL_DQ,
234 		EMC_FDPD_CTRL_CMD,
235 		EMC_FBIO_SPARE,
236 		EMC_ZCAL_INTERVAL,
237 		EMC_ZCAL_WAIT_CNT,
238 		EMC_MRS_WAIT_CNT,
239 		EMC_MRS_WAIT_CNT2,
240 		EMC_AUTO_CAL_CHANNEL,
241 		EMC_DLL_CFG_0,
242 		EMC_DLL_CFG_1,
243 		EMC_PMACRO_AUTOCAL_CFG_COMMON,
244 		EMC_PMACRO_ZCTRL,
245 		EMC_CFG,
246 		EMC_CFG_PIPE,
247 		EMC_DYN_SELF_REF_CONTROL,
248 		EMC_QPOP,
249 		EMC_DQS_BRLSHFT_0,
250 		EMC_DQS_BRLSHFT_1,
251 		EMC_CMD_BRLSHFT_2,
252 		EMC_CMD_BRLSHFT_3,
253 		EMC_PMACRO_PAD_CFG_CTRL,
254 		EMC_PMACRO_DATA_PAD_RX_CTRL,
255 		EMC_PMACRO_CMD_PAD_RX_CTRL,
256 		EMC_PMACRO_DATA_RX_TERM_MODE,
257 		EMC_PMACRO_CMD_RX_TERM_MODE,
258 		EMC_PMACRO_CMD_PAD_TX_CTRL,
259 		EMC_PMACRO_DATA_PAD_TX_CTRL,
260 		EMC_PMACRO_COMMON_PAD_TX_CTRL,
261 		EMC_PMACRO_VTTGEN_CTRL_0,
262 		EMC_PMACRO_VTTGEN_CTRL_1,
263 		EMC_PMACRO_VTTGEN_CTRL_2,
264 		EMC_PMACRO_BRICK_CTRL_RFU1,
265 		EMC_PMACRO_CMD_BRICK_CTRL_FDPD,
266 		EMC_PMACRO_BRICK_CTRL_RFU2,
267 		EMC_PMACRO_DATA_BRICK_CTRL_FDPD,
268 		EMC_PMACRO_BG_BIAS_CTRL_0,
269 		EMC_CFG_3,
270 		EMC_PMACRO_TX_PWRD_0,
271 		EMC_PMACRO_TX_PWRD_1,
272 		EMC_PMACRO_TX_PWRD_2,
273 		EMC_PMACRO_TX_PWRD_3,
274 		EMC_PMACRO_TX_PWRD_4,
275 		EMC_PMACRO_TX_PWRD_5,
276 		EMC_CONFIG_SAMPLE_DELAY,
277 		EMC_PMACRO_TX_SEL_CLK_SRC_0,
278 		EMC_PMACRO_TX_SEL_CLK_SRC_1,
279 		EMC_PMACRO_TX_SEL_CLK_SRC_2,
280 		EMC_PMACRO_TX_SEL_CLK_SRC_3,
281 		EMC_PMACRO_TX_SEL_CLK_SRC_4,
282 		EMC_PMACRO_TX_SEL_CLK_SRC_5,
283 		EMC_PMACRO_DDLL_BYPASS,
284 		EMC_PMACRO_DDLL_PWRD_0,
285 		EMC_PMACRO_DDLL_PWRD_1,
286 		EMC_PMACRO_DDLL_PWRD_2,
287 		EMC_PMACRO_CMD_CTRL_0,
288 		EMC_PMACRO_CMD_CTRL_1,
289 		EMC_PMACRO_CMD_CTRL_2,
290 		EMC_TR_TIMING_0,
291 		EMC_TR_DVFS,
292 		EMC_TR_CTRL_1,
293 		EMC_TR_RDV,
294 		EMC_TR_QPOP,
295 		EMC_TR_RDV_MASK,
296 		EMC_MRW14,
297 		EMC_TR_QSAFE,
298 		EMC_TR_QRST,
299 		EMC_TRAINING_CTRL,
300 		EMC_TRAINING_SETTLE,
301 		EMC_TRAINING_VREF_SETTLE,
302 		EMC_TRAINING_CA_FINE_CTRL,
303 		EMC_TRAINING_CA_CTRL_MISC,
304 		EMC_TRAINING_CA_CTRL_MISC1,
305 		EMC_TRAINING_CA_VREF_CTRL,
306 		EMC_TRAINING_QUSE_CORS_CTRL,
307 		EMC_TRAINING_QUSE_FINE_CTRL,
308 		EMC_TRAINING_QUSE_CTRL_MISC,
309 		EMC_TRAINING_QUSE_VREF_CTRL,
310 		EMC_TRAINING_READ_FINE_CTRL,
311 		EMC_TRAINING_READ_CTRL_MISC,
312 		EMC_TRAINING_READ_VREF_CTRL,
313 		EMC_TRAINING_WRITE_FINE_CTRL,
314 		EMC_TRAINING_WRITE_CTRL_MISC,
315 		EMC_TRAINING_WRITE_VREF_CTRL,
316 		EMC_TRAINING_MPC,
317 		EMC_MRW15,
318 	},
319 	.trim = {
320 		EMC_PMACRO_IB_DDLL_LONG_DQS_RANK0_0,
321 		EMC_PMACRO_IB_DDLL_LONG_DQS_RANK0_1,
322 		EMC_PMACRO_IB_DDLL_LONG_DQS_RANK0_2,
323 		EMC_PMACRO_IB_DDLL_LONG_DQS_RANK0_3,
324 		EMC_PMACRO_IB_DDLL_LONG_DQS_RANK1_0,
325 		EMC_PMACRO_IB_DDLL_LONG_DQS_RANK1_1,
326 		EMC_PMACRO_IB_DDLL_LONG_DQS_RANK1_2,
327 		EMC_PMACRO_IB_DDLL_LONG_DQS_RANK1_3,
328 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE0_0,
329 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE0_1,
330 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE0_2,
331 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE1_0,
332 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE1_1,
333 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE1_2,
334 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE2_0,
335 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE2_1,
336 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE2_2,
337 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE3_0,
338 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE3_1,
339 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE3_2,
340 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE4_0,
341 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE4_1,
342 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE4_2,
343 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE5_0,
344 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE5_1,
345 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE5_2,
346 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE6_0,
347 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE6_1,
348 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE6_2,
349 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE7_0,
350 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE7_1,
351 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE7_2,
352 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE0_0,
353 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE0_1,
354 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE0_2,
355 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE1_0,
356 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE1_1,
357 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE1_2,
358 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE2_0,
359 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE2_1,
360 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE2_2,
361 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE3_0,
362 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE3_1,
363 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE3_2,
364 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE4_0,
365 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE4_1,
366 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE4_2,
367 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE5_0,
368 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE5_1,
369 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE5_2,
370 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE6_0,
371 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE6_1,
372 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE6_2,
373 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE7_0,
374 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE7_1,
375 		EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE7_2,
376 		EMC_PMACRO_IB_VREF_DQS_0,
377 		EMC_PMACRO_IB_VREF_DQS_1,
378 		EMC_PMACRO_IB_VREF_DQ_0,
379 		EMC_PMACRO_IB_VREF_DQ_1,
380 		EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0,
381 		EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1,
382 		EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2,
383 		EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3,
384 		EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_4,
385 		EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_5,
386 		EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0,
387 		EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1,
388 		EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2,
389 		EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3,
390 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE0_0,
391 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE0_1,
392 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE0_2,
393 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE1_0,
394 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE1_1,
395 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE1_2,
396 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE2_0,
397 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE2_1,
398 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE2_2,
399 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE3_0,
400 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE3_1,
401 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE3_2,
402 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE4_0,
403 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE4_1,
404 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE4_2,
405 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE5_0,
406 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE5_1,
407 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE5_2,
408 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE6_0,
409 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE6_1,
410 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE6_2,
411 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE7_0,
412 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE7_1,
413 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE7_2,
414 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD0_0,
415 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD0_1,
416 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD0_2,
417 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD1_0,
418 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD1_1,
419 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD1_2,
420 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD2_0,
421 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD2_1,
422 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD2_2,
423 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD3_0,
424 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD3_1,
425 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD3_2,
426 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE0_0,
427 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE0_1,
428 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE0_2,
429 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE1_0,
430 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE1_1,
431 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE1_2,
432 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE2_0,
433 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE2_1,
434 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE2_2,
435 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE3_0,
436 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE3_1,
437 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE3_2,
438 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE4_0,
439 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE4_1,
440 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE4_2,
441 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE5_0,
442 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE5_1,
443 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE5_2,
444 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE6_0,
445 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE6_1,
446 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE6_2,
447 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE7_0,
448 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE7_1,
449 		EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE7_2,
450 		EMC_PMACRO_QUSE_DDLL_RANK0_0,
451 		EMC_PMACRO_QUSE_DDLL_RANK0_1,
452 		EMC_PMACRO_QUSE_DDLL_RANK0_2,
453 		EMC_PMACRO_QUSE_DDLL_RANK0_3,
454 		EMC_PMACRO_QUSE_DDLL_RANK1_0,
455 		EMC_PMACRO_QUSE_DDLL_RANK1_1,
456 		EMC_PMACRO_QUSE_DDLL_RANK1_2,
457 		EMC_PMACRO_QUSE_DDLL_RANK1_3
458 	},
459 	.burst_mc = {
460 		MC_EMEM_ARB_CFG,
461 		MC_EMEM_ARB_OUTSTANDING_REQ,
462 		MC_EMEM_ARB_REFPB_HP_CTRL,
463 		MC_EMEM_ARB_REFPB_BANK_CTRL,
464 		MC_EMEM_ARB_TIMING_RCD,
465 		MC_EMEM_ARB_TIMING_RP,
466 		MC_EMEM_ARB_TIMING_RC,
467 		MC_EMEM_ARB_TIMING_RAS,
468 		MC_EMEM_ARB_TIMING_FAW,
469 		MC_EMEM_ARB_TIMING_RRD,
470 		MC_EMEM_ARB_TIMING_RAP2PRE,
471 		MC_EMEM_ARB_TIMING_WAP2PRE,
472 		MC_EMEM_ARB_TIMING_R2R,
473 		MC_EMEM_ARB_TIMING_W2W,
474 		MC_EMEM_ARB_TIMING_R2W,
475 		MC_EMEM_ARB_TIMING_CCDMW,
476 		MC_EMEM_ARB_TIMING_W2R,
477 		MC_EMEM_ARB_TIMING_RFCPB,
478 		MC_EMEM_ARB_DA_TURNS,
479 		MC_EMEM_ARB_DA_COVERS,
480 		MC_EMEM_ARB_MISC0,
481 		MC_EMEM_ARB_MISC1,
482 		MC_EMEM_ARB_MISC2,
483 		MC_EMEM_ARB_RING1_THROTTLE,
484 		MC_EMEM_ARB_DHYST_CTRL,
485 		MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_0,
486 		MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_1,
487 		MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_2,
488 		MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_3,
489 		MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_4,
490 		MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_5,
491 		MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_6,
492 		MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_7,
493 	},
494 	.la_scale = {
495 		MC_MLL_MPCORER_PTSA_RATE,
496 		MC_FTOP_PTSA_RATE,
497 		MC_PTSA_GRANT_DECREMENT,
498 		MC_LATENCY_ALLOWANCE_XUSB_0,
499 		MC_LATENCY_ALLOWANCE_XUSB_1,
500 		MC_LATENCY_ALLOWANCE_TSEC_0,
501 		MC_LATENCY_ALLOWANCE_SDMMCA_0,
502 		MC_LATENCY_ALLOWANCE_SDMMCAA_0,
503 		MC_LATENCY_ALLOWANCE_SDMMC_0,
504 		MC_LATENCY_ALLOWANCE_SDMMCAB_0,
505 		MC_LATENCY_ALLOWANCE_PPCS_0,
506 		MC_LATENCY_ALLOWANCE_PPCS_1,
507 		MC_LATENCY_ALLOWANCE_MPCORE_0,
508 		MC_LATENCY_ALLOWANCE_HC_0,
509 		MC_LATENCY_ALLOWANCE_HC_1,
510 		MC_LATENCY_ALLOWANCE_AVPC_0,
511 		MC_LATENCY_ALLOWANCE_GPU_0,
512 		MC_LATENCY_ALLOWANCE_GPU2_0,
513 		MC_LATENCY_ALLOWANCE_NVENC_0,
514 		MC_LATENCY_ALLOWANCE_NVDEC_0,
515 		MC_LATENCY_ALLOWANCE_VIC_0,
516 		MC_LATENCY_ALLOWANCE_VI2_0,
517 		MC_LATENCY_ALLOWANCE_ISP2_0,
518 		MC_LATENCY_ALLOWANCE_ISP2_1,
519 	},
520 	.burst_per_channel = {
521 		{ .bank = 0, .offset = EMC_MRW10, },
522 		{ .bank = 1, .offset = EMC_MRW10, },
523 		{ .bank = 0, .offset = EMC_MRW11, },
524 		{ .bank = 1, .offset = EMC_MRW11, },
525 		{ .bank = 0, .offset = EMC_MRW12, },
526 		{ .bank = 1, .offset = EMC_MRW12, },
527 		{ .bank = 0, .offset = EMC_MRW13, },
528 		{ .bank = 1, .offset = EMC_MRW13, },
529 	},
530 	.trim_per_channel = {
531 		{ .bank = 0, .offset = EMC_CMD_BRLSHFT_0, },
532 		{ .bank = 1, .offset = EMC_CMD_BRLSHFT_1, },
533 		{ .bank = 0, .offset = EMC_DATA_BRLSHFT_0, },
534 		{ .bank = 1, .offset = EMC_DATA_BRLSHFT_0, },
535 		{ .bank = 0, .offset = EMC_DATA_BRLSHFT_1, },
536 		{ .bank = 1, .offset = EMC_DATA_BRLSHFT_1, },
537 		{ .bank = 0, .offset = EMC_QUSE_BRLSHFT_0, },
538 		{ .bank = 1, .offset = EMC_QUSE_BRLSHFT_1, },
539 		{ .bank = 0, .offset = EMC_QUSE_BRLSHFT_2, },
540 		{ .bank = 1, .offset = EMC_QUSE_BRLSHFT_3, },
541 	},
542 	.vref_per_channel = {
543 		{
544 			.bank = 0,
545 			.offset = EMC_TRAINING_OPT_DQS_IB_VREF_RANK0,
546 		}, {
547 			.bank = 1,
548 			.offset = EMC_TRAINING_OPT_DQS_IB_VREF_RANK0,
549 		}, {
550 			.bank = 0,
551 			.offset = EMC_TRAINING_OPT_DQS_IB_VREF_RANK1,
552 		}, {
553 			.bank = 1,
554 			.offset = EMC_TRAINING_OPT_DQS_IB_VREF_RANK1,
555 		},
556 	},
557 };
558 
559 static void tegra210_emc_train(struct timer_list *timer)
560 {
561 	struct tegra210_emc *emc = from_timer(emc, timer, training);
562 	unsigned long flags;
563 
564 	if (!emc->last)
565 		return;
566 
567 	spin_lock_irqsave(&emc->lock, flags);
568 
569 	if (emc->sequence->periodic_compensation)
570 		emc->sequence->periodic_compensation(emc);
571 
572 	spin_unlock_irqrestore(&emc->lock, flags);
573 
574 	mod_timer(&emc->training,
575 		  jiffies + msecs_to_jiffies(emc->training_interval));
576 }
577 
578 static void tegra210_emc_training_start(struct tegra210_emc *emc)
579 {
580 	mod_timer(&emc->training,
581 		  jiffies + msecs_to_jiffies(emc->training_interval));
582 }
583 
584 static void tegra210_emc_training_stop(struct tegra210_emc *emc)
585 {
586 	del_timer(&emc->training);
587 }
588 
589 static unsigned int tegra210_emc_get_temperature(struct tegra210_emc *emc)
590 {
591 	unsigned long flags;
592 	u32 value, max = 0;
593 	unsigned int i;
594 
595 	spin_lock_irqsave(&emc->lock, flags);
596 
597 	for (i = 0; i < emc->num_devices; i++) {
598 		value = tegra210_emc_mrr_read(emc, i, 4);
599 
600 		if (value & BIT(7))
601 			dev_dbg(emc->dev,
602 				"sensor reading changed for device %u: %08x\n",
603 				i, value);
604 
605 		value = FIELD_GET(LPDDR2_MR4_SRR, value);
606 		if (value > max)
607 			max = value;
608 	}
609 
610 	spin_unlock_irqrestore(&emc->lock, flags);
611 
612 	return max;
613 }
614 
615 static void tegra210_emc_poll_refresh(struct timer_list *timer)
616 {
617 	struct tegra210_emc *emc = from_timer(emc, timer, refresh_timer);
618 	unsigned int temperature;
619 
620 	if (!emc->debugfs.temperature)
621 		temperature = tegra210_emc_get_temperature(emc);
622 	else
623 		temperature = emc->debugfs.temperature;
624 
625 	if (temperature == emc->temperature)
626 		goto reset;
627 
628 	switch (temperature) {
629 	case 0 ... 3:
630 		/* temperature is fine, using regular refresh */
631 		dev_dbg(emc->dev, "switching to nominal refresh...\n");
632 		tegra210_emc_set_refresh(emc, TEGRA210_EMC_REFRESH_NOMINAL);
633 		break;
634 
635 	case 4:
636 		dev_dbg(emc->dev, "switching to 2x refresh...\n");
637 		tegra210_emc_set_refresh(emc, TEGRA210_EMC_REFRESH_2X);
638 		break;
639 
640 	case 5:
641 		dev_dbg(emc->dev, "switching to 4x refresh...\n");
642 		tegra210_emc_set_refresh(emc, TEGRA210_EMC_REFRESH_4X);
643 		break;
644 
645 	case 6 ... 7:
646 		dev_dbg(emc->dev, "switching to throttle refresh...\n");
647 		tegra210_emc_set_refresh(emc, TEGRA210_EMC_REFRESH_THROTTLE);
648 		break;
649 
650 	default:
651 		WARN(1, "invalid DRAM temperature state %u\n", temperature);
652 		return;
653 	}
654 
655 	emc->temperature = temperature;
656 
657 reset:
658 	if (atomic_read(&emc->refresh_poll) > 0) {
659 		unsigned int interval = emc->refresh_poll_interval;
660 		unsigned int timeout = msecs_to_jiffies(interval);
661 
662 		mod_timer(&emc->refresh_timer, jiffies + timeout);
663 	}
664 }
665 
666 static void tegra210_emc_poll_refresh_stop(struct tegra210_emc *emc)
667 {
668 	atomic_set(&emc->refresh_poll, 0);
669 	del_timer_sync(&emc->refresh_timer);
670 }
671 
672 static void tegra210_emc_poll_refresh_start(struct tegra210_emc *emc)
673 {
674 	atomic_set(&emc->refresh_poll, 1);
675 
676 	mod_timer(&emc->refresh_timer,
677 		  jiffies + msecs_to_jiffies(emc->refresh_poll_interval));
678 }
679 
680 static int tegra210_emc_cd_max_state(struct thermal_cooling_device *cd,
681 				     unsigned long *state)
682 {
683 	*state = 1;
684 
685 	return 0;
686 }
687 
688 static int tegra210_emc_cd_get_state(struct thermal_cooling_device *cd,
689 				     unsigned long *state)
690 {
691 	struct tegra210_emc *emc = cd->devdata;
692 
693 	*state = atomic_read(&emc->refresh_poll);
694 
695 	return 0;
696 }
697 
698 static int tegra210_emc_cd_set_state(struct thermal_cooling_device *cd,
699 				     unsigned long state)
700 {
701 	struct tegra210_emc *emc = cd->devdata;
702 
703 	if (state == atomic_read(&emc->refresh_poll))
704 		return 0;
705 
706 	if (state)
707 		tegra210_emc_poll_refresh_start(emc);
708 	else
709 		tegra210_emc_poll_refresh_stop(emc);
710 
711 	return 0;
712 }
713 
714 static const struct thermal_cooling_device_ops tegra210_emc_cd_ops = {
715 	.get_max_state = tegra210_emc_cd_max_state,
716 	.get_cur_state = tegra210_emc_cd_get_state,
717 	.set_cur_state = tegra210_emc_cd_set_state,
718 };
719 
720 static void tegra210_emc_set_clock(struct tegra210_emc *emc, u32 clksrc)
721 {
722 	emc->sequence->set_clock(emc, clksrc);
723 
724 	if (emc->next->periodic_training)
725 		tegra210_emc_training_start(emc);
726 	else
727 		tegra210_emc_training_stop(emc);
728 }
729 
730 static void tegra210_change_dll_src(struct tegra210_emc *emc,
731 				    u32 clksrc)
732 {
733 	u32 dll_setting = emc->next->dll_clk_src;
734 	u32 emc_clk_src;
735 	u32 emc_clk_div;
736 
737 	emc_clk_src = (clksrc & EMC_CLK_EMC_2X_CLK_SRC_MASK) >>
738 		       EMC_CLK_EMC_2X_CLK_SRC_SHIFT;
739 	emc_clk_div = (clksrc & EMC_CLK_EMC_2X_CLK_DIVISOR_MASK) >>
740 		       EMC_CLK_EMC_2X_CLK_DIVISOR_SHIFT;
741 
742 	dll_setting &= ~(DLL_CLK_EMC_DLL_CLK_SRC_MASK |
743 			 DLL_CLK_EMC_DLL_CLK_DIVISOR_MASK);
744 	dll_setting |= emc_clk_src << DLL_CLK_EMC_DLL_CLK_SRC_SHIFT;
745 	dll_setting |= emc_clk_div << DLL_CLK_EMC_DLL_CLK_DIVISOR_SHIFT;
746 
747 	dll_setting &= ~DLL_CLK_EMC_DLL_DDLL_CLK_SEL_MASK;
748 	if (emc_clk_src == EMC_CLK_SOURCE_PLLMB_LJ)
749 		dll_setting |= (PLLM_VCOB <<
750 				DLL_CLK_EMC_DLL_DDLL_CLK_SEL_SHIFT);
751 	else if (emc_clk_src == EMC_CLK_SOURCE_PLLM_LJ)
752 		dll_setting |= (PLLM_VCOA <<
753 				DLL_CLK_EMC_DLL_DDLL_CLK_SEL_SHIFT);
754 	else
755 		dll_setting |= (EMC_DLL_SWITCH_OUT <<
756 				DLL_CLK_EMC_DLL_DDLL_CLK_SEL_SHIFT);
757 
758 	tegra210_clk_emc_dll_update_setting(dll_setting);
759 
760 	if (emc->next->clk_out_enb_x_0_clk_enb_emc_dll)
761 		tegra210_clk_emc_dll_enable(true);
762 	else
763 		tegra210_clk_emc_dll_enable(false);
764 }
765 
766 int tegra210_emc_set_refresh(struct tegra210_emc *emc,
767 			     enum tegra210_emc_refresh refresh)
768 {
769 	struct tegra210_emc_timing *timings;
770 	unsigned long flags;
771 
772 	if ((emc->dram_type != DRAM_TYPE_LPDDR2 &&
773 	     emc->dram_type != DRAM_TYPE_LPDDR4) ||
774 	    !emc->last)
775 		return -ENODEV;
776 
777 	if (refresh > TEGRA210_EMC_REFRESH_THROTTLE)
778 		return -EINVAL;
779 
780 	if (refresh == emc->refresh)
781 		return 0;
782 
783 	spin_lock_irqsave(&emc->lock, flags);
784 
785 	if (refresh == TEGRA210_EMC_REFRESH_THROTTLE && emc->derated)
786 		timings = emc->derated;
787 	else
788 		timings = emc->nominal;
789 
790 	if (timings != emc->timings) {
791 		unsigned int index = emc->last - emc->timings;
792 		u32 clksrc;
793 
794 		clksrc = emc->provider.configs[index].value |
795 			 EMC_CLK_FORCE_CC_TRIGGER;
796 
797 		emc->next = &timings[index];
798 		emc->timings = timings;
799 
800 		tegra210_emc_set_clock(emc, clksrc);
801 	} else {
802 		tegra210_emc_adjust_timing(emc, emc->last);
803 		tegra210_emc_timing_update(emc);
804 
805 		if (refresh != TEGRA210_EMC_REFRESH_NOMINAL)
806 			emc_writel(emc, EMC_REF_REF_CMD, EMC_REF);
807 	}
808 
809 	spin_unlock_irqrestore(&emc->lock, flags);
810 
811 	return 0;
812 }
813 
814 u32 tegra210_emc_mrr_read(struct tegra210_emc *emc, unsigned int chip,
815 			  unsigned int address)
816 {
817 	u32 value, ret = 0;
818 	unsigned int i;
819 
820 	value = (chip & EMC_MRR_DEV_SEL_MASK) << EMC_MRR_DEV_SEL_SHIFT |
821 		(address & EMC_MRR_MA_MASK) << EMC_MRR_MA_SHIFT;
822 	emc_writel(emc, value, EMC_MRR);
823 
824 	for (i = 0; i < emc->num_channels; i++)
825 		WARN(tegra210_emc_wait_for_update(emc, i, EMC_EMC_STATUS,
826 						  EMC_EMC_STATUS_MRR_DIVLD, 1),
827 		     "Timed out waiting for MRR %u (ch=%u)\n", address, i);
828 
829 	for (i = 0; i < emc->num_channels; i++) {
830 		value = emc_channel_readl(emc, i, EMC_MRR);
831 		value &= EMC_MRR_DATA_MASK;
832 
833 		ret = (ret << 16) | value;
834 	}
835 
836 	return ret;
837 }
838 
839 void tegra210_emc_do_clock_change(struct tegra210_emc *emc, u32 clksrc)
840 {
841 	int err;
842 
843 	mc_readl(emc->mc, MC_EMEM_ADR_CFG);
844 	emc_readl(emc, EMC_INTSTATUS);
845 
846 	tegra210_clk_emc_update_setting(clksrc);
847 
848 	err = tegra210_emc_wait_for_update(emc, 0, EMC_INTSTATUS,
849 					   EMC_INTSTATUS_CLKCHANGE_COMPLETE,
850 					   true);
851 	if (err)
852 		dev_warn(emc->dev, "clock change completion error: %d\n", err);
853 }
854 
855 struct tegra210_emc_timing *tegra210_emc_find_timing(struct tegra210_emc *emc,
856 						     unsigned long rate)
857 {
858 	unsigned int i;
859 
860 	for (i = 0; i < emc->num_timings; i++)
861 		if (emc->timings[i].rate * 1000UL == rate)
862 			return &emc->timings[i];
863 
864 	return NULL;
865 }
866 
867 int tegra210_emc_wait_for_update(struct tegra210_emc *emc, unsigned int channel,
868 				 unsigned int offset, u32 bit_mask, bool state)
869 {
870 	unsigned int i;
871 	u32 value;
872 
873 	for (i = 0; i < EMC_STATUS_UPDATE_TIMEOUT; i++) {
874 		value = emc_channel_readl(emc, channel, offset);
875 		if (!!(value & bit_mask) == state)
876 			return 0;
877 
878 		udelay(1);
879 	}
880 
881 	return -ETIMEDOUT;
882 }
883 
884 void tegra210_emc_set_shadow_bypass(struct tegra210_emc *emc, int set)
885 {
886 	u32 emc_dbg = emc_readl(emc, EMC_DBG);
887 
888 	if (set)
889 		emc_writel(emc, emc_dbg | EMC_DBG_WRITE_MUX_ACTIVE, EMC_DBG);
890 	else
891 		emc_writel(emc, emc_dbg & ~EMC_DBG_WRITE_MUX_ACTIVE, EMC_DBG);
892 }
893 
894 u32 tegra210_emc_get_dll_state(struct tegra210_emc_timing *next)
895 {
896 	if (next->emc_emrs & 0x1)
897 		return 0;
898 
899 	return 1;
900 }
901 
902 void tegra210_emc_timing_update(struct tegra210_emc *emc)
903 {
904 	unsigned int i;
905 	int err = 0;
906 
907 	emc_writel(emc, 0x1, EMC_TIMING_CONTROL);
908 
909 	for (i = 0; i < emc->num_channels; i++) {
910 		err |= tegra210_emc_wait_for_update(emc, i, EMC_EMC_STATUS,
911 						    EMC_EMC_STATUS_TIMING_UPDATE_STALLED,
912 						    false);
913 	}
914 
915 	if (err)
916 		dev_warn(emc->dev, "timing update error: %d\n", err);
917 }
918 
919 unsigned long tegra210_emc_actual_osc_clocks(u32 in)
920 {
921 	if (in < 0x40)
922 		return in * 16;
923 	else if (in < 0x80)
924 		return 2048;
925 	else if (in < 0xc0)
926 		return 4096;
927 	else
928 		return 8192;
929 }
930 
931 void tegra210_emc_start_periodic_compensation(struct tegra210_emc *emc)
932 {
933 	u32 mpc_req = 0x4b;
934 
935 	emc_writel(emc, mpc_req, EMC_MPC);
936 	mpc_req = emc_readl(emc, EMC_MPC);
937 }
938 
939 u32 tegra210_emc_compensate(struct tegra210_emc_timing *next, u32 offset)
940 {
941 	u32 temp = 0, rate = next->rate / 1000;
942 	s32 delta[4], delta_taps[4];
943 	s32 new[] = {
944 		TRIM_REG(0, 0, 0, 0),
945 		TRIM_REG(0, 0, 0, 1),
946 		TRIM_REG(0, 0, 1, 2),
947 		TRIM_REG(0, 0, 1, 3),
948 
949 		TRIM_REG(1, 0, 2, 4),
950 		TRIM_REG(1, 0, 2, 5),
951 		TRIM_REG(1, 0, 3, 6),
952 		TRIM_REG(1, 0, 3, 7),
953 
954 		TRIM_REG(0, 1, 0, 0),
955 		TRIM_REG(0, 1, 0, 1),
956 		TRIM_REG(0, 1, 1, 2),
957 		TRIM_REG(0, 1, 1, 3),
958 
959 		TRIM_REG(1, 1, 2, 4),
960 		TRIM_REG(1, 1, 2, 5),
961 		TRIM_REG(1, 1, 3, 6),
962 		TRIM_REG(1, 1, 3, 7)
963 	};
964 	unsigned i;
965 
966 	switch (offset) {
967 	case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0:
968 	case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1:
969 	case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2:
970 	case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3:
971 	case EMC_DATA_BRLSHFT_0:
972 		delta[0] = 128 * (next->current_dram_clktree[C0D0U0] -
973 				  next->trained_dram_clktree[C0D0U0]);
974 		delta[1] = 128 * (next->current_dram_clktree[C0D0U1] -
975 				  next->trained_dram_clktree[C0D0U1]);
976 		delta[2] = 128 * (next->current_dram_clktree[C1D0U0] -
977 				  next->trained_dram_clktree[C1D0U0]);
978 		delta[3] = 128 * (next->current_dram_clktree[C1D0U1] -
979 				  next->trained_dram_clktree[C1D0U1]);
980 
981 		delta_taps[0] = (delta[0] * (s32)rate) / 1000000;
982 		delta_taps[1] = (delta[1] * (s32)rate) / 1000000;
983 		delta_taps[2] = (delta[2] * (s32)rate) / 1000000;
984 		delta_taps[3] = (delta[3] * (s32)rate) / 1000000;
985 
986 		for (i = 0; i < 4; i++) {
987 			if ((delta_taps[i] > next->tree_margin) ||
988 			    (delta_taps[i] < (-1 * next->tree_margin))) {
989 				new[i * 2] = new[i * 2] + delta_taps[i];
990 				new[i * 2 + 1] = new[i * 2 + 1] +
991 							delta_taps[i];
992 			}
993 		}
994 
995 		if (offset == EMC_DATA_BRLSHFT_0) {
996 			for (i = 0; i < 8; i++)
997 				new[i] = new[i] / 64;
998 		} else {
999 			for (i = 0; i < 8; i++)
1000 				new[i] = new[i] % 64;
1001 		}
1002 
1003 		break;
1004 
1005 	case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0:
1006 	case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1:
1007 	case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2:
1008 	case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3:
1009 	case EMC_DATA_BRLSHFT_1:
1010 		delta[0] = 128 * (next->current_dram_clktree[C0D1U0] -
1011 				  next->trained_dram_clktree[C0D1U0]);
1012 		delta[1] = 128 * (next->current_dram_clktree[C0D1U1] -
1013 				  next->trained_dram_clktree[C0D1U1]);
1014 		delta[2] = 128 * (next->current_dram_clktree[C1D1U0] -
1015 				  next->trained_dram_clktree[C1D1U0]);
1016 		delta[3] = 128 * (next->current_dram_clktree[C1D1U1] -
1017 				  next->trained_dram_clktree[C1D1U1]);
1018 
1019 		delta_taps[0] = (delta[0] * (s32)rate) / 1000000;
1020 		delta_taps[1] = (delta[1] * (s32)rate) / 1000000;
1021 		delta_taps[2] = (delta[2] * (s32)rate) / 1000000;
1022 		delta_taps[3] = (delta[3] * (s32)rate) / 1000000;
1023 
1024 		for (i = 0; i < 4; i++) {
1025 			if ((delta_taps[i] > next->tree_margin) ||
1026 			    (delta_taps[i] < (-1 * next->tree_margin))) {
1027 				new[8 + i * 2] = new[8 + i * 2] +
1028 							delta_taps[i];
1029 				new[8 + i * 2 + 1] = new[8 + i * 2 + 1] +
1030 							delta_taps[i];
1031 			}
1032 		}
1033 
1034 		if (offset == EMC_DATA_BRLSHFT_1) {
1035 			for (i = 0; i < 8; i++)
1036 				new[i + 8] = new[i + 8] / 64;
1037 		} else {
1038 			for (i = 0; i < 8; i++)
1039 				new[i + 8] = new[i + 8] % 64;
1040 		}
1041 
1042 		break;
1043 	}
1044 
1045 	switch (offset) {
1046 	case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0:
1047 		temp = CALC_TEMP(0, 0, 0, 1, 0);
1048 		break;
1049 
1050 	case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1:
1051 		temp = CALC_TEMP(0, 1, 2, 3, 2);
1052 		break;
1053 
1054 	case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2:
1055 		temp = CALC_TEMP(0, 2, 4, 5, 4);
1056 		break;
1057 
1058 	case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3:
1059 		temp = CALC_TEMP(0, 3, 6, 7, 6);
1060 		break;
1061 
1062 	case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0:
1063 		temp = CALC_TEMP(1, 0, 0, 1, 8);
1064 		break;
1065 
1066 	case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1:
1067 		temp = CALC_TEMP(1, 1, 2, 3, 10);
1068 		break;
1069 
1070 	case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2:
1071 		temp = CALC_TEMP(1, 2, 4, 5, 12);
1072 		break;
1073 
1074 	case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3:
1075 		temp = CALC_TEMP(1, 3, 6, 7, 14);
1076 		break;
1077 
1078 	case EMC_DATA_BRLSHFT_0:
1079 		temp = ((new[0] <<
1080 			 EMC_DATA_BRLSHFT_0_RANK0_BYTE0_DATA_BRLSHFT_SHIFT) &
1081 			 EMC_DATA_BRLSHFT_0_RANK0_BYTE0_DATA_BRLSHFT_MASK) |
1082 		       ((new[1] <<
1083 			 EMC_DATA_BRLSHFT_0_RANK0_BYTE1_DATA_BRLSHFT_SHIFT) &
1084 			 EMC_DATA_BRLSHFT_0_RANK0_BYTE1_DATA_BRLSHFT_MASK) |
1085 		       ((new[2] <<
1086 			 EMC_DATA_BRLSHFT_0_RANK0_BYTE2_DATA_BRLSHFT_SHIFT) &
1087 			 EMC_DATA_BRLSHFT_0_RANK0_BYTE2_DATA_BRLSHFT_MASK) |
1088 		       ((new[3] <<
1089 			 EMC_DATA_BRLSHFT_0_RANK0_BYTE3_DATA_BRLSHFT_SHIFT) &
1090 			 EMC_DATA_BRLSHFT_0_RANK0_BYTE3_DATA_BRLSHFT_MASK) |
1091 		       ((new[4] <<
1092 			 EMC_DATA_BRLSHFT_0_RANK0_BYTE4_DATA_BRLSHFT_SHIFT) &
1093 			 EMC_DATA_BRLSHFT_0_RANK0_BYTE4_DATA_BRLSHFT_MASK) |
1094 		       ((new[5] <<
1095 			 EMC_DATA_BRLSHFT_0_RANK0_BYTE5_DATA_BRLSHFT_SHIFT) &
1096 			 EMC_DATA_BRLSHFT_0_RANK0_BYTE5_DATA_BRLSHFT_MASK) |
1097 		       ((new[6] <<
1098 			 EMC_DATA_BRLSHFT_0_RANK0_BYTE6_DATA_BRLSHFT_SHIFT) &
1099 			 EMC_DATA_BRLSHFT_0_RANK0_BYTE6_DATA_BRLSHFT_MASK) |
1100 		       ((new[7] <<
1101 			 EMC_DATA_BRLSHFT_0_RANK0_BYTE7_DATA_BRLSHFT_SHIFT) &
1102 			 EMC_DATA_BRLSHFT_0_RANK0_BYTE7_DATA_BRLSHFT_MASK);
1103 		break;
1104 
1105 	case EMC_DATA_BRLSHFT_1:
1106 		temp = ((new[8] <<
1107 			 EMC_DATA_BRLSHFT_1_RANK1_BYTE0_DATA_BRLSHFT_SHIFT) &
1108 			 EMC_DATA_BRLSHFT_1_RANK1_BYTE0_DATA_BRLSHFT_MASK) |
1109 		       ((new[9] <<
1110 			 EMC_DATA_BRLSHFT_1_RANK1_BYTE1_DATA_BRLSHFT_SHIFT) &
1111 			 EMC_DATA_BRLSHFT_1_RANK1_BYTE1_DATA_BRLSHFT_MASK) |
1112 		       ((new[10] <<
1113 			 EMC_DATA_BRLSHFT_1_RANK1_BYTE2_DATA_BRLSHFT_SHIFT) &
1114 			 EMC_DATA_BRLSHFT_1_RANK1_BYTE2_DATA_BRLSHFT_MASK) |
1115 		       ((new[11] <<
1116 			 EMC_DATA_BRLSHFT_1_RANK1_BYTE3_DATA_BRLSHFT_SHIFT) &
1117 			 EMC_DATA_BRLSHFT_1_RANK1_BYTE3_DATA_BRLSHFT_MASK) |
1118 		       ((new[12] <<
1119 			 EMC_DATA_BRLSHFT_1_RANK1_BYTE4_DATA_BRLSHFT_SHIFT) &
1120 			 EMC_DATA_BRLSHFT_1_RANK1_BYTE4_DATA_BRLSHFT_MASK) |
1121 		       ((new[13] <<
1122 			 EMC_DATA_BRLSHFT_1_RANK1_BYTE5_DATA_BRLSHFT_SHIFT) &
1123 			 EMC_DATA_BRLSHFT_1_RANK1_BYTE5_DATA_BRLSHFT_MASK) |
1124 		       ((new[14] <<
1125 			 EMC_DATA_BRLSHFT_1_RANK1_BYTE6_DATA_BRLSHFT_SHIFT) &
1126 			 EMC_DATA_BRLSHFT_1_RANK1_BYTE6_DATA_BRLSHFT_MASK) |
1127 		       ((new[15] <<
1128 			 EMC_DATA_BRLSHFT_1_RANK1_BYTE7_DATA_BRLSHFT_SHIFT) &
1129 			 EMC_DATA_BRLSHFT_1_RANK1_BYTE7_DATA_BRLSHFT_MASK);
1130 		break;
1131 
1132 	default:
1133 		break;
1134 	}
1135 
1136 	return temp;
1137 }
1138 
1139 u32 tegra210_emc_dll_prelock(struct tegra210_emc *emc, u32 clksrc)
1140 {
1141 	unsigned int i;
1142 	u32 value;
1143 
1144 	value = emc_readl(emc, EMC_CFG_DIG_DLL);
1145 	value &= ~EMC_CFG_DIG_DLL_CFG_DLL_LOCK_LIMIT_MASK;
1146 	value |= (3 << EMC_CFG_DIG_DLL_CFG_DLL_LOCK_LIMIT_SHIFT);
1147 	value &= ~EMC_CFG_DIG_DLL_CFG_DLL_EN;
1148 	value &= ~EMC_CFG_DIG_DLL_CFG_DLL_MODE_MASK;
1149 	value |= (3 << EMC_CFG_DIG_DLL_CFG_DLL_MODE_SHIFT);
1150 	value |= EMC_CFG_DIG_DLL_CFG_DLL_STALL_ALL_TRAFFIC;
1151 	value &= ~EMC_CFG_DIG_DLL_CFG_DLL_STALL_RW_UNTIL_LOCK;
1152 	value &= ~EMC_CFG_DIG_DLL_CFG_DLL_STALL_ALL_UNTIL_LOCK;
1153 	emc_writel(emc, value, EMC_CFG_DIG_DLL);
1154 	emc_writel(emc, 1, EMC_TIMING_CONTROL);
1155 
1156 	for (i = 0; i < emc->num_channels; i++)
1157 		tegra210_emc_wait_for_update(emc, i, EMC_EMC_STATUS,
1158 					     EMC_EMC_STATUS_TIMING_UPDATE_STALLED,
1159 					     0);
1160 
1161 	for (i = 0; i < emc->num_channels; i++) {
1162 		while (true) {
1163 			value = emc_channel_readl(emc, i, EMC_CFG_DIG_DLL);
1164 			if ((value & EMC_CFG_DIG_DLL_CFG_DLL_EN) == 0)
1165 				break;
1166 		}
1167 	}
1168 
1169 	value = emc->next->burst_regs[EMC_DLL_CFG_0_INDEX];
1170 	emc_writel(emc, value, EMC_DLL_CFG_0);
1171 
1172 	value = emc_readl(emc, EMC_DLL_CFG_1);
1173 	value &= EMC_DLL_CFG_1_DDLLCAL_CTRL_START_TRIM_MASK;
1174 
1175 	if (emc->next->rate >= 400000 && emc->next->rate < 600000)
1176 		value |= 150;
1177 	else if (emc->next->rate >= 600000 && emc->next->rate < 800000)
1178 		value |= 100;
1179 	else if (emc->next->rate >= 800000 && emc->next->rate < 1000000)
1180 		value |= 70;
1181 	else if (emc->next->rate >= 1000000 && emc->next->rate < 1200000)
1182 		value |= 30;
1183 	else
1184 		value |= 20;
1185 
1186 	emc_writel(emc, value, EMC_DLL_CFG_1);
1187 
1188 	tegra210_change_dll_src(emc, clksrc);
1189 
1190 	value = emc_readl(emc, EMC_CFG_DIG_DLL);
1191 	value |= EMC_CFG_DIG_DLL_CFG_DLL_EN;
1192 	emc_writel(emc, value, EMC_CFG_DIG_DLL);
1193 
1194 	tegra210_emc_timing_update(emc);
1195 
1196 	for (i = 0; i < emc->num_channels; i++) {
1197 		while (true) {
1198 			value = emc_channel_readl(emc, 0, EMC_CFG_DIG_DLL);
1199 			if (value & EMC_CFG_DIG_DLL_CFG_DLL_EN)
1200 				break;
1201 		}
1202 	}
1203 
1204 	while (true) {
1205 		value = emc_readl(emc, EMC_DIG_DLL_STATUS);
1206 
1207 		if ((value & EMC_DIG_DLL_STATUS_DLL_PRIV_UPDATED) == 0)
1208 			continue;
1209 
1210 		if ((value & EMC_DIG_DLL_STATUS_DLL_LOCK) == 0)
1211 			continue;
1212 
1213 		break;
1214 	}
1215 
1216 	value = emc_readl(emc, EMC_DIG_DLL_STATUS);
1217 
1218 	return value & EMC_DIG_DLL_STATUS_DLL_OUT_MASK;
1219 }
1220 
1221 u32 tegra210_emc_dvfs_power_ramp_up(struct tegra210_emc *emc, u32 clk,
1222 				    bool flip_backward)
1223 {
1224 	u32 cmd_pad, dq_pad, rfu1, cfg5, common_tx, ramp_up_wait = 0;
1225 	const struct tegra210_emc_timing *timing;
1226 
1227 	if (flip_backward)
1228 		timing = emc->last;
1229 	else
1230 		timing = emc->next;
1231 
1232 	cmd_pad = timing->burst_regs[EMC_PMACRO_CMD_PAD_TX_CTRL_INDEX];
1233 	dq_pad = timing->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX];
1234 	rfu1 = timing->burst_regs[EMC_PMACRO_BRICK_CTRL_RFU1_INDEX];
1235 	cfg5 = timing->burst_regs[EMC_FBIO_CFG5_INDEX];
1236 	common_tx = timing->burst_regs[EMC_PMACRO_COMMON_PAD_TX_CTRL_INDEX];
1237 
1238 	cmd_pad |= EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_DRVFORCEON;
1239 
1240 	if (clk < 1000000 / DVFS_FGCG_MID_SPEED_THRESHOLD) {
1241 		ccfifo_writel(emc, common_tx & 0xa,
1242 			      EMC_PMACRO_COMMON_PAD_TX_CTRL, 0);
1243 		ccfifo_writel(emc, common_tx & 0xf,
1244 			      EMC_PMACRO_COMMON_PAD_TX_CTRL,
1245 			      (100000 / clk) + 1);
1246 		ramp_up_wait += 100000;
1247 	} else {
1248 		ccfifo_writel(emc, common_tx | 0x8,
1249 			      EMC_PMACRO_COMMON_PAD_TX_CTRL, 0);
1250 	}
1251 
1252 	if (clk < 1000000 / DVFS_FGCG_HIGH_SPEED_THRESHOLD) {
1253 		if (clk < 1000000 / IOBRICK_DCC_THRESHOLD) {
1254 			cmd_pad |=
1255 				EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSP_TX_E_DCC |
1256 				EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSN_TX_E_DCC;
1257 			cmd_pad &=
1258 				~(EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_E_DCC |
1259 				  EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_CMD_TX_E_DCC);
1260 			ccfifo_writel(emc, cmd_pad,
1261 				      EMC_PMACRO_CMD_PAD_TX_CTRL,
1262 				      (100000 / clk) + 1);
1263 			ramp_up_wait += 100000;
1264 
1265 			dq_pad |=
1266 				EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSP_TX_E_DCC |
1267 				EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSN_TX_E_DCC;
1268 			dq_pad &=
1269 			       ~(EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_TX_E_DCC |
1270 				 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_CMD_TX_E_DCC);
1271 			ccfifo_writel(emc, dq_pad,
1272 				      EMC_PMACRO_DATA_PAD_TX_CTRL, 0);
1273 			ccfifo_writel(emc, rfu1 & 0xfe40fe40,
1274 				      EMC_PMACRO_BRICK_CTRL_RFU1, 0);
1275 		} else {
1276 			ccfifo_writel(emc, rfu1 & 0xfe40fe40,
1277 				      EMC_PMACRO_BRICK_CTRL_RFU1,
1278 				      (100000 / clk) + 1);
1279 			ramp_up_wait += 100000;
1280 		}
1281 
1282 		ccfifo_writel(emc, rfu1 & 0xfeedfeed,
1283 			      EMC_PMACRO_BRICK_CTRL_RFU1, (100000 / clk) + 1);
1284 		ramp_up_wait += 100000;
1285 
1286 		if (clk < 1000000 / IOBRICK_DCC_THRESHOLD) {
1287 			cmd_pad |=
1288 				EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSP_TX_E_DCC |
1289 				EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSN_TX_E_DCC |
1290 				EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_E_DCC |
1291 				EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_CMD_TX_E_DCC;
1292 			ccfifo_writel(emc, cmd_pad,
1293 				      EMC_PMACRO_CMD_PAD_TX_CTRL,
1294 				      (100000 / clk) + 1);
1295 			ramp_up_wait += 100000;
1296 
1297 			dq_pad |=
1298 				EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSP_TX_E_DCC |
1299 				EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSN_TX_E_DCC |
1300 				EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_TX_E_DCC |
1301 				EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_CMD_TX_E_DCC;
1302 			ccfifo_writel(emc, dq_pad,
1303 				      EMC_PMACRO_DATA_PAD_TX_CTRL, 0);
1304 			ccfifo_writel(emc, rfu1,
1305 				      EMC_PMACRO_BRICK_CTRL_RFU1, 0);
1306 		} else {
1307 			ccfifo_writel(emc, rfu1,
1308 				      EMC_PMACRO_BRICK_CTRL_RFU1,
1309 				      (100000 / clk) + 1);
1310 			ramp_up_wait += 100000;
1311 		}
1312 
1313 		ccfifo_writel(emc, cfg5 & ~EMC_FBIO_CFG5_CMD_TX_DIS,
1314 			      EMC_FBIO_CFG5, (100000 / clk) + 10);
1315 		ramp_up_wait += 100000 + (10 * clk);
1316 	} else if (clk < 1000000 / DVFS_FGCG_MID_SPEED_THRESHOLD) {
1317 		ccfifo_writel(emc, rfu1 | 0x06000600,
1318 			      EMC_PMACRO_BRICK_CTRL_RFU1, (100000 / clk) + 1);
1319 		ccfifo_writel(emc, cfg5 & ~EMC_FBIO_CFG5_CMD_TX_DIS,
1320 			      EMC_FBIO_CFG5, (100000 / clk) + 10);
1321 		ramp_up_wait += 100000 + 10 * clk;
1322 	} else {
1323 		ccfifo_writel(emc, rfu1 | 0x00000600,
1324 			      EMC_PMACRO_BRICK_CTRL_RFU1, 0);
1325 		ccfifo_writel(emc, cfg5 & ~EMC_FBIO_CFG5_CMD_TX_DIS,
1326 			      EMC_FBIO_CFG5, 12);
1327 		ramp_up_wait += 12 * clk;
1328 	}
1329 
1330 	cmd_pad &= ~EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_DRVFORCEON;
1331 	ccfifo_writel(emc, cmd_pad, EMC_PMACRO_CMD_PAD_TX_CTRL, 5);
1332 
1333 	return ramp_up_wait;
1334 }
1335 
1336 u32 tegra210_emc_dvfs_power_ramp_down(struct tegra210_emc *emc, u32 clk,
1337 				      bool flip_backward)
1338 {
1339 	u32 ramp_down_wait = 0, cmd_pad, dq_pad, rfu1, cfg5, common_tx;
1340 	const struct tegra210_emc_timing *entry;
1341 	u32 seq_wait;
1342 
1343 	if (flip_backward)
1344 		entry = emc->next;
1345 	else
1346 		entry = emc->last;
1347 
1348 	cmd_pad = entry->burst_regs[EMC_PMACRO_CMD_PAD_TX_CTRL_INDEX];
1349 	dq_pad = entry->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX];
1350 	rfu1 = entry->burst_regs[EMC_PMACRO_BRICK_CTRL_RFU1_INDEX];
1351 	cfg5 = entry->burst_regs[EMC_FBIO_CFG5_INDEX];
1352 	common_tx = entry->burst_regs[EMC_PMACRO_COMMON_PAD_TX_CTRL_INDEX];
1353 
1354 	cmd_pad |= EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_DRVFORCEON;
1355 
1356 	ccfifo_writel(emc, cmd_pad, EMC_PMACRO_CMD_PAD_TX_CTRL, 0);
1357 	ccfifo_writel(emc, cfg5 | EMC_FBIO_CFG5_CMD_TX_DIS,
1358 		      EMC_FBIO_CFG5, 12);
1359 	ramp_down_wait = 12 * clk;
1360 
1361 	seq_wait = (100000 / clk) + 1;
1362 
1363 	if (clk < (1000000 / DVFS_FGCG_HIGH_SPEED_THRESHOLD)) {
1364 		if (clk < (1000000 / IOBRICK_DCC_THRESHOLD)) {
1365 			cmd_pad &=
1366 				~(EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_E_DCC |
1367 				  EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_CMD_TX_E_DCC);
1368 			cmd_pad |=
1369 				EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSP_TX_E_DCC |
1370 				EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSN_TX_E_DCC;
1371 			ccfifo_writel(emc, cmd_pad,
1372 				      EMC_PMACRO_CMD_PAD_TX_CTRL, seq_wait);
1373 			ramp_down_wait += 100000;
1374 
1375 			dq_pad &=
1376 			      ~(EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_TX_E_DCC |
1377 				EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_CMD_TX_E_DCC);
1378 			dq_pad |=
1379 				EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSP_TX_E_DCC |
1380 				EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSN_TX_E_DCC;
1381 			ccfifo_writel(emc, dq_pad,
1382 				      EMC_PMACRO_DATA_PAD_TX_CTRL, 0);
1383 			ccfifo_writel(emc, rfu1 & ~0x01120112,
1384 				      EMC_PMACRO_BRICK_CTRL_RFU1, 0);
1385 		} else {
1386 			ccfifo_writel(emc, rfu1 & ~0x01120112,
1387 				      EMC_PMACRO_BRICK_CTRL_RFU1, seq_wait);
1388 			ramp_down_wait += 100000;
1389 		}
1390 
1391 		ccfifo_writel(emc, rfu1 & ~0x01bf01bf,
1392 			      EMC_PMACRO_BRICK_CTRL_RFU1, seq_wait);
1393 		ramp_down_wait += 100000;
1394 
1395 		if (clk < (1000000 / IOBRICK_DCC_THRESHOLD)) {
1396 			cmd_pad &=
1397 				~(EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_E_DCC |
1398 				  EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_CMD_TX_E_DCC |
1399 				  EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSP_TX_E_DCC |
1400 				  EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSN_TX_E_DCC);
1401 			ccfifo_writel(emc, cmd_pad,
1402 				      EMC_PMACRO_CMD_PAD_TX_CTRL, seq_wait);
1403 			ramp_down_wait += 100000;
1404 
1405 			dq_pad &=
1406 			      ~(EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_TX_E_DCC |
1407 				EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_CMD_TX_E_DCC |
1408 				EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSP_TX_E_DCC |
1409 				EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSN_TX_E_DCC);
1410 			ccfifo_writel(emc, dq_pad,
1411 				      EMC_PMACRO_DATA_PAD_TX_CTRL, 0);
1412 			ccfifo_writel(emc, rfu1 & ~0x07ff07ff,
1413 				      EMC_PMACRO_BRICK_CTRL_RFU1, 0);
1414 		} else {
1415 			ccfifo_writel(emc, rfu1 & ~0x07ff07ff,
1416 				      EMC_PMACRO_BRICK_CTRL_RFU1, seq_wait);
1417 			ramp_down_wait += 100000;
1418 		}
1419 	} else {
1420 		ccfifo_writel(emc, rfu1 & ~0xffff07ff,
1421 			      EMC_PMACRO_BRICK_CTRL_RFU1, seq_wait + 19);
1422 		ramp_down_wait += 100000 + (20 * clk);
1423 	}
1424 
1425 	if (clk < (1000000 / DVFS_FGCG_MID_SPEED_THRESHOLD)) {
1426 		ramp_down_wait += 100000;
1427 		ccfifo_writel(emc, common_tx & ~0x5,
1428 			      EMC_PMACRO_COMMON_PAD_TX_CTRL, seq_wait);
1429 		ramp_down_wait += 100000;
1430 		ccfifo_writel(emc, common_tx & ~0xf,
1431 			      EMC_PMACRO_COMMON_PAD_TX_CTRL, seq_wait);
1432 		ramp_down_wait += 100000;
1433 		ccfifo_writel(emc, 0, 0, seq_wait);
1434 		ramp_down_wait += 100000;
1435 	} else {
1436 		ccfifo_writel(emc, common_tx & ~0xf,
1437 			      EMC_PMACRO_COMMON_PAD_TX_CTRL, seq_wait);
1438 	}
1439 
1440 	return ramp_down_wait;
1441 }
1442 
1443 void tegra210_emc_reset_dram_clktree_values(struct tegra210_emc_timing *timing)
1444 {
1445 	timing->current_dram_clktree[C0D0U0] =
1446 		timing->trained_dram_clktree[C0D0U0];
1447 	timing->current_dram_clktree[C0D0U1] =
1448 		timing->trained_dram_clktree[C0D0U1];
1449 	timing->current_dram_clktree[C1D0U0] =
1450 		timing->trained_dram_clktree[C1D0U0];
1451 	timing->current_dram_clktree[C1D0U1] =
1452 		timing->trained_dram_clktree[C1D0U1];
1453 	timing->current_dram_clktree[C1D1U0] =
1454 		timing->trained_dram_clktree[C1D1U0];
1455 	timing->current_dram_clktree[C1D1U1] =
1456 		timing->trained_dram_clktree[C1D1U1];
1457 }
1458 
1459 static void update_dll_control(struct tegra210_emc *emc, u32 value, bool state)
1460 {
1461 	unsigned int i;
1462 
1463 	emc_writel(emc, value, EMC_CFG_DIG_DLL);
1464 	tegra210_emc_timing_update(emc);
1465 
1466 	for (i = 0; i < emc->num_channels; i++)
1467 		tegra210_emc_wait_for_update(emc, i, EMC_CFG_DIG_DLL,
1468 					     EMC_CFG_DIG_DLL_CFG_DLL_EN,
1469 					     state);
1470 }
1471 
1472 void tegra210_emc_dll_disable(struct tegra210_emc *emc)
1473 {
1474 	u32 value;
1475 
1476 	value = emc_readl(emc, EMC_CFG_DIG_DLL);
1477 	value &= ~EMC_CFG_DIG_DLL_CFG_DLL_EN;
1478 
1479 	update_dll_control(emc, value, false);
1480 }
1481 
1482 void tegra210_emc_dll_enable(struct tegra210_emc *emc)
1483 {
1484 	u32 value;
1485 
1486 	value = emc_readl(emc, EMC_CFG_DIG_DLL);
1487 	value |= EMC_CFG_DIG_DLL_CFG_DLL_EN;
1488 
1489 	update_dll_control(emc, value, true);
1490 }
1491 
1492 void tegra210_emc_adjust_timing(struct tegra210_emc *emc,
1493 				struct tegra210_emc_timing *timing)
1494 {
1495 	u32 dsr_cntrl = timing->burst_regs[EMC_DYN_SELF_REF_CONTROL_INDEX];
1496 	u32 pre_ref = timing->burst_regs[EMC_PRE_REFRESH_REQ_CNT_INDEX];
1497 	u32 ref = timing->burst_regs[EMC_REFRESH_INDEX];
1498 
1499 	switch (emc->refresh) {
1500 	case TEGRA210_EMC_REFRESH_NOMINAL:
1501 	case TEGRA210_EMC_REFRESH_THROTTLE:
1502 		break;
1503 
1504 	case TEGRA210_EMC_REFRESH_2X:
1505 		ref = REFRESH_SPEEDUP(ref, 2);
1506 		pre_ref = REFRESH_SPEEDUP(pre_ref, 2);
1507 		dsr_cntrl = REFRESH_SPEEDUP(dsr_cntrl, 2);
1508 		break;
1509 
1510 	case TEGRA210_EMC_REFRESH_4X:
1511 		ref = REFRESH_SPEEDUP(ref, 4);
1512 		pre_ref = REFRESH_SPEEDUP(pre_ref, 4);
1513 		dsr_cntrl = REFRESH_SPEEDUP(dsr_cntrl, 4);
1514 		break;
1515 
1516 	default:
1517 		dev_warn(emc->dev, "failed to set refresh: %d\n", emc->refresh);
1518 		return;
1519 	}
1520 
1521 	emc_writel(emc, ref, emc->offsets->burst[EMC_REFRESH_INDEX]);
1522 	emc_writel(emc, pre_ref,
1523 		   emc->offsets->burst[EMC_PRE_REFRESH_REQ_CNT_INDEX]);
1524 	emc_writel(emc, dsr_cntrl,
1525 		   emc->offsets->burst[EMC_DYN_SELF_REF_CONTROL_INDEX]);
1526 }
1527 
1528 static int tegra210_emc_set_rate(struct device *dev,
1529 				 const struct tegra210_clk_emc_config *config)
1530 {
1531 	struct tegra210_emc *emc = dev_get_drvdata(dev);
1532 	struct tegra210_emc_timing *timing = NULL;
1533 	unsigned long rate = config->rate;
1534 	s64 last_change_delay;
1535 	unsigned long flags;
1536 	unsigned int i;
1537 
1538 	if (rate == emc->last->rate * 1000UL)
1539 		return 0;
1540 
1541 	for (i = 0; i < emc->num_timings; i++) {
1542 		if (emc->timings[i].rate * 1000UL == rate) {
1543 			timing = &emc->timings[i];
1544 			break;
1545 		}
1546 	}
1547 
1548 	if (!timing)
1549 		return -EINVAL;
1550 
1551 	if (rate > 204000000 && !timing->trained)
1552 		return -EINVAL;
1553 
1554 	emc->next = timing;
1555 	last_change_delay = ktime_us_delta(ktime_get(), emc->clkchange_time);
1556 
1557 	/* XXX use non-busy-looping sleep? */
1558 	if ((last_change_delay >= 0) &&
1559 	    (last_change_delay < emc->clkchange_delay))
1560 		udelay(emc->clkchange_delay - (int)last_change_delay);
1561 
1562 	spin_lock_irqsave(&emc->lock, flags);
1563 	tegra210_emc_set_clock(emc, config->value);
1564 	emc->clkchange_time = ktime_get();
1565 	emc->last = timing;
1566 	spin_unlock_irqrestore(&emc->lock, flags);
1567 
1568 	return 0;
1569 }
1570 
1571 /*
1572  * debugfs interface
1573  *
1574  * The memory controller driver exposes some files in debugfs that can be used
1575  * to control the EMC frequency. The top-level directory can be found here:
1576  *
1577  *   /sys/kernel/debug/emc
1578  *
1579  * It contains the following files:
1580  *
1581  *   - available_rates: This file contains a list of valid, space-separated
1582  *     EMC frequencies.
1583  *
1584  *   - min_rate: Writing a value to this file sets the given frequency as the
1585  *       floor of the permitted range. If this is higher than the currently
1586  *       configured EMC frequency, this will cause the frequency to be
1587  *       increased so that it stays within the valid range.
1588  *
1589  *   - max_rate: Similarily to the min_rate file, writing a value to this file
1590  *       sets the given frequency as the ceiling of the permitted range. If
1591  *       the value is lower than the currently configured EMC frequency, this
1592  *       will cause the frequency to be decreased so that it stays within the
1593  *       valid range.
1594  */
1595 
1596 static bool tegra210_emc_validate_rate(struct tegra210_emc *emc,
1597 				       unsigned long rate)
1598 {
1599 	unsigned int i;
1600 
1601 	for (i = 0; i < emc->num_timings; i++)
1602 		if (rate == emc->timings[i].rate * 1000UL)
1603 			return true;
1604 
1605 	return false;
1606 }
1607 
1608 static int tegra210_emc_debug_available_rates_show(struct seq_file *s,
1609 						   void *data)
1610 {
1611 	struct tegra210_emc *emc = s->private;
1612 	const char *prefix = "";
1613 	unsigned int i;
1614 
1615 	for (i = 0; i < emc->num_timings; i++) {
1616 		seq_printf(s, "%s%u", prefix, emc->timings[i].rate * 1000);
1617 		prefix = " ";
1618 	}
1619 
1620 	seq_puts(s, "\n");
1621 
1622 	return 0;
1623 }
1624 DEFINE_SHOW_ATTRIBUTE(tegra210_emc_debug_available_rates);
1625 
1626 static int tegra210_emc_debug_min_rate_get(void *data, u64 *rate)
1627 {
1628 	struct tegra210_emc *emc = data;
1629 
1630 	*rate = emc->debugfs.min_rate;
1631 
1632 	return 0;
1633 }
1634 
1635 static int tegra210_emc_debug_min_rate_set(void *data, u64 rate)
1636 {
1637 	struct tegra210_emc *emc = data;
1638 	int err;
1639 
1640 	if (!tegra210_emc_validate_rate(emc, rate))
1641 		return -EINVAL;
1642 
1643 	err = clk_set_min_rate(emc->clk, rate);
1644 	if (err < 0)
1645 		return err;
1646 
1647 	emc->debugfs.min_rate = rate;
1648 
1649 	return 0;
1650 }
1651 
1652 DEFINE_DEBUGFS_ATTRIBUTE(tegra210_emc_debug_min_rate_fops,
1653 			tegra210_emc_debug_min_rate_get,
1654 			tegra210_emc_debug_min_rate_set, "%llu\n");
1655 
1656 static int tegra210_emc_debug_max_rate_get(void *data, u64 *rate)
1657 {
1658 	struct tegra210_emc *emc = data;
1659 
1660 	*rate = emc->debugfs.max_rate;
1661 
1662 	return 0;
1663 }
1664 
1665 static int tegra210_emc_debug_max_rate_set(void *data, u64 rate)
1666 {
1667 	struct tegra210_emc *emc = data;
1668 	int err;
1669 
1670 	if (!tegra210_emc_validate_rate(emc, rate))
1671 		return -EINVAL;
1672 
1673 	err = clk_set_max_rate(emc->clk, rate);
1674 	if (err < 0)
1675 		return err;
1676 
1677 	emc->debugfs.max_rate = rate;
1678 
1679 	return 0;
1680 }
1681 
1682 DEFINE_DEBUGFS_ATTRIBUTE(tegra210_emc_debug_max_rate_fops,
1683 			tegra210_emc_debug_max_rate_get,
1684 			tegra210_emc_debug_max_rate_set, "%llu\n");
1685 
1686 static int tegra210_emc_debug_temperature_get(void *data, u64 *temperature)
1687 {
1688 	struct tegra210_emc *emc = data;
1689 	unsigned int value;
1690 
1691 	if (!emc->debugfs.temperature)
1692 		value = tegra210_emc_get_temperature(emc);
1693 	else
1694 		value = emc->debugfs.temperature;
1695 
1696 	*temperature = value;
1697 
1698 	return 0;
1699 }
1700 
1701 static int tegra210_emc_debug_temperature_set(void *data, u64 temperature)
1702 {
1703 	struct tegra210_emc *emc = data;
1704 
1705 	if (temperature > 7)
1706 		return -EINVAL;
1707 
1708 	emc->debugfs.temperature = temperature;
1709 
1710 	return 0;
1711 }
1712 
1713 DEFINE_DEBUGFS_ATTRIBUTE(tegra210_emc_debug_temperature_fops,
1714 			tegra210_emc_debug_temperature_get,
1715 			tegra210_emc_debug_temperature_set, "%llu\n");
1716 
1717 static void tegra210_emc_debugfs_init(struct tegra210_emc *emc)
1718 {
1719 	struct device *dev = emc->dev;
1720 	unsigned int i;
1721 	int err;
1722 
1723 	emc->debugfs.min_rate = ULONG_MAX;
1724 	emc->debugfs.max_rate = 0;
1725 
1726 	for (i = 0; i < emc->num_timings; i++) {
1727 		if (emc->timings[i].rate * 1000UL < emc->debugfs.min_rate)
1728 			emc->debugfs.min_rate = emc->timings[i].rate * 1000UL;
1729 
1730 		if (emc->timings[i].rate * 1000UL > emc->debugfs.max_rate)
1731 			emc->debugfs.max_rate = emc->timings[i].rate * 1000UL;
1732 	}
1733 
1734 	if (!emc->num_timings) {
1735 		emc->debugfs.min_rate = clk_get_rate(emc->clk);
1736 		emc->debugfs.max_rate = emc->debugfs.min_rate;
1737 	}
1738 
1739 	err = clk_set_rate_range(emc->clk, emc->debugfs.min_rate,
1740 				 emc->debugfs.max_rate);
1741 	if (err < 0) {
1742 		dev_err(dev, "failed to set rate range [%lu-%lu] for %pC\n",
1743 			emc->debugfs.min_rate, emc->debugfs.max_rate,
1744 			emc->clk);
1745 		return;
1746 	}
1747 
1748 	emc->debugfs.root = debugfs_create_dir("emc", NULL);
1749 
1750 	debugfs_create_file("available_rates", 0444, emc->debugfs.root, emc,
1751 			    &tegra210_emc_debug_available_rates_fops);
1752 	debugfs_create_file("min_rate", 0644, emc->debugfs.root, emc,
1753 			    &tegra210_emc_debug_min_rate_fops);
1754 	debugfs_create_file("max_rate", 0644, emc->debugfs.root, emc,
1755 			    &tegra210_emc_debug_max_rate_fops);
1756 	debugfs_create_file("temperature", 0644, emc->debugfs.root, emc,
1757 			    &tegra210_emc_debug_temperature_fops);
1758 }
1759 
1760 static void tegra210_emc_detect(struct tegra210_emc *emc)
1761 {
1762 	u32 value;
1763 
1764 	/* probe the number of connected DRAM devices */
1765 	value = mc_readl(emc->mc, MC_EMEM_ADR_CFG);
1766 
1767 	if (value & MC_EMEM_ADR_CFG_EMEM_NUMDEV)
1768 		emc->num_devices = 2;
1769 	else
1770 		emc->num_devices = 1;
1771 
1772 	/* probe the type of DRAM */
1773 	value = emc_readl(emc, EMC_FBIO_CFG5);
1774 	emc->dram_type = value & 0x3;
1775 
1776 	/* probe the number of channels */
1777 	value = emc_readl(emc, EMC_FBIO_CFG7);
1778 
1779 	if ((value & EMC_FBIO_CFG7_CH1_ENABLE) &&
1780 	    (value & EMC_FBIO_CFG7_CH0_ENABLE))
1781 		emc->num_channels = 2;
1782 	else
1783 		emc->num_channels = 1;
1784 }
1785 
1786 static int tegra210_emc_validate_timings(struct tegra210_emc *emc,
1787 					 struct tegra210_emc_timing *timings,
1788 					 unsigned int num_timings)
1789 {
1790 	unsigned int i;
1791 
1792 	for (i = 0; i < num_timings; i++) {
1793 		u32 min_volt = timings[i].min_volt;
1794 		u32 rate = timings[i].rate;
1795 
1796 		if (!rate)
1797 			return -EINVAL;
1798 
1799 		if ((i > 0) && ((rate <= timings[i - 1].rate) ||
1800 		    (min_volt < timings[i - 1].min_volt)))
1801 			return -EINVAL;
1802 
1803 		if (timings[i].revision != timings[0].revision)
1804 			continue;
1805 	}
1806 
1807 	return 0;
1808 }
1809 
1810 static int tegra210_emc_probe(struct platform_device *pdev)
1811 {
1812 	struct thermal_cooling_device *cd;
1813 	unsigned long current_rate;
1814 	struct tegra210_emc *emc;
1815 	struct device_node *np;
1816 	unsigned int i;
1817 	int err;
1818 
1819 	emc = devm_kzalloc(&pdev->dev, sizeof(*emc), GFP_KERNEL);
1820 	if (!emc)
1821 		return -ENOMEM;
1822 
1823 	emc->clk = devm_clk_get(&pdev->dev, "emc");
1824 	if (IS_ERR(emc->clk))
1825 		return PTR_ERR(emc->clk);
1826 
1827 	platform_set_drvdata(pdev, emc);
1828 	spin_lock_init(&emc->lock);
1829 	emc->dev = &pdev->dev;
1830 
1831 	emc->mc = devm_tegra_memory_controller_get(&pdev->dev);
1832 	if (IS_ERR(emc->mc))
1833 		return PTR_ERR(emc->mc);
1834 
1835 	emc->regs = devm_platform_ioremap_resource(pdev, 0);
1836 	if (IS_ERR(emc->regs))
1837 		return PTR_ERR(emc->regs);
1838 
1839 	for (i = 0; i < 2; i++) {
1840 		emc->channel[i] = devm_platform_ioremap_resource(pdev, 1 + i);
1841 		if (IS_ERR(emc->channel[i]))
1842 			return PTR_ERR(emc->channel[i]);
1843 
1844 	}
1845 
1846 	tegra210_emc_detect(emc);
1847 	np = pdev->dev.of_node;
1848 
1849 	/* attach to the nominal and (optional) derated tables */
1850 	err = of_reserved_mem_device_init_by_name(emc->dev, np, "nominal");
1851 	if (err < 0) {
1852 		dev_err(emc->dev, "failed to get nominal EMC table: %d\n", err);
1853 		return err;
1854 	}
1855 
1856 	err = of_reserved_mem_device_init_by_name(emc->dev, np, "derated");
1857 	if (err < 0 && err != -ENODEV) {
1858 		dev_err(emc->dev, "failed to get derated EMC table: %d\n", err);
1859 		goto release;
1860 	}
1861 
1862 	/* validate the tables */
1863 	if (emc->nominal) {
1864 		err = tegra210_emc_validate_timings(emc, emc->nominal,
1865 						    emc->num_timings);
1866 		if (err < 0)
1867 			goto release;
1868 	}
1869 
1870 	if (emc->derated) {
1871 		err = tegra210_emc_validate_timings(emc, emc->derated,
1872 						    emc->num_timings);
1873 		if (err < 0)
1874 			goto release;
1875 	}
1876 
1877 	/* default to the nominal table */
1878 	emc->timings = emc->nominal;
1879 
1880 	/* pick the current timing based on the current EMC clock rate */
1881 	current_rate = clk_get_rate(emc->clk) / 1000;
1882 
1883 	for (i = 0; i < emc->num_timings; i++) {
1884 		if (emc->timings[i].rate == current_rate) {
1885 			emc->last = &emc->timings[i];
1886 			break;
1887 		}
1888 	}
1889 
1890 	if (i == emc->num_timings) {
1891 		dev_err(emc->dev, "no EMC table entry found for %lu kHz\n",
1892 			current_rate);
1893 		err = -ENOENT;
1894 		goto release;
1895 	}
1896 
1897 	/* pick a compatible clock change sequence for the EMC table */
1898 	for (i = 0; i < ARRAY_SIZE(tegra210_emc_sequences); i++) {
1899 		const struct tegra210_emc_sequence *sequence =
1900 				tegra210_emc_sequences[i];
1901 
1902 		if (emc->timings[0].revision == sequence->revision) {
1903 			emc->sequence = sequence;
1904 			break;
1905 		}
1906 	}
1907 
1908 	if (!emc->sequence) {
1909 		dev_err(&pdev->dev, "sequence %u not supported\n",
1910 			emc->timings[0].revision);
1911 		err = -ENOTSUPP;
1912 		goto release;
1913 	}
1914 
1915 	emc->offsets = &tegra210_emc_table_register_offsets;
1916 	emc->refresh = TEGRA210_EMC_REFRESH_NOMINAL;
1917 
1918 	emc->provider.owner = THIS_MODULE;
1919 	emc->provider.dev = &pdev->dev;
1920 	emc->provider.set_rate = tegra210_emc_set_rate;
1921 
1922 	emc->provider.configs = devm_kcalloc(&pdev->dev, emc->num_timings,
1923 					     sizeof(*emc->provider.configs),
1924 					     GFP_KERNEL);
1925 	if (!emc->provider.configs) {
1926 		err = -ENOMEM;
1927 		goto release;
1928 	}
1929 
1930 	emc->provider.num_configs = emc->num_timings;
1931 
1932 	for (i = 0; i < emc->provider.num_configs; i++) {
1933 		struct tegra210_emc_timing *timing = &emc->timings[i];
1934 		struct tegra210_clk_emc_config *config =
1935 				&emc->provider.configs[i];
1936 		u32 value;
1937 
1938 		config->rate = timing->rate * 1000UL;
1939 		config->value = timing->clk_src_emc;
1940 
1941 		value = timing->burst_mc_regs[MC_EMEM_ARB_MISC0_INDEX];
1942 
1943 		if ((value & MC_EMEM_ARB_MISC0_EMC_SAME_FREQ) == 0)
1944 			config->same_freq = false;
1945 		else
1946 			config->same_freq = true;
1947 	}
1948 
1949 	err = tegra210_clk_emc_attach(emc->clk, &emc->provider);
1950 	if (err < 0) {
1951 		dev_err(&pdev->dev, "failed to attach to EMC clock: %d\n", err);
1952 		goto release;
1953 	}
1954 
1955 	emc->clkchange_delay = 100;
1956 	emc->training_interval = 100;
1957 	dev_set_drvdata(emc->dev, emc);
1958 
1959 	timer_setup(&emc->refresh_timer, tegra210_emc_poll_refresh,
1960 		    TIMER_DEFERRABLE);
1961 	atomic_set(&emc->refresh_poll, 0);
1962 	emc->refresh_poll_interval = 1000;
1963 
1964 	timer_setup(&emc->training, tegra210_emc_train, 0);
1965 
1966 	tegra210_emc_debugfs_init(emc);
1967 
1968 	cd = devm_thermal_of_cooling_device_register(emc->dev, np, "emc", emc,
1969 						     &tegra210_emc_cd_ops);
1970 	if (IS_ERR(cd)) {
1971 		err = PTR_ERR(cd);
1972 		dev_err(emc->dev, "failed to register cooling device: %d\n",
1973 			err);
1974 		goto detach;
1975 	}
1976 
1977 	return 0;
1978 
1979 detach:
1980 	debugfs_remove_recursive(emc->debugfs.root);
1981 	tegra210_clk_emc_detach(emc->clk);
1982 release:
1983 	of_reserved_mem_device_release(emc->dev);
1984 
1985 	return err;
1986 }
1987 
1988 static int tegra210_emc_remove(struct platform_device *pdev)
1989 {
1990 	struct tegra210_emc *emc = platform_get_drvdata(pdev);
1991 
1992 	debugfs_remove_recursive(emc->debugfs.root);
1993 	tegra210_clk_emc_detach(emc->clk);
1994 	of_reserved_mem_device_release(emc->dev);
1995 
1996 	return 0;
1997 }
1998 
1999 static int __maybe_unused tegra210_emc_suspend(struct device *dev)
2000 {
2001 	struct tegra210_emc *emc = dev_get_drvdata(dev);
2002 	int err;
2003 
2004 	err = clk_rate_exclusive_get(emc->clk);
2005 	if (err < 0) {
2006 		dev_err(emc->dev, "failed to acquire clock: %d\n", err);
2007 		return err;
2008 	}
2009 
2010 	emc->resume_rate = clk_get_rate(emc->clk);
2011 
2012 	clk_set_rate(emc->clk, 204000000);
2013 	tegra210_clk_emc_detach(emc->clk);
2014 
2015 	dev_dbg(dev, "suspending at %lu Hz\n", clk_get_rate(emc->clk));
2016 
2017 	return 0;
2018 }
2019 
2020 static int __maybe_unused tegra210_emc_resume(struct device *dev)
2021 {
2022 	struct tegra210_emc *emc = dev_get_drvdata(dev);
2023 	int err;
2024 
2025 	err = tegra210_clk_emc_attach(emc->clk, &emc->provider);
2026 	if (err < 0) {
2027 		dev_err(dev, "failed to attach to EMC clock: %d\n", err);
2028 		return err;
2029 	}
2030 
2031 	clk_set_rate(emc->clk, emc->resume_rate);
2032 	clk_rate_exclusive_put(emc->clk);
2033 
2034 	dev_dbg(dev, "resuming at %lu Hz\n", clk_get_rate(emc->clk));
2035 
2036 	return 0;
2037 }
2038 
2039 static const struct dev_pm_ops tegra210_emc_pm_ops = {
2040 	SET_SYSTEM_SLEEP_PM_OPS(tegra210_emc_suspend, tegra210_emc_resume)
2041 };
2042 
2043 static const struct of_device_id tegra210_emc_of_match[] = {
2044 	{ .compatible = "nvidia,tegra210-emc", },
2045 	{ },
2046 };
2047 MODULE_DEVICE_TABLE(of, tegra210_emc_of_match);
2048 
2049 static struct platform_driver tegra210_emc_driver = {
2050 	.driver = {
2051 		.name = "tegra210-emc",
2052 		.of_match_table = tegra210_emc_of_match,
2053 		.pm = &tegra210_emc_pm_ops,
2054 	},
2055 	.probe = tegra210_emc_probe,
2056 	.remove = tegra210_emc_remove,
2057 };
2058 
2059 module_platform_driver(tegra210_emc_driver);
2060 
2061 MODULE_AUTHOR("Thierry Reding <treding@nvidia.com>");
2062 MODULE_AUTHOR("Joseph Lo <josephl@nvidia.com>");
2063 MODULE_DESCRIPTION("NVIDIA Tegra210 EMC driver");
2064 MODULE_LICENSE("GPL v2");
2065