1 /* SPDX-License-Identifier: (GPL-2.0+ OR MIT) */
2 /*
3  * Copyright (c) 2018 Microsemi Corporation
4  */
5 
6 #ifndef __ASM_MACH_DDR_H
7 #define __ASM_MACH_DDR_H
8 
9 #include <asm/cacheops.h>
10 #include <asm/io.h>
11 #include <asm/reboot.h>
12 #include <mach/common.h>
13 
14 #define MIPS_VCOREIII_MEMORY_DDR3
15 #define MIPS_VCOREIII_DDR_SIZE CONFIG_SYS_SDRAM_SIZE
16 
17 #if defined(CONFIG_DDRTYPE_H5TQ1G63BFA)	/* Serval1 Refboard */
18 
19 /* Hynix H5TQ1G63BFA (1Gbit DDR3, x16) @ 3.20ns */
20 #define VC3_MPAR_bank_addr_cnt    3
21 #define VC3_MPAR_row_addr_cnt     13
22 #define VC3_MPAR_col_addr_cnt     10
23 #define VC3_MPAR_tREFI            2437
24 #define VC3_MPAR_tRAS_min         12
25 #define VC3_MPAR_CL               6
26 #define VC3_MPAR_tWTR             4
27 #define VC3_MPAR_tRC              16
28 #define VC3_MPR_tFAW             16
29 #define VC3_MPAR_tRP              5
30 #define VC3_MPAR_tRRD             4
31 #define VC3_MPAR_tRCD             5
32 #define VC3_MPAR_tMRD             4
33 #define VC3_MPAR_tRFC             35
34 #define VC3_MPAR_CWL              5
35 #define VC3_MPAR_tXPR             38
36 #define VC3_MPAR_tMOD             12
37 #define VC3_MPAR_tDLLK            512
38 #define VC3_MPAR_tWR              5
39 
40 #elif defined(CONFIG_DDRTYPE_MT41J128M16HA)	/* Validation board */
41 
42 /* Micron MT41J128M16HA-15E:D (2Gbit DDR3, x16) @ 3.20ns */
43 #define VC3_MPAR_bank_addr_cnt    3
44 #define VC3_MPAR_row_addr_cnt     14
45 #define VC3_MPAR_col_addr_cnt     10
46 #define VC3_MPAR_tREFI            2437
47 #define VC3_MPAR_tRAS_min         12
48 #define VC3_MPAR_CL               5
49 #define VC3_MPAR_tWTR             4
50 #define VC3_MPAR_tRC              16
51 #define VC3_MPAR_tFAW             16
52 #define VC3_MPAR_tRP              5
53 #define VC3_MPAR_tRRD             4
54 #define VC3_MPAR_tRCD             5
55 #define VC3_MPAR_tMRD             4
56 #define VC3_MPAR_tRFC             50
57 #define VC3_MPAR_CWL              5
58 #define VC3_MPAR_tXPR             54
59 #define VC3_MPAR_tMOD             12
60 #define VC3_MPAR_tDLLK            512
61 #define VC3_MPAR_tWR              5
62 
63 #elif defined(CONFIG_DDRTYPE_MT41K256M16)	/* JR2 Validation board */
64 
65 /* Micron MT41K256M16 (4Gbit, DDR3L-800, 256Mbitx16) @ 3.20ns */
66 #define VC3_MPAR_bank_addr_cnt    3
67 #define VC3_MPAR_row_addr_cnt     15
68 #define VC3_MPAR_col_addr_cnt     10
69 #define VC3_MPAR_tREFI            2437
70 #define VC3_MPAR_tRAS_min         12
71 #define VC3_MPAR_CL               5
72 #define VC3_MPAR_tWTR             4
73 #define VC3_MPAR_tRC              16
74 #define VC3_MPAR_tFAW             16
75 #define VC3_MPAR_tRP              5
76 #define VC3_MPAR_tRRD             4
77 #define VC3_MPAR_tRCD             5
78 #define VC3_MPAR_tMRD             4
79 #define VC3_MPAR_tRFC             82
80 #define VC3_MPAR_CWL              5
81 #define VC3_MPAR_tXPR             85
82 #define VC3_MPAR_tMOD             12
83 #define VC3_MPAR_tDLLK            512
84 #define VC3_MPAR_tWR              5
85 
86 #elif defined(CONFIG_DDRTYPE_H5TQ4G63MFR)	/* JR2 Reference board */
87 
88 /* Hynix H5TQ4G63MFR-PBC (4Gbit, DDR3-800, 256Mbitx16) - 2kb pages @ 3.20ns */
89 #define VC3_MPAR_bank_addr_cnt    3
90 #define VC3_MPAR_row_addr_cnt     15
91 #define VC3_MPAR_col_addr_cnt     10
92 #define VC3_MPAR_tREFI            2437
93 #define VC3_MPAR_tRAS_min         12
94 #define VC3_MPAR_CL               6
95 #define VC3_MPAR_tWTR             4
96 #define VC3_MPAR_tRC              17
97 #define VC3_MPAR_tFAW             16
98 #define VC3_MPAR_tRP              5
99 #define VC3_MPAR_tRRD             4
100 #define VC3_MPAR_tRCD             5
101 #define VC3_MPAR_tMRD             4
102 #define VC3_MPAR_tRFC             82
103 #define VC3_MPAR_CWL              5
104 #define VC3_MPAR_tXPR             85
105 #define VC3_MPAR_tMOD             12
106 #define VC3_MPAR_tDLLK            512
107 #define VC3_MPAR_tWR              5
108 
109 #elif defined(CONFIG_DDRTYPE_MT41K128M16JT)
110 
111 /* Micron Micron MT41K128M16JT-125 (2Gbit DDR3L, 128Mbitx16) @ 3.20ns */
112 #define VC3_MPAR_bank_addr_cnt    3
113 #define VC3_MPAR_row_addr_cnt     14
114 #define VC3_MPAR_col_addr_cnt     10
115 #define VC3_MPAR_tREFI            2437
116 #define VC3_MPAR_tRAS_min         12
117 #define VC3_MPAR_CL               6
118 #define VC3_MPAR_tWTR             4
119 #define VC3_MPAR_tRC              16
120 #define VC3_MPAR_tFAW             16
121 #define VC3_MPAR_tRP              5
122 #define VC3_MPAR_tRRD             4
123 #define VC3_MPAR_tRCD             5
124 #define VC3_MPAR_tMRD             4
125 #define VC3_MPAR_tRFC             82
126 #define VC3_MPAR_CWL              5
127 #define VC3_MPAR_tXPR             85
128 #define VC3_MPAR_tMOD             12
129 #define VC3_MPAR_tDLLK            512
130 #define VC3_MPAR_tWR              5
131 
132 #elif defined(CONFIG_DDRTYPE_MT47H128M8HQ)	/* Luton10/26 Refboards */
133 
134 /* Micron 1Gb MT47H128M8-3 16Meg x 8 x 8 banks, DDR-533@CL4 @ 4.80ns */
135 #define VC3_MPAR_bank_addr_cnt    3
136 #define VC3_MPAR_row_addr_cnt     14
137 #define VC3_MPAR_col_addr_cnt     10
138 #define VC3_MPAR_tREFI            1625
139 #define VC3_MPAR_tRAS_min         9
140 #define VC3_MPAR_CL               4
141 #define VC3_MPAR_tWTR             2
142 #define VC3_MPAR_tRC              12
143 #define VC3_MPAR_tFAW             8
144 #define VC3_MPAR_tRP              4
145 #define VC3_MPAR_tRRD             2
146 #define VC3_MPAR_tRCD             4
147 
148 #define VC3_MPAR_tRPA             4
149 #define VC3_MPAR_tRP              4
150 
151 #define VC3_MPAR_tMRD             2
152 #define VC3_MPAR_tRFC             27
153 
154 #define VC3_MPAR__400_ns_dly      84
155 
156 #define VC3_MPAR_tWR              4
157 #undef MIPS_VCOREIII_MEMORY_DDR3
158 #else
159 
160 #error Unknown DDR system configuration - please add!
161 
162 #endif
163 
164 #if defined(CONFIG_SOC_OCELOT) || defined(CONFIG_SOC_JR2)
165 #define MIPS_VCOREIII_MEMORY_16BIT 1
166 #endif
167 
168 #define MIPS_VCOREIII_MEMORY_SSTL_ODT 7
169 #define MIPS_VCOREIII_MEMORY_SSTL_DRIVE 7
170 #define VCOREIII_DDR_DQS_MODE_CALIBRATE
171 
172 #ifdef MIPS_VCOREIII_MEMORY_16BIT
173 #define VC3_MPAR_16BIT       1
174 #else
175 #define VC3_MPAR_16BIT       0
176 #endif
177 
178 #ifdef MIPS_VCOREIII_MEMORY_DDR3
179 #define VC3_MPAR_DDR3_MODE    1	/* DDR3 */
180 #define VC3_MPAR_BURST_LENGTH 8	/* Always 8 (1) for DDR3 */
181 #ifdef MIPS_VCOREIII_MEMORY_16BIT
182 #define VC3_MPAR_BURST_SIZE   1	/* Always 1 for DDR3/16bit */
183 #else
184 #define VC3_MPAR_BURST_SIZE   0
185 #endif
186 #else
187 #define VC3_MPAR_DDR3_MODE    0	/* DDR2 */
188 #ifdef MIPS_VCOREIII_MEMORY_16BIT
189 #define VC3_MPAR_BURST_LENGTH 4	/* in DDR2 16-bit mode, use burstlen 4 */
190 #else
191 #define VC3_MPAR_BURST_LENGTH 8	/* For 8-bit IF we must run burst-8 */
192 #endif
193 #define VC3_MPAR_BURST_SIZE   0	/* Always 0 for DDR2 */
194 #endif
195 
196 #define VC3_MPAR_RL VC3_MPAR_CL
197 #if !defined(MIPS_VCOREIII_MEMORY_DDR3)
198 #define VC3_MPAR_WL (VC3_MPAR_RL - 1)
199 #define VC3_MPAR_MD VC3_MPAR_tMRD
200 #define VC3_MPAR_ID VC3_MPAR__400_ns_dly
201 #define VC3_MPAR_SD VC3_MPAR_tXSRD
202 #define VC3_MPAR_OW (VC3_MPAR_WL - 2)
203 #define VC3_MPAR_OR (VC3_MPAR_WL - 3)
204 #define VC3_MPAR_RP (VC3_MPAR_bank_addr_cnt < 3 ? VC3_MPAR_tRP : VC3_MPAR_tRPA)
205 #define VC3_MPAR_FAW (VC3_MPAR_bank_addr_cnt < 3 ? 1 : VC3_MPAR_tFAW)
206 #define VC3_MPAR_BL (VC3_MPAR_BURST_LENGTH == 4 ? 2 : 4)
207 #define MSCC_MEMPARM_MR0 \
208 	(VC3_MPAR_BURST_LENGTH == 8 ? 3 : 2) | (VC3_MPAR_CL << 4) | \
209 	((VC3_MPAR_tWR - 1) << 9)
210 /* DLL-on, Full-OD, AL=0, RTT=off, nDQS-on, RDQS-off, out-en */
211 #define MSCC_MEMPARM_MR1 0x382
212 #define MSCC_MEMPARM_MR2 0
213 #define MSCC_MEMPARM_MR3 0
214 #else
215 #define VC3_MPAR_WL VC3_MPAR_CWL
216 #define VC3_MPAR_MD VC3_MPAR_tMOD
217 #define VC3_MPAR_ID VC3_MPAR_tXPR
218 #define VC3_MPAR_SD VC3_MPAR_tDLLK
219 #define VC3_MPAR_OW 2
220 #define VC3_MPAR_OR 2
221 #define VC3_MPAR_RP VC3_MPAR_tRP
222 #define VC3_MPAR_FAW VC3_MPAR_tFAW
223 #define VC3_MPAR_BL 4
224 #define MSCC_MEMPARM_MR0 ((VC3_MPAR_RL - 4) << 4) | ((VC3_MPAR_tWR - 4) << 9)
225 /* ODT_RTT: “0x0040” for 120ohm, and “0x0004” for 60ohm. */
226 #define MSCC_MEMPARM_MR1 0x0040
227 #define MSCC_MEMPARM_MR2 ((VC3_MPAR_WL - 5) << 3)
228 #define MSCC_MEMPARM_MR3 0
229 #endif				/* MIPS_VCOREIII_MEMORY_DDR3 */
230 
231 #define MSCC_MEMPARM_MEMCFG                                             \
232 	((MIPS_VCOREIII_DDR_SIZE > SZ_512M) ?				\
233 	 ICPU_MEMCTRL_CFG_DDR_512MBYTE_PLUS : 0) |			\
234 	(VC3_MPAR_16BIT ? ICPU_MEMCTRL_CFG_DDR_WIDTH : 0) |		\
235 	(VC3_MPAR_DDR3_MODE ? ICPU_MEMCTRL_CFG_DDR_MODE : 0) |		\
236 	(VC3_MPAR_BURST_SIZE ? ICPU_MEMCTRL_CFG_BURST_SIZE : 0) |	\
237 	(VC3_MPAR_BURST_LENGTH == 8 ? ICPU_MEMCTRL_CFG_BURST_LEN : 0) | \
238 	(VC3_MPAR_bank_addr_cnt == 3 ? ICPU_MEMCTRL_CFG_BANK_CNT : 0) | \
239 	ICPU_MEMCTRL_CFG_MSB_ROW_ADDR(VC3_MPAR_row_addr_cnt - 1) |	\
240 	ICPU_MEMCTRL_CFG_MSB_COL_ADDR(VC3_MPAR_col_addr_cnt - 1)
241 
242 #if defined(CONFIG_SOC_OCELOT) || defined(CONFIG_SOC_JR2)
243 #define MSCC_MEMPARM_PERIOD					\
244 	ICPU_MEMCTRL_REF_PERIOD_MAX_PEND_REF(8) |		\
245 	ICPU_MEMCTRL_REF_PERIOD_REF_PERIOD(VC3_MPAR_tREFI)
246 
247 #define MSCC_MEMPARM_TIMING0                                            \
248 	ICPU_MEMCTRL_TIMING0_RD_TO_WR_DLY(VC3_MPAR_RL + VC3_MPAR_BL + 1 - \
249 					  VC3_MPAR_WL) |		\
250 	ICPU_MEMCTRL_TIMING0_WR_CS_CHANGE_DLY(VC3_MPAR_BL - 1) |	\
251 	ICPU_MEMCTRL_TIMING0_RD_CS_CHANGE_DLY(VC3_MPAR_BL) |		\
252 	ICPU_MEMCTRL_TIMING0_RAS_TO_PRECH_DLY(VC3_MPAR_tRAS_min - 1) |	\
253 	ICPU_MEMCTRL_TIMING0_WR_TO_PRECH_DLY(VC3_MPAR_WL +		\
254 					     VC3_MPAR_BL +		\
255 					     VC3_MPAR_tWR - 1) |	\
256 	ICPU_MEMCTRL_TIMING0_RD_TO_PRECH_DLY(VC3_MPAR_BL - 1) |		\
257 		ICPU_MEMCTRL_TIMING0_WR_DATA_XFR_DLY(VC3_MPAR_WL - 1) |	\
258 	ICPU_MEMCTRL_TIMING0_RD_DATA_XFR_DLY(VC3_MPAR_RL - 3)
259 
260 #define MSCC_MEMPARM_TIMING1                                            \
261 	ICPU_MEMCTRL_TIMING1_RAS_TO_RAS_SAME_BANK_DLY(VC3_MPAR_tRC - 1) | \
262 	ICPU_MEMCTRL_TIMING1_BANK8_FAW_DLY(VC3_MPAR_FAW - 1) |		\
263 	ICPU_MEMCTRL_TIMING1_PRECH_TO_RAS_DLY(VC3_MPAR_RP - 1) |	\
264 	ICPU_MEMCTRL_TIMING1_RAS_TO_RAS_DLY(VC3_MPAR_tRRD - 1) |	\
265 	ICPU_MEMCTRL_TIMING1_RAS_TO_CAS_DLY(VC3_MPAR_tRCD - 1) |	\
266 	ICPU_MEMCTRL_TIMING1_WR_TO_RD_DLY(VC3_MPAR_WL +			\
267 					  VC3_MPAR_BL +			\
268 					  VC3_MPAR_tWTR - 1)
269 
270 #define MSCC_MEMPARM_TIMING2					\
271 	ICPU_MEMCTRL_TIMING2_PRECH_ALL_DLY(VC3_MPAR_RP - 1) |	\
272 	ICPU_MEMCTRL_TIMING2_MDSET_DLY(VC3_MPAR_MD - 1) |		\
273 	ICPU_MEMCTRL_TIMING2_REF_DLY(VC3_MPAR_tRFC - 1) |		\
274 	ICPU_MEMCTRL_TIMING2_INIT_DLY(VC3_MPAR_ID - 1)
275 
276 #define MSCC_MEMPARM_TIMING3						\
277 	ICPU_MEMCTRL_TIMING3_WR_TO_RD_CS_CHANGE_DLY(VC3_MPAR_WL +	\
278 						    VC3_MPAR_tWTR - 1) |\
279 	ICPU_MEMCTRL_TIMING3_ODT_RD_DLY(VC3_MPAR_OR - 1) |		\
280 	ICPU_MEMCTRL_TIMING3_ODT_WR_DLY(VC3_MPAR_OW - 1) |		\
281 	ICPU_MEMCTRL_TIMING3_LOCAL_ODT_RD_DLY(VC3_MPAR_RL - 3)
282 
283 #else
284 #define MSCC_MEMPARM_PERIOD					\
285 	ICPU_MEMCTRL_REF_PERIOD_MAX_PEND_REF(1) |		\
286 	ICPU_MEMCTRL_REF_PERIOD_REF_PERIOD(VC3_MPAR_tREFI)
287 
288 #define MSCC_MEMPARM_TIMING0                                            \
289 	ICPU_MEMCTRL_TIMING0_RAS_TO_PRECH_DLY(VC3_MPAR_tRAS_min - 1) |	\
290 	ICPU_MEMCTRL_TIMING0_WR_TO_PRECH_DLY(VC3_MPAR_CL +		\
291 					     (VC3_MPAR_BURST_LENGTH == 8 ? 2 : 0) + \
292 					     VC3_MPAR_tWR) |		\
293 	ICPU_MEMCTRL_TIMING0_RD_TO_PRECH_DLY(VC3_MPAR_BURST_LENGTH == 8 ? 3 : 1) | \
294 	ICPU_MEMCTRL_TIMING0_WR_DATA_XFR_DLY(VC3_MPAR_CL - 3) |		\
295 	ICPU_MEMCTRL_TIMING0_RD_DATA_XFR_DLY(VC3_MPAR_CL - 3)
296 
297 #define MSCC_MEMPARM_TIMING1                                            \
298 	ICPU_MEMCTRL_TIMING1_RAS_TO_RAS_SAME_BANK_DLY(VC3_MPAR_tRC - 1) | \
299 	ICPU_MEMCTRL_TIMING1_BANK8_FAW_DLY(VC3_MPAR_tFAW - 1) |		\
300 	ICPU_MEMCTRL_TIMING1_PRECH_TO_RAS_DLY(VC3_MPAR_tRP - 1) |	\
301 	ICPU_MEMCTRL_TIMING1_RAS_TO_RAS_DLY(VC3_MPAR_tRRD - 1) |	\
302 	ICPU_MEMCTRL_TIMING1_RAS_TO_CAS_DLY(VC3_MPAR_tRCD - 1) |	\
303 	ICPU_MEMCTRL_TIMING1_WR_TO_RD_DLY(VC3_MPAR_CL +			\
304 					  (VC3_MPAR_BURST_LENGTH == 8 ? 2 : 0) + \
305 					  VC3_MPAR_tWTR)
306 #define MSCC_MEMPARM_TIMING2                                            \
307 	ICPU_MEMCTRL_TIMING2_PRECH_ALL_DLY(VC3_MPAR_tRPA - 1) |		\
308 	ICPU_MEMCTRL_TIMING2_MDSET_DLY(VC3_MPAR_tMRD - 1) |		\
309 	ICPU_MEMCTRL_TIMING2_REF_DLY(VC3_MPAR_tRFC - 1) |		\
310 	ICPU_MEMCTRL_TIMING2_FOUR_HUNDRED_NS_DLY(VC3_MPAR__400_ns_dly)
311 
312 #define MSCC_MEMPARM_TIMING3						\
313 	ICPU_MEMCTRL_TIMING3_WR_TO_RD_CS_CHANGE_DLY(VC3_MPAR_CL - 1) |	\
314 	ICPU_MEMCTRL_TIMING3_ODT_WR_DLY(VC3_MPAR_CL - 1) |		\
315 	ICPU_MEMCTRL_TIMING3_LOCAL_ODT_RD_DLY(VC3_MPAR_CL - 1)
316 
317 #endif
318 
319 enum {
320 	DDR_TRAIN_OK,
321 	DDR_TRAIN_CONTINUE,
322 	DDR_TRAIN_ERROR,
323 };
324 
325 /*
326  * We actually have very few 'pause' possibilities apart from
327  * these assembly nops (at this very early stage).
328  */
329 #define PAUSE() asm volatile("nop; nop; nop; nop; nop; nop; nop; nop")
330 
331 /* NB: Assumes inlining as no stack is available! */
332 static inline void set_dly(u32 bytelane, u32 dly)
333 {
334 	register u32 r = readl(BASE_CFG + ICPU_MEMCTRL_DQS_DLY(bytelane));
335 
336 	r &= ~ICPU_MEMCTRL_DQS_DLY_DQS_DLY_M;
337 	r |= ICPU_MEMCTRL_DQS_DLY_DQS_DLY(dly);
338 	writel(r, BASE_CFG + ICPU_MEMCTRL_DQS_DLY(bytelane));
339 }
340 
341 static inline bool incr_dly(u32 bytelane)
342 {
343 	register u32 r = readl(BASE_CFG + ICPU_MEMCTRL_DQS_DLY(bytelane));
344 
345 	if (ICPU_MEMCTRL_DQS_DLY_DQS_DLY(r) < 31) {
346 		writel(r + 1, BASE_CFG + ICPU_MEMCTRL_DQS_DLY(bytelane));
347 		return true;
348 	}
349 
350 	return false;
351 }
352 
353 static inline bool adjust_dly(int adjust)
354 {
355 	register u32 r = readl(BASE_CFG + ICPU_MEMCTRL_DQS_DLY(0));
356 
357 	if (ICPU_MEMCTRL_DQS_DLY_DQS_DLY(r) < 31) {
358 		writel(r + adjust, BASE_CFG + ICPU_MEMCTRL_DQS_DLY(0));
359 		return true;
360 	}
361 
362 	return false;
363 }
364 
365 /* NB: Assumes inlining as no stack is available! */
366 static inline void center_dly(u32 bytelane, u32 start)
367 {
368 	register u32 r = readl(BASE_CFG + ICPU_MEMCTRL_DQS_DLY(bytelane)) - start;
369 
370 	writel(start + (r >> 1), BASE_CFG + ICPU_MEMCTRL_DQS_DLY(bytelane));
371 }
372 
373 static inline void memphy_soft_reset(void)
374 {
375 	setbits_le32(BASE_CFG + ICPU_MEMPHY_CFG, ICPU_MEMPHY_CFG_PHY_FIFO_RST);
376 	PAUSE();
377 	clrbits_le32(BASE_CFG + ICPU_MEMPHY_CFG, ICPU_MEMPHY_CFG_PHY_FIFO_RST);
378 	PAUSE();
379 }
380 
381 #if defined(CONFIG_SOC_OCELOT) || defined(CONFIG_SOC_JR2)
382 static u8 training_data[] = { 0xfe, 0x11, 0x33, 0x55, 0x77, 0x99, 0xbb, 0xdd };
383 
384 static inline void sleep_100ns(u32 val)
385 {
386 	/* Set the timer tick generator to 100 ns */
387 	writel(VCOREIII_TIMER_DIVIDER - 1, BASE_CFG + ICPU_TIMER_TICK_DIV);
388 
389 	/* Set the timer value */
390 	writel(val, BASE_CFG + ICPU_TIMER_VALUE(0));
391 
392 	/* Enable timer 0 for one-shot */
393 	writel(ICPU_TIMER_CTRL_ONE_SHOT_ENA | ICPU_TIMER_CTRL_TIMER_ENA,
394 	       BASE_CFG + ICPU_TIMER_CTRL(0));
395 
396 	/* Wait for timer 0 to reach 0 */
397 	while (readl(BASE_CFG + ICPU_TIMER_VALUE(0)) != 0)
398 		;
399 }
400 
401 #if defined(CONFIG_SOC_OCELOT)
402 static inline void hal_vcoreiii_ddr_reset_assert(void)
403 {
404 	/* DDR has reset pin on GPIO 19 toggle Low-High to release */
405 	setbits_le32(BASE_DEVCPU_GCB + PERF_GPIO_OE, BIT(19));
406 	writel(BIT(19), BASE_DEVCPU_GCB + PERF_GPIO_OUT_CLR);
407 	sleep_100ns(10000);
408 }
409 
410 static inline void hal_vcoreiii_ddr_reset_release(void)
411 {
412 	/* DDR has reset pin on GPIO 19 toggle Low-High to release */
413 	setbits_le32(BASE_DEVCPU_GCB + PERF_GPIO_OE, BIT(19));
414 	writel(BIT(19), BASE_DEVCPU_GCB + PERF_GPIO_OUT_SET);
415 	sleep_100ns(10000);
416 }
417 
418 /*
419  * DDR memory sanity checking failed, tally and do hard reset
420  *
421  * NB: Assumes inlining as no stack is available!
422  */
423 static inline void hal_vcoreiii_ddr_failed(void)
424 {
425 	register u32 reset;
426 
427 	writel(readl(BASE_CFG + ICPU_GPR(6)) + 1, BASE_CFG + ICPU_GPR(6));
428 
429 	clrbits_le32(BASE_DEVCPU_GCB + PERF_GPIO_OE, BIT(19));
430 
431 	/* We have to execute the reset function from cache. Indeed,
432 	 * the reboot workaround in _machine_restart() will change the
433 	 * SPI NOR into SW bitbang.
434 	 *
435 	 * This will render the CPU unable to execute directly from
436 	 * the NOR, which is why the reset instructions are prefetched
437 	 * into the I-cache.
438 	 *
439 	 * When failing the DDR initialization we are executing from
440 	 * NOR.
441 	 *
442 	 * The last instruction in _machine_restart() will reset the
443 	 * MIPS CPU (and the cache), and the CPU will start executing
444 	 * from the reset vector.
445 	 */
446 	reset = KSEG0ADDR(_machine_restart);
447 	icache_lock((void *)reset, 128);
448 	asm volatile ("jr %0"::"r" (reset));
449 
450 	panic("DDR init failed\n");
451 }
452 #else				/* JR2 */
453 static inline void hal_vcoreiii_ddr_reset_assert(void)
454 {
455 	/* Ensure the memory controller physical iface is forced reset */
456 	writel(readl(BASE_CFG + ICPU_MEMPHY_CFG) |
457 	       ICPU_MEMPHY_CFG_PHY_RST, BASE_CFG + ICPU_MEMPHY_CFG);
458 
459 	/* Ensure the memory controller is forced reset */
460 	writel(readl(BASE_CFG + ICPU_RESET) |
461 	       ICPU_RESET_MEM_RST_FORCE, BASE_CFG + ICPU_RESET);
462 }
463 
464 static inline void hal_vcoreiii_ddr_failed(void)
465 {
466 	writel(0, BASE_CFG + ICPU_RESET);
467 	writel(PERF_SOFT_RST_SOFT_CHIP_RST, BASE_CFG + PERF_SOFT_RST);
468 
469 	panic("DDR init failed\n");
470 }
471 #endif
472 
473 /*
474  * DDR memory sanity checking done, possibly enable ECC.
475  *
476  * NB: Assumes inlining as no stack is available!
477  */
478 static inline void hal_vcoreiii_ddr_verified(void)
479 {
480 #ifdef MIPS_VCOREIII_MEMORY_ECC
481 	/* Finally, enable ECC */
482 	register u32 val = readl(BASE_CFG + ICPU_MEMCTRL_CFG);
483 
484 	val |= ICPU_MEMCTRL_CFG_DDR_ECC_ERR_ENA;
485 	val &= ~ICPU_MEMCTRL_CFG_BURST_SIZE;
486 
487 	writel(val, BASE_CFG + ICPU_MEMCTRL_CFG);
488 #endif
489 
490 	/* Reset Status register - sticky bits */
491 	writel(readl(BASE_CFG + ICPU_MEMCTRL_STAT), BASE_CFG + ICPU_MEMCTRL_STAT);
492 }
493 
494 /* NB: Assumes inlining as no stack is available! */
495 static inline int look_for(u32 bytelane)
496 {
497 	register u32 i;
498 
499 	/* Reset FIFO in case any previous access failed */
500 	for (i = 0; i < sizeof(training_data); i++) {
501 		register u32 byte;
502 
503 		memphy_soft_reset();
504 		/* Reset sticky bits */
505 		writel(readl(BASE_CFG + ICPU_MEMCTRL_STAT),
506 		       BASE_CFG + ICPU_MEMCTRL_STAT);
507 		/* Read data */
508 		byte = __raw_readb((void __iomem *)MSCC_DDR_TO + bytelane +
509 				   (i * 4));
510 
511 		/*
512 		 * Prevent the compiler reordering the instruction so
513 		 * the read of RAM happens after the check of the
514 		 * errors.
515 		 */
516 		rmb();
517 		if (readl(BASE_CFG + ICPU_MEMCTRL_STAT) &
518 		    (ICPU_MEMCTRL_STAT_RDATA_MASKED |
519 		     ICPU_MEMCTRL_STAT_RDATA_DUMMY)) {
520 			/* Noise on the line */
521 			goto read_error;
522 		}
523 		/* If mismatch, increment DQS - if possible */
524 		if (byte != training_data[i]) {
525  read_error:
526 			if (!incr_dly(bytelane))
527 				return DDR_TRAIN_ERROR;
528 			return DDR_TRAIN_CONTINUE;
529 		}
530 	}
531 	return DDR_TRAIN_OK;
532 }
533 
534 /* NB: Assumes inlining as no stack is available! */
535 static inline int look_past(u32 bytelane)
536 {
537 	register u32 i;
538 
539 	/* Reset FIFO in case any previous access failed */
540 	for (i = 0; i < sizeof(training_data); i++) {
541 		register u32 byte;
542 
543 		memphy_soft_reset();
544 		/* Ack sticky bits */
545 		writel(readl(BASE_CFG + ICPU_MEMCTRL_STAT),
546 		       BASE_CFG + ICPU_MEMCTRL_STAT);
547 		byte = __raw_readb((void __iomem *)MSCC_DDR_TO + bytelane +
548 				   (i * 4));
549 		/*
550 		 * Prevent the compiler reordering the instruction so
551 		 * the read of RAM happens after the check of the
552 		 * errors.
553 		 */
554 		rmb();
555 		if (readl(BASE_CFG + ICPU_MEMCTRL_STAT) &
556 		    (ICPU_MEMCTRL_STAT_RDATA_MASKED |
557 		     ICPU_MEMCTRL_STAT_RDATA_DUMMY)) {
558 			/* Noise on the line */
559 			goto read_error;
560 		}
561 		/* Bail out when we see first mismatch */
562 		if (byte != training_data[i]) {
563  read_error:
564 			return DDR_TRAIN_OK;
565 		}
566 	}
567 	/* All data compares OK, increase DQS and retry */
568 	if (!incr_dly(bytelane))
569 		return DDR_TRAIN_ERROR;
570 
571 	return DDR_TRAIN_CONTINUE;
572 }
573 
574 static inline int hal_vcoreiii_train_bytelane(u32 bytelane)
575 {
576 	register int res;
577 	register u32 dqs_s;
578 
579 	set_dly(bytelane, 0);	/* Start training at DQS=0 */
580 	while ((res = look_for(bytelane)) == DDR_TRAIN_CONTINUE)
581 		;
582 	if (res != DDR_TRAIN_OK)
583 		return res;
584 
585 	dqs_s = readl(BASE_CFG + ICPU_MEMCTRL_DQS_DLY(bytelane));
586 	while ((res = look_past(bytelane)) == DDR_TRAIN_CONTINUE)
587 		;
588 	if (res != DDR_TRAIN_OK)
589 		return res;
590 	/* Reset FIFO - for good measure */
591 	memphy_soft_reset();
592 	/* Adjust to center [dqs_s;cur] */
593 	center_dly(bytelane, dqs_s);
594 	return DDR_TRAIN_OK;
595 }
596 
597 /* This algorithm is converted from the TCL training algorithm used
598  * during silicon simulation.
599  * NB: Assumes inlining as no stack is available!
600  */
601 static inline int hal_vcoreiii_init_dqs(void)
602 {
603 #define MAX_DQS 32
604 	register u32 i, j;
605 
606 	for (i = 0; i < MAX_DQS; i++) {
607 		set_dly(0, i);	/* Byte-lane 0 */
608 		for (j = 0; j < MAX_DQS; j++) {
609 			__maybe_unused register u32  byte;
610 
611 			set_dly(1, j);	/* Byte-lane 1 */
612 			/* Reset FIFO in case any previous access failed */
613 			memphy_soft_reset();
614 			writel(readl(BASE_CFG + ICPU_MEMCTRL_STAT),
615 			       BASE_CFG + ICPU_MEMCTRL_STAT);
616 			byte = __raw_readb((void __iomem *)MSCC_DDR_TO);
617 			byte = __raw_readb((void __iomem *)(MSCC_DDR_TO + 1));
618 			if (!(readl(BASE_CFG + ICPU_MEMCTRL_STAT) &
619 			    (ICPU_MEMCTRL_STAT_RDATA_MASKED |
620 			     ICPU_MEMCTRL_STAT_RDATA_DUMMY)))
621 				return 0;
622 		}
623 	}
624 	return -1;
625 }
626 
627 static inline int dram_check(void)
628 {
629 	register u32 i;
630 
631 	for (i = 0; i < 8; i++) {
632 		__raw_writel(~i, (void __iomem *)(MSCC_DDR_TO + (i * 4)));
633 		if (__raw_readl((void __iomem *)(MSCC_DDR_TO + (i * 4))) != ~i)
634 			return 1;
635 	}
636 	return 0;
637 }
638 #else				/* Luton */
639 
640 static inline void sleep_100ns(u32 val)
641 {
642 }
643 
644 static inline void hal_vcoreiii_ddr_reset_assert(void)
645 {
646 	setbits_le32(BASE_CFG + ICPU_MEMPHY_CFG, ICPU_MEMPHY_CFG_PHY_RST);
647 	setbits_le32(BASE_CFG + ICPU_RESET, ICPU_RESET_MEM_RST_FORCE);
648 }
649 
650 static inline void hal_vcoreiii_ddr_reset_release(void)
651 {
652 }
653 
654 static inline void hal_vcoreiii_ddr_failed(void)
655 {
656 	register u32 memphy_cfg = readl(BASE_CFG + ICPU_MEMPHY_CFG);
657 
658 	/* Do a fifo reset and start over */
659 	writel(memphy_cfg | ICPU_MEMPHY_CFG_PHY_FIFO_RST,
660 	       BASE_CFG + ICPU_MEMPHY_CFG);
661 	writel(memphy_cfg & ~ICPU_MEMPHY_CFG_PHY_FIFO_RST,
662 	       BASE_CFG + ICPU_MEMPHY_CFG);
663 	writel(memphy_cfg | ICPU_MEMPHY_CFG_PHY_FIFO_RST,
664 	       BASE_CFG + ICPU_MEMPHY_CFG);
665 }
666 
667 static inline void hal_vcoreiii_ddr_verified(void)
668 {
669 }
670 
671 static inline int look_for(u32 data)
672 {
673 	register u32 byte = __raw_readb((void __iomem *)MSCC_DDR_TO);
674 
675 	if (data != byte) {
676 		if (!incr_dly(0))
677 			return DDR_TRAIN_ERROR;
678 		return DDR_TRAIN_CONTINUE;
679 	}
680 
681 	return DDR_TRAIN_OK;
682 }
683 
684 /* This algorithm is converted from the TCL training algorithm used
685  * during silicon simulation.
686  * NB: Assumes inlining as no stack is available!
687  */
688 static inline int hal_vcoreiii_train_bytelane(u32 bytelane)
689 {
690 	register int res;
691 
692 	set_dly(bytelane, 0);	/* Start training at DQS=0 */
693 	while ((res = look_for(0xff)) == DDR_TRAIN_CONTINUE)
694 		;
695 	if (res != DDR_TRAIN_OK)
696 		return res;
697 
698 	set_dly(bytelane, 0);	/* Start training at DQS=0 */
699 	while ((res = look_for(0x00)) == DDR_TRAIN_CONTINUE)
700 
701 		;
702 
703 	if (res != DDR_TRAIN_OK)
704 		return res;
705 
706 	adjust_dly(-3);
707 
708 	return DDR_TRAIN_OK;
709 }
710 
711 static inline int hal_vcoreiii_init_dqs(void)
712 {
713 	return 0;
714 }
715 
716 static inline int dram_check(void)
717 {
718 	register u32 i;
719 
720 	for (i = 0; i < 8; i++) {
721 		__raw_writel(~i, (void __iomem *)(MSCC_DDR_TO + (i * 4)));
722 
723 		if (__raw_readl((void __iomem *)(MSCC_DDR_TO + (i * 4))) != ~i)
724 			return 1;
725 	}
726 
727 	return 0;
728 }
729 #endif
730 
731 /*
732  * NB: Called *early* to init memory controller - assumes inlining as
733  * no stack is available!
734  */
735 static inline void hal_vcoreiii_init_memctl(void)
736 {
737 	/* Ensure DDR is in reset */
738 	hal_vcoreiii_ddr_reset_assert();
739 
740 	/* Wait maybe not needed, but ... */
741 	PAUSE();
742 
743 	/* Drop sys ctl memory controller forced reset */
744 	clrbits_le32(BASE_CFG + ICPU_RESET, ICPU_RESET_MEM_RST_FORCE);
745 
746 	PAUSE();
747 
748 	/* Drop Reset, enable SSTL */
749 	writel(ICPU_MEMPHY_CFG_PHY_SSTL_ENA, BASE_CFG + ICPU_MEMPHY_CFG);
750 	PAUSE();
751 
752 	/* Start the automatic SSTL output and ODT drive-strength calibration */
753 	writel(ICPU_MEMPHY_ZCAL_ZCAL_PROG_ODT(MIPS_VCOREIII_MEMORY_SSTL_ODT) |
754 	       /* drive strength */
755 	       ICPU_MEMPHY_ZCAL_ZCAL_PROG(MIPS_VCOREIII_MEMORY_SSTL_DRIVE) |
756 	       /* Start calibration process */
757 	       ICPU_MEMPHY_ZCAL_ZCAL_ENA, BASE_CFG + ICPU_MEMPHY_ZCAL);
758 
759 	/* Wait for ZCAL to clear */
760 	while (readl(BASE_CFG + ICPU_MEMPHY_ZCAL) & ICPU_MEMPHY_ZCAL_ZCAL_ENA)
761 		;
762 #if defined(CONFIG_SOC_OCELOT) || defined(CONFIG_SOC_JR2)
763 	/* Check no ZCAL_ERR */
764 	if (readl(BASE_CFG + ICPU_MEMPHY_ZCAL_STAT)
765 	    & ICPU_MEMPHY_ZCAL_STAT_ZCAL_ERR)
766 		hal_vcoreiii_ddr_failed();
767 #endif
768 	/* Drive CL, CK, ODT */
769 	setbits_le32(BASE_CFG + ICPU_MEMPHY_CFG, ICPU_MEMPHY_CFG_PHY_ODT_OE |
770 		     ICPU_MEMPHY_CFG_PHY_CK_OE | ICPU_MEMPHY_CFG_PHY_CL_OE);
771 
772 	/* Initialize memory controller */
773 	writel(MSCC_MEMPARM_MEMCFG, BASE_CFG + ICPU_MEMCTRL_CFG);
774 	writel(MSCC_MEMPARM_PERIOD, BASE_CFG + ICPU_MEMCTRL_REF_PERIOD);
775 
776 #if defined(CONFIG_SOC_OCELOT) || defined(CONFIG_SOC_JR2)
777 	writel(MSCC_MEMPARM_TIMING0, BASE_CFG + ICPU_MEMCTRL_TIMING0);
778 #else /* Luton */
779 	clrbits_le32(BASE_CFG + ICPU_MEMCTRL_TIMING0, ((1 << 20) - 1));
780 	setbits_le32(BASE_CFG + ICPU_MEMCTRL_TIMING0, MSCC_MEMPARM_TIMING0);
781 #endif
782 
783 	writel(MSCC_MEMPARM_TIMING1, BASE_CFG + ICPU_MEMCTRL_TIMING1);
784 	writel(MSCC_MEMPARM_TIMING2, BASE_CFG + ICPU_MEMCTRL_TIMING2);
785 	writel(MSCC_MEMPARM_TIMING3, BASE_CFG + ICPU_MEMCTRL_TIMING3);
786 	writel(MSCC_MEMPARM_MR0, BASE_CFG + ICPU_MEMCTRL_MR0_VAL);
787 	writel(MSCC_MEMPARM_MR1, BASE_CFG + ICPU_MEMCTRL_MR1_VAL);
788 	writel(MSCC_MEMPARM_MR2, BASE_CFG + ICPU_MEMCTRL_MR2_VAL);
789 	writel(MSCC_MEMPARM_MR3, BASE_CFG + ICPU_MEMCTRL_MR3_VAL);
790 
791 #if defined(CONFIG_SOC_OCELOT)
792 	/* Termination setup - enable ODT */
793 	writel(ICPU_MEMCTRL_TERMRES_CTRL_LOCAL_ODT_RD_ENA |
794 	       /* Assert ODT0 for any write */
795 	       ICPU_MEMCTRL_TERMRES_CTRL_ODT_WR_ENA(3),
796 	       BASE_CFG + ICPU_MEMCTRL_TERMRES_CTRL);
797 
798 	/* Release Reset from DDR */
799 	hal_vcoreiii_ddr_reset_release();
800 
801 	writel(readl(BASE_CFG + ICPU_GPR(7)) + 1, BASE_CFG + ICPU_GPR(7));
802 #elif defined(CONFIG_SOC_JR2)
803 	writel(ICPU_MEMCTRL_TERMRES_CTRL_ODT_WR_ENA(3),
804 	       BASE_CFG + ICPU_MEMCTRL_TERMRES_CTRL);
805 #else				/* Luton */
806 	/* Termination setup - disable ODT */
807 	writel(0, BASE_CFG + ICPU_MEMCTRL_TERMRES_CTRL);
808 
809 #endif
810 }
811 
812 static inline void hal_vcoreiii_wait_memctl(void)
813 {
814 	/* Now, rip it! */
815 	writel(ICPU_MEMCTRL_CTRL_INITIALIZE, BASE_CFG + ICPU_MEMCTRL_CTRL);
816 
817 	while (!(readl(BASE_CFG + ICPU_MEMCTRL_STAT)
818 		 & ICPU_MEMCTRL_STAT_INIT_DONE))
819 		;
820 
821 	/* Settle...? */
822 	sleep_100ns(10000);
823 #if defined(CONFIG_SOC_OCELOT) || defined(CONFIG_SOC_JR2)
824 	/* Establish data contents in DDR RAM for training */
825 
826 	__raw_writel(0xcacafefe, ((void __iomem *)MSCC_DDR_TO));
827 	__raw_writel(0x22221111, ((void __iomem *)MSCC_DDR_TO + 0x4));
828 	__raw_writel(0x44443333, ((void __iomem *)MSCC_DDR_TO + 0x8));
829 	__raw_writel(0x66665555, ((void __iomem *)MSCC_DDR_TO + 0xC));
830 	__raw_writel(0x88887777, ((void __iomem *)MSCC_DDR_TO + 0x10));
831 	__raw_writel(0xaaaa9999, ((void __iomem *)MSCC_DDR_TO + 0x14));
832 	__raw_writel(0xccccbbbb, ((void __iomem *)MSCC_DDR_TO + 0x18));
833 	__raw_writel(0xeeeedddd, ((void __iomem *)MSCC_DDR_TO + 0x1C));
834 #else
835 	__raw_writel(0xff, ((void __iomem *)MSCC_DDR_TO));
836 #endif
837 }
838 #endif				/* __ASM_MACH_DDR_H */
839