1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (C) Marvell International Ltd. and its affiliates
4  */
5 
6 #include <common.h>
7 #include <i2c.h>
8 #include <spl.h>
9 #include <asm/io.h>
10 #include <asm/arch/cpu.h>
11 #include <asm/arch/soc.h>
12 
13 #include "ddr3_hw_training.h"
14 
15 /*
16  * Debug
17  */
18 #define DEBUG_WL_C(s, d, l) \
19 	DEBUG_WL_S(s); DEBUG_WL_D(d, l); DEBUG_WL_S("\n")
20 #define DEBUG_WL_FULL_C(s, d, l) \
21 	DEBUG_WL_FULL_S(s); DEBUG_WL_FULL_D(d, l); DEBUG_WL_FULL_S("\n")
22 
23 #ifdef MV_DEBUG_WL
24 #define DEBUG_WL_S(s)			puts(s)
25 #define DEBUG_WL_D(d, l)		printf("%x", d)
26 #define DEBUG_RL_S(s) \
27 	debug_cond(ddr3_get_log_level() >= MV_LOG_LEVEL_2, "%s", s)
28 #define DEBUG_RL_D(d, l) \
29 	debug_cond(ddr3_get_log_level() >= MV_LOG_LEVEL_2, "%x", d)
30 #else
31 #define DEBUG_WL_S(s)
32 #define DEBUG_WL_D(d, l)
33 #endif
34 
35 #ifdef MV_DEBUG_WL_FULL
36 #define DEBUG_WL_FULL_S(s)		puts(s)
37 #define DEBUG_WL_FULL_D(d, l)		printf("%x", d)
38 #else
39 #define DEBUG_WL_FULL_S(s)
40 #define DEBUG_WL_FULL_D(d, l)
41 #endif
42 
43 #define WL_SUP_EXPECTED_DATA		0x21
44 #define WL_SUP_READ_DRAM_ENTRY		0x8
45 
46 static int ddr3_write_leveling_single_cs(u32 cs, u32 freq, int ratio_2to1,
47 					 u32 *result,
48 					 MV_DRAM_INFO *dram_info);
49 static void ddr3_write_ctrl_pup_reg(int bc_acc, u32 pup, u32 reg_addr,
50 				    u32 data);
51 
52 extern u16 odt_static[ODT_OPT][MAX_CS];
53 extern u16 odt_dynamic[ODT_OPT][MAX_CS];
54 extern u32 wl_sup_pattern[LEN_WL_SUP_PATTERN];
55 
56 /*
57  * Name:     ddr3_write_leveling_hw
58  * Desc:     Execute Write leveling phase by HW
59  * Args:     freq      - current sequence frequency
60  *           dram_info   - main struct
61  * Notes:
62  * Returns:  MV_OK if success, MV_FAIL if fail.
63  */
ddr3_write_leveling_hw(u32 freq,MV_DRAM_INFO * dram_info)64 int ddr3_write_leveling_hw(u32 freq, MV_DRAM_INFO *dram_info)
65 {
66 	u32 reg, phase, delay, cs, pup;
67 #ifdef MV88F67XX
68 	int dpde_flag = 0;
69 #endif
70 	/* Debug message - Start Read leveling procedure */
71 	DEBUG_WL_S("DDR3 - Write Leveling - Starting HW WL procedure\n");
72 
73 #ifdef MV88F67XX
74 	/* Dynamic pad issue (BTS669) during WL */
75 	reg = reg_read(REG_DUNIT_CTRL_LOW_ADDR);
76 	if (reg & (1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS)) {
77 		dpde_flag = 1;
78 		reg_write(REG_DUNIT_CTRL_LOW_ADDR,
79 			  reg & ~(1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS));
80 	}
81 #endif
82 
83 	reg = 1 << REG_DRAM_TRAINING_WL_OFFS;
84 	/* Config the retest number */
85 	reg |= (COUNT_HW_WL << REG_DRAM_TRAINING_RETEST_OFFS);
86 	reg |= (dram_info->cs_ena << (REG_DRAM_TRAINING_CS_OFFS));
87 	reg_write(REG_DRAM_TRAINING_ADDR, reg);	/* 0x15B0 - Training Register */
88 
89 	reg =  reg_read(REG_DRAM_TRAINING_SHADOW_ADDR) |
90 		(1 << REG_DRAM_TRAINING_AUTO_OFFS);
91 	reg_write(REG_DRAM_TRAINING_SHADOW_ADDR, reg);
92 
93 	/* Wait */
94 	do {
95 		reg = reg_read(REG_DRAM_TRAINING_SHADOW_ADDR) &
96 			(1 << REG_DRAM_TRAINING_AUTO_OFFS);
97 	} while (reg);		/* Wait for '0' */
98 
99 	reg = reg_read(REG_DRAM_TRAINING_ADDR);
100 	/* Check if Successful */
101 	if (reg & (1 << REG_DRAM_TRAINING_ERROR_OFFS)) {
102 		/*
103 		 * Read results to arrays - Results are required for WL
104 		 * High freq Supplement and DQS Centralization
105 		 */
106 		for (cs = 0; cs < MAX_CS; cs++) {
107 			if (dram_info->cs_ena & (1 << cs)) {
108 				for (pup = 0;
109 				     pup < dram_info->num_of_total_pups;
110 				     pup++) {
111 					if (pup == dram_info->num_of_std_pups
112 					    && dram_info->ecc_ena)
113 						pup = ECC_PUP;
114 					reg =
115 					    ddr3_read_pup_reg(PUP_WL_MODE, cs,
116 							      pup);
117 					phase =
118 					    (reg >> REG_PHY_PHASE_OFFS) &
119 					    PUP_PHASE_MASK;
120 					delay = reg & PUP_DELAY_MASK;
121 					dram_info->wl_val[cs][pup][P] = phase;
122 					dram_info->wl_val[cs][pup][D] = delay;
123 					dram_info->wl_val[cs][pup][S] =
124 					    WL_HI_FREQ_STATE - 1;
125 					reg =
126 					    ddr3_read_pup_reg(PUP_WL_MODE + 0x1,
127 							      cs, pup);
128 					dram_info->wl_val[cs][pup][DQS] =
129 					    (reg & 0x3F);
130 				}
131 
132 #ifdef MV_DEBUG_WL
133 				/* Debug message - Print res for cs[i]: cs,PUP,Phase,Delay */
134 				DEBUG_WL_S("DDR3 - Write Leveling - Write Leveling Cs - ");
135 				DEBUG_WL_D((u32) cs, 1);
136 				DEBUG_WL_S(" Results:\n");
137 				for (pup = 0;
138 				     pup < dram_info->num_of_total_pups;
139 				     pup++) {
140 					if (pup == dram_info->num_of_std_pups
141 					    && dram_info->ecc_ena)
142 						pup = ECC_PUP;
143 					DEBUG_WL_S("DDR3 - Write Leveling - PUP: ");
144 					DEBUG_WL_D((u32) pup, 1);
145 					DEBUG_WL_S(", Phase: ");
146 					DEBUG_WL_D((u32)
147 						   dram_info->wl_val[cs][pup]
148 						   [P], 1);
149 					DEBUG_WL_S(", Delay: ");
150 					DEBUG_WL_D((u32)
151 						   dram_info->wl_val[cs][pup]
152 						   [D], 2);
153 					DEBUG_WL_S("\n");
154 				}
155 #endif
156 			}
157 		}
158 
159 		/* Dynamic pad issue (BTS669) during WL */
160 #ifdef MV88F67XX
161 		if (dpde_flag) {
162 			reg = reg_read(REG_DUNIT_CTRL_LOW_ADDR) |
163 				(1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS);
164 			reg_write(REG_DUNIT_CTRL_LOW_ADDR, reg);
165 		}
166 #endif
167 
168 		DEBUG_WL_S("DDR3 - Write Leveling - HW WL Ended Successfully\n");
169 
170 		return MV_OK;
171 	} else {
172 		DEBUG_WL_S("DDR3 - Write Leveling - HW WL Error\n");
173 		return MV_FAIL;
174 	}
175 }
176 
177 /*
178  * Name:     ddr3_wl_supplement
179  * Desc:     Write Leveling Supplement
180  * Args:     dram_info   - main struct
181  * Notes:
182  * Returns:  MV_OK if success, MV_FAIL if fail.
183  */
ddr3_wl_supplement(MV_DRAM_INFO * dram_info)184 int ddr3_wl_supplement(MV_DRAM_INFO *dram_info)
185 {
186 	u32 cs, cnt, pup_num, sum, phase, delay, max_pup_num, pup, sdram_offset;
187 	u32 tmp_count, ecc, reg;
188 	u32 ddr_width, tmp_pup, idx;
189 	u32 sdram_pup_val, uj;
190 	u32 one_clk_err = 0, align_err = 0, no_err = 0, err = 0, err_n = 0;
191 	u32 sdram_data[LEN_WL_SUP_PATTERN] __aligned(32) = { 0 };
192 
193 	ddr_width = dram_info->ddr_width;
194 	no_err = 0;
195 
196 	DEBUG_WL_S("DDR3 - Write Leveling Hi-Freq Supplement - Starting\n");
197 
198 	switch (ddr_width) {
199 		/* Data error from pos-adge to pos-adge */
200 	case 16:
201 		one_clk_err = 4;
202 		align_err = 4;
203 		break;
204 	case 32:
205 		one_clk_err = 8;
206 		align_err = 8;
207 		break;
208 	case 64:
209 		one_clk_err = 0x10;
210 		align_err = 0x10;
211 		break;
212 	default:
213 		DEBUG_WL_S("Error - bus width!!!\n");
214 		return MV_FAIL;
215 	}
216 
217 	/* Enable SW override */
218 	reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |
219 		(1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
220 
221 	/* [0] = 1 - Enable SW override  */
222 	/* 0x15B8 - Training SW 2 Register */
223 	reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
224 	DEBUG_WL_S("DDR3 - Write Leveling Hi-Freq Supplement - SW Override Enabled\n");
225 	reg = (1 << REG_DRAM_TRAINING_AUTO_OFFS);
226 	reg_write(REG_DRAM_TRAINING_ADDR, reg);	/* 0x15B0 - Training Register */
227 	tmp_count = 0;
228 	for (cs = 0; cs < MAX_CS; cs++) {
229 		if (dram_info->cs_ena & (1 << cs)) {
230 			sum = 0;
231 			/*
232 			 * 2 iterations loop: 1)actual WL results 2) fix WL
233 			 * if needed
234 			 */
235 			for (cnt = 0; cnt < COUNT_WL_HI_FREQ; cnt++) {
236 				DEBUG_WL_C("COUNT = ", cnt, 1);
237 				for (ecc = 0; ecc < (dram_info->ecc_ena + 1);
238 				     ecc++) {
239 					if (ecc) {
240 						DEBUG_WL_S("ECC PUP:\n");
241 					} else {
242 						DEBUG_WL_S("DATA PUP:\n");
243 					}
244 
245 					max_pup_num =
246 					    dram_info->num_of_std_pups * (1 -
247 									  ecc) +
248 					    ecc;
249 					/* ECC Support - Switch ECC Mux on ecc=1 */
250 					reg =
251 					    (reg_read(REG_DRAM_TRAINING_2_ADDR)
252 					     & ~(1 <<
253 						 REG_DRAM_TRAINING_2_ECC_MUX_OFFS));
254 					reg |=
255 					    (dram_info->ecc_ena *
256 					     ecc <<
257 					     REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
258 					reg_write(REG_DRAM_TRAINING_2_ADDR,
259 						  reg);
260 					ddr3_reset_phy_read_fifo();
261 
262 					/* Write to memory */
263 					sdram_offset =
264 					    tmp_count * (SDRAM_CS_SIZE + 1) +
265 					    0x200;
266 					if (MV_OK != ddr3_dram_sram_burst((u32)
267 									  wl_sup_pattern,
268 									  sdram_offset,
269 									  LEN_WL_SUP_PATTERN))
270 						return MV_FAIL;
271 
272 					/* Read from memory */
273 					if (MV_OK !=
274 					    ddr3_dram_sram_burst(sdram_offset,
275 								 (u32)
276 								 sdram_data,
277 								 LEN_WL_SUP_PATTERN))
278 						return MV_FAIL;
279 
280 					/* Print the buffer */
281 					for (uj = 0; uj < LEN_WL_SUP_PATTERN;
282 					     uj++) {
283 						if ((uj % 4 == 0) && (uj != 0)) {
284 							DEBUG_WL_S("\n");
285 						}
286 						DEBUG_WL_D(sdram_data[uj],
287 							   8);
288 						DEBUG_WL_S(" ");
289 					}
290 
291 					/* Check pup which DQS/DATA is error */
292 					for (pup = 0; pup < max_pup_num; pup++) {
293 						/* ECC support - bit 8 */
294 						pup_num = (ecc) ? ECC_PUP : pup;
295 						if (pup < 4) {	/* lower 32 bit */
296 							tmp_pup = pup;
297 							idx =
298 							    WL_SUP_READ_DRAM_ENTRY;
299 						} else {	/* higher 32 bit */
300 							tmp_pup = pup - 4;
301 							idx =
302 							    WL_SUP_READ_DRAM_ENTRY
303 							    + 1;
304 						}
305 						DEBUG_WL_S("\nCS: ");
306 						DEBUG_WL_D((u32) cs, 1);
307 						DEBUG_WL_S(" PUP: ");
308 						DEBUG_WL_D((u32) pup_num, 1);
309 						DEBUG_WL_S("\n");
310 						sdram_pup_val =
311 						    ((sdram_data[idx] >>
312 						      ((tmp_pup) * 8)) & 0xFF);
313 						DEBUG_WL_C("Actual Data = ",
314 							   sdram_pup_val, 2);
315 						DEBUG_WL_C("Expected Data = ",
316 							   (WL_SUP_EXPECTED_DATA
317 							    + pup), 2);
318 						/*
319 						 * ALINGHMENT: calculate
320 						 * expected data vs actual data
321 						 */
322 						err =
323 						    (WL_SUP_EXPECTED_DATA +
324 						     pup) - sdram_pup_val;
325 						/*
326 						 * CLOCK LONG: calculate
327 						 * expected data vs actual data
328 						 */
329 						err_n =
330 						    sdram_pup_val -
331 						    (WL_SUP_EXPECTED_DATA +
332 						     pup);
333 						DEBUG_WL_C("err = ", err, 2);
334 						DEBUG_WL_C("err_n = ", err_n,
335 							   2);
336 						if (err == no_err) {
337 							/* PUP is correct - increment State */
338 							dram_info->wl_val[cs]
339 							    [pup_num]
340 							    [S] = 1;
341 						} else if (err_n == one_clk_err) {
342 							/* clock is longer than DQS */
343 							phase =
344 							    ((dram_info->wl_val
345 							      [cs]
346 							      [pup_num][P] +
347 							      WL_HI_FREQ_SHIFT)
348 							     % MAX_PHASE_2TO1);
349 							dram_info->wl_val[cs]
350 							    [pup_num]
351 							    [P] = phase;
352 							delay =
353 							    dram_info->wl_val
354 							    [cs][pup_num]
355 							    [D];
356 							DEBUG_WL_S("#### Clock is longer than DQS more than one clk cycle ####\n");
357 							ddr3_write_pup_reg
358 							    (PUP_WL_MODE, cs,
359 							     pup * (1 - ecc) +
360 							     ECC_PUP * ecc,
361 							     phase, delay);
362 						} else if (err == align_err) {
363 							/* clock is align to DQS */
364 							phase =
365 							    dram_info->wl_val
366 							    [cs][pup_num]
367 							    [P];
368 							delay =
369 							    dram_info->wl_val
370 							    [cs][pup_num]
371 							    [D];
372 							DEBUG_WL_S("#### Alignment PUPS problem ####\n");
373 							if ((phase == 0)
374 							    || ((phase == 1)
375 								&& (delay <=
376 								    0x10))) {
377 								DEBUG_WL_S("#### Warning - Possible Layout Violation (DQS is longer than CLK)####\n");
378 							}
379 
380 							phase = 0x0;
381 							delay = 0x0;
382 							dram_info->wl_val[cs]
383 							    [pup_num]
384 							    [P] = phase;
385 							dram_info->wl_val[cs]
386 							    [pup_num]
387 							    [D] = delay;
388 							ddr3_write_pup_reg
389 							    (PUP_WL_MODE, cs,
390 							     pup * (1 - ecc) +
391 							     ECC_PUP * ecc,
392 							     phase, delay);
393 						}
394 						/* Stop condition for ECC phase */
395 						pup = (ecc) ? max_pup_num : pup;
396 					}
397 
398 					/* ECC Support - Disable ECC MUX */
399 					reg =
400 					    (reg_read(REG_DRAM_TRAINING_2_ADDR)
401 					     & ~(1 <<
402 						 REG_DRAM_TRAINING_2_ECC_MUX_OFFS));
403 					reg_write(REG_DRAM_TRAINING_2_ADDR,
404 						  reg);
405 				}
406 			}
407 
408 			for (pup = 0; pup < dram_info->num_of_std_pups; pup++)
409 				sum += dram_info->wl_val[cs][pup][S];
410 
411 			if (dram_info->ecc_ena)
412 				sum += dram_info->wl_val[cs][ECC_PUP][S];
413 
414 			/* Checks if any pup is not locked after the change */
415 			if (sum < (WL_HI_FREQ_STATE * (dram_info->num_of_total_pups))) {
416 				DEBUG_WL_C("DDR3 - Write Leveling Hi-Freq Supplement - didn't work for Cs - ",
417 					   (u32) cs, 1);
418 				return MV_FAIL;
419 			}
420 			tmp_count++;
421 		}
422 	}
423 
424 	dram_info->wl_max_phase = 0;
425 	dram_info->wl_min_phase = 10;
426 
427 	/*
428 	 * Read results to arrays - Results are required for DQS Centralization
429 	 */
430 	for (cs = 0; cs < MAX_CS; cs++) {
431 		if (dram_info->cs_ena & (1 << cs)) {
432 			for (pup = 0; pup < dram_info->num_of_total_pups; pup++) {
433 				if (pup == dram_info->num_of_std_pups
434 				    && dram_info->ecc_ena)
435 					pup = ECC_PUP;
436 				reg = ddr3_read_pup_reg(PUP_WL_MODE, cs, pup);
437 				phase =
438 				    (reg >> REG_PHY_PHASE_OFFS) &
439 				    PUP_PHASE_MASK;
440 				if (phase > dram_info->wl_max_phase)
441 					dram_info->wl_max_phase = phase;
442 				if (phase < dram_info->wl_min_phase)
443 					dram_info->wl_min_phase = phase;
444 			}
445 		}
446 	}
447 
448 	/* Disable SW override - Must be in a different stage */
449 	/* [0]=0 - Enable SW override  */
450 	reg = reg_read(REG_DRAM_TRAINING_2_ADDR);
451 	reg &= ~(1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
452 	/* 0x15B8 - Training SW 2 Register */
453 	reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
454 
455 	reg = reg_read(REG_DRAM_TRAINING_1_ADDR) |
456 		(1 << REG_DRAM_TRAINING_1_TRNBPOINT_OFFS);
457 	reg_write(REG_DRAM_TRAINING_1_ADDR, reg);
458 
459 	DEBUG_WL_S("DDR3 - Write Leveling Hi-Freq Supplement - Ended Successfully\n");
460 
461 	return MV_OK;
462 }
463 
464 /*
465  * Name:     ddr3_write_leveling_hw_reg_dimm
466  * Desc:     Execute Write leveling phase by HW
467  * Args:     freq      - current sequence frequency
468  *           dram_info   - main struct
469  * Notes:
470  * Returns:  MV_OK if success, MV_FAIL if fail.
471  */
ddr3_write_leveling_hw_reg_dimm(u32 freq,MV_DRAM_INFO * dram_info)472 int ddr3_write_leveling_hw_reg_dimm(u32 freq, MV_DRAM_INFO *dram_info)
473 {
474 	u32 reg, phase, delay, cs, pup, pup_num;
475 	__maybe_unused int dpde_flag = 0;
476 
477 	/* Debug message - Start Read leveling procedure */
478 	DEBUG_WL_S("DDR3 - Write Leveling - Starting HW WL procedure\n");
479 
480 	if (dram_info->num_cs > 2) {
481 		DEBUG_WL_S("DDR3 - Write Leveling - HW WL Ended Successfully\n");
482 		return MV_NO_CHANGE;
483 	}
484 
485 	/* If target freq = 400 move clock start point */
486 	/* Write to control PUP to Control Deskew Regs */
487 	if (freq <= DDR_400) {
488 		for (pup = 0; pup <= dram_info->num_of_total_pups; pup++) {
489 			/* PUP_DELAY_MASK 0x1F */
490 			/* reg = 0x0C10001F + (uj << 16); */
491 			ddr3_write_ctrl_pup_reg(1, pup, CNTRL_PUP_DESKEW + pup,
492 						0x1F);
493 		}
494 	}
495 
496 #ifdef MV88F67XX
497 	/* Dynamic pad issue (BTS669) during WL */
498 	reg = reg_read(REG_DUNIT_CTRL_LOW_ADDR);
499 	if (reg & (1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS)) {
500 		dpde_flag = 1;
501 		reg_write(REG_DUNIT_CTRL_LOW_ADDR,
502 			  reg & ~(1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS));
503 	}
504 #endif
505 
506 	reg = (1 << REG_DRAM_TRAINING_WL_OFFS);
507 	/* Config the retest number */
508 	reg |= (COUNT_HW_WL << REG_DRAM_TRAINING_RETEST_OFFS);
509 	reg |= (dram_info->cs_ena << (REG_DRAM_TRAINING_CS_OFFS));
510 	reg_write(REG_DRAM_TRAINING_ADDR, reg);	/* 0x15B0 - Training Register */
511 
512 	reg = reg_read(REG_DRAM_TRAINING_SHADOW_ADDR) |
513 		(1 << REG_DRAM_TRAINING_AUTO_OFFS);
514 	reg_write(REG_DRAM_TRAINING_SHADOW_ADDR, reg);
515 
516 	/* Wait */
517 	do {
518 		reg = reg_read(REG_DRAM_TRAINING_SHADOW_ADDR) &
519 			(1 << REG_DRAM_TRAINING_AUTO_OFFS);
520 	} while (reg);		/* Wait for '0' */
521 
522 	reg = reg_read(REG_DRAM_TRAINING_ADDR);
523 	/* Check if Successful */
524 	if (reg & (1 << REG_DRAM_TRAINING_ERROR_OFFS)) {
525 		/*
526 		 * Read results to arrays - Results are required for WL High
527 		 * freq Supplement and DQS Centralization
528 		 */
529 		for (cs = 0; cs < MAX_CS; cs++) {
530 			if (dram_info->cs_ena & (1 << cs)) {
531 				for (pup = 0;
532 				     pup < dram_info->num_of_total_pups;
533 				     pup++) {
534 					if (pup == dram_info->num_of_std_pups
535 					    && dram_info->ecc_ena)
536 						pup = ECC_BIT;
537 					reg =
538 					    ddr3_read_pup_reg(PUP_WL_MODE, cs,
539 							      pup);
540 					phase =
541 					    (reg >> REG_PHY_PHASE_OFFS) &
542 					    PUP_PHASE_MASK;
543 					delay = reg & PUP_DELAY_MASK;
544 					dram_info->wl_val[cs][pup][P] = phase;
545 					dram_info->wl_val[cs][pup][D] = delay;
546 					if ((phase == 1) && (delay >= 0x1D)) {
547 						/*
548 						 * Need to do it here for
549 						 * uncorrect WL values
550 						 */
551 						ddr3_write_pup_reg(PUP_WL_MODE,
552 								   cs, pup, 0,
553 								   0);
554 						dram_info->wl_val[cs][pup][P] =
555 						    0;
556 						dram_info->wl_val[cs][pup][D] =
557 						    0;
558 					}
559 					dram_info->wl_val[cs][pup][S] =
560 					    WL_HI_FREQ_STATE - 1;
561 					reg =
562 					    ddr3_read_pup_reg(PUP_WL_MODE + 0x1,
563 							      cs, pup);
564 					dram_info->wl_val[cs][pup][DQS] =
565 					    (reg & 0x3F);
566 				}
567 #ifdef MV_DEBUG_WL
568 				/*
569 				 * Debug message - Print res for cs[i]:
570 				 * cs,PUP,Phase,Delay
571 				 */
572 				DEBUG_WL_S("DDR3 - Write Leveling - Write Leveling Cs - ");
573 				DEBUG_WL_D((u32) cs, 1);
574 				DEBUG_WL_S(" Results:\n");
575 				for (pup = 0;
576 				     pup < dram_info->num_of_total_pups;
577 				     pup++) {
578 					DEBUG_WL_S
579 					    ("DDR3 - Write Leveling - PUP: ");
580 					DEBUG_WL_D((u32) pup, 1);
581 					DEBUG_WL_S(", Phase: ");
582 					DEBUG_WL_D((u32)
583 						   dram_info->wl_val[cs][pup]
584 						   [P], 1);
585 					DEBUG_WL_S(", Delay: ");
586 					DEBUG_WL_D((u32)
587 						   dram_info->wl_val[cs][pup]
588 						   [D], 2);
589 					DEBUG_WL_S("\n");
590 				}
591 #endif
592 			}
593 		}
594 
595 #ifdef MV88F67XX
596 		/* Dynamic pad issue (BTS669) during WL */
597 		if (dpde_flag) {
598 			reg = reg_read(REG_DUNIT_CTRL_LOW_ADDR) |
599 				(1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS);
600 			reg_write(REG_DUNIT_CTRL_LOW_ADDR, reg);
601 		}
602 #endif
603 		DEBUG_WL_S("DDR3 - Write Leveling - HW WL Ended Successfully\n");
604 
605 		/* If target freq = 400 move clock back */
606 		/* Write to control PUP to Control Deskew Regs */
607 		if (freq <= DDR_400) {
608 			for (pup = 0; pup <= dram_info->num_of_total_pups;
609 			     pup++) {
610 				ddr3_write_ctrl_pup_reg(1, pup,
611 							CNTRL_PUP_DESKEW + pup, 0);
612 			}
613 		}
614 
615 		return MV_OK;
616 	} else {
617 		/* Configure Each PUP with locked leveling settings */
618 		for (cs = 0; cs < MAX_CS; cs++) {
619 			if (dram_info->cs_ena & (1 << cs)) {
620 				for (pup = 0;
621 				     pup < dram_info->num_of_total_pups;
622 				     pup++) {
623 					/* ECC support - bit 8 */
624 					pup_num = (pup == dram_info->num_of_std_pups) ?
625 						ECC_BIT : pup;
626 					ddr3_write_pup_reg(PUP_WL_MODE, cs,
627 							   pup_num, 0, 0);
628 				}
629 			}
630 		}
631 
632 		reg_write(REG_DRAM_TRAINING_ADDR, 0);
633 
634 		/* If target freq = 400 move clock back */
635 		/* Write to control PUP to Control Deskew Regs */
636 		if (freq <= DDR_400) {
637 			for (pup = 0; pup <= dram_info->num_of_total_pups;
638 			     pup++) {
639 				ddr3_write_ctrl_pup_reg(1, pup,
640 							CNTRL_PUP_DESKEW + pup, 0);
641 			}
642 		}
643 
644 		DEBUG_WL_S("DDR3 - Write Leveling - HW WL Ended Successfully\n");
645 		return MV_NO_CHANGE;
646 	}
647 }
648 
649 /*
650  * Name:     ddr3_write_leveling_sw
651  * Desc:     Execute Write leveling phase by SW
652  * Args:     freq      - current sequence frequency
653  *           dram_info   - main struct
654  * Notes:
655  * Returns:  MV_OK if success, MV_FAIL if fail.
656  */
ddr3_write_leveling_sw(u32 freq,int ratio_2to1,MV_DRAM_INFO * dram_info)657 int ddr3_write_leveling_sw(u32 freq, int ratio_2to1, MV_DRAM_INFO *dram_info)
658 {
659 	u32 reg, cs, cnt, pup, max_pup_num;
660 	u32 res[MAX_CS];
661 	max_pup_num = dram_info->num_of_total_pups;
662 	__maybe_unused int dpde_flag = 0;
663 
664 	/* Debug message - Start Write leveling procedure */
665 	DEBUG_WL_S("DDR3 - Write Leveling - Starting SW WL procedure\n");
666 
667 #ifdef MV88F67XX
668 	/* Dynamic pad issue (BTS669) during WL */
669 	reg = reg_read(REG_DUNIT_CTRL_LOW_ADDR);
670 	if (reg & (1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS)) {
671 		dpde_flag = 1;
672 		reg_write(REG_DUNIT_CTRL_LOW_ADDR,
673 			  reg & ~(1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS));
674 	}
675 #endif
676 
677 	/* Set Output buffer-off to all CS and correct ODT values */
678 	for (cs = 0; cs < MAX_CS; cs++) {
679 		if (dram_info->cs_ena & (1 << cs)) {
680 			reg = reg_read(REG_DDR3_MR1_ADDR) &
681 				REG_DDR3_MR1_ODT_MASK;
682 			reg |= odt_static[dram_info->cs_ena][cs];
683 			reg |= (1 << REG_DDR3_MR1_OUTBUF_DIS_OFFS);
684 
685 			/* 0x15D0 - DDR3 MR0 Register */
686 			reg_write(REG_DDR3_MR1_ADDR, reg);
687 			/* Issue MRS Command to current cs */
688 			reg = REG_SDRAM_OPERATION_CMD_MR1 &
689 				~(1 << (REG_SDRAM_OPERATION_CS_OFFS + cs));
690 			/*
691 			 * [3-0] = 0x4 - MR1 Command, [11-8] -
692 			 * enable current cs
693 			 */
694 			/* 0x1418 - SDRAM Operation Register */
695 			reg_write(REG_SDRAM_OPERATION_ADDR, reg);
696 
697 			udelay(MRS_DELAY);
698 		}
699 	}
700 
701 	DEBUG_WL_FULL_S("DDR3 - Write Leveling - Qoff and RTT Values are set for all Cs\n");
702 
703 	/* Enable SW override */
704 	reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |
705 		(1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
706 	/* [0] = 1 - Enable SW override  */
707 	/* 0x15B8 - Training SW 2 Register */
708 	reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
709 	DEBUG_WL_FULL_S("DDR3 - Write Leveling - SW Override Enabled\n");
710 
711 	/* Enable PHY write leveling mode */
712 	reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &
713 		~(1 << REG_DRAM_TRAINING_2_WL_MODE_OFFS);
714 	/* [2] = 0 - TrnWLMode - Enable */
715 	/* 0x15B8 - Training SW 2 Register */
716 	reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
717 	/* Reset WL results arry */
718 	memset(dram_info->wl_val, 0, sizeof(u32) * MAX_CS * MAX_PUP_NUM * 7);
719 
720 	/* Loop for each cs */
721 	for (cs = 0; cs < MAX_CS; cs++) {
722 		if (dram_info->cs_ena & (1 << cs)) {
723 			DEBUG_WL_FULL_C("DDR3 - Write Leveling - Starting working with Cs - ",
724 					(u32) cs, 1);
725 			/* Refresh X9 current cs */
726 			DEBUG_WL_FULL_S("DDR3 - Write Leveling - Refresh X9\n");
727 			for (cnt = 0; cnt < COUNT_WL_RFRS; cnt++) {
728 				reg =
729 				    REG_SDRAM_OPERATION_CMD_RFRS & ~(1 <<
730 								     (REG_SDRAM_OPERATION_CS_OFFS
731 								      + cs));
732 				/* [3-0] = 0x2 - refresh, [11-8] - enable current cs */
733 				reg_write(REG_SDRAM_OPERATION_ADDR, reg);	/* 0x1418 - SDRAM Operation Register */
734 
735 				do {
736 					reg =
737 					    ((reg_read
738 					      (REG_SDRAM_OPERATION_ADDR)) &
739 					     REG_SDRAM_OPERATION_CMD_RFRS_DONE);
740 				} while (reg);	/* Wait for '0' */
741 			}
742 
743 			/* Configure MR1 in Cs[CsNum] - write leveling on, output buffer on */
744 			DEBUG_WL_FULL_S("DDR3 - Write Leveling - Configure MR1 for current Cs: WL-on,OB-on\n");
745 			reg = reg_read(REG_DDR3_MR1_ADDR) &
746 				REG_DDR3_MR1_OUTBUF_WL_MASK;
747 			/* Set ODT Values */
748 			reg &= REG_DDR3_MR1_ODT_MASK;
749 			reg |= odt_static[dram_info->cs_ena][cs];
750 			/* Enable WL MODE */
751 			reg |= (1 << REG_DDR3_MR1_WL_ENA_OFFS);
752 			/* [7]=1, [12]=0 - Output Buffer and write leveling enabled */
753 			reg_write(REG_DDR3_MR1_ADDR, reg);	/* 0x15D4 - DDR3 MR1 Register */
754 			/* Issue MRS Command to current cs */
755 			reg = REG_SDRAM_OPERATION_CMD_MR1 &
756 				~(1 << (REG_SDRAM_OPERATION_CS_OFFS + cs));
757 			/*
758 			 * [3-0] = 0x4 - MR1 Command, [11-8] -
759 			 * enable current cs
760 			 */
761 			/* 0x1418 - SDRAM Operation Register */
762 			reg_write(REG_SDRAM_OPERATION_ADDR, reg);
763 
764 			udelay(MRS_DELAY);
765 
766 			/* Write leveling  cs[cs] */
767 			if (MV_OK !=
768 			    ddr3_write_leveling_single_cs(cs, freq, ratio_2to1,
769 							  (u32 *)(res + cs),
770 							  dram_info)) {
771 				DEBUG_WL_FULL_C("DDR3 - Write Leveling single Cs - FAILED -  Cs - ",
772 						(u32) cs, 1);
773 				for (pup = 0; pup < max_pup_num; pup++) {
774 					if (((res[cs] >> pup) & 0x1) == 0) {
775 						DEBUG_WL_C("Failed Byte : ",
776 							   pup, 1);
777 					}
778 				}
779 				return MV_FAIL;
780 			}
781 
782 			/* Set TrnWLDeUpd - After each CS is done */
783 			reg = reg_read(REG_TRAINING_WL_ADDR) |
784 				(1 << REG_TRAINING_WL_CS_DONE_OFFS);
785 			/* 0x16AC - Training Write leveling register */
786 			reg_write(REG_TRAINING_WL_ADDR, reg);
787 
788 			/*
789 			 * Debug message - Finished Write leveling cs[cs] -
790 			 * each PUP Fail/Success
791 			 */
792 			DEBUG_WL_FULL_C("DDR3 - Write Leveling - Finished Cs - ", (u32) cs,
793 					1);
794 			DEBUG_WL_FULL_C("DDR3 - Write Leveling - The Results: 1-PUP locked, 0-PUP failed -",
795 					(u32) res[cs], 3);
796 
797 			/*
798 			 * Configure MR1 in cs[cs] - write leveling off (0),
799 			 * output buffer off (1)
800 			 */
801 			reg = reg_read(REG_DDR3_MR1_ADDR) &
802 				REG_DDR3_MR1_OUTBUF_WL_MASK;
803 			reg |= (1 << REG_DDR3_MR1_OUTBUF_DIS_OFFS);
804 			/* No need to sort ODT since it is same CS */
805 			/* 0x15D4 - DDR3 MR1 Register */
806 			reg_write(REG_DDR3_MR1_ADDR, reg);
807 			/* Issue MRS Command to current cs */
808 			reg = REG_SDRAM_OPERATION_CMD_MR1 &
809 				~(1 << (REG_SDRAM_OPERATION_CS_OFFS + cs));
810 			/*
811 			 * [3-0] = 0x4 - MR1 Command, [11-8] -
812 			 * enable current cs
813 			 */
814 			/* 0x1418 - SDRAM Operation Register */
815 			reg_write(REG_SDRAM_OPERATION_ADDR, reg);
816 
817 			udelay(MRS_DELAY);
818 		}
819 	}
820 
821 	/* Disable WL Mode */
822 	/* [2]=1 - TrnWLMode - Disable */
823 	reg = reg_read(REG_DRAM_TRAINING_2_ADDR);
824 	reg |= (1 << REG_DRAM_TRAINING_2_WL_MODE_OFFS);
825 	/* 0x15B8 - Training SW 2 Register */
826 	reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
827 
828 	/* Disable SW override - Must be in a different stage */
829 	/* [0]=0 - Enable SW override  */
830 	reg = reg_read(REG_DRAM_TRAINING_2_ADDR);
831 	reg &= ~(1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
832 	/* 0x15B8 - Training SW 2 Register */
833 	reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
834 
835 	/* Set Output buffer-on to all CS and correct ODT values */
836 	for (cs = 0; cs < MAX_CS; cs++) {
837 		if (dram_info->cs_ena & (1 << cs)) {
838 			reg = reg_read(REG_DDR3_MR1_ADDR) &
839 				REG_DDR3_MR1_ODT_MASK;
840 			reg &= REG_DDR3_MR1_OUTBUF_WL_MASK;
841 			reg |= odt_static[dram_info->cs_ena][cs];
842 
843 			/* 0x15D0 - DDR3 MR1 Register */
844 			reg_write(REG_DDR3_MR1_ADDR, reg);
845 			/* Issue MRS Command to current cs */
846 			reg = REG_SDRAM_OPERATION_CMD_MR1 &
847 				~(1 << (REG_SDRAM_OPERATION_CS_OFFS + cs));
848 			/*
849 			 * [3-0] = 0x4 - MR1 Command, [11-8] -
850 			 * enable current cs
851 			 */
852 			/* 0x1418 - SDRAM Operation Register */
853 			reg_write(REG_SDRAM_OPERATION_ADDR, reg);
854 
855 			udelay(MRS_DELAY);
856 		}
857 	}
858 
859 #ifdef MV88F67XX
860 	/* Dynamic pad issue (BTS669) during WL */
861 	if (dpde_flag) {
862 		reg = reg_read(REG_DUNIT_CTRL_LOW_ADDR) |
863 			(1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS);
864 		reg_write(REG_DUNIT_CTRL_LOW_ADDR, reg);
865 	}
866 #endif
867 	DEBUG_WL_FULL_S("DDR3 - Write Leveling - Finished WL procedure for all Cs\n");
868 
869 	return MV_OK;
870 }
871 
872 #if !defined(MV88F672X)
873 /*
874  * Name:     ddr3_write_leveling_sw
875  * Desc:     Execute Write leveling phase by SW
876  * Args:     freq        - current sequence frequency
877  *           dram_info   - main struct
878  * Notes:
879  * Returns:  MV_OK if success, MV_FAIL if fail.
880  */
ddr3_write_leveling_sw_reg_dimm(u32 freq,int ratio_2to1,MV_DRAM_INFO * dram_info)881 int ddr3_write_leveling_sw_reg_dimm(u32 freq, int ratio_2to1,
882 				    MV_DRAM_INFO *dram_info)
883 {
884 	u32 reg, cs, cnt, pup;
885 	u32 res[MAX_CS];
886 	__maybe_unused int dpde_flag = 0;
887 
888 	/* Debug message - Start Write leveling procedure */
889 	DEBUG_WL_S("DDR3 - Write Leveling - Starting SW WL procedure\n");
890 
891 #ifdef MV88F67XX
892 	/* Dynamic pad issue (BTS669) during WL */
893 	reg = reg_read(REG_DUNIT_CTRL_LOW_ADDR);
894 	if (reg & (1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS)) {
895 		dpde_flag = 1;
896 		reg_write(REG_DUNIT_CTRL_LOW_ADDR,
897 			  reg & ~(1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS));
898 	}
899 #endif
900 
901 	/* If target freq = 400 move clock start point */
902 	/* Write to control PUP to Control Deskew Regs */
903 	if (freq <= DDR_400) {
904 		for (pup = 0; pup <= dram_info->num_of_total_pups; pup++) {
905 			/* PUP_DELAY_MASK 0x1F */
906 			/* reg = 0x0C10001F + (uj << 16); */
907 			ddr3_write_ctrl_pup_reg(1, pup, CNTRL_PUP_DESKEW + pup,
908 						0x1F);
909 		}
910 	}
911 
912 	/* Set Output buffer-off to all CS and correct ODT values */
913 	for (cs = 0; cs < MAX_CS; cs++) {
914 		if (dram_info->cs_ena & (1 << cs)) {
915 			reg = reg_read(REG_DDR3_MR1_ADDR) &
916 				REG_DDR3_MR1_ODT_MASK;
917 			reg |= odt_static[dram_info->cs_ena][cs];
918 			reg |= (1 << REG_DDR3_MR1_OUTBUF_DIS_OFFS);
919 
920 			/* 0x15D0 - DDR3 MR0 Register */
921 			reg_write(REG_DDR3_MR1_ADDR, reg);
922 			/* Issue MRS Command to current cs */
923 			reg = REG_SDRAM_OPERATION_CMD_MR1 &
924 				~(1 << (REG_SDRAM_OPERATION_CS_OFFS + cs));
925 			/*
926 			 * [3-0] = 0x4 - MR1 Command, [11-8] -
927 			 * enable current cs
928 			 */
929 			/* 0x1418 - SDRAM Operation Register */
930 			reg_write(REG_SDRAM_OPERATION_ADDR, reg);
931 
932 			udelay(MRS_DELAY);
933 		}
934 	}
935 
936 	DEBUG_WL_FULL_S("DDR3 - Write Leveling - Qoff and RTT Values are set for all Cs\n");
937 
938 	/* Enable SW override */
939 	reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |
940 		(1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
941 	/* [0] = 1 - Enable SW override  */
942 	/* 0x15B8 - Training SW 2 Register */
943 	reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
944 	DEBUG_WL_FULL_S("DDR3 - Write Leveling - SW Override Enabled\n");
945 
946 	/* Enable PHY write leveling mode */
947 	reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &
948 		~(1 << REG_DRAM_TRAINING_2_WL_MODE_OFFS);
949 	/* [2] = 0 - TrnWLMode - Enable */
950 	/* 0x15B8 - Training SW 2 Register */
951 	reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
952 
953 	/* Loop for each cs */
954 	for (cs = 0; cs < MAX_CS; cs++) {
955 		if (dram_info->cs_ena & (1 << cs)) {
956 			DEBUG_WL_FULL_C("DDR3 - Write Leveling - Starting working with Cs - ",
957 					(u32) cs, 1);
958 
959 			/* Refresh X9 current cs */
960 			DEBUG_WL_FULL_S("DDR3 - Write Leveling - Refresh X9\n");
961 			for (cnt = 0; cnt < COUNT_WL_RFRS; cnt++) {
962 				reg =
963 				    REG_SDRAM_OPERATION_CMD_RFRS & ~(1 <<
964 								     (REG_SDRAM_OPERATION_CS_OFFS
965 								      + cs));
966 				/* [3-0] = 0x2 - refresh, [11-8] - enable current cs */
967 				reg_write(REG_SDRAM_OPERATION_ADDR, reg);	/* 0x1418 - SDRAM Operation Register */
968 
969 				do {
970 					reg =
971 					    ((reg_read
972 					      (REG_SDRAM_OPERATION_ADDR)) &
973 					     REG_SDRAM_OPERATION_CMD_RFRS_DONE);
974 				} while (reg);	/* Wait for '0' */
975 			}
976 
977 			/*
978 			 * Configure MR1 in Cs[CsNum] - write leveling on,
979 			 * output buffer on
980 			 */
981 			DEBUG_WL_FULL_S("DDR3 - Write Leveling - Configure MR1 for current Cs: WL-on,OB-on\n");
982 			reg = reg_read(REG_DDR3_MR1_ADDR) &
983 				REG_DDR3_MR1_OUTBUF_WL_MASK;
984 			/* Set ODT Values */
985 			reg &= REG_DDR3_MR1_ODT_MASK;
986 			reg |= odt_static[dram_info->cs_ena][cs];
987 			/* Enable WL MODE */
988 			reg |= (1 << REG_DDR3_MR1_WL_ENA_OFFS);
989 			/*
990 			 * [7]=1, [12]=0 - Output Buffer and write leveling
991 			 * enabled
992 			 */
993 			/* 0x15D4 - DDR3 MR1 Register */
994 			reg_write(REG_DDR3_MR1_ADDR, reg);
995 			/* Issue MRS Command to current cs */
996 			reg = REG_SDRAM_OPERATION_CMD_MR1 &
997 				~(1 << (REG_SDRAM_OPERATION_CS_OFFS + cs));
998 			/*
999 			 * [3-0] = 0x4 - MR1 Command, [11-8] -
1000 			 * enable current cs
1001 			 */
1002 			/* 0x1418 - SDRAM Operation Register */
1003 			reg_write(REG_SDRAM_OPERATION_ADDR, reg);
1004 
1005 			udelay(MRS_DELAY);
1006 
1007 			/* Write leveling  cs[cs] */
1008 			if (MV_OK !=
1009 			    ddr3_write_leveling_single_cs(cs, freq, ratio_2to1,
1010 							  (u32 *)(res + cs),
1011 							  dram_info)) {
1012 				DEBUG_WL_FULL_C("DDR3 - Write Leveling single Cs - FAILED -  Cs - ",
1013 						(u32) cs, 1);
1014 				return MV_FAIL;
1015 			}
1016 
1017 			/* Set TrnWLDeUpd - After each CS is done */
1018 			reg = reg_read(REG_TRAINING_WL_ADDR) |
1019 				(1 << REG_TRAINING_WL_CS_DONE_OFFS);
1020 			/* 0x16AC - Training Write leveling register */
1021 			reg_write(REG_TRAINING_WL_ADDR, reg);
1022 
1023 			/*
1024 			 * Debug message - Finished Write leveling cs[cs] -
1025 			 * each PUP Fail/Success
1026 			 */
1027 			DEBUG_WL_FULL_C("DDR3 - Write Leveling - Finished Cs - ", (u32) cs,
1028 					1);
1029 			DEBUG_WL_FULL_C("DDR3 - Write Leveling - The Results: 1-PUP locked, 0-PUP failed -",
1030 					(u32) res[cs], 3);
1031 
1032 			/* Configure MR1 in cs[cs] - write leveling off (0), output buffer off (1) */
1033 			reg = reg_read(REG_DDR3_MR1_ADDR) &
1034 				REG_DDR3_MR1_OUTBUF_WL_MASK;
1035 			reg |= (1 << REG_DDR3_MR1_OUTBUF_DIS_OFFS);
1036 			/* No need to sort ODT since it is same CS */
1037 			/* 0x15D4 - DDR3 MR1 Register */
1038 			reg_write(REG_DDR3_MR1_ADDR, reg);
1039 			/* Issue MRS Command to current cs */
1040 			reg = REG_SDRAM_OPERATION_CMD_MR1 &
1041 				~(1 << (REG_SDRAM_OPERATION_CS_OFFS + cs));
1042 			/*
1043 			 * [3-0] = 0x4 - MR1 Command, [11-8] -
1044 			 * enable current cs
1045 			 */
1046 			/* 0x1418 - SDRAM Operation Register */
1047 			reg_write(REG_SDRAM_OPERATION_ADDR, reg);
1048 
1049 			udelay(MRS_DELAY);
1050 		}
1051 	}
1052 
1053 	/* Disable WL Mode */
1054 	/* [2]=1 - TrnWLMode - Disable */
1055 	reg = reg_read(REG_DRAM_TRAINING_2_ADDR);
1056 	reg |= (1 << REG_DRAM_TRAINING_2_WL_MODE_OFFS);
1057 	/* 0x15B8 - Training SW 2 Register */
1058 	reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
1059 
1060 	/* Disable SW override - Must be in a different stage */
1061 	/* [0]=0 - Enable SW override  */
1062 	reg = reg_read(REG_DRAM_TRAINING_2_ADDR);
1063 	reg &= ~(1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
1064 	/* 0x15B8 - Training SW 2 Register */
1065 	reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
1066 
1067 	/* Set Output buffer-on to all CS and correct ODT values */
1068 	for (cs = 0; cs < MAX_CS; cs++) {
1069 		if (dram_info->cs_ena & (1 << cs)) {
1070 			reg = reg_read(REG_DDR3_MR1_ADDR) &
1071 				REG_DDR3_MR1_ODT_MASK;
1072 			reg &= REG_DDR3_MR1_OUTBUF_WL_MASK;
1073 			reg |= odt_static[dram_info->cs_ena][cs];
1074 
1075 			/* 0x15D0 - DDR3 MR1 Register */
1076 			reg_write(REG_DDR3_MR1_ADDR, reg);
1077 			/* Issue MRS Command to current cs */
1078 			reg = REG_SDRAM_OPERATION_CMD_MR1 &
1079 				~(1 << (REG_SDRAM_OPERATION_CS_OFFS + cs));
1080 			/*
1081 			 * [3-0] = 0x4 - MR1 Command, [11-8] -
1082 			 * enable current cs
1083 			 */
1084 			/* 0x1418 - SDRAM Operation Register */
1085 			reg_write(REG_SDRAM_OPERATION_ADDR, reg);
1086 
1087 			udelay(MRS_DELAY);
1088 		}
1089 	}
1090 
1091 #ifdef MV88F67XX
1092 	/* Dynamic pad issue (BTS669) during WL */
1093 	if (dpde_flag) {
1094 		reg = reg_read(REG_DUNIT_CTRL_LOW_ADDR) |
1095 			(1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS);
1096 		reg_write(REG_DUNIT_CTRL_LOW_ADDR, reg);
1097 	}
1098 #endif
1099 
1100 	/* If target freq = 400 move clock back */
1101 	/* Write to control PUP to Control Deskew Regs */
1102 	if (freq <= DDR_400) {
1103 		for (pup = 0; pup <= dram_info->num_of_total_pups; pup++) {
1104 			ddr3_write_ctrl_pup_reg(1, pup, CNTRL_PUP_DESKEW + pup,
1105 						0);
1106 		}
1107 	}
1108 
1109 	DEBUG_WL_FULL_S("DDR3 - Write Leveling - Finished WL procedure for all Cs\n");
1110 	return MV_OK;
1111 }
1112 #endif
1113 
1114 /*
1115  * Name:     ddr3_write_leveling_single_cs
1116  * Desc:     Execute Write leveling for single Chip select
1117  * Args:     cs          - current chip select
1118  *           freq        - current sequence frequency
1119  *           result      - res array
1120  *           dram_info   - main struct
1121  * Notes:
1122  * Returns:  MV_OK if success, MV_FAIL if fail.
1123  */
ddr3_write_leveling_single_cs(u32 cs,u32 freq,int ratio_2to1,u32 * result,MV_DRAM_INFO * dram_info)1124 static int ddr3_write_leveling_single_cs(u32 cs, u32 freq, int ratio_2to1,
1125 					 u32 *result, MV_DRAM_INFO *dram_info)
1126 {
1127 	u32 reg, pup_num, delay, phase, phaseMax, max_pup_num, pup,
1128 		max_pup_mask;
1129 
1130 	max_pup_num = dram_info->num_of_total_pups;
1131 	*result = 0;
1132 	u32 flag[MAX_PUP_NUM] = { 0 };
1133 
1134 	DEBUG_WL_FULL_C("DDR3 - Write Leveling Single Cs - WL for Cs - ",
1135 			(u32) cs, 1);
1136 
1137 	switch (max_pup_num) {
1138 	case 2:
1139 		max_pup_mask = 0x3;
1140 		break;
1141 	case 4:
1142 		max_pup_mask = 0xf;
1143 		DEBUG_WL_C("max_pup_mask =  ", max_pup_mask, 3);
1144 		break;
1145 	case 5:
1146 		max_pup_mask = 0x1f;
1147 		DEBUG_WL_C("max_pup_mask =  ", max_pup_mask, 3);
1148 		break;
1149 	case 8:
1150 		max_pup_mask = 0xff;
1151 		DEBUG_WL_C("max_pup_mask =  ", max_pup_mask, 3);
1152 		break;
1153 	case 9:
1154 		max_pup_mask = 0x1ff;
1155 		DEBUG_WL_C("max_pup_mask =  ", max_pup_mask, 3);
1156 		break;
1157 	default:
1158 		DEBUG_WL_C("ddr3_write_leveling_single_cs wrong max_pup_num =  ",
1159 			   max_pup_num, 3);
1160 		return MV_FAIL;
1161 	}
1162 
1163 	/* CS ODT Override */
1164 	reg = reg_read(REG_SDRAM_ODT_CTRL_HIGH_ADDR) &
1165 		REG_SDRAM_ODT_CTRL_HIGH_OVRD_MASK;
1166 	reg |= (REG_SDRAM_ODT_CTRL_HIGH_OVRD_ENA << (2 * cs));
1167 	/* Set 0x3 - Enable ODT on the curent cs and disable on other cs */
1168 	/* 0x1498 - SDRAM ODT Control high */
1169 	reg_write(REG_SDRAM_ODT_CTRL_HIGH_ADDR, reg);
1170 
1171 	DEBUG_WL_FULL_S("DDR3 - Write Leveling Single Cs - ODT Asserted for current Cs\n");
1172 
1173 	/* tWLMRD Delay */
1174 	/* Delay of minimum 40 Dram clock cycles - 20 Tclk cycles */
1175 	udelay(1);
1176 
1177 	/* [1:0] - current cs number */
1178 	reg = (reg_read(REG_TRAINING_WL_ADDR) & REG_TRAINING_WL_CS_MASK) | cs;
1179 	reg |= (1 << REG_TRAINING_WL_UPD_OFFS);	/* [2] - trnWLCsUpd */
1180 	/* 0x16AC - Training Write leveling register */
1181 	reg_write(REG_TRAINING_WL_ADDR, reg);
1182 
1183 	/* Broadcast to all PUPs: Reset DQS phase, reset leveling delay */
1184 	ddr3_write_pup_reg(PUP_WL_MODE, cs, PUP_BC, 0, 0);
1185 
1186 	/* Seek Edge */
1187 	DEBUG_WL_FULL_S("DDR3 - Write Leveling Single Cs - Seek Edge - Current Cs\n");
1188 
1189 	/* Drive DQS high for one cycle - All data PUPs */
1190 	DEBUG_WL_FULL_S("DDR3 - Write Leveling Single Cs - Seek Edge - Driving DQS high for one cycle\n");
1191 	if (!ratio_2to1) {
1192 		reg = (reg_read(REG_TRAINING_WL_ADDR) &
1193 		       REG_TRAINING_WL_RATIO_MASK) | REG_TRAINING_WL_1TO1;
1194 	} else {
1195 		reg = (reg_read(REG_TRAINING_WL_ADDR) &
1196 		       REG_TRAINING_WL_RATIO_MASK) | REG_TRAINING_WL_2TO1;
1197 	}
1198 	/* 0x16AC - Training Write leveling register */
1199 	reg_write(REG_TRAINING_WL_ADDR, reg);
1200 
1201 	/* Wait tWLdelay */
1202 	do {
1203 		/* [29] - trnWLDelayExp */
1204 		reg = (reg_read(REG_TRAINING_WL_ADDR)) &
1205 			REG_TRAINING_WL_DELAYEXP_MASK;
1206 	} while (reg == 0x0);	/* Wait for '1' */
1207 
1208 	/* Read WL res */
1209 	reg = (reg_read(REG_TRAINING_WL_ADDR) >> REG_TRAINING_WL_RESULTS_OFFS) &
1210 		REG_TRAINING_WL_RESULTS_MASK;
1211 	/* [28:20] - TrnWLResult */
1212 
1213 	if (!ratio_2to1) /* Different phase options for 2:1 or 1:1 modes */
1214 		phaseMax = MAX_PHASE_1TO1;
1215 	else
1216 		phaseMax = MAX_PHASE_2TO1;
1217 
1218 	DEBUG_WL_FULL_S("DDR3 - Write Leveling Single Cs - Seek Edge - Shift DQS + Octet Leveling\n");
1219 
1220 	/* Shift DQS + Octet leveling */
1221 	for (phase = 0; phase < phaseMax; phase++) {
1222 		for (delay = 0; delay < MAX_DELAY; delay++) {
1223 			/*  Broadcast to all PUPs: DQS phase,leveling delay */
1224 			ddr3_write_pup_reg(PUP_WL_MODE, cs, PUP_BC, phase,
1225 					   delay);
1226 
1227 			udelay(1);	/* Delay of  3 Tclk cycles */
1228 
1229 			DEBUG_WL_FULL_S("DDR3 - Write Leveling Single Cs - Seek Edge: Phase = ");
1230 			DEBUG_WL_FULL_D((u32) phase, 1);
1231 			DEBUG_WL_FULL_S(", Delay = ");
1232 			DEBUG_WL_FULL_D((u32) delay, 1);
1233 			DEBUG_WL_FULL_S("\n");
1234 
1235 			/* Drive DQS high for one cycle - All data PUPs */
1236 			if (!ratio_2to1) {
1237 				reg = (reg_read(REG_TRAINING_WL_ADDR) &
1238 				       REG_TRAINING_WL_RATIO_MASK) |
1239 					REG_TRAINING_WL_1TO1;
1240 			} else {
1241 				reg = (reg_read(REG_TRAINING_WL_ADDR) &
1242 				       REG_TRAINING_WL_RATIO_MASK) |
1243 					REG_TRAINING_WL_2TO1;
1244 			}
1245 			reg_write(REG_TRAINING_WL_ADDR, reg);	/* 0x16AC  */
1246 
1247 			/* Wait tWLdelay */
1248 			do {
1249 				reg = (reg_read(REG_TRAINING_WL_ADDR)) &
1250 					REG_TRAINING_WL_DELAYEXP_MASK;
1251 			} while (reg == 0x0);	/* [29] Wait for '1' */
1252 
1253 			/* Read WL res */
1254 			reg = reg_read(REG_TRAINING_WL_ADDR);
1255 			reg = (reg >> REG_TRAINING_WL_RESULTS_OFFS) &
1256 				REG_TRAINING_WL_RESULTS_MASK;	/* [28:20] */
1257 
1258 			DEBUG_WL_FULL_C("DDR3 - Write Leveling Single Cs - Seek Edge: Results =  ",
1259 					(u32) reg, 3);
1260 
1261 			/* Update State machine */
1262 			for (pup = 0; pup < (max_pup_num); pup++) {
1263 				/* ECC support - bit 8 */
1264 				pup_num = (pup == dram_info->num_of_std_pups) ?
1265 					ECC_BIT : pup;
1266 				if (dram_info->wl_val[cs][pup][S] == 0) {
1267 					/* Update phase to PUP */
1268 					dram_info->wl_val[cs][pup][P] = phase;
1269 					/* Update delay to PUP */
1270 					dram_info->wl_val[cs][pup][D] = delay;
1271 				}
1272 
1273 				if (((reg >> pup_num) & 0x1) == 0)
1274 					flag[pup_num] = 1;
1275 
1276 				if (((reg >> pup_num) & 0x1)
1277 				    && (flag[pup_num] == 1)
1278 				    && (dram_info->wl_val[cs][pup][S] == 0)) {
1279 					/*
1280 					 * If the PUP is locked now and in last
1281 					 * counter states
1282 					 */
1283 					/* Go to next state */
1284 					dram_info->wl_val[cs][pup][S] = 1;
1285 					/* Set res */
1286 					*result = *result | (1 << pup_num);
1287 				}
1288 			}
1289 
1290 			/* If all locked - Break the loops - Finished */
1291 			if (*result == max_pup_mask) {
1292 				phase = phaseMax;
1293 				delay = MAX_DELAY;
1294 				DEBUG_WL_S("DDR3 - Write Leveling Single Cs - Seek Edge: All Locked\n");
1295 			}
1296 		}
1297 	}
1298 
1299 	/* Debug message - Print res for cs[i]: cs,PUP,Phase,Delay */
1300 	DEBUG_WL_C("DDR3 - Write Leveling - Results for CS - ", (u32) cs, 1);
1301 	for (pup = 0; pup < (max_pup_num); pup++) {
1302 		DEBUG_WL_S("DDR3 - Write Leveling - PUP: ");
1303 		DEBUG_WL_D((u32) pup, 1);
1304 		DEBUG_WL_S(", Phase: ");
1305 		DEBUG_WL_D((u32) dram_info->wl_val[cs][pup][P], 1);
1306 		DEBUG_WL_S(", Delay: ");
1307 		DEBUG_WL_D((u32) dram_info->wl_val[cs][pup][D], 2);
1308 		DEBUG_WL_S("\n");
1309 	}
1310 
1311 	/* Check if some not locked and return error */
1312 	if (*result != max_pup_mask) {
1313 		DEBUG_WL_S("DDR3 - Write Leveling - ERROR - not all PUPS were locked\n");
1314 		return MV_FAIL;
1315 	}
1316 
1317 	/* Configure Each PUP with locked leveling settings */
1318 	for (pup = 0; pup < (max_pup_num); pup++) {
1319 		/* ECC support - bit 8 */
1320 		pup_num = (pup == dram_info->num_of_std_pups) ? ECC_BIT : pup;
1321 		phase = dram_info->wl_val[cs][pup][P];
1322 		delay = dram_info->wl_val[cs][pup][D];
1323 		ddr3_write_pup_reg(PUP_WL_MODE, cs, pup_num, phase, delay);
1324 	}
1325 
1326 	/* CS ODT Override */
1327 	reg =  reg_read(REG_SDRAM_ODT_CTRL_HIGH_ADDR) &
1328 		REG_SDRAM_ODT_CTRL_HIGH_OVRD_MASK;
1329 	/* 0x1498 - SDRAM ODT Control high */
1330 	reg_write(REG_SDRAM_ODT_CTRL_HIGH_ADDR, reg);
1331 
1332 	return MV_OK;
1333 }
1334 
1335 /*
1336  * Perform DDR3 Control PUP Indirect Write
1337  */
ddr3_write_ctrl_pup_reg(int bc_acc,u32 pup,u32 reg_addr,u32 data)1338 static void ddr3_write_ctrl_pup_reg(int bc_acc, u32 pup, u32 reg_addr, u32 data)
1339 {
1340 	u32 reg = 0;
1341 
1342 	/* Store value for write */
1343 	reg = (data & 0xFFFF);
1344 
1345 	/* Set bit 26 for control PHY access */
1346 	reg |= (1 << REG_PHY_CNTRL_OFFS);
1347 
1348 	/* Configure BC or UC access to PHYs */
1349 	if (bc_acc == 1)
1350 		reg |= (1 << REG_PHY_BC_OFFS);
1351 	else
1352 		reg |= (pup << REG_PHY_PUP_OFFS);
1353 
1354 	/* Set PHY register address to write to */
1355 	reg |= (reg_addr << REG_PHY_CS_OFFS);
1356 
1357 	reg_write(REG_PHY_REGISTRY_FILE_ACCESS_ADDR, reg);	/* 0x16A0 */
1358 	reg |= REG_PHY_REGISTRY_FILE_ACCESS_OP_WR;
1359 	reg_write(REG_PHY_REGISTRY_FILE_ACCESS_ADDR, reg);	/* 0x16A0 */
1360 
1361 	do {
1362 		reg = (reg_read(REG_PHY_REGISTRY_FILE_ACCESS_ADDR)) &
1363 			REG_PHY_REGISTRY_FILE_ACCESS_OP_DONE;
1364 	} while (reg);		/* Wait for '0' to mark the end of the transaction */
1365 }
1366