1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (C) Marvell International Ltd. and its affiliates
4  */
5 
6 #include <common.h>
7 #include <i2c.h>
8 #include <spl.h>
9 #include <asm/io.h>
10 #include <asm/arch/cpu.h>
11 #include <asm/arch/soc.h>
12 
13 #include "ddr3_hw_training.h"
14 
15 /*
16  * Debug
17  */
18 #define DEBUG_RL_C(s, d, l) \
19 	DEBUG_RL_S(s); DEBUG_RL_D(d, l); DEBUG_RL_S("\n")
20 #define DEBUG_RL_FULL_C(s, d, l) \
21 	DEBUG_RL_FULL_S(s); DEBUG_RL_FULL_D(d, l); DEBUG_RL_FULL_S("\n")
22 
23 #ifdef MV_DEBUG_RL
24 #define DEBUG_RL_S(s) \
25 	debug_cond(ddr3_get_log_level() >= MV_LOG_LEVEL_2, "%s", s)
26 #define DEBUG_RL_D(d, l) \
27 	debug_cond(ddr3_get_log_level() >= MV_LOG_LEVEL_2, "%x", d)
28 #else
29 #define DEBUG_RL_S(s)
30 #define DEBUG_RL_D(d, l)
31 #endif
32 
33 #ifdef MV_DEBUG_RL_FULL
34 #define DEBUG_RL_FULL_S(s)		puts(s)
35 #define DEBUG_RL_FULL_D(d, l)		printf("%x", d)
36 #else
37 #define DEBUG_RL_FULL_S(s)
38 #define DEBUG_RL_FULL_D(d, l)
39 #endif
40 
41 extern u32 rl_pattern[LEN_STD_PATTERN];
42 
43 #ifdef RL_MODE
44 static int ddr3_read_leveling_single_cs_rl_mode(u32 cs, u32 freq,
45 						int ratio_2to1, u32 ecc,
46 						MV_DRAM_INFO *dram_info);
47 #else
48 static int ddr3_read_leveling_single_cs_window_mode(u32 cs, u32 freq,
49 						    int ratio_2to1, u32 ecc,
50 						    MV_DRAM_INFO *dram_info);
51 #endif
52 
53 /*
54  * Name:     ddr3_read_leveling_hw
55  * Desc:     Execute the Read leveling phase by HW
56  * Args:     dram_info - main struct
57  *           freq      - current sequence frequency
58  * Notes:
59  * Returns:  MV_OK if success, MV_FAIL if fail.
60  */
61 int ddr3_read_leveling_hw(u32 freq, MV_DRAM_INFO *dram_info)
62 {
63 	u32 reg;
64 
65 	/* Debug message - Start Read leveling procedure */
66 	DEBUG_RL_S("DDR3 - Read Leveling - Starting HW RL procedure\n");
67 
68 	/* Start Auto Read Leveling procedure */
69 	reg = 1 << REG_DRAM_TRAINING_RL_OFFS;
70 	/* Config the retest number */
71 	reg |= (COUNT_HW_RL << REG_DRAM_TRAINING_RETEST_OFFS);
72 
73 	/* Enable CS in the automatic process */
74 	reg |= (dram_info->cs_ena << REG_DRAM_TRAINING_CS_OFFS);
75 
76 	reg_write(REG_DRAM_TRAINING_ADDR, reg);	/* 0x15B0 - Training Register */
77 
78 	reg = reg_read(REG_DRAM_TRAINING_SHADOW_ADDR) |
79 		(1 << REG_DRAM_TRAINING_AUTO_OFFS);
80 	reg_write(REG_DRAM_TRAINING_SHADOW_ADDR, reg);
81 
82 	/* Wait */
83 	do {
84 		reg = reg_read(REG_DRAM_TRAINING_SHADOW_ADDR) &
85 			(1 << REG_DRAM_TRAINING_AUTO_OFFS);
86 	} while (reg);		/* Wait for '0' */
87 
88 	/* Check if Successful */
89 	if (reg_read(REG_DRAM_TRAINING_SHADOW_ADDR) &
90 	    (1 << REG_DRAM_TRAINING_ERROR_OFFS)) {
91 		u32 delay, phase, pup, cs;
92 
93 		dram_info->rl_max_phase = 0;
94 		dram_info->rl_min_phase = 10;
95 
96 		/* Read results to arrays */
97 		for (cs = 0; cs < MAX_CS; cs++) {
98 			if (dram_info->cs_ena & (1 << cs)) {
99 				for (pup = 0;
100 				     pup < dram_info->num_of_total_pups;
101 				     pup++) {
102 					if (pup == dram_info->num_of_std_pups
103 					    && dram_info->ecc_ena)
104 						pup = ECC_PUP;
105 					reg =
106 					    ddr3_read_pup_reg(PUP_RL_MODE, cs,
107 							      pup);
108 					phase = (reg >> REG_PHY_PHASE_OFFS) &
109 						PUP_PHASE_MASK;
110 					delay = reg & PUP_DELAY_MASK;
111 					dram_info->rl_val[cs][pup][P] = phase;
112 					if (phase > dram_info->rl_max_phase)
113 						dram_info->rl_max_phase = phase;
114 					if (phase < dram_info->rl_min_phase)
115 						dram_info->rl_min_phase = phase;
116 					dram_info->rl_val[cs][pup][D] = delay;
117 					dram_info->rl_val[cs][pup][S] =
118 					    RL_FINAL_STATE;
119 					reg =
120 					    ddr3_read_pup_reg(PUP_RL_MODE + 0x1,
121 							      cs, pup);
122 					dram_info->rl_val[cs][pup][DQS] =
123 					    (reg & 0x3F);
124 				}
125 #ifdef MV_DEBUG_RL
126 				/* Print results */
127 				DEBUG_RL_C("DDR3 - Read Leveling - Results for CS - ",
128 					   (u32) cs, 1);
129 
130 				for (pup = 0;
131 				     pup < (dram_info->num_of_total_pups);
132 				     pup++) {
133 					if (pup == dram_info->num_of_std_pups
134 					    && dram_info->ecc_ena)
135 						pup = ECC_PUP;
136 					DEBUG_RL_S("DDR3 - Read Leveling - PUP: ");
137 					DEBUG_RL_D((u32) pup, 1);
138 					DEBUG_RL_S(", Phase: ");
139 					DEBUG_RL_D((u32) dram_info->
140 						   rl_val[cs][pup][P], 1);
141 					DEBUG_RL_S(", Delay: ");
142 					DEBUG_RL_D((u32) dram_info->
143 						   rl_val[cs][pup][D], 2);
144 					DEBUG_RL_S("\n");
145 				}
146 #endif
147 			}
148 		}
149 
150 		dram_info->rd_rdy_dly =
151 			reg_read(REG_READ_DATA_READY_DELAYS_ADDR) &
152 			REG_READ_DATA_SAMPLE_DELAYS_MASK;
153 		dram_info->rd_smpl_dly =
154 			reg_read(REG_READ_DATA_SAMPLE_DELAYS_ADDR) &
155 			REG_READ_DATA_READY_DELAYS_MASK;
156 #ifdef MV_DEBUG_RL
157 		DEBUG_RL_C("DDR3 - Read Leveling - Read Sample Delay: ",
158 			   dram_info->rd_smpl_dly, 2);
159 		DEBUG_RL_C("DDR3 - Read Leveling - Read Ready Delay: ",
160 			   dram_info->rd_rdy_dly, 2);
161 		DEBUG_RL_S("DDR3 - Read Leveling - HW RL Ended Successfully\n");
162 #endif
163 		return MV_OK;
164 
165 	} else {
166 		DEBUG_RL_S("DDR3 - Read Leveling - HW RL Error\n");
167 		return MV_FAIL;
168 	}
169 }
170 
171 /*
172  * Name:     ddr3_read_leveling_sw
173  * Desc:     Execute the Read leveling phase by SW
174  * Args:     dram_info - main struct
175  *           freq      - current sequence frequency
176  * Notes:
177  * Returns:  MV_OK if success, MV_FAIL if fail.
178  */
179 int ddr3_read_leveling_sw(u32 freq, int ratio_2to1, MV_DRAM_INFO *dram_info)
180 {
181 	u32 reg, cs, ecc, pup_num, phase, delay, pup;
182 	int status;
183 
184 	/* Debug message - Start Read leveling procedure */
185 	DEBUG_RL_S("DDR3 - Read Leveling - Starting SW RL procedure\n");
186 
187 	/* Enable SW Read Leveling */
188 	reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |
189 		(1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
190 	reg &= ~(1 << REG_DRAM_TRAINING_2_RL_MODE_OFFS);
191 	/* [0]=1 - Enable SW override  */
192 	/* 0x15B8 - Training SW 2 Register */
193 	reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
194 
195 #ifdef RL_MODE
196 	reg = (dram_info->cs_ena << REG_DRAM_TRAINING_CS_OFFS) |
197 		(1 << REG_DRAM_TRAINING_AUTO_OFFS);
198 	reg_write(REG_DRAM_TRAINING_ADDR, reg);	/* 0x15B0 - Training Register */
199 #endif
200 
201 	/* Loop for each CS */
202 	for (cs = 0; cs < dram_info->num_cs; cs++) {
203 		DEBUG_RL_C("DDR3 - Read Leveling - CS - ", (u32) cs, 1);
204 
205 		for (ecc = 0; ecc <= (dram_info->ecc_ena); ecc++) {
206 			/* ECC Support - Switch ECC Mux on ecc=1 */
207 			reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &
208 				~(1 << REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
209 			reg |= (dram_info->ecc_ena *
210 				ecc << REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
211 			reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
212 
213 			if (ecc)
214 				DEBUG_RL_S("DDR3 - Read Leveling - ECC Mux Enabled\n");
215 			else
216 				DEBUG_RL_S("DDR3 - Read Leveling - ECC Mux Disabled\n");
217 
218 			/* Set current sample delays */
219 			reg = reg_read(REG_READ_DATA_SAMPLE_DELAYS_ADDR);
220 			reg &= ~(REG_READ_DATA_SAMPLE_DELAYS_MASK <<
221 				 (REG_READ_DATA_SAMPLE_DELAYS_OFFS * cs));
222 			reg |= (dram_info->cl <<
223 				(REG_READ_DATA_SAMPLE_DELAYS_OFFS * cs));
224 			reg_write(REG_READ_DATA_SAMPLE_DELAYS_ADDR, reg);
225 
226 			/* Set current Ready delay */
227 			reg = reg_read(REG_READ_DATA_READY_DELAYS_ADDR);
228 			reg &= ~(REG_READ_DATA_READY_DELAYS_MASK <<
229 				 (REG_READ_DATA_READY_DELAYS_OFFS * cs));
230 			if (!ratio_2to1) {
231 				/* 1:1 mode */
232 				reg |= ((dram_info->cl + 1) <<
233 					(REG_READ_DATA_READY_DELAYS_OFFS * cs));
234 			} else {
235 				/* 2:1 mode */
236 				reg |= ((dram_info->cl + 2) <<
237 					(REG_READ_DATA_READY_DELAYS_OFFS * cs));
238 			}
239 			reg_write(REG_READ_DATA_READY_DELAYS_ADDR, reg);
240 
241 			/* Read leveling Single CS[cs] */
242 #ifdef RL_MODE
243 			status =
244 			    ddr3_read_leveling_single_cs_rl_mode(cs, freq,
245 								 ratio_2to1,
246 								 ecc,
247 								 dram_info);
248 			if (MV_OK != status)
249 				return status;
250 #else
251 			status =
252 			    ddr3_read_leveling_single_cs_window_mode(cs, freq,
253 								     ratio_2to1,
254 								     ecc,
255 								     dram_info)
256 			    if (MV_OK != status)
257 				return status;
258 #endif
259 		}
260 
261 		/* Print results */
262 		DEBUG_RL_C("DDR3 - Read Leveling - Results for CS - ", (u32) cs,
263 			   1);
264 
265 		for (pup = 0;
266 		     pup < (dram_info->num_of_std_pups + dram_info->ecc_ena);
267 		     pup++) {
268 			DEBUG_RL_S("DDR3 - Read Leveling - PUP: ");
269 			DEBUG_RL_D((u32) pup, 1);
270 			DEBUG_RL_S(", Phase: ");
271 			DEBUG_RL_D((u32) dram_info->rl_val[cs][pup][P], 1);
272 			DEBUG_RL_S(", Delay: ");
273 			DEBUG_RL_D((u32) dram_info->rl_val[cs][pup][D], 2);
274 			DEBUG_RL_S("\n");
275 		}
276 
277 		DEBUG_RL_C("DDR3 - Read Leveling - Read Sample Delay: ",
278 			   dram_info->rd_smpl_dly, 2);
279 		DEBUG_RL_C("DDR3 - Read Leveling - Read Ready Delay: ",
280 			   dram_info->rd_rdy_dly, 2);
281 
282 		/* Configure PHY with average of 3 locked leveling settings */
283 		for (pup = 0;
284 		     pup < (dram_info->num_of_std_pups + dram_info->ecc_ena);
285 		     pup++) {
286 			/* ECC support - bit 8 */
287 			pup_num = (pup == dram_info->num_of_std_pups) ? ECC_BIT : pup;
288 
289 			/* For now, set last cnt result */
290 			phase = dram_info->rl_val[cs][pup][P];
291 			delay = dram_info->rl_val[cs][pup][D];
292 			ddr3_write_pup_reg(PUP_RL_MODE, cs, pup_num, phase,
293 					   delay);
294 		}
295 	}
296 
297 	/* Reset PHY read FIFO */
298 	reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |
299 		(1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS);
300 	/* 0x15B8 - Training SW 2 Register */
301 	reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
302 
303 	do {
304 		reg = (reg_read(REG_DRAM_TRAINING_2_ADDR)) &
305 			(1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS);
306 	} while (reg);		/* Wait for '0' */
307 
308 	/* ECC Support - Switch ECC Mux off ecc=0 */
309 	reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &
310 		~(1 << REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
311 	reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
312 
313 #ifdef RL_MODE
314 	reg_write(REG_DRAM_TRAINING_ADDR, 0);	/* 0x15B0 - Training Register */
315 #endif
316 
317 	/* Disable SW Read Leveling */
318 	reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &
319 		~(1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
320 	/* [0] = 0 - Disable SW override  */
321 	reg = (reg | (0x1 << REG_DRAM_TRAINING_2_RL_MODE_OFFS));
322 	/* [3] = 1 - Disable RL MODE */
323 	/* 0x15B8 - Training SW 2 Register */
324 	reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
325 
326 	DEBUG_RL_S("DDR3 - Read Leveling - Finished RL procedure for all CS\n");
327 	return MV_OK;
328 }
329 
330 #ifdef RL_MODE
331 /*
332  * overrun() extracted from ddr3_read_leveling_single_cs_rl_mode().
333  * This just got too much indented making it hard to read / edit.
334  */
335 static void overrun(u32 cs, MV_DRAM_INFO *info, u32 pup, u32 locked_pups,
336 		    u32 *locked_sum, u32 ecc, int *first_octet_locked,
337 		    int *counter_in_progress, int final_delay, u32 delay,
338 		    u32 phase)
339 {
340 	/* If no OverRun */
341 	if (((~locked_pups >> pup) & 0x1) && (final_delay == 0)) {
342 		int idx;
343 
344 		idx = pup + ecc * ECC_BIT;
345 
346 		/* PUP passed, start examining */
347 		if (info->rl_val[cs][idx][S] == RL_UNLOCK_STATE) {
348 			/* Must be RL_UNLOCK_STATE */
349 			/* Match expected value ? - Update State Machine */
350 			if (info->rl_val[cs][idx][C] < RL_RETRY_COUNT) {
351 				DEBUG_RL_FULL_C("DDR3 - Read Leveling - We have no overrun and a match on pup: ",
352 						(u32)pup, 1);
353 				info->rl_val[cs][idx][C]++;
354 
355 				/* If pup got to last state - lock the delays */
356 				if (info->rl_val[cs][idx][C] == RL_RETRY_COUNT) {
357 					info->rl_val[cs][idx][C] = 0;
358 					info->rl_val[cs][idx][DS] = delay;
359 					info->rl_val[cs][idx][PS] = phase;
360 
361 					/* Go to Final State */
362 					info->rl_val[cs][idx][S] = RL_FINAL_STATE;
363 					*locked_sum = *locked_sum + 1;
364 					DEBUG_RL_FULL_C("DDR3 - Read Leveling - We have locked pup: ",
365 							(u32)pup, 1);
366 
367 					/*
368 					 * If first lock - need to lock delays
369 					 */
370 					if (*first_octet_locked == 0) {
371 						DEBUG_RL_FULL_C("DDR3 - Read Leveling - We got first lock on pup: ",
372 								(u32)pup, 1);
373 						*first_octet_locked = 1;
374 					}
375 
376 					/*
377 					 * If pup is in not in final state but
378 					 * there was match - dont increment
379 					 * counter
380 					 */
381 				} else {
382 					*counter_in_progress = 1;
383 				}
384 			}
385 		}
386 	}
387 }
388 
389 /*
390  * Name:     ddr3_read_leveling_single_cs_rl_mode
391  * Desc:     Execute Read leveling for single Chip select
392  * Args:     cs        - current chip select
393  *           freq      - current sequence frequency
394  *           ecc       - ecc iteration indication
395  *           dram_info - main struct
396  * Notes:
397  * Returns:  MV_OK if success, MV_FAIL if fail.
398  */
399 static int ddr3_read_leveling_single_cs_rl_mode(u32 cs, u32 freq,
400 						int ratio_2to1, u32 ecc,
401 						MV_DRAM_INFO *dram_info)
402 {
403 	u32 reg, delay, phase, pup, rd_sample_delay, add, locked_pups,
404 		repeat_max_cnt, sdram_offset, locked_sum;
405 	u32 phase_min, ui_max_delay;
406 	int all_locked, first_octet_locked, counter_in_progress;
407 	int final_delay = 0;
408 
409 	DEBUG_RL_FULL_C("DDR3 - Read Leveling - Single CS - ", (u32) cs, 1);
410 
411 	/* Init values */
412 	phase = 0;
413 	delay = 0;
414 	rd_sample_delay = dram_info->cl;
415 	all_locked = 0;
416 	first_octet_locked = 0;
417 	repeat_max_cnt = 0;
418 	locked_sum = 0;
419 
420 	for (pup = 0; pup < (dram_info->num_of_std_pups * (1 - ecc) + ecc);
421 	     pup++)
422 		dram_info->rl_val[cs][pup + ecc * ECC_BIT][S] = 0;
423 
424 	/* Main loop */
425 	while (!all_locked) {
426 		counter_in_progress = 0;
427 
428 		DEBUG_RL_FULL_S("DDR3 - Read Leveling - RdSmplDly = ");
429 		DEBUG_RL_FULL_D(rd_sample_delay, 2);
430 		DEBUG_RL_FULL_S(", RdRdyDly = ");
431 		DEBUG_RL_FULL_D(dram_info->rd_rdy_dly, 2);
432 		DEBUG_RL_FULL_S(", Phase = ");
433 		DEBUG_RL_FULL_D(phase, 1);
434 		DEBUG_RL_FULL_S(", Delay = ");
435 		DEBUG_RL_FULL_D(delay, 2);
436 		DEBUG_RL_FULL_S("\n");
437 
438 		/*
439 		 * Broadcast to all PUPs current RL delays: DQS phase,
440 		 * leveling delay
441 		 */
442 		ddr3_write_pup_reg(PUP_RL_MODE, cs, PUP_BC, phase, delay);
443 
444 		/* Reset PHY read FIFO */
445 		reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |
446 			(1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS);
447 		/* 0x15B8 - Training SW 2 Register */
448 		reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
449 
450 		do {
451 			reg = (reg_read(REG_DRAM_TRAINING_2_ADDR)) &
452 				(1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS);
453 		} while (reg);	/* Wait for '0' */
454 
455 		/* Read pattern from SDRAM */
456 		sdram_offset = cs * (SDRAM_CS_SIZE + 1) + SDRAM_RL_OFFS;
457 		locked_pups = 0;
458 		if (MV_OK !=
459 		    ddr3_sdram_compare(dram_info, 0xFF, &locked_pups,
460 				       rl_pattern, LEN_STD_PATTERN,
461 				       sdram_offset, 0, 0, NULL, 0))
462 			return MV_DDR3_TRAINING_ERR_RD_LVL_RL_PATTERN;
463 
464 		/* Octet evaluation */
465 		/* pup_num = Q or 1 for ECC */
466 		for (pup = 0; pup < (dram_info->num_of_std_pups * (1 - ecc) + ecc); pup++) {
467 			/* Check Overrun */
468 			if (!((reg_read(REG_DRAM_TRAINING_2_ADDR) >>
469 			       (REG_DRAM_TRAINING_2_OVERRUN_OFFS + pup)) & 0x1)) {
470 				overrun(cs, dram_info, pup, locked_pups,
471 					&locked_sum, ecc, &first_octet_locked,
472 					&counter_in_progress, final_delay,
473 					delay, phase);
474 			} else {
475 				DEBUG_RL_FULL_C("DDR3 - Read Leveling - We got overrun on pup: ",
476 						(u32)pup, 1);
477 			}
478 		}
479 
480 		if (locked_sum == (dram_info->num_of_std_pups *
481 				   (1 - ecc) + ecc)) {
482 			all_locked = 1;
483 			DEBUG_RL_FULL_S("DDR3 - Read Leveling - Single Cs - All pups locked\n");
484 		}
485 
486 		/*
487 		 * This is a fix for unstable condition where pups are
488 		 * toggling between match and no match
489 		 */
490 		/*
491 		 * If some of the pups is >1 <3, check if we did it too
492 		 * many times
493 		 */
494 		if (counter_in_progress == 1) {
495 			/* Notify at least one Counter is >=1 and < 3 */
496 			if (repeat_max_cnt < RL_RETRY_COUNT) {
497 				repeat_max_cnt++;
498 				counter_in_progress = 1;
499 				DEBUG_RL_FULL_S("DDR3 - Read Leveling - Counter is >=1 and <3\n");
500 				DEBUG_RL_FULL_S("DDR3 - Read Leveling - So we will not increment the delay to see if locked again\n");
501 			} else {
502 				DEBUG_RL_FULL_S("DDR3 - Read Leveling - repeat_max_cnt reached max so now we will increment the delay\n");
503 				counter_in_progress = 0;
504 			}
505 		}
506 
507 		/*
508 		 * Check some of the pups are in the middle of state machine
509 		 * and don't increment the delays
510 		 */
511 		if (!counter_in_progress && !all_locked) {
512 			int idx;
513 
514 			idx = pup + ecc * ECC_BIT;
515 
516 			repeat_max_cnt = 0;
517 			/* if 1:1 mode */
518 			if ((!ratio_2to1) && ((phase == 0) || (phase == 4)))
519 				ui_max_delay = MAX_DELAY_INV;
520 			else
521 				ui_max_delay = MAX_DELAY;
522 
523 			/* Increment Delay */
524 			if (delay < ui_max_delay) {
525 				delay++;
526 				/*
527 				 * Mark the last delay/pahse place for
528 				 * window final place
529 				 */
530 				if (delay == ui_max_delay) {
531 					if ((!ratio_2to1 && phase ==
532 					     MAX_PHASE_RL_L_1TO1)
533 					    || (ratio_2to1 && phase ==
534 						MAX_PHASE_RL_L_2TO1))
535 						final_delay = 1;
536 				}
537 			} else {
538 				/* Phase+CL Incrementation */
539 				delay = 0;
540 
541 				if (!ratio_2to1) {
542 					/* 1:1 mode */
543 					if (first_octet_locked) {
544 						/* some Pup was Locked */
545 						if (phase < MAX_PHASE_RL_L_1TO1) {
546 							if (phase == 1) {
547 								phase = 4;
548 							} else {
549 								phase++;
550 								delay = MIN_DELAY_PHASE_1_LIMIT;
551 							}
552 						} else {
553 							DEBUG_RL_FULL_S("DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n");
554 							DEBUG_RL_S("1)DDR3 - Read Leveling - ERROR - NOT all PUPs Locked n");
555 							return MV_DDR3_TRAINING_ERR_RD_LVL_RL_PUP_UNLOCK;
556 						}
557 					} else {
558 						/* NO Pup was Locked */
559 						if (phase < MAX_PHASE_RL_UL_1TO1) {
560 							phase++;
561 							delay =
562 							    MIN_DELAY_PHASE_1_LIMIT;
563 						} else {
564 							phase = 0;
565 						}
566 					}
567 				} else {
568 					/* 2:1 mode */
569 					if (first_octet_locked) {
570 						/* some Pup was Locked */
571 						if (phase < MAX_PHASE_RL_L_2TO1) {
572 							phase++;
573 						} else {
574 							DEBUG_RL_FULL_S("DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n");
575 							DEBUG_RL_S("2)DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n");
576 							for (pup = 0; pup < (dram_info->num_of_std_pups * (1 - ecc) + ecc); pup++) {
577 								/* pup_num = Q or 1 for ECC */
578 								if (dram_info->rl_val[cs][idx][S]
579 								    == 0) {
580 									DEBUG_RL_C("Failed byte is = ",
581 										   pup, 1);
582 								}
583 							}
584 							return MV_DDR3_TRAINING_ERR_RD_LVL_RL_PUP_UNLOCK;
585 						}
586 					} else {
587 						/* No Pup was Locked */
588 						if (phase < MAX_PHASE_RL_UL_2TO1)
589 							phase++;
590 						else
591 							phase = 0;
592 					}
593 				}
594 
595 				/*
596 				 * If we finished a full Phases cycle (so now
597 				 * phase = 0, need to increment rd_sample_dly
598 				 */
599 				if (phase == 0 && first_octet_locked == 0) {
600 					rd_sample_delay++;
601 					if (rd_sample_delay == 0x10) {
602 						DEBUG_RL_FULL_S("DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n");
603 						DEBUG_RL_S("3)DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n");
604 						for (pup = 0; pup < (dram_info->num_of_std_pups * (1 - ecc) + ecc); pup++) {
605 							/* pup_num = Q or 1 for ECC */
606 							if (dram_info->
607 							    rl_val[cs][idx][S] == 0) {
608 								DEBUG_RL_C("Failed byte is = ",
609 									   pup, 1);
610 							}
611 						}
612 						return MV_DDR3_TRAINING_ERR_RD_LVL_PUP_UNLOCK;
613 					}
614 
615 					/* Set current rd_sample_delay  */
616 					reg = reg_read(REG_READ_DATA_SAMPLE_DELAYS_ADDR);
617 					reg &= ~(REG_READ_DATA_SAMPLE_DELAYS_MASK
618 					      << (REG_READ_DATA_SAMPLE_DELAYS_OFFS
619 						  * cs));
620 					reg |= (rd_sample_delay <<
621 						(REG_READ_DATA_SAMPLE_DELAYS_OFFS *
622 						 cs));
623 					reg_write(REG_READ_DATA_SAMPLE_DELAYS_ADDR,
624 						  reg);
625 				}
626 
627 				/*
628 				 * Set current rdReadyDelay according to the
629 				 * hash table (Need to do this in every phase
630 				 * change)
631 				 */
632 				if (!ratio_2to1) {
633 					/* 1:1 mode */
634 					add = reg_read(REG_TRAINING_DEBUG_2_ADDR);
635 					switch (phase) {
636 					case 0:
637 						add = (add >>
638 						       REG_TRAINING_DEBUG_2_OFFS);
639 						break;
640 					case 1:
641 						add = (add >>
642 						       (REG_TRAINING_DEBUG_2_OFFS
643 							+ 3));
644 						break;
645 					case 4:
646 						add = (add >>
647 						       (REG_TRAINING_DEBUG_2_OFFS
648 							+ 6));
649 						break;
650 					case 5:
651 						add = (add >>
652 						       (REG_TRAINING_DEBUG_2_OFFS
653 							+ 9));
654 						break;
655 					}
656 					add &= REG_TRAINING_DEBUG_2_MASK;
657 				} else {
658 					/* 2:1 mode */
659 					add = reg_read(REG_TRAINING_DEBUG_3_ADDR);
660 					add = (add >>
661 					       (phase *
662 						REG_TRAINING_DEBUG_3_OFFS));
663 					add &= REG_TRAINING_DEBUG_3_MASK;
664 				}
665 
666 				reg = reg_read(REG_READ_DATA_READY_DELAYS_ADDR);
667 				reg &= ~(REG_READ_DATA_READY_DELAYS_MASK <<
668 					 (REG_READ_DATA_READY_DELAYS_OFFS * cs));
669 				reg |= ((rd_sample_delay + add) <<
670 					(REG_READ_DATA_READY_DELAYS_OFFS * cs));
671 				reg_write(REG_READ_DATA_READY_DELAYS_ADDR, reg);
672 				dram_info->rd_smpl_dly = rd_sample_delay;
673 				dram_info->rd_rdy_dly = rd_sample_delay + add;
674 			}
675 
676 			/* Reset counters for pups with states<RD_STATE_COUNT */
677 			for (pup = 0; pup <
678 				     (dram_info->num_of_std_pups * (1 - ecc) + ecc);
679 			     pup++) {
680 				if (dram_info->rl_val[cs][idx][C] < RL_RETRY_COUNT)
681 					dram_info->rl_val[cs][idx][C] = 0;
682 			}
683 		}
684 	}
685 
686 	phase_min = 10;
687 
688 	for (pup = 0; pup < (dram_info->num_of_std_pups); pup++) {
689 		if (dram_info->rl_val[cs][pup][PS] < phase_min)
690 			phase_min = dram_info->rl_val[cs][pup][PS];
691 	}
692 
693 	/*
694 	 * Set current rdReadyDelay according to the hash table (Need to
695 	 * do this in every phase change)
696 	 */
697 	if (!ratio_2to1) {
698 		/* 1:1 mode */
699 		add = reg_read(REG_TRAINING_DEBUG_2_ADDR);
700 		switch (phase_min) {
701 		case 0:
702 			add = (add >> REG_TRAINING_DEBUG_2_OFFS);
703 			break;
704 		case 1:
705 			add = (add >> (REG_TRAINING_DEBUG_2_OFFS + 3));
706 			break;
707 		case 4:
708 			add = (add >> (REG_TRAINING_DEBUG_2_OFFS + 6));
709 			break;
710 		case 5:
711 			add = (add >> (REG_TRAINING_DEBUG_2_OFFS + 9));
712 			break;
713 		}
714 		add &= REG_TRAINING_DEBUG_2_MASK;
715 	} else {
716 		/* 2:1 mode */
717 		add = reg_read(REG_TRAINING_DEBUG_3_ADDR);
718 		add = (add >> (phase_min * REG_TRAINING_DEBUG_3_OFFS));
719 		add &= REG_TRAINING_DEBUG_3_MASK;
720 	}
721 
722 	reg = reg_read(REG_READ_DATA_READY_DELAYS_ADDR);
723 	reg &= ~(REG_READ_DATA_READY_DELAYS_MASK <<
724 		 (REG_READ_DATA_READY_DELAYS_OFFS * cs));
725 	reg |= ((rd_sample_delay + add) << (REG_READ_DATA_READY_DELAYS_OFFS * cs));
726 	reg_write(REG_READ_DATA_READY_DELAYS_ADDR, reg);
727 	dram_info->rd_rdy_dly = rd_sample_delay + add;
728 
729 	for (cs = 0; cs < dram_info->num_cs; cs++) {
730 		for (pup = 0; pup < dram_info->num_of_total_pups; pup++) {
731 			reg = ddr3_read_pup_reg(PUP_RL_MODE + 0x1, cs, pup);
732 			dram_info->rl_val[cs][pup][DQS] = (reg & 0x3F);
733 		}
734 	}
735 
736 	return MV_OK;
737 }
738 
739 #else
740 
741 /*
742  * Name:     ddr3_read_leveling_single_cs_window_mode
743  * Desc:     Execute Read leveling for single Chip select
744  * Args:     cs        - current chip select
745  *           freq      - current sequence frequency
746  *           ecc       - ecc iteration indication
747  *           dram_info - main struct
748  * Notes:
749  * Returns:  MV_OK if success, MV_FAIL if fail.
750  */
751 static int ddr3_read_leveling_single_cs_window_mode(u32 cs, u32 freq,
752 						    int ratio_2to1, u32 ecc,
753 						    MV_DRAM_INFO *dram_info)
754 {
755 	u32 reg, delay, phase, sum, pup, rd_sample_delay, add, locked_pups,
756 	    repeat_max_cnt, sdram_offset, final_sum, locked_sum;
757 	u32 delay_s, delay_e, tmp, phase_min, ui_max_delay;
758 	int all_locked, first_octet_locked, counter_in_progress;
759 	int final_delay = 0;
760 
761 	DEBUG_RL_FULL_C("DDR3 - Read Leveling - Single CS - ", (u32) cs, 1);
762 
763 	/* Init values */
764 	phase = 0;
765 	delay = 0;
766 	rd_sample_delay = dram_info->cl;
767 	all_locked = 0;
768 	first_octet_locked = 0;
769 	repeat_max_cnt = 0;
770 	sum = 0;
771 	final_sum = 0;
772 	locked_sum = 0;
773 
774 	for (pup = 0; pup < (dram_info->num_of_std_pups * (1 - ecc) + ecc);
775 	     pup++)
776 		dram_info->rl_val[cs][pup + ecc * ECC_BIT][S] = 0;
777 
778 	/* Main loop */
779 	while (!all_locked) {
780 		counter_in_progress = 0;
781 
782 		DEBUG_RL_FULL_S("DDR3 - Read Leveling - RdSmplDly = ");
783 		DEBUG_RL_FULL_D(rd_sample_delay, 2);
784 		DEBUG_RL_FULL_S(", RdRdyDly = ");
785 		DEBUG_RL_FULL_D(dram_info->rd_rdy_dly, 2);
786 		DEBUG_RL_FULL_S(", Phase = ");
787 		DEBUG_RL_FULL_D(phase, 1);
788 		DEBUG_RL_FULL_S(", Delay = ");
789 		DEBUG_RL_FULL_D(delay, 2);
790 		DEBUG_RL_FULL_S("\n");
791 
792 		/*
793 		 * Broadcast to all PUPs current RL delays: DQS phase,leveling
794 		 * delay
795 		 */
796 		ddr3_write_pup_reg(PUP_RL_MODE, cs, PUP_BC, phase, delay);
797 
798 		/* Reset PHY read FIFO */
799 		reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |
800 			(1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS);
801 		/* 0x15B8 - Training SW 2 Register */
802 		reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
803 
804 		do {
805 			reg = (reg_read(REG_DRAM_TRAINING_2_ADDR)) &
806 				(1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS);
807 		} while (reg);	/* Wait for '0' */
808 
809 		/* Read pattern from SDRAM */
810 		sdram_offset = cs * (SDRAM_CS_SIZE + 1) + SDRAM_RL_OFFS;
811 		locked_pups = 0;
812 		if (MV_OK !=
813 		    ddr3_sdram_compare(dram_info, 0xFF, &locked_pups,
814 				       rl_pattern, LEN_STD_PATTERN,
815 				       sdram_offset, 0, 0, NULL, 0))
816 			return MV_DDR3_TRAINING_ERR_RD_LVL_WIN_PATTERN;
817 
818 		/* Octet evaluation */
819 		for (pup = 0; pup < (dram_info->num_of_std_pups *
820 				     (1 - ecc) + ecc); pup++) {
821 			/* pup_num = Q or 1 for ECC */
822 			int idx;
823 
824 			idx = pup + ecc * ECC_BIT;
825 
826 			/* Check Overrun */
827 			if (!((reg_read(REG_DRAM_TRAINING_2_ADDR) >>
828 			      (REG_DRAM_TRAINING_2_OVERRUN_OFFS +
829 			       pup)) & 0x1)) {
830 				/* If no OverRun */
831 
832 				/* Inside the window */
833 				if (dram_info->rl_val[cs][idx][S] == RL_WINDOW_STATE) {
834 					/*
835 					 * Match expected value ? - Update
836 					 * State Machine
837 					 */
838 					if (((~locked_pups >> pup) & 0x1)
839 					    && (final_delay == 0)) {
840 						/* Match - Still inside the Window */
841 						DEBUG_RL_FULL_C("DDR3 - Read Leveling - We got another match inside the window  for pup: ",
842 								(u32)pup, 1);
843 
844 					} else {
845 						/* We got fail -> this is the end of the window */
846 						dram_info->rl_val[cs][idx][DE] = delay;
847 						dram_info->rl_val[cs][idx][PE] = phase;
848 						/* Go to Final State */
849 						dram_info->rl_val[cs][idx][S]++;
850 						final_sum++;
851 						DEBUG_RL_FULL_C("DDR3 - Read Leveling - We finished the window for pup: ",
852 								(u32)pup, 1);
853 					}
854 
855 					/* Before the start of the window */
856 				} else if (dram_info->rl_val[cs][idx][S] ==
857 					   RL_UNLOCK_STATE) {
858 					/* Must be RL_UNLOCK_STATE */
859 					/*
860 					 * Match expected value ? - Update
861 					 * State Machine
862 					 */
863 					if (dram_info->rl_val[cs][idx][C] <
864 					    RL_RETRY_COUNT) {
865 						if (((~locked_pups >> pup) & 0x1)) {
866 							/* Match */
867 							DEBUG_RL_FULL_C("DDR3 - Read Leveling - We have no overrun and a match on pup: ",
868 									(u32)pup, 1);
869 							dram_info->rl_val[cs][idx][C]++;
870 
871 							/* If pup got to last state - lock the delays */
872 							if (dram_info->rl_val[cs][idx][C] ==
873 							    RL_RETRY_COUNT) {
874 								dram_info->rl_val[cs][idx][C] = 0;
875 								dram_info->rl_val[cs][idx][DS] =
876 									delay;
877 								dram_info->rl_val[cs][idx][PS] =
878 									phase;
879 								dram_info->rl_val[cs][idx][S]++;	/* Go to Window State */
880 								locked_sum++;
881 								/* Will count the pups that got locked */
882 
883 								/* IF First lock - need to lock delays */
884 								if (first_octet_locked == 0) {
885 									DEBUG_RL_FULL_C("DDR3 - Read Leveling - We got first lock on pup: ",
886 											(u32)pup, 1);
887 									first_octet_locked
888 									    =
889 									    1;
890 								}
891 							}
892 
893 							/* if pup is in not in final state but there was match - dont increment counter */
894 							else {
895 								counter_in_progress
896 								    = 1;
897 							}
898 						}
899 					}
900 				}
901 			} else {
902 				DEBUG_RL_FULL_C("DDR3 - Read Leveling - We got overrun on pup: ",
903 						(u32)pup, 1);
904 				counter_in_progress = 1;
905 			}
906 		}
907 
908 		if (final_sum == (dram_info->num_of_std_pups * (1 - ecc) + ecc)) {
909 			all_locked = 1;
910 			DEBUG_RL_FULL_S("DDR3 - Read Leveling - Single Cs - All pups locked\n");
911 		}
912 
913 		/*
914 		 * This is a fix for unstable condition where pups are
915 		 * toggling between match and no match
916 		 */
917 		/*
918 		 * If some of the pups is >1 <3, check if we did it too many
919 		 * times
920 		 */
921 		if (counter_in_progress == 1) {
922 			if (repeat_max_cnt < RL_RETRY_COUNT) {
923 				/* Notify at least one Counter is >=1 and < 3 */
924 				repeat_max_cnt++;
925 				counter_in_progress = 1;
926 				DEBUG_RL_FULL_S("DDR3 - Read Leveling - Counter is >=1 and <3\n");
927 				DEBUG_RL_FULL_S("DDR3 - Read Leveling - So we will not increment the delay to see if locked again\n");
928 			} else {
929 				DEBUG_RL_FULL_S("DDR3 - Read Leveling - repeat_max_cnt reached max so now we will increment the delay\n");
930 				counter_in_progress = 0;
931 			}
932 		}
933 
934 		/*
935 		 * Check some of the pups are in the middle of state machine
936 		 * and don't increment the delays
937 		 */
938 		if (!counter_in_progress && !all_locked) {
939 			repeat_max_cnt = 0;
940 			if (!ratio_2to1)
941 				ui_max_delay = MAX_DELAY_INV;
942 			else
943 				ui_max_delay = MAX_DELAY;
944 
945 			/* Increment Delay */
946 			if (delay < ui_max_delay) {
947 				/* Delay Incrementation */
948 				delay++;
949 				if (delay == ui_max_delay) {
950 					/*
951 					 * Mark the last delay/pahse place
952 					 * for window final place
953 					 */
954 					if ((!ratio_2to1
955 					     && phase == MAX_PHASE_RL_L_1TO1)
956 					    || (ratio_2to1
957 						&& phase ==
958 						MAX_PHASE_RL_L_2TO1))
959 						final_delay = 1;
960 				}
961 			} else {
962 				/* Phase+CL Incrementation */
963 				delay = 0;
964 				if (!ratio_2to1) {
965 					/* 1:1 mode */
966 					if (first_octet_locked) {
967 						/* some pupet was Locked */
968 						if (phase < MAX_PHASE_RL_L_1TO1) {
969 #ifdef RL_WINDOW_WA
970 							if (phase == 0)
971 #else
972 							if (phase == 1)
973 #endif
974 								phase = 4;
975 							else
976 								phase++;
977 						} else {
978 							DEBUG_RL_FULL_S("DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n");
979 							return MV_DDR3_TRAINING_ERR_RD_LVL_WIN_PUP_UNLOCK;
980 						}
981 					} else {
982 						/* No Pup was Locked */
983 						if (phase < MAX_PHASE_RL_UL_1TO1) {
984 #ifdef RL_WINDOW_WA
985 							if (phase == 0)
986 								phase = 4;
987 #else
988 							phase++;
989 #endif
990 						} else
991 							phase = 0;
992 					}
993 				} else {
994 					/* 2:1 mode */
995 					if (first_octet_locked) {
996 						/* Some Pup was Locked */
997 						if (phase < MAX_PHASE_RL_L_2TO1) {
998 							phase++;
999 						} else {
1000 							DEBUG_RL_FULL_S("DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n");
1001 							return MV_DDR3_TRAINING_ERR_RD_LVL_WIN_PUP_UNLOCK;
1002 						}
1003 					} else {
1004 						/* No Pup was Locked */
1005 						if (phase < MAX_PHASE_RL_UL_2TO1)
1006 							phase++;
1007 						else
1008 							phase = 0;
1009 					}
1010 				}
1011 
1012 				/*
1013 				 * If we finished a full Phases cycle (so
1014 				 * now phase = 0, need to increment
1015 				 * rd_sample_dly
1016 				 */
1017 				if (phase == 0 && first_octet_locked == 0) {
1018 					rd_sample_delay++;
1019 
1020 					/* Set current rd_sample_delay  */
1021 					reg = reg_read(REG_READ_DATA_SAMPLE_DELAYS_ADDR);
1022 					reg &= ~(REG_READ_DATA_SAMPLE_DELAYS_MASK <<
1023 						 (REG_READ_DATA_SAMPLE_DELAYS_OFFS
1024 						  * cs));
1025 					reg |= (rd_sample_delay <<
1026 						(REG_READ_DATA_SAMPLE_DELAYS_OFFS *
1027 						 cs));
1028 					reg_write(REG_READ_DATA_SAMPLE_DELAYS_ADDR,
1029 						  reg);
1030 				}
1031 
1032 				/*
1033 				 * Set current rdReadyDelay according to the
1034 				 * hash table (Need to do this in every phase
1035 				 * change)
1036 				 */
1037 				if (!ratio_2to1) {
1038 					/* 1:1 mode */
1039 					add = reg_read(REG_TRAINING_DEBUG_2_ADDR);
1040 					switch (phase) {
1041 					case 0:
1042 						add = add >>
1043 							REG_TRAINING_DEBUG_2_OFFS;
1044 						break;
1045 					case 1:
1046 						add = add >>
1047 							(REG_TRAINING_DEBUG_2_OFFS
1048 							 + 3);
1049 						break;
1050 					case 4:
1051 						add = add >>
1052 							(REG_TRAINING_DEBUG_2_OFFS
1053 							 + 6);
1054 						break;
1055 					case 5:
1056 						add = add >>
1057 							(REG_TRAINING_DEBUG_2_OFFS
1058 							 + 9);
1059 						break;
1060 					}
1061 				} else {
1062 					/* 2:1 mode */
1063 					add = reg_read(REG_TRAINING_DEBUG_3_ADDR);
1064 					add = (add >> phase *
1065 					       REG_TRAINING_DEBUG_3_OFFS);
1066 				}
1067 				add &= REG_TRAINING_DEBUG_2_MASK;
1068 				reg = reg_read(REG_READ_DATA_READY_DELAYS_ADDR);
1069 				reg &= ~(REG_READ_DATA_READY_DELAYS_MASK <<
1070 					 (REG_READ_DATA_READY_DELAYS_OFFS * cs));
1071 				reg |= ((rd_sample_delay + add) <<
1072 					(REG_READ_DATA_READY_DELAYS_OFFS * cs));
1073 				reg_write(REG_READ_DATA_READY_DELAYS_ADDR, reg);
1074 				dram_info->rd_smpl_dly = rd_sample_delay;
1075 				dram_info->rd_rdy_dly = rd_sample_delay + add;
1076 			}
1077 
1078 			/* Reset counters for pups with states<RD_STATE_COUNT */
1079 			for (pup = 0;
1080 			     pup <
1081 			     (dram_info->num_of_std_pups * (1 - ecc) + ecc);
1082 			     pup++) {
1083 				if (dram_info->rl_val[cs][idx][C] < RL_RETRY_COUNT)
1084 					dram_info->rl_val[cs][idx][C] = 0;
1085 			}
1086 		}
1087 	}
1088 
1089 	phase_min = 10;
1090 
1091 	for (pup = 0; pup < (dram_info->num_of_std_pups); pup++) {
1092 		DEBUG_RL_S("DDR3 - Read Leveling - Window info - PUP: ");
1093 		DEBUG_RL_D((u32) pup, 1);
1094 		DEBUG_RL_S(", PS: ");
1095 		DEBUG_RL_D((u32) dram_info->rl_val[cs][pup][PS], 1);
1096 		DEBUG_RL_S(", DS: ");
1097 		DEBUG_RL_D((u32) dram_info->rl_val[cs][pup][DS], 2);
1098 		DEBUG_RL_S(", PE: ");
1099 		DEBUG_RL_D((u32) dram_info->rl_val[cs][pup][PE], 1);
1100 		DEBUG_RL_S(", DE: ");
1101 		DEBUG_RL_D((u32) dram_info->rl_val[cs][pup][DE], 2);
1102 		DEBUG_RL_S("\n");
1103 	}
1104 
1105 	/* Find center of the window procedure */
1106 	for (pup = 0; pup < (dram_info->num_of_std_pups * (1 - ecc) + ecc);
1107 	     pup++) {
1108 #ifdef RL_WINDOW_WA
1109 		if (!ratio_2to1) {	/* 1:1 mode */
1110 			if (dram_info->rl_val[cs][idx][PS] == 4)
1111 				dram_info->rl_val[cs][idx][PS] = 1;
1112 			if (dram_info->rl_val[cs][idx][PE] == 4)
1113 				dram_info->rl_val[cs][idx][PE] = 1;
1114 
1115 			delay_s = dram_info->rl_val[cs][idx][PS] *
1116 				MAX_DELAY_INV + dram_info->rl_val[cs][idx][DS];
1117 			delay_e = dram_info->rl_val[cs][idx][PE] *
1118 				MAX_DELAY_INV + dram_info->rl_val[cs][idx][DE];
1119 
1120 			tmp = (delay_e - delay_s) / 2 + delay_s;
1121 			phase = tmp / MAX_DELAY_INV;
1122 			if (phase == 1)	/* 1:1 mode */
1123 				phase = 4;
1124 
1125 			if (phase < phase_min)	/* for the read ready delay */
1126 				phase_min = phase;
1127 
1128 			dram_info->rl_val[cs][idx][P] = phase;
1129 			dram_info->rl_val[cs][idx][D] = tmp % MAX_DELAY_INV;
1130 
1131 		} else {
1132 			delay_s = dram_info->rl_val[cs][idx][PS] *
1133 				MAX_DELAY + dram_info->rl_val[cs][idx][DS];
1134 			delay_e = dram_info->rl_val[cs][idx][PE] *
1135 				MAX_DELAY + dram_info->rl_val[cs][idx][DE];
1136 
1137 			tmp = (delay_e - delay_s) / 2 + delay_s;
1138 			phase = tmp / MAX_DELAY;
1139 
1140 			if (phase < phase_min)	/* for the read ready delay */
1141 				phase_min = phase;
1142 
1143 			dram_info->rl_val[cs][idx][P] = phase;
1144 			dram_info->rl_val[cs][idx][D] = tmp % MAX_DELAY;
1145 		}
1146 #else
1147 		if (!ratio_2to1) {	/* 1:1 mode */
1148 			if (dram_info->rl_val[cs][idx][PS] > 1)
1149 				dram_info->rl_val[cs][idx][PS] -= 2;
1150 			if (dram_info->rl_val[cs][idx][PE] > 1)
1151 				dram_info->rl_val[cs][idx][PE] -= 2;
1152 		}
1153 
1154 		delay_s = dram_info->rl_val[cs][idx][PS] * MAX_DELAY +
1155 			dram_info->rl_val[cs][idx][DS];
1156 		delay_e = dram_info->rl_val[cs][idx][PE] * MAX_DELAY +
1157 			dram_info->rl_val[cs][idx][DE];
1158 
1159 		tmp = (delay_e - delay_s) / 2 + delay_s;
1160 		phase = tmp / MAX_DELAY;
1161 		if (!ratio_2to1 && phase > 1)	/* 1:1 mode */
1162 			phase += 2;
1163 
1164 		if (phase < phase_min)	/* for the read ready delay */
1165 			phase_min = phase;
1166 
1167 		dram_info->rl_val[cs][idx][P] = phase;
1168 		dram_info->rl_val[cs][idx][D] = tmp % MAX_DELAY;
1169 #endif
1170 	}
1171 
1172 	/* Set current rdReadyDelay according to the hash table (Need to do this in every phase change) */
1173 	if (!ratio_2to1) {	/* 1:1 mode */
1174 		add = reg_read(REG_TRAINING_DEBUG_2_ADDR);
1175 		switch (phase_min) {
1176 		case 0:
1177 			add = (add >> REG_TRAINING_DEBUG_2_OFFS);
1178 			break;
1179 		case 1:
1180 			add = (add >> (REG_TRAINING_DEBUG_2_OFFS + 3));
1181 			break;
1182 		case 4:
1183 			add = (add >> (REG_TRAINING_DEBUG_2_OFFS + 6));
1184 			break;
1185 		case 5:
1186 			add = (add >> (REG_TRAINING_DEBUG_2_OFFS + 9));
1187 			break;
1188 		}
1189 	} else {		/* 2:1 mode */
1190 		add = reg_read(REG_TRAINING_DEBUG_3_ADDR);
1191 		add = (add >> phase_min * REG_TRAINING_DEBUG_3_OFFS);
1192 	}
1193 
1194 	add &= REG_TRAINING_DEBUG_2_MASK;
1195 	reg = reg_read(REG_READ_DATA_READY_DELAYS_ADDR);
1196 	reg &=
1197 	    ~(REG_READ_DATA_READY_DELAYS_MASK <<
1198 	      (REG_READ_DATA_READY_DELAYS_OFFS * cs));
1199 	reg |=
1200 	    ((rd_sample_delay + add) << (REG_READ_DATA_READY_DELAYS_OFFS * cs));
1201 	reg_write(REG_READ_DATA_READY_DELAYS_ADDR, reg);
1202 	dram_info->rd_rdy_dly = rd_sample_delay + add;
1203 
1204 	for (cs = 0; cs < dram_info->num_cs; cs++) {
1205 		for (pup = 0; pup < dram_info->num_of_total_pups; pup++) {
1206 			reg = ddr3_read_pup_reg(PUP_RL_MODE + 0x1, cs, pup);
1207 			dram_info->rl_val[cs][pup][DQS] = (reg & 0x3F);
1208 		}
1209 	}
1210 
1211 	return MV_OK;
1212 }
1213 #endif
1214