1 /*
2  * Copyright (C) Marvell International Ltd. and its affiliates
3  *
4  * SPDX-License-Identifier:	GPL-2.0
5  */
6 
7 #include <common.h>
8 #include <i2c.h>
9 #include <spl.h>
10 #include <asm/io.h>
11 #include <asm/arch/cpu.h>
12 #include <asm/arch/soc.h>
13 
14 #include "ddr3_hw_training.h"
15 
16 /*
17  * Debug
18  */
19 #define DEBUG_RL_C(s, d, l) \
20 	DEBUG_RL_S(s); DEBUG_RL_D(d, l); DEBUG_RL_S("\n")
21 #define DEBUG_RL_FULL_C(s, d, l) \
22 	DEBUG_RL_FULL_S(s); DEBUG_RL_FULL_D(d, l); DEBUG_RL_FULL_S("\n")
23 
24 #ifdef MV_DEBUG_RL
25 #define DEBUG_RL_S(s) \
26 	debug_cond(ddr3_get_log_level() >= MV_LOG_LEVEL_2, "%s", s)
27 #define DEBUG_RL_D(d, l) \
28 	debug_cond(ddr3_get_log_level() >= MV_LOG_LEVEL_2, "%x", d)
29 #else
30 #define DEBUG_RL_S(s)
31 #define DEBUG_RL_D(d, l)
32 #endif
33 
34 #ifdef MV_DEBUG_RL_FULL
35 #define DEBUG_RL_FULL_S(s)		puts(s)
36 #define DEBUG_RL_FULL_D(d, l)		printf("%x", d)
37 #else
38 #define DEBUG_RL_FULL_S(s)
39 #define DEBUG_RL_FULL_D(d, l)
40 #endif
41 
42 extern u32 rl_pattern[LEN_STD_PATTERN];
43 
44 #ifdef RL_MODE
45 static int ddr3_read_leveling_single_cs_rl_mode(u32 cs, u32 freq,
46 						int ratio_2to1, u32 ecc,
47 						MV_DRAM_INFO *dram_info);
48 #else
49 static int ddr3_read_leveling_single_cs_window_mode(u32 cs, u32 freq,
50 						    int ratio_2to1, u32 ecc,
51 						    MV_DRAM_INFO *dram_info);
52 #endif
53 
54 /*
55  * Name:     ddr3_read_leveling_hw
56  * Desc:     Execute the Read leveling phase by HW
57  * Args:     dram_info - main struct
58  *           freq      - current sequence frequency
59  * Notes:
60  * Returns:  MV_OK if success, MV_FAIL if fail.
61  */
62 int ddr3_read_leveling_hw(u32 freq, MV_DRAM_INFO *dram_info)
63 {
64 	u32 reg;
65 
66 	/* Debug message - Start Read leveling procedure */
67 	DEBUG_RL_S("DDR3 - Read Leveling - Starting HW RL procedure\n");
68 
69 	/* Start Auto Read Leveling procedure */
70 	reg = 1 << REG_DRAM_TRAINING_RL_OFFS;
71 	/* Config the retest number */
72 	reg |= (COUNT_HW_RL << REG_DRAM_TRAINING_RETEST_OFFS);
73 
74 	/* Enable CS in the automatic process */
75 	reg |= (dram_info->cs_ena << REG_DRAM_TRAINING_CS_OFFS);
76 
77 	reg_write(REG_DRAM_TRAINING_ADDR, reg);	/* 0x15B0 - Training Register */
78 
79 	reg = reg_read(REG_DRAM_TRAINING_SHADOW_ADDR) |
80 		(1 << REG_DRAM_TRAINING_AUTO_OFFS);
81 	reg_write(REG_DRAM_TRAINING_SHADOW_ADDR, reg);
82 
83 	/* Wait */
84 	do {
85 		reg = reg_read(REG_DRAM_TRAINING_SHADOW_ADDR) &
86 			(1 << REG_DRAM_TRAINING_AUTO_OFFS);
87 	} while (reg);		/* Wait for '0' */
88 
89 	/* Check if Successful */
90 	if (reg_read(REG_DRAM_TRAINING_SHADOW_ADDR) &
91 	    (1 << REG_DRAM_TRAINING_ERROR_OFFS)) {
92 		u32 delay, phase, pup, cs;
93 
94 		dram_info->rl_max_phase = 0;
95 		dram_info->rl_min_phase = 10;
96 
97 		/* Read results to arrays */
98 		for (cs = 0; cs < MAX_CS; cs++) {
99 			if (dram_info->cs_ena & (1 << cs)) {
100 				for (pup = 0;
101 				     pup < dram_info->num_of_total_pups;
102 				     pup++) {
103 					if (pup == dram_info->num_of_std_pups
104 					    && dram_info->ecc_ena)
105 						pup = ECC_PUP;
106 					reg =
107 					    ddr3_read_pup_reg(PUP_RL_MODE, cs,
108 							      pup);
109 					phase = (reg >> REG_PHY_PHASE_OFFS) &
110 						PUP_PHASE_MASK;
111 					delay = reg & PUP_DELAY_MASK;
112 					dram_info->rl_val[cs][pup][P] = phase;
113 					if (phase > dram_info->rl_max_phase)
114 						dram_info->rl_max_phase = phase;
115 					if (phase < dram_info->rl_min_phase)
116 						dram_info->rl_min_phase = phase;
117 					dram_info->rl_val[cs][pup][D] = delay;
118 					dram_info->rl_val[cs][pup][S] =
119 					    RL_FINAL_STATE;
120 					reg =
121 					    ddr3_read_pup_reg(PUP_RL_MODE + 0x1,
122 							      cs, pup);
123 					dram_info->rl_val[cs][pup][DQS] =
124 					    (reg & 0x3F);
125 				}
126 #ifdef MV_DEBUG_RL
127 				/* Print results */
128 				DEBUG_RL_C("DDR3 - Read Leveling - Results for CS - ",
129 					   (u32) cs, 1);
130 
131 				for (pup = 0;
132 				     pup < (dram_info->num_of_total_pups);
133 				     pup++) {
134 					if (pup == dram_info->num_of_std_pups
135 					    && dram_info->ecc_ena)
136 						pup = ECC_PUP;
137 					DEBUG_RL_S("DDR3 - Read Leveling - PUP: ");
138 					DEBUG_RL_D((u32) pup, 1);
139 					DEBUG_RL_S(", Phase: ");
140 					DEBUG_RL_D((u32) dram_info->
141 						   rl_val[cs][pup][P], 1);
142 					DEBUG_RL_S(", Delay: ");
143 					DEBUG_RL_D((u32) dram_info->
144 						   rl_val[cs][pup][D], 2);
145 					DEBUG_RL_S("\n");
146 				}
147 #endif
148 			}
149 		}
150 
151 		dram_info->rd_rdy_dly =
152 			reg_read(REG_READ_DATA_READY_DELAYS_ADDR) &
153 			REG_READ_DATA_SAMPLE_DELAYS_MASK;
154 		dram_info->rd_smpl_dly =
155 			reg_read(REG_READ_DATA_SAMPLE_DELAYS_ADDR) &
156 			REG_READ_DATA_READY_DELAYS_MASK;
157 #ifdef MV_DEBUG_RL
158 		DEBUG_RL_C("DDR3 - Read Leveling - Read Sample Delay: ",
159 			   dram_info->rd_smpl_dly, 2);
160 		DEBUG_RL_C("DDR3 - Read Leveling - Read Ready Delay: ",
161 			   dram_info->rd_rdy_dly, 2);
162 		DEBUG_RL_S("DDR3 - Read Leveling - HW RL Ended Successfully\n");
163 #endif
164 		return MV_OK;
165 
166 	} else {
167 		DEBUG_RL_S("DDR3 - Read Leveling - HW RL Error\n");
168 		return MV_FAIL;
169 	}
170 }
171 
172 /*
173  * Name:     ddr3_read_leveling_sw
174  * Desc:     Execute the Read leveling phase by SW
175  * Args:     dram_info - main struct
176  *           freq      - current sequence frequency
177  * Notes:
178  * Returns:  MV_OK if success, MV_FAIL if fail.
179  */
180 int ddr3_read_leveling_sw(u32 freq, int ratio_2to1, MV_DRAM_INFO *dram_info)
181 {
182 	u32 reg, cs, ecc, pup_num, phase, delay, pup;
183 	int status;
184 
185 	/* Debug message - Start Read leveling procedure */
186 	DEBUG_RL_S("DDR3 - Read Leveling - Starting SW RL procedure\n");
187 
188 	/* Enable SW Read Leveling */
189 	reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |
190 		(1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
191 	reg &= ~(1 << REG_DRAM_TRAINING_2_RL_MODE_OFFS);
192 	/* [0]=1 - Enable SW override  */
193 	/* 0x15B8 - Training SW 2 Register */
194 	reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
195 
196 #ifdef RL_MODE
197 	reg = (dram_info->cs_ena << REG_DRAM_TRAINING_CS_OFFS) |
198 		(1 << REG_DRAM_TRAINING_AUTO_OFFS);
199 	reg_write(REG_DRAM_TRAINING_ADDR, reg);	/* 0x15B0 - Training Register */
200 #endif
201 
202 	/* Loop for each CS */
203 	for (cs = 0; cs < dram_info->num_cs; cs++) {
204 		DEBUG_RL_C("DDR3 - Read Leveling - CS - ", (u32) cs, 1);
205 
206 		for (ecc = 0; ecc <= (dram_info->ecc_ena); ecc++) {
207 			/* ECC Support - Switch ECC Mux on ecc=1 */
208 			reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &
209 				~(1 << REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
210 			reg |= (dram_info->ecc_ena *
211 				ecc << REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
212 			reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
213 
214 			if (ecc)
215 				DEBUG_RL_S("DDR3 - Read Leveling - ECC Mux Enabled\n");
216 			else
217 				DEBUG_RL_S("DDR3 - Read Leveling - ECC Mux Disabled\n");
218 
219 			/* Set current sample delays */
220 			reg = reg_read(REG_READ_DATA_SAMPLE_DELAYS_ADDR);
221 			reg &= ~(REG_READ_DATA_SAMPLE_DELAYS_MASK <<
222 				 (REG_READ_DATA_SAMPLE_DELAYS_OFFS * cs));
223 			reg |= (dram_info->cl <<
224 				(REG_READ_DATA_SAMPLE_DELAYS_OFFS * cs));
225 			reg_write(REG_READ_DATA_SAMPLE_DELAYS_ADDR, reg);
226 
227 			/* Set current Ready delay */
228 			reg = reg_read(REG_READ_DATA_READY_DELAYS_ADDR);
229 			reg &= ~(REG_READ_DATA_READY_DELAYS_MASK <<
230 				 (REG_READ_DATA_READY_DELAYS_OFFS * cs));
231 			if (!ratio_2to1) {
232 				/* 1:1 mode */
233 				reg |= ((dram_info->cl + 1) <<
234 					(REG_READ_DATA_READY_DELAYS_OFFS * cs));
235 			} else {
236 				/* 2:1 mode */
237 				reg |= ((dram_info->cl + 2) <<
238 					(REG_READ_DATA_READY_DELAYS_OFFS * cs));
239 			}
240 			reg_write(REG_READ_DATA_READY_DELAYS_ADDR, reg);
241 
242 			/* Read leveling Single CS[cs] */
243 #ifdef RL_MODE
244 			status =
245 			    ddr3_read_leveling_single_cs_rl_mode(cs, freq,
246 								 ratio_2to1,
247 								 ecc,
248 								 dram_info);
249 			if (MV_OK != status)
250 				return status;
251 #else
252 			status =
253 			    ddr3_read_leveling_single_cs_window_mode(cs, freq,
254 								     ratio_2to1,
255 								     ecc,
256 								     dram_info)
257 			    if (MV_OK != status)
258 				return status;
259 #endif
260 		}
261 
262 		/* Print results */
263 		DEBUG_RL_C("DDR3 - Read Leveling - Results for CS - ", (u32) cs,
264 			   1);
265 
266 		for (pup = 0;
267 		     pup < (dram_info->num_of_std_pups + dram_info->ecc_ena);
268 		     pup++) {
269 			DEBUG_RL_S("DDR3 - Read Leveling - PUP: ");
270 			DEBUG_RL_D((u32) pup, 1);
271 			DEBUG_RL_S(", Phase: ");
272 			DEBUG_RL_D((u32) dram_info->rl_val[cs][pup][P], 1);
273 			DEBUG_RL_S(", Delay: ");
274 			DEBUG_RL_D((u32) dram_info->rl_val[cs][pup][D], 2);
275 			DEBUG_RL_S("\n");
276 		}
277 
278 		DEBUG_RL_C("DDR3 - Read Leveling - Read Sample Delay: ",
279 			   dram_info->rd_smpl_dly, 2);
280 		DEBUG_RL_C("DDR3 - Read Leveling - Read Ready Delay: ",
281 			   dram_info->rd_rdy_dly, 2);
282 
283 		/* Configure PHY with average of 3 locked leveling settings */
284 		for (pup = 0;
285 		     pup < (dram_info->num_of_std_pups + dram_info->ecc_ena);
286 		     pup++) {
287 			/* ECC support - bit 8 */
288 			pup_num = (pup == dram_info->num_of_std_pups) ? ECC_BIT : pup;
289 
290 			/* For now, set last cnt result */
291 			phase = dram_info->rl_val[cs][pup][P];
292 			delay = dram_info->rl_val[cs][pup][D];
293 			ddr3_write_pup_reg(PUP_RL_MODE, cs, pup_num, phase,
294 					   delay);
295 		}
296 	}
297 
298 	/* Reset PHY read FIFO */
299 	reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |
300 		(1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS);
301 	/* 0x15B8 - Training SW 2 Register */
302 	reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
303 
304 	do {
305 		reg = (reg_read(REG_DRAM_TRAINING_2_ADDR)) &
306 			(1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS);
307 	} while (reg);		/* Wait for '0' */
308 
309 	/* ECC Support - Switch ECC Mux off ecc=0 */
310 	reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &
311 		~(1 << REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
312 	reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
313 
314 #ifdef RL_MODE
315 	reg_write(REG_DRAM_TRAINING_ADDR, 0);	/* 0x15B0 - Training Register */
316 #endif
317 
318 	/* Disable SW Read Leveling */
319 	reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &
320 		~(1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
321 	/* [0] = 0 - Disable SW override  */
322 	reg = (reg | (0x1 << REG_DRAM_TRAINING_2_RL_MODE_OFFS));
323 	/* [3] = 1 - Disable RL MODE */
324 	/* 0x15B8 - Training SW 2 Register */
325 	reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
326 
327 	DEBUG_RL_S("DDR3 - Read Leveling - Finished RL procedure for all CS\n");
328 	return MV_OK;
329 }
330 
331 #ifdef RL_MODE
332 /*
333  * overrun() extracted from ddr3_read_leveling_single_cs_rl_mode().
334  * This just got too much indented making it hard to read / edit.
335  */
336 static void overrun(u32 cs, MV_DRAM_INFO *info, u32 pup, u32 locked_pups,
337 		    u32 *locked_sum, u32 ecc, int *first_octet_locked,
338 		    int *counter_in_progress, int final_delay, u32 delay,
339 		    u32 phase)
340 {
341 	/* If no OverRun */
342 	if (((~locked_pups >> pup) & 0x1) && (final_delay == 0)) {
343 		int idx;
344 
345 		idx = pup + ecc * ECC_BIT;
346 
347 		/* PUP passed, start examining */
348 		if (info->rl_val[cs][idx][S] == RL_UNLOCK_STATE) {
349 			/* Must be RL_UNLOCK_STATE */
350 			/* Match expected value ? - Update State Machine */
351 			if (info->rl_val[cs][idx][C] < RL_RETRY_COUNT) {
352 				DEBUG_RL_FULL_C("DDR3 - Read Leveling - We have no overrun and a match on pup: ",
353 						(u32)pup, 1);
354 				info->rl_val[cs][idx][C]++;
355 
356 				/* If pup got to last state - lock the delays */
357 				if (info->rl_val[cs][idx][C] == RL_RETRY_COUNT) {
358 					info->rl_val[cs][idx][C] = 0;
359 					info->rl_val[cs][idx][DS] = delay;
360 					info->rl_val[cs][idx][PS] = phase;
361 
362 					/* Go to Final State */
363 					info->rl_val[cs][idx][S] = RL_FINAL_STATE;
364 					*locked_sum = *locked_sum + 1;
365 					DEBUG_RL_FULL_C("DDR3 - Read Leveling - We have locked pup: ",
366 							(u32)pup, 1);
367 
368 					/*
369 					 * If first lock - need to lock delays
370 					 */
371 					if (*first_octet_locked == 0) {
372 						DEBUG_RL_FULL_C("DDR3 - Read Leveling - We got first lock on pup: ",
373 								(u32)pup, 1);
374 						*first_octet_locked = 1;
375 					}
376 
377 					/*
378 					 * If pup is in not in final state but
379 					 * there was match - dont increment
380 					 * counter
381 					 */
382 				} else {
383 					*counter_in_progress = 1;
384 				}
385 			}
386 		}
387 	}
388 }
389 
390 /*
391  * Name:     ddr3_read_leveling_single_cs_rl_mode
392  * Desc:     Execute Read leveling for single Chip select
393  * Args:     cs        - current chip select
394  *           freq      - current sequence frequency
395  *           ecc       - ecc iteration indication
396  *           dram_info - main struct
397  * Notes:
398  * Returns:  MV_OK if success, MV_FAIL if fail.
399  */
400 static int ddr3_read_leveling_single_cs_rl_mode(u32 cs, u32 freq,
401 						int ratio_2to1, u32 ecc,
402 						MV_DRAM_INFO *dram_info)
403 {
404 	u32 reg, delay, phase, pup, rd_sample_delay, add, locked_pups,
405 		repeat_max_cnt, sdram_offset, locked_sum;
406 	u32 phase_min, ui_max_delay;
407 	int all_locked, first_octet_locked, counter_in_progress;
408 	int final_delay = 0;
409 
410 	DEBUG_RL_FULL_C("DDR3 - Read Leveling - Single CS - ", (u32) cs, 1);
411 
412 	/* Init values */
413 	phase = 0;
414 	delay = 0;
415 	rd_sample_delay = dram_info->cl;
416 	all_locked = 0;
417 	first_octet_locked = 0;
418 	repeat_max_cnt = 0;
419 	locked_sum = 0;
420 
421 	for (pup = 0; pup < (dram_info->num_of_std_pups * (1 - ecc) + ecc);
422 	     pup++)
423 		dram_info->rl_val[cs][pup + ecc * ECC_BIT][S] = 0;
424 
425 	/* Main loop */
426 	while (!all_locked) {
427 		counter_in_progress = 0;
428 
429 		DEBUG_RL_FULL_S("DDR3 - Read Leveling - RdSmplDly = ");
430 		DEBUG_RL_FULL_D(rd_sample_delay, 2);
431 		DEBUG_RL_FULL_S(", RdRdyDly = ");
432 		DEBUG_RL_FULL_D(dram_info->rd_rdy_dly, 2);
433 		DEBUG_RL_FULL_S(", Phase = ");
434 		DEBUG_RL_FULL_D(phase, 1);
435 		DEBUG_RL_FULL_S(", Delay = ");
436 		DEBUG_RL_FULL_D(delay, 2);
437 		DEBUG_RL_FULL_S("\n");
438 
439 		/*
440 		 * Broadcast to all PUPs current RL delays: DQS phase,
441 		 * leveling delay
442 		 */
443 		ddr3_write_pup_reg(PUP_RL_MODE, cs, PUP_BC, phase, delay);
444 
445 		/* Reset PHY read FIFO */
446 		reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |
447 			(1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS);
448 		/* 0x15B8 - Training SW 2 Register */
449 		reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
450 
451 		do {
452 			reg = (reg_read(REG_DRAM_TRAINING_2_ADDR)) &
453 				(1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS);
454 		} while (reg);	/* Wait for '0' */
455 
456 		/* Read pattern from SDRAM */
457 		sdram_offset = cs * (SDRAM_CS_SIZE + 1) + SDRAM_RL_OFFS;
458 		locked_pups = 0;
459 		if (MV_OK !=
460 		    ddr3_sdram_compare(dram_info, 0xFF, &locked_pups,
461 				       rl_pattern, LEN_STD_PATTERN,
462 				       sdram_offset, 0, 0, NULL, 0))
463 			return MV_DDR3_TRAINING_ERR_RD_LVL_RL_PATTERN;
464 
465 		/* Octet evaluation */
466 		/* pup_num = Q or 1 for ECC */
467 		for (pup = 0; pup < (dram_info->num_of_std_pups * (1 - ecc) + ecc); pup++) {
468 			/* Check Overrun */
469 			if (!((reg_read(REG_DRAM_TRAINING_2_ADDR) >>
470 			       (REG_DRAM_TRAINING_2_OVERRUN_OFFS + pup)) & 0x1)) {
471 				overrun(cs, dram_info, pup, locked_pups,
472 					&locked_sum, ecc, &first_octet_locked,
473 					&counter_in_progress, final_delay,
474 					delay, phase);
475 			} else {
476 				DEBUG_RL_FULL_C("DDR3 - Read Leveling - We got overrun on pup: ",
477 						(u32)pup, 1);
478 			}
479 		}
480 
481 		if (locked_sum == (dram_info->num_of_std_pups *
482 				   (1 - ecc) + ecc)) {
483 			all_locked = 1;
484 			DEBUG_RL_FULL_S("DDR3 - Read Leveling - Single Cs - All pups locked\n");
485 		}
486 
487 		/*
488 		 * This is a fix for unstable condition where pups are
489 		 * toggling between match and no match
490 		 */
491 		/*
492 		 * If some of the pups is >1 <3, check if we did it too
493 		 * many times
494 		 */
495 		if (counter_in_progress == 1) {
496 			/* Notify at least one Counter is >=1 and < 3 */
497 			if (repeat_max_cnt < RL_RETRY_COUNT) {
498 				repeat_max_cnt++;
499 				counter_in_progress = 1;
500 				DEBUG_RL_FULL_S("DDR3 - Read Leveling - Counter is >=1 and <3\n");
501 				DEBUG_RL_FULL_S("DDR3 - Read Leveling - So we will not increment the delay to see if locked again\n");
502 			} else {
503 				DEBUG_RL_FULL_S("DDR3 - Read Leveling - repeat_max_cnt reached max so now we will increment the delay\n");
504 				counter_in_progress = 0;
505 			}
506 		}
507 
508 		/*
509 		 * Check some of the pups are in the middle of state machine
510 		 * and don't increment the delays
511 		 */
512 		if (!counter_in_progress && !all_locked) {
513 			int idx;
514 
515 			idx = pup + ecc * ECC_BIT;
516 
517 			repeat_max_cnt = 0;
518 			/* if 1:1 mode */
519 			if ((!ratio_2to1) && ((phase == 0) || (phase == 4)))
520 				ui_max_delay = MAX_DELAY_INV;
521 			else
522 				ui_max_delay = MAX_DELAY;
523 
524 			/* Increment Delay */
525 			if (delay < ui_max_delay) {
526 				delay++;
527 				/*
528 				 * Mark the last delay/pahse place for
529 				 * window final place
530 				 */
531 				if (delay == ui_max_delay) {
532 					if ((!ratio_2to1 && phase ==
533 					     MAX_PHASE_RL_L_1TO1)
534 					    || (ratio_2to1 && phase ==
535 						MAX_PHASE_RL_L_2TO1))
536 						final_delay = 1;
537 				}
538 			} else {
539 				/* Phase+CL Incrementation */
540 				delay = 0;
541 
542 				if (!ratio_2to1) {
543 					/* 1:1 mode */
544 					if (first_octet_locked) {
545 						/* some Pup was Locked */
546 						if (phase < MAX_PHASE_RL_L_1TO1) {
547 							if (phase == 1) {
548 								phase = 4;
549 							} else {
550 								phase++;
551 								delay = MIN_DELAY_PHASE_1_LIMIT;
552 							}
553 						} else {
554 							DEBUG_RL_FULL_S("DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n");
555 							DEBUG_RL_S("1)DDR3 - Read Leveling - ERROR - NOT all PUPs Locked n");
556 							return MV_DDR3_TRAINING_ERR_RD_LVL_RL_PUP_UNLOCK;
557 						}
558 					} else {
559 						/* NO Pup was Locked */
560 						if (phase < MAX_PHASE_RL_UL_1TO1) {
561 							phase++;
562 							delay =
563 							    MIN_DELAY_PHASE_1_LIMIT;
564 						} else {
565 							phase = 0;
566 						}
567 					}
568 				} else {
569 					/* 2:1 mode */
570 					if (first_octet_locked) {
571 						/* some Pup was Locked */
572 						if (phase < MAX_PHASE_RL_L_2TO1) {
573 							phase++;
574 						} else {
575 							DEBUG_RL_FULL_S("DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n");
576 							DEBUG_RL_S("2)DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n");
577 							for (pup = 0; pup < (dram_info->num_of_std_pups * (1 - ecc) + ecc); pup++) {
578 								/* pup_num = Q or 1 for ECC */
579 								if (dram_info->rl_val[cs][idx][S]
580 								    == 0) {
581 									DEBUG_RL_C("Failed byte is = ",
582 										   pup, 1);
583 								}
584 							}
585 							return MV_DDR3_TRAINING_ERR_RD_LVL_RL_PUP_UNLOCK;
586 						}
587 					} else {
588 						/* No Pup was Locked */
589 						if (phase < MAX_PHASE_RL_UL_2TO1)
590 							phase++;
591 						else
592 							phase = 0;
593 					}
594 				}
595 
596 				/*
597 				 * If we finished a full Phases cycle (so now
598 				 * phase = 0, need to increment rd_sample_dly
599 				 */
600 				if (phase == 0 && first_octet_locked == 0) {
601 					rd_sample_delay++;
602 					if (rd_sample_delay == 0x10) {
603 						DEBUG_RL_FULL_S("DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n");
604 						DEBUG_RL_S("3)DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n");
605 						for (pup = 0; pup < (dram_info->num_of_std_pups * (1 - ecc) + ecc); pup++) {
606 							/* pup_num = Q or 1 for ECC */
607 							if (dram_info->
608 							    rl_val[cs][idx][S] == 0) {
609 								DEBUG_RL_C("Failed byte is = ",
610 									   pup, 1);
611 							}
612 						}
613 						return MV_DDR3_TRAINING_ERR_RD_LVL_PUP_UNLOCK;
614 					}
615 
616 					/* Set current rd_sample_delay  */
617 					reg = reg_read(REG_READ_DATA_SAMPLE_DELAYS_ADDR);
618 					reg &= ~(REG_READ_DATA_SAMPLE_DELAYS_MASK
619 					      << (REG_READ_DATA_SAMPLE_DELAYS_OFFS
620 						  * cs));
621 					reg |= (rd_sample_delay <<
622 						(REG_READ_DATA_SAMPLE_DELAYS_OFFS *
623 						 cs));
624 					reg_write(REG_READ_DATA_SAMPLE_DELAYS_ADDR,
625 						  reg);
626 				}
627 
628 				/*
629 				 * Set current rdReadyDelay according to the
630 				 * hash table (Need to do this in every phase
631 				 * change)
632 				 */
633 				if (!ratio_2to1) {
634 					/* 1:1 mode */
635 					add = reg_read(REG_TRAINING_DEBUG_2_ADDR);
636 					switch (phase) {
637 					case 0:
638 						add = (add >>
639 						       REG_TRAINING_DEBUG_2_OFFS);
640 						break;
641 					case 1:
642 						add = (add >>
643 						       (REG_TRAINING_DEBUG_2_OFFS
644 							+ 3));
645 						break;
646 					case 4:
647 						add = (add >>
648 						       (REG_TRAINING_DEBUG_2_OFFS
649 							+ 6));
650 						break;
651 					case 5:
652 						add = (add >>
653 						       (REG_TRAINING_DEBUG_2_OFFS
654 							+ 9));
655 						break;
656 					}
657 					add &= REG_TRAINING_DEBUG_2_MASK;
658 				} else {
659 					/* 2:1 mode */
660 					add = reg_read(REG_TRAINING_DEBUG_3_ADDR);
661 					add = (add >>
662 					       (phase *
663 						REG_TRAINING_DEBUG_3_OFFS));
664 					add &= REG_TRAINING_DEBUG_3_MASK;
665 				}
666 
667 				reg = reg_read(REG_READ_DATA_READY_DELAYS_ADDR);
668 				reg &= ~(REG_READ_DATA_READY_DELAYS_MASK <<
669 					 (REG_READ_DATA_READY_DELAYS_OFFS * cs));
670 				reg |= ((rd_sample_delay + add) <<
671 					(REG_READ_DATA_READY_DELAYS_OFFS * cs));
672 				reg_write(REG_READ_DATA_READY_DELAYS_ADDR, reg);
673 				dram_info->rd_smpl_dly = rd_sample_delay;
674 				dram_info->rd_rdy_dly = rd_sample_delay + add;
675 			}
676 
677 			/* Reset counters for pups with states<RD_STATE_COUNT */
678 			for (pup = 0; pup <
679 				     (dram_info->num_of_std_pups * (1 - ecc) + ecc);
680 			     pup++) {
681 				if (dram_info->rl_val[cs][idx][C] < RL_RETRY_COUNT)
682 					dram_info->rl_val[cs][idx][C] = 0;
683 			}
684 		}
685 	}
686 
687 	phase_min = 10;
688 
689 	for (pup = 0; pup < (dram_info->num_of_std_pups); pup++) {
690 		if (dram_info->rl_val[cs][pup][PS] < phase_min)
691 			phase_min = dram_info->rl_val[cs][pup][PS];
692 	}
693 
694 	/*
695 	 * Set current rdReadyDelay according to the hash table (Need to
696 	 * do this in every phase change)
697 	 */
698 	if (!ratio_2to1) {
699 		/* 1:1 mode */
700 		add = reg_read(REG_TRAINING_DEBUG_2_ADDR);
701 		switch (phase_min) {
702 		case 0:
703 			add = (add >> REG_TRAINING_DEBUG_2_OFFS);
704 			break;
705 		case 1:
706 			add = (add >> (REG_TRAINING_DEBUG_2_OFFS + 3));
707 			break;
708 		case 4:
709 			add = (add >> (REG_TRAINING_DEBUG_2_OFFS + 6));
710 			break;
711 		case 5:
712 			add = (add >> (REG_TRAINING_DEBUG_2_OFFS + 9));
713 			break;
714 		}
715 		add &= REG_TRAINING_DEBUG_2_MASK;
716 	} else {
717 		/* 2:1 mode */
718 		add = reg_read(REG_TRAINING_DEBUG_3_ADDR);
719 		add = (add >> (phase_min * REG_TRAINING_DEBUG_3_OFFS));
720 		add &= REG_TRAINING_DEBUG_3_MASK;
721 	}
722 
723 	reg = reg_read(REG_READ_DATA_READY_DELAYS_ADDR);
724 	reg &= ~(REG_READ_DATA_READY_DELAYS_MASK <<
725 		 (REG_READ_DATA_READY_DELAYS_OFFS * cs));
726 	reg |= ((rd_sample_delay + add) << (REG_READ_DATA_READY_DELAYS_OFFS * cs));
727 	reg_write(REG_READ_DATA_READY_DELAYS_ADDR, reg);
728 	dram_info->rd_rdy_dly = rd_sample_delay + add;
729 
730 	for (cs = 0; cs < dram_info->num_cs; cs++) {
731 		for (pup = 0; pup < dram_info->num_of_total_pups; pup++) {
732 			reg = ddr3_read_pup_reg(PUP_RL_MODE + 0x1, cs, pup);
733 			dram_info->rl_val[cs][pup][DQS] = (reg & 0x3F);
734 		}
735 	}
736 
737 	return MV_OK;
738 }
739 
740 #else
741 
742 /*
743  * Name:     ddr3_read_leveling_single_cs_window_mode
744  * Desc:     Execute Read leveling for single Chip select
745  * Args:     cs        - current chip select
746  *           freq      - current sequence frequency
747  *           ecc       - ecc iteration indication
748  *           dram_info - main struct
749  * Notes:
750  * Returns:  MV_OK if success, MV_FAIL if fail.
751  */
752 static int ddr3_read_leveling_single_cs_window_mode(u32 cs, u32 freq,
753 						    int ratio_2to1, u32 ecc,
754 						    MV_DRAM_INFO *dram_info)
755 {
756 	u32 reg, delay, phase, sum, pup, rd_sample_delay, add, locked_pups,
757 	    repeat_max_cnt, sdram_offset, final_sum, locked_sum;
758 	u32 delay_s, delay_e, tmp, phase_min, ui_max_delay;
759 	int all_locked, first_octet_locked, counter_in_progress;
760 	int final_delay = 0;
761 
762 	DEBUG_RL_FULL_C("DDR3 - Read Leveling - Single CS - ", (u32) cs, 1);
763 
764 	/* Init values */
765 	phase = 0;
766 	delay = 0;
767 	rd_sample_delay = dram_info->cl;
768 	all_locked = 0;
769 	first_octet_locked = 0;
770 	repeat_max_cnt = 0;
771 	sum = 0;
772 	final_sum = 0;
773 	locked_sum = 0;
774 
775 	for (pup = 0; pup < (dram_info->num_of_std_pups * (1 - ecc) + ecc);
776 	     pup++)
777 		dram_info->rl_val[cs][pup + ecc * ECC_BIT][S] = 0;
778 
779 	/* Main loop */
780 	while (!all_locked) {
781 		counter_in_progress = 0;
782 
783 		DEBUG_RL_FULL_S("DDR3 - Read Leveling - RdSmplDly = ");
784 		DEBUG_RL_FULL_D(rd_sample_delay, 2);
785 		DEBUG_RL_FULL_S(", RdRdyDly = ");
786 		DEBUG_RL_FULL_D(dram_info->rd_rdy_dly, 2);
787 		DEBUG_RL_FULL_S(", Phase = ");
788 		DEBUG_RL_FULL_D(phase, 1);
789 		DEBUG_RL_FULL_S(", Delay = ");
790 		DEBUG_RL_FULL_D(delay, 2);
791 		DEBUG_RL_FULL_S("\n");
792 
793 		/*
794 		 * Broadcast to all PUPs current RL delays: DQS phase,leveling
795 		 * delay
796 		 */
797 		ddr3_write_pup_reg(PUP_RL_MODE, cs, PUP_BC, phase, delay);
798 
799 		/* Reset PHY read FIFO */
800 		reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |
801 			(1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS);
802 		/* 0x15B8 - Training SW 2 Register */
803 		reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
804 
805 		do {
806 			reg = (reg_read(REG_DRAM_TRAINING_2_ADDR)) &
807 				(1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS);
808 		} while (reg);	/* Wait for '0' */
809 
810 		/* Read pattern from SDRAM */
811 		sdram_offset = cs * (SDRAM_CS_SIZE + 1) + SDRAM_RL_OFFS;
812 		locked_pups = 0;
813 		if (MV_OK !=
814 		    ddr3_sdram_compare(dram_info, 0xFF, &locked_pups,
815 				       rl_pattern, LEN_STD_PATTERN,
816 				       sdram_offset, 0, 0, NULL, 0))
817 			return MV_DDR3_TRAINING_ERR_RD_LVL_WIN_PATTERN;
818 
819 		/* Octet evaluation */
820 		for (pup = 0; pup < (dram_info->num_of_std_pups *
821 				     (1 - ecc) + ecc); pup++) {
822 			/* pup_num = Q or 1 for ECC */
823 			int idx;
824 
825 			idx = pup + ecc * ECC_BIT;
826 
827 			/* Check Overrun */
828 			if (!((reg_read(REG_DRAM_TRAINING_2_ADDR) >>
829 			      (REG_DRAM_TRAINING_2_OVERRUN_OFFS +
830 			       pup)) & 0x1)) {
831 				/* If no OverRun */
832 
833 				/* Inside the window */
834 				if (dram_info->rl_val[cs][idx][S] == RL_WINDOW_STATE) {
835 					/*
836 					 * Match expected value ? - Update
837 					 * State Machine
838 					 */
839 					if (((~locked_pups >> pup) & 0x1)
840 					    && (final_delay == 0)) {
841 						/* Match - Still inside the Window */
842 						DEBUG_RL_FULL_C("DDR3 - Read Leveling - We got another match inside the window  for pup: ",
843 								(u32)pup, 1);
844 
845 					} else {
846 						/* We got fail -> this is the end of the window */
847 						dram_info->rl_val[cs][idx][DE] = delay;
848 						dram_info->rl_val[cs][idx][PE] = phase;
849 						/* Go to Final State */
850 						dram_info->rl_val[cs][idx][S]++;
851 						final_sum++;
852 						DEBUG_RL_FULL_C("DDR3 - Read Leveling - We finished the window for pup: ",
853 								(u32)pup, 1);
854 					}
855 
856 					/* Before the start of the window */
857 				} else if (dram_info->rl_val[cs][idx][S] ==
858 					   RL_UNLOCK_STATE) {
859 					/* Must be RL_UNLOCK_STATE */
860 					/*
861 					 * Match expected value ? - Update
862 					 * State Machine
863 					 */
864 					if (dram_info->rl_val[cs][idx][C] <
865 					    RL_RETRY_COUNT) {
866 						if (((~locked_pups >> pup) & 0x1)) {
867 							/* Match */
868 							DEBUG_RL_FULL_C("DDR3 - Read Leveling - We have no overrun and a match on pup: ",
869 									(u32)pup, 1);
870 							dram_info->rl_val[cs][idx][C]++;
871 
872 							/* If pup got to last state - lock the delays */
873 							if (dram_info->rl_val[cs][idx][C] ==
874 							    RL_RETRY_COUNT) {
875 								dram_info->rl_val[cs][idx][C] = 0;
876 								dram_info->rl_val[cs][idx][DS] =
877 									delay;
878 								dram_info->rl_val[cs][idx][PS] =
879 									phase;
880 								dram_info->rl_val[cs][idx][S]++;	/* Go to Window State */
881 								locked_sum++;
882 								/* Will count the pups that got locked */
883 
884 								/* IF First lock - need to lock delays */
885 								if (first_octet_locked == 0) {
886 									DEBUG_RL_FULL_C("DDR3 - Read Leveling - We got first lock on pup: ",
887 											(u32)pup, 1);
888 									first_octet_locked
889 									    =
890 									    1;
891 								}
892 							}
893 
894 							/* if pup is in not in final state but there was match - dont increment counter */
895 							else {
896 								counter_in_progress
897 								    = 1;
898 							}
899 						}
900 					}
901 				}
902 			} else {
903 				DEBUG_RL_FULL_C("DDR3 - Read Leveling - We got overrun on pup: ",
904 						(u32)pup, 1);
905 				counter_in_progress = 1;
906 			}
907 		}
908 
909 		if (final_sum == (dram_info->num_of_std_pups * (1 - ecc) + ecc)) {
910 			all_locked = 1;
911 			DEBUG_RL_FULL_S("DDR3 - Read Leveling - Single Cs - All pups locked\n");
912 		}
913 
914 		/*
915 		 * This is a fix for unstable condition where pups are
916 		 * toggling between match and no match
917 		 */
918 		/*
919 		 * If some of the pups is >1 <3, check if we did it too many
920 		 * times
921 		 */
922 		if (counter_in_progress == 1) {
923 			if (repeat_max_cnt < RL_RETRY_COUNT) {
924 				/* Notify at least one Counter is >=1 and < 3 */
925 				repeat_max_cnt++;
926 				counter_in_progress = 1;
927 				DEBUG_RL_FULL_S("DDR3 - Read Leveling - Counter is >=1 and <3\n");
928 				DEBUG_RL_FULL_S("DDR3 - Read Leveling - So we will not increment the delay to see if locked again\n");
929 			} else {
930 				DEBUG_RL_FULL_S("DDR3 - Read Leveling - repeat_max_cnt reached max so now we will increment the delay\n");
931 				counter_in_progress = 0;
932 			}
933 		}
934 
935 		/*
936 		 * Check some of the pups are in the middle of state machine
937 		 * and don't increment the delays
938 		 */
939 		if (!counter_in_progress && !all_locked) {
940 			repeat_max_cnt = 0;
941 			if (!ratio_2to1)
942 				ui_max_delay = MAX_DELAY_INV;
943 			else
944 				ui_max_delay = MAX_DELAY;
945 
946 			/* Increment Delay */
947 			if (delay < ui_max_delay) {
948 				/* Delay Incrementation */
949 				delay++;
950 				if (delay == ui_max_delay) {
951 					/*
952 					 * Mark the last delay/pahse place
953 					 * for window final place
954 					 */
955 					if ((!ratio_2to1
956 					     && phase == MAX_PHASE_RL_L_1TO1)
957 					    || (ratio_2to1
958 						&& phase ==
959 						MAX_PHASE_RL_L_2TO1))
960 						final_delay = 1;
961 				}
962 			} else {
963 				/* Phase+CL Incrementation */
964 				delay = 0;
965 				if (!ratio_2to1) {
966 					/* 1:1 mode */
967 					if (first_octet_locked) {
968 						/* some pupet was Locked */
969 						if (phase < MAX_PHASE_RL_L_1TO1) {
970 #ifdef RL_WINDOW_WA
971 							if (phase == 0)
972 #else
973 							if (phase == 1)
974 #endif
975 								phase = 4;
976 							else
977 								phase++;
978 						} else {
979 							DEBUG_RL_FULL_S("DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n");
980 							return MV_DDR3_TRAINING_ERR_RD_LVL_WIN_PUP_UNLOCK;
981 						}
982 					} else {
983 						/* No Pup was Locked */
984 						if (phase < MAX_PHASE_RL_UL_1TO1) {
985 #ifdef RL_WINDOW_WA
986 							if (phase == 0)
987 								phase = 4;
988 #else
989 							phase++;
990 #endif
991 						} else
992 							phase = 0;
993 					}
994 				} else {
995 					/* 2:1 mode */
996 					if (first_octet_locked) {
997 						/* Some Pup was Locked */
998 						if (phase < MAX_PHASE_RL_L_2TO1) {
999 							phase++;
1000 						} else {
1001 							DEBUG_RL_FULL_S("DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n");
1002 							return MV_DDR3_TRAINING_ERR_RD_LVL_WIN_PUP_UNLOCK;
1003 						}
1004 					} else {
1005 						/* No Pup was Locked */
1006 						if (phase < MAX_PHASE_RL_UL_2TO1)
1007 							phase++;
1008 						else
1009 							phase = 0;
1010 					}
1011 				}
1012 
1013 				/*
1014 				 * If we finished a full Phases cycle (so
1015 				 * now phase = 0, need to increment
1016 				 * rd_sample_dly
1017 				 */
1018 				if (phase == 0 && first_octet_locked == 0) {
1019 					rd_sample_delay++;
1020 
1021 					/* Set current rd_sample_delay  */
1022 					reg = reg_read(REG_READ_DATA_SAMPLE_DELAYS_ADDR);
1023 					reg &= ~(REG_READ_DATA_SAMPLE_DELAYS_MASK <<
1024 						 (REG_READ_DATA_SAMPLE_DELAYS_OFFS
1025 						  * cs));
1026 					reg |= (rd_sample_delay <<
1027 						(REG_READ_DATA_SAMPLE_DELAYS_OFFS *
1028 						 cs));
1029 					reg_write(REG_READ_DATA_SAMPLE_DELAYS_ADDR,
1030 						  reg);
1031 				}
1032 
1033 				/*
1034 				 * Set current rdReadyDelay according to the
1035 				 * hash table (Need to do this in every phase
1036 				 * change)
1037 				 */
1038 				if (!ratio_2to1) {
1039 					/* 1:1 mode */
1040 					add = reg_read(REG_TRAINING_DEBUG_2_ADDR);
1041 					switch (phase) {
1042 					case 0:
1043 						add = add >>
1044 							REG_TRAINING_DEBUG_2_OFFS;
1045 						break;
1046 					case 1:
1047 						add = add >>
1048 							(REG_TRAINING_DEBUG_2_OFFS
1049 							 + 3);
1050 						break;
1051 					case 4:
1052 						add = add >>
1053 							(REG_TRAINING_DEBUG_2_OFFS
1054 							 + 6);
1055 						break;
1056 					case 5:
1057 						add = add >>
1058 							(REG_TRAINING_DEBUG_2_OFFS
1059 							 + 9);
1060 						break;
1061 					}
1062 				} else {
1063 					/* 2:1 mode */
1064 					add = reg_read(REG_TRAINING_DEBUG_3_ADDR);
1065 					add = (add >> phase *
1066 					       REG_TRAINING_DEBUG_3_OFFS);
1067 				}
1068 				add &= REG_TRAINING_DEBUG_2_MASK;
1069 				reg = reg_read(REG_READ_DATA_READY_DELAYS_ADDR);
1070 				reg &= ~(REG_READ_DATA_READY_DELAYS_MASK <<
1071 					 (REG_READ_DATA_READY_DELAYS_OFFS * cs));
1072 				reg |= ((rd_sample_delay + add) <<
1073 					(REG_READ_DATA_READY_DELAYS_OFFS * cs));
1074 				reg_write(REG_READ_DATA_READY_DELAYS_ADDR, reg);
1075 				dram_info->rd_smpl_dly = rd_sample_delay;
1076 				dram_info->rd_rdy_dly = rd_sample_delay + add;
1077 			}
1078 
1079 			/* Reset counters for pups with states<RD_STATE_COUNT */
1080 			for (pup = 0;
1081 			     pup <
1082 			     (dram_info->num_of_std_pups * (1 - ecc) + ecc);
1083 			     pup++) {
1084 				if (dram_info->rl_val[cs][idx][C] < RL_RETRY_COUNT)
1085 					dram_info->rl_val[cs][idx][C] = 0;
1086 			}
1087 		}
1088 	}
1089 
1090 	phase_min = 10;
1091 
1092 	for (pup = 0; pup < (dram_info->num_of_std_pups); pup++) {
1093 		DEBUG_RL_S("DDR3 - Read Leveling - Window info - PUP: ");
1094 		DEBUG_RL_D((u32) pup, 1);
1095 		DEBUG_RL_S(", PS: ");
1096 		DEBUG_RL_D((u32) dram_info->rl_val[cs][pup][PS], 1);
1097 		DEBUG_RL_S(", DS: ");
1098 		DEBUG_RL_D((u32) dram_info->rl_val[cs][pup][DS], 2);
1099 		DEBUG_RL_S(", PE: ");
1100 		DEBUG_RL_D((u32) dram_info->rl_val[cs][pup][PE], 1);
1101 		DEBUG_RL_S(", DE: ");
1102 		DEBUG_RL_D((u32) dram_info->rl_val[cs][pup][DE], 2);
1103 		DEBUG_RL_S("\n");
1104 	}
1105 
1106 	/* Find center of the window procedure */
1107 	for (pup = 0; pup < (dram_info->num_of_std_pups * (1 - ecc) + ecc);
1108 	     pup++) {
1109 #ifdef RL_WINDOW_WA
1110 		if (!ratio_2to1) {	/* 1:1 mode */
1111 			if (dram_info->rl_val[cs][idx][PS] == 4)
1112 				dram_info->rl_val[cs][idx][PS] = 1;
1113 			if (dram_info->rl_val[cs][idx][PE] == 4)
1114 				dram_info->rl_val[cs][idx][PE] = 1;
1115 
1116 			delay_s = dram_info->rl_val[cs][idx][PS] *
1117 				MAX_DELAY_INV + dram_info->rl_val[cs][idx][DS];
1118 			delay_e = dram_info->rl_val[cs][idx][PE] *
1119 				MAX_DELAY_INV + dram_info->rl_val[cs][idx][DE];
1120 
1121 			tmp = (delay_e - delay_s) / 2 + delay_s;
1122 			phase = tmp / MAX_DELAY_INV;
1123 			if (phase == 1)	/* 1:1 mode */
1124 				phase = 4;
1125 
1126 			if (phase < phase_min)	/* for the read ready delay */
1127 				phase_min = phase;
1128 
1129 			dram_info->rl_val[cs][idx][P] = phase;
1130 			dram_info->rl_val[cs][idx][D] = tmp % MAX_DELAY_INV;
1131 
1132 		} else {
1133 			delay_s = dram_info->rl_val[cs][idx][PS] *
1134 				MAX_DELAY + dram_info->rl_val[cs][idx][DS];
1135 			delay_e = dram_info->rl_val[cs][idx][PE] *
1136 				MAX_DELAY + dram_info->rl_val[cs][idx][DE];
1137 
1138 			tmp = (delay_e - delay_s) / 2 + delay_s;
1139 			phase = tmp / MAX_DELAY;
1140 
1141 			if (phase < phase_min)	/* for the read ready delay */
1142 				phase_min = phase;
1143 
1144 			dram_info->rl_val[cs][idx][P] = phase;
1145 			dram_info->rl_val[cs][idx][D] = tmp % MAX_DELAY;
1146 		}
1147 #else
1148 		if (!ratio_2to1) {	/* 1:1 mode */
1149 			if (dram_info->rl_val[cs][idx][PS] > 1)
1150 				dram_info->rl_val[cs][idx][PS] -= 2;
1151 			if (dram_info->rl_val[cs][idx][PE] > 1)
1152 				dram_info->rl_val[cs][idx][PE] -= 2;
1153 		}
1154 
1155 		delay_s = dram_info->rl_val[cs][idx][PS] * MAX_DELAY +
1156 			dram_info->rl_val[cs][idx][DS];
1157 		delay_e = dram_info->rl_val[cs][idx][PE] * MAX_DELAY +
1158 			dram_info->rl_val[cs][idx][DE];
1159 
1160 		tmp = (delay_e - delay_s) / 2 + delay_s;
1161 		phase = tmp / MAX_DELAY;
1162 		if (!ratio_2to1 && phase > 1)	/* 1:1 mode */
1163 			phase += 2;
1164 
1165 		if (phase < phase_min)	/* for the read ready delay */
1166 			phase_min = phase;
1167 
1168 		dram_info->rl_val[cs][idx][P] = phase;
1169 		dram_info->rl_val[cs][idx][D] = tmp % MAX_DELAY;
1170 #endif
1171 	}
1172 
1173 	/* Set current rdReadyDelay according to the hash table (Need to do this in every phase change) */
1174 	if (!ratio_2to1) {	/* 1:1 mode */
1175 		add = reg_read(REG_TRAINING_DEBUG_2_ADDR);
1176 		switch (phase_min) {
1177 		case 0:
1178 			add = (add >> REG_TRAINING_DEBUG_2_OFFS);
1179 			break;
1180 		case 1:
1181 			add = (add >> (REG_TRAINING_DEBUG_2_OFFS + 3));
1182 			break;
1183 		case 4:
1184 			add = (add >> (REG_TRAINING_DEBUG_2_OFFS + 6));
1185 			break;
1186 		case 5:
1187 			add = (add >> (REG_TRAINING_DEBUG_2_OFFS + 9));
1188 			break;
1189 		}
1190 	} else {		/* 2:1 mode */
1191 		add = reg_read(REG_TRAINING_DEBUG_3_ADDR);
1192 		add = (add >> phase_min * REG_TRAINING_DEBUG_3_OFFS);
1193 	}
1194 
1195 	add &= REG_TRAINING_DEBUG_2_MASK;
1196 	reg = reg_read(REG_READ_DATA_READY_DELAYS_ADDR);
1197 	reg &=
1198 	    ~(REG_READ_DATA_READY_DELAYS_MASK <<
1199 	      (REG_READ_DATA_READY_DELAYS_OFFS * cs));
1200 	reg |=
1201 	    ((rd_sample_delay + add) << (REG_READ_DATA_READY_DELAYS_OFFS * cs));
1202 	reg_write(REG_READ_DATA_READY_DELAYS_ADDR, reg);
1203 	dram_info->rd_rdy_dly = rd_sample_delay + add;
1204 
1205 	for (cs = 0; cs < dram_info->num_cs; cs++) {
1206 		for (pup = 0; pup < dram_info->num_of_total_pups; pup++) {
1207 			reg = ddr3_read_pup_reg(PUP_RL_MODE + 0x1, cs, pup);
1208 			dram_info->rl_val[cs][pup][DQS] = (reg & 0x3F);
1209 		}
1210 	}
1211 
1212 	return MV_OK;
1213 }
1214 #endif
1215