1 /*
2  * Copyright (C) Marvell International Ltd. and its affiliates
3  *
4  * SPDX-License-Identifier:	GPL-2.0
5  */
6 
7 #include <common.h>
8 #include <i2c.h>
9 #include <spl.h>
10 #include <asm/io.h>
11 #include <asm/arch/cpu.h>
12 #include <asm/arch/soc.h>
13 
14 #include "ddr3_hw_training.h"
15 
16 /*
17  * Debug
18  */
19 #define DEBUG_DQS_C(s, d, l) \
20 	DEBUG_DQS_S(s); DEBUG_DQS_D(d, l); DEBUG_DQS_S("\n")
21 #define DEBUG_DQS_FULL_C(s, d, l) \
22 	DEBUG_DQS_FULL_S(s); DEBUG_DQS_FULL_D(d, l); DEBUG_DQS_FULL_S("\n")
23 #define DEBUG_DQS_RESULTS_C(s, d, l) \
24 	DEBUG_DQS_RESULTS_S(s); DEBUG_DQS_RESULTS_D(d, l); DEBUG_DQS_RESULTS_S("\n")
25 #define DEBUG_PER_DQ_C(s, d, l) \
26 	puts(s); printf("%x", d); puts("\n")
27 
28 #define DEBUG_DQS_RESULTS_S(s) \
29 	debug_cond(ddr3_get_log_level() >= MV_LOG_LEVEL_2, "%s", s)
30 #define DEBUG_DQS_RESULTS_D(d, l) \
31 	debug_cond(ddr3_get_log_level() >= MV_LOG_LEVEL_2, "%x", d)
32 
33 #define DEBUG_PER_DQ_S(s) \
34 	debug_cond(ddr3_get_log_level() >= MV_LOG_LEVEL_3, "%s", s)
35 #define DEBUG_PER_DQ_D(d, l) \
36 	debug_cond(ddr3_get_log_level() >= MV_LOG_LEVEL_3, "%x", d)
37 #define DEBUG_PER_DQ_DD(d, l) \
38 	debug_cond(ddr3_get_log_level() >= MV_LOG_LEVEL_3, "%d", d)
39 
40 #ifdef MV_DEBUG_DQS
41 #define DEBUG_DQS_S(s)			puts(s)
42 #define DEBUG_DQS_D(d, l)		printf("%x", d)
43 #else
44 #define DEBUG_DQS_S(s)
45 #define DEBUG_DQS_D(d, l)
46 #endif
47 
48 #ifdef MV_DEBUG_DQS_FULL
49 #define DEBUG_DQS_FULL_S(s)		puts(s)
50 #define DEBUG_DQS_FULL_D(d, l)		printf("%x", d)
51 #else
52 #define DEBUG_DQS_FULL_S(s)
53 #define DEBUG_DQS_FULL_D(d, l)
54 #endif
55 
56 /* State machine for centralization - find low & high limit */
57 enum {
58 	PUP_ADLL_LIMITS_STATE_FAIL,
59 	PUP_ADLL_LIMITS_STATE_PASS,
60 	PUP_ADLL_LIMITS_STATE_FAIL_AFTER_PASS,
61 };
62 
63 /* Hold centralization low results */
64 static int centralization_low_limit[MAX_PUP_NUM] = { 0 };
65 /* Hold centralization high results */
66 static int centralization_high_limit[MAX_PUP_NUM] = { 0 };
67 
68 int ddr3_find_adll_limits(MV_DRAM_INFO *dram_info, u32 cs, u32 ecc, int is_tx);
69 int ddr3_check_window_limits(u32 pup, int high_limit, int low_limit, int is_tx,
70 			  int *size_valid);
71 static int ddr3_center_calc(MV_DRAM_INFO *dram_info, u32 cs, u32 ecc,
72 			    int is_tx);
73 int ddr3_special_pattern_i_search(MV_DRAM_INFO *dram_info, u32 cs, u32 ecc,
74 			      int is_tx, u32 special_pattern_pup);
75 int ddr3_special_pattern_ii_search(MV_DRAM_INFO *dram_info, u32 cs, u32 ecc,
76 				   int is_tx, u32 special_pattern_pup);
77 int ddr3_set_dqs_centralization_results(MV_DRAM_INFO *dram_info, u32 cs, u32 ecc,
78 				    int is_tx);
79 
80 #ifdef MV88F78X60
81 extern u32 killer_pattern_32b[DQ_NUM][LEN_SPECIAL_PATTERN];
82 extern u32 killer_pattern_64b[DQ_NUM][LEN_SPECIAL_PATTERN];
83 extern int per_bit_data[MAX_PUP_NUM][DQ_NUM];
84 #else
85 extern u32 killer_pattern[DQ_NUM][LEN_16BIT_KILLER_PATTERN];
86 extern u32 killer_pattern_32b[DQ_NUM][LEN_SPECIAL_PATTERN];
87 #if defined(MV88F672X)
88 extern int per_bit_data[MAX_PUP_NUM][DQ_NUM];
89 #endif
90 #endif
91 extern u32 special_pattern[DQ_NUM][LEN_SPECIAL_PATTERN];
92 
93 static u32 *ddr3_dqs_choose_pattern(MV_DRAM_INFO *dram_info, u32 victim_dq)
94 {
95 	u32 *pattern_ptr;
96 
97 	/* Choose pattern */
98 	switch (dram_info->ddr_width) {
99 #if defined(MV88F672X)
100 	case 16:
101 		pattern_ptr = (u32 *)&killer_pattern[victim_dq];
102 		break;
103 #endif
104 	case 32:
105 		pattern_ptr = (u32 *)&killer_pattern_32b[victim_dq];
106 		break;
107 #if defined(MV88F78X60)
108 	case 64:
109 		pattern_ptr = (u32 *)&killer_pattern_64b[victim_dq];
110 		break;
111 #endif
112 	default:
113 #if defined(MV88F78X60)
114 		pattern_ptr = (u32 *)&killer_pattern_32b[victim_dq];
115 #else
116 		pattern_ptr = (u32 *)&killer_pattern[victim_dq];
117 #endif
118 		break;
119 	}
120 
121 	return pattern_ptr;
122 }
123 
124 /*
125  * Name:     ddr3_dqs_centralization_rx
126  * Desc:     Execute the DQS centralization RX phase.
127  * Args:     dram_info
128  * Notes:
129  * Returns:  MV_OK if success, other error code if fail.
130  */
131 int ddr3_dqs_centralization_rx(MV_DRAM_INFO *dram_info)
132 {
133 	u32 cs, ecc, reg;
134 	int status;
135 
136 	DEBUG_DQS_S("DDR3 - DQS Centralization RX - Starting procedure\n");
137 
138 	/* Enable SW override */
139 	reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |
140 		(1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
141 
142 	/* [0] = 1 - Enable SW override  */
143 	/* 0x15B8 - Training SW 2 Register */
144 	reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
145 	DEBUG_DQS_S("DDR3 - DQS Centralization RX - SW Override Enabled\n");
146 
147 	reg = (1 << REG_DRAM_TRAINING_AUTO_OFFS);
148 	reg_write(REG_DRAM_TRAINING_ADDR, reg);	/* 0x15B0 - Training Register */
149 
150 	/* Loop for each CS */
151 	for (cs = 0; cs < MAX_CS; cs++) {
152 		if (dram_info->cs_ena & (1 << cs)) {
153 			DEBUG_DQS_FULL_C("DDR3 - DQS Centralization RX - CS - ",
154 					 (u32) cs, 1);
155 
156 			for (ecc = 0; ecc < (dram_info->ecc_ena + 1); ecc++) {
157 
158 				/* ECC Support - Switch ECC Mux on ecc=1 */
159 				reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &
160 					~(1 << REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
161 				reg |= (dram_info->ecc_ena *
162 					ecc << REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
163 				reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
164 
165 				if (ecc)
166 					DEBUG_DQS_FULL_S("DDR3 - DQS Centralization RX - ECC Mux Enabled\n");
167 				else
168 					DEBUG_DQS_FULL_S("DDR3 - DQS Centralization RX - ECC Mux Disabled\n");
169 
170 				DEBUG_DQS_FULL_S("DDR3 - DQS Centralization RX - Find all limits\n");
171 
172 				status = ddr3_find_adll_limits(dram_info, cs,
173 							       ecc, 0);
174 				if (MV_OK != status)
175 					return status;
176 
177 				DEBUG_DQS_FULL_S("DDR3 - DQS Centralization RX - Start calculating center\n");
178 
179 				status = ddr3_center_calc(dram_info, cs, ecc,
180 							  0);
181 				if (MV_OK != status)
182 					return status;
183 			}
184 		}
185 	}
186 
187 	/* ECC Support - Disable ECC MUX */
188 	reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &
189 		~(1 << REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
190 	reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
191 
192 	/* Disable SW override - Must be in a different stage */
193 	/* [0]=0 - Enable SW override  */
194 	reg = reg_read(REG_DRAM_TRAINING_2_ADDR);
195 	reg &= ~(1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
196 	/* 0x15B8 - Training SW 2 Register */
197 	reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
198 
199 	reg = reg_read(REG_DRAM_TRAINING_1_ADDR) |
200 		(1 << REG_DRAM_TRAINING_1_TRNBPOINT_OFFS);
201 	reg_write(REG_DRAM_TRAINING_1_ADDR, reg);
202 
203 	return MV_OK;
204 }
205 
206 /*
207  * Name:     ddr3_dqs_centralization_tx
208  * Desc:     Execute the DQS centralization TX phase.
209  * Args:     dram_info
210  * Notes:
211  * Returns:  MV_OK if success, other error code if fail.
212  */
213 int ddr3_dqs_centralization_tx(MV_DRAM_INFO *dram_info)
214 {
215 	u32 cs, ecc, reg;
216 	int status;
217 
218 	DEBUG_DQS_S("DDR3 - DQS Centralization TX - Starting procedure\n");
219 
220 	/* Enable SW override */
221 	reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |
222 		(1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
223 
224 	/* [0] = 1 - Enable SW override  */
225 	/* 0x15B8 - Training SW 2 Register */
226 	reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
227 	DEBUG_DQS_S("DDR3 - DQS Centralization TX - SW Override Enabled\n");
228 
229 	reg = (1 << REG_DRAM_TRAINING_AUTO_OFFS);
230 	reg_write(REG_DRAM_TRAINING_ADDR, reg);	/* 0x15B0 - Training Register */
231 
232 	/* Loop for each CS */
233 	for (cs = 0; cs < MAX_CS; cs++) {
234 		if (dram_info->cs_ena & (1 << cs)) {
235 			DEBUG_DQS_FULL_C("DDR3 - DQS Centralization TX - CS - ",
236 					 (u32) cs, 1);
237 			for (ecc = 0; ecc < (dram_info->ecc_ena + 1); ecc++) {
238 				/* ECC Support - Switch ECC Mux on ecc=1 */
239 				reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &
240 					~(1 << REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
241 				reg |= (dram_info->ecc_ena *
242 					ecc << REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
243 				reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
244 
245 				if (ecc)
246 					DEBUG_DQS_FULL_S("DDR3 - DQS Centralization TX - ECC Mux Enabled\n");
247 				else
248 					DEBUG_DQS_FULL_S("DDR3 - DQS Centralization TX - ECC Mux Disabled\n");
249 
250 				DEBUG_DQS_FULL_S("DDR3 - DQS Centralization TX - Find all limits\n");
251 
252 				status = ddr3_find_adll_limits(dram_info, cs,
253 							       ecc, 1);
254 				if (MV_OK != status)
255 					return status;
256 
257 				DEBUG_DQS_FULL_S("DDR3 - DQS Centralization TX - Start calculating center\n");
258 
259 				status = ddr3_center_calc(dram_info, cs, ecc,
260 							  1);
261 				if (MV_OK != status)
262 					return status;
263 			}
264 		}
265 	}
266 
267 	/* ECC Support - Disable ECC MUX */
268 	reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &
269 		~(1 << REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
270 	reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
271 
272 	/* Disable SW override - Must be in a different stage */
273 	/* [0]=0 - Enable SW override  */
274 	reg = reg_read(REG_DRAM_TRAINING_2_ADDR);
275 	reg &= ~(1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
276 	/* 0x15B8 - Training SW 2 Register */
277 	reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
278 
279 	reg = reg_read(REG_DRAM_TRAINING_1_ADDR) |
280 		(1 << REG_DRAM_TRAINING_1_TRNBPOINT_OFFS);
281 	reg_write(REG_DRAM_TRAINING_1_ADDR, reg);
282 
283 	return MV_OK;
284 }
285 
286 /*
287  * Name:     ddr3_find_adll_limits
288  * Desc:     Execute the Find ADLL limits phase.
289  * Args:     dram_info
290  *           cs
291  *           ecc_ena
292  *           is_tx             Indicate whether Rx or Tx
293  * Notes:
294  * Returns:  MV_OK if success, other error code if fail.
295  */
296 int ddr3_find_adll_limits(MV_DRAM_INFO *dram_info, u32 cs, u32 ecc, int is_tx)
297 {
298 	u32 victim_dq, pup, tmp;
299 	u32 adll_addr;
300 	u32 max_pup;		/* maximal pup index */
301 	u32 pup_mask = 0;
302 	u32 unlock_pup;		/* bit array of un locked pups */
303 	u32 new_unlock_pup;	/* bit array of compare failed pups */
304 	u32 curr_adll;
305 	u32 adll_start_val;	/* adll start loop value - for rx or tx limit */
306 	u32 high_limit;	/* holds found High Limit */
307 	u32 low_limit;		/* holds found Low Limit */
308 	int win_valid;
309 	int update_win;
310 	u32 sdram_offset;
311 	u32 uj, cs_count, cs_tmp, ii;
312 	u32 *pattern_ptr;
313 	u32 dq;
314 	u32 adll_end_val;	/* adll end of loop val - for rx or tx limit */
315 	u8 analog_pbs[DQ_NUM][MAX_PUP_NUM][DQ_NUM][2];
316 	u8 analog_pbs_sum[MAX_PUP_NUM][DQ_NUM][2];
317 	int pup_adll_limit_state[MAX_PUP_NUM];	/* hold state of each pup */
318 
319 	adll_addr = ((is_tx == 1) ? PUP_DQS_WR : PUP_DQS_RD);
320 	adll_end_val = ((is_tx == 1) ? ADLL_MIN : ADLL_MAX);
321 	adll_start_val = ((is_tx == 1) ? ADLL_MAX : ADLL_MIN);
322 	max_pup = (ecc + (1 - ecc) * dram_info->num_of_std_pups);
323 
324 	DEBUG_DQS_FULL_S("DDR3 - DQS Find Limits - Starting Find ADLL Limits\n");
325 
326 	/* init the array */
327 	for (pup = 0; pup < max_pup; pup++) {
328 		centralization_low_limit[pup] = ADLL_MIN;
329 		centralization_high_limit[pup] = ADLL_MAX;
330 	}
331 
332 	/* Killer Pattern */
333 	cs_count = 0;
334 	for (cs_tmp = 0; cs_tmp < cs; cs_tmp++) {
335 		if (dram_info->cs_ena & (1 << cs_tmp))
336 			cs_count++;
337 	}
338 	sdram_offset = cs_count * (SDRAM_CS_SIZE + 1);
339 	sdram_offset += ((is_tx == 1) ?
340 			 SDRAM_DQS_TX_OFFS : SDRAM_DQS_RX_OFFS);
341 
342 	/* Prepare pup masks */
343 	for (pup = 0; pup < max_pup; pup++)
344 		pup_mask |= (1 << pup);
345 
346 	for (pup = 0; pup < max_pup; pup++) {
347 		for (dq = 0; dq < DQ_NUM; dq++) {
348 			analog_pbs_sum[pup][dq][0] = adll_start_val;
349 			analog_pbs_sum[pup][dq][1] = adll_end_val;
350 		}
351 	}
352 
353 	/* Loop - use different pattern for each victim_dq */
354 	for (victim_dq = 0; victim_dq < DQ_NUM; victim_dq++) {
355 		DEBUG_DQS_FULL_C("DDR3 - DQS Find Limits - Victim DQ - ",
356 				 (u32)victim_dq, 1);
357 		/*
358 		 * The pups 3 bit arrays represent state machine. with
359 		 * 3 stages for each pup.
360 		 * 1. fail and didn't get pass in earlier compares.
361 		 * 2. pass compare
362 		 * 3. fail after pass - end state.
363 		 * The window limits are the adll values where the adll
364 		 * was in the pass stage.
365 		 */
366 
367 		/* Set all states to Fail (1st state) */
368 		for (pup = 0; pup < max_pup; pup++)
369 			pup_adll_limit_state[pup] = PUP_ADLL_LIMITS_STATE_FAIL;
370 
371 		/* Set current valid pups */
372 		unlock_pup = pup_mask;
373 
374 		/* Set ADLL to start value */
375 		curr_adll = adll_start_val;
376 
377 #if defined(MV88F78X60)
378 		for (pup = 0; pup < max_pup; pup++) {
379 			for (dq = 0; dq < DQ_NUM; dq++) {
380 				analog_pbs[victim_dq][pup][dq][0] =
381 					adll_start_val;
382 				analog_pbs[victim_dq][pup][dq][1] =
383 					adll_end_val;
384 				per_bit_data[pup][dq] = 0;
385 			}
386 		}
387 #endif
388 
389 		for (uj = 0; uj < ADLL_MAX; uj++) {
390 			DEBUG_DQS_FULL_C("DDR3 - DQS Find Limits - Setting ADLL to ",
391 					 curr_adll, 2);
392 			for (pup = 0; pup < max_pup; pup++) {
393 				if (IS_PUP_ACTIVE(unlock_pup, pup) == 1) {
394 					tmp = ((is_tx == 1) ? curr_adll +
395 					       dram_info->wl_val[cs]
396 					       [pup * (1 - ecc) + ecc * ECC_PUP]
397 					       [D] : curr_adll);
398 					ddr3_write_pup_reg(adll_addr, cs, pup +
399 						(ecc * ECC_PUP), 0, tmp);
400 				}
401 			}
402 
403 			/* Choose pattern */
404 			pattern_ptr = ddr3_dqs_choose_pattern(dram_info,
405 							      victim_dq);
406 
407 			/* '1' - means pup failed, '0' - means pup pass */
408 			new_unlock_pup = 0;
409 
410 			/* Read and compare results for Victim_DQ# */
411 			for (ii = 0; ii < 3; ii++) {
412 				u32 tmp = 0;
413 				if (MV_OK != ddr3_sdram_dqs_compare(dram_info,
414 							   unlock_pup, &tmp,
415 							   pattern_ptr,
416 							   LEN_KILLER_PATTERN,
417 							   sdram_offset +
418 							   LEN_KILLER_PATTERN *
419 							   4 * victim_dq,
420 							   is_tx, 0, NULL,
421 							   0))
422 					return MV_DDR3_TRAINING_ERR_DRAM_COMPARE;
423 
424 				new_unlock_pup |= tmp;
425 			}
426 
427 			pup = 0;
428 			DEBUG_DQS_FULL_C("DDR3 - DQS Find Limits - UnlockPup: ",
429 					 unlock_pup, 2);
430 			DEBUG_DQS_FULL_C("DDR3 - DQS Find Limits - NewUnlockPup: ",
431 					 new_unlock_pup, 2);
432 
433 			/* Update pup state */
434 			for (pup = 0; pup < max_pup; pup++) {
435 				if (IS_PUP_ACTIVE(unlock_pup, pup) == 0) {
436 					DEBUG_DQS_FULL_C("DDR3 - DQS Find Limits - Skipping pup ",
437 							 pup, 1);
438 					continue;
439 				}
440 
441 				/*
442 				 * Still didn't find the window limit of the pup
443 				 */
444 				if (IS_PUP_ACTIVE(new_unlock_pup, pup) == 1) {
445 					/* Current compare result == fail */
446 					if (pup_adll_limit_state[pup] ==
447 					    PUP_ADLL_LIMITS_STATE_PASS) {
448 						/*
449 						 * If now it failed but passed
450 						 * earlier
451 						 */
452 						DEBUG_DQS_S("DDR3 - DQS Find Limits - PASS to FAIL: CS - ");
453 						DEBUG_DQS_D(cs, 1);
454 						DEBUG_DQS_S(", DQ - ");
455 						DEBUG_DQS_D(victim_dq, 1);
456 						DEBUG_DQS_S(", Pup - ");
457 						DEBUG_DQS_D(pup, 1);
458 						DEBUG_DQS_S(", ADLL - ");
459 						DEBUG_DQS_D(curr_adll, 2);
460 						DEBUG_DQS_S("\n");
461 
462 #if defined(MV88F78X60)
463 						for (dq = 0; dq < DQ_NUM; dq++) {
464 							if ((analog_pbs[victim_dq][pup][dq][0] != adll_start_val)
465 							    && (analog_pbs[victim_dq][pup]
466 								[dq][1] == adll_end_val))
467 								analog_pbs
468 									[victim_dq]
469 									[pup][dq]
470 									[1] =
471 									curr_adll;
472 						}
473 #endif
474 						win_valid = 1;
475 						update_win = 0;
476 
477 						/* Keep min / max limit value */
478 						if (is_tx == 0) {
479 							/* RX - found upper limit */
480 							if (centralization_high_limit[pup] >
481 							    (curr_adll - 1)) {
482 								high_limit =
483 									curr_adll - 1;
484 								low_limit =
485 									centralization_low_limit[pup];
486 								update_win = 1;
487 							}
488 						} else {
489 							/* TX - found lower limit */
490 							if (centralization_low_limit[pup] < (curr_adll + 1)) {
491 								high_limit =
492 									centralization_high_limit
493 									[pup];
494 								low_limit =
495 									curr_adll + 1;
496 								update_win =
497 									1;
498 							}
499 						}
500 
501 						if (update_win == 1) {
502 							/*
503 							 * Before updating
504 							 * window limits we need
505 							 * to check that the
506 							 * limits are valid
507 							 */
508 							if (MV_OK !=
509 							    ddr3_check_window_limits
510 							    (pup, high_limit,
511 							     low_limit, is_tx,
512 							     &win_valid))
513 								return MV_DDR3_TRAINING_ERR_WIN_LIMITS;
514 
515 							if (win_valid == 1) {
516 								/*
517 								 * Window limits
518 								 * should be
519 								 * updated
520 								 */
521 								centralization_low_limit
522 									[pup] =
523 									low_limit;
524 								centralization_high_limit
525 									[pup] =
526 									high_limit;
527 							}
528 						}
529 
530 						if (win_valid == 1) {
531 							/* Found end of window - lock the pup */
532 							pup_adll_limit_state[pup] =
533 								PUP_ADLL_LIMITS_STATE_FAIL_AFTER_PASS;
534 							unlock_pup &= ~(1 << pup);
535 						} else {
536 							/* Probably false pass - reset status */
537 							pup_adll_limit_state[pup] =
538 								PUP_ADLL_LIMITS_STATE_FAIL;
539 
540 #if defined(MV88F78X60)
541 							/* Clear logging array of win size (per Dq) */
542 							for (dq = 0;
543 							     dq < DQ_NUM;
544 							     dq++) {
545 								analog_pbs
546 									[victim_dq]
547 									[pup][dq]
548 									[0] =
549 									adll_start_val;
550 								analog_pbs
551 									[victim_dq]
552 									[pup][dq]
553 									[1] =
554 									adll_end_val;
555 								per_bit_data
556 									[pup][dq]
557 									= 0;
558 							}
559 #endif
560 						}
561 					}
562 				} else {
563 					/* Current compare result == pass */
564 					if (pup_adll_limit_state[pup] ==
565 					    PUP_ADLL_LIMITS_STATE_FAIL) {
566 						/* If now it passed but failed earlier */
567 						DEBUG_DQS_S("DDR3 - DQS Find Limits - FAIL to PASS: CS - ");
568 						DEBUG_DQS_D(cs, 1);
569 						DEBUG_DQS_S(", DQ - ");
570 						DEBUG_DQS_D(victim_dq, 1);
571 						DEBUG_DQS_S(", Pup - ");
572 						DEBUG_DQS_D(pup, 1);
573 						DEBUG_DQS_S(", ADLL - ");
574 						DEBUG_DQS_D(curr_adll, 2);
575 						DEBUG_DQS_S("\n");
576 
577 #if defined(MV88F78X60)
578 						for (dq = 0; dq < DQ_NUM;
579 						     dq++) {
580 							if (analog_pbs[victim_dq][pup][dq][0] == adll_start_val)
581 								analog_pbs
582 								    [victim_dq]
583 								    [pup][dq]
584 								    [0] =
585 								    curr_adll;
586 						}
587 #endif
588 						/* Found start of window */
589 						pup_adll_limit_state[pup] =
590 						    PUP_ADLL_LIMITS_STATE_PASS;
591 
592 						/* Keep min / max limit value */
593 						if (is_tx == 0) {
594 							/* RX - found low limit */
595 							if (centralization_low_limit[pup] <= curr_adll)
596 								centralization_low_limit
597 								    [pup] =
598 								    curr_adll;
599 						} else {
600 							/* TX - found high limit */
601 							if (centralization_high_limit[pup] >= curr_adll)
602 								centralization_high_limit
603 								    [pup] =
604 								    curr_adll;
605 						}
606 					}
607 				}
608 			}
609 
610 			if (unlock_pup == 0) {
611 				/* Found limit to all pups */
612 				DEBUG_DQS_FULL_S("DDR3 - DQS Find Limits - found PUP limit\n");
613 				break;
614 			}
615 
616 			/*
617 			 * Increment / decrement (Move to right / left
618 			 * one phase - ADLL) dqs RX / TX delay (for all un
619 			 * lock pups
620 			 */
621 			if (is_tx == 0)
622 				curr_adll++;
623 			else
624 				curr_adll--;
625 		}
626 
627 		if (unlock_pup != 0) {
628 			/*
629 			 * Found pups that didn't reach to the end of the
630 			 * state machine
631 			 */
632 			DEBUG_DQS_C("DDR3 - DQS Find Limits - Pups that didn't reached end of the state machine: ",
633 				    unlock_pup, 1);
634 
635 			for (pup = 0; pup < max_pup; pup++) {
636 				if (IS_PUP_ACTIVE(unlock_pup, pup) == 1) {
637 					if (pup_adll_limit_state[pup] ==
638 					    PUP_ADLL_LIMITS_STATE_FAIL) {
639 						/* ERROR - found fail for all window size */
640 						DEBUG_DQS_S("DDR3 - DQS Find Limits - Got FAIL for the complete range on pup - ");
641 						DEBUG_DQS_D(pup, 1);
642 						DEBUG_DQS_C(" victim DQ ",
643 							    victim_dq, 1);
644 
645 						/* For debug - set min limit to illegal limit */
646 						centralization_low_limit[pup]
647 							= ADLL_ERROR;
648 						/*
649 						 * In case the pup is in mode
650 						 * PASS - the limit is the min
651 						 * / max adll, no need to
652 						 * update because of the results
653 						 * array default value
654 						 */
655 						return MV_DDR3_TRAINING_ERR_PUP_RANGE;
656 					}
657 				}
658 			}
659 		}
660 	}
661 
662 	DEBUG_DQS_S("DDR3 - DQS Find Limits - DQ values per victim results:\n");
663 	for (victim_dq = 0; victim_dq < DQ_NUM; victim_dq++) {
664 		for (pup = 0; pup < max_pup; pup++) {
665 			DEBUG_DQS_S("Victim DQ-");
666 			DEBUG_DQS_D(victim_dq, 1);
667 			DEBUG_DQS_S(", PUP-");
668 			DEBUG_DQS_D(pup, 1);
669 			for (dq = 0; dq < DQ_NUM; dq++) {
670 				DEBUG_DQS_S(", DQ-");
671 				DEBUG_DQS_D(dq, 1);
672 				DEBUG_DQS_S(",S-");
673 				DEBUG_DQS_D(analog_pbs[victim_dq][pup][dq]
674 					    [0], 2);
675 				DEBUG_DQS_S(",E-");
676 				DEBUG_DQS_D(analog_pbs[victim_dq][pup][dq]
677 					    [1], 2);
678 
679 				if (is_tx == 0) {
680 					if (analog_pbs[victim_dq][pup][dq][0]
681 					    > analog_pbs_sum[pup][dq][0])
682 						analog_pbs_sum[pup][dq][0] =
683 						    analog_pbs[victim_dq][pup]
684 						    [dq][0];
685 					if (analog_pbs[victim_dq][pup][dq][1]
686 					    < analog_pbs_sum[pup][dq][1])
687 						analog_pbs_sum[pup][dq][1] =
688 						    analog_pbs[victim_dq][pup]
689 						    [dq][1];
690 				} else {
691 					if (analog_pbs[victim_dq][pup][dq][0]
692 					    < analog_pbs_sum[pup][dq][0])
693 						analog_pbs_sum[pup][dq][0] =
694 						    analog_pbs[victim_dq][pup]
695 						    [dq][0];
696 					if (analog_pbs[victim_dq][pup][dq][1]
697 					    > analog_pbs_sum[pup][dq][1])
698 						analog_pbs_sum[pup][dq][1] =
699 						    analog_pbs[victim_dq][pup]
700 						    [dq][1];
701 				}
702 			}
703 			DEBUG_DQS_S("\n");
704 		}
705 	}
706 
707 	if (ddr3_get_log_level() >= MV_LOG_LEVEL_3) {
708 		u32 dq;
709 
710 		DEBUG_PER_DQ_S("\n########## LOG LEVEL 3(Windows margins per-DQ) ##########\n");
711 		if (is_tx) {
712 			DEBUG_PER_DQ_C("DDR3 - TX  CS: ", cs, 1);
713 		} else {
714 			DEBUG_PER_DQ_C("DDR3 - RX  CS: ", cs, 1);
715 		}
716 
717 		if (ecc == 0) {
718 			DEBUG_PER_DQ_S("\n DATA RESULTS:\n");
719 		} else {
720 			DEBUG_PER_DQ_S("\n ECC RESULTS:\n");
721 		}
722 
723 		/* Since all dq has the same value we take 0 as representive */
724 		dq = 0;
725 		for (pup = 0; pup < max_pup; pup++) {
726 			if (ecc == 0) {
727 				DEBUG_PER_DQ_S("\nBYTE:");
728 				DEBUG_PER_DQ_D(pup, 1);
729 				DEBUG_PER_DQ_S("\n");
730 			} else {
731 				DEBUG_PER_DQ_S("\nECC BYTE:\n");
732 			}
733 			DEBUG_PER_DQ_S("  DQ's        LOW       HIGH       WIN-SIZE\n");
734 			DEBUG_PER_DQ_S("============================================\n");
735 			for (victim_dq = 0; victim_dq < DQ_NUM; victim_dq++) {
736 				if (ecc == 0) {
737 					DEBUG_PER_DQ_S("DQ[");
738 					DEBUG_PER_DQ_DD((victim_dq +
739 							 DQ_NUM * pup), 2);
740 					DEBUG_PER_DQ_S("]");
741 				} else {
742 					DEBUG_PER_DQ_S("CB[");
743 					DEBUG_PER_DQ_DD(victim_dq, 2);
744 					DEBUG_PER_DQ_S("]");
745 				}
746 				if (is_tx) {
747 					DEBUG_PER_DQ_S("      0x");
748 					DEBUG_PER_DQ_D(analog_pbs[victim_dq][pup][dq][1], 2);	/* low value */
749 					DEBUG_PER_DQ_S("        0x");
750 					DEBUG_PER_DQ_D(analog_pbs[victim_dq][pup][dq][0], 2);	/* high value */
751 					DEBUG_PER_DQ_S("        0x");
752 					DEBUG_PER_DQ_D(analog_pbs[victim_dq][pup][dq][0] - analog_pbs[victim_dq][pup][dq][1], 2);	/* win-size */
753 				} else {
754 					DEBUG_PER_DQ_S("     0x");
755 					DEBUG_PER_DQ_D(analog_pbs[victim_dq][pup][dq][0], 2);	/* low value */
756 					DEBUG_PER_DQ_S("       0x");
757 					DEBUG_PER_DQ_D((analog_pbs[victim_dq][pup][dq][1] - 1), 2);	/* high value */
758 					DEBUG_PER_DQ_S("       0x");
759 					DEBUG_PER_DQ_D(analog_pbs[victim_dq][pup][dq][1] - analog_pbs[victim_dq][pup][dq][0], 2);	/* win-size */
760 				}
761 				DEBUG_PER_DQ_S("\n");
762 			}
763 		}
764 		DEBUG_PER_DQ_S("\n");
765 	}
766 
767 	if (is_tx) {
768 		DEBUG_DQS_S("DDR3 - DQS TX - Find Limits - DQ values Summary:\n");
769 	} else {
770 		DEBUG_DQS_S("DDR3 - DQS RX - Find Limits - DQ values Summary:\n");
771 	}
772 
773 	for (pup = 0; pup < max_pup; pup++) {
774 		DEBUG_DQS_S("PUP-");
775 		DEBUG_DQS_D(pup, 1);
776 		for (dq = 0; dq < DQ_NUM; dq++) {
777 			DEBUG_DQS_S(", DQ-");
778 			DEBUG_DQS_D(dq, 1);
779 			DEBUG_DQS_S(",S-");
780 			DEBUG_DQS_D(analog_pbs_sum[pup][dq][0], 2);
781 			DEBUG_DQS_S(",E-");
782 			DEBUG_DQS_D(analog_pbs_sum[pup][dq][1], 2);
783 		}
784 		DEBUG_DQS_S("\n");
785 	}
786 
787 	if (is_tx) {
788 		DEBUG_DQS_S("DDR3 - DQS TX - Find Limits - DQ values Summary:\n");
789 	} else {
790 		DEBUG_DQS_S("DDR3 - DQS RX - Find Limits - DQ values Summary:\n");
791 	}
792 
793 	for (pup = 0; pup < max_pup; pup++) {
794 		if (max_pup == 1) {
795 			/* For ECC PUP */
796 			DEBUG_DQS_S("DDR3 - DQS8");
797 		} else {
798 			DEBUG_DQS_S("DDR3 - DQS");
799 			DEBUG_DQS_D(pup, 1);
800 		}
801 
802 		for (dq = 0; dq < DQ_NUM; dq++) {
803 			DEBUG_DQS_S(", DQ-");
804 			DEBUG_DQS_D(dq, 1);
805 			DEBUG_DQS_S("::S-");
806 			DEBUG_DQS_D(analog_pbs_sum[pup][dq][0], 2);
807 			DEBUG_DQS_S(",E-");
808 			DEBUG_DQS_D(analog_pbs_sum[pup][dq][1], 2);
809 		}
810 		DEBUG_DQS_S("\n");
811 	}
812 
813 	DEBUG_DQS_S("DDR3 - DQS Find Limits - Ended\n");
814 
815 	return MV_OK;
816 }
817 
818 /*
819  * Name:     ddr3_check_window_limits
820  * Desc:     Check window High & Low limits.
821  * Args:     pup                pup index
822  *           high_limit           window high limit
823  *           low_limit            window low limit
824  *           is_tx                Indicate whether Rx or Tx
825  *           size_valid          Indicate whether window size is valid
826  * Notes:
827  * Returns:  MV_OK if success, other error code if fail.
828  */
829 int ddr3_check_window_limits(u32 pup, int high_limit, int low_limit, int is_tx,
830 			     int *size_valid)
831 {
832 	DEBUG_DQS_FULL_S("DDR3 - DQS Check Win Limits - Starting\n");
833 
834 	if (low_limit > high_limit) {
835 		DEBUG_DQS_S("DDR3 - DQS Check Win Limits - Pup ");
836 		DEBUG_DQS_D(pup, 1);
837 		DEBUG_DQS_S(" Low Limit grater than High Limit\n");
838 		*size_valid = 0;
839 		return MV_OK;
840 	}
841 
842 	/*
843 	 * Check that window size is valid, if not it was probably false pass
844 	 * before
845 	 */
846 	if ((high_limit - low_limit) < MIN_WIN_SIZE) {
847 		/*
848 		 * Since window size is too small probably there was false
849 		 * pass
850 		 */
851 		*size_valid = 0;
852 
853 		DEBUG_DQS_S("DDR3 - DQS Check Win Limits - Pup ");
854 		DEBUG_DQS_D(pup, 1);
855 		DEBUG_DQS_S(" Window size is smaller than MIN_WIN_SIZE\n");
856 
857 	} else if ((high_limit - low_limit) > ADLL_MAX) {
858 		*size_valid = 0;
859 
860 		DEBUG_DQS_S("DDR3 - DQS Check Win Limits - Pup ");
861 		DEBUG_DQS_D(pup, 1);
862 		DEBUG_DQS_S
863 		    (" Window size is bigger than max ADLL taps (31)  Exiting.\n");
864 
865 		return MV_FAIL;
866 
867 	} else {
868 		*size_valid = 1;
869 
870 		DEBUG_DQS_FULL_S("DDR3 - DQS Check Win Limits - Pup ");
871 		DEBUG_DQS_FULL_D(pup, 1);
872 		DEBUG_DQS_FULL_C(" window size is ", (high_limit - low_limit),
873 				 2);
874 	}
875 
876 	return MV_OK;
877 }
878 
879 /*
880  * Name:     ddr3_center_calc
881  * Desc:     Execute the calculate the center of windows phase.
882  * Args:     pDram Info
883  *           is_tx             Indicate whether Rx or Tx
884  * Notes:
885  * Returns:  MV_OK if success, other error code if fail.
886  */
887 static int ddr3_center_calc(MV_DRAM_INFO *dram_info, u32 cs, u32 ecc,
888 			    int is_tx)
889 {
890 	/* bit array of pups that need specail search */
891 	u32 special_pattern_i_pup = 0;
892 	u32 special_pattern_ii_pup = 0;
893 	u32 pup;
894 	u32 max_pup;
895 
896 	max_pup = (ecc + (1 - ecc) * dram_info->num_of_std_pups);
897 
898 	for (pup = 0; pup < max_pup; pup++) {
899 		if (is_tx == 0) {
900 			/* Check special pattern I */
901 			/*
902 			 * Special pattern Low limit search - relevant only
903 			 * for Rx, win size < threshold and low limit = 0
904 			 */
905 			if (((centralization_high_limit[pup] -
906 			      centralization_low_limit[pup]) < VALID_WIN_THRS)
907 			    && (centralization_low_limit[pup] == MIN_DELAY))
908 				special_pattern_i_pup |= (1 << pup);
909 
910 			/* Check special pattern II */
911 			/*
912 			 * Special pattern High limit search - relevant only
913 			 * for Rx, win size < threshold and high limit = 31
914 			 */
915 			if (((centralization_high_limit[pup] -
916 			      centralization_low_limit[pup]) < VALID_WIN_THRS)
917 			    && (centralization_high_limit[pup] == MAX_DELAY))
918 				special_pattern_ii_pup |= (1 << pup);
919 		}
920 	}
921 
922 	/* Run special pattern Low limit search - for relevant pup */
923 	if (special_pattern_i_pup != 0) {
924 		DEBUG_DQS_S("DDR3 - DQS Center Calc - Entering special pattern I for Low limit search\n");
925 		if (MV_OK !=
926 		    ddr3_special_pattern_i_search(dram_info, cs, ecc, is_tx,
927 					      special_pattern_i_pup))
928 			return MV_DDR3_TRAINING_ERR_DQS_LOW_LIMIT_SEARCH;
929 	}
930 
931 	/* Run special pattern High limit search - for relevant pup */
932 	if (special_pattern_ii_pup != 0) {
933 		DEBUG_DQS_S("DDR3 - DQS Center Calc - Entering special pattern II for High limit search\n");
934 		if (MV_OK !=
935 		    ddr3_special_pattern_ii_search(dram_info, cs, ecc, is_tx,
936 						   special_pattern_ii_pup))
937 			return MV_DDR3_TRAINING_ERR_DQS_HIGH_LIMIT_SEARCH;
938 	}
939 
940 	/* Set adll to center = (General_High_limit + General_Low_limit)/2 */
941 	return ddr3_set_dqs_centralization_results(dram_info, cs, ecc, is_tx);
942 }
943 
944 /*
945  * Name:     ddr3_special_pattern_i_search
946  * Desc:     Execute special pattern low limit search.
947  * Args:
948  *           special_pattern_pup  The pups that need the special search
949  * Notes:
950  * Returns:  MV_OK if success, other error code if fail.
951  */
952 int ddr3_special_pattern_i_search(MV_DRAM_INFO *dram_info, u32 cs, u32 ecc,
953 				  int is_tx, u32 special_pattern_pup)
954 {
955 	u32 victim_dq;		/* loop index - victim DQ */
956 	u32 adll_idx;
957 	u32 pup;
958 	u32 unlock_pup;		/* bit array of the unlock pups  */
959 	u32 first_fail;	/* bit array - of pups that  get first fail */
960 	u32 new_lockup_pup;	/* bit array of compare failed pups */
961 	u32 pass_pup;		/* bit array of compare pass pup */
962 	u32 sdram_offset;
963 	u32 max_pup;
964 	u32 comp_val;
965 	u32 special_res[MAX_PUP_NUM];	/* hold tmp results */
966 
967 	DEBUG_DQS_S("DDR3 - DQS - Special Pattern I Search - Starting\n");
968 
969 	max_pup = ecc + (1 - ecc) * dram_info->num_of_std_pups;
970 
971 	/* Init the temporary results to max ADLL value */
972 	for (pup = 0; pup < max_pup; pup++)
973 		special_res[pup] = ADLL_MAX;
974 
975 	/* Run special pattern for all DQ - use the same pattern */
976 	for (victim_dq = 0; victim_dq < DQ_NUM; victim_dq++) {
977 		unlock_pup = special_pattern_pup;
978 		first_fail = 0;
979 
980 		sdram_offset = cs * SDRAM_CS_SIZE + SDRAM_DQS_RX_OFFS +
981 			LEN_KILLER_PATTERN * 4 * victim_dq;
982 
983 		for (pup = 0; pup < max_pup; pup++) {
984 			/* Set adll value per PUP. adll = high limit per pup */
985 			if (IS_PUP_ACTIVE(unlock_pup, pup)) {
986 				/* only for pups that need special search */
987 				ddr3_write_pup_reg(PUP_DQS_RD, cs,
988 						   pup + (ecc * ECC_PUP), 0,
989 						   centralization_high_limit
990 						   [pup]);
991 			}
992 		}
993 
994 		adll_idx = 0;
995 		do {
996 			/*
997 			 * Perform read and compare simultaneously for all
998 			 * un-locked MC use the special pattern mask
999 			 */
1000 			new_lockup_pup = 0;
1001 
1002 			if (MV_OK !=
1003 			    ddr3_sdram_dqs_compare(dram_info, unlock_pup,
1004 						   &new_lockup_pup,
1005 						   special_pattern
1006 						   [victim_dq],
1007 						   LEN_SPECIAL_PATTERN,
1008 						   sdram_offset, 0,
1009 						   0, NULL, 1))
1010 				return MV_FAIL;
1011 
1012 			DEBUG_DQS_S("DDR3 - DQS - Special I - ADLL value is: ");
1013 			DEBUG_DQS_D(adll_idx, 2);
1014 			DEBUG_DQS_S(", UnlockPup: ");
1015 			DEBUG_DQS_D(unlock_pup, 2);
1016 			DEBUG_DQS_S(", NewLockPup: ");
1017 			DEBUG_DQS_D(new_lockup_pup, 2);
1018 			DEBUG_DQS_S("\n");
1019 
1020 			if (unlock_pup != new_lockup_pup)
1021 				DEBUG_DQS_S("DDR3 - DQS - Special I - Some Pup passed!\n");
1022 
1023 			/* Search for pups with passed compare & already fail */
1024 			pass_pup = first_fail & ~new_lockup_pup & unlock_pup;
1025 			first_fail |= new_lockup_pup;
1026 			unlock_pup &= ~pass_pup;
1027 
1028 			/* Get pass pups */
1029 			if (pass_pup != 0) {
1030 				for (pup = 0; pup < max_pup; pup++) {
1031 					if (IS_PUP_ACTIVE(pass_pup, pup) ==
1032 					    1) {
1033 						/* If pup passed and has first fail = 1 */
1034 						/* keep min value of ADLL max value - current adll */
1035 						/* (centralization_high_limit[pup] + adll_idx) = current adll !!! */
1036 						comp_val =
1037 						    (ADLL_MAX -
1038 						     (centralization_high_limit
1039 						      [pup] + adll_idx));
1040 
1041 						DEBUG_DQS_C
1042 						    ("DDR3 - DQS - Special I - Pup - ",
1043 						     pup, 1);
1044 						DEBUG_DQS_C
1045 						    (" comp_val = ",
1046 						     comp_val, 2);
1047 
1048 						if (comp_val <
1049 						    special_res[pup]) {
1050 							special_res[pup] =
1051 							    comp_val;
1052 							centralization_low_limit
1053 							    [pup] =
1054 							    (-1) *
1055 							    comp_val;
1056 
1057 							DEBUG_DQS_C
1058 							    ("DDR3 - DQS - Special I - Pup - ",
1059 							     pup, 1);
1060 							DEBUG_DQS_C
1061 							    (" Changed Low limit to ",
1062 							     centralization_low_limit
1063 							     [pup], 2);
1064 						}
1065 					}
1066 				}
1067 			}
1068 
1069 			/*
1070 			 * Did all PUP found missing window?
1071 			 * Check for each pup if adll (different for each pup)
1072 			 * reach maximum if reach max value - lock the pup
1073 			 * if not - increment (Move to right one phase - ADLL)
1074 			 * dqs RX delay
1075 			 */
1076 			adll_idx++;
1077 			for (pup = 0; pup < max_pup; pup++) {
1078 				if (IS_PUP_ACTIVE(unlock_pup, pup) == 1) {
1079 					/* Check only unlocked pups */
1080 					if ((centralization_high_limit[pup] +
1081 					     adll_idx) >= ADLL_MAX) {
1082 						/* reach maximum - lock the pup */
1083 						DEBUG_DQS_C("DDR3 - DQS - Special I - reach maximum - lock pup ",
1084 							    pup, 1);
1085 						unlock_pup &= ~(1 << pup);
1086 					} else {
1087 						/* Didn't reach maximum - increment ADLL */
1088 						ddr3_write_pup_reg(PUP_DQS_RD,
1089 								   cs,
1090 								   pup +
1091 								   (ecc *
1092 								    ECC_PUP), 0,
1093 								   (centralization_high_limit
1094 								    [pup] +
1095 								    adll_idx));
1096 					}
1097 				}
1098 			}
1099 		} while (unlock_pup != 0);
1100 	}
1101 
1102 	return MV_OK;
1103 }
1104 
1105 /*
1106  * Name:     ddr3_special_pattern_ii_search
1107  * Desc:     Execute special pattern high limit search.
1108  * Args:
1109  *           special_pattern_pup  The pups that need the special search
1110  * Notes:
1111  * Returns:  MV_OK if success, other error code if fail.
1112  */
1113 int ddr3_special_pattern_ii_search(MV_DRAM_INFO *dram_info, u32 cs, u32 ecc,
1114 				   int is_tx, u32 special_pattern_pup)
1115 {
1116 	u32 victim_dq;		/* loop index - victim DQ */
1117 	u32 adll_idx;
1118 	u32 pup;
1119 	u32 unlock_pup;		/* bit array of the unlock pups  */
1120 	u32 first_fail;	/* bit array - of pups that  get first fail */
1121 	u32 new_lockup_pup;	/* bit array of compare failed pups */
1122 	u32 pass_pup;		/* bit array of compare pass pup */
1123 	u32 sdram_offset;
1124 	u32 max_pup;
1125 	u32 comp_val;
1126 	u32 special_res[MAX_PUP_NUM];	/* hold tmp results */
1127 
1128 	DEBUG_DQS_S("DDR3 - DQS - Special Pattern II Search - Starting\n");
1129 
1130 	max_pup = (ecc + (1 - ecc) * dram_info->num_of_std_pups);
1131 
1132 	/* init the tmporary results to max ADLL value */
1133 	for (pup = 0; pup < max_pup; pup++)
1134 		special_res[pup] = ADLL_MAX;
1135 
1136 	sdram_offset = cs * SDRAM_CS_SIZE + SDRAM_DQS_RX_OFFS;
1137 
1138 	/* run special pattern for all DQ - use the same pattern */
1139 	for (victim_dq = 0; victim_dq < DQ_NUM; victim_dq++) {
1140 		unlock_pup = special_pattern_pup;
1141 		first_fail = 0;
1142 
1143 		for (pup = 0; pup < max_pup; pup++) {
1144 			/* Set adll value per PUP. adll = 0 */
1145 			if (IS_PUP_ACTIVE(unlock_pup, pup)) {
1146 				/* Only for pups that need special search */
1147 				ddr3_write_pup_reg(PUP_DQS_RD, cs,
1148 						   pup + (ecc * ECC_PUP), 0,
1149 						   ADLL_MIN);
1150 			}
1151 		}
1152 
1153 		adll_idx = 0;
1154 		do {
1155 			/*
1156 			 * Perform read and compare simultaneously for all
1157 			 * un-locked MC use the special pattern mask
1158 			 */
1159 			new_lockup_pup = 0;
1160 
1161 			if (MV_OK != ddr3_sdram_dqs_compare(
1162 				    dram_info, unlock_pup, &new_lockup_pup,
1163 				    special_pattern[victim_dq],
1164 				    LEN_SPECIAL_PATTERN,
1165 				    sdram_offset, 0, 0, NULL, 0))
1166 				return MV_FAIL;
1167 
1168 			DEBUG_DQS_S("DDR3 - DQS - Special II - ADLL value is ");
1169 			DEBUG_DQS_D(adll_idx, 2);
1170 			DEBUG_DQS_S("unlock_pup ");
1171 			DEBUG_DQS_D(unlock_pup, 1);
1172 			DEBUG_DQS_S("new_lockup_pup ");
1173 			DEBUG_DQS_D(new_lockup_pup, 1);
1174 			DEBUG_DQS_S("\n");
1175 
1176 			if (unlock_pup != new_lockup_pup) {
1177 				DEBUG_DQS_S("DDR3 - DQS - Special II - Some Pup passed!\n");
1178 			}
1179 
1180 			/* Search for pups with passed compare & already fail */
1181 			pass_pup = first_fail & ~new_lockup_pup & unlock_pup;
1182 			first_fail |= new_lockup_pup;
1183 			unlock_pup &= ~pass_pup;
1184 
1185 			/* Get pass pups */
1186 			if (pass_pup != 0) {
1187 				for (pup = 0; pup < max_pup; pup++) {
1188 					if (IS_PUP_ACTIVE(pass_pup, pup) ==
1189 					    1) {
1190 						/* If pup passed and has first fail = 1 */
1191 						/* keep min value of ADLL max value - current adll */
1192 						/* (adll_idx) = current adll !!! */
1193 						comp_val = adll_idx;
1194 
1195 						DEBUG_DQS_C("DDR3 - DQS - Special II - Pup - ",
1196 							    pup, 1);
1197 						DEBUG_DQS_C(" comp_val = ",
1198 							    comp_val, 1);
1199 
1200 						if (comp_val <
1201 						    special_res[pup]) {
1202 							special_res[pup] =
1203 							    comp_val;
1204 							centralization_high_limit
1205 							    [pup] =
1206 							    ADLL_MAX +
1207 							    comp_val;
1208 
1209 							DEBUG_DQS_C
1210 							    ("DDR3 - DQS - Special II - Pup - ",
1211 							     pup, 1);
1212 							DEBUG_DQS_C
1213 							    (" Changed High limit to ",
1214 							     centralization_high_limit
1215 							     [pup], 2);
1216 						}
1217 					}
1218 				}
1219 			}
1220 
1221 			/*
1222 			 * Did all PUP found missing window?
1223 			 * Check for each pup if adll (different for each pup)
1224 			 * reach maximum if reach max value - lock the pup
1225 			 * if not - increment (Move to right one phase - ADLL)
1226 			 * dqs RX delay
1227 			 */
1228 			adll_idx++;
1229 			for (pup = 0; pup < max_pup; pup++) {
1230 				if (IS_PUP_ACTIVE(unlock_pup, pup) == 1) {
1231 					/* Check only unlocked pups */
1232 					if ((adll_idx) >= ADLL_MAX) {
1233 						/* Reach maximum - lock the pup */
1234 						DEBUG_DQS_C("DDR3 - DQS - Special II - reach maximum - lock pup ",
1235 							    pup, 1);
1236 						unlock_pup &= ~(1 << pup);
1237 					} else {
1238 						/* Didn't reach maximum - increment ADLL */
1239 						ddr3_write_pup_reg(PUP_DQS_RD,
1240 								   cs,
1241 								   pup +
1242 								   (ecc *
1243 								    ECC_PUP), 0,
1244 								   (adll_idx));
1245 					}
1246 				}
1247 			}
1248 		} while (unlock_pup != 0);
1249 	}
1250 
1251 	return MV_OK;
1252 }
1253 
1254 /*
1255  * Name:     ddr3_set_dqs_centralization_results
1256  * Desc:     Set to HW the DQS centralization phase results.
1257  * Args:
1258  *           is_tx             Indicates whether to set Tx or RX results
1259  * Notes:
1260  * Returns:  MV_OK if success, other error code if fail.
1261  */
1262 int ddr3_set_dqs_centralization_results(MV_DRAM_INFO *dram_info, u32 cs,
1263 					u32 ecc, int is_tx)
1264 {
1265 	u32 pup, pup_num;
1266 	int addl_val;
1267 	u32 max_pup;
1268 
1269 	max_pup = (ecc + (1 - ecc) * dram_info->num_of_std_pups);
1270 
1271 	DEBUG_DQS_RESULTS_S("\n############ LOG LEVEL 2(Windows margins) ############\n");;
1272 
1273 	if (is_tx) {
1274 		DEBUG_DQS_RESULTS_C("DDR3 - DQS TX - Set Dqs Centralization Results - CS: ",
1275 				    cs, 1);
1276 	} else {
1277 		DEBUG_DQS_RESULTS_C("DDR3 - DQS RX - Set Dqs Centralization Results - CS: ",
1278 				    cs, 1);
1279 	}
1280 
1281 	/* Set adll to center = (General_High_limit + General_Low_limit)/2 */
1282 	DEBUG_DQS_RESULTS_S("\nDQS    LOW     HIGH     WIN-SIZE      Set\n");
1283 	DEBUG_DQS_RESULTS_S("==============================================\n");
1284 	for (pup = 0; pup < max_pup; pup++) {
1285 		addl_val = (centralization_high_limit[pup] +
1286 			    centralization_low_limit[pup]) / 2;
1287 
1288 		pup_num = pup * (1 - ecc) + ecc * ECC_PUP;
1289 
1290 		DEBUG_DQS_RESULTS_D(pup_num, 1);
1291 		DEBUG_DQS_RESULTS_S("     0x");
1292 		DEBUG_DQS_RESULTS_D(centralization_low_limit[pup], 2);
1293 		DEBUG_DQS_RESULTS_S("      0x");
1294 		DEBUG_DQS_RESULTS_D(centralization_high_limit[pup], 2);
1295 		DEBUG_DQS_RESULTS_S("      0x");
1296 		DEBUG_DQS_RESULTS_D(centralization_high_limit[pup] -
1297 				    centralization_low_limit[pup], 2);
1298 		DEBUG_DQS_RESULTS_S("       0x");
1299 		DEBUG_DQS_RESULTS_D(addl_val, 2);
1300 		DEBUG_DQS_RESULTS_S("\n");
1301 
1302 		if (addl_val < ADLL_MIN) {
1303 			addl_val = ADLL_MIN;
1304 			DEBUG_DQS_RESULTS_S("DDR3 - DQS - Setting ADLL value for Pup to MIN (since it was lower than 0)\n");
1305 		}
1306 
1307 		if (addl_val > ADLL_MAX) {
1308 			addl_val = ADLL_MAX;
1309 			DEBUG_DQS_RESULTS_S("DDR3 - DQS - Setting ADLL value for Pup to MAX (since it was higher than 31)\n");
1310 		}
1311 
1312 		if (is_tx) {
1313 			ddr3_write_pup_reg(PUP_DQS_WR, cs, pup_num, 0,
1314 					   addl_val +
1315 					   dram_info->wl_val[cs][pup_num][D]);
1316 		} else {
1317 			ddr3_write_pup_reg(PUP_DQS_RD, cs, pup_num, 0,
1318 					   addl_val);
1319 		}
1320 	}
1321 
1322 	return MV_OK;
1323 }
1324 
1325 /*
1326  * Set training patterns
1327  */
1328 int ddr3_load_dqs_patterns(MV_DRAM_INFO *dram_info)
1329 {
1330 	u32 cs, cs_count, cs_tmp, victim_dq;
1331 	u32 sdram_addr;
1332 	u32 *pattern_ptr;
1333 
1334 	/* Loop for each CS */
1335 	for (cs = 0; cs < MAX_CS; cs++) {
1336 		if (dram_info->cs_ena & (1 << cs)) {
1337 			cs_count = 0;
1338 			for (cs_tmp = 0; cs_tmp < cs; cs_tmp++) {
1339 				if (dram_info->cs_ena & (1 << cs_tmp))
1340 					cs_count++;
1341 			}
1342 
1343 			/* Init killer pattern */
1344 			sdram_addr = (cs_count * (SDRAM_CS_SIZE + 1) +
1345 				      SDRAM_DQS_RX_OFFS);
1346 			for (victim_dq = 0; victim_dq < DQ_NUM; victim_dq++) {
1347 				pattern_ptr = ddr3_dqs_choose_pattern(dram_info,
1348 								      victim_dq);
1349 				if (MV_OK != ddr3_sdram_dqs_compare(
1350 					    dram_info, (u32)NULL, NULL,
1351 					    pattern_ptr, LEN_KILLER_PATTERN,
1352 					    sdram_addr + LEN_KILLER_PATTERN *
1353 					    4 * victim_dq, 1, 0, NULL,
1354 					    0))
1355 					return MV_DDR3_TRAINING_ERR_DQS_PATTERN;
1356 			}
1357 
1358 			/* Init special-killer pattern */
1359 			sdram_addr = (cs_count * (SDRAM_CS_SIZE + 1) +
1360 				      SDRAM_DQS_RX_SPECIAL_OFFS);
1361 			for (victim_dq = 0; victim_dq < DQ_NUM; victim_dq++) {
1362 				if (MV_OK != ddr3_sdram_dqs_compare(
1363 					    dram_info, (u32)NULL, NULL,
1364 					    special_pattern[victim_dq],
1365 					    LEN_KILLER_PATTERN, sdram_addr +
1366 					    LEN_KILLER_PATTERN * 4 * victim_dq,
1367 					    1, 0, NULL, 0))
1368 					return MV_DDR3_TRAINING_ERR_DQS_PATTERN;
1369 			}
1370 		}
1371 	}
1372 
1373 	return MV_OK;
1374 }
1375