1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (C) Marvell International Ltd. and its affiliates
4  */
5 
6 #include <common.h>
7 #include <i2c.h>
8 #include <spl.h>
9 #include <asm/io.h>
10 #include <asm/arch/cpu.h>
11 #include <asm/arch/soc.h>
12 
13 #include "ddr3_hw_training.h"
14 
15 /*
16  * Debug
17  */
18 #define DEBUG_DQS_C(s, d, l) \
19 	DEBUG_DQS_S(s); DEBUG_DQS_D(d, l); DEBUG_DQS_S("\n")
20 #define DEBUG_DQS_FULL_C(s, d, l) \
21 	DEBUG_DQS_FULL_S(s); DEBUG_DQS_FULL_D(d, l); DEBUG_DQS_FULL_S("\n")
22 #define DEBUG_DQS_RESULTS_C(s, d, l) \
23 	DEBUG_DQS_RESULTS_S(s); DEBUG_DQS_RESULTS_D(d, l); DEBUG_DQS_RESULTS_S("\n")
24 #define DEBUG_PER_DQ_C(s, d, l) \
25 	puts(s); printf("%x", d); puts("\n")
26 
27 #define DEBUG_DQS_RESULTS_S(s) \
28 	debug_cond(ddr3_get_log_level() >= MV_LOG_LEVEL_2, "%s", s)
29 #define DEBUG_DQS_RESULTS_D(d, l) \
30 	debug_cond(ddr3_get_log_level() >= MV_LOG_LEVEL_2, "%x", d)
31 
32 #define DEBUG_PER_DQ_S(s) \
33 	debug_cond(ddr3_get_log_level() >= MV_LOG_LEVEL_3, "%s", s)
34 #define DEBUG_PER_DQ_D(d, l) \
35 	debug_cond(ddr3_get_log_level() >= MV_LOG_LEVEL_3, "%x", d)
36 #define DEBUG_PER_DQ_DD(d, l) \
37 	debug_cond(ddr3_get_log_level() >= MV_LOG_LEVEL_3, "%d", d)
38 
39 #ifdef MV_DEBUG_DQS
40 #define DEBUG_DQS_S(s)			puts(s)
41 #define DEBUG_DQS_D(d, l)		printf("%x", d)
42 #else
43 #define DEBUG_DQS_S(s)
44 #define DEBUG_DQS_D(d, l)
45 #endif
46 
47 #ifdef MV_DEBUG_DQS_FULL
48 #define DEBUG_DQS_FULL_S(s)		puts(s)
49 #define DEBUG_DQS_FULL_D(d, l)		printf("%x", d)
50 #else
51 #define DEBUG_DQS_FULL_S(s)
52 #define DEBUG_DQS_FULL_D(d, l)
53 #endif
54 
55 /* State machine for centralization - find low & high limit */
56 enum {
57 	PUP_ADLL_LIMITS_STATE_FAIL,
58 	PUP_ADLL_LIMITS_STATE_PASS,
59 	PUP_ADLL_LIMITS_STATE_FAIL_AFTER_PASS,
60 };
61 
62 /* Hold centralization low results */
63 static int centralization_low_limit[MAX_PUP_NUM] = { 0 };
64 /* Hold centralization high results */
65 static int centralization_high_limit[MAX_PUP_NUM] = { 0 };
66 
67 int ddr3_find_adll_limits(MV_DRAM_INFO *dram_info, u32 cs, u32 ecc, int is_tx);
68 int ddr3_check_window_limits(u32 pup, int high_limit, int low_limit, int is_tx,
69 			  int *size_valid);
70 static int ddr3_center_calc(MV_DRAM_INFO *dram_info, u32 cs, u32 ecc,
71 			    int is_tx);
72 int ddr3_special_pattern_i_search(MV_DRAM_INFO *dram_info, u32 cs, u32 ecc,
73 			      int is_tx, u32 special_pattern_pup);
74 int ddr3_special_pattern_ii_search(MV_DRAM_INFO *dram_info, u32 cs, u32 ecc,
75 				   int is_tx, u32 special_pattern_pup);
76 int ddr3_set_dqs_centralization_results(MV_DRAM_INFO *dram_info, u32 cs, u32 ecc,
77 				    int is_tx);
78 
79 #ifdef MV88F78X60
80 extern u32 killer_pattern_32b[DQ_NUM][LEN_SPECIAL_PATTERN];
81 extern u32 killer_pattern_64b[DQ_NUM][LEN_SPECIAL_PATTERN];
82 extern int per_bit_data[MAX_PUP_NUM][DQ_NUM];
83 #else
84 extern u32 killer_pattern[DQ_NUM][LEN_16BIT_KILLER_PATTERN];
85 extern u32 killer_pattern_32b[DQ_NUM][LEN_SPECIAL_PATTERN];
86 #if defined(MV88F672X)
87 extern int per_bit_data[MAX_PUP_NUM][DQ_NUM];
88 #endif
89 #endif
90 extern u32 special_pattern[DQ_NUM][LEN_SPECIAL_PATTERN];
91 
92 static u32 *ddr3_dqs_choose_pattern(MV_DRAM_INFO *dram_info, u32 victim_dq)
93 {
94 	u32 *pattern_ptr;
95 
96 	/* Choose pattern */
97 	switch (dram_info->ddr_width) {
98 #if defined(MV88F672X)
99 	case 16:
100 		pattern_ptr = (u32 *)&killer_pattern[victim_dq];
101 		break;
102 #endif
103 	case 32:
104 		pattern_ptr = (u32 *)&killer_pattern_32b[victim_dq];
105 		break;
106 #if defined(MV88F78X60)
107 	case 64:
108 		pattern_ptr = (u32 *)&killer_pattern_64b[victim_dq];
109 		break;
110 #endif
111 	default:
112 #if defined(MV88F78X60)
113 		pattern_ptr = (u32 *)&killer_pattern_32b[victim_dq];
114 #else
115 		pattern_ptr = (u32 *)&killer_pattern[victim_dq];
116 #endif
117 		break;
118 	}
119 
120 	return pattern_ptr;
121 }
122 
123 /*
124  * Name:     ddr3_dqs_centralization_rx
125  * Desc:     Execute the DQS centralization RX phase.
126  * Args:     dram_info
127  * Notes:
128  * Returns:  MV_OK if success, other error code if fail.
129  */
130 int ddr3_dqs_centralization_rx(MV_DRAM_INFO *dram_info)
131 {
132 	u32 cs, ecc, reg;
133 	int status;
134 
135 	DEBUG_DQS_S("DDR3 - DQS Centralization RX - Starting procedure\n");
136 
137 	/* Enable SW override */
138 	reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |
139 		(1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
140 
141 	/* [0] = 1 - Enable SW override  */
142 	/* 0x15B8 - Training SW 2 Register */
143 	reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
144 	DEBUG_DQS_S("DDR3 - DQS Centralization RX - SW Override Enabled\n");
145 
146 	reg = (1 << REG_DRAM_TRAINING_AUTO_OFFS);
147 	reg_write(REG_DRAM_TRAINING_ADDR, reg);	/* 0x15B0 - Training Register */
148 
149 	/* Loop for each CS */
150 	for (cs = 0; cs < MAX_CS; cs++) {
151 		if (dram_info->cs_ena & (1 << cs)) {
152 			DEBUG_DQS_FULL_C("DDR3 - DQS Centralization RX - CS - ",
153 					 (u32) cs, 1);
154 
155 			for (ecc = 0; ecc < (dram_info->ecc_ena + 1); ecc++) {
156 
157 				/* ECC Support - Switch ECC Mux on ecc=1 */
158 				reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &
159 					~(1 << REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
160 				reg |= (dram_info->ecc_ena *
161 					ecc << REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
162 				reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
163 
164 				if (ecc)
165 					DEBUG_DQS_FULL_S("DDR3 - DQS Centralization RX - ECC Mux Enabled\n");
166 				else
167 					DEBUG_DQS_FULL_S("DDR3 - DQS Centralization RX - ECC Mux Disabled\n");
168 
169 				DEBUG_DQS_FULL_S("DDR3 - DQS Centralization RX - Find all limits\n");
170 
171 				status = ddr3_find_adll_limits(dram_info, cs,
172 							       ecc, 0);
173 				if (MV_OK != status)
174 					return status;
175 
176 				DEBUG_DQS_FULL_S("DDR3 - DQS Centralization RX - Start calculating center\n");
177 
178 				status = ddr3_center_calc(dram_info, cs, ecc,
179 							  0);
180 				if (MV_OK != status)
181 					return status;
182 			}
183 		}
184 	}
185 
186 	/* ECC Support - Disable ECC MUX */
187 	reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &
188 		~(1 << REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
189 	reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
190 
191 	/* Disable SW override - Must be in a different stage */
192 	/* [0]=0 - Enable SW override  */
193 	reg = reg_read(REG_DRAM_TRAINING_2_ADDR);
194 	reg &= ~(1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
195 	/* 0x15B8 - Training SW 2 Register */
196 	reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
197 
198 	reg = reg_read(REG_DRAM_TRAINING_1_ADDR) |
199 		(1 << REG_DRAM_TRAINING_1_TRNBPOINT_OFFS);
200 	reg_write(REG_DRAM_TRAINING_1_ADDR, reg);
201 
202 	return MV_OK;
203 }
204 
205 /*
206  * Name:     ddr3_dqs_centralization_tx
207  * Desc:     Execute the DQS centralization TX phase.
208  * Args:     dram_info
209  * Notes:
210  * Returns:  MV_OK if success, other error code if fail.
211  */
212 int ddr3_dqs_centralization_tx(MV_DRAM_INFO *dram_info)
213 {
214 	u32 cs, ecc, reg;
215 	int status;
216 
217 	DEBUG_DQS_S("DDR3 - DQS Centralization TX - Starting procedure\n");
218 
219 	/* Enable SW override */
220 	reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |
221 		(1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
222 
223 	/* [0] = 1 - Enable SW override  */
224 	/* 0x15B8 - Training SW 2 Register */
225 	reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
226 	DEBUG_DQS_S("DDR3 - DQS Centralization TX - SW Override Enabled\n");
227 
228 	reg = (1 << REG_DRAM_TRAINING_AUTO_OFFS);
229 	reg_write(REG_DRAM_TRAINING_ADDR, reg);	/* 0x15B0 - Training Register */
230 
231 	/* Loop for each CS */
232 	for (cs = 0; cs < MAX_CS; cs++) {
233 		if (dram_info->cs_ena & (1 << cs)) {
234 			DEBUG_DQS_FULL_C("DDR3 - DQS Centralization TX - CS - ",
235 					 (u32) cs, 1);
236 			for (ecc = 0; ecc < (dram_info->ecc_ena + 1); ecc++) {
237 				/* ECC Support - Switch ECC Mux on ecc=1 */
238 				reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &
239 					~(1 << REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
240 				reg |= (dram_info->ecc_ena *
241 					ecc << REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
242 				reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
243 
244 				if (ecc)
245 					DEBUG_DQS_FULL_S("DDR3 - DQS Centralization TX - ECC Mux Enabled\n");
246 				else
247 					DEBUG_DQS_FULL_S("DDR3 - DQS Centralization TX - ECC Mux Disabled\n");
248 
249 				DEBUG_DQS_FULL_S("DDR3 - DQS Centralization TX - Find all limits\n");
250 
251 				status = ddr3_find_adll_limits(dram_info, cs,
252 							       ecc, 1);
253 				if (MV_OK != status)
254 					return status;
255 
256 				DEBUG_DQS_FULL_S("DDR3 - DQS Centralization TX - Start calculating center\n");
257 
258 				status = ddr3_center_calc(dram_info, cs, ecc,
259 							  1);
260 				if (MV_OK != status)
261 					return status;
262 			}
263 		}
264 	}
265 
266 	/* ECC Support - Disable ECC MUX */
267 	reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &
268 		~(1 << REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
269 	reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
270 
271 	/* Disable SW override - Must be in a different stage */
272 	/* [0]=0 - Enable SW override  */
273 	reg = reg_read(REG_DRAM_TRAINING_2_ADDR);
274 	reg &= ~(1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
275 	/* 0x15B8 - Training SW 2 Register */
276 	reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
277 
278 	reg = reg_read(REG_DRAM_TRAINING_1_ADDR) |
279 		(1 << REG_DRAM_TRAINING_1_TRNBPOINT_OFFS);
280 	reg_write(REG_DRAM_TRAINING_1_ADDR, reg);
281 
282 	return MV_OK;
283 }
284 
285 /*
286  * Name:     ddr3_find_adll_limits
287  * Desc:     Execute the Find ADLL limits phase.
288  * Args:     dram_info
289  *           cs
290  *           ecc_ena
291  *           is_tx             Indicate whether Rx or Tx
292  * Notes:
293  * Returns:  MV_OK if success, other error code if fail.
294  */
295 int ddr3_find_adll_limits(MV_DRAM_INFO *dram_info, u32 cs, u32 ecc, int is_tx)
296 {
297 	u32 victim_dq, pup, tmp;
298 	u32 adll_addr;
299 	u32 max_pup;		/* maximal pup index */
300 	u32 pup_mask = 0;
301 	u32 unlock_pup;		/* bit array of un locked pups */
302 	u32 new_unlock_pup;	/* bit array of compare failed pups */
303 	u32 curr_adll;
304 	u32 adll_start_val;	/* adll start loop value - for rx or tx limit */
305 	u32 high_limit;	/* holds found High Limit */
306 	u32 low_limit;		/* holds found Low Limit */
307 	int win_valid;
308 	int update_win;
309 	u32 sdram_offset;
310 	u32 uj, cs_count, cs_tmp, ii;
311 	u32 *pattern_ptr;
312 	u32 dq;
313 	u32 adll_end_val;	/* adll end of loop val - for rx or tx limit */
314 	u8 analog_pbs[DQ_NUM][MAX_PUP_NUM][DQ_NUM][2];
315 	u8 analog_pbs_sum[MAX_PUP_NUM][DQ_NUM][2];
316 	int pup_adll_limit_state[MAX_PUP_NUM];	/* hold state of each pup */
317 
318 	adll_addr = ((is_tx == 1) ? PUP_DQS_WR : PUP_DQS_RD);
319 	adll_end_val = ((is_tx == 1) ? ADLL_MIN : ADLL_MAX);
320 	adll_start_val = ((is_tx == 1) ? ADLL_MAX : ADLL_MIN);
321 	max_pup = (ecc + (1 - ecc) * dram_info->num_of_std_pups);
322 
323 	DEBUG_DQS_FULL_S("DDR3 - DQS Find Limits - Starting Find ADLL Limits\n");
324 
325 	/* init the array */
326 	for (pup = 0; pup < max_pup; pup++) {
327 		centralization_low_limit[pup] = ADLL_MIN;
328 		centralization_high_limit[pup] = ADLL_MAX;
329 	}
330 
331 	/* Killer Pattern */
332 	cs_count = 0;
333 	for (cs_tmp = 0; cs_tmp < cs; cs_tmp++) {
334 		if (dram_info->cs_ena & (1 << cs_tmp))
335 			cs_count++;
336 	}
337 	sdram_offset = cs_count * (SDRAM_CS_SIZE + 1);
338 	sdram_offset += ((is_tx == 1) ?
339 			 SDRAM_DQS_TX_OFFS : SDRAM_DQS_RX_OFFS);
340 
341 	/* Prepare pup masks */
342 	for (pup = 0; pup < max_pup; pup++)
343 		pup_mask |= (1 << pup);
344 
345 	for (pup = 0; pup < max_pup; pup++) {
346 		for (dq = 0; dq < DQ_NUM; dq++) {
347 			analog_pbs_sum[pup][dq][0] = adll_start_val;
348 			analog_pbs_sum[pup][dq][1] = adll_end_val;
349 		}
350 	}
351 
352 	/* Loop - use different pattern for each victim_dq */
353 	for (victim_dq = 0; victim_dq < DQ_NUM; victim_dq++) {
354 		DEBUG_DQS_FULL_C("DDR3 - DQS Find Limits - Victim DQ - ",
355 				 (u32)victim_dq, 1);
356 		/*
357 		 * The pups 3 bit arrays represent state machine. with
358 		 * 3 stages for each pup.
359 		 * 1. fail and didn't get pass in earlier compares.
360 		 * 2. pass compare
361 		 * 3. fail after pass - end state.
362 		 * The window limits are the adll values where the adll
363 		 * was in the pass stage.
364 		 */
365 
366 		/* Set all states to Fail (1st state) */
367 		for (pup = 0; pup < max_pup; pup++)
368 			pup_adll_limit_state[pup] = PUP_ADLL_LIMITS_STATE_FAIL;
369 
370 		/* Set current valid pups */
371 		unlock_pup = pup_mask;
372 
373 		/* Set ADLL to start value */
374 		curr_adll = adll_start_val;
375 
376 #if defined(MV88F78X60)
377 		for (pup = 0; pup < max_pup; pup++) {
378 			for (dq = 0; dq < DQ_NUM; dq++) {
379 				analog_pbs[victim_dq][pup][dq][0] =
380 					adll_start_val;
381 				analog_pbs[victim_dq][pup][dq][1] =
382 					adll_end_val;
383 				per_bit_data[pup][dq] = 0;
384 			}
385 		}
386 #endif
387 
388 		for (uj = 0; uj < ADLL_MAX; uj++) {
389 			DEBUG_DQS_FULL_C("DDR3 - DQS Find Limits - Setting ADLL to ",
390 					 curr_adll, 2);
391 			for (pup = 0; pup < max_pup; pup++) {
392 				if (IS_PUP_ACTIVE(unlock_pup, pup) == 1) {
393 					tmp = ((is_tx == 1) ? curr_adll +
394 					       dram_info->wl_val[cs]
395 					       [pup * (1 - ecc) + ecc * ECC_PUP]
396 					       [D] : curr_adll);
397 					ddr3_write_pup_reg(adll_addr, cs, pup +
398 						(ecc * ECC_PUP), 0, tmp);
399 				}
400 			}
401 
402 			/* Choose pattern */
403 			pattern_ptr = ddr3_dqs_choose_pattern(dram_info,
404 							      victim_dq);
405 
406 			/* '1' - means pup failed, '0' - means pup pass */
407 			new_unlock_pup = 0;
408 
409 			/* Read and compare results for Victim_DQ# */
410 			for (ii = 0; ii < 3; ii++) {
411 				u32 tmp = 0;
412 				if (MV_OK != ddr3_sdram_dqs_compare(dram_info,
413 							   unlock_pup, &tmp,
414 							   pattern_ptr,
415 							   LEN_KILLER_PATTERN,
416 							   sdram_offset +
417 							   LEN_KILLER_PATTERN *
418 							   4 * victim_dq,
419 							   is_tx, 0, NULL,
420 							   0))
421 					return MV_DDR3_TRAINING_ERR_DRAM_COMPARE;
422 
423 				new_unlock_pup |= tmp;
424 			}
425 
426 			pup = 0;
427 			DEBUG_DQS_FULL_C("DDR3 - DQS Find Limits - UnlockPup: ",
428 					 unlock_pup, 2);
429 			DEBUG_DQS_FULL_C("DDR3 - DQS Find Limits - NewUnlockPup: ",
430 					 new_unlock_pup, 2);
431 
432 			/* Update pup state */
433 			for (pup = 0; pup < max_pup; pup++) {
434 				if (IS_PUP_ACTIVE(unlock_pup, pup) == 0) {
435 					DEBUG_DQS_FULL_C("DDR3 - DQS Find Limits - Skipping pup ",
436 							 pup, 1);
437 					continue;
438 				}
439 
440 				/*
441 				 * Still didn't find the window limit of the pup
442 				 */
443 				if (IS_PUP_ACTIVE(new_unlock_pup, pup) == 1) {
444 					/* Current compare result == fail */
445 					if (pup_adll_limit_state[pup] ==
446 					    PUP_ADLL_LIMITS_STATE_PASS) {
447 						/*
448 						 * If now it failed but passed
449 						 * earlier
450 						 */
451 						DEBUG_DQS_S("DDR3 - DQS Find Limits - PASS to FAIL: CS - ");
452 						DEBUG_DQS_D(cs, 1);
453 						DEBUG_DQS_S(", DQ - ");
454 						DEBUG_DQS_D(victim_dq, 1);
455 						DEBUG_DQS_S(", Pup - ");
456 						DEBUG_DQS_D(pup, 1);
457 						DEBUG_DQS_S(", ADLL - ");
458 						DEBUG_DQS_D(curr_adll, 2);
459 						DEBUG_DQS_S("\n");
460 
461 #if defined(MV88F78X60)
462 						for (dq = 0; dq < DQ_NUM; dq++) {
463 							if ((analog_pbs[victim_dq][pup][dq][0] != adll_start_val)
464 							    && (analog_pbs[victim_dq][pup]
465 								[dq][1] == adll_end_val))
466 								analog_pbs
467 									[victim_dq]
468 									[pup][dq]
469 									[1] =
470 									curr_adll;
471 						}
472 #endif
473 						win_valid = 1;
474 						update_win = 0;
475 
476 						/* Keep min / max limit value */
477 						if (is_tx == 0) {
478 							/* RX - found upper limit */
479 							if (centralization_high_limit[pup] >
480 							    (curr_adll - 1)) {
481 								high_limit =
482 									curr_adll - 1;
483 								low_limit =
484 									centralization_low_limit[pup];
485 								update_win = 1;
486 							}
487 						} else {
488 							/* TX - found lower limit */
489 							if (centralization_low_limit[pup] < (curr_adll + 1)) {
490 								high_limit =
491 									centralization_high_limit
492 									[pup];
493 								low_limit =
494 									curr_adll + 1;
495 								update_win =
496 									1;
497 							}
498 						}
499 
500 						if (update_win == 1) {
501 							/*
502 							 * Before updating
503 							 * window limits we need
504 							 * to check that the
505 							 * limits are valid
506 							 */
507 							if (MV_OK !=
508 							    ddr3_check_window_limits
509 							    (pup, high_limit,
510 							     low_limit, is_tx,
511 							     &win_valid))
512 								return MV_DDR3_TRAINING_ERR_WIN_LIMITS;
513 
514 							if (win_valid == 1) {
515 								/*
516 								 * Window limits
517 								 * should be
518 								 * updated
519 								 */
520 								centralization_low_limit
521 									[pup] =
522 									low_limit;
523 								centralization_high_limit
524 									[pup] =
525 									high_limit;
526 							}
527 						}
528 
529 						if (win_valid == 1) {
530 							/* Found end of window - lock the pup */
531 							pup_adll_limit_state[pup] =
532 								PUP_ADLL_LIMITS_STATE_FAIL_AFTER_PASS;
533 							unlock_pup &= ~(1 << pup);
534 						} else {
535 							/* Probably false pass - reset status */
536 							pup_adll_limit_state[pup] =
537 								PUP_ADLL_LIMITS_STATE_FAIL;
538 
539 #if defined(MV88F78X60)
540 							/* Clear logging array of win size (per Dq) */
541 							for (dq = 0;
542 							     dq < DQ_NUM;
543 							     dq++) {
544 								analog_pbs
545 									[victim_dq]
546 									[pup][dq]
547 									[0] =
548 									adll_start_val;
549 								analog_pbs
550 									[victim_dq]
551 									[pup][dq]
552 									[1] =
553 									adll_end_val;
554 								per_bit_data
555 									[pup][dq]
556 									= 0;
557 							}
558 #endif
559 						}
560 					}
561 				} else {
562 					/* Current compare result == pass */
563 					if (pup_adll_limit_state[pup] ==
564 					    PUP_ADLL_LIMITS_STATE_FAIL) {
565 						/* If now it passed but failed earlier */
566 						DEBUG_DQS_S("DDR3 - DQS Find Limits - FAIL to PASS: CS - ");
567 						DEBUG_DQS_D(cs, 1);
568 						DEBUG_DQS_S(", DQ - ");
569 						DEBUG_DQS_D(victim_dq, 1);
570 						DEBUG_DQS_S(", Pup - ");
571 						DEBUG_DQS_D(pup, 1);
572 						DEBUG_DQS_S(", ADLL - ");
573 						DEBUG_DQS_D(curr_adll, 2);
574 						DEBUG_DQS_S("\n");
575 
576 #if defined(MV88F78X60)
577 						for (dq = 0; dq < DQ_NUM;
578 						     dq++) {
579 							if (analog_pbs[victim_dq][pup][dq][0] == adll_start_val)
580 								analog_pbs
581 								    [victim_dq]
582 								    [pup][dq]
583 								    [0] =
584 								    curr_adll;
585 						}
586 #endif
587 						/* Found start of window */
588 						pup_adll_limit_state[pup] =
589 						    PUP_ADLL_LIMITS_STATE_PASS;
590 
591 						/* Keep min / max limit value */
592 						if (is_tx == 0) {
593 							/* RX - found low limit */
594 							if (centralization_low_limit[pup] <= curr_adll)
595 								centralization_low_limit
596 								    [pup] =
597 								    curr_adll;
598 						} else {
599 							/* TX - found high limit */
600 							if (centralization_high_limit[pup] >= curr_adll)
601 								centralization_high_limit
602 								    [pup] =
603 								    curr_adll;
604 						}
605 					}
606 				}
607 			}
608 
609 			if (unlock_pup == 0) {
610 				/* Found limit to all pups */
611 				DEBUG_DQS_FULL_S("DDR3 - DQS Find Limits - found PUP limit\n");
612 				break;
613 			}
614 
615 			/*
616 			 * Increment / decrement (Move to right / left
617 			 * one phase - ADLL) dqs RX / TX delay (for all un
618 			 * lock pups
619 			 */
620 			if (is_tx == 0)
621 				curr_adll++;
622 			else
623 				curr_adll--;
624 		}
625 
626 		if (unlock_pup != 0) {
627 			/*
628 			 * Found pups that didn't reach to the end of the
629 			 * state machine
630 			 */
631 			DEBUG_DQS_C("DDR3 - DQS Find Limits - Pups that didn't reached end of the state machine: ",
632 				    unlock_pup, 1);
633 
634 			for (pup = 0; pup < max_pup; pup++) {
635 				if (IS_PUP_ACTIVE(unlock_pup, pup) == 1) {
636 					if (pup_adll_limit_state[pup] ==
637 					    PUP_ADLL_LIMITS_STATE_FAIL) {
638 						/* ERROR - found fail for all window size */
639 						DEBUG_DQS_S("DDR3 - DQS Find Limits - Got FAIL for the complete range on pup - ");
640 						DEBUG_DQS_D(pup, 1);
641 						DEBUG_DQS_C(" victim DQ ",
642 							    victim_dq, 1);
643 
644 						/* For debug - set min limit to illegal limit */
645 						centralization_low_limit[pup]
646 							= ADLL_ERROR;
647 						/*
648 						 * In case the pup is in mode
649 						 * PASS - the limit is the min
650 						 * / max adll, no need to
651 						 * update because of the results
652 						 * array default value
653 						 */
654 						return MV_DDR3_TRAINING_ERR_PUP_RANGE;
655 					}
656 				}
657 			}
658 		}
659 	}
660 
661 	DEBUG_DQS_S("DDR3 - DQS Find Limits - DQ values per victim results:\n");
662 	for (victim_dq = 0; victim_dq < DQ_NUM; victim_dq++) {
663 		for (pup = 0; pup < max_pup; pup++) {
664 			DEBUG_DQS_S("Victim DQ-");
665 			DEBUG_DQS_D(victim_dq, 1);
666 			DEBUG_DQS_S(", PUP-");
667 			DEBUG_DQS_D(pup, 1);
668 			for (dq = 0; dq < DQ_NUM; dq++) {
669 				DEBUG_DQS_S(", DQ-");
670 				DEBUG_DQS_D(dq, 1);
671 				DEBUG_DQS_S(",S-");
672 				DEBUG_DQS_D(analog_pbs[victim_dq][pup][dq]
673 					    [0], 2);
674 				DEBUG_DQS_S(",E-");
675 				DEBUG_DQS_D(analog_pbs[victim_dq][pup][dq]
676 					    [1], 2);
677 
678 				if (is_tx == 0) {
679 					if (analog_pbs[victim_dq][pup][dq][0]
680 					    > analog_pbs_sum[pup][dq][0])
681 						analog_pbs_sum[pup][dq][0] =
682 						    analog_pbs[victim_dq][pup]
683 						    [dq][0];
684 					if (analog_pbs[victim_dq][pup][dq][1]
685 					    < analog_pbs_sum[pup][dq][1])
686 						analog_pbs_sum[pup][dq][1] =
687 						    analog_pbs[victim_dq][pup]
688 						    [dq][1];
689 				} else {
690 					if (analog_pbs[victim_dq][pup][dq][0]
691 					    < analog_pbs_sum[pup][dq][0])
692 						analog_pbs_sum[pup][dq][0] =
693 						    analog_pbs[victim_dq][pup]
694 						    [dq][0];
695 					if (analog_pbs[victim_dq][pup][dq][1]
696 					    > analog_pbs_sum[pup][dq][1])
697 						analog_pbs_sum[pup][dq][1] =
698 						    analog_pbs[victim_dq][pup]
699 						    [dq][1];
700 				}
701 			}
702 			DEBUG_DQS_S("\n");
703 		}
704 	}
705 
706 	if (ddr3_get_log_level() >= MV_LOG_LEVEL_3) {
707 		u32 dq;
708 
709 		DEBUG_PER_DQ_S("\n########## LOG LEVEL 3(Windows margins per-DQ) ##########\n");
710 		if (is_tx) {
711 			DEBUG_PER_DQ_C("DDR3 - TX  CS: ", cs, 1);
712 		} else {
713 			DEBUG_PER_DQ_C("DDR3 - RX  CS: ", cs, 1);
714 		}
715 
716 		if (ecc == 0) {
717 			DEBUG_PER_DQ_S("\n DATA RESULTS:\n");
718 		} else {
719 			DEBUG_PER_DQ_S("\n ECC RESULTS:\n");
720 		}
721 
722 		/* Since all dq has the same value we take 0 as representive */
723 		dq = 0;
724 		for (pup = 0; pup < max_pup; pup++) {
725 			if (ecc == 0) {
726 				DEBUG_PER_DQ_S("\nBYTE:");
727 				DEBUG_PER_DQ_D(pup, 1);
728 				DEBUG_PER_DQ_S("\n");
729 			} else {
730 				DEBUG_PER_DQ_S("\nECC BYTE:\n");
731 			}
732 			DEBUG_PER_DQ_S("  DQ's        LOW       HIGH       WIN-SIZE\n");
733 			DEBUG_PER_DQ_S("============================================\n");
734 			for (victim_dq = 0; victim_dq < DQ_NUM; victim_dq++) {
735 				if (ecc == 0) {
736 					DEBUG_PER_DQ_S("DQ[");
737 					DEBUG_PER_DQ_DD((victim_dq +
738 							 DQ_NUM * pup), 2);
739 					DEBUG_PER_DQ_S("]");
740 				} else {
741 					DEBUG_PER_DQ_S("CB[");
742 					DEBUG_PER_DQ_DD(victim_dq, 2);
743 					DEBUG_PER_DQ_S("]");
744 				}
745 				if (is_tx) {
746 					DEBUG_PER_DQ_S("      0x");
747 					DEBUG_PER_DQ_D(analog_pbs[victim_dq][pup][dq][1], 2);	/* low value */
748 					DEBUG_PER_DQ_S("        0x");
749 					DEBUG_PER_DQ_D(analog_pbs[victim_dq][pup][dq][0], 2);	/* high value */
750 					DEBUG_PER_DQ_S("        0x");
751 					DEBUG_PER_DQ_D(analog_pbs[victim_dq][pup][dq][0] - analog_pbs[victim_dq][pup][dq][1], 2);	/* win-size */
752 				} else {
753 					DEBUG_PER_DQ_S("     0x");
754 					DEBUG_PER_DQ_D(analog_pbs[victim_dq][pup][dq][0], 2);	/* low value */
755 					DEBUG_PER_DQ_S("       0x");
756 					DEBUG_PER_DQ_D((analog_pbs[victim_dq][pup][dq][1] - 1), 2);	/* high value */
757 					DEBUG_PER_DQ_S("       0x");
758 					DEBUG_PER_DQ_D(analog_pbs[victim_dq][pup][dq][1] - analog_pbs[victim_dq][pup][dq][0], 2);	/* win-size */
759 				}
760 				DEBUG_PER_DQ_S("\n");
761 			}
762 		}
763 		DEBUG_PER_DQ_S("\n");
764 	}
765 
766 	if (is_tx) {
767 		DEBUG_DQS_S("DDR3 - DQS TX - Find Limits - DQ values Summary:\n");
768 	} else {
769 		DEBUG_DQS_S("DDR3 - DQS RX - Find Limits - DQ values Summary:\n");
770 	}
771 
772 	for (pup = 0; pup < max_pup; pup++) {
773 		DEBUG_DQS_S("PUP-");
774 		DEBUG_DQS_D(pup, 1);
775 		for (dq = 0; dq < DQ_NUM; dq++) {
776 			DEBUG_DQS_S(", DQ-");
777 			DEBUG_DQS_D(dq, 1);
778 			DEBUG_DQS_S(",S-");
779 			DEBUG_DQS_D(analog_pbs_sum[pup][dq][0], 2);
780 			DEBUG_DQS_S(",E-");
781 			DEBUG_DQS_D(analog_pbs_sum[pup][dq][1], 2);
782 		}
783 		DEBUG_DQS_S("\n");
784 	}
785 
786 	if (is_tx) {
787 		DEBUG_DQS_S("DDR3 - DQS TX - Find Limits - DQ values Summary:\n");
788 	} else {
789 		DEBUG_DQS_S("DDR3 - DQS RX - Find Limits - DQ values Summary:\n");
790 	}
791 
792 	for (pup = 0; pup < max_pup; pup++) {
793 		if (max_pup == 1) {
794 			/* For ECC PUP */
795 			DEBUG_DQS_S("DDR3 - DQS8");
796 		} else {
797 			DEBUG_DQS_S("DDR3 - DQS");
798 			DEBUG_DQS_D(pup, 1);
799 		}
800 
801 		for (dq = 0; dq < DQ_NUM; dq++) {
802 			DEBUG_DQS_S(", DQ-");
803 			DEBUG_DQS_D(dq, 1);
804 			DEBUG_DQS_S("::S-");
805 			DEBUG_DQS_D(analog_pbs_sum[pup][dq][0], 2);
806 			DEBUG_DQS_S(",E-");
807 			DEBUG_DQS_D(analog_pbs_sum[pup][dq][1], 2);
808 		}
809 		DEBUG_DQS_S("\n");
810 	}
811 
812 	DEBUG_DQS_S("DDR3 - DQS Find Limits - Ended\n");
813 
814 	return MV_OK;
815 }
816 
817 /*
818  * Name:     ddr3_check_window_limits
819  * Desc:     Check window High & Low limits.
820  * Args:     pup                pup index
821  *           high_limit           window high limit
822  *           low_limit            window low limit
823  *           is_tx                Indicate whether Rx or Tx
824  *           size_valid          Indicate whether window size is valid
825  * Notes:
826  * Returns:  MV_OK if success, other error code if fail.
827  */
828 int ddr3_check_window_limits(u32 pup, int high_limit, int low_limit, int is_tx,
829 			     int *size_valid)
830 {
831 	DEBUG_DQS_FULL_S("DDR3 - DQS Check Win Limits - Starting\n");
832 
833 	if (low_limit > high_limit) {
834 		DEBUG_DQS_S("DDR3 - DQS Check Win Limits - Pup ");
835 		DEBUG_DQS_D(pup, 1);
836 		DEBUG_DQS_S(" Low Limit grater than High Limit\n");
837 		*size_valid = 0;
838 		return MV_OK;
839 	}
840 
841 	/*
842 	 * Check that window size is valid, if not it was probably false pass
843 	 * before
844 	 */
845 	if ((high_limit - low_limit) < MIN_WIN_SIZE) {
846 		/*
847 		 * Since window size is too small probably there was false
848 		 * pass
849 		 */
850 		*size_valid = 0;
851 
852 		DEBUG_DQS_S("DDR3 - DQS Check Win Limits - Pup ");
853 		DEBUG_DQS_D(pup, 1);
854 		DEBUG_DQS_S(" Window size is smaller than MIN_WIN_SIZE\n");
855 
856 	} else if ((high_limit - low_limit) > ADLL_MAX) {
857 		*size_valid = 0;
858 
859 		DEBUG_DQS_S("DDR3 - DQS Check Win Limits - Pup ");
860 		DEBUG_DQS_D(pup, 1);
861 		DEBUG_DQS_S
862 		    (" Window size is bigger than max ADLL taps (31)  Exiting.\n");
863 
864 		return MV_FAIL;
865 
866 	} else {
867 		*size_valid = 1;
868 
869 		DEBUG_DQS_FULL_S("DDR3 - DQS Check Win Limits - Pup ");
870 		DEBUG_DQS_FULL_D(pup, 1);
871 		DEBUG_DQS_FULL_C(" window size is ", (high_limit - low_limit),
872 				 2);
873 	}
874 
875 	return MV_OK;
876 }
877 
878 /*
879  * Name:     ddr3_center_calc
880  * Desc:     Execute the calculate the center of windows phase.
881  * Args:     pDram Info
882  *           is_tx             Indicate whether Rx or Tx
883  * Notes:
884  * Returns:  MV_OK if success, other error code if fail.
885  */
886 static int ddr3_center_calc(MV_DRAM_INFO *dram_info, u32 cs, u32 ecc,
887 			    int is_tx)
888 {
889 	/* bit array of pups that need specail search */
890 	u32 special_pattern_i_pup = 0;
891 	u32 special_pattern_ii_pup = 0;
892 	u32 pup;
893 	u32 max_pup;
894 
895 	max_pup = (ecc + (1 - ecc) * dram_info->num_of_std_pups);
896 
897 	for (pup = 0; pup < max_pup; pup++) {
898 		if (is_tx == 0) {
899 			/* Check special pattern I */
900 			/*
901 			 * Special pattern Low limit search - relevant only
902 			 * for Rx, win size < threshold and low limit = 0
903 			 */
904 			if (((centralization_high_limit[pup] -
905 			      centralization_low_limit[pup]) < VALID_WIN_THRS)
906 			    && (centralization_low_limit[pup] == MIN_DELAY))
907 				special_pattern_i_pup |= (1 << pup);
908 
909 			/* Check special pattern II */
910 			/*
911 			 * Special pattern High limit search - relevant only
912 			 * for Rx, win size < threshold and high limit = 31
913 			 */
914 			if (((centralization_high_limit[pup] -
915 			      centralization_low_limit[pup]) < VALID_WIN_THRS)
916 			    && (centralization_high_limit[pup] == MAX_DELAY))
917 				special_pattern_ii_pup |= (1 << pup);
918 		}
919 	}
920 
921 	/* Run special pattern Low limit search - for relevant pup */
922 	if (special_pattern_i_pup != 0) {
923 		DEBUG_DQS_S("DDR3 - DQS Center Calc - Entering special pattern I for Low limit search\n");
924 		if (MV_OK !=
925 		    ddr3_special_pattern_i_search(dram_info, cs, ecc, is_tx,
926 					      special_pattern_i_pup))
927 			return MV_DDR3_TRAINING_ERR_DQS_LOW_LIMIT_SEARCH;
928 	}
929 
930 	/* Run special pattern High limit search - for relevant pup */
931 	if (special_pattern_ii_pup != 0) {
932 		DEBUG_DQS_S("DDR3 - DQS Center Calc - Entering special pattern II for High limit search\n");
933 		if (MV_OK !=
934 		    ddr3_special_pattern_ii_search(dram_info, cs, ecc, is_tx,
935 						   special_pattern_ii_pup))
936 			return MV_DDR3_TRAINING_ERR_DQS_HIGH_LIMIT_SEARCH;
937 	}
938 
939 	/* Set adll to center = (General_High_limit + General_Low_limit)/2 */
940 	return ddr3_set_dqs_centralization_results(dram_info, cs, ecc, is_tx);
941 }
942 
943 /*
944  * Name:     ddr3_special_pattern_i_search
945  * Desc:     Execute special pattern low limit search.
946  * Args:
947  *           special_pattern_pup  The pups that need the special search
948  * Notes:
949  * Returns:  MV_OK if success, other error code if fail.
950  */
951 int ddr3_special_pattern_i_search(MV_DRAM_INFO *dram_info, u32 cs, u32 ecc,
952 				  int is_tx, u32 special_pattern_pup)
953 {
954 	u32 victim_dq;		/* loop index - victim DQ */
955 	u32 adll_idx;
956 	u32 pup;
957 	u32 unlock_pup;		/* bit array of the unlock pups  */
958 	u32 first_fail;	/* bit array - of pups that  get first fail */
959 	u32 new_lockup_pup;	/* bit array of compare failed pups */
960 	u32 pass_pup;		/* bit array of compare pass pup */
961 	u32 sdram_offset;
962 	u32 max_pup;
963 	u32 comp_val;
964 	u32 special_res[MAX_PUP_NUM];	/* hold tmp results */
965 
966 	DEBUG_DQS_S("DDR3 - DQS - Special Pattern I Search - Starting\n");
967 
968 	max_pup = ecc + (1 - ecc) * dram_info->num_of_std_pups;
969 
970 	/* Init the temporary results to max ADLL value */
971 	for (pup = 0; pup < max_pup; pup++)
972 		special_res[pup] = ADLL_MAX;
973 
974 	/* Run special pattern for all DQ - use the same pattern */
975 	for (victim_dq = 0; victim_dq < DQ_NUM; victim_dq++) {
976 		unlock_pup = special_pattern_pup;
977 		first_fail = 0;
978 
979 		sdram_offset = cs * SDRAM_CS_SIZE + SDRAM_DQS_RX_OFFS +
980 			LEN_KILLER_PATTERN * 4 * victim_dq;
981 
982 		for (pup = 0; pup < max_pup; pup++) {
983 			/* Set adll value per PUP. adll = high limit per pup */
984 			if (IS_PUP_ACTIVE(unlock_pup, pup)) {
985 				/* only for pups that need special search */
986 				ddr3_write_pup_reg(PUP_DQS_RD, cs,
987 						   pup + (ecc * ECC_PUP), 0,
988 						   centralization_high_limit
989 						   [pup]);
990 			}
991 		}
992 
993 		adll_idx = 0;
994 		do {
995 			/*
996 			 * Perform read and compare simultaneously for all
997 			 * un-locked MC use the special pattern mask
998 			 */
999 			new_lockup_pup = 0;
1000 
1001 			if (MV_OK !=
1002 			    ddr3_sdram_dqs_compare(dram_info, unlock_pup,
1003 						   &new_lockup_pup,
1004 						   special_pattern
1005 						   [victim_dq],
1006 						   LEN_SPECIAL_PATTERN,
1007 						   sdram_offset, 0,
1008 						   0, NULL, 1))
1009 				return MV_FAIL;
1010 
1011 			DEBUG_DQS_S("DDR3 - DQS - Special I - ADLL value is: ");
1012 			DEBUG_DQS_D(adll_idx, 2);
1013 			DEBUG_DQS_S(", UnlockPup: ");
1014 			DEBUG_DQS_D(unlock_pup, 2);
1015 			DEBUG_DQS_S(", NewLockPup: ");
1016 			DEBUG_DQS_D(new_lockup_pup, 2);
1017 			DEBUG_DQS_S("\n");
1018 
1019 			if (unlock_pup != new_lockup_pup)
1020 				DEBUG_DQS_S("DDR3 - DQS - Special I - Some Pup passed!\n");
1021 
1022 			/* Search for pups with passed compare & already fail */
1023 			pass_pup = first_fail & ~new_lockup_pup & unlock_pup;
1024 			first_fail |= new_lockup_pup;
1025 			unlock_pup &= ~pass_pup;
1026 
1027 			/* Get pass pups */
1028 			if (pass_pup != 0) {
1029 				for (pup = 0; pup < max_pup; pup++) {
1030 					if (IS_PUP_ACTIVE(pass_pup, pup) ==
1031 					    1) {
1032 						/* If pup passed and has first fail = 1 */
1033 						/* keep min value of ADLL max value - current adll */
1034 						/* (centralization_high_limit[pup] + adll_idx) = current adll !!! */
1035 						comp_val =
1036 						    (ADLL_MAX -
1037 						     (centralization_high_limit
1038 						      [pup] + adll_idx));
1039 
1040 						DEBUG_DQS_C
1041 						    ("DDR3 - DQS - Special I - Pup - ",
1042 						     pup, 1);
1043 						DEBUG_DQS_C
1044 						    (" comp_val = ",
1045 						     comp_val, 2);
1046 
1047 						if (comp_val <
1048 						    special_res[pup]) {
1049 							special_res[pup] =
1050 							    comp_val;
1051 							centralization_low_limit
1052 							    [pup] =
1053 							    (-1) *
1054 							    comp_val;
1055 
1056 							DEBUG_DQS_C
1057 							    ("DDR3 - DQS - Special I - Pup - ",
1058 							     pup, 1);
1059 							DEBUG_DQS_C
1060 							    (" Changed Low limit to ",
1061 							     centralization_low_limit
1062 							     [pup], 2);
1063 						}
1064 					}
1065 				}
1066 			}
1067 
1068 			/*
1069 			 * Did all PUP found missing window?
1070 			 * Check for each pup if adll (different for each pup)
1071 			 * reach maximum if reach max value - lock the pup
1072 			 * if not - increment (Move to right one phase - ADLL)
1073 			 * dqs RX delay
1074 			 */
1075 			adll_idx++;
1076 			for (pup = 0; pup < max_pup; pup++) {
1077 				if (IS_PUP_ACTIVE(unlock_pup, pup) == 1) {
1078 					/* Check only unlocked pups */
1079 					if ((centralization_high_limit[pup] +
1080 					     adll_idx) >= ADLL_MAX) {
1081 						/* reach maximum - lock the pup */
1082 						DEBUG_DQS_C("DDR3 - DQS - Special I - reach maximum - lock pup ",
1083 							    pup, 1);
1084 						unlock_pup &= ~(1 << pup);
1085 					} else {
1086 						/* Didn't reach maximum - increment ADLL */
1087 						ddr3_write_pup_reg(PUP_DQS_RD,
1088 								   cs,
1089 								   pup +
1090 								   (ecc *
1091 								    ECC_PUP), 0,
1092 								   (centralization_high_limit
1093 								    [pup] +
1094 								    adll_idx));
1095 					}
1096 				}
1097 			}
1098 		} while (unlock_pup != 0);
1099 	}
1100 
1101 	return MV_OK;
1102 }
1103 
1104 /*
1105  * Name:     ddr3_special_pattern_ii_search
1106  * Desc:     Execute special pattern high limit search.
1107  * Args:
1108  *           special_pattern_pup  The pups that need the special search
1109  * Notes:
1110  * Returns:  MV_OK if success, other error code if fail.
1111  */
1112 int ddr3_special_pattern_ii_search(MV_DRAM_INFO *dram_info, u32 cs, u32 ecc,
1113 				   int is_tx, u32 special_pattern_pup)
1114 {
1115 	u32 victim_dq;		/* loop index - victim DQ */
1116 	u32 adll_idx;
1117 	u32 pup;
1118 	u32 unlock_pup;		/* bit array of the unlock pups  */
1119 	u32 first_fail;	/* bit array - of pups that  get first fail */
1120 	u32 new_lockup_pup;	/* bit array of compare failed pups */
1121 	u32 pass_pup;		/* bit array of compare pass pup */
1122 	u32 sdram_offset;
1123 	u32 max_pup;
1124 	u32 comp_val;
1125 	u32 special_res[MAX_PUP_NUM];	/* hold tmp results */
1126 
1127 	DEBUG_DQS_S("DDR3 - DQS - Special Pattern II Search - Starting\n");
1128 
1129 	max_pup = (ecc + (1 - ecc) * dram_info->num_of_std_pups);
1130 
1131 	/* init the tmporary results to max ADLL value */
1132 	for (pup = 0; pup < max_pup; pup++)
1133 		special_res[pup] = ADLL_MAX;
1134 
1135 	sdram_offset = cs * SDRAM_CS_SIZE + SDRAM_DQS_RX_OFFS;
1136 
1137 	/* run special pattern for all DQ - use the same pattern */
1138 	for (victim_dq = 0; victim_dq < DQ_NUM; victim_dq++) {
1139 		unlock_pup = special_pattern_pup;
1140 		first_fail = 0;
1141 
1142 		for (pup = 0; pup < max_pup; pup++) {
1143 			/* Set adll value per PUP. adll = 0 */
1144 			if (IS_PUP_ACTIVE(unlock_pup, pup)) {
1145 				/* Only for pups that need special search */
1146 				ddr3_write_pup_reg(PUP_DQS_RD, cs,
1147 						   pup + (ecc * ECC_PUP), 0,
1148 						   ADLL_MIN);
1149 			}
1150 		}
1151 
1152 		adll_idx = 0;
1153 		do {
1154 			/*
1155 			 * Perform read and compare simultaneously for all
1156 			 * un-locked MC use the special pattern mask
1157 			 */
1158 			new_lockup_pup = 0;
1159 
1160 			if (MV_OK != ddr3_sdram_dqs_compare(
1161 				    dram_info, unlock_pup, &new_lockup_pup,
1162 				    special_pattern[victim_dq],
1163 				    LEN_SPECIAL_PATTERN,
1164 				    sdram_offset, 0, 0, NULL, 0))
1165 				return MV_FAIL;
1166 
1167 			DEBUG_DQS_S("DDR3 - DQS - Special II - ADLL value is ");
1168 			DEBUG_DQS_D(adll_idx, 2);
1169 			DEBUG_DQS_S("unlock_pup ");
1170 			DEBUG_DQS_D(unlock_pup, 1);
1171 			DEBUG_DQS_S("new_lockup_pup ");
1172 			DEBUG_DQS_D(new_lockup_pup, 1);
1173 			DEBUG_DQS_S("\n");
1174 
1175 			if (unlock_pup != new_lockup_pup) {
1176 				DEBUG_DQS_S("DDR3 - DQS - Special II - Some Pup passed!\n");
1177 			}
1178 
1179 			/* Search for pups with passed compare & already fail */
1180 			pass_pup = first_fail & ~new_lockup_pup & unlock_pup;
1181 			first_fail |= new_lockup_pup;
1182 			unlock_pup &= ~pass_pup;
1183 
1184 			/* Get pass pups */
1185 			if (pass_pup != 0) {
1186 				for (pup = 0; pup < max_pup; pup++) {
1187 					if (IS_PUP_ACTIVE(pass_pup, pup) ==
1188 					    1) {
1189 						/* If pup passed and has first fail = 1 */
1190 						/* keep min value of ADLL max value - current adll */
1191 						/* (adll_idx) = current adll !!! */
1192 						comp_val = adll_idx;
1193 
1194 						DEBUG_DQS_C("DDR3 - DQS - Special II - Pup - ",
1195 							    pup, 1);
1196 						DEBUG_DQS_C(" comp_val = ",
1197 							    comp_val, 1);
1198 
1199 						if (comp_val <
1200 						    special_res[pup]) {
1201 							special_res[pup] =
1202 							    comp_val;
1203 							centralization_high_limit
1204 							    [pup] =
1205 							    ADLL_MAX +
1206 							    comp_val;
1207 
1208 							DEBUG_DQS_C
1209 							    ("DDR3 - DQS - Special II - Pup - ",
1210 							     pup, 1);
1211 							DEBUG_DQS_C
1212 							    (" Changed High limit to ",
1213 							     centralization_high_limit
1214 							     [pup], 2);
1215 						}
1216 					}
1217 				}
1218 			}
1219 
1220 			/*
1221 			 * Did all PUP found missing window?
1222 			 * Check for each pup if adll (different for each pup)
1223 			 * reach maximum if reach max value - lock the pup
1224 			 * if not - increment (Move to right one phase - ADLL)
1225 			 * dqs RX delay
1226 			 */
1227 			adll_idx++;
1228 			for (pup = 0; pup < max_pup; pup++) {
1229 				if (IS_PUP_ACTIVE(unlock_pup, pup) == 1) {
1230 					/* Check only unlocked pups */
1231 					if ((adll_idx) >= ADLL_MAX) {
1232 						/* Reach maximum - lock the pup */
1233 						DEBUG_DQS_C("DDR3 - DQS - Special II - reach maximum - lock pup ",
1234 							    pup, 1);
1235 						unlock_pup &= ~(1 << pup);
1236 					} else {
1237 						/* Didn't reach maximum - increment ADLL */
1238 						ddr3_write_pup_reg(PUP_DQS_RD,
1239 								   cs,
1240 								   pup +
1241 								   (ecc *
1242 								    ECC_PUP), 0,
1243 								   (adll_idx));
1244 					}
1245 				}
1246 			}
1247 		} while (unlock_pup != 0);
1248 	}
1249 
1250 	return MV_OK;
1251 }
1252 
1253 /*
1254  * Name:     ddr3_set_dqs_centralization_results
1255  * Desc:     Set to HW the DQS centralization phase results.
1256  * Args:
1257  *           is_tx             Indicates whether to set Tx or RX results
1258  * Notes:
1259  * Returns:  MV_OK if success, other error code if fail.
1260  */
1261 int ddr3_set_dqs_centralization_results(MV_DRAM_INFO *dram_info, u32 cs,
1262 					u32 ecc, int is_tx)
1263 {
1264 	u32 pup, pup_num;
1265 	int addl_val;
1266 	u32 max_pup;
1267 
1268 	max_pup = (ecc + (1 - ecc) * dram_info->num_of_std_pups);
1269 
1270 	DEBUG_DQS_RESULTS_S("\n############ LOG LEVEL 2(Windows margins) ############\n");
1271 
1272 	if (is_tx) {
1273 		DEBUG_DQS_RESULTS_C("DDR3 - DQS TX - Set Dqs Centralization Results - CS: ",
1274 				    cs, 1);
1275 	} else {
1276 		DEBUG_DQS_RESULTS_C("DDR3 - DQS RX - Set Dqs Centralization Results - CS: ",
1277 				    cs, 1);
1278 	}
1279 
1280 	/* Set adll to center = (General_High_limit + General_Low_limit)/2 */
1281 	DEBUG_DQS_RESULTS_S("\nDQS    LOW     HIGH     WIN-SIZE      Set\n");
1282 	DEBUG_DQS_RESULTS_S("==============================================\n");
1283 	for (pup = 0; pup < max_pup; pup++) {
1284 		addl_val = (centralization_high_limit[pup] +
1285 			    centralization_low_limit[pup]) / 2;
1286 
1287 		pup_num = pup * (1 - ecc) + ecc * ECC_PUP;
1288 
1289 		DEBUG_DQS_RESULTS_D(pup_num, 1);
1290 		DEBUG_DQS_RESULTS_S("     0x");
1291 		DEBUG_DQS_RESULTS_D(centralization_low_limit[pup], 2);
1292 		DEBUG_DQS_RESULTS_S("      0x");
1293 		DEBUG_DQS_RESULTS_D(centralization_high_limit[pup], 2);
1294 		DEBUG_DQS_RESULTS_S("      0x");
1295 		DEBUG_DQS_RESULTS_D(centralization_high_limit[pup] -
1296 				    centralization_low_limit[pup], 2);
1297 		DEBUG_DQS_RESULTS_S("       0x");
1298 		DEBUG_DQS_RESULTS_D(addl_val, 2);
1299 		DEBUG_DQS_RESULTS_S("\n");
1300 
1301 		if (addl_val < ADLL_MIN) {
1302 			addl_val = ADLL_MIN;
1303 			DEBUG_DQS_RESULTS_S("DDR3 - DQS - Setting ADLL value for Pup to MIN (since it was lower than 0)\n");
1304 		}
1305 
1306 		if (addl_val > ADLL_MAX) {
1307 			addl_val = ADLL_MAX;
1308 			DEBUG_DQS_RESULTS_S("DDR3 - DQS - Setting ADLL value for Pup to MAX (since it was higher than 31)\n");
1309 		}
1310 
1311 		if (is_tx) {
1312 			ddr3_write_pup_reg(PUP_DQS_WR, cs, pup_num, 0,
1313 					   addl_val +
1314 					   dram_info->wl_val[cs][pup_num][D]);
1315 		} else {
1316 			ddr3_write_pup_reg(PUP_DQS_RD, cs, pup_num, 0,
1317 					   addl_val);
1318 		}
1319 	}
1320 
1321 	return MV_OK;
1322 }
1323 
1324 /*
1325  * Set training patterns
1326  */
1327 int ddr3_load_dqs_patterns(MV_DRAM_INFO *dram_info)
1328 {
1329 	u32 cs, cs_count, cs_tmp, victim_dq;
1330 	u32 sdram_addr;
1331 	u32 *pattern_ptr;
1332 
1333 	/* Loop for each CS */
1334 	for (cs = 0; cs < MAX_CS; cs++) {
1335 		if (dram_info->cs_ena & (1 << cs)) {
1336 			cs_count = 0;
1337 			for (cs_tmp = 0; cs_tmp < cs; cs_tmp++) {
1338 				if (dram_info->cs_ena & (1 << cs_tmp))
1339 					cs_count++;
1340 			}
1341 
1342 			/* Init killer pattern */
1343 			sdram_addr = (cs_count * (SDRAM_CS_SIZE + 1) +
1344 				      SDRAM_DQS_RX_OFFS);
1345 			for (victim_dq = 0; victim_dq < DQ_NUM; victim_dq++) {
1346 				pattern_ptr = ddr3_dqs_choose_pattern(dram_info,
1347 								      victim_dq);
1348 				if (MV_OK != ddr3_sdram_dqs_compare(
1349 					    dram_info, (u32)NULL, NULL,
1350 					    pattern_ptr, LEN_KILLER_PATTERN,
1351 					    sdram_addr + LEN_KILLER_PATTERN *
1352 					    4 * victim_dq, 1, 0, NULL,
1353 					    0))
1354 					return MV_DDR3_TRAINING_ERR_DQS_PATTERN;
1355 			}
1356 
1357 			/* Init special-killer pattern */
1358 			sdram_addr = (cs_count * (SDRAM_CS_SIZE + 1) +
1359 				      SDRAM_DQS_RX_SPECIAL_OFFS);
1360 			for (victim_dq = 0; victim_dq < DQ_NUM; victim_dq++) {
1361 				if (MV_OK != ddr3_sdram_dqs_compare(
1362 					    dram_info, (u32)NULL, NULL,
1363 					    special_pattern[victim_dq],
1364 					    LEN_KILLER_PATTERN, sdram_addr +
1365 					    LEN_KILLER_PATTERN * 4 * victim_dq,
1366 					    1, 0, NULL, 0))
1367 					return MV_DDR3_TRAINING_ERR_DQS_PATTERN;
1368 			}
1369 		}
1370 	}
1371 
1372 	return MV_OK;
1373 }
1374