1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (C) Marvell International Ltd. and its affiliates
4  */
5 
6 #include <common.h>
7 #include <i2c.h>
8 #include <spl.h>
9 #include <asm/io.h>
10 #include <asm/arch/cpu.h>
11 #include <asm/arch/soc.h>
12 
13 #include "ddr3_hw_training.h"
14 
15 /*
16  * Debug
17  */
18 #define DEBUG_PBS_FULL_C(s, d, l) \
19 	DEBUG_PBS_FULL_S(s); DEBUG_PBS_FULL_D(d, l); DEBUG_PBS_FULL_S("\n")
20 #define DEBUG_PBS_C(s, d, l) \
21 	DEBUG_PBS_S(s); DEBUG_PBS_D(d, l); DEBUG_PBS_S("\n")
22 
23 #ifdef MV_DEBUG_PBS
24 #define DEBUG_PBS_S(s)			puts(s)
25 #define DEBUG_PBS_D(d, l)		printf("%x", d)
26 #else
27 #define DEBUG_PBS_S(s)
28 #define DEBUG_PBS_D(d, l)
29 #endif
30 
31 #ifdef MV_DEBUG_FULL_PBS
32 #define DEBUG_PBS_FULL_S(s)		puts(s)
33 #define DEBUG_PBS_FULL_D(d, l)		printf("%x", d)
34 #else
35 #define DEBUG_PBS_FULL_S(s)
36 #define DEBUG_PBS_FULL_D(d, l)
37 #endif
38 
39 #if defined(MV88F78X60) || defined(MV88F672X)
40 
41 /* Temp array for skew data storage */
42 static u32 skew_array[(MAX_PUP_NUM) * DQ_NUM] = { 0 };
43 
44 /* PBS locked dq (per pup) */
45 extern u32 pbs_locked_dq[MAX_PUP_NUM][DQ_NUM];
46 extern u32 pbs_locked_dm[MAX_PUP_NUM];
47 extern u32 pbs_locked_value[MAX_PUP_NUM][DQ_NUM];
48 
49 #if defined(MV88F672X)
50 extern u32 pbs_pattern[2][LEN_16BIT_PBS_PATTERN];
51 extern u32 pbs_pattern_32b[2][LEN_PBS_PATTERN];
52 #else
53 extern u32 pbs_pattern_32b[2][LEN_PBS_PATTERN];
54 extern u32 pbs_pattern_64b[2][LEN_PBS_PATTERN];
55 #endif
56 
57 extern u32 pbs_dq_mapping[PUP_NUM_64BIT + 1][DQ_NUM];
58 
59 static int ddr3_tx_shift_dqs_adll_step_before_fail(MV_DRAM_INFO *dram_info,
60 		u32 cur_pup, u32 pbs_pattern_idx, u32 ecc);
61 static int ddr3_rx_shift_dqs_to_first_fail(MV_DRAM_INFO *dram_info, u32 cur_pup,
62 		u32 pbs_pattern_idx, u32 ecc);
63 static int ddr3_pbs_per_bit(MV_DRAM_INFO *dram_info, int *start_over, int is_tx,
64 		u32 *pcur_pup, u32 pbs_pattern_idx, u32 ecc);
65 static int ddr3_set_pbs_results(MV_DRAM_INFO *dram_info, int is_tx);
66 static void ddr3_pbs_write_pup_dqs_reg(u32 cs, u32 pup, u32 dqs_delay);
67 
68 /*
69  * Name:     ddr3_pbs_tx
70  * Desc:     Execute the PBS TX phase.
71  * Args:     dram_info   ddr3 training information struct
72  * Notes:
73  * Returns:  MV_OK if success, other error code if fail.
74  */
75 int ddr3_pbs_tx(MV_DRAM_INFO *dram_info)
76 {
77 	/* Array of Deskew results */
78 
79 	/*
80 	 * Array to hold the total sum of skew from all iterations
81 	 * (for average purpose)
82 	 */
83 	u32 skew_sum_array[MAX_PUP_NUM][DQ_NUM] = { {0} };
84 
85 	/*
86 	 * Array to hold the total average skew from both patterns
87 	 * (for average purpose)
88 	 */
89 	u32 pattern_skew_array[MAX_PUP_NUM][DQ_NUM] = { {0} };
90 
91 	u32 pbs_rep_time = 0;	/* counts number of loop in case of fail */
92 	/* bit array for unlock pups - used to repeat on the RX operation */
93 	u32 cur_pup;
94 	u32 max_pup;
95 	u32 pbs_retry;
96 	u32 pup, dq, pups, cur_max_pup, valid_pup, reg;
97 	u32 pattern_idx;
98 	u32 ecc;
99 	/* indicates whether we need to start the loop again */
100 	int start_over;
101 
102 	DEBUG_PBS_S("DDR3 - PBS TX - Starting PBS TX procedure\n");
103 
104 	pups = dram_info->num_of_total_pups;
105 	max_pup = dram_info->num_of_total_pups;
106 
107 	/* Enable SW override */
108 	reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |
109 		(1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
110 	/* [0] = 1 - Enable SW override  */
111 	/* 0x15B8 - Training SW 2 Register */
112 	reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
113 	DEBUG_PBS_S("DDR3 - PBS RX - SW Override Enabled\n");
114 
115 	reg = 1 << REG_DRAM_TRAINING_AUTO_OFFS;
116 	reg_write(REG_DRAM_TRAINING_ADDR, reg);	/* 0x15B0 - Training Register */
117 
118 	/* Running twice for 2 different patterns. each patterns - 3 times */
119 	for (pattern_idx = 0; pattern_idx < COUNT_PBS_PATTERN; pattern_idx++) {
120 		DEBUG_PBS_C("DDR3 - PBS TX - Working with pattern - ",
121 			    pattern_idx, 1);
122 
123 		/* Reset sum array */
124 		for (pup = 0; pup < pups; pup++) {
125 			for (dq = 0; dq < DQ_NUM; dq++)
126 				skew_sum_array[pup][dq] = 0;
127 		}
128 
129 		/*
130 		 * Perform PBS several of times (3 for each pattern).
131 		 * At the end, we'll use the average
132 		 */
133 		/* If there is ECC, do each PBS again with mux change */
134 		for (pbs_retry = 0; pbs_retry < COUNT_PBS_REPEAT; pbs_retry++) {
135 			for (ecc = 0; ecc < (dram_info->ecc_ena + 1); ecc++) {
136 
137 				/*
138 				 * This parameter stores the current PUP
139 				 * num - ecc mode dependent - 4-8 / 1 pups
140 				 */
141 				cur_max_pup = (1 - ecc) *
142 					dram_info->num_of_std_pups + ecc;
143 
144 				if (ecc) {
145 					/* Only 1 pup in this case */
146 					valid_pup = 0x1;
147 				} else if (cur_max_pup > 4) {
148 					/* 64 bit - 8 pups */
149 					valid_pup = 0xFF;
150 				} else if (cur_max_pup == 4) {
151 					/* 32 bit - 4 pups */
152 					valid_pup = 0xF;
153 				} else {
154 					/* 16 bit - 2 pups */
155 					valid_pup = 0x3;
156 				}
157 
158 				/* ECC Support - Switch ECC Mux on ecc=1 */
159 				reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &
160 					~(1 << REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
161 				reg |= (dram_info->ecc_ena * ecc <<
162 					REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
163 				reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
164 
165 				if (ecc)
166 					DEBUG_PBS_S("DDR3 - PBS Tx - ECC Mux Enabled\n");
167 				else
168 					DEBUG_PBS_S("DDR3 - PBS Tx - ECC Mux Disabled\n");
169 
170 				/* Init iteration values */
171 				/* Clear the locked DQs */
172 				for (pup = 0; pup < cur_max_pup; pup++) {
173 					for (dq = 0; dq < DQ_NUM; dq++) {
174 						pbs_locked_dq[
175 							pup + ecc *
176 							(max_pup - 1)][dq] =
177 							0;
178 					}
179 				}
180 
181 				pbs_rep_time = 0;
182 				cur_pup = valid_pup;
183 				start_over = 0;
184 
185 				/*
186 				 * Run loop On current Pattern and current
187 				 * pattern iteration (just to cover the false
188 				 * fail problem)
189 				 */
190 				do {
191 					DEBUG_PBS_S("DDR3 - PBS Tx - Pbs Rep Loop is ");
192 					DEBUG_PBS_D(pbs_rep_time, 1);
193 					DEBUG_PBS_S(", for Retry No.");
194 					DEBUG_PBS_D(pbs_retry, 1);
195 					DEBUG_PBS_S("\n");
196 
197 					/* Set all PBS values to MIN (0) */
198 					DEBUG_PBS_S("DDR3 - PBS Tx - Set all PBS values to MIN\n");
199 
200 					for (dq = 0; dq < DQ_NUM; dq++) {
201 						ddr3_write_pup_reg(
202 							PUP_PBS_TX +
203 							pbs_dq_mapping[pup *
204 								(1 - ecc) +
205 								ecc * ECC_PUP]
206 							[dq], CS0, (1 - ecc) *
207 							PUP_BC + ecc * ECC_PUP, 0,
208 							0);
209 					}
210 
211 					/*
212 					 * Shift DQ ADLL right, One step before
213 					 * fail
214 					 */
215 					DEBUG_PBS_S("DDR3 - PBS Tx - ADLL shift right one phase before fail\n");
216 
217 					if (MV_OK != ddr3_tx_shift_dqs_adll_step_before_fail
218 					    (dram_info, cur_pup, pattern_idx,
219 					     ecc))
220 						return MV_DDR3_TRAINING_ERR_PBS_ADLL_SHR_1PHASE;
221 
222 					/* PBS For each bit */
223 					DEBUG_PBS_S("DDR3 - PBS Tx - perform PBS for each bit\n");
224 
225 					/*
226 					 * In this stage - start_over = 0
227 					 */
228 					if (MV_OK != ddr3_pbs_per_bit(
229 						    dram_info, &start_over, 1,
230 						    &cur_pup, pattern_idx, ecc))
231 						return MV_DDR3_TRAINING_ERR_PBS_TX_PER_BIT;
232 
233 				} while ((start_over == 1) &&
234 					 (++pbs_rep_time < COUNT_PBS_STARTOVER));
235 
236 				if (pbs_rep_time == COUNT_PBS_STARTOVER &&
237 				    start_over == 1) {
238 					DEBUG_PBS_S("DDR3 - PBS Tx - FAIL - Adll reach max value\n");
239 					return MV_DDR3_TRAINING_ERR_PBS_TX_MAX_VAL;
240 				}
241 
242 				DEBUG_PBS_FULL_C("DDR3 - PBS TX - values for iteration - ",
243 						 pbs_retry, 1);
244 				for (pup = 0; pup < cur_max_pup; pup++) {
245 					/*
246 					 * To minimize delay elements, inc
247 					 * from pbs value the min pbs val
248 					 */
249 					DEBUG_PBS_S("DDR3 - PBS - PUP");
250 					DEBUG_PBS_D((pup + (ecc * ECC_PUP)), 1);
251 					DEBUG_PBS_S(": ");
252 
253 					for (dq = 0; dq < DQ_NUM; dq++) {
254 						/* Set skew value for all dq */
255 						/*
256 						 * Bit# Deskew <- Bit# Deskew -
257 						 * last / first  failing bit
258 						 * Deskew For all bits (per PUP)
259 						 * (minimize delay elements)
260 						 */
261 						DEBUG_PBS_S("DQ");
262 						DEBUG_PBS_D(dq, 1);
263 						DEBUG_PBS_S("-");
264 						DEBUG_PBS_D(skew_array
265 							    [((pup) * DQ_NUM) +
266 							     dq], 2);
267 						DEBUG_PBS_S(", ");
268 					}
269 					DEBUG_PBS_S("\n");
270 				}
271 
272 				/*
273 				 * Collect the results we got on this trial
274 				 * of PBS
275 				 */
276 				for (pup = 0; pup < cur_max_pup; pup++) {
277 					for (dq = 0; dq < DQ_NUM; dq++) {
278 						skew_sum_array[pup + (ecc * (max_pup - 1))]
279 							[dq] += skew_array
280 							[((pup) * DQ_NUM) + dq];
281 					}
282 				}
283 
284 				/* ECC Support - Disable ECC MUX */
285 				reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &
286 					~(1 << REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
287 				reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
288 			}
289 		}
290 
291 		DEBUG_PBS_C("DDR3 - PBS TX - values for current pattern - ",
292 			    pattern_idx, 1);
293 		for (pup = 0; pup < max_pup; pup++) {
294 			/*
295 			 * To minimize delay elements, inc from pbs value the
296 			 * min pbs val
297 			 */
298 			DEBUG_PBS_S("DDR3 - PBS - PUP");
299 			DEBUG_PBS_D(pup, 1);
300 			DEBUG_PBS_S(": ");
301 
302 			for (dq = 0; dq < DQ_NUM; dq++) {
303 				/* set skew value for all dq */
304 				/* Bit# Deskew <- Bit# Deskew - last / first  failing bit Deskew For all bits (per PUP) (minimize delay elements) */
305 				DEBUG_PBS_S("DQ");
306 				DEBUG_PBS_D(dq, 1);
307 				DEBUG_PBS_S("-");
308 				DEBUG_PBS_D(skew_sum_array[pup][dq] /
309 					    COUNT_PBS_REPEAT, 2);
310 				DEBUG_PBS_S(", ");
311 			}
312 			DEBUG_PBS_S("\n");
313 		}
314 
315 		/*
316 		 * Calculate the average skew for current pattern for each
317 		 * pup and each bit
318 		 */
319 		DEBUG_PBS_C("DDR3 - PBS TX - Average for pattern - ",
320 			    pattern_idx, 1);
321 
322 		for (pup = 0; pup < max_pup; pup++) {
323 			/*
324 			 * FOR ECC only :: found min and max value for current
325 			 * pattern skew array
326 			 */
327 			/* Loop for all dqs */
328 			for (dq = 0; dq < DQ_NUM; dq++) {
329 				pattern_skew_array[pup][dq] +=
330 					(skew_sum_array[pup][dq] /
331 					 COUNT_PBS_REPEAT);
332 			}
333 		}
334 	}
335 
336 	/* Calculate the average skew */
337 	for (pup = 0; pup < max_pup; pup++) {
338 		for (dq = 0; dq < DQ_NUM; dq++)
339 			skew_array[((pup) * DQ_NUM) + dq] =
340 				pattern_skew_array[pup][dq] / COUNT_PBS_PATTERN;
341 	}
342 
343 	DEBUG_PBS_S("DDR3 - PBS TX - Average for all patterns:\n");
344 	for (pup = 0; pup < max_pup; pup++) {
345 		/*
346 		 * To minimize delay elements, inc from pbs value the min
347 		 * pbs val
348 		 */
349 		DEBUG_PBS_S("DDR3 - PBS - PUP");
350 		DEBUG_PBS_D(pup, 1);
351 		DEBUG_PBS_S(": ");
352 
353 		for (dq = 0; dq < DQ_NUM; dq++) {
354 			/* Set skew value for all dq */
355 			/*
356 			 * Bit# Deskew <- Bit# Deskew - last / first
357 			 * failing bit Deskew For all bits (per PUP)
358 			 * (minimize delay elements)
359 			 */
360 			DEBUG_PBS_S("DQ");
361 			DEBUG_PBS_D(dq, 1);
362 			DEBUG_PBS_S("-");
363 			DEBUG_PBS_D(skew_array[(pup * DQ_NUM) + dq], 2);
364 			DEBUG_PBS_S(", ");
365 		}
366 		DEBUG_PBS_S("\n");
367 	}
368 
369 	/* Return ADLL to default value */
370 	for (pup = 0; pup < max_pup; pup++) {
371 		if (pup == (max_pup - 1) && dram_info->ecc_ena)
372 			pup = ECC_PUP;
373 		ddr3_pbs_write_pup_dqs_reg(CS0, pup, INIT_WL_DELAY);
374 	}
375 
376 	/* Set averaged PBS results */
377 	ddr3_set_pbs_results(dram_info, 1);
378 
379 	/* Disable SW override - Must be in a different stage */
380 	/* [0]=0 - Enable SW override  */
381 	reg = reg_read(REG_DRAM_TRAINING_2_ADDR);
382 	reg &= ~(1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
383 	/* 0x15B8 - Training SW 2 Register */
384 	reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
385 
386 	reg = reg_read(REG_DRAM_TRAINING_1_ADDR) |
387 		(1 << REG_DRAM_TRAINING_1_TRNBPOINT_OFFS);
388 	reg_write(REG_DRAM_TRAINING_1_ADDR, reg);
389 
390 	DEBUG_PBS_S("DDR3 - PBS Tx - PBS TX ended successfuly\n");
391 
392 	return MV_OK;
393 }
394 
395 /*
396  * Name:     ddr3_tx_shift_dqs_adll_step_before_fail
397  * Desc:     Execute the Tx shift DQ phase.
398  * Args:     dram_info            ddr3 training information struct
399  *           cur_pup              bit array of the function active pups.
400  *           pbs_pattern_idx      Index of PBS pattern
401  * Notes:
402  * Returns:  MV_OK if success, other error code if fail.
403  */
404 static int ddr3_tx_shift_dqs_adll_step_before_fail(MV_DRAM_INFO *dram_info,
405 						   u32 cur_pup,
406 						   u32 pbs_pattern_idx, u32 ecc)
407 {
408 	u32 unlock_pup;		/* bit array of unlock pups  */
409 	u32 new_lockup_pup;	/* bit array of compare failed pups */
410 	u32 adll_val = 4;	/* INIT_WL_DELAY */
411 	u32 cur_max_pup, pup;
412 	u32 dqs_dly_set[MAX_PUP_NUM] = { 0 };
413 	u32 *pattern_ptr;
414 
415 	/* Choose pattern */
416 	switch (dram_info->ddr_width) {
417 #if defined(MV88F672X)
418 	case 16:
419 		pattern_ptr = (u32 *)&pbs_pattern[pbs_pattern_idx];
420 		break;
421 #endif
422 	case 32:
423 		pattern_ptr = (u32 *)&pbs_pattern_32b[pbs_pattern_idx];
424 		break;
425 #if defined(MV88F78X60)
426 	case 64:
427 		pattern_ptr = (u32 *)&pbs_pattern_64b[pbs_pattern_idx];
428 		break;
429 #endif
430 	default:
431 		return MV_FAIL;
432 	}
433 
434 	/* Set current pup number */
435 	if (cur_pup == 0x1)	/* Ecc mode */
436 		cur_max_pup = 1;
437 	else
438 		cur_max_pup = dram_info->num_of_std_pups;
439 
440 	unlock_pup = cur_pup;	/* '1' for each unlocked pup */
441 
442 	/* Loop on all ADLL Vaules */
443 	do {
444 		/* Loop until found first fail */
445 		adll_val++;
446 
447 		/*
448 		 * Increment (Move to right - ADLL) DQ TX delay
449 		 * (broadcast to all Data PUPs)
450 		 */
451 		for (pup = 0; pup < cur_max_pup; pup++)
452 			ddr3_pbs_write_pup_dqs_reg(CS0,
453 						   pup * (1 - ecc) +
454 						   ECC_PUP * ecc, adll_val);
455 
456 		/*
457 		 * Write and Read, compare results (read was already verified)
458 		 */
459 		/* 0 - all locked */
460 		new_lockup_pup = 0;
461 
462 		if (MV_OK != ddr3_sdram_compare(dram_info, unlock_pup,
463 						&new_lockup_pup,
464 						pattern_ptr, LEN_PBS_PATTERN,
465 						SDRAM_PBS_TX_OFFS, 1, 0,
466 						NULL,
467 						0))
468 			return MV_FAIL;
469 
470 		unlock_pup &= ~new_lockup_pup;
471 
472 		DEBUG_PBS_FULL_S("Shift DQS by 2 steps for PUPs: ");
473 		DEBUG_PBS_FULL_D(unlock_pup, 2);
474 		DEBUG_PBS_FULL_C(", Set ADLL value = ", adll_val, 2);
475 
476 		/* If any PUP failed there is '1' to mark the PUP */
477 		if (new_lockup_pup != 0) {
478 			/*
479 			 * Decrement (Move Back to Left two steps - ADLL)
480 			 * DQ TX delay for current failed pups and save
481 			 */
482 			for (pup = 0; pup < cur_max_pup; pup++) {
483 				if (((new_lockup_pup >> pup) & 0x1) &&
484 				    dqs_dly_set[pup] == 0)
485 					dqs_dly_set[pup] = adll_val - 1;
486 			}
487 		}
488 	} while ((unlock_pup != 0) && (adll_val != ADLL_MAX));
489 
490 	if (unlock_pup != 0) {
491 		DEBUG_PBS_FULL_S("DDR3 - PBS Tx - Shift DQ - Adll value reached maximum\n");
492 
493 		for (pup = 0; pup < cur_max_pup; pup++) {
494 			if (((unlock_pup >> pup) & 0x1) &&
495 			    dqs_dly_set[pup] == 0)
496 				dqs_dly_set[pup] = adll_val - 1;
497 		}
498 	}
499 
500 	DEBUG_PBS_FULL_C("PBS TX one step before fail last pups locked Adll ",
501 			 adll_val - 2, 2);
502 
503 	/* Set the PUP DQS DLY Values */
504 	for (pup = 0; pup < cur_max_pup; pup++)
505 		ddr3_pbs_write_pup_dqs_reg(CS0, pup * (1 - ecc) + ECC_PUP * ecc,
506 					   dqs_dly_set[pup]);
507 
508 	/* Found one phase before fail */
509 	return MV_OK;
510 }
511 
512 /*
513  * Name:     ddr3_pbs_rx
514  * Desc:     Execute the PBS RX phase.
515  * Args:     dram_info   ddr3 training information struct
516  * Notes:
517  * Returns:  MV_OK if success, other error code if fail.
518  */
519 int ddr3_pbs_rx(MV_DRAM_INFO *dram_info)
520 {
521 	/*
522 	 * Array to hold the total sum of skew from all iterations
523 	 * (for average purpose)
524 	 */
525 	u32 skew_sum_array[MAX_PUP_NUM][DQ_NUM] = { {0} };
526 
527 	/*
528 	 * Array to hold the total average skew from both patterns
529 	 * (for average purpose)
530 	 */
531 	u32 pattern_skew_array[MAX_PUP_NUM][DQ_NUM] = { {0} };
532 
533 	u32 pbs_rep_time = 0;	/* counts number of loop in case of fail */
534 	/* bit array for unlock pups - used to repeat on the RX operation */
535 	u32 cur_pup;
536 	u32 max_pup;
537 	u32 pbs_retry;
538 	u32 pup, dq, pups, cur_max_pup, valid_pup, reg;
539 	u32 pattern_idx;
540 	u32 ecc;
541 	/* indicates whether we need to start the loop again */
542 	int start_over;
543 	int status;
544 
545 	DEBUG_PBS_S("DDR3 - PBS RX - Starting PBS RX procedure\n");
546 
547 	pups = dram_info->num_of_total_pups;
548 	max_pup = dram_info->num_of_total_pups;
549 
550 	/* Enable SW override */
551 	reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |
552 		(1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
553 	/* [0] = 1 - Enable SW override  */
554 	/* 0x15B8 - Training SW 2 Register */
555 	reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
556 	DEBUG_PBS_FULL_S("DDR3 - PBS RX - SW Override Enabled\n");
557 
558 	reg = 1 << REG_DRAM_TRAINING_AUTO_OFFS;
559 	reg_write(REG_DRAM_TRAINING_ADDR, reg);	/* 0x15B0 - Training Register */
560 
561 	/* Running twice for 2 different patterns. each patterns - 3 times */
562 	for (pattern_idx = 0; pattern_idx < COUNT_PBS_PATTERN; pattern_idx++) {
563 		DEBUG_PBS_FULL_C("DDR3 - PBS RX - Working with pattern - ",
564 				 pattern_idx, 1);
565 
566 		/* Reset sum array */
567 		for (pup = 0; pup < pups; pup++) {
568 			for (dq = 0; dq < DQ_NUM; dq++)
569 				skew_sum_array[pup][dq] = 0;
570 		}
571 
572 		/*
573 		 * Perform PBS several of times (3 for each pattern).
574 		 * At the end, we'll use the average
575 		 */
576 		/* If there is ECC, do each PBS again with mux change */
577 		for (pbs_retry = 0; pbs_retry < COUNT_PBS_REPEAT; pbs_retry++) {
578 			for (ecc = 0; ecc < (dram_info->ecc_ena + 1); ecc++) {
579 				/*
580 				 * This parameter stores the current PUP
581 				 * num - ecc mode dependent - 4-8 / 1 pups
582 				 */
583 				cur_max_pup = (1 - ecc) *
584 					dram_info->num_of_std_pups + ecc;
585 
586 				if (ecc) {
587 					/* Only 1 pup in this case */
588 					valid_pup = 0x1;
589 				} else if (cur_max_pup > 4) {
590 					/* 64 bit - 8 pups */
591 					valid_pup = 0xFF;
592 				} else if (cur_max_pup == 4) {
593 					/* 32 bit - 4 pups */
594 					valid_pup = 0xF;
595 				} else {
596 					/* 16 bit - 2 pups */
597 					valid_pup = 0x3;
598 				}
599 
600 				/* ECC Support - Switch ECC Mux on ecc=1 */
601 				reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &
602 					~(1 << REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
603 				reg |= (dram_info->ecc_ena * ecc <<
604 					REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
605 				reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
606 
607 				if (ecc)
608 					DEBUG_PBS_FULL_S("DDR3 - PBS Rx - ECC Mux Enabled\n");
609 				else
610 					DEBUG_PBS_FULL_S("DDR3 - PBS Rx - ECC Mux Disabled\n");
611 
612 				/* Init iteration values */
613 				/* Clear the locked DQs */
614 				for (pup = 0; pup < cur_max_pup; pup++) {
615 					for (dq = 0; dq < DQ_NUM; dq++) {
616 						pbs_locked_dq[
617 							pup + ecc * (max_pup - 1)][dq] =
618 							0;
619 					}
620 				}
621 
622 				pbs_rep_time = 0;
623 				cur_pup = valid_pup;
624 				start_over = 0;
625 
626 				/*
627 				 * Run loop On current Pattern and current
628 				 * pattern iteration (just to cover the false
629 				 * fail problem
630 				 */
631 				do {
632 					DEBUG_PBS_FULL_S("DDR3 - PBS Rx - Pbs Rep Loop is ");
633 					DEBUG_PBS_FULL_D(pbs_rep_time, 1);
634 					DEBUG_PBS_FULL_S(", for Retry No.");
635 					DEBUG_PBS_FULL_D(pbs_retry, 1);
636 					DEBUG_PBS_FULL_S("\n");
637 
638 					/* Set all PBS values to MAX (31) */
639 					for (pup = 0; pup < cur_max_pup; pup++) {
640 						for (dq = 0; dq < DQ_NUM; dq++)
641 							ddr3_write_pup_reg(
642 								PUP_PBS_RX +
643 								pbs_dq_mapping[
644 								pup * (1 - ecc)
645 								+ ecc * ECC_PUP]
646 								[dq], CS0,
647 								pup + ecc * ECC_PUP,
648 								0, MAX_PBS);
649 					}
650 
651 					/* Set all DQS PBS values to MIN (0) */
652 					for (pup = 0; pup < cur_max_pup; pup++) {
653 						ddr3_write_pup_reg(PUP_PBS_RX +
654 								   DQ_NUM, CS0,
655 								   pup +
656 								   ecc *
657 								   ECC_PUP, 0,
658 								   0);
659 					}
660 
661 					/* Shift DQS, To first Fail */
662 					DEBUG_PBS_FULL_S("DDR3 - PBS Rx - Shift RX DQS to first fail\n");
663 
664 					status = ddr3_rx_shift_dqs_to_first_fail
665 						(dram_info, cur_pup,
666 						 pattern_idx, ecc);
667 					if (MV_OK != status) {
668 						DEBUG_PBS_S("DDR3 - PBS Rx - ddr3_rx_shift_dqs_to_first_fail failed.\n");
669 						DEBUG_PBS_D(status, 8);
670 						DEBUG_PBS_S("\nDDR3 - PBS Rx - SKIP.\n");
671 
672 						/* Reset read FIFO */
673 						reg = reg_read(REG_DRAM_TRAINING_ADDR);
674 						/* Start Auto Read Leveling procedure */
675 						reg |= (1 << REG_DRAM_TRAINING_RL_OFFS);
676 						/* 0x15B0 - Training Register */
677 						reg_write(REG_DRAM_TRAINING_ADDR, reg);
678 
679 						reg = reg_read(REG_DRAM_TRAINING_2_ADDR);
680 						reg |= ((1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS)
681 							+ (1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS));
682 						/* [0] = 1 - Enable SW override, [4] = 1 - FIFO reset  */
683 						/* 0x15B8 - Training SW 2 Register */
684 						reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
685 
686 						do {
687 							reg = (reg_read(REG_DRAM_TRAINING_2_ADDR))
688 								& (1 <<	REG_DRAM_TRAINING_2_FIFO_RST_OFFS);
689 						} while (reg);	/* Wait for '0' */
690 
691 						reg = reg_read(REG_DRAM_TRAINING_ADDR);
692 						/* Clear Auto Read Leveling procedure */
693 						reg &= ~(1 << REG_DRAM_TRAINING_RL_OFFS);
694 						/* 0x15B0 - Training Register */
695 						reg_write(REG_DRAM_TRAINING_ADDR, reg);
696 
697 						/* Set ADLL to 15 */
698 						for (pup = 0; pup < max_pup;
699 						     pup++) {
700 							ddr3_write_pup_reg
701 							    (PUP_DQS_RD, CS0,
702 							     pup +
703 							     (ecc * ECC_PUP), 0,
704 							     15);
705 						}
706 
707 						/* Set all PBS values to MIN (0) */
708 						for (pup = 0; pup < cur_max_pup;
709 						     pup++) {
710 							for (dq = 0;
711 							     dq < DQ_NUM; dq++)
712 								ddr3_write_pup_reg
713 								    (PUP_PBS_RX +
714 								     pbs_dq_mapping
715 								     [pup * (1 - ecc) +
716 								      ecc * ECC_PUP]
717 								     [dq], CS0,
718 								     pup + ecc * ECC_PUP,
719 								     0, MIN_PBS);
720 						}
721 
722 						return MV_OK;
723 					}
724 
725 					/* PBS For each bit */
726 					DEBUG_PBS_FULL_S("DDR3 - PBS Rx - perform PBS for each bit\n");
727 					/* in this stage - start_over = 0; */
728 					if (MV_OK != ddr3_pbs_per_bit(
729 						    dram_info, &start_over,
730 						    0, &cur_pup,
731 						    pattern_idx, ecc)) {
732 						DEBUG_PBS_S("DDR3 - PBS Rx - ddr3_pbs_per_bit failed.");
733 						return MV_DDR3_TRAINING_ERR_PBS_RX_PER_BIT;
734 					}
735 
736 				} while ((start_over == 1) &&
737 					 (++pbs_rep_time < COUNT_PBS_STARTOVER));
738 
739 				if (pbs_rep_time == COUNT_PBS_STARTOVER &&
740 				    start_over == 1) {
741 					DEBUG_PBS_FULL_S("DDR3 - PBS Rx - FAIL - Algorithm failed doing RX PBS\n");
742 					return MV_DDR3_TRAINING_ERR_PBS_RX_MAX_VAL;
743 				}
744 
745 				/* Return DQS ADLL to default value - 15 */
746 				/* Set all DQS PBS values to MIN (0) */
747 				for (pup = 0; pup < cur_max_pup; pup++)
748 					ddr3_write_pup_reg(PUP_DQS_RD, CS0,
749 							   pup + ecc * ECC_PUP,
750 							   0, INIT_RL_DELAY);
751 
752 				DEBUG_PBS_FULL_C("DDR3 - PBS RX - values for iteration - ",
753 						 pbs_retry, 1);
754 				for (pup = 0; pup < cur_max_pup; pup++) {
755 					/*
756 					 * To minimize delay elements, inc from
757 					 * pbs value the min pbs val
758 					 */
759 					DEBUG_PBS_FULL_S("DDR3 - PBS - PUP");
760 					DEBUG_PBS_FULL_D((pup +
761 							  (ecc * ECC_PUP)), 1);
762 					DEBUG_PBS_FULL_S(": ");
763 
764 					for (dq = 0; dq < DQ_NUM; dq++) {
765 						/* Set skew value for all dq */
766 						/*
767 						 * Bit# Deskew <- Bit# Deskew -
768 						 * last / first  failing bit
769 						 * Deskew For all bits (per PUP)
770 						 * (minimize delay elements)
771 						 */
772 						DEBUG_PBS_FULL_S("DQ");
773 						DEBUG_PBS_FULL_D(dq, 1);
774 						DEBUG_PBS_FULL_S("-");
775 						DEBUG_PBS_FULL_D(skew_array
776 								 [((pup) *
777 								   DQ_NUM) +
778 								  dq], 2);
779 						DEBUG_PBS_FULL_S(", ");
780 					}
781 					DEBUG_PBS_FULL_S("\n");
782 				}
783 
784 				/*
785 				 * Collect the results we got on this trial
786 				 * of PBS
787 				 */
788 				for (pup = 0; pup < cur_max_pup; pup++) {
789 					for (dq = 0; dq < DQ_NUM; dq++) {
790 						skew_sum_array
791 							[pup + (ecc * (max_pup - 1))]
792 							[dq] +=
793 							skew_array[((pup) * DQ_NUM) + dq];
794 					}
795 				}
796 
797 				/* ECC Support - Disable ECC MUX */
798 				reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &
799 					~(1 << REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
800 				reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
801 			}
802 		}
803 
804 		/*
805 		 * Calculate the average skew for current pattern for each
806 		 * pup and each bit
807 		 */
808 		DEBUG_PBS_FULL_C("DDR3 - PBS RX - Average for pattern - ",
809 				 pattern_idx, 1);
810 		for (pup = 0; pup < max_pup; pup++) {
811 			/*
812 			 * FOR ECC only :: found min and max value for
813 			 * current pattern skew array
814 			 */
815 			/* Loop for all dqs */
816 			for (dq = 0; dq < DQ_NUM; dq++) {
817 				pattern_skew_array[pup][dq] +=
818 					(skew_sum_array[pup][dq] /
819 					 COUNT_PBS_REPEAT);
820 			}
821 		}
822 
823 		DEBUG_PBS_C("DDR3 - PBS RX - values for current pattern - ",
824 			    pattern_idx, 1);
825 		for (pup = 0; pup < max_pup; pup++) {
826 			/*
827 			 * To minimize delay elements, inc from pbs value the
828 			 * min pbs val
829 			 */
830 			DEBUG_PBS_S("DDR3 - PBS RX - PUP");
831 			DEBUG_PBS_D(pup, 1);
832 			DEBUG_PBS_S(": ");
833 
834 			for (dq = 0; dq < DQ_NUM; dq++) {
835 				/* Set skew value for all dq */
836 				/*
837 				 * Bit# Deskew <- Bit# Deskew - last / first
838 				 * failing bit Deskew For all bits (per PUP)
839 				 * (minimize delay elements)
840 				 */
841 				DEBUG_PBS_S("DQ");
842 				DEBUG_PBS_D(dq, 1);
843 				DEBUG_PBS_S("-");
844 				DEBUG_PBS_D(skew_sum_array[pup][dq] /
845 					    COUNT_PBS_REPEAT, 2);
846 				DEBUG_PBS_S(", ");
847 			}
848 			DEBUG_PBS_S("\n");
849 		}
850 	}
851 
852 	/* Calculate the average skew */
853 	for (pup = 0; pup < max_pup; pup++) {
854 		for (dq = 0; dq < DQ_NUM; dq++)
855 			skew_array[((pup) * DQ_NUM) + dq] =
856 				pattern_skew_array[pup][dq] / COUNT_PBS_PATTERN;
857 	}
858 
859 	DEBUG_PBS_S("DDR3 - PBS RX - Average for all patterns:\n");
860 	for (pup = 0; pup < max_pup; pup++) {
861 		/*
862 		 * To minimize delay elements, inc from pbs value the
863 		 * min pbs val
864 		 */
865 		DEBUG_PBS_S("DDR3 - PBS - PUP");
866 		DEBUG_PBS_D(pup, 1);
867 		DEBUG_PBS_S(": ");
868 
869 		for (dq = 0; dq < DQ_NUM; dq++) {
870 			/* Set skew value for all dq */
871 			/*
872 			 * Bit# Deskew <- Bit# Deskew - last / first
873 			 * failing bit Deskew For all bits (per PUP)
874 			 * (minimize delay elements)
875 			 */
876 			DEBUG_PBS_S("DQ");
877 			DEBUG_PBS_D(dq, 1);
878 			DEBUG_PBS_S("-");
879 			DEBUG_PBS_D(skew_array[(pup * DQ_NUM) + dq], 2);
880 			DEBUG_PBS_S(", ");
881 		}
882 		DEBUG_PBS_S("\n");
883 	}
884 
885 	/* Return ADLL to default value */
886 	ddr3_write_pup_reg(PUP_DQS_RD, CS0, PUP_BC, 0, INIT_RL_DELAY);
887 
888 	/* Set averaged PBS results */
889 	ddr3_set_pbs_results(dram_info, 0);
890 
891 	/* Disable SW override - Must be in a different stage */
892 	/* [0]=0 - Enable SW override  */
893 	reg = reg_read(REG_DRAM_TRAINING_2_ADDR);
894 	reg &= ~(1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
895 	/* 0x15B8 - Training SW 2 Register */
896 	reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
897 
898 	reg = reg_read(REG_DRAM_TRAINING_1_ADDR) |
899 		(1 << REG_DRAM_TRAINING_1_TRNBPOINT_OFFS);
900 	reg_write(REG_DRAM_TRAINING_1_ADDR, reg);
901 
902 	DEBUG_PBS_FULL_S("DDR3 - PBS RX - ended successfuly\n");
903 
904 	return MV_OK;
905 }
906 
907 /*
908  * Name:     ddr3_rx_shift_dqs_to_first_fail
909  * Desc:     Execute the Rx shift DQ phase.
910  * Args:     dram_info           ddr3 training information struct
911  *           cur_pup             bit array of the function active pups.
912  *           pbs_pattern_idx     Index of PBS pattern
913  * Notes:
914  * Returns:  MV_OK if success, other error code if fail.
915  */
916 static int ddr3_rx_shift_dqs_to_first_fail(MV_DRAM_INFO *dram_info, u32 cur_pup,
917 					   u32 pbs_pattern_idx, u32 ecc)
918 {
919 	u32 unlock_pup;		/* bit array of unlock pups  */
920 	u32 new_lockup_pup;	/* bit array of compare failed pups */
921 	u32 adll_val = MAX_DELAY;
922 	u32 dqs_deskew_val = 0;	/* current value of DQS PBS deskew */
923 	u32 cur_max_pup, pup, pass_pup;
924 	u32 *pattern_ptr;
925 
926 	/* Choose pattern */
927 	switch (dram_info->ddr_width) {
928 #if defined(MV88F672X)
929 	case 16:
930 		pattern_ptr = (u32 *)&pbs_pattern[pbs_pattern_idx];
931 		break;
932 #endif
933 	case 32:
934 		pattern_ptr = (u32 *)&pbs_pattern_32b[pbs_pattern_idx];
935 		break;
936 #if defined(MV88F78X60)
937 	case 64:
938 		pattern_ptr = (u32 *)&pbs_pattern_64b[pbs_pattern_idx];
939 		break;
940 #endif
941 	default:
942 		return MV_FAIL;
943 	}
944 
945 	/* Set current pup number */
946 	if (cur_pup == 0x1)	/* Ecc mode */
947 		cur_max_pup = 1;
948 	else
949 		cur_max_pup = dram_info->num_of_std_pups;
950 
951 	unlock_pup = cur_pup;	/* '1' for each unlocked pup */
952 
953 	DEBUG_PBS_FULL_S("DDR3 - PBS RX - Shift DQS - Starting...\n");
954 
955 	/* Set DQS ADLL to MAX */
956 	DEBUG_PBS_FULL_S("DDR3 - PBS RX - Shift DQS - Set DQS ADLL to Max for all PUPs\n");
957 	for (pup = 0; pup < cur_max_pup; pup++)
958 		ddr3_write_pup_reg(PUP_DQS_RD, CS0, pup + ecc * ECC_PUP, 0,
959 				   MAX_DELAY);
960 
961 	/* Loop on all ADLL Vaules */
962 	do {
963 		/* Loop until found fail for all pups */
964 		new_lockup_pup = 0;
965 		if (MV_OK != ddr3_sdram_compare(dram_info, unlock_pup,
966 						&new_lockup_pup,
967 						pattern_ptr, LEN_PBS_PATTERN,
968 						SDRAM_PBS_I_OFFS +
969 						pbs_pattern_idx * SDRAM_PBS_NEXT_OFFS,
970 						0, 0, NULL, 0)) {
971 			DEBUG_PBS_S("DDR3 - PBS Rx - Shift DQS - MV_DDR3_TRAINING_ERR_PBS_SHIFT_QDS_SRAM_CMP(ddr3_sdram_compare)\n");
972 			return MV_DDR3_TRAINING_ERR_PBS_SHIFT_QDS_SRAM_CMP;
973 		}
974 
975 		if ((new_lockup_pup != 0) && (dqs_deskew_val <= 1)) {
976 			/* Fail on start with first deskew value */
977 			/* Decrement DQS ADLL */
978 			--adll_val;
979 			if (adll_val == ADLL_MIN) {
980 				DEBUG_PBS_S("DDR3 - PBS Rx - Shift DQS - fail on start with first deskew value\n");
981 				return MV_DDR3_TRAINING_ERR_PBS_SHIFT_QDS_SRAM_CMP;
982 			}
983 			ddr3_write_pup_reg(PUP_DQS_RD, CS0, pup + ecc * ECC_PUP,
984 					   0, adll_val);
985 			continue;
986 		}
987 
988 		/* Update all new locked pups */
989 		unlock_pup &= ~new_lockup_pup;
990 
991 		if ((unlock_pup == 0) || (dqs_deskew_val == MAX_PBS)) {
992 			if (dqs_deskew_val == MAX_PBS) {
993 				/*
994 				 * Reach max value of dqs deskew or get fail
995 				 * for all pups
996 				 */
997 				DEBUG_PBS_FULL_S("DDR3 - PBS RX - Shift DQS - DQS deskew reached maximum value\n");
998 			}
999 			break;
1000 		}
1001 
1002 		DEBUG_PBS_FULL_S("DDR3 - PBS RX - Shift DQS - Inc DQS deskew for PUPs: ");
1003 		DEBUG_PBS_FULL_D(unlock_pup, 2);
1004 		DEBUG_PBS_FULL_C(", deskew = ", dqs_deskew_val, 2);
1005 
1006 		/* Increment DQS deskew elements - Only for unlocked pups */
1007 		dqs_deskew_val++;
1008 		for (pup = 0; pup < cur_max_pup; pup++) {
1009 			if (IS_PUP_ACTIVE(unlock_pup, pup) == 1) {
1010 				ddr3_write_pup_reg(PUP_PBS_RX + DQS_DQ_NUM, CS0,
1011 						   pup + ecc * ECC_PUP, 0,
1012 						   dqs_deskew_val);
1013 			}
1014 		}
1015 	} while (1);
1016 
1017 	DEBUG_PBS_FULL_S("DDR3 - PBS RX - Shift DQS - ADLL shift one step before fail\n");
1018 	/* Continue to ADLL shift one step before fail */
1019 	unlock_pup = cur_pup;
1020 	do {
1021 		/* Loop until pass compare for all pups */
1022 		new_lockup_pup = 0;
1023 		/* Read and compare results  */
1024 		if (MV_OK != ddr3_sdram_compare(dram_info, unlock_pup, &new_lockup_pup,
1025 						pattern_ptr, LEN_PBS_PATTERN,
1026 						SDRAM_PBS_I_OFFS +
1027 						pbs_pattern_idx * SDRAM_PBS_NEXT_OFFS,
1028 						1, 0, NULL, 0)) {
1029 			DEBUG_PBS_S("DDR3 - PBS Rx - Shift DQS - MV_DDR3_TRAINING_ERR_PBS_SHIFT_QDS_SRAM_CMP(ddr3_sdram_compare)\n");
1030 			return MV_DDR3_TRAINING_ERR_PBS_SHIFT_QDS_SRAM_CMP;
1031 		}
1032 
1033 		/*
1034 		 * Get mask for pup which passed so their adll will be
1035 		 * changed to 2 steps before fails
1036 		 */
1037 		pass_pup = unlock_pup & ~new_lockup_pup;
1038 
1039 		DEBUG_PBS_FULL_S("Shift DQS by 2 steps for PUPs: ");
1040 		DEBUG_PBS_FULL_D(pass_pup, 2);
1041 		DEBUG_PBS_FULL_C(", Set ADLL value = ", (adll_val - 2), 2);
1042 
1043 		/* Only for pass pups   */
1044 		for (pup = 0; pup < cur_max_pup; pup++) {
1045 			if (IS_PUP_ACTIVE(pass_pup, pup) == 1) {
1046 				ddr3_write_pup_reg(PUP_DQS_RD, CS0,
1047 						   pup + ecc * ECC_PUP, 0,
1048 						   (adll_val - 2));
1049 			}
1050 		}
1051 
1052 		/* Locked pups that compare success  */
1053 		unlock_pup &= new_lockup_pup;
1054 
1055 		if (unlock_pup == 0) {
1056 			/* All pups locked */
1057 			break;
1058 		}
1059 
1060 		/* Found error */
1061 		if (adll_val == 0) {
1062 			DEBUG_PBS_FULL_S("DDR3 - PBS Rx - Shift DQS - Adll reach min value\n");
1063 			return MV_DDR3_TRAINING_ERR_PBS_SHIFT_QDS_MAX_VAL;
1064 		}
1065 
1066 		/*
1067 		 * Decrement (Move Back to Left one phase - ADLL) dqs RX delay
1068 		 */
1069 		adll_val--;
1070 		for (pup = 0; pup < cur_max_pup; pup++) {
1071 			if (IS_PUP_ACTIVE(unlock_pup, pup) == 1) {
1072 				ddr3_write_pup_reg(PUP_DQS_RD, CS0,
1073 						   pup + ecc * ECC_PUP, 0,
1074 						   adll_val);
1075 			}
1076 		}
1077 	} while (1);
1078 
1079 	return MV_OK;
1080 }
1081 
1082 /*
1083  * lock_pups() extracted from ddr3_pbs_per_bit(). This just got too
1084  * much indented making it hard to read / edit.
1085  */
1086 static void lock_pups(u32 pup, u32 *pup_locked, u8 *unlock_pup_dq_array,
1087 		      u32 pbs_curr_val, u32 start_pbs, u32 ecc, int is_tx)
1088 {
1089 	u32 dq;
1090 	int idx;
1091 
1092 	/* Lock PBS value for all remaining PUPs bits */
1093 	DEBUG_PBS_FULL_S("DDR3 - PBS Per bit - Lock PBS value for all remaining PUPs bits, pup ");
1094 	DEBUG_PBS_FULL_D(pup, 1);
1095 	DEBUG_PBS_FULL_C(" pbs value ", pbs_curr_val, 2);
1096 
1097 	idx = pup * (1 - ecc) + ecc * ECC_PUP;
1098 	*pup_locked &= ~(1 << pup);
1099 
1100 	for (dq = 0; dq < DQ_NUM; dq++) {
1101 		if (IS_PUP_ACTIVE(unlock_pup_dq_array[dq], pup) == 1) {
1102 			int offs;
1103 
1104 			/* Lock current dq */
1105 			unlock_pup_dq_array[dq] &= ~(1 << pup);
1106 			skew_array[(pup * DQ_NUM) + dq] = pbs_curr_val;
1107 
1108 			if (is_tx == 1)
1109 				offs = PUP_PBS_TX;
1110 			else
1111 				offs = PUP_PBS_RX;
1112 
1113 			ddr3_write_pup_reg(offs +
1114 					   pbs_dq_mapping[idx][dq], CS0,
1115 					   idx, 0, start_pbs);
1116 		}
1117 	}
1118 }
1119 
1120 /*
1121  * Name:     ddr3_pbs_per_bit
1122  * Desc:     Execute the Per Bit Skew phase.
1123  * Args:     start_over      Return whether need to start over the algorithm
1124  *           is_tx           Indicate whether Rx or Tx
1125  *           pcur_pup        bit array of the function active pups. return the
1126  *                           pups that need to repeat on the PBS
1127  *           pbs_pattern_idx Index of PBS pattern
1128  *
1129  * Notes:    Current implementation supports double activation of this function.
1130  *           i.e. in order to activate this function (using start_over) more than
1131  *           twice, the implementation should change.
1132  *           imlementation limitation are marked using
1133  *           ' CHIP-ONLY! - Implementation Limitation '
1134  * Returns:  MV_OK if success, other error code if fail.
1135  */
1136 static int ddr3_pbs_per_bit(MV_DRAM_INFO *dram_info, int *start_over, int is_tx,
1137 			    u32 *pcur_pup, u32 pbs_pattern_idx, u32 ecc)
1138 {
1139 	/*
1140 	 * Bit array to indicate if we already get fail on bit per pup & dq bit
1141 	 */
1142 	u8 unlock_pup_dq_array[DQ_NUM] = {
1143 		*pcur_pup, *pcur_pup, *pcur_pup, *pcur_pup, *pcur_pup,
1144 		*pcur_pup, *pcur_pup, *pcur_pup
1145 	};
1146 
1147 	u8 cmp_unlock_pup_dq_array[COUNT_PBS_COMP_RETRY_NUM][DQ_NUM];
1148 	u32 pup, dq;
1149 	/* value of pbs is according to RX or TX */
1150 	u32 start_pbs, last_pbs;
1151 	u32 pbs_curr_val;
1152 	/* bit array that indicates all dq of the pup locked */
1153 	u32 pup_locked;
1154 	u32 first_fail[MAX_PUP_NUM] = { 0 };	/* count first fail per pup */
1155 	/* indicates whether we get first fail per pup */
1156 	int first_failed[MAX_PUP_NUM] = { 0 };
1157 	/* bit array that indicates pup already get fail */
1158 	u32 sum_pup_fail;
1159 	/* use to calculate diff between curr pbs to first fail pbs */
1160 	u32 calc_pbs_diff;
1161 	u32 pbs_cmp_retry;
1162 	u32 max_pup;
1163 
1164 	/* Set init values for retry array - 8 retry */
1165 	for (pbs_cmp_retry = 0; pbs_cmp_retry < COUNT_PBS_COMP_RETRY_NUM;
1166 	     pbs_cmp_retry++) {
1167 		for (dq = 0; dq < DQ_NUM; dq++)
1168 			cmp_unlock_pup_dq_array[pbs_cmp_retry][dq] = *pcur_pup;
1169 	}
1170 
1171 	memset(&skew_array, 0, MAX_PUP_NUM * DQ_NUM * sizeof(u32));
1172 
1173 	DEBUG_PBS_FULL_S("DDR3 - PBS Per bit - Started\n");
1174 
1175 	/* The pbs value depends if rx or tx */
1176 	if (is_tx == 1) {
1177 		start_pbs = MIN_PBS;
1178 		last_pbs = MAX_PBS;
1179 	} else {
1180 		start_pbs = MAX_PBS;
1181 		last_pbs = MIN_PBS;
1182 	}
1183 
1184 	pbs_curr_val = start_pbs;
1185 	pup_locked = *pcur_pup;
1186 
1187 	/* Set current pup number */
1188 	if (pup_locked == 0x1)	/* Ecc mode */
1189 		max_pup = 1;
1190 	else
1191 		max_pup = dram_info->num_of_std_pups;
1192 
1193 	do {
1194 		/* Increment/ decrement PBS for un-lock bits only */
1195 		if (is_tx == 1)
1196 			pbs_curr_val++;
1197 		else
1198 			pbs_curr_val--;
1199 
1200 		/* Set Current PBS delay  */
1201 		for (dq = 0; dq < DQ_NUM; dq++) {
1202 			/* Check DQ bits to see if locked in all pups */
1203 			if (unlock_pup_dq_array[dq] == 0) {
1204 				DEBUG_PBS_FULL_S("DDR3 - PBS Per bit - All pups are locked for DQ ");
1205 				DEBUG_PBS_FULL_D(dq, 1);
1206 				DEBUG_PBS_FULL_S("\n");
1207 				continue;
1208 			}
1209 
1210 			for (pup = 0; pup < max_pup; pup++) {
1211 				int idx;
1212 
1213 				idx = pup * (1 - ecc) + ecc * ECC_PUP;
1214 
1215 				if (IS_PUP_ACTIVE(unlock_pup_dq_array[dq], pup)
1216 				    == 0)
1217 					continue;
1218 
1219 				if (is_tx == 1)
1220 					ddr3_write_pup_reg(
1221 						PUP_PBS_TX + pbs_dq_mapping[idx][dq],
1222 						CS0, idx, 0, pbs_curr_val);
1223 				else
1224 					ddr3_write_pup_reg(
1225 						PUP_PBS_RX + pbs_dq_mapping[idx][dq],
1226 						CS0, idx, 0, pbs_curr_val);
1227 			}
1228 		}
1229 
1230 		/*
1231 		 * Write Read and compare results - run the test
1232 		 * DDR_PBS_COMP_RETRY_NUM times
1233 		 */
1234 		/* Run number of read and write to verify */
1235 		for (pbs_cmp_retry = 0;
1236 		     pbs_cmp_retry < COUNT_PBS_COMP_RETRY_NUM;
1237 		     pbs_cmp_retry++) {
1238 
1239 			if (MV_OK !=
1240 			    ddr3_sdram_pbs_compare(dram_info, pup_locked, is_tx,
1241 						   pbs_pattern_idx,
1242 						   pbs_curr_val, start_pbs,
1243 						   skew_array,
1244 						   cmp_unlock_pup_dq_array
1245 						   [pbs_cmp_retry], ecc))
1246 				return MV_FAIL;
1247 
1248 			for (pup = 0; pup < max_pup; pup++) {
1249 				for (dq = 0; dq < DQ_NUM; dq++) {
1250 					if ((IS_PUP_ACTIVE(unlock_pup_dq_array[dq],
1251 							   pup) == 1)
1252 					    && (IS_PUP_ACTIVE(cmp_unlock_pup_dq_array
1253 					      [pbs_cmp_retry][dq],
1254 					      pup) == 0)) {
1255 						DEBUG_PBS_FULL_S("DDR3 - PBS Per bit - PbsCurrVal: ");
1256 						DEBUG_PBS_FULL_D(pbs_curr_val, 2);
1257 						DEBUG_PBS_FULL_S(" PUP: ");
1258 						DEBUG_PBS_FULL_D(pup, 1);
1259 						DEBUG_PBS_FULL_S(" DQ: ");
1260 						DEBUG_PBS_FULL_D(dq, 1);
1261 						DEBUG_PBS_FULL_S(" - failed\n");
1262 					}
1263 				}
1264 			}
1265 
1266 			for (dq = 0; dq < DQ_NUM; dq++) {
1267 				unlock_pup_dq_array[dq] &=
1268 				    cmp_unlock_pup_dq_array[pbs_cmp_retry][dq];
1269 			}
1270 		}
1271 
1272 		pup_locked = 0;
1273 		sum_pup_fail = *pcur_pup;
1274 
1275 		/* Check which DQ is failed */
1276 		for (dq = 0; dq < DQ_NUM; dq++) {
1277 			/* Summarize the locked pup */
1278 			pup_locked |= unlock_pup_dq_array[dq];
1279 
1280 			/* Check if get fail */
1281 			sum_pup_fail &= unlock_pup_dq_array[dq];
1282 		}
1283 
1284 		/* If all PUPS are locked in all DQ - Break */
1285 		if (pup_locked == 0) {
1286 			/* All pups are locked */
1287 			*start_over = 0;
1288 			DEBUG_PBS_FULL_S("DDR3 - PBS Per bit -  All bit in all pups are successfully locked\n");
1289 			break;
1290 		}
1291 
1292 		/* PBS deskew elements reach max ? */
1293 		if (pbs_curr_val == last_pbs) {
1294 			DEBUG_PBS_FULL_S("DDR3 - PBS Per bit - PBS deskew elements reach max\n");
1295 			/* CHIP-ONLY! - Implementation Limitation */
1296 			*start_over = (sum_pup_fail != 0) && (!(*start_over));
1297 			*pcur_pup = pup_locked;
1298 
1299 			DEBUG_PBS_FULL_S("DDR3 - PBS Per bit - StartOver: ");
1300 			DEBUG_PBS_FULL_D(*start_over, 1);
1301 			DEBUG_PBS_FULL_S("  pup_locked: ");
1302 			DEBUG_PBS_FULL_D(pup_locked, 2);
1303 			DEBUG_PBS_FULL_S("  sum_pup_fail: ");
1304 			DEBUG_PBS_FULL_D(sum_pup_fail, 2);
1305 			DEBUG_PBS_FULL_S("\n");
1306 
1307 			/* Lock PBS value for all remaining  bits */
1308 			for (pup = 0; pup < max_pup; pup++) {
1309 				/* Check if current pup already received error */
1310 				if (IS_PUP_ACTIVE(pup_locked, pup) == 1) {
1311 					/* Valid pup for current function */
1312 					if (IS_PUP_ACTIVE(sum_pup_fail, pup) ==
1313 					    1 && (*start_over == 1)) {
1314 						DEBUG_PBS_FULL_C("DDR3 - PBS Per bit - skipping lock of pup (first loop of pbs)",
1315 								 pup, 1);
1316 						continue;
1317 					} else
1318 					    if (IS_PUP_ACTIVE(sum_pup_fail, pup)
1319 						== 1) {
1320 						DEBUG_PBS_FULL_C("DDR3 - PBS Per bit - Locking pup %d (even though it wasn't supposed to be locked)",
1321 								 pup, 1);
1322 					}
1323 
1324 					/* Already got fail on the PUP */
1325 					/* Lock PBS value for all remaining bits */
1326 					DEBUG_PBS_FULL_S("DDR3 - PBS Per bit - Locking remaning DQs for pup - ");
1327 					DEBUG_PBS_FULL_D(pup, 1);
1328 					DEBUG_PBS_FULL_S(": ");
1329 
1330 					for (dq = 0; dq < DQ_NUM; dq++) {
1331 						if (IS_PUP_ACTIVE
1332 						    (unlock_pup_dq_array[dq],
1333 						     pup) == 1) {
1334 							DEBUG_PBS_FULL_D(dq, 1);
1335 							DEBUG_PBS_FULL_S(",");
1336 							/* set current PBS */
1337 							skew_array[((pup) *
1338 								    DQ_NUM) +
1339 								   dq] =
1340 							    pbs_curr_val;
1341 						}
1342 					}
1343 
1344 					if (*start_over == 1) {
1345 						/*
1346 						 * Reset this pup bit - when
1347 						 * restart the PBS, ignore this
1348 						 * pup
1349 						 */
1350 						*pcur_pup &= ~(1 << pup);
1351 					}
1352 					DEBUG_PBS_FULL_S("\n");
1353 				} else {
1354 					DEBUG_PBS_FULL_S("DDR3 - PBS Per bit - Pup ");
1355 					DEBUG_PBS_FULL_D(pup, 1);
1356 					DEBUG_PBS_FULL_C(" is not set in puplocked - ",
1357 							 pup_locked, 1);
1358 				}
1359 			}
1360 
1361 			/* Need to start the PBS again */
1362 			if (*start_over == 1) {
1363 				DEBUG_PBS_FULL_S("DDR3 - PBS Per bit - false fail - returning to start\n");
1364 				return MV_OK;
1365 			}
1366 			break;
1367 		}
1368 
1369 		/* Diff Check */
1370 		for (pup = 0; pup < max_pup; pup++) {
1371 			if (IS_PUP_ACTIVE(pup_locked, pup) == 1) {
1372 				/* pup is not locked */
1373 				if (first_failed[pup] == 0) {
1374 					/* No first fail until now */
1375 					if (IS_PUP_ACTIVE(sum_pup_fail, pup) ==
1376 					    0) {
1377 						/* Get first fail */
1378 						DEBUG_PBS_FULL_C("DDR3 - PBS Per bit - First fail in pup ",
1379 								 pup, 1);
1380 						first_failed[pup] = 1;
1381 						first_fail[pup] = pbs_curr_val;
1382 					}
1383 				} else {
1384 					/* Already got first fail */
1385 					if (is_tx == 1) {
1386 						/* TX - inc pbs */
1387 						calc_pbs_diff =	pbs_curr_val -
1388 							first_fail[pup];
1389 					} else {
1390 						/* RX - dec pbs */
1391 						calc_pbs_diff = first_fail[pup] -
1392 							pbs_curr_val;
1393 					}
1394 
1395 					if (calc_pbs_diff >= PBS_DIFF_LIMIT) {
1396 						lock_pups(pup, &pup_locked,
1397 							  unlock_pup_dq_array,
1398 							  pbs_curr_val,
1399 							  start_pbs, ecc, is_tx);
1400 					}
1401 				}
1402 			}
1403 		}
1404 	} while (1);
1405 
1406 	return MV_OK;
1407 }
1408 
1409 /*
1410  * Name:         ddr3_set_pbs_results
1411  * Desc:         Set to HW the PBS phase results.
1412  * Args:         is_tx       Indicates whether to set Tx or RX results
1413  * Notes:
1414  * Returns:      MV_OK if success, other error code if fail.
1415  */
1416 static int ddr3_set_pbs_results(MV_DRAM_INFO *dram_info, int is_tx)
1417 {
1418 	u32 pup, phys_pup, dq;
1419 	u32 max_pup;		/* number of valid pups */
1420 	u32 pbs_min;		/* minimal pbs val per pup */
1421 	u32 pbs_max;		/* maximum pbs val per pup */
1422 	u32 val[9];
1423 
1424 	max_pup = dram_info->num_of_total_pups;
1425 	DEBUG_PBS_FULL_S("DDR3 - PBS - ddr3_set_pbs_results:\n");
1426 
1427 	/* Loop for all dqs & pups */
1428 	for (pup = 0; pup < max_pup; pup++) {
1429 		if (pup == (max_pup - 1) && dram_info->ecc_ena)
1430 			phys_pup = ECC_PUP;
1431 		else
1432 			phys_pup = pup;
1433 
1434 		/*
1435 		 * To minimize delay elements, inc from pbs value the min
1436 		 * pbs val
1437 		 */
1438 		pbs_min = MAX_PBS;
1439 		pbs_max = 0;
1440 		for (dq = 0; dq < DQ_NUM; dq++) {
1441 			if (pbs_min > skew_array[(pup * DQ_NUM) + dq])
1442 				pbs_min = skew_array[(pup * DQ_NUM) + dq];
1443 
1444 			if (pbs_max < skew_array[(pup * DQ_NUM) + dq])
1445 				pbs_max = skew_array[(pup * DQ_NUM) + dq];
1446 		}
1447 
1448 		pbs_max -= pbs_min;
1449 
1450 		DEBUG_PBS_FULL_S("DDR3 - PBS - PUP");
1451 		DEBUG_PBS_FULL_D(phys_pup, 1);
1452 		DEBUG_PBS_FULL_S(": Min Val = ");
1453 		DEBUG_PBS_FULL_D(pbs_min, 2);
1454 		DEBUG_PBS_FULL_C(", Max Val = ", pbs_max, 2);
1455 
1456 		val[pup] = 0;
1457 
1458 		for (dq = 0; dq < DQ_NUM; dq++) {
1459 			int idx;
1460 			int offs;
1461 
1462 			/* Set skew value for all dq */
1463 			/*
1464 			 * Bit# Deskew <- Bit# Deskew - last / first
1465 			 * failing bit Deskew For all bits (per PUP)
1466 			 * (minimize delay elements)
1467 			 */
1468 
1469 			DEBUG_PBS_FULL_S("DQ");
1470 			DEBUG_PBS_FULL_D(dq, 1);
1471 			DEBUG_PBS_FULL_S("-");
1472 			DEBUG_PBS_FULL_D((skew_array[(pup * DQ_NUM) + dq] -
1473 					  pbs_min), 2);
1474 			DEBUG_PBS_FULL_S(", ");
1475 
1476 			idx = (pup * DQ_NUM) + dq;
1477 
1478 			if (is_tx == 1)
1479 				offs = PUP_PBS_TX;
1480 			else
1481 				offs = PUP_PBS_RX;
1482 
1483 			ddr3_write_pup_reg(offs + pbs_dq_mapping[phys_pup][dq],
1484 					   CS0, phys_pup, 0,
1485 					   skew_array[idx] - pbs_min);
1486 
1487 			if (is_tx == 1)
1488 				val[pup] += skew_array[idx] - pbs_min;
1489 		}
1490 
1491 		DEBUG_PBS_FULL_S("\n");
1492 
1493 		/* Set the DQS the half of the Max PBS of the DQs  */
1494 		if (is_tx == 1) {
1495 			ddr3_write_pup_reg(PUP_PBS_TX + 8, CS0, phys_pup, 0,
1496 					   pbs_max / 2);
1497 			ddr3_write_pup_reg(PUP_PBS_TX + 0xa, CS0, phys_pup, 0,
1498 					   val[pup] / 8);
1499 		} else
1500 			ddr3_write_pup_reg(PUP_PBS_RX + 8, CS0, phys_pup, 0,
1501 					   pbs_max / 2);
1502 	}
1503 
1504 	return MV_OK;
1505 }
1506 
1507 static void ddr3_pbs_write_pup_dqs_reg(u32 cs, u32 pup, u32 dqs_delay)
1508 {
1509 	u32 reg, delay;
1510 
1511 	reg = (ddr3_read_pup_reg(PUP_WL_MODE, cs, pup) & 0x3FF);
1512 	delay = reg & PUP_DELAY_MASK;
1513 	reg |= ((dqs_delay + delay) << REG_PHY_DQS_REF_DLY_OFFS);
1514 	reg |= REG_PHY_REGISTRY_FILE_ACCESS_OP_WR;
1515 	reg |= (pup << REG_PHY_PUP_OFFS);
1516 	reg |= ((0x4 * cs + PUP_WL_MODE) << REG_PHY_CS_OFFS);
1517 
1518 	reg_write(REG_PHY_REGISTRY_FILE_ACCESS_ADDR, reg);	/* 0x16A0 */
1519 	do {
1520 		reg = reg_read(REG_PHY_REGISTRY_FILE_ACCESS_ADDR) &
1521 			REG_PHY_REGISTRY_FILE_ACCESS_OP_DONE;
1522 	} while (reg);	/* Wait for '0' to mark the end of the transaction */
1523 
1524 	udelay(10);
1525 }
1526 
1527 /*
1528  * Set training patterns
1529  */
1530 int ddr3_load_pbs_patterns(MV_DRAM_INFO *dram_info)
1531 {
1532 	u32 cs, cs_count, cs_tmp;
1533 	u32 sdram_addr;
1534 	u32 *pattern_ptr0, *pattern_ptr1;
1535 
1536 	/* Choose pattern */
1537 	switch (dram_info->ddr_width) {
1538 #if defined(MV88F672X)
1539 	case 16:
1540 		pattern_ptr0 = (u32 *)&pbs_pattern[0];
1541 		pattern_ptr1 = (u32 *)&pbs_pattern[1];
1542 		break;
1543 #endif
1544 	case 32:
1545 		pattern_ptr0 = (u32 *)&pbs_pattern_32b[0];
1546 		pattern_ptr1 = (u32 *)&pbs_pattern_32b[1];
1547 		break;
1548 #if defined(MV88F78X60)
1549 	case 64:
1550 		pattern_ptr0 = (u32 *)&pbs_pattern_64b[0];
1551 		pattern_ptr1 = (u32 *)&pbs_pattern_64b[1];
1552 		break;
1553 #endif
1554 	default:
1555 		return MV_FAIL;
1556 	}
1557 
1558 	/* Loop for each CS */
1559 	for (cs = 0; cs < MAX_CS; cs++) {
1560 		if (dram_info->cs_ena & (1 << cs)) {
1561 			cs_count = 0;
1562 			for (cs_tmp = 0; cs_tmp < cs; cs_tmp++) {
1563 				if (dram_info->cs_ena & (1 << cs_tmp))
1564 					cs_count++;
1565 			}
1566 
1567 			/* Init PBS I pattern */
1568 			sdram_addr = (cs_count * (SDRAM_CS_SIZE + 1) +
1569 				      SDRAM_PBS_I_OFFS);
1570 			if (MV_OK !=
1571 			    ddr3_sdram_compare(dram_info, (u32) NULL, NULL,
1572 					       pattern_ptr0, LEN_STD_PATTERN,
1573 					       sdram_addr, 1, 0, NULL,
1574 					       0))
1575 				return MV_FAIL;
1576 
1577 			/* Init PBS II pattern */
1578 			sdram_addr = (cs_count * (SDRAM_CS_SIZE + 1) +
1579 				      SDRAM_PBS_II_OFFS);
1580 			if (MV_OK !=
1581 			    ddr3_sdram_compare(dram_info, (u32) NULL, NULL,
1582 					       pattern_ptr1, LEN_STD_PATTERN,
1583 					       sdram_addr, 1, 0, NULL,
1584 					       0))
1585 				return MV_FAIL;
1586 		}
1587 	}
1588 
1589 	return MV_OK;
1590 }
1591 #endif
1592