1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (C) Marvell International Ltd. and its affiliates
4  */
5 
6 #include "ddr3_init.h"
7 #include "mv_ddr_training_db.h"
8 #include "ddr_training_ip_db.h"
9 #include "mv_ddr_regs.h"
10 
11 #define WL_ITERATION_NUM	10
12 
13 static u32 pup_mask_table[] = {
14 	0x000000ff,
15 	0x0000ff00,
16 	0x00ff0000,
17 	0xff000000
18 };
19 
20 static struct write_supp_result wr_supp_res[MAX_INTERFACE_NUM][MAX_BUS_NUM];
21 
22 static int ddr3_tip_dynamic_write_leveling_seq(u32 dev_num);
23 static int ddr3_tip_dynamic_read_leveling_seq(u32 dev_num);
24 static int ddr3_tip_dynamic_per_bit_read_leveling_seq(u32 dev_num);
25 static int ddr3_tip_wl_supp_align_phase_shift(u32 dev_num, u32 if_id,
26 					      u32 bus_id);
27 static int ddr3_tip_xsb_compare_test(u32 dev_num, u32 if_id, u32 bus_id,
28 				     u32 edge_offset);
29 
30 enum {
31 	PASS,
32 	FAIL
33 };
34 /*****************************************************************************
35 Dynamic read leveling
36 ******************************************************************************/
37 int ddr3_tip_dynamic_read_leveling(u32 dev_num, u32 freq)
38 {
39 	u32 data, mask;
40 	unsigned int max_cs = mv_ddr_cs_num_get();
41 	u32 bus_num, if_id, cl_val;
42 	enum mv_ddr_speed_bin speed_bin_index;
43 	/* save current CS value */
44 	u32 cs_enable_reg_val[MAX_INTERFACE_NUM] = { 0 };
45 	int is_any_pup_fail = 0;
46 	u32 data_read[MAX_INTERFACE_NUM + 1] = { 0 };
47 	u8 rl_values[MAX_CS_NUM][MAX_BUS_NUM][MAX_INTERFACE_NUM];
48 	struct pattern_info *pattern_table = ddr3_tip_get_pattern_table();
49 	u16 *mask_results_pup_reg_map = ddr3_tip_get_mask_results_pup_reg_map();
50 	u32 octets_per_if_num = ddr3_tip_dev_attr_get(dev_num, MV_ATTR_OCTET_PER_INTERFACE);
51 	struct mv_ddr_topology_map *tm = mv_ddr_topology_map_get();
52 
53 	for (effective_cs = 0; effective_cs < MAX_CS_NUM; effective_cs++)
54 		for (bus_num = 0; bus_num < MAX_BUS_NUM; bus_num++)
55 			for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++)
56 				rl_values[effective_cs][bus_num][if_id] = 0;
57 
58 	for (effective_cs = 0; effective_cs < max_cs; effective_cs++) {
59 		for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
60 			VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
61 			training_result[training_stage][if_id] = TEST_SUCCESS;
62 
63 			/* save current cs enable reg val */
64 			CHECK_STATUS(ddr3_tip_if_read
65 				     (dev_num, ACCESS_TYPE_UNICAST, if_id,
66 				      DUAL_DUNIT_CFG_REG, cs_enable_reg_val,
67 				      MASK_ALL_BITS));
68 			/* enable single cs */
69 			CHECK_STATUS(ddr3_tip_if_write
70 				     (dev_num, ACCESS_TYPE_UNICAST, if_id,
71 				      DUAL_DUNIT_CFG_REG, (1 << 3), (1 << 3)));
72 		}
73 
74 		ddr3_tip_reset_fifo_ptr(dev_num);
75 
76 		/*
77 		 *     Phase 1: Load pattern (using ODPG)
78 		 *
79 		 * enter Read Leveling mode
80 		 * only 27 bits are masked
81 		 * assuming non multi-CS configuration
82 		 * write to CS = 0 for the non multi CS configuration, note
83 		 * that the results shall be read back to the required CS !!!
84 		 */
85 
86 		/* BUS count is 0 shifted 26 */
87 		CHECK_STATUS(ddr3_tip_if_write
88 			     (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
89 			      ODPG_DATA_CTRL_REG, 0x3, 0x3));
90 		CHECK_STATUS(ddr3_tip_configure_odpg
91 			     (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE, 0,
92 			      pattern_table[PATTERN_RL].num_of_phases_tx, 0,
93 			      pattern_table[PATTERN_RL].num_of_phases_rx, 0, 0,
94 			      effective_cs, STRESS_NONE, DURATION_SINGLE));
95 
96 		/* load pattern to ODPG */
97 		ddr3_tip_load_pattern_to_odpg(dev_num, ACCESS_TYPE_MULTICAST,
98 					      PARAM_NOT_CARE, PATTERN_RL,
99 					      pattern_table[PATTERN_RL].
100 					      start_addr);
101 
102 		/*
103 		 *     Phase 2: ODPG to Read Leveling mode
104 		 */
105 
106 		/* General Training Opcode register */
107 		CHECK_STATUS(ddr3_tip_if_write
108 			     (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
109 			      ODPG_WR_RD_MODE_ENA_REG, 0,
110 			      MASK_ALL_BITS));
111 
112 		CHECK_STATUS(ddr3_tip_if_write
113 			     (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
114 			      GENERAL_TRAINING_OPCODE_REG,
115 			      (0x301b01 | effective_cs << 2), 0x3c3fef));
116 
117 		/* Object1 opcode register 0 & 1 */
118 		for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
119 			VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
120 			speed_bin_index =
121 				tm->interface_params[if_id].speed_bin_index;
122 			cl_val = mv_ddr_cl_val_get(speed_bin_index, freq);
123 			data = (cl_val << 17) | (0x3 << 25);
124 			mask = (0xff << 9) | (0x1f << 17) | (0x3 << 25);
125 			CHECK_STATUS(ddr3_tip_if_write
126 				     (dev_num, ACCESS_TYPE_UNICAST, if_id,
127 				      OPCODE_REG0_REG(1), data, mask));
128 		}
129 
130 		/* Set iteration count to max value */
131 		CHECK_STATUS(ddr3_tip_if_write
132 			     (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
133 			      OPCODE_REG1_REG(1), 0xd00, 0xd00));
134 
135 		/*
136 		 *     Phase 2: Mask config
137 		 */
138 
139 		ddr3_tip_dynamic_read_leveling_seq(dev_num);
140 
141 		/*
142 		 *     Phase 3: Read Leveling execution
143 		 */
144 
145 		/* temporary jira dunit=14751 */
146 		CHECK_STATUS(ddr3_tip_if_write
147 			     (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
148 			      TRAINING_DBG_1_REG, 0, (u32)(1 << 31)));
149 		/* configure phy reset value */
150 		CHECK_STATUS(ddr3_tip_if_write
151 			     (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
152 			      TRAINING_DBG_3_REG, (0x7f << 24),
153 			      (u32)(0xff << 24)));
154 		/* data pup rd reset enable  */
155 		CHECK_STATUS(ddr3_tip_if_write
156 			     (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
157 			      SDRAM_CFG_REG, 0, (1 << 30)));
158 		/* data pup rd reset disable */
159 		CHECK_STATUS(ddr3_tip_if_write
160 			     (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
161 			      SDRAM_CFG_REG, (1 << 30), (1 << 30)));
162 		/* training SW override & training RL mode */
163 		CHECK_STATUS(ddr3_tip_if_write
164 			     (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
165 			      TRAINING_SW_2_REG, 0x1, 0x9));
166 		/* training enable */
167 		CHECK_STATUS(ddr3_tip_if_write
168 			     (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
169 			      TRAINING_REG, (1 << 24) | (1 << 20),
170 			      (1 << 24) | (1 << 20)));
171 		CHECK_STATUS(ddr3_tip_if_write
172 			     (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
173 			      TRAINING_REG, (u32)(1 << 31), (u32)(1 << 31)));
174 
175 		/* trigger training */
176 		mv_ddr_training_enable();
177 
178 		/* check for training done */
179 		if (mv_ddr_is_training_done(MAX_POLLING_ITERATIONS, &data) != MV_OK) {
180 			DEBUG_LEVELING(DEBUG_LEVEL_ERROR, ("training done failed\n"));
181 			return MV_FAIL;
182 		}
183 		/* check for training pass */
184 		if (data != PASS)
185 			DEBUG_LEVELING(DEBUG_LEVEL_INFO, ("training result failed\n"));
186 
187 		/* disable odpg; switch back to functional mode */
188 		mv_ddr_odpg_disable();
189 
190 		if (mv_ddr_is_odpg_done(MAX_POLLING_ITERATIONS) != MV_OK) {
191 			DEBUG_LEVELING(DEBUG_LEVEL_ERROR, ("odpg disable failed\n"));
192 			return MV_FAIL;
193 		}
194 
195 		ddr3_tip_if_write(0, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
196 				  ODPG_DATA_CTRL_REG, 0, MASK_ALL_BITS);
197 
198 		/* double loop on bus, pup */
199 		for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
200 			VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
201 			/* check training done */
202 			is_any_pup_fail = 0;
203 			for (bus_num = 0;
204 			     bus_num < octets_per_if_num;
205 			     bus_num++) {
206 				VALIDATE_BUS_ACTIVE(tm->bus_act_mask, bus_num);
207 				if (ddr3_tip_if_polling
208 				    (dev_num, ACCESS_TYPE_UNICAST,
209 				     if_id, (1 << 25), (1 << 25),
210 				     mask_results_pup_reg_map[bus_num],
211 				     MAX_POLLING_ITERATIONS) != MV_OK) {
212 					DEBUG_LEVELING(DEBUG_LEVEL_ERROR,
213 						       ("\n_r_l: DDR3 poll failed(2) for IF %d CS %d bus %d",
214 							if_id, effective_cs, bus_num));
215 					is_any_pup_fail = 1;
216 				} else {
217 					/* read result per pup */
218 					CHECK_STATUS(ddr3_tip_if_read
219 						     (dev_num,
220 						      ACCESS_TYPE_UNICAST,
221 						      if_id,
222 						      mask_results_pup_reg_map
223 						      [bus_num], data_read,
224 						      0xff));
225 					rl_values[effective_cs][bus_num]
226 						[if_id] = (u8)data_read[if_id];
227 				}
228 			}
229 
230 			if (is_any_pup_fail == 1) {
231 				training_result[training_stage][if_id] =
232 					TEST_FAILED;
233 				if (debug_mode == 0)
234 					return MV_FAIL;
235 			}
236 		}
237 
238 		DEBUG_LEVELING(DEBUG_LEVEL_INFO, ("RL exit read leveling\n"));
239 
240 		/*
241 		 *     Phase 3: Exit Read Leveling
242 		 */
243 
244 		CHECK_STATUS(ddr3_tip_if_write
245 			     (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
246 			      TRAINING_SW_2_REG, (1 << 3), (1 << 3)));
247 		CHECK_STATUS(ddr3_tip_if_write
248 			     (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
249 			      TRAINING_SW_1_REG, (1 << 16), (1 << 16)));
250 		/* set ODPG to functional */
251 		CHECK_STATUS(ddr3_tip_if_write
252 			     (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
253 			      ODPG_DATA_CTRL_REG, 0x0, MASK_ALL_BITS));
254 
255 		/*
256 		 * Copy the result from the effective CS search to the
257 		 * real Functional CS
258 		 */
259 		/*ddr3_tip_write_cs_result(dev_num, RL_PHY_REG(0); */
260 		CHECK_STATUS(ddr3_tip_if_write
261 			     (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
262 			      ODPG_DATA_CTRL_REG, 0x0, MASK_ALL_BITS));
263 	}
264 
265 	for (effective_cs = 0; effective_cs < max_cs; effective_cs++) {
266 		/* double loop on bus, pup */
267 		for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
268 			VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
269 			for (bus_num = 0;
270 			     bus_num < octets_per_if_num;
271 			     bus_num++) {
272 				VALIDATE_BUS_ACTIVE(tm->bus_act_mask, bus_num);
273 				/* read result per pup from arry */
274 				data = rl_values[effective_cs][bus_num][if_id];
275 				data = (data & 0x1f) |
276 					(((data & 0xe0) >> 5) << 6);
277 				ddr3_tip_bus_write(dev_num,
278 						   ACCESS_TYPE_UNICAST,
279 						   if_id,
280 						   ACCESS_TYPE_UNICAST,
281 						   bus_num, DDR_PHY_DATA,
282 						   RL_PHY_REG(effective_cs),
283 						   data);
284 			}
285 		}
286 	}
287 	/* Set to 0 after each loop to avoid illegal value may be used */
288 	effective_cs = 0;
289 
290 	for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
291 		VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
292 		/* restore cs enable value */
293 		CHECK_STATUS(ddr3_tip_if_write
294 			     (dev_num, ACCESS_TYPE_UNICAST, if_id,
295 			      DUAL_DUNIT_CFG_REG, cs_enable_reg_val[if_id],
296 			      MASK_ALL_BITS));
297 		if (odt_config != 0) {
298 			CHECK_STATUS(ddr3_tip_write_additional_odt_setting
299 				     (dev_num, if_id));
300 		}
301 	}
302 
303 	for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
304 		VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
305 		if (training_result[training_stage][if_id] == TEST_FAILED)
306 			return MV_FAIL;
307 	}
308 
309 	return MV_OK;
310 }
311 
312 /*
313  * Legacy Dynamic write leveling
314  */
315 int ddr3_tip_legacy_dynamic_write_leveling(u32 dev_num)
316 {
317 	u32 c_cs, if_id, cs_mask = 0;
318 	unsigned int max_cs = mv_ddr_cs_num_get();
319 	struct mv_ddr_topology_map *tm = mv_ddr_topology_map_get();
320 
321 	/*
322 	 * In TRAINIUNG reg (0x15b0) write 0x80000008 | cs_mask:
323 	 * Trn_start
324 	 * cs_mask = 0x1 <<20 Trn_CS0 - CS0 is included in the DDR3 training
325 	 * cs_mask = 0x1 <<21 Trn_CS1 - CS1 is included in the DDR3 training
326 	 * cs_mask = 0x1 <<22 Trn_CS2 - CS2 is included in the DDR3 training
327 	 * cs_mask = 0x1 <<23 Trn_CS3 - CS3 is included in the DDR3 training
328 	 * Trn_auto_seq =  write leveling
329 	 */
330 	for (c_cs = 0; c_cs < max_cs; c_cs++)
331 		cs_mask = cs_mask | 1 << (20 + c_cs);
332 
333 	for (if_id = 0; if_id < MAX_INTERFACE_NUM; if_id++) {
334 		VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
335 		CHECK_STATUS(ddr3_tip_if_write
336 			     (dev_num, ACCESS_TYPE_MULTICAST, 0,
337 			      TRAINING_REG, (0x80000008 | cs_mask),
338 			      0xffffffff));
339 		mdelay(20);
340 		if (ddr3_tip_if_polling
341 		    (dev_num, ACCESS_TYPE_UNICAST, if_id, 0,
342 		     (u32)0x80000000, TRAINING_REG,
343 		     MAX_POLLING_ITERATIONS) != MV_OK) {
344 			DEBUG_LEVELING(DEBUG_LEVEL_ERROR,
345 				       ("polling failed for Old WL result\n"));
346 			return MV_FAIL;
347 		}
348 	}
349 
350 	return MV_OK;
351 }
352 
353 /*
354  * Legacy Dynamic read leveling
355  */
356 int ddr3_tip_legacy_dynamic_read_leveling(u32 dev_num)
357 {
358 	u32 c_cs, if_id, cs_mask = 0;
359 	unsigned int max_cs = mv_ddr_cs_num_get();
360 	struct mv_ddr_topology_map *tm = mv_ddr_topology_map_get();
361 
362 	/*
363 	 * In TRAINIUNG reg (0x15b0) write 0x80000040 | cs_mask:
364 	 * Trn_start
365 	 * cs_mask = 0x1 <<20 Trn_CS0 - CS0 is included in the DDR3 training
366 	 * cs_mask = 0x1 <<21 Trn_CS1 - CS1 is included in the DDR3 training
367 	 * cs_mask = 0x1 <<22 Trn_CS2 - CS2 is included in the DDR3 training
368 	 * cs_mask = 0x1 <<23 Trn_CS3 - CS3 is included in the DDR3 training
369 	 * Trn_auto_seq =  Read Leveling using training pattern
370 	 */
371 	for (c_cs = 0; c_cs < max_cs; c_cs++)
372 		cs_mask = cs_mask | 1 << (20 + c_cs);
373 
374 	CHECK_STATUS(ddr3_tip_if_write
375 		     (dev_num, ACCESS_TYPE_MULTICAST, 0, TRAINING_REG,
376 		      (0x80000040 | cs_mask), 0xffffffff));
377 	mdelay(100);
378 
379 	for (if_id = 0; if_id < MAX_INTERFACE_NUM; if_id++) {
380 		VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
381 		if (ddr3_tip_if_polling
382 		    (dev_num, ACCESS_TYPE_UNICAST, if_id, 0,
383 		     (u32)0x80000000, TRAINING_REG,
384 		     MAX_POLLING_ITERATIONS) != MV_OK) {
385 			DEBUG_LEVELING(DEBUG_LEVEL_ERROR,
386 				       ("polling failed for Old RL result\n"));
387 			return MV_FAIL;
388 		}
389 	}
390 
391 	return MV_OK;
392 }
393 
394 /*
395  * Dynamic per bit read leveling
396  */
397 int ddr3_tip_dynamic_per_bit_read_leveling(u32 dev_num, u32 freq)
398 {
399 	u32 data, mask;
400 	u32 bus_num, if_id, cl_val, bit_num;
401 	u32 curr_numb, curr_min_delay;
402 	int adll_array[3] = { 0, -0xa, 0x14 };
403 	u32 phyreg3_arr[MAX_INTERFACE_NUM][MAX_BUS_NUM];
404 	enum mv_ddr_speed_bin speed_bin_index;
405 	int is_any_pup_fail = 0;
406 	int break_loop = 0;
407 	u32 cs_enable_reg_val[MAX_INTERFACE_NUM]; /* save current CS value */
408 	u32 data_read[MAX_INTERFACE_NUM];
409 	int per_bit_rl_pup_status[MAX_INTERFACE_NUM][MAX_BUS_NUM];
410 	u32 data2_write[MAX_INTERFACE_NUM][MAX_BUS_NUM];
411 	struct pattern_info *pattern_table = ddr3_tip_get_pattern_table();
412 	u16 *mask_results_dq_reg_map = ddr3_tip_get_mask_results_dq_reg();
413 	u32 octets_per_if_num = ddr3_tip_dev_attr_get(dev_num, MV_ATTR_OCTET_PER_INTERFACE);
414 	struct mv_ddr_topology_map *tm = mv_ddr_topology_map_get();
415 
416 	for (if_id = 0; if_id < MAX_INTERFACE_NUM; if_id++) {
417 		VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
418 		for (bus_num = 0;
419 		     bus_num <= octets_per_if_num; bus_num++) {
420 			VALIDATE_BUS_ACTIVE(tm->bus_act_mask, bus_num);
421 			per_bit_rl_pup_status[if_id][bus_num] = 0;
422 			data2_write[if_id][bus_num] = 0;
423 			/* read current value of phy register 0x3 */
424 			CHECK_STATUS(ddr3_tip_bus_read
425 				     (dev_num, if_id, ACCESS_TYPE_UNICAST,
426 				      bus_num, DDR_PHY_DATA,
427 				      CRX_PHY_REG(0),
428 				      &phyreg3_arr[if_id][bus_num]));
429 		}
430 	}
431 
432 	/* NEW RL machine */
433 	for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
434 		VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
435 		training_result[training_stage][if_id] = TEST_SUCCESS;
436 
437 		/* save current cs enable reg val */
438 		CHECK_STATUS(ddr3_tip_if_read
439 			     (dev_num, ACCESS_TYPE_UNICAST, if_id,
440 			      DUAL_DUNIT_CFG_REG, &cs_enable_reg_val[if_id],
441 			      MASK_ALL_BITS));
442 		/* enable single cs */
443 		CHECK_STATUS(ddr3_tip_if_write
444 			     (dev_num, ACCESS_TYPE_UNICAST, if_id,
445 			      DUAL_DUNIT_CFG_REG, (1 << 3), (1 << 3)));
446 	}
447 
448 	ddr3_tip_reset_fifo_ptr(dev_num);
449 	for (curr_numb = 0; curr_numb < 3; curr_numb++) {
450 		/*
451 		 *     Phase 1: Load pattern (using ODPG)
452 		 *
453 		 * enter Read Leveling mode
454 		 * only 27 bits are masked
455 		 * assuming non multi-CS configuration
456 		 * write to CS = 0 for the non multi CS configuration, note that
457 		 * the results shall be read back to the required CS !!!
458 		 */
459 
460 		/* BUS count is 0 shifted 26 */
461 		CHECK_STATUS(ddr3_tip_if_write
462 			     (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
463 			      ODPG_DATA_CTRL_REG, 0x3, 0x3));
464 		CHECK_STATUS(ddr3_tip_configure_odpg
465 			     (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE, 0,
466 			      pattern_table[PATTERN_TEST].num_of_phases_tx, 0,
467 			      pattern_table[PATTERN_TEST].num_of_phases_rx, 0,
468 			      0, 0, STRESS_NONE, DURATION_SINGLE));
469 
470 		/* load pattern to ODPG */
471 		ddr3_tip_load_pattern_to_odpg(dev_num, ACCESS_TYPE_MULTICAST,
472 					      PARAM_NOT_CARE, PATTERN_TEST,
473 					      pattern_table[PATTERN_TEST].
474 					      start_addr);
475 
476 		/*
477 		 *     Phase 2: ODPG to Read Leveling mode
478 		 */
479 
480 		/* General Training Opcode register */
481 		CHECK_STATUS(ddr3_tip_if_write
482 			     (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
483 			      ODPG_WR_RD_MODE_ENA_REG, 0,
484 			      MASK_ALL_BITS));
485 		CHECK_STATUS(ddr3_tip_if_write
486 			     (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
487 			      GENERAL_TRAINING_OPCODE_REG, 0x301b01, 0x3c3fef));
488 
489 		/* Object1 opcode register 0 & 1 */
490 		for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
491 			VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
492 			speed_bin_index =
493 				tm->interface_params[if_id].speed_bin_index;
494 			cl_val = mv_ddr_cl_val_get(speed_bin_index, freq);
495 			data = (cl_val << 17) | (0x3 << 25);
496 			mask = (0xff << 9) | (0x1f << 17) | (0x3 << 25);
497 			CHECK_STATUS(ddr3_tip_if_write
498 				     (dev_num, ACCESS_TYPE_UNICAST, if_id,
499 				      OPCODE_REG0_REG(1), data, mask));
500 		}
501 
502 		/* Set iteration count to max value */
503 		CHECK_STATUS(ddr3_tip_if_write
504 			     (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
505 			      OPCODE_REG1_REG(1), 0xd00, 0xd00));
506 
507 		/*
508 		 *     Phase 2: Mask config
509 		 */
510 
511 		ddr3_tip_dynamic_per_bit_read_leveling_seq(dev_num);
512 
513 		/*
514 		 *     Phase 3: Read Leveling execution
515 		 */
516 
517 		/* temporary jira dunit=14751 */
518 		CHECK_STATUS(ddr3_tip_if_write
519 			     (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
520 			      TRAINING_DBG_1_REG, 0, (u32)(1 << 31)));
521 		/* configure phy reset value */
522 		CHECK_STATUS(ddr3_tip_if_write
523 			     (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
524 			      TRAINING_DBG_3_REG, (0x7f << 24),
525 			      (u32)(0xff << 24)));
526 		/* data pup rd reset enable  */
527 		CHECK_STATUS(ddr3_tip_if_write
528 			     (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
529 			      SDRAM_CFG_REG, 0, (1 << 30)));
530 		/* data pup rd reset disable */
531 		CHECK_STATUS(ddr3_tip_if_write
532 			     (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
533 			      SDRAM_CFG_REG, (1 << 30), (1 << 30)));
534 		/* training SW override & training RL mode */
535 		CHECK_STATUS(ddr3_tip_if_write
536 			     (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
537 			      TRAINING_SW_2_REG, 0x1, 0x9));
538 		/* training enable */
539 		CHECK_STATUS(ddr3_tip_if_write
540 			     (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
541 			      TRAINING_REG, (1 << 24) | (1 << 20),
542 			      (1 << 24) | (1 << 20)));
543 		CHECK_STATUS(ddr3_tip_if_write
544 			     (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
545 			      TRAINING_REG, (u32)(1 << 31), (u32)(1 << 31)));
546 
547 		/* trigger training */
548 		mv_ddr_training_enable();
549 
550 		/* check for training done */
551 		if (mv_ddr_is_training_done(MAX_POLLING_ITERATIONS, &data) != MV_OK) {
552 			DEBUG_LEVELING(DEBUG_LEVEL_ERROR, ("training done failed\n"));
553 			return MV_FAIL;
554 		}
555 		/* check for training pass */
556 		if (data != PASS)
557 			DEBUG_LEVELING(DEBUG_LEVEL_INFO, ("training result failed\n"));
558 
559 		/* disable odpg; switch back to functional mode */
560 		mv_ddr_odpg_disable();
561 
562 		if (mv_ddr_is_odpg_done(MAX_POLLING_ITERATIONS) != MV_OK) {
563 			DEBUG_LEVELING(DEBUG_LEVEL_ERROR, ("odpg disable failed\n"));
564 			return MV_FAIL;
565 		}
566 
567 		ddr3_tip_if_write(0, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
568 				  ODPG_DATA_CTRL_REG, 0, MASK_ALL_BITS);
569 
570 		/* double loop on bus, pup */
571 		for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
572 			VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
573 			/* check training done */
574 			for (bus_num = 0;
575 			     bus_num < octets_per_if_num;
576 			     bus_num++) {
577 				VALIDATE_BUS_ACTIVE(tm->bus_act_mask, bus_num);
578 
579 				if (per_bit_rl_pup_status[if_id][bus_num]
580 				    == 0) {
581 					curr_min_delay = 0;
582 					for (bit_num = 0; bit_num < 8;
583 					     bit_num++) {
584 						if (ddr3_tip_if_polling
585 						    (dev_num,
586 						     ACCESS_TYPE_UNICAST,
587 						     if_id, (1 << 25),
588 						     (1 << 25),
589 						     mask_results_dq_reg_map
590 						     [bus_num * 8 + bit_num],
591 						     MAX_POLLING_ITERATIONS) !=
592 						    MV_OK) {
593 							DEBUG_LEVELING
594 								(DEBUG_LEVEL_ERROR,
595 								 ("\n_r_l: DDR3 poll failed(2) for bus %d bit %d\n",
596 								  bus_num,
597 								  bit_num));
598 						} else {
599 							/* read result per pup */
600 							CHECK_STATUS
601 								(ddr3_tip_if_read
602 								 (dev_num,
603 								  ACCESS_TYPE_UNICAST,
604 								  if_id,
605 								  mask_results_dq_reg_map
606 								  [bus_num * 8 +
607 								   bit_num],
608 								  data_read,
609 								  MASK_ALL_BITS));
610 							data =
611 								(data_read
612 								 [if_id] &
613 								 0x1f) |
614 								((data_read
615 								  [if_id] &
616 								  0xe0) << 1);
617 							if (curr_min_delay == 0)
618 								curr_min_delay =
619 									data;
620 							else if (data <
621 								 curr_min_delay)
622 								curr_min_delay =
623 									data;
624 							if (data > data2_write[if_id][bus_num])
625 								data2_write
626 									[if_id]
627 									[bus_num] =
628 									data;
629 						}
630 					}
631 
632 					if (data2_write[if_id][bus_num] <=
633 					    (curr_min_delay +
634 					     MAX_DQ_READ_LEVELING_DELAY)) {
635 						per_bit_rl_pup_status[if_id]
636 							[bus_num] = 1;
637 					}
638 				}
639 			}
640 		}
641 
642 		/* check if there is need to search new phyreg3 value */
643 		if (curr_numb < 2) {
644 			/* if there is DLL that is not checked yet */
645 			for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1;
646 			     if_id++) {
647 				VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
648 				for (bus_num = 0;
649 				     bus_num < octets_per_if_num;
650 				     bus_num++) {
651 					VALIDATE_BUS_ACTIVE(tm->bus_act_mask,
652 							bus_num);
653 					if (per_bit_rl_pup_status[if_id]
654 					    [bus_num] != 1) {
655 						/* go to next ADLL value */
656 						CHECK_STATUS
657 							(ddr3_tip_bus_write
658 							 (dev_num,
659 							  ACCESS_TYPE_UNICAST,
660 							  if_id,
661 							  ACCESS_TYPE_UNICAST,
662 							  bus_num, DDR_PHY_DATA,
663 							  CRX_PHY_REG(0),
664 							  (phyreg3_arr[if_id]
665 							   [bus_num] +
666 							   adll_array[curr_numb])));
667 						break_loop = 1;
668 						break;
669 					}
670 				}
671 				if (break_loop)
672 					break;
673 			}
674 		}		/* if (curr_numb < 2) */
675 		if (!break_loop)
676 			break;
677 	}		/* for ( curr_numb = 0; curr_numb <3; curr_numb++) */
678 
679 	for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
680 		VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
681 		for (bus_num = 0; bus_num < octets_per_if_num;
682 		     bus_num++) {
683 			VALIDATE_BUS_ACTIVE(tm->bus_act_mask, bus_num);
684 			if (per_bit_rl_pup_status[if_id][bus_num] == 1)
685 				ddr3_tip_bus_write(dev_num,
686 						   ACCESS_TYPE_UNICAST,
687 						   if_id,
688 						   ACCESS_TYPE_UNICAST,
689 						   bus_num, DDR_PHY_DATA,
690 						   RL_PHY_REG(effective_cs),
691 						   data2_write[if_id]
692 						   [bus_num]);
693 			else
694 				is_any_pup_fail = 1;
695 		}
696 
697 		/* TBD flow does not support multi CS */
698 		/*
699 		 * cs_bitmask = tm->interface_params[if_id].
700 		 * as_bus_params[bus_num].cs_bitmask;
701 		 */
702 		/* divide by 4 is used for retrieving the CS number */
703 		/*
704 		 * TBD BC2 - what is the PHY address for other
705 		 * CS ddr3_tip_write_cs_result() ???
706 		 */
707 		/*
708 		 * find what should be written to PHY
709 		 * - max delay that is less than threshold
710 		 */
711 		if (is_any_pup_fail == 1) {
712 			training_result[training_stage][if_id] = TEST_FAILED;
713 			if (debug_mode == 0)
714 				return MV_FAIL;
715 		}
716 	}
717 	DEBUG_LEVELING(DEBUG_LEVEL_INFO, ("RL exit read leveling\n"));
718 
719 	/*
720 	 *     Phase 3: Exit Read Leveling
721 	 */
722 
723 	CHECK_STATUS(ddr3_tip_if_write
724 		     (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
725 		      TRAINING_SW_2_REG, (1 << 3), (1 << 3)));
726 	CHECK_STATUS(ddr3_tip_if_write
727 		     (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
728 		      TRAINING_SW_1_REG, (1 << 16), (1 << 16)));
729 	/* set ODPG to functional */
730 	CHECK_STATUS(ddr3_tip_if_write
731 		     (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
732 		      ODPG_DATA_CTRL_REG, 0x0, MASK_ALL_BITS));
733 	/*
734 	 * Copy the result from the effective CS search to the real
735 	 * Functional CS
736 	 */
737 	ddr3_tip_write_cs_result(dev_num, RL_PHY_REG(0));
738 	CHECK_STATUS(ddr3_tip_if_write
739 		     (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
740 		      ODPG_DATA_CTRL_REG, 0x0, MASK_ALL_BITS));
741 
742 	for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
743 		VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
744 		/* restore cs enable value */
745 		CHECK_STATUS(ddr3_tip_if_write
746 			     (dev_num, ACCESS_TYPE_UNICAST, if_id,
747 			      DUAL_DUNIT_CFG_REG, cs_enable_reg_val[if_id],
748 			      MASK_ALL_BITS));
749 		if (odt_config != 0) {
750 			CHECK_STATUS(ddr3_tip_write_additional_odt_setting
751 				     (dev_num, if_id));
752 		}
753 	}
754 
755 	for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
756 		VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
757 		if (training_result[training_stage][if_id] == TEST_FAILED)
758 			return MV_FAIL;
759 	}
760 
761 	return MV_OK;
762 }
763 
764 int ddr3_tip_calc_cs_mask(u32 dev_num, u32 if_id, u32 effective_cs,
765 			  u32 *cs_mask)
766 {
767 	u32 all_bus_cs = 0, same_bus_cs;
768 	u32 bus_cnt;
769 	u32 octets_per_if_num = ddr3_tip_dev_attr_get(dev_num, MV_ATTR_OCTET_PER_INTERFACE);
770 	struct mv_ddr_topology_map *tm = mv_ddr_topology_map_get();
771 
772 	*cs_mask = same_bus_cs = CS_BIT_MASK;
773 
774 	/*
775 	 * In some of the devices (such as BC2), the CS is per pup and there
776 	 * for mixed mode is valid on like other devices where CS configuration
777 	 * is per interface.
778 	 * In order to know that, we do 'Or' and 'And' operation between all
779 	 * CS (of the pups).
780 	 * If they are they are not the same then it's mixed mode so all CS
781 	 * should be configured (when configuring the MRS)
782 	 */
783 	for (bus_cnt = 0; bus_cnt < octets_per_if_num; bus_cnt++) {
784 		VALIDATE_BUS_ACTIVE(tm->bus_act_mask, bus_cnt);
785 
786 		all_bus_cs |= tm->interface_params[if_id].
787 			as_bus_params[bus_cnt].cs_bitmask;
788 		same_bus_cs &= tm->interface_params[if_id].
789 			as_bus_params[bus_cnt].cs_bitmask;
790 
791 		/* cs enable is active low */
792 		*cs_mask &= ~tm->interface_params[if_id].
793 			as_bus_params[bus_cnt].cs_bitmask;
794 	}
795 
796 	if (all_bus_cs == same_bus_cs)
797 		*cs_mask = (*cs_mask | (~(1 << effective_cs))) & CS_BIT_MASK;
798 
799 	return MV_OK;
800 }
801 
802 /*
803  * Dynamic write leveling
804  */
805 int ddr3_tip_dynamic_write_leveling(u32 dev_num, int phase_remove)
806 {
807 	u32 reg_data = 0, temp = 0, iter, if_id, bus_cnt;
808 	u32 cs_enable_reg_val[MAX_INTERFACE_NUM] = { 0 };
809 	u32 cs_mask[MAX_INTERFACE_NUM];
810 	u32 read_data_sample_delay_vals[MAX_INTERFACE_NUM] = { 0 };
811 	u32 read_data_ready_delay_vals[MAX_INTERFACE_NUM] = { 0 };
812 	/* 0 for failure */
813 	u32 res_values[MAX_INTERFACE_NUM * MAX_BUS_NUM] = { 0 };
814 	u32 test_res = 0;	/* 0 - success for all pup */
815 	u32 data_read[MAX_INTERFACE_NUM];
816 	u8 wl_values[MAX_CS_NUM][MAX_BUS_NUM][MAX_INTERFACE_NUM];
817 	u16 *mask_results_pup_reg_map = ddr3_tip_get_mask_results_pup_reg_map();
818 	u32 cs_mask0[MAX_INTERFACE_NUM] = { 0 };
819 	unsigned int max_cs = mv_ddr_cs_num_get();
820 	u32 octets_per_if_num = ddr3_tip_dev_attr_get(dev_num, MV_ATTR_OCTET_PER_INTERFACE);
821 	struct mv_ddr_topology_map *tm = mv_ddr_topology_map_get();
822 
823 	for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
824 		VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
825 
826 		training_result[training_stage][if_id] = TEST_SUCCESS;
827 
828 		/* save Read Data Sample Delay */
829 		CHECK_STATUS(ddr3_tip_if_read
830 			     (dev_num, ACCESS_TYPE_UNICAST, if_id,
831 			      RD_DATA_SMPL_DLYS_REG,
832 			      read_data_sample_delay_vals, MASK_ALL_BITS));
833 		/* save Read Data Ready Delay */
834 		CHECK_STATUS(ddr3_tip_if_read
835 			     (dev_num, ACCESS_TYPE_UNICAST, if_id,
836 			      RD_DATA_RDY_DLYS_REG, read_data_ready_delay_vals,
837 			      MASK_ALL_BITS));
838 		/* save current cs reg val */
839 		CHECK_STATUS(ddr3_tip_if_read
840 			     (dev_num, ACCESS_TYPE_UNICAST, if_id,
841 			      DUAL_DUNIT_CFG_REG, cs_enable_reg_val, MASK_ALL_BITS));
842 	}
843 
844 	if (ddr3_tip_dev_attr_get(dev_num, MV_ATTR_TIP_REV) < MV_TIP_REV_3) {
845 		/* Enable multi-CS */
846 		CHECK_STATUS(ddr3_tip_if_write
847 			     (dev_num, ACCESS_TYPE_UNICAST, if_id,
848 			     DUAL_DUNIT_CFG_REG, 0, (1 << 3)));
849 	}
850 
851 	/*
852 	 *     Phase 1: DRAM 2 Write Leveling mode
853 	 */
854 
855 	/*Assert 10 refresh commands to DRAM to all CS */
856 	for (iter = 0; iter < WL_ITERATION_NUM; iter++) {
857 		for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
858 			VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
859 			CHECK_STATUS(ddr3_tip_if_write
860 				     (dev_num, ACCESS_TYPE_UNICAST,
861 				      if_id, SDRAM_OP_REG,
862 				      (u32)((~(0xf) << 8) | 0x2), 0xf1f));
863 		}
864 	}
865 	/* check controller back to normal */
866 	for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
867 		VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
868 		if (ddr3_tip_if_polling
869 		    (dev_num, ACCESS_TYPE_UNICAST, if_id, 0, 0x1f,
870 		     SDRAM_OP_REG, MAX_POLLING_ITERATIONS) != MV_OK) {
871 			DEBUG_LEVELING(DEBUG_LEVEL_ERROR,
872 				       ("WL: DDR3 poll failed(3)"));
873 		}
874 	}
875 
876 	for (effective_cs = 0; effective_cs < max_cs; effective_cs++) {
877 		/*enable write leveling to all cs  - Q off , WL n */
878 		/* calculate interface cs mask */
879 		CHECK_STATUS(ddr3_tip_write_mrs_cmd(dev_num, cs_mask0, MR_CMD1,
880 						    0x1000, 0x1080));
881 
882 		for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
883 			VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
884 			/* cs enable is active low */
885 			ddr3_tip_calc_cs_mask(dev_num, if_id, effective_cs,
886 					      &cs_mask[if_id]);
887 		}
888 
889 		if (ddr3_tip_dev_attr_get(dev_num, MV_ATTR_TIP_REV) >= MV_TIP_REV_3) {
890 			/* Enable Output buffer to relevant CS - Q on , WL on */
891 			CHECK_STATUS(ddr3_tip_write_mrs_cmd
892 				     (dev_num, cs_mask, MR_CMD1, 0x80, 0x1080));
893 
894 			/*enable odt for relevant CS */
895 			CHECK_STATUS(ddr3_tip_if_write
896 				     (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
897 				      0x1498, (0x3 << (effective_cs * 2)), 0xf));
898 		} else {
899 			/* FIXME: should be the same as _CPU case */
900 			CHECK_STATUS(ddr3_tip_write_mrs_cmd
901 				     (dev_num, cs_mask, MR_CMD1, 0xc0, 0x12c4));
902 		}
903 
904 		/*
905 		 *     Phase 2: Set training IP to write leveling mode
906 		 */
907 
908 		CHECK_STATUS(ddr3_tip_dynamic_write_leveling_seq(dev_num));
909 
910 		/* phase 3: trigger training */
911 		mv_ddr_training_enable();
912 
913 		/* check for training done */
914 		if (mv_ddr_is_training_done(MAX_POLLING_ITERATIONS, data_read) != MV_OK) {
915 			DEBUG_LEVELING(DEBUG_LEVEL_ERROR, ("training done failed\n"));
916 		} else { /* check for training pass */
917 			reg_data = data_read[0];
918 #if defined(CONFIG_ARMADA_38X) /* JIRA #1498 for 16 bit with ECC */
919 			if (tm->bus_act_mask == 0xb) /* set to data to 0 to skip the check */
920 				reg_data = 0;
921 #endif
922 			if (reg_data != PASS)
923 				DEBUG_LEVELING(DEBUG_LEVEL_INFO, ("training result failed\n"));
924 
925 			/* check for training completion per bus */
926 			for (bus_cnt = 0; bus_cnt < octets_per_if_num; bus_cnt++) {
927 				VALIDATE_BUS_ACTIVE(tm->bus_act_mask, bus_cnt);
928 				/* training status */
929 				ddr3_tip_if_read(0, ACCESS_TYPE_UNICAST, 0,
930 					      mask_results_pup_reg_map[bus_cnt],
931 					      data_read, MASK_ALL_BITS);
932 				reg_data = data_read[0];
933 				DEBUG_LEVELING(DEBUG_LEVEL_TRACE, ("WL: IF %d BUS %d reg 0x%x\n",
934 								   0, bus_cnt, reg_data));
935 				if ((reg_data & (1 << 25)) == 0)
936 					res_values[bus_cnt] = 1;
937 				ddr3_tip_if_read(0, ACCESS_TYPE_UNICAST, 0,
938 					      mask_results_pup_reg_map[bus_cnt],
939 					      data_read, 0xff);
940 				/*
941 				 * Save the read value that should be
942 				 * write to PHY register
943 				 */
944 				wl_values[effective_cs][bus_cnt][0] = (u8)data_read[0];
945 			}
946 		}
947 
948 		/*
949 		 *     Phase 3.5: Validate result
950 		 */
951 		for (if_id = 0; if_id < MAX_INTERFACE_NUM; if_id++) {
952 			VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
953 			for (bus_cnt = 0; bus_cnt < octets_per_if_num; bus_cnt++) {
954 				VALIDATE_BUS_ACTIVE(tm->bus_act_mask, bus_cnt);
955 				/*
956 				 * Read result control register according to subphy
957 				 * "16" below is for a half-phase
958 				 */
959 				reg_data = wl_values[effective_cs][bus_cnt][if_id] + 16;
960 				/*
961 				 * Write to WL register: ADLL [4:0], Phase [8:6],
962 				 * Centralization ADLL [15:10] + 0x10
963 				 */
964 				reg_data = (reg_data & 0x1f) |
965 					   (((reg_data & 0xe0) >> 5) << 6) |
966 					   (((reg_data & 0x1f) + phy_reg1_val) << 10);
967 				/* Search with WL CS0 subphy reg */
968 				ddr3_tip_bus_write(dev_num, ACCESS_TYPE_UNICAST, if_id,
969 						   ACCESS_TYPE_UNICAST, bus_cnt,
970 						   DDR_PHY_DATA, WL_PHY_REG(0), reg_data);
971 				/*
972 				 * Check for change in data read from DRAM.
973 				 * If changed, fix the result
974 				 */
975 				CHECK_STATUS(ddr3_tip_if_read
976 					     (dev_num,
977 					      ACCESS_TYPE_UNICAST,
978 					      if_id,
979 					      TRAINING_WL_REG,
980 					      data_read, MASK_ALL_BITS));
981 				if (((data_read[if_id] & (1 << (bus_cnt + 20))) >>
982 				     (bus_cnt + 20)) == 0) {
983 					DEBUG_LEVELING(
984 						DEBUG_LEVEL_ERROR,
985 						("WLValues was changed from 0x%X",
986 						 wl_values[effective_cs]
987 						 [bus_cnt][if_id]));
988 					wl_values[effective_cs]
989 					[bus_cnt][if_id] += 32;
990 					DEBUG_LEVELING(
991 						DEBUG_LEVEL_ERROR,
992 						("to 0x%X",
993 						 wl_values[effective_cs]
994 						 [bus_cnt][if_id]));
995 				}
996 			}
997 		}
998 
999 		/*
1000 		 *     Phase 4: Exit write leveling mode
1001 		 */
1002 
1003 		/* disable DQs toggling */
1004 		CHECK_STATUS(ddr3_tip_if_write
1005 			     (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1006 			      WL_DQS_PATTERN_REG, 0x0, 0x1));
1007 
1008 		/* Update MRS 1 (WL off) */
1009 		if (ddr3_tip_dev_attr_get(dev_num, MV_ATTR_TIP_REV) >= MV_TIP_REV_3) {
1010 			CHECK_STATUS(ddr3_tip_write_mrs_cmd(dev_num, cs_mask0, MR_CMD1,
1011 							    0x1000, 0x1080));
1012 		} else {
1013 			/* FIXME: should be same as _CPU case */
1014 			CHECK_STATUS(ddr3_tip_write_mrs_cmd(dev_num, cs_mask0, MR_CMD1,
1015 							    0x1000, 0x12c4));
1016 		}
1017 
1018 		/* Update MRS 1 (return to functional mode - Q on , WL off) */
1019 		CHECK_STATUS(ddr3_tip_write_mrs_cmd
1020 			     (dev_num, cs_mask0, MR_CMD1, 0x0, 0x1080));
1021 
1022 		/* set phy to normal mode */
1023 		CHECK_STATUS(ddr3_tip_if_write
1024 			     (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1025 			      TRAINING_SW_2_REG, 0x5, 0x7));
1026 
1027 		/* exit sw override mode  */
1028 		CHECK_STATUS(ddr3_tip_if_write
1029 			     (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1030 			      TRAINING_SW_2_REG, 0x4, 0x7));
1031 	}
1032 
1033 	/*
1034 	 *     Phase 5: Load WL values to each PHY
1035 	 */
1036 
1037 	for (effective_cs = 0; effective_cs < max_cs; effective_cs++) {
1038 		for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
1039 			VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
1040 			test_res = 0;
1041 			for (bus_cnt = 0;
1042 			     bus_cnt < octets_per_if_num;
1043 			     bus_cnt++) {
1044 				VALIDATE_BUS_ACTIVE(tm->bus_act_mask, bus_cnt);
1045 				/* check if result == pass */
1046 				if (res_values
1047 				    [(if_id *
1048 				      octets_per_if_num) +
1049 				     bus_cnt] == 0) {
1050 					/*
1051 					 * read result control register
1052 					 * according to pup
1053 					 */
1054 					reg_data =
1055 						wl_values[effective_cs][bus_cnt]
1056 						[if_id];
1057 					/*
1058 					 * Write into write leveling register
1059 					 * ([4:0] ADLL, [8:6] Phase, [15:10]
1060 					 * (centralization) ADLL + 0x10)
1061 					 */
1062 					reg_data =
1063 						(reg_data & 0x1f) |
1064 						(((reg_data & 0xe0) >> 5) << 6) |
1065 						(((reg_data & 0x1f) +
1066 						  phy_reg1_val) << 10);
1067 					/*
1068 					 * in case phase remove should be executed
1069 					 * need to remove more than one phase.
1070 					 * this will take place only in low frequency,
1071 					 * where there could be more than one phase between sub-phys
1072 					 */
1073 					if (phase_remove == 1) {
1074 						temp = (reg_data >> WR_LVL_PH_SEL_OFFS) & WR_LVL_PH_SEL_PHASE1;
1075 						reg_data &= ~(WR_LVL_PH_SEL_MASK << WR_LVL_PH_SEL_OFFS);
1076 						reg_data |= (temp << WR_LVL_PH_SEL_OFFS);
1077 					}
1078 
1079 					ddr3_tip_bus_write(
1080 						dev_num,
1081 						ACCESS_TYPE_UNICAST,
1082 						if_id,
1083 						ACCESS_TYPE_UNICAST,
1084 						bus_cnt,
1085 						DDR_PHY_DATA,
1086 						WL_PHY_REG(effective_cs),
1087 						reg_data);
1088 				} else {
1089 					test_res = 1;
1090 					/*
1091 					 * read result control register
1092 					 * according to pup
1093 					 */
1094 					CHECK_STATUS(ddr3_tip_if_read
1095 						     (dev_num,
1096 						      ACCESS_TYPE_UNICAST,
1097 						      if_id,
1098 						      mask_results_pup_reg_map
1099 						      [bus_cnt], data_read,
1100 						      0xff));
1101 					reg_data = data_read[if_id];
1102 					DEBUG_LEVELING(
1103 						DEBUG_LEVEL_ERROR,
1104 						("WL: IF %d BUS %d failed, reg 0x%x\n",
1105 						 if_id, bus_cnt, reg_data));
1106 				}
1107 			}
1108 
1109 			if (test_res != 0) {
1110 				training_result[training_stage][if_id] =
1111 					TEST_FAILED;
1112 			}
1113 		}
1114 	}
1115 	/* Set to 0 after each loop to avoid illegal value may be used */
1116 	effective_cs = 0;
1117 
1118 	/*
1119 	 * Copy the result from the effective CS search to the real
1120 	 * Functional CS
1121 	 */
1122 	/* ddr3_tip_write_cs_result(dev_num, WL_PHY_REG(0); */
1123 	/* restore saved values */
1124 	for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
1125 		VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
1126 		/* restore Read Data Sample Delay */
1127 		CHECK_STATUS(ddr3_tip_if_write
1128 			     (dev_num, ACCESS_TYPE_UNICAST, if_id,
1129 			      RD_DATA_SMPL_DLYS_REG,
1130 			      read_data_sample_delay_vals[if_id],
1131 			      MASK_ALL_BITS));
1132 
1133 		/* restore Read Data Ready Delay */
1134 		CHECK_STATUS(ddr3_tip_if_write
1135 			     (dev_num, ACCESS_TYPE_UNICAST, if_id,
1136 			      RD_DATA_RDY_DLYS_REG,
1137 			      read_data_ready_delay_vals[if_id],
1138 			      MASK_ALL_BITS));
1139 
1140 		/* enable multi cs */
1141 		CHECK_STATUS(ddr3_tip_if_write
1142 			     (dev_num, ACCESS_TYPE_UNICAST, if_id,
1143 			      DUAL_DUNIT_CFG_REG, cs_enable_reg_val[if_id],
1144 			      MASK_ALL_BITS));
1145 	}
1146 
1147 	if (ddr3_tip_dev_attr_get(dev_num, MV_ATTR_TIP_REV) >= MV_TIP_REV_3) {
1148 		/* Disable modt0 for CS0 training - need to adjust for multi-CS
1149 		 * in case of ddr4 set 0xf else 0
1150 		 */
1151 		if (odt_config != 0) {
1152 			CHECK_STATUS(ddr3_tip_if_write(dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1153 						       SDRAM_ODT_CTRL_HIGH_REG, 0x0, 0xf));
1154 		}
1155 		else {
1156 			CHECK_STATUS(ddr3_tip_if_write(dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1157 						       SDRAM_ODT_CTRL_HIGH_REG, 0xf, 0xf));
1158 		}
1159 
1160 	}
1161 
1162 	for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
1163 		VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
1164 		if (training_result[training_stage][if_id] == TEST_FAILED)
1165 			return MV_FAIL;
1166 	}
1167 
1168 	return MV_OK;
1169 }
1170 
1171 /*
1172  * Dynamic write leveling supplementary
1173  */
1174 int ddr3_tip_dynamic_write_leveling_supp(u32 dev_num)
1175 {
1176 	int adll_offset;
1177 	u32 if_id, bus_id, data, data_tmp;
1178 	int is_if_fail = 0;
1179 	u32 octets_per_if_num = ddr3_tip_dev_attr_get(dev_num, MV_ATTR_OCTET_PER_INTERFACE);
1180 	struct mv_ddr_topology_map *tm = mv_ddr_topology_map_get();
1181 
1182 	for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
1183 		VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
1184 		is_if_fail = 0;
1185 
1186 		for (bus_id = 0; bus_id < octets_per_if_num; bus_id++) {
1187 			VALIDATE_BUS_ACTIVE(tm->bus_act_mask, bus_id);
1188 			wr_supp_res[if_id][bus_id].is_pup_fail = 1;
1189 			CHECK_STATUS(ddr3_tip_bus_read
1190 				     (dev_num, if_id, ACCESS_TYPE_UNICAST,
1191 				      bus_id, DDR_PHY_DATA,
1192 				      CTX_PHY_REG(effective_cs),
1193 				      &data));
1194 			DEBUG_LEVELING(
1195 				DEBUG_LEVEL_TRACE,
1196 				("WL Supp: adll_offset=0 data delay = %d\n",
1197 				 data));
1198 			if (ddr3_tip_wl_supp_align_phase_shift
1199 			    (dev_num, if_id, bus_id) == MV_OK) {
1200 				DEBUG_LEVELING(
1201 					DEBUG_LEVEL_TRACE,
1202 					("WL Supp: IF %d bus_id %d adll_offset=0 Success !\n",
1203 					 if_id, bus_id));
1204 				continue;
1205 			}
1206 
1207 			/* change adll */
1208 			adll_offset = 5;
1209 			CHECK_STATUS(ddr3_tip_bus_write
1210 				     (dev_num, ACCESS_TYPE_UNICAST, if_id,
1211 				      ACCESS_TYPE_UNICAST, bus_id, DDR_PHY_DATA,
1212 				      CTX_PHY_REG(effective_cs),
1213 				      data + adll_offset));
1214 			CHECK_STATUS(ddr3_tip_bus_read
1215 				     (dev_num, if_id, ACCESS_TYPE_UNICAST,
1216 				      bus_id, DDR_PHY_DATA,
1217 				      CTX_PHY_REG(effective_cs),
1218 				      &data_tmp));
1219 			DEBUG_LEVELING(
1220 				DEBUG_LEVEL_TRACE,
1221 				("WL Supp: adll_offset= %d data delay = %d\n",
1222 				 adll_offset, data_tmp));
1223 
1224 			if (ddr3_tip_wl_supp_align_phase_shift
1225 			    (dev_num, if_id, bus_id) == MV_OK) {
1226 				DEBUG_LEVELING(
1227 					DEBUG_LEVEL_TRACE,
1228 					("WL Supp: IF %d bus_id %d adll_offset= %d Success !\n",
1229 					 if_id, bus_id, adll_offset));
1230 				continue;
1231 			}
1232 
1233 			/* change adll */
1234 			adll_offset = -5;
1235 			CHECK_STATUS(ddr3_tip_bus_write
1236 				     (dev_num, ACCESS_TYPE_UNICAST, if_id,
1237 				      ACCESS_TYPE_UNICAST, bus_id, DDR_PHY_DATA,
1238 				      CTX_PHY_REG(effective_cs),
1239 				      data + adll_offset));
1240 			CHECK_STATUS(ddr3_tip_bus_read
1241 				     (dev_num, if_id, ACCESS_TYPE_UNICAST,
1242 				      bus_id, DDR_PHY_DATA,
1243 				      CTX_PHY_REG(effective_cs),
1244 				      &data_tmp));
1245 			DEBUG_LEVELING(
1246 				DEBUG_LEVEL_TRACE,
1247 				("WL Supp: adll_offset= %d data delay = %d\n",
1248 				 adll_offset, data_tmp));
1249 			if (ddr3_tip_wl_supp_align_phase_shift
1250 			    (dev_num, if_id, bus_id) == MV_OK) {
1251 				DEBUG_LEVELING(
1252 					DEBUG_LEVEL_TRACE,
1253 					("WL Supp: IF %d bus_id %d adll_offset= %d Success !\n",
1254 					 if_id, bus_id, adll_offset));
1255 				continue;
1256 			} else {
1257 				DEBUG_LEVELING(
1258 					DEBUG_LEVEL_ERROR,
1259 					("WL Supp: IF %d bus_id %d Failed !\n",
1260 					 if_id, bus_id));
1261 				is_if_fail = 1;
1262 			}
1263 		}
1264 
1265 		if (is_if_fail == 1) {
1266 			DEBUG_LEVELING(DEBUG_LEVEL_ERROR,
1267 				       ("WL Supp: CS# %d: IF %d failed\n",
1268 					effective_cs, if_id));
1269 			training_result[training_stage][if_id] = TEST_FAILED;
1270 		} else {
1271 			training_result[training_stage][if_id] = TEST_SUCCESS;
1272 		}
1273 	}
1274 
1275 	for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
1276 		VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
1277 		if (training_result[training_stage][if_id] == TEST_FAILED)
1278 			return MV_FAIL;
1279 	}
1280 
1281 	return MV_OK;
1282 }
1283 
1284 /*
1285  * Phase Shift
1286  */
1287 static int ddr3_tip_wl_supp_align_phase_shift(u32 dev_num, u32 if_id,
1288 					      u32 bus_id)
1289 {
1290 	u32 original_phase;
1291 	u32 data, write_data;
1292 
1293 	wr_supp_res[if_id][bus_id].stage = PHASE_SHIFT;
1294 	if (ddr3_tip_xsb_compare_test
1295 	    (dev_num, if_id, bus_id, 0) == MV_OK)
1296 		return MV_OK;
1297 
1298 	/* Read current phase */
1299 	CHECK_STATUS(ddr3_tip_bus_read
1300 		     (dev_num, if_id, ACCESS_TYPE_UNICAST, bus_id,
1301 		      DDR_PHY_DATA, WL_PHY_REG(effective_cs), &data));
1302 	original_phase = (data >> 6) & 0x7;
1303 
1304 	/* Set phase (0x0[6-8]) -2 */
1305 	if (original_phase >= 1) {
1306 		if (original_phase == 1)
1307 			write_data = data & ~0x1df;
1308 		else
1309 			write_data = (data & ~0x1c0) |
1310 				     ((original_phase - 2) << 6);
1311 		ddr3_tip_bus_write(dev_num, ACCESS_TYPE_UNICAST, if_id,
1312 				   ACCESS_TYPE_UNICAST, bus_id, DDR_PHY_DATA,
1313 				   WL_PHY_REG(effective_cs), write_data);
1314 		if (ddr3_tip_xsb_compare_test
1315 		    (dev_num, if_id, bus_id, -2) == MV_OK)
1316 			return MV_OK;
1317 	}
1318 
1319 	/* Set phase (0x0[6-8]) +2 */
1320 	if (original_phase <= 5) {
1321 		write_data = (data & ~0x1c0) |
1322 			     ((original_phase + 2) << 6);
1323 		ddr3_tip_bus_write(dev_num, ACCESS_TYPE_UNICAST, if_id,
1324 				   ACCESS_TYPE_UNICAST, bus_id, DDR_PHY_DATA,
1325 				   WL_PHY_REG(effective_cs), write_data);
1326 		if (ddr3_tip_xsb_compare_test
1327 		    (dev_num, if_id, bus_id, 2) == MV_OK)
1328 			return MV_OK;
1329 	}
1330 
1331 	/* Set phase (0x0[6-8]) +4 */
1332 	if (original_phase <= 3) {
1333 		write_data = (data & ~0x1c0) |
1334 			     ((original_phase + 4) << 6);
1335 		ddr3_tip_bus_write(dev_num, ACCESS_TYPE_UNICAST, if_id,
1336 				   ACCESS_TYPE_UNICAST, bus_id, DDR_PHY_DATA,
1337 				   WL_PHY_REG(effective_cs), write_data);
1338 		if (ddr3_tip_xsb_compare_test
1339 		    (dev_num, if_id, bus_id, 4) == MV_OK)
1340 			return MV_OK;
1341 	}
1342 
1343 	/* Set phase (0x0[6-8]) +6 */
1344 	if (original_phase <= 1) {
1345 		write_data = (data & ~0x1c0) |
1346 			     ((original_phase + 6) << 6);
1347 		ddr3_tip_bus_write(dev_num, ACCESS_TYPE_UNICAST, if_id,
1348 				   ACCESS_TYPE_UNICAST, bus_id, DDR_PHY_DATA,
1349 				   WL_PHY_REG(effective_cs), write_data);
1350 		if (ddr3_tip_xsb_compare_test
1351 		    (dev_num, if_id, bus_id, 6) == MV_OK)
1352 			return MV_OK;
1353 	}
1354 
1355 	/* Write original WL result back */
1356 	ddr3_tip_bus_write(dev_num, ACCESS_TYPE_UNICAST, if_id,
1357 			   ACCESS_TYPE_UNICAST, bus_id, DDR_PHY_DATA,
1358 			   WL_PHY_REG(effective_cs), data);
1359 	wr_supp_res[if_id][bus_id].is_pup_fail = 1;
1360 
1361 	return MV_FAIL;
1362 }
1363 
1364 /*
1365  * Compare Test
1366  */
1367 static int ddr3_tip_xsb_compare_test(u32 dev_num, u32 if_id, u32 bus_id,
1368 				     u32 edge_offset)
1369 {
1370 	u32 num_of_succ_byte_compare, word_in_pattern;
1371 	u32 word_offset, i, num_of_word_mult;
1372 	u32 read_pattern[TEST_PATTERN_LENGTH * 2];
1373 	struct pattern_info *pattern_table = ddr3_tip_get_pattern_table();
1374 	u32 pattern_test_pattern_table[8];
1375 	struct mv_ddr_topology_map *tm = mv_ddr_topology_map_get();
1376 
1377 	/* 3 below for INTERFACE_BUS_MASK_16BIT */
1378 	num_of_word_mult = (tm->bus_act_mask == 3) ? 1 : 2;
1379 
1380 	for (i = 0; i < 8; i++) {
1381 		pattern_test_pattern_table[i] =
1382 			pattern_table_get_word(dev_num, PATTERN_TEST, (u8)i);
1383 	}
1384 
1385 	/* External write, read and compare */
1386 	CHECK_STATUS(ddr3_tip_load_pattern_to_mem(dev_num, PATTERN_TEST));
1387 
1388 	CHECK_STATUS(ddr3_tip_reset_fifo_ptr(dev_num));
1389 
1390 	CHECK_STATUS(ddr3_tip_ext_read
1391 		     (dev_num, if_id,
1392 		      ((pattern_table[PATTERN_TEST].start_addr << 3) +
1393 		       ((SDRAM_CS_SIZE + 1) * effective_cs)), 1, read_pattern));
1394 
1395 	DEBUG_LEVELING(
1396 		DEBUG_LEVEL_TRACE,
1397 		("XSB-compt CS#%d: IF %d bus_id %d 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x\n",
1398 		 effective_cs, if_id, bus_id,
1399 		 read_pattern[0], read_pattern[1],
1400 		 read_pattern[2], read_pattern[3],
1401 		 read_pattern[4], read_pattern[5],
1402 		 read_pattern[6], read_pattern[7]));
1403 
1404 	/* compare byte per pup */
1405 	num_of_succ_byte_compare = 0;
1406 	for (word_in_pattern = start_xsb_offset;
1407 	     word_in_pattern < (TEST_PATTERN_LENGTH * num_of_word_mult);
1408 	     word_in_pattern++) {
1409 		word_offset = word_in_pattern;
1410 		if ((word_offset > (TEST_PATTERN_LENGTH * 2 - 1)))
1411 			continue;
1412 
1413 		if ((read_pattern[word_in_pattern] & pup_mask_table[bus_id]) ==
1414 		    (pattern_test_pattern_table[word_offset] &
1415 		     pup_mask_table[bus_id]))
1416 			num_of_succ_byte_compare++;
1417 	}
1418 
1419 	if ((TEST_PATTERN_LENGTH * num_of_word_mult - start_xsb_offset) ==
1420 	    num_of_succ_byte_compare) {
1421 		wr_supp_res[if_id][bus_id].stage = edge_offset;
1422 		DEBUG_LEVELING(DEBUG_LEVEL_TRACE,
1423 			       ("supplementary: shift to %d for if %d pup %d success\n",
1424 				edge_offset, if_id, bus_id));
1425 		wr_supp_res[if_id][bus_id].is_pup_fail = 0;
1426 
1427 		return MV_OK;
1428 	} else {
1429 		DEBUG_LEVELING(
1430 			DEBUG_LEVEL_TRACE,
1431 			("XSB-compt CS#%d: IF %d bus_id %d num_of_succ_byte_compare %d - Fail!\n",
1432 			 effective_cs, if_id, bus_id, num_of_succ_byte_compare));
1433 
1434 		DEBUG_LEVELING(
1435 			DEBUG_LEVEL_TRACE,
1436 			("XSB-compt: expected 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x\n",
1437 			 pattern_test_pattern_table[0],
1438 			 pattern_test_pattern_table[1],
1439 			 pattern_test_pattern_table[2],
1440 			 pattern_test_pattern_table[3],
1441 			 pattern_test_pattern_table[4],
1442 			 pattern_test_pattern_table[5],
1443 			 pattern_test_pattern_table[6],
1444 			 pattern_test_pattern_table[7]));
1445 		DEBUG_LEVELING(
1446 			DEBUG_LEVEL_TRACE,
1447 			("XSB-compt: recieved 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x\n",
1448 			 read_pattern[0], read_pattern[1],
1449 			 read_pattern[2], read_pattern[3],
1450 			 read_pattern[4], read_pattern[5],
1451 			 read_pattern[6], read_pattern[7]));
1452 
1453 		return MV_FAIL;
1454 	}
1455 }
1456 
1457 /*
1458  * Dynamic write leveling sequence
1459  */
1460 static int ddr3_tip_dynamic_write_leveling_seq(u32 dev_num)
1461 {
1462 	u32 bus_id, dq_id;
1463 	u16 *mask_results_pup_reg_map = ddr3_tip_get_mask_results_pup_reg_map();
1464 	u16 *mask_results_dq_reg_map = ddr3_tip_get_mask_results_dq_reg();
1465 	u32 octets_per_if_num = ddr3_tip_dev_attr_get(dev_num, MV_ATTR_OCTET_PER_INTERFACE);
1466 	struct mv_ddr_topology_map *tm = mv_ddr_topology_map_get();
1467 
1468 	CHECK_STATUS(ddr3_tip_if_write
1469 		     (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1470 		      TRAINING_SW_2_REG, 0x1, 0x5));
1471 	CHECK_STATUS(ddr3_tip_if_write
1472 		     (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1473 		      TRAINING_WL_REG, 0x50, 0xff));
1474 	CHECK_STATUS(ddr3_tip_if_write
1475 		     (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1476 		      TRAINING_WL_REG, 0x5c, 0xff));
1477 	CHECK_STATUS(ddr3_tip_if_write
1478 		     (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1479 		      GENERAL_TRAINING_OPCODE_REG, 0x381b82, 0x3c3faf));
1480 	CHECK_STATUS(ddr3_tip_if_write
1481 		     (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1482 		      OPCODE_REG0_REG(1), (0x3 << 25), (0x3ffff << 9)));
1483 	CHECK_STATUS(ddr3_tip_if_write
1484 		     (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1485 		      OPCODE_REG1_REG(1), 0x80, 0xffff));
1486 	CHECK_STATUS(ddr3_tip_if_write
1487 		     (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1488 		      WL_DONE_CNTR_REF_REG, 0x14, 0xff));
1489 	CHECK_STATUS(ddr3_tip_if_write
1490 		     (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1491 		      TRAINING_WL_REG, 0xff5c, 0xffff));
1492 
1493 	/* mask PBS */
1494 	for (dq_id = 0; dq_id < MAX_DQ_NUM; dq_id++) {
1495 		CHECK_STATUS(ddr3_tip_if_write
1496 			     (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1497 			      mask_results_dq_reg_map[dq_id], 0x1 << 24,
1498 			      0x1 << 24));
1499 	}
1500 
1501 	/* Mask all results */
1502 	for (bus_id = 0; bus_id < octets_per_if_num; bus_id++) {
1503 		CHECK_STATUS(ddr3_tip_if_write
1504 			     (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1505 			      mask_results_pup_reg_map[bus_id], 0x1 << 24,
1506 			      0x1 << 24));
1507 	}
1508 
1509 	/* Unmask only wanted */
1510 	for (bus_id = 0; bus_id < octets_per_if_num; bus_id++) {
1511 		VALIDATE_BUS_ACTIVE(tm->bus_act_mask, bus_id);
1512 		CHECK_STATUS(ddr3_tip_if_write
1513 			     (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1514 			      mask_results_pup_reg_map[bus_id], 0, 0x1 << 24));
1515 	}
1516 
1517 	CHECK_STATUS(ddr3_tip_if_write
1518 		     (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1519 		      WL_DQS_PATTERN_REG, 0x1, 0x1));
1520 
1521 	return MV_OK;
1522 }
1523 
1524 /*
1525  * Dynamic read leveling sequence
1526  */
1527 static int ddr3_tip_dynamic_read_leveling_seq(u32 dev_num)
1528 {
1529 	u32 bus_id, dq_id;
1530 	u16 *mask_results_pup_reg_map = ddr3_tip_get_mask_results_pup_reg_map();
1531 	u16 *mask_results_dq_reg_map = ddr3_tip_get_mask_results_dq_reg();
1532 	u32 octets_per_if_num = ddr3_tip_dev_attr_get(dev_num, MV_ATTR_OCTET_PER_INTERFACE);
1533 	struct mv_ddr_topology_map *tm = mv_ddr_topology_map_get();
1534 
1535 	/* mask PBS */
1536 	for (dq_id = 0; dq_id < MAX_DQ_NUM; dq_id++) {
1537 		CHECK_STATUS(ddr3_tip_if_write
1538 			     (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1539 			      mask_results_dq_reg_map[dq_id], 0x1 << 24,
1540 			      0x1 << 24));
1541 	}
1542 
1543 	/* Mask all results */
1544 	for (bus_id = 0; bus_id < octets_per_if_num; bus_id++) {
1545 		CHECK_STATUS(ddr3_tip_if_write
1546 			     (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1547 			      mask_results_pup_reg_map[bus_id], 0x1 << 24,
1548 			      0x1 << 24));
1549 	}
1550 
1551 	/* Unmask only wanted */
1552 	for (bus_id = 0; bus_id < octets_per_if_num; bus_id++) {
1553 		VALIDATE_BUS_ACTIVE(tm->bus_act_mask, bus_id);
1554 		CHECK_STATUS(ddr3_tip_if_write
1555 			     (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1556 			      mask_results_pup_reg_map[bus_id], 0, 0x1 << 24));
1557 	}
1558 
1559 	return MV_OK;
1560 }
1561 
1562 /*
1563  * Dynamic read leveling sequence
1564  */
1565 static int ddr3_tip_dynamic_per_bit_read_leveling_seq(u32 dev_num)
1566 {
1567 	u32 bus_id, dq_id;
1568 	u16 *mask_results_pup_reg_map = ddr3_tip_get_mask_results_pup_reg_map();
1569 	u16 *mask_results_dq_reg_map = ddr3_tip_get_mask_results_dq_reg();
1570 	u32 octets_per_if_num = ddr3_tip_dev_attr_get(dev_num, MV_ATTR_OCTET_PER_INTERFACE);
1571 	struct mv_ddr_topology_map *tm = mv_ddr_topology_map_get();
1572 
1573 	/* mask PBS */
1574 	for (dq_id = 0; dq_id < MAX_DQ_NUM; dq_id++) {
1575 		CHECK_STATUS(ddr3_tip_if_write
1576 			     (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1577 			      mask_results_dq_reg_map[dq_id], 0x1 << 24,
1578 			      0x1 << 24));
1579 	}
1580 
1581 	/* Mask all results */
1582 	for (bus_id = 0; bus_id < octets_per_if_num; bus_id++) {
1583 		CHECK_STATUS(ddr3_tip_if_write
1584 			     (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1585 			      mask_results_pup_reg_map[bus_id], 0x1 << 24,
1586 			      0x1 << 24));
1587 	}
1588 
1589 	/* Unmask only wanted */
1590 	for (dq_id = 0; dq_id < MAX_DQ_NUM; dq_id++) {
1591 		VALIDATE_BUS_ACTIVE(tm->bus_act_mask, dq_id / 8);
1592 		CHECK_STATUS(ddr3_tip_if_write
1593 			     (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1594 			      mask_results_dq_reg_map[dq_id], 0x0 << 24,
1595 			      0x1 << 24));
1596 	}
1597 
1598 	return MV_OK;
1599 }
1600 
1601 /*
1602  * Print write leveling supplementary results
1603  */
1604 int ddr3_tip_print_wl_supp_result(u32 dev_num)
1605 {
1606 	u32 bus_id = 0, if_id = 0;
1607 	u32 octets_per_if_num = ddr3_tip_dev_attr_get(dev_num, MV_ATTR_OCTET_PER_INTERFACE);
1608 	struct mv_ddr_topology_map *tm = mv_ddr_topology_map_get();
1609 
1610 	DEBUG_LEVELING(DEBUG_LEVEL_INFO,
1611 		       ("I/F0 PUP0 Result[0 - success, 1-fail] ...\n"));
1612 
1613 	for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
1614 		VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
1615 		for (bus_id = 0; bus_id < octets_per_if_num;
1616 		     bus_id++) {
1617 			VALIDATE_BUS_ACTIVE(tm->bus_act_mask, bus_id);
1618 			DEBUG_LEVELING(DEBUG_LEVEL_INFO,
1619 				       ("%d ,", wr_supp_res[if_id]
1620 					[bus_id].is_pup_fail));
1621 		}
1622 	}
1623 	DEBUG_LEVELING(
1624 		DEBUG_LEVEL_INFO,
1625 		("I/F0 PUP0 Stage[0-phase_shift, 1-clock_shift, 2-align_shift] ...\n"));
1626 
1627 	for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
1628 		VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
1629 		for (bus_id = 0; bus_id < octets_per_if_num;
1630 		     bus_id++) {
1631 			VALIDATE_BUS_ACTIVE(tm->bus_act_mask, bus_id);
1632 			DEBUG_LEVELING(DEBUG_LEVEL_INFO,
1633 				       ("%d ,", wr_supp_res[if_id]
1634 					[bus_id].stage));
1635 		}
1636 	}
1637 
1638 	return MV_OK;
1639 }
1640 
1641 #define RD_FIFO_PTR_LOW_STAT_INDIR_ADDR		0x9a
1642 #define RD_FIFO_PTR_HIGH_STAT_INDIR_ADDR	0x9b
1643 /* position of falling dqs edge in fifo; walking 1 */
1644 #define RD_FIFO_DQS_FALL_EDGE_POS_0		0x1
1645 #define RD_FIFO_DQS_FALL_EDGE_POS_1		0x2
1646 #define RD_FIFO_DQS_FALL_EDGE_POS_2		0x4
1647 #define RD_FIFO_DQS_FALL_EDGE_POS_3		0x8
1648 #define RD_FIFO_DQS_FALL_EDGE_POS_4		0x10 /* lock */
1649 /* position of rising dqs edge in fifo; walking 0 */
1650 #define RD_FIFO_DQS_RISE_EDGE_POS_0		0x1fff
1651 #define RD_FIFO_DQS_RISE_EDGE_POS_1		0x3ffe
1652 #define RD_FIFO_DQS_RISE_EDGE_POS_2		0x3ffd
1653 #define RD_FIFO_DQS_RISE_EDGE_POS_3		0x3ffb
1654 #define RD_FIFO_DQS_RISE_EDGE_POS_4		0x3ff7 /* lock */
1655 #define TEST_ADDR		0x8
1656 #define TAPS_PER_UI		32
1657 #define UI_PER_RD_SAMPLE	4
1658 #define TAPS_PER_RD_SAMPLE	((UI_PER_RD_SAMPLE) * (TAPS_PER_UI))
1659 #define MAX_RD_SAMPLES		32
1660 #define MAX_RL_VALUE		((MAX_RD_SAMPLES) * (TAPS_PER_RD_SAMPLE))
1661 #define RD_FIFO_DLY		8
1662 #define STEP_SIZE		64
1663 #define RL_JITTER_WIDTH_LMT	20
1664 #define ADLL_TAPS_IN_CYCLE	64
1665 
1666 enum rl_dqs_burst_state {
1667 	RL_AHEAD = 0,
1668 	RL_INSIDE,
1669 	RL_BEHIND
1670 };
1671 
1672 
1673 int mv_ddr_rl_dqs_burst(u32 dev_num, u32 if_id, u32 freq)
1674 {
1675 	enum rl_dqs_burst_state rl_state[MAX_CS_NUM][MAX_BUS_NUM][MAX_INTERFACE_NUM] = { { {0} } };
1676 	enum hws_ddr_phy subphy_type = DDR_PHY_DATA;
1677 	struct mv_ddr_topology_map *tm = mv_ddr_topology_map_get();
1678 	int cl_val = tm->interface_params[0].cas_l;
1679 	int rl_adll_val, rl_phase_val, sdr_cycle_incr, rd_sample, rd_ready;
1680 	int final_rd_sample, final_rd_ready;
1681 	int i, subphy_id, step;
1682 	int pass_lock_num = 0;
1683 	int init_pass_lock_num;
1684 	int phase_delta;
1685 	int min_phase, max_phase;
1686 	unsigned int max_cs = mv_ddr_cs_num_get();
1687 	u32 rl_values[MAX_CS_NUM][MAX_BUS_NUM][MAX_INTERFACE_NUM] = { { {0} } };
1688 	u32 rl_min_values[MAX_CS_NUM][MAX_BUS_NUM][MAX_INTERFACE_NUM] = { { {0} } };
1689 	u32 rl_max_values[MAX_CS_NUM][MAX_BUS_NUM][MAX_INTERFACE_NUM] = { { {0} } };
1690 	u32 rl_val, rl_min_val[MAX_CS_NUM], rl_max_val[MAX_CS_NUM];
1691 	u32 reg_val_low, reg_val_high;
1692 	u32 reg_val, reg_mask;
1693 	uintptr_t test_addr = TEST_ADDR;
1694 
1695 
1696 	/* initialization */
1697 	if (mv_ddr_is_ecc_ena()) {
1698 		ddr3_tip_if_read(dev_num, ACCESS_TYPE_UNICAST, if_id, TRAINING_SW_2_REG,
1699 				 &reg_val, MASK_ALL_BITS);
1700 		reg_mask = (TRAINING_ECC_MUX_MASK << TRAINING_ECC_MUX_OFFS) |
1701 			   (TRAINING_SW_OVRD_MASK << TRAINING_SW_OVRD_OFFS);
1702 		reg_val &= ~reg_mask;
1703 		reg_val |= (TRAINING_ECC_MUX_DIS << TRAINING_ECC_MUX_OFFS) |
1704 			   (TRAINING_SW_OVRD_ENA << TRAINING_SW_OVRD_OFFS);
1705 		ddr3_tip_if_write(0, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE, TRAINING_SW_2_REG,
1706 				  reg_val, MASK_ALL_BITS);
1707 		ddr3_tip_if_read(dev_num, ACCESS_TYPE_UNICAST, if_id, TRAINING_REG,
1708 				 &reg_val, MASK_ALL_BITS);
1709 		reg_mask = (TRN_START_MASK << TRN_START_OFFS);
1710 		reg_val &= ~reg_mask;
1711 		reg_val |= TRN_START_ENA << TRN_START_OFFS;
1712 		ddr3_tip_if_write(0, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE, TRAINING_REG,
1713 				  reg_val, MASK_ALL_BITS);
1714 	}
1715 
1716 	for (effective_cs = 0; effective_cs < max_cs; effective_cs++)
1717 		for (subphy_id = 0; subphy_id < MAX_BUS_NUM; subphy_id++)
1718 			for (if_id = 0; if_id < MAX_INTERFACE_NUM; if_id++)
1719 				if (IS_BUS_ACTIVE(tm->bus_act_mask, subphy_id) == 0)
1720 					pass_lock_num++; /* increment on inactive subphys */
1721 
1722 	init_pass_lock_num = pass_lock_num / max_cs;
1723 	for (effective_cs = 0; effective_cs < max_cs; effective_cs++) {
1724 		for (if_id = 0; if_id < MAX_INTERFACE_NUM; if_id++) {
1725 			VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
1726 			training_result[training_stage][if_id] = TEST_SUCCESS;
1727 		}
1728 	}
1729 
1730 	/* search for dqs edges per subphy */
1731 	if_id = 0;
1732 	for (effective_cs = 0; effective_cs < max_cs; effective_cs++) {
1733 
1734 		pass_lock_num = init_pass_lock_num;
1735 		ddr3_tip_if_write(0, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE, ODPG_DATA_CTRL_REG,
1736 				  effective_cs << ODPG_DATA_CS_OFFS,
1737 				  ODPG_DATA_CS_MASK << ODPG_DATA_CS_OFFS);
1738 		rl_min_val[effective_cs] = MAX_RL_VALUE;
1739 		rl_max_val[effective_cs] = 0;
1740 		step = STEP_SIZE;
1741 		for (i = 0; i < MAX_RL_VALUE; i += step) {
1742 			rl_val = 0;
1743 			sdr_cycle_incr = i / TAPS_PER_RD_SAMPLE; /* sdr cycle increment */
1744 			rd_sample = cl_val + 2 * sdr_cycle_incr;
1745 			/* fifo out to in delay in search is constant */
1746 			rd_ready = rd_sample + RD_FIFO_DLY;
1747 
1748 			ddr3_tip_if_write(0, ACCESS_TYPE_UNICAST, 0, RD_DATA_SMPL_DLYS_REG,
1749 					  rd_sample << RD_SMPL_DLY_CS_OFFS(effective_cs),
1750 					  RD_SMPL_DLY_CS_MASK << RD_SMPL_DLY_CS_OFFS(effective_cs));
1751 			ddr3_tip_if_write(0, ACCESS_TYPE_UNICAST, 0, RD_DATA_RDY_DLYS_REG,
1752 					  rd_ready << RD_RDY_DLY_CS_OFFS(effective_cs),
1753 					  RD_RDY_DLY_CS_MASK << RD_RDY_DLY_CS_OFFS(effective_cs));
1754 
1755 			/* one sdr (single data rate) cycle incremented on every four phases of ddr clock */
1756 			sdr_cycle_incr = i % TAPS_PER_RD_SAMPLE;
1757 			rl_adll_val = sdr_cycle_incr % MAX_RD_SAMPLES;
1758 			rl_phase_val = sdr_cycle_incr / MAX_RD_SAMPLES;
1759 			rl_val = ((rl_adll_val & RL_REF_DLY_MASK) << RL_REF_DLY_OFFS) |
1760 				 ((rl_phase_val & RL_PH_SEL_MASK) << RL_PH_SEL_OFFS);
1761 
1762 			/* write to all subphys (even to not connected or locked) */
1763 			ddr3_tip_bus_write(dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE, ACCESS_TYPE_MULTICAST,
1764 					   0, DDR_PHY_DATA, RL_PHY_REG(effective_cs), rl_val);
1765 
1766 			/* reset read fifo assertion */
1767 			ddr3_tip_if_write(dev_num, ACCESS_TYPE_MULTICAST, if_id, SDRAM_CFG_REG,
1768 					  DATA_PUP_RD_RESET_ENA << DATA_PUP_RD_RESET_OFFS,
1769 					  DATA_PUP_RD_RESET_MASK << DATA_PUP_RD_RESET_OFFS);
1770 
1771 			/* reset read fifo deassertion */
1772 			ddr3_tip_if_write(dev_num, ACCESS_TYPE_MULTICAST, if_id, SDRAM_CFG_REG,
1773 					  DATA_PUP_RD_RESET_DIS << DATA_PUP_RD_RESET_OFFS,
1774 					  DATA_PUP_RD_RESET_MASK << DATA_PUP_RD_RESET_OFFS);
1775 
1776 			/* perform one read burst */
1777 			if (MV_DDR_IS_64BIT_DRAM_MODE(tm->bus_act_mask))
1778 				readq(test_addr);
1779 			else
1780 				readl(test_addr);
1781 
1782 			/* progress read ptr; decide on rl state per byte */
1783 			for (subphy_id = 0; subphy_id < MAX_BUS_NUM; subphy_id++) {
1784 				if (rl_state[effective_cs][subphy_id][if_id] == RL_BEHIND)
1785 					continue; /* skip locked subphys */
1786 				ddr3_tip_bus_read(dev_num, if_id, ACCESS_TYPE_UNICAST, subphy_id, DDR_PHY_DATA,
1787 						  RD_FIFO_PTR_LOW_STAT_INDIR_ADDR, &reg_val_low);
1788 				ddr3_tip_bus_read(dev_num, if_id, ACCESS_TYPE_UNICAST, subphy_id, DDR_PHY_DATA,
1789 						  RD_FIFO_PTR_HIGH_STAT_INDIR_ADDR, &reg_val_high);
1790 				DEBUG_LEVELING(DEBUG_LEVEL_TRACE,
1791 					       ("%s: cs %d, step %d, subphy %d, state %d, low 0x%04x, high 0x%04x; move to ",
1792 						__func__, effective_cs, i, subphy_id,
1793 						rl_state[effective_cs][subphy_id][if_id],
1794 						reg_val_low, reg_val_high));
1795 
1796 				switch (rl_state[effective_cs][subphy_id][if_id]) {
1797 				case RL_AHEAD:
1798 					/* improve search resolution getting closer to the window */
1799 					if (reg_val_low == RD_FIFO_DQS_FALL_EDGE_POS_4 &&
1800 					    reg_val_high == RD_FIFO_DQS_RISE_EDGE_POS_4) {
1801 						rl_state[effective_cs][subphy_id][if_id] = RL_INSIDE;
1802 						rl_values[effective_cs][subphy_id][if_id] = i;
1803 						rl_min_values[effective_cs][subphy_id][if_id] = i;
1804 						DEBUG_LEVELING(DEBUG_LEVEL_TRACE,
1805 							       ("new state %d\n",
1806 								rl_state[effective_cs][subphy_id][if_id]));
1807 					} else if (reg_val_low == RD_FIFO_DQS_FALL_EDGE_POS_3 &&
1808 						   reg_val_high == RD_FIFO_DQS_RISE_EDGE_POS_3) {
1809 						step = (step < 2) ? step : 2;
1810 					} else if (reg_val_low == RD_FIFO_DQS_FALL_EDGE_POS_2 &&
1811 						   reg_val_high == RD_FIFO_DQS_RISE_EDGE_POS_2) {
1812 						step = (step < 16) ? step : 16;
1813 					} else if (reg_val_low == RD_FIFO_DQS_FALL_EDGE_POS_1 &&
1814 						   reg_val_high == RD_FIFO_DQS_RISE_EDGE_POS_1) {
1815 						step = (step < 32) ? step : 32;
1816 					} else if (reg_val_low == RD_FIFO_DQS_FALL_EDGE_POS_0 &&
1817 						   reg_val_high == RD_FIFO_DQS_RISE_EDGE_POS_0) {
1818 						step = (step < 64) ? step : 64;
1819 					} else {
1820 						/* otherwise, step is unchanged */
1821 					}
1822 					break;
1823 				case RL_INSIDE:
1824 					if (reg_val_low == RD_FIFO_DQS_FALL_EDGE_POS_4 &&
1825 					    reg_val_high == RD_FIFO_DQS_RISE_EDGE_POS_4) {
1826 						rl_max_values[effective_cs][subphy_id][if_id] = i;
1827 						if ((rl_max_values[effective_cs][subphy_id][if_id] -
1828 						     rl_min_values[effective_cs][subphy_id][if_id]) >
1829 						    ADLL_TAPS_IN_CYCLE) {
1830 							rl_state[effective_cs][subphy_id][if_id] = RL_BEHIND;
1831 							rl_values[effective_cs][subphy_id][if_id] =
1832 								(i + rl_values[effective_cs][subphy_id][if_id]) / 2;
1833 							pass_lock_num++;
1834 							DEBUG_LEVELING(DEBUG_LEVEL_TRACE,
1835 								       ("new lock %d\n", pass_lock_num));
1836 							if (rl_min_val[effective_cs] >
1837 							    rl_values[effective_cs][subphy_id][if_id])
1838 								rl_min_val[effective_cs] =
1839 									rl_values[effective_cs][subphy_id][if_id];
1840 							if (rl_max_val[effective_cs] <
1841 							    rl_values[effective_cs][subphy_id][if_id])
1842 								rl_max_val[effective_cs] =
1843 									rl_values[effective_cs][subphy_id][if_id];
1844 							step = 2;
1845 						}
1846 					}
1847 					if (reg_val_low != RD_FIFO_DQS_FALL_EDGE_POS_4 ||
1848 					    reg_val_high != RD_FIFO_DQS_RISE_EDGE_POS_4) {
1849 						if ((i - rl_values[effective_cs][subphy_id][if_id]) <
1850 						    RL_JITTER_WIDTH_LMT) {
1851 							/* inside the jitter; not valid segment */
1852 							rl_state[effective_cs][subphy_id][if_id] = RL_AHEAD;
1853 							DEBUG_LEVELING(DEBUG_LEVEL_TRACE,
1854 								       ("new state %d; jitter on mask\n",
1855 									rl_state[effective_cs][subphy_id][if_id]));
1856 						} else { /* finished valid segment */
1857 							rl_state[effective_cs][subphy_id][if_id] = RL_BEHIND;
1858 							rl_values[effective_cs][subphy_id][if_id] =
1859 								(i + rl_values[effective_cs][subphy_id][if_id]) / 2;
1860 							DEBUG_LEVELING(DEBUG_LEVEL_TRACE,
1861 								       ("new state %d, solution %d\n",
1862 									rl_state[effective_cs][subphy_id][if_id],
1863 									rl_values[effective_cs][subphy_id][if_id]));
1864 							pass_lock_num++;
1865 							DEBUG_LEVELING(DEBUG_LEVEL_TRACE,
1866 								       ("new lock %d\n", pass_lock_num));
1867 							if (rl_min_val[effective_cs] >
1868 							    rl_values[effective_cs][subphy_id][if_id])
1869 								rl_min_val[effective_cs] =
1870 									rl_values[effective_cs][subphy_id][if_id];
1871 							if (rl_max_val[effective_cs] <
1872 							    rl_values[effective_cs][subphy_id][if_id])
1873 								rl_max_val[effective_cs] =
1874 									rl_values[effective_cs][subphy_id][if_id];
1875 							step = 2;
1876 						}
1877 					}
1878 					break;
1879 				case RL_BEHIND: /* do nothing */
1880 					break;
1881 				}
1882 				DEBUG_LEVELING(DEBUG_LEVEL_TRACE, ("\n"));
1883 			}
1884 			DEBUG_LEVELING(DEBUG_LEVEL_TRACE, ("pass_lock_num %d\n", pass_lock_num));
1885 			/* exit condition */
1886 			if (pass_lock_num == MAX_BUS_NUM)
1887 				break;
1888 		} /* for-loop on i */
1889 
1890 		if (pass_lock_num != MAX_BUS_NUM) {
1891 			DEBUG_LEVELING(DEBUG_LEVEL_ERROR,
1892 				       ("%s: cs %d, pass_lock_num %d, max_bus_num %d, init_pass_lock_num %d\n",
1893 				       __func__, effective_cs, pass_lock_num, MAX_BUS_NUM, init_pass_lock_num));
1894 			for (subphy_id = 0; subphy_id < MAX_BUS_NUM; subphy_id++) {
1895 				VALIDATE_BUS_ACTIVE(tm->bus_act_mask, subphy_id);
1896 				DEBUG_LEVELING(DEBUG_LEVEL_ERROR,
1897 					       ("%s: subphy %d %s\n",
1898 						__func__, subphy_id,
1899 						(rl_state[effective_cs][subphy_id][if_id] == RL_BEHIND) ?
1900 						"locked" : "not locked"));
1901 			}
1902 		}
1903 	} /* for-loop on effective_cs */
1904 
1905 	/* post-processing read leveling results */
1906 	if_id = 0;
1907 	for (effective_cs = 0; effective_cs < max_cs; effective_cs++) {
1908 		phase_delta = 0;
1909 		i = rl_min_val[effective_cs];
1910 		sdr_cycle_incr = i / TAPS_PER_RD_SAMPLE; /* sdr cycle increment */
1911 		rd_sample = cl_val + 2 * sdr_cycle_incr;
1912 		rd_ready = rd_sample + RD_FIFO_DLY;
1913 		min_phase = (rl_min_val[effective_cs] - (sdr_cycle_incr * TAPS_PER_RD_SAMPLE)) % MAX_RD_SAMPLES;
1914 		max_phase = (rl_max_val[effective_cs] - (sdr_cycle_incr * TAPS_PER_RD_SAMPLE)) % MAX_RD_SAMPLES;
1915 		final_rd_sample = rd_sample;
1916 		final_rd_ready = rd_ready;
1917 
1918 		ddr3_tip_if_write(0, ACCESS_TYPE_UNICAST, 0, RD_DATA_SMPL_DLYS_REG,
1919 				  rd_sample << RD_SMPL_DLY_CS_OFFS(effective_cs),
1920 				  RD_SMPL_DLY_CS_MASK << RD_SMPL_DLY_CS_OFFS(effective_cs));
1921 		ddr3_tip_if_write(0, ACCESS_TYPE_UNICAST, 0, RD_DATA_RDY_DLYS_REG,
1922 				  rd_ready << RD_RDY_DLY_CS_OFFS(effective_cs),
1923 				  RD_RDY_DLY_CS_MASK << RD_RDY_DLY_CS_OFFS(effective_cs));
1924 		DEBUG_LEVELING(DEBUG_LEVEL_INFO,
1925 			       ("%s: cs %d, min phase %d, max phase %d, read sample %d\n",
1926 				__func__, effective_cs, min_phase, max_phase, rd_sample));
1927 
1928 		for (subphy_id = 0; subphy_id < MAX_BUS_NUM; subphy_id++) {
1929 			VALIDATE_BUS_ACTIVE(tm->bus_act_mask, subphy_id);
1930 			/* reduce sdr cycle per cs; extract rl adll and phase values */
1931 			i = rl_values[effective_cs][subphy_id][if_id] - (sdr_cycle_incr * TAPS_PER_RD_SAMPLE);
1932 			rl_adll_val = i % MAX_RD_SAMPLES;
1933 			rl_phase_val = i / MAX_RD_SAMPLES;
1934 			rl_phase_val -= phase_delta;
1935 			DEBUG_LEVELING(DEBUG_LEVEL_INFO,
1936 				       ("%s: final results: cs %d, subphy %d, read sample %d read ready %d, rl_phase_val %d, rl_adll_val %d\n",
1937 					__func__, effective_cs, subphy_id, final_rd_sample,
1938 					final_rd_ready, rl_phase_val, rl_adll_val));
1939 
1940 			rl_val = ((rl_adll_val & RL_REF_DLY_MASK) << RL_REF_DLY_OFFS) |
1941 				 ((rl_phase_val & RL_PH_SEL_MASK) << RL_PH_SEL_OFFS);
1942 			ddr3_tip_bus_write(dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE, ACCESS_TYPE_UNICAST,
1943 					   subphy_id, subphy_type, RL_PHY_REG(effective_cs), rl_val);
1944 		}
1945 	} /* for-loop on effective cs */
1946 
1947 	for (if_id = 0; if_id < MAX_INTERFACE_NUM; if_id++) {
1948 		VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
1949 		if (odt_config != 0)
1950 			CHECK_STATUS(ddr3_tip_write_additional_odt_setting(dev_num, if_id));
1951 	}
1952 
1953 
1954 	/* reset read fifo assertion */
1955 	ddr3_tip_if_write(dev_num, ACCESS_TYPE_MULTICAST, if_id, SDRAM_CFG_REG,
1956 			  DATA_PUP_RD_RESET_ENA << DATA_PUP_RD_RESET_OFFS,
1957 			  DATA_PUP_RD_RESET_MASK << DATA_PUP_RD_RESET_OFFS);
1958 
1959 	/* reset read fifo deassertion */
1960 	ddr3_tip_if_write(dev_num, ACCESS_TYPE_MULTICAST, if_id, SDRAM_CFG_REG,
1961 			  DATA_PUP_RD_RESET_DIS << DATA_PUP_RD_RESET_OFFS,
1962 			  DATA_PUP_RD_RESET_MASK << DATA_PUP_RD_RESET_OFFS);
1963 
1964 	return MV_OK;
1965 }
1966