1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) Marvell International Ltd. and its affiliates
4 */
5
6 #include "ddr3_init.h"
7 #include "mv_ddr_training_db.h"
8 #include "ddr_training_ip_db.h"
9 #include "mv_ddr_regs.h"
10
11 #define WL_ITERATION_NUM 10
12
13 static u32 pup_mask_table[] = {
14 0x000000ff,
15 0x0000ff00,
16 0x00ff0000,
17 0xff000000
18 };
19
20 static struct write_supp_result wr_supp_res[MAX_INTERFACE_NUM][MAX_BUS_NUM];
21
22 static int ddr3_tip_dynamic_write_leveling_seq(u32 dev_num);
23 static int ddr3_tip_dynamic_read_leveling_seq(u32 dev_num);
24 static int ddr3_tip_dynamic_per_bit_read_leveling_seq(u32 dev_num);
25 static int ddr3_tip_wl_supp_align_phase_shift(u32 dev_num, u32 if_id,
26 u32 bus_id);
27 static int ddr3_tip_xsb_compare_test(u32 dev_num, u32 if_id, u32 bus_id,
28 u32 edge_offset);
29
30 enum {
31 PASS,
32 FAIL
33 };
34 /*****************************************************************************
35 Dynamic read leveling
36 ******************************************************************************/
ddr3_tip_dynamic_read_leveling(u32 dev_num,u32 freq)37 int ddr3_tip_dynamic_read_leveling(u32 dev_num, u32 freq)
38 {
39 u32 data, mask;
40 unsigned int max_cs = mv_ddr_cs_num_get();
41 u32 bus_num, if_id, cl_val;
42 enum mv_ddr_speed_bin speed_bin_index;
43 /* save current CS value */
44 u32 cs_enable_reg_val[MAX_INTERFACE_NUM] = { 0 };
45 int is_any_pup_fail = 0;
46 u32 data_read[MAX_INTERFACE_NUM + 1] = { 0 };
47 u8 rl_values[MAX_CS_NUM][MAX_BUS_NUM][MAX_INTERFACE_NUM];
48 struct pattern_info *pattern_table = ddr3_tip_get_pattern_table();
49 u16 *mask_results_pup_reg_map = ddr3_tip_get_mask_results_pup_reg_map();
50 u32 octets_per_if_num = ddr3_tip_dev_attr_get(dev_num, MV_ATTR_OCTET_PER_INTERFACE);
51 struct mv_ddr_topology_map *tm = mv_ddr_topology_map_get();
52
53 for (effective_cs = 0; effective_cs < MAX_CS_NUM; effective_cs++)
54 for (bus_num = 0; bus_num < MAX_BUS_NUM; bus_num++)
55 for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++)
56 rl_values[effective_cs][bus_num][if_id] = 0;
57
58 for (effective_cs = 0; effective_cs < max_cs; effective_cs++) {
59 for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
60 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
61 training_result[training_stage][if_id] = TEST_SUCCESS;
62
63 /* save current cs enable reg val */
64 CHECK_STATUS(ddr3_tip_if_read
65 (dev_num, ACCESS_TYPE_UNICAST, if_id,
66 DUAL_DUNIT_CFG_REG, cs_enable_reg_val,
67 MASK_ALL_BITS));
68 /* enable single cs */
69 CHECK_STATUS(ddr3_tip_if_write
70 (dev_num, ACCESS_TYPE_UNICAST, if_id,
71 DUAL_DUNIT_CFG_REG, (1 << 3), (1 << 3)));
72 }
73
74 ddr3_tip_reset_fifo_ptr(dev_num);
75
76 /*
77 * Phase 1: Load pattern (using ODPG)
78 *
79 * enter Read Leveling mode
80 * only 27 bits are masked
81 * assuming non multi-CS configuration
82 * write to CS = 0 for the non multi CS configuration, note
83 * that the results shall be read back to the required CS !!!
84 */
85
86 /* BUS count is 0 shifted 26 */
87 CHECK_STATUS(ddr3_tip_if_write
88 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
89 ODPG_DATA_CTRL_REG, 0x3, 0x3));
90 CHECK_STATUS(ddr3_tip_configure_odpg
91 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE, 0,
92 pattern_table[PATTERN_RL].num_of_phases_tx, 0,
93 pattern_table[PATTERN_RL].num_of_phases_rx, 0, 0,
94 effective_cs, STRESS_NONE, DURATION_SINGLE));
95
96 /* load pattern to ODPG */
97 ddr3_tip_load_pattern_to_odpg(dev_num, ACCESS_TYPE_MULTICAST,
98 PARAM_NOT_CARE, PATTERN_RL,
99 pattern_table[PATTERN_RL].
100 start_addr);
101
102 /*
103 * Phase 2: ODPG to Read Leveling mode
104 */
105
106 /* General Training Opcode register */
107 CHECK_STATUS(ddr3_tip_if_write
108 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
109 ODPG_WR_RD_MODE_ENA_REG, 0,
110 MASK_ALL_BITS));
111
112 CHECK_STATUS(ddr3_tip_if_write
113 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
114 GENERAL_TRAINING_OPCODE_REG,
115 (0x301b01 | effective_cs << 2), 0x3c3fef));
116
117 /* Object1 opcode register 0 & 1 */
118 for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
119 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
120 speed_bin_index =
121 tm->interface_params[if_id].speed_bin_index;
122 cl_val = mv_ddr_cl_val_get(speed_bin_index, freq);
123 data = (cl_val << 17) | (0x3 << 25);
124 mask = (0xff << 9) | (0x1f << 17) | (0x3 << 25);
125 CHECK_STATUS(ddr3_tip_if_write
126 (dev_num, ACCESS_TYPE_UNICAST, if_id,
127 OPCODE_REG0_REG(1), data, mask));
128 }
129
130 /* Set iteration count to max value */
131 CHECK_STATUS(ddr3_tip_if_write
132 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
133 OPCODE_REG1_REG(1), 0xd00, 0xd00));
134
135 /*
136 * Phase 2: Mask config
137 */
138
139 ddr3_tip_dynamic_read_leveling_seq(dev_num);
140
141 /*
142 * Phase 3: Read Leveling execution
143 */
144
145 /* temporary jira dunit=14751 */
146 CHECK_STATUS(ddr3_tip_if_write
147 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
148 TRAINING_DBG_1_REG, 0, (u32)(1 << 31)));
149 /* configure phy reset value */
150 CHECK_STATUS(ddr3_tip_if_write
151 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
152 TRAINING_DBG_3_REG, (0x7f << 24),
153 (u32)(0xff << 24)));
154 /* data pup rd reset enable */
155 CHECK_STATUS(ddr3_tip_if_write
156 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
157 SDRAM_CFG_REG, 0, (1 << 30)));
158 /* data pup rd reset disable */
159 CHECK_STATUS(ddr3_tip_if_write
160 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
161 SDRAM_CFG_REG, (1 << 30), (1 << 30)));
162 /* training SW override & training RL mode */
163 CHECK_STATUS(ddr3_tip_if_write
164 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
165 TRAINING_SW_2_REG, 0x1, 0x9));
166 /* training enable */
167 CHECK_STATUS(ddr3_tip_if_write
168 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
169 TRAINING_REG, (1 << 24) | (1 << 20),
170 (1 << 24) | (1 << 20)));
171 CHECK_STATUS(ddr3_tip_if_write
172 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
173 TRAINING_REG, (u32)(1 << 31), (u32)(1 << 31)));
174
175 /* trigger training */
176 mv_ddr_training_enable();
177
178 /* check for training done */
179 if (mv_ddr_is_training_done(MAX_POLLING_ITERATIONS, &data) != MV_OK) {
180 DEBUG_LEVELING(DEBUG_LEVEL_ERROR, ("training done failed\n"));
181 return MV_FAIL;
182 }
183 /* check for training pass */
184 if (data != PASS)
185 DEBUG_LEVELING(DEBUG_LEVEL_INFO, ("training result failed\n"));
186
187 /* disable odpg; switch back to functional mode */
188 mv_ddr_odpg_disable();
189
190 if (mv_ddr_is_odpg_done(MAX_POLLING_ITERATIONS) != MV_OK) {
191 DEBUG_LEVELING(DEBUG_LEVEL_ERROR, ("odpg disable failed\n"));
192 return MV_FAIL;
193 }
194
195 ddr3_tip_if_write(0, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
196 ODPG_DATA_CTRL_REG, 0, MASK_ALL_BITS);
197
198 /* double loop on bus, pup */
199 for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
200 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
201 /* check training done */
202 is_any_pup_fail = 0;
203 for (bus_num = 0;
204 bus_num < octets_per_if_num;
205 bus_num++) {
206 VALIDATE_BUS_ACTIVE(tm->bus_act_mask, bus_num);
207 if (ddr3_tip_if_polling
208 (dev_num, ACCESS_TYPE_UNICAST,
209 if_id, (1 << 25), (1 << 25),
210 mask_results_pup_reg_map[bus_num],
211 MAX_POLLING_ITERATIONS) != MV_OK) {
212 DEBUG_LEVELING(DEBUG_LEVEL_ERROR,
213 ("\n_r_l: DDR3 poll failed(2) for IF %d CS %d bus %d",
214 if_id, effective_cs, bus_num));
215 is_any_pup_fail = 1;
216 } else {
217 /* read result per pup */
218 CHECK_STATUS(ddr3_tip_if_read
219 (dev_num,
220 ACCESS_TYPE_UNICAST,
221 if_id,
222 mask_results_pup_reg_map
223 [bus_num], data_read,
224 0xff));
225 rl_values[effective_cs][bus_num]
226 [if_id] = (u8)data_read[if_id];
227 }
228 }
229
230 if (is_any_pup_fail == 1) {
231 training_result[training_stage][if_id] =
232 TEST_FAILED;
233 if (debug_mode == 0)
234 return MV_FAIL;
235 }
236 }
237
238 DEBUG_LEVELING(DEBUG_LEVEL_INFO, ("RL exit read leveling\n"));
239
240 /*
241 * Phase 3: Exit Read Leveling
242 */
243
244 CHECK_STATUS(ddr3_tip_if_write
245 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
246 TRAINING_SW_2_REG, (1 << 3), (1 << 3)));
247 CHECK_STATUS(ddr3_tip_if_write
248 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
249 TRAINING_SW_1_REG, (1 << 16), (1 << 16)));
250 /* set ODPG to functional */
251 CHECK_STATUS(ddr3_tip_if_write
252 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
253 ODPG_DATA_CTRL_REG, 0x0, MASK_ALL_BITS));
254
255 /*
256 * Copy the result from the effective CS search to the
257 * real Functional CS
258 */
259 /*ddr3_tip_write_cs_result(dev_num, RL_PHY_REG(0); */
260 CHECK_STATUS(ddr3_tip_if_write
261 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
262 ODPG_DATA_CTRL_REG, 0x0, MASK_ALL_BITS));
263 }
264
265 for (effective_cs = 0; effective_cs < max_cs; effective_cs++) {
266 /* double loop on bus, pup */
267 for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
268 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
269 for (bus_num = 0;
270 bus_num < octets_per_if_num;
271 bus_num++) {
272 VALIDATE_BUS_ACTIVE(tm->bus_act_mask, bus_num);
273 /* read result per pup from arry */
274 data = rl_values[effective_cs][bus_num][if_id];
275 data = (data & 0x1f) |
276 (((data & 0xe0) >> 5) << 6);
277 ddr3_tip_bus_write(dev_num,
278 ACCESS_TYPE_UNICAST,
279 if_id,
280 ACCESS_TYPE_UNICAST,
281 bus_num, DDR_PHY_DATA,
282 RL_PHY_REG(effective_cs),
283 data);
284 }
285 }
286 }
287 /* Set to 0 after each loop to avoid illegal value may be used */
288 effective_cs = 0;
289
290 for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
291 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
292 /* restore cs enable value */
293 CHECK_STATUS(ddr3_tip_if_write
294 (dev_num, ACCESS_TYPE_UNICAST, if_id,
295 DUAL_DUNIT_CFG_REG, cs_enable_reg_val[if_id],
296 MASK_ALL_BITS));
297 if (odt_config != 0) {
298 CHECK_STATUS(ddr3_tip_write_additional_odt_setting
299 (dev_num, if_id));
300 }
301 }
302
303 for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
304 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
305 if (training_result[training_stage][if_id] == TEST_FAILED)
306 return MV_FAIL;
307 }
308
309 return MV_OK;
310 }
311
312 /*
313 * Legacy Dynamic write leveling
314 */
ddr3_tip_legacy_dynamic_write_leveling(u32 dev_num)315 int ddr3_tip_legacy_dynamic_write_leveling(u32 dev_num)
316 {
317 u32 c_cs, if_id, cs_mask = 0;
318 unsigned int max_cs = mv_ddr_cs_num_get();
319 struct mv_ddr_topology_map *tm = mv_ddr_topology_map_get();
320
321 /*
322 * In TRAINIUNG reg (0x15b0) write 0x80000008 | cs_mask:
323 * Trn_start
324 * cs_mask = 0x1 <<20 Trn_CS0 - CS0 is included in the DDR3 training
325 * cs_mask = 0x1 <<21 Trn_CS1 - CS1 is included in the DDR3 training
326 * cs_mask = 0x1 <<22 Trn_CS2 - CS2 is included in the DDR3 training
327 * cs_mask = 0x1 <<23 Trn_CS3 - CS3 is included in the DDR3 training
328 * Trn_auto_seq = write leveling
329 */
330 for (c_cs = 0; c_cs < max_cs; c_cs++)
331 cs_mask = cs_mask | 1 << (20 + c_cs);
332
333 for (if_id = 0; if_id < MAX_INTERFACE_NUM; if_id++) {
334 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
335 CHECK_STATUS(ddr3_tip_if_write
336 (dev_num, ACCESS_TYPE_MULTICAST, 0,
337 TRAINING_REG, (0x80000008 | cs_mask),
338 0xffffffff));
339 mdelay(20);
340 if (ddr3_tip_if_polling
341 (dev_num, ACCESS_TYPE_UNICAST, if_id, 0,
342 (u32)0x80000000, TRAINING_REG,
343 MAX_POLLING_ITERATIONS) != MV_OK) {
344 DEBUG_LEVELING(DEBUG_LEVEL_ERROR,
345 ("polling failed for Old WL result\n"));
346 return MV_FAIL;
347 }
348 }
349
350 return MV_OK;
351 }
352
353 /*
354 * Legacy Dynamic read leveling
355 */
ddr3_tip_legacy_dynamic_read_leveling(u32 dev_num)356 int ddr3_tip_legacy_dynamic_read_leveling(u32 dev_num)
357 {
358 u32 c_cs, if_id, cs_mask = 0;
359 unsigned int max_cs = mv_ddr_cs_num_get();
360 struct mv_ddr_topology_map *tm = mv_ddr_topology_map_get();
361
362 /*
363 * In TRAINIUNG reg (0x15b0) write 0x80000040 | cs_mask:
364 * Trn_start
365 * cs_mask = 0x1 <<20 Trn_CS0 - CS0 is included in the DDR3 training
366 * cs_mask = 0x1 <<21 Trn_CS1 - CS1 is included in the DDR3 training
367 * cs_mask = 0x1 <<22 Trn_CS2 - CS2 is included in the DDR3 training
368 * cs_mask = 0x1 <<23 Trn_CS3 - CS3 is included in the DDR3 training
369 * Trn_auto_seq = Read Leveling using training pattern
370 */
371 for (c_cs = 0; c_cs < max_cs; c_cs++)
372 cs_mask = cs_mask | 1 << (20 + c_cs);
373
374 CHECK_STATUS(ddr3_tip_if_write
375 (dev_num, ACCESS_TYPE_MULTICAST, 0, TRAINING_REG,
376 (0x80000040 | cs_mask), 0xffffffff));
377 mdelay(100);
378
379 for (if_id = 0; if_id < MAX_INTERFACE_NUM; if_id++) {
380 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
381 if (ddr3_tip_if_polling
382 (dev_num, ACCESS_TYPE_UNICAST, if_id, 0,
383 (u32)0x80000000, TRAINING_REG,
384 MAX_POLLING_ITERATIONS) != MV_OK) {
385 DEBUG_LEVELING(DEBUG_LEVEL_ERROR,
386 ("polling failed for Old RL result\n"));
387 return MV_FAIL;
388 }
389 }
390
391 return MV_OK;
392 }
393
394 /*
395 * Dynamic per bit read leveling
396 */
ddr3_tip_dynamic_per_bit_read_leveling(u32 dev_num,u32 freq)397 int ddr3_tip_dynamic_per_bit_read_leveling(u32 dev_num, u32 freq)
398 {
399 u32 data, mask;
400 u32 bus_num, if_id, cl_val, bit_num;
401 u32 curr_numb, curr_min_delay;
402 int adll_array[3] = { 0, -0xa, 0x14 };
403 u32 phyreg3_arr[MAX_INTERFACE_NUM][MAX_BUS_NUM];
404 enum mv_ddr_speed_bin speed_bin_index;
405 int is_any_pup_fail = 0;
406 int break_loop = 0;
407 u32 cs_enable_reg_val[MAX_INTERFACE_NUM]; /* save current CS value */
408 u32 data_read[MAX_INTERFACE_NUM];
409 int per_bit_rl_pup_status[MAX_INTERFACE_NUM][MAX_BUS_NUM];
410 u32 data2_write[MAX_INTERFACE_NUM][MAX_BUS_NUM];
411 struct pattern_info *pattern_table = ddr3_tip_get_pattern_table();
412 u16 *mask_results_dq_reg_map = ddr3_tip_get_mask_results_dq_reg();
413 u32 octets_per_if_num = ddr3_tip_dev_attr_get(dev_num, MV_ATTR_OCTET_PER_INTERFACE);
414 struct mv_ddr_topology_map *tm = mv_ddr_topology_map_get();
415
416 for (if_id = 0; if_id < MAX_INTERFACE_NUM; if_id++) {
417 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
418 for (bus_num = 0;
419 bus_num <= octets_per_if_num; bus_num++) {
420 VALIDATE_BUS_ACTIVE(tm->bus_act_mask, bus_num);
421 per_bit_rl_pup_status[if_id][bus_num] = 0;
422 data2_write[if_id][bus_num] = 0;
423 /* read current value of phy register 0x3 */
424 CHECK_STATUS(ddr3_tip_bus_read
425 (dev_num, if_id, ACCESS_TYPE_UNICAST,
426 bus_num, DDR_PHY_DATA,
427 CRX_PHY_REG(0),
428 &phyreg3_arr[if_id][bus_num]));
429 }
430 }
431
432 /* NEW RL machine */
433 for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
434 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
435 training_result[training_stage][if_id] = TEST_SUCCESS;
436
437 /* save current cs enable reg val */
438 CHECK_STATUS(ddr3_tip_if_read
439 (dev_num, ACCESS_TYPE_UNICAST, if_id,
440 DUAL_DUNIT_CFG_REG, &cs_enable_reg_val[if_id],
441 MASK_ALL_BITS));
442 /* enable single cs */
443 CHECK_STATUS(ddr3_tip_if_write
444 (dev_num, ACCESS_TYPE_UNICAST, if_id,
445 DUAL_DUNIT_CFG_REG, (1 << 3), (1 << 3)));
446 }
447
448 ddr3_tip_reset_fifo_ptr(dev_num);
449 for (curr_numb = 0; curr_numb < 3; curr_numb++) {
450 /*
451 * Phase 1: Load pattern (using ODPG)
452 *
453 * enter Read Leveling mode
454 * only 27 bits are masked
455 * assuming non multi-CS configuration
456 * write to CS = 0 for the non multi CS configuration, note that
457 * the results shall be read back to the required CS !!!
458 */
459
460 /* BUS count is 0 shifted 26 */
461 CHECK_STATUS(ddr3_tip_if_write
462 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
463 ODPG_DATA_CTRL_REG, 0x3, 0x3));
464 CHECK_STATUS(ddr3_tip_configure_odpg
465 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE, 0,
466 pattern_table[PATTERN_TEST].num_of_phases_tx, 0,
467 pattern_table[PATTERN_TEST].num_of_phases_rx, 0,
468 0, 0, STRESS_NONE, DURATION_SINGLE));
469
470 /* load pattern to ODPG */
471 ddr3_tip_load_pattern_to_odpg(dev_num, ACCESS_TYPE_MULTICAST,
472 PARAM_NOT_CARE, PATTERN_TEST,
473 pattern_table[PATTERN_TEST].
474 start_addr);
475
476 /*
477 * Phase 2: ODPG to Read Leveling mode
478 */
479
480 /* General Training Opcode register */
481 CHECK_STATUS(ddr3_tip_if_write
482 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
483 ODPG_WR_RD_MODE_ENA_REG, 0,
484 MASK_ALL_BITS));
485 CHECK_STATUS(ddr3_tip_if_write
486 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
487 GENERAL_TRAINING_OPCODE_REG, 0x301b01, 0x3c3fef));
488
489 /* Object1 opcode register 0 & 1 */
490 for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
491 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
492 speed_bin_index =
493 tm->interface_params[if_id].speed_bin_index;
494 cl_val = mv_ddr_cl_val_get(speed_bin_index, freq);
495 data = (cl_val << 17) | (0x3 << 25);
496 mask = (0xff << 9) | (0x1f << 17) | (0x3 << 25);
497 CHECK_STATUS(ddr3_tip_if_write
498 (dev_num, ACCESS_TYPE_UNICAST, if_id,
499 OPCODE_REG0_REG(1), data, mask));
500 }
501
502 /* Set iteration count to max value */
503 CHECK_STATUS(ddr3_tip_if_write
504 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
505 OPCODE_REG1_REG(1), 0xd00, 0xd00));
506
507 /*
508 * Phase 2: Mask config
509 */
510
511 ddr3_tip_dynamic_per_bit_read_leveling_seq(dev_num);
512
513 /*
514 * Phase 3: Read Leveling execution
515 */
516
517 /* temporary jira dunit=14751 */
518 CHECK_STATUS(ddr3_tip_if_write
519 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
520 TRAINING_DBG_1_REG, 0, (u32)(1 << 31)));
521 /* configure phy reset value */
522 CHECK_STATUS(ddr3_tip_if_write
523 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
524 TRAINING_DBG_3_REG, (0x7f << 24),
525 (u32)(0xff << 24)));
526 /* data pup rd reset enable */
527 CHECK_STATUS(ddr3_tip_if_write
528 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
529 SDRAM_CFG_REG, 0, (1 << 30)));
530 /* data pup rd reset disable */
531 CHECK_STATUS(ddr3_tip_if_write
532 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
533 SDRAM_CFG_REG, (1 << 30), (1 << 30)));
534 /* training SW override & training RL mode */
535 CHECK_STATUS(ddr3_tip_if_write
536 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
537 TRAINING_SW_2_REG, 0x1, 0x9));
538 /* training enable */
539 CHECK_STATUS(ddr3_tip_if_write
540 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
541 TRAINING_REG, (1 << 24) | (1 << 20),
542 (1 << 24) | (1 << 20)));
543 CHECK_STATUS(ddr3_tip_if_write
544 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
545 TRAINING_REG, (u32)(1 << 31), (u32)(1 << 31)));
546
547 /* trigger training */
548 mv_ddr_training_enable();
549
550 /* check for training done */
551 if (mv_ddr_is_training_done(MAX_POLLING_ITERATIONS, &data) != MV_OK) {
552 DEBUG_LEVELING(DEBUG_LEVEL_ERROR, ("training done failed\n"));
553 return MV_FAIL;
554 }
555 /* check for training pass */
556 if (data != PASS)
557 DEBUG_LEVELING(DEBUG_LEVEL_INFO, ("training result failed\n"));
558
559 /* disable odpg; switch back to functional mode */
560 mv_ddr_odpg_disable();
561
562 if (mv_ddr_is_odpg_done(MAX_POLLING_ITERATIONS) != MV_OK) {
563 DEBUG_LEVELING(DEBUG_LEVEL_ERROR, ("odpg disable failed\n"));
564 return MV_FAIL;
565 }
566
567 ddr3_tip_if_write(0, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
568 ODPG_DATA_CTRL_REG, 0, MASK_ALL_BITS);
569
570 /* double loop on bus, pup */
571 for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
572 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
573 /* check training done */
574 for (bus_num = 0;
575 bus_num < octets_per_if_num;
576 bus_num++) {
577 VALIDATE_BUS_ACTIVE(tm->bus_act_mask, bus_num);
578
579 if (per_bit_rl_pup_status[if_id][bus_num]
580 == 0) {
581 curr_min_delay = 0;
582 for (bit_num = 0; bit_num < 8;
583 bit_num++) {
584 if (ddr3_tip_if_polling
585 (dev_num,
586 ACCESS_TYPE_UNICAST,
587 if_id, (1 << 25),
588 (1 << 25),
589 mask_results_dq_reg_map
590 [bus_num * 8 + bit_num],
591 MAX_POLLING_ITERATIONS) !=
592 MV_OK) {
593 DEBUG_LEVELING
594 (DEBUG_LEVEL_ERROR,
595 ("\n_r_l: DDR3 poll failed(2) for bus %d bit %d\n",
596 bus_num,
597 bit_num));
598 } else {
599 /* read result per pup */
600 CHECK_STATUS
601 (ddr3_tip_if_read
602 (dev_num,
603 ACCESS_TYPE_UNICAST,
604 if_id,
605 mask_results_dq_reg_map
606 [bus_num * 8 +
607 bit_num],
608 data_read,
609 MASK_ALL_BITS));
610 data =
611 (data_read
612 [if_id] &
613 0x1f) |
614 ((data_read
615 [if_id] &
616 0xe0) << 1);
617 if (curr_min_delay == 0)
618 curr_min_delay =
619 data;
620 else if (data <
621 curr_min_delay)
622 curr_min_delay =
623 data;
624 if (data > data2_write[if_id][bus_num])
625 data2_write
626 [if_id]
627 [bus_num] =
628 data;
629 }
630 }
631
632 if (data2_write[if_id][bus_num] <=
633 (curr_min_delay +
634 MAX_DQ_READ_LEVELING_DELAY)) {
635 per_bit_rl_pup_status[if_id]
636 [bus_num] = 1;
637 }
638 }
639 }
640 }
641
642 /* check if there is need to search new phyreg3 value */
643 if (curr_numb < 2) {
644 /* if there is DLL that is not checked yet */
645 for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1;
646 if_id++) {
647 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
648 for (bus_num = 0;
649 bus_num < octets_per_if_num;
650 bus_num++) {
651 VALIDATE_BUS_ACTIVE(tm->bus_act_mask,
652 bus_num);
653 if (per_bit_rl_pup_status[if_id]
654 [bus_num] != 1) {
655 /* go to next ADLL value */
656 CHECK_STATUS
657 (ddr3_tip_bus_write
658 (dev_num,
659 ACCESS_TYPE_UNICAST,
660 if_id,
661 ACCESS_TYPE_UNICAST,
662 bus_num, DDR_PHY_DATA,
663 CRX_PHY_REG(0),
664 (phyreg3_arr[if_id]
665 [bus_num] +
666 adll_array[curr_numb])));
667 break_loop = 1;
668 break;
669 }
670 }
671 if (break_loop)
672 break;
673 }
674 } /* if (curr_numb < 2) */
675 if (!break_loop)
676 break;
677 } /* for ( curr_numb = 0; curr_numb <3; curr_numb++) */
678
679 for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
680 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
681 for (bus_num = 0; bus_num < octets_per_if_num;
682 bus_num++) {
683 VALIDATE_BUS_ACTIVE(tm->bus_act_mask, bus_num);
684 if (per_bit_rl_pup_status[if_id][bus_num] == 1)
685 ddr3_tip_bus_write(dev_num,
686 ACCESS_TYPE_UNICAST,
687 if_id,
688 ACCESS_TYPE_UNICAST,
689 bus_num, DDR_PHY_DATA,
690 RL_PHY_REG(effective_cs),
691 data2_write[if_id]
692 [bus_num]);
693 else
694 is_any_pup_fail = 1;
695 }
696
697 /* TBD flow does not support multi CS */
698 /*
699 * cs_bitmask = tm->interface_params[if_id].
700 * as_bus_params[bus_num].cs_bitmask;
701 */
702 /* divide by 4 is used for retrieving the CS number */
703 /*
704 * TBD BC2 - what is the PHY address for other
705 * CS ddr3_tip_write_cs_result() ???
706 */
707 /*
708 * find what should be written to PHY
709 * - max delay that is less than threshold
710 */
711 if (is_any_pup_fail == 1) {
712 training_result[training_stage][if_id] = TEST_FAILED;
713 if (debug_mode == 0)
714 return MV_FAIL;
715 }
716 }
717 DEBUG_LEVELING(DEBUG_LEVEL_INFO, ("RL exit read leveling\n"));
718
719 /*
720 * Phase 3: Exit Read Leveling
721 */
722
723 CHECK_STATUS(ddr3_tip_if_write
724 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
725 TRAINING_SW_2_REG, (1 << 3), (1 << 3)));
726 CHECK_STATUS(ddr3_tip_if_write
727 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
728 TRAINING_SW_1_REG, (1 << 16), (1 << 16)));
729 /* set ODPG to functional */
730 CHECK_STATUS(ddr3_tip_if_write
731 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
732 ODPG_DATA_CTRL_REG, 0x0, MASK_ALL_BITS));
733 /*
734 * Copy the result from the effective CS search to the real
735 * Functional CS
736 */
737 ddr3_tip_write_cs_result(dev_num, RL_PHY_REG(0));
738 CHECK_STATUS(ddr3_tip_if_write
739 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
740 ODPG_DATA_CTRL_REG, 0x0, MASK_ALL_BITS));
741
742 for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
743 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
744 /* restore cs enable value */
745 CHECK_STATUS(ddr3_tip_if_write
746 (dev_num, ACCESS_TYPE_UNICAST, if_id,
747 DUAL_DUNIT_CFG_REG, cs_enable_reg_val[if_id],
748 MASK_ALL_BITS));
749 if (odt_config != 0) {
750 CHECK_STATUS(ddr3_tip_write_additional_odt_setting
751 (dev_num, if_id));
752 }
753 }
754
755 for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
756 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
757 if (training_result[training_stage][if_id] == TEST_FAILED)
758 return MV_FAIL;
759 }
760
761 return MV_OK;
762 }
763
ddr3_tip_calc_cs_mask(u32 dev_num,u32 if_id,u32 effective_cs,u32 * cs_mask)764 int ddr3_tip_calc_cs_mask(u32 dev_num, u32 if_id, u32 effective_cs,
765 u32 *cs_mask)
766 {
767 u32 all_bus_cs = 0, same_bus_cs;
768 u32 bus_cnt;
769 u32 octets_per_if_num = ddr3_tip_dev_attr_get(dev_num, MV_ATTR_OCTET_PER_INTERFACE);
770 struct mv_ddr_topology_map *tm = mv_ddr_topology_map_get();
771
772 *cs_mask = same_bus_cs = CS_BIT_MASK;
773
774 /*
775 * In some of the devices (such as BC2), the CS is per pup and there
776 * for mixed mode is valid on like other devices where CS configuration
777 * is per interface.
778 * In order to know that, we do 'Or' and 'And' operation between all
779 * CS (of the pups).
780 * If they are they are not the same then it's mixed mode so all CS
781 * should be configured (when configuring the MRS)
782 */
783 for (bus_cnt = 0; bus_cnt < octets_per_if_num; bus_cnt++) {
784 VALIDATE_BUS_ACTIVE(tm->bus_act_mask, bus_cnt);
785
786 all_bus_cs |= tm->interface_params[if_id].
787 as_bus_params[bus_cnt].cs_bitmask;
788 same_bus_cs &= tm->interface_params[if_id].
789 as_bus_params[bus_cnt].cs_bitmask;
790
791 /* cs enable is active low */
792 *cs_mask &= ~tm->interface_params[if_id].
793 as_bus_params[bus_cnt].cs_bitmask;
794 }
795
796 if (all_bus_cs == same_bus_cs)
797 *cs_mask = (*cs_mask | (~(1 << effective_cs))) & CS_BIT_MASK;
798
799 return MV_OK;
800 }
801
802 /*
803 * Dynamic write leveling
804 */
ddr3_tip_dynamic_write_leveling(u32 dev_num,int phase_remove)805 int ddr3_tip_dynamic_write_leveling(u32 dev_num, int phase_remove)
806 {
807 u32 reg_data = 0, temp = 0, iter, if_id, bus_cnt;
808 u32 cs_enable_reg_val[MAX_INTERFACE_NUM] = { 0 };
809 u32 cs_mask[MAX_INTERFACE_NUM];
810 u32 read_data_sample_delay_vals[MAX_INTERFACE_NUM] = { 0 };
811 u32 read_data_ready_delay_vals[MAX_INTERFACE_NUM] = { 0 };
812 /* 0 for failure */
813 u32 res_values[MAX_INTERFACE_NUM * MAX_BUS_NUM] = { 0 };
814 u32 test_res = 0; /* 0 - success for all pup */
815 u32 data_read[MAX_INTERFACE_NUM];
816 u8 wl_values[MAX_CS_NUM][MAX_BUS_NUM][MAX_INTERFACE_NUM];
817 u16 *mask_results_pup_reg_map = ddr3_tip_get_mask_results_pup_reg_map();
818 u32 cs_mask0[MAX_INTERFACE_NUM] = { 0 };
819 unsigned int max_cs = mv_ddr_cs_num_get();
820 u32 octets_per_if_num = ddr3_tip_dev_attr_get(dev_num, MV_ATTR_OCTET_PER_INTERFACE);
821 struct mv_ddr_topology_map *tm = mv_ddr_topology_map_get();
822
823 for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
824 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
825
826 training_result[training_stage][if_id] = TEST_SUCCESS;
827
828 /* save Read Data Sample Delay */
829 CHECK_STATUS(ddr3_tip_if_read
830 (dev_num, ACCESS_TYPE_UNICAST, if_id,
831 RD_DATA_SMPL_DLYS_REG,
832 read_data_sample_delay_vals, MASK_ALL_BITS));
833 /* save Read Data Ready Delay */
834 CHECK_STATUS(ddr3_tip_if_read
835 (dev_num, ACCESS_TYPE_UNICAST, if_id,
836 RD_DATA_RDY_DLYS_REG, read_data_ready_delay_vals,
837 MASK_ALL_BITS));
838 /* save current cs reg val */
839 CHECK_STATUS(ddr3_tip_if_read
840 (dev_num, ACCESS_TYPE_UNICAST, if_id,
841 DUAL_DUNIT_CFG_REG, cs_enable_reg_val, MASK_ALL_BITS));
842 }
843
844 if (ddr3_tip_dev_attr_get(dev_num, MV_ATTR_TIP_REV) < MV_TIP_REV_3) {
845 /* Enable multi-CS */
846 CHECK_STATUS(ddr3_tip_if_write
847 (dev_num, ACCESS_TYPE_UNICAST, if_id,
848 DUAL_DUNIT_CFG_REG, 0, (1 << 3)));
849 }
850
851 /*
852 * Phase 1: DRAM 2 Write Leveling mode
853 */
854
855 /*Assert 10 refresh commands to DRAM to all CS */
856 for (iter = 0; iter < WL_ITERATION_NUM; iter++) {
857 for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
858 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
859 CHECK_STATUS(ddr3_tip_if_write
860 (dev_num, ACCESS_TYPE_UNICAST,
861 if_id, SDRAM_OP_REG,
862 (u32)((~(0xf) << 8) | 0x2), 0xf1f));
863 }
864 }
865 /* check controller back to normal */
866 for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
867 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
868 if (ddr3_tip_if_polling
869 (dev_num, ACCESS_TYPE_UNICAST, if_id, 0, 0x1f,
870 SDRAM_OP_REG, MAX_POLLING_ITERATIONS) != MV_OK) {
871 DEBUG_LEVELING(DEBUG_LEVEL_ERROR,
872 ("WL: DDR3 poll failed(3)"));
873 }
874 }
875
876 for (effective_cs = 0; effective_cs < max_cs; effective_cs++) {
877 /*enable write leveling to all cs - Q off , WL n */
878 /* calculate interface cs mask */
879 CHECK_STATUS(ddr3_tip_write_mrs_cmd(dev_num, cs_mask0, MR_CMD1,
880 0x1000, 0x1080));
881
882 for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
883 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
884 /* cs enable is active low */
885 ddr3_tip_calc_cs_mask(dev_num, if_id, effective_cs,
886 &cs_mask[if_id]);
887 }
888
889 if (ddr3_tip_dev_attr_get(dev_num, MV_ATTR_TIP_REV) >= MV_TIP_REV_3) {
890 /* Enable Output buffer to relevant CS - Q on , WL on */
891 CHECK_STATUS(ddr3_tip_write_mrs_cmd
892 (dev_num, cs_mask, MR_CMD1, 0x80, 0x1080));
893
894 /*enable odt for relevant CS */
895 CHECK_STATUS(ddr3_tip_if_write
896 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
897 0x1498, (0x3 << (effective_cs * 2)), 0xf));
898 } else {
899 /* FIXME: should be the same as _CPU case */
900 CHECK_STATUS(ddr3_tip_write_mrs_cmd
901 (dev_num, cs_mask, MR_CMD1, 0xc0, 0x12c4));
902 }
903
904 /*
905 * Phase 2: Set training IP to write leveling mode
906 */
907
908 CHECK_STATUS(ddr3_tip_dynamic_write_leveling_seq(dev_num));
909
910 /* phase 3: trigger training */
911 mv_ddr_training_enable();
912
913 /* check for training done */
914 if (mv_ddr_is_training_done(MAX_POLLING_ITERATIONS, data_read) != MV_OK) {
915 DEBUG_LEVELING(DEBUG_LEVEL_ERROR, ("training done failed\n"));
916 } else { /* check for training pass */
917 reg_data = data_read[0];
918 #if defined(CONFIG_ARMADA_38X) /* JIRA #1498 for 16 bit with ECC */
919 if (tm->bus_act_mask == 0xb) /* set to data to 0 to skip the check */
920 reg_data = 0;
921 #endif
922 if (reg_data != PASS)
923 DEBUG_LEVELING(DEBUG_LEVEL_INFO, ("training result failed\n"));
924
925 /* check for training completion per bus */
926 for (bus_cnt = 0; bus_cnt < octets_per_if_num; bus_cnt++) {
927 VALIDATE_BUS_ACTIVE(tm->bus_act_mask, bus_cnt);
928 /* training status */
929 ddr3_tip_if_read(0, ACCESS_TYPE_UNICAST, 0,
930 mask_results_pup_reg_map[bus_cnt],
931 data_read, MASK_ALL_BITS);
932 reg_data = data_read[0];
933 DEBUG_LEVELING(DEBUG_LEVEL_TRACE, ("WL: IF %d BUS %d reg 0x%x\n",
934 0, bus_cnt, reg_data));
935 if ((reg_data & (1 << 25)) == 0)
936 res_values[bus_cnt] = 1;
937 ddr3_tip_if_read(0, ACCESS_TYPE_UNICAST, 0,
938 mask_results_pup_reg_map[bus_cnt],
939 data_read, 0xff);
940 /*
941 * Save the read value that should be
942 * write to PHY register
943 */
944 wl_values[effective_cs][bus_cnt][0] = (u8)data_read[0];
945 }
946 }
947
948 /*
949 * Phase 3.5: Validate result
950 */
951 for (if_id = 0; if_id < MAX_INTERFACE_NUM; if_id++) {
952 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
953 for (bus_cnt = 0; bus_cnt < octets_per_if_num; bus_cnt++) {
954 VALIDATE_BUS_ACTIVE(tm->bus_act_mask, bus_cnt);
955 /*
956 * Read result control register according to subphy
957 * "16" below is for a half-phase
958 */
959 reg_data = wl_values[effective_cs][bus_cnt][if_id] + 16;
960 /*
961 * Write to WL register: ADLL [4:0], Phase [8:6],
962 * Centralization ADLL [15:10] + 0x10
963 */
964 reg_data = (reg_data & 0x1f) |
965 (((reg_data & 0xe0) >> 5) << 6) |
966 (((reg_data & 0x1f) + phy_reg1_val) << 10);
967 /* Search with WL CS0 subphy reg */
968 ddr3_tip_bus_write(dev_num, ACCESS_TYPE_UNICAST, if_id,
969 ACCESS_TYPE_UNICAST, bus_cnt,
970 DDR_PHY_DATA, WL_PHY_REG(0), reg_data);
971 /*
972 * Check for change in data read from DRAM.
973 * If changed, fix the result
974 */
975 CHECK_STATUS(ddr3_tip_if_read
976 (dev_num,
977 ACCESS_TYPE_UNICAST,
978 if_id,
979 TRAINING_WL_REG,
980 data_read, MASK_ALL_BITS));
981 if (((data_read[if_id] & (1 << (bus_cnt + 20))) >>
982 (bus_cnt + 20)) == 0) {
983 DEBUG_LEVELING(
984 DEBUG_LEVEL_ERROR,
985 ("WLValues was changed from 0x%X",
986 wl_values[effective_cs]
987 [bus_cnt][if_id]));
988 wl_values[effective_cs]
989 [bus_cnt][if_id] += 32;
990 DEBUG_LEVELING(
991 DEBUG_LEVEL_ERROR,
992 ("to 0x%X",
993 wl_values[effective_cs]
994 [bus_cnt][if_id]));
995 }
996 }
997 }
998
999 /*
1000 * Phase 4: Exit write leveling mode
1001 */
1002
1003 /* disable DQs toggling */
1004 CHECK_STATUS(ddr3_tip_if_write
1005 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1006 WL_DQS_PATTERN_REG, 0x0, 0x1));
1007
1008 /* Update MRS 1 (WL off) */
1009 if (ddr3_tip_dev_attr_get(dev_num, MV_ATTR_TIP_REV) >= MV_TIP_REV_3) {
1010 CHECK_STATUS(ddr3_tip_write_mrs_cmd(dev_num, cs_mask0, MR_CMD1,
1011 0x1000, 0x1080));
1012 } else {
1013 /* FIXME: should be same as _CPU case */
1014 CHECK_STATUS(ddr3_tip_write_mrs_cmd(dev_num, cs_mask0, MR_CMD1,
1015 0x1000, 0x12c4));
1016 }
1017
1018 /* Update MRS 1 (return to functional mode - Q on , WL off) */
1019 CHECK_STATUS(ddr3_tip_write_mrs_cmd
1020 (dev_num, cs_mask0, MR_CMD1, 0x0, 0x1080));
1021
1022 /* set phy to normal mode */
1023 CHECK_STATUS(ddr3_tip_if_write
1024 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1025 TRAINING_SW_2_REG, 0x5, 0x7));
1026
1027 /* exit sw override mode */
1028 CHECK_STATUS(ddr3_tip_if_write
1029 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1030 TRAINING_SW_2_REG, 0x4, 0x7));
1031 }
1032
1033 /*
1034 * Phase 5: Load WL values to each PHY
1035 */
1036
1037 for (effective_cs = 0; effective_cs < max_cs; effective_cs++) {
1038 for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
1039 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
1040 test_res = 0;
1041 for (bus_cnt = 0;
1042 bus_cnt < octets_per_if_num;
1043 bus_cnt++) {
1044 VALIDATE_BUS_ACTIVE(tm->bus_act_mask, bus_cnt);
1045 /* check if result == pass */
1046 if (res_values
1047 [(if_id *
1048 octets_per_if_num) +
1049 bus_cnt] == 0) {
1050 /*
1051 * read result control register
1052 * according to pup
1053 */
1054 reg_data =
1055 wl_values[effective_cs][bus_cnt]
1056 [if_id];
1057 /*
1058 * Write into write leveling register
1059 * ([4:0] ADLL, [8:6] Phase, [15:10]
1060 * (centralization) ADLL + 0x10)
1061 */
1062 reg_data =
1063 (reg_data & 0x1f) |
1064 (((reg_data & 0xe0) >> 5) << 6) |
1065 (((reg_data & 0x1f) +
1066 phy_reg1_val) << 10);
1067 /*
1068 * in case phase remove should be executed
1069 * need to remove more than one phase.
1070 * this will take place only in low frequency,
1071 * where there could be more than one phase between sub-phys
1072 */
1073 if (phase_remove == 1) {
1074 temp = (reg_data >> WR_LVL_PH_SEL_OFFS) & WR_LVL_PH_SEL_PHASE1;
1075 reg_data &= ~(WR_LVL_PH_SEL_MASK << WR_LVL_PH_SEL_OFFS);
1076 reg_data |= (temp << WR_LVL_PH_SEL_OFFS);
1077 }
1078
1079 ddr3_tip_bus_write(
1080 dev_num,
1081 ACCESS_TYPE_UNICAST,
1082 if_id,
1083 ACCESS_TYPE_UNICAST,
1084 bus_cnt,
1085 DDR_PHY_DATA,
1086 WL_PHY_REG(effective_cs),
1087 reg_data);
1088 } else {
1089 test_res = 1;
1090 /*
1091 * read result control register
1092 * according to pup
1093 */
1094 CHECK_STATUS(ddr3_tip_if_read
1095 (dev_num,
1096 ACCESS_TYPE_UNICAST,
1097 if_id,
1098 mask_results_pup_reg_map
1099 [bus_cnt], data_read,
1100 0xff));
1101 reg_data = data_read[if_id];
1102 DEBUG_LEVELING(
1103 DEBUG_LEVEL_ERROR,
1104 ("WL: IF %d BUS %d failed, reg 0x%x\n",
1105 if_id, bus_cnt, reg_data));
1106 }
1107 }
1108
1109 if (test_res != 0) {
1110 training_result[training_stage][if_id] =
1111 TEST_FAILED;
1112 }
1113 }
1114 }
1115 /* Set to 0 after each loop to avoid illegal value may be used */
1116 effective_cs = 0;
1117
1118 /*
1119 * Copy the result from the effective CS search to the real
1120 * Functional CS
1121 */
1122 /* ddr3_tip_write_cs_result(dev_num, WL_PHY_REG(0); */
1123 /* restore saved values */
1124 for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
1125 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
1126 /* restore Read Data Sample Delay */
1127 CHECK_STATUS(ddr3_tip_if_write
1128 (dev_num, ACCESS_TYPE_UNICAST, if_id,
1129 RD_DATA_SMPL_DLYS_REG,
1130 read_data_sample_delay_vals[if_id],
1131 MASK_ALL_BITS));
1132
1133 /* restore Read Data Ready Delay */
1134 CHECK_STATUS(ddr3_tip_if_write
1135 (dev_num, ACCESS_TYPE_UNICAST, if_id,
1136 RD_DATA_RDY_DLYS_REG,
1137 read_data_ready_delay_vals[if_id],
1138 MASK_ALL_BITS));
1139
1140 /* enable multi cs */
1141 CHECK_STATUS(ddr3_tip_if_write
1142 (dev_num, ACCESS_TYPE_UNICAST, if_id,
1143 DUAL_DUNIT_CFG_REG, cs_enable_reg_val[if_id],
1144 MASK_ALL_BITS));
1145 }
1146
1147 if (ddr3_tip_dev_attr_get(dev_num, MV_ATTR_TIP_REV) >= MV_TIP_REV_3) {
1148 /* Disable modt0 for CS0 training - need to adjust for multi-CS
1149 * in case of ddr4 set 0xf else 0
1150 */
1151 if (odt_config != 0) {
1152 CHECK_STATUS(ddr3_tip_if_write(dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1153 SDRAM_ODT_CTRL_HIGH_REG, 0x0, 0xf));
1154 }
1155 else {
1156 CHECK_STATUS(ddr3_tip_if_write(dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1157 SDRAM_ODT_CTRL_HIGH_REG, 0xf, 0xf));
1158 }
1159
1160 }
1161
1162 for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
1163 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
1164 if (training_result[training_stage][if_id] == TEST_FAILED)
1165 return MV_FAIL;
1166 }
1167
1168 return MV_OK;
1169 }
1170
1171 /*
1172 * Dynamic write leveling supplementary
1173 */
ddr3_tip_dynamic_write_leveling_supp(u32 dev_num)1174 int ddr3_tip_dynamic_write_leveling_supp(u32 dev_num)
1175 {
1176 int adll_offset;
1177 u32 if_id, bus_id, data, data_tmp;
1178 int is_if_fail = 0;
1179 u32 octets_per_if_num = ddr3_tip_dev_attr_get(dev_num, MV_ATTR_OCTET_PER_INTERFACE);
1180 struct mv_ddr_topology_map *tm = mv_ddr_topology_map_get();
1181
1182 for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
1183 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
1184 is_if_fail = 0;
1185
1186 for (bus_id = 0; bus_id < octets_per_if_num; bus_id++) {
1187 VALIDATE_BUS_ACTIVE(tm->bus_act_mask, bus_id);
1188 wr_supp_res[if_id][bus_id].is_pup_fail = 1;
1189 CHECK_STATUS(ddr3_tip_bus_read
1190 (dev_num, if_id, ACCESS_TYPE_UNICAST,
1191 bus_id, DDR_PHY_DATA,
1192 CTX_PHY_REG(effective_cs),
1193 &data));
1194 DEBUG_LEVELING(
1195 DEBUG_LEVEL_TRACE,
1196 ("WL Supp: adll_offset=0 data delay = %d\n",
1197 data));
1198 if (ddr3_tip_wl_supp_align_phase_shift
1199 (dev_num, if_id, bus_id) == MV_OK) {
1200 DEBUG_LEVELING(
1201 DEBUG_LEVEL_TRACE,
1202 ("WL Supp: IF %d bus_id %d adll_offset=0 Success !\n",
1203 if_id, bus_id));
1204 continue;
1205 }
1206
1207 /* change adll */
1208 adll_offset = 5;
1209 CHECK_STATUS(ddr3_tip_bus_write
1210 (dev_num, ACCESS_TYPE_UNICAST, if_id,
1211 ACCESS_TYPE_UNICAST, bus_id, DDR_PHY_DATA,
1212 CTX_PHY_REG(effective_cs),
1213 data + adll_offset));
1214 CHECK_STATUS(ddr3_tip_bus_read
1215 (dev_num, if_id, ACCESS_TYPE_UNICAST,
1216 bus_id, DDR_PHY_DATA,
1217 CTX_PHY_REG(effective_cs),
1218 &data_tmp));
1219 DEBUG_LEVELING(
1220 DEBUG_LEVEL_TRACE,
1221 ("WL Supp: adll_offset= %d data delay = %d\n",
1222 adll_offset, data_tmp));
1223
1224 if (ddr3_tip_wl_supp_align_phase_shift
1225 (dev_num, if_id, bus_id) == MV_OK) {
1226 DEBUG_LEVELING(
1227 DEBUG_LEVEL_TRACE,
1228 ("WL Supp: IF %d bus_id %d adll_offset= %d Success !\n",
1229 if_id, bus_id, adll_offset));
1230 continue;
1231 }
1232
1233 /* change adll */
1234 adll_offset = -5;
1235 CHECK_STATUS(ddr3_tip_bus_write
1236 (dev_num, ACCESS_TYPE_UNICAST, if_id,
1237 ACCESS_TYPE_UNICAST, bus_id, DDR_PHY_DATA,
1238 CTX_PHY_REG(effective_cs),
1239 data + adll_offset));
1240 CHECK_STATUS(ddr3_tip_bus_read
1241 (dev_num, if_id, ACCESS_TYPE_UNICAST,
1242 bus_id, DDR_PHY_DATA,
1243 CTX_PHY_REG(effective_cs),
1244 &data_tmp));
1245 DEBUG_LEVELING(
1246 DEBUG_LEVEL_TRACE,
1247 ("WL Supp: adll_offset= %d data delay = %d\n",
1248 adll_offset, data_tmp));
1249 if (ddr3_tip_wl_supp_align_phase_shift
1250 (dev_num, if_id, bus_id) == MV_OK) {
1251 DEBUG_LEVELING(
1252 DEBUG_LEVEL_TRACE,
1253 ("WL Supp: IF %d bus_id %d adll_offset= %d Success !\n",
1254 if_id, bus_id, adll_offset));
1255 continue;
1256 } else {
1257 DEBUG_LEVELING(
1258 DEBUG_LEVEL_ERROR,
1259 ("WL Supp: IF %d bus_id %d Failed !\n",
1260 if_id, bus_id));
1261 is_if_fail = 1;
1262 }
1263 }
1264
1265 if (is_if_fail == 1) {
1266 DEBUG_LEVELING(DEBUG_LEVEL_ERROR,
1267 ("WL Supp: CS# %d: IF %d failed\n",
1268 effective_cs, if_id));
1269 training_result[training_stage][if_id] = TEST_FAILED;
1270 } else {
1271 training_result[training_stage][if_id] = TEST_SUCCESS;
1272 }
1273 }
1274
1275 for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
1276 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
1277 if (training_result[training_stage][if_id] == TEST_FAILED)
1278 return MV_FAIL;
1279 }
1280
1281 return MV_OK;
1282 }
1283
1284 /*
1285 * Phase Shift
1286 */
ddr3_tip_wl_supp_align_phase_shift(u32 dev_num,u32 if_id,u32 bus_id)1287 static int ddr3_tip_wl_supp_align_phase_shift(u32 dev_num, u32 if_id,
1288 u32 bus_id)
1289 {
1290 u32 original_phase;
1291 u32 data, write_data;
1292
1293 wr_supp_res[if_id][bus_id].stage = PHASE_SHIFT;
1294 if (ddr3_tip_xsb_compare_test
1295 (dev_num, if_id, bus_id, 0) == MV_OK)
1296 return MV_OK;
1297
1298 /* Read current phase */
1299 CHECK_STATUS(ddr3_tip_bus_read
1300 (dev_num, if_id, ACCESS_TYPE_UNICAST, bus_id,
1301 DDR_PHY_DATA, WL_PHY_REG(effective_cs), &data));
1302 original_phase = (data >> 6) & 0x7;
1303
1304 /* Set phase (0x0[6-8]) -2 */
1305 if (original_phase >= 1) {
1306 if (original_phase == 1)
1307 write_data = data & ~0x1df;
1308 else
1309 write_data = (data & ~0x1c0) |
1310 ((original_phase - 2) << 6);
1311 ddr3_tip_bus_write(dev_num, ACCESS_TYPE_UNICAST, if_id,
1312 ACCESS_TYPE_UNICAST, bus_id, DDR_PHY_DATA,
1313 WL_PHY_REG(effective_cs), write_data);
1314 if (ddr3_tip_xsb_compare_test
1315 (dev_num, if_id, bus_id, -2) == MV_OK)
1316 return MV_OK;
1317 }
1318
1319 /* Set phase (0x0[6-8]) +2 */
1320 if (original_phase <= 5) {
1321 write_data = (data & ~0x1c0) |
1322 ((original_phase + 2) << 6);
1323 ddr3_tip_bus_write(dev_num, ACCESS_TYPE_UNICAST, if_id,
1324 ACCESS_TYPE_UNICAST, bus_id, DDR_PHY_DATA,
1325 WL_PHY_REG(effective_cs), write_data);
1326 if (ddr3_tip_xsb_compare_test
1327 (dev_num, if_id, bus_id, 2) == MV_OK)
1328 return MV_OK;
1329 }
1330
1331 /* Set phase (0x0[6-8]) +4 */
1332 if (original_phase <= 3) {
1333 write_data = (data & ~0x1c0) |
1334 ((original_phase + 4) << 6);
1335 ddr3_tip_bus_write(dev_num, ACCESS_TYPE_UNICAST, if_id,
1336 ACCESS_TYPE_UNICAST, bus_id, DDR_PHY_DATA,
1337 WL_PHY_REG(effective_cs), write_data);
1338 if (ddr3_tip_xsb_compare_test
1339 (dev_num, if_id, bus_id, 4) == MV_OK)
1340 return MV_OK;
1341 }
1342
1343 /* Set phase (0x0[6-8]) +6 */
1344 if (original_phase <= 1) {
1345 write_data = (data & ~0x1c0) |
1346 ((original_phase + 6) << 6);
1347 ddr3_tip_bus_write(dev_num, ACCESS_TYPE_UNICAST, if_id,
1348 ACCESS_TYPE_UNICAST, bus_id, DDR_PHY_DATA,
1349 WL_PHY_REG(effective_cs), write_data);
1350 if (ddr3_tip_xsb_compare_test
1351 (dev_num, if_id, bus_id, 6) == MV_OK)
1352 return MV_OK;
1353 }
1354
1355 /* Write original WL result back */
1356 ddr3_tip_bus_write(dev_num, ACCESS_TYPE_UNICAST, if_id,
1357 ACCESS_TYPE_UNICAST, bus_id, DDR_PHY_DATA,
1358 WL_PHY_REG(effective_cs), data);
1359 wr_supp_res[if_id][bus_id].is_pup_fail = 1;
1360
1361 return MV_FAIL;
1362 }
1363
1364 /*
1365 * Compare Test
1366 */
ddr3_tip_xsb_compare_test(u32 dev_num,u32 if_id,u32 bus_id,u32 edge_offset)1367 static int ddr3_tip_xsb_compare_test(u32 dev_num, u32 if_id, u32 bus_id,
1368 u32 edge_offset)
1369 {
1370 u32 num_of_succ_byte_compare, word_in_pattern;
1371 u32 word_offset, i, num_of_word_mult;
1372 u32 read_pattern[TEST_PATTERN_LENGTH * 2];
1373 struct pattern_info *pattern_table = ddr3_tip_get_pattern_table();
1374 u32 pattern_test_pattern_table[8];
1375 struct mv_ddr_topology_map *tm = mv_ddr_topology_map_get();
1376
1377 /* 3 below for INTERFACE_BUS_MASK_16BIT */
1378 num_of_word_mult = (tm->bus_act_mask == 3) ? 1 : 2;
1379
1380 for (i = 0; i < 8; i++) {
1381 pattern_test_pattern_table[i] =
1382 pattern_table_get_word(dev_num, PATTERN_TEST, (u8)i);
1383 }
1384
1385 /* External write, read and compare */
1386 CHECK_STATUS(ddr3_tip_load_pattern_to_mem(dev_num, PATTERN_TEST));
1387
1388 CHECK_STATUS(ddr3_tip_reset_fifo_ptr(dev_num));
1389
1390 CHECK_STATUS(ddr3_tip_ext_read
1391 (dev_num, if_id,
1392 ((pattern_table[PATTERN_TEST].start_addr << 3) +
1393 ((SDRAM_CS_SIZE + 1) * effective_cs)), 1, read_pattern));
1394
1395 DEBUG_LEVELING(
1396 DEBUG_LEVEL_TRACE,
1397 ("XSB-compt CS#%d: IF %d bus_id %d 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x\n",
1398 effective_cs, if_id, bus_id,
1399 read_pattern[0], read_pattern[1],
1400 read_pattern[2], read_pattern[3],
1401 read_pattern[4], read_pattern[5],
1402 read_pattern[6], read_pattern[7]));
1403
1404 /* compare byte per pup */
1405 num_of_succ_byte_compare = 0;
1406 for (word_in_pattern = start_xsb_offset;
1407 word_in_pattern < (TEST_PATTERN_LENGTH * num_of_word_mult);
1408 word_in_pattern++) {
1409 word_offset = word_in_pattern;
1410 if ((word_offset > (TEST_PATTERN_LENGTH * 2 - 1)))
1411 continue;
1412
1413 if ((read_pattern[word_in_pattern] & pup_mask_table[bus_id]) ==
1414 (pattern_test_pattern_table[word_offset] &
1415 pup_mask_table[bus_id]))
1416 num_of_succ_byte_compare++;
1417 }
1418
1419 if ((TEST_PATTERN_LENGTH * num_of_word_mult - start_xsb_offset) ==
1420 num_of_succ_byte_compare) {
1421 wr_supp_res[if_id][bus_id].stage = edge_offset;
1422 DEBUG_LEVELING(DEBUG_LEVEL_TRACE,
1423 ("supplementary: shift to %d for if %d pup %d success\n",
1424 edge_offset, if_id, bus_id));
1425 wr_supp_res[if_id][bus_id].is_pup_fail = 0;
1426
1427 return MV_OK;
1428 } else {
1429 DEBUG_LEVELING(
1430 DEBUG_LEVEL_TRACE,
1431 ("XSB-compt CS#%d: IF %d bus_id %d num_of_succ_byte_compare %d - Fail!\n",
1432 effective_cs, if_id, bus_id, num_of_succ_byte_compare));
1433
1434 DEBUG_LEVELING(
1435 DEBUG_LEVEL_TRACE,
1436 ("XSB-compt: expected 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x\n",
1437 pattern_test_pattern_table[0],
1438 pattern_test_pattern_table[1],
1439 pattern_test_pattern_table[2],
1440 pattern_test_pattern_table[3],
1441 pattern_test_pattern_table[4],
1442 pattern_test_pattern_table[5],
1443 pattern_test_pattern_table[6],
1444 pattern_test_pattern_table[7]));
1445 DEBUG_LEVELING(
1446 DEBUG_LEVEL_TRACE,
1447 ("XSB-compt: recieved 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x\n",
1448 read_pattern[0], read_pattern[1],
1449 read_pattern[2], read_pattern[3],
1450 read_pattern[4], read_pattern[5],
1451 read_pattern[6], read_pattern[7]));
1452
1453 return MV_FAIL;
1454 }
1455 }
1456
1457 /*
1458 * Dynamic write leveling sequence
1459 */
ddr3_tip_dynamic_write_leveling_seq(u32 dev_num)1460 static int ddr3_tip_dynamic_write_leveling_seq(u32 dev_num)
1461 {
1462 u32 bus_id, dq_id;
1463 u16 *mask_results_pup_reg_map = ddr3_tip_get_mask_results_pup_reg_map();
1464 u16 *mask_results_dq_reg_map = ddr3_tip_get_mask_results_dq_reg();
1465 u32 octets_per_if_num = ddr3_tip_dev_attr_get(dev_num, MV_ATTR_OCTET_PER_INTERFACE);
1466 struct mv_ddr_topology_map *tm = mv_ddr_topology_map_get();
1467
1468 CHECK_STATUS(ddr3_tip_if_write
1469 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1470 TRAINING_SW_2_REG, 0x1, 0x5));
1471 CHECK_STATUS(ddr3_tip_if_write
1472 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1473 TRAINING_WL_REG, 0x50, 0xff));
1474 CHECK_STATUS(ddr3_tip_if_write
1475 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1476 TRAINING_WL_REG, 0x5c, 0xff));
1477 CHECK_STATUS(ddr3_tip_if_write
1478 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1479 GENERAL_TRAINING_OPCODE_REG, 0x381b82, 0x3c3faf));
1480 CHECK_STATUS(ddr3_tip_if_write
1481 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1482 OPCODE_REG0_REG(1), (0x3 << 25), (0x3ffff << 9)));
1483 CHECK_STATUS(ddr3_tip_if_write
1484 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1485 OPCODE_REG1_REG(1), 0x80, 0xffff));
1486 CHECK_STATUS(ddr3_tip_if_write
1487 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1488 WL_DONE_CNTR_REF_REG, 0x14, 0xff));
1489 CHECK_STATUS(ddr3_tip_if_write
1490 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1491 TRAINING_WL_REG, 0xff5c, 0xffff));
1492
1493 /* mask PBS */
1494 for (dq_id = 0; dq_id < MAX_DQ_NUM; dq_id++) {
1495 CHECK_STATUS(ddr3_tip_if_write
1496 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1497 mask_results_dq_reg_map[dq_id], 0x1 << 24,
1498 0x1 << 24));
1499 }
1500
1501 /* Mask all results */
1502 for (bus_id = 0; bus_id < octets_per_if_num; bus_id++) {
1503 CHECK_STATUS(ddr3_tip_if_write
1504 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1505 mask_results_pup_reg_map[bus_id], 0x1 << 24,
1506 0x1 << 24));
1507 }
1508
1509 /* Unmask only wanted */
1510 for (bus_id = 0; bus_id < octets_per_if_num; bus_id++) {
1511 VALIDATE_BUS_ACTIVE(tm->bus_act_mask, bus_id);
1512 CHECK_STATUS(ddr3_tip_if_write
1513 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1514 mask_results_pup_reg_map[bus_id], 0, 0x1 << 24));
1515 }
1516
1517 CHECK_STATUS(ddr3_tip_if_write
1518 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1519 WL_DQS_PATTERN_REG, 0x1, 0x1));
1520
1521 return MV_OK;
1522 }
1523
1524 /*
1525 * Dynamic read leveling sequence
1526 */
ddr3_tip_dynamic_read_leveling_seq(u32 dev_num)1527 static int ddr3_tip_dynamic_read_leveling_seq(u32 dev_num)
1528 {
1529 u32 bus_id, dq_id;
1530 u16 *mask_results_pup_reg_map = ddr3_tip_get_mask_results_pup_reg_map();
1531 u16 *mask_results_dq_reg_map = ddr3_tip_get_mask_results_dq_reg();
1532 u32 octets_per_if_num = ddr3_tip_dev_attr_get(dev_num, MV_ATTR_OCTET_PER_INTERFACE);
1533 struct mv_ddr_topology_map *tm = mv_ddr_topology_map_get();
1534
1535 /* mask PBS */
1536 for (dq_id = 0; dq_id < MAX_DQ_NUM; dq_id++) {
1537 CHECK_STATUS(ddr3_tip_if_write
1538 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1539 mask_results_dq_reg_map[dq_id], 0x1 << 24,
1540 0x1 << 24));
1541 }
1542
1543 /* Mask all results */
1544 for (bus_id = 0; bus_id < octets_per_if_num; bus_id++) {
1545 CHECK_STATUS(ddr3_tip_if_write
1546 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1547 mask_results_pup_reg_map[bus_id], 0x1 << 24,
1548 0x1 << 24));
1549 }
1550
1551 /* Unmask only wanted */
1552 for (bus_id = 0; bus_id < octets_per_if_num; bus_id++) {
1553 VALIDATE_BUS_ACTIVE(tm->bus_act_mask, bus_id);
1554 CHECK_STATUS(ddr3_tip_if_write
1555 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1556 mask_results_pup_reg_map[bus_id], 0, 0x1 << 24));
1557 }
1558
1559 return MV_OK;
1560 }
1561
1562 /*
1563 * Dynamic read leveling sequence
1564 */
ddr3_tip_dynamic_per_bit_read_leveling_seq(u32 dev_num)1565 static int ddr3_tip_dynamic_per_bit_read_leveling_seq(u32 dev_num)
1566 {
1567 u32 bus_id, dq_id;
1568 u16 *mask_results_pup_reg_map = ddr3_tip_get_mask_results_pup_reg_map();
1569 u16 *mask_results_dq_reg_map = ddr3_tip_get_mask_results_dq_reg();
1570 u32 octets_per_if_num = ddr3_tip_dev_attr_get(dev_num, MV_ATTR_OCTET_PER_INTERFACE);
1571 struct mv_ddr_topology_map *tm = mv_ddr_topology_map_get();
1572
1573 /* mask PBS */
1574 for (dq_id = 0; dq_id < MAX_DQ_NUM; dq_id++) {
1575 CHECK_STATUS(ddr3_tip_if_write
1576 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1577 mask_results_dq_reg_map[dq_id], 0x1 << 24,
1578 0x1 << 24));
1579 }
1580
1581 /* Mask all results */
1582 for (bus_id = 0; bus_id < octets_per_if_num; bus_id++) {
1583 CHECK_STATUS(ddr3_tip_if_write
1584 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1585 mask_results_pup_reg_map[bus_id], 0x1 << 24,
1586 0x1 << 24));
1587 }
1588
1589 /* Unmask only wanted */
1590 for (dq_id = 0; dq_id < MAX_DQ_NUM; dq_id++) {
1591 VALIDATE_BUS_ACTIVE(tm->bus_act_mask, dq_id / 8);
1592 CHECK_STATUS(ddr3_tip_if_write
1593 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1594 mask_results_dq_reg_map[dq_id], 0x0 << 24,
1595 0x1 << 24));
1596 }
1597
1598 return MV_OK;
1599 }
1600
1601 /*
1602 * Print write leveling supplementary results
1603 */
ddr3_tip_print_wl_supp_result(u32 dev_num)1604 int ddr3_tip_print_wl_supp_result(u32 dev_num)
1605 {
1606 u32 bus_id = 0, if_id = 0;
1607 u32 octets_per_if_num = ddr3_tip_dev_attr_get(dev_num, MV_ATTR_OCTET_PER_INTERFACE);
1608 struct mv_ddr_topology_map *tm = mv_ddr_topology_map_get();
1609
1610 DEBUG_LEVELING(DEBUG_LEVEL_INFO,
1611 ("I/F0 PUP0 Result[0 - success, 1-fail] ...\n"));
1612
1613 for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
1614 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
1615 for (bus_id = 0; bus_id < octets_per_if_num;
1616 bus_id++) {
1617 VALIDATE_BUS_ACTIVE(tm->bus_act_mask, bus_id);
1618 DEBUG_LEVELING(DEBUG_LEVEL_INFO,
1619 ("%d ,", wr_supp_res[if_id]
1620 [bus_id].is_pup_fail));
1621 }
1622 }
1623 DEBUG_LEVELING(
1624 DEBUG_LEVEL_INFO,
1625 ("I/F0 PUP0 Stage[0-phase_shift, 1-clock_shift, 2-align_shift] ...\n"));
1626
1627 for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
1628 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
1629 for (bus_id = 0; bus_id < octets_per_if_num;
1630 bus_id++) {
1631 VALIDATE_BUS_ACTIVE(tm->bus_act_mask, bus_id);
1632 DEBUG_LEVELING(DEBUG_LEVEL_INFO,
1633 ("%d ,", wr_supp_res[if_id]
1634 [bus_id].stage));
1635 }
1636 }
1637
1638 return MV_OK;
1639 }
1640
1641 #define RD_FIFO_PTR_LOW_STAT_INDIR_ADDR 0x9a
1642 #define RD_FIFO_PTR_HIGH_STAT_INDIR_ADDR 0x9b
1643 /* position of falling dqs edge in fifo; walking 1 */
1644 #define RD_FIFO_DQS_FALL_EDGE_POS_0 0x1
1645 #define RD_FIFO_DQS_FALL_EDGE_POS_1 0x2
1646 #define RD_FIFO_DQS_FALL_EDGE_POS_2 0x4
1647 #define RD_FIFO_DQS_FALL_EDGE_POS_3 0x8
1648 #define RD_FIFO_DQS_FALL_EDGE_POS_4 0x10 /* lock */
1649 /* position of rising dqs edge in fifo; walking 0 */
1650 #define RD_FIFO_DQS_RISE_EDGE_POS_0 0x1fff
1651 #define RD_FIFO_DQS_RISE_EDGE_POS_1 0x3ffe
1652 #define RD_FIFO_DQS_RISE_EDGE_POS_2 0x3ffd
1653 #define RD_FIFO_DQS_RISE_EDGE_POS_3 0x3ffb
1654 #define RD_FIFO_DQS_RISE_EDGE_POS_4 0x3ff7 /* lock */
1655 #define TEST_ADDR 0x8
1656 #define TAPS_PER_UI 32
1657 #define UI_PER_RD_SAMPLE 4
1658 #define TAPS_PER_RD_SAMPLE ((UI_PER_RD_SAMPLE) * (TAPS_PER_UI))
1659 #define MAX_RD_SAMPLES 32
1660 #define MAX_RL_VALUE ((MAX_RD_SAMPLES) * (TAPS_PER_RD_SAMPLE))
1661 #define RD_FIFO_DLY 8
1662 #define STEP_SIZE 64
1663 #define RL_JITTER_WIDTH_LMT 20
1664 #define ADLL_TAPS_IN_CYCLE 64
1665
1666 enum rl_dqs_burst_state {
1667 RL_AHEAD = 0,
1668 RL_INSIDE,
1669 RL_BEHIND
1670 };
1671
1672
mv_ddr_rl_dqs_burst(u32 dev_num,u32 if_id,u32 freq)1673 int mv_ddr_rl_dqs_burst(u32 dev_num, u32 if_id, u32 freq)
1674 {
1675 enum rl_dqs_burst_state rl_state[MAX_CS_NUM][MAX_BUS_NUM][MAX_INTERFACE_NUM] = { { {0} } };
1676 enum hws_ddr_phy subphy_type = DDR_PHY_DATA;
1677 struct mv_ddr_topology_map *tm = mv_ddr_topology_map_get();
1678 int cl_val = tm->interface_params[0].cas_l;
1679 int rl_adll_val, rl_phase_val, sdr_cycle_incr, rd_sample, rd_ready;
1680 int final_rd_sample, final_rd_ready;
1681 int i, subphy_id, step;
1682 int pass_lock_num = 0;
1683 int init_pass_lock_num;
1684 int phase_delta;
1685 int min_phase, max_phase;
1686 unsigned int max_cs = mv_ddr_cs_num_get();
1687 u32 rl_values[MAX_CS_NUM][MAX_BUS_NUM][MAX_INTERFACE_NUM] = { { {0} } };
1688 u32 rl_min_values[MAX_CS_NUM][MAX_BUS_NUM][MAX_INTERFACE_NUM] = { { {0} } };
1689 u32 rl_max_values[MAX_CS_NUM][MAX_BUS_NUM][MAX_INTERFACE_NUM] = { { {0} } };
1690 u32 rl_val, rl_min_val[MAX_CS_NUM], rl_max_val[MAX_CS_NUM];
1691 u32 reg_val_low, reg_val_high;
1692 u32 reg_val, reg_mask;
1693 uintptr_t test_addr = TEST_ADDR;
1694
1695
1696 /* initialization */
1697 if (mv_ddr_is_ecc_ena()) {
1698 ddr3_tip_if_read(dev_num, ACCESS_TYPE_UNICAST, if_id, TRAINING_SW_2_REG,
1699 ®_val, MASK_ALL_BITS);
1700 reg_mask = (TRAINING_ECC_MUX_MASK << TRAINING_ECC_MUX_OFFS) |
1701 (TRAINING_SW_OVRD_MASK << TRAINING_SW_OVRD_OFFS);
1702 reg_val &= ~reg_mask;
1703 reg_val |= (TRAINING_ECC_MUX_DIS << TRAINING_ECC_MUX_OFFS) |
1704 (TRAINING_SW_OVRD_ENA << TRAINING_SW_OVRD_OFFS);
1705 ddr3_tip_if_write(0, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE, TRAINING_SW_2_REG,
1706 reg_val, MASK_ALL_BITS);
1707 ddr3_tip_if_read(dev_num, ACCESS_TYPE_UNICAST, if_id, TRAINING_REG,
1708 ®_val, MASK_ALL_BITS);
1709 reg_mask = (TRN_START_MASK << TRN_START_OFFS);
1710 reg_val &= ~reg_mask;
1711 reg_val |= TRN_START_ENA << TRN_START_OFFS;
1712 ddr3_tip_if_write(0, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE, TRAINING_REG,
1713 reg_val, MASK_ALL_BITS);
1714 }
1715
1716 for (effective_cs = 0; effective_cs < max_cs; effective_cs++)
1717 for (subphy_id = 0; subphy_id < MAX_BUS_NUM; subphy_id++)
1718 for (if_id = 0; if_id < MAX_INTERFACE_NUM; if_id++)
1719 if (IS_BUS_ACTIVE(tm->bus_act_mask, subphy_id) == 0)
1720 pass_lock_num++; /* increment on inactive subphys */
1721
1722 init_pass_lock_num = pass_lock_num / max_cs;
1723 for (effective_cs = 0; effective_cs < max_cs; effective_cs++) {
1724 for (if_id = 0; if_id < MAX_INTERFACE_NUM; if_id++) {
1725 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
1726 training_result[training_stage][if_id] = TEST_SUCCESS;
1727 }
1728 }
1729
1730 /* search for dqs edges per subphy */
1731 if_id = 0;
1732 for (effective_cs = 0; effective_cs < max_cs; effective_cs++) {
1733
1734 pass_lock_num = init_pass_lock_num;
1735 ddr3_tip_if_write(0, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE, ODPG_DATA_CTRL_REG,
1736 effective_cs << ODPG_DATA_CS_OFFS,
1737 ODPG_DATA_CS_MASK << ODPG_DATA_CS_OFFS);
1738 rl_min_val[effective_cs] = MAX_RL_VALUE;
1739 rl_max_val[effective_cs] = 0;
1740 step = STEP_SIZE;
1741 for (i = 0; i < MAX_RL_VALUE; i += step) {
1742 rl_val = 0;
1743 sdr_cycle_incr = i / TAPS_PER_RD_SAMPLE; /* sdr cycle increment */
1744 rd_sample = cl_val + 2 * sdr_cycle_incr;
1745 /* fifo out to in delay in search is constant */
1746 rd_ready = rd_sample + RD_FIFO_DLY;
1747
1748 ddr3_tip_if_write(0, ACCESS_TYPE_UNICAST, 0, RD_DATA_SMPL_DLYS_REG,
1749 rd_sample << RD_SMPL_DLY_CS_OFFS(effective_cs),
1750 RD_SMPL_DLY_CS_MASK << RD_SMPL_DLY_CS_OFFS(effective_cs));
1751 ddr3_tip_if_write(0, ACCESS_TYPE_UNICAST, 0, RD_DATA_RDY_DLYS_REG,
1752 rd_ready << RD_RDY_DLY_CS_OFFS(effective_cs),
1753 RD_RDY_DLY_CS_MASK << RD_RDY_DLY_CS_OFFS(effective_cs));
1754
1755 /* one sdr (single data rate) cycle incremented on every four phases of ddr clock */
1756 sdr_cycle_incr = i % TAPS_PER_RD_SAMPLE;
1757 rl_adll_val = sdr_cycle_incr % MAX_RD_SAMPLES;
1758 rl_phase_val = sdr_cycle_incr / MAX_RD_SAMPLES;
1759 rl_val = ((rl_adll_val & RL_REF_DLY_MASK) << RL_REF_DLY_OFFS) |
1760 ((rl_phase_val & RL_PH_SEL_MASK) << RL_PH_SEL_OFFS);
1761
1762 /* write to all subphys (even to not connected or locked) */
1763 ddr3_tip_bus_write(dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE, ACCESS_TYPE_MULTICAST,
1764 0, DDR_PHY_DATA, RL_PHY_REG(effective_cs), rl_val);
1765
1766 /* reset read fifo assertion */
1767 ddr3_tip_if_write(dev_num, ACCESS_TYPE_MULTICAST, if_id, SDRAM_CFG_REG,
1768 DATA_PUP_RD_RESET_ENA << DATA_PUP_RD_RESET_OFFS,
1769 DATA_PUP_RD_RESET_MASK << DATA_PUP_RD_RESET_OFFS);
1770
1771 /* reset read fifo deassertion */
1772 ddr3_tip_if_write(dev_num, ACCESS_TYPE_MULTICAST, if_id, SDRAM_CFG_REG,
1773 DATA_PUP_RD_RESET_DIS << DATA_PUP_RD_RESET_OFFS,
1774 DATA_PUP_RD_RESET_MASK << DATA_PUP_RD_RESET_OFFS);
1775
1776 /* perform one read burst */
1777 if (MV_DDR_IS_64BIT_DRAM_MODE(tm->bus_act_mask))
1778 readq(test_addr);
1779 else
1780 readl(test_addr);
1781
1782 /* progress read ptr; decide on rl state per byte */
1783 for (subphy_id = 0; subphy_id < MAX_BUS_NUM; subphy_id++) {
1784 if (rl_state[effective_cs][subphy_id][if_id] == RL_BEHIND)
1785 continue; /* skip locked subphys */
1786 ddr3_tip_bus_read(dev_num, if_id, ACCESS_TYPE_UNICAST, subphy_id, DDR_PHY_DATA,
1787 RD_FIFO_PTR_LOW_STAT_INDIR_ADDR, ®_val_low);
1788 ddr3_tip_bus_read(dev_num, if_id, ACCESS_TYPE_UNICAST, subphy_id, DDR_PHY_DATA,
1789 RD_FIFO_PTR_HIGH_STAT_INDIR_ADDR, ®_val_high);
1790 DEBUG_LEVELING(DEBUG_LEVEL_TRACE,
1791 ("%s: cs %d, step %d, subphy %d, state %d, low 0x%04x, high 0x%04x; move to ",
1792 __func__, effective_cs, i, subphy_id,
1793 rl_state[effective_cs][subphy_id][if_id],
1794 reg_val_low, reg_val_high));
1795
1796 switch (rl_state[effective_cs][subphy_id][if_id]) {
1797 case RL_AHEAD:
1798 /* improve search resolution getting closer to the window */
1799 if (reg_val_low == RD_FIFO_DQS_FALL_EDGE_POS_4 &&
1800 reg_val_high == RD_FIFO_DQS_RISE_EDGE_POS_4) {
1801 rl_state[effective_cs][subphy_id][if_id] = RL_INSIDE;
1802 rl_values[effective_cs][subphy_id][if_id] = i;
1803 rl_min_values[effective_cs][subphy_id][if_id] = i;
1804 DEBUG_LEVELING(DEBUG_LEVEL_TRACE,
1805 ("new state %d\n",
1806 rl_state[effective_cs][subphy_id][if_id]));
1807 } else if (reg_val_low == RD_FIFO_DQS_FALL_EDGE_POS_3 &&
1808 reg_val_high == RD_FIFO_DQS_RISE_EDGE_POS_3) {
1809 step = (step < 2) ? step : 2;
1810 } else if (reg_val_low == RD_FIFO_DQS_FALL_EDGE_POS_2 &&
1811 reg_val_high == RD_FIFO_DQS_RISE_EDGE_POS_2) {
1812 step = (step < 16) ? step : 16;
1813 } else if (reg_val_low == RD_FIFO_DQS_FALL_EDGE_POS_1 &&
1814 reg_val_high == RD_FIFO_DQS_RISE_EDGE_POS_1) {
1815 step = (step < 32) ? step : 32;
1816 } else if (reg_val_low == RD_FIFO_DQS_FALL_EDGE_POS_0 &&
1817 reg_val_high == RD_FIFO_DQS_RISE_EDGE_POS_0) {
1818 step = (step < 64) ? step : 64;
1819 } else {
1820 /* otherwise, step is unchanged */
1821 }
1822 break;
1823 case RL_INSIDE:
1824 if (reg_val_low == RD_FIFO_DQS_FALL_EDGE_POS_4 &&
1825 reg_val_high == RD_FIFO_DQS_RISE_EDGE_POS_4) {
1826 rl_max_values[effective_cs][subphy_id][if_id] = i;
1827 if ((rl_max_values[effective_cs][subphy_id][if_id] -
1828 rl_min_values[effective_cs][subphy_id][if_id]) >
1829 ADLL_TAPS_IN_CYCLE) {
1830 rl_state[effective_cs][subphy_id][if_id] = RL_BEHIND;
1831 rl_values[effective_cs][subphy_id][if_id] =
1832 (i + rl_values[effective_cs][subphy_id][if_id]) / 2;
1833 pass_lock_num++;
1834 DEBUG_LEVELING(DEBUG_LEVEL_TRACE,
1835 ("new lock %d\n", pass_lock_num));
1836 if (rl_min_val[effective_cs] >
1837 rl_values[effective_cs][subphy_id][if_id])
1838 rl_min_val[effective_cs] =
1839 rl_values[effective_cs][subphy_id][if_id];
1840 if (rl_max_val[effective_cs] <
1841 rl_values[effective_cs][subphy_id][if_id])
1842 rl_max_val[effective_cs] =
1843 rl_values[effective_cs][subphy_id][if_id];
1844 step = 2;
1845 }
1846 }
1847 if (reg_val_low != RD_FIFO_DQS_FALL_EDGE_POS_4 ||
1848 reg_val_high != RD_FIFO_DQS_RISE_EDGE_POS_4) {
1849 if ((i - rl_values[effective_cs][subphy_id][if_id]) <
1850 RL_JITTER_WIDTH_LMT) {
1851 /* inside the jitter; not valid segment */
1852 rl_state[effective_cs][subphy_id][if_id] = RL_AHEAD;
1853 DEBUG_LEVELING(DEBUG_LEVEL_TRACE,
1854 ("new state %d; jitter on mask\n",
1855 rl_state[effective_cs][subphy_id][if_id]));
1856 } else { /* finished valid segment */
1857 rl_state[effective_cs][subphy_id][if_id] = RL_BEHIND;
1858 rl_values[effective_cs][subphy_id][if_id] =
1859 (i + rl_values[effective_cs][subphy_id][if_id]) / 2;
1860 DEBUG_LEVELING(DEBUG_LEVEL_TRACE,
1861 ("new state %d, solution %d\n",
1862 rl_state[effective_cs][subphy_id][if_id],
1863 rl_values[effective_cs][subphy_id][if_id]));
1864 pass_lock_num++;
1865 DEBUG_LEVELING(DEBUG_LEVEL_TRACE,
1866 ("new lock %d\n", pass_lock_num));
1867 if (rl_min_val[effective_cs] >
1868 rl_values[effective_cs][subphy_id][if_id])
1869 rl_min_val[effective_cs] =
1870 rl_values[effective_cs][subphy_id][if_id];
1871 if (rl_max_val[effective_cs] <
1872 rl_values[effective_cs][subphy_id][if_id])
1873 rl_max_val[effective_cs] =
1874 rl_values[effective_cs][subphy_id][if_id];
1875 step = 2;
1876 }
1877 }
1878 break;
1879 case RL_BEHIND: /* do nothing */
1880 break;
1881 }
1882 DEBUG_LEVELING(DEBUG_LEVEL_TRACE, ("\n"));
1883 }
1884 DEBUG_LEVELING(DEBUG_LEVEL_TRACE, ("pass_lock_num %d\n", pass_lock_num));
1885 /* exit condition */
1886 if (pass_lock_num == MAX_BUS_NUM)
1887 break;
1888 } /* for-loop on i */
1889
1890 if (pass_lock_num != MAX_BUS_NUM) {
1891 DEBUG_LEVELING(DEBUG_LEVEL_ERROR,
1892 ("%s: cs %d, pass_lock_num %d, max_bus_num %d, init_pass_lock_num %d\n",
1893 __func__, effective_cs, pass_lock_num, MAX_BUS_NUM, init_pass_lock_num));
1894 for (subphy_id = 0; subphy_id < MAX_BUS_NUM; subphy_id++) {
1895 VALIDATE_BUS_ACTIVE(tm->bus_act_mask, subphy_id);
1896 DEBUG_LEVELING(DEBUG_LEVEL_ERROR,
1897 ("%s: subphy %d %s\n",
1898 __func__, subphy_id,
1899 (rl_state[effective_cs][subphy_id][if_id] == RL_BEHIND) ?
1900 "locked" : "not locked"));
1901 }
1902 }
1903 } /* for-loop on effective_cs */
1904
1905 /* post-processing read leveling results */
1906 if_id = 0;
1907 for (effective_cs = 0; effective_cs < max_cs; effective_cs++) {
1908 phase_delta = 0;
1909 i = rl_min_val[effective_cs];
1910 sdr_cycle_incr = i / TAPS_PER_RD_SAMPLE; /* sdr cycle increment */
1911 rd_sample = cl_val + 2 * sdr_cycle_incr;
1912 rd_ready = rd_sample + RD_FIFO_DLY;
1913 min_phase = (rl_min_val[effective_cs] - (sdr_cycle_incr * TAPS_PER_RD_SAMPLE)) % MAX_RD_SAMPLES;
1914 max_phase = (rl_max_val[effective_cs] - (sdr_cycle_incr * TAPS_PER_RD_SAMPLE)) % MAX_RD_SAMPLES;
1915 final_rd_sample = rd_sample;
1916 final_rd_ready = rd_ready;
1917
1918 ddr3_tip_if_write(0, ACCESS_TYPE_UNICAST, 0, RD_DATA_SMPL_DLYS_REG,
1919 rd_sample << RD_SMPL_DLY_CS_OFFS(effective_cs),
1920 RD_SMPL_DLY_CS_MASK << RD_SMPL_DLY_CS_OFFS(effective_cs));
1921 ddr3_tip_if_write(0, ACCESS_TYPE_UNICAST, 0, RD_DATA_RDY_DLYS_REG,
1922 rd_ready << RD_RDY_DLY_CS_OFFS(effective_cs),
1923 RD_RDY_DLY_CS_MASK << RD_RDY_DLY_CS_OFFS(effective_cs));
1924 DEBUG_LEVELING(DEBUG_LEVEL_INFO,
1925 ("%s: cs %d, min phase %d, max phase %d, read sample %d\n",
1926 __func__, effective_cs, min_phase, max_phase, rd_sample));
1927
1928 for (subphy_id = 0; subphy_id < MAX_BUS_NUM; subphy_id++) {
1929 VALIDATE_BUS_ACTIVE(tm->bus_act_mask, subphy_id);
1930 /* reduce sdr cycle per cs; extract rl adll and phase values */
1931 i = rl_values[effective_cs][subphy_id][if_id] - (sdr_cycle_incr * TAPS_PER_RD_SAMPLE);
1932 rl_adll_val = i % MAX_RD_SAMPLES;
1933 rl_phase_val = i / MAX_RD_SAMPLES;
1934 rl_phase_val -= phase_delta;
1935 DEBUG_LEVELING(DEBUG_LEVEL_INFO,
1936 ("%s: final results: cs %d, subphy %d, read sample %d read ready %d, rl_phase_val %d, rl_adll_val %d\n",
1937 __func__, effective_cs, subphy_id, final_rd_sample,
1938 final_rd_ready, rl_phase_val, rl_adll_val));
1939
1940 rl_val = ((rl_adll_val & RL_REF_DLY_MASK) << RL_REF_DLY_OFFS) |
1941 ((rl_phase_val & RL_PH_SEL_MASK) << RL_PH_SEL_OFFS);
1942 ddr3_tip_bus_write(dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE, ACCESS_TYPE_UNICAST,
1943 subphy_id, subphy_type, RL_PHY_REG(effective_cs), rl_val);
1944 }
1945 } /* for-loop on effective cs */
1946
1947 for (if_id = 0; if_id < MAX_INTERFACE_NUM; if_id++) {
1948 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
1949 if (odt_config != 0)
1950 CHECK_STATUS(ddr3_tip_write_additional_odt_setting(dev_num, if_id));
1951 }
1952
1953
1954 /* reset read fifo assertion */
1955 ddr3_tip_if_write(dev_num, ACCESS_TYPE_MULTICAST, if_id, SDRAM_CFG_REG,
1956 DATA_PUP_RD_RESET_ENA << DATA_PUP_RD_RESET_OFFS,
1957 DATA_PUP_RD_RESET_MASK << DATA_PUP_RD_RESET_OFFS);
1958
1959 /* reset read fifo deassertion */
1960 ddr3_tip_if_write(dev_num, ACCESS_TYPE_MULTICAST, if_id, SDRAM_CFG_REG,
1961 DATA_PUP_RD_RESET_DIS << DATA_PUP_RD_RESET_OFFS,
1962 DATA_PUP_RD_RESET_MASK << DATA_PUP_RD_RESET_OFFS);
1963
1964 return MV_OK;
1965 }
1966