Lines Matching +full:positive +full:- +full:phase
1 // SPDX-License-Identifier: BSD-3-Clause
3 * Copyright Altera Corporation (C) 2012-2015
44 * However, to support simulation-time selection of fast simulation mode, where
47 * check, which is based on the rtl-supplied value, or we dynamically compute
48 * the value to use based on the dynamically-chosen calibration mode
64 * non-skip and skip values
66 * The mask is set to include all bits when not-skipping, but is
70 static u16 skip_delay_mask; /* mask off bits when skipping/not-skipping */
85 if (gbl->error_stage == CAL_STAGE_NIL) { in set_failing_group_stage()
86 gbl->error_substage = substage; in set_failing_group_stage()
87 gbl->error_stage = stage; in set_failing_group_stage()
88 gbl->error_group = group; in set_failing_group_stage()
94 clrsetbits_le32(&sdr_reg_file->cur_stage, 0xffff0000, set_group << 16); in reg_file_set_group()
99 clrsetbits_le32(&sdr_reg_file->cur_stage, 0xffff, set_stage & 0xff); in reg_file_set_stage()
105 clrsetbits_le32(&sdr_reg_file->cur_stage, 0xff00, set_sub_stage << 8); in reg_file_set_sub_stage()
109 * phy_mgr_initialize() - Initialize PHY Manager
120 * In Hard PHY this is a 2-bit control: in phy_mgr_initialize()
124 writel(0x3, &phy_mgr_cfg->mux_sel); in phy_mgr_initialize()
127 writel(0, &phy_mgr_cfg->reset_mem_stbl); in phy_mgr_initialize()
130 writel(0, &phy_mgr_cfg->cal_status); in phy_mgr_initialize()
132 writel(0, &phy_mgr_cfg->cal_debug_info); in phy_mgr_initialize()
138 ratio = rwcfg->mem_dq_per_read_dqs / in phy_mgr_initialize()
139 rwcfg->mem_virtual_groups_per_read_dqs; in phy_mgr_initialize()
140 param->read_correct_mask_vg = (1 << ratio) - 1; in phy_mgr_initialize()
141 param->write_correct_mask_vg = (1 << ratio) - 1; in phy_mgr_initialize()
142 param->read_correct_mask = (1 << rwcfg->mem_dq_per_read_dqs) - 1; in phy_mgr_initialize()
143 param->write_correct_mask = (1 << rwcfg->mem_dq_per_write_dqs) - 1; in phy_mgr_initialize()
147 * set_rank_and_odt_mask() - Set Rank and ODT mask
151 * Set Rank and ODT mask (On-Die Termination).
163 switch (rwcfg->mem_number_of_ranks) { in set_rank_and_odt_mask()
170 if (rwcfg->mem_number_of_cs_per_dimm == 1) { in set_rank_and_odt_mask()
172 * - Dual-Slot , Single-Rank (1 CS per DIMM) in set_rank_and_odt_mask()
174 * - RDIMM, 4 total CS (2 CS per DIMM, 2 DIMM) in set_rank_and_odt_mask()
187 * - Single-Slot , Dual-Rank (2 CS per DIMM) in set_rank_and_odt_mask()
198 * ----------+-----------------------+ in set_rank_and_odt_mask()
200 * Read From +-----------------------+ in set_rank_and_odt_mask()
202 * ----------+-----+-----+-----+-----+ in set_rank_and_odt_mask()
207 * ----------+-----+-----+-----+-----+ in set_rank_and_odt_mask()
210 * ----------+-----------------------+ in set_rank_and_odt_mask()
212 * Write To +-----------------------+ in set_rank_and_odt_mask()
214 * ----------+-----+-----+-----+-----+ in set_rank_and_odt_mask()
219 * ----------+-----+-----+-----+-----+ in set_rank_and_odt_mask()
251 * scc_mgr_set() - Set SCC Manager register
264 * scc_mgr_initialize() - Initialize SCC Manager registers
274 * MEM_IF_READ_DQS_WIDTH - 1); in scc_mgr_initialize()
285 static void scc_mgr_set_dqdqs_output_phase(u32 write_group, u32 phase) in scc_mgr_set_dqdqs_output_phase() argument
287 scc_mgr_set(SCC_MGR_DQDQS_OUT_PHASE_OFFSET, write_group, phase); in scc_mgr_set_dqdqs_output_phase()
295 static void scc_mgr_set_dqs_en_phase(u32 read_group, u32 phase) in scc_mgr_set_dqs_en_phase() argument
297 scc_mgr_set(SCC_MGR_DQS_EN_PHASE_OFFSET, read_group, phase); in scc_mgr_set_dqs_en_phase()
312 scc_mgr_set(SCC_MGR_IO_IN_DELAY_OFFSET, rwcfg->mem_dq_per_write_dqs, in scc_mgr_set_dqs_io_in_delay()
319 rwcfg->mem_dq_per_write_dqs + 1 + dm, in scc_mgr_set_dm_in_delay()
330 scc_mgr_set(SCC_MGR_IO_OUT1_DELAY_OFFSET, rwcfg->mem_dq_per_write_dqs, in scc_mgr_set_dqs_out1_delay()
337 rwcfg->mem_dq_per_write_dqs + 1 + dm, in scc_mgr_set_dm_out1_delay()
344 writel(dqs, &sdr_scc_mgr->dqs_ena); in scc_mgr_load_dqs()
350 writel(0, &sdr_scc_mgr->dqs_io_ena); in scc_mgr_load_dqs_io()
356 writel(dq_in_group, &sdr_scc_mgr->dq_ena); in scc_mgr_load_dq()
362 writel(dm, &sdr_scc_mgr->dm_ena); in scc_mgr_load_dm()
366 * scc_mgr_set_all_ranks() - Set SCC Manager register for all ranks
370 * @update: If non-zero, trigger SCC Manager update for all ranks
380 for (r = 0; r < rwcfg->mem_number_of_ranks; in scc_mgr_set_all_ranks()
385 writel(grp, &sdr_scc_mgr->dqs_ena); in scc_mgr_set_all_ranks()
386 writel(0, &sdr_scc_mgr->update); in scc_mgr_set_all_ranks()
391 static void scc_mgr_set_dqs_en_phase_all_ranks(u32 read_group, u32 phase) in scc_mgr_set_dqs_en_phase_all_ranks() argument
396 * keeps different phase settings per shadow reg, and it's in scc_mgr_set_dqs_en_phase_all_ranks()
402 read_group, phase, 0); in scc_mgr_set_dqs_en_phase_all_ranks()
406 u32 phase) in scc_mgr_set_dqdqs_output_phase_all_ranks() argument
411 * keeps different phase settings per shadow reg, and it's in scc_mgr_set_dqdqs_output_phase_all_ranks()
417 write_group, phase, 0); in scc_mgr_set_dqdqs_output_phase_all_ranks()
436 * scc_mgr_set_oct_out1_delay() - Set OCT output delay
444 const int ratio = rwcfg->mem_if_read_dqs_width / in scc_mgr_set_oct_out1_delay()
445 rwcfg->mem_if_write_dqs_width; in scc_mgr_set_oct_out1_delay()
460 * scc_mgr_set_hhp_extras() - Set HHP extras.
468 * bits: 0:0 = 1'b1 - DQS bypass in scc_mgr_set_hhp_extras()
469 * bits: 1:1 = 1'b1 - DQ bypass in scc_mgr_set_hhp_extras()
470 * bits: 4:2 = 3'b001 - rfifo_mode in scc_mgr_set_hhp_extras()
471 * bits: 6:5 = 2'b01 - rfifo clock_select in scc_mgr_set_hhp_extras()
472 * bits: 7:7 = 1'b0 - separate gating from ungating setting in scc_mgr_set_hhp_extras()
473 * bits: 8:8 = 1'b0 - separate OE from Output delay setting in scc_mgr_set_hhp_extras()
489 * scc_mgr_zero_all() - Zero all DQS config
501 for (r = 0; r < rwcfg->mem_number_of_ranks; in scc_mgr_zero_all()
503 for (i = 0; i < rwcfg->mem_if_read_dqs_width; i++) { in scc_mgr_zero_all()
505 * The phases actually don't exist on a per-rank basis, in scc_mgr_zero_all()
509 scc_mgr_set_dqs_bus_in_delay(i, iocfg->dqs_in_reserve); in scc_mgr_zero_all()
514 for (i = 0; i < rwcfg->mem_if_write_dqs_width; i++) { in scc_mgr_zero_all()
517 scc_mgr_set_oct_out1_delay(i, iocfg->dqs_out_reserve); in scc_mgr_zero_all()
522 writel(0xff, &sdr_scc_mgr->dqs_ena); in scc_mgr_zero_all()
523 writel(0, &sdr_scc_mgr->update); in scc_mgr_zero_all()
527 * scc_set_bypass_mode() - Set bypass mode and trigger SCC update
535 writel(0xff, &sdr_scc_mgr->dq_ena); in scc_set_bypass_mode()
536 writel(0xff, &sdr_scc_mgr->dm_ena); in scc_set_bypass_mode()
539 writel(0, &sdr_scc_mgr->dqs_io_ena); in scc_set_bypass_mode()
542 writel(write_group, &sdr_scc_mgr->dqs_ena); in scc_set_bypass_mode()
545 writel(0, &sdr_scc_mgr->update); in scc_set_bypass_mode()
549 * scc_mgr_load_dqs_for_write_group() - Load DQS settings for Write Group
556 const int ratio = rwcfg->mem_if_read_dqs_width / in scc_mgr_load_dqs_for_write_group()
557 rwcfg->mem_if_write_dqs_width; in scc_mgr_load_dqs_for_write_group()
568 writel(base + i, &sdr_scc_mgr->dqs_ena); in scc_mgr_load_dqs_for_write_group()
572 * scc_mgr_zero_group() - Zero all configs for a group
580 for (r = 0; r < rwcfg->mem_number_of_ranks; in scc_mgr_zero_group()
583 for (i = 0; i < rwcfg->mem_dq_per_write_dqs; i++) { in scc_mgr_zero_group()
590 writel(0xff, &sdr_scc_mgr->dq_ena); in scc_mgr_zero_group()
600 writel(0xff, &sdr_scc_mgr->dm_ena); in scc_mgr_zero_group()
607 scc_mgr_set_dqs_out1_delay(iocfg->dqs_out_reserve); in scc_mgr_zero_group()
608 scc_mgr_set_oct_out1_delay(write_group, iocfg->dqs_out_reserve); in scc_mgr_zero_group()
612 writel(0, &sdr_scc_mgr->dqs_io_ena); in scc_mgr_zero_group()
615 writel(0, &sdr_scc_mgr->update); in scc_mgr_zero_group()
627 for (i = 0, p = group_bgn; i < rwcfg->mem_dq_per_read_dqs; i++, p++) { in scc_mgr_apply_group_dq_in_delay()
634 * scc_mgr_apply_group_dq_out1_delay() - Apply and load an output delay for the DQ pins in a group
643 for (i = 0; i < rwcfg->mem_dq_per_write_dqs; i++) { in scc_mgr_apply_group_dq_out1_delay()
673 …* scc_mgr_apply_group_all_out_delay_add() - Apply a delay to the entire output side: DQ, DM, DQS, …
685 for (i = 0; i < rwcfg->mem_dq_per_write_dqs; i++) in scc_mgr_apply_group_all_out_delay_add()
694 if (new_delay > iocfg->io_out2_delay_max) { in scc_mgr_apply_group_all_out_delay_add()
698 iocfg->io_out2_delay_max, in scc_mgr_apply_group_all_out_delay_add()
699 new_delay - iocfg->io_out2_delay_max); in scc_mgr_apply_group_all_out_delay_add()
700 new_delay -= iocfg->io_out2_delay_max; in scc_mgr_apply_group_all_out_delay_add()
708 if (new_delay > iocfg->io_out2_delay_max) { in scc_mgr_apply_group_all_out_delay_add()
712 new_delay, iocfg->io_out2_delay_max, in scc_mgr_apply_group_all_out_delay_add()
713 new_delay - iocfg->io_out2_delay_max); in scc_mgr_apply_group_all_out_delay_add()
714 new_delay -= iocfg->io_out2_delay_max; in scc_mgr_apply_group_all_out_delay_add()
722 * scc_mgr_apply_group_all_out_delay_add() - Apply a delay to the entire output side to all ranks
734 for (r = 0; r < rwcfg->mem_number_of_ranks; in scc_mgr_apply_group_all_out_delay_add_all_ranks()
737 writel(0, &sdr_scc_mgr->update); in scc_mgr_apply_group_all_out_delay_add_all_ranks()
742 * set_jump_as_return() - Return instruction optimization
754 writel(0xff, &sdr_rw_load_mgr_regs->load_cntr0); in set_jump_as_return()
755 writel(rwcfg->rreturn, &sdr_rw_load_jump_mgr_regs->load_jump_add0); in set_jump_as_return()
759 * delay_for_n_mem_clocks() - Delay for N memory clocks
774 afi_clocks = DIV_ROUND_UP(clocks, misccfg->afi_rate_ratio); in delay_for_n_mem_clocks()
776 afi_clocks--; in delay_for_n_mem_clocks()
808 &sdr_rw_load_mgr_regs->load_cntr1); in delay_for_n_mem_clocks()
810 writel(rwcfg->idle_loop1, in delay_for_n_mem_clocks()
811 &sdr_rw_load_jump_mgr_regs->load_jump_add1); in delay_for_n_mem_clocks()
813 writel(rwcfg->idle_loop1, SDR_PHYGRP_RWMGRGRP_ADDRESS | in delay_for_n_mem_clocks()
817 &sdr_rw_load_mgr_regs->load_cntr0); in delay_for_n_mem_clocks()
820 &sdr_rw_load_mgr_regs->load_cntr1); in delay_for_n_mem_clocks()
822 writel(rwcfg->idle_loop2, in delay_for_n_mem_clocks()
823 &sdr_rw_load_jump_mgr_regs->load_jump_add0); in delay_for_n_mem_clocks()
825 writel(rwcfg->idle_loop2, in delay_for_n_mem_clocks()
826 &sdr_rw_load_jump_mgr_regs->load_jump_add1); in delay_for_n_mem_clocks()
829 writel(rwcfg->idle_loop2, in delay_for_n_mem_clocks()
832 } while (c_loop-- != 0); in delay_for_n_mem_clocks()
838 * rw_mgr_mem_init_load_regs() - Load instruction registers
853 &sdr_rw_load_mgr_regs->load_cntr0); in rw_mgr_mem_init_load_regs()
855 &sdr_rw_load_mgr_regs->load_cntr1); in rw_mgr_mem_init_load_regs()
857 &sdr_rw_load_mgr_regs->load_cntr2); in rw_mgr_mem_init_load_regs()
860 writel(jump, &sdr_rw_load_jump_mgr_regs->load_jump_add0); in rw_mgr_mem_init_load_regs()
861 writel(jump, &sdr_rw_load_jump_mgr_regs->load_jump_add1); in rw_mgr_mem_init_load_regs()
862 writel(jump, &sdr_rw_load_jump_mgr_regs->load_jump_add2); in rw_mgr_mem_init_load_regs()
869 * rw_mgr_mem_load_user() - Load user calibration values
883 for (r = 0; r < rwcfg->mem_number_of_ranks; r++) { in rw_mgr_mem_load_user()
889 writel(rwcfg->precharge_all, grpaddr); in rw_mgr_mem_load_user()
892 * USER Use Mirror-ed commands for odd ranks if address in rw_mgr_mem_load_user()
895 if ((rwcfg->mem_address_mirroring >> r) & 0x1) { in rw_mgr_mem_load_user()
897 writel(rwcfg->mrs2_mirr, grpaddr); in rw_mgr_mem_load_user()
900 writel(rwcfg->mrs3_mirr, grpaddr); in rw_mgr_mem_load_user()
903 writel(rwcfg->mrs1_mirr, grpaddr); in rw_mgr_mem_load_user()
909 writel(rwcfg->mrs2, grpaddr); in rw_mgr_mem_load_user()
912 writel(rwcfg->mrs3, grpaddr); in rw_mgr_mem_load_user()
915 writel(rwcfg->mrs1, grpaddr); in rw_mgr_mem_load_user()
924 writel(rwcfg->zqcl, grpaddr); in rw_mgr_mem_load_user()
932 * rw_mgr_mem_initialize() - Initialize RW Manager
967 rw_mgr_mem_init_load_regs(misccfg->tinit_cntr0_val, in rw_mgr_mem_initialize()
968 misccfg->tinit_cntr1_val, in rw_mgr_mem_initialize()
969 misccfg->tinit_cntr2_val, in rw_mgr_mem_initialize()
970 rwcfg->init_reset_0_cke_0); in rw_mgr_mem_initialize()
973 writel(1, &phy_mgr_cfg->reset_mem_stbl); in rw_mgr_mem_initialize()
989 rw_mgr_mem_init_load_regs(misccfg->treset_cntr0_val, in rw_mgr_mem_initialize()
990 misccfg->treset_cntr1_val, in rw_mgr_mem_initialize()
991 misccfg->treset_cntr2_val, in rw_mgr_mem_initialize()
992 rwcfg->init_reset_1_cke_0); in rw_mgr_mem_initialize()
999 rw_mgr_mem_load_user(rwcfg->mrs0_dll_reset_mirr, rwcfg->mrs0_dll_reset, in rw_mgr_mem_initialize()
1004 * rw_mgr_mem_handoff() - Hand off the memory to user
1011 rw_mgr_mem_load_user(rwcfg->mrs0_user_mirr, rwcfg->mrs0_user, 1); in rw_mgr_mem_handoff()
1020 * rw_mgr_mem_calibrate_write_test_issue() - Issue write test command
1032 misccfg->enable_super_quick_calibration; in rw_mgr_mem_calibrate_write_test_issue()
1039 * The number of supported NOP cycles can range from -1 to infinity in rw_mgr_mem_calibrate_write_test_issue()
1045 * 2. For a number of NOP cycles equals to 0, the micro-instruction in rw_mgr_mem_calibrate_write_test_issue()
1047 * micro-instruction that turns on DQS (for DDRx), or outputs write in rw_mgr_mem_calibrate_write_test_issue()
1049 * the NOP micro-instruction all together in rw_mgr_mem_calibrate_write_test_issue()
1051 * 3. A number of NOP cycles equal to -1 indicates that DQS must be in rw_mgr_mem_calibrate_write_test_issue()
1052 * turned on in the same micro-instruction that issues the write in rw_mgr_mem_calibrate_write_test_issue()
1054 * to directly jump to the micro-instruction that sends out the data in rw_mgr_mem_calibrate_write_test_issue()
1056 * NOTE: Implementing this mechanism uses 2 RW Mgr jump-counters in rw_mgr_mem_calibrate_write_test_issue()
1057 * (2 and 3). One jump-counter (0) is used to perform multiple in rw_mgr_mem_calibrate_write_test_issue()
1058 * write-read operations. in rw_mgr_mem_calibrate_write_test_issue()
1059 * one counter left to issue this command in "multiple-group" mode in rw_mgr_mem_calibrate_write_test_issue()
1062 rw_wl_nop_cycles = gbl->rw_wl_nop_cycles; in rw_mgr_mem_calibrate_write_test_issue()
1064 if (rw_wl_nop_cycles == -1) { in rw_mgr_mem_calibrate_write_test_issue()
1066 * CNTR 2 - We want to execute the special write operation that in rw_mgr_mem_calibrate_write_test_issue()
1071 writel(0xFF, &sdr_rw_load_mgr_regs->load_cntr2); in rw_mgr_mem_calibrate_write_test_issue()
1073 /* CNTR 3 - Not used */ in rw_mgr_mem_calibrate_write_test_issue()
1075 mcc_instruction = rwcfg->lfsr_wr_rd_dm_bank_0_wl_1; in rw_mgr_mem_calibrate_write_test_issue()
1076 writel(rwcfg->lfsr_wr_rd_dm_bank_0_data, in rw_mgr_mem_calibrate_write_test_issue()
1077 &sdr_rw_load_jump_mgr_regs->load_jump_add2); in rw_mgr_mem_calibrate_write_test_issue()
1078 writel(rwcfg->lfsr_wr_rd_dm_bank_0_nop, in rw_mgr_mem_calibrate_write_test_issue()
1079 &sdr_rw_load_jump_mgr_regs->load_jump_add3); in rw_mgr_mem_calibrate_write_test_issue()
1081 mcc_instruction = rwcfg->lfsr_wr_rd_bank_0_wl_1; in rw_mgr_mem_calibrate_write_test_issue()
1082 writel(rwcfg->lfsr_wr_rd_bank_0_data, in rw_mgr_mem_calibrate_write_test_issue()
1083 &sdr_rw_load_jump_mgr_regs->load_jump_add2); in rw_mgr_mem_calibrate_write_test_issue()
1084 writel(rwcfg->lfsr_wr_rd_bank_0_nop, in rw_mgr_mem_calibrate_write_test_issue()
1085 &sdr_rw_load_jump_mgr_regs->load_jump_add3); in rw_mgr_mem_calibrate_write_test_issue()
1089 * CNTR 2 - We want to skip the NOP operation and go straight in rw_mgr_mem_calibrate_write_test_issue()
1093 writel(0xFF, &sdr_rw_load_mgr_regs->load_cntr2); in rw_mgr_mem_calibrate_write_test_issue()
1095 /* CNTR 3 - Not used */ in rw_mgr_mem_calibrate_write_test_issue()
1097 mcc_instruction = rwcfg->lfsr_wr_rd_dm_bank_0; in rw_mgr_mem_calibrate_write_test_issue()
1098 writel(rwcfg->lfsr_wr_rd_dm_bank_0_dqs, in rw_mgr_mem_calibrate_write_test_issue()
1099 &sdr_rw_load_jump_mgr_regs->load_jump_add2); in rw_mgr_mem_calibrate_write_test_issue()
1101 mcc_instruction = rwcfg->lfsr_wr_rd_bank_0; in rw_mgr_mem_calibrate_write_test_issue()
1102 writel(rwcfg->lfsr_wr_rd_bank_0_dqs, in rw_mgr_mem_calibrate_write_test_issue()
1103 &sdr_rw_load_jump_mgr_regs->load_jump_add2); in rw_mgr_mem_calibrate_write_test_issue()
1107 * CNTR 2 - In this case we want to execute the next instruction in rw_mgr_mem_calibrate_write_test_issue()
1111 writel(0x0, &sdr_rw_load_mgr_regs->load_cntr2); in rw_mgr_mem_calibrate_write_test_issue()
1112 writel(0x0, &sdr_rw_load_jump_mgr_regs->load_jump_add2); in rw_mgr_mem_calibrate_write_test_issue()
1115 * CNTR 3 - Set the nop counter to the number of cycles we in rw_mgr_mem_calibrate_write_test_issue()
1118 writel(rw_wl_nop_cycles - 1, &sdr_rw_load_mgr_regs->load_cntr3); in rw_mgr_mem_calibrate_write_test_issue()
1120 mcc_instruction = rwcfg->lfsr_wr_rd_dm_bank_0; in rw_mgr_mem_calibrate_write_test_issue()
1121 writel(rwcfg->lfsr_wr_rd_dm_bank_0_nop, in rw_mgr_mem_calibrate_write_test_issue()
1122 &sdr_rw_load_jump_mgr_regs->load_jump_add3); in rw_mgr_mem_calibrate_write_test_issue()
1124 mcc_instruction = rwcfg->lfsr_wr_rd_bank_0; in rw_mgr_mem_calibrate_write_test_issue()
1125 writel(rwcfg->lfsr_wr_rd_bank_0_nop, in rw_mgr_mem_calibrate_write_test_issue()
1126 &sdr_rw_load_jump_mgr_regs->load_jump_add3); in rw_mgr_mem_calibrate_write_test_issue()
1134 writel(0x08, &sdr_rw_load_mgr_regs->load_cntr0); in rw_mgr_mem_calibrate_write_test_issue()
1136 writel(0x40, &sdr_rw_load_mgr_regs->load_cntr0); in rw_mgr_mem_calibrate_write_test_issue()
1138 writel(mcc_instruction, &sdr_rw_load_jump_mgr_regs->load_jump_add0); in rw_mgr_mem_calibrate_write_test_issue()
1141 * CNTR 1 - This is used to ensure enough time elapses in rw_mgr_mem_calibrate_write_test_issue()
1144 writel(0x30, &sdr_rw_load_mgr_regs->load_cntr1); in rw_mgr_mem_calibrate_write_test_issue()
1147 writel(rwcfg->lfsr_wr_rd_dm_bank_0_wait, in rw_mgr_mem_calibrate_write_test_issue()
1148 &sdr_rw_load_jump_mgr_regs->load_jump_add1); in rw_mgr_mem_calibrate_write_test_issue()
1150 writel(rwcfg->lfsr_wr_rd_bank_0_wait, in rw_mgr_mem_calibrate_write_test_issue()
1151 &sdr_rw_load_jump_mgr_regs->load_jump_add1); in rw_mgr_mem_calibrate_write_test_issue()
1160 * rw_mgr_mem_calibrate_write_test() - Test writes, check for single/multiple pass
1176 rwcfg->mem_number_of_ranks : in rw_mgr_mem_calibrate_write_test()
1178 const u32 shift_ratio = rwcfg->mem_dq_per_write_dqs / in rw_mgr_mem_calibrate_write_test()
1179 rwcfg->mem_virtual_groups_per_write_dqs; in rw_mgr_mem_calibrate_write_test()
1180 const u32 correct_mask_vg = param->write_correct_mask_vg; in rw_mgr_mem_calibrate_write_test()
1185 *bit_chk = param->write_correct_mask; in rw_mgr_mem_calibrate_write_test()
1192 for (vg = rwcfg->mem_virtual_groups_per_write_dqs - 1; in rw_mgr_mem_calibrate_write_test()
1193 vg >= 0; vg--) { in rw_mgr_mem_calibrate_write_test()
1195 writel(0, &phy_mgr_cmd->fifo_reset); in rw_mgr_mem_calibrate_write_test()
1199 rwcfg->mem_virtual_groups_per_write_dqs + vg, in rw_mgr_mem_calibrate_write_test()
1215 param->write_correct_mask, in rw_mgr_mem_calibrate_write_test()
1216 *bit_chk == param->write_correct_mask); in rw_mgr_mem_calibrate_write_test()
1217 return *bit_chk == param->write_correct_mask; in rw_mgr_mem_calibrate_write_test()
1227 * rw_mgr_mem_calibrate_read_test_patterns() - Read back test patterns
1242 (group * rwcfg->mem_virtual_groups_per_read_dqs) << 2; in rw_mgr_mem_calibrate_read_test_patterns()
1244 rwcfg->mem_number_of_ranks : in rw_mgr_mem_calibrate_read_test_patterns()
1246 const u32 shift_ratio = rwcfg->mem_dq_per_read_dqs / in rw_mgr_mem_calibrate_read_test_patterns()
1247 rwcfg->mem_virtual_groups_per_read_dqs; in rw_mgr_mem_calibrate_read_test_patterns()
1248 const u32 correct_mask_vg = param->read_correct_mask_vg; in rw_mgr_mem_calibrate_read_test_patterns()
1254 bit_chk = param->read_correct_mask; in rw_mgr_mem_calibrate_read_test_patterns()
1261 writel(0x20, &sdr_rw_load_mgr_regs->load_cntr0); in rw_mgr_mem_calibrate_read_test_patterns()
1262 writel(rwcfg->guaranteed_read, in rw_mgr_mem_calibrate_read_test_patterns()
1263 &sdr_rw_load_jump_mgr_regs->load_jump_add0); in rw_mgr_mem_calibrate_read_test_patterns()
1265 writel(0x20, &sdr_rw_load_mgr_regs->load_cntr1); in rw_mgr_mem_calibrate_read_test_patterns()
1266 writel(rwcfg->guaranteed_read_cont, in rw_mgr_mem_calibrate_read_test_patterns()
1267 &sdr_rw_load_jump_mgr_regs->load_jump_add1); in rw_mgr_mem_calibrate_read_test_patterns()
1270 for (vg = rwcfg->mem_virtual_groups_per_read_dqs - 1; in rw_mgr_mem_calibrate_read_test_patterns()
1271 vg >= 0; vg--) { in rw_mgr_mem_calibrate_read_test_patterns()
1273 writel(0, &phy_mgr_cmd->fifo_reset); in rw_mgr_mem_calibrate_read_test_patterns()
1276 writel(rwcfg->guaranteed_read, in rw_mgr_mem_calibrate_read_test_patterns()
1287 writel(rwcfg->clear_dqs_enable, addr + (group << 2)); in rw_mgr_mem_calibrate_read_test_patterns()
1291 if (bit_chk != param->read_correct_mask) in rw_mgr_mem_calibrate_read_test_patterns()
1292 ret = -EIO; in rw_mgr_mem_calibrate_read_test_patterns()
1297 param->read_correct_mask, ret); in rw_mgr_mem_calibrate_read_test_patterns()
1303 * rw_mgr_mem_calibrate_read_load_patterns() - Load up the patterns for read test
1313 rwcfg->mem_number_of_ranks : in rw_mgr_mem_calibrate_read_load_patterns()
1324 writel(0x20, &sdr_rw_load_mgr_regs->load_cntr0); in rw_mgr_mem_calibrate_read_load_patterns()
1326 writel(rwcfg->guaranteed_write_wait0, in rw_mgr_mem_calibrate_read_load_patterns()
1327 &sdr_rw_load_jump_mgr_regs->load_jump_add0); in rw_mgr_mem_calibrate_read_load_patterns()
1329 writel(0x20, &sdr_rw_load_mgr_regs->load_cntr1); in rw_mgr_mem_calibrate_read_load_patterns()
1331 writel(rwcfg->guaranteed_write_wait1, in rw_mgr_mem_calibrate_read_load_patterns()
1332 &sdr_rw_load_jump_mgr_regs->load_jump_add1); in rw_mgr_mem_calibrate_read_load_patterns()
1334 writel(0x04, &sdr_rw_load_mgr_regs->load_cntr2); in rw_mgr_mem_calibrate_read_load_patterns()
1336 writel(rwcfg->guaranteed_write_wait2, in rw_mgr_mem_calibrate_read_load_patterns()
1337 &sdr_rw_load_jump_mgr_regs->load_jump_add2); in rw_mgr_mem_calibrate_read_load_patterns()
1339 writel(0x04, &sdr_rw_load_mgr_regs->load_cntr3); in rw_mgr_mem_calibrate_read_load_patterns()
1341 writel(rwcfg->guaranteed_write_wait3, in rw_mgr_mem_calibrate_read_load_patterns()
1342 &sdr_rw_load_jump_mgr_regs->load_jump_add3); in rw_mgr_mem_calibrate_read_load_patterns()
1344 writel(rwcfg->guaranteed_write, SDR_PHYGRP_RWMGRGRP_ADDRESS | in rw_mgr_mem_calibrate_read_load_patterns()
1352 * rw_mgr_mem_calibrate_read_test() - Perform READ test on single rank
1371 const u32 rank_end = all_ranks ? rwcfg->mem_number_of_ranks : in rw_mgr_mem_calibrate_read_test()
1375 misccfg->enable_super_quick_calibration); in rw_mgr_mem_calibrate_read_test()
1376 u32 correct_mask_vg = param->read_correct_mask_vg; in rw_mgr_mem_calibrate_read_test()
1383 *bit_chk = param->read_correct_mask; in rw_mgr_mem_calibrate_read_test()
1389 writel(0x10, &sdr_rw_load_mgr_regs->load_cntr1); in rw_mgr_mem_calibrate_read_test()
1391 writel(rwcfg->read_b2b_wait1, in rw_mgr_mem_calibrate_read_test()
1392 &sdr_rw_load_jump_mgr_regs->load_jump_add1); in rw_mgr_mem_calibrate_read_test()
1394 writel(0x10, &sdr_rw_load_mgr_regs->load_cntr2); in rw_mgr_mem_calibrate_read_test()
1395 writel(rwcfg->read_b2b_wait2, in rw_mgr_mem_calibrate_read_test()
1396 &sdr_rw_load_jump_mgr_regs->load_jump_add2); in rw_mgr_mem_calibrate_read_test()
1399 writel(0x1, &sdr_rw_load_mgr_regs->load_cntr0); in rw_mgr_mem_calibrate_read_test()
1402 writel(0x06, &sdr_rw_load_mgr_regs->load_cntr0); in rw_mgr_mem_calibrate_read_test()
1404 writel(0x32, &sdr_rw_load_mgr_regs->load_cntr0); in rw_mgr_mem_calibrate_read_test()
1406 writel(rwcfg->read_b2b, in rw_mgr_mem_calibrate_read_test()
1407 &sdr_rw_load_jump_mgr_regs->load_jump_add0); in rw_mgr_mem_calibrate_read_test()
1409 writel(rwcfg->mem_if_read_dqs_width * in rw_mgr_mem_calibrate_read_test()
1410 rwcfg->mem_virtual_groups_per_read_dqs - 1, in rw_mgr_mem_calibrate_read_test()
1411 &sdr_rw_load_mgr_regs->load_cntr3); in rw_mgr_mem_calibrate_read_test()
1413 writel(0x0, &sdr_rw_load_mgr_regs->load_cntr3); in rw_mgr_mem_calibrate_read_test()
1415 writel(rwcfg->read_b2b, in rw_mgr_mem_calibrate_read_test()
1416 &sdr_rw_load_jump_mgr_regs->load_jump_add3); in rw_mgr_mem_calibrate_read_test()
1419 for (vg = rwcfg->mem_virtual_groups_per_read_dqs - 1; vg >= 0; in rw_mgr_mem_calibrate_read_test()
1420 vg--) { in rw_mgr_mem_calibrate_read_test()
1422 writel(0, &phy_mgr_cmd->fifo_reset); in rw_mgr_mem_calibrate_read_test()
1434 writel(rwcfg->read_b2b, addr + in rw_mgr_mem_calibrate_read_test()
1436 rwcfg->mem_virtual_groups_per_read_dqs + in rw_mgr_mem_calibrate_read_test()
1440 tmp_bit_chk <<= rwcfg->mem_dq_per_read_dqs / in rw_mgr_mem_calibrate_read_test()
1441 rwcfg->mem_virtual_groups_per_read_dqs; in rw_mgr_mem_calibrate_read_test()
1449 writel(rwcfg->clear_dqs_enable, addr + (group << 2)); in rw_mgr_mem_calibrate_read_test()
1454 ret = (*bit_chk == param->read_correct_mask); in rw_mgr_mem_calibrate_read_test()
1458 param->read_correct_mask, ret); in rw_mgr_mem_calibrate_read_test()
1471 * rw_mgr_mem_calibrate_read_test_all_ranks() - Perform READ test on all ranks
1490 * rw_mgr_incr_vfifo() - Increase VFIFO value
1497 writel(grp, &phy_mgr_cmd->inc_vfifo_hard_phy); in rw_mgr_incr_vfifo()
1501 * rw_mgr_decr_vfifo() - Decrease VFIFO value
1510 for (i = 0; i < misccfg->read_valid_fifo_size - 1; i++) in rw_mgr_decr_vfifo()
1515 * find_vfifo_failing_read() - Push VFIFO to get a failing read
1524 for (v = 0; v < misccfg->read_valid_fifo_size; v++) { in find_vfifo_failing_read()
1546 * sdr_find_phase_delay() - Find DQS enable phase or delay
1547 * @working: If 1, look for working phase/delay, if 0, look for non-working
1548 * @delay: If 1, look for delay, if 0, look for phase
1552 * @pd: DQS Phase/Delay Iterator
1554 * Find working or non-working DQS enable phase setting.
1559 const u32 max = delay ? iocfg->dqs_en_delay_max : in sdr_find_phase_delay()
1560 iocfg->dqs_en_phase_max; in sdr_find_phase_delay()
1581 return -EINVAL; in sdr_find_phase_delay()
1584 * sdr_find_phase() - Find DQS enable phase
1585 * @working: If 1, look for working phase, if 0, look for non-working phase
1589 * @p: DQS Phase Iterator
1591 * Find working or non-working DQS enable phase setting.
1596 const u32 end = misccfg->read_valid_fifo_size + (working ? 0 : 1); in sdr_find_phase()
1604 iocfg->delay_per_opa_tap, p); in sdr_find_phase()
1608 if (*p > iocfg->dqs_en_phase_max) { in sdr_find_phase()
1616 return -EINVAL; in sdr_find_phase()
1620 * sdr_working_phase() - Find working DQS enable phase
1624 * @p: DQS Phase Iterator
1627 * Find working DQS enable phase setting.
1632 const u32 dtaps_per_ptap = iocfg->delay_per_opa_tap / in sdr_working_phase()
1633 iocfg->delay_per_dqs_en_dchain_tap; in sdr_working_phase()
1644 *work_bgn += iocfg->delay_per_dqs_en_dchain_tap; in sdr_working_phase()
1650 return -EINVAL; in sdr_working_phase()
1654 * sdr_backup_phase() - Find DQS enable backup phase
1657 * @p: DQS Phase Iterator
1659 * Find DQS enable backup phase setting.
1666 /* Special case code for backing up a phase */ in sdr_backup_phase()
1668 *p = iocfg->dqs_en_phase_max; in sdr_backup_phase()
1671 (*p)--; in sdr_backup_phase()
1673 tmp_delay = *work_bgn - iocfg->delay_per_opa_tap; in sdr_backup_phase()
1676 for (d = 0; d <= iocfg->dqs_en_delay_max && tmp_delay < *work_bgn; in sdr_backup_phase()
1687 tmp_delay += iocfg->delay_per_dqs_en_dchain_tap; in sdr_backup_phase()
1692 if (*p > iocfg->dqs_en_phase_max) { in sdr_backup_phase()
1701 * sdr_nonworking_phase() - Find non-working DQS enable phase
1704 * @p: DQS Phase Iterator
1707 * Find non-working DQS enable phase setting.
1714 *work_end += iocfg->delay_per_opa_tap; in sdr_nonworking_phase()
1715 if (*p > iocfg->dqs_en_phase_max) { in sdr_nonworking_phase()
1732 * sdr_find_window_center() - Find center of the working DQS window.
1751 tmp_delay = (iocfg->dqs_en_phase_max + 1) * iocfg->delay_per_opa_tap; in sdr_find_window_center()
1757 tmp_delay = rounddown(work_mid, iocfg->delay_per_opa_tap); in sdr_find_window_center()
1758 if (tmp_delay > iocfg->dqs_en_phase_max * iocfg->delay_per_opa_tap) in sdr_find_window_center()
1759 tmp_delay = iocfg->dqs_en_phase_max * iocfg->delay_per_opa_tap; in sdr_find_window_center()
1760 p = tmp_delay / iocfg->delay_per_opa_tap; in sdr_find_window_center()
1764 d = DIV_ROUND_UP(work_mid - tmp_delay, in sdr_find_window_center()
1765 iocfg->delay_per_dqs_en_dchain_tap); in sdr_find_window_center()
1766 if (d > iocfg->dqs_en_delay_max) in sdr_find_window_center()
1767 d = iocfg->dqs_en_delay_max; in sdr_find_window_center()
1768 tmp_delay += d * iocfg->delay_per_dqs_en_dchain_tap; in sdr_find_window_center()
1779 for (i = 0; i < misccfg->read_valid_fifo_size; i++) { in sdr_find_window_center()
1796 return -EINVAL; in sdr_find_window_center()
1800 * rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase() - Find a good DQS enable to use
1820 /* Step 0: Determine number of delay taps for each phase tap. */ in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
1821 dtaps_per_ptap = iocfg->delay_per_opa_tap / in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
1822 iocfg->delay_per_dqs_en_dchain_tap; in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
1827 /* Step 2: Find first working phase, increment in ptaps. */ in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
1836 * If d is 0 then the working window covers a phase tap and we can in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
1848 * Step 4a: go forward from working phase to non working in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
1849 * phase, increment in ptaps. in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
1857 /* Special case code for backing up a phase */ in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
1859 p = iocfg->dqs_en_phase_max; in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
1862 p = p - 1; in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
1865 work_end -= iocfg->delay_per_opa_tap; in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
1876 iocfg->delay_per_dqs_en_dchain_tap, &d); in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
1880 work_end -= iocfg->delay_per_dqs_en_dchain_tap; in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
1884 __func__, __LINE__, p, d - 1, work_end); in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
1888 debug_cond(DLEVEL >= 2, "%s:%d end-2: failed\n", in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
1890 return -EINVAL; in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
1898 * To do that we'll back up a ptap and re-find the edge of the in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
1904 /* Special case code for backing up a phase */ in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
1906 p = iocfg->dqs_en_phase_max; in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
1908 debug_cond(DLEVEL >= 2, "%s:%d backedup cycle/phase: p=%u\n", in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
1911 p = p - 1; in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
1912 debug_cond(DLEVEL >= 2, "%s:%d backedup phase only: p=%u", in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
1947 * (iocfg->dqs_en_delay_max). Otherwise, dtaps_per_ptap retains its in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
1951 dtaps_per_ptap = d - initial_failing_dtap; in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
1953 writel(dtaps_per_ptap, &sdr_reg_file->dtaps_per_ptap); in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
1954 debug_cond(DLEVEL >= 2, "%s:%d dtaps_per_ptap=%u - %u = %u", in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
1964 * search_stop_check() - Check if the detected edge is valid
1981 const u32 ratio = rwcfg->mem_if_read_dqs_width / in search_stop_check()
1982 rwcfg->mem_if_write_dqs_width; in search_stop_check()
1983 const u32 correct_mask = write ? param->write_correct_mask : in search_stop_check()
1984 param->read_correct_mask; in search_stop_check()
1985 const u32 per_dqs = write ? rwcfg->mem_dq_per_write_dqs : in search_stop_check()
1986 rwcfg->mem_dq_per_read_dqs; in search_stop_check()
1992 if (write) { /* WRITE-ONLY */ in search_stop_check()
1996 } else if (use_read_test) { /* READ-ONLY */ in search_stop_check()
2001 } else { /* READ-ONLY */ in search_stop_check()
2005 (read_group - (write_group * ratio))); in search_stop_check()
2018 * search_left_edge() - Find left edge of DQ/DQS working phase
2025 * @left_edge: Left edge of the DQ/DQS phase
2026 * @right_edge: Right edge of the DQ/DQS phase
2029 * Find left edge of DQ/DQS working phase.
2036 const u32 delay_max = write ? iocfg->io_out1_delay_max : in search_left_edge()
2037 iocfg->io_in_delay_max; in search_left_edge()
2038 const u32 dqs_max = write ? iocfg->io_out1_delay_max : in search_left_edge()
2039 iocfg->dqs_in_delay_max; in search_left_edge()
2040 const u32 per_dqs = write ? rwcfg->mem_dq_per_write_dqs : in search_left_edge()
2041 rwcfg->mem_dq_per_read_dqs; in search_left_edge()
2051 writel(0, &sdr_scc_mgr->update); in search_left_edge()
2075 right_edge[i] = -(d + 1); in search_left_edge()
2088 for (i = per_dqs - 1; i >= 0; i--) { in search_left_edge()
2127 * search_right_edge() - Find right edge of DQ/DQS working phase
2132 * @start_dqs: DQS start phase
2133 * @start_dqs_en: DQS enable start phase
2135 * @left_edge: Left edge of the DQ/DQS phase
2136 * @right_edge: Right edge of the DQ/DQS phase
2139 * Find right edge of DQ/DQS working phase.
2147 const u32 delay_max = write ? iocfg->io_out1_delay_max : in search_right_edge()
2148 iocfg->io_in_delay_max; in search_right_edge()
2149 const u32 dqs_max = write ? iocfg->io_out1_delay_max : in search_right_edge()
2150 iocfg->dqs_in_delay_max; in search_right_edge()
2151 const u32 per_dqs = write ? rwcfg->mem_dq_per_write_dqs : in search_right_edge()
2152 rwcfg->mem_dq_per_read_dqs; in search_right_edge()
2156 for (d = 0; d <= dqs_max - start_dqs; d++) { in search_right_edge()
2157 if (write) { /* WRITE-ONLY */ in search_right_edge()
2160 } else { /* READ-ONLY */ in search_right_edge()
2162 if (iocfg->shift_dqs_en_when_shift_dqs) { in search_right_edge()
2164 if (delay > iocfg->dqs_en_delay_max) in search_right_edge()
2165 delay = iocfg->dqs_en_delay_max; in search_right_edge()
2171 writel(0, &sdr_scc_mgr->update); in search_right_edge()
2177 if (write && (d == 0)) { /* WRITE-ONLY */ in search_right_edge()
2178 for (i = 0; i < rwcfg->mem_dq_per_write_dqs; in search_right_edge()
2183 * marginal, set it to -1 in search_right_edge()
2187 right_edge[i] = -1; in search_right_edge()
2210 left_edge[i] = -(d + 1); in search_right_edge()
2216 * it to -1 in search_right_edge()
2220 right_edge[i] = -1; in search_right_edge()
2228 left_edge[i] = -(d + 1); in search_right_edge()
2258 * get_window_mid_index() - Find the best middle setting of DQ/DQS phase
2260 * @left_edge: Left edge of the DQ/DQS phase
2261 * @right_edge: Right edge of the DQ/DQS phase
2262 * @mid_min: Best DQ/DQS phase middle setting
2264 * Find index and value of the middle of the DQ/DQS working phase.
2269 const u32 per_dqs = write ? rwcfg->mem_dq_per_write_dqs : in get_window_mid_index()
2270 rwcfg->mem_dq_per_read_dqs; in get_window_mid_index()
2274 *mid_min = left_edge[0] - right_edge[0]; in get_window_mid_index()
2277 mid = left_edge[i] - right_edge[i]; in get_window_mid_index()
2285 * -mid_min/2 represents the amount that we need to move DQS. in get_window_mid_index()
2286 * If mid_min is odd and positive we'll need to add one to make in get_window_mid_index()
2288 * bias to the right), so just add 1 for all positive values. in get_window_mid_index()
2300 * center_dq_windows() - Center the DQ/DQS windows
2302 * @left_edge: Left edge of the DQ/DQS phase
2303 * @right_edge: Right edge of the DQ/DQS phase
2304 * @mid_min: Adjusted DQ/DQS phase middle setting
2305 * @orig_mid_min: Original DQ/DQS phase middle setting
2306 * @min_index: DQ/DQS phase middle setting index
2318 const s32 delay_max = write ? iocfg->io_out1_delay_max : in center_dq_windows()
2319 iocfg->io_in_delay_max; in center_dq_windows()
2320 const s32 per_dqs = write ? rwcfg->mem_dq_per_write_dqs : in center_dq_windows()
2321 rwcfg->mem_dq_per_read_dqs; in center_dq_windows()
2336 shift_dq = (left_edge[i] - right_edge[i] - in center_dq_windows()
2337 (left_edge[min_index] - right_edge[min_index]))/2 + in center_dq_windows()
2338 (orig_mid_min - mid_min); in center_dq_windows()
2347 shift_dq = delay_max - temp_dq_io_delay1; in center_dq_windows()
2349 shift_dq = -temp_dq_io_delay1; in center_dq_windows()
2366 left_edge[i] - shift_dq + (-mid_min), in center_dq_windows()
2367 right_edge[i] + shift_dq - (-mid_min)); in center_dq_windows()
2370 if (left_edge[i] - shift_dq + (-mid_min) < *dq_margin) in center_dq_windows()
2371 *dq_margin = left_edge[i] - shift_dq + (-mid_min); in center_dq_windows()
2373 if (right_edge[i] + shift_dq - (-mid_min) < *dqs_margin) in center_dq_windows()
2374 *dqs_margin = right_edge[i] + shift_dq - (-mid_min); in center_dq_windows()
2379 * rw_mgr_mem_calibrate_vfifo_center() - Per-bit deskew DQ and centering
2386 * Per-bit deskew DQ and centering.
2400 int32_t left_edge[rwcfg->mem_dq_per_read_dqs]; in rw_mgr_mem_calibrate_vfifo_center()
2401 int32_t right_edge[rwcfg->mem_dq_per_read_dqs]; in rw_mgr_mem_calibrate_vfifo_center()
2411 if (iocfg->shift_dqs_en_when_shift_dqs) in rw_mgr_mem_calibrate_vfifo_center()
2412 start_dqs_en = readl(addr - iocfg->dqs_en_delay_offset); in rw_mgr_mem_calibrate_vfifo_center()
2415 /* use (iocfg->io_in_delay_max + 1) as an illegal value */ in rw_mgr_mem_calibrate_vfifo_center()
2417 for (i = 0; i < rwcfg->mem_dq_per_read_dqs; i++) { in rw_mgr_mem_calibrate_vfifo_center()
2418 left_edge[i] = iocfg->io_in_delay_max + 1; in rw_mgr_mem_calibrate_vfifo_center()
2419 right_edge[i] = iocfg->io_in_delay_max + 1; in rw_mgr_mem_calibrate_vfifo_center()
2440 if (iocfg->shift_dqs_en_when_shift_dqs) in rw_mgr_mem_calibrate_vfifo_center()
2444 writel(0, &sdr_scc_mgr->update); in rw_mgr_mem_calibrate_vfifo_center()
2451 rwcfg->mem_dq_per_read_dqs + i, in rw_mgr_mem_calibrate_vfifo_center()
2456 rwcfg->mem_dq_per_read_dqs + i, in rw_mgr_mem_calibrate_vfifo_center()
2460 return -EIO; in rw_mgr_mem_calibrate_vfifo_center()
2465 /* Determine the amount we can change DQS (which is -mid_min) */ in rw_mgr_mem_calibrate_vfifo_center()
2467 new_dqs = start_dqs - mid_min; in rw_mgr_mem_calibrate_vfifo_center()
2468 if (new_dqs > iocfg->dqs_in_delay_max) in rw_mgr_mem_calibrate_vfifo_center()
2469 new_dqs = iocfg->dqs_in_delay_max; in rw_mgr_mem_calibrate_vfifo_center()
2473 mid_min = start_dqs - new_dqs; in rw_mgr_mem_calibrate_vfifo_center()
2477 if (iocfg->shift_dqs_en_when_shift_dqs) { in rw_mgr_mem_calibrate_vfifo_center()
2478 if (start_dqs_en - mid_min > iocfg->dqs_en_delay_max) in rw_mgr_mem_calibrate_vfifo_center()
2479 mid_min += start_dqs_en - mid_min - in rw_mgr_mem_calibrate_vfifo_center()
2480 iocfg->dqs_en_delay_max; in rw_mgr_mem_calibrate_vfifo_center()
2481 else if (start_dqs_en - mid_min < 0) in rw_mgr_mem_calibrate_vfifo_center()
2482 mid_min += start_dqs_en - mid_min; in rw_mgr_mem_calibrate_vfifo_center()
2484 new_dqs = start_dqs - mid_min; in rw_mgr_mem_calibrate_vfifo_center()
2489 iocfg->shift_dqs_en_when_shift_dqs ? start_dqs_en : -1, in rw_mgr_mem_calibrate_vfifo_center()
2496 /* Move DQS-en */ in rw_mgr_mem_calibrate_vfifo_center()
2497 if (iocfg->shift_dqs_en_when_shift_dqs) { in rw_mgr_mem_calibrate_vfifo_center()
2498 final_dqs_en = start_dqs_en - mid_min; in rw_mgr_mem_calibrate_vfifo_center()
2514 writel(0, &sdr_scc_mgr->update); in rw_mgr_mem_calibrate_vfifo_center()
2517 return -EINVAL; in rw_mgr_mem_calibrate_vfifo_center()
2523 * rw_mgr_mem_calibrate_guaranteed_write() - Perform guaranteed write into the device
2525 * @phase: DQ/DQS phase
2532 const u32 phase) in rw_mgr_mem_calibrate_guaranteed_write() argument
2536 /* Set a particular DQ/DQS phase. */ in rw_mgr_mem_calibrate_guaranteed_write()
2537 scc_mgr_set_dqdqs_output_phase_all_ranks(rw_group, phase); in rw_mgr_mem_calibrate_guaranteed_write()
2540 __func__, __LINE__, rw_group, phase); in rw_mgr_mem_calibrate_guaranteed_write()
2543 * Altera EMI_RM 2015.05.04 :: Figure 1-25 in rw_mgr_mem_calibrate_guaranteed_write()
2545 * current DQDQS phase. in rw_mgr_mem_calibrate_guaranteed_write()
2549 if (gbl->phy_debug_mode_flags & PHY_DEBUG_DISABLE_GUARANTEED_READ) in rw_mgr_mem_calibrate_guaranteed_write()
2553 * Altera EMI_RM 2015.05.04 :: Figure 1-26 in rw_mgr_mem_calibrate_guaranteed_write()
2554 * Back-to-Back reads of the patterns used for calibration. in rw_mgr_mem_calibrate_guaranteed_write()
2560 __func__, __LINE__, rw_group, phase); in rw_mgr_mem_calibrate_guaranteed_write()
2565 * rw_mgr_mem_calibrate_dqs_enable_calibration() - DQS Enable Calibration
2576 * Altera EMI_RM 2015.05.04 :: Figure 1-27 in rw_mgr_mem_calibrate_dqs_enable_calibration()
2581 const u32 delay_step = iocfg->io_in_delay_max / in rw_mgr_mem_calibrate_dqs_enable_calibration()
2582 (rwcfg->mem_dq_per_read_dqs - 1); in rw_mgr_mem_calibrate_dqs_enable_calibration()
2589 for (r = 0; r < rwcfg->mem_number_of_ranks; in rw_mgr_mem_calibrate_dqs_enable_calibration()
2592 i < rwcfg->mem_dq_per_read_dqs; in rw_mgr_mem_calibrate_dqs_enable_calibration()
2602 writel(0, &sdr_scc_mgr->update); in rw_mgr_mem_calibrate_dqs_enable_calibration()
2615 for (r = 0; r < rwcfg->mem_number_of_ranks; in rw_mgr_mem_calibrate_dqs_enable_calibration()
2618 writel(0, &sdr_scc_mgr->update); in rw_mgr_mem_calibrate_dqs_enable_calibration()
2625 * rw_mgr_mem_calibrate_dq_dqs_centering() - Centering DQ/DQS
2644 * Altera EMI_RM 2015.05.04 :: Figure 1-28 in rw_mgr_mem_calibrate_dq_dqs_centering()
2645 * Read per-bit deskew can be done on a per shadow register basis. in rw_mgr_mem_calibrate_dq_dqs_centering()
2649 rank_bgn < rwcfg->mem_number_of_ranks; in rw_mgr_mem_calibrate_dq_dqs_centering()
2662 return -EIO; in rw_mgr_mem_calibrate_dq_dqs_centering()
2668 * rw_mgr_mem_calibrate_vfifo() - Calibrate the read valid prediction FIFO
2677 * - read valid prediction will consist of finding:
2678 * - DQS enable phase and DQS enable delay (DQS Enable Calibration)
2679 * - DQS input phase and DQS input delay (DQ/DQS Centering)
2680 * - we also do a per-bit deskew on the DQ lines.
2699 /* USER Determine number of delay taps for each phase tap. */ in rw_mgr_mem_calibrate_vfifo()
2700 dtaps_per_ptap = DIV_ROUND_UP(iocfg->delay_per_opa_tap, in rw_mgr_mem_calibrate_vfifo()
2701 iocfg->delay_per_dqs_en_dchain_tap) - 1; in rw_mgr_mem_calibrate_vfifo()
2715 for (p = 0; p <= iocfg->dqdqs_out_phase_max; p++) { in rw_mgr_mem_calibrate_vfifo()
2764 * rw_mgr_mem_calibrate_vfifo_end() - DQ/DQS Centering.
2794 * rw_mgr_mem_calibrate_lfifo() - Minimize latency
2816 writel(gbl->curr_read_lat, &phy_mgr_cfg->phy_rlat); in rw_mgr_mem_calibrate_lfifo()
2818 __func__, __LINE__, gbl->curr_read_lat); in rw_mgr_mem_calibrate_lfifo()
2829 gbl->curr_read_lat--; in rw_mgr_mem_calibrate_lfifo()
2830 } while (gbl->curr_read_lat > 0); in rw_mgr_mem_calibrate_lfifo()
2833 writel(0, &phy_mgr_cmd->fifo_reset); in rw_mgr_mem_calibrate_lfifo()
2837 gbl->curr_read_lat += 2; in rw_mgr_mem_calibrate_lfifo()
2838 writel(gbl->curr_read_lat, &phy_mgr_cfg->phy_rlat); in rw_mgr_mem_calibrate_lfifo()
2841 __func__, __LINE__, gbl->curr_read_lat); in rw_mgr_mem_calibrate_lfifo()
2848 __func__, __LINE__, gbl->curr_read_lat); in rw_mgr_mem_calibrate_lfifo()
2855 * search_window() - Search for the/part of the window with DM/DQS shift
2874 const int max = iocfg->io_out1_delay_max - new_dqs; in search_window()
2878 for (di = max; di >= 0; di -= DELTA_D) { in search_window()
2884 d = max - di; in search_window()
2893 writel(0, &sdr_scc_mgr->update); in search_window()
2899 *end_curr = search_dm ? -d : d; in search_window()
2905 if (*bgn_curr == iocfg->io_out1_delay_max + 1) in search_window()
2906 *bgn_curr = search_dm ? -d : d; in search_window()
2912 if ((*end_curr - *bgn_curr + 1) > *win_best) { in search_window()
2913 *win_best = *end_curr - *bgn_curr + 1; in search_window()
2919 *bgn_curr = iocfg->io_out1_delay_max + 1; in search_window()
2920 *end_curr = iocfg->io_out1_delay_max + 1; in search_window()
2931 if (*win_best - 1 > iocfg->io_out1_delay_max - new_dqs - d) in search_window()
2938 * rw_mgr_mem_calibrate_writes_center() - Center all windows
2943 * Center all windows. Do per-bit-deskew to possibly increase size of
2953 int left_edge[rwcfg->mem_dq_per_write_dqs]; in rw_mgr_mem_calibrate_writes_center()
2954 int right_edge[rwcfg->mem_dq_per_write_dqs]; in rw_mgr_mem_calibrate_writes_center()
2959 int bgn_curr = iocfg->io_out1_delay_max + 1; in rw_mgr_mem_calibrate_writes_center()
2960 int end_curr = iocfg->io_out1_delay_max + 1; in rw_mgr_mem_calibrate_writes_center()
2961 int bgn_best = iocfg->io_out1_delay_max + 1; in rw_mgr_mem_calibrate_writes_center()
2962 int end_best = iocfg->io_out1_delay_max + 1; in rw_mgr_mem_calibrate_writes_center()
2973 (rwcfg->mem_dq_per_write_dqs << 2)); in rw_mgr_mem_calibrate_writes_center()
2975 /* Per-bit deskew. */ in rw_mgr_mem_calibrate_writes_center()
2979 * Use (iocfg->io_out1_delay_max + 1) as an illegal value. in rw_mgr_mem_calibrate_writes_center()
2982 for (i = 0; i < rwcfg->mem_dq_per_write_dqs; i++) { in rw_mgr_mem_calibrate_writes_center()
2983 left_edge[i] = iocfg->io_out1_delay_max + 1; in rw_mgr_mem_calibrate_writes_center()
2984 right_edge[i] = iocfg->io_out1_delay_max + 1; in rw_mgr_mem_calibrate_writes_center()
2998 set_failing_group_stage(test_bgn + ret - 1, CAL_STAGE_WRITES, in rw_mgr_mem_calibrate_writes_center()
3000 return -EINVAL; in rw_mgr_mem_calibrate_writes_center()
3005 /* Determine the amount we can change DQS (which is -mid_min). */ in rw_mgr_mem_calibrate_writes_center()
3019 writel(0, &sdr_scc_mgr->update); in rw_mgr_mem_calibrate_writes_center()
3026 * Use (iocfg->io_out1_delay_max + 1) as an illegal value. in rw_mgr_mem_calibrate_writes_center()
3028 left_edge[0] = iocfg->io_out1_delay_max + 1; in rw_mgr_mem_calibrate_writes_center()
3029 right_edge[0] = iocfg->io_out1_delay_max + 1; in rw_mgr_mem_calibrate_writes_center()
3044 bgn_curr = iocfg->io_out1_delay_max + 1; in rw_mgr_mem_calibrate_writes_center()
3045 end_curr = iocfg->io_out1_delay_max + 1; in rw_mgr_mem_calibrate_writes_center()
3053 left_edge[0] = -1 * bgn_best; in rw_mgr_mem_calibrate_writes_center()
3065 mid = (left_edge[0] - right_edge[0]) / 2; in rw_mgr_mem_calibrate_writes_center()
3073 dm_margin = -1; in rw_mgr_mem_calibrate_writes_center()
3075 dm_margin = left_edge[0] - mid; in rw_mgr_mem_calibrate_writes_center()
3078 writel(0, &sdr_scc_mgr->update); in rw_mgr_mem_calibrate_writes_center()
3085 gbl->fom_out += dq_margin + dqs_margin; in rw_mgr_mem_calibrate_writes_center()
3095 writel(0, &sdr_scc_mgr->update); in rw_mgr_mem_calibrate_writes_center()
3098 return -EINVAL; in rw_mgr_mem_calibrate_writes_center()
3104 * rw_mgr_mem_calibrate_writes() - Write Calibration Part One
3135 * mem_precharge_and_activate() - Precharge all banks and activate
3143 for (r = 0; r < rwcfg->mem_number_of_ranks; r++) { in mem_precharge_and_activate()
3148 writel(rwcfg->precharge_all, SDR_PHYGRP_RWMGRGRP_ADDRESS | in mem_precharge_and_activate()
3151 writel(0x0F, &sdr_rw_load_mgr_regs->load_cntr0); in mem_precharge_and_activate()
3152 writel(rwcfg->activate_0_and_1_wait1, in mem_precharge_and_activate()
3153 &sdr_rw_load_jump_mgr_regs->load_jump_add0); in mem_precharge_and_activate()
3155 writel(0x0F, &sdr_rw_load_mgr_regs->load_cntr1); in mem_precharge_and_activate()
3156 writel(rwcfg->activate_0_and_1_wait2, in mem_precharge_and_activate()
3157 &sdr_rw_load_jump_mgr_regs->load_jump_add1); in mem_precharge_and_activate()
3160 writel(rwcfg->activate_0_and_1, SDR_PHYGRP_RWMGRGRP_ADDRESS | in mem_precharge_and_activate()
3166 * mem_init_latency() - Configure memory RLAT and WLAT settings
3177 const u32 max_latency = (1 << misccfg->max_latency_count_width) - 1; in mem_init_latency()
3186 wlat = readl(&data_mgr->t_wl_add); in mem_init_latency()
3187 wlat += readl(&data_mgr->mem_t_add); in mem_init_latency()
3189 gbl->rw_wl_nop_cycles = wlat - 1; in mem_init_latency()
3192 rlat = readl(&data_mgr->t_rl_add); in mem_init_latency()
3195 gbl->curr_read_lat = rlat + 16; in mem_init_latency()
3196 if (gbl->curr_read_lat > max_latency) in mem_init_latency()
3197 gbl->curr_read_lat = max_latency; in mem_init_latency()
3199 writel(gbl->curr_read_lat, &phy_mgr_cfg->phy_rlat); in mem_init_latency()
3202 writel(wlat, &phy_mgr_cfg->afi_wlat); in mem_init_latency()
3206 * @mem_skip_calibrate() - Set VFIFO and LFIFO to instant-on settings
3208 * Set VFIFO and LFIFO to instant-on settings in skip calibration mode.
3217 for (r = 0; r < rwcfg->mem_number_of_ranks; in mem_skip_calibrate()
3220 * Set output phase alignment settings appropriate for in mem_skip_calibrate()
3223 for (i = 0; i < rwcfg->mem_if_read_dqs_width; i++) { in mem_skip_calibrate()
3225 if (iocfg->dll_chain_length == 6) in mem_skip_calibrate()
3234 * -> due to bit-slip in a/c bus in mem_skip_calibrate()
3235 * -> to allow board skew where dqs is longer than ck in mem_skip_calibrate()
3236 * -> how often can this happen!? in mem_skip_calibrate()
3237 * -> can claim back some ptaps for high freq in mem_skip_calibrate()
3247 * (720 - 90 - 180 - 2) * in mem_skip_calibrate()
3248 * (360 / iocfg->dll_chain_length) in mem_skip_calibrate()
3250 * Dividing the above by (360 / iocfg->dll_chain_length) in mem_skip_calibrate()
3253 * (1.25 * iocfg->dll_chain_length - 2) in mem_skip_calibrate()
3256 ((125 * iocfg->dll_chain_length) / 100) - 2); in mem_skip_calibrate()
3258 writel(0xff, &sdr_scc_mgr->dqs_ena); in mem_skip_calibrate()
3259 writel(0xff, &sdr_scc_mgr->dqs_io_ena); in mem_skip_calibrate()
3261 for (i = 0; i < rwcfg->mem_if_write_dqs_width; i++) { in mem_skip_calibrate()
3265 writel(0xff, &sdr_scc_mgr->dq_ena); in mem_skip_calibrate()
3266 writel(0xff, &sdr_scc_mgr->dm_ena); in mem_skip_calibrate()
3267 writel(0, &sdr_scc_mgr->update); in mem_skip_calibrate()
3271 for (i = 0; i < rwcfg->mem_if_read_dqs_width; i++) { in mem_skip_calibrate()
3275 writel(0, &sdr_scc_mgr->update); in mem_skip_calibrate()
3281 vfifo_offset = misccfg->calib_vfifo_offset; in mem_skip_calibrate()
3283 writel(0xff, &phy_mgr_cmd->inc_vfifo_hard_phy); in mem_skip_calibrate()
3284 writel(0, &phy_mgr_cmd->fifo_reset); in mem_skip_calibrate()
3287 * For Arria V and Cyclone V with hard LFIFO, we get the skip-cal in mem_skip_calibrate()
3288 * setting from generation-time constant. in mem_skip_calibrate()
3290 gbl->curr_read_lat = misccfg->calib_lfifo_offset; in mem_skip_calibrate()
3291 writel(gbl->curr_read_lat, &phy_mgr_cfg->phy_rlat); in mem_skip_calibrate()
3295 * mem_calibrate() - Memory calibration entry point.
3309 const u32 rwdqs_ratio = rwcfg->mem_if_read_dqs_width / in mem_calibrate()
3310 rwcfg->mem_if_write_dqs_width; in mem_calibrate()
3315 gbl->error_substage = CAL_SUBSTAGE_NIL; in mem_calibrate()
3316 gbl->error_stage = CAL_STAGE_NIL; in mem_calibrate()
3317 gbl->error_group = 0xff; in mem_calibrate()
3318 gbl->fom_in = 0; in mem_calibrate()
3319 gbl->fom_out = 0; in mem_calibrate()
3327 for (i = 0; i < rwcfg->mem_if_read_dqs_width; i++) { in mem_calibrate()
3340 * Set VFIFO and LFIFO to instant-on settings in skip in mem_calibrate()
3349 writel(0, &sdr_scc_mgr->update); in mem_calibrate()
3356 * Zero all delay chain/phase settings for all in mem_calibrate()
3364 < rwcfg->mem_if_write_dqs_width; write_group++, in mem_calibrate()
3365 write_test_bgn += rwcfg->mem_dq_per_write_dqs) { in mem_calibrate()
3370 RW_MGR_NUM_DQS_PER_WRITE_GROUP) - 1); in mem_calibrate()
3385 read_test_bgn += rwcfg->mem_dq_per_read_dqs) { in mem_calibrate()
3394 if (!(gbl->phy_debug_mode_flags & in mem_calibrate()
3404 rank_bgn < rwcfg->mem_number_of_ranks; in mem_calibrate()
3421 if (!(gbl->phy_debug_mode_flags & in mem_calibrate()
3434 read_test_bgn += rwcfg->mem_dq_per_read_dqs) { in mem_calibrate()
3442 if (!(gbl->phy_debug_mode_flags & in mem_calibrate()
3476 writel(0, &sdr_scc_mgr->update); in mem_calibrate()
3481 * run_mem_calibrate() - Perform memory calibration
3493 writel(PHY_MGR_CAL_RESET, &phy_mgr_cfg->cal_status); in run_mem_calibrate()
3496 ctrl_cfg = readl(&sdr_ctrl->ctrl_cfg); in run_mem_calibrate()
3498 &sdr_ctrl->ctrl_cfg); in run_mem_calibrate()
3507 writel(0, &phy_mgr_cmd->fifo_reset); in run_mem_calibrate()
3512 * In Hard PHY this is a 2-bit control: in run_mem_calibrate()
3516 writel(0x2, &phy_mgr_cfg->mux_sel); in run_mem_calibrate()
3519 writel(ctrl_cfg, &sdr_ctrl->ctrl_cfg); in run_mem_calibrate()
3525 * debug_mem_calibrate() - Report result of memory calibration
3538 gbl->fom_in /= 2; in debug_mem_calibrate()
3539 gbl->fom_out /= 2; in debug_mem_calibrate()
3541 if (gbl->fom_in > 0xff) in debug_mem_calibrate()
3542 gbl->fom_in = 0xff; in debug_mem_calibrate()
3544 if (gbl->fom_out > 0xff) in debug_mem_calibrate()
3545 gbl->fom_out = 0xff; in debug_mem_calibrate()
3548 debug_info = gbl->fom_in; in debug_mem_calibrate()
3549 debug_info |= gbl->fom_out << 8; in debug_mem_calibrate()
3550 writel(debug_info, &sdr_reg_file->fom); in debug_mem_calibrate()
3552 writel(debug_info, &phy_mgr_cfg->cal_debug_info); in debug_mem_calibrate()
3553 writel(PHY_MGR_CAL_SUCCESS, &phy_mgr_cfg->cal_status); in debug_mem_calibrate()
3557 debug_info = gbl->error_stage; in debug_mem_calibrate()
3558 debug_info |= gbl->error_substage << 8; in debug_mem_calibrate()
3559 debug_info |= gbl->error_group << 16; in debug_mem_calibrate()
3561 writel(debug_info, &sdr_reg_file->failing_stage); in debug_mem_calibrate()
3562 writel(debug_info, &phy_mgr_cfg->cal_debug_info); in debug_mem_calibrate()
3563 writel(PHY_MGR_CAL_FAIL, &phy_mgr_cfg->cal_status); in debug_mem_calibrate()
3566 debug_info = gbl->error_stage; in debug_mem_calibrate()
3567 debug_info |= gbl->error_substage << 8; in debug_mem_calibrate()
3568 debug_info |= gbl->error_group << 16; in debug_mem_calibrate()
3569 writel(debug_info, &sdr_reg_file->failing_stage); in debug_mem_calibrate()
3576 * hc_initialize_rom_data() - Initialize ROM data
3598 * initialize_reg_file() - Initialize SDR register file
3605 writel(misccfg->reg_file_init_seq_signature, &sdr_reg_file->signature); in initialize_reg_file()
3606 writel(0, &sdr_reg_file->debug_data_addr); in initialize_reg_file()
3607 writel(0, &sdr_reg_file->cur_stage); in initialize_reg_file()
3608 writel(0, &sdr_reg_file->fom); in initialize_reg_file()
3609 writel(0, &sdr_reg_file->failing_stage); in initialize_reg_file()
3610 writel(0, &sdr_reg_file->debug1); in initialize_reg_file()
3611 writel(0, &sdr_reg_file->debug2); in initialize_reg_file()
3615 * initialize_hps_phy() - Initialize HPS PHY
3642 * 00-bypass, 01- add 5 cycles, 10- add 10 cycles, 11- add 15 cycles. in initialize_hps_phy()
3647 writel(reg, &sdr_ctrl->phy_ctrl0); in initialize_hps_phy()
3655 writel(reg, &sdr_ctrl->phy_ctrl1); in initialize_hps_phy()
3661 writel(reg, &sdr_ctrl->phy_ctrl2); in initialize_hps_phy()
3665 * initialize_tracking() - Initialize tracking
3676 writel(DIV_ROUND_UP(iocfg->delay_per_opa_tap, in initialize_tracking()
3677 iocfg->delay_per_dchain_tap) - 1, in initialize_tracking()
3678 &sdr_reg_file->dtaps_per_ptap); in initialize_tracking()
3681 writel(7500, &sdr_reg_file->trk_sample_count); in initialize_tracking()
3684 writel((10 << 16) | (100 << 0), &sdr_reg_file->trk_longidle); in initialize_tracking()
3693 &sdr_reg_file->delays); in initialize_tracking()
3696 writel((rwcfg->idle << 24) | (rwcfg->activate_1 << 16) | in initialize_tracking()
3697 (rwcfg->sgle_read << 8) | (rwcfg->precharge_all << 0), in initialize_tracking()
3698 &sdr_reg_file->trk_rw_mgr_addr); in initialize_tracking()
3700 writel(rwcfg->mem_if_read_dqs_width, in initialize_tracking()
3701 &sdr_reg_file->trk_read_dqs_width); in initialize_tracking()
3704 writel((rwcfg->refresh_all << 24) | (1000 << 0), in initialize_tracking()
3705 &sdr_reg_file->trk_rfsh); in initialize_tracking()
3725 gbl->phy_debug_mode_flags |= PHY_DEBUG_ENABLE_CAL_RPT; in sdram_calibration_full()
3731 gbl->phy_debug_mode_flags |= PHY_DEBUG_DISABLE_GUARANTEED_READ; in sdram_calibration_full()
3748 rwcfg->mem_number_of_ranks, rwcfg->mem_number_of_cs_per_dimm, in sdram_calibration_full()
3749 rwcfg->mem_dq_per_read_dqs, rwcfg->mem_dq_per_write_dqs, in sdram_calibration_full()
3750 rwcfg->mem_virtual_groups_per_read_dqs, in sdram_calibration_full()
3751 rwcfg->mem_virtual_groups_per_write_dqs); in sdram_calibration_full()
3754 rwcfg->mem_if_read_dqs_width, rwcfg->mem_if_write_dqs_width, in sdram_calibration_full()
3755 rwcfg->mem_data_width, rwcfg->mem_data_mask_width, in sdram_calibration_full()
3756 iocfg->delay_per_opa_tap, iocfg->delay_per_dchain_tap); in sdram_calibration_full()
3758 iocfg->delay_per_dqs_en_dchain_tap, iocfg->dll_chain_length); in sdram_calibration_full()
3761 iocfg->dqs_en_phase_max, iocfg->dqdqs_out_phase_max, in sdram_calibration_full()
3762 iocfg->dqs_en_delay_max, iocfg->dqs_in_delay_max); in sdram_calibration_full()
3764 iocfg->io_in_delay_max, iocfg->io_out1_delay_max, in sdram_calibration_full()
3765 iocfg->io_out2_delay_max); in sdram_calibration_full()
3767 iocfg->dqs_in_reserve, iocfg->dqs_out_reserve); in sdram_calibration_full()