1 // SPDX-License-Identifier: GPL-2.0+
31 	writel(DDR3PHY_CTRL_PHY_RESET_OFF, &clk->lpddr3phy_ctrl);  in reset_phy_ctrl()
32 	writel(DDR3PHY_CTRL_PHY_RESET, &clk->lpddr3phy_ctrl);  in reset_phy_ctrl()
50 	/* Set Impedance Output Driver */  in ddr3_mem_ctrl_init()
51 	val = (mem->impedance << CA_CK_DRVR_DS_OFFSET) |  in ddr3_mem_ctrl_init()
52 		(mem->impedance << CA_CKE_DRVR_DS_OFFSET) |  in ddr3_mem_ctrl_init()
53 		(mem->impedance << CA_CS_DRVR_DS_OFFSET) |  in ddr3_mem_ctrl_init()
54 		(mem->impedance << CA_ADR_DRVR_DS_OFFSET);  in ddr3_mem_ctrl_init()
55 	writel(val, &phy0_ctrl->phy_con39);  in ddr3_mem_ctrl_init()
56 	writel(val, &phy1_ctrl->phy_con39);  in ddr3_mem_ctrl_init()
59 	val = (mem->ctrl_bstlen << PHY_CON42_CTRL_BSTLEN_SHIFT) |  in ddr3_mem_ctrl_init()
60 		(mem->ctrl_rdlat << PHY_CON42_CTRL_RDLAT_SHIFT);  in ddr3_mem_ctrl_init()
61 	writel(val, &phy0_ctrl->phy_con42);  in ddr3_mem_ctrl_init()
62 	writel(val, &phy1_ctrl->phy_con42);  in ddr3_mem_ctrl_init()
65 	if (dmc_config_zq(mem, &phy0_ctrl->phy_con16, &phy1_ctrl->phy_con16,  in ddr3_mem_ctrl_init()
66 			  &phy0_ctrl->phy_con17, &phy1_ctrl->phy_con17))  in ddr3_mem_ctrl_init()
70 	writel(mem->phy0_pulld_dqs, &phy0_ctrl->phy_con14);  in ddr3_mem_ctrl_init()
71 	writel(mem->phy1_pulld_dqs, &phy1_ctrl->phy_con14);  in ddr3_mem_ctrl_init()
73 	writel(mem->concontrol | (mem->rd_fetch << CONCONTROL_RD_FETCH_SHIFT)  in ddr3_mem_ctrl_init()
74 		| (mem->dfi_init_start << CONCONTROL_DFI_INIT_START_SHIFT),  in ddr3_mem_ctrl_init()
75 		&dmc->concontrol);  in ddr3_mem_ctrl_init()
77 	update_reset_dll(&dmc->phycontrol0, DDR_MODE_DDR3);  in ddr3_mem_ctrl_init()
80 	writel(mem->phy0_dqs, &phy0_ctrl->phy_con4);  in ddr3_mem_ctrl_init()
81 	writel(mem->phy1_dqs, &phy1_ctrl->phy_con4);  in ddr3_mem_ctrl_init()
83 	writel(mem->phy0_dq, &phy0_ctrl->phy_con6);  in ddr3_mem_ctrl_init()
84 	writel(mem->phy1_dq, &phy1_ctrl->phy_con6);  in ddr3_mem_ctrl_init()
86 	writel(mem->phy0_tFS, &phy0_ctrl->phy_con10);  in ddr3_mem_ctrl_init()
87 	writel(mem->phy1_tFS, &phy1_ctrl->phy_con10);  in ddr3_mem_ctrl_init()
89 	val = (mem->ctrl_start_point << PHY_CON12_CTRL_START_POINT_SHIFT) |  in ddr3_mem_ctrl_init()
90 		(mem->ctrl_inc << PHY_CON12_CTRL_INC_SHIFT) |  in ddr3_mem_ctrl_init()
91 		(mem->ctrl_dll_on << PHY_CON12_CTRL_DLL_ON_SHIFT) |  in ddr3_mem_ctrl_init()
92 		(mem->ctrl_ref << PHY_CON12_CTRL_REF_SHIFT);  in ddr3_mem_ctrl_init()
93 	writel(val, &phy0_ctrl->phy_con12);  in ddr3_mem_ctrl_init()
94 	writel(val, &phy1_ctrl->phy_con12);  in ddr3_mem_ctrl_init()
97 	writel(val | (mem->ctrl_start << PHY_CON12_CTRL_START_SHIFT),  in ddr3_mem_ctrl_init()
98 	       &phy0_ctrl->phy_con12);  in ddr3_mem_ctrl_init()
99 	writel(val | (mem->ctrl_start << PHY_CON12_CTRL_START_SHIFT),  in ddr3_mem_ctrl_init()
100 	       &phy1_ctrl->phy_con12);  in ddr3_mem_ctrl_init()
102 	update_reset_dll(&dmc->phycontrol0, DDR_MODE_DDR3);  in ddr3_mem_ctrl_init()
104 	writel(mem->concontrol | (mem->rd_fetch << CONCONTROL_RD_FETCH_SHIFT),  in ddr3_mem_ctrl_init()
105 	       &dmc->concontrol);  in ddr3_mem_ctrl_init()
108 	writel(mem->iv_size, &dmc->ivcontrol);  in ddr3_mem_ctrl_init()
110 	writel(mem->memconfig, &dmc->memconfig0);  in ddr3_mem_ctrl_init()
111 	writel(mem->memconfig, &dmc->memconfig1);  in ddr3_mem_ctrl_init()
112 	writel(mem->membaseconfig0, &dmc->membaseconfig0);  in ddr3_mem_ctrl_init()
113 	writel(mem->membaseconfig1, &dmc->membaseconfig1);  in ddr3_mem_ctrl_init()
116 	writel(mem->prechconfig_tp_cnt << PRECHCONFIG_TP_CNT_SHIFT,  in ddr3_mem_ctrl_init()
117 	       &dmc->prechconfig);  in ddr3_mem_ctrl_init()
120 	writel(mem->dpwrdn_cyc << PWRDNCONFIG_DPWRDN_CYC_SHIFT |  in ddr3_mem_ctrl_init()
121 		mem->dsref_cyc << PWRDNCONFIG_DSREF_CYC_SHIFT,  in ddr3_mem_ctrl_init()
122 		&dmc->pwrdnconfig);  in ddr3_mem_ctrl_init()
127 	writel(mem->timing_ref, &dmc->timingref);  in ddr3_mem_ctrl_init()
128 	writel(mem->timing_row, &dmc->timingrow);  in ddr3_mem_ctrl_init()
129 	writel(mem->timing_data, &dmc->timingdata);  in ddr3_mem_ctrl_init()
130 	writel(mem->timing_power, &dmc->timingpower);  in ddr3_mem_ctrl_init()
133 	dmc_config_prech(mem, &dmc->directcmd);  in ddr3_mem_ctrl_init()
136 	dmc_config_mrs(mem, &dmc->directcmd);  in ddr3_mem_ctrl_init()
138 	if (mem->gate_leveling_enable) {  in ddr3_mem_ctrl_init()
141 		writel(val, &phy0_ctrl->phy_con0);  in ddr3_mem_ctrl_init()
142 		writel(val, &phy1_ctrl->phy_con0);  in ddr3_mem_ctrl_init()
146 		writel(val, &phy0_ctrl->phy_con2);  in ddr3_mem_ctrl_init()
147 		writel(val, &phy1_ctrl->phy_con2);  in ddr3_mem_ctrl_init()
152 		writel(val, &phy0_ctrl->phy_con0);  in ddr3_mem_ctrl_init()
153 		writel(val, &phy1_ctrl->phy_con0);  in ddr3_mem_ctrl_init()
155 		val = (mem->ctrl_start_point <<  in ddr3_mem_ctrl_init()
157 			(mem->ctrl_inc << PHY_CON12_CTRL_INC_SHIFT) |  in ddr3_mem_ctrl_init()
158 			(mem->ctrl_force << PHY_CON12_CTRL_FORCE_SHIFT) |  in ddr3_mem_ctrl_init()
159 			(mem->ctrl_start << PHY_CON12_CTRL_START_SHIFT) |  in ddr3_mem_ctrl_init()
160 			(mem->ctrl_ref << PHY_CON12_CTRL_REF_SHIFT);  in ddr3_mem_ctrl_init()
161 		writel(val, &phy0_ctrl->phy_con12);  in ddr3_mem_ctrl_init()
162 		writel(val, &phy1_ctrl->phy_con12);  in ddr3_mem_ctrl_init()
167 		writel(val, &phy0_ctrl->phy_con2);  in ddr3_mem_ctrl_init()
168 		writel(val, &phy1_ctrl->phy_con2);  in ddr3_mem_ctrl_init()
174 		writel(val, &phy0_ctrl->phy_con0);  in ddr3_mem_ctrl_init()
175 		writel(val, &phy1_ctrl->phy_con0);  in ddr3_mem_ctrl_init()
179 		writel(val, &phy0_ctrl->phy_con1);  in ddr3_mem_ctrl_init()
180 		writel(val, &phy1_ctrl->phy_con1);  in ddr3_mem_ctrl_init()
182 		writel(CTRL_RDLVL_GATE_ENABLE, &dmc->rdlvl_config);  in ddr3_mem_ctrl_init()
184 		while ((readl(&dmc->phystatus) &  in ddr3_mem_ctrl_init()
192 			i--;  in ddr3_mem_ctrl_init()
196 		writel(CTRL_RDLVL_GATE_DISABLE, &dmc->rdlvl_config);  in ddr3_mem_ctrl_init()
198 		writel(0, &phy0_ctrl->phy_con14);  in ddr3_mem_ctrl_init()
199 		writel(0, &phy1_ctrl->phy_con14);  in ddr3_mem_ctrl_init()
201 		val = (mem->ctrl_start_point <<  in ddr3_mem_ctrl_init()
203 			(mem->ctrl_inc << PHY_CON12_CTRL_INC_SHIFT) |  in ddr3_mem_ctrl_init()
204 			(mem->ctrl_force << PHY_CON12_CTRL_FORCE_SHIFT) |  in ddr3_mem_ctrl_init()
205 			(mem->ctrl_start << PHY_CON12_CTRL_START_SHIFT) |  in ddr3_mem_ctrl_init()
206 			(mem->ctrl_dll_on << PHY_CON12_CTRL_DLL_ON_SHIFT) |  in ddr3_mem_ctrl_init()
207 			(mem->ctrl_ref << PHY_CON12_CTRL_REF_SHIFT);  in ddr3_mem_ctrl_init()
208 		writel(val, &phy0_ctrl->phy_con12);  in ddr3_mem_ctrl_init()
209 		writel(val, &phy1_ctrl->phy_con12);  in ddr3_mem_ctrl_init()
211 		update_reset_dll(&dmc->phycontrol0, DDR_MODE_DDR3);  in ddr3_mem_ctrl_init()
215 	dmc_config_prech(mem, &dmc->directcmd);  in ddr3_mem_ctrl_init()
217 	writel(mem->memcontrol, &dmc->memcontrol);  in ddr3_mem_ctrl_init()
219 	/* Set DMC Concontrol and enable auto-refresh counter */  in ddr3_mem_ctrl_init()
220 	writel(mem->concontrol | (mem->rd_fetch << CONCONTROL_RD_FETCH_SHIFT)  in ddr3_mem_ctrl_init()
221 		| (mem->aref_en << CONCONTROL_AREF_EN_SHIFT), &dmc->concontrol);  in ddr3_mem_ctrl_init()
283 	return readl(&phy_ctrl->phy_con4);  in dmc_get_read_offset_value()
293 	setbits_le32(&phy_ctrl->phy_con10, PHY_CON10_CTRL_OFFSETR3);  in ddr_phy_set_do_resync()
294 	clrbits_le32(&phy_ctrl->phy_con10, PHY_CON10_CTRL_OFFSETR3);  in ddr_phy_set_do_resync()
308 	writel(offset, &phy_ctrl->phy_con4);  in dmc_set_read_offset_value()
316  *   As an example, -10 = 0xf6.  After this function -10 = 0x8a.  If you wanted
317  *   to do math and get the average of 10 and -10 (should be 0):
318  *     0x8a + 0xa = 0x94 (-108)
319  *     0x94 / 2   = 0xca (-54)
320  *   ...and 0xca = sign bit plus 0x4a, or -74
322  * Also note that you lose the ability to represent -128 since there are two
326  * @return	The 7-bit value + sign bit.
332 		return 0x80 | -b;  in make_signed_byte()
346  * @param start			the start shift.  -127 to 127
347  * @param end			the end shift.  -127 to 127
354 	int incr = (start < end) ? 1 : -1;  in test_shifts()
383  * This function performs SW read leveling to compensate DQ-DQS skew at
387  * Read offset value has its min margin and max margin. If read offset
388  * value exceeds its min or max margin, read data will have corruption.
400  *				(0 - 0x7f)
417 	/* Figure out the limits we'll test with; keep -127 < limit < 127 */  in software_find_read_offset()
418 	left_limit = DEFAULT_DQS - coarse_lock_val;  in software_find_read_offset()
468 		mem->memcontrol |= DMC_MEMCONTROL_NUM_CHIP_2;  in ddr3_mem_ctrl_init()
469 		mem->chips_per_channel = 2;  in ddr3_mem_ctrl_init()
470 		mem->chips_to_configure = 2;  in ddr3_mem_ctrl_init()
473 		mem->memcontrol |= DMC_MEMCONTROL_NUM_CHIP_1;  in ddr3_mem_ctrl_init()
477 	setbits_le32(&clk->pause, ENABLE_BIT);  in ddr3_mem_ctrl_init()
480 	setbits_le32(&clk->bpll_con1, BYPASS_EN);  in ddr3_mem_ctrl_init()
482 	writel(MUX_BPLL_SEL_FOUTBPLL, &clk->src_cdrex);  in ddr3_mem_ctrl_init()
484 		val = readl(&clk->mux_stat_cdrex);  in ddr3_mem_ctrl_init()
488 	clrbits_le32(&clk->bpll_con1, BYPASS_EN);  in ddr3_mem_ctrl_init()
491 	val = readl(&phy0_ctrl->phy_con0);  in ddr3_mem_ctrl_init()
494 	writel(val, &phy0_ctrl->phy_con0);  in ddr3_mem_ctrl_init()
496 	val = readl(&phy1_ctrl->phy_con0);  in ddr3_mem_ctrl_init()
499 	writel(val, &phy1_ctrl->phy_con0);  in ddr3_mem_ctrl_init()
502 	val = (mem->ctrl_bstlen << PHY_CON42_CTRL_BSTLEN_SHIFT) |  in ddr3_mem_ctrl_init()
503 		(mem->ctrl_rdlat << PHY_CON42_CTRL_RDLAT_SHIFT);  in ddr3_mem_ctrl_init()
504 	writel(val, &phy0_ctrl->phy_con42);  in ddr3_mem_ctrl_init()
505 	writel(val, &phy1_ctrl->phy_con42);  in ddr3_mem_ctrl_init()
507 	val = readl(&phy0_ctrl->phy_con26);  in ddr3_mem_ctrl_init()
510 	writel(val, &phy0_ctrl->phy_con26);  in ddr3_mem_ctrl_init()
512 	val = readl(&phy1_ctrl->phy_con26);  in ddr3_mem_ctrl_init()
515 	writel(val, &phy1_ctrl->phy_con26);  in ddr3_mem_ctrl_init()
525 	writel(val, &phy0_ctrl->phy_con39);  in ddr3_mem_ctrl_init()
526 	writel(val, &phy1_ctrl->phy_con39);  in ddr3_mem_ctrl_init()
529 	if (dmc_config_zq(mem, &phy0_ctrl->phy_con16, &phy1_ctrl->phy_con16,  in ddr3_mem_ctrl_init()
530 			  &phy0_ctrl->phy_con17, &phy1_ctrl->phy_con17))  in ddr3_mem_ctrl_init()
533 	clrbits_le32(&phy0_ctrl->phy_con16, ZQ_CLK_DIV_EN);  in ddr3_mem_ctrl_init()
534 	clrbits_le32(&phy1_ctrl->phy_con16, ZQ_CLK_DIV_EN);  in ddr3_mem_ctrl_init()
537 	val = readl(&phy0_ctrl->phy_con14);  in ddr3_mem_ctrl_init()
538 	val |= mem->phy0_pulld_dqs;  in ddr3_mem_ctrl_init()
539 	writel(val, &phy0_ctrl->phy_con14);  in ddr3_mem_ctrl_init()
540 	val = readl(&phy1_ctrl->phy_con14);  in ddr3_mem_ctrl_init()
541 	val |= mem->phy1_pulld_dqs;  in ddr3_mem_ctrl_init()
542 	writel(val, &phy1_ctrl->phy_con14);  in ddr3_mem_ctrl_init()
545 	writel(val, &drex0->phycontrol0);  in ddr3_mem_ctrl_init()
546 	writel(val, &drex1->phycontrol0);  in ddr3_mem_ctrl_init()
548 	writel(mem->concontrol |  in ddr3_mem_ctrl_init()
549 		(mem->dfi_init_start << CONCONTROL_DFI_INIT_START_SHIFT) |  in ddr3_mem_ctrl_init()
550 		(mem->rd_fetch << CONCONTROL_RD_FETCH_SHIFT),  in ddr3_mem_ctrl_init()
551 		&drex0->concontrol);  in ddr3_mem_ctrl_init()
552 	writel(mem->concontrol |  in ddr3_mem_ctrl_init()
553 		(mem->dfi_init_start << CONCONTROL_DFI_INIT_START_SHIFT) |  in ddr3_mem_ctrl_init()
554 		(mem->rd_fetch << CONCONTROL_RD_FETCH_SHIFT),  in ddr3_mem_ctrl_init()
555 		&drex1->concontrol);  in ddr3_mem_ctrl_init()
558 		val = readl(&drex0->phystatus);  in ddr3_mem_ctrl_init()
561 		val = readl(&drex1->phystatus);  in ddr3_mem_ctrl_init()
564 	clrbits_le32(&drex0->concontrol, DFI_INIT_START);  in ddr3_mem_ctrl_init()
565 	clrbits_le32(&drex1->concontrol, DFI_INIT_START);  in ddr3_mem_ctrl_init()
567 	update_reset_dll(&drex0->phycontrol0, DDR_MODE_DDR3);  in ddr3_mem_ctrl_init()
568 	update_reset_dll(&drex1->phycontrol0, DDR_MODE_DDR3);  in ddr3_mem_ctrl_init()
578 	writel(val, &tzasc0->membaseconfig0);  in ddr3_mem_ctrl_init()
579 	writel(val, &tzasc1->membaseconfig0);  in ddr3_mem_ctrl_init()
584 	writel(val, &tzasc0->membaseconfig1);  in ddr3_mem_ctrl_init()
585 	writel(val, &tzasc1->membaseconfig1);  in ddr3_mem_ctrl_init()
592 	writel(mem->memconfig, &tzasc0->memconfig0);  in ddr3_mem_ctrl_init()
593 	writel(mem->memconfig, &tzasc1->memconfig0);  in ddr3_mem_ctrl_init()
594 	writel(mem->memconfig, &tzasc0->memconfig1);  in ddr3_mem_ctrl_init()
595 	writel(mem->memconfig, &tzasc1->memconfig1);  in ddr3_mem_ctrl_init()
598 	writel(mem->prechconfig_tp_cnt << PRECHCONFIG_TP_CNT_SHIFT,  in ddr3_mem_ctrl_init()
599 	       &drex0->prechconfig0);  in ddr3_mem_ctrl_init()
600 	writel(mem->prechconfig_tp_cnt << PRECHCONFIG_TP_CNT_SHIFT,  in ddr3_mem_ctrl_init()
601 	       &drex1->prechconfig0);  in ddr3_mem_ctrl_init()
607 	writel(mem->timing_ref, &drex0->timingref);  in ddr3_mem_ctrl_init()
608 	writel(mem->timing_ref, &drex1->timingref);  in ddr3_mem_ctrl_init()
609 	writel(mem->timing_row, &drex0->timingrow0);  in ddr3_mem_ctrl_init()
610 	writel(mem->timing_row, &drex1->timingrow0);  in ddr3_mem_ctrl_init()
611 	writel(mem->timing_data, &drex0->timingdata0);  in ddr3_mem_ctrl_init()
612 	writel(mem->timing_data, &drex1->timingdata0);  in ddr3_mem_ctrl_init()
613 	writel(mem->timing_power, &drex0->timingpower0);  in ddr3_mem_ctrl_init()
614 	writel(mem->timing_power, &drex1->timingpower0);  in ddr3_mem_ctrl_init()
623 		dmc_config_mrs(mem, &drex0->directcmd);  in ddr3_mem_ctrl_init()
624 		dmc_config_mrs(mem, &drex1->directcmd);  in ddr3_mem_ctrl_init()
637 	val = readl(&clk->gate_bus_cdrex);  in ddr3_mem_ctrl_init()
639 		writel(val & ~0x1, &clk->gate_bus_cdrex);  in ddr3_mem_ctrl_init()
640 		lock0_info = readl(&phy0_ctrl->phy_con13);  in ddr3_mem_ctrl_init()
641 		writel(val, &clk->gate_bus_cdrex);  in ddr3_mem_ctrl_init()
649 		writel(val & ~0x2, &clk->gate_bus_cdrex);  in ddr3_mem_ctrl_init()
650 		lock1_info = readl(&phy1_ctrl->phy_con13);  in ddr3_mem_ctrl_init()
651 		writel(val, &clk->gate_bus_cdrex);  in ddr3_mem_ctrl_init()
661 		 * During Suspend-Resume & S/W-Reset, as soon as PMU releases  in ddr3_mem_ctrl_init()
669 		       &power->pad_retention_dram_coreblk_option);  in ddr3_mem_ctrl_init()
671 			val = readl(&power->pad_retention_dram_status);  in ddr3_mem_ctrl_init()
675 		 * CKE PAD retention disables DRAM self-refresh mode.  in ddr3_mem_ctrl_init()
679 			for (chip = 0; chip < mem->chips_to_configure; chip++) {  in ddr3_mem_ctrl_init()
682 				       &drex0->directcmd);  in ddr3_mem_ctrl_init()
685 				       &drex1->directcmd);  in ddr3_mem_ctrl_init()
690 	if (mem->gate_leveling_enable) {  in ddr3_mem_ctrl_init()
691 		writel(PHY_CON0_RESET_VAL, &phy0_ctrl->phy_con0);  in ddr3_mem_ctrl_init()
692 		writel(PHY_CON0_RESET_VAL, &phy1_ctrl->phy_con0);  in ddr3_mem_ctrl_init()
694 		setbits_le32(&phy0_ctrl->phy_con0, P0_CMD_EN);  in ddr3_mem_ctrl_init()
695 		setbits_le32(&phy1_ctrl->phy_con0, P0_CMD_EN);  in ddr3_mem_ctrl_init()
699 		writel(val, &phy0_ctrl->phy_con2);  in ddr3_mem_ctrl_init()
700 		writel(val, &phy1_ctrl->phy_con2);  in ddr3_mem_ctrl_init()
702 		val =  readl(&phy0_ctrl->phy_con1);  in ddr3_mem_ctrl_init()
704 		writel(val, &phy0_ctrl->phy_con1);  in ddr3_mem_ctrl_init()
706 		val =  readl(&phy1_ctrl->phy_con1);  in ddr3_mem_ctrl_init()
708 		writel(val, &phy1_ctrl->phy_con1);  in ddr3_mem_ctrl_init()
711 		n_lock_r = readl(&phy0_ctrl->phy_con12);  in ddr3_mem_ctrl_init()
714 		writel(n_lock_r, &phy0_ctrl->phy_con12);  in ddr3_mem_ctrl_init()
717 		n_lock_r = readl(&phy1_ctrl->phy_con12);  in ddr3_mem_ctrl_init()
720 		writel(n_lock_r, &phy1_ctrl->phy_con12);  in ddr3_mem_ctrl_init()
723 		for (chip = 0; chip < mem->chips_to_configure; chip++) {  in ddr3_mem_ctrl_init()
725 			       &drex0->directcmd);  in ddr3_mem_ctrl_init()
727 			       &drex1->directcmd);  in ddr3_mem_ctrl_init()
730 		setbits_le32(&phy0_ctrl->phy_con2, RDLVL_GATE_EN);  in ddr3_mem_ctrl_init()
731 		setbits_le32(&phy1_ctrl->phy_con2, RDLVL_GATE_EN);  in ddr3_mem_ctrl_init()
733 		setbits_le32(&phy0_ctrl->phy_con0, CTRL_SHGATE);  in ddr3_mem_ctrl_init()
734 		setbits_le32(&phy1_ctrl->phy_con0, CTRL_SHGATE);  in ddr3_mem_ctrl_init()
736 		val = readl(&phy0_ctrl->phy_con1);  in ddr3_mem_ctrl_init()
738 		writel(val, &phy0_ctrl->phy_con1);  in ddr3_mem_ctrl_init()
740 		val = readl(&phy1_ctrl->phy_con1);  in ddr3_mem_ctrl_init()
742 		writel(val, &phy1_ctrl->phy_con1);  in ddr3_mem_ctrl_init()
744 		writel(CTRL_RDLVL_GATE_ENABLE, &drex0->rdlvl_config);  in ddr3_mem_ctrl_init()
746 		while (((readl(&drex0->phystatus) & RDLVL_COMPLETE_CHO) !=  in ddr3_mem_ctrl_init()
753 			i--;  in ddr3_mem_ctrl_init()
757 		writel(CTRL_RDLVL_GATE_DISABLE, &drex0->rdlvl_config);  in ddr3_mem_ctrl_init()
759 		writel(CTRL_RDLVL_GATE_ENABLE, &drex1->rdlvl_config);  in ddr3_mem_ctrl_init()
761 		while (((readl(&drex1->phystatus) & RDLVL_COMPLETE_CHO) !=  in ddr3_mem_ctrl_init()
768 			i--;  in ddr3_mem_ctrl_init()
772 		writel(CTRL_RDLVL_GATE_DISABLE, &drex1->rdlvl_config);  in ddr3_mem_ctrl_init()
774 		writel(0, &phy0_ctrl->phy_con14);  in ddr3_mem_ctrl_init()
775 		writel(0, &phy1_ctrl->phy_con14);  in ddr3_mem_ctrl_init()
778 		for (chip = 0; chip < mem->chips_to_configure; chip++) {  in ddr3_mem_ctrl_init()
780 			       &drex0->directcmd);  in ddr3_mem_ctrl_init()
782 			       &drex1->directcmd);  in ddr3_mem_ctrl_init()
787 		writel((val + n_lock_w_phy0), &phy0_ctrl->phy_con12);  in ddr3_mem_ctrl_init()
788 		writel((val + n_lock_w_phy1), &phy1_ctrl->phy_con12);  in ddr3_mem_ctrl_init()
790 		setbits_le32(&phy0_ctrl->phy_con2, DLL_DESKEW_EN);  in ddr3_mem_ctrl_init()
791 		setbits_le32(&phy1_ctrl->phy_con2, DLL_DESKEW_EN);  in ddr3_mem_ctrl_init()
803 		dmc_set_read_offset_value(phy0_ctrl, readl(&pmu->pmu_spare1));  in ddr3_mem_ctrl_init()
804 		dmc_set_read_offset_value(phy1_ctrl, readl(&pmu->pmu_spare2));  in ddr3_mem_ctrl_init()
811 		writel(dmc_get_read_offset_value(phy0_ctrl), &pmu->pmu_spare1);  in ddr3_mem_ctrl_init()
812 		writel(dmc_get_read_offset_value(phy1_ctrl), &pmu->pmu_spare2);  in ddr3_mem_ctrl_init()
816 	dmc_config_prech(mem, &drex0->directcmd);  in ddr3_mem_ctrl_init()
817 	dmc_config_prech(mem, &drex1->directcmd);  in ddr3_mem_ctrl_init()
819 	writel(mem->memcontrol, &drex0->memcontrol);  in ddr3_mem_ctrl_init()
820 	writel(mem->memcontrol, &drex1->memcontrol);  in ddr3_mem_ctrl_init()
823 	 * Set DMC Concontrol: Enable auto-refresh counter, provide  in ddr3_mem_ctrl_init()
827 	writel(mem->concontrol | (mem->aref_en << CONCONTROL_AREF_EN_SHIFT) |  in ddr3_mem_ctrl_init()
828 		(mem->rd_fetch << CONCONTROL_RD_FETCH_SHIFT)|  in ddr3_mem_ctrl_init()
830 		&drex0->concontrol);  in ddr3_mem_ctrl_init()
831 	writel(mem->concontrol | (mem->aref_en << CONCONTROL_AREF_EN_SHIFT) |  in ddr3_mem_ctrl_init()
832 		(mem->rd_fetch << CONCONTROL_RD_FETCH_SHIFT)|  in ddr3_mem_ctrl_init()
834 		&drex1->concontrol);  in ddr3_mem_ctrl_init()
841 	setbits_le32(&drex0->cgcontrol, DMC_INTERNAL_CG);  in ddr3_mem_ctrl_init()
842 	setbits_le32(&drex1->cgcontrol, DMC_INTERNAL_CG);  in ddr3_mem_ctrl_init()
850 	 * When we apply PHY-initiated mode, DLL lock value is determined  in ddr3_mem_ctrl_init()
852 	 * voltage based on ASV group in kernel. Applying MC-initiated mode  in ddr3_mem_ctrl_init()
856 	val = readl(&drex0->concontrol);  in ddr3_mem_ctrl_init()
858 	writel(val, &drex0->concontrol);  in ddr3_mem_ctrl_init()
859 	val = readl(&drex1->concontrol);  in ddr3_mem_ctrl_init()
861 	writel(val, &drex1->concontrol);  in ddr3_mem_ctrl_init()