91 		u32 delay, phase, pup, cs;  in ddr3_read_leveling_hw()  local
108 					phase = (reg >> REG_PHY_PHASE_OFFS) &  in ddr3_read_leveling_hw()
111 					dram_info->rl_val[cs][pup][P] = phase;  in ddr3_read_leveling_hw()
112 					if (phase > dram_info->rl_max_phase)  in ddr3_read_leveling_hw()
113 						dram_info->rl_max_phase = phase;  in ddr3_read_leveling_hw()
114 					if (phase < dram_info->rl_min_phase)  in ddr3_read_leveling_hw()
115 						dram_info->rl_min_phase = phase;  in ddr3_read_leveling_hw()
181 	u32 reg, cs, ecc, pup_num, phase, delay, pup;  in ddr3_read_leveling_sw()  local
290 			phase = dram_info->rl_val[cs][pup][P];  in ddr3_read_leveling_sw()
292 			ddr3_write_pup_reg(PUP_RL_MODE, cs, pup_num, phase,  in ddr3_read_leveling_sw()
338 		    u32 phase)  in overrun()  argument
359 					info->rl_val[cs][idx][PS] = phase;  in overrun()
403 	u32 reg, delay, phase, pup, rd_sample_delay, add, locked_pups,  in ddr3_read_leveling_single_cs_rl_mode()  local
412 	phase = 0;  in ddr3_read_leveling_single_cs_rl_mode()
433 		DEBUG_RL_FULL_D(phase, 1);  in ddr3_read_leveling_single_cs_rl_mode()
442 		ddr3_write_pup_reg(PUP_RL_MODE, cs, PUP_BC, phase, delay);  in ddr3_read_leveling_single_cs_rl_mode()
473 					delay, phase);  in ddr3_read_leveling_single_cs_rl_mode()
518 			if ((!ratio_2to1) && ((phase == 0) || (phase == 4)))  in ddr3_read_leveling_single_cs_rl_mode()
531 					if ((!ratio_2to1 && phase ==  in ddr3_read_leveling_single_cs_rl_mode()
533 					    || (ratio_2to1 && phase ==  in ddr3_read_leveling_single_cs_rl_mode()
545 						if (phase < MAX_PHASE_RL_L_1TO1) {  in ddr3_read_leveling_single_cs_rl_mode()
546 							if (phase == 1) {  in ddr3_read_leveling_single_cs_rl_mode()
547 								phase = 4;  in ddr3_read_leveling_single_cs_rl_mode()
549 								phase++;  in ddr3_read_leveling_single_cs_rl_mode()
559 						if (phase < MAX_PHASE_RL_UL_1TO1) {  in ddr3_read_leveling_single_cs_rl_mode()
560 							phase++;  in ddr3_read_leveling_single_cs_rl_mode()
564 							phase = 0;  in ddr3_read_leveling_single_cs_rl_mode()
571 						if (phase < MAX_PHASE_RL_L_2TO1) {  in ddr3_read_leveling_single_cs_rl_mode()
572 							phase++;  in ddr3_read_leveling_single_cs_rl_mode()
588 						if (phase < MAX_PHASE_RL_UL_2TO1)  in ddr3_read_leveling_single_cs_rl_mode()
589 							phase++;  in ddr3_read_leveling_single_cs_rl_mode()
591 							phase = 0;  in ddr3_read_leveling_single_cs_rl_mode()
599 				if (phase == 0 && first_octet_locked == 0) {  in ddr3_read_leveling_single_cs_rl_mode()
635 					switch (phase) {  in ddr3_read_leveling_single_cs_rl_mode()
661 					       (phase *  in ddr3_read_leveling_single_cs_rl_mode()
755 	u32 reg, delay, phase, sum, pup, rd_sample_delay, add, locked_pups,  in ddr3_read_leveling_single_cs_window_mode()  local
764 	phase = 0;  in ddr3_read_leveling_single_cs_window_mode()
787 		DEBUG_RL_FULL_D(phase, 1);  in ddr3_read_leveling_single_cs_window_mode()
796 		ddr3_write_pup_reg(PUP_RL_MODE, cs, PUP_BC, phase, delay);  in ddr3_read_leveling_single_cs_window_mode()
847 						dram_info->rl_val[cs][idx][PE] = phase;  in ddr3_read_leveling_single_cs_window_mode()
878 									phase;  in ddr3_read_leveling_single_cs_window_mode()
955 					     && phase == MAX_PHASE_RL_L_1TO1)  in ddr3_read_leveling_single_cs_window_mode()
957 						&& phase ==  in ddr3_read_leveling_single_cs_window_mode()
968 						if (phase < MAX_PHASE_RL_L_1TO1) {  in ddr3_read_leveling_single_cs_window_mode()
970 							if (phase == 0)  in ddr3_read_leveling_single_cs_window_mode()
972 							if (phase == 1)  in ddr3_read_leveling_single_cs_window_mode()
974 								phase = 4;  in ddr3_read_leveling_single_cs_window_mode()
976 								phase++;  in ddr3_read_leveling_single_cs_window_mode()
983 						if (phase < MAX_PHASE_RL_UL_1TO1) {  in ddr3_read_leveling_single_cs_window_mode()
985 							if (phase == 0)  in ddr3_read_leveling_single_cs_window_mode()
986 								phase = 4;  in ddr3_read_leveling_single_cs_window_mode()
988 							phase++;  in ddr3_read_leveling_single_cs_window_mode()
991 							phase = 0;  in ddr3_read_leveling_single_cs_window_mode()
997 						if (phase < MAX_PHASE_RL_L_2TO1) {  in ddr3_read_leveling_single_cs_window_mode()
998 							phase++;  in ddr3_read_leveling_single_cs_window_mode()
1005 						if (phase < MAX_PHASE_RL_UL_2TO1)  in ddr3_read_leveling_single_cs_window_mode()
1006 							phase++;  in ddr3_read_leveling_single_cs_window_mode()
1008 							phase = 0;  in ddr3_read_leveling_single_cs_window_mode()
1017 				if (phase == 0 && first_octet_locked == 0) {  in ddr3_read_leveling_single_cs_window_mode()
1040 					switch (phase) {  in ddr3_read_leveling_single_cs_window_mode()
1064 					add = (add >> phase *  in ddr3_read_leveling_single_cs_window_mode()
1121 			phase = tmp / MAX_DELAY_INV;  in ddr3_read_leveling_single_cs_window_mode()
1122 			if (phase == 1)	/* 1:1 mode */  in ddr3_read_leveling_single_cs_window_mode()
1123 				phase = 4;  in ddr3_read_leveling_single_cs_window_mode()
1125 			if (phase < phase_min)	/* for the read ready delay */  in ddr3_read_leveling_single_cs_window_mode()
1126 				phase_min = phase;  in ddr3_read_leveling_single_cs_window_mode()
1128 			dram_info->rl_val[cs][idx][P] = phase;  in ddr3_read_leveling_single_cs_window_mode()
1138 			phase = tmp / MAX_DELAY;  in ddr3_read_leveling_single_cs_window_mode()
1140 			if (phase < phase_min)	/* for the read ready delay */  in ddr3_read_leveling_single_cs_window_mode()
1141 				phase_min = phase;  in ddr3_read_leveling_single_cs_window_mode()
1143 			dram_info->rl_val[cs][idx][P] = phase;  in ddr3_read_leveling_single_cs_window_mode()
1160 		phase = tmp / MAX_DELAY;  in ddr3_read_leveling_single_cs_window_mode()
1161 		if (!ratio_2to1 && phase > 1)	/* 1:1 mode */  in ddr3_read_leveling_single_cs_window_mode()
1162 			phase += 2;  in ddr3_read_leveling_single_cs_window_mode()
1164 		if (phase < phase_min)	/* for the read ready delay */  in ddr3_read_leveling_single_cs_window_mode()
1165 			phase_min = phase;  in ddr3_read_leveling_single_cs_window_mode()
1167 		dram_info->rl_val[cs][idx][P] = phase;  in ddr3_read_leveling_single_cs_window_mode()