258 static void scc_mgr_set(u32 off, u32 grp, u32 val)  in scc_mgr_set()  argument
260 	writel(val, SDR_PHYGRP_SCCGRP_ADDRESS | off | (grp << 2));  in scc_mgr_set()
375 static void scc_mgr_set_all_ranks(const u32 off, const u32 grp, const u32 val,  in scc_mgr_set_all_ranks()  argument
382 		scc_mgr_set(off, grp, val);  in scc_mgr_set_all_ranks()
385 			writel(grp, &sdr_scc_mgr->dqs_ena);  in scc_mgr_set_all_ranks()
1480 rw_mgr_mem_calibrate_read_test_all_ranks(const u32 grp, const u32 num_tries,  in rw_mgr_mem_calibrate_read_test_all_ranks()  argument
1485 	return rw_mgr_mem_calibrate_read_test(0, grp, num_tries, all_correct,  in rw_mgr_mem_calibrate_read_test_all_ranks()
1495 static void rw_mgr_incr_vfifo(const u32 grp)  in rw_mgr_incr_vfifo()  argument
1497 	writel(grp, &phy_mgr_cmd->inc_vfifo_hard_phy);  in rw_mgr_incr_vfifo()
1506 static void rw_mgr_decr_vfifo(const u32 grp)  in rw_mgr_decr_vfifo()  argument
1511 		rw_mgr_incr_vfifo(grp);  in rw_mgr_decr_vfifo()
1520 static int find_vfifo_failing_read(const u32 grp)  in find_vfifo_failing_read()  argument
1527 		ret = rw_mgr_mem_calibrate_read_test_all_ranks(grp, 1,  in find_vfifo_failing_read()
1537 		rw_mgr_incr_vfifo(grp);  in find_vfifo_failing_read()
1556 static int sdr_find_phase_delay(int working, int delay, const u32 grp,  in sdr_find_phase_delay()  argument
1565 			scc_mgr_set_dqs_en_delay_all_ranks(grp, *pd);  in sdr_find_phase_delay()
1567 			scc_mgr_set_dqs_en_phase_all_ranks(grp, *pd);  in sdr_find_phase_delay()
1569 		ret = rw_mgr_mem_calibrate_read_test_all_ranks(grp, 1,  in sdr_find_phase_delay()
1593 static int sdr_find_phase(int working, const u32 grp, u32 *work,  in sdr_find_phase()  argument
1603 		ret = sdr_find_phase_delay(working, 0, grp, work,  in sdr_find_phase()
1610 			rw_mgr_incr_vfifo(grp);  in sdr_find_phase()
1629 static int sdr_working_phase(const u32 grp, u32 *work_bgn, u32 *d,  in sdr_working_phase()  argument
1640 		scc_mgr_set_dqs_en_delay_all_ranks(grp, *d);  in sdr_working_phase()
1641 		ret = sdr_find_phase(1, grp, work_bgn, i, p);  in sdr_working_phase()
1661 static void sdr_backup_phase(const u32 grp, u32 *work_bgn, u32 *p)  in sdr_backup_phase()  argument
1669 		rw_mgr_decr_vfifo(grp);  in sdr_backup_phase()
1674 	scc_mgr_set_dqs_en_phase_all_ranks(grp, *p);  in sdr_backup_phase()
1678 		scc_mgr_set_dqs_en_delay_all_ranks(grp, d);  in sdr_backup_phase()
1680 		ret = rw_mgr_mem_calibrate_read_test_all_ranks(grp, 1,  in sdr_backup_phase()
1694 		rw_mgr_incr_vfifo(grp);  in sdr_backup_phase()
1697 	scc_mgr_set_dqs_en_delay_all_ranks(grp, 0);  in sdr_backup_phase()
1709 static int sdr_nonworking_phase(const u32 grp, u32 *work_end, u32 *p, u32 *i)  in sdr_nonworking_phase()  argument
1718 		rw_mgr_incr_vfifo(grp);  in sdr_nonworking_phase()
1721 	ret = sdr_find_phase(0, grp, work_end, i, p);  in sdr_nonworking_phase()
1739 static int sdr_find_window_center(const u32 grp, const u32 work_bgn,  in sdr_find_window_center()  argument
1772 	scc_mgr_set_dqs_en_phase_all_ranks(grp, p);  in sdr_find_window_center()
1773 	scc_mgr_set_dqs_en_delay_all_ranks(grp, d);  in sdr_find_window_center()
1781 		if (rw_mgr_mem_calibrate_read_test_all_ranks(grp, 1,  in sdr_find_window_center()
1791 		rw_mgr_incr_vfifo(grp);  in sdr_find_window_center()
1805 static int rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase(const u32 grp)  in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()  argument
1813 	debug("%s:%d %u\n", __func__, __LINE__, grp);  in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
1817 	scc_mgr_set_dqs_en_delay_all_ranks(grp, 0);  in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
1818 	scc_mgr_set_dqs_en_phase_all_ranks(grp, 0);  in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
1825 	find_vfifo_failing_read(grp);  in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
1829 	ret = sdr_working_phase(grp, &work_bgn, &d, &p, &i);  in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
1845 		sdr_backup_phase(grp, &work_bgn, &p);  in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
1851 		ret = sdr_nonworking_phase(grp, &work_end, &p, &i);  in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
1860 			rw_mgr_decr_vfifo(grp);  in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
1866 		scc_mgr_set_dqs_en_phase_all_ranks(grp, p);  in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
1875 	sdr_find_phase_delay(0, 1, grp, &work_end,  in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
1907 		rw_mgr_decr_vfifo(grp);  in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
1916 	scc_mgr_set_dqs_en_phase_all_ranks(grp, p);  in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
1930 	found_passing_read = !sdr_find_phase_delay(1, 1, grp, NULL, 0, &d);  in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
1936 		found_failing_read = !sdr_find_phase_delay(0, 1, grp, NULL, 0,  in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
1958 	ret = sdr_find_window_center(grp, work_bgn, work_end);  in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()