1 // SPDX-License-Identifier: GPL-2.0+
2 /*
3  * ddrmc DDR3 calibration code for NXP's VF610
4  *
5  * Copyright (C) 2018 DENX Software Engineering
6  * Lukasz Majewski, DENX Software Engineering, lukma@denx.de
7  *
8  */
9 /* #define DEBUG */
10 #include <common.h>
11 #include <asm/io.h>
12 #include <asm/arch/imx-regs.h>
13 #include <linux/bitmap.h>
14 
15 #include "ddrmc-vf610-calibration.h"
16 
17 /*
18  * Documents:
19  *
20  * [1] "Vybrid: About DDR leveling feature on DDRMC."
21  * https://community.nxp.com/thread/395323
22  *
23  * [2] VFxxx Controller Reference Manual, Rev. 0, 10/2016
24  *
25  *
26  * NOTE
27  * ====
28  *
29  * NXP recommends setting 'fixed' parameters instead of performing the
30  * training at each boot.
31  *
32  * Use those functions to determine those values on new HW, read the
33  * calculated value from registers and add them to the board specific
34  * struct ddrmc_cr_setting.
35  *
36  * SW leveling supported operations - CR93[SW_LVL_MODE]:
37  *
38  * - 0x0 (b'00) - No leveling
39  *
40  * - 0x1 (b'01) - WRLVL_DL_X - It is not recommended to perform this tuning
41  *                             on HW designs utilizing non-flyback topology
42  *                             (Single DDR3 with x16).
43  *                             Instead the WRLVL_DL_0/1 fields shall be set
44  *                             based on trace length differences from their
45  *                             layout.
46  *                             Mismatches up to 25% or tCK (clock period) are
47  *                             allowed, so the value in the filed doesn’t have
48  *                             to be very accurate.
49  *
50  * - 0x2 (b'10) - RDLVL_DL_0/1 - refers to adjusting the DQS strobe in relation
51  *                             to the DQ signals so that the strobe edge is
52  *                             centered in the window of valid read data.
53  *
54  * - 0x3 (b'11) - RDLVL_GTDL_0/1 - refers to the delay the PHY uses to un-gate
55  *                             the Read DQS strobe pad from the time that the
56  *                             PHY enables the pad to input the strobe signal.
57  *
58  */
59 static int ddr_cal_get_first_edge_index(unsigned long *bmap, enum edge e,
60 					int samples, int start, int max)
61 {
62 	int i, ret = -1;
63 
64 	/*
65 	 * We look only for the first value (and filter out
66 	 * some wrong data)
67 	 */
68 	switch (e) {
69 	case RISING_EDGE:
70 		for (i = start; i <= max - samples; i++) {
71 			if (test_bit(i, bmap)) {
72 				if (!test_bit(i - 1, bmap) &&
73 				    test_bit(i + 1, bmap) &&
74 				    test_bit(i + 2, bmap) &&
75 				    test_bit(i + 3, bmap)) {
76 					return i;
77 				}
78 			}
79 		}
80 		break;
81 	case FALLING_EDGE:
82 		for (i = start; i <= max - samples; i++) {
83 			if (!test_bit(i, bmap)) {
84 				if (test_bit(i - 1, bmap) &&
85 				    test_bit(i - 2, bmap) &&
86 				    test_bit(i - 3, bmap)) {
87 					return i;
88 				}
89 			}
90 		}
91 	}
92 
93 	return ret;
94 }
95 
96 static void bitmap_print(unsigned long *bmap, int max)
97 {
98 	int i;
99 
100 	debug("BITMAP [0x%p]:\n", bmap);
101 	for (i = 0; i <= max; i++) {
102 		debug("%d ", test_bit(i, bmap) ? 1 : 0);
103 		if (i && (i % 32) == (32 - 1))
104 			debug("\n");
105 	}
106 	debug("\n");
107 }
108 
109 #define sw_leveling_op_done \
110 	while (!(readl(&ddrmr->cr[94]) & DDRMC_CR94_SWLVL_OP_DONE))
111 
112 #define sw_leveling_load_value \
113 	do { clrsetbits_le32(&ddrmr->cr[93], DDRMC_CR93_SWLVL_LOAD, \
114 			     DDRMC_CR93_SWLVL_LOAD); } while (0)
115 
116 #define sw_leveling_start \
117 	do { clrsetbits_le32(&ddrmr->cr[93], DDRMC_CR93_SWLVL_START, \
118 			     DDRMC_CR93_SWLVL_START); } while (0)
119 
120 #define sw_leveling_exit \
121 	do { clrsetbits_le32(&ddrmr->cr[94], DDRMC_CR94_SWLVL_EXIT, \
122 			     DDRMC_CR94_SWLVL_EXIT); } while (0)
123 
124 /*
125  * RDLVL_DL calibration:
126  *
127  * NXP is _NOT_ recommending performing the leveling at each
128  * boot. Instead - one shall run this procedure on new boards
129  * and then use hardcoded values.
130  *
131  */
132 static int ddrmc_cal_dqs_to_dq(struct ddrmr_regs *ddrmr)
133 {
134 	DECLARE_BITMAP(rdlvl_rsp, DDRMC_DQS_DQ_MAX_DELAY + 1);
135 	int rdlvl_dl_0_min = -1, rdlvl_dl_0_max = -1;
136 	int rdlvl_dl_1_min = -1, rdlvl_dl_1_max = -1;
137 	int rdlvl_dl_0, rdlvl_dl_1;
138 	u8 swlvl_rsp;
139 	u32 tmp;
140 	int i;
141 
142 	/* Read defaults */
143 	u16 rdlvl_dl_0_def =
144 		(readl(&ddrmr->cr[105]) >> DDRMC_CR105_RDLVL_DL_0_OFF) & 0xFFFF;
145 	u16 rdlvl_dl_1_def = readl(&ddrmr->cr[110]) & 0xFFFF;
146 
147 	debug("\nRDLVL: ======================\n");
148 	debug("RDLVL: DQS to DQ (RDLVL)\n");
149 
150 	debug("RDLVL: RDLVL_DL_0_DFL:\t 0x%x\n", rdlvl_dl_0_def);
151 	debug("RDLVL: RDLVL_DL_1_DFL:\t 0x%x\n", rdlvl_dl_1_def);
152 
153 	/*
154 	 * Set/Read setup for calibration
155 	 *
156 	 * Values necessary for leveling from Vybrid RM [2] - page 1600
157 	 */
158 	writel(0x40703030, &ddrmr->cr[144]);
159 	writel(0x40, &ddrmr->cr[145]);
160 	writel(0x40, &ddrmr->cr[146]);
161 
162 	tmp = readl(&ddrmr->cr[144]);
163 	debug("RDLVL: PHY_RDLVL_RES:\t 0x%x\n", (tmp >> 24) & 0xFF);// set 0x40
164 	debug("RDLVL: PHY_RDLV_LOAD:\t 0x%x\n", (tmp >> 16) & 0xFF);// set 0x70
165 	debug("RDLVL: PHY_RDLV_DLL:\t 0x%x\n", (tmp >> 8) & 0xFF); // set 0x30
166 	debug("RDLVL: PHY_RDLV_EN:\t 0x%x\n", tmp & 0xFF); //set 0x30
167 
168 	tmp = readl(&ddrmr->cr[145]);
169 	debug("RDLVL: PHY_RDLV_RR:\t 0x%x\n", tmp & 0x3FF); //set 0x40
170 
171 	tmp = readl(&ddrmr->cr[146]);
172 	debug("RDLVL: PHY_RDLV_RESP:\t 0x%x\n", tmp); //set 0x40
173 
174 	/*
175 	 * Program/read the leveling edge RDLVL_EDGE = 0
176 	 *
177 	 * 0x00 is the correct output on SWLVL_RSP_X
178 	 * If by any chance 1s are visible -> wrong number read
179 	 */
180 	clrbits_le32(&ddrmr->cr[101], DDRMC_CR101_PHY_RDLVL_EDGE);
181 
182 	tmp = readl(&ddrmr->cr[101]);
183 	debug("RDLVL: PHY_RDLVL_EDGE:\t 0x%x\n",
184 	      (tmp >> DDRMC_CR101_PHY_RDLVL_EDGE_OFF) & 0x1); //set 0
185 
186 	/* Program Leveling mode - CR93[SW_LVL_MODE] to ’b10 */
187 	clrsetbits_le32(&ddrmr->cr[93], DDRMC_CR93_SW_LVL_MODE(0x3),
188 			DDRMC_CR93_SW_LVL_MODE(0x2));
189 	tmp = readl(&ddrmr->cr[93]);
190 	debug("RDLVL: SW_LVL_MODE:\t 0x%x\n",
191 	      (tmp >> DDRMC_CR93_SW_LVL_MODE_OFF) & 0x3);
192 
193 	/* Start procedure - CR93[SWLVL_START] to ’b1 */
194 	sw_leveling_start;
195 
196 	/* Poll CR94[SWLVL_OP_DONE] */
197 	sw_leveling_op_done;
198 
199 	/*
200 	 * Program delays for RDLVL_DL_0
201 	 *
202 	 * The procedure is to increase the delay values from 0 to 0xFF
203 	 * and read the response from the DDRMC
204 	 */
205 	debug("\nRDLVL: ---> RDLVL_DL_0\n");
206 	bitmap_zero(rdlvl_rsp, DDRMC_DQS_DQ_MAX_DELAY + 1);
207 
208 	for (i = 0; i <= DDRMC_DQS_DQ_MAX_DELAY; i++) {
209 		clrsetbits_le32(&ddrmr->cr[105],
210 				0xFFFF << DDRMC_CR105_RDLVL_DL_0_OFF,
211 				i << DDRMC_CR105_RDLVL_DL_0_OFF);
212 
213 		/* Load values CR93[SWLVL_LOAD] to ’b1 */
214 		sw_leveling_load_value;
215 
216 		/* Poll CR94[SWLVL_OP_DONE] */
217 		sw_leveling_op_done;
218 
219 		/*
220 		 * Read Responses - SWLVL_RESP_0
221 		 *
222 		 * The 0x00 (correct response when PHY_RDLVL_EDGE = 0)
223 		 * -> 1 in the bit vector
224 		 */
225 		swlvl_rsp = (readl(&ddrmr->cr[94]) >>
226 			     DDRMC_CR94_SWLVL_RESP_0_OFF) & 0xF;
227 		if (swlvl_rsp == 0)
228 			generic_set_bit(i, rdlvl_rsp);
229 	}
230 
231 	bitmap_print(rdlvl_rsp, DDRMC_DQS_DQ_MAX_DELAY);
232 
233 	/*
234 	 * First test for rising edge 0x0 -> 0x1 in bitmap
235 	 */
236 	rdlvl_dl_0_min = ddr_cal_get_first_edge_index(rdlvl_rsp, RISING_EDGE,
237 						      N_SAMPLES, N_SAMPLES,
238 						      DDRMC_DQS_DQ_MAX_DELAY);
239 
240 	/*
241 	 * Secondly test for falling edge 0x1 -> 0x0 in bitmap
242 	 */
243 	rdlvl_dl_0_max = ddr_cal_get_first_edge_index(rdlvl_rsp, FALLING_EDGE,
244 						      N_SAMPLES, rdlvl_dl_0_min,
245 						      DDRMC_DQS_DQ_MAX_DELAY);
246 
247 	debug("RDLVL: DL_0 min: %d [0x%x] DL_0 max: %d [0x%x]\n",
248 	      rdlvl_dl_0_min, rdlvl_dl_0_min, rdlvl_dl_0_max, rdlvl_dl_0_max);
249 	rdlvl_dl_0 = (rdlvl_dl_0_max - rdlvl_dl_0_min) / 2;
250 
251 	if (rdlvl_dl_0_max == -1 || rdlvl_dl_0_min == -1 || rdlvl_dl_0 <= 0) {
252 		debug("RDLVL: The DQS to DQ delay cannot be found!\n");
253 		debug("RDLVL: Using default - slice 0: %d!\n", rdlvl_dl_0_def);
254 		rdlvl_dl_0 = rdlvl_dl_0_def;
255 	}
256 
257 	debug("\nRDLVL: ---> RDLVL_DL_1\n");
258 	bitmap_zero(rdlvl_rsp, DDRMC_DQS_DQ_MAX_DELAY + 1);
259 
260 	for (i = 0; i <= DDRMC_DQS_DQ_MAX_DELAY; i++) {
261 		clrsetbits_le32(&ddrmr->cr[110],
262 				0xFFFF << DDRMC_CR110_RDLVL_DL_1_OFF,
263 				i << DDRMC_CR110_RDLVL_DL_1_OFF);
264 
265 		/* Load values CR93[SWLVL_LOAD] to ’b1 */
266 		sw_leveling_load_value;
267 
268 		/* Poll CR94[SWLVL_OP_DONE] */
269 		sw_leveling_op_done;
270 
271 		/*
272 		 * Read Responses - SWLVL_RESP_1
273 		 *
274 		 * The 0x00 (correct response when PHY_RDLVL_EDGE = 0)
275 		 * -> 1 in the bit vector
276 		 */
277 		swlvl_rsp = (readl(&ddrmr->cr[95]) >>
278 			     DDRMC_CR95_SWLVL_RESP_1_OFF) & 0xF;
279 		if (swlvl_rsp == 0)
280 			generic_set_bit(i, rdlvl_rsp);
281 	}
282 
283 	bitmap_print(rdlvl_rsp, DDRMC_DQS_DQ_MAX_DELAY);
284 
285 	/*
286 	 * First test for rising edge 0x0 -> 0x1 in bitmap
287 	 */
288 	rdlvl_dl_1_min = ddr_cal_get_first_edge_index(rdlvl_rsp, RISING_EDGE,
289 						      N_SAMPLES, N_SAMPLES,
290 						      DDRMC_DQS_DQ_MAX_DELAY);
291 
292 	/*
293 	 * Secondly test for falling edge 0x1 -> 0x0 in bitmap
294 	 */
295 	rdlvl_dl_1_max = ddr_cal_get_first_edge_index(rdlvl_rsp, FALLING_EDGE,
296 						      N_SAMPLES, rdlvl_dl_1_min,
297 						      DDRMC_DQS_DQ_MAX_DELAY);
298 
299 	debug("RDLVL: DL_1 min: %d [0x%x] DL_1 max: %d [0x%x]\n",
300 	      rdlvl_dl_1_min, rdlvl_dl_1_min, rdlvl_dl_1_max, rdlvl_dl_1_max);
301 	rdlvl_dl_1 = (rdlvl_dl_1_max - rdlvl_dl_1_min) / 2;
302 
303 	if (rdlvl_dl_1_max == -1 || rdlvl_dl_1_min == -1 || rdlvl_dl_1 <= 0) {
304 		debug("RDLVL: The DQS to DQ delay cannot be found!\n");
305 		debug("RDLVL: Using default - slice 1: %d!\n", rdlvl_dl_1_def);
306 		rdlvl_dl_1 = rdlvl_dl_1_def;
307 	}
308 
309 	debug("RDLVL: CALIBRATED: rdlvl_dl_0: 0x%x\t rdlvl_dl_1: 0x%x\n",
310 	      rdlvl_dl_0, rdlvl_dl_1);
311 
312 	/* Write new delay values */
313 	writel(DDRMC_CR105_RDLVL_DL_0(rdlvl_dl_0), &ddrmr->cr[105]);
314 	writel(DDRMC_CR110_RDLVL_DL_1(rdlvl_dl_1), &ddrmr->cr[110]);
315 
316 	sw_leveling_load_value;
317 	sw_leveling_op_done;
318 
319 	/* Exit procedure - CR94[SWLVL_EXIT] to ’b1 */
320 	sw_leveling_exit;
321 
322 	/* Poll CR94[SWLVL_OP_DONE] */
323 	sw_leveling_op_done;
324 
325 	return 0;
326 }
327 
328 /*
329  * WRLVL_DL calibration:
330  *
331  * For non-flyback memory architecture - where one have a single DDR3 x16
332  * memory - it is NOT necessary to perform "Write Leveling"
333  * [3] 'Vybrid DDR3 write leveling' https://community.nxp.com/thread/429362
334  *
335  */
336 
337 int ddrmc_calibration(struct ddrmr_regs *ddrmr)
338 {
339 	ddrmc_cal_dqs_to_dq(ddrmr);
340 
341 	return 0;
342 }
343