xref: /openbmc/u-boot/arch/x86/cpu/quark/smc.c (revision 5381c285)
1 // SPDX-License-Identifier: Intel
2 /*
3  * Copyright (C) 2013, Intel Corporation
4  * Copyright (C) 2015, Bin Meng <bmeng.cn@gmail.com>
5  *
6  * Ported from Intel released Quark UEFI BIOS
7  * QuarkSocPkg/QuarkNorthCluster/MemoryInit/Pei
8  */
9 
10 #include <common.h>
11 #include <pci.h>
12 #include <asm/arch/device.h>
13 #include <asm/arch/mrc.h>
14 #include <asm/arch/msg_port.h>
15 #include "mrc_util.h"
16 #include "hte.h"
17 #include "smc.h"
18 
19 /* t_ck clock period in picoseconds per speed index 800, 1066, 1333 */
20 static const uint32_t t_ck[3] = {
21 	2500,
22 	1875,
23 	1500
24 };
25 
26 /* Global variables */
27 static const uint16_t ddr_wclk[] = {193, 158};
28 #ifdef BACKUP_WCTL
29 static const uint16_t ddr_wctl[] = {1, 217};
30 #endif
31 #ifdef BACKUP_WCMD
32 static const uint16_t ddr_wcmd[] = {1, 220};
33 #endif
34 
35 #ifdef BACKUP_RCVN
36 static const uint16_t ddr_rcvn[] = {129, 498};
37 #endif
38 
39 #ifdef BACKUP_WDQS
40 static const uint16_t ddr_wdqs[] = {65, 289};
41 #endif
42 
43 #ifdef BACKUP_RDQS
44 static const uint8_t ddr_rdqs[] = {32, 24};
45 #endif
46 
47 #ifdef BACKUP_WDQ
48 static const uint16_t ddr_wdq[] = {32, 257};
49 #endif
50 
51 /* Stop self refresh driven by MCU */
52 void clear_self_refresh(struct mrc_params *mrc_params)
53 {
54 	ENTERFN();
55 
56 	/* clear the PMSTS Channel Self Refresh bits */
57 	mrc_write_mask(MEM_CTLR, PMSTS, PMSTS_DISR, PMSTS_DISR);
58 
59 	LEAVEFN();
60 }
61 
62 /* It will initialize timing registers in the MCU (DTR0..DTR4) */
63 void prog_ddr_timing_control(struct mrc_params *mrc_params)
64 {
65 	uint8_t tcl, wl;
66 	uint8_t trp, trcd, tras, twr, twtr, trrd, trtp, tfaw;
67 	uint32_t tck;
68 	u32 dtr0, dtr1, dtr2, dtr3, dtr4;
69 	u32 tmp1, tmp2;
70 
71 	ENTERFN();
72 
73 	/* mcu_init starts */
74 	mrc_post_code(0x02, 0x00);
75 
76 	dtr0 = msg_port_read(MEM_CTLR, DTR0);
77 	dtr1 = msg_port_read(MEM_CTLR, DTR1);
78 	dtr2 = msg_port_read(MEM_CTLR, DTR2);
79 	dtr3 = msg_port_read(MEM_CTLR, DTR3);
80 	dtr4 = msg_port_read(MEM_CTLR, DTR4);
81 
82 	tck = t_ck[mrc_params->ddr_speed];	/* Clock in picoseconds */
83 	tcl = mrc_params->params.cl;		/* CAS latency in clocks */
84 	trp = tcl;	/* Per CAT MRC */
85 	trcd = tcl;	/* Per CAT MRC */
86 	tras = MCEIL(mrc_params->params.ras, tck);
87 
88 	/* Per JEDEC: tWR=15000ps DDR2/3 from 800-1600 */
89 	twr = MCEIL(15000, tck);
90 
91 	twtr = MCEIL(mrc_params->params.wtr, tck);
92 	trrd = MCEIL(mrc_params->params.rrd, tck);
93 	trtp = 4;	/* Valid for 800 and 1066, use 5 for 1333 */
94 	tfaw = MCEIL(mrc_params->params.faw, tck);
95 
96 	wl = 5 + mrc_params->ddr_speed;
97 
98 	dtr0 &= ~DTR0_DFREQ_MASK;
99 	dtr0 |= mrc_params->ddr_speed;
100 	dtr0 &= ~DTR0_TCL_MASK;
101 	tmp1 = tcl - 5;
102 	dtr0 |= ((tcl - 5) << 12);
103 	dtr0 &= ~DTR0_TRP_MASK;
104 	dtr0 |= ((trp - 5) << 4);	/* 5 bit DRAM Clock */
105 	dtr0 &= ~DTR0_TRCD_MASK;
106 	dtr0 |= ((trcd - 5) << 8);	/* 5 bit DRAM Clock */
107 
108 	dtr1 &= ~DTR1_TWCL_MASK;
109 	tmp2 = wl - 3;
110 	dtr1 |= (wl - 3);
111 	dtr1 &= ~DTR1_TWTP_MASK;
112 	dtr1 |= ((wl + 4 + twr - 14) << 8);	/* Change to tWTP */
113 	dtr1 &= ~DTR1_TRTP_MASK;
114 	dtr1 |= ((MMAX(trtp, 4) - 3) << 28);	/* 4 bit DRAM Clock */
115 	dtr1 &= ~DTR1_TRRD_MASK;
116 	dtr1 |= ((trrd - 4) << 24);		/* 4 bit DRAM Clock */
117 	dtr1 &= ~DTR1_TCMD_MASK;
118 	dtr1 |= (1 << 4);
119 	dtr1 &= ~DTR1_TRAS_MASK;
120 	dtr1 |= ((tras - 14) << 20);		/* 6 bit DRAM Clock */
121 	dtr1 &= ~DTR1_TFAW_MASK;
122 	dtr1 |= ((((tfaw + 1) >> 1) - 5) << 16);/* 4 bit DRAM Clock */
123 	/* Set 4 Clock CAS to CAS delay (multi-burst) */
124 	dtr1 &= ~DTR1_TCCD_MASK;
125 
126 	dtr2 &= ~DTR2_TRRDR_MASK;
127 	dtr2 |= 1;
128 	dtr2 &= ~DTR2_TWWDR_MASK;
129 	dtr2 |= (2 << 8);
130 	dtr2 &= ~DTR2_TRWDR_MASK;
131 	dtr2 |= (2 << 16);
132 
133 	dtr3 &= ~DTR3_TWRDR_MASK;
134 	dtr3 |= 2;
135 	dtr3 &= ~DTR3_TXXXX_MASK;
136 	dtr3 |= (2 << 4);
137 
138 	dtr3 &= ~DTR3_TRWSR_MASK;
139 	if (mrc_params->ddr_speed == DDRFREQ_800) {
140 		/* Extended RW delay (+1) */
141 		dtr3 |= ((tcl - 5 + 1) << 8);
142 	} else if (mrc_params->ddr_speed == DDRFREQ_1066) {
143 		/* Extended RW delay (+1) */
144 		dtr3 |= ((tcl - 5 + 1) << 8);
145 	}
146 
147 	dtr3 &= ~DTR3_TWRSR_MASK;
148 	dtr3 |= ((4 + wl + twtr - 11) << 13);
149 
150 	dtr3 &= ~DTR3_TXP_MASK;
151 	if (mrc_params->ddr_speed == DDRFREQ_800)
152 		dtr3 |= ((MMAX(0, 1 - 1)) << 22);
153 	else
154 		dtr3 |= ((MMAX(0, 2 - 1)) << 22);
155 
156 	dtr4 &= ~DTR4_WRODTSTRT_MASK;
157 	dtr4 |= 1;
158 	dtr4 &= ~DTR4_WRODTSTOP_MASK;
159 	dtr4 |= (1 << 4);
160 	dtr4 &= ~DTR4_XXXX1_MASK;
161 	dtr4 |= ((1 + tmp1 - tmp2 + 2) << 8);
162 	dtr4 &= ~DTR4_XXXX2_MASK;
163 	dtr4 |= ((1 + tmp1 - tmp2 + 2) << 12);
164 	dtr4 &= ~(DTR4_ODTDIS | DTR4_TRGSTRDIS);
165 
166 	msg_port_write(MEM_CTLR, DTR0, dtr0);
167 	msg_port_write(MEM_CTLR, DTR1, dtr1);
168 	msg_port_write(MEM_CTLR, DTR2, dtr2);
169 	msg_port_write(MEM_CTLR, DTR3, dtr3);
170 	msg_port_write(MEM_CTLR, DTR4, dtr4);
171 
172 	LEAVEFN();
173 }
174 
175 /* Configure MCU before jedec init sequence */
176 void prog_decode_before_jedec(struct mrc_params *mrc_params)
177 {
178 	u32 drp;
179 	u32 drfc;
180 	u32 dcal;
181 	u32 dsch;
182 	u32 dpmc0;
183 
184 	ENTERFN();
185 
186 	/* Disable power saving features */
187 	dpmc0 = msg_port_read(MEM_CTLR, DPMC0);
188 	dpmc0 |= (DPMC0_CLKGTDIS | DPMC0_DISPWRDN);
189 	dpmc0 &= ~DPMC0_PCLSTO_MASK;
190 	dpmc0 &= ~DPMC0_DYNSREN;
191 	msg_port_write(MEM_CTLR, DPMC0, dpmc0);
192 
193 	/* Disable out of order transactions */
194 	dsch = msg_port_read(MEM_CTLR, DSCH);
195 	dsch |= (DSCH_OOODIS | DSCH_NEWBYPDIS);
196 	msg_port_write(MEM_CTLR, DSCH, dsch);
197 
198 	/* Disable issuing the REF command */
199 	drfc = msg_port_read(MEM_CTLR, DRFC);
200 	drfc &= ~DRFC_TREFI_MASK;
201 	msg_port_write(MEM_CTLR, DRFC, drfc);
202 
203 	/* Disable ZQ calibration short */
204 	dcal = msg_port_read(MEM_CTLR, DCAL);
205 	dcal &= ~DCAL_ZQCINT_MASK;
206 	dcal &= ~DCAL_SRXZQCL_MASK;
207 	msg_port_write(MEM_CTLR, DCAL, dcal);
208 
209 	/*
210 	 * Training performed in address mode 0, rank population has limited
211 	 * impact, however simulator complains if enabled non-existing rank.
212 	 */
213 	drp = 0;
214 	if (mrc_params->rank_enables & 1)
215 		drp |= DRP_RKEN0;
216 	if (mrc_params->rank_enables & 2)
217 		drp |= DRP_RKEN1;
218 	msg_port_write(MEM_CTLR, DRP, drp);
219 
220 	LEAVEFN();
221 }
222 
223 /*
224  * After Cold Reset, BIOS should set COLDWAKE bit to 1 before
225  * sending the WAKE message to the Dunit.
226  *
227  * For Standby Exit, or any other mode in which the DRAM is in
228  * SR, this bit must be set to 0.
229  */
230 void perform_ddr_reset(struct mrc_params *mrc_params)
231 {
232 	ENTERFN();
233 
234 	/* Set COLDWAKE bit before sending the WAKE message */
235 	mrc_write_mask(MEM_CTLR, DRMC, DRMC_COLDWAKE, DRMC_COLDWAKE);
236 
237 	/* Send wake command to DUNIT (MUST be done before JEDEC) */
238 	dram_wake_command();
239 
240 	/* Set default value */
241 	msg_port_write(MEM_CTLR, DRMC,
242 		       mrc_params->rd_odt_value == 0 ? DRMC_ODTMODE : 0);
243 
244 	LEAVEFN();
245 }
246 
247 
248 /*
249  * This function performs some initialization on the DDRIO unit.
250  * This function is dependent on BOARD_ID, DDR_SPEED, and CHANNEL_ENABLES.
251  */
252 void ddrphy_init(struct mrc_params *mrc_params)
253 {
254 	uint32_t temp;
255 	uint8_t ch;	/* channel counter */
256 	uint8_t rk;	/* rank counter */
257 	uint8_t bl_grp;	/*  byte lane group counter (2 BLs per module) */
258 	uint8_t bl_divisor = 1;	/* byte lane divisor */
259 	/* For DDR3 --> 0 == 800, 1 == 1066, 2 == 1333 */
260 	uint8_t speed = mrc_params->ddr_speed & 3;
261 	uint8_t cas;
262 	uint8_t cwl;
263 
264 	ENTERFN();
265 
266 	cas = mrc_params->params.cl;
267 	cwl = 5 + mrc_params->ddr_speed;
268 
269 	/* ddrphy_init starts */
270 	mrc_post_code(0x03, 0x00);
271 
272 	/*
273 	 * HSD#231531
274 	 * Make sure IOBUFACT is deasserted before initializing the DDR PHY
275 	 *
276 	 * HSD#234845
277 	 * Make sure WRPTRENABLE is deasserted before initializing the DDR PHY
278 	 */
279 	for (ch = 0; ch < NUM_CHANNELS; ch++) {
280 		if (mrc_params->channel_enables & (1 << ch)) {
281 			/* Deassert DDRPHY Initialization Complete */
282 			mrc_alt_write_mask(DDRPHY,
283 				CMDPMCONFIG0 + ch * DDRIOCCC_CH_OFFSET,
284 				~(1 << 20), 1 << 20);	/* SPID_INIT_COMPLETE=0 */
285 			/* Deassert IOBUFACT */
286 			mrc_alt_write_mask(DDRPHY,
287 				CMDCFGREG0 + ch * DDRIOCCC_CH_OFFSET,
288 				~(1 << 2), 1 << 2);	/* IOBUFACTRST_N=0 */
289 			/* Disable WRPTR */
290 			mrc_alt_write_mask(DDRPHY,
291 				CMDPTRREG + ch * DDRIOCCC_CH_OFFSET,
292 				~(1 << 0), 1 << 0);	/* WRPTRENABLE=0 */
293 		}
294 	}
295 
296 	/* Put PHY in reset */
297 	mrc_alt_write_mask(DDRPHY, MASTERRSTN, 0, 1);
298 
299 	/* Initialize DQ01, DQ23, CMD, CLK-CTL, COMP modules */
300 
301 	/* STEP0 */
302 	mrc_post_code(0x03, 0x10);
303 	for (ch = 0; ch < NUM_CHANNELS; ch++) {
304 		if (mrc_params->channel_enables & (1 << ch)) {
305 			/* DQ01-DQ23 */
306 			for (bl_grp = 0;
307 			     bl_grp < (NUM_BYTE_LANES / bl_divisor) / 2;
308 			     bl_grp++) {
309 				/* Analog MUX select - IO2xCLKSEL */
310 				mrc_alt_write_mask(DDRPHY,
311 					DQOBSCKEBBCTL +
312 					bl_grp * DDRIODQ_BL_OFFSET +
313 					ch * DDRIODQ_CH_OFFSET,
314 					bl_grp ? 0 : (1 << 22), 1 << 22);
315 
316 				/* ODT Strength */
317 				switch (mrc_params->rd_odt_value) {
318 				case 1:
319 					temp = 0x3;
320 					break;	/* 60 ohm */
321 				case 2:
322 					temp = 0x3;
323 					break;	/* 120 ohm */
324 				case 3:
325 					temp = 0x3;
326 					break;	/* 180 ohm */
327 				default:
328 					temp = 0x3;
329 					break;	/* 120 ohm */
330 				}
331 
332 				/* ODT strength */
333 				mrc_alt_write_mask(DDRPHY,
334 					B0RXIOBUFCTL +
335 					bl_grp * DDRIODQ_BL_OFFSET +
336 					ch * DDRIODQ_CH_OFFSET,
337 					temp << 5, 0x60);
338 				/* ODT strength */
339 				mrc_alt_write_mask(DDRPHY,
340 					B1RXIOBUFCTL +
341 					bl_grp * DDRIODQ_BL_OFFSET +
342 					ch * DDRIODQ_CH_OFFSET,
343 					temp << 5, 0x60);
344 
345 				/* Dynamic ODT/DIFFAMP */
346 				temp = (cas << 24) | (cas << 16) |
347 					(cas << 8) | (cas << 0);
348 				switch (speed) {
349 				case 0:
350 					temp -= 0x01010101;
351 					break;	/* 800 */
352 				case 1:
353 					temp -= 0x02020202;
354 					break;	/* 1066 */
355 				case 2:
356 					temp -= 0x03030303;
357 					break;	/* 1333 */
358 				case 3:
359 					temp -= 0x04040404;
360 					break;	/* 1600 */
361 				}
362 
363 				/* Launch Time: ODT, DIFFAMP, ODT, DIFFAMP */
364 				mrc_alt_write_mask(DDRPHY,
365 					B01LATCTL1 +
366 					bl_grp * DDRIODQ_BL_OFFSET +
367 					ch * DDRIODQ_CH_OFFSET,
368 					temp, 0x1f1f1f1f);
369 				switch (speed) {
370 				/* HSD#234715 */
371 				case 0:
372 					temp = (0x06 << 16) | (0x07 << 8);
373 					break;	/* 800 */
374 				case 1:
375 					temp = (0x07 << 16) | (0x08 << 8);
376 					break;	/* 1066 */
377 				case 2:
378 					temp = (0x09 << 16) | (0x0a << 8);
379 					break;	/* 1333 */
380 				case 3:
381 					temp = (0x0a << 16) | (0x0b << 8);
382 					break;	/* 1600 */
383 				}
384 
385 				/* On Duration: ODT, DIFFAMP */
386 				mrc_alt_write_mask(DDRPHY,
387 					B0ONDURCTL +
388 					bl_grp * DDRIODQ_BL_OFFSET +
389 					ch * DDRIODQ_CH_OFFSET,
390 					temp, 0x003f3f00);
391 				/* On Duration: ODT, DIFFAMP */
392 				mrc_alt_write_mask(DDRPHY,
393 					B1ONDURCTL +
394 					bl_grp * DDRIODQ_BL_OFFSET +
395 					ch * DDRIODQ_CH_OFFSET,
396 					temp, 0x003f3f00);
397 
398 				switch (mrc_params->rd_odt_value) {
399 				case 0:
400 					/* override DIFFAMP=on, ODT=off */
401 					temp = (0x3f << 16) | (0x3f << 10);
402 					break;
403 				default:
404 					/* override DIFFAMP=on, ODT=on */
405 					temp = (0x3f << 16) | (0x2a << 10);
406 					break;
407 				}
408 
409 				/* Override: DIFFAMP, ODT */
410 				mrc_alt_write_mask(DDRPHY,
411 					B0OVRCTL +
412 					bl_grp * DDRIODQ_BL_OFFSET +
413 					ch * DDRIODQ_CH_OFFSET,
414 					temp, 0x003ffc00);
415 				/* Override: DIFFAMP, ODT */
416 				mrc_alt_write_mask(DDRPHY,
417 					B1OVRCTL +
418 					bl_grp * DDRIODQ_BL_OFFSET +
419 					ch * DDRIODQ_CH_OFFSET,
420 					temp, 0x003ffc00);
421 
422 				/* DLL Setup */
423 
424 				/* 1xCLK Domain Timings: tEDP,RCVEN,WDQS (PO) */
425 				mrc_alt_write_mask(DDRPHY,
426 					B0LATCTL0 +
427 					bl_grp * DDRIODQ_BL_OFFSET +
428 					ch * DDRIODQ_CH_OFFSET,
429 					((cas + 7) << 16) | ((cas - 4) << 8) |
430 					((cwl - 2) << 0), 0x003f1f1f);
431 				mrc_alt_write_mask(DDRPHY,
432 					B1LATCTL0 +
433 					bl_grp * DDRIODQ_BL_OFFSET +
434 					ch * DDRIODQ_CH_OFFSET,
435 					((cas + 7) << 16) | ((cas - 4) << 8) |
436 					((cwl - 2) << 0), 0x003f1f1f);
437 
438 				/* RCVEN Bypass (PO) */
439 				mrc_alt_write_mask(DDRPHY,
440 					B0RXIOBUFCTL +
441 					bl_grp * DDRIODQ_BL_OFFSET +
442 					ch * DDRIODQ_CH_OFFSET,
443 					0, 0x81);
444 				mrc_alt_write_mask(DDRPHY,
445 					B1RXIOBUFCTL +
446 					bl_grp * DDRIODQ_BL_OFFSET +
447 					ch * DDRIODQ_CH_OFFSET,
448 					0, 0x81);
449 
450 				/* TX */
451 				mrc_alt_write_mask(DDRPHY,
452 					DQCTL +
453 					bl_grp * DDRIODQ_BL_OFFSET +
454 					ch * DDRIODQ_CH_OFFSET,
455 					1 << 16, 1 << 16);
456 				mrc_alt_write_mask(DDRPHY,
457 					B01PTRCTL1 +
458 					bl_grp * DDRIODQ_BL_OFFSET +
459 					ch * DDRIODQ_CH_OFFSET,
460 					1 << 8, 1 << 8);
461 
462 				/* RX (PO) */
463 				/* Internal Vref Code, Enable#, Ext_or_Int (1=Ext) */
464 				mrc_alt_write_mask(DDRPHY,
465 					B0VREFCTL +
466 					bl_grp * DDRIODQ_BL_OFFSET +
467 					ch * DDRIODQ_CH_OFFSET,
468 					(0x03 << 2) | (0x0 << 1) | (0x0 << 0),
469 					0xff);
470 				/* Internal Vref Code, Enable#, Ext_or_Int (1=Ext) */
471 				mrc_alt_write_mask(DDRPHY,
472 					B1VREFCTL +
473 					bl_grp * DDRIODQ_BL_OFFSET +
474 					ch * DDRIODQ_CH_OFFSET,
475 					(0x03 << 2) | (0x0 << 1) | (0x0 << 0),
476 					0xff);
477 				/* Per-Bit De-Skew Enable */
478 				mrc_alt_write_mask(DDRPHY,
479 					B0RXIOBUFCTL +
480 					bl_grp * DDRIODQ_BL_OFFSET +
481 					ch * DDRIODQ_CH_OFFSET,
482 					0, 0x10);
483 				/* Per-Bit De-Skew Enable */
484 				mrc_alt_write_mask(DDRPHY,
485 					B1RXIOBUFCTL +
486 					bl_grp * DDRIODQ_BL_OFFSET +
487 					ch * DDRIODQ_CH_OFFSET,
488 					0, 0x10);
489 			}
490 
491 			/* CLKEBB */
492 			mrc_alt_write_mask(DDRPHY,
493 				CMDOBSCKEBBCTL + ch * DDRIOCCC_CH_OFFSET,
494 				0, 1 << 23);
495 
496 			/* Enable tristate control of cmd/address bus */
497 			mrc_alt_write_mask(DDRPHY,
498 				CMDCFGREG0 + ch * DDRIOCCC_CH_OFFSET,
499 				0, 0x03);
500 
501 			/* ODT RCOMP */
502 			mrc_alt_write_mask(DDRPHY,
503 				CMDRCOMPODT + ch * DDRIOCCC_CH_OFFSET,
504 				(0x03 << 5) | (0x03 << 0), 0x3ff);
505 
506 			/* CMDPM* registers must be programmed in this order */
507 
508 			/* Turn On Delays: SFR (regulator), MPLL */
509 			mrc_alt_write_mask(DDRPHY,
510 				CMDPMDLYREG4 + ch * DDRIOCCC_CH_OFFSET,
511 				0xffffffff, 0xffffffff);
512 			/*
513 			 * Delays: ASSERT_IOBUFACT_to_ALLON0_for_PM_MSG_3,
514 			 * VREG (MDLL) Turn On, ALLON0_to_DEASSERT_IOBUFACT
515 			 * for_PM_MSG_gt0, MDLL Turn On
516 			 */
517 			mrc_alt_write_mask(DDRPHY,
518 				CMDPMDLYREG3 + ch * DDRIOCCC_CH_OFFSET,
519 				0xfffff616, 0xffffffff);
520 			/* MPLL Divider Reset Delays */
521 			mrc_alt_write_mask(DDRPHY,
522 				CMDPMDLYREG2 + ch * DDRIOCCC_CH_OFFSET,
523 				0xffffffff, 0xffffffff);
524 			/* Turn Off Delays: VREG, Staggered MDLL, MDLL, PI */
525 			mrc_alt_write_mask(DDRPHY,
526 				CMDPMDLYREG1 + ch * DDRIOCCC_CH_OFFSET,
527 				0xffffffff, 0xffffffff);
528 			/* Turn On Delays: MPLL, Staggered MDLL, PI, IOBUFACT */
529 			mrc_alt_write_mask(DDRPHY,
530 				CMDPMDLYREG0 + ch * DDRIOCCC_CH_OFFSET,
531 				0xffffffff, 0xffffffff);
532 			/* Allow PUnit signals */
533 			mrc_alt_write_mask(DDRPHY,
534 				CMDPMCONFIG0 + ch * DDRIOCCC_CH_OFFSET,
535 				(0x6 << 8) | (0x1 << 6) | (0x4 << 0),
536 				0xffe00f4f);
537 			/* DLL_VREG Bias Trim, VREF Tuning for DLL_VREG */
538 			mrc_alt_write_mask(DDRPHY,
539 				CMDMDLLCTL + ch * DDRIOCCC_CH_OFFSET,
540 				(0x3 << 4) | (0x7 << 0), 0x7f);
541 
542 			/* CLK-CTL */
543 			mrc_alt_write_mask(DDRPHY,
544 				CCOBSCKEBBCTL + ch * DDRIOCCC_CH_OFFSET,
545 				0, 1 << 24);	/* CLKEBB */
546 			/* Buffer Enable: CS,CKE,ODT,CLK */
547 			mrc_alt_write_mask(DDRPHY,
548 				CCCFGREG0 + ch * DDRIOCCC_CH_OFFSET,
549 				0x1f, 0x000ffff1);
550 			/* ODT RCOMP */
551 			mrc_alt_write_mask(DDRPHY,
552 				CCRCOMPODT + ch * DDRIOCCC_CH_OFFSET,
553 				(0x03 << 8) | (0x03 << 0), 0x00001f1f);
554 			/* DLL_VREG Bias Trim, VREF Tuning for DLL_VREG */
555 			mrc_alt_write_mask(DDRPHY,
556 				CCMDLLCTL + ch * DDRIOCCC_CH_OFFSET,
557 				(0x3 << 4) | (0x7 << 0), 0x7f);
558 
559 			/*
560 			 * COMP (RON channel specific)
561 			 * - DQ/DQS/DM RON: 32 Ohm
562 			 * - CTRL/CMD RON: 27 Ohm
563 			 * - CLK RON: 26 Ohm
564 			 */
565 			/* RCOMP Vref PU/PD */
566 			mrc_alt_write_mask(DDRPHY,
567 				DQVREFCH0 +  ch * DDRCOMP_CH_OFFSET,
568 				(0x08 << 24) | (0x03 << 16), 0x3f3f0000);
569 			/* RCOMP Vref PU/PD */
570 			mrc_alt_write_mask(DDRPHY,
571 				CMDVREFCH0 + ch * DDRCOMP_CH_OFFSET,
572 				(0x0C << 24) | (0x03 << 16), 0x3f3f0000);
573 			/* RCOMP Vref PU/PD */
574 			mrc_alt_write_mask(DDRPHY,
575 				CLKVREFCH0 + ch * DDRCOMP_CH_OFFSET,
576 				(0x0F << 24) | (0x03 << 16), 0x3f3f0000);
577 			/* RCOMP Vref PU/PD */
578 			mrc_alt_write_mask(DDRPHY,
579 				DQSVREFCH0 + ch * DDRCOMP_CH_OFFSET,
580 				(0x08 << 24) | (0x03 << 16), 0x3f3f0000);
581 			/* RCOMP Vref PU/PD */
582 			mrc_alt_write_mask(DDRPHY,
583 				CTLVREFCH0 + ch * DDRCOMP_CH_OFFSET,
584 				(0x0C << 24) | (0x03 << 16), 0x3f3f0000);
585 
586 			/* DQS Swapped Input Enable */
587 			mrc_alt_write_mask(DDRPHY,
588 				COMPEN1CH0 + ch * DDRCOMP_CH_OFFSET,
589 				(1 << 19) | (1 << 17), 0xc00ac000);
590 
591 			/* ODT VREF = 1.5 x 274/360+274 = 0.65V (code of ~50) */
592 			/* ODT Vref PU/PD */
593 			mrc_alt_write_mask(DDRPHY,
594 				DQVREFCH0 + ch * DDRCOMP_CH_OFFSET,
595 				(0x32 << 8) | (0x03 << 0), 0x00003f3f);
596 			/* ODT Vref PU/PD */
597 			mrc_alt_write_mask(DDRPHY,
598 				DQSVREFCH0 + ch * DDRCOMP_CH_OFFSET,
599 				(0x32 << 8) | (0x03 << 0), 0x00003f3f);
600 			/* ODT Vref PU/PD */
601 			mrc_alt_write_mask(DDRPHY,
602 				CLKVREFCH0 + ch * DDRCOMP_CH_OFFSET,
603 				(0x0E << 8) | (0x05 << 0), 0x00003f3f);
604 
605 			/*
606 			 * Slew rate settings are frequency specific,
607 			 * numbers below are for 800Mhz (speed == 0)
608 			 * - DQ/DQS/DM/CLK SR: 4V/ns,
609 			 * - CTRL/CMD SR: 1.5V/ns
610 			 */
611 			temp = (0x0e << 16) | (0x0e << 12) | (0x08 << 8) |
612 				(0x0b << 4) | (0x0b << 0);
613 			/* DCOMP Delay Select: CTL,CMD,CLK,DQS,DQ */
614 			mrc_alt_write_mask(DDRPHY,
615 				DLYSELCH0 + ch * DDRCOMP_CH_OFFSET,
616 				temp, 0x000fffff);
617 			/* TCO Vref CLK,DQS,DQ */
618 			mrc_alt_write_mask(DDRPHY,
619 				TCOVREFCH0 + ch * DDRCOMP_CH_OFFSET,
620 				(0x05 << 16) | (0x05 << 8) | (0x05 << 0),
621 				0x003f3f3f);
622 			/* ODTCOMP CMD/CTL PU/PD */
623 			mrc_alt_write_mask(DDRPHY,
624 				CCBUFODTCH0 + ch * DDRCOMP_CH_OFFSET,
625 				(0x03 << 8) | (0x03 << 0),
626 				0x00001f1f);
627 			/* COMP */
628 			mrc_alt_write_mask(DDRPHY,
629 				COMPEN0CH0 + ch * DDRCOMP_CH_OFFSET,
630 				0, 0xc0000100);
631 
632 #ifdef BACKUP_COMPS
633 			/* DQ COMP Overrides */
634 			/* RCOMP PU */
635 			mrc_alt_write_mask(DDRPHY,
636 				DQDRVPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
637 				(1 << 31) | (0x0a << 16),
638 				0x801f0000);
639 			/* RCOMP PD */
640 			mrc_alt_write_mask(DDRPHY,
641 				DQDRVPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
642 				(1 << 31) | (0x0a << 16),
643 				0x801f0000);
644 			/* DCOMP PU */
645 			mrc_alt_write_mask(DDRPHY,
646 				DQDLYPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
647 				(1 << 31) | (0x10 << 16),
648 				0x801f0000);
649 			/* DCOMP PD */
650 			mrc_alt_write_mask(DDRPHY,
651 				DQDLYPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
652 				(1 << 31) | (0x10 << 16),
653 				0x801f0000);
654 			/* ODTCOMP PU */
655 			mrc_alt_write_mask(DDRPHY,
656 				DQODTPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
657 				(1 << 31) | (0x0b << 16),
658 				0x801f0000);
659 			/* ODTCOMP PD */
660 			mrc_alt_write_mask(DDRPHY,
661 				DQODTPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
662 				(1 << 31) | (0x0b << 16),
663 				0x801f0000);
664 			/* TCOCOMP PU */
665 			mrc_alt_write_mask(DDRPHY,
666 				DQTCOPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
667 				1 << 31, 1 << 31);
668 			/* TCOCOMP PD */
669 			mrc_alt_write_mask(DDRPHY,
670 				DQTCOPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
671 				1 << 31, 1 << 31);
672 
673 			/* DQS COMP Overrides */
674 			/* RCOMP PU */
675 			mrc_alt_write_mask(DDRPHY,
676 				DQSDRVPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
677 				(1 << 31) | (0x0a << 16),
678 				0x801f0000);
679 			/* RCOMP PD */
680 			mrc_alt_write_mask(DDRPHY,
681 				DQSDRVPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
682 				(1 << 31) | (0x0a << 16),
683 				0x801f0000);
684 			/* DCOMP PU */
685 			mrc_alt_write_mask(DDRPHY,
686 				DQSDLYPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
687 				(1 << 31) | (0x10 << 16),
688 				0x801f0000);
689 			/* DCOMP PD */
690 			mrc_alt_write_mask(DDRPHY,
691 				DQSDLYPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
692 				(1 << 31) | (0x10 << 16),
693 				0x801f0000);
694 			/* ODTCOMP PU */
695 			mrc_alt_write_mask(DDRPHY,
696 				DQSODTPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
697 				(1 << 31) | (0x0b << 16),
698 				0x801f0000);
699 			/* ODTCOMP PD */
700 			mrc_alt_write_mask(DDRPHY,
701 				DQSODTPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
702 				(1 << 31) | (0x0b << 16),
703 				0x801f0000);
704 			/* TCOCOMP PU */
705 			mrc_alt_write_mask(DDRPHY,
706 				DQSTCOPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
707 				1 << 31, 1 << 31);
708 			/* TCOCOMP PD */
709 			mrc_alt_write_mask(DDRPHY,
710 				DQSTCOPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
711 				1 << 31, 1 << 31);
712 
713 			/* CLK COMP Overrides */
714 			/* RCOMP PU */
715 			mrc_alt_write_mask(DDRPHY,
716 				CLKDRVPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
717 				(1 << 31) | (0x0c << 16),
718 				0x801f0000);
719 			/* RCOMP PD */
720 			mrc_alt_write_mask(DDRPHY,
721 				CLKDRVPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
722 				(1 << 31) | (0x0c << 16),
723 				0x801f0000);
724 			/* DCOMP PU */
725 			mrc_alt_write_mask(DDRPHY,
726 				CLKDLYPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
727 				(1 << 31) | (0x07 << 16),
728 				0x801f0000);
729 			/* DCOMP PD */
730 			mrc_alt_write_mask(DDRPHY,
731 				CLKDLYPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
732 				(1 << 31) | (0x07 << 16),
733 				0x801f0000);
734 			/* ODTCOMP PU */
735 			mrc_alt_write_mask(DDRPHY,
736 				CLKODTPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
737 				(1 << 31) | (0x0b << 16),
738 				0x801f0000);
739 			/* ODTCOMP PD */
740 			mrc_alt_write_mask(DDRPHY,
741 				CLKODTPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
742 				(1 << 31) | (0x0b << 16),
743 				0x801f0000);
744 			/* TCOCOMP PU */
745 			mrc_alt_write_mask(DDRPHY,
746 				CLKTCOPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
747 				1 << 31, 1 << 31);
748 			/* TCOCOMP PD */
749 			mrc_alt_write_mask(DDRPHY,
750 				CLKTCOPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
751 				1 << 31, 1 << 31);
752 
753 			/* CMD COMP Overrides */
754 			/* RCOMP PU */
755 			mrc_alt_write_mask(DDRPHY,
756 				CMDDRVPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
757 				(1 << 31) | (0x0d << 16),
758 				0x803f0000);
759 			/* RCOMP PD */
760 			mrc_alt_write_mask(DDRPHY,
761 				CMDDRVPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
762 				(1 << 31) | (0x0d << 16),
763 				0x803f0000);
764 			/* DCOMP PU */
765 			mrc_alt_write_mask(DDRPHY,
766 				CMDDLYPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
767 				(1 << 31) | (0x0a << 16),
768 				0x801f0000);
769 			/* DCOMP PD */
770 			mrc_alt_write_mask(DDRPHY,
771 				CMDDLYPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
772 				(1 << 31) | (0x0a << 16),
773 				0x801f0000);
774 
775 			/* CTL COMP Overrides */
776 			/* RCOMP PU */
777 			mrc_alt_write_mask(DDRPHY,
778 				CTLDRVPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
779 				(1 << 31) | (0x0d << 16),
780 				0x803f0000);
781 			/* RCOMP PD */
782 			mrc_alt_write_mask(DDRPHY,
783 				CTLDRVPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
784 				(1 << 31) | (0x0d << 16),
785 				0x803f0000);
786 			/* DCOMP PU */
787 			mrc_alt_write_mask(DDRPHY,
788 				CTLDLYPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
789 				(1 << 31) | (0x0a << 16),
790 				0x801f0000);
791 			/* DCOMP PD */
792 			mrc_alt_write_mask(DDRPHY,
793 				CTLDLYPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
794 				(1 << 31) | (0x0a << 16),
795 				0x801f0000);
796 #else
797 			/* DQ TCOCOMP Overrides */
798 			/* TCOCOMP PU */
799 			mrc_alt_write_mask(DDRPHY,
800 				DQTCOPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
801 				(1 << 31) | (0x1f << 16),
802 				0x801f0000);
803 			/* TCOCOMP PD */
804 			mrc_alt_write_mask(DDRPHY,
805 				DQTCOPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
806 				(1 << 31) | (0x1f << 16),
807 				0x801f0000);
808 
809 			/* DQS TCOCOMP Overrides */
810 			/* TCOCOMP PU */
811 			mrc_alt_write_mask(DDRPHY,
812 				DQSTCOPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
813 				(1 << 31) | (0x1f << 16),
814 				0x801f0000);
815 			/* TCOCOMP PD */
816 			mrc_alt_write_mask(DDRPHY,
817 				DQSTCOPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
818 				(1 << 31) | (0x1f << 16),
819 				0x801f0000);
820 
821 			/* CLK TCOCOMP Overrides */
822 			/* TCOCOMP PU */
823 			mrc_alt_write_mask(DDRPHY,
824 				CLKTCOPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
825 				(1 << 31) | (0x1f << 16),
826 				0x801f0000);
827 			/* TCOCOMP PD */
828 			mrc_alt_write_mask(DDRPHY,
829 				CLKTCOPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
830 				(1 << 31) | (0x1f << 16),
831 				0x801f0000);
832 #endif
833 
834 			/* program STATIC delays */
835 #ifdef BACKUP_WCMD
836 			set_wcmd(ch, ddr_wcmd[PLATFORM_ID]);
837 #else
838 			set_wcmd(ch, ddr_wclk[PLATFORM_ID] + HALF_CLK);
839 #endif
840 
841 			for (rk = 0; rk < NUM_RANKS; rk++) {
842 				if (mrc_params->rank_enables & (1 << rk)) {
843 					set_wclk(ch, rk, ddr_wclk[PLATFORM_ID]);
844 #ifdef BACKUP_WCTL
845 					set_wctl(ch, rk, ddr_wctl[PLATFORM_ID]);
846 #else
847 					set_wctl(ch, rk, ddr_wclk[PLATFORM_ID] + HALF_CLK);
848 #endif
849 				}
850 			}
851 		}
852 	}
853 
854 	/* COMP (non channel specific) */
855 	/* RCOMP: Dither PU Enable */
856 	mrc_alt_write_mask(DDRPHY, DQANADRVPUCTL, 1 << 30, 1 << 30);
857 	/* RCOMP: Dither PD Enable */
858 	mrc_alt_write_mask(DDRPHY, DQANADRVPDCTL, 1 << 30, 1 << 30);
859 	/* RCOMP: Dither PU Enable */
860 	mrc_alt_write_mask(DDRPHY, CMDANADRVPUCTL, 1 << 30, 1 << 30);
861 	/* RCOMP: Dither PD Enable */
862 	mrc_alt_write_mask(DDRPHY, CMDANADRVPDCTL, 1 << 30, 1 << 30);
863 	/* RCOMP: Dither PU Enable */
864 	mrc_alt_write_mask(DDRPHY, CLKANADRVPUCTL, 1 << 30, 1 << 30);
865 	/* RCOMP: Dither PD Enable */
866 	mrc_alt_write_mask(DDRPHY, CLKANADRVPDCTL, 1 << 30, 1 << 30);
867 	/* RCOMP: Dither PU Enable */
868 	mrc_alt_write_mask(DDRPHY, DQSANADRVPUCTL, 1 << 30, 1 << 30);
869 	/* RCOMP: Dither PD Enable */
870 	mrc_alt_write_mask(DDRPHY, DQSANADRVPDCTL, 1 << 30, 1 << 30);
871 	/* RCOMP: Dither PU Enable */
872 	mrc_alt_write_mask(DDRPHY, CTLANADRVPUCTL, 1 << 30, 1 << 30);
873 	/* RCOMP: Dither PD Enable */
874 	mrc_alt_write_mask(DDRPHY, CTLANADRVPDCTL, 1 << 30, 1 << 30);
875 	/* ODT: Dither PU Enable */
876 	mrc_alt_write_mask(DDRPHY, DQANAODTPUCTL, 1 << 30, 1 << 30);
877 	/* ODT: Dither PD Enable */
878 	mrc_alt_write_mask(DDRPHY, DQANAODTPDCTL, 1 << 30, 1 << 30);
879 	/* ODT: Dither PU Enable */
880 	mrc_alt_write_mask(DDRPHY, CLKANAODTPUCTL, 1 << 30, 1 << 30);
881 	/* ODT: Dither PD Enable */
882 	mrc_alt_write_mask(DDRPHY, CLKANAODTPDCTL, 1 << 30, 1 << 30);
883 	/* ODT: Dither PU Enable */
884 	mrc_alt_write_mask(DDRPHY, DQSANAODTPUCTL, 1 << 30, 1 << 30);
885 	/* ODT: Dither PD Enable */
886 	mrc_alt_write_mask(DDRPHY, DQSANAODTPDCTL, 1 << 30, 1 << 30);
887 	/* DCOMP: Dither PU Enable */
888 	mrc_alt_write_mask(DDRPHY, DQANADLYPUCTL, 1 << 30, 1 << 30);
889 	/* DCOMP: Dither PD Enable */
890 	mrc_alt_write_mask(DDRPHY, DQANADLYPDCTL, 1 << 30, 1 << 30);
891 	/* DCOMP: Dither PU Enable */
892 	mrc_alt_write_mask(DDRPHY, CMDANADLYPUCTL, 1 << 30, 1 << 30);
893 	/* DCOMP: Dither PD Enable */
894 	mrc_alt_write_mask(DDRPHY, CMDANADLYPDCTL, 1 << 30, 1 << 30);
895 	/* DCOMP: Dither PU Enable */
896 	mrc_alt_write_mask(DDRPHY, CLKANADLYPUCTL, 1 << 30, 1 << 30);
897 	/* DCOMP: Dither PD Enable */
898 	mrc_alt_write_mask(DDRPHY, CLKANADLYPDCTL, 1 << 30, 1 << 30);
899 	/* DCOMP: Dither PU Enable */
900 	mrc_alt_write_mask(DDRPHY, DQSANADLYPUCTL, 1 << 30, 1 << 30);
901 	/* DCOMP: Dither PD Enable */
902 	mrc_alt_write_mask(DDRPHY, DQSANADLYPDCTL, 1 << 30, 1 << 30);
903 	/* DCOMP: Dither PU Enable */
904 	mrc_alt_write_mask(DDRPHY, CTLANADLYPUCTL, 1 << 30, 1 << 30);
905 	/* DCOMP: Dither PD Enable */
906 	mrc_alt_write_mask(DDRPHY, CTLANADLYPDCTL, 1 << 30, 1 << 30);
907 	/* TCO: Dither PU Enable */
908 	mrc_alt_write_mask(DDRPHY, DQANATCOPUCTL, 1 << 30, 1 << 30);
909 	/* TCO: Dither PD Enable */
910 	mrc_alt_write_mask(DDRPHY, DQANATCOPDCTL, 1 << 30, 1 << 30);
911 	/* TCO: Dither PU Enable */
912 	mrc_alt_write_mask(DDRPHY, CLKANATCOPUCTL, 1 << 30, 1 << 30);
913 	/* TCO: Dither PD Enable */
914 	mrc_alt_write_mask(DDRPHY, CLKANATCOPDCTL, 1 << 30, 1 << 30);
915 	/* TCO: Dither PU Enable */
916 	mrc_alt_write_mask(DDRPHY, DQSANATCOPUCTL, 1 << 30, 1 << 30);
917 	/* TCO: Dither PD Enable */
918 	mrc_alt_write_mask(DDRPHY, DQSANATCOPDCTL, 1 << 30, 1 << 30);
919 	/* TCOCOMP: Pulse Count */
920 	mrc_alt_write_mask(DDRPHY, TCOCNTCTRL, 1, 3);
921 	/* ODT: CMD/CTL PD/PU */
922 	mrc_alt_write_mask(DDRPHY, CHNLBUFSTATIC,
923 		(0x03 << 24) | (0x03 << 16), 0x1f1f0000);
924 	/* Set 1us counter */
925 	mrc_alt_write_mask(DDRPHY, MSCNTR, 0x64, 0xff);
926 	mrc_alt_write_mask(DDRPHY, LATCH1CTL, 0x1 << 28, 0x70000000);
927 
928 	/* Release PHY from reset */
929 	mrc_alt_write_mask(DDRPHY, MASTERRSTN, 1, 1);
930 
931 	/* STEP1 */
932 	mrc_post_code(0x03, 0x11);
933 
934 	for (ch = 0; ch < NUM_CHANNELS; ch++) {
935 		if (mrc_params->channel_enables & (1 << ch)) {
936 			/* DQ01-DQ23 */
937 			for (bl_grp = 0;
938 			     bl_grp < (NUM_BYTE_LANES / bl_divisor) / 2;
939 			     bl_grp++) {
940 				mrc_alt_write_mask(DDRPHY,
941 					DQMDLLCTL +
942 					bl_grp * DDRIODQ_BL_OFFSET +
943 					ch * DDRIODQ_CH_OFFSET,
944 					1 << 13,
945 					1 << 13);	/* Enable VREG */
946 				delay_n(3);
947 			}
948 
949 			/* ECC */
950 			mrc_alt_write_mask(DDRPHY, ECCMDLLCTL,
951 				1 << 13, 1 << 13);	/* Enable VREG */
952 			delay_n(3);
953 			/* CMD */
954 			mrc_alt_write_mask(DDRPHY,
955 				CMDMDLLCTL + ch * DDRIOCCC_CH_OFFSET,
956 				1 << 13, 1 << 13);	/* Enable VREG */
957 			delay_n(3);
958 			/* CLK-CTL */
959 			mrc_alt_write_mask(DDRPHY,
960 				CCMDLLCTL + ch * DDRIOCCC_CH_OFFSET,
961 				1 << 13, 1 << 13);	/* Enable VREG */
962 			delay_n(3);
963 		}
964 	}
965 
966 	/* STEP2 */
967 	mrc_post_code(0x03, 0x12);
968 	delay_n(200);
969 
970 	for (ch = 0; ch < NUM_CHANNELS; ch++) {
971 		if (mrc_params->channel_enables & (1 << ch)) {
972 			/* DQ01-DQ23 */
973 			for (bl_grp = 0;
974 			     bl_grp < (NUM_BYTE_LANES / bl_divisor) / 2;
975 			     bl_grp++) {
976 				mrc_alt_write_mask(DDRPHY,
977 					DQMDLLCTL +
978 					bl_grp * DDRIODQ_BL_OFFSET +
979 					ch * DDRIODQ_CH_OFFSET,
980 					1 << 17,
981 					1 << 17);	/* Enable MCDLL */
982 				delay_n(50);
983 			}
984 
985 		/* ECC */
986 		mrc_alt_write_mask(DDRPHY, ECCMDLLCTL,
987 			1 << 17, 1 << 17);	/* Enable MCDLL */
988 		delay_n(50);
989 		/* CMD */
990 		mrc_alt_write_mask(DDRPHY,
991 			CMDMDLLCTL + ch * DDRIOCCC_CH_OFFSET,
992 			1 << 18, 1 << 18);	/* Enable MCDLL */
993 		delay_n(50);
994 		/* CLK-CTL */
995 		mrc_alt_write_mask(DDRPHY,
996 			CCMDLLCTL + ch * DDRIOCCC_CH_OFFSET,
997 			1 << 18, 1 << 18);	/* Enable MCDLL */
998 		delay_n(50);
999 		}
1000 	}
1001 
1002 	/* STEP3: */
1003 	mrc_post_code(0x03, 0x13);
1004 	delay_n(100);
1005 
1006 	for (ch = 0; ch < NUM_CHANNELS; ch++) {
1007 		if (mrc_params->channel_enables & (1 << ch)) {
1008 			/* DQ01-DQ23 */
1009 			for (bl_grp = 0;
1010 			     bl_grp < (NUM_BYTE_LANES / bl_divisor) / 2;
1011 			     bl_grp++) {
1012 #ifdef FORCE_16BIT_DDRIO
1013 				temp = (bl_grp &&
1014 					(mrc_params->channel_width == X16)) ?
1015 					0x11ff : 0xffff;
1016 #else
1017 				temp = 0xffff;
1018 #endif
1019 				/* Enable TXDLL */
1020 				mrc_alt_write_mask(DDRPHY,
1021 					DQDLLTXCTL +
1022 					bl_grp * DDRIODQ_BL_OFFSET +
1023 					ch * DDRIODQ_CH_OFFSET,
1024 					temp, 0xffff);
1025 				delay_n(3);
1026 				/* Enable RXDLL */
1027 				mrc_alt_write_mask(DDRPHY,
1028 					DQDLLRXCTL +
1029 					bl_grp * DDRIODQ_BL_OFFSET +
1030 					ch * DDRIODQ_CH_OFFSET,
1031 					0xf, 0xf);
1032 				delay_n(3);
1033 				/* Enable RXDLL Overrides BL0 */
1034 				mrc_alt_write_mask(DDRPHY,
1035 					B0OVRCTL +
1036 					bl_grp * DDRIODQ_BL_OFFSET +
1037 					ch * DDRIODQ_CH_OFFSET,
1038 					0xf, 0xf);
1039 			}
1040 
1041 			/* ECC */
1042 			temp = 0xffff;
1043 			mrc_alt_write_mask(DDRPHY, ECCDLLTXCTL,
1044 				temp, 0xffff);
1045 			delay_n(3);
1046 
1047 			/* CMD (PO) */
1048 			mrc_alt_write_mask(DDRPHY,
1049 				CMDDLLTXCTL + ch * DDRIOCCC_CH_OFFSET,
1050 				temp, 0xffff);
1051 			delay_n(3);
1052 		}
1053 	}
1054 
1055 	/* STEP4 */
1056 	mrc_post_code(0x03, 0x14);
1057 
1058 	for (ch = 0; ch < NUM_CHANNELS; ch++) {
1059 		if (mrc_params->channel_enables & (1 << ch)) {
1060 			/* Host To Memory Clock Alignment (HMC) for 800/1066 */
1061 			for (bl_grp = 0;
1062 			     bl_grp < (NUM_BYTE_LANES / bl_divisor) / 2;
1063 			     bl_grp++) {
1064 				/* CLK_ALIGN_MOD_ID */
1065 				mrc_alt_write_mask(DDRPHY,
1066 					DQCLKALIGNREG2 +
1067 					bl_grp * DDRIODQ_BL_OFFSET +
1068 					ch * DDRIODQ_CH_OFFSET,
1069 					bl_grp ? 3 : 1,
1070 					0xf);
1071 			}
1072 
1073 			mrc_alt_write_mask(DDRPHY,
1074 				ECCCLKALIGNREG2 + ch * DDRIODQ_CH_OFFSET,
1075 				0x2, 0xf);
1076 			mrc_alt_write_mask(DDRPHY,
1077 				CMDCLKALIGNREG2 + ch * DDRIODQ_CH_OFFSET,
1078 				0x0, 0xf);
1079 			mrc_alt_write_mask(DDRPHY,
1080 				CCCLKALIGNREG2 + ch * DDRIODQ_CH_OFFSET,
1081 				0x2, 0xf);
1082 			mrc_alt_write_mask(DDRPHY,
1083 				CMDCLKALIGNREG0 + ch * DDRIOCCC_CH_OFFSET,
1084 				0x20, 0x30);
1085 			/*
1086 			 * NUM_SAMPLES, MAX_SAMPLES,
1087 			 * MACRO_PI_STEP, MICRO_PI_STEP
1088 			 */
1089 			mrc_alt_write_mask(DDRPHY,
1090 				CMDCLKALIGNREG1 + ch * DDRIOCCC_CH_OFFSET,
1091 				(0x18 << 16) | (0x10 << 8) |
1092 				(0x8 << 2) | (0x1 << 0),
1093 				0x007f7fff);
1094 			/* TOTAL_NUM_MODULES, FIRST_U_PARTITION */
1095 			mrc_alt_write_mask(DDRPHY,
1096 				CMDCLKALIGNREG2 + ch * DDRIOCCC_CH_OFFSET,
1097 				(0x10 << 16) | (0x4 << 8) | (0x2 << 4),
1098 				0x001f0ff0);
1099 #ifdef HMC_TEST
1100 			/* START_CLK_ALIGN=1 */
1101 			mrc_alt_write_mask(DDRPHY,
1102 				CMDCLKALIGNREG0 + ch * DDRIOCCC_CH_OFFSET,
1103 				1 << 24, 1 << 24);
1104 			while (msg_port_alt_read(DDRPHY,
1105 				CMDCLKALIGNREG0 + ch * DDRIOCCC_CH_OFFSET) &
1106 				(1 << 24))
1107 				;	/* wait for START_CLK_ALIGN=0 */
1108 #endif
1109 
1110 			/* Set RD/WR Pointer Seperation & COUNTEN & FIFOPTREN */
1111 			mrc_alt_write_mask(DDRPHY,
1112 				CMDPTRREG + ch * DDRIOCCC_CH_OFFSET,
1113 				1, 1);	/* WRPTRENABLE=1 */
1114 
1115 			/* COMP initial */
1116 			/* enable bypass for CLK buffer (PO) */
1117 			mrc_alt_write_mask(DDRPHY,
1118 				COMPEN0CH0 + ch * DDRCOMP_CH_OFFSET,
1119 				1 << 5, 1 << 5);
1120 			/* Initial COMP Enable */
1121 			mrc_alt_write_mask(DDRPHY, CMPCTRL, 1, 1);
1122 			/* wait for Initial COMP Enable = 0 */
1123 			while (msg_port_alt_read(DDRPHY, CMPCTRL) & 1)
1124 				;
1125 			/* disable bypass for CLK buffer (PO) */
1126 			mrc_alt_write_mask(DDRPHY,
1127 				COMPEN0CH0 + ch * DDRCOMP_CH_OFFSET,
1128 				~(1 << 5), 1 << 5);
1129 
1130 			/* IOBUFACT */
1131 
1132 			/* STEP4a */
1133 			mrc_alt_write_mask(DDRPHY,
1134 				CMDCFGREG0 + ch * DDRIOCCC_CH_OFFSET,
1135 				1 << 2, 1 << 2);	/* IOBUFACTRST_N=1 */
1136 
1137 			/* DDRPHY initialization complete */
1138 			mrc_alt_write_mask(DDRPHY,
1139 				CMDPMCONFIG0 + ch * DDRIOCCC_CH_OFFSET,
1140 				1 << 20, 1 << 20);	/* SPID_INIT_COMPLETE=1 */
1141 		}
1142 	}
1143 
1144 	LEAVEFN();
1145 }
1146 
1147 /* This function performs JEDEC initialization on all enabled channels */
1148 void perform_jedec_init(struct mrc_params *mrc_params)
1149 {
1150 	uint8_t twr, wl, rank;
1151 	uint32_t tck;
1152 	u32 dtr0;
1153 	u32 drp;
1154 	u32 drmc;
1155 	u32 mrs0_cmd = 0;
1156 	u32 emrs1_cmd = 0;
1157 	u32 emrs2_cmd = 0;
1158 	u32 emrs3_cmd = 0;
1159 
1160 	ENTERFN();
1161 
1162 	/* jedec_init starts */
1163 	mrc_post_code(0x04, 0x00);
1164 
1165 	/* DDR3_RESET_SET=0, DDR3_RESET_RESET=1 */
1166 	mrc_alt_write_mask(DDRPHY, CCDDR3RESETCTL, 2, 0x102);
1167 
1168 	/* Assert RESET# for 200us */
1169 	delay_u(200);
1170 
1171 	/* DDR3_RESET_SET=1, DDR3_RESET_RESET=0 */
1172 	mrc_alt_write_mask(DDRPHY, CCDDR3RESETCTL, 0x100, 0x102);
1173 
1174 	dtr0 = msg_port_read(MEM_CTLR, DTR0);
1175 
1176 	/*
1177 	 * Set CKEVAL for populated ranks
1178 	 * then send NOP to each rank (#4550197)
1179 	 */
1180 
1181 	drp = msg_port_read(MEM_CTLR, DRP);
1182 	drp &= 0x3;
1183 
1184 	drmc = msg_port_read(MEM_CTLR, DRMC);
1185 	drmc &= 0xfffffffc;
1186 	drmc |= (DRMC_CKEMODE | drp);
1187 
1188 	msg_port_write(MEM_CTLR, DRMC, drmc);
1189 
1190 	for (rank = 0; rank < NUM_RANKS; rank++) {
1191 		/* Skip to next populated rank */
1192 		if ((mrc_params->rank_enables & (1 << rank)) == 0)
1193 			continue;
1194 
1195 		dram_init_command(DCMD_NOP(rank));
1196 	}
1197 
1198 	msg_port_write(MEM_CTLR, DRMC,
1199 		(mrc_params->rd_odt_value == 0 ? DRMC_ODTMODE : 0));
1200 
1201 	/*
1202 	 * setup for emrs 2
1203 	 * BIT[15:11] --> Always "0"
1204 	 * BIT[10:09] --> Rtt_WR: want "Dynamic ODT Off" (0)
1205 	 * BIT[08]    --> Always "0"
1206 	 * BIT[07]    --> SRT: use sr_temp_range
1207 	 * BIT[06]    --> ASR: want "Manual SR Reference" (0)
1208 	 * BIT[05:03] --> CWL: use oem_tCWL
1209 	 * BIT[02:00] --> PASR: want "Full Array" (0)
1210 	 */
1211 	emrs2_cmd |= (2 << 3);
1212 	wl = 5 + mrc_params->ddr_speed;
1213 	emrs2_cmd |= ((wl - 5) << 9);
1214 	emrs2_cmd |= (mrc_params->sr_temp_range << 13);
1215 
1216 	/*
1217 	 * setup for emrs 3
1218 	 * BIT[15:03] --> Always "0"
1219 	 * BIT[02]    --> MPR: want "Normal Operation" (0)
1220 	 * BIT[01:00] --> MPR_Loc: want "Predefined Pattern" (0)
1221 	 */
1222 	emrs3_cmd |= (3 << 3);
1223 
1224 	/*
1225 	 * setup for emrs 1
1226 	 * BIT[15:13]     --> Always "0"
1227 	 * BIT[12:12]     --> Qoff: want "Output Buffer Enabled" (0)
1228 	 * BIT[11:11]     --> TDQS: want "Disabled" (0)
1229 	 * BIT[10:10]     --> Always "0"
1230 	 * BIT[09,06,02]  --> Rtt_nom: use rtt_nom_value
1231 	 * BIT[08]        --> Always "0"
1232 	 * BIT[07]        --> WR_LVL: want "Disabled" (0)
1233 	 * BIT[05,01]     --> DIC: use ron_value
1234 	 * BIT[04:03]     --> AL: additive latency want "0" (0)
1235 	 * BIT[00]        --> DLL: want "Enable" (0)
1236 	 *
1237 	 * (BIT5|BIT1) set Ron value
1238 	 * 00 --> RZQ/6 (40ohm)
1239 	 * 01 --> RZQ/7 (34ohm)
1240 	 * 1* --> RESERVED
1241 	 *
1242 	 * (BIT9|BIT6|BIT2) set Rtt_nom value
1243 	 * 000 --> Disabled
1244 	 * 001 --> RZQ/4 ( 60ohm)
1245 	 * 010 --> RZQ/2 (120ohm)
1246 	 * 011 --> RZQ/6 ( 40ohm)
1247 	 * 1** --> RESERVED
1248 	 */
1249 	emrs1_cmd |= (1 << 3);
1250 	emrs1_cmd &= ~(1 << 6);
1251 
1252 	if (mrc_params->ron_value == 0)
1253 		emrs1_cmd |= (1 << 7);
1254 	else
1255 		emrs1_cmd &= ~(1 << 7);
1256 
1257 	if (mrc_params->rtt_nom_value == 0)
1258 		emrs1_cmd |= (DDR3_EMRS1_RTTNOM_40 << 6);
1259 	else if (mrc_params->rtt_nom_value == 1)
1260 		emrs1_cmd |= (DDR3_EMRS1_RTTNOM_60 << 6);
1261 	else if (mrc_params->rtt_nom_value == 2)
1262 		emrs1_cmd |= (DDR3_EMRS1_RTTNOM_120 << 6);
1263 
1264 	/* save MRS1 value (excluding control fields) */
1265 	mrc_params->mrs1 = emrs1_cmd >> 6;
1266 
1267 	/*
1268 	 * setup for mrs 0
1269 	 * BIT[15:13]     --> Always "0"
1270 	 * BIT[12]        --> PPD: for Quark (1)
1271 	 * BIT[11:09]     --> WR: use oem_tWR
1272 	 * BIT[08]        --> DLL: want "Reset" (1, self clearing)
1273 	 * BIT[07]        --> MODE: want "Normal" (0)
1274 	 * BIT[06:04,02]  --> CL: use oem_tCAS
1275 	 * BIT[03]        --> RD_BURST_TYPE: want "Interleave" (1)
1276 	 * BIT[01:00]     --> BL: want "8 Fixed" (0)
1277 	 * WR:
1278 	 * 0 --> 16
1279 	 * 1 --> 5
1280 	 * 2 --> 6
1281 	 * 3 --> 7
1282 	 * 4 --> 8
1283 	 * 5 --> 10
1284 	 * 6 --> 12
1285 	 * 7 --> 14
1286 	 * CL:
1287 	 * BIT[02:02] "0" if oem_tCAS <= 11 (1866?)
1288 	 * BIT[06:04] use oem_tCAS-4
1289 	 */
1290 	mrs0_cmd |= (1 << 14);
1291 	mrs0_cmd |= (1 << 18);
1292 	mrs0_cmd |= ((((dtr0 >> 12) & 7) + 1) << 10);
1293 
1294 	tck = t_ck[mrc_params->ddr_speed];
1295 	/* Per JEDEC: tWR=15000ps DDR2/3 from 800-1600 */
1296 	twr = MCEIL(15000, tck);
1297 	mrs0_cmd |= ((twr - 4) << 15);
1298 
1299 	for (rank = 0; rank < NUM_RANKS; rank++) {
1300 		/* Skip to next populated rank */
1301 		if ((mrc_params->rank_enables & (1 << rank)) == 0)
1302 			continue;
1303 
1304 		emrs2_cmd |= (rank << 22);
1305 		dram_init_command(emrs2_cmd);
1306 
1307 		emrs3_cmd |= (rank << 22);
1308 		dram_init_command(emrs3_cmd);
1309 
1310 		emrs1_cmd |= (rank << 22);
1311 		dram_init_command(emrs1_cmd);
1312 
1313 		mrs0_cmd |= (rank << 22);
1314 		dram_init_command(mrs0_cmd);
1315 
1316 		dram_init_command(DCMD_ZQCL(rank));
1317 	}
1318 
1319 	LEAVEFN();
1320 }
1321 
1322 /*
1323  * Dunit Initialization Complete
1324  *
1325  * Indicates that initialization of the Dunit has completed.
1326  *
1327  * Memory accesses are permitted and maintenance operation begins.
1328  * Until this bit is set to a 1, the memory controller will not accept
1329  * DRAM requests from the MEMORY_MANAGER or HTE.
1330  */
1331 void set_ddr_init_complete(struct mrc_params *mrc_params)
1332 {
1333 	u32 dco;
1334 
1335 	ENTERFN();
1336 
1337 	dco = msg_port_read(MEM_CTLR, DCO);
1338 	dco &= ~DCO_PMICTL;
1339 	dco |= DCO_IC;
1340 	msg_port_write(MEM_CTLR, DCO, dco);
1341 
1342 	LEAVEFN();
1343 }
1344 
1345 /*
1346  * This function will retrieve relevant timing data
1347  *
1348  * This data will be used on subsequent boots to speed up boot times
1349  * and is required for Suspend To RAM capabilities.
1350  */
1351 void restore_timings(struct mrc_params *mrc_params)
1352 {
1353 	uint8_t ch, rk, bl;
1354 	const struct mrc_timings *mt = &mrc_params->timings;
1355 
1356 	for (ch = 0; ch < NUM_CHANNELS; ch++) {
1357 		for (rk = 0; rk < NUM_RANKS; rk++) {
1358 			for (bl = 0; bl < NUM_BYTE_LANES; bl++) {
1359 				set_rcvn(ch, rk, bl, mt->rcvn[ch][rk][bl]);
1360 				set_rdqs(ch, rk, bl, mt->rdqs[ch][rk][bl]);
1361 				set_wdqs(ch, rk, bl, mt->wdqs[ch][rk][bl]);
1362 				set_wdq(ch, rk, bl, mt->wdq[ch][rk][bl]);
1363 				if (rk == 0) {
1364 					/* VREF (RANK0 only) */
1365 					set_vref(ch, bl, mt->vref[ch][bl]);
1366 				}
1367 			}
1368 			set_wctl(ch, rk, mt->wctl[ch][rk]);
1369 		}
1370 		set_wcmd(ch, mt->wcmd[ch]);
1371 	}
1372 }
1373 
1374 /*
1375  * Configure default settings normally set as part of read training
1376  *
1377  * Some defaults have to be set earlier as they may affect earlier
1378  * training steps.
1379  */
1380 void default_timings(struct mrc_params *mrc_params)
1381 {
1382 	uint8_t ch, rk, bl;
1383 
1384 	for (ch = 0; ch < NUM_CHANNELS; ch++) {
1385 		for (rk = 0; rk < NUM_RANKS; rk++) {
1386 			for (bl = 0; bl < NUM_BYTE_LANES; bl++) {
1387 				set_rdqs(ch, rk, bl, 24);
1388 				if (rk == 0) {
1389 					/* VREF (RANK0 only) */
1390 					set_vref(ch, bl, 32);
1391 				}
1392 			}
1393 		}
1394 	}
1395 }
1396 
1397 /*
1398  * This function will perform our RCVEN Calibration Algorithm.
1399  * We will only use the 2xCLK domain timings to perform RCVEN Calibration.
1400  * All byte lanes will be calibrated "simultaneously" per channel per rank.
1401  */
1402 void rcvn_cal(struct mrc_params *mrc_params)
1403 {
1404 	uint8_t ch;	/* channel counter */
1405 	uint8_t rk;	/* rank counter */
1406 	uint8_t bl;	/* byte lane counter */
1407 	uint8_t bl_divisor = (mrc_params->channel_width == X16) ? 2 : 1;
1408 
1409 #ifdef R2R_SHARING
1410 	/* used to find placement for rank2rank sharing configs */
1411 	uint32_t final_delay[NUM_CHANNELS][NUM_BYTE_LANES];
1412 #ifndef BACKUP_RCVN
1413 	/* used to find placement for rank2rank sharing configs */
1414 	uint32_t num_ranks_enabled = 0;
1415 #endif
1416 #endif
1417 
1418 #ifdef BACKUP_RCVN
1419 #else
1420 	uint32_t temp;
1421 	/* absolute PI value to be programmed on the byte lane */
1422 	uint32_t delay[NUM_BYTE_LANES];
1423 	u32 dtr1, dtr1_save;
1424 #endif
1425 
1426 	ENTERFN();
1427 
1428 	/* rcvn_cal starts */
1429 	mrc_post_code(0x05, 0x00);
1430 
1431 #ifndef BACKUP_RCVN
1432 	/* need separate burst to sample DQS preamble */
1433 	dtr1 = msg_port_read(MEM_CTLR, DTR1);
1434 	dtr1_save = dtr1;
1435 	dtr1 |= DTR1_TCCD_12CLK;
1436 	msg_port_write(MEM_CTLR, DTR1, dtr1);
1437 #endif
1438 
1439 #ifdef R2R_SHARING
1440 	/* need to set "final_delay[][]" elements to "0" */
1441 	memset((void *)(final_delay), 0x00, (size_t)sizeof(final_delay));
1442 #endif
1443 
1444 	/* loop through each enabled channel */
1445 	for (ch = 0; ch < NUM_CHANNELS; ch++) {
1446 		if (mrc_params->channel_enables & (1 << ch)) {
1447 			/* perform RCVEN Calibration on a per rank basis */
1448 			for (rk = 0; rk < NUM_RANKS; rk++) {
1449 				if (mrc_params->rank_enables & (1 << rk)) {
1450 					/*
1451 					 * POST_CODE here indicates the current
1452 					 * channel and rank being calibrated
1453 					 */
1454 					mrc_post_code(0x05, 0x10 + ((ch << 4) | rk));
1455 
1456 #ifdef BACKUP_RCVN
1457 					/* et hard-coded timing values */
1458 					for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++)
1459 						set_rcvn(ch, rk, bl, ddr_rcvn[PLATFORM_ID]);
1460 #else
1461 					/* enable FIFORST */
1462 					for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl += 2) {
1463 						mrc_alt_write_mask(DDRPHY,
1464 							B01PTRCTL1 +
1465 							(bl >> 1) * DDRIODQ_BL_OFFSET +
1466 							ch * DDRIODQ_CH_OFFSET,
1467 							0, 1 << 8);
1468 					}
1469 					/* initialize the starting delay to 128 PI (cas +1 CLK) */
1470 					for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
1471 						/* 1x CLK domain timing is cas-4 */
1472 						delay[bl] = (4 + 1) * FULL_CLK;
1473 
1474 						set_rcvn(ch, rk, bl, delay[bl]);
1475 					}
1476 
1477 					/* now find the rising edge */
1478 					find_rising_edge(mrc_params, delay, ch, rk, true);
1479 
1480 					/* Now increase delay by 32 PI (1/4 CLK) to place in center of high pulse */
1481 					for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
1482 						delay[bl] += QRTR_CLK;
1483 						set_rcvn(ch, rk, bl, delay[bl]);
1484 					}
1485 					/* Now decrement delay by 128 PI (1 CLK) until we sample a "0" */
1486 					do {
1487 						temp = sample_dqs(mrc_params, ch, rk, true);
1488 						for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
1489 							if (temp & (1 << bl)) {
1490 								if (delay[bl] >= FULL_CLK) {
1491 									delay[bl] -= FULL_CLK;
1492 									set_rcvn(ch, rk, bl, delay[bl]);
1493 								} else {
1494 									/* not enough delay */
1495 									training_message(ch, rk, bl);
1496 									mrc_post_code(0xee, 0x50);
1497 								}
1498 							}
1499 						}
1500 					} while (temp & 0xff);
1501 
1502 #ifdef R2R_SHARING
1503 					/* increment "num_ranks_enabled" */
1504 					num_ranks_enabled++;
1505 					/* Finally increment delay by 32 PI (1/4 CLK) to place in center of preamble */
1506 					for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
1507 						delay[bl] += QRTR_CLK;
1508 						/* add "delay[]" values to "final_delay[][]" for rolling average */
1509 						final_delay[ch][bl] += delay[bl];
1510 						/* set timing based on rolling average values */
1511 						set_rcvn(ch, rk, bl, final_delay[ch][bl] / num_ranks_enabled);
1512 					}
1513 #else
1514 					/* Finally increment delay by 32 PI (1/4 CLK) to place in center of preamble */
1515 					for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
1516 						delay[bl] += QRTR_CLK;
1517 						set_rcvn(ch, rk, bl, delay[bl]);
1518 					}
1519 #endif
1520 
1521 					/* disable FIFORST */
1522 					for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl += 2) {
1523 						mrc_alt_write_mask(DDRPHY,
1524 							B01PTRCTL1 +
1525 							(bl >> 1) * DDRIODQ_BL_OFFSET +
1526 							ch * DDRIODQ_CH_OFFSET,
1527 							1 << 8, 1 << 8);
1528 					}
1529 #endif
1530 				}
1531 			}
1532 		}
1533 	}
1534 
1535 #ifndef BACKUP_RCVN
1536 	/* restore original */
1537 	msg_port_write(MEM_CTLR, DTR1, dtr1_save);
1538 #endif
1539 
1540 	LEAVEFN();
1541 }
1542 
1543 /*
1544  * This function will perform the Write Levelling algorithm
1545  * (align WCLK and WDQS).
1546  *
1547  * This algorithm will act on each rank in each channel separately.
1548  */
1549 void wr_level(struct mrc_params *mrc_params)
1550 {
1551 	uint8_t ch;	/* channel counter */
1552 	uint8_t rk;	/* rank counter */
1553 	uint8_t bl;	/* byte lane counter */
1554 	uint8_t bl_divisor = (mrc_params->channel_width == X16) ? 2 : 1;
1555 
1556 #ifdef R2R_SHARING
1557 	/* used to find placement for rank2rank sharing configs */
1558 	uint32_t final_delay[NUM_CHANNELS][NUM_BYTE_LANES];
1559 #ifndef BACKUP_WDQS
1560 	/* used to find placement for rank2rank sharing configs */
1561 	uint32_t num_ranks_enabled = 0;
1562 #endif
1563 #endif
1564 
1565 #ifdef BACKUP_WDQS
1566 #else
1567 	/* determines stop condition for CRS_WR_LVL */
1568 	bool all_edges_found;
1569 	/* absolute PI value to be programmed on the byte lane */
1570 	uint32_t delay[NUM_BYTE_LANES];
1571 	/*
1572 	 * static makes it so the data is loaded in the heap once by shadow(),
1573 	 * where non-static copies the data onto the stack every time this
1574 	 * function is called
1575 	 */
1576 	uint32_t address;	/* address to be checked during COARSE_WR_LVL */
1577 	u32 dtr4, dtr4_save;
1578 #endif
1579 
1580 	ENTERFN();
1581 
1582 	/* wr_level starts */
1583 	mrc_post_code(0x06, 0x00);
1584 
1585 #ifdef R2R_SHARING
1586 	/* need to set "final_delay[][]" elements to "0" */
1587 	memset((void *)(final_delay), 0x00, (size_t)sizeof(final_delay));
1588 #endif
1589 
1590 	/* loop through each enabled channel */
1591 	for (ch = 0; ch < NUM_CHANNELS; ch++) {
1592 		if (mrc_params->channel_enables & (1 << ch)) {
1593 			/* perform WRITE LEVELING algorithm on a per rank basis */
1594 			for (rk = 0; rk < NUM_RANKS; rk++) {
1595 				if (mrc_params->rank_enables & (1 << rk)) {
1596 					/*
1597 					 * POST_CODE here indicates the current
1598 					 * rank and channel being calibrated
1599 					 */
1600 					mrc_post_code(0x06, 0x10 + ((ch << 4) | rk));
1601 
1602 #ifdef BACKUP_WDQS
1603 					for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
1604 						set_wdqs(ch, rk, bl, ddr_wdqs[PLATFORM_ID]);
1605 						set_wdq(ch, rk, bl, ddr_wdqs[PLATFORM_ID] - QRTR_CLK);
1606 					}
1607 #else
1608 					/*
1609 					 * perform a single PRECHARGE_ALL command to
1610 					 * make DRAM state machine go to IDLE state
1611 					 */
1612 					dram_init_command(DCMD_PREA(rk));
1613 
1614 					/*
1615 					 * enable Write Levelling Mode
1616 					 * (EMRS1 w/ Write Levelling Mode Enable)
1617 					 */
1618 					dram_init_command(DCMD_MRS1(rk, 0x82));
1619 
1620 					/*
1621 					 * set ODT DRAM Full Time Termination
1622 					 * disable in MCU
1623 					 */
1624 
1625 					dtr4 = msg_port_read(MEM_CTLR, DTR4);
1626 					dtr4_save = dtr4;
1627 					dtr4 |= DTR4_ODTDIS;
1628 					msg_port_write(MEM_CTLR, DTR4, dtr4);
1629 
1630 					for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor) / 2; bl++) {
1631 						/*
1632 						 * Enable Sandy Bridge Mode (WDQ Tri-State) &
1633 						 * Ensure 5 WDQS pulses during Write Leveling
1634 						 */
1635 						mrc_alt_write_mask(DDRPHY,
1636 							DQCTL + DDRIODQ_BL_OFFSET * bl + DDRIODQ_CH_OFFSET * ch,
1637 							0x10000154,
1638 							0x100003fc);
1639 					}
1640 
1641 					/* Write Leveling Mode enabled in IO */
1642 					mrc_alt_write_mask(DDRPHY,
1643 						CCDDR3RESETCTL + DDRIOCCC_CH_OFFSET * ch,
1644 						1 << 16, 1 << 16);
1645 
1646 					/* Initialize the starting delay to WCLK */
1647 					for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
1648 						/*
1649 						 * CLK0 --> RK0
1650 						 * CLK1 --> RK1
1651 						 */
1652 						delay[bl] = get_wclk(ch, rk);
1653 
1654 						set_wdqs(ch, rk, bl, delay[bl]);
1655 					}
1656 
1657 					/* now find the rising edge */
1658 					find_rising_edge(mrc_params, delay, ch, rk, false);
1659 
1660 					/* disable Write Levelling Mode */
1661 					mrc_alt_write_mask(DDRPHY,
1662 						CCDDR3RESETCTL + DDRIOCCC_CH_OFFSET * ch,
1663 						0, 1 << 16);
1664 
1665 					for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor) / 2; bl++) {
1666 						/* Disable Sandy Bridge Mode & Ensure 4 WDQS pulses during normal operation */
1667 						mrc_alt_write_mask(DDRPHY,
1668 							DQCTL + DDRIODQ_BL_OFFSET * bl + DDRIODQ_CH_OFFSET * ch,
1669 							0x00000154,
1670 							0x100003fc);
1671 					}
1672 
1673 					/* restore original DTR4 */
1674 					msg_port_write(MEM_CTLR, DTR4, dtr4_save);
1675 
1676 					/*
1677 					 * restore original value
1678 					 * (Write Levelling Mode Disable)
1679 					 */
1680 					dram_init_command(DCMD_MRS1(rk, mrc_params->mrs1));
1681 
1682 					/*
1683 					 * perform a single PRECHARGE_ALL command to
1684 					 * make DRAM state machine go to IDLE state
1685 					 */
1686 					dram_init_command(DCMD_PREA(rk));
1687 
1688 					mrc_post_code(0x06, 0x30 + ((ch << 4) | rk));
1689 
1690 					/*
1691 					 * COARSE WRITE LEVEL:
1692 					 * check that we're on the correct clock edge
1693 					 */
1694 
1695 					/* hte reconfiguration request */
1696 					mrc_params->hte_setup = 1;
1697 
1698 					/* start CRS_WR_LVL with WDQS = WDQS + 128 PI */
1699 					for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
1700 						delay[bl] = get_wdqs(ch, rk, bl) + FULL_CLK;
1701 						set_wdqs(ch, rk, bl, delay[bl]);
1702 						/*
1703 						 * program WDQ timings based on WDQS
1704 						 * (WDQ = WDQS - 32 PI)
1705 						 */
1706 						set_wdq(ch, rk, bl, (delay[bl] - QRTR_CLK));
1707 					}
1708 
1709 					/* get an address in the targeted channel/rank */
1710 					address = get_addr(ch, rk);
1711 					do {
1712 						uint32_t coarse_result = 0x00;
1713 						uint32_t coarse_result_mask = byte_lane_mask(mrc_params);
1714 						/* assume pass */
1715 						all_edges_found = true;
1716 
1717 						mrc_params->hte_setup = 1;
1718 						coarse_result = check_rw_coarse(mrc_params, address);
1719 
1720 						/* check for failures and margin the byte lane back 128 PI (1 CLK) */
1721 						for (bl = 0; bl < NUM_BYTE_LANES / bl_divisor; bl++) {
1722 							if (coarse_result & (coarse_result_mask << bl)) {
1723 								all_edges_found = false;
1724 								delay[bl] -= FULL_CLK;
1725 								set_wdqs(ch, rk, bl, delay[bl]);
1726 								/* program WDQ timings based on WDQS (WDQ = WDQS - 32 PI) */
1727 								set_wdq(ch, rk, bl, delay[bl] - QRTR_CLK);
1728 							}
1729 						}
1730 					} while (!all_edges_found);
1731 
1732 #ifdef R2R_SHARING
1733 					/* increment "num_ranks_enabled" */
1734 					 num_ranks_enabled++;
1735 					/* accumulate "final_delay[][]" values from "delay[]" values for rolling average */
1736 					for (bl = 0; bl < NUM_BYTE_LANES / bl_divisor; bl++) {
1737 						final_delay[ch][bl] += delay[bl];
1738 						set_wdqs(ch, rk, bl, final_delay[ch][bl] / num_ranks_enabled);
1739 						/* program WDQ timings based on WDQS (WDQ = WDQS - 32 PI) */
1740 						set_wdq(ch, rk, bl, final_delay[ch][bl] / num_ranks_enabled - QRTR_CLK);
1741 					}
1742 #endif
1743 #endif
1744 				}
1745 			}
1746 		}
1747 	}
1748 
1749 	LEAVEFN();
1750 }
1751 
1752 void prog_page_ctrl(struct mrc_params *mrc_params)
1753 {
1754 	u32 dpmc0;
1755 
1756 	ENTERFN();
1757 
1758 	dpmc0 = msg_port_read(MEM_CTLR, DPMC0);
1759 	dpmc0 &= ~DPMC0_PCLSTO_MASK;
1760 	dpmc0 |= (4 << 16);
1761 	dpmc0 |= DPMC0_PREAPWDEN;
1762 	msg_port_write(MEM_CTLR, DPMC0, dpmc0);
1763 }
1764 
1765 /*
1766  * This function will perform the READ TRAINING Algorithm on all
1767  * channels/ranks/byte_lanes simultaneously to minimize execution time.
1768  *
1769  * The idea here is to train the VREF and RDQS (and eventually RDQ) values
1770  * to achieve maximum READ margins. The algorithm will first determine the
1771  * X coordinate (RDQS setting). This is done by collapsing the VREF eye
1772  * until we find a minimum required RDQS eye for VREF_MIN and VREF_MAX.
1773  * Then we take the averages of the RDQS eye at VREF_MIN and VREF_MAX,
1774  * then average those; this will be the final X coordinate. The algorithm
1775  * will then determine the Y coordinate (VREF setting). This is done by
1776  * collapsing the RDQS eye until we find a minimum required VREF eye for
1777  * RDQS_MIN and RDQS_MAX. Then we take the averages of the VREF eye at
1778  * RDQS_MIN and RDQS_MAX, then average those; this will be the final Y
1779  * coordinate.
1780  *
1781  * NOTE: this algorithm assumes the eye curves have a one-to-one relationship,
1782  * meaning for each X the curve has only one Y and vice-a-versa.
1783  */
1784 void rd_train(struct mrc_params *mrc_params)
1785 {
1786 	uint8_t ch;	/* channel counter */
1787 	uint8_t rk;	/* rank counter */
1788 	uint8_t bl;	/* byte lane counter */
1789 	uint8_t bl_divisor = (mrc_params->channel_width == X16) ? 2 : 1;
1790 #ifdef BACKUP_RDQS
1791 #else
1792 	uint8_t side_x;	/* tracks LEFT/RIGHT approach vectors */
1793 	uint8_t side_y;	/* tracks BOTTOM/TOP approach vectors */
1794 	/* X coordinate data (passing RDQS values) for approach vectors */
1795 	uint8_t x_coordinate[2][2][NUM_CHANNELS][NUM_RANKS][NUM_BYTE_LANES];
1796 	/* Y coordinate data (passing VREF values) for approach vectors */
1797 	uint8_t y_coordinate[2][2][NUM_CHANNELS][NUM_BYTE_LANES];
1798 	/* centered X (RDQS) */
1799 	uint8_t x_center[NUM_CHANNELS][NUM_RANKS][NUM_BYTE_LANES];
1800 	/* centered Y (VREF) */
1801 	uint8_t y_center[NUM_CHANNELS][NUM_BYTE_LANES];
1802 	uint32_t address;	/* target address for check_bls_ex() */
1803 	uint32_t result;	/* result of check_bls_ex() */
1804 	uint32_t bl_mask;	/* byte lane mask for result checking */
1805 #ifdef R2R_SHARING
1806 	/* used to find placement for rank2rank sharing configs */
1807 	uint32_t final_delay[NUM_CHANNELS][NUM_BYTE_LANES];
1808 	/* used to find placement for rank2rank sharing configs */
1809 	uint32_t num_ranks_enabled = 0;
1810 #endif
1811 #endif
1812 
1813 	/* rd_train starts */
1814 	mrc_post_code(0x07, 0x00);
1815 
1816 	ENTERFN();
1817 
1818 #ifdef BACKUP_RDQS
1819 	for (ch = 0; ch < NUM_CHANNELS; ch++) {
1820 		if (mrc_params->channel_enables & (1 << ch)) {
1821 			for (rk = 0; rk < NUM_RANKS; rk++) {
1822 				if (mrc_params->rank_enables & (1 << rk)) {
1823 					for (bl = 0;
1824 					     bl < NUM_BYTE_LANES / bl_divisor;
1825 					     bl++) {
1826 						set_rdqs(ch, rk, bl, ddr_rdqs[PLATFORM_ID]);
1827 					}
1828 				}
1829 			}
1830 		}
1831 	}
1832 #else
1833 	/* initialize x/y_coordinate arrays */
1834 	for (ch = 0; ch < NUM_CHANNELS; ch++) {
1835 		if (mrc_params->channel_enables & (1 << ch)) {
1836 			for (rk = 0; rk < NUM_RANKS; rk++) {
1837 				if (mrc_params->rank_enables & (1 << rk)) {
1838 					for (bl = 0;
1839 					     bl < NUM_BYTE_LANES / bl_divisor;
1840 					     bl++) {
1841 						/* x_coordinate */
1842 						x_coordinate[L][B][ch][rk][bl] = RDQS_MIN;
1843 						x_coordinate[R][B][ch][rk][bl] = RDQS_MAX;
1844 						x_coordinate[L][T][ch][rk][bl] = RDQS_MIN;
1845 						x_coordinate[R][T][ch][rk][bl] = RDQS_MAX;
1846 						/* y_coordinate */
1847 						y_coordinate[L][B][ch][bl] = VREF_MIN;
1848 						y_coordinate[R][B][ch][bl] = VREF_MIN;
1849 						y_coordinate[L][T][ch][bl] = VREF_MAX;
1850 						y_coordinate[R][T][ch][bl] = VREF_MAX;
1851 					}
1852 				}
1853 			}
1854 		}
1855 	}
1856 
1857 	/* initialize other variables */
1858 	bl_mask = byte_lane_mask(mrc_params);
1859 	address = get_addr(0, 0);
1860 
1861 #ifdef R2R_SHARING
1862 	/* need to set "final_delay[][]" elements to "0" */
1863 	memset((void *)(final_delay), 0x00, (size_t)sizeof(final_delay));
1864 #endif
1865 
1866 	/* look for passing coordinates */
1867 	for (side_y = B; side_y <= T; side_y++) {
1868 		for (side_x = L; side_x <= R; side_x++) {
1869 			mrc_post_code(0x07, 0x10 + side_y * 2 + side_x);
1870 
1871 			/* find passing values */
1872 			for (ch = 0; ch < NUM_CHANNELS; ch++) {
1873 				if (mrc_params->channel_enables & (0x1 << ch)) {
1874 					for (rk = 0; rk < NUM_RANKS; rk++) {
1875 						if (mrc_params->rank_enables &
1876 							(0x1 << rk)) {
1877 							/* set x/y_coordinate search starting settings */
1878 							for (bl = 0;
1879 							     bl < NUM_BYTE_LANES / bl_divisor;
1880 							     bl++) {
1881 								set_rdqs(ch, rk, bl,
1882 									 x_coordinate[side_x][side_y][ch][rk][bl]);
1883 								set_vref(ch, bl,
1884 									 y_coordinate[side_x][side_y][ch][bl]);
1885 							}
1886 
1887 							/* get an address in the target channel/rank */
1888 							address = get_addr(ch, rk);
1889 
1890 							/* request HTE reconfiguration */
1891 							mrc_params->hte_setup = 1;
1892 
1893 							/* test the settings */
1894 							do {
1895 								/* result[07:00] == failing byte lane (MAX 8) */
1896 								result = check_bls_ex(mrc_params, address);
1897 
1898 								/* check for failures */
1899 								if (result & 0xff) {
1900 									/* at least 1 byte lane failed */
1901 									for (bl = 0; bl < NUM_BYTE_LANES / bl_divisor; bl++) {
1902 										if (result &
1903 											(bl_mask << bl)) {
1904 											/* adjust the RDQS values accordingly */
1905 											if (side_x == L)
1906 												x_coordinate[L][side_y][ch][rk][bl] += RDQS_STEP;
1907 											else
1908 												x_coordinate[R][side_y][ch][rk][bl] -= RDQS_STEP;
1909 
1910 											/* check that we haven't closed the RDQS_EYE too much */
1911 											if ((x_coordinate[L][side_y][ch][rk][bl] > (RDQS_MAX - MIN_RDQS_EYE)) ||
1912 												(x_coordinate[R][side_y][ch][rk][bl] < (RDQS_MIN + MIN_RDQS_EYE)) ||
1913 												(x_coordinate[L][side_y][ch][rk][bl] ==
1914 												x_coordinate[R][side_y][ch][rk][bl])) {
1915 												/*
1916 												 * not enough RDQS margin available at this VREF
1917 												 * update VREF values accordingly
1918 												 */
1919 												if (side_y == B)
1920 													y_coordinate[side_x][B][ch][bl] += VREF_STEP;
1921 												else
1922 													y_coordinate[side_x][T][ch][bl] -= VREF_STEP;
1923 
1924 												/* check that we haven't closed the VREF_EYE too much */
1925 												if ((y_coordinate[side_x][B][ch][bl] > (VREF_MAX - MIN_VREF_EYE)) ||
1926 													(y_coordinate[side_x][T][ch][bl] < (VREF_MIN + MIN_VREF_EYE)) ||
1927 													(y_coordinate[side_x][B][ch][bl] == y_coordinate[side_x][T][ch][bl])) {
1928 													/* VREF_EYE collapsed below MIN_VREF_EYE */
1929 													training_message(ch, rk, bl);
1930 													mrc_post_code(0xEE, 0x70 + side_y * 2 + side_x);
1931 												} else {
1932 													/* update the VREF setting */
1933 													set_vref(ch, bl, y_coordinate[side_x][side_y][ch][bl]);
1934 													/* reset the X coordinate to begin the search at the new VREF */
1935 													x_coordinate[side_x][side_y][ch][rk][bl] =
1936 														(side_x == L) ? RDQS_MIN : RDQS_MAX;
1937 												}
1938 											}
1939 
1940 											/* update the RDQS setting */
1941 											set_rdqs(ch, rk, bl, x_coordinate[side_x][side_y][ch][rk][bl]);
1942 										}
1943 									}
1944 								}
1945 							} while (result & 0xff);
1946 						}
1947 					}
1948 				}
1949 			}
1950 		}
1951 	}
1952 
1953 	mrc_post_code(0x07, 0x20);
1954 
1955 	/* find final RDQS (X coordinate) & final VREF (Y coordinate) */
1956 	for (ch = 0; ch < NUM_CHANNELS; ch++) {
1957 		if (mrc_params->channel_enables & (1 << ch)) {
1958 			for (rk = 0; rk < NUM_RANKS; rk++) {
1959 				if (mrc_params->rank_enables & (1 << rk)) {
1960 					for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
1961 						uint32_t temp1;
1962 						uint32_t temp2;
1963 
1964 						/* x_coordinate */
1965 						DPF(D_INFO,
1966 						    "RDQS T/B eye rank%d lane%d : %d-%d %d-%d\n",
1967 						    rk, bl,
1968 						    x_coordinate[L][T][ch][rk][bl],
1969 						    x_coordinate[R][T][ch][rk][bl],
1970 						    x_coordinate[L][B][ch][rk][bl],
1971 						    x_coordinate[R][B][ch][rk][bl]);
1972 
1973 						/* average the TOP side LEFT & RIGHT values */
1974 						temp1 = (x_coordinate[R][T][ch][rk][bl] + x_coordinate[L][T][ch][rk][bl]) / 2;
1975 						/* average the BOTTOM side LEFT & RIGHT values */
1976 						temp2 = (x_coordinate[R][B][ch][rk][bl] + x_coordinate[L][B][ch][rk][bl]) / 2;
1977 						/* average the above averages */
1978 						x_center[ch][rk][bl] = (uint8_t) ((temp1 + temp2) / 2);
1979 
1980 						/* y_coordinate */
1981 						DPF(D_INFO,
1982 						    "VREF R/L eye lane%d : %d-%d %d-%d\n",
1983 						    bl,
1984 						    y_coordinate[R][B][ch][bl],
1985 						    y_coordinate[R][T][ch][bl],
1986 						    y_coordinate[L][B][ch][bl],
1987 						    y_coordinate[L][T][ch][bl]);
1988 
1989 						/* average the RIGHT side TOP & BOTTOM values */
1990 						temp1 = (y_coordinate[R][T][ch][bl] + y_coordinate[R][B][ch][bl]) / 2;
1991 						/* average the LEFT side TOP & BOTTOM values */
1992 						temp2 = (y_coordinate[L][T][ch][bl] + y_coordinate[L][B][ch][bl]) / 2;
1993 						/* average the above averages */
1994 						y_center[ch][bl] = (uint8_t) ((temp1 + temp2) / 2);
1995 					}
1996 				}
1997 			}
1998 		}
1999 	}
2000 
2001 #ifdef RX_EYE_CHECK
2002 	/* perform an eye check */
2003 	for (side_y = B; side_y <= T; side_y++) {
2004 		for (side_x = L; side_x <= R; side_x++) {
2005 			mrc_post_code(0x07, 0x30 + side_y * 2 + side_x);
2006 
2007 			/* update the settings for the eye check */
2008 			for (ch = 0; ch < NUM_CHANNELS; ch++) {
2009 				if (mrc_params->channel_enables & (1 << ch)) {
2010 					for (rk = 0; rk < NUM_RANKS; rk++) {
2011 						if (mrc_params->rank_enables & (1 << rk)) {
2012 							for (bl = 0; bl < NUM_BYTE_LANES / bl_divisor; bl++) {
2013 								if (side_x == L)
2014 									set_rdqs(ch, rk, bl, x_center[ch][rk][bl] - (MIN_RDQS_EYE / 2));
2015 								else
2016 									set_rdqs(ch, rk, bl, x_center[ch][rk][bl] + (MIN_RDQS_EYE / 2));
2017 
2018 								if (side_y == B)
2019 									set_vref(ch, bl, y_center[ch][bl] - (MIN_VREF_EYE / 2));
2020 								else
2021 									set_vref(ch, bl, y_center[ch][bl] + (MIN_VREF_EYE / 2));
2022 							}
2023 						}
2024 					}
2025 				}
2026 			}
2027 
2028 			/* request HTE reconfiguration */
2029 			mrc_params->hte_setup = 1;
2030 
2031 			/* check the eye */
2032 			if (check_bls_ex(mrc_params, address) & 0xff) {
2033 				/* one or more byte lanes failed */
2034 				mrc_post_code(0xee, 0x74 + side_x * 2 + side_y);
2035 			}
2036 		}
2037 	}
2038 #endif
2039 
2040 	mrc_post_code(0x07, 0x40);
2041 
2042 	/* set final placements */
2043 	for (ch = 0; ch < NUM_CHANNELS; ch++) {
2044 		if (mrc_params->channel_enables & (1 << ch)) {
2045 			for (rk = 0; rk < NUM_RANKS; rk++) {
2046 				if (mrc_params->rank_enables & (1 << rk)) {
2047 #ifdef R2R_SHARING
2048 					/* increment "num_ranks_enabled" */
2049 					num_ranks_enabled++;
2050 #endif
2051 					for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
2052 						/* x_coordinate */
2053 #ifdef R2R_SHARING
2054 						final_delay[ch][bl] += x_center[ch][rk][bl];
2055 						set_rdqs(ch, rk, bl, final_delay[ch][bl] / num_ranks_enabled);
2056 #else
2057 						set_rdqs(ch, rk, bl, x_center[ch][rk][bl]);
2058 #endif
2059 						/* y_coordinate */
2060 						set_vref(ch, bl, y_center[ch][bl]);
2061 					}
2062 				}
2063 			}
2064 		}
2065 	}
2066 #endif
2067 
2068 	LEAVEFN();
2069 }
2070 
2071 /*
2072  * This function will perform the WRITE TRAINING Algorithm on all
2073  * channels/ranks/byte_lanes simultaneously to minimize execution time.
2074  *
2075  * The idea here is to train the WDQ timings to achieve maximum WRITE margins.
2076  * The algorithm will start with WDQ at the current WDQ setting (tracks WDQS
2077  * in WR_LVL) +/- 32 PIs (+/- 1/4 CLK) and collapse the eye until all data
2078  * patterns pass. This is because WDQS will be aligned to WCLK by the
2079  * Write Leveling algorithm and WDQ will only ever have a 1/2 CLK window
2080  * of validity.
2081  */
2082 void wr_train(struct mrc_params *mrc_params)
2083 {
2084 	uint8_t ch;	/* channel counter */
2085 	uint8_t rk;	/* rank counter */
2086 	uint8_t bl;	/* byte lane counter */
2087 	uint8_t bl_divisor = (mrc_params->channel_width == X16) ? 2 : 1;
2088 #ifdef BACKUP_WDQ
2089 #else
2090 	uint8_t side;		/* LEFT/RIGHT side indicator (0=L, 1=R) */
2091 	uint32_t temp;		/* temporary DWORD */
2092 	/* 2 arrays, for L & R side passing delays */
2093 	uint32_t delay[2][NUM_CHANNELS][NUM_RANKS][NUM_BYTE_LANES];
2094 	uint32_t address;	/* target address for check_bls_ex() */
2095 	uint32_t result;	/* result of check_bls_ex() */
2096 	uint32_t bl_mask;	/* byte lane mask for result checking */
2097 #ifdef R2R_SHARING
2098 	/* used to find placement for rank2rank sharing configs */
2099 	uint32_t final_delay[NUM_CHANNELS][NUM_BYTE_LANES];
2100 	/* used to find placement for rank2rank sharing configs */
2101 	uint32_t num_ranks_enabled = 0;
2102 #endif
2103 #endif
2104 
2105 	/* wr_train starts */
2106 	mrc_post_code(0x08, 0x00);
2107 
2108 	ENTERFN();
2109 
2110 #ifdef BACKUP_WDQ
2111 	for (ch = 0; ch < NUM_CHANNELS; ch++) {
2112 		if (mrc_params->channel_enables & (1 << ch)) {
2113 			for (rk = 0; rk < NUM_RANKS; rk++) {
2114 				if (mrc_params->rank_enables & (1 << rk)) {
2115 					for (bl = 0;
2116 					     bl < NUM_BYTE_LANES / bl_divisor;
2117 					     bl++) {
2118 						set_wdq(ch, rk, bl, ddr_wdq[PLATFORM_ID]);
2119 					}
2120 				}
2121 			}
2122 		}
2123 	}
2124 #else
2125 	/* initialize "delay" */
2126 	for (ch = 0; ch < NUM_CHANNELS; ch++) {
2127 		if (mrc_params->channel_enables & (1 << ch)) {
2128 			for (rk = 0; rk < NUM_RANKS; rk++) {
2129 				if (mrc_params->rank_enables & (1 << rk)) {
2130 					for (bl = 0;
2131 					     bl < NUM_BYTE_LANES / bl_divisor;
2132 					     bl++) {
2133 						/*
2134 						 * want to start with
2135 						 * WDQ = (WDQS - QRTR_CLK)
2136 						 * +/- QRTR_CLK
2137 						 */
2138 						temp = get_wdqs(ch, rk, bl) - QRTR_CLK;
2139 						delay[L][ch][rk][bl] = temp - QRTR_CLK;
2140 						delay[R][ch][rk][bl] = temp + QRTR_CLK;
2141 					}
2142 				}
2143 			}
2144 		}
2145 	}
2146 
2147 	/* initialize other variables */
2148 	bl_mask = byte_lane_mask(mrc_params);
2149 	address = get_addr(0, 0);
2150 
2151 #ifdef R2R_SHARING
2152 	/* need to set "final_delay[][]" elements to "0" */
2153 	memset((void *)(final_delay), 0x00, (size_t)sizeof(final_delay));
2154 #endif
2155 
2156 	/*
2157 	 * start algorithm on the LEFT side and train each channel/bl
2158 	 * until no failures are observed, then repeat for the RIGHT side.
2159 	 */
2160 	for (side = L; side <= R; side++) {
2161 		mrc_post_code(0x08, 0x10 + side);
2162 
2163 		/* set starting values */
2164 		for (ch = 0; ch < NUM_CHANNELS; ch++) {
2165 			if (mrc_params->channel_enables & (1 << ch)) {
2166 				for (rk = 0; rk < NUM_RANKS; rk++) {
2167 					if (mrc_params->rank_enables &
2168 						(1 << rk)) {
2169 						for (bl = 0;
2170 						     bl < NUM_BYTE_LANES / bl_divisor;
2171 						     bl++) {
2172 							set_wdq(ch, rk, bl, delay[side][ch][rk][bl]);
2173 						}
2174 					}
2175 				}
2176 			}
2177 		}
2178 
2179 		/* find passing values */
2180 		for (ch = 0; ch < NUM_CHANNELS; ch++) {
2181 			if (mrc_params->channel_enables & (1 << ch)) {
2182 				for (rk = 0; rk < NUM_RANKS; rk++) {
2183 					if (mrc_params->rank_enables &
2184 						(1 << rk)) {
2185 						/* get an address in the target channel/rank */
2186 						address = get_addr(ch, rk);
2187 
2188 						/* request HTE reconfiguration */
2189 						mrc_params->hte_setup = 1;
2190 
2191 						/* check the settings */
2192 						do {
2193 							/* result[07:00] == failing byte lane (MAX 8) */
2194 							result = check_bls_ex(mrc_params, address);
2195 							/* check for failures */
2196 							if (result & 0xff) {
2197 								/* at least 1 byte lane failed */
2198 								for (bl = 0; bl < NUM_BYTE_LANES / bl_divisor; bl++) {
2199 									if (result &
2200 										(bl_mask << bl)) {
2201 										if (side == L)
2202 											delay[L][ch][rk][bl] += WDQ_STEP;
2203 										else
2204 											delay[R][ch][rk][bl] -= WDQ_STEP;
2205 
2206 										/* check for algorithm failure */
2207 										if (delay[L][ch][rk][bl] != delay[R][ch][rk][bl]) {
2208 											/*
2209 											 * margin available
2210 											 * update delay setting
2211 											 */
2212 											set_wdq(ch, rk, bl,
2213 												delay[side][ch][rk][bl]);
2214 										} else {
2215 											/*
2216 											 * no margin available
2217 											 * notify the user and halt
2218 											 */
2219 											training_message(ch, rk, bl);
2220 											mrc_post_code(0xee, 0x80 + side);
2221 										}
2222 									}
2223 								}
2224 							}
2225 						/* stop when all byte lanes pass */
2226 						} while (result & 0xff);
2227 					}
2228 				}
2229 			}
2230 		}
2231 	}
2232 
2233 	/* program WDQ to the middle of passing window */
2234 	for (ch = 0; ch < NUM_CHANNELS; ch++) {
2235 		if (mrc_params->channel_enables & (1 << ch)) {
2236 			for (rk = 0; rk < NUM_RANKS; rk++) {
2237 				if (mrc_params->rank_enables & (1 << rk)) {
2238 #ifdef R2R_SHARING
2239 					/* increment "num_ranks_enabled" */
2240 					num_ranks_enabled++;
2241 #endif
2242 					for (bl = 0; bl < NUM_BYTE_LANES / bl_divisor; bl++) {
2243 						DPF(D_INFO,
2244 						    "WDQ eye rank%d lane%d : %d-%d\n",
2245 						    rk, bl,
2246 						    delay[L][ch][rk][bl],
2247 						    delay[R][ch][rk][bl]);
2248 
2249 						temp = (delay[R][ch][rk][bl] + delay[L][ch][rk][bl]) / 2;
2250 
2251 #ifdef R2R_SHARING
2252 						final_delay[ch][bl] += temp;
2253 						set_wdq(ch, rk, bl,
2254 							final_delay[ch][bl] / num_ranks_enabled);
2255 #else
2256 						set_wdq(ch, rk, bl, temp);
2257 #endif
2258 					}
2259 				}
2260 			}
2261 		}
2262 	}
2263 #endif
2264 
2265 	LEAVEFN();
2266 }
2267 
2268 /*
2269  * This function will store relevant timing data
2270  *
2271  * This data will be used on subsequent boots to speed up boot times
2272  * and is required for Suspend To RAM capabilities.
2273  */
2274 void store_timings(struct mrc_params *mrc_params)
2275 {
2276 	uint8_t ch, rk, bl;
2277 	struct mrc_timings *mt = &mrc_params->timings;
2278 
2279 	for (ch = 0; ch < NUM_CHANNELS; ch++) {
2280 		for (rk = 0; rk < NUM_RANKS; rk++) {
2281 			for (bl = 0; bl < NUM_BYTE_LANES; bl++) {
2282 				mt->rcvn[ch][rk][bl] = get_rcvn(ch, rk, bl);
2283 				mt->rdqs[ch][rk][bl] = get_rdqs(ch, rk, bl);
2284 				mt->wdqs[ch][rk][bl] = get_wdqs(ch, rk, bl);
2285 				mt->wdq[ch][rk][bl] = get_wdq(ch, rk, bl);
2286 
2287 				if (rk == 0)
2288 					mt->vref[ch][bl] = get_vref(ch, bl);
2289 			}
2290 
2291 			mt->wctl[ch][rk] = get_wctl(ch, rk);
2292 		}
2293 
2294 		mt->wcmd[ch] = get_wcmd(ch);
2295 	}
2296 
2297 	/* need to save for a case of changing frequency after warm reset */
2298 	mt->ddr_speed = mrc_params->ddr_speed;
2299 }
2300 
2301 /*
2302  * The purpose of this function is to ensure the SEC comes out of reset
2303  * and IA initiates the SEC enabling Memory Scrambling.
2304  */
2305 void enable_scrambling(struct mrc_params *mrc_params)
2306 {
2307 	uint32_t lfsr = 0;
2308 	uint8_t i;
2309 
2310 	if (mrc_params->scrambling_enables == 0)
2311 		return;
2312 
2313 	ENTERFN();
2314 
2315 	/* 32 bit seed is always stored in BIOS NVM */
2316 	lfsr = mrc_params->timings.scrambler_seed;
2317 
2318 	if (mrc_params->boot_mode == BM_COLD) {
2319 		/*
2320 		 * factory value is 0 and in first boot,
2321 		 * a clock based seed is loaded.
2322 		 */
2323 		if (lfsr == 0) {
2324 			/*
2325 			 * get seed from system clock
2326 			 * and make sure it is not all 1's
2327 			 */
2328 			lfsr = rdtsc() & 0x0fffffff;
2329 		} else {
2330 			/*
2331 			 * Need to replace scrambler
2332 			 *
2333 			 * get next 32bit LFSR 16 times which is the last
2334 			 * part of the previous scrambler vector
2335 			 */
2336 			for (i = 0; i < 16; i++)
2337 				lfsr32(&lfsr);
2338 		}
2339 
2340 		/* save new seed */
2341 		mrc_params->timings.scrambler_seed = lfsr;
2342 	}
2343 
2344 	/*
2345 	 * In warm boot or S3 exit, we have the previous seed.
2346 	 * In cold boot, we have the last 32bit LFSR which is the new seed.
2347 	 */
2348 	lfsr32(&lfsr);	/* shift to next value */
2349 	msg_port_write(MEM_CTLR, SCRMSEED, (lfsr & 0x0003ffff));
2350 
2351 	for (i = 0; i < 2; i++)
2352 		msg_port_write(MEM_CTLR, SCRMLO + i, (lfsr & 0xaaaaaaaa));
2353 
2354 	LEAVEFN();
2355 }
2356 
2357 /*
2358  * Configure MCU Power Management Control Register
2359  * and Scheduler Control Register
2360  */
2361 void prog_ddr_control(struct mrc_params *mrc_params)
2362 {
2363 	u32 dsch;
2364 	u32 dpmc0;
2365 
2366 	ENTERFN();
2367 
2368 	dsch = msg_port_read(MEM_CTLR, DSCH);
2369 	dsch &= ~(DSCH_OOODIS | DSCH_OOOST3DIS | DSCH_NEWBYPDIS);
2370 	msg_port_write(MEM_CTLR, DSCH, dsch);
2371 
2372 	dpmc0 = msg_port_read(MEM_CTLR, DPMC0);
2373 	dpmc0 &= ~DPMC0_DISPWRDN;
2374 	dpmc0 |= (mrc_params->power_down_disable << 25);
2375 	dpmc0 &= ~DPMC0_CLKGTDIS;
2376 	dpmc0 &= ~DPMC0_PCLSTO_MASK;
2377 	dpmc0 |= (4 << 16);
2378 	dpmc0 |= DPMC0_PREAPWDEN;
2379 	msg_port_write(MEM_CTLR, DPMC0, dpmc0);
2380 
2381 	/* CMDTRIST = 2h - CMD/ADDR are tristated when no valid command */
2382 	mrc_write_mask(MEM_CTLR, DPMC1, 0x20, 0x30);
2383 
2384 	LEAVEFN();
2385 }
2386 
2387 /*
2388  * After training complete configure MCU Rank Population Register
2389  * specifying: ranks enabled, device width, density, address mode
2390  */
2391 void prog_dra_drb(struct mrc_params *mrc_params)
2392 {
2393 	u32 drp;
2394 	u32 dco;
2395 	u8 density = mrc_params->params.density;
2396 
2397 	ENTERFN();
2398 
2399 	dco = msg_port_read(MEM_CTLR, DCO);
2400 	dco &= ~DCO_IC;
2401 	msg_port_write(MEM_CTLR, DCO, dco);
2402 
2403 	drp = 0;
2404 	if (mrc_params->rank_enables & 1)
2405 		drp |= DRP_RKEN0;
2406 	if (mrc_params->rank_enables & 2)
2407 		drp |= DRP_RKEN1;
2408 	if (mrc_params->dram_width == X16) {
2409 		drp |= (1 << 4);
2410 		drp |= (1 << 9);
2411 	}
2412 
2413 	/*
2414 	 * Density encoding in struct dram_params: 0=512Mb, 1=Gb, 2=2Gb, 3=4Gb
2415 	 * has to be mapped RANKDENSx encoding (0=1Gb)
2416 	 */
2417 	if (density == 0)
2418 		density = 4;
2419 
2420 	drp |= ((density - 1) << 6);
2421 	drp |= ((density - 1) << 11);
2422 
2423 	/* Address mode can be overwritten if ECC enabled */
2424 	drp |= (mrc_params->address_mode << 14);
2425 
2426 	msg_port_write(MEM_CTLR, DRP, drp);
2427 
2428 	dco &= ~DCO_PMICTL;
2429 	dco |= DCO_IC;
2430 	msg_port_write(MEM_CTLR, DCO, dco);
2431 
2432 	LEAVEFN();
2433 }
2434 
2435 /* Send DRAM wake command */
2436 void perform_wake(struct mrc_params *mrc_params)
2437 {
2438 	ENTERFN();
2439 
2440 	dram_wake_command();
2441 
2442 	LEAVEFN();
2443 }
2444 
2445 /*
2446  * Configure refresh rate and short ZQ calibration interval
2447  * Activate dynamic self refresh
2448  */
2449 void change_refresh_period(struct mrc_params *mrc_params)
2450 {
2451 	u32 drfc;
2452 	u32 dcal;
2453 	u32 dpmc0;
2454 
2455 	ENTERFN();
2456 
2457 	drfc = msg_port_read(MEM_CTLR, DRFC);
2458 	drfc &= ~DRFC_TREFI_MASK;
2459 	drfc |= (mrc_params->refresh_rate << 12);
2460 	drfc |= DRFC_REFDBTCLR;
2461 	msg_port_write(MEM_CTLR, DRFC, drfc);
2462 
2463 	dcal = msg_port_read(MEM_CTLR, DCAL);
2464 	dcal &= ~DCAL_ZQCINT_MASK;
2465 	dcal |= (3 << 8);	/* 63ms */
2466 	msg_port_write(MEM_CTLR, DCAL, dcal);
2467 
2468 	dpmc0 = msg_port_read(MEM_CTLR, DPMC0);
2469 	dpmc0 |= (DPMC0_DYNSREN | DPMC0_ENPHYCLKGATE);
2470 	msg_port_write(MEM_CTLR, DPMC0, dpmc0);
2471 
2472 	LEAVEFN();
2473 }
2474 
2475 /*
2476  * Configure DDRPHY for Auto-Refresh, Periodic Compensations,
2477  * Dynamic Diff-Amp, ZQSPERIOD, Auto-Precharge, CKE Power-Down
2478  */
2479 void set_auto_refresh(struct mrc_params *mrc_params)
2480 {
2481 	uint32_t channel;
2482 	uint32_t rank;
2483 	uint32_t bl;
2484 	uint32_t bl_divisor = 1;
2485 	uint32_t temp;
2486 
2487 	ENTERFN();
2488 
2489 	/*
2490 	 * Enable Auto-Refresh, Periodic Compensations, Dynamic Diff-Amp,
2491 	 * ZQSPERIOD, Auto-Precharge, CKE Power-Down
2492 	 */
2493 	for (channel = 0; channel < NUM_CHANNELS; channel++) {
2494 		if (mrc_params->channel_enables & (1 << channel)) {
2495 			/* Enable Periodic RCOMPS */
2496 			mrc_alt_write_mask(DDRPHY, CMPCTRL, 2, 2);
2497 
2498 			/* Enable Dynamic DiffAmp & Set Read ODT Value */
2499 			switch (mrc_params->rd_odt_value) {
2500 			case 0:
2501 				temp = 0x3f;	/* OFF */
2502 				break;
2503 			default:
2504 				temp = 0x00;	/* Auto */
2505 				break;
2506 			}
2507 
2508 			for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor) / 2; bl++) {
2509 				/* Override: DIFFAMP, ODT */
2510 				mrc_alt_write_mask(DDRPHY,
2511 					B0OVRCTL + bl * DDRIODQ_BL_OFFSET +
2512 					channel * DDRIODQ_CH_OFFSET,
2513 					temp << 10,
2514 					0x003ffc00);
2515 
2516 				/* Override: DIFFAMP, ODT */
2517 				mrc_alt_write_mask(DDRPHY,
2518 					B1OVRCTL + bl * DDRIODQ_BL_OFFSET +
2519 					channel * DDRIODQ_CH_OFFSET,
2520 					temp << 10,
2521 					0x003ffc00);
2522 			}
2523 
2524 			/* Issue ZQCS command */
2525 			for (rank = 0; rank < NUM_RANKS; rank++) {
2526 				if (mrc_params->rank_enables & (1 << rank))
2527 					dram_init_command(DCMD_ZQCS(rank));
2528 			}
2529 		}
2530 	}
2531 
2532 	clear_pointers();
2533 
2534 	LEAVEFN();
2535 }
2536 
2537 /*
2538  * Depending on configuration enables ECC support
2539  *
2540  * Available memory size is decreased, and updated with 0s
2541  * in order to clear error status. Address mode 2 forced.
2542  */
2543 void ecc_enable(struct mrc_params *mrc_params)
2544 {
2545 	u32 drp;
2546 	u32 dsch;
2547 	u32 ecc_ctrl;
2548 
2549 	if (mrc_params->ecc_enables == 0)
2550 		return;
2551 
2552 	ENTERFN();
2553 
2554 	/* Configuration required in ECC mode */
2555 	drp = msg_port_read(MEM_CTLR, DRP);
2556 	drp &= ~DRP_ADDRMAP_MASK;
2557 	drp |= DRP_ADDRMAP_MAP1;
2558 	drp |= DRP_PRI64BSPLITEN;
2559 	msg_port_write(MEM_CTLR, DRP, drp);
2560 
2561 	/* Disable new request bypass */
2562 	dsch = msg_port_read(MEM_CTLR, DSCH);
2563 	dsch |= DSCH_NEWBYPDIS;
2564 	msg_port_write(MEM_CTLR, DSCH, dsch);
2565 
2566 	/* Enable ECC */
2567 	ecc_ctrl = (DECCCTRL_SBEEN | DECCCTRL_DBEEN | DECCCTRL_ENCBGEN);
2568 	msg_port_write(MEM_CTLR, DECCCTRL, ecc_ctrl);
2569 
2570 	/* Assume 8 bank memory, one bank is gone for ECC */
2571 	mrc_params->mem_size -= mrc_params->mem_size / 8;
2572 
2573 	/* For S3 resume memory content has to be preserved */
2574 	if (mrc_params->boot_mode != BM_S3) {
2575 		select_hte();
2576 		hte_mem_init(mrc_params, MRC_MEM_INIT);
2577 		select_mem_mgr();
2578 	}
2579 
2580 	LEAVEFN();
2581 }
2582 
2583 /*
2584  * Execute memory test
2585  * if error detected it is indicated in mrc_params->status
2586  */
2587 void memory_test(struct mrc_params *mrc_params)
2588 {
2589 	uint32_t result = 0;
2590 
2591 	ENTERFN();
2592 
2593 	select_hte();
2594 	result = hte_mem_init(mrc_params, MRC_MEM_TEST);
2595 	select_mem_mgr();
2596 
2597 	DPF(D_INFO, "Memory test result %x\n", result);
2598 	mrc_params->status = ((result == 0) ? MRC_SUCCESS : MRC_E_MEMTEST);
2599 	LEAVEFN();
2600 }
2601 
2602 /* Lock MCU registers at the end of initialization sequence */
2603 void lock_registers(struct mrc_params *mrc_params)
2604 {
2605 	u32 dco;
2606 
2607 	ENTERFN();
2608 
2609 	dco = msg_port_read(MEM_CTLR, DCO);
2610 	dco &= ~(DCO_PMICTL | DCO_PMIDIS);
2611 	dco |= (DCO_DRPLOCK | DCO_CPGCLOCK);
2612 	msg_port_write(MEM_CTLR, DCO, dco);
2613 
2614 	LEAVEFN();
2615 }
2616