1 /* 2 * Copyright Altera Corporation (C) 2012-2015 3 * 4 * SPDX-License-Identifier: BSD-3-Clause 5 */ 6 7 #include <common.h> 8 #include <asm/io.h> 9 #include <asm/arch/sdram.h> 10 #include "sequencer.h" 11 #include "sequencer_auto.h" 12 #include "sequencer_auto_ac_init.h" 13 #include "sequencer_auto_inst_init.h" 14 #include "sequencer_defines.h" 15 16 static void scc_mgr_load_dqs_for_write_group(uint32_t write_group); 17 18 static struct socfpga_sdr_rw_load_manager *sdr_rw_load_mgr_regs = 19 (struct socfpga_sdr_rw_load_manager *)(SDR_PHYGRP_RWMGRGRP_ADDRESS | 0x800); 20 21 static struct socfpga_sdr_rw_load_jump_manager *sdr_rw_load_jump_mgr_regs = 22 (struct socfpga_sdr_rw_load_jump_manager *)(SDR_PHYGRP_RWMGRGRP_ADDRESS | 0xC00); 23 24 static struct socfpga_sdr_reg_file *sdr_reg_file = 25 (struct socfpga_sdr_reg_file *)SDR_PHYGRP_REGFILEGRP_ADDRESS; 26 27 static struct socfpga_sdr_scc_mgr *sdr_scc_mgr = 28 (struct socfpga_sdr_scc_mgr *)(SDR_PHYGRP_SCCGRP_ADDRESS | 0xe00); 29 30 static struct socfpga_phy_mgr_cmd *phy_mgr_cmd = 31 (struct socfpga_phy_mgr_cmd *)SDR_PHYGRP_PHYMGRGRP_ADDRESS; 32 33 static struct socfpga_phy_mgr_cfg *phy_mgr_cfg = 34 (struct socfpga_phy_mgr_cfg *)(SDR_PHYGRP_PHYMGRGRP_ADDRESS | 0x40); 35 36 static struct socfpga_data_mgr *data_mgr = 37 (struct socfpga_data_mgr *)SDR_PHYGRP_DATAMGRGRP_ADDRESS; 38 39 static struct socfpga_sdr_ctrl *sdr_ctrl = 40 (struct socfpga_sdr_ctrl *)SDR_CTRLGRP_ADDRESS; 41 42 #define DELTA_D 1 43 44 /* 45 * In order to reduce ROM size, most of the selectable calibration steps are 46 * decided at compile time based on the user's calibration mode selection, 47 * as captured by the STATIC_CALIB_STEPS selection below. 48 * 49 * However, to support simulation-time selection of fast simulation mode, where 50 * we skip everything except the bare minimum, we need a few of the steps to 51 * be dynamic. In those cases, we either use the DYNAMIC_CALIB_STEPS for the 52 * check, which is based on the rtl-supplied value, or we dynamically compute 53 * the value to use based on the dynamically-chosen calibration mode 54 */ 55 56 #define DLEVEL 0 57 #define STATIC_IN_RTL_SIM 0 58 #define STATIC_SKIP_DELAY_LOOPS 0 59 60 #define STATIC_CALIB_STEPS (STATIC_IN_RTL_SIM | CALIB_SKIP_FULL_TEST | \ 61 STATIC_SKIP_DELAY_LOOPS) 62 63 /* calibration steps requested by the rtl */ 64 uint16_t dyn_calib_steps; 65 66 /* 67 * To make CALIB_SKIP_DELAY_LOOPS a dynamic conditional option 68 * instead of static, we use boolean logic to select between 69 * non-skip and skip values 70 * 71 * The mask is set to include all bits when not-skipping, but is 72 * zero when skipping 73 */ 74 75 uint16_t skip_delay_mask; /* mask off bits when skipping/not-skipping */ 76 77 #define SKIP_DELAY_LOOP_VALUE_OR_ZERO(non_skip_value) \ 78 ((non_skip_value) & skip_delay_mask) 79 80 struct gbl_type *gbl; 81 struct param_type *param; 82 uint32_t curr_shadow_reg; 83 84 static uint32_t rw_mgr_mem_calibrate_write_test(uint32_t rank_bgn, 85 uint32_t write_group, uint32_t use_dm, 86 uint32_t all_correct, uint32_t *bit_chk, uint32_t all_ranks); 87 88 static void set_failing_group_stage(uint32_t group, uint32_t stage, 89 uint32_t substage) 90 { 91 /* 92 * Only set the global stage if there was not been any other 93 * failing group 94 */ 95 if (gbl->error_stage == CAL_STAGE_NIL) { 96 gbl->error_substage = substage; 97 gbl->error_stage = stage; 98 gbl->error_group = group; 99 } 100 } 101 102 static void reg_file_set_group(uint32_t set_group) 103 { 104 u32 addr = (u32)&sdr_reg_file->cur_stage; 105 106 /* Read the current group and stage */ 107 uint32_t cur_stage_group = readl(addr); 108 109 /* Clear the group */ 110 cur_stage_group &= 0x0000FFFF; 111 112 /* Set the group */ 113 cur_stage_group |= (set_group << 16); 114 115 /* Write the data back */ 116 writel(cur_stage_group, addr); 117 } 118 119 static void reg_file_set_stage(uint32_t set_stage) 120 { 121 u32 addr = (u32)&sdr_reg_file->cur_stage; 122 123 /* Read the current group and stage */ 124 uint32_t cur_stage_group = readl(addr); 125 126 /* Clear the stage and substage */ 127 cur_stage_group &= 0xFFFF0000; 128 129 /* Set the stage */ 130 cur_stage_group |= (set_stage & 0x000000FF); 131 132 /* Write the data back */ 133 writel(cur_stage_group, addr); 134 } 135 136 static void reg_file_set_sub_stage(uint32_t set_sub_stage) 137 { 138 u32 addr = (u32)&sdr_reg_file->cur_stage; 139 140 /* Read the current group and stage */ 141 uint32_t cur_stage_group = readl(addr); 142 143 /* Clear the substage */ 144 cur_stage_group &= 0xFFFF00FF; 145 146 /* Set the sub stage */ 147 cur_stage_group |= ((set_sub_stage << 8) & 0x0000FF00); 148 149 /* Write the data back */ 150 writel(cur_stage_group, addr); 151 } 152 153 static void initialize(void) 154 { 155 u32 addr = (u32)&phy_mgr_cfg->mux_sel; 156 157 debug("%s:%d\n", __func__, __LINE__); 158 /* USER calibration has control over path to memory */ 159 /* 160 * In Hard PHY this is a 2-bit control: 161 * 0: AFI Mux Select 162 * 1: DDIO Mux Select 163 */ 164 writel(0x3, addr); 165 166 /* USER memory clock is not stable we begin initialization */ 167 addr = (u32)&phy_mgr_cfg->reset_mem_stbl; 168 writel(0, addr); 169 170 /* USER calibration status all set to zero */ 171 addr = (u32)&phy_mgr_cfg->cal_status; 172 writel(0, addr); 173 174 addr = (u32)&phy_mgr_cfg->cal_debug_info; 175 writel(0, addr); 176 177 if ((dyn_calib_steps & CALIB_SKIP_ALL) != CALIB_SKIP_ALL) { 178 param->read_correct_mask_vg = ((uint32_t)1 << 179 (RW_MGR_MEM_DQ_PER_READ_DQS / 180 RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS)) - 1; 181 param->write_correct_mask_vg = ((uint32_t)1 << 182 (RW_MGR_MEM_DQ_PER_READ_DQS / 183 RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS)) - 1; 184 param->read_correct_mask = ((uint32_t)1 << 185 RW_MGR_MEM_DQ_PER_READ_DQS) - 1; 186 param->write_correct_mask = ((uint32_t)1 << 187 RW_MGR_MEM_DQ_PER_WRITE_DQS) - 1; 188 param->dm_correct_mask = ((uint32_t)1 << 189 (RW_MGR_MEM_DATA_WIDTH / RW_MGR_MEM_DATA_MASK_WIDTH)) 190 - 1; 191 } 192 } 193 194 static void set_rank_and_odt_mask(uint32_t rank, uint32_t odt_mode) 195 { 196 uint32_t odt_mask_0 = 0; 197 uint32_t odt_mask_1 = 0; 198 uint32_t cs_and_odt_mask; 199 uint32_t addr; 200 201 if (odt_mode == RW_MGR_ODT_MODE_READ_WRITE) { 202 if (RW_MGR_MEM_NUMBER_OF_RANKS == 1) { 203 /* 204 * 1 Rank 205 * Read: ODT = 0 206 * Write: ODT = 1 207 */ 208 odt_mask_0 = 0x0; 209 odt_mask_1 = 0x1; 210 } else if (RW_MGR_MEM_NUMBER_OF_RANKS == 2) { 211 /* 2 Ranks */ 212 if (RW_MGR_MEM_NUMBER_OF_CS_PER_DIMM == 1) { 213 /* - Dual-Slot , Single-Rank 214 * (1 chip-select per DIMM) 215 * OR 216 * - RDIMM, 4 total CS (2 CS per DIMM) 217 * means 2 DIMM 218 * Since MEM_NUMBER_OF_RANKS is 2 they are 219 * both single rank 220 * with 2 CS each (special for RDIMM) 221 * Read: Turn on ODT on the opposite rank 222 * Write: Turn on ODT on all ranks 223 */ 224 odt_mask_0 = 0x3 & ~(1 << rank); 225 odt_mask_1 = 0x3; 226 } else { 227 /* 228 * USER - Single-Slot , Dual-rank DIMMs 229 * (2 chip-selects per DIMM) 230 * USER Read: Turn on ODT off on all ranks 231 * USER Write: Turn on ODT on active rank 232 */ 233 odt_mask_0 = 0x0; 234 odt_mask_1 = 0x3 & (1 << rank); 235 } 236 } else { 237 /* 4 Ranks 238 * Read: 239 * ----------+-----------------------+ 240 * | | 241 * | ODT | 242 * Read From +-----------------------+ 243 * Rank | 3 | 2 | 1 | 0 | 244 * ----------+-----+-----+-----+-----+ 245 * 0 | 0 | 1 | 0 | 0 | 246 * 1 | 1 | 0 | 0 | 0 | 247 * 2 | 0 | 0 | 0 | 1 | 248 * 3 | 0 | 0 | 1 | 0 | 249 * ----------+-----+-----+-----+-----+ 250 * 251 * Write: 252 * ----------+-----------------------+ 253 * | | 254 * | ODT | 255 * Write To +-----------------------+ 256 * Rank | 3 | 2 | 1 | 0 | 257 * ----------+-----+-----+-----+-----+ 258 * 0 | 0 | 1 | 0 | 1 | 259 * 1 | 1 | 0 | 1 | 0 | 260 * 2 | 0 | 1 | 0 | 1 | 261 * 3 | 1 | 0 | 1 | 0 | 262 * ----------+-----+-----+-----+-----+ 263 */ 264 switch (rank) { 265 case 0: 266 odt_mask_0 = 0x4; 267 odt_mask_1 = 0x5; 268 break; 269 case 1: 270 odt_mask_0 = 0x8; 271 odt_mask_1 = 0xA; 272 break; 273 case 2: 274 odt_mask_0 = 0x1; 275 odt_mask_1 = 0x5; 276 break; 277 case 3: 278 odt_mask_0 = 0x2; 279 odt_mask_1 = 0xA; 280 break; 281 } 282 } 283 } else { 284 odt_mask_0 = 0x0; 285 odt_mask_1 = 0x0; 286 } 287 288 cs_and_odt_mask = 289 (0xFF & ~(1 << rank)) | 290 ((0xFF & odt_mask_0) << 8) | 291 ((0xFF & odt_mask_1) << 16); 292 addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_SET_CS_AND_ODT_MASK_OFFSET; 293 writel(cs_and_odt_mask, addr); 294 } 295 296 static void scc_mgr_initialize(void) 297 { 298 u32 addr = SDR_PHYGRP_SCCGRP_ADDRESS | SCC_MGR_HHP_RFILE_OFFSET; 299 300 /* 301 * Clear register file for HPS 302 * 16 (2^4) is the size of the full register file in the scc mgr: 303 * RFILE_DEPTH = log2(MEM_DQ_PER_DQS + 1 + MEM_DM_PER_DQS + 304 * MEM_IF_READ_DQS_WIDTH - 1) + 1; 305 */ 306 uint32_t i; 307 for (i = 0; i < 16; i++) { 308 debug_cond(DLEVEL == 1, "%s:%d: Clearing SCC RFILE index %u\n", 309 __func__, __LINE__, i); 310 writel(0, addr + (i << 2)); 311 } 312 } 313 314 static void scc_mgr_set_dqs_bus_in_delay(uint32_t read_group, 315 uint32_t delay) 316 { 317 u32 addr = SDR_PHYGRP_SCCGRP_ADDRESS | SCC_MGR_DQS_IN_DELAY_OFFSET; 318 319 /* Load the setting in the SCC manager */ 320 writel(delay, addr + (read_group << 2)); 321 } 322 323 static void scc_mgr_set_dqs_io_in_delay(uint32_t write_group, 324 uint32_t delay) 325 { 326 u32 addr = SDR_PHYGRP_SCCGRP_ADDRESS | SCC_MGR_IO_IN_DELAY_OFFSET; 327 328 writel(delay, addr + (RW_MGR_MEM_DQ_PER_WRITE_DQS << 2)); 329 } 330 331 static void scc_mgr_set_dqs_en_phase(uint32_t read_group, uint32_t phase) 332 { 333 u32 addr = SDR_PHYGRP_SCCGRP_ADDRESS | SCC_MGR_DQS_EN_PHASE_OFFSET; 334 335 /* Load the setting in the SCC manager */ 336 writel(phase, addr + (read_group << 2)); 337 } 338 339 static void scc_mgr_set_dqs_en_phase_all_ranks(uint32_t read_group, 340 uint32_t phase) 341 { 342 uint32_t r; 343 uint32_t update_scan_chains; 344 uint32_t addr; 345 346 for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; 347 r += NUM_RANKS_PER_SHADOW_REG) { 348 /* 349 * USER although the h/w doesn't support different phases per 350 * shadow register, for simplicity our scc manager modeling 351 * keeps different phase settings per shadow reg, and it's 352 * important for us to keep them in sync to match h/w. 353 * for efficiency, the scan chain update should occur only 354 * once to sr0. 355 */ 356 update_scan_chains = (r == 0) ? 1 : 0; 357 358 scc_mgr_set_dqs_en_phase(read_group, phase); 359 360 if (update_scan_chains) { 361 addr = (u32)&sdr_scc_mgr->dqs_ena; 362 writel(read_group, addr); 363 364 addr = (u32)&sdr_scc_mgr->update; 365 writel(0, addr); 366 } 367 } 368 } 369 370 static void scc_mgr_set_dqdqs_output_phase(uint32_t write_group, 371 uint32_t phase) 372 { 373 u32 addr = SDR_PHYGRP_SCCGRP_ADDRESS | SCC_MGR_DQDQS_OUT_PHASE_OFFSET; 374 375 /* Load the setting in the SCC manager */ 376 writel(phase, addr + (write_group << 2)); 377 } 378 379 static void scc_mgr_set_dqdqs_output_phase_all_ranks(uint32_t write_group, 380 uint32_t phase) 381 { 382 uint32_t r; 383 uint32_t update_scan_chains; 384 uint32_t addr; 385 386 for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; 387 r += NUM_RANKS_PER_SHADOW_REG) { 388 /* 389 * USER although the h/w doesn't support different phases per 390 * shadow register, for simplicity our scc manager modeling 391 * keeps different phase settings per shadow reg, and it's 392 * important for us to keep them in sync to match h/w. 393 * for efficiency, the scan chain update should occur only 394 * once to sr0. 395 */ 396 update_scan_chains = (r == 0) ? 1 : 0; 397 398 scc_mgr_set_dqdqs_output_phase(write_group, phase); 399 400 if (update_scan_chains) { 401 addr = (u32)&sdr_scc_mgr->dqs_ena; 402 writel(write_group, addr); 403 404 addr = (u32)&sdr_scc_mgr->update; 405 writel(0, addr); 406 } 407 } 408 } 409 410 static void scc_mgr_set_dqs_en_delay(uint32_t read_group, uint32_t delay) 411 { 412 uint32_t addr = SDR_PHYGRP_SCCGRP_ADDRESS | SCC_MGR_DQS_EN_DELAY_OFFSET; 413 414 /* Load the setting in the SCC manager */ 415 writel(delay + IO_DQS_EN_DELAY_OFFSET, addr + 416 (read_group << 2)); 417 } 418 419 static void scc_mgr_set_dqs_en_delay_all_ranks(uint32_t read_group, 420 uint32_t delay) 421 { 422 uint32_t r; 423 uint32_t addr; 424 425 for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; 426 r += NUM_RANKS_PER_SHADOW_REG) { 427 scc_mgr_set_dqs_en_delay(read_group, delay); 428 429 addr = (u32)&sdr_scc_mgr->dqs_ena; 430 writel(read_group, addr); 431 /* 432 * In shadow register mode, the T11 settings are stored in 433 * registers in the core, which are updated by the DQS_ENA 434 * signals. Not issuing the SCC_MGR_UPD command allows us to 435 * save lots of rank switching overhead, by calling 436 * select_shadow_regs_for_update with update_scan_chains 437 * set to 0. 438 */ 439 addr = (u32)&sdr_scc_mgr->update; 440 writel(0, addr); 441 } 442 /* 443 * In shadow register mode, the T11 settings are stored in 444 * registers in the core, which are updated by the DQS_ENA 445 * signals. Not issuing the SCC_MGR_UPD command allows us to 446 * save lots of rank switching overhead, by calling 447 * select_shadow_regs_for_update with update_scan_chains 448 * set to 0. 449 */ 450 addr = (u32)&sdr_scc_mgr->update; 451 writel(0, addr); 452 } 453 454 static void scc_mgr_set_oct_out1_delay(uint32_t write_group, uint32_t delay) 455 { 456 uint32_t read_group; 457 uint32_t addr = SDR_PHYGRP_SCCGRP_ADDRESS | SCC_MGR_OCT_OUT1_DELAY_OFFSET; 458 459 /* 460 * Load the setting in the SCC manager 461 * Although OCT affects only write data, the OCT delay is controlled 462 * by the DQS logic block which is instantiated once per read group. 463 * For protocols where a write group consists of multiple read groups, 464 * the setting must be set multiple times. 465 */ 466 for (read_group = write_group * RW_MGR_MEM_IF_READ_DQS_WIDTH / 467 RW_MGR_MEM_IF_WRITE_DQS_WIDTH; 468 read_group < (write_group + 1) * RW_MGR_MEM_IF_READ_DQS_WIDTH / 469 RW_MGR_MEM_IF_WRITE_DQS_WIDTH; ++read_group) 470 writel(delay, addr + (read_group << 2)); 471 } 472 473 static void scc_mgr_set_dq_out1_delay(uint32_t write_group, 474 uint32_t dq_in_group, uint32_t delay) 475 { 476 uint32_t addr = SDR_PHYGRP_SCCGRP_ADDRESS | SCC_MGR_IO_OUT1_DELAY_OFFSET; 477 478 /* Load the setting in the SCC manager */ 479 writel(delay, addr + (dq_in_group << 2)); 480 } 481 482 static void scc_mgr_set_dq_in_delay(uint32_t write_group, 483 uint32_t dq_in_group, uint32_t delay) 484 { 485 uint32_t addr = SDR_PHYGRP_SCCGRP_ADDRESS | SCC_MGR_IO_IN_DELAY_OFFSET; 486 487 /* Load the setting in the SCC manager */ 488 writel(delay, addr + (dq_in_group << 2)); 489 } 490 491 static void scc_mgr_set_hhp_extras(void) 492 { 493 /* 494 * Load the fixed setting in the SCC manager 495 * bits: 0:0 = 1'b1 - dqs bypass 496 * bits: 1:1 = 1'b1 - dq bypass 497 * bits: 4:2 = 3'b001 - rfifo_mode 498 * bits: 6:5 = 2'b01 - rfifo clock_select 499 * bits: 7:7 = 1'b0 - separate gating from ungating setting 500 * bits: 8:8 = 1'b0 - separate OE from Output delay setting 501 */ 502 uint32_t value = (0<<8) | (0<<7) | (1<<5) | (1<<2) | (1<<1) | (1<<0); 503 uint32_t addr = SDR_PHYGRP_SCCGRP_ADDRESS | SCC_MGR_HHP_GLOBALS_OFFSET; 504 505 writel(value, addr + SCC_MGR_HHP_EXTRAS_OFFSET); 506 } 507 508 static void scc_mgr_set_dqs_out1_delay(uint32_t write_group, 509 uint32_t delay) 510 { 511 uint32_t addr = SDR_PHYGRP_SCCGRP_ADDRESS | SCC_MGR_IO_OUT1_DELAY_OFFSET; 512 513 /* Load the setting in the SCC manager */ 514 writel(delay, addr + (RW_MGR_MEM_DQ_PER_WRITE_DQS << 2)); 515 } 516 517 static void scc_mgr_set_dm_out1_delay(uint32_t write_group, 518 uint32_t dm, uint32_t delay) 519 { 520 uint32_t addr = SDR_PHYGRP_SCCGRP_ADDRESS | SCC_MGR_IO_OUT1_DELAY_OFFSET; 521 522 /* Load the setting in the SCC manager */ 523 writel(delay, addr + 524 ((RW_MGR_MEM_DQ_PER_WRITE_DQS + 1 + dm) << 2)); 525 } 526 527 /* 528 * USER Zero all DQS config 529 * TODO: maybe rename to scc_mgr_zero_dqs_config (or something) 530 */ 531 static void scc_mgr_zero_all(void) 532 { 533 uint32_t i, r; 534 uint32_t addr; 535 536 /* 537 * USER Zero all DQS config settings, across all groups and all 538 * shadow registers 539 */ 540 for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; r += 541 NUM_RANKS_PER_SHADOW_REG) { 542 for (i = 0; i < RW_MGR_MEM_IF_READ_DQS_WIDTH; i++) { 543 /* 544 * The phases actually don't exist on a per-rank basis, 545 * but there's no harm updating them several times, so 546 * let's keep the code simple. 547 */ 548 scc_mgr_set_dqs_bus_in_delay(i, IO_DQS_IN_RESERVE); 549 scc_mgr_set_dqs_en_phase(i, 0); 550 scc_mgr_set_dqs_en_delay(i, 0); 551 } 552 553 for (i = 0; i < RW_MGR_MEM_IF_WRITE_DQS_WIDTH; i++) { 554 scc_mgr_set_dqdqs_output_phase(i, 0); 555 /* av/cv don't have out2 */ 556 scc_mgr_set_oct_out1_delay(i, IO_DQS_OUT_RESERVE); 557 } 558 } 559 560 /* multicast to all DQS group enables */ 561 addr = (u32)&sdr_scc_mgr->dqs_ena; 562 writel(0xff, addr); 563 564 addr = (u32)&sdr_scc_mgr->update; 565 writel(0, addr); 566 } 567 568 static void scc_set_bypass_mode(uint32_t write_group, uint32_t mode) 569 { 570 uint32_t addr; 571 /* mode = 0 : Do NOT bypass - Half Rate Mode */ 572 /* mode = 1 : Bypass - Full Rate Mode */ 573 574 /* only need to set once for all groups, pins, dq, dqs, dm */ 575 if (write_group == 0) { 576 debug_cond(DLEVEL == 1, "%s:%d Setting HHP Extras\n", __func__, 577 __LINE__); 578 scc_mgr_set_hhp_extras(); 579 debug_cond(DLEVEL == 1, "%s:%d Done Setting HHP Extras\n", 580 __func__, __LINE__); 581 } 582 /* multicast to all DQ enables */ 583 addr = (u32)&sdr_scc_mgr->dq_ena; 584 writel(0xff, addr); 585 586 addr = (u32)&sdr_scc_mgr->dm_ena; 587 writel(0xff, addr); 588 589 /* update current DQS IO enable */ 590 addr = (u32)&sdr_scc_mgr->dqs_io_ena; 591 writel(0, addr); 592 593 /* update the DQS logic */ 594 addr = (u32)&sdr_scc_mgr->dqs_ena; 595 writel(write_group, addr); 596 597 /* hit update */ 598 addr = (u32)&sdr_scc_mgr->update; 599 writel(0, addr); 600 } 601 602 static void scc_mgr_zero_group(uint32_t write_group, uint32_t test_begin, 603 int32_t out_only) 604 { 605 uint32_t i, r; 606 uint32_t addr; 607 608 for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; r += 609 NUM_RANKS_PER_SHADOW_REG) { 610 /* Zero all DQ config settings */ 611 for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) { 612 scc_mgr_set_dq_out1_delay(write_group, i, 0); 613 if (!out_only) 614 scc_mgr_set_dq_in_delay(write_group, i, 0); 615 } 616 617 /* multicast to all DQ enables */ 618 addr = (u32)&sdr_scc_mgr->dq_ena; 619 writel(0xff, addr); 620 621 /* Zero all DM config settings */ 622 for (i = 0; i < RW_MGR_NUM_DM_PER_WRITE_GROUP; i++) { 623 scc_mgr_set_dm_out1_delay(write_group, i, 0); 624 } 625 626 /* multicast to all DM enables */ 627 addr = (u32)&sdr_scc_mgr->dm_ena; 628 writel(0xff, addr); 629 630 /* zero all DQS io settings */ 631 if (!out_only) 632 scc_mgr_set_dqs_io_in_delay(write_group, 0); 633 /* av/cv don't have out2 */ 634 scc_mgr_set_dqs_out1_delay(write_group, IO_DQS_OUT_RESERVE); 635 scc_mgr_set_oct_out1_delay(write_group, IO_DQS_OUT_RESERVE); 636 scc_mgr_load_dqs_for_write_group(write_group); 637 638 /* multicast to all DQS IO enables (only 1) */ 639 addr = (u32)&sdr_scc_mgr->dqs_io_ena; 640 writel(0, addr); 641 642 /* hit update to zero everything */ 643 addr = (u32)&sdr_scc_mgr->update; 644 writel(0, addr); 645 } 646 } 647 648 /* load up dqs config settings */ 649 static void scc_mgr_load_dqs(uint32_t dqs) 650 { 651 uint32_t addr = (u32)&sdr_scc_mgr->dqs_ena; 652 653 writel(dqs, addr); 654 } 655 656 static void scc_mgr_load_dqs_for_write_group(uint32_t write_group) 657 { 658 uint32_t read_group; 659 uint32_t addr = (u32)&sdr_scc_mgr->dqs_ena; 660 /* 661 * Although OCT affects only write data, the OCT delay is controlled 662 * by the DQS logic block which is instantiated once per read group. 663 * For protocols where a write group consists of multiple read groups, 664 * the setting must be scanned multiple times. 665 */ 666 for (read_group = write_group * RW_MGR_MEM_IF_READ_DQS_WIDTH / 667 RW_MGR_MEM_IF_WRITE_DQS_WIDTH; 668 read_group < (write_group + 1) * RW_MGR_MEM_IF_READ_DQS_WIDTH / 669 RW_MGR_MEM_IF_WRITE_DQS_WIDTH; ++read_group) 670 writel(read_group, addr); 671 } 672 673 /* load up dqs io config settings */ 674 static void scc_mgr_load_dqs_io(void) 675 { 676 uint32_t addr = (u32)&sdr_scc_mgr->dqs_io_ena; 677 678 writel(0, addr); 679 } 680 681 /* load up dq config settings */ 682 static void scc_mgr_load_dq(uint32_t dq_in_group) 683 { 684 uint32_t addr = (u32)&sdr_scc_mgr->dq_ena; 685 686 writel(dq_in_group, addr); 687 } 688 689 /* load up dm config settings */ 690 static void scc_mgr_load_dm(uint32_t dm) 691 { 692 uint32_t addr = (u32)&sdr_scc_mgr->dm_ena; 693 694 writel(dm, addr); 695 } 696 697 /* 698 * apply and load a particular input delay for the DQ pins in a group 699 * group_bgn is the index of the first dq pin (in the write group) 700 */ 701 static void scc_mgr_apply_group_dq_in_delay(uint32_t write_group, 702 uint32_t group_bgn, uint32_t delay) 703 { 704 uint32_t i, p; 705 706 for (i = 0, p = group_bgn; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++, p++) { 707 scc_mgr_set_dq_in_delay(write_group, p, delay); 708 scc_mgr_load_dq(p); 709 } 710 } 711 712 /* apply and load a particular output delay for the DQ pins in a group */ 713 static void scc_mgr_apply_group_dq_out1_delay(uint32_t write_group, 714 uint32_t group_bgn, 715 uint32_t delay1) 716 { 717 uint32_t i, p; 718 719 for (i = 0, p = group_bgn; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++, p++) { 720 scc_mgr_set_dq_out1_delay(write_group, i, delay1); 721 scc_mgr_load_dq(i); 722 } 723 } 724 725 /* apply and load a particular output delay for the DM pins in a group */ 726 static void scc_mgr_apply_group_dm_out1_delay(uint32_t write_group, 727 uint32_t delay1) 728 { 729 uint32_t i; 730 731 for (i = 0; i < RW_MGR_NUM_DM_PER_WRITE_GROUP; i++) { 732 scc_mgr_set_dm_out1_delay(write_group, i, delay1); 733 scc_mgr_load_dm(i); 734 } 735 } 736 737 738 /* apply and load delay on both DQS and OCT out1 */ 739 static void scc_mgr_apply_group_dqs_io_and_oct_out1(uint32_t write_group, 740 uint32_t delay) 741 { 742 scc_mgr_set_dqs_out1_delay(write_group, delay); 743 scc_mgr_load_dqs_io(); 744 745 scc_mgr_set_oct_out1_delay(write_group, delay); 746 scc_mgr_load_dqs_for_write_group(write_group); 747 } 748 749 /* apply a delay to the entire output side: DQ, DM, DQS, OCT */ 750 static void scc_mgr_apply_group_all_out_delay_add(uint32_t write_group, 751 uint32_t group_bgn, 752 uint32_t delay) 753 { 754 uint32_t i, p, new_delay; 755 756 /* dq shift */ 757 for (i = 0, p = group_bgn; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++, p++) { 758 new_delay = READ_SCC_DQ_OUT2_DELAY; 759 new_delay += delay; 760 761 if (new_delay > IO_IO_OUT2_DELAY_MAX) { 762 debug_cond(DLEVEL == 1, "%s:%d (%u, %u, %u) DQ[%u,%u]:\ 763 %u > %lu => %lu", __func__, __LINE__, 764 write_group, group_bgn, delay, i, p, new_delay, 765 (long unsigned int)IO_IO_OUT2_DELAY_MAX, 766 (long unsigned int)IO_IO_OUT2_DELAY_MAX); 767 new_delay = IO_IO_OUT2_DELAY_MAX; 768 } 769 770 scc_mgr_load_dq(i); 771 } 772 773 /* dm shift */ 774 for (i = 0; i < RW_MGR_NUM_DM_PER_WRITE_GROUP; i++) { 775 new_delay = READ_SCC_DM_IO_OUT2_DELAY; 776 new_delay += delay; 777 778 if (new_delay > IO_IO_OUT2_DELAY_MAX) { 779 debug_cond(DLEVEL == 1, "%s:%d (%u, %u, %u) DM[%u]:\ 780 %u > %lu => %lu\n", __func__, __LINE__, 781 write_group, group_bgn, delay, i, new_delay, 782 (long unsigned int)IO_IO_OUT2_DELAY_MAX, 783 (long unsigned int)IO_IO_OUT2_DELAY_MAX); 784 new_delay = IO_IO_OUT2_DELAY_MAX; 785 } 786 787 scc_mgr_load_dm(i); 788 } 789 790 /* dqs shift */ 791 new_delay = READ_SCC_DQS_IO_OUT2_DELAY; 792 new_delay += delay; 793 794 if (new_delay > IO_IO_OUT2_DELAY_MAX) { 795 debug_cond(DLEVEL == 1, "%s:%d (%u, %u, %u) DQS: %u > %d => %d;" 796 " adding %u to OUT1\n", __func__, __LINE__, 797 write_group, group_bgn, delay, new_delay, 798 IO_IO_OUT2_DELAY_MAX, IO_IO_OUT2_DELAY_MAX, 799 new_delay - IO_IO_OUT2_DELAY_MAX); 800 scc_mgr_set_dqs_out1_delay(write_group, new_delay - 801 IO_IO_OUT2_DELAY_MAX); 802 new_delay = IO_IO_OUT2_DELAY_MAX; 803 } 804 805 scc_mgr_load_dqs_io(); 806 807 /* oct shift */ 808 new_delay = READ_SCC_OCT_OUT2_DELAY; 809 new_delay += delay; 810 811 if (new_delay > IO_IO_OUT2_DELAY_MAX) { 812 debug_cond(DLEVEL == 1, "%s:%d (%u, %u, %u) DQS: %u > %d => %d;" 813 " adding %u to OUT1\n", __func__, __LINE__, 814 write_group, group_bgn, delay, new_delay, 815 IO_IO_OUT2_DELAY_MAX, IO_IO_OUT2_DELAY_MAX, 816 new_delay - IO_IO_OUT2_DELAY_MAX); 817 scc_mgr_set_oct_out1_delay(write_group, new_delay - 818 IO_IO_OUT2_DELAY_MAX); 819 new_delay = IO_IO_OUT2_DELAY_MAX; 820 } 821 822 scc_mgr_load_dqs_for_write_group(write_group); 823 } 824 825 /* 826 * USER apply a delay to the entire output side (DQ, DM, DQS, OCT) 827 * and to all ranks 828 */ 829 static void scc_mgr_apply_group_all_out_delay_add_all_ranks( 830 uint32_t write_group, uint32_t group_bgn, uint32_t delay) 831 { 832 uint32_t r; 833 uint32_t addr = (u32)&sdr_scc_mgr->update; 834 835 for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; 836 r += NUM_RANKS_PER_SHADOW_REG) { 837 scc_mgr_apply_group_all_out_delay_add(write_group, 838 group_bgn, delay); 839 writel(0, addr); 840 } 841 } 842 843 /* optimization used to recover some slots in ddr3 inst_rom */ 844 /* could be applied to other protocols if we wanted to */ 845 static void set_jump_as_return(void) 846 { 847 uint32_t addr = (u32)&sdr_rw_load_mgr_regs->load_cntr0; 848 849 /* 850 * to save space, we replace return with jump to special shared 851 * RETURN instruction so we set the counter to large value so that 852 * we always jump 853 */ 854 writel(0xff, addr); 855 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add0; 856 writel(RW_MGR_RETURN, addr); 857 } 858 859 /* 860 * should always use constants as argument to ensure all computations are 861 * performed at compile time 862 */ 863 static void delay_for_n_mem_clocks(const uint32_t clocks) 864 { 865 uint32_t afi_clocks; 866 uint8_t inner = 0; 867 uint8_t outer = 0; 868 uint16_t c_loop = 0; 869 uint32_t addr; 870 871 debug("%s:%d: clocks=%u ... start\n", __func__, __LINE__, clocks); 872 873 874 afi_clocks = (clocks + AFI_RATE_RATIO-1) / AFI_RATE_RATIO; 875 /* scale (rounding up) to get afi clocks */ 876 877 /* 878 * Note, we don't bother accounting for being off a little bit 879 * because of a few extra instructions in outer loops 880 * Note, the loops have a test at the end, and do the test before 881 * the decrement, and so always perform the loop 882 * 1 time more than the counter value 883 */ 884 if (afi_clocks == 0) { 885 ; 886 } else if (afi_clocks <= 0x100) { 887 inner = afi_clocks-1; 888 outer = 0; 889 c_loop = 0; 890 } else if (afi_clocks <= 0x10000) { 891 inner = 0xff; 892 outer = (afi_clocks-1) >> 8; 893 c_loop = 0; 894 } else { 895 inner = 0xff; 896 outer = 0xff; 897 c_loop = (afi_clocks-1) >> 16; 898 } 899 900 /* 901 * rom instructions are structured as follows: 902 * 903 * IDLE_LOOP2: jnz cntr0, TARGET_A 904 * IDLE_LOOP1: jnz cntr1, TARGET_B 905 * return 906 * 907 * so, when doing nested loops, TARGET_A is set to IDLE_LOOP2, and 908 * TARGET_B is set to IDLE_LOOP2 as well 909 * 910 * if we have no outer loop, though, then we can use IDLE_LOOP1 only, 911 * and set TARGET_B to IDLE_LOOP1 and we skip IDLE_LOOP2 entirely 912 * 913 * a little confusing, but it helps save precious space in the inst_rom 914 * and sequencer rom and keeps the delays more accurate and reduces 915 * overhead 916 */ 917 if (afi_clocks <= 0x100) { 918 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr1; 919 writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(inner), addr); 920 921 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add1; 922 writel(RW_MGR_IDLE_LOOP1, addr); 923 924 addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_SINGLE_GROUP_OFFSET; 925 writel(RW_MGR_IDLE_LOOP1, addr); 926 } else { 927 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr0; 928 writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(inner), addr); 929 930 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr1; 931 writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(outer), addr); 932 933 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add0; 934 writel(RW_MGR_IDLE_LOOP2, addr); 935 936 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add1; 937 writel(RW_MGR_IDLE_LOOP2, addr); 938 939 /* hack to get around compiler not being smart enough */ 940 if (afi_clocks <= 0x10000) { 941 /* only need to run once */ 942 addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_SINGLE_GROUP_OFFSET; 943 writel(RW_MGR_IDLE_LOOP2, addr); 944 } else { 945 do { 946 addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_SINGLE_GROUP_OFFSET; 947 writel(RW_MGR_IDLE_LOOP2, addr); 948 } while (c_loop-- != 0); 949 } 950 } 951 debug("%s:%d clocks=%u ... end\n", __func__, __LINE__, clocks); 952 } 953 954 static void rw_mgr_mem_initialize(void) 955 { 956 uint32_t r; 957 uint32_t addr; 958 959 debug("%s:%d\n", __func__, __LINE__); 960 961 /* The reset / cke part of initialization is broadcasted to all ranks */ 962 addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_SET_CS_AND_ODT_MASK_OFFSET; 963 writel(RW_MGR_RANK_ALL, addr); 964 965 /* 966 * Here's how you load register for a loop 967 * Counters are located @ 0x800 968 * Jump address are located @ 0xC00 969 * For both, registers 0 to 3 are selected using bits 3 and 2, like 970 * in 0x800, 0x804, 0x808, 0x80C and 0xC00, 0xC04, 0xC08, 0xC0C 971 * I know this ain't pretty, but Avalon bus throws away the 2 least 972 * significant bits 973 */ 974 975 /* start with memory RESET activated */ 976 977 /* tINIT = 200us */ 978 979 /* 980 * 200us @ 266MHz (3.75 ns) ~ 54000 clock cycles 981 * If a and b are the number of iteration in 2 nested loops 982 * it takes the following number of cycles to complete the operation: 983 * number_of_cycles = ((2 + n) * a + 2) * b 984 * where n is the number of instruction in the inner loop 985 * One possible solution is n = 0 , a = 256 , b = 106 => a = FF, 986 * b = 6A 987 */ 988 989 /* Load counters */ 990 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr0; 991 writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(SEQ_TINIT_CNTR0_VAL), 992 addr); 993 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr1; 994 writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(SEQ_TINIT_CNTR1_VAL), 995 addr); 996 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr2; 997 writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(SEQ_TINIT_CNTR2_VAL), 998 addr); 999 1000 /* Load jump address */ 1001 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add0; 1002 writel(RW_MGR_INIT_RESET_0_CKE_0, addr); 1003 1004 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add1; 1005 writel(RW_MGR_INIT_RESET_0_CKE_0, addr); 1006 1007 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add2; 1008 writel(RW_MGR_INIT_RESET_0_CKE_0, addr); 1009 1010 /* Execute count instruction */ 1011 addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_SINGLE_GROUP_OFFSET; 1012 writel(RW_MGR_INIT_RESET_0_CKE_0, addr); 1013 1014 /* indicate that memory is stable */ 1015 addr = (u32)&phy_mgr_cfg->reset_mem_stbl; 1016 writel(1, addr); 1017 1018 /* 1019 * transition the RESET to high 1020 * Wait for 500us 1021 */ 1022 1023 /* 1024 * 500us @ 266MHz (3.75 ns) ~ 134000 clock cycles 1025 * If a and b are the number of iteration in 2 nested loops 1026 * it takes the following number of cycles to complete the operation 1027 * number_of_cycles = ((2 + n) * a + 2) * b 1028 * where n is the number of instruction in the inner loop 1029 * One possible solution is n = 2 , a = 131 , b = 256 => a = 83, 1030 * b = FF 1031 */ 1032 1033 /* Load counters */ 1034 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr0; 1035 writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(SEQ_TRESET_CNTR0_VAL), 1036 addr); 1037 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr1; 1038 writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(SEQ_TRESET_CNTR1_VAL), 1039 addr); 1040 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr2; 1041 writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(SEQ_TRESET_CNTR2_VAL), 1042 addr); 1043 1044 /* Load jump address */ 1045 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add0; 1046 writel(RW_MGR_INIT_RESET_1_CKE_0, addr); 1047 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add1; 1048 writel(RW_MGR_INIT_RESET_1_CKE_0, addr); 1049 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add2; 1050 writel(RW_MGR_INIT_RESET_1_CKE_0, addr); 1051 1052 addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_SINGLE_GROUP_OFFSET; 1053 writel(RW_MGR_INIT_RESET_1_CKE_0, addr); 1054 1055 /* bring up clock enable */ 1056 1057 /* tXRP < 250 ck cycles */ 1058 delay_for_n_mem_clocks(250); 1059 1060 for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; r++) { 1061 if (param->skip_ranks[r]) { 1062 /* request to skip the rank */ 1063 continue; 1064 } 1065 1066 /* set rank */ 1067 set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_OFF); 1068 1069 /* 1070 * USER Use Mirror-ed commands for odd ranks if address 1071 * mirrorring is on 1072 */ 1073 if ((RW_MGR_MEM_ADDRESS_MIRRORING >> r) & 0x1) { 1074 set_jump_as_return(); 1075 addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_SINGLE_GROUP_OFFSET; 1076 writel(RW_MGR_MRS2_MIRR, addr); 1077 delay_for_n_mem_clocks(4); 1078 set_jump_as_return(); 1079 writel(RW_MGR_MRS3_MIRR, addr); 1080 delay_for_n_mem_clocks(4); 1081 set_jump_as_return(); 1082 writel(RW_MGR_MRS1_MIRR, addr); 1083 delay_for_n_mem_clocks(4); 1084 set_jump_as_return(); 1085 writel(RW_MGR_MRS0_DLL_RESET_MIRR, addr); 1086 } else { 1087 set_jump_as_return(); 1088 addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_SINGLE_GROUP_OFFSET; 1089 writel(RW_MGR_MRS2, addr); 1090 delay_for_n_mem_clocks(4); 1091 set_jump_as_return(); 1092 writel(RW_MGR_MRS3, addr); 1093 delay_for_n_mem_clocks(4); 1094 set_jump_as_return(); 1095 writel(RW_MGR_MRS1, addr); 1096 set_jump_as_return(); 1097 writel(RW_MGR_MRS0_DLL_RESET, addr); 1098 } 1099 set_jump_as_return(); 1100 addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_SINGLE_GROUP_OFFSET; 1101 writel(RW_MGR_ZQCL, addr); 1102 1103 /* tZQinit = tDLLK = 512 ck cycles */ 1104 delay_for_n_mem_clocks(512); 1105 } 1106 } 1107 1108 /* 1109 * At the end of calibration we have to program the user settings in, and 1110 * USER hand off the memory to the user. 1111 */ 1112 static void rw_mgr_mem_handoff(void) 1113 { 1114 uint32_t r; 1115 uint32_t addr; 1116 1117 debug("%s:%d\n", __func__, __LINE__); 1118 for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; r++) { 1119 if (param->skip_ranks[r]) 1120 /* request to skip the rank */ 1121 continue; 1122 /* set rank */ 1123 set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_OFF); 1124 1125 /* precharge all banks ... */ 1126 addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_SINGLE_GROUP_OFFSET; 1127 writel(RW_MGR_PRECHARGE_ALL, addr); 1128 1129 /* load up MR settings specified by user */ 1130 1131 /* 1132 * Use Mirror-ed commands for odd ranks if address 1133 * mirrorring is on 1134 */ 1135 addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_SINGLE_GROUP_OFFSET; 1136 if ((RW_MGR_MEM_ADDRESS_MIRRORING >> r) & 0x1) { 1137 set_jump_as_return(); 1138 writel(RW_MGR_MRS2_MIRR, addr); 1139 delay_for_n_mem_clocks(4); 1140 set_jump_as_return(); 1141 writel(RW_MGR_MRS3_MIRR, addr); 1142 delay_for_n_mem_clocks(4); 1143 set_jump_as_return(); 1144 writel(RW_MGR_MRS1_MIRR, addr); 1145 delay_for_n_mem_clocks(4); 1146 set_jump_as_return(); 1147 writel(RW_MGR_MRS0_USER_MIRR, addr); 1148 } else { 1149 set_jump_as_return(); 1150 writel(RW_MGR_MRS2, addr); 1151 delay_for_n_mem_clocks(4); 1152 set_jump_as_return(); 1153 writel(RW_MGR_MRS3, addr); 1154 delay_for_n_mem_clocks(4); 1155 set_jump_as_return(); 1156 writel(RW_MGR_MRS1, addr); 1157 delay_for_n_mem_clocks(4); 1158 set_jump_as_return(); 1159 writel(RW_MGR_MRS0_USER, addr); 1160 } 1161 /* 1162 * USER need to wait tMOD (12CK or 15ns) time before issuing 1163 * other commands, but we will have plenty of NIOS cycles before 1164 * actual handoff so its okay. 1165 */ 1166 } 1167 } 1168 1169 /* 1170 * performs a guaranteed read on the patterns we are going to use during a 1171 * read test to ensure memory works 1172 */ 1173 static uint32_t rw_mgr_mem_calibrate_read_test_patterns(uint32_t rank_bgn, 1174 uint32_t group, uint32_t num_tries, uint32_t *bit_chk, 1175 uint32_t all_ranks) 1176 { 1177 uint32_t r, vg; 1178 uint32_t correct_mask_vg; 1179 uint32_t tmp_bit_chk; 1180 uint32_t rank_end = all_ranks ? RW_MGR_MEM_NUMBER_OF_RANKS : 1181 (rank_bgn + NUM_RANKS_PER_SHADOW_REG); 1182 uint32_t addr; 1183 uint32_t base_rw_mgr; 1184 1185 *bit_chk = param->read_correct_mask; 1186 correct_mask_vg = param->read_correct_mask_vg; 1187 1188 for (r = rank_bgn; r < rank_end; r++) { 1189 if (param->skip_ranks[r]) 1190 /* request to skip the rank */ 1191 continue; 1192 1193 /* set rank */ 1194 set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_READ_WRITE); 1195 1196 /* Load up a constant bursts of read commands */ 1197 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr0; 1198 writel(0x20, addr); 1199 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add0; 1200 writel(RW_MGR_GUARANTEED_READ, addr); 1201 1202 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr1; 1203 writel(0x20, addr); 1204 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add1; 1205 writel(RW_MGR_GUARANTEED_READ_CONT, addr); 1206 1207 tmp_bit_chk = 0; 1208 for (vg = RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS-1; ; vg--) { 1209 /* reset the fifos to get pointers to known state */ 1210 1211 addr = (u32)&phy_mgr_cmd->fifo_reset; 1212 writel(0, addr); 1213 addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RESET_READ_DATAPATH_OFFSET; 1214 writel(0, addr); 1215 1216 tmp_bit_chk = tmp_bit_chk << (RW_MGR_MEM_DQ_PER_READ_DQS 1217 / RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS); 1218 1219 addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_SINGLE_GROUP_OFFSET; 1220 writel(RW_MGR_GUARANTEED_READ, addr + 1221 ((group * RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS + 1222 vg) << 2)); 1223 1224 addr = SDR_PHYGRP_RWMGRGRP_ADDRESS; 1225 base_rw_mgr = readl(addr); 1226 tmp_bit_chk = tmp_bit_chk | (correct_mask_vg & (~base_rw_mgr)); 1227 1228 if (vg == 0) 1229 break; 1230 } 1231 *bit_chk &= tmp_bit_chk; 1232 } 1233 1234 addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_SINGLE_GROUP_OFFSET; 1235 writel(RW_MGR_CLEAR_DQS_ENABLE, addr + (group << 2)); 1236 1237 set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF); 1238 debug_cond(DLEVEL == 1, "%s:%d test_load_patterns(%u,ALL) => (%u == %u) =>\ 1239 %lu\n", __func__, __LINE__, group, *bit_chk, param->read_correct_mask, 1240 (long unsigned int)(*bit_chk == param->read_correct_mask)); 1241 return *bit_chk == param->read_correct_mask; 1242 } 1243 1244 static uint32_t rw_mgr_mem_calibrate_read_test_patterns_all_ranks 1245 (uint32_t group, uint32_t num_tries, uint32_t *bit_chk) 1246 { 1247 return rw_mgr_mem_calibrate_read_test_patterns(0, group, 1248 num_tries, bit_chk, 1); 1249 } 1250 1251 /* load up the patterns we are going to use during a read test */ 1252 static void rw_mgr_mem_calibrate_read_load_patterns(uint32_t rank_bgn, 1253 uint32_t all_ranks) 1254 { 1255 uint32_t r; 1256 uint32_t addr; 1257 uint32_t rank_end = all_ranks ? RW_MGR_MEM_NUMBER_OF_RANKS : 1258 (rank_bgn + NUM_RANKS_PER_SHADOW_REG); 1259 1260 debug("%s:%d\n", __func__, __LINE__); 1261 for (r = rank_bgn; r < rank_end; r++) { 1262 if (param->skip_ranks[r]) 1263 /* request to skip the rank */ 1264 continue; 1265 1266 /* set rank */ 1267 set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_READ_WRITE); 1268 1269 /* Load up a constant bursts */ 1270 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr0; 1271 writel(0x20, addr); 1272 1273 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add0; 1274 writel(RW_MGR_GUARANTEED_WRITE_WAIT0, addr); 1275 1276 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr1; 1277 writel(0x20, addr); 1278 1279 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add1; 1280 writel(RW_MGR_GUARANTEED_WRITE_WAIT1, addr); 1281 1282 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr2; 1283 writel(0x04, addr); 1284 1285 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add2; 1286 writel(RW_MGR_GUARANTEED_WRITE_WAIT2, addr); 1287 1288 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr3; 1289 writel(0x04, addr); 1290 1291 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add3; 1292 writel(RW_MGR_GUARANTEED_WRITE_WAIT3, addr); 1293 1294 addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_SINGLE_GROUP_OFFSET; 1295 writel(RW_MGR_GUARANTEED_WRITE, addr); 1296 } 1297 1298 set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF); 1299 } 1300 1301 /* 1302 * try a read and see if it returns correct data back. has dummy reads 1303 * inserted into the mix used to align dqs enable. has more thorough checks 1304 * than the regular read test. 1305 */ 1306 static uint32_t rw_mgr_mem_calibrate_read_test(uint32_t rank_bgn, uint32_t group, 1307 uint32_t num_tries, uint32_t all_correct, uint32_t *bit_chk, 1308 uint32_t all_groups, uint32_t all_ranks) 1309 { 1310 uint32_t r, vg; 1311 uint32_t correct_mask_vg; 1312 uint32_t tmp_bit_chk; 1313 uint32_t rank_end = all_ranks ? RW_MGR_MEM_NUMBER_OF_RANKS : 1314 (rank_bgn + NUM_RANKS_PER_SHADOW_REG); 1315 uint32_t addr; 1316 uint32_t base_rw_mgr; 1317 1318 *bit_chk = param->read_correct_mask; 1319 correct_mask_vg = param->read_correct_mask_vg; 1320 1321 uint32_t quick_read_mode = (((STATIC_CALIB_STEPS) & 1322 CALIB_SKIP_DELAY_SWEEPS) && ENABLE_SUPER_QUICK_CALIBRATION); 1323 1324 for (r = rank_bgn; r < rank_end; r++) { 1325 if (param->skip_ranks[r]) 1326 /* request to skip the rank */ 1327 continue; 1328 1329 /* set rank */ 1330 set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_READ_WRITE); 1331 1332 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr1; 1333 writel(0x10, addr); 1334 1335 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add1; 1336 writel(RW_MGR_READ_B2B_WAIT1, addr); 1337 1338 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr2; 1339 writel(0x10, addr); 1340 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add2; 1341 writel(RW_MGR_READ_B2B_WAIT2, addr); 1342 1343 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr0; 1344 if (quick_read_mode) 1345 writel(0x1, addr); 1346 /* need at least two (1+1) reads to capture failures */ 1347 else if (all_groups) 1348 writel(0x06, addr); 1349 else 1350 writel(0x32, addr); 1351 1352 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add0; 1353 writel(RW_MGR_READ_B2B, addr); 1354 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr3; 1355 if (all_groups) 1356 writel(RW_MGR_MEM_IF_READ_DQS_WIDTH * 1357 RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS - 1, 1358 addr); 1359 else 1360 writel(0x0, addr); 1361 1362 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add3; 1363 writel(RW_MGR_READ_B2B, addr); 1364 1365 tmp_bit_chk = 0; 1366 for (vg = RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS-1; ; vg--) { 1367 /* reset the fifos to get pointers to known state */ 1368 addr = (u32)&phy_mgr_cmd->fifo_reset; 1369 writel(0, addr); 1370 addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RESET_READ_DATAPATH_OFFSET; 1371 writel(0, addr); 1372 1373 tmp_bit_chk = tmp_bit_chk << (RW_MGR_MEM_DQ_PER_READ_DQS 1374 / RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS); 1375 1376 if (all_groups) 1377 addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_ALL_GROUPS_OFFSET; 1378 else 1379 addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_SINGLE_GROUP_OFFSET; 1380 1381 writel(RW_MGR_READ_B2B, addr + 1382 ((group * RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS + 1383 vg) << 2)); 1384 1385 addr = SDR_PHYGRP_RWMGRGRP_ADDRESS; 1386 base_rw_mgr = readl(addr); 1387 tmp_bit_chk = tmp_bit_chk | (correct_mask_vg & ~(base_rw_mgr)); 1388 1389 if (vg == 0) 1390 break; 1391 } 1392 *bit_chk &= tmp_bit_chk; 1393 } 1394 1395 addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_SINGLE_GROUP_OFFSET; 1396 writel(RW_MGR_CLEAR_DQS_ENABLE, addr + (group << 2)); 1397 1398 if (all_correct) { 1399 set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF); 1400 debug_cond(DLEVEL == 2, "%s:%d read_test(%u,ALL,%u) =>\ 1401 (%u == %u) => %lu", __func__, __LINE__, group, 1402 all_groups, *bit_chk, param->read_correct_mask, 1403 (long unsigned int)(*bit_chk == 1404 param->read_correct_mask)); 1405 return *bit_chk == param->read_correct_mask; 1406 } else { 1407 set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF); 1408 debug_cond(DLEVEL == 2, "%s:%d read_test(%u,ONE,%u) =>\ 1409 (%u != %lu) => %lu\n", __func__, __LINE__, 1410 group, all_groups, *bit_chk, (long unsigned int)0, 1411 (long unsigned int)(*bit_chk != 0x00)); 1412 return *bit_chk != 0x00; 1413 } 1414 } 1415 1416 static uint32_t rw_mgr_mem_calibrate_read_test_all_ranks(uint32_t group, 1417 uint32_t num_tries, uint32_t all_correct, uint32_t *bit_chk, 1418 uint32_t all_groups) 1419 { 1420 return rw_mgr_mem_calibrate_read_test(0, group, num_tries, all_correct, 1421 bit_chk, all_groups, 1); 1422 } 1423 1424 static void rw_mgr_incr_vfifo(uint32_t grp, uint32_t *v) 1425 { 1426 uint32_t addr = (u32)&phy_mgr_cmd->inc_vfifo_hard_phy; 1427 1428 writel(grp, addr); 1429 (*v)++; 1430 } 1431 1432 static void rw_mgr_decr_vfifo(uint32_t grp, uint32_t *v) 1433 { 1434 uint32_t i; 1435 1436 for (i = 0; i < VFIFO_SIZE-1; i++) 1437 rw_mgr_incr_vfifo(grp, v); 1438 } 1439 1440 static int find_vfifo_read(uint32_t grp, uint32_t *bit_chk) 1441 { 1442 uint32_t v; 1443 uint32_t fail_cnt = 0; 1444 uint32_t test_status; 1445 1446 for (v = 0; v < VFIFO_SIZE; ) { 1447 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: vfifo %u\n", 1448 __func__, __LINE__, v); 1449 test_status = rw_mgr_mem_calibrate_read_test_all_ranks 1450 (grp, 1, PASS_ONE_BIT, bit_chk, 0); 1451 if (!test_status) { 1452 fail_cnt++; 1453 1454 if (fail_cnt == 2) 1455 break; 1456 } 1457 1458 /* fiddle with FIFO */ 1459 rw_mgr_incr_vfifo(grp, &v); 1460 } 1461 1462 if (v >= VFIFO_SIZE) { 1463 /* no failing read found!! Something must have gone wrong */ 1464 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: vfifo failed\n", 1465 __func__, __LINE__); 1466 return 0; 1467 } else { 1468 return v; 1469 } 1470 } 1471 1472 static int find_working_phase(uint32_t *grp, uint32_t *bit_chk, 1473 uint32_t dtaps_per_ptap, uint32_t *work_bgn, 1474 uint32_t *v, uint32_t *d, uint32_t *p, 1475 uint32_t *i, uint32_t *max_working_cnt) 1476 { 1477 uint32_t found_begin = 0; 1478 uint32_t tmp_delay = 0; 1479 uint32_t test_status; 1480 1481 for (*d = 0; *d <= dtaps_per_ptap; (*d)++, tmp_delay += 1482 IO_DELAY_PER_DQS_EN_DCHAIN_TAP) { 1483 *work_bgn = tmp_delay; 1484 scc_mgr_set_dqs_en_delay_all_ranks(*grp, *d); 1485 1486 for (*i = 0; *i < VFIFO_SIZE; (*i)++) { 1487 for (*p = 0; *p <= IO_DQS_EN_PHASE_MAX; (*p)++, *work_bgn += 1488 IO_DELAY_PER_OPA_TAP) { 1489 scc_mgr_set_dqs_en_phase_all_ranks(*grp, *p); 1490 1491 test_status = 1492 rw_mgr_mem_calibrate_read_test_all_ranks 1493 (*grp, 1, PASS_ONE_BIT, bit_chk, 0); 1494 1495 if (test_status) { 1496 *max_working_cnt = 1; 1497 found_begin = 1; 1498 break; 1499 } 1500 } 1501 1502 if (found_begin) 1503 break; 1504 1505 if (*p > IO_DQS_EN_PHASE_MAX) 1506 /* fiddle with FIFO */ 1507 rw_mgr_incr_vfifo(*grp, v); 1508 } 1509 1510 if (found_begin) 1511 break; 1512 } 1513 1514 if (*i >= VFIFO_SIZE) { 1515 /* cannot find working solution */ 1516 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: no vfifo/\ 1517 ptap/dtap\n", __func__, __LINE__); 1518 return 0; 1519 } else { 1520 return 1; 1521 } 1522 } 1523 1524 static void sdr_backup_phase(uint32_t *grp, uint32_t *bit_chk, 1525 uint32_t *work_bgn, uint32_t *v, uint32_t *d, 1526 uint32_t *p, uint32_t *max_working_cnt) 1527 { 1528 uint32_t found_begin = 0; 1529 uint32_t tmp_delay; 1530 1531 /* Special case code for backing up a phase */ 1532 if (*p == 0) { 1533 *p = IO_DQS_EN_PHASE_MAX; 1534 rw_mgr_decr_vfifo(*grp, v); 1535 } else { 1536 (*p)--; 1537 } 1538 tmp_delay = *work_bgn - IO_DELAY_PER_OPA_TAP; 1539 scc_mgr_set_dqs_en_phase_all_ranks(*grp, *p); 1540 1541 for (*d = 0; *d <= IO_DQS_EN_DELAY_MAX && tmp_delay < *work_bgn; 1542 (*d)++, tmp_delay += IO_DELAY_PER_DQS_EN_DCHAIN_TAP) { 1543 scc_mgr_set_dqs_en_delay_all_ranks(*grp, *d); 1544 1545 if (rw_mgr_mem_calibrate_read_test_all_ranks(*grp, 1, 1546 PASS_ONE_BIT, 1547 bit_chk, 0)) { 1548 found_begin = 1; 1549 *work_bgn = tmp_delay; 1550 break; 1551 } 1552 } 1553 1554 /* We have found a working dtap before the ptap found above */ 1555 if (found_begin == 1) 1556 (*max_working_cnt)++; 1557 1558 /* 1559 * Restore VFIFO to old state before we decremented it 1560 * (if needed). 1561 */ 1562 (*p)++; 1563 if (*p > IO_DQS_EN_PHASE_MAX) { 1564 *p = 0; 1565 rw_mgr_incr_vfifo(*grp, v); 1566 } 1567 1568 scc_mgr_set_dqs_en_delay_all_ranks(*grp, 0); 1569 } 1570 1571 static int sdr_nonworking_phase(uint32_t *grp, uint32_t *bit_chk, 1572 uint32_t *work_bgn, uint32_t *v, uint32_t *d, 1573 uint32_t *p, uint32_t *i, uint32_t *max_working_cnt, 1574 uint32_t *work_end) 1575 { 1576 uint32_t found_end = 0; 1577 1578 (*p)++; 1579 *work_end += IO_DELAY_PER_OPA_TAP; 1580 if (*p > IO_DQS_EN_PHASE_MAX) { 1581 /* fiddle with FIFO */ 1582 *p = 0; 1583 rw_mgr_incr_vfifo(*grp, v); 1584 } 1585 1586 for (; *i < VFIFO_SIZE + 1; (*i)++) { 1587 for (; *p <= IO_DQS_EN_PHASE_MAX; (*p)++, *work_end 1588 += IO_DELAY_PER_OPA_TAP) { 1589 scc_mgr_set_dqs_en_phase_all_ranks(*grp, *p); 1590 1591 if (!rw_mgr_mem_calibrate_read_test_all_ranks 1592 (*grp, 1, PASS_ONE_BIT, bit_chk, 0)) { 1593 found_end = 1; 1594 break; 1595 } else { 1596 (*max_working_cnt)++; 1597 } 1598 } 1599 1600 if (found_end) 1601 break; 1602 1603 if (*p > IO_DQS_EN_PHASE_MAX) { 1604 /* fiddle with FIFO */ 1605 rw_mgr_incr_vfifo(*grp, v); 1606 *p = 0; 1607 } 1608 } 1609 1610 if (*i >= VFIFO_SIZE + 1) { 1611 /* cannot see edge of failing read */ 1612 debug_cond(DLEVEL == 2, "%s:%d sdr_nonworking_phase: end:\ 1613 failed\n", __func__, __LINE__); 1614 return 0; 1615 } else { 1616 return 1; 1617 } 1618 } 1619 1620 static int sdr_find_window_centre(uint32_t *grp, uint32_t *bit_chk, 1621 uint32_t *work_bgn, uint32_t *v, uint32_t *d, 1622 uint32_t *p, uint32_t *work_mid, 1623 uint32_t *work_end) 1624 { 1625 int i; 1626 int tmp_delay = 0; 1627 1628 *work_mid = (*work_bgn + *work_end) / 2; 1629 1630 debug_cond(DLEVEL == 2, "work_bgn=%d work_end=%d work_mid=%d\n", 1631 *work_bgn, *work_end, *work_mid); 1632 /* Get the middle delay to be less than a VFIFO delay */ 1633 for (*p = 0; *p <= IO_DQS_EN_PHASE_MAX; 1634 (*p)++, tmp_delay += IO_DELAY_PER_OPA_TAP) 1635 ; 1636 debug_cond(DLEVEL == 2, "vfifo ptap delay %d\n", tmp_delay); 1637 while (*work_mid > tmp_delay) 1638 *work_mid -= tmp_delay; 1639 debug_cond(DLEVEL == 2, "new work_mid %d\n", *work_mid); 1640 1641 tmp_delay = 0; 1642 for (*p = 0; *p <= IO_DQS_EN_PHASE_MAX && tmp_delay < *work_mid; 1643 (*p)++, tmp_delay += IO_DELAY_PER_OPA_TAP) 1644 ; 1645 tmp_delay -= IO_DELAY_PER_OPA_TAP; 1646 debug_cond(DLEVEL == 2, "new p %d, tmp_delay=%d\n", (*p) - 1, tmp_delay); 1647 for (*d = 0; *d <= IO_DQS_EN_DELAY_MAX && tmp_delay < *work_mid; (*d)++, 1648 tmp_delay += IO_DELAY_PER_DQS_EN_DCHAIN_TAP) 1649 ; 1650 debug_cond(DLEVEL == 2, "new d %d, tmp_delay=%d\n", *d, tmp_delay); 1651 1652 scc_mgr_set_dqs_en_phase_all_ranks(*grp, (*p) - 1); 1653 scc_mgr_set_dqs_en_delay_all_ranks(*grp, *d); 1654 1655 /* 1656 * push vfifo until we can successfully calibrate. We can do this 1657 * because the largest possible margin in 1 VFIFO cycle. 1658 */ 1659 for (i = 0; i < VFIFO_SIZE; i++) { 1660 debug_cond(DLEVEL == 2, "find_dqs_en_phase: center: vfifo=%u\n", 1661 *v); 1662 if (rw_mgr_mem_calibrate_read_test_all_ranks(*grp, 1, 1663 PASS_ONE_BIT, 1664 bit_chk, 0)) { 1665 break; 1666 } 1667 1668 /* fiddle with FIFO */ 1669 rw_mgr_incr_vfifo(*grp, v); 1670 } 1671 1672 if (i >= VFIFO_SIZE) { 1673 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: center: \ 1674 failed\n", __func__, __LINE__); 1675 return 0; 1676 } else { 1677 return 1; 1678 } 1679 } 1680 1681 /* find a good dqs enable to use */ 1682 static uint32_t rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase(uint32_t grp) 1683 { 1684 uint32_t v, d, p, i; 1685 uint32_t max_working_cnt; 1686 uint32_t bit_chk; 1687 uint32_t dtaps_per_ptap; 1688 uint32_t work_bgn, work_mid, work_end; 1689 uint32_t found_passing_read, found_failing_read, initial_failing_dtap; 1690 uint32_t addr; 1691 1692 debug("%s:%d %u\n", __func__, __LINE__, grp); 1693 1694 reg_file_set_sub_stage(CAL_SUBSTAGE_VFIFO_CENTER); 1695 1696 scc_mgr_set_dqs_en_delay_all_ranks(grp, 0); 1697 scc_mgr_set_dqs_en_phase_all_ranks(grp, 0); 1698 1699 /* ************************************************************** */ 1700 /* * Step 0 : Determine number of delay taps for each phase tap * */ 1701 dtaps_per_ptap = IO_DELAY_PER_OPA_TAP/IO_DELAY_PER_DQS_EN_DCHAIN_TAP; 1702 1703 /* ********************************************************* */ 1704 /* * Step 1 : First push vfifo until we get a failing read * */ 1705 v = find_vfifo_read(grp, &bit_chk); 1706 1707 max_working_cnt = 0; 1708 1709 /* ******************************************************** */ 1710 /* * step 2: find first working phase, increment in ptaps * */ 1711 work_bgn = 0; 1712 if (find_working_phase(&grp, &bit_chk, dtaps_per_ptap, &work_bgn, &v, &d, 1713 &p, &i, &max_working_cnt) == 0) 1714 return 0; 1715 1716 work_end = work_bgn; 1717 1718 /* 1719 * If d is 0 then the working window covers a phase tap and 1720 * we can follow the old procedure otherwise, we've found the beginning, 1721 * and we need to increment the dtaps until we find the end. 1722 */ 1723 if (d == 0) { 1724 /* ********************************************************* */ 1725 /* * step 3a: if we have room, back off by one and 1726 increment in dtaps * */ 1727 1728 sdr_backup_phase(&grp, &bit_chk, &work_bgn, &v, &d, &p, 1729 &max_working_cnt); 1730 1731 /* ********************************************************* */ 1732 /* * step 4a: go forward from working phase to non working 1733 phase, increment in ptaps * */ 1734 if (sdr_nonworking_phase(&grp, &bit_chk, &work_bgn, &v, &d, &p, 1735 &i, &max_working_cnt, &work_end) == 0) 1736 return 0; 1737 1738 /* ********************************************************* */ 1739 /* * step 5a: back off one from last, increment in dtaps * */ 1740 1741 /* Special case code for backing up a phase */ 1742 if (p == 0) { 1743 p = IO_DQS_EN_PHASE_MAX; 1744 rw_mgr_decr_vfifo(grp, &v); 1745 } else { 1746 p = p - 1; 1747 } 1748 1749 work_end -= IO_DELAY_PER_OPA_TAP; 1750 scc_mgr_set_dqs_en_phase_all_ranks(grp, p); 1751 1752 /* * The actual increment of dtaps is done outside of 1753 the if/else loop to share code */ 1754 d = 0; 1755 1756 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: v/p: \ 1757 vfifo=%u ptap=%u\n", __func__, __LINE__, 1758 v, p); 1759 } else { 1760 /* ******************************************************* */ 1761 /* * step 3-5b: Find the right edge of the window using 1762 delay taps * */ 1763 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase:vfifo=%u \ 1764 ptap=%u dtap=%u bgn=%u\n", __func__, __LINE__, 1765 v, p, d, work_bgn); 1766 1767 work_end = work_bgn; 1768 1769 /* * The actual increment of dtaps is done outside of the 1770 if/else loop to share code */ 1771 1772 /* Only here to counterbalance a subtract later on which is 1773 not needed if this branch of the algorithm is taken */ 1774 max_working_cnt++; 1775 } 1776 1777 /* The dtap increment to find the failing edge is done here */ 1778 for (; d <= IO_DQS_EN_DELAY_MAX; d++, work_end += 1779 IO_DELAY_PER_DQS_EN_DCHAIN_TAP) { 1780 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: \ 1781 end-2: dtap=%u\n", __func__, __LINE__, d); 1782 scc_mgr_set_dqs_en_delay_all_ranks(grp, d); 1783 1784 if (!rw_mgr_mem_calibrate_read_test_all_ranks(grp, 1, 1785 PASS_ONE_BIT, 1786 &bit_chk, 0)) { 1787 break; 1788 } 1789 } 1790 1791 /* Go back to working dtap */ 1792 if (d != 0) 1793 work_end -= IO_DELAY_PER_DQS_EN_DCHAIN_TAP; 1794 1795 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: v/p/d: vfifo=%u \ 1796 ptap=%u dtap=%u end=%u\n", __func__, __LINE__, 1797 v, p, d-1, work_end); 1798 1799 if (work_end < work_bgn) { 1800 /* nil range */ 1801 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: end-2: \ 1802 failed\n", __func__, __LINE__); 1803 return 0; 1804 } 1805 1806 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: found range [%u,%u]\n", 1807 __func__, __LINE__, work_bgn, work_end); 1808 1809 /* *************************************************************** */ 1810 /* 1811 * * We need to calculate the number of dtaps that equal a ptap 1812 * * To do that we'll back up a ptap and re-find the edge of the 1813 * * window using dtaps 1814 */ 1815 1816 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: calculate dtaps_per_ptap \ 1817 for tracking\n", __func__, __LINE__); 1818 1819 /* Special case code for backing up a phase */ 1820 if (p == 0) { 1821 p = IO_DQS_EN_PHASE_MAX; 1822 rw_mgr_decr_vfifo(grp, &v); 1823 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: backedup \ 1824 cycle/phase: v=%u p=%u\n", __func__, __LINE__, 1825 v, p); 1826 } else { 1827 p = p - 1; 1828 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: backedup \ 1829 phase only: v=%u p=%u", __func__, __LINE__, 1830 v, p); 1831 } 1832 1833 scc_mgr_set_dqs_en_phase_all_ranks(grp, p); 1834 1835 /* 1836 * Increase dtap until we first see a passing read (in case the 1837 * window is smaller than a ptap), 1838 * and then a failing read to mark the edge of the window again 1839 */ 1840 1841 /* Find a passing read */ 1842 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: find passing read\n", 1843 __func__, __LINE__); 1844 found_passing_read = 0; 1845 found_failing_read = 0; 1846 initial_failing_dtap = d; 1847 for (; d <= IO_DQS_EN_DELAY_MAX; d++) { 1848 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: testing \ 1849 read d=%u\n", __func__, __LINE__, d); 1850 scc_mgr_set_dqs_en_delay_all_ranks(grp, d); 1851 1852 if (rw_mgr_mem_calibrate_read_test_all_ranks(grp, 1, 1853 PASS_ONE_BIT, 1854 &bit_chk, 0)) { 1855 found_passing_read = 1; 1856 break; 1857 } 1858 } 1859 1860 if (found_passing_read) { 1861 /* Find a failing read */ 1862 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: find failing \ 1863 read\n", __func__, __LINE__); 1864 for (d = d + 1; d <= IO_DQS_EN_DELAY_MAX; d++) { 1865 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: \ 1866 testing read d=%u\n", __func__, __LINE__, d); 1867 scc_mgr_set_dqs_en_delay_all_ranks(grp, d); 1868 1869 if (!rw_mgr_mem_calibrate_read_test_all_ranks 1870 (grp, 1, PASS_ONE_BIT, &bit_chk, 0)) { 1871 found_failing_read = 1; 1872 break; 1873 } 1874 } 1875 } else { 1876 debug_cond(DLEVEL == 1, "%s:%d find_dqs_en_phase: failed to \ 1877 calculate dtaps", __func__, __LINE__); 1878 debug_cond(DLEVEL == 1, "per ptap. Fall back on static value\n"); 1879 } 1880 1881 /* 1882 * The dynamically calculated dtaps_per_ptap is only valid if we 1883 * found a passing/failing read. If we didn't, it means d hit the max 1884 * (IO_DQS_EN_DELAY_MAX). Otherwise, dtaps_per_ptap retains its 1885 * statically calculated value. 1886 */ 1887 if (found_passing_read && found_failing_read) 1888 dtaps_per_ptap = d - initial_failing_dtap; 1889 1890 addr = (u32)&sdr_reg_file->dtaps_per_ptap; 1891 writel(dtaps_per_ptap, addr); 1892 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: dtaps_per_ptap=%u \ 1893 - %u = %u", __func__, __LINE__, d, 1894 initial_failing_dtap, dtaps_per_ptap); 1895 1896 /* ******************************************** */ 1897 /* * step 6: Find the centre of the window * */ 1898 if (sdr_find_window_centre(&grp, &bit_chk, &work_bgn, &v, &d, &p, 1899 &work_mid, &work_end) == 0) 1900 return 0; 1901 1902 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: center found: \ 1903 vfifo=%u ptap=%u dtap=%u\n", __func__, __LINE__, 1904 v, p-1, d); 1905 return 1; 1906 } 1907 1908 /* 1909 * Try rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase across different 1910 * dq_in_delay values 1911 */ 1912 static uint32_t 1913 rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase_sweep_dq_in_delay 1914 (uint32_t write_group, uint32_t read_group, uint32_t test_bgn) 1915 { 1916 uint32_t found; 1917 uint32_t i; 1918 uint32_t p; 1919 uint32_t d; 1920 uint32_t r; 1921 uint32_t addr; 1922 1923 const uint32_t delay_step = IO_IO_IN_DELAY_MAX / 1924 (RW_MGR_MEM_DQ_PER_READ_DQS-1); 1925 /* we start at zero, so have one less dq to devide among */ 1926 1927 debug("%s:%d (%u,%u,%u)", __func__, __LINE__, write_group, read_group, 1928 test_bgn); 1929 1930 /* try different dq_in_delays since the dq path is shorter than dqs */ 1931 1932 for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; 1933 r += NUM_RANKS_PER_SHADOW_REG) { 1934 for (i = 0, p = test_bgn, d = 0; i < RW_MGR_MEM_DQ_PER_READ_DQS; 1935 i++, p++, d += delay_step) { 1936 debug_cond(DLEVEL == 1, "%s:%d rw_mgr_mem_calibrate_\ 1937 vfifo_find_dqs_", __func__, __LINE__); 1938 debug_cond(DLEVEL == 1, "en_phase_sweep_dq_in_delay: g=%u/%u ", 1939 write_group, read_group); 1940 debug_cond(DLEVEL == 1, "r=%u, i=%u p=%u d=%u\n", r, i , p, d); 1941 scc_mgr_set_dq_in_delay(write_group, p, d); 1942 scc_mgr_load_dq(p); 1943 } 1944 addr = (u32)&sdr_scc_mgr->update; 1945 writel(0, addr); 1946 } 1947 1948 found = rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase(read_group); 1949 1950 debug_cond(DLEVEL == 1, "%s:%d rw_mgr_mem_calibrate_vfifo_find_dqs_\ 1951 en_phase_sweep_dq", __func__, __LINE__); 1952 debug_cond(DLEVEL == 1, "_in_delay: g=%u/%u found=%u; Reseting delay \ 1953 chain to zero\n", write_group, read_group, found); 1954 1955 for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; 1956 r += NUM_RANKS_PER_SHADOW_REG) { 1957 for (i = 0, p = test_bgn; i < RW_MGR_MEM_DQ_PER_READ_DQS; 1958 i++, p++) { 1959 scc_mgr_set_dq_in_delay(write_group, p, 0); 1960 scc_mgr_load_dq(p); 1961 } 1962 addr = (u32)&sdr_scc_mgr->update; 1963 writel(0, addr); 1964 } 1965 1966 return found; 1967 } 1968 1969 /* per-bit deskew DQ and center */ 1970 static uint32_t rw_mgr_mem_calibrate_vfifo_center(uint32_t rank_bgn, 1971 uint32_t write_group, uint32_t read_group, uint32_t test_bgn, 1972 uint32_t use_read_test, uint32_t update_fom) 1973 { 1974 uint32_t i, p, d, min_index; 1975 /* 1976 * Store these as signed since there are comparisons with 1977 * signed numbers. 1978 */ 1979 uint32_t bit_chk; 1980 uint32_t sticky_bit_chk; 1981 int32_t left_edge[RW_MGR_MEM_DQ_PER_READ_DQS]; 1982 int32_t right_edge[RW_MGR_MEM_DQ_PER_READ_DQS]; 1983 int32_t final_dq[RW_MGR_MEM_DQ_PER_READ_DQS]; 1984 int32_t mid; 1985 int32_t orig_mid_min, mid_min; 1986 int32_t new_dqs, start_dqs, start_dqs_en, shift_dq, final_dqs, 1987 final_dqs_en; 1988 int32_t dq_margin, dqs_margin; 1989 uint32_t stop; 1990 uint32_t temp_dq_in_delay1, temp_dq_in_delay2; 1991 uint32_t addr; 1992 1993 debug("%s:%d: %u %u", __func__, __LINE__, read_group, test_bgn); 1994 1995 addr = SDR_PHYGRP_SCCGRP_ADDRESS | SCC_MGR_DQS_IN_DELAY_OFFSET; 1996 start_dqs = readl(addr + (read_group << 2)); 1997 if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS) 1998 start_dqs_en = readl(addr + ((read_group << 2) 1999 - IO_DQS_EN_DELAY_OFFSET)); 2000 2001 /* set the left and right edge of each bit to an illegal value */ 2002 /* use (IO_IO_IN_DELAY_MAX + 1) as an illegal value */ 2003 sticky_bit_chk = 0; 2004 for (i = 0; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++) { 2005 left_edge[i] = IO_IO_IN_DELAY_MAX + 1; 2006 right_edge[i] = IO_IO_IN_DELAY_MAX + 1; 2007 } 2008 2009 addr = (u32)&sdr_scc_mgr->update; 2010 /* Search for the left edge of the window for each bit */ 2011 for (d = 0; d <= IO_IO_IN_DELAY_MAX; d++) { 2012 scc_mgr_apply_group_dq_in_delay(write_group, test_bgn, d); 2013 2014 writel(0, addr); 2015 2016 /* 2017 * Stop searching when the read test doesn't pass AND when 2018 * we've seen a passing read on every bit. 2019 */ 2020 if (use_read_test) { 2021 stop = !rw_mgr_mem_calibrate_read_test(rank_bgn, 2022 read_group, NUM_READ_PB_TESTS, PASS_ONE_BIT, 2023 &bit_chk, 0, 0); 2024 } else { 2025 rw_mgr_mem_calibrate_write_test(rank_bgn, write_group, 2026 0, PASS_ONE_BIT, 2027 &bit_chk, 0); 2028 bit_chk = bit_chk >> (RW_MGR_MEM_DQ_PER_READ_DQS * 2029 (read_group - (write_group * 2030 RW_MGR_MEM_IF_READ_DQS_WIDTH / 2031 RW_MGR_MEM_IF_WRITE_DQS_WIDTH))); 2032 stop = (bit_chk == 0); 2033 } 2034 sticky_bit_chk = sticky_bit_chk | bit_chk; 2035 stop = stop && (sticky_bit_chk == param->read_correct_mask); 2036 debug_cond(DLEVEL == 2, "%s:%d vfifo_center(left): dtap=%u => %u == %u \ 2037 && %u", __func__, __LINE__, d, 2038 sticky_bit_chk, 2039 param->read_correct_mask, stop); 2040 2041 if (stop == 1) { 2042 break; 2043 } else { 2044 for (i = 0; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++) { 2045 if (bit_chk & 1) { 2046 /* Remember a passing test as the 2047 left_edge */ 2048 left_edge[i] = d; 2049 } else { 2050 /* If a left edge has not been seen yet, 2051 then a future passing test will mark 2052 this edge as the right edge */ 2053 if (left_edge[i] == 2054 IO_IO_IN_DELAY_MAX + 1) { 2055 right_edge[i] = -(d + 1); 2056 } 2057 } 2058 bit_chk = bit_chk >> 1; 2059 } 2060 } 2061 } 2062 2063 /* Reset DQ delay chains to 0 */ 2064 scc_mgr_apply_group_dq_in_delay(write_group, test_bgn, 0); 2065 sticky_bit_chk = 0; 2066 for (i = RW_MGR_MEM_DQ_PER_READ_DQS - 1;; i--) { 2067 debug_cond(DLEVEL == 2, "%s:%d vfifo_center: left_edge[%u]: \ 2068 %d right_edge[%u]: %d\n", __func__, __LINE__, 2069 i, left_edge[i], i, right_edge[i]); 2070 2071 /* 2072 * Check for cases where we haven't found the left edge, 2073 * which makes our assignment of the the right edge invalid. 2074 * Reset it to the illegal value. 2075 */ 2076 if ((left_edge[i] == IO_IO_IN_DELAY_MAX + 1) && ( 2077 right_edge[i] != IO_IO_IN_DELAY_MAX + 1)) { 2078 right_edge[i] = IO_IO_IN_DELAY_MAX + 1; 2079 debug_cond(DLEVEL == 2, "%s:%d vfifo_center: reset \ 2080 right_edge[%u]: %d\n", __func__, __LINE__, 2081 i, right_edge[i]); 2082 } 2083 2084 /* 2085 * Reset sticky bit (except for bits where we have seen 2086 * both the left and right edge). 2087 */ 2088 sticky_bit_chk = sticky_bit_chk << 1; 2089 if ((left_edge[i] != IO_IO_IN_DELAY_MAX + 1) && 2090 (right_edge[i] != IO_IO_IN_DELAY_MAX + 1)) { 2091 sticky_bit_chk = sticky_bit_chk | 1; 2092 } 2093 2094 if (i == 0) 2095 break; 2096 } 2097 2098 addr = (u32)&sdr_scc_mgr->update; 2099 /* Search for the right edge of the window for each bit */ 2100 for (d = 0; d <= IO_DQS_IN_DELAY_MAX - start_dqs; d++) { 2101 scc_mgr_set_dqs_bus_in_delay(read_group, d + start_dqs); 2102 if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS) { 2103 uint32_t delay = d + start_dqs_en; 2104 if (delay > IO_DQS_EN_DELAY_MAX) 2105 delay = IO_DQS_EN_DELAY_MAX; 2106 scc_mgr_set_dqs_en_delay(read_group, delay); 2107 } 2108 scc_mgr_load_dqs(read_group); 2109 2110 writel(0, addr); 2111 2112 /* 2113 * Stop searching when the read test doesn't pass AND when 2114 * we've seen a passing read on every bit. 2115 */ 2116 if (use_read_test) { 2117 stop = !rw_mgr_mem_calibrate_read_test(rank_bgn, 2118 read_group, NUM_READ_PB_TESTS, PASS_ONE_BIT, 2119 &bit_chk, 0, 0); 2120 } else { 2121 rw_mgr_mem_calibrate_write_test(rank_bgn, write_group, 2122 0, PASS_ONE_BIT, 2123 &bit_chk, 0); 2124 bit_chk = bit_chk >> (RW_MGR_MEM_DQ_PER_READ_DQS * 2125 (read_group - (write_group * 2126 RW_MGR_MEM_IF_READ_DQS_WIDTH / 2127 RW_MGR_MEM_IF_WRITE_DQS_WIDTH))); 2128 stop = (bit_chk == 0); 2129 } 2130 sticky_bit_chk = sticky_bit_chk | bit_chk; 2131 stop = stop && (sticky_bit_chk == param->read_correct_mask); 2132 2133 debug_cond(DLEVEL == 2, "%s:%d vfifo_center(right): dtap=%u => %u == \ 2134 %u && %u", __func__, __LINE__, d, 2135 sticky_bit_chk, param->read_correct_mask, stop); 2136 2137 if (stop == 1) { 2138 break; 2139 } else { 2140 for (i = 0; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++) { 2141 if (bit_chk & 1) { 2142 /* Remember a passing test as 2143 the right_edge */ 2144 right_edge[i] = d; 2145 } else { 2146 if (d != 0) { 2147 /* If a right edge has not been 2148 seen yet, then a future passing 2149 test will mark this edge as the 2150 left edge */ 2151 if (right_edge[i] == 2152 IO_IO_IN_DELAY_MAX + 1) { 2153 left_edge[i] = -(d + 1); 2154 } 2155 } else { 2156 /* d = 0 failed, but it passed 2157 when testing the left edge, 2158 so it must be marginal, 2159 set it to -1 */ 2160 if (right_edge[i] == 2161 IO_IO_IN_DELAY_MAX + 1 && 2162 left_edge[i] != 2163 IO_IO_IN_DELAY_MAX 2164 + 1) { 2165 right_edge[i] = -1; 2166 } 2167 /* If a right edge has not been 2168 seen yet, then a future passing 2169 test will mark this edge as the 2170 left edge */ 2171 else if (right_edge[i] == 2172 IO_IO_IN_DELAY_MAX + 2173 1) { 2174 left_edge[i] = -(d + 1); 2175 } 2176 } 2177 } 2178 2179 debug_cond(DLEVEL == 2, "%s:%d vfifo_center[r,\ 2180 d=%u]: ", __func__, __LINE__, d); 2181 debug_cond(DLEVEL == 2, "bit_chk_test=%d left_edge[%u]: %d ", 2182 (int)(bit_chk & 1), i, left_edge[i]); 2183 debug_cond(DLEVEL == 2, "right_edge[%u]: %d\n", i, 2184 right_edge[i]); 2185 bit_chk = bit_chk >> 1; 2186 } 2187 } 2188 } 2189 2190 /* Check that all bits have a window */ 2191 addr = (u32)&sdr_scc_mgr->update; 2192 for (i = 0; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++) { 2193 debug_cond(DLEVEL == 2, "%s:%d vfifo_center: left_edge[%u]: \ 2194 %d right_edge[%u]: %d", __func__, __LINE__, 2195 i, left_edge[i], i, right_edge[i]); 2196 if ((left_edge[i] == IO_IO_IN_DELAY_MAX + 1) || (right_edge[i] 2197 == IO_IO_IN_DELAY_MAX + 1)) { 2198 /* 2199 * Restore delay chain settings before letting the loop 2200 * in rw_mgr_mem_calibrate_vfifo to retry different 2201 * dqs/ck relationships. 2202 */ 2203 scc_mgr_set_dqs_bus_in_delay(read_group, start_dqs); 2204 if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS) { 2205 scc_mgr_set_dqs_en_delay(read_group, 2206 start_dqs_en); 2207 } 2208 scc_mgr_load_dqs(read_group); 2209 writel(0, addr); 2210 2211 debug_cond(DLEVEL == 1, "%s:%d vfifo_center: failed to \ 2212 find edge [%u]: %d %d", __func__, __LINE__, 2213 i, left_edge[i], right_edge[i]); 2214 if (use_read_test) { 2215 set_failing_group_stage(read_group * 2216 RW_MGR_MEM_DQ_PER_READ_DQS + i, 2217 CAL_STAGE_VFIFO, 2218 CAL_SUBSTAGE_VFIFO_CENTER); 2219 } else { 2220 set_failing_group_stage(read_group * 2221 RW_MGR_MEM_DQ_PER_READ_DQS + i, 2222 CAL_STAGE_VFIFO_AFTER_WRITES, 2223 CAL_SUBSTAGE_VFIFO_CENTER); 2224 } 2225 return 0; 2226 } 2227 } 2228 2229 /* Find middle of window for each DQ bit */ 2230 mid_min = left_edge[0] - right_edge[0]; 2231 min_index = 0; 2232 for (i = 1; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++) { 2233 mid = left_edge[i] - right_edge[i]; 2234 if (mid < mid_min) { 2235 mid_min = mid; 2236 min_index = i; 2237 } 2238 } 2239 2240 /* 2241 * -mid_min/2 represents the amount that we need to move DQS. 2242 * If mid_min is odd and positive we'll need to add one to 2243 * make sure the rounding in further calculations is correct 2244 * (always bias to the right), so just add 1 for all positive values. 2245 */ 2246 if (mid_min > 0) 2247 mid_min++; 2248 2249 mid_min = mid_min / 2; 2250 2251 debug_cond(DLEVEL == 1, "%s:%d vfifo_center: mid_min=%d (index=%u)\n", 2252 __func__, __LINE__, mid_min, min_index); 2253 2254 /* Determine the amount we can change DQS (which is -mid_min) */ 2255 orig_mid_min = mid_min; 2256 new_dqs = start_dqs - mid_min; 2257 if (new_dqs > IO_DQS_IN_DELAY_MAX) 2258 new_dqs = IO_DQS_IN_DELAY_MAX; 2259 else if (new_dqs < 0) 2260 new_dqs = 0; 2261 2262 mid_min = start_dqs - new_dqs; 2263 debug_cond(DLEVEL == 1, "vfifo_center: new mid_min=%d new_dqs=%d\n", 2264 mid_min, new_dqs); 2265 2266 if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS) { 2267 if (start_dqs_en - mid_min > IO_DQS_EN_DELAY_MAX) 2268 mid_min += start_dqs_en - mid_min - IO_DQS_EN_DELAY_MAX; 2269 else if (start_dqs_en - mid_min < 0) 2270 mid_min += start_dqs_en - mid_min; 2271 } 2272 new_dqs = start_dqs - mid_min; 2273 2274 debug_cond(DLEVEL == 1, "vfifo_center: start_dqs=%d start_dqs_en=%d \ 2275 new_dqs=%d mid_min=%d\n", start_dqs, 2276 IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS ? start_dqs_en : -1, 2277 new_dqs, mid_min); 2278 2279 /* Initialize data for export structures */ 2280 dqs_margin = IO_IO_IN_DELAY_MAX + 1; 2281 dq_margin = IO_IO_IN_DELAY_MAX + 1; 2282 2283 addr = SDR_PHYGRP_SCCGRP_ADDRESS | SCC_MGR_IO_IN_DELAY_OFFSET; 2284 /* add delay to bring centre of all DQ windows to the same "level" */ 2285 for (i = 0, p = test_bgn; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++, p++) { 2286 /* Use values before divide by 2 to reduce round off error */ 2287 shift_dq = (left_edge[i] - right_edge[i] - 2288 (left_edge[min_index] - right_edge[min_index]))/2 + 2289 (orig_mid_min - mid_min); 2290 2291 debug_cond(DLEVEL == 2, "vfifo_center: before: \ 2292 shift_dq[%u]=%d\n", i, shift_dq); 2293 2294 temp_dq_in_delay1 = readl(addr + (p << 2)); 2295 temp_dq_in_delay2 = readl(addr + (i << 2)); 2296 2297 if (shift_dq + (int32_t)temp_dq_in_delay1 > 2298 (int32_t)IO_IO_IN_DELAY_MAX) { 2299 shift_dq = (int32_t)IO_IO_IN_DELAY_MAX - temp_dq_in_delay2; 2300 } else if (shift_dq + (int32_t)temp_dq_in_delay1 < 0) { 2301 shift_dq = -(int32_t)temp_dq_in_delay1; 2302 } 2303 debug_cond(DLEVEL == 2, "vfifo_center: after: \ 2304 shift_dq[%u]=%d\n", i, shift_dq); 2305 final_dq[i] = temp_dq_in_delay1 + shift_dq; 2306 scc_mgr_set_dq_in_delay(write_group, p, final_dq[i]); 2307 scc_mgr_load_dq(p); 2308 2309 debug_cond(DLEVEL == 2, "vfifo_center: margin[%u]=[%d,%d]\n", i, 2310 left_edge[i] - shift_dq + (-mid_min), 2311 right_edge[i] + shift_dq - (-mid_min)); 2312 /* To determine values for export structures */ 2313 if (left_edge[i] - shift_dq + (-mid_min) < dq_margin) 2314 dq_margin = left_edge[i] - shift_dq + (-mid_min); 2315 2316 if (right_edge[i] + shift_dq - (-mid_min) < dqs_margin) 2317 dqs_margin = right_edge[i] + shift_dq - (-mid_min); 2318 } 2319 2320 final_dqs = new_dqs; 2321 if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS) 2322 final_dqs_en = start_dqs_en - mid_min; 2323 2324 /* Move DQS-en */ 2325 if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS) { 2326 scc_mgr_set_dqs_en_delay(read_group, final_dqs_en); 2327 scc_mgr_load_dqs(read_group); 2328 } 2329 2330 /* Move DQS */ 2331 scc_mgr_set_dqs_bus_in_delay(read_group, final_dqs); 2332 scc_mgr_load_dqs(read_group); 2333 debug_cond(DLEVEL == 2, "%s:%d vfifo_center: dq_margin=%d \ 2334 dqs_margin=%d", __func__, __LINE__, 2335 dq_margin, dqs_margin); 2336 2337 /* 2338 * Do not remove this line as it makes sure all of our decisions 2339 * have been applied. Apply the update bit. 2340 */ 2341 addr = (u32)&sdr_scc_mgr->update; 2342 writel(0, addr); 2343 2344 return (dq_margin >= 0) && (dqs_margin >= 0); 2345 } 2346 2347 /* 2348 * calibrate the read valid prediction FIFO. 2349 * 2350 * - read valid prediction will consist of finding a good DQS enable phase, 2351 * DQS enable delay, DQS input phase, and DQS input delay. 2352 * - we also do a per-bit deskew on the DQ lines. 2353 */ 2354 static uint32_t rw_mgr_mem_calibrate_vfifo(uint32_t read_group, 2355 uint32_t test_bgn) 2356 { 2357 uint32_t p, d, rank_bgn, sr; 2358 uint32_t dtaps_per_ptap; 2359 uint32_t tmp_delay; 2360 uint32_t bit_chk; 2361 uint32_t grp_calibrated; 2362 uint32_t write_group, write_test_bgn; 2363 uint32_t failed_substage; 2364 2365 debug("%s:%d: %u %u\n", __func__, __LINE__, read_group, test_bgn); 2366 2367 /* update info for sims */ 2368 reg_file_set_stage(CAL_STAGE_VFIFO); 2369 2370 write_group = read_group; 2371 write_test_bgn = test_bgn; 2372 2373 /* USER Determine number of delay taps for each phase tap */ 2374 dtaps_per_ptap = 0; 2375 tmp_delay = 0; 2376 while (tmp_delay < IO_DELAY_PER_OPA_TAP) { 2377 dtaps_per_ptap++; 2378 tmp_delay += IO_DELAY_PER_DQS_EN_DCHAIN_TAP; 2379 } 2380 dtaps_per_ptap--; 2381 tmp_delay = 0; 2382 2383 /* update info for sims */ 2384 reg_file_set_group(read_group); 2385 2386 grp_calibrated = 0; 2387 2388 reg_file_set_sub_stage(CAL_SUBSTAGE_GUARANTEED_READ); 2389 failed_substage = CAL_SUBSTAGE_GUARANTEED_READ; 2390 2391 for (d = 0; d <= dtaps_per_ptap && grp_calibrated == 0; d += 2) { 2392 /* 2393 * In RLDRAMX we may be messing the delay of pins in 2394 * the same write group but outside of the current read 2395 * the group, but that's ok because we haven't 2396 * calibrated output side yet. 2397 */ 2398 if (d > 0) { 2399 scc_mgr_apply_group_all_out_delay_add_all_ranks 2400 (write_group, write_test_bgn, d); 2401 } 2402 2403 for (p = 0; p <= IO_DQDQS_OUT_PHASE_MAX && grp_calibrated == 0; 2404 p++) { 2405 /* set a particular dqdqs phase */ 2406 scc_mgr_set_dqdqs_output_phase_all_ranks(read_group, p); 2407 2408 debug_cond(DLEVEL == 1, "%s:%d calibrate_vfifo: g=%u \ 2409 p=%u d=%u\n", __func__, __LINE__, 2410 read_group, p, d); 2411 2412 /* 2413 * Load up the patterns used by read calibration 2414 * using current DQDQS phase. 2415 */ 2416 rw_mgr_mem_calibrate_read_load_patterns(0, 1); 2417 if (!(gbl->phy_debug_mode_flags & 2418 PHY_DEBUG_DISABLE_GUARANTEED_READ)) { 2419 if (!rw_mgr_mem_calibrate_read_test_patterns_all_ranks 2420 (read_group, 1, &bit_chk)) { 2421 debug_cond(DLEVEL == 1, "%s:%d Guaranteed read test failed:", 2422 __func__, __LINE__); 2423 debug_cond(DLEVEL == 1, " g=%u p=%u d=%u\n", 2424 read_group, p, d); 2425 break; 2426 } 2427 } 2428 2429 /* case:56390 */ 2430 grp_calibrated = 1; 2431 if (rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase_sweep_dq_in_delay 2432 (write_group, read_group, test_bgn)) { 2433 /* 2434 * USER Read per-bit deskew can be done on a 2435 * per shadow register basis. 2436 */ 2437 for (rank_bgn = 0, sr = 0; 2438 rank_bgn < RW_MGR_MEM_NUMBER_OF_RANKS; 2439 rank_bgn += NUM_RANKS_PER_SHADOW_REG, 2440 ++sr) { 2441 /* 2442 * Determine if this set of ranks 2443 * should be skipped entirely. 2444 */ 2445 if (!param->skip_shadow_regs[sr]) { 2446 /* 2447 * If doing read after write 2448 * calibration, do not update 2449 * FOM, now - do it then. 2450 */ 2451 if (!rw_mgr_mem_calibrate_vfifo_center 2452 (rank_bgn, write_group, 2453 read_group, test_bgn, 1, 0)) { 2454 grp_calibrated = 0; 2455 failed_substage = 2456 CAL_SUBSTAGE_VFIFO_CENTER; 2457 } 2458 } 2459 } 2460 } else { 2461 grp_calibrated = 0; 2462 failed_substage = CAL_SUBSTAGE_DQS_EN_PHASE; 2463 } 2464 } 2465 } 2466 2467 if (grp_calibrated == 0) { 2468 set_failing_group_stage(write_group, CAL_STAGE_VFIFO, 2469 failed_substage); 2470 return 0; 2471 } 2472 2473 /* 2474 * Reset the delay chains back to zero if they have moved > 1 2475 * (check for > 1 because loop will increase d even when pass in 2476 * first case). 2477 */ 2478 if (d > 2) 2479 scc_mgr_zero_group(write_group, write_test_bgn, 1); 2480 2481 return 1; 2482 } 2483 2484 /* VFIFO Calibration -- Read Deskew Calibration after write deskew */ 2485 static uint32_t rw_mgr_mem_calibrate_vfifo_end(uint32_t read_group, 2486 uint32_t test_bgn) 2487 { 2488 uint32_t rank_bgn, sr; 2489 uint32_t grp_calibrated; 2490 uint32_t write_group; 2491 2492 debug("%s:%d %u %u", __func__, __LINE__, read_group, test_bgn); 2493 2494 /* update info for sims */ 2495 2496 reg_file_set_stage(CAL_STAGE_VFIFO_AFTER_WRITES); 2497 reg_file_set_sub_stage(CAL_SUBSTAGE_VFIFO_CENTER); 2498 2499 write_group = read_group; 2500 2501 /* update info for sims */ 2502 reg_file_set_group(read_group); 2503 2504 grp_calibrated = 1; 2505 /* Read per-bit deskew can be done on a per shadow register basis */ 2506 for (rank_bgn = 0, sr = 0; rank_bgn < RW_MGR_MEM_NUMBER_OF_RANKS; 2507 rank_bgn += NUM_RANKS_PER_SHADOW_REG, ++sr) { 2508 /* Determine if this set of ranks should be skipped entirely */ 2509 if (!param->skip_shadow_regs[sr]) { 2510 /* This is the last calibration round, update FOM here */ 2511 if (!rw_mgr_mem_calibrate_vfifo_center(rank_bgn, 2512 write_group, 2513 read_group, 2514 test_bgn, 0, 2515 1)) { 2516 grp_calibrated = 0; 2517 } 2518 } 2519 } 2520 2521 2522 if (grp_calibrated == 0) { 2523 set_failing_group_stage(write_group, 2524 CAL_STAGE_VFIFO_AFTER_WRITES, 2525 CAL_SUBSTAGE_VFIFO_CENTER); 2526 return 0; 2527 } 2528 2529 return 1; 2530 } 2531 2532 /* Calibrate LFIFO to find smallest read latency */ 2533 static uint32_t rw_mgr_mem_calibrate_lfifo(void) 2534 { 2535 uint32_t found_one; 2536 uint32_t bit_chk; 2537 uint32_t addr; 2538 2539 debug("%s:%d\n", __func__, __LINE__); 2540 2541 /* update info for sims */ 2542 reg_file_set_stage(CAL_STAGE_LFIFO); 2543 reg_file_set_sub_stage(CAL_SUBSTAGE_READ_LATENCY); 2544 2545 /* Load up the patterns used by read calibration for all ranks */ 2546 rw_mgr_mem_calibrate_read_load_patterns(0, 1); 2547 found_one = 0; 2548 2549 addr = (u32)&phy_mgr_cfg->phy_rlat; 2550 do { 2551 writel(gbl->curr_read_lat, addr); 2552 debug_cond(DLEVEL == 2, "%s:%d lfifo: read_lat=%u", 2553 __func__, __LINE__, gbl->curr_read_lat); 2554 2555 if (!rw_mgr_mem_calibrate_read_test_all_ranks(0, 2556 NUM_READ_TESTS, 2557 PASS_ALL_BITS, 2558 &bit_chk, 1)) { 2559 break; 2560 } 2561 2562 found_one = 1; 2563 /* reduce read latency and see if things are working */ 2564 /* correctly */ 2565 gbl->curr_read_lat--; 2566 } while (gbl->curr_read_lat > 0); 2567 2568 /* reset the fifos to get pointers to known state */ 2569 2570 addr = (u32)&phy_mgr_cmd->fifo_reset; 2571 writel(0, addr); 2572 2573 if (found_one) { 2574 /* add a fudge factor to the read latency that was determined */ 2575 gbl->curr_read_lat += 2; 2576 addr = (u32)&phy_mgr_cfg->phy_rlat; 2577 writel(gbl->curr_read_lat, addr); 2578 debug_cond(DLEVEL == 2, "%s:%d lfifo: success: using \ 2579 read_lat=%u\n", __func__, __LINE__, 2580 gbl->curr_read_lat); 2581 return 1; 2582 } else { 2583 set_failing_group_stage(0xff, CAL_STAGE_LFIFO, 2584 CAL_SUBSTAGE_READ_LATENCY); 2585 2586 debug_cond(DLEVEL == 2, "%s:%d lfifo: failed at initial \ 2587 read_lat=%u\n", __func__, __LINE__, 2588 gbl->curr_read_lat); 2589 return 0; 2590 } 2591 } 2592 2593 /* 2594 * issue write test command. 2595 * two variants are provided. one that just tests a write pattern and 2596 * another that tests datamask functionality. 2597 */ 2598 static void rw_mgr_mem_calibrate_write_test_issue(uint32_t group, 2599 uint32_t test_dm) 2600 { 2601 uint32_t mcc_instruction; 2602 uint32_t quick_write_mode = (((STATIC_CALIB_STEPS) & CALIB_SKIP_WRITES) && 2603 ENABLE_SUPER_QUICK_CALIBRATION); 2604 uint32_t rw_wl_nop_cycles; 2605 uint32_t addr; 2606 2607 /* 2608 * Set counter and jump addresses for the right 2609 * number of NOP cycles. 2610 * The number of supported NOP cycles can range from -1 to infinity 2611 * Three different cases are handled: 2612 * 2613 * 1. For a number of NOP cycles greater than 0, the RW Mgr looping 2614 * mechanism will be used to insert the right number of NOPs 2615 * 2616 * 2. For a number of NOP cycles equals to 0, the micro-instruction 2617 * issuing the write command will jump straight to the 2618 * micro-instruction that turns on DQS (for DDRx), or outputs write 2619 * data (for RLD), skipping 2620 * the NOP micro-instruction all together 2621 * 2622 * 3. A number of NOP cycles equal to -1 indicates that DQS must be 2623 * turned on in the same micro-instruction that issues the write 2624 * command. Then we need 2625 * to directly jump to the micro-instruction that sends out the data 2626 * 2627 * NOTE: Implementing this mechanism uses 2 RW Mgr jump-counters 2628 * (2 and 3). One jump-counter (0) is used to perform multiple 2629 * write-read operations. 2630 * one counter left to issue this command in "multiple-group" mode 2631 */ 2632 2633 rw_wl_nop_cycles = gbl->rw_wl_nop_cycles; 2634 2635 if (rw_wl_nop_cycles == -1) { 2636 /* 2637 * CNTR 2 - We want to execute the special write operation that 2638 * turns on DQS right away and then skip directly to the 2639 * instruction that sends out the data. We set the counter to a 2640 * large number so that the jump is always taken. 2641 */ 2642 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr2; 2643 writel(0xFF, addr); 2644 2645 /* CNTR 3 - Not used */ 2646 if (test_dm) { 2647 mcc_instruction = RW_MGR_LFSR_WR_RD_DM_BANK_0_WL_1; 2648 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add2; 2649 writel(RW_MGR_LFSR_WR_RD_DM_BANK_0_DATA, 2650 addr); 2651 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add3; 2652 writel(RW_MGR_LFSR_WR_RD_DM_BANK_0_NOP, 2653 addr); 2654 } else { 2655 mcc_instruction = RW_MGR_LFSR_WR_RD_BANK_0_WL_1; 2656 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add2; 2657 writel(RW_MGR_LFSR_WR_RD_BANK_0_DATA, addr); 2658 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add3; 2659 writel(RW_MGR_LFSR_WR_RD_BANK_0_NOP, addr); 2660 } 2661 } else if (rw_wl_nop_cycles == 0) { 2662 /* 2663 * CNTR 2 - We want to skip the NOP operation and go straight 2664 * to the DQS enable instruction. We set the counter to a large 2665 * number so that the jump is always taken. 2666 */ 2667 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr2; 2668 writel(0xFF, addr); 2669 2670 /* CNTR 3 - Not used */ 2671 if (test_dm) { 2672 mcc_instruction = RW_MGR_LFSR_WR_RD_DM_BANK_0; 2673 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add2; 2674 writel(RW_MGR_LFSR_WR_RD_DM_BANK_0_DQS, 2675 addr); 2676 } else { 2677 mcc_instruction = RW_MGR_LFSR_WR_RD_BANK_0; 2678 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add2; 2679 writel(RW_MGR_LFSR_WR_RD_BANK_0_DQS, addr); 2680 } 2681 } else { 2682 /* 2683 * CNTR 2 - In this case we want to execute the next instruction 2684 * and NOT take the jump. So we set the counter to 0. The jump 2685 * address doesn't count. 2686 */ 2687 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr2; 2688 writel(0x0, addr); 2689 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add2; 2690 writel(0x0, addr); 2691 2692 /* 2693 * CNTR 3 - Set the nop counter to the number of cycles we 2694 * need to loop for, minus 1. 2695 */ 2696 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr3; 2697 writel(rw_wl_nop_cycles - 1, addr); 2698 if (test_dm) { 2699 mcc_instruction = RW_MGR_LFSR_WR_RD_DM_BANK_0; 2700 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add3; 2701 writel(RW_MGR_LFSR_WR_RD_DM_BANK_0_NOP, addr); 2702 } else { 2703 mcc_instruction = RW_MGR_LFSR_WR_RD_BANK_0; 2704 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add3; 2705 writel(RW_MGR_LFSR_WR_RD_BANK_0_NOP, addr); 2706 } 2707 } 2708 2709 addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RESET_READ_DATAPATH_OFFSET; 2710 writel(0, addr); 2711 2712 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr0; 2713 if (quick_write_mode) 2714 writel(0x08, addr); 2715 else 2716 writel(0x40, addr); 2717 2718 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add0; 2719 writel(mcc_instruction, addr); 2720 2721 /* 2722 * CNTR 1 - This is used to ensure enough time elapses 2723 * for read data to come back. 2724 */ 2725 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr1; 2726 writel(0x30, addr); 2727 2728 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add1; 2729 if (test_dm) { 2730 writel(RW_MGR_LFSR_WR_RD_DM_BANK_0_WAIT, addr); 2731 } else { 2732 writel(RW_MGR_LFSR_WR_RD_BANK_0_WAIT, addr); 2733 } 2734 2735 addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_SINGLE_GROUP_OFFSET; 2736 writel(mcc_instruction, addr + (group << 2)); 2737 } 2738 2739 /* Test writes, can check for a single bit pass or multiple bit pass */ 2740 static uint32_t rw_mgr_mem_calibrate_write_test(uint32_t rank_bgn, 2741 uint32_t write_group, uint32_t use_dm, uint32_t all_correct, 2742 uint32_t *bit_chk, uint32_t all_ranks) 2743 { 2744 uint32_t addr; 2745 uint32_t r; 2746 uint32_t correct_mask_vg; 2747 uint32_t tmp_bit_chk; 2748 uint32_t vg; 2749 uint32_t rank_end = all_ranks ? RW_MGR_MEM_NUMBER_OF_RANKS : 2750 (rank_bgn + NUM_RANKS_PER_SHADOW_REG); 2751 uint32_t addr_rw_mgr; 2752 uint32_t base_rw_mgr; 2753 2754 *bit_chk = param->write_correct_mask; 2755 correct_mask_vg = param->write_correct_mask_vg; 2756 2757 for (r = rank_bgn; r < rank_end; r++) { 2758 if (param->skip_ranks[r]) { 2759 /* request to skip the rank */ 2760 continue; 2761 } 2762 2763 /* set rank */ 2764 set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_READ_WRITE); 2765 2766 tmp_bit_chk = 0; 2767 addr = (u32)&phy_mgr_cmd->fifo_reset; 2768 addr_rw_mgr = SDR_PHYGRP_RWMGRGRP_ADDRESS; 2769 for (vg = RW_MGR_MEM_VIRTUAL_GROUPS_PER_WRITE_DQS-1; ; vg--) { 2770 /* reset the fifos to get pointers to known state */ 2771 writel(0, addr); 2772 2773 tmp_bit_chk = tmp_bit_chk << 2774 (RW_MGR_MEM_DQ_PER_WRITE_DQS / 2775 RW_MGR_MEM_VIRTUAL_GROUPS_PER_WRITE_DQS); 2776 rw_mgr_mem_calibrate_write_test_issue(write_group * 2777 RW_MGR_MEM_VIRTUAL_GROUPS_PER_WRITE_DQS+vg, 2778 use_dm); 2779 2780 base_rw_mgr = readl(addr_rw_mgr); 2781 tmp_bit_chk = tmp_bit_chk | (correct_mask_vg & ~(base_rw_mgr)); 2782 if (vg == 0) 2783 break; 2784 } 2785 *bit_chk &= tmp_bit_chk; 2786 } 2787 2788 if (all_correct) { 2789 set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF); 2790 debug_cond(DLEVEL == 2, "write_test(%u,%u,ALL) : %u == \ 2791 %u => %lu", write_group, use_dm, 2792 *bit_chk, param->write_correct_mask, 2793 (long unsigned int)(*bit_chk == 2794 param->write_correct_mask)); 2795 return *bit_chk == param->write_correct_mask; 2796 } else { 2797 set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF); 2798 debug_cond(DLEVEL == 2, "write_test(%u,%u,ONE) : %u != ", 2799 write_group, use_dm, *bit_chk); 2800 debug_cond(DLEVEL == 2, "%lu" " => %lu", (long unsigned int)0, 2801 (long unsigned int)(*bit_chk != 0)); 2802 return *bit_chk != 0x00; 2803 } 2804 } 2805 2806 /* 2807 * center all windows. do per-bit-deskew to possibly increase size of 2808 * certain windows. 2809 */ 2810 static uint32_t rw_mgr_mem_calibrate_writes_center(uint32_t rank_bgn, 2811 uint32_t write_group, uint32_t test_bgn) 2812 { 2813 uint32_t i, p, min_index; 2814 int32_t d; 2815 /* 2816 * Store these as signed since there are comparisons with 2817 * signed numbers. 2818 */ 2819 uint32_t bit_chk; 2820 uint32_t sticky_bit_chk; 2821 int32_t left_edge[RW_MGR_MEM_DQ_PER_WRITE_DQS]; 2822 int32_t right_edge[RW_MGR_MEM_DQ_PER_WRITE_DQS]; 2823 int32_t mid; 2824 int32_t mid_min, orig_mid_min; 2825 int32_t new_dqs, start_dqs, shift_dq; 2826 int32_t dq_margin, dqs_margin, dm_margin; 2827 uint32_t stop; 2828 uint32_t temp_dq_out1_delay; 2829 uint32_t addr; 2830 2831 debug("%s:%d %u %u", __func__, __LINE__, write_group, test_bgn); 2832 2833 dm_margin = 0; 2834 2835 addr = SDR_PHYGRP_SCCGRP_ADDRESS | SCC_MGR_IO_OUT1_DELAY_OFFSET; 2836 start_dqs = readl(addr + 2837 (RW_MGR_MEM_DQ_PER_WRITE_DQS << 2)); 2838 2839 /* per-bit deskew */ 2840 2841 /* 2842 * set the left and right edge of each bit to an illegal value 2843 * use (IO_IO_OUT1_DELAY_MAX + 1) as an illegal value. 2844 */ 2845 sticky_bit_chk = 0; 2846 for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) { 2847 left_edge[i] = IO_IO_OUT1_DELAY_MAX + 1; 2848 right_edge[i] = IO_IO_OUT1_DELAY_MAX + 1; 2849 } 2850 2851 /* Search for the left edge of the window for each bit */ 2852 addr = (u32)&sdr_scc_mgr->update; 2853 for (d = 0; d <= IO_IO_OUT1_DELAY_MAX; d++) { 2854 scc_mgr_apply_group_dq_out1_delay(write_group, test_bgn, d); 2855 2856 writel(0, addr); 2857 2858 /* 2859 * Stop searching when the read test doesn't pass AND when 2860 * we've seen a passing read on every bit. 2861 */ 2862 stop = !rw_mgr_mem_calibrate_write_test(rank_bgn, write_group, 2863 0, PASS_ONE_BIT, &bit_chk, 0); 2864 sticky_bit_chk = sticky_bit_chk | bit_chk; 2865 stop = stop && (sticky_bit_chk == param->write_correct_mask); 2866 debug_cond(DLEVEL == 2, "write_center(left): dtap=%d => %u \ 2867 == %u && %u [bit_chk= %u ]\n", 2868 d, sticky_bit_chk, param->write_correct_mask, 2869 stop, bit_chk); 2870 2871 if (stop == 1) { 2872 break; 2873 } else { 2874 for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) { 2875 if (bit_chk & 1) { 2876 /* 2877 * Remember a passing test as the 2878 * left_edge. 2879 */ 2880 left_edge[i] = d; 2881 } else { 2882 /* 2883 * If a left edge has not been seen 2884 * yet, then a future passing test will 2885 * mark this edge as the right edge. 2886 */ 2887 if (left_edge[i] == 2888 IO_IO_OUT1_DELAY_MAX + 1) { 2889 right_edge[i] = -(d + 1); 2890 } 2891 } 2892 debug_cond(DLEVEL == 2, "write_center[l,d=%d):", d); 2893 debug_cond(DLEVEL == 2, "bit_chk_test=%d left_edge[%u]: %d", 2894 (int)(bit_chk & 1), i, left_edge[i]); 2895 debug_cond(DLEVEL == 2, "right_edge[%u]: %d\n", i, 2896 right_edge[i]); 2897 bit_chk = bit_chk >> 1; 2898 } 2899 } 2900 } 2901 2902 /* Reset DQ delay chains to 0 */ 2903 scc_mgr_apply_group_dq_out1_delay(write_group, test_bgn, 0); 2904 sticky_bit_chk = 0; 2905 for (i = RW_MGR_MEM_DQ_PER_WRITE_DQS - 1;; i--) { 2906 debug_cond(DLEVEL == 2, "%s:%d write_center: left_edge[%u]: \ 2907 %d right_edge[%u]: %d\n", __func__, __LINE__, 2908 i, left_edge[i], i, right_edge[i]); 2909 2910 /* 2911 * Check for cases where we haven't found the left edge, 2912 * which makes our assignment of the the right edge invalid. 2913 * Reset it to the illegal value. 2914 */ 2915 if ((left_edge[i] == IO_IO_OUT1_DELAY_MAX + 1) && 2916 (right_edge[i] != IO_IO_OUT1_DELAY_MAX + 1)) { 2917 right_edge[i] = IO_IO_OUT1_DELAY_MAX + 1; 2918 debug_cond(DLEVEL == 2, "%s:%d write_center: reset \ 2919 right_edge[%u]: %d\n", __func__, __LINE__, 2920 i, right_edge[i]); 2921 } 2922 2923 /* 2924 * Reset sticky bit (except for bits where we have 2925 * seen the left edge). 2926 */ 2927 sticky_bit_chk = sticky_bit_chk << 1; 2928 if ((left_edge[i] != IO_IO_OUT1_DELAY_MAX + 1)) 2929 sticky_bit_chk = sticky_bit_chk | 1; 2930 2931 if (i == 0) 2932 break; 2933 } 2934 2935 /* Search for the right edge of the window for each bit */ 2936 addr = (u32)&sdr_scc_mgr->update; 2937 for (d = 0; d <= IO_IO_OUT1_DELAY_MAX - start_dqs; d++) { 2938 scc_mgr_apply_group_dqs_io_and_oct_out1(write_group, 2939 d + start_dqs); 2940 2941 writel(0, addr); 2942 2943 /* 2944 * Stop searching when the read test doesn't pass AND when 2945 * we've seen a passing read on every bit. 2946 */ 2947 stop = !rw_mgr_mem_calibrate_write_test(rank_bgn, write_group, 2948 0, PASS_ONE_BIT, &bit_chk, 0); 2949 2950 sticky_bit_chk = sticky_bit_chk | bit_chk; 2951 stop = stop && (sticky_bit_chk == param->write_correct_mask); 2952 2953 debug_cond(DLEVEL == 2, "write_center (right): dtap=%u => %u == \ 2954 %u && %u\n", d, sticky_bit_chk, 2955 param->write_correct_mask, stop); 2956 2957 if (stop == 1) { 2958 if (d == 0) { 2959 for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; 2960 i++) { 2961 /* d = 0 failed, but it passed when 2962 testing the left edge, so it must be 2963 marginal, set it to -1 */ 2964 if (right_edge[i] == 2965 IO_IO_OUT1_DELAY_MAX + 1 && 2966 left_edge[i] != 2967 IO_IO_OUT1_DELAY_MAX + 1) { 2968 right_edge[i] = -1; 2969 } 2970 } 2971 } 2972 break; 2973 } else { 2974 for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) { 2975 if (bit_chk & 1) { 2976 /* 2977 * Remember a passing test as 2978 * the right_edge. 2979 */ 2980 right_edge[i] = d; 2981 } else { 2982 if (d != 0) { 2983 /* 2984 * If a right edge has not 2985 * been seen yet, then a future 2986 * passing test will mark this 2987 * edge as the left edge. 2988 */ 2989 if (right_edge[i] == 2990 IO_IO_OUT1_DELAY_MAX + 1) 2991 left_edge[i] = -(d + 1); 2992 } else { 2993 /* 2994 * d = 0 failed, but it passed 2995 * when testing the left edge, 2996 * so it must be marginal, set 2997 * it to -1. 2998 */ 2999 if (right_edge[i] == 3000 IO_IO_OUT1_DELAY_MAX + 1 && 3001 left_edge[i] != 3002 IO_IO_OUT1_DELAY_MAX + 1) 3003 right_edge[i] = -1; 3004 /* 3005 * If a right edge has not been 3006 * seen yet, then a future 3007 * passing test will mark this 3008 * edge as the left edge. 3009 */ 3010 else if (right_edge[i] == 3011 IO_IO_OUT1_DELAY_MAX + 3012 1) 3013 left_edge[i] = -(d + 1); 3014 } 3015 } 3016 debug_cond(DLEVEL == 2, "write_center[r,d=%d):", d); 3017 debug_cond(DLEVEL == 2, "bit_chk_test=%d left_edge[%u]: %d", 3018 (int)(bit_chk & 1), i, left_edge[i]); 3019 debug_cond(DLEVEL == 2, "right_edge[%u]: %d\n", i, 3020 right_edge[i]); 3021 bit_chk = bit_chk >> 1; 3022 } 3023 } 3024 } 3025 3026 /* Check that all bits have a window */ 3027 for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) { 3028 debug_cond(DLEVEL == 2, "%s:%d write_center: left_edge[%u]: \ 3029 %d right_edge[%u]: %d", __func__, __LINE__, 3030 i, left_edge[i], i, right_edge[i]); 3031 if ((left_edge[i] == IO_IO_OUT1_DELAY_MAX + 1) || 3032 (right_edge[i] == IO_IO_OUT1_DELAY_MAX + 1)) { 3033 set_failing_group_stage(test_bgn + i, 3034 CAL_STAGE_WRITES, 3035 CAL_SUBSTAGE_WRITES_CENTER); 3036 return 0; 3037 } 3038 } 3039 3040 /* Find middle of window for each DQ bit */ 3041 mid_min = left_edge[0] - right_edge[0]; 3042 min_index = 0; 3043 for (i = 1; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) { 3044 mid = left_edge[i] - right_edge[i]; 3045 if (mid < mid_min) { 3046 mid_min = mid; 3047 min_index = i; 3048 } 3049 } 3050 3051 /* 3052 * -mid_min/2 represents the amount that we need to move DQS. 3053 * If mid_min is odd and positive we'll need to add one to 3054 * make sure the rounding in further calculations is correct 3055 * (always bias to the right), so just add 1 for all positive values. 3056 */ 3057 if (mid_min > 0) 3058 mid_min++; 3059 mid_min = mid_min / 2; 3060 debug_cond(DLEVEL == 1, "%s:%d write_center: mid_min=%d\n", __func__, 3061 __LINE__, mid_min); 3062 3063 /* Determine the amount we can change DQS (which is -mid_min) */ 3064 orig_mid_min = mid_min; 3065 new_dqs = start_dqs; 3066 mid_min = 0; 3067 debug_cond(DLEVEL == 1, "%s:%d write_center: start_dqs=%d new_dqs=%d \ 3068 mid_min=%d\n", __func__, __LINE__, start_dqs, new_dqs, mid_min); 3069 /* Initialize data for export structures */ 3070 dqs_margin = IO_IO_OUT1_DELAY_MAX + 1; 3071 dq_margin = IO_IO_OUT1_DELAY_MAX + 1; 3072 3073 /* add delay to bring centre of all DQ windows to the same "level" */ 3074 addr = SDR_PHYGRP_SCCGRP_ADDRESS | SCC_MGR_IO_OUT1_DELAY_OFFSET; 3075 for (i = 0, p = test_bgn; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++, p++) { 3076 /* Use values before divide by 2 to reduce round off error */ 3077 shift_dq = (left_edge[i] - right_edge[i] - 3078 (left_edge[min_index] - right_edge[min_index]))/2 + 3079 (orig_mid_min - mid_min); 3080 3081 debug_cond(DLEVEL == 2, "%s:%d write_center: before: shift_dq \ 3082 [%u]=%d\n", __func__, __LINE__, i, shift_dq); 3083 3084 temp_dq_out1_delay = readl(addr + (i << 2)); 3085 if (shift_dq + (int32_t)temp_dq_out1_delay > 3086 (int32_t)IO_IO_OUT1_DELAY_MAX) { 3087 shift_dq = (int32_t)IO_IO_OUT1_DELAY_MAX - temp_dq_out1_delay; 3088 } else if (shift_dq + (int32_t)temp_dq_out1_delay < 0) { 3089 shift_dq = -(int32_t)temp_dq_out1_delay; 3090 } 3091 debug_cond(DLEVEL == 2, "write_center: after: shift_dq[%u]=%d\n", 3092 i, shift_dq); 3093 scc_mgr_set_dq_out1_delay(write_group, i, temp_dq_out1_delay + 3094 shift_dq); 3095 scc_mgr_load_dq(i); 3096 3097 debug_cond(DLEVEL == 2, "write_center: margin[%u]=[%d,%d]\n", i, 3098 left_edge[i] - shift_dq + (-mid_min), 3099 right_edge[i] + shift_dq - (-mid_min)); 3100 /* To determine values for export structures */ 3101 if (left_edge[i] - shift_dq + (-mid_min) < dq_margin) 3102 dq_margin = left_edge[i] - shift_dq + (-mid_min); 3103 3104 if (right_edge[i] + shift_dq - (-mid_min) < dqs_margin) 3105 dqs_margin = right_edge[i] + shift_dq - (-mid_min); 3106 } 3107 3108 /* Move DQS */ 3109 scc_mgr_apply_group_dqs_io_and_oct_out1(write_group, new_dqs); 3110 addr = (u32)&sdr_scc_mgr->update; 3111 writel(0, addr); 3112 3113 /* Centre DM */ 3114 debug_cond(DLEVEL == 2, "%s:%d write_center: DM\n", __func__, __LINE__); 3115 3116 /* 3117 * set the left and right edge of each bit to an illegal value, 3118 * use (IO_IO_OUT1_DELAY_MAX + 1) as an illegal value, 3119 */ 3120 left_edge[0] = IO_IO_OUT1_DELAY_MAX + 1; 3121 right_edge[0] = IO_IO_OUT1_DELAY_MAX + 1; 3122 int32_t bgn_curr = IO_IO_OUT1_DELAY_MAX + 1; 3123 int32_t end_curr = IO_IO_OUT1_DELAY_MAX + 1; 3124 int32_t bgn_best = IO_IO_OUT1_DELAY_MAX + 1; 3125 int32_t end_best = IO_IO_OUT1_DELAY_MAX + 1; 3126 int32_t win_best = 0; 3127 3128 /* Search for the/part of the window with DM shift */ 3129 addr = (u32)&sdr_scc_mgr->update; 3130 for (d = IO_IO_OUT1_DELAY_MAX; d >= 0; d -= DELTA_D) { 3131 scc_mgr_apply_group_dm_out1_delay(write_group, d); 3132 writel(0, addr); 3133 3134 if (rw_mgr_mem_calibrate_write_test(rank_bgn, write_group, 1, 3135 PASS_ALL_BITS, &bit_chk, 3136 0)) { 3137 /* USE Set current end of the window */ 3138 end_curr = -d; 3139 /* 3140 * If a starting edge of our window has not been seen 3141 * this is our current start of the DM window. 3142 */ 3143 if (bgn_curr == IO_IO_OUT1_DELAY_MAX + 1) 3144 bgn_curr = -d; 3145 3146 /* 3147 * If current window is bigger than best seen. 3148 * Set best seen to be current window. 3149 */ 3150 if ((end_curr-bgn_curr+1) > win_best) { 3151 win_best = end_curr-bgn_curr+1; 3152 bgn_best = bgn_curr; 3153 end_best = end_curr; 3154 } 3155 } else { 3156 /* We just saw a failing test. Reset temp edge */ 3157 bgn_curr = IO_IO_OUT1_DELAY_MAX + 1; 3158 end_curr = IO_IO_OUT1_DELAY_MAX + 1; 3159 } 3160 } 3161 3162 3163 /* Reset DM delay chains to 0 */ 3164 scc_mgr_apply_group_dm_out1_delay(write_group, 0); 3165 3166 /* 3167 * Check to see if the current window nudges up aganist 0 delay. 3168 * If so we need to continue the search by shifting DQS otherwise DQS 3169 * search begins as a new search. */ 3170 if (end_curr != 0) { 3171 bgn_curr = IO_IO_OUT1_DELAY_MAX + 1; 3172 end_curr = IO_IO_OUT1_DELAY_MAX + 1; 3173 } 3174 3175 /* Search for the/part of the window with DQS shifts */ 3176 addr = (u32)&sdr_scc_mgr->update; 3177 for (d = 0; d <= IO_IO_OUT1_DELAY_MAX - new_dqs; d += DELTA_D) { 3178 /* 3179 * Note: This only shifts DQS, so are we limiting ourselve to 3180 * width of DQ unnecessarily. 3181 */ 3182 scc_mgr_apply_group_dqs_io_and_oct_out1(write_group, 3183 d + new_dqs); 3184 3185 writel(0, addr); 3186 if (rw_mgr_mem_calibrate_write_test(rank_bgn, write_group, 1, 3187 PASS_ALL_BITS, &bit_chk, 3188 0)) { 3189 /* USE Set current end of the window */ 3190 end_curr = d; 3191 /* 3192 * If a beginning edge of our window has not been seen 3193 * this is our current begin of the DM window. 3194 */ 3195 if (bgn_curr == IO_IO_OUT1_DELAY_MAX + 1) 3196 bgn_curr = d; 3197 3198 /* 3199 * If current window is bigger than best seen. Set best 3200 * seen to be current window. 3201 */ 3202 if ((end_curr-bgn_curr+1) > win_best) { 3203 win_best = end_curr-bgn_curr+1; 3204 bgn_best = bgn_curr; 3205 end_best = end_curr; 3206 } 3207 } else { 3208 /* We just saw a failing test. Reset temp edge */ 3209 bgn_curr = IO_IO_OUT1_DELAY_MAX + 1; 3210 end_curr = IO_IO_OUT1_DELAY_MAX + 1; 3211 3212 /* Early exit optimization: if ther remaining delay 3213 chain space is less than already seen largest window 3214 we can exit */ 3215 if ((win_best-1) > 3216 (IO_IO_OUT1_DELAY_MAX - new_dqs - d)) { 3217 break; 3218 } 3219 } 3220 } 3221 3222 /* assign left and right edge for cal and reporting; */ 3223 left_edge[0] = -1*bgn_best; 3224 right_edge[0] = end_best; 3225 3226 debug_cond(DLEVEL == 2, "%s:%d dm_calib: left=%d right=%d\n", __func__, 3227 __LINE__, left_edge[0], right_edge[0]); 3228 3229 /* Move DQS (back to orig) */ 3230 scc_mgr_apply_group_dqs_io_and_oct_out1(write_group, new_dqs); 3231 3232 /* Move DM */ 3233 3234 /* Find middle of window for the DM bit */ 3235 mid = (left_edge[0] - right_edge[0]) / 2; 3236 3237 /* only move right, since we are not moving DQS/DQ */ 3238 if (mid < 0) 3239 mid = 0; 3240 3241 /* dm_marign should fail if we never find a window */ 3242 if (win_best == 0) 3243 dm_margin = -1; 3244 else 3245 dm_margin = left_edge[0] - mid; 3246 3247 scc_mgr_apply_group_dm_out1_delay(write_group, mid); 3248 addr = (u32)&sdr_scc_mgr->update; 3249 writel(0, addr); 3250 3251 debug_cond(DLEVEL == 2, "%s:%d dm_calib: left=%d right=%d mid=%d \ 3252 dm_margin=%d\n", __func__, __LINE__, left_edge[0], 3253 right_edge[0], mid, dm_margin); 3254 /* Export values */ 3255 gbl->fom_out += dq_margin + dqs_margin; 3256 3257 debug_cond(DLEVEL == 2, "%s:%d write_center: dq_margin=%d \ 3258 dqs_margin=%d dm_margin=%d\n", __func__, __LINE__, 3259 dq_margin, dqs_margin, dm_margin); 3260 3261 /* 3262 * Do not remove this line as it makes sure all of our 3263 * decisions have been applied. 3264 */ 3265 addr = (u32)&sdr_scc_mgr->update; 3266 writel(0, addr); 3267 return (dq_margin >= 0) && (dqs_margin >= 0) && (dm_margin >= 0); 3268 } 3269 3270 /* calibrate the write operations */ 3271 static uint32_t rw_mgr_mem_calibrate_writes(uint32_t rank_bgn, uint32_t g, 3272 uint32_t test_bgn) 3273 { 3274 /* update info for sims */ 3275 debug("%s:%d %u %u\n", __func__, __LINE__, g, test_bgn); 3276 3277 reg_file_set_stage(CAL_STAGE_WRITES); 3278 reg_file_set_sub_stage(CAL_SUBSTAGE_WRITES_CENTER); 3279 3280 reg_file_set_group(g); 3281 3282 if (!rw_mgr_mem_calibrate_writes_center(rank_bgn, g, test_bgn)) { 3283 set_failing_group_stage(g, CAL_STAGE_WRITES, 3284 CAL_SUBSTAGE_WRITES_CENTER); 3285 return 0; 3286 } 3287 3288 return 1; 3289 } 3290 3291 /* precharge all banks and activate row 0 in bank "000..." and bank "111..." */ 3292 static void mem_precharge_and_activate(void) 3293 { 3294 uint32_t r; 3295 uint32_t addr; 3296 3297 for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; r++) { 3298 if (param->skip_ranks[r]) { 3299 /* request to skip the rank */ 3300 continue; 3301 } 3302 3303 /* set rank */ 3304 set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_OFF); 3305 3306 /* precharge all banks ... */ 3307 addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_SINGLE_GROUP_OFFSET; 3308 writel(RW_MGR_PRECHARGE_ALL, addr); 3309 3310 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr0; 3311 writel(0x0F, addr); 3312 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add0; 3313 writel(RW_MGR_ACTIVATE_0_AND_1_WAIT1, addr); 3314 3315 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr1; 3316 writel(0x0F, addr); 3317 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add1; 3318 writel(RW_MGR_ACTIVATE_0_AND_1_WAIT2, addr); 3319 3320 /* activate rows */ 3321 addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_SINGLE_GROUP_OFFSET; 3322 writel(RW_MGR_ACTIVATE_0_AND_1, addr); 3323 } 3324 } 3325 3326 /* Configure various memory related parameters. */ 3327 static void mem_config(void) 3328 { 3329 uint32_t rlat, wlat; 3330 uint32_t rw_wl_nop_cycles; 3331 uint32_t max_latency; 3332 uint32_t addr; 3333 3334 debug("%s:%d\n", __func__, __LINE__); 3335 /* read in write and read latency */ 3336 addr = (u32)&data_mgr->t_wl_add; 3337 wlat = readl(addr); 3338 3339 addr = (u32)&data_mgr->mem_t_add; 3340 wlat += readl(addr); 3341 /* WL for hard phy does not include additive latency */ 3342 3343 /* 3344 * add addtional write latency to offset the address/command extra 3345 * clock cycle. We change the AC mux setting causing AC to be delayed 3346 * by one mem clock cycle. Only do this for DDR3 3347 */ 3348 wlat = wlat + 1; 3349 3350 addr = (u32)&data_mgr->t_rl_add; 3351 rlat = readl(addr); 3352 3353 rw_wl_nop_cycles = wlat - 2; 3354 gbl->rw_wl_nop_cycles = rw_wl_nop_cycles; 3355 3356 /* 3357 * For AV/CV, lfifo is hardened and always runs at full rate so 3358 * max latency in AFI clocks, used here, is correspondingly smaller. 3359 */ 3360 max_latency = (1<<MAX_LATENCY_COUNT_WIDTH)/1 - 1; 3361 /* configure for a burst length of 8 */ 3362 3363 /* write latency */ 3364 /* Adjust Write Latency for Hard PHY */ 3365 wlat = wlat + 1; 3366 3367 /* set a pretty high read latency initially */ 3368 gbl->curr_read_lat = rlat + 16; 3369 3370 if (gbl->curr_read_lat > max_latency) 3371 gbl->curr_read_lat = max_latency; 3372 3373 addr = (u32)&phy_mgr_cfg->phy_rlat; 3374 writel(gbl->curr_read_lat, addr); 3375 3376 /* advertise write latency */ 3377 gbl->curr_write_lat = wlat; 3378 addr = (u32)&phy_mgr_cfg->afi_wlat; 3379 writel(wlat - 2, addr); 3380 3381 /* initialize bit slips */ 3382 mem_precharge_and_activate(); 3383 } 3384 3385 /* Set VFIFO and LFIFO to instant-on settings in skip calibration mode */ 3386 static void mem_skip_calibrate(void) 3387 { 3388 uint32_t vfifo_offset; 3389 uint32_t i, j, r; 3390 uint32_t addr; 3391 3392 debug("%s:%d\n", __func__, __LINE__); 3393 /* Need to update every shadow register set used by the interface */ 3394 for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; 3395 r += NUM_RANKS_PER_SHADOW_REG) { 3396 /* 3397 * Set output phase alignment settings appropriate for 3398 * skip calibration. 3399 */ 3400 for (i = 0; i < RW_MGR_MEM_IF_READ_DQS_WIDTH; i++) { 3401 scc_mgr_set_dqs_en_phase(i, 0); 3402 #if IO_DLL_CHAIN_LENGTH == 6 3403 scc_mgr_set_dqdqs_output_phase(i, 6); 3404 #else 3405 scc_mgr_set_dqdqs_output_phase(i, 7); 3406 #endif 3407 /* 3408 * Case:33398 3409 * 3410 * Write data arrives to the I/O two cycles before write 3411 * latency is reached (720 deg). 3412 * -> due to bit-slip in a/c bus 3413 * -> to allow board skew where dqs is longer than ck 3414 * -> how often can this happen!? 3415 * -> can claim back some ptaps for high freq 3416 * support if we can relax this, but i digress... 3417 * 3418 * The write_clk leads mem_ck by 90 deg 3419 * The minimum ptap of the OPA is 180 deg 3420 * Each ptap has (360 / IO_DLL_CHAIN_LENGH) deg of delay 3421 * The write_clk is always delayed by 2 ptaps 3422 * 3423 * Hence, to make DQS aligned to CK, we need to delay 3424 * DQS by: 3425 * (720 - 90 - 180 - 2 * (360 / IO_DLL_CHAIN_LENGTH)) 3426 * 3427 * Dividing the above by (360 / IO_DLL_CHAIN_LENGTH) 3428 * gives us the number of ptaps, which simplies to: 3429 * 3430 * (1.25 * IO_DLL_CHAIN_LENGTH - 2) 3431 */ 3432 scc_mgr_set_dqdqs_output_phase(i, (1.25 * 3433 IO_DLL_CHAIN_LENGTH - 2)); 3434 } 3435 addr = (u32)&sdr_scc_mgr->dqs_ena; 3436 writel(0xff, addr); 3437 addr = (u32)&sdr_scc_mgr->dqs_io_ena; 3438 writel(0xff, addr); 3439 3440 addr = SDR_PHYGRP_SCCGRP_ADDRESS | SCC_MGR_GROUP_COUNTER_OFFSET; 3441 for (i = 0; i < RW_MGR_MEM_IF_WRITE_DQS_WIDTH; i++) { 3442 writel(i, addr); 3443 } 3444 addr = (u32)&sdr_scc_mgr->dq_ena; 3445 writel(0xff, addr); 3446 addr = (u32)&sdr_scc_mgr->dm_ena; 3447 writel(0xff, addr); 3448 addr = (u32)&sdr_scc_mgr->update; 3449 writel(0, addr); 3450 } 3451 3452 /* Compensate for simulation model behaviour */ 3453 for (i = 0; i < RW_MGR_MEM_IF_READ_DQS_WIDTH; i++) { 3454 scc_mgr_set_dqs_bus_in_delay(i, 10); 3455 scc_mgr_load_dqs(i); 3456 } 3457 addr = (u32)&sdr_scc_mgr->update; 3458 writel(0, addr); 3459 3460 /* 3461 * ArriaV has hard FIFOs that can only be initialized by incrementing 3462 * in sequencer. 3463 */ 3464 vfifo_offset = CALIB_VFIFO_OFFSET; 3465 addr = (u32)&phy_mgr_cmd->inc_vfifo_hard_phy; 3466 for (j = 0; j < vfifo_offset; j++) { 3467 writel(0xff, addr); 3468 } 3469 addr = (u32)&phy_mgr_cmd->fifo_reset; 3470 writel(0, addr); 3471 3472 /* 3473 * For ACV with hard lfifo, we get the skip-cal setting from 3474 * generation-time constant. 3475 */ 3476 gbl->curr_read_lat = CALIB_LFIFO_OFFSET; 3477 addr = (u32)&phy_mgr_cfg->phy_rlat; 3478 writel(gbl->curr_read_lat, addr); 3479 } 3480 3481 /* Memory calibration entry point */ 3482 static uint32_t mem_calibrate(void) 3483 { 3484 uint32_t i; 3485 uint32_t rank_bgn, sr; 3486 uint32_t write_group, write_test_bgn; 3487 uint32_t read_group, read_test_bgn; 3488 uint32_t run_groups, current_run; 3489 uint32_t failing_groups = 0; 3490 uint32_t group_failed = 0; 3491 uint32_t sr_failed = 0; 3492 uint32_t addr; 3493 3494 debug("%s:%d\n", __func__, __LINE__); 3495 /* Initialize the data settings */ 3496 3497 gbl->error_substage = CAL_SUBSTAGE_NIL; 3498 gbl->error_stage = CAL_STAGE_NIL; 3499 gbl->error_group = 0xff; 3500 gbl->fom_in = 0; 3501 gbl->fom_out = 0; 3502 3503 mem_config(); 3504 3505 uint32_t bypass_mode = 0x1; 3506 addr = SDR_PHYGRP_SCCGRP_ADDRESS | SCC_MGR_GROUP_COUNTER_OFFSET; 3507 for (i = 0; i < RW_MGR_MEM_IF_READ_DQS_WIDTH; i++) { 3508 writel(i, addr); 3509 scc_set_bypass_mode(i, bypass_mode); 3510 } 3511 3512 if ((dyn_calib_steps & CALIB_SKIP_ALL) == CALIB_SKIP_ALL) { 3513 /* 3514 * Set VFIFO and LFIFO to instant-on settings in skip 3515 * calibration mode. 3516 */ 3517 mem_skip_calibrate(); 3518 } else { 3519 for (i = 0; i < NUM_CALIB_REPEAT; i++) { 3520 /* 3521 * Zero all delay chain/phase settings for all 3522 * groups and all shadow register sets. 3523 */ 3524 scc_mgr_zero_all(); 3525 3526 run_groups = ~param->skip_groups; 3527 3528 for (write_group = 0, write_test_bgn = 0; write_group 3529 < RW_MGR_MEM_IF_WRITE_DQS_WIDTH; write_group++, 3530 write_test_bgn += RW_MGR_MEM_DQ_PER_WRITE_DQS) { 3531 /* Initialized the group failure */ 3532 group_failed = 0; 3533 3534 current_run = run_groups & ((1 << 3535 RW_MGR_NUM_DQS_PER_WRITE_GROUP) - 1); 3536 run_groups = run_groups >> 3537 RW_MGR_NUM_DQS_PER_WRITE_GROUP; 3538 3539 if (current_run == 0) 3540 continue; 3541 3542 addr = SDR_PHYGRP_SCCGRP_ADDRESS | SCC_MGR_GROUP_COUNTER_OFFSET; 3543 writel(write_group, addr); 3544 scc_mgr_zero_group(write_group, write_test_bgn, 3545 0); 3546 3547 for (read_group = write_group * 3548 RW_MGR_MEM_IF_READ_DQS_WIDTH / 3549 RW_MGR_MEM_IF_WRITE_DQS_WIDTH, 3550 read_test_bgn = 0; 3551 read_group < (write_group + 1) * 3552 RW_MGR_MEM_IF_READ_DQS_WIDTH / 3553 RW_MGR_MEM_IF_WRITE_DQS_WIDTH && 3554 group_failed == 0; 3555 read_group++, read_test_bgn += 3556 RW_MGR_MEM_DQ_PER_READ_DQS) { 3557 /* Calibrate the VFIFO */ 3558 if (!((STATIC_CALIB_STEPS) & 3559 CALIB_SKIP_VFIFO)) { 3560 if (!rw_mgr_mem_calibrate_vfifo 3561 (read_group, 3562 read_test_bgn)) { 3563 group_failed = 1; 3564 3565 if (!(gbl-> 3566 phy_debug_mode_flags & 3567 PHY_DEBUG_SWEEP_ALL_GROUPS)) { 3568 return 0; 3569 } 3570 } 3571 } 3572 } 3573 3574 /* Calibrate the output side */ 3575 if (group_failed == 0) { 3576 for (rank_bgn = 0, sr = 0; rank_bgn 3577 < RW_MGR_MEM_NUMBER_OF_RANKS; 3578 rank_bgn += 3579 NUM_RANKS_PER_SHADOW_REG, 3580 ++sr) { 3581 sr_failed = 0; 3582 if (!((STATIC_CALIB_STEPS) & 3583 CALIB_SKIP_WRITES)) { 3584 if ((STATIC_CALIB_STEPS) 3585 & CALIB_SKIP_DELAY_SWEEPS) { 3586 /* not needed in quick mode! */ 3587 } else { 3588 /* 3589 * Determine if this set of 3590 * ranks should be skipped 3591 * entirely. 3592 */ 3593 if (!param->skip_shadow_regs[sr]) { 3594 if (!rw_mgr_mem_calibrate_writes 3595 (rank_bgn, write_group, 3596 write_test_bgn)) { 3597 sr_failed = 1; 3598 if (!(gbl-> 3599 phy_debug_mode_flags & 3600 PHY_DEBUG_SWEEP_ALL_GROUPS)) { 3601 return 0; 3602 } 3603 } 3604 } 3605 } 3606 } 3607 if (sr_failed != 0) 3608 group_failed = 1; 3609 } 3610 } 3611 3612 if (group_failed == 0) { 3613 for (read_group = write_group * 3614 RW_MGR_MEM_IF_READ_DQS_WIDTH / 3615 RW_MGR_MEM_IF_WRITE_DQS_WIDTH, 3616 read_test_bgn = 0; 3617 read_group < (write_group + 1) 3618 * RW_MGR_MEM_IF_READ_DQS_WIDTH 3619 / RW_MGR_MEM_IF_WRITE_DQS_WIDTH && 3620 group_failed == 0; 3621 read_group++, read_test_bgn += 3622 RW_MGR_MEM_DQ_PER_READ_DQS) { 3623 if (!((STATIC_CALIB_STEPS) & 3624 CALIB_SKIP_WRITES)) { 3625 if (!rw_mgr_mem_calibrate_vfifo_end 3626 (read_group, read_test_bgn)) { 3627 group_failed = 1; 3628 3629 if (!(gbl->phy_debug_mode_flags 3630 & PHY_DEBUG_SWEEP_ALL_GROUPS)) { 3631 return 0; 3632 } 3633 } 3634 } 3635 } 3636 } 3637 3638 if (group_failed != 0) 3639 failing_groups++; 3640 } 3641 3642 /* 3643 * USER If there are any failing groups then report 3644 * the failure. 3645 */ 3646 if (failing_groups != 0) 3647 return 0; 3648 3649 /* Calibrate the LFIFO */ 3650 if (!((STATIC_CALIB_STEPS) & CALIB_SKIP_LFIFO)) { 3651 /* 3652 * If we're skipping groups as part of debug, 3653 * don't calibrate LFIFO. 3654 */ 3655 if (param->skip_groups == 0) { 3656 if (!rw_mgr_mem_calibrate_lfifo()) 3657 return 0; 3658 } 3659 } 3660 } 3661 } 3662 3663 /* 3664 * Do not remove this line as it makes sure all of our decisions 3665 * have been applied. 3666 */ 3667 addr = (u32)&sdr_scc_mgr->update; 3668 writel(0, addr); 3669 return 1; 3670 } 3671 3672 static uint32_t run_mem_calibrate(void) 3673 { 3674 uint32_t pass; 3675 uint32_t debug_info; 3676 uint32_t addr; 3677 3678 debug("%s:%d\n", __func__, __LINE__); 3679 3680 /* Reset pass/fail status shown on afi_cal_success/fail */ 3681 addr = (u32)&phy_mgr_cfg->cal_status; 3682 writel(PHY_MGR_CAL_RESET, addr); 3683 3684 /* stop tracking manger */ 3685 uint32_t ctrlcfg = readl(&sdr_ctrl->ctrl_cfg); 3686 3687 writel(ctrlcfg & 0xFFBFFFFF, &sdr_ctrl->ctrl_cfg); 3688 3689 initialize(); 3690 rw_mgr_mem_initialize(); 3691 3692 pass = mem_calibrate(); 3693 3694 mem_precharge_and_activate(); 3695 addr = (u32)&phy_mgr_cmd->fifo_reset; 3696 writel(0, addr); 3697 3698 /* 3699 * Handoff: 3700 * Don't return control of the PHY back to AFI when in debug mode. 3701 */ 3702 if ((gbl->phy_debug_mode_flags & PHY_DEBUG_IN_DEBUG_MODE) == 0) { 3703 rw_mgr_mem_handoff(); 3704 /* 3705 * In Hard PHY this is a 2-bit control: 3706 * 0: AFI Mux Select 3707 * 1: DDIO Mux Select 3708 */ 3709 addr = (u32)&phy_mgr_cfg->mux_sel; 3710 writel(0x2, addr); 3711 } 3712 3713 writel(ctrlcfg, &sdr_ctrl->ctrl_cfg); 3714 3715 if (pass) { 3716 printf("%s: CALIBRATION PASSED\n", __FILE__); 3717 3718 gbl->fom_in /= 2; 3719 gbl->fom_out /= 2; 3720 3721 if (gbl->fom_in > 0xff) 3722 gbl->fom_in = 0xff; 3723 3724 if (gbl->fom_out > 0xff) 3725 gbl->fom_out = 0xff; 3726 3727 /* Update the FOM in the register file */ 3728 debug_info = gbl->fom_in; 3729 debug_info |= gbl->fom_out << 8; 3730 addr = (u32)&sdr_reg_file->fom; 3731 writel(debug_info, addr); 3732 3733 addr = (u32)&phy_mgr_cfg->cal_debug_info; 3734 writel(debug_info, addr); 3735 addr = (u32)&phy_mgr_cfg->cal_status; 3736 writel(PHY_MGR_CAL_SUCCESS, addr); 3737 } else { 3738 printf("%s: CALIBRATION FAILED\n", __FILE__); 3739 3740 debug_info = gbl->error_stage; 3741 debug_info |= gbl->error_substage << 8; 3742 debug_info |= gbl->error_group << 16; 3743 3744 addr = (u32)&sdr_reg_file->failing_stage; 3745 writel(debug_info, addr); 3746 addr = (u32)&phy_mgr_cfg->cal_debug_info; 3747 writel(debug_info, addr); 3748 addr = (u32)&phy_mgr_cfg->cal_status; 3749 writel(PHY_MGR_CAL_FAIL, addr); 3750 3751 /* Update the failing group/stage in the register file */ 3752 debug_info = gbl->error_stage; 3753 debug_info |= gbl->error_substage << 8; 3754 debug_info |= gbl->error_group << 16; 3755 addr = (u32)&sdr_reg_file->failing_stage; 3756 writel(debug_info, addr); 3757 } 3758 3759 return pass; 3760 } 3761 3762 static void hc_initialize_rom_data(void) 3763 { 3764 uint32_t i; 3765 uint32_t addr; 3766 3767 addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_INST_ROM_WRITE_OFFSET; 3768 for (i = 0; i < ARRAY_SIZE(inst_rom_init); i++) { 3769 uint32_t data = inst_rom_init[i]; 3770 writel(data, addr + (i << 2)); 3771 } 3772 3773 addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_AC_ROM_WRITE_OFFSET; 3774 for (i = 0; i < ARRAY_SIZE(ac_rom_init); i++) { 3775 uint32_t data = ac_rom_init[i]; 3776 writel(data, addr + (i << 2)); 3777 } 3778 } 3779 3780 static void initialize_reg_file(void) 3781 { 3782 uint32_t addr; 3783 3784 /* Initialize the register file with the correct data */ 3785 addr = (u32)&sdr_reg_file->signature; 3786 writel(REG_FILE_INIT_SEQ_SIGNATURE, addr); 3787 3788 addr = (u32)&sdr_reg_file->debug_data_addr; 3789 writel(0, addr); 3790 3791 addr = (u32)&sdr_reg_file->cur_stage; 3792 writel(0, addr); 3793 3794 addr = (u32)&sdr_reg_file->fom; 3795 writel(0, addr); 3796 3797 addr = (u32)&sdr_reg_file->failing_stage; 3798 writel(0, addr); 3799 3800 addr = (u32)&sdr_reg_file->debug1; 3801 writel(0, addr); 3802 3803 addr = (u32)&sdr_reg_file->debug2; 3804 writel(0, addr); 3805 } 3806 3807 static void initialize_hps_phy(void) 3808 { 3809 uint32_t reg; 3810 /* 3811 * Tracking also gets configured here because it's in the 3812 * same register. 3813 */ 3814 uint32_t trk_sample_count = 7500; 3815 uint32_t trk_long_idle_sample_count = (10 << 16) | 100; 3816 /* 3817 * Format is number of outer loops in the 16 MSB, sample 3818 * count in 16 LSB. 3819 */ 3820 3821 reg = 0; 3822 reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_ACDELAYEN_SET(2); 3823 reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_DQDELAYEN_SET(1); 3824 reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_DQSDELAYEN_SET(1); 3825 reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_DQSLOGICDELAYEN_SET(1); 3826 reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_RESETDELAYEN_SET(0); 3827 reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_LPDDRDIS_SET(1); 3828 /* 3829 * This field selects the intrinsic latency to RDATA_EN/FULL path. 3830 * 00-bypass, 01- add 5 cycles, 10- add 10 cycles, 11- add 15 cycles. 3831 */ 3832 reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_ADDLATSEL_SET(0); 3833 reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_SAMPLECOUNT_19_0_SET( 3834 trk_sample_count); 3835 writel(reg, &sdr_ctrl->phy_ctrl0); 3836 3837 reg = 0; 3838 reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_1_SAMPLECOUNT_31_20_SET( 3839 trk_sample_count >> 3840 SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_SAMPLECOUNT_19_0_WIDTH); 3841 reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_1_LONGIDLESAMPLECOUNT_19_0_SET( 3842 trk_long_idle_sample_count); 3843 writel(reg, &sdr_ctrl->phy_ctrl1); 3844 3845 reg = 0; 3846 reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_2_LONGIDLESAMPLECOUNT_31_20_SET( 3847 trk_long_idle_sample_count >> 3848 SDR_CTRLGRP_PHYCTRL_PHYCTRL_1_LONGIDLESAMPLECOUNT_19_0_WIDTH); 3849 writel(reg, &sdr_ctrl->phy_ctrl2); 3850 } 3851 3852 static void initialize_tracking(void) 3853 { 3854 uint32_t concatenated_longidle = 0x0; 3855 uint32_t concatenated_delays = 0x0; 3856 uint32_t concatenated_rw_addr = 0x0; 3857 uint32_t concatenated_refresh = 0x0; 3858 uint32_t trk_sample_count = 7500; 3859 uint32_t dtaps_per_ptap; 3860 uint32_t tmp_delay; 3861 uint32_t addr; 3862 3863 /* 3864 * compute usable version of value in case we skip full 3865 * computation later 3866 */ 3867 dtaps_per_ptap = 0; 3868 tmp_delay = 0; 3869 while (tmp_delay < IO_DELAY_PER_OPA_TAP) { 3870 dtaps_per_ptap++; 3871 tmp_delay += IO_DELAY_PER_DCHAIN_TAP; 3872 } 3873 dtaps_per_ptap--; 3874 3875 concatenated_longidle = concatenated_longidle ^ 10; 3876 /*longidle outer loop */ 3877 concatenated_longidle = concatenated_longidle << 16; 3878 concatenated_longidle = concatenated_longidle ^ 100; 3879 /*longidle sample count */ 3880 concatenated_delays = concatenated_delays ^ 243; 3881 /* trfc, worst case of 933Mhz 4Gb */ 3882 concatenated_delays = concatenated_delays << 8; 3883 concatenated_delays = concatenated_delays ^ 14; 3884 /* trcd, worst case */ 3885 concatenated_delays = concatenated_delays << 8; 3886 concatenated_delays = concatenated_delays ^ 10; 3887 /* vfifo wait */ 3888 concatenated_delays = concatenated_delays << 8; 3889 concatenated_delays = concatenated_delays ^ 4; 3890 /* mux delay */ 3891 3892 concatenated_rw_addr = concatenated_rw_addr ^ RW_MGR_IDLE; 3893 concatenated_rw_addr = concatenated_rw_addr << 8; 3894 concatenated_rw_addr = concatenated_rw_addr ^ RW_MGR_ACTIVATE_1; 3895 concatenated_rw_addr = concatenated_rw_addr << 8; 3896 concatenated_rw_addr = concatenated_rw_addr ^ RW_MGR_SGLE_READ; 3897 concatenated_rw_addr = concatenated_rw_addr << 8; 3898 concatenated_rw_addr = concatenated_rw_addr ^ RW_MGR_PRECHARGE_ALL; 3899 3900 concatenated_refresh = concatenated_refresh ^ RW_MGR_REFRESH_ALL; 3901 concatenated_refresh = concatenated_refresh << 24; 3902 concatenated_refresh = concatenated_refresh ^ 1000; /* trefi */ 3903 3904 /* Initialize the register file with the correct data */ 3905 addr = (u32)&sdr_reg_file->dtaps_per_ptap; 3906 writel(dtaps_per_ptap, addr); 3907 3908 addr = (u32)&sdr_reg_file->trk_sample_count; 3909 writel(trk_sample_count, addr); 3910 3911 addr = (u32)&sdr_reg_file->trk_longidle; 3912 writel(concatenated_longidle, addr); 3913 3914 addr = (u32)&sdr_reg_file->delays; 3915 writel(concatenated_delays, addr); 3916 3917 addr = (u32)&sdr_reg_file->trk_rw_mgr_addr; 3918 writel(concatenated_rw_addr, addr); 3919 3920 addr = (u32)&sdr_reg_file->trk_read_dqs_width; 3921 writel(RW_MGR_MEM_IF_READ_DQS_WIDTH, addr); 3922 3923 addr = (u32)&sdr_reg_file->trk_rfsh; 3924 writel(concatenated_refresh, addr); 3925 } 3926 3927 int sdram_calibration_full(void) 3928 { 3929 struct param_type my_param; 3930 struct gbl_type my_gbl; 3931 uint32_t pass; 3932 uint32_t i; 3933 3934 param = &my_param; 3935 gbl = &my_gbl; 3936 3937 /* Initialize the debug mode flags */ 3938 gbl->phy_debug_mode_flags = 0; 3939 /* Set the calibration enabled by default */ 3940 gbl->phy_debug_mode_flags |= PHY_DEBUG_ENABLE_CAL_RPT; 3941 /* 3942 * Only sweep all groups (regardless of fail state) by default 3943 * Set enabled read test by default. 3944 */ 3945 #if DISABLE_GUARANTEED_READ 3946 gbl->phy_debug_mode_flags |= PHY_DEBUG_DISABLE_GUARANTEED_READ; 3947 #endif 3948 /* Initialize the register file */ 3949 initialize_reg_file(); 3950 3951 /* Initialize any PHY CSR */ 3952 initialize_hps_phy(); 3953 3954 scc_mgr_initialize(); 3955 3956 initialize_tracking(); 3957 3958 /* USER Enable all ranks, groups */ 3959 for (i = 0; i < RW_MGR_MEM_NUMBER_OF_RANKS; i++) 3960 param->skip_ranks[i] = 0; 3961 for (i = 0; i < NUM_SHADOW_REGS; ++i) 3962 param->skip_shadow_regs[i] = 0; 3963 param->skip_groups = 0; 3964 3965 printf("%s: Preparing to start memory calibration\n", __FILE__); 3966 3967 debug("%s:%d\n", __func__, __LINE__); 3968 debug_cond(DLEVEL == 1, 3969 "DDR3 FULL_RATE ranks=%u cs/dimm=%u dq/dqs=%u,%u vg/dqs=%u,%u ", 3970 RW_MGR_MEM_NUMBER_OF_RANKS, RW_MGR_MEM_NUMBER_OF_CS_PER_DIMM, 3971 RW_MGR_MEM_DQ_PER_READ_DQS, RW_MGR_MEM_DQ_PER_WRITE_DQS, 3972 RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS, 3973 RW_MGR_MEM_VIRTUAL_GROUPS_PER_WRITE_DQS); 3974 debug_cond(DLEVEL == 1, 3975 "dqs=%u,%u dq=%u dm=%u ptap_delay=%u dtap_delay=%u ", 3976 RW_MGR_MEM_IF_READ_DQS_WIDTH, RW_MGR_MEM_IF_WRITE_DQS_WIDTH, 3977 RW_MGR_MEM_DATA_WIDTH, RW_MGR_MEM_DATA_MASK_WIDTH, 3978 IO_DELAY_PER_OPA_TAP, IO_DELAY_PER_DCHAIN_TAP); 3979 debug_cond(DLEVEL == 1, "dtap_dqsen_delay=%u, dll=%u", 3980 IO_DELAY_PER_DQS_EN_DCHAIN_TAP, IO_DLL_CHAIN_LENGTH); 3981 debug_cond(DLEVEL == 1, "max values: en_p=%u dqdqs_p=%u en_d=%u dqs_in_d=%u ", 3982 IO_DQS_EN_PHASE_MAX, IO_DQDQS_OUT_PHASE_MAX, 3983 IO_DQS_EN_DELAY_MAX, IO_DQS_IN_DELAY_MAX); 3984 debug_cond(DLEVEL == 1, "io_in_d=%u io_out1_d=%u io_out2_d=%u ", 3985 IO_IO_IN_DELAY_MAX, IO_IO_OUT1_DELAY_MAX, 3986 IO_IO_OUT2_DELAY_MAX); 3987 debug_cond(DLEVEL == 1, "dqs_in_reserve=%u dqs_out_reserve=%u\n", 3988 IO_DQS_IN_RESERVE, IO_DQS_OUT_RESERVE); 3989 3990 hc_initialize_rom_data(); 3991 3992 /* update info for sims */ 3993 reg_file_set_stage(CAL_STAGE_NIL); 3994 reg_file_set_group(0); 3995 3996 /* 3997 * Load global needed for those actions that require 3998 * some dynamic calibration support. 3999 */ 4000 dyn_calib_steps = STATIC_CALIB_STEPS; 4001 /* 4002 * Load global to allow dynamic selection of delay loop settings 4003 * based on calibration mode. 4004 */ 4005 if (!(dyn_calib_steps & CALIB_SKIP_DELAY_LOOPS)) 4006 skip_delay_mask = 0xff; 4007 else 4008 skip_delay_mask = 0x0; 4009 4010 pass = run_mem_calibrate(); 4011 4012 printf("%s: Calibration complete\n", __FILE__); 4013 return pass; 4014 } 4015