1 /* 2 * Copyright Altera Corporation (C) 2012-2015 3 * 4 * SPDX-License-Identifier: BSD-3-Clause 5 */ 6 7 #include <common.h> 8 #include <asm/io.h> 9 #include <asm/arch/sdram.h> 10 #include "sequencer.h" 11 #include "sequencer_auto.h" 12 #include "sequencer_auto_ac_init.h" 13 #include "sequencer_auto_inst_init.h" 14 #include "sequencer_defines.h" 15 16 static void scc_mgr_load_dqs_for_write_group(uint32_t write_group); 17 18 static struct socfpga_sdr_rw_load_manager *sdr_rw_load_mgr_regs = 19 (struct socfpga_sdr_rw_load_manager *)(SDR_PHYGRP_RWMGRGRP_ADDRESS | 0x800); 20 21 static struct socfpga_sdr_rw_load_jump_manager *sdr_rw_load_jump_mgr_regs = 22 (struct socfpga_sdr_rw_load_jump_manager *)(SDR_PHYGRP_RWMGRGRP_ADDRESS | 0xC00); 23 24 static struct socfpga_sdr_reg_file *sdr_reg_file = 25 (struct socfpga_sdr_reg_file *)SDR_PHYGRP_REGFILEGRP_ADDRESS; 26 27 static struct socfpga_sdr_scc_mgr *sdr_scc_mgr = 28 (struct socfpga_sdr_scc_mgr *)(SDR_PHYGRP_SCCGRP_ADDRESS | 0xe00); 29 30 static struct socfpga_phy_mgr_cmd *phy_mgr_cmd = 31 (struct socfpga_phy_mgr_cmd *)(BASE_PHY_MGR); 32 33 static struct socfpga_phy_mgr_cfg *phy_mgr_cfg = 34 (struct socfpga_phy_mgr_cfg *)(BASE_PHY_MGR + 0x4000); 35 36 static struct socfpga_data_mgr *data_mgr = 37 (struct socfpga_data_mgr *)(BASE_DATA_MGR); 38 39 #define DELTA_D 1 40 #define MGR_SELECT_MASK 0xf8000 41 42 /* 43 * In order to reduce ROM size, most of the selectable calibration steps are 44 * decided at compile time based on the user's calibration mode selection, 45 * as captured by the STATIC_CALIB_STEPS selection below. 46 * 47 * However, to support simulation-time selection of fast simulation mode, where 48 * we skip everything except the bare minimum, we need a few of the steps to 49 * be dynamic. In those cases, we either use the DYNAMIC_CALIB_STEPS for the 50 * check, which is based on the rtl-supplied value, or we dynamically compute 51 * the value to use based on the dynamically-chosen calibration mode 52 */ 53 54 #define DLEVEL 0 55 #define STATIC_IN_RTL_SIM 0 56 #define STATIC_SKIP_DELAY_LOOPS 0 57 58 #define STATIC_CALIB_STEPS (STATIC_IN_RTL_SIM | CALIB_SKIP_FULL_TEST | \ 59 STATIC_SKIP_DELAY_LOOPS) 60 61 /* calibration steps requested by the rtl */ 62 uint16_t dyn_calib_steps; 63 64 /* 65 * To make CALIB_SKIP_DELAY_LOOPS a dynamic conditional option 66 * instead of static, we use boolean logic to select between 67 * non-skip and skip values 68 * 69 * The mask is set to include all bits when not-skipping, but is 70 * zero when skipping 71 */ 72 73 uint16_t skip_delay_mask; /* mask off bits when skipping/not-skipping */ 74 75 #define SKIP_DELAY_LOOP_VALUE_OR_ZERO(non_skip_value) \ 76 ((non_skip_value) & skip_delay_mask) 77 78 struct gbl_type *gbl; 79 struct param_type *param; 80 uint32_t curr_shadow_reg; 81 82 static uint32_t rw_mgr_mem_calibrate_write_test(uint32_t rank_bgn, 83 uint32_t write_group, uint32_t use_dm, 84 uint32_t all_correct, uint32_t *bit_chk, uint32_t all_ranks); 85 86 static u32 sdr_get_addr(u32 *base) 87 { 88 u32 addr = (u32)base & MGR_SELECT_MASK; 89 90 switch (addr) { 91 case BASE_PHY_MGR: 92 addr = (((u32)base >> 8) & (1 << 6)) | ((u32)base & 0x3f) | 93 SDR_PHYGRP_PHYMGRGRP_ADDRESS; 94 break; 95 case BASE_RW_MGR: 96 addr = ((u32)base & 0x1fff) | SDR_PHYGRP_RWMGRGRP_ADDRESS; 97 break; 98 case BASE_DATA_MGR: 99 addr = ((u32)base & 0x7ff) | SDR_PHYGRP_DATAMGRGRP_ADDRESS; 100 break; 101 case BASE_SCC_MGR: 102 addr = ((u32)base & 0xfff) | SDR_PHYGRP_SCCGRP_ADDRESS; 103 break; 104 case BASE_REG_FILE: 105 addr = ((u32)base & 0x7ff) | SDR_PHYGRP_REGFILEGRP_ADDRESS; 106 break; 107 case BASE_MMR: 108 addr = ((u32)base & 0xfff) | SDR_CTRLGRP_ADDRESS; 109 break; 110 default: 111 return -1; 112 } 113 114 return addr; 115 } 116 117 static void set_failing_group_stage(uint32_t group, uint32_t stage, 118 uint32_t substage) 119 { 120 /* 121 * Only set the global stage if there was not been any other 122 * failing group 123 */ 124 if (gbl->error_stage == CAL_STAGE_NIL) { 125 gbl->error_substage = substage; 126 gbl->error_stage = stage; 127 gbl->error_group = group; 128 } 129 } 130 131 static void reg_file_set_group(uint32_t set_group) 132 { 133 u32 addr = (u32)&sdr_reg_file->cur_stage; 134 135 /* Read the current group and stage */ 136 uint32_t cur_stage_group = readl(SOCFPGA_SDR_ADDRESS + addr); 137 138 /* Clear the group */ 139 cur_stage_group &= 0x0000FFFF; 140 141 /* Set the group */ 142 cur_stage_group |= (set_group << 16); 143 144 /* Write the data back */ 145 writel(cur_stage_group, SOCFPGA_SDR_ADDRESS + addr); 146 } 147 148 static void reg_file_set_stage(uint32_t set_stage) 149 { 150 u32 addr = (u32)&sdr_reg_file->cur_stage; 151 152 /* Read the current group and stage */ 153 uint32_t cur_stage_group = readl(SOCFPGA_SDR_ADDRESS + addr); 154 155 /* Clear the stage and substage */ 156 cur_stage_group &= 0xFFFF0000; 157 158 /* Set the stage */ 159 cur_stage_group |= (set_stage & 0x000000FF); 160 161 /* Write the data back */ 162 writel(cur_stage_group, SOCFPGA_SDR_ADDRESS + addr); 163 } 164 165 static void reg_file_set_sub_stage(uint32_t set_sub_stage) 166 { 167 u32 addr = (u32)&sdr_reg_file->cur_stage; 168 169 /* Read the current group and stage */ 170 uint32_t cur_stage_group = readl(SOCFPGA_SDR_ADDRESS + addr); 171 172 /* Clear the substage */ 173 cur_stage_group &= 0xFFFF00FF; 174 175 /* Set the sub stage */ 176 cur_stage_group |= ((set_sub_stage << 8) & 0x0000FF00); 177 178 /* Write the data back */ 179 writel(cur_stage_group, SOCFPGA_SDR_ADDRESS + addr); 180 } 181 182 static void initialize(void) 183 { 184 u32 addr = sdr_get_addr(&phy_mgr_cfg->mux_sel); 185 186 debug("%s:%d\n", __func__, __LINE__); 187 /* USER calibration has control over path to memory */ 188 /* 189 * In Hard PHY this is a 2-bit control: 190 * 0: AFI Mux Select 191 * 1: DDIO Mux Select 192 */ 193 writel(0x3, SOCFPGA_SDR_ADDRESS + addr); 194 195 /* USER memory clock is not stable we begin initialization */ 196 addr = sdr_get_addr(&phy_mgr_cfg->reset_mem_stbl); 197 writel(0, SOCFPGA_SDR_ADDRESS + addr); 198 199 /* USER calibration status all set to zero */ 200 addr = sdr_get_addr(&phy_mgr_cfg->cal_status); 201 writel(0, SOCFPGA_SDR_ADDRESS + addr); 202 203 addr = sdr_get_addr(&phy_mgr_cfg->cal_debug_info); 204 writel(0, SOCFPGA_SDR_ADDRESS + addr); 205 206 if ((dyn_calib_steps & CALIB_SKIP_ALL) != CALIB_SKIP_ALL) { 207 param->read_correct_mask_vg = ((uint32_t)1 << 208 (RW_MGR_MEM_DQ_PER_READ_DQS / 209 RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS)) - 1; 210 param->write_correct_mask_vg = ((uint32_t)1 << 211 (RW_MGR_MEM_DQ_PER_READ_DQS / 212 RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS)) - 1; 213 param->read_correct_mask = ((uint32_t)1 << 214 RW_MGR_MEM_DQ_PER_READ_DQS) - 1; 215 param->write_correct_mask = ((uint32_t)1 << 216 RW_MGR_MEM_DQ_PER_WRITE_DQS) - 1; 217 param->dm_correct_mask = ((uint32_t)1 << 218 (RW_MGR_MEM_DATA_WIDTH / RW_MGR_MEM_DATA_MASK_WIDTH)) 219 - 1; 220 } 221 } 222 223 static void set_rank_and_odt_mask(uint32_t rank, uint32_t odt_mode) 224 { 225 uint32_t odt_mask_0 = 0; 226 uint32_t odt_mask_1 = 0; 227 uint32_t cs_and_odt_mask; 228 uint32_t addr; 229 230 if (odt_mode == RW_MGR_ODT_MODE_READ_WRITE) { 231 if (RW_MGR_MEM_NUMBER_OF_RANKS == 1) { 232 /* 233 * 1 Rank 234 * Read: ODT = 0 235 * Write: ODT = 1 236 */ 237 odt_mask_0 = 0x0; 238 odt_mask_1 = 0x1; 239 } else if (RW_MGR_MEM_NUMBER_OF_RANKS == 2) { 240 /* 2 Ranks */ 241 if (RW_MGR_MEM_NUMBER_OF_CS_PER_DIMM == 1) { 242 /* - Dual-Slot , Single-Rank 243 * (1 chip-select per DIMM) 244 * OR 245 * - RDIMM, 4 total CS (2 CS per DIMM) 246 * means 2 DIMM 247 * Since MEM_NUMBER_OF_RANKS is 2 they are 248 * both single rank 249 * with 2 CS each (special for RDIMM) 250 * Read: Turn on ODT on the opposite rank 251 * Write: Turn on ODT on all ranks 252 */ 253 odt_mask_0 = 0x3 & ~(1 << rank); 254 odt_mask_1 = 0x3; 255 } else { 256 /* 257 * USER - Single-Slot , Dual-rank DIMMs 258 * (2 chip-selects per DIMM) 259 * USER Read: Turn on ODT off on all ranks 260 * USER Write: Turn on ODT on active rank 261 */ 262 odt_mask_0 = 0x0; 263 odt_mask_1 = 0x3 & (1 << rank); 264 } 265 } else { 266 /* 4 Ranks 267 * Read: 268 * ----------+-----------------------+ 269 * | | 270 * | ODT | 271 * Read From +-----------------------+ 272 * Rank | 3 | 2 | 1 | 0 | 273 * ----------+-----+-----+-----+-----+ 274 * 0 | 0 | 1 | 0 | 0 | 275 * 1 | 1 | 0 | 0 | 0 | 276 * 2 | 0 | 0 | 0 | 1 | 277 * 3 | 0 | 0 | 1 | 0 | 278 * ----------+-----+-----+-----+-----+ 279 * 280 * Write: 281 * ----------+-----------------------+ 282 * | | 283 * | ODT | 284 * Write To +-----------------------+ 285 * Rank | 3 | 2 | 1 | 0 | 286 * ----------+-----+-----+-----+-----+ 287 * 0 | 0 | 1 | 0 | 1 | 288 * 1 | 1 | 0 | 1 | 0 | 289 * 2 | 0 | 1 | 0 | 1 | 290 * 3 | 1 | 0 | 1 | 0 | 291 * ----------+-----+-----+-----+-----+ 292 */ 293 switch (rank) { 294 case 0: 295 odt_mask_0 = 0x4; 296 odt_mask_1 = 0x5; 297 break; 298 case 1: 299 odt_mask_0 = 0x8; 300 odt_mask_1 = 0xA; 301 break; 302 case 2: 303 odt_mask_0 = 0x1; 304 odt_mask_1 = 0x5; 305 break; 306 case 3: 307 odt_mask_0 = 0x2; 308 odt_mask_1 = 0xA; 309 break; 310 } 311 } 312 } else { 313 odt_mask_0 = 0x0; 314 odt_mask_1 = 0x0; 315 } 316 317 cs_and_odt_mask = 318 (0xFF & ~(1 << rank)) | 319 ((0xFF & odt_mask_0) << 8) | 320 ((0xFF & odt_mask_1) << 16); 321 addr = sdr_get_addr((u32 *)RW_MGR_SET_CS_AND_ODT_MASK); 322 writel(cs_and_odt_mask, SOCFPGA_SDR_ADDRESS + addr); 323 } 324 325 static void scc_mgr_initialize(void) 326 { 327 u32 addr = sdr_get_addr((u32 *)SCC_MGR_HHP_RFILE); 328 329 /* 330 * Clear register file for HPS 331 * 16 (2^4) is the size of the full register file in the scc mgr: 332 * RFILE_DEPTH = log2(MEM_DQ_PER_DQS + 1 + MEM_DM_PER_DQS + 333 * MEM_IF_READ_DQS_WIDTH - 1) + 1; 334 */ 335 uint32_t i; 336 for (i = 0; i < 16; i++) { 337 debug_cond(DLEVEL == 1, "%s:%d: Clearing SCC RFILE index %u\n", 338 __func__, __LINE__, i); 339 writel(0, SOCFPGA_SDR_ADDRESS + addr + (i << 2)); 340 } 341 } 342 343 static void scc_mgr_set_dqs_bus_in_delay(uint32_t read_group, 344 uint32_t delay) 345 { 346 u32 addr = sdr_get_addr((u32 *)SCC_MGR_DQS_IN_DELAY); 347 348 /* Load the setting in the SCC manager */ 349 writel(delay, SOCFPGA_SDR_ADDRESS + addr + (read_group << 2)); 350 } 351 352 static void scc_mgr_set_dqs_io_in_delay(uint32_t write_group, 353 uint32_t delay) 354 { 355 u32 addr = sdr_get_addr((u32 *)SCC_MGR_IO_IN_DELAY); 356 357 writel(delay, SOCFPGA_SDR_ADDRESS + addr + (RW_MGR_MEM_DQ_PER_WRITE_DQS << 2)); 358 } 359 360 static void scc_mgr_set_dqs_en_phase(uint32_t read_group, uint32_t phase) 361 { 362 u32 addr = sdr_get_addr((u32 *)SCC_MGR_DQS_EN_PHASE); 363 364 /* Load the setting in the SCC manager */ 365 writel(phase, SOCFPGA_SDR_ADDRESS + addr + (read_group << 2)); 366 } 367 368 static void scc_mgr_set_dqs_en_phase_all_ranks(uint32_t read_group, 369 uint32_t phase) 370 { 371 uint32_t r; 372 uint32_t update_scan_chains; 373 uint32_t addr; 374 375 for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; 376 r += NUM_RANKS_PER_SHADOW_REG) { 377 /* 378 * USER although the h/w doesn't support different phases per 379 * shadow register, for simplicity our scc manager modeling 380 * keeps different phase settings per shadow reg, and it's 381 * important for us to keep them in sync to match h/w. 382 * for efficiency, the scan chain update should occur only 383 * once to sr0. 384 */ 385 update_scan_chains = (r == 0) ? 1 : 0; 386 387 scc_mgr_set_dqs_en_phase(read_group, phase); 388 389 if (update_scan_chains) { 390 addr = (u32)&sdr_scc_mgr->dqs_ena; 391 writel(read_group, SOCFPGA_SDR_ADDRESS + addr); 392 393 addr = (u32)&sdr_scc_mgr->update; 394 writel(0, SOCFPGA_SDR_ADDRESS + addr); 395 } 396 } 397 } 398 399 static void scc_mgr_set_dqdqs_output_phase(uint32_t write_group, 400 uint32_t phase) 401 { 402 u32 addr = sdr_get_addr((u32 *)SCC_MGR_DQDQS_OUT_PHASE); 403 404 /* Load the setting in the SCC manager */ 405 writel(phase, SOCFPGA_SDR_ADDRESS + addr + (write_group << 2)); 406 } 407 408 static void scc_mgr_set_dqdqs_output_phase_all_ranks(uint32_t write_group, 409 uint32_t phase) 410 { 411 uint32_t r; 412 uint32_t update_scan_chains; 413 uint32_t addr; 414 415 for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; 416 r += NUM_RANKS_PER_SHADOW_REG) { 417 /* 418 * USER although the h/w doesn't support different phases per 419 * shadow register, for simplicity our scc manager modeling 420 * keeps different phase settings per shadow reg, and it's 421 * important for us to keep them in sync to match h/w. 422 * for efficiency, the scan chain update should occur only 423 * once to sr0. 424 */ 425 update_scan_chains = (r == 0) ? 1 : 0; 426 427 scc_mgr_set_dqdqs_output_phase(write_group, phase); 428 429 if (update_scan_chains) { 430 addr = (u32)&sdr_scc_mgr->dqs_ena; 431 writel(write_group, SOCFPGA_SDR_ADDRESS + addr); 432 433 addr = (u32)&sdr_scc_mgr->update; 434 writel(0, SOCFPGA_SDR_ADDRESS + addr); 435 } 436 } 437 } 438 439 static void scc_mgr_set_dqs_en_delay(uint32_t read_group, uint32_t delay) 440 { 441 uint32_t addr = sdr_get_addr((u32 *)SCC_MGR_DQS_EN_DELAY); 442 443 /* Load the setting in the SCC manager */ 444 writel(delay + IO_DQS_EN_DELAY_OFFSET, SOCFPGA_SDR_ADDRESS + addr + 445 (read_group << 2)); 446 } 447 448 static void scc_mgr_set_dqs_en_delay_all_ranks(uint32_t read_group, 449 uint32_t delay) 450 { 451 uint32_t r; 452 uint32_t addr; 453 454 for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; 455 r += NUM_RANKS_PER_SHADOW_REG) { 456 scc_mgr_set_dqs_en_delay(read_group, delay); 457 458 addr = (u32)&sdr_scc_mgr->dqs_ena; 459 writel(read_group, SOCFPGA_SDR_ADDRESS + addr); 460 /* 461 * In shadow register mode, the T11 settings are stored in 462 * registers in the core, which are updated by the DQS_ENA 463 * signals. Not issuing the SCC_MGR_UPD command allows us to 464 * save lots of rank switching overhead, by calling 465 * select_shadow_regs_for_update with update_scan_chains 466 * set to 0. 467 */ 468 addr = (u32)&sdr_scc_mgr->update; 469 writel(0, SOCFPGA_SDR_ADDRESS + addr); 470 } 471 /* 472 * In shadow register mode, the T11 settings are stored in 473 * registers in the core, which are updated by the DQS_ENA 474 * signals. Not issuing the SCC_MGR_UPD command allows us to 475 * save lots of rank switching overhead, by calling 476 * select_shadow_regs_for_update with update_scan_chains 477 * set to 0. 478 */ 479 addr = (u32)&sdr_scc_mgr->update; 480 writel(0, SOCFPGA_SDR_ADDRESS + addr); 481 } 482 483 static void scc_mgr_set_oct_out1_delay(uint32_t write_group, uint32_t delay) 484 { 485 uint32_t read_group; 486 uint32_t addr = sdr_get_addr((u32 *)SCC_MGR_OCT_OUT1_DELAY); 487 488 /* 489 * Load the setting in the SCC manager 490 * Although OCT affects only write data, the OCT delay is controlled 491 * by the DQS logic block which is instantiated once per read group. 492 * For protocols where a write group consists of multiple read groups, 493 * the setting must be set multiple times. 494 */ 495 for (read_group = write_group * RW_MGR_MEM_IF_READ_DQS_WIDTH / 496 RW_MGR_MEM_IF_WRITE_DQS_WIDTH; 497 read_group < (write_group + 1) * RW_MGR_MEM_IF_READ_DQS_WIDTH / 498 RW_MGR_MEM_IF_WRITE_DQS_WIDTH; ++read_group) 499 writel(delay, SOCFPGA_SDR_ADDRESS + addr + (read_group << 2)); 500 } 501 502 static void scc_mgr_set_dq_out1_delay(uint32_t write_group, 503 uint32_t dq_in_group, uint32_t delay) 504 { 505 uint32_t addr = sdr_get_addr((u32 *)SCC_MGR_IO_OUT1_DELAY); 506 507 /* Load the setting in the SCC manager */ 508 writel(delay, SOCFPGA_SDR_ADDRESS + addr + (dq_in_group << 2)); 509 } 510 511 static void scc_mgr_set_dq_in_delay(uint32_t write_group, 512 uint32_t dq_in_group, uint32_t delay) 513 { 514 uint32_t addr = sdr_get_addr((u32 *)SCC_MGR_IO_IN_DELAY); 515 516 /* Load the setting in the SCC manager */ 517 writel(delay, SOCFPGA_SDR_ADDRESS + addr + (dq_in_group << 2)); 518 } 519 520 static void scc_mgr_set_hhp_extras(void) 521 { 522 /* 523 * Load the fixed setting in the SCC manager 524 * bits: 0:0 = 1'b1 - dqs bypass 525 * bits: 1:1 = 1'b1 - dq bypass 526 * bits: 4:2 = 3'b001 - rfifo_mode 527 * bits: 6:5 = 2'b01 - rfifo clock_select 528 * bits: 7:7 = 1'b0 - separate gating from ungating setting 529 * bits: 8:8 = 1'b0 - separate OE from Output delay setting 530 */ 531 uint32_t value = (0<<8) | (0<<7) | (1<<5) | (1<<2) | (1<<1) | (1<<0); 532 uint32_t addr = sdr_get_addr((u32 *)SCC_MGR_HHP_GLOBALS); 533 534 writel(value, SOCFPGA_SDR_ADDRESS + addr + SCC_MGR_HHP_EXTRAS_OFFSET); 535 } 536 537 static void scc_mgr_set_dqs_out1_delay(uint32_t write_group, 538 uint32_t delay) 539 { 540 uint32_t addr = sdr_get_addr((u32 *)SCC_MGR_IO_OUT1_DELAY); 541 542 /* Load the setting in the SCC manager */ 543 writel(delay, SOCFPGA_SDR_ADDRESS + addr + (RW_MGR_MEM_DQ_PER_WRITE_DQS << 2)); 544 } 545 546 static void scc_mgr_set_dm_out1_delay(uint32_t write_group, 547 uint32_t dm, uint32_t delay) 548 { 549 uint32_t addr = sdr_get_addr((u32 *)SCC_MGR_IO_OUT1_DELAY); 550 551 /* Load the setting in the SCC manager */ 552 writel(delay, SOCFPGA_SDR_ADDRESS + addr + 553 ((RW_MGR_MEM_DQ_PER_WRITE_DQS + 1 + dm) << 2)); 554 } 555 556 /* 557 * USER Zero all DQS config 558 * TODO: maybe rename to scc_mgr_zero_dqs_config (or something) 559 */ 560 static void scc_mgr_zero_all(void) 561 { 562 uint32_t i, r; 563 uint32_t addr; 564 565 /* 566 * USER Zero all DQS config settings, across all groups and all 567 * shadow registers 568 */ 569 for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; r += 570 NUM_RANKS_PER_SHADOW_REG) { 571 for (i = 0; i < RW_MGR_MEM_IF_READ_DQS_WIDTH; i++) { 572 /* 573 * The phases actually don't exist on a per-rank basis, 574 * but there's no harm updating them several times, so 575 * let's keep the code simple. 576 */ 577 scc_mgr_set_dqs_bus_in_delay(i, IO_DQS_IN_RESERVE); 578 scc_mgr_set_dqs_en_phase(i, 0); 579 scc_mgr_set_dqs_en_delay(i, 0); 580 } 581 582 for (i = 0; i < RW_MGR_MEM_IF_WRITE_DQS_WIDTH; i++) { 583 scc_mgr_set_dqdqs_output_phase(i, 0); 584 /* av/cv don't have out2 */ 585 scc_mgr_set_oct_out1_delay(i, IO_DQS_OUT_RESERVE); 586 } 587 } 588 589 /* multicast to all DQS group enables */ 590 addr = (u32)&sdr_scc_mgr->dqs_ena; 591 writel(0xff, SOCFPGA_SDR_ADDRESS + addr); 592 593 addr = (u32)&sdr_scc_mgr->update; 594 writel(0, SOCFPGA_SDR_ADDRESS + addr); 595 } 596 597 static void scc_set_bypass_mode(uint32_t write_group, uint32_t mode) 598 { 599 uint32_t addr; 600 /* mode = 0 : Do NOT bypass - Half Rate Mode */ 601 /* mode = 1 : Bypass - Full Rate Mode */ 602 603 /* only need to set once for all groups, pins, dq, dqs, dm */ 604 if (write_group == 0) { 605 debug_cond(DLEVEL == 1, "%s:%d Setting HHP Extras\n", __func__, 606 __LINE__); 607 scc_mgr_set_hhp_extras(); 608 debug_cond(DLEVEL == 1, "%s:%d Done Setting HHP Extras\n", 609 __func__, __LINE__); 610 } 611 /* multicast to all DQ enables */ 612 addr = (u32)&sdr_scc_mgr->dq_ena; 613 writel(0xff, SOCFPGA_SDR_ADDRESS + addr); 614 615 addr = (u32)&sdr_scc_mgr->dm_ena; 616 writel(0xff, SOCFPGA_SDR_ADDRESS + addr); 617 618 /* update current DQS IO enable */ 619 addr = (u32)&sdr_scc_mgr->dqs_io_ena; 620 writel(0, SOCFPGA_SDR_ADDRESS + addr); 621 622 /* update the DQS logic */ 623 addr = (u32)&sdr_scc_mgr->dqs_ena; 624 writel(write_group, SOCFPGA_SDR_ADDRESS + addr); 625 626 /* hit update */ 627 addr = (u32)&sdr_scc_mgr->update; 628 writel(0, SOCFPGA_SDR_ADDRESS + addr); 629 } 630 631 static void scc_mgr_zero_group(uint32_t write_group, uint32_t test_begin, 632 int32_t out_only) 633 { 634 uint32_t i, r; 635 uint32_t addr; 636 637 for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; r += 638 NUM_RANKS_PER_SHADOW_REG) { 639 /* Zero all DQ config settings */ 640 for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) { 641 scc_mgr_set_dq_out1_delay(write_group, i, 0); 642 if (!out_only) 643 scc_mgr_set_dq_in_delay(write_group, i, 0); 644 } 645 646 /* multicast to all DQ enables */ 647 addr = (u32)&sdr_scc_mgr->dq_ena; 648 writel(0xff, SOCFPGA_SDR_ADDRESS + addr); 649 650 /* Zero all DM config settings */ 651 for (i = 0; i < RW_MGR_NUM_DM_PER_WRITE_GROUP; i++) { 652 scc_mgr_set_dm_out1_delay(write_group, i, 0); 653 } 654 655 /* multicast to all DM enables */ 656 addr = (u32)&sdr_scc_mgr->dm_ena; 657 writel(0xff, SOCFPGA_SDR_ADDRESS + addr); 658 659 /* zero all DQS io settings */ 660 if (!out_only) 661 scc_mgr_set_dqs_io_in_delay(write_group, 0); 662 /* av/cv don't have out2 */ 663 scc_mgr_set_dqs_out1_delay(write_group, IO_DQS_OUT_RESERVE); 664 scc_mgr_set_oct_out1_delay(write_group, IO_DQS_OUT_RESERVE); 665 scc_mgr_load_dqs_for_write_group(write_group); 666 667 /* multicast to all DQS IO enables (only 1) */ 668 addr = (u32)&sdr_scc_mgr->dqs_io_ena; 669 writel(0, SOCFPGA_SDR_ADDRESS + addr); 670 671 /* hit update to zero everything */ 672 addr = (u32)&sdr_scc_mgr->update; 673 writel(0, SOCFPGA_SDR_ADDRESS + addr); 674 } 675 } 676 677 /* load up dqs config settings */ 678 static void scc_mgr_load_dqs(uint32_t dqs) 679 { 680 uint32_t addr = (u32)&sdr_scc_mgr->dqs_ena; 681 682 writel(dqs, SOCFPGA_SDR_ADDRESS + addr); 683 } 684 685 static void scc_mgr_load_dqs_for_write_group(uint32_t write_group) 686 { 687 uint32_t read_group; 688 uint32_t addr = (u32)&sdr_scc_mgr->dqs_ena; 689 /* 690 * Although OCT affects only write data, the OCT delay is controlled 691 * by the DQS logic block which is instantiated once per read group. 692 * For protocols where a write group consists of multiple read groups, 693 * the setting must be scanned multiple times. 694 */ 695 for (read_group = write_group * RW_MGR_MEM_IF_READ_DQS_WIDTH / 696 RW_MGR_MEM_IF_WRITE_DQS_WIDTH; 697 read_group < (write_group + 1) * RW_MGR_MEM_IF_READ_DQS_WIDTH / 698 RW_MGR_MEM_IF_WRITE_DQS_WIDTH; ++read_group) 699 writel(read_group, SOCFPGA_SDR_ADDRESS + addr); 700 } 701 702 /* load up dqs io config settings */ 703 static void scc_mgr_load_dqs_io(void) 704 { 705 uint32_t addr = (u32)&sdr_scc_mgr->dqs_io_ena; 706 707 writel(0, SOCFPGA_SDR_ADDRESS + addr); 708 } 709 710 /* load up dq config settings */ 711 static void scc_mgr_load_dq(uint32_t dq_in_group) 712 { 713 uint32_t addr = (u32)&sdr_scc_mgr->dq_ena; 714 715 writel(dq_in_group, SOCFPGA_SDR_ADDRESS + addr); 716 } 717 718 /* load up dm config settings */ 719 static void scc_mgr_load_dm(uint32_t dm) 720 { 721 uint32_t addr = (u32)&sdr_scc_mgr->dm_ena; 722 723 writel(dm, SOCFPGA_SDR_ADDRESS + addr); 724 } 725 726 /* 727 * apply and load a particular input delay for the DQ pins in a group 728 * group_bgn is the index of the first dq pin (in the write group) 729 */ 730 static void scc_mgr_apply_group_dq_in_delay(uint32_t write_group, 731 uint32_t group_bgn, uint32_t delay) 732 { 733 uint32_t i, p; 734 735 for (i = 0, p = group_bgn; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++, p++) { 736 scc_mgr_set_dq_in_delay(write_group, p, delay); 737 scc_mgr_load_dq(p); 738 } 739 } 740 741 /* apply and load a particular output delay for the DQ pins in a group */ 742 static void scc_mgr_apply_group_dq_out1_delay(uint32_t write_group, 743 uint32_t group_bgn, 744 uint32_t delay1) 745 { 746 uint32_t i, p; 747 748 for (i = 0, p = group_bgn; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++, p++) { 749 scc_mgr_set_dq_out1_delay(write_group, i, delay1); 750 scc_mgr_load_dq(i); 751 } 752 } 753 754 /* apply and load a particular output delay for the DM pins in a group */ 755 static void scc_mgr_apply_group_dm_out1_delay(uint32_t write_group, 756 uint32_t delay1) 757 { 758 uint32_t i; 759 760 for (i = 0; i < RW_MGR_NUM_DM_PER_WRITE_GROUP; i++) { 761 scc_mgr_set_dm_out1_delay(write_group, i, delay1); 762 scc_mgr_load_dm(i); 763 } 764 } 765 766 767 /* apply and load delay on both DQS and OCT out1 */ 768 static void scc_mgr_apply_group_dqs_io_and_oct_out1(uint32_t write_group, 769 uint32_t delay) 770 { 771 scc_mgr_set_dqs_out1_delay(write_group, delay); 772 scc_mgr_load_dqs_io(); 773 774 scc_mgr_set_oct_out1_delay(write_group, delay); 775 scc_mgr_load_dqs_for_write_group(write_group); 776 } 777 778 /* apply a delay to the entire output side: DQ, DM, DQS, OCT */ 779 static void scc_mgr_apply_group_all_out_delay_add(uint32_t write_group, 780 uint32_t group_bgn, 781 uint32_t delay) 782 { 783 uint32_t i, p, new_delay; 784 785 /* dq shift */ 786 for (i = 0, p = group_bgn; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++, p++) { 787 new_delay = READ_SCC_DQ_OUT2_DELAY; 788 new_delay += delay; 789 790 if (new_delay > IO_IO_OUT2_DELAY_MAX) { 791 debug_cond(DLEVEL == 1, "%s:%d (%u, %u, %u) DQ[%u,%u]:\ 792 %u > %lu => %lu", __func__, __LINE__, 793 write_group, group_bgn, delay, i, p, new_delay, 794 (long unsigned int)IO_IO_OUT2_DELAY_MAX, 795 (long unsigned int)IO_IO_OUT2_DELAY_MAX); 796 new_delay = IO_IO_OUT2_DELAY_MAX; 797 } 798 799 scc_mgr_load_dq(i); 800 } 801 802 /* dm shift */ 803 for (i = 0; i < RW_MGR_NUM_DM_PER_WRITE_GROUP; i++) { 804 new_delay = READ_SCC_DM_IO_OUT2_DELAY; 805 new_delay += delay; 806 807 if (new_delay > IO_IO_OUT2_DELAY_MAX) { 808 debug_cond(DLEVEL == 1, "%s:%d (%u, %u, %u) DM[%u]:\ 809 %u > %lu => %lu\n", __func__, __LINE__, 810 write_group, group_bgn, delay, i, new_delay, 811 (long unsigned int)IO_IO_OUT2_DELAY_MAX, 812 (long unsigned int)IO_IO_OUT2_DELAY_MAX); 813 new_delay = IO_IO_OUT2_DELAY_MAX; 814 } 815 816 scc_mgr_load_dm(i); 817 } 818 819 /* dqs shift */ 820 new_delay = READ_SCC_DQS_IO_OUT2_DELAY; 821 new_delay += delay; 822 823 if (new_delay > IO_IO_OUT2_DELAY_MAX) { 824 debug_cond(DLEVEL == 1, "%s:%d (%u, %u, %u) DQS: %u > %d => %d;" 825 " adding %u to OUT1\n", __func__, __LINE__, 826 write_group, group_bgn, delay, new_delay, 827 IO_IO_OUT2_DELAY_MAX, IO_IO_OUT2_DELAY_MAX, 828 new_delay - IO_IO_OUT2_DELAY_MAX); 829 scc_mgr_set_dqs_out1_delay(write_group, new_delay - 830 IO_IO_OUT2_DELAY_MAX); 831 new_delay = IO_IO_OUT2_DELAY_MAX; 832 } 833 834 scc_mgr_load_dqs_io(); 835 836 /* oct shift */ 837 new_delay = READ_SCC_OCT_OUT2_DELAY; 838 new_delay += delay; 839 840 if (new_delay > IO_IO_OUT2_DELAY_MAX) { 841 debug_cond(DLEVEL == 1, "%s:%d (%u, %u, %u) DQS: %u > %d => %d;" 842 " adding %u to OUT1\n", __func__, __LINE__, 843 write_group, group_bgn, delay, new_delay, 844 IO_IO_OUT2_DELAY_MAX, IO_IO_OUT2_DELAY_MAX, 845 new_delay - IO_IO_OUT2_DELAY_MAX); 846 scc_mgr_set_oct_out1_delay(write_group, new_delay - 847 IO_IO_OUT2_DELAY_MAX); 848 new_delay = IO_IO_OUT2_DELAY_MAX; 849 } 850 851 scc_mgr_load_dqs_for_write_group(write_group); 852 } 853 854 /* 855 * USER apply a delay to the entire output side (DQ, DM, DQS, OCT) 856 * and to all ranks 857 */ 858 static void scc_mgr_apply_group_all_out_delay_add_all_ranks( 859 uint32_t write_group, uint32_t group_bgn, uint32_t delay) 860 { 861 uint32_t r; 862 uint32_t addr = (u32)&sdr_scc_mgr->update; 863 864 for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; 865 r += NUM_RANKS_PER_SHADOW_REG) { 866 scc_mgr_apply_group_all_out_delay_add(write_group, 867 group_bgn, delay); 868 writel(0, SOCFPGA_SDR_ADDRESS + addr); 869 } 870 } 871 872 /* optimization used to recover some slots in ddr3 inst_rom */ 873 /* could be applied to other protocols if we wanted to */ 874 static void set_jump_as_return(void) 875 { 876 uint32_t addr = (u32)&sdr_rw_load_mgr_regs->load_cntr0; 877 878 /* 879 * to save space, we replace return with jump to special shared 880 * RETURN instruction so we set the counter to large value so that 881 * we always jump 882 */ 883 writel(0xff, SOCFPGA_SDR_ADDRESS + addr); 884 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add0; 885 writel(RW_MGR_RETURN, SOCFPGA_SDR_ADDRESS + addr); 886 } 887 888 /* 889 * should always use constants as argument to ensure all computations are 890 * performed at compile time 891 */ 892 static void delay_for_n_mem_clocks(const uint32_t clocks) 893 { 894 uint32_t afi_clocks; 895 uint8_t inner = 0; 896 uint8_t outer = 0; 897 uint16_t c_loop = 0; 898 uint32_t addr; 899 900 debug("%s:%d: clocks=%u ... start\n", __func__, __LINE__, clocks); 901 902 903 afi_clocks = (clocks + AFI_RATE_RATIO-1) / AFI_RATE_RATIO; 904 /* scale (rounding up) to get afi clocks */ 905 906 /* 907 * Note, we don't bother accounting for being off a little bit 908 * because of a few extra instructions in outer loops 909 * Note, the loops have a test at the end, and do the test before 910 * the decrement, and so always perform the loop 911 * 1 time more than the counter value 912 */ 913 if (afi_clocks == 0) { 914 ; 915 } else if (afi_clocks <= 0x100) { 916 inner = afi_clocks-1; 917 outer = 0; 918 c_loop = 0; 919 } else if (afi_clocks <= 0x10000) { 920 inner = 0xff; 921 outer = (afi_clocks-1) >> 8; 922 c_loop = 0; 923 } else { 924 inner = 0xff; 925 outer = 0xff; 926 c_loop = (afi_clocks-1) >> 16; 927 } 928 929 /* 930 * rom instructions are structured as follows: 931 * 932 * IDLE_LOOP2: jnz cntr0, TARGET_A 933 * IDLE_LOOP1: jnz cntr1, TARGET_B 934 * return 935 * 936 * so, when doing nested loops, TARGET_A is set to IDLE_LOOP2, and 937 * TARGET_B is set to IDLE_LOOP2 as well 938 * 939 * if we have no outer loop, though, then we can use IDLE_LOOP1 only, 940 * and set TARGET_B to IDLE_LOOP1 and we skip IDLE_LOOP2 entirely 941 * 942 * a little confusing, but it helps save precious space in the inst_rom 943 * and sequencer rom and keeps the delays more accurate and reduces 944 * overhead 945 */ 946 if (afi_clocks <= 0x100) { 947 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr1; 948 writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(inner), SOCFPGA_SDR_ADDRESS + addr); 949 950 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add1; 951 writel(RW_MGR_IDLE_LOOP1, SOCFPGA_SDR_ADDRESS + addr); 952 953 addr = sdr_get_addr((u32 *)RW_MGR_RUN_SINGLE_GROUP); 954 writel(RW_MGR_IDLE_LOOP1, SOCFPGA_SDR_ADDRESS + addr); 955 } else { 956 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr0; 957 writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(inner), SOCFPGA_SDR_ADDRESS + addr); 958 959 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr1; 960 writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(outer), SOCFPGA_SDR_ADDRESS + addr); 961 962 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add0; 963 writel(RW_MGR_IDLE_LOOP2, SOCFPGA_SDR_ADDRESS + addr); 964 965 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add1; 966 writel(RW_MGR_IDLE_LOOP2, SOCFPGA_SDR_ADDRESS + addr); 967 968 /* hack to get around compiler not being smart enough */ 969 if (afi_clocks <= 0x10000) { 970 /* only need to run once */ 971 addr = sdr_get_addr((u32 *)RW_MGR_RUN_SINGLE_GROUP); 972 writel(RW_MGR_IDLE_LOOP2, SOCFPGA_SDR_ADDRESS + addr); 973 } else { 974 do { 975 addr = sdr_get_addr((u32 *)RW_MGR_RUN_SINGLE_GROUP); 976 writel(RW_MGR_IDLE_LOOP2, SOCFPGA_SDR_ADDRESS + addr); 977 } while (c_loop-- != 0); 978 } 979 } 980 debug("%s:%d clocks=%u ... end\n", __func__, __LINE__, clocks); 981 } 982 983 static void rw_mgr_mem_initialize(void) 984 { 985 uint32_t r; 986 uint32_t addr; 987 988 debug("%s:%d\n", __func__, __LINE__); 989 990 /* The reset / cke part of initialization is broadcasted to all ranks */ 991 addr = sdr_get_addr((u32 *)RW_MGR_SET_CS_AND_ODT_MASK); 992 writel(RW_MGR_RANK_ALL, SOCFPGA_SDR_ADDRESS + addr); 993 994 /* 995 * Here's how you load register for a loop 996 * Counters are located @ 0x800 997 * Jump address are located @ 0xC00 998 * For both, registers 0 to 3 are selected using bits 3 and 2, like 999 * in 0x800, 0x804, 0x808, 0x80C and 0xC00, 0xC04, 0xC08, 0xC0C 1000 * I know this ain't pretty, but Avalon bus throws away the 2 least 1001 * significant bits 1002 */ 1003 1004 /* start with memory RESET activated */ 1005 1006 /* tINIT = 200us */ 1007 1008 /* 1009 * 200us @ 266MHz (3.75 ns) ~ 54000 clock cycles 1010 * If a and b are the number of iteration in 2 nested loops 1011 * it takes the following number of cycles to complete the operation: 1012 * number_of_cycles = ((2 + n) * a + 2) * b 1013 * where n is the number of instruction in the inner loop 1014 * One possible solution is n = 0 , a = 256 , b = 106 => a = FF, 1015 * b = 6A 1016 */ 1017 1018 /* Load counters */ 1019 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr0; 1020 writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(SEQ_TINIT_CNTR0_VAL), 1021 SOCFPGA_SDR_ADDRESS + addr); 1022 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr1; 1023 writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(SEQ_TINIT_CNTR1_VAL), 1024 SOCFPGA_SDR_ADDRESS + addr); 1025 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr2; 1026 writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(SEQ_TINIT_CNTR2_VAL), 1027 SOCFPGA_SDR_ADDRESS + addr); 1028 1029 /* Load jump address */ 1030 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add0; 1031 writel(RW_MGR_INIT_RESET_0_CKE_0, SOCFPGA_SDR_ADDRESS + addr); 1032 1033 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add1; 1034 writel(RW_MGR_INIT_RESET_0_CKE_0, SOCFPGA_SDR_ADDRESS + addr); 1035 1036 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add2; 1037 writel(RW_MGR_INIT_RESET_0_CKE_0, SOCFPGA_SDR_ADDRESS + addr); 1038 1039 /* Execute count instruction */ 1040 addr = sdr_get_addr((u32 *)RW_MGR_RUN_SINGLE_GROUP); 1041 writel(RW_MGR_INIT_RESET_0_CKE_0, SOCFPGA_SDR_ADDRESS + addr); 1042 1043 /* indicate that memory is stable */ 1044 addr = sdr_get_addr(&phy_mgr_cfg->reset_mem_stbl); 1045 writel(1, SOCFPGA_SDR_ADDRESS + addr); 1046 1047 /* 1048 * transition the RESET to high 1049 * Wait for 500us 1050 */ 1051 1052 /* 1053 * 500us @ 266MHz (3.75 ns) ~ 134000 clock cycles 1054 * If a and b are the number of iteration in 2 nested loops 1055 * it takes the following number of cycles to complete the operation 1056 * number_of_cycles = ((2 + n) * a + 2) * b 1057 * where n is the number of instruction in the inner loop 1058 * One possible solution is n = 2 , a = 131 , b = 256 => a = 83, 1059 * b = FF 1060 */ 1061 1062 /* Load counters */ 1063 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr0; 1064 writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(SEQ_TRESET_CNTR0_VAL), 1065 SOCFPGA_SDR_ADDRESS + addr); 1066 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr1; 1067 writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(SEQ_TRESET_CNTR1_VAL), 1068 SOCFPGA_SDR_ADDRESS + addr); 1069 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr2; 1070 writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(SEQ_TRESET_CNTR2_VAL), 1071 SOCFPGA_SDR_ADDRESS + addr); 1072 1073 /* Load jump address */ 1074 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add0; 1075 writel(RW_MGR_INIT_RESET_1_CKE_0, SOCFPGA_SDR_ADDRESS + addr); 1076 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add1; 1077 writel(RW_MGR_INIT_RESET_1_CKE_0, SOCFPGA_SDR_ADDRESS + addr); 1078 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add2; 1079 writel(RW_MGR_INIT_RESET_1_CKE_0, SOCFPGA_SDR_ADDRESS + addr); 1080 1081 addr = sdr_get_addr((u32 *)RW_MGR_RUN_SINGLE_GROUP); 1082 writel(RW_MGR_INIT_RESET_1_CKE_0, SOCFPGA_SDR_ADDRESS + addr); 1083 1084 /* bring up clock enable */ 1085 1086 /* tXRP < 250 ck cycles */ 1087 delay_for_n_mem_clocks(250); 1088 1089 for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; r++) { 1090 if (param->skip_ranks[r]) { 1091 /* request to skip the rank */ 1092 continue; 1093 } 1094 1095 /* set rank */ 1096 set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_OFF); 1097 1098 /* 1099 * USER Use Mirror-ed commands for odd ranks if address 1100 * mirrorring is on 1101 */ 1102 if ((RW_MGR_MEM_ADDRESS_MIRRORING >> r) & 0x1) { 1103 set_jump_as_return(); 1104 addr = sdr_get_addr((u32 *)RW_MGR_RUN_SINGLE_GROUP); 1105 writel(RW_MGR_MRS2_MIRR, SOCFPGA_SDR_ADDRESS + addr); 1106 delay_for_n_mem_clocks(4); 1107 set_jump_as_return(); 1108 writel(RW_MGR_MRS3_MIRR, SOCFPGA_SDR_ADDRESS + addr); 1109 delay_for_n_mem_clocks(4); 1110 set_jump_as_return(); 1111 writel(RW_MGR_MRS1_MIRR, SOCFPGA_SDR_ADDRESS + addr); 1112 delay_for_n_mem_clocks(4); 1113 set_jump_as_return(); 1114 writel(RW_MGR_MRS0_DLL_RESET_MIRR, SOCFPGA_SDR_ADDRESS + addr); 1115 } else { 1116 set_jump_as_return(); 1117 addr = sdr_get_addr((u32 *)RW_MGR_RUN_SINGLE_GROUP); 1118 writel(RW_MGR_MRS2, SOCFPGA_SDR_ADDRESS + addr); 1119 delay_for_n_mem_clocks(4); 1120 set_jump_as_return(); 1121 writel(RW_MGR_MRS3, SOCFPGA_SDR_ADDRESS + addr); 1122 delay_for_n_mem_clocks(4); 1123 set_jump_as_return(); 1124 writel(RW_MGR_MRS1, SOCFPGA_SDR_ADDRESS + addr); 1125 set_jump_as_return(); 1126 writel(RW_MGR_MRS0_DLL_RESET, SOCFPGA_SDR_ADDRESS + addr); 1127 } 1128 set_jump_as_return(); 1129 addr = sdr_get_addr((u32 *)RW_MGR_RUN_SINGLE_GROUP); 1130 writel(RW_MGR_ZQCL, SOCFPGA_SDR_ADDRESS + addr); 1131 1132 /* tZQinit = tDLLK = 512 ck cycles */ 1133 delay_for_n_mem_clocks(512); 1134 } 1135 } 1136 1137 /* 1138 * At the end of calibration we have to program the user settings in, and 1139 * USER hand off the memory to the user. 1140 */ 1141 static void rw_mgr_mem_handoff(void) 1142 { 1143 uint32_t r; 1144 uint32_t addr; 1145 1146 debug("%s:%d\n", __func__, __LINE__); 1147 for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; r++) { 1148 if (param->skip_ranks[r]) 1149 /* request to skip the rank */ 1150 continue; 1151 /* set rank */ 1152 set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_OFF); 1153 1154 /* precharge all banks ... */ 1155 addr = sdr_get_addr((u32 *)RW_MGR_RUN_SINGLE_GROUP); 1156 writel(RW_MGR_PRECHARGE_ALL, SOCFPGA_SDR_ADDRESS + addr); 1157 1158 /* load up MR settings specified by user */ 1159 1160 /* 1161 * Use Mirror-ed commands for odd ranks if address 1162 * mirrorring is on 1163 */ 1164 addr = sdr_get_addr((u32 *)RW_MGR_RUN_SINGLE_GROUP); 1165 if ((RW_MGR_MEM_ADDRESS_MIRRORING >> r) & 0x1) { 1166 set_jump_as_return(); 1167 writel(RW_MGR_MRS2_MIRR, SOCFPGA_SDR_ADDRESS + addr); 1168 delay_for_n_mem_clocks(4); 1169 set_jump_as_return(); 1170 writel(RW_MGR_MRS3_MIRR, SOCFPGA_SDR_ADDRESS + addr); 1171 delay_for_n_mem_clocks(4); 1172 set_jump_as_return(); 1173 writel(RW_MGR_MRS1_MIRR, SOCFPGA_SDR_ADDRESS + addr); 1174 delay_for_n_mem_clocks(4); 1175 set_jump_as_return(); 1176 writel(RW_MGR_MRS0_USER_MIRR, SOCFPGA_SDR_ADDRESS + addr); 1177 } else { 1178 set_jump_as_return(); 1179 writel(RW_MGR_MRS2, SOCFPGA_SDR_ADDRESS + addr); 1180 delay_for_n_mem_clocks(4); 1181 set_jump_as_return(); 1182 writel(RW_MGR_MRS3, SOCFPGA_SDR_ADDRESS + addr); 1183 delay_for_n_mem_clocks(4); 1184 set_jump_as_return(); 1185 writel(RW_MGR_MRS1, SOCFPGA_SDR_ADDRESS + addr); 1186 delay_for_n_mem_clocks(4); 1187 set_jump_as_return(); 1188 writel(RW_MGR_MRS0_USER, SOCFPGA_SDR_ADDRESS + addr); 1189 } 1190 /* 1191 * USER need to wait tMOD (12CK or 15ns) time before issuing 1192 * other commands, but we will have plenty of NIOS cycles before 1193 * actual handoff so its okay. 1194 */ 1195 } 1196 } 1197 1198 /* 1199 * performs a guaranteed read on the patterns we are going to use during a 1200 * read test to ensure memory works 1201 */ 1202 static uint32_t rw_mgr_mem_calibrate_read_test_patterns(uint32_t rank_bgn, 1203 uint32_t group, uint32_t num_tries, uint32_t *bit_chk, 1204 uint32_t all_ranks) 1205 { 1206 uint32_t r, vg; 1207 uint32_t correct_mask_vg; 1208 uint32_t tmp_bit_chk; 1209 uint32_t rank_end = all_ranks ? RW_MGR_MEM_NUMBER_OF_RANKS : 1210 (rank_bgn + NUM_RANKS_PER_SHADOW_REG); 1211 uint32_t addr; 1212 uint32_t base_rw_mgr; 1213 1214 *bit_chk = param->read_correct_mask; 1215 correct_mask_vg = param->read_correct_mask_vg; 1216 1217 for (r = rank_bgn; r < rank_end; r++) { 1218 if (param->skip_ranks[r]) 1219 /* request to skip the rank */ 1220 continue; 1221 1222 /* set rank */ 1223 set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_READ_WRITE); 1224 1225 /* Load up a constant bursts of read commands */ 1226 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr0; 1227 writel(0x20, SOCFPGA_SDR_ADDRESS + addr); 1228 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add0; 1229 writel(RW_MGR_GUARANTEED_READ, SOCFPGA_SDR_ADDRESS + addr); 1230 1231 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr1; 1232 writel(0x20, SOCFPGA_SDR_ADDRESS + addr); 1233 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add1; 1234 writel(RW_MGR_GUARANTEED_READ_CONT, SOCFPGA_SDR_ADDRESS + addr); 1235 1236 tmp_bit_chk = 0; 1237 for (vg = RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS-1; ; vg--) { 1238 /* reset the fifos to get pointers to known state */ 1239 1240 addr = sdr_get_addr(&phy_mgr_cmd->fifo_reset); 1241 writel(0, SOCFPGA_SDR_ADDRESS + addr); 1242 addr = sdr_get_addr((u32 *)RW_MGR_RESET_READ_DATAPATH); 1243 writel(0, SOCFPGA_SDR_ADDRESS + addr); 1244 1245 tmp_bit_chk = tmp_bit_chk << (RW_MGR_MEM_DQ_PER_READ_DQS 1246 / RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS); 1247 1248 addr = sdr_get_addr((u32 *)RW_MGR_RUN_SINGLE_GROUP); 1249 writel(RW_MGR_GUARANTEED_READ, SOCFPGA_SDR_ADDRESS + addr + 1250 ((group * RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS + 1251 vg) << 2)); 1252 1253 addr = SDR_PHYGRP_RWMGRGRP_ADDRESS; 1254 base_rw_mgr = readl(SOCFPGA_SDR_ADDRESS + addr); 1255 tmp_bit_chk = tmp_bit_chk | (correct_mask_vg & (~base_rw_mgr)); 1256 1257 if (vg == 0) 1258 break; 1259 } 1260 *bit_chk &= tmp_bit_chk; 1261 } 1262 1263 addr = sdr_get_addr((u32 *)RW_MGR_RUN_SINGLE_GROUP); 1264 writel(RW_MGR_CLEAR_DQS_ENABLE, SOCFPGA_SDR_ADDRESS + addr + (group << 2)); 1265 1266 set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF); 1267 debug_cond(DLEVEL == 1, "%s:%d test_load_patterns(%u,ALL) => (%u == %u) =>\ 1268 %lu\n", __func__, __LINE__, group, *bit_chk, param->read_correct_mask, 1269 (long unsigned int)(*bit_chk == param->read_correct_mask)); 1270 return *bit_chk == param->read_correct_mask; 1271 } 1272 1273 static uint32_t rw_mgr_mem_calibrate_read_test_patterns_all_ranks 1274 (uint32_t group, uint32_t num_tries, uint32_t *bit_chk) 1275 { 1276 return rw_mgr_mem_calibrate_read_test_patterns(0, group, 1277 num_tries, bit_chk, 1); 1278 } 1279 1280 /* load up the patterns we are going to use during a read test */ 1281 static void rw_mgr_mem_calibrate_read_load_patterns(uint32_t rank_bgn, 1282 uint32_t all_ranks) 1283 { 1284 uint32_t r; 1285 uint32_t addr; 1286 uint32_t rank_end = all_ranks ? RW_MGR_MEM_NUMBER_OF_RANKS : 1287 (rank_bgn + NUM_RANKS_PER_SHADOW_REG); 1288 1289 debug("%s:%d\n", __func__, __LINE__); 1290 for (r = rank_bgn; r < rank_end; r++) { 1291 if (param->skip_ranks[r]) 1292 /* request to skip the rank */ 1293 continue; 1294 1295 /* set rank */ 1296 set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_READ_WRITE); 1297 1298 /* Load up a constant bursts */ 1299 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr0; 1300 writel(0x20, SOCFPGA_SDR_ADDRESS + addr); 1301 1302 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add0; 1303 writel(RW_MGR_GUARANTEED_WRITE_WAIT0, SOCFPGA_SDR_ADDRESS + addr); 1304 1305 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr1; 1306 writel(0x20, SOCFPGA_SDR_ADDRESS + addr); 1307 1308 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add1; 1309 writel(RW_MGR_GUARANTEED_WRITE_WAIT1, SOCFPGA_SDR_ADDRESS + addr); 1310 1311 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr2; 1312 writel(0x04, SOCFPGA_SDR_ADDRESS + addr); 1313 1314 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add2; 1315 writel(RW_MGR_GUARANTEED_WRITE_WAIT2, SOCFPGA_SDR_ADDRESS + addr); 1316 1317 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr3; 1318 writel(0x04, SOCFPGA_SDR_ADDRESS + addr); 1319 1320 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add3; 1321 writel(RW_MGR_GUARANTEED_WRITE_WAIT3, SOCFPGA_SDR_ADDRESS + addr); 1322 1323 addr = sdr_get_addr((u32 *)RW_MGR_RUN_SINGLE_GROUP); 1324 writel(RW_MGR_GUARANTEED_WRITE, SOCFPGA_SDR_ADDRESS + addr); 1325 } 1326 1327 set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF); 1328 } 1329 1330 /* 1331 * try a read and see if it returns correct data back. has dummy reads 1332 * inserted into the mix used to align dqs enable. has more thorough checks 1333 * than the regular read test. 1334 */ 1335 static uint32_t rw_mgr_mem_calibrate_read_test(uint32_t rank_bgn, uint32_t group, 1336 uint32_t num_tries, uint32_t all_correct, uint32_t *bit_chk, 1337 uint32_t all_groups, uint32_t all_ranks) 1338 { 1339 uint32_t r, vg; 1340 uint32_t correct_mask_vg; 1341 uint32_t tmp_bit_chk; 1342 uint32_t rank_end = all_ranks ? RW_MGR_MEM_NUMBER_OF_RANKS : 1343 (rank_bgn + NUM_RANKS_PER_SHADOW_REG); 1344 uint32_t addr; 1345 uint32_t base_rw_mgr; 1346 1347 *bit_chk = param->read_correct_mask; 1348 correct_mask_vg = param->read_correct_mask_vg; 1349 1350 uint32_t quick_read_mode = (((STATIC_CALIB_STEPS) & 1351 CALIB_SKIP_DELAY_SWEEPS) && ENABLE_SUPER_QUICK_CALIBRATION); 1352 1353 for (r = rank_bgn; r < rank_end; r++) { 1354 if (param->skip_ranks[r]) 1355 /* request to skip the rank */ 1356 continue; 1357 1358 /* set rank */ 1359 set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_READ_WRITE); 1360 1361 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr1; 1362 writel(0x10, SOCFPGA_SDR_ADDRESS + addr); 1363 1364 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add1; 1365 writel(RW_MGR_READ_B2B_WAIT1, SOCFPGA_SDR_ADDRESS + addr); 1366 1367 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr2; 1368 writel(0x10, SOCFPGA_SDR_ADDRESS + addr); 1369 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add2; 1370 writel(RW_MGR_READ_B2B_WAIT2, SOCFPGA_SDR_ADDRESS + addr); 1371 1372 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr0; 1373 if (quick_read_mode) 1374 writel(0x1, SOCFPGA_SDR_ADDRESS + addr); 1375 /* need at least two (1+1) reads to capture failures */ 1376 else if (all_groups) 1377 writel(0x06, SOCFPGA_SDR_ADDRESS + addr); 1378 else 1379 writel(0x32, SOCFPGA_SDR_ADDRESS + addr); 1380 1381 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add0; 1382 writel(RW_MGR_READ_B2B, SOCFPGA_SDR_ADDRESS + addr); 1383 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr3; 1384 if (all_groups) 1385 writel(RW_MGR_MEM_IF_READ_DQS_WIDTH * 1386 RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS - 1, 1387 SOCFPGA_SDR_ADDRESS + addr); 1388 else 1389 writel(0x0, SOCFPGA_SDR_ADDRESS + addr); 1390 1391 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add3; 1392 writel(RW_MGR_READ_B2B, SOCFPGA_SDR_ADDRESS + addr); 1393 1394 tmp_bit_chk = 0; 1395 for (vg = RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS-1; ; vg--) { 1396 /* reset the fifos to get pointers to known state */ 1397 addr = sdr_get_addr(&phy_mgr_cmd->fifo_reset); 1398 writel(0, SOCFPGA_SDR_ADDRESS + addr); 1399 addr = sdr_get_addr((u32 *)RW_MGR_RESET_READ_DATAPATH); 1400 writel(0, SOCFPGA_SDR_ADDRESS + addr); 1401 1402 tmp_bit_chk = tmp_bit_chk << (RW_MGR_MEM_DQ_PER_READ_DQS 1403 / RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS); 1404 1405 addr = sdr_get_addr((u32 *)(all_groups ? RW_MGR_RUN_ALL_GROUPS : 1406 RW_MGR_RUN_SINGLE_GROUP)); 1407 writel(RW_MGR_READ_B2B, SOCFPGA_SDR_ADDRESS + addr + 1408 ((group * RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS + 1409 vg) << 2)); 1410 1411 addr = SDR_PHYGRP_RWMGRGRP_ADDRESS; 1412 base_rw_mgr = readl(SOCFPGA_SDR_ADDRESS + addr); 1413 tmp_bit_chk = tmp_bit_chk | (correct_mask_vg & ~(base_rw_mgr)); 1414 1415 if (vg == 0) 1416 break; 1417 } 1418 *bit_chk &= tmp_bit_chk; 1419 } 1420 1421 addr = sdr_get_addr((u32 *)RW_MGR_RUN_SINGLE_GROUP); 1422 writel(RW_MGR_CLEAR_DQS_ENABLE, SOCFPGA_SDR_ADDRESS + addr + (group << 2)); 1423 1424 if (all_correct) { 1425 set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF); 1426 debug_cond(DLEVEL == 2, "%s:%d read_test(%u,ALL,%u) =>\ 1427 (%u == %u) => %lu", __func__, __LINE__, group, 1428 all_groups, *bit_chk, param->read_correct_mask, 1429 (long unsigned int)(*bit_chk == 1430 param->read_correct_mask)); 1431 return *bit_chk == param->read_correct_mask; 1432 } else { 1433 set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF); 1434 debug_cond(DLEVEL == 2, "%s:%d read_test(%u,ONE,%u) =>\ 1435 (%u != %lu) => %lu\n", __func__, __LINE__, 1436 group, all_groups, *bit_chk, (long unsigned int)0, 1437 (long unsigned int)(*bit_chk != 0x00)); 1438 return *bit_chk != 0x00; 1439 } 1440 } 1441 1442 static uint32_t rw_mgr_mem_calibrate_read_test_all_ranks(uint32_t group, 1443 uint32_t num_tries, uint32_t all_correct, uint32_t *bit_chk, 1444 uint32_t all_groups) 1445 { 1446 return rw_mgr_mem_calibrate_read_test(0, group, num_tries, all_correct, 1447 bit_chk, all_groups, 1); 1448 } 1449 1450 static void rw_mgr_incr_vfifo(uint32_t grp, uint32_t *v) 1451 { 1452 uint32_t addr = sdr_get_addr(&phy_mgr_cmd->inc_vfifo_hard_phy); 1453 1454 writel(grp, SOCFPGA_SDR_ADDRESS + addr); 1455 (*v)++; 1456 } 1457 1458 static void rw_mgr_decr_vfifo(uint32_t grp, uint32_t *v) 1459 { 1460 uint32_t i; 1461 1462 for (i = 0; i < VFIFO_SIZE-1; i++) 1463 rw_mgr_incr_vfifo(grp, v); 1464 } 1465 1466 static int find_vfifo_read(uint32_t grp, uint32_t *bit_chk) 1467 { 1468 uint32_t v; 1469 uint32_t fail_cnt = 0; 1470 uint32_t test_status; 1471 1472 for (v = 0; v < VFIFO_SIZE; ) { 1473 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: vfifo %u\n", 1474 __func__, __LINE__, v); 1475 test_status = rw_mgr_mem_calibrate_read_test_all_ranks 1476 (grp, 1, PASS_ONE_BIT, bit_chk, 0); 1477 if (!test_status) { 1478 fail_cnt++; 1479 1480 if (fail_cnt == 2) 1481 break; 1482 } 1483 1484 /* fiddle with FIFO */ 1485 rw_mgr_incr_vfifo(grp, &v); 1486 } 1487 1488 if (v >= VFIFO_SIZE) { 1489 /* no failing read found!! Something must have gone wrong */ 1490 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: vfifo failed\n", 1491 __func__, __LINE__); 1492 return 0; 1493 } else { 1494 return v; 1495 } 1496 } 1497 1498 static int find_working_phase(uint32_t *grp, uint32_t *bit_chk, 1499 uint32_t dtaps_per_ptap, uint32_t *work_bgn, 1500 uint32_t *v, uint32_t *d, uint32_t *p, 1501 uint32_t *i, uint32_t *max_working_cnt) 1502 { 1503 uint32_t found_begin = 0; 1504 uint32_t tmp_delay = 0; 1505 uint32_t test_status; 1506 1507 for (*d = 0; *d <= dtaps_per_ptap; (*d)++, tmp_delay += 1508 IO_DELAY_PER_DQS_EN_DCHAIN_TAP) { 1509 *work_bgn = tmp_delay; 1510 scc_mgr_set_dqs_en_delay_all_ranks(*grp, *d); 1511 1512 for (*i = 0; *i < VFIFO_SIZE; (*i)++) { 1513 for (*p = 0; *p <= IO_DQS_EN_PHASE_MAX; (*p)++, *work_bgn += 1514 IO_DELAY_PER_OPA_TAP) { 1515 scc_mgr_set_dqs_en_phase_all_ranks(*grp, *p); 1516 1517 test_status = 1518 rw_mgr_mem_calibrate_read_test_all_ranks 1519 (*grp, 1, PASS_ONE_BIT, bit_chk, 0); 1520 1521 if (test_status) { 1522 *max_working_cnt = 1; 1523 found_begin = 1; 1524 break; 1525 } 1526 } 1527 1528 if (found_begin) 1529 break; 1530 1531 if (*p > IO_DQS_EN_PHASE_MAX) 1532 /* fiddle with FIFO */ 1533 rw_mgr_incr_vfifo(*grp, v); 1534 } 1535 1536 if (found_begin) 1537 break; 1538 } 1539 1540 if (*i >= VFIFO_SIZE) { 1541 /* cannot find working solution */ 1542 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: no vfifo/\ 1543 ptap/dtap\n", __func__, __LINE__); 1544 return 0; 1545 } else { 1546 return 1; 1547 } 1548 } 1549 1550 static void sdr_backup_phase(uint32_t *grp, uint32_t *bit_chk, 1551 uint32_t *work_bgn, uint32_t *v, uint32_t *d, 1552 uint32_t *p, uint32_t *max_working_cnt) 1553 { 1554 uint32_t found_begin = 0; 1555 uint32_t tmp_delay; 1556 1557 /* Special case code for backing up a phase */ 1558 if (*p == 0) { 1559 *p = IO_DQS_EN_PHASE_MAX; 1560 rw_mgr_decr_vfifo(*grp, v); 1561 } else { 1562 (*p)--; 1563 } 1564 tmp_delay = *work_bgn - IO_DELAY_PER_OPA_TAP; 1565 scc_mgr_set_dqs_en_phase_all_ranks(*grp, *p); 1566 1567 for (*d = 0; *d <= IO_DQS_EN_DELAY_MAX && tmp_delay < *work_bgn; 1568 (*d)++, tmp_delay += IO_DELAY_PER_DQS_EN_DCHAIN_TAP) { 1569 scc_mgr_set_dqs_en_delay_all_ranks(*grp, *d); 1570 1571 if (rw_mgr_mem_calibrate_read_test_all_ranks(*grp, 1, 1572 PASS_ONE_BIT, 1573 bit_chk, 0)) { 1574 found_begin = 1; 1575 *work_bgn = tmp_delay; 1576 break; 1577 } 1578 } 1579 1580 /* We have found a working dtap before the ptap found above */ 1581 if (found_begin == 1) 1582 (*max_working_cnt)++; 1583 1584 /* 1585 * Restore VFIFO to old state before we decremented it 1586 * (if needed). 1587 */ 1588 (*p)++; 1589 if (*p > IO_DQS_EN_PHASE_MAX) { 1590 *p = 0; 1591 rw_mgr_incr_vfifo(*grp, v); 1592 } 1593 1594 scc_mgr_set_dqs_en_delay_all_ranks(*grp, 0); 1595 } 1596 1597 static int sdr_nonworking_phase(uint32_t *grp, uint32_t *bit_chk, 1598 uint32_t *work_bgn, uint32_t *v, uint32_t *d, 1599 uint32_t *p, uint32_t *i, uint32_t *max_working_cnt, 1600 uint32_t *work_end) 1601 { 1602 uint32_t found_end = 0; 1603 1604 (*p)++; 1605 *work_end += IO_DELAY_PER_OPA_TAP; 1606 if (*p > IO_DQS_EN_PHASE_MAX) { 1607 /* fiddle with FIFO */ 1608 *p = 0; 1609 rw_mgr_incr_vfifo(*grp, v); 1610 } 1611 1612 for (; *i < VFIFO_SIZE + 1; (*i)++) { 1613 for (; *p <= IO_DQS_EN_PHASE_MAX; (*p)++, *work_end 1614 += IO_DELAY_PER_OPA_TAP) { 1615 scc_mgr_set_dqs_en_phase_all_ranks(*grp, *p); 1616 1617 if (!rw_mgr_mem_calibrate_read_test_all_ranks 1618 (*grp, 1, PASS_ONE_BIT, bit_chk, 0)) { 1619 found_end = 1; 1620 break; 1621 } else { 1622 (*max_working_cnt)++; 1623 } 1624 } 1625 1626 if (found_end) 1627 break; 1628 1629 if (*p > IO_DQS_EN_PHASE_MAX) { 1630 /* fiddle with FIFO */ 1631 rw_mgr_incr_vfifo(*grp, v); 1632 *p = 0; 1633 } 1634 } 1635 1636 if (*i >= VFIFO_SIZE + 1) { 1637 /* cannot see edge of failing read */ 1638 debug_cond(DLEVEL == 2, "%s:%d sdr_nonworking_phase: end:\ 1639 failed\n", __func__, __LINE__); 1640 return 0; 1641 } else { 1642 return 1; 1643 } 1644 } 1645 1646 static int sdr_find_window_centre(uint32_t *grp, uint32_t *bit_chk, 1647 uint32_t *work_bgn, uint32_t *v, uint32_t *d, 1648 uint32_t *p, uint32_t *work_mid, 1649 uint32_t *work_end) 1650 { 1651 int i; 1652 int tmp_delay = 0; 1653 1654 *work_mid = (*work_bgn + *work_end) / 2; 1655 1656 debug_cond(DLEVEL == 2, "work_bgn=%d work_end=%d work_mid=%d\n", 1657 *work_bgn, *work_end, *work_mid); 1658 /* Get the middle delay to be less than a VFIFO delay */ 1659 for (*p = 0; *p <= IO_DQS_EN_PHASE_MAX; 1660 (*p)++, tmp_delay += IO_DELAY_PER_OPA_TAP) 1661 ; 1662 debug_cond(DLEVEL == 2, "vfifo ptap delay %d\n", tmp_delay); 1663 while (*work_mid > tmp_delay) 1664 *work_mid -= tmp_delay; 1665 debug_cond(DLEVEL == 2, "new work_mid %d\n", *work_mid); 1666 1667 tmp_delay = 0; 1668 for (*p = 0; *p <= IO_DQS_EN_PHASE_MAX && tmp_delay < *work_mid; 1669 (*p)++, tmp_delay += IO_DELAY_PER_OPA_TAP) 1670 ; 1671 tmp_delay -= IO_DELAY_PER_OPA_TAP; 1672 debug_cond(DLEVEL == 2, "new p %d, tmp_delay=%d\n", (*p) - 1, tmp_delay); 1673 for (*d = 0; *d <= IO_DQS_EN_DELAY_MAX && tmp_delay < *work_mid; (*d)++, 1674 tmp_delay += IO_DELAY_PER_DQS_EN_DCHAIN_TAP) 1675 ; 1676 debug_cond(DLEVEL == 2, "new d %d, tmp_delay=%d\n", *d, tmp_delay); 1677 1678 scc_mgr_set_dqs_en_phase_all_ranks(*grp, (*p) - 1); 1679 scc_mgr_set_dqs_en_delay_all_ranks(*grp, *d); 1680 1681 /* 1682 * push vfifo until we can successfully calibrate. We can do this 1683 * because the largest possible margin in 1 VFIFO cycle. 1684 */ 1685 for (i = 0; i < VFIFO_SIZE; i++) { 1686 debug_cond(DLEVEL == 2, "find_dqs_en_phase: center: vfifo=%u\n", 1687 *v); 1688 if (rw_mgr_mem_calibrate_read_test_all_ranks(*grp, 1, 1689 PASS_ONE_BIT, 1690 bit_chk, 0)) { 1691 break; 1692 } 1693 1694 /* fiddle with FIFO */ 1695 rw_mgr_incr_vfifo(*grp, v); 1696 } 1697 1698 if (i >= VFIFO_SIZE) { 1699 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: center: \ 1700 failed\n", __func__, __LINE__); 1701 return 0; 1702 } else { 1703 return 1; 1704 } 1705 } 1706 1707 /* find a good dqs enable to use */ 1708 static uint32_t rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase(uint32_t grp) 1709 { 1710 uint32_t v, d, p, i; 1711 uint32_t max_working_cnt; 1712 uint32_t bit_chk; 1713 uint32_t dtaps_per_ptap; 1714 uint32_t work_bgn, work_mid, work_end; 1715 uint32_t found_passing_read, found_failing_read, initial_failing_dtap; 1716 uint32_t addr; 1717 1718 debug("%s:%d %u\n", __func__, __LINE__, grp); 1719 1720 reg_file_set_sub_stage(CAL_SUBSTAGE_VFIFO_CENTER); 1721 1722 scc_mgr_set_dqs_en_delay_all_ranks(grp, 0); 1723 scc_mgr_set_dqs_en_phase_all_ranks(grp, 0); 1724 1725 /* ************************************************************** */ 1726 /* * Step 0 : Determine number of delay taps for each phase tap * */ 1727 dtaps_per_ptap = IO_DELAY_PER_OPA_TAP/IO_DELAY_PER_DQS_EN_DCHAIN_TAP; 1728 1729 /* ********************************************************* */ 1730 /* * Step 1 : First push vfifo until we get a failing read * */ 1731 v = find_vfifo_read(grp, &bit_chk); 1732 1733 max_working_cnt = 0; 1734 1735 /* ******************************************************** */ 1736 /* * step 2: find first working phase, increment in ptaps * */ 1737 work_bgn = 0; 1738 if (find_working_phase(&grp, &bit_chk, dtaps_per_ptap, &work_bgn, &v, &d, 1739 &p, &i, &max_working_cnt) == 0) 1740 return 0; 1741 1742 work_end = work_bgn; 1743 1744 /* 1745 * If d is 0 then the working window covers a phase tap and 1746 * we can follow the old procedure otherwise, we've found the beginning, 1747 * and we need to increment the dtaps until we find the end. 1748 */ 1749 if (d == 0) { 1750 /* ********************************************************* */ 1751 /* * step 3a: if we have room, back off by one and 1752 increment in dtaps * */ 1753 1754 sdr_backup_phase(&grp, &bit_chk, &work_bgn, &v, &d, &p, 1755 &max_working_cnt); 1756 1757 /* ********************************************************* */ 1758 /* * step 4a: go forward from working phase to non working 1759 phase, increment in ptaps * */ 1760 if (sdr_nonworking_phase(&grp, &bit_chk, &work_bgn, &v, &d, &p, 1761 &i, &max_working_cnt, &work_end) == 0) 1762 return 0; 1763 1764 /* ********************************************************* */ 1765 /* * step 5a: back off one from last, increment in dtaps * */ 1766 1767 /* Special case code for backing up a phase */ 1768 if (p == 0) { 1769 p = IO_DQS_EN_PHASE_MAX; 1770 rw_mgr_decr_vfifo(grp, &v); 1771 } else { 1772 p = p - 1; 1773 } 1774 1775 work_end -= IO_DELAY_PER_OPA_TAP; 1776 scc_mgr_set_dqs_en_phase_all_ranks(grp, p); 1777 1778 /* * The actual increment of dtaps is done outside of 1779 the if/else loop to share code */ 1780 d = 0; 1781 1782 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: v/p: \ 1783 vfifo=%u ptap=%u\n", __func__, __LINE__, 1784 v, p); 1785 } else { 1786 /* ******************************************************* */ 1787 /* * step 3-5b: Find the right edge of the window using 1788 delay taps * */ 1789 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase:vfifo=%u \ 1790 ptap=%u dtap=%u bgn=%u\n", __func__, __LINE__, 1791 v, p, d, work_bgn); 1792 1793 work_end = work_bgn; 1794 1795 /* * The actual increment of dtaps is done outside of the 1796 if/else loop to share code */ 1797 1798 /* Only here to counterbalance a subtract later on which is 1799 not needed if this branch of the algorithm is taken */ 1800 max_working_cnt++; 1801 } 1802 1803 /* The dtap increment to find the failing edge is done here */ 1804 for (; d <= IO_DQS_EN_DELAY_MAX; d++, work_end += 1805 IO_DELAY_PER_DQS_EN_DCHAIN_TAP) { 1806 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: \ 1807 end-2: dtap=%u\n", __func__, __LINE__, d); 1808 scc_mgr_set_dqs_en_delay_all_ranks(grp, d); 1809 1810 if (!rw_mgr_mem_calibrate_read_test_all_ranks(grp, 1, 1811 PASS_ONE_BIT, 1812 &bit_chk, 0)) { 1813 break; 1814 } 1815 } 1816 1817 /* Go back to working dtap */ 1818 if (d != 0) 1819 work_end -= IO_DELAY_PER_DQS_EN_DCHAIN_TAP; 1820 1821 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: v/p/d: vfifo=%u \ 1822 ptap=%u dtap=%u end=%u\n", __func__, __LINE__, 1823 v, p, d-1, work_end); 1824 1825 if (work_end < work_bgn) { 1826 /* nil range */ 1827 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: end-2: \ 1828 failed\n", __func__, __LINE__); 1829 return 0; 1830 } 1831 1832 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: found range [%u,%u]\n", 1833 __func__, __LINE__, work_bgn, work_end); 1834 1835 /* *************************************************************** */ 1836 /* 1837 * * We need to calculate the number of dtaps that equal a ptap 1838 * * To do that we'll back up a ptap and re-find the edge of the 1839 * * window using dtaps 1840 */ 1841 1842 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: calculate dtaps_per_ptap \ 1843 for tracking\n", __func__, __LINE__); 1844 1845 /* Special case code for backing up a phase */ 1846 if (p == 0) { 1847 p = IO_DQS_EN_PHASE_MAX; 1848 rw_mgr_decr_vfifo(grp, &v); 1849 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: backedup \ 1850 cycle/phase: v=%u p=%u\n", __func__, __LINE__, 1851 v, p); 1852 } else { 1853 p = p - 1; 1854 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: backedup \ 1855 phase only: v=%u p=%u", __func__, __LINE__, 1856 v, p); 1857 } 1858 1859 scc_mgr_set_dqs_en_phase_all_ranks(grp, p); 1860 1861 /* 1862 * Increase dtap until we first see a passing read (in case the 1863 * window is smaller than a ptap), 1864 * and then a failing read to mark the edge of the window again 1865 */ 1866 1867 /* Find a passing read */ 1868 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: find passing read\n", 1869 __func__, __LINE__); 1870 found_passing_read = 0; 1871 found_failing_read = 0; 1872 initial_failing_dtap = d; 1873 for (; d <= IO_DQS_EN_DELAY_MAX; d++) { 1874 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: testing \ 1875 read d=%u\n", __func__, __LINE__, d); 1876 scc_mgr_set_dqs_en_delay_all_ranks(grp, d); 1877 1878 if (rw_mgr_mem_calibrate_read_test_all_ranks(grp, 1, 1879 PASS_ONE_BIT, 1880 &bit_chk, 0)) { 1881 found_passing_read = 1; 1882 break; 1883 } 1884 } 1885 1886 if (found_passing_read) { 1887 /* Find a failing read */ 1888 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: find failing \ 1889 read\n", __func__, __LINE__); 1890 for (d = d + 1; d <= IO_DQS_EN_DELAY_MAX; d++) { 1891 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: \ 1892 testing read d=%u\n", __func__, __LINE__, d); 1893 scc_mgr_set_dqs_en_delay_all_ranks(grp, d); 1894 1895 if (!rw_mgr_mem_calibrate_read_test_all_ranks 1896 (grp, 1, PASS_ONE_BIT, &bit_chk, 0)) { 1897 found_failing_read = 1; 1898 break; 1899 } 1900 } 1901 } else { 1902 debug_cond(DLEVEL == 1, "%s:%d find_dqs_en_phase: failed to \ 1903 calculate dtaps", __func__, __LINE__); 1904 debug_cond(DLEVEL == 1, "per ptap. Fall back on static value\n"); 1905 } 1906 1907 /* 1908 * The dynamically calculated dtaps_per_ptap is only valid if we 1909 * found a passing/failing read. If we didn't, it means d hit the max 1910 * (IO_DQS_EN_DELAY_MAX). Otherwise, dtaps_per_ptap retains its 1911 * statically calculated value. 1912 */ 1913 if (found_passing_read && found_failing_read) 1914 dtaps_per_ptap = d - initial_failing_dtap; 1915 1916 addr = (u32)&sdr_reg_file->dtaps_per_ptap; 1917 writel(dtaps_per_ptap, SOCFPGA_SDR_ADDRESS + addr); 1918 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: dtaps_per_ptap=%u \ 1919 - %u = %u", __func__, __LINE__, d, 1920 initial_failing_dtap, dtaps_per_ptap); 1921 1922 /* ******************************************** */ 1923 /* * step 6: Find the centre of the window * */ 1924 if (sdr_find_window_centre(&grp, &bit_chk, &work_bgn, &v, &d, &p, 1925 &work_mid, &work_end) == 0) 1926 return 0; 1927 1928 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: center found: \ 1929 vfifo=%u ptap=%u dtap=%u\n", __func__, __LINE__, 1930 v, p-1, d); 1931 return 1; 1932 } 1933 1934 /* 1935 * Try rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase across different 1936 * dq_in_delay values 1937 */ 1938 static uint32_t 1939 rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase_sweep_dq_in_delay 1940 (uint32_t write_group, uint32_t read_group, uint32_t test_bgn) 1941 { 1942 uint32_t found; 1943 uint32_t i; 1944 uint32_t p; 1945 uint32_t d; 1946 uint32_t r; 1947 uint32_t addr; 1948 1949 const uint32_t delay_step = IO_IO_IN_DELAY_MAX / 1950 (RW_MGR_MEM_DQ_PER_READ_DQS-1); 1951 /* we start at zero, so have one less dq to devide among */ 1952 1953 debug("%s:%d (%u,%u,%u)", __func__, __LINE__, write_group, read_group, 1954 test_bgn); 1955 1956 /* try different dq_in_delays since the dq path is shorter than dqs */ 1957 1958 for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; 1959 r += NUM_RANKS_PER_SHADOW_REG) { 1960 for (i = 0, p = test_bgn, d = 0; i < RW_MGR_MEM_DQ_PER_READ_DQS; 1961 i++, p++, d += delay_step) { 1962 debug_cond(DLEVEL == 1, "%s:%d rw_mgr_mem_calibrate_\ 1963 vfifo_find_dqs_", __func__, __LINE__); 1964 debug_cond(DLEVEL == 1, "en_phase_sweep_dq_in_delay: g=%u/%u ", 1965 write_group, read_group); 1966 debug_cond(DLEVEL == 1, "r=%u, i=%u p=%u d=%u\n", r, i , p, d); 1967 scc_mgr_set_dq_in_delay(write_group, p, d); 1968 scc_mgr_load_dq(p); 1969 } 1970 addr = (u32)&sdr_scc_mgr->update; 1971 writel(0, SOCFPGA_SDR_ADDRESS + addr); 1972 } 1973 1974 found = rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase(read_group); 1975 1976 debug_cond(DLEVEL == 1, "%s:%d rw_mgr_mem_calibrate_vfifo_find_dqs_\ 1977 en_phase_sweep_dq", __func__, __LINE__); 1978 debug_cond(DLEVEL == 1, "_in_delay: g=%u/%u found=%u; Reseting delay \ 1979 chain to zero\n", write_group, read_group, found); 1980 1981 for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; 1982 r += NUM_RANKS_PER_SHADOW_REG) { 1983 for (i = 0, p = test_bgn; i < RW_MGR_MEM_DQ_PER_READ_DQS; 1984 i++, p++) { 1985 scc_mgr_set_dq_in_delay(write_group, p, 0); 1986 scc_mgr_load_dq(p); 1987 } 1988 addr = (u32)&sdr_scc_mgr->update; 1989 writel(0, SOCFPGA_SDR_ADDRESS + addr); 1990 } 1991 1992 return found; 1993 } 1994 1995 /* per-bit deskew DQ and center */ 1996 static uint32_t rw_mgr_mem_calibrate_vfifo_center(uint32_t rank_bgn, 1997 uint32_t write_group, uint32_t read_group, uint32_t test_bgn, 1998 uint32_t use_read_test, uint32_t update_fom) 1999 { 2000 uint32_t i, p, d, min_index; 2001 /* 2002 * Store these as signed since there are comparisons with 2003 * signed numbers. 2004 */ 2005 uint32_t bit_chk; 2006 uint32_t sticky_bit_chk; 2007 int32_t left_edge[RW_MGR_MEM_DQ_PER_READ_DQS]; 2008 int32_t right_edge[RW_MGR_MEM_DQ_PER_READ_DQS]; 2009 int32_t final_dq[RW_MGR_MEM_DQ_PER_READ_DQS]; 2010 int32_t mid; 2011 int32_t orig_mid_min, mid_min; 2012 int32_t new_dqs, start_dqs, start_dqs_en, shift_dq, final_dqs, 2013 final_dqs_en; 2014 int32_t dq_margin, dqs_margin; 2015 uint32_t stop; 2016 uint32_t temp_dq_in_delay1, temp_dq_in_delay2; 2017 uint32_t addr; 2018 2019 debug("%s:%d: %u %u", __func__, __LINE__, read_group, test_bgn); 2020 2021 addr = sdr_get_addr((u32 *)SCC_MGR_DQS_IN_DELAY); 2022 start_dqs = readl(SOCFPGA_SDR_ADDRESS + addr + (read_group << 2)); 2023 if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS) 2024 start_dqs_en = readl(SOCFPGA_SDR_ADDRESS + addr + ((read_group << 2) 2025 - IO_DQS_EN_DELAY_OFFSET)); 2026 2027 /* set the left and right edge of each bit to an illegal value */ 2028 /* use (IO_IO_IN_DELAY_MAX + 1) as an illegal value */ 2029 sticky_bit_chk = 0; 2030 for (i = 0; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++) { 2031 left_edge[i] = IO_IO_IN_DELAY_MAX + 1; 2032 right_edge[i] = IO_IO_IN_DELAY_MAX + 1; 2033 } 2034 2035 addr = (u32)&sdr_scc_mgr->update; 2036 /* Search for the left edge of the window for each bit */ 2037 for (d = 0; d <= IO_IO_IN_DELAY_MAX; d++) { 2038 scc_mgr_apply_group_dq_in_delay(write_group, test_bgn, d); 2039 2040 writel(0, SOCFPGA_SDR_ADDRESS + addr); 2041 2042 /* 2043 * Stop searching when the read test doesn't pass AND when 2044 * we've seen a passing read on every bit. 2045 */ 2046 if (use_read_test) { 2047 stop = !rw_mgr_mem_calibrate_read_test(rank_bgn, 2048 read_group, NUM_READ_PB_TESTS, PASS_ONE_BIT, 2049 &bit_chk, 0, 0); 2050 } else { 2051 rw_mgr_mem_calibrate_write_test(rank_bgn, write_group, 2052 0, PASS_ONE_BIT, 2053 &bit_chk, 0); 2054 bit_chk = bit_chk >> (RW_MGR_MEM_DQ_PER_READ_DQS * 2055 (read_group - (write_group * 2056 RW_MGR_MEM_IF_READ_DQS_WIDTH / 2057 RW_MGR_MEM_IF_WRITE_DQS_WIDTH))); 2058 stop = (bit_chk == 0); 2059 } 2060 sticky_bit_chk = sticky_bit_chk | bit_chk; 2061 stop = stop && (sticky_bit_chk == param->read_correct_mask); 2062 debug_cond(DLEVEL == 2, "%s:%d vfifo_center(left): dtap=%u => %u == %u \ 2063 && %u", __func__, __LINE__, d, 2064 sticky_bit_chk, 2065 param->read_correct_mask, stop); 2066 2067 if (stop == 1) { 2068 break; 2069 } else { 2070 for (i = 0; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++) { 2071 if (bit_chk & 1) { 2072 /* Remember a passing test as the 2073 left_edge */ 2074 left_edge[i] = d; 2075 } else { 2076 /* If a left edge has not been seen yet, 2077 then a future passing test will mark 2078 this edge as the right edge */ 2079 if (left_edge[i] == 2080 IO_IO_IN_DELAY_MAX + 1) { 2081 right_edge[i] = -(d + 1); 2082 } 2083 } 2084 bit_chk = bit_chk >> 1; 2085 } 2086 } 2087 } 2088 2089 /* Reset DQ delay chains to 0 */ 2090 scc_mgr_apply_group_dq_in_delay(write_group, test_bgn, 0); 2091 sticky_bit_chk = 0; 2092 for (i = RW_MGR_MEM_DQ_PER_READ_DQS - 1;; i--) { 2093 debug_cond(DLEVEL == 2, "%s:%d vfifo_center: left_edge[%u]: \ 2094 %d right_edge[%u]: %d\n", __func__, __LINE__, 2095 i, left_edge[i], i, right_edge[i]); 2096 2097 /* 2098 * Check for cases where we haven't found the left edge, 2099 * which makes our assignment of the the right edge invalid. 2100 * Reset it to the illegal value. 2101 */ 2102 if ((left_edge[i] == IO_IO_IN_DELAY_MAX + 1) && ( 2103 right_edge[i] != IO_IO_IN_DELAY_MAX + 1)) { 2104 right_edge[i] = IO_IO_IN_DELAY_MAX + 1; 2105 debug_cond(DLEVEL == 2, "%s:%d vfifo_center: reset \ 2106 right_edge[%u]: %d\n", __func__, __LINE__, 2107 i, right_edge[i]); 2108 } 2109 2110 /* 2111 * Reset sticky bit (except for bits where we have seen 2112 * both the left and right edge). 2113 */ 2114 sticky_bit_chk = sticky_bit_chk << 1; 2115 if ((left_edge[i] != IO_IO_IN_DELAY_MAX + 1) && 2116 (right_edge[i] != IO_IO_IN_DELAY_MAX + 1)) { 2117 sticky_bit_chk = sticky_bit_chk | 1; 2118 } 2119 2120 if (i == 0) 2121 break; 2122 } 2123 2124 addr = (u32)&sdr_scc_mgr->update; 2125 /* Search for the right edge of the window for each bit */ 2126 for (d = 0; d <= IO_DQS_IN_DELAY_MAX - start_dqs; d++) { 2127 scc_mgr_set_dqs_bus_in_delay(read_group, d + start_dqs); 2128 if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS) { 2129 uint32_t delay = d + start_dqs_en; 2130 if (delay > IO_DQS_EN_DELAY_MAX) 2131 delay = IO_DQS_EN_DELAY_MAX; 2132 scc_mgr_set_dqs_en_delay(read_group, delay); 2133 } 2134 scc_mgr_load_dqs(read_group); 2135 2136 writel(0, SOCFPGA_SDR_ADDRESS + addr); 2137 2138 /* 2139 * Stop searching when the read test doesn't pass AND when 2140 * we've seen a passing read on every bit. 2141 */ 2142 if (use_read_test) { 2143 stop = !rw_mgr_mem_calibrate_read_test(rank_bgn, 2144 read_group, NUM_READ_PB_TESTS, PASS_ONE_BIT, 2145 &bit_chk, 0, 0); 2146 } else { 2147 rw_mgr_mem_calibrate_write_test(rank_bgn, write_group, 2148 0, PASS_ONE_BIT, 2149 &bit_chk, 0); 2150 bit_chk = bit_chk >> (RW_MGR_MEM_DQ_PER_READ_DQS * 2151 (read_group - (write_group * 2152 RW_MGR_MEM_IF_READ_DQS_WIDTH / 2153 RW_MGR_MEM_IF_WRITE_DQS_WIDTH))); 2154 stop = (bit_chk == 0); 2155 } 2156 sticky_bit_chk = sticky_bit_chk | bit_chk; 2157 stop = stop && (sticky_bit_chk == param->read_correct_mask); 2158 2159 debug_cond(DLEVEL == 2, "%s:%d vfifo_center(right): dtap=%u => %u == \ 2160 %u && %u", __func__, __LINE__, d, 2161 sticky_bit_chk, param->read_correct_mask, stop); 2162 2163 if (stop == 1) { 2164 break; 2165 } else { 2166 for (i = 0; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++) { 2167 if (bit_chk & 1) { 2168 /* Remember a passing test as 2169 the right_edge */ 2170 right_edge[i] = d; 2171 } else { 2172 if (d != 0) { 2173 /* If a right edge has not been 2174 seen yet, then a future passing 2175 test will mark this edge as the 2176 left edge */ 2177 if (right_edge[i] == 2178 IO_IO_IN_DELAY_MAX + 1) { 2179 left_edge[i] = -(d + 1); 2180 } 2181 } else { 2182 /* d = 0 failed, but it passed 2183 when testing the left edge, 2184 so it must be marginal, 2185 set it to -1 */ 2186 if (right_edge[i] == 2187 IO_IO_IN_DELAY_MAX + 1 && 2188 left_edge[i] != 2189 IO_IO_IN_DELAY_MAX 2190 + 1) { 2191 right_edge[i] = -1; 2192 } 2193 /* If a right edge has not been 2194 seen yet, then a future passing 2195 test will mark this edge as the 2196 left edge */ 2197 else if (right_edge[i] == 2198 IO_IO_IN_DELAY_MAX + 2199 1) { 2200 left_edge[i] = -(d + 1); 2201 } 2202 } 2203 } 2204 2205 debug_cond(DLEVEL == 2, "%s:%d vfifo_center[r,\ 2206 d=%u]: ", __func__, __LINE__, d); 2207 debug_cond(DLEVEL == 2, "bit_chk_test=%d left_edge[%u]: %d ", 2208 (int)(bit_chk & 1), i, left_edge[i]); 2209 debug_cond(DLEVEL == 2, "right_edge[%u]: %d\n", i, 2210 right_edge[i]); 2211 bit_chk = bit_chk >> 1; 2212 } 2213 } 2214 } 2215 2216 /* Check that all bits have a window */ 2217 addr = (u32)&sdr_scc_mgr->update; 2218 for (i = 0; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++) { 2219 debug_cond(DLEVEL == 2, "%s:%d vfifo_center: left_edge[%u]: \ 2220 %d right_edge[%u]: %d", __func__, __LINE__, 2221 i, left_edge[i], i, right_edge[i]); 2222 if ((left_edge[i] == IO_IO_IN_DELAY_MAX + 1) || (right_edge[i] 2223 == IO_IO_IN_DELAY_MAX + 1)) { 2224 /* 2225 * Restore delay chain settings before letting the loop 2226 * in rw_mgr_mem_calibrate_vfifo to retry different 2227 * dqs/ck relationships. 2228 */ 2229 scc_mgr_set_dqs_bus_in_delay(read_group, start_dqs); 2230 if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS) { 2231 scc_mgr_set_dqs_en_delay(read_group, 2232 start_dqs_en); 2233 } 2234 scc_mgr_load_dqs(read_group); 2235 writel(0, SOCFPGA_SDR_ADDRESS + addr); 2236 2237 debug_cond(DLEVEL == 1, "%s:%d vfifo_center: failed to \ 2238 find edge [%u]: %d %d", __func__, __LINE__, 2239 i, left_edge[i], right_edge[i]); 2240 if (use_read_test) { 2241 set_failing_group_stage(read_group * 2242 RW_MGR_MEM_DQ_PER_READ_DQS + i, 2243 CAL_STAGE_VFIFO, 2244 CAL_SUBSTAGE_VFIFO_CENTER); 2245 } else { 2246 set_failing_group_stage(read_group * 2247 RW_MGR_MEM_DQ_PER_READ_DQS + i, 2248 CAL_STAGE_VFIFO_AFTER_WRITES, 2249 CAL_SUBSTAGE_VFIFO_CENTER); 2250 } 2251 return 0; 2252 } 2253 } 2254 2255 /* Find middle of window for each DQ bit */ 2256 mid_min = left_edge[0] - right_edge[0]; 2257 min_index = 0; 2258 for (i = 1; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++) { 2259 mid = left_edge[i] - right_edge[i]; 2260 if (mid < mid_min) { 2261 mid_min = mid; 2262 min_index = i; 2263 } 2264 } 2265 2266 /* 2267 * -mid_min/2 represents the amount that we need to move DQS. 2268 * If mid_min is odd and positive we'll need to add one to 2269 * make sure the rounding in further calculations is correct 2270 * (always bias to the right), so just add 1 for all positive values. 2271 */ 2272 if (mid_min > 0) 2273 mid_min++; 2274 2275 mid_min = mid_min / 2; 2276 2277 debug_cond(DLEVEL == 1, "%s:%d vfifo_center: mid_min=%d (index=%u)\n", 2278 __func__, __LINE__, mid_min, min_index); 2279 2280 /* Determine the amount we can change DQS (which is -mid_min) */ 2281 orig_mid_min = mid_min; 2282 new_dqs = start_dqs - mid_min; 2283 if (new_dqs > IO_DQS_IN_DELAY_MAX) 2284 new_dqs = IO_DQS_IN_DELAY_MAX; 2285 else if (new_dqs < 0) 2286 new_dqs = 0; 2287 2288 mid_min = start_dqs - new_dqs; 2289 debug_cond(DLEVEL == 1, "vfifo_center: new mid_min=%d new_dqs=%d\n", 2290 mid_min, new_dqs); 2291 2292 if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS) { 2293 if (start_dqs_en - mid_min > IO_DQS_EN_DELAY_MAX) 2294 mid_min += start_dqs_en - mid_min - IO_DQS_EN_DELAY_MAX; 2295 else if (start_dqs_en - mid_min < 0) 2296 mid_min += start_dqs_en - mid_min; 2297 } 2298 new_dqs = start_dqs - mid_min; 2299 2300 debug_cond(DLEVEL == 1, "vfifo_center: start_dqs=%d start_dqs_en=%d \ 2301 new_dqs=%d mid_min=%d\n", start_dqs, 2302 IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS ? start_dqs_en : -1, 2303 new_dqs, mid_min); 2304 2305 /* Initialize data for export structures */ 2306 dqs_margin = IO_IO_IN_DELAY_MAX + 1; 2307 dq_margin = IO_IO_IN_DELAY_MAX + 1; 2308 2309 addr = sdr_get_addr((u32 *)SCC_MGR_IO_IN_DELAY); 2310 /* add delay to bring centre of all DQ windows to the same "level" */ 2311 for (i = 0, p = test_bgn; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++, p++) { 2312 /* Use values before divide by 2 to reduce round off error */ 2313 shift_dq = (left_edge[i] - right_edge[i] - 2314 (left_edge[min_index] - right_edge[min_index]))/2 + 2315 (orig_mid_min - mid_min); 2316 2317 debug_cond(DLEVEL == 2, "vfifo_center: before: \ 2318 shift_dq[%u]=%d\n", i, shift_dq); 2319 2320 temp_dq_in_delay1 = readl(SOCFPGA_SDR_ADDRESS + addr + (p << 2)); 2321 temp_dq_in_delay2 = readl(SOCFPGA_SDR_ADDRESS + addr + (i << 2)); 2322 2323 if (shift_dq + (int32_t)temp_dq_in_delay1 > 2324 (int32_t)IO_IO_IN_DELAY_MAX) { 2325 shift_dq = (int32_t)IO_IO_IN_DELAY_MAX - temp_dq_in_delay2; 2326 } else if (shift_dq + (int32_t)temp_dq_in_delay1 < 0) { 2327 shift_dq = -(int32_t)temp_dq_in_delay1; 2328 } 2329 debug_cond(DLEVEL == 2, "vfifo_center: after: \ 2330 shift_dq[%u]=%d\n", i, shift_dq); 2331 final_dq[i] = temp_dq_in_delay1 + shift_dq; 2332 scc_mgr_set_dq_in_delay(write_group, p, final_dq[i]); 2333 scc_mgr_load_dq(p); 2334 2335 debug_cond(DLEVEL == 2, "vfifo_center: margin[%u]=[%d,%d]\n", i, 2336 left_edge[i] - shift_dq + (-mid_min), 2337 right_edge[i] + shift_dq - (-mid_min)); 2338 /* To determine values for export structures */ 2339 if (left_edge[i] - shift_dq + (-mid_min) < dq_margin) 2340 dq_margin = left_edge[i] - shift_dq + (-mid_min); 2341 2342 if (right_edge[i] + shift_dq - (-mid_min) < dqs_margin) 2343 dqs_margin = right_edge[i] + shift_dq - (-mid_min); 2344 } 2345 2346 final_dqs = new_dqs; 2347 if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS) 2348 final_dqs_en = start_dqs_en - mid_min; 2349 2350 /* Move DQS-en */ 2351 if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS) { 2352 scc_mgr_set_dqs_en_delay(read_group, final_dqs_en); 2353 scc_mgr_load_dqs(read_group); 2354 } 2355 2356 /* Move DQS */ 2357 scc_mgr_set_dqs_bus_in_delay(read_group, final_dqs); 2358 scc_mgr_load_dqs(read_group); 2359 debug_cond(DLEVEL == 2, "%s:%d vfifo_center: dq_margin=%d \ 2360 dqs_margin=%d", __func__, __LINE__, 2361 dq_margin, dqs_margin); 2362 2363 /* 2364 * Do not remove this line as it makes sure all of our decisions 2365 * have been applied. Apply the update bit. 2366 */ 2367 addr = (u32)&sdr_scc_mgr->update; 2368 writel(0, SOCFPGA_SDR_ADDRESS + addr); 2369 2370 return (dq_margin >= 0) && (dqs_margin >= 0); 2371 } 2372 2373 /* 2374 * calibrate the read valid prediction FIFO. 2375 * 2376 * - read valid prediction will consist of finding a good DQS enable phase, 2377 * DQS enable delay, DQS input phase, and DQS input delay. 2378 * - we also do a per-bit deskew on the DQ lines. 2379 */ 2380 static uint32_t rw_mgr_mem_calibrate_vfifo(uint32_t read_group, 2381 uint32_t test_bgn) 2382 { 2383 uint32_t p, d, rank_bgn, sr; 2384 uint32_t dtaps_per_ptap; 2385 uint32_t tmp_delay; 2386 uint32_t bit_chk; 2387 uint32_t grp_calibrated; 2388 uint32_t write_group, write_test_bgn; 2389 uint32_t failed_substage; 2390 2391 debug("%s:%d: %u %u\n", __func__, __LINE__, read_group, test_bgn); 2392 2393 /* update info for sims */ 2394 reg_file_set_stage(CAL_STAGE_VFIFO); 2395 2396 write_group = read_group; 2397 write_test_bgn = test_bgn; 2398 2399 /* USER Determine number of delay taps for each phase tap */ 2400 dtaps_per_ptap = 0; 2401 tmp_delay = 0; 2402 while (tmp_delay < IO_DELAY_PER_OPA_TAP) { 2403 dtaps_per_ptap++; 2404 tmp_delay += IO_DELAY_PER_DQS_EN_DCHAIN_TAP; 2405 } 2406 dtaps_per_ptap--; 2407 tmp_delay = 0; 2408 2409 /* update info for sims */ 2410 reg_file_set_group(read_group); 2411 2412 grp_calibrated = 0; 2413 2414 reg_file_set_sub_stage(CAL_SUBSTAGE_GUARANTEED_READ); 2415 failed_substage = CAL_SUBSTAGE_GUARANTEED_READ; 2416 2417 for (d = 0; d <= dtaps_per_ptap && grp_calibrated == 0; d += 2) { 2418 /* 2419 * In RLDRAMX we may be messing the delay of pins in 2420 * the same write group but outside of the current read 2421 * the group, but that's ok because we haven't 2422 * calibrated output side yet. 2423 */ 2424 if (d > 0) { 2425 scc_mgr_apply_group_all_out_delay_add_all_ranks 2426 (write_group, write_test_bgn, d); 2427 } 2428 2429 for (p = 0; p <= IO_DQDQS_OUT_PHASE_MAX && grp_calibrated == 0; 2430 p++) { 2431 /* set a particular dqdqs phase */ 2432 scc_mgr_set_dqdqs_output_phase_all_ranks(read_group, p); 2433 2434 debug_cond(DLEVEL == 1, "%s:%d calibrate_vfifo: g=%u \ 2435 p=%u d=%u\n", __func__, __LINE__, 2436 read_group, p, d); 2437 2438 /* 2439 * Load up the patterns used by read calibration 2440 * using current DQDQS phase. 2441 */ 2442 rw_mgr_mem_calibrate_read_load_patterns(0, 1); 2443 if (!(gbl->phy_debug_mode_flags & 2444 PHY_DEBUG_DISABLE_GUARANTEED_READ)) { 2445 if (!rw_mgr_mem_calibrate_read_test_patterns_all_ranks 2446 (read_group, 1, &bit_chk)) { 2447 debug_cond(DLEVEL == 1, "%s:%d Guaranteed read test failed:", 2448 __func__, __LINE__); 2449 debug_cond(DLEVEL == 1, " g=%u p=%u d=%u\n", 2450 read_group, p, d); 2451 break; 2452 } 2453 } 2454 2455 /* case:56390 */ 2456 grp_calibrated = 1; 2457 if (rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase_sweep_dq_in_delay 2458 (write_group, read_group, test_bgn)) { 2459 /* 2460 * USER Read per-bit deskew can be done on a 2461 * per shadow register basis. 2462 */ 2463 for (rank_bgn = 0, sr = 0; 2464 rank_bgn < RW_MGR_MEM_NUMBER_OF_RANKS; 2465 rank_bgn += NUM_RANKS_PER_SHADOW_REG, 2466 ++sr) { 2467 /* 2468 * Determine if this set of ranks 2469 * should be skipped entirely. 2470 */ 2471 if (!param->skip_shadow_regs[sr]) { 2472 /* 2473 * If doing read after write 2474 * calibration, do not update 2475 * FOM, now - do it then. 2476 */ 2477 if (!rw_mgr_mem_calibrate_vfifo_center 2478 (rank_bgn, write_group, 2479 read_group, test_bgn, 1, 0)) { 2480 grp_calibrated = 0; 2481 failed_substage = 2482 CAL_SUBSTAGE_VFIFO_CENTER; 2483 } 2484 } 2485 } 2486 } else { 2487 grp_calibrated = 0; 2488 failed_substage = CAL_SUBSTAGE_DQS_EN_PHASE; 2489 } 2490 } 2491 } 2492 2493 if (grp_calibrated == 0) { 2494 set_failing_group_stage(write_group, CAL_STAGE_VFIFO, 2495 failed_substage); 2496 return 0; 2497 } 2498 2499 /* 2500 * Reset the delay chains back to zero if they have moved > 1 2501 * (check for > 1 because loop will increase d even when pass in 2502 * first case). 2503 */ 2504 if (d > 2) 2505 scc_mgr_zero_group(write_group, write_test_bgn, 1); 2506 2507 return 1; 2508 } 2509 2510 /* VFIFO Calibration -- Read Deskew Calibration after write deskew */ 2511 static uint32_t rw_mgr_mem_calibrate_vfifo_end(uint32_t read_group, 2512 uint32_t test_bgn) 2513 { 2514 uint32_t rank_bgn, sr; 2515 uint32_t grp_calibrated; 2516 uint32_t write_group; 2517 2518 debug("%s:%d %u %u", __func__, __LINE__, read_group, test_bgn); 2519 2520 /* update info for sims */ 2521 2522 reg_file_set_stage(CAL_STAGE_VFIFO_AFTER_WRITES); 2523 reg_file_set_sub_stage(CAL_SUBSTAGE_VFIFO_CENTER); 2524 2525 write_group = read_group; 2526 2527 /* update info for sims */ 2528 reg_file_set_group(read_group); 2529 2530 grp_calibrated = 1; 2531 /* Read per-bit deskew can be done on a per shadow register basis */ 2532 for (rank_bgn = 0, sr = 0; rank_bgn < RW_MGR_MEM_NUMBER_OF_RANKS; 2533 rank_bgn += NUM_RANKS_PER_SHADOW_REG, ++sr) { 2534 /* Determine if this set of ranks should be skipped entirely */ 2535 if (!param->skip_shadow_regs[sr]) { 2536 /* This is the last calibration round, update FOM here */ 2537 if (!rw_mgr_mem_calibrate_vfifo_center(rank_bgn, 2538 write_group, 2539 read_group, 2540 test_bgn, 0, 2541 1)) { 2542 grp_calibrated = 0; 2543 } 2544 } 2545 } 2546 2547 2548 if (grp_calibrated == 0) { 2549 set_failing_group_stage(write_group, 2550 CAL_STAGE_VFIFO_AFTER_WRITES, 2551 CAL_SUBSTAGE_VFIFO_CENTER); 2552 return 0; 2553 } 2554 2555 return 1; 2556 } 2557 2558 /* Calibrate LFIFO to find smallest read latency */ 2559 static uint32_t rw_mgr_mem_calibrate_lfifo(void) 2560 { 2561 uint32_t found_one; 2562 uint32_t bit_chk; 2563 uint32_t addr; 2564 2565 debug("%s:%d\n", __func__, __LINE__); 2566 2567 /* update info for sims */ 2568 reg_file_set_stage(CAL_STAGE_LFIFO); 2569 reg_file_set_sub_stage(CAL_SUBSTAGE_READ_LATENCY); 2570 2571 /* Load up the patterns used by read calibration for all ranks */ 2572 rw_mgr_mem_calibrate_read_load_patterns(0, 1); 2573 found_one = 0; 2574 2575 addr = sdr_get_addr(&phy_mgr_cfg->phy_rlat); 2576 do { 2577 writel(gbl->curr_read_lat, SOCFPGA_SDR_ADDRESS + addr); 2578 debug_cond(DLEVEL == 2, "%s:%d lfifo: read_lat=%u", 2579 __func__, __LINE__, gbl->curr_read_lat); 2580 2581 if (!rw_mgr_mem_calibrate_read_test_all_ranks(0, 2582 NUM_READ_TESTS, 2583 PASS_ALL_BITS, 2584 &bit_chk, 1)) { 2585 break; 2586 } 2587 2588 found_one = 1; 2589 /* reduce read latency and see if things are working */ 2590 /* correctly */ 2591 gbl->curr_read_lat--; 2592 } while (gbl->curr_read_lat > 0); 2593 2594 /* reset the fifos to get pointers to known state */ 2595 2596 addr = sdr_get_addr(&phy_mgr_cmd->fifo_reset); 2597 writel(0, SOCFPGA_SDR_ADDRESS + addr); 2598 2599 if (found_one) { 2600 /* add a fudge factor to the read latency that was determined */ 2601 gbl->curr_read_lat += 2; 2602 addr = sdr_get_addr(&phy_mgr_cfg->phy_rlat); 2603 writel(gbl->curr_read_lat, SOCFPGA_SDR_ADDRESS + addr); 2604 debug_cond(DLEVEL == 2, "%s:%d lfifo: success: using \ 2605 read_lat=%u\n", __func__, __LINE__, 2606 gbl->curr_read_lat); 2607 return 1; 2608 } else { 2609 set_failing_group_stage(0xff, CAL_STAGE_LFIFO, 2610 CAL_SUBSTAGE_READ_LATENCY); 2611 2612 debug_cond(DLEVEL == 2, "%s:%d lfifo: failed at initial \ 2613 read_lat=%u\n", __func__, __LINE__, 2614 gbl->curr_read_lat); 2615 return 0; 2616 } 2617 } 2618 2619 /* 2620 * issue write test command. 2621 * two variants are provided. one that just tests a write pattern and 2622 * another that tests datamask functionality. 2623 */ 2624 static void rw_mgr_mem_calibrate_write_test_issue(uint32_t group, 2625 uint32_t test_dm) 2626 { 2627 uint32_t mcc_instruction; 2628 uint32_t quick_write_mode = (((STATIC_CALIB_STEPS) & CALIB_SKIP_WRITES) && 2629 ENABLE_SUPER_QUICK_CALIBRATION); 2630 uint32_t rw_wl_nop_cycles; 2631 uint32_t addr; 2632 2633 /* 2634 * Set counter and jump addresses for the right 2635 * number of NOP cycles. 2636 * The number of supported NOP cycles can range from -1 to infinity 2637 * Three different cases are handled: 2638 * 2639 * 1. For a number of NOP cycles greater than 0, the RW Mgr looping 2640 * mechanism will be used to insert the right number of NOPs 2641 * 2642 * 2. For a number of NOP cycles equals to 0, the micro-instruction 2643 * issuing the write command will jump straight to the 2644 * micro-instruction that turns on DQS (for DDRx), or outputs write 2645 * data (for RLD), skipping 2646 * the NOP micro-instruction all together 2647 * 2648 * 3. A number of NOP cycles equal to -1 indicates that DQS must be 2649 * turned on in the same micro-instruction that issues the write 2650 * command. Then we need 2651 * to directly jump to the micro-instruction that sends out the data 2652 * 2653 * NOTE: Implementing this mechanism uses 2 RW Mgr jump-counters 2654 * (2 and 3). One jump-counter (0) is used to perform multiple 2655 * write-read operations. 2656 * one counter left to issue this command in "multiple-group" mode 2657 */ 2658 2659 rw_wl_nop_cycles = gbl->rw_wl_nop_cycles; 2660 2661 if (rw_wl_nop_cycles == -1) { 2662 /* 2663 * CNTR 2 - We want to execute the special write operation that 2664 * turns on DQS right away and then skip directly to the 2665 * instruction that sends out the data. We set the counter to a 2666 * large number so that the jump is always taken. 2667 */ 2668 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr2; 2669 writel(0xFF, SOCFPGA_SDR_ADDRESS + addr); 2670 2671 /* CNTR 3 - Not used */ 2672 if (test_dm) { 2673 mcc_instruction = RW_MGR_LFSR_WR_RD_DM_BANK_0_WL_1; 2674 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add2; 2675 writel(RW_MGR_LFSR_WR_RD_DM_BANK_0_DATA, 2676 SOCFPGA_SDR_ADDRESS + addr); 2677 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add3; 2678 writel(RW_MGR_LFSR_WR_RD_DM_BANK_0_NOP, 2679 SOCFPGA_SDR_ADDRESS + addr); 2680 } else { 2681 mcc_instruction = RW_MGR_LFSR_WR_RD_BANK_0_WL_1; 2682 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add2; 2683 writel(RW_MGR_LFSR_WR_RD_BANK_0_DATA, SOCFPGA_SDR_ADDRESS + addr); 2684 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add3; 2685 writel(RW_MGR_LFSR_WR_RD_BANK_0_NOP, SOCFPGA_SDR_ADDRESS + addr); 2686 } 2687 } else if (rw_wl_nop_cycles == 0) { 2688 /* 2689 * CNTR 2 - We want to skip the NOP operation and go straight 2690 * to the DQS enable instruction. We set the counter to a large 2691 * number so that the jump is always taken. 2692 */ 2693 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr2; 2694 writel(0xFF, SOCFPGA_SDR_ADDRESS + addr); 2695 2696 /* CNTR 3 - Not used */ 2697 if (test_dm) { 2698 mcc_instruction = RW_MGR_LFSR_WR_RD_DM_BANK_0; 2699 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add2; 2700 writel(RW_MGR_LFSR_WR_RD_DM_BANK_0_DQS, 2701 SOCFPGA_SDR_ADDRESS + addr); 2702 } else { 2703 mcc_instruction = RW_MGR_LFSR_WR_RD_BANK_0; 2704 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add2; 2705 writel(RW_MGR_LFSR_WR_RD_BANK_0_DQS, SOCFPGA_SDR_ADDRESS + addr); 2706 } 2707 } else { 2708 /* 2709 * CNTR 2 - In this case we want to execute the next instruction 2710 * and NOT take the jump. So we set the counter to 0. The jump 2711 * address doesn't count. 2712 */ 2713 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr2; 2714 writel(0x0, SOCFPGA_SDR_ADDRESS + addr); 2715 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add2; 2716 writel(0x0, SOCFPGA_SDR_ADDRESS + addr); 2717 2718 /* 2719 * CNTR 3 - Set the nop counter to the number of cycles we 2720 * need to loop for, minus 1. 2721 */ 2722 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr3; 2723 writel(rw_wl_nop_cycles - 1, SOCFPGA_SDR_ADDRESS + addr); 2724 if (test_dm) { 2725 mcc_instruction = RW_MGR_LFSR_WR_RD_DM_BANK_0; 2726 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add3; 2727 writel(RW_MGR_LFSR_WR_RD_DM_BANK_0_NOP, SOCFPGA_SDR_ADDRESS + addr); 2728 } else { 2729 mcc_instruction = RW_MGR_LFSR_WR_RD_BANK_0; 2730 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add3; 2731 writel(RW_MGR_LFSR_WR_RD_BANK_0_NOP, SOCFPGA_SDR_ADDRESS + addr); 2732 } 2733 } 2734 2735 addr = sdr_get_addr((u32 *)RW_MGR_RESET_READ_DATAPATH); 2736 writel(0, SOCFPGA_SDR_ADDRESS + addr); 2737 2738 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr0; 2739 if (quick_write_mode) 2740 writel(0x08, SOCFPGA_SDR_ADDRESS + addr); 2741 else 2742 writel(0x40, SOCFPGA_SDR_ADDRESS + addr); 2743 2744 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add0; 2745 writel(mcc_instruction, SOCFPGA_SDR_ADDRESS + addr); 2746 2747 /* 2748 * CNTR 1 - This is used to ensure enough time elapses 2749 * for read data to come back. 2750 */ 2751 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr1; 2752 writel(0x30, SOCFPGA_SDR_ADDRESS + addr); 2753 2754 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add1; 2755 if (test_dm) { 2756 writel(RW_MGR_LFSR_WR_RD_DM_BANK_0_WAIT, SOCFPGA_SDR_ADDRESS + addr); 2757 } else { 2758 writel(RW_MGR_LFSR_WR_RD_BANK_0_WAIT, SOCFPGA_SDR_ADDRESS + addr); 2759 } 2760 2761 addr = sdr_get_addr((u32 *)RW_MGR_RUN_SINGLE_GROUP); 2762 writel(mcc_instruction, SOCFPGA_SDR_ADDRESS + addr + (group << 2)); 2763 } 2764 2765 /* Test writes, can check for a single bit pass or multiple bit pass */ 2766 static uint32_t rw_mgr_mem_calibrate_write_test(uint32_t rank_bgn, 2767 uint32_t write_group, uint32_t use_dm, uint32_t all_correct, 2768 uint32_t *bit_chk, uint32_t all_ranks) 2769 { 2770 uint32_t addr; 2771 uint32_t r; 2772 uint32_t correct_mask_vg; 2773 uint32_t tmp_bit_chk; 2774 uint32_t vg; 2775 uint32_t rank_end = all_ranks ? RW_MGR_MEM_NUMBER_OF_RANKS : 2776 (rank_bgn + NUM_RANKS_PER_SHADOW_REG); 2777 uint32_t addr_rw_mgr; 2778 uint32_t base_rw_mgr; 2779 2780 *bit_chk = param->write_correct_mask; 2781 correct_mask_vg = param->write_correct_mask_vg; 2782 2783 for (r = rank_bgn; r < rank_end; r++) { 2784 if (param->skip_ranks[r]) { 2785 /* request to skip the rank */ 2786 continue; 2787 } 2788 2789 /* set rank */ 2790 set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_READ_WRITE); 2791 2792 tmp_bit_chk = 0; 2793 addr = sdr_get_addr(&phy_mgr_cmd->fifo_reset); 2794 addr_rw_mgr = SDR_PHYGRP_RWMGRGRP_ADDRESS; 2795 for (vg = RW_MGR_MEM_VIRTUAL_GROUPS_PER_WRITE_DQS-1; ; vg--) { 2796 /* reset the fifos to get pointers to known state */ 2797 writel(0, SOCFPGA_SDR_ADDRESS + addr); 2798 2799 tmp_bit_chk = tmp_bit_chk << 2800 (RW_MGR_MEM_DQ_PER_WRITE_DQS / 2801 RW_MGR_MEM_VIRTUAL_GROUPS_PER_WRITE_DQS); 2802 rw_mgr_mem_calibrate_write_test_issue(write_group * 2803 RW_MGR_MEM_VIRTUAL_GROUPS_PER_WRITE_DQS+vg, 2804 use_dm); 2805 2806 base_rw_mgr = readl(SOCFPGA_SDR_ADDRESS + addr_rw_mgr); 2807 tmp_bit_chk = tmp_bit_chk | (correct_mask_vg & ~(base_rw_mgr)); 2808 if (vg == 0) 2809 break; 2810 } 2811 *bit_chk &= tmp_bit_chk; 2812 } 2813 2814 if (all_correct) { 2815 set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF); 2816 debug_cond(DLEVEL == 2, "write_test(%u,%u,ALL) : %u == \ 2817 %u => %lu", write_group, use_dm, 2818 *bit_chk, param->write_correct_mask, 2819 (long unsigned int)(*bit_chk == 2820 param->write_correct_mask)); 2821 return *bit_chk == param->write_correct_mask; 2822 } else { 2823 set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF); 2824 debug_cond(DLEVEL == 2, "write_test(%u,%u,ONE) : %u != ", 2825 write_group, use_dm, *bit_chk); 2826 debug_cond(DLEVEL == 2, "%lu" " => %lu", (long unsigned int)0, 2827 (long unsigned int)(*bit_chk != 0)); 2828 return *bit_chk != 0x00; 2829 } 2830 } 2831 2832 /* 2833 * center all windows. do per-bit-deskew to possibly increase size of 2834 * certain windows. 2835 */ 2836 static uint32_t rw_mgr_mem_calibrate_writes_center(uint32_t rank_bgn, 2837 uint32_t write_group, uint32_t test_bgn) 2838 { 2839 uint32_t i, p, min_index; 2840 int32_t d; 2841 /* 2842 * Store these as signed since there are comparisons with 2843 * signed numbers. 2844 */ 2845 uint32_t bit_chk; 2846 uint32_t sticky_bit_chk; 2847 int32_t left_edge[RW_MGR_MEM_DQ_PER_WRITE_DQS]; 2848 int32_t right_edge[RW_MGR_MEM_DQ_PER_WRITE_DQS]; 2849 int32_t mid; 2850 int32_t mid_min, orig_mid_min; 2851 int32_t new_dqs, start_dqs, shift_dq; 2852 int32_t dq_margin, dqs_margin, dm_margin; 2853 uint32_t stop; 2854 uint32_t temp_dq_out1_delay; 2855 uint32_t addr; 2856 2857 debug("%s:%d %u %u", __func__, __LINE__, write_group, test_bgn); 2858 2859 dm_margin = 0; 2860 2861 addr = sdr_get_addr((u32 *)SCC_MGR_IO_OUT1_DELAY); 2862 start_dqs = readl(SOCFPGA_SDR_ADDRESS + addr + 2863 (RW_MGR_MEM_DQ_PER_WRITE_DQS << 2)); 2864 2865 /* per-bit deskew */ 2866 2867 /* 2868 * set the left and right edge of each bit to an illegal value 2869 * use (IO_IO_OUT1_DELAY_MAX + 1) as an illegal value. 2870 */ 2871 sticky_bit_chk = 0; 2872 for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) { 2873 left_edge[i] = IO_IO_OUT1_DELAY_MAX + 1; 2874 right_edge[i] = IO_IO_OUT1_DELAY_MAX + 1; 2875 } 2876 2877 /* Search for the left edge of the window for each bit */ 2878 addr = (u32)&sdr_scc_mgr->update; 2879 for (d = 0; d <= IO_IO_OUT1_DELAY_MAX; d++) { 2880 scc_mgr_apply_group_dq_out1_delay(write_group, test_bgn, d); 2881 2882 writel(0, SOCFPGA_SDR_ADDRESS + addr); 2883 2884 /* 2885 * Stop searching when the read test doesn't pass AND when 2886 * we've seen a passing read on every bit. 2887 */ 2888 stop = !rw_mgr_mem_calibrate_write_test(rank_bgn, write_group, 2889 0, PASS_ONE_BIT, &bit_chk, 0); 2890 sticky_bit_chk = sticky_bit_chk | bit_chk; 2891 stop = stop && (sticky_bit_chk == param->write_correct_mask); 2892 debug_cond(DLEVEL == 2, "write_center(left): dtap=%d => %u \ 2893 == %u && %u [bit_chk= %u ]\n", 2894 d, sticky_bit_chk, param->write_correct_mask, 2895 stop, bit_chk); 2896 2897 if (stop == 1) { 2898 break; 2899 } else { 2900 for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) { 2901 if (bit_chk & 1) { 2902 /* 2903 * Remember a passing test as the 2904 * left_edge. 2905 */ 2906 left_edge[i] = d; 2907 } else { 2908 /* 2909 * If a left edge has not been seen 2910 * yet, then a future passing test will 2911 * mark this edge as the right edge. 2912 */ 2913 if (left_edge[i] == 2914 IO_IO_OUT1_DELAY_MAX + 1) { 2915 right_edge[i] = -(d + 1); 2916 } 2917 } 2918 debug_cond(DLEVEL == 2, "write_center[l,d=%d):", d); 2919 debug_cond(DLEVEL == 2, "bit_chk_test=%d left_edge[%u]: %d", 2920 (int)(bit_chk & 1), i, left_edge[i]); 2921 debug_cond(DLEVEL == 2, "right_edge[%u]: %d\n", i, 2922 right_edge[i]); 2923 bit_chk = bit_chk >> 1; 2924 } 2925 } 2926 } 2927 2928 /* Reset DQ delay chains to 0 */ 2929 scc_mgr_apply_group_dq_out1_delay(write_group, test_bgn, 0); 2930 sticky_bit_chk = 0; 2931 for (i = RW_MGR_MEM_DQ_PER_WRITE_DQS - 1;; i--) { 2932 debug_cond(DLEVEL == 2, "%s:%d write_center: left_edge[%u]: \ 2933 %d right_edge[%u]: %d\n", __func__, __LINE__, 2934 i, left_edge[i], i, right_edge[i]); 2935 2936 /* 2937 * Check for cases where we haven't found the left edge, 2938 * which makes our assignment of the the right edge invalid. 2939 * Reset it to the illegal value. 2940 */ 2941 if ((left_edge[i] == IO_IO_OUT1_DELAY_MAX + 1) && 2942 (right_edge[i] != IO_IO_OUT1_DELAY_MAX + 1)) { 2943 right_edge[i] = IO_IO_OUT1_DELAY_MAX + 1; 2944 debug_cond(DLEVEL == 2, "%s:%d write_center: reset \ 2945 right_edge[%u]: %d\n", __func__, __LINE__, 2946 i, right_edge[i]); 2947 } 2948 2949 /* 2950 * Reset sticky bit (except for bits where we have 2951 * seen the left edge). 2952 */ 2953 sticky_bit_chk = sticky_bit_chk << 1; 2954 if ((left_edge[i] != IO_IO_OUT1_DELAY_MAX + 1)) 2955 sticky_bit_chk = sticky_bit_chk | 1; 2956 2957 if (i == 0) 2958 break; 2959 } 2960 2961 /* Search for the right edge of the window for each bit */ 2962 addr = (u32)&sdr_scc_mgr->update; 2963 for (d = 0; d <= IO_IO_OUT1_DELAY_MAX - start_dqs; d++) { 2964 scc_mgr_apply_group_dqs_io_and_oct_out1(write_group, 2965 d + start_dqs); 2966 2967 writel(0, SOCFPGA_SDR_ADDRESS + addr); 2968 2969 /* 2970 * Stop searching when the read test doesn't pass AND when 2971 * we've seen a passing read on every bit. 2972 */ 2973 stop = !rw_mgr_mem_calibrate_write_test(rank_bgn, write_group, 2974 0, PASS_ONE_BIT, &bit_chk, 0); 2975 2976 sticky_bit_chk = sticky_bit_chk | bit_chk; 2977 stop = stop && (sticky_bit_chk == param->write_correct_mask); 2978 2979 debug_cond(DLEVEL == 2, "write_center (right): dtap=%u => %u == \ 2980 %u && %u\n", d, sticky_bit_chk, 2981 param->write_correct_mask, stop); 2982 2983 if (stop == 1) { 2984 if (d == 0) { 2985 for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; 2986 i++) { 2987 /* d = 0 failed, but it passed when 2988 testing the left edge, so it must be 2989 marginal, set it to -1 */ 2990 if (right_edge[i] == 2991 IO_IO_OUT1_DELAY_MAX + 1 && 2992 left_edge[i] != 2993 IO_IO_OUT1_DELAY_MAX + 1) { 2994 right_edge[i] = -1; 2995 } 2996 } 2997 } 2998 break; 2999 } else { 3000 for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) { 3001 if (bit_chk & 1) { 3002 /* 3003 * Remember a passing test as 3004 * the right_edge. 3005 */ 3006 right_edge[i] = d; 3007 } else { 3008 if (d != 0) { 3009 /* 3010 * If a right edge has not 3011 * been seen yet, then a future 3012 * passing test will mark this 3013 * edge as the left edge. 3014 */ 3015 if (right_edge[i] == 3016 IO_IO_OUT1_DELAY_MAX + 1) 3017 left_edge[i] = -(d + 1); 3018 } else { 3019 /* 3020 * d = 0 failed, but it passed 3021 * when testing the left edge, 3022 * so it must be marginal, set 3023 * it to -1. 3024 */ 3025 if (right_edge[i] == 3026 IO_IO_OUT1_DELAY_MAX + 1 && 3027 left_edge[i] != 3028 IO_IO_OUT1_DELAY_MAX + 1) 3029 right_edge[i] = -1; 3030 /* 3031 * If a right edge has not been 3032 * seen yet, then a future 3033 * passing test will mark this 3034 * edge as the left edge. 3035 */ 3036 else if (right_edge[i] == 3037 IO_IO_OUT1_DELAY_MAX + 3038 1) 3039 left_edge[i] = -(d + 1); 3040 } 3041 } 3042 debug_cond(DLEVEL == 2, "write_center[r,d=%d):", d); 3043 debug_cond(DLEVEL == 2, "bit_chk_test=%d left_edge[%u]: %d", 3044 (int)(bit_chk & 1), i, left_edge[i]); 3045 debug_cond(DLEVEL == 2, "right_edge[%u]: %d\n", i, 3046 right_edge[i]); 3047 bit_chk = bit_chk >> 1; 3048 } 3049 } 3050 } 3051 3052 /* Check that all bits have a window */ 3053 for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) { 3054 debug_cond(DLEVEL == 2, "%s:%d write_center: left_edge[%u]: \ 3055 %d right_edge[%u]: %d", __func__, __LINE__, 3056 i, left_edge[i], i, right_edge[i]); 3057 if ((left_edge[i] == IO_IO_OUT1_DELAY_MAX + 1) || 3058 (right_edge[i] == IO_IO_OUT1_DELAY_MAX + 1)) { 3059 set_failing_group_stage(test_bgn + i, 3060 CAL_STAGE_WRITES, 3061 CAL_SUBSTAGE_WRITES_CENTER); 3062 return 0; 3063 } 3064 } 3065 3066 /* Find middle of window for each DQ bit */ 3067 mid_min = left_edge[0] - right_edge[0]; 3068 min_index = 0; 3069 for (i = 1; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) { 3070 mid = left_edge[i] - right_edge[i]; 3071 if (mid < mid_min) { 3072 mid_min = mid; 3073 min_index = i; 3074 } 3075 } 3076 3077 /* 3078 * -mid_min/2 represents the amount that we need to move DQS. 3079 * If mid_min is odd and positive we'll need to add one to 3080 * make sure the rounding in further calculations is correct 3081 * (always bias to the right), so just add 1 for all positive values. 3082 */ 3083 if (mid_min > 0) 3084 mid_min++; 3085 mid_min = mid_min / 2; 3086 debug_cond(DLEVEL == 1, "%s:%d write_center: mid_min=%d\n", __func__, 3087 __LINE__, mid_min); 3088 3089 /* Determine the amount we can change DQS (which is -mid_min) */ 3090 orig_mid_min = mid_min; 3091 new_dqs = start_dqs; 3092 mid_min = 0; 3093 debug_cond(DLEVEL == 1, "%s:%d write_center: start_dqs=%d new_dqs=%d \ 3094 mid_min=%d\n", __func__, __LINE__, start_dqs, new_dqs, mid_min); 3095 /* Initialize data for export structures */ 3096 dqs_margin = IO_IO_OUT1_DELAY_MAX + 1; 3097 dq_margin = IO_IO_OUT1_DELAY_MAX + 1; 3098 3099 /* add delay to bring centre of all DQ windows to the same "level" */ 3100 addr = sdr_get_addr((u32 *)SCC_MGR_IO_OUT1_DELAY); 3101 for (i = 0, p = test_bgn; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++, p++) { 3102 /* Use values before divide by 2 to reduce round off error */ 3103 shift_dq = (left_edge[i] - right_edge[i] - 3104 (left_edge[min_index] - right_edge[min_index]))/2 + 3105 (orig_mid_min - mid_min); 3106 3107 debug_cond(DLEVEL == 2, "%s:%d write_center: before: shift_dq \ 3108 [%u]=%d\n", __func__, __LINE__, i, shift_dq); 3109 3110 temp_dq_out1_delay = readl(SOCFPGA_SDR_ADDRESS + addr + (i << 2)); 3111 if (shift_dq + (int32_t)temp_dq_out1_delay > 3112 (int32_t)IO_IO_OUT1_DELAY_MAX) { 3113 shift_dq = (int32_t)IO_IO_OUT1_DELAY_MAX - temp_dq_out1_delay; 3114 } else if (shift_dq + (int32_t)temp_dq_out1_delay < 0) { 3115 shift_dq = -(int32_t)temp_dq_out1_delay; 3116 } 3117 debug_cond(DLEVEL == 2, "write_center: after: shift_dq[%u]=%d\n", 3118 i, shift_dq); 3119 scc_mgr_set_dq_out1_delay(write_group, i, temp_dq_out1_delay + 3120 shift_dq); 3121 scc_mgr_load_dq(i); 3122 3123 debug_cond(DLEVEL == 2, "write_center: margin[%u]=[%d,%d]\n", i, 3124 left_edge[i] - shift_dq + (-mid_min), 3125 right_edge[i] + shift_dq - (-mid_min)); 3126 /* To determine values for export structures */ 3127 if (left_edge[i] - shift_dq + (-mid_min) < dq_margin) 3128 dq_margin = left_edge[i] - shift_dq + (-mid_min); 3129 3130 if (right_edge[i] + shift_dq - (-mid_min) < dqs_margin) 3131 dqs_margin = right_edge[i] + shift_dq - (-mid_min); 3132 } 3133 3134 /* Move DQS */ 3135 scc_mgr_apply_group_dqs_io_and_oct_out1(write_group, new_dqs); 3136 addr = (u32)&sdr_scc_mgr->update; 3137 writel(0, SOCFPGA_SDR_ADDRESS + addr); 3138 3139 /* Centre DM */ 3140 debug_cond(DLEVEL == 2, "%s:%d write_center: DM\n", __func__, __LINE__); 3141 3142 /* 3143 * set the left and right edge of each bit to an illegal value, 3144 * use (IO_IO_OUT1_DELAY_MAX + 1) as an illegal value, 3145 */ 3146 left_edge[0] = IO_IO_OUT1_DELAY_MAX + 1; 3147 right_edge[0] = IO_IO_OUT1_DELAY_MAX + 1; 3148 int32_t bgn_curr = IO_IO_OUT1_DELAY_MAX + 1; 3149 int32_t end_curr = IO_IO_OUT1_DELAY_MAX + 1; 3150 int32_t bgn_best = IO_IO_OUT1_DELAY_MAX + 1; 3151 int32_t end_best = IO_IO_OUT1_DELAY_MAX + 1; 3152 int32_t win_best = 0; 3153 3154 /* Search for the/part of the window with DM shift */ 3155 addr = (u32)&sdr_scc_mgr->update; 3156 for (d = IO_IO_OUT1_DELAY_MAX; d >= 0; d -= DELTA_D) { 3157 scc_mgr_apply_group_dm_out1_delay(write_group, d); 3158 writel(0, SOCFPGA_SDR_ADDRESS + addr); 3159 3160 if (rw_mgr_mem_calibrate_write_test(rank_bgn, write_group, 1, 3161 PASS_ALL_BITS, &bit_chk, 3162 0)) { 3163 /* USE Set current end of the window */ 3164 end_curr = -d; 3165 /* 3166 * If a starting edge of our window has not been seen 3167 * this is our current start of the DM window. 3168 */ 3169 if (bgn_curr == IO_IO_OUT1_DELAY_MAX + 1) 3170 bgn_curr = -d; 3171 3172 /* 3173 * If current window is bigger than best seen. 3174 * Set best seen to be current window. 3175 */ 3176 if ((end_curr-bgn_curr+1) > win_best) { 3177 win_best = end_curr-bgn_curr+1; 3178 bgn_best = bgn_curr; 3179 end_best = end_curr; 3180 } 3181 } else { 3182 /* We just saw a failing test. Reset temp edge */ 3183 bgn_curr = IO_IO_OUT1_DELAY_MAX + 1; 3184 end_curr = IO_IO_OUT1_DELAY_MAX + 1; 3185 } 3186 } 3187 3188 3189 /* Reset DM delay chains to 0 */ 3190 scc_mgr_apply_group_dm_out1_delay(write_group, 0); 3191 3192 /* 3193 * Check to see if the current window nudges up aganist 0 delay. 3194 * If so we need to continue the search by shifting DQS otherwise DQS 3195 * search begins as a new search. */ 3196 if (end_curr != 0) { 3197 bgn_curr = IO_IO_OUT1_DELAY_MAX + 1; 3198 end_curr = IO_IO_OUT1_DELAY_MAX + 1; 3199 } 3200 3201 /* Search for the/part of the window with DQS shifts */ 3202 addr = (u32)&sdr_scc_mgr->update; 3203 for (d = 0; d <= IO_IO_OUT1_DELAY_MAX - new_dqs; d += DELTA_D) { 3204 /* 3205 * Note: This only shifts DQS, so are we limiting ourselve to 3206 * width of DQ unnecessarily. 3207 */ 3208 scc_mgr_apply_group_dqs_io_and_oct_out1(write_group, 3209 d + new_dqs); 3210 3211 writel(0, SOCFPGA_SDR_ADDRESS + addr); 3212 if (rw_mgr_mem_calibrate_write_test(rank_bgn, write_group, 1, 3213 PASS_ALL_BITS, &bit_chk, 3214 0)) { 3215 /* USE Set current end of the window */ 3216 end_curr = d; 3217 /* 3218 * If a beginning edge of our window has not been seen 3219 * this is our current begin of the DM window. 3220 */ 3221 if (bgn_curr == IO_IO_OUT1_DELAY_MAX + 1) 3222 bgn_curr = d; 3223 3224 /* 3225 * If current window is bigger than best seen. Set best 3226 * seen to be current window. 3227 */ 3228 if ((end_curr-bgn_curr+1) > win_best) { 3229 win_best = end_curr-bgn_curr+1; 3230 bgn_best = bgn_curr; 3231 end_best = end_curr; 3232 } 3233 } else { 3234 /* We just saw a failing test. Reset temp edge */ 3235 bgn_curr = IO_IO_OUT1_DELAY_MAX + 1; 3236 end_curr = IO_IO_OUT1_DELAY_MAX + 1; 3237 3238 /* Early exit optimization: if ther remaining delay 3239 chain space is less than already seen largest window 3240 we can exit */ 3241 if ((win_best-1) > 3242 (IO_IO_OUT1_DELAY_MAX - new_dqs - d)) { 3243 break; 3244 } 3245 } 3246 } 3247 3248 /* assign left and right edge for cal and reporting; */ 3249 left_edge[0] = -1*bgn_best; 3250 right_edge[0] = end_best; 3251 3252 debug_cond(DLEVEL == 2, "%s:%d dm_calib: left=%d right=%d\n", __func__, 3253 __LINE__, left_edge[0], right_edge[0]); 3254 3255 /* Move DQS (back to orig) */ 3256 scc_mgr_apply_group_dqs_io_and_oct_out1(write_group, new_dqs); 3257 3258 /* Move DM */ 3259 3260 /* Find middle of window for the DM bit */ 3261 mid = (left_edge[0] - right_edge[0]) / 2; 3262 3263 /* only move right, since we are not moving DQS/DQ */ 3264 if (mid < 0) 3265 mid = 0; 3266 3267 /* dm_marign should fail if we never find a window */ 3268 if (win_best == 0) 3269 dm_margin = -1; 3270 else 3271 dm_margin = left_edge[0] - mid; 3272 3273 scc_mgr_apply_group_dm_out1_delay(write_group, mid); 3274 addr = (u32)&sdr_scc_mgr->update; 3275 writel(0, SOCFPGA_SDR_ADDRESS + addr); 3276 3277 debug_cond(DLEVEL == 2, "%s:%d dm_calib: left=%d right=%d mid=%d \ 3278 dm_margin=%d\n", __func__, __LINE__, left_edge[0], 3279 right_edge[0], mid, dm_margin); 3280 /* Export values */ 3281 gbl->fom_out += dq_margin + dqs_margin; 3282 3283 debug_cond(DLEVEL == 2, "%s:%d write_center: dq_margin=%d \ 3284 dqs_margin=%d dm_margin=%d\n", __func__, __LINE__, 3285 dq_margin, dqs_margin, dm_margin); 3286 3287 /* 3288 * Do not remove this line as it makes sure all of our 3289 * decisions have been applied. 3290 */ 3291 addr = (u32)&sdr_scc_mgr->update; 3292 writel(0, SOCFPGA_SDR_ADDRESS + addr); 3293 return (dq_margin >= 0) && (dqs_margin >= 0) && (dm_margin >= 0); 3294 } 3295 3296 /* calibrate the write operations */ 3297 static uint32_t rw_mgr_mem_calibrate_writes(uint32_t rank_bgn, uint32_t g, 3298 uint32_t test_bgn) 3299 { 3300 /* update info for sims */ 3301 debug("%s:%d %u %u\n", __func__, __LINE__, g, test_bgn); 3302 3303 reg_file_set_stage(CAL_STAGE_WRITES); 3304 reg_file_set_sub_stage(CAL_SUBSTAGE_WRITES_CENTER); 3305 3306 reg_file_set_group(g); 3307 3308 if (!rw_mgr_mem_calibrate_writes_center(rank_bgn, g, test_bgn)) { 3309 set_failing_group_stage(g, CAL_STAGE_WRITES, 3310 CAL_SUBSTAGE_WRITES_CENTER); 3311 return 0; 3312 } 3313 3314 return 1; 3315 } 3316 3317 /* precharge all banks and activate row 0 in bank "000..." and bank "111..." */ 3318 static void mem_precharge_and_activate(void) 3319 { 3320 uint32_t r; 3321 uint32_t addr; 3322 3323 for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; r++) { 3324 if (param->skip_ranks[r]) { 3325 /* request to skip the rank */ 3326 continue; 3327 } 3328 3329 /* set rank */ 3330 set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_OFF); 3331 3332 /* precharge all banks ... */ 3333 addr = sdr_get_addr((u32 *)RW_MGR_RUN_SINGLE_GROUP); 3334 writel(RW_MGR_PRECHARGE_ALL, SOCFPGA_SDR_ADDRESS + addr); 3335 3336 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr0; 3337 writel(0x0F, SOCFPGA_SDR_ADDRESS + addr); 3338 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add0; 3339 writel(RW_MGR_ACTIVATE_0_AND_1_WAIT1, SOCFPGA_SDR_ADDRESS + addr); 3340 3341 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr1; 3342 writel(0x0F, SOCFPGA_SDR_ADDRESS + addr); 3343 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add1; 3344 writel(RW_MGR_ACTIVATE_0_AND_1_WAIT2, SOCFPGA_SDR_ADDRESS + addr); 3345 3346 /* activate rows */ 3347 addr = sdr_get_addr((u32 *)RW_MGR_RUN_SINGLE_GROUP); 3348 writel(RW_MGR_ACTIVATE_0_AND_1, SOCFPGA_SDR_ADDRESS + addr); 3349 } 3350 } 3351 3352 /* Configure various memory related parameters. */ 3353 static void mem_config(void) 3354 { 3355 uint32_t rlat, wlat; 3356 uint32_t rw_wl_nop_cycles; 3357 uint32_t max_latency; 3358 uint32_t addr; 3359 3360 debug("%s:%d\n", __func__, __LINE__); 3361 /* read in write and read latency */ 3362 addr = sdr_get_addr(&data_mgr->t_wl_add); 3363 wlat = readl(SOCFPGA_SDR_ADDRESS + addr); 3364 3365 addr = sdr_get_addr(&data_mgr->mem_t_add); 3366 wlat += readl(SOCFPGA_SDR_ADDRESS + addr); 3367 /* WL for hard phy does not include additive latency */ 3368 3369 /* 3370 * add addtional write latency to offset the address/command extra 3371 * clock cycle. We change the AC mux setting causing AC to be delayed 3372 * by one mem clock cycle. Only do this for DDR3 3373 */ 3374 wlat = wlat + 1; 3375 3376 addr = sdr_get_addr(&data_mgr->t_rl_add); 3377 rlat = readl(SOCFPGA_SDR_ADDRESS + addr); 3378 3379 rw_wl_nop_cycles = wlat - 2; 3380 gbl->rw_wl_nop_cycles = rw_wl_nop_cycles; 3381 3382 /* 3383 * For AV/CV, lfifo is hardened and always runs at full rate so 3384 * max latency in AFI clocks, used here, is correspondingly smaller. 3385 */ 3386 max_latency = (1<<MAX_LATENCY_COUNT_WIDTH)/1 - 1; 3387 /* configure for a burst length of 8 */ 3388 3389 /* write latency */ 3390 /* Adjust Write Latency for Hard PHY */ 3391 wlat = wlat + 1; 3392 3393 /* set a pretty high read latency initially */ 3394 gbl->curr_read_lat = rlat + 16; 3395 3396 if (gbl->curr_read_lat > max_latency) 3397 gbl->curr_read_lat = max_latency; 3398 3399 addr = sdr_get_addr(&phy_mgr_cfg->phy_rlat); 3400 writel(gbl->curr_read_lat, SOCFPGA_SDR_ADDRESS + addr); 3401 3402 /* advertise write latency */ 3403 gbl->curr_write_lat = wlat; 3404 addr = sdr_get_addr(&phy_mgr_cfg->afi_wlat); 3405 writel(wlat - 2, SOCFPGA_SDR_ADDRESS + addr); 3406 3407 /* initialize bit slips */ 3408 mem_precharge_and_activate(); 3409 } 3410 3411 /* Set VFIFO and LFIFO to instant-on settings in skip calibration mode */ 3412 static void mem_skip_calibrate(void) 3413 { 3414 uint32_t vfifo_offset; 3415 uint32_t i, j, r; 3416 uint32_t addr; 3417 3418 debug("%s:%d\n", __func__, __LINE__); 3419 /* Need to update every shadow register set used by the interface */ 3420 for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; 3421 r += NUM_RANKS_PER_SHADOW_REG) { 3422 /* 3423 * Set output phase alignment settings appropriate for 3424 * skip calibration. 3425 */ 3426 for (i = 0; i < RW_MGR_MEM_IF_READ_DQS_WIDTH; i++) { 3427 scc_mgr_set_dqs_en_phase(i, 0); 3428 #if IO_DLL_CHAIN_LENGTH == 6 3429 scc_mgr_set_dqdqs_output_phase(i, 6); 3430 #else 3431 scc_mgr_set_dqdqs_output_phase(i, 7); 3432 #endif 3433 /* 3434 * Case:33398 3435 * 3436 * Write data arrives to the I/O two cycles before write 3437 * latency is reached (720 deg). 3438 * -> due to bit-slip in a/c bus 3439 * -> to allow board skew where dqs is longer than ck 3440 * -> how often can this happen!? 3441 * -> can claim back some ptaps for high freq 3442 * support if we can relax this, but i digress... 3443 * 3444 * The write_clk leads mem_ck by 90 deg 3445 * The minimum ptap of the OPA is 180 deg 3446 * Each ptap has (360 / IO_DLL_CHAIN_LENGH) deg of delay 3447 * The write_clk is always delayed by 2 ptaps 3448 * 3449 * Hence, to make DQS aligned to CK, we need to delay 3450 * DQS by: 3451 * (720 - 90 - 180 - 2 * (360 / IO_DLL_CHAIN_LENGTH)) 3452 * 3453 * Dividing the above by (360 / IO_DLL_CHAIN_LENGTH) 3454 * gives us the number of ptaps, which simplies to: 3455 * 3456 * (1.25 * IO_DLL_CHAIN_LENGTH - 2) 3457 */ 3458 scc_mgr_set_dqdqs_output_phase(i, (1.25 * 3459 IO_DLL_CHAIN_LENGTH - 2)); 3460 } 3461 addr = (u32)&sdr_scc_mgr->dqs_ena; 3462 writel(0xff, SOCFPGA_SDR_ADDRESS + addr); 3463 addr = (u32)&sdr_scc_mgr->dqs_io_ena; 3464 writel(0xff, SOCFPGA_SDR_ADDRESS + addr); 3465 3466 addr = sdr_get_addr((u32 *)SCC_MGR_GROUP_COUNTER); 3467 for (i = 0; i < RW_MGR_MEM_IF_WRITE_DQS_WIDTH; i++) { 3468 writel(i, SOCFPGA_SDR_ADDRESS + addr); 3469 } 3470 addr = (u32)&sdr_scc_mgr->dq_ena; 3471 writel(0xff, SOCFPGA_SDR_ADDRESS + addr); 3472 addr = (u32)&sdr_scc_mgr->dm_ena; 3473 writel(0xff, SOCFPGA_SDR_ADDRESS + addr); 3474 addr = (u32)&sdr_scc_mgr->update; 3475 writel(0, SOCFPGA_SDR_ADDRESS + addr); 3476 } 3477 3478 /* Compensate for simulation model behaviour */ 3479 for (i = 0; i < RW_MGR_MEM_IF_READ_DQS_WIDTH; i++) { 3480 scc_mgr_set_dqs_bus_in_delay(i, 10); 3481 scc_mgr_load_dqs(i); 3482 } 3483 addr = (u32)&sdr_scc_mgr->update; 3484 writel(0, SOCFPGA_SDR_ADDRESS + addr); 3485 3486 /* 3487 * ArriaV has hard FIFOs that can only be initialized by incrementing 3488 * in sequencer. 3489 */ 3490 vfifo_offset = CALIB_VFIFO_OFFSET; 3491 addr = sdr_get_addr(&phy_mgr_cmd->inc_vfifo_hard_phy); 3492 for (j = 0; j < vfifo_offset; j++) { 3493 writel(0xff, SOCFPGA_SDR_ADDRESS + addr); 3494 } 3495 addr = sdr_get_addr(&phy_mgr_cmd->fifo_reset); 3496 writel(0, SOCFPGA_SDR_ADDRESS + addr); 3497 3498 /* 3499 * For ACV with hard lfifo, we get the skip-cal setting from 3500 * generation-time constant. 3501 */ 3502 gbl->curr_read_lat = CALIB_LFIFO_OFFSET; 3503 addr = sdr_get_addr(&phy_mgr_cfg->phy_rlat); 3504 writel(gbl->curr_read_lat, SOCFPGA_SDR_ADDRESS + addr); 3505 } 3506 3507 /* Memory calibration entry point */ 3508 static uint32_t mem_calibrate(void) 3509 { 3510 uint32_t i; 3511 uint32_t rank_bgn, sr; 3512 uint32_t write_group, write_test_bgn; 3513 uint32_t read_group, read_test_bgn; 3514 uint32_t run_groups, current_run; 3515 uint32_t failing_groups = 0; 3516 uint32_t group_failed = 0; 3517 uint32_t sr_failed = 0; 3518 uint32_t addr; 3519 3520 debug("%s:%d\n", __func__, __LINE__); 3521 /* Initialize the data settings */ 3522 3523 gbl->error_substage = CAL_SUBSTAGE_NIL; 3524 gbl->error_stage = CAL_STAGE_NIL; 3525 gbl->error_group = 0xff; 3526 gbl->fom_in = 0; 3527 gbl->fom_out = 0; 3528 3529 mem_config(); 3530 3531 uint32_t bypass_mode = 0x1; 3532 addr = sdr_get_addr((u32 *)SCC_MGR_GROUP_COUNTER); 3533 for (i = 0; i < RW_MGR_MEM_IF_READ_DQS_WIDTH; i++) { 3534 writel(i, SOCFPGA_SDR_ADDRESS + addr); 3535 scc_set_bypass_mode(i, bypass_mode); 3536 } 3537 3538 if ((dyn_calib_steps & CALIB_SKIP_ALL) == CALIB_SKIP_ALL) { 3539 /* 3540 * Set VFIFO and LFIFO to instant-on settings in skip 3541 * calibration mode. 3542 */ 3543 mem_skip_calibrate(); 3544 } else { 3545 for (i = 0; i < NUM_CALIB_REPEAT; i++) { 3546 /* 3547 * Zero all delay chain/phase settings for all 3548 * groups and all shadow register sets. 3549 */ 3550 scc_mgr_zero_all(); 3551 3552 run_groups = ~param->skip_groups; 3553 3554 for (write_group = 0, write_test_bgn = 0; write_group 3555 < RW_MGR_MEM_IF_WRITE_DQS_WIDTH; write_group++, 3556 write_test_bgn += RW_MGR_MEM_DQ_PER_WRITE_DQS) { 3557 /* Initialized the group failure */ 3558 group_failed = 0; 3559 3560 current_run = run_groups & ((1 << 3561 RW_MGR_NUM_DQS_PER_WRITE_GROUP) - 1); 3562 run_groups = run_groups >> 3563 RW_MGR_NUM_DQS_PER_WRITE_GROUP; 3564 3565 if (current_run == 0) 3566 continue; 3567 3568 addr = sdr_get_addr((u32 *)SCC_MGR_GROUP_COUNTER); 3569 writel(write_group, SOCFPGA_SDR_ADDRESS + addr); 3570 scc_mgr_zero_group(write_group, write_test_bgn, 3571 0); 3572 3573 for (read_group = write_group * 3574 RW_MGR_MEM_IF_READ_DQS_WIDTH / 3575 RW_MGR_MEM_IF_WRITE_DQS_WIDTH, 3576 read_test_bgn = 0; 3577 read_group < (write_group + 1) * 3578 RW_MGR_MEM_IF_READ_DQS_WIDTH / 3579 RW_MGR_MEM_IF_WRITE_DQS_WIDTH && 3580 group_failed == 0; 3581 read_group++, read_test_bgn += 3582 RW_MGR_MEM_DQ_PER_READ_DQS) { 3583 /* Calibrate the VFIFO */ 3584 if (!((STATIC_CALIB_STEPS) & 3585 CALIB_SKIP_VFIFO)) { 3586 if (!rw_mgr_mem_calibrate_vfifo 3587 (read_group, 3588 read_test_bgn)) { 3589 group_failed = 1; 3590 3591 if (!(gbl-> 3592 phy_debug_mode_flags & 3593 PHY_DEBUG_SWEEP_ALL_GROUPS)) { 3594 return 0; 3595 } 3596 } 3597 } 3598 } 3599 3600 /* Calibrate the output side */ 3601 if (group_failed == 0) { 3602 for (rank_bgn = 0, sr = 0; rank_bgn 3603 < RW_MGR_MEM_NUMBER_OF_RANKS; 3604 rank_bgn += 3605 NUM_RANKS_PER_SHADOW_REG, 3606 ++sr) { 3607 sr_failed = 0; 3608 if (!((STATIC_CALIB_STEPS) & 3609 CALIB_SKIP_WRITES)) { 3610 if ((STATIC_CALIB_STEPS) 3611 & CALIB_SKIP_DELAY_SWEEPS) { 3612 /* not needed in quick mode! */ 3613 } else { 3614 /* 3615 * Determine if this set of 3616 * ranks should be skipped 3617 * entirely. 3618 */ 3619 if (!param->skip_shadow_regs[sr]) { 3620 if (!rw_mgr_mem_calibrate_writes 3621 (rank_bgn, write_group, 3622 write_test_bgn)) { 3623 sr_failed = 1; 3624 if (!(gbl-> 3625 phy_debug_mode_flags & 3626 PHY_DEBUG_SWEEP_ALL_GROUPS)) { 3627 return 0; 3628 } 3629 } 3630 } 3631 } 3632 } 3633 if (sr_failed != 0) 3634 group_failed = 1; 3635 } 3636 } 3637 3638 if (group_failed == 0) { 3639 for (read_group = write_group * 3640 RW_MGR_MEM_IF_READ_DQS_WIDTH / 3641 RW_MGR_MEM_IF_WRITE_DQS_WIDTH, 3642 read_test_bgn = 0; 3643 read_group < (write_group + 1) 3644 * RW_MGR_MEM_IF_READ_DQS_WIDTH 3645 / RW_MGR_MEM_IF_WRITE_DQS_WIDTH && 3646 group_failed == 0; 3647 read_group++, read_test_bgn += 3648 RW_MGR_MEM_DQ_PER_READ_DQS) { 3649 if (!((STATIC_CALIB_STEPS) & 3650 CALIB_SKIP_WRITES)) { 3651 if (!rw_mgr_mem_calibrate_vfifo_end 3652 (read_group, read_test_bgn)) { 3653 group_failed = 1; 3654 3655 if (!(gbl->phy_debug_mode_flags 3656 & PHY_DEBUG_SWEEP_ALL_GROUPS)) { 3657 return 0; 3658 } 3659 } 3660 } 3661 } 3662 } 3663 3664 if (group_failed != 0) 3665 failing_groups++; 3666 } 3667 3668 /* 3669 * USER If there are any failing groups then report 3670 * the failure. 3671 */ 3672 if (failing_groups != 0) 3673 return 0; 3674 3675 /* Calibrate the LFIFO */ 3676 if (!((STATIC_CALIB_STEPS) & CALIB_SKIP_LFIFO)) { 3677 /* 3678 * If we're skipping groups as part of debug, 3679 * don't calibrate LFIFO. 3680 */ 3681 if (param->skip_groups == 0) { 3682 if (!rw_mgr_mem_calibrate_lfifo()) 3683 return 0; 3684 } 3685 } 3686 } 3687 } 3688 3689 /* 3690 * Do not remove this line as it makes sure all of our decisions 3691 * have been applied. 3692 */ 3693 addr = (u32)&sdr_scc_mgr->update; 3694 writel(0, SOCFPGA_SDR_ADDRESS + addr); 3695 return 1; 3696 } 3697 3698 static uint32_t run_mem_calibrate(void) 3699 { 3700 uint32_t pass; 3701 uint32_t debug_info; 3702 uint32_t addr; 3703 3704 debug("%s:%d\n", __func__, __LINE__); 3705 3706 /* Reset pass/fail status shown on afi_cal_success/fail */ 3707 addr = sdr_get_addr(&phy_mgr_cfg->cal_status); 3708 writel(PHY_MGR_CAL_RESET, SOCFPGA_SDR_ADDRESS + addr); 3709 3710 addr = sdr_get_addr((u32 *)BASE_MMR); 3711 /* stop tracking manger */ 3712 uint32_t ctrlcfg = readl(SOCFPGA_SDR_ADDRESS + addr); 3713 3714 addr = sdr_get_addr((u32 *)BASE_MMR); 3715 writel(ctrlcfg & 0xFFBFFFFF, SOCFPGA_SDR_ADDRESS + addr); 3716 3717 initialize(); 3718 rw_mgr_mem_initialize(); 3719 3720 pass = mem_calibrate(); 3721 3722 mem_precharge_and_activate(); 3723 addr = sdr_get_addr(&phy_mgr_cmd->fifo_reset); 3724 writel(0, SOCFPGA_SDR_ADDRESS + addr); 3725 3726 /* 3727 * Handoff: 3728 * Don't return control of the PHY back to AFI when in debug mode. 3729 */ 3730 if ((gbl->phy_debug_mode_flags & PHY_DEBUG_IN_DEBUG_MODE) == 0) { 3731 rw_mgr_mem_handoff(); 3732 /* 3733 * In Hard PHY this is a 2-bit control: 3734 * 0: AFI Mux Select 3735 * 1: DDIO Mux Select 3736 */ 3737 addr = sdr_get_addr(&phy_mgr_cfg->mux_sel); 3738 writel(0x2, SOCFPGA_SDR_ADDRESS + addr); 3739 } 3740 3741 addr = sdr_get_addr((u32 *)BASE_MMR); 3742 writel(ctrlcfg, SOCFPGA_SDR_ADDRESS + addr); 3743 3744 if (pass) { 3745 printf("%s: CALIBRATION PASSED\n", __FILE__); 3746 3747 gbl->fom_in /= 2; 3748 gbl->fom_out /= 2; 3749 3750 if (gbl->fom_in > 0xff) 3751 gbl->fom_in = 0xff; 3752 3753 if (gbl->fom_out > 0xff) 3754 gbl->fom_out = 0xff; 3755 3756 /* Update the FOM in the register file */ 3757 debug_info = gbl->fom_in; 3758 debug_info |= gbl->fom_out << 8; 3759 addr = (u32)&sdr_reg_file->fom; 3760 writel(debug_info, SOCFPGA_SDR_ADDRESS + addr); 3761 3762 addr = sdr_get_addr(&phy_mgr_cfg->cal_debug_info); 3763 writel(debug_info, SOCFPGA_SDR_ADDRESS + addr); 3764 addr = sdr_get_addr(&phy_mgr_cfg->cal_status); 3765 writel(PHY_MGR_CAL_SUCCESS, SOCFPGA_SDR_ADDRESS + addr); 3766 } else { 3767 printf("%s: CALIBRATION FAILED\n", __FILE__); 3768 3769 debug_info = gbl->error_stage; 3770 debug_info |= gbl->error_substage << 8; 3771 debug_info |= gbl->error_group << 16; 3772 3773 addr = (u32)&sdr_reg_file->failing_stage; 3774 writel(debug_info, SOCFPGA_SDR_ADDRESS + addr); 3775 addr = sdr_get_addr(&phy_mgr_cfg->cal_debug_info); 3776 writel(debug_info, SOCFPGA_SDR_ADDRESS + addr); 3777 addr = sdr_get_addr(&phy_mgr_cfg->cal_status); 3778 writel(PHY_MGR_CAL_FAIL, SOCFPGA_SDR_ADDRESS + addr); 3779 3780 /* Update the failing group/stage in the register file */ 3781 debug_info = gbl->error_stage; 3782 debug_info |= gbl->error_substage << 8; 3783 debug_info |= gbl->error_group << 16; 3784 addr = (u32)&sdr_reg_file->failing_stage; 3785 writel(debug_info, SOCFPGA_SDR_ADDRESS + addr); 3786 } 3787 3788 return pass; 3789 } 3790 3791 static void hc_initialize_rom_data(void) 3792 { 3793 uint32_t i; 3794 uint32_t addr; 3795 3796 addr = sdr_get_addr((u32 *)(RW_MGR_INST_ROM_WRITE)); 3797 for (i = 0; i < ARRAY_SIZE(inst_rom_init); i++) { 3798 uint32_t data = inst_rom_init[i]; 3799 writel(data, SOCFPGA_SDR_ADDRESS + addr + (i << 2)); 3800 } 3801 3802 addr = sdr_get_addr((u32 *)(RW_MGR_AC_ROM_WRITE)); 3803 for (i = 0; i < ARRAY_SIZE(ac_rom_init); i++) { 3804 uint32_t data = ac_rom_init[i]; 3805 writel(data, SOCFPGA_SDR_ADDRESS + addr + (i << 2)); 3806 } 3807 } 3808 3809 static void initialize_reg_file(void) 3810 { 3811 uint32_t addr; 3812 3813 /* Initialize the register file with the correct data */ 3814 addr = (u32)&sdr_reg_file->signature; 3815 writel(REG_FILE_INIT_SEQ_SIGNATURE, SOCFPGA_SDR_ADDRESS + addr); 3816 3817 addr = (u32)&sdr_reg_file->debug_data_addr; 3818 writel(0, SOCFPGA_SDR_ADDRESS + addr); 3819 3820 addr = (u32)&sdr_reg_file->cur_stage; 3821 writel(0, SOCFPGA_SDR_ADDRESS + addr); 3822 3823 addr = (u32)&sdr_reg_file->fom; 3824 writel(0, SOCFPGA_SDR_ADDRESS + addr); 3825 3826 addr = (u32)&sdr_reg_file->failing_stage; 3827 writel(0, SOCFPGA_SDR_ADDRESS + addr); 3828 3829 addr = (u32)&sdr_reg_file->debug1; 3830 writel(0, SOCFPGA_SDR_ADDRESS + addr); 3831 3832 addr = (u32)&sdr_reg_file->debug2; 3833 writel(0, SOCFPGA_SDR_ADDRESS + addr); 3834 } 3835 3836 static void initialize_hps_phy(void) 3837 { 3838 uint32_t reg; 3839 uint32_t addr; 3840 /* 3841 * Tracking also gets configured here because it's in the 3842 * same register. 3843 */ 3844 uint32_t trk_sample_count = 7500; 3845 uint32_t trk_long_idle_sample_count = (10 << 16) | 100; 3846 /* 3847 * Format is number of outer loops in the 16 MSB, sample 3848 * count in 16 LSB. 3849 */ 3850 3851 reg = 0; 3852 reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_ACDELAYEN_SET(2); 3853 reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_DQDELAYEN_SET(1); 3854 reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_DQSDELAYEN_SET(1); 3855 reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_DQSLOGICDELAYEN_SET(1); 3856 reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_RESETDELAYEN_SET(0); 3857 reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_LPDDRDIS_SET(1); 3858 /* 3859 * This field selects the intrinsic latency to RDATA_EN/FULL path. 3860 * 00-bypass, 01- add 5 cycles, 10- add 10 cycles, 11- add 15 cycles. 3861 */ 3862 reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_ADDLATSEL_SET(0); 3863 reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_SAMPLECOUNT_19_0_SET( 3864 trk_sample_count); 3865 addr = sdr_get_addr((u32 *)BASE_MMR); 3866 writel(reg, SOCFPGA_SDR_ADDRESS + addr + SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_OFFSET); 3867 3868 reg = 0; 3869 reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_1_SAMPLECOUNT_31_20_SET( 3870 trk_sample_count >> 3871 SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_SAMPLECOUNT_19_0_WIDTH); 3872 reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_1_LONGIDLESAMPLECOUNT_19_0_SET( 3873 trk_long_idle_sample_count); 3874 writel(reg, SOCFPGA_SDR_ADDRESS + addr + SDR_CTRLGRP_PHYCTRL_PHYCTRL_1_OFFSET); 3875 3876 reg = 0; 3877 reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_2_LONGIDLESAMPLECOUNT_31_20_SET( 3878 trk_long_idle_sample_count >> 3879 SDR_CTRLGRP_PHYCTRL_PHYCTRL_1_LONGIDLESAMPLECOUNT_19_0_WIDTH); 3880 writel(reg, SOCFPGA_SDR_ADDRESS + addr + SDR_CTRLGRP_PHYCTRL_PHYCTRL_2_OFFSET); 3881 } 3882 3883 static void initialize_tracking(void) 3884 { 3885 uint32_t concatenated_longidle = 0x0; 3886 uint32_t concatenated_delays = 0x0; 3887 uint32_t concatenated_rw_addr = 0x0; 3888 uint32_t concatenated_refresh = 0x0; 3889 uint32_t trk_sample_count = 7500; 3890 uint32_t dtaps_per_ptap; 3891 uint32_t tmp_delay; 3892 uint32_t addr; 3893 3894 /* 3895 * compute usable version of value in case we skip full 3896 * computation later 3897 */ 3898 dtaps_per_ptap = 0; 3899 tmp_delay = 0; 3900 while (tmp_delay < IO_DELAY_PER_OPA_TAP) { 3901 dtaps_per_ptap++; 3902 tmp_delay += IO_DELAY_PER_DCHAIN_TAP; 3903 } 3904 dtaps_per_ptap--; 3905 3906 concatenated_longidle = concatenated_longidle ^ 10; 3907 /*longidle outer loop */ 3908 concatenated_longidle = concatenated_longidle << 16; 3909 concatenated_longidle = concatenated_longidle ^ 100; 3910 /*longidle sample count */ 3911 concatenated_delays = concatenated_delays ^ 243; 3912 /* trfc, worst case of 933Mhz 4Gb */ 3913 concatenated_delays = concatenated_delays << 8; 3914 concatenated_delays = concatenated_delays ^ 14; 3915 /* trcd, worst case */ 3916 concatenated_delays = concatenated_delays << 8; 3917 concatenated_delays = concatenated_delays ^ 10; 3918 /* vfifo wait */ 3919 concatenated_delays = concatenated_delays << 8; 3920 concatenated_delays = concatenated_delays ^ 4; 3921 /* mux delay */ 3922 3923 concatenated_rw_addr = concatenated_rw_addr ^ RW_MGR_IDLE; 3924 concatenated_rw_addr = concatenated_rw_addr << 8; 3925 concatenated_rw_addr = concatenated_rw_addr ^ RW_MGR_ACTIVATE_1; 3926 concatenated_rw_addr = concatenated_rw_addr << 8; 3927 concatenated_rw_addr = concatenated_rw_addr ^ RW_MGR_SGLE_READ; 3928 concatenated_rw_addr = concatenated_rw_addr << 8; 3929 concatenated_rw_addr = concatenated_rw_addr ^ RW_MGR_PRECHARGE_ALL; 3930 3931 concatenated_refresh = concatenated_refresh ^ RW_MGR_REFRESH_ALL; 3932 concatenated_refresh = concatenated_refresh << 24; 3933 concatenated_refresh = concatenated_refresh ^ 1000; /* trefi */ 3934 3935 /* Initialize the register file with the correct data */ 3936 addr = (u32)&sdr_reg_file->dtaps_per_ptap; 3937 writel(dtaps_per_ptap, SOCFPGA_SDR_ADDRESS + addr); 3938 3939 addr = (u32)&sdr_reg_file->trk_sample_count; 3940 writel(trk_sample_count, SOCFPGA_SDR_ADDRESS + addr); 3941 3942 addr = (u32)&sdr_reg_file->trk_longidle; 3943 writel(concatenated_longidle, SOCFPGA_SDR_ADDRESS + addr); 3944 3945 addr = (u32)&sdr_reg_file->delays; 3946 writel(concatenated_delays, SOCFPGA_SDR_ADDRESS + addr); 3947 3948 addr = (u32)&sdr_reg_file->trk_rw_mgr_addr; 3949 writel(concatenated_rw_addr, SOCFPGA_SDR_ADDRESS + addr); 3950 3951 addr = (u32)&sdr_reg_file->trk_read_dqs_width; 3952 writel(RW_MGR_MEM_IF_READ_DQS_WIDTH, SOCFPGA_SDR_ADDRESS + addr); 3953 3954 addr = (u32)&sdr_reg_file->trk_rfsh; 3955 writel(concatenated_refresh, SOCFPGA_SDR_ADDRESS + addr); 3956 } 3957 3958 int sdram_calibration_full(void) 3959 { 3960 struct param_type my_param; 3961 struct gbl_type my_gbl; 3962 uint32_t pass; 3963 uint32_t i; 3964 3965 param = &my_param; 3966 gbl = &my_gbl; 3967 3968 /* Initialize the debug mode flags */ 3969 gbl->phy_debug_mode_flags = 0; 3970 /* Set the calibration enabled by default */ 3971 gbl->phy_debug_mode_flags |= PHY_DEBUG_ENABLE_CAL_RPT; 3972 /* 3973 * Only sweep all groups (regardless of fail state) by default 3974 * Set enabled read test by default. 3975 */ 3976 #if DISABLE_GUARANTEED_READ 3977 gbl->phy_debug_mode_flags |= PHY_DEBUG_DISABLE_GUARANTEED_READ; 3978 #endif 3979 /* Initialize the register file */ 3980 initialize_reg_file(); 3981 3982 /* Initialize any PHY CSR */ 3983 initialize_hps_phy(); 3984 3985 scc_mgr_initialize(); 3986 3987 initialize_tracking(); 3988 3989 /* USER Enable all ranks, groups */ 3990 for (i = 0; i < RW_MGR_MEM_NUMBER_OF_RANKS; i++) 3991 param->skip_ranks[i] = 0; 3992 for (i = 0; i < NUM_SHADOW_REGS; ++i) 3993 param->skip_shadow_regs[i] = 0; 3994 param->skip_groups = 0; 3995 3996 printf("%s: Preparing to start memory calibration\n", __FILE__); 3997 3998 debug("%s:%d\n", __func__, __LINE__); 3999 debug_cond(DLEVEL == 1, 4000 "DDR3 FULL_RATE ranks=%u cs/dimm=%u dq/dqs=%u,%u vg/dqs=%u,%u ", 4001 RW_MGR_MEM_NUMBER_OF_RANKS, RW_MGR_MEM_NUMBER_OF_CS_PER_DIMM, 4002 RW_MGR_MEM_DQ_PER_READ_DQS, RW_MGR_MEM_DQ_PER_WRITE_DQS, 4003 RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS, 4004 RW_MGR_MEM_VIRTUAL_GROUPS_PER_WRITE_DQS); 4005 debug_cond(DLEVEL == 1, 4006 "dqs=%u,%u dq=%u dm=%u ptap_delay=%u dtap_delay=%u ", 4007 RW_MGR_MEM_IF_READ_DQS_WIDTH, RW_MGR_MEM_IF_WRITE_DQS_WIDTH, 4008 RW_MGR_MEM_DATA_WIDTH, RW_MGR_MEM_DATA_MASK_WIDTH, 4009 IO_DELAY_PER_OPA_TAP, IO_DELAY_PER_DCHAIN_TAP); 4010 debug_cond(DLEVEL == 1, "dtap_dqsen_delay=%u, dll=%u", 4011 IO_DELAY_PER_DQS_EN_DCHAIN_TAP, IO_DLL_CHAIN_LENGTH); 4012 debug_cond(DLEVEL == 1, "max values: en_p=%u dqdqs_p=%u en_d=%u dqs_in_d=%u ", 4013 IO_DQS_EN_PHASE_MAX, IO_DQDQS_OUT_PHASE_MAX, 4014 IO_DQS_EN_DELAY_MAX, IO_DQS_IN_DELAY_MAX); 4015 debug_cond(DLEVEL == 1, "io_in_d=%u io_out1_d=%u io_out2_d=%u ", 4016 IO_IO_IN_DELAY_MAX, IO_IO_OUT1_DELAY_MAX, 4017 IO_IO_OUT2_DELAY_MAX); 4018 debug_cond(DLEVEL == 1, "dqs_in_reserve=%u dqs_out_reserve=%u\n", 4019 IO_DQS_IN_RESERVE, IO_DQS_OUT_RESERVE); 4020 4021 hc_initialize_rom_data(); 4022 4023 /* update info for sims */ 4024 reg_file_set_stage(CAL_STAGE_NIL); 4025 reg_file_set_group(0); 4026 4027 /* 4028 * Load global needed for those actions that require 4029 * some dynamic calibration support. 4030 */ 4031 dyn_calib_steps = STATIC_CALIB_STEPS; 4032 /* 4033 * Load global to allow dynamic selection of delay loop settings 4034 * based on calibration mode. 4035 */ 4036 if (!(dyn_calib_steps & CALIB_SKIP_DELAY_LOOPS)) 4037 skip_delay_mask = 0xff; 4038 else 4039 skip_delay_mask = 0x0; 4040 4041 pass = run_mem_calibrate(); 4042 4043 printf("%s: Calibration complete\n", __FILE__); 4044 return pass; 4045 } 4046