drivers/ddr/altera/sequencer.c

   1 /*
   2  * Copyright Altera Corporation (C) 2012-2015
   3  *
   4  * SPDX-License-Identifier:    BSD-3-Clause
   5  */
   6
   7 #include <common.h>
   8 #include <asm/io.h>
   9 #include <asm/arch/sdram.h>
  10 #include "sequencer.h"
  11 #include "sequencer_auto.h"
  12 #include "sequencer_auto_ac_init.h"
  13 #include "sequencer_auto_inst_init.h"
  14 #include "sequencer_defines.h"
  15
  16 static struct socfpga_sdr_rw_load_manager *sdr_rw_load_mgr_regs =
  17         (struct socfpga_sdr_rw_load_manager *)(SDR_PHYGRP_RWMGRGRP_ADDRESS | 0x800);
  18
  19 static struct socfpga_sdr_rw_load_jump_manager *sdr_rw_load_jump_mgr_regs =
  20         (struct socfpga_sdr_rw_load_jump_manager *)(SDR_PHYGRP_RWMGRGRP_ADDRESS | 0xC00);
  21
  22 static struct socfpga_sdr_reg_file *sdr_reg_file =
  23         (struct socfpga_sdr_reg_file *)SDR_PHYGRP_REGFILEGRP_ADDRESS;
  24
  25 static struct socfpga_sdr_scc_mgr *sdr_scc_mgr =
  26         (struct socfpga_sdr_scc_mgr *)(SDR_PHYGRP_SCCGRP_ADDRESS | 0xe00);
  27
  28 static struct socfpga_phy_mgr_cmd *phy_mgr_cmd =
  29         (struct socfpga_phy_mgr_cmd *)SDR_PHYGRP_PHYMGRGRP_ADDRESS;
  30
  31 static struct socfpga_phy_mgr_cfg *phy_mgr_cfg =
  32         (struct socfpga_phy_mgr_cfg *)(SDR_PHYGRP_PHYMGRGRP_ADDRESS | 0x40);
  33
  34 static struct socfpga_data_mgr *data_mgr =
  35         (struct socfpga_data_mgr *)SDR_PHYGRP_DATAMGRGRP_ADDRESS;
  36
  37 static struct socfpga_sdr_ctrl *sdr_ctrl =
  38         (struct socfpga_sdr_ctrl *)SDR_CTRLGRP_ADDRESS;
  39
  40 #define DELTA_D         1
  41
  42 /*
  43  * In order to reduce ROM size, most of the selectable calibration steps are
  44  * decided at compile time based on the user's calibration mode selection,
  45  * as captured by the STATIC_CALIB_STEPS selection below.
  46  *
  47  * However, to support simulation-time selection of fast simulation mode, where
  48  * we skip everything except the bare minimum, we need a few of the steps to
  49  * be dynamic.  In those cases, we either use the DYNAMIC_CALIB_STEPS for the
  50  * check, which is based on the rtl-supplied value, or we dynamically compute
  51  * the value to use based on the dynamically-chosen calibration mode
  52  */
  53
  54 #define DLEVEL 0
  55 #define STATIC_IN_RTL_SIM 0
  56 #define STATIC_SKIP_DELAY_LOOPS 0
  57
  58 #define STATIC_CALIB_STEPS (STATIC_IN_RTL_SIM | CALIB_SKIP_FULL_TEST | \
  59         STATIC_SKIP_DELAY_LOOPS)
  60
  61 /* calibration steps requested by the rtl */
  62 uint16_t dyn_calib_steps;
  63
  64 /*
  65  * To make CALIB_SKIP_DELAY_LOOPS a dynamic conditional option
  66  * instead of static, we use boolean logic to select between
  67  * non-skip and skip values
  68  *
  69  * The mask is set to include all bits when not-skipping, but is
  70  * zero when skipping
  71  */
  72
  73 uint16_t skip_delay_mask;       /* mask off bits when skipping/not-skipping */
  74
  75 #define SKIP_DELAY_LOOP_VALUE_OR_ZERO(non_skip_value) \
  76         ((non_skip_value) & skip_delay_mask)
  77
  78 struct gbl_type *gbl;
  79 struct param_type *param;
  80 uint32_t curr_shadow_reg;
  81
  82 static uint32_t rw_mgr_mem_calibrate_write_test(uint32_t rank_bgn,
  83         uint32_t write_group, uint32_t use_dm,
  84         uint32_t all_correct, uint32_t *bit_chk, uint32_t all_ranks);
  85
  86 static void set_failing_group_stage(uint32_t group, uint32_t stage,
  87         uint32_t substage)
  88 {
  89         /*
  90          * Only set the global stage if there was not been any other
  91          * failing group
  92          */
  93         if (gbl->error_stage == CAL_STAGE_NIL)  {
  94                 gbl->error_substage = substage;
  95                 gbl->error_stage = stage;
  96                 gbl->error_group = group;
  97         }
  98 }
  99
 100 static void reg_file_set_group(u16 set_group)
 101 {
 102         clrsetbits_le32(&sdr_reg_file->cur_stage, 0xffff0000, set_group << 16);
 103 }
 104
 105 static void reg_file_set_stage(u8 set_stage)
 106 {
 107         clrsetbits_le32(&sdr_reg_file->cur_stage, 0xffff, set_stage & 0xff);
 108 }
 109
 110 static void reg_file_set_sub_stage(u8 set_sub_stage)
 111 {
 112         set_sub_stage &= 0xff;
 113         clrsetbits_le32(&sdr_reg_file->cur_stage, 0xff00, set_sub_stage << 8);
 114 }
 115
 116 /**
 117  * phy_mgr_initialize() - Initialize PHY Manager
 118  *
 119  * Initialize PHY Manager.
 120  */
 121 static void phy_mgr_initialize(void)
 122 {
 123         u32 ratio;
 124
 125         debug("%s:%d\n", __func__, __LINE__);
 126         /* Calibration has control over path to memory */
 127         /*
 128          * In Hard PHY this is a 2-bit control:
 129          * 0: AFI Mux Select
 130          * 1: DDIO Mux Select
 131          */
 132         writel(0x3, &phy_mgr_cfg->mux_sel);
 133
 134         /* USER memory clock is not stable we begin initialization  */
 135         writel(0, &phy_mgr_cfg->reset_mem_stbl);
 136
 137         /* USER calibration status all set to zero */
 138         writel(0, &phy_mgr_cfg->cal_status);
 139
 140         writel(0, &phy_mgr_cfg->cal_debug_info);
 141
 142         /* Init params only if we do NOT skip calibration. */
 143         if ((dyn_calib_steps & CALIB_SKIP_ALL) == CALIB_SKIP_ALL)
 144                 return;
 145
 146         ratio = RW_MGR_MEM_DQ_PER_READ_DQS /
 147                 RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS;
 148         param->read_correct_mask_vg = (1 << ratio) - 1;
 149         param->write_correct_mask_vg = (1 << ratio) - 1;
 150         param->read_correct_mask = (1 << RW_MGR_MEM_DQ_PER_READ_DQS) - 1;
 151         param->write_correct_mask = (1 << RW_MGR_MEM_DQ_PER_WRITE_DQS) - 1;
 152         ratio = RW_MGR_MEM_DATA_WIDTH /
 153                 RW_MGR_MEM_DATA_MASK_WIDTH;
 154         param->dm_correct_mask = (1 << ratio) - 1;
 155 }
 156
 157 static void set_rank_and_odt_mask(const u32 rank, const u32 odt_mode)
 158 {
 159         u32 odt_mask_0 = 0;
 160         u32 odt_mask_1 = 0;
 161         u32 cs_and_odt_mask;
 162
 163         if (odt_mode == RW_MGR_ODT_MODE_OFF) {
 164                 odt_mask_0 = 0x0;
 165                 odt_mask_1 = 0x0;
 166         } else {        /* RW_MGR_ODT_MODE_READ_WRITE */
 167                 if (RW_MGR_MEM_NUMBER_OF_RANKS == 1) {
 168                         /*
 169                          * 1 Rank
 170                          * Read: ODT = 0
 171                          * Write: ODT = 1
 172                          */
 173                         odt_mask_0 = 0x0;
 174                         odt_mask_1 = 0x1;
 175                 } else if (RW_MGR_MEM_NUMBER_OF_RANKS == 2) {
 176                         /* 2 Ranks */
 177                         if (RW_MGR_MEM_NUMBER_OF_CS_PER_DIMM == 1) {
 178                                 /* - Dual-Slot , Single-Rank
 179                                  * (1 chip-select per DIMM)
 180                                  * OR
 181                                  * - RDIMM, 4 total CS (2 CS per DIMM)
 182                                  * means 2 DIMM
 183                                  * Since MEM_NUMBER_OF_RANKS is 2 they are
 184                                  * both single rank
 185                                  * with 2 CS each (special for RDIMM)
 186                                  * Read: Turn on ODT on the opposite rank
 187                                  * Write: Turn on ODT on all ranks
 188                                  */
 189                                 odt_mask_0 = 0x3 & ~(1 << rank);
 190                                 odt_mask_1 = 0x3;
 191                         } else {
 192                                 /*
 193                                  * USER - Single-Slot , Dual-rank DIMMs
 194                                  * (2 chip-selects per DIMM)
 195                                  * USER Read: Turn on ODT off on all ranks
 196                                  * USER Write: Turn on ODT on active rank
 197                                  */
 198                                 odt_mask_0 = 0x0;
 199                                 odt_mask_1 = 0x3 & (1 << rank);
 200                         }
 201                 } else {
 202                         /* 4 Ranks
 203                          * Read:
 204                          * ----------+-----------------------+
 205                          *           |                       |
 206                          *           |         ODT           |
 207                          * Read From +-----------------------+
 208                          *   Rank    |  3  |  2  |  1  |  0  |
 209                          * ----------+-----+-----+-----+-----+
 210                          *     0     |  0  |  1  |  0  |  0  |
 211                          *     1     |  1  |  0  |  0  |  0  |
 212                          *     2     |  0  |  0  |  0  |  1  |
 213                          *     3     |  0  |  0  |  1  |  0  |
 214                          * ----------+-----+-----+-----+-----+
 215                          *
 216                          * Write:
 217                          * ----------+-----------------------+
 218                          *           |                       |
 219                          *           |         ODT           |
 220                          * Write To  +-----------------------+
 221                          *   Rank    |  3  |  2  |  1  |  0  |
 222                          * ----------+-----+-----+-----+-----+
 223                          *     0     |  0  |  1  |  0  |  1  |
 224                          *     1     |  1  |  0  |  1  |  0  |
 225                          *     2     |  0  |  1  |  0  |  1  |
 226                          *     3     |  1  |  0  |  1  |  0  |
 227                          * ----------+-----+-----+-----+-----+
 228                          */
 229                         switch (rank) {
 230                         case 0:
 231                                 odt_mask_0 = 0x4;
 232                                 odt_mask_1 = 0x5;
 233                                 break;
 234                         case 1:
 235                                 odt_mask_0 = 0x8;
 236                                 odt_mask_1 = 0xA;
 237                                 break;
 238                         case 2:
 239                                 odt_mask_0 = 0x1;
 240                                 odt_mask_1 = 0x5;
 241                                 break;
 242                         case 3:
 243                                 odt_mask_0 = 0x2;
 244                                 odt_mask_1 = 0xA;
 245                                 break;
 246                         }
 247                 }
 248         }
 249
 250         cs_and_odt_mask = (0xFF & ~(1 << rank)) |
 251                           ((0xFF & odt_mask_0) << 8) |
 252                           ((0xFF & odt_mask_1) << 16);
 253         writel(cs_and_odt_mask, SDR_PHYGRP_RWMGRGRP_ADDRESS |
 254                                 RW_MGR_SET_CS_AND_ODT_MASK_OFFSET);
 255 }
 256
 257 /**
 258  * scc_mgr_set() - Set SCC Manager register
 259  * @off:        Base offset in SCC Manager space
 260  * @grp:        Read/Write group
 261  * @val:        Value to be set
 262  *
 263  * This function sets the SCC Manager (Scan Chain Control Manager) register.
 264  */
 265 static void scc_mgr_set(u32 off, u32 grp, u32 val)
 266 {
 267         writel(val, SDR_PHYGRP_SCCGRP_ADDRESS | off | (grp << 2));
 268 }
 269
 270 /**
 271  * scc_mgr_initialize() - Initialize SCC Manager registers
 272  *
 273  * Initialize SCC Manager registers.
 274  */
 275 static void scc_mgr_initialize(void)
 276 {
 277         /*
 278          * Clear register file for HPS. 16 (2^4) is the size of the
 279          * full register file in the scc mgr:
 280          *      RFILE_DEPTH = 1 + log2(MEM_DQ_PER_DQS + 1 + MEM_DM_PER_DQS +
 281          *                             MEM_IF_READ_DQS_WIDTH - 1);
 282          */
 283         int i;
 284
 285         for (i = 0; i < 16; i++) {
 286                 debug_cond(DLEVEL == 1, "%s:%d: Clearing SCC RFILE index %u\n",
 287                            __func__, __LINE__, i);
 288                 scc_mgr_set(SCC_MGR_HHP_RFILE_OFFSET, 0, i);
 289         }
 290 }
 291
 292 static void scc_mgr_set_dqdqs_output_phase(uint32_t write_group, uint32_t phase)
 293 {
 294         scc_mgr_set(SCC_MGR_DQDQS_OUT_PHASE_OFFSET, write_group, phase);
 295 }
 296
 297 static void scc_mgr_set_dqs_bus_in_delay(uint32_t read_group, uint32_t delay)
 298 {
 299         scc_mgr_set(SCC_MGR_DQS_IN_DELAY_OFFSET, read_group, delay);
 300 }
 301
 302 static void scc_mgr_set_dqs_en_phase(uint32_t read_group, uint32_t phase)
 303 {
 304         scc_mgr_set(SCC_MGR_DQS_EN_PHASE_OFFSET, read_group, phase);
 305 }
 306
 307 static void scc_mgr_set_dqs_en_delay(uint32_t read_group, uint32_t delay)
 308 {
 309         scc_mgr_set(SCC_MGR_DQS_EN_DELAY_OFFSET, read_group, delay);
 310 }
 311
 312 static void scc_mgr_set_dqs_io_in_delay(uint32_t delay)
 313 {
 314         scc_mgr_set(SCC_MGR_IO_IN_DELAY_OFFSET, RW_MGR_MEM_DQ_PER_WRITE_DQS,
 315                     delay);
 316 }
 317
 318 static void scc_mgr_set_dq_in_delay(uint32_t dq_in_group, uint32_t delay)
 319 {
 320         scc_mgr_set(SCC_MGR_IO_IN_DELAY_OFFSET, dq_in_group, delay);
 321 }
 322
 323 static void scc_mgr_set_dq_out1_delay(uint32_t dq_in_group, uint32_t delay)
 324 {
 325         scc_mgr_set(SCC_MGR_IO_OUT1_DELAY_OFFSET, dq_in_group, delay);
 326 }
 327
 328 static void scc_mgr_set_dqs_out1_delay(uint32_t delay)
 329 {
 330         scc_mgr_set(SCC_MGR_IO_OUT1_DELAY_OFFSET, RW_MGR_MEM_DQ_PER_WRITE_DQS,
 331                     delay);
 332 }
 333
 334 static void scc_mgr_set_dm_out1_delay(uint32_t dm, uint32_t delay)
 335 {
 336         scc_mgr_set(SCC_MGR_IO_OUT1_DELAY_OFFSET,
 337                     RW_MGR_MEM_DQ_PER_WRITE_DQS + 1 + dm,
 338                     delay);
 339 }
 340
 341 /* load up dqs config settings */
 342 static void scc_mgr_load_dqs(uint32_t dqs)
 343 {
 344         writel(dqs, &sdr_scc_mgr->dqs_ena);
 345 }
 346
 347 /* load up dqs io config settings */
 348 static void scc_mgr_load_dqs_io(void)
 349 {
 350         writel(0, &sdr_scc_mgr->dqs_io_ena);
 351 }
 352
 353 /* load up dq config settings */
 354 static void scc_mgr_load_dq(uint32_t dq_in_group)
 355 {
 356         writel(dq_in_group, &sdr_scc_mgr->dq_ena);
 357 }
 358
 359 /* load up dm config settings */
 360 static void scc_mgr_load_dm(uint32_t dm)
 361 {
 362         writel(dm, &sdr_scc_mgr->dm_ena);
 363 }
 364
 365 /**
 366  * scc_mgr_set_all_ranks() - Set SCC Manager register for all ranks
 367  * @off:        Base offset in SCC Manager space
 368  * @grp:        Read/Write group
 369  * @val:        Value to be set
 370  * @update:     If non-zero, trigger SCC Manager update for all ranks
 371  *
 372  * This function sets the SCC Manager (Scan Chain Control Manager) register
 373  * and optionally triggers the SCC update for all ranks.
 374  */
 375 static void scc_mgr_set_all_ranks(const u32 off, const u32 grp, const u32 val,
 376                                   const int update)
 377 {
 378         u32 r;
 379
 380         for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS;
 381              r += NUM_RANKS_PER_SHADOW_REG) {
 382                 scc_mgr_set(off, grp, val);
 383
 384                 if (update || (r == 0)) {
 385                         writel(grp, &sdr_scc_mgr->dqs_ena);
 386                         writel(0, &sdr_scc_mgr->update);
 387                 }
 388         }
 389 }
 390
 391 static void scc_mgr_set_dqs_en_phase_all_ranks(u32 read_group, u32 phase)
 392 {
 393         /*
 394          * USER although the h/w doesn't support different phases per
 395          * shadow register, for simplicity our scc manager modeling
 396          * keeps different phase settings per shadow reg, and it's
 397          * important for us to keep them in sync to match h/w.
 398          * for efficiency, the scan chain update should occur only
 399          * once to sr0.
 400          */
 401         scc_mgr_set_all_ranks(SCC_MGR_DQS_EN_PHASE_OFFSET,
 402                               read_group, phase, 0);
 403 }
 404
 405 static void scc_mgr_set_dqdqs_output_phase_all_ranks(uint32_t write_group,
 406                                                      uint32_t phase)
 407 {
 408         /*
 409          * USER although the h/w doesn't support different phases per
 410          * shadow register, for simplicity our scc manager modeling
 411          * keeps different phase settings per shadow reg, and it's
 412          * important for us to keep them in sync to match h/w.
 413          * for efficiency, the scan chain update should occur only
 414          * once to sr0.
 415          */
 416         scc_mgr_set_all_ranks(SCC_MGR_DQDQS_OUT_PHASE_OFFSET,
 417                               write_group, phase, 0);
 418 }
 419
 420 static void scc_mgr_set_dqs_en_delay_all_ranks(uint32_t read_group,
 421                                                uint32_t delay)
 422 {
 423         /*
 424          * In shadow register mode, the T11 settings are stored in
 425          * registers in the core, which are updated by the DQS_ENA
 426          * signals. Not issuing the SCC_MGR_UPD command allows us to
 427          * save lots of rank switching overhead, by calling
 428          * select_shadow_regs_for_update with update_scan_chains
 429          * set to 0.
 430          */
 431         scc_mgr_set_all_ranks(SCC_MGR_DQS_EN_DELAY_OFFSET,
 432                               read_group, delay, 1);
 433         writel(0, &sdr_scc_mgr->update);
 434 }
 435
 436 /**
 437  * scc_mgr_set_oct_out1_delay() - Set OCT output delay
 438  * @write_group:        Write group
 439  * @delay:              Delay value
 440  *
 441  * This function sets the OCT output delay in SCC manager.
 442  */
 443 static void scc_mgr_set_oct_out1_delay(const u32 write_group, const u32 delay)
 444 {
 445         const int ratio = RW_MGR_MEM_IF_READ_DQS_WIDTH /
 446                           RW_MGR_MEM_IF_WRITE_DQS_WIDTH;
 447         const int base = write_group * ratio;
 448         int i;
 449         /*
 450          * Load the setting in the SCC manager
 451          * Although OCT affects only write data, the OCT delay is controlled
 452          * by the DQS logic block which is instantiated once per read group.
 453          * For protocols where a write group consists of multiple read groups,
 454          * the setting must be set multiple times.
 455          */
 456         for (i = 0; i < ratio; i++)
 457                 scc_mgr_set(SCC_MGR_OCT_OUT1_DELAY_OFFSET, base + i, delay);
 458 }
 459
 460 /**
 461  * scc_mgr_set_hhp_extras() - Set HHP extras.
 462  *
 463  * Load the fixed setting in the SCC manager HHP extras.
 464  */
 465 static void scc_mgr_set_hhp_extras(void)
 466 {
 467         /*
 468          * Load the fixed setting in the SCC manager
 469          * bits: 0:0 = 1'b1     - DQS bypass
 470          * bits: 1:1 = 1'b1     - DQ bypass
 471          * bits: 4:2 = 3'b001   - rfifo_mode
 472          * bits: 6:5 = 2'b01    - rfifo clock_select
 473          * bits: 7:7 = 1'b0     - separate gating from ungating setting
 474          * bits: 8:8 = 1'b0     - separate OE from Output delay setting
 475          */
 476         const u32 value = (0 << 8) | (0 << 7) | (1 << 5) |
 477                           (1 << 2) | (1 << 1) | (1 << 0);
 478         const u32 addr = SDR_PHYGRP_SCCGRP_ADDRESS |
 479                          SCC_MGR_HHP_GLOBALS_OFFSET |
 480                          SCC_MGR_HHP_EXTRAS_OFFSET;
 481
 482         debug_cond(DLEVEL == 1, "%s:%d Setting HHP Extras\n",
 483                    __func__, __LINE__);
 484         writel(value, addr);
 485         debug_cond(DLEVEL == 1, "%s:%d Done Setting HHP Extras\n",
 486                    __func__, __LINE__);
 487 }
 488
 489 /**
 490  * scc_mgr_zero_all() - Zero all DQS config
 491  *
 492  * Zero all DQS config.
 493  */
 494 static void scc_mgr_zero_all(void)
 495 {
 496         int i, r;
 497
 498         /*
 499          * USER Zero all DQS config settings, across all groups and all
 500          * shadow registers
 501          */
 502         for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS;
 503              r += NUM_RANKS_PER_SHADOW_REG) {
 504                 for (i = 0; i < RW_MGR_MEM_IF_READ_DQS_WIDTH; i++) {
 505                         /*
 506                          * The phases actually don't exist on a per-rank basis,
 507                          * but there's no harm updating them several times, so
 508                          * let's keep the code simple.
 509                          */
 510                         scc_mgr_set_dqs_bus_in_delay(i, IO_DQS_IN_RESERVE);
 511                         scc_mgr_set_dqs_en_phase(i, 0);
 512                         scc_mgr_set_dqs_en_delay(i, 0);
 513                 }
 514
 515                 for (i = 0; i < RW_MGR_MEM_IF_WRITE_DQS_WIDTH; i++) {
 516                         scc_mgr_set_dqdqs_output_phase(i, 0);
 517                         /* Arria V/Cyclone V don't have out2. */
 518                         scc_mgr_set_oct_out1_delay(i, IO_DQS_OUT_RESERVE);
 519                 }
 520         }
 521
 522         /* Multicast to all DQS group enables. */
 523         writel(0xff, &sdr_scc_mgr->dqs_ena);
 524         writel(0, &sdr_scc_mgr->update);
 525 }
 526
 527 /**
 528  * scc_set_bypass_mode() - Set bypass mode and trigger SCC update
 529  * @write_group:        Write group
 530  *
 531  * Set bypass mode and trigger SCC update.
 532  */
 533 static void scc_set_bypass_mode(const u32 write_group)
 534 {
 535         /* Multicast to all DQ enables. */
 536         writel(0xff, &sdr_scc_mgr->dq_ena);
 537         writel(0xff, &sdr_scc_mgr->dm_ena);
 538
 539         /* Update current DQS IO enable. */
 540         writel(0, &sdr_scc_mgr->dqs_io_ena);
 541
 542         /* Update the DQS logic. */
 543         writel(write_group, &sdr_scc_mgr->dqs_ena);
 544
 545         /* Hit update. */
 546         writel(0, &sdr_scc_mgr->update);
 547 }
 548
 549 /**
 550  * scc_mgr_load_dqs_for_write_group() - Load DQS settings for Write Group
 551  * @write_group:        Write group
 552  *
 553  * Load DQS settings for Write Group, do not trigger SCC update.
 554  */
 555 static void scc_mgr_load_dqs_for_write_group(const u32 write_group)
 556 {
 557         const int ratio = RW_MGR_MEM_IF_READ_DQS_WIDTH /
 558                           RW_MGR_MEM_IF_WRITE_DQS_WIDTH;
 559         const int base = write_group * ratio;
 560         int i;
 561         /*
 562          * Load the setting in the SCC manager
 563          * Although OCT affects only write data, the OCT delay is controlled
 564          * by the DQS logic block which is instantiated once per read group.
 565          * For protocols where a write group consists of multiple read groups,
 566          * the setting must be set multiple times.
 567          */
 568         for (i = 0; i < ratio; i++)
 569                 writel(base + i, &sdr_scc_mgr->dqs_ena);
 570 }
 571
 572 /**
 573  * scc_mgr_zero_group() - Zero all configs for a group
 574  *
 575  * Zero DQ, DM, DQS and OCT configs for a group.
 576  */
 577 static void scc_mgr_zero_group(const u32 write_group, const int out_only)
 578 {
 579         int i, r;
 580
 581         for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS;
 582              r += NUM_RANKS_PER_SHADOW_REG) {
 583                 /* Zero all DQ config settings. */
 584                 for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) {
 585                         scc_mgr_set_dq_out1_delay(i, 0);
 586                         if (!out_only)
 587                                 scc_mgr_set_dq_in_delay(i, 0);
 588                 }
 589
 590                 /* Multicast to all DQ enables. */
 591                 writel(0xff, &sdr_scc_mgr->dq_ena);
 592
 593                 /* Zero all DM config settings. */
 594                 for (i = 0; i < RW_MGR_NUM_DM_PER_WRITE_GROUP; i++)
 595                         scc_mgr_set_dm_out1_delay(i, 0);
 596
 597                 /* Multicast to all DM enables. */
 598                 writel(0xff, &sdr_scc_mgr->dm_ena);
 599
 600                 /* Zero all DQS IO settings. */
 601                 if (!out_only)
 602                         scc_mgr_set_dqs_io_in_delay(0);
 603
 604                 /* Arria V/Cyclone V don't have out2. */
 605                 scc_mgr_set_dqs_out1_delay(IO_DQS_OUT_RESERVE);
 606                 scc_mgr_set_oct_out1_delay(write_group, IO_DQS_OUT_RESERVE);
 607                 scc_mgr_load_dqs_for_write_group(write_group);
 608
 609                 /* Multicast to all DQS IO enables (only 1 in total). */
 610                 writel(0, &sdr_scc_mgr->dqs_io_ena);
 611
 612                 /* Hit update to zero everything. */
 613                 writel(0, &sdr_scc_mgr->update);
 614         }
 615 }
 616
 617 /*
 618  * apply and load a particular input delay for the DQ pins in a group
 619  * group_bgn is the index of the first dq pin (in the write group)
 620  */
 621 static void scc_mgr_apply_group_dq_in_delay(uint32_t group_bgn, uint32_t delay)
 622 {
 623         uint32_t i, p;
 624
 625         for (i = 0, p = group_bgn; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++, p++) {
 626                 scc_mgr_set_dq_in_delay(p, delay);
 627                 scc_mgr_load_dq(p);
 628         }
 629 }
 630
 631 /**
 632  * scc_mgr_apply_group_dq_out1_delay() - Apply and load an output delay for the DQ pins in a group
 633  * @delay:              Delay value
 634  *
 635  * Apply and load a particular output delay for the DQ pins in a group.
 636  */
 637 static void scc_mgr_apply_group_dq_out1_delay(const u32 delay)
 638 {
 639         int i;
 640
 641         for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) {
 642                 scc_mgr_set_dq_out1_delay(i, delay);
 643                 scc_mgr_load_dq(i);
 644         }
 645 }
 646
 647 /* apply and load a particular output delay for the DM pins in a group */
 648 static void scc_mgr_apply_group_dm_out1_delay(uint32_t delay1)
 649 {
 650         uint32_t i;
 651
 652         for (i = 0; i < RW_MGR_NUM_DM_PER_WRITE_GROUP; i++) {
 653                 scc_mgr_set_dm_out1_delay(i, delay1);
 654                 scc_mgr_load_dm(i);
 655         }
 656 }
 657
 658
 659 /* apply and load delay on both DQS and OCT out1 */
 660 static void scc_mgr_apply_group_dqs_io_and_oct_out1(uint32_t write_group,
 661                                                     uint32_t delay)
 662 {
 663         scc_mgr_set_dqs_out1_delay(delay);
 664         scc_mgr_load_dqs_io();
 665
 666         scc_mgr_set_oct_out1_delay(write_group, delay);
 667         scc_mgr_load_dqs_for_write_group(write_group);
 668 }
 669
 670 /**
 671  * scc_mgr_apply_group_all_out_delay_add() - Apply a delay to the entire output side: DQ, DM, DQS, OCT
 672  * @write_group:        Write group
 673  * @delay:              Delay value
 674  *
 675  * Apply a delay to the entire output side: DQ, DM, DQS, OCT.
 676  */
 677 static void scc_mgr_apply_group_all_out_delay_add(const u32 write_group,
 678                                                   const u32 delay)
 679 {
 680         u32 i, new_delay;
 681
 682         /* DQ shift */
 683         for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++)
 684                 scc_mgr_load_dq(i);
 685
 686         /* DM shift */
 687         for (i = 0; i < RW_MGR_NUM_DM_PER_WRITE_GROUP; i++)
 688                 scc_mgr_load_dm(i);
 689
 690         /* DQS shift */
 691         new_delay = READ_SCC_DQS_IO_OUT2_DELAY + delay;
 692         if (new_delay > IO_IO_OUT2_DELAY_MAX) {
 693                 debug_cond(DLEVEL == 1,
 694                            "%s:%d (%u, %u) DQS: %u > %d; adding %u to OUT1\n",
 695                            __func__, __LINE__, write_group, delay, new_delay,
 696                            IO_IO_OUT2_DELAY_MAX,
 697                            new_delay - IO_IO_OUT2_DELAY_MAX);
 698                 new_delay -= IO_IO_OUT2_DELAY_MAX;
 699                 scc_mgr_set_dqs_out1_delay(new_delay);
 700         }
 701
 702         scc_mgr_load_dqs_io();
 703
 704         /* OCT shift */
 705         new_delay = READ_SCC_OCT_OUT2_DELAY + delay;
 706         if (new_delay > IO_IO_OUT2_DELAY_MAX) {
 707                 debug_cond(DLEVEL == 1,
 708                            "%s:%d (%u, %u) DQS: %u > %d; adding %u to OUT1\n",
 709                            __func__, __LINE__, write_group, delay,
 710                            new_delay, IO_IO_OUT2_DELAY_MAX,
 711                            new_delay - IO_IO_OUT2_DELAY_MAX);
 712                 new_delay -= IO_IO_OUT2_DELAY_MAX;
 713                 scc_mgr_set_oct_out1_delay(write_group, new_delay);
 714         }
 715
 716         scc_mgr_load_dqs_for_write_group(write_group);
 717 }
 718
 719 /**
 720  * scc_mgr_apply_group_all_out_delay_add() - Apply a delay to the entire output side to all ranks
 721  * @write_group:        Write group
 722  * @delay:              Delay value
 723  *
 724  * Apply a delay to the entire output side (DQ, DM, DQS, OCT) to all ranks.
 725  */
 726 static void
 727 scc_mgr_apply_group_all_out_delay_add_all_ranks(const u32 write_group,
 728                                                 const u32 delay)
 729 {
 730         int r;
 731
 732         for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS;
 733              r += NUM_RANKS_PER_SHADOW_REG) {
 734                 scc_mgr_apply_group_all_out_delay_add(write_group, delay);
 735                 writel(0, &sdr_scc_mgr->update);
 736         }
 737 }
 738
 739 /**
 740  * set_jump_as_return() - Return instruction optimization
 741  *
 742  * Optimization used to recover some slots in ddr3 inst_rom could be
 743  * applied to other protocols if we wanted to
 744  */
 745 static void set_jump_as_return(void)
 746 {
 747         /*
 748          * To save space, we replace return with jump to special shared
 749          * RETURN instruction so we set the counter to large value so that
 750          * we always jump.
 751          */
 752         writel(0xff, &sdr_rw_load_mgr_regs->load_cntr0);
 753         writel(RW_MGR_RETURN, &sdr_rw_load_jump_mgr_regs->load_jump_add0);
 754 }
 755
 756 /*
 757  * should always use constants as argument to ensure all computations are
 758  * performed at compile time
 759  */
 760 static void delay_for_n_mem_clocks(const uint32_t clocks)
 761 {
 762         uint32_t afi_clocks;
 763         uint8_t inner = 0;
 764         uint8_t outer = 0;
 765         uint16_t c_loop = 0;
 766
 767         debug("%s:%d: clocks=%u ... start\n", __func__, __LINE__, clocks);
 768
 769
 770         afi_clocks = (clocks + AFI_RATE_RATIO-1) / AFI_RATE_RATIO;
 771         /* scale (rounding up) to get afi clocks */
 772
 773         /*
 774          * Note, we don't bother accounting for being off a little bit
 775          * because of a few extra instructions in outer loops
 776          * Note, the loops have a test at the end, and do the test before
 777          * the decrement, and so always perform the loop
 778          * 1 time more than the counter value
 779          */
 780         if (afi_clocks == 0) {
 781                 ;
 782         } else if (afi_clocks <= 0x100) {
 783                 inner = afi_clocks-1;
 784                 outer = 0;
 785                 c_loop = 0;
 786         } else if (afi_clocks <= 0x10000) {
 787                 inner = 0xff;
 788                 outer = (afi_clocks-1) >> 8;
 789                 c_loop = 0;
 790         } else {
 791                 inner = 0xff;
 792                 outer = 0xff;
 793                 c_loop = (afi_clocks-1) >> 16;
 794         }
 795
 796         /*
 797          * rom instructions are structured as follows:
 798          *
 799          *    IDLE_LOOP2: jnz cntr0, TARGET_A
 800          *    IDLE_LOOP1: jnz cntr1, TARGET_B
 801          *                return
 802          *
 803          * so, when doing nested loops, TARGET_A is set to IDLE_LOOP2, and
 804          * TARGET_B is set to IDLE_LOOP2 as well
 805          *
 806          * if we have no outer loop, though, then we can use IDLE_LOOP1 only,
 807          * and set TARGET_B to IDLE_LOOP1 and we skip IDLE_LOOP2 entirely
 808          *
 809          * a little confusing, but it helps save precious space in the inst_rom
 810          * and sequencer rom and keeps the delays more accurate and reduces
 811          * overhead
 812          */
 813         if (afi_clocks <= 0x100) {
 814                 writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(inner),
 815                         &sdr_rw_load_mgr_regs->load_cntr1);
 816
 817                 writel(RW_MGR_IDLE_LOOP1,
 818                         &sdr_rw_load_jump_mgr_regs->load_jump_add1);
 819
 820                 writel(RW_MGR_IDLE_LOOP1, SDR_PHYGRP_RWMGRGRP_ADDRESS |
 821                                           RW_MGR_RUN_SINGLE_GROUP_OFFSET);
 822         } else {
 823                 writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(inner),
 824                         &sdr_rw_load_mgr_regs->load_cntr0);
 825
 826                 writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(outer),
 827                         &sdr_rw_load_mgr_regs->load_cntr1);
 828
 829                 writel(RW_MGR_IDLE_LOOP2,
 830                         &sdr_rw_load_jump_mgr_regs->load_jump_add0);
 831
 832                 writel(RW_MGR_IDLE_LOOP2,
 833                         &sdr_rw_load_jump_mgr_regs->load_jump_add1);
 834
 835                 /* hack to get around compiler not being smart enough */
 836                 if (afi_clocks <= 0x10000) {
 837                         /* only need to run once */
 838                         writel(RW_MGR_IDLE_LOOP2, SDR_PHYGRP_RWMGRGRP_ADDRESS |
 839                                                   RW_MGR_RUN_SINGLE_GROUP_OFFSET);
 840                 } else {
 841                         do {
 842                                 writel(RW_MGR_IDLE_LOOP2,
 843                                         SDR_PHYGRP_RWMGRGRP_ADDRESS |
 844                                         RW_MGR_RUN_SINGLE_GROUP_OFFSET);
 845                         } while (c_loop-- != 0);
 846                 }
 847         }
 848         debug("%s:%d clocks=%u ... end\n", __func__, __LINE__, clocks);
 849 }
 850
 851 /**
 852  * rw_mgr_mem_init_load_regs() - Load instruction registers
 853  * @cntr0:      Counter 0 value
 854  * @cntr1:      Counter 1 value
 855  * @cntr2:      Counter 2 value
 856  * @jump:       Jump instruction value
 857  *
 858  * Load instruction registers.
 859  */
 860 static void rw_mgr_mem_init_load_regs(u32 cntr0, u32 cntr1, u32 cntr2, u32 jump)
 861 {
 862         uint32_t grpaddr = SDR_PHYGRP_RWMGRGRP_ADDRESS |
 863                            RW_MGR_RUN_SINGLE_GROUP_OFFSET;
 864
 865         /* Load counters */
 866         writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(cntr0),
 867                &sdr_rw_load_mgr_regs->load_cntr0);
 868         writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(cntr1),
 869                &sdr_rw_load_mgr_regs->load_cntr1);
 870         writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(cntr2),
 871                &sdr_rw_load_mgr_regs->load_cntr2);
 872
 873         /* Load jump address */
 874         writel(jump, &sdr_rw_load_jump_mgr_regs->load_jump_add0);
 875         writel(jump, &sdr_rw_load_jump_mgr_regs->load_jump_add1);
 876         writel(jump, &sdr_rw_load_jump_mgr_regs->load_jump_add2);
 877
 878         /* Execute count instruction */
 879         writel(jump, grpaddr);
 880 }
 881
 882 /**
 883  * rw_mgr_mem_load_user() - Load user calibration values
 884  * @fin1:       Final instruction 1
 885  * @fin2:       Final instruction 2
 886  * @precharge:  If 1, precharge the banks at the end
 887  *
 888  * Load user calibration values and optionally precharge the banks.
 889  */
 890 static void rw_mgr_mem_load_user(const u32 fin1, const u32 fin2,
 891                                  const int precharge)
 892 {
 893         u32 grpaddr = SDR_PHYGRP_RWMGRGRP_ADDRESS |
 894                       RW_MGR_RUN_SINGLE_GROUP_OFFSET;
 895         u32 r;
 896
 897         for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; r++) {
 898                 if (param->skip_ranks[r]) {
 899                         /* request to skip the rank */
 900                         continue;
 901                 }
 902
 903                 /* set rank */
 904                 set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_OFF);
 905
 906                 /* precharge all banks ... */
 907                 if (precharge)
 908                         writel(RW_MGR_PRECHARGE_ALL, grpaddr);
 909
 910                 /*
 911                  * USER Use Mirror-ed commands for odd ranks if address
 912                  * mirrorring is on
 913                  */
 914                 if ((RW_MGR_MEM_ADDRESS_MIRRORING >> r) & 0x1) {
 915                         set_jump_as_return();
 916                         writel(RW_MGR_MRS2_MIRR, grpaddr);
 917                         delay_for_n_mem_clocks(4);
 918                         set_jump_as_return();
 919                         writel(RW_MGR_MRS3_MIRR, grpaddr);
 920                         delay_for_n_mem_clocks(4);
 921                         set_jump_as_return();
 922                         writel(RW_MGR_MRS1_MIRR, grpaddr);
 923                         delay_for_n_mem_clocks(4);
 924                         set_jump_as_return();
 925                         writel(fin1, grpaddr);
 926                 } else {
 927                         set_jump_as_return();
 928                         writel(RW_MGR_MRS2, grpaddr);
 929                         delay_for_n_mem_clocks(4);
 930                         set_jump_as_return();
 931                         writel(RW_MGR_MRS3, grpaddr);
 932                         delay_for_n_mem_clocks(4);
 933                         set_jump_as_return();
 934                         writel(RW_MGR_MRS1, grpaddr);
 935                         set_jump_as_return();
 936                         writel(fin2, grpaddr);
 937                 }
 938
 939                 if (precharge)
 940                         continue;
 941
 942                 set_jump_as_return();
 943                 writel(RW_MGR_ZQCL, grpaddr);
 944
 945                 /* tZQinit = tDLLK = 512 ck cycles */
 946                 delay_for_n_mem_clocks(512);
 947         }
 948 }
 949
 950 static void rw_mgr_mem_initialize(void)
 951 {
 952         debug("%s:%d\n", __func__, __LINE__);
 953
 954         /* The reset / cke part of initialization is broadcasted to all ranks */
 955         writel(RW_MGR_RANK_ALL, SDR_PHYGRP_RWMGRGRP_ADDRESS |
 956                                 RW_MGR_SET_CS_AND_ODT_MASK_OFFSET);
 957
 958         /*
 959          * Here's how you load register for a loop
 960          * Counters are located @ 0x800
 961          * Jump address are located @ 0xC00
 962          * For both, registers 0 to 3 are selected using bits 3 and 2, like
 963          * in 0x800, 0x804, 0x808, 0x80C and 0xC00, 0xC04, 0xC08, 0xC0C
 964          * I know this ain't pretty, but Avalon bus throws away the 2 least
 965          * significant bits
 966          */
 967
 968         /* start with memory RESET activated */
 969
 970         /* tINIT = 200us */
 971
 972         /*
 973          * 200us @ 266MHz (3.75 ns) ~ 54000 clock cycles
 974          * If a and b are the number of iteration in 2 nested loops
 975          * it takes the following number of cycles to complete the operation:
 976          * number_of_cycles = ((2 + n) * a + 2) * b
 977          * where n is the number of instruction in the inner loop
 978          * One possible solution is n = 0 , a = 256 , b = 106 => a = FF,
 979          * b = 6A
 980          */
 981         rw_mgr_mem_init_load_regs(SEQ_TINIT_CNTR0_VAL, SEQ_TINIT_CNTR1_VAL,
 982                                   SEQ_TINIT_CNTR2_VAL,
 983                                   RW_MGR_INIT_RESET_0_CKE_0);
 984
 985         /* indicate that memory is stable */
 986         writel(1, &phy_mgr_cfg->reset_mem_stbl);
 987
 988         /*
 989          * transition the RESET to high
 990          * Wait for 500us
 991          */
 992
 993         /*
 994          * 500us @ 266MHz (3.75 ns) ~ 134000 clock cycles
 995          * If a and b are the number of iteration in 2 nested loops
 996          * it takes the following number of cycles to complete the operation
 997          * number_of_cycles = ((2 + n) * a + 2) * b
 998          * where n is the number of instruction in the inner loop
 999          * One possible solution is n = 2 , a = 131 , b = 256 => a = 83,
1000          * b = FF
1001          */
1002         rw_mgr_mem_init_load_regs(SEQ_TRESET_CNTR0_VAL, SEQ_TRESET_CNTR1_VAL,
1003                                   SEQ_TRESET_CNTR2_VAL,
1004                                   RW_MGR_INIT_RESET_1_CKE_0);
1005
1006         /* bring up clock enable */
1007
1008         /* tXRP < 250 ck cycles */
1009         delay_for_n_mem_clocks(250);
1010
1011         rw_mgr_mem_load_user(RW_MGR_MRS0_DLL_RESET_MIRR, RW_MGR_MRS0_DLL_RESET,
1012                              0);
1013 }
1014
1015 /*
1016  * At the end of calibration we have to program the user settings in, and
1017  * USER  hand off the memory to the user.
1018  */
1019 static void rw_mgr_mem_handoff(void)
1020 {
1021         rw_mgr_mem_load_user(RW_MGR_MRS0_USER_MIRR, RW_MGR_MRS0_USER, 1);
1022         /*
1023          * USER  need to wait tMOD (12CK or 15ns) time before issuing
1024          * other commands, but we will have plenty of NIOS cycles before
1025          * actual handoff so its okay.
1026          */
1027 }
1028
1029 /*
1030  * performs a guaranteed read on the patterns we are going to use during a
1031  * read test to ensure memory works
1032  */
1033 static uint32_t rw_mgr_mem_calibrate_read_test_patterns(uint32_t rank_bgn,
1034         uint32_t group, uint32_t num_tries, uint32_t *bit_chk,
1035         uint32_t all_ranks)
1036 {
1037         uint32_t r, vg;
1038         uint32_t correct_mask_vg;
1039         uint32_t tmp_bit_chk;
1040         uint32_t rank_end = all_ranks ? RW_MGR_MEM_NUMBER_OF_RANKS :
1041                 (rank_bgn + NUM_RANKS_PER_SHADOW_REG);
1042         uint32_t addr;
1043         uint32_t base_rw_mgr;
1044
1045         *bit_chk = param->read_correct_mask;
1046         correct_mask_vg = param->read_correct_mask_vg;
1047
1048         for (r = rank_bgn; r < rank_end; r++) {
1049                 if (param->skip_ranks[r])
1050                         /* request to skip the rank */
1051                         continue;
1052
1053                 /* set rank */
1054                 set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_READ_WRITE);
1055
1056                 /* Load up a constant bursts of read commands */
1057                 writel(0x20, &sdr_rw_load_mgr_regs->load_cntr0);
1058                 writel(RW_MGR_GUARANTEED_READ,
1059                         &sdr_rw_load_jump_mgr_regs->load_jump_add0);
1060
1061                 writel(0x20, &sdr_rw_load_mgr_regs->load_cntr1);
1062                 writel(RW_MGR_GUARANTEED_READ_CONT,
1063                         &sdr_rw_load_jump_mgr_regs->load_jump_add1);
1064
1065                 tmp_bit_chk = 0;
1066                 for (vg = RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS-1; ; vg--) {
1067                         /* reset the fifos to get pointers to known state */
1068
1069                         writel(0, &phy_mgr_cmd->fifo_reset);
1070                         writel(0, SDR_PHYGRP_RWMGRGRP_ADDRESS |
1071                                   RW_MGR_RESET_READ_DATAPATH_OFFSET);
1072
1073                         tmp_bit_chk = tmp_bit_chk << (RW_MGR_MEM_DQ_PER_READ_DQS
1074                                 / RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS);
1075
1076                         addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_SINGLE_GROUP_OFFSET;
1077                         writel(RW_MGR_GUARANTEED_READ, addr +
1078                                ((group * RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS +
1079                                 vg) << 2));
1080
1081                         base_rw_mgr = readl(SDR_PHYGRP_RWMGRGRP_ADDRESS);
1082                         tmp_bit_chk = tmp_bit_chk | (correct_mask_vg & (~base_rw_mgr));
1083
1084                         if (vg == 0)
1085                                 break;
1086                 }
1087                 *bit_chk &= tmp_bit_chk;
1088         }
1089
1090         addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_SINGLE_GROUP_OFFSET;
1091         writel(RW_MGR_CLEAR_DQS_ENABLE, addr + (group << 2));
1092
1093         set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF);
1094         debug_cond(DLEVEL == 1, "%s:%d test_load_patterns(%u,ALL) => (%u == %u) =>\
1095                    %lu\n", __func__, __LINE__, group, *bit_chk, param->read_correct_mask,
1096                    (long unsigned int)(*bit_chk == param->read_correct_mask));
1097         return *bit_chk == param->read_correct_mask;
1098 }
1099
1100 static uint32_t rw_mgr_mem_calibrate_read_test_patterns_all_ranks
1101         (uint32_t group, uint32_t num_tries, uint32_t *bit_chk)
1102 {
1103         return rw_mgr_mem_calibrate_read_test_patterns(0, group,
1104                 num_tries, bit_chk, 1);
1105 }
1106
1107 /* load up the patterns we are going to use during a read test */
1108 static void rw_mgr_mem_calibrate_read_load_patterns(uint32_t rank_bgn,
1109         uint32_t all_ranks)
1110 {
1111         uint32_t r;
1112         uint32_t rank_end = all_ranks ? RW_MGR_MEM_NUMBER_OF_RANKS :
1113                 (rank_bgn + NUM_RANKS_PER_SHADOW_REG);
1114
1115         debug("%s:%d\n", __func__, __LINE__);
1116         for (r = rank_bgn; r < rank_end; r++) {
1117                 if (param->skip_ranks[r])
1118                         /* request to skip the rank */
1119                         continue;
1120
1121                 /* set rank */
1122                 set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_READ_WRITE);
1123
1124                 /* Load up a constant bursts */
1125                 writel(0x20, &sdr_rw_load_mgr_regs->load_cntr0);
1126
1127                 writel(RW_MGR_GUARANTEED_WRITE_WAIT0,
1128                         &sdr_rw_load_jump_mgr_regs->load_jump_add0);
1129
1130                 writel(0x20, &sdr_rw_load_mgr_regs->load_cntr1);
1131
1132                 writel(RW_MGR_GUARANTEED_WRITE_WAIT1,
1133                         &sdr_rw_load_jump_mgr_regs->load_jump_add1);
1134
1135                 writel(0x04, &sdr_rw_load_mgr_regs->load_cntr2);
1136
1137                 writel(RW_MGR_GUARANTEED_WRITE_WAIT2,
1138                         &sdr_rw_load_jump_mgr_regs->load_jump_add2);
1139
1140                 writel(0x04, &sdr_rw_load_mgr_regs->load_cntr3);
1141
1142                 writel(RW_MGR_GUARANTEED_WRITE_WAIT3,
1143                         &sdr_rw_load_jump_mgr_regs->load_jump_add3);
1144
1145                 writel(RW_MGR_GUARANTEED_WRITE, SDR_PHYGRP_RWMGRGRP_ADDRESS |
1146                                                 RW_MGR_RUN_SINGLE_GROUP_OFFSET);
1147         }
1148
1149         set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF);
1150 }
1151
1152 /*
1153  * try a read and see if it returns correct data back. has dummy reads
1154  * inserted into the mix used to align dqs enable. has more thorough checks
1155  * than the regular read test.
1156  */
1157 static uint32_t rw_mgr_mem_calibrate_read_test(uint32_t rank_bgn, uint32_t group,
1158         uint32_t num_tries, uint32_t all_correct, uint32_t *bit_chk,
1159         uint32_t all_groups, uint32_t all_ranks)
1160 {
1161         uint32_t r, vg;
1162         uint32_t correct_mask_vg;
1163         uint32_t tmp_bit_chk;
1164         uint32_t rank_end = all_ranks ? RW_MGR_MEM_NUMBER_OF_RANKS :
1165                 (rank_bgn + NUM_RANKS_PER_SHADOW_REG);
1166         uint32_t addr;
1167         uint32_t base_rw_mgr;
1168
1169         *bit_chk = param->read_correct_mask;
1170         correct_mask_vg = param->read_correct_mask_vg;
1171
1172         uint32_t quick_read_mode = (((STATIC_CALIB_STEPS) &
1173                 CALIB_SKIP_DELAY_SWEEPS) && ENABLE_SUPER_QUICK_CALIBRATION);
1174
1175         for (r = rank_bgn; r < rank_end; r++) {
1176                 if (param->skip_ranks[r])
1177                         /* request to skip the rank */
1178                         continue;
1179
1180                 /* set rank */
1181                 set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_READ_WRITE);
1182
1183                 writel(0x10, &sdr_rw_load_mgr_regs->load_cntr1);
1184
1185                 writel(RW_MGR_READ_B2B_WAIT1,
1186                         &sdr_rw_load_jump_mgr_regs->load_jump_add1);
1187
1188                 writel(0x10, &sdr_rw_load_mgr_regs->load_cntr2);
1189                 writel(RW_MGR_READ_B2B_WAIT2,
1190                         &sdr_rw_load_jump_mgr_regs->load_jump_add2);
1191
1192                 if (quick_read_mode)
1193                         writel(0x1, &sdr_rw_load_mgr_regs->load_cntr0);
1194                         /* need at least two (1+1) reads to capture failures */
1195                 else if (all_groups)
1196                         writel(0x06, &sdr_rw_load_mgr_regs->load_cntr0);
1197                 else
1198                         writel(0x32, &sdr_rw_load_mgr_regs->load_cntr0);
1199
1200                 writel(RW_MGR_READ_B2B,
1201                         &sdr_rw_load_jump_mgr_regs->load_jump_add0);
1202                 if (all_groups)
1203                         writel(RW_MGR_MEM_IF_READ_DQS_WIDTH *
1204                                RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS - 1,
1205                                &sdr_rw_load_mgr_regs->load_cntr3);
1206                 else
1207                         writel(0x0, &sdr_rw_load_mgr_regs->load_cntr3);
1208
1209                 writel(RW_MGR_READ_B2B,
1210                         &sdr_rw_load_jump_mgr_regs->load_jump_add3);
1211
1212                 tmp_bit_chk = 0;
1213                 for (vg = RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS-1; ; vg--) {
1214                         /* reset the fifos to get pointers to known state */
1215                         writel(0, &phy_mgr_cmd->fifo_reset);
1216                         writel(0, SDR_PHYGRP_RWMGRGRP_ADDRESS |
1217                                   RW_MGR_RESET_READ_DATAPATH_OFFSET);
1218
1219                         tmp_bit_chk = tmp_bit_chk << (RW_MGR_MEM_DQ_PER_READ_DQS
1220                                 / RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS);
1221
1222                         if (all_groups)
1223                                 addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_ALL_GROUPS_OFFSET;
1224                         else
1225                                 addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_SINGLE_GROUP_OFFSET;
1226
1227                         writel(RW_MGR_READ_B2B, addr +
1228                                ((group * RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS +
1229                                vg) << 2));
1230
1231                         base_rw_mgr = readl(SDR_PHYGRP_RWMGRGRP_ADDRESS);
1232                         tmp_bit_chk = tmp_bit_chk | (correct_mask_vg & ~(base_rw_mgr));
1233
1234                         if (vg == 0)
1235                                 break;
1236                 }
1237                 *bit_chk &= tmp_bit_chk;
1238         }
1239
1240         addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_SINGLE_GROUP_OFFSET;
1241         writel(RW_MGR_CLEAR_DQS_ENABLE, addr + (group << 2));
1242
1243         if (all_correct) {
1244                 set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF);
1245                 debug_cond(DLEVEL == 2, "%s:%d read_test(%u,ALL,%u) =>\
1246                            (%u == %u) => %lu", __func__, __LINE__, group,
1247                            all_groups, *bit_chk, param->read_correct_mask,
1248                            (long unsigned int)(*bit_chk ==
1249                            param->read_correct_mask));
1250                 return *bit_chk == param->read_correct_mask;
1251         } else  {
1252                 set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF);
1253                 debug_cond(DLEVEL == 2, "%s:%d read_test(%u,ONE,%u) =>\
1254                            (%u != %lu) => %lu\n", __func__, __LINE__,
1255                            group, all_groups, *bit_chk, (long unsigned int)0,
1256                            (long unsigned int)(*bit_chk != 0x00));
1257                 return *bit_chk != 0x00;
1258         }
1259 }
1260
1261 static uint32_t rw_mgr_mem_calibrate_read_test_all_ranks(uint32_t group,
1262         uint32_t num_tries, uint32_t all_correct, uint32_t *bit_chk,
1263         uint32_t all_groups)
1264 {
1265         return rw_mgr_mem_calibrate_read_test(0, group, num_tries, all_correct,
1266                                               bit_chk, all_groups, 1);
1267 }
1268
1269 static void rw_mgr_incr_vfifo(uint32_t grp, uint32_t *v)
1270 {
1271         writel(grp, &phy_mgr_cmd->inc_vfifo_hard_phy);
1272         (*v)++;
1273 }
1274
1275 static void rw_mgr_decr_vfifo(uint32_t grp, uint32_t *v)
1276 {
1277         uint32_t i;
1278
1279         for (i = 0; i < VFIFO_SIZE-1; i++)
1280                 rw_mgr_incr_vfifo(grp, v);
1281 }
1282
1283 static int find_vfifo_read(uint32_t grp, uint32_t *bit_chk)
1284 {
1285         uint32_t  v;
1286         uint32_t fail_cnt = 0;
1287         uint32_t test_status;
1288
1289         for (v = 0; v < VFIFO_SIZE; ) {
1290                 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: vfifo %u\n",
1291                            __func__, __LINE__, v);
1292                 test_status = rw_mgr_mem_calibrate_read_test_all_ranks
1293                         (grp, 1, PASS_ONE_BIT, bit_chk, 0);
1294                 if (!test_status) {
1295                         fail_cnt++;
1296
1297                         if (fail_cnt == 2)
1298                                 break;
1299                 }
1300
1301                 /* fiddle with FIFO */
1302                 rw_mgr_incr_vfifo(grp, &v);
1303         }
1304
1305         if (v >= VFIFO_SIZE) {
1306                 /* no failing read found!! Something must have gone wrong */
1307                 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: vfifo failed\n",
1308                            __func__, __LINE__);
1309                 return 0;
1310         } else {
1311                 return v;
1312         }
1313 }
1314
1315 static int find_working_phase(uint32_t *grp, uint32_t *bit_chk,
1316                               uint32_t dtaps_per_ptap, uint32_t *work_bgn,
1317                               uint32_t *v, uint32_t *d, uint32_t *p,
1318                               uint32_t *i, uint32_t *max_working_cnt)
1319 {
1320         uint32_t found_begin = 0;
1321         uint32_t tmp_delay = 0;
1322         uint32_t test_status;
1323
1324         for (*d = 0; *d <= dtaps_per_ptap; (*d)++, tmp_delay +=
1325                 IO_DELAY_PER_DQS_EN_DCHAIN_TAP) {
1326                 *work_bgn = tmp_delay;
1327                 scc_mgr_set_dqs_en_delay_all_ranks(*grp, *d);
1328
1329                 for (*i = 0; *i < VFIFO_SIZE; (*i)++) {
1330                         for (*p = 0; *p <= IO_DQS_EN_PHASE_MAX; (*p)++, *work_bgn +=
1331                                 IO_DELAY_PER_OPA_TAP) {
1332                                 scc_mgr_set_dqs_en_phase_all_ranks(*grp, *p);
1333
1334                                 test_status =
1335                                 rw_mgr_mem_calibrate_read_test_all_ranks
1336                                 (*grp, 1, PASS_ONE_BIT, bit_chk, 0);
1337
1338                                 if (test_status) {
1339                                         *max_working_cnt = 1;
1340                                         found_begin = 1;
1341                                         break;
1342                                 }
1343                         }
1344
1345                         if (found_begin)
1346                                 break;
1347
1348                         if (*p > IO_DQS_EN_PHASE_MAX)
1349                                 /* fiddle with FIFO */
1350                                 rw_mgr_incr_vfifo(*grp, v);
1351                 }
1352
1353                 if (found_begin)
1354                         break;
1355         }
1356
1357         if (*i >= VFIFO_SIZE) {
1358                 /* cannot find working solution */
1359                 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: no vfifo/\
1360                            ptap/dtap\n", __func__, __LINE__);
1361                 return 0;
1362         } else {
1363                 return 1;
1364         }
1365 }
1366
1367 static void sdr_backup_phase(uint32_t *grp, uint32_t *bit_chk,
1368                              uint32_t *work_bgn, uint32_t *v, uint32_t *d,
1369                              uint32_t *p, uint32_t *max_working_cnt)
1370 {
1371         uint32_t found_begin = 0;
1372         uint32_t tmp_delay;
1373
1374         /* Special case code for backing up a phase */
1375         if (*p == 0) {
1376                 *p = IO_DQS_EN_PHASE_MAX;
1377                 rw_mgr_decr_vfifo(*grp, v);
1378         } else {
1379                 (*p)--;
1380         }
1381         tmp_delay = *work_bgn - IO_DELAY_PER_OPA_TAP;
1382         scc_mgr_set_dqs_en_phase_all_ranks(*grp, *p);
1383
1384         for (*d = 0; *d <= IO_DQS_EN_DELAY_MAX && tmp_delay < *work_bgn;
1385                 (*d)++, tmp_delay += IO_DELAY_PER_DQS_EN_DCHAIN_TAP) {
1386                 scc_mgr_set_dqs_en_delay_all_ranks(*grp, *d);
1387
1388                 if (rw_mgr_mem_calibrate_read_test_all_ranks(*grp, 1,
1389                                                              PASS_ONE_BIT,
1390                                                              bit_chk, 0)) {
1391                         found_begin = 1;
1392                         *work_bgn = tmp_delay;
1393                         break;
1394                 }
1395         }
1396
1397         /* We have found a working dtap before the ptap found above */
1398         if (found_begin == 1)
1399                 (*max_working_cnt)++;
1400
1401         /*
1402          * Restore VFIFO to old state before we decremented it
1403          * (if needed).
1404          */
1405         (*p)++;
1406         if (*p > IO_DQS_EN_PHASE_MAX) {
1407                 *p = 0;
1408                 rw_mgr_incr_vfifo(*grp, v);
1409         }
1410
1411         scc_mgr_set_dqs_en_delay_all_ranks(*grp, 0);
1412 }
1413
1414 static int sdr_nonworking_phase(uint32_t *grp, uint32_t *bit_chk,
1415                              uint32_t *work_bgn, uint32_t *v, uint32_t *d,
1416                              uint32_t *p, uint32_t *i, uint32_t *max_working_cnt,
1417                              uint32_t *work_end)
1418 {
1419         uint32_t found_end = 0;
1420
1421         (*p)++;
1422         *work_end += IO_DELAY_PER_OPA_TAP;
1423         if (*p > IO_DQS_EN_PHASE_MAX) {
1424                 /* fiddle with FIFO */
1425                 *p = 0;
1426                 rw_mgr_incr_vfifo(*grp, v);
1427         }
1428
1429         for (; *i < VFIFO_SIZE + 1; (*i)++) {
1430                 for (; *p <= IO_DQS_EN_PHASE_MAX; (*p)++, *work_end
1431                         += IO_DELAY_PER_OPA_TAP) {
1432                         scc_mgr_set_dqs_en_phase_all_ranks(*grp, *p);
1433
1434                         if (!rw_mgr_mem_calibrate_read_test_all_ranks
1435                                 (*grp, 1, PASS_ONE_BIT, bit_chk, 0)) {
1436                                 found_end = 1;
1437                                 break;
1438                         } else {
1439                                 (*max_working_cnt)++;
1440                         }
1441                 }
1442
1443                 if (found_end)
1444                         break;
1445
1446                 if (*p > IO_DQS_EN_PHASE_MAX) {
1447                         /* fiddle with FIFO */
1448                         rw_mgr_incr_vfifo(*grp, v);
1449                         *p = 0;
1450                 }
1451         }
1452
1453         if (*i >= VFIFO_SIZE + 1) {
1454                 /* cannot see edge of failing read */
1455                 debug_cond(DLEVEL == 2, "%s:%d sdr_nonworking_phase: end:\
1456                            failed\n", __func__, __LINE__);
1457                 return 0;
1458         } else {
1459                 return 1;
1460         }
1461 }
1462
1463 static int sdr_find_window_centre(uint32_t *grp, uint32_t *bit_chk,
1464                                   uint32_t *work_bgn, uint32_t *v, uint32_t *d,
1465                                   uint32_t *p, uint32_t *work_mid,
1466                                   uint32_t *work_end)
1467 {
1468         int i;
1469         int tmp_delay = 0;
1470
1471         *work_mid = (*work_bgn + *work_end) / 2;
1472
1473         debug_cond(DLEVEL == 2, "work_bgn=%d work_end=%d work_mid=%d\n",
1474                    *work_bgn, *work_end, *work_mid);
1475         /* Get the middle delay to be less than a VFIFO delay */
1476         for (*p = 0; *p <= IO_DQS_EN_PHASE_MAX;
1477                 (*p)++, tmp_delay += IO_DELAY_PER_OPA_TAP)
1478                 ;
1479         debug_cond(DLEVEL == 2, "vfifo ptap delay %d\n", tmp_delay);
1480         while (*work_mid > tmp_delay)
1481                 *work_mid -= tmp_delay;
1482         debug_cond(DLEVEL == 2, "new work_mid %d\n", *work_mid);
1483
1484         tmp_delay = 0;
1485         for (*p = 0; *p <= IO_DQS_EN_PHASE_MAX && tmp_delay < *work_mid;
1486                 (*p)++, tmp_delay += IO_DELAY_PER_OPA_TAP)
1487                 ;
1488         tmp_delay -= IO_DELAY_PER_OPA_TAP;
1489         debug_cond(DLEVEL == 2, "new p %d, tmp_delay=%d\n", (*p) - 1, tmp_delay);
1490         for (*d = 0; *d <= IO_DQS_EN_DELAY_MAX && tmp_delay < *work_mid; (*d)++,
1491                 tmp_delay += IO_DELAY_PER_DQS_EN_DCHAIN_TAP)
1492                 ;
1493         debug_cond(DLEVEL == 2, "new d %d, tmp_delay=%d\n", *d, tmp_delay);
1494
1495         scc_mgr_set_dqs_en_phase_all_ranks(*grp, (*p) - 1);
1496         scc_mgr_set_dqs_en_delay_all_ranks(*grp, *d);
1497
1498         /*
1499          * push vfifo until we can successfully calibrate. We can do this
1500          * because the largest possible margin in 1 VFIFO cycle.
1501          */
1502         for (i = 0; i < VFIFO_SIZE; i++) {
1503                 debug_cond(DLEVEL == 2, "find_dqs_en_phase: center: vfifo=%u\n",
1504                            *v);
1505                 if (rw_mgr_mem_calibrate_read_test_all_ranks(*grp, 1,
1506                                                              PASS_ONE_BIT,
1507                                                              bit_chk, 0)) {
1508                         break;
1509                 }
1510
1511                 /* fiddle with FIFO */
1512                 rw_mgr_incr_vfifo(*grp, v);
1513         }
1514
1515         if (i >= VFIFO_SIZE) {
1516                 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: center: \
1517                            failed\n", __func__, __LINE__);
1518                 return 0;
1519         } else {
1520                 return 1;
1521         }
1522 }
1523
1524 /* find a good dqs enable to use */
1525 static uint32_t rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase(uint32_t grp)
1526 {
1527         uint32_t v, d, p, i;
1528         uint32_t max_working_cnt;
1529         uint32_t bit_chk;
1530         uint32_t dtaps_per_ptap;
1531         uint32_t work_bgn, work_mid, work_end;
1532         uint32_t found_passing_read, found_failing_read, initial_failing_dtap;
1533
1534         debug("%s:%d %u\n", __func__, __LINE__, grp);
1535
1536         reg_file_set_sub_stage(CAL_SUBSTAGE_VFIFO_CENTER);
1537
1538         scc_mgr_set_dqs_en_delay_all_ranks(grp, 0);
1539         scc_mgr_set_dqs_en_phase_all_ranks(grp, 0);
1540
1541         /* ************************************************************** */
1542         /* * Step 0 : Determine number of delay taps for each phase tap * */
1543         dtaps_per_ptap = IO_DELAY_PER_OPA_TAP/IO_DELAY_PER_DQS_EN_DCHAIN_TAP;
1544
1545         /* ********************************************************* */
1546         /* * Step 1 : First push vfifo until we get a failing read * */
1547         v = find_vfifo_read(grp, &bit_chk);
1548
1549         max_working_cnt = 0;
1550
1551         /* ******************************************************** */
1552         /* * step 2: find first working phase, increment in ptaps * */
1553         work_bgn = 0;
1554         if (find_working_phase(&grp, &bit_chk, dtaps_per_ptap, &work_bgn, &v, &d,
1555                                 &p, &i, &max_working_cnt) == 0)
1556                 return 0;
1557
1558         work_end = work_bgn;
1559
1560         /*
1561          * If d is 0 then the working window covers a phase tap and
1562          * we can follow the old procedure otherwise, we've found the beginning,
1563          * and we need to increment the dtaps until we find the end.
1564          */
1565         if (d == 0) {
1566                 /* ********************************************************* */
1567                 /* * step 3a: if we have room, back off by one and
1568                 increment in dtaps * */
1569
1570                 sdr_backup_phase(&grp, &bit_chk, &work_bgn, &v, &d, &p,
1571                                  &max_working_cnt);
1572
1573                 /* ********************************************************* */
1574                 /* * step 4a: go forward from working phase to non working
1575                 phase, increment in ptaps * */
1576                 if (sdr_nonworking_phase(&grp, &bit_chk, &work_bgn, &v, &d, &p,
1577                                          &i, &max_working_cnt, &work_end) == 0)
1578                         return 0;
1579
1580                 /* ********************************************************* */
1581                 /* * step 5a:  back off one from last, increment in dtaps  * */
1582
1583                 /* Special case code for backing up a phase */
1584                 if (p == 0) {
1585                         p = IO_DQS_EN_PHASE_MAX;
1586                         rw_mgr_decr_vfifo(grp, &v);
1587                 } else {
1588                         p = p - 1;
1589                 }
1590
1591                 work_end -= IO_DELAY_PER_OPA_TAP;
1592                 scc_mgr_set_dqs_en_phase_all_ranks(grp, p);
1593
1594                 /* * The actual increment of dtaps is done outside of
1595                 the if/else loop to share code */
1596                 d = 0;
1597
1598                 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: v/p: \
1599                            vfifo=%u ptap=%u\n", __func__, __LINE__,
1600                            v, p);
1601         } else {
1602                 /* ******************************************************* */
1603                 /* * step 3-5b:  Find the right edge of the window using
1604                 delay taps   * */
1605                 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase:vfifo=%u \
1606                            ptap=%u dtap=%u bgn=%u\n", __func__, __LINE__,
1607                            v, p, d, work_bgn);
1608
1609                 work_end = work_bgn;
1610
1611                 /* * The actual increment of dtaps is done outside of the
1612                 if/else loop to share code */
1613
1614                 /* Only here to counterbalance a subtract later on which is
1615                 not needed if this branch of the algorithm is taken */
1616                 max_working_cnt++;
1617         }
1618
1619         /* The dtap increment to find the failing edge is done here */
1620         for (; d <= IO_DQS_EN_DELAY_MAX; d++, work_end +=
1621                 IO_DELAY_PER_DQS_EN_DCHAIN_TAP) {
1622                         debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: \
1623                                    end-2: dtap=%u\n", __func__, __LINE__, d);
1624                         scc_mgr_set_dqs_en_delay_all_ranks(grp, d);
1625
1626                         if (!rw_mgr_mem_calibrate_read_test_all_ranks(grp, 1,
1627                                                                       PASS_ONE_BIT,
1628                                                                       &bit_chk, 0)) {
1629                                 break;
1630                         }
1631         }
1632
1633         /* Go back to working dtap */
1634         if (d != 0)
1635                 work_end -= IO_DELAY_PER_DQS_EN_DCHAIN_TAP;
1636
1637         debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: v/p/d: vfifo=%u \
1638                    ptap=%u dtap=%u end=%u\n", __func__, __LINE__,
1639                    v, p, d-1, work_end);
1640
1641         if (work_end < work_bgn) {
1642                 /* nil range */
1643                 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: end-2: \
1644                            failed\n", __func__, __LINE__);
1645                 return 0;
1646         }
1647
1648         debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: found range [%u,%u]\n",
1649                    __func__, __LINE__, work_bgn, work_end);
1650
1651         /* *************************************************************** */
1652         /*
1653          * * We need to calculate the number of dtaps that equal a ptap
1654          * * To do that we'll back up a ptap and re-find the edge of the
1655          * * window using dtaps
1656          */
1657
1658         debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: calculate dtaps_per_ptap \
1659                    for tracking\n", __func__, __LINE__);
1660
1661         /* Special case code for backing up a phase */
1662         if (p == 0) {
1663                 p = IO_DQS_EN_PHASE_MAX;
1664                 rw_mgr_decr_vfifo(grp, &v);
1665                 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: backedup \
1666                            cycle/phase: v=%u p=%u\n", __func__, __LINE__,
1667                            v, p);
1668         } else {
1669                 p = p - 1;
1670                 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: backedup \
1671                            phase only: v=%u p=%u", __func__, __LINE__,
1672                            v, p);
1673         }
1674
1675         scc_mgr_set_dqs_en_phase_all_ranks(grp, p);
1676
1677         /*
1678          * Increase dtap until we first see a passing read (in case the
1679          * window is smaller than a ptap),
1680          * and then a failing read to mark the edge of the window again
1681          */
1682
1683         /* Find a passing read */
1684         debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: find passing read\n",
1685                    __func__, __LINE__);
1686         found_passing_read = 0;
1687         found_failing_read = 0;
1688         initial_failing_dtap = d;
1689         for (; d <= IO_DQS_EN_DELAY_MAX; d++) {
1690                 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: testing \
1691                            read d=%u\n", __func__, __LINE__, d);
1692                 scc_mgr_set_dqs_en_delay_all_ranks(grp, d);
1693
1694                 if (rw_mgr_mem_calibrate_read_test_all_ranks(grp, 1,
1695                                                              PASS_ONE_BIT,
1696                                                              &bit_chk, 0)) {
1697                         found_passing_read = 1;
1698                         break;
1699                 }
1700         }
1701
1702         if (found_passing_read) {
1703                 /* Find a failing read */
1704                 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: find failing \
1705                            read\n", __func__, __LINE__);
1706                 for (d = d + 1; d <= IO_DQS_EN_DELAY_MAX; d++) {
1707                         debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: \
1708                                    testing read d=%u\n", __func__, __LINE__, d);
1709                         scc_mgr_set_dqs_en_delay_all_ranks(grp, d);
1710
1711                         if (!rw_mgr_mem_calibrate_read_test_all_ranks
1712                                 (grp, 1, PASS_ONE_BIT, &bit_chk, 0)) {
1713                                 found_failing_read = 1;
1714                                 break;
1715                         }
1716                 }
1717         } else {
1718                 debug_cond(DLEVEL == 1, "%s:%d find_dqs_en_phase: failed to \
1719                            calculate dtaps", __func__, __LINE__);
1720                 debug_cond(DLEVEL == 1, "per ptap. Fall back on static value\n");
1721         }
1722
1723         /*
1724          * The dynamically calculated dtaps_per_ptap is only valid if we
1725          * found a passing/failing read. If we didn't, it means d hit the max
1726          * (IO_DQS_EN_DELAY_MAX). Otherwise, dtaps_per_ptap retains its
1727          * statically calculated value.
1728          */
1729         if (found_passing_read && found_failing_read)
1730                 dtaps_per_ptap = d - initial_failing_dtap;
1731
1732         writel(dtaps_per_ptap, &sdr_reg_file->dtaps_per_ptap);
1733         debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: dtaps_per_ptap=%u \
1734                    - %u = %u",  __func__, __LINE__, d,
1735                    initial_failing_dtap, dtaps_per_ptap);
1736
1737         /* ******************************************** */
1738         /* * step 6:  Find the centre of the window   * */
1739         if (sdr_find_window_centre(&grp, &bit_chk, &work_bgn, &v, &d, &p,
1740                                    &work_mid, &work_end) == 0)
1741                 return 0;
1742
1743         debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: center found: \
1744                    vfifo=%u ptap=%u dtap=%u\n", __func__, __LINE__,
1745                    v, p-1, d);
1746         return 1;
1747 }
1748
1749 /*
1750  * Try rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase across different
1751  * dq_in_delay values
1752  */
1753 static uint32_t
1754 rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase_sweep_dq_in_delay
1755 (uint32_t write_group, uint32_t read_group, uint32_t test_bgn)
1756 {
1757         uint32_t found;
1758         uint32_t i;
1759         uint32_t p;
1760         uint32_t d;
1761         uint32_t r;
1762
1763         const uint32_t delay_step = IO_IO_IN_DELAY_MAX /
1764                 (RW_MGR_MEM_DQ_PER_READ_DQS-1);
1765                 /* we start at zero, so have one less dq to devide among */
1766
1767         debug("%s:%d (%u,%u,%u)", __func__, __LINE__, write_group, read_group,
1768               test_bgn);
1769
1770         /* try different dq_in_delays since the dq path is shorter than dqs */
1771
1772         for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS;
1773              r += NUM_RANKS_PER_SHADOW_REG) {
1774                 for (i = 0, p = test_bgn, d = 0; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++, p++, d += delay_step) {
1775                         debug_cond(DLEVEL == 1, "%s:%d rw_mgr_mem_calibrate_\
1776                                    vfifo_find_dqs_", __func__, __LINE__);
1777                         debug_cond(DLEVEL == 1, "en_phase_sweep_dq_in_delay: g=%u/%u ",
1778                                write_group, read_group);
1779                         debug_cond(DLEVEL == 1, "r=%u, i=%u p=%u d=%u\n", r, i , p, d);
1780                         scc_mgr_set_dq_in_delay(p, d);
1781                         scc_mgr_load_dq(p);
1782                 }
1783                 writel(0, &sdr_scc_mgr->update);
1784         }
1785
1786         found = rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase(read_group);
1787
1788         debug_cond(DLEVEL == 1, "%s:%d rw_mgr_mem_calibrate_vfifo_find_dqs_\
1789                    en_phase_sweep_dq", __func__, __LINE__);
1790         debug_cond(DLEVEL == 1, "_in_delay: g=%u/%u found=%u; Reseting delay \
1791                    chain to zero\n", write_group, read_group, found);
1792
1793         for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS;
1794              r += NUM_RANKS_PER_SHADOW_REG) {
1795                 for (i = 0, p = test_bgn; i < RW_MGR_MEM_DQ_PER_READ_DQS;
1796                         i++, p++) {
1797                         scc_mgr_set_dq_in_delay(p, 0);
1798                         scc_mgr_load_dq(p);
1799                 }
1800                 writel(0, &sdr_scc_mgr->update);
1801         }
1802
1803         return found;
1804 }
1805
1806 /* per-bit deskew DQ and center */
1807 static uint32_t rw_mgr_mem_calibrate_vfifo_center(uint32_t rank_bgn,
1808         uint32_t write_group, uint32_t read_group, uint32_t test_bgn,
1809         uint32_t use_read_test, uint32_t update_fom)
1810 {
1811         uint32_t i, p, d, min_index;
1812         /*
1813          * Store these as signed since there are comparisons with
1814          * signed numbers.
1815          */
1816         uint32_t bit_chk;
1817         uint32_t sticky_bit_chk;
1818         int32_t left_edge[RW_MGR_MEM_DQ_PER_READ_DQS];
1819         int32_t right_edge[RW_MGR_MEM_DQ_PER_READ_DQS];
1820         int32_t final_dq[RW_MGR_MEM_DQ_PER_READ_DQS];
1821         int32_t mid;
1822         int32_t orig_mid_min, mid_min;
1823         int32_t new_dqs, start_dqs, start_dqs_en, shift_dq, final_dqs,
1824                 final_dqs_en;
1825         int32_t dq_margin, dqs_margin;
1826         uint32_t stop;
1827         uint32_t temp_dq_in_delay1, temp_dq_in_delay2;
1828         uint32_t addr;
1829
1830         debug("%s:%d: %u %u", __func__, __LINE__, read_group, test_bgn);
1831
1832         addr = SDR_PHYGRP_SCCGRP_ADDRESS | SCC_MGR_DQS_IN_DELAY_OFFSET;
1833         start_dqs = readl(addr + (read_group << 2));
1834         if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS)
1835                 start_dqs_en = readl(addr + ((read_group << 2)
1836                                      - IO_DQS_EN_DELAY_OFFSET));
1837
1838         /* set the left and right edge of each bit to an illegal value */
1839         /* use (IO_IO_IN_DELAY_MAX + 1) as an illegal value */
1840         sticky_bit_chk = 0;
1841         for (i = 0; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++) {
1842                 left_edge[i]  = IO_IO_IN_DELAY_MAX + 1;
1843                 right_edge[i] = IO_IO_IN_DELAY_MAX + 1;
1844         }
1845
1846         /* Search for the left edge of the window for each bit */
1847         for (d = 0; d <= IO_IO_IN_DELAY_MAX; d++) {
1848                 scc_mgr_apply_group_dq_in_delay(write_group, test_bgn, d);
1849
1850                 writel(0, &sdr_scc_mgr->update);
1851
1852                 /*
1853                  * Stop searching when the read test doesn't pass AND when
1854                  * we've seen a passing read on every bit.
1855                  */
1856                 if (use_read_test) {
1857                         stop = !rw_mgr_mem_calibrate_read_test(rank_bgn,
1858                                 read_group, NUM_READ_PB_TESTS, PASS_ONE_BIT,
1859                                 &bit_chk, 0, 0);
1860                 } else {
1861                         rw_mgr_mem_calibrate_write_test(rank_bgn, write_group,
1862                                                         0, PASS_ONE_BIT,
1863                                                         &bit_chk, 0);
1864                         bit_chk = bit_chk >> (RW_MGR_MEM_DQ_PER_READ_DQS *
1865                                 (read_group - (write_group *
1866                                         RW_MGR_MEM_IF_READ_DQS_WIDTH /
1867                                         RW_MGR_MEM_IF_WRITE_DQS_WIDTH)));
1868                         stop = (bit_chk == 0);
1869                 }
1870                 sticky_bit_chk = sticky_bit_chk | bit_chk;
1871                 stop = stop && (sticky_bit_chk == param->read_correct_mask);
1872                 debug_cond(DLEVEL == 2, "%s:%d vfifo_center(left): dtap=%u => %u == %u \
1873                            && %u", __func__, __LINE__, d,
1874                            sticky_bit_chk,
1875                         param->read_correct_mask, stop);
1876
1877                 if (stop == 1) {
1878                         break;
1879                 } else {
1880                         for (i = 0; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++) {
1881                                 if (bit_chk & 1) {
1882                                         /* Remember a passing test as the
1883                                         left_edge */
1884                                         left_edge[i] = d;
1885                                 } else {
1886                                         /* If a left edge has not been seen yet,
1887                                         then a future passing test will mark
1888                                         this edge as the right edge */
1889                                         if (left_edge[i] ==
1890                                                 IO_IO_IN_DELAY_MAX + 1) {
1891                                                 right_edge[i] = -(d + 1);
1892                                         }
1893                                 }
1894                                 bit_chk = bit_chk >> 1;
1895                         }
1896                 }
1897         }
1898
1899         /* Reset DQ delay chains to 0 */
1900         scc_mgr_apply_group_dq_in_delay(test_bgn, 0);
1901         sticky_bit_chk = 0;
1902         for (i = RW_MGR_MEM_DQ_PER_READ_DQS - 1;; i--) {
1903                 debug_cond(DLEVEL == 2, "%s:%d vfifo_center: left_edge[%u]: \
1904                            %d right_edge[%u]: %d\n", __func__, __LINE__,
1905                            i, left_edge[i], i, right_edge[i]);
1906
1907                 /*
1908                  * Check for cases where we haven't found the left edge,
1909                  * which makes our assignment of the the right edge invalid.
1910                  * Reset it to the illegal value.
1911                  */
1912                 if ((left_edge[i] == IO_IO_IN_DELAY_MAX + 1) && (
1913                         right_edge[i] != IO_IO_IN_DELAY_MAX + 1)) {
1914                         right_edge[i] = IO_IO_IN_DELAY_MAX + 1;
1915                         debug_cond(DLEVEL == 2, "%s:%d vfifo_center: reset \
1916                                    right_edge[%u]: %d\n", __func__, __LINE__,
1917                                    i, right_edge[i]);
1918                 }
1919
1920                 /*
1921                  * Reset sticky bit (except for bits where we have seen
1922                  * both the left and right edge).
1923                  */
1924                 sticky_bit_chk = sticky_bit_chk << 1;
1925                 if ((left_edge[i] != IO_IO_IN_DELAY_MAX + 1) &&
1926                     (right_edge[i] != IO_IO_IN_DELAY_MAX + 1)) {
1927                         sticky_bit_chk = sticky_bit_chk | 1;
1928                 }
1929
1930                 if (i == 0)
1931                         break;
1932         }
1933
1934         /* Search for the right edge of the window for each bit */
1935         for (d = 0; d <= IO_DQS_IN_DELAY_MAX - start_dqs; d++) {
1936                 scc_mgr_set_dqs_bus_in_delay(read_group, d + start_dqs);
1937                 if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS) {
1938                         uint32_t delay = d + start_dqs_en;
1939                         if (delay > IO_DQS_EN_DELAY_MAX)
1940                                 delay = IO_DQS_EN_DELAY_MAX;
1941                         scc_mgr_set_dqs_en_delay(read_group, delay);
1942                 }
1943                 scc_mgr_load_dqs(read_group);
1944
1945                 writel(0, &sdr_scc_mgr->update);
1946
1947                 /*
1948                  * Stop searching when the read test doesn't pass AND when
1949                  * we've seen a passing read on every bit.
1950                  */
1951                 if (use_read_test) {
1952                         stop = !rw_mgr_mem_calibrate_read_test(rank_bgn,
1953                                 read_group, NUM_READ_PB_TESTS, PASS_ONE_BIT,
1954                                 &bit_chk, 0, 0);
1955                 } else {
1956                         rw_mgr_mem_calibrate_write_test(rank_bgn, write_group,
1957                                                         0, PASS_ONE_BIT,
1958                                                         &bit_chk, 0);
1959                         bit_chk = bit_chk >> (RW_MGR_MEM_DQ_PER_READ_DQS *
1960                                 (read_group - (write_group *
1961                                         RW_MGR_MEM_IF_READ_DQS_WIDTH /
1962                                         RW_MGR_MEM_IF_WRITE_DQS_WIDTH)));
1963                         stop = (bit_chk == 0);
1964                 }
1965                 sticky_bit_chk = sticky_bit_chk | bit_chk;
1966                 stop = stop && (sticky_bit_chk == param->read_correct_mask);
1967
1968                 debug_cond(DLEVEL == 2, "%s:%d vfifo_center(right): dtap=%u => %u == \
1969                            %u && %u", __func__, __LINE__, d,
1970                            sticky_bit_chk, param->read_correct_mask, stop);
1971
1972                 if (stop == 1) {
1973                         break;
1974                 } else {
1975                         for (i = 0; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++) {
1976                                 if (bit_chk & 1) {
1977                                         /* Remember a passing test as
1978                                         the right_edge */
1979                                         right_edge[i] = d;
1980                                 } else {
1981                                         if (d != 0) {
1982                                                 /* If a right edge has not been
1983                                                 seen yet, then a future passing
1984                                                 test will mark this edge as the
1985                                                 left edge */
1986                                                 if (right_edge[i] ==
1987                                                 IO_IO_IN_DELAY_MAX + 1) {
1988                                                         left_edge[i] = -(d + 1);
1989                                                 }
1990                                         } else {
1991                                                 /* d = 0 failed, but it passed
1992                                                 when testing the left edge,
1993                                                 so it must be marginal,
1994                                                 set it to -1 */
1995                                                 if (right_edge[i] ==
1996                                                         IO_IO_IN_DELAY_MAX + 1 &&
1997                                                         left_edge[i] !=
1998                                                         IO_IO_IN_DELAY_MAX
1999                                                         + 1) {
2000                                                         right_edge[i] = -1;
2001                                                 }
2002                                                 /* If a right edge has not been
2003                                                 seen yet, then a future passing
2004                                                 test will mark this edge as the
2005                                                 left edge */
2006                                                 else if (right_edge[i] ==
2007                                                         IO_IO_IN_DELAY_MAX +
2008                                                         1) {
2009                                                         left_edge[i] = -(d + 1);
2010                                                 }
2011                                         }
2012                                 }
2013
2014                                 debug_cond(DLEVEL == 2, "%s:%d vfifo_center[r,\
2015                                            d=%u]: ", __func__, __LINE__, d);
2016                                 debug_cond(DLEVEL == 2, "bit_chk_test=%d left_edge[%u]: %d ",
2017                                            (int)(bit_chk & 1), i, left_edge[i]);
2018                                 debug_cond(DLEVEL == 2, "right_edge[%u]: %d\n", i,
2019                                            right_edge[i]);
2020                                 bit_chk = bit_chk >> 1;
2021                         }
2022                 }
2023         }
2024
2025         /* Check that all bits have a window */
2026         for (i = 0; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++) {
2027                 debug_cond(DLEVEL == 2, "%s:%d vfifo_center: left_edge[%u]: \
2028                            %d right_edge[%u]: %d", __func__, __LINE__,
2029                            i, left_edge[i], i, right_edge[i]);
2030                 if ((left_edge[i] == IO_IO_IN_DELAY_MAX + 1) || (right_edge[i]
2031                         == IO_IO_IN_DELAY_MAX + 1)) {
2032                         /*
2033                          * Restore delay chain settings before letting the loop
2034                          * in rw_mgr_mem_calibrate_vfifo to retry different
2035                          * dqs/ck relationships.
2036                          */
2037                         scc_mgr_set_dqs_bus_in_delay(read_group, start_dqs);
2038                         if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS) {
2039                                 scc_mgr_set_dqs_en_delay(read_group,
2040                                                          start_dqs_en);
2041                         }
2042                         scc_mgr_load_dqs(read_group);
2043                         writel(0, &sdr_scc_mgr->update);
2044
2045                         debug_cond(DLEVEL == 1, "%s:%d vfifo_center: failed to \
2046                                    find edge [%u]: %d %d", __func__, __LINE__,
2047                                    i, left_edge[i], right_edge[i]);
2048                         if (use_read_test) {
2049                                 set_failing_group_stage(read_group *
2050                                         RW_MGR_MEM_DQ_PER_READ_DQS + i,
2051                                         CAL_STAGE_VFIFO,
2052                                         CAL_SUBSTAGE_VFIFO_CENTER);
2053                         } else {
2054                                 set_failing_group_stage(read_group *
2055                                         RW_MGR_MEM_DQ_PER_READ_DQS + i,
2056                                         CAL_STAGE_VFIFO_AFTER_WRITES,
2057                                         CAL_SUBSTAGE_VFIFO_CENTER);
2058                         }
2059                         return 0;
2060                 }
2061         }
2062
2063         /* Find middle of window for each DQ bit */
2064         mid_min = left_edge[0] - right_edge[0];
2065         min_index = 0;
2066         for (i = 1; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++) {
2067                 mid = left_edge[i] - right_edge[i];
2068                 if (mid < mid_min) {
2069                         mid_min = mid;
2070                         min_index = i;
2071                 }
2072         }
2073
2074         /*
2075          * -mid_min/2 represents the amount that we need to move DQS.
2076          * If mid_min is odd and positive we'll need to add one to
2077          * make sure the rounding in further calculations is correct
2078          * (always bias to the right), so just add 1 for all positive values.
2079          */
2080         if (mid_min > 0)
2081                 mid_min++;
2082
2083         mid_min = mid_min / 2;
2084
2085         debug_cond(DLEVEL == 1, "%s:%d vfifo_center: mid_min=%d (index=%u)\n",
2086                    __func__, __LINE__, mid_min, min_index);
2087
2088         /* Determine the amount we can change DQS (which is -mid_min) */
2089         orig_mid_min = mid_min;
2090         new_dqs = start_dqs - mid_min;
2091         if (new_dqs > IO_DQS_IN_DELAY_MAX)
2092                 new_dqs = IO_DQS_IN_DELAY_MAX;
2093         else if (new_dqs < 0)
2094                 new_dqs = 0;
2095
2096         mid_min = start_dqs - new_dqs;
2097         debug_cond(DLEVEL == 1, "vfifo_center: new mid_min=%d new_dqs=%d\n",
2098                    mid_min, new_dqs);
2099
2100         if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS) {
2101                 if (start_dqs_en - mid_min > IO_DQS_EN_DELAY_MAX)
2102                         mid_min += start_dqs_en - mid_min - IO_DQS_EN_DELAY_MAX;
2103                 else if (start_dqs_en - mid_min < 0)
2104                         mid_min += start_dqs_en - mid_min;
2105         }
2106         new_dqs = start_dqs - mid_min;
2107
2108         debug_cond(DLEVEL == 1, "vfifo_center: start_dqs=%d start_dqs_en=%d \
2109                    new_dqs=%d mid_min=%d\n", start_dqs,
2110                    IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS ? start_dqs_en : -1,
2111                    new_dqs, mid_min);
2112
2113         /* Initialize data for export structures */
2114         dqs_margin = IO_IO_IN_DELAY_MAX + 1;
2115         dq_margin  = IO_IO_IN_DELAY_MAX + 1;
2116
2117         /* add delay to bring centre of all DQ windows to the same "level" */
2118         for (i = 0, p = test_bgn; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++, p++) {
2119                 /* Use values before divide by 2 to reduce round off error */
2120                 shift_dq = (left_edge[i] - right_edge[i] -
2121                         (left_edge[min_index] - right_edge[min_index]))/2  +
2122                         (orig_mid_min - mid_min);
2123
2124                 debug_cond(DLEVEL == 2, "vfifo_center: before: \
2125                            shift_dq[%u]=%d\n", i, shift_dq);
2126
2127                 addr = SDR_PHYGRP_SCCGRP_ADDRESS | SCC_MGR_IO_IN_DELAY_OFFSET;
2128                 temp_dq_in_delay1 = readl(addr + (p << 2));
2129                 temp_dq_in_delay2 = readl(addr + (i << 2));
2130
2131                 if (shift_dq + (int32_t)temp_dq_in_delay1 >
2132                         (int32_t)IO_IO_IN_DELAY_MAX) {
2133                         shift_dq = (int32_t)IO_IO_IN_DELAY_MAX - temp_dq_in_delay2;
2134                 } else if (shift_dq + (int32_t)temp_dq_in_delay1 < 0) {
2135                         shift_dq = -(int32_t)temp_dq_in_delay1;
2136                 }
2137                 debug_cond(DLEVEL == 2, "vfifo_center: after: \
2138                            shift_dq[%u]=%d\n", i, shift_dq);
2139                 final_dq[i] = temp_dq_in_delay1 + shift_dq;
2140                 scc_mgr_set_dq_in_delay(p, final_dq[i]);
2141                 scc_mgr_load_dq(p);
2142
2143                 debug_cond(DLEVEL == 2, "vfifo_center: margin[%u]=[%d,%d]\n", i,
2144                            left_edge[i] - shift_dq + (-mid_min),
2145                            right_edge[i] + shift_dq - (-mid_min));
2146                 /* To determine values for export structures */
2147                 if (left_edge[i] - shift_dq + (-mid_min) < dq_margin)
2148                         dq_margin = left_edge[i] - shift_dq + (-mid_min);
2149
2150                 if (right_edge[i] + shift_dq - (-mid_min) < dqs_margin)
2151                         dqs_margin = right_edge[i] + shift_dq - (-mid_min);
2152         }
2153
2154         final_dqs = new_dqs;
2155         if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS)
2156                 final_dqs_en = start_dqs_en - mid_min;
2157
2158         /* Move DQS-en */
2159         if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS) {
2160                 scc_mgr_set_dqs_en_delay(read_group, final_dqs_en);
2161                 scc_mgr_load_dqs(read_group);
2162         }
2163
2164         /* Move DQS */
2165         scc_mgr_set_dqs_bus_in_delay(read_group, final_dqs);
2166         scc_mgr_load_dqs(read_group);
2167         debug_cond(DLEVEL == 2, "%s:%d vfifo_center: dq_margin=%d \
2168                    dqs_margin=%d", __func__, __LINE__,
2169                    dq_margin, dqs_margin);
2170
2171         /*
2172          * Do not remove this line as it makes sure all of our decisions
2173          * have been applied. Apply the update bit.
2174          */
2175         writel(0, &sdr_scc_mgr->update);
2176
2177         return (dq_margin >= 0) && (dqs_margin >= 0);
2178 }
2179
2180 /*
2181  * calibrate the read valid prediction FIFO.
2182  *
2183  *  - read valid prediction will consist of finding a good DQS enable phase,
2184  * DQS enable delay, DQS input phase, and DQS input delay.
2185  *  - we also do a per-bit deskew on the DQ lines.
2186  */
2187 static uint32_t rw_mgr_mem_calibrate_vfifo(uint32_t read_group,
2188                                            uint32_t test_bgn)
2189 {
2190         uint32_t p, d, rank_bgn, sr;
2191         uint32_t dtaps_per_ptap;
2192         uint32_t bit_chk;
2193         uint32_t grp_calibrated;
2194         uint32_t write_group, write_test_bgn;
2195         uint32_t failed_substage;
2196
2197         debug("%s:%d: %u %u\n", __func__, __LINE__, read_group, test_bgn);
2198
2199         /* update info for sims */
2200         reg_file_set_stage(CAL_STAGE_VFIFO);
2201
2202         write_group = read_group;
2203         write_test_bgn = test_bgn;
2204
2205         /* USER Determine number of delay taps for each phase tap */
2206         dtaps_per_ptap = DIV_ROUND_UP(IO_DELAY_PER_OPA_TAP,
2207                                       IO_DELAY_PER_DQS_EN_DCHAIN_TAP) - 1;
2208
2209         /* update info for sims */
2210         reg_file_set_group(read_group);
2211
2212         grp_calibrated = 0;
2213
2214         reg_file_set_sub_stage(CAL_SUBSTAGE_GUARANTEED_READ);
2215         failed_substage = CAL_SUBSTAGE_GUARANTEED_READ;
2216
2217         for (d = 0; d <= dtaps_per_ptap && grp_calibrated == 0; d += 2) {
2218                 /*
2219                  * In RLDRAMX we may be messing the delay of pins in
2220                  * the same write group but outside of the current read
2221                  * the group, but that's ok because we haven't
2222                  * calibrated output side yet.
2223                  */
2224                 if (d > 0) {
2225                         scc_mgr_apply_group_all_out_delay_add_all_ranks(
2226                                                                 write_group, d);
2227                 }
2228
2229                 for (p = 0; p <= IO_DQDQS_OUT_PHASE_MAX && grp_calibrated == 0;
2230                         p++) {
2231                         /* set a particular dqdqs phase */
2232                         scc_mgr_set_dqdqs_output_phase_all_ranks(read_group, p);
2233
2234                         debug_cond(DLEVEL == 1, "%s:%d calibrate_vfifo: g=%u \
2235                                    p=%u d=%u\n", __func__, __LINE__,
2236                                    read_group, p, d);
2237
2238                         /*
2239                          * Load up the patterns used by read calibration
2240                          * using current DQDQS phase.
2241                          */
2242                         rw_mgr_mem_calibrate_read_load_patterns(0, 1);
2243                         if (!(gbl->phy_debug_mode_flags &
2244                                 PHY_DEBUG_DISABLE_GUARANTEED_READ)) {
2245                                 if (!rw_mgr_mem_calibrate_read_test_patterns_all_ranks
2246                                     (read_group, 1, &bit_chk)) {
2247                                         debug_cond(DLEVEL == 1, "%s:%d Guaranteed read test failed:",
2248                                                    __func__, __LINE__);
2249                                         debug_cond(DLEVEL == 1, " g=%u p=%u d=%u\n",
2250                                                    read_group, p, d);
2251                                         break;
2252                                 }
2253                         }
2254
2255 /* case:56390 */
2256                         grp_calibrated = 1;
2257                 if (rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase_sweep_dq_in_delay
2258                     (write_group, read_group, test_bgn)) {
2259                                 /*
2260                                  * USER Read per-bit deskew can be done on a
2261                                  * per shadow register basis.
2262                                  */
2263                                 for (rank_bgn = 0, sr = 0;
2264                                         rank_bgn < RW_MGR_MEM_NUMBER_OF_RANKS;
2265                                         rank_bgn += NUM_RANKS_PER_SHADOW_REG,
2266                                         ++sr) {
2267                                         /*
2268                                          * Determine if this set of ranks
2269                                          * should be skipped entirely.
2270                                          */
2271                                         if (!param->skip_shadow_regs[sr]) {
2272                                                 /*
2273                                                  * If doing read after write
2274                                                  * calibration, do not update
2275                                                  * FOM, now - do it then.
2276                                                  */
2277                                         if (!rw_mgr_mem_calibrate_vfifo_center
2278                                                 (rank_bgn, write_group,
2279                                                 read_group, test_bgn, 1, 0)) {
2280                                                         grp_calibrated = 0;
2281                                                         failed_substage =
2282                                                 CAL_SUBSTAGE_VFIFO_CENTER;
2283                                                 }
2284                                         }
2285                                 }
2286                         } else {
2287                                 grp_calibrated = 0;
2288                                 failed_substage = CAL_SUBSTAGE_DQS_EN_PHASE;
2289                         }
2290                 }
2291         }
2292
2293         if (grp_calibrated == 0) {
2294                 set_failing_group_stage(write_group, CAL_STAGE_VFIFO,
2295                                         failed_substage);
2296                 return 0;
2297         }
2298
2299         /*
2300          * Reset the delay chains back to zero if they have moved > 1
2301          * (check for > 1 because loop will increase d even when pass in
2302          * first case).
2303          */
2304         if (d > 2)
2305                 scc_mgr_zero_group(write_group, 1);
2306
2307         return 1;
2308 }
2309
2310 /* VFIFO Calibration -- Read Deskew Calibration after write deskew */
2311 static uint32_t rw_mgr_mem_calibrate_vfifo_end(uint32_t read_group,
2312                                                uint32_t test_bgn)
2313 {
2314         uint32_t rank_bgn, sr;
2315         uint32_t grp_calibrated;
2316         uint32_t write_group;
2317
2318         debug("%s:%d %u %u", __func__, __LINE__, read_group, test_bgn);
2319
2320         /* update info for sims */
2321
2322         reg_file_set_stage(CAL_STAGE_VFIFO_AFTER_WRITES);
2323         reg_file_set_sub_stage(CAL_SUBSTAGE_VFIFO_CENTER);
2324
2325         write_group = read_group;
2326
2327         /* update info for sims */
2328         reg_file_set_group(read_group);
2329
2330         grp_calibrated = 1;
2331         /* Read per-bit deskew can be done on a per shadow register basis */
2332         for (rank_bgn = 0, sr = 0; rank_bgn < RW_MGR_MEM_NUMBER_OF_RANKS;
2333                 rank_bgn += NUM_RANKS_PER_SHADOW_REG, ++sr) {
2334                 /* Determine if this set of ranks should be skipped entirely */
2335                 if (!param->skip_shadow_regs[sr]) {
2336                 /* This is the last calibration round, update FOM here */
2337                         if (!rw_mgr_mem_calibrate_vfifo_center(rank_bgn,
2338                                                                 write_group,
2339                                                                 read_group,
2340                                                                 test_bgn, 0,
2341                                                                 1)) {
2342                                 grp_calibrated = 0;
2343                         }
2344                 }
2345         }
2346
2347
2348         if (grp_calibrated == 0) {
2349                 set_failing_group_stage(write_group,
2350                                         CAL_STAGE_VFIFO_AFTER_WRITES,
2351                                         CAL_SUBSTAGE_VFIFO_CENTER);
2352                 return 0;
2353         }
2354
2355         return 1;
2356 }
2357
2358 /* Calibrate LFIFO to find smallest read latency */
2359 static uint32_t rw_mgr_mem_calibrate_lfifo(void)
2360 {
2361         uint32_t found_one;
2362         uint32_t bit_chk;
2363
2364         debug("%s:%d\n", __func__, __LINE__);
2365
2366         /* update info for sims */
2367         reg_file_set_stage(CAL_STAGE_LFIFO);
2368         reg_file_set_sub_stage(CAL_SUBSTAGE_READ_LATENCY);
2369
2370         /* Load up the patterns used by read calibration for all ranks */
2371         rw_mgr_mem_calibrate_read_load_patterns(0, 1);
2372         found_one = 0;
2373
2374         do {
2375                 writel(gbl->curr_read_lat, &phy_mgr_cfg->phy_rlat);
2376                 debug_cond(DLEVEL == 2, "%s:%d lfifo: read_lat=%u",
2377                            __func__, __LINE__, gbl->curr_read_lat);
2378
2379                 if (!rw_mgr_mem_calibrate_read_test_all_ranks(0,
2380                                                               NUM_READ_TESTS,
2381                                                               PASS_ALL_BITS,
2382                                                               &bit_chk, 1)) {
2383                         break;
2384                 }
2385
2386                 found_one = 1;
2387                 /* reduce read latency and see if things are working */
2388                 /* correctly */
2389                 gbl->curr_read_lat--;
2390         } while (gbl->curr_read_lat > 0);
2391
2392         /* reset the fifos to get pointers to known state */
2393
2394         writel(0, &phy_mgr_cmd->fifo_reset);
2395
2396         if (found_one) {
2397                 /* add a fudge factor to the read latency that was determined */
2398                 gbl->curr_read_lat += 2;
2399                 writel(gbl->curr_read_lat, &phy_mgr_cfg->phy_rlat);
2400                 debug_cond(DLEVEL == 2, "%s:%d lfifo: success: using \
2401                            read_lat=%u\n", __func__, __LINE__,
2402                            gbl->curr_read_lat);
2403                 return 1;
2404         } else {
2405                 set_failing_group_stage(0xff, CAL_STAGE_LFIFO,
2406                                         CAL_SUBSTAGE_READ_LATENCY);
2407
2408                 debug_cond(DLEVEL == 2, "%s:%d lfifo: failed at initial \
2409                            read_lat=%u\n", __func__, __LINE__,
2410                            gbl->curr_read_lat);
2411                 return 0;
2412         }
2413 }
2414
2415 /*
2416  * issue write test command.
2417  * two variants are provided. one that just tests a write pattern and
2418  * another that tests datamask functionality.
2419  */
2420 static void rw_mgr_mem_calibrate_write_test_issue(uint32_t group,
2421                                                   uint32_t test_dm)
2422 {
2423         uint32_t mcc_instruction;
2424         uint32_t quick_write_mode = (((STATIC_CALIB_STEPS) & CALIB_SKIP_WRITES) &&
2425                 ENABLE_SUPER_QUICK_CALIBRATION);
2426         uint32_t rw_wl_nop_cycles;
2427         uint32_t addr;
2428
2429         /*
2430          * Set counter and jump addresses for the right
2431          * number of NOP cycles.
2432          * The number of supported NOP cycles can range from -1 to infinity
2433          * Three different cases are handled:
2434          *
2435          * 1. For a number of NOP cycles greater than 0, the RW Mgr looping
2436          *    mechanism will be used to insert the right number of NOPs
2437          *
2438          * 2. For a number of NOP cycles equals to 0, the micro-instruction
2439          *    issuing the write command will jump straight to the
2440          *    micro-instruction that turns on DQS (for DDRx), or outputs write
2441          *    data (for RLD), skipping
2442          *    the NOP micro-instruction all together
2443          *
2444          * 3. A number of NOP cycles equal to -1 indicates that DQS must be
2445          *    turned on in the same micro-instruction that issues the write
2446          *    command. Then we need
2447          *    to directly jump to the micro-instruction that sends out the data
2448          *
2449          * NOTE: Implementing this mechanism uses 2 RW Mgr jump-counters
2450          *       (2 and 3). One jump-counter (0) is used to perform multiple
2451          *       write-read operations.
2452          *       one counter left to issue this command in "multiple-group" mode
2453          */
2454
2455         rw_wl_nop_cycles = gbl->rw_wl_nop_cycles;
2456
2457         if (rw_wl_nop_cycles == -1) {
2458                 /*
2459                  * CNTR 2 - We want to execute the special write operation that
2460                  * turns on DQS right away and then skip directly to the
2461                  * instruction that sends out the data. We set the counter to a
2462                  * large number so that the jump is always taken.
2463                  */
2464                 writel(0xFF, &sdr_rw_load_mgr_regs->load_cntr2);
2465
2466                 /* CNTR 3 - Not used */
2467                 if (test_dm) {
2468                         mcc_instruction = RW_MGR_LFSR_WR_RD_DM_BANK_0_WL_1;
2469                         writel(RW_MGR_LFSR_WR_RD_DM_BANK_0_DATA,
2470                                &sdr_rw_load_jump_mgr_regs->load_jump_add2);
2471                         writel(RW_MGR_LFSR_WR_RD_DM_BANK_0_NOP,
2472                                &sdr_rw_load_jump_mgr_regs->load_jump_add3);
2473                 } else {
2474                         mcc_instruction = RW_MGR_LFSR_WR_RD_BANK_0_WL_1;
2475                         writel(RW_MGR_LFSR_WR_RD_BANK_0_DATA,
2476                                 &sdr_rw_load_jump_mgr_regs->load_jump_add2);
2477                         writel(RW_MGR_LFSR_WR_RD_BANK_0_NOP,
2478                                 &sdr_rw_load_jump_mgr_regs->load_jump_add3);
2479                 }
2480         } else if (rw_wl_nop_cycles == 0) {
2481                 /*
2482                  * CNTR 2 - We want to skip the NOP operation and go straight
2483                  * to the DQS enable instruction. We set the counter to a large
2484                  * number so that the jump is always taken.
2485                  */
2486                 writel(0xFF, &sdr_rw_load_mgr_regs->load_cntr2);
2487
2488                 /* CNTR 3 - Not used */
2489                 if (test_dm) {
2490                         mcc_instruction = RW_MGR_LFSR_WR_RD_DM_BANK_0;
2491                         writel(RW_MGR_LFSR_WR_RD_DM_BANK_0_DQS,
2492                                &sdr_rw_load_jump_mgr_regs->load_jump_add2);
2493                 } else {
2494                         mcc_instruction = RW_MGR_LFSR_WR_RD_BANK_0;
2495                         writel(RW_MGR_LFSR_WR_RD_BANK_0_DQS,
2496                                 &sdr_rw_load_jump_mgr_regs->load_jump_add2);
2497                 }
2498         } else {
2499                 /*
2500                  * CNTR 2 - In this case we want to execute the next instruction
2501                  * and NOT take the jump. So we set the counter to 0. The jump
2502                  * address doesn't count.
2503                  */
2504                 writel(0x0, &sdr_rw_load_mgr_regs->load_cntr2);
2505                 writel(0x0, &sdr_rw_load_jump_mgr_regs->load_jump_add2);
2506
2507                 /*
2508                  * CNTR 3 - Set the nop counter to the number of cycles we
2509                  * need to loop for, minus 1.
2510                  */
2511                 writel(rw_wl_nop_cycles - 1, &sdr_rw_load_mgr_regs->load_cntr3);
2512                 if (test_dm) {
2513                         mcc_instruction = RW_MGR_LFSR_WR_RD_DM_BANK_0;
2514                         writel(RW_MGR_LFSR_WR_RD_DM_BANK_0_NOP,
2515                                 &sdr_rw_load_jump_mgr_regs->load_jump_add3);
2516                 } else {
2517                         mcc_instruction = RW_MGR_LFSR_WR_RD_BANK_0;
2518                         writel(RW_MGR_LFSR_WR_RD_BANK_0_NOP,
2519                                 &sdr_rw_load_jump_mgr_regs->load_jump_add3);
2520                 }
2521         }
2522
2523         writel(0, SDR_PHYGRP_RWMGRGRP_ADDRESS |
2524                   RW_MGR_RESET_READ_DATAPATH_OFFSET);
2525
2526         if (quick_write_mode)
2527                 writel(0x08, &sdr_rw_load_mgr_regs->load_cntr0);
2528         else
2529                 writel(0x40, &sdr_rw_load_mgr_regs->load_cntr0);
2530
2531         writel(mcc_instruction, &sdr_rw_load_jump_mgr_regs->load_jump_add0);
2532
2533         /*
2534          * CNTR 1 - This is used to ensure enough time elapses
2535          * for read data to come back.
2536          */
2537         writel(0x30, &sdr_rw_load_mgr_regs->load_cntr1);
2538
2539         if (test_dm) {
2540                 writel(RW_MGR_LFSR_WR_RD_DM_BANK_0_WAIT,
2541                         &sdr_rw_load_jump_mgr_regs->load_jump_add1);
2542         } else {
2543                 writel(RW_MGR_LFSR_WR_RD_BANK_0_WAIT,
2544                         &sdr_rw_load_jump_mgr_regs->load_jump_add1);
2545         }
2546
2547         addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_SINGLE_GROUP_OFFSET;
2548         writel(mcc_instruction, addr + (group << 2));
2549 }
2550
2551 /* Test writes, can check for a single bit pass or multiple bit pass */
2552 static uint32_t rw_mgr_mem_calibrate_write_test(uint32_t rank_bgn,
2553         uint32_t write_group, uint32_t use_dm, uint32_t all_correct,
2554         uint32_t *bit_chk, uint32_t all_ranks)
2555 {
2556         uint32_t r;
2557         uint32_t correct_mask_vg;
2558         uint32_t tmp_bit_chk;
2559         uint32_t vg;
2560         uint32_t rank_end = all_ranks ? RW_MGR_MEM_NUMBER_OF_RANKS :
2561                 (rank_bgn + NUM_RANKS_PER_SHADOW_REG);
2562         uint32_t addr_rw_mgr;
2563         uint32_t base_rw_mgr;
2564
2565         *bit_chk = param->write_correct_mask;
2566         correct_mask_vg = param->write_correct_mask_vg;
2567
2568         for (r = rank_bgn; r < rank_end; r++) {
2569                 if (param->skip_ranks[r]) {
2570                         /* request to skip the rank */
2571                         continue;
2572                 }
2573
2574                 /* set rank */
2575                 set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_READ_WRITE);
2576
2577                 tmp_bit_chk = 0;
2578                 addr_rw_mgr = SDR_PHYGRP_RWMGRGRP_ADDRESS;
2579                 for (vg = RW_MGR_MEM_VIRTUAL_GROUPS_PER_WRITE_DQS-1; ; vg--) {
2580                         /* reset the fifos to get pointers to known state */
2581                         writel(0, &phy_mgr_cmd->fifo_reset);
2582
2583                         tmp_bit_chk = tmp_bit_chk <<
2584                                 (RW_MGR_MEM_DQ_PER_WRITE_DQS /
2585                                 RW_MGR_MEM_VIRTUAL_GROUPS_PER_WRITE_DQS);
2586                         rw_mgr_mem_calibrate_write_test_issue(write_group *
2587                                 RW_MGR_MEM_VIRTUAL_GROUPS_PER_WRITE_DQS+vg,
2588                                 use_dm);
2589
2590                         base_rw_mgr = readl(addr_rw_mgr);
2591                         tmp_bit_chk = tmp_bit_chk | (correct_mask_vg & ~(base_rw_mgr));
2592                         if (vg == 0)
2593                                 break;
2594                 }
2595                 *bit_chk &= tmp_bit_chk;
2596         }
2597
2598         if (all_correct) {
2599                 set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF);
2600                 debug_cond(DLEVEL == 2, "write_test(%u,%u,ALL) : %u == \
2601                            %u => %lu", write_group, use_dm,
2602                            *bit_chk, param->write_correct_mask,
2603                            (long unsigned int)(*bit_chk ==
2604                            param->write_correct_mask));
2605                 return *bit_chk == param->write_correct_mask;
2606         } else {
2607                 set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF);
2608                 debug_cond(DLEVEL == 2, "write_test(%u,%u,ONE) : %u != ",
2609                        write_group, use_dm, *bit_chk);
2610                 debug_cond(DLEVEL == 2, "%lu" " => %lu", (long unsigned int)0,
2611                         (long unsigned int)(*bit_chk != 0));
2612                 return *bit_chk != 0x00;
2613         }
2614 }
2615
2616 /*
2617  * center all windows. do per-bit-deskew to possibly increase size of
2618  * certain windows.
2619  */
2620 static uint32_t rw_mgr_mem_calibrate_writes_center(uint32_t rank_bgn,
2621         uint32_t write_group, uint32_t test_bgn)
2622 {
2623         uint32_t i, p, min_index;
2624         int32_t d;
2625         /*
2626          * Store these as signed since there are comparisons with
2627          * signed numbers.
2628          */
2629         uint32_t bit_chk;
2630         uint32_t sticky_bit_chk;
2631         int32_t left_edge[RW_MGR_MEM_DQ_PER_WRITE_DQS];
2632         int32_t right_edge[RW_MGR_MEM_DQ_PER_WRITE_DQS];
2633         int32_t mid;
2634         int32_t mid_min, orig_mid_min;
2635         int32_t new_dqs, start_dqs, shift_dq;
2636         int32_t dq_margin, dqs_margin, dm_margin;
2637         uint32_t stop;
2638         uint32_t temp_dq_out1_delay;
2639         uint32_t addr;
2640
2641         debug("%s:%d %u %u", __func__, __LINE__, write_group, test_bgn);
2642
2643         dm_margin = 0;
2644
2645         addr = SDR_PHYGRP_SCCGRP_ADDRESS | SCC_MGR_IO_OUT1_DELAY_OFFSET;
2646         start_dqs = readl(addr +
2647                           (RW_MGR_MEM_DQ_PER_WRITE_DQS << 2));
2648
2649         /* per-bit deskew */
2650
2651         /*
2652          * set the left and right edge of each bit to an illegal value
2653          * use (IO_IO_OUT1_DELAY_MAX + 1) as an illegal value.
2654          */
2655         sticky_bit_chk = 0;
2656         for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) {
2657                 left_edge[i]  = IO_IO_OUT1_DELAY_MAX + 1;
2658                 right_edge[i] = IO_IO_OUT1_DELAY_MAX + 1;
2659         }
2660
2661         /* Search for the left edge of the window for each bit */
2662         for (d = 0; d <= IO_IO_OUT1_DELAY_MAX; d++) {
2663                 scc_mgr_apply_group_dq_out1_delay(write_group, d);
2664
2665                 writel(0, &sdr_scc_mgr->update);
2666
2667                 /*
2668                  * Stop searching when the read test doesn't pass AND when
2669                  * we've seen a passing read on every bit.
2670                  */
2671                 stop = !rw_mgr_mem_calibrate_write_test(rank_bgn, write_group,
2672                         0, PASS_ONE_BIT, &bit_chk, 0);
2673                 sticky_bit_chk = sticky_bit_chk | bit_chk;
2674                 stop = stop && (sticky_bit_chk == param->write_correct_mask);
2675                 debug_cond(DLEVEL == 2, "write_center(left): dtap=%d => %u \
2676                            == %u && %u [bit_chk= %u ]\n",
2677                         d, sticky_bit_chk, param->write_correct_mask,
2678                         stop, bit_chk);
2679
2680                 if (stop == 1) {
2681                         break;
2682                 } else {
2683                         for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) {
2684                                 if (bit_chk & 1) {
2685                                         /*
2686                                          * Remember a passing test as the
2687                                          * left_edge.
2688                                          */
2689                                         left_edge[i] = d;
2690                                 } else {
2691                                         /*
2692                                          * If a left edge has not been seen
2693                                          * yet, then a future passing test will
2694                                          * mark this edge as the right edge.
2695                                          */
2696                                         if (left_edge[i] ==
2697                                                 IO_IO_OUT1_DELAY_MAX + 1) {
2698                                                 right_edge[i] = -(d + 1);
2699                                         }
2700                                 }
2701                                 debug_cond(DLEVEL == 2, "write_center[l,d=%d):", d);
2702                                 debug_cond(DLEVEL == 2, "bit_chk_test=%d left_edge[%u]: %d",
2703                                            (int)(bit_chk & 1), i, left_edge[i]);
2704                                 debug_cond(DLEVEL == 2, "right_edge[%u]: %d\n", i,
2705                                        right_edge[i]);
2706                                 bit_chk = bit_chk >> 1;
2707                         }
2708                 }
2709         }
2710
2711         /* Reset DQ delay chains to 0 */
2712         scc_mgr_apply_group_dq_out1_delay(0);
2713         sticky_bit_chk = 0;
2714         for (i = RW_MGR_MEM_DQ_PER_WRITE_DQS - 1;; i--) {
2715                 debug_cond(DLEVEL == 2, "%s:%d write_center: left_edge[%u]: \
2716                            %d right_edge[%u]: %d\n", __func__, __LINE__,
2717                            i, left_edge[i], i, right_edge[i]);
2718
2719                 /*
2720                  * Check for cases where we haven't found the left edge,
2721                  * which makes our assignment of the the right edge invalid.
2722                  * Reset it to the illegal value.
2723                  */
2724                 if ((left_edge[i] == IO_IO_OUT1_DELAY_MAX + 1) &&
2725                     (right_edge[i] != IO_IO_OUT1_DELAY_MAX + 1)) {
2726                         right_edge[i] = IO_IO_OUT1_DELAY_MAX + 1;
2727                         debug_cond(DLEVEL == 2, "%s:%d write_center: reset \
2728                                    right_edge[%u]: %d\n", __func__, __LINE__,
2729                                    i, right_edge[i]);
2730                 }
2731
2732                 /*
2733                  * Reset sticky bit (except for bits where we have
2734                  * seen the left edge).
2735                  */
2736                 sticky_bit_chk = sticky_bit_chk << 1;
2737                 if ((left_edge[i] != IO_IO_OUT1_DELAY_MAX + 1))
2738                         sticky_bit_chk = sticky_bit_chk | 1;
2739
2740                 if (i == 0)
2741                         break;
2742         }
2743
2744         /* Search for the right edge of the window for each bit */
2745         for (d = 0; d <= IO_IO_OUT1_DELAY_MAX - start_dqs; d++) {
2746                 scc_mgr_apply_group_dqs_io_and_oct_out1(write_group,
2747                                                         d + start_dqs);
2748
2749                 writel(0, &sdr_scc_mgr->update);
2750
2751                 /*
2752                  * Stop searching when the read test doesn't pass AND when
2753                  * we've seen a passing read on every bit.
2754                  */
2755                 stop = !rw_mgr_mem_calibrate_write_test(rank_bgn, write_group,
2756                         0, PASS_ONE_BIT, &bit_chk, 0);
2757
2758                 sticky_bit_chk = sticky_bit_chk | bit_chk;
2759                 stop = stop && (sticky_bit_chk == param->write_correct_mask);
2760
2761                 debug_cond(DLEVEL == 2, "write_center (right): dtap=%u => %u == \
2762                            %u && %u\n", d, sticky_bit_chk,
2763                            param->write_correct_mask, stop);
2764
2765                 if (stop == 1) {
2766                         if (d == 0) {
2767                                 for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS;
2768                                         i++) {
2769                                         /* d = 0 failed, but it passed when
2770                                         testing the left edge, so it must be
2771                                         marginal, set it to -1 */
2772                                         if (right_edge[i] ==
2773                                                 IO_IO_OUT1_DELAY_MAX + 1 &&
2774                                                 left_edge[i] !=
2775                                                 IO_IO_OUT1_DELAY_MAX + 1) {
2776                                                 right_edge[i] = -1;
2777                                         }
2778                                 }
2779                         }
2780                         break;
2781                 } else {
2782                         for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) {
2783                                 if (bit_chk & 1) {
2784                                         /*
2785                                          * Remember a passing test as
2786                                          * the right_edge.
2787                                          */
2788                                         right_edge[i] = d;
2789                                 } else {
2790                                         if (d != 0) {
2791                                                 /*
2792                                                  * If a right edge has not
2793                                                  * been seen yet, then a future
2794                                                  * passing test will mark this
2795                                                  * edge as the left edge.
2796                                                  */
2797                                                 if (right_edge[i] ==
2798                                                     IO_IO_OUT1_DELAY_MAX + 1)
2799                                                         left_edge[i] = -(d + 1);
2800                                         } else {
2801                                                 /*
2802                                                  * d = 0 failed, but it passed
2803                                                  * when testing the left edge,
2804                                                  * so it must be marginal, set
2805                                                  * it to -1.
2806                                                  */
2807                                                 if (right_edge[i] ==
2808                                                     IO_IO_OUT1_DELAY_MAX + 1 &&
2809                                                     left_edge[i] !=
2810                                                     IO_IO_OUT1_DELAY_MAX + 1)
2811                                                         right_edge[i] = -1;
2812                                                 /*
2813                                                  * If a right edge has not been
2814                                                  * seen yet, then a future
2815                                                  * passing test will mark this
2816                                                  * edge as the left edge.
2817                                                  */
2818                                                 else if (right_edge[i] ==
2819                                                         IO_IO_OUT1_DELAY_MAX +
2820                                                         1)
2821                                                         left_edge[i] = -(d + 1);
2822                                         }
2823                                 }
2824                                 debug_cond(DLEVEL == 2, "write_center[r,d=%d):", d);
2825                                 debug_cond(DLEVEL == 2, "bit_chk_test=%d left_edge[%u]: %d",
2826                                            (int)(bit_chk & 1), i, left_edge[i]);
2827                                 debug_cond(DLEVEL == 2, "right_edge[%u]: %d\n", i,
2828                                            right_edge[i]);
2829                                 bit_chk = bit_chk >> 1;
2830                         }
2831                 }
2832         }
2833
2834         /* Check that all bits have a window */
2835         for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) {
2836                 debug_cond(DLEVEL == 2, "%s:%d write_center: left_edge[%u]: \
2837                            %d right_edge[%u]: %d", __func__, __LINE__,
2838                            i, left_edge[i], i, right_edge[i]);
2839                 if ((left_edge[i] == IO_IO_OUT1_DELAY_MAX + 1) ||
2840                     (right_edge[i] == IO_IO_OUT1_DELAY_MAX + 1)) {
2841                         set_failing_group_stage(test_bgn + i,
2842                                                 CAL_STAGE_WRITES,
2843                                                 CAL_SUBSTAGE_WRITES_CENTER);
2844                         return 0;
2845                 }
2846         }
2847
2848         /* Find middle of window for each DQ bit */
2849         mid_min = left_edge[0] - right_edge[0];
2850         min_index = 0;
2851         for (i = 1; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) {
2852                 mid = left_edge[i] - right_edge[i];
2853                 if (mid < mid_min) {
2854                         mid_min = mid;
2855                         min_index = i;
2856                 }
2857         }
2858
2859         /*
2860          * -mid_min/2 represents the amount that we need to move DQS.
2861          * If mid_min is odd and positive we'll need to add one to
2862          * make sure the rounding in further calculations is correct
2863          * (always bias to the right), so just add 1 for all positive values.
2864          */
2865         if (mid_min > 0)
2866                 mid_min++;
2867         mid_min = mid_min / 2;
2868         debug_cond(DLEVEL == 1, "%s:%d write_center: mid_min=%d\n", __func__,
2869                    __LINE__, mid_min);
2870
2871         /* Determine the amount we can change DQS (which is -mid_min) */
2872         orig_mid_min = mid_min;
2873         new_dqs = start_dqs;
2874         mid_min = 0;
2875         debug_cond(DLEVEL == 1, "%s:%d write_center: start_dqs=%d new_dqs=%d \
2876                    mid_min=%d\n", __func__, __LINE__, start_dqs, new_dqs, mid_min);
2877         /* Initialize data for export structures */
2878         dqs_margin = IO_IO_OUT1_DELAY_MAX + 1;
2879         dq_margin  = IO_IO_OUT1_DELAY_MAX + 1;
2880
2881         /* add delay to bring centre of all DQ windows to the same "level" */
2882         for (i = 0, p = test_bgn; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++, p++) {
2883                 /* Use values before divide by 2 to reduce round off error */
2884                 shift_dq = (left_edge[i] - right_edge[i] -
2885                         (left_edge[min_index] - right_edge[min_index]))/2  +
2886                 (orig_mid_min - mid_min);
2887
2888                 debug_cond(DLEVEL == 2, "%s:%d write_center: before: shift_dq \
2889                            [%u]=%d\n", __func__, __LINE__, i, shift_dq);
2890
2891                 addr = SDR_PHYGRP_SCCGRP_ADDRESS | SCC_MGR_IO_OUT1_DELAY_OFFSET;
2892                 temp_dq_out1_delay = readl(addr + (i << 2));
2893                 if (shift_dq + (int32_t)temp_dq_out1_delay >
2894                         (int32_t)IO_IO_OUT1_DELAY_MAX) {
2895                         shift_dq = (int32_t)IO_IO_OUT1_DELAY_MAX - temp_dq_out1_delay;
2896                 } else if (shift_dq + (int32_t)temp_dq_out1_delay < 0) {
2897                         shift_dq = -(int32_t)temp_dq_out1_delay;
2898                 }
2899                 debug_cond(DLEVEL == 2, "write_center: after: shift_dq[%u]=%d\n",
2900                            i, shift_dq);
2901                 scc_mgr_set_dq_out1_delay(i, temp_dq_out1_delay + shift_dq);
2902                 scc_mgr_load_dq(i);
2903
2904                 debug_cond(DLEVEL == 2, "write_center: margin[%u]=[%d,%d]\n", i,
2905                            left_edge[i] - shift_dq + (-mid_min),
2906                            right_edge[i] + shift_dq - (-mid_min));
2907                 /* To determine values for export structures */
2908                 if (left_edge[i] - shift_dq + (-mid_min) < dq_margin)
2909                         dq_margin = left_edge[i] - shift_dq + (-mid_min);
2910
2911                 if (right_edge[i] + shift_dq - (-mid_min) < dqs_margin)
2912                         dqs_margin = right_edge[i] + shift_dq - (-mid_min);
2913         }
2914
2915         /* Move DQS */
2916         scc_mgr_apply_group_dqs_io_and_oct_out1(write_group, new_dqs);
2917         writel(0, &sdr_scc_mgr->update);
2918
2919         /* Centre DM */
2920         debug_cond(DLEVEL == 2, "%s:%d write_center: DM\n", __func__, __LINE__);
2921
2922         /*
2923          * set the left and right edge of each bit to an illegal value,
2924          * use (IO_IO_OUT1_DELAY_MAX + 1) as an illegal value,
2925          */
2926         left_edge[0]  = IO_IO_OUT1_DELAY_MAX + 1;
2927         right_edge[0] = IO_IO_OUT1_DELAY_MAX + 1;
2928         int32_t bgn_curr = IO_IO_OUT1_DELAY_MAX + 1;
2929         int32_t end_curr = IO_IO_OUT1_DELAY_MAX + 1;
2930         int32_t bgn_best = IO_IO_OUT1_DELAY_MAX + 1;
2931         int32_t end_best = IO_IO_OUT1_DELAY_MAX + 1;
2932         int32_t win_best = 0;
2933
2934         /* Search for the/part of the window with DM shift */
2935         for (d = IO_IO_OUT1_DELAY_MAX; d >= 0; d -= DELTA_D) {
2936                 scc_mgr_apply_group_dm_out1_delay(d);
2937                 writel(0, &sdr_scc_mgr->update);
2938
2939                 if (rw_mgr_mem_calibrate_write_test(rank_bgn, write_group, 1,
2940                                                     PASS_ALL_BITS, &bit_chk,
2941                                                     0)) {
2942                         /* USE Set current end of the window */
2943                         end_curr = -d;
2944                         /*
2945                          * If a starting edge of our window has not been seen
2946                          * this is our current start of the DM window.
2947                          */
2948                         if (bgn_curr == IO_IO_OUT1_DELAY_MAX + 1)
2949                                 bgn_curr = -d;
2950
2951                         /*
2952                          * If current window is bigger than best seen.
2953                          * Set best seen to be current window.
2954                          */
2955                         if ((end_curr-bgn_curr+1) > win_best) {
2956                                 win_best = end_curr-bgn_curr+1;
2957                                 bgn_best = bgn_curr;
2958                                 end_best = end_curr;
2959                         }
2960                 } else {
2961                         /* We just saw a failing test. Reset temp edge */
2962                         bgn_curr = IO_IO_OUT1_DELAY_MAX + 1;
2963                         end_curr = IO_IO_OUT1_DELAY_MAX + 1;
2964                         }
2965                 }
2966
2967
2968         /* Reset DM delay chains to 0 */
2969         scc_mgr_apply_group_dm_out1_delay(0);
2970
2971         /*
2972          * Check to see if the current window nudges up aganist 0 delay.
2973          * If so we need to continue the search by shifting DQS otherwise DQS
2974          * search begins as a new search. */
2975         if (end_curr != 0) {
2976                 bgn_curr = IO_IO_OUT1_DELAY_MAX + 1;
2977                 end_curr = IO_IO_OUT1_DELAY_MAX + 1;
2978         }
2979
2980         /* Search for the/part of the window with DQS shifts */
2981         for (d = 0; d <= IO_IO_OUT1_DELAY_MAX - new_dqs; d += DELTA_D) {
2982                 /*
2983                  * Note: This only shifts DQS, so are we limiting ourselve to
2984                  * width of DQ unnecessarily.
2985                  */
2986                 scc_mgr_apply_group_dqs_io_and_oct_out1(write_group,
2987                                                         d + new_dqs);
2988
2989                 writel(0, &sdr_scc_mgr->update);
2990                 if (rw_mgr_mem_calibrate_write_test(rank_bgn, write_group, 1,
2991                                                     PASS_ALL_BITS, &bit_chk,
2992                                                     0)) {
2993                         /* USE Set current end of the window */
2994                         end_curr = d;
2995                         /*
2996                          * If a beginning edge of our window has not been seen
2997                          * this is our current begin of the DM window.
2998                          */
2999                         if (bgn_curr == IO_IO_OUT1_DELAY_MAX + 1)
3000                                 bgn_curr = d;
3001
3002                         /*
3003                          * If current window is bigger than best seen. Set best
3004                          * seen to be current window.
3005                          */
3006                         if ((end_curr-bgn_curr+1) > win_best) {
3007                                 win_best = end_curr-bgn_curr+1;
3008                                 bgn_best = bgn_curr;
3009                                 end_best = end_curr;
3010                         }
3011                 } else {
3012                         /* We just saw a failing test. Reset temp edge */
3013                         bgn_curr = IO_IO_OUT1_DELAY_MAX + 1;
3014                         end_curr = IO_IO_OUT1_DELAY_MAX + 1;
3015
3016                         /* Early exit optimization: if ther remaining delay
3017                         chain space is less than already seen largest window
3018                         we can exit */
3019                         if ((win_best-1) >
3020                                 (IO_IO_OUT1_DELAY_MAX - new_dqs - d)) {
3021                                         break;
3022                                 }
3023                         }
3024                 }
3025
3026         /* assign left and right edge for cal and reporting; */
3027         left_edge[0] = -1*bgn_best;
3028         right_edge[0] = end_best;
3029
3030         debug_cond(DLEVEL == 2, "%s:%d dm_calib: left=%d right=%d\n", __func__,
3031                    __LINE__, left_edge[0], right_edge[0]);
3032
3033         /* Move DQS (back to orig) */
3034         scc_mgr_apply_group_dqs_io_and_oct_out1(write_group, new_dqs);
3035
3036         /* Move DM */
3037
3038         /* Find middle of window for the DM bit */
3039         mid = (left_edge[0] - right_edge[0]) / 2;
3040
3041         /* only move right, since we are not moving DQS/DQ */
3042         if (mid < 0)
3043                 mid = 0;
3044
3045         /* dm_marign should fail if we never find a window */
3046         if (win_best == 0)
3047                 dm_margin = -1;
3048         else
3049                 dm_margin = left_edge[0] - mid;
3050
3051         scc_mgr_apply_group_dm_out1_delay(mid);
3052         writel(0, &sdr_scc_mgr->update);
3053
3054         debug_cond(DLEVEL == 2, "%s:%d dm_calib: left=%d right=%d mid=%d \
3055                    dm_margin=%d\n", __func__, __LINE__, left_edge[0],
3056                    right_edge[0], mid, dm_margin);
3057         /* Export values */
3058         gbl->fom_out += dq_margin + dqs_margin;
3059
3060         debug_cond(DLEVEL == 2, "%s:%d write_center: dq_margin=%d \
3061                    dqs_margin=%d dm_margin=%d\n", __func__, __LINE__,
3062                    dq_margin, dqs_margin, dm_margin);
3063
3064         /*
3065          * Do not remove this line as it makes sure all of our
3066          * decisions have been applied.
3067          */
3068         writel(0, &sdr_scc_mgr->update);
3069         return (dq_margin >= 0) && (dqs_margin >= 0) && (dm_margin >= 0);
3070 }
3071
3072 /* calibrate the write operations */
3073 static uint32_t rw_mgr_mem_calibrate_writes(uint32_t rank_bgn, uint32_t g,
3074         uint32_t test_bgn)
3075 {
3076         /* update info for sims */
3077         debug("%s:%d %u %u\n", __func__, __LINE__, g, test_bgn);
3078
3079         reg_file_set_stage(CAL_STAGE_WRITES);
3080         reg_file_set_sub_stage(CAL_SUBSTAGE_WRITES_CENTER);
3081
3082         reg_file_set_group(g);
3083
3084         if (!rw_mgr_mem_calibrate_writes_center(rank_bgn, g, test_bgn)) {
3085                 set_failing_group_stage(g, CAL_STAGE_WRITES,
3086                                         CAL_SUBSTAGE_WRITES_CENTER);
3087                 return 0;
3088         }
3089
3090         return 1;
3091 }
3092
3093 /**
3094  * mem_precharge_and_activate() - Precharge all banks and activate
3095  *
3096  * Precharge all banks and activate row 0 in bank "000..." and bank "111...".
3097  */
3098 static void mem_precharge_and_activate(void)
3099 {
3100         int r;
3101
3102         for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; r++) {
3103                 /* Test if the rank should be skipped. */
3104                 if (param->skip_ranks[r])
3105                         continue;
3106
3107                 /* Set rank. */
3108                 set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_OFF);
3109
3110                 /* Precharge all banks. */
3111                 writel(RW_MGR_PRECHARGE_ALL, SDR_PHYGRP_RWMGRGRP_ADDRESS |
3112                                              RW_MGR_RUN_SINGLE_GROUP_OFFSET);
3113
3114                 writel(0x0F, &sdr_rw_load_mgr_regs->load_cntr0);
3115                 writel(RW_MGR_ACTIVATE_0_AND_1_WAIT1,
3116                         &sdr_rw_load_jump_mgr_regs->load_jump_add0);
3117
3118                 writel(0x0F, &sdr_rw_load_mgr_regs->load_cntr1);
3119                 writel(RW_MGR_ACTIVATE_0_AND_1_WAIT2,
3120                         &sdr_rw_load_jump_mgr_regs->load_jump_add1);
3121
3122                 /* Activate rows. */
3123                 writel(RW_MGR_ACTIVATE_0_AND_1, SDR_PHYGRP_RWMGRGRP_ADDRESS |
3124                                                 RW_MGR_RUN_SINGLE_GROUP_OFFSET);
3125         }
3126 }
3127
3128 /**
3129  * mem_init_latency() - Configure memory RLAT and WLAT settings
3130  *
3131  * Configure memory RLAT and WLAT parameters.
3132  */
3133 static void mem_init_latency(void)
3134 {
3135         /*
3136          * For AV/CV, LFIFO is hardened and always runs at full rate
3137          * so max latency in AFI clocks, used here, is correspondingly
3138          * smaller.
3139          */
3140         const u32 max_latency = (1 << MAX_LATENCY_COUNT_WIDTH) - 1;
3141         u32 rlat, wlat;
3142
3143         debug("%s:%d\n", __func__, __LINE__);
3144
3145         /*
3146          * Read in write latency.
3147          * WL for Hard PHY does not include additive latency.
3148          */
3149         wlat = readl(&data_mgr->t_wl_add);
3150         wlat += readl(&data_mgr->mem_t_add);
3151
3152         gbl->rw_wl_nop_cycles = wlat - 1;
3153
3154         /* Read in readl latency. */
3155         rlat = readl(&data_mgr->t_rl_add);
3156
3157         /* Set a pretty high read latency initially. */
3158         gbl->curr_read_lat = rlat + 16;
3159         if (gbl->curr_read_lat > max_latency)
3160                 gbl->curr_read_lat = max_latency;
3161
3162         writel(gbl->curr_read_lat, &phy_mgr_cfg->phy_rlat);
3163
3164         /* Advertise write latency. */
3165         writel(wlat, &phy_mgr_cfg->afi_wlat);
3166 }
3167
3168 /* Set VFIFO and LFIFO to instant-on settings in skip calibration mode */
3169 static void mem_skip_calibrate(void)
3170 {
3171         uint32_t vfifo_offset;
3172         uint32_t i, j, r;
3173
3174         debug("%s:%d\n", __func__, __LINE__);
3175         /* Need to update every shadow register set used by the interface */
3176         for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS;
3177                 r += NUM_RANKS_PER_SHADOW_REG) {
3178                 /*
3179                  * Set output phase alignment settings appropriate for
3180                  * skip calibration.
3181                  */
3182                 for (i = 0; i < RW_MGR_MEM_IF_READ_DQS_WIDTH; i++) {
3183                         scc_mgr_set_dqs_en_phase(i, 0);
3184 #if IO_DLL_CHAIN_LENGTH == 6
3185                         scc_mgr_set_dqdqs_output_phase(i, 6);
3186 #else
3187                         scc_mgr_set_dqdqs_output_phase(i, 7);
3188 #endif
3189                         /*
3190                          * Case:33398
3191                          *
3192                          * Write data arrives to the I/O two cycles before write
3193                          * latency is reached (720 deg).
3194                          *   -> due to bit-slip in a/c bus
3195                          *   -> to allow board skew where dqs is longer than ck
3196                          *      -> how often can this happen!?
3197                          *      -> can claim back some ptaps for high freq
3198                          *       support if we can relax this, but i digress...
3199                          *
3200                          * The write_clk leads mem_ck by 90 deg
3201                          * The minimum ptap of the OPA is 180 deg
3202                          * Each ptap has (360 / IO_DLL_CHAIN_LENGH) deg of delay
3203                          * The write_clk is always delayed by 2 ptaps
3204                          *
3205                          * Hence, to make DQS aligned to CK, we need to delay
3206                          * DQS by:
3207                          *    (720 - 90 - 180 - 2 * (360 / IO_DLL_CHAIN_LENGTH))
3208                          *
3209                          * Dividing the above by (360 / IO_DLL_CHAIN_LENGTH)
3210                          * gives us the number of ptaps, which simplies to:
3211                          *
3212                          *    (1.25 * IO_DLL_CHAIN_LENGTH - 2)
3213                          */
3214                         scc_mgr_set_dqdqs_output_phase(i, (1.25 *
3215                                 IO_DLL_CHAIN_LENGTH - 2));
3216                 }
3217                 writel(0xff, &sdr_scc_mgr->dqs_ena);
3218                 writel(0xff, &sdr_scc_mgr->dqs_io_ena);
3219
3220                 for (i = 0; i < RW_MGR_MEM_IF_WRITE_DQS_WIDTH; i++) {
3221                         writel(i, SDR_PHYGRP_SCCGRP_ADDRESS |
3222                                   SCC_MGR_GROUP_COUNTER_OFFSET);
3223                 }
3224                 writel(0xff, &sdr_scc_mgr->dq_ena);
3225                 writel(0xff, &sdr_scc_mgr->dm_ena);
3226                 writel(0, &sdr_scc_mgr->update);
3227         }
3228
3229         /* Compensate for simulation model behaviour */
3230         for (i = 0; i < RW_MGR_MEM_IF_READ_DQS_WIDTH; i++) {
3231                 scc_mgr_set_dqs_bus_in_delay(i, 10);
3232                 scc_mgr_load_dqs(i);
3233         }
3234         writel(0, &sdr_scc_mgr->update);
3235
3236         /*
3237          * ArriaV has hard FIFOs that can only be initialized by incrementing
3238          * in sequencer.
3239          */
3240         vfifo_offset = CALIB_VFIFO_OFFSET;
3241         for (j = 0; j < vfifo_offset; j++) {
3242                 writel(0xff, &phy_mgr_cmd->inc_vfifo_hard_phy);
3243         }
3244         writel(0, &phy_mgr_cmd->fifo_reset);
3245
3246         /*
3247          * For ACV with hard lfifo, we get the skip-cal setting from
3248          * generation-time constant.
3249          */
3250         gbl->curr_read_lat = CALIB_LFIFO_OFFSET;
3251         writel(gbl->curr_read_lat, &phy_mgr_cfg->phy_rlat);
3252 }
3253
3254 /* Memory calibration entry point */
3255 static uint32_t mem_calibrate(void)
3256 {
3257         uint32_t i;
3258         uint32_t rank_bgn, sr;
3259         uint32_t write_group, write_test_bgn;
3260         uint32_t read_group, read_test_bgn;
3261         uint32_t run_groups, current_run;
3262         uint32_t failing_groups = 0;
3263         uint32_t group_failed = 0;
3264         uint32_t sr_failed = 0;
3265
3266         debug("%s:%d\n", __func__, __LINE__);
3267
3268         /* Initialize the data settings */
3269         gbl->error_substage = CAL_SUBSTAGE_NIL;
3270         gbl->error_stage = CAL_STAGE_NIL;
3271         gbl->error_group = 0xff;
3272         gbl->fom_in = 0;
3273         gbl->fom_out = 0;
3274
3275         /* Initialize WLAT and RLAT. */
3276         mem_init_latency();
3277
3278         /* Initialize bit slips. */
3279         mem_precharge_and_activate();
3280
3281         for (i = 0; i < RW_MGR_MEM_IF_READ_DQS_WIDTH; i++) {
3282                 writel(i, SDR_PHYGRP_SCCGRP_ADDRESS |
3283                           SCC_MGR_GROUP_COUNTER_OFFSET);
3284                 /* Only needed once to set all groups, pins, DQ, DQS, DM. */
3285                 if (i == 0)
3286                         scc_mgr_set_hhp_extras();
3287
3288                 scc_set_bypass_mode(i);
3289         }
3290
3291         if ((dyn_calib_steps & CALIB_SKIP_ALL) == CALIB_SKIP_ALL) {
3292                 /*
3293                  * Set VFIFO and LFIFO to instant-on settings in skip
3294                  * calibration mode.
3295                  */
3296                 mem_skip_calibrate();
3297         } else {
3298                 for (i = 0; i < NUM_CALIB_REPEAT; i++) {
3299                         /*
3300                          * Zero all delay chain/phase settings for all
3301                          * groups and all shadow register sets.
3302                          */
3303                         scc_mgr_zero_all();
3304
3305                         run_groups = ~param->skip_groups;
3306
3307                         for (write_group = 0, write_test_bgn = 0; write_group
3308                                 < RW_MGR_MEM_IF_WRITE_DQS_WIDTH; write_group++,
3309                                 write_test_bgn += RW_MGR_MEM_DQ_PER_WRITE_DQS) {
3310                                 /* Initialized the group failure */
3311                                 group_failed = 0;
3312
3313                                 current_run = run_groups & ((1 <<
3314                                         RW_MGR_NUM_DQS_PER_WRITE_GROUP) - 1);
3315                                 run_groups = run_groups >>
3316                                         RW_MGR_NUM_DQS_PER_WRITE_GROUP;
3317
3318                                 if (current_run == 0)
3319                                         continue;
3320
3321                                 writel(write_group, SDR_PHYGRP_SCCGRP_ADDRESS |
3322                                                     SCC_MGR_GROUP_COUNTER_OFFSET);
3323                                 scc_mgr_zero_group(write_group, 0);
3324
3325                                 for (read_group = write_group *
3326                                         RW_MGR_MEM_IF_READ_DQS_WIDTH /
3327                                         RW_MGR_MEM_IF_WRITE_DQS_WIDTH,
3328                                         read_test_bgn = 0;
3329                                         read_group < (write_group + 1) *
3330                                         RW_MGR_MEM_IF_READ_DQS_WIDTH /
3331                                         RW_MGR_MEM_IF_WRITE_DQS_WIDTH &&
3332                                         group_failed == 0;
3333                                         read_group++, read_test_bgn +=
3334                                         RW_MGR_MEM_DQ_PER_READ_DQS) {
3335                                         /* Calibrate the VFIFO */
3336                                         if (!((STATIC_CALIB_STEPS) &
3337                                                 CALIB_SKIP_VFIFO)) {
3338                                                 if (!rw_mgr_mem_calibrate_vfifo
3339                                                         (read_group,
3340                                                         read_test_bgn)) {
3341                                                         group_failed = 1;
3342
3343                                                         if (!(gbl->
3344                                                         phy_debug_mode_flags &
3345                                                 PHY_DEBUG_SWEEP_ALL_GROUPS)) {
3346                                                                 return 0;
3347                                                         }
3348                                                 }
3349                                         }
3350                                 }
3351
3352                                 /* Calibrate the output side */
3353                                 if (group_failed == 0)  {
3354                                         for (rank_bgn = 0, sr = 0; rank_bgn
3355                                                 < RW_MGR_MEM_NUMBER_OF_RANKS;
3356                                                 rank_bgn +=
3357                                                 NUM_RANKS_PER_SHADOW_REG,
3358                                                 ++sr) {
3359                                                 sr_failed = 0;
3360                                                 if (!((STATIC_CALIB_STEPS) &
3361                                                 CALIB_SKIP_WRITES)) {
3362                                                         if ((STATIC_CALIB_STEPS)
3363                                                 & CALIB_SKIP_DELAY_SWEEPS) {
3364                                                 /* not needed in quick mode! */
3365                                                         } else {
3366                                                 /*
3367                                                  * Determine if this set of
3368                                                  * ranks should be skipped
3369                                                  * entirely.
3370                                                  */
3371                                         if (!param->skip_shadow_regs[sr]) {
3372                                                 if (!rw_mgr_mem_calibrate_writes
3373                                                 (rank_bgn, write_group,
3374                                                 write_test_bgn)) {
3375                                                         sr_failed = 1;
3376                                                         if (!(gbl->
3377                                                         phy_debug_mode_flags &
3378                                                 PHY_DEBUG_SWEEP_ALL_GROUPS)) {
3379                                                                 return 0;
3380                                                                         }
3381                                                                         }
3382                                                                 }
3383                                                         }
3384                                                 }
3385                                                 if (sr_failed != 0)
3386                                                         group_failed = 1;
3387                                         }
3388                                 }
3389
3390                                 if (group_failed == 0) {
3391                                         for (read_group = write_group *
3392                                         RW_MGR_MEM_IF_READ_DQS_WIDTH /
3393                                         RW_MGR_MEM_IF_WRITE_DQS_WIDTH,
3394                                         read_test_bgn = 0;
3395                                                 read_group < (write_group + 1)
3396                                                 * RW_MGR_MEM_IF_READ_DQS_WIDTH
3397                                                 / RW_MGR_MEM_IF_WRITE_DQS_WIDTH &&
3398                                                 group_failed == 0;
3399                                                 read_group++, read_test_bgn +=
3400                                                 RW_MGR_MEM_DQ_PER_READ_DQS) {
3401                                                 if (!((STATIC_CALIB_STEPS) &
3402                                                         CALIB_SKIP_WRITES)) {
3403                                         if (!rw_mgr_mem_calibrate_vfifo_end
3404                                                 (read_group, read_test_bgn)) {
3405                                                         group_failed = 1;
3406
3407                                                 if (!(gbl->phy_debug_mode_flags
3408                                                 & PHY_DEBUG_SWEEP_ALL_GROUPS)) {
3409                                                                 return 0;
3410                                                                 }
3411                                                         }
3412                                                 }
3413                                         }
3414                                 }
3415
3416                                 if (group_failed != 0)
3417                                         failing_groups++;
3418                         }
3419
3420                         /*
3421                          * USER If there are any failing groups then report
3422                          * the failure.
3423                          */
3424                         if (failing_groups != 0)
3425                                 return 0;
3426
3427                         /* Calibrate the LFIFO */
3428                         if (!((STATIC_CALIB_STEPS) & CALIB_SKIP_LFIFO)) {
3429                                 /*
3430                                  * If we're skipping groups as part of debug,
3431                                  * don't calibrate LFIFO.
3432                                  */
3433                                 if (param->skip_groups == 0) {
3434                                         if (!rw_mgr_mem_calibrate_lfifo())
3435                                                 return 0;
3436                                 }
3437                         }
3438                 }
3439         }
3440
3441         /*
3442          * Do not remove this line as it makes sure all of our decisions
3443          * have been applied.
3444          */
3445         writel(0, &sdr_scc_mgr->update);
3446         return 1;
3447 }
3448
3449 /**
3450  * run_mem_calibrate() - Perform memory calibration
3451  *
3452  * This function triggers the entire memory calibration procedure.
3453  */
3454 static int run_mem_calibrate(void)
3455 {
3456         int pass;
3457
3458         debug("%s:%d\n", __func__, __LINE__);
3459
3460         /* Reset pass/fail status shown on afi_cal_success/fail */
3461         writel(PHY_MGR_CAL_RESET, &phy_mgr_cfg->cal_status);
3462
3463         /* Stop tracking manager. */
3464         clrbits_le32(&sdr_ctrl->ctrl_cfg, 1 << 22);
3465
3466         phy_mgr_initialize();
3467         rw_mgr_mem_initialize();
3468
3469         /* Perform the actual memory calibration. */
3470         pass = mem_calibrate();
3471
3472         mem_precharge_and_activate();
3473         writel(0, &phy_mgr_cmd->fifo_reset);
3474
3475         /* Handoff. */
3476         rw_mgr_mem_handoff();
3477         /*
3478          * In Hard PHY this is a 2-bit control:
3479          * 0: AFI Mux Select
3480          * 1: DDIO Mux Select
3481          */
3482         writel(0x2, &phy_mgr_cfg->mux_sel);
3483
3484         /* Start tracking manager. */
3485         setbits_le32(&sdr_ctrl->ctrl_cfg, 1 << 22);
3486
3487         return pass;
3488 }
3489
3490 /**
3491  * debug_mem_calibrate() - Report result of memory calibration
3492  * @pass:       Value indicating whether calibration passed or failed
3493  *
3494  * This function reports the results of the memory calibration
3495  * and writes debug information into the register file.
3496  */
3497 static void debug_mem_calibrate(int pass)
3498 {
3499         uint32_t debug_info;
3500
3501         if (pass) {
3502                 printf("%s: CALIBRATION PASSED\n", __FILE__);
3503
3504                 gbl->fom_in /= 2;
3505                 gbl->fom_out /= 2;
3506
3507                 if (gbl->fom_in > 0xff)
3508                         gbl->fom_in = 0xff;
3509
3510                 if (gbl->fom_out > 0xff)
3511                         gbl->fom_out = 0xff;
3512
3513                 /* Update the FOM in the register file */
3514                 debug_info = gbl->fom_in;
3515                 debug_info |= gbl->fom_out << 8;
3516                 writel(debug_info, &sdr_reg_file->fom);
3517
3518                 writel(debug_info, &phy_mgr_cfg->cal_debug_info);
3519                 writel(PHY_MGR_CAL_SUCCESS, &phy_mgr_cfg->cal_status);
3520         } else {
3521                 printf("%s: CALIBRATION FAILED\n", __FILE__);
3522
3523                 debug_info = gbl->error_stage;
3524                 debug_info |= gbl->error_substage << 8;
3525                 debug_info |= gbl->error_group << 16;
3526
3527                 writel(debug_info, &sdr_reg_file->failing_stage);
3528                 writel(debug_info, &phy_mgr_cfg->cal_debug_info);
3529                 writel(PHY_MGR_CAL_FAIL, &phy_mgr_cfg->cal_status);
3530
3531                 /* Update the failing group/stage in the register file */
3532                 debug_info = gbl->error_stage;
3533                 debug_info |= gbl->error_substage << 8;
3534                 debug_info |= gbl->error_group << 16;
3535                 writel(debug_info, &sdr_reg_file->failing_stage);
3536         }
3537
3538         printf("%s: Calibration complete\n", __FILE__);
3539 }
3540
3541 /**
3542  * hc_initialize_rom_data() - Initialize ROM data
3543  *
3544  * Initialize ROM data.
3545  */
3546 static void hc_initialize_rom_data(void)
3547 {
3548         u32 i, addr;
3549
3550         addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_INST_ROM_WRITE_OFFSET;
3551         for (i = 0; i < ARRAY_SIZE(inst_rom_init); i++)
3552                 writel(inst_rom_init[i], addr + (i << 2));
3553
3554         addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_AC_ROM_WRITE_OFFSET;
3555         for (i = 0; i < ARRAY_SIZE(ac_rom_init); i++)
3556                 writel(ac_rom_init[i], addr + (i << 2));
3557 }
3558
3559 /**
3560  * initialize_reg_file() - Initialize SDR register file
3561  *
3562  * Initialize SDR register file.
3563  */
3564 static void initialize_reg_file(void)
3565 {
3566         /* Initialize the register file with the correct data */
3567         writel(REG_FILE_INIT_SEQ_SIGNATURE, &sdr_reg_file->signature);
3568         writel(0, &sdr_reg_file->debug_data_addr);
3569         writel(0, &sdr_reg_file->cur_stage);
3570         writel(0, &sdr_reg_file->fom);
3571         writel(0, &sdr_reg_file->failing_stage);
3572         writel(0, &sdr_reg_file->debug1);
3573         writel(0, &sdr_reg_file->debug2);
3574 }
3575
3576 /**
3577  * initialize_hps_phy() - Initialize HPS PHY
3578  *
3579  * Initialize HPS PHY.
3580  */
3581 static void initialize_hps_phy(void)
3582 {
3583         uint32_t reg;
3584         /*
3585          * Tracking also gets configured here because it's in the
3586          * same register.
3587          */
3588         uint32_t trk_sample_count = 7500;
3589         uint32_t trk_long_idle_sample_count = (10 << 16) | 100;
3590         /*
3591          * Format is number of outer loops in the 16 MSB, sample
3592          * count in 16 LSB.
3593          */
3594
3595         reg = 0;
3596         reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_ACDELAYEN_SET(2);
3597         reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_DQDELAYEN_SET(1);
3598         reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_DQSDELAYEN_SET(1);
3599         reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_DQSLOGICDELAYEN_SET(1);
3600         reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_RESETDELAYEN_SET(0);
3601         reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_LPDDRDIS_SET(1);
3602         /*
3603          * This field selects the intrinsic latency to RDATA_EN/FULL path.
3604          * 00-bypass, 01- add 5 cycles, 10- add 10 cycles, 11- add 15 cycles.
3605          */
3606         reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_ADDLATSEL_SET(0);
3607         reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_SAMPLECOUNT_19_0_SET(
3608                 trk_sample_count);
3609         writel(reg, &sdr_ctrl->phy_ctrl0);
3610
3611         reg = 0;
3612         reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_1_SAMPLECOUNT_31_20_SET(
3613                 trk_sample_count >>
3614                 SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_SAMPLECOUNT_19_0_WIDTH);
3615         reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_1_LONGIDLESAMPLECOUNT_19_0_SET(
3616                 trk_long_idle_sample_count);
3617         writel(reg, &sdr_ctrl->phy_ctrl1);
3618
3619         reg = 0;
3620         reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_2_LONGIDLESAMPLECOUNT_31_20_SET(
3621                 trk_long_idle_sample_count >>
3622                 SDR_CTRLGRP_PHYCTRL_PHYCTRL_1_LONGIDLESAMPLECOUNT_19_0_WIDTH);
3623         writel(reg, &sdr_ctrl->phy_ctrl2);
3624 }
3625
3626 /**
3627  * initialize_tracking() - Initialize tracking
3628  *
3629  * Initialize the register file with usable initial data.
3630  */
3631 static void initialize_tracking(void)
3632 {
3633         /*
3634          * Initialize the register file with the correct data.
3635          * Compute usable version of value in case we skip full
3636          * computation later.
3637          */
3638         writel(DIV_ROUND_UP(IO_DELAY_PER_OPA_TAP, IO_DELAY_PER_DCHAIN_TAP) - 1,
3639                &sdr_reg_file->dtaps_per_ptap);
3640
3641         /* trk_sample_count */
3642         writel(7500, &sdr_reg_file->trk_sample_count);
3643
3644         /* longidle outer loop [15:0] */
3645         writel((10 << 16) | (100 << 0), &sdr_reg_file->trk_longidle);
3646
3647         /*
3648          * longidle sample count [31:24]
3649          * trfc, worst case of 933Mhz 4Gb [23:16]
3650          * trcd, worst case [15:8]
3651          * vfifo wait [7:0]
3652          */
3653         writel((243 << 24) | (14 << 16) | (10 << 8) | (4 << 0),
3654                &sdr_reg_file->delays);
3655
3656         /* mux delay */
3657         writel((RW_MGR_IDLE << 24) | (RW_MGR_ACTIVATE_1 << 16) |
3658                (RW_MGR_SGLE_READ << 8) | (RW_MGR_PRECHARGE_ALL << 0),
3659                &sdr_reg_file->trk_rw_mgr_addr);
3660
3661         writel(RW_MGR_MEM_IF_READ_DQS_WIDTH,
3662                &sdr_reg_file->trk_read_dqs_width);
3663
3664         /* trefi [7:0] */
3665         writel((RW_MGR_REFRESH_ALL << 24) | (1000 << 0),
3666                &sdr_reg_file->trk_rfsh);
3667 }
3668
3669 int sdram_calibration_full(void)
3670 {
3671         struct param_type my_param;
3672         struct gbl_type my_gbl;
3673         uint32_t pass;
3674
3675         memset(&my_param, 0, sizeof(my_param));
3676         memset(&my_gbl, 0, sizeof(my_gbl));
3677
3678         param = &my_param;
3679         gbl = &my_gbl;
3680
3681         /* Set the calibration enabled by default */
3682         gbl->phy_debug_mode_flags |= PHY_DEBUG_ENABLE_CAL_RPT;
3683         /*
3684          * Only sweep all groups (regardless of fail state) by default
3685          * Set enabled read test by default.
3686          */
3687 #if DISABLE_GUARANTEED_READ
3688         gbl->phy_debug_mode_flags |= PHY_DEBUG_DISABLE_GUARANTEED_READ;
3689 #endif
3690         /* Initialize the register file */
3691         initialize_reg_file();
3692
3693         /* Initialize any PHY CSR */
3694         initialize_hps_phy();
3695
3696         scc_mgr_initialize();
3697
3698         initialize_tracking();
3699
3700         printf("%s: Preparing to start memory calibration\n", __FILE__);
3701
3702         debug("%s:%d\n", __func__, __LINE__);
3703         debug_cond(DLEVEL == 1,
3704                    "DDR3 FULL_RATE ranks=%u cs/dimm=%u dq/dqs=%u,%u vg/dqs=%u,%u ",
3705                    RW_MGR_MEM_NUMBER_OF_RANKS, RW_MGR_MEM_NUMBER_OF_CS_PER_DIMM,
3706                    RW_MGR_MEM_DQ_PER_READ_DQS, RW_MGR_MEM_DQ_PER_WRITE_DQS,
3707                    RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS,
3708                    RW_MGR_MEM_VIRTUAL_GROUPS_PER_WRITE_DQS);
3709         debug_cond(DLEVEL == 1,
3710                    "dqs=%u,%u dq=%u dm=%u ptap_delay=%u dtap_delay=%u ",
3711                    RW_MGR_MEM_IF_READ_DQS_WIDTH, RW_MGR_MEM_IF_WRITE_DQS_WIDTH,
3712                    RW_MGR_MEM_DATA_WIDTH, RW_MGR_MEM_DATA_MASK_WIDTH,
3713                    IO_DELAY_PER_OPA_TAP, IO_DELAY_PER_DCHAIN_TAP);
3714         debug_cond(DLEVEL == 1, "dtap_dqsen_delay=%u, dll=%u",
3715                    IO_DELAY_PER_DQS_EN_DCHAIN_TAP, IO_DLL_CHAIN_LENGTH);
3716         debug_cond(DLEVEL == 1, "max values: en_p=%u dqdqs_p=%u en_d=%u dqs_in_d=%u ",
3717                    IO_DQS_EN_PHASE_MAX, IO_DQDQS_OUT_PHASE_MAX,
3718                    IO_DQS_EN_DELAY_MAX, IO_DQS_IN_DELAY_MAX);
3719         debug_cond(DLEVEL == 1, "io_in_d=%u io_out1_d=%u io_out2_d=%u ",
3720                    IO_IO_IN_DELAY_MAX, IO_IO_OUT1_DELAY_MAX,
3721                    IO_IO_OUT2_DELAY_MAX);
3722         debug_cond(DLEVEL == 1, "dqs_in_reserve=%u dqs_out_reserve=%u\n",
3723                    IO_DQS_IN_RESERVE, IO_DQS_OUT_RESERVE);
3724
3725         hc_initialize_rom_data();
3726
3727         /* update info for sims */
3728         reg_file_set_stage(CAL_STAGE_NIL);
3729         reg_file_set_group(0);
3730
3731         /*
3732          * Load global needed for those actions that require
3733          * some dynamic calibration support.
3734          */
3735         dyn_calib_steps = STATIC_CALIB_STEPS;
3736         /*
3737          * Load global to allow dynamic selection of delay loop settings
3738          * based on calibration mode.
3739          */
3740         if (!(dyn_calib_steps & CALIB_SKIP_DELAY_LOOPS))
3741                 skip_delay_mask = 0xff;
3742         else
3743                 skip_delay_mask = 0x0;
3744
3745         pass = run_mem_calibrate();
3746         debug_mem_calibrate(pass);
3747         return pass;
3748 }