arch/x86/cpu/quark/smc.c

   1 /*
   2  * Copyright (C) 2013, Intel Corporation
   3  * Copyright (C) 2015, Bin Meng <bmeng.cn@gmail.com>
   4  *
   5  * Ported from Intel released Quark UEFI BIOS
   6  * QuarkSocPkg/QuarkNorthCluster/MemoryInit/Pei
   7  *
   8  * SPDX-License-Identifier:     Intel
   9  */
  10
  11 #include <common.h>
  12 #include <pci.h>
  13 #include <asm/arch/device.h>
  14 #include <asm/arch/mrc.h>
  15 #include <asm/arch/msg_port.h>
  16 #include "mrc_util.h"
  17 #include "hte.h"
  18 #include "smc.h"
  19
  20 /* t_rfc values (in picoseconds) per density */
  21 static const uint32_t t_rfc[5] = {
  22         90000,  /* 512Mb */
  23         110000, /* 1Gb */
  24         160000, /* 2Gb */
  25         300000, /* 4Gb */
  26         350000, /* 8Gb */
  27 };
  28
  29 /* t_ck clock period in picoseconds per speed index 800, 1066, 1333 */
  30 static const uint32_t t_ck[3] = {
  31         2500,
  32         1875,
  33         1500
  34 };
  35
  36 /* Global variables */
  37 static const uint16_t ddr_wclk[] = {193, 158};
  38 static const uint16_t ddr_wctl[] = {1, 217};
  39 static const uint16_t ddr_wcmd[] = {1, 220};
  40
  41 #ifdef BACKUP_RCVN
  42 static const uint16_t ddr_rcvn[] = {129, 498};
  43 #endif
  44
  45 #ifdef BACKUP_WDQS
  46 static const uint16_t ddr_wdqs[] = {65, 289};
  47 #endif
  48
  49 #ifdef BACKUP_RDQS
  50 static const uint8_t ddr_rdqs[] = {32, 24};
  51 #endif
  52
  53 #ifdef BACKUP_WDQ
  54 static const uint16_t ddr_wdq[] = {32, 257};
  55 #endif
  56
  57 /* Stop self refresh driven by MCU */
  58 void clear_self_refresh(struct mrc_params *mrc_params)
  59 {
  60         ENTERFN();
  61
  62         /* clear the PMSTS Channel Self Refresh bits */
  63         mrc_write_mask(MEM_CTLR, PMSTS, PMSTS_DISR, PMSTS_DISR);
  64
  65         LEAVEFN();
  66 }
  67
  68 /* It will initialize timing registers in the MCU (DTR0..DTR4) */
  69 void prog_ddr_timing_control(struct mrc_params *mrc_params)
  70 {
  71         uint8_t tcl, wl;
  72         uint8_t trp, trcd, tras, twr, twtr, trrd, trtp, tfaw;
  73         uint32_t tck;
  74         u32 dtr0, dtr1, dtr2, dtr3, dtr4;
  75         u32 tmp1, tmp2;
  76
  77         ENTERFN();
  78
  79         /* mcu_init starts */
  80         mrc_post_code(0x02, 0x00);
  81
  82         dtr0 = msg_port_read(MEM_CTLR, DTR0);
  83         dtr1 = msg_port_read(MEM_CTLR, DTR1);
  84         dtr2 = msg_port_read(MEM_CTLR, DTR2);
  85         dtr3 = msg_port_read(MEM_CTLR, DTR3);
  86         dtr4 = msg_port_read(MEM_CTLR, DTR4);
  87
  88         tck = t_ck[mrc_params->ddr_speed];      /* Clock in picoseconds */
  89         tcl = mrc_params->params.cl;            /* CAS latency in clocks */
  90         trp = tcl;      /* Per CAT MRC */
  91         trcd = tcl;     /* Per CAT MRC */
  92         tras = MCEIL(mrc_params->params.ras, tck);
  93
  94         /* Per JEDEC: tWR=15000ps DDR2/3 from 800-1600 */
  95         twr = MCEIL(15000, tck);
  96
  97         twtr = MCEIL(mrc_params->params.wtr, tck);
  98         trrd = MCEIL(mrc_params->params.rrd, tck);
  99         trtp = 4;       /* Valid for 800 and 1066, use 5 for 1333 */
 100         tfaw = MCEIL(mrc_params->params.faw, tck);
 101
 102         wl = 5 + mrc_params->ddr_speed;
 103
 104         dtr0 &= ~DTR0_DFREQ_MASK;
 105         dtr0 |= mrc_params->ddr_speed;
 106         dtr0 &= ~DTR0_TCL_MASK;
 107         tmp1 = tcl - 5;
 108         dtr0 |= ((tcl - 5) << 12);
 109         dtr0 &= ~DTR0_TRP_MASK;
 110         dtr0 |= ((trp - 5) << 4);       /* 5 bit DRAM Clock */
 111         dtr0 &= ~DTR0_TRCD_MASK;
 112         dtr0 |= ((trcd - 5) << 8);      /* 5 bit DRAM Clock */
 113
 114         dtr1 &= ~DTR1_TWCL_MASK;
 115         tmp2 = wl - 3;
 116         dtr1 |= (wl - 3);
 117         dtr1 &= ~DTR1_TWTP_MASK;
 118         dtr1 |= ((wl + 4 + twr - 14) << 8);     /* Change to tWTP */
 119         dtr1 &= ~DTR1_TRTP_MASK;
 120         dtr1 |= ((MMAX(trtp, 4) - 3) << 28);    /* 4 bit DRAM Clock */
 121         dtr1 &= ~DTR1_TRRD_MASK;
 122         dtr1 |= ((trrd - 4) << 24);             /* 4 bit DRAM Clock */
 123         dtr1 &= ~DTR1_TCMD_MASK;
 124         dtr1 |= (1 << 4);
 125         dtr1 &= ~DTR1_TRAS_MASK;
 126         dtr1 |= ((tras - 14) << 20);            /* 6 bit DRAM Clock */
 127         dtr1 &= ~DTR1_TFAW_MASK;
 128         dtr1 |= ((((tfaw + 1) >> 1) - 5) << 16);/* 4 bit DRAM Clock */
 129         /* Set 4 Clock CAS to CAS delay (multi-burst) */
 130         dtr1 &= ~DTR1_TCCD_MASK;
 131
 132         dtr2 &= ~DTR2_TRRDR_MASK;
 133         dtr2 |= 1;
 134         dtr2 &= ~DTR2_TWWDR_MASK;
 135         dtr2 |= (2 << 8);
 136         dtr2 &= ~DTR2_TRWDR_MASK;
 137         dtr2 |= (2 << 16);
 138
 139         dtr3 &= ~DTR3_TWRDR_MASK;
 140         dtr3 |= 2;
 141         dtr3 &= ~DTR3_TXXXX_MASK;
 142         dtr3 |= (2 << 4);
 143
 144         dtr3 &= ~DTR3_TRWSR_MASK;
 145         if (mrc_params->ddr_speed == DDRFREQ_800) {
 146                 /* Extended RW delay (+1) */
 147                 dtr3 |= ((tcl - 5 + 1) << 8);
 148         } else if (mrc_params->ddr_speed == DDRFREQ_1066) {
 149                 /* Extended RW delay (+1) */
 150                 dtr3 |= ((tcl - 5 + 1) << 8);
 151         }
 152
 153         dtr3 &= ~DTR3_TWRSR_MASK;
 154         dtr3 |= ((4 + wl + twtr - 11) << 13);
 155
 156         dtr3 &= ~DTR3_TXP_MASK;
 157         if (mrc_params->ddr_speed == DDRFREQ_800)
 158                 dtr3 |= ((MMAX(0, 1 - 1)) << 22);
 159         else
 160                 dtr3 |= ((MMAX(0, 2 - 1)) << 22);
 161
 162         dtr4 &= ~DTR4_WRODTSTRT_MASK;
 163         dtr4 |= 1;
 164         dtr4 &= ~DTR4_WRODTSTOP_MASK;
 165         dtr4 |= (1 << 4);
 166         dtr4 &= ~DTR4_XXXX1_MASK;
 167         dtr4 |= ((1 + tmp1 - tmp2 + 2) << 8);
 168         dtr4 &= ~DTR4_XXXX2_MASK;
 169         dtr4 |= ((1 + tmp1 - tmp2 + 2) << 12);
 170         dtr4 &= ~(DTR4_ODTDIS | DTR4_TRGSTRDIS);
 171
 172         msg_port_write(MEM_CTLR, DTR0, dtr0);
 173         msg_port_write(MEM_CTLR, DTR1, dtr1);
 174         msg_port_write(MEM_CTLR, DTR2, dtr2);
 175         msg_port_write(MEM_CTLR, DTR3, dtr3);
 176         msg_port_write(MEM_CTLR, DTR4, dtr4);
 177
 178         LEAVEFN();
 179 }
 180
 181 /* Configure MCU before jedec init sequence */
 182 void prog_decode_before_jedec(struct mrc_params *mrc_params)
 183 {
 184         u32 drp;
 185         u32 drfc;
 186         u32 dcal;
 187         u32 dsch;
 188         u32 dpmc0;
 189
 190         ENTERFN();
 191
 192         /* Disable power saving features */
 193         dpmc0 = msg_port_read(MEM_CTLR, DPMC0);
 194         dpmc0 |= (DPMC0_CLKGTDIS | DPMC0_DISPWRDN);
 195         dpmc0 &= ~DPMC0_PCLSTO_MASK;
 196         dpmc0 &= ~DPMC0_DYNSREN;
 197         msg_port_write(MEM_CTLR, DPMC0, dpmc0);
 198
 199         /* Disable out of order transactions */
 200         dsch = msg_port_read(MEM_CTLR, DSCH);
 201         dsch |= (DSCH_OOODIS | DSCH_NEWBYPDIS);
 202         msg_port_write(MEM_CTLR, DSCH, dsch);
 203
 204         /* Disable issuing the REF command */
 205         drfc = msg_port_read(MEM_CTLR, DRFC);
 206         drfc &= ~DRFC_TREFI_MASK;
 207         msg_port_write(MEM_CTLR, DRFC, drfc);
 208
 209         /* Disable ZQ calibration short */
 210         dcal = msg_port_read(MEM_CTLR, DCAL);
 211         dcal &= ~DCAL_ZQCINT_MASK;
 212         dcal &= ~DCAL_SRXZQCL_MASK;
 213         msg_port_write(MEM_CTLR, DCAL, dcal);
 214
 215         /*
 216          * Training performed in address mode 0, rank population has limited
 217          * impact, however simulator complains if enabled non-existing rank.
 218          */
 219         drp = 0;
 220         if (mrc_params->rank_enables & 1)
 221                 drp |= DRP_RKEN0;
 222         if (mrc_params->rank_enables & 2)
 223                 drp |= DRP_RKEN1;
 224         msg_port_write(MEM_CTLR, DRP, drp);
 225
 226         LEAVEFN();
 227 }
 228
 229 /*
 230  * After Cold Reset, BIOS should set COLDWAKE bit to 1 before
 231  * sending the WAKE message to the Dunit.
 232  *
 233  * For Standby Exit, or any other mode in which the DRAM is in
 234  * SR, this bit must be set to 0.
 235  */
 236 void perform_ddr_reset(struct mrc_params *mrc_params)
 237 {
 238         ENTERFN();
 239
 240         /* Set COLDWAKE bit before sending the WAKE message */
 241         mrc_write_mask(MEM_CTLR, DRMC, DRMC_COLDWAKE, DRMC_COLDWAKE);
 242
 243         /* Send wake command to DUNIT (MUST be done before JEDEC) */
 244         dram_wake_command();
 245
 246         /* Set default value */
 247         msg_port_write(MEM_CTLR, DRMC,
 248                        mrc_params->rd_odt_value == 0 ? DRMC_ODTMODE : 0);
 249
 250         LEAVEFN();
 251 }
 252
 253
 254 /*
 255  * This function performs some initialization on the DDRIO unit.
 256  * This function is dependent on BOARD_ID, DDR_SPEED, and CHANNEL_ENABLES.
 257  */
 258 void ddrphy_init(struct mrc_params *mrc_params)
 259 {
 260         uint32_t temp;
 261         uint8_t ch;     /* channel counter */
 262         uint8_t rk;     /* rank counter */
 263         uint8_t bl_grp; /*  byte lane group counter (2 BLs per module) */
 264         uint8_t bl_divisor = 1; /* byte lane divisor */
 265         /* For DDR3 --> 0 == 800, 1 == 1066, 2 == 1333 */
 266         uint8_t speed = mrc_params->ddr_speed & 3;
 267         uint8_t cas;
 268         uint8_t cwl;
 269
 270         ENTERFN();
 271
 272         cas = mrc_params->params.cl;
 273         cwl = 5 + mrc_params->ddr_speed;
 274
 275         /* ddrphy_init starts */
 276         mrc_post_code(0x03, 0x00);
 277
 278         /*
 279          * HSD#231531
 280          * Make sure IOBUFACT is deasserted before initializing the DDR PHY
 281          *
 282          * HSD#234845
 283          * Make sure WRPTRENABLE is deasserted before initializing the DDR PHY
 284          */
 285         for (ch = 0; ch < NUM_CHANNELS; ch++) {
 286                 if (mrc_params->channel_enables & (1 << ch)) {
 287                         /* Deassert DDRPHY Initialization Complete */
 288                         mrc_alt_write_mask(DDRPHY,
 289                                 CMDPMCONFIG0 + ch * DDRIOCCC_CH_OFFSET,
 290                                 ~(1 << 20), 1 << 20);   /* SPID_INIT_COMPLETE=0 */
 291                         /* Deassert IOBUFACT */
 292                         mrc_alt_write_mask(DDRPHY,
 293                                 CMDCFGREG0 + ch * DDRIOCCC_CH_OFFSET,
 294                                 ~(1 << 2), 1 << 2);     /* IOBUFACTRST_N=0 */
 295                         /* Disable WRPTR */
 296                         mrc_alt_write_mask(DDRPHY,
 297                                 CMDPTRREG + ch * DDRIOCCC_CH_OFFSET,
 298                                 ~(1 << 0), 1 << 0);     /* WRPTRENABLE=0 */
 299                 }
 300         }
 301
 302         /* Put PHY in reset */
 303         mrc_alt_write_mask(DDRPHY, MASTERRSTN, 0, 1);
 304
 305         /* Initialize DQ01, DQ23, CMD, CLK-CTL, COMP modules */
 306
 307         /* STEP0 */
 308         mrc_post_code(0x03, 0x10);
 309         for (ch = 0; ch < NUM_CHANNELS; ch++) {
 310                 if (mrc_params->channel_enables & (1 << ch)) {
 311                         /* DQ01-DQ23 */
 312                         for (bl_grp = 0;
 313                              bl_grp < (NUM_BYTE_LANES / bl_divisor) / 2;
 314                              bl_grp++) {
 315                                 /* Analog MUX select - IO2xCLKSEL */
 316                                 mrc_alt_write_mask(DDRPHY,
 317                                         DQOBSCKEBBCTL +
 318                                         bl_grp * DDRIODQ_BL_OFFSET +
 319                                         ch * DDRIODQ_CH_OFFSET,
 320                                         bl_grp ? 0 : (1 << 22), 1 << 22);
 321
 322                                 /* ODT Strength */
 323                                 switch (mrc_params->rd_odt_value) {
 324                                 case 1:
 325                                         temp = 0x3;
 326                                         break;  /* 60 ohm */
 327                                 case 2:
 328                                         temp = 0x3;
 329                                         break;  /* 120 ohm */
 330                                 case 3:
 331                                         temp = 0x3;
 332                                         break;  /* 180 ohm */
 333                                 default:
 334                                         temp = 0x3;
 335                                         break;  /* 120 ohm */
 336                                 }
 337
 338                                 /* ODT strength */
 339                                 mrc_alt_write_mask(DDRPHY,
 340                                         B0RXIOBUFCTL +
 341                                         bl_grp * DDRIODQ_BL_OFFSET +
 342                                         ch * DDRIODQ_CH_OFFSET,
 343                                         temp << 5, 0x60);
 344                                 /* ODT strength */
 345                                 mrc_alt_write_mask(DDRPHY,
 346                                         B1RXIOBUFCTL +
 347                                         bl_grp * DDRIODQ_BL_OFFSET +
 348                                         ch * DDRIODQ_CH_OFFSET,
 349                                         temp << 5, 0x60);
 350
 351                                 /* Dynamic ODT/DIFFAMP */
 352                                 temp = (cas << 24) | (cas << 16) |
 353                                         (cas << 8) | (cas << 0);
 354                                 switch (speed) {
 355                                 case 0:
 356                                         temp -= 0x01010101;
 357                                         break;  /* 800 */
 358                                 case 1:
 359                                         temp -= 0x02020202;
 360                                         break;  /* 1066 */
 361                                 case 2:
 362                                         temp -= 0x03030303;
 363                                         break;  /* 1333 */
 364                                 case 3:
 365                                         temp -= 0x04040404;
 366                                         break;  /* 1600 */
 367                                 }
 368
 369                                 /* Launch Time: ODT, DIFFAMP, ODT, DIFFAMP */
 370                                 mrc_alt_write_mask(DDRPHY,
 371                                         B01LATCTL1 +
 372                                         bl_grp * DDRIODQ_BL_OFFSET +
 373                                         ch * DDRIODQ_CH_OFFSET,
 374                                         temp, 0x1f1f1f1f);
 375                                 switch (speed) {
 376                                 /* HSD#234715 */
 377                                 case 0:
 378                                         temp = (0x06 << 16) | (0x07 << 8);
 379                                         break;  /* 800 */
 380                                 case 1:
 381                                         temp = (0x07 << 16) | (0x08 << 8);
 382                                         break;  /* 1066 */
 383                                 case 2:
 384                                         temp = (0x09 << 16) | (0x0a << 8);
 385                                         break;  /* 1333 */
 386                                 case 3:
 387                                         temp = (0x0a << 16) | (0x0b << 8);
 388                                         break;  /* 1600 */
 389                                 }
 390
 391                                 /* On Duration: ODT, DIFFAMP */
 392                                 mrc_alt_write_mask(DDRPHY,
 393                                         B0ONDURCTL +
 394                                         bl_grp * DDRIODQ_BL_OFFSET +
 395                                         ch * DDRIODQ_CH_OFFSET,
 396                                         temp, 0x003f3f00);
 397                                 /* On Duration: ODT, DIFFAMP */
 398                                 mrc_alt_write_mask(DDRPHY,
 399                                         B1ONDURCTL +
 400                                         bl_grp * DDRIODQ_BL_OFFSET +
 401                                         ch * DDRIODQ_CH_OFFSET,
 402                                         temp, 0x003f3f00);
 403
 404                                 switch (mrc_params->rd_odt_value) {
 405                                 case 0:
 406                                         /* override DIFFAMP=on, ODT=off */
 407                                         temp = (0x3f << 16) | (0x3f << 10);
 408                                         break;
 409                                 default:
 410                                         /* override DIFFAMP=on, ODT=on */
 411                                         temp = (0x3f << 16) | (0x2a << 10);
 412                                         break;
 413                                 }
 414
 415                                 /* Override: DIFFAMP, ODT */
 416                                 mrc_alt_write_mask(DDRPHY,
 417                                         B0OVRCTL +
 418                                         bl_grp * DDRIODQ_BL_OFFSET +
 419                                         ch * DDRIODQ_CH_OFFSET,
 420                                         temp, 0x003ffc00);
 421                                 /* Override: DIFFAMP, ODT */
 422                                 mrc_alt_write_mask(DDRPHY,
 423                                         B1OVRCTL +
 424                                         bl_grp * DDRIODQ_BL_OFFSET +
 425                                         ch * DDRIODQ_CH_OFFSET,
 426                                         temp, 0x003ffc00);
 427
 428                                 /* DLL Setup */
 429
 430                                 /* 1xCLK Domain Timings: tEDP,RCVEN,WDQS (PO) */
 431                                 mrc_alt_write_mask(DDRPHY,
 432                                         B0LATCTL0 +
 433                                         bl_grp * DDRIODQ_BL_OFFSET +
 434                                         ch * DDRIODQ_CH_OFFSET,
 435                                         ((cas + 7) << 16) | ((cas - 4) << 8) |
 436                                         ((cwl - 2) << 0), 0x003f1f1f);
 437                                 mrc_alt_write_mask(DDRPHY,
 438                                         B1LATCTL0 +
 439                                         bl_grp * DDRIODQ_BL_OFFSET +
 440                                         ch * DDRIODQ_CH_OFFSET,
 441                                         ((cas + 7) << 16) | ((cas - 4) << 8) |
 442                                         ((cwl - 2) << 0), 0x003f1f1f);
 443
 444                                 /* RCVEN Bypass (PO) */
 445                                 mrc_alt_write_mask(DDRPHY,
 446                                         B0RXIOBUFCTL +
 447                                         bl_grp * DDRIODQ_BL_OFFSET +
 448                                         ch * DDRIODQ_CH_OFFSET,
 449                                         0, 0x81);
 450                                 mrc_alt_write_mask(DDRPHY,
 451                                         B1RXIOBUFCTL +
 452                                         bl_grp * DDRIODQ_BL_OFFSET +
 453                                         ch * DDRIODQ_CH_OFFSET,
 454                                         0, 0x81);
 455
 456                                 /* TX */
 457                                 mrc_alt_write_mask(DDRPHY,
 458                                         DQCTL +
 459                                         bl_grp * DDRIODQ_BL_OFFSET +
 460                                         ch * DDRIODQ_CH_OFFSET,
 461                                         1 << 16, 1 << 16);
 462                                 mrc_alt_write_mask(DDRPHY,
 463                                         B01PTRCTL1 +
 464                                         bl_grp * DDRIODQ_BL_OFFSET +
 465                                         ch * DDRIODQ_CH_OFFSET,
 466                                         1 << 8, 1 << 8);
 467
 468                                 /* RX (PO) */
 469                                 /* Internal Vref Code, Enable#, Ext_or_Int (1=Ext) */
 470                                 mrc_alt_write_mask(DDRPHY,
 471                                         B0VREFCTL +
 472                                         bl_grp * DDRIODQ_BL_OFFSET +
 473                                         ch * DDRIODQ_CH_OFFSET,
 474                                         (0x03 << 2) | (0x0 << 1) | (0x0 << 0),
 475                                         0xff);
 476                                 /* Internal Vref Code, Enable#, Ext_or_Int (1=Ext) */
 477                                 mrc_alt_write_mask(DDRPHY,
 478                                         B1VREFCTL +
 479                                         bl_grp * DDRIODQ_BL_OFFSET +
 480                                         ch * DDRIODQ_CH_OFFSET,
 481                                         (0x03 << 2) | (0x0 << 1) | (0x0 << 0),
 482                                         0xff);
 483                                 /* Per-Bit De-Skew Enable */
 484                                 mrc_alt_write_mask(DDRPHY,
 485                                         B0RXIOBUFCTL +
 486                                         bl_grp * DDRIODQ_BL_OFFSET +
 487                                         ch * DDRIODQ_CH_OFFSET,
 488                                         0, 0x10);
 489                                 /* Per-Bit De-Skew Enable */
 490                                 mrc_alt_write_mask(DDRPHY,
 491                                         B1RXIOBUFCTL +
 492                                         bl_grp * DDRIODQ_BL_OFFSET +
 493                                         ch * DDRIODQ_CH_OFFSET,
 494                                         0, 0x10);
 495                         }
 496
 497                         /* CLKEBB */
 498                         mrc_alt_write_mask(DDRPHY,
 499                                 CMDOBSCKEBBCTL + ch * DDRIOCCC_CH_OFFSET,
 500                                 0, 1 << 23);
 501
 502                         /* Enable tristate control of cmd/address bus */
 503                         mrc_alt_write_mask(DDRPHY,
 504                                 CMDCFGREG0 + ch * DDRIOCCC_CH_OFFSET,
 505                                 0, 0x03);
 506
 507                         /* ODT RCOMP */
 508                         mrc_alt_write_mask(DDRPHY,
 509                                 CMDRCOMPODT + ch * DDRIOCCC_CH_OFFSET,
 510                                 (0x03 << 5) | (0x03 << 0), 0x3ff);
 511
 512                         /* CMDPM* registers must be programmed in this order */
 513
 514                         /* Turn On Delays: SFR (regulator), MPLL */
 515                         mrc_alt_write_mask(DDRPHY,
 516                                 CMDPMDLYREG4 + ch * DDRIOCCC_CH_OFFSET,
 517                                 0xffffffff, 0xffffffff);
 518                         /*
 519                          * Delays: ASSERT_IOBUFACT_to_ALLON0_for_PM_MSG_3,
 520                          * VREG (MDLL) Turn On, ALLON0_to_DEASSERT_IOBUFACT
 521                          * for_PM_MSG_gt0, MDLL Turn On
 522                          */
 523                         mrc_alt_write_mask(DDRPHY,
 524                                 CMDPMDLYREG3 + ch * DDRIOCCC_CH_OFFSET,
 525                                 0xfffff616, 0xffffffff);
 526                         /* MPLL Divider Reset Delays */
 527                         mrc_alt_write_mask(DDRPHY,
 528                                 CMDPMDLYREG2 + ch * DDRIOCCC_CH_OFFSET,
 529                                 0xffffffff, 0xffffffff);
 530                         /* Turn Off Delays: VREG, Staggered MDLL, MDLL, PI */
 531                         mrc_alt_write_mask(DDRPHY,
 532                                 CMDPMDLYREG1 + ch * DDRIOCCC_CH_OFFSET,
 533                                 0xffffffff, 0xffffffff);
 534                         /* Turn On Delays: MPLL, Staggered MDLL, PI, IOBUFACT */
 535                         mrc_alt_write_mask(DDRPHY,
 536                                 CMDPMDLYREG0 + ch * DDRIOCCC_CH_OFFSET,
 537                                 0xffffffff, 0xffffffff);
 538                         /* Allow PUnit signals */
 539                         mrc_alt_write_mask(DDRPHY,
 540                                 CMDPMCONFIG0 + ch * DDRIOCCC_CH_OFFSET,
 541                                 (0x6 << 8) | (0x1 << 6) | (0x4 << 0),
 542                                 0xffe00f4f);
 543                         /* DLL_VREG Bias Trim, VREF Tuning for DLL_VREG */
 544                         mrc_alt_write_mask(DDRPHY,
 545                                 CMDMDLLCTL + ch * DDRIOCCC_CH_OFFSET,
 546                                 (0x3 << 4) | (0x7 << 0), 0x7f);
 547
 548                         /* CLK-CTL */
 549                         mrc_alt_write_mask(DDRPHY,
 550                                 CCOBSCKEBBCTL + ch * DDRIOCCC_CH_OFFSET,
 551                                 0, 1 << 24);    /* CLKEBB */
 552                         /* Buffer Enable: CS,CKE,ODT,CLK */
 553                         mrc_alt_write_mask(DDRPHY,
 554                                 CCCFGREG0 + ch * DDRIOCCC_CH_OFFSET,
 555                                 0x1f, 0x000ffff1);
 556                         /* ODT RCOMP */
 557                         mrc_alt_write_mask(DDRPHY,
 558                                 CCRCOMPODT + ch * DDRIOCCC_CH_OFFSET,
 559                                 (0x03 << 8) | (0x03 << 0), 0x00001f1f);
 560                         /* DLL_VREG Bias Trim, VREF Tuning for DLL_VREG */
 561                         mrc_alt_write_mask(DDRPHY,
 562                                 CCMDLLCTL + ch * DDRIOCCC_CH_OFFSET,
 563                                 (0x3 << 4) | (0x7 << 0), 0x7f);
 564
 565                         /*
 566                          * COMP (RON channel specific)
 567                          * - DQ/DQS/DM RON: 32 Ohm
 568                          * - CTRL/CMD RON: 27 Ohm
 569                          * - CLK RON: 26 Ohm
 570                          */
 571                         /* RCOMP Vref PU/PD */
 572                         mrc_alt_write_mask(DDRPHY,
 573                                 DQVREFCH0 +  ch * DDRCOMP_CH_OFFSET,
 574                                 (0x08 << 24) | (0x03 << 16), 0x3f3f0000);
 575                         /* RCOMP Vref PU/PD */
 576                         mrc_alt_write_mask(DDRPHY,
 577                                 CMDVREFCH0 + ch * DDRCOMP_CH_OFFSET,
 578                                 (0x0C << 24) | (0x03 << 16), 0x3f3f0000);
 579                         /* RCOMP Vref PU/PD */
 580                         mrc_alt_write_mask(DDRPHY,
 581                                 CLKVREFCH0 + ch * DDRCOMP_CH_OFFSET,
 582                                 (0x0F << 24) | (0x03 << 16), 0x3f3f0000);
 583                         /* RCOMP Vref PU/PD */
 584                         mrc_alt_write_mask(DDRPHY,
 585                                 DQSVREFCH0 + ch * DDRCOMP_CH_OFFSET,
 586                                 (0x08 << 24) | (0x03 << 16), 0x3f3f0000);
 587                         /* RCOMP Vref PU/PD */
 588                         mrc_alt_write_mask(DDRPHY,
 589                                 CTLVREFCH0 + ch * DDRCOMP_CH_OFFSET,
 590                                 (0x0C << 24) | (0x03 << 16), 0x3f3f0000);
 591
 592                         /* DQS Swapped Input Enable */
 593                         mrc_alt_write_mask(DDRPHY,
 594                                 COMPEN1CH0 + ch * DDRCOMP_CH_OFFSET,
 595                                 (1 << 19) | (1 << 17), 0xc00ac000);
 596
 597                         /* ODT VREF = 1.5 x 274/360+274 = 0.65V (code of ~50) */
 598                         /* ODT Vref PU/PD */
 599                         mrc_alt_write_mask(DDRPHY,
 600                                 DQVREFCH0 + ch * DDRCOMP_CH_OFFSET,
 601                                 (0x32 << 8) | (0x03 << 0), 0x00003f3f);
 602                         /* ODT Vref PU/PD */
 603                         mrc_alt_write_mask(DDRPHY,
 604                                 DQSVREFCH0 + ch * DDRCOMP_CH_OFFSET,
 605                                 (0x32 << 8) | (0x03 << 0), 0x00003f3f);
 606                         /* ODT Vref PU/PD */
 607                         mrc_alt_write_mask(DDRPHY,
 608                                 CLKVREFCH0 + ch * DDRCOMP_CH_OFFSET,
 609                                 (0x0E << 8) | (0x05 << 0), 0x00003f3f);
 610
 611                         /*
 612                          * Slew rate settings are frequency specific,
 613                          * numbers below are for 800Mhz (speed == 0)
 614                          * - DQ/DQS/DM/CLK SR: 4V/ns,
 615                          * - CTRL/CMD SR: 1.5V/ns
 616                          */
 617                         temp = (0x0e << 16) | (0x0e << 12) | (0x08 << 8) |
 618                                 (0x0b << 4) | (0x0b << 0);
 619                         /* DCOMP Delay Select: CTL,CMD,CLK,DQS,DQ */
 620                         mrc_alt_write_mask(DDRPHY,
 621                                 DLYSELCH0 + ch * DDRCOMP_CH_OFFSET,
 622                                 temp, 0x000fffff);
 623                         /* TCO Vref CLK,DQS,DQ */
 624                         mrc_alt_write_mask(DDRPHY,
 625                                 TCOVREFCH0 + ch * DDRCOMP_CH_OFFSET,
 626                                 (0x05 << 16) | (0x05 << 8) | (0x05 << 0),
 627                                 0x003f3f3f);
 628                         /* ODTCOMP CMD/CTL PU/PD */
 629                         mrc_alt_write_mask(DDRPHY,
 630                                 CCBUFODTCH0 + ch * DDRCOMP_CH_OFFSET,
 631                                 (0x03 << 8) | (0x03 << 0),
 632                                 0x00001f1f);
 633                         /* COMP */
 634                         mrc_alt_write_mask(DDRPHY,
 635                                 COMPEN0CH0 + ch * DDRCOMP_CH_OFFSET,
 636                                 0, 0xc0000100);
 637
 638 #ifdef BACKUP_COMPS
 639                         /* DQ COMP Overrides */
 640                         /* RCOMP PU */
 641                         mrc_alt_write_mask(DDRPHY,
 642                                 DQDRVPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 643                                 (1 << 31) | (0x0a << 16),
 644                                 0x801f0000);
 645                         /* RCOMP PD */
 646                         mrc_alt_write_mask(DDRPHY,
 647                                 DQDRVPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 648                                 (1 << 31) | (0x0a << 16),
 649                                 0x801f0000);
 650                         /* DCOMP PU */
 651                         mrc_alt_write_mask(DDRPHY,
 652                                 DQDLYPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 653                                 (1 << 31) | (0x10 << 16),
 654                                 0x801f0000);
 655                         /* DCOMP PD */
 656                         mrc_alt_write_mask(DDRPHY,
 657                                 DQDLYPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 658                                 (1 << 31) | (0x10 << 16),
 659                                 0x801f0000);
 660                         /* ODTCOMP PU */
 661                         mrc_alt_write_mask(DDRPHY,
 662                                 DQODTPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 663                                 (1 << 31) | (0x0b << 16),
 664                                 0x801f0000);
 665                         /* ODTCOMP PD */
 666                         mrc_alt_write_mask(DDRPHY,
 667                                 DQODTPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 668                                 (1 << 31) | (0x0b << 16),
 669                                 0x801f0000);
 670                         /* TCOCOMP PU */
 671                         mrc_alt_write_mask(DDRPHY,
 672                                 DQTCOPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 673                                 1 << 31, 1 << 31);
 674                         /* TCOCOMP PD */
 675                         mrc_alt_write_mask(DDRPHY,
 676                                 DQTCOPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 677                                 1 << 31, 1 << 31);
 678
 679                         /* DQS COMP Overrides */
 680                         /* RCOMP PU */
 681                         mrc_alt_write_mask(DDRPHY,
 682                                 DQSDRVPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 683                                 (1 << 31) | (0x0a << 16),
 684                                 0x801f0000);
 685                         /* RCOMP PD */
 686                         mrc_alt_write_mask(DDRPHY,
 687                                 DQSDRVPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 688                                 (1 << 31) | (0x0a << 16),
 689                                 0x801f0000);
 690                         /* DCOMP PU */
 691                         mrc_alt_write_mask(DDRPHY,
 692                                 DQSDLYPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 693                                 (1 << 31) | (0x10 << 16),
 694                                 0x801f0000);
 695                         /* DCOMP PD */
 696                         mrc_alt_write_mask(DDRPHY,
 697                                 DQSDLYPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 698                                 (1 << 31) | (0x10 << 16),
 699                                 0x801f0000);
 700                         /* ODTCOMP PU */
 701                         mrc_alt_write_mask(DDRPHY,
 702                                 DQSODTPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 703                                 (1 << 31) | (0x0b << 16),
 704                                 0x801f0000);
 705                         /* ODTCOMP PD */
 706                         mrc_alt_write_mask(DDRPHY,
 707                                 DQSODTPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 708                                 (1 << 31) | (0x0b << 16),
 709                                 0x801f0000);
 710                         /* TCOCOMP PU */
 711                         mrc_alt_write_mask(DDRPHY,
 712                                 DQSTCOPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 713                                 1 << 31, 1 << 31);
 714                         /* TCOCOMP PD */
 715                         mrc_alt_write_mask(DDRPHY,
 716                                 DQSTCOPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 717                                 1 << 31, 1 << 31);
 718
 719                         /* CLK COMP Overrides */
 720                         /* RCOMP PU */
 721                         mrc_alt_write_mask(DDRPHY,
 722                                 CLKDRVPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 723                                 (1 << 31) | (0x0c << 16),
 724                                 0x801f0000);
 725                         /* RCOMP PD */
 726                         mrc_alt_write_mask(DDRPHY,
 727                                 CLKDRVPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 728                                 (1 << 31) | (0x0c << 16),
 729                                 0x801f0000);
 730                         /* DCOMP PU */
 731                         mrc_alt_write_mask(DDRPHY,
 732                                 CLKDLYPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 733                                 (1 << 31) | (0x07 << 16),
 734                                 0x801f0000);
 735                         /* DCOMP PD */
 736                         mrc_alt_write_mask(DDRPHY,
 737                                 CLKDLYPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 738                                 (1 << 31) | (0x07 << 16),
 739                                 0x801f0000);
 740                         /* ODTCOMP PU */
 741                         mrc_alt_write_mask(DDRPHY,
 742                                 CLKODTPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 743                                 (1 << 31) | (0x0b << 16),
 744                                 0x801f0000);
 745                         /* ODTCOMP PD */
 746                         mrc_alt_write_mask(DDRPHY,
 747                                 CLKODTPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 748                                 (1 << 31) | (0x0b << 16),
 749                                 0x801f0000);
 750                         /* TCOCOMP PU */
 751                         mrc_alt_write_mask(DDRPHY,
 752                                 CLKTCOPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 753                                 1 << 31, 1 << 31);
 754                         /* TCOCOMP PD */
 755                         mrc_alt_write_mask(DDRPHY,
 756                                 CLKTCOPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 757                                 1 << 31, 1 << 31);
 758
 759                         /* CMD COMP Overrides */
 760                         /* RCOMP PU */
 761                         mrc_alt_write_mask(DDRPHY,
 762                                 CMDDRVPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 763                                 (1 << 31) | (0x0d << 16),
 764                                 0x803f0000);
 765                         /* RCOMP PD */
 766                         mrc_alt_write_mask(DDRPHY,
 767                                 CMDDRVPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 768                                 (1 << 31) | (0x0d << 16),
 769                                 0x803f0000);
 770                         /* DCOMP PU */
 771                         mrc_alt_write_mask(DDRPHY,
 772                                 CMDDLYPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 773                                 (1 << 31) | (0x0a << 16),
 774                                 0x801f0000);
 775                         /* DCOMP PD */
 776                         mrc_alt_write_mask(DDRPHY,
 777                                 CMDDLYPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 778                                 (1 << 31) | (0x0a << 16),
 779                                 0x801f0000);
 780
 781                         /* CTL COMP Overrides */
 782                         /* RCOMP PU */
 783                         mrc_alt_write_mask(DDRPHY,
 784                                 CTLDRVPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 785                                 (1 << 31) | (0x0d << 16),
 786                                 0x803f0000);
 787                         /* RCOMP PD */
 788                         mrc_alt_write_mask(DDRPHY,
 789                                 CTLDRVPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 790                                 (1 << 31) | (0x0d << 16),
 791                                 0x803f0000);
 792                         /* DCOMP PU */
 793                         mrc_alt_write_mask(DDRPHY,
 794                                 CTLDLYPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 795                                 (1 << 31) | (0x0a << 16),
 796                                 0x801f0000);
 797                         /* DCOMP PD */
 798                         mrc_alt_write_mask(DDRPHY,
 799                                 CTLDLYPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 800                                 (1 << 31) | (0x0a << 16),
 801                                 0x801f0000);
 802 #else
 803                         /* DQ TCOCOMP Overrides */
 804                         /* TCOCOMP PU */
 805                         mrc_alt_write_mask(DDRPHY,
 806                                 DQTCOPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 807                                 (1 << 31) | (0x1f << 16),
 808                                 0x801f0000);
 809                         /* TCOCOMP PD */
 810                         mrc_alt_write_mask(DDRPHY,
 811                                 DQTCOPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 812                                 (1 << 31) | (0x1f << 16),
 813                                 0x801f0000);
 814
 815                         /* DQS TCOCOMP Overrides */
 816                         /* TCOCOMP PU */
 817                         mrc_alt_write_mask(DDRPHY,
 818                                 DQSTCOPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 819                                 (1 << 31) | (0x1f << 16),
 820                                 0x801f0000);
 821                         /* TCOCOMP PD */
 822                         mrc_alt_write_mask(DDRPHY,
 823                                 DQSTCOPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 824                                 (1 << 31) | (0x1f << 16),
 825                                 0x801f0000);
 826
 827                         /* CLK TCOCOMP Overrides */
 828                         /* TCOCOMP PU */
 829                         mrc_alt_write_mask(DDRPHY,
 830                                 CLKTCOPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 831                                 (1 << 31) | (0x1f << 16),
 832                                 0x801f0000);
 833                         /* TCOCOMP PD */
 834                         mrc_alt_write_mask(DDRPHY,
 835                                 CLKTCOPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 836                                 (1 << 31) | (0x1f << 16),
 837                                 0x801f0000);
 838 #endif
 839
 840                         /* program STATIC delays */
 841 #ifdef BACKUP_WCMD
 842                         set_wcmd(ch, ddr_wcmd[PLATFORM_ID]);
 843 #else
 844                         set_wcmd(ch, ddr_wclk[PLATFORM_ID] + HALF_CLK);
 845 #endif
 846
 847                         for (rk = 0; rk < NUM_RANKS; rk++) {
 848                                 if (mrc_params->rank_enables & (1 << rk)) {
 849                                         set_wclk(ch, rk, ddr_wclk[PLATFORM_ID]);
 850 #ifdef BACKUP_WCTL
 851                                         set_wctl(ch, rk, ddr_wctl[PLATFORM_ID]);
 852 #else
 853                                         set_wctl(ch, rk, ddr_wclk[PLATFORM_ID] + HALF_CLK);
 854 #endif
 855                                 }
 856                         }
 857                 }
 858         }
 859
 860         /* COMP (non channel specific) */
 861         /* RCOMP: Dither PU Enable */
 862         mrc_alt_write_mask(DDRPHY, DQANADRVPUCTL, 1 << 30, 1 << 30);
 863         /* RCOMP: Dither PD Enable */
 864         mrc_alt_write_mask(DDRPHY, DQANADRVPDCTL, 1 << 30, 1 << 30);
 865         /* RCOMP: Dither PU Enable */
 866         mrc_alt_write_mask(DDRPHY, CMDANADRVPUCTL, 1 << 30, 1 << 30);
 867         /* RCOMP: Dither PD Enable */
 868         mrc_alt_write_mask(DDRPHY, CMDANADRVPDCTL, 1 << 30, 1 << 30);
 869         /* RCOMP: Dither PU Enable */
 870         mrc_alt_write_mask(DDRPHY, CLKANADRVPUCTL, 1 << 30, 1 << 30);
 871         /* RCOMP: Dither PD Enable */
 872         mrc_alt_write_mask(DDRPHY, CLKANADRVPDCTL, 1 << 30, 1 << 30);
 873         /* RCOMP: Dither PU Enable */
 874         mrc_alt_write_mask(DDRPHY, DQSANADRVPUCTL, 1 << 30, 1 << 30);
 875         /* RCOMP: Dither PD Enable */
 876         mrc_alt_write_mask(DDRPHY, DQSANADRVPDCTL, 1 << 30, 1 << 30);
 877         /* RCOMP: Dither PU Enable */
 878         mrc_alt_write_mask(DDRPHY, CTLANADRVPUCTL, 1 << 30, 1 << 30);
 879         /* RCOMP: Dither PD Enable */
 880         mrc_alt_write_mask(DDRPHY, CTLANADRVPDCTL, 1 << 30, 1 << 30);
 881         /* ODT: Dither PU Enable */
 882         mrc_alt_write_mask(DDRPHY, DQANAODTPUCTL, 1 << 30, 1 << 30);
 883         /* ODT: Dither PD Enable */
 884         mrc_alt_write_mask(DDRPHY, DQANAODTPDCTL, 1 << 30, 1 << 30);
 885         /* ODT: Dither PU Enable */
 886         mrc_alt_write_mask(DDRPHY, CLKANAODTPUCTL, 1 << 30, 1 << 30);
 887         /* ODT: Dither PD Enable */
 888         mrc_alt_write_mask(DDRPHY, CLKANAODTPDCTL, 1 << 30, 1 << 30);
 889         /* ODT: Dither PU Enable */
 890         mrc_alt_write_mask(DDRPHY, DQSANAODTPUCTL, 1 << 30, 1 << 30);
 891         /* ODT: Dither PD Enable */
 892         mrc_alt_write_mask(DDRPHY, DQSANAODTPDCTL, 1 << 30, 1 << 30);
 893         /* DCOMP: Dither PU Enable */
 894         mrc_alt_write_mask(DDRPHY, DQANADLYPUCTL, 1 << 30, 1 << 30);
 895         /* DCOMP: Dither PD Enable */
 896         mrc_alt_write_mask(DDRPHY, DQANADLYPDCTL, 1 << 30, 1 << 30);
 897         /* DCOMP: Dither PU Enable */
 898         mrc_alt_write_mask(DDRPHY, CMDANADLYPUCTL, 1 << 30, 1 << 30);
 899         /* DCOMP: Dither PD Enable */
 900         mrc_alt_write_mask(DDRPHY, CMDANADLYPDCTL, 1 << 30, 1 << 30);
 901         /* DCOMP: Dither PU Enable */
 902         mrc_alt_write_mask(DDRPHY, CLKANADLYPUCTL, 1 << 30, 1 << 30);
 903         /* DCOMP: Dither PD Enable */
 904         mrc_alt_write_mask(DDRPHY, CLKANADLYPDCTL, 1 << 30, 1 << 30);
 905         /* DCOMP: Dither PU Enable */
 906         mrc_alt_write_mask(DDRPHY, DQSANADLYPUCTL, 1 << 30, 1 << 30);
 907         /* DCOMP: Dither PD Enable */
 908         mrc_alt_write_mask(DDRPHY, DQSANADLYPDCTL, 1 << 30, 1 << 30);
 909         /* DCOMP: Dither PU Enable */
 910         mrc_alt_write_mask(DDRPHY, CTLANADLYPUCTL, 1 << 30, 1 << 30);
 911         /* DCOMP: Dither PD Enable */
 912         mrc_alt_write_mask(DDRPHY, CTLANADLYPDCTL, 1 << 30, 1 << 30);
 913         /* TCO: Dither PU Enable */
 914         mrc_alt_write_mask(DDRPHY, DQANATCOPUCTL, 1 << 30, 1 << 30);
 915         /* TCO: Dither PD Enable */
 916         mrc_alt_write_mask(DDRPHY, DQANATCOPDCTL, 1 << 30, 1 << 30);
 917         /* TCO: Dither PU Enable */
 918         mrc_alt_write_mask(DDRPHY, CLKANATCOPUCTL, 1 << 30, 1 << 30);
 919         /* TCO: Dither PD Enable */
 920         mrc_alt_write_mask(DDRPHY, CLKANATCOPDCTL, 1 << 30, 1 << 30);
 921         /* TCO: Dither PU Enable */
 922         mrc_alt_write_mask(DDRPHY, DQSANATCOPUCTL, 1 << 30, 1 << 30);
 923         /* TCO: Dither PD Enable */
 924         mrc_alt_write_mask(DDRPHY, DQSANATCOPDCTL, 1 << 30, 1 << 30);
 925         /* TCOCOMP: Pulse Count */
 926         mrc_alt_write_mask(DDRPHY, TCOCNTCTRL, 1, 3);
 927         /* ODT: CMD/CTL PD/PU */
 928         mrc_alt_write_mask(DDRPHY, CHNLBUFSTATIC,
 929                 (0x03 << 24) | (0x03 << 16), 0x1f1f0000);
 930         /* Set 1us counter */
 931         mrc_alt_write_mask(DDRPHY, MSCNTR, 0x64, 0xff);
 932         mrc_alt_write_mask(DDRPHY, LATCH1CTL, 0x1 << 28, 0x70000000);
 933
 934         /* Release PHY from reset */
 935         mrc_alt_write_mask(DDRPHY, MASTERRSTN, 1, 1);
 936
 937         /* STEP1 */
 938         mrc_post_code(0x03, 0x11);
 939
 940         for (ch = 0; ch < NUM_CHANNELS; ch++) {
 941                 if (mrc_params->channel_enables & (1 << ch)) {
 942                         /* DQ01-DQ23 */
 943                         for (bl_grp = 0;
 944                              bl_grp < (NUM_BYTE_LANES / bl_divisor) / 2;
 945                              bl_grp++) {
 946                                 mrc_alt_write_mask(DDRPHY,
 947                                         DQMDLLCTL +
 948                                         bl_grp * DDRIODQ_BL_OFFSET +
 949                                         ch * DDRIODQ_CH_OFFSET,
 950                                         1 << 13,
 951                                         1 << 13);       /* Enable VREG */
 952                                 delay_n(3);
 953                         }
 954
 955                         /* ECC */
 956                         mrc_alt_write_mask(DDRPHY, ECCMDLLCTL,
 957                                 1 << 13, 1 << 13);      /* Enable VREG */
 958                         delay_n(3);
 959                         /* CMD */
 960                         mrc_alt_write_mask(DDRPHY,
 961                                 CMDMDLLCTL + ch * DDRIOCCC_CH_OFFSET,
 962                                 1 << 13, 1 << 13);      /* Enable VREG */
 963                         delay_n(3);
 964                         /* CLK-CTL */
 965                         mrc_alt_write_mask(DDRPHY,
 966                                 CCMDLLCTL + ch * DDRIOCCC_CH_OFFSET,
 967                                 1 << 13, 1 << 13);      /* Enable VREG */
 968                         delay_n(3);
 969                 }
 970         }
 971
 972         /* STEP2 */
 973         mrc_post_code(0x03, 0x12);
 974         delay_n(200);
 975
 976         for (ch = 0; ch < NUM_CHANNELS; ch++) {
 977                 if (mrc_params->channel_enables & (1 << ch)) {
 978                         /* DQ01-DQ23 */
 979                         for (bl_grp = 0;
 980                              bl_grp < (NUM_BYTE_LANES / bl_divisor) / 2;
 981                              bl_grp++) {
 982                                 mrc_alt_write_mask(DDRPHY,
 983                                         DQMDLLCTL +
 984                                         bl_grp * DDRIODQ_BL_OFFSET +
 985                                         ch * DDRIODQ_CH_OFFSET,
 986                                         1 << 17,
 987                                         1 << 17);       /* Enable MCDLL */
 988                                 delay_n(50);
 989                         }
 990
 991                 /* ECC */
 992                 mrc_alt_write_mask(DDRPHY, ECCMDLLCTL,
 993                         1 << 17, 1 << 17);      /* Enable MCDLL */
 994                 delay_n(50);
 995                 /* CMD */
 996                 mrc_alt_write_mask(DDRPHY,
 997                         CMDMDLLCTL + ch * DDRIOCCC_CH_OFFSET,
 998                         1 << 18, 1 << 18);      /* Enable MCDLL */
 999                 delay_n(50);
1000                 /* CLK-CTL */
1001                 mrc_alt_write_mask(DDRPHY,
1002                         CCMDLLCTL + ch * DDRIOCCC_CH_OFFSET,
1003                         1 << 18, 1 << 18);      /* Enable MCDLL */
1004                 delay_n(50);
1005                 }
1006         }
1007
1008         /* STEP3: */
1009         mrc_post_code(0x03, 0x13);
1010         delay_n(100);
1011
1012         for (ch = 0; ch < NUM_CHANNELS; ch++) {
1013                 if (mrc_params->channel_enables & (1 << ch)) {
1014                         /* DQ01-DQ23 */
1015                         for (bl_grp = 0;
1016                              bl_grp < (NUM_BYTE_LANES / bl_divisor) / 2;
1017                              bl_grp++) {
1018 #ifdef FORCE_16BIT_DDRIO
1019                                 temp = (bl_grp &&
1020                                         (mrc_params->channel_width == X16)) ?
1021                                         0x11ff : 0xffff;
1022 #else
1023                                 temp = 0xffff;
1024 #endif
1025                                 /* Enable TXDLL */
1026                                 mrc_alt_write_mask(DDRPHY,
1027                                         DQDLLTXCTL +
1028                                         bl_grp * DDRIODQ_BL_OFFSET +
1029                                         ch * DDRIODQ_CH_OFFSET,
1030                                         temp, 0xffff);
1031                                 delay_n(3);
1032                                 /* Enable RXDLL */
1033                                 mrc_alt_write_mask(DDRPHY,
1034                                         DQDLLRXCTL +
1035                                         bl_grp * DDRIODQ_BL_OFFSET +
1036                                         ch * DDRIODQ_CH_OFFSET,
1037                                         0xf, 0xf);
1038                                 delay_n(3);
1039                                 /* Enable RXDLL Overrides BL0 */
1040                                 mrc_alt_write_mask(DDRPHY,
1041                                         B0OVRCTL +
1042                                         bl_grp * DDRIODQ_BL_OFFSET +
1043                                         ch * DDRIODQ_CH_OFFSET,
1044                                         0xf, 0xf);
1045                         }
1046
1047                         /* ECC */
1048                         temp = 0xffff;
1049                         mrc_alt_write_mask(DDRPHY, ECCDLLTXCTL,
1050                                 temp, 0xffff);
1051                         delay_n(3);
1052
1053                         /* CMD (PO) */
1054                         mrc_alt_write_mask(DDRPHY,
1055                                 CMDDLLTXCTL + ch * DDRIOCCC_CH_OFFSET,
1056                                 temp, 0xffff);
1057                         delay_n(3);
1058                 }
1059         }
1060
1061         /* STEP4 */
1062         mrc_post_code(0x03, 0x14);
1063
1064         for (ch = 0; ch < NUM_CHANNELS; ch++) {
1065                 if (mrc_params->channel_enables & (1 << ch)) {
1066                         /* Host To Memory Clock Alignment (HMC) for 800/1066 */
1067                         for (bl_grp = 0;
1068                              bl_grp < (NUM_BYTE_LANES / bl_divisor) / 2;
1069                              bl_grp++) {
1070                                 /* CLK_ALIGN_MOD_ID */
1071                                 mrc_alt_write_mask(DDRPHY,
1072                                         DQCLKALIGNREG2 +
1073                                         bl_grp * DDRIODQ_BL_OFFSET +
1074                                         ch * DDRIODQ_CH_OFFSET,
1075                                         bl_grp ? 3 : 1,
1076                                         0xf);
1077                         }
1078
1079                         mrc_alt_write_mask(DDRPHY,
1080                                 ECCCLKALIGNREG2 + ch * DDRIODQ_CH_OFFSET,
1081                                 0x2, 0xf);
1082                         mrc_alt_write_mask(DDRPHY,
1083                                 CMDCLKALIGNREG2 + ch * DDRIODQ_CH_OFFSET,
1084                                 0x0, 0xf);
1085                         mrc_alt_write_mask(DDRPHY,
1086                                 CCCLKALIGNREG2 + ch * DDRIODQ_CH_OFFSET,
1087                                 0x2, 0xf);
1088                         mrc_alt_write_mask(DDRPHY,
1089                                 CMDCLKALIGNREG0 + ch * DDRIOCCC_CH_OFFSET,
1090                                 0x20, 0x30);
1091                         /*
1092                          * NUM_SAMPLES, MAX_SAMPLES,
1093                          * MACRO_PI_STEP, MICRO_PI_STEP
1094                          */
1095                         mrc_alt_write_mask(DDRPHY,
1096                                 CMDCLKALIGNREG1 + ch * DDRIOCCC_CH_OFFSET,
1097                                 (0x18 << 16) | (0x10 << 8) |
1098                                 (0x8 << 2) | (0x1 << 0),
1099                                 0x007f7fff);
1100                         /* TOTAL_NUM_MODULES, FIRST_U_PARTITION */
1101                         mrc_alt_write_mask(DDRPHY,
1102                                 CMDCLKALIGNREG2 + ch * DDRIOCCC_CH_OFFSET,
1103                                 (0x10 << 16) | (0x4 << 8) | (0x2 << 4),
1104                                 0x001f0ff0);
1105 #ifdef HMC_TEST
1106                         /* START_CLK_ALIGN=1 */
1107                         mrc_alt_write_mask(DDRPHY,
1108                                 CMDCLKALIGNREG0 + ch * DDRIOCCC_CH_OFFSET,
1109                                 1 << 24, 1 << 24);
1110                         while (msg_port_alt_read(DDRPHY,
1111                                 CMDCLKALIGNREG0 + ch * DDRIOCCC_CH_OFFSET) &
1112                                 (1 << 24))
1113                                 ;       /* wait for START_CLK_ALIGN=0 */
1114 #endif
1115
1116                         /* Set RD/WR Pointer Seperation & COUNTEN & FIFOPTREN */
1117                         mrc_alt_write_mask(DDRPHY,
1118                                 CMDPTRREG + ch * DDRIOCCC_CH_OFFSET,
1119                                 1, 1);  /* WRPTRENABLE=1 */
1120
1121                         /* COMP initial */
1122                         /* enable bypass for CLK buffer (PO) */
1123                         mrc_alt_write_mask(DDRPHY,
1124                                 COMPEN0CH0 + ch * DDRCOMP_CH_OFFSET,
1125                                 1 << 5, 1 << 5);
1126                         /* Initial COMP Enable */
1127                         mrc_alt_write_mask(DDRPHY, CMPCTRL, 1, 1);
1128                         /* wait for Initial COMP Enable = 0 */
1129                         while (msg_port_alt_read(DDRPHY, CMPCTRL) & 1)
1130                                 ;
1131                         /* disable bypass for CLK buffer (PO) */
1132                         mrc_alt_write_mask(DDRPHY,
1133                                 COMPEN0CH0 + ch * DDRCOMP_CH_OFFSET,
1134                                 ~(1 << 5), 1 << 5);
1135
1136                         /* IOBUFACT */
1137
1138                         /* STEP4a */
1139                         mrc_alt_write_mask(DDRPHY,
1140                                 CMDCFGREG0 + ch * DDRIOCCC_CH_OFFSET,
1141                                 1 << 2, 1 << 2);        /* IOBUFACTRST_N=1 */
1142
1143                         /* DDRPHY initialization complete */
1144                         mrc_alt_write_mask(DDRPHY,
1145                                 CMDPMCONFIG0 + ch * DDRIOCCC_CH_OFFSET,
1146                                 1 << 20, 1 << 20);      /* SPID_INIT_COMPLETE=1 */
1147                 }
1148         }
1149
1150         LEAVEFN();
1151 }
1152
1153 /* This function performs JEDEC initialization on all enabled channels */
1154 void perform_jedec_init(struct mrc_params *mrc_params)
1155 {
1156         uint8_t twr, wl, rank;
1157         uint32_t tck;
1158         u32 dtr0;
1159         u32 drp;
1160         u32 drmc;
1161         u32 mrs0_cmd = 0;
1162         u32 emrs1_cmd = 0;
1163         u32 emrs2_cmd = 0;
1164         u32 emrs3_cmd = 0;
1165
1166         ENTERFN();
1167
1168         /* jedec_init starts */
1169         mrc_post_code(0x04, 0x00);
1170
1171         /* DDR3_RESET_SET=0, DDR3_RESET_RESET=1 */
1172         mrc_alt_write_mask(DDRPHY, CCDDR3RESETCTL, 2, 0x102);
1173
1174         /* Assert RESET# for 200us */
1175         delay_u(200);
1176
1177         /* DDR3_RESET_SET=1, DDR3_RESET_RESET=0 */
1178         mrc_alt_write_mask(DDRPHY, CCDDR3RESETCTL, 0x100, 0x102);
1179
1180         dtr0 = msg_port_read(MEM_CTLR, DTR0);
1181
1182         /*
1183          * Set CKEVAL for populated ranks
1184          * then send NOP to each rank (#4550197)
1185          */
1186
1187         drp = msg_port_read(MEM_CTLR, DRP);
1188         drp &= 0x3;
1189
1190         drmc = msg_port_read(MEM_CTLR, DRMC);
1191         drmc &= 0xfffffffc;
1192         drmc |= (DRMC_CKEMODE | drp);
1193
1194         msg_port_write(MEM_CTLR, DRMC, drmc);
1195
1196         for (rank = 0; rank < NUM_RANKS; rank++) {
1197                 /* Skip to next populated rank */
1198                 if ((mrc_params->rank_enables & (1 << rank)) == 0)
1199                         continue;
1200
1201                 dram_init_command(DCMD_NOP(rank));
1202         }
1203
1204         msg_port_write(MEM_CTLR, DRMC,
1205                 (mrc_params->rd_odt_value == 0 ? DRMC_ODTMODE : 0));
1206
1207         /*
1208          * setup for emrs 2
1209          * BIT[15:11] --> Always "0"
1210          * BIT[10:09] --> Rtt_WR: want "Dynamic ODT Off" (0)
1211          * BIT[08]    --> Always "0"
1212          * BIT[07]    --> SRT: use sr_temp_range
1213          * BIT[06]    --> ASR: want "Manual SR Reference" (0)
1214          * BIT[05:03] --> CWL: use oem_tCWL
1215          * BIT[02:00] --> PASR: want "Full Array" (0)
1216          */
1217         emrs2_cmd |= (2 << 3);
1218         wl = 5 + mrc_params->ddr_speed;
1219         emrs2_cmd |= ((wl - 5) << 9);
1220         emrs2_cmd |= (mrc_params->sr_temp_range << 13);
1221
1222         /*
1223          * setup for emrs 3
1224          * BIT[15:03] --> Always "0"
1225          * BIT[02]    --> MPR: want "Normal Operation" (0)
1226          * BIT[01:00] --> MPR_Loc: want "Predefined Pattern" (0)
1227          */
1228         emrs3_cmd |= (3 << 3);
1229
1230         /*
1231          * setup for emrs 1
1232          * BIT[15:13]     --> Always "0"
1233          * BIT[12:12]     --> Qoff: want "Output Buffer Enabled" (0)
1234          * BIT[11:11]     --> TDQS: want "Disabled" (0)
1235          * BIT[10:10]     --> Always "0"
1236          * BIT[09,06,02]  --> Rtt_nom: use rtt_nom_value
1237          * BIT[08]        --> Always "0"
1238          * BIT[07]        --> WR_LVL: want "Disabled" (0)
1239          * BIT[05,01]     --> DIC: use ron_value
1240          * BIT[04:03]     --> AL: additive latency want "0" (0)
1241          * BIT[00]        --> DLL: want "Enable" (0)
1242          *
1243          * (BIT5|BIT1) set Ron value
1244          * 00 --> RZQ/6 (40ohm)
1245          * 01 --> RZQ/7 (34ohm)
1246          * 1* --> RESERVED
1247          *
1248          * (BIT9|BIT6|BIT2) set Rtt_nom value
1249          * 000 --> Disabled
1250          * 001 --> RZQ/4 ( 60ohm)
1251          * 010 --> RZQ/2 (120ohm)
1252          * 011 --> RZQ/6 ( 40ohm)
1253          * 1** --> RESERVED
1254          */
1255         emrs1_cmd |= (1 << 3);
1256         emrs1_cmd &= ~(1 << 6);
1257
1258         if (mrc_params->ron_value == 0)
1259                 emrs1_cmd |= (1 << 7);
1260         else
1261                 emrs1_cmd &= ~(1 << 7);
1262
1263         if (mrc_params->rtt_nom_value == 0)
1264                 emrs1_cmd |= (DDR3_EMRS1_RTTNOM_40 << 6);
1265         else if (mrc_params->rtt_nom_value == 1)
1266                 emrs1_cmd |= (DDR3_EMRS1_RTTNOM_60 << 6);
1267         else if (mrc_params->rtt_nom_value == 2)
1268                 emrs1_cmd |= (DDR3_EMRS1_RTTNOM_120 << 6);
1269
1270         /* save MRS1 value (excluding control fields) */
1271         mrc_params->mrs1 = emrs1_cmd >> 6;
1272
1273         /*
1274          * setup for mrs 0
1275          * BIT[15:13]     --> Always "0"
1276          * BIT[12]        --> PPD: for Quark (1)
1277          * BIT[11:09]     --> WR: use oem_tWR
1278          * BIT[08]        --> DLL: want "Reset" (1, self clearing)
1279          * BIT[07]        --> MODE: want "Normal" (0)
1280          * BIT[06:04,02]  --> CL: use oem_tCAS
1281          * BIT[03]        --> RD_BURST_TYPE: want "Interleave" (1)
1282          * BIT[01:00]     --> BL: want "8 Fixed" (0)
1283          * WR:
1284          * 0 --> 16
1285          * 1 --> 5
1286          * 2 --> 6
1287          * 3 --> 7
1288          * 4 --> 8
1289          * 5 --> 10
1290          * 6 --> 12
1291          * 7 --> 14
1292          * CL:
1293          * BIT[02:02] "0" if oem_tCAS <= 11 (1866?)
1294          * BIT[06:04] use oem_tCAS-4
1295          */
1296         mrs0_cmd |= (1 << 14);
1297         mrs0_cmd |= (1 << 18);
1298         mrs0_cmd |= ((((dtr0 >> 12) & 7) + 1) << 10);
1299
1300         tck = t_ck[mrc_params->ddr_speed];
1301         /* Per JEDEC: tWR=15000ps DDR2/3 from 800-1600 */
1302         twr = MCEIL(15000, tck);
1303         mrs0_cmd |= ((twr - 4) << 15);
1304
1305         for (rank = 0; rank < NUM_RANKS; rank++) {
1306                 /* Skip to next populated rank */
1307                 if ((mrc_params->rank_enables & (1 << rank)) == 0)
1308                         continue;
1309
1310                 emrs2_cmd |= (rank << 22);
1311                 dram_init_command(emrs2_cmd);
1312
1313                 emrs3_cmd |= (rank << 22);
1314                 dram_init_command(emrs3_cmd);
1315
1316                 emrs1_cmd |= (rank << 22);
1317                 dram_init_command(emrs1_cmd);
1318
1319                 mrs0_cmd |= (rank << 22);
1320                 dram_init_command(mrs0_cmd);
1321
1322                 dram_init_command(DCMD_ZQCL(rank));
1323         }
1324
1325         LEAVEFN();
1326 }
1327
1328 /*
1329  * Dunit Initialization Complete
1330  *
1331  * Indicates that initialization of the Dunit has completed.
1332  *
1333  * Memory accesses are permitted and maintenance operation begins.
1334  * Until this bit is set to a 1, the memory controller will not accept
1335  * DRAM requests from the MEMORY_MANAGER or HTE.
1336  */
1337 void set_ddr_init_complete(struct mrc_params *mrc_params)
1338 {
1339         u32 dco;
1340
1341         ENTERFN();
1342
1343         dco = msg_port_read(MEM_CTLR, DCO);
1344         dco &= ~DCO_PMICTL;
1345         dco |= DCO_IC;
1346         msg_port_write(MEM_CTLR, DCO, dco);
1347
1348         LEAVEFN();
1349 }
1350
1351 /*
1352  * This function will retrieve relevant timing data
1353  *
1354  * This data will be used on subsequent boots to speed up boot times
1355  * and is required for Suspend To RAM capabilities.
1356  */
1357 void restore_timings(struct mrc_params *mrc_params)
1358 {
1359         uint8_t ch, rk, bl;
1360         const struct mrc_timings *mt = &mrc_params->timings;
1361
1362         for (ch = 0; ch < NUM_CHANNELS; ch++) {
1363                 for (rk = 0; rk < NUM_RANKS; rk++) {
1364                         for (bl = 0; bl < NUM_BYTE_LANES; bl++) {
1365                                 set_rcvn(ch, rk, bl, mt->rcvn[ch][rk][bl]);
1366                                 set_rdqs(ch, rk, bl, mt->rdqs[ch][rk][bl]);
1367                                 set_wdqs(ch, rk, bl, mt->wdqs[ch][rk][bl]);
1368                                 set_wdq(ch, rk, bl, mt->wdq[ch][rk][bl]);
1369                                 if (rk == 0) {
1370                                         /* VREF (RANK0 only) */
1371                                         set_vref(ch, bl, mt->vref[ch][bl]);
1372                                 }
1373                         }
1374                         set_wctl(ch, rk, mt->wctl[ch][rk]);
1375                 }
1376                 set_wcmd(ch, mt->wcmd[ch]);
1377         }
1378 }
1379
1380 /*
1381  * Configure default settings normally set as part of read training
1382  *
1383  * Some defaults have to be set earlier as they may affect earlier
1384  * training steps.
1385  */
1386 void default_timings(struct mrc_params *mrc_params)
1387 {
1388         uint8_t ch, rk, bl;
1389
1390         for (ch = 0; ch < NUM_CHANNELS; ch++) {
1391                 for (rk = 0; rk < NUM_RANKS; rk++) {
1392                         for (bl = 0; bl < NUM_BYTE_LANES; bl++) {
1393                                 set_rdqs(ch, rk, bl, 24);
1394                                 if (rk == 0) {
1395                                         /* VREF (RANK0 only) */
1396                                         set_vref(ch, bl, 32);
1397                                 }
1398                         }
1399                 }
1400         }
1401 }
1402
1403 /*
1404  * This function will perform our RCVEN Calibration Algorithm.
1405  * We will only use the 2xCLK domain timings to perform RCVEN Calibration.
1406  * All byte lanes will be calibrated "simultaneously" per channel per rank.
1407  */
1408 void rcvn_cal(struct mrc_params *mrc_params)
1409 {
1410         uint8_t ch;     /* channel counter */
1411         uint8_t rk;     /* rank counter */
1412         uint8_t bl;     /* byte lane counter */
1413         uint8_t bl_divisor = (mrc_params->channel_width == X16) ? 2 : 1;
1414
1415 #ifdef R2R_SHARING
1416         /* used to find placement for rank2rank sharing configs */
1417         uint32_t final_delay[NUM_CHANNELS][NUM_BYTE_LANES];
1418 #ifndef BACKUP_RCVN
1419         /* used to find placement for rank2rank sharing configs */
1420         uint32_t num_ranks_enabled = 0;
1421 #endif
1422 #endif
1423
1424 #ifdef BACKUP_RCVN
1425 #else
1426         uint32_t temp;
1427         /* absolute PI value to be programmed on the byte lane */
1428         uint32_t delay[NUM_BYTE_LANES];
1429         u32 dtr1, dtr1_save;
1430 #endif
1431
1432         ENTERFN();
1433
1434         /* rcvn_cal starts */
1435         mrc_post_code(0x05, 0x00);
1436
1437 #ifndef BACKUP_RCVN
1438         /* need separate burst to sample DQS preamble */
1439         dtr1 = msg_port_read(MEM_CTLR, DTR1);
1440         dtr1_save = dtr1;
1441         dtr1 |= DTR1_TCCD_12CLK;
1442         msg_port_write(MEM_CTLR, DTR1, dtr1);
1443 #endif
1444
1445 #ifdef R2R_SHARING
1446         /* need to set "final_delay[][]" elements to "0" */
1447         memset((void *)(final_delay), 0x00, (size_t)sizeof(final_delay));
1448 #endif
1449
1450         /* loop through each enabled channel */
1451         for (ch = 0; ch < NUM_CHANNELS; ch++) {
1452                 if (mrc_params->channel_enables & (1 << ch)) {
1453                         /* perform RCVEN Calibration on a per rank basis */
1454                         for (rk = 0; rk < NUM_RANKS; rk++) {
1455                                 if (mrc_params->rank_enables & (1 << rk)) {
1456                                         /*
1457                                          * POST_CODE here indicates the current
1458                                          * channel and rank being calibrated
1459                                          */
1460                                         mrc_post_code(0x05, 0x10 + ((ch << 4) | rk));
1461
1462 #ifdef BACKUP_RCVN
1463                                         /* et hard-coded timing values */
1464                                         for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++)
1465                                                 set_rcvn(ch, rk, bl, ddr_rcvn[PLATFORM_ID]);
1466 #else
1467                                         /* enable FIFORST */
1468                                         for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl += 2) {
1469                                                 mrc_alt_write_mask(DDRPHY,
1470                                                         B01PTRCTL1 +
1471                                                         (bl >> 1) * DDRIODQ_BL_OFFSET +
1472                                                         ch * DDRIODQ_CH_OFFSET,
1473                                                         0, 1 << 8);
1474                                         }
1475                                         /* initialize the starting delay to 128 PI (cas +1 CLK) */
1476                                         for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
1477                                                 /* 1x CLK domain timing is cas-4 */
1478                                                 delay[bl] = (4 + 1) * FULL_CLK;
1479
1480                                                 set_rcvn(ch, rk, bl, delay[bl]);
1481                                         }
1482
1483                                         /* now find the rising edge */
1484                                         find_rising_edge(mrc_params, delay, ch, rk, true);
1485
1486                                         /* Now increase delay by 32 PI (1/4 CLK) to place in center of high pulse */
1487                                         for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
1488                                                 delay[bl] += QRTR_CLK;
1489                                                 set_rcvn(ch, rk, bl, delay[bl]);
1490                                         }
1491                                         /* Now decrement delay by 128 PI (1 CLK) until we sample a "0" */
1492                                         do {
1493                                                 temp = sample_dqs(mrc_params, ch, rk, true);
1494                                                 for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
1495                                                         if (temp & (1 << bl)) {
1496                                                                 if (delay[bl] >= FULL_CLK) {
1497                                                                         delay[bl] -= FULL_CLK;
1498                                                                         set_rcvn(ch, rk, bl, delay[bl]);
1499                                                                 } else {
1500                                                                         /* not enough delay */
1501                                                                         training_message(ch, rk, bl);
1502                                                                         mrc_post_code(0xee, 0x50);
1503                                                                 }
1504                                                         }
1505                                                 }
1506                                         } while (temp & 0xff);
1507
1508 #ifdef R2R_SHARING
1509                                         /* increment "num_ranks_enabled" */
1510                                         num_ranks_enabled++;
1511                                         /* Finally increment delay by 32 PI (1/4 CLK) to place in center of preamble */
1512                                         for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
1513                                                 delay[bl] += QRTR_CLK;
1514                                                 /* add "delay[]" values to "final_delay[][]" for rolling average */
1515                                                 final_delay[ch][bl] += delay[bl];
1516                                                 /* set timing based on rolling average values */
1517                                                 set_rcvn(ch, rk, bl, final_delay[ch][bl] / num_ranks_enabled);
1518                                         }
1519 #else
1520                                         /* Finally increment delay by 32 PI (1/4 CLK) to place in center of preamble */
1521                                         for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
1522                                                 delay[bl] += QRTR_CLK;
1523                                                 set_rcvn(ch, rk, bl, delay[bl]);
1524                                         }
1525 #endif
1526
1527                                         /* disable FIFORST */
1528                                         for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl += 2) {
1529                                                 mrc_alt_write_mask(DDRPHY,
1530                                                         B01PTRCTL1 +
1531                                                         (bl >> 1) * DDRIODQ_BL_OFFSET +
1532                                                         ch * DDRIODQ_CH_OFFSET,
1533                                                         1 << 8, 1 << 8);
1534                                         }
1535 #endif
1536                                 }
1537                         }
1538                 }
1539         }
1540
1541 #ifndef BACKUP_RCVN
1542         /* restore original */
1543         msg_port_write(MEM_CTLR, DTR1, dtr1_save);
1544 #endif
1545
1546         LEAVEFN();
1547 }
1548
1549 /*
1550  * This function will perform the Write Levelling algorithm
1551  * (align WCLK and WDQS).
1552  *
1553  * This algorithm will act on each rank in each channel separately.
1554  */
1555 void wr_level(struct mrc_params *mrc_params)
1556 {
1557         uint8_t ch;     /* channel counter */
1558         uint8_t rk;     /* rank counter */
1559         uint8_t bl;     /* byte lane counter */
1560         uint8_t bl_divisor = (mrc_params->channel_width == X16) ? 2 : 1;
1561
1562 #ifdef R2R_SHARING
1563         /* used to find placement for rank2rank sharing configs */
1564         uint32_t final_delay[NUM_CHANNELS][NUM_BYTE_LANES];
1565 #ifndef BACKUP_WDQS
1566         /* used to find placement for rank2rank sharing configs */
1567         uint32_t num_ranks_enabled = 0;
1568 #endif
1569 #endif
1570
1571 #ifdef BACKUP_WDQS
1572 #else
1573         /* determines stop condition for CRS_WR_LVL */
1574         bool all_edges_found;
1575         /* absolute PI value to be programmed on the byte lane */
1576         uint32_t delay[NUM_BYTE_LANES];
1577         /*
1578          * static makes it so the data is loaded in the heap once by shadow(),
1579          * where non-static copies the data onto the stack every time this
1580          * function is called
1581          */
1582         uint32_t address;       /* address to be checked during COARSE_WR_LVL */
1583         u32 dtr4, dtr4_save;
1584 #endif
1585
1586         ENTERFN();
1587
1588         /* wr_level starts */
1589         mrc_post_code(0x06, 0x00);
1590
1591 #ifdef R2R_SHARING
1592         /* need to set "final_delay[][]" elements to "0" */
1593         memset((void *)(final_delay), 0x00, (size_t)sizeof(final_delay));
1594 #endif
1595
1596         /* loop through each enabled channel */
1597         for (ch = 0; ch < NUM_CHANNELS; ch++) {
1598                 if (mrc_params->channel_enables & (1 << ch)) {
1599                         /* perform WRITE LEVELING algorithm on a per rank basis */
1600                         for (rk = 0; rk < NUM_RANKS; rk++) {
1601                                 if (mrc_params->rank_enables & (1 << rk)) {
1602                                         /*
1603                                          * POST_CODE here indicates the current
1604                                          * rank and channel being calibrated
1605                                          */
1606                                         mrc_post_code(0x06, 0x10 + ((ch << 4) | rk));
1607
1608 #ifdef BACKUP_WDQS
1609                                         for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
1610                                                 set_wdqs(ch, rk, bl, ddr_wdqs[PLATFORM_ID]);
1611                                                 set_wdq(ch, rk, bl, ddr_wdqs[PLATFORM_ID] - QRTR_CLK);
1612                                         }
1613 #else
1614                                         /*
1615                                          * perform a single PRECHARGE_ALL command to
1616                                          * make DRAM state machine go to IDLE state
1617                                          */
1618                                         dram_init_command(DCMD_PREA(rk));
1619
1620                                         /*
1621                                          * enable Write Levelling Mode
1622                                          * (EMRS1 w/ Write Levelling Mode Enable)
1623                                          */
1624                                         dram_init_command(DCMD_MRS1(rk, 0x82));
1625
1626                                         /*
1627                                          * set ODT DRAM Full Time Termination
1628                                          * disable in MCU
1629                                          */
1630
1631                                         dtr4 = msg_port_read(MEM_CTLR, DTR4);
1632                                         dtr4_save = dtr4;
1633                                         dtr4 |= DTR4_ODTDIS;
1634                                         msg_port_write(MEM_CTLR, DTR4, dtr4);
1635
1636                                         for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor) / 2; bl++) {
1637                                                 /*
1638                                                  * Enable Sandy Bridge Mode (WDQ Tri-State) &
1639                                                  * Ensure 5 WDQS pulses during Write Leveling
1640                                                  */
1641                                                 mrc_alt_write_mask(DDRPHY,
1642                                                         DQCTL + DDRIODQ_BL_OFFSET * bl + DDRIODQ_CH_OFFSET * ch,
1643                                                         0x10000154,
1644                                                         0x100003fc);
1645                                         }
1646
1647                                         /* Write Leveling Mode enabled in IO */
1648                                         mrc_alt_write_mask(DDRPHY,
1649                                                 CCDDR3RESETCTL + DDRIOCCC_CH_OFFSET * ch,
1650                                                 1 << 16, 1 << 16);
1651
1652                                         /* Initialize the starting delay to WCLK */
1653                                         for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
1654                                                 /*
1655                                                  * CLK0 --> RK0
1656                                                  * CLK1 --> RK1
1657                                                  */
1658                                                 delay[bl] = get_wclk(ch, rk);
1659
1660                                                 set_wdqs(ch, rk, bl, delay[bl]);
1661                                         }
1662
1663                                         /* now find the rising edge */
1664                                         find_rising_edge(mrc_params, delay, ch, rk, false);
1665
1666                                         /* disable Write Levelling Mode */
1667                                         mrc_alt_write_mask(DDRPHY,
1668                                                 CCDDR3RESETCTL + DDRIOCCC_CH_OFFSET * ch,
1669                                                 0, 1 << 16);
1670
1671                                         for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor) / 2; bl++) {
1672                                                 /* Disable Sandy Bridge Mode & Ensure 4 WDQS pulses during normal operation */
1673                                                 mrc_alt_write_mask(DDRPHY,
1674                                                         DQCTL + DDRIODQ_BL_OFFSET * bl + DDRIODQ_CH_OFFSET * ch,
1675                                                         0x00000154,
1676                                                         0x100003fc);
1677                                         }
1678
1679                                         /* restore original DTR4 */
1680                                         msg_port_write(MEM_CTLR, DTR4, dtr4_save);
1681
1682                                         /*
1683                                          * restore original value
1684                                          * (Write Levelling Mode Disable)
1685                                          */
1686                                         dram_init_command(DCMD_MRS1(rk, mrc_params->mrs1));
1687
1688                                         /*
1689                                          * perform a single PRECHARGE_ALL command to
1690                                          * make DRAM state machine go to IDLE state
1691                                          */
1692                                         dram_init_command(DCMD_PREA(rk));
1693
1694                                         mrc_post_code(0x06, 0x30 + ((ch << 4) | rk));
1695
1696                                         /*
1697                                          * COARSE WRITE LEVEL:
1698                                          * check that we're on the correct clock edge
1699                                          */
1700
1701                                         /* hte reconfiguration request */
1702                                         mrc_params->hte_setup = 1;
1703
1704                                         /* start CRS_WR_LVL with WDQS = WDQS + 128 PI */
1705                                         for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
1706                                                 delay[bl] = get_wdqs(ch, rk, bl) + FULL_CLK;
1707                                                 set_wdqs(ch, rk, bl, delay[bl]);
1708                                                 /*
1709                                                  * program WDQ timings based on WDQS
1710                                                  * (WDQ = WDQS - 32 PI)
1711                                                  */
1712                                                 set_wdq(ch, rk, bl, (delay[bl] - QRTR_CLK));
1713                                         }
1714
1715                                         /* get an address in the targeted channel/rank */
1716                                         address = get_addr(ch, rk);
1717                                         do {
1718                                                 uint32_t coarse_result = 0x00;
1719                                                 uint32_t coarse_result_mask = byte_lane_mask(mrc_params);
1720                                                 /* assume pass */
1721                                                 all_edges_found = true;
1722
1723                                                 mrc_params->hte_setup = 1;
1724                                                 coarse_result = check_rw_coarse(mrc_params, address);
1725
1726                                                 /* check for failures and margin the byte lane back 128 PI (1 CLK) */
1727                                                 for (bl = 0; bl < NUM_BYTE_LANES / bl_divisor; bl++) {
1728                                                         if (coarse_result & (coarse_result_mask << bl)) {
1729                                                                 all_edges_found = false;
1730                                                                 delay[bl] -= FULL_CLK;
1731                                                                 set_wdqs(ch, rk, bl, delay[bl]);
1732                                                                 /* program WDQ timings based on WDQS (WDQ = WDQS - 32 PI) */
1733                                                                 set_wdq(ch, rk, bl, delay[bl] - QRTR_CLK);
1734                                                         }
1735                                                 }
1736                                         } while (!all_edges_found);
1737
1738 #ifdef R2R_SHARING
1739                                         /* increment "num_ranks_enabled" */
1740                                          num_ranks_enabled++;
1741                                         /* accumulate "final_delay[][]" values from "delay[]" values for rolling average */
1742                                         for (bl = 0; bl < NUM_BYTE_LANES / bl_divisor; bl++) {
1743                                                 final_delay[ch][bl] += delay[bl];
1744                                                 set_wdqs(ch, rk, bl, final_delay[ch][bl] / num_ranks_enabled);
1745                                                 /* program WDQ timings based on WDQS (WDQ = WDQS - 32 PI) */
1746                                                 set_wdq(ch, rk, bl, final_delay[ch][bl] / num_ranks_enabled - QRTR_CLK);
1747                                         }
1748 #endif
1749 #endif
1750                                 }
1751                         }
1752                 }
1753         }
1754
1755         LEAVEFN();
1756 }
1757
1758 void prog_page_ctrl(struct mrc_params *mrc_params)
1759 {
1760         u32 dpmc0;
1761
1762         ENTERFN();
1763
1764         dpmc0 = msg_port_read(MEM_CTLR, DPMC0);
1765         dpmc0 &= ~DPMC0_PCLSTO_MASK;
1766         dpmc0 |= (4 << 16);
1767         dpmc0 |= DPMC0_PREAPWDEN;
1768         msg_port_write(MEM_CTLR, DPMC0, dpmc0);
1769 }
1770
1771 /*
1772  * This function will perform the READ TRAINING Algorithm on all
1773  * channels/ranks/byte_lanes simultaneously to minimize execution time.
1774  *
1775  * The idea here is to train the VREF and RDQS (and eventually RDQ) values
1776  * to achieve maximum READ margins. The algorithm will first determine the
1777  * X coordinate (RDQS setting). This is done by collapsing the VREF eye
1778  * until we find a minimum required RDQS eye for VREF_MIN and VREF_MAX.
1779  * Then we take the averages of the RDQS eye at VREF_MIN and VREF_MAX,
1780  * then average those; this will be the final X coordinate. The algorithm
1781  * will then determine the Y coordinate (VREF setting). This is done by
1782  * collapsing the RDQS eye until we find a minimum required VREF eye for
1783  * RDQS_MIN and RDQS_MAX. Then we take the averages of the VREF eye at
1784  * RDQS_MIN and RDQS_MAX, then average those; this will be the final Y
1785  * coordinate.
1786  *
1787  * NOTE: this algorithm assumes the eye curves have a one-to-one relationship,
1788  * meaning for each X the curve has only one Y and vice-a-versa.
1789  */
1790 void rd_train(struct mrc_params *mrc_params)
1791 {
1792         uint8_t ch;     /* channel counter */
1793         uint8_t rk;     /* rank counter */
1794         uint8_t bl;     /* byte lane counter */
1795         uint8_t bl_divisor = (mrc_params->channel_width == X16) ? 2 : 1;
1796 #ifdef BACKUP_RDQS
1797 #else
1798         uint8_t side_x; /* tracks LEFT/RIGHT approach vectors */
1799         uint8_t side_y; /* tracks BOTTOM/TOP approach vectors */
1800         /* X coordinate data (passing RDQS values) for approach vectors */
1801         uint8_t x_coordinate[2][2][NUM_CHANNELS][NUM_RANKS][NUM_BYTE_LANES];
1802         /* Y coordinate data (passing VREF values) for approach vectors */
1803         uint8_t y_coordinate[2][2][NUM_CHANNELS][NUM_BYTE_LANES];
1804         /* centered X (RDQS) */
1805         uint8_t x_center[NUM_CHANNELS][NUM_RANKS][NUM_BYTE_LANES];
1806         /* centered Y (VREF) */
1807         uint8_t y_center[NUM_CHANNELS][NUM_BYTE_LANES];
1808         uint32_t address;       /* target address for check_bls_ex() */
1809         uint32_t result;        /* result of check_bls_ex() */
1810         uint32_t bl_mask;       /* byte lane mask for result checking */
1811 #ifdef R2R_SHARING
1812         /* used to find placement for rank2rank sharing configs */
1813         uint32_t final_delay[NUM_CHANNELS][NUM_BYTE_LANES];
1814         /* used to find placement for rank2rank sharing configs */
1815         uint32_t num_ranks_enabled = 0;
1816 #endif
1817 #endif
1818
1819         /* rd_train starts */
1820         mrc_post_code(0x07, 0x00);
1821
1822         ENTERFN();
1823
1824 #ifdef BACKUP_RDQS
1825         for (ch = 0; ch < NUM_CHANNELS; ch++) {
1826                 if (mrc_params->channel_enables & (1 << ch)) {
1827                         for (rk = 0; rk < NUM_RANKS; rk++) {
1828                                 if (mrc_params->rank_enables & (1 << rk)) {
1829                                         for (bl = 0;
1830                                              bl < NUM_BYTE_LANES / bl_divisor;
1831                                              bl++) {
1832                                                 set_rdqs(ch, rk, bl, ddr_rdqs[PLATFORM_ID]);
1833                                         }
1834                                 }
1835                         }
1836                 }
1837         }
1838 #else
1839         /* initialize x/y_coordinate arrays */
1840         for (ch = 0; ch < NUM_CHANNELS; ch++) {
1841                 if (mrc_params->channel_enables & (1 << ch)) {
1842                         for (rk = 0; rk < NUM_RANKS; rk++) {
1843                                 if (mrc_params->rank_enables & (1 << rk)) {
1844                                         for (bl = 0;
1845                                              bl < NUM_BYTE_LANES / bl_divisor;
1846                                              bl++) {
1847                                                 /* x_coordinate */
1848                                                 x_coordinate[L][B][ch][rk][bl] = RDQS_MIN;
1849                                                 x_coordinate[R][B][ch][rk][bl] = RDQS_MAX;
1850                                                 x_coordinate[L][T][ch][rk][bl] = RDQS_MIN;
1851                                                 x_coordinate[R][T][ch][rk][bl] = RDQS_MAX;
1852                                                 /* y_coordinate */
1853                                                 y_coordinate[L][B][ch][bl] = VREF_MIN;
1854                                                 y_coordinate[R][B][ch][bl] = VREF_MIN;
1855                                                 y_coordinate[L][T][ch][bl] = VREF_MAX;
1856                                                 y_coordinate[R][T][ch][bl] = VREF_MAX;
1857                                         }
1858                                 }
1859                         }
1860                 }
1861         }
1862
1863         /* initialize other variables */
1864         bl_mask = byte_lane_mask(mrc_params);
1865         address = get_addr(0, 0);
1866
1867 #ifdef R2R_SHARING
1868         /* need to set "final_delay[][]" elements to "0" */
1869         memset((void *)(final_delay), 0x00, (size_t)sizeof(final_delay));
1870 #endif
1871
1872         /* look for passing coordinates */
1873         for (side_y = B; side_y <= T; side_y++) {
1874                 for (side_x = L; side_x <= R; side_x++) {
1875                         mrc_post_code(0x07, 0x10 + side_y * 2 + side_x);
1876
1877                         /* find passing values */
1878                         for (ch = 0; ch < NUM_CHANNELS; ch++) {
1879                                 if (mrc_params->channel_enables & (0x1 << ch)) {
1880                                         for (rk = 0; rk < NUM_RANKS; rk++) {
1881                                                 if (mrc_params->rank_enables &
1882                                                         (0x1 << rk)) {
1883                                                         /* set x/y_coordinate search starting settings */
1884                                                         for (bl = 0;
1885                                                              bl < NUM_BYTE_LANES / bl_divisor;
1886                                                              bl++) {
1887                                                                 set_rdqs(ch, rk, bl,
1888                                                                          x_coordinate[side_x][side_y][ch][rk][bl]);
1889                                                                 set_vref(ch, bl,
1890                                                                          y_coordinate[side_x][side_y][ch][bl]);
1891                                                         }
1892
1893                                                         /* get an address in the target channel/rank */
1894                                                         address = get_addr(ch, rk);
1895
1896                                                         /* request HTE reconfiguration */
1897                                                         mrc_params->hte_setup = 1;
1898
1899                                                         /* test the settings */
1900                                                         do {
1901                                                                 /* result[07:00] == failing byte lane (MAX 8) */
1902                                                                 result = check_bls_ex(mrc_params, address);
1903
1904                                                                 /* check for failures */
1905                                                                 if (result & 0xff) {
1906                                                                         /* at least 1 byte lane failed */
1907                                                                         for (bl = 0; bl < NUM_BYTE_LANES / bl_divisor; bl++) {
1908                                                                                 if (result &
1909                                                                                         (bl_mask << bl)) {
1910                                                                                         /* adjust the RDQS values accordingly */
1911                                                                                         if (side_x == L)
1912                                                                                                 x_coordinate[L][side_y][ch][rk][bl] += RDQS_STEP;
1913                                                                                         else
1914                                                                                                 x_coordinate[R][side_y][ch][rk][bl] -= RDQS_STEP;
1915
1916                                                                                         /* check that we haven't closed the RDQS_EYE too much */
1917                                                                                         if ((x_coordinate[L][side_y][ch][rk][bl] > (RDQS_MAX - MIN_RDQS_EYE)) ||
1918                                                                                                 (x_coordinate[R][side_y][ch][rk][bl] < (RDQS_MIN + MIN_RDQS_EYE)) ||
1919                                                                                                 (x_coordinate[L][side_y][ch][rk][bl] ==
1920                                                                                                 x_coordinate[R][side_y][ch][rk][bl])) {
1921                                                                                                 /*
1922                                                                                                  * not enough RDQS margin available at this VREF
1923                                                                                                  * update VREF values accordingly
1924                                                                                                  */
1925                                                                                                 if (side_y == B)
1926                                                                                                         y_coordinate[side_x][B][ch][bl] += VREF_STEP;
1927                                                                                                 else
1928                                                                                                         y_coordinate[side_x][T][ch][bl] -= VREF_STEP;
1929
1930                                                                                                 /* check that we haven't closed the VREF_EYE too much */
1931                                                                                                 if ((y_coordinate[side_x][B][ch][bl] > (VREF_MAX - MIN_VREF_EYE)) ||
1932                                                                                                         (y_coordinate[side_x][T][ch][bl] < (VREF_MIN + MIN_VREF_EYE)) ||
1933                                                                                                         (y_coordinate[side_x][B][ch][bl] == y_coordinate[side_x][T][ch][bl])) {
1934                                                                                                         /* VREF_EYE collapsed below MIN_VREF_EYE */
1935                                                                                                         training_message(ch, rk, bl);
1936                                                                                                         mrc_post_code(0xEE, 0x70 + side_y * 2 + side_x);
1937                                                                                                 } else {
1938                                                                                                         /* update the VREF setting */
1939                                                                                                         set_vref(ch, bl, y_coordinate[side_x][side_y][ch][bl]);
1940                                                                                                         /* reset the X coordinate to begin the search at the new VREF */
1941                                                                                                         x_coordinate[side_x][side_y][ch][rk][bl] =
1942                                                                                                                 (side_x == L) ? RDQS_MIN : RDQS_MAX;
1943                                                                                                 }
1944                                                                                         }
1945
1946                                                                                         /* update the RDQS setting */
1947                                                                                         set_rdqs(ch, rk, bl, x_coordinate[side_x][side_y][ch][rk][bl]);
1948                                                                                 }
1949                                                                         }
1950                                                                 }
1951                                                         } while (result & 0xff);
1952                                                 }
1953                                         }
1954                                 }
1955                         }
1956                 }
1957         }
1958
1959         mrc_post_code(0x07, 0x20);
1960
1961         /* find final RDQS (X coordinate) & final VREF (Y coordinate) */
1962         for (ch = 0; ch < NUM_CHANNELS; ch++) {
1963                 if (mrc_params->channel_enables & (1 << ch)) {
1964                         for (rk = 0; rk < NUM_RANKS; rk++) {
1965                                 if (mrc_params->rank_enables & (1 << rk)) {
1966                                         for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
1967                                                 uint32_t temp1;
1968                                                 uint32_t temp2;
1969
1970                                                 /* x_coordinate */
1971                                                 DPF(D_INFO,
1972                                                     "RDQS T/B eye rank%d lane%d : %d-%d %d-%d\n",
1973                                                     rk, bl,
1974                                                     x_coordinate[L][T][ch][rk][bl],
1975                                                     x_coordinate[R][T][ch][rk][bl],
1976                                                     x_coordinate[L][B][ch][rk][bl],
1977                                                     x_coordinate[R][B][ch][rk][bl]);
1978
1979                                                 /* average the TOP side LEFT & RIGHT values */
1980                                                 temp1 = (x_coordinate[R][T][ch][rk][bl] + x_coordinate[L][T][ch][rk][bl]) / 2;
1981                                                 /* average the BOTTOM side LEFT & RIGHT values */
1982                                                 temp2 = (x_coordinate[R][B][ch][rk][bl] + x_coordinate[L][B][ch][rk][bl]) / 2;
1983                                                 /* average the above averages */
1984                                                 x_center[ch][rk][bl] = (uint8_t) ((temp1 + temp2) / 2);
1985
1986                                                 /* y_coordinate */
1987                                                 DPF(D_INFO,
1988                                                     "VREF R/L eye lane%d : %d-%d %d-%d\n",
1989                                                     bl,
1990                                                     y_coordinate[R][B][ch][bl],
1991                                                     y_coordinate[R][T][ch][bl],
1992                                                     y_coordinate[L][B][ch][bl],
1993                                                     y_coordinate[L][T][ch][bl]);
1994
1995                                                 /* average the RIGHT side TOP & BOTTOM values */
1996                                                 temp1 = (y_coordinate[R][T][ch][bl] + y_coordinate[R][B][ch][bl]) / 2;
1997                                                 /* average the LEFT side TOP & BOTTOM values */
1998                                                 temp2 = (y_coordinate[L][T][ch][bl] + y_coordinate[L][B][ch][bl]) / 2;
1999                                                 /* average the above averages */
2000                                                 y_center[ch][bl] = (uint8_t) ((temp1 + temp2) / 2);
2001                                         }
2002                                 }
2003                         }
2004                 }
2005         }
2006
2007 #ifdef RX_EYE_CHECK
2008         /* perform an eye check */
2009         for (side_y = B; side_y <= T; side_y++) {
2010                 for (side_x = L; side_x <= R; side_x++) {
2011                         mrc_post_code(0x07, 0x30 + side_y * 2 + side_x);
2012
2013                         /* update the settings for the eye check */
2014                         for (ch = 0; ch < NUM_CHANNELS; ch++) {
2015                                 if (mrc_params->channel_enables & (1 << ch)) {
2016                                         for (rk = 0; rk < NUM_RANKS; rk++) {
2017                                                 if (mrc_params->rank_enables & (1 << rk)) {
2018                                                         for (bl = 0; bl < NUM_BYTE_LANES / bl_divisor; bl++) {
2019                                                                 if (side_x == L)
2020                                                                         set_rdqs(ch, rk, bl, x_center[ch][rk][bl] - (MIN_RDQS_EYE / 2));
2021                                                                 else
2022                                                                         set_rdqs(ch, rk, bl, x_center[ch][rk][bl] + (MIN_RDQS_EYE / 2));
2023
2024                                                                 if (side_y == B)
2025                                                                         set_vref(ch, bl, y_center[ch][bl] - (MIN_VREF_EYE / 2));
2026                                                                 else
2027                                                                         set_vref(ch, bl, y_center[ch][bl] + (MIN_VREF_EYE / 2));
2028                                                         }
2029                                                 }
2030                                         }
2031                                 }
2032                         }
2033
2034                         /* request HTE reconfiguration */
2035                         mrc_params->hte_setup = 1;
2036
2037                         /* check the eye */
2038                         if (check_bls_ex(mrc_params, address) & 0xff) {
2039                                 /* one or more byte lanes failed */
2040                                 mrc_post_code(0xee, 0x74 + side_x * 2 + side_y);
2041                         }
2042                 }
2043         }
2044 #endif
2045
2046         mrc_post_code(0x07, 0x40);
2047
2048         /* set final placements */
2049         for (ch = 0; ch < NUM_CHANNELS; ch++) {
2050                 if (mrc_params->channel_enables & (1 << ch)) {
2051                         for (rk = 0; rk < NUM_RANKS; rk++) {
2052                                 if (mrc_params->rank_enables & (1 << rk)) {
2053 #ifdef R2R_SHARING
2054                                         /* increment "num_ranks_enabled" */
2055                                         num_ranks_enabled++;
2056 #endif
2057                                         for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
2058                                                 /* x_coordinate */
2059 #ifdef R2R_SHARING
2060                                                 final_delay[ch][bl] += x_center[ch][rk][bl];
2061                                                 set_rdqs(ch, rk, bl, final_delay[ch][bl] / num_ranks_enabled);
2062 #else
2063                                                 set_rdqs(ch, rk, bl, x_center[ch][rk][bl]);
2064 #endif
2065                                                 /* y_coordinate */
2066                                                 set_vref(ch, bl, y_center[ch][bl]);
2067                                         }
2068                                 }
2069                         }
2070                 }
2071         }
2072 #endif
2073
2074         LEAVEFN();
2075 }
2076
2077 /*
2078  * This function will perform the WRITE TRAINING Algorithm on all
2079  * channels/ranks/byte_lanes simultaneously to minimize execution time.
2080  *
2081  * The idea here is to train the WDQ timings to achieve maximum WRITE margins.
2082  * The algorithm will start with WDQ at the current WDQ setting (tracks WDQS
2083  * in WR_LVL) +/- 32 PIs (+/- 1/4 CLK) and collapse the eye until all data
2084  * patterns pass. This is because WDQS will be aligned to WCLK by the
2085  * Write Leveling algorithm and WDQ will only ever have a 1/2 CLK window
2086  * of validity.
2087  */
2088 void wr_train(struct mrc_params *mrc_params)
2089 {
2090         uint8_t ch;     /* channel counter */
2091         uint8_t rk;     /* rank counter */
2092         uint8_t bl;     /* byte lane counter */
2093         uint8_t bl_divisor = (mrc_params->channel_width == X16) ? 2 : 1;
2094 #ifdef BACKUP_WDQ
2095 #else
2096         uint8_t side;           /* LEFT/RIGHT side indicator (0=L, 1=R) */
2097         uint32_t temp;          /* temporary DWORD */
2098         /* 2 arrays, for L & R side passing delays */
2099         uint32_t delay[2][NUM_CHANNELS][NUM_RANKS][NUM_BYTE_LANES];
2100         uint32_t address;       /* target address for check_bls_ex() */
2101         uint32_t result;        /* result of check_bls_ex() */
2102         uint32_t bl_mask;       /* byte lane mask for result checking */
2103 #ifdef R2R_SHARING
2104         /* used to find placement for rank2rank sharing configs */
2105         uint32_t final_delay[NUM_CHANNELS][NUM_BYTE_LANES];
2106         /* used to find placement for rank2rank sharing configs */
2107         uint32_t num_ranks_enabled = 0;
2108 #endif
2109 #endif
2110
2111         /* wr_train starts */
2112         mrc_post_code(0x08, 0x00);
2113
2114         ENTERFN();
2115
2116 #ifdef BACKUP_WDQ
2117         for (ch = 0; ch < NUM_CHANNELS; ch++) {
2118                 if (mrc_params->channel_enables & (1 << ch)) {
2119                         for (rk = 0; rk < NUM_RANKS; rk++) {
2120                                 if (mrc_params->rank_enables & (1 << rk)) {
2121                                         for (bl = 0;
2122                                              bl < NUM_BYTE_LANES / bl_divisor;
2123                                              bl++) {
2124                                                 set_wdq(ch, rk, bl, ddr_wdq[PLATFORM_ID]);
2125                                         }
2126                                 }
2127                         }
2128                 }
2129         }
2130 #else
2131         /* initialize "delay" */
2132         for (ch = 0; ch < NUM_CHANNELS; ch++) {
2133                 if (mrc_params->channel_enables & (1 << ch)) {
2134                         for (rk = 0; rk < NUM_RANKS; rk++) {
2135                                 if (mrc_params->rank_enables & (1 << rk)) {
2136                                         for (bl = 0;
2137                                              bl < NUM_BYTE_LANES / bl_divisor;
2138                                              bl++) {
2139                                                 /*
2140                                                  * want to start with
2141                                                  * WDQ = (WDQS - QRTR_CLK)
2142                                                  * +/- QRTR_CLK
2143                                                  */
2144                                                 temp = get_wdqs(ch, rk, bl) - QRTR_CLK;
2145                                                 delay[L][ch][rk][bl] = temp - QRTR_CLK;
2146                                                 delay[R][ch][rk][bl] = temp + QRTR_CLK;
2147                                         }
2148                                 }
2149                         }
2150                 }
2151         }
2152
2153         /* initialize other variables */
2154         bl_mask = byte_lane_mask(mrc_params);
2155         address = get_addr(0, 0);
2156
2157 #ifdef R2R_SHARING
2158         /* need to set "final_delay[][]" elements to "0" */
2159         memset((void *)(final_delay), 0x00, (size_t)sizeof(final_delay));
2160 #endif
2161
2162         /*
2163          * start algorithm on the LEFT side and train each channel/bl
2164          * until no failures are observed, then repeat for the RIGHT side.
2165          */
2166         for (side = L; side <= R; side++) {
2167                 mrc_post_code(0x08, 0x10 + side);
2168
2169                 /* set starting values */
2170                 for (ch = 0; ch < NUM_CHANNELS; ch++) {
2171                         if (mrc_params->channel_enables & (1 << ch)) {
2172                                 for (rk = 0; rk < NUM_RANKS; rk++) {
2173                                         if (mrc_params->rank_enables &
2174                                                 (1 << rk)) {
2175                                                 for (bl = 0;
2176                                                      bl < NUM_BYTE_LANES / bl_divisor;
2177                                                      bl++) {
2178                                                         set_wdq(ch, rk, bl, delay[side][ch][rk][bl]);
2179                                                 }
2180                                         }
2181                                 }
2182                         }
2183                 }
2184
2185                 /* find passing values */
2186                 for (ch = 0; ch < NUM_CHANNELS; ch++) {
2187                         if (mrc_params->channel_enables & (1 << ch)) {
2188                                 for (rk = 0; rk < NUM_RANKS; rk++) {
2189                                         if (mrc_params->rank_enables &
2190                                                 (1 << rk)) {
2191                                                 /* get an address in the target channel/rank */
2192                                                 address = get_addr(ch, rk);
2193
2194                                                 /* request HTE reconfiguration */
2195                                                 mrc_params->hte_setup = 1;
2196
2197                                                 /* check the settings */
2198                                                 do {
2199                                                         /* result[07:00] == failing byte lane (MAX 8) */
2200                                                         result = check_bls_ex(mrc_params, address);
2201                                                         /* check for failures */
2202                                                         if (result & 0xff) {
2203                                                                 /* at least 1 byte lane failed */
2204                                                                 for (bl = 0; bl < NUM_BYTE_LANES / bl_divisor; bl++) {
2205                                                                         if (result &
2206                                                                                 (bl_mask << bl)) {
2207                                                                                 if (side == L)
2208                                                                                         delay[L][ch][rk][bl] += WDQ_STEP;
2209                                                                                 else
2210                                                                                         delay[R][ch][rk][bl] -= WDQ_STEP;
2211
2212                                                                                 /* check for algorithm failure */
2213                                                                                 if (delay[L][ch][rk][bl] != delay[R][ch][rk][bl]) {
2214                                                                                         /*
2215                                                                                          * margin available
2216                                                                                          * update delay setting
2217                                                                                          */
2218                                                                                         set_wdq(ch, rk, bl,
2219                                                                                                 delay[side][ch][rk][bl]);
2220                                                                                 } else {
2221                                                                                         /*
2222                                                                                          * no margin available
2223                                                                                          * notify the user and halt
2224                                                                                          */
2225                                                                                         training_message(ch, rk, bl);
2226                                                                                         mrc_post_code(0xee, 0x80 + side);
2227                                                                                 }
2228                                                                         }
2229                                                                 }
2230                                                         }
2231                                                 /* stop when all byte lanes pass */
2232                                                 } while (result & 0xff);
2233                                         }
2234                                 }
2235                         }
2236                 }
2237         }
2238
2239         /* program WDQ to the middle of passing window */
2240         for (ch = 0; ch < NUM_CHANNELS; ch++) {
2241                 if (mrc_params->channel_enables & (1 << ch)) {
2242                         for (rk = 0; rk < NUM_RANKS; rk++) {
2243                                 if (mrc_params->rank_enables & (1 << rk)) {
2244 #ifdef R2R_SHARING
2245                                         /* increment "num_ranks_enabled" */
2246                                         num_ranks_enabled++;
2247 #endif
2248                                         for (bl = 0; bl < NUM_BYTE_LANES / bl_divisor; bl++) {
2249                                                 DPF(D_INFO,
2250                                                     "WDQ eye rank%d lane%d : %d-%d\n",
2251                                                     rk, bl,
2252                                                     delay[L][ch][rk][bl],
2253                                                     delay[R][ch][rk][bl]);
2254
2255                                                 temp = (delay[R][ch][rk][bl] + delay[L][ch][rk][bl]) / 2;
2256
2257 #ifdef R2R_SHARING
2258                                                 final_delay[ch][bl] += temp;
2259                                                 set_wdq(ch, rk, bl,
2260                                                         final_delay[ch][bl] / num_ranks_enabled);
2261 #else
2262                                                 set_wdq(ch, rk, bl, temp);
2263 #endif
2264                                         }
2265                                 }
2266                         }
2267                 }
2268         }
2269 #endif
2270
2271         LEAVEFN();
2272 }
2273
2274 /*
2275  * This function will store relevant timing data
2276  *
2277  * This data will be used on subsequent boots to speed up boot times
2278  * and is required for Suspend To RAM capabilities.
2279  */
2280 void store_timings(struct mrc_params *mrc_params)
2281 {
2282         uint8_t ch, rk, bl;
2283         struct mrc_timings *mt = &mrc_params->timings;
2284
2285         for (ch = 0; ch < NUM_CHANNELS; ch++) {
2286                 for (rk = 0; rk < NUM_RANKS; rk++) {
2287                         for (bl = 0; bl < NUM_BYTE_LANES; bl++) {
2288                                 mt->rcvn[ch][rk][bl] = get_rcvn(ch, rk, bl);
2289                                 mt->rdqs[ch][rk][bl] = get_rdqs(ch, rk, bl);
2290                                 mt->wdqs[ch][rk][bl] = get_wdqs(ch, rk, bl);
2291                                 mt->wdq[ch][rk][bl] = get_wdq(ch, rk, bl);
2292
2293                                 if (rk == 0)
2294                                         mt->vref[ch][bl] = get_vref(ch, bl);
2295                         }
2296
2297                         mt->wctl[ch][rk] = get_wctl(ch, rk);
2298                 }
2299
2300                 mt->wcmd[ch] = get_wcmd(ch);
2301         }
2302
2303         /* need to save for a case of changing frequency after warm reset */
2304         mt->ddr_speed = mrc_params->ddr_speed;
2305 }
2306
2307 /*
2308  * The purpose of this function is to ensure the SEC comes out of reset
2309  * and IA initiates the SEC enabling Memory Scrambling.
2310  */
2311 void enable_scrambling(struct mrc_params *mrc_params)
2312 {
2313         uint32_t lfsr = 0;
2314         uint8_t i;
2315
2316         if (mrc_params->scrambling_enables == 0)
2317                 return;
2318
2319         ENTERFN();
2320
2321         /* 32 bit seed is always stored in BIOS NVM */
2322         lfsr = mrc_params->timings.scrambler_seed;
2323
2324         if (mrc_params->boot_mode == BM_COLD) {
2325                 /*
2326                  * factory value is 0 and in first boot,
2327                  * a clock based seed is loaded.
2328                  */
2329                 if (lfsr == 0) {
2330                         /*
2331                          * get seed from system clock
2332                          * and make sure it is not all 1's
2333                          */
2334                         lfsr = rdtsc() & 0x0fffffff;
2335                 } else {
2336                         /*
2337                          * Need to replace scrambler
2338                          *
2339                          * get next 32bit LFSR 16 times which is the last
2340                          * part of the previous scrambler vector
2341                          */
2342                         for (i = 0; i < 16; i++)
2343                                 lfsr32(&lfsr);
2344                 }
2345
2346                 /* save new seed */
2347                 mrc_params->timings.scrambler_seed = lfsr;
2348         }
2349
2350         /*
2351          * In warm boot or S3 exit, we have the previous seed.
2352          * In cold boot, we have the last 32bit LFSR which is the new seed.
2353          */
2354         lfsr32(&lfsr);  /* shift to next value */
2355         msg_port_write(MEM_CTLR, SCRMSEED, (lfsr & 0x0003ffff));
2356
2357         for (i = 0; i < 2; i++)
2358                 msg_port_write(MEM_CTLR, SCRMLO + i, (lfsr & 0xaaaaaaaa));
2359
2360         LEAVEFN();
2361 }
2362
2363 /*
2364  * Configure MCU Power Management Control Register
2365  * and Scheduler Control Register
2366  */
2367 void prog_ddr_control(struct mrc_params *mrc_params)
2368 {
2369         u32 dsch;
2370         u32 dpmc0;
2371
2372         ENTERFN();
2373
2374         dsch = msg_port_read(MEM_CTLR, DSCH);
2375         dsch &= ~(DSCH_OOODIS | DSCH_OOOST3DIS | DSCH_NEWBYPDIS);
2376         msg_port_write(MEM_CTLR, DSCH, dsch);
2377
2378         dpmc0 = msg_port_read(MEM_CTLR, DPMC0);
2379         dpmc0 &= ~DPMC0_DISPWRDN;
2380         dpmc0 |= (mrc_params->power_down_disable << 25);
2381         dpmc0 &= ~DPMC0_CLKGTDIS;
2382         dpmc0 &= ~DPMC0_PCLSTO_MASK;
2383         dpmc0 |= (4 << 16);
2384         dpmc0 |= DPMC0_PREAPWDEN;
2385         msg_port_write(MEM_CTLR, DPMC0, dpmc0);
2386
2387         /* CMDTRIST = 2h - CMD/ADDR are tristated when no valid command */
2388         mrc_write_mask(MEM_CTLR, DPMC1, 0x20, 0x30);
2389
2390         LEAVEFN();
2391 }
2392
2393 /*
2394  * After training complete configure MCU Rank Population Register
2395  * specifying: ranks enabled, device width, density, address mode
2396  */
2397 void prog_dra_drb(struct mrc_params *mrc_params)
2398 {
2399         u32 drp;
2400         u32 dco;
2401         u8 density = mrc_params->params.density;
2402
2403         ENTERFN();
2404
2405         dco = msg_port_read(MEM_CTLR, DCO);
2406         dco &= ~DCO_IC;
2407         msg_port_write(MEM_CTLR, DCO, dco);
2408
2409         drp = 0;
2410         if (mrc_params->rank_enables & 1)
2411                 drp |= DRP_RKEN0;
2412         if (mrc_params->rank_enables & 2)
2413                 drp |= DRP_RKEN1;
2414         if (mrc_params->dram_width == X16) {
2415                 drp |= (1 << 4);
2416                 drp |= (1 << 9);
2417         }
2418
2419         /*
2420          * Density encoding in struct dram_params: 0=512Mb, 1=Gb, 2=2Gb, 3=4Gb
2421          * has to be mapped RANKDENSx encoding (0=1Gb)
2422          */
2423         if (density == 0)
2424                 density = 4;
2425
2426         drp |= ((density - 1) << 6);
2427         drp |= ((density - 1) << 11);
2428
2429         /* Address mode can be overwritten if ECC enabled */
2430         drp |= (mrc_params->address_mode << 14);
2431
2432         msg_port_write(MEM_CTLR, DRP, drp);
2433
2434         dco &= ~DCO_PMICTL;
2435         dco |= DCO_IC;
2436         msg_port_write(MEM_CTLR, DCO, dco);
2437
2438         LEAVEFN();
2439 }
2440
2441 /* Send DRAM wake command */
2442 void perform_wake(struct mrc_params *mrc_params)
2443 {
2444         ENTERFN();
2445
2446         dram_wake_command();
2447
2448         LEAVEFN();
2449 }
2450
2451 /*
2452  * Configure refresh rate and short ZQ calibration interval
2453  * Activate dynamic self refresh
2454  */
2455 void change_refresh_period(struct mrc_params *mrc_params)
2456 {
2457         u32 drfc;
2458         u32 dcal;
2459         u32 dpmc0;
2460
2461         ENTERFN();
2462
2463         drfc = msg_port_read(MEM_CTLR, DRFC);
2464         drfc &= ~DRFC_TREFI_MASK;
2465         drfc |= (mrc_params->refresh_rate << 12);
2466         drfc |= DRFC_REFDBTCLR;
2467         msg_port_write(MEM_CTLR, DRFC, drfc);
2468
2469         dcal = msg_port_read(MEM_CTLR, DCAL);
2470         dcal &= ~DCAL_ZQCINT_MASK;
2471         dcal |= (3 << 8);       /* 63ms */
2472         msg_port_write(MEM_CTLR, DCAL, dcal);
2473
2474         dpmc0 = msg_port_read(MEM_CTLR, DPMC0);
2475         dpmc0 |= (DPMC0_DYNSREN | DPMC0_ENPHYCLKGATE);
2476         msg_port_write(MEM_CTLR, DPMC0, dpmc0);
2477
2478         LEAVEFN();
2479 }
2480
2481 /*
2482  * Configure DDRPHY for Auto-Refresh, Periodic Compensations,
2483  * Dynamic Diff-Amp, ZQSPERIOD, Auto-Precharge, CKE Power-Down
2484  */
2485 void set_auto_refresh(struct mrc_params *mrc_params)
2486 {
2487         uint32_t channel;
2488         uint32_t rank;
2489         uint32_t bl;
2490         uint32_t bl_divisor = 1;
2491         uint32_t temp;
2492
2493         ENTERFN();
2494
2495         /*
2496          * Enable Auto-Refresh, Periodic Compensations, Dynamic Diff-Amp,
2497          * ZQSPERIOD, Auto-Precharge, CKE Power-Down
2498          */
2499         for (channel = 0; channel < NUM_CHANNELS; channel++) {
2500                 if (mrc_params->channel_enables & (1 << channel)) {
2501                         /* Enable Periodic RCOMPS */
2502                         mrc_alt_write_mask(DDRPHY, CMPCTRL, 2, 2);
2503
2504                         /* Enable Dynamic DiffAmp & Set Read ODT Value */
2505                         switch (mrc_params->rd_odt_value) {
2506                         case 0:
2507                                 temp = 0x3f;    /* OFF */
2508                                 break;
2509                         default:
2510                                 temp = 0x00;    /* Auto */
2511                                 break;
2512                         }
2513
2514                         for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor) / 2; bl++) {
2515                                 /* Override: DIFFAMP, ODT */
2516                                 mrc_alt_write_mask(DDRPHY,
2517                                         B0OVRCTL + bl * DDRIODQ_BL_OFFSET +
2518                                         channel * DDRIODQ_CH_OFFSET,
2519                                         temp << 10,
2520                                         0x003ffc00);
2521
2522                                 /* Override: DIFFAMP, ODT */
2523                                 mrc_alt_write_mask(DDRPHY,
2524                                         B1OVRCTL + bl * DDRIODQ_BL_OFFSET +
2525                                         channel * DDRIODQ_CH_OFFSET,
2526                                         temp << 10,
2527                                         0x003ffc00);
2528                         }
2529
2530                         /* Issue ZQCS command */
2531                         for (rank = 0; rank < NUM_RANKS; rank++) {
2532                                 if (mrc_params->rank_enables & (1 << rank))
2533                                         dram_init_command(DCMD_ZQCS(rank));
2534                         }
2535                 }
2536         }
2537
2538         clear_pointers();
2539
2540         LEAVEFN();
2541 }
2542
2543 /*
2544  * Depending on configuration enables ECC support
2545  *
2546  * Available memory size is decreased, and updated with 0s
2547  * in order to clear error status. Address mode 2 forced.
2548  */
2549 void ecc_enable(struct mrc_params *mrc_params)
2550 {
2551         u32 drp;
2552         u32 dsch;
2553         u32 ecc_ctrl;
2554
2555         if (mrc_params->ecc_enables == 0)
2556                 return;
2557
2558         ENTERFN();
2559
2560         /* Configuration required in ECC mode */
2561         drp = msg_port_read(MEM_CTLR, DRP);
2562         drp &= ~DRP_ADDRMAP_MASK;
2563         drp |= DRP_ADDRMAP_MAP1;
2564         drp |= DRP_PRI64BSPLITEN;
2565         msg_port_write(MEM_CTLR, DRP, drp);
2566
2567         /* Disable new request bypass */
2568         dsch = msg_port_read(MEM_CTLR, DSCH);
2569         dsch |= DSCH_NEWBYPDIS;
2570         msg_port_write(MEM_CTLR, DSCH, dsch);
2571
2572         /* Enable ECC */
2573         ecc_ctrl = (DECCCTRL_SBEEN | DECCCTRL_DBEEN | DECCCTRL_ENCBGEN);
2574         msg_port_write(MEM_CTLR, DECCCTRL, ecc_ctrl);
2575
2576         /* Assume 8 bank memory, one bank is gone for ECC */
2577         mrc_params->mem_size -= mrc_params->mem_size / 8;
2578
2579         /* For S3 resume memory content has to be preserved */
2580         if (mrc_params->boot_mode != BM_S3) {
2581                 select_hte();
2582                 hte_mem_init(mrc_params, MRC_MEM_INIT);
2583                 select_mem_mgr();
2584         }
2585
2586         LEAVEFN();
2587 }
2588
2589 /*
2590  * Execute memory test
2591  * if error detected it is indicated in mrc_params->status
2592  */
2593 void memory_test(struct mrc_params *mrc_params)
2594 {
2595         uint32_t result = 0;
2596
2597         ENTERFN();
2598
2599         select_hte();
2600         result = hte_mem_init(mrc_params, MRC_MEM_TEST);
2601         select_mem_mgr();
2602
2603         DPF(D_INFO, "Memory test result %x\n", result);
2604         mrc_params->status = ((result == 0) ? MRC_SUCCESS : MRC_E_MEMTEST);
2605         LEAVEFN();
2606 }
2607
2608 /* Lock MCU registers at the end of initialization sequence */
2609 void lock_registers(struct mrc_params *mrc_params)
2610 {
2611         u32 dco;
2612
2613         ENTERFN();
2614
2615         dco = msg_port_read(MEM_CTLR, DCO);
2616         dco &= ~(DCO_PMICTL | DCO_PMIDIS);
2617         dco |= (DCO_DRPLOCK | DCO_CPGCLOCK);
2618         msg_port_write(MEM_CTLR, DCO, dco);
2619
2620         LEAVEFN();
2621 }