+ /*
+ * Set counter and jump addresses for the right
+ * number of NOP cycles.
+ * The number of supported NOP cycles can range from -1 to infinity
+ * Three different cases are handled:
+ *
+ * 1. For a number of NOP cycles greater than 0, the RW Mgr looping
+ * mechanism will be used to insert the right number of NOPs
+ *
+ * 2. For a number of NOP cycles equals to 0, the micro-instruction
+ * issuing the write command will jump straight to the
+ * micro-instruction that turns on DQS (for DDRx), or outputs write
+ * data (for RLD), skipping
+ * the NOP micro-instruction all together
+ *
+ * 3. A number of NOP cycles equal to -1 indicates that DQS must be
+ * turned on in the same micro-instruction that issues the write
+ * command. Then we need
+ * to directly jump to the micro-instruction that sends out the data
+ *
+ * NOTE: Implementing this mechanism uses 2 RW Mgr jump-counters
+ * (2 and 3). One jump-counter (0) is used to perform multiple
+ * write-read operations.
+ * one counter left to issue this command in "multiple-group" mode
+ */
+
+ rw_wl_nop_cycles = gbl->rw_wl_nop_cycles;
+
+ if (rw_wl_nop_cycles == -1) {
+ /*
+ * CNTR 2 - We want to execute the special write operation that
+ * turns on DQS right away and then skip directly to the
+ * instruction that sends out the data. We set the counter to a
+ * large number so that the jump is always taken.
+ */
+ writel(0xFF, &sdr_rw_load_mgr_regs->load_cntr2);
+
+ /* CNTR 3 - Not used */
+ if (test_dm) {
+ mcc_instruction = rwcfg->lfsr_wr_rd_dm_bank_0_wl_1;
+ writel(rwcfg->lfsr_wr_rd_dm_bank_0_data,
+ &sdr_rw_load_jump_mgr_regs->load_jump_add2);
+ writel(rwcfg->lfsr_wr_rd_dm_bank_0_nop,
+ &sdr_rw_load_jump_mgr_regs->load_jump_add3);
+ } else {
+ mcc_instruction = rwcfg->lfsr_wr_rd_bank_0_wl_1;
+ writel(rwcfg->lfsr_wr_rd_bank_0_data,
+ &sdr_rw_load_jump_mgr_regs->load_jump_add2);
+ writel(rwcfg->lfsr_wr_rd_bank_0_nop,
+ &sdr_rw_load_jump_mgr_regs->load_jump_add3);
+ }
+ } else if (rw_wl_nop_cycles == 0) {
+ /*
+ * CNTR 2 - We want to skip the NOP operation and go straight
+ * to the DQS enable instruction. We set the counter to a large
+ * number so that the jump is always taken.
+ */
+ writel(0xFF, &sdr_rw_load_mgr_regs->load_cntr2);
+
+ /* CNTR 3 - Not used */
+ if (test_dm) {
+ mcc_instruction = rwcfg->lfsr_wr_rd_dm_bank_0;
+ writel(rwcfg->lfsr_wr_rd_dm_bank_0_dqs,
+ &sdr_rw_load_jump_mgr_regs->load_jump_add2);
+ } else {
+ mcc_instruction = rwcfg->lfsr_wr_rd_bank_0;
+ writel(rwcfg->lfsr_wr_rd_bank_0_dqs,
+ &sdr_rw_load_jump_mgr_regs->load_jump_add2);
+ }
+ } else {
+ /*
+ * CNTR 2 - In this case we want to execute the next instruction
+ * and NOT take the jump. So we set the counter to 0. The jump
+ * address doesn't count.
+ */
+ writel(0x0, &sdr_rw_load_mgr_regs->load_cntr2);
+ writel(0x0, &sdr_rw_load_jump_mgr_regs->load_jump_add2);
+
+ /*
+ * CNTR 3 - Set the nop counter to the number of cycles we
+ * need to loop for, minus 1.
+ */
+ writel(rw_wl_nop_cycles - 1, &sdr_rw_load_mgr_regs->load_cntr3);
+ if (test_dm) {
+ mcc_instruction = rwcfg->lfsr_wr_rd_dm_bank_0;
+ writel(rwcfg->lfsr_wr_rd_dm_bank_0_nop,
+ &sdr_rw_load_jump_mgr_regs->load_jump_add3);
+ } else {
+ mcc_instruction = rwcfg->lfsr_wr_rd_bank_0;
+ writel(rwcfg->lfsr_wr_rd_bank_0_nop,
+ &sdr_rw_load_jump_mgr_regs->load_jump_add3);
+ }
+ }
+
+ writel(0, SDR_PHYGRP_RWMGRGRP_ADDRESS |
+ RW_MGR_RESET_READ_DATAPATH_OFFSET);
+
+ if (quick_write_mode)
+ writel(0x08, &sdr_rw_load_mgr_regs->load_cntr0);
+ else
+ writel(0x40, &sdr_rw_load_mgr_regs->load_cntr0);
+
+ writel(mcc_instruction, &sdr_rw_load_jump_mgr_regs->load_jump_add0);
+
+ /*
+ * CNTR 1 - This is used to ensure enough time elapses
+ * for read data to come back.
+ */
+ writel(0x30, &sdr_rw_load_mgr_regs->load_cntr1);
+
+ if (test_dm) {
+ writel(rwcfg->lfsr_wr_rd_dm_bank_0_wait,
+ &sdr_rw_load_jump_mgr_regs->load_jump_add1);
+ } else {
+ writel(rwcfg->lfsr_wr_rd_bank_0_wait,
+ &sdr_rw_load_jump_mgr_regs->load_jump_add1);
+ }
+
+ writel(mcc_instruction, (SDR_PHYGRP_RWMGRGRP_ADDRESS |
+ RW_MGR_RUN_SINGLE_GROUP_OFFSET) +
+ (group << 2));
+}
+
+/**
+ * rw_mgr_mem_calibrate_write_test() - Test writes, check for single/multiple pass
+ * @rank_bgn: Rank number
+ * @write_group: Write Group
+ * @use_dm: Use DM
+ * @all_correct: All bits must be correct in the mask
+ * @bit_chk: Resulting bit mask after the test
+ * @all_ranks: Test all ranks
+ *
+ * Test writes, can check for a single bit pass or multiple bit pass.
+ */
+static int
+rw_mgr_mem_calibrate_write_test(const u32 rank_bgn, const u32 write_group,
+ const u32 use_dm, const u32 all_correct,
+ u32 *bit_chk, const u32 all_ranks)
+{
+ const u32 rank_end = all_ranks ?
+ rwcfg->mem_number_of_ranks :
+ (rank_bgn + NUM_RANKS_PER_SHADOW_REG);
+ const u32 shift_ratio = rwcfg->mem_dq_per_write_dqs /
+ rwcfg->mem_virtual_groups_per_write_dqs;
+ const u32 correct_mask_vg = param->write_correct_mask_vg;
+
+ u32 tmp_bit_chk, base_rw_mgr;
+ int vg, r;
+
+ *bit_chk = param->write_correct_mask;