drivers/mtd/nand/qcom_nandc.c

   1 /*
   2  * Copyright (c) 2016, The Linux Foundation. All rights reserved.
   3  *
   4  * This software is licensed under the terms of the GNU General Public
   5  * License version 2, as published by the Free Software Foundation, and
   6  * may be copied, distributed, and modified under those terms.
   7  *
   8  * This program is distributed in the hope that it will be useful,
   9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  11  * GNU General Public License for more details.
  12  */
  13
  14 #include <linux/clk.h>
  15 #include <linux/slab.h>
  16 #include <linux/bitops.h>
  17 #include <linux/dma-mapping.h>
  18 #include <linux/dmaengine.h>
  19 #include <linux/module.h>
  20 #include <linux/mtd/nand.h>
  21 #include <linux/mtd/partitions.h>
  22 #include <linux/of.h>
  23 #include <linux/of_device.h>
  24 #include <linux/delay.h>
  25
  26 /* NANDc reg offsets */
  27 #define NAND_FLASH_CMD                  0x00
  28 #define NAND_ADDR0                      0x04
  29 #define NAND_ADDR1                      0x08
  30 #define NAND_FLASH_CHIP_SELECT          0x0c
  31 #define NAND_EXEC_CMD                   0x10
  32 #define NAND_FLASH_STATUS               0x14
  33 #define NAND_BUFFER_STATUS              0x18
  34 #define NAND_DEV0_CFG0                  0x20
  35 #define NAND_DEV0_CFG1                  0x24
  36 #define NAND_DEV0_ECC_CFG               0x28
  37 #define NAND_DEV1_ECC_CFG               0x2c
  38 #define NAND_DEV1_CFG0                  0x30
  39 #define NAND_DEV1_CFG1                  0x34
  40 #define NAND_READ_ID                    0x40
  41 #define NAND_READ_STATUS                0x44
  42 #define NAND_DEV_CMD0                   0xa0
  43 #define NAND_DEV_CMD1                   0xa4
  44 #define NAND_DEV_CMD2                   0xa8
  45 #define NAND_DEV_CMD_VLD                0xac
  46 #define SFLASHC_BURST_CFG               0xe0
  47 #define NAND_ERASED_CW_DETECT_CFG       0xe8
  48 #define NAND_ERASED_CW_DETECT_STATUS    0xec
  49 #define NAND_EBI2_ECC_BUF_CFG           0xf0
  50 #define FLASH_BUF_ACC                   0x100
  51
  52 #define NAND_CTRL                       0xf00
  53 #define NAND_VERSION                    0xf08
  54 #define NAND_READ_LOCATION_0            0xf20
  55 #define NAND_READ_LOCATION_1            0xf24
  56
  57 /* dummy register offsets, used by write_reg_dma */
  58 #define NAND_DEV_CMD1_RESTORE           0xdead
  59 #define NAND_DEV_CMD_VLD_RESTORE        0xbeef
  60
  61 /* NAND_FLASH_CMD bits */
  62 #define PAGE_ACC                        BIT(4)
  63 #define LAST_PAGE                       BIT(5)
  64
  65 /* NAND_FLASH_CHIP_SELECT bits */
  66 #define NAND_DEV_SEL                    0
  67 #define DM_EN                           BIT(2)
  68
  69 /* NAND_FLASH_STATUS bits */
  70 #define FS_OP_ERR                       BIT(4)
  71 #define FS_READY_BSY_N                  BIT(5)
  72 #define FS_MPU_ERR                      BIT(8)
  73 #define FS_DEVICE_STS_ERR               BIT(16)
  74 #define FS_DEVICE_WP                    BIT(23)
  75
  76 /* NAND_BUFFER_STATUS bits */
  77 #define BS_UNCORRECTABLE_BIT            BIT(8)
  78 #define BS_CORRECTABLE_ERR_MSK          0x1f
  79
  80 /* NAND_DEVn_CFG0 bits */
  81 #define DISABLE_STATUS_AFTER_WRITE      4
  82 #define CW_PER_PAGE                     6
  83 #define UD_SIZE_BYTES                   9
  84 #define ECC_PARITY_SIZE_BYTES_RS        19
  85 #define SPARE_SIZE_BYTES                23
  86 #define NUM_ADDR_CYCLES                 27
  87 #define STATUS_BFR_READ                 30
  88 #define SET_RD_MODE_AFTER_STATUS        31
  89
  90 /* NAND_DEVn_CFG0 bits */
  91 #define DEV0_CFG1_ECC_DISABLE           0
  92 #define WIDE_FLASH                      1
  93 #define NAND_RECOVERY_CYCLES            2
  94 #define CS_ACTIVE_BSY                   5
  95 #define BAD_BLOCK_BYTE_NUM              6
  96 #define BAD_BLOCK_IN_SPARE_AREA         16
  97 #define WR_RD_BSY_GAP                   17
  98 #define ENABLE_BCH_ECC                  27
  99
 100 /* NAND_DEV0_ECC_CFG bits */
 101 #define ECC_CFG_ECC_DISABLE             0
 102 #define ECC_SW_RESET                    1
 103 #define ECC_MODE                        4
 104 #define ECC_PARITY_SIZE_BYTES_BCH       8
 105 #define ECC_NUM_DATA_BYTES              16
 106 #define ECC_FORCE_CLK_OPEN              30
 107
 108 /* NAND_DEV_CMD1 bits */
 109 #define READ_ADDR                       0
 110
 111 /* NAND_DEV_CMD_VLD bits */
 112 #define READ_START_VLD                  0
 113
 114 /* NAND_EBI2_ECC_BUF_CFG bits */
 115 #define NUM_STEPS                       0
 116
 117 /* NAND_ERASED_CW_DETECT_CFG bits */
 118 #define ERASED_CW_ECC_MASK              1
 119 #define AUTO_DETECT_RES                 0
 120 #define MASK_ECC                        (1 << ERASED_CW_ECC_MASK)
 121 #define RESET_ERASED_DET                (1 << AUTO_DETECT_RES)
 122 #define ACTIVE_ERASED_DET               (0 << AUTO_DETECT_RES)
 123 #define CLR_ERASED_PAGE_DET             (RESET_ERASED_DET | MASK_ECC)
 124 #define SET_ERASED_PAGE_DET             (ACTIVE_ERASED_DET | MASK_ECC)
 125
 126 /* NAND_ERASED_CW_DETECT_STATUS bits */
 127 #define PAGE_ALL_ERASED                 BIT(7)
 128 #define CODEWORD_ALL_ERASED             BIT(6)
 129 #define PAGE_ERASED                     BIT(5)
 130 #define CODEWORD_ERASED                 BIT(4)
 131 #define ERASED_PAGE                     (PAGE_ALL_ERASED | PAGE_ERASED)
 132 #define ERASED_CW                       (CODEWORD_ALL_ERASED | CODEWORD_ERASED)
 133
 134 /* Version Mask */
 135 #define NAND_VERSION_MAJOR_MASK         0xf0000000
 136 #define NAND_VERSION_MAJOR_SHIFT        28
 137 #define NAND_VERSION_MINOR_MASK         0x0fff0000
 138 #define NAND_VERSION_MINOR_SHIFT        16
 139
 140 /* NAND OP_CMDs */
 141 #define PAGE_READ                       0x2
 142 #define PAGE_READ_WITH_ECC              0x3
 143 #define PAGE_READ_WITH_ECC_SPARE        0x4
 144 #define PROGRAM_PAGE                    0x6
 145 #define PAGE_PROGRAM_WITH_ECC           0x7
 146 #define PROGRAM_PAGE_SPARE              0x9
 147 #define BLOCK_ERASE                     0xa
 148 #define FETCH_ID                        0xb
 149 #define RESET_DEVICE                    0xd
 150
 151 /*
 152  * the NAND controller performs reads/writes with ECC in 516 byte chunks.
 153  * the driver calls the chunks 'step' or 'codeword' interchangeably
 154  */
 155 #define NANDC_STEP_SIZE                 512
 156
 157 /*
 158  * the largest page size we support is 8K, this will have 16 steps/codewords
 159  * of 512 bytes each
 160  */
 161 #define MAX_NUM_STEPS                   (SZ_8K / NANDC_STEP_SIZE)
 162
 163 /* we read at most 3 registers per codeword scan */
 164 #define MAX_REG_RD                      (3 * MAX_NUM_STEPS)
 165
 166 /* ECC modes supported by the controller */
 167 #define ECC_NONE        BIT(0)
 168 #define ECC_RS_4BIT     BIT(1)
 169 #define ECC_BCH_4BIT    BIT(2)
 170 #define ECC_BCH_8BIT    BIT(3)
 171
 172 struct desc_info {
 173         struct list_head node;
 174
 175         enum dma_data_direction dir;
 176         struct scatterlist sgl;
 177         struct dma_async_tx_descriptor *dma_desc;
 178 };
 179
 180 /*
 181  * holds the current register values that we want to write. acts as a contiguous
 182  * chunk of memory which we use to write the controller registers through DMA.
 183  */
 184 struct nandc_regs {
 185         __le32 cmd;
 186         __le32 addr0;
 187         __le32 addr1;
 188         __le32 chip_sel;
 189         __le32 exec;
 190
 191         __le32 cfg0;
 192         __le32 cfg1;
 193         __le32 ecc_bch_cfg;
 194
 195         __le32 clrflashstatus;
 196         __le32 clrreadstatus;
 197
 198         __le32 cmd1;
 199         __le32 vld;
 200
 201         __le32 orig_cmd1;
 202         __le32 orig_vld;
 203
 204         __le32 ecc_buf_cfg;
 205 };
 206
 207 /*
 208  * NAND controller data struct
 209  *
 210  * @controller:                 base controller structure
 211  * @host_list:                  list containing all the chips attached to the
 212  *                              controller
 213  * @dev:                        parent device
 214  * @base:                       MMIO base
 215  * @base_dma:                   physical base address of controller registers
 216  * @core_clk:                   controller clock
 217  * @aon_clk:                    another controller clock
 218  *
 219  * @chan:                       dma channel
 220  * @cmd_crci:                   ADM DMA CRCI for command flow control
 221  * @data_crci:                  ADM DMA CRCI for data flow control
 222  * @desc_list:                  DMA descriptor list (list of desc_infos)
 223  *
 224  * @data_buffer:                our local DMA buffer for page read/writes,
 225  *                              used when we can't use the buffer provided
 226  *                              by upper layers directly
 227  * @buf_size/count/start:       markers for chip->read_buf/write_buf functions
 228  * @reg_read_buf:               local buffer for reading back registers via DMA
 229  * @reg_read_pos:               marker for data read in reg_read_buf
 230  *
 231  * @regs:                       a contiguous chunk of memory for DMA register
 232  *                              writes. contains the register values to be
 233  *                              written to controller
 234  * @cmd1/vld:                   some fixed controller register values
 235  * @ecc_modes:                  supported ECC modes by the current controller,
 236  *                              initialized via DT match data
 237  */
 238 struct qcom_nand_controller {
 239         struct nand_hw_control controller;
 240         struct list_head host_list;
 241
 242         struct device *dev;
 243
 244         void __iomem *base;
 245         dma_addr_t base_dma;
 246
 247         struct clk *core_clk;
 248         struct clk *aon_clk;
 249
 250         struct dma_chan *chan;
 251         unsigned int cmd_crci;
 252         unsigned int data_crci;
 253         struct list_head desc_list;
 254
 255         u8              *data_buffer;
 256         int             buf_size;
 257         int             buf_count;
 258         int             buf_start;
 259
 260         __le32 *reg_read_buf;
 261         int reg_read_pos;
 262
 263         struct nandc_regs *regs;
 264
 265         u32 cmd1, vld;
 266         u32 ecc_modes;
 267 };
 268
 269 /*
 270  * NAND chip structure
 271  *
 272  * @chip:                       base NAND chip structure
 273  * @node:                       list node to add itself to host_list in
 274  *                              qcom_nand_controller
 275  *
 276  * @cs:                         chip select value for this chip
 277  * @cw_size:                    the number of bytes in a single step/codeword
 278  *                              of a page, consisting of all data, ecc, spare
 279  *                              and reserved bytes
 280  * @cw_data:                    the number of bytes within a codeword protected
 281  *                              by ECC
 282  * @use_ecc:                    request the controller to use ECC for the
 283  *                              upcoming read/write
 284  * @bch_enabled:                flag to tell whether BCH ECC mode is used
 285  * @ecc_bytes_hw:               ECC bytes used by controller hardware for this
 286  *                              chip
 287  * @status:                     value to be returned if NAND_CMD_STATUS command
 288  *                              is executed
 289  * @last_command:               keeps track of last command on this chip. used
 290  *                              for reading correct status
 291  *
 292  * @cfg0, cfg1, cfg0_raw..:     NANDc register configurations needed for
 293  *                              ecc/non-ecc mode for the current nand flash
 294  *                              device
 295  */
 296 struct qcom_nand_host {
 297         struct nand_chip chip;
 298         struct list_head node;
 299
 300         int cs;
 301         int cw_size;
 302         int cw_data;
 303         bool use_ecc;
 304         bool bch_enabled;
 305         int ecc_bytes_hw;
 306         int spare_bytes;
 307         int bbm_size;
 308         u8 status;
 309         int last_command;
 310
 311         u32 cfg0, cfg1;
 312         u32 cfg0_raw, cfg1_raw;
 313         u32 ecc_buf_cfg;
 314         u32 ecc_bch_cfg;
 315         u32 clrflashstatus;
 316         u32 clrreadstatus;
 317 };
 318
 319 static inline struct qcom_nand_host *to_qcom_nand_host(struct nand_chip *chip)
 320 {
 321         return container_of(chip, struct qcom_nand_host, chip);
 322 }
 323
 324 static inline struct qcom_nand_controller *
 325 get_qcom_nand_controller(struct nand_chip *chip)
 326 {
 327         return container_of(chip->controller, struct qcom_nand_controller,
 328                             controller);
 329 }
 330
 331 static inline u32 nandc_read(struct qcom_nand_controller *nandc, int offset)
 332 {
 333         return ioread32(nandc->base + offset);
 334 }
 335
 336 static inline void nandc_write(struct qcom_nand_controller *nandc, int offset,
 337                                u32 val)
 338 {
 339         iowrite32(val, nandc->base + offset);
 340 }
 341
 342 static __le32 *offset_to_nandc_reg(struct nandc_regs *regs, int offset)
 343 {
 344         switch (offset) {
 345         case NAND_FLASH_CMD:
 346                 return &regs->cmd;
 347         case NAND_ADDR0:
 348                 return &regs->addr0;
 349         case NAND_ADDR1:
 350                 return &regs->addr1;
 351         case NAND_FLASH_CHIP_SELECT:
 352                 return &regs->chip_sel;
 353         case NAND_EXEC_CMD:
 354                 return &regs->exec;
 355         case NAND_FLASH_STATUS:
 356                 return &regs->clrflashstatus;
 357         case NAND_DEV0_CFG0:
 358                 return &regs->cfg0;
 359         case NAND_DEV0_CFG1:
 360                 return &regs->cfg1;
 361         case NAND_DEV0_ECC_CFG:
 362                 return &regs->ecc_bch_cfg;
 363         case NAND_READ_STATUS:
 364                 return &regs->clrreadstatus;
 365         case NAND_DEV_CMD1:
 366                 return &regs->cmd1;
 367         case NAND_DEV_CMD1_RESTORE:
 368                 return &regs->orig_cmd1;
 369         case NAND_DEV_CMD_VLD:
 370                 return &regs->vld;
 371         case NAND_DEV_CMD_VLD_RESTORE:
 372                 return &regs->orig_vld;
 373         case NAND_EBI2_ECC_BUF_CFG:
 374                 return &regs->ecc_buf_cfg;
 375         default:
 376                 return NULL;
 377         }
 378 }
 379
 380 static void nandc_set_reg(struct qcom_nand_controller *nandc, int offset,
 381                           u32 val)
 382 {
 383         struct nandc_regs *regs = nandc->regs;
 384         __le32 *reg;
 385
 386         reg = offset_to_nandc_reg(regs, offset);
 387
 388         if (reg)
 389                 *reg = cpu_to_le32(val);
 390 }
 391
 392 /* helper to configure address register values */
 393 static void set_address(struct qcom_nand_host *host, u16 column, int page)
 394 {
 395         struct nand_chip *chip = &host->chip;
 396         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
 397
 398         if (chip->options & NAND_BUSWIDTH_16)
 399                 column >>= 1;
 400
 401         nandc_set_reg(nandc, NAND_ADDR0, page << 16 | column);
 402         nandc_set_reg(nandc, NAND_ADDR1, page >> 16 & 0xff);
 403 }
 404
 405 /*
 406  * update_rw_regs:      set up read/write register values, these will be
 407  *                      written to the NAND controller registers via DMA
 408  *
 409  * @num_cw:             number of steps for the read/write operation
 410  * @read:               read or write operation
 411  */
 412 static void update_rw_regs(struct qcom_nand_host *host, int num_cw, bool read)
 413 {
 414         struct nand_chip *chip = &host->chip;
 415         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
 416         u32 cmd, cfg0, cfg1, ecc_bch_cfg;
 417
 418         if (read) {
 419                 if (host->use_ecc)
 420                         cmd = PAGE_READ_WITH_ECC | PAGE_ACC | LAST_PAGE;
 421                 else
 422                         cmd = PAGE_READ | PAGE_ACC | LAST_PAGE;
 423         } else {
 424                         cmd = PROGRAM_PAGE | PAGE_ACC | LAST_PAGE;
 425         }
 426
 427         if (host->use_ecc) {
 428                 cfg0 = (host->cfg0 & ~(7U << CW_PER_PAGE)) |
 429                                 (num_cw - 1) << CW_PER_PAGE;
 430
 431                 cfg1 = host->cfg1;
 432                 ecc_bch_cfg = host->ecc_bch_cfg;
 433         } else {
 434                 cfg0 = (host->cfg0_raw & ~(7U << CW_PER_PAGE)) |
 435                                 (num_cw - 1) << CW_PER_PAGE;
 436
 437                 cfg1 = host->cfg1_raw;
 438                 ecc_bch_cfg = 1 << ECC_CFG_ECC_DISABLE;
 439         }
 440
 441         nandc_set_reg(nandc, NAND_FLASH_CMD, cmd);
 442         nandc_set_reg(nandc, NAND_DEV0_CFG0, cfg0);
 443         nandc_set_reg(nandc, NAND_DEV0_CFG1, cfg1);
 444         nandc_set_reg(nandc, NAND_DEV0_ECC_CFG, ecc_bch_cfg);
 445         nandc_set_reg(nandc, NAND_EBI2_ECC_BUF_CFG, host->ecc_buf_cfg);
 446         nandc_set_reg(nandc, NAND_FLASH_STATUS, host->clrflashstatus);
 447         nandc_set_reg(nandc, NAND_READ_STATUS, host->clrreadstatus);
 448         nandc_set_reg(nandc, NAND_EXEC_CMD, 1);
 449 }
 450
 451 static int prep_dma_desc(struct qcom_nand_controller *nandc, bool read,
 452                          int reg_off, const void *vaddr, int size,
 453                          bool flow_control)
 454 {
 455         struct desc_info *desc;
 456         struct dma_async_tx_descriptor *dma_desc;
 457         struct scatterlist *sgl;
 458         struct dma_slave_config slave_conf;
 459         enum dma_transfer_direction dir_eng;
 460         int ret;
 461
 462         desc = kzalloc(sizeof(*desc), GFP_KERNEL);
 463         if (!desc)
 464                 return -ENOMEM;
 465
 466         sgl = &desc->sgl;
 467
 468         sg_init_one(sgl, vaddr, size);
 469
 470         if (read) {
 471                 dir_eng = DMA_DEV_TO_MEM;
 472                 desc->dir = DMA_FROM_DEVICE;
 473         } else {
 474                 dir_eng = DMA_MEM_TO_DEV;
 475                 desc->dir = DMA_TO_DEVICE;
 476         }
 477
 478         ret = dma_map_sg(nandc->dev, sgl, 1, desc->dir);
 479         if (ret == 0) {
 480                 ret = -ENOMEM;
 481                 goto err;
 482         }
 483
 484         memset(&slave_conf, 0x00, sizeof(slave_conf));
 485
 486         slave_conf.device_fc = flow_control;
 487         if (read) {
 488                 slave_conf.src_maxburst = 16;
 489                 slave_conf.src_addr = nandc->base_dma + reg_off;
 490                 slave_conf.slave_id = nandc->data_crci;
 491         } else {
 492                 slave_conf.dst_maxburst = 16;
 493                 slave_conf.dst_addr = nandc->base_dma + reg_off;
 494                 slave_conf.slave_id = nandc->cmd_crci;
 495         }
 496
 497         ret = dmaengine_slave_config(nandc->chan, &slave_conf);
 498         if (ret) {
 499                 dev_err(nandc->dev, "failed to configure dma channel\n");
 500                 goto err;
 501         }
 502
 503         dma_desc = dmaengine_prep_slave_sg(nandc->chan, sgl, 1, dir_eng, 0);
 504         if (!dma_desc) {
 505                 dev_err(nandc->dev, "failed to prepare desc\n");
 506                 ret = -EINVAL;
 507                 goto err;
 508         }
 509
 510         desc->dma_desc = dma_desc;
 511
 512         list_add_tail(&desc->node, &nandc->desc_list);
 513
 514         return 0;
 515 err:
 516         kfree(desc);
 517
 518         return ret;
 519 }
 520
 521 /*
 522  * read_reg_dma:        prepares a descriptor to read a given number of
 523  *                      contiguous registers to the reg_read_buf pointer
 524  *
 525  * @first:              offset of the first register in the contiguous block
 526  * @num_regs:           number of registers to read
 527  */
 528 static int read_reg_dma(struct qcom_nand_controller *nandc, int first,
 529                         int num_regs)
 530 {
 531         bool flow_control = false;
 532         void *vaddr;
 533         int size;
 534
 535         if (first == NAND_READ_ID || first == NAND_FLASH_STATUS)
 536                 flow_control = true;
 537
 538         size = num_regs * sizeof(u32);
 539         vaddr = nandc->reg_read_buf + nandc->reg_read_pos;
 540         nandc->reg_read_pos += num_regs;
 541
 542         return prep_dma_desc(nandc, true, first, vaddr, size, flow_control);
 543 }
 544
 545 /*
 546  * write_reg_dma:       prepares a descriptor to write a given number of
 547  *                      contiguous registers
 548  *
 549  * @first:              offset of the first register in the contiguous block
 550  * @num_regs:           number of registers to write
 551  */
 552 static int write_reg_dma(struct qcom_nand_controller *nandc, int first,
 553                          int num_regs)
 554 {
 555         bool flow_control = false;
 556         struct nandc_regs *regs = nandc->regs;
 557         void *vaddr;
 558         int size;
 559
 560         vaddr = offset_to_nandc_reg(regs, first);
 561
 562         if (first == NAND_FLASH_CMD)
 563                 flow_control = true;
 564
 565         if (first == NAND_DEV_CMD1_RESTORE)
 566                 first = NAND_DEV_CMD1;
 567
 568         if (first == NAND_DEV_CMD_VLD_RESTORE)
 569                 first = NAND_DEV_CMD_VLD;
 570
 571         size = num_regs * sizeof(u32);
 572
 573         return prep_dma_desc(nandc, false, first, vaddr, size, flow_control);
 574 }
 575
 576 /*
 577  * read_data_dma:       prepares a DMA descriptor to transfer data from the
 578  *                      controller's internal buffer to the buffer 'vaddr'
 579  *
 580  * @reg_off:            offset within the controller's data buffer
 581  * @vaddr:              virtual address of the buffer we want to write to
 582  * @size:               DMA transaction size in bytes
 583  */
 584 static int read_data_dma(struct qcom_nand_controller *nandc, int reg_off,
 585                          const u8 *vaddr, int size)
 586 {
 587         return prep_dma_desc(nandc, true, reg_off, vaddr, size, false);
 588 }
 589
 590 /*
 591  * write_data_dma:      prepares a DMA descriptor to transfer data from
 592  *                      'vaddr' to the controller's internal buffer
 593  *
 594  * @reg_off:            offset within the controller's data buffer
 595  * @vaddr:              virtual address of the buffer we want to read from
 596  * @size:               DMA transaction size in bytes
 597  */
 598 static int write_data_dma(struct qcom_nand_controller *nandc, int reg_off,
 599                           const u8 *vaddr, int size)
 600 {
 601         return prep_dma_desc(nandc, false, reg_off, vaddr, size, false);
 602 }
 603
 604 /*
 605  * helper to prepare dma descriptors to configure registers needed for reading a
 606  * codeword/step in a page
 607  */
 608 static void config_cw_read(struct qcom_nand_controller *nandc)
 609 {
 610         write_reg_dma(nandc, NAND_FLASH_CMD, 3);
 611         write_reg_dma(nandc, NAND_DEV0_CFG0, 3);
 612         write_reg_dma(nandc, NAND_EBI2_ECC_BUF_CFG, 1);
 613
 614         write_reg_dma(nandc, NAND_EXEC_CMD, 1);
 615
 616         read_reg_dma(nandc, NAND_FLASH_STATUS, 2);
 617         read_reg_dma(nandc, NAND_ERASED_CW_DETECT_STATUS, 1);
 618 }
 619
 620 /*
 621  * helpers to prepare dma descriptors used to configure registers needed for
 622  * writing a codeword/step in a page
 623  */
 624 static void config_cw_write_pre(struct qcom_nand_controller *nandc)
 625 {
 626         write_reg_dma(nandc, NAND_FLASH_CMD, 3);
 627         write_reg_dma(nandc, NAND_DEV0_CFG0, 3);
 628         write_reg_dma(nandc, NAND_EBI2_ECC_BUF_CFG, 1);
 629 }
 630
 631 static void config_cw_write_post(struct qcom_nand_controller *nandc)
 632 {
 633         write_reg_dma(nandc, NAND_EXEC_CMD, 1);
 634
 635         read_reg_dma(nandc, NAND_FLASH_STATUS, 1);
 636
 637         write_reg_dma(nandc, NAND_FLASH_STATUS, 1);
 638         write_reg_dma(nandc, NAND_READ_STATUS, 1);
 639 }
 640
 641 /*
 642  * the following functions are used within chip->cmdfunc() to perform different
 643  * NAND_CMD_* commands
 644  */
 645
 646 /* sets up descriptors for NAND_CMD_PARAM */
 647 static int nandc_param(struct qcom_nand_host *host)
 648 {
 649         struct nand_chip *chip = &host->chip;
 650         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
 651
 652         /*
 653          * NAND_CMD_PARAM is called before we know much about the FLASH chip
 654          * in use. we configure the controller to perform a raw read of 512
 655          * bytes to read onfi params
 656          */
 657         nandc_set_reg(nandc, NAND_FLASH_CMD, PAGE_READ | PAGE_ACC | LAST_PAGE);
 658         nandc_set_reg(nandc, NAND_ADDR0, 0);
 659         nandc_set_reg(nandc, NAND_ADDR1, 0);
 660         nandc_set_reg(nandc, NAND_DEV0_CFG0, 0 << CW_PER_PAGE
 661                                         | 512 << UD_SIZE_BYTES
 662                                         | 5 << NUM_ADDR_CYCLES
 663                                         | 0 << SPARE_SIZE_BYTES);
 664         nandc_set_reg(nandc, NAND_DEV0_CFG1, 7 << NAND_RECOVERY_CYCLES
 665                                         | 0 << CS_ACTIVE_BSY
 666                                         | 17 << BAD_BLOCK_BYTE_NUM
 667                                         | 1 << BAD_BLOCK_IN_SPARE_AREA
 668                                         | 2 << WR_RD_BSY_GAP
 669                                         | 0 << WIDE_FLASH
 670                                         | 1 << DEV0_CFG1_ECC_DISABLE);
 671         nandc_set_reg(nandc, NAND_EBI2_ECC_BUF_CFG, 1 << ECC_CFG_ECC_DISABLE);
 672
 673         /* configure CMD1 and VLD for ONFI param probing */
 674         nandc_set_reg(nandc, NAND_DEV_CMD_VLD,
 675                       (nandc->vld & ~(1 << READ_START_VLD))
 676                       | 0 << READ_START_VLD);
 677         nandc_set_reg(nandc, NAND_DEV_CMD1,
 678                       (nandc->cmd1 & ~(0xFF << READ_ADDR))
 679                       | NAND_CMD_PARAM << READ_ADDR);
 680
 681         nandc_set_reg(nandc, NAND_EXEC_CMD, 1);
 682
 683         nandc_set_reg(nandc, NAND_DEV_CMD1_RESTORE, nandc->cmd1);
 684         nandc_set_reg(nandc, NAND_DEV_CMD_VLD_RESTORE, nandc->vld);
 685
 686         write_reg_dma(nandc, NAND_DEV_CMD_VLD, 1);
 687         write_reg_dma(nandc, NAND_DEV_CMD1, 1);
 688
 689         nandc->buf_count = 512;
 690         memset(nandc->data_buffer, 0xff, nandc->buf_count);
 691
 692         config_cw_read(nandc);
 693
 694         read_data_dma(nandc, FLASH_BUF_ACC, nandc->data_buffer,
 695                       nandc->buf_count);
 696
 697         /* restore CMD1 and VLD regs */
 698         write_reg_dma(nandc, NAND_DEV_CMD1_RESTORE, 1);
 699         write_reg_dma(nandc, NAND_DEV_CMD_VLD_RESTORE, 1);
 700
 701         return 0;
 702 }
 703
 704 /* sets up descriptors for NAND_CMD_ERASE1 */
 705 static int erase_block(struct qcom_nand_host *host, int page_addr)
 706 {
 707         struct nand_chip *chip = &host->chip;
 708         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
 709
 710         nandc_set_reg(nandc, NAND_FLASH_CMD,
 711                       BLOCK_ERASE | PAGE_ACC | LAST_PAGE);
 712         nandc_set_reg(nandc, NAND_ADDR0, page_addr);
 713         nandc_set_reg(nandc, NAND_ADDR1, 0);
 714         nandc_set_reg(nandc, NAND_DEV0_CFG0,
 715                       host->cfg0_raw & ~(7 << CW_PER_PAGE));
 716         nandc_set_reg(nandc, NAND_DEV0_CFG1, host->cfg1_raw);
 717         nandc_set_reg(nandc, NAND_EXEC_CMD, 1);
 718         nandc_set_reg(nandc, NAND_FLASH_STATUS, host->clrflashstatus);
 719         nandc_set_reg(nandc, NAND_READ_STATUS, host->clrreadstatus);
 720
 721         write_reg_dma(nandc, NAND_FLASH_CMD, 3);
 722         write_reg_dma(nandc, NAND_DEV0_CFG0, 2);
 723         write_reg_dma(nandc, NAND_EXEC_CMD, 1);
 724
 725         read_reg_dma(nandc, NAND_FLASH_STATUS, 1);
 726
 727         write_reg_dma(nandc, NAND_FLASH_STATUS, 1);
 728         write_reg_dma(nandc, NAND_READ_STATUS, 1);
 729
 730         return 0;
 731 }
 732
 733 /* sets up descriptors for NAND_CMD_READID */
 734 static int read_id(struct qcom_nand_host *host, int column)
 735 {
 736         struct nand_chip *chip = &host->chip;
 737         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
 738
 739         if (column == -1)
 740                 return 0;
 741
 742         nandc_set_reg(nandc, NAND_FLASH_CMD, FETCH_ID);
 743         nandc_set_reg(nandc, NAND_ADDR0, column);
 744         nandc_set_reg(nandc, NAND_ADDR1, 0);
 745         nandc_set_reg(nandc, NAND_FLASH_CHIP_SELECT, DM_EN);
 746         nandc_set_reg(nandc, NAND_EXEC_CMD, 1);
 747
 748         write_reg_dma(nandc, NAND_FLASH_CMD, 4);
 749         write_reg_dma(nandc, NAND_EXEC_CMD, 1);
 750
 751         read_reg_dma(nandc, NAND_READ_ID, 1);
 752
 753         return 0;
 754 }
 755
 756 /* sets up descriptors for NAND_CMD_RESET */
 757 static int reset(struct qcom_nand_host *host)
 758 {
 759         struct nand_chip *chip = &host->chip;
 760         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
 761
 762         nandc_set_reg(nandc, NAND_FLASH_CMD, RESET_DEVICE);
 763         nandc_set_reg(nandc, NAND_EXEC_CMD, 1);
 764
 765         write_reg_dma(nandc, NAND_FLASH_CMD, 1);
 766         write_reg_dma(nandc, NAND_EXEC_CMD, 1);
 767
 768         read_reg_dma(nandc, NAND_FLASH_STATUS, 1);
 769
 770         return 0;
 771 }
 772
 773 /* helpers to submit/free our list of dma descriptors */
 774 static int submit_descs(struct qcom_nand_controller *nandc)
 775 {
 776         struct desc_info *desc;
 777         dma_cookie_t cookie = 0;
 778
 779         list_for_each_entry(desc, &nandc->desc_list, node)
 780                 cookie = dmaengine_submit(desc->dma_desc);
 781
 782         if (dma_sync_wait(nandc->chan, cookie) != DMA_COMPLETE)
 783                 return -ETIMEDOUT;
 784
 785         return 0;
 786 }
 787
 788 static void free_descs(struct qcom_nand_controller *nandc)
 789 {
 790         struct desc_info *desc, *n;
 791
 792         list_for_each_entry_safe(desc, n, &nandc->desc_list, node) {
 793                 list_del(&desc->node);
 794                 dma_unmap_sg(nandc->dev, &desc->sgl, 1, desc->dir);
 795                 kfree(desc);
 796         }
 797 }
 798
 799 /* reset the register read buffer for next NAND operation */
 800 static void clear_read_regs(struct qcom_nand_controller *nandc)
 801 {
 802         nandc->reg_read_pos = 0;
 803         memset(nandc->reg_read_buf, 0,
 804                MAX_REG_RD * sizeof(*nandc->reg_read_buf));
 805 }
 806
 807 static void pre_command(struct qcom_nand_host *host, int command)
 808 {
 809         struct nand_chip *chip = &host->chip;
 810         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
 811
 812         nandc->buf_count = 0;
 813         nandc->buf_start = 0;
 814         host->use_ecc = false;
 815         host->last_command = command;
 816
 817         clear_read_regs(nandc);
 818 }
 819
 820 /*
 821  * this is called after NAND_CMD_PAGEPROG and NAND_CMD_ERASE1 to set our
 822  * privately maintained status byte, this status byte can be read after
 823  * NAND_CMD_STATUS is called
 824  */
 825 static void parse_erase_write_errors(struct qcom_nand_host *host, int command)
 826 {
 827         struct nand_chip *chip = &host->chip;
 828         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
 829         struct nand_ecc_ctrl *ecc = &chip->ecc;
 830         int num_cw;
 831         int i;
 832
 833         num_cw = command == NAND_CMD_PAGEPROG ? ecc->steps : 1;
 834
 835         for (i = 0; i < num_cw; i++) {
 836                 u32 flash_status = le32_to_cpu(nandc->reg_read_buf[i]);
 837
 838                 if (flash_status & FS_MPU_ERR)
 839                         host->status &= ~NAND_STATUS_WP;
 840
 841                 if (flash_status & FS_OP_ERR || (i == (num_cw - 1) &&
 842                                                  (flash_status &
 843                                                   FS_DEVICE_STS_ERR)))
 844                         host->status |= NAND_STATUS_FAIL;
 845         }
 846 }
 847
 848 static void post_command(struct qcom_nand_host *host, int command)
 849 {
 850         struct nand_chip *chip = &host->chip;
 851         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
 852
 853         switch (command) {
 854         case NAND_CMD_READID:
 855                 memcpy(nandc->data_buffer, nandc->reg_read_buf,
 856                        nandc->buf_count);
 857                 break;
 858         case NAND_CMD_PAGEPROG:
 859         case NAND_CMD_ERASE1:
 860                 parse_erase_write_errors(host, command);
 861                 break;
 862         default:
 863                 break;
 864         }
 865 }
 866
 867 /*
 868  * Implements chip->cmdfunc. It's  only used for a limited set of commands.
 869  * The rest of the commands wouldn't be called by upper layers. For example,
 870  * NAND_CMD_READOOB would never be called because we have our own versions
 871  * of read_oob ops for nand_ecc_ctrl.
 872  */
 873 static void qcom_nandc_command(struct mtd_info *mtd, unsigned int command,
 874                                int column, int page_addr)
 875 {
 876         struct nand_chip *chip = mtd_to_nand(mtd);
 877         struct qcom_nand_host *host = to_qcom_nand_host(chip);
 878         struct nand_ecc_ctrl *ecc = &chip->ecc;
 879         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
 880         bool wait = false;
 881         int ret = 0;
 882
 883         pre_command(host, command);
 884
 885         switch (command) {
 886         case NAND_CMD_RESET:
 887                 ret = reset(host);
 888                 wait = true;
 889                 break;
 890
 891         case NAND_CMD_READID:
 892                 nandc->buf_count = 4;
 893                 ret = read_id(host, column);
 894                 wait = true;
 895                 break;
 896
 897         case NAND_CMD_PARAM:
 898                 ret = nandc_param(host);
 899                 wait = true;
 900                 break;
 901
 902         case NAND_CMD_ERASE1:
 903                 ret = erase_block(host, page_addr);
 904                 wait = true;
 905                 break;
 906
 907         case NAND_CMD_READ0:
 908                 /* we read the entire page for now */
 909                 WARN_ON(column != 0);
 910
 911                 host->use_ecc = true;
 912                 set_address(host, 0, page_addr);
 913                 update_rw_regs(host, ecc->steps, true);
 914                 break;
 915
 916         case NAND_CMD_SEQIN:
 917                 WARN_ON(column != 0);
 918                 set_address(host, 0, page_addr);
 919                 break;
 920
 921         case NAND_CMD_PAGEPROG:
 922         case NAND_CMD_STATUS:
 923         case NAND_CMD_NONE:
 924         default:
 925                 break;
 926         }
 927
 928         if (ret) {
 929                 dev_err(nandc->dev, "failure executing command %d\n",
 930                         command);
 931                 free_descs(nandc);
 932                 return;
 933         }
 934
 935         if (wait) {
 936                 ret = submit_descs(nandc);
 937                 if (ret)
 938                         dev_err(nandc->dev,
 939                                 "failure submitting descs for command %d\n",
 940                                 command);
 941         }
 942
 943         free_descs(nandc);
 944
 945         post_command(host, command);
 946 }
 947
 948 /*
 949  * when using BCH ECC, the HW flags an error in NAND_FLASH_STATUS if it read
 950  * an erased CW, and reports an erased CW in NAND_ERASED_CW_DETECT_STATUS.
 951  *
 952  * when using RS ECC, the HW reports the same erros when reading an erased CW,
 953  * but it notifies that it is an erased CW by placing special characters at
 954  * certain offsets in the buffer.
 955  *
 956  * verify if the page is erased or not, and fix up the page for RS ECC by
 957  * replacing the special characters with 0xff.
 958  */
 959 static bool erased_chunk_check_and_fixup(u8 *data_buf, int data_len)
 960 {
 961         u8 empty1, empty2;
 962
 963         /*
 964          * an erased page flags an error in NAND_FLASH_STATUS, check if the page
 965          * is erased by looking for 0x54s at offsets 3 and 175 from the
 966          * beginning of each codeword
 967          */
 968
 969         empty1 = data_buf[3];
 970         empty2 = data_buf[175];
 971
 972         /*
 973          * if the erased codework markers, if they exist override them with
 974          * 0xffs
 975          */
 976         if ((empty1 == 0x54 && empty2 == 0xff) ||
 977             (empty1 == 0xff && empty2 == 0x54)) {
 978                 data_buf[3] = 0xff;
 979                 data_buf[175] = 0xff;
 980         }
 981
 982         /*
 983          * check if the entire chunk contains 0xffs or not. if it doesn't, then
 984          * restore the original values at the special offsets
 985          */
 986         if (memchr_inv(data_buf, 0xff, data_len)) {
 987                 data_buf[3] = empty1;
 988                 data_buf[175] = empty2;
 989
 990                 return false;
 991         }
 992
 993         return true;
 994 }
 995
 996 struct read_stats {
 997         __le32 flash;
 998         __le32 buffer;
 999         __le32 erased_cw;
1000 };
1001
1002 /*
1003  * reads back status registers set by the controller to notify page read
1004  * errors. this is equivalent to what 'ecc->correct()' would do.
1005  */
1006 static int parse_read_errors(struct qcom_nand_host *host, u8 *data_buf,
1007                              u8 *oob_buf)
1008 {
1009         struct nand_chip *chip = &host->chip;
1010         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1011         struct mtd_info *mtd = nand_to_mtd(chip);
1012         struct nand_ecc_ctrl *ecc = &chip->ecc;
1013         unsigned int max_bitflips = 0;
1014         struct read_stats *buf;
1015         int i;
1016
1017         buf = (struct read_stats *)nandc->reg_read_buf;
1018
1019         for (i = 0; i < ecc->steps; i++, buf++) {
1020                 u32 flash, buffer, erased_cw;
1021                 int data_len, oob_len;
1022
1023                 if (i == (ecc->steps - 1)) {
1024                         data_len = ecc->size - ((ecc->steps - 1) << 2);
1025                         oob_len = ecc->steps << 2;
1026                 } else {
1027                         data_len = host->cw_data;
1028                         oob_len = 0;
1029                 }
1030
1031                 flash = le32_to_cpu(buf->flash);
1032                 buffer = le32_to_cpu(buf->buffer);
1033                 erased_cw = le32_to_cpu(buf->erased_cw);
1034
1035                 if (flash & (FS_OP_ERR | FS_MPU_ERR)) {
1036                         bool erased;
1037
1038                         /* ignore erased codeword errors */
1039                         if (host->bch_enabled) {
1040                                 erased = (erased_cw & ERASED_CW) == ERASED_CW ?
1041                                          true : false;
1042                         } else {
1043                                 erased = erased_chunk_check_and_fixup(data_buf,
1044                                                                       data_len);
1045                         }
1046
1047                         if (erased) {
1048                                 data_buf += data_len;
1049                                 if (oob_buf)
1050                                         oob_buf += oob_len + ecc->bytes;
1051                                 continue;
1052                         }
1053
1054                         if (buffer & BS_UNCORRECTABLE_BIT) {
1055                                 int ret, ecclen, extraooblen;
1056                                 void *eccbuf;
1057
1058                                 eccbuf = oob_buf ? oob_buf + oob_len : NULL;
1059                                 ecclen = oob_buf ? host->ecc_bytes_hw : 0;
1060                                 extraooblen = oob_buf ? oob_len : 0;
1061
1062                                 /*
1063                                  * make sure it isn't an erased page reported
1064                                  * as not-erased by HW because of a few bitflips
1065                                  */
1066                                 ret = nand_check_erased_ecc_chunk(data_buf,
1067                                         data_len, eccbuf, ecclen, oob_buf,
1068                                         extraooblen, ecc->strength);
1069                                 if (ret < 0) {
1070                                         mtd->ecc_stats.failed++;
1071                                 } else {
1072                                         mtd->ecc_stats.corrected += ret;
1073                                         max_bitflips =
1074                                                 max_t(unsigned int, max_bitflips, ret);
1075                                 }
1076                         }
1077                 } else {
1078                         unsigned int stat;
1079
1080                         stat = buffer & BS_CORRECTABLE_ERR_MSK;
1081                         mtd->ecc_stats.corrected += stat;
1082                         max_bitflips = max(max_bitflips, stat);
1083                 }
1084
1085                 data_buf += data_len;
1086                 if (oob_buf)
1087                         oob_buf += oob_len + ecc->bytes;
1088         }
1089
1090         return max_bitflips;
1091 }
1092
1093 /*
1094  * helper to perform the actual page read operation, used by ecc->read_page(),
1095  * ecc->read_oob()
1096  */
1097 static int read_page_ecc(struct qcom_nand_host *host, u8 *data_buf,
1098                          u8 *oob_buf)
1099 {
1100         struct nand_chip *chip = &host->chip;
1101         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1102         struct nand_ecc_ctrl *ecc = &chip->ecc;
1103         int i, ret;
1104
1105         /* queue cmd descs for each codeword */
1106         for (i = 0; i < ecc->steps; i++) {
1107                 int data_size, oob_size;
1108
1109                 if (i == (ecc->steps - 1)) {
1110                         data_size = ecc->size - ((ecc->steps - 1) << 2);
1111                         oob_size = (ecc->steps << 2) + host->ecc_bytes_hw +
1112                                    host->spare_bytes;
1113                 } else {
1114                         data_size = host->cw_data;
1115                         oob_size = host->ecc_bytes_hw + host->spare_bytes;
1116                 }
1117
1118                 config_cw_read(nandc);
1119
1120                 if (data_buf)
1121                         read_data_dma(nandc, FLASH_BUF_ACC, data_buf,
1122                                       data_size);
1123
1124                 /*
1125                  * when ecc is enabled, the controller doesn't read the real
1126                  * or dummy bad block markers in each chunk. To maintain a
1127                  * consistent layout across RAW and ECC reads, we just
1128                  * leave the real/dummy BBM offsets empty (i.e, filled with
1129                  * 0xffs)
1130                  */
1131                 if (oob_buf) {
1132                         int j;
1133
1134                         for (j = 0; j < host->bbm_size; j++)
1135                                 *oob_buf++ = 0xff;
1136
1137                         read_data_dma(nandc, FLASH_BUF_ACC + data_size,
1138                                       oob_buf, oob_size);
1139                 }
1140
1141                 if (data_buf)
1142                         data_buf += data_size;
1143                 if (oob_buf)
1144                         oob_buf += oob_size;
1145         }
1146
1147         ret = submit_descs(nandc);
1148         if (ret)
1149                 dev_err(nandc->dev, "failure to read page/oob\n");
1150
1151         free_descs(nandc);
1152
1153         return ret;
1154 }
1155
1156 /*
1157  * a helper that copies the last step/codeword of a page (containing free oob)
1158  * into our local buffer
1159  */
1160 static int copy_last_cw(struct qcom_nand_host *host, int page)
1161 {
1162         struct nand_chip *chip = &host->chip;
1163         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1164         struct nand_ecc_ctrl *ecc = &chip->ecc;
1165         int size;
1166         int ret;
1167
1168         clear_read_regs(nandc);
1169
1170         size = host->use_ecc ? host->cw_data : host->cw_size;
1171
1172         /* prepare a clean read buffer */
1173         memset(nandc->data_buffer, 0xff, size);
1174
1175         set_address(host, host->cw_size * (ecc->steps - 1), page);
1176         update_rw_regs(host, 1, true);
1177
1178         config_cw_read(nandc);
1179
1180         read_data_dma(nandc, FLASH_BUF_ACC, nandc->data_buffer, size);
1181
1182         ret = submit_descs(nandc);
1183         if (ret)
1184                 dev_err(nandc->dev, "failed to copy last codeword\n");
1185
1186         free_descs(nandc);
1187
1188         return ret;
1189 }
1190
1191 /* implements ecc->read_page() */
1192 static int qcom_nandc_read_page(struct mtd_info *mtd, struct nand_chip *chip,
1193                                 uint8_t *buf, int oob_required, int page)
1194 {
1195         struct qcom_nand_host *host = to_qcom_nand_host(chip);
1196         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1197         u8 *data_buf, *oob_buf = NULL;
1198         int ret;
1199
1200         data_buf = buf;
1201         oob_buf = oob_required ? chip->oob_poi : NULL;
1202
1203         ret = read_page_ecc(host, data_buf, oob_buf);
1204         if (ret) {
1205                 dev_err(nandc->dev, "failure to read page\n");
1206                 return ret;
1207         }
1208
1209         return parse_read_errors(host, data_buf, oob_buf);
1210 }
1211
1212 /* implements ecc->read_page_raw() */
1213 static int qcom_nandc_read_page_raw(struct mtd_info *mtd,
1214                                     struct nand_chip *chip, uint8_t *buf,
1215                                     int oob_required, int page)
1216 {
1217         struct qcom_nand_host *host = to_qcom_nand_host(chip);
1218         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1219         u8 *data_buf, *oob_buf;
1220         struct nand_ecc_ctrl *ecc = &chip->ecc;
1221         int i, ret;
1222
1223         data_buf = buf;
1224         oob_buf = chip->oob_poi;
1225
1226         host->use_ecc = false;
1227         update_rw_regs(host, ecc->steps, true);
1228
1229         for (i = 0; i < ecc->steps; i++) {
1230                 int data_size1, data_size2, oob_size1, oob_size2;
1231                 int reg_off = FLASH_BUF_ACC;
1232
1233                 data_size1 = mtd->writesize - host->cw_size * (ecc->steps - 1);
1234                 oob_size1 = host->bbm_size;
1235
1236                 if (i == (ecc->steps - 1)) {
1237                         data_size2 = ecc->size - data_size1 -
1238                                      ((ecc->steps - 1) << 2);
1239                         oob_size2 = (ecc->steps << 2) + host->ecc_bytes_hw +
1240                                     host->spare_bytes;
1241                 } else {
1242                         data_size2 = host->cw_data - data_size1;
1243                         oob_size2 = host->ecc_bytes_hw + host->spare_bytes;
1244                 }
1245
1246                 config_cw_read(nandc);
1247
1248                 read_data_dma(nandc, reg_off, data_buf, data_size1);
1249                 reg_off += data_size1;
1250                 data_buf += data_size1;
1251
1252                 read_data_dma(nandc, reg_off, oob_buf, oob_size1);
1253                 reg_off += oob_size1;
1254                 oob_buf += oob_size1;
1255
1256                 read_data_dma(nandc, reg_off, data_buf, data_size2);
1257                 reg_off += data_size2;
1258                 data_buf += data_size2;
1259
1260                 read_data_dma(nandc, reg_off, oob_buf, oob_size2);
1261                 oob_buf += oob_size2;
1262         }
1263
1264         ret = submit_descs(nandc);
1265         if (ret)
1266                 dev_err(nandc->dev, "failure to read raw page\n");
1267
1268         free_descs(nandc);
1269
1270         return 0;
1271 }
1272
1273 /* implements ecc->read_oob() */
1274 static int qcom_nandc_read_oob(struct mtd_info *mtd, struct nand_chip *chip,
1275                                int page)
1276 {
1277         struct qcom_nand_host *host = to_qcom_nand_host(chip);
1278         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1279         struct nand_ecc_ctrl *ecc = &chip->ecc;
1280         int ret;
1281
1282         clear_read_regs(nandc);
1283
1284         host->use_ecc = true;
1285         set_address(host, 0, page);
1286         update_rw_regs(host, ecc->steps, true);
1287
1288         ret = read_page_ecc(host, NULL, chip->oob_poi);
1289         if (ret)
1290                 dev_err(nandc->dev, "failure to read oob\n");
1291
1292         return ret;
1293 }
1294
1295 /* implements ecc->write_page() */
1296 static int qcom_nandc_write_page(struct mtd_info *mtd, struct nand_chip *chip,
1297                                  const uint8_t *buf, int oob_required, int page)
1298 {
1299         struct qcom_nand_host *host = to_qcom_nand_host(chip);
1300         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1301         struct nand_ecc_ctrl *ecc = &chip->ecc;
1302         u8 *data_buf, *oob_buf;
1303         int i, ret;
1304
1305         clear_read_regs(nandc);
1306
1307         data_buf = (u8 *)buf;
1308         oob_buf = chip->oob_poi;
1309
1310         host->use_ecc = true;
1311         update_rw_regs(host, ecc->steps, false);
1312
1313         for (i = 0; i < ecc->steps; i++) {
1314                 int data_size, oob_size;
1315
1316                 if (i == (ecc->steps - 1)) {
1317                         data_size = ecc->size - ((ecc->steps - 1) << 2);
1318                         oob_size = (ecc->steps << 2) + host->ecc_bytes_hw +
1319                                    host->spare_bytes;
1320                 } else {
1321                         data_size = host->cw_data;
1322                         oob_size = ecc->bytes;
1323                 }
1324
1325                 config_cw_write_pre(nandc);
1326
1327                 write_data_dma(nandc, FLASH_BUF_ACC, data_buf, data_size);
1328
1329                 /*
1330                  * when ECC is enabled, we don't really need to write anything
1331                  * to oob for the first n - 1 codewords since these oob regions
1332                  * just contain ECC bytes that's written by the controller
1333                  * itself. For the last codeword, we skip the bbm positions and
1334                  * write to the free oob area.
1335                  */
1336                 if (i == (ecc->steps - 1)) {
1337                         oob_buf += host->bbm_size;
1338
1339                         write_data_dma(nandc, FLASH_BUF_ACC + data_size,
1340                                        oob_buf, oob_size);
1341                 }
1342
1343                 config_cw_write_post(nandc);
1344
1345                 data_buf += data_size;
1346                 oob_buf += oob_size;
1347         }
1348
1349         ret = submit_descs(nandc);
1350         if (ret)
1351                 dev_err(nandc->dev, "failure to write page\n");
1352
1353         free_descs(nandc);
1354
1355         return ret;
1356 }
1357
1358 /* implements ecc->write_page_raw() */
1359 static int qcom_nandc_write_page_raw(struct mtd_info *mtd,
1360                                      struct nand_chip *chip, const uint8_t *buf,
1361                                      int oob_required, int page)
1362 {
1363         struct qcom_nand_host *host = to_qcom_nand_host(chip);
1364         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1365         struct nand_ecc_ctrl *ecc = &chip->ecc;
1366         u8 *data_buf, *oob_buf;
1367         int i, ret;
1368
1369         clear_read_regs(nandc);
1370
1371         data_buf = (u8 *)buf;
1372         oob_buf = chip->oob_poi;
1373
1374         host->use_ecc = false;
1375         update_rw_regs(host, ecc->steps, false);
1376
1377         for (i = 0; i < ecc->steps; i++) {
1378                 int data_size1, data_size2, oob_size1, oob_size2;
1379                 int reg_off = FLASH_BUF_ACC;
1380
1381                 data_size1 = mtd->writesize - host->cw_size * (ecc->steps - 1);
1382                 oob_size1 = host->bbm_size;
1383
1384                 if (i == (ecc->steps - 1)) {
1385                         data_size2 = ecc->size - data_size1 -
1386                                      ((ecc->steps - 1) << 2);
1387                         oob_size2 = (ecc->steps << 2) + host->ecc_bytes_hw +
1388                                     host->spare_bytes;
1389                 } else {
1390                         data_size2 = host->cw_data - data_size1;
1391                         oob_size2 = host->ecc_bytes_hw + host->spare_bytes;
1392                 }
1393
1394                 config_cw_write_pre(nandc);
1395
1396                 write_data_dma(nandc, reg_off, data_buf, data_size1);
1397                 reg_off += data_size1;
1398                 data_buf += data_size1;
1399
1400                 write_data_dma(nandc, reg_off, oob_buf, oob_size1);
1401                 reg_off += oob_size1;
1402                 oob_buf += oob_size1;
1403
1404                 write_data_dma(nandc, reg_off, data_buf, data_size2);
1405                 reg_off += data_size2;
1406                 data_buf += data_size2;
1407
1408                 write_data_dma(nandc, reg_off, oob_buf, oob_size2);
1409                 oob_buf += oob_size2;
1410
1411                 config_cw_write_post(nandc);
1412         }
1413
1414         ret = submit_descs(nandc);
1415         if (ret)
1416                 dev_err(nandc->dev, "failure to write raw page\n");
1417
1418         free_descs(nandc);
1419
1420         return ret;
1421 }
1422
1423 /*
1424  * implements ecc->write_oob()
1425  *
1426  * the NAND controller cannot write only data or only oob within a codeword,
1427  * since ecc is calculated for the combined codeword. we first copy the
1428  * entire contents for the last codeword(data + oob), replace the old oob
1429  * with the new one in chip->oob_poi, and then write the entire codeword.
1430  * this read-copy-write operation results in a slight performance loss.
1431  */
1432 static int qcom_nandc_write_oob(struct mtd_info *mtd, struct nand_chip *chip,
1433                                 int page)
1434 {
1435         struct qcom_nand_host *host = to_qcom_nand_host(chip);
1436         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1437         struct nand_ecc_ctrl *ecc = &chip->ecc;
1438         u8 *oob = chip->oob_poi;
1439         int data_size, oob_size;
1440         int ret, status = 0;
1441
1442         host->use_ecc = true;
1443
1444         ret = copy_last_cw(host, page);
1445         if (ret)
1446                 return ret;
1447
1448         clear_read_regs(nandc);
1449
1450         /* calculate the data and oob size for the last codeword/step */
1451         data_size = ecc->size - ((ecc->steps - 1) << 2);
1452         oob_size = mtd->oobavail;
1453
1454         /* override new oob content to last codeword */
1455         mtd_ooblayout_get_databytes(mtd, nandc->data_buffer + data_size, oob,
1456                                     0, mtd->oobavail);
1457
1458         set_address(host, host->cw_size * (ecc->steps - 1), page);
1459         update_rw_regs(host, 1, false);
1460
1461         config_cw_write_pre(nandc);
1462         write_data_dma(nandc, FLASH_BUF_ACC, nandc->data_buffer,
1463                        data_size + oob_size);
1464         config_cw_write_post(nandc);
1465
1466         ret = submit_descs(nandc);
1467
1468         free_descs(nandc);
1469
1470         if (ret) {
1471                 dev_err(nandc->dev, "failure to write oob\n");
1472                 return -EIO;
1473         }
1474
1475         chip->cmdfunc(mtd, NAND_CMD_PAGEPROG, -1, -1);
1476
1477         status = chip->waitfunc(mtd, chip);
1478
1479         return status & NAND_STATUS_FAIL ? -EIO : 0;
1480 }
1481
1482 static int qcom_nandc_block_bad(struct mtd_info *mtd, loff_t ofs)
1483 {
1484         struct nand_chip *chip = mtd_to_nand(mtd);
1485         struct qcom_nand_host *host = to_qcom_nand_host(chip);
1486         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1487         struct nand_ecc_ctrl *ecc = &chip->ecc;
1488         int page, ret, bbpos, bad = 0;
1489         u32 flash_status;
1490
1491         page = (int)(ofs >> chip->page_shift) & chip->pagemask;
1492
1493         /*
1494          * configure registers for a raw sub page read, the address is set to
1495          * the beginning of the last codeword, we don't care about reading ecc
1496          * portion of oob. we just want the first few bytes from this codeword
1497          * that contains the BBM
1498          */
1499         host->use_ecc = false;
1500
1501         ret = copy_last_cw(host, page);
1502         if (ret)
1503                 goto err;
1504
1505         flash_status = le32_to_cpu(nandc->reg_read_buf[0]);
1506
1507         if (flash_status & (FS_OP_ERR | FS_MPU_ERR)) {
1508                 dev_warn(nandc->dev, "error when trying to read BBM\n");
1509                 goto err;
1510         }
1511
1512         bbpos = mtd->writesize - host->cw_size * (ecc->steps - 1);
1513
1514         bad = nandc->data_buffer[bbpos] != 0xff;
1515
1516         if (chip->options & NAND_BUSWIDTH_16)
1517                 bad = bad || (nandc->data_buffer[bbpos + 1] != 0xff);
1518 err:
1519         return bad;
1520 }
1521
1522 static int qcom_nandc_block_markbad(struct mtd_info *mtd, loff_t ofs)
1523 {
1524         struct nand_chip *chip = mtd_to_nand(mtd);
1525         struct qcom_nand_host *host = to_qcom_nand_host(chip);
1526         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1527         struct nand_ecc_ctrl *ecc = &chip->ecc;
1528         int page, ret, status = 0;
1529
1530         clear_read_regs(nandc);
1531
1532         /*
1533          * to mark the BBM as bad, we flash the entire last codeword with 0s.
1534          * we don't care about the rest of the content in the codeword since
1535          * we aren't going to use this block again
1536          */
1537         memset(nandc->data_buffer, 0x00, host->cw_size);
1538
1539         page = (int)(ofs >> chip->page_shift) & chip->pagemask;
1540
1541         /* prepare write */
1542         host->use_ecc = false;
1543         set_address(host, host->cw_size * (ecc->steps - 1), page);
1544         update_rw_regs(host, 1, false);
1545
1546         config_cw_write_pre(nandc);
1547         write_data_dma(nandc, FLASH_BUF_ACC, nandc->data_buffer, host->cw_size);
1548         config_cw_write_post(nandc);
1549
1550         ret = submit_descs(nandc);
1551
1552         free_descs(nandc);
1553
1554         if (ret) {
1555                 dev_err(nandc->dev, "failure to update BBM\n");
1556                 return -EIO;
1557         }
1558
1559         chip->cmdfunc(mtd, NAND_CMD_PAGEPROG, -1, -1);
1560
1561         status = chip->waitfunc(mtd, chip);
1562
1563         return status & NAND_STATUS_FAIL ? -EIO : 0;
1564 }
1565
1566 /*
1567  * the three functions below implement chip->read_byte(), chip->read_buf()
1568  * and chip->write_buf() respectively. these aren't used for
1569  * reading/writing page data, they are used for smaller data like reading
1570  * id, status etc
1571  */
1572 static uint8_t qcom_nandc_read_byte(struct mtd_info *mtd)
1573 {
1574         struct nand_chip *chip = mtd_to_nand(mtd);
1575         struct qcom_nand_host *host = to_qcom_nand_host(chip);
1576         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1577         u8 *buf = nandc->data_buffer;
1578         u8 ret = 0x0;
1579
1580         if (host->last_command == NAND_CMD_STATUS) {
1581                 ret = host->status;
1582
1583                 host->status = NAND_STATUS_READY | NAND_STATUS_WP;
1584
1585                 return ret;
1586         }
1587
1588         if (nandc->buf_start < nandc->buf_count)
1589                 ret = buf[nandc->buf_start++];
1590
1591         return ret;
1592 }
1593
1594 static void qcom_nandc_read_buf(struct mtd_info *mtd, uint8_t *buf, int len)
1595 {
1596         struct nand_chip *chip = mtd_to_nand(mtd);
1597         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1598         int real_len = min_t(size_t, len, nandc->buf_count - nandc->buf_start);
1599
1600         memcpy(buf, nandc->data_buffer + nandc->buf_start, real_len);
1601         nandc->buf_start += real_len;
1602 }
1603
1604 static void qcom_nandc_write_buf(struct mtd_info *mtd, const uint8_t *buf,
1605                                  int len)
1606 {
1607         struct nand_chip *chip = mtd_to_nand(mtd);
1608         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1609         int real_len = min_t(size_t, len, nandc->buf_count - nandc->buf_start);
1610
1611         memcpy(nandc->data_buffer + nandc->buf_start, buf, real_len);
1612
1613         nandc->buf_start += real_len;
1614 }
1615
1616 /* we support only one external chip for now */
1617 static void qcom_nandc_select_chip(struct mtd_info *mtd, int chipnr)
1618 {
1619         struct nand_chip *chip = mtd_to_nand(mtd);
1620         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1621
1622         if (chipnr <= 0)
1623                 return;
1624
1625         dev_warn(nandc->dev, "invalid chip select\n");
1626 }
1627
1628 /*
1629  * NAND controller page layout info
1630  *
1631  * Layout with ECC enabled:
1632  *
1633  * |----------------------|  |---------------------------------|
1634  * |           xx.......yy|  |             *********xx.......yy|
1635  * |    DATA   xx..ECC..yy|  |    DATA     **SPARE**xx..ECC..yy|
1636  * |   (516)   xx.......yy|  |  (516-n*4)  **(n*4)**xx.......yy|
1637  * |           xx.......yy|  |             *********xx.......yy|
1638  * |----------------------|  |---------------------------------|
1639  *     codeword 1,2..n-1                  codeword n
1640  *  <---(528/532 Bytes)-->    <-------(528/532 Bytes)--------->
1641  *
1642  * n = Number of codewords in the page
1643  * . = ECC bytes
1644  * * = Spare/free bytes
1645  * x = Unused byte(s)
1646  * y = Reserved byte(s)
1647  *
1648  * 2K page: n = 4, spare = 16 bytes
1649  * 4K page: n = 8, spare = 32 bytes
1650  * 8K page: n = 16, spare = 64 bytes
1651  *
1652  * the qcom nand controller operates at a sub page/codeword level. each
1653  * codeword is 528 and 532 bytes for 4 bit and 8 bit ECC modes respectively.
1654  * the number of ECC bytes vary based on the ECC strength and the bus width.
1655  *
1656  * the first n - 1 codewords contains 516 bytes of user data, the remaining
1657  * 12/16 bytes consist of ECC and reserved data. The nth codeword contains
1658  * both user data and spare(oobavail) bytes that sum up to 516 bytes.
1659  *
1660  * When we access a page with ECC enabled, the reserved bytes(s) are not
1661  * accessible at all. When reading, we fill up these unreadable positions
1662  * with 0xffs. When writing, the controller skips writing the inaccessible
1663  * bytes.
1664  *
1665  * Layout with ECC disabled:
1666  *
1667  * |------------------------------|  |---------------------------------------|
1668  * |         yy          xx.......|  |         bb          *********xx.......|
1669  * |  DATA1  yy  DATA2   xx..ECC..|  |  DATA1  bb  DATA2   **SPARE**xx..ECC..|
1670  * | (size1) yy (size2)  xx.......|  | (size1) bb (size2)  **(n*4)**xx.......|
1671  * |         yy          xx.......|  |         bb          *********xx.......|
1672  * |------------------------------|  |---------------------------------------|
1673  *         codeword 1,2..n-1                        codeword n
1674  *  <-------(528/532 Bytes)------>    <-----------(528/532 Bytes)----------->
1675  *
1676  * n = Number of codewords in the page
1677  * . = ECC bytes
1678  * * = Spare/free bytes
1679  * x = Unused byte(s)
1680  * y = Dummy Bad Bock byte(s)
1681  * b = Real Bad Block byte(s)
1682  * size1/size2 = function of codeword size and 'n'
1683  *
1684  * when the ECC block is disabled, one reserved byte (or two for 16 bit bus
1685  * width) is now accessible. For the first n - 1 codewords, these are dummy Bad
1686  * Block Markers. In the last codeword, this position contains the real BBM
1687  *
1688  * In order to have a consistent layout between RAW and ECC modes, we assume
1689  * the following OOB layout arrangement:
1690  *
1691  * |-----------|  |--------------------|
1692  * |yyxx.......|  |bb*********xx.......|
1693  * |yyxx..ECC..|  |bb*FREEOOB*xx..ECC..|
1694  * |yyxx.......|  |bb*********xx.......|
1695  * |yyxx.......|  |bb*********xx.......|
1696  * |-----------|  |--------------------|
1697  *  first n - 1       nth OOB region
1698  *  OOB regions
1699  *
1700  * n = Number of codewords in the page
1701  * . = ECC bytes
1702  * * = FREE OOB bytes
1703  * y = Dummy bad block byte(s) (inaccessible when ECC enabled)
1704  * x = Unused byte(s)
1705  * b = Real bad block byte(s) (inaccessible when ECC enabled)
1706  *
1707  * This layout is read as is when ECC is disabled. When ECC is enabled, the
1708  * inaccessible Bad Block byte(s) are ignored when we write to a page/oob,
1709  * and assumed as 0xffs when we read a page/oob. The ECC, unused and
1710  * dummy/real bad block bytes are grouped as ecc bytes (i.e, ecc->bytes is
1711  * the sum of the three).
1712  */
1713 static int qcom_nand_ooblayout_ecc(struct mtd_info *mtd, int section,
1714                                    struct mtd_oob_region *oobregion)
1715 {
1716         struct nand_chip *chip = mtd_to_nand(mtd);
1717         struct qcom_nand_host *host = to_qcom_nand_host(chip);
1718         struct nand_ecc_ctrl *ecc = &chip->ecc;
1719
1720         if (section > 1)
1721                 return -ERANGE;
1722
1723         if (!section) {
1724                 oobregion->length = (ecc->bytes * (ecc->steps - 1)) +
1725                                     host->bbm_size;
1726                 oobregion->offset = 0;
1727         } else {
1728                 oobregion->length = host->ecc_bytes_hw + host->spare_bytes;
1729                 oobregion->offset = mtd->oobsize - oobregion->length;
1730         }
1731
1732         return 0;
1733 }
1734
1735 static int qcom_nand_ooblayout_free(struct mtd_info *mtd, int section,
1736                                      struct mtd_oob_region *oobregion)
1737 {
1738         struct nand_chip *chip = mtd_to_nand(mtd);
1739         struct qcom_nand_host *host = to_qcom_nand_host(chip);
1740         struct nand_ecc_ctrl *ecc = &chip->ecc;
1741
1742         if (section)
1743                 return -ERANGE;
1744
1745         oobregion->length = ecc->steps * 4;
1746         oobregion->offset = ((ecc->steps - 1) * ecc->bytes) + host->bbm_size;
1747
1748         return 0;
1749 }
1750
1751 static const struct mtd_ooblayout_ops qcom_nand_ooblayout_ops = {
1752         .ecc = qcom_nand_ooblayout_ecc,
1753         .free = qcom_nand_ooblayout_free,
1754 };
1755
1756 static int qcom_nand_host_setup(struct qcom_nand_host *host)
1757 {
1758         struct nand_chip *chip = &host->chip;
1759         struct mtd_info *mtd = nand_to_mtd(chip);
1760         struct nand_ecc_ctrl *ecc = &chip->ecc;
1761         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1762         int cwperpage, bad_block_byte;
1763         bool wide_bus;
1764         int ecc_mode = 1;
1765
1766         /*
1767          * the controller requires each step consists of 512 bytes of data.
1768          * bail out if DT has populated a wrong step size.
1769          */
1770         if (ecc->size != NANDC_STEP_SIZE) {
1771                 dev_err(nandc->dev, "invalid ecc size\n");
1772                 return -EINVAL;
1773         }
1774
1775         wide_bus = chip->options & NAND_BUSWIDTH_16 ? true : false;
1776
1777         if (ecc->strength >= 8) {
1778                 /* 8 bit ECC defaults to BCH ECC on all platforms */
1779                 host->bch_enabled = true;
1780                 ecc_mode = 1;
1781
1782                 if (wide_bus) {
1783                         host->ecc_bytes_hw = 14;
1784                         host->spare_bytes = 0;
1785                         host->bbm_size = 2;
1786                 } else {
1787                         host->ecc_bytes_hw = 13;
1788                         host->spare_bytes = 2;
1789                         host->bbm_size = 1;
1790                 }
1791         } else {
1792                 /*
1793                  * if the controller supports BCH for 4 bit ECC, the controller
1794                  * uses lesser bytes for ECC. If RS is used, the ECC bytes is
1795                  * always 10 bytes
1796                  */
1797                 if (nandc->ecc_modes & ECC_BCH_4BIT) {
1798                         /* BCH */
1799                         host->bch_enabled = true;
1800                         ecc_mode = 0;
1801
1802                         if (wide_bus) {
1803                                 host->ecc_bytes_hw = 8;
1804                                 host->spare_bytes = 2;
1805                                 host->bbm_size = 2;
1806                         } else {
1807                                 host->ecc_bytes_hw = 7;
1808                                 host->spare_bytes = 4;
1809                                 host->bbm_size = 1;
1810                         }
1811                 } else {
1812                         /* RS */
1813                         host->ecc_bytes_hw = 10;
1814
1815                         if (wide_bus) {
1816                                 host->spare_bytes = 0;
1817                                 host->bbm_size = 2;
1818                         } else {
1819                                 host->spare_bytes = 1;
1820                                 host->bbm_size = 1;
1821                         }
1822                 }
1823         }
1824
1825         /*
1826          * we consider ecc->bytes as the sum of all the non-data content in a
1827          * step. It gives us a clean representation of the oob area (even if
1828          * all the bytes aren't used for ECC).It is always 16 bytes for 8 bit
1829          * ECC and 12 bytes for 4 bit ECC
1830          */
1831         ecc->bytes = host->ecc_bytes_hw + host->spare_bytes + host->bbm_size;
1832
1833         ecc->read_page          = qcom_nandc_read_page;
1834         ecc->read_page_raw      = qcom_nandc_read_page_raw;
1835         ecc->read_oob           = qcom_nandc_read_oob;
1836         ecc->write_page         = qcom_nandc_write_page;
1837         ecc->write_page_raw     = qcom_nandc_write_page_raw;
1838         ecc->write_oob          = qcom_nandc_write_oob;
1839
1840         ecc->mode = NAND_ECC_HW;
1841
1842         mtd_set_ooblayout(mtd, &qcom_nand_ooblayout_ops);
1843
1844         cwperpage = mtd->writesize / ecc->size;
1845
1846         /*
1847          * DATA_UD_BYTES varies based on whether the read/write command protects
1848          * spare data with ECC too. We protect spare data by default, so we set
1849          * it to main + spare data, which are 512 and 4 bytes respectively.
1850          */
1851         host->cw_data = 516;
1852
1853         /*
1854          * total bytes in a step, either 528 bytes for 4 bit ECC, or 532 bytes
1855          * for 8 bit ECC
1856          */
1857         host->cw_size = host->cw_data + ecc->bytes;
1858
1859         if (ecc->bytes * (mtd->writesize / ecc->size) > mtd->oobsize) {
1860                 dev_err(nandc->dev, "ecc data doesn't fit in OOB area\n");
1861                 return -EINVAL;
1862         }
1863
1864         bad_block_byte = mtd->writesize - host->cw_size * (cwperpage - 1) + 1;
1865
1866         host->cfg0 = (cwperpage - 1) << CW_PER_PAGE
1867                                 | host->cw_data << UD_SIZE_BYTES
1868                                 | 0 << DISABLE_STATUS_AFTER_WRITE
1869                                 | 5 << NUM_ADDR_CYCLES
1870                                 | host->ecc_bytes_hw << ECC_PARITY_SIZE_BYTES_RS
1871                                 | 0 << STATUS_BFR_READ
1872                                 | 1 << SET_RD_MODE_AFTER_STATUS
1873                                 | host->spare_bytes << SPARE_SIZE_BYTES;
1874
1875         host->cfg1 = 7 << NAND_RECOVERY_CYCLES
1876                                 | 0 <<  CS_ACTIVE_BSY
1877                                 | bad_block_byte << BAD_BLOCK_BYTE_NUM
1878                                 | 0 << BAD_BLOCK_IN_SPARE_AREA
1879                                 | 2 << WR_RD_BSY_GAP
1880                                 | wide_bus << WIDE_FLASH
1881                                 | host->bch_enabled << ENABLE_BCH_ECC;
1882
1883         host->cfg0_raw = (cwperpage - 1) << CW_PER_PAGE
1884                                 | host->cw_size << UD_SIZE_BYTES
1885                                 | 5 << NUM_ADDR_CYCLES
1886                                 | 0 << SPARE_SIZE_BYTES;
1887
1888         host->cfg1_raw = 7 << NAND_RECOVERY_CYCLES
1889                                 | 0 << CS_ACTIVE_BSY
1890                                 | 17 << BAD_BLOCK_BYTE_NUM
1891                                 | 1 << BAD_BLOCK_IN_SPARE_AREA
1892                                 | 2 << WR_RD_BSY_GAP
1893                                 | wide_bus << WIDE_FLASH
1894                                 | 1 << DEV0_CFG1_ECC_DISABLE;
1895
1896         host->ecc_bch_cfg = host->bch_enabled << ECC_CFG_ECC_DISABLE
1897                                 | 0 << ECC_SW_RESET
1898                                 | host->cw_data << ECC_NUM_DATA_BYTES
1899                                 | 1 << ECC_FORCE_CLK_OPEN
1900                                 | ecc_mode << ECC_MODE
1901                                 | host->ecc_bytes_hw << ECC_PARITY_SIZE_BYTES_BCH;
1902
1903         host->ecc_buf_cfg = 0x203 << NUM_STEPS;
1904
1905         host->clrflashstatus = FS_READY_BSY_N;
1906         host->clrreadstatus = 0xc0;
1907
1908         dev_dbg(nandc->dev,
1909                 "cfg0 %x cfg1 %x ecc_buf_cfg %x ecc_bch cfg %x cw_size %d cw_data %d strength %d parity_bytes %d steps %d\n",
1910                 host->cfg0, host->cfg1, host->ecc_buf_cfg, host->ecc_bch_cfg,
1911                 host->cw_size, host->cw_data, ecc->strength, ecc->bytes,
1912                 cwperpage);
1913
1914         return 0;
1915 }
1916
1917 static int qcom_nandc_alloc(struct qcom_nand_controller *nandc)
1918 {
1919         int ret;
1920
1921         ret = dma_set_coherent_mask(nandc->dev, DMA_BIT_MASK(32));
1922         if (ret) {
1923                 dev_err(nandc->dev, "failed to set DMA mask\n");
1924                 return ret;
1925         }
1926
1927         /*
1928          * we use the internal buffer for reading ONFI params, reading small
1929          * data like ID and status, and preforming read-copy-write operations
1930          * when writing to a codeword partially. 532 is the maximum possible
1931          * size of a codeword for our nand controller
1932          */
1933         nandc->buf_size = 532;
1934
1935         nandc->data_buffer = devm_kzalloc(nandc->dev, nandc->buf_size,
1936                                         GFP_KERNEL);
1937         if (!nandc->data_buffer)
1938                 return -ENOMEM;
1939
1940         nandc->regs = devm_kzalloc(nandc->dev, sizeof(*nandc->regs),
1941                                         GFP_KERNEL);
1942         if (!nandc->regs)
1943                 return -ENOMEM;
1944
1945         nandc->reg_read_buf = devm_kzalloc(nandc->dev,
1946                                 MAX_REG_RD * sizeof(*nandc->reg_read_buf),
1947                                 GFP_KERNEL);
1948         if (!nandc->reg_read_buf)
1949                 return -ENOMEM;
1950
1951         nandc->chan = dma_request_slave_channel(nandc->dev, "rxtx");
1952         if (!nandc->chan) {
1953                 dev_err(nandc->dev, "failed to request slave channel\n");
1954                 return -ENODEV;
1955         }
1956
1957         INIT_LIST_HEAD(&nandc->desc_list);
1958         INIT_LIST_HEAD(&nandc->host_list);
1959
1960         nand_hw_control_init(&nandc->controller);
1961
1962         return 0;
1963 }
1964
1965 static void qcom_nandc_unalloc(struct qcom_nand_controller *nandc)
1966 {
1967         dma_release_channel(nandc->chan);
1968 }
1969
1970 /* one time setup of a few nand controller registers */
1971 static int qcom_nandc_setup(struct qcom_nand_controller *nandc)
1972 {
1973         /* kill onenand */
1974         nandc_write(nandc, SFLASHC_BURST_CFG, 0);
1975
1976         /* enable ADM DMA */
1977         nandc_write(nandc, NAND_FLASH_CHIP_SELECT, DM_EN);
1978
1979         /* save the original values of these registers */
1980         nandc->cmd1 = nandc_read(nandc, NAND_DEV_CMD1);
1981         nandc->vld = nandc_read(nandc, NAND_DEV_CMD_VLD);
1982
1983         return 0;
1984 }
1985
1986 static int qcom_nand_host_init(struct qcom_nand_controller *nandc,
1987                                struct qcom_nand_host *host,
1988                                struct device_node *dn)
1989 {
1990         struct nand_chip *chip = &host->chip;
1991         struct mtd_info *mtd = nand_to_mtd(chip);
1992         struct device *dev = nandc->dev;
1993         int ret;
1994
1995         ret = of_property_read_u32(dn, "reg", &host->cs);
1996         if (ret) {
1997                 dev_err(dev, "can't get chip-select\n");
1998                 return -ENXIO;
1999         }
2000
2001         nand_set_flash_node(chip, dn);
2002         mtd->name = devm_kasprintf(dev, GFP_KERNEL, "qcom_nand.%d", host->cs);
2003         mtd->owner = THIS_MODULE;
2004         mtd->dev.parent = dev;
2005
2006         chip->cmdfunc           = qcom_nandc_command;
2007         chip->select_chip       = qcom_nandc_select_chip;
2008         chip->read_byte         = qcom_nandc_read_byte;
2009         chip->read_buf          = qcom_nandc_read_buf;
2010         chip->write_buf         = qcom_nandc_write_buf;
2011         chip->onfi_set_features = nand_onfi_get_set_features_notsupp;
2012         chip->onfi_get_features = nand_onfi_get_set_features_notsupp;
2013
2014         /*
2015          * the bad block marker is readable only when we read the last codeword
2016          * of a page with ECC disabled. currently, the nand_base and nand_bbt
2017          * helpers don't allow us to read BB from a nand chip with ECC
2018          * disabled (MTD_OPS_PLACE_OOB is set by default). use the block_bad
2019          * and block_markbad helpers until we permanently switch to using
2020          * MTD_OPS_RAW for all drivers (with the help of badblockbits)
2021          */
2022         chip->block_bad         = qcom_nandc_block_bad;
2023         chip->block_markbad     = qcom_nandc_block_markbad;
2024
2025         chip->controller = &nandc->controller;
2026         chip->options |= NAND_NO_SUBPAGE_WRITE | NAND_USE_BOUNCE_BUFFER |
2027                          NAND_SKIP_BBTSCAN;
2028
2029         /* set up initial status value */
2030         host->status = NAND_STATUS_READY | NAND_STATUS_WP;
2031
2032         ret = nand_scan_ident(mtd, 1, NULL);
2033         if (ret)
2034                 return ret;
2035
2036         ret = qcom_nand_host_setup(host);
2037         if (ret)
2038                 return ret;
2039
2040         ret = nand_scan_tail(mtd);
2041         if (ret)
2042                 return ret;
2043
2044         return mtd_device_register(mtd, NULL, 0);
2045 }
2046
2047 /* parse custom DT properties here */
2048 static int qcom_nandc_parse_dt(struct platform_device *pdev)
2049 {
2050         struct qcom_nand_controller *nandc = platform_get_drvdata(pdev);
2051         struct device_node *np = nandc->dev->of_node;
2052         int ret;
2053
2054         ret = of_property_read_u32(np, "qcom,cmd-crci", &nandc->cmd_crci);
2055         if (ret) {
2056                 dev_err(nandc->dev, "command CRCI unspecified\n");
2057                 return ret;
2058         }
2059
2060         ret = of_property_read_u32(np, "qcom,data-crci", &nandc->data_crci);
2061         if (ret) {
2062                 dev_err(nandc->dev, "data CRCI unspecified\n");
2063                 return ret;
2064         }
2065
2066         return 0;
2067 }
2068
2069 static int qcom_nandc_probe(struct platform_device *pdev)
2070 {
2071         struct qcom_nand_controller *nandc;
2072         struct qcom_nand_host *host;
2073         const void *dev_data;
2074         struct device *dev = &pdev->dev;
2075         struct device_node *dn = dev->of_node, *child;
2076         struct resource *res;
2077         int ret;
2078
2079         nandc = devm_kzalloc(&pdev->dev, sizeof(*nandc), GFP_KERNEL);
2080         if (!nandc)
2081                 return -ENOMEM;
2082
2083         platform_set_drvdata(pdev, nandc);
2084         nandc->dev = dev;
2085
2086         dev_data = of_device_get_match_data(dev);
2087         if (!dev_data) {
2088                 dev_err(&pdev->dev, "failed to get device data\n");
2089                 return -ENODEV;
2090         }
2091
2092         nandc->ecc_modes = (unsigned long)dev_data;
2093
2094         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2095         nandc->base = devm_ioremap_resource(dev, res);
2096         if (IS_ERR(nandc->base))
2097                 return PTR_ERR(nandc->base);
2098
2099         nandc->base_dma = phys_to_dma(dev, (phys_addr_t)res->start);
2100
2101         nandc->core_clk = devm_clk_get(dev, "core");
2102         if (IS_ERR(nandc->core_clk))
2103                 return PTR_ERR(nandc->core_clk);
2104
2105         nandc->aon_clk = devm_clk_get(dev, "aon");
2106         if (IS_ERR(nandc->aon_clk))
2107                 return PTR_ERR(nandc->aon_clk);
2108
2109         ret = qcom_nandc_parse_dt(pdev);
2110         if (ret)
2111                 return ret;
2112
2113         ret = qcom_nandc_alloc(nandc);
2114         if (ret)
2115                 return ret;
2116
2117         ret = clk_prepare_enable(nandc->core_clk);
2118         if (ret)
2119                 goto err_core_clk;
2120
2121         ret = clk_prepare_enable(nandc->aon_clk);
2122         if (ret)
2123                 goto err_aon_clk;
2124
2125         ret = qcom_nandc_setup(nandc);
2126         if (ret)
2127                 goto err_setup;
2128
2129         for_each_available_child_of_node(dn, child) {
2130                 if (of_device_is_compatible(child, "qcom,nandcs")) {
2131                         host = devm_kzalloc(dev, sizeof(*host), GFP_KERNEL);
2132                         if (!host) {
2133                                 of_node_put(child);
2134                                 ret = -ENOMEM;
2135                                 goto err_cs_init;
2136                         }
2137
2138                         ret = qcom_nand_host_init(nandc, host, child);
2139                         if (ret) {
2140                                 devm_kfree(dev, host);
2141                                 continue;
2142                         }
2143
2144                         list_add_tail(&host->node, &nandc->host_list);
2145                 }
2146         }
2147
2148         if (list_empty(&nandc->host_list)) {
2149                 ret = -ENODEV;
2150                 goto err_cs_init;
2151         }
2152
2153         return 0;
2154
2155 err_cs_init:
2156         list_for_each_entry(host, &nandc->host_list, node)
2157                 nand_release(nand_to_mtd(&host->chip));
2158 err_setup:
2159         clk_disable_unprepare(nandc->aon_clk);
2160 err_aon_clk:
2161         clk_disable_unprepare(nandc->core_clk);
2162 err_core_clk:
2163         qcom_nandc_unalloc(nandc);
2164
2165         return ret;
2166 }
2167
2168 static int qcom_nandc_remove(struct platform_device *pdev)
2169 {
2170         struct qcom_nand_controller *nandc = platform_get_drvdata(pdev);
2171         struct qcom_nand_host *host;
2172
2173         list_for_each_entry(host, &nandc->host_list, node)
2174                 nand_release(nand_to_mtd(&host->chip));
2175
2176         qcom_nandc_unalloc(nandc);
2177
2178         clk_disable_unprepare(nandc->aon_clk);
2179         clk_disable_unprepare(nandc->core_clk);
2180
2181         return 0;
2182 }
2183
2184 #define EBI2_NANDC_ECC_MODES    (ECC_RS_4BIT | ECC_BCH_8BIT)
2185
2186 /*
2187  * data will hold a struct pointer containing more differences once we support
2188  * more controller variants
2189  */
2190 static const struct of_device_id qcom_nandc_of_match[] = {
2191         {       .compatible = "qcom,ipq806x-nand",
2192                 .data = (void *)EBI2_NANDC_ECC_MODES,
2193         },
2194         {}
2195 };
2196 MODULE_DEVICE_TABLE(of, qcom_nandc_of_match);
2197
2198 static struct platform_driver qcom_nandc_driver = {
2199         .driver = {
2200                 .name = "qcom-nandc",
2201                 .of_match_table = qcom_nandc_of_match,
2202         },
2203         .probe   = qcom_nandc_probe,
2204         .remove  = qcom_nandc_remove,
2205 };
2206 module_platform_driver(qcom_nandc_driver);
2207
2208 MODULE_AUTHOR("Archit Taneja <architt@codeaurora.org>");
2209 MODULE_DESCRIPTION("Qualcomm NAND Controller driver");
2210 MODULE_LICENSE("GPL v2");