drivers/mtd/nand/sunxi_nand.c

   1 /*
   2  * Copyright (C) 2013 Boris BREZILLON <b.brezillon.dev@gmail.com>
   3  *
   4  * Derived from:
   5  *      https://github.com/yuq/sunxi-nfc-mtd
   6  *      Copyright (C) 2013 Qiang Yu <yuq825@gmail.com>
   7  *
   8  *      https://github.com/hno/Allwinner-Info
   9  *      Copyright (C) 2013 Henrik Nordström <Henrik Nordström>
  10  *
  11  *      Copyright (C) 2013 Dmitriy B. <rzk333@gmail.com>
  12  *      Copyright (C) 2013 Sergey Lapin <slapin@ossfans.org>
  13  *
  14  * This program is free software; you can redistribute it and/or modify
  15  * it under the terms of the GNU General Public License as published by
  16  * the Free Software Foundation; either version 2 of the License, or
  17  * (at your option) any later version.
  18  *
  19  * This program is distributed in the hope that it will be useful,
  20  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  21  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  22  * GNU General Public License for more details.
  23  */
  24
  25 #include <linux/dma-mapping.h>
  26 #include <linux/slab.h>
  27 #include <linux/module.h>
  28 #include <linux/moduleparam.h>
  29 #include <linux/platform_device.h>
  30 #include <linux/of.h>
  31 #include <linux/of_device.h>
  32 #include <linux/of_gpio.h>
  33 #include <linux/mtd/mtd.h>
  34 #include <linux/mtd/nand.h>
  35 #include <linux/mtd/partitions.h>
  36 #include <linux/clk.h>
  37 #include <linux/delay.h>
  38 #include <linux/dmaengine.h>
  39 #include <linux/gpio.h>
  40 #include <linux/interrupt.h>
  41 #include <linux/iopoll.h>
  42 #include <linux/reset.h>
  43
  44 #define NFC_REG_CTL             0x0000
  45 #define NFC_REG_ST              0x0004
  46 #define NFC_REG_INT             0x0008
  47 #define NFC_REG_TIMING_CTL      0x000C
  48 #define NFC_REG_TIMING_CFG      0x0010
  49 #define NFC_REG_ADDR_LOW        0x0014
  50 #define NFC_REG_ADDR_HIGH       0x0018
  51 #define NFC_REG_SECTOR_NUM      0x001C
  52 #define NFC_REG_CNT             0x0020
  53 #define NFC_REG_CMD             0x0024
  54 #define NFC_REG_RCMD_SET        0x0028
  55 #define NFC_REG_WCMD_SET        0x002C
  56 #define NFC_REG_IO_DATA         0x0030
  57 #define NFC_REG_ECC_CTL         0x0034
  58 #define NFC_REG_ECC_ST          0x0038
  59 #define NFC_REG_DEBUG           0x003C
  60 #define NFC_REG_ECC_ERR_CNT(x)  ((0x0040 + (x)) & ~0x3)
  61 #define NFC_REG_USER_DATA(x)    (0x0050 + ((x) * 4))
  62 #define NFC_REG_SPARE_AREA      0x00A0
  63 #define NFC_REG_PAT_ID          0x00A4
  64 #define NFC_RAM0_BASE           0x0400
  65 #define NFC_RAM1_BASE           0x0800
  66
  67 /* define bit use in NFC_CTL */
  68 #define NFC_EN                  BIT(0)
  69 #define NFC_RESET               BIT(1)
  70 #define NFC_BUS_WIDTH_MSK       BIT(2)
  71 #define NFC_BUS_WIDTH_8         (0 << 2)
  72 #define NFC_BUS_WIDTH_16        (1 << 2)
  73 #define NFC_RB_SEL_MSK          BIT(3)
  74 #define NFC_RB_SEL(x)           ((x) << 3)
  75 #define NFC_CE_SEL_MSK          GENMASK(26, 24)
  76 #define NFC_CE_SEL(x)           ((x) << 24)
  77 #define NFC_CE_CTL              BIT(6)
  78 #define NFC_PAGE_SHIFT_MSK      GENMASK(11, 8)
  79 #define NFC_PAGE_SHIFT(x)       (((x) < 10 ? 0 : (x) - 10) << 8)
  80 #define NFC_SAM                 BIT(12)
  81 #define NFC_RAM_METHOD          BIT(14)
  82 #define NFC_DEBUG_CTL           BIT(31)
  83
  84 /* define bit use in NFC_ST */
  85 #define NFC_RB_B2R              BIT(0)
  86 #define NFC_CMD_INT_FLAG        BIT(1)
  87 #define NFC_DMA_INT_FLAG        BIT(2)
  88 #define NFC_CMD_FIFO_STATUS     BIT(3)
  89 #define NFC_STA                 BIT(4)
  90 #define NFC_NATCH_INT_FLAG      BIT(5)
  91 #define NFC_RB_STATE(x)         BIT(x + 8)
  92
  93 /* define bit use in NFC_INT */
  94 #define NFC_B2R_INT_ENABLE      BIT(0)
  95 #define NFC_CMD_INT_ENABLE      BIT(1)
  96 #define NFC_DMA_INT_ENABLE      BIT(2)
  97 #define NFC_INT_MASK            (NFC_B2R_INT_ENABLE | \
  98                                  NFC_CMD_INT_ENABLE | \
  99                                  NFC_DMA_INT_ENABLE)
 100
 101 /* define bit use in NFC_TIMING_CTL */
 102 #define NFC_TIMING_CTL_EDO      BIT(8)
 103
 104 /* define NFC_TIMING_CFG register layout */
 105 #define NFC_TIMING_CFG(tWB, tADL, tWHR, tRHW, tCAD)             \
 106         (((tWB) & 0x3) | (((tADL) & 0x3) << 2) |                \
 107         (((tWHR) & 0x3) << 4) | (((tRHW) & 0x3) << 6) |         \
 108         (((tCAD) & 0x7) << 8))
 109
 110 /* define bit use in NFC_CMD */
 111 #define NFC_CMD_LOW_BYTE_MSK    GENMASK(7, 0)
 112 #define NFC_CMD_HIGH_BYTE_MSK   GENMASK(15, 8)
 113 #define NFC_CMD(x)              (x)
 114 #define NFC_ADR_NUM_MSK         GENMASK(18, 16)
 115 #define NFC_ADR_NUM(x)          (((x) - 1) << 16)
 116 #define NFC_SEND_ADR            BIT(19)
 117 #define NFC_ACCESS_DIR          BIT(20)
 118 #define NFC_DATA_TRANS          BIT(21)
 119 #define NFC_SEND_CMD1           BIT(22)
 120 #define NFC_WAIT_FLAG           BIT(23)
 121 #define NFC_SEND_CMD2           BIT(24)
 122 #define NFC_SEQ                 BIT(25)
 123 #define NFC_DATA_SWAP_METHOD    BIT(26)
 124 #define NFC_ROW_AUTO_INC        BIT(27)
 125 #define NFC_SEND_CMD3           BIT(28)
 126 #define NFC_SEND_CMD4           BIT(29)
 127 #define NFC_CMD_TYPE_MSK        GENMASK(31, 30)
 128 #define NFC_NORMAL_OP           (0 << 30)
 129 #define NFC_ECC_OP              (1 << 30)
 130 #define NFC_PAGE_OP             (2 << 30)
 131
 132 /* define bit use in NFC_RCMD_SET */
 133 #define NFC_READ_CMD_MSK        GENMASK(7, 0)
 134 #define NFC_RND_READ_CMD0_MSK   GENMASK(15, 8)
 135 #define NFC_RND_READ_CMD1_MSK   GENMASK(23, 16)
 136
 137 /* define bit use in NFC_WCMD_SET */
 138 #define NFC_PROGRAM_CMD_MSK     GENMASK(7, 0)
 139 #define NFC_RND_WRITE_CMD_MSK   GENMASK(15, 8)
 140 #define NFC_READ_CMD0_MSK       GENMASK(23, 16)
 141 #define NFC_READ_CMD1_MSK       GENMASK(31, 24)
 142
 143 /* define bit use in NFC_ECC_CTL */
 144 #define NFC_ECC_EN              BIT(0)
 145 #define NFC_ECC_PIPELINE        BIT(3)
 146 #define NFC_ECC_EXCEPTION       BIT(4)
 147 #define NFC_ECC_BLOCK_SIZE_MSK  BIT(5)
 148 #define NFC_ECC_BLOCK_512       BIT(5)
 149 #define NFC_RANDOM_EN           BIT(9)
 150 #define NFC_RANDOM_DIRECTION    BIT(10)
 151 #define NFC_ECC_MODE_MSK        GENMASK(15, 12)
 152 #define NFC_ECC_MODE(x)         ((x) << 12)
 153 #define NFC_RANDOM_SEED_MSK     GENMASK(30, 16)
 154 #define NFC_RANDOM_SEED(x)      ((x) << 16)
 155
 156 /* define bit use in NFC_ECC_ST */
 157 #define NFC_ECC_ERR(x)          BIT(x)
 158 #define NFC_ECC_ERR_MSK         GENMASK(15, 0)
 159 #define NFC_ECC_PAT_FOUND(x)    BIT(x + 16)
 160 #define NFC_ECC_ERR_CNT(b, x)   (((x) >> (((b) % 4) * 8)) & 0xff)
 161
 162 #define NFC_DEFAULT_TIMEOUT_MS  1000
 163
 164 #define NFC_SRAM_SIZE           1024
 165
 166 #define NFC_MAX_CS              7
 167
 168 /*
 169  * Ready/Busy detection type: describes the Ready/Busy detection modes
 170  *
 171  * @RB_NONE:    no external detection available, rely on STATUS command
 172  *              and software timeouts
 173  * @RB_NATIVE:  use sunxi NAND controller Ready/Busy support. The Ready/Busy
 174  *              pin of the NAND flash chip must be connected to one of the
 175  *              native NAND R/B pins (those which can be muxed to the NAND
 176  *              Controller)
 177  * @RB_GPIO:    use a simple GPIO to handle Ready/Busy status. The Ready/Busy
 178  *              pin of the NAND flash chip must be connected to a GPIO capable
 179  *              pin.
 180  */
 181 enum sunxi_nand_rb_type {
 182         RB_NONE,
 183         RB_NATIVE,
 184         RB_GPIO,
 185 };
 186
 187 /*
 188  * Ready/Busy structure: stores information related to Ready/Busy detection
 189  *
 190  * @type:       the Ready/Busy detection mode
 191  * @info:       information related to the R/B detection mode. Either a gpio
 192  *              id or a native R/B id (those supported by the NAND controller).
 193  */
 194 struct sunxi_nand_rb {
 195         enum sunxi_nand_rb_type type;
 196         union {
 197                 int gpio;
 198                 int nativeid;
 199         } info;
 200 };
 201
 202 /*
 203  * Chip Select structure: stores information related to NAND Chip Select
 204  *
 205  * @cs:         the NAND CS id used to communicate with a NAND Chip
 206  * @rb:         the Ready/Busy description
 207  */
 208 struct sunxi_nand_chip_sel {
 209         u8 cs;
 210         struct sunxi_nand_rb rb;
 211 };
 212
 213 /*
 214  * sunxi HW ECC infos: stores information related to HW ECC support
 215  *
 216  * @mode:       the sunxi ECC mode field deduced from ECC requirements
 217  */
 218 struct sunxi_nand_hw_ecc {
 219         int mode;
 220 };
 221
 222 /*
 223  * NAND chip structure: stores NAND chip device related information
 224  *
 225  * @node:               used to store NAND chips into a list
 226  * @nand:               base NAND chip structure
 227  * @mtd:                base MTD structure
 228  * @clk_rate:           clk_rate required for this NAND chip
 229  * @timing_cfg          TIMING_CFG register value for this NAND chip
 230  * @selected:           current active CS
 231  * @nsels:              number of CS lines required by the NAND chip
 232  * @sels:               array of CS lines descriptions
 233  */
 234 struct sunxi_nand_chip {
 235         struct list_head node;
 236         struct nand_chip nand;
 237         unsigned long clk_rate;
 238         u32 timing_cfg;
 239         u32 timing_ctl;
 240         int selected;
 241         int addr_cycles;
 242         u32 addr[2];
 243         int cmd_cycles;
 244         u8 cmd[2];
 245         int nsels;
 246         struct sunxi_nand_chip_sel sels[0];
 247 };
 248
 249 static inline struct sunxi_nand_chip *to_sunxi_nand(struct nand_chip *nand)
 250 {
 251         return container_of(nand, struct sunxi_nand_chip, nand);
 252 }
 253
 254 /*
 255  * NAND Controller structure: stores sunxi NAND controller information
 256  *
 257  * @controller:         base controller structure
 258  * @dev:                parent device (used to print error messages)
 259  * @regs:               NAND controller registers
 260  * @ahb_clk:            NAND Controller AHB clock
 261  * @mod_clk:            NAND Controller mod clock
 262  * @assigned_cs:        bitmask describing already assigned CS lines
 263  * @clk_rate:           NAND controller current clock rate
 264  * @chips:              a list containing all the NAND chips attached to
 265  *                      this NAND controller
 266  * @complete:           a completion object used to wait for NAND
 267  *                      controller events
 268  */
 269 struct sunxi_nfc {
 270         struct nand_hw_control controller;
 271         struct device *dev;
 272         void __iomem *regs;
 273         struct clk *ahb_clk;
 274         struct clk *mod_clk;
 275         struct reset_control *reset;
 276         unsigned long assigned_cs;
 277         unsigned long clk_rate;
 278         struct list_head chips;
 279         struct completion complete;
 280         struct dma_chan *dmac;
 281 };
 282
 283 static inline struct sunxi_nfc *to_sunxi_nfc(struct nand_hw_control *ctrl)
 284 {
 285         return container_of(ctrl, struct sunxi_nfc, controller);
 286 }
 287
 288 static irqreturn_t sunxi_nfc_interrupt(int irq, void *dev_id)
 289 {
 290         struct sunxi_nfc *nfc = dev_id;
 291         u32 st = readl(nfc->regs + NFC_REG_ST);
 292         u32 ien = readl(nfc->regs + NFC_REG_INT);
 293
 294         if (!(ien & st))
 295                 return IRQ_NONE;
 296
 297         if ((ien & st) == ien)
 298                 complete(&nfc->complete);
 299
 300         writel(st & NFC_INT_MASK, nfc->regs + NFC_REG_ST);
 301         writel(~st & ien & NFC_INT_MASK, nfc->regs + NFC_REG_INT);
 302
 303         return IRQ_HANDLED;
 304 }
 305
 306 static int sunxi_nfc_wait_events(struct sunxi_nfc *nfc, u32 events,
 307                                  bool use_polling, unsigned int timeout_ms)
 308 {
 309         int ret;
 310
 311         if (events & ~NFC_INT_MASK)
 312                 return -EINVAL;
 313
 314         if (!timeout_ms)
 315                 timeout_ms = NFC_DEFAULT_TIMEOUT_MS;
 316
 317         if (!use_polling) {
 318                 init_completion(&nfc->complete);
 319
 320                 writel(events, nfc->regs + NFC_REG_INT);
 321
 322                 ret = wait_for_completion_timeout(&nfc->complete,
 323                                                 msecs_to_jiffies(timeout_ms));
 324                 if (!ret)
 325                         ret = -ETIMEDOUT;
 326                 else
 327                         ret = 0;
 328
 329                 writel(0, nfc->regs + NFC_REG_INT);
 330         } else {
 331                 u32 status;
 332
 333                 ret = readl_poll_timeout(nfc->regs + NFC_REG_ST, status,
 334                                          (status & events) == events, 1,
 335                                          timeout_ms * 1000);
 336         }
 337
 338         writel(events & NFC_INT_MASK, nfc->regs + NFC_REG_ST);
 339
 340         if (ret)
 341                 dev_err(nfc->dev, "wait interrupt timedout\n");
 342
 343         return ret;
 344 }
 345
 346 static int sunxi_nfc_wait_cmd_fifo_empty(struct sunxi_nfc *nfc)
 347 {
 348         u32 status;
 349         int ret;
 350
 351         ret = readl_poll_timeout(nfc->regs + NFC_REG_ST, status,
 352                                  !(status & NFC_CMD_FIFO_STATUS), 1,
 353                                  NFC_DEFAULT_TIMEOUT_MS * 1000);
 354         if (ret)
 355                 dev_err(nfc->dev, "wait for empty cmd FIFO timedout\n");
 356
 357         return ret;
 358 }
 359
 360 static int sunxi_nfc_rst(struct sunxi_nfc *nfc)
 361 {
 362         u32 ctl;
 363         int ret;
 364
 365         writel(0, nfc->regs + NFC_REG_ECC_CTL);
 366         writel(NFC_RESET, nfc->regs + NFC_REG_CTL);
 367
 368         ret = readl_poll_timeout(nfc->regs + NFC_REG_CTL, ctl,
 369                                  !(ctl & NFC_RESET), 1,
 370                                  NFC_DEFAULT_TIMEOUT_MS * 1000);
 371         if (ret)
 372                 dev_err(nfc->dev, "wait for NAND controller reset timedout\n");
 373
 374         return ret;
 375 }
 376
 377 static int sunxi_nfc_dma_op_prepare(struct mtd_info *mtd, const void *buf,
 378                                     int chunksize, int nchunks,
 379                                     enum dma_data_direction ddir,
 380                                     struct scatterlist *sg)
 381 {
 382         struct nand_chip *nand = mtd_to_nand(mtd);
 383         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 384         struct dma_async_tx_descriptor *dmad;
 385         enum dma_transfer_direction tdir;
 386         dma_cookie_t dmat;
 387         int ret;
 388
 389         if (ddir == DMA_FROM_DEVICE)
 390                 tdir = DMA_DEV_TO_MEM;
 391         else
 392                 tdir = DMA_MEM_TO_DEV;
 393
 394         sg_init_one(sg, buf, nchunks * chunksize);
 395         ret = dma_map_sg(nfc->dev, sg, 1, ddir);
 396         if (!ret)
 397                 return -ENOMEM;
 398
 399         dmad = dmaengine_prep_slave_sg(nfc->dmac, sg, 1, tdir, DMA_CTRL_ACK);
 400         if (!dmad) {
 401                 ret = -EINVAL;
 402                 goto err_unmap_buf;
 403         }
 404
 405         writel(readl(nfc->regs + NFC_REG_CTL) | NFC_RAM_METHOD,
 406                nfc->regs + NFC_REG_CTL);
 407         writel(nchunks, nfc->regs + NFC_REG_SECTOR_NUM);
 408         writel(chunksize, nfc->regs + NFC_REG_CNT);
 409         dmat = dmaengine_submit(dmad);
 410
 411         ret = dma_submit_error(dmat);
 412         if (ret)
 413                 goto err_clr_dma_flag;
 414
 415         return 0;
 416
 417 err_clr_dma_flag:
 418         writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD,
 419                nfc->regs + NFC_REG_CTL);
 420
 421 err_unmap_buf:
 422         dma_unmap_sg(nfc->dev, sg, 1, ddir);
 423         return ret;
 424 }
 425
 426 static void sunxi_nfc_dma_op_cleanup(struct mtd_info *mtd,
 427                                      enum dma_data_direction ddir,
 428                                      struct scatterlist *sg)
 429 {
 430         struct nand_chip *nand = mtd_to_nand(mtd);
 431         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 432
 433         dma_unmap_sg(nfc->dev, sg, 1, ddir);
 434         writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD,
 435                nfc->regs + NFC_REG_CTL);
 436 }
 437
 438 static int sunxi_nfc_dev_ready(struct mtd_info *mtd)
 439 {
 440         struct nand_chip *nand = mtd_to_nand(mtd);
 441         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
 442         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
 443         struct sunxi_nand_rb *rb;
 444         int ret;
 445
 446         if (sunxi_nand->selected < 0)
 447                 return 0;
 448
 449         rb = &sunxi_nand->sels[sunxi_nand->selected].rb;
 450
 451         switch (rb->type) {
 452         case RB_NATIVE:
 453                 ret = !!(readl(nfc->regs + NFC_REG_ST) &
 454                          NFC_RB_STATE(rb->info.nativeid));
 455                 break;
 456         case RB_GPIO:
 457                 ret = gpio_get_value(rb->info.gpio);
 458                 break;
 459         case RB_NONE:
 460         default:
 461                 ret = 0;
 462                 dev_err(nfc->dev, "cannot check R/B NAND status!\n");
 463                 break;
 464         }
 465
 466         return ret;
 467 }
 468
 469 static void sunxi_nfc_select_chip(struct mtd_info *mtd, int chip)
 470 {
 471         struct nand_chip *nand = mtd_to_nand(mtd);
 472         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
 473         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
 474         struct sunxi_nand_chip_sel *sel;
 475         u32 ctl;
 476
 477         if (chip > 0 && chip >= sunxi_nand->nsels)
 478                 return;
 479
 480         if (chip == sunxi_nand->selected)
 481                 return;
 482
 483         ctl = readl(nfc->regs + NFC_REG_CTL) &
 484               ~(NFC_PAGE_SHIFT_MSK | NFC_CE_SEL_MSK | NFC_RB_SEL_MSK | NFC_EN);
 485
 486         if (chip >= 0) {
 487                 sel = &sunxi_nand->sels[chip];
 488
 489                 ctl |= NFC_CE_SEL(sel->cs) | NFC_EN |
 490                        NFC_PAGE_SHIFT(nand->page_shift);
 491                 if (sel->rb.type == RB_NONE) {
 492                         nand->dev_ready = NULL;
 493                 } else {
 494                         nand->dev_ready = sunxi_nfc_dev_ready;
 495                         if (sel->rb.type == RB_NATIVE)
 496                                 ctl |= NFC_RB_SEL(sel->rb.info.nativeid);
 497                 }
 498
 499                 writel(mtd->writesize, nfc->regs + NFC_REG_SPARE_AREA);
 500
 501                 if (nfc->clk_rate != sunxi_nand->clk_rate) {
 502                         clk_set_rate(nfc->mod_clk, sunxi_nand->clk_rate);
 503                         nfc->clk_rate = sunxi_nand->clk_rate;
 504                 }
 505         }
 506
 507         writel(sunxi_nand->timing_ctl, nfc->regs + NFC_REG_TIMING_CTL);
 508         writel(sunxi_nand->timing_cfg, nfc->regs + NFC_REG_TIMING_CFG);
 509         writel(ctl, nfc->regs + NFC_REG_CTL);
 510
 511         sunxi_nand->selected = chip;
 512 }
 513
 514 static void sunxi_nfc_read_buf(struct mtd_info *mtd, uint8_t *buf, int len)
 515 {
 516         struct nand_chip *nand = mtd_to_nand(mtd);
 517         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
 518         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
 519         int ret;
 520         int cnt;
 521         int offs = 0;
 522         u32 tmp;
 523
 524         while (len > offs) {
 525                 bool poll = false;
 526
 527                 cnt = min(len - offs, NFC_SRAM_SIZE);
 528
 529                 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
 530                 if (ret)
 531                         break;
 532
 533                 writel(cnt, nfc->regs + NFC_REG_CNT);
 534                 tmp = NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD;
 535                 writel(tmp, nfc->regs + NFC_REG_CMD);
 536
 537                 /* Arbitrary limit for polling mode */
 538                 if (cnt < 64)
 539                         poll = true;
 540
 541                 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, poll, 0);
 542                 if (ret)
 543                         break;
 544
 545                 if (buf)
 546                         memcpy_fromio(buf + offs, nfc->regs + NFC_RAM0_BASE,
 547                                       cnt);
 548                 offs += cnt;
 549         }
 550 }
 551
 552 static void sunxi_nfc_write_buf(struct mtd_info *mtd, const uint8_t *buf,
 553                                 int len)
 554 {
 555         struct nand_chip *nand = mtd_to_nand(mtd);
 556         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
 557         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
 558         int ret;
 559         int cnt;
 560         int offs = 0;
 561         u32 tmp;
 562
 563         while (len > offs) {
 564                 bool poll = false;
 565
 566                 cnt = min(len - offs, NFC_SRAM_SIZE);
 567
 568                 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
 569                 if (ret)
 570                         break;
 571
 572                 writel(cnt, nfc->regs + NFC_REG_CNT);
 573                 memcpy_toio(nfc->regs + NFC_RAM0_BASE, buf + offs, cnt);
 574                 tmp = NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD |
 575                       NFC_ACCESS_DIR;
 576                 writel(tmp, nfc->regs + NFC_REG_CMD);
 577
 578                 /* Arbitrary limit for polling mode */
 579                 if (cnt < 64)
 580                         poll = true;
 581
 582                 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, poll, 0);
 583                 if (ret)
 584                         break;
 585
 586                 offs += cnt;
 587         }
 588 }
 589
 590 static uint8_t sunxi_nfc_read_byte(struct mtd_info *mtd)
 591 {
 592         uint8_t ret;
 593
 594         sunxi_nfc_read_buf(mtd, &ret, 1);
 595
 596         return ret;
 597 }
 598
 599 static void sunxi_nfc_cmd_ctrl(struct mtd_info *mtd, int dat,
 600                                unsigned int ctrl)
 601 {
 602         struct nand_chip *nand = mtd_to_nand(mtd);
 603         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
 604         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
 605         int ret;
 606
 607         if (dat == NAND_CMD_NONE && (ctrl & NAND_NCE) &&
 608             !(ctrl & (NAND_CLE | NAND_ALE))) {
 609                 u32 cmd = 0;
 610
 611                 if (!sunxi_nand->addr_cycles && !sunxi_nand->cmd_cycles)
 612                         return;
 613
 614                 if (sunxi_nand->cmd_cycles--)
 615                         cmd |= NFC_SEND_CMD1 | sunxi_nand->cmd[0];
 616
 617                 if (sunxi_nand->cmd_cycles--) {
 618                         cmd |= NFC_SEND_CMD2;
 619                         writel(sunxi_nand->cmd[1],
 620                                nfc->regs + NFC_REG_RCMD_SET);
 621                 }
 622
 623                 sunxi_nand->cmd_cycles = 0;
 624
 625                 if (sunxi_nand->addr_cycles) {
 626                         cmd |= NFC_SEND_ADR |
 627                                NFC_ADR_NUM(sunxi_nand->addr_cycles);
 628                         writel(sunxi_nand->addr[0],
 629                                nfc->regs + NFC_REG_ADDR_LOW);
 630                 }
 631
 632                 if (sunxi_nand->addr_cycles > 4)
 633                         writel(sunxi_nand->addr[1],
 634                                nfc->regs + NFC_REG_ADDR_HIGH);
 635
 636                 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
 637                 if (ret)
 638                         return;
 639
 640                 writel(cmd, nfc->regs + NFC_REG_CMD);
 641                 sunxi_nand->addr[0] = 0;
 642                 sunxi_nand->addr[1] = 0;
 643                 sunxi_nand->addr_cycles = 0;
 644                 sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0);
 645         }
 646
 647         if (ctrl & NAND_CLE) {
 648                 sunxi_nand->cmd[sunxi_nand->cmd_cycles++] = dat;
 649         } else if (ctrl & NAND_ALE) {
 650                 sunxi_nand->addr[sunxi_nand->addr_cycles / 4] |=
 651                                 dat << ((sunxi_nand->addr_cycles % 4) * 8);
 652                 sunxi_nand->addr_cycles++;
 653         }
 654 }
 655
 656 /* These seed values have been extracted from Allwinner's BSP */
 657 static const u16 sunxi_nfc_randomizer_page_seeds[] = {
 658         0x2b75, 0x0bd0, 0x5ca3, 0x62d1, 0x1c93, 0x07e9, 0x2162, 0x3a72,
 659         0x0d67, 0x67f9, 0x1be7, 0x077d, 0x032f, 0x0dac, 0x2716, 0x2436,
 660         0x7922, 0x1510, 0x3860, 0x5287, 0x480f, 0x4252, 0x1789, 0x5a2d,
 661         0x2a49, 0x5e10, 0x437f, 0x4b4e, 0x2f45, 0x216e, 0x5cb7, 0x7130,
 662         0x2a3f, 0x60e4, 0x4dc9, 0x0ef0, 0x0f52, 0x1bb9, 0x6211, 0x7a56,
 663         0x226d, 0x4ea7, 0x6f36, 0x3692, 0x38bf, 0x0c62, 0x05eb, 0x4c55,
 664         0x60f4, 0x728c, 0x3b6f, 0x2037, 0x7f69, 0x0936, 0x651a, 0x4ceb,
 665         0x6218, 0x79f3, 0x383f, 0x18d9, 0x4f05, 0x5c82, 0x2912, 0x6f17,
 666         0x6856, 0x5938, 0x1007, 0x61ab, 0x3e7f, 0x57c2, 0x542f, 0x4f62,
 667         0x7454, 0x2eac, 0x7739, 0x42d4, 0x2f90, 0x435a, 0x2e52, 0x2064,
 668         0x637c, 0x66ad, 0x2c90, 0x0bad, 0x759c, 0x0029, 0x0986, 0x7126,
 669         0x1ca7, 0x1605, 0x386a, 0x27f5, 0x1380, 0x6d75, 0x24c3, 0x0f8e,
 670         0x2b7a, 0x1418, 0x1fd1, 0x7dc1, 0x2d8e, 0x43af, 0x2267, 0x7da3,
 671         0x4e3d, 0x1338, 0x50db, 0x454d, 0x764d, 0x40a3, 0x42e6, 0x262b,
 672         0x2d2e, 0x1aea, 0x2e17, 0x173d, 0x3a6e, 0x71bf, 0x25f9, 0x0a5d,
 673         0x7c57, 0x0fbe, 0x46ce, 0x4939, 0x6b17, 0x37bb, 0x3e91, 0x76db,
 674 };
 675
 676 /*
 677  * sunxi_nfc_randomizer_ecc512_seeds and sunxi_nfc_randomizer_ecc1024_seeds
 678  * have been generated using
 679  * sunxi_nfc_randomizer_step(seed, (step_size * 8) + 15), which is what
 680  * the randomizer engine does internally before de/scrambling OOB data.
 681  *
 682  * Those tables are statically defined to avoid calculating randomizer state
 683  * at runtime.
 684  */
 685 static const u16 sunxi_nfc_randomizer_ecc512_seeds[] = {
 686         0x3346, 0x367f, 0x1f18, 0x769a, 0x4f64, 0x068c, 0x2ef1, 0x6b64,
 687         0x28a9, 0x15d7, 0x30f8, 0x3659, 0x53db, 0x7c5f, 0x71d4, 0x4409,
 688         0x26eb, 0x03cc, 0x655d, 0x47d4, 0x4daa, 0x0877, 0x712d, 0x3617,
 689         0x3264, 0x49aa, 0x7f9e, 0x588e, 0x4fbc, 0x7176, 0x7f91, 0x6c6d,
 690         0x4b95, 0x5fb7, 0x3844, 0x4037, 0x0184, 0x081b, 0x0ee8, 0x5b91,
 691         0x293d, 0x1f71, 0x0e6f, 0x402b, 0x5122, 0x1e52, 0x22be, 0x3d2d,
 692         0x75bc, 0x7c60, 0x6291, 0x1a2f, 0x61d4, 0x74aa, 0x4140, 0x29ab,
 693         0x472d, 0x2852, 0x017e, 0x15e8, 0x5ec2, 0x17cf, 0x7d0f, 0x06b8,
 694         0x117a, 0x6b94, 0x789b, 0x3126, 0x6ac5, 0x5be7, 0x150f, 0x51f8,
 695         0x7889, 0x0aa5, 0x663d, 0x77e8, 0x0b87, 0x3dcb, 0x360d, 0x218b,
 696         0x512f, 0x7dc9, 0x6a4d, 0x630a, 0x3547, 0x1dd2, 0x5aea, 0x69a5,
 697         0x7bfa, 0x5e4f, 0x1519, 0x6430, 0x3a0e, 0x5eb3, 0x5425, 0x0c7a,
 698         0x5540, 0x3670, 0x63c1, 0x31e9, 0x5a39, 0x2de7, 0x5979, 0x2891,
 699         0x1562, 0x014b, 0x5b05, 0x2756, 0x5a34, 0x13aa, 0x6cb5, 0x2c36,
 700         0x5e72, 0x1306, 0x0861, 0x15ef, 0x1ee8, 0x5a37, 0x7ac4, 0x45dd,
 701         0x44c4, 0x7266, 0x2f41, 0x3ccc, 0x045e, 0x7d40, 0x7c66, 0x0fa0,
 702 };
 703
 704 static const u16 sunxi_nfc_randomizer_ecc1024_seeds[] = {
 705         0x2cf5, 0x35f1, 0x63a4, 0x5274, 0x2bd2, 0x778b, 0x7285, 0x32b6,
 706         0x6a5c, 0x70d6, 0x757d, 0x6769, 0x5375, 0x1e81, 0x0cf3, 0x3982,
 707         0x6787, 0x042a, 0x6c49, 0x1925, 0x56a8, 0x40a9, 0x063e, 0x7bd9,
 708         0x4dbf, 0x55ec, 0x672e, 0x7334, 0x5185, 0x4d00, 0x232a, 0x7e07,
 709         0x445d, 0x6b92, 0x528f, 0x4255, 0x53ba, 0x7d82, 0x2a2e, 0x3a4e,
 710         0x75eb, 0x450c, 0x6844, 0x1b5d, 0x581a, 0x4cc6, 0x0379, 0x37b2,
 711         0x419f, 0x0e92, 0x6b27, 0x5624, 0x01e3, 0x07c1, 0x44a5, 0x130c,
 712         0x13e8, 0x5910, 0x0876, 0x60c5, 0x54e3, 0x5b7f, 0x2269, 0x509f,
 713         0x7665, 0x36fd, 0x3e9a, 0x0579, 0x6295, 0x14ef, 0x0a81, 0x1bcc,
 714         0x4b16, 0x64db, 0x0514, 0x4f07, 0x0591, 0x3576, 0x6853, 0x0d9e,
 715         0x259f, 0x38b7, 0x64fb, 0x3094, 0x4693, 0x6ddd, 0x29bb, 0x0bc8,
 716         0x3f47, 0x490e, 0x0c0e, 0x7933, 0x3c9e, 0x5840, 0x398d, 0x3e68,
 717         0x4af1, 0x71f5, 0x57cf, 0x1121, 0x64eb, 0x3579, 0x15ac, 0x584d,
 718         0x5f2a, 0x47e2, 0x6528, 0x6eac, 0x196e, 0x6b96, 0x0450, 0x0179,
 719         0x609c, 0x06e1, 0x4626, 0x42c7, 0x273e, 0x486f, 0x0705, 0x1601,
 720         0x145b, 0x407e, 0x062b, 0x57a5, 0x53f9, 0x5659, 0x4410, 0x3ccd,
 721 };
 722
 723 static u16 sunxi_nfc_randomizer_step(u16 state, int count)
 724 {
 725         state &= 0x7fff;
 726
 727         /*
 728          * This loop is just a simple implementation of a Fibonacci LFSR using
 729          * the x16 + x15 + 1 polynomial.
 730          */
 731         while (count--)
 732                 state = ((state >> 1) |
 733                          (((state ^ (state >> 1)) & 1) << 14)) & 0x7fff;
 734
 735         return state;
 736 }
 737
 738 static u16 sunxi_nfc_randomizer_state(struct mtd_info *mtd, int page, bool ecc)
 739 {
 740         const u16 *seeds = sunxi_nfc_randomizer_page_seeds;
 741         int mod = mtd_div_by_ws(mtd->erasesize, mtd);
 742
 743         if (mod > ARRAY_SIZE(sunxi_nfc_randomizer_page_seeds))
 744                 mod = ARRAY_SIZE(sunxi_nfc_randomizer_page_seeds);
 745
 746         if (ecc) {
 747                 if (mtd->ecc_step_size == 512)
 748                         seeds = sunxi_nfc_randomizer_ecc512_seeds;
 749                 else
 750                         seeds = sunxi_nfc_randomizer_ecc1024_seeds;
 751         }
 752
 753         return seeds[page % mod];
 754 }
 755
 756 static void sunxi_nfc_randomizer_config(struct mtd_info *mtd,
 757                                         int page, bool ecc)
 758 {
 759         struct nand_chip *nand = mtd_to_nand(mtd);
 760         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 761         u32 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
 762         u16 state;
 763
 764         if (!(nand->options & NAND_NEED_SCRAMBLING))
 765                 return;
 766
 767         ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
 768         state = sunxi_nfc_randomizer_state(mtd, page, ecc);
 769         ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_RANDOM_SEED_MSK;
 770         writel(ecc_ctl | NFC_RANDOM_SEED(state), nfc->regs + NFC_REG_ECC_CTL);
 771 }
 772
 773 static void sunxi_nfc_randomizer_enable(struct mtd_info *mtd)
 774 {
 775         struct nand_chip *nand = mtd_to_nand(mtd);
 776         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 777
 778         if (!(nand->options & NAND_NEED_SCRAMBLING))
 779                 return;
 780
 781         writel(readl(nfc->regs + NFC_REG_ECC_CTL) | NFC_RANDOM_EN,
 782                nfc->regs + NFC_REG_ECC_CTL);
 783 }
 784
 785 static void sunxi_nfc_randomizer_disable(struct mtd_info *mtd)
 786 {
 787         struct nand_chip *nand = mtd_to_nand(mtd);
 788         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 789
 790         if (!(nand->options & NAND_NEED_SCRAMBLING))
 791                 return;
 792
 793         writel(readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_RANDOM_EN,
 794                nfc->regs + NFC_REG_ECC_CTL);
 795 }
 796
 797 static void sunxi_nfc_randomize_bbm(struct mtd_info *mtd, int page, u8 *bbm)
 798 {
 799         u16 state = sunxi_nfc_randomizer_state(mtd, page, true);
 800
 801         bbm[0] ^= state;
 802         bbm[1] ^= sunxi_nfc_randomizer_step(state, 8);
 803 }
 804
 805 static void sunxi_nfc_randomizer_write_buf(struct mtd_info *mtd,
 806                                            const uint8_t *buf, int len,
 807                                            bool ecc, int page)
 808 {
 809         sunxi_nfc_randomizer_config(mtd, page, ecc);
 810         sunxi_nfc_randomizer_enable(mtd);
 811         sunxi_nfc_write_buf(mtd, buf, len);
 812         sunxi_nfc_randomizer_disable(mtd);
 813 }
 814
 815 static void sunxi_nfc_randomizer_read_buf(struct mtd_info *mtd, uint8_t *buf,
 816                                           int len, bool ecc, int page)
 817 {
 818         sunxi_nfc_randomizer_config(mtd, page, ecc);
 819         sunxi_nfc_randomizer_enable(mtd);
 820         sunxi_nfc_read_buf(mtd, buf, len);
 821         sunxi_nfc_randomizer_disable(mtd);
 822 }
 823
 824 static void sunxi_nfc_hw_ecc_enable(struct mtd_info *mtd)
 825 {
 826         struct nand_chip *nand = mtd_to_nand(mtd);
 827         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 828         struct sunxi_nand_hw_ecc *data = nand->ecc.priv;
 829         u32 ecc_ctl;
 830
 831         ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
 832         ecc_ctl &= ~(NFC_ECC_MODE_MSK | NFC_ECC_PIPELINE |
 833                      NFC_ECC_BLOCK_SIZE_MSK);
 834         ecc_ctl |= NFC_ECC_EN | NFC_ECC_MODE(data->mode) | NFC_ECC_EXCEPTION |
 835                    NFC_ECC_PIPELINE;
 836
 837         if (nand->ecc.size == 512)
 838                 ecc_ctl |= NFC_ECC_BLOCK_512;
 839
 840         writel(ecc_ctl, nfc->regs + NFC_REG_ECC_CTL);
 841 }
 842
 843 static void sunxi_nfc_hw_ecc_disable(struct mtd_info *mtd)
 844 {
 845         struct nand_chip *nand = mtd_to_nand(mtd);
 846         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 847
 848         writel(readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_ECC_EN,
 849                nfc->regs + NFC_REG_ECC_CTL);
 850 }
 851
 852 static inline void sunxi_nfc_user_data_to_buf(u32 user_data, u8 *buf)
 853 {
 854         buf[0] = user_data;
 855         buf[1] = user_data >> 8;
 856         buf[2] = user_data >> 16;
 857         buf[3] = user_data >> 24;
 858 }
 859
 860 static inline u32 sunxi_nfc_buf_to_user_data(const u8 *buf)
 861 {
 862         return buf[0] | (buf[1] << 8) | (buf[2] << 16) | (buf[3] << 24);
 863 }
 864
 865 static void sunxi_nfc_hw_ecc_get_prot_oob_bytes(struct mtd_info *mtd, u8 *oob,
 866                                                 int step, bool bbm, int page)
 867 {
 868         struct nand_chip *nand = mtd_to_nand(mtd);
 869         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 870
 871         sunxi_nfc_user_data_to_buf(readl(nfc->regs + NFC_REG_USER_DATA(step)),
 872                                    oob);
 873
 874         /* De-randomize the Bad Block Marker. */
 875         if (bbm && (nand->options & NAND_NEED_SCRAMBLING))
 876                 sunxi_nfc_randomize_bbm(mtd, page, oob);
 877 }
 878
 879 static void sunxi_nfc_hw_ecc_set_prot_oob_bytes(struct mtd_info *mtd,
 880                                                 const u8 *oob, int step,
 881                                                 bool bbm, int page)
 882 {
 883         struct nand_chip *nand = mtd_to_nand(mtd);
 884         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 885         u8 user_data[4];
 886
 887         /* Randomize the Bad Block Marker. */
 888         if (bbm && (nand->options & NAND_NEED_SCRAMBLING)) {
 889                 memcpy(user_data, oob, sizeof(user_data));
 890                 sunxi_nfc_randomize_bbm(mtd, page, user_data);
 891                 oob = user_data;
 892         }
 893
 894         writel(sunxi_nfc_buf_to_user_data(oob),
 895                nfc->regs + NFC_REG_USER_DATA(step));
 896 }
 897
 898 static void sunxi_nfc_hw_ecc_update_stats(struct mtd_info *mtd,
 899                                           unsigned int *max_bitflips, int ret)
 900 {
 901         if (ret < 0) {
 902                 mtd->ecc_stats.failed++;
 903         } else {
 904                 mtd->ecc_stats.corrected += ret;
 905                 *max_bitflips = max_t(unsigned int, *max_bitflips, ret);
 906         }
 907 }
 908
 909 static int sunxi_nfc_hw_ecc_correct(struct mtd_info *mtd, u8 *data, u8 *oob,
 910                                     int step, u32 status, bool *erased)
 911 {
 912         struct nand_chip *nand = mtd_to_nand(mtd);
 913         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 914         struct nand_ecc_ctrl *ecc = &nand->ecc;
 915         u32 tmp;
 916
 917         *erased = false;
 918
 919         if (status & NFC_ECC_ERR(step))
 920                 return -EBADMSG;
 921
 922         if (status & NFC_ECC_PAT_FOUND(step)) {
 923                 u8 pattern;
 924
 925                 if (unlikely(!(readl(nfc->regs + NFC_REG_PAT_ID) & 0x1))) {
 926                         pattern = 0x0;
 927                 } else {
 928                         pattern = 0xff;
 929                         *erased = true;
 930                 }
 931
 932                 if (data)
 933                         memset(data, pattern, ecc->size);
 934
 935                 if (oob)
 936                         memset(oob, pattern, ecc->bytes + 4);
 937
 938                 return 0;
 939         }
 940
 941         tmp = readl(nfc->regs + NFC_REG_ECC_ERR_CNT(step));
 942
 943         return NFC_ECC_ERR_CNT(step, tmp);
 944 }
 945
 946 static int sunxi_nfc_hw_ecc_read_chunk(struct mtd_info *mtd,
 947                                        u8 *data, int data_off,
 948                                        u8 *oob, int oob_off,
 949                                        int *cur_off,
 950                                        unsigned int *max_bitflips,
 951                                        bool bbm, bool oob_required, int page)
 952 {
 953         struct nand_chip *nand = mtd_to_nand(mtd);
 954         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 955         struct nand_ecc_ctrl *ecc = &nand->ecc;
 956         int raw_mode = 0;
 957         bool erased;
 958         int ret;
 959
 960         if (*cur_off != data_off)
 961                 nand->cmdfunc(mtd, NAND_CMD_RNDOUT, data_off, -1);
 962
 963         sunxi_nfc_randomizer_read_buf(mtd, NULL, ecc->size, false, page);
 964
 965         if (data_off + ecc->size != oob_off)
 966                 nand->cmdfunc(mtd, NAND_CMD_RNDOUT, oob_off, -1);
 967
 968         ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
 969         if (ret)
 970                 return ret;
 971
 972         sunxi_nfc_randomizer_enable(mtd);
 973         writel(NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD | NFC_ECC_OP,
 974                nfc->regs + NFC_REG_CMD);
 975
 976         ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0);
 977         sunxi_nfc_randomizer_disable(mtd);
 978         if (ret)
 979                 return ret;
 980
 981         *cur_off = oob_off + ecc->bytes + 4;
 982
 983         ret = sunxi_nfc_hw_ecc_correct(mtd, data, oob_required ? oob : NULL, 0,
 984                                        readl(nfc->regs + NFC_REG_ECC_ST),
 985                                        &erased);
 986         if (erased)
 987                 return 1;
 988
 989         if (ret < 0) {
 990                 /*
 991                  * Re-read the data with the randomizer disabled to identify
 992                  * bitflips in erased pages.
 993                  */
 994                 if (nand->options & NAND_NEED_SCRAMBLING) {
 995                         nand->cmdfunc(mtd, NAND_CMD_RNDOUT, data_off, -1);
 996                         nand->read_buf(mtd, data, ecc->size);
 997                 } else {
 998                         memcpy_fromio(data, nfc->regs + NFC_RAM0_BASE,
 999                                       ecc->size);
1000                 }
1001
1002                 nand->cmdfunc(mtd, NAND_CMD_RNDOUT, oob_off, -1);
1003                 nand->read_buf(mtd, oob, ecc->bytes + 4);
1004
1005                 ret = nand_check_erased_ecc_chunk(data, ecc->size,
1006                                                   oob, ecc->bytes + 4,
1007                                                   NULL, 0, ecc->strength);
1008                 if (ret >= 0)
1009                         raw_mode = 1;
1010         } else {
1011                 memcpy_fromio(data, nfc->regs + NFC_RAM0_BASE, ecc->size);
1012
1013                 if (oob_required) {
1014                         nand->cmdfunc(mtd, NAND_CMD_RNDOUT, oob_off, -1);
1015                         sunxi_nfc_randomizer_read_buf(mtd, oob, ecc->bytes + 4,
1016                                                       true, page);
1017
1018                         sunxi_nfc_hw_ecc_get_prot_oob_bytes(mtd, oob, 0,
1019                                                             bbm, page);
1020                 }
1021         }
1022
1023         sunxi_nfc_hw_ecc_update_stats(mtd, max_bitflips, ret);
1024
1025         return raw_mode;
1026 }
1027
1028 static void sunxi_nfc_hw_ecc_read_extra_oob(struct mtd_info *mtd,
1029                                             u8 *oob, int *cur_off,
1030                                             bool randomize, int page)
1031 {
1032         struct nand_chip *nand = mtd_to_nand(mtd);
1033         struct nand_ecc_ctrl *ecc = &nand->ecc;
1034         int offset = ((ecc->bytes + 4) * ecc->steps);
1035         int len = mtd->oobsize - offset;
1036
1037         if (len <= 0)
1038                 return;
1039
1040         if (!cur_off || *cur_off != offset)
1041                 nand->cmdfunc(mtd, NAND_CMD_RNDOUT,
1042                               offset + mtd->writesize, -1);
1043
1044         if (!randomize)
1045                 sunxi_nfc_read_buf(mtd, oob + offset, len);
1046         else
1047                 sunxi_nfc_randomizer_read_buf(mtd, oob + offset, len,
1048                                               false, page);
1049
1050         if (cur_off)
1051                 *cur_off = mtd->oobsize + mtd->writesize;
1052 }
1053
1054 static int sunxi_nfc_hw_ecc_read_chunks_dma(struct mtd_info *mtd, uint8_t *buf,
1055                                             int oob_required, int page,
1056                                             int nchunks)
1057 {
1058         struct nand_chip *nand = mtd_to_nand(mtd);
1059         bool randomized = nand->options & NAND_NEED_SCRAMBLING;
1060         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1061         struct nand_ecc_ctrl *ecc = &nand->ecc;
1062         unsigned int max_bitflips = 0;
1063         int ret, i, raw_mode = 0;
1064         struct scatterlist sg;
1065         u32 status;
1066
1067         ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1068         if (ret)
1069                 return ret;
1070
1071         ret = sunxi_nfc_dma_op_prepare(mtd, buf, ecc->size, nchunks,
1072                                        DMA_FROM_DEVICE, &sg);
1073         if (ret)
1074                 return ret;
1075
1076         sunxi_nfc_hw_ecc_enable(mtd);
1077         sunxi_nfc_randomizer_config(mtd, page, false);
1078         sunxi_nfc_randomizer_enable(mtd);
1079
1080         writel((NAND_CMD_RNDOUTSTART << 16) | (NAND_CMD_RNDOUT << 8) |
1081                NAND_CMD_READSTART, nfc->regs + NFC_REG_RCMD_SET);
1082
1083         dma_async_issue_pending(nfc->dmac);
1084
1085         writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD | NFC_DATA_TRANS,
1086                nfc->regs + NFC_REG_CMD);
1087
1088         ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0);
1089         if (ret)
1090                 dmaengine_terminate_all(nfc->dmac);
1091
1092         sunxi_nfc_randomizer_disable(mtd);
1093         sunxi_nfc_hw_ecc_disable(mtd);
1094
1095         sunxi_nfc_dma_op_cleanup(mtd, DMA_FROM_DEVICE, &sg);
1096
1097         if (ret)
1098                 return ret;
1099
1100         status = readl(nfc->regs + NFC_REG_ECC_ST);
1101
1102         for (i = 0; i < nchunks; i++) {
1103                 int data_off = i * ecc->size;
1104                 int oob_off = i * (ecc->bytes + 4);
1105                 u8 *data = buf + data_off;
1106                 u8 *oob = nand->oob_poi + oob_off;
1107                 bool erased;
1108
1109                 ret = sunxi_nfc_hw_ecc_correct(mtd, randomized ? data : NULL,
1110                                                oob_required ? oob : NULL,
1111                                                i, status, &erased);
1112
1113                 /* ECC errors are handled in the second loop. */
1114                 if (ret < 0)
1115                         continue;
1116
1117                 if (oob_required && !erased) {
1118                         /* TODO: use DMA to retrieve OOB */
1119                         nand->cmdfunc(mtd, NAND_CMD_RNDOUT,
1120                                       mtd->writesize + oob_off, -1);
1121                         nand->read_buf(mtd, oob, ecc->bytes + 4);
1122
1123                         sunxi_nfc_hw_ecc_get_prot_oob_bytes(mtd, oob, i,
1124                                                             !i, page);
1125                 }
1126
1127                 if (erased)
1128                         raw_mode = 1;
1129
1130                 sunxi_nfc_hw_ecc_update_stats(mtd, &max_bitflips, ret);
1131         }
1132
1133         if (status & NFC_ECC_ERR_MSK) {
1134                 for (i = 0; i < nchunks; i++) {
1135                         int data_off = i * ecc->size;
1136                         int oob_off = i * (ecc->bytes + 4);
1137                         u8 *data = buf + data_off;
1138                         u8 *oob = nand->oob_poi + oob_off;
1139
1140                         if (!(status & NFC_ECC_ERR(i)))
1141                                 continue;
1142
1143                         /*
1144                          * Re-read the data with the randomizer disabled to
1145                          * identify bitflips in erased pages.
1146                          */
1147                         if (randomized) {
1148                                 /* TODO: use DMA to read page in raw mode */
1149                                 nand->cmdfunc(mtd, NAND_CMD_RNDOUT,
1150                                               data_off, -1);
1151                                 nand->read_buf(mtd, data, ecc->size);
1152                         }
1153
1154                         /* TODO: use DMA to retrieve OOB */
1155                         nand->cmdfunc(mtd, NAND_CMD_RNDOUT,
1156                                       mtd->writesize + oob_off, -1);
1157                         nand->read_buf(mtd, oob, ecc->bytes + 4);
1158
1159                         ret = nand_check_erased_ecc_chunk(data, ecc->size,
1160                                                           oob, ecc->bytes + 4,
1161                                                           NULL, 0,
1162                                                           ecc->strength);
1163                         if (ret >= 0)
1164                                 raw_mode = 1;
1165
1166                         sunxi_nfc_hw_ecc_update_stats(mtd, &max_bitflips, ret);
1167                 }
1168         }
1169
1170         if (oob_required)
1171                 sunxi_nfc_hw_ecc_read_extra_oob(mtd, nand->oob_poi,
1172                                                 NULL, !raw_mode,
1173                                                 page);
1174
1175         return max_bitflips;
1176 }
1177
1178 static int sunxi_nfc_hw_ecc_write_chunk(struct mtd_info *mtd,
1179                                         const u8 *data, int data_off,
1180                                         const u8 *oob, int oob_off,
1181                                         int *cur_off, bool bbm,
1182                                         int page)
1183 {
1184         struct nand_chip *nand = mtd_to_nand(mtd);
1185         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1186         struct nand_ecc_ctrl *ecc = &nand->ecc;
1187         int ret;
1188
1189         if (data_off != *cur_off)
1190                 nand->cmdfunc(mtd, NAND_CMD_RNDIN, data_off, -1);
1191
1192         sunxi_nfc_randomizer_write_buf(mtd, data, ecc->size, false, page);
1193
1194         if (data_off + ecc->size != oob_off)
1195                 nand->cmdfunc(mtd, NAND_CMD_RNDIN, oob_off, -1);
1196
1197         ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1198         if (ret)
1199                 return ret;
1200
1201         sunxi_nfc_randomizer_enable(mtd);
1202         sunxi_nfc_hw_ecc_set_prot_oob_bytes(mtd, oob, 0, bbm, page);
1203
1204         writel(NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD |
1205                NFC_ACCESS_DIR | NFC_ECC_OP,
1206                nfc->regs + NFC_REG_CMD);
1207
1208         ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0);
1209         sunxi_nfc_randomizer_disable(mtd);
1210         if (ret)
1211                 return ret;
1212
1213         *cur_off = oob_off + ecc->bytes + 4;
1214
1215         return 0;
1216 }
1217
1218 static void sunxi_nfc_hw_ecc_write_extra_oob(struct mtd_info *mtd,
1219                                              u8 *oob, int *cur_off,
1220                                              int page)
1221 {
1222         struct nand_chip *nand = mtd_to_nand(mtd);
1223         struct nand_ecc_ctrl *ecc = &nand->ecc;
1224         int offset = ((ecc->bytes + 4) * ecc->steps);
1225         int len = mtd->oobsize - offset;
1226
1227         if (len <= 0)
1228                 return;
1229
1230         if (!cur_off || *cur_off != offset)
1231                 nand->cmdfunc(mtd, NAND_CMD_RNDIN,
1232                               offset + mtd->writesize, -1);
1233
1234         sunxi_nfc_randomizer_write_buf(mtd, oob + offset, len, false, page);
1235
1236         if (cur_off)
1237                 *cur_off = mtd->oobsize + mtd->writesize;
1238 }
1239
1240 static int sunxi_nfc_hw_ecc_read_page(struct mtd_info *mtd,
1241                                       struct nand_chip *chip, uint8_t *buf,
1242                                       int oob_required, int page)
1243 {
1244         struct nand_ecc_ctrl *ecc = &chip->ecc;
1245         unsigned int max_bitflips = 0;
1246         int ret, i, cur_off = 0;
1247         bool raw_mode = false;
1248
1249         sunxi_nfc_hw_ecc_enable(mtd);
1250
1251         for (i = 0; i < ecc->steps; i++) {
1252                 int data_off = i * ecc->size;
1253                 int oob_off = i * (ecc->bytes + 4);
1254                 u8 *data = buf + data_off;
1255                 u8 *oob = chip->oob_poi + oob_off;
1256
1257                 ret = sunxi_nfc_hw_ecc_read_chunk(mtd, data, data_off, oob,
1258                                                   oob_off + mtd->writesize,
1259                                                   &cur_off, &max_bitflips,
1260                                                   !i, oob_required, page);
1261                 if (ret < 0)
1262                         return ret;
1263                 else if (ret)
1264                         raw_mode = true;
1265         }
1266
1267         if (oob_required)
1268                 sunxi_nfc_hw_ecc_read_extra_oob(mtd, chip->oob_poi, &cur_off,
1269                                                 !raw_mode, page);
1270
1271         sunxi_nfc_hw_ecc_disable(mtd);
1272
1273         return max_bitflips;
1274 }
1275
1276 static int sunxi_nfc_hw_ecc_read_page_dma(struct mtd_info *mtd,
1277                                           struct nand_chip *chip, u8 *buf,
1278                                           int oob_required, int page)
1279 {
1280         int ret;
1281
1282         ret = sunxi_nfc_hw_ecc_read_chunks_dma(mtd, buf, oob_required, page,
1283                                                chip->ecc.steps);
1284         if (ret >= 0)
1285                 return ret;
1286
1287         /* Fallback to PIO mode */
1288         chip->cmdfunc(mtd, NAND_CMD_RNDOUT, 0, -1);
1289
1290         return sunxi_nfc_hw_ecc_read_page(mtd, chip, buf, oob_required, page);
1291 }
1292
1293 static int sunxi_nfc_hw_ecc_read_subpage(struct mtd_info *mtd,
1294                                          struct nand_chip *chip,
1295                                          u32 data_offs, u32 readlen,
1296                                          u8 *bufpoi, int page)
1297 {
1298         struct nand_ecc_ctrl *ecc = &chip->ecc;
1299         int ret, i, cur_off = 0;
1300         unsigned int max_bitflips = 0;
1301
1302         sunxi_nfc_hw_ecc_enable(mtd);
1303
1304         chip->cmdfunc(mtd, NAND_CMD_READ0, 0, page);
1305         for (i = data_offs / ecc->size;
1306              i < DIV_ROUND_UP(data_offs + readlen, ecc->size); i++) {
1307                 int data_off = i * ecc->size;
1308                 int oob_off = i * (ecc->bytes + 4);
1309                 u8 *data = bufpoi + data_off;
1310                 u8 *oob = chip->oob_poi + oob_off;
1311
1312                 ret = sunxi_nfc_hw_ecc_read_chunk(mtd, data, data_off,
1313                                                   oob,
1314                                                   oob_off + mtd->writesize,
1315                                                   &cur_off, &max_bitflips, !i,
1316                                                   false, page);
1317                 if (ret < 0)
1318                         return ret;
1319         }
1320
1321         sunxi_nfc_hw_ecc_disable(mtd);
1322
1323         return max_bitflips;
1324 }
1325
1326 static int sunxi_nfc_hw_ecc_read_subpage_dma(struct mtd_info *mtd,
1327                                              struct nand_chip *chip,
1328                                              u32 data_offs, u32 readlen,
1329                                              u8 *buf, int page)
1330 {
1331         int nchunks = DIV_ROUND_UP(data_offs + readlen, chip->ecc.size);
1332         int ret;
1333
1334         ret = sunxi_nfc_hw_ecc_read_chunks_dma(mtd, buf, false, page, nchunks);
1335         if (ret >= 0)
1336                 return ret;
1337
1338         /* Fallback to PIO mode */
1339         chip->cmdfunc(mtd, NAND_CMD_RNDOUT, 0, -1);
1340
1341         return sunxi_nfc_hw_ecc_read_subpage(mtd, chip, data_offs, readlen,
1342                                              buf, page);
1343 }
1344
1345 static int sunxi_nfc_hw_ecc_write_page(struct mtd_info *mtd,
1346                                        struct nand_chip *chip,
1347                                        const uint8_t *buf, int oob_required,
1348                                        int page)
1349 {
1350         struct nand_ecc_ctrl *ecc = &chip->ecc;
1351         int ret, i, cur_off = 0;
1352
1353         sunxi_nfc_hw_ecc_enable(mtd);
1354
1355         for (i = 0; i < ecc->steps; i++) {
1356                 int data_off = i * ecc->size;
1357                 int oob_off = i * (ecc->bytes + 4);
1358                 const u8 *data = buf + data_off;
1359                 const u8 *oob = chip->oob_poi + oob_off;
1360
1361                 ret = sunxi_nfc_hw_ecc_write_chunk(mtd, data, data_off, oob,
1362                                                    oob_off + mtd->writesize,
1363                                                    &cur_off, !i, page);
1364                 if (ret)
1365                         return ret;
1366         }
1367
1368         if (oob_required || (chip->options & NAND_NEED_SCRAMBLING))
1369                 sunxi_nfc_hw_ecc_write_extra_oob(mtd, chip->oob_poi,
1370                                                  &cur_off, page);
1371
1372         sunxi_nfc_hw_ecc_disable(mtd);
1373
1374         return 0;
1375 }
1376
1377 static int sunxi_nfc_hw_ecc_write_subpage(struct mtd_info *mtd,
1378                                           struct nand_chip *chip,
1379                                           u32 data_offs, u32 data_len,
1380                                           const u8 *buf, int oob_required,
1381                                           int page)
1382 {
1383         struct nand_ecc_ctrl *ecc = &chip->ecc;
1384         int ret, i, cur_off = 0;
1385
1386         sunxi_nfc_hw_ecc_enable(mtd);
1387
1388         for (i = data_offs / ecc->size;
1389              i < DIV_ROUND_UP(data_offs + data_len, ecc->size); i++) {
1390                 int data_off = i * ecc->size;
1391                 int oob_off = i * (ecc->bytes + 4);
1392                 const u8 *data = buf + data_off;
1393                 const u8 *oob = chip->oob_poi + oob_off;
1394
1395                 ret = sunxi_nfc_hw_ecc_write_chunk(mtd, data, data_off, oob,
1396                                                    oob_off + mtd->writesize,
1397                                                    &cur_off, !i, page);
1398                 if (ret)
1399                         return ret;
1400         }
1401
1402         sunxi_nfc_hw_ecc_disable(mtd);
1403
1404         return 0;
1405 }
1406
1407 static int sunxi_nfc_hw_ecc_write_page_dma(struct mtd_info *mtd,
1408                                            struct nand_chip *chip,
1409                                            const u8 *buf,
1410                                            int oob_required,
1411                                            int page)
1412 {
1413         struct nand_chip *nand = mtd_to_nand(mtd);
1414         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1415         struct nand_ecc_ctrl *ecc = &nand->ecc;
1416         struct scatterlist sg;
1417         int ret, i;
1418
1419         ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1420         if (ret)
1421                 return ret;
1422
1423         ret = sunxi_nfc_dma_op_prepare(mtd, buf, ecc->size, ecc->steps,
1424                                        DMA_TO_DEVICE, &sg);
1425         if (ret)
1426                 goto pio_fallback;
1427
1428         for (i = 0; i < ecc->steps; i++) {
1429                 const u8 *oob = nand->oob_poi + (i * (ecc->bytes + 4));
1430
1431                 sunxi_nfc_hw_ecc_set_prot_oob_bytes(mtd, oob, i, !i, page);
1432         }
1433
1434         sunxi_nfc_hw_ecc_enable(mtd);
1435         sunxi_nfc_randomizer_config(mtd, page, false);
1436         sunxi_nfc_randomizer_enable(mtd);
1437
1438         writel((NAND_CMD_RNDIN << 8) | NAND_CMD_PAGEPROG,
1439                nfc->regs + NFC_REG_RCMD_SET);
1440
1441         dma_async_issue_pending(nfc->dmac);
1442
1443         writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD |
1444                NFC_DATA_TRANS | NFC_ACCESS_DIR,
1445                nfc->regs + NFC_REG_CMD);
1446
1447         ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0);
1448         if (ret)
1449                 dmaengine_terminate_all(nfc->dmac);
1450
1451         sunxi_nfc_randomizer_disable(mtd);
1452         sunxi_nfc_hw_ecc_disable(mtd);
1453
1454         sunxi_nfc_dma_op_cleanup(mtd, DMA_TO_DEVICE, &sg);
1455
1456         if (ret)
1457                 return ret;
1458
1459         if (oob_required || (chip->options & NAND_NEED_SCRAMBLING))
1460                 /* TODO: use DMA to transfer extra OOB bytes ? */
1461                 sunxi_nfc_hw_ecc_write_extra_oob(mtd, chip->oob_poi,
1462                                                  NULL, page);
1463
1464         return 0;
1465
1466 pio_fallback:
1467         return sunxi_nfc_hw_ecc_write_page(mtd, chip, buf, oob_required, page);
1468 }
1469
1470 static int sunxi_nfc_hw_syndrome_ecc_read_page(struct mtd_info *mtd,
1471                                                struct nand_chip *chip,
1472                                                uint8_t *buf, int oob_required,
1473                                                int page)
1474 {
1475         struct nand_ecc_ctrl *ecc = &chip->ecc;
1476         unsigned int max_bitflips = 0;
1477         int ret, i, cur_off = 0;
1478         bool raw_mode = false;
1479
1480         sunxi_nfc_hw_ecc_enable(mtd);
1481
1482         for (i = 0; i < ecc->steps; i++) {
1483                 int data_off = i * (ecc->size + ecc->bytes + 4);
1484                 int oob_off = data_off + ecc->size;
1485                 u8 *data = buf + (i * ecc->size);
1486                 u8 *oob = chip->oob_poi + (i * (ecc->bytes + 4));
1487
1488                 ret = sunxi_nfc_hw_ecc_read_chunk(mtd, data, data_off, oob,
1489                                                   oob_off, &cur_off,
1490                                                   &max_bitflips, !i,
1491                                                   oob_required,
1492                                                   page);
1493                 if (ret < 0)
1494                         return ret;
1495                 else if (ret)
1496                         raw_mode = true;
1497         }
1498
1499         if (oob_required)
1500                 sunxi_nfc_hw_ecc_read_extra_oob(mtd, chip->oob_poi, &cur_off,
1501                                                 !raw_mode, page);
1502
1503         sunxi_nfc_hw_ecc_disable(mtd);
1504
1505         return max_bitflips;
1506 }
1507
1508 static int sunxi_nfc_hw_syndrome_ecc_write_page(struct mtd_info *mtd,
1509                                                 struct nand_chip *chip,
1510                                                 const uint8_t *buf,
1511                                                 int oob_required, int page)
1512 {
1513         struct nand_ecc_ctrl *ecc = &chip->ecc;
1514         int ret, i, cur_off = 0;
1515
1516         sunxi_nfc_hw_ecc_enable(mtd);
1517
1518         for (i = 0; i < ecc->steps; i++) {
1519                 int data_off = i * (ecc->size + ecc->bytes + 4);
1520                 int oob_off = data_off + ecc->size;
1521                 const u8 *data = buf + (i * ecc->size);
1522                 const u8 *oob = chip->oob_poi + (i * (ecc->bytes + 4));
1523
1524                 ret = sunxi_nfc_hw_ecc_write_chunk(mtd, data, data_off,
1525                                                    oob, oob_off, &cur_off,
1526                                                    false, page);
1527                 if (ret)
1528                         return ret;
1529         }
1530
1531         if (oob_required || (chip->options & NAND_NEED_SCRAMBLING))
1532                 sunxi_nfc_hw_ecc_write_extra_oob(mtd, chip->oob_poi,
1533                                                  &cur_off, page);
1534
1535         sunxi_nfc_hw_ecc_disable(mtd);
1536
1537         return 0;
1538 }
1539
1540 static int sunxi_nfc_hw_common_ecc_read_oob(struct mtd_info *mtd,
1541                                             struct nand_chip *chip,
1542                                             int page)
1543 {
1544         chip->cmdfunc(mtd, NAND_CMD_READ0, 0, page);
1545
1546         chip->pagebuf = -1;
1547
1548         return chip->ecc.read_page(mtd, chip, chip->buffers->databuf, 1, page);
1549 }
1550
1551 static int sunxi_nfc_hw_common_ecc_write_oob(struct mtd_info *mtd,
1552                                              struct nand_chip *chip,
1553                                              int page)
1554 {
1555         int ret, status;
1556
1557         chip->cmdfunc(mtd, NAND_CMD_SEQIN, 0, page);
1558
1559         chip->pagebuf = -1;
1560
1561         memset(chip->buffers->databuf, 0xff, mtd->writesize);
1562         ret = chip->ecc.write_page(mtd, chip, chip->buffers->databuf, 1, page);
1563         if (ret)
1564                 return ret;
1565
1566         /* Send command to program the OOB data */
1567         chip->cmdfunc(mtd, NAND_CMD_PAGEPROG, -1, -1);
1568
1569         status = chip->waitfunc(mtd, chip);
1570
1571         return status & NAND_STATUS_FAIL ? -EIO : 0;
1572 }
1573
1574 static const s32 tWB_lut[] = {6, 12, 16, 20};
1575 static const s32 tRHW_lut[] = {4, 8, 12, 20};
1576
1577 static int _sunxi_nand_lookup_timing(const s32 *lut, int lut_size, u32 duration,
1578                 u32 clk_period)
1579 {
1580         u32 clk_cycles = DIV_ROUND_UP(duration, clk_period);
1581         int i;
1582
1583         for (i = 0; i < lut_size; i++) {
1584                 if (clk_cycles <= lut[i])
1585                         return i;
1586         }
1587
1588         /* Doesn't fit */
1589         return -EINVAL;
1590 }
1591
1592 #define sunxi_nand_lookup_timing(l, p, c) \
1593                         _sunxi_nand_lookup_timing(l, ARRAY_SIZE(l), p, c)
1594
1595 static int sunxi_nfc_setup_data_interface(struct mtd_info *mtd,
1596                                         const struct nand_data_interface *conf,
1597                                         bool check_only)
1598 {
1599         struct nand_chip *nand = mtd_to_nand(mtd);
1600         struct sunxi_nand_chip *chip = to_sunxi_nand(nand);
1601         struct sunxi_nfc *nfc = to_sunxi_nfc(chip->nand.controller);
1602         const struct nand_sdr_timings *timings;
1603         u32 min_clk_period = 0;
1604         s32 tWB, tADL, tWHR, tRHW, tCAD;
1605         long real_clk_rate;
1606
1607         timings = nand_get_sdr_timings(conf);
1608         if (IS_ERR(timings))
1609                 return -ENOTSUPP;
1610
1611         /* T1 <=> tCLS */
1612         if (timings->tCLS_min > min_clk_period)
1613                 min_clk_period = timings->tCLS_min;
1614
1615         /* T2 <=> tCLH */
1616         if (timings->tCLH_min > min_clk_period)
1617                 min_clk_period = timings->tCLH_min;
1618
1619         /* T3 <=> tCS */
1620         if (timings->tCS_min > min_clk_period)
1621                 min_clk_period = timings->tCS_min;
1622
1623         /* T4 <=> tCH */
1624         if (timings->tCH_min > min_clk_period)
1625                 min_clk_period = timings->tCH_min;
1626
1627         /* T5 <=> tWP */
1628         if (timings->tWP_min > min_clk_period)
1629                 min_clk_period = timings->tWP_min;
1630
1631         /* T6 <=> tWH */
1632         if (timings->tWH_min > min_clk_period)
1633                 min_clk_period = timings->tWH_min;
1634
1635         /* T7 <=> tALS */
1636         if (timings->tALS_min > min_clk_period)
1637                 min_clk_period = timings->tALS_min;
1638
1639         /* T8 <=> tDS */
1640         if (timings->tDS_min > min_clk_period)
1641                 min_clk_period = timings->tDS_min;
1642
1643         /* T9 <=> tDH */
1644         if (timings->tDH_min > min_clk_period)
1645                 min_clk_period = timings->tDH_min;
1646
1647         /* T10 <=> tRR */
1648         if (timings->tRR_min > (min_clk_period * 3))
1649                 min_clk_period = DIV_ROUND_UP(timings->tRR_min, 3);
1650
1651         /* T11 <=> tALH */
1652         if (timings->tALH_min > min_clk_period)
1653                 min_clk_period = timings->tALH_min;
1654
1655         /* T12 <=> tRP */
1656         if (timings->tRP_min > min_clk_period)
1657                 min_clk_period = timings->tRP_min;
1658
1659         /* T13 <=> tREH */
1660         if (timings->tREH_min > min_clk_period)
1661                 min_clk_period = timings->tREH_min;
1662
1663         /* T14 <=> tRC */
1664         if (timings->tRC_min > (min_clk_period * 2))
1665                 min_clk_period = DIV_ROUND_UP(timings->tRC_min, 2);
1666
1667         /* T15 <=> tWC */
1668         if (timings->tWC_min > (min_clk_period * 2))
1669                 min_clk_period = DIV_ROUND_UP(timings->tWC_min, 2);
1670
1671         /* T16 - T19 + tCAD */
1672         if (timings->tWB_max > (min_clk_period * 20))
1673                 min_clk_period = DIV_ROUND_UP(timings->tWB_max, 20);
1674
1675         if (timings->tADL_min > (min_clk_period * 32))
1676                 min_clk_period = DIV_ROUND_UP(timings->tADL_min, 32);
1677
1678         if (timings->tWHR_min > (min_clk_period * 32))
1679                 min_clk_period = DIV_ROUND_UP(timings->tWHR_min, 32);
1680
1681         if (timings->tRHW_min > (min_clk_period * 20))
1682                 min_clk_period = DIV_ROUND_UP(timings->tRHW_min, 20);
1683
1684         tWB  = sunxi_nand_lookup_timing(tWB_lut, timings->tWB_max,
1685                                         min_clk_period);
1686         if (tWB < 0) {
1687                 dev_err(nfc->dev, "unsupported tWB\n");
1688                 return tWB;
1689         }
1690
1691         tADL = DIV_ROUND_UP(timings->tADL_min, min_clk_period) >> 3;
1692         if (tADL > 3) {
1693                 dev_err(nfc->dev, "unsupported tADL\n");
1694                 return -EINVAL;
1695         }
1696
1697         tWHR = DIV_ROUND_UP(timings->tWHR_min, min_clk_period) >> 3;
1698         if (tWHR > 3) {
1699                 dev_err(nfc->dev, "unsupported tWHR\n");
1700                 return -EINVAL;
1701         }
1702
1703         tRHW = sunxi_nand_lookup_timing(tRHW_lut, timings->tRHW_min,
1704                                         min_clk_period);
1705         if (tRHW < 0) {
1706                 dev_err(nfc->dev, "unsupported tRHW\n");
1707                 return tRHW;
1708         }
1709
1710         if (check_only)
1711                 return 0;
1712
1713         /*
1714          * TODO: according to ONFI specs this value only applies for DDR NAND,
1715          * but Allwinner seems to set this to 0x7. Mimic them for now.
1716          */
1717         tCAD = 0x7;
1718
1719         /* TODO: A83 has some more bits for CDQSS, CS, CLHZ, CCS, WC */
1720         chip->timing_cfg = NFC_TIMING_CFG(tWB, tADL, tWHR, tRHW, tCAD);
1721
1722         /* Convert min_clk_period from picoseconds to nanoseconds */
1723         min_clk_period = DIV_ROUND_UP(min_clk_period, 1000);
1724
1725         /*
1726          * Unlike what is stated in Allwinner datasheet, the clk_rate should
1727          * be set to (1 / min_clk_period), and not (2 / min_clk_period).
1728          * This new formula was verified with a scope and validated by
1729          * Allwinner engineers.
1730          */
1731         chip->clk_rate = NSEC_PER_SEC / min_clk_period;
1732         real_clk_rate = clk_round_rate(nfc->mod_clk, chip->clk_rate);
1733
1734         /*
1735          * ONFI specification 3.1, paragraph 4.15.2 dictates that EDO data
1736          * output cycle timings shall be used if the host drives tRC less than
1737          * 30 ns.
1738          */
1739         min_clk_period = NSEC_PER_SEC / real_clk_rate;
1740         chip->timing_ctl = ((min_clk_period * 2) < 30) ?
1741                            NFC_TIMING_CTL_EDO : 0;
1742
1743         return 0;
1744 }
1745
1746 static int sunxi_nand_ooblayout_ecc(struct mtd_info *mtd, int section,
1747                                     struct mtd_oob_region *oobregion)
1748 {
1749         struct nand_chip *nand = mtd_to_nand(mtd);
1750         struct nand_ecc_ctrl *ecc = &nand->ecc;
1751
1752         if (section >= ecc->steps)
1753                 return -ERANGE;
1754
1755         oobregion->offset = section * (ecc->bytes + 4) + 4;
1756         oobregion->length = ecc->bytes;
1757
1758         return 0;
1759 }
1760
1761 static int sunxi_nand_ooblayout_free(struct mtd_info *mtd, int section,
1762                                      struct mtd_oob_region *oobregion)
1763 {
1764         struct nand_chip *nand = mtd_to_nand(mtd);
1765         struct nand_ecc_ctrl *ecc = &nand->ecc;
1766
1767         if (section > ecc->steps)
1768                 return -ERANGE;
1769
1770         /*
1771          * The first 2 bytes are used for BB markers, hence we
1772          * only have 2 bytes available in the first user data
1773          * section.
1774          */
1775         if (!section && ecc->mode == NAND_ECC_HW) {
1776                 oobregion->offset = 2;
1777                 oobregion->length = 2;
1778
1779                 return 0;
1780         }
1781
1782         oobregion->offset = section * (ecc->bytes + 4);
1783
1784         if (section < ecc->steps)
1785                 oobregion->length = 4;
1786         else
1787                 oobregion->offset = mtd->oobsize - oobregion->offset;
1788
1789         return 0;
1790 }
1791
1792 static const struct mtd_ooblayout_ops sunxi_nand_ooblayout_ops = {
1793         .ecc = sunxi_nand_ooblayout_ecc,
1794         .free = sunxi_nand_ooblayout_free,
1795 };
1796
1797 static int sunxi_nand_hw_common_ecc_ctrl_init(struct mtd_info *mtd,
1798                                               struct nand_ecc_ctrl *ecc,
1799                                               struct device_node *np)
1800 {
1801         static const u8 strengths[] = { 16, 24, 28, 32, 40, 48, 56, 60, 64 };
1802         struct nand_chip *nand = mtd_to_nand(mtd);
1803         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
1804         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
1805         struct sunxi_nand_hw_ecc *data;
1806         int nsectors;
1807         int ret;
1808         int i;
1809
1810         if (ecc->options & NAND_ECC_MAXIMIZE) {
1811                 int bytes;
1812
1813                 ecc->size = 1024;
1814                 nsectors = mtd->writesize / ecc->size;
1815
1816                 /* Reserve 2 bytes for the BBM */
1817                 bytes = (mtd->oobsize - 2) / nsectors;
1818
1819                 /* 4 non-ECC bytes are added before each ECC bytes section */
1820                 bytes -= 4;
1821
1822                 /* and bytes has to be even. */
1823                 if (bytes % 2)
1824                         bytes--;
1825
1826                 ecc->strength = bytes * 8 / fls(8 * ecc->size);
1827
1828                 for (i = 0; i < ARRAY_SIZE(strengths); i++) {
1829                         if (strengths[i] > ecc->strength)
1830                                 break;
1831                 }
1832
1833                 if (!i)
1834                         ecc->strength = 0;
1835                 else
1836                         ecc->strength = strengths[i - 1];
1837         }
1838
1839         if (ecc->size != 512 && ecc->size != 1024)
1840                 return -EINVAL;
1841
1842         data = kzalloc(sizeof(*data), GFP_KERNEL);
1843         if (!data)
1844                 return -ENOMEM;
1845
1846         /* Prefer 1k ECC chunk over 512 ones */
1847         if (ecc->size == 512 && mtd->writesize > 512) {
1848                 ecc->size = 1024;
1849                 ecc->strength *= 2;
1850         }
1851
1852         /* Add ECC info retrieval from DT */
1853         for (i = 0; i < ARRAY_SIZE(strengths); i++) {
1854                 if (ecc->strength <= strengths[i])
1855                         break;
1856         }
1857
1858         if (i >= ARRAY_SIZE(strengths)) {
1859                 dev_err(nfc->dev, "unsupported strength\n");
1860                 ret = -ENOTSUPP;
1861                 goto err;
1862         }
1863
1864         data->mode = i;
1865
1866         /* HW ECC always request ECC bytes for 1024 bytes blocks */
1867         ecc->bytes = DIV_ROUND_UP(ecc->strength * fls(8 * 1024), 8);
1868
1869         /* HW ECC always work with even numbers of ECC bytes */
1870         ecc->bytes = ALIGN(ecc->bytes, 2);
1871
1872         nsectors = mtd->writesize / ecc->size;
1873
1874         if (mtd->oobsize < ((ecc->bytes + 4) * nsectors)) {
1875                 ret = -EINVAL;
1876                 goto err;
1877         }
1878
1879         ecc->read_oob = sunxi_nfc_hw_common_ecc_read_oob;
1880         ecc->write_oob = sunxi_nfc_hw_common_ecc_write_oob;
1881         mtd_set_ooblayout(mtd, &sunxi_nand_ooblayout_ops);
1882         ecc->priv = data;
1883
1884         return 0;
1885
1886 err:
1887         kfree(data);
1888
1889         return ret;
1890 }
1891
1892 static void sunxi_nand_hw_common_ecc_ctrl_cleanup(struct nand_ecc_ctrl *ecc)
1893 {
1894         kfree(ecc->priv);
1895 }
1896
1897 static int sunxi_nand_hw_ecc_ctrl_init(struct mtd_info *mtd,
1898                                        struct nand_ecc_ctrl *ecc,
1899                                        struct device_node *np)
1900 {
1901         struct nand_chip *nand = mtd_to_nand(mtd);
1902         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
1903         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
1904         int ret;
1905
1906         ret = sunxi_nand_hw_common_ecc_ctrl_init(mtd, ecc, np);
1907         if (ret)
1908                 return ret;
1909
1910         if (nfc->dmac) {
1911                 ecc->read_page = sunxi_nfc_hw_ecc_read_page_dma;
1912                 ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage_dma;
1913                 ecc->write_page = sunxi_nfc_hw_ecc_write_page_dma;
1914                 nand->options |= NAND_USE_BOUNCE_BUFFER;
1915         } else {
1916                 ecc->read_page = sunxi_nfc_hw_ecc_read_page;
1917                 ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage;
1918                 ecc->write_page = sunxi_nfc_hw_ecc_write_page;
1919         }
1920
1921         /* TODO: support DMA for raw accesses and subpage write */
1922         ecc->write_subpage = sunxi_nfc_hw_ecc_write_subpage;
1923         ecc->read_oob_raw = nand_read_oob_std;
1924         ecc->write_oob_raw = nand_write_oob_std;
1925         ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage;
1926
1927         return 0;
1928 }
1929
1930 static int sunxi_nand_hw_syndrome_ecc_ctrl_init(struct mtd_info *mtd,
1931                                                 struct nand_ecc_ctrl *ecc,
1932                                                 struct device_node *np)
1933 {
1934         int ret;
1935
1936         ret = sunxi_nand_hw_common_ecc_ctrl_init(mtd, ecc, np);
1937         if (ret)
1938                 return ret;
1939
1940         ecc->prepad = 4;
1941         ecc->read_page = sunxi_nfc_hw_syndrome_ecc_read_page;
1942         ecc->write_page = sunxi_nfc_hw_syndrome_ecc_write_page;
1943         ecc->read_oob_raw = nand_read_oob_syndrome;
1944         ecc->write_oob_raw = nand_write_oob_syndrome;
1945
1946         return 0;
1947 }
1948
1949 static void sunxi_nand_ecc_cleanup(struct nand_ecc_ctrl *ecc)
1950 {
1951         switch (ecc->mode) {
1952         case NAND_ECC_HW:
1953         case NAND_ECC_HW_SYNDROME:
1954                 sunxi_nand_hw_common_ecc_ctrl_cleanup(ecc);
1955                 break;
1956         case NAND_ECC_NONE:
1957         default:
1958                 break;
1959         }
1960 }
1961
1962 static int sunxi_nand_ecc_init(struct mtd_info *mtd, struct nand_ecc_ctrl *ecc,
1963                                struct device_node *np)
1964 {
1965         struct nand_chip *nand = mtd_to_nand(mtd);
1966         int ret;
1967
1968         if (!ecc->size) {
1969                 ecc->size = nand->ecc_step_ds;
1970                 ecc->strength = nand->ecc_strength_ds;
1971         }
1972
1973         if (!ecc->size || !ecc->strength)
1974                 return -EINVAL;
1975
1976         switch (ecc->mode) {
1977         case NAND_ECC_HW:
1978                 ret = sunxi_nand_hw_ecc_ctrl_init(mtd, ecc, np);
1979                 if (ret)
1980                         return ret;
1981                 break;
1982         case NAND_ECC_HW_SYNDROME:
1983                 ret = sunxi_nand_hw_syndrome_ecc_ctrl_init(mtd, ecc, np);
1984                 if (ret)
1985                         return ret;
1986                 break;
1987         case NAND_ECC_NONE:
1988         case NAND_ECC_SOFT:
1989                 break;
1990         default:
1991                 return -EINVAL;
1992         }
1993
1994         return 0;
1995 }
1996
1997 static int sunxi_nand_chip_init(struct device *dev, struct sunxi_nfc *nfc,
1998                                 struct device_node *np)
1999 {
2000         struct sunxi_nand_chip *chip;
2001         struct mtd_info *mtd;
2002         struct nand_chip *nand;
2003         int nsels;
2004         int ret;
2005         int i;
2006         u32 tmp;
2007
2008         if (!of_get_property(np, "reg", &nsels))
2009                 return -EINVAL;
2010
2011         nsels /= sizeof(u32);
2012         if (!nsels) {
2013                 dev_err(dev, "invalid reg property size\n");
2014                 return -EINVAL;
2015         }
2016
2017         chip = devm_kzalloc(dev,
2018                             sizeof(*chip) +
2019                             (nsels * sizeof(struct sunxi_nand_chip_sel)),
2020                             GFP_KERNEL);
2021         if (!chip) {
2022                 dev_err(dev, "could not allocate chip\n");
2023                 return -ENOMEM;
2024         }
2025
2026         chip->nsels = nsels;
2027         chip->selected = -1;
2028
2029         for (i = 0; i < nsels; i++) {
2030                 ret = of_property_read_u32_index(np, "reg", i, &tmp);
2031                 if (ret) {
2032                         dev_err(dev, "could not retrieve reg property: %d\n",
2033                                 ret);
2034                         return ret;
2035                 }
2036
2037                 if (tmp > NFC_MAX_CS) {
2038                         dev_err(dev,
2039                                 "invalid reg value: %u (max CS = 7)\n",
2040                                 tmp);
2041                         return -EINVAL;
2042                 }
2043
2044                 if (test_and_set_bit(tmp, &nfc->assigned_cs)) {
2045                         dev_err(dev, "CS %d already assigned\n", tmp);
2046                         return -EINVAL;
2047                 }
2048
2049                 chip->sels[i].cs = tmp;
2050
2051                 if (!of_property_read_u32_index(np, "allwinner,rb", i, &tmp) &&
2052                     tmp < 2) {
2053                         chip->sels[i].rb.type = RB_NATIVE;
2054                         chip->sels[i].rb.info.nativeid = tmp;
2055                 } else {
2056                         ret = of_get_named_gpio(np, "rb-gpios", i);
2057                         if (ret >= 0) {
2058                                 tmp = ret;
2059                                 chip->sels[i].rb.type = RB_GPIO;
2060                                 chip->sels[i].rb.info.gpio = tmp;
2061                                 ret = devm_gpio_request(dev, tmp, "nand-rb");
2062                                 if (ret)
2063                                         return ret;
2064
2065                                 ret = gpio_direction_input(tmp);
2066                                 if (ret)
2067                                         return ret;
2068                         } else {
2069                                 chip->sels[i].rb.type = RB_NONE;
2070                         }
2071                 }
2072         }
2073
2074         nand = &chip->nand;
2075         /* Default tR value specified in the ONFI spec (chapter 4.15.1) */
2076         nand->chip_delay = 200;
2077         nand->controller = &nfc->controller;
2078         /*
2079          * Set the ECC mode to the default value in case nothing is specified
2080          * in the DT.
2081          */
2082         nand->ecc.mode = NAND_ECC_HW;
2083         nand_set_flash_node(nand, np);
2084         nand->select_chip = sunxi_nfc_select_chip;
2085         nand->cmd_ctrl = sunxi_nfc_cmd_ctrl;
2086         nand->read_buf = sunxi_nfc_read_buf;
2087         nand->write_buf = sunxi_nfc_write_buf;
2088         nand->read_byte = sunxi_nfc_read_byte;
2089         nand->setup_data_interface = sunxi_nfc_setup_data_interface;
2090
2091         mtd = nand_to_mtd(nand);
2092         mtd->dev.parent = dev;
2093
2094         ret = nand_scan_ident(mtd, nsels, NULL);
2095         if (ret)
2096                 return ret;
2097
2098         if (nand->bbt_options & NAND_BBT_USE_FLASH)
2099                 nand->bbt_options |= NAND_BBT_NO_OOB;
2100
2101         if (nand->options & NAND_NEED_SCRAMBLING)
2102                 nand->options |= NAND_NO_SUBPAGE_WRITE;
2103
2104         nand->options |= NAND_SUBPAGE_READ;
2105
2106         ret = sunxi_nand_ecc_init(mtd, &nand->ecc, np);
2107         if (ret) {
2108                 dev_err(dev, "ECC init failed: %d\n", ret);
2109                 return ret;
2110         }
2111
2112         ret = nand_scan_tail(mtd);
2113         if (ret) {
2114                 dev_err(dev, "nand_scan_tail failed: %d\n", ret);
2115                 return ret;
2116         }
2117
2118         ret = mtd_device_register(mtd, NULL, 0);
2119         if (ret) {
2120                 dev_err(dev, "failed to register mtd device: %d\n", ret);
2121                 nand_release(mtd);
2122                 return ret;
2123         }
2124
2125         list_add_tail(&chip->node, &nfc->chips);
2126
2127         return 0;
2128 }
2129
2130 static int sunxi_nand_chips_init(struct device *dev, struct sunxi_nfc *nfc)
2131 {
2132         struct device_node *np = dev->of_node;
2133         struct device_node *nand_np;
2134         int nchips = of_get_child_count(np);
2135         int ret;
2136
2137         if (nchips > 8) {
2138                 dev_err(dev, "too many NAND chips: %d (max = 8)\n", nchips);
2139                 return -EINVAL;
2140         }
2141
2142         for_each_child_of_node(np, nand_np) {
2143                 ret = sunxi_nand_chip_init(dev, nfc, nand_np);
2144                 if (ret) {
2145                         of_node_put(nand_np);
2146                         return ret;
2147                 }
2148         }
2149
2150         return 0;
2151 }
2152
2153 static void sunxi_nand_chips_cleanup(struct sunxi_nfc *nfc)
2154 {
2155         struct sunxi_nand_chip *chip;
2156
2157         while (!list_empty(&nfc->chips)) {
2158                 chip = list_first_entry(&nfc->chips, struct sunxi_nand_chip,
2159                                         node);
2160                 nand_release(nand_to_mtd(&chip->nand));
2161                 sunxi_nand_ecc_cleanup(&chip->nand.ecc);
2162                 list_del(&chip->node);
2163         }
2164 }
2165
2166 static int sunxi_nfc_probe(struct platform_device *pdev)
2167 {
2168         struct device *dev = &pdev->dev;
2169         struct resource *r;
2170         struct sunxi_nfc *nfc;
2171         int irq;
2172         int ret;
2173
2174         nfc = devm_kzalloc(dev, sizeof(*nfc), GFP_KERNEL);
2175         if (!nfc)
2176                 return -ENOMEM;
2177
2178         nfc->dev = dev;
2179         nand_hw_control_init(&nfc->controller);
2180         INIT_LIST_HEAD(&nfc->chips);
2181
2182         r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2183         nfc->regs = devm_ioremap_resource(dev, r);
2184         if (IS_ERR(nfc->regs))
2185                 return PTR_ERR(nfc->regs);
2186
2187         irq = platform_get_irq(pdev, 0);
2188         if (irq < 0) {
2189                 dev_err(dev, "failed to retrieve irq\n");
2190                 return irq;
2191         }
2192
2193         nfc->ahb_clk = devm_clk_get(dev, "ahb");
2194         if (IS_ERR(nfc->ahb_clk)) {
2195                 dev_err(dev, "failed to retrieve ahb clk\n");
2196                 return PTR_ERR(nfc->ahb_clk);
2197         }
2198
2199         ret = clk_prepare_enable(nfc->ahb_clk);
2200         if (ret)
2201                 return ret;
2202
2203         nfc->mod_clk = devm_clk_get(dev, "mod");
2204         if (IS_ERR(nfc->mod_clk)) {
2205                 dev_err(dev, "failed to retrieve mod clk\n");
2206                 ret = PTR_ERR(nfc->mod_clk);
2207                 goto out_ahb_clk_unprepare;
2208         }
2209
2210         ret = clk_prepare_enable(nfc->mod_clk);
2211         if (ret)
2212                 goto out_ahb_clk_unprepare;
2213
2214         nfc->reset = devm_reset_control_get_optional(dev, "ahb");
2215         if (!IS_ERR(nfc->reset)) {
2216                 ret = reset_control_deassert(nfc->reset);
2217                 if (ret) {
2218                         dev_err(dev, "reset err %d\n", ret);
2219                         goto out_mod_clk_unprepare;
2220                 }
2221         } else if (PTR_ERR(nfc->reset) != -ENOENT) {
2222                 ret = PTR_ERR(nfc->reset);
2223                 goto out_mod_clk_unprepare;
2224         }
2225
2226         ret = sunxi_nfc_rst(nfc);
2227         if (ret)
2228                 goto out_ahb_reset_reassert;
2229
2230         writel(0, nfc->regs + NFC_REG_INT);
2231         ret = devm_request_irq(dev, irq, sunxi_nfc_interrupt,
2232                                0, "sunxi-nand", nfc);
2233         if (ret)
2234                 goto out_ahb_reset_reassert;
2235
2236         nfc->dmac = dma_request_slave_channel(dev, "rxtx");
2237         if (nfc->dmac) {
2238                 struct dma_slave_config dmac_cfg = { };
2239
2240                 dmac_cfg.src_addr = r->start + NFC_REG_IO_DATA;
2241                 dmac_cfg.dst_addr = dmac_cfg.src_addr;
2242                 dmac_cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
2243                 dmac_cfg.dst_addr_width = dmac_cfg.src_addr_width;
2244                 dmac_cfg.src_maxburst = 4;
2245                 dmac_cfg.dst_maxburst = 4;
2246                 dmaengine_slave_config(nfc->dmac, &dmac_cfg);
2247         } else {
2248                 dev_warn(dev, "failed to request rxtx DMA channel\n");
2249         }
2250
2251         platform_set_drvdata(pdev, nfc);
2252
2253         ret = sunxi_nand_chips_init(dev, nfc);
2254         if (ret) {
2255                 dev_err(dev, "failed to init nand chips\n");
2256                 goto out_release_dmac;
2257         }
2258
2259         return 0;
2260
2261 out_release_dmac:
2262         if (nfc->dmac)
2263                 dma_release_channel(nfc->dmac);
2264 out_ahb_reset_reassert:
2265         if (!IS_ERR(nfc->reset))
2266                 reset_control_assert(nfc->reset);
2267 out_mod_clk_unprepare:
2268         clk_disable_unprepare(nfc->mod_clk);
2269 out_ahb_clk_unprepare:
2270         clk_disable_unprepare(nfc->ahb_clk);
2271
2272         return ret;
2273 }
2274
2275 static int sunxi_nfc_remove(struct platform_device *pdev)
2276 {
2277         struct sunxi_nfc *nfc = platform_get_drvdata(pdev);
2278
2279         sunxi_nand_chips_cleanup(nfc);
2280
2281         if (!IS_ERR(nfc->reset))
2282                 reset_control_assert(nfc->reset);
2283
2284         if (nfc->dmac)
2285                 dma_release_channel(nfc->dmac);
2286         clk_disable_unprepare(nfc->mod_clk);
2287         clk_disable_unprepare(nfc->ahb_clk);
2288
2289         return 0;
2290 }
2291
2292 static const struct of_device_id sunxi_nfc_ids[] = {
2293         { .compatible = "allwinner,sun4i-a10-nand" },
2294         { /* sentinel */ }
2295 };
2296 MODULE_DEVICE_TABLE(of, sunxi_nfc_ids);
2297
2298 static struct platform_driver sunxi_nfc_driver = {
2299         .driver = {
2300                 .name = "sunxi_nand",
2301                 .of_match_table = sunxi_nfc_ids,
2302         },
2303         .probe = sunxi_nfc_probe,
2304         .remove = sunxi_nfc_remove,
2305 };
2306 module_platform_driver(sunxi_nfc_driver);
2307
2308 MODULE_LICENSE("GPL v2");
2309 MODULE_AUTHOR("Boris BREZILLON");
2310 MODULE_DESCRIPTION("Allwinner NAND Flash Controller driver");
2311 MODULE_ALIAS("platform:sunxi_nand");