drivers/mtd/nand/sunxi_nand.c

   1 /*
   2  * Copyright (C) 2013 Boris BREZILLON <b.brezillon.dev@gmail.com>
   3  *
   4  * Derived from:
   5  *      https://github.com/yuq/sunxi-nfc-mtd
   6  *      Copyright (C) 2013 Qiang Yu <yuq825@gmail.com>
   7  *
   8  *      https://github.com/hno/Allwinner-Info
   9  *      Copyright (C) 2013 Henrik Nordström <Henrik Nordström>
  10  *
  11  *      Copyright (C) 2013 Dmitriy B. <rzk333@gmail.com>
  12  *      Copyright (C) 2013 Sergey Lapin <slapin@ossfans.org>
  13  *
  14  * This program is free software; you can redistribute it and/or modify
  15  * it under the terms of the GNU General Public License as published by
  16  * the Free Software Foundation; either version 2 of the License, or
  17  * (at your option) any later version.
  18  *
  19  * This program is distributed in the hope that it will be useful,
  20  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  21  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  22  * GNU General Public License for more details.
  23  */
  24
  25 #include <linux/dma-mapping.h>
  26 #include <linux/slab.h>
  27 #include <linux/module.h>
  28 #include <linux/moduleparam.h>
  29 #include <linux/platform_device.h>
  30 #include <linux/of.h>
  31 #include <linux/of_device.h>
  32 #include <linux/of_gpio.h>
  33 #include <linux/mtd/mtd.h>
  34 #include <linux/mtd/nand.h>
  35 #include <linux/mtd/partitions.h>
  36 #include <linux/clk.h>
  37 #include <linux/delay.h>
  38 #include <linux/dmaengine.h>
  39 #include <linux/gpio.h>
  40 #include <linux/interrupt.h>
  41 #include <linux/iopoll.h>
  42 #include <linux/reset.h>
  43
  44 #define NFC_REG_CTL             0x0000
  45 #define NFC_REG_ST              0x0004
  46 #define NFC_REG_INT             0x0008
  47 #define NFC_REG_TIMING_CTL      0x000C
  48 #define NFC_REG_TIMING_CFG      0x0010
  49 #define NFC_REG_ADDR_LOW        0x0014
  50 #define NFC_REG_ADDR_HIGH       0x0018
  51 #define NFC_REG_SECTOR_NUM      0x001C
  52 #define NFC_REG_CNT             0x0020
  53 #define NFC_REG_CMD             0x0024
  54 #define NFC_REG_RCMD_SET        0x0028
  55 #define NFC_REG_WCMD_SET        0x002C
  56 #define NFC_REG_IO_DATA         0x0030
  57 #define NFC_REG_ECC_CTL         0x0034
  58 #define NFC_REG_ECC_ST          0x0038
  59 #define NFC_REG_DEBUG           0x003C
  60 #define NFC_REG_ECC_ERR_CNT(x)  ((0x0040 + (x)) & ~0x3)
  61 #define NFC_REG_USER_DATA(x)    (0x0050 + ((x) * 4))
  62 #define NFC_REG_SPARE_AREA      0x00A0
  63 #define NFC_REG_PAT_ID          0x00A4
  64 #define NFC_RAM0_BASE           0x0400
  65 #define NFC_RAM1_BASE           0x0800
  66
  67 /* define bit use in NFC_CTL */
  68 #define NFC_EN                  BIT(0)
  69 #define NFC_RESET               BIT(1)
  70 #define NFC_BUS_WIDTH_MSK       BIT(2)
  71 #define NFC_BUS_WIDTH_8         (0 << 2)
  72 #define NFC_BUS_WIDTH_16        (1 << 2)
  73 #define NFC_RB_SEL_MSK          BIT(3)
  74 #define NFC_RB_SEL(x)           ((x) << 3)
  75 #define NFC_CE_SEL_MSK          GENMASK(26, 24)
  76 #define NFC_CE_SEL(x)           ((x) << 24)
  77 #define NFC_CE_CTL              BIT(6)
  78 #define NFC_PAGE_SHIFT_MSK      GENMASK(11, 8)
  79 #define NFC_PAGE_SHIFT(x)       (((x) < 10 ? 0 : (x) - 10) << 8)
  80 #define NFC_SAM                 BIT(12)
  81 #define NFC_RAM_METHOD          BIT(14)
  82 #define NFC_DEBUG_CTL           BIT(31)
  83
  84 /* define bit use in NFC_ST */
  85 #define NFC_RB_B2R              BIT(0)
  86 #define NFC_CMD_INT_FLAG        BIT(1)
  87 #define NFC_DMA_INT_FLAG        BIT(2)
  88 #define NFC_CMD_FIFO_STATUS     BIT(3)
  89 #define NFC_STA                 BIT(4)
  90 #define NFC_NATCH_INT_FLAG      BIT(5)
  91 #define NFC_RB_STATE(x)         BIT(x + 8)
  92
  93 /* define bit use in NFC_INT */
  94 #define NFC_B2R_INT_ENABLE      BIT(0)
  95 #define NFC_CMD_INT_ENABLE      BIT(1)
  96 #define NFC_DMA_INT_ENABLE      BIT(2)
  97 #define NFC_INT_MASK            (NFC_B2R_INT_ENABLE | \
  98                                  NFC_CMD_INT_ENABLE | \
  99                                  NFC_DMA_INT_ENABLE)
 100
 101 /* define bit use in NFC_TIMING_CTL */
 102 #define NFC_TIMING_CTL_EDO      BIT(8)
 103
 104 /* define NFC_TIMING_CFG register layout */
 105 #define NFC_TIMING_CFG(tWB, tADL, tWHR, tRHW, tCAD)             \
 106         (((tWB) & 0x3) | (((tADL) & 0x3) << 2) |                \
 107         (((tWHR) & 0x3) << 4) | (((tRHW) & 0x3) << 6) |         \
 108         (((tCAD) & 0x7) << 8))
 109
 110 /* define bit use in NFC_CMD */
 111 #define NFC_CMD_LOW_BYTE_MSK    GENMASK(7, 0)
 112 #define NFC_CMD_HIGH_BYTE_MSK   GENMASK(15, 8)
 113 #define NFC_CMD(x)              (x)
 114 #define NFC_ADR_NUM_MSK         GENMASK(18, 16)
 115 #define NFC_ADR_NUM(x)          (((x) - 1) << 16)
 116 #define NFC_SEND_ADR            BIT(19)
 117 #define NFC_ACCESS_DIR          BIT(20)
 118 #define NFC_DATA_TRANS          BIT(21)
 119 #define NFC_SEND_CMD1           BIT(22)
 120 #define NFC_WAIT_FLAG           BIT(23)
 121 #define NFC_SEND_CMD2           BIT(24)
 122 #define NFC_SEQ                 BIT(25)
 123 #define NFC_DATA_SWAP_METHOD    BIT(26)
 124 #define NFC_ROW_AUTO_INC        BIT(27)
 125 #define NFC_SEND_CMD3           BIT(28)
 126 #define NFC_SEND_CMD4           BIT(29)
 127 #define NFC_CMD_TYPE_MSK        GENMASK(31, 30)
 128 #define NFC_NORMAL_OP           (0 << 30)
 129 #define NFC_ECC_OP              (1 << 30)
 130 #define NFC_PAGE_OP             (2 << 30)
 131
 132 /* define bit use in NFC_RCMD_SET */
 133 #define NFC_READ_CMD_MSK        GENMASK(7, 0)
 134 #define NFC_RND_READ_CMD0_MSK   GENMASK(15, 8)
 135 #define NFC_RND_READ_CMD1_MSK   GENMASK(23, 16)
 136
 137 /* define bit use in NFC_WCMD_SET */
 138 #define NFC_PROGRAM_CMD_MSK     GENMASK(7, 0)
 139 #define NFC_RND_WRITE_CMD_MSK   GENMASK(15, 8)
 140 #define NFC_READ_CMD0_MSK       GENMASK(23, 16)
 141 #define NFC_READ_CMD1_MSK       GENMASK(31, 24)
 142
 143 /* define bit use in NFC_ECC_CTL */
 144 #define NFC_ECC_EN              BIT(0)
 145 #define NFC_ECC_PIPELINE        BIT(3)
 146 #define NFC_ECC_EXCEPTION       BIT(4)
 147 #define NFC_ECC_BLOCK_SIZE_MSK  BIT(5)
 148 #define NFC_ECC_BLOCK_512       BIT(5)
 149 #define NFC_RANDOM_EN           BIT(9)
 150 #define NFC_RANDOM_DIRECTION    BIT(10)
 151 #define NFC_ECC_MODE_MSK        GENMASK(15, 12)
 152 #define NFC_ECC_MODE(x)         ((x) << 12)
 153 #define NFC_RANDOM_SEED_MSK     GENMASK(30, 16)
 154 #define NFC_RANDOM_SEED(x)      ((x) << 16)
 155
 156 /* define bit use in NFC_ECC_ST */
 157 #define NFC_ECC_ERR(x)          BIT(x)
 158 #define NFC_ECC_ERR_MSK         GENMASK(15, 0)
 159 #define NFC_ECC_PAT_FOUND(x)    BIT(x + 16)
 160 #define NFC_ECC_ERR_CNT(b, x)   (((x) >> (((b) % 4) * 8)) & 0xff)
 161
 162 #define NFC_DEFAULT_TIMEOUT_MS  1000
 163
 164 #define NFC_SRAM_SIZE           1024
 165
 166 #define NFC_MAX_CS              7
 167
 168 /*
 169  * Ready/Busy detection type: describes the Ready/Busy detection modes
 170  *
 171  * @RB_NONE:    no external detection available, rely on STATUS command
 172  *              and software timeouts
 173  * @RB_NATIVE:  use sunxi NAND controller Ready/Busy support. The Ready/Busy
 174  *              pin of the NAND flash chip must be connected to one of the
 175  *              native NAND R/B pins (those which can be muxed to the NAND
 176  *              Controller)
 177  * @RB_GPIO:    use a simple GPIO to handle Ready/Busy status. The Ready/Busy
 178  *              pin of the NAND flash chip must be connected to a GPIO capable
 179  *              pin.
 180  */
 181 enum sunxi_nand_rb_type {
 182         RB_NONE,
 183         RB_NATIVE,
 184         RB_GPIO,
 185 };
 186
 187 /*
 188  * Ready/Busy structure: stores information related to Ready/Busy detection
 189  *
 190  * @type:       the Ready/Busy detection mode
 191  * @info:       information related to the R/B detection mode. Either a gpio
 192  *              id or a native R/B id (those supported by the NAND controller).
 193  */
 194 struct sunxi_nand_rb {
 195         enum sunxi_nand_rb_type type;
 196         union {
 197                 int gpio;
 198                 int nativeid;
 199         } info;
 200 };
 201
 202 /*
 203  * Chip Select structure: stores information related to NAND Chip Select
 204  *
 205  * @cs:         the NAND CS id used to communicate with a NAND Chip
 206  * @rb:         the Ready/Busy description
 207  */
 208 struct sunxi_nand_chip_sel {
 209         u8 cs;
 210         struct sunxi_nand_rb rb;
 211 };
 212
 213 /*
 214  * sunxi HW ECC infos: stores information related to HW ECC support
 215  *
 216  * @mode:       the sunxi ECC mode field deduced from ECC requirements
 217  */
 218 struct sunxi_nand_hw_ecc {
 219         int mode;
 220 };
 221
 222 /*
 223  * NAND chip structure: stores NAND chip device related information
 224  *
 225  * @node:               used to store NAND chips into a list
 226  * @nand:               base NAND chip structure
 227  * @mtd:                base MTD structure
 228  * @clk_rate:           clk_rate required for this NAND chip
 229  * @timing_cfg          TIMING_CFG register value for this NAND chip
 230  * @selected:           current active CS
 231  * @nsels:              number of CS lines required by the NAND chip
 232  * @sels:               array of CS lines descriptions
 233  */
 234 struct sunxi_nand_chip {
 235         struct list_head node;
 236         struct nand_chip nand;
 237         unsigned long clk_rate;
 238         u32 timing_cfg;
 239         u32 timing_ctl;
 240         int selected;
 241         int addr_cycles;
 242         u32 addr[2];
 243         int cmd_cycles;
 244         u8 cmd[2];
 245         int nsels;
 246         struct sunxi_nand_chip_sel sels[0];
 247 };
 248
 249 static inline struct sunxi_nand_chip *to_sunxi_nand(struct nand_chip *nand)
 250 {
 251         return container_of(nand, struct sunxi_nand_chip, nand);
 252 }
 253
 254 /*
 255  * NAND Controller structure: stores sunxi NAND controller information
 256  *
 257  * @controller:         base controller structure
 258  * @dev:                parent device (used to print error messages)
 259  * @regs:               NAND controller registers
 260  * @ahb_clk:            NAND Controller AHB clock
 261  * @mod_clk:            NAND Controller mod clock
 262  * @assigned_cs:        bitmask describing already assigned CS lines
 263  * @clk_rate:           NAND controller current clock rate
 264  * @chips:              a list containing all the NAND chips attached to
 265  *                      this NAND controller
 266  * @complete:           a completion object used to wait for NAND
 267  *                      controller events
 268  */
 269 struct sunxi_nfc {
 270         struct nand_hw_control controller;
 271         struct device *dev;
 272         void __iomem *regs;
 273         struct clk *ahb_clk;
 274         struct clk *mod_clk;
 275         struct reset_control *reset;
 276         unsigned long assigned_cs;
 277         unsigned long clk_rate;
 278         struct list_head chips;
 279         struct completion complete;
 280         struct dma_chan *dmac;
 281 };
 282
 283 static inline struct sunxi_nfc *to_sunxi_nfc(struct nand_hw_control *ctrl)
 284 {
 285         return container_of(ctrl, struct sunxi_nfc, controller);
 286 }
 287
 288 static irqreturn_t sunxi_nfc_interrupt(int irq, void *dev_id)
 289 {
 290         struct sunxi_nfc *nfc = dev_id;
 291         u32 st = readl(nfc->regs + NFC_REG_ST);
 292         u32 ien = readl(nfc->regs + NFC_REG_INT);
 293
 294         if (!(ien & st))
 295                 return IRQ_NONE;
 296
 297         if ((ien & st) == ien)
 298                 complete(&nfc->complete);
 299
 300         writel(st & NFC_INT_MASK, nfc->regs + NFC_REG_ST);
 301         writel(~st & ien & NFC_INT_MASK, nfc->regs + NFC_REG_INT);
 302
 303         return IRQ_HANDLED;
 304 }
 305
 306 static int sunxi_nfc_wait_events(struct sunxi_nfc *nfc, u32 events,
 307                                  bool use_polling, unsigned int timeout_ms)
 308 {
 309         int ret;
 310
 311         if (events & ~NFC_INT_MASK)
 312                 return -EINVAL;
 313
 314         if (!timeout_ms)
 315                 timeout_ms = NFC_DEFAULT_TIMEOUT_MS;
 316
 317         if (!use_polling) {
 318                 init_completion(&nfc->complete);
 319
 320                 writel(events, nfc->regs + NFC_REG_INT);
 321
 322                 ret = wait_for_completion_timeout(&nfc->complete,
 323                                                 msecs_to_jiffies(timeout_ms));
 324                 if (!ret)
 325                         ret = -ETIMEDOUT;
 326                 else
 327                         ret = 0;
 328
 329                 writel(0, nfc->regs + NFC_REG_INT);
 330         } else {
 331                 u32 status;
 332
 333                 ret = readl_poll_timeout(nfc->regs + NFC_REG_ST, status,
 334                                          (status & events) == events, 1,
 335                                          timeout_ms * 1000);
 336         }
 337
 338         writel(events & NFC_INT_MASK, nfc->regs + NFC_REG_ST);
 339
 340         if (ret)
 341                 dev_err(nfc->dev, "wait interrupt timedout\n");
 342
 343         return ret;
 344 }
 345
 346 static int sunxi_nfc_wait_cmd_fifo_empty(struct sunxi_nfc *nfc)
 347 {
 348         u32 status;
 349         int ret;
 350
 351         ret = readl_poll_timeout(nfc->regs + NFC_REG_ST, status,
 352                                  !(status & NFC_CMD_FIFO_STATUS), 1,
 353                                  NFC_DEFAULT_TIMEOUT_MS * 1000);
 354         if (ret)
 355                 dev_err(nfc->dev, "wait for empty cmd FIFO timedout\n");
 356
 357         return ret;
 358 }
 359
 360 static int sunxi_nfc_rst(struct sunxi_nfc *nfc)
 361 {
 362         u32 ctl;
 363         int ret;
 364
 365         writel(0, nfc->regs + NFC_REG_ECC_CTL);
 366         writel(NFC_RESET, nfc->regs + NFC_REG_CTL);
 367
 368         ret = readl_poll_timeout(nfc->regs + NFC_REG_CTL, ctl,
 369                                  !(ctl & NFC_RESET), 1,
 370                                  NFC_DEFAULT_TIMEOUT_MS * 1000);
 371         if (ret)
 372                 dev_err(nfc->dev, "wait for NAND controller reset timedout\n");
 373
 374         return ret;
 375 }
 376
 377 static int sunxi_nfc_dma_op_prepare(struct mtd_info *mtd, const void *buf,
 378                                     int chunksize, int nchunks,
 379                                     enum dma_data_direction ddir,
 380                                     struct scatterlist *sg)
 381 {
 382         struct nand_chip *nand = mtd_to_nand(mtd);
 383         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 384         struct dma_async_tx_descriptor *dmad;
 385         enum dma_transfer_direction tdir;
 386         dma_cookie_t dmat;
 387         int ret;
 388
 389         if (ddir == DMA_FROM_DEVICE)
 390                 tdir = DMA_DEV_TO_MEM;
 391         else
 392                 tdir = DMA_MEM_TO_DEV;
 393
 394         sg_init_one(sg, buf, nchunks * chunksize);
 395         ret = dma_map_sg(nfc->dev, sg, 1, ddir);
 396         if (!ret)
 397                 return -ENOMEM;
 398
 399         dmad = dmaengine_prep_slave_sg(nfc->dmac, sg, 1, tdir, DMA_CTRL_ACK);
 400         if (!dmad) {
 401                 ret = -EINVAL;
 402                 goto err_unmap_buf;
 403         }
 404
 405         writel(readl(nfc->regs + NFC_REG_CTL) | NFC_RAM_METHOD,
 406                nfc->regs + NFC_REG_CTL);
 407         writel(nchunks, nfc->regs + NFC_REG_SECTOR_NUM);
 408         writel(chunksize, nfc->regs + NFC_REG_CNT);
 409         dmat = dmaengine_submit(dmad);
 410
 411         ret = dma_submit_error(dmat);
 412         if (ret)
 413                 goto err_clr_dma_flag;
 414
 415         return 0;
 416
 417 err_clr_dma_flag:
 418         writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD,
 419                nfc->regs + NFC_REG_CTL);
 420
 421 err_unmap_buf:
 422         dma_unmap_sg(nfc->dev, sg, 1, ddir);
 423         return ret;
 424 }
 425
 426 static void sunxi_nfc_dma_op_cleanup(struct mtd_info *mtd,
 427                                      enum dma_data_direction ddir,
 428                                      struct scatterlist *sg)
 429 {
 430         struct nand_chip *nand = mtd_to_nand(mtd);
 431         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 432
 433         dma_unmap_sg(nfc->dev, sg, 1, ddir);
 434         writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD,
 435                nfc->regs + NFC_REG_CTL);
 436 }
 437
 438 static int sunxi_nfc_dev_ready(struct mtd_info *mtd)
 439 {
 440         struct nand_chip *nand = mtd_to_nand(mtd);
 441         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
 442         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
 443         struct sunxi_nand_rb *rb;
 444         int ret;
 445
 446         if (sunxi_nand->selected < 0)
 447                 return 0;
 448
 449         rb = &sunxi_nand->sels[sunxi_nand->selected].rb;
 450
 451         switch (rb->type) {
 452         case RB_NATIVE:
 453                 ret = !!(readl(nfc->regs + NFC_REG_ST) &
 454                          NFC_RB_STATE(rb->info.nativeid));
 455                 break;
 456         case RB_GPIO:
 457                 ret = gpio_get_value(rb->info.gpio);
 458                 break;
 459         case RB_NONE:
 460         default:
 461                 ret = 0;
 462                 dev_err(nfc->dev, "cannot check R/B NAND status!\n");
 463                 break;
 464         }
 465
 466         return ret;
 467 }
 468
 469 static void sunxi_nfc_select_chip(struct mtd_info *mtd, int chip)
 470 {
 471         struct nand_chip *nand = mtd_to_nand(mtd);
 472         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
 473         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
 474         struct sunxi_nand_chip_sel *sel;
 475         u32 ctl;
 476
 477         if (chip > 0 && chip >= sunxi_nand->nsels)
 478                 return;
 479
 480         if (chip == sunxi_nand->selected)
 481                 return;
 482
 483         ctl = readl(nfc->regs + NFC_REG_CTL) &
 484               ~(NFC_PAGE_SHIFT_MSK | NFC_CE_SEL_MSK | NFC_RB_SEL_MSK | NFC_EN);
 485
 486         if (chip >= 0) {
 487                 sel = &sunxi_nand->sels[chip];
 488
 489                 ctl |= NFC_CE_SEL(sel->cs) | NFC_EN |
 490                        NFC_PAGE_SHIFT(nand->page_shift);
 491                 if (sel->rb.type == RB_NONE) {
 492                         nand->dev_ready = NULL;
 493                 } else {
 494                         nand->dev_ready = sunxi_nfc_dev_ready;
 495                         if (sel->rb.type == RB_NATIVE)
 496                                 ctl |= NFC_RB_SEL(sel->rb.info.nativeid);
 497                 }
 498
 499                 writel(mtd->writesize, nfc->regs + NFC_REG_SPARE_AREA);
 500
 501                 if (nfc->clk_rate != sunxi_nand->clk_rate) {
 502                         clk_set_rate(nfc->mod_clk, sunxi_nand->clk_rate);
 503                         nfc->clk_rate = sunxi_nand->clk_rate;
 504                 }
 505         }
 506
 507         writel(sunxi_nand->timing_ctl, nfc->regs + NFC_REG_TIMING_CTL);
 508         writel(sunxi_nand->timing_cfg, nfc->regs + NFC_REG_TIMING_CFG);
 509         writel(ctl, nfc->regs + NFC_REG_CTL);
 510
 511         sunxi_nand->selected = chip;
 512 }
 513
 514 static void sunxi_nfc_read_buf(struct mtd_info *mtd, uint8_t *buf, int len)
 515 {
 516         struct nand_chip *nand = mtd_to_nand(mtd);
 517         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
 518         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
 519         int ret;
 520         int cnt;
 521         int offs = 0;
 522         u32 tmp;
 523
 524         while (len > offs) {
 525                 bool poll = false;
 526
 527                 cnt = min(len - offs, NFC_SRAM_SIZE);
 528
 529                 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
 530                 if (ret)
 531                         break;
 532
 533                 writel(cnt, nfc->regs + NFC_REG_CNT);
 534                 tmp = NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD;
 535                 writel(tmp, nfc->regs + NFC_REG_CMD);
 536
 537                 /* Arbitrary limit for polling mode */
 538                 if (cnt < 64)
 539                         poll = true;
 540
 541                 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, poll, 0);
 542                 if (ret)
 543                         break;
 544
 545                 if (buf)
 546                         memcpy_fromio(buf + offs, nfc->regs + NFC_RAM0_BASE,
 547                                       cnt);
 548                 offs += cnt;
 549         }
 550 }
 551
 552 static void sunxi_nfc_write_buf(struct mtd_info *mtd, const uint8_t *buf,
 553                                 int len)
 554 {
 555         struct nand_chip *nand = mtd_to_nand(mtd);
 556         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
 557         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
 558         int ret;
 559         int cnt;
 560         int offs = 0;
 561         u32 tmp;
 562
 563         while (len > offs) {
 564                 bool poll = false;
 565
 566                 cnt = min(len - offs, NFC_SRAM_SIZE);
 567
 568                 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
 569                 if (ret)
 570                         break;
 571
 572                 writel(cnt, nfc->regs + NFC_REG_CNT);
 573                 memcpy_toio(nfc->regs + NFC_RAM0_BASE, buf + offs, cnt);
 574                 tmp = NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD |
 575                       NFC_ACCESS_DIR;
 576                 writel(tmp, nfc->regs + NFC_REG_CMD);
 577
 578                 /* Arbitrary limit for polling mode */
 579                 if (cnt < 64)
 580                         poll = true;
 581
 582                 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, poll, 0);
 583                 if (ret)
 584                         break;
 585
 586                 offs += cnt;
 587         }
 588 }
 589
 590 static uint8_t sunxi_nfc_read_byte(struct mtd_info *mtd)
 591 {
 592         uint8_t ret;
 593
 594         sunxi_nfc_read_buf(mtd, &ret, 1);
 595
 596         return ret;
 597 }
 598
 599 static void sunxi_nfc_cmd_ctrl(struct mtd_info *mtd, int dat,
 600                                unsigned int ctrl)
 601 {
 602         struct nand_chip *nand = mtd_to_nand(mtd);
 603         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
 604         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
 605         int ret;
 606
 607         if (dat == NAND_CMD_NONE && (ctrl & NAND_NCE) &&
 608             !(ctrl & (NAND_CLE | NAND_ALE))) {
 609                 u32 cmd = 0;
 610
 611                 if (!sunxi_nand->addr_cycles && !sunxi_nand->cmd_cycles)
 612                         return;
 613
 614                 if (sunxi_nand->cmd_cycles--)
 615                         cmd |= NFC_SEND_CMD1 | sunxi_nand->cmd[0];
 616
 617                 if (sunxi_nand->cmd_cycles--) {
 618                         cmd |= NFC_SEND_CMD2;
 619                         writel(sunxi_nand->cmd[1],
 620                                nfc->regs + NFC_REG_RCMD_SET);
 621                 }
 622
 623                 sunxi_nand->cmd_cycles = 0;
 624
 625                 if (sunxi_nand->addr_cycles) {
 626                         cmd |= NFC_SEND_ADR |
 627                                NFC_ADR_NUM(sunxi_nand->addr_cycles);
 628                         writel(sunxi_nand->addr[0],
 629                                nfc->regs + NFC_REG_ADDR_LOW);
 630                 }
 631
 632                 if (sunxi_nand->addr_cycles > 4)
 633                         writel(sunxi_nand->addr[1],
 634                                nfc->regs + NFC_REG_ADDR_HIGH);
 635
 636                 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
 637                 if (ret)
 638                         return;
 639
 640                 writel(cmd, nfc->regs + NFC_REG_CMD);
 641                 sunxi_nand->addr[0] = 0;
 642                 sunxi_nand->addr[1] = 0;
 643                 sunxi_nand->addr_cycles = 0;
 644                 sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0);
 645         }
 646
 647         if (ctrl & NAND_CLE) {
 648                 sunxi_nand->cmd[sunxi_nand->cmd_cycles++] = dat;
 649         } else if (ctrl & NAND_ALE) {
 650                 sunxi_nand->addr[sunxi_nand->addr_cycles / 4] |=
 651                                 dat << ((sunxi_nand->addr_cycles % 4) * 8);
 652                 sunxi_nand->addr_cycles++;
 653         }
 654 }
 655
 656 /* These seed values have been extracted from Allwinner's BSP */
 657 static const u16 sunxi_nfc_randomizer_page_seeds[] = {
 658         0x2b75, 0x0bd0, 0x5ca3, 0x62d1, 0x1c93, 0x07e9, 0x2162, 0x3a72,
 659         0x0d67, 0x67f9, 0x1be7, 0x077d, 0x032f, 0x0dac, 0x2716, 0x2436,
 660         0x7922, 0x1510, 0x3860, 0x5287, 0x480f, 0x4252, 0x1789, 0x5a2d,
 661         0x2a49, 0x5e10, 0x437f, 0x4b4e, 0x2f45, 0x216e, 0x5cb7, 0x7130,
 662         0x2a3f, 0x60e4, 0x4dc9, 0x0ef0, 0x0f52, 0x1bb9, 0x6211, 0x7a56,
 663         0x226d, 0x4ea7, 0x6f36, 0x3692, 0x38bf, 0x0c62, 0x05eb, 0x4c55,
 664         0x60f4, 0x728c, 0x3b6f, 0x2037, 0x7f69, 0x0936, 0x651a, 0x4ceb,
 665         0x6218, 0x79f3, 0x383f, 0x18d9, 0x4f05, 0x5c82, 0x2912, 0x6f17,
 666         0x6856, 0x5938, 0x1007, 0x61ab, 0x3e7f, 0x57c2, 0x542f, 0x4f62,
 667         0x7454, 0x2eac, 0x7739, 0x42d4, 0x2f90, 0x435a, 0x2e52, 0x2064,
 668         0x637c, 0x66ad, 0x2c90, 0x0bad, 0x759c, 0x0029, 0x0986, 0x7126,
 669         0x1ca7, 0x1605, 0x386a, 0x27f5, 0x1380, 0x6d75, 0x24c3, 0x0f8e,
 670         0x2b7a, 0x1418, 0x1fd1, 0x7dc1, 0x2d8e, 0x43af, 0x2267, 0x7da3,
 671         0x4e3d, 0x1338, 0x50db, 0x454d, 0x764d, 0x40a3, 0x42e6, 0x262b,
 672         0x2d2e, 0x1aea, 0x2e17, 0x173d, 0x3a6e, 0x71bf, 0x25f9, 0x0a5d,
 673         0x7c57, 0x0fbe, 0x46ce, 0x4939, 0x6b17, 0x37bb, 0x3e91, 0x76db,
 674 };
 675
 676 /*
 677  * sunxi_nfc_randomizer_ecc512_seeds and sunxi_nfc_randomizer_ecc1024_seeds
 678  * have been generated using
 679  * sunxi_nfc_randomizer_step(seed, (step_size * 8) + 15), which is what
 680  * the randomizer engine does internally before de/scrambling OOB data.
 681  *
 682  * Those tables are statically defined to avoid calculating randomizer state
 683  * at runtime.
 684  */
 685 static const u16 sunxi_nfc_randomizer_ecc512_seeds[] = {
 686         0x3346, 0x367f, 0x1f18, 0x769a, 0x4f64, 0x068c, 0x2ef1, 0x6b64,
 687         0x28a9, 0x15d7, 0x30f8, 0x3659, 0x53db, 0x7c5f, 0x71d4, 0x4409,
 688         0x26eb, 0x03cc, 0x655d, 0x47d4, 0x4daa, 0x0877, 0x712d, 0x3617,
 689         0x3264, 0x49aa, 0x7f9e, 0x588e, 0x4fbc, 0x7176, 0x7f91, 0x6c6d,
 690         0x4b95, 0x5fb7, 0x3844, 0x4037, 0x0184, 0x081b, 0x0ee8, 0x5b91,
 691         0x293d, 0x1f71, 0x0e6f, 0x402b, 0x5122, 0x1e52, 0x22be, 0x3d2d,
 692         0x75bc, 0x7c60, 0x6291, 0x1a2f, 0x61d4, 0x74aa, 0x4140, 0x29ab,
 693         0x472d, 0x2852, 0x017e, 0x15e8, 0x5ec2, 0x17cf, 0x7d0f, 0x06b8,
 694         0x117a, 0x6b94, 0x789b, 0x3126, 0x6ac5, 0x5be7, 0x150f, 0x51f8,
 695         0x7889, 0x0aa5, 0x663d, 0x77e8, 0x0b87, 0x3dcb, 0x360d, 0x218b,
 696         0x512f, 0x7dc9, 0x6a4d, 0x630a, 0x3547, 0x1dd2, 0x5aea, 0x69a5,
 697         0x7bfa, 0x5e4f, 0x1519, 0x6430, 0x3a0e, 0x5eb3, 0x5425, 0x0c7a,
 698         0x5540, 0x3670, 0x63c1, 0x31e9, 0x5a39, 0x2de7, 0x5979, 0x2891,
 699         0x1562, 0x014b, 0x5b05, 0x2756, 0x5a34, 0x13aa, 0x6cb5, 0x2c36,
 700         0x5e72, 0x1306, 0x0861, 0x15ef, 0x1ee8, 0x5a37, 0x7ac4, 0x45dd,
 701         0x44c4, 0x7266, 0x2f41, 0x3ccc, 0x045e, 0x7d40, 0x7c66, 0x0fa0,
 702 };
 703
 704 static const u16 sunxi_nfc_randomizer_ecc1024_seeds[] = {
 705         0x2cf5, 0x35f1, 0x63a4, 0x5274, 0x2bd2, 0x778b, 0x7285, 0x32b6,
 706         0x6a5c, 0x70d6, 0x757d, 0x6769, 0x5375, 0x1e81, 0x0cf3, 0x3982,
 707         0x6787, 0x042a, 0x6c49, 0x1925, 0x56a8, 0x40a9, 0x063e, 0x7bd9,
 708         0x4dbf, 0x55ec, 0x672e, 0x7334, 0x5185, 0x4d00, 0x232a, 0x7e07,
 709         0x445d, 0x6b92, 0x528f, 0x4255, 0x53ba, 0x7d82, 0x2a2e, 0x3a4e,
 710         0x75eb, 0x450c, 0x6844, 0x1b5d, 0x581a, 0x4cc6, 0x0379, 0x37b2,
 711         0x419f, 0x0e92, 0x6b27, 0x5624, 0x01e3, 0x07c1, 0x44a5, 0x130c,
 712         0x13e8, 0x5910, 0x0876, 0x60c5, 0x54e3, 0x5b7f, 0x2269, 0x509f,
 713         0x7665, 0x36fd, 0x3e9a, 0x0579, 0x6295, 0x14ef, 0x0a81, 0x1bcc,
 714         0x4b16, 0x64db, 0x0514, 0x4f07, 0x0591, 0x3576, 0x6853, 0x0d9e,
 715         0x259f, 0x38b7, 0x64fb, 0x3094, 0x4693, 0x6ddd, 0x29bb, 0x0bc8,
 716         0x3f47, 0x490e, 0x0c0e, 0x7933, 0x3c9e, 0x5840, 0x398d, 0x3e68,
 717         0x4af1, 0x71f5, 0x57cf, 0x1121, 0x64eb, 0x3579, 0x15ac, 0x584d,
 718         0x5f2a, 0x47e2, 0x6528, 0x6eac, 0x196e, 0x6b96, 0x0450, 0x0179,
 719         0x609c, 0x06e1, 0x4626, 0x42c7, 0x273e, 0x486f, 0x0705, 0x1601,
 720         0x145b, 0x407e, 0x062b, 0x57a5, 0x53f9, 0x5659, 0x4410, 0x3ccd,
 721 };
 722
 723 static u16 sunxi_nfc_randomizer_step(u16 state, int count)
 724 {
 725         state &= 0x7fff;
 726
 727         /*
 728          * This loop is just a simple implementation of a Fibonacci LFSR using
 729          * the x16 + x15 + 1 polynomial.
 730          */
 731         while (count--)
 732                 state = ((state >> 1) |
 733                          (((state ^ (state >> 1)) & 1) << 14)) & 0x7fff;
 734
 735         return state;
 736 }
 737
 738 static u16 sunxi_nfc_randomizer_state(struct mtd_info *mtd, int page, bool ecc)
 739 {
 740         const u16 *seeds = sunxi_nfc_randomizer_page_seeds;
 741         int mod = mtd_div_by_ws(mtd->erasesize, mtd);
 742
 743         if (mod > ARRAY_SIZE(sunxi_nfc_randomizer_page_seeds))
 744                 mod = ARRAY_SIZE(sunxi_nfc_randomizer_page_seeds);
 745
 746         if (ecc) {
 747                 if (mtd->ecc_step_size == 512)
 748                         seeds = sunxi_nfc_randomizer_ecc512_seeds;
 749                 else
 750                         seeds = sunxi_nfc_randomizer_ecc1024_seeds;
 751         }
 752
 753         return seeds[page % mod];
 754 }
 755
 756 static void sunxi_nfc_randomizer_config(struct mtd_info *mtd,
 757                                         int page, bool ecc)
 758 {
 759         struct nand_chip *nand = mtd_to_nand(mtd);
 760         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 761         u32 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
 762         u16 state;
 763
 764         if (!(nand->options & NAND_NEED_SCRAMBLING))
 765                 return;
 766
 767         ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
 768         state = sunxi_nfc_randomizer_state(mtd, page, ecc);
 769         ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_RANDOM_SEED_MSK;
 770         writel(ecc_ctl | NFC_RANDOM_SEED(state), nfc->regs + NFC_REG_ECC_CTL);
 771 }
 772
 773 static void sunxi_nfc_randomizer_enable(struct mtd_info *mtd)
 774 {
 775         struct nand_chip *nand = mtd_to_nand(mtd);
 776         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 777
 778         if (!(nand->options & NAND_NEED_SCRAMBLING))
 779                 return;
 780
 781         writel(readl(nfc->regs + NFC_REG_ECC_CTL) | NFC_RANDOM_EN,
 782                nfc->regs + NFC_REG_ECC_CTL);
 783 }
 784
 785 static void sunxi_nfc_randomizer_disable(struct mtd_info *mtd)
 786 {
 787         struct nand_chip *nand = mtd_to_nand(mtd);
 788         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 789
 790         if (!(nand->options & NAND_NEED_SCRAMBLING))
 791                 return;
 792
 793         writel(readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_RANDOM_EN,
 794                nfc->regs + NFC_REG_ECC_CTL);
 795 }
 796
 797 static void sunxi_nfc_randomize_bbm(struct mtd_info *mtd, int page, u8 *bbm)
 798 {
 799         u16 state = sunxi_nfc_randomizer_state(mtd, page, true);
 800
 801         bbm[0] ^= state;
 802         bbm[1] ^= sunxi_nfc_randomizer_step(state, 8);
 803 }
 804
 805 static void sunxi_nfc_randomizer_write_buf(struct mtd_info *mtd,
 806                                            const uint8_t *buf, int len,
 807                                            bool ecc, int page)
 808 {
 809         sunxi_nfc_randomizer_config(mtd, page, ecc);
 810         sunxi_nfc_randomizer_enable(mtd);
 811         sunxi_nfc_write_buf(mtd, buf, len);
 812         sunxi_nfc_randomizer_disable(mtd);
 813 }
 814
 815 static void sunxi_nfc_randomizer_read_buf(struct mtd_info *mtd, uint8_t *buf,
 816                                           int len, bool ecc, int page)
 817 {
 818         sunxi_nfc_randomizer_config(mtd, page, ecc);
 819         sunxi_nfc_randomizer_enable(mtd);
 820         sunxi_nfc_read_buf(mtd, buf, len);
 821         sunxi_nfc_randomizer_disable(mtd);
 822 }
 823
 824 static void sunxi_nfc_hw_ecc_enable(struct mtd_info *mtd)
 825 {
 826         struct nand_chip *nand = mtd_to_nand(mtd);
 827         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 828         struct sunxi_nand_hw_ecc *data = nand->ecc.priv;
 829         u32 ecc_ctl;
 830
 831         ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
 832         ecc_ctl &= ~(NFC_ECC_MODE_MSK | NFC_ECC_PIPELINE |
 833                      NFC_ECC_BLOCK_SIZE_MSK);
 834         ecc_ctl |= NFC_ECC_EN | NFC_ECC_MODE(data->mode) | NFC_ECC_EXCEPTION |
 835                    NFC_ECC_PIPELINE;
 836
 837         if (nand->ecc.size == 512)
 838                 ecc_ctl |= NFC_ECC_BLOCK_512;
 839
 840         writel(ecc_ctl, nfc->regs + NFC_REG_ECC_CTL);
 841 }
 842
 843 static void sunxi_nfc_hw_ecc_disable(struct mtd_info *mtd)
 844 {
 845         struct nand_chip *nand = mtd_to_nand(mtd);
 846         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 847
 848         writel(readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_ECC_EN,
 849                nfc->regs + NFC_REG_ECC_CTL);
 850 }
 851
 852 static inline void sunxi_nfc_user_data_to_buf(u32 user_data, u8 *buf)
 853 {
 854         buf[0] = user_data;
 855         buf[1] = user_data >> 8;
 856         buf[2] = user_data >> 16;
 857         buf[3] = user_data >> 24;
 858 }
 859
 860 static inline u32 sunxi_nfc_buf_to_user_data(const u8 *buf)
 861 {
 862         return buf[0] | (buf[1] << 8) | (buf[2] << 16) | (buf[3] << 24);
 863 }
 864
 865 static void sunxi_nfc_hw_ecc_get_prot_oob_bytes(struct mtd_info *mtd, u8 *oob,
 866                                                 int step, bool bbm, int page)
 867 {
 868         struct nand_chip *nand = mtd_to_nand(mtd);
 869         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 870
 871         sunxi_nfc_user_data_to_buf(readl(nfc->regs + NFC_REG_USER_DATA(step)),
 872                                    oob);
 873
 874         /* De-randomize the Bad Block Marker. */
 875         if (bbm && (nand->options & NAND_NEED_SCRAMBLING))
 876                 sunxi_nfc_randomize_bbm(mtd, page, oob);
 877 }
 878
 879 static void sunxi_nfc_hw_ecc_set_prot_oob_bytes(struct mtd_info *mtd,
 880                                                 const u8 *oob, int step,
 881                                                 bool bbm, int page)
 882 {
 883         struct nand_chip *nand = mtd_to_nand(mtd);
 884         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 885         u8 user_data[4];
 886
 887         /* Randomize the Bad Block Marker. */
 888         if (bbm && (nand->options & NAND_NEED_SCRAMBLING)) {
 889                 memcpy(user_data, oob, sizeof(user_data));
 890                 sunxi_nfc_randomize_bbm(mtd, page, user_data);
 891                 oob = user_data;
 892         }
 893
 894         writel(sunxi_nfc_buf_to_user_data(oob),
 895                nfc->regs + NFC_REG_USER_DATA(step));
 896 }
 897
 898 static void sunxi_nfc_hw_ecc_update_stats(struct mtd_info *mtd,
 899                                           unsigned int *max_bitflips, int ret)
 900 {
 901         if (ret < 0) {
 902                 mtd->ecc_stats.failed++;
 903         } else {
 904                 mtd->ecc_stats.corrected += ret;
 905                 *max_bitflips = max_t(unsigned int, *max_bitflips, ret);
 906         }
 907 }
 908
 909 static int sunxi_nfc_hw_ecc_correct(struct mtd_info *mtd, u8 *data, u8 *oob,
 910                                     int step, u32 status, bool *erased)
 911 {
 912         struct nand_chip *nand = mtd_to_nand(mtd);
 913         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 914         struct nand_ecc_ctrl *ecc = &nand->ecc;
 915         u32 tmp;
 916
 917         *erased = false;
 918
 919         if (status & NFC_ECC_ERR(step))
 920                 return -EBADMSG;
 921
 922         if (status & NFC_ECC_PAT_FOUND(step)) {
 923                 u8 pattern;
 924
 925                 if (unlikely(!(readl(nfc->regs + NFC_REG_PAT_ID) & 0x1))) {
 926                         pattern = 0x0;
 927                 } else {
 928                         pattern = 0xff;
 929                         *erased = true;
 930                 }
 931
 932                 if (data)
 933                         memset(data, pattern, ecc->size);
 934
 935                 if (oob)
 936                         memset(oob, pattern, ecc->bytes + 4);
 937
 938                 return 0;
 939         }
 940
 941         tmp = readl(nfc->regs + NFC_REG_ECC_ERR_CNT(step));
 942
 943         return NFC_ECC_ERR_CNT(step, tmp);
 944 }
 945
 946 static int sunxi_nfc_hw_ecc_read_chunk(struct mtd_info *mtd,
 947                                        u8 *data, int data_off,
 948                                        u8 *oob, int oob_off,
 949                                        int *cur_off,
 950                                        unsigned int *max_bitflips,
 951                                        bool bbm, bool oob_required, int page)
 952 {
 953         struct nand_chip *nand = mtd_to_nand(mtd);
 954         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 955         struct nand_ecc_ctrl *ecc = &nand->ecc;
 956         int raw_mode = 0;
 957         bool erased;
 958         int ret;
 959
 960         if (*cur_off != data_off)
 961                 nand->cmdfunc(mtd, NAND_CMD_RNDOUT, data_off, -1);
 962
 963         sunxi_nfc_randomizer_read_buf(mtd, NULL, ecc->size, false, page);
 964
 965         if (data_off + ecc->size != oob_off)
 966                 nand->cmdfunc(mtd, NAND_CMD_RNDOUT, oob_off, -1);
 967
 968         ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
 969         if (ret)
 970                 return ret;
 971
 972         sunxi_nfc_randomizer_enable(mtd);
 973         writel(NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD | NFC_ECC_OP,
 974                nfc->regs + NFC_REG_CMD);
 975
 976         ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0);
 977         sunxi_nfc_randomizer_disable(mtd);
 978         if (ret)
 979                 return ret;
 980
 981         *cur_off = oob_off + ecc->bytes + 4;
 982
 983         ret = sunxi_nfc_hw_ecc_correct(mtd, data, oob_required ? oob : NULL, 0,
 984                                        readl(nfc->regs + NFC_REG_ECC_ST),
 985                                        &erased);
 986         if (erased)
 987                 return 1;
 988
 989         if (ret < 0) {
 990                 /*
 991                  * Re-read the data with the randomizer disabled to identify
 992                  * bitflips in erased pages.
 993                  */
 994                 if (nand->options & NAND_NEED_SCRAMBLING) {
 995                         nand->cmdfunc(mtd, NAND_CMD_RNDOUT, data_off, -1);
 996                         nand->read_buf(mtd, data, ecc->size);
 997                 } else {
 998                         memcpy_fromio(data, nfc->regs + NFC_RAM0_BASE,
 999                                       ecc->size);
1000                 }
1001
1002                 nand->cmdfunc(mtd, NAND_CMD_RNDOUT, oob_off, -1);
1003                 nand->read_buf(mtd, oob, ecc->bytes + 4);
1004
1005                 ret = nand_check_erased_ecc_chunk(data, ecc->size,
1006                                                   oob, ecc->bytes + 4,
1007                                                   NULL, 0, ecc->strength);
1008                 if (ret >= 0)
1009                         raw_mode = 1;
1010         } else {
1011                 memcpy_fromio(data, nfc->regs + NFC_RAM0_BASE, ecc->size);
1012
1013                 if (oob_required) {
1014                         nand->cmdfunc(mtd, NAND_CMD_RNDOUT, oob_off, -1);
1015                         sunxi_nfc_randomizer_read_buf(mtd, oob, ecc->bytes + 4,
1016                                                       true, page);
1017
1018                         sunxi_nfc_hw_ecc_get_prot_oob_bytes(mtd, oob, 0,
1019                                                             bbm, page);
1020                 }
1021         }
1022
1023         sunxi_nfc_hw_ecc_update_stats(mtd, max_bitflips, ret);
1024
1025         return raw_mode;
1026 }
1027
1028 static void sunxi_nfc_hw_ecc_read_extra_oob(struct mtd_info *mtd,
1029                                             u8 *oob, int *cur_off,
1030                                             bool randomize, int page)
1031 {
1032         struct nand_chip *nand = mtd_to_nand(mtd);
1033         struct nand_ecc_ctrl *ecc = &nand->ecc;
1034         int offset = ((ecc->bytes + 4) * ecc->steps);
1035         int len = mtd->oobsize - offset;
1036
1037         if (len <= 0)
1038                 return;
1039
1040         if (!cur_off || *cur_off != offset)
1041                 nand->cmdfunc(mtd, NAND_CMD_RNDOUT,
1042                               offset + mtd->writesize, -1);
1043
1044         if (!randomize)
1045                 sunxi_nfc_read_buf(mtd, oob + offset, len);
1046         else
1047                 sunxi_nfc_randomizer_read_buf(mtd, oob + offset, len,
1048                                               false, page);
1049
1050         if (cur_off)
1051                 *cur_off = mtd->oobsize + mtd->writesize;
1052 }
1053
1054 static int sunxi_nfc_hw_ecc_read_chunks_dma(struct mtd_info *mtd, uint8_t *buf,
1055                                             int oob_required, int page,
1056                                             int nchunks)
1057 {
1058         struct nand_chip *nand = mtd_to_nand(mtd);
1059         bool randomized = nand->options & NAND_NEED_SCRAMBLING;
1060         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1061         struct nand_ecc_ctrl *ecc = &nand->ecc;
1062         unsigned int max_bitflips = 0;
1063         int ret, i, raw_mode = 0;
1064         struct scatterlist sg;
1065         u32 status;
1066
1067         ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1068         if (ret)
1069                 return ret;
1070
1071         ret = sunxi_nfc_dma_op_prepare(mtd, buf, ecc->size, nchunks,
1072                                        DMA_FROM_DEVICE, &sg);
1073         if (ret)
1074                 return ret;
1075
1076         sunxi_nfc_hw_ecc_enable(mtd);
1077         sunxi_nfc_randomizer_config(mtd, page, false);
1078         sunxi_nfc_randomizer_enable(mtd);
1079
1080         writel((NAND_CMD_RNDOUTSTART << 16) | (NAND_CMD_RNDOUT << 8) |
1081                NAND_CMD_READSTART, nfc->regs + NFC_REG_RCMD_SET);
1082
1083         dma_async_issue_pending(nfc->dmac);
1084
1085         writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD | NFC_DATA_TRANS,
1086                nfc->regs + NFC_REG_CMD);
1087
1088         ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0);
1089         if (ret)
1090                 dmaengine_terminate_all(nfc->dmac);
1091
1092         sunxi_nfc_randomizer_disable(mtd);
1093         sunxi_nfc_hw_ecc_disable(mtd);
1094
1095         sunxi_nfc_dma_op_cleanup(mtd, DMA_FROM_DEVICE, &sg);
1096
1097         if (ret)
1098                 return ret;
1099
1100         status = readl(nfc->regs + NFC_REG_ECC_ST);
1101
1102         for (i = 0; i < nchunks; i++) {
1103                 int data_off = i * ecc->size;
1104                 int oob_off = i * (ecc->bytes + 4);
1105                 u8 *data = buf + data_off;
1106                 u8 *oob = nand->oob_poi + oob_off;
1107                 bool erased;
1108
1109                 ret = sunxi_nfc_hw_ecc_correct(mtd, randomized ? data : NULL,
1110                                                oob_required ? oob : NULL,
1111                                                i, status, &erased);
1112
1113                 /* ECC errors are handled in the second loop. */
1114                 if (ret < 0)
1115                         continue;
1116
1117                 if (oob_required && !erased) {
1118                         /* TODO: use DMA to retrieve OOB */
1119                         nand->cmdfunc(mtd, NAND_CMD_RNDOUT,
1120                                       mtd->writesize + oob_off, -1);
1121                         nand->read_buf(mtd, oob, ecc->bytes + 4);
1122
1123                         sunxi_nfc_hw_ecc_get_prot_oob_bytes(mtd, oob, i,
1124                                                             !i, page);
1125                 }
1126
1127                 if (erased)
1128                         raw_mode = 1;
1129
1130                 sunxi_nfc_hw_ecc_update_stats(mtd, &max_bitflips, ret);
1131         }
1132
1133         if (status & NFC_ECC_ERR_MSK) {
1134                 for (i = 0; i < nchunks; i++) {
1135                         int data_off = i * ecc->size;
1136                         int oob_off = i * (ecc->bytes + 4);
1137                         u8 *data = buf + data_off;
1138                         u8 *oob = nand->oob_poi + oob_off;
1139
1140                         if (!(status & NFC_ECC_ERR(i)))
1141                                 continue;
1142
1143                         /*
1144                          * Re-read the data with the randomizer disabled to
1145                          * identify bitflips in erased pages.
1146                          */
1147                         if (randomized) {
1148                                 /* TODO: use DMA to read page in raw mode */
1149                                 nand->cmdfunc(mtd, NAND_CMD_RNDOUT,
1150                                               data_off, -1);
1151                                 nand->read_buf(mtd, data, ecc->size);
1152                         }
1153
1154                         /* TODO: use DMA to retrieve OOB */
1155                         nand->cmdfunc(mtd, NAND_CMD_RNDOUT,
1156                                       mtd->writesize + oob_off, -1);
1157                         nand->read_buf(mtd, oob, ecc->bytes + 4);
1158
1159                         ret = nand_check_erased_ecc_chunk(data, ecc->size,
1160                                                           oob, ecc->bytes + 4,
1161                                                           NULL, 0,
1162                                                           ecc->strength);
1163                         if (ret >= 0)
1164                                 raw_mode = 1;
1165
1166                         sunxi_nfc_hw_ecc_update_stats(mtd, &max_bitflips, ret);
1167                 }
1168         }
1169
1170         if (oob_required)
1171                 sunxi_nfc_hw_ecc_read_extra_oob(mtd, nand->oob_poi,
1172                                                 NULL, !raw_mode,
1173                                                 page);
1174
1175         return max_bitflips;
1176 }
1177
1178 static int sunxi_nfc_hw_ecc_write_chunk(struct mtd_info *mtd,
1179                                         const u8 *data, int data_off,
1180                                         const u8 *oob, int oob_off,
1181                                         int *cur_off, bool bbm,
1182                                         int page)
1183 {
1184         struct nand_chip *nand = mtd_to_nand(mtd);
1185         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1186         struct nand_ecc_ctrl *ecc = &nand->ecc;
1187         int ret;
1188
1189         if (data_off != *cur_off)
1190                 nand->cmdfunc(mtd, NAND_CMD_RNDIN, data_off, -1);
1191
1192         sunxi_nfc_randomizer_write_buf(mtd, data, ecc->size, false, page);
1193
1194         if (data_off + ecc->size != oob_off)
1195                 nand->cmdfunc(mtd, NAND_CMD_RNDIN, oob_off, -1);
1196
1197         ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1198         if (ret)
1199                 return ret;
1200
1201         sunxi_nfc_randomizer_enable(mtd);
1202         sunxi_nfc_hw_ecc_set_prot_oob_bytes(mtd, oob, 0, bbm, page);
1203
1204         writel(NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD |
1205                NFC_ACCESS_DIR | NFC_ECC_OP,
1206                nfc->regs + NFC_REG_CMD);
1207
1208         ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0);
1209         sunxi_nfc_randomizer_disable(mtd);
1210         if (ret)
1211                 return ret;
1212
1213         *cur_off = oob_off + ecc->bytes + 4;
1214
1215         return 0;
1216 }
1217
1218 static void sunxi_nfc_hw_ecc_write_extra_oob(struct mtd_info *mtd,
1219                                              u8 *oob, int *cur_off,
1220                                              int page)
1221 {
1222         struct nand_chip *nand = mtd_to_nand(mtd);
1223         struct nand_ecc_ctrl *ecc = &nand->ecc;
1224         int offset = ((ecc->bytes + 4) * ecc->steps);
1225         int len = mtd->oobsize - offset;
1226
1227         if (len <= 0)
1228                 return;
1229
1230         if (!cur_off || *cur_off != offset)
1231                 nand->cmdfunc(mtd, NAND_CMD_RNDIN,
1232                               offset + mtd->writesize, -1);
1233
1234         sunxi_nfc_randomizer_write_buf(mtd, oob + offset, len, false, page);
1235
1236         if (cur_off)
1237                 *cur_off = mtd->oobsize + mtd->writesize;
1238 }
1239
1240 static int sunxi_nfc_hw_ecc_read_page(struct mtd_info *mtd,
1241                                       struct nand_chip *chip, uint8_t *buf,
1242                                       int oob_required, int page)
1243 {
1244         struct nand_ecc_ctrl *ecc = &chip->ecc;
1245         unsigned int max_bitflips = 0;
1246         int ret, i, cur_off = 0;
1247         bool raw_mode = false;
1248
1249         sunxi_nfc_hw_ecc_enable(mtd);
1250
1251         for (i = 0; i < ecc->steps; i++) {
1252                 int data_off = i * ecc->size;
1253                 int oob_off = i * (ecc->bytes + 4);
1254                 u8 *data = buf + data_off;
1255                 u8 *oob = chip->oob_poi + oob_off;
1256
1257                 ret = sunxi_nfc_hw_ecc_read_chunk(mtd, data, data_off, oob,
1258                                                   oob_off + mtd->writesize,
1259                                                   &cur_off, &max_bitflips,
1260                                                   !i, oob_required, page);
1261                 if (ret < 0)
1262                         return ret;
1263                 else if (ret)
1264                         raw_mode = true;
1265         }
1266
1267         if (oob_required)
1268                 sunxi_nfc_hw_ecc_read_extra_oob(mtd, chip->oob_poi, &cur_off,
1269                                                 !raw_mode, page);
1270
1271         sunxi_nfc_hw_ecc_disable(mtd);
1272
1273         return max_bitflips;
1274 }
1275
1276 static int sunxi_nfc_hw_ecc_read_page_dma(struct mtd_info *mtd,
1277                                           struct nand_chip *chip, u8 *buf,
1278                                           int oob_required, int page)
1279 {
1280         int ret;
1281
1282         ret = sunxi_nfc_hw_ecc_read_chunks_dma(mtd, buf, oob_required, page,
1283                                                chip->ecc.steps);
1284         if (ret >= 0)
1285                 return ret;
1286
1287         /* Fallback to PIO mode */
1288         chip->cmdfunc(mtd, NAND_CMD_RNDOUT, 0, -1);
1289
1290         return sunxi_nfc_hw_ecc_read_page(mtd, chip, buf, oob_required, page);
1291 }
1292
1293 static int sunxi_nfc_hw_ecc_read_subpage(struct mtd_info *mtd,
1294                                          struct nand_chip *chip,
1295                                          u32 data_offs, u32 readlen,
1296                                          u8 *bufpoi, int page)
1297 {
1298         struct nand_ecc_ctrl *ecc = &chip->ecc;
1299         int ret, i, cur_off = 0;
1300         unsigned int max_bitflips = 0;
1301
1302         sunxi_nfc_hw_ecc_enable(mtd);
1303
1304         chip->cmdfunc(mtd, NAND_CMD_READ0, 0, page);
1305         for (i = data_offs / ecc->size;
1306              i < DIV_ROUND_UP(data_offs + readlen, ecc->size); i++) {
1307                 int data_off = i * ecc->size;
1308                 int oob_off = i * (ecc->bytes + 4);
1309                 u8 *data = bufpoi + data_off;
1310                 u8 *oob = chip->oob_poi + oob_off;
1311
1312                 ret = sunxi_nfc_hw_ecc_read_chunk(mtd, data, data_off,
1313                                                   oob,
1314                                                   oob_off + mtd->writesize,
1315                                                   &cur_off, &max_bitflips, !i,
1316                                                   false, page);
1317                 if (ret < 0)
1318                         return ret;
1319         }
1320
1321         sunxi_nfc_hw_ecc_disable(mtd);
1322
1323         return max_bitflips;
1324 }
1325
1326 static int sunxi_nfc_hw_ecc_read_subpage_dma(struct mtd_info *mtd,
1327                                              struct nand_chip *chip,
1328                                              u32 data_offs, u32 readlen,
1329                                              u8 *buf, int page)
1330 {
1331         int nchunks = DIV_ROUND_UP(data_offs + readlen, chip->ecc.size);
1332         int ret;
1333
1334         ret = sunxi_nfc_hw_ecc_read_chunks_dma(mtd, buf, false, page, nchunks);
1335         if (ret >= 0)
1336                 return ret;
1337
1338         /* Fallback to PIO mode */
1339         chip->cmdfunc(mtd, NAND_CMD_RNDOUT, 0, -1);
1340
1341         return sunxi_nfc_hw_ecc_read_subpage(mtd, chip, data_offs, readlen,
1342                                              buf, page);
1343 }
1344
1345 static int sunxi_nfc_hw_ecc_write_page(struct mtd_info *mtd,
1346                                        struct nand_chip *chip,
1347                                        const uint8_t *buf, int oob_required,
1348                                        int page)
1349 {
1350         struct nand_ecc_ctrl *ecc = &chip->ecc;
1351         int ret, i, cur_off = 0;
1352
1353         sunxi_nfc_hw_ecc_enable(mtd);
1354
1355         for (i = 0; i < ecc->steps; i++) {
1356                 int data_off = i * ecc->size;
1357                 int oob_off = i * (ecc->bytes + 4);
1358                 const u8 *data = buf + data_off;
1359                 const u8 *oob = chip->oob_poi + oob_off;
1360
1361                 ret = sunxi_nfc_hw_ecc_write_chunk(mtd, data, data_off, oob,
1362                                                    oob_off + mtd->writesize,
1363                                                    &cur_off, !i, page);
1364                 if (ret)
1365                         return ret;
1366         }
1367
1368         if (oob_required || (chip->options & NAND_NEED_SCRAMBLING))
1369                 sunxi_nfc_hw_ecc_write_extra_oob(mtd, chip->oob_poi,
1370                                                  &cur_off, page);
1371
1372         sunxi_nfc_hw_ecc_disable(mtd);
1373
1374         return 0;
1375 }
1376
1377 static int sunxi_nfc_hw_ecc_write_subpage(struct mtd_info *mtd,
1378                                           struct nand_chip *chip,
1379                                           u32 data_offs, u32 data_len,
1380                                           const u8 *buf, int oob_required,
1381                                           int page)
1382 {
1383         struct nand_ecc_ctrl *ecc = &chip->ecc;
1384         int ret, i, cur_off = 0;
1385
1386         sunxi_nfc_hw_ecc_enable(mtd);
1387
1388         for (i = data_offs / ecc->size;
1389              i < DIV_ROUND_UP(data_offs + data_len, ecc->size); i++) {
1390                 int data_off = i * ecc->size;
1391                 int oob_off = i * (ecc->bytes + 4);
1392                 const u8 *data = buf + data_off;
1393                 const u8 *oob = chip->oob_poi + oob_off;
1394
1395                 ret = sunxi_nfc_hw_ecc_write_chunk(mtd, data, data_off, oob,
1396                                                    oob_off + mtd->writesize,
1397                                                    &cur_off, !i, page);
1398                 if (ret)
1399                         return ret;
1400         }
1401
1402         sunxi_nfc_hw_ecc_disable(mtd);
1403
1404         return 0;
1405 }
1406
1407 static int sunxi_nfc_hw_ecc_write_page_dma(struct mtd_info *mtd,
1408                                            struct nand_chip *chip,
1409                                            const u8 *buf,
1410                                            int oob_required,
1411                                            int page)
1412 {
1413         struct nand_chip *nand = mtd_to_nand(mtd);
1414         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1415         struct nand_ecc_ctrl *ecc = &nand->ecc;
1416         struct scatterlist sg;
1417         int ret, i;
1418
1419         ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1420         if (ret)
1421                 return ret;
1422
1423         ret = sunxi_nfc_dma_op_prepare(mtd, buf, ecc->size, ecc->steps,
1424                                        DMA_TO_DEVICE, &sg);
1425         if (ret)
1426                 goto pio_fallback;
1427
1428         for (i = 0; i < ecc->steps; i++) {
1429                 const u8 *oob = nand->oob_poi + (i * (ecc->bytes + 4));
1430
1431                 sunxi_nfc_hw_ecc_set_prot_oob_bytes(mtd, oob, i, !i, page);
1432         }
1433
1434         sunxi_nfc_hw_ecc_enable(mtd);
1435         sunxi_nfc_randomizer_config(mtd, page, false);
1436         sunxi_nfc_randomizer_enable(mtd);
1437
1438         writel((NAND_CMD_RNDIN << 8) | NAND_CMD_PAGEPROG,
1439                nfc->regs + NFC_REG_RCMD_SET);
1440
1441         dma_async_issue_pending(nfc->dmac);
1442
1443         writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD |
1444                NFC_DATA_TRANS | NFC_ACCESS_DIR,
1445                nfc->regs + NFC_REG_CMD);
1446
1447         ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0);
1448         if (ret)
1449                 dmaengine_terminate_all(nfc->dmac);
1450
1451         sunxi_nfc_randomizer_disable(mtd);
1452         sunxi_nfc_hw_ecc_disable(mtd);
1453
1454         sunxi_nfc_dma_op_cleanup(mtd, DMA_TO_DEVICE, &sg);
1455
1456         if (ret)
1457                 return ret;
1458
1459         if (oob_required || (chip->options & NAND_NEED_SCRAMBLING))
1460                 /* TODO: use DMA to transfer extra OOB bytes ? */
1461                 sunxi_nfc_hw_ecc_write_extra_oob(mtd, chip->oob_poi,
1462                                                  NULL, page);
1463
1464         return 0;
1465
1466 pio_fallback:
1467         return sunxi_nfc_hw_ecc_write_page(mtd, chip, buf, oob_required, page);
1468 }
1469
1470 static int sunxi_nfc_hw_syndrome_ecc_read_page(struct mtd_info *mtd,
1471                                                struct nand_chip *chip,
1472                                                uint8_t *buf, int oob_required,
1473                                                int page)
1474 {
1475         struct nand_ecc_ctrl *ecc = &chip->ecc;
1476         unsigned int max_bitflips = 0;
1477         int ret, i, cur_off = 0;
1478         bool raw_mode = false;
1479
1480         sunxi_nfc_hw_ecc_enable(mtd);
1481
1482         for (i = 0; i < ecc->steps; i++) {
1483                 int data_off = i * (ecc->size + ecc->bytes + 4);
1484                 int oob_off = data_off + ecc->size;
1485                 u8 *data = buf + (i * ecc->size);
1486                 u8 *oob = chip->oob_poi + (i * (ecc->bytes + 4));
1487
1488                 ret = sunxi_nfc_hw_ecc_read_chunk(mtd, data, data_off, oob,
1489                                                   oob_off, &cur_off,
1490                                                   &max_bitflips, !i,
1491                                                   oob_required,
1492                                                   page);
1493                 if (ret < 0)
1494                         return ret;
1495                 else if (ret)
1496                         raw_mode = true;
1497         }
1498
1499         if (oob_required)
1500                 sunxi_nfc_hw_ecc_read_extra_oob(mtd, chip->oob_poi, &cur_off,
1501                                                 !raw_mode, page);
1502
1503         sunxi_nfc_hw_ecc_disable(mtd);
1504
1505         return max_bitflips;
1506 }
1507
1508 static int sunxi_nfc_hw_syndrome_ecc_write_page(struct mtd_info *mtd,
1509                                                 struct nand_chip *chip,
1510                                                 const uint8_t *buf,
1511                                                 int oob_required, int page)
1512 {
1513         struct nand_ecc_ctrl *ecc = &chip->ecc;
1514         int ret, i, cur_off = 0;
1515
1516         sunxi_nfc_hw_ecc_enable(mtd);
1517
1518         for (i = 0; i < ecc->steps; i++) {
1519                 int data_off = i * (ecc->size + ecc->bytes + 4);
1520                 int oob_off = data_off + ecc->size;
1521                 const u8 *data = buf + (i * ecc->size);
1522                 const u8 *oob = chip->oob_poi + (i * (ecc->bytes + 4));
1523
1524                 ret = sunxi_nfc_hw_ecc_write_chunk(mtd, data, data_off,
1525                                                    oob, oob_off, &cur_off,
1526                                                    false, page);
1527                 if (ret)
1528                         return ret;
1529         }
1530
1531         if (oob_required || (chip->options & NAND_NEED_SCRAMBLING))
1532                 sunxi_nfc_hw_ecc_write_extra_oob(mtd, chip->oob_poi,
1533                                                  &cur_off, page);
1534
1535         sunxi_nfc_hw_ecc_disable(mtd);
1536
1537         return 0;
1538 }
1539
1540 static int sunxi_nfc_hw_common_ecc_read_oob(struct mtd_info *mtd,
1541                                             struct nand_chip *chip,
1542                                             int page)
1543 {
1544         chip->cmdfunc(mtd, NAND_CMD_READ0, 0, page);
1545
1546         chip->pagebuf = -1;
1547
1548         return chip->ecc.read_page(mtd, chip, chip->buffers->databuf, 1, page);
1549 }
1550
1551 static int sunxi_nfc_hw_common_ecc_write_oob(struct mtd_info *mtd,
1552                                              struct nand_chip *chip,
1553                                              int page)
1554 {
1555         int ret, status;
1556
1557         chip->cmdfunc(mtd, NAND_CMD_SEQIN, 0, page);
1558
1559         chip->pagebuf = -1;
1560
1561         memset(chip->buffers->databuf, 0xff, mtd->writesize);
1562         ret = chip->ecc.write_page(mtd, chip, chip->buffers->databuf, 1, page);
1563         if (ret)
1564                 return ret;
1565
1566         /* Send command to program the OOB data */
1567         chip->cmdfunc(mtd, NAND_CMD_PAGEPROG, -1, -1);
1568
1569         status = chip->waitfunc(mtd, chip);
1570
1571         return status & NAND_STATUS_FAIL ? -EIO : 0;
1572 }
1573
1574 static const s32 tWB_lut[] = {6, 12, 16, 20};
1575 static const s32 tRHW_lut[] = {4, 8, 12, 20};
1576
1577 static int _sunxi_nand_lookup_timing(const s32 *lut, int lut_size, u32 duration,
1578                 u32 clk_period)
1579 {
1580         u32 clk_cycles = DIV_ROUND_UP(duration, clk_period);
1581         int i;
1582
1583         for (i = 0; i < lut_size; i++) {
1584                 if (clk_cycles <= lut[i])
1585                         return i;
1586         }
1587
1588         /* Doesn't fit */
1589         return -EINVAL;
1590 }
1591
1592 #define sunxi_nand_lookup_timing(l, p, c) \
1593                         _sunxi_nand_lookup_timing(l, ARRAY_SIZE(l), p, c)
1594
1595 static int sunxi_nfc_setup_data_interface(struct mtd_info *mtd, int csline,
1596                                         const struct nand_data_interface *conf)
1597 {
1598         struct nand_chip *nand = mtd_to_nand(mtd);
1599         struct sunxi_nand_chip *chip = to_sunxi_nand(nand);
1600         struct sunxi_nfc *nfc = to_sunxi_nfc(chip->nand.controller);
1601         const struct nand_sdr_timings *timings;
1602         u32 min_clk_period = 0;
1603         s32 tWB, tADL, tWHR, tRHW, tCAD;
1604         long real_clk_rate;
1605
1606         timings = nand_get_sdr_timings(conf);
1607         if (IS_ERR(timings))
1608                 return -ENOTSUPP;
1609
1610         /* T1 <=> tCLS */
1611         if (timings->tCLS_min > min_clk_period)
1612                 min_clk_period = timings->tCLS_min;
1613
1614         /* T2 <=> tCLH */
1615         if (timings->tCLH_min > min_clk_period)
1616                 min_clk_period = timings->tCLH_min;
1617
1618         /* T3 <=> tCS */
1619         if (timings->tCS_min > min_clk_period)
1620                 min_clk_period = timings->tCS_min;
1621
1622         /* T4 <=> tCH */
1623         if (timings->tCH_min > min_clk_period)
1624                 min_clk_period = timings->tCH_min;
1625
1626         /* T5 <=> tWP */
1627         if (timings->tWP_min > min_clk_period)
1628                 min_clk_period = timings->tWP_min;
1629
1630         /* T6 <=> tWH */
1631         if (timings->tWH_min > min_clk_period)
1632                 min_clk_period = timings->tWH_min;
1633
1634         /* T7 <=> tALS */
1635         if (timings->tALS_min > min_clk_period)
1636                 min_clk_period = timings->tALS_min;
1637
1638         /* T8 <=> tDS */
1639         if (timings->tDS_min > min_clk_period)
1640                 min_clk_period = timings->tDS_min;
1641
1642         /* T9 <=> tDH */
1643         if (timings->tDH_min > min_clk_period)
1644                 min_clk_period = timings->tDH_min;
1645
1646         /* T10 <=> tRR */
1647         if (timings->tRR_min > (min_clk_period * 3))
1648                 min_clk_period = DIV_ROUND_UP(timings->tRR_min, 3);
1649
1650         /* T11 <=> tALH */
1651         if (timings->tALH_min > min_clk_period)
1652                 min_clk_period = timings->tALH_min;
1653
1654         /* T12 <=> tRP */
1655         if (timings->tRP_min > min_clk_period)
1656                 min_clk_period = timings->tRP_min;
1657
1658         /* T13 <=> tREH */
1659         if (timings->tREH_min > min_clk_period)
1660                 min_clk_period = timings->tREH_min;
1661
1662         /* T14 <=> tRC */
1663         if (timings->tRC_min > (min_clk_period * 2))
1664                 min_clk_period = DIV_ROUND_UP(timings->tRC_min, 2);
1665
1666         /* T15 <=> tWC */
1667         if (timings->tWC_min > (min_clk_period * 2))
1668                 min_clk_period = DIV_ROUND_UP(timings->tWC_min, 2);
1669
1670         /* T16 - T19 + tCAD */
1671         if (timings->tWB_max > (min_clk_period * 20))
1672                 min_clk_period = DIV_ROUND_UP(timings->tWB_max, 20);
1673
1674         if (timings->tADL_min > (min_clk_period * 32))
1675                 min_clk_period = DIV_ROUND_UP(timings->tADL_min, 32);
1676
1677         if (timings->tWHR_min > (min_clk_period * 32))
1678                 min_clk_period = DIV_ROUND_UP(timings->tWHR_min, 32);
1679
1680         if (timings->tRHW_min > (min_clk_period * 20))
1681                 min_clk_period = DIV_ROUND_UP(timings->tRHW_min, 20);
1682
1683         tWB  = sunxi_nand_lookup_timing(tWB_lut, timings->tWB_max,
1684                                         min_clk_period);
1685         if (tWB < 0) {
1686                 dev_err(nfc->dev, "unsupported tWB\n");
1687                 return tWB;
1688         }
1689
1690         tADL = DIV_ROUND_UP(timings->tADL_min, min_clk_period) >> 3;
1691         if (tADL > 3) {
1692                 dev_err(nfc->dev, "unsupported tADL\n");
1693                 return -EINVAL;
1694         }
1695
1696         tWHR = DIV_ROUND_UP(timings->tWHR_min, min_clk_period) >> 3;
1697         if (tWHR > 3) {
1698                 dev_err(nfc->dev, "unsupported tWHR\n");
1699                 return -EINVAL;
1700         }
1701
1702         tRHW = sunxi_nand_lookup_timing(tRHW_lut, timings->tRHW_min,
1703                                         min_clk_period);
1704         if (tRHW < 0) {
1705                 dev_err(nfc->dev, "unsupported tRHW\n");
1706                 return tRHW;
1707         }
1708
1709         if (csline == NAND_DATA_IFACE_CHECK_ONLY)
1710                 return 0;
1711
1712         /*
1713          * TODO: according to ONFI specs this value only applies for DDR NAND,
1714          * but Allwinner seems to set this to 0x7. Mimic them for now.
1715          */
1716         tCAD = 0x7;
1717
1718         /* TODO: A83 has some more bits for CDQSS, CS, CLHZ, CCS, WC */
1719         chip->timing_cfg = NFC_TIMING_CFG(tWB, tADL, tWHR, tRHW, tCAD);
1720
1721         /* Convert min_clk_period from picoseconds to nanoseconds */
1722         min_clk_period = DIV_ROUND_UP(min_clk_period, 1000);
1723
1724         /*
1725          * Unlike what is stated in Allwinner datasheet, the clk_rate should
1726          * be set to (1 / min_clk_period), and not (2 / min_clk_period).
1727          * This new formula was verified with a scope and validated by
1728          * Allwinner engineers.
1729          */
1730         chip->clk_rate = NSEC_PER_SEC / min_clk_period;
1731         real_clk_rate = clk_round_rate(nfc->mod_clk, chip->clk_rate);
1732
1733         /*
1734          * ONFI specification 3.1, paragraph 4.15.2 dictates that EDO data
1735          * output cycle timings shall be used if the host drives tRC less than
1736          * 30 ns.
1737          */
1738         min_clk_period = NSEC_PER_SEC / real_clk_rate;
1739         chip->timing_ctl = ((min_clk_period * 2) < 30) ?
1740                            NFC_TIMING_CTL_EDO : 0;
1741
1742         return 0;
1743 }
1744
1745 static int sunxi_nand_ooblayout_ecc(struct mtd_info *mtd, int section,
1746                                     struct mtd_oob_region *oobregion)
1747 {
1748         struct nand_chip *nand = mtd_to_nand(mtd);
1749         struct nand_ecc_ctrl *ecc = &nand->ecc;
1750
1751         if (section >= ecc->steps)
1752                 return -ERANGE;
1753
1754         oobregion->offset = section * (ecc->bytes + 4) + 4;
1755         oobregion->length = ecc->bytes;
1756
1757         return 0;
1758 }
1759
1760 static int sunxi_nand_ooblayout_free(struct mtd_info *mtd, int section,
1761                                      struct mtd_oob_region *oobregion)
1762 {
1763         struct nand_chip *nand = mtd_to_nand(mtd);
1764         struct nand_ecc_ctrl *ecc = &nand->ecc;
1765
1766         if (section > ecc->steps)
1767                 return -ERANGE;
1768
1769         /*
1770          * The first 2 bytes are used for BB markers, hence we
1771          * only have 2 bytes available in the first user data
1772          * section.
1773          */
1774         if (!section && ecc->mode == NAND_ECC_HW) {
1775                 oobregion->offset = 2;
1776                 oobregion->length = 2;
1777
1778                 return 0;
1779         }
1780
1781         oobregion->offset = section * (ecc->bytes + 4);
1782
1783         if (section < ecc->steps)
1784                 oobregion->length = 4;
1785         else
1786                 oobregion->offset = mtd->oobsize - oobregion->offset;
1787
1788         return 0;
1789 }
1790
1791 static const struct mtd_ooblayout_ops sunxi_nand_ooblayout_ops = {
1792         .ecc = sunxi_nand_ooblayout_ecc,
1793         .free = sunxi_nand_ooblayout_free,
1794 };
1795
1796 static int sunxi_nand_hw_common_ecc_ctrl_init(struct mtd_info *mtd,
1797                                               struct nand_ecc_ctrl *ecc,
1798                                               struct device_node *np)
1799 {
1800         static const u8 strengths[] = { 16, 24, 28, 32, 40, 48, 56, 60, 64 };
1801         struct nand_chip *nand = mtd_to_nand(mtd);
1802         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
1803         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
1804         struct sunxi_nand_hw_ecc *data;
1805         int nsectors;
1806         int ret;
1807         int i;
1808
1809         if (ecc->options & NAND_ECC_MAXIMIZE) {
1810                 int bytes;
1811
1812                 ecc->size = 1024;
1813                 nsectors = mtd->writesize / ecc->size;
1814
1815                 /* Reserve 2 bytes for the BBM */
1816                 bytes = (mtd->oobsize - 2) / nsectors;
1817
1818                 /* 4 non-ECC bytes are added before each ECC bytes section */
1819                 bytes -= 4;
1820
1821                 /* and bytes has to be even. */
1822                 if (bytes % 2)
1823                         bytes--;
1824
1825                 ecc->strength = bytes * 8 / fls(8 * ecc->size);
1826
1827                 for (i = 0; i < ARRAY_SIZE(strengths); i++) {
1828                         if (strengths[i] > ecc->strength)
1829                                 break;
1830                 }
1831
1832                 if (!i)
1833                         ecc->strength = 0;
1834                 else
1835                         ecc->strength = strengths[i - 1];
1836         }
1837
1838         if (ecc->size != 512 && ecc->size != 1024)
1839                 return -EINVAL;
1840
1841         data = kzalloc(sizeof(*data), GFP_KERNEL);
1842         if (!data)
1843                 return -ENOMEM;
1844
1845         /* Prefer 1k ECC chunk over 512 ones */
1846         if (ecc->size == 512 && mtd->writesize > 512) {
1847                 ecc->size = 1024;
1848                 ecc->strength *= 2;
1849         }
1850
1851         /* Add ECC info retrieval from DT */
1852         for (i = 0; i < ARRAY_SIZE(strengths); i++) {
1853                 if (ecc->strength <= strengths[i])
1854                         break;
1855         }
1856
1857         if (i >= ARRAY_SIZE(strengths)) {
1858                 dev_err(nfc->dev, "unsupported strength\n");
1859                 ret = -ENOTSUPP;
1860                 goto err;
1861         }
1862
1863         data->mode = i;
1864
1865         /* HW ECC always request ECC bytes for 1024 bytes blocks */
1866         ecc->bytes = DIV_ROUND_UP(ecc->strength * fls(8 * 1024), 8);
1867
1868         /* HW ECC always work with even numbers of ECC bytes */
1869         ecc->bytes = ALIGN(ecc->bytes, 2);
1870
1871         nsectors = mtd->writesize / ecc->size;
1872
1873         if (mtd->oobsize < ((ecc->bytes + 4) * nsectors)) {
1874                 ret = -EINVAL;
1875                 goto err;
1876         }
1877
1878         ecc->read_oob = sunxi_nfc_hw_common_ecc_read_oob;
1879         ecc->write_oob = sunxi_nfc_hw_common_ecc_write_oob;
1880         mtd_set_ooblayout(mtd, &sunxi_nand_ooblayout_ops);
1881         ecc->priv = data;
1882
1883         return 0;
1884
1885 err:
1886         kfree(data);
1887
1888         return ret;
1889 }
1890
1891 static void sunxi_nand_hw_common_ecc_ctrl_cleanup(struct nand_ecc_ctrl *ecc)
1892 {
1893         kfree(ecc->priv);
1894 }
1895
1896 static int sunxi_nand_hw_ecc_ctrl_init(struct mtd_info *mtd,
1897                                        struct nand_ecc_ctrl *ecc,
1898                                        struct device_node *np)
1899 {
1900         struct nand_chip *nand = mtd_to_nand(mtd);
1901         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
1902         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
1903         int ret;
1904
1905         ret = sunxi_nand_hw_common_ecc_ctrl_init(mtd, ecc, np);
1906         if (ret)
1907                 return ret;
1908
1909         if (nfc->dmac) {
1910                 ecc->read_page = sunxi_nfc_hw_ecc_read_page_dma;
1911                 ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage_dma;
1912                 ecc->write_page = sunxi_nfc_hw_ecc_write_page_dma;
1913                 nand->options |= NAND_USE_BOUNCE_BUFFER;
1914         } else {
1915                 ecc->read_page = sunxi_nfc_hw_ecc_read_page;
1916                 ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage;
1917                 ecc->write_page = sunxi_nfc_hw_ecc_write_page;
1918         }
1919
1920         /* TODO: support DMA for raw accesses and subpage write */
1921         ecc->write_subpage = sunxi_nfc_hw_ecc_write_subpage;
1922         ecc->read_oob_raw = nand_read_oob_std;
1923         ecc->write_oob_raw = nand_write_oob_std;
1924         ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage;
1925
1926         return 0;
1927 }
1928
1929 static int sunxi_nand_hw_syndrome_ecc_ctrl_init(struct mtd_info *mtd,
1930                                                 struct nand_ecc_ctrl *ecc,
1931                                                 struct device_node *np)
1932 {
1933         int ret;
1934
1935         ret = sunxi_nand_hw_common_ecc_ctrl_init(mtd, ecc, np);
1936         if (ret)
1937                 return ret;
1938
1939         ecc->prepad = 4;
1940         ecc->read_page = sunxi_nfc_hw_syndrome_ecc_read_page;
1941         ecc->write_page = sunxi_nfc_hw_syndrome_ecc_write_page;
1942         ecc->read_oob_raw = nand_read_oob_syndrome;
1943         ecc->write_oob_raw = nand_write_oob_syndrome;
1944
1945         return 0;
1946 }
1947
1948 static void sunxi_nand_ecc_cleanup(struct nand_ecc_ctrl *ecc)
1949 {
1950         switch (ecc->mode) {
1951         case NAND_ECC_HW:
1952         case NAND_ECC_HW_SYNDROME:
1953                 sunxi_nand_hw_common_ecc_ctrl_cleanup(ecc);
1954                 break;
1955         case NAND_ECC_NONE:
1956         default:
1957                 break;
1958         }
1959 }
1960
1961 static int sunxi_nand_ecc_init(struct mtd_info *mtd, struct nand_ecc_ctrl *ecc,
1962                                struct device_node *np)
1963 {
1964         struct nand_chip *nand = mtd_to_nand(mtd);
1965         int ret;
1966
1967         if (!ecc->size) {
1968                 ecc->size = nand->ecc_step_ds;
1969                 ecc->strength = nand->ecc_strength_ds;
1970         }
1971
1972         if (!ecc->size || !ecc->strength)
1973                 return -EINVAL;
1974
1975         switch (ecc->mode) {
1976         case NAND_ECC_HW:
1977                 ret = sunxi_nand_hw_ecc_ctrl_init(mtd, ecc, np);
1978                 if (ret)
1979                         return ret;
1980                 break;
1981         case NAND_ECC_HW_SYNDROME:
1982                 ret = sunxi_nand_hw_syndrome_ecc_ctrl_init(mtd, ecc, np);
1983                 if (ret)
1984                         return ret;
1985                 break;
1986         case NAND_ECC_NONE:
1987         case NAND_ECC_SOFT:
1988                 break;
1989         default:
1990                 return -EINVAL;
1991         }
1992
1993         return 0;
1994 }
1995
1996 static int sunxi_nand_chip_init(struct device *dev, struct sunxi_nfc *nfc,
1997                                 struct device_node *np)
1998 {
1999         struct sunxi_nand_chip *chip;
2000         struct mtd_info *mtd;
2001         struct nand_chip *nand;
2002         int nsels;
2003         int ret;
2004         int i;
2005         u32 tmp;
2006
2007         if (!of_get_property(np, "reg", &nsels))
2008                 return -EINVAL;
2009
2010         nsels /= sizeof(u32);
2011         if (!nsels) {
2012                 dev_err(dev, "invalid reg property size\n");
2013                 return -EINVAL;
2014         }
2015
2016         chip = devm_kzalloc(dev,
2017                             sizeof(*chip) +
2018                             (nsels * sizeof(struct sunxi_nand_chip_sel)),
2019                             GFP_KERNEL);
2020         if (!chip) {
2021                 dev_err(dev, "could not allocate chip\n");
2022                 return -ENOMEM;
2023         }
2024
2025         chip->nsels = nsels;
2026         chip->selected = -1;
2027
2028         for (i = 0; i < nsels; i++) {
2029                 ret = of_property_read_u32_index(np, "reg", i, &tmp);
2030                 if (ret) {
2031                         dev_err(dev, "could not retrieve reg property: %d\n",
2032                                 ret);
2033                         return ret;
2034                 }
2035
2036                 if (tmp > NFC_MAX_CS) {
2037                         dev_err(dev,
2038                                 "invalid reg value: %u (max CS = 7)\n",
2039                                 tmp);
2040                         return -EINVAL;
2041                 }
2042
2043                 if (test_and_set_bit(tmp, &nfc->assigned_cs)) {
2044                         dev_err(dev, "CS %d already assigned\n", tmp);
2045                         return -EINVAL;
2046                 }
2047
2048                 chip->sels[i].cs = tmp;
2049
2050                 if (!of_property_read_u32_index(np, "allwinner,rb", i, &tmp) &&
2051                     tmp < 2) {
2052                         chip->sels[i].rb.type = RB_NATIVE;
2053                         chip->sels[i].rb.info.nativeid = tmp;
2054                 } else {
2055                         ret = of_get_named_gpio(np, "rb-gpios", i);
2056                         if (ret >= 0) {
2057                                 tmp = ret;
2058                                 chip->sels[i].rb.type = RB_GPIO;
2059                                 chip->sels[i].rb.info.gpio = tmp;
2060                                 ret = devm_gpio_request(dev, tmp, "nand-rb");
2061                                 if (ret)
2062                                         return ret;
2063
2064                                 ret = gpio_direction_input(tmp);
2065                                 if (ret)
2066                                         return ret;
2067                         } else {
2068                                 chip->sels[i].rb.type = RB_NONE;
2069                         }
2070                 }
2071         }
2072
2073         nand = &chip->nand;
2074         /* Default tR value specified in the ONFI spec (chapter 4.15.1) */
2075         nand->chip_delay = 200;
2076         nand->controller = &nfc->controller;
2077         /*
2078          * Set the ECC mode to the default value in case nothing is specified
2079          * in the DT.
2080          */
2081         nand->ecc.mode = NAND_ECC_HW;
2082         nand_set_flash_node(nand, np);
2083         nand->select_chip = sunxi_nfc_select_chip;
2084         nand->cmd_ctrl = sunxi_nfc_cmd_ctrl;
2085         nand->read_buf = sunxi_nfc_read_buf;
2086         nand->write_buf = sunxi_nfc_write_buf;
2087         nand->read_byte = sunxi_nfc_read_byte;
2088         nand->setup_data_interface = sunxi_nfc_setup_data_interface;
2089
2090         mtd = nand_to_mtd(nand);
2091         mtd->dev.parent = dev;
2092
2093         ret = nand_scan_ident(mtd, nsels, NULL);
2094         if (ret)
2095                 return ret;
2096
2097         if (nand->bbt_options & NAND_BBT_USE_FLASH)
2098                 nand->bbt_options |= NAND_BBT_NO_OOB;
2099
2100         if (nand->options & NAND_NEED_SCRAMBLING)
2101                 nand->options |= NAND_NO_SUBPAGE_WRITE;
2102
2103         nand->options |= NAND_SUBPAGE_READ;
2104
2105         ret = sunxi_nand_ecc_init(mtd, &nand->ecc, np);
2106         if (ret) {
2107                 dev_err(dev, "ECC init failed: %d\n", ret);
2108                 return ret;
2109         }
2110
2111         ret = nand_scan_tail(mtd);
2112         if (ret) {
2113                 dev_err(dev, "nand_scan_tail failed: %d\n", ret);
2114                 return ret;
2115         }
2116
2117         ret = mtd_device_register(mtd, NULL, 0);
2118         if (ret) {
2119                 dev_err(dev, "failed to register mtd device: %d\n", ret);
2120                 nand_release(mtd);
2121                 return ret;
2122         }
2123
2124         list_add_tail(&chip->node, &nfc->chips);
2125
2126         return 0;
2127 }
2128
2129 static int sunxi_nand_chips_init(struct device *dev, struct sunxi_nfc *nfc)
2130 {
2131         struct device_node *np = dev->of_node;
2132         struct device_node *nand_np;
2133         int nchips = of_get_child_count(np);
2134         int ret;
2135
2136         if (nchips > 8) {
2137                 dev_err(dev, "too many NAND chips: %d (max = 8)\n", nchips);
2138                 return -EINVAL;
2139         }
2140
2141         for_each_child_of_node(np, nand_np) {
2142                 ret = sunxi_nand_chip_init(dev, nfc, nand_np);
2143                 if (ret) {
2144                         of_node_put(nand_np);
2145                         return ret;
2146                 }
2147         }
2148
2149         return 0;
2150 }
2151
2152 static void sunxi_nand_chips_cleanup(struct sunxi_nfc *nfc)
2153 {
2154         struct sunxi_nand_chip *chip;
2155
2156         while (!list_empty(&nfc->chips)) {
2157                 chip = list_first_entry(&nfc->chips, struct sunxi_nand_chip,
2158                                         node);
2159                 nand_release(nand_to_mtd(&chip->nand));
2160                 sunxi_nand_ecc_cleanup(&chip->nand.ecc);
2161                 list_del(&chip->node);
2162         }
2163 }
2164
2165 static int sunxi_nfc_probe(struct platform_device *pdev)
2166 {
2167         struct device *dev = &pdev->dev;
2168         struct resource *r;
2169         struct sunxi_nfc *nfc;
2170         int irq;
2171         int ret;
2172
2173         nfc = devm_kzalloc(dev, sizeof(*nfc), GFP_KERNEL);
2174         if (!nfc)
2175                 return -ENOMEM;
2176
2177         nfc->dev = dev;
2178         nand_hw_control_init(&nfc->controller);
2179         INIT_LIST_HEAD(&nfc->chips);
2180
2181         r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2182         nfc->regs = devm_ioremap_resource(dev, r);
2183         if (IS_ERR(nfc->regs))
2184                 return PTR_ERR(nfc->regs);
2185
2186         irq = platform_get_irq(pdev, 0);
2187         if (irq < 0) {
2188                 dev_err(dev, "failed to retrieve irq\n");
2189                 return irq;
2190         }
2191
2192         nfc->ahb_clk = devm_clk_get(dev, "ahb");
2193         if (IS_ERR(nfc->ahb_clk)) {
2194                 dev_err(dev, "failed to retrieve ahb clk\n");
2195                 return PTR_ERR(nfc->ahb_clk);
2196         }
2197
2198         ret = clk_prepare_enable(nfc->ahb_clk);
2199         if (ret)
2200                 return ret;
2201
2202         nfc->mod_clk = devm_clk_get(dev, "mod");
2203         if (IS_ERR(nfc->mod_clk)) {
2204                 dev_err(dev, "failed to retrieve mod clk\n");
2205                 ret = PTR_ERR(nfc->mod_clk);
2206                 goto out_ahb_clk_unprepare;
2207         }
2208
2209         ret = clk_prepare_enable(nfc->mod_clk);
2210         if (ret)
2211                 goto out_ahb_clk_unprepare;
2212
2213         nfc->reset = devm_reset_control_get_optional(dev, "ahb");
2214         if (IS_ERR(nfc->reset)) {
2215                 ret = PTR_ERR(nfc->reset);
2216                 goto out_mod_clk_unprepare;
2217         }
2218
2219         ret = reset_control_deassert(nfc->reset);
2220         if (ret) {
2221                 dev_err(dev, "reset err %d\n", ret);
2222                 goto out_mod_clk_unprepare;
2223         }
2224
2225         ret = sunxi_nfc_rst(nfc);
2226         if (ret)
2227                 goto out_ahb_reset_reassert;
2228
2229         writel(0, nfc->regs + NFC_REG_INT);
2230         ret = devm_request_irq(dev, irq, sunxi_nfc_interrupt,
2231                                0, "sunxi-nand", nfc);
2232         if (ret)
2233                 goto out_ahb_reset_reassert;
2234
2235         nfc->dmac = dma_request_slave_channel(dev, "rxtx");
2236         if (nfc->dmac) {
2237                 struct dma_slave_config dmac_cfg = { };
2238
2239                 dmac_cfg.src_addr = r->start + NFC_REG_IO_DATA;
2240                 dmac_cfg.dst_addr = dmac_cfg.src_addr;
2241                 dmac_cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
2242                 dmac_cfg.dst_addr_width = dmac_cfg.src_addr_width;
2243                 dmac_cfg.src_maxburst = 4;
2244                 dmac_cfg.dst_maxburst = 4;
2245                 dmaengine_slave_config(nfc->dmac, &dmac_cfg);
2246         } else {
2247                 dev_warn(dev, "failed to request rxtx DMA channel\n");
2248         }
2249
2250         platform_set_drvdata(pdev, nfc);
2251
2252         ret = sunxi_nand_chips_init(dev, nfc);
2253         if (ret) {
2254                 dev_err(dev, "failed to init nand chips\n");
2255                 goto out_release_dmac;
2256         }
2257
2258         return 0;
2259
2260 out_release_dmac:
2261         if (nfc->dmac)
2262                 dma_release_channel(nfc->dmac);
2263 out_ahb_reset_reassert:
2264         reset_control_assert(nfc->reset);
2265 out_mod_clk_unprepare:
2266         clk_disable_unprepare(nfc->mod_clk);
2267 out_ahb_clk_unprepare:
2268         clk_disable_unprepare(nfc->ahb_clk);
2269
2270         return ret;
2271 }
2272
2273 static int sunxi_nfc_remove(struct platform_device *pdev)
2274 {
2275         struct sunxi_nfc *nfc = platform_get_drvdata(pdev);
2276
2277         sunxi_nand_chips_cleanup(nfc);
2278
2279         reset_control_assert(nfc->reset);
2280
2281         if (nfc->dmac)
2282                 dma_release_channel(nfc->dmac);
2283         clk_disable_unprepare(nfc->mod_clk);
2284         clk_disable_unprepare(nfc->ahb_clk);
2285
2286         return 0;
2287 }
2288
2289 static const struct of_device_id sunxi_nfc_ids[] = {
2290         { .compatible = "allwinner,sun4i-a10-nand" },
2291         { /* sentinel */ }
2292 };
2293 MODULE_DEVICE_TABLE(of, sunxi_nfc_ids);
2294
2295 static struct platform_driver sunxi_nfc_driver = {
2296         .driver = {
2297                 .name = "sunxi_nand",
2298                 .of_match_table = sunxi_nfc_ids,
2299         },
2300         .probe = sunxi_nfc_probe,
2301         .remove = sunxi_nfc_remove,
2302 };
2303 module_platform_driver(sunxi_nfc_driver);
2304
2305 MODULE_LICENSE("GPL v2");
2306 MODULE_AUTHOR("Boris BREZILLON");
2307 MODULE_DESCRIPTION("Allwinner NAND Flash Controller driver");
2308 MODULE_ALIAS("platform:sunxi_nand");