drivers/mtd/nand/atmel_nand.c

   1 /*
   2  * (C) Copyright 2007-2008
   3  * Stelian Pop <stelian@popies.net>
   4  * Lead Tech Design <www.leadtechdesign.com>
   5  *
   6  * (C) Copyright 2006 ATMEL Rousset, Lacressonniere Nicolas
   7  *
   8  * Add Programmable Multibit ECC support for various AT91 SoC
   9  *     (C) Copyright 2012 ATMEL, Hong Xu
  10  *
  11  * See file CREDITS for list of people who contributed to this
  12  * project.
  13  *
  14  * This program is free software; you can redistribute it and/or
  15  * modify it under the terms of the GNU General Public License as
  16  * published by the Free Software Foundation; either version 2 of
  17  * the License, or (at your option) any later version.
  18  *
  19  * This program is distributed in the hope that it will be useful,
  20  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  21  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  22  * GNU General Public License for more details.
  23  *
  24  * You should have received a copy of the GNU General Public License
  25  * along with this program; if not, write to the Free Software
  26  * Foundation, Inc., 59 Temple Place, Suite 330, Boston,
  27  * MA 02111-1307 USA
  28  */
  29
  30 #include <common.h>
  31 #include <asm/arch/hardware.h>
  32 #include <asm/arch/gpio.h>
  33 #include <asm/arch/at91_pio.h>
  34
  35 #include <nand.h>
  36 #include <watchdog.h>
  37
  38 #ifdef CONFIG_ATMEL_NAND_HWECC
  39
  40 /* Register access macros */
  41 #define ecc_readl(add, reg)                             \
  42         readl(AT91_BASE_SYS + add + ATMEL_ECC_##reg)
  43 #define ecc_writel(add, reg, value)                     \
  44         writel((value), AT91_BASE_SYS + add + ATMEL_ECC_##reg)
  45
  46 #include "atmel_nand_ecc.h"     /* Hardware ECC registers */
  47
  48 #ifdef CONFIG_ATMEL_NAND_HW_PMECC
  49
  50 struct atmel_nand_host {
  51         struct pmecc_regs __iomem *pmecc;
  52         struct pmecc_errloc_regs __iomem *pmerrloc;
  53         void __iomem            *pmecc_rom_base;
  54
  55         u8              pmecc_corr_cap;
  56         u16             pmecc_sector_size;
  57         u32             pmecc_index_table_offset;
  58
  59         int             pmecc_bytes_per_sector;
  60         int             pmecc_sector_number;
  61         int             pmecc_degree;   /* Degree of remainders */
  62         int             pmecc_cw_len;   /* Length of codeword */
  63
  64         /* lookup table for alpha_to and index_of */
  65         void __iomem    *pmecc_alpha_to;
  66         void __iomem    *pmecc_index_of;
  67
  68         /* data for pmecc computation */
  69         int16_t pmecc_smu[(CONFIG_PMECC_CAP + 2) * (2 * CONFIG_PMECC_CAP + 1)];
  70         int16_t pmecc_partial_syn[2 * CONFIG_PMECC_CAP + 1];
  71         int16_t pmecc_si[2 * CONFIG_PMECC_CAP + 1];
  72         int16_t pmecc_lmu[CONFIG_PMECC_CAP + 1]; /* polynomal order */
  73         int     pmecc_mu[CONFIG_PMECC_CAP + 1];
  74         int     pmecc_dmu[CONFIG_PMECC_CAP + 1];
  75         int     pmecc_delta[CONFIG_PMECC_CAP + 1];
  76 };
  77
  78 static struct atmel_nand_host pmecc_host;
  79 static struct nand_ecclayout atmel_pmecc_oobinfo;
  80
  81 /*
  82  * Return number of ecc bytes per sector according to sector size and
  83  * correction capability
  84  *
  85  * Following table shows what at91 PMECC supported:
  86  * Correction Capability        Sector_512_bytes        Sector_1024_bytes
  87  * =====================        ================        =================
  88  *                2-bits                 4-bytes                  4-bytes
  89  *                4-bits                 7-bytes                  7-bytes
  90  *                8-bits                13-bytes                 14-bytes
  91  *               12-bits                20-bytes                 21-bytes
  92  *               24-bits                39-bytes                 42-bytes
  93  */
  94 static int pmecc_get_ecc_bytes(int cap, int sector_size)
  95 {
  96         int m = 12 + sector_size / 512;
  97         return (m * cap + 7) / 8;
  98 }
  99
 100 static void pmecc_config_ecc_layout(struct nand_ecclayout *layout,
 101         int oobsize, int ecc_len)
 102 {
 103         int i;
 104
 105         layout->eccbytes = ecc_len;
 106
 107         /* ECC will occupy the last ecc_len bytes continuously */
 108         for (i = 0; i < ecc_len; i++)
 109                 layout->eccpos[i] = oobsize - ecc_len + i;
 110
 111         layout->oobfree[0].offset = 2;
 112         layout->oobfree[0].length =
 113                 oobsize - ecc_len - layout->oobfree[0].offset;
 114 }
 115
 116 static void __iomem *pmecc_get_alpha_to(struct atmel_nand_host *host)
 117 {
 118         int table_size;
 119
 120         table_size = host->pmecc_sector_size == 512 ?
 121                 PMECC_INDEX_TABLE_SIZE_512 : PMECC_INDEX_TABLE_SIZE_1024;
 122
 123         /* the ALPHA lookup table is right behind the INDEX lookup table. */
 124         return host->pmecc_rom_base + host->pmecc_index_table_offset +
 125                         table_size * sizeof(int16_t);
 126 }
 127
 128 static void pmecc_gen_syndrome(struct mtd_info *mtd, int sector)
 129 {
 130         struct nand_chip *nand_chip = mtd->priv;
 131         struct atmel_nand_host *host = nand_chip->priv;
 132         int i;
 133         uint32_t value;
 134
 135         /* Fill odd syndromes */
 136         for (i = 0; i < host->pmecc_corr_cap; i++) {
 137                 value = readl(&host->pmecc->rem_port[sector].rem[i / 2]);
 138                 if (i & 1)
 139                         value >>= 16;
 140                 value &= 0xffff;
 141                 host->pmecc_partial_syn[(2 * i) + 1] = (int16_t)value;
 142         }
 143 }
 144
 145 static void pmecc_substitute(struct mtd_info *mtd)
 146 {
 147         struct nand_chip *nand_chip = mtd->priv;
 148         struct atmel_nand_host *host = nand_chip->priv;
 149         int16_t __iomem *alpha_to = host->pmecc_alpha_to;
 150         int16_t __iomem *index_of = host->pmecc_index_of;
 151         int16_t *partial_syn = host->pmecc_partial_syn;
 152         const int cap = host->pmecc_corr_cap;
 153         int16_t *si;
 154         int i, j;
 155
 156         /* si[] is a table that holds the current syndrome value,
 157          * an element of that table belongs to the field
 158          */
 159         si = host->pmecc_si;
 160
 161         memset(&si[1], 0, sizeof(int16_t) * (2 * cap - 1));
 162
 163         /* Computation 2t syndromes based on S(x) */
 164         /* Odd syndromes */
 165         for (i = 1; i < 2 * cap; i += 2) {
 166                 for (j = 0; j < host->pmecc_degree; j++) {
 167                         if (partial_syn[i] & (0x1 << j))
 168                                 si[i] = readw(alpha_to + i * j) ^ si[i];
 169                 }
 170         }
 171         /* Even syndrome = (Odd syndrome) ** 2 */
 172         for (i = 2, j = 1; j <= cap; i = ++j << 1) {
 173                 if (si[j] == 0) {
 174                         si[i] = 0;
 175                 } else {
 176                         int16_t tmp;
 177
 178                         tmp = readw(index_of + si[j]);
 179                         tmp = (tmp * 2) % host->pmecc_cw_len;
 180                         si[i] = readw(alpha_to + tmp);
 181                 }
 182         }
 183 }
 184
 185 /*
 186  * This function defines a Berlekamp iterative procedure for
 187  * finding the value of the error location polynomial.
 188  * The input is si[], initialize by pmecc_substitute().
 189  * The output is smu[][].
 190  *
 191  * This function is written according to chip datasheet Chapter:
 192  * Find the Error Location Polynomial Sigma(x) of Section:
 193  * Programmable Multibit ECC Control (PMECC).
 194  */
 195 static void pmecc_get_sigma(struct mtd_info *mtd)
 196 {
 197         struct nand_chip *nand_chip = mtd->priv;
 198         struct atmel_nand_host *host = nand_chip->priv;
 199
 200         int16_t *lmu = host->pmecc_lmu;
 201         int16_t *si = host->pmecc_si;
 202         int *mu = host->pmecc_mu;
 203         int *dmu = host->pmecc_dmu;     /* Discrepancy */
 204         int *delta = host->pmecc_delta; /* Delta order */
 205         int cw_len = host->pmecc_cw_len;
 206         const int16_t cap = host->pmecc_corr_cap;
 207         const int num = 2 * cap + 1;
 208         int16_t __iomem *index_of = host->pmecc_index_of;
 209         int16_t __iomem *alpha_to = host->pmecc_alpha_to;
 210         int i, j, k;
 211         uint32_t dmu_0_count, tmp;
 212         int16_t *smu = host->pmecc_smu;
 213
 214         /* index of largest delta */
 215         int ro;
 216         int largest;
 217         int diff;
 218
 219         /* Init the Sigma(x) */
 220         memset(smu, 0, sizeof(int16_t) * ARRAY_SIZE(smu));
 221
 222         dmu_0_count = 0;
 223
 224         /* First Row */
 225
 226         /* Mu */
 227         mu[0] = -1;
 228
 229         smu[0] = 1;
 230
 231         /* discrepancy set to 1 */
 232         dmu[0] = 1;
 233         /* polynom order set to 0 */
 234         lmu[0] = 0;
 235         /* delta[0] = (mu[0] * 2 - lmu[0]) >> 1; */
 236         delta[0] = -1;
 237
 238         /* Second Row */
 239
 240         /* Mu */
 241         mu[1] = 0;
 242         /* Sigma(x) set to 1 */
 243         smu[num] = 1;
 244
 245         /* discrepancy set to S1 */
 246         dmu[1] = si[1];
 247
 248         /* polynom order set to 0 */
 249         lmu[1] = 0;
 250
 251         /* delta[1] = (mu[1] * 2 - lmu[1]) >> 1; */
 252         delta[1] = 0;
 253
 254         for (i = 1; i <= cap; i++) {
 255                 mu[i + 1] = i << 1;
 256                 /* Begin Computing Sigma (Mu+1) and L(mu) */
 257                 /* check if discrepancy is set to 0 */
 258                 if (dmu[i] == 0) {
 259                         dmu_0_count++;
 260
 261                         tmp = ((cap - (lmu[i] >> 1) - 1) / 2);
 262                         if ((cap - (lmu[i] >> 1) - 1) & 0x1)
 263                                 tmp += 2;
 264                         else
 265                                 tmp += 1;
 266
 267                         if (dmu_0_count == tmp) {
 268                                 for (j = 0; j <= (lmu[i] >> 1) + 1; j++)
 269                                         smu[(cap + 1) * num + j] =
 270                                                         smu[i * num + j];
 271
 272                                 lmu[cap + 1] = lmu[i];
 273                                 return;
 274                         }
 275
 276                         /* copy polynom */
 277                         for (j = 0; j <= lmu[i] >> 1; j++)
 278                                 smu[(i + 1) * num + j] = smu[i * num + j];
 279
 280                         /* copy previous polynom order to the next */
 281                         lmu[i + 1] = lmu[i];
 282                 } else {
 283                         ro = 0;
 284                         largest = -1;
 285                         /* find largest delta with dmu != 0 */
 286                         for (j = 0; j < i; j++) {
 287                                 if ((dmu[j]) && (delta[j] > largest)) {
 288                                         largest = delta[j];
 289                                         ro = j;
 290                                 }
 291                         }
 292
 293                         /* compute difference */
 294                         diff = (mu[i] - mu[ro]);
 295
 296                         /* Compute degree of the new smu polynomial */
 297                         if ((lmu[i] >> 1) > ((lmu[ro] >> 1) + diff))
 298                                 lmu[i + 1] = lmu[i];
 299                         else
 300                                 lmu[i + 1] = ((lmu[ro] >> 1) + diff) * 2;
 301
 302                         /* Init smu[i+1] with 0 */
 303                         for (k = 0; k < num; k++)
 304                                 smu[(i + 1) * num + k] = 0;
 305
 306                         /* Compute smu[i+1] */
 307                         for (k = 0; k <= lmu[ro] >> 1; k++) {
 308                                 int16_t a, b, c;
 309
 310                                 if (!(smu[ro * num + k] && dmu[i]))
 311                                         continue;
 312                                 a = readw(index_of + dmu[i]);
 313                                 b = readw(index_of + dmu[ro]);
 314                                 c = readw(index_of + smu[ro * num + k]);
 315                                 tmp = a + (cw_len - b) + c;
 316                                 a = readw(alpha_to + tmp % cw_len);
 317                                 smu[(i + 1) * num + (k + diff)] = a;
 318                         }
 319
 320                         for (k = 0; k <= lmu[i] >> 1; k++)
 321                                 smu[(i + 1) * num + k] ^= smu[i * num + k];
 322                 }
 323
 324                 /* End Computing Sigma (Mu+1) and L(mu) */
 325                 /* In either case compute delta */
 326                 delta[i + 1] = (mu[i + 1] * 2 - lmu[i + 1]) >> 1;
 327
 328                 /* Do not compute discrepancy for the last iteration */
 329                 if (i >= cap)
 330                         continue;
 331
 332                 for (k = 0; k <= (lmu[i + 1] >> 1); k++) {
 333                         tmp = 2 * (i - 1);
 334                         if (k == 0) {
 335                                 dmu[i + 1] = si[tmp + 3];
 336                         } else if (smu[(i + 1) * num + k] && si[tmp + 3 - k]) {
 337                                 int16_t a, b, c;
 338                                 a = readw(index_of +
 339                                                 smu[(i + 1) * num + k]);
 340                                 b = si[2 * (i - 1) + 3 - k];
 341                                 c = readw(index_of + b);
 342                                 tmp = a + c;
 343                                 tmp %= cw_len;
 344                                 dmu[i + 1] = readw(alpha_to + tmp) ^
 345                                         dmu[i + 1];
 346                         }
 347                 }
 348         }
 349 }
 350
 351 static int pmecc_err_location(struct mtd_info *mtd)
 352 {
 353         struct nand_chip *nand_chip = mtd->priv;
 354         struct atmel_nand_host *host = nand_chip->priv;
 355         const int cap = host->pmecc_corr_cap;
 356         const int num = 2 * cap + 1;
 357         int sector_size = host->pmecc_sector_size;
 358         int err_nbr = 0;        /* number of error */
 359         int roots_nbr;          /* number of roots */
 360         int i;
 361         uint32_t val;
 362         int16_t *smu = host->pmecc_smu;
 363         int timeout = PMECC_MAX_TIMEOUT_US;
 364
 365         writel(PMERRLOC_DISABLE, &host->pmerrloc->eldis);
 366
 367         for (i = 0; i <= host->pmecc_lmu[cap + 1] >> 1; i++) {
 368                 writel(smu[(cap + 1) * num + i], &host->pmerrloc->sigma[i]);
 369                 err_nbr++;
 370         }
 371
 372         val = PMERRLOC_ELCFG_NUM_ERRORS(err_nbr - 1);
 373         if (sector_size == 1024)
 374                 val |= PMERRLOC_ELCFG_SECTOR_1024;
 375
 376         writel(val, &host->pmerrloc->elcfg);
 377         writel(sector_size * 8 + host->pmecc_degree * cap,
 378                         &host->pmerrloc->elen);
 379
 380         while (--timeout) {
 381                 if (readl(&host->pmerrloc->elisr) & PMERRLOC_CALC_DONE)
 382                         break;
 383                 WATCHDOG_RESET();
 384                 udelay(1);
 385         }
 386
 387         if (!timeout) {
 388                 printk(KERN_ERR "atmel_nand : Timeout to calculate PMECC error location\n");
 389                 return -1;
 390         }
 391
 392         roots_nbr = (readl(&host->pmerrloc->elisr) & PMERRLOC_ERR_NUM_MASK)
 393                         >> 8;
 394         /* Number of roots == degree of smu hence <= cap */
 395         if (roots_nbr == host->pmecc_lmu[cap + 1] >> 1)
 396                 return err_nbr - 1;
 397
 398         /* Number of roots does not match the degree of smu
 399          * unable to correct error */
 400         return -1;
 401 }
 402
 403 static void pmecc_correct_data(struct mtd_info *mtd, uint8_t *buf, uint8_t *ecc,
 404                 int sector_num, int extra_bytes, int err_nbr)
 405 {
 406         struct nand_chip *nand_chip = mtd->priv;
 407         struct atmel_nand_host *host = nand_chip->priv;
 408         int i = 0;
 409         int byte_pos, bit_pos, sector_size, pos;
 410         uint32_t tmp;
 411         uint8_t err_byte;
 412
 413         sector_size = host->pmecc_sector_size;
 414
 415         while (err_nbr) {
 416                 tmp = readl(&host->pmerrloc->el[i]) - 1;
 417                 byte_pos = tmp / 8;
 418                 bit_pos  = tmp % 8;
 419
 420                 if (byte_pos >= (sector_size + extra_bytes))
 421                         BUG();  /* should never happen */
 422
 423                 if (byte_pos < sector_size) {
 424                         err_byte = *(buf + byte_pos);
 425                         *(buf + byte_pos) ^= (1 << bit_pos);
 426
 427                         pos = sector_num * host->pmecc_sector_size + byte_pos;
 428                         printk(KERN_INFO "Bit flip in data area, byte_pos: %d, bit_pos: %d, 0x%02x -> 0x%02x\n",
 429                                 pos, bit_pos, err_byte, *(buf + byte_pos));
 430                 } else {
 431                         /* Bit flip in OOB area */
 432                         tmp = sector_num * host->pmecc_bytes_per_sector
 433                                         + (byte_pos - sector_size);
 434                         err_byte = ecc[tmp];
 435                         ecc[tmp] ^= (1 << bit_pos);
 436
 437                         pos = tmp + nand_chip->ecc.layout->eccpos[0];
 438                         printk(KERN_INFO "Bit flip in OOB, oob_byte_pos: %d, bit_pos: %d, 0x%02x -> 0x%02x\n",
 439                                 pos, bit_pos, err_byte, ecc[tmp]);
 440                 }
 441
 442                 i++;
 443                 err_nbr--;
 444         }
 445
 446         return;
 447 }
 448
 449 static int pmecc_correction(struct mtd_info *mtd, u32 pmecc_stat, uint8_t *buf,
 450         u8 *ecc)
 451 {
 452         struct nand_chip *nand_chip = mtd->priv;
 453         struct atmel_nand_host *host = nand_chip->priv;
 454         int i, err_nbr, eccbytes;
 455         uint8_t *buf_pos;
 456
 457         eccbytes = nand_chip->ecc.bytes;
 458         for (i = 0; i < eccbytes; i++)
 459                 if (ecc[i] != 0xff)
 460                         goto normal_check;
 461         /* Erased page, return OK */
 462         return 0;
 463
 464 normal_check:
 465         for (i = 0; i < host->pmecc_sector_number; i++) {
 466                 err_nbr = 0;
 467                 if (pmecc_stat & 0x1) {
 468                         buf_pos = buf + i * host->pmecc_sector_size;
 469
 470                         pmecc_gen_syndrome(mtd, i);
 471                         pmecc_substitute(mtd);
 472                         pmecc_get_sigma(mtd);
 473
 474                         err_nbr = pmecc_err_location(mtd);
 475                         if (err_nbr == -1) {
 476                                 printk(KERN_ERR "PMECC: Too many errors\n");
 477                                 mtd->ecc_stats.failed++;
 478                                 return -EIO;
 479                         } else {
 480                                 pmecc_correct_data(mtd, buf_pos, ecc, i,
 481                                         host->pmecc_bytes_per_sector, err_nbr);
 482                                 mtd->ecc_stats.corrected += err_nbr;
 483                         }
 484                 }
 485                 pmecc_stat >>= 1;
 486         }
 487
 488         return 0;
 489 }
 490
 491 static int atmel_nand_pmecc_read_page(struct mtd_info *mtd,
 492         struct nand_chip *chip, uint8_t *buf, int oob_required, int page)
 493 {
 494         struct atmel_nand_host *host = chip->priv;
 495         int eccsize = chip->ecc.size;
 496         uint8_t *oob = chip->oob_poi;
 497         uint32_t *eccpos = chip->ecc.layout->eccpos;
 498         uint32_t stat;
 499         int timeout = PMECC_MAX_TIMEOUT_US;
 500
 501         pmecc_writel(host->pmecc, ctrl, PMECC_CTRL_RST);
 502         pmecc_writel(host->pmecc, ctrl, PMECC_CTRL_DISABLE);
 503         pmecc_writel(host->pmecc, cfg, ((pmecc_readl(host->pmecc, cfg))
 504                 & ~PMECC_CFG_WRITE_OP) | PMECC_CFG_AUTO_ENABLE);
 505
 506         pmecc_writel(host->pmecc, ctrl, PMECC_CTRL_ENABLE);
 507         pmecc_writel(host->pmecc, ctrl, PMECC_CTRL_DATA);
 508
 509         chip->read_buf(mtd, buf, eccsize);
 510         chip->read_buf(mtd, oob, mtd->oobsize);
 511
 512         while (--timeout) {
 513                 if (!(pmecc_readl(host->pmecc, sr) & PMECC_SR_BUSY))
 514                         break;
 515                 WATCHDOG_RESET();
 516                 udelay(1);
 517         }
 518
 519         if (!timeout) {
 520                 printk(KERN_ERR "atmel_nand : Timeout to read PMECC page\n");
 521                 return -1;
 522         }
 523
 524         stat = pmecc_readl(host->pmecc, isr);
 525         if (stat != 0)
 526                 if (pmecc_correction(mtd, stat, buf, &oob[eccpos[0]]) != 0)
 527                         return -EIO;
 528
 529         return 0;
 530 }
 531
 532 static int atmel_nand_pmecc_write_page(struct mtd_info *mtd,
 533                 struct nand_chip *chip, const uint8_t *buf,
 534                 int oob_required)
 535 {
 536         struct atmel_nand_host *host = chip->priv;
 537         uint32_t *eccpos = chip->ecc.layout->eccpos;
 538         int i, j;
 539         int timeout = PMECC_MAX_TIMEOUT_US;
 540
 541         pmecc_writel(host->pmecc, ctrl, PMECC_CTRL_RST);
 542         pmecc_writel(host->pmecc, ctrl, PMECC_CTRL_DISABLE);
 543
 544         pmecc_writel(host->pmecc, cfg, (pmecc_readl(host->pmecc, cfg) |
 545                 PMECC_CFG_WRITE_OP) & ~PMECC_CFG_AUTO_ENABLE);
 546
 547         pmecc_writel(host->pmecc, ctrl, PMECC_CTRL_ENABLE);
 548         pmecc_writel(host->pmecc, ctrl, PMECC_CTRL_DATA);
 549
 550         chip->write_buf(mtd, (u8 *)buf, mtd->writesize);
 551
 552         while (--timeout) {
 553                 if (!(pmecc_readl(host->pmecc, sr) & PMECC_SR_BUSY))
 554                         break;
 555                 WATCHDOG_RESET();
 556                 udelay(1);
 557         }
 558
 559         if (!timeout) {
 560                 printk(KERN_ERR "atmel_nand : Timeout to read PMECC status, fail to write PMECC in oob\n");
 561                 goto out;
 562         }
 563
 564         for (i = 0; i < host->pmecc_sector_number; i++) {
 565                 for (j = 0; j < host->pmecc_bytes_per_sector; j++) {
 566                         int pos;
 567
 568                         pos = i * host->pmecc_bytes_per_sector + j;
 569                         chip->oob_poi[eccpos[pos]] =
 570                                 readb(&host->pmecc->ecc_port[i].ecc[j]);
 571                 }
 572         }
 573         chip->write_buf(mtd, chip->oob_poi, mtd->oobsize);
 574 out:
 575         return 0;
 576 }
 577
 578 static void atmel_pmecc_core_init(struct mtd_info *mtd)
 579 {
 580         struct nand_chip *nand_chip = mtd->priv;
 581         struct atmel_nand_host *host = nand_chip->priv;
 582         uint32_t val = 0;
 583         struct nand_ecclayout *ecc_layout;
 584
 585         pmecc_writel(host->pmecc, ctrl, PMECC_CTRL_RST);
 586         pmecc_writel(host->pmecc, ctrl, PMECC_CTRL_DISABLE);
 587
 588         switch (host->pmecc_corr_cap) {
 589         case 2:
 590                 val = PMECC_CFG_BCH_ERR2;
 591                 break;
 592         case 4:
 593                 val = PMECC_CFG_BCH_ERR4;
 594                 break;
 595         case 8:
 596                 val = PMECC_CFG_BCH_ERR8;
 597                 break;
 598         case 12:
 599                 val = PMECC_CFG_BCH_ERR12;
 600                 break;
 601         case 24:
 602                 val = PMECC_CFG_BCH_ERR24;
 603                 break;
 604         }
 605
 606         if (host->pmecc_sector_size == 512)
 607                 val |= PMECC_CFG_SECTOR512;
 608         else if (host->pmecc_sector_size == 1024)
 609                 val |= PMECC_CFG_SECTOR1024;
 610
 611         switch (host->pmecc_sector_number) {
 612         case 1:
 613                 val |= PMECC_CFG_PAGE_1SECTOR;
 614                 break;
 615         case 2:
 616                 val |= PMECC_CFG_PAGE_2SECTORS;
 617                 break;
 618         case 4:
 619                 val |= PMECC_CFG_PAGE_4SECTORS;
 620                 break;
 621         case 8:
 622                 val |= PMECC_CFG_PAGE_8SECTORS;
 623                 break;
 624         }
 625
 626         val |= (PMECC_CFG_READ_OP | PMECC_CFG_SPARE_DISABLE
 627                 | PMECC_CFG_AUTO_DISABLE);
 628         pmecc_writel(host->pmecc, cfg, val);
 629
 630         ecc_layout = nand_chip->ecc.layout;
 631         pmecc_writel(host->pmecc, sarea, mtd->oobsize - 1);
 632         pmecc_writel(host->pmecc, saddr, ecc_layout->eccpos[0]);
 633         pmecc_writel(host->pmecc, eaddr,
 634                         ecc_layout->eccpos[ecc_layout->eccbytes - 1]);
 635         /* See datasheet about PMECC Clock Control Register */
 636         pmecc_writel(host->pmecc, clk, PMECC_CLK_133MHZ);
 637         pmecc_writel(host->pmecc, idr, 0xff);
 638         pmecc_writel(host->pmecc, ctrl, PMECC_CTRL_ENABLE);
 639 }
 640
 641 static int atmel_pmecc_nand_init_params(struct nand_chip *nand,
 642                 struct mtd_info *mtd)
 643 {
 644         struct atmel_nand_host *host;
 645         int cap, sector_size;
 646
 647         host = nand->priv = &pmecc_host;
 648
 649         nand->ecc.mode = NAND_ECC_HW;
 650         nand->ecc.calculate = NULL;
 651         nand->ecc.correct = NULL;
 652         nand->ecc.hwctl = NULL;
 653
 654         cap = host->pmecc_corr_cap = CONFIG_PMECC_CAP;
 655         sector_size = host->pmecc_sector_size = CONFIG_PMECC_SECTOR_SIZE;
 656         host->pmecc_index_table_offset = CONFIG_PMECC_INDEX_TABLE_OFFSET;
 657
 658         MTDDEBUG(MTD_DEBUG_LEVEL1,
 659                 "Initialize PMECC params, cap: %d, sector: %d\n",
 660                 cap, sector_size);
 661
 662         host->pmecc = (struct pmecc_regs __iomem *) ATMEL_BASE_PMECC;
 663         host->pmerrloc = (struct pmecc_errloc_regs __iomem *)
 664                         ATMEL_BASE_PMERRLOC;
 665         host->pmecc_rom_base = (void __iomem *) ATMEL_BASE_ROM;
 666
 667         /* ECC is calculated for the whole page (1 step) */
 668         nand->ecc.size = mtd->writesize;
 669
 670         /* set ECC page size and oob layout */
 671         switch (mtd->writesize) {
 672         case 2048:
 673         case 4096:
 674                 host->pmecc_degree = PMECC_GF_DIMENSION_13;
 675                 host->pmecc_cw_len = (1 << host->pmecc_degree) - 1;
 676                 host->pmecc_sector_number = mtd->writesize / sector_size;
 677                 host->pmecc_bytes_per_sector = pmecc_get_ecc_bytes(
 678                         cap, sector_size);
 679                 host->pmecc_alpha_to = pmecc_get_alpha_to(host);
 680                 host->pmecc_index_of = host->pmecc_rom_base +
 681                         host->pmecc_index_table_offset;
 682
 683                 nand->ecc.steps = 1;
 684                 nand->ecc.bytes = host->pmecc_bytes_per_sector *
 685                                        host->pmecc_sector_number;
 686                 if (nand->ecc.bytes > mtd->oobsize - 2) {
 687                         printk(KERN_ERR "No room for ECC bytes\n");
 688                         return -EINVAL;
 689                 }
 690                 pmecc_config_ecc_layout(&atmel_pmecc_oobinfo,
 691                                         mtd->oobsize,
 692                                         nand->ecc.bytes);
 693                 nand->ecc.layout = &atmel_pmecc_oobinfo;
 694                 break;
 695         case 512:
 696         case 1024:
 697                 /* TODO */
 698                 printk(KERN_ERR "Unsupported page size for PMECC, use Software ECC\n");
 699         default:
 700                 /* page size not handled by HW ECC */
 701                 /* switching back to soft ECC */
 702                 nand->ecc.mode = NAND_ECC_SOFT;
 703                 nand->ecc.read_page = NULL;
 704                 nand->ecc.postpad = 0;
 705                 nand->ecc.prepad = 0;
 706                 nand->ecc.bytes = 0;
 707                 return 0;
 708         }
 709
 710         nand->ecc.read_page = atmel_nand_pmecc_read_page;
 711         nand->ecc.write_page = atmel_nand_pmecc_write_page;
 712         nand->ecc.strength = cap;
 713
 714         atmel_pmecc_core_init(mtd);
 715
 716         return 0;
 717 }
 718
 719 #else
 720
 721 /* oob layout for large page size
 722  * bad block info is on bytes 0 and 1
 723  * the bytes have to be consecutives to avoid
 724  * several NAND_CMD_RNDOUT during read
 725  */
 726 static struct nand_ecclayout atmel_oobinfo_large = {
 727         .eccbytes = 4,
 728         .eccpos = {60, 61, 62, 63},
 729         .oobfree = {
 730                 {2, 58}
 731         },
 732 };
 733
 734 /* oob layout for small page size
 735  * bad block info is on bytes 4 and 5
 736  * the bytes have to be consecutives to avoid
 737  * several NAND_CMD_RNDOUT during read
 738  */
 739 static struct nand_ecclayout atmel_oobinfo_small = {
 740         .eccbytes = 4,
 741         .eccpos = {0, 1, 2, 3},
 742         .oobfree = {
 743                 {6, 10}
 744         },
 745 };
 746
 747 /*
 748  * Calculate HW ECC
 749  *
 750  * function called after a write
 751  *
 752  * mtd:        MTD block structure
 753  * dat:        raw data (unused)
 754  * ecc_code:   buffer for ECC
 755  */
 756 static int atmel_nand_calculate(struct mtd_info *mtd,
 757                 const u_char *dat, unsigned char *ecc_code)
 758 {
 759         unsigned int ecc_value;
 760
 761         /* get the first 2 ECC bytes */
 762         ecc_value = ecc_readl(CONFIG_SYS_NAND_ECC_BASE, PR);
 763
 764         ecc_code[0] = ecc_value & 0xFF;
 765         ecc_code[1] = (ecc_value >> 8) & 0xFF;
 766
 767         /* get the last 2 ECC bytes */
 768         ecc_value = ecc_readl(CONFIG_SYS_NAND_ECC_BASE, NPR) & ATMEL_ECC_NPARITY;
 769
 770         ecc_code[2] = ecc_value & 0xFF;
 771         ecc_code[3] = (ecc_value >> 8) & 0xFF;
 772
 773         return 0;
 774 }
 775
 776 /*
 777  * HW ECC read page function
 778  *
 779  * mtd:        mtd info structure
 780  * chip:       nand chip info structure
 781  * buf:        buffer to store read data
 782  * oob_required:    caller expects OOB data read to chip->oob_poi
 783  */
 784 static int atmel_nand_read_page(struct mtd_info *mtd, struct nand_chip *chip,
 785                                 uint8_t *buf, int oob_required, int page)
 786 {
 787         int eccsize = chip->ecc.size;
 788         int eccbytes = chip->ecc.bytes;
 789         uint32_t *eccpos = chip->ecc.layout->eccpos;
 790         uint8_t *p = buf;
 791         uint8_t *oob = chip->oob_poi;
 792         uint8_t *ecc_pos;
 793         int stat;
 794
 795         /* read the page */
 796         chip->read_buf(mtd, p, eccsize);
 797
 798         /* move to ECC position if needed */
 799         if (eccpos[0] != 0) {
 800                 /* This only works on large pages
 801                  * because the ECC controller waits for
 802                  * NAND_CMD_RNDOUTSTART after the
 803                  * NAND_CMD_RNDOUT.
 804                  * anyway, for small pages, the eccpos[0] == 0
 805                  */
 806                 chip->cmdfunc(mtd, NAND_CMD_RNDOUT,
 807                                 mtd->writesize + eccpos[0], -1);
 808         }
 809
 810         /* the ECC controller needs to read the ECC just after the data */
 811         ecc_pos = oob + eccpos[0];
 812         chip->read_buf(mtd, ecc_pos, eccbytes);
 813
 814         /* check if there's an error */
 815         stat = chip->ecc.correct(mtd, p, oob, NULL);
 816
 817         if (stat < 0)
 818                 mtd->ecc_stats.failed++;
 819         else
 820                 mtd->ecc_stats.corrected += stat;
 821
 822         /* get back to oob start (end of page) */
 823         chip->cmdfunc(mtd, NAND_CMD_RNDOUT, mtd->writesize, -1);
 824
 825         /* read the oob */
 826         chip->read_buf(mtd, oob, mtd->oobsize);
 827
 828         return 0;
 829 }
 830
 831 /*
 832  * HW ECC Correction
 833  *
 834  * function called after a read
 835  *
 836  * mtd:        MTD block structure
 837  * dat:        raw data read from the chip
 838  * read_ecc:   ECC from the chip (unused)
 839  * isnull:     unused
 840  *
 841  * Detect and correct a 1 bit error for a page
 842  */
 843 static int atmel_nand_correct(struct mtd_info *mtd, u_char *dat,
 844                 u_char *read_ecc, u_char *isnull)
 845 {
 846         struct nand_chip *nand_chip = mtd->priv;
 847         unsigned int ecc_status;
 848         unsigned int ecc_word, ecc_bit;
 849
 850         /* get the status from the Status Register */
 851         ecc_status = ecc_readl(CONFIG_SYS_NAND_ECC_BASE, SR);
 852
 853         /* if there's no error */
 854         if (likely(!(ecc_status & ATMEL_ECC_RECERR)))
 855                 return 0;
 856
 857         /* get error bit offset (4 bits) */
 858         ecc_bit = ecc_readl(CONFIG_SYS_NAND_ECC_BASE, PR) & ATMEL_ECC_BITADDR;
 859         /* get word address (12 bits) */
 860         ecc_word = ecc_readl(CONFIG_SYS_NAND_ECC_BASE, PR) & ATMEL_ECC_WORDADDR;
 861         ecc_word >>= 4;
 862
 863         /* if there are multiple errors */
 864         if (ecc_status & ATMEL_ECC_MULERR) {
 865                 /* check if it is a freshly erased block
 866                  * (filled with 0xff) */
 867                 if ((ecc_bit == ATMEL_ECC_BITADDR)
 868                                 && (ecc_word == (ATMEL_ECC_WORDADDR >> 4))) {
 869                         /* the block has just been erased, return OK */
 870                         return 0;
 871                 }
 872                 /* it doesn't seems to be a freshly
 873                  * erased block.
 874                  * We can't correct so many errors */
 875                 printk(KERN_WARNING "atmel_nand : multiple errors detected."
 876                                 " Unable to correct.\n");
 877                 return -EIO;
 878         }
 879
 880         /* if there's a single bit error : we can correct it */
 881         if (ecc_status & ATMEL_ECC_ECCERR) {
 882                 /* there's nothing much to do here.
 883                  * the bit error is on the ECC itself.
 884                  */
 885                 printk(KERN_WARNING "atmel_nand : one bit error on ECC code."
 886                                 " Nothing to correct\n");
 887                 return 0;
 888         }
 889
 890         printk(KERN_WARNING "atmel_nand : one bit error on data."
 891                         " (word offset in the page :"
 892                         " 0x%x bit offset : 0x%x)\n",
 893                         ecc_word, ecc_bit);
 894         /* correct the error */
 895         if (nand_chip->options & NAND_BUSWIDTH_16) {
 896                 /* 16 bits words */
 897                 ((unsigned short *) dat)[ecc_word] ^= (1 << ecc_bit);
 898         } else {
 899                 /* 8 bits words */
 900                 dat[ecc_word] ^= (1 << ecc_bit);
 901         }
 902         printk(KERN_WARNING "atmel_nand : error corrected\n");
 903         return 1;
 904 }
 905
 906 /*
 907  * Enable HW ECC : unused on most chips
 908  */
 909 static void atmel_nand_hwctl(struct mtd_info *mtd, int mode)
 910 {
 911 }
 912
 913 int atmel_hwecc_nand_init_param(struct nand_chip *nand, struct mtd_info *mtd)
 914 {
 915         nand->ecc.mode = NAND_ECC_HW;
 916         nand->ecc.calculate = atmel_nand_calculate;
 917         nand->ecc.correct = atmel_nand_correct;
 918         nand->ecc.hwctl = atmel_nand_hwctl;
 919         nand->ecc.read_page = atmel_nand_read_page;
 920         nand->ecc.bytes = 4;
 921
 922         if (nand->ecc.mode == NAND_ECC_HW) {
 923                 /* ECC is calculated for the whole page (1 step) */
 924                 nand->ecc.size = mtd->writesize;
 925
 926                 /* set ECC page size and oob layout */
 927                 switch (mtd->writesize) {
 928                 case 512:
 929                         nand->ecc.layout = &atmel_oobinfo_small;
 930                         ecc_writel(CONFIG_SYS_NAND_ECC_BASE, MR,
 931                                         ATMEL_ECC_PAGESIZE_528);
 932                         break;
 933                 case 1024:
 934                         nand->ecc.layout = &atmel_oobinfo_large;
 935                         ecc_writel(CONFIG_SYS_NAND_ECC_BASE, MR,
 936                                         ATMEL_ECC_PAGESIZE_1056);
 937                         break;
 938                 case 2048:
 939                         nand->ecc.layout = &atmel_oobinfo_large;
 940                         ecc_writel(CONFIG_SYS_NAND_ECC_BASE, MR,
 941                                         ATMEL_ECC_PAGESIZE_2112);
 942                         break;
 943                 case 4096:
 944                         nand->ecc.layout = &atmel_oobinfo_large;
 945                         ecc_writel(CONFIG_SYS_NAND_ECC_BASE, MR,
 946                                         ATMEL_ECC_PAGESIZE_4224);
 947                         break;
 948                 default:
 949                         /* page size not handled by HW ECC */
 950                         /* switching back to soft ECC */
 951                         nand->ecc.mode = NAND_ECC_SOFT;
 952                         nand->ecc.calculate = NULL;
 953                         nand->ecc.correct = NULL;
 954                         nand->ecc.hwctl = NULL;
 955                         nand->ecc.read_page = NULL;
 956                         nand->ecc.postpad = 0;
 957                         nand->ecc.prepad = 0;
 958                         nand->ecc.bytes = 0;
 959                         break;
 960                 }
 961         }
 962
 963         return 0;
 964 }
 965
 966 #endif /* CONFIG_ATMEL_NAND_HW_PMECC */
 967
 968 #endif /* CONFIG_ATMEL_NAND_HWECC */
 969
 970 static void at91_nand_hwcontrol(struct mtd_info *mtd,
 971                                          int cmd, unsigned int ctrl)
 972 {
 973         struct nand_chip *this = mtd->priv;
 974
 975         if (ctrl & NAND_CTRL_CHANGE) {
 976                 ulong IO_ADDR_W = (ulong) this->IO_ADDR_W;
 977                 IO_ADDR_W &= ~(CONFIG_SYS_NAND_MASK_ALE
 978                              | CONFIG_SYS_NAND_MASK_CLE);
 979
 980                 if (ctrl & NAND_CLE)
 981                         IO_ADDR_W |= CONFIG_SYS_NAND_MASK_CLE;
 982                 if (ctrl & NAND_ALE)
 983                         IO_ADDR_W |= CONFIG_SYS_NAND_MASK_ALE;
 984
 985 #ifdef CONFIG_SYS_NAND_ENABLE_PIN
 986                 at91_set_gpio_value(CONFIG_SYS_NAND_ENABLE_PIN,
 987                                     !(ctrl & NAND_NCE));
 988 #endif
 989                 this->IO_ADDR_W = (void *) IO_ADDR_W;
 990         }
 991
 992         if (cmd != NAND_CMD_NONE)
 993                 writeb(cmd, this->IO_ADDR_W);
 994 }
 995
 996 #ifdef CONFIG_SYS_NAND_READY_PIN
 997 static int at91_nand_ready(struct mtd_info *mtd)
 998 {
 999         return at91_get_gpio_value(CONFIG_SYS_NAND_READY_PIN);
1000 }
1001 #endif
1002
1003 #ifndef CONFIG_SYS_NAND_BASE_LIST
1004 #define CONFIG_SYS_NAND_BASE_LIST { CONFIG_SYS_NAND_BASE }
1005 #endif
1006 static struct nand_chip nand_chip[CONFIG_SYS_MAX_NAND_DEVICE];
1007 static ulong base_addr[CONFIG_SYS_MAX_NAND_DEVICE] = CONFIG_SYS_NAND_BASE_LIST;
1008
1009 int atmel_nand_chip_init(int devnum, ulong base_addr)
1010 {
1011         int ret;
1012         struct mtd_info *mtd = &nand_info[devnum];
1013         struct nand_chip *nand = &nand_chip[devnum];
1014
1015         mtd->priv = nand;
1016         nand->IO_ADDR_R = nand->IO_ADDR_W = (void  __iomem *)base_addr;
1017
1018         nand->ecc.mode = NAND_ECC_SOFT;
1019 #ifdef CONFIG_SYS_NAND_DBW_16
1020         nand->options = NAND_BUSWIDTH_16;
1021 #endif
1022         nand->cmd_ctrl = at91_nand_hwcontrol;
1023 #ifdef CONFIG_SYS_NAND_READY_PIN
1024         nand->dev_ready = at91_nand_ready;
1025 #endif
1026         nand->chip_delay = 20;
1027
1028         ret = nand_scan_ident(mtd, CONFIG_SYS_NAND_MAX_CHIPS, NULL);
1029         if (ret)
1030                 return ret;
1031
1032 #ifdef CONFIG_ATMEL_NAND_HWECC
1033 #ifdef CONFIG_ATMEL_NAND_HW_PMECC
1034         ret = atmel_pmecc_nand_init_params(nand, mtd);
1035 #else
1036         ret = atmel_hwecc_nand_init_param(nand, mtd);
1037 #endif
1038         if (ret)
1039                 return ret;
1040 #endif
1041
1042         ret = nand_scan_tail(mtd);
1043         if (!ret)
1044                 nand_register(devnum);
1045
1046         return ret;
1047 }
1048
1049 void board_nand_init(void)
1050 {
1051         int i;
1052         for (i = 0; i < CONFIG_SYS_MAX_NAND_DEVICE; i++)
1053                 if (atmel_nand_chip_init(i, base_addr[i]))
1054                         printk(KERN_ERR "atmel_nand: Fail to initialize #%d chip",
1055                                 i);
1056 }