drivers/mtd/nand/atmel_nand.c

   1 /*
   2  * (C) Copyright 2007-2008
   3  * Stelian Pop <stelian@popies.net>
   4  * Lead Tech Design <www.leadtechdesign.com>
   5  *
   6  * (C) Copyright 2006 ATMEL Rousset, Lacressonniere Nicolas
   7  *
   8  * Add Programmable Multibit ECC support for various AT91 SoC
   9  *     (C) Copyright 2012 ATMEL, Hong Xu
  10  *
  11  * SPDX-License-Identifier:     GPL-2.0+
  12  */
  13
  14 #include <common.h>
  15 #include <asm/arch/hardware.h>
  16 #include <asm/arch/gpio.h>
  17 #include <asm/arch/at91_pio.h>
  18
  19 #include <nand.h>
  20 #include <watchdog.h>
  21
  22 #ifdef CONFIG_ATMEL_NAND_HWECC
  23
  24 /* Register access macros */
  25 #define ecc_readl(add, reg)                             \
  26         readl(AT91_BASE_SYS + add + ATMEL_ECC_##reg)
  27 #define ecc_writel(add, reg, value)                     \
  28         writel((value), AT91_BASE_SYS + add + ATMEL_ECC_##reg)
  29
  30 #include "atmel_nand_ecc.h"     /* Hardware ECC registers */
  31
  32 #ifdef CONFIG_ATMEL_NAND_HW_PMECC
  33
  34 struct atmel_nand_host {
  35         struct pmecc_regs __iomem *pmecc;
  36         struct pmecc_errloc_regs __iomem *pmerrloc;
  37         void __iomem            *pmecc_rom_base;
  38
  39         u8              pmecc_corr_cap;
  40         u16             pmecc_sector_size;
  41         u32             pmecc_index_table_offset;
  42
  43         int             pmecc_bytes_per_sector;
  44         int             pmecc_sector_number;
  45         int             pmecc_degree;   /* Degree of remainders */
  46         int             pmecc_cw_len;   /* Length of codeword */
  47
  48         /* lookup table for alpha_to and index_of */
  49         void __iomem    *pmecc_alpha_to;
  50         void __iomem    *pmecc_index_of;
  51
  52         /* data for pmecc computation */
  53         int16_t pmecc_smu[(CONFIG_PMECC_CAP + 2) * (2 * CONFIG_PMECC_CAP + 1)];
  54         int16_t pmecc_partial_syn[2 * CONFIG_PMECC_CAP + 1];
  55         int16_t pmecc_si[2 * CONFIG_PMECC_CAP + 1];
  56         int16_t pmecc_lmu[CONFIG_PMECC_CAP + 1]; /* polynomal order */
  57         int     pmecc_mu[CONFIG_PMECC_CAP + 1];
  58         int     pmecc_dmu[CONFIG_PMECC_CAP + 1];
  59         int     pmecc_delta[CONFIG_PMECC_CAP + 1];
  60 };
  61
  62 static struct atmel_nand_host pmecc_host;
  63 static struct nand_ecclayout atmel_pmecc_oobinfo;
  64
  65 /*
  66  * Return number of ecc bytes per sector according to sector size and
  67  * correction capability
  68  *
  69  * Following table shows what at91 PMECC supported:
  70  * Correction Capability        Sector_512_bytes        Sector_1024_bytes
  71  * =====================        ================        =================
  72  *                2-bits                 4-bytes                  4-bytes
  73  *                4-bits                 7-bytes                  7-bytes
  74  *                8-bits                13-bytes                 14-bytes
  75  *               12-bits                20-bytes                 21-bytes
  76  *               24-bits                39-bytes                 42-bytes
  77  */
  78 static int pmecc_get_ecc_bytes(int cap, int sector_size)
  79 {
  80         int m = 12 + sector_size / 512;
  81         return (m * cap + 7) / 8;
  82 }
  83
  84 static void pmecc_config_ecc_layout(struct nand_ecclayout *layout,
  85         int oobsize, int ecc_len)
  86 {
  87         int i;
  88
  89         layout->eccbytes = ecc_len;
  90
  91         /* ECC will occupy the last ecc_len bytes continuously */
  92         for (i = 0; i < ecc_len; i++)
  93                 layout->eccpos[i] = oobsize - ecc_len + i;
  94
  95         layout->oobfree[0].offset = 2;
  96         layout->oobfree[0].length =
  97                 oobsize - ecc_len - layout->oobfree[0].offset;
  98 }
  99
 100 static void __iomem *pmecc_get_alpha_to(struct atmel_nand_host *host)
 101 {
 102         int table_size;
 103
 104         table_size = host->pmecc_sector_size == 512 ?
 105                 PMECC_INDEX_TABLE_SIZE_512 : PMECC_INDEX_TABLE_SIZE_1024;
 106
 107         /* the ALPHA lookup table is right behind the INDEX lookup table. */
 108         return host->pmecc_rom_base + host->pmecc_index_table_offset +
 109                         table_size * sizeof(int16_t);
 110 }
 111
 112 static void pmecc_gen_syndrome(struct mtd_info *mtd, int sector)
 113 {
 114         struct nand_chip *nand_chip = mtd->priv;
 115         struct atmel_nand_host *host = nand_chip->priv;
 116         int i;
 117         uint32_t value;
 118
 119         /* Fill odd syndromes */
 120         for (i = 0; i < host->pmecc_corr_cap; i++) {
 121                 value = readl(&host->pmecc->rem_port[sector].rem[i / 2]);
 122                 if (i & 1)
 123                         value >>= 16;
 124                 value &= 0xffff;
 125                 host->pmecc_partial_syn[(2 * i) + 1] = (int16_t)value;
 126         }
 127 }
 128
 129 static void pmecc_substitute(struct mtd_info *mtd)
 130 {
 131         struct nand_chip *nand_chip = mtd->priv;
 132         struct atmel_nand_host *host = nand_chip->priv;
 133         int16_t __iomem *alpha_to = host->pmecc_alpha_to;
 134         int16_t __iomem *index_of = host->pmecc_index_of;
 135         int16_t *partial_syn = host->pmecc_partial_syn;
 136         const int cap = host->pmecc_corr_cap;
 137         int16_t *si;
 138         int i, j;
 139
 140         /* si[] is a table that holds the current syndrome value,
 141          * an element of that table belongs to the field
 142          */
 143         si = host->pmecc_si;
 144
 145         memset(&si[1], 0, sizeof(int16_t) * (2 * cap - 1));
 146
 147         /* Computation 2t syndromes based on S(x) */
 148         /* Odd syndromes */
 149         for (i = 1; i < 2 * cap; i += 2) {
 150                 for (j = 0; j < host->pmecc_degree; j++) {
 151                         if (partial_syn[i] & (0x1 << j))
 152                                 si[i] = readw(alpha_to + i * j) ^ si[i];
 153                 }
 154         }
 155         /* Even syndrome = (Odd syndrome) ** 2 */
 156         for (i = 2, j = 1; j <= cap; i = ++j << 1) {
 157                 if (si[j] == 0) {
 158                         si[i] = 0;
 159                 } else {
 160                         int16_t tmp;
 161
 162                         tmp = readw(index_of + si[j]);
 163                         tmp = (tmp * 2) % host->pmecc_cw_len;
 164                         si[i] = readw(alpha_to + tmp);
 165                 }
 166         }
 167 }
 168
 169 /*
 170  * This function defines a Berlekamp iterative procedure for
 171  * finding the value of the error location polynomial.
 172  * The input is si[], initialize by pmecc_substitute().
 173  * The output is smu[][].
 174  *
 175  * This function is written according to chip datasheet Chapter:
 176  * Find the Error Location Polynomial Sigma(x) of Section:
 177  * Programmable Multibit ECC Control (PMECC).
 178  */
 179 static void pmecc_get_sigma(struct mtd_info *mtd)
 180 {
 181         struct nand_chip *nand_chip = mtd->priv;
 182         struct atmel_nand_host *host = nand_chip->priv;
 183
 184         int16_t *lmu = host->pmecc_lmu;
 185         int16_t *si = host->pmecc_si;
 186         int *mu = host->pmecc_mu;
 187         int *dmu = host->pmecc_dmu;     /* Discrepancy */
 188         int *delta = host->pmecc_delta; /* Delta order */
 189         int cw_len = host->pmecc_cw_len;
 190         const int16_t cap = host->pmecc_corr_cap;
 191         const int num = 2 * cap + 1;
 192         int16_t __iomem *index_of = host->pmecc_index_of;
 193         int16_t __iomem *alpha_to = host->pmecc_alpha_to;
 194         int i, j, k;
 195         uint32_t dmu_0_count, tmp;
 196         int16_t *smu = host->pmecc_smu;
 197
 198         /* index of largest delta */
 199         int ro;
 200         int largest;
 201         int diff;
 202
 203         /* Init the Sigma(x) */
 204         memset(smu, 0, sizeof(int16_t) * ARRAY_SIZE(smu));
 205
 206         dmu_0_count = 0;
 207
 208         /* First Row */
 209
 210         /* Mu */
 211         mu[0] = -1;
 212
 213         smu[0] = 1;
 214
 215         /* discrepancy set to 1 */
 216         dmu[0] = 1;
 217         /* polynom order set to 0 */
 218         lmu[0] = 0;
 219         /* delta[0] = (mu[0] * 2 - lmu[0]) >> 1; */
 220         delta[0] = -1;
 221
 222         /* Second Row */
 223
 224         /* Mu */
 225         mu[1] = 0;
 226         /* Sigma(x) set to 1 */
 227         smu[num] = 1;
 228
 229         /* discrepancy set to S1 */
 230         dmu[1] = si[1];
 231
 232         /* polynom order set to 0 */
 233         lmu[1] = 0;
 234
 235         /* delta[1] = (mu[1] * 2 - lmu[1]) >> 1; */
 236         delta[1] = 0;
 237
 238         for (i = 1; i <= cap; i++) {
 239                 mu[i + 1] = i << 1;
 240                 /* Begin Computing Sigma (Mu+1) and L(mu) */
 241                 /* check if discrepancy is set to 0 */
 242                 if (dmu[i] == 0) {
 243                         dmu_0_count++;
 244
 245                         tmp = ((cap - (lmu[i] >> 1) - 1) / 2);
 246                         if ((cap - (lmu[i] >> 1) - 1) & 0x1)
 247                                 tmp += 2;
 248                         else
 249                                 tmp += 1;
 250
 251                         if (dmu_0_count == tmp) {
 252                                 for (j = 0; j <= (lmu[i] >> 1) + 1; j++)
 253                                         smu[(cap + 1) * num + j] =
 254                                                         smu[i * num + j];
 255
 256                                 lmu[cap + 1] = lmu[i];
 257                                 return;
 258                         }
 259
 260                         /* copy polynom */
 261                         for (j = 0; j <= lmu[i] >> 1; j++)
 262                                 smu[(i + 1) * num + j] = smu[i * num + j];
 263
 264                         /* copy previous polynom order to the next */
 265                         lmu[i + 1] = lmu[i];
 266                 } else {
 267                         ro = 0;
 268                         largest = -1;
 269                         /* find largest delta with dmu != 0 */
 270                         for (j = 0; j < i; j++) {
 271                                 if ((dmu[j]) && (delta[j] > largest)) {
 272                                         largest = delta[j];
 273                                         ro = j;
 274                                 }
 275                         }
 276
 277                         /* compute difference */
 278                         diff = (mu[i] - mu[ro]);
 279
 280                         /* Compute degree of the new smu polynomial */
 281                         if ((lmu[i] >> 1) > ((lmu[ro] >> 1) + diff))
 282                                 lmu[i + 1] = lmu[i];
 283                         else
 284                                 lmu[i + 1] = ((lmu[ro] >> 1) + diff) * 2;
 285
 286                         /* Init smu[i+1] with 0 */
 287                         for (k = 0; k < num; k++)
 288                                 smu[(i + 1) * num + k] = 0;
 289
 290                         /* Compute smu[i+1] */
 291                         for (k = 0; k <= lmu[ro] >> 1; k++) {
 292                                 int16_t a, b, c;
 293
 294                                 if (!(smu[ro * num + k] && dmu[i]))
 295                                         continue;
 296                                 a = readw(index_of + dmu[i]);
 297                                 b = readw(index_of + dmu[ro]);
 298                                 c = readw(index_of + smu[ro * num + k]);
 299                                 tmp = a + (cw_len - b) + c;
 300                                 a = readw(alpha_to + tmp % cw_len);
 301                                 smu[(i + 1) * num + (k + diff)] = a;
 302                         }
 303
 304                         for (k = 0; k <= lmu[i] >> 1; k++)
 305                                 smu[(i + 1) * num + k] ^= smu[i * num + k];
 306                 }
 307
 308                 /* End Computing Sigma (Mu+1) and L(mu) */
 309                 /* In either case compute delta */
 310                 delta[i + 1] = (mu[i + 1] * 2 - lmu[i + 1]) >> 1;
 311
 312                 /* Do not compute discrepancy for the last iteration */
 313                 if (i >= cap)
 314                         continue;
 315
 316                 for (k = 0; k <= (lmu[i + 1] >> 1); k++) {
 317                         tmp = 2 * (i - 1);
 318                         if (k == 0) {
 319                                 dmu[i + 1] = si[tmp + 3];
 320                         } else if (smu[(i + 1) * num + k] && si[tmp + 3 - k]) {
 321                                 int16_t a, b, c;
 322                                 a = readw(index_of +
 323                                                 smu[(i + 1) * num + k]);
 324                                 b = si[2 * (i - 1) + 3 - k];
 325                                 c = readw(index_of + b);
 326                                 tmp = a + c;
 327                                 tmp %= cw_len;
 328                                 dmu[i + 1] = readw(alpha_to + tmp) ^
 329                                         dmu[i + 1];
 330                         }
 331                 }
 332         }
 333 }
 334
 335 static int pmecc_err_location(struct mtd_info *mtd)
 336 {
 337         struct nand_chip *nand_chip = mtd->priv;
 338         struct atmel_nand_host *host = nand_chip->priv;
 339         const int cap = host->pmecc_corr_cap;
 340         const int num = 2 * cap + 1;
 341         int sector_size = host->pmecc_sector_size;
 342         int err_nbr = 0;        /* number of error */
 343         int roots_nbr;          /* number of roots */
 344         int i;
 345         uint32_t val;
 346         int16_t *smu = host->pmecc_smu;
 347         int timeout = PMECC_MAX_TIMEOUT_US;
 348
 349         writel(PMERRLOC_DISABLE, &host->pmerrloc->eldis);
 350
 351         for (i = 0; i <= host->pmecc_lmu[cap + 1] >> 1; i++) {
 352                 writel(smu[(cap + 1) * num + i], &host->pmerrloc->sigma[i]);
 353                 err_nbr++;
 354         }
 355
 356         val = PMERRLOC_ELCFG_NUM_ERRORS(err_nbr - 1);
 357         if (sector_size == 1024)
 358                 val |= PMERRLOC_ELCFG_SECTOR_1024;
 359
 360         writel(val, &host->pmerrloc->elcfg);
 361         writel(sector_size * 8 + host->pmecc_degree * cap,
 362                         &host->pmerrloc->elen);
 363
 364         while (--timeout) {
 365                 if (readl(&host->pmerrloc->elisr) & PMERRLOC_CALC_DONE)
 366                         break;
 367                 WATCHDOG_RESET();
 368                 udelay(1);
 369         }
 370
 371         if (!timeout) {
 372                 printk(KERN_ERR "atmel_nand : Timeout to calculate PMECC error location\n");
 373                 return -1;
 374         }
 375
 376         roots_nbr = (readl(&host->pmerrloc->elisr) & PMERRLOC_ERR_NUM_MASK)
 377                         >> 8;
 378         /* Number of roots == degree of smu hence <= cap */
 379         if (roots_nbr == host->pmecc_lmu[cap + 1] >> 1)
 380                 return err_nbr - 1;
 381
 382         /* Number of roots does not match the degree of smu
 383          * unable to correct error */
 384         return -1;
 385 }
 386
 387 static void pmecc_correct_data(struct mtd_info *mtd, uint8_t *buf, uint8_t *ecc,
 388                 int sector_num, int extra_bytes, int err_nbr)
 389 {
 390         struct nand_chip *nand_chip = mtd->priv;
 391         struct atmel_nand_host *host = nand_chip->priv;
 392         int i = 0;
 393         int byte_pos, bit_pos, sector_size, pos;
 394         uint32_t tmp;
 395         uint8_t err_byte;
 396
 397         sector_size = host->pmecc_sector_size;
 398
 399         while (err_nbr) {
 400                 tmp = readl(&host->pmerrloc->el[i]) - 1;
 401                 byte_pos = tmp / 8;
 402                 bit_pos  = tmp % 8;
 403
 404                 if (byte_pos >= (sector_size + extra_bytes))
 405                         BUG();  /* should never happen */
 406
 407                 if (byte_pos < sector_size) {
 408                         err_byte = *(buf + byte_pos);
 409                         *(buf + byte_pos) ^= (1 << bit_pos);
 410
 411                         pos = sector_num * host->pmecc_sector_size + byte_pos;
 412                         printk(KERN_INFO "Bit flip in data area, byte_pos: %d, bit_pos: %d, 0x%02x -> 0x%02x\n",
 413                                 pos, bit_pos, err_byte, *(buf + byte_pos));
 414                 } else {
 415                         /* Bit flip in OOB area */
 416                         tmp = sector_num * host->pmecc_bytes_per_sector
 417                                         + (byte_pos - sector_size);
 418                         err_byte = ecc[tmp];
 419                         ecc[tmp] ^= (1 << bit_pos);
 420
 421                         pos = tmp + nand_chip->ecc.layout->eccpos[0];
 422                         printk(KERN_INFO "Bit flip in OOB, oob_byte_pos: %d, bit_pos: %d, 0x%02x -> 0x%02x\n",
 423                                 pos, bit_pos, err_byte, ecc[tmp]);
 424                 }
 425
 426                 i++;
 427                 err_nbr--;
 428         }
 429
 430         return;
 431 }
 432
 433 static int pmecc_correction(struct mtd_info *mtd, u32 pmecc_stat, uint8_t *buf,
 434         u8 *ecc)
 435 {
 436         struct nand_chip *nand_chip = mtd->priv;
 437         struct atmel_nand_host *host = nand_chip->priv;
 438         int i, err_nbr, eccbytes;
 439         uint8_t *buf_pos;
 440
 441         eccbytes = nand_chip->ecc.bytes;
 442         for (i = 0; i < eccbytes; i++)
 443                 if (ecc[i] != 0xff)
 444                         goto normal_check;
 445         /* Erased page, return OK */
 446         return 0;
 447
 448 normal_check:
 449         for (i = 0; i < host->pmecc_sector_number; i++) {
 450                 err_nbr = 0;
 451                 if (pmecc_stat & 0x1) {
 452                         buf_pos = buf + i * host->pmecc_sector_size;
 453
 454                         pmecc_gen_syndrome(mtd, i);
 455                         pmecc_substitute(mtd);
 456                         pmecc_get_sigma(mtd);
 457
 458                         err_nbr = pmecc_err_location(mtd);
 459                         if (err_nbr == -1) {
 460                                 printk(KERN_ERR "PMECC: Too many errors\n");
 461                                 mtd->ecc_stats.failed++;
 462                                 return -EIO;
 463                         } else {
 464                                 pmecc_correct_data(mtd, buf_pos, ecc, i,
 465                                         host->pmecc_bytes_per_sector, err_nbr);
 466                                 mtd->ecc_stats.corrected += err_nbr;
 467                         }
 468                 }
 469                 pmecc_stat >>= 1;
 470         }
 471
 472         return 0;
 473 }
 474
 475 static int atmel_nand_pmecc_read_page(struct mtd_info *mtd,
 476         struct nand_chip *chip, uint8_t *buf, int oob_required, int page)
 477 {
 478         struct atmel_nand_host *host = chip->priv;
 479         int eccsize = chip->ecc.size;
 480         uint8_t *oob = chip->oob_poi;
 481         uint32_t *eccpos = chip->ecc.layout->eccpos;
 482         uint32_t stat;
 483         int timeout = PMECC_MAX_TIMEOUT_US;
 484
 485         pmecc_writel(host->pmecc, ctrl, PMECC_CTRL_RST);
 486         pmecc_writel(host->pmecc, ctrl, PMECC_CTRL_DISABLE);
 487         pmecc_writel(host->pmecc, cfg, ((pmecc_readl(host->pmecc, cfg))
 488                 & ~PMECC_CFG_WRITE_OP) | PMECC_CFG_AUTO_ENABLE);
 489
 490         pmecc_writel(host->pmecc, ctrl, PMECC_CTRL_ENABLE);
 491         pmecc_writel(host->pmecc, ctrl, PMECC_CTRL_DATA);
 492
 493         chip->read_buf(mtd, buf, eccsize);
 494         chip->read_buf(mtd, oob, mtd->oobsize);
 495
 496         while (--timeout) {
 497                 if (!(pmecc_readl(host->pmecc, sr) & PMECC_SR_BUSY))
 498                         break;
 499                 WATCHDOG_RESET();
 500                 udelay(1);
 501         }
 502
 503         if (!timeout) {
 504                 printk(KERN_ERR "atmel_nand : Timeout to read PMECC page\n");
 505                 return -1;
 506         }
 507
 508         stat = pmecc_readl(host->pmecc, isr);
 509         if (stat != 0)
 510                 if (pmecc_correction(mtd, stat, buf, &oob[eccpos[0]]) != 0)
 511                         return -EIO;
 512
 513         return 0;
 514 }
 515
 516 static int atmel_nand_pmecc_write_page(struct mtd_info *mtd,
 517                 struct nand_chip *chip, const uint8_t *buf,
 518                 int oob_required)
 519 {
 520         struct atmel_nand_host *host = chip->priv;
 521         uint32_t *eccpos = chip->ecc.layout->eccpos;
 522         int i, j;
 523         int timeout = PMECC_MAX_TIMEOUT_US;
 524
 525         pmecc_writel(host->pmecc, ctrl, PMECC_CTRL_RST);
 526         pmecc_writel(host->pmecc, ctrl, PMECC_CTRL_DISABLE);
 527
 528         pmecc_writel(host->pmecc, cfg, (pmecc_readl(host->pmecc, cfg) |
 529                 PMECC_CFG_WRITE_OP) & ~PMECC_CFG_AUTO_ENABLE);
 530
 531         pmecc_writel(host->pmecc, ctrl, PMECC_CTRL_ENABLE);
 532         pmecc_writel(host->pmecc, ctrl, PMECC_CTRL_DATA);
 533
 534         chip->write_buf(mtd, (u8 *)buf, mtd->writesize);
 535
 536         while (--timeout) {
 537                 if (!(pmecc_readl(host->pmecc, sr) & PMECC_SR_BUSY))
 538                         break;
 539                 WATCHDOG_RESET();
 540                 udelay(1);
 541         }
 542
 543         if (!timeout) {
 544                 printk(KERN_ERR "atmel_nand : Timeout to read PMECC status, fail to write PMECC in oob\n");
 545                 goto out;
 546         }
 547
 548         for (i = 0; i < host->pmecc_sector_number; i++) {
 549                 for (j = 0; j < host->pmecc_bytes_per_sector; j++) {
 550                         int pos;
 551
 552                         pos = i * host->pmecc_bytes_per_sector + j;
 553                         chip->oob_poi[eccpos[pos]] =
 554                                 readb(&host->pmecc->ecc_port[i].ecc[j]);
 555                 }
 556         }
 557         chip->write_buf(mtd, chip->oob_poi, mtd->oobsize);
 558 out:
 559         return 0;
 560 }
 561
 562 static void atmel_pmecc_core_init(struct mtd_info *mtd)
 563 {
 564         struct nand_chip *nand_chip = mtd->priv;
 565         struct atmel_nand_host *host = nand_chip->priv;
 566         uint32_t val = 0;
 567         struct nand_ecclayout *ecc_layout;
 568
 569         pmecc_writel(host->pmecc, ctrl, PMECC_CTRL_RST);
 570         pmecc_writel(host->pmecc, ctrl, PMECC_CTRL_DISABLE);
 571
 572         switch (host->pmecc_corr_cap) {
 573         case 2:
 574                 val = PMECC_CFG_BCH_ERR2;
 575                 break;
 576         case 4:
 577                 val = PMECC_CFG_BCH_ERR4;
 578                 break;
 579         case 8:
 580                 val = PMECC_CFG_BCH_ERR8;
 581                 break;
 582         case 12:
 583                 val = PMECC_CFG_BCH_ERR12;
 584                 break;
 585         case 24:
 586                 val = PMECC_CFG_BCH_ERR24;
 587                 break;
 588         }
 589
 590         if (host->pmecc_sector_size == 512)
 591                 val |= PMECC_CFG_SECTOR512;
 592         else if (host->pmecc_sector_size == 1024)
 593                 val |= PMECC_CFG_SECTOR1024;
 594
 595         switch (host->pmecc_sector_number) {
 596         case 1:
 597                 val |= PMECC_CFG_PAGE_1SECTOR;
 598                 break;
 599         case 2:
 600                 val |= PMECC_CFG_PAGE_2SECTORS;
 601                 break;
 602         case 4:
 603                 val |= PMECC_CFG_PAGE_4SECTORS;
 604                 break;
 605         case 8:
 606                 val |= PMECC_CFG_PAGE_8SECTORS;
 607                 break;
 608         }
 609
 610         val |= (PMECC_CFG_READ_OP | PMECC_CFG_SPARE_DISABLE
 611                 | PMECC_CFG_AUTO_DISABLE);
 612         pmecc_writel(host->pmecc, cfg, val);
 613
 614         ecc_layout = nand_chip->ecc.layout;
 615         pmecc_writel(host->pmecc, sarea, mtd->oobsize - 1);
 616         pmecc_writel(host->pmecc, saddr, ecc_layout->eccpos[0]);
 617         pmecc_writel(host->pmecc, eaddr,
 618                         ecc_layout->eccpos[ecc_layout->eccbytes - 1]);
 619         /* See datasheet about PMECC Clock Control Register */
 620         pmecc_writel(host->pmecc, clk, PMECC_CLK_133MHZ);
 621         pmecc_writel(host->pmecc, idr, 0xff);
 622         pmecc_writel(host->pmecc, ctrl, PMECC_CTRL_ENABLE);
 623 }
 624
 625 static int atmel_pmecc_nand_init_params(struct nand_chip *nand,
 626                 struct mtd_info *mtd)
 627 {
 628         struct atmel_nand_host *host;
 629         int cap, sector_size;
 630
 631         host = nand->priv = &pmecc_host;
 632
 633         nand->ecc.mode = NAND_ECC_HW;
 634         nand->ecc.calculate = NULL;
 635         nand->ecc.correct = NULL;
 636         nand->ecc.hwctl = NULL;
 637
 638         cap = host->pmecc_corr_cap = CONFIG_PMECC_CAP;
 639         sector_size = host->pmecc_sector_size = CONFIG_PMECC_SECTOR_SIZE;
 640         host->pmecc_index_table_offset = CONFIG_PMECC_INDEX_TABLE_OFFSET;
 641
 642         MTDDEBUG(MTD_DEBUG_LEVEL1,
 643                 "Initialize PMECC params, cap: %d, sector: %d\n",
 644                 cap, sector_size);
 645
 646         host->pmecc = (struct pmecc_regs __iomem *) ATMEL_BASE_PMECC;
 647         host->pmerrloc = (struct pmecc_errloc_regs __iomem *)
 648                         ATMEL_BASE_PMERRLOC;
 649         host->pmecc_rom_base = (void __iomem *) ATMEL_BASE_ROM;
 650
 651         /* ECC is calculated for the whole page (1 step) */
 652         nand->ecc.size = mtd->writesize;
 653
 654         /* set ECC page size and oob layout */
 655         switch (mtd->writesize) {
 656         case 2048:
 657         case 4096:
 658                 host->pmecc_degree = PMECC_GF_DIMENSION_13;
 659                 host->pmecc_cw_len = (1 << host->pmecc_degree) - 1;
 660                 host->pmecc_sector_number = mtd->writesize / sector_size;
 661                 host->pmecc_bytes_per_sector = pmecc_get_ecc_bytes(
 662                         cap, sector_size);
 663                 host->pmecc_alpha_to = pmecc_get_alpha_to(host);
 664                 host->pmecc_index_of = host->pmecc_rom_base +
 665                         host->pmecc_index_table_offset;
 666
 667                 nand->ecc.steps = 1;
 668                 nand->ecc.bytes = host->pmecc_bytes_per_sector *
 669                                        host->pmecc_sector_number;
 670                 if (nand->ecc.bytes > mtd->oobsize - 2) {
 671                         printk(KERN_ERR "No room for ECC bytes\n");
 672                         return -EINVAL;
 673                 }
 674                 pmecc_config_ecc_layout(&atmel_pmecc_oobinfo,
 675                                         mtd->oobsize,
 676                                         nand->ecc.bytes);
 677                 nand->ecc.layout = &atmel_pmecc_oobinfo;
 678                 break;
 679         case 512:
 680         case 1024:
 681                 /* TODO */
 682                 printk(KERN_ERR "Unsupported page size for PMECC, use Software ECC\n");
 683         default:
 684                 /* page size not handled by HW ECC */
 685                 /* switching back to soft ECC */
 686                 nand->ecc.mode = NAND_ECC_SOFT;
 687                 nand->ecc.read_page = NULL;
 688                 nand->ecc.postpad = 0;
 689                 nand->ecc.prepad = 0;
 690                 nand->ecc.bytes = 0;
 691                 return 0;
 692         }
 693
 694         nand->ecc.read_page = atmel_nand_pmecc_read_page;
 695         nand->ecc.write_page = atmel_nand_pmecc_write_page;
 696         nand->ecc.strength = cap;
 697
 698         atmel_pmecc_core_init(mtd);
 699
 700         return 0;
 701 }
 702
 703 #else
 704
 705 /* oob layout for large page size
 706  * bad block info is on bytes 0 and 1
 707  * the bytes have to be consecutives to avoid
 708  * several NAND_CMD_RNDOUT during read
 709  */
 710 static struct nand_ecclayout atmel_oobinfo_large = {
 711         .eccbytes = 4,
 712         .eccpos = {60, 61, 62, 63},
 713         .oobfree = {
 714                 {2, 58}
 715         },
 716 };
 717
 718 /* oob layout for small page size
 719  * bad block info is on bytes 4 and 5
 720  * the bytes have to be consecutives to avoid
 721  * several NAND_CMD_RNDOUT during read
 722  */
 723 static struct nand_ecclayout atmel_oobinfo_small = {
 724         .eccbytes = 4,
 725         .eccpos = {0, 1, 2, 3},
 726         .oobfree = {
 727                 {6, 10}
 728         },
 729 };
 730
 731 /*
 732  * Calculate HW ECC
 733  *
 734  * function called after a write
 735  *
 736  * mtd:        MTD block structure
 737  * dat:        raw data (unused)
 738  * ecc_code:   buffer for ECC
 739  */
 740 static int atmel_nand_calculate(struct mtd_info *mtd,
 741                 const u_char *dat, unsigned char *ecc_code)
 742 {
 743         unsigned int ecc_value;
 744
 745         /* get the first 2 ECC bytes */
 746         ecc_value = ecc_readl(CONFIG_SYS_NAND_ECC_BASE, PR);
 747
 748         ecc_code[0] = ecc_value & 0xFF;
 749         ecc_code[1] = (ecc_value >> 8) & 0xFF;
 750
 751         /* get the last 2 ECC bytes */
 752         ecc_value = ecc_readl(CONFIG_SYS_NAND_ECC_BASE, NPR) & ATMEL_ECC_NPARITY;
 753
 754         ecc_code[2] = ecc_value & 0xFF;
 755         ecc_code[3] = (ecc_value >> 8) & 0xFF;
 756
 757         return 0;
 758 }
 759
 760 /*
 761  * HW ECC read page function
 762  *
 763  * mtd:        mtd info structure
 764  * chip:       nand chip info structure
 765  * buf:        buffer to store read data
 766  * oob_required:    caller expects OOB data read to chip->oob_poi
 767  */
 768 static int atmel_nand_read_page(struct mtd_info *mtd, struct nand_chip *chip,
 769                                 uint8_t *buf, int oob_required, int page)
 770 {
 771         int eccsize = chip->ecc.size;
 772         int eccbytes = chip->ecc.bytes;
 773         uint32_t *eccpos = chip->ecc.layout->eccpos;
 774         uint8_t *p = buf;
 775         uint8_t *oob = chip->oob_poi;
 776         uint8_t *ecc_pos;
 777         int stat;
 778
 779         /* read the page */
 780         chip->read_buf(mtd, p, eccsize);
 781
 782         /* move to ECC position if needed */
 783         if (eccpos[0] != 0) {
 784                 /* This only works on large pages
 785                  * because the ECC controller waits for
 786                  * NAND_CMD_RNDOUTSTART after the
 787                  * NAND_CMD_RNDOUT.
 788                  * anyway, for small pages, the eccpos[0] == 0
 789                  */
 790                 chip->cmdfunc(mtd, NAND_CMD_RNDOUT,
 791                                 mtd->writesize + eccpos[0], -1);
 792         }
 793
 794         /* the ECC controller needs to read the ECC just after the data */
 795         ecc_pos = oob + eccpos[0];
 796         chip->read_buf(mtd, ecc_pos, eccbytes);
 797
 798         /* check if there's an error */
 799         stat = chip->ecc.correct(mtd, p, oob, NULL);
 800
 801         if (stat < 0)
 802                 mtd->ecc_stats.failed++;
 803         else
 804                 mtd->ecc_stats.corrected += stat;
 805
 806         /* get back to oob start (end of page) */
 807         chip->cmdfunc(mtd, NAND_CMD_RNDOUT, mtd->writesize, -1);
 808
 809         /* read the oob */
 810         chip->read_buf(mtd, oob, mtd->oobsize);
 811
 812         return 0;
 813 }
 814
 815 /*
 816  * HW ECC Correction
 817  *
 818  * function called after a read
 819  *
 820  * mtd:        MTD block structure
 821  * dat:        raw data read from the chip
 822  * read_ecc:   ECC from the chip (unused)
 823  * isnull:     unused
 824  *
 825  * Detect and correct a 1 bit error for a page
 826  */
 827 static int atmel_nand_correct(struct mtd_info *mtd, u_char *dat,
 828                 u_char *read_ecc, u_char *isnull)
 829 {
 830         struct nand_chip *nand_chip = mtd->priv;
 831         unsigned int ecc_status;
 832         unsigned int ecc_word, ecc_bit;
 833
 834         /* get the status from the Status Register */
 835         ecc_status = ecc_readl(CONFIG_SYS_NAND_ECC_BASE, SR);
 836
 837         /* if there's no error */
 838         if (likely(!(ecc_status & ATMEL_ECC_RECERR)))
 839                 return 0;
 840
 841         /* get error bit offset (4 bits) */
 842         ecc_bit = ecc_readl(CONFIG_SYS_NAND_ECC_BASE, PR) & ATMEL_ECC_BITADDR;
 843         /* get word address (12 bits) */
 844         ecc_word = ecc_readl(CONFIG_SYS_NAND_ECC_BASE, PR) & ATMEL_ECC_WORDADDR;
 845         ecc_word >>= 4;
 846
 847         /* if there are multiple errors */
 848         if (ecc_status & ATMEL_ECC_MULERR) {
 849                 /* check if it is a freshly erased block
 850                  * (filled with 0xff) */
 851                 if ((ecc_bit == ATMEL_ECC_BITADDR)
 852                                 && (ecc_word == (ATMEL_ECC_WORDADDR >> 4))) {
 853                         /* the block has just been erased, return OK */
 854                         return 0;
 855                 }
 856                 /* it doesn't seems to be a freshly
 857                  * erased block.
 858                  * We can't correct so many errors */
 859                 printk(KERN_WARNING "atmel_nand : multiple errors detected."
 860                                 " Unable to correct.\n");
 861                 return -EIO;
 862         }
 863
 864         /* if there's a single bit error : we can correct it */
 865         if (ecc_status & ATMEL_ECC_ECCERR) {
 866                 /* there's nothing much to do here.
 867                  * the bit error is on the ECC itself.
 868                  */
 869                 printk(KERN_WARNING "atmel_nand : one bit error on ECC code."
 870                                 " Nothing to correct\n");
 871                 return 0;
 872         }
 873
 874         printk(KERN_WARNING "atmel_nand : one bit error on data."
 875                         " (word offset in the page :"
 876                         " 0x%x bit offset : 0x%x)\n",
 877                         ecc_word, ecc_bit);
 878         /* correct the error */
 879         if (nand_chip->options & NAND_BUSWIDTH_16) {
 880                 /* 16 bits words */
 881                 ((unsigned short *) dat)[ecc_word] ^= (1 << ecc_bit);
 882         } else {
 883                 /* 8 bits words */
 884                 dat[ecc_word] ^= (1 << ecc_bit);
 885         }
 886         printk(KERN_WARNING "atmel_nand : error corrected\n");
 887         return 1;
 888 }
 889
 890 /*
 891  * Enable HW ECC : unused on most chips
 892  */
 893 static void atmel_nand_hwctl(struct mtd_info *mtd, int mode)
 894 {
 895 }
 896
 897 int atmel_hwecc_nand_init_param(struct nand_chip *nand, struct mtd_info *mtd)
 898 {
 899         nand->ecc.mode = NAND_ECC_HW;
 900         nand->ecc.calculate = atmel_nand_calculate;
 901         nand->ecc.correct = atmel_nand_correct;
 902         nand->ecc.hwctl = atmel_nand_hwctl;
 903         nand->ecc.read_page = atmel_nand_read_page;
 904         nand->ecc.bytes = 4;
 905
 906         if (nand->ecc.mode == NAND_ECC_HW) {
 907                 /* ECC is calculated for the whole page (1 step) */
 908                 nand->ecc.size = mtd->writesize;
 909
 910                 /* set ECC page size and oob layout */
 911                 switch (mtd->writesize) {
 912                 case 512:
 913                         nand->ecc.layout = &atmel_oobinfo_small;
 914                         ecc_writel(CONFIG_SYS_NAND_ECC_BASE, MR,
 915                                         ATMEL_ECC_PAGESIZE_528);
 916                         break;
 917                 case 1024:
 918                         nand->ecc.layout = &atmel_oobinfo_large;
 919                         ecc_writel(CONFIG_SYS_NAND_ECC_BASE, MR,
 920                                         ATMEL_ECC_PAGESIZE_1056);
 921                         break;
 922                 case 2048:
 923                         nand->ecc.layout = &atmel_oobinfo_large;
 924                         ecc_writel(CONFIG_SYS_NAND_ECC_BASE, MR,
 925                                         ATMEL_ECC_PAGESIZE_2112);
 926                         break;
 927                 case 4096:
 928                         nand->ecc.layout = &atmel_oobinfo_large;
 929                         ecc_writel(CONFIG_SYS_NAND_ECC_BASE, MR,
 930                                         ATMEL_ECC_PAGESIZE_4224);
 931                         break;
 932                 default:
 933                         /* page size not handled by HW ECC */
 934                         /* switching back to soft ECC */
 935                         nand->ecc.mode = NAND_ECC_SOFT;
 936                         nand->ecc.calculate = NULL;
 937                         nand->ecc.correct = NULL;
 938                         nand->ecc.hwctl = NULL;
 939                         nand->ecc.read_page = NULL;
 940                         nand->ecc.postpad = 0;
 941                         nand->ecc.prepad = 0;
 942                         nand->ecc.bytes = 0;
 943                         break;
 944                 }
 945         }
 946
 947         return 0;
 948 }
 949
 950 #endif /* CONFIG_ATMEL_NAND_HW_PMECC */
 951
 952 #endif /* CONFIG_ATMEL_NAND_HWECC */
 953
 954 static void at91_nand_hwcontrol(struct mtd_info *mtd,
 955                                          int cmd, unsigned int ctrl)
 956 {
 957         struct nand_chip *this = mtd->priv;
 958
 959         if (ctrl & NAND_CTRL_CHANGE) {
 960                 ulong IO_ADDR_W = (ulong) this->IO_ADDR_W;
 961                 IO_ADDR_W &= ~(CONFIG_SYS_NAND_MASK_ALE
 962                              | CONFIG_SYS_NAND_MASK_CLE);
 963
 964                 if (ctrl & NAND_CLE)
 965                         IO_ADDR_W |= CONFIG_SYS_NAND_MASK_CLE;
 966                 if (ctrl & NAND_ALE)
 967                         IO_ADDR_W |= CONFIG_SYS_NAND_MASK_ALE;
 968
 969 #ifdef CONFIG_SYS_NAND_ENABLE_PIN
 970                 at91_set_gpio_value(CONFIG_SYS_NAND_ENABLE_PIN,
 971                                     !(ctrl & NAND_NCE));
 972 #endif
 973                 this->IO_ADDR_W = (void *) IO_ADDR_W;
 974         }
 975
 976         if (cmd != NAND_CMD_NONE)
 977                 writeb(cmd, this->IO_ADDR_W);
 978 }
 979
 980 #ifdef CONFIG_SYS_NAND_READY_PIN
 981 static int at91_nand_ready(struct mtd_info *mtd)
 982 {
 983         return at91_get_gpio_value(CONFIG_SYS_NAND_READY_PIN);
 984 }
 985 #endif
 986
 987 #ifndef CONFIG_SYS_NAND_BASE_LIST
 988 #define CONFIG_SYS_NAND_BASE_LIST { CONFIG_SYS_NAND_BASE }
 989 #endif
 990 static struct nand_chip nand_chip[CONFIG_SYS_MAX_NAND_DEVICE];
 991 static ulong base_addr[CONFIG_SYS_MAX_NAND_DEVICE] = CONFIG_SYS_NAND_BASE_LIST;
 992
 993 int atmel_nand_chip_init(int devnum, ulong base_addr)
 994 {
 995         int ret;
 996         struct mtd_info *mtd = &nand_info[devnum];
 997         struct nand_chip *nand = &nand_chip[devnum];
 998
 999         mtd->priv = nand;
1000         nand->IO_ADDR_R = nand->IO_ADDR_W = (void  __iomem *)base_addr;
1001
1002         nand->ecc.mode = NAND_ECC_SOFT;
1003 #ifdef CONFIG_SYS_NAND_DBW_16
1004         nand->options = NAND_BUSWIDTH_16;
1005 #endif
1006         nand->cmd_ctrl = at91_nand_hwcontrol;
1007 #ifdef CONFIG_SYS_NAND_READY_PIN
1008         nand->dev_ready = at91_nand_ready;
1009 #endif
1010         nand->chip_delay = 20;
1011
1012         ret = nand_scan_ident(mtd, CONFIG_SYS_NAND_MAX_CHIPS, NULL);
1013         if (ret)
1014                 return ret;
1015
1016 #ifdef CONFIG_ATMEL_NAND_HWECC
1017 #ifdef CONFIG_ATMEL_NAND_HW_PMECC
1018         ret = atmel_pmecc_nand_init_params(nand, mtd);
1019 #else
1020         ret = atmel_hwecc_nand_init_param(nand, mtd);
1021 #endif
1022         if (ret)
1023                 return ret;
1024 #endif
1025
1026         ret = nand_scan_tail(mtd);
1027         if (!ret)
1028                 nand_register(devnum);
1029
1030         return ret;
1031 }
1032
1033 void board_nand_init(void)
1034 {
1035         int i;
1036         for (i = 0; i < CONFIG_SYS_MAX_NAND_DEVICE; i++)
1037                 if (atmel_nand_chip_init(i, base_addr[i]))
1038                         printk(KERN_ERR "atmel_nand: Fail to initialize #%d chip",
1039                                 i);
1040 }