]> git.kernelconcepts.de Git - karo-tx-uboot.git/blob - drivers/mtd/nand/omap_gpmc.c
karo: tx6: improve pad ctrl for SD card interfaces
[karo-tx-uboot.git] / drivers / mtd / nand / omap_gpmc.c
1 /*
2  * (C) Copyright 2004-2008 Texas Instruments, <www.ti.com>
3  * Rohit Choraria <rohitkc@ti.com>
4  *
5  * SPDX-License-Identifier:     GPL-2.0+
6  */
7
8 #include <common.h>
9 #include <asm/io.h>
10 #include <asm/errno.h>
11 #include <asm/arch/mem.h>
12 #include <linux/mtd/omap_gpmc.h>
13 #include <linux/mtd/nand_ecc.h>
14 #include <linux/bch.h>
15 #include <linux/compiler.h>
16 #include <nand.h>
17 #include <linux/mtd/omap_elm.h>
18
19 #define BADBLOCK_MARKER_LENGTH  2
20 #define SECTOR_BYTES            512
21 #define ECCCLEAR                (0x1 << 8)
22 #define ECCRESULTREG1           (0x1 << 0)
23 /* 4 bit padding to make byte aligned, 56 = 52 + 4 */
24 #define BCH4_BIT_PAD            4
25
26 #ifdef CONFIG_BCH
27 static u8  bch8_polynomial[] = {0xef, 0x51, 0x2e, 0x09, 0xed, 0x93, 0x9a, 0xc2,
28                                 0x97, 0x79, 0xe5, 0x24, 0xb5};
29 #endif
30 static uint8_t cs_next;
31 static __maybe_unused struct nand_ecclayout omap_ecclayout;
32
33 /*
34  * Driver configurations
35  */
36 struct omap_nand_info {
37         struct bch_control *control;
38         enum omap_ecc ecc_scheme;
39         int cs;
40 };
41
42 /* We are wasting a bit of memory but al least we are safe */
43 static struct omap_nand_info omap_nand_info[GPMC_MAX_CS];
44
45 static struct gpmc __iomem *gpmc_cfg = (void __iomem *)GPMC_BASE;
46
47 #ifdef CONFIG_SYS_NAND_USE_FLASH_BBT
48 static uint8_t bbt_pattern[] = {'B', 'b', 't', '0' };
49 static uint8_t mirror_pattern[] = {'1', 't', 'b', 'B' };
50
51 static struct nand_bbt_descr bbt_main_descr = {
52         .options = NAND_BBT_LASTBLOCK | NAND_BBT_CREATE | NAND_BBT_WRITE |
53                 NAND_BBT_2BIT | NAND_BBT_VERSION | NAND_BBT_PERCHIP,
54         .offs = 0, /* may be overwritten depending on ECC layout */
55         .len = 4,
56         .veroffs = 4, /* may be overwritten depending on ECC layout */
57         .maxblocks = 4,
58         .pattern = bbt_pattern,
59 };
60
61 static struct nand_bbt_descr bbt_mirror_descr = {
62         .options = NAND_BBT_LASTBLOCK | NAND_BBT_CREATE | NAND_BBT_WRITE |
63                 NAND_BBT_2BIT | NAND_BBT_VERSION | NAND_BBT_PERCHIP,
64         .offs = 0, /* may be overwritten depending on ECC layout */
65         .len = 4,
66         .veroffs = 4, /* may be overwritten depending on ECC layout */
67         .maxblocks = 4,
68         .pattern = mirror_pattern,
69 };
70 #endif
71
72 #define PREFETCH_FIFOTHRESHOLD_MAX              0x40
73 #define PREFETCH_FIFOTHRESHOLD(val)             ((val) << 8)
74
75 #define PREFETCH_ENABLEOPTIMIZEDACCESS          (0x1 << 27)
76
77 #define GPMC_PREFETCH_STATUS_FIFO_CNT(val)      (((val) >> 24) & 0x7F)
78 #define GPMC_PREFETCH_STATUS_COUNT(val)         ((val) & 0x00003fff)
79
80 #define CS_NUM_SHIFT                            24
81 #define ENABLE_PREFETCH                         (0x1 << 7)
82 #define DMA_MPU_MODE                            2
83
84 #define OMAP_NAND_TIMEOUT_MS                    5000
85
86 #define PRINT_REG(x) debug("+++ %.15s (0x%08x)=0x%08x\n", #x, &gpmc_cfg->x, readl(&gpmc_cfg->x))
87
88 #ifdef CONFIG_SYS_GPMC_PREFETCH_ENABLE
89 /**
90  * gpmc_prefetch_enable - configures and starts prefetch transfer
91  * @cs: cs (chip select) number
92  * @fifo_th: fifo threshold to be used for read/ write
93  * @count: number of bytes to be transferred
94  * @is_write: prefetch read(0) or write post(1) mode
95  */
96 static inline void gpmc_prefetch_enable(int cs, int fifo_th,
97                                         unsigned int count, int is_write)
98 {
99         writel(count, &gpmc_cfg->pref_config2);
100
101         /* Set the prefetch read / post write and enable the engine.
102          * Set which cs is has requested for.
103          */
104         uint32_t val = (cs << CS_NUM_SHIFT) |
105                 PREFETCH_ENABLEOPTIMIZEDACCESS |
106                 PREFETCH_FIFOTHRESHOLD(fifo_th) |
107                 ENABLE_PREFETCH |
108                 !!is_write;
109         writel(val, &gpmc_cfg->pref_config1);
110
111         /*  Start the prefetch engine */
112         writel(0x1, &gpmc_cfg->pref_control);
113 }
114
115 /**
116  * gpmc_prefetch_reset - disables and stops the prefetch engine
117  */
118 static inline void gpmc_prefetch_reset(void)
119 {
120         /* Stop the PFPW engine */
121         writel(0x0, &gpmc_cfg->pref_control);
122
123         /* Reset/disable the PFPW engine */
124         writel(0x0, &gpmc_cfg->pref_config1);
125 }
126
127 //#define FIFO_IOADDR           (nand->IO_ADDR_R)
128 #define FIFO_IOADDR             PISMO1_NAND_BASE
129
130 /**
131  * read_buf_pref - read data from NAND controller into buffer
132  * @mtd: MTD device structure
133  * @buf: buffer to store date
134  * @len: number of bytes to read
135  */
136 static void read_buf_pref(struct mtd_info *mtd, u_char *buf, int len)
137 {
138         gpmc_prefetch_enable(cs, PREFETCH_FIFOTHRESHOLD_MAX, len, 0);
139         do {
140                 // Get number of bytes waiting in the FIFO
141                 uint32_t read_bytes = GPMC_PREFETCH_STATUS_FIFO_CNT(readl(&gpmc_cfg->pref_status));
142
143                 if (read_bytes == 0)
144                         continue;
145                 // Alignment of Destination Buffer
146                 while (read_bytes && ((unsigned int)buf & 3)) {
147                         *buf++ = readb(FIFO_IOADDR);
148                         read_bytes--;
149                         len--;
150                 }
151                 // Use maximum word size (32bit) inside this loop, because speed is limited by
152                 // GPMC bus arbitration with a maximum transfer rate of 3.000.000/sec.
153                 len -= read_bytes & ~3;
154                 while (read_bytes >= 4) {
155                         *((uint32_t*)buf) = readl(FIFO_IOADDR);
156                         buf += 4;
157                         read_bytes -= 4;
158                 }
159                 // Transfer the last (non-aligned) bytes only at the last iteration,
160                 // to maintain full speed up to the end of the transfer.
161                 if (read_bytes == len) {
162                         while (read_bytes) {
163                                 *buf++ = readb(FIFO_IOADDR);
164                                 read_bytes--;
165                         }
166                         len = 0;
167                 }
168         } while (len > 0);
169         gpmc_prefetch_reset();
170 }
171
172 /*
173  * write_buf_pref - write buffer to NAND controller
174  * @mtd: MTD device structure
175  * @buf: data buffer
176  * @len: number of bytes to write
177  */
178 static void write_buf_pref(struct mtd_info *mtd, const u_char *buf, int len)
179 {
180         /*  configure and start prefetch transfer */
181         gpmc_prefetch_enable(cs, PREFETCH_FIFOTHRESHOLD_MAX, len, 1);
182
183         while (len) {
184                 // Get number of free bytes in the FIFO
185                 uint32_t write_bytes = GPMC_PREFETCH_STATUS_FIFO_CNT(readl(&gpmc_cfg->pref_status));
186
187                 // don't write more bytes than requested
188                 if (write_bytes > len)
189                         write_bytes = len;
190
191                 // Alignment of Source Buffer
192                 while (write_bytes && ((unsigned int)buf & 3)) {
193                         writeb(*buf++, FIFO_IOADDR);
194                         write_bytes--;
195                         len--;
196                 }
197
198                 // Use maximum word size (32bit) inside this loop, because speed is limited by
199                 // GPMC bus arbitration with a maximum transfer rate of 3.000.000/sec.
200                 len -= write_bytes & ~3;
201                 while (write_bytes >= 4) {
202                         writel(*((uint32_t*)buf), FIFO_IOADDR);
203                         buf += 4;
204                         write_bytes -= 4;
205                 }
206
207                 // Transfer the last (non-aligned) bytes only at the last iteration,
208                 // to maintain full speed up to the end of the transfer.
209                 if (write_bytes == len) {
210                         while (write_bytes) {
211                                 writeb(*buf++, FIFO_IOADDR);
212                                 write_bytes--;
213                         }
214                         len = 0;
215                 }
216         }
217
218         /* wait for data to be flushed out before resetting the prefetch */
219         while ((len = GPMC_PREFETCH_STATUS_COUNT(readl(&gpmc_cfg->pref_status)))) {
220                 debug("%u bytes still in FIFO\n", PREFETCH_FIFOTHRESHOLD_MAX - len);
221                 ndelay(1);
222         }
223
224         /* disable and stop the PFPW engine */
225         gpmc_prefetch_reset();
226 }
227 #endif /* CONFIG_SYS_GPMC_PREFETCH_ENABLE */
228
229 /*
230  * omap_nand_hwcontrol - Set the address pointers corretly for the
231  *                      following address/data/command operation
232  */
233 static void omap_nand_hwcontrol(struct mtd_info *mtd, int32_t cmd,
234                                 uint32_t ctrl)
235 {
236         register struct nand_chip *this = mtd->priv;
237         struct omap_nand_info *info = this->priv;
238         int cs = info->cs;
239
240         /*
241          * Point the IO_ADDR to DATA and ADDRESS registers instead
242          * of chip address
243          */
244         switch (ctrl) {
245         case NAND_CTRL_CHANGE | NAND_CTRL_CLE:
246                 this->IO_ADDR_W = (void __iomem *)&gpmc_cfg->cs[cs].nand_cmd;
247                 break;
248         case NAND_CTRL_CHANGE | NAND_CTRL_ALE:
249                 this->IO_ADDR_W = (void __iomem *)&gpmc_cfg->cs[cs].nand_adr;
250                 break;
251         case NAND_CTRL_CHANGE | NAND_NCE:
252                 this->IO_ADDR_W = (void __iomem *)&gpmc_cfg->cs[cs].nand_dat;
253                 break;
254         }
255
256         if (cmd != NAND_CMD_NONE)
257                 writeb(cmd, this->IO_ADDR_W);
258 }
259
260 /* Check wait pin as dev ready indicator */
261 static int omap_dev_ready(struct mtd_info *mtd)
262 {
263         return readl(&gpmc_cfg->status) & (1 << 8);
264 }
265
266 /*
267  * gen_true_ecc - This function will generate true ECC value, which
268  * can be used when correcting data read from NAND flash memory core
269  *
270  * @ecc_buf:    buffer to store ecc code
271  *
272  * @return:     re-formatted ECC value
273  */
274 static uint32_t gen_true_ecc(uint8_t *ecc_buf)
275 {
276         return ecc_buf[0] | (ecc_buf[1] << 16) | ((ecc_buf[2] & 0xF0) << 20) |
277                 ((ecc_buf[2] & 0x0F) << 8);
278 }
279
280 /*
281  * omap_correct_data - Compares the ecc read from nand spare area with ECC
282  * registers values and corrects one bit error if it has occured
283  * Further details can be had from OMAP TRM and the following selected links:
284  * http://en.wikipedia.org/wiki/Hamming_code
285  * http://www.cs.utexas.edu/users/plaxton/c/337/05f/slides/ErrorCorrection-4.pdf
286  *
287  * @mtd:                 MTD device structure
288  * @dat:                 page data
289  * @read_ecc:            ecc read from nand flash
290  * @calc_ecc:            ecc read from ECC registers
291  *
292  * @return 0 if data is OK or corrected, else returns -1
293  */
294 static int __maybe_unused omap_correct_data(struct mtd_info *mtd, uint8_t *dat,
295                                 uint8_t *read_ecc, uint8_t *calc_ecc)
296 {
297         uint32_t orig_ecc, new_ecc, res, hm;
298         uint16_t parity_bits, byte;
299         uint8_t bit;
300
301         /* Regenerate the orginal ECC */
302         orig_ecc = gen_true_ecc(read_ecc);
303         new_ecc = gen_true_ecc(calc_ecc);
304         /* Get the XOR of real ecc */
305         res = orig_ecc ^ new_ecc;
306         if (res) {
307                 /* Get the hamming width */
308                 hm = hweight32(res);
309                 /* Single bit errors can be corrected! */
310                 if (hm == 12) {
311                         /* Correctable data! */
312                         parity_bits = res >> 16;
313                         bit = (parity_bits & 0x7);
314                         byte = (parity_bits >> 3) & 0x1FF;
315                         /* Flip the bit to correct */
316                         dat[byte] ^= (0x1 << bit);
317                 } else if (hm == 1) {
318                         printf("Error: Ecc is wrong\n");
319                         /* ECC itself is corrupted */
320                         return 2;
321                 } else {
322                         /*
323                          * hm distance != parity pairs OR one, could mean 2 bit
324                          * error OR potentially be on a blank page..
325                          * orig_ecc: contains spare area data from nand flash.
326                          * new_ecc: generated ecc while reading data area.
327                          * Note: if the ecc = 0, all data bits from which it was
328                          * generated are 0xFF.
329                          * The 3 byte(24 bits) ecc is generated per 512byte
330                          * chunk of a page. If orig_ecc(from spare area)
331                          * is 0xFF && new_ecc(computed now from data area)=0x0,
332                          * this means that data area is 0xFF and spare area is
333                          * 0xFF. A sure sign of a erased page!
334                          */
335                         if ((orig_ecc == 0x0FFF0FFF) && (new_ecc == 0x00000000))
336                                 return 0;
337                         printf("Error: Bad compare! failed\n");
338                         /* detected 2 bit error */
339                         return -1;
340                 }
341         }
342         return 0;
343 }
344
345 /*
346  * omap_enable_hwecc - configures GPMC as per ECC scheme before read/write
347  * @mtd:        MTD device structure
348  * @mode:       Read/Write mode
349  */
350 __maybe_unused
351 static void omap_enable_hwecc(struct mtd_info *mtd, int32_t mode)
352 {
353         struct nand_chip        *nand   = mtd->priv;
354         struct omap_nand_info   *info   = nand->priv;
355         unsigned int dev_width = (nand->options & NAND_BUSWIDTH_16) ? 1 : 0;
356         unsigned int ecc_algo = 0;
357         unsigned int bch_type = 0;
358         unsigned int eccsize1 = 0x00, eccsize0 = 0x00, bch_wrapmode = 0x00;
359         u32 ecc_size_config_val = 0;
360         u32 ecc_config_val = 0;
361         int cs = info->cs;
362
363         /* configure GPMC for specific ecc-scheme */
364         switch (info->ecc_scheme) {
365         case OMAP_ECC_HAM1_CODE_SW:
366                 return;
367         case OMAP_ECC_HAM1_CODE_HW:
368                 ecc_algo = 0x0;
369                 bch_type = 0x0;
370                 bch_wrapmode = 0x00;
371                 eccsize0 = 0xFF;
372                 eccsize1 = 0xFF;
373                 break;
374         case OMAP_ECC_BCH8_CODE_HW_DETECTION_SW:
375         case OMAP_ECC_BCH8_CODE_HW:
376                 ecc_algo = 0x1;
377                 bch_type = 0x1;
378                 if (mode == NAND_ECC_WRITE) {
379                         bch_wrapmode = 0x01;
380                         eccsize0 = 0;  /* extra bits in nibbles per sector */
381                         eccsize1 = 28; /* OOB bits in nibbles per sector */
382                 } else {
383                         bch_wrapmode = 0x01;
384                         eccsize0 = 26; /* ECC bits in nibbles per sector */
385                         eccsize1 = 2;  /* non-ECC bits in nibbles per sector */
386                 }
387                 break;
388         case OMAP_ECC_BCH16_CODE_HW:
389                 ecc_algo = 0x1;
390                 bch_type = 0x2;
391                 if (mode == NAND_ECC_WRITE) {
392                         bch_wrapmode = 0x01;
393                         eccsize0 = 0;  /* extra bits in nibbles per sector */
394                         eccsize1 = 52; /* OOB bits in nibbles per sector */
395                 } else {
396                         bch_wrapmode = 0x01;
397                         eccsize0 = 52; /* ECC bits in nibbles per sector */
398                         eccsize1 = 0;  /* non-ECC bits in nibbles per sector */
399                 }
400                 break;
401         default:
402                 return;
403         }
404         /* Clear ecc and enable bits */
405         writel(ECCCLEAR | ECCRESULTREG1, &gpmc_cfg->ecc_control);
406         /* Configure ecc size for BCH */
407         ecc_size_config_val = (eccsize1 << 22) | (eccsize0 << 12);
408         writel(ecc_size_config_val, &gpmc_cfg->ecc_size_config);
409
410         /* Configure device details for BCH engine */
411         ecc_config_val = ((ecc_algo << 16)      | /* HAM1 | BCHx */
412                         (bch_type << 12)        | /* BCH4/BCH8/BCH16 */
413                         (bch_wrapmode << 8)     | /* wrap mode */
414                         (dev_width << 7)        | /* bus width */
415                         (0x0 << 4)              | /* number of sectors */
416                         (cs <<  1)              | /* ECC CS */
417                         (0x1));                   /* enable ECC */
418         writel(ecc_config_val, &gpmc_cfg->ecc_config);
419 }
420
421 /*
422  *  omap_calculate_ecc - Read ECC result
423  *  @mtd:       MTD structure
424  *  @dat:       unused
425  *  @ecc_code:  ecc_code buffer
426  *  Using noninverted ECC can be considered ugly since writing a blank
427  *  page ie. padding will clear the ECC bytes. This is no problem as
428  *  long nobody is trying to write data on the seemingly unused page.
429  *  Reading an erased page will produce an ECC mismatch between
430  *  generated and read ECC bytes that has to be dealt with separately.
431  *  E.g. if page is 0xFF (fresh erased), and if HW ECC engine within GPMC
432  *  is used, the result of read will be 0x0 while the ECC offsets of the
433  *  spare area will be 0xFF which will result in an ECC mismatch.
434  */
435 static int omap_calculate_ecc(struct mtd_info *mtd, const uint8_t *dat,
436                                 uint8_t *ecc_code)
437 {
438         struct nand_chip *chip = mtd->priv;
439         struct omap_nand_info *info = chip->priv;
440         uint32_t *ptr, val = 0;
441         int8_t i = 0, j;
442
443         switch (info->ecc_scheme) {
444         case OMAP_ECC_HAM1_CODE_HW:
445                 val = readl(&gpmc_cfg->ecc1_result);
446                 ecc_code[0] = val & 0xFF;
447                 ecc_code[1] = (val >> 16) & 0xFF;
448                 ecc_code[2] = ((val >> 8) & 0x0F) | ((val >> 20) & 0xF0);
449                 break;
450 #ifdef CONFIG_BCH
451         case OMAP_ECC_BCH8_CODE_HW_DETECTION_SW:
452 #endif
453         case OMAP_ECC_BCH8_CODE_HW:
454                 ptr = &gpmc_cfg->bch_result_0_3[0].bch_result_x[3];
455                 val = readl(ptr);
456                 ecc_code[i++] = (val >>  0) & 0xFF;
457                 ptr--;
458                 for (j = 0; j < 3; j++) {
459                         val = readl(ptr);
460                         ecc_code[i++] = (val >> 24) & 0xFF;
461                         ecc_code[i++] = (val >> 16) & 0xFF;
462                         ecc_code[i++] = (val >>  8) & 0xFF;
463                         ecc_code[i++] = (val >>  0) & 0xFF;
464                         ptr--;
465                 }
466                 break;
467         case OMAP_ECC_BCH16_CODE_HW:
468                 val = readl(&gpmc_cfg->bch_result_4_6[0].bch_result_x[2]);
469                 ecc_code[i++] = (val >>  8) & 0xFF;
470                 ecc_code[i++] = (val >>  0) & 0xFF;
471                 val = readl(&gpmc_cfg->bch_result_4_6[0].bch_result_x[1]);
472                 ecc_code[i++] = (val >> 24) & 0xFF;
473                 ecc_code[i++] = (val >> 16) & 0xFF;
474                 ecc_code[i++] = (val >>  8) & 0xFF;
475                 ecc_code[i++] = (val >>  0) & 0xFF;
476                 val = readl(&gpmc_cfg->bch_result_4_6[0].bch_result_x[0]);
477                 ecc_code[i++] = (val >> 24) & 0xFF;
478                 ecc_code[i++] = (val >> 16) & 0xFF;
479                 ecc_code[i++] = (val >>  8) & 0xFF;
480                 ecc_code[i++] = (val >>  0) & 0xFF;
481                 for (j = 3; j >= 0; j--) {
482                         val = readl(&gpmc_cfg->bch_result_0_3[0].bch_result_x[j]
483                                                                         );
484                         ecc_code[i++] = (val >> 24) & 0xFF;
485                         ecc_code[i++] = (val >> 16) & 0xFF;
486                         ecc_code[i++] = (val >>  8) & 0xFF;
487                         ecc_code[i++] = (val >>  0) & 0xFF;
488                 }
489                 break;
490         default:
491                 return -EINVAL;
492         }
493         /* ECC scheme specific syndrome customizations */
494         switch (info->ecc_scheme) {
495         case OMAP_ECC_HAM1_CODE_HW:
496                 break;
497 #ifdef CONFIG_BCH
498         case OMAP_ECC_BCH8_CODE_HW_DETECTION_SW:
499
500                 for (i = 0; i < chip->ecc.bytes; i++)
501                         *(ecc_code + i) = *(ecc_code + i) ^
502                                                 bch8_polynomial[i];
503                 break;
504 #endif
505         case OMAP_ECC_BCH8_CODE_HW:
506                 ecc_code[chip->ecc.bytes - 1] = 0x00;
507                 break;
508         case OMAP_ECC_BCH16_CODE_HW:
509                 break;
510         default:
511                 return -EINVAL;
512         }
513         return 0;
514 }
515
516 #ifdef CONFIG_NAND_OMAP_ELM
517 /*
518  * omap_reverse_list - re-orders list elements in reverse order [internal]
519  * @list:       pointer to start of list
520  * @length:     length of list
521 */
522 static void omap_reverse_list(u8 *list, unsigned int length)
523 {
524         unsigned int i, j;
525         unsigned int half_length = length / 2;
526         u8 tmp;
527         for (i = 0, j = length - 1; i < half_length; i++, j--) {
528                 tmp = list[i];
529                 list[i] = list[j];
530                 list[j] = tmp;
531         }
532 }
533
534 /*
535  * omap_correct_data_bch - Compares the ecc read from nand spare area
536  * with ECC registers values and corrects one bit error if it has occured
537  *
538  * @mtd:        MTD device structure
539  * @dat:        page data
540  * @read_ecc:   ecc read from nand flash (ignored)
541  * @calc_ecc:   ecc read from ECC registers
542  *
543  * @return 0 if data is OK or corrected, else returns -1
544  */
545 static int omap_correct_data_bch(struct mtd_info *mtd, uint8_t *dat,
546                                 uint8_t *read_ecc, uint8_t *calc_ecc)
547 {
548         struct nand_chip *chip = mtd->priv;
549         struct omap_nand_info *info = chip->priv;
550         struct nand_ecc_ctrl *ecc = &chip->ecc;
551         uint32_t error_count = 0, error_max;
552         uint32_t error_loc[ELM_MAX_ERROR_COUNT];
553         enum bch_level bch_type;
554         uint32_t i, ecc_flag = 0;
555         uint8_t count;
556         uint32_t byte_pos, bit_pos;
557         int err = 0;
558
559         /* check calculated ecc */
560         for (i = 0; i < ecc->bytes && !ecc_flag; i++) {
561                 if (calc_ecc[i] != 0x00)
562                         ecc_flag = 1;
563         }
564         if (!ecc_flag)
565                 return 0;
566
567         /* check for whether its a erased-page */
568         ecc_flag = 0;
569         for (i = 0; i < ecc->bytes && !ecc_flag; i++) {
570                 if (read_ecc[i] != 0xff)
571                         ecc_flag = 1;
572         }
573         if (!ecc_flag)
574                 return 0;
575
576         /*
577          * while reading ECC result we read it in big endian.
578          * Hence while loading to ELM we have rotate to get the right endian.
579          */
580         switch (info->ecc_scheme) {
581         case OMAP_ECC_BCH8_CODE_HW:
582                 bch_type = BCH_8_BIT;
583                 omap_reverse_list(calc_ecc, ecc->bytes - 1);
584                 break;
585         case OMAP_ECC_BCH16_CODE_HW:
586                 bch_type = BCH_16_BIT;
587                 omap_reverse_list(calc_ecc, ecc->bytes);
588                 break;
589         default:
590                 return -EINVAL;
591         }
592         /* use elm module to check for errors */
593         elm_config(bch_type);
594         err = elm_check_error(calc_ecc, bch_type, &error_count, error_loc);
595         if (err)
596                 return err;
597
598         /* correct bch error */
599         for (count = 0; count < error_count; count++) {
600                 switch (info->ecc_scheme) {
601                 case OMAP_ECC_BCH8_CODE_HW:
602                         /* 14th byte in ECC is reserved to match ROM layout */
603                         error_max = SECTOR_BYTES + (ecc->bytes - 1);
604                         break;
605                 case OMAP_ECC_BCH16_CODE_HW:
606                         error_max = SECTOR_BYTES + ecc->bytes;
607                         break;
608                 default:
609                         return -EINVAL;
610                 }
611                 byte_pos = error_max - (error_loc[count] / 8) - 1;
612                 bit_pos  = error_loc[count] % 8;
613                 if (byte_pos < SECTOR_BYTES) {
614                         dat[byte_pos] ^= 1 << bit_pos;
615                         printf("nand: bit-flip corrected @data=%d\n", byte_pos);
616                 } else if (byte_pos < error_max) {
617                         read_ecc[byte_pos - SECTOR_BYTES] ^= 1 << bit_pos;
618                         printf("nand: bit-flip corrected @oob=%d\n", byte_pos -
619                                                                 SECTOR_BYTES);
620                 } else {
621                         err = -EBADMSG;
622                         printf("nand: error: invalid bit-flip location\n");
623                 }
624         }
625         return (err) ? err : error_count;
626 }
627
628 /**
629  * omap_read_page_bch - hardware ecc based page read function
630  * @mtd:        mtd info structure
631  * @chip:       nand chip info structure
632  * @buf:        buffer to store read data
633  * @oob_required: caller expects OOB data read to chip->oob_poi
634  * @page:       page number to read
635  *
636  */
637 static int omap_read_page_bch(struct mtd_info *mtd, struct nand_chip *chip,
638                                 uint8_t *buf, int oob_required, int page)
639 {
640         int i, eccsize = chip->ecc.size;
641         int eccbytes = chip->ecc.bytes;
642         int eccsteps = chip->ecc.steps;
643         uint8_t *p = buf;
644         uint8_t *ecc_calc = chip->buffers->ecccalc;
645         uint8_t *ecc_code = chip->buffers->ecccode;
646         uint32_t *eccpos = chip->ecc.layout->eccpos;
647         uint8_t *oob = &chip->oob_poi[eccpos[0]];
648         uint32_t data_pos;
649         uint32_t oob_pos;
650
651         data_pos = 0;
652         /* oob area start */
653         oob_pos = (eccsize * eccsteps) + eccpos[0];
654
655         for (i = 0; eccsteps; eccsteps--, i += eccbytes, p += eccsize,
656                                 oob += eccbytes) {
657                 chip->ecc.hwctl(mtd, NAND_ECC_READ);
658                 /* read data */
659                 chip->cmdfunc(mtd, NAND_CMD_RNDOUT, data_pos, -1);
660                 chip->read_buf(mtd, p, eccsize);
661
662                 /* read respective ecc from oob area */
663                 chip->cmdfunc(mtd, NAND_CMD_RNDOUT, oob_pos, -1);
664                 chip->read_buf(mtd, oob, eccbytes);
665                 /* read syndrome */
666                 chip->ecc.calculate(mtd, p, &ecc_calc[i]);
667
668                 if (oob_required) {
669                         /* reread the OOB area to get the metadata */
670                         chip->cmdfunc(mtd, NAND_CMD_RNDOUT, mtd->writesize, page);
671                         chip->read_buf(mtd, chip->oob_poi, mtd->oobsize);
672                 }
673
674                 data_pos += eccsize;
675                 oob_pos += eccbytes;
676         }
677
678         for (i = 0; i < chip->ecc.total; i++)
679                 ecc_code[i] = chip->oob_poi[eccpos[i]];
680
681         eccsteps = chip->ecc.steps;
682         p = buf;
683
684         for (i = 0 ; eccsteps; eccsteps--, i += eccbytes, p += eccsize) {
685                 int stat;
686
687                 stat = chip->ecc.correct(mtd, p, &ecc_code[i], &ecc_calc[i]);
688                 if (stat < 0)
689                         mtd->ecc_stats.failed++;
690                 else
691                         mtd->ecc_stats.corrected += stat;
692         }
693         return 0;
694 }
695 #endif /* CONFIG_NAND_OMAP_ELM */
696
697 /*
698  * OMAP3 BCH8 support (with BCH library)
699  */
700 #ifdef CONFIG_BCH
701 /**
702  * omap_correct_data_bch_sw - Decode received data and correct errors
703  * @mtd: MTD device structure
704  * @data: page data
705  * @read_ecc: ecc read from nand flash
706  * @calc_ecc: ecc read from HW ECC registers
707  */
708 static int omap_correct_data_bch_sw(struct mtd_info *mtd, u_char *data,
709                                  u_char *read_ecc, u_char *calc_ecc)
710 {
711         int i, count;
712         /* cannot correct more than 8 errors */
713         unsigned int errloc[8];
714         struct nand_chip *chip = mtd->priv;
715         struct omap_nand_info *info = chip->priv;
716
717         count = decode_bch(info->control, NULL, 512, read_ecc, calc_ecc,
718                                                         NULL, errloc);
719         if (count > 0) {
720                 /* correct errors */
721                 for (i = 0; i < count; i++) {
722                         /* correct data only, not ecc bytes */
723                         if (errloc[i] < 8*512)
724                                 data[errloc[i]/8] ^= 1 << (errloc[i] & 7);
725                         printf("corrected bitflip %u\n", errloc[i]);
726 #ifdef DEBUG
727                         printf("read_ecc: ");
728                         /*
729                          * BCH8 have 13 bytes of ECC; BCH4 needs adoption
730                          * here!
731                          */
732                         for (i = 0; i < 13; i++)
733                                 printf("%02x ", read_ecc[i]);
734                         printf("\n");
735                         printf("calc_ecc: ");
736                         for (i = 0; i < 13; i++)
737                                 printf("%02x ", calc_ecc[i]);
738                         printf("\n");
739 #endif
740                 }
741         } else if (count < 0) {
742                 printf("ecc unrecoverable error\n");
743         }
744         return count;
745 }
746
747 /**
748  * omap_free_bch - Release BCH ecc resources
749  * @mtd: MTD device structure
750  */
751 static void __maybe_unused omap_free_bch(struct mtd_info *mtd)
752 {
753         struct nand_chip *chip = mtd->priv;
754         struct omap_nand_info *info = chip->priv;
755
756         if (info->control) {
757                 free_bch(info->control);
758                 info->control = NULL;
759         }
760 }
761 #endif /* CONFIG_BCH */
762
763 /**
764  * omap_select_ecc_scheme - configures driver for particular ecc-scheme
765  * @nand: NAND chip device structure
766  * @ecc_scheme: ecc scheme to configure
767  * @pagesize: number of main-area bytes per page of NAND device
768  * @oobsize: number of OOB/spare bytes per page of NAND device
769  */
770 static int omap_select_ecc_scheme(struct nand_chip *nand,
771         enum omap_ecc ecc_scheme, unsigned int pagesize, unsigned int oobsize) {
772         struct omap_nand_info   *info           = nand->priv;
773         struct nand_ecclayout   *ecclayout      = &omap_ecclayout;
774         int eccsteps = pagesize / SECTOR_BYTES;
775         int i;
776
777         switch (ecc_scheme) {
778         case OMAP_ECC_HAM1_CODE_SW:
779                 debug("nand: selected OMAP_ECC_HAM1_CODE_SW\n");
780                 /* For this ecc-scheme, ecc.bytes, ecc.layout, ... are
781                  * initialized in nand_scan_tail(), so just set ecc.mode */
782                 info->control           = NULL;
783                 nand->ecc.mode          = NAND_ECC_SOFT;
784                 nand->ecc.layout        = NULL;
785                 nand->ecc.size          = 0;
786                 break;
787
788         case OMAP_ECC_HAM1_CODE_HW:
789                 debug("nand: selected OMAP_ECC_HAM1_CODE_HW\n");
790                 /* check ecc-scheme requirements before updating ecc info */
791                 if ((3 * eccsteps) + BADBLOCK_MARKER_LENGTH > oobsize) {
792                         printf("nand: error: insufficient OOB: require=%d\n", (
793                                 (3 * eccsteps) + BADBLOCK_MARKER_LENGTH));
794                         return -EINVAL;
795                 }
796                 info->control           = NULL;
797                 /* populate ecc specific fields */
798                 memset(&nand->ecc, 0, sizeof(struct nand_ecc_ctrl));
799                 nand->ecc.mode          = NAND_ECC_HW;
800                 nand->ecc.strength      = 1;
801                 nand->ecc.size          = SECTOR_BYTES;
802                 nand->ecc.bytes         = 3;
803                 nand->ecc.hwctl         = omap_enable_hwecc;
804                 nand->ecc.correct       = omap_correct_data;
805                 nand->ecc.calculate     = omap_calculate_ecc;
806                 /* define ecc-layout */
807                 ecclayout->eccbytes     = nand->ecc.bytes * eccsteps;
808                 for (i = 0; i < ecclayout->eccbytes; i++) {
809                         if (nand->options & NAND_BUSWIDTH_16)
810                                 ecclayout->eccpos[i] = i + 2;
811                         else
812                                 ecclayout->eccpos[i] = i + 1;
813                 }
814                 ecclayout->oobfree[0].offset = i + BADBLOCK_MARKER_LENGTH;
815                 ecclayout->oobfree[0].length = oobsize - ecclayout->eccbytes -
816                                                 BADBLOCK_MARKER_LENGTH;
817                 break;
818
819         case OMAP_ECC_BCH8_CODE_HW_DETECTION_SW:
820 #ifdef CONFIG_BCH
821                 debug("nand: selected OMAP_ECC_BCH8_CODE_HW_DETECTION_SW\n");
822                 /* check ecc-scheme requirements before updating ecc info */
823                 if ((13 * eccsteps) + BADBLOCK_MARKER_LENGTH > oobsize) {
824                         printf("nand: error: insufficient OOB: require=%d\n", (
825                                 (13 * eccsteps) + BADBLOCK_MARKER_LENGTH));
826                         return -EINVAL;
827                 }
828                 /* check if BCH S/W library can be used for error detection */
829                 info->control = init_bch(13, 8, 0x201b);
830                 if (!info->control) {
831                         printf("nand: error: could not init_bch()\n");
832                         return -ENODEV;
833                 }
834                 /* populate ecc specific fields */
835                 memset(&nand->ecc, 0, sizeof(struct nand_ecc_ctrl));
836                 nand->ecc.mode          = NAND_ECC_HW;
837                 nand->ecc.strength      = 8;
838                 nand->ecc.size          = SECTOR_BYTES;
839                 nand->ecc.bytes         = 13;
840                 nand->ecc.hwctl         = omap_enable_hwecc;
841                 nand->ecc.correct       = omap_correct_data_bch_sw;
842                 nand->ecc.calculate     = omap_calculate_ecc;
843                 /* define ecc-layout */
844                 ecclayout->eccbytes     = nand->ecc.bytes * eccsteps;
845                 ecclayout->eccpos[0]    = BADBLOCK_MARKER_LENGTH;
846                 for (i = 1; i < ecclayout->eccbytes; i++) {
847                         if (i % nand->ecc.bytes)
848                                 ecclayout->eccpos[i] =
849                                                 ecclayout->eccpos[i - 1] + 1;
850                         else
851                                 ecclayout->eccpos[i] =
852                                                 ecclayout->eccpos[i - 1] + 2;
853                 }
854                 ecclayout->oobfree[0].offset = i + BADBLOCK_MARKER_LENGTH;
855                 ecclayout->oobfree[0].length = oobsize - ecclayout->eccbytes -
856                                                 BADBLOCK_MARKER_LENGTH;
857                 break;
858 #else
859                 printf("nand: error: CONFIG_BCH required for ECC\n");
860                 return -EINVAL;
861 #endif
862
863         case OMAP_ECC_BCH8_CODE_HW:
864 #ifdef CONFIG_NAND_OMAP_ELM
865                 debug("nand: selected OMAP_ECC_BCH8_CODE_HW\n");
866                 /* check ecc-scheme requirements before updating ecc info */
867                 if ((14 * eccsteps) + BADBLOCK_MARKER_LENGTH > oobsize) {
868                         printf("nand: error: insufficient OOB: require=%d\n", (
869                                 (14 * eccsteps) + BADBLOCK_MARKER_LENGTH));
870                         return -EINVAL;
871                 }
872                 /* intialize ELM for ECC error detection */
873                 elm_init();
874                 info->control           = NULL;
875                 /* populate ecc specific fields */
876                 memset(&nand->ecc, 0, sizeof(struct nand_ecc_ctrl));
877                 nand->ecc.mode          = NAND_ECC_HW;
878                 nand->ecc.strength      = 8;
879                 nand->ecc.size          = SECTOR_BYTES;
880                 nand->ecc.bytes         = 14;
881                 nand->ecc.hwctl         = omap_enable_hwecc;
882                 nand->ecc.correct       = omap_correct_data_bch;
883                 nand->ecc.calculate     = omap_calculate_ecc;
884                 nand->ecc.read_page     = omap_read_page_bch;
885                 /* define ecc-layout */
886                 ecclayout->eccbytes     = nand->ecc.bytes * eccsteps;
887                 for (i = 0; i < ecclayout->eccbytes; i++)
888                         ecclayout->eccpos[i] = i + BADBLOCK_MARKER_LENGTH;
889                 ecclayout->oobfree[0].offset = i + BADBLOCK_MARKER_LENGTH;
890                 ecclayout->oobfree[0].length = oobsize - ecclayout->eccbytes -
891                                                 BADBLOCK_MARKER_LENGTH;
892                 break;
893 #else
894                 printf("nand: error: CONFIG_NAND_OMAP_ELM required for ECC\n");
895                 return -EINVAL;
896 #endif
897
898         case OMAP_ECC_BCH16_CODE_HW:
899 #ifdef CONFIG_NAND_OMAP_ELM
900                 debug("nand: using OMAP_ECC_BCH16_CODE_HW\n");
901                 /* check ecc-scheme requirements before updating ecc info */
902                 if ((26 * eccsteps) + BADBLOCK_MARKER_LENGTH > oobsize) {
903                         printf("nand: error: insufficient OOB: require=%d\n", (
904                                 (26 * eccsteps) + BADBLOCK_MARKER_LENGTH));
905                         return -EINVAL;
906                 }
907                 /* intialize ELM for ECC error detection */
908                 elm_init();
909                 /* populate ecc specific fields */
910                 nand->ecc.mode          = NAND_ECC_HW;
911                 nand->ecc.size          = SECTOR_BYTES;
912                 nand->ecc.bytes         = 26;
913                 nand->ecc.strength      = 16;
914                 nand->ecc.hwctl         = omap_enable_hwecc;
915                 nand->ecc.correct       = omap_correct_data_bch;
916                 nand->ecc.calculate     = omap_calculate_ecc;
917                 nand->ecc.read_page     = omap_read_page_bch;
918                 /* define ecc-layout */
919                 ecclayout->eccbytes     = nand->ecc.bytes * eccsteps;
920                 for (i = 0; i < ecclayout->eccbytes; i++)
921                         ecclayout->eccpos[i] = i + BADBLOCK_MARKER_LENGTH;
922                 ecclayout->oobfree[0].offset = i + BADBLOCK_MARKER_LENGTH;
923                 ecclayout->oobfree[0].length = oobsize - nand->ecc.bytes -
924                                                 BADBLOCK_MARKER_LENGTH;
925                 break;
926 #else
927                 printf("nand: error: CONFIG_NAND_OMAP_ELM required for ECC\n");
928                 return -EINVAL;
929 #endif
930         default:
931                 debug("nand: error: ecc scheme not enabled or supported\n");
932                 return -EINVAL;
933         }
934
935         /* nand_scan_tail() sets ham1 sw ecc; hw ecc layout is set by driver */
936         if (ecc_scheme != OMAP_ECC_HAM1_CODE_SW)
937                 nand->ecc.layout = ecclayout;
938
939         info->ecc_scheme = ecc_scheme;
940         return 0;
941 }
942
943 #ifndef CONFIG_SPL_BUILD
944 /*
945  * omap_nand_switch_ecc - switch the ECC operation between different engines
946  * (h/w and s/w) and different algorithms (hamming and BCHx)
947  *
948  * @hardware            - true if one of the HW engines should be used
949  * @eccstrength         - the number of bits that could be corrected
950  *                        (1 - hamming, 4 - BCH4, 8 - BCH8, 16 - BCH16)
951  */
952 int __maybe_unused omap_nand_switch_ecc(uint32_t hardware, uint32_t eccstrength)
953 {
954         struct nand_chip *nand;
955         struct mtd_info *mtd;
956         int err = 0;
957
958         if (nand_curr_device < 0 ||
959             nand_curr_device >= CONFIG_SYS_MAX_NAND_DEVICE ||
960             !nand_info[nand_curr_device].name) {
961                 printf("nand: error: no NAND devices found\n");
962                 return -ENODEV;
963         }
964
965         mtd = &nand_info[nand_curr_device];
966         nand = mtd->priv;
967         nand->options |= NAND_OWN_BUFFERS;
968         nand->options &= ~NAND_SUBPAGE_READ;
969         /* Setup the ecc configurations again */
970         if (hardware) {
971                 if (eccstrength == 1) {
972                         err = omap_select_ecc_scheme(nand,
973                                         OMAP_ECC_HAM1_CODE_HW,
974                                         mtd->writesize, mtd->oobsize);
975                 } else if (eccstrength == 8) {
976                         err = omap_select_ecc_scheme(nand,
977                                         OMAP_ECC_BCH8_CODE_HW,
978                                         mtd->writesize, mtd->oobsize);
979                 } else {
980                         printf("nand: error: unsupported ECC scheme\n");
981                         return -EINVAL;
982                 }
983         } else {
984                 err = omap_select_ecc_scheme(nand, OMAP_ECC_HAM1_CODE_SW,
985                                         mtd->writesize, mtd->oobsize);
986         }
987
988         /* Update NAND handling after ECC mode switch */
989         if (!err)
990                 err = nand_scan_tail(mtd);
991         return err;
992 }
993 #endif /* CONFIG_SPL_BUILD */
994
995 /*
996  * Board-specific NAND initialization. The following members of the
997  * argument are board-specific:
998  * - IO_ADDR_R: address to read the 8 I/O lines of the flash device
999  * - IO_ADDR_W: address to write the 8 I/O lines of the flash device
1000  * - cmd_ctrl: hardwarespecific function for accesing control-lines
1001  * - waitfunc: hardwarespecific function for accesing device ready/busy line
1002  * - ecc.hwctl: function to enable (reset) hardware ecc generator
1003  * - ecc.mode: mode of ecc, see defines
1004  * - chip_delay: chip dependent delay for transfering data from array to
1005  *   read regs (tR)
1006  * - options: various chip options. They can partly be set to inform
1007  *   nand_scan about special functionality. See the defines for further
1008  *   explanation
1009  */
1010 int board_nand_init(struct nand_chip *nand)
1011 {
1012         int32_t gpmc_config = 0;
1013         int cs = cs_next++;
1014         int err = 0;
1015         /*
1016          * xloader/Uboot's gpmc configuration would have configured GPMC for
1017          * nand type of memory. The following logic scans and latches on to the
1018          * first CS with NAND type memory.
1019          * TBD: need to make this logic generic to handle multiple CS NAND
1020          * devices.
1021          */
1022         while (cs < GPMC_MAX_CS) {
1023                 /* Check if NAND type is set */
1024                 if ((readl(&gpmc_cfg->cs[cs].config1) & 0xC00) == 0x800) {
1025                         /* Found it!! */
1026                         break;
1027                 }
1028                 cs++;
1029         }
1030         if (cs >= GPMC_MAX_CS) {
1031                 printf("nand: error: Unable to find NAND settings in "
1032                         "GPMC Configuration - quitting\n");
1033                 return -ENODEV;
1034         }
1035
1036         gpmc_config = readl(&gpmc_cfg->config);
1037         /* Disable Write protect */
1038         gpmc_config |= 0x10;
1039         writel(gpmc_config, &gpmc_cfg->config);
1040
1041         nand->IO_ADDR_R = (void __iomem *)&gpmc_cfg->cs[cs].nand_dat;
1042         nand->IO_ADDR_W = (void __iomem *)&gpmc_cfg->cs[cs].nand_cmd;
1043         omap_nand_info[cs].control = NULL;
1044         omap_nand_info[cs].cs = cs;
1045         nand->priv      = &omap_nand_info[cs];
1046         nand->cmd_ctrl  = omap_nand_hwcontrol;
1047         nand->options   |= NAND_NO_PADDING | NAND_CACHEPRG;
1048         nand->chip_delay = 100;
1049         nand->ecc.layout = &omap_ecclayout;
1050
1051         /* configure driver and controller based on NAND device bus-width */
1052         gpmc_config = readl(&gpmc_cfg->cs[cs].config1);
1053 #if defined(CONFIG_SYS_NAND_BUSWIDTH_16BIT)
1054         nand->options |= NAND_BUSWIDTH_16;
1055         writel(gpmc_config | (0x1 << 12), &gpmc_cfg->cs[cs].config1);
1056 #else
1057         nand->options &= ~NAND_BUSWIDTH_16;
1058         writel(gpmc_config & ~(0x1 << 12), &gpmc_cfg->cs[cs].config1);
1059 #endif
1060         /* select ECC scheme */
1061 #if defined(CONFIG_NAND_OMAP_ECCSCHEME)
1062         err = omap_select_ecc_scheme(nand, CONFIG_NAND_OMAP_ECCSCHEME,
1063                         CONFIG_SYS_NAND_PAGE_SIZE, CONFIG_SYS_NAND_OOBSIZE);
1064 #else
1065         /* pagesize and oobsize are not required to configure sw ecc-scheme */
1066         err = omap_select_ecc_scheme(nand, OMAP_ECC_HAM1_CODE_SW,
1067                         0, 0);
1068 #endif
1069         if (err)
1070                 return err;
1071 #ifdef CONFIG_SYS_NAND_USE_FLASH_BBT
1072         if (nand->ecc.layout) {
1073                 bbt_main_descr.offs = nand->ecc.layout->oobfree[0].offset;
1074                 bbt_main_descr.veroffs = bbt_main_descr.offs +
1075                         sizeof(bbt_pattern);
1076
1077                 bbt_mirror_descr.offs = nand->ecc.layout->oobfree[0].offset;
1078                 bbt_mirror_descr.veroffs = bbt_mirror_descr.offs +
1079                         sizeof(mirror_pattern);
1080         }
1081
1082         nand->bbt_options |= NAND_BBT_USE_FLASH;
1083         nand->bbt_td = &bbt_main_descr;
1084         nand->bbt_md = &bbt_mirror_descr;
1085 #endif
1086
1087 #ifdef CONFIG_SPL_BUILD
1088         if (nand->options & NAND_BUSWIDTH_16)
1089                 nand->read_buf = nand_read_buf16;
1090         else
1091                 nand->read_buf = nand_read_buf;
1092 #endif
1093
1094         nand->dev_ready = omap_dev_ready;
1095
1096         return 0;
1097 }