2 * Freescale GPMI NAND Flash Driver
4 * Copyright (C) 2010-2015 Freescale Semiconductor, Inc.
5 * Copyright (C) 2008 Embedded Alley Solutions, Inc.
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation; either version 2 of the License, or
10 * (at your option) any later version.
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
17 * You should have received a copy of the GNU General Public License along
18 * with this program; if not, write to the Free Software Foundation, Inc.,
19 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
21 #include <linux/clk.h>
22 #include <linux/slab.h>
23 #include <linux/sched/task_stack.h>
24 #include <linux/interrupt.h>
25 #include <linux/module.h>
26 #include <linux/mtd/partitions.h>
28 #include <linux/of_device.h>
29 #include "gpmi-nand.h"
32 /* Resource names for the GPMI NAND driver. */
33 #define GPMI_NAND_GPMI_REGS_ADDR_RES_NAME "gpmi-nand"
34 #define GPMI_NAND_BCH_REGS_ADDR_RES_NAME "bch"
35 #define GPMI_NAND_BCH_INTERRUPT_RES_NAME "bch"
37 /* add our owner bbt descriptor */
38 static uint8_t scan_ff_pattern[] = { 0xff };
39 static struct nand_bbt_descr gpmi_bbt_descr = {
43 .pattern = scan_ff_pattern
47 * We may change the layout if we can get the ECC info from the datasheet,
48 * else we will use all the (page + OOB).
50 static int gpmi_ooblayout_ecc(struct mtd_info *mtd, int section,
51 struct mtd_oob_region *oobregion)
53 struct nand_chip *chip = mtd_to_nand(mtd);
54 struct gpmi_nand_data *this = nand_get_controller_data(chip);
55 struct bch_geometry *geo = &this->bch_geometry;
60 oobregion->offset = 0;
61 oobregion->length = geo->page_size - mtd->writesize;
66 static int gpmi_ooblayout_free(struct mtd_info *mtd, int section,
67 struct mtd_oob_region *oobregion)
69 struct nand_chip *chip = mtd_to_nand(mtd);
70 struct gpmi_nand_data *this = nand_get_controller_data(chip);
71 struct bch_geometry *geo = &this->bch_geometry;
76 /* The available oob size we have. */
77 if (geo->page_size < mtd->writesize + mtd->oobsize) {
78 oobregion->offset = geo->page_size - mtd->writesize;
79 oobregion->length = mtd->oobsize - oobregion->offset;
85 static const char * const gpmi_clks_for_mx2x[] = {
89 static const struct mtd_ooblayout_ops gpmi_ooblayout_ops = {
90 .ecc = gpmi_ooblayout_ecc,
91 .free = gpmi_ooblayout_free,
94 static const struct gpmi_devdata gpmi_devdata_imx23 = {
96 .bch_max_ecc_strength = 20,
97 .max_chain_delay = 16,
98 .clks = gpmi_clks_for_mx2x,
99 .clks_count = ARRAY_SIZE(gpmi_clks_for_mx2x),
102 static const struct gpmi_devdata gpmi_devdata_imx28 = {
104 .bch_max_ecc_strength = 20,
105 .max_chain_delay = 16,
106 .clks = gpmi_clks_for_mx2x,
107 .clks_count = ARRAY_SIZE(gpmi_clks_for_mx2x),
110 static const char * const gpmi_clks_for_mx6[] = {
111 "gpmi_io", "gpmi_apb", "gpmi_bch", "gpmi_bch_apb", "per1_bch",
114 static const struct gpmi_devdata gpmi_devdata_imx6q = {
116 .bch_max_ecc_strength = 40,
117 .max_chain_delay = 12,
118 .clks = gpmi_clks_for_mx6,
119 .clks_count = ARRAY_SIZE(gpmi_clks_for_mx6),
122 static const struct gpmi_devdata gpmi_devdata_imx6sx = {
124 .bch_max_ecc_strength = 62,
125 .max_chain_delay = 12,
126 .clks = gpmi_clks_for_mx6,
127 .clks_count = ARRAY_SIZE(gpmi_clks_for_mx6),
130 static irqreturn_t bch_irq(int irq, void *cookie)
132 struct gpmi_nand_data *this = cookie;
134 gpmi_clear_bch(this);
135 complete(&this->bch_done);
140 * Calculate the ECC strength by hand:
141 * E : The ECC strength.
142 * G : the length of Galois Field.
143 * N : The chunk count of per page.
144 * O : the oobsize of the NAND chip.
145 * M : the metasize of per page.
149 * ------------ <= (O - M)
157 static inline int get_ecc_strength(struct gpmi_nand_data *this)
159 struct bch_geometry *geo = &this->bch_geometry;
160 struct mtd_info *mtd = nand_to_mtd(&this->nand);
163 ecc_strength = ((mtd->oobsize - geo->metadata_size) * 8)
164 / (geo->gf_len * geo->ecc_chunk_count);
166 /* We need the minor even number. */
167 return round_down(ecc_strength, 2);
170 static inline bool gpmi_check_ecc(struct gpmi_nand_data *this)
172 struct bch_geometry *geo = &this->bch_geometry;
174 /* Do the sanity check. */
175 if (GPMI_IS_MX23(this) || GPMI_IS_MX28(this)) {
176 /* The mx23/mx28 only support the GF13. */
177 if (geo->gf_len == 14)
180 return geo->ecc_strength <= this->devdata->bch_max_ecc_strength;
184 * If we can get the ECC information from the nand chip, we do not
185 * need to calculate them ourselves.
187 * We may have available oob space in this case.
189 static int set_geometry_by_ecc_info(struct gpmi_nand_data *this)
191 struct bch_geometry *geo = &this->bch_geometry;
192 struct nand_chip *chip = &this->nand;
193 struct mtd_info *mtd = nand_to_mtd(chip);
194 unsigned int block_mark_bit_offset;
196 if (!(chip->ecc_strength_ds > 0 && chip->ecc_step_ds > 0))
199 switch (chip->ecc_step_ds) {
208 "unsupported nand chip. ecc bits : %d, ecc size : %d\n",
209 chip->ecc_strength_ds, chip->ecc_step_ds);
212 geo->ecc_chunk_size = chip->ecc_step_ds;
213 geo->ecc_strength = round_up(chip->ecc_strength_ds, 2);
214 if (!gpmi_check_ecc(this))
217 /* Keep the C >= O */
218 if (geo->ecc_chunk_size < mtd->oobsize) {
220 "unsupported nand chip. ecc size: %d, oob size : %d\n",
221 chip->ecc_step_ds, mtd->oobsize);
225 /* The default value, see comment in the legacy_set_geometry(). */
226 geo->metadata_size = 10;
228 geo->ecc_chunk_count = mtd->writesize / geo->ecc_chunk_size;
231 * Now, the NAND chip with 2K page(data chunk is 512byte) shows below:
234 * |<----------------------------------------------------->|
238 * |<-------------------------------------------->| D | | O' |
241 * +---+----------+-+----------+-+----------+-+----------+-+-----+
242 * | M | data |E| data |E| data |E| data |E| |
243 * +---+----------+-+----------+-+----------+-+----------+-+-----+
249 * P : the page size for BCH module.
250 * E : The ECC strength.
251 * G : the length of Galois Field.
252 * N : The chunk count of per page.
253 * M : the metasize of per page.
254 * C : the ecc chunk size, aka the "data" above.
255 * P': the nand chip's page size.
256 * O : the nand chip's oob size.
259 * The formula for P is :
262 * P = ------------ + P' + M
265 * The position of block mark moves forward in the ECC-based view
266 * of page, and the delta is:
269 * D = (---------------- + M)
272 * Please see the comment in legacy_set_geometry().
273 * With the condition C >= O , we still can get same result.
274 * So the bit position of the physical block mark within the ECC-based
275 * view of the page is :
278 geo->page_size = mtd->writesize + geo->metadata_size +
279 (geo->gf_len * geo->ecc_strength * geo->ecc_chunk_count) / 8;
281 geo->payload_size = mtd->writesize;
283 geo->auxiliary_status_offset = ALIGN(geo->metadata_size, 4);
284 geo->auxiliary_size = ALIGN(geo->metadata_size, 4)
285 + ALIGN(geo->ecc_chunk_count, 4);
287 if (!this->swap_block_mark)
291 block_mark_bit_offset = mtd->writesize * 8 -
292 (geo->ecc_strength * geo->gf_len * (geo->ecc_chunk_count - 1)
293 + geo->metadata_size * 8);
295 geo->block_mark_byte_offset = block_mark_bit_offset / 8;
296 geo->block_mark_bit_offset = block_mark_bit_offset % 8;
300 static int legacy_set_geometry(struct gpmi_nand_data *this)
302 struct bch_geometry *geo = &this->bch_geometry;
303 struct mtd_info *mtd = nand_to_mtd(&this->nand);
304 unsigned int metadata_size;
305 unsigned int status_size;
306 unsigned int block_mark_bit_offset;
309 * The size of the metadata can be changed, though we set it to 10
310 * bytes now. But it can't be too large, because we have to save
311 * enough space for BCH.
313 geo->metadata_size = 10;
315 /* The default for the length of Galois Field. */
318 /* The default for chunk size. */
319 geo->ecc_chunk_size = 512;
320 while (geo->ecc_chunk_size < mtd->oobsize) {
321 geo->ecc_chunk_size *= 2; /* keep C >= O */
325 geo->ecc_chunk_count = mtd->writesize / geo->ecc_chunk_size;
327 /* We use the same ECC strength for all chunks. */
328 geo->ecc_strength = get_ecc_strength(this);
329 if (!gpmi_check_ecc(this)) {
331 "ecc strength: %d cannot be supported by the controller (%d)\n"
332 "try to use minimum ecc strength that NAND chip required\n",
334 this->devdata->bch_max_ecc_strength);
338 geo->page_size = mtd->writesize + geo->metadata_size +
339 (geo->gf_len * geo->ecc_strength * geo->ecc_chunk_count) / 8;
340 geo->payload_size = mtd->writesize;
343 * The auxiliary buffer contains the metadata and the ECC status. The
344 * metadata is padded to the nearest 32-bit boundary. The ECC status
345 * contains one byte for every ECC chunk, and is also padded to the
346 * nearest 32-bit boundary.
348 metadata_size = ALIGN(geo->metadata_size, 4);
349 status_size = ALIGN(geo->ecc_chunk_count, 4);
351 geo->auxiliary_size = metadata_size + status_size;
352 geo->auxiliary_status_offset = metadata_size;
354 if (!this->swap_block_mark)
358 * We need to compute the byte and bit offsets of
359 * the physical block mark within the ECC-based view of the page.
361 * NAND chip with 2K page shows below:
367 * +---+----------+-+----------+-+----------+-+----------+-+
368 * | M | data |E| data |E| data |E| data |E|
369 * +---+----------+-+----------+-+----------+-+----------+-+
371 * The position of block mark moves forward in the ECC-based view
372 * of page, and the delta is:
375 * D = (---------------- + M)
378 * With the formula to compute the ECC strength, and the condition
379 * : C >= O (C is the ecc chunk size)
381 * It's easy to deduce to the following result:
383 * E * G (O - M) C - M C - M
384 * ----------- <= ------- <= -------- < ---------
390 * D = (---------------- + M) < C
393 * The above inequality means the position of block mark
394 * within the ECC-based view of the page is still in the data chunk,
395 * and it's NOT in the ECC bits of the chunk.
397 * Use the following to compute the bit position of the
398 * physical block mark within the ECC-based view of the page:
399 * (page_size - D) * 8
403 block_mark_bit_offset = mtd->writesize * 8 -
404 (geo->ecc_strength * geo->gf_len * (geo->ecc_chunk_count - 1)
405 + geo->metadata_size * 8);
407 geo->block_mark_byte_offset = block_mark_bit_offset / 8;
408 geo->block_mark_bit_offset = block_mark_bit_offset % 8;
412 int common_nfc_set_geometry(struct gpmi_nand_data *this)
414 if ((of_property_read_bool(this->dev->of_node, "fsl,use-minimum-ecc"))
415 || legacy_set_geometry(this))
416 return set_geometry_by_ecc_info(this);
421 struct dma_chan *get_dma_chan(struct gpmi_nand_data *this)
423 /* We use the DMA channel 0 to access all the nand chips. */
424 return this->dma_chans[0];
427 /* Can we use the upper's buffer directly for DMA? */
428 void prepare_data_dma(struct gpmi_nand_data *this, enum dma_data_direction dr)
430 struct scatterlist *sgl = &this->data_sgl;
433 /* first try to map the upper buffer directly */
434 if (virt_addr_valid(this->upper_buf) &&
435 !object_is_on_stack(this->upper_buf)) {
436 sg_init_one(sgl, this->upper_buf, this->upper_len);
437 ret = dma_map_sg(this->dev, sgl, 1, dr);
441 this->direct_dma_map_ok = true;
446 /* We have to use our own DMA buffer. */
447 sg_init_one(sgl, this->data_buffer_dma, this->upper_len);
449 if (dr == DMA_TO_DEVICE)
450 memcpy(this->data_buffer_dma, this->upper_buf, this->upper_len);
452 dma_map_sg(this->dev, sgl, 1, dr);
454 this->direct_dma_map_ok = false;
457 /* This will be called after the DMA operation is finished. */
458 static void dma_irq_callback(void *param)
460 struct gpmi_nand_data *this = param;
461 struct completion *dma_c = &this->dma_done;
463 switch (this->dma_type) {
464 case DMA_FOR_COMMAND:
465 dma_unmap_sg(this->dev, &this->cmd_sgl, 1, DMA_TO_DEVICE);
468 case DMA_FOR_READ_DATA:
469 dma_unmap_sg(this->dev, &this->data_sgl, 1, DMA_FROM_DEVICE);
470 if (this->direct_dma_map_ok == false)
471 memcpy(this->upper_buf, this->data_buffer_dma,
475 case DMA_FOR_WRITE_DATA:
476 dma_unmap_sg(this->dev, &this->data_sgl, 1, DMA_TO_DEVICE);
479 case DMA_FOR_READ_ECC_PAGE:
480 case DMA_FOR_WRITE_ECC_PAGE:
481 /* We have to wait the BCH interrupt to finish. */
485 dev_err(this->dev, "in wrong DMA operation.\n");
491 int start_dma_without_bch_irq(struct gpmi_nand_data *this,
492 struct dma_async_tx_descriptor *desc)
494 struct completion *dma_c = &this->dma_done;
495 unsigned long timeout;
497 init_completion(dma_c);
499 desc->callback = dma_irq_callback;
500 desc->callback_param = this;
501 dmaengine_submit(desc);
502 dma_async_issue_pending(get_dma_chan(this));
504 /* Wait for the interrupt from the DMA block. */
505 timeout = wait_for_completion_timeout(dma_c, msecs_to_jiffies(1000));
507 dev_err(this->dev, "DMA timeout, last DMA :%d\n",
508 this->last_dma_type);
509 gpmi_dump_info(this);
516 * This function is used in BCH reading or BCH writing pages.
517 * It will wait for the BCH interrupt as long as ONE second.
518 * Actually, we must wait for two interrupts :
519 * [1] firstly the DMA interrupt and
520 * [2] secondly the BCH interrupt.
522 int start_dma_with_bch_irq(struct gpmi_nand_data *this,
523 struct dma_async_tx_descriptor *desc)
525 struct completion *bch_c = &this->bch_done;
526 unsigned long timeout;
528 /* Prepare to receive an interrupt from the BCH block. */
529 init_completion(bch_c);
532 start_dma_without_bch_irq(this, desc);
534 /* Wait for the interrupt from the BCH block. */
535 timeout = wait_for_completion_timeout(bch_c, msecs_to_jiffies(1000));
537 dev_err(this->dev, "BCH timeout, last DMA :%d\n",
538 this->last_dma_type);
539 gpmi_dump_info(this);
545 static int acquire_register_block(struct gpmi_nand_data *this,
546 const char *res_name)
548 struct platform_device *pdev = this->pdev;
549 struct resources *res = &this->resources;
553 r = platform_get_resource_byname(pdev, IORESOURCE_MEM, res_name);
554 p = devm_ioremap_resource(&pdev->dev, r);
558 if (!strcmp(res_name, GPMI_NAND_GPMI_REGS_ADDR_RES_NAME))
560 else if (!strcmp(res_name, GPMI_NAND_BCH_REGS_ADDR_RES_NAME))
563 dev_err(this->dev, "unknown resource name : %s\n", res_name);
568 static int acquire_bch_irq(struct gpmi_nand_data *this, irq_handler_t irq_h)
570 struct platform_device *pdev = this->pdev;
571 const char *res_name = GPMI_NAND_BCH_INTERRUPT_RES_NAME;
575 r = platform_get_resource_byname(pdev, IORESOURCE_IRQ, res_name);
577 dev_err(this->dev, "Can't get resource for %s\n", res_name);
581 err = devm_request_irq(this->dev, r->start, irq_h, 0, res_name, this);
583 dev_err(this->dev, "error requesting BCH IRQ\n");
588 static void release_dma_channels(struct gpmi_nand_data *this)
591 for (i = 0; i < DMA_CHANS; i++)
592 if (this->dma_chans[i]) {
593 dma_release_channel(this->dma_chans[i]);
594 this->dma_chans[i] = NULL;
598 static int acquire_dma_channels(struct gpmi_nand_data *this)
600 struct platform_device *pdev = this->pdev;
601 struct dma_chan *dma_chan;
603 /* request dma channel */
604 dma_chan = dma_request_slave_channel(&pdev->dev, "rx-tx");
606 dev_err(this->dev, "Failed to request DMA channel.\n");
610 this->dma_chans[0] = dma_chan;
614 release_dma_channels(this);
618 static int gpmi_get_clks(struct gpmi_nand_data *this)
620 struct resources *r = &this->resources;
624 for (i = 0; i < this->devdata->clks_count; i++) {
625 clk = devm_clk_get(this->dev, this->devdata->clks[i]);
634 if (GPMI_IS_MX6(this))
636 * Set the default value for the gpmi clock.
638 * If you want to use the ONFI nand which is in the
639 * Synchronous Mode, you should change the clock as you need.
641 clk_set_rate(r->clock[0], 22000000);
646 dev_dbg(this->dev, "failed in finding the clocks.\n");
650 static int acquire_resources(struct gpmi_nand_data *this)
654 ret = acquire_register_block(this, GPMI_NAND_GPMI_REGS_ADDR_RES_NAME);
658 ret = acquire_register_block(this, GPMI_NAND_BCH_REGS_ADDR_RES_NAME);
662 ret = acquire_bch_irq(this, bch_irq);
666 ret = acquire_dma_channels(this);
670 ret = gpmi_get_clks(this);
676 release_dma_channels(this);
681 static void release_resources(struct gpmi_nand_data *this)
683 release_dma_channels(this);
686 static int init_hardware(struct gpmi_nand_data *this)
691 * This structure contains the "safe" GPMI timing that should succeed
692 * with any NAND Flash device
693 * (although, with less-than-optimal performance).
695 struct nand_timing safe_timing = {
696 .data_setup_in_ns = 80,
697 .data_hold_in_ns = 60,
698 .address_setup_in_ns = 25,
699 .gpmi_sample_delay_in_ns = 6,
705 /* Initialize the hardwares. */
706 ret = gpmi_init(this);
710 this->timing = safe_timing;
714 static int read_page_prepare(struct gpmi_nand_data *this,
715 void *destination, unsigned length,
716 void *alt_virt, dma_addr_t alt_phys, unsigned alt_size,
717 void **use_virt, dma_addr_t *use_phys)
719 struct device *dev = this->dev;
721 if (virt_addr_valid(destination)) {
722 dma_addr_t dest_phys;
724 dest_phys = dma_map_single(dev, destination,
725 length, DMA_FROM_DEVICE);
726 if (dma_mapping_error(dev, dest_phys)) {
727 if (alt_size < length) {
728 dev_err(dev, "Alternate buffer is too small\n");
733 *use_virt = destination;
734 *use_phys = dest_phys;
735 this->direct_dma_map_ok = true;
740 *use_virt = alt_virt;
741 *use_phys = alt_phys;
742 this->direct_dma_map_ok = false;
746 static inline void read_page_end(struct gpmi_nand_data *this,
747 void *destination, unsigned length,
748 void *alt_virt, dma_addr_t alt_phys, unsigned alt_size,
749 void *used_virt, dma_addr_t used_phys)
751 if (this->direct_dma_map_ok)
752 dma_unmap_single(this->dev, used_phys, length, DMA_FROM_DEVICE);
755 static inline void read_page_swap_end(struct gpmi_nand_data *this,
756 void *destination, unsigned length,
757 void *alt_virt, dma_addr_t alt_phys, unsigned alt_size,
758 void *used_virt, dma_addr_t used_phys)
760 if (!this->direct_dma_map_ok)
761 memcpy(destination, alt_virt, length);
764 static int send_page_prepare(struct gpmi_nand_data *this,
765 const void *source, unsigned length,
766 void *alt_virt, dma_addr_t alt_phys, unsigned alt_size,
767 const void **use_virt, dma_addr_t *use_phys)
769 struct device *dev = this->dev;
771 if (virt_addr_valid(source)) {
772 dma_addr_t source_phys;
774 source_phys = dma_map_single(dev, (void *)source, length,
776 if (dma_mapping_error(dev, source_phys)) {
777 if (alt_size < length) {
778 dev_err(dev, "Alternate buffer is too small\n");
784 *use_phys = source_phys;
789 * Copy the content of the source buffer into the alternate
790 * buffer and set up the return values accordingly.
792 memcpy(alt_virt, source, length);
794 *use_virt = alt_virt;
795 *use_phys = alt_phys;
799 static void send_page_end(struct gpmi_nand_data *this,
800 const void *source, unsigned length,
801 void *alt_virt, dma_addr_t alt_phys, unsigned alt_size,
802 const void *used_virt, dma_addr_t used_phys)
804 struct device *dev = this->dev;
805 if (used_virt == source)
806 dma_unmap_single(dev, used_phys, length, DMA_TO_DEVICE);
809 static void gpmi_free_dma_buffer(struct gpmi_nand_data *this)
811 struct device *dev = this->dev;
813 if (this->page_buffer_virt && virt_addr_valid(this->page_buffer_virt))
814 dma_free_coherent(dev, this->page_buffer_size,
815 this->page_buffer_virt,
816 this->page_buffer_phys);
817 kfree(this->cmd_buffer);
818 kfree(this->data_buffer_dma);
819 kfree(this->raw_buffer);
821 this->cmd_buffer = NULL;
822 this->data_buffer_dma = NULL;
823 this->raw_buffer = NULL;
824 this->page_buffer_virt = NULL;
825 this->page_buffer_size = 0;
828 /* Allocate the DMA buffers */
829 static int gpmi_alloc_dma_buffer(struct gpmi_nand_data *this)
831 struct bch_geometry *geo = &this->bch_geometry;
832 struct device *dev = this->dev;
833 struct mtd_info *mtd = nand_to_mtd(&this->nand);
835 /* [1] Allocate a command buffer. PAGE_SIZE is enough. */
836 this->cmd_buffer = kzalloc(PAGE_SIZE, GFP_DMA | GFP_KERNEL);
837 if (this->cmd_buffer == NULL)
841 * [2] Allocate a read/write data buffer.
842 * The gpmi_alloc_dma_buffer can be called twice.
843 * We allocate a PAGE_SIZE length buffer if gpmi_alloc_dma_buffer
844 * is called before the nand_scan_ident; and we allocate a buffer
845 * of the real NAND page size when the gpmi_alloc_dma_buffer is
846 * called after the nand_scan_ident.
848 this->data_buffer_dma = kzalloc(mtd->writesize ?: PAGE_SIZE,
849 GFP_DMA | GFP_KERNEL);
850 if (this->data_buffer_dma == NULL)
854 * [3] Allocate the page buffer.
856 * Both the payload buffer and the auxiliary buffer must appear on
857 * 32-bit boundaries. We presume the size of the payload buffer is a
858 * power of two and is much larger than four, which guarantees the
859 * auxiliary buffer will appear on a 32-bit boundary.
861 this->page_buffer_size = geo->payload_size + geo->auxiliary_size;
862 this->page_buffer_virt = dma_alloc_coherent(dev, this->page_buffer_size,
863 &this->page_buffer_phys, GFP_DMA);
864 if (!this->page_buffer_virt)
867 this->raw_buffer = kzalloc(mtd->writesize + mtd->oobsize, GFP_KERNEL);
868 if (!this->raw_buffer)
871 /* Slice up the page buffer. */
872 this->payload_virt = this->page_buffer_virt;
873 this->payload_phys = this->page_buffer_phys;
874 this->auxiliary_virt = this->payload_virt + geo->payload_size;
875 this->auxiliary_phys = this->payload_phys + geo->payload_size;
879 gpmi_free_dma_buffer(this);
883 static void gpmi_cmd_ctrl(struct mtd_info *mtd, int data, unsigned int ctrl)
885 struct nand_chip *chip = mtd_to_nand(mtd);
886 struct gpmi_nand_data *this = nand_get_controller_data(chip);
890 * Every operation begins with a command byte and a series of zero or
891 * more address bytes. These are distinguished by either the Address
892 * Latch Enable (ALE) or Command Latch Enable (CLE) signals being
893 * asserted. When MTD is ready to execute the command, it will deassert
894 * both latch enables.
896 * Rather than run a separate DMA operation for every single byte, we
897 * queue them up and run a single DMA operation for the entire series
898 * of command and data bytes. NAND_CMD_NONE means the END of the queue.
900 if ((ctrl & (NAND_ALE | NAND_CLE))) {
901 if (data != NAND_CMD_NONE)
902 this->cmd_buffer[this->command_length++] = data;
906 if (!this->command_length)
909 ret = gpmi_send_command(this);
911 dev_err(this->dev, "Chip: %u, Error %d\n",
912 this->current_chip, ret);
914 this->command_length = 0;
917 static int gpmi_dev_ready(struct mtd_info *mtd)
919 struct nand_chip *chip = mtd_to_nand(mtd);
920 struct gpmi_nand_data *this = nand_get_controller_data(chip);
922 return gpmi_is_ready(this, this->current_chip);
925 static void gpmi_select_chip(struct mtd_info *mtd, int chipnr)
927 struct nand_chip *chip = mtd_to_nand(mtd);
928 struct gpmi_nand_data *this = nand_get_controller_data(chip);
930 if ((this->current_chip < 0) && (chipnr >= 0))
932 else if ((this->current_chip >= 0) && (chipnr < 0))
935 this->current_chip = chipnr;
938 static void gpmi_read_buf(struct mtd_info *mtd, uint8_t *buf, int len)
940 struct nand_chip *chip = mtd_to_nand(mtd);
941 struct gpmi_nand_data *this = nand_get_controller_data(chip);
943 dev_dbg(this->dev, "len is %d\n", len);
944 this->upper_buf = buf;
945 this->upper_len = len;
947 gpmi_read_data(this);
950 static void gpmi_write_buf(struct mtd_info *mtd, const uint8_t *buf, int len)
952 struct nand_chip *chip = mtd_to_nand(mtd);
953 struct gpmi_nand_data *this = nand_get_controller_data(chip);
955 dev_dbg(this->dev, "len is %d\n", len);
956 this->upper_buf = (uint8_t *)buf;
957 this->upper_len = len;
959 gpmi_send_data(this);
962 static uint8_t gpmi_read_byte(struct mtd_info *mtd)
964 struct nand_chip *chip = mtd_to_nand(mtd);
965 struct gpmi_nand_data *this = nand_get_controller_data(chip);
966 uint8_t *buf = this->data_buffer_dma;
968 gpmi_read_buf(mtd, buf, 1);
973 * Handles block mark swapping.
974 * It can be called in swapping the block mark, or swapping it back,
975 * because the the operations are the same.
977 static void block_mark_swapping(struct gpmi_nand_data *this,
978 void *payload, void *auxiliary)
980 struct bch_geometry *nfc_geo = &this->bch_geometry;
985 unsigned char from_data;
986 unsigned char from_oob;
988 if (!this->swap_block_mark)
992 * If control arrives here, we're swapping. Make some convenience
995 bit = nfc_geo->block_mark_bit_offset;
996 p = payload + nfc_geo->block_mark_byte_offset;
1000 * Get the byte from the data area that overlays the block mark. Since
1001 * the ECC engine applies its own view to the bits in the page, the
1002 * physical block mark won't (in general) appear on a byte boundary in
1005 from_data = (p[0] >> bit) | (p[1] << (8 - bit));
1007 /* Get the byte from the OOB. */
1013 mask = (0x1 << bit) - 1;
1014 p[0] = (p[0] & mask) | (from_oob << bit);
1017 p[1] = (p[1] & mask) | (from_oob >> (8 - bit));
1020 static int gpmi_ecc_read_page(struct mtd_info *mtd, struct nand_chip *chip,
1021 uint8_t *buf, int oob_required, int page)
1023 struct gpmi_nand_data *this = nand_get_controller_data(chip);
1024 struct bch_geometry *nfc_geo = &this->bch_geometry;
1026 dma_addr_t payload_phys;
1027 void *auxiliary_virt;
1028 dma_addr_t auxiliary_phys;
1030 unsigned char *status;
1031 unsigned int max_bitflips = 0;
1034 dev_dbg(this->dev, "page number is : %d\n", page);
1035 ret = read_page_prepare(this, buf, nfc_geo->payload_size,
1036 this->payload_virt, this->payload_phys,
1037 nfc_geo->payload_size,
1038 &payload_virt, &payload_phys);
1040 dev_err(this->dev, "Inadequate DMA buffer\n");
1044 auxiliary_virt = this->auxiliary_virt;
1045 auxiliary_phys = this->auxiliary_phys;
1048 ret = gpmi_read_page(this, payload_phys, auxiliary_phys);
1049 read_page_end(this, buf, nfc_geo->payload_size,
1050 this->payload_virt, this->payload_phys,
1051 nfc_geo->payload_size,
1052 payload_virt, payload_phys);
1054 dev_err(this->dev, "Error in ECC-based read: %d\n", ret);
1058 /* handle the block mark swapping */
1059 block_mark_swapping(this, payload_virt, auxiliary_virt);
1061 /* Loop over status bytes, accumulating ECC status. */
1062 status = auxiliary_virt + nfc_geo->auxiliary_status_offset;
1064 read_page_swap_end(this, buf, nfc_geo->payload_size,
1065 this->payload_virt, this->payload_phys,
1066 nfc_geo->payload_size,
1067 payload_virt, payload_phys);
1069 for (i = 0; i < nfc_geo->ecc_chunk_count; i++, status++) {
1070 if ((*status == STATUS_GOOD) || (*status == STATUS_ERASED))
1073 if (*status == STATUS_UNCORRECTABLE) {
1074 int eccbits = nfc_geo->ecc_strength * nfc_geo->gf_len;
1075 u8 *eccbuf = this->raw_buffer;
1076 int offset, bitoffset;
1080 /* Read ECC bytes into our internal raw_buffer */
1081 offset = nfc_geo->metadata_size * 8;
1082 offset += ((8 * nfc_geo->ecc_chunk_size) + eccbits) * (i + 1);
1084 bitoffset = offset % 8;
1085 eccbytes = DIV_ROUND_UP(offset + eccbits, 8);
1088 chip->cmdfunc(mtd, NAND_CMD_RNDOUT, offset, -1);
1089 chip->read_buf(mtd, eccbuf, eccbytes);
1092 * ECC data are not byte aligned and we may have
1093 * in-band data in the first and last byte of
1094 * eccbuf. Set non-eccbits to one so that
1095 * nand_check_erased_ecc_chunk() does not count them
1099 eccbuf[0] |= GENMASK(bitoffset - 1, 0);
1101 bitoffset = (bitoffset + eccbits) % 8;
1103 eccbuf[eccbytes - 1] |= GENMASK(7, bitoffset);
1106 * The ECC hardware has an uncorrectable ECC status
1107 * code in case we have bitflips in an erased page. As
1108 * nothing was written into this subpage the ECC is
1109 * obviously wrong and we can not trust it. We assume
1110 * at this point that we are reading an erased page and
1111 * try to correct the bitflips in buffer up to
1112 * ecc_strength bitflips. If this is a page with random
1113 * data, we exceed this number of bitflips and have a
1114 * ECC failure. Otherwise we use the corrected buffer.
1117 /* The first block includes metadata */
1118 flips = nand_check_erased_ecc_chunk(
1119 buf + i * nfc_geo->ecc_chunk_size,
1120 nfc_geo->ecc_chunk_size,
1123 nfc_geo->metadata_size,
1124 nfc_geo->ecc_strength);
1126 flips = nand_check_erased_ecc_chunk(
1127 buf + i * nfc_geo->ecc_chunk_size,
1128 nfc_geo->ecc_chunk_size,
1131 nfc_geo->ecc_strength);
1135 max_bitflips = max_t(unsigned int, max_bitflips,
1137 mtd->ecc_stats.corrected += flips;
1141 mtd->ecc_stats.failed++;
1145 mtd->ecc_stats.corrected += *status;
1146 max_bitflips = max_t(unsigned int, max_bitflips, *status);
1151 * It's time to deliver the OOB bytes. See gpmi_ecc_read_oob()
1152 * for details about our policy for delivering the OOB.
1154 * We fill the caller's buffer with set bits, and then copy the
1155 * block mark to th caller's buffer. Note that, if block mark
1156 * swapping was necessary, it has already been done, so we can
1157 * rely on the first byte of the auxiliary buffer to contain
1160 memset(chip->oob_poi, ~0, mtd->oobsize);
1161 chip->oob_poi[0] = ((uint8_t *) auxiliary_virt)[0];
1164 return max_bitflips;
1167 /* Fake a virtual small page for the subpage read */
1168 static int gpmi_ecc_read_subpage(struct mtd_info *mtd, struct nand_chip *chip,
1169 uint32_t offs, uint32_t len, uint8_t *buf, int page)
1171 struct gpmi_nand_data *this = nand_get_controller_data(chip);
1172 void __iomem *bch_regs = this->resources.bch_regs;
1173 struct bch_geometry old_geo = this->bch_geometry;
1174 struct bch_geometry *geo = &this->bch_geometry;
1175 int size = chip->ecc.size; /* ECC chunk size */
1176 int meta, n, page_size;
1177 u32 r1_old, r2_old, r1_new, r2_new;
1178 unsigned int max_bitflips;
1179 int first, last, marker_pos;
1180 int ecc_parity_size;
1182 int old_swap_block_mark = this->swap_block_mark;
1184 /* The size of ECC parity */
1185 ecc_parity_size = geo->gf_len * geo->ecc_strength / 8;
1187 /* Align it with the chunk size */
1188 first = offs / size;
1189 last = (offs + len - 1) / size;
1191 if (this->swap_block_mark) {
1193 * Find the chunk which contains the Block Marker.
1194 * If this chunk is in the range of [first, last],
1195 * we have to read out the whole page.
1196 * Why? since we had swapped the data at the position of Block
1197 * Marker to the metadata which is bound with the chunk 0.
1199 marker_pos = geo->block_mark_byte_offset / size;
1200 if (last >= marker_pos && first <= marker_pos) {
1202 "page:%d, first:%d, last:%d, marker at:%d\n",
1203 page, first, last, marker_pos);
1204 return gpmi_ecc_read_page(mtd, chip, buf, 0, page);
1208 meta = geo->metadata_size;
1210 col = meta + (size + ecc_parity_size) * first;
1211 chip->cmdfunc(mtd, NAND_CMD_RNDOUT, col, -1);
1214 buf = buf + first * size;
1217 /* Save the old environment */
1218 r1_old = r1_new = readl(bch_regs + HW_BCH_FLASH0LAYOUT0);
1219 r2_old = r2_new = readl(bch_regs + HW_BCH_FLASH0LAYOUT1);
1221 /* change the BCH registers and bch_geometry{} */
1222 n = last - first + 1;
1223 page_size = meta + (size + ecc_parity_size) * n;
1225 r1_new &= ~(BM_BCH_FLASH0LAYOUT0_NBLOCKS |
1226 BM_BCH_FLASH0LAYOUT0_META_SIZE);
1227 r1_new |= BF_BCH_FLASH0LAYOUT0_NBLOCKS(n - 1)
1228 | BF_BCH_FLASH0LAYOUT0_META_SIZE(meta);
1229 writel(r1_new, bch_regs + HW_BCH_FLASH0LAYOUT0);
1231 r2_new &= ~BM_BCH_FLASH0LAYOUT1_PAGE_SIZE;
1232 r2_new |= BF_BCH_FLASH0LAYOUT1_PAGE_SIZE(page_size);
1233 writel(r2_new, bch_regs + HW_BCH_FLASH0LAYOUT1);
1235 geo->ecc_chunk_count = n;
1236 geo->payload_size = n * size;
1237 geo->page_size = page_size;
1238 geo->auxiliary_status_offset = ALIGN(meta, 4);
1240 dev_dbg(this->dev, "page:%d(%d:%d)%d, chunk:(%d:%d), BCH PG size:%d\n",
1241 page, offs, len, col, first, n, page_size);
1243 /* Read the subpage now */
1244 this->swap_block_mark = false;
1245 max_bitflips = gpmi_ecc_read_page(mtd, chip, buf, 0, page);
1248 writel(r1_old, bch_regs + HW_BCH_FLASH0LAYOUT0);
1249 writel(r2_old, bch_regs + HW_BCH_FLASH0LAYOUT1);
1250 this->bch_geometry = old_geo;
1251 this->swap_block_mark = old_swap_block_mark;
1253 return max_bitflips;
1256 static int gpmi_ecc_write_page(struct mtd_info *mtd, struct nand_chip *chip,
1257 const uint8_t *buf, int oob_required, int page)
1259 struct gpmi_nand_data *this = nand_get_controller_data(chip);
1260 struct bch_geometry *nfc_geo = &this->bch_geometry;
1261 const void *payload_virt;
1262 dma_addr_t payload_phys;
1263 const void *auxiliary_virt;
1264 dma_addr_t auxiliary_phys;
1267 dev_dbg(this->dev, "ecc write page.\n");
1268 if (this->swap_block_mark) {
1270 * If control arrives here, we're doing block mark swapping.
1271 * Since we can't modify the caller's buffers, we must copy them
1274 memcpy(this->payload_virt, buf, mtd->writesize);
1275 payload_virt = this->payload_virt;
1276 payload_phys = this->payload_phys;
1278 memcpy(this->auxiliary_virt, chip->oob_poi,
1279 nfc_geo->auxiliary_size);
1280 auxiliary_virt = this->auxiliary_virt;
1281 auxiliary_phys = this->auxiliary_phys;
1283 /* Handle block mark swapping. */
1284 block_mark_swapping(this,
1285 (void *)payload_virt, (void *)auxiliary_virt);
1288 * If control arrives here, we're not doing block mark swapping,
1289 * so we can to try and use the caller's buffers.
1291 ret = send_page_prepare(this,
1292 buf, mtd->writesize,
1293 this->payload_virt, this->payload_phys,
1294 nfc_geo->payload_size,
1295 &payload_virt, &payload_phys);
1297 dev_err(this->dev, "Inadequate payload DMA buffer\n");
1301 ret = send_page_prepare(this,
1302 chip->oob_poi, mtd->oobsize,
1303 this->auxiliary_virt, this->auxiliary_phys,
1304 nfc_geo->auxiliary_size,
1305 &auxiliary_virt, &auxiliary_phys);
1307 dev_err(this->dev, "Inadequate auxiliary DMA buffer\n");
1308 goto exit_auxiliary;
1313 ret = gpmi_send_page(this, payload_phys, auxiliary_phys);
1315 dev_err(this->dev, "Error in ECC-based write: %d\n", ret);
1317 if (!this->swap_block_mark) {
1318 send_page_end(this, chip->oob_poi, mtd->oobsize,
1319 this->auxiliary_virt, this->auxiliary_phys,
1320 nfc_geo->auxiliary_size,
1321 auxiliary_virt, auxiliary_phys);
1323 send_page_end(this, buf, mtd->writesize,
1324 this->payload_virt, this->payload_phys,
1325 nfc_geo->payload_size,
1326 payload_virt, payload_phys);
1333 * There are several places in this driver where we have to handle the OOB and
1334 * block marks. This is the function where things are the most complicated, so
1335 * this is where we try to explain it all. All the other places refer back to
1338 * These are the rules, in order of decreasing importance:
1340 * 1) Nothing the caller does can be allowed to imperil the block mark.
1342 * 2) In read operations, the first byte of the OOB we return must reflect the
1343 * true state of the block mark, no matter where that block mark appears in
1344 * the physical page.
1346 * 3) ECC-based read operations return an OOB full of set bits (since we never
1347 * allow ECC-based writes to the OOB, it doesn't matter what ECC-based reads
1350 * 4) "Raw" read operations return a direct view of the physical bytes in the
1351 * page, using the conventional definition of which bytes are data and which
1352 * are OOB. This gives the caller a way to see the actual, physical bytes
1353 * in the page, without the distortions applied by our ECC engine.
1356 * What we do for this specific read operation depends on two questions:
1358 * 1) Are we doing a "raw" read, or an ECC-based read?
1360 * 2) Are we using block mark swapping or transcription?
1362 * There are four cases, illustrated by the following Karnaugh map:
1364 * | Raw | ECC-based |
1365 * -------------+-------------------------+-------------------------+
1366 * | Read the conventional | |
1367 * | OOB at the end of the | |
1368 * Swapping | page and return it. It | |
1369 * | contains exactly what | |
1370 * | we want. | Read the block mark and |
1371 * -------------+-------------------------+ return it in a buffer |
1372 * | Read the conventional | full of set bits. |
1373 * | OOB at the end of the | |
1374 * | page and also the block | |
1375 * Transcribing | mark in the metadata. | |
1376 * | Copy the block mark | |
1377 * | into the first byte of | |
1379 * -------------+-------------------------+-------------------------+
1381 * Note that we break rule #4 in the Transcribing/Raw case because we're not
1382 * giving an accurate view of the actual, physical bytes in the page (we're
1383 * overwriting the block mark). That's OK because it's more important to follow
1386 * It turns out that knowing whether we want an "ECC-based" or "raw" read is not
1387 * easy. When reading a page, for example, the NAND Flash MTD code calls our
1388 * ecc.read_page or ecc.read_page_raw function. Thus, the fact that MTD wants an
1389 * ECC-based or raw view of the page is implicit in which function it calls
1390 * (there is a similar pair of ECC-based/raw functions for writing).
1392 static int gpmi_ecc_read_oob(struct mtd_info *mtd, struct nand_chip *chip,
1395 struct gpmi_nand_data *this = nand_get_controller_data(chip);
1397 dev_dbg(this->dev, "page number is %d\n", page);
1398 /* clear the OOB buffer */
1399 memset(chip->oob_poi, ~0, mtd->oobsize);
1401 /* Read out the conventional OOB. */
1402 chip->cmdfunc(mtd, NAND_CMD_READ0, mtd->writesize, page);
1403 chip->read_buf(mtd, chip->oob_poi, mtd->oobsize);
1406 * Now, we want to make sure the block mark is correct. In the
1407 * non-transcribing case (!GPMI_IS_MX23()), we already have it.
1408 * Otherwise, we need to explicitly read it.
1410 if (GPMI_IS_MX23(this)) {
1411 /* Read the block mark into the first byte of the OOB buffer. */
1412 chip->cmdfunc(mtd, NAND_CMD_READ0, 0, page);
1413 chip->oob_poi[0] = chip->read_byte(mtd);
1420 gpmi_ecc_write_oob(struct mtd_info *mtd, struct nand_chip *chip, int page)
1422 struct mtd_oob_region of = { };
1425 /* Do we have available oob area? */
1426 mtd_ooblayout_free(mtd, 0, &of);
1430 if (!nand_is_slc(chip))
1433 chip->cmdfunc(mtd, NAND_CMD_SEQIN, mtd->writesize + of.offset, page);
1434 chip->write_buf(mtd, chip->oob_poi + of.offset, of.length);
1435 chip->cmdfunc(mtd, NAND_CMD_PAGEPROG, -1, -1);
1437 status = chip->waitfunc(mtd, chip);
1438 return status & NAND_STATUS_FAIL ? -EIO : 0;
1442 * This function reads a NAND page without involving the ECC engine (no HW
1444 * The tricky part in the GPMI/BCH controller is that it stores ECC bits
1445 * inline (interleaved with payload DATA), and do not align data chunk on
1447 * We thus need to take care moving the payload data and ECC bits stored in the
1448 * page into the provided buffers, which is why we're using gpmi_copy_bits.
1450 * See set_geometry_by_ecc_info inline comments to have a full description
1451 * of the layout used by the GPMI controller.
1453 static int gpmi_ecc_read_page_raw(struct mtd_info *mtd,
1454 struct nand_chip *chip, uint8_t *buf,
1455 int oob_required, int page)
1457 struct gpmi_nand_data *this = nand_get_controller_data(chip);
1458 struct bch_geometry *nfc_geo = &this->bch_geometry;
1459 int eccsize = nfc_geo->ecc_chunk_size;
1460 int eccbits = nfc_geo->ecc_strength * nfc_geo->gf_len;
1461 u8 *tmp_buf = this->raw_buffer;
1464 size_t oob_byte_off;
1465 uint8_t *oob = chip->oob_poi;
1468 chip->read_buf(mtd, tmp_buf,
1469 mtd->writesize + mtd->oobsize);
1472 * If required, swap the bad block marker and the data stored in the
1473 * metadata section, so that we don't wrongly consider a block as bad.
1475 * See the layout description for a detailed explanation on why this
1478 if (this->swap_block_mark) {
1479 u8 swap = tmp_buf[0];
1481 tmp_buf[0] = tmp_buf[mtd->writesize];
1482 tmp_buf[mtd->writesize] = swap;
1486 * Copy the metadata section into the oob buffer (this section is
1487 * guaranteed to be aligned on a byte boundary).
1490 memcpy(oob, tmp_buf, nfc_geo->metadata_size);
1492 oob_bit_off = nfc_geo->metadata_size * 8;
1493 src_bit_off = oob_bit_off;
1495 /* Extract interleaved payload data and ECC bits */
1496 for (step = 0; step < nfc_geo->ecc_chunk_count; step++) {
1498 gpmi_copy_bits(buf, step * eccsize * 8,
1499 tmp_buf, src_bit_off,
1501 src_bit_off += eccsize * 8;
1503 /* Align last ECC block to align a byte boundary */
1504 if (step == nfc_geo->ecc_chunk_count - 1 &&
1505 (oob_bit_off + eccbits) % 8)
1506 eccbits += 8 - ((oob_bit_off + eccbits) % 8);
1509 gpmi_copy_bits(oob, oob_bit_off,
1510 tmp_buf, src_bit_off,
1513 src_bit_off += eccbits;
1514 oob_bit_off += eccbits;
1518 oob_byte_off = oob_bit_off / 8;
1520 if (oob_byte_off < mtd->oobsize)
1521 memcpy(oob + oob_byte_off,
1522 tmp_buf + mtd->writesize + oob_byte_off,
1523 mtd->oobsize - oob_byte_off);
1530 * This function writes a NAND page without involving the ECC engine (no HW
1532 * The tricky part in the GPMI/BCH controller is that it stores ECC bits
1533 * inline (interleaved with payload DATA), and do not align data chunk on
1535 * We thus need to take care moving the OOB area at the right place in the
1536 * final page, which is why we're using gpmi_copy_bits.
1538 * See set_geometry_by_ecc_info inline comments to have a full description
1539 * of the layout used by the GPMI controller.
1541 static int gpmi_ecc_write_page_raw(struct mtd_info *mtd,
1542 struct nand_chip *chip,
1544 int oob_required, int page)
1546 struct gpmi_nand_data *this = nand_get_controller_data(chip);
1547 struct bch_geometry *nfc_geo = &this->bch_geometry;
1548 int eccsize = nfc_geo->ecc_chunk_size;
1549 int eccbits = nfc_geo->ecc_strength * nfc_geo->gf_len;
1550 u8 *tmp_buf = this->raw_buffer;
1551 uint8_t *oob = chip->oob_poi;
1554 size_t oob_byte_off;
1558 * Initialize all bits to 1 in case we don't have a buffer for the
1559 * payload or oob data in order to leave unspecified bits of data
1560 * to their initial state.
1562 if (!buf || !oob_required)
1563 memset(tmp_buf, 0xff, mtd->writesize + mtd->oobsize);
1566 * First copy the metadata section (stored in oob buffer) at the
1567 * beginning of the page, as imposed by the GPMI layout.
1569 memcpy(tmp_buf, oob, nfc_geo->metadata_size);
1570 oob_bit_off = nfc_geo->metadata_size * 8;
1571 dst_bit_off = oob_bit_off;
1573 /* Interleave payload data and ECC bits */
1574 for (step = 0; step < nfc_geo->ecc_chunk_count; step++) {
1576 gpmi_copy_bits(tmp_buf, dst_bit_off,
1577 buf, step * eccsize * 8, eccsize * 8);
1578 dst_bit_off += eccsize * 8;
1580 /* Align last ECC block to align a byte boundary */
1581 if (step == nfc_geo->ecc_chunk_count - 1 &&
1582 (oob_bit_off + eccbits) % 8)
1583 eccbits += 8 - ((oob_bit_off + eccbits) % 8);
1586 gpmi_copy_bits(tmp_buf, dst_bit_off,
1587 oob, oob_bit_off, eccbits);
1589 dst_bit_off += eccbits;
1590 oob_bit_off += eccbits;
1593 oob_byte_off = oob_bit_off / 8;
1595 if (oob_required && oob_byte_off < mtd->oobsize)
1596 memcpy(tmp_buf + mtd->writesize + oob_byte_off,
1597 oob + oob_byte_off, mtd->oobsize - oob_byte_off);
1600 * If required, swap the bad block marker and the first byte of the
1601 * metadata section, so that we don't modify the bad block marker.
1603 * See the layout description for a detailed explanation on why this
1606 if (this->swap_block_mark) {
1607 u8 swap = tmp_buf[0];
1609 tmp_buf[0] = tmp_buf[mtd->writesize];
1610 tmp_buf[mtd->writesize] = swap;
1613 chip->write_buf(mtd, tmp_buf, mtd->writesize + mtd->oobsize);
1618 static int gpmi_ecc_read_oob_raw(struct mtd_info *mtd, struct nand_chip *chip,
1621 chip->cmdfunc(mtd, NAND_CMD_READ0, 0, page);
1623 return gpmi_ecc_read_page_raw(mtd, chip, NULL, 1, page);
1626 static int gpmi_ecc_write_oob_raw(struct mtd_info *mtd, struct nand_chip *chip,
1629 chip->cmdfunc(mtd, NAND_CMD_SEQIN, 0, page);
1631 return gpmi_ecc_write_page_raw(mtd, chip, NULL, 1, page);
1634 static int gpmi_block_markbad(struct mtd_info *mtd, loff_t ofs)
1636 struct nand_chip *chip = mtd_to_nand(mtd);
1637 struct gpmi_nand_data *this = nand_get_controller_data(chip);
1639 uint8_t *block_mark;
1640 int column, page, status, chipnr;
1642 chipnr = (int)(ofs >> chip->chip_shift);
1643 chip->select_chip(mtd, chipnr);
1645 column = !GPMI_IS_MX23(this) ? mtd->writesize : 0;
1647 /* Write the block mark. */
1648 block_mark = this->data_buffer_dma;
1649 block_mark[0] = 0; /* bad block marker */
1651 /* Shift to get page */
1652 page = (int)(ofs >> chip->page_shift);
1654 chip->cmdfunc(mtd, NAND_CMD_SEQIN, column, page);
1655 chip->write_buf(mtd, block_mark, 1);
1656 chip->cmdfunc(mtd, NAND_CMD_PAGEPROG, -1, -1);
1658 status = chip->waitfunc(mtd, chip);
1659 if (status & NAND_STATUS_FAIL)
1662 chip->select_chip(mtd, -1);
1667 static int nand_boot_set_geometry(struct gpmi_nand_data *this)
1669 struct boot_rom_geometry *geometry = &this->rom_geometry;
1672 * Set the boot block stride size.
1674 * In principle, we should be reading this from the OTP bits, since
1675 * that's where the ROM is going to get it. In fact, we don't have any
1676 * way to read the OTP bits, so we go with the default and hope for the
1679 geometry->stride_size_in_pages = 64;
1682 * Set the search area stride exponent.
1684 * In principle, we should be reading this from the OTP bits, since
1685 * that's where the ROM is going to get it. In fact, we don't have any
1686 * way to read the OTP bits, so we go with the default and hope for the
1689 geometry->search_area_stride_exponent = 2;
1693 static const char *fingerprint = "STMP";
1694 static int mx23_check_transcription_stamp(struct gpmi_nand_data *this)
1696 struct boot_rom_geometry *rom_geo = &this->rom_geometry;
1697 struct device *dev = this->dev;
1698 struct nand_chip *chip = &this->nand;
1699 struct mtd_info *mtd = nand_to_mtd(chip);
1700 unsigned int search_area_size_in_strides;
1701 unsigned int stride;
1703 uint8_t *buffer = chip->buffers->databuf;
1704 int saved_chip_number;
1705 int found_an_ncb_fingerprint = false;
1707 /* Compute the number of strides in a search area. */
1708 search_area_size_in_strides = 1 << rom_geo->search_area_stride_exponent;
1710 saved_chip_number = this->current_chip;
1711 chip->select_chip(mtd, 0);
1714 * Loop through the first search area, looking for the NCB fingerprint.
1716 dev_dbg(dev, "Scanning for an NCB fingerprint...\n");
1718 for (stride = 0; stride < search_area_size_in_strides; stride++) {
1719 /* Compute the page addresses. */
1720 page = stride * rom_geo->stride_size_in_pages;
1722 dev_dbg(dev, "Looking for a fingerprint in page 0x%x\n", page);
1725 * Read the NCB fingerprint. The fingerprint is four bytes long
1726 * and starts in the 12th byte of the page.
1728 chip->cmdfunc(mtd, NAND_CMD_READ0, 12, page);
1729 chip->read_buf(mtd, buffer, strlen(fingerprint));
1731 /* Look for the fingerprint. */
1732 if (!memcmp(buffer, fingerprint, strlen(fingerprint))) {
1733 found_an_ncb_fingerprint = true;
1739 chip->select_chip(mtd, saved_chip_number);
1741 if (found_an_ncb_fingerprint)
1742 dev_dbg(dev, "\tFound a fingerprint\n");
1744 dev_dbg(dev, "\tNo fingerprint found\n");
1745 return found_an_ncb_fingerprint;
1748 /* Writes a transcription stamp. */
1749 static int mx23_write_transcription_stamp(struct gpmi_nand_data *this)
1751 struct device *dev = this->dev;
1752 struct boot_rom_geometry *rom_geo = &this->rom_geometry;
1753 struct nand_chip *chip = &this->nand;
1754 struct mtd_info *mtd = nand_to_mtd(chip);
1755 unsigned int block_size_in_pages;
1756 unsigned int search_area_size_in_strides;
1757 unsigned int search_area_size_in_pages;
1758 unsigned int search_area_size_in_blocks;
1760 unsigned int stride;
1762 uint8_t *buffer = chip->buffers->databuf;
1763 int saved_chip_number;
1766 /* Compute the search area geometry. */
1767 block_size_in_pages = mtd->erasesize / mtd->writesize;
1768 search_area_size_in_strides = 1 << rom_geo->search_area_stride_exponent;
1769 search_area_size_in_pages = search_area_size_in_strides *
1770 rom_geo->stride_size_in_pages;
1771 search_area_size_in_blocks =
1772 (search_area_size_in_pages + (block_size_in_pages - 1)) /
1773 block_size_in_pages;
1775 dev_dbg(dev, "Search Area Geometry :\n");
1776 dev_dbg(dev, "\tin Blocks : %u\n", search_area_size_in_blocks);
1777 dev_dbg(dev, "\tin Strides: %u\n", search_area_size_in_strides);
1778 dev_dbg(dev, "\tin Pages : %u\n", search_area_size_in_pages);
1780 /* Select chip 0. */
1781 saved_chip_number = this->current_chip;
1782 chip->select_chip(mtd, 0);
1784 /* Loop over blocks in the first search area, erasing them. */
1785 dev_dbg(dev, "Erasing the search area...\n");
1787 for (block = 0; block < search_area_size_in_blocks; block++) {
1788 /* Compute the page address. */
1789 page = block * block_size_in_pages;
1791 /* Erase this block. */
1792 dev_dbg(dev, "\tErasing block 0x%x\n", block);
1793 chip->cmdfunc(mtd, NAND_CMD_ERASE1, -1, page);
1794 chip->cmdfunc(mtd, NAND_CMD_ERASE2, -1, -1);
1796 /* Wait for the erase to finish. */
1797 status = chip->waitfunc(mtd, chip);
1798 if (status & NAND_STATUS_FAIL)
1799 dev_err(dev, "[%s] Erase failed.\n", __func__);
1802 /* Write the NCB fingerprint into the page buffer. */
1803 memset(buffer, ~0, mtd->writesize);
1804 memcpy(buffer + 12, fingerprint, strlen(fingerprint));
1806 /* Loop through the first search area, writing NCB fingerprints. */
1807 dev_dbg(dev, "Writing NCB fingerprints...\n");
1808 for (stride = 0; stride < search_area_size_in_strides; stride++) {
1809 /* Compute the page addresses. */
1810 page = stride * rom_geo->stride_size_in_pages;
1812 /* Write the first page of the current stride. */
1813 dev_dbg(dev, "Writing an NCB fingerprint in page 0x%x\n", page);
1814 chip->cmdfunc(mtd, NAND_CMD_SEQIN, 0x00, page);
1815 chip->ecc.write_page_raw(mtd, chip, buffer, 0, page);
1816 chip->cmdfunc(mtd, NAND_CMD_PAGEPROG, -1, -1);
1818 /* Wait for the write to finish. */
1819 status = chip->waitfunc(mtd, chip);
1820 if (status & NAND_STATUS_FAIL)
1821 dev_err(dev, "[%s] Write failed.\n", __func__);
1824 /* Deselect chip 0. */
1825 chip->select_chip(mtd, saved_chip_number);
1829 static int mx23_boot_init(struct gpmi_nand_data *this)
1831 struct device *dev = this->dev;
1832 struct nand_chip *chip = &this->nand;
1833 struct mtd_info *mtd = nand_to_mtd(chip);
1834 unsigned int block_count;
1843 * If control arrives here, we can't use block mark swapping, which
1844 * means we're forced to use transcription. First, scan for the
1845 * transcription stamp. If we find it, then we don't have to do
1846 * anything -- the block marks are already transcribed.
1848 if (mx23_check_transcription_stamp(this))
1852 * If control arrives here, we couldn't find a transcription stamp, so
1853 * so we presume the block marks are in the conventional location.
1855 dev_dbg(dev, "Transcribing bad block marks...\n");
1857 /* Compute the number of blocks in the entire medium. */
1858 block_count = chip->chipsize >> chip->phys_erase_shift;
1861 * Loop over all the blocks in the medium, transcribing block marks as
1864 for (block = 0; block < block_count; block++) {
1866 * Compute the chip, page and byte addresses for this block's
1867 * conventional mark.
1869 chipnr = block >> (chip->chip_shift - chip->phys_erase_shift);
1870 page = block << (chip->phys_erase_shift - chip->page_shift);
1871 byte = block << chip->phys_erase_shift;
1873 /* Send the command to read the conventional block mark. */
1874 chip->select_chip(mtd, chipnr);
1875 chip->cmdfunc(mtd, NAND_CMD_READ0, mtd->writesize, page);
1876 block_mark = chip->read_byte(mtd);
1877 chip->select_chip(mtd, -1);
1880 * Check if the block is marked bad. If so, we need to mark it
1881 * again, but this time the result will be a mark in the
1882 * location where we transcribe block marks.
1884 if (block_mark != 0xff) {
1885 dev_dbg(dev, "Transcribing mark in block %u\n", block);
1886 ret = chip->block_markbad(mtd, byte);
1889 "Failed to mark block bad with ret %d\n",
1894 /* Write the stamp that indicates we've transcribed the block marks. */
1895 mx23_write_transcription_stamp(this);
1899 static int nand_boot_init(struct gpmi_nand_data *this)
1901 nand_boot_set_geometry(this);
1903 /* This is ROM arch-specific initilization before the BBT scanning. */
1904 if (GPMI_IS_MX23(this))
1905 return mx23_boot_init(this);
1909 static int gpmi_set_geometry(struct gpmi_nand_data *this)
1913 /* Free the temporary DMA memory for reading ID. */
1914 gpmi_free_dma_buffer(this);
1916 /* Set up the NFC geometry which is used by BCH. */
1917 ret = bch_set_geometry(this);
1919 dev_err(this->dev, "Error setting BCH geometry : %d\n", ret);
1923 /* Alloc the new DMA buffers according to the pagesize and oobsize */
1924 return gpmi_alloc_dma_buffer(this);
1927 static void gpmi_nand_exit(struct gpmi_nand_data *this)
1929 nand_release(nand_to_mtd(&this->nand));
1930 gpmi_free_dma_buffer(this);
1933 static int gpmi_init_last(struct gpmi_nand_data *this)
1935 struct nand_chip *chip = &this->nand;
1936 struct mtd_info *mtd = nand_to_mtd(chip);
1937 struct nand_ecc_ctrl *ecc = &chip->ecc;
1938 struct bch_geometry *bch_geo = &this->bch_geometry;
1941 /* Set up the medium geometry */
1942 ret = gpmi_set_geometry(this);
1946 /* Init the nand_ecc_ctrl{} */
1947 ecc->read_page = gpmi_ecc_read_page;
1948 ecc->write_page = gpmi_ecc_write_page;
1949 ecc->read_oob = gpmi_ecc_read_oob;
1950 ecc->write_oob = gpmi_ecc_write_oob;
1951 ecc->read_page_raw = gpmi_ecc_read_page_raw;
1952 ecc->write_page_raw = gpmi_ecc_write_page_raw;
1953 ecc->read_oob_raw = gpmi_ecc_read_oob_raw;
1954 ecc->write_oob_raw = gpmi_ecc_write_oob_raw;
1955 ecc->mode = NAND_ECC_HW;
1956 ecc->size = bch_geo->ecc_chunk_size;
1957 ecc->strength = bch_geo->ecc_strength;
1958 mtd_set_ooblayout(mtd, &gpmi_ooblayout_ops);
1961 * We only enable the subpage read when:
1962 * (1) the chip is imx6, and
1963 * (2) the size of the ECC parity is byte aligned.
1965 if (GPMI_IS_MX6(this) &&
1966 ((bch_geo->gf_len * bch_geo->ecc_strength) % 8) == 0) {
1967 ecc->read_subpage = gpmi_ecc_read_subpage;
1968 chip->options |= NAND_SUBPAGE_READ;
1972 * Can we enable the extra features? such as EDO or Sync mode.
1974 * We do not check the return value now. That's means if we fail in
1975 * enable the extra features, we still can run in the normal way.
1977 gpmi_extra_init(this);
1982 static int gpmi_nand_init(struct gpmi_nand_data *this)
1984 struct nand_chip *chip = &this->nand;
1985 struct mtd_info *mtd = nand_to_mtd(chip);
1988 /* init current chip */
1989 this->current_chip = -1;
1991 /* init the MTD data structures */
1992 mtd->name = "gpmi-nand";
1993 mtd->dev.parent = this->dev;
1995 /* init the nand_chip{}, we don't support a 16-bit NAND Flash bus. */
1996 nand_set_controller_data(chip, this);
1997 nand_set_flash_node(chip, this->pdev->dev.of_node);
1998 chip->select_chip = gpmi_select_chip;
1999 chip->cmd_ctrl = gpmi_cmd_ctrl;
2000 chip->dev_ready = gpmi_dev_ready;
2001 chip->read_byte = gpmi_read_byte;
2002 chip->read_buf = gpmi_read_buf;
2003 chip->write_buf = gpmi_write_buf;
2004 chip->badblock_pattern = &gpmi_bbt_descr;
2005 chip->block_markbad = gpmi_block_markbad;
2006 chip->options |= NAND_NO_SUBPAGE_WRITE;
2008 /* Set up swap_block_mark, must be set before the gpmi_set_geometry() */
2009 this->swap_block_mark = !GPMI_IS_MX23(this);
2012 * Allocate a temporary DMA buffer for reading ID in the
2013 * nand_scan_ident().
2015 this->bch_geometry.payload_size = 1024;
2016 this->bch_geometry.auxiliary_size = 128;
2017 ret = gpmi_alloc_dma_buffer(this);
2021 ret = nand_scan_ident(mtd, GPMI_IS_MX6(this) ? 2 : 1, NULL);
2025 if (chip->bbt_options & NAND_BBT_USE_FLASH) {
2026 chip->bbt_options |= NAND_BBT_NO_OOB;
2028 if (of_property_read_bool(this->dev->of_node,
2029 "fsl,no-blockmark-swap"))
2030 this->swap_block_mark = false;
2032 dev_dbg(this->dev, "Blockmark swapping %sabled\n",
2033 this->swap_block_mark ? "en" : "dis");
2035 ret = gpmi_init_last(this);
2039 chip->options |= NAND_SKIP_BBTSCAN;
2040 ret = nand_scan_tail(mtd);
2044 ret = nand_boot_init(this);
2047 ret = chip->scan_bbt(mtd);
2051 ret = mtd_device_register(mtd, NULL, 0);
2057 gpmi_nand_exit(this);
2061 static const struct of_device_id gpmi_nand_id_table[] = {
2063 .compatible = "fsl,imx23-gpmi-nand",
2064 .data = &gpmi_devdata_imx23,
2066 .compatible = "fsl,imx28-gpmi-nand",
2067 .data = &gpmi_devdata_imx28,
2069 .compatible = "fsl,imx6q-gpmi-nand",
2070 .data = &gpmi_devdata_imx6q,
2072 .compatible = "fsl,imx6sx-gpmi-nand",
2073 .data = &gpmi_devdata_imx6sx,
2076 MODULE_DEVICE_TABLE(of, gpmi_nand_id_table);
2078 static int gpmi_nand_probe(struct platform_device *pdev)
2080 struct gpmi_nand_data *this;
2081 const struct of_device_id *of_id;
2084 this = devm_kzalloc(&pdev->dev, sizeof(*this), GFP_KERNEL);
2088 of_id = of_match_device(gpmi_nand_id_table, &pdev->dev);
2090 this->devdata = of_id->data;
2092 dev_err(&pdev->dev, "Failed to find the right device id.\n");
2096 platform_set_drvdata(pdev, this);
2098 this->dev = &pdev->dev;
2100 ret = acquire_resources(this);
2102 goto exit_acquire_resources;
2104 ret = init_hardware(this);
2108 ret = gpmi_nand_init(this);
2112 dev_info(this->dev, "driver registered.\n");
2117 release_resources(this);
2118 exit_acquire_resources:
2123 static int gpmi_nand_remove(struct platform_device *pdev)
2125 struct gpmi_nand_data *this = platform_get_drvdata(pdev);
2127 gpmi_nand_exit(this);
2128 release_resources(this);
2132 #ifdef CONFIG_PM_SLEEP
2133 static int gpmi_pm_suspend(struct device *dev)
2135 struct gpmi_nand_data *this = dev_get_drvdata(dev);
2137 release_dma_channels(this);
2141 static int gpmi_pm_resume(struct device *dev)
2143 struct gpmi_nand_data *this = dev_get_drvdata(dev);
2146 ret = acquire_dma_channels(this);
2150 /* re-init the GPMI registers */
2151 this->flags &= ~GPMI_TIMING_INIT_OK;
2152 ret = gpmi_init(this);
2154 dev_err(this->dev, "Error setting GPMI : %d\n", ret);
2158 /* re-init the BCH registers */
2159 ret = bch_set_geometry(this);
2161 dev_err(this->dev, "Error setting BCH : %d\n", ret);
2165 /* re-init others */
2166 gpmi_extra_init(this);
2170 #endif /* CONFIG_PM_SLEEP */
2172 static const struct dev_pm_ops gpmi_pm_ops = {
2173 SET_SYSTEM_SLEEP_PM_OPS(gpmi_pm_suspend, gpmi_pm_resume)
2176 static struct platform_driver gpmi_nand_driver = {
2178 .name = "gpmi-nand",
2180 .of_match_table = gpmi_nand_id_table,
2182 .probe = gpmi_nand_probe,
2183 .remove = gpmi_nand_remove,
2185 module_platform_driver(gpmi_nand_driver);
2187 MODULE_AUTHOR("Freescale Semiconductor, Inc.");
2188 MODULE_DESCRIPTION("i.MX GPMI NAND Flash Controller Driver");
2189 MODULE_LICENSE("GPL");