2 * Freescale i.MX28 APBH DMA driver
4 * Copyright (C) 2011 Marek Vasut <marek.vasut@gmail.com>
5 * on behalf of DENX Software Engineering GmbH
7 * Based on code from LTIB:
8 * Copyright (C) 2010 Freescale Semiconductor, Inc. All Rights Reserved.
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of the GNU General Public License as published by
12 * the Free Software Foundation; either version 2 of the License, or
13 * (at your option) any later version.
15 * This program is distributed in the hope that it will be useful,
16 * but WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 * GNU General Public License for more details.
20 * You should have received a copy of the GNU General Public License along
21 * with this program; if not, write to the Free Software Foundation, Inc.,
22 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
25 #include <linux/list.h>
29 #include <asm/errno.h>
31 #include <asm/arch/clock.h>
32 #include <asm/arch/imx-regs.h>
33 #include <asm/arch/regs-apbh.h>
34 #include <asm/arch/sys_proto.h>
35 #include <asm/arch/dma.h>
38 static inline u32 mxs_readl(void *addr,
39 const char *fn, int ln)
41 u32 val = readl(addr);
42 static void *last_addr;
45 if (addr != last_addr || last_val != val) {
46 printf("%s@%d: Read %08x from %p\n", fn, ln, val, addr);
53 static inline void mxs_writel(u32 val, void *addr,
54 const char *name, const char *fn, int ln)
57 printf("%s@%d: Writing %08x to %s[%p]...", fn, ln, val, name, addr);
59 printf("%s@%d: Writing %08x to %p...", fn, ln, val, addr);
62 printf(" result: %08x\n", readl(addr));
66 #define readl(a) mxs_readl(a, __func__, __LINE__)
69 #define writel(v, a) mxs_writel(v, a, #a, __func__, __LINE__)
72 #define pr_dma_flag(c,f) do { if ((c) & MXS_DMA_DESC_##f) printf("%s ", #f); } while (0)
73 static inline void dump_dma_desc(struct mxs_dma_desc *desc)
75 struct mxs_dma_cmd *cmd = &desc->cmd;
77 printf("DMA desc %p:\n", desc);
78 printf("NXT: %08lx\n", cmd->next);
79 printf("CMD: %08lx - ", cmd->data);
80 printf("CNT: %04lx ", (cmd->data & MXS_DMA_DESC_BYTES_MASK) >> MXS_DMA_DESC_BYTES_OFFSET);
81 printf("PIO: %ld ", (cmd->data & MXS_DMA_DESC_PIO_WORDS_MASK) >> MXS_DMA_DESC_PIO_WORDS_OFFSET);
82 pr_dma_flag(cmd->data, HALT_ON_TERMINATE);
83 pr_dma_flag(cmd->data, WAIT4END);
84 pr_dma_flag(cmd->data, DEC_SEM);
85 pr_dma_flag(cmd->data, NAND_WAIT_4_READY);
86 pr_dma_flag(cmd->data, NAND_LOCK);
87 pr_dma_flag(cmd->data, IRQ);
88 pr_dma_flag(cmd->data, CHAIN);
90 switch (cmd->data & MXS_DMA_DESC_COMMAND_MASK) {
91 case MXS_DMA_DESC_COMMAND_NO_DMAXFER:
94 case MXS_DMA_DESC_COMMAND_DMA_WRITE:
97 case MXS_DMA_DESC_COMMAND_DMA_READ:
100 case MXS_DMA_DESC_COMMAND_DMA_SENSE:
103 if (cmd->data & MXS_DMA_DESC_PIO_WORDS_MASK) {
104 int pio_words = (cmd->data & MXS_DMA_DESC_PIO_WORDS_MASK) >> MXS_DMA_DESC_PIO_WORDS_OFFSET;
108 for (i = 0; i < pio_words; i++) {
109 printf("%08lx ", cmd->pio_words[i]);
115 static struct mxs_dma_chan mxs_dma_channels[MXS_MAX_DMA_CHANNELS];
116 static struct apbh_regs *apbh_regs = (struct apbh_regs *)MXS_APBH_BASE;
119 * Test is the DMA channel is valid channel
121 int mxs_dma_validate_chan(int channel)
123 struct mxs_dma_chan *pchan;
125 if ((channel < 0) || (channel >= MXS_MAX_DMA_CHANNELS)) {
126 printf("Invalid DMA channel %d\n", channel);
130 pchan = mxs_dma_channels + channel;
131 if (!(pchan->flags & MXS_DMA_FLAGS_ALLOCATED)) {
132 printf("DMA channel %d not allocated\n", channel);
140 * Return the address of the command within a descriptor.
142 static unsigned int mxs_dma_cmd_address(struct mxs_dma_desc *desc)
144 return desc->address + offsetof(struct mxs_dma_desc, cmd);
148 * Read a DMA channel's hardware semaphore.
150 * As used by the MXS platform's DMA software, the DMA channel's hardware
151 * semaphore reflects the number of DMA commands the hardware will process, but
152 * has not yet finished. This is a volatile value read directly from hardware,
153 * so it must be be viewed as immediately stale.
155 * If the channel is not marked busy, or has finished processing all its
156 * commands, this value should be zero.
158 * See mxs_dma_append() for details on how DMA command blocks must be configured
159 * to maintain the expected behavior of the semaphore's value.
161 static int mxs_dma_read_semaphore(int channel)
166 ret = mxs_dma_validate_chan(channel);
170 tmp = readl(&apbh_regs->ch[channel].hw_apbh_ch_sema);
172 tmp &= APBH_CHn_SEMA_PHORE_MASK;
173 tmp >>= APBH_CHn_SEMA_PHORE_OFFSET;
178 #ifndef CONFIG_SYS_DCACHE_OFF
179 void mxs_dma_flush_desc(struct mxs_dma_desc *desc)
184 addr = (uint32_t)desc;
185 size = roundup(sizeof(struct mxs_dma_desc), MXS_DMA_ALIGNMENT);
187 flush_dcache_range(addr, addr + size);
190 inline void mxs_dma_flush_desc(struct mxs_dma_desc *desc) {}
194 * Enable a DMA channel.
196 * If the given channel has any DMA descriptors on its active list, this
197 * function causes the DMA hardware to begin processing them.
199 * This function marks the DMA channel as "busy," whether or not there are any
200 * descriptors to process.
202 static int mxs_dma_enable(int channel)
205 struct mxs_dma_chan *pchan;
206 struct mxs_dma_desc *pdesc;
209 ret = mxs_dma_validate_chan(channel);
213 pchan = mxs_dma_channels + channel;
215 if (pchan->pending_num == 0) {
216 pchan->flags |= MXS_DMA_FLAGS_BUSY;
220 pdesc = list_first_entry(&pchan->active, struct mxs_dma_desc, node);
224 if (pchan->flags & MXS_DMA_FLAGS_BUSY) {
225 if (!(pdesc->cmd.data & MXS_DMA_DESC_CHAIN))
228 sem = mxs_dma_read_semaphore(channel);
233 pdesc = list_entry(pdesc->node.next,
234 struct mxs_dma_desc, node);
235 writel(mxs_dma_cmd_address(pdesc),
236 &apbh_regs->ch[channel].hw_apbh_ch_nxtcmdar);
238 writel(pchan->pending_num,
239 &apbh_regs->ch[channel].hw_apbh_ch_sema);
240 pchan->active_num += pchan->pending_num;
241 pchan->pending_num = 0;
243 pchan->active_num += pchan->pending_num;
244 pchan->pending_num = 0;
246 writel(1 << (channel + APBH_CTRL0_CLKGATE_CHANNEL_OFFSET),
247 &apbh_regs->hw_apbh_ctrl0_clr);
249 writel(mxs_dma_cmd_address(pdesc),
250 &apbh_regs->ch[channel].hw_apbh_ch_nxtcmdar);
251 writel(pchan->active_num,
252 &apbh_regs->ch[channel].hw_apbh_ch_sema);
255 pchan->flags |= MXS_DMA_FLAGS_BUSY;
260 * Disable a DMA channel.
262 * This function shuts down a DMA channel and marks it as "not busy." Any
263 * descriptors on the active list are immediately moved to the head of the
264 * "done" list, whether or not they have actually been processed by the
265 * hardware. The "ready" flags of these descriptors are NOT cleared, so they
266 * still appear to be active.
268 * This function immediately shuts down a DMA channel's hardware, aborting any
269 * I/O that may be in progress, potentially leaving I/O hardware in an undefined
270 * state. It is unwise to call this function if there is ANY chance the hardware
271 * is still processing a command.
273 static int mxs_dma_disable(int channel)
275 struct mxs_dma_chan *pchan;
278 ret = mxs_dma_validate_chan(channel);
282 pchan = mxs_dma_channels + channel;
284 if ((pchan->flags & MXS_DMA_FLAGS_BUSY)) {
285 printf("%s: DMA channel %d busy\n", __func__, channel);
289 writel(1 << (channel + APBH_CTRL0_CLKGATE_CHANNEL_OFFSET),
290 &apbh_regs->hw_apbh_ctrl0_set);
292 pchan->active_num = 0;
293 pchan->pending_num = 0;
294 list_splice_init(&pchan->active, &pchan->done);
300 * Resets the DMA channel hardware.
302 static int mxs_dma_reset(int channel)
306 ret = mxs_dma_validate_chan(channel);
310 writel(1 << (channel + APBH_CHANNEL_CTRL_RESET_CHANNEL_OFFSET),
311 &apbh_regs->hw_apbh_channel_ctrl_set);
317 * Enable or disable DMA interrupt.
319 * This function enables the given DMA channel to interrupt the CPU.
321 static int mxs_dma_enable_irq(int channel, int enable)
325 ret = mxs_dma_validate_chan(channel);
330 writel(1 << (channel + APBH_CTRL1_CH_CMDCMPLT_IRQ_EN_OFFSET),
331 &apbh_regs->hw_apbh_ctrl1_set);
333 writel(1 << (channel + APBH_CTRL1_CH_CMDCMPLT_IRQ_EN_OFFSET),
334 &apbh_regs->hw_apbh_ctrl1_clr);
340 * Clear DMA interrupt.
342 * The software that is using the DMA channel must register to receive its
343 * interrupts and, when they arrive, must call this function to clear them.
345 static int mxs_dma_ack_irq(int channel)
349 ret = mxs_dma_validate_chan(channel);
353 writel(1 << channel, &apbh_regs->hw_apbh_ctrl1_clr);
354 writel(1 << channel, &apbh_regs->hw_apbh_ctrl2_clr);
360 * Request to reserve a DMA channel
362 static int mxs_dma_request(int channel)
364 struct mxs_dma_chan *pchan;
366 if ((channel < 0) || (channel >= MXS_MAX_DMA_CHANNELS))
369 pchan = mxs_dma_channels + channel;
370 if ((pchan->flags & MXS_DMA_FLAGS_VALID) != MXS_DMA_FLAGS_VALID)
373 if (pchan->flags & MXS_DMA_FLAGS_ALLOCATED)
376 pchan->flags |= MXS_DMA_FLAGS_ALLOCATED;
377 pchan->active_num = 0;
378 pchan->pending_num = 0;
380 INIT_LIST_HEAD(&pchan->active);
381 INIT_LIST_HEAD(&pchan->done);
387 * Release a DMA channel.
389 * This function releases a DMA channel from its current owner.
391 * The channel will NOT be released if it's marked "busy" (see
394 int mxs_dma_release(int channel)
396 struct mxs_dma_chan *pchan;
399 ret = mxs_dma_validate_chan(channel);
403 pchan = mxs_dma_channels + channel;
405 if (pchan->flags & MXS_DMA_FLAGS_BUSY)
409 pchan->active_num = 0;
410 pchan->pending_num = 0;
411 pchan->flags &= ~MXS_DMA_FLAGS_ALLOCATED;
417 * Allocate DMA descriptor
419 struct mxs_dma_desc *mxs_dma_desc_alloc(void)
421 struct mxs_dma_desc *pdesc;
424 size = roundup(sizeof(struct mxs_dma_desc), MXS_DMA_ALIGNMENT);
425 pdesc = memalign(MXS_DMA_ALIGNMENT, size);
430 memset(pdesc, 0, sizeof(*pdesc));
431 pdesc->address = (dma_addr_t)pdesc;
437 * Free DMA descriptor
439 void mxs_dma_desc_free(struct mxs_dma_desc *pdesc)
448 * Add a DMA descriptor to a channel.
450 * If the descriptor list for this channel is not empty, this function sets the
451 * CHAIN bit and the NEXTCMD_ADDR fields in the last descriptor's DMA command so
452 * it will chain to the new descriptor's command.
454 * Then, this function marks the new descriptor as "ready," adds it to the end
455 * of the active descriptor list, and increments the count of pending
458 * The MXS platform DMA software imposes some rules on DMA commands to maintain
459 * important invariants. These rules are NOT checked, but they must be carefully
460 * applied by software that uses MXS DMA channels.
463 * The DMA channel's hardware semaphore must reflect the number of DMA
464 * commands the hardware will process, but has not yet finished.
467 * A DMA channel begins processing commands when its hardware semaphore is
468 * written with a value greater than zero, and it stops processing commands
469 * when the semaphore returns to zero.
471 * When a channel finishes a DMA command, it will decrement its semaphore if
472 * the DECREMENT_SEMAPHORE bit is set in that command's flags bits.
474 * In principle, it's not necessary for the DECREMENT_SEMAPHORE to be set,
475 * unless it suits the purposes of the software. For example, one could
476 * construct a series of five DMA commands, with the DECREMENT_SEMAPHORE
477 * bit set only in the last one. Then, setting the DMA channel's hardware
478 * semaphore to one would cause the entire series of five commands to be
479 * processed. However, this example would violate the invariant given above.
482 * ALL DMA commands MUST have the DECREMENT_SEMAPHORE bit set so that the DMA
483 * channel's hardware semaphore will be decremented EVERY time a command is
486 int mxs_dma_desc_append(int channel, struct mxs_dma_desc *pdesc)
488 struct mxs_dma_chan *pchan;
489 struct mxs_dma_desc *last;
492 ret = mxs_dma_validate_chan(channel);
496 pchan = mxs_dma_channels + channel;
498 pdesc->cmd.next = mxs_dma_cmd_address(pdesc);
499 pdesc->flags |= MXS_DMA_DESC_FIRST | MXS_DMA_DESC_LAST;
501 if (!list_empty(&pchan->active)) {
502 last = list_entry(pchan->active.prev, struct mxs_dma_desc,
505 pdesc->flags &= ~MXS_DMA_DESC_FIRST;
506 last->flags &= ~MXS_DMA_DESC_LAST;
508 last->cmd.next = mxs_dma_cmd_address(pdesc);
509 last->cmd.data |= MXS_DMA_DESC_CHAIN;
511 mxs_dma_flush_desc(last);
513 pdesc->flags |= MXS_DMA_DESC_READY;
514 if (pdesc->flags & MXS_DMA_DESC_FIRST)
515 pchan->pending_num++;
516 list_add_tail(&pdesc->node, &pchan->active);
518 mxs_dma_flush_desc(pdesc);
524 * Clean up processed DMA descriptors.
526 * This function removes processed DMA descriptors from the "active" list. Pass
527 * in a non-NULL list head to get the descriptors moved to your list. Pass NULL
528 * to get the descriptors moved to the channel's "done" list. Descriptors on
529 * the "done" list can be retrieved with mxs_dma_get_finished().
531 * This function marks the DMA channel as "not busy" if no unprocessed
532 * descriptors remain on the "active" list.
534 static int mxs_dma_finish(int channel, struct list_head *head)
537 struct mxs_dma_chan *pchan;
538 struct list_head *p, *q;
539 struct mxs_dma_desc *pdesc;
542 ret = mxs_dma_validate_chan(channel);
546 pchan = mxs_dma_channels + channel;
548 sem = mxs_dma_read_semaphore(channel);
552 if (sem == pchan->active_num)
555 list_for_each_safe(p, q, &pchan->active) {
556 if ((pchan->active_num) <= sem)
559 pdesc = list_entry(p, struct mxs_dma_desc, node);
560 pdesc->flags &= ~MXS_DMA_DESC_READY;
563 list_move_tail(p, head);
565 list_move_tail(p, &pchan->done);
567 if (pdesc->flags & MXS_DMA_DESC_LAST)
572 pchan->flags &= ~MXS_DMA_FLAGS_BUSY;
578 * Wait for DMA channel to complete
580 static int mxs_dma_wait_complete(uint32_t timeout, unsigned int chan)
584 ret = mxs_dma_validate_chan(chan);
588 if (mxs_wait_mask_set(&apbh_regs->hw_apbh_ctrl1_reg,
589 1 << chan, timeout)) {
598 * Execute the DMA channel
600 int mxs_dma_go(int chan)
602 uint32_t timeout = 10000;
605 LIST_HEAD(tmp_desc_list);
607 mxs_dma_enable_irq(chan, 1);
608 mxs_dma_enable(chan);
610 /* Wait for DMA to finish. */
611 ret = mxs_dma_wait_complete(timeout, chan);
613 /* Clear out the descriptors we just ran. */
614 mxs_dma_finish(chan, &tmp_desc_list);
616 /* Shut the DMA channel down. */
617 mxs_dma_ack_irq(chan);
619 mxs_dma_enable_irq(chan, 0);
620 mxs_dma_disable(chan);
626 * Initialize the DMA hardware
628 void mxs_dma_init(void)
630 mxs_reset_block(&apbh_regs->hw_apbh_ctrl0_reg);
632 #ifdef CONFIG_APBH_DMA_BURST8
633 writel(APBH_CTRL0_AHB_BURST8_EN,
634 &apbh_regs->hw_apbh_ctrl0_set);
636 writel(APBH_CTRL0_AHB_BURST8_EN,
637 &apbh_regs->hw_apbh_ctrl0_clr);
640 #ifdef CONFIG_APBH_DMA_BURST
641 writel(APBH_CTRL0_APB_BURST_EN,
642 &apbh_regs->hw_apbh_ctrl0_set);
644 writel(APBH_CTRL0_APB_BURST_EN,
645 &apbh_regs->hw_apbh_ctrl0_clr);
649 int mxs_dma_init_channel(int channel)
651 struct mxs_dma_chan *pchan;
654 pchan = mxs_dma_channels + channel;
655 pchan->flags = MXS_DMA_FLAGS_VALID;
657 ret = mxs_dma_request(channel);
660 printf("MXS DMA: Can't acquire DMA channel %i\n",
665 mxs_dma_reset(channel);
666 mxs_dma_ack_irq(channel);