drivers/dma/edma.c

   1 /*
   2  * TI EDMA DMA engine driver
   3  *
   4  * Copyright 2012 Texas Instruments
   5  *
   6  * This program is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU General Public License as
   8  * published by the Free Software Foundation version 2.
   9  *
  10  * This program is distributed "as is" WITHOUT ANY WARRANTY of any
  11  * kind, whether express or implied; without even the implied warranty
  12  * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  */
  15
  16 #include <linux/dmaengine.h>
  17 #include <linux/dma-mapping.h>
  18 #include <linux/edma.h>
  19 #include <linux/err.h>
  20 #include <linux/init.h>
  21 #include <linux/interrupt.h>
  22 #include <linux/list.h>
  23 #include <linux/module.h>
  24 #include <linux/platform_device.h>
  25 #include <linux/slab.h>
  26 #include <linux/spinlock.h>
  27 #include <linux/of.h>
  28
  29 #include <linux/platform_data/edma.h>
  30
  31 #include "dmaengine.h"
  32 #include "virt-dma.h"
  33
  34 /*
  35  * This will go away when the private EDMA API is folded
  36  * into this driver and the platform device(s) are
  37  * instantiated in the arch code. We can only get away
  38  * with this simplification because DA8XX may not be built
  39  * in the same kernel image with other DaVinci parts. This
  40  * avoids having to sprinkle dmaengine driver platform devices
  41  * and data throughout all the existing board files.
  42  */
  43 #ifdef CONFIG_ARCH_DAVINCI_DA8XX
  44 #define EDMA_CTLRS      2
  45 #define EDMA_CHANS      32
  46 #else
  47 #define EDMA_CTLRS      1
  48 #define EDMA_CHANS      64
  49 #endif /* CONFIG_ARCH_DAVINCI_DA8XX */
  50
  51 /*
  52  * Max of 20 segments per channel to conserve PaRAM slots
  53  * Also note that MAX_NR_SG should be atleast the no.of periods
  54  * that are required for ASoC, otherwise DMA prep calls will
  55  * fail. Today davinci-pcm is the only user of this driver and
  56  * requires atleast 17 slots, so we setup the default to 20.
  57  */
  58 #define MAX_NR_SG               20
  59 #define EDMA_MAX_SLOTS          MAX_NR_SG
  60 #define EDMA_DESCRIPTORS        16
  61
  62 struct edma_pset {
  63         u32                             len;
  64         dma_addr_t                      addr;
  65         struct edmacc_param             param;
  66 };
  67
  68 struct edma_desc {
  69         struct virt_dma_desc            vdesc;
  70         struct list_head                node;
  71         enum dma_transfer_direction     direction;
  72         int                             cyclic;
  73         int                             absync;
  74         int                             pset_nr;
  75         struct edma_chan                *echan;
  76         int                             processed;
  77
  78         /*
  79          * The following 4 elements are used for residue accounting.
  80          *
  81          * - processed_stat: the number of SG elements we have traversed
  82          * so far to cover accounting. This is updated directly to processed
  83          * during edma_callback and is always <= processed, because processed
  84          * refers to the number of pending transfer (programmed to EDMA
  85          * controller), where as processed_stat tracks number of transfers
  86          * accounted for so far.
  87          *
  88          * - residue: The amount of bytes we have left to transfer for this desc
  89          *
  90          * - residue_stat: The residue in bytes of data we have covered
  91          * so far for accounting. This is updated directly to residue
  92          * during callbacks to keep it current.
  93          *
  94          * - sg_len: Tracks the length of the current intermediate transfer,
  95          * this is required to update the residue during intermediate transfer
  96          * completion callback.
  97          */
  98         int                             processed_stat;
  99         u32                             sg_len;
 100         u32                             residue;
 101         u32                             residue_stat;
 102
 103         struct edma_pset                pset[0];
 104 };
 105
 106 struct edma_cc;
 107
 108 struct edma_chan {
 109         struct virt_dma_chan            vchan;
 110         struct list_head                node;
 111         struct edma_desc                *edesc;
 112         struct edma_cc                  *ecc;
 113         int                             ch_num;
 114         bool                            alloced;
 115         int                             slot[EDMA_MAX_SLOTS];
 116         int                             missed;
 117         struct dma_slave_config         cfg;
 118 };
 119
 120 struct edma_cc {
 121         int                             ctlr;
 122         struct dma_device               dma_slave;
 123         struct edma_chan                slave_chans[EDMA_CHANS];
 124         int                             num_slave_chans;
 125         int                             dummy_slot;
 126 };
 127
 128 static inline struct edma_cc *to_edma_cc(struct dma_device *d)
 129 {
 130         return container_of(d, struct edma_cc, dma_slave);
 131 }
 132
 133 static inline struct edma_chan *to_edma_chan(struct dma_chan *c)
 134 {
 135         return container_of(c, struct edma_chan, vchan.chan);
 136 }
 137
 138 static inline struct edma_desc
 139 *to_edma_desc(struct dma_async_tx_descriptor *tx)
 140 {
 141         return container_of(tx, struct edma_desc, vdesc.tx);
 142 }
 143
 144 static void edma_desc_free(struct virt_dma_desc *vdesc)
 145 {
 146         kfree(container_of(vdesc, struct edma_desc, vdesc));
 147 }
 148
 149 /* Dispatch a queued descriptor to the controller (caller holds lock) */
 150 static void edma_execute(struct edma_chan *echan)
 151 {
 152         struct virt_dma_desc *vdesc;
 153         struct edma_desc *edesc;
 154         struct device *dev = echan->vchan.chan.device->dev;
 155         int i, j, left, nslots;
 156
 157         /* If either we processed all psets or we're still not started */
 158         if (!echan->edesc ||
 159             echan->edesc->pset_nr == echan->edesc->processed) {
 160                 /* Get next vdesc */
 161                 vdesc = vchan_next_desc(&echan->vchan);
 162                 if (!vdesc) {
 163                         echan->edesc = NULL;
 164                         return;
 165                 }
 166                 list_del(&vdesc->node);
 167                 echan->edesc = to_edma_desc(&vdesc->tx);
 168         }
 169
 170         edesc = echan->edesc;
 171
 172         /* Find out how many left */
 173         left = edesc->pset_nr - edesc->processed;
 174         nslots = min(MAX_NR_SG, left);
 175         edesc->sg_len = 0;
 176
 177         /* Write descriptor PaRAM set(s) */
 178         for (i = 0; i < nslots; i++) {
 179                 j = i + edesc->processed;
 180                 edma_write_slot(echan->slot[i], &edesc->pset[j].param);
 181                 edesc->sg_len += edesc->pset[j].len;
 182                 dev_vdbg(echan->vchan.chan.device->dev,
 183                         "\n pset[%d]:\n"
 184                         "  chnum\t%d\n"
 185                         "  slot\t%d\n"
 186                         "  opt\t%08x\n"
 187                         "  src\t%08x\n"
 188                         "  dst\t%08x\n"
 189                         "  abcnt\t%08x\n"
 190                         "  ccnt\t%08x\n"
 191                         "  bidx\t%08x\n"
 192                         "  cidx\t%08x\n"
 193                         "  lkrld\t%08x\n",
 194                         j, echan->ch_num, echan->slot[i],
 195                         edesc->pset[j].param.opt,
 196                         edesc->pset[j].param.src,
 197                         edesc->pset[j].param.dst,
 198                         edesc->pset[j].param.a_b_cnt,
 199                         edesc->pset[j].param.ccnt,
 200                         edesc->pset[j].param.src_dst_bidx,
 201                         edesc->pset[j].param.src_dst_cidx,
 202                         edesc->pset[j].param.link_bcntrld);
 203                 /* Link to the previous slot if not the last set */
 204                 if (i != (nslots - 1))
 205                         edma_link(echan->slot[i], echan->slot[i+1]);
 206         }
 207
 208         edesc->processed += nslots;
 209
 210         /*
 211          * If this is either the last set in a set of SG-list transactions
 212          * then setup a link to the dummy slot, this results in all future
 213          * events being absorbed and that's OK because we're done
 214          */
 215         if (edesc->processed == edesc->pset_nr) {
 216                 if (edesc->cyclic)
 217                         edma_link(echan->slot[nslots-1], echan->slot[1]);
 218                 else
 219                         edma_link(echan->slot[nslots-1],
 220                                   echan->ecc->dummy_slot);
 221         }
 222
 223         if (edesc->processed <= MAX_NR_SG) {
 224                 dev_dbg(dev, "first transfer starting on channel %d\n",
 225                         echan->ch_num);
 226                 edma_start(echan->ch_num);
 227         } else {
 228                 dev_dbg(dev, "chan: %d: completed %d elements, resuming\n",
 229                         echan->ch_num, edesc->processed);
 230                 edma_resume(echan->ch_num);
 231         }
 232
 233         /*
 234          * This happens due to setup times between intermediate transfers
 235          * in long SG lists which have to be broken up into transfers of
 236          * MAX_NR_SG
 237          */
 238         if (echan->missed) {
 239                 dev_dbg(dev, "missed event on channel %d\n", echan->ch_num);
 240                 edma_clean_channel(echan->ch_num);
 241                 edma_stop(echan->ch_num);
 242                 edma_start(echan->ch_num);
 243                 edma_trigger_channel(echan->ch_num);
 244                 echan->missed = 0;
 245         }
 246 }
 247
 248 static int edma_terminate_all(struct dma_chan *chan)
 249 {
 250         struct edma_chan *echan = to_edma_chan(chan);
 251         unsigned long flags;
 252         LIST_HEAD(head);
 253
 254         spin_lock_irqsave(&echan->vchan.lock, flags);
 255
 256         /*
 257          * Stop DMA activity: we assume the callback will not be called
 258          * after edma_dma() returns (even if it does, it will see
 259          * echan->edesc is NULL and exit.)
 260          */
 261         if (echan->edesc) {
 262                 int cyclic = echan->edesc->cyclic;
 263                 echan->edesc = NULL;
 264                 edma_stop(echan->ch_num);
 265                 /* Move the cyclic channel back to default queue */
 266                 if (cyclic)
 267                         edma_assign_channel_eventq(echan->ch_num,
 268                                                    EVENTQ_DEFAULT);
 269         }
 270
 271         vchan_get_all_descriptors(&echan->vchan, &head);
 272         spin_unlock_irqrestore(&echan->vchan.lock, flags);
 273         vchan_dma_desc_free_list(&echan->vchan, &head);
 274
 275         return 0;
 276 }
 277
 278 static int edma_slave_config(struct dma_chan *chan,
 279         struct dma_slave_config *cfg)
 280 {
 281         struct edma_chan *echan = to_edma_chan(chan);
 282
 283         if (cfg->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES ||
 284             cfg->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)
 285                 return -EINVAL;
 286
 287         memcpy(&echan->cfg, cfg, sizeof(echan->cfg));
 288
 289         return 0;
 290 }
 291
 292 static int edma_dma_pause(struct dma_chan *chan)
 293 {
 294         struct edma_chan *echan = to_edma_chan(chan);
 295
 296         /* Pause/Resume only allowed with cyclic mode */
 297         if (!echan->edesc || !echan->edesc->cyclic)
 298                 return -EINVAL;
 299
 300         edma_pause(echan->ch_num);
 301         return 0;
 302 }
 303
 304 static int edma_dma_resume(struct dma_chan *chan)
 305 {
 306         struct edma_chan *echan = to_edma_chan(chan);
 307
 308         /* Pause/Resume only allowed with cyclic mode */
 309         if (!echan->edesc->cyclic)
 310                 return -EINVAL;
 311
 312         edma_resume(echan->ch_num);
 313         return 0;
 314 }
 315
 316 /*
 317  * A PaRAM set configuration abstraction used by other modes
 318  * @chan: Channel who's PaRAM set we're configuring
 319  * @pset: PaRAM set to initialize and setup.
 320  * @src_addr: Source address of the DMA
 321  * @dst_addr: Destination address of the DMA
 322  * @burst: In units of dev_width, how much to send
 323  * @dev_width: How much is the dev_width
 324  * @dma_length: Total length of the DMA transfer
 325  * @direction: Direction of the transfer
 326  */
 327 static int edma_config_pset(struct dma_chan *chan, struct edma_pset *epset,
 328         dma_addr_t src_addr, dma_addr_t dst_addr, u32 burst,
 329         enum dma_slave_buswidth dev_width, unsigned int dma_length,
 330         enum dma_transfer_direction direction)
 331 {
 332         struct edma_chan *echan = to_edma_chan(chan);
 333         struct device *dev = chan->device->dev;
 334         struct edmacc_param *param = &epset->param;
 335         int acnt, bcnt, ccnt, cidx;
 336         int src_bidx, dst_bidx, src_cidx, dst_cidx;
 337         int absync;
 338
 339         acnt = dev_width;
 340
 341         /* src/dst_maxburst == 0 is the same case as src/dst_maxburst == 1 */
 342         if (!burst)
 343                 burst = 1;
 344         /*
 345          * If the maxburst is equal to the fifo width, use
 346          * A-synced transfers. This allows for large contiguous
 347          * buffer transfers using only one PaRAM set.
 348          */
 349         if (burst == 1) {
 350                 /*
 351                  * For the A-sync case, bcnt and ccnt are the remainder
 352                  * and quotient respectively of the division of:
 353                  * (dma_length / acnt) by (SZ_64K -1). This is so
 354                  * that in case bcnt over flows, we have ccnt to use.
 355                  * Note: In A-sync tranfer only, bcntrld is used, but it
 356                  * only applies for sg_dma_len(sg) >= SZ_64K.
 357                  * In this case, the best way adopted is- bccnt for the
 358                  * first frame will be the remainder below. Then for
 359                  * every successive frame, bcnt will be SZ_64K-1. This
 360                  * is assured as bcntrld = 0xffff in end of function.
 361                  */
 362                 absync = false;
 363                 ccnt = dma_length / acnt / (SZ_64K - 1);
 364                 bcnt = dma_length / acnt - ccnt * (SZ_64K - 1);
 365                 /*
 366                  * If bcnt is non-zero, we have a remainder and hence an
 367                  * extra frame to transfer, so increment ccnt.
 368                  */
 369                 if (bcnt)
 370                         ccnt++;
 371                 else
 372                         bcnt = SZ_64K - 1;
 373                 cidx = acnt;
 374         } else {
 375                 /*
 376                  * If maxburst is greater than the fifo address_width,
 377                  * use AB-synced transfers where A count is the fifo
 378                  * address_width and B count is the maxburst. In this
 379                  * case, we are limited to transfers of C count frames
 380                  * of (address_width * maxburst) where C count is limited
 381                  * to SZ_64K-1. This places an upper bound on the length
 382                  * of an SG segment that can be handled.
 383                  */
 384                 absync = true;
 385                 bcnt = burst;
 386                 ccnt = dma_length / (acnt * bcnt);
 387                 if (ccnt > (SZ_64K - 1)) {
 388                         dev_err(dev, "Exceeded max SG segment size\n");
 389                         return -EINVAL;
 390                 }
 391                 cidx = acnt * bcnt;
 392         }
 393
 394         epset->len = dma_length;
 395
 396         if (direction == DMA_MEM_TO_DEV) {
 397                 src_bidx = acnt;
 398                 src_cidx = cidx;
 399                 dst_bidx = 0;
 400                 dst_cidx = 0;
 401                 epset->addr = src_addr;
 402         } else if (direction == DMA_DEV_TO_MEM)  {
 403                 src_bidx = 0;
 404                 src_cidx = 0;
 405                 dst_bidx = acnt;
 406                 dst_cidx = cidx;
 407                 epset->addr = dst_addr;
 408         } else if (direction == DMA_MEM_TO_MEM)  {
 409                 src_bidx = acnt;
 410                 src_cidx = cidx;
 411                 dst_bidx = acnt;
 412                 dst_cidx = cidx;
 413         } else {
 414                 dev_err(dev, "%s: direction not implemented yet\n", __func__);
 415                 return -EINVAL;
 416         }
 417
 418         param->opt = EDMA_TCC(EDMA_CHAN_SLOT(echan->ch_num));
 419         /* Configure A or AB synchronized transfers */
 420         if (absync)
 421                 param->opt |= SYNCDIM;
 422
 423         param->src = src_addr;
 424         param->dst = dst_addr;
 425
 426         param->src_dst_bidx = (dst_bidx << 16) | src_bidx;
 427         param->src_dst_cidx = (dst_cidx << 16) | src_cidx;
 428
 429         param->a_b_cnt = bcnt << 16 | acnt;
 430         param->ccnt = ccnt;
 431         /*
 432          * Only time when (bcntrld) auto reload is required is for
 433          * A-sync case, and in this case, a requirement of reload value
 434          * of SZ_64K-1 only is assured. 'link' is initially set to NULL
 435          * and then later will be populated by edma_execute.
 436          */
 437         param->link_bcntrld = 0xffffffff;
 438         return absync;
 439 }
 440
 441 static struct dma_async_tx_descriptor *edma_prep_slave_sg(
 442         struct dma_chan *chan, struct scatterlist *sgl,
 443         unsigned int sg_len, enum dma_transfer_direction direction,
 444         unsigned long tx_flags, void *context)
 445 {
 446         struct edma_chan *echan = to_edma_chan(chan);
 447         struct device *dev = chan->device->dev;
 448         struct edma_desc *edesc;
 449         dma_addr_t src_addr = 0, dst_addr = 0;
 450         enum dma_slave_buswidth dev_width;
 451         u32 burst;
 452         struct scatterlist *sg;
 453         int i, nslots, ret;
 454
 455         if (unlikely(!echan || !sgl || !sg_len))
 456                 return NULL;
 457
 458         if (direction == DMA_DEV_TO_MEM) {
 459                 src_addr = echan->cfg.src_addr;
 460                 dev_width = echan->cfg.src_addr_width;
 461                 burst = echan->cfg.src_maxburst;
 462         } else if (direction == DMA_MEM_TO_DEV) {
 463                 dst_addr = echan->cfg.dst_addr;
 464                 dev_width = echan->cfg.dst_addr_width;
 465                 burst = echan->cfg.dst_maxburst;
 466         } else {
 467                 dev_err(dev, "%s: bad direction: %d\n", __func__, direction);
 468                 return NULL;
 469         }
 470
 471         if (dev_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) {
 472                 dev_err(dev, "%s: Undefined slave buswidth\n", __func__);
 473                 return NULL;
 474         }
 475
 476         edesc = kzalloc(sizeof(*edesc) + sg_len *
 477                 sizeof(edesc->pset[0]), GFP_ATOMIC);
 478         if (!edesc) {
 479                 dev_err(dev, "%s: Failed to allocate a descriptor\n", __func__);
 480                 return NULL;
 481         }
 482
 483         edesc->pset_nr = sg_len;
 484         edesc->residue = 0;
 485         edesc->direction = direction;
 486         edesc->echan = echan;
 487
 488         /* Allocate a PaRAM slot, if needed */
 489         nslots = min_t(unsigned, MAX_NR_SG, sg_len);
 490
 491         for (i = 0; i < nslots; i++) {
 492                 if (echan->slot[i] < 0) {
 493                         echan->slot[i] =
 494                                 edma_alloc_slot(EDMA_CTLR(echan->ch_num),
 495                                                 EDMA_SLOT_ANY);
 496                         if (echan->slot[i] < 0) {
 497                                 kfree(edesc);
 498                                 dev_err(dev, "%s: Failed to allocate slot\n",
 499                                         __func__);
 500                                 return NULL;
 501                         }
 502                 }
 503         }
 504
 505         /* Configure PaRAM sets for each SG */
 506         for_each_sg(sgl, sg, sg_len, i) {
 507                 /* Get address for each SG */
 508                 if (direction == DMA_DEV_TO_MEM)
 509                         dst_addr = sg_dma_address(sg);
 510                 else
 511                         src_addr = sg_dma_address(sg);
 512
 513                 ret = edma_config_pset(chan, &edesc->pset[i], src_addr,
 514                                        dst_addr, burst, dev_width,
 515                                        sg_dma_len(sg), direction);
 516                 if (ret < 0) {
 517                         kfree(edesc);
 518                         return NULL;
 519                 }
 520
 521                 edesc->absync = ret;
 522                 edesc->residue += sg_dma_len(sg);
 523
 524                 /* If this is the last in a current SG set of transactions,
 525                    enable interrupts so that next set is processed */
 526                 if (!((i+1) % MAX_NR_SG))
 527                         edesc->pset[i].param.opt |= TCINTEN;
 528
 529                 /* If this is the last set, enable completion interrupt flag */
 530                 if (i == sg_len - 1)
 531                         edesc->pset[i].param.opt |= TCINTEN;
 532         }
 533         edesc->residue_stat = edesc->residue;
 534
 535         return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags);
 536 }
 537
 538 static struct dma_async_tx_descriptor *edma_prep_dma_memcpy(
 539         struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
 540         size_t len, unsigned long tx_flags)
 541 {
 542         int ret;
 543         struct edma_desc *edesc;
 544         struct device *dev = chan->device->dev;
 545         struct edma_chan *echan = to_edma_chan(chan);
 546
 547         if (unlikely(!echan || !len))
 548                 return NULL;
 549
 550         edesc = kzalloc(sizeof(*edesc) + sizeof(edesc->pset[0]), GFP_ATOMIC);
 551         if (!edesc) {
 552                 dev_dbg(dev, "Failed to allocate a descriptor\n");
 553                 return NULL;
 554         }
 555
 556         edesc->pset_nr = 1;
 557
 558         ret = edma_config_pset(chan, &edesc->pset[0], src, dest, 1,
 559                                DMA_SLAVE_BUSWIDTH_4_BYTES, len, DMA_MEM_TO_MEM);
 560         if (ret < 0)
 561                 return NULL;
 562
 563         edesc->absync = ret;
 564
 565         /*
 566          * Enable intermediate transfer chaining to re-trigger channel
 567          * on completion of every TR, and enable transfer-completion
 568          * interrupt on completion of the whole transfer.
 569          */
 570         edesc->pset[0].param.opt |= ITCCHEN;
 571         edesc->pset[0].param.opt |= TCINTEN;
 572
 573         return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags);
 574 }
 575
 576 static struct dma_async_tx_descriptor *edma_prep_dma_cyclic(
 577         struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
 578         size_t period_len, enum dma_transfer_direction direction,
 579         unsigned long tx_flags)
 580 {
 581         struct edma_chan *echan = to_edma_chan(chan);
 582         struct device *dev = chan->device->dev;
 583         struct edma_desc *edesc;
 584         dma_addr_t src_addr, dst_addr;
 585         enum dma_slave_buswidth dev_width;
 586         u32 burst;
 587         int i, ret, nslots;
 588
 589         if (unlikely(!echan || !buf_len || !period_len))
 590                 return NULL;
 591
 592         if (direction == DMA_DEV_TO_MEM) {
 593                 src_addr = echan->cfg.src_addr;
 594                 dst_addr = buf_addr;
 595                 dev_width = echan->cfg.src_addr_width;
 596                 burst = echan->cfg.src_maxburst;
 597         } else if (direction == DMA_MEM_TO_DEV) {
 598                 src_addr = buf_addr;
 599                 dst_addr = echan->cfg.dst_addr;
 600                 dev_width = echan->cfg.dst_addr_width;
 601                 burst = echan->cfg.dst_maxburst;
 602         } else {
 603                 dev_err(dev, "%s: bad direction: %d\n", __func__, direction);
 604                 return NULL;
 605         }
 606
 607         if (dev_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) {
 608                 dev_err(dev, "%s: Undefined slave buswidth\n", __func__);
 609                 return NULL;
 610         }
 611
 612         if (unlikely(buf_len % period_len)) {
 613                 dev_err(dev, "Period should be multiple of Buffer length\n");
 614                 return NULL;
 615         }
 616
 617         nslots = (buf_len / period_len) + 1;
 618
 619         /*
 620          * Cyclic DMA users such as audio cannot tolerate delays introduced
 621          * by cases where the number of periods is more than the maximum
 622          * number of SGs the EDMA driver can handle at a time. For DMA types
 623          * such as Slave SGs, such delays are tolerable and synchronized,
 624          * but the synchronization is difficult to achieve with Cyclic and
 625          * cannot be guaranteed, so we error out early.
 626          */
 627         if (nslots > MAX_NR_SG)
 628                 return NULL;
 629
 630         edesc = kzalloc(sizeof(*edesc) + nslots *
 631                 sizeof(edesc->pset[0]), GFP_ATOMIC);
 632         if (!edesc) {
 633                 dev_err(dev, "%s: Failed to allocate a descriptor\n", __func__);
 634                 return NULL;
 635         }
 636
 637         edesc->cyclic = 1;
 638         edesc->pset_nr = nslots;
 639         edesc->residue = edesc->residue_stat = buf_len;
 640         edesc->direction = direction;
 641         edesc->echan = echan;
 642
 643         dev_dbg(dev, "%s: channel=%d nslots=%d period_len=%zu buf_len=%zu\n",
 644                 __func__, echan->ch_num, nslots, period_len, buf_len);
 645
 646         for (i = 0; i < nslots; i++) {
 647                 /* Allocate a PaRAM slot, if needed */
 648                 if (echan->slot[i] < 0) {
 649                         echan->slot[i] =
 650                                 edma_alloc_slot(EDMA_CTLR(echan->ch_num),
 651                                                 EDMA_SLOT_ANY);
 652                         if (echan->slot[i] < 0) {
 653                                 kfree(edesc);
 654                                 dev_err(dev, "%s: Failed to allocate slot\n",
 655                                         __func__);
 656                                 return NULL;
 657                         }
 658                 }
 659
 660                 if (i == nslots - 1) {
 661                         memcpy(&edesc->pset[i], &edesc->pset[0],
 662                                sizeof(edesc->pset[0]));
 663                         break;
 664                 }
 665
 666                 ret = edma_config_pset(chan, &edesc->pset[i], src_addr,
 667                                        dst_addr, burst, dev_width, period_len,
 668                                        direction);
 669                 if (ret < 0) {
 670                         kfree(edesc);
 671                         return NULL;
 672                 }
 673
 674                 if (direction == DMA_DEV_TO_MEM)
 675                         dst_addr += period_len;
 676                 else
 677                         src_addr += period_len;
 678
 679                 dev_vdbg(dev, "%s: Configure period %d of buf:\n", __func__, i);
 680                 dev_vdbg(dev,
 681                         "\n pset[%d]:\n"
 682                         "  chnum\t%d\n"
 683                         "  slot\t%d\n"
 684                         "  opt\t%08x\n"
 685                         "  src\t%08x\n"
 686                         "  dst\t%08x\n"
 687                         "  abcnt\t%08x\n"
 688                         "  ccnt\t%08x\n"
 689                         "  bidx\t%08x\n"
 690                         "  cidx\t%08x\n"
 691                         "  lkrld\t%08x\n",
 692                         i, echan->ch_num, echan->slot[i],
 693                         edesc->pset[i].param.opt,
 694                         edesc->pset[i].param.src,
 695                         edesc->pset[i].param.dst,
 696                         edesc->pset[i].param.a_b_cnt,
 697                         edesc->pset[i].param.ccnt,
 698                         edesc->pset[i].param.src_dst_bidx,
 699                         edesc->pset[i].param.src_dst_cidx,
 700                         edesc->pset[i].param.link_bcntrld);
 701
 702                 edesc->absync = ret;
 703
 704                 /*
 705                  * Enable period interrupt only if it is requested
 706                  */
 707                 if (tx_flags & DMA_PREP_INTERRUPT)
 708                         edesc->pset[i].param.opt |= TCINTEN;
 709         }
 710
 711         /* Place the cyclic channel to highest priority queue */
 712         edma_assign_channel_eventq(echan->ch_num, EVENTQ_0);
 713
 714         return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags);
 715 }
 716
 717 static void edma_callback(unsigned ch_num, u16 ch_status, void *data)
 718 {
 719         struct edma_chan *echan = data;
 720         struct device *dev = echan->vchan.chan.device->dev;
 721         struct edma_desc *edesc;
 722         struct edmacc_param p;
 723
 724         edesc = echan->edesc;
 725
 726         /* Pause the channel for non-cyclic */
 727         if (!edesc || (edesc && !edesc->cyclic))
 728                 edma_pause(echan->ch_num);
 729
 730         switch (ch_status) {
 731         case EDMA_DMA_COMPLETE:
 732                 spin_lock(&echan->vchan.lock);
 733
 734                 if (edesc) {
 735                         if (edesc->cyclic) {
 736                                 vchan_cyclic_callback(&edesc->vdesc);
 737                         } else if (edesc->processed == edesc->pset_nr) {
 738                                 dev_dbg(dev, "Transfer complete, stopping channel %d\n", ch_num);
 739                                 edesc->residue = 0;
 740                                 edma_stop(echan->ch_num);
 741                                 vchan_cookie_complete(&edesc->vdesc);
 742                                 edma_execute(echan);
 743                         } else {
 744                                 dev_dbg(dev, "Intermediate transfer complete on channel %d\n", ch_num);
 745
 746                                 /* Update statistics for tx_status */
 747                                 edesc->residue -= edesc->sg_len;
 748                                 edesc->residue_stat = edesc->residue;
 749                                 edesc->processed_stat = edesc->processed;
 750
 751                                 edma_execute(echan);
 752                         }
 753                 }
 754
 755                 spin_unlock(&echan->vchan.lock);
 756
 757                 break;
 758         case EDMA_DMA_CC_ERROR:
 759                 spin_lock(&echan->vchan.lock);
 760
 761                 edma_read_slot(EDMA_CHAN_SLOT(echan->slot[0]), &p);
 762
 763                 /*
 764                  * Issue later based on missed flag which will be sure
 765                  * to happen as:
 766                  * (1) we finished transmitting an intermediate slot and
 767                  *     edma_execute is coming up.
 768                  * (2) or we finished current transfer and issue will
 769                  *     call edma_execute.
 770                  *
 771                  * Important note: issuing can be dangerous here and
 772                  * lead to some nasty recursion when we are in a NULL
 773                  * slot. So we avoid doing so and set the missed flag.
 774                  */
 775                 if (p.a_b_cnt == 0 && p.ccnt == 0) {
 776                         dev_dbg(dev, "Error occurred, looks like slot is null, just setting miss\n");
 777                         echan->missed = 1;
 778                 } else {
 779                         /*
 780                          * The slot is already programmed but the event got
 781                          * missed, so its safe to issue it here.
 782                          */
 783                         dev_dbg(dev, "Error occurred but slot is non-null, TRIGGERING\n");
 784                         edma_clean_channel(echan->ch_num);
 785                         edma_stop(echan->ch_num);
 786                         edma_start(echan->ch_num);
 787                         edma_trigger_channel(echan->ch_num);
 788                 }
 789
 790                 spin_unlock(&echan->vchan.lock);
 791
 792                 break;
 793         default:
 794                 break;
 795         }
 796 }
 797
 798 /* Alloc channel resources */
 799 static int edma_alloc_chan_resources(struct dma_chan *chan)
 800 {
 801         struct edma_chan *echan = to_edma_chan(chan);
 802         struct device *dev = chan->device->dev;
 803         int ret;
 804         int a_ch_num;
 805         LIST_HEAD(descs);
 806
 807         a_ch_num = edma_alloc_channel(echan->ch_num, edma_callback,
 808                                         chan, EVENTQ_DEFAULT);
 809
 810         if (a_ch_num < 0) {
 811                 ret = -ENODEV;
 812                 goto err_no_chan;
 813         }
 814
 815         if (a_ch_num != echan->ch_num) {
 816                 dev_err(dev, "failed to allocate requested channel %u:%u\n",
 817                         EDMA_CTLR(echan->ch_num),
 818                         EDMA_CHAN_SLOT(echan->ch_num));
 819                 ret = -ENODEV;
 820                 goto err_wrong_chan;
 821         }
 822
 823         echan->alloced = true;
 824         echan->slot[0] = echan->ch_num;
 825
 826         dev_dbg(dev, "allocated channel %d for %u:%u\n", echan->ch_num,
 827                 EDMA_CTLR(echan->ch_num), EDMA_CHAN_SLOT(echan->ch_num));
 828
 829         return 0;
 830
 831 err_wrong_chan:
 832         edma_free_channel(a_ch_num);
 833 err_no_chan:
 834         return ret;
 835 }
 836
 837 /* Free channel resources */
 838 static void edma_free_chan_resources(struct dma_chan *chan)
 839 {
 840         struct edma_chan *echan = to_edma_chan(chan);
 841         struct device *dev = chan->device->dev;
 842         int i;
 843
 844         /* Terminate transfers */
 845         edma_stop(echan->ch_num);
 846
 847         vchan_free_chan_resources(&echan->vchan);
 848
 849         /* Free EDMA PaRAM slots */
 850         for (i = 1; i < EDMA_MAX_SLOTS; i++) {
 851                 if (echan->slot[i] >= 0) {
 852                         edma_free_slot(echan->slot[i]);
 853                         echan->slot[i] = -1;
 854                 }
 855         }
 856
 857         /* Free EDMA channel */
 858         if (echan->alloced) {
 859                 edma_free_channel(echan->ch_num);
 860                 echan->alloced = false;
 861         }
 862
 863         dev_dbg(dev, "freeing channel for %u\n", echan->ch_num);
 864 }
 865
 866 /* Send pending descriptor to hardware */
 867 static void edma_issue_pending(struct dma_chan *chan)
 868 {
 869         struct edma_chan *echan = to_edma_chan(chan);
 870         unsigned long flags;
 871
 872         spin_lock_irqsave(&echan->vchan.lock, flags);
 873         if (vchan_issue_pending(&echan->vchan) && !echan->edesc)
 874                 edma_execute(echan);
 875         spin_unlock_irqrestore(&echan->vchan.lock, flags);
 876 }
 877
 878 static u32 edma_residue(struct edma_desc *edesc)
 879 {
 880         bool dst = edesc->direction == DMA_DEV_TO_MEM;
 881         struct edma_pset *pset = edesc->pset;
 882         dma_addr_t done, pos;
 883         int i;
 884
 885         /*
 886          * We always read the dst/src position from the first RamPar
 887          * pset. That's the one which is active now.
 888          */
 889         pos = edma_get_position(edesc->echan->slot[0], dst);
 890
 891         /*
 892          * Cyclic is simple. Just subtract pset[0].addr from pos.
 893          *
 894          * We never update edesc->residue in the cyclic case, so we
 895          * can tell the remaining room to the end of the circular
 896          * buffer.
 897          */
 898         if (edesc->cyclic) {
 899                 done = pos - pset->addr;
 900                 edesc->residue_stat = edesc->residue - done;
 901                 return edesc->residue_stat;
 902         }
 903
 904         /*
 905          * For SG operation we catch up with the last processed
 906          * status.
 907          */
 908         pset += edesc->processed_stat;
 909
 910         for (i = edesc->processed_stat; i < edesc->processed; i++, pset++) {
 911                 /*
 912                  * If we are inside this pset address range, we know
 913                  * this is the active one. Get the current delta and
 914                  * stop walking the psets.
 915                  */
 916                 if (pos >= pset->addr && pos < pset->addr + pset->len)
 917                         return edesc->residue_stat - (pos - pset->addr);
 918
 919                 /* Otherwise mark it done and update residue_stat. */
 920                 edesc->processed_stat++;
 921                 edesc->residue_stat -= pset->len;
 922         }
 923         return edesc->residue_stat;
 924 }
 925
 926 /* Check request completion status */
 927 static enum dma_status edma_tx_status(struct dma_chan *chan,
 928                                       dma_cookie_t cookie,
 929                                       struct dma_tx_state *txstate)
 930 {
 931         struct edma_chan *echan = to_edma_chan(chan);
 932         struct virt_dma_desc *vdesc;
 933         enum dma_status ret;
 934         unsigned long flags;
 935
 936         ret = dma_cookie_status(chan, cookie, txstate);
 937         if (ret == DMA_COMPLETE || !txstate)
 938                 return ret;
 939
 940         spin_lock_irqsave(&echan->vchan.lock, flags);
 941         if (echan->edesc && echan->edesc->vdesc.tx.cookie == cookie)
 942                 txstate->residue = edma_residue(echan->edesc);
 943         else if ((vdesc = vchan_find_desc(&echan->vchan, cookie)))
 944                 txstate->residue = to_edma_desc(&vdesc->tx)->residue;
 945         spin_unlock_irqrestore(&echan->vchan.lock, flags);
 946
 947         return ret;
 948 }
 949
 950 static void __init edma_chan_init(struct edma_cc *ecc,
 951                                   struct dma_device *dma,
 952                                   struct edma_chan *echans)
 953 {
 954         int i, j;
 955
 956         for (i = 0; i < EDMA_CHANS; i++) {
 957                 struct edma_chan *echan = &echans[i];
 958                 echan->ch_num = EDMA_CTLR_CHAN(ecc->ctlr, i);
 959                 echan->ecc = ecc;
 960                 echan->vchan.desc_free = edma_desc_free;
 961
 962                 vchan_init(&echan->vchan, dma);
 963
 964                 INIT_LIST_HEAD(&echan->node);
 965                 for (j = 0; j < EDMA_MAX_SLOTS; j++)
 966                         echan->slot[j] = -1;
 967         }
 968 }
 969
 970 #define EDMA_DMA_BUSWIDTHS      (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
 971                                  BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
 972                                  BIT(DMA_SLAVE_BUSWIDTH_3_BYTES) | \
 973                                  BIT(DMA_SLAVE_BUSWIDTH_4_BYTES))
 974
 975 static void edma_dma_init(struct edma_cc *ecc, struct dma_device *dma,
 976                           struct device *dev)
 977 {
 978         dma->device_prep_slave_sg = edma_prep_slave_sg;
 979         dma->device_prep_dma_cyclic = edma_prep_dma_cyclic;
 980         dma->device_prep_dma_memcpy = edma_prep_dma_memcpy;
 981         dma->device_alloc_chan_resources = edma_alloc_chan_resources;
 982         dma->device_free_chan_resources = edma_free_chan_resources;
 983         dma->device_issue_pending = edma_issue_pending;
 984         dma->device_tx_status = edma_tx_status;
 985         dma->device_config = edma_slave_config;
 986         dma->device_pause = edma_dma_pause;
 987         dma->device_resume = edma_dma_resume;
 988         dma->device_terminate_all = edma_terminate_all;
 989
 990         dma->src_addr_widths = EDMA_DMA_BUSWIDTHS;
 991         dma->dst_addr_widths = EDMA_DMA_BUSWIDTHS;
 992         dma->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
 993         dma->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
 994
 995         dma->dev = dev;
 996
 997         /*
 998          * code using dma memcpy must make sure alignment of
 999          * length is at dma->copy_align boundary.
1000          */
1001         dma->copy_align = DMA_SLAVE_BUSWIDTH_4_BYTES;
1002
1003         INIT_LIST_HEAD(&dma->channels);
1004 }
1005
1006 static int edma_probe(struct platform_device *pdev)
1007 {
1008         struct edma_cc *ecc;
1009         int ret;
1010
1011         ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
1012         if (ret)
1013                 return ret;
1014
1015         ecc = devm_kzalloc(&pdev->dev, sizeof(*ecc), GFP_KERNEL);
1016         if (!ecc) {
1017                 dev_err(&pdev->dev, "Can't allocate controller\n");
1018                 return -ENOMEM;
1019         }
1020
1021         ecc->ctlr = pdev->id;
1022         ecc->dummy_slot = edma_alloc_slot(ecc->ctlr, EDMA_SLOT_ANY);
1023         if (ecc->dummy_slot < 0) {
1024                 dev_err(&pdev->dev, "Can't allocate PaRAM dummy slot\n");
1025                 return ecc->dummy_slot;
1026         }
1027
1028         dma_cap_zero(ecc->dma_slave.cap_mask);
1029         dma_cap_set(DMA_SLAVE, ecc->dma_slave.cap_mask);
1030         dma_cap_set(DMA_CYCLIC, ecc->dma_slave.cap_mask);
1031         dma_cap_set(DMA_MEMCPY, ecc->dma_slave.cap_mask);
1032
1033         edma_dma_init(ecc, &ecc->dma_slave, &pdev->dev);
1034
1035         edma_chan_init(ecc, &ecc->dma_slave, ecc->slave_chans);
1036
1037         ret = dma_async_device_register(&ecc->dma_slave);
1038         if (ret)
1039                 goto err_reg1;
1040
1041         platform_set_drvdata(pdev, ecc);
1042
1043         dev_info(&pdev->dev, "TI EDMA DMA engine driver\n");
1044
1045         return 0;
1046
1047 err_reg1:
1048         edma_free_slot(ecc->dummy_slot);
1049         return ret;
1050 }
1051
1052 static int edma_remove(struct platform_device *pdev)
1053 {
1054         struct device *dev = &pdev->dev;
1055         struct edma_cc *ecc = dev_get_drvdata(dev);
1056
1057         dma_async_device_unregister(&ecc->dma_slave);
1058         edma_free_slot(ecc->dummy_slot);
1059
1060         return 0;
1061 }
1062
1063 static struct platform_driver edma_driver = {
1064         .probe          = edma_probe,
1065         .remove         = edma_remove,
1066         .driver = {
1067                 .name = "edma-dma-engine",
1068         },
1069 };
1070
1071 bool edma_filter_fn(struct dma_chan *chan, void *param)
1072 {
1073         if (chan->device->dev->driver == &edma_driver.driver) {
1074                 struct edma_chan *echan = to_edma_chan(chan);
1075                 unsigned ch_req = *(unsigned *)param;
1076                 return ch_req == echan->ch_num;
1077         }
1078         return false;
1079 }
1080 EXPORT_SYMBOL(edma_filter_fn);
1081
1082 static int edma_init(void)
1083 {
1084         return platform_driver_register(&edma_driver);
1085 }
1086 subsys_initcall(edma_init);
1087
1088 static void __exit edma_exit(void)
1089 {
1090         platform_driver_unregister(&edma_driver);
1091 }
1092 module_exit(edma_exit);
1093
1094 MODULE_AUTHOR("Matt Porter <matt.porter@linaro.org>");
1095 MODULE_DESCRIPTION("TI EDMA DMA engine driver");
1096 MODULE_LICENSE("GPL v2");