drivers/dma/edma.c

   1 /*
   2  * TI EDMA DMA engine driver
   3  *
   4  * Copyright 2012 Texas Instruments
   5  *
   6  * This program is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU General Public License as
   8  * published by the Free Software Foundation version 2.
   9  *
  10  * This program is distributed "as is" WITHOUT ANY WARRANTY of any
  11  * kind, whether express or implied; without even the implied warranty
  12  * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  */
  15
  16 #include <linux/dmaengine.h>
  17 #include <linux/dma-mapping.h>
  18 #include <linux/err.h>
  19 #include <linux/init.h>
  20 #include <linux/interrupt.h>
  21 #include <linux/list.h>
  22 #include <linux/module.h>
  23 #include <linux/platform_device.h>
  24 #include <linux/slab.h>
  25 #include <linux/spinlock.h>
  26
  27 #include <linux/platform_data/edma.h>
  28
  29 #include "dmaengine.h"
  30 #include "virt-dma.h"
  31
  32 /*
  33  * This will go away when the private EDMA API is folded
  34  * into this driver and the platform device(s) are
  35  * instantiated in the arch code. We can only get away
  36  * with this simplification because DA8XX may not be built
  37  * in the same kernel image with other DaVinci parts. This
  38  * avoids having to sprinkle dmaengine driver platform devices
  39  * and data throughout all the existing board files.
  40  */
  41 #ifdef CONFIG_ARCH_DAVINCI_DA8XX
  42 #define EDMA_CTLRS      2
  43 #define EDMA_CHANS      32
  44 #else
  45 #define EDMA_CTLRS      1
  46 #define EDMA_CHANS      64
  47 #endif /* CONFIG_ARCH_DAVINCI_DA8XX */
  48
  49 /*
  50  * Max of 20 segments per channel to conserve PaRAM slots
  51  * Also note that MAX_NR_SG should be atleast the no.of periods
  52  * that are required for ASoC, otherwise DMA prep calls will
  53  * fail. Today davinci-pcm is the only user of this driver and
  54  * requires atleast 17 slots, so we setup the default to 20.
  55  */
  56 #define MAX_NR_SG               20
  57 #define EDMA_MAX_SLOTS          MAX_NR_SG
  58 #define EDMA_DESCRIPTORS        16
  59
  60 struct edma_pset {
  61         u32                             len;
  62         dma_addr_t                      addr;
  63         struct edmacc_param             param;
  64 };
  65
  66 struct edma_desc {
  67         struct virt_dma_desc            vdesc;
  68         struct list_head                node;
  69         enum dma_transfer_direction     direction;
  70         int                             cyclic;
  71         int                             absync;
  72         int                             pset_nr;
  73         struct edma_chan                *echan;
  74         int                             processed;
  75
  76         /*
  77          * The following 4 elements are used for residue accounting.
  78          *
  79          * - processed_stat: the number of SG elements we have traversed
  80          * so far to cover accounting. This is updated directly to processed
  81          * during edma_callback and is always <= processed, because processed
  82          * refers to the number of pending transfer (programmed to EDMA
  83          * controller), where as processed_stat tracks number of transfers
  84          * accounted for so far.
  85          *
  86          * - residue: The amount of bytes we have left to transfer for this desc
  87          *
  88          * - residue_stat: The residue in bytes of data we have covered
  89          * so far for accounting. This is updated directly to residue
  90          * during callbacks to keep it current.
  91          *
  92          * - sg_len: Tracks the length of the current intermediate transfer,
  93          * this is required to update the residue during intermediate transfer
  94          * completion callback.
  95          */
  96         int                             processed_stat;
  97         u32                             sg_len;
  98         u32                             residue;
  99         u32                             residue_stat;
 100
 101         struct edma_pset                pset[0];
 102 };
 103
 104 struct edma_cc;
 105
 106 struct edma_chan {
 107         struct virt_dma_chan            vchan;
 108         struct list_head                node;
 109         struct edma_desc                *edesc;
 110         struct edma_cc                  *ecc;
 111         int                             ch_num;
 112         bool                            alloced;
 113         int                             slot[EDMA_MAX_SLOTS];
 114         int                             missed;
 115         struct dma_slave_config         cfg;
 116 };
 117
 118 struct edma_cc {
 119         int                             ctlr;
 120         struct dma_device               dma_slave;
 121         struct edma_chan                slave_chans[EDMA_CHANS];
 122         int                             num_slave_chans;
 123         int                             dummy_slot;
 124 };
 125
 126 static inline struct edma_cc *to_edma_cc(struct dma_device *d)
 127 {
 128         return container_of(d, struct edma_cc, dma_slave);
 129 }
 130
 131 static inline struct edma_chan *to_edma_chan(struct dma_chan *c)
 132 {
 133         return container_of(c, struct edma_chan, vchan.chan);
 134 }
 135
 136 static inline struct edma_desc
 137 *to_edma_desc(struct dma_async_tx_descriptor *tx)
 138 {
 139         return container_of(tx, struct edma_desc, vdesc.tx);
 140 }
 141
 142 static void edma_desc_free(struct virt_dma_desc *vdesc)
 143 {
 144         kfree(container_of(vdesc, struct edma_desc, vdesc));
 145 }
 146
 147 /* Dispatch a queued descriptor to the controller (caller holds lock) */
 148 static void edma_execute(struct edma_chan *echan)
 149 {
 150         struct virt_dma_desc *vdesc;
 151         struct edma_desc *edesc;
 152         struct device *dev = echan->vchan.chan.device->dev;
 153         int i, j, left, nslots;
 154
 155         /* If either we processed all psets or we're still not started */
 156         if (!echan->edesc ||
 157             echan->edesc->pset_nr == echan->edesc->processed) {
 158                 /* Get next vdesc */
 159                 vdesc = vchan_next_desc(&echan->vchan);
 160                 if (!vdesc) {
 161                         echan->edesc = NULL;
 162                         return;
 163                 }
 164                 list_del(&vdesc->node);
 165                 echan->edesc = to_edma_desc(&vdesc->tx);
 166         }
 167
 168         edesc = echan->edesc;
 169
 170         /* Find out how many left */
 171         left = edesc->pset_nr - edesc->processed;
 172         nslots = min(MAX_NR_SG, left);
 173         edesc->sg_len = 0;
 174
 175         /* Write descriptor PaRAM set(s) */
 176         for (i = 0; i < nslots; i++) {
 177                 j = i + edesc->processed;
 178                 edma_write_slot(echan->slot[i], &edesc->pset[j].param);
 179                 edesc->sg_len += edesc->pset[j].len;
 180                 dev_vdbg(echan->vchan.chan.device->dev,
 181                         "\n pset[%d]:\n"
 182                         "  chnum\t%d\n"
 183                         "  slot\t%d\n"
 184                         "  opt\t%08x\n"
 185                         "  src\t%08x\n"
 186                         "  dst\t%08x\n"
 187                         "  abcnt\t%08x\n"
 188                         "  ccnt\t%08x\n"
 189                         "  bidx\t%08x\n"
 190                         "  cidx\t%08x\n"
 191                         "  lkrld\t%08x\n",
 192                         j, echan->ch_num, echan->slot[i],
 193                         edesc->pset[j].param.opt,
 194                         edesc->pset[j].param.src,
 195                         edesc->pset[j].param.dst,
 196                         edesc->pset[j].param.a_b_cnt,
 197                         edesc->pset[j].param.ccnt,
 198                         edesc->pset[j].param.src_dst_bidx,
 199                         edesc->pset[j].param.src_dst_cidx,
 200                         edesc->pset[j].param.link_bcntrld);
 201                 /* Link to the previous slot if not the last set */
 202                 if (i != (nslots - 1))
 203                         edma_link(echan->slot[i], echan->slot[i+1]);
 204         }
 205
 206         edesc->processed += nslots;
 207
 208         /*
 209          * If this is either the last set in a set of SG-list transactions
 210          * then setup a link to the dummy slot, this results in all future
 211          * events being absorbed and that's OK because we're done
 212          */
 213         if (edesc->processed == edesc->pset_nr) {
 214                 if (edesc->cyclic)
 215                         edma_link(echan->slot[nslots-1], echan->slot[1]);
 216                 else
 217                         edma_link(echan->slot[nslots-1],
 218                                   echan->ecc->dummy_slot);
 219         }
 220
 221         if (edesc->processed <= MAX_NR_SG) {
 222                 dev_dbg(dev, "first transfer starting on channel %d\n",
 223                         echan->ch_num);
 224                 edma_start(echan->ch_num);
 225         } else {
 226                 dev_dbg(dev, "chan: %d: completed %d elements, resuming\n",
 227                         echan->ch_num, edesc->processed);
 228                 edma_resume(echan->ch_num);
 229         }
 230
 231         /*
 232          * This happens due to setup times between intermediate transfers
 233          * in long SG lists which have to be broken up into transfers of
 234          * MAX_NR_SG
 235          */
 236         if (echan->missed) {
 237                 dev_dbg(dev, "missed event on channel %d\n", echan->ch_num);
 238                 edma_clean_channel(echan->ch_num);
 239                 edma_stop(echan->ch_num);
 240                 edma_start(echan->ch_num);
 241                 edma_trigger_channel(echan->ch_num);
 242                 echan->missed = 0;
 243         }
 244 }
 245
 246 static int edma_terminate_all(struct edma_chan *echan)
 247 {
 248         unsigned long flags;
 249         LIST_HEAD(head);
 250
 251         spin_lock_irqsave(&echan->vchan.lock, flags);
 252
 253         /*
 254          * Stop DMA activity: we assume the callback will not be called
 255          * after edma_dma() returns (even if it does, it will see
 256          * echan->edesc is NULL and exit.)
 257          */
 258         if (echan->edesc) {
 259                 echan->edesc = NULL;
 260                 edma_stop(echan->ch_num);
 261         }
 262
 263         vchan_get_all_descriptors(&echan->vchan, &head);
 264         spin_unlock_irqrestore(&echan->vchan.lock, flags);
 265         vchan_dma_desc_free_list(&echan->vchan, &head);
 266
 267         return 0;
 268 }
 269
 270 static int edma_slave_config(struct edma_chan *echan,
 271         struct dma_slave_config *cfg)
 272 {
 273         if (cfg->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES ||
 274             cfg->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)
 275                 return -EINVAL;
 276
 277         memcpy(&echan->cfg, cfg, sizeof(echan->cfg));
 278
 279         return 0;
 280 }
 281
 282 static int edma_dma_pause(struct edma_chan *echan)
 283 {
 284         /* Pause/Resume only allowed with cyclic mode */
 285         if (!echan->edesc->cyclic)
 286                 return -EINVAL;
 287
 288         edma_pause(echan->ch_num);
 289         return 0;
 290 }
 291
 292 static int edma_dma_resume(struct edma_chan *echan)
 293 {
 294         /* Pause/Resume only allowed with cyclic mode */
 295         if (!echan->edesc->cyclic)
 296                 return -EINVAL;
 297
 298         edma_resume(echan->ch_num);
 299         return 0;
 300 }
 301
 302 static int edma_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
 303                         unsigned long arg)
 304 {
 305         int ret = 0;
 306         struct dma_slave_config *config;
 307         struct edma_chan *echan = to_edma_chan(chan);
 308
 309         switch (cmd) {
 310         case DMA_TERMINATE_ALL:
 311                 edma_terminate_all(echan);
 312                 break;
 313         case DMA_SLAVE_CONFIG:
 314                 config = (struct dma_slave_config *)arg;
 315                 ret = edma_slave_config(echan, config);
 316                 break;
 317         case DMA_PAUSE:
 318                 ret = edma_dma_pause(echan);
 319                 break;
 320
 321         case DMA_RESUME:
 322                 ret = edma_dma_resume(echan);
 323                 break;
 324
 325         default:
 326                 ret = -ENOSYS;
 327         }
 328
 329         return ret;
 330 }
 331
 332 /*
 333  * A PaRAM set configuration abstraction used by other modes
 334  * @chan: Channel who's PaRAM set we're configuring
 335  * @pset: PaRAM set to initialize and setup.
 336  * @src_addr: Source address of the DMA
 337  * @dst_addr: Destination address of the DMA
 338  * @burst: In units of dev_width, how much to send
 339  * @dev_width: How much is the dev_width
 340  * @dma_length: Total length of the DMA transfer
 341  * @direction: Direction of the transfer
 342  */
 343 static int edma_config_pset(struct dma_chan *chan, struct edma_pset *epset,
 344         dma_addr_t src_addr, dma_addr_t dst_addr, u32 burst,
 345         enum dma_slave_buswidth dev_width, unsigned int dma_length,
 346         enum dma_transfer_direction direction)
 347 {
 348         struct edma_chan *echan = to_edma_chan(chan);
 349         struct device *dev = chan->device->dev;
 350         struct edmacc_param *param = &epset->param;
 351         int acnt, bcnt, ccnt, cidx;
 352         int src_bidx, dst_bidx, src_cidx, dst_cidx;
 353         int absync;
 354
 355         acnt = dev_width;
 356
 357         /* src/dst_maxburst == 0 is the same case as src/dst_maxburst == 1 */
 358         if (!burst)
 359                 burst = 1;
 360         /*
 361          * If the maxburst is equal to the fifo width, use
 362          * A-synced transfers. This allows for large contiguous
 363          * buffer transfers using only one PaRAM set.
 364          */
 365         if (burst == 1) {
 366                 /*
 367                  * For the A-sync case, bcnt and ccnt are the remainder
 368                  * and quotient respectively of the division of:
 369                  * (dma_length / acnt) by (SZ_64K -1). This is so
 370                  * that in case bcnt over flows, we have ccnt to use.
 371                  * Note: In A-sync tranfer only, bcntrld is used, but it
 372                  * only applies for sg_dma_len(sg) >= SZ_64K.
 373                  * In this case, the best way adopted is- bccnt for the
 374                  * first frame will be the remainder below. Then for
 375                  * every successive frame, bcnt will be SZ_64K-1. This
 376                  * is assured as bcntrld = 0xffff in end of function.
 377                  */
 378                 absync = false;
 379                 ccnt = dma_length / acnt / (SZ_64K - 1);
 380                 bcnt = dma_length / acnt - ccnt * (SZ_64K - 1);
 381                 /*
 382                  * If bcnt is non-zero, we have a remainder and hence an
 383                  * extra frame to transfer, so increment ccnt.
 384                  */
 385                 if (bcnt)
 386                         ccnt++;
 387                 else
 388                         bcnt = SZ_64K - 1;
 389                 cidx = acnt;
 390         } else {
 391                 /*
 392                  * If maxburst is greater than the fifo address_width,
 393                  * use AB-synced transfers where A count is the fifo
 394                  * address_width and B count is the maxburst. In this
 395                  * case, we are limited to transfers of C count frames
 396                  * of (address_width * maxburst) where C count is limited
 397                  * to SZ_64K-1. This places an upper bound on the length
 398                  * of an SG segment that can be handled.
 399                  */
 400                 absync = true;
 401                 bcnt = burst;
 402                 ccnt = dma_length / (acnt * bcnt);
 403                 if (ccnt > (SZ_64K - 1)) {
 404                         dev_err(dev, "Exceeded max SG segment size\n");
 405                         return -EINVAL;
 406                 }
 407                 cidx = acnt * bcnt;
 408         }
 409
 410         epset->len = dma_length;
 411
 412         if (direction == DMA_MEM_TO_DEV) {
 413                 src_bidx = acnt;
 414                 src_cidx = cidx;
 415                 dst_bidx = 0;
 416                 dst_cidx = 0;
 417                 epset->addr = src_addr;
 418         } else if (direction == DMA_DEV_TO_MEM)  {
 419                 src_bidx = 0;
 420                 src_cidx = 0;
 421                 dst_bidx = acnt;
 422                 dst_cidx = cidx;
 423                 epset->addr = dst_addr;
 424         } else if (direction == DMA_MEM_TO_MEM)  {
 425                 src_bidx = acnt;
 426                 src_cidx = cidx;
 427                 dst_bidx = acnt;
 428                 dst_cidx = cidx;
 429         } else {
 430                 dev_err(dev, "%s: direction not implemented yet\n", __func__);
 431                 return -EINVAL;
 432         }
 433
 434         param->opt = EDMA_TCC(EDMA_CHAN_SLOT(echan->ch_num));
 435         /* Configure A or AB synchronized transfers */
 436         if (absync)
 437                 param->opt |= SYNCDIM;
 438
 439         param->src = src_addr;
 440         param->dst = dst_addr;
 441
 442         param->src_dst_bidx = (dst_bidx << 16) | src_bidx;
 443         param->src_dst_cidx = (dst_cidx << 16) | src_cidx;
 444
 445         param->a_b_cnt = bcnt << 16 | acnt;
 446         param->ccnt = ccnt;
 447         /*
 448          * Only time when (bcntrld) auto reload is required is for
 449          * A-sync case, and in this case, a requirement of reload value
 450          * of SZ_64K-1 only is assured. 'link' is initially set to NULL
 451          * and then later will be populated by edma_execute.
 452          */
 453         param->link_bcntrld = 0xffffffff;
 454         return absync;
 455 }
 456
 457 static struct dma_async_tx_descriptor *edma_prep_slave_sg(
 458         struct dma_chan *chan, struct scatterlist *sgl,
 459         unsigned int sg_len, enum dma_transfer_direction direction,
 460         unsigned long tx_flags, void *context)
 461 {
 462         struct edma_chan *echan = to_edma_chan(chan);
 463         struct device *dev = chan->device->dev;
 464         struct edma_desc *edesc;
 465         dma_addr_t src_addr = 0, dst_addr = 0;
 466         enum dma_slave_buswidth dev_width;
 467         u32 burst;
 468         struct scatterlist *sg;
 469         int i, nslots, ret;
 470
 471         if (unlikely(!echan || !sgl || !sg_len))
 472                 return NULL;
 473
 474         if (direction == DMA_DEV_TO_MEM) {
 475                 src_addr = echan->cfg.src_addr;
 476                 dev_width = echan->cfg.src_addr_width;
 477                 burst = echan->cfg.src_maxburst;
 478         } else if (direction == DMA_MEM_TO_DEV) {
 479                 dst_addr = echan->cfg.dst_addr;
 480                 dev_width = echan->cfg.dst_addr_width;
 481                 burst = echan->cfg.dst_maxburst;
 482         } else {
 483                 dev_err(dev, "%s: bad direction: %d\n", __func__, direction);
 484                 return NULL;
 485         }
 486
 487         if (dev_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) {
 488                 dev_err(dev, "%s: Undefined slave buswidth\n", __func__);
 489                 return NULL;
 490         }
 491
 492         edesc = kzalloc(sizeof(*edesc) + sg_len *
 493                 sizeof(edesc->pset[0]), GFP_ATOMIC);
 494         if (!edesc) {
 495                 dev_err(dev, "%s: Failed to allocate a descriptor\n", __func__);
 496                 return NULL;
 497         }
 498
 499         edesc->pset_nr = sg_len;
 500         edesc->residue = 0;
 501         edesc->direction = direction;
 502         edesc->echan = echan;
 503
 504         /* Allocate a PaRAM slot, if needed */
 505         nslots = min_t(unsigned, MAX_NR_SG, sg_len);
 506
 507         for (i = 0; i < nslots; i++) {
 508                 if (echan->slot[i] < 0) {
 509                         echan->slot[i] =
 510                                 edma_alloc_slot(EDMA_CTLR(echan->ch_num),
 511                                                 EDMA_SLOT_ANY);
 512                         if (echan->slot[i] < 0) {
 513                                 kfree(edesc);
 514                                 dev_err(dev, "%s: Failed to allocate slot\n",
 515                                         __func__);
 516                                 return NULL;
 517                         }
 518                 }
 519         }
 520
 521         /* Configure PaRAM sets for each SG */
 522         for_each_sg(sgl, sg, sg_len, i) {
 523                 /* Get address for each SG */
 524                 if (direction == DMA_DEV_TO_MEM)
 525                         dst_addr = sg_dma_address(sg);
 526                 else
 527                         src_addr = sg_dma_address(sg);
 528
 529                 ret = edma_config_pset(chan, &edesc->pset[i], src_addr,
 530                                        dst_addr, burst, dev_width,
 531                                        sg_dma_len(sg), direction);
 532                 if (ret < 0) {
 533                         kfree(edesc);
 534                         return NULL;
 535                 }
 536
 537                 edesc->absync = ret;
 538                 edesc->residue += sg_dma_len(sg);
 539
 540                 /* If this is the last in a current SG set of transactions,
 541                    enable interrupts so that next set is processed */
 542                 if (!((i+1) % MAX_NR_SG))
 543                         edesc->pset[i].param.opt |= TCINTEN;
 544
 545                 /* If this is the last set, enable completion interrupt flag */
 546                 if (i == sg_len - 1)
 547                         edesc->pset[i].param.opt |= TCINTEN;
 548         }
 549         edesc->residue_stat = edesc->residue;
 550
 551         return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags);
 552 }
 553
 554 struct dma_async_tx_descriptor *edma_prep_dma_memcpy(
 555         struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
 556         size_t len, unsigned long tx_flags)
 557 {
 558         int ret;
 559         struct edma_desc *edesc;
 560         struct device *dev = chan->device->dev;
 561         struct edma_chan *echan = to_edma_chan(chan);
 562
 563         if (unlikely(!echan || !len))
 564                 return NULL;
 565
 566         edesc = kzalloc(sizeof(*edesc) + sizeof(edesc->pset[0]), GFP_ATOMIC);
 567         if (!edesc) {
 568                 dev_dbg(dev, "Failed to allocate a descriptor\n");
 569                 return NULL;
 570         }
 571
 572         edesc->pset_nr = 1;
 573
 574         ret = edma_config_pset(chan, &edesc->pset[0], src, dest, 1,
 575                                DMA_SLAVE_BUSWIDTH_4_BYTES, len, DMA_MEM_TO_MEM);
 576         if (ret < 0)
 577                 return NULL;
 578
 579         edesc->absync = ret;
 580
 581         /*
 582          * Enable intermediate transfer chaining to re-trigger channel
 583          * on completion of every TR, and enable transfer-completion
 584          * interrupt on completion of the whole transfer.
 585          */
 586         edesc->pset[0].param.opt |= ITCCHEN;
 587         edesc->pset[0].param.opt |= TCINTEN;
 588
 589         return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags);
 590 }
 591
 592 static struct dma_async_tx_descriptor *edma_prep_dma_cyclic(
 593         struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
 594         size_t period_len, enum dma_transfer_direction direction,
 595         unsigned long tx_flags, void *context)
 596 {
 597         struct edma_chan *echan = to_edma_chan(chan);
 598         struct device *dev = chan->device->dev;
 599         struct edma_desc *edesc;
 600         dma_addr_t src_addr, dst_addr;
 601         enum dma_slave_buswidth dev_width;
 602         u32 burst;
 603         int i, ret, nslots;
 604
 605         if (unlikely(!echan || !buf_len || !period_len))
 606                 return NULL;
 607
 608         if (direction == DMA_DEV_TO_MEM) {
 609                 src_addr = echan->cfg.src_addr;
 610                 dst_addr = buf_addr;
 611                 dev_width = echan->cfg.src_addr_width;
 612                 burst = echan->cfg.src_maxburst;
 613         } else if (direction == DMA_MEM_TO_DEV) {
 614                 src_addr = buf_addr;
 615                 dst_addr = echan->cfg.dst_addr;
 616                 dev_width = echan->cfg.dst_addr_width;
 617                 burst = echan->cfg.dst_maxburst;
 618         } else {
 619                 dev_err(dev, "%s: bad direction: %d\n", __func__, direction);
 620                 return NULL;
 621         }
 622
 623         if (dev_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) {
 624                 dev_err(dev, "%s: Undefined slave buswidth\n", __func__);
 625                 return NULL;
 626         }
 627
 628         if (unlikely(buf_len % period_len)) {
 629                 dev_err(dev, "Period should be multiple of Buffer length\n");
 630                 return NULL;
 631         }
 632
 633         nslots = (buf_len / period_len) + 1;
 634
 635         /*
 636          * Cyclic DMA users such as audio cannot tolerate delays introduced
 637          * by cases where the number of periods is more than the maximum
 638          * number of SGs the EDMA driver can handle at a time. For DMA types
 639          * such as Slave SGs, such delays are tolerable and synchronized,
 640          * but the synchronization is difficult to achieve with Cyclic and
 641          * cannot be guaranteed, so we error out early.
 642          */
 643         if (nslots > MAX_NR_SG)
 644                 return NULL;
 645
 646         edesc = kzalloc(sizeof(*edesc) + nslots *
 647                 sizeof(edesc->pset[0]), GFP_ATOMIC);
 648         if (!edesc) {
 649                 dev_err(dev, "%s: Failed to allocate a descriptor\n", __func__);
 650                 return NULL;
 651         }
 652
 653         edesc->cyclic = 1;
 654         edesc->pset_nr = nslots;
 655         edesc->residue = edesc->residue_stat = buf_len;
 656         edesc->direction = direction;
 657         edesc->echan = echan;
 658
 659         dev_dbg(dev, "%s: channel=%d nslots=%d period_len=%zu buf_len=%zu\n",
 660                 __func__, echan->ch_num, nslots, period_len, buf_len);
 661
 662         for (i = 0; i < nslots; i++) {
 663                 /* Allocate a PaRAM slot, if needed */
 664                 if (echan->slot[i] < 0) {
 665                         echan->slot[i] =
 666                                 edma_alloc_slot(EDMA_CTLR(echan->ch_num),
 667                                                 EDMA_SLOT_ANY);
 668                         if (echan->slot[i] < 0) {
 669                                 kfree(edesc);
 670                                 dev_err(dev, "%s: Failed to allocate slot\n",
 671                                         __func__);
 672                                 return NULL;
 673                         }
 674                 }
 675
 676                 if (i == nslots - 1) {
 677                         memcpy(&edesc->pset[i], &edesc->pset[0],
 678                                sizeof(edesc->pset[0]));
 679                         break;
 680                 }
 681
 682                 ret = edma_config_pset(chan, &edesc->pset[i], src_addr,
 683                                        dst_addr, burst, dev_width, period_len,
 684                                        direction);
 685                 if (ret < 0) {
 686                         kfree(edesc);
 687                         return NULL;
 688                 }
 689
 690                 if (direction == DMA_DEV_TO_MEM)
 691                         dst_addr += period_len;
 692                 else
 693                         src_addr += period_len;
 694
 695                 dev_vdbg(dev, "%s: Configure period %d of buf:\n", __func__, i);
 696                 dev_vdbg(dev,
 697                         "\n pset[%d]:\n"
 698                         "  chnum\t%d\n"
 699                         "  slot\t%d\n"
 700                         "  opt\t%08x\n"
 701                         "  src\t%08x\n"
 702                         "  dst\t%08x\n"
 703                         "  abcnt\t%08x\n"
 704                         "  ccnt\t%08x\n"
 705                         "  bidx\t%08x\n"
 706                         "  cidx\t%08x\n"
 707                         "  lkrld\t%08x\n",
 708                         i, echan->ch_num, echan->slot[i],
 709                         edesc->pset[i].param.opt,
 710                         edesc->pset[i].param.src,
 711                         edesc->pset[i].param.dst,
 712                         edesc->pset[i].param.a_b_cnt,
 713                         edesc->pset[i].param.ccnt,
 714                         edesc->pset[i].param.src_dst_bidx,
 715                         edesc->pset[i].param.src_dst_cidx,
 716                         edesc->pset[i].param.link_bcntrld);
 717
 718                 edesc->absync = ret;
 719
 720                 /*
 721                  * Enable interrupts for every period because callback
 722                  * has to be called for every period.
 723                  */
 724                 edesc->pset[i].param.opt |= TCINTEN;
 725         }
 726
 727         return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags);
 728 }
 729
 730 static void edma_callback(unsigned ch_num, u16 ch_status, void *data)
 731 {
 732         struct edma_chan *echan = data;
 733         struct device *dev = echan->vchan.chan.device->dev;
 734         struct edma_desc *edesc;
 735         struct edmacc_param p;
 736
 737         edesc = echan->edesc;
 738
 739         /* Pause the channel for non-cyclic */
 740         if (!edesc || (edesc && !edesc->cyclic))
 741                 edma_pause(echan->ch_num);
 742
 743         switch (ch_status) {
 744         case EDMA_DMA_COMPLETE:
 745                 spin_lock(&echan->vchan.lock);
 746
 747                 if (edesc) {
 748                         if (edesc->cyclic) {
 749                                 vchan_cyclic_callback(&edesc->vdesc);
 750                         } else if (edesc->processed == edesc->pset_nr) {
 751                                 dev_dbg(dev, "Transfer complete, stopping channel %d\n", ch_num);
 752                                 edesc->residue = 0;
 753                                 edma_stop(echan->ch_num);
 754                                 vchan_cookie_complete(&edesc->vdesc);
 755                                 edma_execute(echan);
 756                         } else {
 757                                 dev_dbg(dev, "Intermediate transfer complete on channel %d\n", ch_num);
 758
 759                                 /* Update statistics for tx_status */
 760                                 edesc->residue -= edesc->sg_len;
 761                                 edesc->residue_stat = edesc->residue;
 762                                 edesc->processed_stat = edesc->processed;
 763
 764                                 edma_execute(echan);
 765                         }
 766                 }
 767
 768                 spin_unlock(&echan->vchan.lock);
 769
 770                 break;
 771         case EDMA_DMA_CC_ERROR:
 772                 spin_lock(&echan->vchan.lock);
 773
 774                 edma_read_slot(EDMA_CHAN_SLOT(echan->slot[0]), &p);
 775
 776                 /*
 777                  * Issue later based on missed flag which will be sure
 778                  * to happen as:
 779                  * (1) we finished transmitting an intermediate slot and
 780                  *     edma_execute is coming up.
 781                  * (2) or we finished current transfer and issue will
 782                  *     call edma_execute.
 783                  *
 784                  * Important note: issuing can be dangerous here and
 785                  * lead to some nasty recursion when we are in a NULL
 786                  * slot. So we avoid doing so and set the missed flag.
 787                  */
 788                 if (p.a_b_cnt == 0 && p.ccnt == 0) {
 789                         dev_dbg(dev, "Error occurred, looks like slot is null, just setting miss\n");
 790                         echan->missed = 1;
 791                 } else {
 792                         /*
 793                          * The slot is already programmed but the event got
 794                          * missed, so its safe to issue it here.
 795                          */
 796                         dev_dbg(dev, "Error occurred but slot is non-null, TRIGGERING\n");
 797                         edma_clean_channel(echan->ch_num);
 798                         edma_stop(echan->ch_num);
 799                         edma_start(echan->ch_num);
 800                         edma_trigger_channel(echan->ch_num);
 801                 }
 802
 803                 spin_unlock(&echan->vchan.lock);
 804
 805                 break;
 806         default:
 807                 break;
 808         }
 809 }
 810
 811 /* Alloc channel resources */
 812 static int edma_alloc_chan_resources(struct dma_chan *chan)
 813 {
 814         struct edma_chan *echan = to_edma_chan(chan);
 815         struct device *dev = chan->device->dev;
 816         int ret;
 817         int a_ch_num;
 818         LIST_HEAD(descs);
 819
 820         a_ch_num = edma_alloc_channel(echan->ch_num, edma_callback,
 821                                         chan, EVENTQ_DEFAULT);
 822
 823         if (a_ch_num < 0) {
 824                 ret = -ENODEV;
 825                 goto err_no_chan;
 826         }
 827
 828         if (a_ch_num != echan->ch_num) {
 829                 dev_err(dev, "failed to allocate requested channel %u:%u\n",
 830                         EDMA_CTLR(echan->ch_num),
 831                         EDMA_CHAN_SLOT(echan->ch_num));
 832                 ret = -ENODEV;
 833                 goto err_wrong_chan;
 834         }
 835
 836         echan->alloced = true;
 837         echan->slot[0] = echan->ch_num;
 838
 839         dev_dbg(dev, "allocated channel %d for %u:%u\n", echan->ch_num,
 840                 EDMA_CTLR(echan->ch_num), EDMA_CHAN_SLOT(echan->ch_num));
 841
 842         return 0;
 843
 844 err_wrong_chan:
 845         edma_free_channel(a_ch_num);
 846 err_no_chan:
 847         return ret;
 848 }
 849
 850 /* Free channel resources */
 851 static void edma_free_chan_resources(struct dma_chan *chan)
 852 {
 853         struct edma_chan *echan = to_edma_chan(chan);
 854         struct device *dev = chan->device->dev;
 855         int i;
 856
 857         /* Terminate transfers */
 858         edma_stop(echan->ch_num);
 859
 860         vchan_free_chan_resources(&echan->vchan);
 861
 862         /* Free EDMA PaRAM slots */
 863         for (i = 1; i < EDMA_MAX_SLOTS; i++) {
 864                 if (echan->slot[i] >= 0) {
 865                         edma_free_slot(echan->slot[i]);
 866                         echan->slot[i] = -1;
 867                 }
 868         }
 869
 870         /* Free EDMA channel */
 871         if (echan->alloced) {
 872                 edma_free_channel(echan->ch_num);
 873                 echan->alloced = false;
 874         }
 875
 876         dev_dbg(dev, "freeing channel for %u\n", echan->ch_num);
 877 }
 878
 879 /* Send pending descriptor to hardware */
 880 static void edma_issue_pending(struct dma_chan *chan)
 881 {
 882         struct edma_chan *echan = to_edma_chan(chan);
 883         unsigned long flags;
 884
 885         spin_lock_irqsave(&echan->vchan.lock, flags);
 886         if (vchan_issue_pending(&echan->vchan) && !echan->edesc)
 887                 edma_execute(echan);
 888         spin_unlock_irqrestore(&echan->vchan.lock, flags);
 889 }
 890
 891 static u32 edma_residue(struct edma_desc *edesc)
 892 {
 893         bool dst = edesc->direction == DMA_DEV_TO_MEM;
 894         struct edma_pset *pset = edesc->pset;
 895         dma_addr_t done, pos;
 896         int i;
 897
 898         /*
 899          * We always read the dst/src position from the first RamPar
 900          * pset. That's the one which is active now.
 901          */
 902         pos = edma_get_position(edesc->echan->slot[0], dst);
 903
 904         /*
 905          * Cyclic is simple. Just subtract pset[0].addr from pos.
 906          *
 907          * We never update edesc->residue in the cyclic case, so we
 908          * can tell the remaining room to the end of the circular
 909          * buffer.
 910          */
 911         if (edesc->cyclic) {
 912                 done = pos - pset->addr;
 913                 edesc->residue_stat = edesc->residue - done;
 914                 return edesc->residue_stat;
 915         }
 916
 917         /*
 918          * For SG operation we catch up with the last processed
 919          * status.
 920          */
 921         pset += edesc->processed_stat;
 922
 923         for (i = edesc->processed_stat; i < edesc->processed; i++, pset++) {
 924                 /*
 925                  * If we are inside this pset address range, we know
 926                  * this is the active one. Get the current delta and
 927                  * stop walking the psets.
 928                  */
 929                 if (pos >= pset->addr && pos < pset->addr + pset->len)
 930                         return edesc->residue_stat - (pos - pset->addr);
 931
 932                 /* Otherwise mark it done and update residue_stat. */
 933                 edesc->processed_stat++;
 934                 edesc->residue_stat -= pset->len;
 935         }
 936         return edesc->residue_stat;
 937 }
 938
 939 /* Check request completion status */
 940 static enum dma_status edma_tx_status(struct dma_chan *chan,
 941                                       dma_cookie_t cookie,
 942                                       struct dma_tx_state *txstate)
 943 {
 944         struct edma_chan *echan = to_edma_chan(chan);
 945         struct virt_dma_desc *vdesc;
 946         enum dma_status ret;
 947         unsigned long flags;
 948
 949         ret = dma_cookie_status(chan, cookie, txstate);
 950         if (ret == DMA_COMPLETE || !txstate)
 951                 return ret;
 952
 953         spin_lock_irqsave(&echan->vchan.lock, flags);
 954         if (echan->edesc && echan->edesc->vdesc.tx.cookie == cookie)
 955                 txstate->residue = edma_residue(echan->edesc);
 956         else if ((vdesc = vchan_find_desc(&echan->vchan, cookie)))
 957                 txstate->residue = to_edma_desc(&vdesc->tx)->residue;
 958         spin_unlock_irqrestore(&echan->vchan.lock, flags);
 959
 960         return ret;
 961 }
 962
 963 static void __init edma_chan_init(struct edma_cc *ecc,
 964                                   struct dma_device *dma,
 965                                   struct edma_chan *echans)
 966 {
 967         int i, j;
 968
 969         for (i = 0; i < EDMA_CHANS; i++) {
 970                 struct edma_chan *echan = &echans[i];
 971                 echan->ch_num = EDMA_CTLR_CHAN(ecc->ctlr, i);
 972                 echan->ecc = ecc;
 973                 echan->vchan.desc_free = edma_desc_free;
 974
 975                 vchan_init(&echan->vchan, dma);
 976
 977                 INIT_LIST_HEAD(&echan->node);
 978                 for (j = 0; j < EDMA_MAX_SLOTS; j++)
 979                         echan->slot[j] = -1;
 980         }
 981 }
 982
 983 #define EDMA_DMA_BUSWIDTHS      (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
 984                                  BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
 985                                  BIT(DMA_SLAVE_BUSWIDTH_4_BYTES))
 986
 987 static int edma_dma_device_slave_caps(struct dma_chan *dchan,
 988                                       struct dma_slave_caps *caps)
 989 {
 990         caps->src_addr_widths = EDMA_DMA_BUSWIDTHS;
 991         caps->dstn_addr_widths = EDMA_DMA_BUSWIDTHS;
 992         caps->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
 993         caps->cmd_pause = true;
 994         caps->cmd_terminate = true;
 995         caps->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR;
 996
 997         return 0;
 998 }
 999
1000 static void edma_dma_init(struct edma_cc *ecc, struct dma_device *dma,
1001                           struct device *dev)
1002 {
1003         dma->device_prep_slave_sg = edma_prep_slave_sg;
1004         dma->device_prep_dma_cyclic = edma_prep_dma_cyclic;
1005         dma->device_prep_dma_memcpy = edma_prep_dma_memcpy;
1006         dma->device_alloc_chan_resources = edma_alloc_chan_resources;
1007         dma->device_free_chan_resources = edma_free_chan_resources;
1008         dma->device_issue_pending = edma_issue_pending;
1009         dma->device_tx_status = edma_tx_status;
1010         dma->device_control = edma_control;
1011         dma->device_slave_caps = edma_dma_device_slave_caps;
1012         dma->dev = dev;
1013
1014         /*
1015          * code using dma memcpy must make sure alignment of
1016          * length is at dma->copy_align boundary.
1017          */
1018         dma->copy_align = DMA_SLAVE_BUSWIDTH_4_BYTES;
1019
1020         INIT_LIST_HEAD(&dma->channels);
1021 }
1022
1023 static int edma_probe(struct platform_device *pdev)
1024 {
1025         struct edma_cc *ecc;
1026         int ret;
1027
1028         ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
1029         if (ret)
1030                 return ret;
1031
1032         ecc = devm_kzalloc(&pdev->dev, sizeof(*ecc), GFP_KERNEL);
1033         if (!ecc) {
1034                 dev_err(&pdev->dev, "Can't allocate controller\n");
1035                 return -ENOMEM;
1036         }
1037
1038         ecc->ctlr = pdev->id;
1039         ecc->dummy_slot = edma_alloc_slot(ecc->ctlr, EDMA_SLOT_ANY);
1040         if (ecc->dummy_slot < 0) {
1041                 dev_err(&pdev->dev, "Can't allocate PaRAM dummy slot\n");
1042                 return -EIO;
1043         }
1044
1045         dma_cap_zero(ecc->dma_slave.cap_mask);
1046         dma_cap_set(DMA_SLAVE, ecc->dma_slave.cap_mask);
1047         dma_cap_set(DMA_CYCLIC, ecc->dma_slave.cap_mask);
1048         dma_cap_set(DMA_MEMCPY, ecc->dma_slave.cap_mask);
1049
1050         edma_dma_init(ecc, &ecc->dma_slave, &pdev->dev);
1051
1052         edma_chan_init(ecc, &ecc->dma_slave, ecc->slave_chans);
1053
1054         ret = dma_async_device_register(&ecc->dma_slave);
1055         if (ret)
1056                 goto err_reg1;
1057
1058         platform_set_drvdata(pdev, ecc);
1059
1060         dev_info(&pdev->dev, "TI EDMA DMA engine driver\n");
1061
1062         return 0;
1063
1064 err_reg1:
1065         edma_free_slot(ecc->dummy_slot);
1066         return ret;
1067 }
1068
1069 static int edma_remove(struct platform_device *pdev)
1070 {
1071         struct device *dev = &pdev->dev;
1072         struct edma_cc *ecc = dev_get_drvdata(dev);
1073
1074         dma_async_device_unregister(&ecc->dma_slave);
1075         edma_free_slot(ecc->dummy_slot);
1076
1077         return 0;
1078 }
1079
1080 static struct platform_driver edma_driver = {
1081         .probe          = edma_probe,
1082         .remove         = edma_remove,
1083         .driver = {
1084                 .name = "edma-dma-engine",
1085                 .owner = THIS_MODULE,
1086         },
1087 };
1088
1089 bool edma_filter_fn(struct dma_chan *chan, void *param)
1090 {
1091         if (chan->device->dev->driver == &edma_driver.driver) {
1092                 struct edma_chan *echan = to_edma_chan(chan);
1093                 unsigned ch_req = *(unsigned *)param;
1094                 return ch_req == echan->ch_num;
1095         }
1096         return false;
1097 }
1098 EXPORT_SYMBOL(edma_filter_fn);
1099
1100 static struct platform_device *pdev0, *pdev1;
1101
1102 static const struct platform_device_info edma_dev_info0 = {
1103         .name = "edma-dma-engine",
1104         .id = 0,
1105         .dma_mask = DMA_BIT_MASK(32),
1106 };
1107
1108 static const struct platform_device_info edma_dev_info1 = {
1109         .name = "edma-dma-engine",
1110         .id = 1,
1111         .dma_mask = DMA_BIT_MASK(32),
1112 };
1113
1114 static int edma_init(void)
1115 {
1116         int ret = platform_driver_register(&edma_driver);
1117
1118         if (ret == 0) {
1119                 pdev0 = platform_device_register_full(&edma_dev_info0);
1120                 if (IS_ERR(pdev0)) {
1121                         platform_driver_unregister(&edma_driver);
1122                         ret = PTR_ERR(pdev0);
1123                         goto out;
1124                 }
1125         }
1126
1127         if (EDMA_CTLRS == 2) {
1128                 pdev1 = platform_device_register_full(&edma_dev_info1);
1129                 if (IS_ERR(pdev1)) {
1130                         platform_driver_unregister(&edma_driver);
1131                         platform_device_unregister(pdev0);
1132                         ret = PTR_ERR(pdev1);
1133                 }
1134         }
1135
1136 out:
1137         return ret;
1138 }
1139 subsys_initcall(edma_init);
1140
1141 static void __exit edma_exit(void)
1142 {
1143         platform_device_unregister(pdev0);
1144         if (pdev1)
1145                 platform_device_unregister(pdev1);
1146         platform_driver_unregister(&edma_driver);
1147 }
1148 module_exit(edma_exit);
1149
1150 MODULE_AUTHOR("Matt Porter <matt.porter@linaro.org>");
1151 MODULE_DESCRIPTION("TI EDMA DMA engine driver");
1152 MODULE_LICENSE("GPL v2");