drivers/dma/edma.c

   1 /*
   2  * TI EDMA DMA engine driver
   3  *
   4  * Copyright 2012 Texas Instruments
   5  *
   6  * This program is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU General Public License as
   8  * published by the Free Software Foundation version 2.
   9  *
  10  * This program is distributed "as is" WITHOUT ANY WARRANTY of any
  11  * kind, whether express or implied; without even the implied warranty
  12  * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  */
  15
  16 #include <linux/dmaengine.h>
  17 #include <linux/dma-mapping.h>
  18 #include <linux/edma.h>
  19 #include <linux/err.h>
  20 #include <linux/init.h>
  21 #include <linux/interrupt.h>
  22 #include <linux/list.h>
  23 #include <linux/module.h>
  24 #include <linux/platform_device.h>
  25 #include <linux/slab.h>
  26 #include <linux/spinlock.h>
  27 #include <linux/of.h>
  28
  29 #include <linux/platform_data/edma.h>
  30
  31 #include "dmaengine.h"
  32 #include "virt-dma.h"
  33
  34 /*
  35  * This will go away when the private EDMA API is folded
  36  * into this driver and the platform device(s) are
  37  * instantiated in the arch code. We can only get away
  38  * with this simplification because DA8XX may not be built
  39  * in the same kernel image with other DaVinci parts. This
  40  * avoids having to sprinkle dmaengine driver platform devices
  41  * and data throughout all the existing board files.
  42  */
  43 #ifdef CONFIG_ARCH_DAVINCI_DA8XX
  44 #define EDMA_CTLRS      2
  45 #define EDMA_CHANS      32
  46 #else
  47 #define EDMA_CTLRS      1
  48 #define EDMA_CHANS      64
  49 #endif /* CONFIG_ARCH_DAVINCI_DA8XX */
  50
  51 /*
  52  * Max of 20 segments per channel to conserve PaRAM slots
  53  * Also note that MAX_NR_SG should be atleast the no.of periods
  54  * that are required for ASoC, otherwise DMA prep calls will
  55  * fail. Today davinci-pcm is the only user of this driver and
  56  * requires atleast 17 slots, so we setup the default to 20.
  57  */
  58 #define MAX_NR_SG               20
  59 #define EDMA_MAX_SLOTS          MAX_NR_SG
  60 #define EDMA_DESCRIPTORS        16
  61
  62 struct edma_pset {
  63         u32                             len;
  64         dma_addr_t                      addr;
  65         struct edmacc_param             param;
  66 };
  67
  68 struct edma_desc {
  69         struct virt_dma_desc            vdesc;
  70         struct list_head                node;
  71         enum dma_transfer_direction     direction;
  72         int                             cyclic;
  73         int                             absync;
  74         int                             pset_nr;
  75         struct edma_chan                *echan;
  76         int                             processed;
  77
  78         /*
  79          * The following 4 elements are used for residue accounting.
  80          *
  81          * - processed_stat: the number of SG elements we have traversed
  82          * so far to cover accounting. This is updated directly to processed
  83          * during edma_callback and is always <= processed, because processed
  84          * refers to the number of pending transfer (programmed to EDMA
  85          * controller), where as processed_stat tracks number of transfers
  86          * accounted for so far.
  87          *
  88          * - residue: The amount of bytes we have left to transfer for this desc
  89          *
  90          * - residue_stat: The residue in bytes of data we have covered
  91          * so far for accounting. This is updated directly to residue
  92          * during callbacks to keep it current.
  93          *
  94          * - sg_len: Tracks the length of the current intermediate transfer,
  95          * this is required to update the residue during intermediate transfer
  96          * completion callback.
  97          */
  98         int                             processed_stat;
  99         u32                             sg_len;
 100         u32                             residue;
 101         u32                             residue_stat;
 102
 103         struct edma_pset                pset[0];
 104 };
 105
 106 struct edma_cc;
 107
 108 struct edma_chan {
 109         struct virt_dma_chan            vchan;
 110         struct list_head                node;
 111         struct edma_desc                *edesc;
 112         struct edma_cc                  *ecc;
 113         int                             ch_num;
 114         bool                            alloced;
 115         int                             slot[EDMA_MAX_SLOTS];
 116         int                             missed;
 117         struct dma_slave_config         cfg;
 118 };
 119
 120 struct edma_cc {
 121         int                             ctlr;
 122         struct dma_device               dma_slave;
 123         struct edma_chan                slave_chans[EDMA_CHANS];
 124         int                             num_slave_chans;
 125         int                             dummy_slot;
 126 };
 127
 128 static inline struct edma_cc *to_edma_cc(struct dma_device *d)
 129 {
 130         return container_of(d, struct edma_cc, dma_slave);
 131 }
 132
 133 static inline struct edma_chan *to_edma_chan(struct dma_chan *c)
 134 {
 135         return container_of(c, struct edma_chan, vchan.chan);
 136 }
 137
 138 static inline struct edma_desc
 139 *to_edma_desc(struct dma_async_tx_descriptor *tx)
 140 {
 141         return container_of(tx, struct edma_desc, vdesc.tx);
 142 }
 143
 144 static void edma_desc_free(struct virt_dma_desc *vdesc)
 145 {
 146         kfree(container_of(vdesc, struct edma_desc, vdesc));
 147 }
 148
 149 /* Dispatch a queued descriptor to the controller (caller holds lock) */
 150 static void edma_execute(struct edma_chan *echan)
 151 {
 152         struct virt_dma_desc *vdesc;
 153         struct edma_desc *edesc;
 154         struct device *dev = echan->vchan.chan.device->dev;
 155         int i, j, left, nslots;
 156
 157         /* If either we processed all psets or we're still not started */
 158         if (!echan->edesc ||
 159             echan->edesc->pset_nr == echan->edesc->processed) {
 160                 /* Get next vdesc */
 161                 vdesc = vchan_next_desc(&echan->vchan);
 162                 if (!vdesc) {
 163                         echan->edesc = NULL;
 164                         return;
 165                 }
 166                 list_del(&vdesc->node);
 167                 echan->edesc = to_edma_desc(&vdesc->tx);
 168         }
 169
 170         edesc = echan->edesc;
 171
 172         /* Find out how many left */
 173         left = edesc->pset_nr - edesc->processed;
 174         nslots = min(MAX_NR_SG, left);
 175         edesc->sg_len = 0;
 176
 177         /* Write descriptor PaRAM set(s) */
 178         for (i = 0; i < nslots; i++) {
 179                 j = i + edesc->processed;
 180                 edma_write_slot(echan->slot[i], &edesc->pset[j].param);
 181                 edesc->sg_len += edesc->pset[j].len;
 182                 dev_vdbg(echan->vchan.chan.device->dev,
 183                         "\n pset[%d]:\n"
 184                         "  chnum\t%d\n"
 185                         "  slot\t%d\n"
 186                         "  opt\t%08x\n"
 187                         "  src\t%08x\n"
 188                         "  dst\t%08x\n"
 189                         "  abcnt\t%08x\n"
 190                         "  ccnt\t%08x\n"
 191                         "  bidx\t%08x\n"
 192                         "  cidx\t%08x\n"
 193                         "  lkrld\t%08x\n",
 194                         j, echan->ch_num, echan->slot[i],
 195                         edesc->pset[j].param.opt,
 196                         edesc->pset[j].param.src,
 197                         edesc->pset[j].param.dst,
 198                         edesc->pset[j].param.a_b_cnt,
 199                         edesc->pset[j].param.ccnt,
 200                         edesc->pset[j].param.src_dst_bidx,
 201                         edesc->pset[j].param.src_dst_cidx,
 202                         edesc->pset[j].param.link_bcntrld);
 203                 /* Link to the previous slot if not the last set */
 204                 if (i != (nslots - 1))
 205                         edma_link(echan->slot[i], echan->slot[i+1]);
 206         }
 207
 208         edesc->processed += nslots;
 209
 210         /*
 211          * If this is either the last set in a set of SG-list transactions
 212          * then setup a link to the dummy slot, this results in all future
 213          * events being absorbed and that's OK because we're done
 214          */
 215         if (edesc->processed == edesc->pset_nr) {
 216                 if (edesc->cyclic)
 217                         edma_link(echan->slot[nslots-1], echan->slot[1]);
 218                 else
 219                         edma_link(echan->slot[nslots-1],
 220                                   echan->ecc->dummy_slot);
 221         }
 222
 223         if (edesc->processed <= MAX_NR_SG) {
 224                 dev_dbg(dev, "first transfer starting on channel %d\n",
 225                         echan->ch_num);
 226                 edma_start(echan->ch_num);
 227         } else {
 228                 dev_dbg(dev, "chan: %d: completed %d elements, resuming\n",
 229                         echan->ch_num, edesc->processed);
 230                 edma_resume(echan->ch_num);
 231         }
 232
 233         /*
 234          * This happens due to setup times between intermediate transfers
 235          * in long SG lists which have to be broken up into transfers of
 236          * MAX_NR_SG
 237          */
 238         if (echan->missed) {
 239                 dev_dbg(dev, "missed event on channel %d\n", echan->ch_num);
 240                 edma_clean_channel(echan->ch_num);
 241                 edma_stop(echan->ch_num);
 242                 edma_start(echan->ch_num);
 243                 edma_trigger_channel(echan->ch_num);
 244                 echan->missed = 0;
 245         }
 246 }
 247
 248 static int edma_terminate_all(struct dma_chan *chan)
 249 {
 250         struct edma_chan *echan = to_edma_chan(chan);
 251         unsigned long flags;
 252         LIST_HEAD(head);
 253
 254         spin_lock_irqsave(&echan->vchan.lock, flags);
 255
 256         /*
 257          * Stop DMA activity: we assume the callback will not be called
 258          * after edma_dma() returns (even if it does, it will see
 259          * echan->edesc is NULL and exit.)
 260          */
 261         if (echan->edesc) {
 262                 int cyclic = echan->edesc->cyclic;
 263
 264                 /*
 265                  * free the running request descriptor
 266                  * since it is not in any of the vdesc lists
 267                  */
 268                 edma_desc_free(&echan->edesc->vdesc);
 269
 270                 echan->edesc = NULL;
 271                 edma_stop(echan->ch_num);
 272                 /* Move the cyclic channel back to default queue */
 273                 if (cyclic)
 274                         edma_assign_channel_eventq(echan->ch_num,
 275                                                    EVENTQ_DEFAULT);
 276         }
 277
 278         vchan_get_all_descriptors(&echan->vchan, &head);
 279         spin_unlock_irqrestore(&echan->vchan.lock, flags);
 280         vchan_dma_desc_free_list(&echan->vchan, &head);
 281
 282         return 0;
 283 }
 284
 285 static int edma_slave_config(struct dma_chan *chan,
 286         struct dma_slave_config *cfg)
 287 {
 288         struct edma_chan *echan = to_edma_chan(chan);
 289
 290         if (cfg->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES ||
 291             cfg->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)
 292                 return -EINVAL;
 293
 294         memcpy(&echan->cfg, cfg, sizeof(echan->cfg));
 295
 296         return 0;
 297 }
 298
 299 static int edma_dma_pause(struct dma_chan *chan)
 300 {
 301         struct edma_chan *echan = to_edma_chan(chan);
 302
 303         if (!echan->edesc)
 304                 return -EINVAL;
 305
 306         edma_pause(echan->ch_num);
 307         return 0;
 308 }
 309
 310 static int edma_dma_resume(struct dma_chan *chan)
 311 {
 312         struct edma_chan *echan = to_edma_chan(chan);
 313
 314         edma_resume(echan->ch_num);
 315         return 0;
 316 }
 317
 318 /*
 319  * A PaRAM set configuration abstraction used by other modes
 320  * @chan: Channel who's PaRAM set we're configuring
 321  * @pset: PaRAM set to initialize and setup.
 322  * @src_addr: Source address of the DMA
 323  * @dst_addr: Destination address of the DMA
 324  * @burst: In units of dev_width, how much to send
 325  * @dev_width: How much is the dev_width
 326  * @dma_length: Total length of the DMA transfer
 327  * @direction: Direction of the transfer
 328  */
 329 static int edma_config_pset(struct dma_chan *chan, struct edma_pset *epset,
 330         dma_addr_t src_addr, dma_addr_t dst_addr, u32 burst,
 331         enum dma_slave_buswidth dev_width, unsigned int dma_length,
 332         enum dma_transfer_direction direction)
 333 {
 334         struct edma_chan *echan = to_edma_chan(chan);
 335         struct device *dev = chan->device->dev;
 336         struct edmacc_param *param = &epset->param;
 337         int acnt, bcnt, ccnt, cidx;
 338         int src_bidx, dst_bidx, src_cidx, dst_cidx;
 339         int absync;
 340
 341         acnt = dev_width;
 342
 343         /* src/dst_maxburst == 0 is the same case as src/dst_maxburst == 1 */
 344         if (!burst)
 345                 burst = 1;
 346         /*
 347          * If the maxburst is equal to the fifo width, use
 348          * A-synced transfers. This allows for large contiguous
 349          * buffer transfers using only one PaRAM set.
 350          */
 351         if (burst == 1) {
 352                 /*
 353                  * For the A-sync case, bcnt and ccnt are the remainder
 354                  * and quotient respectively of the division of:
 355                  * (dma_length / acnt) by (SZ_64K -1). This is so
 356                  * that in case bcnt over flows, we have ccnt to use.
 357                  * Note: In A-sync tranfer only, bcntrld is used, but it
 358                  * only applies for sg_dma_len(sg) >= SZ_64K.
 359                  * In this case, the best way adopted is- bccnt for the
 360                  * first frame will be the remainder below. Then for
 361                  * every successive frame, bcnt will be SZ_64K-1. This
 362                  * is assured as bcntrld = 0xffff in end of function.
 363                  */
 364                 absync = false;
 365                 ccnt = dma_length / acnt / (SZ_64K - 1);
 366                 bcnt = dma_length / acnt - ccnt * (SZ_64K - 1);
 367                 /*
 368                  * If bcnt is non-zero, we have a remainder and hence an
 369                  * extra frame to transfer, so increment ccnt.
 370                  */
 371                 if (bcnt)
 372                         ccnt++;
 373                 else
 374                         bcnt = SZ_64K - 1;
 375                 cidx = acnt;
 376         } else {
 377                 /*
 378                  * If maxburst is greater than the fifo address_width,
 379                  * use AB-synced transfers where A count is the fifo
 380                  * address_width and B count is the maxburst. In this
 381                  * case, we are limited to transfers of C count frames
 382                  * of (address_width * maxburst) where C count is limited
 383                  * to SZ_64K-1. This places an upper bound on the length
 384                  * of an SG segment that can be handled.
 385                  */
 386                 absync = true;
 387                 bcnt = burst;
 388                 ccnt = dma_length / (acnt * bcnt);
 389                 if (ccnt > (SZ_64K - 1)) {
 390                         dev_err(dev, "Exceeded max SG segment size\n");
 391                         return -EINVAL;
 392                 }
 393                 cidx = acnt * bcnt;
 394         }
 395
 396         epset->len = dma_length;
 397
 398         if (direction == DMA_MEM_TO_DEV) {
 399                 src_bidx = acnt;
 400                 src_cidx = cidx;
 401                 dst_bidx = 0;
 402                 dst_cidx = 0;
 403                 epset->addr = src_addr;
 404         } else if (direction == DMA_DEV_TO_MEM)  {
 405                 src_bidx = 0;
 406                 src_cidx = 0;
 407                 dst_bidx = acnt;
 408                 dst_cidx = cidx;
 409                 epset->addr = dst_addr;
 410         } else if (direction == DMA_MEM_TO_MEM)  {
 411                 src_bidx = acnt;
 412                 src_cidx = cidx;
 413                 dst_bidx = acnt;
 414                 dst_cidx = cidx;
 415         } else {
 416                 dev_err(dev, "%s: direction not implemented yet\n", __func__);
 417                 return -EINVAL;
 418         }
 419
 420         param->opt = EDMA_TCC(EDMA_CHAN_SLOT(echan->ch_num));
 421         /* Configure A or AB synchronized transfers */
 422         if (absync)
 423                 param->opt |= SYNCDIM;
 424
 425         param->src = src_addr;
 426         param->dst = dst_addr;
 427
 428         param->src_dst_bidx = (dst_bidx << 16) | src_bidx;
 429         param->src_dst_cidx = (dst_cidx << 16) | src_cidx;
 430
 431         param->a_b_cnt = bcnt << 16 | acnt;
 432         param->ccnt = ccnt;
 433         /*
 434          * Only time when (bcntrld) auto reload is required is for
 435          * A-sync case, and in this case, a requirement of reload value
 436          * of SZ_64K-1 only is assured. 'link' is initially set to NULL
 437          * and then later will be populated by edma_execute.
 438          */
 439         param->link_bcntrld = 0xffffffff;
 440         return absync;
 441 }
 442
 443 static struct dma_async_tx_descriptor *edma_prep_slave_sg(
 444         struct dma_chan *chan, struct scatterlist *sgl,
 445         unsigned int sg_len, enum dma_transfer_direction direction,
 446         unsigned long tx_flags, void *context)
 447 {
 448         struct edma_chan *echan = to_edma_chan(chan);
 449         struct device *dev = chan->device->dev;
 450         struct edma_desc *edesc;
 451         dma_addr_t src_addr = 0, dst_addr = 0;
 452         enum dma_slave_buswidth dev_width;
 453         u32 burst;
 454         struct scatterlist *sg;
 455         int i, nslots, ret;
 456
 457         if (unlikely(!echan || !sgl || !sg_len))
 458                 return NULL;
 459
 460         if (direction == DMA_DEV_TO_MEM) {
 461                 src_addr = echan->cfg.src_addr;
 462                 dev_width = echan->cfg.src_addr_width;
 463                 burst = echan->cfg.src_maxburst;
 464         } else if (direction == DMA_MEM_TO_DEV) {
 465                 dst_addr = echan->cfg.dst_addr;
 466                 dev_width = echan->cfg.dst_addr_width;
 467                 burst = echan->cfg.dst_maxburst;
 468         } else {
 469                 dev_err(dev, "%s: bad direction: %d\n", __func__, direction);
 470                 return NULL;
 471         }
 472
 473         if (dev_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) {
 474                 dev_err(dev, "%s: Undefined slave buswidth\n", __func__);
 475                 return NULL;
 476         }
 477
 478         edesc = kzalloc(sizeof(*edesc) + sg_len *
 479                 sizeof(edesc->pset[0]), GFP_ATOMIC);
 480         if (!edesc) {
 481                 dev_err(dev, "%s: Failed to allocate a descriptor\n", __func__);
 482                 return NULL;
 483         }
 484
 485         edesc->pset_nr = sg_len;
 486         edesc->residue = 0;
 487         edesc->direction = direction;
 488         edesc->echan = echan;
 489
 490         /* Allocate a PaRAM slot, if needed */
 491         nslots = min_t(unsigned, MAX_NR_SG, sg_len);
 492
 493         for (i = 0; i < nslots; i++) {
 494                 if (echan->slot[i] < 0) {
 495                         echan->slot[i] =
 496                                 edma_alloc_slot(EDMA_CTLR(echan->ch_num),
 497                                                 EDMA_SLOT_ANY);
 498                         if (echan->slot[i] < 0) {
 499                                 kfree(edesc);
 500                                 dev_err(dev, "%s: Failed to allocate slot\n",
 501                                         __func__);
 502                                 return NULL;
 503                         }
 504                 }
 505         }
 506
 507         /* Configure PaRAM sets for each SG */
 508         for_each_sg(sgl, sg, sg_len, i) {
 509                 /* Get address for each SG */
 510                 if (direction == DMA_DEV_TO_MEM)
 511                         dst_addr = sg_dma_address(sg);
 512                 else
 513                         src_addr = sg_dma_address(sg);
 514
 515                 ret = edma_config_pset(chan, &edesc->pset[i], src_addr,
 516                                        dst_addr, burst, dev_width,
 517                                        sg_dma_len(sg), direction);
 518                 if (ret < 0) {
 519                         kfree(edesc);
 520                         return NULL;
 521                 }
 522
 523                 edesc->absync = ret;
 524                 edesc->residue += sg_dma_len(sg);
 525
 526                 /* If this is the last in a current SG set of transactions,
 527                    enable interrupts so that next set is processed */
 528                 if (!((i+1) % MAX_NR_SG))
 529                         edesc->pset[i].param.opt |= TCINTEN;
 530
 531                 /* If this is the last set, enable completion interrupt flag */
 532                 if (i == sg_len - 1)
 533                         edesc->pset[i].param.opt |= TCINTEN;
 534         }
 535         edesc->residue_stat = edesc->residue;
 536
 537         return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags);
 538 }
 539
 540 static struct dma_async_tx_descriptor *edma_prep_dma_memcpy(
 541         struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
 542         size_t len, unsigned long tx_flags)
 543 {
 544         int ret;
 545         struct edma_desc *edesc;
 546         struct device *dev = chan->device->dev;
 547         struct edma_chan *echan = to_edma_chan(chan);
 548
 549         if (unlikely(!echan || !len))
 550                 return NULL;
 551
 552         edesc = kzalloc(sizeof(*edesc) + sizeof(edesc->pset[0]), GFP_ATOMIC);
 553         if (!edesc) {
 554                 dev_dbg(dev, "Failed to allocate a descriptor\n");
 555                 return NULL;
 556         }
 557
 558         edesc->pset_nr = 1;
 559
 560         ret = edma_config_pset(chan, &edesc->pset[0], src, dest, 1,
 561                                DMA_SLAVE_BUSWIDTH_4_BYTES, len, DMA_MEM_TO_MEM);
 562         if (ret < 0)
 563                 return NULL;
 564
 565         edesc->absync = ret;
 566
 567         /*
 568          * Enable intermediate transfer chaining to re-trigger channel
 569          * on completion of every TR, and enable transfer-completion
 570          * interrupt on completion of the whole transfer.
 571          */
 572         edesc->pset[0].param.opt |= ITCCHEN;
 573         edesc->pset[0].param.opt |= TCINTEN;
 574
 575         return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags);
 576 }
 577
 578 static struct dma_async_tx_descriptor *edma_prep_dma_cyclic(
 579         struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
 580         size_t period_len, enum dma_transfer_direction direction,
 581         unsigned long tx_flags)
 582 {
 583         struct edma_chan *echan = to_edma_chan(chan);
 584         struct device *dev = chan->device->dev;
 585         struct edma_desc *edesc;
 586         dma_addr_t src_addr, dst_addr;
 587         enum dma_slave_buswidth dev_width;
 588         u32 burst;
 589         int i, ret, nslots;
 590
 591         if (unlikely(!echan || !buf_len || !period_len))
 592                 return NULL;
 593
 594         if (direction == DMA_DEV_TO_MEM) {
 595                 src_addr = echan->cfg.src_addr;
 596                 dst_addr = buf_addr;
 597                 dev_width = echan->cfg.src_addr_width;
 598                 burst = echan->cfg.src_maxburst;
 599         } else if (direction == DMA_MEM_TO_DEV) {
 600                 src_addr = buf_addr;
 601                 dst_addr = echan->cfg.dst_addr;
 602                 dev_width = echan->cfg.dst_addr_width;
 603                 burst = echan->cfg.dst_maxburst;
 604         } else {
 605                 dev_err(dev, "%s: bad direction: %d\n", __func__, direction);
 606                 return NULL;
 607         }
 608
 609         if (dev_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) {
 610                 dev_err(dev, "%s: Undefined slave buswidth\n", __func__);
 611                 return NULL;
 612         }
 613
 614         if (unlikely(buf_len % period_len)) {
 615                 dev_err(dev, "Period should be multiple of Buffer length\n");
 616                 return NULL;
 617         }
 618
 619         nslots = (buf_len / period_len) + 1;
 620
 621         /*
 622          * Cyclic DMA users such as audio cannot tolerate delays introduced
 623          * by cases where the number of periods is more than the maximum
 624          * number of SGs the EDMA driver can handle at a time. For DMA types
 625          * such as Slave SGs, such delays are tolerable and synchronized,
 626          * but the synchronization is difficult to achieve with Cyclic and
 627          * cannot be guaranteed, so we error out early.
 628          */
 629         if (nslots > MAX_NR_SG)
 630                 return NULL;
 631
 632         edesc = kzalloc(sizeof(*edesc) + nslots *
 633                 sizeof(edesc->pset[0]), GFP_ATOMIC);
 634         if (!edesc) {
 635                 dev_err(dev, "%s: Failed to allocate a descriptor\n", __func__);
 636                 return NULL;
 637         }
 638
 639         edesc->cyclic = 1;
 640         edesc->pset_nr = nslots;
 641         edesc->residue = edesc->residue_stat = buf_len;
 642         edesc->direction = direction;
 643         edesc->echan = echan;
 644
 645         dev_dbg(dev, "%s: channel=%d nslots=%d period_len=%zu buf_len=%zu\n",
 646                 __func__, echan->ch_num, nslots, period_len, buf_len);
 647
 648         for (i = 0; i < nslots; i++) {
 649                 /* Allocate a PaRAM slot, if needed */
 650                 if (echan->slot[i] < 0) {
 651                         echan->slot[i] =
 652                                 edma_alloc_slot(EDMA_CTLR(echan->ch_num),
 653                                                 EDMA_SLOT_ANY);
 654                         if (echan->slot[i] < 0) {
 655                                 kfree(edesc);
 656                                 dev_err(dev, "%s: Failed to allocate slot\n",
 657                                         __func__);
 658                                 return NULL;
 659                         }
 660                 }
 661
 662                 if (i == nslots - 1) {
 663                         memcpy(&edesc->pset[i], &edesc->pset[0],
 664                                sizeof(edesc->pset[0]));
 665                         break;
 666                 }
 667
 668                 ret = edma_config_pset(chan, &edesc->pset[i], src_addr,
 669                                        dst_addr, burst, dev_width, period_len,
 670                                        direction);
 671                 if (ret < 0) {
 672                         kfree(edesc);
 673                         return NULL;
 674                 }
 675
 676                 if (direction == DMA_DEV_TO_MEM)
 677                         dst_addr += period_len;
 678                 else
 679                         src_addr += period_len;
 680
 681                 dev_vdbg(dev, "%s: Configure period %d of buf:\n", __func__, i);
 682                 dev_vdbg(dev,
 683                         "\n pset[%d]:\n"
 684                         "  chnum\t%d\n"
 685                         "  slot\t%d\n"
 686                         "  opt\t%08x\n"
 687                         "  src\t%08x\n"
 688                         "  dst\t%08x\n"
 689                         "  abcnt\t%08x\n"
 690                         "  ccnt\t%08x\n"
 691                         "  bidx\t%08x\n"
 692                         "  cidx\t%08x\n"
 693                         "  lkrld\t%08x\n",
 694                         i, echan->ch_num, echan->slot[i],
 695                         edesc->pset[i].param.opt,
 696                         edesc->pset[i].param.src,
 697                         edesc->pset[i].param.dst,
 698                         edesc->pset[i].param.a_b_cnt,
 699                         edesc->pset[i].param.ccnt,
 700                         edesc->pset[i].param.src_dst_bidx,
 701                         edesc->pset[i].param.src_dst_cidx,
 702                         edesc->pset[i].param.link_bcntrld);
 703
 704                 edesc->absync = ret;
 705
 706                 /*
 707                  * Enable period interrupt only if it is requested
 708                  */
 709                 if (tx_flags & DMA_PREP_INTERRUPT)
 710                         edesc->pset[i].param.opt |= TCINTEN;
 711         }
 712
 713         /* Place the cyclic channel to highest priority queue */
 714         edma_assign_channel_eventq(echan->ch_num, EVENTQ_0);
 715
 716         return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags);
 717 }
 718
 719 static void edma_callback(unsigned ch_num, u16 ch_status, void *data)
 720 {
 721         struct edma_chan *echan = data;
 722         struct device *dev = echan->vchan.chan.device->dev;
 723         struct edma_desc *edesc;
 724         struct edmacc_param p;
 725
 726         edesc = echan->edesc;
 727
 728         /* Pause the channel for non-cyclic */
 729         if (!edesc || (edesc && !edesc->cyclic))
 730                 edma_pause(echan->ch_num);
 731
 732         switch (ch_status) {
 733         case EDMA_DMA_COMPLETE:
 734                 spin_lock(&echan->vchan.lock);
 735
 736                 if (edesc) {
 737                         if (edesc->cyclic) {
 738                                 vchan_cyclic_callback(&edesc->vdesc);
 739                         } else if (edesc->processed == edesc->pset_nr) {
 740                                 dev_dbg(dev, "Transfer complete, stopping channel %d\n", ch_num);
 741                                 edesc->residue = 0;
 742                                 edma_stop(echan->ch_num);
 743                                 vchan_cookie_complete(&edesc->vdesc);
 744                                 edma_execute(echan);
 745                         } else {
 746                                 dev_dbg(dev, "Intermediate transfer complete on channel %d\n", ch_num);
 747
 748                                 /* Update statistics for tx_status */
 749                                 edesc->residue -= edesc->sg_len;
 750                                 edesc->residue_stat = edesc->residue;
 751                                 edesc->processed_stat = edesc->processed;
 752
 753                                 edma_execute(echan);
 754                         }
 755                 }
 756
 757                 spin_unlock(&echan->vchan.lock);
 758
 759                 break;
 760         case EDMA_DMA_CC_ERROR:
 761                 spin_lock(&echan->vchan.lock);
 762
 763                 edma_read_slot(EDMA_CHAN_SLOT(echan->slot[0]), &p);
 764
 765                 /*
 766                  * Issue later based on missed flag which will be sure
 767                  * to happen as:
 768                  * (1) we finished transmitting an intermediate slot and
 769                  *     edma_execute is coming up.
 770                  * (2) or we finished current transfer and issue will
 771                  *     call edma_execute.
 772                  *
 773                  * Important note: issuing can be dangerous here and
 774                  * lead to some nasty recursion when we are in a NULL
 775                  * slot. So we avoid doing so and set the missed flag.
 776                  */
 777                 if (p.a_b_cnt == 0 && p.ccnt == 0) {
 778                         dev_dbg(dev, "Error occurred, looks like slot is null, just setting miss\n");
 779                         echan->missed = 1;
 780                 } else {
 781                         /*
 782                          * The slot is already programmed but the event got
 783                          * missed, so its safe to issue it here.
 784                          */
 785                         dev_dbg(dev, "Error occurred but slot is non-null, TRIGGERING\n");
 786                         edma_clean_channel(echan->ch_num);
 787                         edma_stop(echan->ch_num);
 788                         edma_start(echan->ch_num);
 789                         edma_trigger_channel(echan->ch_num);
 790                 }
 791
 792                 spin_unlock(&echan->vchan.lock);
 793
 794                 break;
 795         default:
 796                 break;
 797         }
 798 }
 799
 800 /* Alloc channel resources */
 801 static int edma_alloc_chan_resources(struct dma_chan *chan)
 802 {
 803         struct edma_chan *echan = to_edma_chan(chan);
 804         struct device *dev = chan->device->dev;
 805         int ret;
 806         int a_ch_num;
 807         LIST_HEAD(descs);
 808
 809         a_ch_num = edma_alloc_channel(echan->ch_num, edma_callback,
 810                                         echan, EVENTQ_DEFAULT);
 811
 812         if (a_ch_num < 0) {
 813                 ret = -ENODEV;
 814                 goto err_no_chan;
 815         }
 816
 817         if (a_ch_num != echan->ch_num) {
 818                 dev_err(dev, "failed to allocate requested channel %u:%u\n",
 819                         EDMA_CTLR(echan->ch_num),
 820                         EDMA_CHAN_SLOT(echan->ch_num));
 821                 ret = -ENODEV;
 822                 goto err_wrong_chan;
 823         }
 824
 825         echan->alloced = true;
 826         echan->slot[0] = echan->ch_num;
 827
 828         dev_dbg(dev, "allocated channel %d for %u:%u\n", echan->ch_num,
 829                 EDMA_CTLR(echan->ch_num), EDMA_CHAN_SLOT(echan->ch_num));
 830
 831         return 0;
 832
 833 err_wrong_chan:
 834         edma_free_channel(a_ch_num);
 835 err_no_chan:
 836         return ret;
 837 }
 838
 839 /* Free channel resources */
 840 static void edma_free_chan_resources(struct dma_chan *chan)
 841 {
 842         struct edma_chan *echan = to_edma_chan(chan);
 843         struct device *dev = chan->device->dev;
 844         int i;
 845
 846         /* Terminate transfers */
 847         edma_stop(echan->ch_num);
 848
 849         vchan_free_chan_resources(&echan->vchan);
 850
 851         /* Free EDMA PaRAM slots */
 852         for (i = 1; i < EDMA_MAX_SLOTS; i++) {
 853                 if (echan->slot[i] >= 0) {
 854                         edma_free_slot(echan->slot[i]);
 855                         echan->slot[i] = -1;
 856                 }
 857         }
 858
 859         /* Free EDMA channel */
 860         if (echan->alloced) {
 861                 edma_free_channel(echan->ch_num);
 862                 echan->alloced = false;
 863         }
 864
 865         dev_dbg(dev, "freeing channel for %u\n", echan->ch_num);
 866 }
 867
 868 /* Send pending descriptor to hardware */
 869 static void edma_issue_pending(struct dma_chan *chan)
 870 {
 871         struct edma_chan *echan = to_edma_chan(chan);
 872         unsigned long flags;
 873
 874         spin_lock_irqsave(&echan->vchan.lock, flags);
 875         if (vchan_issue_pending(&echan->vchan) && !echan->edesc)
 876                 edma_execute(echan);
 877         spin_unlock_irqrestore(&echan->vchan.lock, flags);
 878 }
 879
 880 static u32 edma_residue(struct edma_desc *edesc)
 881 {
 882         bool dst = edesc->direction == DMA_DEV_TO_MEM;
 883         struct edma_pset *pset = edesc->pset;
 884         dma_addr_t done, pos;
 885         int i;
 886
 887         /*
 888          * We always read the dst/src position from the first RamPar
 889          * pset. That's the one which is active now.
 890          */
 891         pos = edma_get_position(edesc->echan->slot[0], dst);
 892
 893         /*
 894          * Cyclic is simple. Just subtract pset[0].addr from pos.
 895          *
 896          * We never update edesc->residue in the cyclic case, so we
 897          * can tell the remaining room to the end of the circular
 898          * buffer.
 899          */
 900         if (edesc->cyclic) {
 901                 done = pos - pset->addr;
 902                 edesc->residue_stat = edesc->residue - done;
 903                 return edesc->residue_stat;
 904         }
 905
 906         /*
 907          * For SG operation we catch up with the last processed
 908          * status.
 909          */
 910         pset += edesc->processed_stat;
 911
 912         for (i = edesc->processed_stat; i < edesc->processed; i++, pset++) {
 913                 /*
 914                  * If we are inside this pset address range, we know
 915                  * this is the active one. Get the current delta and
 916                  * stop walking the psets.
 917                  */
 918                 if (pos >= pset->addr && pos < pset->addr + pset->len)
 919                         return edesc->residue_stat - (pos - pset->addr);
 920
 921                 /* Otherwise mark it done and update residue_stat. */
 922                 edesc->processed_stat++;
 923                 edesc->residue_stat -= pset->len;
 924         }
 925         return edesc->residue_stat;
 926 }
 927
 928 /* Check request completion status */
 929 static enum dma_status edma_tx_status(struct dma_chan *chan,
 930                                       dma_cookie_t cookie,
 931                                       struct dma_tx_state *txstate)
 932 {
 933         struct edma_chan *echan = to_edma_chan(chan);
 934         struct virt_dma_desc *vdesc;
 935         enum dma_status ret;
 936         unsigned long flags;
 937
 938         ret = dma_cookie_status(chan, cookie, txstate);
 939         if (ret == DMA_COMPLETE || !txstate)
 940                 return ret;
 941
 942         spin_lock_irqsave(&echan->vchan.lock, flags);
 943         if (echan->edesc && echan->edesc->vdesc.tx.cookie == cookie)
 944                 txstate->residue = edma_residue(echan->edesc);
 945         else if ((vdesc = vchan_find_desc(&echan->vchan, cookie)))
 946                 txstate->residue = to_edma_desc(&vdesc->tx)->residue;
 947         spin_unlock_irqrestore(&echan->vchan.lock, flags);
 948
 949         return ret;
 950 }
 951
 952 static void __init edma_chan_init(struct edma_cc *ecc,
 953                                   struct dma_device *dma,
 954                                   struct edma_chan *echans)
 955 {
 956         int i, j;
 957
 958         for (i = 0; i < EDMA_CHANS; i++) {
 959                 struct edma_chan *echan = &echans[i];
 960                 echan->ch_num = EDMA_CTLR_CHAN(ecc->ctlr, i);
 961                 echan->ecc = ecc;
 962                 echan->vchan.desc_free = edma_desc_free;
 963
 964                 vchan_init(&echan->vchan, dma);
 965
 966                 INIT_LIST_HEAD(&echan->node);
 967                 for (j = 0; j < EDMA_MAX_SLOTS; j++)
 968                         echan->slot[j] = -1;
 969         }
 970 }
 971
 972 #define EDMA_DMA_BUSWIDTHS      (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
 973                                  BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
 974                                  BIT(DMA_SLAVE_BUSWIDTH_3_BYTES) | \
 975                                  BIT(DMA_SLAVE_BUSWIDTH_4_BYTES))
 976
 977 static void edma_dma_init(struct edma_cc *ecc, struct dma_device *dma,
 978                           struct device *dev)
 979 {
 980         dma->device_prep_slave_sg = edma_prep_slave_sg;
 981         dma->device_prep_dma_cyclic = edma_prep_dma_cyclic;
 982         dma->device_prep_dma_memcpy = edma_prep_dma_memcpy;
 983         dma->device_alloc_chan_resources = edma_alloc_chan_resources;
 984         dma->device_free_chan_resources = edma_free_chan_resources;
 985         dma->device_issue_pending = edma_issue_pending;
 986         dma->device_tx_status = edma_tx_status;
 987         dma->device_config = edma_slave_config;
 988         dma->device_pause = edma_dma_pause;
 989         dma->device_resume = edma_dma_resume;
 990         dma->device_terminate_all = edma_terminate_all;
 991
 992         dma->src_addr_widths = EDMA_DMA_BUSWIDTHS;
 993         dma->dst_addr_widths = EDMA_DMA_BUSWIDTHS;
 994         dma->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
 995         dma->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
 996
 997         dma->dev = dev;
 998
 999         /*
1000          * code using dma memcpy must make sure alignment of
1001          * length is at dma->copy_align boundary.
1002          */
1003         dma->copy_align = DMAENGINE_ALIGN_4_BYTES;
1004
1005         INIT_LIST_HEAD(&dma->channels);
1006 }
1007
1008 static int edma_probe(struct platform_device *pdev)
1009 {
1010         struct edma_cc *ecc;
1011         int ret;
1012
1013         ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
1014         if (ret)
1015                 return ret;
1016
1017         ecc = devm_kzalloc(&pdev->dev, sizeof(*ecc), GFP_KERNEL);
1018         if (!ecc) {
1019                 dev_err(&pdev->dev, "Can't allocate controller\n");
1020                 return -ENOMEM;
1021         }
1022
1023         ecc->ctlr = pdev->id;
1024         ecc->dummy_slot = edma_alloc_slot(ecc->ctlr, EDMA_SLOT_ANY);
1025         if (ecc->dummy_slot < 0) {
1026                 dev_err(&pdev->dev, "Can't allocate PaRAM dummy slot\n");
1027                 return ecc->dummy_slot;
1028         }
1029
1030         dma_cap_zero(ecc->dma_slave.cap_mask);
1031         dma_cap_set(DMA_SLAVE, ecc->dma_slave.cap_mask);
1032         dma_cap_set(DMA_CYCLIC, ecc->dma_slave.cap_mask);
1033         dma_cap_set(DMA_MEMCPY, ecc->dma_slave.cap_mask);
1034
1035         edma_dma_init(ecc, &ecc->dma_slave, &pdev->dev);
1036
1037         edma_chan_init(ecc, &ecc->dma_slave, ecc->slave_chans);
1038
1039         ret = dma_async_device_register(&ecc->dma_slave);
1040         if (ret)
1041                 goto err_reg1;
1042
1043         platform_set_drvdata(pdev, ecc);
1044
1045         dev_info(&pdev->dev, "TI EDMA DMA engine driver\n");
1046
1047         return 0;
1048
1049 err_reg1:
1050         edma_free_slot(ecc->dummy_slot);
1051         return ret;
1052 }
1053
1054 static int edma_remove(struct platform_device *pdev)
1055 {
1056         struct device *dev = &pdev->dev;
1057         struct edma_cc *ecc = dev_get_drvdata(dev);
1058
1059         dma_async_device_unregister(&ecc->dma_slave);
1060         edma_free_slot(ecc->dummy_slot);
1061
1062         return 0;
1063 }
1064
1065 static struct platform_driver edma_driver = {
1066         .probe          = edma_probe,
1067         .remove         = edma_remove,
1068         .driver = {
1069                 .name = "edma-dma-engine",
1070         },
1071 };
1072
1073 bool edma_filter_fn(struct dma_chan *chan, void *param)
1074 {
1075         if (chan->device->dev->driver == &edma_driver.driver) {
1076                 struct edma_chan *echan = to_edma_chan(chan);
1077                 unsigned ch_req = *(unsigned *)param;
1078                 return ch_req == echan->ch_num;
1079         }
1080         return false;
1081 }
1082 EXPORT_SYMBOL(edma_filter_fn);
1083
1084 static int edma_init(void)
1085 {
1086         return platform_driver_register(&edma_driver);
1087 }
1088 subsys_initcall(edma_init);
1089
1090 static void __exit edma_exit(void)
1091 {
1092         platform_driver_unregister(&edma_driver);
1093 }
1094 module_exit(edma_exit);
1095
1096 MODULE_AUTHOR("Matt Porter <matt.porter@linaro.org>");
1097 MODULE_DESCRIPTION("TI EDMA DMA engine driver");
1098 MODULE_LICENSE("GPL v2");