drivers/dma/qcom_adm.c

   1 /*
   2  * Copyright (c) 2013-2015, The Linux Foundation. All rights reserved.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License version 2 and
   6  * only version 2 as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it will be useful,
   9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  11  * GNU General Public License for more details.
  12  *
  13  */
  14
  15 #include <linux/kernel.h>
  16 #include <linux/io.h>
  17 #include <linux/init.h>
  18 #include <linux/slab.h>
  19 #include <linux/module.h>
  20 #include <linux/interrupt.h>
  21 #include <linux/dma-mapping.h>
  22 #include <linux/scatterlist.h>
  23 #include <linux/device.h>
  24 #include <linux/platform_device.h>
  25 #include <linux/of.h>
  26 #include <linux/of_address.h>
  27 #include <linux/of_irq.h>
  28 #include <linux/of_dma.h>
  29 #include <linux/reset.h>
  30 #include <linux/clk.h>
  31 #include <linux/dmaengine.h>
  32
  33 #include "dmaengine.h"
  34 #include "virt-dma.h"
  35
  36 /* ADM registers - calculated from channel number and security domain */
  37 #define ADM_CHAN_MULTI                  0x4
  38 #define ADM_CI_MULTI                    0x4
  39 #define ADM_CRCI_MULTI                  0x4
  40 #define ADM_EE_MULTI                    0x800
  41 #define ADM_CHAN_OFFS(chan)             (ADM_CHAN_MULTI * chan)
  42 #define ADM_EE_OFFS(ee)                 (ADM_EE_MULTI * ee)
  43 #define ADM_CHAN_EE_OFFS(chan, ee)      (ADM_CHAN_OFFS(chan) + ADM_EE_OFFS(ee))
  44 #define ADM_CHAN_OFFS(chan)             (ADM_CHAN_MULTI * chan)
  45 #define ADM_CI_OFFS(ci)                 (ADM_CHAN_OFF(ci))
  46 #define ADM_CH_CMD_PTR(chan, ee)        (ADM_CHAN_EE_OFFS(chan, ee))
  47 #define ADM_CH_RSLT(chan, ee)           (0x40 + ADM_CHAN_EE_OFFS(chan, ee))
  48 #define ADM_CH_FLUSH_STATE0(chan, ee)   (0x80 + ADM_CHAN_EE_OFFS(chan, ee))
  49 #define ADM_CH_STATUS_SD(chan, ee)      (0x200 + ADM_CHAN_EE_OFFS(chan, ee))
  50 #define ADM_CH_CONF(chan)               (0x240 + ADM_CHAN_OFFS(chan))
  51 #define ADM_CH_RSLT_CONF(chan, ee)      (0x300 + ADM_CHAN_EE_OFFS(chan, ee))
  52 #define ADM_SEC_DOMAIN_IRQ_STATUS(ee)   (0x380 + ADM_EE_OFFS(ee))
  53 #define ADM_CI_CONF(ci)                 (0x390 + ci * ADM_CI_MULTI)
  54 #define ADM_GP_CTL                      0x3d8
  55 #define ADM_CRCI_CTL(crci, ee)          (0x400 + crci * ADM_CRCI_MULTI + \
  56                                                 ADM_EE_OFFS(ee))
  57
  58 /* channel status */
  59 #define ADM_CH_STATUS_VALID     BIT(1)
  60
  61 /* channel result */
  62 #define ADM_CH_RSLT_VALID       BIT(31)
  63 #define ADM_CH_RSLT_ERR         BIT(3)
  64 #define ADM_CH_RSLT_FLUSH       BIT(2)
  65 #define ADM_CH_RSLT_TPD         BIT(1)
  66
  67 /* channel conf */
  68 #define ADM_CH_CONF_SHADOW_EN           BIT(12)
  69 #define ADM_CH_CONF_MPU_DISABLE         BIT(11)
  70 #define ADM_CH_CONF_PERM_MPU_CONF       BIT(9)
  71 #define ADM_CH_CONF_FORCE_RSLT_EN       BIT(7)
  72 #define ADM_CH_CONF_SEC_DOMAIN(ee)      (((ee & 0x3) << 4) | ((ee & 0x4) << 11))
  73
  74 /* channel result conf */
  75 #define ADM_CH_RSLT_CONF_FLUSH_EN       BIT(1)
  76 #define ADM_CH_RSLT_CONF_IRQ_EN         BIT(0)
  77
  78 /* CRCI CTL */
  79 #define ADM_CRCI_CTL_MUX_SEL    BIT(18)
  80 #define ADM_CRCI_CTL_RST        BIT(17)
  81
  82 /* CI configuration */
  83 #define ADM_CI_RANGE_END(x)     (x << 24)
  84 #define ADM_CI_RANGE_START(x)   (x << 16)
  85 #define ADM_CI_BURST_4_WORDS    BIT(2)
  86 #define ADM_CI_BURST_8_WORDS    BIT(3)
  87
  88 /* GP CTL */
  89 #define ADM_GP_CTL_LP_EN        BIT(12)
  90 #define ADM_GP_CTL_LP_CNT(x)    (x << 8)
  91
  92 /* Command pointer list entry */
  93 #define ADM_CPLE_LP             BIT(31)
  94 #define ADM_CPLE_CMD_PTR_LIST   BIT(29)
  95
  96 /* Command list entry */
  97 #define ADM_CMD_LC              BIT(31)
  98 #define ADM_CMD_DST_CRCI(n)     (((n) & 0xf) << 7)
  99 #define ADM_CMD_SRC_CRCI(n)     (((n) & 0xf) << 3)
 100
 101 #define ADM_CMD_TYPE_SINGLE     0x0
 102 #define ADM_CMD_TYPE_BOX        0x3
 103
 104 #define ADM_CRCI_MUX_SEL        BIT(4)
 105 #define ADM_DESC_ALIGN          8
 106 #define ADM_MAX_XFER            (SZ_64K-1)
 107 #define ADM_MAX_ROWS            (SZ_64K-1)
 108 #define ADM_MAX_CHANNELS        16
 109
 110 struct adm_desc_hw_box {
 111         u32 cmd;
 112         u32 src_addr;
 113         u32 dst_addr;
 114         u32 row_len;
 115         u32 num_rows;
 116         u32 row_offset;
 117 };
 118
 119 struct adm_desc_hw_single {
 120         u32 cmd;
 121         u32 src_addr;
 122         u32 dst_addr;
 123         u32 len;
 124 };
 125
 126 struct adm_async_desc {
 127         struct virt_dma_desc vd;
 128         struct adm_device *adev;
 129
 130         size_t length;
 131         enum dma_transfer_direction dir;
 132         dma_addr_t dma_addr;
 133         size_t dma_len;
 134
 135         void *cpl;
 136         dma_addr_t cp_addr;
 137         u32 crci;
 138         u32 mux;
 139         u32 blk_size;
 140 };
 141
 142 struct adm_chan {
 143         struct virt_dma_chan vc;
 144         struct adm_device *adev;
 145
 146         /* parsed from DT */
 147         u32 id;                 /* channel id */
 148
 149         struct adm_async_desc *curr_txd;
 150         struct dma_slave_config slave;
 151         struct list_head node;
 152
 153         int error;
 154         int initialized;
 155 };
 156
 157 static inline struct adm_chan *to_adm_chan(struct dma_chan *common)
 158 {
 159         return container_of(common, struct adm_chan, vc.chan);
 160 }
 161
 162 struct adm_device {
 163         void __iomem *regs;
 164         struct device *dev;
 165         struct dma_device common;
 166         struct device_dma_parameters dma_parms;
 167         struct adm_chan *channels;
 168
 169         u32 ee;
 170
 171         struct clk *core_clk;
 172         struct clk *iface_clk;
 173
 174         struct reset_control *clk_reset;
 175         struct reset_control *c0_reset;
 176         struct reset_control *c1_reset;
 177         struct reset_control *c2_reset;
 178         int irq;
 179 };
 180
 181 /**
 182  * adm_free_chan - Frees dma resources associated with the specific channel
 183  *
 184  * Free all allocated descriptors associated with this channel
 185  *
 186  */
 187 static void adm_free_chan(struct dma_chan *chan)
 188 {
 189         /* free all queued descriptors */
 190         vchan_free_chan_resources(to_virt_chan(chan));
 191 }
 192
 193 /**
 194  * adm_get_blksize - Get block size from burst value
 195  *
 196  */
 197 static int adm_get_blksize(unsigned int burst)
 198 {
 199         int ret;
 200
 201         switch (burst) {
 202         case 16:
 203         case 32:
 204         case 64:
 205         case 128:
 206                 ret = ffs(burst>>4) - 1;
 207                 break;
 208         case 192:
 209                 ret = 4;
 210                 break;
 211         case 256:
 212                 ret = 5;
 213                 break;
 214         default:
 215                 ret = -EINVAL;
 216                 break;
 217         }
 218
 219         return ret;
 220 }
 221
 222 /**
 223  * adm_process_fc_descriptors - Process descriptors for flow controlled xfers
 224  *
 225  * @achan: ADM channel
 226  * @desc: Descriptor memory pointer
 227  * @sg: Scatterlist entry
 228  * @crci: CRCI value
 229  * @burst: Burst size of transaction
 230  * @direction: DMA transfer direction
 231  */
 232 static void *adm_process_fc_descriptors(struct adm_chan *achan,
 233         void *desc, struct scatterlist *sg, u32 crci, u32 burst,
 234         enum dma_transfer_direction direction)
 235 {
 236         struct adm_desc_hw_box *box_desc;
 237         struct adm_desc_hw_single *single_desc;
 238         u32 remainder = sg_dma_len(sg);
 239         u32 rows, row_offset, crci_cmd;
 240         u32 mem_addr = sg_dma_address(sg);
 241         u32 *incr_addr = &mem_addr;
 242         u32 *src, *dst;
 243
 244         if (direction == DMA_DEV_TO_MEM) {
 245                 crci_cmd = ADM_CMD_SRC_CRCI(crci);
 246                 row_offset = burst;
 247                 src = &achan->slave.src_addr;
 248                 dst = &mem_addr;
 249         } else {
 250                 crci_cmd = ADM_CMD_DST_CRCI(crci);
 251                 row_offset = burst << 16;
 252                 src = &mem_addr;
 253                 dst = &achan->slave.dst_addr;
 254         }
 255
 256         do {
 257                 box_desc = desc;
 258                 box_desc->cmd = ADM_CMD_TYPE_BOX | crci_cmd;
 259                 box_desc->row_offset = row_offset;
 260                 box_desc->src_addr = *src;
 261                 box_desc->dst_addr = *dst;
 262
 263                 rows = remainder / burst;
 264                 rows = min_t(u32, rows, ADM_MAX_ROWS);
 265                 box_desc->num_rows = rows << 16 | rows;
 266                 box_desc->row_len = burst << 16 | burst;
 267
 268                 *incr_addr += burst * rows;
 269                 remainder -= burst * rows;
 270                 desc += sizeof(*box_desc);
 271         } while (remainder >= burst);
 272
 273         /* if leftover bytes, do one single descriptor */
 274         if (remainder) {
 275                 single_desc = desc;
 276                 single_desc->cmd = ADM_CMD_TYPE_SINGLE | crci_cmd;
 277                 single_desc->len = remainder;
 278                 single_desc->src_addr = *src;
 279                 single_desc->dst_addr = *dst;
 280                 desc += sizeof(*single_desc);
 281
 282                 if (sg_is_last(sg))
 283                         single_desc->cmd |= ADM_CMD_LC;
 284         } else {
 285                 if (sg_is_last(sg))
 286                         box_desc->cmd |= ADM_CMD_LC;
 287         }
 288
 289         return desc;
 290 }
 291
 292 /**
 293  * adm_process_non_fc_descriptors - Process descriptors for non-fc xfers
 294  *
 295  * @achan: ADM channel
 296  * @desc: Descriptor memory pointer
 297  * @sg: Scatterlist entry
 298  * @direction: DMA transfer direction
 299  */
 300 static void *adm_process_non_fc_descriptors(struct adm_chan *achan,
 301         void *desc, struct scatterlist *sg,
 302         enum dma_transfer_direction direction)
 303 {
 304         struct adm_desc_hw_single *single_desc;
 305         u32 remainder = sg_dma_len(sg);
 306         u32 mem_addr = sg_dma_address(sg);
 307         u32 *incr_addr = &mem_addr;
 308         u32 *src, *dst;
 309
 310         if (direction == DMA_DEV_TO_MEM) {
 311                 src = &achan->slave.src_addr;
 312                 dst = &mem_addr;
 313         } else {
 314                 src = &mem_addr;
 315                 dst = &achan->slave.dst_addr;
 316         }
 317
 318         do {
 319                 single_desc = desc;
 320                 single_desc->cmd = ADM_CMD_TYPE_SINGLE;
 321                 single_desc->src_addr = *src;
 322                 single_desc->dst_addr = *dst;
 323                 single_desc->len = (remainder > ADM_MAX_XFER) ?
 324                                 ADM_MAX_XFER : remainder;
 325
 326                 remainder -= single_desc->len;
 327                 *incr_addr += single_desc->len;
 328                 desc += sizeof(*single_desc);
 329         } while (remainder);
 330
 331         /* set last command if this is the end of the whole transaction */
 332         if (sg_is_last(sg))
 333                 single_desc->cmd |= ADM_CMD_LC;
 334
 335         return desc;
 336 }
 337
 338 /**
 339  * adm_prep_slave_sg - Prep slave sg transaction
 340  *
 341  * @chan: dma channel
 342  * @sgl: scatter gather list
 343  * @sg_len: length of sg
 344  * @direction: DMA transfer direction
 345  * @flags: DMA flags
 346  * @context: transfer context (unused)
 347  */
 348 static struct dma_async_tx_descriptor *adm_prep_slave_sg(struct dma_chan *chan,
 349         struct scatterlist *sgl, unsigned int sg_len,
 350         enum dma_transfer_direction direction, unsigned long flags,
 351         void *context)
 352 {
 353         struct adm_chan *achan = to_adm_chan(chan);
 354         struct adm_device *adev = achan->adev;
 355         struct adm_async_desc *async_desc;
 356         struct scatterlist *sg;
 357         u32 i, burst;
 358         u32 single_count = 0, box_count = 0, crci = 0;
 359         void *desc;
 360         u32 *cple;
 361         int blk_size = 0;
 362
 363         if (!is_slave_direction(direction)) {
 364                 dev_err(adev->dev, "invalid dma direction\n");
 365                 return NULL;
 366         }
 367
 368         /*
 369          * get burst value from slave configuration
 370          */
 371         burst = (direction == DMA_MEM_TO_DEV) ?
 372                 achan->slave.dst_maxburst :
 373                 achan->slave.src_maxburst;
 374
 375         /* if using flow control, validate burst and crci values */
 376         if (achan->slave.device_fc) {
 377
 378                 blk_size = adm_get_blksize(burst);
 379                 if (blk_size < 0) {
 380                         dev_err(adev->dev, "invalid burst value: %d\n",
 381                                 burst);
 382                         return ERR_PTR(-EINVAL);
 383                 }
 384
 385                 crci = achan->slave.slave_id & 0xf;
 386                 if (!crci || achan->slave.slave_id > 0x1f) {
 387                         dev_err(adev->dev, "invalid crci value\n");
 388                         return ERR_PTR(-EINVAL);
 389                 }
 390         }
 391
 392         /* iterate through sgs and compute allocation size of structures */
 393         for_each_sg(sgl, sg, sg_len, i) {
 394                 if (achan->slave.device_fc) {
 395                         box_count += DIV_ROUND_UP(sg_dma_len(sg) / burst,
 396                                                   ADM_MAX_ROWS);
 397                         if (sg_dma_len(sg) % burst)
 398                                 single_count++;
 399                 } else {
 400                         single_count += DIV_ROUND_UP(sg_dma_len(sg),
 401                                                      ADM_MAX_XFER);
 402                 }
 403         }
 404
 405         async_desc = kzalloc(sizeof(*async_desc), GFP_NOWAIT);
 406         if (!async_desc)
 407                 return ERR_PTR(-ENOMEM);
 408
 409         if (crci)
 410                 async_desc->mux = achan->slave.slave_id & ADM_CRCI_MUX_SEL ?
 411                                         ADM_CRCI_CTL_MUX_SEL : 0;
 412         async_desc->crci = crci;
 413         async_desc->blk_size = blk_size;
 414         async_desc->dma_len = single_count * sizeof(struct adm_desc_hw_single) +
 415                                 box_count * sizeof(struct adm_desc_hw_box) +
 416                                 sizeof(*cple) + 2 * ADM_DESC_ALIGN;
 417
 418         async_desc->cpl = dma_alloc_writecombine(adev->dev, async_desc->dma_len,
 419                                 &async_desc->dma_addr, GFP_NOWAIT);
 420
 421         if (!async_desc->cpl) {
 422                 kfree(async_desc);
 423                 return ERR_PTR(-ENOMEM);
 424         }
 425
 426         async_desc->adev = adev;
 427
 428         /* both command list entry and descriptors must be 8 byte aligned */
 429         cple = PTR_ALIGN(async_desc->cpl, ADM_DESC_ALIGN);
 430         desc = PTR_ALIGN(cple + 1, ADM_DESC_ALIGN);
 431
 432         /* init cmd list */
 433         *cple = ADM_CPLE_LP;
 434         *cple |= (desc - async_desc->cpl + async_desc->dma_addr) >> 3;
 435
 436         for_each_sg(sgl, sg, sg_len, i) {
 437                 async_desc->length += sg_dma_len(sg);
 438
 439                 if (achan->slave.device_fc)
 440                         desc = adm_process_fc_descriptors(achan, desc, sg, crci,
 441                                                         burst, direction);
 442                 else
 443                         desc = adm_process_non_fc_descriptors(achan, desc, sg,
 444                                                            direction);
 445         }
 446
 447         return vchan_tx_prep(&achan->vc, &async_desc->vd, flags);
 448 }
 449
 450 /**
 451  * adm_terminate_all - terminate all transactions on a channel
 452  * @achan: adm dma channel
 453  *
 454  * Dequeues and frees all transactions, aborts current transaction
 455  * No callbacks are done
 456  *
 457  */
 458 static int adm_terminate_all(struct dma_chan *chan)
 459 {
 460         struct adm_chan *achan = to_adm_chan(chan);
 461         struct adm_device *adev = achan->adev;
 462         unsigned long flags;
 463         LIST_HEAD(head);
 464
 465         spin_lock_irqsave(&achan->vc.lock, flags);
 466         vchan_get_all_descriptors(&achan->vc, &head);
 467
 468         /* send flush command to terminate current transaction */
 469         writel_relaxed(0x0,
 470                 adev->regs + ADM_CH_FLUSH_STATE0(achan->id, adev->ee));
 471
 472         spin_unlock_irqrestore(&achan->vc.lock, flags);
 473
 474         vchan_dma_desc_free_list(&achan->vc, &head);
 475
 476         return 0;
 477 }
 478
 479 static int adm_slave_config(struct dma_chan *chan, struct dma_slave_config *cfg)
 480 {
 481         struct adm_chan *achan = to_adm_chan(chan);
 482         unsigned long flag;
 483
 484         spin_lock_irqsave(&achan->vc.lock, flag);
 485         memcpy(&achan->slave, cfg, sizeof(struct dma_slave_config));
 486         spin_unlock_irqrestore(&achan->vc.lock, flag);
 487
 488         return 0;
 489 }
 490
 491 /**
 492  * adm_start_dma - start next transaction
 493  * @achan - ADM dma channel
 494  */
 495 static void adm_start_dma(struct adm_chan *achan)
 496 {
 497         struct virt_dma_desc *vd = vchan_next_desc(&achan->vc);
 498         struct adm_device *adev = achan->adev;
 499         struct adm_async_desc *async_desc;
 500
 501         lockdep_assert_held(&achan->vc.lock);
 502
 503         if (!vd)
 504                 return;
 505
 506         list_del(&vd->node);
 507
 508         /* write next command list out to the CMD FIFO */
 509         async_desc = container_of(vd, struct adm_async_desc, vd);
 510         achan->curr_txd = async_desc;
 511
 512         /* reset channel error */
 513         achan->error = 0;
 514
 515         if (!achan->initialized) {
 516                 /* enable interrupts */
 517                 writel(ADM_CH_CONF_SHADOW_EN |
 518                        ADM_CH_CONF_PERM_MPU_CONF |
 519                        ADM_CH_CONF_MPU_DISABLE |
 520                        ADM_CH_CONF_SEC_DOMAIN(adev->ee),
 521                        adev->regs + ADM_CH_CONF(achan->id));
 522
 523                 writel(ADM_CH_RSLT_CONF_IRQ_EN | ADM_CH_RSLT_CONF_FLUSH_EN,
 524                         adev->regs + ADM_CH_RSLT_CONF(achan->id, adev->ee));
 525
 526                 achan->initialized = 1;
 527         }
 528
 529         /* set the crci block size if this transaction requires CRCI */
 530         if (async_desc->crci) {
 531                 writel(async_desc->mux | async_desc->blk_size,
 532                         adev->regs + ADM_CRCI_CTL(async_desc->crci, adev->ee));
 533         }
 534
 535         /* make sure IRQ enable doesn't get reordered */
 536         wmb();
 537
 538         /* write next command list out to the CMD FIFO */
 539         writel(ALIGN(async_desc->dma_addr, ADM_DESC_ALIGN) >> 3,
 540                 adev->regs + ADM_CH_CMD_PTR(achan->id, adev->ee));
 541 }
 542
 543 /**
 544  * adm_dma_irq - irq handler for ADM controller
 545  * @irq: IRQ of interrupt
 546  * @data: callback data
 547  *
 548  * IRQ handler for the bam controller
 549  */
 550 static irqreturn_t adm_dma_irq(int irq, void *data)
 551 {
 552         struct adm_device *adev = data;
 553         u32 srcs, i;
 554         struct adm_async_desc *async_desc;
 555         unsigned long flags;
 556
 557         srcs = readl_relaxed(adev->regs +
 558                         ADM_SEC_DOMAIN_IRQ_STATUS(adev->ee));
 559
 560         for (i = 0; i < ADM_MAX_CHANNELS; i++) {
 561                 struct adm_chan *achan = &adev->channels[i];
 562                 u32 status, result;
 563
 564                 if (srcs & BIT(i)) {
 565                         status = readl_relaxed(adev->regs +
 566                                 ADM_CH_STATUS_SD(i, adev->ee));
 567
 568                         /* if no result present, skip */
 569                         if (!(status & ADM_CH_STATUS_VALID))
 570                                 continue;
 571
 572                         result = readl_relaxed(adev->regs +
 573                                 ADM_CH_RSLT(i, adev->ee));
 574
 575                         /* no valid results, skip */
 576                         if (!(result & ADM_CH_RSLT_VALID))
 577                                 continue;
 578
 579                         /* flag error if transaction was flushed or failed */
 580                         if (result & (ADM_CH_RSLT_ERR | ADM_CH_RSLT_FLUSH))
 581                                 achan->error = 1;
 582
 583                         spin_lock_irqsave(&achan->vc.lock, flags);
 584                         async_desc = achan->curr_txd;
 585
 586                         achan->curr_txd = NULL;
 587
 588                         if (async_desc) {
 589                                 vchan_cookie_complete(&async_desc->vd);
 590
 591                                 /* kick off next DMA */
 592                                 adm_start_dma(achan);
 593                         }
 594
 595                         spin_unlock_irqrestore(&achan->vc.lock, flags);
 596                 }
 597         }
 598
 599         return IRQ_HANDLED;
 600 }
 601
 602 /**
 603  * adm_tx_status - returns status of transaction
 604  * @chan: dma channel
 605  * @cookie: transaction cookie
 606  * @txstate: DMA transaction state
 607  *
 608  * Return status of dma transaction
 609  */
 610 static enum dma_status adm_tx_status(struct dma_chan *chan, dma_cookie_t cookie,
 611         struct dma_tx_state *txstate)
 612 {
 613         struct adm_chan *achan = to_adm_chan(chan);
 614         struct virt_dma_desc *vd;
 615         enum dma_status ret;
 616         unsigned long flags;
 617         size_t residue = 0;
 618
 619         ret = dma_cookie_status(chan, cookie, txstate);
 620         if (ret == DMA_COMPLETE || !txstate)
 621                 return ret;
 622
 623         spin_lock_irqsave(&achan->vc.lock, flags);
 624
 625         vd = vchan_find_desc(&achan->vc, cookie);
 626         if (vd)
 627                 residue = container_of(vd, struct adm_async_desc, vd)->length;
 628
 629         spin_unlock_irqrestore(&achan->vc.lock, flags);
 630
 631         /*
 632          * residue is either the full length if it is in the issued list, or 0
 633          * if it is in progress.  We have no reliable way of determining
 634          * anything inbetween
 635         */
 636         dma_set_residue(txstate, residue);
 637
 638         if (achan->error)
 639                 return DMA_ERROR;
 640
 641         return ret;
 642 }
 643
 644 /**
 645  * adm_issue_pending - starts pending transactions
 646  * @chan: dma channel
 647  *
 648  * Issues all pending transactions and starts DMA
 649  */
 650 static void adm_issue_pending(struct dma_chan *chan)
 651 {
 652         struct adm_chan *achan = to_adm_chan(chan);
 653         unsigned long flags;
 654
 655         spin_lock_irqsave(&achan->vc.lock, flags);
 656
 657         if (vchan_issue_pending(&achan->vc) && !achan->curr_txd)
 658                 adm_start_dma(achan);
 659         spin_unlock_irqrestore(&achan->vc.lock, flags);
 660 }
 661
 662 /**
 663  * adm_dma_free_desc - free descriptor memory
 664  * @vd: virtual descriptor
 665  *
 666  */
 667 static void adm_dma_free_desc(struct virt_dma_desc *vd)
 668 {
 669         struct adm_async_desc *async_desc = container_of(vd,
 670                         struct adm_async_desc, vd);
 671
 672         dma_free_writecombine(async_desc->adev->dev, async_desc->dma_len,
 673                 async_desc->cpl, async_desc->dma_addr);
 674         kfree(async_desc);
 675 }
 676
 677 static void adm_channel_init(struct adm_device *adev, struct adm_chan *achan,
 678         u32 index)
 679 {
 680         achan->id = index;
 681         achan->adev = adev;
 682
 683         vchan_init(&achan->vc, &adev->common);
 684         achan->vc.desc_free = adm_dma_free_desc;
 685 }
 686
 687 static int adm_dma_probe(struct platform_device *pdev)
 688 {
 689         struct adm_device *adev;
 690         struct resource *iores;
 691         int ret;
 692         u32 i;
 693
 694         adev = devm_kzalloc(&pdev->dev, sizeof(*adev), GFP_KERNEL);
 695         if (!adev)
 696                 return -ENOMEM;
 697
 698         adev->dev = &pdev->dev;
 699
 700         iores = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 701         adev->regs = devm_ioremap_resource(&pdev->dev, iores);
 702         if (IS_ERR(adev->regs))
 703                 return PTR_ERR(adev->regs);
 704
 705         adev->irq = platform_get_irq(pdev, 0);
 706         if (adev->irq < 0)
 707                 return adev->irq;
 708
 709         ret = of_property_read_u32(pdev->dev.of_node, "qcom,ee", &adev->ee);
 710         if (ret) {
 711                 dev_err(adev->dev, "Execution environment unspecified\n");
 712                 return ret;
 713         }
 714
 715         adev->core_clk = devm_clk_get(adev->dev, "core");
 716         if (IS_ERR(adev->core_clk))
 717                 return PTR_ERR(adev->core_clk);
 718
 719         ret = clk_prepare_enable(adev->core_clk);
 720         if (ret) {
 721                 dev_err(adev->dev, "failed to prepare/enable core clock\n");
 722                 return ret;
 723         }
 724
 725         adev->iface_clk = devm_clk_get(adev->dev, "iface");
 726         if (IS_ERR(adev->iface_clk)) {
 727                 ret = PTR_ERR(adev->iface_clk);
 728                 goto err_disable_core_clk;
 729         }
 730
 731         ret = clk_prepare_enable(adev->iface_clk);
 732         if (ret) {
 733                 dev_err(adev->dev, "failed to prepare/enable iface clock\n");
 734                 goto err_disable_core_clk;
 735         }
 736
 737         adev->clk_reset = devm_reset_control_get(&pdev->dev, "clk");
 738         if (IS_ERR(adev->clk_reset)) {
 739                 dev_err(adev->dev, "failed to get ADM0 reset\n");
 740                 ret = PTR_ERR(adev->clk_reset);
 741                 goto err_disable_clks;
 742         }
 743
 744         adev->c0_reset = devm_reset_control_get(&pdev->dev, "c0");
 745         if (IS_ERR(adev->c0_reset)) {
 746                 dev_err(adev->dev, "failed to get ADM0 C0 reset\n");
 747                 ret = PTR_ERR(adev->c0_reset);
 748                 goto err_disable_clks;
 749         }
 750
 751         adev->c1_reset = devm_reset_control_get(&pdev->dev, "c1");
 752         if (IS_ERR(adev->c1_reset)) {
 753                 dev_err(adev->dev, "failed to get ADM0 C1 reset\n");
 754                 ret = PTR_ERR(adev->c1_reset);
 755                 goto err_disable_clks;
 756         }
 757
 758         adev->c2_reset = devm_reset_control_get(&pdev->dev, "c2");
 759         if (IS_ERR(adev->c2_reset)) {
 760                 dev_err(adev->dev, "failed to get ADM0 C2 reset\n");
 761                 ret = PTR_ERR(adev->c2_reset);
 762                 goto err_disable_clks;
 763         }
 764
 765         reset_control_assert(adev->clk_reset);
 766         reset_control_assert(adev->c0_reset);
 767         reset_control_assert(adev->c1_reset);
 768         reset_control_assert(adev->c2_reset);
 769
 770         reset_control_deassert(adev->clk_reset);
 771         reset_control_deassert(adev->c0_reset);
 772         reset_control_deassert(adev->c1_reset);
 773         reset_control_deassert(adev->c2_reset);
 774
 775         adev->channels = devm_kcalloc(adev->dev, ADM_MAX_CHANNELS,
 776                                 sizeof(*adev->channels), GFP_KERNEL);
 777
 778         if (!adev->channels) {
 779                 ret = -ENOMEM;
 780                 goto err_disable_clks;
 781         }
 782
 783         /* allocate and initialize channels */
 784         INIT_LIST_HEAD(&adev->common.channels);
 785
 786         for (i = 0; i < ADM_MAX_CHANNELS; i++)
 787                 adm_channel_init(adev, &adev->channels[i], i);
 788
 789         /* reset CRCIs */
 790         for (i = 0; i < 16; i++)
 791                 writel(ADM_CRCI_CTL_RST, adev->regs +
 792                         ADM_CRCI_CTL(i, adev->ee));
 793
 794         /* configure client interfaces */
 795         writel(ADM_CI_RANGE_START(0x40) | ADM_CI_RANGE_END(0xb0) |
 796                 ADM_CI_BURST_8_WORDS, adev->regs + ADM_CI_CONF(0));
 797         writel(ADM_CI_RANGE_START(0x2a) | ADM_CI_RANGE_END(0x2c) |
 798                 ADM_CI_BURST_8_WORDS, adev->regs + ADM_CI_CONF(1));
 799         writel(ADM_CI_RANGE_START(0x12) | ADM_CI_RANGE_END(0x28) |
 800                 ADM_CI_BURST_8_WORDS, adev->regs + ADM_CI_CONF(2));
 801         writel(ADM_GP_CTL_LP_EN | ADM_GP_CTL_LP_CNT(0xf),
 802                 adev->regs + ADM_GP_CTL);
 803
 804         ret = devm_request_irq(adev->dev, adev->irq, adm_dma_irq,
 805                         0, "adm_dma", adev);
 806         if (ret)
 807                 goto err_disable_clks;
 808
 809         platform_set_drvdata(pdev, adev);
 810
 811         adev->common.dev = adev->dev;
 812         adev->common.dev->dma_parms = &adev->dma_parms;
 813
 814         /* set capabilities */
 815         dma_cap_zero(adev->common.cap_mask);
 816         dma_cap_set(DMA_SLAVE, adev->common.cap_mask);
 817         dma_cap_set(DMA_PRIVATE, adev->common.cap_mask);
 818
 819         /* initialize dmaengine apis */
 820         adev->common.directions = BIT(DMA_DEV_TO_MEM | DMA_MEM_TO_DEV);
 821         adev->common.residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR;
 822         adev->common.src_addr_widths = DMA_SLAVE_BUSWIDTH_4_BYTES;
 823         adev->common.dst_addr_widths = DMA_SLAVE_BUSWIDTH_4_BYTES;
 824         adev->common.device_free_chan_resources = adm_free_chan;
 825         adev->common.device_prep_slave_sg = adm_prep_slave_sg;
 826         adev->common.device_issue_pending = adm_issue_pending;
 827         adev->common.device_tx_status = adm_tx_status;
 828         adev->common.device_terminate_all = adm_terminate_all;
 829         adev->common.device_config = adm_slave_config;
 830
 831         ret = dma_async_device_register(&adev->common);
 832         if (ret) {
 833                 dev_err(adev->dev, "failed to register dma async device\n");
 834                 goto err_disable_clks;
 835         }
 836
 837         ret = of_dma_controller_register(pdev->dev.of_node,
 838                                          of_dma_xlate_by_chan_id,
 839                                          &adev->common);
 840         if (ret)
 841                 goto err_unregister_dma;
 842
 843         return 0;
 844
 845 err_unregister_dma:
 846         dma_async_device_unregister(&adev->common);
 847 err_disable_clks:
 848         clk_disable_unprepare(adev->iface_clk);
 849 err_disable_core_clk:
 850         clk_disable_unprepare(adev->core_clk);
 851
 852         return ret;
 853 }
 854
 855 static int adm_dma_remove(struct platform_device *pdev)
 856 {
 857         struct adm_device *adev = platform_get_drvdata(pdev);
 858         struct adm_chan *achan;
 859         u32 i;
 860
 861         of_dma_controller_free(pdev->dev.of_node);
 862         dma_async_device_unregister(&adev->common);
 863
 864         for (i = 0; i < ADM_MAX_CHANNELS; i++) {
 865                 achan = &adev->channels[i];
 866
 867                 /* mask IRQs for this channel/EE pair */
 868                 writel(0, adev->regs + ADM_CH_RSLT_CONF(achan->id, adev->ee));
 869
 870                 adm_terminate_all(&adev->channels[i].vc.chan);
 871         }
 872
 873         devm_free_irq(adev->dev, adev->irq, adev);
 874
 875         clk_disable_unprepare(adev->core_clk);
 876         clk_disable_unprepare(adev->iface_clk);
 877
 878         return 0;
 879 }
 880
 881 static const struct of_device_id adm_of_match[] = {
 882         { .compatible = "qcom,adm", },
 883         {}
 884 };
 885 MODULE_DEVICE_TABLE(of, adm_of_match);
 886
 887 static struct platform_driver adm_dma_driver = {
 888         .probe = adm_dma_probe,
 889         .remove = adm_dma_remove,
 890         .driver = {
 891                 .name = "adm-dma-engine",
 892                 .of_match_table = adm_of_match,
 893         },
 894 };
 895
 896 module_platform_driver(adm_dma_driver);
 897
 898 MODULE_AUTHOR("Andy Gross <agross@codeaurora.org>");
 899 MODULE_DESCRIPTION("QCOM ADM DMA engine driver");
 900 MODULE_LICENSE("GPL v2");