drivers/dma/ioat/dma.c

   1 /*
   2  * Intel I/OAT DMA Linux driver
   3  * Copyright(c) 2004 - 2009 Intel Corporation.
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms and conditions of the GNU General Public License,
   7  * version 2, as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12  * more details.
  13  *
  14  * You should have received a copy of the GNU General Public License along with
  15  * this program; if not, write to the Free Software Foundation, Inc.,
  16  * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  17  *
  18  * The full GNU General Public License is included in this distribution in
  19  * the file called "COPYING".
  20  *
  21  */
  22
  23 /*
  24  * This driver supports an Intel I/OAT DMA engine, which does asynchronous
  25  * copy operations.
  26  */
  27
  28 #include <linux/init.h>
  29 #include <linux/module.h>
  30 #include <linux/pci.h>
  31 #include <linux/interrupt.h>
  32 #include <linux/dmaengine.h>
  33 #include <linux/delay.h>
  34 #include <linux/dma-mapping.h>
  35 #include <linux/workqueue.h>
  36 #include <linux/i7300_idle.h>
  37 #include "dma.h"
  38 #include "registers.h"
  39 #include "hw.h"
  40
  41 static int ioat_pending_level = 4;
  42 module_param(ioat_pending_level, int, 0644);
  43 MODULE_PARM_DESC(ioat_pending_level,
  44                  "high-water mark for pushing ioat descriptors (default: 4)");
  45
  46 static void ioat_dma_chan_reset_part2(struct work_struct *work);
  47 static void ioat_dma_chan_watchdog(struct work_struct *work);
  48
  49 /* internal functions */
  50 static void ioat_dma_start_null_desc(struct ioat_dma_chan *ioat);
  51 static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *ioat);
  52
  53 static struct ioat_desc_sw *
  54 ioat1_dma_get_next_descriptor(struct ioat_dma_chan *ioat);
  55 static struct ioat_desc_sw *
  56 ioat2_dma_get_next_descriptor(struct ioat_dma_chan *ioat);
  57
  58 static inline struct ioat_chan_common *
  59 ioat_chan_by_index(struct ioatdma_device *device, int index)
  60 {
  61         return device->idx[index];
  62 }
  63
  64 /**
  65  * ioat_dma_do_interrupt - handler used for single vector interrupt mode
  66  * @irq: interrupt id
  67  * @data: interrupt data
  68  */
  69 static irqreturn_t ioat_dma_do_interrupt(int irq, void *data)
  70 {
  71         struct ioatdma_device *instance = data;
  72         struct ioat_chan_common *chan;
  73         unsigned long attnstatus;
  74         int bit;
  75         u8 intrctrl;
  76
  77         intrctrl = readb(instance->reg_base + IOAT_INTRCTRL_OFFSET);
  78
  79         if (!(intrctrl & IOAT_INTRCTRL_MASTER_INT_EN))
  80                 return IRQ_NONE;
  81
  82         if (!(intrctrl & IOAT_INTRCTRL_INT_STATUS)) {
  83                 writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET);
  84                 return IRQ_NONE;
  85         }
  86
  87         attnstatus = readl(instance->reg_base + IOAT_ATTNSTATUS_OFFSET);
  88         for_each_bit(bit, &attnstatus, BITS_PER_LONG) {
  89                 chan = ioat_chan_by_index(instance, bit);
  90                 tasklet_schedule(&chan->cleanup_task);
  91         }
  92
  93         writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET);
  94         return IRQ_HANDLED;
  95 }
  96
  97 /**
  98  * ioat_dma_do_interrupt_msix - handler used for vector-per-channel interrupt mode
  99  * @irq: interrupt id
 100  * @data: interrupt data
 101  */
 102 static irqreturn_t ioat_dma_do_interrupt_msix(int irq, void *data)
 103 {
 104         struct ioat_chan_common *chan = data;
 105
 106         tasklet_schedule(&chan->cleanup_task);
 107
 108         return IRQ_HANDLED;
 109 }
 110
 111 static void ioat_dma_cleanup_tasklet(unsigned long data);
 112
 113 /**
 114  * ioat_dma_enumerate_channels - find and initialize the device's channels
 115  * @device: the device to be enumerated
 116  */
 117 static int ioat_dma_enumerate_channels(struct ioatdma_device *device)
 118 {
 119         u8 xfercap_scale;
 120         u32 xfercap;
 121         int i;
 122         struct ioat_chan_common *chan;
 123         struct ioat_dma_chan *ioat;
 124         struct device *dev = &device->pdev->dev;
 125         struct dma_device *dma = &device->common;
 126
 127         INIT_LIST_HEAD(&dma->channels);
 128         dma->chancnt = readb(device->reg_base + IOAT_CHANCNT_OFFSET);
 129         xfercap_scale = readb(device->reg_base + IOAT_XFERCAP_OFFSET);
 130         xfercap = (xfercap_scale == 0 ? -1 : (1UL << xfercap_scale));
 131
 132 #ifdef  CONFIG_I7300_IDLE_IOAT_CHANNEL
 133         if (i7300_idle_platform_probe(NULL, NULL, 1) == 0)
 134                 dma->chancnt--;
 135 #endif
 136         for (i = 0; i < dma->chancnt; i++) {
 137                 ioat = devm_kzalloc(dev, sizeof(*ioat), GFP_KERNEL);
 138                 if (!ioat) {
 139                         dma->chancnt = i;
 140                         break;
 141                 }
 142
 143                 chan = &ioat->base;
 144                 chan->device = device;
 145                 chan->reg_base = device->reg_base + (0x80 * (i + 1));
 146                 ioat->xfercap = xfercap;
 147                 ioat->desccount = 0;
 148                 INIT_DELAYED_WORK(&chan->work, ioat_dma_chan_reset_part2);
 149                 spin_lock_init(&chan->cleanup_lock);
 150                 spin_lock_init(&ioat->desc_lock);
 151                 INIT_LIST_HEAD(&ioat->free_desc);
 152                 INIT_LIST_HEAD(&ioat->used_desc);
 153                 /* This should be made common somewhere in dmaengine.c */
 154                 chan->common.device = &device->common;
 155                 list_add_tail(&chan->common.device_node, &dma->channels);
 156                 device->idx[i] = chan;
 157                 tasklet_init(&chan->cleanup_task,
 158                              ioat_dma_cleanup_tasklet,
 159                              (unsigned long) ioat);
 160                 tasklet_disable(&chan->cleanup_task);
 161         }
 162         return dma->chancnt;
 163 }
 164
 165 /**
 166  * ioat_dma_memcpy_issue_pending - push potentially unrecognized appended
 167  *                                 descriptors to hw
 168  * @chan: DMA channel handle
 169  */
 170 static inline void
 171 __ioat1_dma_memcpy_issue_pending(struct ioat_dma_chan *ioat)
 172 {
 173         void __iomem *reg_base = ioat->base.reg_base;
 174
 175         ioat->pending = 0;
 176         writeb(IOAT_CHANCMD_APPEND, reg_base + IOAT1_CHANCMD_OFFSET);
 177 }
 178
 179 static void ioat1_dma_memcpy_issue_pending(struct dma_chan *chan)
 180 {
 181         struct ioat_dma_chan *ioat = to_ioat_chan(chan);
 182
 183         if (ioat->pending > 0) {
 184                 spin_lock_bh(&ioat->desc_lock);
 185                 __ioat1_dma_memcpy_issue_pending(ioat);
 186                 spin_unlock_bh(&ioat->desc_lock);
 187         }
 188 }
 189
 190 static inline void
 191 __ioat2_dma_memcpy_issue_pending(struct ioat_dma_chan *ioat)
 192 {
 193         void __iomem *reg_base = ioat->base.reg_base;
 194
 195         ioat->pending = 0;
 196         writew(ioat->dmacount, reg_base + IOAT_CHAN_DMACOUNT_OFFSET);
 197 }
 198
 199 static void ioat2_dma_memcpy_issue_pending(struct dma_chan *chan)
 200 {
 201         struct ioat_dma_chan *ioat = to_ioat_chan(chan);
 202
 203         if (ioat->pending > 0) {
 204                 spin_lock_bh(&ioat->desc_lock);
 205                 __ioat2_dma_memcpy_issue_pending(ioat);
 206                 spin_unlock_bh(&ioat->desc_lock);
 207         }
 208 }
 209
 210
 211 /**
 212  * ioat_dma_chan_reset_part2 - reinit the channel after a reset
 213  */
 214 static void ioat_dma_chan_reset_part2(struct work_struct *work)
 215 {
 216         struct ioat_chan_common *chan;
 217         struct ioat_dma_chan *ioat;
 218         struct ioat_desc_sw *desc;
 219
 220         chan = container_of(work, struct ioat_chan_common, work.work);
 221         ioat = container_of(chan, struct ioat_dma_chan, base);
 222         spin_lock_bh(&chan->cleanup_lock);
 223         spin_lock_bh(&ioat->desc_lock);
 224
 225         chan->completion_virt->low = 0;
 226         chan->completion_virt->high = 0;
 227         ioat->pending = 0;
 228
 229         /*
 230          * count the descriptors waiting, and be sure to do it
 231          * right for both the CB1 line and the CB2 ring
 232          */
 233         ioat->dmacount = 0;
 234         if (ioat->used_desc.prev) {
 235                 desc = to_ioat_desc(ioat->used_desc.prev);
 236                 do {
 237                         ioat->dmacount++;
 238                         desc = to_ioat_desc(desc->node.next);
 239                 } while (&desc->node != ioat->used_desc.next);
 240         }
 241
 242         /*
 243          * write the new starting descriptor address
 244          * this puts channel engine into ARMED state
 245          */
 246         desc = to_ioat_desc(ioat->used_desc.prev);
 247         switch (chan->device->version) {
 248         case IOAT_VER_1_2:
 249                 writel(((u64) desc->txd.phys) & 0x00000000FFFFFFFF,
 250                        chan->reg_base + IOAT1_CHAINADDR_OFFSET_LOW);
 251                 writel(((u64) desc->txd.phys) >> 32,
 252                        chan->reg_base + IOAT1_CHAINADDR_OFFSET_HIGH);
 253
 254                 writeb(IOAT_CHANCMD_START, chan->reg_base
 255                         + IOAT_CHANCMD_OFFSET(chan->device->version));
 256                 break;
 257         case IOAT_VER_2_0:
 258                 writel(((u64) desc->txd.phys) & 0x00000000FFFFFFFF,
 259                        chan->reg_base + IOAT2_CHAINADDR_OFFSET_LOW);
 260                 writel(((u64) desc->txd.phys) >> 32,
 261                        chan->reg_base + IOAT2_CHAINADDR_OFFSET_HIGH);
 262
 263                 /* tell the engine to go with what's left to be done */
 264                 writew(ioat->dmacount,
 265                        chan->reg_base + IOAT_CHAN_DMACOUNT_OFFSET);
 266
 267                 break;
 268         }
 269         dev_err(to_dev(chan),
 270                 "chan%d reset - %d descs waiting, %d total desc\n",
 271                 chan_num(chan), ioat->dmacount, ioat->desccount);
 272
 273         spin_unlock_bh(&ioat->desc_lock);
 274         spin_unlock_bh(&chan->cleanup_lock);
 275 }
 276
 277 /**
 278  * ioat_dma_reset_channel - restart a channel
 279  * @ioat: IOAT DMA channel handle
 280  */
 281 static void ioat_dma_reset_channel(struct ioat_dma_chan *ioat)
 282 {
 283         struct ioat_chan_common *chan = &ioat->base;
 284         void __iomem *reg_base = chan->reg_base;
 285         u32 chansts, chanerr;
 286
 287         if (!ioat->used_desc.prev)
 288                 return;
 289
 290         chanerr = readl(reg_base + IOAT_CHANERR_OFFSET);
 291         chansts = (chan->completion_virt->low
 292                                         & IOAT_CHANSTS_DMA_TRANSFER_STATUS);
 293         if (chanerr) {
 294                 dev_err(to_dev(chan),
 295                         "chan%d, CHANSTS = 0x%08x CHANERR = 0x%04x, clearing\n",
 296                         chan_num(chan), chansts, chanerr);
 297                 writel(chanerr, reg_base + IOAT_CHANERR_OFFSET);
 298         }
 299
 300         /*
 301          * whack it upside the head with a reset
 302          * and wait for things to settle out.
 303          * force the pending count to a really big negative
 304          * to make sure no one forces an issue_pending
 305          * while we're waiting.
 306          */
 307
 308         spin_lock_bh(&ioat->desc_lock);
 309         ioat->pending = INT_MIN;
 310         writeb(IOAT_CHANCMD_RESET,
 311                reg_base + IOAT_CHANCMD_OFFSET(chan->device->version));
 312         spin_unlock_bh(&ioat->desc_lock);
 313
 314         /* schedule the 2nd half instead of sleeping a long time */
 315         schedule_delayed_work(&chan->work, RESET_DELAY);
 316 }
 317
 318 /**
 319  * ioat_dma_chan_watchdog - watch for stuck channels
 320  */
 321 static void ioat_dma_chan_watchdog(struct work_struct *work)
 322 {
 323         struct ioatdma_device *device =
 324                 container_of(work, struct ioatdma_device, work.work);
 325         struct ioat_dma_chan *ioat;
 326         struct ioat_chan_common *chan;
 327         int i;
 328
 329         union {
 330                 u64 full;
 331                 struct {
 332                         u32 low;
 333                         u32 high;
 334                 };
 335         } completion_hw;
 336         unsigned long compl_desc_addr_hw;
 337
 338         for (i = 0; i < device->common.chancnt; i++) {
 339                 chan = ioat_chan_by_index(device, i);
 340                 ioat = container_of(chan, struct ioat_dma_chan, base);
 341
 342                 if (chan->device->version == IOAT_VER_1_2
 343                         /* have we started processing anything yet */
 344                     && chan->last_completion
 345                         /* have we completed any since last watchdog cycle? */
 346                     && (chan->last_completion == chan->watchdog_completion)
 347                         /* has TCP stuck on one cookie since last watchdog? */
 348                     && (chan->watchdog_tcp_cookie == chan->watchdog_last_tcp_cookie)
 349                     && (chan->watchdog_tcp_cookie != chan->completed_cookie)
 350                         /* is there something in the chain to be processed? */
 351                         /* CB1 chain always has at least the last one processed */
 352                     && (ioat->used_desc.prev != ioat->used_desc.next)
 353                     && ioat->pending == 0) {
 354
 355                         /*
 356                          * check CHANSTS register for completed
 357                          * descriptor address.
 358                          * if it is different than completion writeback,
 359                          * it is not zero
 360                          * and it has changed since the last watchdog
 361                          *     we can assume that channel
 362                          *     is still working correctly
 363                          *     and the problem is in completion writeback.
 364                          *     update completion writeback
 365                          *     with actual CHANSTS value
 366                          * else
 367                          *     try resetting the channel
 368                          */
 369
 370                         completion_hw.low = readl(chan->reg_base +
 371                                 IOAT_CHANSTS_OFFSET_LOW(chan->device->version));
 372                         completion_hw.high = readl(chan->reg_base +
 373                                 IOAT_CHANSTS_OFFSET_HIGH(chan->device->version));
 374 #if (BITS_PER_LONG == 64)
 375                         compl_desc_addr_hw =
 376                                 completion_hw.full
 377                                 & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR;
 378 #else
 379                         compl_desc_addr_hw =
 380                                 completion_hw.low & IOAT_LOW_COMPLETION_MASK;
 381 #endif
 382
 383                         if ((compl_desc_addr_hw != 0)
 384                            && (compl_desc_addr_hw != chan->watchdog_completion)
 385                            && (compl_desc_addr_hw != chan->last_compl_desc_addr_hw)) {
 386                                 chan->last_compl_desc_addr_hw = compl_desc_addr_hw;
 387                                 chan->completion_virt->low = completion_hw.low;
 388                                 chan->completion_virt->high = completion_hw.high;
 389                         } else {
 390                                 ioat_dma_reset_channel(ioat);
 391                                 chan->watchdog_completion = 0;
 392                                 chan->last_compl_desc_addr_hw = 0;
 393                         }
 394
 395                 /*
 396                  * for version 2.0 if there are descriptors yet to be processed
 397                  * and the last completed hasn't changed since the last watchdog
 398                  *      if they haven't hit the pending level
 399                  *          issue the pending to push them through
 400                  *      else
 401                  *          try resetting the channel
 402                  */
 403                 } else if (chan->device->version == IOAT_VER_2_0
 404                     && ioat->used_desc.prev
 405                     && chan->last_completion
 406                     && chan->last_completion == chan->watchdog_completion) {
 407
 408                         if (ioat->pending < ioat_pending_level)
 409                                 ioat2_dma_memcpy_issue_pending(&chan->common);
 410                         else {
 411                                 ioat_dma_reset_channel(ioat);
 412                                 chan->watchdog_completion = 0;
 413                         }
 414                 } else {
 415                         chan->last_compl_desc_addr_hw = 0;
 416                         chan->watchdog_completion = chan->last_completion;
 417                 }
 418                 chan->watchdog_last_tcp_cookie = chan->watchdog_tcp_cookie;
 419         }
 420
 421         schedule_delayed_work(&device->work, WATCHDOG_DELAY);
 422 }
 423
 424 static dma_cookie_t ioat1_tx_submit(struct dma_async_tx_descriptor *tx)
 425 {
 426         struct dma_chan *c = tx->chan;
 427         struct ioat_dma_chan *ioat = to_ioat_chan(c);
 428         struct ioat_desc_sw *desc = tx_to_ioat_desc(tx);
 429         struct ioat_desc_sw *first;
 430         struct ioat_desc_sw *chain_tail;
 431         dma_cookie_t cookie;
 432
 433         spin_lock_bh(&ioat->desc_lock);
 434         /* cookie incr and addition to used_list must be atomic */
 435         cookie = c->cookie;
 436         cookie++;
 437         if (cookie < 0)
 438                 cookie = 1;
 439         c->cookie = cookie;
 440         tx->cookie = cookie;
 441
 442         /* write address into NextDescriptor field of last desc in chain */
 443         first = to_ioat_desc(tx->tx_list.next);
 444         chain_tail = to_ioat_desc(ioat->used_desc.prev);
 445         /* make descriptor updates globally visible before chaining */
 446         wmb();
 447         chain_tail->hw->next = first->txd.phys;
 448         list_splice_tail_init(&tx->tx_list, &ioat->used_desc);
 449
 450         ioat->dmacount += desc->tx_cnt;
 451         ioat->pending += desc->tx_cnt;
 452         if (ioat->pending >= ioat_pending_level)
 453                 __ioat1_dma_memcpy_issue_pending(ioat);
 454         spin_unlock_bh(&ioat->desc_lock);
 455
 456         return cookie;
 457 }
 458
 459 static dma_cookie_t ioat2_tx_submit(struct dma_async_tx_descriptor *tx)
 460 {
 461         struct ioat_dma_chan *ioat = to_ioat_chan(tx->chan);
 462         struct ioat_desc_sw *first = tx_to_ioat_desc(tx);
 463         struct ioat_desc_sw *new;
 464         struct ioat_dma_descriptor *hw;
 465         dma_cookie_t cookie;
 466         u32 copy;
 467         size_t len;
 468         dma_addr_t src, dst;
 469         unsigned long orig_flags;
 470         unsigned int desc_count = 0;
 471
 472         /* src and dest and len are stored in the initial descriptor */
 473         len = first->len;
 474         src = first->src;
 475         dst = first->dst;
 476         orig_flags = first->txd.flags;
 477         new = first;
 478
 479         /*
 480          * ioat->desc_lock is still in force in version 2 path
 481          * it gets unlocked at end of this function
 482          */
 483         do {
 484                 copy = min_t(size_t, len, ioat->xfercap);
 485
 486                 async_tx_ack(&new->txd);
 487
 488                 hw = new->hw;
 489                 hw->size = copy;
 490                 hw->ctl = 0;
 491                 hw->src_addr = src;
 492                 hw->dst_addr = dst;
 493
 494                 len -= copy;
 495                 dst += copy;
 496                 src += copy;
 497                 desc_count++;
 498         } while (len && (new = ioat2_dma_get_next_descriptor(ioat)));
 499
 500         if (!new) {
 501                 dev_err(to_dev(&ioat->base), "tx submit failed\n");
 502                 spin_unlock_bh(&ioat->desc_lock);
 503                 return -ENOMEM;
 504         }
 505
 506         hw->ctl_f.compl_write = 1;
 507         if (first->txd.callback) {
 508                 hw->ctl_f.int_en = 1;
 509                 if (first != new) {
 510                         /* move callback into to last desc */
 511                         new->txd.callback = first->txd.callback;
 512                         new->txd.callback_param
 513                                         = first->txd.callback_param;
 514                         first->txd.callback = NULL;
 515                         first->txd.callback_param = NULL;
 516                 }
 517         }
 518
 519         new->tx_cnt = desc_count;
 520         new->txd.flags = orig_flags; /* client is in control of this ack */
 521
 522         /* store the original values for use in later cleanup */
 523         if (new != first) {
 524                 new->src = first->src;
 525                 new->dst = first->dst;
 526                 new->len = first->len;
 527         }
 528
 529         /* cookie incr and addition to used_list must be atomic */
 530         cookie = ioat->base.common.cookie;
 531         cookie++;
 532         if (cookie < 0)
 533                 cookie = 1;
 534         ioat->base.common.cookie = new->txd.cookie = cookie;
 535
 536         ioat->dmacount += desc_count;
 537         ioat->pending += desc_count;
 538         if (ioat->pending >= ioat_pending_level)
 539                 __ioat2_dma_memcpy_issue_pending(ioat);
 540         spin_unlock_bh(&ioat->desc_lock);
 541
 542         return cookie;
 543 }
 544
 545 /**
 546  * ioat_dma_alloc_descriptor - allocate and return a sw and hw descriptor pair
 547  * @ioat: the channel supplying the memory pool for the descriptors
 548  * @flags: allocation flags
 549  */
 550 static struct ioat_desc_sw *
 551 ioat_dma_alloc_descriptor(struct ioat_dma_chan *ioat, gfp_t flags)
 552 {
 553         struct ioat_dma_descriptor *desc;
 554         struct ioat_desc_sw *desc_sw;
 555         struct ioatdma_device *ioatdma_device;
 556         dma_addr_t phys;
 557
 558         ioatdma_device = ioat->base.device;
 559         desc = pci_pool_alloc(ioatdma_device->dma_pool, flags, &phys);
 560         if (unlikely(!desc))
 561                 return NULL;
 562
 563         desc_sw = kzalloc(sizeof(*desc_sw), flags);
 564         if (unlikely(!desc_sw)) {
 565                 pci_pool_free(ioatdma_device->dma_pool, desc, phys);
 566                 return NULL;
 567         }
 568
 569         memset(desc, 0, sizeof(*desc));
 570         dma_async_tx_descriptor_init(&desc_sw->txd, &ioat->base.common);
 571         switch (ioatdma_device->version) {
 572         case IOAT_VER_1_2:
 573                 desc_sw->txd.tx_submit = ioat1_tx_submit;
 574                 break;
 575         case IOAT_VER_2_0:
 576         case IOAT_VER_3_0:
 577                 desc_sw->txd.tx_submit = ioat2_tx_submit;
 578                 break;
 579         }
 580
 581         desc_sw->hw = desc;
 582         desc_sw->txd.phys = phys;
 583
 584         return desc_sw;
 585 }
 586
 587 static int ioat_initial_desc_count = 256;
 588 module_param(ioat_initial_desc_count, int, 0644);
 589 MODULE_PARM_DESC(ioat_initial_desc_count,
 590                  "initial descriptors per channel (default: 256)");
 591
 592 /**
 593  * ioat2_dma_massage_chan_desc - link the descriptors into a circle
 594  * @ioat: the channel to be massaged
 595  */
 596 static void ioat2_dma_massage_chan_desc(struct ioat_dma_chan *ioat)
 597 {
 598         struct ioat_desc_sw *desc, *_desc;
 599
 600         /* setup used_desc */
 601         ioat->used_desc.next = ioat->free_desc.next;
 602         ioat->used_desc.prev = NULL;
 603
 604         /* pull free_desc out of the circle so that every node is a hw
 605          * descriptor, but leave it pointing to the list
 606          */
 607         ioat->free_desc.prev->next = ioat->free_desc.next;
 608         ioat->free_desc.next->prev = ioat->free_desc.prev;
 609
 610         /* circle link the hw descriptors */
 611         desc = to_ioat_desc(ioat->free_desc.next);
 612         desc->hw->next = to_ioat_desc(desc->node.next)->txd.phys;
 613         list_for_each_entry_safe(desc, _desc, ioat->free_desc.next, node) {
 614                 desc->hw->next = to_ioat_desc(desc->node.next)->txd.phys;
 615         }
 616 }
 617
 618 /**
 619  * ioat_dma_alloc_chan_resources - returns the number of allocated descriptors
 620  * @chan: the channel to be filled out
 621  */
 622 static int ioat_dma_alloc_chan_resources(struct dma_chan *c)
 623 {
 624         struct ioat_dma_chan *ioat = to_ioat_chan(c);
 625         struct ioat_chan_common *chan = &ioat->base;
 626         struct ioat_desc_sw *desc;
 627         u16 chanctrl;
 628         u32 chanerr;
 629         int i;
 630         LIST_HEAD(tmp_list);
 631
 632         /* have we already been set up? */
 633         if (!list_empty(&ioat->free_desc))
 634                 return ioat->desccount;
 635
 636         /* Setup register to interrupt and write completion status on error */
 637         chanctrl = IOAT_CHANCTRL_ERR_INT_EN |
 638                 IOAT_CHANCTRL_ANY_ERR_ABORT_EN |
 639                 IOAT_CHANCTRL_ERR_COMPLETION_EN;
 640         writew(chanctrl, chan->reg_base + IOAT_CHANCTRL_OFFSET);
 641
 642         chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
 643         if (chanerr) {
 644                 dev_err(to_dev(chan), "CHANERR = %x, clearing\n", chanerr);
 645                 writel(chanerr, chan->reg_base + IOAT_CHANERR_OFFSET);
 646         }
 647
 648         /* Allocate descriptors */
 649         for (i = 0; i < ioat_initial_desc_count; i++) {
 650                 desc = ioat_dma_alloc_descriptor(ioat, GFP_KERNEL);
 651                 if (!desc) {
 652                         dev_err(to_dev(chan), "Only %d initial descriptors\n", i);
 653                         break;
 654                 }
 655                 list_add_tail(&desc->node, &tmp_list);
 656         }
 657         spin_lock_bh(&ioat->desc_lock);
 658         ioat->desccount = i;
 659         list_splice(&tmp_list, &ioat->free_desc);
 660         if (chan->device->version != IOAT_VER_1_2)
 661                 ioat2_dma_massage_chan_desc(ioat);
 662         spin_unlock_bh(&ioat->desc_lock);
 663
 664         /* allocate a completion writeback area */
 665         /* doing 2 32bit writes to mmio since 1 64b write doesn't work */
 666         chan->completion_virt = pci_pool_alloc(chan->device->completion_pool,
 667                                                GFP_KERNEL,
 668                                                &chan->completion_addr);
 669         memset(chan->completion_virt, 0,
 670                sizeof(*chan->completion_virt));
 671         writel(((u64) chan->completion_addr) & 0x00000000FFFFFFFF,
 672                chan->reg_base + IOAT_CHANCMP_OFFSET_LOW);
 673         writel(((u64) chan->completion_addr) >> 32,
 674                chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH);
 675
 676         tasklet_enable(&chan->cleanup_task);
 677         ioat_dma_start_null_desc(ioat);  /* give chain to dma device */
 678         return ioat->desccount;
 679 }
 680
 681 /**
 682  * ioat_dma_free_chan_resources - release all the descriptors
 683  * @chan: the channel to be cleaned
 684  */
 685 static void ioat_dma_free_chan_resources(struct dma_chan *c)
 686 {
 687         struct ioat_dma_chan *ioat = to_ioat_chan(c);
 688         struct ioat_chan_common *chan = &ioat->base;
 689         struct ioatdma_device *ioatdma_device = chan->device;
 690         struct ioat_desc_sw *desc, *_desc;
 691         int in_use_descs = 0;
 692
 693         /* Before freeing channel resources first check
 694          * if they have been previously allocated for this channel.
 695          */
 696         if (ioat->desccount == 0)
 697                 return;
 698
 699         tasklet_disable(&chan->cleanup_task);
 700         ioat_dma_memcpy_cleanup(ioat);
 701
 702         /* Delay 100ms after reset to allow internal DMA logic to quiesce
 703          * before removing DMA descriptor resources.
 704          */
 705         writeb(IOAT_CHANCMD_RESET,
 706                chan->reg_base + IOAT_CHANCMD_OFFSET(chan->device->version));
 707         mdelay(100);
 708
 709         spin_lock_bh(&ioat->desc_lock);
 710         switch (chan->device->version) {
 711         case IOAT_VER_1_2:
 712                 list_for_each_entry_safe(desc, _desc,
 713                                          &ioat->used_desc, node) {
 714                         in_use_descs++;
 715                         list_del(&desc->node);
 716                         pci_pool_free(ioatdma_device->dma_pool, desc->hw,
 717                                       desc->txd.phys);
 718                         kfree(desc);
 719                 }
 720                 list_for_each_entry_safe(desc, _desc,
 721                                          &ioat->free_desc, node) {
 722                         list_del(&desc->node);
 723                         pci_pool_free(ioatdma_device->dma_pool, desc->hw,
 724                                       desc->txd.phys);
 725                         kfree(desc);
 726                 }
 727                 break;
 728         case IOAT_VER_2_0:
 729         case IOAT_VER_3_0:
 730                 list_for_each_entry_safe(desc, _desc,
 731                                          ioat->free_desc.next, node) {
 732                         list_del(&desc->node);
 733                         pci_pool_free(ioatdma_device->dma_pool, desc->hw,
 734                                       desc->txd.phys);
 735                         kfree(desc);
 736                 }
 737                 desc = to_ioat_desc(ioat->free_desc.next);
 738                 pci_pool_free(ioatdma_device->dma_pool, desc->hw,
 739                               desc->txd.phys);
 740                 kfree(desc);
 741                 INIT_LIST_HEAD(&ioat->free_desc);
 742                 INIT_LIST_HEAD(&ioat->used_desc);
 743                 break;
 744         }
 745         spin_unlock_bh(&ioat->desc_lock);
 746
 747         pci_pool_free(ioatdma_device->completion_pool,
 748                       chan->completion_virt,
 749                       chan->completion_addr);
 750
 751         /* one is ok since we left it on there on purpose */
 752         if (in_use_descs > 1)
 753                 dev_err(to_dev(chan), "Freeing %d in use descriptors!\n",
 754                         in_use_descs - 1);
 755
 756         chan->last_completion = chan->completion_addr = 0;
 757         chan->watchdog_completion = 0;
 758         chan->last_compl_desc_addr_hw = 0;
 759         chan->watchdog_tcp_cookie = chan->watchdog_last_tcp_cookie = 0;
 760         ioat->pending = 0;
 761         ioat->dmacount = 0;
 762         ioat->desccount = 0;
 763 }
 764
 765 /**
 766  * ioat1_dma_get_next_descriptor - return the next available descriptor
 767  * @ioat: IOAT DMA channel handle
 768  *
 769  * Gets the next descriptor from the chain, and must be called with the
 770  * channel's desc_lock held.  Allocates more descriptors if the channel
 771  * has run out.
 772  */
 773 static struct ioat_desc_sw *
 774 ioat1_dma_get_next_descriptor(struct ioat_dma_chan *ioat)
 775 {
 776         struct ioat_desc_sw *new;
 777
 778         if (!list_empty(&ioat->free_desc)) {
 779                 new = to_ioat_desc(ioat->free_desc.next);
 780                 list_del(&new->node);
 781         } else {
 782                 /* try to get another desc */
 783                 new = ioat_dma_alloc_descriptor(ioat, GFP_ATOMIC);
 784                 if (!new) {
 785                         dev_err(to_dev(&ioat->base), "alloc failed\n");
 786                         return NULL;
 787                 }
 788         }
 789
 790         prefetch(new->hw);
 791         return new;
 792 }
 793
 794 static struct ioat_desc_sw *
 795 ioat2_dma_get_next_descriptor(struct ioat_dma_chan *ioat)
 796 {
 797         struct ioat_desc_sw *new;
 798
 799         /*
 800          * used.prev points to where to start processing
 801          * used.next points to next free descriptor
 802          * if used.prev == NULL, there are none waiting to be processed
 803          * if used.next == used.prev.prev, there is only one free descriptor,
 804          *      and we need to use it to as a noop descriptor before
 805          *      linking in a new set of descriptors, since the device
 806          *      has probably already read the pointer to it
 807          */
 808         if (ioat->used_desc.prev &&
 809             ioat->used_desc.next == ioat->used_desc.prev->prev) {
 810
 811                 struct ioat_desc_sw *desc;
 812                 struct ioat_desc_sw *noop_desc;
 813                 int i;
 814
 815                 /* set up the noop descriptor */
 816                 noop_desc = to_ioat_desc(ioat->used_desc.next);
 817                 /* set size to non-zero value (channel returns error when size is 0) */
 818                 noop_desc->hw->size = NULL_DESC_BUFFER_SIZE;
 819                 noop_desc->hw->ctl = 0;
 820                 noop_desc->hw->ctl_f.null = 1;
 821                 noop_desc->hw->src_addr = 0;
 822                 noop_desc->hw->dst_addr = 0;
 823
 824                 ioat->used_desc.next = ioat->used_desc.next->next;
 825                 ioat->pending++;
 826                 ioat->dmacount++;
 827
 828                 /* try to get a few more descriptors */
 829                 for (i = 16; i; i--) {
 830                         desc = ioat_dma_alloc_descriptor(ioat, GFP_ATOMIC);
 831                         if (!desc) {
 832                                 dev_err(to_dev(&ioat->base),
 833                                         "alloc failed\n");
 834                                 break;
 835                         }
 836                         list_add_tail(&desc->node, ioat->used_desc.next);
 837
 838                         desc->hw->next
 839                                 = to_ioat_desc(desc->node.next)->txd.phys;
 840                         to_ioat_desc(desc->node.prev)->hw->next
 841                                 = desc->txd.phys;
 842                         ioat->desccount++;
 843                 }
 844
 845                 ioat->used_desc.next = noop_desc->node.next;
 846         }
 847         new = to_ioat_desc(ioat->used_desc.next);
 848         prefetch(new);
 849         ioat->used_desc.next = new->node.next;
 850
 851         if (ioat->used_desc.prev == NULL)
 852                 ioat->used_desc.prev = &new->node;
 853
 854         prefetch(new->hw);
 855         return new;
 856 }
 857
 858 static struct ioat_desc_sw *
 859 ioat_dma_get_next_descriptor(struct ioat_dma_chan *ioat)
 860 {
 861         if (!ioat)
 862                 return NULL;
 863
 864         switch (ioat->base.device->version) {
 865         case IOAT_VER_1_2:
 866                 return ioat1_dma_get_next_descriptor(ioat);
 867         case IOAT_VER_2_0:
 868         case IOAT_VER_3_0:
 869                 return ioat2_dma_get_next_descriptor(ioat);
 870         }
 871         return NULL;
 872 }
 873
 874 static struct dma_async_tx_descriptor *
 875 ioat1_dma_prep_memcpy(struct dma_chan *c, dma_addr_t dma_dest,
 876                       dma_addr_t dma_src, size_t len, unsigned long flags)
 877 {
 878         struct ioat_dma_chan *ioat = to_ioat_chan(c);
 879         struct ioat_desc_sw *desc;
 880         size_t copy;
 881         LIST_HEAD(chain);
 882         dma_addr_t src = dma_src;
 883         dma_addr_t dest = dma_dest;
 884         size_t total_len = len;
 885         struct ioat_dma_descriptor *hw = NULL;
 886         int tx_cnt = 0;
 887
 888         spin_lock_bh(&ioat->desc_lock);
 889         desc = ioat_dma_get_next_descriptor(ioat);
 890         do {
 891                 if (!desc)
 892                         break;
 893
 894                 tx_cnt++;
 895                 copy = min_t(size_t, len, ioat->xfercap);
 896
 897                 hw = desc->hw;
 898                 hw->size = copy;
 899                 hw->ctl = 0;
 900                 hw->src_addr = src;
 901                 hw->dst_addr = dest;
 902
 903                 list_add_tail(&desc->node, &chain);
 904
 905                 len -= copy;
 906                 dest += copy;
 907                 src += copy;
 908                 if (len) {
 909                         struct ioat_desc_sw *next;
 910
 911                         async_tx_ack(&desc->txd);
 912                         next = ioat_dma_get_next_descriptor(ioat);
 913                         hw->next = next ? next->txd.phys : 0;
 914                         desc = next;
 915                 } else
 916                         hw->next = 0;
 917         } while (len);
 918
 919         if (!desc) {
 920                 struct ioat_chan_common *chan = &ioat->base;
 921
 922                 dev_err(to_dev(chan),
 923                         "chan%d - get_next_desc failed: %d descs waiting, %d total desc\n",
 924                         chan_num(chan), ioat->dmacount, ioat->desccount);
 925                 list_splice(&chain, &ioat->free_desc);
 926                 spin_unlock_bh(&ioat->desc_lock);
 927                 return NULL;
 928         }
 929         spin_unlock_bh(&ioat->desc_lock);
 930
 931         desc->txd.flags = flags;
 932         desc->tx_cnt = tx_cnt;
 933         desc->src = dma_src;
 934         desc->dst = dma_dest;
 935         desc->len = total_len;
 936         list_splice(&chain, &desc->txd.tx_list);
 937         hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);
 938         hw->ctl_f.compl_write = 1;
 939
 940         return &desc->txd;
 941 }
 942
 943 static struct dma_async_tx_descriptor *
 944 ioat2_dma_prep_memcpy(struct dma_chan *c, dma_addr_t dma_dest,
 945                       dma_addr_t dma_src, size_t len, unsigned long flags)
 946 {
 947         struct ioat_dma_chan *ioat = to_ioat_chan(c);
 948         struct ioat_desc_sw *new;
 949
 950         spin_lock_bh(&ioat->desc_lock);
 951         new = ioat2_dma_get_next_descriptor(ioat);
 952
 953         /*
 954          * leave ioat->desc_lock set in ioat 2 path
 955          * it will get unlocked at end of tx_submit
 956          */
 957
 958         if (new) {
 959                 new->len = len;
 960                 new->dst = dma_dest;
 961                 new->src = dma_src;
 962                 new->txd.flags = flags;
 963                 return &new->txd;
 964         } else {
 965                 struct ioat_chan_common *chan = &ioat->base;
 966
 967                 spin_unlock_bh(&ioat->desc_lock);
 968                 dev_err(to_dev(chan),
 969                         "chan%d - get_next_desc failed: %d descs waiting, %d total desc\n",
 970                         chan_num(chan), ioat->dmacount, ioat->desccount);
 971                 return NULL;
 972         }
 973 }
 974
 975 static void ioat_dma_cleanup_tasklet(unsigned long data)
 976 {
 977         struct ioat_dma_chan *chan = (void *)data;
 978         ioat_dma_memcpy_cleanup(chan);
 979         writew(IOAT_CHANCTRL_INT_DISABLE,
 980                chan->base.reg_base + IOAT_CHANCTRL_OFFSET);
 981 }
 982
 983 static void
 984 ioat_dma_unmap(struct ioat_chan_common *chan, struct ioat_desc_sw *desc)
 985 {
 986         if (!(desc->txd.flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
 987                 if (desc->txd.flags & DMA_COMPL_DEST_UNMAP_SINGLE)
 988                         pci_unmap_single(chan->device->pdev,
 989                                          pci_unmap_addr(desc, dst),
 990                                          pci_unmap_len(desc, len),
 991                                          PCI_DMA_FROMDEVICE);
 992                 else
 993                         pci_unmap_page(chan->device->pdev,
 994                                        pci_unmap_addr(desc, dst),
 995                                        pci_unmap_len(desc, len),
 996                                        PCI_DMA_FROMDEVICE);
 997         }
 998
 999         if (!(desc->txd.flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
1000                 if (desc->txd.flags & DMA_COMPL_SRC_UNMAP_SINGLE)
1001                         pci_unmap_single(chan->device->pdev,
1002                                          pci_unmap_addr(desc, src),
1003                                          pci_unmap_len(desc, len),
1004                                          PCI_DMA_TODEVICE);
1005                 else
1006                         pci_unmap_page(chan->device->pdev,
1007                                        pci_unmap_addr(desc, src),
1008                                        pci_unmap_len(desc, len),
1009                                        PCI_DMA_TODEVICE);
1010         }
1011 }
1012
1013 /**
1014  * ioat_dma_memcpy_cleanup - cleanup up finished descriptors
1015  * @chan: ioat channel to be cleaned up
1016  */
1017 static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *ioat)
1018 {
1019         struct ioat_chan_common *chan = &ioat->base;
1020         unsigned long phys_complete;
1021         struct ioat_desc_sw *desc, *_desc;
1022         dma_cookie_t cookie = 0;
1023         unsigned long desc_phys;
1024         struct ioat_desc_sw *latest_desc;
1025         struct dma_async_tx_descriptor *tx;
1026
1027         prefetch(chan->completion_virt);
1028
1029         if (!spin_trylock_bh(&chan->cleanup_lock))
1030                 return;
1031
1032         /* The completion writeback can happen at any time,
1033            so reads by the driver need to be atomic operations
1034            The descriptor physical addresses are limited to 32-bits
1035            when the CPU can only do a 32-bit mov */
1036
1037 #if (BITS_PER_LONG == 64)
1038         phys_complete =
1039                 chan->completion_virt->full
1040                 & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR;
1041 #else
1042         phys_complete = chan->completion_virt->low & IOAT_LOW_COMPLETION_MASK;
1043 #endif
1044
1045         if ((chan->completion_virt->full
1046                 & IOAT_CHANSTS_DMA_TRANSFER_STATUS) ==
1047                                 IOAT_CHANSTS_DMA_TRANSFER_STATUS_HALTED) {
1048                 dev_err(to_dev(chan), "Channel halted, chanerr = %x\n",
1049                         readl(chan->reg_base + IOAT_CHANERR_OFFSET));
1050
1051                 /* TODO do something to salvage the situation */
1052         }
1053
1054         if (phys_complete == chan->last_completion) {
1055                 spin_unlock_bh(&chan->cleanup_lock);
1056                 /*
1057                  * perhaps we're stuck so hard that the watchdog can't go off?
1058                  * try to catch it after 2 seconds
1059                  */
1060                 if (chan->device->version != IOAT_VER_3_0) {
1061                         if (time_after(jiffies,
1062                                        chan->last_completion_time + HZ*WATCHDOG_DELAY)) {
1063                                 ioat_dma_chan_watchdog(&(chan->device->work.work));
1064                                 chan->last_completion_time = jiffies;
1065                         }
1066                 }
1067                 return;
1068         }
1069         chan->last_completion_time = jiffies;
1070
1071         cookie = 0;
1072         if (!spin_trylock_bh(&ioat->desc_lock)) {
1073                 spin_unlock_bh(&chan->cleanup_lock);
1074                 return;
1075         }
1076
1077         switch (chan->device->version) {
1078         case IOAT_VER_1_2:
1079                 list_for_each_entry_safe(desc, _desc, &ioat->used_desc, node) {
1080                         tx = &desc->txd;
1081                         /*
1082                          * Incoming DMA requests may use multiple descriptors,
1083                          * due to exceeding xfercap, perhaps. If so, only the
1084                          * last one will have a cookie, and require unmapping.
1085                          */
1086                         if (tx->cookie) {
1087                                 cookie = tx->cookie;
1088                                 ioat_dma_unmap(chan, desc);
1089                                 if (tx->callback) {
1090                                         tx->callback(tx->callback_param);
1091                                         tx->callback = NULL;
1092                                 }
1093                         }
1094
1095                         if (tx->phys != phys_complete) {
1096                                 /*
1097                                  * a completed entry, but not the last, so clean
1098                                  * up if the client is done with the descriptor
1099                                  */
1100                                 if (async_tx_test_ack(tx)) {
1101                                         list_move_tail(&desc->node,
1102                                                        &ioat->free_desc);
1103                                 } else
1104                                         tx->cookie = 0;
1105                         } else {
1106                                 /*
1107                                  * last used desc. Do not remove, so we can
1108                                  * append from it, but don't look at it next
1109                                  * time, either
1110                                  */
1111                                 tx->cookie = 0;
1112
1113                                 /* TODO check status bits? */
1114                                 break;
1115                         }
1116                 }
1117                 break;
1118         case IOAT_VER_2_0:
1119         case IOAT_VER_3_0:
1120                 /* has some other thread has already cleaned up? */
1121                 if (ioat->used_desc.prev == NULL)
1122                         break;
1123
1124                 /* work backwards to find latest finished desc */
1125                 desc = to_ioat_desc(ioat->used_desc.next);
1126                 tx = &desc->txd;
1127                 latest_desc = NULL;
1128                 do {
1129                         desc = to_ioat_desc(desc->node.prev);
1130                         desc_phys = (unsigned long)tx->phys
1131                                        & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR;
1132                         if (desc_phys == phys_complete) {
1133                                 latest_desc = desc;
1134                                 break;
1135                         }
1136                 } while (&desc->node != ioat->used_desc.prev);
1137
1138                 if (latest_desc != NULL) {
1139                         /* work forwards to clear finished descriptors */
1140                         for (desc = to_ioat_desc(ioat->used_desc.prev);
1141                              &desc->node != latest_desc->node.next &&
1142                              &desc->node != ioat->used_desc.next;
1143                              desc = to_ioat_desc(desc->node.next)) {
1144                                 if (tx->cookie) {
1145                                         cookie = tx->cookie;
1146                                         tx->cookie = 0;
1147                                         ioat_dma_unmap(chan, desc);
1148                                         if (tx->callback) {
1149                                                 tx->callback(tx->callback_param);
1150                                                 tx->callback = NULL;
1151                                         }
1152                                 }
1153                         }
1154
1155                         /* move used.prev up beyond those that are finished */
1156                         if (&desc->node == ioat->used_desc.next)
1157                                 ioat->used_desc.prev = NULL;
1158                         else
1159                                 ioat->used_desc.prev = &desc->node;
1160                 }
1161                 break;
1162         }
1163
1164         spin_unlock_bh(&ioat->desc_lock);
1165
1166         chan->last_completion = phys_complete;
1167         if (cookie != 0)
1168                 chan->completed_cookie = cookie;
1169
1170         spin_unlock_bh(&chan->cleanup_lock);
1171 }
1172
1173 /**
1174  * ioat_dma_is_complete - poll the status of a IOAT DMA transaction
1175  * @chan: IOAT DMA channel handle
1176  * @cookie: DMA transaction identifier
1177  * @done: if not %NULL, updated with last completed transaction
1178  * @used: if not %NULL, updated with last used transaction
1179  */
1180 static enum dma_status
1181 ioat_dma_is_complete(struct dma_chan *c, dma_cookie_t cookie,
1182                      dma_cookie_t *done, dma_cookie_t *used)
1183 {
1184         struct ioat_dma_chan *ioat = to_ioat_chan(c);
1185         struct ioat_chan_common *chan = &ioat->base;
1186         dma_cookie_t last_used;
1187         dma_cookie_t last_complete;
1188         enum dma_status ret;
1189
1190         last_used = c->cookie;
1191         last_complete = chan->completed_cookie;
1192         chan->watchdog_tcp_cookie = cookie;
1193
1194         if (done)
1195                 *done = last_complete;
1196         if (used)
1197                 *used = last_used;
1198
1199         ret = dma_async_is_complete(cookie, last_complete, last_used);
1200         if (ret == DMA_SUCCESS)
1201                 return ret;
1202
1203         ioat_dma_memcpy_cleanup(ioat);
1204
1205         last_used = c->cookie;
1206         last_complete = chan->completed_cookie;
1207
1208         if (done)
1209                 *done = last_complete;
1210         if (used)
1211                 *used = last_used;
1212
1213         return dma_async_is_complete(cookie, last_complete, last_used);
1214 }
1215
1216 static void ioat_dma_start_null_desc(struct ioat_dma_chan *ioat)
1217 {
1218         struct ioat_chan_common *chan = &ioat->base;
1219         struct ioat_desc_sw *desc;
1220         struct ioat_dma_descriptor *hw;
1221
1222         spin_lock_bh(&ioat->desc_lock);
1223
1224         desc = ioat_dma_get_next_descriptor(ioat);
1225
1226         if (!desc) {
1227                 dev_err(to_dev(chan),
1228                         "Unable to start null desc - get next desc failed\n");
1229                 spin_unlock_bh(&ioat->desc_lock);
1230                 return;
1231         }
1232
1233         hw = desc->hw;
1234         hw->ctl = 0;
1235         hw->ctl_f.null = 1;
1236         hw->ctl_f.int_en = 1;
1237         hw->ctl_f.compl_write = 1;
1238         /* set size to non-zero value (channel returns error when size is 0) */
1239         hw->size = NULL_DESC_BUFFER_SIZE;
1240         hw->src_addr = 0;
1241         hw->dst_addr = 0;
1242         async_tx_ack(&desc->txd);
1243         switch (chan->device->version) {
1244         case IOAT_VER_1_2:
1245                 hw->next = 0;
1246                 list_add_tail(&desc->node, &ioat->used_desc);
1247
1248                 writel(((u64) desc->txd.phys) & 0x00000000FFFFFFFF,
1249                        chan->reg_base + IOAT1_CHAINADDR_OFFSET_LOW);
1250                 writel(((u64) desc->txd.phys) >> 32,
1251                        chan->reg_base + IOAT1_CHAINADDR_OFFSET_HIGH);
1252
1253                 writeb(IOAT_CHANCMD_START, chan->reg_base
1254                         + IOAT_CHANCMD_OFFSET(chan->device->version));
1255                 break;
1256         case IOAT_VER_2_0:
1257         case IOAT_VER_3_0:
1258                 writel(((u64) desc->txd.phys) & 0x00000000FFFFFFFF,
1259                        chan->reg_base + IOAT2_CHAINADDR_OFFSET_LOW);
1260                 writel(((u64) desc->txd.phys) >> 32,
1261                        chan->reg_base + IOAT2_CHAINADDR_OFFSET_HIGH);
1262
1263                 ioat->dmacount++;
1264                 __ioat2_dma_memcpy_issue_pending(ioat);
1265                 break;
1266         }
1267         spin_unlock_bh(&ioat->desc_lock);
1268 }
1269
1270 /*
1271  * Perform a IOAT transaction to verify the HW works.
1272  */
1273 #define IOAT_TEST_SIZE 2000
1274
1275 static void ioat_dma_test_callback(void *dma_async_param)
1276 {
1277         struct completion *cmp = dma_async_param;
1278
1279         complete(cmp);
1280 }
1281
1282 /**
1283  * ioat_dma_self_test - Perform a IOAT transaction to verify the HW works.
1284  * @device: device to be tested
1285  */
1286 static int ioat_dma_self_test(struct ioatdma_device *device)
1287 {
1288         int i;
1289         u8 *src;
1290         u8 *dest;
1291         struct dma_device *dma = &device->common;
1292         struct device *dev = &device->pdev->dev;
1293         struct dma_chan *dma_chan;
1294         struct dma_async_tx_descriptor *tx;
1295         dma_addr_t dma_dest, dma_src;
1296         dma_cookie_t cookie;
1297         int err = 0;
1298         struct completion cmp;
1299         unsigned long tmo;
1300         unsigned long flags;
1301
1302         src = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, GFP_KERNEL);
1303         if (!src)
1304                 return -ENOMEM;
1305         dest = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, GFP_KERNEL);
1306         if (!dest) {
1307                 kfree(src);
1308                 return -ENOMEM;
1309         }
1310
1311         /* Fill in src buffer */
1312         for (i = 0; i < IOAT_TEST_SIZE; i++)
1313                 src[i] = (u8)i;
1314
1315         /* Start copy, using first DMA channel */
1316         dma_chan = container_of(dma->channels.next, struct dma_chan,
1317                                 device_node);
1318         if (dma->device_alloc_chan_resources(dma_chan) < 1) {
1319                 dev_err(dev, "selftest cannot allocate chan resource\n");
1320                 err = -ENODEV;
1321                 goto out;
1322         }
1323
1324         dma_src = dma_map_single(dev, src, IOAT_TEST_SIZE, DMA_TO_DEVICE);
1325         dma_dest = dma_map_single(dev, dest, IOAT_TEST_SIZE, DMA_FROM_DEVICE);
1326         flags = DMA_COMPL_SRC_UNMAP_SINGLE | DMA_COMPL_DEST_UNMAP_SINGLE |
1327                 DMA_PREP_INTERRUPT;
1328         tx = device->common.device_prep_dma_memcpy(dma_chan, dma_dest, dma_src,
1329                                                    IOAT_TEST_SIZE, flags);
1330         if (!tx) {
1331                 dev_err(dev, "Self-test prep failed, disabling\n");
1332                 err = -ENODEV;
1333                 goto free_resources;
1334         }
1335
1336         async_tx_ack(tx);
1337         init_completion(&cmp);
1338         tx->callback = ioat_dma_test_callback;
1339         tx->callback_param = &cmp;
1340         cookie = tx->tx_submit(tx);
1341         if (cookie < 0) {
1342                 dev_err(dev, "Self-test setup failed, disabling\n");
1343                 err = -ENODEV;
1344                 goto free_resources;
1345         }
1346         dma->device_issue_pending(dma_chan);
1347
1348         tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000));
1349
1350         if (tmo == 0 ||
1351             dma->device_is_tx_complete(dma_chan, cookie, NULL, NULL)
1352                                         != DMA_SUCCESS) {
1353                 dev_err(dev, "Self-test copy timed out, disabling\n");
1354                 err = -ENODEV;
1355                 goto free_resources;
1356         }
1357         if (memcmp(src, dest, IOAT_TEST_SIZE)) {
1358                 dev_err(dev, "Self-test copy failed compare, disabling\n");
1359                 err = -ENODEV;
1360                 goto free_resources;
1361         }
1362
1363 free_resources:
1364         dma->device_free_chan_resources(dma_chan);
1365 out:
1366         kfree(src);
1367         kfree(dest);
1368         return err;
1369 }
1370
1371 static char ioat_interrupt_style[32] = "msix";
1372 module_param_string(ioat_interrupt_style, ioat_interrupt_style,
1373                     sizeof(ioat_interrupt_style), 0644);
1374 MODULE_PARM_DESC(ioat_interrupt_style,
1375                  "set ioat interrupt style: msix (default), "
1376                  "msix-single-vector, msi, intx)");
1377
1378 /**
1379  * ioat_dma_setup_interrupts - setup interrupt handler
1380  * @device: ioat device
1381  */
1382 static int ioat_dma_setup_interrupts(struct ioatdma_device *device)
1383 {
1384         struct ioat_chan_common *chan;
1385         struct pci_dev *pdev = device->pdev;
1386         struct device *dev = &pdev->dev;
1387         struct msix_entry *msix;
1388         int i, j, msixcnt;
1389         int err = -EINVAL;
1390         u8 intrctrl = 0;
1391
1392         if (!strcmp(ioat_interrupt_style, "msix"))
1393                 goto msix;
1394         if (!strcmp(ioat_interrupt_style, "msix-single-vector"))
1395                 goto msix_single_vector;
1396         if (!strcmp(ioat_interrupt_style, "msi"))
1397                 goto msi;
1398         if (!strcmp(ioat_interrupt_style, "intx"))
1399                 goto intx;
1400         dev_err(dev, "invalid ioat_interrupt_style %s\n", ioat_interrupt_style);
1401         goto err_no_irq;
1402
1403 msix:
1404         /* The number of MSI-X vectors should equal the number of channels */
1405         msixcnt = device->common.chancnt;
1406         for (i = 0; i < msixcnt; i++)
1407                 device->msix_entries[i].entry = i;
1408
1409         err = pci_enable_msix(pdev, device->msix_entries, msixcnt);
1410         if (err < 0)
1411                 goto msi;
1412         if (err > 0)
1413                 goto msix_single_vector;
1414
1415         for (i = 0; i < msixcnt; i++) {
1416                 msix = &device->msix_entries[i];
1417                 chan = ioat_chan_by_index(device, i);
1418                 err = devm_request_irq(dev, msix->vector,
1419                                        ioat_dma_do_interrupt_msix, 0,
1420                                        "ioat-msix", chan);
1421                 if (err) {
1422                         for (j = 0; j < i; j++) {
1423                                 msix = &device->msix_entries[j];
1424                                 chan = ioat_chan_by_index(device, j);
1425                                 devm_free_irq(dev, msix->vector, chan);
1426                         }
1427                         goto msix_single_vector;
1428                 }
1429         }
1430         intrctrl |= IOAT_INTRCTRL_MSIX_VECTOR_CONTROL;
1431         goto done;
1432
1433 msix_single_vector:
1434         msix = &device->msix_entries[0];
1435         msix->entry = 0;
1436         err = pci_enable_msix(pdev, device->msix_entries, 1);
1437         if (err)
1438                 goto msi;
1439
1440         err = devm_request_irq(dev, msix->vector, ioat_dma_do_interrupt, 0,
1441                                "ioat-msix", device);
1442         if (err) {
1443                 pci_disable_msix(pdev);
1444                 goto msi;
1445         }
1446         goto done;
1447
1448 msi:
1449         err = pci_enable_msi(pdev);
1450         if (err)
1451                 goto intx;
1452
1453         err = devm_request_irq(dev, pdev->irq, ioat_dma_do_interrupt, 0,
1454                                "ioat-msi", device);
1455         if (err) {
1456                 pci_disable_msi(pdev);
1457                 goto intx;
1458         }
1459         goto done;
1460
1461 intx:
1462         err = devm_request_irq(dev, pdev->irq, ioat_dma_do_interrupt,
1463                                IRQF_SHARED, "ioat-intx", device);
1464         if (err)
1465                 goto err_no_irq;
1466
1467 done:
1468         if (device->intr_quirk)
1469                 device->intr_quirk(device);
1470         intrctrl |= IOAT_INTRCTRL_MASTER_INT_EN;
1471         writeb(intrctrl, device->reg_base + IOAT_INTRCTRL_OFFSET);
1472         return 0;
1473
1474 err_no_irq:
1475         /* Disable all interrupt generation */
1476         writeb(0, device->reg_base + IOAT_INTRCTRL_OFFSET);
1477         dev_err(dev, "no usable interrupts\n");
1478         return err;
1479 }
1480
1481 static void ioat_disable_interrupts(struct ioatdma_device *device)
1482 {
1483         /* Disable all interrupt generation */
1484         writeb(0, device->reg_base + IOAT_INTRCTRL_OFFSET);
1485 }
1486
1487 static int ioat_probe(struct ioatdma_device *device)
1488 {
1489         int err = -ENODEV;
1490         struct dma_device *dma = &device->common;
1491         struct pci_dev *pdev = device->pdev;
1492         struct device *dev = &pdev->dev;
1493
1494         /* DMA coherent memory pool for DMA descriptor allocations */
1495         device->dma_pool = pci_pool_create("dma_desc_pool", pdev,
1496                                            sizeof(struct ioat_dma_descriptor),
1497                                            64, 0);
1498         if (!device->dma_pool) {
1499                 err = -ENOMEM;
1500                 goto err_dma_pool;
1501         }
1502
1503         device->completion_pool = pci_pool_create("completion_pool", pdev,
1504                                                   sizeof(u64), SMP_CACHE_BYTES,
1505                                                   SMP_CACHE_BYTES);
1506         if (!device->completion_pool) {
1507                 err = -ENOMEM;
1508                 goto err_completion_pool;
1509         }
1510
1511         ioat_dma_enumerate_channels(device);
1512
1513         dma_cap_set(DMA_MEMCPY, dma->cap_mask);
1514         dma->device_alloc_chan_resources = ioat_dma_alloc_chan_resources;
1515         dma->device_free_chan_resources = ioat_dma_free_chan_resources;
1516         dma->device_is_tx_complete = ioat_dma_is_complete;
1517         dma->dev = &pdev->dev;
1518
1519         dev_err(dev, "Intel(R) I/OAT DMA Engine found,"
1520                 " %d channels, device version 0x%02x, driver version %s\n",
1521                 dma->chancnt, device->version, IOAT_DMA_VERSION);
1522
1523         if (!dma->chancnt) {
1524                 dev_err(dev, "Intel(R) I/OAT DMA Engine problem found: "
1525                         "zero channels detected\n");
1526                 goto err_setup_interrupts;
1527         }
1528
1529         err = ioat_dma_setup_interrupts(device);
1530         if (err)
1531                 goto err_setup_interrupts;
1532
1533         err = ioat_dma_self_test(device);
1534         if (err)
1535                 goto err_self_test;
1536
1537         return 0;
1538
1539 err_self_test:
1540         ioat_disable_interrupts(device);
1541 err_setup_interrupts:
1542         pci_pool_destroy(device->completion_pool);
1543 err_completion_pool:
1544         pci_pool_destroy(device->dma_pool);
1545 err_dma_pool:
1546         return err;
1547 }
1548
1549 static int ioat_register(struct ioatdma_device *device)
1550 {
1551         int err = dma_async_device_register(&device->common);
1552
1553         if (err) {
1554                 ioat_disable_interrupts(device);
1555                 pci_pool_destroy(device->completion_pool);
1556                 pci_pool_destroy(device->dma_pool);
1557         }
1558
1559         return err;
1560 }
1561
1562 /* ioat1_intr_quirk - fix up dma ctrl register to enable / disable msi */
1563 static void ioat1_intr_quirk(struct ioatdma_device *device)
1564 {
1565         struct pci_dev *pdev = device->pdev;
1566         u32 dmactrl;
1567
1568         pci_read_config_dword(pdev, IOAT_PCI_DMACTRL_OFFSET, &dmactrl);
1569         if (pdev->msi_enabled)
1570                 dmactrl |= IOAT_PCI_DMACTRL_MSI_EN;
1571         else
1572                 dmactrl &= ~IOAT_PCI_DMACTRL_MSI_EN;
1573         pci_write_config_dword(pdev, IOAT_PCI_DMACTRL_OFFSET, dmactrl);
1574 }
1575
1576 int ioat1_dma_probe(struct ioatdma_device *device, int dca)
1577 {
1578         struct pci_dev *pdev = device->pdev;
1579         struct dma_device *dma;
1580         int err;
1581
1582         device->intr_quirk = ioat1_intr_quirk;
1583         dma = &device->common;
1584         dma->device_prep_dma_memcpy = ioat1_dma_prep_memcpy;
1585         dma->device_issue_pending = ioat1_dma_memcpy_issue_pending;
1586
1587         err = ioat_probe(device);
1588         if (err)
1589                 return err;
1590         ioat_set_tcp_copy_break(4096);
1591         err = ioat_register(device);
1592         if (err)
1593                 return err;
1594         if (dca)
1595                 device->dca = ioat_dca_init(pdev, device->reg_base);
1596
1597         INIT_DELAYED_WORK(&device->work, ioat_dma_chan_watchdog);
1598         schedule_delayed_work(&device->work, WATCHDOG_DELAY);
1599
1600         return err;
1601 }
1602
1603 int ioat2_dma_probe(struct ioatdma_device *device, int dca)
1604 {
1605         struct pci_dev *pdev = device->pdev;
1606         struct dma_device *dma;
1607         struct dma_chan *c;
1608         struct ioat_chan_common *chan;
1609         int err;
1610
1611         dma = &device->common;
1612         dma->device_prep_dma_memcpy = ioat2_dma_prep_memcpy;
1613         dma->device_issue_pending = ioat2_dma_memcpy_issue_pending;
1614
1615         err = ioat_probe(device);
1616         if (err)
1617                 return err;
1618         ioat_set_tcp_copy_break(2048);
1619
1620         list_for_each_entry(c, &dma->channels, device_node) {
1621                 chan = to_chan_common(c);
1622                 writel(IOAT_DCACTRL_CMPL_WRITE_ENABLE | IOAT_DMA_DCA_ANY_CPU,
1623                        chan->reg_base + IOAT_DCACTRL_OFFSET);
1624         }
1625
1626         err = ioat_register(device);
1627         if (err)
1628                 return err;
1629         if (dca)
1630                 device->dca = ioat2_dca_init(pdev, device->reg_base);
1631
1632         INIT_DELAYED_WORK(&device->work, ioat_dma_chan_watchdog);
1633         schedule_delayed_work(&device->work, WATCHDOG_DELAY);
1634
1635         return err;
1636 }
1637
1638 int ioat3_dma_probe(struct ioatdma_device *device, int dca)
1639 {
1640         struct pci_dev *pdev = device->pdev;
1641         struct dma_device *dma;
1642         struct dma_chan *c;
1643         struct ioat_chan_common *chan;
1644         int err;
1645         u16 dev_id;
1646
1647         dma = &device->common;
1648         dma->device_prep_dma_memcpy = ioat2_dma_prep_memcpy;
1649         dma->device_issue_pending = ioat2_dma_memcpy_issue_pending;
1650
1651         /* -= IOAT ver.3 workarounds =- */
1652         /* Write CHANERRMSK_INT with 3E07h to mask out the errors
1653          * that can cause stability issues for IOAT ver.3
1654          */
1655         pci_write_config_dword(pdev, IOAT_PCI_CHANERRMASK_INT_OFFSET, 0x3e07);
1656
1657         /* Clear DMAUNCERRSTS Cfg-Reg Parity Error status bit
1658          * (workaround for spurious config parity error after restart)
1659          */
1660         pci_read_config_word(pdev, IOAT_PCI_DEVICE_ID_OFFSET, &dev_id);
1661         if (dev_id == PCI_DEVICE_ID_INTEL_IOAT_TBG0)
1662                 pci_write_config_dword(pdev, IOAT_PCI_DMAUNCERRSTS_OFFSET, 0x10);
1663
1664         err = ioat_probe(device);
1665         if (err)
1666                 return err;
1667         ioat_set_tcp_copy_break(262144);
1668
1669         list_for_each_entry(c, &dma->channels, device_node) {
1670                 chan = to_chan_common(c);
1671                 writel(IOAT_DMA_DCA_ANY_CPU,
1672                        chan->reg_base + IOAT_DCACTRL_OFFSET);
1673         }
1674
1675         err = ioat_register(device);
1676         if (err)
1677                 return err;
1678         if (dca)
1679                 device->dca = ioat3_dca_init(pdev, device->reg_base);
1680
1681         return err;
1682 }
1683
1684 void ioat_dma_remove(struct ioatdma_device *device)
1685 {
1686         struct dma_device *dma = &device->common;
1687
1688         if (device->version != IOAT_VER_3_0)
1689                 cancel_delayed_work(&device->work);
1690
1691         ioat_disable_interrupts(device);
1692
1693         dma_async_device_unregister(dma);
1694
1695         pci_pool_destroy(device->dma_pool);
1696         pci_pool_destroy(device->completion_pool);
1697
1698         INIT_LIST_HEAD(&dma->channels);
1699 }
1700